Fix v5 for Fedora 41 (#5839)

Signed-off-by: Adam Warner <me@adamwarner.co.uk>

.github/workflows/codeql-analysis.yml

@@ -25,16 +25,16 @@ jobs:
     steps:
     -
       name: Checkout repository
-      uses: actions/checkout@v3.3.0
+      uses: actions/checkout@v4.1.7
     # Initializes the CodeQL tools for scanning.
     -
       name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v3
       with:
         languages: 'python'
     -
       name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      uses: github/codeql-action/autobuild@v3
     -
       name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v3

.github/workflows/merge-conflict.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check if PRs are have merge conflicts
-        uses: eps1lon/actions-label-merge-conflict@v2.1.0
+        uses: eps1lon/actions-label-merge-conflict@v3.0.2
         with:
           dirtyLabel: "PR: Merge Conflict"
           repoToken: "${{ secrets.GITHUB_TOKEN }}"

.github/workflows/stale.yml

@@ -4,23 +4,45 @@ on:
   schedule:
     - cron: '0 8 * * *'
   workflow_dispatch:
+  issue_comment:
+
+env:
+  stale_label: stale
 
 jobs:
-  stale:
-
+  stale_action:
+    if: github.event_name != 'issue_comment'
     runs-on: ubuntu-latest
     permissions:
       issues: write
 
     steps:
-      - uses: actions/stale@v7.0.0
+      - uses: actions/stale@v9.0.0
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           days-before-stale: 30
           days-before-close: 5
           stale-issue-message: 'This issue is stale because it has been open 30 days with no activity. Please comment or update this issue or it will be closed in 5 days.'
-          stale-issue-label: 'stale'
+          stale-issue-label: '${{ env.stale_label }}'
           exempt-issue-labels: 'Internal, Fixed in next release, Bug: Confirmed, Documentation Needed'
           exempt-all-issue-assignees: true
           operations-per-run: 300
           close-issue-reason: 'not_planned'
+
+  remove_stale:
+    # trigger "stale" removal immediately when stale issues are commented on
+    # we need to explicitly check that the trigger does not run on comment on a PR as
+    # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#issue_comment-on-issues-only-or-pull-requests-only
+    if: ${{ !github.event.issue.pull_request && github.event_name != 'schedule' }}
+    permissions:
+      contents: read #  for actions/checkout
+      issues: write #  to edit issues label
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4.1.7
+      - name: Remove 'stale' label
+        run: gh issue edit ${{ github.event.issue.number }} --remove-label ${{ env.stale_label }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+

.github/workflows/stale_pr.yml

@@ -17,7 +17,7 @@ jobs:
       pull-requests: write
 
     steps:
-      - uses: actions/stale@v7.0.0
+      - uses: actions/stale@v9.0.0
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           # Do not automatically mark PR/issue as stale

.github/workflows/sync-back-to-dev.yml

@@ -5,13 +5,35 @@ on:
     branches:
       - master
 
+# The section is needed to drop the default write-all permissions for all jobs
+# that are granted on `push` event. By specifying any permission explicitly
+# all others are set to none. By using the principle of least privilege the damage a compromised
+# workflow can do (because of an injection or compromised third party tool or
+# action) is restricted. Adding labels to issues, commenting
+# on pull-requests, etc. may need additional permissions:
+#
+# Syntax for this section:
+# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#permissions
+#
+# Reference for how to assign permissions on a job-by-job basis:
+# https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs
+#
+# Reference for available permissions that we can enable if needed:
+# https://docs.github.com/en/actions/security-guides/automatic-token-authentication#permissions-for-the-github_token
+permissions: {}
+
 jobs:
   sync-branches:
+    # The job needs to be able to pull the code and create a pull request.
+    permissions:
+      contents: read #  for actions/checkout
+      pull-requests: write #  to create pull request
+
     runs-on: ubuntu-latest
     name: Syncing branches
     steps:
       - name: Checkout
-        uses: actions/checkout@v3.3.0
+        uses: actions/checkout@v4.1.7
       - name: Opening pull request
         run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal'
         env:

.github/workflows/test.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3.3.0
+        uses: actions/checkout@v4.1.7
 
       - name: Check scripts in repository are executable
         run: |
@@ -49,24 +49,23 @@ jobs:
       matrix:
         distro:
           [
-            debian_10,
             debian_11,
+            debian_12,
             ubuntu_20,
             ubuntu_22,
-            centos_8,
+            ubuntu_24,
             centos_9,
-            fedora_35,
-            fedora_36,
-            fedora_37,
+            fedora_40,
+            fedora_41,
           ]
     env:
       DISTRO: ${{matrix.distro}}
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3.3.0
+        uses: actions/checkout@v4.1.7
 
       - name: Set up Python 3.10
-        uses: actions/setup-python@v4.5.0
+        uses: actions/setup-python@v5.1.0
         with:
           python-version: "3.10"
 

README.md

@@ -33,7 +33,9 @@ The Pi-hole® is a [DNS sinkhole](https://en.wikipedia.org/wiki/DNS_Sinkhole) th
 
 Those who want to get started quickly and conveniently may install Pi-hole using the following command:
 
-### `curl -sSL https://install.pi-hole.net | bash`
+```bash
+curl -sSL https://install.pi-hole.net | bash
+```
 
 ## Alternative Install Methods
 
@@ -150,7 +152,7 @@ You can read our [Core Feature Breakdown](https://docs.pi-hole.net/core/pihole-c
 
 ### The Web Interface Dashboard
 
-This [optional dashboard](https://github.com/pi-hole/AdminLTE) allows you to view stats, change settings, and configure your Pi-hole. It's the power of the Command Line Interface, with none of the learning curve!
+This [optional dashboard](https://github.com/pi-hole/web) allows you to view stats, change settings, and configure your Pi-hole. It's the power of the Command Line Interface, with none of the learning curve!
 
 Some notable features include:
 

advanced/Scripts/chronometer.sh

@@ -233,7 +233,7 @@ get_sys_stats() {
         if [[ -n "${ph_ver_raw[0]}" ]]; then
             ph_core_ver="${ph_ver_raw[0]}"
             if [[ ${#ph_ver_raw[@]} -eq 2 ]]; then
-                # AdminLTE not installed
+                # web not installed
                 ph_lte_ver="(not installed)"
                 ph_ftl_ver="${ph_ver_raw[1]}"
             else

advanced/Scripts/database_migration/gravity-db.sh

@@ -13,119 +13,119 @@
 readonly scriptPath="/etc/.pihole/advanced/Scripts/database_migration/gravity"
 
 upgrade_gravityDB(){
-	local database piholeDir auditFile version
-	database="${1}"
-	piholeDir="${2}"
-	auditFile="${piholeDir}/auditlog.list"
+    local database piholeDir auditFile version
+    database="${1}"
+    piholeDir="${2}"
+    auditFile="${piholeDir}/auditlog.list"
 
-	# Get database version
-	version="$(pihole-FTL sqlite3 "${database}" "SELECT \"value\" FROM \"info\" WHERE \"property\" = 'version';")"
+    # Get database version
+    version="$(pihole-FTL sqlite3 -ni "${database}" "SELECT \"value\" FROM \"info\" WHERE \"property\" = 'version';")"
 
-	if [[ "$version" == "1" ]]; then
-		# This migration script upgrades the gravity.db file by
-		# adding the domain_audit table
-		echo -e "  ${INFO} Upgrading gravity database from version 1 to 2"
-		pihole-FTL sqlite3 "${database}" < "${scriptPath}/1_to_2.sql"
-		version=2
+    if [[ "$version" == "1" ]]; then
+        # This migration script upgrades the gravity.db file by
+        # adding the domain_audit table
+        echo -e "  ${INFO} Upgrading gravity database from version 1 to 2"
+        pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/1_to_2.sql"
+        version=2
 
-		# Store audit domains in database table
-		if [ -e "${auditFile}" ]; then
-			echo -e "  ${INFO} Migrating content of ${auditFile} into new database"
-			# database_table_from_file is defined in gravity.sh
-			database_table_from_file "domain_audit" "${auditFile}"
-		fi
-	fi
-	if [[ "$version" == "2" ]]; then
-		# This migration script upgrades the gravity.db file by
-		# renaming the regex table to regex_blacklist, and
-		# creating a new regex_whitelist table + corresponding linking table and views
-		echo -e "  ${INFO} Upgrading gravity database from version 2 to 3"
-		pihole-FTL sqlite3 "${database}" < "${scriptPath}/2_to_3.sql"
-		version=3
-	fi
-	if [[ "$version" == "3" ]]; then
-		# This migration script unifies the formally separated domain
-		# lists into a single table with a UNIQUE domain constraint
-		echo -e "  ${INFO} Upgrading gravity database from version 3 to 4"
-		pihole-FTL sqlite3 "${database}" < "${scriptPath}/3_to_4.sql"
-		version=4
-	fi
-	if [[ "$version" == "4" ]]; then
-		# This migration script upgrades the gravity and list views
-		# implementing necessary changes for per-client blocking
-		echo -e "  ${INFO} Upgrading gravity database from version 4 to 5"
-		pihole-FTL sqlite3 "${database}" < "${scriptPath}/4_to_5.sql"
-		version=5
-	fi
-	if [[ "$version" == "5" ]]; then
-		# This migration script upgrades the adlist view
-		# to return an ID used in gravity.sh
-		echo -e "  ${INFO} Upgrading gravity database from version 5 to 6"
-		pihole-FTL sqlite3 "${database}" < "${scriptPath}/5_to_6.sql"
-		version=6
-	fi
-	if [[ "$version" == "6" ]]; then
-		# This migration script adds a special group with ID 0
-		# which is automatically associated to all clients not
-		# having their own group assignments
-		echo -e "  ${INFO} Upgrading gravity database from version 6 to 7"
-		pihole-FTL sqlite3 "${database}" < "${scriptPath}/6_to_7.sql"
-		version=7
-	fi
-	if [[ "$version" == "7" ]]; then
-		# This migration script recreated the group table
-		# to ensure uniqueness on the group name
-		# We also add date_added and date_modified columns
-		echo -e "  ${INFO} Upgrading gravity database from version 7 to 8"
-		pihole-FTL sqlite3 "${database}" < "${scriptPath}/7_to_8.sql"
-		version=8
-	fi
-	if [[ "$version" == "8" ]]; then
-		# This migration fixes some issues that were introduced
-		# in the previous migration script.
-		echo -e "  ${INFO} Upgrading gravity database from version 8 to 9"
-		pihole-FTL sqlite3 "${database}" < "${scriptPath}/8_to_9.sql"
-		version=9
-	fi
-	if [[ "$version" == "9" ]]; then
-		# This migration drops unused tables and creates triggers to remove
-		# obsolete groups assignments when the linked items are deleted
-		echo -e "  ${INFO} Upgrading gravity database from version 9 to 10"
-		pihole-FTL sqlite3 "${database}" < "${scriptPath}/9_to_10.sql"
-		version=10
-	fi
-	if [[ "$version" == "10" ]]; then
-		# This adds timestamp and an optional comment field to the client table
-		# These fields are only temporary and will be replaces by the columns
-		# defined in gravity.db.sql during gravity swapping. We add them here
-		# to keep the copying process generic (needs the same columns in both the
-		# source and the destination databases).
-		echo -e "  ${INFO} Upgrading gravity database from version 10 to 11"
-		pihole-FTL sqlite3 "${database}" < "${scriptPath}/10_to_11.sql"
-		version=11
-	fi
-	if [[ "$version" == "11" ]]; then
-		# Rename group 0 from "Unassociated" to "Default"
-		echo -e "  ${INFO} Upgrading gravity database from version 11 to 12"
-		pihole-FTL sqlite3 "${database}" < "${scriptPath}/11_to_12.sql"
-		version=12
-	fi
-	if [[ "$version" == "12" ]]; then
-		# Add column date_updated to adlist table
-		echo -e "  ${INFO} Upgrading gravity database from version 12 to 13"
-		pihole-FTL sqlite3 "${database}" < "${scriptPath}/12_to_13.sql"
-		version=13
-	fi
-	if [[ "$version" == "13" ]]; then
-		# Add columns number and status to adlist table
-		echo -e "  ${INFO} Upgrading gravity database from version 13 to 14"
-		pihole-FTL sqlite3 "${database}" < "${scriptPath}/13_to_14.sql"
-		version=14
-	fi
-	if [[ "$version" == "14" ]]; then
-		# Changes the vw_adlist created in 5_to_6
-		echo -e "  ${INFO} Upgrading gravity database from version 14 to 15"
-		pihole-FTL sqlite3 "${database}" < "${scriptPath}/14_to_15.sql"
-		version=15
-	fi
+        # Store audit domains in database table
+        if [ -e "${auditFile}" ]; then
+            echo -e "  ${INFO} Migrating content of ${auditFile} into new database"
+            # database_table_from_file is defined in gravity.sh
+            database_table_from_file "domain_audit" "${auditFile}"
+        fi
+    fi
+    if [[ "$version" == "2" ]]; then
+        # This migration script upgrades the gravity.db file by
+        # renaming the regex table to regex_blacklist, and
+        # creating a new regex_whitelist table + corresponding linking table and views
+        echo -e "  ${INFO} Upgrading gravity database from version 2 to 3"
+        pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/2_to_3.sql"
+        version=3
+    fi
+    if [[ "$version" == "3" ]]; then
+        # This migration script unifies the formally separated domain
+        # lists into a single table with a UNIQUE domain constraint
+        echo -e "  ${INFO} Upgrading gravity database from version 3 to 4"
+        pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/3_to_4.sql"
+        version=4
+    fi
+    if [[ "$version" == "4" ]]; then
+        # This migration script upgrades the gravity and list views
+        # implementing necessary changes for per-client blocking
+        echo -e "  ${INFO} Upgrading gravity database from version 4 to 5"
+        pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/4_to_5.sql"
+        version=5
+    fi
+    if [[ "$version" == "5" ]]; then
+        # This migration script upgrades the adlist view
+        # to return an ID used in gravity.sh
+        echo -e "  ${INFO} Upgrading gravity database from version 5 to 6"
+        pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/5_to_6.sql"
+        version=6
+    fi
+    if [[ "$version" == "6" ]]; then
+        # This migration script adds a special group with ID 0
+        # which is automatically associated to all clients not
+        # having their own group assignments
+        echo -e "  ${INFO} Upgrading gravity database from version 6 to 7"
+        pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/6_to_7.sql"
+        version=7
+    fi
+    if [[ "$version" == "7" ]]; then
+        # This migration script recreated the group table
+        # to ensure uniqueness on the group name
+        # We also add date_added and date_modified columns
+        echo -e "  ${INFO} Upgrading gravity database from version 7 to 8"
+        pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/7_to_8.sql"
+        version=8
+    fi
+    if [[ "$version" == "8" ]]; then
+        # This migration fixes some issues that were introduced
+        # in the previous migration script.
+        echo -e "  ${INFO} Upgrading gravity database from version 8 to 9"
+        pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/8_to_9.sql"
+        version=9
+    fi
+    if [[ "$version" == "9" ]]; then
+        # This migration drops unused tables and creates triggers to remove
+        # obsolete groups assignments when the linked items are deleted
+        echo -e "  ${INFO} Upgrading gravity database from version 9 to 10"
+        pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/9_to_10.sql"
+        version=10
+    fi
+    if [[ "$version" == "10" ]]; then
+        # This adds timestamp and an optional comment field to the client table
+        # These fields are only temporary and will be replaces by the columns
+        # defined in gravity.db.sql during gravity swapping. We add them here
+        # to keep the copying process generic (needs the same columns in both the
+        # source and the destination databases).
+        echo -e "  ${INFO} Upgrading gravity database from version 10 to 11"
+        pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/10_to_11.sql"
+        version=11
+    fi
+    if [[ "$version" == "11" ]]; then
+        # Rename group 0 from "Unassociated" to "Default"
+        echo -e "  ${INFO} Upgrading gravity database from version 11 to 12"
+        pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/11_to_12.sql"
+        version=12
+    fi
+    if [[ "$version" == "12" ]]; then
+        # Add column date_updated to adlist table
+        echo -e "  ${INFO} Upgrading gravity database from version 12 to 13"
+        pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/12_to_13.sql"
+        version=13
+    fi
+    if [[ "$version" == "13" ]]; then
+        # Add columns number and status to adlist table
+        echo -e "  ${INFO} Upgrading gravity database from version 13 to 14"
+        pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/13_to_14.sql"
+        version=14
+    fi
+    if [[ "$version" == "14" ]]; then
+        # Changes the vw_adlist created in 5_to_6
+        echo -e "  ${INFO} Upgrading gravity database from version 14 to 15"
+        pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/14_to_15.sql"
+        version=15
+    fi
 }

advanced/Scripts/database_migration/gravity/1_to_2.sql

@@ -4,9 +4,9 @@ BEGIN TRANSACTION;
 
 CREATE TABLE domain_audit
 (
-	id INTEGER PRIMARY KEY AUTOINCREMENT,
-	domain TEXT UNIQUE NOT NULL,
-	date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int))
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    domain TEXT UNIQUE NOT NULL,
+    date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int))
 );
 
 UPDATE info SET value = 2 WHERE property = 'version';

advanced/Scripts/database_migration/gravity/2_to_3.sql

@@ -8,9 +8,9 @@ ALTER TABLE regex RENAME TO regex_blacklist;
 
 CREATE TABLE regex_blacklist_by_group
 (
-	regex_blacklist_id INTEGER NOT NULL REFERENCES regex_blacklist (id),
-	group_id INTEGER NOT NULL REFERENCES "group" (id),
-	PRIMARY KEY (regex_blacklist_id, group_id)
+    regex_blacklist_id INTEGER NOT NULL REFERENCES regex_blacklist (id),
+    group_id INTEGER NOT NULL REFERENCES "group" (id),
+    PRIMARY KEY (regex_blacklist_id, group_id)
 );
 
 INSERT INTO regex_blacklist_by_group SELECT * FROM regex_by_group;
@@ -32,19 +32,19 @@ CREATE TRIGGER tr_regex_blacklist_update AFTER UPDATE ON regex_blacklist
 
 CREATE TABLE regex_whitelist
 (
-	id INTEGER PRIMARY KEY AUTOINCREMENT,
-	domain TEXT UNIQUE NOT NULL,
-	enabled BOOLEAN NOT NULL DEFAULT 1,
-	date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
-	date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
-	comment TEXT
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    domain TEXT UNIQUE NOT NULL,
+    enabled BOOLEAN NOT NULL DEFAULT 1,
+    date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
+    date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
+    comment TEXT
 );
 
 CREATE TABLE regex_whitelist_by_group
 (
-	regex_whitelist_id INTEGER NOT NULL REFERENCES regex_whitelist (id),
-	group_id INTEGER NOT NULL REFERENCES "group" (id),
-	PRIMARY KEY (regex_whitelist_id, group_id)
+    regex_whitelist_id INTEGER NOT NULL REFERENCES regex_whitelist (id),
+    group_id INTEGER NOT NULL REFERENCES "group" (id),
+    PRIMARY KEY (regex_whitelist_id, group_id)
 );
 
 CREATE VIEW vw_regex_whitelist AS SELECT DISTINCT domain

advanced/Scripts/database_migration/gravity/3_to_4.sql

@@ -6,13 +6,13 @@ BEGIN TRANSACTION;
 
 CREATE TABLE domainlist
 (
-	id INTEGER PRIMARY KEY AUTOINCREMENT,
-	type INTEGER NOT NULL DEFAULT 0,
-	domain TEXT UNIQUE NOT NULL,
-	enabled BOOLEAN NOT NULL DEFAULT 1,
-	date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
-	date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
-	comment TEXT
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    type INTEGER NOT NULL DEFAULT 0,
+    domain TEXT UNIQUE NOT NULL,
+    enabled BOOLEAN NOT NULL DEFAULT 1,
+    date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
+    date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
+    comment TEXT
 );
 
 ALTER TABLE whitelist ADD COLUMN type INTEGER;
@@ -41,9 +41,9 @@ DROP TABLE regex_whitelist_by_group;
 DROP TABLE regex_blacklist_by_group;
 CREATE TABLE domainlist_by_group
 (
-	domainlist_id INTEGER NOT NULL REFERENCES domainlist (id),
-	group_id INTEGER NOT NULL REFERENCES "group" (id),
-	PRIMARY KEY (domainlist_id, group_id)
+    domainlist_id INTEGER NOT NULL REFERENCES domainlist (id),
+    group_id INTEGER NOT NULL REFERENCES "group" (id),
+    PRIMARY KEY (domainlist_id, group_id)
 );
 
 DROP TRIGGER tr_whitelist_update;

advanced/Scripts/database_migration/gravity/4_to_5.sql

@@ -7,9 +7,9 @@ BEGIN TRANSACTION;
 DROP TABLE gravity;
 CREATE TABLE gravity
 (
-	domain TEXT NOT NULL,
-	adlist_id INTEGER NOT NULL REFERENCES adlist (id),
-	PRIMARY KEY(domain, adlist_id)
+    domain TEXT NOT NULL,
+    adlist_id INTEGER NOT NULL REFERENCES adlist (id),
+    PRIMARY KEY(domain, adlist_id)
 );
 
 DROP VIEW vw_gravity;
@@ -22,15 +22,15 @@ CREATE VIEW vw_gravity AS SELECT domain, adlist_by_group.group_id AS group_id
 
 CREATE TABLE client
 (
-	id INTEGER PRIMARY KEY AUTOINCREMENT,
-	ip TEXT NOL NULL UNIQUE
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    ip TEXT NOL NULL UNIQUE
 );
 
 CREATE TABLE client_by_group
 (
-	client_id INTEGER NOT NULL REFERENCES client (id),
-	group_id INTEGER NOT NULL REFERENCES "group" (id),
-	PRIMARY KEY (client_id, group_id)
+    client_id INTEGER NOT NULL REFERENCES client (id),
+    group_id INTEGER NOT NULL REFERENCES "group" (id),
+    PRIMARY KEY (client_id, group_id)
 );
 
 UPDATE info SET value = 5 WHERE property = 'version';

advanced/Scripts/database_migration/gravity/7_to_8.sql

@@ -8,12 +8,12 @@ ALTER TABLE "group" RENAME TO "group__";
 
 CREATE TABLE "group"
 (
-	id INTEGER PRIMARY KEY AUTOINCREMENT,
-	enabled BOOLEAN NOT NULL DEFAULT 1,
-	name TEXT UNIQUE NOT NULL,
-	date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
-	date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
-	description TEXT
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    enabled BOOLEAN NOT NULL DEFAULT 1,
+    name TEXT UNIQUE NOT NULL,
+    date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
+    date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
+    description TEXT
 );
 
 CREATE TRIGGER tr_group_update AFTER UPDATE ON "group"

advanced/Scripts/list.sh

@@ -150,18 +150,18 @@ AddDomain() {
     domain="$1"
 
     # Is the domain in the list we want to add it to?
-    num="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}';")"
+    num="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}';")"
     requestedListname="$(GetListnameFromTypeId "${typeId}")"
 
     if [[ "${num}" -ne 0 ]]; then
-        existingTypeId="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT type FROM domainlist WHERE domain = '${domain}';")"
+        existingTypeId="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT type FROM domainlist WHERE domain = '${domain}';")"
         if [[ "${existingTypeId}" == "${typeId}" ]]; then
             if [[ "${verbose}" == true ]]; then
                 echo -e "  ${INFO} ${1} already exists in ${requestedListname}, no need to add!"
             fi
         else
             existingListname="$(GetListnameFromTypeId "${existingTypeId}")"
-            pihole-FTL sqlite3 "${gravityDBfile}" "UPDATE domainlist SET type = ${typeId} WHERE domain='${domain}';"
+            pihole-FTL sqlite3 -ni "${gravityDBfile}" "UPDATE domainlist SET type = ${typeId} WHERE domain='${domain}';"
             if [[ "${verbose}" == true ]]; then
                 echo -e "  ${INFO} ${1} already exists in ${existingListname}, it has been moved to ${requestedListname}!"
             fi
@@ -177,10 +177,10 @@ AddDomain() {
     # Insert only the domain here. The enabled and date_added fields will be filled
     # with their default values (enabled = true, date_added = current timestamp)
     if [[ -z "${comment}" ]]; then
-        pihole-FTL sqlite3 "${gravityDBfile}" "INSERT INTO domainlist (domain,type) VALUES ('${domain}',${typeId});"
+        pihole-FTL sqlite3 -ni "${gravityDBfile}" "INSERT INTO domainlist (domain,type) VALUES ('${domain}',${typeId});"
     else
         # also add comment when variable has been set through the "--comment" option
-        pihole-FTL sqlite3 "${gravityDBfile}" "INSERT INTO domainlist (domain,type,comment) VALUES ('${domain}',${typeId},'${comment}');"
+        pihole-FTL sqlite3 -ni "${gravityDBfile}" "INSERT INTO domainlist (domain,type,comment) VALUES ('${domain}',${typeId},'${comment}');"
     fi
 }
 
@@ -189,7 +189,7 @@ RemoveDomain() {
     domain="$1"
 
     # Is the domain in the list we want to remove it from?
-    num="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}' AND type = ${typeId};")"
+    num="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}' AND type = ${typeId};")"
 
     requestedListname="$(GetListnameFromTypeId "${typeId}")"
 
@@ -206,14 +206,14 @@ RemoveDomain() {
     fi
     reload=true
     # Remove it from the current list
-    pihole-FTL sqlite3 "${gravityDBfile}" "DELETE FROM domainlist WHERE domain = '${domain}' AND type = ${typeId};"
+    pihole-FTL sqlite3 -ni "${gravityDBfile}" "DELETE FROM domainlist WHERE domain = '${domain}' AND type = ${typeId};"
 }
 
 Displaylist() {
     local count num_pipes domain enabled status nicedate requestedListname
 
     requestedListname="$(GetListnameFromTypeId "${typeId}")"
-    data="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT domain,enabled,date_modified FROM domainlist WHERE type = ${typeId};" 2> /dev/null)"
+    data="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT domain,enabled,date_modified FROM domainlist WHERE type = ${typeId};" 2> /dev/null)"
 
     if [[ -z $data ]]; then
         echo -e "Not showing empty list"
@@ -251,10 +251,10 @@ Displaylist() {
 }
 
 NukeList() {
-    count=$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(1) FROM domainlist WHERE type = ${typeId};")
+    count=$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT COUNT(1) FROM domainlist WHERE type = ${typeId};")
     listname="$(GetListnameFromTypeId "${typeId}")"
     if [ "$count" -gt 0 ];then
-        pihole-FTL sqlite3 "${gravityDBfile}" "DELETE FROM domainlist WHERE type = ${typeId};"
+        pihole-FTL sqlite3 -ni "${gravityDBfile}" "DELETE FROM domainlist WHERE type = ${typeId};"
         echo "  ${TICK} Removed ${count} domain(s) from the ${listname}"
     else
         echo "  ${INFO} ${listname} already empty. Nothing to do!"

advanced/Scripts/piholeARPTable.sh

@@ -39,7 +39,7 @@ flushARP(){
     # Truncate network_addresses table in pihole-FTL.db
     # This needs to be done before we can truncate the network table due to
     # foreign key constraints
-    if ! output=$(pihole-FTL sqlite3 "${DBFILE}" "DELETE FROM network_addresses" 2>&1); then
+    if ! output=$(pihole-FTL sqlite3 -ni "${DBFILE}" "DELETE FROM network_addresses" 2>&1); then
         echo -e "${OVER}  ${CROSS} Failed to truncate network_addresses table"
         echo "  Database location: ${DBFILE}"
         echo "  Output: ${output}"
@@ -47,7 +47,7 @@ flushARP(){
     fi
 
     # Truncate network table in pihole-FTL.db
-    if ! output=$(pihole-FTL sqlite3 "${DBFILE}" "DELETE FROM network" 2>&1); then
+    if ! output=$(pihole-FTL sqlite3 -ni "${DBFILE}" "DELETE FROM network" 2>&1); then
         echo -e "${OVER}  ${CROSS} Failed to truncate network table"
         echo "  Database location: ${DBFILE}"
         echo "  Output: ${output}"

advanced/Scripts/piholeDebug.sh

@@ -66,6 +66,8 @@ RUN_DIRECTORY="/run"
 LOG_DIRECTORY="/var/log/pihole"
 WEB_SERVER_LOG_DIRECTORY="/var/log/lighttpd"
 WEB_SERVER_CONFIG_DIRECTORY="/etc/lighttpd"
+WEB_SERVER_CONFIG_DIRECTORY_FEDORA="${WEB_SERVER_CONFIG_DIRECTORY}/conf.d"
+WEB_SERVER_CONFIG_DIRECTORY_DEBIAN="${WEB_SERVER_CONFIG_DIRECTORY}/conf-enabled"
 HTML_DIRECTORY="/var/www/html"
 WEB_GIT_DIRECTORY="${HTML_DIRECTORY}/admin"
 SHM_DIRECTORY="/dev/shm"
@@ -77,6 +79,8 @@ PIHOLE_CRON_FILE="${CRON_D_DIRECTORY}/pihole"
 
 WEB_SERVER_CONFIG_FILE="${WEB_SERVER_CONFIG_DIRECTORY}/lighttpd.conf"
 WEB_SERVER_CUSTOM_CONFIG_FILE="${WEB_SERVER_CONFIG_DIRECTORY}/external.conf"
+WEB_SERVER_PIHOLE_CONFIG_FILE_DEBIAN="${WEB_SERVER_CONFIG_DIRECTORY_DEBIAN}/15-pihole-admin.conf"
+WEB_SERVER_PIHOLE_CONFIG_FILE_FEDORA="${WEB_SERVER_CONFIG_DIRECTORY_FEDORA}/pihole-admin.conf"
 
 PIHOLE_INSTALL_LOG_FILE="${PIHOLE_DIRECTORY}/install.log"
 PIHOLE_RAW_BLOCKLIST_FILES="${PIHOLE_DIRECTORY}/list.*"
@@ -140,6 +144,8 @@ PIHOLE_PROCESSES=( "lighttpd" "pihole-FTL" )
 REQUIRED_FILES=("${PIHOLE_CRON_FILE}"
 "${WEB_SERVER_CONFIG_FILE}"
 "${WEB_SERVER_CUSTOM_CONFIG_FILE}"
+"${WEB_SERVER_PIHOLE_CONFIG_FILE_DEBIAN}"
+"${WEB_SERVER_PIHOLE_CONFIG_FILE_FEDORA}"
 "${PIHOLE_INSTALL_LOG_FILE}"
 "${PIHOLE_RAW_BLOCKLIST_FILES}"
 "${PIHOLE_LOCAL_HOSTS_FILE}"
@@ -224,10 +230,8 @@ initialize_debug() {
 
 # This is a function for visually displaying the current test that is being run.
 # Accepts one variable: the name of what is being diagnosed
-# Colors do not show in the dasboard, but the icons do: [i], [✓], and [✗]
 echo_current_diagnostic() {
     # Colors are used for visually distinguishing each test in the output
-    # These colors do not show in the GUI, but the formatting will
     log_write "\\n${COL_PURPLE}*** [ DIAGNOSING ]:${COL_NC} ${1}"
 }
 
@@ -445,7 +449,7 @@ os_check() {
 }
 
 diagnose_operating_system() {
-    # error message in a variable so we can easily modify it later (or re-use it)
+    # error message in a variable so we can easily modify it later (or reuse it)
     local error_msg="Distribution unknown -- most likely you are on an unsupported platform and may run into issues."
     # Display the current test that is running
     echo_current_diagnostic "Operating system"
@@ -857,11 +861,15 @@ dig_at() {
         local record_type="A"
     fi
 
-    # Find a random blocked url that has not been whitelisted.
+    # Find a random blocked url that has not been whitelisted and is not ABP style.
     # This helps emulate queries to different domains that a user might query
     # It will also give extra assurance that Pi-hole is correctly resolving and blocking domains
     local random_url
-    random_url=$(pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity ORDER BY RANDOM() LIMIT 1")
+    random_url=$(pihole-FTL sqlite3 -ni "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity WHERE domain not like '||%^' ORDER BY RANDOM() LIMIT 1")
+    # Fallback if no non-ABP style domains were found
+    if [ -z "${random_url}" ]; then
+        random_url="flurry.com"
+    fi
 
     # Next we need to check if Pi-hole can resolve a domain when the query is sent to it's IP address
     # This better emulates how clients will interact with Pi-hole as opposed to above where Pi-hole is
@@ -977,6 +985,20 @@ ftl_full_status(){
     fi
 }
 
+lighttpd_test_configuration(){
+    # let lighttpd test it's own configuration
+    local lighttpd_conf_test
+    echo_current_diagnostic "Lighttpd configuration test"
+    lighttpd_conf_test=$(lighttpd -tt -f /etc/lighttpd/lighttpd.conf)
+    if [ -z "${lighttpd_conf_test}" ]; then
+        # empty output
+        log_write "${TICK} ${COL_GREEN}No error in lighttpd configuration${COL_NC}"
+    else
+        log_write "${CROSS} ${COL_RED}Error in lighttpd configuration${COL_NC}"
+        log_write "   ${lighttpd_conf_test}"
+    fi
+}
+
 make_array_from_file() {
     local filename="${1}"
     # The second argument can put a limit on how many line should be read from the file
@@ -1069,10 +1091,13 @@ dir_check() {
         # check if exists first; if it does,
         if ls "${filename}" 1> /dev/null 2>&1; then
             # do nothing
-            :
+            true
+            return
         else
             # Otherwise, show an error
             log_write "${COL_RED}${directory} does not exist.${COL_NC}"
+            false
+            return
         fi
     done
 }
@@ -1080,6 +1105,19 @@ dir_check() {
 list_files_in_dir() {
     # Set the first argument passed to this function as a named variable for better readability
     local dir_to_parse="${1}"
+
+    # show files and sizes of some directories, don't print the file content (yet)
+    if [[ "${dir_to_parse}" == "${SHM_DIRECTORY}" ]]; then
+        # SHM file - we do not want to see the content, but we want to see the files and their sizes
+        log_write "$(ls -lh "${dir_to_parse}/")"
+    elif [[ "${dir_to_parse}" == "${WEB_SERVER_CONFIG_DIRECTORY_FEDORA}" ]]; then
+        # we want to see all files files in /etc/lighttpd/conf.d
+        log_write "$(ls -lh "${dir_to_parse}/" 2> /dev/null )"
+    elif [[ "${dir_to_parse}" == "${WEB_SERVER_CONFIG_DIRECTORY_DEBIAN}" ]]; then
+        # we want to see all files files in /etc/lighttpd/conf.d
+        log_write "$(ls -lh "${dir_to_parse}/"/ 2> /dev/null )"
+    fi
+
     # Store the files found in an array
     mapfile -t files_found < <(ls "${dir_to_parse}")
     # For each file in the array,
@@ -1095,11 +1133,8 @@ list_files_in_dir() {
             [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_WEB_SERVER_ACCESS_LOG_FILE}" ]] || \
             [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_LOG_GZIPS}" ]]; then
             :
-        elif [[ "${dir_to_parse}" == "${SHM_DIRECTORY}" ]]; then
-            # SHM file - we do not want to see the content, but we want to see the files and their sizes
-            log_write "$(ls -lhd "${dir_to_parse}"/"${each_file}")"
         elif [[ "${dir_to_parse}" == "${DNSMASQ_D_DIRECTORY}" ]]; then
-            # in case of the dnsmasq directory inlcuede all files in the debug output
+            # in case of the dnsmasq directory include all files in the debug output
             log_write "\\n${COL_GREEN}$(ls -lhd "${dir_to_parse}"/"${each_file}")${COL_NC}"
             make_array_from_file "${dir_to_parse}/${each_file}"
         else
@@ -1132,9 +1167,10 @@ show_content_of_files_in_dir() {
     # Set a local variable for better readability
     local directory="${1}"
     # Check if the directory exists
-    dir_check "${directory}"
-    # if it does, list the files in it
-    list_files_in_dir "${directory}"
+    if dir_check "${directory}"; then
+        # if it does, list the files in it
+        list_files_in_dir "${directory}"
+    fi
 }
 
 show_content_of_pihole_files() {
@@ -1142,6 +1178,8 @@ show_content_of_pihole_files() {
     show_content_of_files_in_dir "${PIHOLE_DIRECTORY}"
     show_content_of_files_in_dir "${DNSMASQ_D_DIRECTORY}"
     show_content_of_files_in_dir "${WEB_SERVER_CONFIG_DIRECTORY}"
+    show_content_of_files_in_dir "${WEB_SERVER_CONFIG_DIRECTORY_FEDORA}"
+    show_content_of_files_in_dir "${WEB_SERVER_CONFIG_DIRECTORY_DEBIAN}"
     show_content_of_files_in_dir "${CRON_D_DIRECTORY}"
     show_content_of_files_in_dir "${WEB_SERVER_LOG_DIRECTORY}"
     show_content_of_files_in_dir "${LOG_DIRECTORY}"
@@ -1188,7 +1226,7 @@ show_db_entries() {
     IFS=$'\r\n'
     local entries=()
     mapfile -t entries < <(\
-        pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" \
+        pihole-FTL sqlite3 -ni "${PIHOLE_GRAVITY_DB_FILE}" \
             -cmd ".headers on" \
             -cmd ".mode column" \
             -cmd ".width ${widths}" \
@@ -1213,7 +1251,7 @@ show_FTL_db_entries() {
     IFS=$'\r\n'
     local entries=()
     mapfile -t entries < <(\
-        pihole-FTL sqlite3 "${PIHOLE_FTL_DB_FILE}" \
+        pihole-FTL sqlite3 -ni "${PIHOLE_FTL_DB_FILE}" \
             -cmd ".headers on" \
             -cmd ".mode column" \
             -cmd ".width ${widths}" \
@@ -1279,7 +1317,7 @@ analyze_gravity_list() {
     fi
 
     show_db_entries "Info table" "SELECT property,value FROM info" "20 40"
-    gravity_updated_raw="$(pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT value FROM info where property = 'updated'")"
+    gravity_updated_raw="$(pihole-FTL sqlite3 -ni "${PIHOLE_GRAVITY_DB_FILE}" "SELECT value FROM info where property = 'updated'")"
     gravity_updated="$(date -d @"${gravity_updated_raw}")"
     log_write "   Last gravity run finished at: ${COL_CYAN}${gravity_updated}${COL_NC}"
     log_write ""
@@ -1287,7 +1325,7 @@ analyze_gravity_list() {
     OLD_IFS="$IFS"
     IFS=$'\r\n'
     local gravity_sample=()
-    mapfile -t gravity_sample < <(pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity LIMIT 10")
+    mapfile -t gravity_sample < <(pihole-FTL sqlite3 -ni "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity LIMIT 10")
     log_write "   ${COL_CYAN}----- First 10 Gravity Domains -----${COL_NC}"
 
     for line in "${gravity_sample[@]}"; do
@@ -1319,7 +1357,7 @@ database_integrity_check(){
 
       log_write "${INFO} Checking foreign key constraints of ${database} ... (this can take several minutes)"
       unset result
-      result="$(pihole-FTL sqlite3 "${database}" -cmd ".headers on" -cmd ".mode column" "PRAGMA foreign_key_check" 2>&1 & spinner)"
+      result="$(pihole-FTL sqlite3 -ni "${database}" -cmd ".headers on" -cmd ".mode column" "PRAGMA foreign_key_check" 2>&1 & spinner)"
       if [[ -z ${result} ]]; then
         log_write "${TICK} No foreign key errors in ${database}"
       else
@@ -1465,7 +1503,7 @@ upload_to_tricorder() {
     # If no token was generated
     else
         # Show an error and some help instructions
-        # Skip this if being called from web interface and autmatic mode was not chosen (users opt-out to upload)
+        # Skip this if being called from web interface and automatic mode was not chosen (users opt-out to upload)
         if [[ "${WEBCALL}" ]] && [[ ! "${AUTOMATED}" ]]; then
             :
         else
@@ -1496,6 +1534,7 @@ check_name_resolution
 check_dhcp_servers
 process_status
 ftl_full_status
+lighttpd_test_configuration
 parse_setup_vars
 check_x_headers
 analyze_ftl_db

advanced/Scripts/piholeLogFlush.sh

@@ -63,7 +63,7 @@ else
         fi
     fi
     # Delete most recent 24 hours from FTL's database, leave even older data intact (don't wipe out all history)
-    deleted=$(pihole-FTL sqlite3 "${DBFILE}" "DELETE FROM query_storage WHERE timestamp >= strftime('%s','now')-86400; select changes() from query_storage limit 1")
+    deleted=$(pihole-FTL sqlite3 -ni "${DBFILE}" "DELETE FROM query_storage WHERE timestamp >= strftime('%s','now')-86400; select changes() from query_storage limit 1")
 
     # Restart pihole-FTL to force reloading history
     sudo pihole restartdns

advanced/Scripts/query.sh

@@ -30,33 +30,6 @@ gravityDBfile="${GRAVITYDB}"
 colfile="/opt/pihole/COL_TABLE"
 source "${colfile}"
 
-# Scan an array of files for matching strings
-scanList(){
-    # Escape full stops
-    local domain="${1}" esc_domain="${1//./\\.}" lists="${2}" list_type="${3:-}"
-
-    # Prevent grep from printing file path
-    cd "$piholeDir" || exit 1
-
-    # Prevent grep -i matching slowly: https://bit.ly/2xFXtUX
-    export LC_CTYPE=C
-
-    # /dev/null forces filename to be printed when only one list has been generated
-    case "${list_type}" in
-        "exact" ) grep -i -E -l "(^|(?<!#)\\s)${esc_domain}($|\\s|#)" ${lists} /dev/null 2>/dev/null;;
-        # Iterate through each regexp and check whether it matches the domainQuery
-        # If it does, print the matching regexp and continue looping
-        # Input 1 - regexps | Input 2 - domainQuery
-        "regex" )
-            for list in ${lists}; do
-                if [[ "${domain}" =~ ${list} ]]; then
-                    printf "%b\n" "${list}";
-                fi
-            done;;
-        *       ) grep -i "${esc_domain}" ${lists} /dev/null 2>/dev/null;;
-    esac
-}
-
 if [[ "${options}" == "-h" ]] || [[ "${options}" == "--help" ]]; then
     echo "Usage: pihole -q [option] <domain>
 Example: 'pihole -q -exact domain.com'
@@ -84,17 +57,32 @@ options=$(sed -E 's/ ?-(all|exact) ?//g' <<< "${options}")
 case "${options}" in
     ""             ) str="No domain specified";;
     *" "*          ) str="Unknown query option specified";;
-    *[![:ascii:]]* ) domainQuery=$(idn2 "${options}");;
-    *              ) domainQuery="${options}";;
+    *[![:ascii:]]* ) rawDomainQuery=$(idn2 "${options}");;
+    *              ) rawDomainQuery="${options}";;
 esac
 
+# convert the domain to lowercase
+domainQuery=$(echo "${rawDomainQuery}" | tr '[:upper:]' '[:lower:]')
+
 if [[ -n "${str:-}" ]]; then
     echo -e "${str}${COL_NC}\\nTry 'pihole -q --help' for more information."
     exit 1
 fi
 
+# Scan a domain again a list of RegEX
+scanRegExList(){
+    local domain="${1}" list="${2}"
+
+    for entry in ${list}; do
+        if [[ "${domain}" =~ ${entry} ]]; then
+            printf "%b\n" "${entry}";
+        fi
+    done
+
+}
+
 scanDatabaseTable() {
-    local domain table list_type querystr result extra
+    local domain table list_type querystr result extra abpquerystr abpfound abpentry searchstr
     domain="$(printf "%q" "${1}")"
     table="${2}"
     list_type="${3:-}"
@@ -104,9 +92,34 @@ scanDatabaseTable() {
     # behavior. The "ESCAPE '\'" clause specifies that an underscore preceded by an '\' should be matched
     # as a literal underscore character. We pretreat the $domain variable accordingly to escape underscores.
     if [[ "${table}" == "gravity" ]]; then
+
+        # Are there ABP entries on gravity?
+        # Return 1 if abp_domain=1 or Zero if abp_domain=0 or not set
+        abpquerystr="SELECT EXISTS (SELECT 1 FROM info WHERE property='abp_domains' and value='1')"
+        abpfound="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "${abpquerystr}")" 2> /dev/null
+
+        # Create search string for ABP entries only if needed
+        if [ "${abpfound}" -eq 1 ]; then
+            abpentry="${domain}"
+
+            searchstr="'||${abpentry}^'"
+
+            # While a dot is found ...
+            while [ "${abpentry}" != "${abpentry/./}" ]
+            do
+                # ... remove text before the dot (including the dot) and append the result to $searchstr
+                abpentry=$(echo "${abpentry}" | cut -f 2- -d '.')
+                searchstr="$searchstr, '||${abpentry}^'"
+            done
+
+            # The final search string will look like:
+            # "domain IN ('||sub2.sub1.domain.com^', '||sub1.domain.com^', '||domain.com^', '||com^') OR"
+            searchstr="domain IN (${searchstr}) OR "
+        fi
+
         case "${exact}" in
             "exact" ) querystr="SELECT gravity.domain,adlist.address,adlist.enabled FROM gravity LEFT JOIN adlist ON adlist.id = gravity.adlist_id WHERE domain = '${domain}'";;
-            *       ) querystr="SELECT gravity.domain,adlist.address,adlist.enabled FROM gravity LEFT JOIN adlist ON adlist.id = gravity.adlist_id WHERE domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";;
+            *       ) querystr="SELECT gravity.domain,adlist.address,adlist.enabled FROM gravity LEFT JOIN adlist ON adlist.id = gravity.adlist_id WHERE ${searchstr} domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";;
         esac
     else
         case "${exact}" in
@@ -116,7 +129,7 @@ scanDatabaseTable() {
     fi
 
     # Send prepared query to gravity database
-    result="$(pihole-FTL sqlite3 "${gravityDBfile}" "${querystr}")" 2> /dev/null
+    result="$(pihole-FTL sqlite3 -ni -separator ',' "${gravityDBfile}" "${querystr}")" 2> /dev/null
     if [[ -z "${result}" ]]; then
         # Return early when there are no matches in this table
         return
@@ -136,8 +149,8 @@ scanDatabaseTable() {
     # Loop over results and print them
     mapfile -t results <<< "${result}"
     for result in "${results[@]}"; do
-        domain="${result/|*}"
-        if [[ "${result#*|}" == "0" ]]; then
+        domain="${result/,*}"
+        if [[ "${result#*,}" == "0" ]]; then
             extra=" (disabled)"
         else
             extra=""
@@ -153,14 +166,14 @@ scanRegexDatabaseTable() {
     list_type="${3:-}"
 
     # Query all regex from the corresponding database tables
-    mapfile -t regexList < <(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT domain FROM domainlist WHERE type = ${list_type}" 2> /dev/null)
+    mapfile -t regexList < <(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT domain FROM domainlist WHERE type = ${list_type}" 2> /dev/null)
 
     # If we have regexps to process
     if [[ "${#regexList[@]}" -ne 0 ]]; then
         # Split regexps over a new line
         str_regexList=$(printf '%s\n' "${regexList[@]}")
         # Check domain against regexps
-        mapfile -t regexMatches < <(scanList "${domain}" "${str_regexList}" "regex")
+        mapfile -t regexMatches < <(scanRegExList "${domain}" "${str_regexList}")
         # If there were regex matches
         if [[  "${#regexMatches[@]}" -ne 0 ]]; then
             # Split matching regexps over a new line
@@ -212,10 +225,10 @@ if [[ -n "${exact}" ]]; then
 fi
 
 for result in "${results[@]}"; do
-    match="${result/|*/}"
-    extra="${result#*|}"
-    adlistAddress="${extra/|*/}"
-    extra="${extra#*|}"
+    match="${result/,*/}"
+    extra="${result#*,}"
+    adlistAddress="${extra/,*/}"
+    extra="${extra#*,}"
     if [[ "${extra}" == "0" ]]; then
         extra=" (disabled)"
     else

advanced/Scripts/update.sh

@@ -11,7 +11,7 @@
 # Please see LICENSE file for your rights under this license.
 
 # Variables
-readonly ADMIN_INTERFACE_GIT_URL="https://github.com/pi-hole/AdminLTE.git"
+readonly ADMIN_INTERFACE_GIT_URL="https://github.com/pi-hole/web.git"
 readonly ADMIN_INTERFACE_DIR="/var/www/html/admin"
 readonly PI_HOLE_GIT_URL="https://github.com/pi-hole/pi-hole.git"
 readonly PI_HOLE_FILES_DIR="/etc/.pihole"

advanced/Scripts/updatecheck.sh

@@ -97,10 +97,10 @@ if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then
     WEB_HASH="$(get_local_hash /var/www/html/admin)"
     addOrEditKeyValPair "${VERSION_FILE}" "WEB_HASH" "${WEB_HASH}"
 
-    GITHUB_WEB_VERSION="$(get_remote_version AdminLTE)"
+    GITHUB_WEB_VERSION="$(get_remote_version web)"
     addOrEditKeyValPair "${VERSION_FILE}" "GITHUB_WEB_VERSION" "${GITHUB_WEB_VERSION}"
 
-    GITHUB_WEB_HASH="$(get_remote_hash AdminLTE "${WEB_BRANCH}")"
+    GITHUB_WEB_HASH="$(get_remote_hash web "${WEB_BRANCH}")"
     addOrEditKeyValPair "${VERSION_FILE}" "GITHUB_WEB_HASH" "${GITHUB_WEB_HASH}"
 
 fi

advanced/Scripts/utils.sh

@@ -57,7 +57,11 @@ addKey(){
   # touch file to prevent grep error if file does not exist yet
   touch "${file}"
 
-  if ! grep -q "^${key}" "${file}"; then
+  # Match key against entire line, using both anchors. We assume
+  # that the file's keys never have bounding whitespace. Anchors
+  # are necessary to ensure the key is considered absent when it
+  # is a substring of another key present in the file.
+  if ! grep -q "^${key}$" "${file}"; then
     # Key does not exist, add it.
     echo "${key}" >> "${file}"
   fi

advanced/Scripts/version.sh

@@ -28,7 +28,7 @@ fi
 getLocalVersion() {
     case ${1} in
         "Pi-hole"   )  echo "${CORE_VERSION:=N/A}";;
-        "AdminLTE"  )  [ "${INSTALL_WEB_INTERFACE}" = true ] && echo "${WEB_VERSION:=N/A}";;
+        "web"  )  [ "${INSTALL_WEB_INTERFACE}" = true ] && echo "${WEB_VERSION:=N/A}";;
         "FTL"       )  echo "${FTL_VERSION:=N/A}";;
     esac
 }
@@ -36,7 +36,7 @@ getLocalVersion() {
 getLocalHash() {
     case ${1} in
         "Pi-hole"   )  echo "${CORE_HASH:=N/A}";;
-        "AdminLTE"  )  [ "${INSTALL_WEB_INTERFACE}" = true ] && echo "${WEB_HASH:=N/A}";;
+        "web"  )  [ "${INSTALL_WEB_INTERFACE}" = true ] && echo "${WEB_HASH:=N/A}";;
         "FTL"       )  echo "${FTL_HASH:=N/A}";;
     esac
 }
@@ -44,7 +44,7 @@ getLocalHash() {
 getRemoteHash(){
     case ${1} in
         "Pi-hole"   )  echo "${GITHUB_CORE_HASH:=N/A}";;
-        "AdminLTE"  )  [ "${INSTALL_WEB_INTERFACE}" = true ] && echo "${GITHUB_WEB_HASH:=N/A}";;
+        "web"  )  [ "${INSTALL_WEB_INTERFACE}" = true ] && echo "${GITHUB_WEB_HASH:=N/A}";;
         "FTL"       )  echo "${GITHUB_FTL_HASH:=N/A}";;
     esac
 }
@@ -52,7 +52,7 @@ getRemoteHash(){
 getRemoteVersion(){
     case ${1} in
         "Pi-hole"   )  echo "${GITHUB_CORE_VERSION:=N/A}";;
-        "AdminLTE"  )  [ "${INSTALL_WEB_INTERFACE}" = true ] && echo "${GITHUB_WEB_VERSION:=N/A}";;
+        "web"  )  [ "${INSTALL_WEB_INTERFACE}" = true ] && echo "${GITHUB_WEB_VERSION:=N/A}";;
         "FTL"       )  echo "${GITHUB_FTL_VERSION:=N/A}";;
     esac
 }
@@ -60,13 +60,13 @@ getRemoteVersion(){
 getLocalBranch(){
     case ${1} in
         "Pi-hole"   )  echo "${CORE_BRANCH:=N/A}";;
-        "AdminLTE"  )  [ "${INSTALL_WEB_INTERFACE}" = true ] && echo "${WEB_BRANCH:=N/A}";;
+        "web"  )  [ "${INSTALL_WEB_INTERFACE}" = true ] && echo "${WEB_BRANCH:=N/A}";;
         "FTL"       )  echo "${FTL_BRANCH:=N/A}";;
     esac
 }
 
 versionOutput() {
-    if [ "$1" = "AdminLTE" ] && [ "${INSTALL_WEB_INTERFACE}" != true ]; then
+    if [ "$1" = "web" ] && [ "${INSTALL_WEB_INTERFACE}" != true ]; then
         echo "  WebAdmin not installed"
         return 1
     fi
@@ -117,7 +117,7 @@ defaultOutput() {
     versionOutput "Pi-hole" "$@"
 
     if [ "${INSTALL_WEB_INTERFACE}" = true ]; then
-        versionOutput "AdminLTE" "$@"
+        versionOutput "web" "$@"
     fi
 
     versionOutput "FTL" "$@"
@@ -130,7 +130,7 @@ Show Pi-hole, Admin Console & FTL versions
 
 Repositories:
   -p, --pihole         Only retrieve info regarding Pi-hole repository
-  -a, --admin          Only retrieve info regarding AdminLTE repository
+  -a, --admin          Only retrieve info regarding web repository
   -f, --ftl            Only retrieve info regarding FTL repository
 
 Options:
@@ -143,7 +143,7 @@ Options:
 
 case "${1}" in
     "-p" | "--pihole"    ) shift; versionOutput "Pi-hole" "$@";;
-    "-a" | "--admin"     ) shift; versionOutput "AdminLTE" "$@";;
+    "-a" | "--admin"     ) shift; versionOutput "web" "$@";;
     "-f" | "--ftl"       ) shift; versionOutput "FTL" "$@";;
     "-h" | "--help"      ) helpFunc;;
     *                    ) defaultOutput "$@";;

advanced/Scripts/webpage.sh

@@ -22,12 +22,14 @@ readonly dnscustomcnamefile="/etc/dnsmasq.d/05-pihole-custom-cname.conf"
 
 readonly gravityDBfile="/etc/pihole/gravity.db"
 
-# Source install script for ${setupVars}, ${PI_HOLE_BIN_DIR} and valid_ip()
-readonly PI_HOLE_FILES_DIR="/etc/.pihole"
-# shellcheck disable=SC2034  # used in basic-install to source the script without running it
-SKIP_INSTALL="true"
-source "${PI_HOLE_FILES_DIR}/automated install/basic-install.sh"
 
+readonly setupVars="/etc/pihole/setupVars.conf"
+readonly PI_HOLE_BIN_DIR="/usr/local/bin"
+
+# Root of the web server
+readonly webroot="/var/www/html"
+
+# Source utils script
 utilsfile="/opt/pihole/utils.sh"
 source "${utilsfile}"
 
@@ -98,6 +100,47 @@ HashPassword() {
     echo "${return}"
 }
 
+# Check an IP address to see if it is a valid one
+valid_ip() {
+    # Local, named variables
+    local ip=${1}
+    local stat=1
+
+    # Regex matching one IPv4 component, i.e. an integer from 0 to 255.
+    # See https://tools.ietf.org/html/rfc1340
+    local ipv4elem="(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]?|0)";
+    # Regex matching an optional port (starting with '#') range of 1-65536
+    local portelem="(#(6553[0-5]|655[0-2][0-9]|65[0-4][0-9]{2}|6[0-4][0-9]{3}|[1-5][0-9]{4}|[1-9][0-9]{0,3}|0))?";
+    # Build a full IPv4 regex from the above subexpressions
+    local regex="^${ipv4elem}\\.${ipv4elem}\\.${ipv4elem}\\.${ipv4elem}${portelem}$"
+
+    # Evaluate the regex, and return the result
+    [[ $ip =~ ${regex} ]]
+
+    stat=$?
+    return "${stat}"
+}
+
+valid_ip6() {
+    local ip=${1}
+    local stat=1
+
+    # Regex matching one IPv6 element, i.e. a hex value from 0000 to FFFF
+    local ipv6elem="[0-9a-fA-F]{1,4}"
+    # Regex matching an IPv6 CIDR, i.e. 1 to 128
+    local v6cidr="(\\/([1-9]|[1-9][0-9]|1[0-1][0-9]|12[0-8])){0,1}"
+    # Regex matching an optional port (starting with '#') range of 1-65536
+    local portelem="(#(6553[0-5]|655[0-2][0-9]|65[0-4][0-9]{2}|6[0-4][0-9]{3}|[1-5][0-9]{4}|[1-9][0-9]{0,3}|0))?";
+    # Build a full IPv6 regex from the above subexpressions
+    local regex="^(((${ipv6elem}))*((:${ipv6elem}))*::((${ipv6elem}))*((:${ipv6elem}))*|((${ipv6elem}))((:${ipv6elem})){7})${v6cidr}${portelem}$"
+
+    # Evaluate the regex, and return the result
+    [[ ${ip} =~ ${regex} ]]
+
+    stat=$?
+    return "${stat}"
+}
+
 SetWebPassword() {
     if [ "${SUDO_USER}" == "www-data" ]; then
         echo "Security measure: user www-data is not allowed to change webUI password!"
@@ -305,7 +348,7 @@ SetDNSServers() {
     IFS=',' read -r -a array <<< "${args[2]}"
     for index in "${!array[@]}"
     do
-        # Replace possible "\#" by "#". This fixes AdminLTE#1427
+        # Replace possible "\#" by "#". This fixes web#1427
         local ip
         ip="${array[index]//\\#/#}"
 
@@ -517,13 +560,13 @@ CustomizeAdLists() {
 
     if CheckUrl "${address}"; then
         if [[ "${args[2]}" == "enable" ]]; then
-            pihole-FTL sqlite3 "${gravityDBfile}" "UPDATE adlist SET enabled = 1 WHERE address = '${address}'"
+            pihole-FTL sqlite3 -ni "${gravityDBfile}" "UPDATE adlist SET enabled = 1 WHERE address = '${address}'"
         elif [[ "${args[2]}" == "disable" ]]; then
-            pihole-FTL sqlite3 "${gravityDBfile}" "UPDATE adlist SET enabled = 0 WHERE address = '${address}'"
+            pihole-FTL sqlite3 -ni "${gravityDBfile}" "UPDATE adlist SET enabled = 0 WHERE address = '${address}'"
         elif [[ "${args[2]}" == "add" ]]; then
-            pihole-FTL sqlite3 "${gravityDBfile}" "INSERT OR IGNORE INTO adlist (address, comment) VALUES ('${address}', '${comment}')"
+            pihole-FTL sqlite3 -ni "${gravityDBfile}" "INSERT OR IGNORE INTO adlist (address, comment) VALUES ('${address}', '${comment}')"
         elif [[ "${args[2]}" == "del" ]]; then
-            pihole-FTL sqlite3 "${gravityDBfile}" "DELETE FROM adlist WHERE address = '${address}'"
+            pihole-FTL sqlite3 -ni "${gravityDBfile}" "DELETE FROM adlist WHERE address = '${address}'"
         else
             echo "Not permitted"
             return 1
@@ -613,7 +656,6 @@ Teleporter() {
         host="${host//./_}"
         filename="pi-hole-${host:-noname}-teleporter_${datetimestamp}.tar.gz"
     fi
-    # webroot is sourced from basic-install above
     php "${webroot}/admin/scripts/pi-hole/php/teleporter.php" > "${filename}"
 }
 
@@ -622,7 +664,7 @@ checkDomain()
     local domain validDomain
     # Convert to lowercase
     domain="${1,,}"
-    validDomain=$(grep -P "^((-|_)*[a-z\\d]((-|_)*[a-z\\d])*(-|_)*)(\\.(-|_)*([a-z\\d]((-|_)*[a-z\\d])*))*$" <<< "${domain}") # Valid chars check
+    validDomain=$(grep -P "^((-|_)*[a-z0-9]((-|_)*[a-z0-9])*(-|_)*)(\\.(-|_)*([a-z0-9]((-|_)*[a-z0-9])*))*$" <<< "${domain}") # Valid chars check
     validDomain=$(grep -P "^[^\\.]{1,63}(\\.[^\\.]{1,63})*$" <<< "${validDomain}") # Length of each label
     echo "${validDomain}"
 }
@@ -658,12 +700,12 @@ addAudit()
     done
     # Insert only the domain here. The date_added field will be
     # filled with its default value (date_added = current timestamp)
-    pihole-FTL sqlite3 "${gravityDBfile}" "INSERT INTO domain_audit (domain) VALUES ${domains};"
+    pihole-FTL sqlite3 -ni "${gravityDBfile}" "INSERT INTO domain_audit (domain) VALUES ${domains};"
 }
 
 clearAudit()
 {
-    pihole-FTL sqlite3 "${gravityDBfile}" "DELETE FROM domain_audit;"
+    pihole-FTL sqlite3 -ni "${gravityDBfile}" "DELETE FROM domain_audit;"
 }
 
 SetPrivacyLevel() {

advanced/Templates/gravity.db.sql

@@ -3,90 +3,90 @@ BEGIN TRANSACTION;
 
 CREATE TABLE "group"
 (
-	id INTEGER PRIMARY KEY AUTOINCREMENT,
-	enabled BOOLEAN NOT NULL DEFAULT 1,
-	name TEXT UNIQUE NOT NULL,
-	date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
-	date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
-	description TEXT
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    enabled BOOLEAN NOT NULL DEFAULT 1,
+    name TEXT UNIQUE NOT NULL,
+    date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
+    date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
+    description TEXT
 );
 INSERT INTO "group" (id,enabled,name,description) VALUES (0,1,'Default','The default group');
 
 CREATE TABLE domainlist
 (
-	id INTEGER PRIMARY KEY AUTOINCREMENT,
-	type INTEGER NOT NULL DEFAULT 0,
-	domain TEXT NOT NULL,
-	enabled BOOLEAN NOT NULL DEFAULT 1,
-	date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
-	date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
-	comment TEXT,
-	UNIQUE(domain, type)
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    type INTEGER NOT NULL DEFAULT 0,
+    domain TEXT NOT NULL,
+    enabled BOOLEAN NOT NULL DEFAULT 1,
+    date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
+    date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
+    comment TEXT,
+    UNIQUE(domain, type)
 );
 
 CREATE TABLE adlist
 (
-	id INTEGER PRIMARY KEY AUTOINCREMENT,
-	address TEXT UNIQUE NOT NULL,
-	enabled BOOLEAN NOT NULL DEFAULT 1,
-	date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
-	date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
-	comment TEXT,
-	date_updated INTEGER,
-	number INTEGER NOT NULL DEFAULT 0,
-	invalid_domains INTEGER NOT NULL DEFAULT 0,
-	status INTEGER NOT NULL DEFAULT 0
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    address TEXT UNIQUE NOT NULL,
+    enabled BOOLEAN NOT NULL DEFAULT 1,
+    date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
+    date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
+    comment TEXT,
+    date_updated INTEGER,
+    number INTEGER NOT NULL DEFAULT 0,
+    invalid_domains INTEGER NOT NULL DEFAULT 0,
+    status INTEGER NOT NULL DEFAULT 0
 );
 
 CREATE TABLE adlist_by_group
 (
-	adlist_id INTEGER NOT NULL REFERENCES adlist (id),
-	group_id INTEGER NOT NULL REFERENCES "group" (id),
-	PRIMARY KEY (adlist_id, group_id)
+    adlist_id INTEGER NOT NULL REFERENCES adlist (id),
+    group_id INTEGER NOT NULL REFERENCES "group" (id),
+    PRIMARY KEY (adlist_id, group_id)
 );
 
 CREATE TABLE gravity
 (
-	domain TEXT NOT NULL,
-	adlist_id INTEGER NOT NULL REFERENCES adlist (id)
+    domain TEXT NOT NULL,
+    adlist_id INTEGER NOT NULL REFERENCES adlist (id)
 );
 
 CREATE TABLE info
 (
-	property TEXT PRIMARY KEY,
-	value TEXT NOT NULL
+    property TEXT PRIMARY KEY,
+    value TEXT NOT NULL
 );
 
 INSERT INTO "info" VALUES('version','15');
 
 CREATE TABLE domain_audit
 (
-	id INTEGER PRIMARY KEY AUTOINCREMENT,
-	domain TEXT UNIQUE NOT NULL,
-	date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int))
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    domain TEXT UNIQUE NOT NULL,
+    date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int))
 );
 
 CREATE TABLE domainlist_by_group
 (
-	domainlist_id INTEGER NOT NULL REFERENCES domainlist (id),
-	group_id INTEGER NOT NULL REFERENCES "group" (id),
-	PRIMARY KEY (domainlist_id, group_id)
+    domainlist_id INTEGER NOT NULL REFERENCES domainlist (id),
+    group_id INTEGER NOT NULL REFERENCES "group" (id),
+    PRIMARY KEY (domainlist_id, group_id)
 );
 
 CREATE TABLE client
 (
-	id INTEGER PRIMARY KEY AUTOINCREMENT,
-	ip TEXT NOT NULL UNIQUE,
-	date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
-	date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
-	comment TEXT
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    ip TEXT NOT NULL UNIQUE,
+    date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
+    date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
+    comment TEXT
 );
 
 CREATE TABLE client_by_group
 (
-	client_id INTEGER NOT NULL REFERENCES client (id),
-	group_id INTEGER NOT NULL REFERENCES "group" (id),
-	PRIMARY KEY (client_id, group_id)
+    client_id INTEGER NOT NULL REFERENCES client (id),
+    group_id INTEGER NOT NULL REFERENCES "group" (id),
+    PRIMARY KEY (client_id, group_id)
 );
 
 CREATE TRIGGER tr_adlist_update AFTER UPDATE OF address,enabled,comment ON adlist

advanced/Templates/gravity_copy.sql

@@ -19,8 +19,6 @@ INSERT OR REPLACE INTO adlist SELECT * FROM OLD.adlist;
 DELETE FROM OLD.adlist_by_group WHERE adlist_id NOT IN (SELECT id FROM OLD.adlist);
 INSERT OR REPLACE INTO adlist_by_group SELECT * FROM OLD.adlist_by_group;
 
-INSERT OR REPLACE INTO info SELECT * FROM OLD.info;
-
 INSERT OR REPLACE INTO client SELECT * FROM OLD.client;
 DELETE FROM OLD.client_by_group WHERE client_id NOT IN (SELECT id FROM OLD.client);
 INSERT OR REPLACE INTO client_by_group SELECT * FROM OLD.client_by_group;

advanced/Templates/logrotate

@@ -1,21 +1,21 @@
 /var/log/pihole/pihole.log {
-	# su #
-	daily
-	copytruncate
-	rotate 5
-	compress
-	delaycompress
-	notifempty
-	nomail
+    # su #
+    daily
+    copytruncate
+    rotate 5
+    compress
+    delaycompress
+    notifempty
+    nomail
 }
 
 /var/log/pihole/FTL.log {
-	# su #
-	weekly
-	copytruncate
-	rotate 3
-	compress
-	delaycompress
-	notifempty
-	nomail
+    # su #
+    weekly
+    copytruncate
+    rotate 3
+    compress
+    delaycompress
+    notifempty
+    nomail
 }

advanced/Templates/pihole-FTL.service

@@ -23,6 +23,11 @@ is_running() {
   return 1
 }
 
+cleanup() {
+    # Run post-stop script, which does cleanup among runtime files
+    sh "${PI_HOLE_SCRIPT_DIR}/pihole-FTL-poststop.sh"
+}
+
 
 # Start the service
 start() {
@@ -33,10 +38,16 @@ start() {
     sh "${PI_HOLE_SCRIPT_DIR}/pihole-FTL-prestart.sh"
 
     if setcap CAP_NET_BIND_SERVICE,CAP_NET_RAW,CAP_NET_ADMIN,CAP_SYS_NICE,CAP_IPC_LOCK,CAP_CHOWN+eip "/usr/bin/pihole-FTL"; then
-      su -s /bin/sh -c "/usr/bin/pihole-FTL" pihole || exit $?
+      su -s /bin/sh -c "/usr/bin/pihole-FTL" pihole
     else
       echo "Warning: Starting pihole-FTL as root because setting capabilities is not supported on this system"
-      /usr/bin/pihole-FTL || exit $?
+      /usr/bin/pihole-FTL
+    fi
+    rc=$?
+    # Cleanup if startup failed
+    if [ "${rc}" != 0 ]; then
+        cleanup
+        exit $rc
     fi
     echo
   fi
@@ -65,8 +76,7 @@ stop() {
   else
     echo "Not running"
   fi
-  # Run post-stop script, which does cleanup among runtime files
-  sh "${PI_HOLE_SCRIPT_DIR}/pihole-FTL-poststop.sh"
+  cleanup
   echo
 }
 
@@ -84,6 +94,9 @@ status() {
 
 ### main logic ###
 
+# catch sudden termination
+trap 'cleanup; exit 1' INT HUP TERM ABRT
+
 # Get FTL's PID file path
 FTL_PID_FILE="$(getFTLPIDFile)"
 

advanced/bash-completion/pihole

@@ -1,79 +1,79 @@
 _pihole() {
-	local cur prev opts opts_admin opts_checkout opts_chronometer opts_debug opts_interface  opts_logging opts_privacy opts_query opts_update opts_version
-	COMPREPLY=()
-	cur="${COMP_WORDS[COMP_CWORD]}"
-	prev="${COMP_WORDS[COMP_CWORD-1]}"
-	prev2="${COMP_WORDS[COMP_CWORD-2]}"
+    local cur prev opts opts_admin opts_checkout opts_chronometer opts_debug opts_interface  opts_logging opts_privacy opts_query opts_update opts_version
+    COMPREPLY=()
+    cur="${COMP_WORDS[COMP_CWORD]}"
+    prev="${COMP_WORDS[COMP_CWORD-1]}"
+    prev2="${COMP_WORDS[COMP_CWORD-2]}"
 
-	case "${prev}" in
-		"pihole")
-			opts="admin blacklist checkout chronometer debug disable enable flush help logging query reconfigure regex restartdns status tail uninstall updateGravity updatePihole version wildcard whitelist arpflush"
-			COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
-		;;
-		"whitelist"|"blacklist"|"wildcard"|"regex")
-			opts_lists="\--delmode \--noreload \--quiet \--list \--nuke"
-			COMPREPLY=( $(compgen -W "${opts_lists}" -- ${cur}) )
-		;;
-		"admin")
-			opts_admin="celsius fahrenheit interface kelvin password privacylevel"
-			COMPREPLY=( $(compgen -W "${opts_admin}" -- ${cur}) )
-		;;
-		"checkout")
-			opts_checkout="core ftl web master dev"
-			COMPREPLY=( $(compgen -W "${opts_checkout}" -- ${cur}) )
-		;;
-		"chronometer")
-			opts_chronometer="\--exit \--json \--refresh"
-			COMPREPLY=( $(compgen -W "${opts_chronometer}" -- ${cur}) )
-		;;
-		"debug")
-			opts_debug="-a"
-			COMPREPLY=( $(compgen -W "${opts_debug}" -- ${cur}) )
-		;;
-		"logging")
-			opts_logging="on off 'off noflush'"
-			COMPREPLY=( $(compgen -W "${opts_logging}" -- ${cur}) )
-		;;
-		"query")
-			opts_query="-adlist -all -exact"
-			COMPREPLY=( $(compgen -W "${opts_query}" -- ${cur}) )
-		;;
-		"updatePihole"|"-up")
-			opts_update="--check-only"
-			COMPREPLY=( $(compgen -W "${opts_update}" -- ${cur}) )
-		;;
-		"version")
-			opts_version="\--admin \--current \--ftl \--hash \--latest \--pihole"
-			COMPREPLY=( $(compgen -W "${opts_version}" -- ${cur}) )
-		;;
-		"interface")
-			if ( [[ "$prev2" == "admin" ]] || [[ "$prev2" == "-a" ]] ); then
-				opts_interface="$(cat /proc/net/dev | cut -d: -s -f1)"
-				COMPREPLY=( $(compgen -W "${opts_interface}" -- ${cur}) )
-			else
-				return 1
-			fi
-		;;
-		"privacylevel")
-			if ( [[ "$prev2" == "admin" ]] || [[ "$prev2" == "-a" ]] ); then
-				opts_privacy="0 1 2 3"
-				COMPREPLY=( $(compgen -W "${opts_privacy}" -- ${cur}) )
-			else
-				return 1
-			fi
-		;;
-		"core"|"admin"|"ftl")
-			if [[ "$prev2" == "checkout" ]]; then
-				opts_checkout="master dev"
-				COMPREPLY=( $(compgen -W "${opts_checkout}" -- ${cur}) )
-			else
-				return 1
-			fi
-		;;
-		*)
-		return 1
-		;;
-	esac
-	return 0
+    case "${prev}" in
+        "pihole")
+            opts="admin blacklist checkout chronometer debug disable enable flush help logging query reconfigure regex restartdns status tail uninstall updateGravity updatePihole version wildcard whitelist arpflush"
+            COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
+        ;;
+        "whitelist"|"blacklist"|"wildcard"|"regex")
+            opts_lists="\--delmode \--noreload \--quiet \--list \--nuke"
+            COMPREPLY=( $(compgen -W "${opts_lists}" -- ${cur}) )
+        ;;
+        "admin")
+            opts_admin="celsius fahrenheit interface kelvin password privacylevel"
+            COMPREPLY=( $(compgen -W "${opts_admin}" -- ${cur}) )
+        ;;
+        "checkout")
+            opts_checkout="core ftl web master dev"
+            COMPREPLY=( $(compgen -W "${opts_checkout}" -- ${cur}) )
+        ;;
+        "chronometer")
+            opts_chronometer="\--exit \--json \--refresh"
+            COMPREPLY=( $(compgen -W "${opts_chronometer}" -- ${cur}) )
+        ;;
+        "debug")
+            opts_debug="-a"
+            COMPREPLY=( $(compgen -W "${opts_debug}" -- ${cur}) )
+        ;;
+        "logging")
+            opts_logging="on off 'off noflush'"
+            COMPREPLY=( $(compgen -W "${opts_logging}" -- ${cur}) )
+        ;;
+        "query")
+            opts_query="-adlist -all -exact"
+            COMPREPLY=( $(compgen -W "${opts_query}" -- ${cur}) )
+        ;;
+        "updatePihole"|"-up")
+            opts_update="--check-only"
+            COMPREPLY=( $(compgen -W "${opts_update}" -- ${cur}) )
+        ;;
+        "version")
+            opts_version="\--admin \--current \--ftl \--hash \--latest \--pihole"
+            COMPREPLY=( $(compgen -W "${opts_version}" -- ${cur}) )
+        ;;
+        "interface")
+            if ( [[ "$prev2" == "admin" ]] || [[ "$prev2" == "-a" ]] ); then
+                opts_interface="$(cat /proc/net/dev | cut -d: -s -f1)"
+                COMPREPLY=( $(compgen -W "${opts_interface}" -- ${cur}) )
+            else
+                return 1
+            fi
+        ;;
+        "privacylevel")
+            if ( [[ "$prev2" == "admin" ]] || [[ "$prev2" == "-a" ]] ); then
+                opts_privacy="0 1 2 3"
+                COMPREPLY=( $(compgen -W "${opts_privacy}" -- ${cur}) )
+            else
+                return 1
+            fi
+        ;;
+        "core"|"admin"|"ftl")
+            if [[ "$prev2" == "checkout" ]]; then
+                opts_checkout="master dev"
+                COMPREPLY=( $(compgen -W "${opts_checkout}" -- ${cur}) )
+            else
+                return 1
+            fi
+        ;;
+        *)
+        return 1
+        ;;
+    esac
+    return 0
 }
 complete -F _pihole pihole

advanced/lighttpd.conf.debian

@@ -7,17 +7,18 @@
 # This file is copyright under the latest version of the EUPL.
 # Please see LICENSE file for your rights under this license.
 
-###############################################################################
-#     FILE AUTOMATICALLY OVERWRITTEN BY PI-HOLE INSTALL/UPDATE PROCEDURE.     #
-# ANY CHANGES MADE TO THIS FILE AFTER INSTALL WILL BE LOST ON THE NEXT UPDATE #
-#                                                                             #
-#              CHANGES SHOULD BE MADE IN A SEPARATE CONFIG FILE:              #
-#                         /etc/lighttpd/external.conf                         #
-###############################################################################
+###################################################################################################
+#   IF THIS HEADER EXISTS, THE FILE WILL BE OVERWRITTEN BY PI-HOLE'S UPDATE PROCEDURE.            #
+#   ANY CHANGES MADE TO THIS FILE WILL BE LOST ON THE NEXT UPDATE UNLESS YOU REMOVE THIS HEADER   #
+#                                                                                                 #
+#   ENSURE THAT YOU DO NOT REMOVE THE REQUIRED LINE:                                              #
+#                                                                                                 #
+#   include "/etc/lighttpd/conf-enabled/*.conf"                                                   #
+#                                                                                                 #
+###################################################################################################
 
 server.modules = (
     "mod_access",
-    "mod_accesslog",
     "mod_auth",
     "mod_expire",
     "mod_redirect",
@@ -34,8 +35,6 @@ server.groupname            = "www-data"
 # For lighttpd version 1.4.46 or above, the port can be overwritten in `/etc/lighttpd/external.conf` using the := operator
 # e.g. server.port := 8000
 server.port                 = 80
-accesslog.filename          = "/var/log/lighttpd/access-pihole.log"
-accesslog.format            = "%{%s}t|%V|%r|%s|%b"
 
 # Allow streaming response
 # reference: https://redmine.lighttpd.net/projects/lighttpd/wiki/Server_stream-response-bodyDetails

advanced/lighttpd.conf.fedora

@@ -7,13 +7,15 @@
 # This file is copyright under the latest version of the EUPL.
 # Please see LICENSE file for your rights under this license.
 
-###############################################################################
-#     FILE AUTOMATICALLY OVERWRITTEN BY PI-HOLE INSTALL/UPDATE PROCEDURE.     #
-# ANY CHANGES MADE TO THIS FILE AFTER INSTALL WILL BE LOST ON THE NEXT UPDATE #
-#                                                                             #
-#              CHANGES SHOULD BE MADE IN A SEPARATE CONFIG FILE:              #
-#                         /etc/lighttpd/external.conf                         #
-###############################################################################
+###################################################################################################
+#   IF THIS HEADER EXISTS, THE FILE WILL BE OVERWRITTEN BY PI-HOLE'S UPDATE PROCEDURE.            #
+#   ANY CHANGES MADE TO THIS FILE WILL BE LOST ON THE NEXT UPDATE UNLESS YOU REMOVE THIS HEADER   #
+#                                                                                                 #
+#   ENSURE THAT YOU DO NOT REMOVE THE REQUIRED LINE:                                              #
+#                                                                                                 #
+#   include "/etc/lighttpd/conf.d/pihole-admin.conf"                                              #
+#                                                                                                 #
+###################################################################################################
 
 server.modules = (
     "mod_access",
@@ -35,8 +37,6 @@ server.groupname            = "lighttpd"
 # For lighttpd version 1.4.46 or above, the port can be overwritten in `/etc/lighttpd/external.conf` using the := operator
 # e.g. server.port := 8000
 server.port                 = 80
-accesslog.filename          = "/var/log/lighttpd/access-pihole.log"
-accesslog.format            = "%{%s}t|%V|%r|%s|%b"
 
 # Allow streaming response
 # reference: https://redmine.lighttpd.net/projects/lighttpd/wiki/Server_stream-response-bodyDetails

advanced/pihole-admin.conf

@@ -23,9 +23,9 @@ $HTTP["url"] =~ "^/admin/" {
     fastcgi.server = (
         ".php" => (
             "localhost" => (
-                "socket" => "/tmp/pihole-php-fastcgi.socket",
+                "socket" => "/run/lighttpd/pihole-php-fastcgi.socket",
                 "bin-path" => "/usr/bin/php-cgi",
-                "min-procs" => 0,
+                "min-procs" => 1,
                 "max-procs" => 1,
                 "bin-environment" => (
                     "PHP_FCGI_CHILDREN" => "4",
@@ -79,4 +79,4 @@ $HTTP["host"] == "pi.hole" {
 }
 
 # (keep this on one line for basic-install.sh filtering during install)
-server.modules += ( "mod_access", "mod_redirect", "mod_fastcgi", "mod_setenv" )
+server.modules += ( "mod_access", "mod_accesslog", "mod_redirect", "mod_fastcgi", "mod_setenv" )

automated install/basic-install.sh

@@ -68,11 +68,11 @@ webroot="/var/www/html"
 
 
 # We clone (or update) two git repositories during the install. This helps to make sure that we always have the latest versions of the relevant files.
-# AdminLTE is used to set up the Web admin interface.
+# web is used to set up the Web admin interface.
 # Pi-hole contains various setup scripts and files which are critical to the installation.
 # Search for "PI_HOLE_LOCAL_REPO" in this file to see all such scripts.
 # Two notable scripts are gravity.sh (used to generate the HOSTS file) and advanced/Scripts/webpage.sh (used to install the Web admin interface)
-webInterfaceGitUrl="https://github.com/pi-hole/AdminLTE.git"
+webInterfaceGitUrl="https://github.com/pi-hole/web.git"
 webInterfaceDir="${webroot}/admin"
 piholeGitUrl="https://github.com/pi-hole/pi-hole.git"
 PI_HOLE_LOCAL_REPO="/etc/.pihole"
@@ -334,7 +334,7 @@ package_manager_detect() {
         # Packages required for the Web admin interface (stored as an array)
         # It's useful to separate this from Pi-hole, since the two repos are also setup separately
         PIHOLE_WEB_DEPS=(lighttpd "${phpVer}-common" "${phpVer}-cgi" "${phpVer}-sqlite3" "${phpVer}-xml" "${phpVer}-intl")
-        # Prior to PHP8.0, JSON functionality is provided as dedicated module, required by Pi-hole AdminLTE: https://www.php.net/manual/json.installation.php
+        # Prior to PHP8.0, JSON functionality is provided as dedicated module, required by Pi-hole web: https://www.php.net/manual/json.installation.php
         if [[ -z "${phpInsMajor}" || "${phpInsMajor}" -lt 8 ]]; then
             PIHOLE_WEB_DEPS+=("${phpVer}-json")
         fi
@@ -357,7 +357,7 @@ package_manager_detect() {
         # These variable names match the ones for apt-get. See above for an explanation of what they are for.
         PKG_INSTALL=("${PKG_MANAGER}" install -y)
         # CentOS package manager returns 100 when there are packages to update so we need to || true to prevent the script from exiting.
-        PKG_COUNT="${PKG_MANAGER} check-update | grep -E '(.i686|.x86|.noarch|.arm|.src)' | wc -l || true"
+        PKG_COUNT="${PKG_MANAGER} check-update | grep -E '(.i686|.x86|.noarch|.arm|.src|.riscv64)' | wc -l || true"
         OS_CHECK_DEPS=(grep bind-utils)
         INSTALLER_DEPS=(git dialog iproute newt procps-ng chkconfig ca-certificates)
         PIHOLE_DEPS=(cronie curl findutils sudo unzip libidn2 psmisc libcap nmap-ncat jq)
@@ -1400,12 +1400,19 @@ installConfigs() {
 
     # If the user chose to install the dashboard,
     if [[ "${INSTALL_WEB_SERVER}" == true ]]; then
-        if grep -q -F "FILE AUTOMATICALLY OVERWRITTEN BY PI-HOLE" "${lighttpdConfig}"; then
+        # set permissions on /etc/lighttpd/lighttpd.conf so pihole user (other) can read the file
+        chmod o+x /etc/lighttpd
+        chmod o+r "${lighttpdConfig}"
+
+        # Ensure /run/lighttpd exists and is owned by lighttpd user
+        # Needed for the php socket
+        mkdir -p /run/lighttpd
+        chown ${LIGHTTPD_USER}:${LIGHTTPD_GROUP} /run/lighttpd
+
+        if grep -q -F "OVERWRITTEN BY PI-HOLE" "${lighttpdConfig}"; then
             # Attempt to preserve backwards compatibility with older versions
             install -D -m 644 -T ${PI_HOLE_LOCAL_REPO}/advanced/${LIGHTTPD_CFG} "${lighttpdConfig}"
             # Make the directories if they do not exist and set the owners
-            mkdir -p /run/lighttpd
-            chown ${LIGHTTPD_USER}:${LIGHTTPD_GROUP} /run/lighttpd
             mkdir -p /var/cache/lighttpd/compress
             chown ${LIGHTTPD_USER}:${LIGHTTPD_GROUP} /var/cache/lighttpd/compress
             mkdir -p /var/cache/lighttpd/uploads
@@ -1432,17 +1439,26 @@ installConfigs() {
         elif [[ -d "/etc/lighttpd/conf-available" ]]; then
             conf=/etc/lighttpd/conf-available/15-pihole-admin.conf
             install -D -m 644 -T ${PI_HOLE_LOCAL_REPO}/advanced/pihole-admin.conf $conf
-            # disable server.modules += ( ... ) in $conf to avoid module dups
-            # (needed until Debian 10 no longer supported by pi-hole)
-            # (server.modules duplication is ignored in lighttpd 1.4.56+)
-            if awk '!/^server\.modules/{print}' $conf > $conf.$$ && mv $conf.$$ $conf; then
+
+            # Get the version number of lighttpd
+            version=$(dpkg-query -f='${Version}\n' --show lighttpd)
+            # Test if that version is greater than or equal to 1.4.56
+            if dpkg --compare-versions "$version" "ge" "1.4.56"; then
+                # If it is, then we don't need to disable the modules
+                # (server.modules duplication is ignored in lighttpd 1.4.56+)
                 :
             else
-                rm $conf.$$
+                # disable server.modules += ( ... ) in $conf to avoid module dups
+                if awk '!/^server\.modules/{print}' $conf > $conf.$$ && mv $conf.$$ $conf; then
+                :
+                else
+                    rm $conf.$$
+                fi
             fi
+
             chmod 644 $conf
             if is_command lighty-enable-mod ; then
-                lighty-enable-mod pihole-admin access redirect fastcgi setenv > /dev/null || true
+                lighty-enable-mod pihole-admin access accesslog redirect fastcgi setenv > /dev/null || true
             else
                 # Otherwise, show info about installing them
                 printf "  %b Warning: 'lighty-enable-mod' utility not found\\n" "${INFO}"
@@ -1682,7 +1698,7 @@ install_dependent_packages() {
     for i in "$@"; do
         # For each package, check if it's already installed (and if so, don't add it to the installArray)
         printf "  %b Checking for %s..." "${INFO}" "${i}"
-        if "${PKG_MANAGER}" -q list installed "${i}" &> /dev/null; then
+        if rpm -q "${i}" &> /dev/null; then
             printf "%b  %b Checking for %s\\n" "${OVER}" "${TICK}" "${i}"
         else
             printf "%b  %b Checking for %s (will be installed)\\n" "${OVER}" "${INFO}" "${i}"
@@ -2350,6 +2366,9 @@ get_binary_name() {
             # set the binary to be used
             l_binary="pihole-FTL-linux-x86_64"
         fi
+    elif [[ "${machine}" == "riscv64" ]]; then
+        printf "%b  %b Detected riscv64 processor\\n" "${OVER}" "${TICK}"
+        l_binary="pihole-FTL-riscv64-linux-gnu"
     else
         # Something else - we try to use 32bit executable and warn the user
         if [[ ! "${machine}" == "i686" ]]; then
@@ -2596,7 +2615,8 @@ main() {
 
         # Get the privacy level if it exists (default is 0)
         if [[ -f "${FTL_CONFIG_FILE}" ]]; then
-            PRIVACY_LEVEL=$(sed -ne 's/PRIVACYLEVEL=\(.*\)/\1/p' "${FTL_CONFIG_FILE}")
+            # get the value from $FTL_CONFIG_FILE (and ignoring all commented lines)
+            PRIVACY_LEVEL=$(sed -e '/^[[:blank:]]*#/d' "${FTL_CONFIG_FILE}" | grep "PRIVACYLEVEL" | awk -F "=" 'NR==1{printf$2}')
 
             # If no setting was found, default to 0
             PRIVACY_LEVEL="${PRIVACY_LEVEL:-0}"
@@ -2665,7 +2685,7 @@ main() {
     # Check for and disable systemd-resolved-DNSStubListener before reloading resolved
     # DNSStubListener needs to remain in place for installer to download needed files,
     # so this change needs to be made after installation is complete,
-    # but before starting or resarting the dnsmasq or ftl services
+    # but before starting or restarting the dnsmasq or ftl services
     disable_resolved_stublistener
 
     # If the Web server was installed,
@@ -2717,12 +2737,12 @@ main() {
 
     restart_service pihole-FTL
 
-    # Update local and remote versions via updatechecker
-    /opt/pihole/updatecheck.sh
-
     # Download and compile the aggregated block list
     runGravity
 
+    # Update local and remote versions via updatechecker
+    /opt/pihole/updatecheck.sh
+
     if [[ "${useUpdateVars}" == false ]]; then
         displayFinalMessage "${pw}"
     fi

automated install/uninstall.sh

@@ -193,6 +193,18 @@ removeNoPurge() {
         else
             service pihole-FTL stop
         fi
+        ${SUDO} rm -f /etc/systemd/system/pihole-FTL.service
+        if [[ -d '/etc/systemd/system/pihole-FTL.service.d' ]]; then
+            read -rp "  ${QST} FTL service override directory /etc/systemd/system/pihole-FTL.service.d detected. Do you wish to remove this from your system? [y/N] " answer
+            case $answer in
+                [yY]*)
+                    echo -ne "  ${INFO} Removing /etc/systemd/system/pihole-FTL.service.d..."
+                    ${SUDO} rm -R /etc/systemd/system/pihole-FTL.service.d
+                    echo -e "${OVER}  ${INFO} Removed /etc/systemd/system/pihole-FTL.service.d"
+                ;;
+                *) echo -e "  ${INFO} Leaving /etc/systemd/system/pihole-FTL.service.d in place.";;
+            esac
+        fi
         ${SUDO} rm -f /etc/init.d/pihole-FTL
         ${SUDO} rm -f /usr/bin/pihole-FTL
         echo -e "${OVER}  ${TICK} Removed pihole-FTL"

gravity.sh

@@ -52,6 +52,14 @@ else
   exit 1
 fi
 
+# Set up tmp dir variable in case it's not configured
+: "${GRAVITY_TMPDIR:=/tmp}"
+
+if [ ! -d "${GRAVITY_TMPDIR}" ] || [ ! -w "${GRAVITY_TMPDIR}" ]; then
+  echo -e "  ${COL_LIGHT_RED}Gravity temporary directory does not exist or is not a writeable directory, falling back to /tmp. ${COL_NC}"
+  GRAVITY_TMPDIR="/tmp"
+fi
+
 # Source pihole-FTL from install script
 pihole_FTL="${piholeDir}/pihole-FTL.conf"
 if [[ -f "${pihole_FTL}" ]]; then
@@ -76,7 +84,7 @@ fi
 
 # Generate new SQLite3 file from schema template
 generate_gravity_database() {
-  if ! pihole-FTL sqlite3 "${gravityDBfile}" < "${gravityDBschema}"; then
+  if ! pihole-FTL sqlite3 -ni "${gravityDBfile}" < "${gravityDBschema}"; then
     echo -e "   ${CROSS} Unable to create ${gravityDBfile}"
     return 1
   fi
@@ -91,7 +99,7 @@ gravity_swap_databases() {
   echo -ne "  ${INFO} ${str}..."
 
   # The index is intentionally not UNIQUE as poor quality adlists may contain domains more than once
-  output=$( { pihole-FTL sqlite3 "${gravityTEMPfile}" "CREATE INDEX idx_gravity ON gravity (domain, adlist_id);"; } 2>&1 )
+  output=$( { pihole-FTL sqlite3 -ni "${gravityTEMPfile}" "CREATE INDEX idx_gravity ON gravity (domain, adlist_id);"; } 2>&1 )
   status="$?"
 
   if [[ "${status}" -ne 0 ]]; then
@@ -121,13 +129,13 @@ gravity_swap_databases() {
   echo -e "${OVER}  ${TICK} ${str}"
 
   if $oldAvail; then
-    echo -e "  ${TICK} The old database remains available."
+    echo -e "  ${TICK} The old database remains available"
   fi
 }
 
 # Update timestamp when the gravity table was last updated successfully
 update_gravity_timestamp() {
-  output=$( { printf ".timeout 30000\\nINSERT OR REPLACE INTO info (property,value) values ('updated',cast(strftime('%%s', 'now') as int));" | pihole-FTL sqlite3 "${gravityDBfile}"; } 2>&1 )
+  output=$( { printf ".timeout 30000\\nINSERT OR REPLACE INTO info (property,value) values ('updated',cast(strftime('%%s', 'now') as int));" | pihole-FTL sqlite3 -ni "${gravityDBfile}"; } 2>&1 )
   status="$?"
 
   if [[ "${status}" -ne 0 ]]; then
@@ -145,7 +153,10 @@ database_table_from_file() {
   src="${2}"
   backup_path="${piholeDir}/migration_backup"
   backup_file="${backup_path}/$(basename "${2}")"
-  tmpFile="$(mktemp -p "/tmp" --suffix=".gravity")"
+  # Create a temporary file. We don't use '--suffix' here because not all
+  # implementations of mktemp support it, e.g. on Alpine
+  tmpFile="$(mktemp -p "${GRAVITY_TMPDIR}")"
+  mv "${tmpFile}" "${tmpFile%.*}.gravity"
 
   local timestamp
   timestamp="$(date --utc +'%s')"
@@ -168,7 +179,7 @@ database_table_from_file() {
 
   # Get MAX(id) from domainlist when INSERTing into this table
   if [[ "${table}" == "domainlist" ]]; then
-    rowid="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT MAX(id) FROM domainlist;")"
+    rowid="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT MAX(id) FROM domainlist;")"
     if [[ -z "$rowid" ]]; then
       rowid=0
     fi
@@ -198,7 +209,7 @@ database_table_from_file() {
   # Store domains in database table specified by ${table}
   # Use printf as .mode and .import need to be on separate lines
   # see https://unix.stackexchange.com/a/445615/83260
-  output=$( { printf ".timeout 30000\\n.mode csv\\n.import \"%s\" %s\\n" "${tmpFile}" "${table}" | pihole-FTL sqlite3 "${gravityDBfile}"; } 2>&1 )
+  output=$( { printf ".timeout 30000\\n.mode csv\\n.import \"%s\" %s\\n" "${tmpFile}" "${table}" | pihole-FTL sqlite3 -ni "${gravityDBfile}"; } 2>&1 )
   status="$?"
 
   if [[ "${status}" -ne 0 ]]; then
@@ -216,20 +227,9 @@ database_table_from_file() {
     echo -e "  ${CROSS} Unable to remove ${tmpFile}"
 }
 
-# Update timestamp of last update of this list. We store this in the "old" database as all values in the new database will later be overwritten
-database_adlist_updated() {
-  output=$( { printf ".timeout 30000\\nUPDATE adlist SET date_updated = (cast(strftime('%%s', 'now') as int)) WHERE id = %i;\\n" "${1}" | pihole-FTL sqlite3 "${gravityDBfile}"; } 2>&1 )
-  status="$?"
-
-  if [[ "${status}" -ne 0 ]]; then
-    echo -e "\\n  ${CROSS} Unable to update timestamp of adlist with ID ${1} in database ${gravityDBfile}\\n  ${output}"
-    gravity_Cleanup "error"
-  fi
-}
-
 # Check if a column with name ${2} exists in gravity table with name ${1}
 gravity_column_exists() {
-  output=$( { printf ".timeout 30000\\nSELECT EXISTS(SELECT * FROM pragma_table_info('%s') WHERE name='%s');\\n" "${1}" "${2}" | pihole-FTL sqlite3 "${gravityDBfile}"; } 2>&1 )
+  output=$( { printf ".timeout 30000\\nSELECT EXISTS(SELECT * FROM pragma_table_info('%s') WHERE name='%s');\\n" "${1}" "${2}" | pihole-FTL sqlite3 -ni "${gravityTEMPfile}"; } 2>&1 )
   if [[ "${output}" == "1" ]]; then
     return 0 # Bash 0 is success
   fi
@@ -244,11 +244,11 @@ database_adlist_number() {
     return;
   fi
 
-  output=$( { printf ".timeout 30000\\nUPDATE adlist SET number = %i, invalid_domains = %i WHERE id = %i;\\n" "${num_source_lines}" "${num_invalid}" "${1}" | pihole-FTL sqlite3 "${gravityDBfile}"; } 2>&1 )
+  output=$( { printf ".timeout 30000\\nUPDATE adlist SET number = %i, invalid_domains = %i WHERE id = %i;\\n" "${2}" "${3}" "${1}" | pihole-FTL sqlite3 -ni "${gravityTEMPfile}"; } 2>&1 )
   status="$?"
 
   if [[ "${status}" -ne 0 ]]; then
-    echo -e "\\n  ${CROSS} Unable to update number of domains in adlist with ID ${1} in database ${gravityDBfile}\\n  ${output}"
+    echo -e "\\n  ${CROSS} Unable to update number of domains in adlist with ID ${1} in database ${gravityTEMPfile}\\n  ${output}"
     gravity_Cleanup "error"
   fi
 }
@@ -260,11 +260,11 @@ database_adlist_status() {
     return;
   fi
 
-  output=$( { printf ".timeout 30000\\nUPDATE adlist SET status = %i WHERE id = %i;\\n" "${2}" "${1}" | pihole-FTL sqlite3 "${gravityDBfile}"; } 2>&1 )
+  output=$( { printf ".timeout 30000\\nUPDATE adlist SET status = %i WHERE id = %i;\\n" "${2}" "${1}" | pihole-FTL sqlite3 -ni "${gravityTEMPfile}"; } 2>&1 )
   status="$?"
 
   if [[ "${status}" -ne 0 ]]; then
-    echo -e "\\n  ${CROSS} Unable to update status of adlist with ID ${1} in database ${gravityDBfile}\\n  ${output}"
+    echo -e "\\n  ${CROSS} Unable to update status of adlist with ID ${1} in database ${gravityTEMPfile}\\n  ${output}"
     gravity_Cleanup "error"
   fi
 }
@@ -378,8 +378,8 @@ gravity_DownloadBlocklists() {
 
   # Retrieve source URLs from gravity database
   # We source only enabled adlists, SQLite3 stores boolean values as 0 (false) or 1 (true)
-  mapfile -t sources <<< "$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2> /dev/null)"
-  mapfile -t sourceIDs <<< "$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT id FROM vw_adlist;" 2> /dev/null)"
+  mapfile -t sources <<< "$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2> /dev/null)"
+  mapfile -t sourceIDs <<< "$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT id FROM vw_adlist;" 2> /dev/null)"
 
   # Parse source domains from $sources
   mapfile -t sourceDomains <<< "$(
@@ -401,14 +401,14 @@ gravity_DownloadBlocklists() {
     unset sources
   fi
 
-  local url domain agent cmd_ext str target compression
+  local url domain str target compression
   echo ""
 
   # Prepare new gravity database
   str="Preparing new gravity database"
   echo -ne "  ${INFO} ${str}..."
   rm "${gravityTEMPfile}" > /dev/null 2>&1
-  output=$( { pihole-FTL sqlite3 "${gravityTEMPfile}" < "${gravityDBschema}"; } 2>&1 )
+  output=$( { pihole-FTL sqlite3 -ni "${gravityTEMPfile}" < "${gravityDBschema}"; } 2>&1 )
   status="$?"
 
   if [[ "${status}" -ne 0 ]]; then
@@ -418,7 +418,24 @@ gravity_DownloadBlocklists() {
     echo -e "${OVER}  ${TICK} ${str}"
   fi
 
-  target="$(mktemp -p "/tmp" --suffix=".gravity")"
+  str="Creating new gravity databases"
+  echo -ne "  ${INFO} ${str}..."
+
+  # Gravity copying SQL script
+  copyGravity="$(cat "${gravityDBcopy}")"
+  if [[ "${gravityDBfile}" != "${gravityDBfile_default}" ]]; then
+    # Replace default gravity script location by custom location
+    copyGravity="${copyGravity//"${gravityDBfile_default}"/"${gravityDBfile}"}"
+  fi
+
+  output=$( { pihole-FTL sqlite3 -ni "${gravityTEMPfile}" <<< "${copyGravity}"; } 2>&1 )
+  status="$?"
+
+  if [[ "${status}" -ne 0 ]]; then
+    echo -e "\\n  ${CROSS} Unable to copy data from ${gravityDBfile} to ${gravityTEMPfile}\\n  ${output}"
+    return 1
+  fi
+  echo -e "${OVER}  ${TICK} ${str}"
 
   # Use compression to reduce the amount of data that is transferred
   # between the Pi-hole and the ad list provider. Use this feature
@@ -440,15 +457,6 @@ gravity_DownloadBlocklists() {
     saveLocation="${piholeDir}/list.${id}.${domain}.${domainsExtension}"
     activeDomains[$i]="${saveLocation}"
 
-    # Default user-agent (for Cloudflare's Browser Integrity Check: https://support.cloudflare.com/hc/en-us/articles/200170086-What-does-the-Browser-Integrity-Check-do-)
-    agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36"
-
-    # Provide special commands for blocklists which may need them
-    case "${domain}" in
-      "pgl.yoyo.org") cmd_ext="-d mimetype=plaintext -d hostformat=hosts";;
-      *) cmd_ext="";;
-    esac
-
     echo -e "  ${INFO} Target: ${url}"
     local regex check_url
     # Check for characters NOT allowed in URLs
@@ -461,109 +469,14 @@ gravity_DownloadBlocklists() {
     if [[ "${check_url}" =~ ${regex} ]]; then
       echo -e "  ${CROSS} Invalid Target"
     else
-      gravity_DownloadBlocklistFromUrl "${url}" "${cmd_ext}" "${agent}" "${sourceIDs[$i]}" "${saveLocation}" "${target}" "${compression}"
+      gravity_DownloadBlocklistFromUrl "${url}" "${sourceIDs[$i]}" "${saveLocation}" "${target}" "${compression}"
     fi
     echo ""
   done
 
-  str="Creating new gravity databases"
-  echo -ne "  ${INFO} ${str}..."
-
-  # Gravity copying SQL script
-  copyGravity="$(cat "${gravityDBcopy}")"
-  if [[ "${gravityDBfile}" != "${gravityDBfile_default}" ]]; then
-    # Replace default gravity script location by custom location
-    copyGravity="${copyGravity//"${gravityDBfile_default}"/"${gravityDBfile}"}"
-  fi
-
-  output=$( { pihole-FTL sqlite3 "${gravityTEMPfile}" <<< "${copyGravity}"; } 2>&1 )
-  status="$?"
-
-  if [[ "${status}" -ne 0 ]]; then
-    echo -e "\\n  ${CROSS} Unable to copy data from ${gravityDBfile} to ${gravityTEMPfile}\\n  ${output}"
-    return 1
-  fi
-  echo -e "${OVER}  ${TICK} ${str}"
-
-  str="Storing downloaded domains in new gravity database"
-  echo -ne "  ${INFO} ${str}..."
-  output=$( { printf ".timeout 30000\\n.mode csv\\n.import \"%s\" gravity\\n" "${target}" | pihole-FTL sqlite3 "${gravityTEMPfile}"; } 2>&1 )
-  status="$?"
-
-  if [[ "${status}" -ne 0 ]]; then
-    echo -e "\\n  ${CROSS} Unable to fill gravity table in database ${gravityTEMPfile}\\n  ${output}"
-    gravity_Cleanup "error"
-  else
-    echo -e "${OVER}  ${TICK} ${str}"
-  fi
-
-  if [[ "${status}" -eq 0 && -n "${output}" ]]; then
-    echo -e "  Encountered non-critical SQL warnings. Please check the suitability of the lists you're using!\\n\\n  SQL warnings:"
-    local warning file line lineno
-    while IFS= read -r line; do
-      echo "  - ${line}"
-      warning="$(grep -oh "^[^:]*:[0-9]*" <<< "${line}")"
-      file="${warning%:*}"
-      lineno="${warning#*:}"
-      if [[ -n "${file}" && -n "${lineno}" ]]; then
-        echo -n "    Line contains: "
-        awk "NR==${lineno}" < "${file}"
-      fi
-    done <<< "${output}"
-    echo ""
-  fi
-
-  rm "${target}" > /dev/null 2>&1 || \
-    echo -e "  ${CROSS} Unable to remove ${target}"
-
   gravity_Blackbody=true
 }
 
-# num_target_lines does increase for every correctly added domain in pareseList()
-num_target_lines=0
-num_source_lines=0
-num_invalid=0
-parseList() {
-  local adlistID="${1}" src="${2}" target="${3}" incorrect_lines sample_incorrect_lines
-  # This sed does the following things:
-  # 1. Remove all lines containing no domains
-  # 2. Remove all domains containing invalid characters. Valid are: a-z, A-Z, 0-9, dot (.), minus (-), underscore (_)
-  # 3. Append ,adlistID to every line
-  # 4. Remove trailing period (see https://github.com/pi-hole/pi-hole/issues/4701)
-  # 5. Ensures there is a newline on the last line
-  sed -r  "/([^\.]+\.)+[^\.]{2,}/!d;/[^a-zA-Z0-9.\_-]/d;s/\.$//;s/$/,${adlistID}/;/.$/a\\" "${src}" >> "${target}"
-
-  # Find lines containing no domains or with invalid characters (see above)
-  # Remove duplicates and limit to 5 domains
-  mapfile -t incorrect_lines <<< "$(sed -r "/([^\.]+\.)+[^\.]{2,}/d" < "${src}")"
-  mapfile -t -O "${#incorrect_lines[@]}" incorrect_lines <<< "$(sed -r "/[^a-zA-Z0-9.\_-]/!d" < "${src}")"
-  IFS=" " read -r -a sample_incorrect_lines <<< "$(tr ' ' '\n' <<< "${incorrect_lines[@]}" | sort -u | head -n 5| tr '\n' ' ')"
-
-  local num_target_lines_new num_correct_lines
-  # Get number of lines in source file
-  num_source_lines="$(grep -c "^" "${src}")"
-  # Get the new number of lines in destination file
-  num_target_lines_new="$(grep -c "^" "${target}")"
-  # Number of new correctly added lines
-  num_correct_lines="$(( num_target_lines_new-num_target_lines ))"
-  # Update number of lines in target file
-  num_target_lines="$num_target_lines_new"
-  num_invalid="$(( num_source_lines-num_correct_lines ))"
-  if [[ "${num_invalid}" -eq 0 ]]; then
-    echo "  ${INFO} Analyzed ${num_source_lines} domains"
-  else
-    echo "  ${INFO} Analyzed ${num_source_lines} domains, ${num_invalid} domains invalid!"
-  fi
-
-  # Display sample of invalid lines if we found some
-  if [ ${#sample_incorrect_lines[@]} -ne 0 ]; then
-    echo "      Sample of invalid domains:"
-    for each in "${sample_incorrect_lines[@]}"
-    do
-        echo "      - ${each}"
-    done
-  fi
-}
 compareLists() {
   local adlistID="${1}" target="${2}"
 
@@ -574,7 +487,6 @@ compareLists() {
       sha1sum "${target}" > "${target}.sha1"
       echo "  ${INFO} List has been updated"
       database_adlist_status "${adlistID}" "1"
-      database_adlist_updated "${adlistID}"
     else
       echo "  ${INFO} List stayed unchanged"
       database_adlist_status "${adlistID}" "2"
@@ -584,17 +496,19 @@ compareLists() {
     sha1sum "${target}" > "${target}.sha1"
     # We assume here it was changed upstream
     database_adlist_status "${adlistID}" "1"
-    database_adlist_updated "${adlistID}"
   fi
 }
 
 # Download specified URL and perform checks on HTTP status and file content
 gravity_DownloadBlocklistFromUrl() {
-  local url="${1}" cmd_ext="${2}" agent="${3}" adlistID="${4}" saveLocation="${5}" target="${6}" compression="${7}"
-  local heisenbergCompensator="" patternBuffer str httpCode success="" ip
+  local url="${1}" adlistID="${2}" saveLocation="${3}" target="${4}" compression="${5}"
+  local heisenbergCompensator="" listCurlBuffer str httpCode success="" ip cmd_ext
+  local file_path permissions ip_addr port blocked=false download=true
 
   # Create temp file to store content on disk instead of RAM
-  patternBuffer=$(mktemp -p "/tmp" --suffix=".phgpb")
+  # We don't use '--suffix' here because not all implementations of mktemp support it, e.g. on Alpine
+  listCurlBuffer="$(mktemp -p "${GRAVITY_TMPDIR}")"
+  mv "${listCurlBuffer}" "${listCurlBuffer%.*}.phgpb"
 
   # Determine if $saveLocation has read permission
   if [[ -r "${saveLocation}" && $url != "file"* ]]; then
@@ -606,7 +520,6 @@ gravity_DownloadBlocklistFromUrl() {
 
   str="Status:"
   echo -ne "  ${INFO} ${str} Pending..."
-  blocked=false
   case $BLOCKINGMODE in
     "IP-NODATA-AAAA"|"IP")
       # Get IP address of this domain
@@ -644,19 +557,53 @@ gravity_DownloadBlocklistFromUrl() {
     bad_list=$(pihole -q -adlist "${domain}" | head -n1 | awk -F 'Match found in ' '{print $2}')
     echo -e "${OVER}  ${CROSS} ${str} ${domain} is blocked by ${bad_list%:}. Using DNS on ${PIHOLE_DNS_1} to download ${url}";
     echo -ne "  ${INFO} ${str} Pending..."
-    cmd_ext="--resolve $domain:$port:$ip $cmd_ext"
+    cmd_ext="--resolve $domain:$port:$ip"
   fi
 
-  # shellcheck disable=SC2086
-  httpCode=$(curl --connect-timeout ${curl_connect_timeout} -s -L ${compression} ${cmd_ext} ${heisenbergCompensator} -w "%{http_code}" -A "${agent}" "${url}" -o "${patternBuffer}" 2> /dev/null)
+  # If we are going to "download" a local file, we first check if the target
+  # file has a+r permission. We explicitly check for all+read because we want
+  # to make sure that the file is readable by everyone and not just the user
+  # running the script.
+  if [[ $url == "file://"* ]]; then
+    # Get the file path
+    file_path=$(echo "$url" | cut -d'/' -f3-)
+    # Check if the file exists and is a regular file (i.e. not a socket, fifo, tty, block). Might still be a symlink.
+    if [[ ! -f $file_path ]]; then
+      # Output that the file does not exist
+      echo -e "${OVER}  ${CROSS} ${file_path} does not exist"
+      download=false
+    else
+      # Check if the file or a file referenced by the symlink has a+r permissions
+      permissions=$(stat -L -c "%a" "$file_path")
+      if [[ $permissions == *4 || $permissions == *5 || $permissions == *6 || $permissions == *7 ]]; then
+        # Output that we are using the local file
+        echo -e "${OVER}  ${INFO} Using local file ${file_path}"
+      else
+        # Output that the file does not have the correct permissions
+        echo -e "${OVER}  ${CROSS} Cannot read file (file needs to have a+r permission)"
+        download=false
+      fi
+    fi
+  fi
+
+  # Check for allowed protocols
+  if [[ $url != "http"* && $url != "https"* && $url != "file"* && $url != "ftp"* && $url != "ftps"* && $url != "sftp"* ]]; then
+    echo -e "${OVER}  ${CROSS} ${str} Invalid protocol specified, ignoring list"
+    download=false
+  fi
+
+  if [[ "${download}" == true ]]; then
+    # shellcheck disable=SC2086
+    httpCode=$(curl --connect-timeout ${curl_connect_timeout} -s -L ${compression} ${cmd_ext} ${heisenbergCompensator} -w "%{http_code}" "${url}" -o "${listCurlBuffer}" 2> /dev/null)
+  fi
 
   case $url in
     # Did we "download" a local file?
     "file"*)
-      if [[ -s "${patternBuffer}" ]]; then
+      if [[ -s "${listCurlBuffer}" ]]; then
         echo -e "${OVER}  ${TICK} ${str} Retrieval successful"; success=true
       else
-        echo -e "${OVER}  ${CROSS} ${str} Not found / empty list"
+        echo -e "${OVER}  ${CROSS} ${str} Retrieval failed / empty list"
       fi;;
     # Did we "download" a remote file?
     *)
@@ -682,24 +629,22 @@ gravity_DownloadBlocklistFromUrl() {
   if [[ "${success}" == true ]]; then
     if [[ "${httpCode}" == "304" ]]; then
       # Add domains to database table file
-      parseList "${adlistID}" "${saveLocation}" "${target}"
+      pihole-FTL gravity parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}"
       database_adlist_status "${adlistID}" "2"
-      database_adlist_number "${adlistID}"
       done="true"
-    # Check if $patternbuffer is a non-zero length file
-    elif [[ -s "${patternBuffer}" ]]; then
+    # Check if $listCurlBuffer is a non-zero length file
+    elif [[ -s "${listCurlBuffer}" ]]; then
       # Determine if blocklist is non-standard and parse as appropriate
-      gravity_ParseFileIntoDomains "${patternBuffer}" "${saveLocation}"
+      gravity_ParseFileIntoDomains "${listCurlBuffer}" "${saveLocation}"
+      # Remove curl buffer file after its use
+      rm "${listCurlBuffer}"
       # Add domains to database table file
-      parseList "${adlistID}" "${saveLocation}" "${target}"
+      pihole-FTL gravity parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}"
       # Compare lists, are they identical?
       compareLists "${adlistID}" "${saveLocation}"
-      # Update gravity database table (status and updated timestamp are set in
-      # compareLists)
-      database_adlist_number "${adlistID}"
       done="true"
     else
-      # Fall back to previously cached list if $patternBuffer is empty
+      # Fall back to previously cached list if $listCurlBuffer is empty
       echo -e "  ${INFO} Received empty file"
     fi
   fi
@@ -710,15 +655,12 @@ gravity_DownloadBlocklistFromUrl() {
     if [[ -r "${saveLocation}" ]]; then
       echo -e "  ${CROSS} List download failed: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}"
       # Add domains to database table file
-      parseList "${adlistID}" "${saveLocation}" "${target}"
-      database_adlist_number "${adlistID}"
+      pihole-FTL gravity parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}"
       database_adlist_status "${adlistID}" "3"
     else
       echo -e "  ${CROSS} List download failed: ${COL_LIGHT_RED}no cached list available${COL_NC}"
       # Manually reset these two numbers because we do not call parseList here
-      num_source_lines=0
-      num_invalid=0
-      database_adlist_number "${adlistID}"
+      database_adlist_number "${adlistID}" 0 0
       database_adlist_status "${adlistID}" "4"
     fi
   fi
@@ -732,18 +674,26 @@ gravity_ParseFileIntoDomains() {
   # Most of the lists downloaded are already in hosts file format but the spacing/formatting is not contiguous
   # This helps with that and makes it easier to read
   # It also helps with debugging so each stage of the script can be researched more in depth
-  # 1) Remove carriage returns
-  # 2) Convert all characters to lowercase
-  # 3) Remove comments (text starting with "#", include possible spaces before the hash sign)
-  # 4) Remove lines containing "/"
-  # 5) Remove leading tabs, spaces, etc.
-  # 6) Remove empty lines
-  < "${src}" tr -d '\r' | \
-  tr '[:upper:]' '[:lower:]' | \
-  sed 's/\s*#.*//g' | \
-  sed -r '/(\/).*$/d' | \
-  sed -r 's/^.*\s+//g' | \
-  sed '/^$/d'>  "${destination}"
+  # 1) Convert all characters to lowercase
+  tr '[:upper:]' '[:lower:]' < "${src}" > "${destination}"
+
+  # 2) Remove carriage returns
+  # 3) Remove lines starting with ! (ABP Comments)
+  # 4) Remove lines starting with [ (ABP Header)
+  # 5) Remove lines containing ABP extended CSS selectors ("##", "#!#", "#@#", "#?#") preceded by a letter
+  # 6) Remove comments (text starting with "#", include possible spaces before the hash sign)
+  # 7) Remove leading tabs, spaces, etc. (Also removes leading IP addresses)
+  # 8) Remove empty lines
+
+    sed -i -r \
+    -e 's/\r$//' \
+    -e 's/\s*!.*//g' \
+    -e 's/\s*\[.*//g' \
+    -e '/[a-z]\#[$?@]{0,1}\#/d' \
+    -e 's/\s*#.*//g' \
+    -e 's/^.*\s+//g' \
+    -e '/^$/d' "${destination}"
+
   chmod 644 "${destination}"
 }
 
@@ -752,12 +702,12 @@ gravity_Table_Count() {
   local table="${1}"
   local str="${2}"
   local num
-  num="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${table};")"
-  if [[ "${table}" == "vw_gravity" ]]; then
+  num="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT COUNT(*) FROM ${table};")"
+  if [[ "${table}" == "gravity" ]]; then
     local unique
-    unique="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(DISTINCT domain) FROM ${table};")"
+    unique="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT COUNT(*) FROM (SELECT DISTINCT domain FROM ${table});")"
     echo -e "  ${INFO} Number of ${str}: ${num} (${COL_BOLD}${unique} unique domains${COL_NC})"
-    pihole-FTL sqlite3 "${gravityDBfile}" "INSERT OR REPLACE INTO info (property,value) VALUES ('gravity_count',${unique});"
+    pihole-FTL sqlite3 -ni "${gravityDBfile}" "INSERT OR REPLACE INTO info (property,value) VALUES ('gravity_count',${unique});"
   else
     echo -e "  ${INFO} Number of ${str}: ${num}"
   fi
@@ -765,7 +715,9 @@ gravity_Table_Count() {
 
 # Output count of blacklisted domains and regex filters
 gravity_ShowCount() {
-  gravity_Table_Count "vw_gravity" "gravity domains" ""
+  # Here we use the table "gravity" instead of the view "vw_gravity" for speed.
+  # It's safe to replace it here, because right after a gravity run both will show the exactly same number of domains.
+  gravity_Table_Count "gravity" "gravity domains" ""
   gravity_Table_Count "vw_blacklist" "exact blacklisted domains"
   gravity_Table_Count "vw_regex_blacklist" "regex blacklist filters"
   gravity_Table_Count "vw_whitelist" "exact whitelisted domains"
@@ -799,7 +751,10 @@ gravity_Cleanup() {
   # Delete tmp content generated by Gravity
   rm ${piholeDir}/pihole.*.txt 2> /dev/null
   rm ${piholeDir}/*.tmp 2> /dev/null
-  rm /tmp/*.phgpb 2> /dev/null
+  # listCurlBuffer location
+  rm "${GRAVITY_TMPDIR}"/*.phgpb 2> /dev/null
+  # invalid_domains location
+  rm "${GRAVITY_TMPDIR}"/*.ph-non-domains 2> /dev/null
 
   # Ensure this function only runs when gravity_SetDownloadOptions() has completed
   if [[ "${gravity_Blackbody:-}" == true ]]; then
@@ -833,7 +788,7 @@ database_recovery() {
   local str="Checking integrity of existing gravity database (this can take a while)"
   local option="${1}"
   echo -ne "  ${INFO} ${str}..."
-  result="$(pihole-FTL sqlite3 "${gravityDBfile}" "PRAGMA integrity_check" 2>&1)"
+  result="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "PRAGMA integrity_check" 2>&1)"
 
   if [[ ${result} = "ok" ]]; then
     echo -e "${OVER}  ${TICK} ${str} - no errors found"
@@ -841,7 +796,7 @@ database_recovery() {
     str="Checking foreign keys of existing gravity database (this can take a while)"
     echo -ne "  ${INFO} ${str}..."
     unset result
-    result="$(pihole-FTL sqlite3 "${gravityDBfile}" "PRAGMA foreign_key_check" 2>&1)"
+    result="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "PRAGMA foreign_key_check" 2>&1)"
     if [[ -z ${result} ]]; then
       echo -e "${OVER}  ${TICK} ${str} - no errors found"
       if [[ "${option}" != "force" ]]; then
@@ -860,7 +815,7 @@ database_recovery() {
   echo -ne "  ${INFO} ${str}..."
   # We have to remove any possibly existing recovery database or this will fail
   rm -f "${gravityDBfile}.recovered" > /dev/null 2>&1
-  if result="$(pihole-FTL sqlite3 "${gravityDBfile}" ".recover" | pihole-FTL sqlite3 "${gravityDBfile}.recovered" 2>&1)"; then
+  if result="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" ".recover" | pihole-FTL sqlite3 -ni "${gravityDBfile}.recovered" 2>&1)"; then
     echo -e "${OVER}  ${TICK} ${str} - success"
     mv "${gravityDBfile}" "${gravityDBfile}.old"
     mv "${gravityDBfile}.recovered" "${gravityDBfile}"
@@ -962,7 +917,10 @@ if ! gravity_CheckDNSResolutionAvailable; then
   exit 1
 fi
 
-gravity_DownloadBlocklists
+if ! gravity_DownloadBlocklists; then
+  echo -e "   ${CROSS} Unable to create gravity database. Please try again later. If the problem persists, please contact support."
+  exit 1
+fi
 
 # Create local.list
 gravity_generateLocalList

manpages/pihole.8

@@ -184,7 +184,7 @@ Available commands and options:
     Specify whether the Pi-hole log should be used
 .br
 
-	(Logging options):
+    (Logging options):
 .br
       on                Enable the Pi-hole log at /var/log/pihole/pihole.log
 .br
@@ -212,7 +212,7 @@ Available commands and options:
 .br
       -p, --pihole      Only retrieve info regarding Pi-hole repository
 .br
-      -a, --admin       Only retrieve info regarding AdminLTE
+      -a, --admin       Only retrieve info regarding web
                         repository
 .br
       -f, --ftl         Only retrieve info regarding FTL repository
@@ -339,7 +339,7 @@ Displaying version information
 
 \fBpihole -v -a -c\fR
 .br
-    Display the current version of AdminLTE
+    Display the current version of web
 .br
 
 Temporarily disabling Pi-hole

pihole

@@ -24,7 +24,12 @@ utilsfile="${PI_HOLE_SCRIPT_DIR}/utils.sh"
 source "${utilsfile}"
 
 versionsfile="/etc/pihole/versions"
-source "${versionsfile}"
+if [ -f "${versionsfile}" ]; then
+    # Only source versionsfile if the file exits
+    # fixes a warning during installation where versionsfile does not exist yet
+    # but gravity calls `pihole -status` and thereby sourcing the file
+    source "${versionsfile}"
+fi
 
 webpageFunc() {
   source "${PI_HOLE_SCRIPT_DIR}/webpage.sh"

test/_centos_8.Dockerfile

@@ -1,18 +0,0 @@
-FROM quay.io/centos/centos:stream8
-RUN yum install -y git initscripts
-
-ENV GITDIR /etc/.pihole
-ENV SCRIPTDIR /opt/pihole
-
-RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole
-ADD . $GITDIR
-RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/
-ENV PATH /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR
-
-RUN true && \
-    chmod +x $SCRIPTDIR/*
-
-ENV SKIP_INSTALL true
-ENV OS_CHECK_DOMAIN_NAME dev-supportedos.pi-hole.net
-
-#sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \

test/_debian_12.Dockerfile

@@ -1,4 +1,4 @@
-FROM buildpack-deps:buster-scm
+FROM buildpack-deps:bookworm-scm
 
 ENV GITDIR /etc/.pihole
 ENV SCRIPTDIR /opt/pihole

test/_fedora_40.Dockerfile

@@ -1,4 +1,4 @@
-FROM fedora:37
+FROM fedora:40
 RUN dnf install -y git initscripts
 
 ENV GITDIR /etc/.pihole

test/_fedora_41.Dockerfile

@@ -1,4 +1,4 @@
-FROM fedora:35
+FROM fedora:41
 RUN dnf install -y git initscripts
 
 ENV GITDIR /etc/.pihole

test/_ubuntu_24.Dockerfile

@@ -1,5 +1,4 @@
-FROM fedora:36
-RUN dnf install -y git initscripts
+FROM buildpack-deps:24.04-scm
 
 ENV GITDIR /etc/.pihole
 ENV SCRIPTDIR /opt/pihole
@@ -8,6 +7,7 @@ RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole
 ADD . $GITDIR
 RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/
 ENV PATH /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR
+ENV DEBIAN_FRONTEND=noninteractive
 
 RUN true && \
     chmod +x $SCRIPTDIR/*

test/requirements.txt

@@ -1,6 +1,6 @@
-docker-compose == 1.29.2
-pytest == 7.2.1
-pytest-xdist == 3.1.0
-pytest-testinfra == 7.0.0
-tox == 4.2.8
+pyyaml == 6.0.1
+pytest == 8.2.2
+pytest-xdist == 3.6.1
+pytest-testinfra == 10.1.1
+tox == 4.15.1
 

test/test_any_automated_install.py

@@ -70,7 +70,7 @@ def test_setupVars_are_sourced_to_global_scope(host):
 
 def test_setupVars_saved_to_file(host):
     """
-    confirm saved settings are written to a file for future updates to re-use
+    confirm saved settings are written to a file for future updates to reuse
     """
     # dedent works better with this and padding matching script below
     set_setup_vars = "\n"
@@ -176,6 +176,12 @@ def test_installPihole_fresh_install_readableFiles(host):
     setup_var_file += "INSTALL_WEB_INTERFACE=true\n"
     setup_var_file += "EOF\n"
     host.run(setup_var_file)
+    # Install FTL's development branch to get the latest features
+    host.run(
+        """
+    echo "master" > /etc/pihole/ftlbranch
+    """
+    )
     install = host.run(
         """
     export TERM=xterm
@@ -431,6 +437,12 @@ def test_installPihole_fresh_install_readableBlockpage(host, test_webpage):
     setup_var_file += "INSTALL_WEB_INTERFACE=true\n"
     setup_var_file += "EOF\n"
     host.run(setup_var_file)
+    # Install FTL's development branch to get the latest features
+    host.run(
+        """
+    echo "master" > /etc/pihole/ftlbranch
+    """
+    )
     installWeb = host.run(
         """
     export TERM=xterm
@@ -881,7 +893,7 @@ def test_FTL_binary_installed_and_responsive_no_errors(host):
     source /opt/pihole/basic-install.sh
     create_pihole_user
     funcOutput=$(get_binary_name)
-    echo "development" > /etc/pihole/ftlbranch
+    echo "master" > /etc/pihole/ftlbranch
     binary="pihole-FTL${funcOutput##*pihole-FTL}"
     theRest="${funcOutput%pihole-FTL*}"
     FTLdetect "${binary}" "${theRest}"
@@ -1151,3 +1163,30 @@ def test_package_manager_has_web_deps(host):
 
     assert "No package" not in output.stdout
     assert output.rc == 0
+
+
+def test_webpage_sh_valid_domain(host):
+    """Confirms checkDomain function in webpage.sh works as expected"""
+    check1 = host.run(
+        """
+    source /opt/pihole/webpage.sh
+    checkDomain "pi-hole.net"
+    """
+    )
+    check2 = host.run(
+        """
+    source /opt/pihole/webpage.sh
+    checkDomain "ab.pi-hole.net"
+    """
+    )
+
+    check3 = host.run(
+        """
+    source /opt/pihole/webpage.sh
+    checkDomain "abc.pi-hole.net"
+    """
+    )
+
+    assert "pi-hole.net" in check1.stdout
+    assert "ab.pi-hole.net" in check2.stdout
+    assert "abc.pi-hole.net" in check3.stdout

test/test_any_utils.py

@@ -40,6 +40,26 @@ def test_key_addition_works(host):
     assert expected_stdout == output.stdout
 
 
+def test_key_addition_substr(host):
+    """Confirms addKey adds substring keys (no value) to a file"""
+    host.run(
+        """
+    source /opt/pihole/utils.sh
+    addKey "./testoutput" "KEY_ONE"
+    addKey "./testoutput" "KEY_O"
+    addKey "./testoutput" "KEY_TWO"
+    addKey "./testoutput" "Y_TWO"
+    """
+    )
+    output = host.run(
+        """
+    cat ./testoutput
+    """
+    )
+    expected_stdout = "KEY_ONE\nKEY_O\nKEY_TWO\nY_TWO\n"
+    assert expected_stdout == output.stdout
+
+
 def test_key_removal_works(host):
     """Confirms removeKey removes a key or key/value pair"""
     host.run(

test/tox.centos_8.ini

@@ -1,8 +0,0 @@
-[tox]
-envlist = py3
-
-[testenv:py3]
-allowlist_externals = docker
-deps = -rrequirements.txt
-commands =  docker build -f _centos_8.Dockerfile -t pytest_pihole:test_container ../
-            pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_centos_common_support.py

test/tox.centos_9.ini

@@ -4,5 +4,5 @@ envlist = py3
 [testenv:py3]
 allowlist_externals = docker
 deps = -rrequirements.txt
-commands = docker build -f _centos_9.Dockerfile -t pytest_pihole:test_container ../
+commands = docker buildx build --load --progress plain -f _centos_9.Dockerfile -t pytest_pihole:test_container ../
            pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_centos_common_support.py

test/tox.debian_11.ini

@@ -4,5 +4,5 @@ envlist = py3
 [testenv:py3]
 allowlist_externals = docker
 deps = -rrequirements.txt
-commands = docker build -f _debian_11.Dockerfile -t pytest_pihole:test_container ../
+commands = docker buildx build --load --progress plain -f _debian_11.Dockerfile -t pytest_pihole:test_container ../
            pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py

test/tox.debian_12.ini

@@ -4,5 +4,5 @@ envlist = py3
 [testenv:py3]
 allowlist_externals = docker
 deps = -rrequirements.txt
-commands = docker build -f _debian_10.Dockerfile -t pytest_pihole:test_container ../
+commands = docker buildx build --load --progress plain -f _debian_12.Dockerfile -t pytest_pihole:test_container ../
            pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py

test/tox.fedora_35.ini

@@ -1,8 +0,0 @@
-[tox]
-envlist = py3
-
-[testenv:py3]
-allowlist_externals = docker
-deps = -rrequirements.txt
-commands = docker build -f _fedora_35.Dockerfile -t pytest_pihole:test_container ../
-           pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_fedora_support.py

test/tox.fedora_40.ini

@@ -4,5 +4,5 @@ envlist = py3
 [testenv]
 allowlist_externals = docker
 deps = -rrequirements.txt
-commands = docker build -f _fedora_37.Dockerfile -t pytest_pihole:test_container ../
+commands = docker buildx build --load --progress plain -f _fedora_40.Dockerfile -t pytest_pihole:test_container ../
            pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_fedora_support.py

test/tox.fedora_41.ini

@@ -1,8 +1,8 @@
 [tox]
 envlist = py3
 
-[testenv:py3]
+[testenv]
 allowlist_externals = docker
 deps = -rrequirements.txt
-commands = docker build -f _fedora_36.Dockerfile -t pytest_pihole:test_container ../
+commands = docker buildx build --load --progress plain -f _fedora_41.Dockerfile -t pytest_pihole:test_container ../
            pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_fedora_support.py

test/tox.ubuntu_20.ini

@@ -4,5 +4,5 @@ envlist = py3
 [testenv:py3]
 allowlist_externals = docker
 deps = -rrequirements.txt
-commands = docker build -f _ubuntu_20.Dockerfile -t pytest_pihole:test_container ../
+commands = docker buildx build --load --progress plain -f _ubuntu_20.Dockerfile -t pytest_pihole:test_container ../
            pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py

test/tox.ubuntu_22.ini

@@ -4,5 +4,5 @@ envlist = py3
 [testenv:py3]
 allowlist_externals = docker
 deps = -rrequirements.txt
-commands = docker build -f _ubuntu_22.Dockerfile -t pytest_pihole:test_container ../
+commands = docker buildx build --load --progress plain -f _ubuntu_22.Dockerfile -t pytest_pihole:test_container ../
            pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py

test/tox.ubuntu_24.ini

@@ -0,0 +1,8 @@
+[tox]
+envlist = py3
+
+[testenv:py3]
+allowlist_externals = docker
+deps = -rrequirements.txt
+commands = docker buildx build --load --progress plain -f _ubuntu_24.Dockerfile -t pytest_pihole:test_container ../
+           pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py