v5.18.1 (#5618)

Signed-off-by: Adam Warner <me@adamwarner.co.uk>

.github/workflows/codeql-analysis.yml

@@ -25,16 +25,16 @@ jobs:
     steps:
     -
       name: Checkout repository
-      uses: actions/checkout@v4.1.0
+      uses: actions/checkout@v4.1.1
     # Initializes the CodeQL tools for scanning.
     -
       name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v3
       with:
         languages: 'python'
     -
       name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      uses: github/codeql-action/autobuild@v3
     -
       name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v3

.github/workflows/stale.yml

@@ -17,7 +17,7 @@ jobs:
       issues: write
 
     steps:
-      - uses: actions/stale@v8.0.0
+      - uses: actions/stale@v9.0.0
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           days-before-stale: 30
@@ -40,7 +40,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v4.1.0
+        uses: actions/checkout@v4.1.1
       - name: Remove 'stale' label
         run: gh issue edit ${{ github.event.issue.number }} --remove-label ${{ env.stale_label }}
         env:

.github/workflows/stale_pr.yml

@@ -17,7 +17,7 @@ jobs:
       pull-requests: write
 
     steps:
-      - uses: actions/stale@v8.0.0
+      - uses: actions/stale@v9.0.0
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           # Do not automatically mark PR/issue as stale

.github/workflows/sync-back-to-dev.yml

@@ -33,7 +33,7 @@ jobs:
     name: Syncing branches
     steps:
       - name: Checkout
-        uses: actions/checkout@v4.1.0
+        uses: actions/checkout@v4.1.1
       - name: Opening pull request
         run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal'
         env:

.github/workflows/test.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4.1.0
+        uses: actions/checkout@v4.1.1
 
       - name: Check scripts in repository are executable
         run: |
@@ -57,18 +57,17 @@ jobs:
             ubuntu_23,
             centos_8,
             centos_9,
-            fedora_36,
-            fedora_37,
-            fedora_38,
+             fedora_38,
+            fedora_39,
           ]
     env:
       DISTRO: ${{matrix.distro}}
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4.1.0
+        uses: actions/checkout@v4.1.1
 
       - name: Set up Python 3.10
-        uses: actions/setup-python@v4.7.1
+        uses: actions/setup-python@v5.0.0
         with:
           python-version: "3.10"
 

advanced/Scripts/database_migration/gravity-db.sh

@@ -19,13 +19,13 @@ upgrade_gravityDB(){
 	auditFile="${piholeDir}/auditlog.list"
 
 	# Get database version
-	version="$(pihole-FTL sqlite3 "${database}" "SELECT \"value\" FROM \"info\" WHERE \"property\" = 'version';")"
+	version="$(pihole-FTL sqlite3 -ni "${database}" "SELECT \"value\" FROM \"info\" WHERE \"property\" = 'version';")"
 
 	if [[ "$version" == "1" ]]; then
 		# This migration script upgrades the gravity.db file by
 		# adding the domain_audit table
 		echo -e "  ${INFO} Upgrading gravity database from version 1 to 2"
-		pihole-FTL sqlite3 "${database}" < "${scriptPath}/1_to_2.sql"
+		pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/1_to_2.sql"
 		version=2
 
 		# Store audit domains in database table
@@ -40,28 +40,28 @@ upgrade_gravityDB(){
 		# renaming the regex table to regex_blacklist, and
 		# creating a new regex_whitelist table + corresponding linking table and views
 		echo -e "  ${INFO} Upgrading gravity database from version 2 to 3"
-		pihole-FTL sqlite3 "${database}" < "${scriptPath}/2_to_3.sql"
+		pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/2_to_3.sql"
 		version=3
 	fi
 	if [[ "$version" == "3" ]]; then
 		# This migration script unifies the formally separated domain
 		# lists into a single table with a UNIQUE domain constraint
 		echo -e "  ${INFO} Upgrading gravity database from version 3 to 4"
-		pihole-FTL sqlite3 "${database}" < "${scriptPath}/3_to_4.sql"
+		pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/3_to_4.sql"
 		version=4
 	fi
 	if [[ "$version" == "4" ]]; then
 		# This migration script upgrades the gravity and list views
 		# implementing necessary changes for per-client blocking
 		echo -e "  ${INFO} Upgrading gravity database from version 4 to 5"
-		pihole-FTL sqlite3 "${database}" < "${scriptPath}/4_to_5.sql"
+		pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/4_to_5.sql"
 		version=5
 	fi
 	if [[ "$version" == "5" ]]; then
 		# This migration script upgrades the adlist view
 		# to return an ID used in gravity.sh
 		echo -e "  ${INFO} Upgrading gravity database from version 5 to 6"
-		pihole-FTL sqlite3 "${database}" < "${scriptPath}/5_to_6.sql"
+		pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/5_to_6.sql"
 		version=6
 	fi
 	if [[ "$version" == "6" ]]; then
@@ -69,7 +69,7 @@ upgrade_gravityDB(){
 		# which is automatically associated to all clients not
 		# having their own group assignments
 		echo -e "  ${INFO} Upgrading gravity database from version 6 to 7"
-		pihole-FTL sqlite3 "${database}" < "${scriptPath}/6_to_7.sql"
+		pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/6_to_7.sql"
 		version=7
 	fi
 	if [[ "$version" == "7" ]]; then
@@ -77,21 +77,21 @@ upgrade_gravityDB(){
 		# to ensure uniqueness on the group name
 		# We also add date_added and date_modified columns
 		echo -e "  ${INFO} Upgrading gravity database from version 7 to 8"
-		pihole-FTL sqlite3 "${database}" < "${scriptPath}/7_to_8.sql"
+		pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/7_to_8.sql"
 		version=8
 	fi
 	if [[ "$version" == "8" ]]; then
 		# This migration fixes some issues that were introduced
 		# in the previous migration script.
 		echo -e "  ${INFO} Upgrading gravity database from version 8 to 9"
-		pihole-FTL sqlite3 "${database}" < "${scriptPath}/8_to_9.sql"
+		pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/8_to_9.sql"
 		version=9
 	fi
 	if [[ "$version" == "9" ]]; then
 		# This migration drops unused tables and creates triggers to remove
 		# obsolete groups assignments when the linked items are deleted
 		echo -e "  ${INFO} Upgrading gravity database from version 9 to 10"
-		pihole-FTL sqlite3 "${database}" < "${scriptPath}/9_to_10.sql"
+		pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/9_to_10.sql"
 		version=10
 	fi
 	if [[ "$version" == "10" ]]; then
@@ -101,31 +101,31 @@ upgrade_gravityDB(){
 		# to keep the copying process generic (needs the same columns in both the
 		# source and the destination databases).
 		echo -e "  ${INFO} Upgrading gravity database from version 10 to 11"
-		pihole-FTL sqlite3 "${database}" < "${scriptPath}/10_to_11.sql"
+		pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/10_to_11.sql"
 		version=11
 	fi
 	if [[ "$version" == "11" ]]; then
 		# Rename group 0 from "Unassociated" to "Default"
 		echo -e "  ${INFO} Upgrading gravity database from version 11 to 12"
-		pihole-FTL sqlite3 "${database}" < "${scriptPath}/11_to_12.sql"
+		pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/11_to_12.sql"
 		version=12
 	fi
 	if [[ "$version" == "12" ]]; then
 		# Add column date_updated to adlist table
 		echo -e "  ${INFO} Upgrading gravity database from version 12 to 13"
-		pihole-FTL sqlite3 "${database}" < "${scriptPath}/12_to_13.sql"
+		pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/12_to_13.sql"
 		version=13
 	fi
 	if [[ "$version" == "13" ]]; then
 		# Add columns number and status to adlist table
 		echo -e "  ${INFO} Upgrading gravity database from version 13 to 14"
-		pihole-FTL sqlite3 "${database}" < "${scriptPath}/13_to_14.sql"
+		pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/13_to_14.sql"
 		version=14
 	fi
 	if [[ "$version" == "14" ]]; then
 		# Changes the vw_adlist created in 5_to_6
 		echo -e "  ${INFO} Upgrading gravity database from version 14 to 15"
-		pihole-FTL sqlite3 "${database}" < "${scriptPath}/14_to_15.sql"
+		pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/14_to_15.sql"
 		version=15
 	fi
 }

advanced/Scripts/list.sh

@@ -150,18 +150,18 @@ AddDomain() {
     domain="$1"
 
     # Is the domain in the list we want to add it to?
-    num="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}';")"
+    num="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}';")"
     requestedListname="$(GetListnameFromTypeId "${typeId}")"
 
     if [[ "${num}" -ne 0 ]]; then
-        existingTypeId="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT type FROM domainlist WHERE domain = '${domain}';")"
+        existingTypeId="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT type FROM domainlist WHERE domain = '${domain}';")"
         if [[ "${existingTypeId}" == "${typeId}" ]]; then
             if [[ "${verbose}" == true ]]; then
                 echo -e "  ${INFO} ${1} already exists in ${requestedListname}, no need to add!"
             fi
         else
             existingListname="$(GetListnameFromTypeId "${existingTypeId}")"
-            pihole-FTL sqlite3 "${gravityDBfile}" "UPDATE domainlist SET type = ${typeId} WHERE domain='${domain}';"
+            pihole-FTL sqlite3 -ni "${gravityDBfile}" "UPDATE domainlist SET type = ${typeId} WHERE domain='${domain}';"
             if [[ "${verbose}" == true ]]; then
                 echo -e "  ${INFO} ${1} already exists in ${existingListname}, it has been moved to ${requestedListname}!"
             fi
@@ -177,10 +177,10 @@ AddDomain() {
     # Insert only the domain here. The enabled and date_added fields will be filled
     # with their default values (enabled = true, date_added = current timestamp)
     if [[ -z "${comment}" ]]; then
-        pihole-FTL sqlite3 "${gravityDBfile}" "INSERT INTO domainlist (domain,type) VALUES ('${domain}',${typeId});"
+        pihole-FTL sqlite3 -ni "${gravityDBfile}" "INSERT INTO domainlist (domain,type) VALUES ('${domain}',${typeId});"
     else
         # also add comment when variable has been set through the "--comment" option
-        pihole-FTL sqlite3 "${gravityDBfile}" "INSERT INTO domainlist (domain,type,comment) VALUES ('${domain}',${typeId},'${comment}');"
+        pihole-FTL sqlite3 -ni "${gravityDBfile}" "INSERT INTO domainlist (domain,type,comment) VALUES ('${domain}',${typeId},'${comment}');"
     fi
 }
 
@@ -189,7 +189,7 @@ RemoveDomain() {
     domain="$1"
 
     # Is the domain in the list we want to remove it from?
-    num="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}' AND type = ${typeId};")"
+    num="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}' AND type = ${typeId};")"
 
     requestedListname="$(GetListnameFromTypeId "${typeId}")"
 
@@ -206,14 +206,14 @@ RemoveDomain() {
     fi
     reload=true
     # Remove it from the current list
-    pihole-FTL sqlite3 "${gravityDBfile}" "DELETE FROM domainlist WHERE domain = '${domain}' AND type = ${typeId};"
+    pihole-FTL sqlite3 -ni "${gravityDBfile}" "DELETE FROM domainlist WHERE domain = '${domain}' AND type = ${typeId};"
 }
 
 Displaylist() {
     local count num_pipes domain enabled status nicedate requestedListname
 
     requestedListname="$(GetListnameFromTypeId "${typeId}")"
-    data="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT domain,enabled,date_modified FROM domainlist WHERE type = ${typeId};" 2> /dev/null)"
+    data="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT domain,enabled,date_modified FROM domainlist WHERE type = ${typeId};" 2> /dev/null)"
 
     if [[ -z $data ]]; then
         echo -e "Not showing empty list"
@@ -251,10 +251,10 @@ Displaylist() {
 }
 
 NukeList() {
-    count=$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(1) FROM domainlist WHERE type = ${typeId};")
+    count=$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT COUNT(1) FROM domainlist WHERE type = ${typeId};")
     listname="$(GetListnameFromTypeId "${typeId}")"
     if [ "$count" -gt 0 ];then
-        pihole-FTL sqlite3 "${gravityDBfile}" "DELETE FROM domainlist WHERE type = ${typeId};"
+        pihole-FTL sqlite3 -ni "${gravityDBfile}" "DELETE FROM domainlist WHERE type = ${typeId};"
         echo "  ${TICK} Removed ${count} domain(s) from the ${listname}"
     else
         echo "  ${INFO} ${listname} already empty. Nothing to do!"

advanced/Scripts/piholeARPTable.sh

@@ -39,7 +39,7 @@ flushARP(){
     # Truncate network_addresses table in pihole-FTL.db
     # This needs to be done before we can truncate the network table due to
     # foreign key constraints
-    if ! output=$(pihole-FTL sqlite3 "${DBFILE}" "DELETE FROM network_addresses" 2>&1); then
+    if ! output=$(pihole-FTL sqlite3 -ni "${DBFILE}" "DELETE FROM network_addresses" 2>&1); then
         echo -e "${OVER}  ${CROSS} Failed to truncate network_addresses table"
         echo "  Database location: ${DBFILE}"
         echo "  Output: ${output}"
@@ -47,7 +47,7 @@ flushARP(){
     fi
 
     # Truncate network table in pihole-FTL.db
-    if ! output=$(pihole-FTL sqlite3 "${DBFILE}" "DELETE FROM network" 2>&1); then
+    if ! output=$(pihole-FTL sqlite3 -ni "${DBFILE}" "DELETE FROM network" 2>&1); then
         echo -e "${OVER}  ${CROSS} Failed to truncate network table"
         echo "  Database location: ${DBFILE}"
         echo "  Output: ${output}"

advanced/Scripts/piholeDebug.sh

@@ -865,7 +865,7 @@ dig_at() {
     # This helps emulate queries to different domains that a user might query
     # It will also give extra assurance that Pi-hole is correctly resolving and blocking domains
     local random_url
-    random_url=$(pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity WHERE domain not like '||%^' ORDER BY RANDOM() LIMIT 1")
+    random_url=$(pihole-FTL sqlite3 -ni "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity WHERE domain not like '||%^' ORDER BY RANDOM() LIMIT 1")
     # Fallback if no non-ABP style domains were found
     if [ -z "${random_url}" ]; then
         random_url="flurry.com"
@@ -1226,7 +1226,7 @@ show_db_entries() {
     IFS=$'\r\n'
     local entries=()
     mapfile -t entries < <(\
-        pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" \
+        pihole-FTL sqlite3 -ni "${PIHOLE_GRAVITY_DB_FILE}" \
             -cmd ".headers on" \
             -cmd ".mode column" \
             -cmd ".width ${widths}" \
@@ -1251,7 +1251,7 @@ show_FTL_db_entries() {
     IFS=$'\r\n'
     local entries=()
     mapfile -t entries < <(\
-        pihole-FTL sqlite3 "${PIHOLE_FTL_DB_FILE}" \
+        pihole-FTL sqlite3 -ni "${PIHOLE_FTL_DB_FILE}" \
             -cmd ".headers on" \
             -cmd ".mode column" \
             -cmd ".width ${widths}" \
@@ -1317,7 +1317,7 @@ analyze_gravity_list() {
     fi
 
     show_db_entries "Info table" "SELECT property,value FROM info" "20 40"
-    gravity_updated_raw="$(pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT value FROM info where property = 'updated'")"
+    gravity_updated_raw="$(pihole-FTL sqlite3 -ni "${PIHOLE_GRAVITY_DB_FILE}" "SELECT value FROM info where property = 'updated'")"
     gravity_updated="$(date -d @"${gravity_updated_raw}")"
     log_write "   Last gravity run finished at: ${COL_CYAN}${gravity_updated}${COL_NC}"
     log_write ""
@@ -1325,7 +1325,7 @@ analyze_gravity_list() {
     OLD_IFS="$IFS"
     IFS=$'\r\n'
     local gravity_sample=()
-    mapfile -t gravity_sample < <(pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity LIMIT 10")
+    mapfile -t gravity_sample < <(pihole-FTL sqlite3 -ni "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity LIMIT 10")
     log_write "   ${COL_CYAN}----- First 10 Gravity Domains -----${COL_NC}"
 
     for line in "${gravity_sample[@]}"; do
@@ -1357,7 +1357,7 @@ database_integrity_check(){
 
       log_write "${INFO} Checking foreign key constraints of ${database} ... (this can take several minutes)"
       unset result
-      result="$(pihole-FTL sqlite3 "${database}" -cmd ".headers on" -cmd ".mode column" "PRAGMA foreign_key_check" 2>&1 & spinner)"
+      result="$(pihole-FTL sqlite3 -ni "${database}" -cmd ".headers on" -cmd ".mode column" "PRAGMA foreign_key_check" 2>&1 & spinner)"
       if [[ -z ${result} ]]; then
         log_write "${TICK} No foreign key errors in ${database}"
       else

advanced/Scripts/piholeLogFlush.sh

@@ -63,7 +63,7 @@ else
         fi
     fi
     # Delete most recent 24 hours from FTL's database, leave even older data intact (don't wipe out all history)
-    deleted=$(pihole-FTL sqlite3 "${DBFILE}" "DELETE FROM query_storage WHERE timestamp >= strftime('%s','now')-86400; select changes() from query_storage limit 1")
+    deleted=$(pihole-FTL sqlite3 -ni "${DBFILE}" "DELETE FROM query_storage WHERE timestamp >= strftime('%s','now')-86400; select changes() from query_storage limit 1")
 
     # Restart pihole-FTL to force reloading history
     sudo pihole restartdns

advanced/Scripts/query.sh

@@ -96,7 +96,7 @@ scanDatabaseTable() {
         # Are there ABP entries on gravity?
         # Return 1 if abp_domain=1 or Zero if abp_domain=0 or not set
         abpquerystr="SELECT EXISTS (SELECT 1 FROM info WHERE property='abp_domains' and value='1')"
-        abpfound="$(pihole-FTL sqlite3 "${gravityDBfile}" "${abpquerystr}")" 2> /dev/null
+        abpfound="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "${abpquerystr}")" 2> /dev/null
 
         # Create search string for ABP entries only if needed
         if [ "${abpfound}" -eq 1 ]; then
@@ -129,7 +129,7 @@ scanDatabaseTable() {
     fi
 
     # Send prepared query to gravity database
-    result="$(pihole-FTL sqlite3 -separator ',' "${gravityDBfile}" "${querystr}")" 2> /dev/null
+    result="$(pihole-FTL sqlite3 -ni -separator ',' "${gravityDBfile}" "${querystr}")" 2> /dev/null
     if [[ -z "${result}" ]]; then
         # Return early when there are no matches in this table
         return
@@ -166,7 +166,7 @@ scanRegexDatabaseTable() {
     list_type="${3:-}"
 
     # Query all regex from the corresponding database tables
-    mapfile -t regexList < <(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT domain FROM domainlist WHERE type = ${list_type}" 2> /dev/null)
+    mapfile -t regexList < <(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT domain FROM domainlist WHERE type = ${list_type}" 2> /dev/null)
 
     # If we have regexps to process
     if [[ "${#regexList[@]}" -ne 0 ]]; then

advanced/Scripts/webpage.sh

@@ -560,13 +560,13 @@ CustomizeAdLists() {
 
     if CheckUrl "${address}"; then
         if [[ "${args[2]}" == "enable" ]]; then
-            pihole-FTL sqlite3 "${gravityDBfile}" "UPDATE adlist SET enabled = 1 WHERE address = '${address}'"
+            pihole-FTL sqlite3 -ni "${gravityDBfile}" "UPDATE adlist SET enabled = 1 WHERE address = '${address}'"
         elif [[ "${args[2]}" == "disable" ]]; then
-            pihole-FTL sqlite3 "${gravityDBfile}" "UPDATE adlist SET enabled = 0 WHERE address = '${address}'"
+            pihole-FTL sqlite3 -ni "${gravityDBfile}" "UPDATE adlist SET enabled = 0 WHERE address = '${address}'"
         elif [[ "${args[2]}" == "add" ]]; then
-            pihole-FTL sqlite3 "${gravityDBfile}" "INSERT OR IGNORE INTO adlist (address, comment) VALUES ('${address}', '${comment}')"
+            pihole-FTL sqlite3 -ni "${gravityDBfile}" "INSERT OR IGNORE INTO adlist (address, comment) VALUES ('${address}', '${comment}')"
         elif [[ "${args[2]}" == "del" ]]; then
-            pihole-FTL sqlite3 "${gravityDBfile}" "DELETE FROM adlist WHERE address = '${address}'"
+            pihole-FTL sqlite3 -ni "${gravityDBfile}" "DELETE FROM adlist WHERE address = '${address}'"
         else
             echo "Not permitted"
             return 1
@@ -700,12 +700,12 @@ addAudit()
     done
     # Insert only the domain here. The date_added field will be
     # filled with its default value (date_added = current timestamp)
-    pihole-FTL sqlite3 "${gravityDBfile}" "INSERT INTO domain_audit (domain) VALUES ${domains};"
+    pihole-FTL sqlite3 -ni "${gravityDBfile}" "INSERT INTO domain_audit (domain) VALUES ${domains};"
 }
 
 clearAudit()
 {
-    pihole-FTL sqlite3 "${gravityDBfile}" "DELETE FROM domain_audit;"
+    pihole-FTL sqlite3 -ni "${gravityDBfile}" "DELETE FROM domain_audit;"
 }
 
 SetPrivacyLevel() {

gravity.sh

@@ -84,7 +84,7 @@ fi
 
 # Generate new SQLite3 file from schema template
 generate_gravity_database() {
-  if ! pihole-FTL sqlite3 "${gravityDBfile}" < "${gravityDBschema}"; then
+  if ! pihole-FTL sqlite3 -ni "${gravityDBfile}" < "${gravityDBschema}"; then
     echo -e "   ${CROSS} Unable to create ${gravityDBfile}"
     return 1
   fi
@@ -99,7 +99,7 @@ gravity_swap_databases() {
   echo -ne "  ${INFO} ${str}..."
 
   # The index is intentionally not UNIQUE as poor quality adlists may contain domains more than once
-  output=$( { pihole-FTL sqlite3 "${gravityTEMPfile}" "CREATE INDEX idx_gravity ON gravity (domain, adlist_id);"; } 2>&1 )
+  output=$( { pihole-FTL sqlite3 -ni "${gravityTEMPfile}" "CREATE INDEX idx_gravity ON gravity (domain, adlist_id);"; } 2>&1 )
   status="$?"
 
   if [[ "${status}" -ne 0 ]]; then
@@ -135,7 +135,7 @@ gravity_swap_databases() {
 
 # Update timestamp when the gravity table was last updated successfully
 update_gravity_timestamp() {
-  output=$( { printf ".timeout 30000\\nINSERT OR REPLACE INTO info (property,value) values ('updated',cast(strftime('%%s', 'now') as int));" | pihole-FTL sqlite3 "${gravityDBfile}"; } 2>&1 )
+  output=$( { printf ".timeout 30000\\nINSERT OR REPLACE INTO info (property,value) values ('updated',cast(strftime('%%s', 'now') as int));" | pihole-FTL sqlite3 -ni "${gravityDBfile}"; } 2>&1 )
   status="$?"
 
   if [[ "${status}" -ne 0 ]]; then
@@ -179,7 +179,7 @@ database_table_from_file() {
 
   # Get MAX(id) from domainlist when INSERTing into this table
   if [[ "${table}" == "domainlist" ]]; then
-    rowid="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT MAX(id) FROM domainlist;")"
+    rowid="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT MAX(id) FROM domainlist;")"
     if [[ -z "$rowid" ]]; then
       rowid=0
     fi
@@ -209,7 +209,7 @@ database_table_from_file() {
   # Store domains in database table specified by ${table}
   # Use printf as .mode and .import need to be on separate lines
   # see https://unix.stackexchange.com/a/445615/83260
-  output=$( { printf ".timeout 30000\\n.mode csv\\n.import \"%s\" %s\\n" "${tmpFile}" "${table}" | pihole-FTL sqlite3 "${gravityDBfile}"; } 2>&1 )
+  output=$( { printf ".timeout 30000\\n.mode csv\\n.import \"%s\" %s\\n" "${tmpFile}" "${table}" | pihole-FTL sqlite3 -ni "${gravityDBfile}"; } 2>&1 )
   status="$?"
 
   if [[ "${status}" -ne 0 ]]; then
@@ -229,7 +229,7 @@ database_table_from_file() {
 
 # Check if a column with name ${2} exists in gravity table with name ${1}
 gravity_column_exists() {
-  output=$( { printf ".timeout 30000\\nSELECT EXISTS(SELECT * FROM pragma_table_info('%s') WHERE name='%s');\\n" "${1}" "${2}" | pihole-FTL sqlite3 "${gravityTEMPfile}"; } 2>&1 )
+  output=$( { printf ".timeout 30000\\nSELECT EXISTS(SELECT * FROM pragma_table_info('%s') WHERE name='%s');\\n" "${1}" "${2}" | pihole-FTL sqlite3 -ni "${gravityTEMPfile}"; } 2>&1 )
   if [[ "${output}" == "1" ]]; then
     return 0 # Bash 0 is success
   fi
@@ -244,7 +244,7 @@ database_adlist_number() {
     return;
   fi
 
-  output=$( { printf ".timeout 30000\\nUPDATE adlist SET number = %i, invalid_domains = %i WHERE id = %i;\\n" "${2}" "${3}" "${1}" | pihole-FTL sqlite3 "${gravityTEMPfile}"; } 2>&1 )
+  output=$( { printf ".timeout 30000\\nUPDATE adlist SET number = %i, invalid_domains = %i WHERE id = %i;\\n" "${2}" "${3}" "${1}" | pihole-FTL sqlite3 -ni "${gravityTEMPfile}"; } 2>&1 )
   status="$?"
 
   if [[ "${status}" -ne 0 ]]; then
@@ -260,7 +260,7 @@ database_adlist_status() {
     return;
   fi
 
-  output=$( { printf ".timeout 30000\\nUPDATE adlist SET status = %i WHERE id = %i;\\n" "${2}" "${1}" | pihole-FTL sqlite3 "${gravityTEMPfile}"; } 2>&1 )
+  output=$( { printf ".timeout 30000\\nUPDATE adlist SET status = %i WHERE id = %i;\\n" "${2}" "${1}" | pihole-FTL sqlite3 -ni "${gravityTEMPfile}"; } 2>&1 )
   status="$?"
 
   if [[ "${status}" -ne 0 ]]; then
@@ -378,8 +378,8 @@ gravity_DownloadBlocklists() {
 
   # Retrieve source URLs from gravity database
   # We source only enabled adlists, SQLite3 stores boolean values as 0 (false) or 1 (true)
-  mapfile -t sources <<< "$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2> /dev/null)"
-  mapfile -t sourceIDs <<< "$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT id FROM vw_adlist;" 2> /dev/null)"
+  mapfile -t sources <<< "$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2> /dev/null)"
+  mapfile -t sourceIDs <<< "$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT id FROM vw_adlist;" 2> /dev/null)"
 
   # Parse source domains from $sources
   mapfile -t sourceDomains <<< "$(
@@ -408,7 +408,7 @@ gravity_DownloadBlocklists() {
   str="Preparing new gravity database"
   echo -ne "  ${INFO} ${str}..."
   rm "${gravityTEMPfile}" > /dev/null 2>&1
-  output=$( { pihole-FTL sqlite3 "${gravityTEMPfile}" < "${gravityDBschema}"; } 2>&1 )
+  output=$( { pihole-FTL sqlite3 -ni "${gravityTEMPfile}" < "${gravityDBschema}"; } 2>&1 )
   status="$?"
 
   if [[ "${status}" -ne 0 ]]; then
@@ -428,7 +428,7 @@ gravity_DownloadBlocklists() {
     copyGravity="${copyGravity//"${gravityDBfile_default}"/"${gravityDBfile}"}"
   fi
 
-  output=$( { pihole-FTL sqlite3 "${gravityTEMPfile}" <<< "${copyGravity}"; } 2>&1 )
+  output=$( { pihole-FTL sqlite3 -ni "${gravityTEMPfile}" <<< "${copyGravity}"; } 2>&1 )
   status="$?"
 
   if [[ "${status}" -ne 0 ]]; then
@@ -503,6 +503,7 @@ compareLists() {
 gravity_DownloadBlocklistFromUrl() {
   local url="${1}" adlistID="${2}" saveLocation="${3}" target="${4}" compression="${5}"
   local heisenbergCompensator="" listCurlBuffer str httpCode success="" ip cmd_ext
+  local file_path permissions ip_addr port blocked=false download=true
 
   # Create temp file to store content on disk instead of RAM
   # We don't use '--suffix' here because not all implementations of mktemp support it, e.g. on Alpine
@@ -519,7 +520,6 @@ gravity_DownloadBlocklistFromUrl() {
 
   str="Status:"
   echo -ne "  ${INFO} ${str} Pending..."
-  blocked=false
   case $BLOCKINGMODE in
     "IP-NODATA-AAAA"|"IP")
       # Get IP address of this domain
@@ -560,8 +560,36 @@ gravity_DownloadBlocklistFromUrl() {
     cmd_ext="--resolve $domain:$port:$ip"
   fi
 
-  # shellcheck disable=SC2086
-  httpCode=$(curl --connect-timeout ${curl_connect_timeout} -s -L ${compression} ${cmd_ext} ${heisenbergCompensator} -w "%{http_code}" "${url}" -o "${listCurlBuffer}" 2> /dev/null)
+  # If we are going to "download" a local file, we first check if the target
+  # file has a+r permission. We explicitly check for all+read because we want
+  # to make sure that the file is readable by everyone and not just the user
+  # running the script.
+  if [[ $url == "file://"* ]]; then
+    # Get the file path
+    file_path=$(echo "$url" | cut -d'/' -f3-)
+    # Check if the file exists
+    if [[ ! -e $file_path ]]; then
+      # Output that the file does not exist
+      echo -e "${OVER}  ${CROSS} ${file_path} does not exist"
+      download=false
+    else
+      # Check if the file has a+r permissions
+      permissions=$(stat -c "%a" "$file_path")
+      if [[ $permissions == *4 || $permissions == *5 || $permissions == *6 || $permissions == *7 ]]; then
+        # Output that we are using the local file
+        echo -e "${OVER}  ${INFO} Using local file ${file_path}"
+      else
+        # Output that the file does not have the correct permissions
+        echo -e "${OVER}  ${CROSS} Cannot read file (file needs to have a+r permission)"
+        download=false
+      fi
+    fi
+  fi
+
+  if [[ "${download}" == true ]]; then
+    # shellcheck disable=SC2086
+    httpCode=$(curl --connect-timeout ${curl_connect_timeout} -s -L ${compression} ${cmd_ext} ${heisenbergCompensator} -w "%{http_code}" "${url}" -o "${listCurlBuffer}" 2> /dev/null)
+  fi
 
   case $url in
     # Did we "download" a local file?
@@ -569,7 +597,7 @@ gravity_DownloadBlocklistFromUrl() {
       if [[ -s "${listCurlBuffer}" ]]; then
         echo -e "${OVER}  ${TICK} ${str} Retrieval successful"; success=true
       else
-        echo -e "${OVER}  ${CROSS} ${str} Not found / empty list"
+        echo -e "${OVER}  ${CROSS} ${str} Retrieval failed / empty list"
       fi;;
     # Did we "download" a remote file?
     *)
@@ -668,12 +696,12 @@ gravity_Table_Count() {
   local table="${1}"
   local str="${2}"
   local num
-  num="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${table};")"
+  num="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT COUNT(*) FROM ${table};")"
   if [[ "${table}" == "gravity" ]]; then
     local unique
-    unique="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM (SELECT DISTINCT domain FROM ${table});")"
+    unique="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT COUNT(*) FROM (SELECT DISTINCT domain FROM ${table});")"
     echo -e "  ${INFO} Number of ${str}: ${num} (${COL_BOLD}${unique} unique domains${COL_NC})"
-    pihole-FTL sqlite3 "${gravityDBfile}" "INSERT OR REPLACE INTO info (property,value) VALUES ('gravity_count',${unique});"
+    pihole-FTL sqlite3 -ni "${gravityDBfile}" "INSERT OR REPLACE INTO info (property,value) VALUES ('gravity_count',${unique});"
   else
     echo -e "  ${INFO} Number of ${str}: ${num}"
   fi
@@ -754,7 +782,7 @@ database_recovery() {
   local str="Checking integrity of existing gravity database (this can take a while)"
   local option="${1}"
   echo -ne "  ${INFO} ${str}..."
-  result="$(pihole-FTL sqlite3 "${gravityDBfile}" "PRAGMA integrity_check" 2>&1)"
+  result="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "PRAGMA integrity_check" 2>&1)"
 
   if [[ ${result} = "ok" ]]; then
     echo -e "${OVER}  ${TICK} ${str} - no errors found"
@@ -762,7 +790,7 @@ database_recovery() {
     str="Checking foreign keys of existing gravity database (this can take a while)"
     echo -ne "  ${INFO} ${str}..."
     unset result
-    result="$(pihole-FTL sqlite3 "${gravityDBfile}" "PRAGMA foreign_key_check" 2>&1)"
+    result="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "PRAGMA foreign_key_check" 2>&1)"
     if [[ -z ${result} ]]; then
       echo -e "${OVER}  ${TICK} ${str} - no errors found"
       if [[ "${option}" != "force" ]]; then
@@ -781,7 +809,7 @@ database_recovery() {
   echo -ne "  ${INFO} ${str}..."
   # We have to remove any possibly existing recovery database or this will fail
   rm -f "${gravityDBfile}.recovered" > /dev/null 2>&1
-  if result="$(pihole-FTL sqlite3 "${gravityDBfile}" ".recover" | pihole-FTL sqlite3 "${gravityDBfile}.recovered" 2>&1)"; then
+  if result="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" ".recover" | pihole-FTL sqlite3 -ni "${gravityDBfile}.recovered" 2>&1)"; then
     echo -e "${OVER}  ${TICK} ${str} - success"
     mv "${gravityDBfile}" "${gravityDBfile}.old"
     mv "${gravityDBfile}.recovered" "${gravityDBfile}"

test/_fedora_37.Dockerfile

@@ -1,18 +0,0 @@
-FROM fedora:37
-RUN dnf install -y git initscripts
-
-ENV GITDIR /etc/.pihole
-ENV SCRIPTDIR /opt/pihole
-
-RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole
-ADD . $GITDIR
-RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/
-ENV PATH /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR
-
-RUN true && \
-    chmod +x $SCRIPTDIR/*
-
-ENV SKIP_INSTALL true
-ENV OS_CHECK_DOMAIN_NAME dev-supportedos.pi-hole.net
-
-#sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \

test/_fedora_39.Dockerfile

@@ -1,4 +1,4 @@
-FROM fedora:36
+FROM fedora:39
 RUN dnf install -y git initscripts
 
 ENV GITDIR /etc/.pihole

test/requirements.txt

@@ -1,6 +1,6 @@
 pyyaml == 6.0.1
-pytest == 7.4.2
-pytest-xdist == 3.3.1
-pytest-testinfra == 9.0.0
-tox == 4.11.3
+pytest == 8.0.2
+pytest-xdist == 3.5.0
+pytest-testinfra == 10.1.0
+tox == 4.14.1
 

test/test_any_automated_install.py

@@ -176,6 +176,12 @@ def test_installPihole_fresh_install_readableFiles(host):
     setup_var_file += "INSTALL_WEB_INTERFACE=true\n"
     setup_var_file += "EOF\n"
     host.run(setup_var_file)
+    # Install FTL's development branch to get the latest features
+    host.run(
+        """
+    echo "development" > /etc/pihole/ftlbranch
+    """
+    )
     install = host.run(
         """
     export TERM=xterm
@@ -431,6 +437,12 @@ def test_installPihole_fresh_install_readableBlockpage(host, test_webpage):
     setup_var_file += "INSTALL_WEB_INTERFACE=true\n"
     setup_var_file += "EOF\n"
     host.run(setup_var_file)
+    # Install FTL's development branch to get the latest features
+    host.run(
+        """
+    echo "development" > /etc/pihole/ftlbranch
+    """
+    )
     installWeb = host.run(
         """
     export TERM=xterm

test/tox.fedora_36.ini

@@ -1,8 +0,0 @@
-[tox]
-envlist = py3
-
-[testenv:py3]
-allowlist_externals = docker
-deps = -rrequirements.txt
-commands = docker buildx build --load --progress plain -f _fedora_36.Dockerfile -t pytest_pihole:test_container ../
-           pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_fedora_support.py

test/tox.fedora_39.ini

@@ -4,5 +4,5 @@ envlist = py3
 [testenv]
 allowlist_externals = docker
 deps = -rrequirements.txt
-commands = docker buildx build --load --progress plain -f _fedora_37.Dockerfile -t pytest_pihole:test_container ../
+commands = docker buildx build --load --progress plain -f _fedora_39.Dockerfile -t pytest_pihole:test_container ../
            pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_fedora_support.py