Compare commits
48 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
741717aa38 | ||
|
|
2d13cd2f1d | ||
|
|
788e7cc777 | ||
|
|
0ac89ac2e3 | ||
|
|
e5ea361b53 | ||
|
|
df0155abe1 | ||
|
|
ca00ffa101 | ||
|
|
3ad8965959 | ||
|
|
460f83580e | ||
|
|
c7ad7113d7 | ||
|
|
0a6ebadb52 | ||
|
|
b9e401aaa3 | ||
|
|
18ab94135f | ||
|
|
be0efa2332 | ||
|
|
9bde5de601 | ||
|
|
db42ed1e4f | ||
|
|
262ffe458b | ||
|
|
e59f5db145 | ||
|
|
362c604744 | ||
|
|
68a03cc877 | ||
|
|
671da760d3 | ||
|
|
3c3d913934 | ||
|
|
50be36400c | ||
|
|
5f9523afa6 | ||
|
|
81a31b9e7b | ||
|
|
d0affcb376 | ||
|
|
9939cf1d77 | ||
|
|
ce7f926e92 | ||
|
|
6b4f77bdfe | ||
|
|
267792aa1e | ||
|
|
9331cbff4b | ||
|
|
d30a5f1b95 | ||
|
|
d6e25403ee | ||
|
|
79f4a7cef0 | ||
|
|
9ed3ede0d9 | ||
|
|
771b7cfcc7 | ||
|
|
7bb86e4118 | ||
|
|
c992fd48b0 | ||
|
|
afc8241c2c | ||
|
|
6cae37e720 | ||
|
|
d604aec9f1 | ||
|
|
bb4698429f | ||
|
|
634e3b0e46 | ||
|
|
21c7c8a008 | ||
|
|
9e47b61c8f | ||
|
|
b9ebb05246 | ||
|
|
c59e11a332 | ||
|
|
f713b14ba0 |
1
.github/workflows/test.yml
vendored
1
.github/workflows/test.yml
vendored
@@ -55,7 +55,6 @@ jobs:
|
||||
ubuntu_22,
|
||||
centos_8,
|
||||
centos_9,
|
||||
fedora_35,
|
||||
fedora_36,
|
||||
fedora_37,
|
||||
]
|
||||
|
||||
@@ -66,6 +66,8 @@ RUN_DIRECTORY="/run"
|
||||
LOG_DIRECTORY="/var/log/pihole"
|
||||
WEB_SERVER_LOG_DIRECTORY="/var/log/lighttpd"
|
||||
WEB_SERVER_CONFIG_DIRECTORY="/etc/lighttpd"
|
||||
WEB_SERVER_CONFIG_DIRECTORY_FEDORA="${WEB_SERVER_CONFIG_DIRECTORY}/conf.d"
|
||||
WEB_SERVER_CONFIG_DIRECTORY_DEBIAN="${WEB_SERVER_CONFIG_DIRECTORY}/conf-enabled"
|
||||
HTML_DIRECTORY="/var/www/html"
|
||||
WEB_GIT_DIRECTORY="${HTML_DIRECTORY}/admin"
|
||||
SHM_DIRECTORY="/dev/shm"
|
||||
@@ -77,6 +79,8 @@ PIHOLE_CRON_FILE="${CRON_D_DIRECTORY}/pihole"
|
||||
|
||||
WEB_SERVER_CONFIG_FILE="${WEB_SERVER_CONFIG_DIRECTORY}/lighttpd.conf"
|
||||
WEB_SERVER_CUSTOM_CONFIG_FILE="${WEB_SERVER_CONFIG_DIRECTORY}/external.conf"
|
||||
WEB_SERVER_PIHOLE_CONFIG_FILE_DEBIAN="${WEB_SERVER_CONFIG_DIRECTORY_DEBIAN}/15-pihole-admin.conf"
|
||||
WEB_SERVER_PIHOLE_CONFIG_FILE_FEDORA="${WEB_SERVER_CONFIG_DIRECTORY_FEDORA}/pihole-admin.conf"
|
||||
|
||||
PIHOLE_INSTALL_LOG_FILE="${PIHOLE_DIRECTORY}/install.log"
|
||||
PIHOLE_RAW_BLOCKLIST_FILES="${PIHOLE_DIRECTORY}/list.*"
|
||||
@@ -140,6 +144,8 @@ PIHOLE_PROCESSES=( "lighttpd" "pihole-FTL" )
|
||||
REQUIRED_FILES=("${PIHOLE_CRON_FILE}"
|
||||
"${WEB_SERVER_CONFIG_FILE}"
|
||||
"${WEB_SERVER_CUSTOM_CONFIG_FILE}"
|
||||
"${WEB_SERVER_PIHOLE_CONFIG_FILE_DEBIAN}"
|
||||
"${WEB_SERVER_PIHOLE_CONFIG_FILE_FEDORA}"
|
||||
"${PIHOLE_INSTALL_LOG_FILE}"
|
||||
"${PIHOLE_RAW_BLOCKLIST_FILES}"
|
||||
"${PIHOLE_LOCAL_HOSTS_FILE}"
|
||||
@@ -977,6 +983,20 @@ ftl_full_status(){
|
||||
fi
|
||||
}
|
||||
|
||||
lighttpd_test_configuration(){
|
||||
# let lighttpd test it's own configuration
|
||||
local lighttpd_conf_test
|
||||
echo_current_diagnostic "Lighttpd configuration test"
|
||||
lighttpd_conf_test=$(lighttpd -tt -f /etc/lighttpd/lighttpd.conf)
|
||||
if [ -z "${lighttpd_conf_test}" ]; then
|
||||
# empty output
|
||||
log_write "${TICK} ${COL_GREEN}No error in lighttpd configuration${COL_NC}"
|
||||
else
|
||||
log_write "${CROSS} ${COL_RED}Error in lighttpd configuration${COL_NC}"
|
||||
log_write " ${lighttpd_conf_test}"
|
||||
fi
|
||||
}
|
||||
|
||||
make_array_from_file() {
|
||||
local filename="${1}"
|
||||
# The second argument can put a limit on how many line should be read from the file
|
||||
@@ -1069,10 +1089,13 @@ dir_check() {
|
||||
# check if exists first; if it does,
|
||||
if ls "${filename}" 1> /dev/null 2>&1; then
|
||||
# do nothing
|
||||
:
|
||||
true
|
||||
return
|
||||
else
|
||||
# Otherwise, show an error
|
||||
log_write "${COL_RED}${directory} does not exist.${COL_NC}"
|
||||
false
|
||||
return
|
||||
fi
|
||||
done
|
||||
}
|
||||
@@ -1080,6 +1103,19 @@ dir_check() {
|
||||
list_files_in_dir() {
|
||||
# Set the first argument passed to this function as a named variable for better readability
|
||||
local dir_to_parse="${1}"
|
||||
|
||||
# show files and sizes of some directories, don't print the file content (yet)
|
||||
if [[ "${dir_to_parse}" == "${SHM_DIRECTORY}" ]]; then
|
||||
# SHM file - we do not want to see the content, but we want to see the files and their sizes
|
||||
log_write "$(ls -lh "${dir_to_parse}/")"
|
||||
elif [[ "${dir_to_parse}" == "${WEB_SERVER_CONFIG_DIRECTORY_FEDORA}" ]]; then
|
||||
# we want to see all files files in /etc/lighttpd/conf.d
|
||||
log_write "$(ls -lh "${dir_to_parse}/" 2> /dev/null )"
|
||||
elif [[ "${dir_to_parse}" == "${WEB_SERVER_CONFIG_DIRECTORY_DEBIAN}" ]]; then
|
||||
# we want to see all files files in /etc/lighttpd/conf.d
|
||||
log_write "$(ls -lh "${dir_to_parse}/"/ 2> /dev/null )"
|
||||
fi
|
||||
|
||||
# Store the files found in an array
|
||||
mapfile -t files_found < <(ls "${dir_to_parse}")
|
||||
# For each file in the array,
|
||||
@@ -1095,11 +1131,8 @@ list_files_in_dir() {
|
||||
[[ "${dir_to_parse}/${each_file}" == "${PIHOLE_WEB_SERVER_ACCESS_LOG_FILE}" ]] || \
|
||||
[[ "${dir_to_parse}/${each_file}" == "${PIHOLE_LOG_GZIPS}" ]]; then
|
||||
:
|
||||
elif [[ "${dir_to_parse}" == "${SHM_DIRECTORY}" ]]; then
|
||||
# SHM file - we do not want to see the content, but we want to see the files and their sizes
|
||||
log_write "$(ls -lhd "${dir_to_parse}"/"${each_file}")"
|
||||
elif [[ "${dir_to_parse}" == "${DNSMASQ_D_DIRECTORY}" ]]; then
|
||||
# in case of the dnsmasq directory inlcuede all files in the debug output
|
||||
# in case of the dnsmasq directory include all files in the debug output
|
||||
log_write "\\n${COL_GREEN}$(ls -lhd "${dir_to_parse}"/"${each_file}")${COL_NC}"
|
||||
make_array_from_file "${dir_to_parse}/${each_file}"
|
||||
else
|
||||
@@ -1132,9 +1165,10 @@ show_content_of_files_in_dir() {
|
||||
# Set a local variable for better readability
|
||||
local directory="${1}"
|
||||
# Check if the directory exists
|
||||
dir_check "${directory}"
|
||||
# if it does, list the files in it
|
||||
list_files_in_dir "${directory}"
|
||||
if dir_check "${directory}"; then
|
||||
# if it does, list the files in it
|
||||
list_files_in_dir "${directory}"
|
||||
fi
|
||||
}
|
||||
|
||||
show_content_of_pihole_files() {
|
||||
@@ -1142,6 +1176,8 @@ show_content_of_pihole_files() {
|
||||
show_content_of_files_in_dir "${PIHOLE_DIRECTORY}"
|
||||
show_content_of_files_in_dir "${DNSMASQ_D_DIRECTORY}"
|
||||
show_content_of_files_in_dir "${WEB_SERVER_CONFIG_DIRECTORY}"
|
||||
show_content_of_files_in_dir "${WEB_SERVER_CONFIG_DIRECTORY_FEDORA}"
|
||||
show_content_of_files_in_dir "${WEB_SERVER_CONFIG_DIRECTORY_DEBIAN}"
|
||||
show_content_of_files_in_dir "${CRON_D_DIRECTORY}"
|
||||
show_content_of_files_in_dir "${WEB_SERVER_LOG_DIRECTORY}"
|
||||
show_content_of_files_in_dir "${LOG_DIRECTORY}"
|
||||
@@ -1496,6 +1532,7 @@ check_name_resolution
|
||||
check_dhcp_servers
|
||||
process_status
|
||||
ftl_full_status
|
||||
lighttpd_test_configuration
|
||||
parse_setup_vars
|
||||
check_x_headers
|
||||
analyze_ftl_db
|
||||
|
||||
@@ -23,6 +23,11 @@ is_running() {
|
||||
return 1
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
# Run post-stop script, which does cleanup among runtime files
|
||||
sh "${PI_HOLE_SCRIPT_DIR}/pihole-FTL-poststop.sh"
|
||||
}
|
||||
|
||||
|
||||
# Start the service
|
||||
start() {
|
||||
@@ -33,10 +38,16 @@ start() {
|
||||
sh "${PI_HOLE_SCRIPT_DIR}/pihole-FTL-prestart.sh"
|
||||
|
||||
if setcap CAP_NET_BIND_SERVICE,CAP_NET_RAW,CAP_NET_ADMIN,CAP_SYS_NICE,CAP_IPC_LOCK,CAP_CHOWN+eip "/usr/bin/pihole-FTL"; then
|
||||
su -s /bin/sh -c "/usr/bin/pihole-FTL" pihole || exit $?
|
||||
su -s /bin/sh -c "/usr/bin/pihole-FTL" pihole
|
||||
else
|
||||
echo "Warning: Starting pihole-FTL as root because setting capabilities is not supported on this system"
|
||||
/usr/bin/pihole-FTL || exit $?
|
||||
/usr/bin/pihole-FTL
|
||||
fi
|
||||
rc=$?
|
||||
# Cleanup if startup failed
|
||||
if [ "${rc}" != 0 ]; then
|
||||
cleanup
|
||||
exit $rc
|
||||
fi
|
||||
echo
|
||||
fi
|
||||
@@ -65,8 +76,7 @@ stop() {
|
||||
else
|
||||
echo "Not running"
|
||||
fi
|
||||
# Run post-stop script, which does cleanup among runtime files
|
||||
sh "${PI_HOLE_SCRIPT_DIR}/pihole-FTL-poststop.sh"
|
||||
cleanup
|
||||
echo
|
||||
}
|
||||
|
||||
@@ -84,6 +94,9 @@ status() {
|
||||
|
||||
### main logic ###
|
||||
|
||||
# catch sudden termination
|
||||
trap 'cleanup; exit 1' INT HUP TERM ABRT
|
||||
|
||||
# Get FTL's PID file path
|
||||
FTL_PID_FILE="$(getFTLPIDFile)"
|
||||
|
||||
|
||||
@@ -7,17 +7,18 @@
|
||||
# This file is copyright under the latest version of the EUPL.
|
||||
# Please see LICENSE file for your rights under this license.
|
||||
|
||||
###############################################################################
|
||||
# FILE AUTOMATICALLY OVERWRITTEN BY PI-HOLE INSTALL/UPDATE PROCEDURE. #
|
||||
# ANY CHANGES MADE TO THIS FILE AFTER INSTALL WILL BE LOST ON THE NEXT UPDATE #
|
||||
# #
|
||||
# CHANGES SHOULD BE MADE IN A SEPARATE CONFIG FILE: #
|
||||
# /etc/lighttpd/external.conf #
|
||||
###############################################################################
|
||||
###################################################################################################
|
||||
# IF THIS HEADER EXISTS, THE FILE WILL BE OVERWRITTEN BY PI-HOLE'S UPDATE PROCEDURE. #
|
||||
# ANY CHANGES MADE TO THIS FILE WILL BE LOST ON THE NEXT UPDATE UNLESS YOU REMOVE THIS HEADER #
|
||||
# #
|
||||
# ENSURE THAT YOU DO NOT REMOVE THE REQUIRED LINE: #
|
||||
# #
|
||||
# include "/etc/lighttpd/conf-enabled/*.conf" #
|
||||
# #
|
||||
###################################################################################################
|
||||
|
||||
server.modules = (
|
||||
"mod_access",
|
||||
"mod_accesslog",
|
||||
"mod_auth",
|
||||
"mod_expire",
|
||||
"mod_redirect",
|
||||
@@ -34,8 +35,6 @@ server.groupname = "www-data"
|
||||
# For lighttpd version 1.4.46 or above, the port can be overwritten in `/etc/lighttpd/external.conf` using the := operator
|
||||
# e.g. server.port := 8000
|
||||
server.port = 80
|
||||
accesslog.filename = "/var/log/lighttpd/access-pihole.log"
|
||||
accesslog.format = "%{%s}t|%V|%r|%s|%b"
|
||||
|
||||
# Allow streaming response
|
||||
# reference: https://redmine.lighttpd.net/projects/lighttpd/wiki/Server_stream-response-bodyDetails
|
||||
|
||||
@@ -7,13 +7,15 @@
|
||||
# This file is copyright under the latest version of the EUPL.
|
||||
# Please see LICENSE file for your rights under this license.
|
||||
|
||||
###############################################################################
|
||||
# FILE AUTOMATICALLY OVERWRITTEN BY PI-HOLE INSTALL/UPDATE PROCEDURE. #
|
||||
# ANY CHANGES MADE TO THIS FILE AFTER INSTALL WILL BE LOST ON THE NEXT UPDATE #
|
||||
# #
|
||||
# CHANGES SHOULD BE MADE IN A SEPARATE CONFIG FILE: #
|
||||
# /etc/lighttpd/external.conf #
|
||||
###############################################################################
|
||||
###################################################################################################
|
||||
# IF THIS HEADER EXISTS, THE FILE WILL BE OVERWRITTEN BY PI-HOLE'S UPDATE PROCEDURE. #
|
||||
# ANY CHANGES MADE TO THIS FILE WILL BE LOST ON THE NEXT UPDATE UNLESS YOU REMOVE THIS HEADER #
|
||||
# #
|
||||
# ENSURE THAT YOU DO NOT REMOVE THE REQUIRED LINE: #
|
||||
# #
|
||||
# include "/etc/lighttpd/conf.d/pihole-admin.conf" #
|
||||
# #
|
||||
###################################################################################################
|
||||
|
||||
server.modules = (
|
||||
"mod_access",
|
||||
@@ -35,8 +37,6 @@ server.groupname = "lighttpd"
|
||||
# For lighttpd version 1.4.46 or above, the port can be overwritten in `/etc/lighttpd/external.conf` using the := operator
|
||||
# e.g. server.port := 8000
|
||||
server.port = 80
|
||||
accesslog.filename = "/var/log/lighttpd/access-pihole.log"
|
||||
accesslog.format = "%{%s}t|%V|%r|%s|%b"
|
||||
|
||||
# Allow streaming response
|
||||
# reference: https://redmine.lighttpd.net/projects/lighttpd/wiki/Server_stream-response-bodyDetails
|
||||
|
||||
@@ -23,9 +23,9 @@ $HTTP["url"] =~ "^/admin/" {
|
||||
fastcgi.server = (
|
||||
".php" => (
|
||||
"localhost" => (
|
||||
"socket" => "/tmp/pihole-php-fastcgi.socket",
|
||||
"socket" => "/run/lighttpd/pihole-php-fastcgi.socket",
|
||||
"bin-path" => "/usr/bin/php-cgi",
|
||||
"min-procs" => 0,
|
||||
"min-procs" => 1,
|
||||
"max-procs" => 1,
|
||||
"bin-environment" => (
|
||||
"PHP_FCGI_CHILDREN" => "4",
|
||||
@@ -79,4 +79,4 @@ $HTTP["host"] == "pi.hole" {
|
||||
}
|
||||
|
||||
# (keep this on one line for basic-install.sh filtering during install)
|
||||
server.modules += ( "mod_access", "mod_redirect", "mod_fastcgi", "mod_setenv" )
|
||||
server.modules += ( "mod_access", "mod_accesslog", "mod_redirect", "mod_fastcgi", "mod_setenv" )
|
||||
|
||||
@@ -1400,12 +1400,19 @@ installConfigs() {
|
||||
|
||||
# If the user chose to install the dashboard,
|
||||
if [[ "${INSTALL_WEB_SERVER}" == true ]]; then
|
||||
if grep -q -F "FILE AUTOMATICALLY OVERWRITTEN BY PI-HOLE" "${lighttpdConfig}"; then
|
||||
# set permissions on /etc/lighttpd/lighttpd.conf so pihole user (other) can read the file
|
||||
chmod o+x /etc/lighttpd
|
||||
chmod o+r "${lighttpdConfig}"
|
||||
|
||||
# Ensure /run/lighttpd exists and is owned by lighttpd user
|
||||
# Needed for the php socket
|
||||
mkdir -p /run/lighttpd
|
||||
chown ${LIGHTTPD_USER}:${LIGHTTPD_GROUP} /run/lighttpd
|
||||
|
||||
if grep -q -F "FILE WILL BE OVERWRITTEN BY PI-HOLE" "${lighttpdConfig}"; then
|
||||
# Attempt to preserve backwards compatibility with older versions
|
||||
install -D -m 644 -T ${PI_HOLE_LOCAL_REPO}/advanced/${LIGHTTPD_CFG} "${lighttpdConfig}"
|
||||
# Make the directories if they do not exist and set the owners
|
||||
mkdir -p /run/lighttpd
|
||||
chown ${LIGHTTPD_USER}:${LIGHTTPD_GROUP} /run/lighttpd
|
||||
mkdir -p /var/cache/lighttpd/compress
|
||||
chown ${LIGHTTPD_USER}:${LIGHTTPD_GROUP} /var/cache/lighttpd/compress
|
||||
mkdir -p /var/cache/lighttpd/uploads
|
||||
@@ -1432,17 +1439,26 @@ installConfigs() {
|
||||
elif [[ -d "/etc/lighttpd/conf-available" ]]; then
|
||||
conf=/etc/lighttpd/conf-available/15-pihole-admin.conf
|
||||
install -D -m 644 -T ${PI_HOLE_LOCAL_REPO}/advanced/pihole-admin.conf $conf
|
||||
# disable server.modules += ( ... ) in $conf to avoid module dups
|
||||
# (needed until Debian 10 no longer supported by pi-hole)
|
||||
# (server.modules duplication is ignored in lighttpd 1.4.56+)
|
||||
if awk '!/^server\.modules/{print}' $conf > $conf.$$ && mv $conf.$$ $conf; then
|
||||
|
||||
# Get the version number of lighttpd
|
||||
version=$(dpkg-query -f='${Version}\n' --show lighttpd)
|
||||
# Test if that version is greater than or euqal to 1.4.56
|
||||
if dpkg --compare-versions "$version" "ge" "1.4.56"; then
|
||||
# If it is, then we don't need to disable the modules
|
||||
# (server.modules duplication is ignored in lighttpd 1.4.56+)
|
||||
:
|
||||
else
|
||||
rm $conf.$$
|
||||
# disable server.modules += ( ... ) in $conf to avoid module dups
|
||||
if awk '!/^server\.modules/{print}' $conf > $conf.$$ && mv $conf.$$ $conf; then
|
||||
:
|
||||
else
|
||||
rm $conf.$$
|
||||
fi
|
||||
fi
|
||||
|
||||
chmod 644 $conf
|
||||
if is_command lighty-enable-mod ; then
|
||||
lighty-enable-mod pihole-admin access redirect fastcgi setenv > /dev/null || true
|
||||
lighty-enable-mod pihole-admin access accesslog redirect fastcgi setenv > /dev/null || true
|
||||
else
|
||||
# Otherwise, show info about installing them
|
||||
printf " %b Warning: 'lighty-enable-mod' utility not found\\n" "${INFO}"
|
||||
@@ -2717,12 +2733,12 @@ main() {
|
||||
|
||||
restart_service pihole-FTL
|
||||
|
||||
# Update local and remote versions via updatechecker
|
||||
/opt/pihole/updatecheck.sh
|
||||
|
||||
# Download and compile the aggregated block list
|
||||
runGravity
|
||||
|
||||
# Update local and remote versions via updatechecker
|
||||
/opt/pihole/updatecheck.sh
|
||||
|
||||
if [[ "${useUpdateVars}" == false ]]; then
|
||||
displayFinalMessage "${pw}"
|
||||
fi
|
||||
|
||||
93
gravity.sh
93
gravity.sh
@@ -244,7 +244,7 @@ database_adlist_number() {
|
||||
return;
|
||||
fi
|
||||
|
||||
output=$( { printf ".timeout 30000\\nUPDATE adlist SET number = %i, invalid_domains = %i WHERE id = %i;\\n" "${num_source_lines}" "${num_invalid}" "${1}" | pihole-FTL sqlite3 "${gravityDBfile}"; } 2>&1 )
|
||||
output=$( { printf ".timeout 30000\\nUPDATE adlist SET number = %i, invalid_domains = %i WHERE id = %i;\\n" "${num_domains}" "${num_non_domains}" "${1}" | pihole-FTL sqlite3 "${gravityDBfile}"; } 2>&1 )
|
||||
status="$?"
|
||||
|
||||
if [[ "${status}" -ne 0 ]]; then
|
||||
@@ -519,12 +519,12 @@ gravity_DownloadBlocklists() {
|
||||
gravity_Blackbody=true
|
||||
}
|
||||
|
||||
# num_target_lines does increase for every correctly added domain in pareseList()
|
||||
num_target_lines=0
|
||||
num_source_lines=0
|
||||
num_invalid=0
|
||||
# num_total_imported_domains increases for each list processed
|
||||
num_total_imported_domains=0
|
||||
num_domains=0
|
||||
num_non_domains=0
|
||||
parseList() {
|
||||
local adlistID="${1}" src="${2}" target="${3}" incorrect_lines sample_incorrect_lines
|
||||
local adlistID="${1}" src="${2}" target="${3}" non_domains sample_non_domains tmp_non_domains_str false_positive
|
||||
# This sed does the following things:
|
||||
# 1. Remove all lines containing no domains
|
||||
# 2. Remove all domains containing invalid characters. Valid are: a-z, A-Z, 0-9, dot (.), minus (-), underscore (_)
|
||||
@@ -534,36 +534,65 @@ parseList() {
|
||||
sed -r "/([^\.]+\.)+[^\.]{2,}/!d;/[^a-zA-Z0-9.\_-]/d;s/\.$//;s/$/,${adlistID}/;/.$/a\\" "${src}" >> "${target}"
|
||||
|
||||
# Find lines containing no domains or with invalid characters (see above)
|
||||
# Remove duplicates and limit to 5 domains
|
||||
mapfile -t incorrect_lines <<< "$(sed -r "/([^\.]+\.)+[^\.]{2,}/d" < "${src}")"
|
||||
mapfile -t -O "${#incorrect_lines[@]}" incorrect_lines <<< "$(sed -r "/[^a-zA-Z0-9.\_-]/!d" < "${src}")"
|
||||
IFS=" " read -r -a sample_incorrect_lines <<< "$(tr ' ' '\n' <<< "${incorrect_lines[@]}" | sort -u | head -n 5| tr '\n' ' ')"
|
||||
# Remove duplicates from the list
|
||||
mapfile -t non_domains <<< "$(sed -r "/([^\.]+\.)+[^\.]{2,}/d" < "${src}")"
|
||||
mapfile -t -O "${#non_domains[@]}" non_domains <<< "$(sed -r "/[^a-zA-Z0-9.\_-]/!d" < "${src}")"
|
||||
IFS=" " read -r -a non_domains <<< "$(tr ' ' '\n' <<< "${non_domains[@]}" | sort -u | tr '\n' ' ')"
|
||||
|
||||
local num_target_lines_new num_correct_lines
|
||||
# Get number of lines in source file
|
||||
num_source_lines="$(grep -c "^" "${src}")"
|
||||
# Get the new number of lines in destination file
|
||||
num_target_lines_new="$(grep -c "^" "${target}")"
|
||||
# Number of new correctly added lines
|
||||
num_correct_lines="$(( num_target_lines_new-num_target_lines ))"
|
||||
# Update number of lines in target file
|
||||
num_target_lines="$num_target_lines_new"
|
||||
num_invalid="$(( num_source_lines-num_correct_lines ))"
|
||||
if [[ "${num_invalid}" -eq 0 ]]; then
|
||||
echo " ${INFO} Analyzed ${num_source_lines} domains"
|
||||
else
|
||||
echo " ${INFO} Analyzed ${num_source_lines} domains, ${num_invalid} domains invalid!"
|
||||
fi
|
||||
# A list of items of common local hostnames not to report as unusable
|
||||
# Some lists (i.e StevenBlack's) contain these as they are supposed to be used as HOST files
|
||||
# but flagging them as unusable causes more confusion than it's worth - so we suppress them from the output
|
||||
false_positives=(
|
||||
"localhost"
|
||||
"localhost.localdomain"
|
||||
"local"
|
||||
"broadcasthost"
|
||||
"localhost"
|
||||
"ip6-localhost"
|
||||
"ip6-loopback"
|
||||
"lo0 localhost"
|
||||
"ip6-localnet"
|
||||
"ip6-mcastprefix"
|
||||
"ip6-allnodes"
|
||||
"ip6-allrouters"
|
||||
"ip6-allhosts"
|
||||
)
|
||||
|
||||
# Display sample of invalid lines if we found some
|
||||
if [ ${#sample_incorrect_lines[@]} -ne 0 ]; then
|
||||
echo " Sample of invalid domains:"
|
||||
for each in "${sample_incorrect_lines[@]}"
|
||||
# Read the unusable lines into a string
|
||||
tmp_non_domains_str=" ${non_domains[*]} "
|
||||
for false_positive in "${false_positives[@]}"; do
|
||||
# Remove false positives from tmp_non_domains_str
|
||||
tmp_non_domains_str="${tmp_non_domains_str/ ${false_positive} / }"
|
||||
done
|
||||
# Read the string back into an array
|
||||
IFS=" " read -r -a non_domains <<< "${tmp_non_domains_str}"
|
||||
|
||||
# Get a sample of non-domain entries, limited to 5 (the list should already have been de-duplicated)
|
||||
IFS=" " read -r -a sample_non_domains <<< "$(tr ' ' '\n' <<< "${non_domains[@]}" | head -n 5 | tr '\n' ' ')"
|
||||
|
||||
local tmp_new_imported_total
|
||||
# Get the new number of domains in destination file
|
||||
tmp_new_imported_total="$(grep -c "^" "${target}")"
|
||||
# Number of imported lines for this file is the difference between the new total and the old total. (Or, the number of domains we just added.)
|
||||
num_domains="$(( tmp_new_imported_total-num_total_imported_domains ))"
|
||||
# Replace the running total with the new total.
|
||||
num_total_imported_domains="$tmp_new_imported_total"
|
||||
# Get the number of non_domains (this is the number of entries left after stripping the source of comments/duplicates/false positives/domains)
|
||||
num_non_domains="${#non_domains[@]}"
|
||||
|
||||
# If there are unusable lines, we display some information about them. This is not error or major cause for concern.
|
||||
if [[ "${num_non_domains}" -ne 0 ]]; then
|
||||
echo " ${INFO} Imported ${num_domains} domains, ignoring ${num_non_domains} non-domain entries"
|
||||
echo " Sample of non-domain entries:"
|
||||
for each in "${sample_non_domains[@]}"
|
||||
do
|
||||
echo " - ${each}"
|
||||
echo " - ${each}"
|
||||
done
|
||||
else
|
||||
echo " ${INFO} Imported ${num_domains} domains"
|
||||
fi
|
||||
}
|
||||
|
||||
compareLists() {
|
||||
local adlistID="${1}" target="${2}"
|
||||
|
||||
@@ -716,8 +745,8 @@ gravity_DownloadBlocklistFromUrl() {
|
||||
else
|
||||
echo -e " ${CROSS} List download failed: ${COL_LIGHT_RED}no cached list available${COL_NC}"
|
||||
# Manually reset these two numbers because we do not call parseList here
|
||||
num_source_lines=0
|
||||
num_invalid=0
|
||||
num_domains=0
|
||||
num_non_domains=0
|
||||
database_adlist_number "${adlistID}"
|
||||
database_adlist_status "${adlistID}" "4"
|
||||
fi
|
||||
|
||||
7
pihole
7
pihole
@@ -24,7 +24,12 @@ utilsfile="${PI_HOLE_SCRIPT_DIR}/utils.sh"
|
||||
source "${utilsfile}"
|
||||
|
||||
versionsfile="/etc/pihole/versions"
|
||||
source "${versionsfile}"
|
||||
if [ -f "${versionsfile}" ]; then
|
||||
# Only source versionsfile if the file exits
|
||||
# fixes a warning during installation where versionsfile does not exist yet
|
||||
# but gravity calls `pihole -status` and thereby sourcing the file
|
||||
source "${versionsfile}"
|
||||
fi
|
||||
|
||||
webpageFunc() {
|
||||
source "${PI_HOLE_SCRIPT_DIR}/webpage.sh"
|
||||
|
||||
@@ -1,18 +0,0 @@
|
||||
FROM fedora:35
|
||||
RUN dnf install -y git initscripts
|
||||
|
||||
ENV GITDIR /etc/.pihole
|
||||
ENV SCRIPTDIR /opt/pihole
|
||||
|
||||
RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole
|
||||
ADD . $GITDIR
|
||||
RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/
|
||||
ENV PATH /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR
|
||||
|
||||
RUN true && \
|
||||
chmod +x $SCRIPTDIR/*
|
||||
|
||||
ENV SKIP_INSTALL true
|
||||
ENV OS_CHECK_DOMAIN_NAME dev-supportedos.pi-hole.net
|
||||
|
||||
#sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \
|
||||
@@ -2,5 +2,5 @@ docker-compose == 1.29.2
|
||||
pytest == 7.2.1
|
||||
pytest-xdist == 3.1.0
|
||||
pytest-testinfra == 7.0.0
|
||||
tox == 4.2.8
|
||||
tox == 4.4.4
|
||||
|
||||
|
||||
@@ -1,8 +0,0 @@
|
||||
[tox]
|
||||
envlist = py3
|
||||
|
||||
[testenv:py3]
|
||||
allowlist_externals = docker
|
||||
deps = -rrequirements.txt
|
||||
commands = docker build -f _fedora_35.Dockerfile -t pytest_pihole:test_container ../
|
||||
pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_fedora_support.py
|
||||
Reference in New Issue
Block a user