Merge pull request #314 from pi-hole/development

Pi-hole v2.5.3

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,7 @@
+##### Expected Behaviour:
+
+
+##### Actual Behaviour:
+
+
+##### Steps to reproduce this issue:

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,11 @@
+Fixes #[issue number] .
+
+Changes proposed in this pull request:
+
+- 
+
+- 
+
+- 
+
+@pihole/gravity

README.md

@@ -24,7 +24,9 @@ Once installed, [configure your router to have **DHCP clients use the Pi as thei
 [![Pi-hole exlplained](http://i.imgur.com/qNybJDX.png)](https://vimeo.com/135965232)
 
 ## Pi-hole Projects
+- [Pi-hole on Ubuntu 14.04 on VirtualBox](http://hbalagtas.blogspot.com/2016/02/adblocking-with-pi-hole-and-ubuntu-1404.html)
 - [x86 Docker container that runs Pi-hole](https://hub.docker.com/r/diginc/pi-hole/)
+- [Splunk: Pi-hole Visualizser](https://splunkbase.splunk.com/app/3023/)
 - [Pi-hole Chrome extension](https://chrome.google.com/webstore/detail/pi-hole-list-editor/hlnoeoejkllgkjbnnnhfolapllcnaglh) ([open source](https://github.com/packtloss/pihole-extension))
 - [Go Bananas for CHiP-hole ad blocking](https://www.hackster.io/jacobsalmela/chip-hole-network-wide-ad-blocker-98e037)
 - [Sky-Hole](http://dlaa.me/blog/post/skyhole)
@@ -34,7 +36,6 @@ Once installed, [configure your router to have **DHCP clients use the Pi as thei
 - [Minibian Pi-hole](http://munkjensen.net/wiki/index.php/See_my_Pi-Hole#Minibian_Pi-hole)
 
 ## Coverage
-- [Splunk: Pi-hole Visualizser](https://splunkbase.splunk.com/app/3023/)
 - [The Defrag Show - MSDN/Channel 9](https://channel9.msdn.com/Shows/The-Defrag-Show/Defrag-Endoscope-USB-Camera-The-Final-HoloLens-Vote-Adblock-Pi-and-more?WT.mc_id=dlvr_twitter_ch9#time=20m39s)
 - [MacObserver Podcast 585](http://www.macobserver.com/tmo/podcast/macgeekgab-585)
 - [Medium: Block All Ads For $53](https://medium.com/@robleathern/block-ads-on-all-home-devices-for-53-18-a5f1ec139693#.gj1xpgr5d)

adlists.default

@@ -1,17 +1,46 @@
-https://adaway.org/hosts.txt
+## Pi-hole ad-list default sources. Updated 21/02/2016 #########################
+#                                                                              #
+#  To make changes to this file:                                               #
+#    1. run `cp /etc/pihole/adlists.default /etc/pihole/adlists.list`           #
+#    2. run `nano /etc/pihole/adlists.list`                                    #
+#    3. Uncomment or comment any of the below lists                            #
+#                                                                              #
+#  Know of any other lists? Feel free to let us know about them, or add them   # 
+#  to this file!                                                               #
+################################################################################
+
+# The below list amalgamates several lists we used previously. 
+# See `https://github.com/StevenBlack/hosts` for details
+https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts
+
+# Other lists we consider safe:
 http://adblock.gjtech.net/?format=unix-hosts
-http://hosts-file.net/ad_servers.txt
-http://www.malwaredomainlist.com/hostslist/hosts.txt
-http://pgl.yoyo.org/adservers/serverlist.php?
-http://someonewhocares.org/hosts/hosts
-http://winhelp2002.mvps.org/hosts.txt
 http://mirror1.malwaredomains.com/files/justdomains
-http://securemecca.com/Downloads/hosts.txt
-https://hosts.neocities.org/
 http://sysctl.org/cameleon/hosts
 https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist
 https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt
 https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt
-http://optimate.dl.sourceforge.net/project/adzhosts/HOSTS.txt
+
+# hosts-file.net list. Updated frequently, but has been known to block legitimate sites.
+http://hosts-file.net/ad_servers.txt
+
+# Mahakala list. Has been known to block legitimage domains including the entire .com range.
+#http://adblock.mahakala.is/
+
+# ADZHOSTS list. Has been known to block legitimate domains
+#http://optimate.dl.sourceforge.net/project/adzhosts/HOSTS.txt
+
+# Windows 10 telemetry list - warning this one may block windows update
+#https://raw.githubusercontent.com/crazy-max/HostsWindowsBlocker/master/hosts.txt
+
+# Securemecca.com list - Also blocks "adult" sites (pornography/gambling etc)
+#http://securemecca.com/Downloads/hosts.txt
+
+# Quidsup's tracker list
 https://raw.githubusercontent.com/quidsup/notrack/master/trackers.txt
 
+
+# Untested Lists:
+#https://raw.githubusercontent.com/reek/anti-adblock-killer/master/anti-adblock-killer-filters.txt
+#http://spam404bl.com/spam404scamlist.txt
+#http://malwaredomains.lehigh.edu/files/domains.txt

advanced/Scripts/chronometer.sh

@@ -88,6 +88,7 @@ function normalChrono(){
 		echo "        $(ifconfig eth0 | awk '/inet addr/ {print $2}' | cut -d':' -f2)"
 		echo ""
 		uptime | cut -d' ' -f11-
+		uptime -p
 		echo "-------------------------------"
 		# Uncomment to continually read the log file and display the current domain being blocked
 		#tail -f /var/log/pihole.log | awk '/\/etc\/pihole\/gravity.list/ {if ($7 != "address" && $7 != "name" && $7 != "/etc/pihole/gravity.list") print $7; else;}'

advanced/Scripts/setupLCD.sh

@@ -0,0 +1,89 @@
+#!/usr/bin/env bash
+# Pi-hole: A black hole for Internet advertisements
+# (c) 2015 by Jacob Salmela
+# Network-wide ad blocking via your Raspberry Pi
+# http://pi-hole.net
+# Automatically configures the Pi to use the 2.8 LCD screen to display stats on it (also works over ssh)
+#
+# Pi-hole is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+
+############ FUNCTIONS ###########
+# Run this script as root or under sudo
+echo ":::"
+if [[ $EUID -eq 0 ]];then
+	echo "::: You are root."
+else
+	echo "::: sudo will be used."
+  # Check if it is actually installed
+  # If it isn't, exit because the install cannot complete
+  if [[ $(dpkg-query -s sudo) ]];then
+		export SUDO="sudo"
+  else
+		echo "::: Please install sudo or run this script as root."
+    exit 1
+  fi
+fi
+
+# Borrowed from adafruit-pitft-helper < borrowed from raspi-config
+# https://github.com/adafruit/Adafruit-PiTFT-Helper/blob/master/adafruit-pitft-helper#L324-L334
+getInitSys() {
+  if command -v systemctl > /dev/null && systemctl | grep -q '\-\.mount'; then
+    SYSTEMD=1
+  elif [ -f /etc/init.d/cron ] && [ ! -h /etc/init.d/cron ]; then
+    SYSTEMD=0
+  else
+    echo "Unrecognised init system"
+    return 1
+  fi
+}
+
+# Borrowed from adafruit-pitft-helper:
+# https://github.com/adafruit/Adafruit-PiTFT-Helper/blob/master/adafruit-pitft-helper#L274-L285
+autoLoginPiToConsole() {
+  if [ -e /etc/init.d/lightdm ]; then
+    if [ $SYSTEMD -eq 1 ]; then
+      $SUDO systemctl set-default multi-user.target
+      $SUDO ln -fs /etc/systemd/system/autologin@.service /etc/systemd/system/getty.target.wants/getty@tty1.service
+    else
+      $SUDO update-rc.d lightdm disable 2
+      $SUDO sed /etc/inittab -i -e "s/1:2345:respawn:\/sbin\/getty --noclear 38400 tty1/1:2345:respawn:\/bin\/login -f pi tty1 <\/dev\/tty1 >\/dev\/tty1 2>&1/"
+      fi
+  fi
+}
+
+######### SCRIPT ###########
+# Set pi to log in automatically
+getInitSys
+autoLoginPiToConsole
+
+# Set chronomter to run automatically when pi logs in
+echo /usr/local/bin/chronometer.sh >> /home/pi/.bashrc
+# OR
+#$SUDO echo /usr/local/bin/chronometer.sh >> /etc/profile
+
+# Set up the LCD screen based on Adafruits instuctions:
+# https://learn.adafruit.com/adafruit-pitft-28-inch-resistive-touchscreen-display-raspberry-pi/easy-install
+curl -SLs https://apt.adafruit.com/add-pin | $SUDO bash
+$SUDO apt-get -y install raspberrypi-bootloader
+$SUDO apt-get -y install adafruit-pitft-helper
+$SUDO adafruit-pitft-helper -t 28r
+
+# Download the cmdline.txt file that prevents the screen from going blank after a period of time
+$SUDO mv /boot/cmdline.txt /boot/cmdline.orig
+$SUDO curl -o /boot/cmdline.txt https://raw.githubusercontent.com/pi-hole/pi-hole/master/advanced/cmdline.txt
+
+# Back up the original file and download the new one
+$SUDO mv /etc/default/console-setup /etc/default/console-setup.orig
+$SUDO curl -o /etc/default/console-setup https://raw.githubusercontent.com/pi-hole/pi-hole/master/advanced/console-setup
+
+# Instantly apply the font change to the LCD screen
+$SUDO setupcon
+
+$SUDO reboot
+
+# Start showing the stats on the screen by running the command on another tty:
+# http://unix.stackexchange.com/questions/170063/start-a-process-on-a-different-tty
+#setsid sh -c 'exec /usr/local/bin/chronometer.sh <> /dev/tty1 >&0 2>&1'

advanced/cmdline.txt

@@ -0,0 +1 @@
+dwc_otg.lpm_enable=0 console=ttyAMA0,115200 console=tty1 root=/dev/mmcblk0p2 rootfstype=ext4 elevator=deadline fsck.repair=yes rootwait fbcon=map:10 fbcon=font:VGA8x8 consoleblank=0

advanced/console-setup

@@ -0,0 +1,17 @@
+# CONFIGURATION FILE FOR SETUPCON
+
+# Consult the console-setup(5) manual page.
+
+ACTIVE_CONSOLES="/dev/tty[1-6]"
+
+CHARMAP="UTF-8"
+
+# For best results with the Adafruit 2.8 LCD and Pi-hole's chronometer
+CODESET="guess"
+FONTFACE="Terminus"
+FONTSIZE="10x20"
+
+VIDEOMODE=
+
+# The following is an example how to use a braille font
+# FONT='lat9w-08.psf.gz brl-8x8.psf'

automated install/basic-install.sh

@@ -117,15 +117,20 @@ welcomeDialogs() {
 
 
 verifyFreeDiskSpace() {
-    # 25MB is the minimum space needed (20MB install + 5MB one day of logs.)
-    requiredFreeBytes=25600
-
-    existingFreeBytes=`df -lkP / | awk '{print $4}' | tail -1`
-
-    if [[ $existingFreeBytes -lt $requiredFreeBytes ]]; then
-        whiptail --msgbox --backtitle "Insufficient Disk Space" --title "Insufficient Disk Space" "\nYour system appears to be low on disk space. pi-hole recomends a minimum of $requiredFreeBytes Bytes.\nYou only have $existingFreeBytes Free.\n\nIf this is a new install you may need to expand your disk.\n\nTry running:\n    'sudo raspi-config'\nChoose the 'expand file system option'\n\nAfter rebooting, run this installation again.\n\ncurl -L install.pi-hole.net | bash\n" $r $c
-        exit 1
-    fi
+	# 25MB is the minimum space needed (20MB install + 5MB one day of logs.)
+	requiredFreeBytes=51200
+	
+	existingFreeBytes=`df -lk / 2>&1 | awk '{print $4}' | head -2 | tail -1`    	
+	if ! [[ "$existingFreeBytes" =~ ^([0-9])+$ ]]; then       
+		existingFreeBytes=`df -lk /dev 2>&1 | awk '{print $4}' | head -2 | tail -1`		
+	fi
+	
+	if [[ $existingFreeBytes -lt $requiredFreeBytes ]]; then
+		whiptail --msgbox --backtitle "Insufficient Disk Space" --title "Insufficient Disk Space" "\nYour system appears to be low on disk space. pi-hole recomends a minimum of $requiredFreeBytes Bytes.\nYou only have $existingFreeBytes Free.\n\nIf this is a new install you may need to expand your disk.\n\nTry running:\n    'sudo raspi-config'\nChoose the 'expand file system option'\n\nAfter rebooting, run this installation again.\n\ncurl -L install.pi-hole.net | bash\n" $r $c
+		echo "$existingFreeBytes is less than $requiredFreeBytes"
+		echo "Insufficient free space, exiting..."
+		exit 1
+	fi
 }
 
 
@@ -299,10 +304,31 @@ setStaticIPv4() {
 	fi
 }
 
+function valid_ip()
+{
+	local  ip=$1
+	local  stat=1
+	
+	if [[ $ip =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then
+		OIFS=$IFS
+		IFS='.'
+		ip=($ip)
+		IFS=$OIFS
+		[[ ${ip[0]} -le 255 && ${ip[1]} -le 255 \
+		&& ${ip[2]} -le 255 && ${ip[3]} -le 255 ]]
+		stat=$?
+	fi
+	return $stat
+}
+
 setDNS(){
-	DNSChoseCmd=(whiptail --separate-output --radiolist "Select Upstream DNS Provider" $r $c 2)
+	DNSChoseCmd=(whiptail --separate-output --radiolist "Select Upstream DNS Provider. To use your own, select Custom." $r $c 6)
 	DNSChooseOptions=(Google "" on
-					  OpenDNS "" off)
+					  OpenDNS "" off
+					  Level3 "" off
+					  Norton "" off
+					  Comodo "" off
+					  Custom "" off)
 	DNSchoices=$("${DNSChoseCmd[@]}" "${DNSChooseOptions[@]}" 2>&1 >/dev/tty)
 	if [[ $? = 0 ]];then
 		case $DNSchoices in
@@ -316,6 +342,79 @@ setDNS(){
 				piholeDNS1="208.67.222.222"
 				piholeDNS2="208.67.220.220"
 				;;
+			Level3)
+				echo "::: Using Level3 servers."
+				piholeDNS1="4.2.2.1"
+				piholeDNS2="4.2.2.2"
+				;;
+			Norton)
+				echo "::: Using Norton ConnectSafe servers."
+				piholeDNS1="199.85.126.10"
+				piholeDNS2="199.85.127.10"
+				;;
+			Comodo)
+				echo "::: Using Comodo Secure servers."
+				piholeDNS1="8.26.56.26"
+				piholeDNS2="8.20.247.20"
+				;;
+			Custom)
+				until [[ $DNSSettingsCorrect = True ]]
+				do
+					
+					strInvalid="Invalid"
+				
+					if [ ! $piholeDNS1 ]; then
+						if [ ! $piholeDNS2 ]; then
+							prePopulate=""
+						else
+							prePopulate=", $piholeDNS2"
+						fi
+					elif  [ $piholeDNS1 ] && [ ! $piholeDNS2 ]; then
+						prePopulate="$piholeDNS1"
+					elif [ $piholeDNS1 ] && [ $piholeDNS2 ]; then
+						prePopulate="$piholeDNS1, $piholeDNS2"
+					fi
+					
+					piholeDNS=$(whiptail --backtitle "Specify Upstream DNS Provider(s)"  --inputbox "Enter your desired upstream DNS provider(s), seperated by a comma.\n\nFor example '8.8.8.8, 8.8.4.4'" $r $c "$prePopulate" 3>&1 1>&2 2>&3)
+					if [[ $? = 0 ]];then
+						piholeDNS1=$(echo $piholeDNS | sed 's/[, \t]\+/,/g' | awk -F, '{print$1}')
+						piholeDNS2=$(echo $piholeDNS | sed 's/[, \t]\+/,/g' | awk -F, '{print$2}')
+						
+						if ! valid_ip $piholeDNS1 || [ ! $piholeDNS1 ]; then
+							piholeDNS1=$strInvalid
+						fi
+												
+						if ! valid_ip $piholeDNS2 && [ $piholeDNS2 ]; then
+							piholeDNS2=$strInvalid
+						fi
+						
+					else
+						echo "::: Cancel selected, exiting...."
+						exit 1
+					fi
+					
+					if [[ $piholeDNS1 == $strInvalid ]] || [[ $piholeDNS2 == $strInvalid ]]; then
+						whiptail --msgbox --backtitle "Invalid IP" --title "Invalid IP" "One or both entered IP addresses were invalid. Please try again.\n\n    DNS Server 1:   $piholeDNS1\n    DNS Server 2:   $piholeDNS2" $r $c						
+						
+						if [[ $piholeDNS1 == $strInvalid ]]; then
+							piholeDNS1=""
+						fi
+						
+						if [[ $piholeDNS2 == $strInvalid ]]; then
+							piholeDNS2=""
+						fi
+						
+						DNSSettingsCorrect=False
+					else					
+						if (whiptail --backtitle "Specify Upstream DNS Provider(s)" --title "Upstream DNS Provider(s)" --yesno "Are these settings correct?\n    DNS Server 1:   $piholeDNS1\n    DNS Server 2:   $piholeDNS2" $r $c) then
+								DNSSettingsCorrect=True
+						else
+							# If the settings are wrong, the loop continues
+							DNSSettingsCorrect=False
+						fi
+					fi
+				done
+				;;
 		esac
 	else
 		echo "::: Cancel selected. Exiting..."
@@ -356,8 +455,16 @@ versionCheckDNSmasq(){
   $SUDO cp $newFileToInstall $newFileFinalLocation
   echo " done."
   $SUDO sed -i "s/@INT@/$piholeInterface/" $newFileFinalLocation
-  $SUDO sed -i "s/@DNS1@/$piholeDNS1/" $newFileFinalLocation
-  $SUDO sed -i "s/@DNS2@/$piholeDNS2/" $newFileFinalLocation
+  if [[ "$piholeDNS1" != "" ]]; then
+    $SUDO sed -i "s/@DNS1@/$piholeDNS1/" $newFileFinalLocation
+  else
+    $SUDO sed -i '/^server=@DNS1@/d' $newFileFinalLocation
+  fi
+  if [[ "$piholeDNS2" != "" ]]; then
+    $SUDO sed -i "s/@DNS2@/$piholeDNS2/" $newFileFinalLocation
+  else
+    $SUDO sed -i '/^server=@DNS2@/d' $newFileFinalLocation
+  fi
 }
 
 installScripts() {
@@ -387,7 +494,7 @@ stopServices() {
 	# Stop dnsmasq and lighttpd
 	$SUDO echo ":::"
 	$SUDO echo -n "::: Stopping services..."
-	$SUDO service dnsmasq stop & spinner $! || true
+	#$SUDO service dnsmasq stop & spinner $! || true
 	$SUDO service lighttpd stop & spinner $! || true
 	$SUDO echo " done."
 }
@@ -540,21 +647,31 @@ runGravity() {
 	/usr/local/bin/gravity.sh
 }
 
+setUser(){
+	# Check if user pihole exists and create if not
+	echo "::: Checking if user 'pihole' exists..."
+	if id -u pihole > /dev/null 2>&1; then
+		echo "::: User 'pihole' already exists"
+	else
+        echo "::: User 'pihole' doesn't exist.  Creating..."
+		$SUDO useradd -r -s /usr/sbin/nologin pihole
+	fi
+}
 
 installPihole() {
 	# Install base files and web interface
 	checkForDependencies # done
 	stopServices
+	setUser
 	$SUDO mkdir -p /etc/pihole/
 	$SUDO chown www-data:www-data /var/www/html
 	$SUDO chmod 775 /var/www/html
-	$SUDO usermod -a -G www-data pi
+	$SUDO usermod -a -G www-data pihole
 	$SUDO lighty-enable-mod fastcgi fastcgi-php > /dev/null
 
 	getGitFiles
 	installScripts
 	installConfigs
-	#installWebAdmin
 	CreateLogFile
 	installPiholeWeb
 	installCron
@@ -601,7 +718,7 @@ displayFinalMessage
 
 echo -n "::: Restarting services..."
 # Start services
-$SUDO service dnsmasq start
+$SUDO service dnsmasq restart
 $SUDO service lighttpd start
 echo " done."
 

gravity.sh

@@ -100,7 +100,12 @@ function gravity_collapse() {
 		echo -n "::: Custom adList file detected. Reading..."
 		sources=()
 		while read -a line; do
-			sources+=($line)
+			#Do not read commented out or blank lines
+			if [[ $line = \#* ]] || [[ ! $line ]]; then
+				echo "" > /dev/null
+			else
+				sources+=($line)
+			fi
 		done < $adListFile
 		echo " done!"	
 	else
@@ -108,7 +113,12 @@ function gravity_collapse() {
 		echo -n "::: No custom adlist file detected, reading from default file..."
 				sources=()
 		while read -a line; do
-			sources+=($line)
+			#Do not read commented out or blank lines
+			if [[ $line = \#* ]] || [[ ! $line ]]; then
+				echo "" > /dev/null
+			else
+				sources+=($line)
+			fi
 		done < $adListDefault
 		echo " done!"	
 	fi	
@@ -318,11 +328,19 @@ function gravity_advanced() {
 }
 
 function gravity_reload() {
+	#Clear no longer needed files...
+	echo ":::"
+	echo -n "::: Cleaning up un-needed files..."
+	$SUDO rm /etc/pihole/pihole.*
+	echo " done!"
+	
 	# Reload hosts file
 	echo ":::"
 	echo -n "::: Refresh lists in dnsmasq..."
 	dnsmasqPid=$(pidof dnsmasq)
 
+    find "$piholeDir" -type f -exec $SUDO chmod 666 {} \; & spinner $!
+
 	if [[ $dnsmasqPid ]]; then
 		# service already running - reload config
 		$SUDO kill -HUP $dnsmasqPid & spinner $!