adblock: small changes

* add whocares.org as additional source
* add adblock statistics (requires query log)
* cosmetics

Signed-off-by: Dirk Brenken <dirk@brenken.org>

diff --git a/net/adblock/Makefile b/net/adblock/Makefile
index 36f394e6750f734a86712e3facc2c1b0d36ef87d..db029039411da4c7c9294acb62d5f8ad16b1f8f5 100644 (file)
--- a/net/adblock/Makefile
+++ b/net/adblock/Makefile
@@ -7,7 +7,7 @@
 include $(TOPDIR)/rules.mk
 
 PKG_NAME:=adblock
-PKG_VERSION:=0.40.0
+PKG_VERSION:=0.40.1
 PKG_RELEASE:=1
 PKG_LICENSE:=GPL-3.0+
 PKG_MAINTAINER:=Dirk Brenken <dirk@brenken.org>
@@ -28,17 +28,18 @@ When the dns server on your router receives dns requests, we will sort out queri
 resource records of ad servers and return the local ip address of your router and the internal web server
 delivers a transparent pixel instead.
 The script supports the following domain blacklist sites:
-  http://pgl.yoyo.org/adservers
-  http://malwaredomains.com
-  https://zeustracker.abuse.ch
+  https://adaway.org
+  https://disconnect.me
+  http://dshield.org
   https://feodotracker.abuse.ch
+  http://malwaredomains.com
   https://palevotracker.abuse.ch
-  http://dshield.org
   http://www.shallalist.de
   http://www.spam404.com
+  http://someonewhocares.org
   http://winhelp2002.mvps.org
-  https://adaway.org
-  https://disconnect.me
+  http://pgl.yoyo.org/adservers
+  https://zeustracker.abuse.ch
 
 Please see README.md in /etc/adblock for further information.
 

diff --git a/net/adblock/files/README.md b/net/adblock/files/README.md
index 0f7d9f6e341f558ce1656bf6f01af2a45cd50888..301e0f6a4ddfa1ff2f589c4c53ee26757eb7c2d2 100644 (file)
--- a/net/adblock/files/README.md
+++ b/net/adblock/files/README.md
@@ -5,33 +5,34 @@ A lot of people already use adblocker plugins within their desktop browsers,
 but what if you are using your (smart) phone, tablet, watch or any other wlan gadget...  
 ...getting rid of annoying ads, trackers and other abuse sites (like facebook ;-) is simple: block them with your router.  
 
-When the dns server on your router receives dns requests, you’ll sort out queries that ask for the [A] resource records of ad servers  
-and return the local ip address of your router and the internal web server delivers a transparent pixel instead.
+When the dns server on your router receives dns requests, you’ll sort out queries that ask for the [A] resource records of ad servers and return the local ip address of your router and the internal web server delivers a transparent pixel instead.
 
 ## Main Features
 * support of the following domain blocklist sources (free for private usage, for commercial use please check their individual licenses):
-    * [pgl.yoyo.org](http://pgl.yoyo.org/adservers)
-    * => weekly updates, approx. 2.500 entries (enabled by default)
-    * [malwaredomains.com](http://malwaredomains.com)
-    * => daily updates, approx. 16.000 entries
-    * [zeustracker.abuse.ch](https://zeustracker.abuse.ch)
-    * => daily updates, approx. 440 entries
+    * [adaway.org](https://adaway.org)
+    * => infrequent updates, approx. 400 entries
+    * [disconnect.me](https://disconnect.me)
+    * => numerous updates on the same day, approx. 6.500 entries
+    * [dshield.org](http://dshield.org)
+    * => daily updates, approx. 4.500 entries
     * [feodotracker.abuse.ch](https://feodotracker.abuse.ch)
     * => daily updates, approx. 0-10 entries
+    * [malwaredomains.com](http://malwaredomains.com)
+    * => daily updates, approx. 16.000 entries
     * [palevotracker.abuse.ch](https://palevotracker.abuse.ch)
     * => daily updates, approx. 15 entries
-    * [dshield.org](http://dshield.org)
-    * => daily updates, approx. 4.500 entries
     * [shallalist.de](http://www.shallalist.de) (categories "adv" "costtraps" "spyware" "tracker" "warez" enabled by default)
     * => daily updates, approx. 32.000 entries (a short description of all shallalist categories can be found [online](http://www.shallalist.de/categories.html))
     * [spam404.com](http://www.spam404.com)
     * => infrequent updates, approx. 5.000 entries
+    * [whocares.org](http://someonewhocares.org)
+    * => weekly updates, approx. 12.000 entries
     * [winhelp2002.mvps.org](http://winhelp2002.mvps.org)
     * => infrequent updates, approx. 15.000 entries
-    * [adaway.org](https://adaway.org)
-    * => infrequent updates, approx. 400 entries
-    * [disconnect.me](https://disconnect.me)
-    * => numerous updates on the same day, approx. 6.500 entries
+    * [yoyo.org](http://pgl.yoyo.org/adservers)
+    * => weekly updates, approx. 2.500 entries (enabled by default)
+    * [zeustracker.abuse.ch](https://zeustracker.abuse.ch)
+    * => daily updates, approx. 440 entries
 * each blocklist source will be updated and processed separately
 * timestamp check to download and process only updated blocklists
 * overall duplicate removal in separate blocklists (will be automatically disabled on low memory systems)
@@ -40,10 +41,11 @@ and return the local ip address of your router and the internal web server deliv
 * use of dynamic adblock network interface
 * use of dynamic uhttpd instance as pixel server
 * use of quality checks after adblocklist updates to ensure a reliable dnsmasq service
-* optional features (disabled by default): 
+* optional features (disabled by default):
     * powerful adblock list backup/restore handling
-    * adblock updates only on pre-defined wan interfaces (useful for (mobile) multiwan setups)
+    * wan interface check, useful for rc.local based autostart and (mobile) multiwan setups to update only on pre-defined wan interfaces
     * domain query logging as a background service to easily identify free and already blocked domains (see example output below)
+    * adblock statistics (req. query logging)
     * ntp time sync
     * status & error logging to separate file (req. ntp time sync)
 
@@ -78,44 +80,16 @@ and return the local ip address of your router and the internal web server deliv
 
 ## Examples
 
-  log of a full adblock run:  
+  adblock log file excerpt:  
     
-    adblock[1586] info : domain adblock processing started (0.40.0, r47670, 17.12.2015 10:50:09)  
-    adblock[1586] info : wan update check will be disabled  
-    adblock[1586] info : get ntp time sync (192.168.2.254), after 0 loops  
-    adblock[1586] info : created new dynamic/volatile network interface (adblock, 192.168.2.1)  
-    adblock[1586] info : created new dynamic/volatile uhttpd instance (adblock, 192.168.2.1)  
-    adblock[1586] info : shallalist (pre-)processing started ...  
-    adblock[1586] info : source download finished (http://pgl.yoyo.org/adservers/serverlist.php?hostformat=one-line&showintro=0&mimetype=plaintext, 2432 entries)  
-    adblock[1586] info : domain merging finished (yoyo)  
-    adblock[1586] info : source download finished (http://mirror1.malwaredomains.com/files/justdomains, 17368 entries)  
-    adblock[1586] info : domain merging finished (malware)  
-    adblock[1586] info : source download finished (https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist, 440 entries)  
-    adblock[1586] info : domain merging finished (zeus)  
-    adblock[1586] info : no online timestamp received, current date will be used (feodo)  
-    adblock[1586] info : source download finished (https://feodotracker.abuse.ch/blocklist/?download=domainblocklist, 0 entries)  
-    adblock[1586] info : empty domain input received (feodo)  
-    adblock[1586] info : no online timestamp received, current date will be used (palevo)  
-    adblock[1586] info : source download finished (https://palevotracker.abuse.ch/blocklists.php?download=domainblocklist, 16 entries)  
-    adblock[1586] info : domain merging finished (palevo)  
-    adblock[1586] info : source download finished (http://www.dshield.org/feeds/suspiciousdomains_Low.txt, 4542 entries)  
-    adblock[1586] info : domain merging finished (dshield)  
-    adblock[1586] info : source download finished (http://spam404bl.com/spam404scamlist.txt, 5193 entries)  
-    adblock[1586] info : domain merging finished (spam404)  
-    adblock[1586] info : source download finished (http://winhelp2002.mvps.org/hosts.txt, 13635 entries)  
-    adblock[1586] info : domain merging finished (winhelp)  
-    adblock[1586] info : source download finished (https://adaway.org/hosts.txt, 410 entries)  
-    adblock[1586] info : domain merging finished (adaway)  
-    adblock[1586] info : source download finished (https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt, 6343 entries)  
-    adblock[1586] info : domain merging finished (disconnect)  
-    adblock[1586] info : source download finished (file:////tmp/tmp.FIhIBh/shallalist.txt, 32458 entries)  
-    adblock[1586] info : domain merging finished (shalla)  
-    adblock[1586] info : source download finished (file:///etc/adblock/adblock.blacklist, 1 entries)  
-    adblock[1586] info : domain merging finished (blacklist)  
-    adblock[1586] info : remove duplicates in separate adblocklists ...  
-    adblock[1586] info : adblocklists with overall 71552 domains loaded, new backups generated  
-    adblock[1586] info : new domain query log background process started (pid: 2416)  
-    adblock[1586] info : domain adblock processing finished (0.40.0, r47670, 17.12.2015 10:52:47)  
+    adblock[11780] info : domain adblock processing started (0.40.1, r47929, 20.12.2015 21:59:56)  
+    adblock[11780] info : backup/restore will be disabled  
+    adblock[11780] info : wan update check will be disabled  
+    adblock[11780] info : get ntp time sync (192.168.254.254), after 0 loops  
+    adblock[11780] info : source doesn't change, no update required (whocares)  
+    adblock[11780] info : adblocklists with overall 11619 domains loaded, backups disabled  
+    adblock[11780] info : adblock statistics for query date 20151220 (total: 2495, blocked: 154)  
+    adblock[11780] info : domain adblock processing finished (0.40.1, r47929, 20.12.2015 22:00:03)  
     
 
   domain blocklist for dnsmasq (disconnect.me after overall duplicate removal):  

diff --git a/net/adblock/files/adblock-helper.sh b/net/adblock/files/adblock-helper.sh
index c3cc52d2c6315931b03b668c3f1187ca66d9113c..f13b9acceab8fb0f6bab3806d6b7af1db276b53c 100644 (file)
--- a/net/adblock/files/adblock-helper.sh
+++ b/net/adblock/files/adblock-helper.sh
@@ -166,7 +166,7 @@ f_envparse()
 
     # set adblock source ruleset definitions
     #
-    rset_start="sed -r 's/[[:space:]]|[\[!#/:;_].*|[0-9\.]*localhost//g; s/[\^#/:;_\.\t ]*$//g'"
+    rset_start="sed -r 's/[[:space:]]|[\[!#/:;_].*|[0-9\.]*localhost.*//g; s/[\^#/:;_\.\t ]*$//g'"
     rset_end="sed '/^[#/:;_\s]*$/d'"
     rset_adaway="${rset_start} | sed 's/\([0-9]\{1,3\}\.\)\{3\}[0-1]\{1,1\}//g' | ${rset_end}"
     rset_blacklist="${rset_start} | ${rset_end}"
@@ -177,6 +177,7 @@ f_envparse()
     rset_palevo="${rset_start} | ${rset_end}"
     rset_shalla="${rset_start} | sed 's/\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}$//g' | ${rset_end}"
     rset_spam404="${rset_start} | sed 's/^\|\|//g' | ${rset_end}"
+    rset_whocares="${rset_start} | sed 's/\([0-9]\{1,3\}\.\)\{3\}[0-1]\{1,1\}//g' | ${rset_end}"
     rset_winhelp="${rset_start} | sed 's/\([0-9]\{1,3\}\.\)\{3\}[0-1]\{1,1\}//g' | ${rset_end}"
     rset_yoyo="${rset_start} | sed 's/,/\n/g' | ${rset_end}"
     rset_zeus="${rset_start} | ${rset_end}"
@@ -538,6 +539,12 @@ f_remove()
     if [ "${query_ok}" = "true" ] && [ "${ntp_ok}" = "true" ]
     then
         query_date="$(date "+%Y%m%d")"
+        if [ -s "${adb_querypid}" ] && [ -f "${adb_queryfile}.${query_date}" ]
+        then
+            query_total="$(grep -F "query[A]" "${adb_queryfile}.${query_date}" 2>/dev/null | wc -l)"
+            query_blocked="$(grep -Fv "query[A]" "${adb_queryfile}.${query_date}" 2>/dev/null | wc -l)"
+            f_log "adblock statistics for query date ${query_date} (total: ${query_total}, blocked: ${query_blocked})"
+        fi
         if [ -s "${adb_querypid}" ] && [ ! -f "${adb_queryfile}.${query_date}" ]
         then
             kill -9 "$(cat "${adb_querypid}")" >/dev/null 2>&1
@@ -551,7 +558,7 @@ f_remove()
             then
                 f_log "remove old domain query log background process (pid: $(cat "${adb_querypid}")) and do logfile housekeeping"
             else
-                f_log "error during domain query removal/housekeeping (pid: $(cat "${adb_querypid}"))"
+                f_log "error during domain query removal/housekeeping (pid: $(cat "${adb_querypid}"))" "${rc}"
             fi
             > "${adb_querypid}"
         fi
@@ -564,7 +571,7 @@ f_remove()
                 sleep 1
                 f_log "new domain query log background process started (pid: $(cat "${adb_querypid}"))"
             else
-                f_log "error during domain query start"
+                f_log "error during domain query start" "${rc}"
             fi
         fi
     fi

diff --git a/net/adblock/files/adblock-update.sh b/net/adblock/files/adblock-update.sh
index bbe197ad77f8d60b63bb4be0d6c8465256d8a403..9435f6c8fa0a6cc38719cd396b832ce5c2aac3ea 100755 (executable)
--- a/net/adblock/files/adblock-update.sh
+++ b/net/adblock/files/adblock-update.sh
@@ -25,7 +25,7 @@
 
 # set script version
 #
-adb_version="0.40.0"
+adb_version="0.40.1"
 
 # get current pid, script directory and openwrt version
 #

diff --git a/net/adblock/files/adblock.conf b/net/adblock/files/adblock.conf
index e8bb721d185119d11a5d177fda19532190f34579..8dff6e30d767d411d40314ff62142629dc2c6575 100644 (file)
--- a/net/adblock/files/adblock.conf
+++ b/net/adblock/files/adblock.conf
@@ -28,51 +28,55 @@ config service "querylog"
        option adb_queryfile "/tmp/adb_query.log"
        option adb_queryhistory "1"
 
-config source "yoyo"
-       option enabled "1"
-       option adb_src_yoyo "http://pgl.yoyo.org/adservers/serverlist.php?hostformat=one-line&showintro=0&mimetype=plaintext&ruleset=rset_yoyo"
+config source "adaway"
+       option enabled "0"
+       option adb_src_adaway "https://adaway.org/hosts.txt&ruleset=rset_adaway"
 
-config source "malware"
+config source "disconnect"
        option enabled "0"
-       option adb_src_malware "http://mirror1.malwaredomains.com/files/justdomains&ruleset=rset_malware"
+       option adb_src_disconnect "https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt&ruleset=rset_disconnect"
 
-config source "zeus"
+config source "dshield"
        option enabled "0"
-       option adb_src_zeus "https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist&ruleset=rset_zeus"
+       option adb_src_dshield "http://www.dshield.org/feeds/suspiciousdomains_Low.txt&ruleset=rset_dshield"
 
 config source "feodo"
        option enabled "0"
        option adb_src_feodo "https://feodotracker.abuse.ch/blocklist/?download=domainblocklist&ruleset=rset_feodo"
 
+config source "malware"
+       option enabled "0"
+       option adb_src_malware "http://mirror1.malwaredomains.com/files/justdomains&ruleset=rset_malware"
+
 config source "palevo"
        option enabled "0"
        option adb_src_palevo "https://palevotracker.abuse.ch/blocklists.php?download=domainblocklist&ruleset=rset_palevo"
 
-config source "dshield"
+config source "shalla"
        option enabled "0"
-       option adb_src_dshield "http://www.dshield.org/feeds/suspiciousdomains_Low.txt&ruleset=rset_dshield"
+       option adb_arc_shalla "http://www.shallalist.de/Downloads/shallalist.tar.gz"
+       list adb_catlist "adv"
+       list adb_catlist "costtraps"
+       list adb_catlist "spyware"
+       list adb_catlist "tracker"
+       list adb_catlist "warez"
 
 config source "spam404"
        option enabled "0"
        option adb_src_spam404 "http://spam404bl.com/spam404scamlist.txt&ruleset=rset_spam404"
 
-config source "winhelp"
+config source "whocares"
        option enabled "0"
-       option adb_src_winhelp "http://winhelp2002.mvps.org/hosts.txt&ruleset=rset_winhelp"
+       option adb_src_whocares "http://someonewhocares.org/hosts/hosts&ruleset=rset_whocares"
 
-config source "adaway"
+config source "winhelp"
        option enabled "0"
-       option adb_src_adaway "https://adaway.org/hosts.txt&ruleset=rset_adaway"
+       option adb_src_winhelp "http://winhelp2002.mvps.org/hosts.txt&ruleset=rset_winhelp"
 
-config source "disconnect"
-       option enabled "0"
-       option adb_src_disconnect "https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt&ruleset=rset_disconnect"
+config source "yoyo"
+       option enabled "1"
+       option adb_src_yoyo "http://pgl.yoyo.org/adservers/serverlist.php?hostformat=one-line&showintro=0&mimetype=plaintext&ruleset=rset_yoyo"
 
-config source "shalla"
+config source "zeus"
        option enabled "0"
-       option adb_arc_shalla "http://www.shallalist.de/Downloads/shallalist.tar.gz"
-       list adb_catlist "adv"
-       list adb_catlist "costtraps"
-       list adb_catlist "spyware"
-       list adb_catlist "tracker"
-       list adb_catlist "warez"
+       option adb_src_zeus "https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist&ruleset=rset_zeus"

diff --git a/net/adblock/files/samples/adblock.conf.sample b/net/adblock/files/samples/adblock.conf.sample
index be0bba6d0513e724df84a5c795175d43b20b08cc..7f7a1d7543dc2fa2a918b6af6404e5e360751106 100644 (file)
--- a/net/adblock/files/samples/adblock.conf.sample
+++ b/net/adblock/files/samples/adblock.conf.sample
@@ -63,51 +63,55 @@ config service "querylog"
 # please do not change the urls listed below, 
 # enable/disable sources as needed
 # for shallalist you can also enable/disable different ad categories
-config source "yoyo"
-       option enabled "1"
-       option adb_src_yoyo "http://pgl.yoyo.org/adservers/serverlist.php?hostformat=one-line&showintro=0&mimetype=plaintext&ruleset=rset_yoyo"
+config source "adaway"
+       option enabled "0"
+       option adb_src_adaway "https://adaway.org/hosts.txt&ruleset=rset_adaway"
 
-config source "malware"
+config source "disconnect"
        option enabled "0"
-       option adb_src_malware "http://mirror1.malwaredomains.com/files/justdomains&ruleset=rset_malware"
+       option adb_src_disconnect "https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt&ruleset=rset_disconnect"
 
-config source "zeus"
+config source "dshield"
        option enabled "0"
-       option adb_src_zeus "https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist&ruleset=rset_zeus"
+       option adb_src_dshield "http://www.dshield.org/feeds/suspiciousdomains_Low.txt&ruleset=rset_dshield"
 
 config source "feodo"
        option enabled "0"
        option adb_src_feodo "https://feodotracker.abuse.ch/blocklist/?download=domainblocklist&ruleset=rset_feodo"
 
+config source "malware"
+       option enabled "0"
+       option adb_src_malware "http://mirror1.malwaredomains.com/files/justdomains&ruleset=rset_malware"
+
 config source "palevo"
        option enabled "0"
        option adb_src_palevo "https://palevotracker.abuse.ch/blocklists.php?download=domainblocklist&ruleset=rset_palevo"
 
-config source "dshield"
+config source "shalla"
        option enabled "0"
-       option adb_src_dshield "http://www.dshield.org/feeds/suspiciousdomains_Low.txt&ruleset=rset_dshield"
+       option adb_arc_shalla "http://www.shallalist.de/Downloads/shallalist.tar.gz"
+       list adb_catlist "adv"
+       list adb_catlist "costtraps"
+       list adb_catlist "spyware"
+       list adb_catlist "tracker"
+       list adb_catlist "warez"
 
 config source "spam404"
        option enabled "0"
        option adb_src_spam404 "http://spam404bl.com/spam404scamlist.txt&ruleset=rset_spam404"
 
-config source "winhelp"
+config source "whocares"
        option enabled "0"
-       option adb_src_winhelp "http://winhelp2002.mvps.org/hosts.txt&ruleset=rset_winhelp"
+       option adb_src_whocares "http://someonewhocares.org/hosts/hosts&ruleset=rset_whocares"
 
-config source "adaway"
+config source "winhelp"
        option enabled "0"
-       option adb_src_adaway "https://adaway.org/hosts.txt&ruleset=rset_adaway"
+       option adb_src_winhelp "http://winhelp2002.mvps.org/hosts.txt&ruleset=rset_winhelp"
 
-config source "disconnect"
-       option enabled "0"
-       option adb_src_disconnect "https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt&ruleset=rset_disconnect"
+config source "yoyo"
+       option enabled "1"
+       option adb_src_yoyo "http://pgl.yoyo.org/adservers/serverlist.php?hostformat=one-line&showintro=0&mimetype=plaintext&ruleset=rset_yoyo"
 
-config source "shalla"
+config source "zeus"
        option enabled "0"
-       option adb_arc_shalla "http://www.shallalist.de/Downloads/shallalist.tar.gz"
-       list adb_catlist "adv"
-       list adb_catlist "costtraps"
-       list adb_catlist "spyware"
-       list adb_catlist "tracker"
-       list adb_catlist "warez"
+       option adb_src_zeus "https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist&ruleset=rset_zeus"