net/adblock/files/adblock.sh

   1 #!/bin/sh
   2 # dns based ad/abuse domain blocking
   3 # written by Dirk Brenken (dev@brenken.org)
   4
   5 # This is free software, licensed under the GNU General Public License v3.
   6 # You should have received a copy of the GNU General Public License
   7 # along with this program. If not, see <http://www.gnu.org/licenses/>.
   8
   9 # set initial defaults
  10 #
  11 LC_ALL=C
  12 PATH="/usr/sbin:/usr/bin:/sbin:/bin"
  13 adb_ver="2.8.0-2"
  14 adb_sysver="$(ubus -S call system board | jsonfilter -e '@.release.description')"
  15 adb_enabled=1
  16 adb_debug=0
  17 adb_minfree=2
  18 adb_manmode=0
  19 adb_forcesrt=0
  20 adb_forcedns=0
  21 adb_backup=0
  22 adb_backupdir="/mnt"
  23 adb_whitelist="/etc/adblock/adblock.whitelist"
  24 adb_whitelist_rset="\$1 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\"^\"\$1\"\\\|[.]\"\$1)}"
  25 adb_fetch="/usr/bin/wget"
  26 adb_fetchparm="--quiet --no-cache --no-cookies --max-redirect=0 --timeout=10 --no-check-certificate -O"
  27 adb_dnslist="dnsmasq unbound named"
  28 adb_dnsprefix="adb_list"
  29 adb_dnsfile="${adb_dnsprefix}.overall"
  30 adb_rtfile="/tmp/adb_runtime.json"
  31 adb_sources=""
  32 adb_src_cat_shalla=""
  33 adb_action="${1}"
  34
  35 # f_envload: load adblock environment
  36 #
  37 f_envload()
  38 {
  39     local services dns_up cnt=0
  40
  41     # source in system libraries
  42     #
  43     if [ -r "/lib/functions.sh" ] && [ -r "/usr/share/libubox/jshn.sh" ]
  44     then
  45         . "/lib/functions.sh"
  46         . "/usr/share/libubox/jshn.sh"
  47     else
  48         f_log "error" "system libraries not found"
  49     fi
  50
  51     # parse global section by callback
  52     #
  53     config_cb()
  54     {
  55         local type="${1}"
  56         if [ "${type}" = "adblock" ]
  57         then
  58             option_cb()
  59             {
  60                 local option="${1}"
  61                 local value="${2}"
  62                 eval "${option}=\"${value}\""
  63             }
  64         else
  65             reset_cb
  66         fi
  67     }
  68
  69     # parse 'source' sections
  70     #
  71     parse_config()
  72     {
  73         local value opt section="${1}" options="enabled adb_src adb_src_rset adb_src_cat"
  74         eval "adb_sources=\"${adb_sources} ${section}\""
  75         for opt in ${options}
  76         do
  77             config_get value "${section}" "${opt}"
  78             if [ -n "${value}" ]
  79             then
  80                 eval "${opt}_${section}=\"${value}\""
  81             fi
  82         done
  83     }
  84
  85     # load adblock config
  86     #
  87     config_load adblock
  88     config_foreach parse_config source
  89
  90     # set dns backend environment
  91     #
  92     while [ ${cnt} -le 20 ]
  93     do
  94         services="$(ubus -S call service list 2>/dev/null)"
  95         if [ -n "${services}" ]
  96         then
  97             for dns in ${adb_dnslist}
  98             do
  99                 dns_up="$(printf "%s" "${services}" | jsonfilter -l1 -e "@.${dns}.instances.*.running")"
 100                 if [ "${dns_up}" = "true" ]
 101                 then
 102                     case "${dns}" in
 103                         dnsmasq)
 104                             adb_dns="${dns}"
 105                             adb_dnsdir="${adb_dnsdir:="/tmp/dnsmasq.d"}"
 106                             adb_dnshidedir="${adb_dnsdir}/.adb_hidden"
 107                             adb_dnsformat="awk '{print \"local=/\"\$0\"/\"}'"
 108                             break 2
 109                             ;;
 110                         unbound)
 111                             adb_dns="${dns}"
 112                             adb_dnsdir="${adb_dnsdir:="/var/lib/unbound"}"
 113                             adb_dnshidedir="${adb_dnsdir}/.adb_hidden"
 114                             adb_dnsformat="awk '{print \"local-zone: \042\"\$0\"\042 static\"}'"
 115                             break 2
 116                             ;;
 117                         named)
 118                             adb_dns="${dns}"
 119                             adb_dnsdir="${adb_dnsdir:="/var/lib/bind"}"
 120                             adb_dnshidedir="${adb_dnsdir}/.adb_hidden"
 121                             adb_dnsformat="awk '{print \"\"\$0\" IN CNAME .\n*.\"\$0\" IN CNAME .\"}'"
 122                             break 2
 123                             ;;
 124                     esac
 125                 fi
 126             done
 127         fi
 128         sleep 1
 129         cnt=$((cnt+1))
 130     done
 131     if [ -z "${adb_dns}" ] || [ -z "${adb_dnsformat}" ] || [ ! -x "$(command -v ${adb_dns})" ] || [ ! -d "${adb_dnsdir}" ]
 132     then
 133         f_log "error" "no active/supported DNS backend found"
 134     fi
 135
 136     # force dns to local resolver
 137     #
 138     if [ ${adb_forcedns} -eq 1 ] && [ -z "$(uci -q get firewall.adblock_dns)" ]
 139     then
 140         uci -q set firewall.adblock_dns="redirect"
 141         uci -q set firewall.adblock_dns.name="Adblock DNS"
 142         uci -q set firewall.adblock_dns.src="lan"
 143         uci -q set firewall.adblock_dns.proto="tcp udp"
 144         uci -q set firewall.adblock_dns.src_dport="53"
 145         uci -q set firewall.adblock_dns.dest_port="53"
 146         uci -q set firewall.adblock_dns.target="DNAT"
 147     elif [ ${adb_forcedns} -eq 0 ] && [ -n "$(uci -q get firewall.adblock_dns)" ]
 148     then
 149         uci -q delete firewall.adblock_dns
 150     fi
 151     if [ -n "$(uci -q changes firewall)" ]
 152     then
 153         uci -q commit firewall
 154         if [ $(/etc/init.d/firewall enabled; printf "%u" ${?}) -eq 0 ]
 155         then
 156             /etc/init.d/firewall reload >/dev/null 2>&1
 157         fi
 158     fi
 159 }
 160
 161 # f_envcheck: check/set environment prerequisites
 162 #
 163 f_envcheck()
 164 {
 165     local ssl_lib
 166
 167     # check 'enabled' option
 168     #
 169     if [ ${adb_enabled} -ne 1 ]
 170     then
 171         if [ -s "${adb_dnsdir}/${adb_dnsfile}" ]
 172         then
 173             f_rmdns
 174             f_dnsrestart
 175         fi
 176         f_log "info " "adblock is currently disabled, please set adb_enabled to '1' to use this service"
 177         exit 0
 178     fi
 179
 180     # check fetch utility
 181     #
 182     ssl_lib="-"
 183     if [ -x "${adb_fetch}" ]
 184     then
 185         if [ "$(readlink -fn "${adb_fetch}")" = "/usr/bin/wget-nossl" ]
 186         then
 187             adb_fetchparm="--quiet --no-cache --no-cookies --max-redirect=0 --timeout=10 -O"
 188         elif [ "$(readlink -fn "/bin/wget")" = "/bin/busybox" ] || [ "$(readlink -fn "${adb_fetch}")" = "/bin/busybox" ]
 189         then
 190             adb_fetch="/bin/busybox"
 191             adb_fetchparm="-q -O"
 192         else
 193             ssl_lib="built-in"
 194         fi
 195     fi
 196     if [ ! -x "${adb_fetch}" ] && [ "$(readlink -fn "/bin/wget")" = "/bin/uclient-fetch" ]
 197     then
 198         adb_fetch="/bin/uclient-fetch"
 199         if [ -f "/lib/libustream-ssl.so" ]
 200         then
 201             adb_fetchparm="-q --timeout=10 --no-check-certificate -O"
 202             ssl_lib="libustream-ssl"
 203         else
 204             adb_fetchparm="-q --timeout=10 -O"
 205         fi
 206     fi
 207     if [ ! -x "${adb_fetch}" ] || [ -z "${adb_fetch}" ] || [ -z "${adb_fetchparm}" ]
 208     then
 209         f_log "error" "no download utility found, please install 'uclient-fetch' with 'libustream-mbedtls' or the full 'wget' package"
 210     fi
 211     adb_fetchinfo="${adb_fetch##*/} (${ssl_lib})"
 212
 213     # create dns hideout directory
 214     #
 215     if [ ! -d "${adb_dnshidedir}" ]
 216     then
 217         mkdir -p -m 660 "${adb_dnshidedir}"
 218         chown -R "${adb_dns}":"${adb_dns}" "${adb_dnshidedir}" 2>/dev/null
 219     else
 220         rm -f "${adb_dnshidedir}/${adb_dnsprefix}"*
 221     fi
 222
 223     # create adblock temp file/directory
 224     #
 225     adb_tmpload="$(mktemp -tu)"
 226     adb_tmpfile="$(mktemp -tu)"
 227     adb_tmpdir="$(mktemp -p /tmp -d)"
 228
 229     # prepare whitelist entries
 230     #
 231     if [ -s "${adb_whitelist}" ]
 232     then
 233         awk "${adb_whitelist_rset}" "${adb_whitelist}" > "${adb_tmpdir}/tmp.whitelist"
 234     fi
 235 }
 236
 237 # f_rmtemp: remove temporary files & directories
 238 #
 239 f_rmtemp()
 240 {
 241     if [ -d "${adb_tmpdir}" ]
 242     then
 243         rm -f "${adb_tmpload}"
 244         rm -f "${adb_tmpfile}"
 245         rm -rf "${adb_tmpdir}"
 246     fi
 247 }
 248
 249 # f_rmdns: remove dns related files & directories
 250 #
 251 f_rmdns()
 252 {
 253     if [ -n "${adb_dns}" ]
 254     then
 255         rm -f "${adb_dnsdir}/${adb_dnsprefix}"*
 256         rm -f "${adb_backupdir}/${adb_dnsprefix}"*.gz
 257         rm -rf "${adb_dnshidedir}"
 258         > "${adb_rtfile}"
 259     fi
 260 }
 261
 262 # f_dnsrestart: restart the dns backend
 263 #
 264 f_dnsrestart()
 265 {
 266     local dns_up mem_free cnt=0
 267
 268     "/etc/init.d/${adb_dns}" restart >/dev/null 2>&1
 269     while [ ${cnt} -le 10 ]
 270     do
 271         dns_up="$(ubus -S call service list "{\"name\":\"${adb_dns}\"}" | jsonfilter -l1 -e "@.${adb_dns}.instances.*.running")"
 272         if [ "${dns_up}" = "true" ]
 273         then
 274             mem_free="$(awk '/^MemFree/ {print int($2/1000)}' "/proc/meminfo")"
 275             if [ ${mem_free} -ge ${adb_minfree} ]
 276             then
 277                 return 0
 278             fi
 279         fi
 280         cnt=$((cnt+1))
 281         sleep 1
 282     done
 283     return 1
 284 }
 285
 286 # f_list: backup/restore/remove block lists
 287 #
 288 f_list()
 289 {
 290     local mode="${1}" in_rc="${adb_rc}" cnt=0
 291
 292     case "${mode}" in
 293         backup)
 294             cnt="$(wc -l < "${adb_tmpfile}")"
 295             if [ ${adb_backup} -eq 1 ] && [ -d "${adb_backupdir}" ]
 296             then
 297                 gzip -cf "${adb_tmpfile}" > "${adb_backupdir}/${adb_dnsprefix}.${src_name}.gz"
 298                 adb_rc=${?}
 299             fi
 300             ;;
 301         restore)
 302             if [ ${adb_backup} -eq 1 ] && [ -d "${adb_backupdir}" ] &&
 303                 [ -f "${adb_backupdir}/${adb_dnsprefix}.${src_name}.gz" ]
 304             then
 305                 gunzip -cf "${adb_backupdir}/${adb_dnsprefix}.${src_name}.gz" > "${adb_tmpfile}"
 306                 adb_rc=${?}
 307             fi
 308             ;;
 309         remove)
 310             if [ -d "${adb_backupdir}" ]
 311             then
 312                 rm -f "${adb_backupdir}/${adb_dnsprefix}.${src_name}.gz"
 313             fi
 314             adb_rc=${?}
 315             ;;
 316         format)
 317             if [ -s "${adb_tmpdir}/tmp.whitelist" ]
 318             then
 319                 grep -vf "${adb_tmpdir}/tmp.whitelist" "${adb_tmpfile}" | eval "${adb_dnsformat}" >> "${adb_tmpdir}/${adb_dnsfile}"
 320             else
 321                 eval "${adb_dnsformat}" "${adb_tmpfile}" >> "${adb_tmpdir}/${adb_dnsfile}"
 322             fi
 323             adb_rc=${?}
 324             ;;
 325     esac
 326     f_log "debug" "name: ${src_name}, mode: ${mode}, count: ${cnt}, in_rc: ${in_rc}, out_rc: ${adb_rc}"
 327 }
 328
 329 # f_switch: suspend/resume adblock processing
 330 #
 331 f_switch()
 332 {
 333     local source target status mode="${1}"
 334
 335     if [ -d "${adb_dnshidedir}" ]
 336     then
 337         if [ -s "${adb_dnsdir}/${adb_dnsfile}" ] && [ "${mode}" = "suspend" ]
 338         then
 339             source="${adb_dnsdir}/${adb_dnsfile}"
 340             target="${adb_dnshidedir}"
 341             status="suspended"
 342         elif [ -s "${adb_dnshidedir}/${adb_dnsfile}" ] && [ "${mode}" = "resume" ]
 343         then
 344             source="${adb_dnshidedir}/${adb_dnsfile}"
 345             target="${adb_dnsdir}"
 346             status="resumed"
 347         fi
 348         if [ -n "${status}" ]
 349         then
 350             mv -f "${source}"* "${target}"
 351             f_dnsrestart
 352             f_log "info " "adblock processing ${status}"
 353         fi
 354     fi
 355 }
 356
 357 # f_query: query block list for certain (sub-)domains
 358 #
 359 f_query()
 360 {
 361     local search result cnt
 362     local domain="${1}"
 363     local tld="${domain#*.}"
 364
 365     if [ ! -s "${adb_dnsdir}/${adb_dnsfile}" ]
 366     then
 367          printf "%s\n" "::: no active block list found, please start / resume adblock first"
 368     elif [ -z "${domain}" ] || [ "${domain}" = "${tld}" ]
 369     then
 370         printf "%s\n" "::: invalid domain input, please submit a specific (sub-)domain, e.g. 'www.abc.xyz'"
 371     else
 372         cd "${adb_dnsdir}"
 373         while [ "${domain}" != "${tld}" ]
 374         do
 375             search="${domain//./\.}"
 376             result="$(grep -Hm5 "[/\"\.]${search}[/\"]" "${adb_dnsfile}" | awk -F ':|=|/|\"' '{printf(" + %s\n",$4)}')"
 377             printf "%s\n" "::: results for (sub-)domain '${domain}' (max. 5)"
 378             printf "%s\n" "${result:=" - no match"}"
 379             domain="${tld}"
 380             tld="${domain#*.}"
 381         done
 382     fi
 383 }
 384
 385 # f_status: output runtime information
 386 #
 387 f_status()
 388 {
 389     local key keylist value
 390
 391     if [ -s "${adb_rtfile}" ]
 392     then
 393         if [ -s "${adb_dnsdir}/${adb_dnsfile}" ]
 394         then
 395             value="active"
 396         else
 397             value="no domains blocked"
 398         fi
 399         printf "%s\n" "::: adblock runtime information"
 400         printf " %-15s : %s\n" "status" "${value}"
 401         json_load "$(cat "${adb_rtfile}" 2>/dev/null)"
 402         json_select data
 403         json_get_keys keylist
 404         for key in ${keylist}
 405         do
 406             json_get_var value "${key}"
 407             printf " %-15s : %s\n" "${key}" "${value}"
 408         done
 409     fi
 410 }
 411
 412 # f_log: write to syslog, exit on error
 413 #
 414 f_log()
 415 {
 416     local class="${1}" log_msg="${2}"
 417
 418     if [ -n "${log_msg}" ] && ([ "${class}" != "debug" ] || [ ${adb_debug} -eq 1 ])
 419     then
 420         logger -t "adblock-[${adb_ver}] ${class}" "${log_msg}"
 421         if [ "${class}" = "error" ]
 422         then
 423             logger -t "adblock-[${adb_ver}] ${class}" "Please check 'https://github.com/openwrt/packages/blob/master/net/adblock/files/README.md' (${adb_sysver})"
 424             f_rmtemp
 425             if [ -s "${adb_dnsdir}/${adb_dnsfile}" ]
 426             then
 427                 f_rmdns
 428                 f_dnsrestart
 429             fi
 430             exit 1
 431         fi
 432     fi
 433 }
 434
 435 # main function for block list processing
 436 #
 437 f_main()
 438 {
 439     local src_name src_rset shalla_archive enabled url cnt=0
 440     local mem_total="$(awk '/^MemTotal/ {print int($2/1000)}' "/proc/meminfo")"
 441
 442     f_log "info " "start adblock processing ..."
 443     f_log "debug" "action: ${adb_action}, manual_mode:${adb_manmode}, backup: ${adb_backup}, dns: ${adb_dns}, fetch: ${adb_fetchinfo}, mem_total: ${mem_total}, force_srt/_dns: ${adb_forcesrt}/${adb_forcedns}"
 444     > "${adb_rtfile}"
 445     for src_name in ${adb_sources}
 446     do
 447         eval "enabled=\"\${enabled_${src_name}}\""
 448         eval "url=\"\${adb_src_${src_name}}\""
 449         eval "src_rset=\"\${adb_src_rset_${src_name}}\""
 450         > "${adb_tmpload}"
 451         > "${adb_tmpfile}"
 452         adb_rc=4
 453
 454         # basic pre-checks
 455         #
 456         f_log "debug" "name: ${src_name}, enabled: ${enabled}, url: ${url}, rset: ${src_rset}"
 457         if [ "${enabled}" != "1" ] || [ -z "${url}" ] || [ -z "${src_rset}" ]
 458         then
 459             f_list remove
 460             continue
 461         fi
 462
 463         # manual mode
 464         #
 465         if [ ${adb_manmode} -eq 1 ] && [ -z "${adb_action}" ]
 466         then
 467             f_list restore
 468             if [ ${adb_rc} -eq 0 ] && [ -s "${adb_tmpfile}" ]
 469             then
 470                 f_list format
 471                 continue
 472             fi
 473         fi
 474
 475         # download block list
 476         #
 477         if [ "${src_name}" = "blacklist" ] && [ -s "${url}" ]
 478         then
 479             cat "${url}" > "${adb_tmpload}"
 480             adb_rc=${?}
 481         elif [ "${src_name}" = "shalla" ]
 482         then
 483             shalla_archive="${adb_tmpdir}/shallalist.tar.gz"
 484             "${adb_fetch}" ${adb_fetchparm} "${shalla_archive}" "${url}" 2>/dev/null
 485             adb_rc=${?}
 486             if [ ${adb_rc} -eq 0 ]
 487             then
 488                 for category in ${adb_src_cat_shalla}
 489                 do
 490                     tar -xOzf "${shalla_archive}" "BL/${category}/domains" >> "${adb_tmpload}"
 491                     adb_rc=${?}
 492                     if [ ${adb_rc} -ne 0 ]
 493                     then
 494                         break
 495                     fi
 496                 done
 497             fi
 498             rm -f "${shalla_archive}"
 499             rm -rf "${adb_tmpdir}/BL"
 500         else
 501             "${adb_fetch}" ${adb_fetchparm} "${adb_tmpload}" "${url}" 2>/dev/null
 502             adb_rc=${?}
 503         fi
 504
 505         # check download result and prepare domain output (incl. tld compression, list backup & restore)
 506         #
 507         if [ ${adb_rc} -eq 0 ] && [ -s "${adb_tmpload}" ]
 508         then
 509             awk "${src_rset}" "${adb_tmpload}" 2>/dev/null > "${adb_tmpfile}"
 510             if [ -s "${adb_tmpfile}" ]
 511             then
 512                 awk -F "." '{for(f=NF;f > 1;f--) printf "%s.", $f;print $1}' "${adb_tmpfile}" 2>/dev/null | sort -u > "${adb_tmpload}"
 513                 awk '{if(NR==1){tld=$NF};while(getline){if($NF !~ tld"\\."){print tld;tld=$NF}}print tld}' "${adb_tmpload}" 2>/dev/null > "${adb_tmpfile}"
 514                 awk -F "." '{for(f=NF;f > 1;f--) printf "%s.", $f;print $1}' "${adb_tmpfile}" 2>/dev/null > "${adb_tmpload}"
 515                 mv -f "${adb_tmpload}" "${adb_tmpfile}"
 516                 f_list backup
 517             else
 518                 f_list restore
 519             fi
 520         else
 521             f_list restore
 522         fi
 523
 524         # remove whitelist domains, final list preparation
 525         #
 526         if [ ${adb_rc} -eq 0 ] && [ -s "${adb_tmpfile}" ]
 527         then
 528             f_list format
 529             if [ ${adb_rc} -ne 0 ]
 530             then
 531                 f_list remove
 532             fi
 533         else
 534             f_list remove
 535         fi
 536     done
 537
 538     # overall sort
 539     #
 540     if [ -s "${adb_tmpdir}/${adb_dnsfile}" ]
 541     then
 542         if [ ${mem_total} -ge 64 ] || [ ${adb_forcesrt} -eq 1 ]
 543         then
 544             sort -u "${adb_tmpdir}/${adb_dnsfile}" > "${adb_dnsdir}/${adb_dnsfile}"
 545         else
 546             mv -f "${adb_tmpdir}/${adb_dnsfile}" "${adb_dnsdir}" 2>/dev/null
 547         fi
 548     else
 549         > "${adb_dnsdir}/${adb_dnsfile}"
 550     fi
 551     cnt="$(wc -l < "${adb_dnsdir}/${adb_dnsfile}")"
 552
 553     # restart the dns backend and export runtime information
 554     #
 555     chown "${adb_dns}":"${adb_dns}" "${adb_dnsdir}/${adb_dnsfile}" 2>/dev/null
 556     f_rmtemp
 557     f_dnsrestart
 558     if [ ${?} -eq 0 ]
 559     then
 560         json_init
 561         json_add_object "data"
 562         json_add_string "adblock_version" "${adb_ver}"
 563         json_add_string "blocked_domains" "${cnt}"
 564         json_add_string "fetch_info" "${adb_fetchinfo}"
 565         json_add_string "dns_backend" "${adb_dns}"
 566         json_add_string "last_rundate" "$(/bin/date "+%d.%m.%Y %H:%M:%S")"
 567         json_add_string "system" "${adb_sysver}"
 568         json_close_object
 569         json_dump > "${adb_rtfile}"
 570         f_log "info " "block list with overall ${cnt} domains loaded successfully (${adb_sysver})"
 571     else
 572         f_log "error" "dns backend restart with active block list failed"
 573     fi
 574 }
 575
 576 # handle different adblock actions
 577 #
 578 f_envload
 579 case "${adb_action}" in
 580     stop)
 581         f_rmtemp
 582         f_rmdns
 583         f_dnsrestart
 584         ;;
 585     restart)
 586         f_rmtemp
 587         f_rmdns
 588         f_envcheck
 589         f_main
 590         ;;
 591     suspend)
 592         f_switch suspend
 593         ;;
 594     resume)
 595         f_switch resume
 596         ;;
 597     query)
 598         f_query "${2}"
 599         ;;
 600     status)
 601         f_status
 602         ;;
 603     *)
 604         f_envcheck
 605         f_main
 606         ;;
 607 esac
 608 exit 0