X-Git-Url: http://git.openwrt.org/?a=blobdiff_plain;f=net%2Fadblock%2Ffiles%2Fadblock.sh;h=cd0357617898520076c30f664ec669bae1a2bd80;hb=5c934e5352b6d700d6b3771261d0ad585e95a547;hp=028f42110149f0278ba5cd097395642bd64a9a72;hpb=e660813798832ffd8d7aad0c53161318c93eb024;p=feed%2Fpackages.git diff --git a/net/adblock/files/adblock.sh b/net/adblock/files/adblock.sh index 028f421101..cd03576178 100755 --- a/net/adblock/files/adblock.sh +++ b/net/adblock/files/adblock.sh @@ -10,33 +10,46 @@ # LC_ALL=C PATH="/usr/sbin:/usr/bin:/sbin:/bin" -adb_ver="2.8.0" -adb_sysver="$(ubus -S call system board | jsonfilter -e '@.release.description')" -adb_enabled=1 +adb_ver="3.0.3" +adb_sysver="unknown" +adb_enabled=0 adb_debug=0 -adb_minfree=2 adb_manmode=0 adb_forcesrt=0 adb_forcedns=0 +adb_triggerdelay=0 adb_backup=0 adb_backupdir="/mnt" -adb_whitelist="/etc/adblock/adblock.whitelist" -adb_whitelist_rset="\$1 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\"^\"\$1\"\\\|[.]\"\$1)}" adb_fetch="/usr/bin/wget" adb_fetchparm="--quiet --no-cache --no-cookies --max-redirect=0 --timeout=10 --no-check-certificate -O" -adb_dnslist="dnsmasq unbound named" +adb_dns="dnsmasq" adb_dnsprefix="adb_list" adb_dnsfile="${adb_dnsprefix}.overall" adb_rtfile="/tmp/adb_runtime.json" -adb_sources="" -adb_src_cat_shalla="" -adb_action="${1}" +adb_action="${1:-"start"}" +adb_cnt=0 +adb_rc=0 # f_envload: load adblock environment # f_envload() { - local services dns_up cnt=0 + local dns_up sys_call sys_desc sys_model sys_ver cnt=0 + + # get system information + # + sys_call="$(ubus -S call system board 2>/dev/null)" + if [ -n "${sys_call}" ] + then + sys_desc="$(printf '%s' "${sys_call}" | jsonfilter -e '@.release.description')" + sys_model="$(printf '%s' "${sys_call}" | jsonfilter -e '@.model')" + sys_ver="$(cat /etc/turris-version 2>/dev/null)" + if [ -n "${sys_ver}" ] + then + sys_desc="${sys_desc}/${sys_ver}" + fi + adb_sysver="${sys_model}, ${sys_desc}" + fi # source in system libraries # @@ -87,50 +100,66 @@ f_envload() config_load adblock config_foreach parse_config source - # set dns backend environment + # set/check dns backend environment # - while [ ${cnt} -le 20 ] + case "${adb_dns}" in + dnsmasq) + adb_dnsuser="${adb_dns}" + adb_dnsdir="${adb_dnsdir:-"/tmp/dnsmasq.d"}" + adb_dnsformat="awk '{print \"local=/\"\$0\"/\"}'" + ;; + unbound) + adb_dnsuser="${adb_dns}" + adb_dnsdir="${adb_dnsdir:-"/var/lib/unbound"}" + adb_dnsformat="awk '{print \"local-zone: \042\"\$0\"\042 static\"}'" + ;; + named) + adb_dnsuser="bind" + adb_dnsdir="${adb_dnsdir:-"/var/lib/bind"}" + adb_dnsformat="awk '{print \"\"\$0\" IN CNAME .\n*.\"\$0\" IN CNAME .\"}'" + ;; + kresd) + adb_dnsuser="root" + adb_dnsdir="${adb_dnsdir:-"/etc/kresd"}" + adb_dnsformat="awk '{print \"\"\$0\" CNAME .\n*.\"\$0\" CNAME .\"}'" + adb_dnsheader="\$TTL 2h"$'\n'"@ IN SOA localhost. root.localhost. (2 6h 1h 1w 2h)"$'\n'" IN NS localhost." + ;; + dnscrypt-proxy) + adb_dnsuser="nobody" + adb_dnsdir="${adb_dnsdir:-"/tmp"}" + adb_dnsformat="awk '{print \$0}'" + ;; + esac + + if [ -d "${adb_dnsdir}" ] && [ ! -f "${adb_dnsdir}/${adb_dnsfile}" ] + then + > "${adb_dnsdir}/${adb_dnsfile}" + fi + + case "${adb_action}" in + start|restart|reload) + > "${adb_rtfile}" + if [ "${adb_action}" = "start" ] && [ "${adb_trigger}" = "timed" ] + then + sleep ${adb_triggerdelay} + fi + ;; + esac + + while [ ${cnt} -le 30 ] do - services="$(ubus -S call service list 2>/dev/null)" - if [ -n "${services}" ] + dns_up="$(ubus -S call service list "{\"name\":\"${adb_dns}\"}" 2>/dev/null | jsonfilter -l1 -e "@[\"${adb_dns}\"].instances.*.running" 2>/dev/null)" + if [ "${dns_up}" = "true" ] then - for dns in ${adb_dnslist} - do - dns_up="$(printf "%s" "${services}" | jsonfilter -l1 -e "@.${dns}.instances.*.running")" - if [ "${dns_up}" = "true" ] - then - case "${dns}" in - dnsmasq) - adb_dns="${dns}" - adb_dnsdir="${adb_dnsdir:="/tmp/dnsmasq.d"}" - adb_dnshidedir="${adb_dnsdir}/.adb_hidden" - adb_dnsformat="awk '{print \"local=/\"\$0\"/\"}'" - break 2 - ;; - unbound) - adb_dns="${dns}" - adb_dnsdir="${adb_dnsdir:="/var/lib/unbound"}" - adb_dnshidedir="${adb_dnsdir}/.adb_hidden" - adb_dnsformat="awk '{print \"local-zone: \042\"\$0\"\042 static\"}'" - break 2 - ;; - named) - adb_dns="${dns}" - adb_dnsdir="${adb_dnsdir:="/var/lib/bind"}" - adb_dnshidedir="${adb_dnsdir}/.adb_hidden" - adb_dnsformat="awk '{print \"\"\$0\" IN CNAME .\n*.\"\$0\" IN CNAME .\"}'" - break 2 - ;; - esac - fi - done + break fi sleep 1 cnt=$((cnt+1)) done + if [ -z "${adb_dns}" ] || [ -z "${adb_dnsformat}" ] || [ ! -x "$(command -v ${adb_dns})" ] || [ ! -d "${adb_dnsdir}" ] then - f_log "error" "no active/supported DNS backend found" + f_log "error" "'${adb_dns}' not running, DNS backend not found" fi # force dns to local resolver @@ -173,6 +202,7 @@ f_envcheck() f_rmdns f_dnsrestart fi + f_jsnupdate f_log "info " "adblock is currently disabled, please set adb_enabled to '1' to use this service" exit 0 fi @@ -185,7 +215,8 @@ f_envcheck() if [ "$(readlink -fn "${adb_fetch}")" = "/usr/bin/wget-nossl" ] then adb_fetchparm="--quiet --no-cache --no-cookies --max-redirect=0 --timeout=10 -O" - elif [ "$(readlink -fn "/bin/wget")" = "/bin/busybox" ] || [ "$(readlink -fn "${adb_fetch}")" = "/bin/busybox" ] + elif [ "$(readlink -fn "${adb_fetch}")" = "/bin/busybox" ] || + ([ "$(readlink -fn "/bin/wget")" = "/bin/busybox" ] && [ "$(readlink -fn "${adb_fetch}")" != "/usr/bin/wget" ]) then adb_fetch="/bin/busybox" adb_fetchparm="-q -O" @@ -210,16 +241,6 @@ f_envcheck() fi adb_fetchinfo="${adb_fetch##*/} (${ssl_lib})" - # create dns hideout directory - # - if [ ! -d "${adb_dnshidedir}" ] - then - mkdir -p -m 660 "${adb_dnshidedir}" - chown -R "${adb_dns}":"${adb_dns}" "${adb_dnshidedir}" 2>/dev/null - else - rm -f "${adb_dnshidedir}/${adb_dnsprefix}"* - fi - # create adblock temp file/directory # adb_tmpload="$(mktemp -tu)" @@ -228,7 +249,7 @@ f_envcheck() # prepare whitelist entries # - if [ -s "${adb_whitelist}" ] + if [ -s "${adb_whitelist}" ] && [ -n "${adb_whitelist_rset}" ] then awk "${adb_whitelist_rset}" "${adb_whitelist}" > "${adb_tmpdir}/tmp.whitelist" fi @@ -252,10 +273,10 @@ f_rmdns() { if [ -n "${adb_dns}" ] then - rm -f "${adb_dnsdir}/${adb_dnsprefix}"* - rm -f "${adb_backupdir}/${adb_dnsprefix}"*.gz - rm -rf "${adb_dnshidedir}" + > "${adb_dnsdir}/${adb_dnsfile}" > "${adb_rtfile}" + rm -f "${adb_dnsdir}/.${adb_dnsfile}" + rm -f "${adb_backupdir}/${adb_dnsprefix}"*.gz fi } @@ -263,19 +284,15 @@ f_rmdns() # f_dnsrestart() { - local dns_up mem_free cnt=0 + local dns_up cnt=0 "/etc/init.d/${adb_dns}" restart >/dev/null 2>&1 while [ ${cnt} -le 10 ] do - dns_up="$(ubus -S call service list "{\"name\":\"${adb_dns}\"}" | jsonfilter -l1 -e "@.${adb_dns}.instances.*.running")" + dns_up="$(ubus -S call service list "{\"name\":\"${adb_dns}\"}" | jsonfilter -l1 -e "@[\"${adb_dns}\"].instances.*.running")" if [ "${dns_up}" = "true" ] then - mem_free="$(awk '/^MemFree/ {print int($2/1000)}' "/proc/meminfo")" - if [ ${mem_free} -ge ${adb_minfree} ] - then - return 0 - fi + return 0 fi cnt=$((cnt+1)) sleep 1 @@ -313,12 +330,24 @@ f_list() fi adb_rc=${?} ;; + merge) + if [ -s "${adb_tmpfile}" ] + then + cat "${adb_tmpfile}" >> "${adb_tmpdir}/${adb_dnsfile}" + adb_rc=${?} + fi + ;; format) if [ -s "${adb_tmpdir}/tmp.whitelist" ] then - grep -vf "${adb_tmpdir}/tmp.whitelist" "${adb_tmpfile}" | eval "${adb_dnsformat}" >> "${adb_tmpdir}/${adb_dnsfile}" + grep -vf "${adb_tmpdir}/tmp.whitelist" "${adb_tmpdir}/${adb_dnsfile}" | eval "${adb_dnsformat}" > "${adb_dnsdir}/${adb_dnsfile}" else - eval "${adb_dnsformat}" "${adb_tmpfile}" >> "${adb_tmpdir}/${adb_dnsfile}" + eval "${adb_dnsformat}" "${adb_tmpdir}/${adb_dnsfile}" > "${adb_dnsdir}/${adb_dnsfile}" + fi + if [ -n "${adb_dnsheader}" ] + then + printf '%s\n' "${adb_dnsheader}" | cat - "${adb_dnsdir}/${adb_dnsfile}" > "${adb_tmpdir}/${adb_dnsfile}" + cat "${adb_tmpdir}/${adb_dnsfile}" > "${adb_dnsdir}/${adb_dnsfile}" fi adb_rc=${?} ;; @@ -326,31 +355,42 @@ f_list() f_log "debug" "name: ${src_name}, mode: ${mode}, count: ${cnt}, in_rc: ${in_rc}, out_rc: ${adb_rc}" } +# f_tldcompression: top level domain compression +# +f_tldcompression() +{ + local source="${1}" temp="${adb_tmpload}" + + awk -F "." '{for(f=NF;f > 1;f--) printf "%s.", $f;print $1}' "${source}" 2>/dev/null | sort -u > "${temp}" + awk '{if(NR==1){tld=$NF};while(getline){if($NF !~ tld"\\."){print tld;tld=$NF}}print tld}' "${temp}" 2>/dev/null > "${source}" + awk -F "." '{for(f=NF;f > 1;f--) printf "%s.", $f;print $1}' "${source}" 2>/dev/null > "${temp}" + sort -u "${temp}" > "${source}" +} + # f_switch: suspend/resume adblock processing # f_switch() { local source target status mode="${1}" - if [ -d "${adb_dnshidedir}" ] + if [ -s "${adb_dnsdir}/${adb_dnsfile}" ] && [ "${mode}" = "suspend" ] then - if [ -s "${adb_dnsdir}/${adb_dnsfile}" ] && [ "${mode}" = "suspend" ] - then - source="${adb_dnsdir}/${adb_dnsfile}" - target="${adb_dnshidedir}" - status="suspended" - elif [ -s "${adb_dnshidedir}/${adb_dnsfile}" ] && [ "${mode}" = "resume" ] - then - source="${adb_dnshidedir}/${adb_dnsfile}" - target="${adb_dnsdir}" - status="resumed" - fi - if [ -n "${status}" ] - then - mv -f "${source}"* "${target}" - f_dnsrestart - f_log "info " "adblock processing ${status}" - fi + source="${adb_dnsdir}/${adb_dnsfile}" + target="${adb_dnsdir}/.${adb_dnsfile}" + status="suspended" + elif [ -s "${adb_dnsdir}/.${adb_dnsfile}" ] && [ "${mode}" = "resume" ] + then + source="${adb_dnsdir}/.${adb_dnsfile}" + target="${adb_dnsdir}/${adb_dnsfile}" + status="resumed" + fi + if [ -n "${status}" ] + then + cat "${source}" > "${target}" + > "${source}" + f_dnsrestart + f_jsnupdate + f_log "info " "adblock processing ${status}" fi } @@ -367,21 +407,74 @@ f_query() printf "%s\n" "::: no active block list found, please start / resume adblock first" elif [ -z "${domain}" ] || [ "${domain}" = "${tld}" ] then - printf "%s\n" "::: invalid domain input, please submit a specific (sub-)domain, e.g. 'www.abc.xyz'" + printf "%s\n" "::: invalid domain input, please submit a single domain, e.g. 'doubleclick.net'" else cd "${adb_dnsdir}" while [ "${domain}" != "${tld}" ] do search="${domain//./\.}" - result="$(grep -Hm5 "[/\"\.]${search}[/\"]" "${adb_dnsfile}" | awk -F ':|=|/|\"' '{printf(" + %s\n",$4)}')" - printf "%s\n" "::: results for (sub-)domain '${domain}' (max. 5)" - printf "%s\n" "${result:=" - no match"}" + if [ "${adb_dns}" = "dnsmasq" ] || [ "${adb_dns}" = "unbound" ] + then + result="$(awk -F '/|\"' "/[\/\"\.]${search}/{i++;{printf(\" + %s\n\",\$2)};if(i>9){exit}}" "${adb_dnsfile}")" + else + result="$(awk "/(^[^\*][a-z]*[\.]+${search}|^${search})/{i++;{printf(\" + %s\n\",\$1)};if(i>9){exit}}" "${adb_dnsfile}")" + fi + printf "%s\n" "::: max. ten results for domain '${domain}'" + printf "%s\n" "${result:-" - no match"}" domain="${tld}" tld="${domain#*.}" done fi } +# f_jsnupdate: update runtime information +# +f_jsnupdate() +{ + local status rundate="$(/bin/date "+%d.%m.%Y %H:%M:%S")" + + if [ ${adb_rc} -gt 0 ] + then + status="error" + elif [ ${adb_enabled} -ne 1 ] + then + status="disabled" + elif [ -s "${adb_dnsdir}/.${adb_dnsfile}" ] + then + status="paused" + else + status="enabled" + if [ -s "${adb_dnsdir}/${adb_dnsfile}" ] + then + if [ "${adb_dns}" = "named" ] || [ "${adb_dns}" = "kresd" ] + then + adb_cnt="$(( ( $(wc -l < "${adb_dnsdir}/${adb_dnsfile}") - $(printf "%s" "${adb_dnsheader}" | grep -c "^") ) / 2 ))" + else + adb_cnt="$(( $(wc -l < "${adb_dnsdir}/${adb_dnsfile}") - $(printf "%s" "${adb_dnsheader}" | grep -c "^") ))" + fi + fi + fi + + if [ -z "${adb_fetchinfo}" ] && [ -s "${adb_rtfile}" ] + then + json_load "$(cat "${adb_rtfile}" 2>/dev/null)" + json_select data + json_get_var adb_fetchinfo "fetch_utility" + fi + + json_init + json_add_object "data" + json_add_string "adblock_status" "${status}" + json_add_string "adblock_version" "${adb_ver}" + json_add_string "blocked_domains" "${adb_cnt}" + json_add_string "fetch_utility" "${adb_fetchinfo}" + json_add_string "dns_backend" "${adb_dns} (${adb_dnsdir})" + json_add_string "last_rundate" "${rundate}" + json_add_string "system_release" "${adb_sysver}" + json_close_object + json_dump > "${adb_rtfile}" +} + # f_status: output runtime information # f_status() @@ -390,21 +483,14 @@ f_status() if [ -s "${adb_rtfile}" ] then - if [ -s "${adb_dnsdir}/${adb_dnsfile}" ] - then - value="active" - else - value="no domains blocked" - fi printf "%s\n" "::: adblock runtime information" - printf " %-15s : %s\n" "status" "${value}" json_load "$(cat "${adb_rtfile}" 2>/dev/null)" json_select data json_get_keys keylist for key in ${keylist} do json_get_var value "${key}" - printf " %-15s : %s\n" "${key}" "${value}" + printf " + %-15s : %s\n" "${key}" "${value}" done fi } @@ -427,6 +513,8 @@ f_log() f_rmdns f_dnsrestart fi + adb_rc=1 + f_jsnupdate exit 1 fi fi @@ -436,7 +524,7 @@ f_log() # f_main() { - local src_name src_rset shalla_archive enabled url cnt=0 + local src_name src_rset shalla_archive enabled url hash_old hash_new local mem_total="$(awk '/^MemTotal/ {print int($2/1000)}' "/proc/meminfo")" f_log "info " "start adblock processing ..." @@ -460,14 +548,14 @@ f_main() continue fi - # manual mode + # manual / backup mode # - if [ ${adb_manmode} -eq 1 ] && [ -z "${adb_action}" ] + if [ ${adb_manmode} -eq 1 ] && [ "${adb_action}" = "start" ] && [ "${src_name}" != "blacklist" ] then f_list restore if [ ${adb_rc} -eq 0 ] && [ -s "${adb_tmpfile}" ] then - f_list format + f_list merge continue fi fi @@ -502,18 +590,18 @@ f_main() adb_rc=${?} fi - # check download result and prepare domain output (incl. tld compression, list backup & restore) + # check download result and prepare list output (incl. tld compression, list backup & restore) # if [ ${adb_rc} -eq 0 ] && [ -s "${adb_tmpload}" ] then awk "${src_rset}" "${adb_tmpload}" 2>/dev/null > "${adb_tmpfile}" if [ -s "${adb_tmpfile}" ] then - awk -F "." '{for(f=NF;f > 1;f--) printf "%s.", $f;print $1}' "${adb_tmpfile}" 2>/dev/null | sort -u > "${adb_tmpload}" - awk '{if(NR==1){tld=$NF};while(getline){if($NF !~ tld"\\."){print tld;tld=$NF}}print tld}' "${adb_tmpload}" 2>/dev/null > "${adb_tmpfile}" - awk -F "." '{for(f=NF;f > 1;f--) printf "%s.", $f;print $1}' "${adb_tmpfile}" 2>/dev/null > "${adb_tmpload}" - mv -f "${adb_tmpload}" "${adb_tmpfile}" - f_list backup + f_tldcompression "${adb_tmpfile}" + if [ "${src_name}" != "blacklist" ] + then + f_list backup + fi else f_list restore fi @@ -521,11 +609,11 @@ f_main() f_list restore fi - # remove whitelist domains, final list preparation + # list merge # if [ ${adb_rc} -eq 0 ] && [ -s "${adb_tmpfile}" ] then - f_list format + f_list merge if [ ${adb_rc} -ne 0 ] then f_list remove @@ -535,37 +623,36 @@ f_main() fi done - # overall sort + # hash preparation, whitelist removal and overall sort # - if [ ${mem_total} -ge 64 ] || [ ${adb_forcesrt} -eq 1 ] + if [ -f "${adb_dnsdir}/${adb_dnsfile}" ] then - if [ -s "${adb_tmpdir}/${adb_dnsfile}" ] + hash_old="$(sha256sum "${adb_dnsdir}/${adb_dnsfile}" 2>/dev/null | awk '{print $1}')" + fi + if [ -s "${adb_tmpdir}/${adb_dnsfile}" ] + then + if [ ${mem_total} -ge 64 ] || [ ${adb_forcesrt} -eq 1 ] then - sort -u "${adb_tmpdir}/${adb_dnsfile}" > "${adb_dnsdir}/${adb_dnsfile}" - else - mv -f "${adb_tmpdir}/${adb_dnsfile}" "${adb_dnsdir}" 2>/dev/null + f_tldcompression "${adb_tmpdir}/${adb_dnsfile}" fi - cnt="$(wc -l < "${adb_dnsdir}/${adb_dnsfile}")" + f_list format + else + > "${adb_dnsdir}/${adb_dnsfile}" fi + chown "${adb_dnsuser}" "${adb_dnsdir}/${adb_dnsfile}" 2>/dev/null + f_rmtemp - # restart the dns backend and export runtime information + # conditional restart of the dns backend and runtime information export # - chown "${adb_dns}":"${adb_dns}" "${adb_dnsdir}/${adb_dnsfile}" 2>/dev/null - f_rmtemp - f_dnsrestart + hash_new="$(sha256sum "${adb_dnsdir}/${adb_dnsfile}" 2>/dev/null | awk '{print $1}')" + if [ -z "${hash_old}" ] || [ -z "${hash_new}" ] || [ "${hash_old}" != "${hash_new}" ] + then + f_dnsrestart + fi if [ ${?} -eq 0 ] then - json_init - json_add_object "data" - json_add_string "adblock_version" "${adb_ver}" - json_add_string "blocked_domains" "${cnt}" - json_add_string "fetch_info" "${adb_fetchinfo}" - json_add_string "dns_backend" "${adb_dns}" - json_add_string "last_rundate" "$(/bin/date "+%d.%m.%Y %H:%M:%S")" - json_add_string "system" "${adb_sysver}" - json_close_object - json_dump > "${adb_rtfile}" - f_log "info " "block list with overall ${cnt} domains loaded successfully (${adb_sysver})" + f_jsnupdate "${adb_cnt}" + f_log "info " "block list with overall ${adb_cnt} domains loaded successfully (${adb_sysver})" else f_log "error" "dns backend restart with active block list failed" fi