banip: release 0.8.4-1
authorDirk Brenken <dev@brenken.org>
Sun, 23 Apr 2023 20:03:09 +0000 (22:03 +0200)
committerDirk Brenken <dev@brenken.org>
Sun, 23 Apr 2023 20:03:09 +0000 (22:03 +0200)
* add support for a custom feeds file (/etc/banip/banip.custom.feeds). Add new or edit existing banIP feeds on your own with the integrated custom feed editor (LuCI-component
* add a new option 'ban_blockpolicy' to overrule the default bblock policy (block all chains), see readme for details
* change the feed file format and add a new ipthreat feed, see readme
* refine (debug) logging
* multiple small fixes and improvements
* readme update
* luci update (separate commit)

Signed-off-by: Dirk Brenken <dev@brenken.org>
net/banip/Makefile
net/banip/files/README.md
net/banip/files/banip-functions.sh
net/banip/files/banip-service.sh
net/banip/files/banip.custom.feeds [new file with mode: 0644]
net/banip/files/banip.feeds

index eabaadb58eadb17b908ec6fd60f4236bb156fc75..dbee9b9926de31b9b8affe09a6102eaf7cb08c64 100644 (file)
@@ -7,8 +7,8 @@
 include $(TOPDIR)/rules.mk
 
 PKG_NAME:=banip
-PKG_VERSION:=0.8.3
-PKG_RELEASE:=2
+PKG_VERSION:=0.8.4
+PKG_RELEASE:=1
 PKG_LICENSE:=GPL-3.0-or-later
 PKG_MAINTAINER:=Dirk Brenken <dev@brenken.org>
 
@@ -33,6 +33,7 @@ define Package/banip/conffiles
 /etc/config/banip
 /etc/banip/banip.allowlist
 /etc/banip/banip.blocklist
+/etc/banip/banip.custom.feeds
 endef
 
 define Build/Prepare
@@ -63,6 +64,7 @@ define Package/banip/install
        $(INSTALL_CONF) ./files/banip.blocklist $(1)/etc/banip
        $(INSTALL_CONF) ./files/banip.countries $(1)/etc/banip
        $(INSTALL_CONF) ./files/banip.feeds $(1)/etc/banip
+       $(INSTALL_CONF) ./files/banip.custom.feeds $(1)/etc/banip
 endef
 
 $(eval $(call BuildPackage,banip))
index f4116889b71fa03e1f1db7322fa642f2367f52ed..00cb83f5b04df87869c1d5c9db7a9d8acba394c1 100644 (file)
@@ -7,7 +7,7 @@ IP address blocking is commonly used to protect against brute force attacks, pre
 
 ## Main Features
 * banIP supports the following fully pre-configured domain blocklist feeds (free for private usage, for commercial use please check their individual licenses).  
-  **Please note:** the columns "WAN-INP", "WAN-FWD" and "LAN-FWD" show for which chains the feeds are suitable in common scenarios, e.g. the first entry should be limited to the LAN forward chain - see the config options 'ban\_blockinput', 'ban\_blockforwardwan' and 'ban\_blockforwardlan' below.  
+  **Please note:** By default every feed blocks all supported chains. The columns "WAN-INP", "WAN-FWD" and "LAN-FWD" show for which chains the feeds are suitable in common scenarios, e.g. the first entry should be limited to the LAN forward chain - see the config options 'ban\_blockpolicy', 'ban\_blockinput', 'ban\_blockforwardwan' and 'ban\_blockforwardlan' below.  
 
 | Feed                | Focus                          | WAN-INP | WAN-FWD | LAN-FWD | Information                                                   |
 | :------------------ | :----------------------------- | :-----: | :-----: | :-----: | :-----------------------------------------------------------  |
@@ -34,6 +34,7 @@ IP address blocking is commonly used to protect against brute force attacks, pre
 | greensnow           | suspicious server IPs          |    x    |    x    |         | [Link](https://greensnow.co)                                  |
 | iblockads           | Advertising IPs                |         |         |    x    | [Link](https://www.iblocklist.com)                            |
 | iblockspy           | Malicious spyware IPs          |    x    |    x    |         | [Link](https://www.iblocklist.com)                            |
+| ipthreat            | hacker and botnet TPs          |    x    |    x    |         | [Link](https://ipthreat.net)                                  |
 | myip                | real-time IP blocklist         |    x    |    x    |         | [Link](https://myip.ms)                                       |
 | nixspam             | iX spam protection             |    x    |    x    |         | [Link](http://www.nixspam.org)                                |
 | oisdbig             | OISD-big IPs                   |         |         |    x    | [Link](https://github.com/dibdot/banIP-IP-blocklists)         |
@@ -77,7 +78,7 @@ IP address blocking is commonly used to protect against brute force attacks, pre
 * Minimal status & error logging to syslog, enable debug logging to receive more output
 * Procd based init system support (start/stop/restart/reload/status/report/search/survey/lookup)
 * Procd network interface trigger support
-* Ability to add new banIP feeds on your own
+* Add new or edit existing banIP feeds on your own with the integrated custom feed editor
 
 ## Prerequisites
 * **[OpenWrt](https://openwrt.org)**, latest stable release or a snapshot with nft/firewall 4 support
@@ -123,56 +124,57 @@ Available commands:
 
 ## banIP config options
 
-| Option                  | Type   | Default                       | Description                                                                           |
-| :---------------------- | :----- | :---------------------------- | :------------------------------------------------------------------------------------ |
-| ban_enabled             | option | 0                             | enable the banIP service                                                              |
-| ban_nicelimit           | option | 0                             | ulimit nice level of the banIP service (range 0-19)                                   |
-| ban_filelimit           | option | 1024                          | ulimit max open/number of files (range 1024-4096)                                     |
-| ban_loglimit            | option | 100                           | the logread monitor scans only the last n lines of the logfile                        |
-| ban_logcount            | option | 1                             | how many times the IP must appear in the log to be considered as suspicious           |
-| ban_logterm             | list   | regex                         | various regex for logfile parsing (default: dropbear, sshd, luci, nginx, asterisk)    |
-| ban_autodetect          | option | 1                             | auto-detect wan interfaces, devices and subnets                                       |
-| ban_debug               | option | 0                             | enable banIP related debug logging                                                    |
-| ban_loginput            | option | 1                             | log drops in the wan-input chain                                                      |
-| ban_logforwardwan       | option | 1                             | log drops in the wan-forward chain                                                    |
-| ban_logforwardlan       | option | 0                             | log rejects in the lan-forward chain                                                  |
-| ban_autoallowlist       | option | 1                             | add wan IPs/subnets automatically to the local allowlist                              |
-| ban_autoblocklist       | option | 1                             | add suspicious attacker IPs automatically to the local blocklist                      |
-| ban_allowlistonly       | option | 0                             | restrict the internet access from/to a small number of secure websites/IPs            |
-| ban_basedir             | option | /tmp                          | base working directory while banIP processing                                         |
-| ban_reportdir           | option | /tmp/banIP-report             | directory where banIP stores the report files                                         |
-| ban_backupdir           | option | /tmp/banIP-backup             | directory where banIP stores the compressed backup files                              |
-| ban_protov4             | option | - / autodetect                | enable IPv4 support                                                                   |
-| ban_protov6             | option | - / autodetect                | enable IPv4 support                                                                   |
-| ban_ifv4                | list   | - / autodetect                | logical wan IPv4 interfaces, e.g. 'wan'                                               |
-| ban_ifv6                | list   | - / autodetect                | logical wan IPv6 interfaces, e.g. 'wan6'                                              |
-| ban_dev                 | list   | - / autodetect                | wan device(s), e.g. 'eth2'                                                            |
-| ban_trigger             | list   | -                             | logical startup trigger interface(s), e.g. 'wan'                                      |
-| ban_triggerdelay        | option | 10                            | trigger timeout before banIP processing begins                                        |
-| ban_triggeraction       | option | start                         | trigger action on ifup events, e.g. start, restart or reload                          |
-| ban_deduplicate         | option | 1                             | deduplicate IP addresses across all active sets                                       |
-| ban_splitsize           | option | 0                             | split ext. sets after every n lines/members (saves RAM)                               |
-| ban_cores               | option | - / autodetect                | limit the cpu cores used by banIP (saves RAM)                                         |
-| ban_nftloglevel         | option | warn                          | nft loglevel, values: emerg, alert, crit, err, warn, notice, info, debug, audit       |
-| ban_nftpriority         | option | -200                          | nft priority for the banIP table (default is the prerouting table priority)           |
-| ban_nftpolicy           | option | memory                        | nft policy for banIP-related sets, values: memory, performance                        |
-| ban_nftexpiry           | option | -                             | expiry time for auto added blocklist members, e.g. '5m', '2h' or '1d'                 |
-| ban_feed                | list   | -                             | external download feeds, e.g. 'yoyo', 'doh', 'country' or 'talos' (see feed table)    |
-| ban_asn                 | list   | -                             | ASNs for the 'asn' feed, e.g.'32934'                                                  |
-| ban_country             | list   | -                             | country iso codes for the 'country' feed, e.g. 'ru'                                   |
-| ban_blockinput          | list   | -                             | limit a feed to the wan-input chain, e.g. 'country'                                   |
-| ban_blockforwardwan     | list   | -                             | limit a feed to the wan-forward chain, e.g. 'debl'                                    |
-| ban_blockforwardlan     | list   | -                             | limit a feed to the lan-forward chain, e.g. 'doh'                                     |
-| ban_fetchcmd            | option | - / autodetect                | 'uclient-fetch', 'wget', 'curl' or 'aria2c'                                           |
-| ban_fetchparm           | option | - / autodetect                | set the config options for the selected download utility                              |
-| ban_fetchinsecure       | option | 0                             | don't check SSL server certificates during download                                   |
-| ban_mailreceiver        | option | -                             | receiver address for banIP related notification E-Mails                               |
-| ban_mailsender          | option | no-reply@banIP                | sender address for banIP related notification E-Mails                                 |
-| ban_mailtopic           | option | banIP notification            | topic for banIP related notification E-Mails                                          |
-| ban_mailprofile         | option | ban_notify                    | mail profile used in 'msmtp' for banIP related notification E-Mails                   |
-| ban_mailnotification    | option | 0                             | receive E-Mail notifications with every banIP run                                     |
-| ban_reportelements      | option | 1                             | list set elements in the report, disable this to speed up the report significantly    |
-| ban_resolver            | option | -                             | external resolver used for DNS lookups                                                |
+| Option                  | Type   | Default                       | Description                                                                                   |
+| :---------------------- | :----- | :---------------------------- | :-------------------------------------------------------------------------------------------- |
+| ban_enabled             | option | 0                             | enable the banIP service                                                                      |
+| ban_nicelimit           | option | 0                             | ulimit nice level of the banIP service (range 0-19)                                           |
+| ban_filelimit           | option | 1024                          | ulimit max open/number of files (range 1024-4096)                                             |
+| ban_loglimit            | option | 100                           | scan only the last n log entries permanently. Set it to '0' to disable the monitor            |
+| ban_logcount            | option | 1                             | how many times the IP must appear in the log to be considered as suspicious                   |
+| ban_logterm             | list   | regex                         | various regex for logfile parsing (default: dropbear, sshd, luci, nginx, asterisk)            |
+| ban_autodetect          | option | 1                             | auto-detect wan interfaces, devices and subnets                                               |
+| ban_debug               | option | 0                             | enable banIP related debug logging                                                            |
+| ban_loginput            | option | 1                             | log drops in the wan-input chain                                                              |
+| ban_logforwardwan       | option | 1                             | log drops in the wan-forward chain                                                            |
+| ban_logforwardlan       | option | 0                             | log rejects in the lan-forward chain                                                          |
+| ban_autoallowlist       | option | 1                             | add wan IPs/subnets automatically to the local allowlist                                      |
+| ban_autoblocklist       | option | 1                             | add suspicious attacker IPs automatically to the local blocklist                              |
+| ban_allowlistonly       | option | 0                             | restrict the internet access from/to a small number of secure websites/IPs                    |
+| ban_basedir             | option | /tmp                          | base working directory while banIP processing                                                 |
+| ban_reportdir           | option | /tmp/banIP-report             | directory where banIP stores the report files                                                 |
+| ban_backupdir           | option | /tmp/banIP-backup             | directory where banIP stores the compressed backup files                                      |
+| ban_protov4             | option | - / autodetect                | enable IPv4 support                                                                           |
+| ban_protov6             | option | - / autodetect                | enable IPv4 support                                                                           |
+| ban_ifv4                | list   | - / autodetect                | logical wan IPv4 interfaces, e.g. 'wan'                                                       |
+| ban_ifv6                | list   | - / autodetect                | logical wan IPv6 interfaces, e.g. 'wan6'                                                      |
+| ban_dev                 | list   | - / autodetect                | wan device(s), e.g. 'eth2'                                                                    |
+| ban_trigger             | list   | -                             | logical startup trigger interface(s), e.g. 'wan'                                              |
+| ban_triggerdelay        | option | 10                            | trigger timeout before banIP processing begins                                                |
+| ban_triggeraction       | option | start                         | trigger action on ifup events, e.g. start, restart or reload                                  |
+| ban_deduplicate         | option | 1                             | deduplicate IP addresses across all active sets                                               |
+| ban_splitsize           | option | 0                             | split ext. sets after every n lines/members (saves RAM)                                       |
+| ban_cores               | option | - / autodetect                | limit the cpu cores used by banIP (saves RAM)                                                 |
+| ban_nftloglevel         | option | warn                          | nft loglevel, values: emerg, alert, crit, err, warn, notice, info, debug                      |
+| ban_nftpriority         | option | -200                          | nft priority for the banIP table (default is the prerouting table priority)                   |
+| ban_nftpolicy           | option | memory                        | nft policy for banIP-related sets, values: memory, performance                                |
+| ban_nftexpiry           | option | -                             | expiry time for auto added blocklist members, e.g. '5m', '2h' or '1d'                         |
+| ban_feed                | list   | -                             | external download feeds, e.g. 'yoyo', 'doh', 'country' or 'talos' (see feed table)            |
+| ban_asn                 | list   | -                             | ASNs for the 'asn' feed, e.g.'32934'                                                          |
+| ban_country             | list   | -                             | country iso codes for the 'country' feed, e.g. 'ru'                                           |
+| ban_blockpolicy         | option | -                             | limit the default block policy to a certain chain, e.g. 'input', 'forwardwan' or 'forwardlan' |
+| ban_blockinput          | list   | -                             | limit a feed to the wan-input chain, e.g. 'country'                                           |
+| ban_blockforwardwan     | list   | -                             | limit a feed to the wan-forward chain, e.g. 'debl'                                            |
+| ban_blockforwardlan     | list   | -                             | limit a feed to the lan-forward chain, e.g. 'doh'                                             |
+| ban_fetchcmd            | option | - / autodetect                | 'uclient-fetch', 'wget', 'curl' or 'aria2c'                                                   |
+| ban_fetchparm           | option | - / autodetect                | set the config options for the selected download utility                                      |
+| ban_fetchinsecure       | option | 0                             | don't check SSL server certificates during download                                           |
+| ban_mailreceiver        | option | -                             | receiver address for banIP related notification E-Mails                                       |
+| ban_mailsender          | option | no-reply@banIP                | sender address for banIP related notification E-Mails                                         |
+| ban_mailtopic           | option | banIP notification            | topic for banIP related notification E-Mails                                                  |
+| ban_mailprofile         | option | ban_notify                    | mail profile used in 'msmtp' for banIP related notification E-Mails                           |
+| ban_mailnotification    | option | 0                             | receive E-Mail notifications with every banIP run                                             |
+| ban_reportelements      | option | 1                             | list set elements in the report, disable this to speed up the report significantly            |
+| ban_resolver            | option | -                             | external resolver used for DNS lookups                                                        |
 
 ## Examples
 **banIP report information**  
@@ -343,8 +345,8 @@ password        <password>
 Finally add a valid E-Mail receiver address.
 
 **change existing banIP feeds or add a new one**  
-The banIP blocklist feeds are stored in an external JSON file '/etc/banip/banip.feeds'.  
-A valid JSON source object contains the following required information, e.g.:
+The banIP default blocklist feeds are stored in an external JSON file '/etc/banip/banip.feeds'. All custom changes should be stored in an external JSON file '/etc/banip/banip.custom.feeds' (empty by default). It's recommended to use the LuCI based Custom Feed Editor to make changes to this file.  
+A valid JSON source object contains the following information, e.g.:
 ```
        [...]
        "tor": {
@@ -352,13 +354,12 @@ A valid JSON source object contains the following required information, e.g.:
                "url_6": "https://raw.githubusercontent.com/SecOps-Institute/Tor-IP-Addresses/master/tor-exit-nodes.lst",
                "rule_4": "/^(([0-9]{1,3}\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\\/(1?[0-9]|2?[0-9]|3?[0-2]))?)$/{printf \"%s,\\n\",$1}",
                "rule_6": "/^(([0-9A-f]{0,4}:){1,7}[0-9A-f]{0,4}:?(\\/(1?[0-2][0-8]|[0-9][0-9]))?)$/{printf \"%s,\\n\",$1}",
-               "focus": "tor exit nodes",
-               "descurl": "https://github.com/SecOps-Institute/Tor-IP-Addresses"
+               "descr": "tor exit nodes",
+               "flag": ""
        },
        [...]
 ```
-Add an unique object name (no spaces, no special chars) and make the required changes: adapt at least the URL the regex to the new feed.  
-**Please note:** if you're going to add new feeds, **always** make a backup of your work, cause this file is always overwritten with the maintainers version on every banIP update.  
+Add an unique feed name (no spaces, no special chars) and make the required changes: adapt at least the URL, the regex and the description for a new feed. The flag is optional, currently only 'gz' is supported to process archive downloads.  
 
 ## Support
 Please join the banIP discussion in this [forum thread](https://forum.openwrt.org/t/banip-support-thread/16985) or contact me by mail <dev@brenken.org>
index 3b485e3bdf47c4ca6eb644136231f50f04b23d5f..18fd331d890ce094cbb827c6c7ec31d78c4fcb1c 100644 (file)
@@ -14,6 +14,7 @@ ban_basedir="/tmp"
 ban_backupdir="/tmp/banIP-backup"
 ban_reportdir="/tmp/banIP-report"
 ban_feedfile="/etc/banip/banip.feeds"
+ban_customfeedfile="/etc/banip/banip.custom.feeds"
 ban_allowlist="/etc/banip/banip.allowlist"
 ban_blocklist="/etc/banip/banip.blocklist"
 ban_mailtemplate="/etc/banip/banip.tpl"
@@ -55,6 +56,7 @@ ban_deduplicate="1"
 ban_splitsize="0"
 ban_autodetect="1"
 ban_feed=""
+ban_blockpolicy=""
 ban_blockinput=""
 ban_blockforwardwan=""
 ban_blockforwardlan=""
@@ -93,8 +95,6 @@ f_system() {
                [ "${core}" = "0" ] && core="1"
                ban_cores="$((cpu * core))"
        fi
-
-       f_log "debug" "f_system  ::: system: ${ban_sysver:-"n/a"}, version: ${ban_ver:-"n/a"}, memory: ${ban_memory:-"0"}, cpu_cores: ${ban_cores}"
 }
 
 # create directories
@@ -146,7 +146,13 @@ f_rmdir() {
 f_char() {
        local char="${1}"
 
-       [ "${char}" = "1" ] && printf "%s" "✔" || printf "%s" "✘"
+       if [ "${char}" = "1" ]; then
+               printf "%s" "✔"
+       elif [ "${char}" = "0" ] || [ -z "${char}" ]; then
+               printf "%s" "✘"
+       else
+               printf "%s" "${char}"
+       fi
 }
 
 # trim strings
@@ -425,6 +431,22 @@ f_getsub() {
        f_log "debug" "f_getsub  ::: auto/update: ${ban_autoallowlist}/${update}, subnet(s): ${ban_sub:-"-"}"
 }
 
+# get feed information
+#
+f_getfeed() {
+       json_init
+       if [ -s "${ban_customfeedfile}" ]; then
+               if ! json_load_file "${ban_customfeedfile}" >/dev/null 2>&1; then
+                       f_log "info" "banIP custom feed file can't be loaded"
+                       if ! json_load_file "${ban_feedfile}" >/dev/null 2>&1; then
+                               f_log "err" "banIP feed file can't be loaded"
+                       fi
+               fi
+       elif ! json_load_file "${ban_feedfile}" >/dev/null 2>&1; then
+               f_log "err" "banIP feed file can't be loaded"
+       fi
+}
+
 # get set elements
 #
 f_getelements() {
@@ -500,8 +522,27 @@ f_down() {
        [ "${ban_logforwardwan}" = "1" ] && log_forwardwan="log level ${ban_nftloglevel} prefix \"banIP/fwd-wan/drp/${feed}: \""
        [ "${ban_logforwardlan}" = "1" ] && log_forwardlan="log level ${ban_nftloglevel} prefix \"banIP/fwd-lan/rej/${feed}: \""
 
-       # set source block direction
+       # set feed block direction
        #
+       if [ "${ban_blockpolicy}" = "input" ]; then
+               if ! printf "%s" "${ban_blockinput}" | "${ban_grepcmd}" -q "${feed%v*}" &&
+                       ! printf "%s" "${ban_blockforwardwan}" | "${ban_grepcmd}" -q "${feed%v*}" &&
+                       ! printf "%s" "${ban_blockforwardlan}" | "${ban_grepcmd}" -q "${feed%v*}"; then
+                       ban_blockinput="${ban_blockinput} ${feed%v*}"
+               fi
+       elif [ "${ban_blockpolicy}" = "forwardwan" ]; then
+               if ! printf "%s" "${ban_blockinput}" | "${ban_grepcmd}" -q "${feed%v*}" &&
+                       ! printf "%s" "${ban_blockforwardwan}" | "${ban_grepcmd}" -q "${feed%v*}" &&
+                       ! printf "%s" "${ban_blockforwardlan}" | "${ban_grepcmd}" -q "${feed%v*}"; then
+                       ban_blockforwardwan="${ban_blockforwardwan} ${feed%v*}"
+               fi
+       elif [ "${ban_blockpolicy}" = "forwardlan" ]; then
+               if ! printf "%s" "${ban_blockinput}" | "${ban_grepcmd}" -q "${feed%v*}" &&
+                       ! printf "%s" "${ban_blockforwardwan}" | "${ban_grepcmd}" -q "${feed%v*}" &&
+                       ! printf "%s" "${ban_blockforwardlan}" | "${ban_grepcmd}" -q "${feed%v*}"; then
+                       ban_blockforwardlan="${ban_blockforwardlan} ${feed%v*}"
+               fi
+       fi
        if printf "%s" "${ban_blockinput}" | "${ban_grepcmd}" -q "${feed%v*}"; then
                feed_direction="input"
        fi
@@ -596,7 +637,7 @@ f_down() {
                                fi
                        fi
                } >"${tmp_nft}"
-               feed_rc="${?}"
+               feed_rc="0"
        elif [ "${feed%v*}" = "blocklist" ]; then
                {
                        printf "%s\n\n" "#!/usr/sbin/nft -f"
@@ -637,7 +678,7 @@ f_down() {
                                [ -z "${feed_direction##*forwardlan*}" ] && printf "%s\n" "add rule inet banIP lan-forward ip6 daddr @${feed} ${log_forwardlan} counter reject with icmpv6 type admin-prohibited"
                        fi
                } >"${tmp_nft}"
-               feed_rc="${?}"
+               feed_rc="0"
        # handle external downloads
        #
        elif [ "${restore_rc}" != "0" ] && [ "${feed_url}" != "local" ]; then
@@ -712,7 +753,7 @@ f_down() {
                        if [ -n "${ban_splitsize//[![:digit]]/}" ] && [ "${ban_splitsize//[![:digit]]/}" -gt "0" ]; then
                                if ! "${ban_awkcmd}" "NR%${ban_splitsize//[![:digit]]/}==1{file=\"${tmp_file}.\"++i;}{ORS=\" \";print > file}" "${tmp_split}" 2>/dev/null; then
                                        rm -f "${tmp_file}".*
-                                       f_log "info" "failed to split ${feed} set to size '${ban_splitsize//[![:digit]]/}'"
+                                       f_log "info" "failed to split '${feed}' Set to size '${ban_splitsize//[![:digit]]/}'"
                                fi
                        else
                                "${ban_awkcmd}" '{ORS=" ";print}' "${tmp_split}" 2>/dev/null >"${tmp_file}.1"
@@ -768,7 +809,7 @@ f_down() {
                                                continue
                                        fi
                                        if ! "${ban_nftcmd}" add element inet banIP "${feed}" "{ $(cat "${split_file}") }" >/dev/null 2>&1; then
-                                               f_log "info" "failed to add split file '${split_file##*.}' to ${feed} set"
+                                               f_log "info" "failed to add split file '${split_file##*.}' to '${feed}' Set"
                                        fi
                                        rm -f "${split_file}"
                                done
@@ -777,7 +818,7 @@ f_down() {
                                fi
                        fi
                else
-                       f_log "info" "empty feed ${feed} will be skipped"
+                       f_log "info" "empty feed '${feed}' will be skipped"
                fi
        fi
        rm -f "${tmp_split}" "${tmp_nft}"
@@ -817,15 +858,18 @@ f_restore() {
 # remove disabled feeds
 #
 f_rmset() {
-       local tmp_del ruleset_raw table_sets handle set del_set feed_log feed_rc
+       local feedlist tmp_del ruleset_raw table_sets handle set del_set feed_log feed_rc
 
+       f_getfeed
+       json_get_keys feedlist
        tmp_del="${ban_tmpfile}.final.delete"
        ruleset_raw="$("${ban_nftcmd}" -tj list ruleset 2>/dev/null)"
        table_sets="$(printf "%s\n" "${ruleset_raw}" | jsonfilter -qe '@.nftables[@.set.table="banIP"].set.name')"
        {
                printf "%s\n\n" "#!/usr/sbin/nft -f"
                for set in ${table_sets}; do
-                       if ! printf "%s" "allowlist blocklist ${ban_feed}" | "${ban_grepcmd}" -q "${set%v*}"; then
+                       if ! printf "%s" "allowlist blocklist ${ban_feed}" | "${ban_grepcmd}" -q "${set%v*}" ||
+                               ! printf "%s" "allowlist blocklist ${feedlist}" | "${ban_grepcmd}" -q "${set%v*}"; then
                                del_set="${del_set}${set}, "
                                rm -f "${ban_backupdir}/banIP.${set}.gz"
                                printf "%s\n" "flush set inet banIP ${set}"
@@ -853,7 +897,7 @@ f_rmset() {
 # generate status information
 #
 f_genstatus() {
-       local object duration set table_sets cnt_elements="0" split="0" status="${1}"
+       local object duration set table_sets cnt_elements="0" custom="0" split="0" status="${1}"
 
        [ -z "${ban_dev}" ] && f_conf
        if [ "${status}" = "active" ]; then
@@ -869,6 +913,7 @@ f_genstatus() {
                fi
                runtime="action: ${ban_action:-"-"}, duration: ${duration:-"-"}, date: $(date "+%Y-%m-%d %H:%M:%S")"
        fi
+       [ -s ${ban_customfeedfile} ] && custom="1"
        [ ${ban_splitsize:-"0"} -gt "0" ] && split="1"
 
        : >"${ban_rtfile}"
@@ -878,51 +923,33 @@ f_genstatus() {
        json_add_string "version" "${ban_ver}"
        json_add_string "element_count" "${cnt_elements}"
        json_add_array "active_feeds"
-       if [ "${status}" != "active" ]; then
+       for object in ${table_sets:-"-"}; do
                json_add_object
-               json_add_string "feed" "-"
+               json_add_string "feed" "${object}"
                json_close_object
-       else
-               for object in ${table_sets}; do
-                       json_add_object
-                       json_add_string "feed" "${object}"
-                       json_close_object
-               done
-       fi
+       done
        json_close_array
        json_add_array "active_devices"
-       if [ "${status}" != "active" ]; then
+       for object in ${ban_dev:-"-"}; do
                json_add_object
-               json_add_string "device" "-"
+               json_add_string "device" "${object}"
                json_close_object
-       else
-               for object in ${ban_dev}; do
-                       json_add_object
-                       json_add_string "device" "${object}"
-                       json_close_object
-               done
-               for object in ${ban_ifv4} ${ban_ifv6}; do
-                       json_add_object
-                       json_add_string "interface" "${object}"
-                       json_close_object
-               done
-       fi
+       done
+       for object in ${ban_ifv4:-"-"} ${ban_ifv6:-"-"}; do
+               json_add_object
+               json_add_string "interface" "${object}"
+               json_close_object
+       done
        json_close_array
        json_add_array "active_subnets"
-       if [ "${status}" != "active" ]; then
+       for object in ${ban_sub:-"-"}; do
                json_add_object
-               json_add_string "subnet" "-"
+               json_add_string "subnet" "${object}"
                json_close_object
-       else
-               for object in ${ban_sub}; do
-                       json_add_object
-                       json_add_string "subnet" "${object}"
-                       json_close_object
-               done
-       fi
+       done
        json_close_array
        json_add_string "nft_info" "priority: ${ban_nftpriority}, policy: ${ban_nftpolicy}, loglevel: ${ban_nftloglevel}, expiry: ${ban_nftexpiry:-"-"}"
-       json_add_string "run_info" "base: ${ban_basedir}, backup: ${ban_backupdir}, report: ${ban_reportdir}, feed: ${ban_feedfile}"
+       json_add_string "run_info" "base: ${ban_basedir}, backup: ${ban_backupdir}, report: ${ban_reportdir}, feed/custom: ${ban_feedfile}/$(f_char ${custom})"
        json_add_string "run_flags" "auto: $(f_char ${ban_autodetect}), proto (4/6): $(f_char ${ban_protov4})/$(f_char ${ban_protov6}), log (wan-inp/wan-fwd/lan-fwd): $(f_char ${ban_loginput})/$(f_char ${ban_logforwardwan})/$(f_char ${ban_logforwardlan}), dedup: $(f_char ${ban_deduplicate}), split: $(f_char ${split}), allowed only: $(f_char ${ban_allowlistonly})"
        json_add_string "last_run" "${runtime:-"-"}"
        json_add_string "system_info" "cores: ${ban_cores}, memory: ${ban_memory}, device: ${ban_sysver}"
@@ -1024,18 +1051,18 @@ f_lookup() {
        done
        if [ -n "${elementsv4}" ]; then
                if ! "${ban_nftcmd}" add element inet banIP "${feed}v4" "{ ${elementsv4} }" >/dev/null 2>&1; then
-                       f_log "info" "failed to add lookup file to ${feed}v4 set"
+                       f_log "info" "failed to add lookup file to '${feed}v4' Set"
                fi
        fi
        if [ -n "${elementsv6}" ]; then
                if ! "${ban_nftcmd}" add element inet banIP "${feed}v6" "{ ${elementsv6} }" >/dev/null 2>&1; then
-                       f_log "info" "failed to add lookup file to ${feed}v6 set"
+                       f_log "info" "failed to add lookup file to '${feed}v6' Set"
                fi
        fi
        end_time="$(date "+%s")"
        duration="$(((end_time - start_time) / 60))m $(((end_time - start_time) % 60))s"
 
-       f_log "info" "Lookup summary for the local ${feed}: Domains processed: ${cnt_domain}, IPs added: ${cnt_ip}, Duration: ${duration}"
+       f_log "debug" "feed: ${feed}, domains: ${cnt_domain}, IPs: ${cnt_ip}, duration: ${duration}"
 }
 
 # table statistics
index 10f0b974781dad83eca74cac881f68896f48c5d8..ed2b9914c9a84ce5817e549e69078e07c483e0f3 100755 (executable)
@@ -15,6 +15,7 @@ ban_funlib="/usr/lib/banip-functions.sh"
 #
 f_conf
 f_log "info" "start banIP processing (${ban_action})"
+f_log "debug" "f_system  ::: system: ${ban_sysver:-"n/a"}, version: ${ban_ver:-"n/a"}, memory: ${ban_memory:-"0"}, cpu_cores: ${ban_cores}"
 f_genstatus "processing"
 f_tmp
 f_fetch
@@ -58,10 +59,7 @@ f_log "info" "start banIP download processes"
 if [ "${ban_allowlistonly}" = "1" ]; then
        ban_feed=""
 else
-       json_init
-       if ! json_load_file "${ban_feedfile}" >/dev/null 2>&1; then
-               f_log "err" "banIP feed file can't be loaded"
-       fi
+       f_getfeed
        [ "${ban_deduplicate}" = "1" ] && printf "\n" >"${ban_tmpfile}.deduplicate"
 fi
 
@@ -82,9 +80,12 @@ for feed in allowlist ${ban_feed} blocklist; do
                continue
        fi
 
-       # read external feed information
+       # external feeds
        #
        if ! json_select "${feed}" >/dev/null 2>&1; then
+               f_log "info" "unknown feed '${feed}' will be removed"
+               uci_remove_list banip global ban_feed "${feed}"
+               uci_commit "banip"
                continue
        fi
        json_objects="url_4 rule_4 url_6 rule_6 flag"
@@ -92,6 +93,16 @@ for feed in allowlist ${ban_feed} blocklist; do
                eval json_get_var feed_"${object}" '${object}' >/dev/null 2>&1
        done
        json_select ..
+       
+       # skip incomplete feeds
+       #
+       if { { [ -n "${feed_url_4}" ] && [ -z "${feed_rule_4}" ]; } || { [ -z "${feed_url_4}" ] && [ -n "${feed_rule_4}" ]; }; } ||
+               { { [ -n "${feed_url_6}" ] && [ -z "${feed_rule_6}" ]; } || { [ -z "${feed_url_6}" ] && [ -n "${feed_rule_6}" ]; }; } ||
+               { [ -z "${feed_url_4}" ] && [ -z "${feed_rule_4}" ] && [ -z "${feed_url_6}" ] && [ -z "${feed_rule_6}" ]; }; then
+               f_log "info" "incomplete feed '${feed}' will be skipped"
+               continue
+       fi
+
        # handle IPv4/IPv6 feeds with the same/single download URL
        #
        if [ "${feed_url_4}" = "${feed_url_6}" ]; then
@@ -149,11 +160,12 @@ if [ "${ban_mailnotification}" = "1" ] && [ -n "${ban_mailreceiver}" ] && [ -x "
                f_mail
        ) &
 fi
+json_cleanup
 rm -rf "${ban_lock}"
 
 # start detached log service
 #
-if [ -x "${ban_logreadcmd}" ] && [ -n "${ban_logterm%%??}" ]; then
+if [ -x "${ban_logreadcmd}" ] && [ -n "${ban_logterm%%??}" ] && [ "${ban_loglimit}" != "0" ]; then
        f_log "info" "start detached banIP log service"
 
        nft_expiry="$(printf "%s" "${ban_nftexpiry}" | grep -oE "([0-9]+[h|m|s]$)")"
@@ -197,7 +209,7 @@ if [ -x "${ban_logreadcmd}" ] && [ -n "${ban_logterm%%??}" ]; then
 # start detached no-op service loop
 #
 else
-       f_log "info" "start detached no-op banIP service (logterms are missing)"
+       f_log "info" "start detached no-op banIP service"
        while :; do
                sleep 1
        done
diff --git a/net/banip/files/banip.custom.feeds b/net/banip/files/banip.custom.feeds
new file mode 100644 (file)
index 0000000..e69de29
index 527c844e302ad59fd9ab6a595208afc5e47d0807..90f6d63be48e4c25adfffdf099d8ba8b7ce9fe90 100644 (file)
@@ -4,46 +4,40 @@
                "url_6": "https://raw.githubusercontent.com/dibdot/banIP-IP-blocklists/main/adaway-ipv6.txt",
                "rule_4": "/^(([0-9]{1,3}\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\\/(1?[0-9]|2?[0-9]|3?[0-2]))?)[[:space:]]/{printf \"%s,\\n\",$1}",
                "rule_6": "/^(([0-9A-f]{0,4}:){1,7}[0-9A-f]{0,4}:?(\\/(1?[0-2][0-8]|[0-9][0-9]))?)[[:space:]]/{printf \"%s,\\n\",$1}",
-               "focus": "adaway IPs",
-               "descurl": "https://github.com/dibdot/banIP-IP-blocklists"
+               "descr": "adaway IPs"
        },
        "adguard": {
                "url_4": "https://raw.githubusercontent.com/dibdot/banIP-IP-blocklists/main/adguard-ipv4.txt",
                "url_6": "https://raw.githubusercontent.com/dibdot/banIP-IP-blocklists/main/adguard-ipv6.txt",
                "rule_4": "/^(([0-9]{1,3}\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\\/(1?[0-9]|2?[0-9]|3?[0-2]))?)[[:space:]]/{printf \"%s,\\n\",$1}",
                "rule_6": "/^(([0-9A-f]{0,4}:){1,7}[0-9A-f]{0,4}:?(\\/(1?[0-2][0-8]|[0-9][0-9]))?)[[:space:]]/{printf \"%s,\\n\",$1}",
-               "focus": "adguard IPs",
-               "descurl": "https://github.com/dibdot/banIP-IP-blocklists"
+               "descr": "adguard IPs"
        },
        "adguardtrackers": {
                "url_4": "https://raw.githubusercontent.com/dibdot/banIP-IP-blocklists/main/adguardtrackers-ipv4.txt",
                "url_6": "https://raw.githubusercontent.com/dibdot/banIP-IP-blocklists/main/adguardtrackers-ipv6.txt",
                "rule_4": "/^(([0-9]{1,3}\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\\/(1?[0-9]|2?[0-9]|3?[0-2]))?)[[:space:]]/{printf \"%s,\\n\",$1}",
                "rule_6": "/^(([0-9A-f]{0,4}:){1,7}[0-9A-f]{0,4}:?(\\/(1?[0-2][0-8]|[0-9][0-9]))?)[[:space:]]/{printf \"%s,\\n\",$1}",
-               "focus": "adguardtracker IPs",
-               "descurl": "https://github.com/dibdot/banIP-IP-blocklists"
+               "descr": "adguardtracker IPs"
        },
        "antipopads": {
                "url_4": "https://raw.githubusercontent.com/dibdot/banIP-IP-blocklists/main/antipopads-ipv4.txt",
                "url_6": "https://raw.githubusercontent.com/dibdot/banIP-IP-blocklists/main/antipopads-ipv6.txt",
                "rule_4": "/^(([0-9]{1,3}\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\\/(1?[0-9]|2?[0-9]|3?[0-2]))?)[[:space:]]/{printf \"%s,\\n\",$1}",
                "rule_6": "/^(([0-9A-f]{0,4}:){1,7}[0-9A-f]{0,4}:?(\\/(1?[0-2][0-8]|[0-9][0-9]))?)[[:space:]]/{printf \"%s,\\n\",$1}",
-               "focus": "antipopads IPs",
-               "descurl": "https://github.com/dibdot/banIP-IP-blocklists"
+               "descr": "antipopads IPs"
        },
        "asn": {
                "url_4": "https://asn.ipinfo.app/api/text/list/",
                "url_6": "https://asn.ipinfo.app/api/text/list/",
                "rule_4": "/^(([0-9]{1,3}\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\\/(1?[0-9]|2?[0-9]|3?[0-2]))?)$/{printf \"%s,\\n\",$1}",
                "rule_6": "/^(([0-9A-f]{0,4}:){1,7}[0-9A-f]{0,4}:?(\\/(1?[0-2][0-8]|[0-9][0-9]))?)$/{printf \"%s,\\n\",$1}",
-               "focus": "ASN IPs",
-               "descurl": "https://asn.ipinfo.app"
+               "descr": "ASN IPs"
        },
        "backscatterer": {
                "url_4": "http://wget-mirrors.uceprotect.net/rbldnsd-all/ips.backscatterer.org.gz",
                "rule_4": "/^(([0-9]{1,3}\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\\/(1?[0-9]|2?[0-9]|3?[0-2]))?)$/{printf \"%s,\\n\",$1}",
-               "focus": "backscatterer IPs",
-               "descurl": "https://www.uceprotect.net/en/index.php",
+               "descr": "backscatterer IPs",
                "flag": "gz"
        },
        "bogon": {
                "url_6": "https://www.team-cymru.org/Services/Bogons/fullbogons-ipv6.txt",
                "rule_4": "/^(([0-9]{1,3}\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\\/(1?[0-9]|2?[0-9]|3?[0-2]))?)$/{printf \"%s,\\n\",$1}",
                "rule_6": "/^(([0-9A-f]{0,4}:){1,7}[0-9A-f]{0,4}:?(\\/(1?[0-2][0-8]|[0-9][0-9]))?)$/{printf \"%s,\\n\",$1}",
-               "focus": "bogon prefixes",
-               "descurl": "https://team-cymru.com"
+               "descr": "bogon prefixes"
        },
        "cinsscore": {
                "url_4": "https://cinsscore.com/list/ci-badguys.txt",
                "rule_4": "/^(([0-9]{1,3}\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\\/(1?[0-9]|2?[0-9]|3?[0-2]))?)$/{printf \"%s,\\n\",$1}",
-               "focus": "suspicious attacker IPs",
-               "descurl": "https://cinsscore.com/#list"
+               "descr": "suspicious attacker IPs"
        },
        "country": {
                "url_4": "https://www.ipdeny.com/ipblocks/data/aggregated/",
                "url_6": "https://www.ipdeny.com/ipv6/ipaddresses/aggregated/",
                "rule_4": "/^(([0-9]{1,3}\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\\/(1?[0-9]|2?[0-9]|3?[0-2]))?)$/{printf \"%s,\\n\",$1}",
                "rule_6": "/^(([0-9A-f]{0,4}:){1,7}[0-9A-f]{0,4}:?(\\/(1?[0-2][0-8]|[0-9][0-9]))?)$/{printf \"%s,\\n\",$1}",
-               "focus": "country blocks",
-               "descurl": "http://www.ipdeny.com/ipblocks"
+               "descr": "country blocks"
        },
        "darklist": {
                "url_4": "https://darklist.de/raw.php",
                "rule_4": "/^(([0-9]{1,3}\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\\/(1?[0-9]|2?[0-9]|3?[0-2]))?)$/{printf \"%s,\\n\",$1}",
-               "focus": "suspicious attacker IPs",
-               "descurl": "https://darklist.de"
+               "descr": "suspicious attacker IPs"
        },
        "debl": {
                "url_4": "https://www.blocklist.de/downloads/export-ips_all.txt",
                "url_6": "https://www.blocklist.de/downloads/export-ips_all.txt",
                "rule_4": "/^(([0-9]{1,3}\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\\/(1?[0-9]|2?[0-9]|3?[0-2]))?)$/{printf \"%s,\\n\",$1}",
                "rule_6": "/^(([0-9A-f]{0,4}:){1,7}[0-9A-f]{0,4}:?(\\/(1?[0-2][0-8]|[0-9][0-9]))?)$/{printf \"%s,\\n\",$1}",
-               "focus": "fail2ban IP blacklist",
-               "descurl": "https://www.blocklist.de"
+               "descr": "fail2ban IP blacklist"
        },
        "doh": {
                "url_4": "https://raw.githubusercontent.com/dibdot/DoH-IP-blocklists/master/doh-ipv4.txt",
                "url_6": "https://raw.githubusercontent.com/dibdot/DoH-IP-blocklists/master/doh-ipv6.txt",
                "rule_4": "/^(([0-9]{1,3}\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\\/(1?[0-9]|2?[0-9]|3?[0-2]))?)[[:space:]]/{printf \"%s,\\n\",$1}",
                "rule_6": "/^(([0-9A-f]{0,4}:){1,7}[0-9A-f]{0,4}:?(\\/(1?[0-2][0-8]|[0-9][0-9]))?)[[:space:]]/{printf \"%s,\\n\",$1}",
-               "focus": "public DoH-Provider",
-               "descurl": "https://github.com/dibdot/DoH-IP-blocklists"
+               "descr": "public DoH-Provider"
        },
        "drop": {
                "url_4": "https://www.spamhaus.org/drop/drop.txt",
                "url_6": "https://www.spamhaus.org/drop/dropv6.txt",
                "rule_4": "/^(([0-9]{1,3}\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\\/(1?[0-9]|2?[0-9]|3?[0-2]))?)[[:space:]]/{printf \"%s,\\n\",$1}",
                "rule_6": "/^(([0-9A-f]{0,4}:){1,7}[0-9A-f]{0,4}:?(\\/(1?[0-2][0-8]|[0-9][0-9]))?)[[:space:]]/{printf \"%s,\\n\",$1}",
-               "focus": "spamhaus drop compilation",
-               "descurl": "https://www.spamhaus.org"
+               "descr": "spamhaus drop compilation"
        },
        "dshield": {
                "url_4": "https://feeds.dshield.org/block.txt",
                "rule_4": "/^(([0-9]{1,3}\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\\/(1?[0-9]|2?[0-9]|3?[0-2]))?)[[:space:]]/{printf \"%s/%s,\\n\",$1,$3}",
-               "focus": "dshield IP blocklist",
-               "descurl": "https://www.dshield.org"
+               "descr": "dshield IP blocklist"
        },
        "edrop": {
                "url_4": "https://www.spamhaus.org/drop/edrop.txt",
                "rule_4": "/^(([0-9]{1,3}\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\\/(1?[0-9]|2?[0-9]|3?[0-2]))?)[[:space:]]/{printf \"%s,\\n\",$1}",
-               "focus": "spamhaus edrop compilation",
-               "descurl": "https://www.spamhaus.org"
+               "descr": "spamhaus edrop compilation"
        },
        "feodo": {
                "url_4": "https://feodotracker.abuse.ch/downloads/ipblocklist.txt",
                "rule_4": "BEGIN{RS=\"\\r\\n\"}/^(([0-9]{1,3}\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\\/(1?[0-9]|2?[0-9]|3?[0-2]))?)$/{printf \"%s,\\n\",$1}",
-               "focus": "feodo tracker",
-               "descurl": "https://feodotracker.abuse.ch"
+               "descr": "feodo tracker"
        },
        "firehol1": {
                "url_4": "https://iplists.firehol.org/files/firehol_level1.netset",
                "rule_4": "/^(([0-9]{1,3}\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\\/(1?[0-9]|2?[0-9]|3?[0-2]))?)$/{printf \"%s,\\n\",$1}",
-               "focus": "firehol level 1 compilation",
-               "descurl": "https://iplists.firehol.org/?ipset=firehol_level1"
+               "descr": "firehol level 1 compilation"
        },
        "firehol2": {
                "url_4": "https://iplists.firehol.org/files/firehol_level2.netset",
                "rule_4": "/^(([0-9]{1,3}\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\\/(1?[0-9]|2?[0-9]|3?[0-2]))?)$/{printf \"%s,\\n\",$1}",
-               "focus": "firehol level 2 compilation",
-               "descurl": "https://iplists.firehol.org/?ipset=firehol_level2"
+               "descr": "firehol level 2 compilation"
        },
        "firehol3": {
                "url_4": "https://iplists.firehol.org/files/firehol_level3.netset",
                "rule_4": "/^(([0-9]{1,3}\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\\/(1?[0-9]|2?[0-9]|3?[0-2]))?)$/{printf \"%s,\\n\",$1}",
-               "focus": "firehol level 3 compilation",
-               "descurl": "https://iplists.firehol.org/?ipset=firehol_level3"
+               "descr": "firehol level 3 compilation"
        },
        "firehol4": {
                "url_4": "https://iplists.firehol.org/files/firehol_level4.netset",
                "rule_4": "/^(([0-9]{1,3}\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\\/(1?[0-9]|2?[0-9]|3?[0-2]))?)$/{if(!seen[$1]++)printf \"%s,\\n\",$1}",
-               "focus": "firehol level 4 compilation",
-               "descurl": "https://iplists.firehol.org/?ipset=firehol_level4"
+               "descr": "firehol level 4 compilation"
        },
        "greensnow": {
                "url_4": "https://blocklist.greensnow.co/greensnow.txt",
                "rule_4": "/^(([0-9]{1,3}\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\\/(1?[0-9]|2?[0-9]|3?[0-2]))?)$/{printf \"%s,\\n\",$1}",
-               "focus": "suspicious server IPs",
-               "descurl": "https://greensnow.co"
+               "descr": "suspicious server IPs"
        },
        "iblockads": {
                "url_4": "https://list.iblocklist.com/?list=dgxtneitpuvgqqcpfulq&fileformat=cidr&archiveformat=gz",
                "rule_4": "/^(([0-9]{1,3}\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\\/(1?[0-9]|2?[0-9]|3?[0-2]))?)$/{printf \"%s,\\n\",$1}",
-               "focus": "advertising IPs",
-               "descurl": "https://www.iblocklist.com",
-               "flag": "gz",
-               "login": true
+               "descr": "advertising IPs",
+               "flag": "gz"
        },
        "iblockspy": {
                "url_4": "https://list.iblocklist.com/?list=llvtlsjyoyiczbkjsxpf&fileformat=cidr&archiveformat=gz",
                "rule_4": "/^(([0-9]{1,3}\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\\/(1?[0-9]|2?[0-9]|3?[0-2]))?)$/{printf \"%s,\\n\",$1}",
-               "focus": "malicious spyware IPs",
-               "descurl": "https://www.iblocklist.com",
-               "flag": "gz",
-               "login": true
+               "descr": "malicious spyware IPs",
+               "flag": "gz"
+       },
+       "ipthreat": {
+               "url_4": "https://lists.ipthreat.net/file/ipthreat-lists/threat/threat-30.txt",
+               "rule_4": "/^(([0-9]{1,3}\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\\/(1?[0-9]|2?[0-9]|3?[0-2]))?)[-[:space:]]?/{printf \"%s,\\n\",$1}",
+               "descr": "hacker and botnet IPs"
        },
        "myip": {
                "url_4": "https://myip.ms/files/blacklist/general/latest_blacklist.txt",
                "url_6": "https://myip.ms/files/blacklist/general/latest_blacklist.txt",
                "rule_4": "/^(([0-9]{1,3}\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\\/(1?[0-9]|2?[0-9]|3?[0-2]))?)[[:space:]]/{printf \"%s,\\n\",$1}",
                "rule_6": "/^(([0-9A-f]{0,4}:){1,7}[0-9A-f]{0,4}:?(\\/(1?[0-2][0-8]|[0-9][0-9]))?)[[:space:]]/{printf \"%s,\\n\",$1}",
-               "focus": "real-time IP blocklist",
-               "descurl": "https://myip.ms"
+               "descr": "real-time IP blocklist"
        },
        "nixspam": {
                "url_4": "https://www.nixspam.net/download/nixspam-ip.dump.gz",
                "rule_4": "/(([0-9]{1,3}\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\\/(1?[0-9]|2?[0-9]|3?[0-2]))?)[[:space:]]/{printf \"%s,\\n\",$2}",
-               "focus": "iX spam protection",
-               "descurl": "https://www.nixspam.net",
+               "descr": "iX spam protection",
                "flag": "gz"
        },
        "oisdbig": {
                "url_6": "https://raw.githubusercontent.com/dibdot/banIP-IP-blocklists/main/oisdbig-ipv6.txt",
                "rule_4": "/^(([0-9]{1,3}\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\\/(1?[0-9]|2?[0-9]|3?[0-2]))?)[[:space:]]/{printf \"%s,\\n\",$1}",
                "rule_6": "/^(([0-9A-f]{0,4}:){1,7}[0-9A-f]{0,4}:?(\\/(1?[0-2][0-8]|[0-9][0-9]))?)[[:space:]]/{printf \"%s,\\n\",$1}",
-               "focus": "OISD-big IPs",
-               "descurl": "https://github.com/dibdot/banIP-IP-blocklists"
+               "descr": "OISD-big IPs"
        },
        "oisdnsfw": {
                "url_4": "https://raw.githubusercontent.com/dibdot/banIP-IP-blocklists/main/oisdnsfw-ipv4.txt",
                "url_6": "https://raw.githubusercontent.com/dibdot/banIP-IP-blocklists/main/oisdnsfw-ipv6.txt",
                "rule_4": "/^(([0-9]{1,3}\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\\/(1?[0-9]|2?[0-9]|3?[0-2]))?)[[:space:]]/{printf \"%s,\\n\",$1}",
                "rule_6": "/^(([0-9A-f]{0,4}:){1,7}[0-9A-f]{0,4}:?(\\/(1?[0-2][0-8]|[0-9][0-9]))?)[[:space:]]/{printf \"%s,\\n\",$1}",
-               "focus": "OISD-nsfw IPs",
-               "descurl": "https://github.com/dibdot/banIP-IP-blocklists"
+               "descr": "OISD-nsfw IPs"
        },
        "oisdsmall": {
                "url_4": "https://raw.githubusercontent.com/dibdot/banIP-IP-blocklists/main/oisdsmall-ipv4.txt",
                "url_6": "https://raw.githubusercontent.com/dibdot/banIP-IP-blocklists/main/oisdsmall-ipv6.txt",
                "rule_4": "/^(([0-9]{1,3}\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\\/(1?[0-9]|2?[0-9]|3?[0-2]))?)[[:space:]]/{printf \"%s,\\n\",$1}",
                "rule_6": "/^(([0-9A-f]{0,4}:){1,7}[0-9A-f]{0,4}:?(\\/(1?[0-2][0-8]|[0-9][0-9]))?)[[:space:]]/{printf \"%s,\\n\",$1}",
-               "focus": "OISD-small IPs",
-               "descurl": "https://github.com/dibdot/banIP-IP-blocklists"
+               "descr": "OISD-small IPs"
        },
        "proxy": {
                "url_4": "https://iplists.firehol.org/files/proxylists.ipset",
                "rule_4": "/^(([0-9]{1,3}\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\\/(1?[0-9]|2?[0-9]|3?[0-2]))?)$/{printf \"%s,\\n\",$1}",
-               "focus": "open proxies",
-               "descurl": "https://iplists.firehol.org/?ipset=proxylists"
+               "descr": "open proxies"
        },
        "sslbl": {
                "url_4": "https://sslbl.abuse.ch/blacklist/sslipblacklist.csv",
                "rule_4": "BEGIN{FS=\",\"}/(([0-9]{1,3}\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\\/(1?[0-9]|2?[0-9]|3?[0-2]))?)/{printf \"%s,\\n\",$2}",
-               "focus": "SSL botnet IPs",
-               "descurl": "https://sslbl.abuse.ch"
+               "descr": "SSL botnet IPs"
        },
        "stevenblack": {
                "url_4": "https://raw.githubusercontent.com/dibdot/banIP-IP-blocklists/main/stevenblack-ipv4.txt",
                "url_6": "https://raw.githubusercontent.com/dibdot/banIP-IP-blocklists/main/stevenblack-ipv6.txt",
                "rule_4": "/^(([0-9]{1,3}\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\\/(1?[0-9]|2?[0-9]|3?[0-2]))?)[[:space:]]/{printf \"%s,\\n\",$1}",
                "rule_6": "/^(([0-9A-f]{0,4}:){1,7}[0-9A-f]{0,4}:?(\\/(1?[0-2][0-8]|[0-9][0-9]))?)[[:space:]]/{printf \"%s,\\n\",$1}",
-               "focus": "stevenblack IPs",
-               "descurl": "https://github.com/dibdot/banIP-IP-blocklists"
+               "descr": "stevenblack IPs"
        },
        "talos": {
                "url_4": "https://www.talosintelligence.com/documents/ip-blacklist",
                "rule_4": "/^(([0-9]{1,3}\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\\/(1?[0-9]|2?[0-9]|3?[0-2]))?)$/{printf \"%s,\\n\",$1}",
-               "focus": "talos IPs",
-               "descurl": "https://talosintelligence.com/reputation_center"
+               "descr": "talos IPs"
        },
        "threat": {
                "url_4": "https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt",
                "rule_4": "/^(([0-9]{1,3}\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\\/(1?[0-9]|2?[0-9]|3?[0-2]))?)$/{printf \"%s,\\n\",$1}",
-               "focus": "emerging threats",
-               "descurl": "https://rules.emergingthreats.net"
+               "descr": "emerging threats"
        },
        "threatview": {
                "url_4": "https://threatview.io/Downloads/IP-High-Confidence-Feed.txt",
                "rule_4": "/^(([0-9]{1,3}\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\\/(1?[0-9]|2?[0-9]|3?[0-2]))?)$/{printf \"%s,\\n\",$1}",
-               "focus": "malicious IPs",
-               "descurl": "https://threatview.io"
+               "descr": "malicious IPs"
        },
        "tor": {
                "url_4": "https://raw.githubusercontent.com/SecOps-Institute/Tor-IP-Addresses/master/tor-exit-nodes.lst",
                "url_6": "https://raw.githubusercontent.com/SecOps-Institute/Tor-IP-Addresses/master/tor-exit-nodes.lst",
                "rule_4": "/^(([0-9]{1,3}\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\\/(1?[0-9]|2?[0-9]|3?[0-2]))?)$/{printf \"%s,\\n\",$1}",
                "rule_6": "/^(([0-9A-f]{0,4}:){1,7}[0-9A-f]{0,4}:?(\\/(1?[0-2][0-8]|[0-9][0-9]))?)$/{printf \"%s,\\n\",$1}",
-               "focus": "tor exit nodes",
-               "descurl": "https://github.com/SecOps-Institute/Tor-IP-Addresses"
+               "descr": "tor exit nodes"
        },
        "uceprotect1": {
                "url_4": "http://wget-mirrors.uceprotect.net/rbldnsd-all/dnsbl-1.uceprotect.net.gz",
                "rule_4": "/^(([0-9]{1,3}\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\\/(1?[0-9]|2?[0-9]|3?[0-2]))?)$/{printf \"%s,\\n\",$1}",
-               "focus": "spam protection level 1",
-               "descurl": "http://www.uceprotect.net/en/index.php",
+               "descr": "spam protection level 1",
                "flag": "gz"
        },
        "uceprotect2": {
                "url_4": "http://wget-mirrors.uceprotect.net/rbldnsd-all/dnsbl-2.uceprotect.net.gz",
                "rule_4": "BEGIN{IGNORECASE=1}/^(([0-9]{1,3}\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\\/(1?[0-9]|2?[0-9]|3?[0-2]))?)([[:space:]]NET)/{printf \"%s,\\n\",$1}",
-               "focus": "spam protection level 2",
-               "descurl": "http://www.uceprotect.net/en/index.php",
+               "descr": "spam protection level 2",
                "flag": "gz"
        },
        "uceprotect3": {
                "url_4": "http://wget-mirrors.uceprotect.net/rbldnsd-all/dnsbl-3.uceprotect.net.gz",
                "rule_4": "BEGIN{IGNORECASE=1}/^(([0-9]{1,3}\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\\/(1?[0-9]|2?[0-9]|3?[0-2]))?)([[:space:]]YOUR)/{printf \"%s,\\n\",$1}",
-               "focus": "spam protection level 3",
-               "descurl": "http://www.uceprotect.net/en/index.php",
+               "descr": "spam protection level 3",
                "flag": "gz"
        },
        "urlhaus": {
                "url_4": "https://urlhaus.abuse.ch/downloads/ids/",
                "rule_4": "match($0,/(([0-9]{1,3}\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5]))/){printf \"%s,\\n\",substr($0,RSTART,RLENGTH)}",
-               "focus": "urlhaus IDS IPs",
-               "descurl": "https://urlhaus.abuse.ch"
+               "descr": "urlhaus IDS IPs"
        },
        "urlvir": {
                "url_4": "https://iplists.firehol.org/files/urlvir.ipset",
                "rule_4": "/^(([0-9]{1,3}\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\\/(1?[0-9]|2?[0-9]|3?[0-2]))?)$/{printf \"%s,\\n\",$1}",
-               "focus": "malware related IPs",
-               "descurl": "https://iplists.firehol.org/?ipset=urlvir"
+               "descr": "malware related IPs"
        },
        "voip": {
                "url_4": "https://voipbl.org/update/",
                "rule_4": "BEGIN{RS=\"(([0-9]{1,3}\\\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\\\\/(1?[0-9]|2?[0-9]|3?[0-2]))?)\"}{if(RT)printf \"%s,\\n\",RT}",
-               "focus": "VoIP fraud blocklist",
-               "descurl": "https://voipbl.org"
+               "descr": "VoIP fraud blocklist"
        },
        "webclient": {
                "url_4": "https://iplists.firehol.org/files/firehol_webclient.netset",
                "rule_4": "/^(([0-9]{1,3}\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\\/(1?[0-9]|2?[0-9]|3?[0-2]))?)$/{printf \"%s,\\n\",$1}",
-               "focus": "malware related IPs",
-               "descurl": "https://iplists.firehol.org/?ipset=firehol_webclient"
+               "descr": "malware related IPs"
        },
        "yoyo": {
                "url_4": "https://raw.githubusercontent.com/dibdot/banIP-IP-blocklists/main/yoyo-ipv4.txt",
                "url_6": "https://raw.githubusercontent.com/dibdot/banIP-IP-blocklists/main/yoyo-ipv6.txt",
                "rule_4": "/^(([0-9]{1,3}\\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\\/(1?[0-9]|2?[0-9]|3?[0-2]))?)[[:space:]]/{printf \"%s,\\n\",$1}",
                "rule_6": "/^(([0-9A-f]{0,4}:){1,7}[0-9A-f]{0,4}:?(\\/(1?[0-2][0-8]|[0-9][0-9]))?)[[:space:]]/{printf \"%s,\\n\",$1}",
-               "focus": "yoyo IPs",
-               "descurl": "https://github.com/dibdot/banIP-IP-blocklists"
+               "descr": "yoyo IPs"
        }
 }