bpf: refactor code to support explicit opt-in for bulk+prio detection

Significantly reduces compiled BPF code size by reducing inlining duplication

Signed-off-by: Felix Fietkau <nbd@nbd.name>

diff --git a/map.c b/map.c
index 303cf39a426366e1dcabb0da3e867fbc8eacb625..466ad3f4016135a735049cf2ae2f4a284019f233 100644 (file)
--- a/map.c
+++ b/map.c
@@ -152,8 +152,8 @@ static void __qosify_map_set_dscp_default(enum qosify_map_id id, struct qosify_d
        int fd = qosify_map_fds[id];
        int i;
 
-       val->ingress |= QOSIFY_DSCP_DEFAULT_FLAG;
-       val->egress |= QOSIFY_DSCP_DEFAULT_FLAG;
+       val->flags |= QOSIFY_VAL_FLAG_PRIO_CHECK |
+                     QOSIFY_VAL_FLAG_BULK_CHECK;
 
        for (i = 0; i < (1 << 16); i++) {
                data.addr.port = htons(i);

diff --git a/qosify-bpf.c b/qosify-bpf.c
index e384ec8a52279ec7da8245d37d6b2c9e56ec5ad6..1e036f00d8e82af3a73a037100c75b08a8dc9bbd 100644 (file)
--- a/qosify-bpf.c
+++ b/qosify-bpf.c
@@ -33,9 +33,9 @@ struct flow_bucket {
        __u32 last_update;
        __u32 pkt_len_avg;
        __u16 pkt_count;
-       __u8 dscp;
+       struct qosify_dscp_val val;
        __u8 bulk_timeout;
-};
+} __packed;
 
 struct {
        __uint(type, BPF_MAP_TYPE_ARRAY);
@@ -220,18 +220,19 @@ parse_ethernet(struct __sk_buff *skb, __u32 *offset)
 
 static void
 parse_l4proto(struct qosify_config *config, struct __sk_buff *skb,
-             __u32 offset, __u8 proto, __u8 *dscp_out, bool ingress)
+             __u32 offset, __u8 proto, bool ingress,
+             struct qosify_dscp_val *out_val)
 {
+       struct qosify_dscp_val *value;
        struct udphdr *udp;
        __u32 src, dest, key;
-       struct qosify_dscp_val *value;
 
        udp = skb_ptr(skb, offset);
        if (skb_check(skb, &udp->len))
                return;
 
        if (config && (proto == IPPROTO_ICMP || proto == IPPROTO_ICMPV6)) {
-               *dscp_out = dscp_val(&config->dscp_icmp, ingress);
+               *out_val = config->dscp_icmp;
                return;
        }
 
@@ -249,175 +250,163 @@ parse_l4proto(struct qosify_config *config, struct __sk_buff *skb,
                value = bpf_map_lookup_elem(&udp_ports, &key);
        }
 
-       if (!value)
-               return;
-
-       *dscp_out = dscp_val(value, ingress);
+       if (value)
+               *out_val = *value;
 }
 
 static __always_inline void
-check_flow(struct qosify_config *config, struct __sk_buff *skb,
-          uint8_t *dscp, bool ingress)
+check_flow_bulk(struct qosify_config *config, struct __sk_buff *skb,
+               struct flow_bucket *flow, struct qosify_dscp_val *out_val)
 {
-       struct flow_bucket flow_data;
-       struct flow_bucket *flow;
+       bool trigger = false;
        __s32 delta;
-       __u32 hash;
        __u32 time;
 
-       if (!(*dscp & QOSIFY_DSCP_DEFAULT_FLAG))
+       if (!config->bulk_trigger_pps)
                return;
 
-       if (!config)
-               return;
-
-       if (!config->bulk_trigger_pps &&
-           !config->prio_max_avg_pkt_len)
-               return;
-
-       time = cur_time();
-       hash = bpf_get_hash_recalc(skb);
-       flow = bpf_map_lookup_elem(&flow_map, &hash);
-       if (!flow) {
-               memset(&flow_data, 0, sizeof(flow_data));
-               bpf_map_update_elem(&flow_map, &hash, &flow_data, BPF_ANY);
-               flow = bpf_map_lookup_elem(&flow_map, &hash);
-               if (!flow)
-                       return;
-       }
-
        if (!flow->last_update)
                goto reset;
 
+       time = cur_time();
        delta = time - flow->last_update;
        if ((u32)delta > FLOW_TIMEOUT)
                goto reset;
 
+       if (flow->pkt_count < 0xffff)
+               flow->pkt_count++;
+
+       if (flow->pkt_count > config->bulk_trigger_pps) {
+               flow->val = config->dscp_bulk;
+               flow->val.flags = QOSIFY_VAL_FLAG_BULK_CHECK;
+               flow->bulk_timeout = config->bulk_trigger_timeout + 1;
+               trigger = true;
+       }
+
        if (delta >= FLOW_CHECK_INTERVAL) {
-               if (flow->bulk_timeout) {
+               if (flow->bulk_timeout && !trigger) {
                        flow->bulk_timeout--;
                        if (!flow->bulk_timeout)
-                               flow->dscp = 0xff;
+                               flow->val.flags = 0;
                }
 
                goto clear;
        }
 
-       if (flow->pkt_count < 0xffff)
-               flow->pkt_count++;
-
-       if (config->bulk_trigger_pps &&
-           flow->pkt_count > config->bulk_trigger_pps) {
-               flow->dscp = dscp_val(&config->dscp_bulk, ingress);
-               flow->bulk_timeout = config->bulk_trigger_timeout;
-       }
-
-out:
-       if (config->prio_max_avg_pkt_len &&
-           flow->dscp != dscp_val(&config->dscp_bulk, ingress)) {
-               if (ewma(&flow->pkt_len_avg, skb->len) <
-                   config->prio_max_avg_pkt_len)
-                       flow->dscp = dscp_val(&config->dscp_prio, ingress);
-               else
-                       flow->dscp = 0xff;
-       }
-
-       if (flow->dscp != 0xff)
-               *dscp = flow->dscp;
-
        return;
 
 reset:
-       flow->dscp = 0xff;
+       flow->val.flags = 0;
        flow->pkt_len_avg = 0;
 clear:
        flow->pkt_count = 1;
        flow->last_update = time;
+}
+
+static __always_inline void
+check_flow_prio(struct qosify_config *config, struct __sk_buff *skb,
+               struct flow_bucket *flow, struct qosify_dscp_val *out_val)
+{
+       if ((flow->val.flags & QOSIFY_VAL_FLAG_BULK_CHECK) ||
+           !config->prio_max_avg_pkt_len)
+               return;
+
+       if (ewma(&flow->pkt_len_avg, skb->len) > config->prio_max_avg_pkt_len) {
+               flow->val.flags = 0;
+               return;
+       }
 
-       goto out;
+       flow->val = config->dscp_prio;
+       flow->val.flags = QOSIFY_VAL_FLAG_PRIO_CHECK;
 }
 
 static __always_inline void
-parse_ipv4(struct __sk_buff *skb, __u32 *offset, bool ingress)
+check_flow(struct qosify_config *config, struct __sk_buff *skb,
+          struct qosify_dscp_val *out_val)
+{
+       struct flow_bucket flow_data;
+       struct flow_bucket *flow;
+       __u32 hash;
+
+       if (!(out_val->flags & (QOSIFY_VAL_FLAG_PRIO_CHECK |
+                               QOSIFY_VAL_FLAG_BULK_CHECK)))
+               return;
+
+       if (!config)
+               return;
+
+       hash = bpf_get_hash_recalc(skb);
+       flow = bpf_map_lookup_elem(&flow_map, &hash);
+       if (!flow) {
+               memset(&flow_data, 0, sizeof(flow_data));
+               bpf_map_update_elem(&flow_map, &hash, &flow_data, BPF_ANY);
+               flow = bpf_map_lookup_elem(&flow_map, &hash);
+               if (!flow)
+                       return;
+       }
+
+
+       if (out_val->flags & QOSIFY_VAL_FLAG_BULK_CHECK)
+               check_flow_bulk(config, skb, flow, out_val);
+       if (out_val->flags & QOSIFY_VAL_FLAG_PRIO_CHECK)
+               check_flow_prio(config, skb, flow, out_val);
+
+       if (flow->val.flags & out_val->flags)
+               *out_val = flow->val;
+}
+
+static __always_inline struct qosify_ip_map_val *
+parse_ipv4(struct qosify_config *config, struct __sk_buff *skb, __u32 *offset,
+          bool ingress, struct qosify_dscp_val *out_val)
 {
-       struct qosify_config *config;
-       struct qosify_ip_map_val *ip_val;
        struct qosify_dscp_val *value;
-       const __u32 zero_port = 0;
        struct iphdr *iph;
-       __u8 dscp = 0xff;
        __u8 ipproto;
        int hdr_len;
        void *key;
-       bool force;
-
-       config = get_config();
 
        iph = skb_ptr(skb, *offset);
        if (skb_check(skb, iph + 1))
-               return;
+               return NULL;
 
        hdr_len = iph->ihl * 4;
        if (bpf_skb_pull_data(skb, *offset + hdr_len + sizeof(struct udphdr)))
-               return;
+               return NULL;
 
        iph = skb_ptr(skb, *offset);
        *offset += hdr_len;
 
        if (skb_check(skb, (void *)(iph + 1)))
-               return;
+               return NULL;
 
        ipproto = iph->protocol;
-       parse_l4proto(config, skb, *offset, ipproto, &dscp, ingress);
+       parse_l4proto(config, skb, *offset, ipproto, ingress, out_val);
 
        if (ingress)
                key = &iph->saddr;
        else
                key = &iph->daddr;
 
-       ip_val = bpf_map_lookup_elem(&ipv4_map, key);
-       if (ip_val) {
-               if (!ip_val->seen)
-                       ip_val->seen = 1;
-               dscp = dscp_val(&ip_val->dscp, ingress);
-       } else if (dscp == 0xff) {
-               /* use udp port 0 entry as fallback for non-tcp/udp */
-               value = bpf_map_lookup_elem(&udp_ports, &zero_port);
-               if (value)
-                       dscp = dscp_val(value, ingress);
-       }
-
-       check_flow(config, skb, &dscp, ingress);
-
-       force = !(dscp & QOSIFY_DSCP_FALLBACK_FLAG);
-       dscp &= GENMASK(5, 0);
-
-       ipv4_change_dsfield(iph, INET_ECN_MASK, dscp << 2, force);
+       return bpf_map_lookup_elem(&ipv4_map, key);
 }
 
-static __always_inline void
-parse_ipv6(struct __sk_buff *skb, __u32 *offset, bool ingress)
+static __always_inline struct qosify_ip_map_val *
+parse_ipv6(struct qosify_config *config, struct __sk_buff *skb, __u32 *offset,
+          bool ingress, struct qosify_dscp_val *out_val)
 {
-       struct qosify_config *config;
-       struct qosify_ip_map_val *ip_val;
        struct qosify_dscp_val *value;
-       const __u32 zero_port = 0;
        struct ipv6hdr *iph;
-       __u8 dscp = 0;
        __u8 ipproto;
        void *key;
-       bool force;
-
-       config = get_config();
 
        if (bpf_skb_pull_data(skb, *offset + sizeof(*iph) + sizeof(struct udphdr)))
-               return;
+               return NULL;
 
        iph = skb_ptr(skb, *offset);
        *offset += sizeof(*iph);
 
        if (skb_check(skb, (void *)(iph + 1)))
-               return;
+               return NULL;
 
        ipproto = iph->nexthdr;
        if (ingress)
@@ -425,44 +414,68 @@ parse_ipv6(struct __sk_buff *skb, __u32 *offset, bool ingress)
        else
                key = &iph->daddr;
 
-       parse_l4proto(config, skb, *offset, ipproto, &dscp, ingress);
-
-       ip_val = bpf_map_lookup_elem(&ipv6_map, key);
-       if (ip_val) {
-               if (!ip_val->seen)
-                       ip_val->seen = 1;
-               dscp = dscp_val(&ip_val->dscp, ingress);
-       } else if (dscp == 0xff) {
-               /* use udp port 0 entry as fallback for non-tcp/udp */
-               value = bpf_map_lookup_elem(&udp_ports, &zero_port);
-               if (value)
-                       dscp = dscp_val(value, ingress);
-       }
-
-       check_flow(config, skb, &dscp, ingress);
+       parse_l4proto(config, skb, *offset, ipproto, ingress, out_val);
 
-       force = !(dscp & QOSIFY_DSCP_FALLBACK_FLAG);
-       dscp &= GENMASK(5, 0);
-
-       ipv6_change_dsfield(iph, INET_ECN_MASK, dscp << 2, force);
+       return bpf_map_lookup_elem(&ipv6_map, key);
 }
 
 SEC("classifier")
 int classify(struct __sk_buff *skb)
 {
        bool ingress = module_flags & QOSIFY_INGRESS;
+       struct qosify_config *config;
+       struct qosify_ip_map_val *ip_val;
+       struct qosify_dscp_val val = {
+               .ingress = 0xff,
+               .egress = 0xff,
+               .flags = 0,
+       };
        __u32 offset = 0;
+       __u32 iph_offset;
+       void *iph;
+       __u8 dscp;
+       bool force;
        int type;
 
+       config = get_config();
+
        if (module_flags & QOSIFY_IP_ONLY)
                type = skb->protocol;
        else
                type = parse_ethernet(skb, &offset);
 
+       iph_offset = offset;
+       if (type == bpf_htons(ETH_P_IP))
+               ip_val = parse_ipv4(config, skb, &offset, ingress, &val);
+       else if (type == bpf_htons(ETH_P_IPV6))
+               ip_val = parse_ipv6(config, skb, &offset, ingress, &val);
+       else
+               return TC_ACT_OK;
+
+       if (ip_val) {
+               if (!ip_val->seen)
+                       ip_val->seen = 1;
+               val = ip_val->dscp;
+       }
+
+       check_flow(config, skb, &val);
+
+       dscp = dscp_val(&val, ingress);
+       if (dscp == 0xff)
+               return TC_ACT_OK;
+
+       dscp &= GENMASK(5, 0);
+       dscp <<= 2;
+       force = !(dscp & QOSIFY_DSCP_FALLBACK_FLAG);
+
+       iph = skb_ptr(skb, iph_offset);
+       if (skb_check(skb, (void *)iph + sizeof(struct ipv6hdr)))
+               return TC_ACT_OK;
+
        if (type == bpf_htons(ETH_P_IP))
-               parse_ipv4(skb, &offset, ingress);
+               ipv4_change_dsfield(iph, INET_ECN_MASK, dscp, force);
        else if (type == bpf_htons(ETH_P_IPV6))
-               parse_ipv6(skb, &offset, ingress);
+               ipv6_change_dsfield(iph, INET_ECN_MASK, dscp, force);
 
        return TC_ACT_OK;
 }

diff --git a/qosify-bpf.h b/qosify-bpf.h
index 3d457a61675734d172be49a9663505c45c2ca895..d9d4e9f7886403459a71b01cd52c7df4c47b1c85 100644 (file)
--- a/qosify-bpf.h
+++ b/qosify-bpf.h
@@ -15,14 +15,16 @@
 #define QOSIFY_INGRESS                 (1 << 0)
 #define QOSIFY_IP_ONLY                 (1 << 1)
 
-
 #define QOSIFY_DSCP_FALLBACK_FLAG      (1 << 6)
-#define QOSIFY_DSCP_DEFAULT_FLAG       (1 << 7)
+
+#define QOSIFY_VAL_FLAG_PRIO_CHECK     (1 << 0)
+#define QOSIFY_VAL_FLAG_BULK_CHECK     (1 << 1)
 
 struct qosify_dscp_val {
        uint8_t ingress;
        uint8_t egress;
-};
+       uint8_t flags;
+} __attribute__((packed));
 
 /* global config data */
 struct qosify_config {