udp6: Run SK_LOOKUP BPF program on socket lookup

Same as for udp4, let BPF program override the socket lookup result, by
selecting a receiving socket of its choice or failing the lookup, if no
connected UDP socket matched packet 4-tuple.

Suggested-by: Marek Majkowski <marek@cloudflare.com>
Signed-off-by: Jakub Sitnicki <jakub@cloudflare.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Acked-by: Andrii Nakryiko <andriin@fb.com>
Link: https://lore.kernel.org/bpf/20200717103536.397595-11-jakub@cloudflare.com

diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c
index 084205c..ff8be20 100644 (file)
--- a/net/ipv6/udp.c
+++ b/net/ipv6/udp.c
@@ -190,6 +190,31 @@ static struct sock *udp6_lib_lookup2(struct net *net,
        return result;
 }
 
+static inline struct sock *udp6_lookup_run_bpf(struct net *net,
+                                              struct udp_table *udptable,
+                                              struct sk_buff *skb,
+                                              const struct in6_addr *saddr,
+                                              __be16 sport,
+                                              const struct in6_addr *daddr,
+                                              u16 hnum)
+{
+       struct sock *sk, *reuse_sk;
+       bool no_reuseport;
+
+       if (udptable != &udp_table)
+               return NULL; /* only UDP is supported */
+
+       no_reuseport = bpf_sk_lookup_run_v6(net, IPPROTO_UDP,
+                                           saddr, sport, daddr, hnum, &sk);
+       if (no_reuseport || IS_ERR_OR_NULL(sk))
+               return sk;
+
+       reuse_sk = lookup_reuseport(net, sk, skb, saddr, sport, daddr, hnum);
+       if (reuse_sk)
+               sk = reuse_sk;
+       return sk;
+}
+
 /* rcu_read_lock() must be held */
 struct sock *__udp6_lib_lookup(struct net *net,
                               const struct in6_addr *saddr, __be16 sport,
@@ -200,25 +225,42 @@ struct sock *__udp6_lib_lookup(struct net *net,
        unsigned short hnum = ntohs(dport);
        unsigned int hash2, slot2;
        struct udp_hslot *hslot2;
-       struct sock *result;
+       struct sock *result, *sk;
 
        hash2 = ipv6_portaddr_hash(net, daddr, hnum);
        slot2 = hash2 & udptable->mask;
        hslot2 = &udptable->hash2[slot2];
 
+       /* Lookup connected or non-wildcard sockets */
        result = udp6_lib_lookup2(net, saddr, sport,
                                  daddr, hnum, dif, sdif,
                                  hslot2, skb);
-       if (!result) {
-               hash2 = ipv6_portaddr_hash(net, &in6addr_any, hnum);
-               slot2 = hash2 & udptable->mask;
+       if (!IS_ERR_OR_NULL(result) && result->sk_state == TCP_ESTABLISHED)
+               goto done;
+
+       /* Lookup redirect from BPF */
+       if (static_branch_unlikely(&bpf_sk_lookup_enabled)) {
+               sk = udp6_lookup_run_bpf(net, udptable, skb,
+                                        saddr, sport, daddr, hnum);
+               if (sk) {
+                       result = sk;
+                       goto done;
+               }
+       }
 
-               hslot2 = &udptable->hash2[slot2];
+       /* Got non-wildcard socket or error on first lookup */
+       if (result)
+               goto done;
 
-               result = udp6_lib_lookup2(net, saddr, sport,
-                                         &in6addr_any, hnum, dif, sdif,
-                                         hslot2, skb);
-       }
+       /* Lookup wildcard sockets */
+       hash2 = ipv6_portaddr_hash(net, &in6addr_any, hnum);
+       slot2 = hash2 & udptable->mask;
+       hslot2 = &udptable->hash2[slot2];
+
+       result = udp6_lib_lookup2(net, saddr, sport,
+                                 &in6addr_any, hnum, dif, sdif,
+                                 hslot2, skb);
+done:
        if (IS_ERR(result))
                return NULL;
        return result;