I have an eBPF program with the following map definitions:
struct bpf_map_def SEC("maps") servers = {
.type = BPF_MAP_TYPE_HASH,
.key_size = sizeof(struct ip_key),
.value_size = sizeof(struct dest_info),
.max_entries = MAX_SERVERS,
};
struct bpf_map_def SEC("maps") client_addrs = {
.type = BPF_MAP_TYPE_HASH,
.key_size = sizeof(struct port_key),
.value_size = sizeof(struct client_port_addr),
.max_entries = MAX_CLIENTS,
};
where the struct definitions are as below:
struct port_key {
__u16 port;
__u16 pad[3];
};
struct ip_key {
__u32 key;
__u32 pad;
};
struct dest_info {
__u32 saddr;
__u32 daddr;
__u64 bytes;
__u64 pkts;
__u8 dmac[6];
__u16 pad;
};
struct client_port_addr {
__u32 client_ip;
__u8 dmac[6];
__u16 pad[3];
};
The program itself, after the pointer verifications and initial checks, is shown below.
struct port_key key = {0};
struct client_port_addr val;
key.port = udp->source;
val.client_ip = iph->saddr;
memcpy (val.dmac, eth->h_source, 6 * sizeof(__u8));
bpf_map_update_elem(&client_addrs, &key, &val, BPF_ANY);
iph->saddr = IP_ADDRESS(BALANCER);
iph->daddr = dest_tnl->daddr;
memcpy (eth->h_source, eth->h_dest, 6 * sizeof(__u8));
memcpy (eth->h_dest, dest_tnl->dmac, 6 * sizeof(__u8));
So, the problem is that I use bpf_map_update() in my code, but while using it, I get the invalid indirect read from the stack error as shown below.
libbpf:
0: (bf) r6 = r1
1: (61) r9 = *(u32 *)(r6 +4)
2: (61) r7 = *(u32 *)(r6 +0)
3: (18) r1 = 0xffffa59ac00b6000
5: (b7) r2 = 24
6: (85) call bpf_trace_printk#6
R1_w=map_value(id=0,off=0,ks=4,vs=50,imm=0) R2_w=inv24 R6_w=ctx(id=0,off=0,imm=0) R7_w=pkt(id=0,off=0,r=0,imm=0) R9_w=pkt_end(id=0,off=0,imm=0) R10=fp0
last_idx 6 first_idx 0
regs=4 stack=0 before 5: (b7) r2 = 24
7: (b7) r8 = 1
8: (bf) r1 = r7
9: (07) r1 += 14
10: (2d) if r1 > r9 goto pc+130
R0_w=inv(id=0) R1_w=pkt(id=0,off=14,r=14,imm=0) R6_w=ctx(id=0,off=0,imm=0) R7_w=pkt(id=0,off=0,r=14,imm=0) R8_w=inv1 R9_w=pkt_end(id=0,off=0,imm=0) R10=fp0
11: (71) r1 = *(u8 *)(r7 +12)
12: (71) r2 = *(u8 *)(r7 +13)
13: (67) r2 <<= 8
14: (4f) r2 |= r1
15: (b7) r8 = 2
16: (55) if r2 != 0x8 goto pc+124
R0=inv(id=0) R1=inv(id=0,umax_value=255,var_off=(0x0; 0xff)) R2=inv8 R6=ctx(id=0,off=0,imm=0) R7=pkt(id=0,off=0,r=14,imm=0) R8=inv2 R9=pkt_end(id=0,off=0,imm=0) R10=fp0
17: (61) r7 = *(u32 *)(r6 +4)
18: (61) r9 = *(u32 *)(r6 +0)
19: (bf) r6 = r9
20: (07) r6 += 14
21: (b7) r8 = 1
22: (2d) if r6 > r7 goto pc+118
R0=inv(id=0) R1=inv(id=0,umax_value=255,var_off=(0x0; 0xff)) R2=inv8 R6_w=pkt(id=0,off=14,r=14,imm=0) R7_w=pkt_end(id=0,off=0,imm=0) R8_w=inv1 R9_w=pkt(id=0,off=0,r=14,imm=0) R10=fp0
23: (bf) r1 = r9
24: (07) r1 += 34
25: (b7) r8 = 1
26: (2d) if r1 > r7 goto pc+114
R0=inv(id=0) R1=pkt(id=0,off=34,r=34,imm=0) R2=inv8 R6=pkt(id=0,off=14,r=34,imm=0) R7=pkt_end(id=0,off=0,imm=0) R8=inv1 R9=pkt(id=0,off=0,r=34,imm=0) R10=fp0
27: (71) r1 = *(u8 *)(r6 +0)
28: (57) r1 &= 15
29: (b7) r8 = 1
30: (55) if r1 != 0x5 goto pc+110
R0=inv(id=0) R1_w=inv5 R2=inv8 R6=pkt(id=0,off=14,r=34,imm=0) R7=pkt_end(id=0,off=0,imm=0) R8_w=inv1 R9=pkt(id=0,off=0,r=34,imm=0) R10=fp0
31: (61) r3 = *(u32 *)(r9 +26)
32: (18) r1 = 0xffffa59ac00b6018
34: (b7) r2 = 26
35: (85) call bpf_trace_printk#6
R0=inv(id=0) R1_w=map_value(id=0,off=24,ks=4,vs=50,imm=0) R2_w=inv26 R3_w=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R6=pkt(id=0,off=14,r=34,imm=0) R7=pkt_end(id=0,off=0,imm=0) R8_w=inv1 R9=pkt(id=0,off=0,r=34,imm=0) R10=fp0
last_idx 35 first_idx 26
regs=4 stack=0 before 34: (b7) r2 = 26
36: (69) r1 = *(u16 *)(r9 +20)
37: (57) r1 &= 65343
38: (b7) r8 = 1
39: (55) if r1 != 0x0 goto pc+101
R0=inv(id=0) R1_w=inv0 R6=pkt(id=0,off=14,r=34,imm=0) R7=pkt_end(id=0,off=0,imm=0) R8_w=inv1 R9=pkt(id=0,off=0,r=34,imm=0) R10=fp0
40: (71) r1 = *(u8 *)(r9 +23)
41: (b7) r8 = 2
42: (55) if r1 != 0x11 goto pc+98
R0=inv(id=0) R1_w=inv17 R6=pkt(id=0,off=14,r=34,imm=0) R7=pkt_end(id=0,off=0,imm=0) R8_w=inv2 R9=pkt(id=0,off=0,r=34,imm=0) R10=fp0
43: (bf) r1 = r9
44: (07) r1 += 42
45: (b7) r8 = 1
46: (2d) if r1 > r7 goto pc+94
R0=inv(id=0) R1=pkt(id=0,off=42,r=42,imm=0) R6=pkt(id=0,off=14,r=42,imm=0) R7=pkt_end(id=0,off=0,imm=0) R8=inv1 R9=pkt(id=0,off=0,r=42,imm=0) R10=fp0
47: (b7) r8 = 0
48: (7b) *(u64 *)(r10 -8) = r8
last_idx 48 first_idx 46
regs=100 stack=0 before 47: (b7) r8 = 0
49: (bf) r2 = r10
50: (07) r2 += -8
51: (18) r1 = 0xffff9a7bed1bc000
53: (85) call bpf_map_lookup_elem#1
54: (bf) r7 = r0
55: (15) if r7 == 0x0 goto pc+85
R0=map_value(id=0,off=0,ks=8,vs=32,imm=0) R6=pkt(id=0,off=14,r=42,imm=0) R7=map_value(id=0,off=0,ks=8,vs=32,imm=0) R8=invP0 R9=pkt(id=0,off=0,r=42,imm=0) R10=fp0 fp-8=mmmmmmmm
56: (b7) r8 = 0
57: (7b) *(u64 *)(r10 -16) = r8
last_idx 57 first_idx 55
regs=100 stack=0 before 56: (b7) r8 = 0
58: (69) r1 = *(u16 *)(r9 +34)
59: (6b) *(u16 *)(r10 -16) = r1
60: (61) r1 = *(u32 *)(r9 +26)
61: (63) *(u32 *)(r10 -32) = r1
62: (71) r1 = *(u8 *)(r9 +11)
63: (73) *(u8 *)(r10 -23) = r1
64: (71) r1 = *(u8 *)(r9 +10)
65: (73) *(u8 *)(r10 -24) = r1
66: (71) r1 = *(u8 *)(r9 +7)
67: (67) r1 <<= 8
68: (71) r2 = *(u8 *)(r9 +6)
69: (4f) r1 |= r2
70: (71) r2 = *(u8 *)(r9 +9)
71: (67) r2 <<= 8
72: (71) r3 = *(u8 *)(r9 +8)
73: (4f) r2 |= r3
74: (67) r2 <<= 16
75: (4f) r2 |= r1
76: (63) *(u32 *)(r10 -28) = r2
77: (bf) r2 = r10
78: (07) r2 += -16
79: (bf) r3 = r10
80: (07) r3 += -32
81: (18) r1 = 0xffff9a7bed1bf400
83: (b7) r4 = 0
84: (85) call bpf_map_update_elem#2
invalid indirect read from stack R3 off -32+10 size 16
processed 81 insns (limit 1000000) max_states_per_insn 0 total_states 5 peak_states 5 mark_read 2
libbpf: -- END LOG --
libbpf: failed to load program 'loadbal'
All of the defined structs for keys and values are padded to their next multiple of 8 bytes. Since I could not find any useful and descriptive explanation on my issue, explanations of this topic and maybe even a bit of detail are much appreciated.
Please let me know if you need more information.
The verifier complains because your code is trying to read uninitialised data from the stack, in particular in your variable val.
If we look at your code:
struct client_port_addr {
__u32 client_ip;
__u8 dmac[6];
__u16 pad[3];
};
struct client_port_addr val;
[...]
val.client_ip = iph->saddr; // val.client_ip
memcpy (val.dmac, eth->h_source, 6 * sizeof(__u8)); // val.dmac
// val.pad where??
bpf_map_update_elem(&client_addrs, &key, &val, BPF_ANY);
You initialised val.client_ip, and val.dmac, but val.pad is never initialised. When you pass val to bpf_map_update_elem(), the eBPF verifier realises that the helper function might read this variable which contains uninitialised memory from kernel space. This is a security risk, therefore, the verifier rejects the program.
To fix the issue, make sure you initialise the memory before using it. You have at least three ways to do so:
val when declaring it, like for your key:
struct client_port_addr val = {0};
val.pad with zeroes with memcpy(). Same as the first option, this won't help if the compiler pads your struct.memset() the struct after declaring it:
struct client_port_addr val;
memset(&val, 0, sizeof(val));