Skip to content

Commit e24bbad

Browse files
peilin-yeAlexei Starovoitov
peilin-ye
authored and
Alexei Starovoitov
committed
bpf: Introduce load-acquire and store-release instructions
Introduce BPF instructions with load-acquire and store-release semantics, as discussed in [1]. Define 2 new flags: #define BPF_LOAD_ACQ 0x100 #define BPF_STORE_REL 0x110 A "load-acquire" is a BPF_STX | BPF_ATOMIC instruction with the 'imm' field set to BPF_LOAD_ACQ (0x100). Similarly, a "store-release" is a BPF_STX | BPF_ATOMIC instruction with the 'imm' field set to BPF_STORE_REL (0x110). Unlike existing atomic read-modify-write operations that only support BPF_W (32-bit) and BPF_DW (64-bit) size modifiers, load-acquires and store-releases also support BPF_B (8-bit) and BPF_H (16-bit). As an exception, however, 64-bit load-acquires/store-releases are not supported on 32-bit architectures (to fix a build error reported by the kernel test robot). An 8- or 16-bit load-acquire zero-extends the value before writing it to a 32-bit register, just like ARM64 instruction LDARH and friends. Similar to existing atomic read-modify-write operations, misaligned load-acquires/store-releases are not allowed (even if BPF_F_ANY_ALIGNMENT is set). As an example, consider the following 64-bit load-acquire BPF instruction (assuming little-endian): db 10 00 00 00 01 00 00 r0 = load_acquire((u64 *)(r1 + 0x0)) opcode (0xdb): BPF_ATOMIC | BPF_DW | BPF_STX imm (0x00000100): BPF_LOAD_ACQ Similarly, a 16-bit BPF store-release: cb 21 00 00 10 01 00 00 store_release((u16 *)(r1 + 0x0), w2) opcode (0xcb): BPF_ATOMIC | BPF_H | BPF_STX imm (0x00000110): BPF_STORE_REL In arch/{arm64,s390,x86}/net/bpf_jit_comp.c, have bpf_jit_supports_insn(..., /*in_arena=*/true) return false for the new instructions, until the corresponding JIT compiler supports them in arena. [1] https://lore.kernel.org/all/20240729183246.4110549-1-yepeilin@google.com/ Acked-by: Eduard Zingerman <eddyz87@gmail.com> Acked-by: Ilya Leoshkevich <iii@linux.ibm.com> Cc: kernel test robot <lkp@intel.com> Signed-off-by: Peilin Ye <yepeilin@google.com> Link: https://lore.kernel.org/r/a217f46f0e445fbd573a1a024be5c6bf1d5fe716.1741049567.git.yepeilin@google.com Signed-off-by: Alexei Starovoitov <ast@kernel.org>
1 parent ad55432 commit e24bbad

File tree

10 files changed

+166
-13
lines changed

10 files changed

+166
-13
lines changed

‎arch/arm64/net/bpf_jit_comp.c‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2667,8 +2667,12 @@ bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena)
26672667
if (!in_arena)
26682668
return true;
26692669
switch (insn->code) {
2670+
case BPF_STX | BPF_ATOMIC | BPF_B:
2671+
case BPF_STX | BPF_ATOMIC | BPF_H:
26702672
case BPF_STX | BPF_ATOMIC | BPF_W:
26712673
case BPF_STX | BPF_ATOMIC | BPF_DW:
2674+
if (bpf_atomic_is_load_store(insn))
2675+
return false;
26722676
if (!cpus_have_cap(ARM64_HAS_LSE_ATOMICS))
26732677
return false;
26742678
}

‎arch/s390/net/bpf_jit_comp.c‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2919,10 +2919,16 @@ bool bpf_jit_supports_arena(void)
29192919

29202920
bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena)
29212921
{
2922-
/*
2923-
* Currently the verifier uses this function only to check which
2924-
* atomic stores to arena are supported, and they all are.
2925-
*/
2922+
if (!in_arena)
2923+
return true;
2924+
switch (insn->code) {
2925+
case BPF_STX | BPF_ATOMIC | BPF_B:
2926+
case BPF_STX | BPF_ATOMIC | BPF_H:
2927+
case BPF_STX | BPF_ATOMIC | BPF_W:
2928+
case BPF_STX | BPF_ATOMIC | BPF_DW:
2929+
if (bpf_atomic_is_load_store(insn))
2930+
return false;
2931+
}
29262932
return true;
29272933
}
29282934

‎arch/x86/net/bpf_jit_comp.c‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3771,8 +3771,12 @@ bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena)
37713771
if (!in_arena)
37723772
return true;
37733773
switch (insn->code) {
3774+
case BPF_STX | BPF_ATOMIC | BPF_B:
3775+
case BPF_STX | BPF_ATOMIC | BPF_H:
37743776
case BPF_STX | BPF_ATOMIC | BPF_W:
37753777
case BPF_STX | BPF_ATOMIC | BPF_DW:
3778+
if (bpf_atomic_is_load_store(insn))
3779+
return false;
37763780
if (insn->imm == (BPF_AND | BPF_FETCH) ||
37773781
insn->imm == (BPF_OR | BPF_FETCH) ||
37783782
insn->imm == (BPF_XOR | BPF_FETCH))

‎include/linux/bpf.h‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -991,6 +991,21 @@ static inline bool bpf_pseudo_func(const struct bpf_insn *insn)
991991
return bpf_is_ldimm64(insn) && insn->src_reg == BPF_PSEUDO_FUNC;
992992
}
993993

994+
/* Given a BPF_ATOMIC instruction @atomic_insn, return true if it is an
995+
* atomic load or store, and false if it is a read-modify-write instruction.
996+
*/
997+
static inline bool
998+
bpf_atomic_is_load_store(const struct bpf_insn *atomic_insn)
999+
{
1000+
switch (atomic_insn->imm) {
1001+
case BPF_LOAD_ACQ:
1002+
case BPF_STORE_REL:
1003+
return true;
1004+
default:
1005+
return false;
1006+
}
1007+
}
1008+
9941009
struct bpf_prog_ops {
9951010
int (*test_run)(struct bpf_prog *prog, const union bpf_attr *kattr,
9961011
union bpf_attr __user *uattr);

‎include/linux/filter.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -364,6 +364,8 @@ static inline bool insn_is_cast_user(const struct bpf_insn *insn)
364364
* BPF_XOR | BPF_FETCH src_reg = atomic_fetch_xor(dst_reg + off16, src_reg);
365365
* BPF_XCHG src_reg = atomic_xchg(dst_reg + off16, src_reg)
366366
* BPF_CMPXCHG r0 = atomic_cmpxchg(dst_reg + off16, r0, src_reg)
367+
* BPF_LOAD_ACQ dst_reg = smp_load_acquire(src_reg + off16)
368+
* BPF_STORE_REL smp_store_release(dst_reg + off16, src_reg)
367369
*/
368370

369371
#define BPF_ATOMIC_OP(SIZE, OP, DST, SRC, OFF) \

‎include/uapi/linux/bpf.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,9 @@
5151
#define BPF_XCHG (0xe0 | BPF_FETCH) /* atomic exchange */
5252
#define BPF_CMPXCHG (0xf0 | BPF_FETCH) /* atomic compare-and-write */
5353

54+
#define BPF_LOAD_ACQ 0x100 /* load-acquire */
55+
#define BPF_STORE_REL 0x110 /* store-release */
56+
5457
enum bpf_cond_pseudo_jmp {
5558
BPF_MAY_GOTO = 0,
5659
};

‎kernel/bpf/core.c‎

Lines changed: 61 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1663,14 +1663,17 @@ EXPORT_SYMBOL_GPL(__bpf_call_base);
16631663
INSN_3(JMP, JSET, K), \
16641664
INSN_2(JMP, JA), \
16651665
INSN_2(JMP32, JA), \
1666+
/* Atomic operations. */ \
1667+
INSN_3(STX, ATOMIC, B), \
1668+
INSN_3(STX, ATOMIC, H), \
1669+
INSN_3(STX, ATOMIC, W), \
1670+
INSN_3(STX, ATOMIC, DW), \
16661671
/* Store instructions. */ \
16671672
/* Register based. */ \
16681673
INSN_3(STX, MEM, B), \
16691674
INSN_3(STX, MEM, H), \
16701675
INSN_3(STX, MEM, W), \
16711676
INSN_3(STX, MEM, DW), \
1672-
INSN_3(STX, ATOMIC, W), \
1673-
INSN_3(STX, ATOMIC, DW), \
16741677
/* Immediate based. */ \
16751678
INSN_3(ST, MEM, B), \
16761679
INSN_3(ST, MEM, H), \
@@ -2152,24 +2155,33 @@ static u64 ___bpf_prog_run(u64 *regs, const struct bpf_insn *insn)
21522155
if (BPF_SIZE(insn->code) == BPF_W) \
21532156
atomic_##KOP((u32) SRC, (atomic_t *)(unsigned long) \
21542157
(DST + insn->off)); \
2155-
else \
2158+
else if (BPF_SIZE(insn->code) == BPF_DW) \
21562159
atomic64_##KOP((u64) SRC, (atomic64_t *)(unsigned long) \
21572160
(DST + insn->off)); \
2161+
else \
2162+
goto default_label; \
21582163
break; \
21592164
case BOP | BPF_FETCH: \
21602165
if (BPF_SIZE(insn->code) == BPF_W) \
21612166
SRC = (u32) atomic_fetch_##KOP( \
21622167
(u32) SRC, \
21632168
(atomic_t *)(unsigned long) (DST + insn->off)); \
2164-
else \
2169+
else if (BPF_SIZE(insn->code) == BPF_DW) \
21652170
SRC = (u64) atomic64_fetch_##KOP( \
21662171
(u64) SRC, \
21672172
(atomic64_t *)(unsigned long) (DST + insn->off)); \
2173+
else \
2174+
goto default_label; \
21682175
break;
21692176

21702177
STX_ATOMIC_DW:
21712178
STX_ATOMIC_W:
2179+
STX_ATOMIC_H:
2180+
STX_ATOMIC_B:
21722181
switch (IMM) {
2182+
/* Atomic read-modify-write instructions support only W and DW
2183+
* size modifiers.
2184+
*/
21732185
ATOMIC_ALU_OP(BPF_ADD, add)
21742186
ATOMIC_ALU_OP(BPF_AND, and)
21752187
ATOMIC_ALU_OP(BPF_OR, or)
@@ -2181,20 +2193,63 @@ static u64 ___bpf_prog_run(u64 *regs, const struct bpf_insn *insn)
21812193
SRC = (u32) atomic_xchg(
21822194
(atomic_t *)(unsigned long) (DST + insn->off),
21832195
(u32) SRC);
2184-
else
2196+
else if (BPF_SIZE(insn->code) == BPF_DW)
21852197
SRC = (u64) atomic64_xchg(
21862198
(atomic64_t *)(unsigned long) (DST + insn->off),
21872199
(u64) SRC);
2200+
else
2201+
goto default_label;
21882202
break;
21892203
case BPF_CMPXCHG:
21902204
if (BPF_SIZE(insn->code) == BPF_W)
21912205
BPF_R0 = (u32) atomic_cmpxchg(
21922206
(atomic_t *)(unsigned long) (DST + insn->off),
21932207
(u32) BPF_R0, (u32) SRC);
2194-
else
2208+
else if (BPF_SIZE(insn->code) == BPF_DW)
21952209
BPF_R0 = (u64) atomic64_cmpxchg(
21962210
(atomic64_t *)(unsigned long) (DST + insn->off),
21972211
(u64) BPF_R0, (u64) SRC);
2212+
else
2213+
goto default_label;
2214+
break;
2215+
/* Atomic load and store instructions support all size
2216+
* modifiers.
2217+
*/
2218+
case BPF_LOAD_ACQ:
2219+
switch (BPF_SIZE(insn->code)) {
2220+
#define LOAD_ACQUIRE(SIZEOP, SIZE) \
2221+
case BPF_##SIZEOP: \
2222+
DST = (SIZE)smp_load_acquire( \
2223+
(SIZE *)(unsigned long)(SRC + insn->off)); \
2224+
break;
2225+
LOAD_ACQUIRE(B, u8)
2226+
LOAD_ACQUIRE(H, u16)
2227+
LOAD_ACQUIRE(W, u32)
2228+
#ifdef CONFIG_64BIT
2229+
LOAD_ACQUIRE(DW, u64)
2230+
#endif
2231+
#undef LOAD_ACQUIRE
2232+
default:
2233+
goto default_label;
2234+
}
2235+
break;
2236+
case BPF_STORE_REL:
2237+
switch (BPF_SIZE(insn->code)) {
2238+
#define STORE_RELEASE(SIZEOP, SIZE) \
2239+
case BPF_##SIZEOP: \
2240+
smp_store_release( \
2241+
(SIZE *)(unsigned long)(DST + insn->off), (SIZE)SRC); \
2242+
break;
2243+
STORE_RELEASE(B, u8)
2244+
STORE_RELEASE(H, u16)
2245+
STORE_RELEASE(W, u32)
2246+
#ifdef CONFIG_64BIT
2247+
STORE_RELEASE(DW, u64)
2248+
#endif
2249+
#undef STORE_RELEASE
2250+
default:
2251+
goto default_label;
2252+
}
21982253
break;
21992254

22002255
default:

‎kernel/bpf/disasm.c‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -267,6 +267,18 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs,
267267
BPF_SIZE(insn->code) == BPF_DW ? "64" : "",
268268
bpf_ldst_string[BPF_SIZE(insn->code) >> 3],
269269
insn->dst_reg, insn->off, insn->src_reg);
270+
} else if (BPF_MODE(insn->code) == BPF_ATOMIC &&
271+
insn->imm == BPF_LOAD_ACQ) {
272+
verbose(cbs->private_data, "(%02x) r%d = load_acquire((%s *)(r%d %+d))\n",
273+
insn->code, insn->dst_reg,
274+
bpf_ldst_string[BPF_SIZE(insn->code) >> 3],
275+
insn->src_reg, insn->off);
276+
} else if (BPF_MODE(insn->code) == BPF_ATOMIC &&
277+
insn->imm == BPF_STORE_REL) {
278+
verbose(cbs->private_data, "(%02x) store_release((%s *)(r%d %+d), r%d)\n",
279+
insn->code,
280+
bpf_ldst_string[BPF_SIZE(insn->code) >> 3],
281+
insn->dst_reg, insn->off, insn->src_reg);
270282
} else {
271283
verbose(cbs->private_data, "BUG_%02x\n", insn->code);
272284
}

‎kernel/bpf/verifier.c‎

Lines changed: 52 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -579,6 +579,13 @@ static bool is_cmpxchg_insn(const struct bpf_insn *insn)
579579
insn->imm == BPF_CMPXCHG;
580580
}
581581

582+
static bool is_atomic_load_insn(const struct bpf_insn *insn)
583+
{
584+
return BPF_CLASS(insn->code) == BPF_STX &&
585+
BPF_MODE(insn->code) == BPF_ATOMIC &&
586+
insn->imm == BPF_LOAD_ACQ;
587+
}
588+
582589
static int __get_spi(s32 off)
583590
{
584591
return (-off - 1) / BPF_REG_SIZE;
@@ -3567,7 +3574,7 @@ static bool is_reg64(struct bpf_verifier_env *env, struct bpf_insn *insn,
35673574
}
35683575

35693576
if (class == BPF_STX) {
3570-
/* BPF_STX (including atomic variants) has multiple source
3577+
/* BPF_STX (including atomic variants) has one or more source
35713578
* operands, one of which is a ptr. Check whether the caller is
35723579
* asking about it.
35733580
*/
@@ -4181,7 +4188,7 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx,
41814188
* dreg still needs precision before this insn
41824189
*/
41834190
}
4184-
} else if (class == BPF_LDX) {
4191+
} else if (class == BPF_LDX || is_atomic_load_insn(insn)) {
41854192
if (!bt_is_reg_set(bt, dreg))
41864193
return 0;
41874194
bt_clear_reg(bt, dreg);
@@ -7766,6 +7773,32 @@ static int check_atomic_rmw(struct bpf_verifier_env *env,
77667773
return 0;
77677774
}
77687775

7776+
static int check_atomic_load(struct bpf_verifier_env *env,
7777+
struct bpf_insn *insn)
7778+
{
7779+
if (!atomic_ptr_type_ok(env, insn->src_reg, insn)) {
7780+
verbose(env, "BPF_ATOMIC loads from R%d %s is not allowed\n",
7781+
insn->src_reg,
7782+
reg_type_str(env, reg_state(env, insn->src_reg)->type));
7783+
return -EACCES;
7784+
}
7785+
7786+
return check_load_mem(env, insn, true, false, false, "atomic_load");
7787+
}
7788+
7789+
static int check_atomic_store(struct bpf_verifier_env *env,
7790+
struct bpf_insn *insn)
7791+
{
7792+
if (!atomic_ptr_type_ok(env, insn->dst_reg, insn)) {
7793+
verbose(env, "BPF_ATOMIC stores into R%d %s is not allowed\n",
7794+
insn->dst_reg,
7795+
reg_type_str(env, reg_state(env, insn->dst_reg)->type));
7796+
return -EACCES;
7797+
}
7798+
7799+
return check_store_reg(env, insn, true);
7800+
}
7801+
77697802
static int check_atomic(struct bpf_verifier_env *env, struct bpf_insn *insn)
77707803
{
77717804
switch (insn->imm) {
@@ -7780,6 +7813,20 @@ static int check_atomic(struct bpf_verifier_env *env, struct bpf_insn *insn)
77807813
case BPF_XCHG:
77817814
case BPF_CMPXCHG:
77827815
return check_atomic_rmw(env, insn);
7816+
case BPF_LOAD_ACQ:
7817+
if (BPF_SIZE(insn->code) == BPF_DW && BITS_PER_LONG != 64) {
7818+
verbose(env,
7819+
"64-bit load-acquires are only supported on 64-bit arches\n");
7820+
return -EOPNOTSUPP;
7821+
}
7822+
return check_atomic_load(env, insn);
7823+
case BPF_STORE_REL:
7824+
if (BPF_SIZE(insn->code) == BPF_DW && BITS_PER_LONG != 64) {
7825+
verbose(env,
7826+
"64-bit store-releases are only supported on 64-bit arches\n");
7827+
return -EOPNOTSUPP;
7828+
}
7829+
return check_atomic_store(env, insn);
77837830
default:
77847831
verbose(env, "BPF_ATOMIC uses invalid atomic opcode %02x\n",
77857832
insn->imm);
@@ -20605,7 +20652,9 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env)
2060520652
insn->code == (BPF_ST | BPF_MEM | BPF_W) ||
2060620653
insn->code == (BPF_ST | BPF_MEM | BPF_DW)) {
2060720654
type = BPF_WRITE;
20608-
} else if ((insn->code == (BPF_STX | BPF_ATOMIC | BPF_W) ||
20655+
} else if ((insn->code == (BPF_STX | BPF_ATOMIC | BPF_B) ||
20656+
insn->code == (BPF_STX | BPF_ATOMIC | BPF_H) ||
20657+
insn->code == (BPF_STX | BPF_ATOMIC | BPF_W) ||
2060920658
insn->code == (BPF_STX | BPF_ATOMIC | BPF_DW)) &&
2061020659
env->insn_aux_data[i + delta].ptr_type == PTR_TO_ARENA) {
2061120660
insn->code = BPF_STX | BPF_PROBE_ATOMIC | BPF_SIZE(insn->code);

‎tools/include/uapi/linux/bpf.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,9 @@
5151
#define BPF_XCHG (0xe0 | BPF_FETCH) /* atomic exchange */
5252
#define BPF_CMPXCHG (0xf0 | BPF_FETCH) /* atomic compare-and-write */
5353

54+
#define BPF_LOAD_ACQ 0x100 /* load-acquire */
55+
#define BPF_STORE_REL 0x110 /* store-release */
56+
5457
enum bpf_cond_pseudo_jmp {
5558
BPF_MAY_GOTO = 0,
5659
};

0 commit comments

Comments
 (0)