Skip to content

Commit d1ef551

Browse files
davem330davem330
committed
Merge branch 'bpf-mips-jit-improvements'
David Daney says: ==================== MIPS,bpf: Improvements for MIPS eBPF JIT Here are several improvements and bug fixes for the MIPS eBPF JIT. The main change is the addition of support for JLT, JLE, JSLT and JSLE ops, that were recently added. Also fix WARN output when used with preemptable kernel, and a small cleanup/optimization in the use of BPF_OP(insn->code). I suggest that the whole thing go via the BPF/net-next path as there are dependencies on code that is not yet merged to Linus' tree. Still pending are changes to reduce stack usage when the verifier can determine the maximum stack size. ==================== Signed-off-by: David S. Miller <davem@davemloft.net>
2 parents da6817e + 6035b3f commit d1ef551

File tree

1 file changed

+103
-59
lines changed

1 file changed

+103
-59
lines changed

‎arch/mips/net/ebpf_jit.c

Lines changed: 103 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,7 @@ struct jit_ctx {
113113
u64 *reg_val_types;
114114
unsigned int long_b_conversion:1;
115115
unsigned int gen_b_offsets:1;
116+
unsigned int use_bbit_insns:1;
116117
};
117118

118119
static void set_reg_val_type(u64 *rvt, int reg, enum reg_val_type type)
@@ -655,19 +656,6 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx, int this_idx)
655656
return build_int_epilogue(ctx, MIPS_R_T9);
656657
}
657658

658-
static bool use_bbit_insns(void)
659-
{
660-
switch (current_cpu_type()) {
661-
case CPU_CAVIUM_OCTEON:
662-
case CPU_CAVIUM_OCTEON_PLUS:
663-
case CPU_CAVIUM_OCTEON2:
664-
case CPU_CAVIUM_OCTEON3:
665-
return true;
666-
default:
667-
return false;
668-
}
669-
}
670-
671659
static bool is_bad_offset(int b_off)
672660
{
673661
return b_off > 0x1ffff || b_off < -0x20000;
@@ -682,6 +670,7 @@ static int build_one_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
682670
unsigned int target;
683671
u64 t64;
684672
s64 t64s;
673+
int bpf_op = BPF_OP(insn->code);
685674

686675
switch (insn->code) {
687676
case BPF_ALU64 | BPF_ADD | BPF_K: /* ALU64_IMM */
@@ -770,13 +759,13 @@ static int build_one_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
770759
emit_instr(ctx, sll, dst, dst, 0);
771760
if (insn->imm == 1) {
772761
/* div by 1 is a nop, mod by 1 is zero */
773-
if (BPF_OP(insn->code) == BPF_MOD)
762+
if (bpf_op == BPF_MOD)
774763
emit_instr(ctx, addu, dst, MIPS_R_ZERO, MIPS_R_ZERO);
775764
break;
776765
}
777766
gen_imm_to_reg(insn, MIPS_R_AT, ctx);
778767
emit_instr(ctx, divu, dst, MIPS_R_AT);
779-
if (BPF_OP(insn->code) == BPF_DIV)
768+
if (bpf_op == BPF_DIV)
780769
emit_instr(ctx, mflo, dst);
781770
else
782771
emit_instr(ctx, mfhi, dst);
@@ -798,13 +787,13 @@ static int build_one_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
798787

799788
if (insn->imm == 1) {
800789
/* div by 1 is a nop, mod by 1 is zero */
801-
if (BPF_OP(insn->code) == BPF_MOD)
790+
if (bpf_op == BPF_MOD)
802791
emit_instr(ctx, addu, dst, MIPS_R_ZERO, MIPS_R_ZERO);
803792
break;
804793
}
805794
gen_imm_to_reg(insn, MIPS_R_AT, ctx);
806795
emit_instr(ctx, ddivu, dst, MIPS_R_AT);
807-
if (BPF_OP(insn->code) == BPF_DIV)
796+
if (bpf_op == BPF_DIV)
808797
emit_instr(ctx, mflo, dst);
809798
else
810799
emit_instr(ctx, mfhi, dst);
@@ -829,7 +818,7 @@ static int build_one_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
829818
emit_instr(ctx, dinsu, dst, MIPS_R_ZERO, 32, 32);
830819
did_move = false;
831820
if (insn->src_reg == BPF_REG_10) {
832-
if (BPF_OP(insn->code) == BPF_MOV) {
821+
if (bpf_op == BPF_MOV) {
833822
emit_instr(ctx, daddiu, dst, MIPS_R_SP, MAX_BPF_STACK);
834823
did_move = true;
835824
} else {
@@ -839,15 +828,15 @@ static int build_one_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
839828
} else if (get_reg_val_type(ctx, this_idx, insn->src_reg) == REG_32BIT) {
840829
int tmp_reg = MIPS_R_AT;
841830

842-
if (BPF_OP(insn->code) == BPF_MOV) {
831+
if (bpf_op == BPF_MOV) {
843832
tmp_reg = dst;
844833
did_move = true;
845834
}
846835
emit_instr(ctx, daddu, tmp_reg, src, MIPS_R_ZERO);
847836
emit_instr(ctx, dinsu, tmp_reg, MIPS_R_ZERO, 32, 32);
848837
src = MIPS_R_AT;
849838
}
850-
switch (BPF_OP(insn->code)) {
839+
switch (bpf_op) {
851840
case BPF_MOV:
852841
if (!did_move)
853842
emit_instr(ctx, daddu, dst, src, MIPS_R_ZERO);
@@ -879,7 +868,7 @@ static int build_one_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
879868
emit_instr(ctx, beq, src, MIPS_R_ZERO, b_off);
880869
emit_instr(ctx, movz, MIPS_R_V0, MIPS_R_ZERO, src);
881870
emit_instr(ctx, ddivu, dst, src);
882-
if (BPF_OP(insn->code) == BPF_DIV)
871+
if (bpf_op == BPF_DIV)
883872
emit_instr(ctx, mflo, dst);
884873
else
885874
emit_instr(ctx, mfhi, dst);
@@ -923,15 +912,15 @@ static int build_one_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
923912
if (ts == REG_64BIT || ts == REG_32BIT_ZERO_EX) {
924913
int tmp_reg = MIPS_R_AT;
925914

926-
if (BPF_OP(insn->code) == BPF_MOV) {
915+
if (bpf_op == BPF_MOV) {
927916
tmp_reg = dst;
928917
did_move = true;
929918
}
930919
/* sign extend */
931920
emit_instr(ctx, sll, tmp_reg, src, 0);
932921
src = MIPS_R_AT;
933922
}
934-
switch (BPF_OP(insn->code)) {
923+
switch (bpf_op) {
935924
case BPF_MOV:
936925
if (!did_move)
937926
emit_instr(ctx, addu, dst, src, MIPS_R_ZERO);
@@ -962,7 +951,7 @@ static int build_one_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
962951
emit_instr(ctx, beq, src, MIPS_R_ZERO, b_off);
963952
emit_instr(ctx, movz, MIPS_R_V0, MIPS_R_ZERO, src);
964953
emit_instr(ctx, divu, dst, src);
965-
if (BPF_OP(insn->code) == BPF_DIV)
954+
if (bpf_op == BPF_DIV)
966955
emit_instr(ctx, mflo, dst);
967956
else
968957
emit_instr(ctx, mfhi, dst);
@@ -989,7 +978,7 @@ static int build_one_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
989978
break;
990979
case BPF_JMP | BPF_JEQ | BPF_K: /* JMP_IMM */
991980
case BPF_JMP | BPF_JNE | BPF_K: /* JMP_IMM */
992-
cmp_eq = (BPF_OP(insn->code) == BPF_JEQ);
981+
cmp_eq = (bpf_op == BPF_JEQ);
993982
dst = ebpf_to_mips_reg(ctx, insn, dst_reg_fp_ok);
994983
if (dst < 0)
995984
return dst;
@@ -1002,8 +991,12 @@ static int build_one_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
1002991
goto jeq_common;
1003992
case BPF_JMP | BPF_JEQ | BPF_X: /* JMP_REG */
1004993
case BPF_JMP | BPF_JNE | BPF_X:
994+
case BPF_JMP | BPF_JSLT | BPF_X:
995+
case BPF_JMP | BPF_JSLE | BPF_X:
1005996
case BPF_JMP | BPF_JSGT | BPF_X:
1006997
case BPF_JMP | BPF_JSGE | BPF_X:
998+
case BPF_JMP | BPF_JLT | BPF_X:
999+
case BPF_JMP | BPF_JLE | BPF_X:
10071000
case BPF_JMP | BPF_JGT | BPF_X:
10081001
case BPF_JMP | BPF_JGE | BPF_X:
10091002
case BPF_JMP | BPF_JSET | BPF_X:
@@ -1020,50 +1013,56 @@ static int build_one_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
10201013
emit_instr(ctx, sll, MIPS_R_AT, dst, 0);
10211014
dst = MIPS_R_AT;
10221015
}
1023-
if (BPF_OP(insn->code) == BPF_JSET) {
1016+
if (bpf_op == BPF_JSET) {
10241017
emit_instr(ctx, and, MIPS_R_AT, dst, src);
10251018
cmp_eq = false;
10261019
dst = MIPS_R_AT;
10271020
src = MIPS_R_ZERO;
1028-
} else if (BPF_OP(insn->code) == BPF_JSGT) {
1021+
} else if (bpf_op == BPF_JSGT || bpf_op == BPF_JSLE) {
10291022
emit_instr(ctx, dsubu, MIPS_R_AT, dst, src);
10301023
if ((insn + 1)->code == (BPF_JMP | BPF_EXIT) && insn->off == 1) {
10311024
b_off = b_imm(exit_idx, ctx);
10321025
if (is_bad_offset(b_off))
10331026
return -E2BIG;
1034-
emit_instr(ctx, blez, MIPS_R_AT, b_off);
1027+
if (bpf_op == BPF_JSGT)
1028+
emit_instr(ctx, blez, MIPS_R_AT, b_off);
1029+
else
1030+
emit_instr(ctx, bgtz, MIPS_R_AT, b_off);
10351031
emit_instr(ctx, nop);
10361032
return 2; /* We consumed the exit. */
10371033
}
10381034
b_off = b_imm(this_idx + insn->off + 1, ctx);
10391035
if (is_bad_offset(b_off))
10401036
return -E2BIG;
1041-
emit_instr(ctx, bgtz, MIPS_R_AT, b_off);
1037+
if (bpf_op == BPF_JSGT)
1038+
emit_instr(ctx, bgtz, MIPS_R_AT, b_off);
1039+
else
1040+
emit_instr(ctx, blez, MIPS_R_AT, b_off);
10421041
emit_instr(ctx, nop);
10431042
break;
1044-
} else if (BPF_OP(insn->code) == BPF_JSGE) {
1043+
} else if (bpf_op == BPF_JSGE || bpf_op == BPF_JSLT) {
10451044
emit_instr(ctx, slt, MIPS_R_AT, dst, src);
1046-
cmp_eq = true;
1045+
cmp_eq = bpf_op == BPF_JSGE;
10471046
dst = MIPS_R_AT;
10481047
src = MIPS_R_ZERO;
1049-
} else if (BPF_OP(insn->code) == BPF_JGT) {
1048+
} else if (bpf_op == BPF_JGT || bpf_op == BPF_JLE) {
10501049
/* dst or src could be AT */
10511050
emit_instr(ctx, dsubu, MIPS_R_T8, dst, src);
10521051
emit_instr(ctx, sltu, MIPS_R_AT, dst, src);
10531052
/* SP known to be non-zero, movz becomes boolean not */
10541053
emit_instr(ctx, movz, MIPS_R_T9, MIPS_R_SP, MIPS_R_T8);
10551054
emit_instr(ctx, movn, MIPS_R_T9, MIPS_R_ZERO, MIPS_R_T8);
10561055
emit_instr(ctx, or, MIPS_R_AT, MIPS_R_T9, MIPS_R_AT);
1057-
cmp_eq = true;
1056+
cmp_eq = bpf_op == BPF_JGT;
10581057
dst = MIPS_R_AT;
10591058
src = MIPS_R_ZERO;
1060-
} else if (BPF_OP(insn->code) == BPF_JGE) {
1059+
} else if (bpf_op == BPF_JGE || bpf_op == BPF_JLT) {
10611060
emit_instr(ctx, sltu, MIPS_R_AT, dst, src);
1062-
cmp_eq = true;
1061+
cmp_eq = bpf_op == BPF_JGE;
10631062
dst = MIPS_R_AT;
10641063
src = MIPS_R_ZERO;
10651064
} else { /* JNE/JEQ case */
1066-
cmp_eq = (BPF_OP(insn->code) == BPF_JEQ);
1065+
cmp_eq = (bpf_op == BPF_JEQ);
10671066
}
10681067
jeq_common:
10691068
/*
@@ -1122,7 +1121,9 @@ static int build_one_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
11221121
break;
11231122
case BPF_JMP | BPF_JSGT | BPF_K: /* JMP_IMM */
11241123
case BPF_JMP | BPF_JSGE | BPF_K: /* JMP_IMM */
1125-
cmp_eq = (BPF_OP(insn->code) == BPF_JSGE);
1124+
case BPF_JMP | BPF_JSLT | BPF_K: /* JMP_IMM */
1125+
case BPF_JMP | BPF_JSLE | BPF_K: /* JMP_IMM */
1126+
cmp_eq = (bpf_op == BPF_JSGE);
11261127
dst = ebpf_to_mips_reg(ctx, insn, dst_reg_fp_ok);
11271128
if (dst < 0)
11281129
return dst;
@@ -1132,73 +1133,100 @@ static int build_one_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
11321133
b_off = b_imm(exit_idx, ctx);
11331134
if (is_bad_offset(b_off))
11341135
return -E2BIG;
1135-
if (cmp_eq)
1136-
emit_instr(ctx, bltz, dst, b_off);
1137-
else
1136+
switch (bpf_op) {
1137+
case BPF_JSGT:
11381138
emit_instr(ctx, blez, dst, b_off);
1139+
break;
1140+
case BPF_JSGE:
1141+
emit_instr(ctx, bltz, dst, b_off);
1142+
break;
1143+
case BPF_JSLT:
1144+
emit_instr(ctx, bgez, dst, b_off);
1145+
break;
1146+
case BPF_JSLE:
1147+
emit_instr(ctx, bgtz, dst, b_off);
1148+
break;
1149+
}
11391150
emit_instr(ctx, nop);
11401151
return 2; /* We consumed the exit. */
11411152
}
11421153
b_off = b_imm(this_idx + insn->off + 1, ctx);
11431154
if (is_bad_offset(b_off))
11441155
return -E2BIG;
1145-
if (cmp_eq)
1146-
emit_instr(ctx, bgez, dst, b_off);
1147-
else
1156+
switch (bpf_op) {
1157+
case BPF_JSGT:
11481158
emit_instr(ctx, bgtz, dst, b_off);
1159+
break;
1160+
case BPF_JSGE:
1161+
emit_instr(ctx, bgez, dst, b_off);
1162+
break;
1163+
case BPF_JSLT:
1164+
emit_instr(ctx, bltz, dst, b_off);
1165+
break;
1166+
case BPF_JSLE:
1167+
emit_instr(ctx, blez, dst, b_off);
1168+
break;
1169+
}
11491170
emit_instr(ctx, nop);
11501171
break;
11511172
}
11521173
/*
11531174
* only "LT" compare available, so we must use imm + 1
1154-
* to generate "GT"
1175+
* to generate "GT" and imm -1 to generate LE
11551176
*/
1156-
t64s = insn->imm + (cmp_eq ? 0 : 1);
1177+
if (bpf_op == BPF_JSGT)
1178+
t64s = insn->imm + 1;
1179+
else if (bpf_op == BPF_JSLE)
1180+
t64s = insn->imm + 1;
1181+
else
1182+
t64s = insn->imm;
1183+
1184+
cmp_eq = bpf_op == BPF_JSGT || bpf_op == BPF_JSGE;
11571185
if (t64s >= S16_MIN && t64s <= S16_MAX) {
11581186
emit_instr(ctx, slti, MIPS_R_AT, dst, (int)t64s);
11591187
src = MIPS_R_AT;
11601188
dst = MIPS_R_ZERO;
1161-
cmp_eq = true;
11621189
goto jeq_common;
11631190
}
11641191
emit_const_to_reg(ctx, MIPS_R_AT, (u64)t64s);
11651192
emit_instr(ctx, slt, MIPS_R_AT, dst, MIPS_R_AT);
11661193
src = MIPS_R_AT;
11671194
dst = MIPS_R_ZERO;
1168-
cmp_eq = true;
11691195
goto jeq_common;
11701196

11711197
case BPF_JMP | BPF_JGT | BPF_K:
11721198
case BPF_JMP | BPF_JGE | BPF_K:
1173-
cmp_eq = (BPF_OP(insn->code) == BPF_JGE);
1199+
case BPF_JMP | BPF_JLT | BPF_K:
1200+
case BPF_JMP | BPF_JLE | BPF_K:
1201+
cmp_eq = (bpf_op == BPF_JGE);
11741202
dst = ebpf_to_mips_reg(ctx, insn, dst_reg_fp_ok);
11751203
if (dst < 0)
11761204
return dst;
11771205
/*
11781206
* only "LT" compare available, so we must use imm + 1
1179-
* to generate "GT"
1207+
* to generate "GT" and imm -1 to generate LE
11801208
*/
1181-
t64s = (u64)(u32)(insn->imm) + (cmp_eq ? 0 : 1);
1182-
if (t64s >= 0 && t64s <= S16_MAX) {
1183-
emit_instr(ctx, sltiu, MIPS_R_AT, dst, (int)t64s);
1184-
src = MIPS_R_AT;
1185-
dst = MIPS_R_ZERO;
1186-
cmp_eq = true;
1187-
goto jeq_common;
1188-
}
1209+
if (bpf_op == BPF_JGT)
1210+
t64s = (u64)(u32)(insn->imm) + 1;
1211+
else if (bpf_op == BPF_JLE)
1212+
t64s = (u64)(u32)(insn->imm) + 1;
1213+
else
1214+
t64s = (u64)(u32)(insn->imm);
1215+
1216+
cmp_eq = bpf_op == BPF_JGT || bpf_op == BPF_JGE;
1217+
11891218
emit_const_to_reg(ctx, MIPS_R_AT, (u64)t64s);
11901219
emit_instr(ctx, sltu, MIPS_R_AT, dst, MIPS_R_AT);
11911220
src = MIPS_R_AT;
11921221
dst = MIPS_R_ZERO;
1193-
cmp_eq = true;
11941222
goto jeq_common;
11951223

11961224
case BPF_JMP | BPF_JSET | BPF_K: /* JMP_IMM */
11971225
dst = ebpf_to_mips_reg(ctx, insn, dst_reg_fp_ok);
11981226
if (dst < 0)
11991227
return dst;
12001228

1201-
if (use_bbit_insns() && hweight32((u32)insn->imm) == 1) {
1229+
if (ctx->use_bbit_insns && hweight32((u32)insn->imm) == 1) {
12021230
if ((insn + 1)->code == (BPF_JMP | BPF_EXIT) && insn->off == 1) {
12031231
b_off = b_imm(exit_idx, ctx);
12041232
if (is_bad_offset(b_off))
@@ -1724,10 +1752,14 @@ static int reg_val_propagate_range(struct jit_ctx *ctx, u64 initial_rvt,
17241752
case BPF_JEQ:
17251753
case BPF_JGT:
17261754
case BPF_JGE:
1755+
case BPF_JLT:
1756+
case BPF_JLE:
17271757
case BPF_JSET:
17281758
case BPF_JNE:
17291759
case BPF_JSGT:
17301760
case BPF_JSGE:
1761+
case BPF_JSLT:
1762+
case BPF_JSLE:
17311763
if (follow_taken) {
17321764
rvt[idx] |= RVT_BRANCH_TAKEN;
17331765
idx += insn->off;
@@ -1853,6 +1885,18 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
18531885

18541886
memset(&ctx, 0, sizeof(ctx));
18551887

1888+
preempt_disable();
1889+
switch (current_cpu_type()) {
1890+
case CPU_CAVIUM_OCTEON:
1891+
case CPU_CAVIUM_OCTEON_PLUS:
1892+
case CPU_CAVIUM_OCTEON2:
1893+
case CPU_CAVIUM_OCTEON3:
1894+
ctx.use_bbit_insns = 1;
1895+
default:
1896+
ctx.use_bbit_insns = 0;
1897+
}
1898+
preempt_enable();
1899+
18561900
ctx.offsets = kcalloc(prog->len + 1, sizeof(*ctx.offsets), GFP_KERNEL);
18571901
if (ctx.offsets == NULL)
18581902
goto out_err;

0 commit comments

Comments
 (0)