Skip to content

Commit 6724a76

Browse files
guoren83palmer-dabbelt
guoren83
authored and
palmer-dabbelt
committed
riscv: ftrace: Reduce the detour code size to half
Use a temporary register to reduce the size of detour code from 16 bytes to 8 bytes. The previous implementation is from 'commit afc76b8 ("riscv: Using PATCHABLE_FUNCTION_ENTRY instead of MCOUNT")'. Before the patch: <func_prolog>: 0: REG_S ra, -SZREG(sp) 4: auipc ra, ? 8: jalr ?(ra) 12: REG_L ra, -SZREG(sp) (func_boddy) After the patch: <func_prolog>: 0: auipc t0, ? 4: jalr t0, ?(t0) (func_boddy) This patch not just reduces the size of detour code, but also fixes an important issue: An Ftrace callback registered with FTRACE_OPS_FL_IPMODIFY flag can actually change the instruction pointer, e.g. to "replace" the given kernel function with a new one, which is needed for livepatching, etc. In this case, the trampoline (ftrace_regs_caller) would not return to <func_prolog+12> but would rather jump to the new function. So, "REG_L ra, -SZREG(sp)" would not run and the original return address would not be restored. The kernel is likely to hang or crash as a result. This can be easily demonstrated if one tries to "replace", say, cmdline_proc_show() with a new function with the same signature using instruction_pointer_set(&fregs->regs, new_func_addr) in the Ftrace callback. Link: https://lore.kernel.org/linux-riscv/20221122075440.1165172-1-suagrfillet@gmail.com/ Link: https://lore.kernel.org/linux-riscv/d7d5730b-ebef-68e5-5046-e763e1ee6164@yadro.com/ Co-developed-by: Song Shuai <suagrfillet@gmail.com> Signed-off-by: Song Shuai <suagrfillet@gmail.com> Signed-off-by: Guo Ren <guoren@linux.alibaba.com> Signed-off-by: Guo Ren <guoren@kernel.org> Cc: Evgenii Shatokhin <e.shatokhin@yadro.com> Reviewed-by: Evgenii Shatokhin <e.shatokhin@yadro.com> Link: https://lore.kernel.org/r/20230112090603.1295340-4-guoren@kernel.org Cc: stable@vger.kernel.org Fixes: 10626c3 ("riscv/ftrace: Add basic support") Signed-off-by: Palmer Dabbelt <palmer@rivosinc.com>
1 parent 409c8fb commit 6724a76

File tree

4 files changed

+75
-86
lines changed

4 files changed

+75
-86
lines changed

‎arch/riscv/Makefile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,9 @@ ifeq ($(CONFIG_DYNAMIC_FTRACE),y)
1212
LDFLAGS_vmlinux := --no-relax
1313
KBUILD_CPPFLAGS += -DCC_USING_PATCHABLE_FUNCTION_ENTRY
1414
ifeq ($(CONFIG_RISCV_ISA_C),y)
15-
CC_FLAGS_FTRACE := -fpatchable-function-entry=8
16-
else
1715
CC_FLAGS_FTRACE := -fpatchable-function-entry=4
16+
else
17+
CC_FLAGS_FTRACE := -fpatchable-function-entry=2
1818
endif
1919
endif
2020

‎arch/riscv/include/asm/ftrace.h‎

Lines changed: 38 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,14 @@ struct dyn_arch_ftrace {
4242
* 2) jalr: setting low-12 offset to ra, jump to ra, and set ra to
4343
* return address (original pc + 4)
4444
*
45+
*<ftrace enable>:
46+
* 0: auipc t0/ra, 0x?
47+
* 4: jalr t0/ra, ?(t0/ra)
48+
*
49+
*<ftrace disable>:
50+
* 0: nop
51+
* 4: nop
52+
*
4553
* Dynamic ftrace generates probes to call sites, so we must deal with
4654
* both auipc and jalr at the same time.
4755
*/
@@ -52,25 +60,43 @@ struct dyn_arch_ftrace {
5260
#define AUIPC_OFFSET_MASK (0xfffff000)
5361
#define AUIPC_PAD (0x00001000)
5462
#define JALR_SHIFT 20
55-
#define JALR_BASIC (0x000080e7)
56-
#define AUIPC_BASIC (0x00000097)
63+
#define JALR_RA (0x000080e7)
64+
#define AUIPC_RA (0x00000097)
65+
#define JALR_T0 (0x000282e7)
66+
#define AUIPC_T0 (0x00000297)
5767
#define NOP4 (0x00000013)
5868

59-
#define make_call(caller, callee, call) \
69+
#define to_jalr_t0(offset) \
70+
(((offset & JALR_OFFSET_MASK) << JALR_SHIFT) | JALR_T0)
71+
72+
#define to_auipc_t0(offset) \
73+
((offset & JALR_SIGN_MASK) ? \
74+
(((offset & AUIPC_OFFSET_MASK) + AUIPC_PAD) | AUIPC_T0) : \
75+
((offset & AUIPC_OFFSET_MASK) | AUIPC_T0))
76+
77+
#define make_call_t0(caller, callee, call) \
6078
do { \
61-
call[0] = to_auipc_insn((unsigned int)((unsigned long)callee - \
62-
(unsigned long)caller)); \
63-
call[1] = to_jalr_insn((unsigned int)((unsigned long)callee - \
64-
(unsigned long)caller)); \
79+
unsigned int offset = \
80+
(unsigned long) callee - (unsigned long) caller; \
81+
call[0] = to_auipc_t0(offset); \
82+
call[1] = to_jalr_t0(offset); \
6583
} while (0)
6684

67-
#define to_jalr_insn(offset) \
68-
(((offset & JALR_OFFSET_MASK) << JALR_SHIFT) | JALR_BASIC)
85+
#define to_jalr_ra(offset) \
86+
(((offset & JALR_OFFSET_MASK) << JALR_SHIFT) | JALR_RA)
6987

70-
#define to_auipc_insn(offset) \
88+
#define to_auipc_ra(offset) \
7189
((offset & JALR_SIGN_MASK) ? \
72-
(((offset & AUIPC_OFFSET_MASK) + AUIPC_PAD) | AUIPC_BASIC) : \
73-
((offset & AUIPC_OFFSET_MASK) | AUIPC_BASIC))
90+
(((offset & AUIPC_OFFSET_MASK) + AUIPC_PAD) | AUIPC_RA) : \
91+
((offset & AUIPC_OFFSET_MASK) | AUIPC_RA))
92+
93+
#define make_call_ra(caller, callee, call) \
94+
do { \
95+
unsigned int offset = \
96+
(unsigned long) callee - (unsigned long) caller; \
97+
call[0] = to_auipc_ra(offset); \
98+
call[1] = to_jalr_ra(offset); \
99+
} while (0)
74100

75101
/*
76102
* Let auipc+jalr be the basic *mcount unit*, so we make it 8 bytes here.

‎arch/riscv/kernel/ftrace.c‎

Lines changed: 19 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -55,12 +55,15 @@ static int ftrace_check_current_call(unsigned long hook_pos,
5555
}
5656

5757
static int __ftrace_modify_call(unsigned long hook_pos, unsigned long target,
58-
bool enable)
58+
bool enable, bool ra)
5959
{
6060
unsigned int call[2];
6161
unsigned int nops[2] = {NOP4, NOP4};
6262

63-
make_call(hook_pos, target, call);
63+
if (ra)
64+
make_call_ra(hook_pos, target, call);
65+
else
66+
make_call_t0(hook_pos, target, call);
6467

6568
/* Replace the auipc-jalr pair at once. Return -EPERM on write error. */
6669
if (patch_text_nosync
@@ -70,42 +73,13 @@ static int __ftrace_modify_call(unsigned long hook_pos, unsigned long target,
7073
return 0;
7174
}
7275

73-
/*
74-
* Put 5 instructions with 16 bytes at the front of function within
75-
* patchable function entry nops' area.
76-
*
77-
* 0: REG_S ra, -SZREG(sp)
78-
* 1: auipc ra, 0x?
79-
* 2: jalr -?(ra)
80-
* 3: REG_L ra, -SZREG(sp)
81-
*
82-
* So the opcodes is:
83-
* 0: 0xfe113c23 (sd)/0xfe112e23 (sw)
84-
* 1: 0x???????? -> auipc
85-
* 2: 0x???????? -> jalr
86-
* 3: 0xff813083 (ld)/0xffc12083 (lw)
87-
*/
88-
#if __riscv_xlen == 64
89-
#define INSN0 0xfe113c23
90-
#define INSN3 0xff813083
91-
#elif __riscv_xlen == 32
92-
#define INSN0 0xfe112e23
93-
#define INSN3 0xffc12083
94-
#endif
95-
96-
#define FUNC_ENTRY_SIZE 16
97-
#define FUNC_ENTRY_JMP 4
98-
9976
int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
10077
{
101-
unsigned int call[4] = {INSN0, 0, 0, INSN3};
102-
unsigned long target = addr;
103-
unsigned long caller = rec->ip + FUNC_ENTRY_JMP;
78+
unsigned int call[2];
10479

105-
call[1] = to_auipc_insn((unsigned int)(target - caller));
106-
call[2] = to_jalr_insn((unsigned int)(target - caller));
80+
make_call_t0(rec->ip, addr, call);
10781

108-
if (patch_text_nosync((void *)rec->ip, call, FUNC_ENTRY_SIZE))
82+
if (patch_text_nosync((void *)rec->ip, call, MCOUNT_INSN_SIZE))
10983
return -EPERM;
11084

11185
return 0;
@@ -114,15 +88,14 @@ int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
11488
int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec,
11589
unsigned long addr)
11690
{
117-
unsigned int nops[4] = {NOP4, NOP4, NOP4, NOP4};
91+
unsigned int nops[2] = {NOP4, NOP4};
11892

119-
if (patch_text_nosync((void *)rec->ip, nops, FUNC_ENTRY_SIZE))
93+
if (patch_text_nosync((void *)rec->ip, nops, MCOUNT_INSN_SIZE))
12094
return -EPERM;
12195

12296
return 0;
12397
}
12498

125-
12699
/*
127100
* This is called early on, and isn't wrapped by
128101
* ftrace_arch_code_modify_{prepare,post_process}() and therefor doesn't hold
@@ -144,10 +117,10 @@ int ftrace_init_nop(struct module *mod, struct dyn_ftrace *rec)
144117
int ftrace_update_ftrace_func(ftrace_func_t func)
145118
{
146119
int ret = __ftrace_modify_call((unsigned long)&ftrace_call,
147-
(unsigned long)func, true);
120+
(unsigned long)func, true, true);
148121
if (!ret) {
149122
ret = __ftrace_modify_call((unsigned long)&ftrace_regs_call,
150-
(unsigned long)func, true);
123+
(unsigned long)func, true, true);
151124
}
152125

153126
return ret;
@@ -159,16 +132,16 @@ int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
159132
unsigned long addr)
160133
{
161134
unsigned int call[2];
162-
unsigned long caller = rec->ip + FUNC_ENTRY_JMP;
135+
unsigned long caller = rec->ip;
163136
int ret;
164137

165-
make_call(caller, old_addr, call);
138+
make_call_t0(caller, old_addr, call);
166139
ret = ftrace_check_current_call(caller, call);
167140

168141
if (ret)
169142
return ret;
170143

171-
return __ftrace_modify_call(caller, addr, true);
144+
return __ftrace_modify_call(caller, addr, true, false);
172145
}
173146
#endif
174147

@@ -203,25 +176,25 @@ int ftrace_enable_ftrace_graph_caller(void)
203176
int ret;
204177

205178
ret = __ftrace_modify_call((unsigned long)&ftrace_graph_call,
206-
(unsigned long)&prepare_ftrace_return, true);
179+
(unsigned long)&prepare_ftrace_return, true, true);
207180
if (ret)
208181
return ret;
209182

210183
return __ftrace_modify_call((unsigned long)&ftrace_graph_regs_call,
211-
(unsigned long)&prepare_ftrace_return, true);
184+
(unsigned long)&prepare_ftrace_return, true, true);
212185
}
213186

214187
int ftrace_disable_ftrace_graph_caller(void)
215188
{
216189
int ret;
217190

218191
ret = __ftrace_modify_call((unsigned long)&ftrace_graph_call,
219-
(unsigned long)&prepare_ftrace_return, false);
192+
(unsigned long)&prepare_ftrace_return, false, true);
220193
if (ret)
221194
return ret;
222195

223196
return __ftrace_modify_call((unsigned long)&ftrace_graph_regs_call,
224-
(unsigned long)&prepare_ftrace_return, false);
197+
(unsigned long)&prepare_ftrace_return, false, true);
225198
}
226199
#endif /* CONFIG_DYNAMIC_FTRACE */
227200
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */

‎arch/riscv/kernel/mcount-dyn.S‎

Lines changed: 16 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,8 @@
1313

1414
.text
1515

16-
#define FENTRY_RA_OFFSET 12
17-
#define ABI_SIZE_ON_STACK 72
16+
#define FENTRY_RA_OFFSET 8
17+
#define ABI_SIZE_ON_STACK 80
1818
#define ABI_A0 0
1919
#define ABI_A1 8
2020
#define ABI_A2 16
@@ -23,10 +23,10 @@
2323
#define ABI_A5 40
2424
#define ABI_A6 48
2525
#define ABI_A7 56
26-
#define ABI_RA 64
26+
#define ABI_T0 64
27+
#define ABI_RA 72
2728

2829
.macro SAVE_ABI
29-
addi sp, sp, -SZREG
3030
addi sp, sp, -ABI_SIZE_ON_STACK
3131

3232
REG_S a0, ABI_A0(sp)
@@ -37,6 +37,7 @@
3737
REG_S a5, ABI_A5(sp)
3838
REG_S a6, ABI_A6(sp)
3939
REG_S a7, ABI_A7(sp)
40+
REG_S t0, ABI_T0(sp)
4041
REG_S ra, ABI_RA(sp)
4142
.endm
4243

@@ -49,24 +50,18 @@
4950
REG_L a5, ABI_A5(sp)
5051
REG_L a6, ABI_A6(sp)
5152
REG_L a7, ABI_A7(sp)
53+
REG_L t0, ABI_T0(sp)
5254
REG_L ra, ABI_RA(sp)
5355

5456
addi sp, sp, ABI_SIZE_ON_STACK
55-
addi sp, sp, SZREG
5657
.endm
5758

5859
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
5960
.macro SAVE_ALL
60-
addi sp, sp, -SZREG
6161
addi sp, sp, -PT_SIZE_ON_STACK
6262

63-
REG_S x1, PT_EPC(sp)
64-
addi sp, sp, PT_SIZE_ON_STACK
65-
REG_L x1, (sp)
66-
addi sp, sp, -PT_SIZE_ON_STACK
63+
REG_S t0, PT_EPC(sp)
6764
REG_S x1, PT_RA(sp)
68-
REG_L x1, PT_EPC(sp)
69-
7065
REG_S x2, PT_SP(sp)
7166
REG_S x3, PT_GP(sp)
7267
REG_S x4, PT_TP(sp)
@@ -100,15 +95,11 @@
10095
.endm
10196

10297
.macro RESTORE_ALL
98+
REG_L t0, PT_EPC(sp)
10399
REG_L x1, PT_RA(sp)
104-
addi sp, sp, PT_SIZE_ON_STACK
105-
REG_S x1, (sp)
106-
addi sp, sp, -PT_SIZE_ON_STACK
107-
REG_L x1, PT_EPC(sp)
108100
REG_L x2, PT_SP(sp)
109101
REG_L x3, PT_GP(sp)
110102
REG_L x4, PT_TP(sp)
111-
REG_L x5, PT_T0(sp)
112103
REG_L x6, PT_T1(sp)
113104
REG_L x7, PT_T2(sp)
114105
REG_L x8, PT_S0(sp)
@@ -137,26 +128,25 @@
137128
REG_L x31, PT_T6(sp)
138129

139130
addi sp, sp, PT_SIZE_ON_STACK
140-
addi sp, sp, SZREG
141131
.endm
142132
#endif /* CONFIG_DYNAMIC_FTRACE_WITH_REGS */
143133

144134
ENTRY(ftrace_caller)
145135
SAVE_ABI
146136

147-
addi a0, ra, -FENTRY_RA_OFFSET
137+
addi a0, t0, -FENTRY_RA_OFFSET
148138
la a1, function_trace_op
149139
REG_L a2, 0(a1)
150-
REG_L a1, ABI_SIZE_ON_STACK(sp)
140+
mv a1, ra
151141
mv a3, sp
152142

153143
ftrace_call:
154144
.global ftrace_call
155145
call ftrace_stub
156146

157147
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
158-
addi a0, sp, ABI_SIZE_ON_STACK
159-
REG_L a1, ABI_RA(sp)
148+
addi a0, sp, ABI_RA
149+
REG_L a1, ABI_T0(sp)
160150
addi a1, a1, -FENTRY_RA_OFFSET
161151
#ifdef HAVE_FUNCTION_GRAPH_FP_TEST
162152
mv a2, s0
@@ -166,17 +156,17 @@ ftrace_graph_call:
166156
call ftrace_stub
167157
#endif
168158
RESTORE_ABI
169-
ret
159+
jr t0
170160
ENDPROC(ftrace_caller)
171161

172162
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
173163
ENTRY(ftrace_regs_caller)
174164
SAVE_ALL
175165

176-
addi a0, ra, -FENTRY_RA_OFFSET
166+
addi a0, t0, -FENTRY_RA_OFFSET
177167
la a1, function_trace_op
178168
REG_L a2, 0(a1)
179-
REG_L a1, PT_SIZE_ON_STACK(sp)
169+
mv a1, ra
180170
mv a3, sp
181171

182172
ftrace_regs_call:
@@ -196,6 +186,6 @@ ftrace_graph_regs_call:
196186
#endif
197187

198188
RESTORE_ALL
199-
ret
189+
jr t0
200190
ENDPROC(ftrace_regs_caller)
201191
#endif /* CONFIG_DYNAMIC_FTRACE_WITH_REGS */

0 commit comments

Comments
 (0)