Skip to content

Commit 841b00f

Browse files
dstogovdstogov
committed
Preallocate stack space for JIT in execute_ex() to eliminate JIT prologue/epilogue.
1 parent 5f36d04 commit 841b00f

File tree

5 files changed

+105
-30
lines changed

5 files changed

+105
-30
lines changed

‎Zend/zend_execute.c

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4757,6 +4757,11 @@ static zend_always_inline zend_execute_data *_zend_vm_stack_push_call_frame(uint
47574757
} \
47584758
} while (0)
47594759

4760+
#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
4761+
/* This callback disables optimization of "vm_stack_data" variable in VM */
4762+
void (*zend_touch_vm_stack_data)(void *vm_stack_data) = NULL;
4763+
#endif
4764+
47604765
#include "zend_vm_execute.h"
47614766

47624767
ZEND_API zend_result zend_set_user_opcode_handler(zend_uchar opcode, user_opcode_handler_t handler)

‎Zend/zend_vm_execute.h

Lines changed: 26 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -50826,15 +50826,34 @@ ZEND_API void execute_ex(zend_execute_data *ex)
5082650826
{
5082750827
DCL_OPLINE
5082850828

50829+
#if defined(ZEND_VM_IP_GLOBAL_REG) || defined(ZEND_VM_IP_GLOBAL_REG)
50830+
struct {
5082950831
#ifdef ZEND_VM_IP_GLOBAL_REG
50830-
const zend_op *orig_opline = opline;
50832+
const zend_op *orig_opline;
5083150833
#endif
5083250834
#ifdef ZEND_VM_FP_GLOBAL_REG
50833-
zend_execute_data *orig_execute_data = execute_data;
50835+
zend_execute_data *orig_execute_data;
50836+
#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
50837+
char hybrid_jit_red_zone[ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE];
50838+
#endif
50839+
#endif
50840+
} vm_stack_data;
50841+
#endif
50842+
#ifdef ZEND_VM_IP_GLOBAL_REG
50843+
vm_stack_data.orig_opline = opline;
50844+
#endif
50845+
#ifdef ZEND_VM_FP_GLOBAL_REG
50846+
vm_stack_data.orig_execute_data = execute_data;
5083450847
execute_data = ex;
5083550848
#else
5083650849
zend_execute_data *execute_data = ex;
5083750850
#endif
50851+
#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
50852+
memset(vm_stack_data.hybrid_jit_red_zone, 0, ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE);
50853+
if (zend_touch_vm_stack_data) {
50854+
zend_touch_vm_stack_data(&vm_stack_data);
50855+
}
50856+
#endif
5083850857

5083950858
#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID)
5084050859
if (UNEXPECTED(execute_data == NULL)) {
@@ -58788,10 +58807,10 @@ ZEND_API void execute_ex(zend_execute_data *ex)
5878858807
HYBRID_BREAK();
5878958808
HYBRID_CASE(HYBRID_HALT):
5879058809
#ifdef ZEND_VM_FP_GLOBAL_REG
58791-
execute_data = orig_execute_data;
58810+
execute_data = vm_stack_data.orig_execute_data;
5879258811
#endif
5879358812
#ifdef ZEND_VM_IP_GLOBAL_REG
58794-
opline = orig_opline;
58813+
opline = vm_stack_data.orig_opline;
5879558814
#endif
5879658815
return;
5879758816
HYBRID_DEFAULT:
@@ -58800,9 +58819,9 @@ ZEND_API void execute_ex(zend_execute_data *ex)
5880058819
HYBRID_BREAK(); /* Never reached */
5880158820
#else
5880258821
#ifdef ZEND_VM_FP_GLOBAL_REG
58803-
execute_data = orig_execute_data;
58822+
execute_data = vm_stack_data.orig_execute_data;
5880458823
# ifdef ZEND_VM_IP_GLOBAL_REG
58805-
opline = orig_opline;
58824+
opline = vm_stack_data.orig_opline;
5880658825
# endif
5880758826
return;
5880858827
#else
@@ -58811,7 +58830,7 @@ ZEND_API void execute_ex(zend_execute_data *ex)
5881158830
ZEND_VM_LOOP_INTERRUPT_CHECK();
5881258831
} else {
5881358832
# ifdef ZEND_VM_IP_GLOBAL_REG
58814-
opline = orig_opline;
58833+
opline = vm_stack_data.orig_opline;
5881558834
# endif
5881658835
return;
5881758836
}

‎Zend/zend_vm_gen.php

Lines changed: 32 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1780,10 +1780,10 @@ function gen_executor_code($f, $spec, $kind, $prolog, &$switch_labels = array())
17801780
case ZEND_VM_KIND_HYBRID:
17811781
out($f,"\t\t\tHYBRID_CASE(HYBRID_HALT):\n");
17821782
out($f,"#ifdef ZEND_VM_FP_GLOBAL_REG\n");
1783-
out($f,"\t\t\t\texecute_data = orig_execute_data;\n");
1783+
out($f,"\t\t\t\texecute_data = vm_stack_data.orig_execute_data;\n");
17841784
out($f,"#endif\n");
17851785
out($f,"#ifdef ZEND_VM_IP_GLOBAL_REG\n");
1786-
out($f,"\t\t\t\topline = orig_opline;\n");
1786+
out($f,"\t\t\t\topline = vm_stack_data.orig_opline;\n");
17871787
out($f,"#endif\n");
17881788
out($f,"\t\t\t\treturn;\n");
17891789
out($f,"\t\t\tHYBRID_DEFAULT:\n");
@@ -2065,15 +2065,34 @@ function gen_executor($f, $skl, $spec, $kind, $executor_name, $initializer_name)
20652065
out($f,$m[1]."zend_execute_data *execute_data = ex;\n");
20662066
out($f,"#endif\n");
20672067
} else {
2068+
out($f,"#if defined(ZEND_VM_IP_GLOBAL_REG) || defined(ZEND_VM_IP_GLOBAL_REG)\n");
2069+
out($f,$m[1]."struct {\n");
20682070
out($f,"#ifdef ZEND_VM_IP_GLOBAL_REG\n");
2069-
out($f,$m[1]."const zend_op *orig_opline = opline;\n");
2071+
out($f,$m[1]."\tconst zend_op *orig_opline;\n");
20702072
out($f,"#endif\n");
20712073
out($f,"#ifdef ZEND_VM_FP_GLOBAL_REG\n");
2072-
out($f,$m[1]."zend_execute_data *orig_execute_data = execute_data;\n");
2074+
out($f,$m[1]."\tzend_execute_data *orig_execute_data;\n");
2075+
out($f,"#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE\n");
2076+
out($f,$m[1]."\tchar hybrid_jit_red_zone[ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE];\n");
2077+
out($f,"#endif\n");
2078+
out($f,"#endif\n");
2079+
out($f,$m[1]."} vm_stack_data;\n");
2080+
out($f,"#endif\n");
2081+
out($f,"#ifdef ZEND_VM_IP_GLOBAL_REG\n");
2082+
out($f,$m[1]."vm_stack_data.orig_opline = opline;\n");
2083+
out($f,"#endif\n");
2084+
out($f,"#ifdef ZEND_VM_FP_GLOBAL_REG\n");
2085+
out($f,$m[1]."vm_stack_data.orig_execute_data = execute_data;\n");
20732086
out($f,$m[1]."execute_data = ex;\n");
20742087
out($f,"#else\n");
20752088
out($f,$m[1]."zend_execute_data *execute_data = ex;\n");
20762089
out($f,"#endif\n");
2090+
out($f,"#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE\n");
2091+
out($f,$m[1]."memset(vm_stack_data.hybrid_jit_red_zone, 0, ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE);\n");
2092+
out($f,$m[1]."if (zend_touch_vm_stack_data) {\n");
2093+
out($f,$m[1]."\tzend_touch_vm_stack_data(&vm_stack_data);\n");
2094+
out($f,$m[1]."}\n");
2095+
out($f,"#endif\n");
20772096
}
20782097
break;
20792098
case "INTERNAL_LABELS":
@@ -2159,9 +2178,9 @@ function gen_executor($f, $skl, $spec, $kind, $executor_name, $initializer_name)
21592178
}
21602179
out($f,
21612180
"#ifdef ZEND_VM_FP_GLOBAL_REG\n" .
2162-
$m[1]."execute_data = orig_execute_data;\n" .
2181+
$m[1]."execute_data = vm_stack_data.orig_execute_data;\n" .
21632182
"# ifdef ZEND_VM_IP_GLOBAL_REG\n" .
2164-
$m[1]."opline = orig_opline;\n" .
2183+
$m[1]."opline = vm_stack_data.orig_opline;\n" .
21652184
"# endif\n" .
21662185
$m[1]."return;\n" .
21672186
"#else\n" .
@@ -2170,7 +2189,7 @@ function gen_executor($f, $skl, $spec, $kind, $executor_name, $initializer_name)
21702189
$m[1]."\tZEND_VM_LOOP_INTERRUPT_CHECK();\n".
21712190
$m[1]."} else {\n" .
21722191
"# ifdef ZEND_VM_IP_GLOBAL_REG\n" .
2173-
$m[1]."\topline = orig_opline;\n" .
2192+
$m[1]."\topline = vm_stack_data.orig_opline;\n" .
21742193
"# endif\n".
21752194
$m[1]."\treturn;\n".
21762195
$m[1]."}\n".
@@ -2578,6 +2597,12 @@ function gen_vm($def, $skel) {
25782597
fputs($f, "#define ZEND_VM_KIND\t\t" . $GLOBALS["vm_kind_name"][ZEND_VM_KIND] . "\n");
25792598
}
25802599
fputs($f, "\n");
2600+
fputs($f, "#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) && !defined(__SANITIZE_ADDRESS__)\n");
2601+
fputs($f, "# if (defined(i386) || defined(__x86_64__) || defined(_M_X64))\n");
2602+
fputs($f, "# define ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 16\n");
2603+
fputs($f, "# endif\n");
2604+
fputs($f, "#endif\n");
2605+
fputs($f, "\n");
25812606
foreach($vm_op_flags as $name => $val) {
25822607
fprintf($f, "#define %-24s 0x%08x\n", $name, $val);
25832608
}

‎Zend/zend_vm_opcodes.h

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,12 @@
3434
# define ZEND_VM_KIND ZEND_VM_KIND_CALL
3535
#endif
3636

37+
#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) && !defined(__SANITIZE_ADDRESS__)
38+
# if (defined(i386) || defined(__x86_64__) || defined(_M_X64))
39+
# define ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 16
40+
# endif
41+
#endif
42+
3743
#define ZEND_VM_OP_SPEC 0x00000001
3844
#define ZEND_VM_OP_CONST 0x00000002
3945
#define ZEND_VM_OP_TMPVAR 0x00000004

‎ext/opcache/jit/zend_jit_x86.dasc

Lines changed: 36 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -171,8 +171,21 @@ static void* dasm_labels[zend_lb_MAX];
171171

172172
#define BP_JIT_IS 6
173173

174+
174175
#define CAN_USE_AVX() (JIT_G(opt_flags) & allowed_opt_flags & ZEND_JIT_CPU_AVX)
175176

177+
|.macro ADD_HYBRID_SPAD
178+
||#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
179+
| add r4, HYBRID_SPAD
180+
||#endif
181+
|.endmacro
182+
183+
|.macro SUB_HYBRID_SPAD
184+
||#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
185+
| sub r4, HYBRID_SPAD
186+
||#endif
187+
|.endmacro
188+
176189
|.macro LOAD_ADDR, reg, addr
177190
| .if X64
178191
|| if (IS_32BIT(addr)) {
@@ -1752,7 +1765,7 @@ static int zend_jit_interrupt_handler_stub(dasm_State **Dst)
17521765
}
17531766
| //ZEND_VM_CONTINUE()
17541767
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
1755-
| add r4, HYBRID_SPAD
1768+
| ADD_HYBRID_SPAD
17561769
| JMP_IP
17571770
} else if (GCC_GLOBAL_REGS) {
17581771
| add r4, SPAD // stack alignment
@@ -1774,7 +1787,7 @@ static int zend_jit_exception_handler_stub(dasm_State **Dst)
17741787
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
17751788
const void *handler = zend_get_opcode_handler_func(EG(exception_op));
17761789

1777-
| add r4, HYBRID_SPAD
1790+
| ADD_HYBRID_SPAD
17781791
| EXT_CALL handler, r0
17791792
| JMP_IP
17801793
} else {
@@ -1827,11 +1840,11 @@ static int zend_jit_leave_function_stub(dasm_State **Dst)
18271840
| test FCARG1d, ZEND_CALL_TOP
18281841
| jnz >1
18291842
| EXT_CALL zend_jit_leave_nested_func_helper, r0
1830-
| add r4, HYBRID_SPAD // stack alignment
1843+
| ADD_HYBRID_SPAD
18311844
| JMP_IP
18321845
|1:
18331846
| EXT_CALL zend_jit_leave_top_func_helper, r0
1834-
| add r4, HYBRID_SPAD // stack alignment
1847+
| ADD_HYBRID_SPAD
18351848
| JMP_IP
18361849
} else {
18371850
if (GCC_GLOBAL_REGS) {
@@ -2413,7 +2426,7 @@ static int zend_jit_trace_halt_stub(dasm_State **Dst)
24132426
{
24142427
|->trace_halt:
24152428
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
2416-
| add r4, HYBRID_SPAD
2429+
| ADD_HYBRID_SPAD
24172430
| EXT_JMP zend_jit_halt_op->handler, r0
24182431
} else if (GCC_GLOBAL_REGS) {
24192432
| add r4, SPAD // stack alignment
@@ -2505,7 +2518,7 @@ static int zend_jit_trace_exit_stub(dasm_State **Dst)
25052518
| LOAD_IP
25062519

25072520
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
2508-
| add r4, HYBRID_SPAD
2521+
| ADD_HYBRID_SPAD
25092522
| JMP_IP
25102523
} else if (GCC_GLOBAL_REGS) {
25112524
| add r4, SPAD // stack alignment
@@ -2531,7 +2544,7 @@ static int zend_jit_trace_exit_stub(dasm_State **Dst)
25312544
| jne ->interrupt_handler
25322545

25332546
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
2534-
| add r4, HYBRID_SPAD
2547+
| ADD_HYBRID_SPAD
25352548
| mov r0, EX->func
25362549
| mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])]
25372550
| mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)]
@@ -2566,7 +2579,7 @@ static int zend_jit_trace_escape_stub(dasm_State **Dst)
25662579
|->trace_escape:
25672580
|
25682581
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
2569-
| add r4, HYBRID_SPAD
2582+
| ADD_HYBRID_SPAD
25702583
| JMP_IP
25712584
} else if (GCC_GLOBAL_REGS) {
25722585
| add r4, SPAD // stack alignment
@@ -2608,7 +2621,7 @@ static int zend_jit_context_threaded_call_stub(dasm_State **Dst)
26082621
|->context_threaded_call:
26092622
| pop r0
26102623
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
2611-
| add r4, HYBRID_SPAD
2624+
| ADD_HYBRID_SPAD
26122625
| jmp aword [IP]
26132626
} else if (GCC_GLOBAL_REGS) {
26142627
| add r4, SPAD // stack alignment
@@ -2990,7 +3003,7 @@ static int zend_jit_align_func(dasm_State **Dst)
29903003
static int zend_jit_prologue(dasm_State **Dst)
29913004
{
29923005
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
2993-
| sub r4, HYBRID_SPAD
3006+
| SUB_HYBRID_SPAD
29943007
} else if (GCC_GLOBAL_REGS) {
29953008
| sub r4, SPAD // stack alignment
29963009
} else {
@@ -3344,10 +3357,13 @@ static int zend_jit_trace_link_to_root(dasm_State **Dst, zend_jit_trace_info *t,
33443357
/* Skip prologue. */
33453358
// TODO: don't hardcode this ???
33463359
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
3360+
#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
3361+
prologue_size = 0;
3362+
#elif defined(__x86_64__) || defined(_M_X64)
33473363
// sub r4, HYBRID_SPAD
3348-
#if defined(__x86_64__) || defined(_M_X64)
33493364
prologue_size = 4;
33503365
#else
3366+
// sub r4, HYBRID_SPAD
33513367
prologue_size = 3;
33523368
#endif
33533369
} else if (GCC_GLOBAL_REGS) {
@@ -3387,7 +3403,7 @@ static int zend_jit_trace_return(dasm_State **Dst, zend_bool original_handler)
33873403
| jmp ->trace_escape
33883404
#else
33893405
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
3390-
| add r4, HYBRID_SPAD
3406+
| ADD_HYBRID_SPAD
33913407
if (!original_handler) {
33923408
| JMP_IP
33933409
} else {
@@ -3652,13 +3668,13 @@ static int zend_jit_tail_handler(dasm_State **Dst, const zend_op *opline)
36523668
/* Use inlined HYBRID VM handler */
36533669
const void *handler = opline->handler;
36543670

3655-
| add r4, HYBRID_SPAD
3671+
| ADD_HYBRID_SPAD
36563672
| EXT_JMP handler, r0
36573673
} else {
36583674
const void *handler = zend_get_opcode_handler_func(opline);
36593675

36603676
| EXT_CALL handler, r0
3661-
| add r4, HYBRID_SPAD
3677+
| ADD_HYBRID_SPAD
36623678
| JMP_IP
36633679
}
36643680
} else {
@@ -10044,7 +10060,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend
1004410060
}
1004510061
#else
1004610062
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
10047-
| add r4, HYBRID_SPAD
10063+
| ADD_HYBRID_SPAD
1004810064
| JMP_IP
1004910065
} else if (GCC_GLOBAL_REGS) {
1005010066
| add r4, SPAD // stack alignment
@@ -11283,7 +11299,11 @@ static int zend_jit_leave_func(dasm_State **Dst,
1128311299
trace_info->flags |= ZEND_JIT_TRACE_LOOP;
1128411300
| CMP_IP next_opline
1128511301
| je =>0 // LOOP
11302+
#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
11303+
| JMP_IP
11304+
#else
1128611305
| jmp ->trace_escape
11306+
#endif
1128711307
} else {
1128811308
| CMP_IP next_opline
1128911309
| jne ->trace_escape
@@ -11313,7 +11333,7 @@ static int zend_jit_leave_func(dasm_State **Dst,
1131311333
}
1131411334

1131511335
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
11316-
| add r4, HYBRID_SPAD
11336+
| ADD_HYBRID_SPAD
1131711337
#ifdef CONTEXT_THREADED_JIT
1131811338
| push aword [IP]
1131911339
| ret

0 commit comments

Comments
 (0)