Skip to content

Aarch64: emit prefix/postfix-indexed memory accesses #9661

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
+579 −8
Open

Aarch64: emit prefix/postfix-indexed memory accesses #9661

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
233 changes: 233 additions & 0 deletions hphp/runtime/vm/jit/vasm-arm.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,8 @@

#include "hphp/vixl/a64/macro-assembler-a64.h"

#include <limits>

TRACE_SET_MOD(vasm)

namespace HPHP::jit {
Expand Down Expand Up @@ -256,7 +258,27 @@ struct Vgen {
void emit(const ldimmw& i);
void emit(const ldundefq& /*i*/) {}
void emit(const load& i);
#define DECL_LOAD_UPDATE_SINGLE(name, pre, post, reg, size, ptr) \
void emit(const pre&); \
void emit(const post&);
VASM_LOAD_UPDATE_SINGLE_LIST(DECL_LOAD_UPDATE_SINGLE)
#undef DECL_LOAD_UPDATE_SINGLE
#define DECL_LOAD_UPDATE_PAIR(name, pre, post, reg, size, lanes, ptr) \
void emit(const pre&); \
void emit(const post&);
VASM_LOAD_UPDATE_PAIR_LIST(DECL_LOAD_UPDATE_PAIR)
#undef DECL_LOAD_UPDATE_PAIR
void emit(const store& i);
#define DECL_STORE_UPDATE_SINGLE(name, pre, post, reg, size, ptr) \
void emit(const pre&); \
void emit(const post&);
VASM_STORE_UPDATE_SINGLE_LIST(DECL_STORE_UPDATE_SINGLE)
#undef DECL_STORE_UPDATE_SINGLE
#define DECL_STORE_UPDATE_PAIR(name, pre, post, reg, size, lanes, ptr) \
void emit(const pre&); \
void emit(const post&);
VASM_STORE_UPDATE_PAIR_LIST(DECL_STORE_UPDATE_PAIR)
#undef DECL_STORE_UPDATE_PAIR
void emit(const mcprep& i);

// native function abi
Expand Down Expand Up @@ -458,6 +480,56 @@ struct Vgen {
void emit_nop() { a->Nop(); }

private:
static bool validSimpleUpdateOffset(int32_t offset) {
return offset >= -256 && offset <= 255;
}

static bool validPairUpdateOffset(int32_t offset, int laneSize) {
if (laneSize <= 0 || (offset % laneSize) != 0) return false;
auto const scaled = offset / laneSize;
return scaled >= -64 && scaled <= 63;
}

static int32_t checkedSimpleOffset(Immed imm) {
auto const value = imm.l();
always_assert_flog(
value >= std::numeric_limits<int32_t>::min() &&
value <= std::numeric_limits<int32_t>::max(),
"Immediate {} out of 32-bit range for simple update", value
);
auto const offset = static_cast<int32_t>(value);
always_assert_flog(
validSimpleUpdateOffset(offset),
"Immediate {} out of range for simple pre/post-index update", value
);
return offset;
}

static int32_t checkedPairOffset(Immed imm, int laneSize) {
auto const value = imm.l();
always_assert_flog(
value >= std::numeric_limits<int32_t>::min() &&
value <= std::numeric_limits<int32_t>::max(),
"Immediate {} out of 32-bit range for pair update", value
);
auto const offset = static_cast<int32_t>(value);
always_assert_flog(
validPairUpdateOffset(offset, laneSize),
"Immediate {} out of range for pair pre/post-index update with lane {}",
value, laneSize
);
return offset;
}

template<class EmitFn>
void emitMemUpdate(Vreg64 s, Vreg64 base, int32_t offset,
AddrMode mode, EmitFn emit_fn) {
auto const baseReg = X(s);
auto const mem = MemOperand(baseReg, offset, mode);
emit_fn(mem);
if (base != s) a->Mov(X(base), baseReg);
}

CodeBlock& frozen() { return env.text.frozen().code; }
static void recordAddressImmediate(Venv& env, TCA addr) {
env.meta.addressImmediates.insert(addr);
Expand Down Expand Up @@ -819,6 +891,96 @@ void Vgen::emit(const load& i) {
}
}

#define VASM_LOAD_UPDATE_SINGLE_BODY_load(mem, inst) \
do { \
if ((inst).d.isGP()) { \
a->Ldr(X((inst).d), (mem)); \
} else { \
a->Ldr(D((inst).d), (mem)); \
} \
} while (false)
#define VASM_LOAD_UPDATE_SINGLE_BODY_loadb(mem, inst) \
a->Ldrb(W((inst).d), (mem))
#define VASM_LOAD_UPDATE_SINGLE_BODY_loadw(mem, inst) \
a->Ldrh(W((inst).d), (mem))
#define VASM_LOAD_UPDATE_SINGLE_BODY_loadl(mem, inst) \
a->Ldr(W((inst).d), (mem))
#define VASM_LOAD_UPDATE_SINGLE_BODY_loadsd(mem, inst) \
a->Ldr(D((inst).d), (mem))
#define VASM_LOAD_UPDATE_SINGLE_BODY_loadzbl(mem, inst) \
a->Ldrb(W((inst).d), (mem))
#define VASM_LOAD_UPDATE_SINGLE_BODY_loadzbq(mem, inst) \
a->Ldrb(W((inst).d), (mem))
#define VASM_LOAD_UPDATE_SINGLE_BODY_loadzwq(mem, inst) \
a->Ldrh(W((inst).d), (mem))
#define VASM_LOAD_UPDATE_SINGLE_BODY_loadzlq(mem, inst) \
a->Ldr(W((inst).d), (mem))
#define VASM_LOAD_UPDATE_SINGLE_BODY_loadsbl(mem, inst) \
a->Ldrsb(W((inst).d), (mem))
#define VASM_LOAD_UPDATE_SINGLE_BODY_loadsbq(mem, inst) \
a->Ldrsb(X((inst).d), (mem))
#define VASM_LOAD_UPDATE_SINGLE_BODY_loadtqb(mem, inst) \
a->Ldrb(W((inst).d), (mem))
#define VASM_LOAD_UPDATE_SINGLE_BODY_loadtql(mem, inst) \
a->Ldr(W((inst).d), (mem))

#define VASM_LOAD_UPDATE_PAIR_BODY_loadpair(mem, inst) \
a->Ldp(X((inst).d0), X((inst).d1), (mem))
#define VASM_LOAD_UPDATE_PAIR_BODY_loadpairl(mem, inst) \
a->Ldp(W((inst).d0), W((inst).d1), (mem))

#define DEFINE_LOAD_UPDATE_SINGLE(name, pre, post, reg, size, ptr) \
void Vgen::emit(const pre& i) { \
auto const offset = checkedSimpleOffset(i.s0); \
emitMemUpdate(i.s, i.base, offset, PreIndex, \
[&](const MemOperand& mem) { \
VASM_LOAD_UPDATE_SINGLE_BODY_##name(mem, i); \
}); \
} \
void Vgen::emit(const post& i) { \
auto const offset = checkedSimpleOffset(i.s0); \
emitMemUpdate(i.s, i.base, offset, PostIndex, \
[&](const MemOperand& mem) { \
VASM_LOAD_UPDATE_SINGLE_BODY_##name(mem, i); \
}); \
}
VASM_LOAD_UPDATE_SINGLE_LIST(DEFINE_LOAD_UPDATE_SINGLE)
#undef DEFINE_LOAD_UPDATE_SINGLE

#define DEFINE_LOAD_UPDATE_PAIR(name, pre, post, reg, laneSize, lanes, ptr) \
void Vgen::emit(const pre& i) { \
auto const offset = checkedPairOffset(i.s0, laneSize); \
emitMemUpdate(i.s, i.base, offset, PreIndex, \
[&](const MemOperand& mem) { \
VASM_LOAD_UPDATE_PAIR_BODY_##name(mem, i); \
}); \
} \
void Vgen::emit(const post& i) { \
auto const offset = checkedPairOffset(i.s0, laneSize); \
emitMemUpdate(i.s, i.base, offset, PostIndex, \
[&](const MemOperand& mem) { \
VASM_LOAD_UPDATE_PAIR_BODY_##name(mem, i); \
}); \
}
VASM_LOAD_UPDATE_PAIR_LIST(DEFINE_LOAD_UPDATE_PAIR)
#undef DEFINE_LOAD_UPDATE_PAIR

#undef VASM_LOAD_UPDATE_SINGLE_BODY_load
#undef VASM_LOAD_UPDATE_SINGLE_BODY_loadb
#undef VASM_LOAD_UPDATE_SINGLE_BODY_loadw
#undef VASM_LOAD_UPDATE_SINGLE_BODY_loadl
#undef VASM_LOAD_UPDATE_SINGLE_BODY_loadsd
#undef VASM_LOAD_UPDATE_SINGLE_BODY_loadzbl
#undef VASM_LOAD_UPDATE_SINGLE_BODY_loadzbq
#undef VASM_LOAD_UPDATE_SINGLE_BODY_loadzwq
#undef VASM_LOAD_UPDATE_SINGLE_BODY_loadzlq
#undef VASM_LOAD_UPDATE_SINGLE_BODY_loadsbl
#undef VASM_LOAD_UPDATE_SINGLE_BODY_loadsbq
#undef VASM_LOAD_UPDATE_SINGLE_BODY_loadtqb
#undef VASM_LOAD_UPDATE_SINGLE_BODY_loadtql
#undef VASM_LOAD_UPDATE_PAIR_BODY_loadpair
#undef VASM_LOAD_UPDATE_PAIR_BODY_loadpairl

void Vgen::emit(const store& i) {
if (i.s.isGP()) {
if (i.s == arm::rsp()) {
Expand All @@ -832,6 +994,77 @@ void Vgen::emit(const store& i) {
}
}

#define VASM_STORE_UPDATE_SINGLE_BODY_store(mem, inst) \
do { \
if ((inst).v.isGP()) { \
if ((inst).v == arm::rsp()) { \
a->Mov(rAsm, X((inst).v)); \
a->Str(rAsm, (mem)); \
} else { \
a->Str(X((inst).v), (mem)); \
} \
} else { \
a->Str(D((inst).v), (mem)); \
} \
} while (false)
#define VASM_STORE_UPDATE_SINGLE_BODY_storeb(mem, inst) \
a->Strb(W((inst).v), (mem))
#define VASM_STORE_UPDATE_SINGLE_BODY_storew(mem, inst) \
a->Strh(W((inst).v), (mem))
#define VASM_STORE_UPDATE_SINGLE_BODY_storel(mem, inst) \
a->Str(W((inst).v), (mem))
#define VASM_STORE_UPDATE_SINGLE_BODY_storesd(mem, inst) \
a->Str(D((inst).v), (mem))

#define VASM_STORE_UPDATE_PAIR_BODY_storepair(mem, inst) \
a->Stp(X((inst).v0), X((inst).v1), (mem))
#define VASM_STORE_UPDATE_PAIR_BODY_storepairl(mem, inst) \
a->Stp(W((inst).v0), W((inst).v1), (mem))

#define DEFINE_STORE_UPDATE_SINGLE(name, pre, post, reg, size, ptr) \
void Vgen::emit(const pre& i) { \
auto const offset = checkedSimpleOffset(i.s0); \
emitMemUpdate(i.s, i.base, offset, PreIndex, \
[&](const MemOperand& mem) { \
VASM_STORE_UPDATE_SINGLE_BODY_##name(mem, i); \
}); \
} \
void Vgen::emit(const post& i) { \
auto const offset = checkedSimpleOffset(i.s0); \
emitMemUpdate(i.s, i.base, offset, PostIndex, \
[&](const MemOperand& mem) { \
VASM_STORE_UPDATE_SINGLE_BODY_##name(mem, i); \
}); \
}
VASM_STORE_UPDATE_SINGLE_LIST(DEFINE_STORE_UPDATE_SINGLE)
#undef DEFINE_STORE_UPDATE_SINGLE

#define DEFINE_STORE_UPDATE_PAIR(name, pre, post, reg, laneSize, lanes, ptr) \
void Vgen::emit(const pre& i) { \
auto const offset = checkedPairOffset(i.s0, laneSize); \
emitMemUpdate(i.s, i.base, offset, PreIndex, \
[&](const MemOperand& mem) { \
VASM_STORE_UPDATE_PAIR_BODY_##name(mem, i); \
}); \
} \
void Vgen::emit(const post& i) { \
auto const offset = checkedPairOffset(i.s0, laneSize); \
emitMemUpdate(i.s, i.base, offset, PostIndex, \
[&](const MemOperand& mem) { \
VASM_STORE_UPDATE_PAIR_BODY_##name(mem, i); \
}); \
}
VASM_STORE_UPDATE_PAIR_LIST(DEFINE_STORE_UPDATE_PAIR)
#undef DEFINE_STORE_UPDATE_PAIR

#undef VASM_STORE_UPDATE_SINGLE_BODY_store
#undef VASM_STORE_UPDATE_SINGLE_BODY_storeb
#undef VASM_STORE_UPDATE_SINGLE_BODY_storew
#undef VASM_STORE_UPDATE_SINGLE_BODY_storel
#undef VASM_STORE_UPDATE_SINGLE_BODY_storesd
#undef VASM_STORE_UPDATE_PAIR_BODY_storepair
#undef VASM_STORE_UPDATE_PAIR_BODY_storepairl

///////////////////////////////////////////////////////////////////////////////

void Vgen::emit(const mcprep& i) {
Expand Down
85 changes: 85 additions & 0 deletions hphp/runtime/vm/jit/vasm-instr.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,67 @@ struct Vunit;
* DH(d,h) define d, try assigning same register as h
* Un,Dn no uses, defs
*/

#if defined(__aarch64__)
#define VASM_STORE_UPDATE_SINGLE_LIST(_) \
_(store, storepri, storepi, Vreg, 8, d) \
_(storeb, storebpri, storebpi, Vreg8, 1, m) \
_(storew, storewpri, storewpi, Vreg16, 2, m) \
_(storel, storelpri, storelpi, Vreg32, 4, m) \
_(storesd, storesdpri, storesdpi, VregDbl, 8, m)

#define VASM_STORE_UPDATE_PAIR_LIST(_) \
_(storepair, storepairpri, storepairpi, Vreg64, 8, 2, d) \
_(storepairl, storepairlpri, storepairlpi, Vreg32, 4, 2, d)

#define VASM_LOAD_UPDATE_SINGLE_LIST(_) \
_(load, loadpri, loadpi, Vreg, 8, s) \
_(loadb, loadbpri, loadbpi, Vreg8, 1, s) \
_(loadw, loadwpri, loadwpi, Vreg16, 2, s) \
_(loadl, loadlpri, loadlpi, Vreg32, 4, s) \
_(loadsd, loadsdpri, loadsdpi, VregDbl, 8, s) \
_(loadzbl, loadzblpri, loadzblpi, Vreg32, 1, s) \
_(loadzbq, loadzbqpri, loadzbqpi, Vreg64, 1, s) \
_(loadzwq, loadzwqpri, loadzwqpi, Vreg64, 2, s) \
_(loadzlq, loadzlpri, loadzlpi, Vreg64, 4, s) \
_(loadsbl, loadsblpri, loadsblpi, Vreg32, 1, s) \
_(loadsbq, loadsbqpri, loadsbqpi, Vreg64, 1, s) \
_(loadtqb, loadtqbpri, loadtqbpi, Vreg8, 1, s) \
_(loadtql, loadtqlpri, loadtqlpi, Vreg32, 4, s)

#define VASM_LOAD_UPDATE_PAIR_LIST(_) \
_(loadpair, loadpairpri, loadpairpi, Vreg64, 8, 2, s) \
_(loadpairl, loadpairlpri, loadpairlpi, Vreg32, 4, 2, s)
#else
#define VASM_STORE_UPDATE_SINGLE_LIST(_)
#define VASM_STORE_UPDATE_PAIR_LIST(_)
#define VASM_LOAD_UPDATE_SINGLE_LIST(_)
#define VASM_LOAD_UPDATE_PAIR_LIST(_)
#endif

#if defined(__aarch64__)
#define VASM_LOAD_UPDATE_SINGLE_OP_ENTRY(name, pre, post, reg, size, ptr) \
O(pre, I(s0), U(s), DH(base,s) D(d))\
O(post, I(s0), U(s), DH(base,s) D(d))

#define VASM_LOAD_UPDATE_PAIR_OP_ENTRY(name, pre, post, reg, size, lanes, ptr) \
O(pre, I(s0), U(s), DH(base,s) D(d0) D(d1))\
O(post, I(s0), U(s), DH(base,s) D(d0) D(d1))

#define VASM_STORE_UPDATE_SINGLE_OP_ENTRY(name, pre, post, reg, size, ptr) \
O(pre, I(s0), U(s) U(v), DH(base,s))\
O(post, I(s0), U(s) U(v), DH(base,s))

#define VASM_STORE_UPDATE_PAIR_OP_ENTRY(name, pre, post, reg, size, lanes, ptr) \
O(pre, I(s0), U(s) U(v0) U(v1), DH(base,s))\
O(post, I(s0), U(s) U(v0) U(v1), DH(base,s))
#else
#define VASM_LOAD_UPDATE_SINGLE_OP_ENTRY(name, pre, post, reg, size, ptr)
#define VASM_LOAD_UPDATE_PAIR_OP_ENTRY(name, pre, post, reg, size, lanes, ptr)
#define VASM_STORE_UPDATE_SINGLE_OP_ENTRY(name, pre, post, reg, size, ptr)
#define VASM_STORE_UPDATE_PAIR_OP_ENTRY(name, pre, post, reg, size, lanes, ptr)
#endif

#define VASM_OPCODES\
/* service requests */\
O(bindjmp, I(target) I(spOff), U(args), Dn)\
Expand All @@ -80,7 +141,11 @@ struct Vunit;
O(ldimmq, I(s), Un, D(d))\
O(ldundefq, Inone, Un, D(d))\
O(load, Inone, U(s), D(d))\
VASM_LOAD_UPDATE_SINGLE_LIST(VASM_LOAD_UPDATE_SINGLE_OP_ENTRY)\
VASM_LOAD_UPDATE_PAIR_LIST(VASM_LOAD_UPDATE_PAIR_OP_ENTRY)\
O(store, Inone, U(s) UW(d), Dn)\
VASM_STORE_UPDATE_SINGLE_LIST(VASM_STORE_UPDATE_SINGLE_OP_ENTRY)\
VASM_STORE_UPDATE_PAIR_LIST(VASM_STORE_UPDATE_PAIR_OP_ENTRY)\
O(mcprep, Inone, Un, D(d))\
O(phidef, Inone, Un, D(defs))\
O(phijmp, Inone, U(uses), Dn)\
Expand Down Expand Up @@ -542,7 +607,27 @@ struct ldundefq { Vreg d; };
* Memory operand load and store.
*/
struct load { Vptr64 s; Vreg d; };
#define VASM_LOAD_UPDATE_SINGLE_STRUCT(name, pre, post, reg, size, ptr) \
struct pre { Immed s0; Vreg64 s, base; reg d; }; \
struct post { Immed s0; Vreg64 s, base; reg d; };
VASM_LOAD_UPDATE_SINGLE_LIST(VASM_LOAD_UPDATE_SINGLE_STRUCT)
#undef VASM_LOAD_UPDATE_SINGLE_STRUCT
#define VASM_LOAD_UPDATE_PAIR_STRUCT(name, pre, post, reg, size, lanes, ptr) \
struct pre { Immed s0; Vreg64 s, base; reg d0, d1; }; \
struct post { Immed s0; Vreg64 s, base; reg d0, d1; };
VASM_LOAD_UPDATE_PAIR_LIST(VASM_LOAD_UPDATE_PAIR_STRUCT)
#undef VASM_LOAD_UPDATE_PAIR_STRUCT
struct store { Vreg s; Vptr64 d; };
#define VASM_STORE_UPDATE_SINGLE_STRUCT(name, pre, post, reg, size, ptr) \
struct pre { Immed s0; Vreg64 s, base; reg v; }; \
struct post { Immed s0; Vreg64 s, base; reg v; };
VASM_STORE_UPDATE_SINGLE_LIST(VASM_STORE_UPDATE_SINGLE_STRUCT)
#undef VASM_STORE_UPDATE_SINGLE_STRUCT
#define VASM_STORE_UPDATE_PAIR_STRUCT(name, pre, post, reg, size, lanes, ptr) \
struct pre { Immed s0; Vreg64 s, base; reg v0, v1; }; \
struct post { Immed s0; Vreg64 s, base; reg v0, v1; };
VASM_STORE_UPDATE_PAIR_LIST(VASM_STORE_UPDATE_PAIR_STRUCT)
#undef VASM_STORE_UPDATE_PAIR_STRUCT

/*
* Method cache smashable prime data.
Expand Down
Loading