flamenco, runtime, vm: clean up bounds on the number of instruction accounts

contrib/test/test-vectors-commit-sha.txt

@@ -1 +1 @@
-ebd84b9c0b327d78fefdc52236c3d91c3df2b946
+e3d072c318b9339379cac22d3278ca137220bfe4

src/flamenco/runtime/fd_runtime_const.h

@@ -105,7 +105,7 @@ static const fd_cluster_version_t FD_RUNTIME_CLUSTER_VERSION = {
 /* The bpf loader's serialization footprint is bounded in the worst case
    by 64 unique writable accounts which are each 10MiB in size (bounded
    by the amount of transaction accounts).  We can also have up to
-   FD_INSTR_ACCT_MAX (256) referenced accounts in an instruction.
+   FD_BPF_INSTR_ACCT_MAX (255) referenced accounts in an instruction.
 
    - 8 bytes for the account count
    For each account:
@@ -147,8 +147,63 @@ static const fd_cluster_version_t FD_RUNTIME_CLUSTER_VERSION = {
 #define FD_ACCOUNT_REC_ALIGN        (8UL)
 /* https://github.com/anza-xyz/sbpf/blob/v0.12.2/src/ebpf.rs#L37-L38 */
 #define FD_RUNTIME_EBPF_HOST_ALIGN  (16UL)
-#define FD_INSTR_ACCT_MAX            (256)
 
+/* FD_INSTR_ACCT_MAX is the maximum number of accounts that can
+   be referenced by a single instruction.
+
+   This is different from FD_BPF_INSTR_ACCT_MAX, which is enforced by the
+   BPF serializer. It is possible to pass in more than FD_BPF_INSTR_ACCT_MAX
+   instruction accounts in a transaction (for example mainnet transaction)
+   3eDdfZE6HswPxFKrtnQPsEmTkyL1iP57gRPEXwaqNGAqF1paGXCYYMwh7z4uQDUMgFor742sikVSQZW1gFRDhPNh).
+
+   A transaction like this will be loaded and sanitized, but will fail in the
+   bpf serialization stage. It is also possible to invoke a native program with
+   more than FD_BPF_INSTR_ACCT_MAX instruction accounts that will execute successfully.
+
+   Therefore we need to derive a bound from a worst-case transaction: one that
+   has the maximum possible number of instruction accounts at the expense of
+   everything else. This is a legacy transaction with a single account address,
+   a single signature, a single instruction with empty data and as many
+   instruction accounts as possible.
+
+   Therefore, the maximum number of instruction accounts is:
+     (MTU - fixed overhead) / (size of instruction account)
+   = (MTU
+       - signature count (1 byte, value=1)
+       - signature (64 bytes)
+       - signature count in header (1 byte)
+       - readonly signed count (1 byte)
+       - readonly unsigned count (1 byte)
+       - account count (1 byte, compact-u16 value=1)
+       - 1 account address (32 bytes)
+       - recent blockhash (32 bytes)
+       - instruction count (1 byte, compact-u16 value=1)
+       - program id index (1 byte)
+       - instruction account count (2 bytes)
+       - data len (1 byte, value=0)
+   = 1232 - 1 - 64 - 1 - 1 - 1 - 1 - 32 - 32 - 1 - 1 - 2 - 1
+   = 1094
+
+   TODO: SIMD-406 (https://github.com/solana-foundation/solana-improvement-documents/pull/406)
+   limits the number of instruction accounts to 255 in transaction sanitization.
+
+   Once the corresponding feature gate has been activated, we can reduce
+   FD_INSTR_ACCT_MAX to 255. We cannot reduce this before as this would cause
+   the result of the get_processed_sibling_instruction syscall to diverge from
+   Agave. */
+#define FD_INSTR_ACCT_MAX           (1094UL)
+
+/* FD_BPF_INSTR_ACCT_MAX is the maximum number of accounts that
+   an instruction that goes through the bpf loader serializer can reference.
+
+   The BPF loader has a lower limit for the number of instruction accounts
+   than is enforced in transaction sanitization.
+
+   TODO: remove this limit once SIMD-406 is activated, as we can then use the
+   same limit everywhere.
+
+   https://github.com/anza-xyz/agave/blob/v3.1.4/transaction-context/src/lib.rs#L30-L32 */
+#define FD_BPF_INSTR_ACCT_MAX       (255UL)
 
 #define FD_BPF_LOADER_UNIQUE_ACCOUNT_FOOTPRINT(direct_mapping)                                                                                              \
                                               (1UL                         /* dup byte          */                                                        + \
@@ -168,7 +223,7 @@ static const fd_cluster_version_t FD_RUNTIME_CLUSTER_VERSION = {
 #define FD_BPF_LOADER_INPUT_REGION_FOOTPRINT(account_lock_limit, direct_mapping)                                                                      \
                                               (FD_ULONG_ALIGN_UP( (sizeof(ulong)         /* acct_cnt       */                                       + \
                                                                    account_lock_limit*FD_BPF_LOADER_UNIQUE_ACCOUNT_FOOTPRINT(direct_mapping)        + \
-                                                                   (FD_INSTR_ACCT_MAX-account_lock_limit)*FD_BPF_LOADER_DUPLICATE_ACCOUNT_FOOTPRINT + \
+                                                                   (FD_BPF_INSTR_ACCT_MAX-account_lock_limit)*FD_BPF_LOADER_DUPLICATE_ACCOUNT_FOOTPRINT + \
                                                                    sizeof(ulong)         /* instr data len */                                       + \
                                                                    FD_TXN_MTU            /* No instr data  */                                       + \
                                                                    sizeof(fd_pubkey_t)), /* program id     */                                          \

src/flamenco/runtime/info/fd_instr_info.c

@@ -33,18 +33,14 @@ fd_instr_info_init_from_txn_instr( fd_instr_info_t *      instr,
 
   /* Set the program id */
   instr->program_id = txn_instr->program_id;
-
-  /* See note in fd_instr_info.h.  TLDR: capping this value at 256
-     should have literally 0 effect on program execution, down to the
-     error codes.  This is purely for the sake of not increasing the
-     overall memory footprint of the transaction context.  If this
-     change causes issues, we may need to increase the array sizes in
-     the instr info. */
-  instr->acct_cnt = fd_ushort_min( txn_instr->acct_cnt, FD_INSTR_ACCT_MAX );
+  instr->acct_cnt   = txn_instr->acct_cnt;
+  if( FD_UNLIKELY( instr->acct_cnt > FD_INSTR_ACCT_MAX ) ) {
+    FD_LOG_CRIT(( "invariant violation: Instruction has too many accounts: %d > %lu", instr->acct_cnt, FD_INSTR_ACCT_MAX ));
+  }
   instr->data_sz  = txn_instr->data_sz;
   memcpy( instr->data, txn_in->txn->payload+txn_instr->data_off, instr->data_sz );
 
-  uchar acc_idx_seen[ FD_INSTR_ACCT_MAX ] = {0};
+  uchar acc_idx_seen[ FD_TXN_ACCT_ADDR_MAX ] = {0};
 
   for( ushort i=0; i<instr->acct_cnt; i++ ) {
     ushort acc_idx = instr_acc_idxs[i];

src/flamenco/runtime/info/fd_instr_info.h

@@ -6,21 +6,6 @@
 #include "../fd_runtime_const.h"
 #include "../../../ballet/txn/fd_txn.h"
 
-/* While the maximum number of instruction accounts allowed for instruction
-   execution is 256, it is entirely possible to have a transaction with more
-   than 256 instruction accounts that passes transaction loading checks and enters
-   `fd_execute_instr` (See mainnet transaction
-   3eDdfZE6HswPxFKrtnQPsEmTkyL1iP57gRPEXwaqNGAqF1paGXCYYMwh7z4uQDUMgFor742sikVSQZW1gFRDhPNh
-   for an example). An instruction that goes into the VM with more than 256 instruction accounts
-   will fail, but you could also theoretically invoke a native program with over 256 random
-   unreferenced instruction accounts that will execute successfully. The true bound for the
-   maximum number of instruction accounts you can pass in is slighly lower than the maximum
-   possible size for a serialized transaction (1232).
-
-   HOWEVER... to keep our memory footprint low, we cap the `acct_cnt` at 256 during setup since
-   any extra accounts should (ideally) have literally 0 impact on program execution, whether
-   or not they are present in the instr info. This keeps the transaction context size from
-   blowing up to around 3MB in size. */
 #define FD_INSTR_ACCT_FLAGS_IS_SIGNER   (0x01U)
 #define FD_INSTR_ACCT_FLAGS_IS_WRITABLE (0x02U)
 
@@ -79,7 +64,7 @@ fd_instruction_account_init( ushort idx_in_txn,
 
 static inline void
 fd_instr_info_setup_instr_account( fd_instr_info_t * instr,
-                                   uchar             acc_idx_seen[ FD_INSTR_ACCT_MAX ],
+                                   uchar             acc_idx_seen[ FD_TXN_ACCT_ADDR_MAX ],
                                    ushort            idx_in_txn,
                                    ushort            idx_in_caller,
                                    ushort            idx_in_callee,

src/flamenco/runtime/program/fd_bpf_loader_serialization.c

@@ -266,8 +266,10 @@ fd_bpf_loader_input_serialize_aligned( fd_exec_instr_ctx_t *     ctx,
                                        ulong *                   instr_data_offset ) {
   fd_pubkey_t * txn_accs = ctx->txn_out->accounts.account_keys;
 
-  uchar  acc_idx_seen[ FD_INSTR_ACCT_MAX ] = {0};
-  ushort dup_acc_idx[ FD_INSTR_ACCT_MAX ]  = {0};
+  /* Transaction sanitisation limits the number of instruction accounts to
+     FD_TXN_ACCT_ADDR_MAX. */
+  uchar  acc_idx_seen[ FD_TXN_ACCT_ADDR_MAX ] = {0};
+  ushort dup_acc_idx[ FD_TXN_ACCT_ADDR_MAX ]  = {0};
 
   /* 16-byte aligned buffer:
      https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L60
@@ -550,8 +552,10 @@ fd_bpf_loader_input_serialize_unaligned( fd_exec_instr_ctx_t *     ctx,
                                          ulong *                   instr_data_offset ) {
   fd_pubkey_t const * txn_accs = ctx->txn_out->accounts.account_keys;
 
-  uchar  acc_idx_seen[FD_INSTR_ACCT_MAX] = {0};
-  ushort dup_acc_idx[FD_INSTR_ACCT_MAX]  = {0};
+  /* Transaction sanitisation limits the number of instruction accounts to
+     FD_TXN_ACCT_ADDR_MAX. */
+  uchar  acc_idx_seen[ FD_TXN_ACCT_ADDR_MAX ] = {0};
+  ushort dup_acc_idx[ FD_TXN_ACCT_ADDR_MAX ]  = {0};
 
   /* 16-byte aligned buffer:
      https://github.com/anza-xyz/agave/blob/v2.2.13/programs/bpf_loader/src/serialization.rs#L32
@@ -761,7 +765,7 @@ fd_bpf_loader_input_serialize_parameters( fd_exec_instr_ctx_t *     instr_ctx,
 
   /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L234-L237 */
   ulong num_ix_accounts = instr_ctx->instr->acct_cnt;
-  if( FD_UNLIKELY( num_ix_accounts>=FD_INSTR_ACCT_MAX ) ) {
+  if( FD_UNLIKELY( num_ix_accounts>FD_BPF_INSTR_ACCT_MAX ) ) {
     return FD_EXECUTOR_INSTR_ERR_MAX_ACCS_EXCEEDED;
   }
 

src/flamenco/runtime/program/fd_native_cpi.c

@@ -15,7 +15,7 @@ fd_native_cpi_native_invoke( fd_exec_instr_ctx_t *             ctx,
                              ulong                             signers_cnt ) {
   /* Set up the instr info */
   fd_instr_info_t *        instr_info = &ctx->runtime->instr.trace[ ctx->runtime->instr.trace_length++ ];
-  fd_instruction_account_t instruction_accounts[ FD_INSTR_ACCT_MAX ];
+  fd_instruction_account_t instruction_accounts[ FD_VM_CPI_MAX_INSTRUCTION_ACCOUNTS ];
   ulong                    instruction_accounts_cnt;
 
   /* Set the stack size */
@@ -28,9 +28,8 @@ fd_native_cpi_native_invoke( fd_exec_instr_ctx_t *             ctx,
     instr_info->program_id = (uchar)program_id;
   }
 
-  fd_pubkey_t instr_acct_keys[ FD_INSTR_ACCT_MAX ];
-  uchar       acc_idx_seen[ FD_INSTR_ACCT_MAX ];
-  memset( acc_idx_seen, 0, FD_INSTR_ACCT_MAX );
+  fd_pubkey_t instr_acct_keys[ FD_VM_CPI_MAX_INSTRUCTION_ACCOUNTS ];
+  uchar       acc_idx_seen[ FD_TXN_ACCT_ADDR_MAX ] = {0};
 
   instr_info->acct_cnt = (ushort)acct_metas_len;
   for( ushort j=0U; j<acct_metas_len; j++ ) {

src/flamenco/runtime/tests/fd_instr_harness.c

@@ -304,7 +304,7 @@ fd_solfuzz_pb_instr_ctx_create( fd_solfuzz_runner_t *                runner,
   ctx->sysvar_cache = fd_bank_sysvar_cache_modify( runner->bank );
   ctx->runtime = runtime;
 
-  uchar acc_idx_seen[ FD_INSTR_ACCT_MAX ] = {0};
+  uchar acc_idx_seen[ FD_TXN_ACCT_ADDR_MAX ] = {0};
   for( ulong j=0UL; j < test_ctx->instr_accounts_count; j++ ) {
     uint index = test_ctx->instr_accounts[j].index;
     if( index >= test_ctx->accounts_count ) {

src/flamenco/vm/fd_vm_base.h

@@ -247,6 +247,17 @@ FD_PROTOTYPES_END
 
 #define FD_VM_MAX_CPI_INSTRUCTION_SIZE                  (            1280UL) /* IPv6 Min MTU size */
 
+/* FD_VM_CPI_MAX_INSTRUCTION_ACCOUNTS is the maximum number of accounts
+   that can be referenced by a single CPI instruction.
+
+   Agave's bound for this is the same as their bound for the bound
+   enforced by the bpf loader serializer.
+   https://github.com/anza-xyz/agave/blob/v3.1.1/transaction-context/src/lib.rs#L32
+
+   TODO: when SIMD-406 is activated, we can use FD_INSTR_ACCT_MAX instead. */
+
+#define FD_VM_CPI_MAX_INSTRUCTION_ACCOUNTS           (FD_BPF_INSTR_ACCT_MAX)
+
 /* FD_VM_CPI_BYTES_PER_UNIT is the number of account data bytes per
    compute unit charged during a cross-program invocation */
 

src/flamenco/vm/syscall/fd_vm_syscall.h

@@ -775,8 +775,8 @@ int
 fd_vm_prepare_instruction( fd_instr_info_t *        callee_instr,
                            fd_exec_instr_ctx_t *    instr_ctx,
                            fd_pubkey_t const *      callee_program_id_pubkey,
-                           fd_pubkey_t const        instr_acct_keys[ FD_INSTR_ACCT_MAX ],
-                           fd_instruction_account_t instruction_accounts[ FD_INSTR_ACCT_MAX ],
+                           fd_pubkey_t const        instr_acct_keys[ FD_VM_CPI_MAX_INSTRUCTION_ACCOUNTS ],
+                           fd_instruction_account_t instruction_accounts[ FD_VM_CPI_MAX_INSTRUCTION_ACCOUNTS ],
                            ulong *                  instruction_accounts_cnt,
                            fd_pubkey_t const *      signers,
                            ulong                    signers_cnt );

src/flamenco/vm/syscall/fd_vm_syscall_cpi.c

@@ -58,17 +58,25 @@ int
 fd_vm_prepare_instruction( fd_instr_info_t *        callee_instr,
                            fd_exec_instr_ctx_t *    instr_ctx,
                            fd_pubkey_t const *      callee_program_id_pubkey,
-                           fd_pubkey_t const        instr_acct_keys[ FD_INSTR_ACCT_MAX ],
-                           fd_instruction_account_t instruction_accounts[ FD_INSTR_ACCT_MAX ],
+                           fd_pubkey_t const        instr_acct_keys[ FD_VM_CPI_MAX_INSTRUCTION_ACCOUNTS ],
+                           fd_instruction_account_t instruction_accounts[ FD_VM_CPI_MAX_INSTRUCTION_ACCOUNTS ],
                            ulong *                  instruction_accounts_cnt,
                            fd_pubkey_t const *      signers,
                            ulong                    signers_cnt ) {
 
   /* De-duplicate the instruction accounts, using the same logic as Solana */
   ulong deduplicated_instruction_accounts_cnt = 0;
-  fd_instruction_account_t deduplicated_instruction_accounts[256] = {0};
-  ulong duplicate_indicies_cnt = 0;
-  ulong duplicate_indices[256] = {0};
+  fd_instruction_account_t deduplicated_instruction_accounts[ FD_VM_CPI_MAX_INSTRUCTION_ACCOUNTS ] = {0};
+  ulong duplicate_indicies_cnt                                                                     = 0;
+  ulong duplicate_indices[ FD_VM_CPI_MAX_INSTRUCTION_ACCOUNTS ]                                    = {0};
+
+  /* This function is either called by a true CPI or by a native cpi invocation.
+     The native CPI invocation is never called with more than 3 instruction
+     accounts, and the true CPI is never called with more than
+     FD_VM_CPI_MAX_INSTRUCTION_ACCOUNTS. */
+  if( FD_UNLIKELY( callee_instr->acct_cnt > FD_VM_CPI_MAX_INSTRUCTION_ACCOUNTS ) ) {
+    FD_LOG_CRIT(( "invariant violation: too many accounts %u", callee_instr->acct_cnt ));
+  }
 
   /* Normalize the privileges of each instruction account in the callee, after de-duping
      the account references.
@@ -237,15 +245,6 @@ get_cpi_max_account_infos( fd_bank_t * bank ) {
   return fd_ulong_if( FD_FEATURE_ACTIVE_BANK( bank, increase_tx_account_lock_limit ), FD_CPI_MAX_ACCOUNT_INFOS, 64UL );
 }
 
-/* Maximum CPI instruction accounts. 255 was chosen to ensure that instruction
-   accounts are always within the maximum instruction account limit for BPF
-   program instructions.
-
-   https://github.com/solana-labs/solana/blob/dbf06e258ae418097049e845035d7d5502fe1327/sdk/program/src/syscalls/mod.rs#L19
-   https://github.com/solana-labs/solana/blob/dbf06e258ae418097049e845035d7d5502fe1327/programs/bpf_loader/src/serialization.rs#L26 */
-
-#define FD_CPI_MAX_INSTRUCTION_ACCOUNTS    (255UL)
-
 /* fd_vm_syscall_cpi_check_instruction contains common instruction acct
    count and data sz checks.  Also consumes compute units proportional
    to instruction data size. */
@@ -256,7 +255,7 @@ fd_vm_syscall_cpi_check_instruction( fd_vm_t const * vm,
                                      ulong           data_sz ) {
   /* https://github.com/anza-xyz/agave/blob/v3.1.2/program-runtime/src/cpi.rs#L146-L161 */
   if( FD_FEATURE_ACTIVE_BANK( vm->instr_ctx->bank, loosen_cpi_size_restriction ) ) {
-    if( FD_UNLIKELY( acct_cnt > FD_CPI_MAX_INSTRUCTION_ACCOUNTS ) ) {
+    if( FD_UNLIKELY( acct_cnt > FD_VM_CPI_MAX_INSTRUCTION_ACCOUNTS ) ) {
       // SyscallError::MaxInstructionAccountsExceeded
       return FD_VM_SYSCALL_ERR_MAX_INSTRUCTION_ACCOUNTS_EXCEEDED;
     }

src/flamenco/vm/syscall/fd_vm_syscall_cpi_common.c

@@ -59,7 +59,7 @@ VM_SYSCALL_CPI_INSTRUCTION_TO_INSTR_FUNC( fd_vm_t *                         vm,
                                           fd_pubkey_t const *               program_id,
                                           uchar const *                     cpi_instr_data,
                                           fd_instr_info_t *                 out_instr,
-                                          fd_pubkey_t                       out_instr_acct_keys[ FD_INSTR_ACCT_MAX ] ) {
+                                          fd_pubkey_t                       out_instr_acct_keys[ FD_VM_CPI_MAX_INSTRUCTION_ACCOUNTS ] ) {
 
   out_instr->program_id   = UCHAR_MAX;
   out_instr->stack_height = vm->instr_ctx->runtime->instr.stack_sz+1;
@@ -73,7 +73,7 @@ VM_SYSCALL_CPI_INSTRUCTION_TO_INSTR_FUNC( fd_vm_t *                         vm,
     out_instr->program_id = (uchar)program_id_idx;
   }
 
-  uchar acc_idx_seen[ FD_INSTR_ACCT_MAX ] = {0};
+  uchar acc_idx_seen[ FD_TXN_ACCT_ADDR_MAX ] = {0};
 
   for( ushort i=0; i<VM_SYSCALL_CPI_INSTR_ACCS_LEN( cpi_instr ); i++ ) {
     VM_SYSCALL_CPI_ACC_META_T const * cpi_acct_meta = &cpi_acct_metas[i];
@@ -782,7 +782,7 @@ VM_SYSCALL_CPI_ENTRYPOINT( void *  _vm,
 
   /* Create the instruction to execute (in the input format the FD runtime expects) from
      the translated CPI ABI inputs. */
-  fd_pubkey_t cpi_instr_acct_keys[ FD_INSTR_ACCT_MAX ];
+  fd_pubkey_t cpi_instr_acct_keys[ FD_VM_CPI_MAX_INSTRUCTION_ACCOUNTS ];
   fd_instr_info_t * instruction_to_execute = &vm->instr_ctx->runtime->instr.trace[ vm->instr_ctx->runtime->instr.trace_length++ ];
 
   err = VM_SYSCALL_CPI_INSTRUCTION_TO_INSTR_FUNC( vm, cpi_instruction, cpi_account_metas, program_id, data, instruction_to_execute, cpi_instr_acct_keys );
@@ -800,7 +800,7 @@ VM_SYSCALL_CPI_ENTRYPOINT( void *  _vm,
 
   /* Prepare the instruction for execution in the runtime. This is required by the runtime
      before we can pass an instruction to the executor. */
-  fd_instruction_account_t instruction_accounts[256];
+  fd_instruction_account_t instruction_accounts[ FD_VM_CPI_MAX_INSTRUCTION_ACCOUNTS ];
   ulong instruction_accounts_cnt;
   err = fd_vm_prepare_instruction( instruction_to_execute, vm->instr_ctx, program_id, cpi_instr_acct_keys, instruction_accounts, &instruction_accounts_cnt, signers, signers_seeds_cnt );
   /* Errors are propagated in the function itself. */
@@ -847,9 +847,9 @@ VM_SYSCALL_CPI_ENTRYPOINT( void *  _vm,
      Update the callee accounts with any changes made by the caller prior to this CPI execution
 
      https://github.com/anza-xyz/agave/blob/v3.0.1/syscalls/src/cpi.rs#L767-L892 */
-  fd_vm_cpi_caller_account_t caller_accounts[ 256 ];
-  ushort callee_account_keys[256];
-  ushort caller_accounts_to_update[256];
+  fd_vm_cpi_caller_account_t caller_accounts[ FD_VM_CPI_MAX_INSTRUCTION_ACCOUNTS ];
+  ushort callee_account_keys[ FD_VM_CPI_MAX_INSTRUCTION_ACCOUNTS ];
+  ushort caller_accounts_to_update[ FD_VM_CPI_MAX_INSTRUCTION_ACCOUNTS ];
   ulong caller_accounts_to_update_len = 0;
   err = VM_SYSCALL_CPI_TRANSLATE_AND_UPDATE_ACCOUNTS_FUNC(
     vm,