Skip to content

Commit

Permalink
fix(mbedtls/crypto_shared_gdma): Enable AXI-DMA enable external memor…
Browse files Browse the repository at this point in the history 
…y AES-ECC access

- When external memory encryption is enabled, set the aes_ecc bit of AXI-DMA to enable memory access
  • Loading branch information
@Harshal5
Harshal5 committed May 13, 2024
1 parent e2f6920 commit 5dfbc47
Show file tree
Hide file tree
Showing 7 changed files with 217 additions and 28 deletions.
16 changes: 16 additions & 0 deletions components/hal/esp32p4/include/hal/axi_dma_ll.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -273,6 +273,14 @@ static inline void axi_dma_ll_rx_enable_etm_task(axi_dma_dev_t *dev, uint32_t ch
dev->in[channel].conf.in_conf0.in_etm_en_chn = enable;
}

/**
* @brief Whether to enable the mean access ecc or aes domain
*/
static inline void axi_dma_ll_rx_enable_ext_mem_ecc_aes_access(axi_dma_dev_t *dev, uint32_t channel, bool enable)
{
dev->in[channel].conf.in_conf0.in_ecc_aec_en_chn = enable;
}

///////////////////////////////////// TX /////////////////////////////////////////
/**
* @brief Get DMA TX channel interrupt status word
Expand Down Expand Up @@ -471,6 +479,14 @@ static inline void axi_dma_ll_tx_enable_etm_task(axi_dma_dev_t *dev, uint32_t ch
dev->out[channel].conf.out_conf0.out_etm_en_chn = enable;
}

/**
* @brief Whether to enable the mean access ecc or aes domain
*/
static inline void axi_dma_ll_tx_enable_ext_mem_ecc_aes_access(axi_dma_dev_t *dev, uint32_t channel, bool enable)
{
dev->out[channel].conf.out_conf0.out_ecc_aec_en_chn = enable;
}

///////////////////////////////////// CRC-TX /////////////////////////////////////////

/**
Expand Down
3 changes: 3 additions & 0 deletions components/mbedtls/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,9 @@ if(SHA_PERIPHERAL_TYPE STREQUAL "dma" OR AES_PERIPHERAL_TYPE STREQUAL "dma")
target_link_libraries(mbedcrypto PRIVATE idf::esp_mm)
if(CONFIG_SOC_SHA_GDMA OR CONFIG_SOC_AES_GDMA)
target_sources(mbedcrypto PRIVATE "${COMPONENT_DIR}/port/crypto_shared_gdma/esp_crypto_shared_gdma.c")
if(CONFIG_SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT)
target_link_libraries(mbedcrypto PRIVATE idf::spi_flash idf::bootloader_support)
endif()
endif()
endif()

Expand Down
93 changes: 67 additions & 26 deletions components/mbedtls/port/aes/dma/esp_aes_dma_core.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@
#include "esp_memory_utils.h"
#include "esp_private/esp_cache_private.h"
#include "esp_private/periph_ctrl.h"
#include "soc/soc_caps.h"
#include "sdkconfig.h"

#if CONFIG_PM_ENABLE
#include "esp_pm.h"
Expand All @@ -36,10 +38,19 @@
#include "aes/esp_aes_gcm.h"
#endif

#ifdef SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT
#include "esp_flash_encrypt.h"
#endif /* SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT */

/* Max size of each chunk to process when output buffer is in unaligned external ram
must be a multiple of block size
*/
#if (CONFIG_IDF_TARGET_ESP32P4 && CONFIG_SPIRAM && SOC_PSRAM_DMA_CAPABLE)
/* As P4 has larger memory than other targets, thus we can support a larger chunk write size */
#define AES_MAX_CHUNK_WRITE_SIZE 8*1024
#else
#define AES_MAX_CHUNK_WRITE_SIZE 1600
#endif

/* Input over this length will yield and wait for interrupt instead of
busy-waiting, 30000 bytes is approx 0.5 ms */
Expand Down Expand Up @@ -163,6 +174,26 @@ static int esp_aes_dma_wait_complete(bool use_intr, crypto_dma_desc_t *output_de
return 0;
}

static inline size_t get_cache_line_size(const void *addr)
{
esp_err_t ret = ESP_FAIL;
size_t cache_line_size = 0;

#if (CONFIG_SPIRAM && SOC_PSRAM_DMA_CAPABLE)
if (esp_ptr_external_ram(addr)) {
ret = esp_cache_get_alignment(MALLOC_CAP_SPIRAM, &cache_line_size);
} else
#endif
{
ret = esp_cache_get_alignment(MALLOC_CAP_DMA, &cache_line_size);
}

if (ret != ESP_OK) {
return 0;
}

return cache_line_size;
}

/* Output buffers in external ram needs to be 16-byte aligned and DMA can't access input in the iCache mem range,
reallocate them into internal memory and encrypt in chunks to avoid
Expand All @@ -176,13 +207,29 @@ static int esp_aes_process_dma_ext_ram(esp_aes_context *ctx, const unsigned char
size_t chunk_len;
int ret = 0;
int offset = 0;
uint32_t heap_caps = 0;
unsigned char *input_buf = NULL;
unsigned char *output_buf = NULL;
const unsigned char *dma_input;
chunk_len = MIN(AES_MAX_CHUNK_WRITE_SIZE, len);

size_t input_alignment = 1;
size_t output_alignment = 1;

/* When AES-DMA operations are carried out using external memory with external memory encryption enabled,
we need to make sure that the addresses and the sizes of the buffers on which the DMA operates are 16 byte-aligned. */
#ifdef SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT
if (esp_flash_encryption_enabled()) {
if (esp_ptr_external_ram(input) || esp_ptr_external_ram(output) || esp_ptr_in_drom(input) || esp_ptr_in_drom(output)) {
input_alignment = MAX(get_cache_line_size(input), SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT);
output_alignment = MAX(get_cache_line_size(output), SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT);
}
}
#endif /* SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT */

if (realloc_input) {
input_buf = heap_caps_malloc(chunk_len, MALLOC_CAP_DMA);
heap_caps = MALLOC_CAP_8BIT | (esp_ptr_external_ram(input) ? MALLOC_CAP_SPIRAM : MALLOC_CAP_DMA | MALLOC_CAP_INTERNAL);
input_buf = heap_caps_aligned_alloc(input_alignment, chunk_len, heap_caps);

if (input_buf == NULL) {
mbedtls_platform_zeroize(output, len);
Expand All @@ -192,7 +239,8 @@ static int esp_aes_process_dma_ext_ram(esp_aes_context *ctx, const unsigned char
}

if (realloc_output) {
output_buf = heap_caps_malloc(chunk_len, MALLOC_CAP_DMA);
heap_caps = MALLOC_CAP_8BIT | (esp_ptr_external_ram(output) ? MALLOC_CAP_SPIRAM : MALLOC_CAP_DMA | MALLOC_CAP_INTERNAL);
output_buf = heap_caps_aligned_alloc(output_alignment, chunk_len, heap_caps);

if (output_buf == NULL) {
mbedtls_platform_zeroize(output, len);
Expand Down Expand Up @@ -284,27 +332,6 @@ static inline void *aes_dma_calloc(size_t num, size_t size, uint32_t caps, size_
return ptr;
}

static inline size_t get_cache_line_size(const void *addr)
{
esp_err_t ret = ESP_FAIL;
size_t cache_line_size = 0;

#if (CONFIG_SPIRAM && SOC_PSRAM_DMA_CAPABLE)
if (esp_ptr_external_ram(addr)) {
ret = esp_cache_get_alignment(MALLOC_CAP_SPIRAM, &cache_line_size);
} else
#endif
{
ret = esp_cache_get_alignment(MALLOC_CAP_DMA, &cache_line_size);
}

if (ret != ESP_OK) {
return 0;
}

return cache_line_size;
}

static inline esp_err_t dma_desc_link(crypto_dma_desc_t *dmadesc, size_t crypto_dma_desc_num, size_t cache_line_size)
{
esp_err_t ret = ESP_OK;
Expand Down Expand Up @@ -404,7 +431,7 @@ static esp_err_t generate_descriptor_list(const uint8_t *buffer, const size_t le
dma_descs_needed = (unaligned_start_bytes ? 1 : 0) + dma_desc_get_required_num(aligned_block_bytes, max_desc_size) + (unaligned_end_bytes ? 1 : 0);

/* Allocate memory for DMA descriptors of total size aligned up to a multiple of cache line size */
dma_descriptors = (crypto_dma_desc_t *) aes_dma_calloc(dma_descs_needed, sizeof(crypto_dma_desc_t), MALLOC_CAP_DMA, NULL);
dma_descriptors = (crypto_dma_desc_t *) aes_dma_calloc(dma_descs_needed, sizeof(crypto_dma_desc_t), MALLOC_CAP_DMA | MALLOC_CAP_INTERNAL, NULL);
if (dma_descriptors == NULL) {
ESP_LOGE(TAG, "Failed to allocate memory for the array of DMA descriptors");
return ESP_FAIL;
Expand All @@ -413,7 +440,7 @@ static esp_err_t generate_descriptor_list(const uint8_t *buffer, const size_t le
size_t populated_dma_descs = 0;

if (unaligned_start_bytes) {
start_alignment_stream_buffer = aes_dma_calloc(alignment_buffer_size, sizeof(uint8_t), AES_DMA_ALLOC_CAPS, NULL);
start_alignment_stream_buffer = aes_dma_calloc(alignment_buffer_size, sizeof(uint8_t), AES_DMA_ALLOC_CAPS | (esp_ptr_external_ram(buffer) ? MALLOC_CAP_SPIRAM : MALLOC_CAP_INTERNAL) , NULL);
if (start_alignment_stream_buffer == NULL) {
ESP_LOGE(TAG, "Failed to allocate memory for start alignment buffer");
return ESP_FAIL;
Expand All @@ -435,7 +462,7 @@ static esp_err_t generate_descriptor_list(const uint8_t *buffer, const size_t le
}

if (unaligned_end_bytes) {
end_alignment_stream_buffer = aes_dma_calloc(alignment_buffer_size, sizeof(uint8_t), AES_DMA_ALLOC_CAPS, NULL);
end_alignment_stream_buffer = aes_dma_calloc(alignment_buffer_size, sizeof(uint8_t), AES_DMA_ALLOC_CAPS | (esp_ptr_external_ram(buffer) ? MALLOC_CAP_SPIRAM : MALLOC_CAP_INTERNAL), NULL);
if (end_alignment_stream_buffer == NULL) {
ESP_LOGE(TAG, "Failed to allocate memory for end alignment buffer");
return ESP_FAIL;
Expand Down Expand Up @@ -499,6 +526,20 @@ int esp_aes_process_dma(esp_aes_context *ctx, const unsigned char *input, unsign
return MBEDTLS_ERR_AES_INVALID_INPUT_LENGTH;
}

#ifdef SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT
if (esp_flash_encryption_enabled()) {
if (esp_ptr_external_ram(input) || esp_ptr_external_ram(output) || esp_ptr_in_drom(input) || esp_ptr_in_drom(output)) {
if (((intptr_t)(input) & (SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT - 1)) != 0) {
input_needs_realloc = true;
}

if (((intptr_t)(output) & (SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT - 1)) != 0) {
output_needs_realloc = true;
}
}
}
#endif /* SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT */

/* DMA cannot access memory in the iCache range, copy input to internal ram */
if (!s_check_dma_capable(input)) {
input_needs_realloc = true;
Expand Down
41 changes: 40 additions & 1 deletion components/mbedtls/port/crypto_shared_gdma/esp_crypto_shared_gdma.c
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: 2021-2023 Espressif Systems (Shanghai) CO LTD
* SPDX-FileCopyrightText: 2021-2024 Espressif Systems (Shanghai) CO LTD
*
* SPDX-License-Identifier: Apache-2.0
*/
Expand All @@ -13,7 +13,13 @@
#include "esp_cache.h"
#include "esp_crypto_dma.h"
#include "esp_crypto_lock.h"
#include "esp_memory_utils.h"
#include "soc/soc_caps.h"
#include "sdkconfig.h"

#ifdef SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT
#include "esp_flash_encrypt.h"
#endif /* SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT */

#if SOC_AHB_GDMA_VERSION == 1
#include "hal/gdma_ll.h"
Expand Down Expand Up @@ -140,6 +146,22 @@ esp_err_t esp_crypto_shared_gdma_start(const lldesc_t *input, const lldesc_t *ou
return ESP_OK;
}

/* The external memory ecc-aes access must be enabled when there exists
at least one buffer in the DMA descriptors that resides in external memory. */
#ifdef SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT
static bool check_dma_descs_need_ext_mem_ecc_aes_access(const crypto_dma_desc_t *dmadesc)
{
crypto_dma_desc_t* desc = (crypto_dma_desc_t*) dmadesc;
while (desc) {
if (esp_ptr_in_drom(desc->buffer) || esp_ptr_external_ram(desc->buffer)) {
return true;
}
desc = desc->next;
}
return false;
}
#endif /* SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT */

esp_err_t esp_crypto_shared_gdma_start_axi_ahb(const crypto_dma_desc_t *input, const crypto_dma_desc_t *output, gdma_trigger_peripheral_t peripheral)
{
int rx_ch_id = 0;
Expand Down Expand Up @@ -173,6 +195,23 @@ esp_err_t esp_crypto_shared_gdma_start_axi_ahb(const crypto_dma_desc_t *input, c
axi_dma_ll_rx_reset_channel(&AXI_DMA, rx_ch_id);
#endif /* SOC_AHB_GDMA_VERSION */

/* When GDMA operations are carried out using external memory with external memory encryption enabled,
we need to enable AXI-DMA's AES-ECC mean access bit. */
#if (SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT)
if (esp_flash_encryption_enabled()) {
int tx_ch_id = 0;
gdma_get_channel_id(tx_channel, &tx_ch_id);

if (check_dma_descs_need_ext_mem_ecc_aes_access(input) || check_dma_descs_need_ext_mem_ecc_aes_access(output)) {
axi_dma_ll_rx_enable_ext_mem_ecc_aes_access(&AXI_DMA, rx_ch_id, true);
axi_dma_ll_tx_enable_ext_mem_ecc_aes_access(&AXI_DMA, tx_ch_id, true);
} else {
axi_dma_ll_rx_enable_ext_mem_ecc_aes_access(&AXI_DMA, rx_ch_id, false);
axi_dma_ll_tx_enable_ext_mem_ecc_aes_access(&AXI_DMA, tx_ch_id, false);
}
}
#endif /* SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT */

gdma_start(tx_channel, (intptr_t)input);
gdma_start(rx_channel, (intptr_t)output);

Expand Down
87 changes: 86 additions & 1 deletion components/mbedtls/port/sha/dma/sha.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,11 @@
#include "hal/sha_ll.h"
#include "soc/soc_caps.h"
#include "esp_sha_dma_priv.h"
#include "sdkconfig.h"

#ifdef SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT
#include "esp_flash_encrypt.h"
#endif /* SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT */

#if SOC_SHA_GDMA
#define SHA_LOCK() esp_crypto_sha_aes_lock_acquire()
Expand Down Expand Up @@ -168,6 +173,63 @@ static void esp_sha_block_mode(esp_sha_type sha_type, const uint8_t *input, uint
static DRAM_ATTR crypto_dma_desc_t s_dma_descr_input;
static DRAM_ATTR crypto_dma_desc_t s_dma_descr_buf;

static esp_err_t esp_sha_dma_process(esp_sha_type sha_type, const void *input, uint32_t ilen,
const void *buf, uint32_t buf_len, bool is_first_block);

#ifdef SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT
static esp_err_t esp_sha_dma_process_ext(esp_sha_type sha_type, const void *input, uint32_t ilen,
const void *buf, uint32_t buf_len, bool is_first_block,
bool realloc_input, bool realloc_buf)
{
int ret = ESP_FAIL;
void *input_copy = NULL;
void *buf_copy = NULL;

const void *dma_input = NULL;
const void *dma_buf = NULL;

uint32_t heap_caps = 0;

if (realloc_input) {
heap_caps = MALLOC_CAP_8BIT | (esp_ptr_external_ram(input) ? MALLOC_CAP_SPIRAM : MALLOC_CAP_DMA | MALLOC_CAP_INTERNAL);
input_copy = heap_caps_aligned_alloc(SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT, ilen, heap_caps);
if (input_copy == NULL) {
ESP_LOGE(TAG, "Failed to allocate aligned SPIRAM memory");
return ret;
}
memcpy(input_copy, input, ilen);
dma_input = input_copy;
} else {
dma_input = input;
}

if (realloc_buf) {
heap_caps = MALLOC_CAP_8BIT | (esp_ptr_external_ram(buf) ? MALLOC_CAP_SPIRAM : MALLOC_CAP_DMA | MALLOC_CAP_INTERNAL);
buf_copy = heap_caps_aligned_alloc(SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT, buf_len, heap_caps);
if (buf_copy == NULL) {
ESP_LOGE(TAG, "Failed to allocate aligned internal memory");
return ret;
}
memcpy(buf_copy, buf, buf_len);
dma_buf = buf_copy;
} else {
dma_buf = buf;
}

ret = esp_sha_dma_process(sha_type, dma_input, ilen, dma_buf, buf_len, is_first_block);

if (realloc_input) {
free(input_copy);
}

if (realloc_buf) {
free(buf_copy);
}

return ret;
}
#endif /* SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT */

/* Performs SHA on multiple blocks at a time */
static esp_err_t esp_sha_dma_process(esp_sha_type sha_type, const void *input, uint32_t ilen,
const void *buf, uint32_t buf_len, bool is_first_block)
Expand All @@ -179,6 +241,29 @@ static esp_err_t esp_sha_dma_process(esp_sha_type sha_type, const void *input, u
memset(&s_dma_descr_input, 0, sizeof(crypto_dma_desc_t));
memset(&s_dma_descr_buf, 0, sizeof(crypto_dma_desc_t));

/* When SHA-DMA operations are carried out using external memory with external memory encryption enabled,
we need to make sure that the addresses and the sizes of the buffers on which the DMA operates are 16 byte-aligned. */
#ifdef SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT
if (esp_flash_encryption_enabled()) {
if (esp_ptr_external_ram(input) || esp_ptr_external_ram(buf) || esp_ptr_in_drom(input) || esp_ptr_in_drom(buf)) {
bool input_needs_realloc = false;
bool buf_needs_realloc = false;

if (ilen && ((intptr_t)(input) & (SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT - 1)) != 0) {
input_needs_realloc = true;
}

if (buf_len && ((intptr_t)(buf) & (SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT - 1)) != 0) {
buf_needs_realloc = true;
}

if (input_needs_realloc || buf_needs_realloc) {
return esp_sha_dma_process_ext(sha_type, input, ilen, buf, buf_len, is_first_block, input_needs_realloc, buf_needs_realloc);
}
}
}
#endif /* SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT */

/* DMA descriptor for Memory to DMA-SHA transfer */
if (ilen) {
s_dma_descr_input.dw0.length = ilen;
Expand All @@ -188,7 +273,7 @@ static esp_err_t esp_sha_dma_process(esp_sha_type sha_type, const void *input, u
s_dma_descr_input.buffer = (void *) input;
dma_descr_head = &s_dma_descr_input;
}
/* Check after input to overide head if there is any buf*/
/* Check after input to override head if there is any buf*/
if (buf_len) {
s_dma_descr_buf.dw0.length = buf_len;
s_dma_descr_buf.dw0.size = buf_len;
Expand Down
Loading

0 comments on commit 5dfbc47

Please sign in to comment.