Merge branch 'bugfix/update_gdma_soc' into 'master'

gdma: alignment setting for PSRAM transfer

Closes IDF-1524

See merge request espressif/esp-idf!13976

components/driver/gdma.c

@@ -74,6 +74,8 @@ struct gdma_channel_t {
     intr_handle_t intr; // per-channel interrupt handle
     gdma_channel_direction_t direction; // channel direction
     int periph_id; // Peripheral instance ID, indicates which peripheral is connected to this GDMA channel
+    size_t sram_alignment;  // alignment for memory in SRAM
+    size_t psram_alignment; // alignment for memory in PSRAM
     esp_err_t (*del)(gdma_channel_t *channel); // channel deletion function, it's polymorphic, see `gdma_del_tx_channel` or `gdma_del_rx_channel`
 };
 
@@ -271,6 +273,67 @@ esp_err_t gdma_disconnect(gdma_channel_handle_t dma_chan)
     return ret;
 }
 
+esp_err_t gdma_set_transfer_ability(gdma_channel_handle_t dma_chan, const gdma_transfer_ability_t *ability)
+{
+    esp_err_t ret = ESP_OK;
+    gdma_pair_t *pair = NULL;
+    gdma_group_t *group = NULL;
+    bool en_burst = true;
+    ESP_GOTO_ON_FALSE(dma_chan, ESP_ERR_INVALID_ARG, err, TAG, "invalid argument");
+    pair = dma_chan->pair;
+    group = pair->group;
+    size_t sram_alignment = ability->sram_trans_align;
+    size_t psram_alignment = ability->psram_trans_align;
+    // alignment should be 2^n
+    ESP_GOTO_ON_FALSE((sram_alignment & (sram_alignment - 1)) == 0, ESP_ERR_INVALID_ARG, err, TAG, "invalid sram alignment: %zu", sram_alignment);
+
+#if SOC_GDMA_SUPPORT_PSRAM
+    int block_size_index = 0;
+    switch (psram_alignment) {
+    case 64: // 64 Bytes alignment
+        block_size_index = GDMA_LL_EXT_MEM_BK_SIZE_64B;
+        break;
+    case 32: // 32 Bytes alignment
+        block_size_index = GDMA_LL_EXT_MEM_BK_SIZE_32B;
+        break;
+    case 16: // 16 Bytes alignment
+        block_size_index = GDMA_LL_EXT_MEM_BK_SIZE_16B;
+        break;
+    case 0: // no alignment is requirement
+        block_size_index = GDMA_LL_EXT_MEM_BK_SIZE_16B;
+        psram_alignment = SOC_GDMA_PSRAM_MIN_ALIGN; // fall back to minimal alignment
+        break;
+    default:
+        ESP_GOTO_ON_FALSE(false, ESP_ERR_INVALID_ARG, err, TAG, "invalid psram alignment: %zu", psram_alignment);
+        break;
+    }
+#endif // #if SOC_GDMA_SUPPORT_PSRAM
+
+    if (dma_chan->direction == GDMA_CHANNEL_DIRECTION_TX) {
+        // TX channel can always enable burst mode, no matter data alignment
+        gdma_ll_tx_enable_data_burst(group->hal.dev, pair->pair_id, true);
+        gdma_ll_tx_enable_descriptor_burst(group->hal.dev, pair->pair_id, true);
+#if SOC_GDMA_SUPPORT_PSRAM
+        gdma_ll_tx_set_block_size_psram(group->hal.dev, pair->pair_id, block_size_index);
+#endif // #if SOC_GDMA_SUPPORT_PSRAM
+    } else {
+        // RX channel burst mode depends on specific data alignment
+        en_burst = sram_alignment >= 4;
+        gdma_ll_rx_enable_data_burst(group->hal.dev, pair->pair_id, en_burst);
+        gdma_ll_rx_enable_descriptor_burst(group->hal.dev, pair->pair_id, en_burst);
+#if SOC_GDMA_SUPPORT_PSRAM
+        gdma_ll_rx_set_block_size_psram(group->hal.dev, pair->pair_id, block_size_index);
+#endif // #if SOC_GDMA_SUPPORT_PSRAM
+    }
+
+    dma_chan->sram_alignment = sram_alignment;
+    dma_chan->psram_alignment = psram_alignment;
+    ESP_LOGD(TAG, "%s channel (%d,%d), (%zu:%zu) bytes aligned, burst %s", dma_chan->direction == GDMA_CHANNEL_DIRECTION_TX ? "tx" : "rx",
+             group->group_id, pair->pair_id, sram_alignment, psram_alignment, en_burst ? "enabled" : "disabled");
+err:
+    return ret;
+}
+
 esp_err_t gdma_apply_strategy(gdma_channel_handle_t dma_chan, const gdma_strategy_config_t *config)
 {
     esp_err_t ret = ESP_OK;

components/driver/include/esp_private/gdma.h

@@ -38,7 +38,8 @@ typedef enum {
     GDMA_TRIG_PERIPH_ADC,  /*!< GDMA trigger peripheral: ADC */
     GDMA_TRIG_PERIPH_DAC,  /*!< GDMA trigger peripheral: DAC */
     GDMA_TRIG_PERIPH_LCD,  /*!< GDMA trigger peripheral: LCD */
-    GDMA_TRIG_PERIPH_CAM   /*!< GDMA trigger peripheral: CAM */
+    GDMA_TRIG_PERIPH_CAM,  /*!< GDMA trigger peripheral: CAM */
+    GDMA_TRIG_PERIPH_RMT,  /*!< GDMA trigger peripheral: RMT */
 } gdma_trigger_peripheral_t;
 
 /**
@@ -58,10 +59,23 @@ typedef struct {
     gdma_channel_handle_t sibling_chan; /*!< DMA sibling channel handle (NULL means having sibling is not necessary) */
     gdma_channel_direction_t direction; /*!< DMA channel direction */
     struct {
-        int reserve_sibling: 1;   /*!< If set, DMA channel allocator would prefer to allocate new channel in a new pair, and reserve sibling channel for future use */
+        int reserve_sibling: 1; /*!< If set, DMA channel allocator would prefer to allocate new channel in a new pair, and reserve sibling channel for future use */
     } flags;
 } gdma_channel_alloc_config_t;
 
+/**
+ * @brief GDMA transfer ability
+ *
+ * @note The alignment set in this structure is **not** a guarantee that gdma driver will take care of the nonalignment cases.
+ *       Actually the GDMA driver has no knowledge about the DMA buffer (address and size) used by upper layer.
+ *       So it's the responsibility of the **upper layer** to take care of the buffer address and size.
+ *
+ */
+typedef struct {
+    size_t sram_trans_align;  /*!< DMA transfer alignment for memory in SRAM, in bytes. The driver enables/disables burst mode based on this value. 0 means no alignment is required */
+    size_t psram_trans_align; /*!< DMA transfer alignment for memory in PSRAM, in bytes. The driver sets proper burst block size based on the alignment value. 0 means no alignment is required */
+} gdma_transfer_ability_t;
+
 /**
  * @brief Type of GDMA event data
  *
@@ -79,6 +93,9 @@ typedef struct {
  * @param event_data GDMA event data
  * @param user_data User registered data from `gdma_register_tx_event_callbacks` or `gdma_register_rx_event_callbacks`
  *
+ * @return Whether a task switch is needed after the callback function returns,
+ *         this is usually due to the callback wakes up some high priority task.
+ *
  */
 typedef bool (*gdma_event_callback_t)(gdma_channel_handle_t dma_chan, gdma_event_data_t *event_data, void *user_data);
 
@@ -171,6 +188,18 @@ esp_err_t gdma_connect(gdma_channel_handle_t dma_chan, gdma_trigger_t trig_perip
  */
 esp_err_t gdma_disconnect(gdma_channel_handle_t dma_chan);
 
+/**
+ * @brief Set DMA channel transfer ability
+ *
+ * @param[in] dma_chan GDMA channel handle, allocated by `gdma_new_channel`
+ * @param[in] ability Transfer ability, e.g. alignment
+ * @return
+ *      - ESP_OK: Set DMA channel transfer ability successfully
+ *      - ESP_ERR_INVALID_ARG: Set DMA channel transfer ability failed because of invalid argument
+ *      - ESP_FAIL: Set DMA channel transfer ability failed because of other error
+ */
+esp_err_t gdma_set_transfer_ability(gdma_channel_handle_t dma_chan, const gdma_transfer_ability_t *ability);
+
 /**
  * @brief Apply channel strategy for GDMA channel
  *

components/esp_hw_support/esp_async_memcpy.c

@@ -11,27 +11,20 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+
+#include <sys/param.h>
 #include "freertos/FreeRTOS.h"
 #include "freertos/semphr.h"
 #include "hal/dma_types.h"
-#include "esp_compiler.h"
+#include "esp_check.h"
 #include "esp_heap_caps.h"
 #include "esp_log.h"
 #include "esp_async_memcpy.h"
 #include "esp_async_memcpy_impl.h"
 
 static const char *TAG = "async_memcpy";
 
-#define ASMCP_CHECK(a, msg, tag, ret, ...)                                        \
-    do                                                                            \
-    {                                                                             \
-        if (unlikely(!(a)))                                                       \
-        {                                                                         \
-            ESP_LOGE(TAG, "%s(%d): " msg, __FUNCTION__, __LINE__, ##__VA_ARGS__); \
-            ret_code = ret;                                                       \
-            goto tag;                                                             \
-        }                                                                         \
-    } while (0)
+#define ALIGN_DOWN(val, align)  ((val) & ~((align) - 1))
 
 /**
  * @brief Type of async mcp stream
@@ -54,25 +47,26 @@ typedef struct async_memcpy_context_t {
     dma_descriptor_t *tx_desc; // pointer to the next free TX descriptor
     dma_descriptor_t *rx_desc; // pointer to the next free RX descriptor
     dma_descriptor_t *next_rx_desc_to_check; // pointer to the next RX descriptor to recycle
-    uint32_t max_stream_num;            // maximum number of streams
+    uint32_t max_stream_num;    // maximum number of streams
+    size_t max_dma_buffer_size; // maximum DMA buffer size
     async_memcpy_stream_t *out_streams;    // pointer to the first TX stream
     async_memcpy_stream_t *in_streams;     // pointer to the first RX stream
     async_memcpy_stream_t streams_pool[0]; // stream pool (TX + RX), the size is configured during driver installation
 } async_memcpy_context_t;
 
 esp_err_t esp_async_memcpy_install(const async_memcpy_config_t *config, async_memcpy_t *asmcp)
 {
-    esp_err_t ret_code = ESP_OK;
+    esp_err_t ret = ESP_OK;
     async_memcpy_context_t *mcp_hdl = NULL;
 
-    ASMCP_CHECK(config, "configuration can't be null", err, ESP_ERR_INVALID_ARG);
-    ASMCP_CHECK(asmcp, "can't assign mcp handle to null", err, ESP_ERR_INVALID_ARG);
+    ESP_GOTO_ON_FALSE(config, ESP_ERR_INVALID_ARG, err, TAG, "configuration can't be null");
+    ESP_GOTO_ON_FALSE(asmcp, ESP_ERR_INVALID_ARG, err, TAG, "can't assign mcp handle to null");
 
     // context memory size + stream pool size
     size_t total_malloc_size = sizeof(async_memcpy_context_t) + sizeof(async_memcpy_stream_t) * config->backlog * 2;
     // to work when cache is disabled, the driver handle should located in SRAM
     mcp_hdl = heap_caps_calloc(1, total_malloc_size, MALLOC_CAP_8BIT | MALLOC_CAP_DMA | MALLOC_CAP_INTERNAL);
-    ASMCP_CHECK(mcp_hdl, "allocate context memory failed", err, ESP_ERR_NO_MEM);
+    ESP_GOTO_ON_FALSE(mcp_hdl, ESP_ERR_NO_MEM, err, TAG, "allocate context memory failed");
 
     mcp_hdl->flags = config->flags;
     mcp_hdl->out_streams = mcp_hdl->streams_pool;
@@ -93,9 +87,14 @@ esp_err_t esp_async_memcpy_install(const async_memcpy_config_t *config, async_me
     mcp_hdl->rx_desc = &mcp_hdl->in_streams[0].desc;
     mcp_hdl->next_rx_desc_to_check = &mcp_hdl->in_streams[0].desc;
     mcp_hdl->spinlock = (portMUX_TYPE)portMUX_INITIALIZER_UNLOCKED;
+    mcp_hdl->mcp_impl.sram_trans_align = config->sram_trans_align;
+    mcp_hdl->mcp_impl.psram_trans_align = config->psram_trans_align;
+    size_t trans_align = MAX(config->sram_trans_align, config->psram_trans_align);
+    mcp_hdl->max_dma_buffer_size = trans_align ? ALIGN_DOWN(DMA_DESCRIPTOR_BUFFER_MAX_SIZE, trans_align) : DMA_DESCRIPTOR_BUFFER_MAX_SIZE;
 
     // initialize implementation layer
-    async_memcpy_impl_init(&mcp_hdl->mcp_impl);
+    ret = async_memcpy_impl_init(&mcp_hdl->mcp_impl);
+    ESP_GOTO_ON_ERROR(ret, err, TAG, "DMA M2M init failed");
 
     *asmcp = mcp_hdl;
 
@@ -109,20 +108,19 @@ esp_err_t esp_async_memcpy_install(const async_memcpy_config_t *config, async_me
     if (asmcp) {
         *asmcp = NULL;
     }
-    return ret_code;
+    return ret;
 }
 
 esp_err_t esp_async_memcpy_uninstall(async_memcpy_t asmcp)
 {
-    esp_err_t ret_code = ESP_OK;
-    ASMCP_CHECK(asmcp, "mcp handle can't be null", err, ESP_ERR_INVALID_ARG);
+    esp_err_t ret = ESP_OK;
+    ESP_GOTO_ON_FALSE(asmcp, ESP_ERR_INVALID_ARG, err, TAG, "mcp handle can't be null");
 
     async_memcpy_impl_stop(&asmcp->mcp_impl);
     async_memcpy_impl_deinit(&asmcp->mcp_impl);
     free(asmcp);
-    return ESP_OK;
 err:
-    return ret_code;
+    return ret;
 }
 
 static int async_memcpy_prepare_receive(async_memcpy_t asmcp, void *buffer, size_t size, dma_descriptor_t **start_desc, dma_descriptor_t **end_desc)
@@ -133,14 +131,14 @@ static int async_memcpy_prepare_receive(async_memcpy_t asmcp, void *buffer, size
     dma_descriptor_t *start = desc;
     dma_descriptor_t *end = desc;
 
-    while (size > DMA_DESCRIPTOR_BUFFER_MAX_SIZE) {
+    while (size > asmcp->max_dma_buffer_size) {
         if (desc->dw0.owner != DMA_DESCRIPTOR_BUFFER_OWNER_DMA) {
             desc->dw0.suc_eof = 0;
-            desc->dw0.size = DMA_DESCRIPTOR_BUFFER_MAX_SIZE;
+            desc->dw0.size = asmcp->max_dma_buffer_size;
             desc->buffer = &buf[prepared_length];
             desc = desc->next; // move to next descriptor
-            prepared_length += DMA_DESCRIPTOR_BUFFER_MAX_SIZE;
-            size -= DMA_DESCRIPTOR_BUFFER_MAX_SIZE;
+            prepared_length += asmcp->max_dma_buffer_size;
+            size -= asmcp->max_dma_buffer_size;
         } else {
             // out of RX descriptors
             goto _exit;
@@ -174,15 +172,15 @@ static int async_memcpy_prepare_transmit(async_memcpy_t asmcp, void *buffer, siz
     dma_descriptor_t *start = desc;
     dma_descriptor_t *end = desc;
 
-    while (len > DMA_DESCRIPTOR_BUFFER_MAX_SIZE) {
+    while (len > asmcp->max_dma_buffer_size) {
         if (desc->dw0.owner != DMA_DESCRIPTOR_BUFFER_OWNER_DMA) {
             desc->dw0.suc_eof = 0; // not the end of the transaction
-            desc->dw0.size = DMA_DESCRIPTOR_BUFFER_MAX_SIZE;
-            desc->dw0.length = DMA_DESCRIPTOR_BUFFER_MAX_SIZE;
+            desc->dw0.size = asmcp->max_dma_buffer_size;
+            desc->dw0.length = asmcp->max_dma_buffer_size;
             desc->buffer = &buf[prepared_length];
             desc = desc->next; // move to next descriptor
-            prepared_length += DMA_DESCRIPTOR_BUFFER_MAX_SIZE;
-            len -= DMA_DESCRIPTOR_BUFFER_MAX_SIZE;
+            prepared_length += asmcp->max_dma_buffer_size;
+            len -= asmcp->max_dma_buffer_size;
         } else {
             // out of TX descriptors
             goto _exit;
@@ -226,22 +224,28 @@ static bool async_memcpy_get_next_rx_descriptor(async_memcpy_t asmcp, dma_descri
 
 esp_err_t esp_async_memcpy(async_memcpy_t asmcp, void *dst, void *src, size_t n, async_memcpy_isr_cb_t cb_isr, void *cb_args)
 {
-    esp_err_t ret_code = ESP_OK;
+    esp_err_t ret = ESP_OK;
     dma_descriptor_t *rx_start_desc = NULL;
     dma_descriptor_t *rx_end_desc = NULL;
     dma_descriptor_t *tx_start_desc = NULL;
     dma_descriptor_t *tx_end_desc = NULL;
     size_t rx_prepared_size = 0;
     size_t tx_prepared_size = 0;
-    ASMCP_CHECK(asmcp, "mcp handle can't be null", err, ESP_ERR_INVALID_ARG);
-    ASMCP_CHECK(async_memcpy_impl_is_buffer_address_valid(&asmcp->mcp_impl, src, dst), "buffer address not valid", err, ESP_ERR_INVALID_ARG);
-    ASMCP_CHECK(n <= DMA_DESCRIPTOR_BUFFER_MAX_SIZE * asmcp->max_stream_num, "buffer size too large", err, ESP_ERR_INVALID_ARG);
+    ESP_GOTO_ON_FALSE(asmcp, ESP_ERR_INVALID_ARG, err, TAG, "mcp handle can't be null");
+    ESP_GOTO_ON_FALSE(async_memcpy_impl_is_buffer_address_valid(&asmcp->mcp_impl, src, dst), ESP_ERR_INVALID_ARG, err, TAG, "buffer address not valid: %p -> %p", src, dst);
+    ESP_GOTO_ON_FALSE(n <= asmcp->max_dma_buffer_size * asmcp->max_stream_num, ESP_ERR_INVALID_ARG, err, TAG, "buffer size too large");
+    if (asmcp->mcp_impl.sram_trans_align) {
+        ESP_GOTO_ON_FALSE(((n & (asmcp->mcp_impl.sram_trans_align - 1)) == 0), ESP_ERR_INVALID_ARG, err, TAG, "copy size should align to %d bytes", asmcp->mcp_impl.sram_trans_align);
+    }
+    if (asmcp->mcp_impl.psram_trans_align) {
+        ESP_GOTO_ON_FALSE(((n & (asmcp->mcp_impl.psram_trans_align - 1)) == 0), ESP_ERR_INVALID_ARG, err, TAG, "copy size should align to %d bytes", asmcp->mcp_impl.psram_trans_align);
+    }
 
     // Prepare TX and RX descriptor
     portENTER_CRITICAL_SAFE(&asmcp->spinlock);
     rx_prepared_size = async_memcpy_prepare_receive(asmcp, dst, n, &rx_start_desc, &rx_end_desc);
     tx_prepared_size = async_memcpy_prepare_transmit(asmcp, src, n, &tx_start_desc, &tx_end_desc);
-    if ((rx_prepared_size == n) && (tx_prepared_size == n)) {
+    if (rx_start_desc && tx_start_desc && (rx_prepared_size == n) && (tx_prepared_size == n)) {
         // register user callback to the last descriptor
         async_memcpy_stream_t *mcp_stream = __containerof(rx_end_desc, async_memcpy_stream_t, desc);
         mcp_stream->cb = cb_isr;
@@ -268,12 +272,11 @@ esp_err_t esp_async_memcpy(async_memcpy_t asmcp, void *dst, void *src, size_t n,
 
     // It's unlikely that we have space for rx descriptor but no space for tx descriptor
     // Both tx and rx descriptor should move in the same pace
-    ASMCP_CHECK(rx_prepared_size == n, "out of rx descriptor", err, ESP_FAIL);
-    ASMCP_CHECK(tx_prepared_size == n, "out of tx descriptor", err, ESP_FAIL);
+    ESP_GOTO_ON_FALSE(rx_prepared_size == n, ESP_FAIL, err, TAG, "out of rx descriptor");
+    ESP_GOTO_ON_FALSE(tx_prepared_size == n, ESP_FAIL, err, TAG, "out of tx descriptor");
 
-    return ESP_OK;
 err:
-    return ret_code;
+    return ret;
 }
 
 IRAM_ATTR void async_memcpy_isr_on_rx_done_event(async_memcpy_impl_t *impl)

components/esp_hw_support/include/esp_async_memcpy.h

@@ -54,18 +54,22 @@ typedef bool (*async_memcpy_isr_cb_t)(async_memcpy_t mcp_hdl, async_memcpy_event
  *
  */
 typedef struct {
-    uint32_t backlog; /*!< Maximum number of streams that can be handled simultaneously */
-    uint32_t flags;   /*!< Extra flags to control async memcpy feature */
+    uint32_t backlog;          /*!< Maximum number of streams that can be handled simultaneously */
+    size_t sram_trans_align;   /*!< DMA transfer alignment (both in size and address) for SRAM memory */
+    size_t psram_trans_align;  /*!< DMA transfer alignment (both in size and address) for PSRAM memory */
+    uint32_t flags;            /*!< Extra flags to control async memcpy feature */
 } async_memcpy_config_t;
 
 /**
  * @brief Default configuration for async memcpy
  *
  */
 #define ASYNC_MEMCPY_DEFAULT_CONFIG() \
-    {                              \
-        .backlog = 8,              \
-        .flags = 0,                \
+    {                                 \
+        .backlog = 8,                 \
+        .sram_trans_align = 0,        \
+        .psram_trans_align = 0,       \
+        .flags = 0,                   \
     }
 
 /**