Merge branch 'feature/partition_api_new_component_v5.0' into 'release…

…/v5.0'

Storage: Partition APIs moved to the new component 'esp_partition' (v5.0)

See merge request espressif/esp-idf!20855

.gitlab/ci/host-test.yml

@@ -437,6 +437,14 @@ test_linux_example:
     - timeout 5 ./build/linux_host_app.elf >test.log || true
     - grep "Restarting" test.log
 
+test_partition_api_host:
+  extends: .host_test_template
+  script:
+    - cd ${IDF_PATH}/components/esp_partition/host_test/partition_api_test
+    - idf.py build
+    - timeout 5 ./build/partition_api_test.elf >test.log
+    - grep " 0 Failures" test.log
+
 test_gen_soc_caps_kconfig:
   extends: .host_test_template
   script:

components/app_update/CMakeLists.txt

@@ -1,7 +1,7 @@
 idf_component_register(SRCS "esp_ota_ops.c" "esp_ota_app_desc.c"
                     INCLUDE_DIRS "include"
-                    REQUIRES spi_flash partition_table bootloader_support esp_app_format
-                    PRIV_REQUIRES esptool_py efuse)
+                    REQUIRES partition_table bootloader_support esp_app_format esp_partition
+                    PRIV_REQUIRES esptool_py efuse spi_flash)
 
 if(NOT BOOTLOADER_BUILD)
     partition_table_get_partition_info(otadata_offset "--partition-type data --partition-subtype ota" "offset")

components/app_update/esp_ota_ops.c

@@ -14,7 +14,6 @@
 
 #include "esp_err.h"
 #include "esp_partition.h"
-#include "spi_flash_mmap.h"
 #include "esp_image_format.h"
 #include "esp_secure_boot.h"
 #include "esp_flash_encrypt.h"
@@ -84,16 +83,16 @@ static const esp_partition_t *read_otadata(esp_ota_select_entry_t *two_otadata)
         return NULL;
     }
 
-    spi_flash_mmap_handle_t ota_data_map;
+    esp_partition_mmap_handle_t ota_data_map;
     const void *result = NULL;
-    esp_err_t err = esp_partition_mmap(otadata_partition, 0, otadata_partition->size, SPI_FLASH_MMAP_DATA, &result, &ota_data_map);
+    esp_err_t err = esp_partition_mmap(otadata_partition, 0, otadata_partition->size, ESP_PARTITION_MMAP_DATA, &result, &ota_data_map);
     if (err != ESP_OK) {
         ESP_LOGE(TAG, "mmap otadata filed. Err=0x%8x", err);
         return NULL;
     } else {
         memcpy(&two_otadata[0], result, sizeof(esp_ota_select_entry_t));
         memcpy(&two_otadata[1], result + SPI_FLASH_SEC_SIZE, sizeof(esp_ota_select_entry_t));
-        spi_flash_munmap(ota_data_map);
+        esp_partition_munmap(ota_data_map);
     }
     return otadata_partition;
 }

components/app_update/test/CMakeLists.txt

@@ -1,5 +1,5 @@
 idf_component_register(SRC_DIRS "."
                        PRIV_INCLUDE_DIRS "."
-                       PRIV_REQUIRES cmock test_utils app_update bootloader_support nvs_flash driver
+                       PRIV_REQUIRES cmock test_utils app_update bootloader_support nvs_flash driver spi_flash
                       )
 target_compile_options(${COMPONENT_LIB} PRIVATE "-Wno-format")

components/driver/test/CMakeLists.txt

@@ -1,7 +1,7 @@
 idf_component_register(SRC_DIRS . param_test touch_sensor_test dac_dma_test
                        PRIV_INCLUDE_DIRS include param_test/include touch_sensor_test/include
                        PRIV_REQUIRES cmock test_utils driver nvs_flash
-                                     esp_timer esp_adc esp_event esp_wifi)
+                                     esp_timer esp_adc esp_event esp_wifi spi_flash)
 target_compile_options(${COMPONENT_LIB} PRIVATE "-Wno-format")
 
 # A local copy of idf-extra-components esp_serial_slave_link, for stabilities of the SDIO test

components/driver/test/test_spi_bus_lock.c

@@ -8,6 +8,7 @@
 #include "driver/spi_master.h"
 #include "driver/gpio.h"
 #include "esp_flash_spi_init.h"
+#include "spi_flash_mmap.h"
 
 #include "test/test_common_spi.h"
 #include "unity.h"

components/esp_hw_support/test/CMakeLists.txt

@@ -1,6 +1,6 @@
 idf_component_register(SRC_DIRS "."
                     PRIV_INCLUDE_DIRS "${include_dirs}"
-                    PRIV_REQUIRES cmock test_utils esp_hw_support driver efuse esp_timer esp_psram)
+                    PRIV_REQUIRES cmock test_utils esp_hw_support driver efuse esp_timer esp_psram spi_flash)
 target_compile_options(${COMPONENT_LIB} PRIVATE "-Wno-format")
 
 target_link_libraries(${COMPONENT_LIB} INTERFACE "-u ld_include_test_dport_xt_highint5")

components/esp_partition/.build-test-rules.yml

@@ -0,0 +1,6 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+components/esp_partition/host_test/partition_api_test:
+  enable:
+    - if: IDF_TARGET == "linux"
+      reason: only test on linux

components/esp_partition/CMakeLists.txt

@@ -0,0 +1,30 @@
+set(srcs "partition.c")
+set(priv_reqs esp_system bootloader_support spi_flash app_update partition_table)
+set(reqs)
+set(include_dirs "include")
+
+idf_build_get_property(target IDF_TARGET)
+if(${target} STREQUAL "linux")
+    list(APPEND srcs "partition_linux.c")
+    set(priv_reqs partition_table linux)
+
+    # Steal some include directories from bootloader_support and hal components:
+    idf_component_get_property(hal_dir hal COMPONENT_DIR)
+    idf_component_get_property(bootloader_support_dir bootloader_support COMPONENT_DIR)
+    list(APPEND include_dirs include ${hal_dir}/include ${bootloader_support_dir}/include)
+else()
+    list(APPEND srcs "partition_target.c")
+endif()
+
+idf_component_register(SRCS "${srcs}"
+    INCLUDE_DIRS ${include_dirs}
+    REQUIRES ${reqs}
+    PRIV_REQUIRES ${priv_reqs})
+
+if(CMAKE_C_COMPILER_ID MATCHES "GNU")
+    # These flags are GCC specific
+    set_property(SOURCE ${cache_srcs} APPEND_STRING PROPERTY COMPILE_FLAGS
+        " -fno-inline-small-functions -fno-inline-functions-called-once")
+endif()
+
+target_compile_options(${COMPONENT_LIB} PRIVATE "-Wno-format")

components/spi_flash/host_test/partition_api_test/CMakeLists.txt → components/esp_partition/host_test/partition_api_test/CMakeLists.txt

@@ -7,3 +7,5 @@ set(COMPONENTS main)
 list(APPEND EXTRA_COMPONENT_DIRS "$ENV{IDF_PATH}/tools/mocks/freertos/")
 
 project(partition_api_test)
+
+add_dependencies(partition_api_test.elf partition-table)

components/esp_partition/host_test/partition_api_test/main/CMakeLists.txt

@@ -0,0 +1,2 @@
+idf_component_register(SRCS "partition_api_test.c"
+                       REQUIRES esp_partition unity)

components/spi_flash/host_test/partition_api_test/main/partition_api_test.cpp → components/esp_partition/host_test/partition_api_test/main/partition_api_test.c

@@ -10,97 +10,120 @@
 #include "esp_err.h"
 #include "esp_partition.h"
 #include "esp_private/partition_linux.h"
+#include "unity.h"
+#include "unity_fixture.h"
 
-int main(int argc, char **argv)
+
+TEST_GROUP(partition_api);
+
+TEST_SETUP(partition_api)
 {
-    printf("Partition API Linux emulation test: ");
+}
 
-    ////////////////////////////////////////
-    //PARTITION LOOKUP:
+TEST_TEAR_DOWN(partition_api)
+{
+}
 
-    //1. esp_partition_find (label=STORAGE)
+TEST(partition_api, test_partition_find_basic)
+{
     esp_partition_iterator_t iter = esp_partition_find(ESP_PARTITION_TYPE_DATA, ESP_PARTITION_SUBTYPE_ANY, "storage");
-    assert(iter);
+    TEST_ASSERT_NOT_NULL(iter);
 
-    //2. esp_partition_get (label=STORAGE)
     const esp_partition_t *part = esp_partition_get(iter);
-    assert(part);
+    TEST_ASSERT_NOT_NULL(part);
 
-    //3. esp_partition_iterator_release (label STORAGE iter): assumed OK
     esp_partition_iterator_release(iter);
+}
 
-    ////////////////////////////////////////
-    //ITERATORS, PARTITION PROPERTIES:
-
-    //4. esp_partition_find_first (type=APP, subtype=ANY)
-    const esp_partition_t *partition_app = esp_partition_find_first(ESP_PARTITION_TYPE_APP, ESP_PARTITION_SUBTYPE_ANY, NULL);
-    assert(partition_app);
-
-    //5. enumerate all APP partitions
-    iter = esp_partition_find(ESP_PARTITION_TYPE_APP, ESP_PARTITION_SUBTYPE_ANY, NULL);
-    assert(iter);
+TEST(partition_api, test_partition_find_app)
+{
+    esp_partition_iterator_t iter = esp_partition_find(ESP_PARTITION_TYPE_APP, ESP_PARTITION_SUBTYPE_ANY, NULL);
+    TEST_ASSERT_NOT_NULL(iter);
     size_t counter = 0;
 
     while (iter != NULL) {
         const esp_partition_t *part_data = esp_partition_get(iter);
         counter++;
-        assert(part_data);
+        TEST_ASSERT_NOT_NULL(part_data);
         iter = esp_partition_next(iter);
     }
     esp_partition_iterator_release(iter);
+}
 
-    //6. enumerate all DATA partitions and print details for each
-    iter = esp_partition_find(ESP_PARTITION_TYPE_DATA, ESP_PARTITION_SUBTYPE_ANY, NULL);
-    assert(iter);
-    counter = 0;
+TEST(partition_api, test_partition_find_data)
+{
+    esp_partition_iterator_t iter = esp_partition_find(ESP_PARTITION_TYPE_DATA, ESP_PARTITION_SUBTYPE_ANY, NULL);
+    TEST_ASSERT_NOT_NULL(iter);
+    size_t counter = 0;
 
     while (iter != NULL) {
         const esp_partition_t *part_data = esp_partition_get(iter);
         counter++;
-        assert(part_data);
+        TEST_ASSERT_NOT_NULL(part_data);
         iter = esp_partition_next(iter);
     }
     esp_partition_iterator_release(iter);
+}
+
+TEST(partition_api, test_partition_find_first)
+{
+    const esp_partition_t *partition_app = esp_partition_find_first(ESP_PARTITION_TYPE_APP, ESP_PARTITION_SUBTYPE_ANY, NULL);
+    TEST_ASSERT_NOT_NULL(partition_app);
 
-    //7. esp_partition_find_first (type=DATA, label=STORAGE)
     const esp_partition_t *partition_data = esp_partition_find_first(ESP_PARTITION_TYPE_DATA, ESP_PARTITION_SUBTYPE_ANY, "storage");
-    assert(partition_data);
+    TEST_ASSERT_NOT_NULL(partition_data);
+}
 
-    /////////////////////////////////////
-    //OPERATIONS
+TEST(partition_api, test_partition_ops)
+{
+    const esp_partition_t *partition_data = esp_partition_find_first(ESP_PARTITION_TYPE_DATA, ESP_PARTITION_SUBTYPE_ANY, "storage");
+    TEST_ASSERT_NOT_NULL(partition_data);
 
     uint8_t buff[] = "ABCDEFGHIJKLMNOP";
     size_t bufsize = sizeof(buff);
     size_t off = 0x100;
 
     //8. esp_partition_write/raw
     esp_err_t err = esp_partition_write(partition_data, off, (const void *)buff, bufsize);
-    assert(err == ESP_OK);
+    TEST_ESP_OK(err);
 
     //9. esp_partition_read/raw
     uint8_t buffout[32] = {0};
     err = esp_partition_read(partition_data, off, (void *)buffout, bufsize);
-    assert(err == ESP_OK);
+    TEST_ESP_OK(err);
 
     //10. esp_partition_erase_range
     uint8_t buferase[bufsize];
     memset(buferase, 0xFF, bufsize);
     memset(buffout, 0, sizeof(buffout));
     size_t sector_off = 0; //erase works per whole sector - offset must be aligned to 4kB boundaries
 
-    err = esp_partition_erase_range(partition_data, sector_off, SPI_FLASH_SEC_SIZE);
+    err = esp_partition_erase_range(partition_data, sector_off, partition_data->erase_size);
     assert(esp_partition_read(partition_data, off, (void *)buffout, bufsize) == ESP_OK);
-    assert(err == ESP_OK && memcmp(buffout, buferase, bufsize) == 0);
+    TEST_ESP_OK(err);
+    TEST_ASSERT_EQUAL(0, memcmp(buffout, buferase, bufsize));
 
     //11. esp_partition_verify (partition_data)
     const esp_partition_t *verified_partition = esp_partition_verify(partition_data);
-    assert(verified_partition != NULL);
+    TEST_ASSERT_NOT_NULL(verified_partition);
+}
 
-    //12. release SPI FLASH emulation block from memory
-    err = esp_partition_file_munmap();
-    assert(err == ESP_OK);
+TEST_GROUP_RUNNER(partition_api)
+{
+    RUN_TEST_CASE(partition_api, test_partition_find_basic);
+    RUN_TEST_CASE(partition_api, test_partition_find_app);
+    RUN_TEST_CASE(partition_api, test_partition_find_data);
+    RUN_TEST_CASE(partition_api, test_partition_find_first);
+    RUN_TEST_CASE(partition_api, test_partition_ops);
+}
 
-    printf("OK\n");
+static void run_all_tests(void)
+{
+    RUN_TEST_GROUP(partition_api);
+}
 
+int main(int argc, char **argv)
+{
+    UNITY_MAIN_FUNC(run_all_tests);
     return 0;
 }

components/spi_flash/host_test/partition_api_test/sdkconfig.defaults → components/esp_partition/host_test/partition_api_test/sdkconfig.defaults

@@ -1,6 +1,7 @@
 CONFIG_IDF_TARGET="linux"
 CONFIG_COMPILER_CXX_EXCEPTIONS=y
 CONFIG_UNITY_ENABLE_IDF_TEST_RUNNER=n
+CONFIG_UNITY_ENABLE_FIXTURE=y
 CONFIG_PARTITION_TABLE_CUSTOM=y
 CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partition_table.csv"
 CONFIG_ESPTOOLPY_FLASHSIZE="4MB"

components/spi_flash/include/esp_partition.h → components/esp_partition/include/esp_partition.h

@@ -11,8 +11,6 @@
 #include <stdbool.h>
 #include <stddef.h>
 #include "esp_err.h"
-#include "esp_flash.h"
-#include "spi_flash_mmap.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -23,6 +21,22 @@ extern "C" {
  * @brief Partition APIs
  */
 
+/** @cond */
+typedef struct esp_flash_t esp_flash_t;
+/** @endcond */
+
+/**
+ * @brief Enumeration which specifies memory space requested in an mmap call
+ */
+typedef enum {
+    ESP_PARTITION_MMAP_DATA,    /**< map to data memory (Vaddr0), allows byte-aligned access, 4 MB total */
+    ESP_PARTITION_MMAP_INST,    /**< map to instruction memory (Vaddr1-3), allows only 4-byte-aligned access, 11 MB total */
+} esp_partition_mmap_memory_t;
+
+/**
+ * @brief Opaque handle for memory region obtained from esp_partition_mmap.
+ */
+typedef uint32_t esp_partition_mmap_handle_t;
 
 /**
  * @brief Partition type
@@ -114,6 +128,7 @@ typedef struct {
     esp_partition_subtype_t subtype;    /*!< partition subtype */
     uint32_t address;                   /*!< starting address of the partition in flash */
     uint32_t size;                      /*!< size of the partition, in bytes */
+    uint32_t erase_size;                /*!< size the erase operation should be aligned to */
     char label[17];                     /*!< partition label, zero-terminated ASCII string */
     bool encrypted;                     /*!< flag is set to true if partition is encrypted */
 } esp_partition_t;
@@ -318,9 +333,9 @@ esp_err_t esp_partition_write_raw(const esp_partition_t* partition,
  *                  esp_partition_find_first or esp_partition_get.
  *                  Must be non-NULL.
  * @param offset Offset from the beginning of partition where erase operation
- *               should start. Must be aligned to 4 kilobytes.
+ *               should start. Must be aligned to partition->erase_size.
  * @param size Size of the range which should be erased, in bytes.
- *                   Must be divisible by 4 kilobytes.
+ *                   Must be divisible by partition->erase_size.
  *
  * @return ESP_OK, if the range was erased successfully;
  *         ESP_ERR_INVALID_ARG, if iterator or dst are NULL;
@@ -342,7 +357,7 @@ esp_err_t esp_partition_erase_range(const esp_partition_t* partition,
  * requested offset (not necessarily to the beginning of mmap-ed region).
  *
  * To release mapped memory, pass handle returned via out_handle argument to
- * spi_flash_munmap function.
+ * esp_partition_munmap function.
  *
  * @param partition Pointer to partition structure obtained using
  *                  esp_partition_find_first or esp_partition_get.
@@ -351,13 +366,25 @@ esp_err_t esp_partition_erase_range(const esp_partition_t* partition,
  * @param size Size of the area to be mapped.
  * @param memory  Memory space where the region should be mapped
  * @param out_ptr  Output, pointer to the mapped memory region
- * @param out_handle  Output, handle which should be used for spi_flash_munmap call
+ * @param out_handle  Output, handle which should be used for esp_partition_munmap call
  *
  * @return ESP_OK, if successful
  */
 esp_err_t esp_partition_mmap(const esp_partition_t* partition, size_t offset, size_t size,
-                             spi_flash_mmap_memory_t memory,
-                             const void** out_ptr, spi_flash_mmap_handle_t* out_handle);
+                             esp_partition_mmap_memory_t memory,
+                             const void** out_ptr, esp_partition_mmap_handle_t* out_handle);
+
+/**
+ * @brief Release region previously obtained using esp_partition_mmap
+ *
+ * @note Calling this function will not necessarily unmap memory region.
+ *       Region will only be unmapped when there are no other handles which
+ *       reference this region. In case of partially overlapping regions
+ *       it is possible that memory will be unmapped partially.
+ *
+ * @param handle  Handle obtained from spi_flash_mmap
+ */
+void esp_partition_munmap(esp_partition_mmap_handle_t handle);
 
 /**
  * @brief Get SHA-256 digest for required partition.

components/spi_flash/include/esp_private/partition_linux.h → components/esp_partition/include/esp_private/partition_linux.h

@@ -19,6 +19,9 @@ extern "C" {
  * @brief Private API functions used for Linux-target emulation of the Partition APIs (host-side testing)
  */
 
+/** @brief emulated sector size for the partition API on Linux */
+#define ESP_PARTITION_EMULATED_SECTOR_SIZE 0x1000
+
 /**
  * @brief Partition type to string conversion routine
  *