components/nvs: batch writes when possible

Introduces new internal function, Page::alterEntryRangeState, which gathers changes to multiple elements of entry state table into a single write, provided that these changes fall into a single word. This allows changing state of up to 16 entries in a single write.
Also adds new function, writeEntryData, which writes the whole payload of SZ and BLOB type entries in one go, instead of splitting it into multiple 32-byte writes.
This reduces number of writes required for SZ and BLOB entries.

components/nvs_flash/src/nvs_page.cpp

@@ -108,6 +108,27 @@ esp_err_t Page::writeEntry(const Item& item)
 
     return ESP_OK;
 }
+    
+esp_err_t Page::writeEntryData(const uint8_t* data, size_t size)
+{
+    assert(size % ENTRY_SIZE == 0);
+    assert(mNextFreeEntry != INVALID_ENTRY);
+    assert(mFirstUsedEntry != INVALID_ENTRY);
+    const uint16_t count = size / ENTRY_SIZE;
+    
+    auto rc = spi_flash_write(getEntryAddress(mNextFreeEntry), reinterpret_cast<const uint32_t*>(data), static_cast<uint32_t>(size));
+    if (rc != ESP_OK) {
+        mState = PageState::INVALID;
+        return rc;
+    }
+    auto err = alterEntryRangeState(mNextFreeEntry, mNextFreeEntry + count, EntryState::WRITTEN);
+    if (err != ESP_OK) {
+        return err;
+    }
+    mUsedEntryCount += count;
+    mNextFreeEntry += count;
+    return ESP_OK;
+}
 
 esp_err_t Page::writeItem(uint8_t nsIndex, ItemType datatype, const char* key, const void* data, size_t dataSize)
 {
@@ -170,13 +191,18 @@ esp_err_t Page::writeItem(uint8_t nsIndex, ItemType datatype, const char* key, c
             return err;
         }
 
-        size_t left = dataSize;
-        while (left != 0) {
-            size_t willWrite = Page::ENTRY_SIZE;
-            willWrite = (left < willWrite)?left:willWrite;
-            memcpy(item.rawData, src, willWrite);
-            src += willWrite;
-            left -= willWrite;
+        size_t left = dataSize / ENTRY_SIZE * ENTRY_SIZE;
+        if (left > 0) {
+            err = writeEntryData(static_cast<const uint8_t*>(data), left);
+            if (err != ESP_OK) {
+                return err;
+            }
+        }
+        
+        size_t tail = dataSize - left;
+        if (tail > 0) {
+            std::fill_n(item.rawData, ENTRY_SIZE / 4, 0xffffffff);
+            memcpy(item.rawData, static_cast<const uint8_t*>(data) + left, tail);
             err = writeEntry(item);
             if (err != ESP_OK) {
                 return err;
@@ -290,12 +316,16 @@ esp_err_t Page::eraseEntryAndSpan(size_t index)
                 if (mEntryTable.get(i) == EntryState::WRITTEN) {
                     --mUsedEntryCount;
                 }
-                rc = alterEntryState(i, EntryState::ERASED);
-                if (rc != ESP_OK) {
-                    return rc;
-                }
                 ++mErasedEntryCount;
             }
+            if (span == 1) {
+                rc = alterEntryState(index, EntryState::ERASED);
+            } else {
+                rc = alterEntryRangeState(index, index + span, EntryState::ERASED);
+            }
+            if (rc != ESP_OK) {
+                return rc;
+            }
         }
     }
     else {
@@ -372,17 +402,7 @@ esp_err_t Page::moveItem(Page& other)
             return err;
         }
     }
-    for (size_t i = mFirstUsedEntry; i < end; ++i) {
-        err = eraseEntry(i);
-        if (err != ESP_OK) {
-            return err;
-        }
-    }
-    updateFirstUsedEntry(mFirstUsedEntry, span);
-    mErasedEntryCount += span;
-    mUsedEntryCount -= span;
-    
-    return ESP_OK;
+    return eraseEntryAndSpan(mFirstUsedEntry);
 }
 
 esp_err_t Page::mLoadEntryTable()
@@ -427,7 +447,7 @@ esp_err_t Page::mLoadEntryTable()
         // but before the entry state table was altered, the entry locacted via
         // entry state table may actually be half-written.
         // this is easy to check by reading EntryHeader (i.e. first word)
-        if (mNextFreeEntry != INVALID_ENTRY) {
+        while (mNextFreeEntry < ENTRY_COUNT) {
             uint32_t entryAddress = getEntryAddress(mNextFreeEntry);
             uint32_t header;
             auto rc = spi_flash_read(entryAddress, &header, sizeof(header));
@@ -436,12 +456,20 @@ esp_err_t Page::mLoadEntryTable()
                 return rc;
             }
             if (header != 0xffffffff) {
+                auto oldState = mEntryTable.get(mNextFreeEntry);
                 auto err = alterEntryState(mNextFreeEntry, EntryState::ERASED);
                 if (err != ESP_OK) {
                     mState = PageState::INVALID;
                     return err;
                 }
                 ++mNextFreeEntry;
+                if (oldState == EntryState::WRITTEN) {
+                    --mUsedEntryCount;
+                }
+                ++mErasedEntryCount;
+            }
+            else {
+                break;
             }
         }
 
@@ -573,6 +601,31 @@ esp_err_t Page::alterEntryState(size_t index, EntryState state)
     return ESP_OK;
 }
 
+esp_err_t Page::alterEntryRangeState(size_t begin, size_t end, EntryState state)
+{
+    assert(end <= ENTRY_COUNT);
+    assert(end > begin);
+    size_t wordIndex = mEntryTable.getWordIndex(end - 1);
+    for (ptrdiff_t i = end - 1; i >= static_cast<ptrdiff_t>(begin); --i) {
+        mEntryTable.set(i, state);
+        size_t nextWordIndex;
+        if (i == static_cast<ptrdiff_t>(begin)) {
+            nextWordIndex = (size_t) -1;
+        } else {
+            nextWordIndex = mEntryTable.getWordIndex(i - 1);
+        }
+        if (nextWordIndex != wordIndex) {
+            uint32_t word = mEntryTable.data()[wordIndex];
+            auto rc = spi_flash_write(mBaseAddress + ENTRY_TABLE_OFFSET + static_cast<uint32_t>(wordIndex) * 4, &word, 4);
+            if (rc != ESP_OK) {
+                return rc;
+            }
+        }
+        wordIndex = nextWordIndex;
+    }
+    return ESP_OK;
+}
+
 esp_err_t Page::alterPageState(PageState state)
 {
     auto rc = spi_flash_write(mBaseAddress, reinterpret_cast<uint32_t*>(&state), sizeof(state));

components/nvs_flash/src/nvs_page.hpp

@@ -194,11 +194,15 @@ protected:
 
     esp_err_t alterEntryState(size_t index, EntryState state);
 
+    esp_err_t alterEntryRangeState(size_t begin, size_t end, EntryState state);
+
     esp_err_t alterPageState(PageState state);
 
     esp_err_t readEntry(size_t index, Item& dst) const;
 
     esp_err_t writeEntry(const Item& item);
+    
+    esp_err_t writeEntryData(const uint8_t* data, size_t size);
 
     esp_err_t eraseEntry(size_t index);
     

components/nvs_flash/src/nvs_storage.cpp

@@ -234,6 +234,7 @@ void Storage::debugCheck()
     
     for (auto p = mPageManager.begin(); p != mPageManager.end(); ++p) {
         size_t itemIndex = 0;
+        size_t usedCount = 0;
         Item item;
         while (p->findItem(Page::NS_ANY, ItemType::ANY, nullptr, itemIndex, item) == ESP_OK) {
             std::stringstream keyrepr;
@@ -246,7 +247,9 @@ void Storage::debugCheck()
             }
             keys.insert(std::make_pair(keystr, static_cast<Page*>(p)));
             itemIndex += item.span;
+            usedCount += item.span;
         }
+        assert(usedCount == p->getUsedEntryCount());
     }
 }
 #endif //ESP_PLATFORM

components/nvs_flash/test/test_nvs.cpp

@@ -443,18 +443,140 @@ TEST_CASE("wifi test", "[nvs]")
     SpiFlashEmulator emu(10);
     emu.randomize(10);
     
-    nvs_handle handle;
+    
     const uint32_t NVS_FLASH_SECTOR = 5;
     const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 3;
     emu.setBounds(NVS_FLASH_SECTOR, NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN);
     TEST_ESP_OK(nvs_flash_init(NVS_FLASH_SECTOR, NVS_FLASH_SECTOR_COUNT_MIN));
     
-    TEST_ESP_OK(nvs_open("nvs.net80211", NVS_READWRITE, &handle));
+    nvs_handle misc_handle;
+    TEST_ESP_OK(nvs_open("nvs.net80211", NVS_READWRITE, &misc_handle));
+    char log[33];
+    size_t log_size = sizeof(log);
+    TEST_ESP_ERR(nvs_get_str(misc_handle, "log", log, &log_size), ESP_ERR_NVS_NOT_FOUND);
+    strcpy(log, "foobarbazfizzz");
+    TEST_ESP_OK(nvs_set_str(misc_handle, "log", log));
+    
+    nvs_handle net80211_handle;
+    TEST_ESP_OK(nvs_open("nvs.net80211", NVS_READWRITE, &net80211_handle));
     
     uint8_t opmode = 2;
-    if (nvs_get_u8(handle, "wifi.opmode", &opmode) != ESP_OK) {
-        TEST_ESP_OK(nvs_set_u8(handle, "wifi.opmode", opmode));
-    }
+    TEST_ESP_ERR(nvs_get_u8(net80211_handle, "wifi.opmode", &opmode), ESP_ERR_NVS_NOT_FOUND);
+    
+    TEST_ESP_OK(nvs_set_u8(net80211_handle, "wifi.opmode", opmode));
+    
+    uint8_t country = 0;
+    TEST_ESP_ERR(nvs_get_u8(net80211_handle, "wifi.country", &opmode), ESP_ERR_NVS_NOT_FOUND);
+    TEST_ESP_OK(nvs_set_u8(net80211_handle, "wifi.country", opmode));
+    
+    char ssid[36];
+    size_t size = sizeof(ssid);
+    TEST_ESP_ERR(nvs_get_blob(net80211_handle, "sta.ssid", ssid, &size), ESP_ERR_NVS_NOT_FOUND);
+    strcpy(ssid, "my android AP");
+    TEST_ESP_OK(nvs_set_blob(net80211_handle, "sta.ssid", ssid, size));
+    
+    char mac[6];
+    size = sizeof(mac);
+    TEST_ESP_ERR(nvs_get_blob(net80211_handle, "sta.mac", mac, &size), ESP_ERR_NVS_NOT_FOUND);
+    memset(mac, 0xab, 6);
+    TEST_ESP_OK(nvs_set_blob(net80211_handle, "sta.mac", mac, size));
+    
+    uint8_t authmode = 1;
+    TEST_ESP_ERR(nvs_get_u8(net80211_handle, "sta.authmode", &authmode), ESP_ERR_NVS_NOT_FOUND);
+    TEST_ESP_OK(nvs_set_u8(net80211_handle, "sta.authmode", authmode));
+
+    char pswd[65];
+    size = sizeof(pswd);
+    TEST_ESP_ERR(nvs_get_blob(net80211_handle, "sta.pswd", pswd, &size), ESP_ERR_NVS_NOT_FOUND);
+    strcpy(pswd, "`123456788990-=");
+    TEST_ESP_OK(nvs_set_blob(net80211_handle, "sta.pswd", pswd, size));
+
+    char pmk[32];
+    size = sizeof(pmk);
+    TEST_ESP_ERR(nvs_get_blob(net80211_handle, "sta.pmk", pmk, &size), ESP_ERR_NVS_NOT_FOUND);
+    memset(pmk, 1, size);
+    TEST_ESP_OK(nvs_set_blob(net80211_handle, "sta.pmk", pmk, size));
+    
+    uint8_t chan = 1;
+    TEST_ESP_ERR(nvs_get_u8(net80211_handle, "sta.chan", &chan), ESP_ERR_NVS_NOT_FOUND);
+    TEST_ESP_OK(nvs_set_u8(net80211_handle, "sta.chan", chan));
+    
+    uint8_t autoconn = 1;
+    TEST_ESP_ERR(nvs_get_u8(net80211_handle, "auto.conn", &autoconn), ESP_ERR_NVS_NOT_FOUND);
+    TEST_ESP_OK(nvs_set_u8(net80211_handle, "auto.conn", autoconn));
+
+    uint8_t bssid_set = 1;
+    TEST_ESP_ERR(nvs_get_u8(net80211_handle, "bssid.set", &bssid_set), ESP_ERR_NVS_NOT_FOUND);
+    TEST_ESP_OK(nvs_set_u8(net80211_handle, "bssid.set", bssid_set));
+
+    char bssid[6];
+    size = sizeof(bssid);
+    TEST_ESP_ERR(nvs_get_blob(net80211_handle, "sta.bssid", bssid, &size), ESP_ERR_NVS_NOT_FOUND);
+    memset(mac, 0xcd, 6);
+    TEST_ESP_OK(nvs_set_blob(net80211_handle, "sta.bssid", bssid, size));
+
+    uint8_t phym = 3;
+    TEST_ESP_ERR(nvs_get_u8(net80211_handle, "sta.phym", &phym), ESP_ERR_NVS_NOT_FOUND);
+    TEST_ESP_OK(nvs_set_u8(net80211_handle, "sta.phym", phym));
+    
+    uint8_t phybw = 2;
+    TEST_ESP_ERR(nvs_get_u8(net80211_handle, "sta.phybw", &phybw), ESP_ERR_NVS_NOT_FOUND);
+    TEST_ESP_OK(nvs_set_u8(net80211_handle, "sta.phybw", phybw));
+    
+    char apsw[2];
+    size = sizeof(apsw);
+    TEST_ESP_ERR(nvs_get_blob(net80211_handle, "sta.apsw", apsw, &size), ESP_ERR_NVS_NOT_FOUND);
+    memset(apsw, 0x2, size);
+    TEST_ESP_OK(nvs_set_blob(net80211_handle, "sta.apsw", apsw, size));
+    
+    char apinfo[700];
+    size = sizeof(apinfo);
+    TEST_ESP_ERR(nvs_get_blob(net80211_handle, "sta.apinfo", apinfo, &size), ESP_ERR_NVS_NOT_FOUND);
+    memset(apinfo, 0, size);
+    TEST_ESP_OK(nvs_set_blob(net80211_handle, "sta.apinfo", apinfo, size));
+    
+    size = sizeof(ssid);
+    TEST_ESP_ERR(nvs_get_blob(net80211_handle, "ap.ssid", ssid, &size), ESP_ERR_NVS_NOT_FOUND);
+    strcpy(ssid, "ESP_A2F340");
+    TEST_ESP_OK(nvs_set_blob(net80211_handle, "ap.ssid", ssid, size));
+    
+    size = sizeof(mac);
+    TEST_ESP_ERR(nvs_get_blob(net80211_handle, "ap.mac", mac, &size), ESP_ERR_NVS_NOT_FOUND);
+    memset(mac, 0xac, 6);
+    TEST_ESP_OK(nvs_set_blob(net80211_handle, "ap.mac", mac, size));
+    
+    size = sizeof(pswd);
+    TEST_ESP_ERR(nvs_get_blob(net80211_handle, "ap.passwd", pswd, &size), ESP_ERR_NVS_NOT_FOUND);
+    strcpy(pswd, "");
+    TEST_ESP_OK(nvs_set_blob(net80211_handle, "ap.passwd", pswd, size));
+    
+    size = sizeof(pmk);
+    TEST_ESP_ERR(nvs_get_blob(net80211_handle, "ap.pmk", pmk, &size), ESP_ERR_NVS_NOT_FOUND);
+    memset(pmk, 1, size);
+    TEST_ESP_OK(nvs_set_blob(net80211_handle, "ap.pmk", pmk, size));
+    
+    chan = 6;
+    TEST_ESP_ERR(nvs_get_u8(net80211_handle, "ap.chan", &chan), ESP_ERR_NVS_NOT_FOUND);
+    TEST_ESP_OK(nvs_set_u8(net80211_handle, "ap.chan", chan));
+
+    authmode = 0;
+    TEST_ESP_ERR(nvs_get_u8(net80211_handle, "ap.authmode", &authmode), ESP_ERR_NVS_NOT_FOUND);
+    TEST_ESP_OK(nvs_set_u8(net80211_handle, "ap.authmode", authmode));
+    
+    uint8_t hidden = 0;
+    TEST_ESP_ERR(nvs_get_u8(net80211_handle, "ap.hidden", &hidden), ESP_ERR_NVS_NOT_FOUND);
+    TEST_ESP_OK(nvs_set_u8(net80211_handle, "ap.hidden", hidden));
+    
+    uint8_t max_conn = 4;
+    TEST_ESP_ERR(nvs_get_u8(net80211_handle, "ap.max.conn", &max_conn), ESP_ERR_NVS_NOT_FOUND);
+    TEST_ESP_OK(nvs_set_u8(net80211_handle, "ap.max.conn", max_conn));
+    
+    uint8_t bcn_interval = 2;
+    TEST_ESP_ERR(nvs_get_u8(net80211_handle, "bcn_interval", &bcn_interval), ESP_ERR_NVS_NOT_FOUND);
+    TEST_ESP_OK(nvs_set_u8(net80211_handle, "bcn_interval", bcn_interval));
+    
+    s_perf << "Time to simulate nvs init with wifi libs: " << emu.getTotalTime() << " us (" << emu.getEraseOps() << "E " << emu.getWriteOps() << "W " << emu.getReadOps() << "R " << emu.getWriteBytes() << "Wb " << emu.getReadBytes() << "Rb)" << std::endl;
+
 }