From 076141aab9eab081d6d06cde42695bb095a57607 Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Thu, 22 Sep 2016 11:42:55 +0800 Subject: [PATCH] components/nvs: batch writes when possible Introduces new internal function, Page::alterEntryRangeState, which gathers changes to multiple elements of entry state table into a single write, provided that these changes fall into a single word. This allows changing state of up to 16 entries in a single write. Also adds new function, writeEntryData, which writes the whole payload of SZ and BLOB type entries in one go, instead of splitting it into multiple 32-byte writes. This reduces number of writes required for SZ and BLOB entries. --- components/nvs_flash/src/nvs_page.cpp | 99 +++++++++++++---- components/nvs_flash/src/nvs_page.hpp | 4 + components/nvs_flash/src/nvs_storage.cpp | 3 + components/nvs_flash/test/test_nvs.cpp | 132 ++++++++++++++++++++++- 4 files changed, 210 insertions(+), 28 deletions(-) diff --git a/components/nvs_flash/src/nvs_page.cpp b/components/nvs_flash/src/nvs_page.cpp index ae067078c6..c20a23677a 100644 --- a/components/nvs_flash/src/nvs_page.cpp +++ b/components/nvs_flash/src/nvs_page.cpp @@ -108,6 +108,27 @@ esp_err_t Page::writeEntry(const Item& item) return ESP_OK; } + +esp_err_t Page::writeEntryData(const uint8_t* data, size_t size) +{ + assert(size % ENTRY_SIZE == 0); + assert(mNextFreeEntry != INVALID_ENTRY); + assert(mFirstUsedEntry != INVALID_ENTRY); + const uint16_t count = size / ENTRY_SIZE; + + auto rc = spi_flash_write(getEntryAddress(mNextFreeEntry), reinterpret_cast(data), static_cast(size)); + if (rc != ESP_OK) { + mState = PageState::INVALID; + return rc; + } + auto err = alterEntryRangeState(mNextFreeEntry, mNextFreeEntry + count, EntryState::WRITTEN); + if (err != ESP_OK) { + return err; + } + mUsedEntryCount += count; + mNextFreeEntry += count; + return ESP_OK; +} esp_err_t Page::writeItem(uint8_t nsIndex, ItemType datatype, const char* key, const void* data, size_t dataSize) { @@ -170,13 +191,18 @@ esp_err_t Page::writeItem(uint8_t nsIndex, ItemType datatype, const char* key, c return err; } - size_t left = dataSize; - while (left != 0) { - size_t willWrite = Page::ENTRY_SIZE; - willWrite = (left < willWrite)?left:willWrite; - memcpy(item.rawData, src, willWrite); - src += willWrite; - left -= willWrite; + size_t left = dataSize / ENTRY_SIZE * ENTRY_SIZE; + if (left > 0) { + err = writeEntryData(static_cast(data), left); + if (err != ESP_OK) { + return err; + } + } + + size_t tail = dataSize - left; + if (tail > 0) { + std::fill_n(item.rawData, ENTRY_SIZE / 4, 0xffffffff); + memcpy(item.rawData, static_cast(data) + left, tail); err = writeEntry(item); if (err != ESP_OK) { return err; @@ -290,12 +316,16 @@ esp_err_t Page::eraseEntryAndSpan(size_t index) if (mEntryTable.get(i) == EntryState::WRITTEN) { --mUsedEntryCount; } - rc = alterEntryState(i, EntryState::ERASED); - if (rc != ESP_OK) { - return rc; - } ++mErasedEntryCount; } + if (span == 1) { + rc = alterEntryState(index, EntryState::ERASED); + } else { + rc = alterEntryRangeState(index, index + span, EntryState::ERASED); + } + if (rc != ESP_OK) { + return rc; + } } } else { @@ -372,17 +402,7 @@ esp_err_t Page::moveItem(Page& other) return err; } } - for (size_t i = mFirstUsedEntry; i < end; ++i) { - err = eraseEntry(i); - if (err != ESP_OK) { - return err; - } - } - updateFirstUsedEntry(mFirstUsedEntry, span); - mErasedEntryCount += span; - mUsedEntryCount -= span; - - return ESP_OK; + return eraseEntryAndSpan(mFirstUsedEntry); } esp_err_t Page::mLoadEntryTable() @@ -427,7 +447,7 @@ esp_err_t Page::mLoadEntryTable() // but before the entry state table was altered, the entry locacted via // entry state table may actually be half-written. // this is easy to check by reading EntryHeader (i.e. first word) - if (mNextFreeEntry != INVALID_ENTRY) { + while (mNextFreeEntry < ENTRY_COUNT) { uint32_t entryAddress = getEntryAddress(mNextFreeEntry); uint32_t header; auto rc = spi_flash_read(entryAddress, &header, sizeof(header)); @@ -436,12 +456,20 @@ esp_err_t Page::mLoadEntryTable() return rc; } if (header != 0xffffffff) { + auto oldState = mEntryTable.get(mNextFreeEntry); auto err = alterEntryState(mNextFreeEntry, EntryState::ERASED); if (err != ESP_OK) { mState = PageState::INVALID; return err; } ++mNextFreeEntry; + if (oldState == EntryState::WRITTEN) { + --mUsedEntryCount; + } + ++mErasedEntryCount; + } + else { + break; } } @@ -573,6 +601,31 @@ esp_err_t Page::alterEntryState(size_t index, EntryState state) return ESP_OK; } +esp_err_t Page::alterEntryRangeState(size_t begin, size_t end, EntryState state) +{ + assert(end <= ENTRY_COUNT); + assert(end > begin); + size_t wordIndex = mEntryTable.getWordIndex(end - 1); + for (ptrdiff_t i = end - 1; i >= static_cast(begin); --i) { + mEntryTable.set(i, state); + size_t nextWordIndex; + if (i == static_cast(begin)) { + nextWordIndex = (size_t) -1; + } else { + nextWordIndex = mEntryTable.getWordIndex(i - 1); + } + if (nextWordIndex != wordIndex) { + uint32_t word = mEntryTable.data()[wordIndex]; + auto rc = spi_flash_write(mBaseAddress + ENTRY_TABLE_OFFSET + static_cast(wordIndex) * 4, &word, 4); + if (rc != ESP_OK) { + return rc; + } + } + wordIndex = nextWordIndex; + } + return ESP_OK; +} + esp_err_t Page::alterPageState(PageState state) { auto rc = spi_flash_write(mBaseAddress, reinterpret_cast(&state), sizeof(state)); diff --git a/components/nvs_flash/src/nvs_page.hpp b/components/nvs_flash/src/nvs_page.hpp index 9598263539..d6fb68f760 100644 --- a/components/nvs_flash/src/nvs_page.hpp +++ b/components/nvs_flash/src/nvs_page.hpp @@ -194,11 +194,15 @@ protected: esp_err_t alterEntryState(size_t index, EntryState state); + esp_err_t alterEntryRangeState(size_t begin, size_t end, EntryState state); + esp_err_t alterPageState(PageState state); esp_err_t readEntry(size_t index, Item& dst) const; esp_err_t writeEntry(const Item& item); + + esp_err_t writeEntryData(const uint8_t* data, size_t size); esp_err_t eraseEntry(size_t index); diff --git a/components/nvs_flash/src/nvs_storage.cpp b/components/nvs_flash/src/nvs_storage.cpp index 9be6580a1d..e60b1df551 100644 --- a/components/nvs_flash/src/nvs_storage.cpp +++ b/components/nvs_flash/src/nvs_storage.cpp @@ -234,6 +234,7 @@ void Storage::debugCheck() for (auto p = mPageManager.begin(); p != mPageManager.end(); ++p) { size_t itemIndex = 0; + size_t usedCount = 0; Item item; while (p->findItem(Page::NS_ANY, ItemType::ANY, nullptr, itemIndex, item) == ESP_OK) { std::stringstream keyrepr; @@ -246,7 +247,9 @@ void Storage::debugCheck() } keys.insert(std::make_pair(keystr, static_cast(p))); itemIndex += item.span; + usedCount += item.span; } + assert(usedCount == p->getUsedEntryCount()); } } #endif //ESP_PLATFORM diff --git a/components/nvs_flash/test/test_nvs.cpp b/components/nvs_flash/test/test_nvs.cpp index 325cdcf45b..c496a09fd4 100644 --- a/components/nvs_flash/test/test_nvs.cpp +++ b/components/nvs_flash/test/test_nvs.cpp @@ -443,18 +443,140 @@ TEST_CASE("wifi test", "[nvs]") SpiFlashEmulator emu(10); emu.randomize(10); - nvs_handle handle; + const uint32_t NVS_FLASH_SECTOR = 5; const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 3; emu.setBounds(NVS_FLASH_SECTOR, NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN); TEST_ESP_OK(nvs_flash_init(NVS_FLASH_SECTOR, NVS_FLASH_SECTOR_COUNT_MIN)); - TEST_ESP_OK(nvs_open("nvs.net80211", NVS_READWRITE, &handle)); + nvs_handle misc_handle; + TEST_ESP_OK(nvs_open("nvs.net80211", NVS_READWRITE, &misc_handle)); + char log[33]; + size_t log_size = sizeof(log); + TEST_ESP_ERR(nvs_get_str(misc_handle, "log", log, &log_size), ESP_ERR_NVS_NOT_FOUND); + strcpy(log, "foobarbazfizzz"); + TEST_ESP_OK(nvs_set_str(misc_handle, "log", log)); + + nvs_handle net80211_handle; + TEST_ESP_OK(nvs_open("nvs.net80211", NVS_READWRITE, &net80211_handle)); uint8_t opmode = 2; - if (nvs_get_u8(handle, "wifi.opmode", &opmode) != ESP_OK) { - TEST_ESP_OK(nvs_set_u8(handle, "wifi.opmode", opmode)); - } + TEST_ESP_ERR(nvs_get_u8(net80211_handle, "wifi.opmode", &opmode), ESP_ERR_NVS_NOT_FOUND); + + TEST_ESP_OK(nvs_set_u8(net80211_handle, "wifi.opmode", opmode)); + + uint8_t country = 0; + TEST_ESP_ERR(nvs_get_u8(net80211_handle, "wifi.country", &opmode), ESP_ERR_NVS_NOT_FOUND); + TEST_ESP_OK(nvs_set_u8(net80211_handle, "wifi.country", opmode)); + + char ssid[36]; + size_t size = sizeof(ssid); + TEST_ESP_ERR(nvs_get_blob(net80211_handle, "sta.ssid", ssid, &size), ESP_ERR_NVS_NOT_FOUND); + strcpy(ssid, "my android AP"); + TEST_ESP_OK(nvs_set_blob(net80211_handle, "sta.ssid", ssid, size)); + + char mac[6]; + size = sizeof(mac); + TEST_ESP_ERR(nvs_get_blob(net80211_handle, "sta.mac", mac, &size), ESP_ERR_NVS_NOT_FOUND); + memset(mac, 0xab, 6); + TEST_ESP_OK(nvs_set_blob(net80211_handle, "sta.mac", mac, size)); + + uint8_t authmode = 1; + TEST_ESP_ERR(nvs_get_u8(net80211_handle, "sta.authmode", &authmode), ESP_ERR_NVS_NOT_FOUND); + TEST_ESP_OK(nvs_set_u8(net80211_handle, "sta.authmode", authmode)); + + char pswd[65]; + size = sizeof(pswd); + TEST_ESP_ERR(nvs_get_blob(net80211_handle, "sta.pswd", pswd, &size), ESP_ERR_NVS_NOT_FOUND); + strcpy(pswd, "`123456788990-="); + TEST_ESP_OK(nvs_set_blob(net80211_handle, "sta.pswd", pswd, size)); + + char pmk[32]; + size = sizeof(pmk); + TEST_ESP_ERR(nvs_get_blob(net80211_handle, "sta.pmk", pmk, &size), ESP_ERR_NVS_NOT_FOUND); + memset(pmk, 1, size); + TEST_ESP_OK(nvs_set_blob(net80211_handle, "sta.pmk", pmk, size)); + + uint8_t chan = 1; + TEST_ESP_ERR(nvs_get_u8(net80211_handle, "sta.chan", &chan), ESP_ERR_NVS_NOT_FOUND); + TEST_ESP_OK(nvs_set_u8(net80211_handle, "sta.chan", chan)); + + uint8_t autoconn = 1; + TEST_ESP_ERR(nvs_get_u8(net80211_handle, "auto.conn", &autoconn), ESP_ERR_NVS_NOT_FOUND); + TEST_ESP_OK(nvs_set_u8(net80211_handle, "auto.conn", autoconn)); + + uint8_t bssid_set = 1; + TEST_ESP_ERR(nvs_get_u8(net80211_handle, "bssid.set", &bssid_set), ESP_ERR_NVS_NOT_FOUND); + TEST_ESP_OK(nvs_set_u8(net80211_handle, "bssid.set", bssid_set)); + + char bssid[6]; + size = sizeof(bssid); + TEST_ESP_ERR(nvs_get_blob(net80211_handle, "sta.bssid", bssid, &size), ESP_ERR_NVS_NOT_FOUND); + memset(mac, 0xcd, 6); + TEST_ESP_OK(nvs_set_blob(net80211_handle, "sta.bssid", bssid, size)); + + uint8_t phym = 3; + TEST_ESP_ERR(nvs_get_u8(net80211_handle, "sta.phym", &phym), ESP_ERR_NVS_NOT_FOUND); + TEST_ESP_OK(nvs_set_u8(net80211_handle, "sta.phym", phym)); + + uint8_t phybw = 2; + TEST_ESP_ERR(nvs_get_u8(net80211_handle, "sta.phybw", &phybw), ESP_ERR_NVS_NOT_FOUND); + TEST_ESP_OK(nvs_set_u8(net80211_handle, "sta.phybw", phybw)); + + char apsw[2]; + size = sizeof(apsw); + TEST_ESP_ERR(nvs_get_blob(net80211_handle, "sta.apsw", apsw, &size), ESP_ERR_NVS_NOT_FOUND); + memset(apsw, 0x2, size); + TEST_ESP_OK(nvs_set_blob(net80211_handle, "sta.apsw", apsw, size)); + + char apinfo[700]; + size = sizeof(apinfo); + TEST_ESP_ERR(nvs_get_blob(net80211_handle, "sta.apinfo", apinfo, &size), ESP_ERR_NVS_NOT_FOUND); + memset(apinfo, 0, size); + TEST_ESP_OK(nvs_set_blob(net80211_handle, "sta.apinfo", apinfo, size)); + + size = sizeof(ssid); + TEST_ESP_ERR(nvs_get_blob(net80211_handle, "ap.ssid", ssid, &size), ESP_ERR_NVS_NOT_FOUND); + strcpy(ssid, "ESP_A2F340"); + TEST_ESP_OK(nvs_set_blob(net80211_handle, "ap.ssid", ssid, size)); + + size = sizeof(mac); + TEST_ESP_ERR(nvs_get_blob(net80211_handle, "ap.mac", mac, &size), ESP_ERR_NVS_NOT_FOUND); + memset(mac, 0xac, 6); + TEST_ESP_OK(nvs_set_blob(net80211_handle, "ap.mac", mac, size)); + + size = sizeof(pswd); + TEST_ESP_ERR(nvs_get_blob(net80211_handle, "ap.passwd", pswd, &size), ESP_ERR_NVS_NOT_FOUND); + strcpy(pswd, ""); + TEST_ESP_OK(nvs_set_blob(net80211_handle, "ap.passwd", pswd, size)); + + size = sizeof(pmk); + TEST_ESP_ERR(nvs_get_blob(net80211_handle, "ap.pmk", pmk, &size), ESP_ERR_NVS_NOT_FOUND); + memset(pmk, 1, size); + TEST_ESP_OK(nvs_set_blob(net80211_handle, "ap.pmk", pmk, size)); + + chan = 6; + TEST_ESP_ERR(nvs_get_u8(net80211_handle, "ap.chan", &chan), ESP_ERR_NVS_NOT_FOUND); + TEST_ESP_OK(nvs_set_u8(net80211_handle, "ap.chan", chan)); + + authmode = 0; + TEST_ESP_ERR(nvs_get_u8(net80211_handle, "ap.authmode", &authmode), ESP_ERR_NVS_NOT_FOUND); + TEST_ESP_OK(nvs_set_u8(net80211_handle, "ap.authmode", authmode)); + + uint8_t hidden = 0; + TEST_ESP_ERR(nvs_get_u8(net80211_handle, "ap.hidden", &hidden), ESP_ERR_NVS_NOT_FOUND); + TEST_ESP_OK(nvs_set_u8(net80211_handle, "ap.hidden", hidden)); + + uint8_t max_conn = 4; + TEST_ESP_ERR(nvs_get_u8(net80211_handle, "ap.max.conn", &max_conn), ESP_ERR_NVS_NOT_FOUND); + TEST_ESP_OK(nvs_set_u8(net80211_handle, "ap.max.conn", max_conn)); + + uint8_t bcn_interval = 2; + TEST_ESP_ERR(nvs_get_u8(net80211_handle, "bcn_interval", &bcn_interval), ESP_ERR_NVS_NOT_FOUND); + TEST_ESP_OK(nvs_set_u8(net80211_handle, "bcn_interval", bcn_interval)); + + s_perf << "Time to simulate nvs init with wifi libs: " << emu.getTotalTime() << " us (" << emu.getEraseOps() << "E " << emu.getWriteOps() << "W " << emu.getReadOps() << "R " << emu.getWriteBytes() << "Wb " << emu.getReadBytes() << "Rb)" << std::endl; + }