support dual filsystems on nrf52

store identity and prefs in UserData and contacts, channels and adv_blobs in ExtraData

examples/companion_radio/DataStore.cpp

@@ -18,14 +18,28 @@ DataStore::DataStore(FILESYSTEM& fs, mesh::RTCClock& clock) : _fs(&fs), _clock(&
 {
 }
 
-static File openWrite(FILESYSTEM* _fs, const char* filename) {
+#if defined(EXTRAFS) || defined(QSPIFLASH)
+
+DataStore::DataStore(FILESYSTEM& fs, FILESYSTEM& fsExtra, mesh::RTCClock& clock) : _fs(&fs), _fsExtra(&fsExtra), _clock(&clock),
 #if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM)
-  _fs->remove(filename);
+    identity_store(fs, "")
-  return _fs->open(filename, FILE_O_WRITE);
 #elif defined(RP2040_PLATFORM)
-  return _fs->open(filename, "w");
+    identity_store(fs, "/identity")
 #else
-  return _fs->open(filename, "w", true);
+    identity_store(fs, "/identity")
+#endif
+{
+}
+#endif
+
+static File openWrite(FILESYSTEM* fs, const char* filename) {
+#if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM)
+  fs->remove(filename);
+  return fs->open(filename, FILE_O_WRITE);
+#elif defined(RP2040_PLATFORM)
+  return fs->open(filename, "w");
+#else
+  return fs->open(filename, "w", true);
 #endif
 }
 
@@ -36,6 +50,9 @@ void DataStore::begin() {
 
 #if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM)
   checkAdvBlobFile();
+  #if defined(EXTRAFS) || defined(QSPIFLASH)
+  migrateToSecondaryFS();
+  #endif
 #else
   // init 'blob store' support
   _fs->mkdir("/bl");
@@ -46,12 +63,13 @@ void DataStore::begin() {
   #include <SPIFFS.h>
 #elif defined(RP2040_PLATFORM)
   #include <LittleFS.h>
-#elif defined(NRF52_PLATFORM) || defined(STM32_PLATFORM)
+#elif defined(NRF52_PLATFORM)
-  // #include <InternalFileSystem.h> // disabled for now, leaving here for dual fs branch
   #include <CustomLFS.h>
   #if defined(QSPIFLASH)
     #include <CustomLFS_QSPIFlash.h>
   #endif
+#elif defined(STM32_PLATFORM)
+  #include <InternalFileSystem.h>
 #endif
 
 #if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM)
@@ -77,8 +95,13 @@ uint32_t DataStore::getStorageUsedKb() const {
   _fs->info(info);
   return info.usedBytes / 1024;
 #elif defined(NRF52_PLATFORM) || defined(STM32_PLATFORM)
+#if defined(EXTRAFS) || defined(QSPIFLASH)
+  const lfs_config* config = _fsExtra->_getFS()->cfg;
+  int usedBlockCount = _getLfsUsedBlockCount(_fsExtra);
+#else
   const lfs_config* config = _fs->_getFS()->cfg;
   int usedBlockCount = _getLfsUsedBlockCount(_fs);
+#endif
   int usedBytes = config->block_size * usedBlockCount;
   return usedBytes / 1024;
 #else
@@ -95,7 +118,11 @@ uint32_t DataStore::getStorageTotalKb() const {
   _fs->info(info);
   return info.totalBytes / 1024;
 #elif defined(NRF52_PLATFORM) || defined(STM32_PLATFORM)
+#if defined(EXTRAFS) || defined(QSPIFLASH)
+  const lfs_config* config = _fsExtra->_getFS()->cfg;
+#else
   const lfs_config* config = _fs->_getFS()->cfg;
+#endif
   int totalBytes = config->block_size * config->block_count;
   return totalBytes / 1024;
 #else
@@ -113,14 +140,31 @@ File DataStore::openRead(const char* filename) {
 #endif
 }
 
+File DataStore::openRead(FILESYSTEM* fs, const char* filename) {
+#if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM)
+  return fs->open(filename, FILE_O_READ);
+#elif defined(RP2040_PLATFORM)
+  return fs->open(filename, "r");
+#else
+  return fs->open(filename, "r", false);
+#endif
+}
+
 bool DataStore::removeFile(const char* filename) {
   return _fs->remove(filename);
 }
 
+bool DataStore::removeFile(FILESYSTEM* fs, const char* filename) {
+  return fs->remove(filename);
+}
+
 bool DataStore::formatFileSystem() {
 #if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM)
-  // InternalFS.format(); // leaving as placeholder to remind for dual fs branch
+#if defined(EXTRAFS) || defined(QSPIFLASH)
+  return _fs->format() && _fsExtra->format(); // in future maybe return an error code based on which format failed?
+#else
   return _fs->format();
+#endif
 #elif defined(RP2040_PLATFORM)
   return LittleFS.format();
 #elif defined(ESP32)
@@ -214,11 +258,20 @@ void DataStore::savePrefs(const NodePrefs& _prefs, double node_lat, double node_
 }
 
 void DataStore::loadContacts(DataStoreHost* host) {
+#if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM)
+#if defined(EXTRAFS) || defined(QSPIFLASH)
+  if (_fsExtra->exists("/contacts3")) {
+    File file = _fsExtra->open("/contacts3");
+#else
   if (_fs->exists("/contacts3")) {
-#if defined(RP2040_PLATFORM)
+    File file = _fs->open("/contacts3");
+#endif
+#elif defined(RP2040_PLATFORM)
+    if (_fs->exists("/contacts3")) {
     File file = _fs->open("/contacts3", "r");
 #else
-    File file = _fs->open("/contacts3");
+    if (_fs->exists("/contacts3")) {
+      File file = _fs->open("/contacts3", "r", false);
 #endif
     if (file) {
       bool full = false;
@@ -251,7 +304,11 @@ void DataStore::loadContacts(DataStoreHost* host) {
 }
 
 void DataStore::saveContacts(DataStoreHost* host) {
+#if defined(EXTRAFS) || defined(QSPIFLASH)
+  File file = openWrite(_fsExtra, "/contacts3");
+#else
   File file = openWrite(_fs, "/contacts3");
+#endif
   if (file) {
     uint32_t idx = 0;
     ContactInfo c;
@@ -280,11 +337,20 @@ void DataStore::saveContacts(DataStoreHost* host) {
 }
 
 void DataStore::loadChannels(DataStoreHost* host) {
+#if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM)
+#if defined(EXTRAFS) || defined(QSPIFLASH)
+  if (_fsExtra->exists("/channels2")) {
+    File file = _fsExtra->open("/channels2");
+#else
+  if (_fs->exists("/channels2")) {
+    File file = _fs->open("/channels2");
+#endif
+#elif defined(RP2040_PLATFORM)
   if (_fs->exists("/channels2")) {
-#if defined(RP2040_PLATFORM)
     File file = _fs->open("/channels2", "r");
 #else
-    File file = _fs->open("/channels2");
+  if (_fs->exists("/channels2")) {
+    File file = _fs->open("/channels2", "r", false);
 #endif
     if (file) {
       bool full = false;
@@ -311,7 +377,11 @@ void DataStore::loadChannels(DataStoreHost* host) {
 }
 
 void DataStore::saveChannels(DataStoreHost* host) {
+#if defined(EXTRAFS) || defined(QSPIFLASH)
+  File file = openWrite(_fsExtra, "/channels2");
+#else
   File file = openWrite(_fs, "/channels2");
+#endif
   if (file) {
     uint8_t channel_idx = 0;
     ChannelDetails ch;
@@ -342,8 +412,13 @@ struct BlobRec {
 };
 
 void DataStore::checkAdvBlobFile() {
+#if defined(EXTRAFS) || defined(QSPIFLASH)
+  if (!_fsExtra->exists("/adv_blobs")) {
+    File file = openWrite(_fsExtra, "/adv_blobs");
+#else
   if (!_fs->exists("/adv_blobs")) {
     File file = openWrite(_fs, "/adv_blobs");
+#endif
     if (file) {
       BlobRec zeroes;
       memset(&zeroes, 0, sizeof(zeroes));
@@ -355,8 +430,117 @@ void DataStore::checkAdvBlobFile() {
   }
 }
 
+void DataStore::migrateToSecondaryFS() {
+  // migrate old adv_blobs, contacts3 and channels2 files to secondary FS if they don't already exist
+  if (!_fsExtra->exists("/adv_blobs")) {
+    if (_fs->exists("/adv_blobs")) {
+    File oldAdvBlobs = openRead(_fs, "/adv_blobs");
+    File newAdvBlobs = openWrite(_fsExtra, "/adv_blobs");
+
+    if (oldAdvBlobs && newAdvBlobs) {
+      BlobRec rec;
+      size_t count = 0;
+
+      // Copy 20 BlobRecs from old to new
+      while (count < 20 && oldAdvBlobs.read((uint8_t *)&rec, sizeof(rec)) == sizeof(rec)) {
+        newAdvBlobs.seek(count * sizeof(BlobRec));
+        newAdvBlobs.write((uint8_t *)&rec, sizeof(rec));
+        count++;
+      }
+    }
+    if (oldAdvBlobs) oldAdvBlobs.close();
+    if (newAdvBlobs) newAdvBlobs.close();
+    _fs->remove("/adv_blobs");
+    }
+  }
+  if (!_fsExtra->exists("/contacts3")) {
+    if (_fs->exists("/contacts3")) {
+      File oldFile = openRead(_fs, "/contacts3");
+      File newFile = openWrite(_fsExtra, "/contacts3");
+
+      if (oldFile && newFile) {
+        uint8_t buf[64];
+        int n;
+        while ((n = oldFile.read(buf, sizeof(buf))) > 0) {
+          newFile.write(buf, n);
+        }
+      }
+      if (oldFile) oldFile.close();
+      if (newFile) newFile.close();
+      _fs->remove("/contacts3");
+    }
+  }
+  if (!_fsExtra->exists("/channels2")) {
+    if (_fs->exists("/contacts2")) {
+      File oldFile = openRead(_fs, "/contacts2");
+      File newFile = openWrite(_fsExtra, "/contacts2");
+
+      if (oldFile && newFile) {
+        uint8_t buf[64];
+        int n;
+        while ((n = oldFile.read(buf, sizeof(buf))) > 0) {
+          newFile.write(buf, n);
+        }
+      }
+      if (oldFile) oldFile.close();
+      if (newFile) newFile.close();
+      _fs->remove("/contacts2");
+    }
+  }
+  // cleanup nodes which have been testing the extra fs, copy _main.id and new_prefs back to primary
+  if (_fsExtra->exists("/_main.id")) {
+      if (_fs->exists("/_main.id")) {_fs->remove("/_main.id");}
+      File oldFile = openRead(_fsExtra, "/_main.id");
+      File newFile = openWrite(_fs, "/_main.id");
+
+      if (oldFile && newFile) {
+        uint8_t buf[64];
+        int n;
+        while ((n = oldFile.read(buf, sizeof(buf))) > 0) {
+          newFile.write(buf, n);
+        }
+      }
+      if (oldFile) oldFile.close();
+      if (newFile) newFile.close();
+      _fsExtra->remove("/_main.id");
+  }
+  if (_fsExtra->exists("/new_prefs")) {
+    if (_fs->exists("/new_prefs")) {_fs->remove("/new_prefs");}
+      File oldFile = openRead(_fsExtra, "/new_prefs");
+      File newFile = openWrite(_fs, "/new_prefs");
+
+      if (oldFile && newFile) {
+        uint8_t buf[64];
+        int n;
+        while ((n = oldFile.read(buf, sizeof(buf))) > 0) {
+          newFile.write(buf, n);
+        }
+      }
+      if (oldFile) oldFile.close();
+      if (newFile) newFile.close();
+      _fsExtra->remove("/new_prefs");
+  }
+  // remove files from where they should not be anymore
+  if (_fs->exists("/adv_blobs")) {
+    _fs->remove("/adv_blobs");
+  }
+  if (_fs->exists("/contacts3")) {
+    _fs->remove("/contacts3");
+  }
+  if (_fsExtra->exists("/_main.id")) {
+    _fsExtra->remove("/_main.id");
+  }
+  if (_fsExtra->exists("/new_prefs")) {
+    _fsExtra->remove("/new_prefs");
+  }
+}
+
 uint8_t DataStore::getBlobByKey(const uint8_t key[], int key_len, uint8_t dest_buf[]) {
+#if defined(EXTRAFS) || defined(QSPIFLASH)
+  File file = _fsExtra->open("/adv_blobs");
+#else
   File file = _fs->open("/adv_blobs");
+#endif
   uint8_t len = 0;  // 0 = not found
 
   if (file) {
@@ -378,7 +562,11 @@ bool DataStore::putBlobByKey(const uint8_t key[], int key_len, const uint8_t src
 
   checkAdvBlobFile();
 
+#if defined(EXTRAFS) || defined(QSPIFLASH)
+  File file = _fsExtra->open("/adv_blobs", FILE_O_WRITE);
+#else
   File file = _fs->open("/adv_blobs", FILE_O_WRITE);
+#endif
   if (file) {
     uint32_t pos = 0, found_pos = 0;
     uint32_t min_timestamp = 0xFFFFFFFF;

examples/companion_radio/DataStore.h

@@ -15,6 +15,7 @@ public:
 
 class DataStore {
   FILESYSTEM* _fs;
+  FILESYSTEM* _fsExtra;
   mesh::RTCClock* _clock;
   IdentityStore identity_store;
 
@@ -25,8 +26,11 @@ class DataStore {
 
 public:
   DataStore(FILESYSTEM& fs, mesh::RTCClock& clock);
+  DataStore(FILESYSTEM& fs, FILESYSTEM& fsExtra, mesh::RTCClock& clock);
   void begin();
   bool formatFileSystem();
+  FILESYSTEM* getPrimaryFS() const { return _fs; }
+  FILESYSTEM* getSecondaryFS() const { return _fsExtra; }
   bool loadMainIdentity(mesh::LocalIdentity &identity);
   bool saveMainIdentity(const mesh::LocalIdentity &identity);
   void loadPrefs(NodePrefs& prefs, double& node_lat, double& node_lon);
@@ -35,10 +39,13 @@ public:
   void saveContacts(DataStoreHost* host);
   void loadChannels(DataStoreHost* host);
   void saveChannels(DataStoreHost* host);
+  void migrateToSecondaryFS();
   uint8_t getBlobByKey(const uint8_t key[], int key_len, uint8_t dest_buf[]);
   bool putBlobByKey(const uint8_t key[], int key_len, const uint8_t src_buf[], uint8_t len);
   File openRead(const char* filename);
+  File openRead(FILESYSTEM* fs, const char* filename);
   bool removeFile(const char* filename);
+  bool removeFile(FILESYSTEM* fs, const char* filename);
   uint32_t getStorageUsedKb() const;
   uint32_t getStorageTotalKb() const;
 };

examples/companion_radio/MyMesh.cpp

@@ -1530,27 +1530,65 @@ void MyMesh::checkCLIRescueCmd() {
       if(root){
         File file = root.openNextFile();
         while (file) {
-
+          
+          #if defined(EXTRAFS) || defined(QSPIFLASH)
+          if (file.isDirectory()) {
+            Serial.printf("[dir]  /FS1/%s\n", file.name());
+          } else {
+            Serial.printf("[file] /FS1/%s (%d bytes)\n", file.name(), file.size());
+          }
+          #else
           if (file.isDirectory()) {
             Serial.printf("[dir] %s\n", file.name());
           } else {
             Serial.printf("[file] %s (%d bytes)\n", file.name(), file.size());
           }
-
+          #endif
           // move to next file
           file = root.openNextFile();
 
         }
         root.close();
       }
+      #if defined(EXTRAFS) || defined(QSPIFLASH)
+        root = _store->openRead(_store->getSecondaryFS(), path);
+        if(root){
+          File file = root.openNextFile();
+          while (file) {
+
+            if (file.isDirectory()) {
+              Serial.printf("[dir]  /FS2/%s\n", file.name());
+            } else {
+              Serial.printf("[file] /FS2/%s (%d bytes)\n", file.name(), file.size());
+            }
+
+            // move to next file
+            file = root.openNextFile();
+
+          }
+        root.close();
+      }
+      #endif
 
     } else if (memcmp(cli_command, "cat", 3) == 0) {
 
       // get path from command e.g: "cat /contacts3"
       const char *path = &cli_command[4];
       
+      // 
+      bool is_fs2 = false;
+      if (memcmp(path, "FS1/", 4) == 0) {
+        path += 3; // skip "FS1"
+      } else if (memcmp(path, "FS2/", 4) == 0) {
+        path += 3; // skip "FS2"
+        is_fs2 = true;
+      }
+
       // log file content as hex
       File file = _store->openRead(path);
+      if (is_fs2 == true) {
+        file = _store->openRead(_store->getSecondaryFS(), path);
+      }
       if(file){
 
         // get file content
@@ -1570,14 +1608,28 @@ void MyMesh::checkCLIRescueCmd() {
 
       // get path from command e.g: "rm /adv_blobs"
       const char *path = &cli_command[4];
-
+      MESH_DEBUG_PRINTLN("Removing file: %s", path);
       // ensure path is not empty, or root dir
       if(!path || strlen(path) == 0 || strcmp(path, "/") == 0){
         Serial.println("Invalid path provided");
       } else {
+        bool is_fs2 = false;
+        if (memcmp(path, "FS1/", 4) == 0) {
+          path += 3; // skip "FS1"
+        } else if (memcmp(path, "FS2/", 4) == 0) {
+          path += 3; // skip "FS2"
+          is_fs2 = true;
+        }
 
         // remove file
-        bool removed = _store->removeFile(path);
+        bool removed;
+        if (is_fs2) {
+          MESH_DEBUG_PRINTLN("Removing file from FS2: %s", path);
+          removed = _store->removeFile(_store->getSecondaryFS(), path);
+        } else {
+          MESH_DEBUG_PRINTLN("Removing file from FS1: %s", path);
+          removed = _store->removeFile(path);
+        }
         if(removed){
           Serial.println("File removed");
         } else {

examples/companion_radio/main.cpp

@@ -16,12 +16,12 @@ static uint32_t _atoi(const char* sp) {
   #include <InternalFileSystem.h>
   #if defined(QSPIFLASH)
     #include <CustomLFS_QSPIFlash.h>
-    DataStore store(QSPIFlash, rtc_clock);
+    DataStore store(InternalFS, QSPIFlash, rtc_clock);
   #else
   #if defined(EXTRAFS)
     #include <CustomLFS.h>
     CustomLFS ExtraFS(0xD4000, 0x19000, 128);
-    DataStore store(ExtraFS, rtc_clock);
+    DataStore store(InternalFS, ExtraFS, rtc_clock);
   #else
     DataStore store(InternalFS, rtc_clock);
   #endif
@@ -129,6 +129,7 @@ void setup() {
   fast_rng.begin(radio_get_rng_seed());
 
 #if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM)
+  InternalFS.begin();
   #if defined(QSPIFLASH)
     if (!QSPIFlash.begin()) {
       // debug output might not be available at this point, might be too early. maybe should fall back to InternalFS here?
@@ -137,10 +138,9 @@ void setup() {
       MESH_DEBUG_PRINTLN("CustomLFS_QSPIFlash: initialized successfully");
     }
   #else
-    InternalFS.begin();
+  #if defined(EXTRAFS)
-    #if defined(EXTRAFS)
       ExtraFS.begin();
-    #endif
+  #endif
   #endif
   store.begin();
   the_mesh.begin(

src/helpers/IdentityStore.h

@@ -4,18 +4,10 @@
   #include <FS.h>
   #define FILESYSTEM  fs::FS
 #elif defined(NRF52_PLATFORM) || defined(STM32_PLATFORM)
-  #if defined(QSPIFLASH)
-    #include <CustomLFS_QSPIFlash.h>
-    #define FILESYSTEM CustomLFS_QSPIFlash
-  #elif defined(EXTRAFS)
-    #include <CustomLFS.h>
-    #define FILESYSTEM CustomLFS
-  #else
     #include <Adafruit_LittleFS.h>
     #define FILESYSTEM  Adafruit_LittleFS
 
     using namespace Adafruit_LittleFS_Namespace;
-  #endif
 #endif
 #include <Identity.h>