Replace SPIFFS with FAT, seems to work well

FollowMe.cfg

@@ -67,12 +67,13 @@
 // #define WITH_KNOB
 // #define WITH_VARIO
 
-// #define WITH_STRATUX                       // talk to Stratux via WiFi and port 30011
 #define WITH_SD                            // use the SD card in SPI mode and FAT file system
+#define WITH_SPIFFS_FAT
 #define WITH_SPIFFS                        // use SPIFFS file system in Flash
 #define WITH_LOG                           // log own positions and other received to SPIFFS
 #define WITH_SDLOG                         // log own position and other data to uSD card
 
+// #define WITH_STRATUX                       // talk to Stratux via WiFi and port 30011
 #define WITH_BT_SPP                        // Bluetooth serial port for smartphone/tablet link
 // #define WITH_WIFI                          // attempt to connect to the wifi router for uploading the log files
 

FollowMe_old.cfg

@@ -54,7 +54,7 @@
 
 #define WITH_FANET
 #define WITH_PFLAA                         // PFLAU and PFLAA for compatibility with XCsoar and LK8000
-#define WITH_POGNT
+// #define WITH_POGNT
 // #define WITH_GDL90
 // #define WITH_PGAV5
 #define WITH_LOOKOUT
@@ -69,6 +69,7 @@
 
 #define WITH_SD                            // use the SD card in SPI mode and FAT file system
 #define WITH_SPIFFS                        // use SPIFFS file system in Flash
+#define WITH_SPIFFS_FAT
 #define WITH_LOG                           // log own positions and other received to SPIFFS
 #define WITH_SDLOG                         // log own position and other data to uSD card
 

T-Beam_v10.cfg

@@ -18,7 +18,7 @@
 // #define WITH_TFT_LCD                       // TFT LCD
 // #define WITH_OLED                          // OLED display on the I2C: some TTGO modules are without OLED display
 // #define WITH_OLED2                         // 2nd OLED display, I2C address next higher
-#define WITH_U8G2_OLED                     // I2C OLED through the U8g2 library
+// #define WITH_U8G2_OLED                     // I2C OLED through the U8g2 library
 // #define WITH_U8G2_SH1106
 
 #define WITH_RFM95                         // RF chip selection:  both HELTEC and TTGO use sx1276 which is same as RFM95
@@ -45,7 +45,7 @@
 
 // #define WITH_BMP180                        // BMP180 pressure sensor
 // #define WITH_BMP280                        // BMP280 pressure sensor
-#define WITH_BME280                        // BMP280 with humidity (but still works with BMP280)
+// #define WITH_BME280                        // BMP280 with humidity (but still works with BMP280)
 // #define WITH_MS5607                        // MS5607 pressure sensor
 // #define WITH_MS5611                        // MS5611 pressure sensor
 
@@ -62,12 +62,14 @@
 // #define WITH_VARIO
 
 // #define WITH_SD                            // use the SD card in SPI mode and FAT file system
+#define WITH_SPIFFS_FAT
 #define WITH_SPIFFS                        // use SPIFFS file system in Flash
 #define WITH_LOG                           // log own positions and other received to SPIFFS and possibly to uSD
 
+// #define WITH_STRATUX
 #define WITH_BT_SPP                        // Bluetooth serial port for smartphone/tablet link
 // #define WITH_WIFI                          // attempt to connect to the wifi router for uploading the log files
 // #define WITH_SPIFFS_LOG                    // log transmitted and received packets to SPIFFS
 
-#define WITH_ENCRYPT                       // Encrypt (optionally) the position
+// #define WITH_ENCRYPT                       // Encrypt (optionally) the position
 

T-Beam_v10_LCD.cfg

@@ -61,8 +61,9 @@
 // #define WITH_KNOB
 // #define WITH_VARIO
 
-#define WITH_STRATUX
+// #define WITH_STRATUX
 // #define WITH_SD                            // use the SD card in SPI mode and FAT file system
+#define WITH_SPIFFS_FAT
 #define WITH_SPIFFS                        // use SPIFFS file system in Flash
 #define WITH_LOG                           // log own positions and other received to SPIFFS and possibly to uSD
 

main/config.h

@@ -10,7 +10,6 @@
 // #define WITH_TTGO                          // TTGO module: PCB LED on GPIO2, GPIO25 free to use as DAC2 output
 // #define WITH_TBEAM                          // T-Beam module
 #define WITH_TBEAM_V10                      // T-Beam module
-// #define WITH_JACEK                         // JACEK ESP32 OGN-Tracker
 // #define WITH_M5_JACEK                         // JACEK M5 ESP32 OGN-Tracker
 // #define WITH_FollowMe                         // by Avionix
 
@@ -19,10 +18,8 @@
 // #define WITH_TFT_LCD                       // TFT LCD
 // #define WITH_OLED                          // OLED display on the I2C: some TTGO modules are without OLED display
 // #define WITH_OLED2                         // 2nd OLED display, I2C address next higher
-#define WITH_U8G2_OLED                     // I2C OLED through the U8g2 library
+// #define WITH_U8G2_OLED                     // I2C OLED through the U8g2 library
 // #define WITH_U8G2_SH1106
-#define U8G2_OLED_PAGES_ENABLED   0b0001111111  // use if WITH_U8G2_OLED is on, 10 pages currently, see end of disp.ccp for the list of pages
-
 
 #define WITH_RFM95                         // RF chip selection:  both HELTEC and TTGO use sx1276 which is same as RFM95
 
@@ -48,7 +45,7 @@
 
 // #define WITH_BMP180                        // BMP180 pressure sensor
 // #define WITH_BMP280                        // BMP280 pressure sensor
-#define WITH_BME280                        // BMP280 with humidity (but still works with BMP280)
+// #define WITH_BME280                        // BMP280 with humidity (but still works with BMP280)
 // #define WITH_MS5607                        // MS5607 pressure sensor
 // #define WITH_MS5611                        // MS5611 pressure sensor
 
@@ -65,6 +62,7 @@
 // #define WITH_VARIO
 
 // #define WITH_SD                            // use the SD card in SPI mode and FAT file system
+#define WITH_SPIFFS_FAT
 #define WITH_SPIFFS                        // use SPIFFS file system in Flash
 #define WITH_LOG                           // log own positions and other received to SPIFFS and possibly to uSD
 
@@ -72,5 +70,5 @@
 // #define WITH_WIFI                          // attempt to connect to the wifi router for uploading the log files
 // #define WITH_SPIFFS_LOG                    // log transmitted and received packets to SPIFFS
 
-#define WITH_ENCRYPT                       // Encrypt (optionally) the position
+// #define WITH_ENCRYPT                       // Encrypt (optionally) the position
 

main/hal.cpp

@@ -24,7 +24,7 @@
 #include "nvs_flash.h"
 
 #ifdef WITH_SPIFFS
-// #include "esp_vfs_fat.h"
+#include "esp_vfs_fat.h"
 #include "esp_spiffs.h"
 #endif
 
@@ -1954,8 +1954,9 @@ int NVS_Init(void)
 // ======================================================================================================
 
 #ifdef WITH_SPIFFS
-/*
+
-// this part is for the FAT filesystem in the internal flash
+#ifdef WITH_SPIFFS_FAT // FAT replaces SPIFFS, hopefully no performace and reliability issues
+
 int SPIFFS_Register(const char *Path, const char *Label, size_t MaxOpenFiles)
 { esp_vfs_fat_mount_config_t FSconf;
   FSconf.max_files = MaxOpenFiles;
@@ -1965,8 +1966,19 @@ int SPIFFS_Register(const char *Path, const char *Label, size_t MaxOpenFiles)
   return esp_vfs_fat_spiflash_mount(Path, Label, &FSconf, &Handle); }
 
 int SPIFFS_Info(size_t &Total, size_t &Used, const char *Label)
-{ Total=0; Used=0; return 0; }
+{ FATFS *FS=0;
-*/
+  Total=0; Used=0;
+  size_t FreeClusters;
+  int Ret = f_getfree("0:", &FreeClusters, &FS);
+  // if(Ret=!FR_OK) return Ret;
+  if(FS==0) return 0;
+  size_t TotalSectors = (FS->n_fatent-2) * FS->csize;
+  size_t FreeSectors = FreeClusters * FS->csize;
+  Total = TotalSectors * FS->ssize;
+  Used  = (TotalSectors-FreeSectors) * FS->ssize;
+  return 0; }
+
+#else // SPIFFS: gives troubles when more than few files are open
 
 int SPIFFS_Register(const char *Path, const char *Label, size_t MaxOpenFiles)
 { esp_vfs_spiffs_conf_t FSconf =
@@ -1979,6 +1991,7 @@ int SPIFFS_Register(const char *Path, const char *Label, size_t MaxOpenFiles)
 int SPIFFS_Info(size_t &Total, size_t &Used, const char *Label)
 { return esp_spiffs_info(Label, &Total, &Used); }
 
+#endif
 #endif
 
 // ======================================================================================================

main/log.cpp

@@ -26,12 +26,16 @@ static void AddPath(char *Name, const char *FileName, const char *Path)
 static const char *SDcard_Path = "/sdcard/TLG";   // with sub-directory which is created if does not exist
 static const char *FlashLog_Path = "/spiffs";     // path to log files
 static const char *FlashLog_Ext  = ".TLG";        // extension for log files, could be as well .TLA
-static const uint32_t FlashLog_MaxTime = 7200;    // 2 hour max. per single log file
+static const uint32_t FlashLog_MaxTime = 3600;    // 1 hour max. per single log file
-static const uint32_t FlashLog_MaxSize = 0x20000; // 128KB max. per single log file
+static const uint32_t FlashLog_MaxSize = 0x10000; // 64KB max. per single log file
-// static uint32_t FlashLog_OldestTime;
+#ifdef WITH_SPIFFS_FAT
-uint32_t FlashLog_FileTime=0;
+static const uint32_t FlashLog_FlushSize = 4096;  // 4kB file flush step
+#endif
+
+uint32_t FlashLog_FileTime=0;                     // [sec] UTC time corresponding to the log file
 char FlashLog_FileName[32];                       // current log file name if open
 static FILE *FlashLog_File=0;                     // current log file if open
+static uint32_t FlashLog_FileFlush=0;             // track where the log file has been forced to be written to flash
 
 FIFO<OGN_LogPacket<OGN_Packet>, 32> FlashLog_FIFO;
 
@@ -237,12 +241,12 @@ static int FlashLog_CleanEmpty(int MinSize=0)                 // delete empty fi
     vTaskDelay(1); }
   return DelFiles; }
 
-static int FlashLog_Clean(size_t MinFree)                          // clean oldest file when running short in space
+static int FlashLog_Clean(size_t MinFree=0)                          // clean oldest file when running short in space
 { size_t Total, Used;
   if(SPIFFS_Info(Total, Used)!=0) return -1;                        // check SPIFFS status, give up if not possible
   size_t Free = Total-Used;                                         // [B] amount of free space
   if(MinFree) { if(Free>= MinFree ) return 0; }                     // give up if enough space
-         else { if(Free>=(Total/2)) return 0; }                     // if MinFree not specified, take Total/4
+         else { if(Free>=(Total/4)) return 0; }                     // if MinFree not specified, take Total/4
   uint32_t Oldest=0xFFFFFFFF;
   int Files=FlashLog_FindOldestFile(Oldest, 0);                    // find the oldest file
   if(Files<0) return Files;
@@ -268,13 +272,14 @@ static int FlashLog_Clean(size_t MinFree, int Loops)                // repeat th
     vTaskDelay(1); Count++; }
   return Count; }
 
-static int FlashLog_Open(uint32_t Time)                             // open a new log file for given start time
+static int FlashLog_Open(uint32_t Time)                            // open a new log file for given start time
-{ if(FlashLog_File) { fclose(FlashLog_File); FlashLog_File=0; }   // if a file open already, close it
+{ if(FlashLog_File) { fclose(FlashLog_File); FlashLog_File=0; }    // if a file open already, close it
-  FlashLog_CleanEmpty(32);                                          // remove empty files or shorter than 32 bytes
+  FlashLog_CleanEmpty(32);                                         // remove empty files or shorter than 32 bytes
-  FlashLog_Clean(2*FlashLog_MaxSize, 8);                           // clean files to get free space at least twice the max. file sie
+  FlashLog_Clean(2*FlashLog_MaxSize, 2);                           // clean files to get free space at least twice the max. file size
   FlashLog_FullFileName(FlashLog_FileName, Time);                  // name of the new log file
-  FlashLog_FileTime=Time;                                           // record the time of the log file
+  FlashLog_FileTime=Time;                                          // record the time of the log file
   FlashLog_File = fopen(FlashLog_FileName, "wb");                  // open the new file
+  FlashLog_FileFlush = 0;
 #ifdef DEBUG_PRINT
   xSemaphoreTake(CONS_Mutex, portMAX_DELAY);
   Format_String(CONS_UART_Write, "FlashLog_Open() ");
@@ -282,13 +287,15 @@ static int FlashLog_Open(uint32_t Time)                             // open a ne
   Format_String(CONS_UART_Write, "\n");
   xSemaphoreGive(CONS_Mutex);
 #endif
-  if(FlashLog_File==0) FlashLog_Clean(0, 8);                       // if the file cannot be open clean again
+  if(FlashLog_File==0) FlashLog_Clean(0, 4);                       // if the file cannot be open clean again
   return FlashLog_File!=0; }                                        // 1=success, 0=failure: new log file could not be open
 
 static int FlashLog(OGN_LogPacket<OGN_Packet> *Packet, int Packets, uint32_t Time)      // log a batch of OGN packets
 { if(FlashLog_File)                                                          // if log file already open
   { uint32_t TimeSinceStart = Time-FlashLog_FileTime;                        // [sec] for how long this file is open already ?
-    if( (TimeSinceStart>=FlashLog_MaxTime) || (ftell(FlashLog_File)>=FlashLog_MaxSize) ) // is it too long in time or in size ?
+    uint32_t WritePos = ftell(FlashLog_File);
+    uint32_t WriteSize = Packets*sizeof(OGN_LogPacket<OGN_Packet>);
+    if( (TimeSinceStart>=FlashLog_MaxTime) || ((WritePos+WriteSize)>FlashLog_MaxSize) ) // is it too long in time or in size ?
     { fclose(FlashLog_File); FlashLog_File=0; }                              // decide to close the current log file
   }
   if(FlashLog_File==0)
@@ -299,7 +306,14 @@ static int FlashLog(OGN_LogPacket<OGN_Packet> *Packet, int Packets, uint32_t Tim
     FlashLog_Open(Time); }                                                   // if file closed, then attempt to open a new one
   if(FlashLog_File==0) return -1;                                            // if file still not open, then give up
   if(fwrite(Packet, Packet->Bytes, Packets, FlashLog_File)!=Packets)         // write the packet to the log file
-  { fclose(FlashLog_File); FlashLog_File=0; FlashLog_Clean(0, 8); return -1; } // if failure then close the log file and report error
+  { fclose(FlashLog_File); FlashLog_File=0; FlashLog_Clean(0, 4); return -1; } // if failure then close the log file and report error
+#ifdef WITH_SPIFFS_FAT
+  uint32_t WritePos = ftell(FlashLog_File);
+  if(WritePos-FlashLog_FileFlush>FlashLog_FlushSize)
+  { fclose(FlashLog_File);
+    FlashLog_File = fopen(FlashLog_FileName, "ab");
+    FlashLog_FileFlush=WritePos; }
+#endif
   return Packets; }                                                          // report success
 #endif // WITH_SPIFFS
 
@@ -310,8 +324,8 @@ static int Copy(void)                                              // copy the p
   uint32_t Time = TimeSync_Time();                                 // Time is to create new log file
 #ifdef WITH_SPIFFS
   int Err=FlashLog(Packet, Packets, Time);                         // log the batch of packets
-  if(Err<0) { FlashLog_Clean(0, 8); Err=FlashLog(Packet, Packets, Time); } // if failed: give it another try
+  if(Err<0) { FlashLog_Clean(0, 4); Err=FlashLog(Packet, Packets, Time); } // if failed: give it another try
-  // if(Err<0) FlashLog_Clean(0, 8);
+  // if(Err<0) FlashLog_Clean(0, 4);
 #ifdef DEBUG_PRINT
   xSemaphoreTake(CONS_Mutex, portMAX_DELAY);
   Format_String(CONS_UART_Write, "vTaskLOG() ");

partitions.csv

@@ -1,7 +1,11 @@
 # Espressif ESP32 Partition Table
 # Name, Type, SubType, Offset, Size, Flags
+# for parameter storage
 nvs,data,nvs,0x9000,24K,
 phy_init,data,phy,0xf000,4K,
+# application (code)
 factory,app,factory,0x10000,1800K,
-intlog,data,spiffs,,0x1F0000,
+# SPIFFS in Flash
-# intlog,data,fat,,0x1F0000,
+# intlog,data,spiffs,,0x1F0000,
+# FAT in Flash
+intlog,data,fat,,0x1F0000,

utils/read_log.cc

@@ -39,7 +39,7 @@ static int List(void)
   { char Byte;
     if(Port.Read(Byte)<=0)                                            // get a byte from the serial port
     { double Now=getTime();                                           // if non, then check time
-      if((Now-Start)>=10.0) break;                                     // if idle for more than 4 sec then stop recording the log files
+      if((Now-Start)>=5.0) break;                                     // if idle for more than 4 sec then stop recording the log files
       usleep(1000); continue; }                                       // if no new bytes on the serial port sleep a little
     // printf("%3d: %02X %c\n", LineIdx, Byte, Byte<=' '?' ':Byte);
     if(Byte<' ')                                                      // if a control (non-printable) character
@@ -80,7 +80,7 @@ static int Download(uint32_t LogFile)
   { char Byte;
     if(Port.Read(Byte)<=0)
     { double Now=getTime();
-      if((Now-Start)>=8.0) break;
+      if((Now-Start)>=4.0) break;
       usleep(1000); continue; }                                       // if no new bytes on the serial port sleep a little
     if(Byte<' ')                                                      // if a control (non-printable) character
     { Line[LineIdx]=0;