AP_HAL_ChibiOS: introduce new DATAFLASH directive and use it

add HAL_OS_POSIX_IO to defaults
ensure explicitly configured DATAFLASH is prioritized
correct inclusion of fat malloc
only create storage backups on fatfs

libraries/AP_HAL_ChibiOS/HAL_ChibiOS_Class.h

@@ -21,7 +21,7 @@
 #include <AP_HAL_Empty/AP_HAL_Empty_Namespace.h>
 #include <AP_HAL_ChibiOS/AP_HAL_ChibiOS_Namespace.h>
 #include "hwdef/common/halconf.h"
-#ifdef USE_POSIX_FATFS
+#if HAL_USE_FATFS
 #include <ff.h>
 #endif
 #include <stdio.h>

libraries/AP_HAL_ChibiOS/Storage.cpp

@@ -127,8 +127,8 @@ void Storage::_storage_open(void)
  */
 void Storage::_save_backup(void)
 {
-#ifdef USE_POSIX_FATFS
+#if HAL_USE_FATFS
-    // allow for fallback to microSD or dataflash based storage
+    // allow for fallback to microSD or littlefs flash based storage
     // create the backup directory if need be
     int ret;
     const char* _storage_bak_directory = HAL_STORAGE_BACKUP_FOLDER;

libraries/AP_HAL_ChibiOS/hwdef/Aocoda-RC-H743Dual/hwdef.dat

@@ -218,5 +218,4 @@ define HAL_OSD_TYPE_DEFAULT 1
 ROMFS_WILDCARD libraries/AP_OSD/fonts/font*.bin
 
 # enable logging to dataflash
-define HAL_OS_LITTLEFS_IO 1
+DATAFLASH littlefs:w25nxx
-define AP_FILESYSTEM_LITTLEFS_FLASH_TYPE AP_FILESYSTEM_FLASH_W25NXX

libraries/AP_HAL_ChibiOS/hwdef/JHEMCU-H743HD/hwdef.dat

@@ -145,8 +145,7 @@ define HAL_GPIO_B_LED_PIN 91
 
 # Dataflash setup
 SPIDEV dataflash SPI3 DEVID1 FLASH1_CS     MODE3 104*MHZ 104*MHZ
-define HAL_OS_LITTLEFS_IO 1
+DATAFLASH littlefs:w25nxx
-define AP_FILESYSTEM_LITTLEFS_FLASH_TYPE AP_FILESYSTEM_FLASH_W25NXX
 
 # OSD setup
 SPIDEV osd SPI2 DEVID1 OSD1_CS   MODE0  10*MHZ  10*MHZ

libraries/AP_HAL_ChibiOS/hwdef/KakuteH7Mini-Nand/hwdef.dat

@@ -10,5 +10,4 @@ SPIDEV bmi270    SPI4 DEVID1 MPU6000_CS    MODE3  1*MHZ  4*MHZ
 IMU BMI270 SPI:bmi270 ROTATION_PITCH_180_YAW_90
 
 undef HAL_LOGGING_DATAFLASH_ENABLED
-define HAL_OS_LITTLEFS_IO 1
+DATAFLASH littlefs:w25nxx
-define AP_FILESYSTEM_LITTLEFS_FLASH_TYPE AP_FILESYSTEM_FLASH_W25NXX

libraries/AP_HAL_ChibiOS/hwdef/KakuteH7v2/hwdef.dat

@@ -64,5 +64,4 @@ DMA_NOSHARE *UP SPI1*
 
 # Motor order implies Betaflight/X for standard ESCs
 define HAL_FRAME_TYPE_DEFAULT 12
-define HAL_OS_LITTLEFS_IO 1
+DATAFLASH littlefs:w25nxx
-define AP_FILESYSTEM_LITTLEFS_FLASH_TYPE AP_FILESYSTEM_FLASH_W25NXX

libraries/AP_HAL_ChibiOS/hwdef/MambaH743v4/hwdef.dat

@@ -180,8 +180,7 @@ IMU BMI270 SPI:bmi270 ROTATION_PITCH_180
 IMU Invensensev3 SPI:icm42688 ROTATION_ROLL_180
 IMU Invensensev3 SPI:icm42688_2 ROTATION_ROLL_180_YAW_270
 
-define HAL_OS_LITTLEFS_IO 1
+DATAFLASH littlefs:w25nxx
-define AP_FILESYSTEM_LITTLEFS_FLASH_TYPE AP_FILESYSTEM_FLASH_W25NXX
 
 # DPS280 or SPL06 integrated on I2C bus
 BARO DPS280 I2C:0:0x76

libraries/AP_HAL_ChibiOS/hwdef/MatekH7A3/hwdef.dat

@@ -162,8 +162,7 @@ ROMFS_WILDCARD libraries/AP_OSD/fonts/font*.bin
 SPIDEV sdcard    SPI2 DEVID3 SD_CS      MODE0 400*KHZ 25*MHZ
 SPIDEV dataflash SPI2 DEVID1 FLASH_CS   MODE3 104*MHZ  104*MHZ
 
-define HAL_OS_LITTLEFS_IO 1
+DATAFLASH littlefs:w25nxx
-define AP_FILESYSTEM_LITTLEFS_FLASH_TYPE AP_FILESYSTEM_FLASH_W25NXX
 
 # ----------------- I2C compass & Baro -----------------
 # no built-in compass, but probe the i2c bus for all possible

libraries/AP_HAL_ChibiOS/hwdef/common/malloc.c

@@ -585,7 +585,7 @@ void __wrap__free_r(void *rptr, void *ptr)
     return free(ptr);
 }
 
-#ifdef USE_POSIX_FATFS
+#if HAL_USE_FATFS
 /*
   allocation functions for FATFS
  */
@@ -615,7 +615,7 @@ void ff_memfree(void* mblock)
 {
     free(mblock);
 }
-#endif // USE_POSIX_FATFS
+#endif // HAL_USE_FATFS
 
 /*
   return true if a memory region is safe for a DMA operation

libraries/AP_HAL_ChibiOS/hwdef/scripts/chibios_hwdef.py

@@ -118,6 +118,9 @@ class ChibiOSHWDef(object):
         self.baro_list = []
         self.airspeed_list = []
 
+        # dataflash config
+        self.dataflash_list = []
+
         # output lines:
         self.all_lines = []
 
@@ -948,17 +951,24 @@ class ChibiOSHWDef(object):
             f.write('#define HAL_STDOUT_SERIAL %s\n\n' % self.get_config('STDOUT_SERIAL'))
             f.write('// baudrate used for stdout (printf)\n')
             f.write('#define HAL_STDOUT_BAUDRATE %u\n\n' % self.get_config('STDOUT_BAUDRATE', type=int))
-        if self.have_type_prefix('SDIO'):
+        if len(self.dataflash_list) > 0:
+            # we only support dataflash OR sdcard, so prioritize dataflash if its been explicitly configured
+            f.write('#define HAL_USE_FATFS FALSE\n\n')
+            f.write('#define HAL_USE_SDC FALSE\n')
+            self.build_flags.append('USE_FATFS=no')
+        elif self.have_type_prefix('SDIO'):
             f.write('// SDIO available, enable POSIX filesystem support\n')
             f.write('#define USE_POSIX\n')
-            f.write('#define USE_POSIX_FATFS\n\n')
+            f.write('#define HAL_OS_POSIX_IO TRUE\n\n')
+            f.write('#define HAL_USE_FATFS TRUE\n\n')
             f.write('#define HAL_USE_SDC TRUE\n')
             self.build_flags.append('USE_FATFS=yes')
             self.env_vars['WITH_FATFS'] = "1"
         elif self.have_type_prefix('SDMMC2'):
             f.write('// SDMMC2 available, enable POSIX filesystem support\n')
             f.write('#define USE_POSIX\n')
-            f.write('#define USE_POSIX_FATFS\n\n')
+            f.write('#define HAL_OS_POSIX_IO TRUE\n\n')
+            f.write('#define HAL_USE_FATFS TRUE\n\n')
             f.write('#define HAL_USE_SDC TRUE\n')
             f.write('#define STM32_SDC_USE_SDMMC2 TRUE\n')
             f.write('#define HAL_USE_SDMMC 1\n')
@@ -967,7 +977,8 @@ class ChibiOSHWDef(object):
         elif self.have_type_prefix('SDMMC'):
             f.write('// SDMMC available, enable POSIX filesystem support\n')
             f.write('#define USE_POSIX\n')
-            f.write('#define USE_POSIX_FATFS\n\n')
+            f.write('#define HAL_USE_FATFS TRUE\n\n')
+            f.write('#define HAL_OS_POSIX_IO TRUE\n\n')
             f.write('#define HAL_USE_SDC TRUE\n')
             f.write('#define STM32_SDC_USE_SDMMC1 TRUE\n')
             f.write('#define HAL_USE_SDMMC 1\n')
@@ -976,20 +987,15 @@ class ChibiOSHWDef(object):
         elif self.has_sdcard_spi():
             f.write('// MMC via SPI available, enable POSIX filesystem support\n')
             f.write('#define USE_POSIX\n')
-            f.write('#define USE_POSIX_FATFS\n\n')
+            f.write('#define HAL_USE_FATFS TRUE\n\n')
+            f.write('#define HAL_OS_POSIX_IO TRUE\n\n')
             f.write('#define HAL_USE_MMC_SPI TRUE\n')
             f.write('#define HAL_USE_SDC FALSE\n')
             f.write('#define HAL_SDCARD_SPI_HOOK TRUE\n')
             self.build_flags.append('USE_FATFS=yes')
             self.env_vars['WITH_FATFS'] = "1"
-        elif self.has_dataflash_spi():
-            f.write('// Dataflash memory via SPI available, enable POSIX filesystem support\n')
-            f.write('#define USE_POSIX\n')
-            f.write('#define USE_POSIX_LITTLEFS\n\n')
-            f.write('#define HAL_USE_SDC FALSE\n')
-            self.build_flags.append('USE_FATFS=no')
-            self.env_vars['WITH_LITTLEFS'] = "1"
         else:
+            f.write('#define HAL_USE_FATFS FALSE\n\n')
             f.write('#define HAL_USE_SDC FALSE\n')
             self.build_flags.append('USE_FATFS=no')
         if 'OTG1' in self.bytype:
@@ -1797,6 +1803,31 @@ INCLUDE common.ld
         if len(devlist) > 0:
             f.write('#define HAL_AIRSPEED_PROBE_LIST %s\n\n' % ';'.join(devlist))
 
+    def write_DATAFLASH_config(self, f):
+        '''write dataflash config defines'''
+        # DATAFLASH block|littlefs:<w25nxx>
+        devlist = []
+        seen = set()
+        for dev in self.dataflash_list:
+            if not self.has_dataflash_spi():
+                self.error("Missing DATAFLASH device: %s" % self.seen_str(dev))
+            if self.seen_str(dev) in seen:
+                self.error("Duplicate DATAFLASH: %s" % self.seen_str(dev))
+            seen.add(self.seen_str(dev))
+            a = dev[0].split(':')
+            if a[0].startswith('block'):
+                if len(a) > 1 and a[1].startswith('w25nxx'):
+                    f.write('#define HAL_LOGGING_DATAFLASH_DRIVER AP_Logger_W25NXX')
+                f.write('#define HAL_LOGGING_DATAFLASH_ENABLED TRUE')
+            elif a[0].startswith('littlefs'):
+                f.write('#define USE_POSIX\n')
+                f.write('#define HAL_OS_LITTLEFS_IO TRUE\n')
+                f.write('#define HAL_OS_POSIX_IO TRUE\n')
+                if len(a) > 1 and a[1].startswith('w25nxx'):
+                    f.write('#define AP_FILESYSTEM_LITTLEFS_FLASH_TYPE AP_FILESYSTEM_FLASH_W25NXX')
+                self.build_flags.append('USE_FATFS=no')
+                self.env_vars['WITH_LITTLEFS'] = "1"
+ 
     def write_board_validate_macro(self, f):
         '''write board validation macro'''
         validate_string = ''
@@ -2661,6 +2692,7 @@ Please run: Tools/scripts/build_bootloaders.py %s
         self.write_MAG_config(f)
         self.write_BARO_config(f)
         self.write_AIRSPEED_config(f)
+        self.write_DATAFLASH_config(f)
         self.write_board_validate_macro(f)
         self.write_check_firmware(f)
 
@@ -3088,6 +3120,8 @@ Please run: Tools/scripts/build_bootloaders.py %s
             self.compass_list.append(a[1:])
         elif a[0] == 'BARO':
             self.baro_list.append(a[1:])
+        elif a[0] == 'DATAFLASH':
+            self.dataflash_list.append(a[1:])
         elif a[0] == 'AIRSPEED':
             self.airspeed_list.append(a[1:])
         elif a[0] == 'ROMFS':
@@ -3127,6 +3161,8 @@ Please run: Tools/scripts/build_bootloaders.py %s
                     self.compass_list = []
                 if u == 'BARO':
                     self.baro_list = []
+                if u == 'DATAFLASH':
+                    self.dataflash_list = []
                 if u == 'AIRSPEED':
                     self.airspeed_list = []
                 if u == 'ROMFS':

libraries/AP_HAL_ChibiOS/hwdef/scripts/defaults_bootloader.h

@@ -62,3 +62,6 @@
 #define STM32_DMA_REQUIRED 1
 #endif
 
+#ifndef HAL_OS_POSIX_IO
+#define HAL_OS_POSIX_IO 0
+#endif

libraries/AP_HAL_ChibiOS/hwdef/scripts/defaults_iofirmware.h

@@ -77,3 +77,7 @@
 #ifndef HAL_GCS_ENABLED
 #define HAL_GCS_ENABLED 0
 #endif
+
+#ifndef HAL_OS_POSIX_IO
+#define HAL_OS_POSIX_IO 0
+#endif

libraries/AP_HAL_ChibiOS/hwdef/scripts/defaults_normal.h

@@ -20,6 +20,10 @@
 #define HAL_OS_LITTLEFS_IO 0
 #endif
 
+#ifndef HAL_OS_POSIX_IO
+#define HAL_OS_POSIX_IO 0
+#endif
+
 #ifndef AP_TERRAIN_AVAILABLE
 // enable terrain only if there's an SD card available:
 #define AP_TERRAIN_AVAILABLE (HAL_OS_FATFS_IO || (HAL_OS_LITTLEFS_IO && (BOARD_FLASH_SIZE>1024)))

libraries/AP_HAL_ChibiOS/hwdef/scripts/defaults_periph.h

@@ -472,3 +472,7 @@
 #ifndef AP_QUICKTUNE_ENABLED
 #define AP_QUICKTUNE_ENABLED 0
 #endif
+
+#ifndef HAL_OS_POSIX_IO
+#define HAL_OS_POSIX_IO 0
+#endif

libraries/AP_HAL_ChibiOS/sdcard.cpp

@@ -26,7 +26,7 @@
 
 extern const AP_HAL::HAL& hal;
 
-#ifdef USE_POSIX_FATFS
+#if HAL_USE_FATFS
 static FATFS SDC_FS; // FATFS object
 #ifndef HAL_BOOTLOADER_BUILD
 static HAL_Semaphore sem;
@@ -54,7 +54,7 @@ static SPIConfig highspeed;
  */
 bool sdcard_init()
 {
-#ifdef USE_POSIX_FATFS
+#if HAL_USE_FATFS
 #ifndef HAL_BOOTLOADER_BUILD
     WITH_SEMAPHORE(sem);
 
@@ -150,7 +150,7 @@ bool sdcard_init()
     }
 #endif
     sdcard_running = false;
-#endif  // USE_POSIX_FATFS
+#endif  // HAL_USE_FATFS
     return false;
 }
 
@@ -159,7 +159,7 @@ bool sdcard_init()
  */
 void sdcard_stop(void)
 {
-#ifdef USE_POSIX_FATFS
+#if HAL_USE_FATFS
     // unmount
     f_mount(nullptr, "/", 1);
 #endif
@@ -185,7 +185,7 @@ void sdcard_stop(void)
 
 bool sdcard_retry(void)
 {
-#ifdef USE_POSIX_FATFS
+#if HAL_USE_FATFS
     if (!sdcard_running) {
         if (sdcard_init()) {
 #if AP_FILESYSTEM_FILE_WRITING_ENABLED