DataFlash: ensure 10% free space when initialising logging

libraries/DataFlash/DataFlash.cpp

@@ -36,9 +36,13 @@ void DataFlash_Class::EraseAll() {
 bool DataFlash_Class::CardInserted(void) {
     return backend->CardInserted();
 }
-// remove me in favour of calling "DoTimeConsumingPreparations" all the time?
+
-bool DataFlash_Class::NeedErase(void) {
+bool DataFlash_Class::NeedPrep() {
-    return backend->NeedErase();
+    return backend->NeedPrep();
+}
+
+void DataFlash_Class::Prep() {
+    backend->Prep();
 }
 
 uint16_t DataFlash_Class::bufferspace_available(void) {

libraries/DataFlash/DataFlash.h

@@ -51,6 +51,10 @@ public:
     bool NeedErase(void);
     void EraseAll();
 
+    // possibly expensive calls to start log system:
+    bool NeedPrep();
+    void Prep();
+
     /* Write a block of data at current offset */
     bool WriteBlock(const void *pBuffer, uint16_t size);
     /* Write an *important* block of data at current offset */

libraries/DataFlash/DataFlash_Backend.h

@@ -26,9 +26,11 @@ public:
     virtual bool CardInserted(void) = 0;
 
     // erase handling
-    virtual bool NeedErase(void) = 0;
     virtual void EraseAll() = 0;
 
+    virtual bool NeedPrep() = 0;
+    virtual void Prep() = 0;
+
     /* Write a block of data at current offset */
     bool WriteBlock(const void *pBuffer, uint16_t size) {
         return WritePrioritisedBlock(pBuffer, size, false);

libraries/DataFlash/DataFlash_Block.cpp

@@ -199,6 +199,22 @@ void DataFlash_Block::EraseAll()
     hal.scheduler->delay(100);
 }
 
+bool DataFlash_Block::NeedPrep(void)
+{
+    return NeedErase();
+}
+
+void DataFlash_Block::Prep()
+{
+    if (hal.util->get_soft_armed()) {
+        // do not want to do any filesystem operations while we are e.g. flying
+        return;
+    }
+    if (NeedErase()) {
+        EraseAll();
+    }
+}
+
 /*
  *  we need to erase if the logging format has changed
  */

libraries/DataFlash/DataFlash_Block.h

@@ -22,9 +22,11 @@ public:
     virtual bool CardInserted(void) = 0;
 
     // erase handling
-    bool NeedErase(void);
     void EraseAll();
 
+    bool NeedPrep(void);
+    void Prep();
+
     /* Write a block of data at current offset */
     bool WritePrioritisedBlock(const void *pBuffer, uint16_t size, bool is_critical);
 
@@ -89,6 +91,9 @@ private:
     // start of the page
     virtual bool BlockRead(uint8_t BufferNum, uint16_t IntPageAdr, void *pBuffer, uint16_t size) = 0;
 
+    // erase handling
+    bool NeedErase(void);
+
     // internal high level functions
     void StartRead(uint16_t PageAdr);
     uint16_t find_last_page(void);

libraries/DataFlash/DataFlash_File.cpp

@@ -26,6 +26,7 @@
 #include <time.h>
 #include <dirent.h>
 #include <AP_HAL/utility/RingBuffer.h>
+#include <sys/statfs.h>
 
 extern const AP_HAL::HAL& hal;
 
@@ -110,7 +111,7 @@ void DataFlash_File::Init(const struct LogStructure *structure, uint8_t num_type
         ret = mkdir(_log_directory, 0777);
     }
     if (ret == -1) {
-        hal.console->printf("Failed to create log directory %s", _log_directory);
+        hal.console->printf("Failed to create log directory %s\n", _log_directory);
         return;
     }
     if (_writebuf != NULL) {
@@ -154,11 +155,135 @@ bool DataFlash_File::CardInserted(void)
     return _initialised && !_open_error;
 }
 
-
+uint64_t DataFlash_File::disk_space_avail()
-// erase handling
-bool DataFlash_File::NeedErase(void)
 {
-    // we could add a format marker at the start of a file?
+    struct statfs stats;
+    if (statfs(_log_directory, &stats) < 0) {
+        return -1;
+    }
+    return (((uint64_t)stats.f_bavail) * stats.f_bsize);
+}
+
+uint64_t DataFlash_File::disk_space()
+{
+    struct statfs stats;
+    if (statfs(_log_directory, &stats) < 0) {
+        return -1;
+    }
+    return (((uint64_t)stats.f_blocks) * stats.f_bsize);
+}
+
+float DataFlash_File::avail_space_percent()
+{
+    uint64_t avail = disk_space_avail();
+    uint64_t space = disk_space();
+
+    return (avail/(float)space) * 100;
+}
+
+// find_first_log - find lowest-numbered log in _log_directory
+// returns 0 if no log was found
+uint16_t DataFlash_File::find_first_log(void)
+{
+    // iterate through directory entries to find lowest log number.
+    // We could count up to find_last_log(), but if people start
+    // relying on the min_avail_space_percent feature we could end up
+    // doing a *lot* of asprintf()s and stat()s
+    DIR *d = opendir(_log_directory);
+    if (d == NULL) {
+        // internal_error();
+        return 0;
+    }
+
+    // we only remove files which look like xxx.BIN
+    uint32_t lowest_number = (uint32_t)-1; // unsigned integer wrap
+    for (struct dirent *de=readdir(d); de; de=readdir(d)) {
+        uint8_t length = strlen(de->d_name);
+        if (length < 5) {
+            // not long enough for \d+[.]BIN
+            continue;
+        }
+        if (strncmp(&de->d_name[length-4], ".BIN", 4)) {
+            // doesn't end in .BIN
+            continue;
+        }
+
+        uint16_t thisnum = strtoul(de->d_name, NULL, 10);
+        if (thisnum < lowest_number) {
+            lowest_number = thisnum;
+        }
+    }
+    closedir(d);
+
+    if (lowest_number == (uint32_t)-1) {
+        return 0;
+    }
+    return lowest_number;
+}
+
+void DataFlash_File::Prep_MinSpace()
+{
+    uint16_t log_to_remove = find_first_log();
+    if (log_to_remove == 0) {
+        // no files to remove
+        return;
+    }
+    uint16_t count = 0;
+    while (avail_space_percent() < min_avail_space_percent) {
+        if (count++ > 500) {
+            // *way* too many deletions going on here.  Possible internal error.
+            // deletion rate seems to be ~50 files/second.
+            // internal_error();
+            break;
+        }
+        char *filename_to_remove = _log_file_name(log_to_remove);
+        if (filename_to_remove == NULL) {
+            // internal_error();
+            break;
+        }
+        hal.console->printf("Removing (%s) for minimum-space requirements\n",
+                              filename_to_remove);
+        if (unlink(filename_to_remove) == -1) {
+            hal.console->printf("Failed to remove %s: (%d/%d) %s\n", filename_to_remove, errno, ENOENT, strerror(errno));
+            free(filename_to_remove);
+            if (errno == ENOENT) {
+                log_to_remove = find_first_log();
+                if (log_to_remove == 0) {
+                    // out of files to remove...
+                    break;
+                }
+                // corruption - should always have a continuous
+                // sequence of files...  however, we now have a file
+                // we can delete, so keep going.
+            } else {
+                break;
+            }
+        } else {
+            free(filename_to_remove);
+            log_to_remove++;
+        }
+    }
+}
+
+void DataFlash_File::Prep() {
+    if (hal.util->get_soft_armed()) {
+        // do not want to do any filesystem operations while we are e.g. flying
+        return;
+    }
+    Prep_MinSpace();
+}
+
+bool DataFlash_File::NeedPrep()
+{
+    if (!CardInserted()) {
+        // should not have been called?!
+        return false;
+    }
+
+    if (avail_space_percent() < min_avail_space_percent) {
+        return true;
+    }
+
     return false;
 }
 

libraries/DataFlash/DataFlash_File.h

@@ -10,6 +10,8 @@
 #ifndef DataFlash_File_h
 #define DataFlash_File_h
 
+#if HAL_OS_POSIX_IO
+
 #if CONFIG_HAL_BOARD == HAL_BOARD_PX4 || CONFIG_HAL_BOARD == HAL_BOARD_VRBRAIN
 #include <systemlib/perf_counter.h>
 #else
@@ -32,9 +34,12 @@ public:
     bool CardInserted(void);
 
     // erase handling
-    bool NeedErase(void);
     void EraseAll();
 
+    // possibly time-consuming preparation handling:
+    bool NeedPrep();
+    void Prep();
+
     /* Write a block of data at current offset */
     bool WritePrioritisedBlock(const void *pBuffer, uint16_t size, bool is_critical);
     uint16_t bufferspace_available();
@@ -75,6 +80,19 @@ private:
     */
     bool ReadBlock(void *pkt, uint16_t size);
 
+    // possibly time-consuming preparations handling
+    void Prep_MinSpace();
+    uint16_t find_first_log(void);
+    uint64_t disk_space_avail();
+    uint64_t disk_space();
+    float avail_space_percent();
+
+#if CONFIG_HAL_BOARD == HAL_BOARD_SITL || CONFIG_HAL_BOARD == HAL_BOARD_LINUX
+    // I always seem to have less than 10% free space on my laptop:
+    const float min_avail_space_percent = 0.1f;
+#else
+    const float min_avail_space_percent = 10.0f;
+#endif
     // write buffer
     uint8_t *_writebuf;
     uint16_t _writebuf_size;
@@ -107,6 +125,6 @@ private:
 #endif
 };
 
+#endif // HAL_OS_POSIX_IO
 
 #endif // DataFlash_File_h
-

libraries/DataFlash/examples/DataFlash_test/DataFlash_test.cpp

@@ -82,9 +82,9 @@ void DataFlashTest::setup(void)
     hal.scheduler->delay(20);
     dataflash.ShowDeviceInfo(hal.console);
 
-    if (dataflash.NeedErase()) {
+    if (dataflash.NeedPrep()) {
-        hal.console->println("Erasing...");
+        hal.console->println("Preparing dataflash...");
-        dataflash.EraseAll();
+        dataflash.Prep();
     }
 
     // We start to write some info (sequentialy) starting from page 1