AP_ROMFS: embed raw DEFLATE streams

Saves size not including unnecessary headers or code to parse them.

Tools/ardupilotwaf/embed.py

@@ -7,7 +7,7 @@ Andrew Tridgell
 May 2017
 '''
 
-import os, sys, tempfile, gzip
+import os, sys, zlib
 
 def write_encode(out, s):
     out.write(s.encode())
@@ -28,28 +28,24 @@ def embed_file(out, f, idx, embedded_name, uncompressed):
             contents += bytes([0xff]*pad)
             print("Padded %u bytes for %s to %u" % (pad, embedded_name, len(contents)))
 
-    crc = crc32(bytearray(contents))
+    crc = crc32(contents)
     write_encode(out, '__EXTFLASHFUNC__ static const uint8_t ap_romfs_%u[] = {' % idx)
 
-    compressed = tempfile.NamedTemporaryFile()
     if uncompressed:
         # ensure nul termination
         if contents[-1] != 0:
             contents += bytes([0])
-        compressed.write(contents)
+        b = contents
     else:
-        # compress it
-        f = open(compressed.name, "wb")
-        with gzip.GzipFile(fileobj=f, mode='wb', filename='', compresslevel=9, mtime=0) as g:
-            g.write(contents)
-        f.close()
+        # compress it (max level, max window size, raw stream, max mem usage)
+        z = zlib.compressobj(level=9, method=zlib.DEFLATED, wbits=-15, memLevel=9)
+        b = z.compress(contents)
+        b += z.flush()
+        # append uncompressed length as little-endian bytes
+        l = len(contents)
+        b += bytes([l & 0xFF, (l >> 8) & 0xFF, (l >> 16) & 0xFF, (l >> 24) & 0xFF])
 
-    compressed.seek(0)
-    b = bytearray(compressed.read())
-    compressed.close()
-    
-    for c in b:
-        write_encode(out, '%u,' % c)
+    write_encode(out, ",".join(str(c) for c in b))
     write_encode(out, '};\n\n');
     return crc
 

libraries/AP_ROMFS/AP_ROMFS.cpp

@@ -65,7 +65,7 @@ const uint8_t *AP_ROMFS::find_decompress(const char *name, uint32_t &size)
     size = compressed_size;
     return compressed_data;
 #else
-    // last 4 bytes of gzip file are length of decompressed data
+    // last 4 bytes of compressed data are length of decompressed data
     const uint8_t *p = &compressed_data[compressed_size-4];
     uint32_t decompressed_size = p[0] | p[1] << 8 | p[2] << 16 | p[3] << 24;
     
@@ -86,21 +86,10 @@ const uint8_t *AP_ROMFS::find_decompress(const char *name, uint32_t &size)
 
     d->source = compressed_data;
     d->source_limit = compressed_data + compressed_size - 4;
-
-    // assume gzip format
-    int res = uzlib_gzip_parse_header(d);
-    if (res != TINF_OK) {
-        ::free(decompressed_data);
-        ::free(d);
-        return nullptr;
-    }
-
     d->dest = decompressed_data;
     d->destSize = decompressed_size;
 
-    // we don't check CRC, as it just wastes flash space for constant
-    // ROMFS data
-    res = uzlib_uncompress(d);
+    int res = uzlib_uncompress(d);
 
     ::free(d);
     

libraries/AP_ROMFS/tinfgzip.cpp

@@ -1,110 +0,0 @@
-/*
- * tinfgzip  -  tiny gzip decompressor
- *
- * Copyright (c) 2003 by Joergen Ibsen / Jibz
- * All Rights Reserved
- *
- * http://www.ibsensoftware.com/
- *
- * Copyright (c) 2014-2016 by Paul Sokolovsky
- *
- * This software is provided 'as-is', without any express
- * or implied warranty.  In no event will the authors be
- * held liable for any damages arising from the use of
- * this software.
- *
- * Permission is granted to anyone to use this software
- * for any purpose, including commercial applications,
- * and to alter it and redistribute it freely, subject to
- * the following restrictions:
- *
- * 1. The origin of this software must not be
- *    misrepresented; you must not claim that you
- *    wrote the original software. If you use this
- *    software in a product, an acknowledgment in
- *    the product documentation would be appreciated
- *    but is not required.
- *
- * 2. Altered source versions must be plainly marked
- *    as such, and must not be misrepresented as
- *    being the original software.
- *
- * 3. This notice may not be removed or altered from
- *    any source distribution.
- */
-
-#include "tinf.h"
-
-#define FTEXT    1
-#define FHCRC    2
-#define FEXTRA   4
-#define FNAME    8
-#define FCOMMENT 16
-
-void tinf_skip_bytes(TINF_DATA *d, int num);
-uint16_t tinf_get_uint16(TINF_DATA *d);
-
-void tinf_skip_bytes(TINF_DATA *d, int num)
-{
-    while (num--) uzlib_get_byte(d);
-}
-
-uint16_t tinf_get_uint16(TINF_DATA *d)
-{
-    unsigned int v = uzlib_get_byte(d);
-    v = (uzlib_get_byte(d) << 8) | v;
-    return v;
-}
-
-int uzlib_gzip_parse_header(TINF_DATA *d)
-{
-    unsigned char flg;
-
-    /* -- check format -- */
-
-    /* check id bytes */
-    if (uzlib_get_byte(d) != 0x1f || uzlib_get_byte(d) != 0x8b) return TINF_DATA_ERROR;
-
-    /* check method is deflate */
-    if (uzlib_get_byte(d) != 8) return TINF_DATA_ERROR;
-
-    /* get flag byte */
-    flg = uzlib_get_byte(d);
-
-    /* check that reserved bits are zero */
-    if (flg & 0xe0) return TINF_DATA_ERROR;
-
-    /* -- find start of compressed data -- */
-
-    /* skip rest of base header of 10 bytes */
-    tinf_skip_bytes(d, 6);
-
-    /* skip extra data if present */
-    if (flg & FEXTRA)
-    {
-       unsigned int xlen = tinf_get_uint16(d);
-       tinf_skip_bytes(d, xlen);
-    }
-
-    /* skip file name if present */
-    if (flg & FNAME) { while (uzlib_get_byte(d)); }
-
-    /* skip file comment if present */
-    if (flg & FCOMMENT) { while (uzlib_get_byte(d)); }
-
-    /* check header crc if present */
-    if (flg & FHCRC)
-    {
-       /*unsigned int hcrc =*/ tinf_get_uint16(d);
-
-        // TODO: Check!
-//       if (hcrc != (tinf_crc32(src, start - src) & 0x0000ffff))
-//          return TINF_DATA_ERROR;
-    }
-
-    /* initialize for crc32 checksum */
-    d->checksum_type = TINF_CHKSUM_CRC;
-    d->checksum = ~0;
-
-    return TINF_OK;
-}