From f5e405ef5b06f4bf26cf3758d962aa7884a7d94c Mon Sep 17 00:00:00 2001 From: Anton Babushkin Date: Sun, 26 May 2013 20:15:46 +0400 Subject: [PATCH] sdlog2_dump.py now generates customizible CSV output, messages/fields filter added. --- Tools/sdlog2_dump.py | 205 +++++++++++++++++++++++++++++++++---------- 1 file changed, 159 insertions(+), 46 deletions(-) diff --git a/Tools/sdlog2_dump.py b/Tools/sdlog2_dump.py index b74bf47396..c8fc00a598 100644 --- a/Tools/sdlog2_dump.py +++ b/Tools/sdlog2_dump.py @@ -1,11 +1,21 @@ #!/usr/bin/env python -"""Parse and dump binary log generated by sdlog2 +"""Dump binary log generated by sdlog2 or APM as CSV - Usage: python sdlog2_dump.py """ +Usage: python sdlog2_dump.py [-v] [-d delimiter] [-n null] [-m MSG[.field1,field2,...]] + + -v Use plain debug output instead of CSV. + + -d Use "delimiter" in CSV. Default is ",". + + -n Use "null" as placeholder for empty values in CSV. Default is empty. + + -m MSG[.field1,field2,...] + Dump only messages of specified type, and only specified fields. + Multiple -m options allowed.""" __author__ = "Anton Babushkin" -__version__ = "0.2" +__version__ = "1.0" import struct, sys @@ -40,54 +50,83 @@ class SDLog2Parser: "q": ("q", None), "Q": ("Q", None), } + __csv_delim = "," + __csv_null = "" + __msg_filter = {} + __debug_out = False def __init__(self): return def reset(self): - self.msg_descrs = {} - self.buffer = "" - self.ptr = 0 + self.__msg_descrs = {} + self.__buffer = "" + self.__ptr = 0 + self.__csv_columns = [] + self.__csv_data = {} + + def setCSVDelimiter(self, csv_delim): + self.__csv_delim = csv_delim + + def setCSVNull(self, csv_null): + self.__csv_null = csv_null + + def setMsgFilter(self, msg_filter): + self.__msg_filter = msg_filter + + def setDebugOut(self, debug_out): + self.__debug_out = debug_out def process(self, fn): self.reset() + first_data_msg = True f = open(fn, "r") while True: chunk = f.read(self.BLOCK_SIZE) if len(chunk) == 0: break - self.buffer = self.buffer[self.ptr:] + chunk - self.ptr = 0 - while self._bytes_left() >= self.MSG_HEADER_LEN: - head1 = ord(self.buffer[self.ptr]) - head2 = ord(self.buffer[self.ptr+1]) + self.__buffer = self.__buffer[self.__ptr:] + chunk + self.__ptr = 0 + while self.__bytesLeft() >= self.MSG_HEADER_LEN: + head1 = ord(self.__buffer[self.__ptr]) + head2 = ord(self.__buffer[self.__ptr+1]) if (head1 != self.MSG_HEAD1 or head2 != self.MSG_HEAD2): raise Exception("Invalid header: %02X %02X, must be %02X %02X" % (head1, head2, self.MSG_HEAD1, self.MSG_HEAD2)) - msg_type = ord(self.buffer[self.ptr+2]) + msg_type = ord(self.__buffer[self.__ptr+2]) if msg_type == self.MSG_TYPE_FORMAT: - self._parse_msg_descr() + self.__parseMsgDescr() else: - msg_descr = self.msg_descrs[msg_type] + msg_descr = self.__msg_descrs[msg_type] if msg_descr == None: raise Exception("Unknown msg type: %i" % msg_type) msg_length = msg_descr[0] - if self._bytes_left() < msg_length: + if self.__bytesLeft() < msg_length: break - self._parse_msg(msg_descr) + if first_data_msg: + print self.__csv_delim.join(self.__csv_columns) + first_data_msg = False + self.__parseMsg(msg_descr) + f.close() - def _bytes_left(self): - return len(self.buffer) - self.ptr + def __bytesLeft(self): + return len(self.__buffer) - self.__ptr - def _parse_msg_descr(self): - if self._bytes_left() < self.MSG_FORMAT_PACKET_LEN: - raise BufferUnderflow("Data is too short: %i bytes, need %i" % (self._bytes_left(), self.MSG_FORMAT_PACKET_LEN)) - data = struct.unpack(self.MSG_FORMAT_STRUCT, self.buffer[self.ptr + 3 : self.ptr + self.MSG_FORMAT_PACKET_LEN]) + def __filterMsg(self, msg_name): + show_fields = "*" + if len(self.__msg_filter) > 0: + show_fields = self.__msg_filter.get(msg_name) + return show_fields + + def __parseMsgDescr(self): + if self.__bytesLeft() < self.MSG_FORMAT_PACKET_LEN: + raise BufferUnderflow("Data is too short: %i bytes, need %i" % (self.__bytesLeft(), self.MSG_FORMAT_PACKET_LEN)) + data = struct.unpack(self.MSG_FORMAT_STRUCT, self.__buffer[self.__ptr + 3 : self.__ptr + self.MSG_FORMAT_PACKET_LEN]) msg_type = data[0] msg_length = data[1] - msg_name = data[2].strip('\0') - msg_format = data[3].strip('\0') - msg_labels = data[4].strip('\0').split(",") + msg_name = data[2].strip("\0") + msg_format = data[3].strip("\0") + msg_labels = data[4].strip("\0").split(",") # Convert msg_format to struct.unpack format string msg_struct = "" msg_mults = [] @@ -97,34 +136,108 @@ class SDLog2Parser: msg_struct += f[0] msg_mults.append(f[1]) except KeyError as e: - raise Exception("Unsupported format char: %s in message %s (0x%02X)" % (c, msg_name, msg_type)) - msg_struct = "<" + msg_struct - print msg_format, msg_struct - print "MSG FORMAT: type = %i, length = %i, name = %s, format = %s, labels = %s, struct = %s, mults = %s" % (msg_type, msg_length, msg_name, msg_format, str(msg_labels), msg_struct, msg_mults) - self.msg_descrs[msg_type] = (msg_length, msg_name, msg_format, msg_labels, msg_struct, msg_mults) - self.ptr += self.MSG_FORMAT_PACKET_LEN + raise Exception("Unsupported format char: %s in message %s (%i)" % (c, msg_name, msg_type)) + msg_struct = "<" + msg_struct # force little-endian + self.__msg_descrs[msg_type] = (msg_length, msg_name, msg_format, msg_labels, msg_struct, msg_mults) + show_fields = self.__filterMsg(msg_name) + if show_fields != None: + if self.__debug_out: + print "MSG FORMAT: type = %i, length = %i, name = %s, format = %s, labels = %s, struct = %s, mults = %s" % ( + msg_type, msg_length, msg_name, msg_format, str(msg_labels), msg_struct, msg_mults) + else: + if show_fields == "*": + fields = msg_labels + else: + fields = [] + for field in show_fields: + if field in msg_labels: + fields.append(field) + for field in fields: + msg_field = msg_name + "." + field + self.__csv_columns.append(msg_field) + self.__csv_data[msg_field] = None + self.__ptr += self.MSG_FORMAT_PACKET_LEN - def _parse_msg(self, msg_descr): + def __parseMsg(self, msg_descr): msg_length, msg_name, msg_format, msg_labels, msg_struct, msg_mults = msg_descr - data = list(struct.unpack(msg_struct, self.buffer[self.ptr+self.MSG_HEADER_LEN:self.ptr+msg_length])) - s = [] - for i in xrange(len(data)): - if type(data[i]) is str: - data[i] = data[i].strip('\0') - m = msg_mults[i] - if m != None: - data[i] = data[i] * m - s.append(msg_labels[i] + "=" + str(data[i])) - - print "MSG %s: %s" % (msg_name, ", ".join(s)) - self.ptr += msg_length - + show_fields = self.__filterMsg(msg_name) + if (show_fields != None): + data = list(struct.unpack(msg_struct, self.__buffer[self.__ptr+self.MSG_HEADER_LEN:self.__ptr+msg_length])) + for i in xrange(len(data)): + if type(data[i]) is str: + data[i] = data[i].strip("\0") + m = msg_mults[i] + if m != None: + data[i] = data[i] * m + if self.__debug_out: + s = [] + for i in xrange(len(data)): + label = msg_labels[i] + if show_fields == "*" or label in show_fields: + s.append(label + "=" + str(data[i])) + print "MSG %s: %s" % (msg_name, ", ".join(s)) + else: + # update CSV data buffer + for i in xrange(len(data)): + label = msg_labels[i] + if show_fields == "*" or label in show_fields: + self.__csv_data[msg_name + "." + label] = data[i] + # format and print CSV row + s = [] + for field in self.__csv_columns: + v = self.__csv_data[field] + if v == None: + v = self.__csv_null + else: + v = str(v) + s.append(v) + print self.__csv_delim.join(s) + self.__ptr += msg_length + def _main(): if len(sys.argv) < 2: - print "Usage:\npython sdlog2_dump.py " + print "Usage: python sdlog2_dump.py [-v] [-d delimiter] [-n null] [-m MSG[.field1,field2,...]]\n" + print "\t-v\tUse plain debug output instead of CSV.\n" + print "\t-d\tUse \"delimiter\" in CSV. Default is \",\".\n" + print "\t-n\tUse \"null\" as placeholder for empty values in CSV. Default is empty.\n" + print "\t-m MSG[.field1,field2,...]\n\t\tDump only messages of specified type, and only specified fields.\n\t\tMultiple -m options allowed." return fn = sys.argv[1] + debug_out = False + msg_filter = {} + csv_null = "" + csv_delim = "," + opt = None + for arg in sys.argv[2:]: + if opt != None: + if opt == "d": + csv_delim = arg + elif opt == "n": + csv_null = arg + if opt == "m": + show_fields = "*" + a = arg.split(".") + if len(a) > 1: + show_fields = set(a[1].split(",")) + msg_filter[a[0]] = show_fields + opt = None + else: + if arg == "-v": + debug_out = True + elif arg == "-d": + opt = "d" + elif arg == "-n": + opt = "n" + elif arg == "-m": + opt = "m" + + if csv_delim == "\\t": + csv_delim = "\t" parser = SDLog2Parser() + parser.setCSVDelimiter(csv_delim) + parser.setCSVNull(csv_null) + parser.setMsgFilter(msg_filter) + parser.setDebugOut(debug_out) parser.process(fn) if __name__ == "__main__":