BLF and ASC log file fixes

can/io/asc.py

@@ -56,12 +56,18 @@ def __iter__(self):
                 continue
 
             timestamp = float(timestamp)
+            try:
+                # See ASCWriter
+                channel = int(channel) - 1
+            except ValueError:
+                pass
 
             if dummy.strip()[0:10] == 'ErrorFrame':
-                msg = Message(timestamp=timestamp, is_error_frame=True)
+                msg = Message(timestamp=timestamp, is_error_frame=True,
+                              channel=channel)
                 yield msg
 
-            elif not channel.isdigit() or dummy.strip()[0:10] == 'Statistic:':
+            elif not isinstance(channel, int) or dummy.strip()[0:10] == 'Statistic:':
                 pass
 
             elif dummy[-1:].lower() == 'r':
@@ -70,7 +76,8 @@ def __iter__(self):
                 msg = Message(timestamp=timestamp,
                               arbitration_id=can_id_num & CAN_ID_MASK,
                               extended_id=is_extended_id,
-                              is_remote_frame=True)
+                              is_remote_frame=True,
+                              channel=channel)
                 yield msg
 
             else:
@@ -97,7 +104,8 @@ def __iter__(self):
                             extended_id=is_extended_id,
                             is_remote_frame=False,
                             dlc=dlc,
-                            data=frame)
+                            data=frame,
+                            channel=channel)
                 yield msg
 
 
@@ -189,7 +197,8 @@ def on_message_received(self, msg):
         if msg.is_extended_id:
             arb_id += 'x'
 
-        channel = msg.channel if isinstance(msg.channel, int) else self.channel
+        # Many interfaces start channel numbering at 0 which is invalid
+        channel = msg.channel + 1 if isinstance(msg.channel, int) else self.channel
 
         serialized = self.FORMAT_MESSAGE.format(channel=channel,
                                                 id=arb_id,

can/io/blf.py

@@ -22,6 +22,7 @@
 
 from can.message import Message
 from can.listener import Listener
+from can.util import len2dlc, dlc2len
 
 
 # 0 = unknown, 2 = CANoe
@@ -35,9 +36,16 @@
 # time start (SYSTEMTIME), time stop (SYSTEMTIME)
 FILE_HEADER_STRUCT = struct.Struct("<4sLBBBBBBBBQQLL8H8H72x")
 
-# signature ("LOBJ"), header size, header version (1), object size, object type,
-# flags, object version, size uncompressed or timestamp
-OBJ_HEADER_STRUCT = struct.Struct("<4sHHLLL2xHQ")
+# signature ("LOBJ"), header size, header version (1), object size, object type
+OBJ_HEADER_BASE_STRUCT = struct.Struct("<4sHHLL")
+
+# flags, client index, object version, timestamp
+OBJ_HEADER_STRUCT = struct.Struct("<LHHQ")
+
+HEADER_SIZE = OBJ_HEADER_BASE_STRUCT.size + OBJ_HEADER_STRUCT.size
+
+# compression method, size uncompressed
+LOG_CONTAINER_STRUCT = struct.Struct("<H6xL4x")
 
 # channel, flags, dlc, arbitration id, data
 CAN_MSG_STRUCT = struct.Struct("<HBBL8s")
@@ -46,8 +54,8 @@
 # valid data bytes, data
 CAN_FD_MSG_STRUCT = struct.Struct("<HBBLLBBB5x64s")
 
-# channel, length
-CAN_ERROR_STRUCT = struct.Struct("<HH4x")
+# channel, length, flags, ecc, position, dlc, frame length, id, flags ext, data
+CAN_ERROR_EXT_STRUCT = struct.Struct("<HHLBBBxLLH2x8s")
 
 # commented event type, foreground color, background color, relocatable,
 # group name length, marker name length, description length
@@ -57,9 +65,13 @@
 CAN_MESSAGE = 1
 CAN_ERROR = 2
 LOG_CONTAINER = 10
+CAN_ERROR_EXT = 73
 GLOBAL_MARKER = 96
 CAN_FD_MESSAGE = 100
 
+NO_COMPRESSION = 0
+ZLIB_DEFLATE = 2
+
 CAN_MSG_EXT = 0x80000000
 REMOTE_FLAG = 0x80
 EDL = 0x1
@@ -109,75 +121,96 @@ def __init__(self, filename):
     def __iter__(self):
         tail = b""
         while True:
-            data = self.fp.read(OBJ_HEADER_STRUCT.size)
+            data = self.fp.read(OBJ_HEADER_BASE_STRUCT.size)
             if not data:
                 # EOF
                 break
-            header = OBJ_HEADER_STRUCT.unpack(data)
+
+            header = OBJ_HEADER_BASE_STRUCT.unpack(data)
             #print(header)
             assert header[0] == b"LOBJ", "Parse error"
             obj_type = header[4]
-            obj_data_size = header[3] - OBJ_HEADER_STRUCT.size
+            obj_data_size = header[3] - OBJ_HEADER_BASE_STRUCT.size
             obj_data = self.fp.read(obj_data_size)
             # Read padding bytes
             self.fp.read(obj_data_size % 4)
+
             if obj_type == LOG_CONTAINER:
-                uncompressed_size = header[7]
-                data = zlib.decompress(obj_data, 15, uncompressed_size)
+                method, uncompressed_size = LOG_CONTAINER_STRUCT.unpack_from(
+                    obj_data)
+                container_data = obj_data[LOG_CONTAINER_STRUCT.size:]
+                if method == NO_COMPRESSION:
+                    data = container_data
+                elif method == ZLIB_DEFLATE:
+                    data = zlib.decompress(container_data, 15, uncompressed_size)
+                else:
+                    # Unknown compression method
+                    continue
+
                 if tail:
                     data = tail + data
                 pos = 0
-                while pos + OBJ_HEADER_STRUCT.size < len(data):
-                    header = OBJ_HEADER_STRUCT.unpack_from(data, pos)
+                while pos + OBJ_HEADER_BASE_STRUCT.size < len(data):
+                    header = OBJ_HEADER_BASE_STRUCT.unpack_from(data, pos)
                     #print(header)
                     assert header[0] == b"LOBJ", "Parse error"
                     obj_size = header[3]
                     if pos + obj_size > len(data):
                         # Object continues in next log container
                         break
+                    pos += OBJ_HEADER_BASE_STRUCT.size
+                    # Read rest of header
+                    header += OBJ_HEADER_STRUCT.unpack_from(data, pos)
+                    pos += OBJ_HEADER_STRUCT.size
+
                     obj_type = header[4]
-                    timestamp = header[7] / 1000000000.0 + self.start_timestamp
+                    timestamp = header[8] * 1e-9 + self.start_timestamp
+
                     if obj_type == CAN_MESSAGE:
-                        assert obj_size == OBJ_HEADER_STRUCT.size + CAN_MSG_STRUCT.size
                         (channel, flags, dlc, can_id,
-                         can_data) = CAN_MSG_STRUCT.unpack_from(
-                             data, pos + OBJ_HEADER_STRUCT.size)
+                         can_data) = CAN_MSG_STRUCT.unpack_from(data, pos)
                         msg = Message(timestamp=timestamp,
                                       arbitration_id=can_id & 0x1FFFFFFF,
                                       extended_id=bool(can_id & CAN_MSG_EXT),
                                       is_remote_frame=bool(flags & REMOTE_FLAG),
                                       dlc=dlc,
                                       data=can_data[:dlc],
-                                      channel=channel)
+                                      channel=channel - 1)
                         yield msg
                     elif obj_type == CAN_FD_MESSAGE:
-                        assert obj_size == OBJ_HEADER_STRUCT.size + CAN_FD_MSG_STRUCT.size
                         (channel, flags, dlc, can_id, _, _, fd_flags,
-                         valid_bytes, can_data) = CAN_FD_MSG_STRUCT.unpack_from(
-                             data, pos + OBJ_HEADER_STRUCT.size)
+                         _, can_data) = CAN_FD_MSG_STRUCT.unpack_from(data, pos)
+                        length = dlc2len(dlc)
                         msg = Message(timestamp=timestamp,
                                       arbitration_id=can_id & 0x1FFFFFFF,
                                       extended_id=bool(can_id & CAN_MSG_EXT),
                                       is_remote_frame=bool(flags & REMOTE_FLAG),
                                       is_fd=bool(fd_flags & EDL),
                                       bitrate_switch=bool(fd_flags & BRS),
                                       error_state_indicator=bool(fd_flags & ESI),
-                                      dlc=dlc,
-                                      data=can_data[:valid_bytes],
-                                      channel=channel)
+                                      dlc=length,
+                                      data=can_data[:length],
+                                      channel=channel - 1)
                         yield msg
-                    elif obj_type == CAN_ERROR:
-                        assert obj_size == OBJ_HEADER_STRUCT.size + CAN_ERROR_STRUCT.size
-                        channel, length = CAN_ERROR_STRUCT.unpack_from(
-                            data, pos + OBJ_HEADER_STRUCT.size)
-                        msg = Message(timestamp=timestamp, is_error_frame=True,
-                                      channel=channel)
+                    elif obj_type == CAN_ERROR_EXT:
+                        (channel, _, _, _, _, dlc, _, can_id, _,
+                         can_data) = CAN_ERROR_EXT_STRUCT.unpack_from(data, pos)
+                        msg = Message(timestamp=timestamp,
+                                      is_error_frame=True,
+                                      extended_id=bool(can_id & CAN_MSG_EXT),
+                                      arbitration_id=can_id & 0x1FFFFFFF,
+                                      dlc=dlc,
+                                      data=can_data[:dlc],
+                                      channel=channel - 1)
                         yield msg
-                    pos += obj_size
+
+                    pos += obj_size - HEADER_SIZE
                     # Add padding bytes
                     pos += obj_size % 4
+
                 # Save remaing data that could not be processed
                 tail = data[pos:]
+
         self.fp.close()
 
 
@@ -187,7 +220,7 @@ class BLFWriter(Listener):
     """
 
     #: Max log container size of uncompressed data
-    MAX_CACHE_SIZE = 0x20000
+    MAX_CACHE_SIZE = 128 * 1024
 
     #: ZLIB compression level
     COMPRESSION_LEVEL = 9
@@ -205,29 +238,38 @@ def __init__(self, filename, channel=1):
         self.stop_timestamp = None
 
     def on_message_received(self, msg):
-        channel = msg.channel if isinstance(msg.channel, int) else self.channel
+        # Many interfaces start channel numbering at 0 which is invalid in BLF
+        channel = msg.channel + 1 if isinstance(msg.channel, int) else self.channel
+        arb_id = msg.arbitration_id
+        if msg.id_type:
+            arb_id |= CAN_MSG_EXT
+        flags = REMOTE_FLAG if msg.is_remote_frame else 0
+        data = bytes(msg.data)
+
         if msg.is_error_frame:
-            data = CAN_ERROR_STRUCT.pack(channel, 0)
-            self._add_object(CAN_ERROR, data, msg.timestamp)
+            data = CAN_ERROR_EXT_STRUCT.pack(channel,
+                                             0,     # length
+                                             0,     # flags
+                                             0,     # ecc
+                                             0,     # position
+                                             len2dlc(msg.dlc),
+                                             0,     # frame length
+                                             arb_id,
+                                             0,     # ext flags
+                                             data)
+            self._add_object(CAN_ERROR_EXT, data, msg.timestamp)
+        elif msg.is_fd:
+            fd_flags = EDL
+            if msg.bitrate_switch:
+                fd_flags |= BRS
+            if msg.error_state_indicator:
+                fd_flags |= ESI
+            data = CAN_FD_MSG_STRUCT.pack(channel, flags, len2dlc(msg.dlc),
+                                          arb_id, 0, 0, fd_flags, msg.dlc, data)
+            self._add_object(CAN_FD_MESSAGE, data, msg.timestamp)
         else:
-            flags = REMOTE_FLAG if msg.is_remote_frame else 0
-            arb_id = msg.arbitration_id
-            if msg.id_type:
-                arb_id |= CAN_MSG_EXT
-            if msg.is_fd:
-                fd_flags = EDL
-                if msg.bitrate_switch:
-                    fd_flags |= BRS
-                if msg.error_state_indicator:
-                    fd_flags |= ESI
-                data = CAN_FD_MSG_STRUCT.pack(channel, flags, msg.dlc, arb_id,
-                                              0, 0, fd_flags, msg.dlc,
-                                              bytes(msg.data))
-                self._add_object(CAN_FD_MESSAGE, data, msg.timestamp)
-            else:
-                data = CAN_MSG_STRUCT.pack(channel, flags, msg.dlc, arb_id,
-                                           bytes(msg.data))
-                self._add_object(CAN_MESSAGE, data, msg.timestamp)
+            data = CAN_MSG_STRUCT.pack(channel, flags, msg.dlc, arb_id, data)
+            self._add_object(CAN_MESSAGE, data, msg.timestamp)
 
     def log_event(self, text, timestamp=None):
         """Add an arbitrary message to the log file as a global marker.
@@ -255,16 +297,19 @@ def _add_object(self, obj_type, data, timestamp=None):
         if self.start_timestamp is None:
             self.start_timestamp = timestamp
         self.stop_timestamp = timestamp
-        timestamp = int((timestamp - self.start_timestamp) * 1000000000)
-        obj_size = OBJ_HEADER_STRUCT.size + len(data)
-        header = OBJ_HEADER_STRUCT.pack(
-            b"LOBJ", OBJ_HEADER_STRUCT.size, 1, obj_size, obj_type,
-            2, 0, max(timestamp, 0))
-        self.cache.append(header)
+        timestamp = int((timestamp - self.start_timestamp) * 1e9)
+        obj_size = HEADER_SIZE + len(data)
+        base_header = OBJ_HEADER_BASE_STRUCT.pack(
+            b"LOBJ", HEADER_SIZE, 1, obj_size, obj_type)
+        obj_header = OBJ_HEADER_STRUCT.pack(2, 0, 0, max(timestamp, 0))
+
+        self.cache.append(base_header)
+        self.cache.append(obj_header)
         self.cache.append(data)
         padding_size = len(data) % 4
         if padding_size:
             self.cache.append(b"\x00" * padding_size)
+
         self.cache_size += obj_size + padding_size
         self.count_of_objects += 1
         if self.cache_size >= self.MAX_CACHE_SIZE:
@@ -285,14 +330,19 @@ def _flush(self):
         self.cache_size = len(tail)
         compressed_data = zlib.compress(uncompressed_data,
                                         self.COMPRESSION_LEVEL)
-        obj_size = OBJ_HEADER_STRUCT.size + len(compressed_data)
-        header = OBJ_HEADER_STRUCT.pack(
-            b"LOBJ", 16, 1, obj_size, LOG_CONTAINER, 2, 0, len(uncompressed_data))
-        self.fp.write(header)
+        obj_size = (OBJ_HEADER_STRUCT.size + LOG_CONTAINER_STRUCT.size +
+                    len(compressed_data))
+        base_header = OBJ_HEADER_BASE_STRUCT.pack(
+            b"LOBJ", OBJ_HEADER_BASE_STRUCT.size, 1, obj_size, LOG_CONTAINER)
+        container_header = LOG_CONTAINER_STRUCT.pack(
+            ZLIB_DEFLATE, len(uncompressed_data))
+        self.fp.write(base_header)
+        self.fp.write(container_header)
         self.fp.write(compressed_data)
         # Write padding bytes
         self.fp.write(b"\x00" * (obj_size % 4))
-        self.uncompressed_size += len(uncompressed_data) + OBJ_HEADER_STRUCT.size
+        self.uncompressed_size += OBJ_HEADER_STRUCT.size + LOG_CONTAINER_STRUCT.size
+        self.uncompressed_size += len(uncompressed_data)
 
     def stop(self):
         """Stops logging and closes the file."""

can/util.py

@@ -23,6 +23,12 @@
 
 log = logging.getLogger('can.util')
 
+# List of valid data lengths for a CAN FD message
+CAN_FD_DLC = [
+    0, 1, 2, 3, 4, 5, 6, 7, 8,
+    12, 16, 20, 24, 32, 48, 64
+]
+
 REQUIRED_KEYS = [
     'interface',
     'channel',
@@ -209,6 +215,33 @@ def set_logging_level(level_name=None):
     log.debug("Logging set to {}".format(level_name))
 
 
+def len2dlc(length):
+    """Calculate the DLC from data length.
+
+    :param int length: Length in number of bytes (0-64)
+
+    :returns: DLC (0-15)
+    :rtype: int
+    """
+    if length <= 8:
+        return length
+    for dlc, nof_bytes in enumerate(CAN_FD_DLC):
+        if nof_bytes >= length:
+            return dlc
+    return 15
+
+
+def dlc2len(dlc):
+    """Calculate the data length from DLC.
+
+    :param int dlc: DLC (0-15)
+
+    :returns: Data length in number of bytes (0-64)
+    :rtype: int
+    """
+    return CAN_FD_DLC[dlc] if dlc <= 15 else 64
+
+
 if __name__ == "__main__":
     print("Searching for configuration named:")
     print("\n".join(CONFIG_FILES))