From 30d17acb664da736f9741015d524950914201647 Mon Sep 17 00:00:00 2001 From: Christian Sandberg Date: Tue, 27 Mar 2018 19:52:54 +0200 Subject: [PATCH 1/3] BLF improvements Add 1 to channel numbers (to avoid use of 0) Better handling of error frames Better handling of CAN FD frames Fixes #279 --- can/io/blf.py | 176 ++++++++++++++++++++++++++-------------- can/util.py | 32 ++++++++ test/logformats_test.py | 7 +- 3 files changed, 155 insertions(+), 60 deletions(-) diff --git a/can/io/blf.py b/can/io/blf.py index 3b96a621a..fb25527f8 100644 --- a/can/io/blf.py +++ b/can/io/blf.py @@ -22,6 +22,7 @@ from can.message import Message from can.listener import Listener +from can.util import len2dlc, dlc2len # 0 = unknown, 2 = CANoe @@ -35,9 +36,16 @@ # time start (SYSTEMTIME), time stop (SYSTEMTIME) FILE_HEADER_STRUCT = struct.Struct("<4sLBBBBBBBBQQLL8H8H72x") -# signature ("LOBJ"), header size, header version (1), object size, object type, -# flags, object version, size uncompressed or timestamp -OBJ_HEADER_STRUCT = struct.Struct("<4sHHLLL2xHQ") +# signature ("LOBJ"), header size, header version (1), object size, object type +OBJ_HEADER_BASE_STRUCT = struct.Struct("<4sHHLL") + +# flags, client index, object version, timestamp +OBJ_HEADER_STRUCT = struct.Struct(" len(data): # Object continues in next log container break + pos += OBJ_HEADER_BASE_STRUCT.size + # Read rest of header + header += OBJ_HEADER_STRUCT.unpack_from(data, pos) + pos += OBJ_HEADER_STRUCT.size + obj_type = header[4] - timestamp = header[7] / 1000000000.0 + self.start_timestamp + timestamp = header[8] * 1e-9 + self.start_timestamp + if obj_type == CAN_MESSAGE: - assert obj_size == OBJ_HEADER_STRUCT.size + CAN_MSG_STRUCT.size (channel, flags, dlc, can_id, - can_data) = CAN_MSG_STRUCT.unpack_from( - data, pos + OBJ_HEADER_STRUCT.size) + can_data) = CAN_MSG_STRUCT.unpack_from(data, pos) msg = Message(timestamp=timestamp, arbitration_id=can_id & 0x1FFFFFFF, extended_id=bool(can_id & CAN_MSG_EXT), is_remote_frame=bool(flags & REMOTE_FLAG), dlc=dlc, data=can_data[:dlc], - channel=channel) + channel=channel - 1) yield msg elif obj_type == CAN_FD_MESSAGE: - assert obj_size == OBJ_HEADER_STRUCT.size + CAN_FD_MSG_STRUCT.size (channel, flags, dlc, can_id, _, _, fd_flags, - valid_bytes, can_data) = CAN_FD_MSG_STRUCT.unpack_from( - data, pos + OBJ_HEADER_STRUCT.size) + _, can_data) = CAN_FD_MSG_STRUCT.unpack_from(data, pos) + length = dlc2len(dlc) msg = Message(timestamp=timestamp, arbitration_id=can_id & 0x1FFFFFFF, extended_id=bool(can_id & CAN_MSG_EXT), @@ -162,22 +191,34 @@ def __iter__(self): is_fd=bool(fd_flags & EDL), bitrate_switch=bool(fd_flags & BRS), error_state_indicator=bool(fd_flags & ESI), - dlc=dlc, - data=can_data[:valid_bytes], - channel=channel) + dlc=length, + data=can_data[:length], + channel=channel - 1) yield msg elif obj_type == CAN_ERROR: - assert obj_size == OBJ_HEADER_STRUCT.size + CAN_ERROR_STRUCT.size - channel, length = CAN_ERROR_STRUCT.unpack_from( - data, pos + OBJ_HEADER_STRUCT.size) + channel, _ = CAN_ERROR_STRUCT.unpack_from(data, pos) msg = Message(timestamp=timestamp, is_error_frame=True, - channel=channel) + channel=channel - 1) yield msg - pos += obj_size + elif obj_type == CAN_ERROR_EXT: + (channel, _, _, _, _, dlc, _, can_id, _, + can_data) = CAN_ERROR_EXT_STRUCT.unpack_from(data, pos) + msg = Message(timestamp=timestamp, + is_error_frame=True, + extended_id=bool(can_id & CAN_MSG_EXT), + arbitration_id=can_id & 0x1FFFFFFF, + dlc=dlc, + data=can_data[:dlc], + channel=channel - 1) + yield msg + + pos += obj_size - HEADER_SIZE # Add padding bytes pos += obj_size % 4 + # Save remaing data that could not be processed tail = data[pos:] + self.fp.close() @@ -187,7 +228,7 @@ class BLFWriter(Listener): """ #: Max log container size of uncompressed data - MAX_CACHE_SIZE = 0x20000 + MAX_CACHE_SIZE = 128 * 1024 #: ZLIB compression level COMPRESSION_LEVEL = 9 @@ -205,29 +246,38 @@ def __init__(self, filename, channel=1): self.stop_timestamp = None def on_message_received(self, msg): - channel = msg.channel if isinstance(msg.channel, int) else self.channel + # Many interfaces start channel numbering at 0 which is invalid in BLF + channel = msg.channel + 1 if isinstance(msg.channel, int) else self.channel + arb_id = msg.arbitration_id + if msg.id_type: + arb_id |= CAN_MSG_EXT + flags = REMOTE_FLAG if msg.is_remote_frame else 0 + data = bytes(msg.data) + if msg.is_error_frame: - data = CAN_ERROR_STRUCT.pack(channel, 0) - self._add_object(CAN_ERROR, data, msg.timestamp) + data = CAN_ERROR_EXT_STRUCT.pack(channel, + 0, # length + 0, # flags + 0, # ecc + 0, # position + len2dlc(msg.dlc), + 0, # frame length + arb_id, + 0, # ext flags + data) + self._add_object(CAN_ERROR_EXT, data, msg.timestamp) + elif msg.is_fd: + fd_flags = EDL + if msg.bitrate_switch: + fd_flags |= BRS + if msg.error_state_indicator: + fd_flags |= ESI + data = CAN_FD_MSG_STRUCT.pack(channel, flags, len2dlc(msg.dlc), + arb_id, 0, 0, fd_flags, msg.dlc, data) + self._add_object(CAN_FD_MESSAGE, data, msg.timestamp) else: - flags = REMOTE_FLAG if msg.is_remote_frame else 0 - arb_id = msg.arbitration_id - if msg.id_type: - arb_id |= CAN_MSG_EXT - if msg.is_fd: - fd_flags = EDL - if msg.bitrate_switch: - fd_flags |= BRS - if msg.error_state_indicator: - fd_flags |= ESI - data = CAN_FD_MSG_STRUCT.pack(channel, flags, msg.dlc, arb_id, - 0, 0, fd_flags, msg.dlc, - bytes(msg.data)) - self._add_object(CAN_FD_MESSAGE, data, msg.timestamp) - else: - data = CAN_MSG_STRUCT.pack(channel, flags, msg.dlc, arb_id, - bytes(msg.data)) - self._add_object(CAN_MESSAGE, data, msg.timestamp) + data = CAN_MSG_STRUCT.pack(channel, flags, msg.dlc, arb_id, data) + self._add_object(CAN_MESSAGE, data, msg.timestamp) def log_event(self, text, timestamp=None): """Add an arbitrary message to the log file as a global marker. @@ -255,16 +305,19 @@ def _add_object(self, obj_type, data, timestamp=None): if self.start_timestamp is None: self.start_timestamp = timestamp self.stop_timestamp = timestamp - timestamp = int((timestamp - self.start_timestamp) * 1000000000) - obj_size = OBJ_HEADER_STRUCT.size + len(data) - header = OBJ_HEADER_STRUCT.pack( - b"LOBJ", OBJ_HEADER_STRUCT.size, 1, obj_size, obj_type, - 2, 0, max(timestamp, 0)) - self.cache.append(header) + timestamp = int((timestamp - self.start_timestamp) * 1e9) + obj_size = HEADER_SIZE + len(data) + base_header = OBJ_HEADER_BASE_STRUCT.pack( + b"LOBJ", HEADER_SIZE, 1, obj_size, obj_type) + obj_header = OBJ_HEADER_STRUCT.pack(2, 0, 0, max(timestamp, 0)) + + self.cache.append(base_header) + self.cache.append(obj_header) self.cache.append(data) padding_size = len(data) % 4 if padding_size: self.cache.append(b"\x00" * padding_size) + self.cache_size += obj_size + padding_size self.count_of_objects += 1 if self.cache_size >= self.MAX_CACHE_SIZE: @@ -285,14 +338,19 @@ def _flush(self): self.cache_size = len(tail) compressed_data = zlib.compress(uncompressed_data, self.COMPRESSION_LEVEL) - obj_size = OBJ_HEADER_STRUCT.size + len(compressed_data) - header = OBJ_HEADER_STRUCT.pack( - b"LOBJ", 16, 1, obj_size, LOG_CONTAINER, 2, 0, len(uncompressed_data)) - self.fp.write(header) + obj_size = (OBJ_HEADER_STRUCT.size + LOG_CONTAINER_STRUCT.size + + len(compressed_data)) + base_header = OBJ_HEADER_BASE_STRUCT.pack( + b"LOBJ", OBJ_HEADER_BASE_STRUCT.size, 1, obj_size, LOG_CONTAINER) + container_header = LOG_CONTAINER_STRUCT.pack( + ZLIB_DEFLATE, len(uncompressed_data)) + self.fp.write(base_header) + self.fp.write(container_header) self.fp.write(compressed_data) # Write padding bytes self.fp.write(b"\x00" * (obj_size % 4)) - self.uncompressed_size += len(uncompressed_data) + OBJ_HEADER_STRUCT.size + self.uncompressed_size += OBJ_HEADER_STRUCT.size + LOG_CONTAINER_STRUCT.size + self.uncompressed_size += len(uncompressed_data) def stop(self): """Stops logging and closes the file.""" diff --git a/can/util.py b/can/util.py index c78b9d674..a52a1ec95 100644 --- a/can/util.py +++ b/can/util.py @@ -23,6 +23,11 @@ log = logging.getLogger('can.util') +CAN_FD_DLC = [ + 0, 1, 2, 3, 4, 5, 6, 7, 8, + 12, 16, 20, 24, 32, 48, 64 +] + REQUIRED_KEYS = [ 'interface', 'channel', @@ -209,6 +214,33 @@ def set_logging_level(level_name=None): log.debug("Logging set to {}".format(level_name)) +def len2dlc(length): + """Calculate the DLC from data length. + + :param int length: Length in number of bytes (0-64) + + :returns: DLC (0-15) + :rtype: int + """ + if length <= 8: + return length + for dlc, nof_bytes in enumerate(CAN_FD_DLC): + if nof_bytes >= length: + return dlc + return 15 + + +def dlc2len(dlc): + """Calculate the data length from DLC. + + :param int dlc: DLC (0-15) + + :returns: Data length in number of bytes (0-64) + :rtype: int + """ + return CAN_FD_DLC[dlc] if dlc <= 15 else 64 + + if __name__ == "__main__": print("Searching for configuration named:") print("\n".join(CONFIG_FILES)) diff --git a/test/logformats_test.py b/test/logformats_test.py index c15511525..11c510a11 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -212,12 +212,17 @@ def test_writer_and_reader(self): def test_reader(self): logfile = os.path.join(os.path.dirname(__file__), "data", "logfile.blf") messages = list(can.BLFReader(logfile)) - self.assertEqual(len(messages), 1) + self.assertEqual(len(messages), 2) self.assertEqual(messages[0], can.Message( extended_id=False, arbitration_id=0x64, data=[0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8])) + self.assertEqual(messages[1], + can.Message( + is_error_frame=True, + extended_id=True, + arbitration_id=0x1FFFFFFF)) if __name__ == '__main__': From 9f7c43fd90aabda1c241fde5259558211dae16bc Mon Sep 17 00:00:00 2001 From: Christian Sandberg Date: Mon, 2 Apr 2018 21:30:57 +0200 Subject: [PATCH 2/3] Add 1 to channel numbers in ASC files --- can/io/asc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/can/io/asc.py b/can/io/asc.py index de3edf807..4ebdbe011 100644 --- a/can/io/asc.py +++ b/can/io/asc.py @@ -155,7 +155,8 @@ def on_message_received(self, msg): if msg.is_extended_id: arb_id += "x" - channel = msg.channel if isinstance(msg.channel, int) else self.channel + # Many interfaces start channel numbering at 0 which is invalid + channel = msg.channel + 1 if isinstance(msg.channel, int) else self.channel line = self.LOG_STRING.format(time=msg.timestamp, channel=channel, From 82e27a01ee00f21dd0e7116140fa3510b2f0df97 Mon Sep 17 00:00:00 2001 From: Christian Sandberg Date: Mon, 9 Apr 2018 22:24:38 +0200 Subject: [PATCH 3/3] Add channel support to ASCReader --- can/io/asc.py | 16 ++++++++++++---- can/io/blf.py | 8 -------- can/util.py | 1 + 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/can/io/asc.py b/can/io/asc.py index a11868832..5636469f9 100644 --- a/can/io/asc.py +++ b/can/io/asc.py @@ -56,12 +56,18 @@ def __iter__(self): continue timestamp = float(timestamp) + try: + # See ASCWriter + channel = int(channel) - 1 + except ValueError: + pass if dummy.strip()[0:10] == 'ErrorFrame': - msg = Message(timestamp=timestamp, is_error_frame=True) + msg = Message(timestamp=timestamp, is_error_frame=True, + channel=channel) yield msg - elif not channel.isdigit() or dummy.strip()[0:10] == 'Statistic:': + elif not isinstance(channel, int) or dummy.strip()[0:10] == 'Statistic:': pass elif dummy[-1:].lower() == 'r': @@ -70,7 +76,8 @@ def __iter__(self): msg = Message(timestamp=timestamp, arbitration_id=can_id_num & CAN_ID_MASK, extended_id=is_extended_id, - is_remote_frame=True) + is_remote_frame=True, + channel=channel) yield msg else: @@ -97,7 +104,8 @@ def __iter__(self): extended_id=is_extended_id, is_remote_frame=False, dlc=dlc, - data=frame) + data=frame, + channel=channel) yield msg diff --git a/can/io/blf.py b/can/io/blf.py index fb25527f8..588f56b32 100644 --- a/can/io/blf.py +++ b/can/io/blf.py @@ -54,9 +54,6 @@ # valid data bytes, data CAN_FD_MSG_STRUCT = struct.Struct("