From 79ff6eb2dbe6d93ffc0fb1f832fcf850c55e1108 Mon Sep 17 00:00:00 2001 From: tony godshall Date: Wed, 13 Mar 2019 12:19:45 -0700 Subject: [PATCH 1/9] don't crash when index out of range (BMS) catch index out of range faults so we get output instead of crashing --- zero_log_parser.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/zero_log_parser.py b/zero_log_parser.py index 4f55a84..6ee29f5 100755 --- a/zero_log_parser.py +++ b/zero_log_parser.py @@ -99,14 +99,30 @@ def parse_entry(log_data, address, unhandled): ''' Parse an individual entry from a LogFile into a human readable form ''' - header = log_data[address] + try: + header = log_data[address] + #IndexError: bytearray index out of range + except IndexError: + #print "IndexError log_data[%r]: forcing header_bad"%(address) + header= 0 # correct header offset as needed to prevent errors header_bad = header != 0xb2 while header_bad: address += 1 - header = log_data[address] + try: + header = log_data[address] + except IndexError: + #IndexError: bytearray index out of range + #print "IndexError log_data[%r]: forcing header_bad"%(address) + header= 0 + header_bad = True + break header_bad = header != 0xb2 - length = log_data[address + 1] + try: + length = log_data[address + 1] + #IndexError: bytearray index out of range + except IndexError: + length= 0 unescaped_block = BinaryTools.unescape_block(log_data[address + 0x2:address + length]) @@ -836,4 +852,5 @@ def parse_log(bin_file, output_file): else: output_file = os.path.splitext(args.bin_file)[0] + '.txt' - parse_log(log_file, output_file) \ No newline at end of file + parse_log(log_file, output_file) + From f8295f24d21890c56425c92b6aa9c3fdc86818fd Mon Sep 17 00:00:00 2001 From: moreel_a Date: Thu, 20 Jun 2019 11:53:14 +0200 Subject: [PATCH 2/9] Update log_structure.md Fix table ### 0x1e --- log_structure.md | 1 + 1 file changed, 1 insertion(+) diff --git a/log_structure.md b/log_structure.md index def12d4..dd07912 100644 --- a/log_structure.md +++ b/log_structure.md @@ -220,6 +220,7 @@ Offset | Length | Contents 0x00 | 8 | ??? ### `0x1e` - MBB unknown +Offset | Length | Contents ------ | :----: | -------- 0x00 | 4 | ??? From a922785a6bbf22b05b8729da981046086f2caeb5 Mon Sep 17 00:00:00 2001 From: John Myers Date: Sat, 18 Sep 2021 22:09:02 -0700 Subject: [PATCH 3/9] add kaitai file --- zlog-bin-format.ksy.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 zlog-bin-format.ksy.yml diff --git a/zlog-bin-format.ksy.yml b/zlog-bin-format.ksy.yml new file mode 100644 index 0000000..e69de29 From 5541d1fb0b6af7266c87cd961b042f047beb59a9 Mon Sep 17 00:00:00 2001 From: John Myers Date: Fri, 24 Sep 2021 23:09:53 -0700 Subject: [PATCH 4/9] add kaitai files --- README.md | 2 + zlog.yml | 163 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 165 insertions(+) create mode 100644 zlog.yml diff --git a/README.md b/README.md index 8885b5f..285c892 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # Zero Log Parser +## **An attempt to rewrite the parser using [Kaitai Struct]https://kaitai.io/#what-is-it ** + This is a small decoder utility to parse Zero Motorcycle main bike board (MBB) and battery management system (BMS) logs. * It is designed to emulate Zero Motorcycles' own official log parser, that turns the binary-encoded event log into a mostly-readable text file. * If you send a log to Zero Customer Support, they should provide you with a text file in response. (Often, a phone call prompts them to process the log file) diff --git a/zlog.yml b/zlog.yml new file mode 100644 index 0000000..eeb977b --- /dev/null +++ b/zlog.yml @@ -0,0 +1,163 @@ +meta: + id: zlog + title: A parser for Zero motorcycle MBB and BMS binary log files + endian: le + #bit-endian: le +seq: + - id: magic + contents: [0xB2, 0x75, 0xFB] # FB probable a record type, actually. Seems 0xF$ are + - id: file_timestamp + type: u4 + - id: unknown_file_header_section + size: 5 + doc: "last byte is # of entries...but more like min number of entries" + - id: file_header_end + contents: [0xF9] + - id: log_file_type + type: str + size: 10 #-MBB is padded with 7 zero's. Is BMS the same? + encoding: ASCII + - id: bike_info + contents: [0xF2] + - id: bike_type + size: 93 # (header:117) - (previous:24) + #- id: unknown_h2 + # size: 6 + #- id: fw_checksum + # type: strz + # encoding: ASCII +# end MBB file header + + - id: garbage #```````````Garbage till next 0xb2 entry + terminator: 0xB2 + consume: false # So next stream can read/start from 0xB2 + doc: "Holds ring buffer garbage. Used to iterate past untill valid log entry is found" + + - id: log_entry + type: log_entries + repeat: until + repeat-until: _io.pos == 0x3FFF or _io.eof +#_io.eof or _io.pos == 0x3FFF +#_io.pos == 0x3FFF or _io.eof + #GEN3::MBB 0x3FFF (16K) + #GEN3::BMS + #GEN2::MBB/BMS 3FFFF (256K) ?? +instances: + file_header_size: + pos: 0x01 + type: u1 + doc: "Looks to always be 0x75 (needs to; for these 'pos' to work)" + header_timestamp: + pos: 0x03 + type: u4 + number_of_entries: + pos: 0x0B + type: u1 + doc: "...but more like min number of entries" + model: + pos: 0x19 + type: strz + encoding: ASCII + bike_v_i_n: + pos: 0x29 + type: strz + encoding: ASCII + board_i_d: + pos: 0x65 + type: u1 + firmware_rev: + pos: 0x67 + type: u1 + firmware_build: + pos: 0x6B + type: strz + encoding: ASCII + +types: +# from ttyConsol: Entry / Time of Log / UTC|Code / Event description; and data + log_entries: + seq: + - id: header_begin + contents: [0xB2] + - id: length + type: u1 + - id: log_type + type: u1 + - id: timestamp + type: u4 + - id: unknown_header_section + terminator: 0xF9 + #size: 4 #--often four but not always(or corrupted entry?) +# - id: count +# type: u1 +# doc: 'seems to increment for every 0xB2 record' +# - id: header_end +# contents: [0xF9] # 0xF8=>Serial Log Console + - id: data + type: + switch-on: log_type + cases: + 0x48: log_type_48 + 0x49: log_type_49 + 0x51: log_type_51 + 0x52: log_type_52 + 0x53: log_type_53 + 0xFB: log_type_f_b + 0xfd: debug_message + _: log_type_unknown + +# instances: +# header: +# pos: unknown_header_section.eos + + + log_type_48: + seq: + - id: data + eos-error: false + terminator: 0xB2 + consume: false + log_type_49: + seq: + - id: data + eos-error: false + terminator: 0xB2 + consume: false + log_type_51: + seq: + - id: data + eos-error: false + terminator: 0xB2 + consume: false + log_type_52: + seq: + - id: data + eos-error: false + terminator: 0xB2 + consume: false + log_type_53: + seq: + - id: data + eos-error: false + terminator: 0xB2 + consume: false + log_type_f_b: + seq: + - id: data + eos-error: false + terminator: 0xB2 + consume: false + debug_message: + seq: + - id: debug_message + type: str + encoding: ASCII + eos-error: false + terminator: 0xB2 + consume: false + log_type_unknown: + seq: + - id: data + eos-error: false + terminator: 0xB2 + consume: false \ No newline at end of file From c3d5ec31e860c05d842b1121aa0fad1b0549b9f7 Mon Sep 17 00:00:00 2001 From: John Myers Date: Fri, 24 Sep 2021 23:39:47 -0700 Subject: [PATCH 5/9] delete empty file, fix readme --- README.md | 2 +- zlog-bin-format.ksy.yml | 0 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 zlog-bin-format.ksy.yml diff --git a/README.md b/README.md index 285c892..b005115 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Zero Log Parser -## **An attempt to rewrite the parser using [Kaitai Struct]https://kaitai.io/#what-is-it ** +## **An attempt to rewrite the parser using [Kaitai Struct](https://kaitai.io/#what-is-it) ** This is a small decoder utility to parse Zero Motorcycle main bike board (MBB) and battery management system (BMS) logs. * It is designed to emulate Zero Motorcycles' own official log parser, that turns the binary-encoded event log into a mostly-readable text file. diff --git a/zlog-bin-format.ksy.yml b/zlog-bin-format.ksy.yml deleted file mode 100644 index e69de29..0000000 From f626bf7ae8caab53a038b3ba6d05c46e44aafebc Mon Sep 17 00:00:00 2001 From: John Myers Date: Mon, 27 Sep 2021 19:15:29 -0700 Subject: [PATCH 6/9] BMS gen3 --- bms-g3.ksy.yml | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 bms-g3.ksy.yml diff --git a/bms-g3.ksy.yml b/bms-g3.ksy.yml new file mode 100644 index 0000000..8fc5f9f --- /dev/null +++ b/bms-g3.ksy.yml @@ -0,0 +1,67 @@ +meta: + id: zlog-bms + title: A parser for Zero motorcycle Gen 3 BMS binary log files + endian: le + #bit-endian: le + +seq: + - id: Partial #```````````Partial log till next 0xb2 entry + terminator: 0xB2 + consume: false # So next stream can read/start from 0xB2 + doc: "Find first valid log entry" + + - id: log_entry + type: log_entries + repeat: until + repeat-until: _io.eof + +types: +# from ttyConsol: Entry / Time of Log / UTC|Code / Event description; and data + log_entries: + seq: + - id: header_begin + contents: [0xB2] + - id: length + type: u1 + doc: "Data is sometimes less then bytes. Calculated different then MBB" + - id: log_type + type: u1 + doc: "0xFD 0x4D" + - id: timestamp + type: u4 + - id: unknown_header_section + terminator: 0xF9 + #size: 4 #--often four but not always(or corrupted entry?) <--MBB... (BMS also?) +# - id: count +# type: u1 +# doc: 'seems to increment for every 0xB2 record' +# - id: header_end +# contents: [0xF9] # 0xF8=>Serial Log Console <--MBB... (BMS also?) + - id: data + type: + switch-on: log_type + cases: + 0x4d: log_type_4_d + 0xfd: capacity_estimate + _: log_type_unknown + + log_type_4_d: + seq: + - id: data + eos-error: false + terminator: 0xB2 + consume: false + capacity_estimate: + seq: + - id: data + type: str + encoding: ASCII + eos-error: false + terminator: 0xB2 + consume: false + log_type_unknown: + seq: + - id: data + eos-error: false + terminator: 0xB2 + consume: false \ No newline at end of file From 80f349389b1038a76bd48b5b7c003d21a3ad3c6d Mon Sep 17 00:00:00 2001 From: John Myers Date: Thu, 30 Sep 2021 00:39:05 -0700 Subject: [PATCH 7/9] update type48 to charger info --- zlog.yml | 44 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/zlog.yml b/zlog.yml index eeb977b..b49ba8d 100644 --- a/zlog.yml +++ b/zlog.yml @@ -97,7 +97,7 @@ types: type: switch-on: log_type cases: - 0x48: log_type_48 + 0x48: charger_info 0x49: log_type_49 0x51: log_type_51 0x52: log_type_52 @@ -110,10 +110,48 @@ types: # header: # pos: unknown_header_section.eos - - log_type_48: +# CHARGERS| EN | DC V A | AC V A HZ |CMD A| Ver | SN | ID | STATUS +# 3kW , 0, 0.0, 0.0, 0, 0.0, 0, 0, 201, 2052093, 0x010, 0x0020 + charger_info: seq: + - id: name + type: str + encoding: ASCII + size: 10 + - id: maybe_bits + size: 2 + doc: "Always? 0x80 0x40" + - id: maybe_floats + terminator: 60 + consume: false + doc: "size varies" + - id: hertz + size: 1 + doc: "Always? 0x3C could be hertz or header end" + - id: val + size: 1 + # - id: en + # size: 1 + # - id: d_c_volts + # type: f4 + # - id: d_c_amps + # type: f4 + # - id: a_c_volts + # size: 1 + # - id: a_c_amps + # type: f4 + # - id: c_m_da + # size: 1 + # - id: status + # size: 1 + - id: i_d + size: 1 + - id: version + type: u2 + - id: serial_number + type: u4 - id: data + # size: 11 eos-error: false terminator: 0xB2 consume: false From 5cb9347eb2a4eaf48488b06ae86ea66c045bb7c0 Mon Sep 17 00:00:00 2001 From: John Myers Date: Thu, 30 Sep 2021 01:07:30 -0700 Subject: [PATCH 8/9] update type48 to charger info --- zlog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zlog.yml b/zlog.yml index b49ba8d..521d3d1 100644 --- a/zlog.yml +++ b/zlog.yml @@ -111,7 +111,7 @@ types: # pos: unknown_header_section.eos # CHARGERS| EN | DC V A | AC V A HZ |CMD A| Ver | SN | ID | STATUS -# 3kW , 0, 0.0, 0.0, 0, 0.0, 0, 0, 201, 2052093, 0x010, 0x0020 +# 3kW , 0, 0.0, 0.0, 0, 0.0, 0, 0, 201, 0000000, 0x010, 0x0020 charger_info: seq: - id: name From fb277710a40e462fd4971767ccf70b96cf38d6cb Mon Sep 17 00:00:00 2001 From: John Myers Date: Thu, 30 Sep 2021 18:54:44 -0700 Subject: [PATCH 9/9] update charger_info --- bms-g3.ksy.yml | 29 +++++++++++++++++++---------- zlog.yml | 2 +- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/bms-g3.ksy.yml b/bms-g3.ksy.yml index 8fc5f9f..e501566 100644 --- a/bms-g3.ksy.yml +++ b/bms-g3.ksy.yml @@ -1,11 +1,11 @@ meta: - id: zlog-bms + id: zlog_bms title: A parser for Zero motorcycle Gen 3 BMS binary log files endian: le #bit-endian: le seq: - - id: Partial #```````````Partial log till next 0xb2 entry + - id: partial #```````````Partial log till next 0xb2 entry terminator: 0xB2 consume: false # So next stream can read/start from 0xB2 doc: "Find first valid log entry" @@ -16,7 +16,6 @@ seq: repeat-until: _io.eof types: -# from ttyConsol: Entry / Time of Log / UTC|Code / Event description; and data log_entries: seq: - id: header_begin @@ -26,12 +25,15 @@ types: doc: "Data is sometimes less then bytes. Calculated different then MBB" - id: log_type type: u1 - doc: "0xFD 0x4D" - - id: timestamp - type: u4 - - id: unknown_header_section + doc: "0xFD 0x4B 0x4D" + # when length 0x6B: 0xFE or 0xFE is inserted before timestamp. variable increments by three + # 0x40->0x43. Sometimes nothing inserted though (smh) +# - id: timestamp +# type: u4 + - id: header terminator: 0xF9 - #size: 4 #--often four but not always(or corrupted entry?) <--MBB... (BMS also?) + doc: "Process ,timestamp,,sequence-number at program SW level for now" + #size: 4 # - id: count # type: u1 # doc: 'seems to increment for every 0xB2 record' @@ -41,10 +43,17 @@ types: type: switch-on: log_type cases: - 0x4d: log_type_4_d - 0xfd: capacity_estimate + 0x4B: log_type_4_b + 0x4D: log_type_4_d + 0xFD: capacity_estimate _: log_type_unknown + log_type_4_b: + seq: + - id: data + eos-error: false + terminator: 0xB2 + consume: false log_type_4_d: seq: - id: data diff --git a/zlog.yml b/zlog.yml index b49ba8d..521d3d1 100644 --- a/zlog.yml +++ b/zlog.yml @@ -111,7 +111,7 @@ types: # pos: unknown_header_section.eos # CHARGERS| EN | DC V A | AC V A HZ |CMD A| Ver | SN | ID | STATUS -# 3kW , 0, 0.0, 0.0, 0, 0.0, 0, 0, 201, 2052093, 0x010, 0x0020 +# 3kW , 0, 0.0, 0.0, 0, 0.0, 0, 0, 201, 0000000, 0x010, 0x0020 charger_info: seq: - id: name