From ab369980214e65b248b5c9dcb15ce988399d7ecc Mon Sep 17 00:00:00 2001 From: deanlee Date: Tue, 26 Mar 2024 12:44:17 +0800 Subject: [PATCH] remove all cereal --- can/SConscript | 3 +- can/parser_pyx.pyx | 36 ++++++++++++---- can/strings_to_candata.cc | 37 ---------------- can/tests/test_checksums.py | 4 +- can/tests/test_packer_parser.py | 64 ++++++---------------------- can/tests/test_parser_performance.py | 4 +- 6 files changed, 44 insertions(+), 104 deletions(-) delete mode 100644 can/strings_to_candata.cc diff --git a/can/SConscript b/can/SConscript index 1a1f79cd4c..3923bf3035 100644 --- a/can/SConscript +++ b/can/SConscript @@ -16,9 +16,8 @@ envDBC.Library('libdbc_static', src, LIBS=libs) # Build packer and parser lenv = envCython.Clone() -lenv.Library('strings_to_candata', ['strings_to_candata.cc']) lenv["LINKFLAGS"] += [libdbc[0].get_labspath()] -parser = lenv.Program('parser_pyx.so', 'parser_pyx.pyx', LIBS=['strings_to_candata', 'capnp', 'kj'] + lenv["LIBS"]) +parser = lenv.Program('parser_pyx.so', 'parser_pyx.pyx') packer = lenv.Program('packer_pyx.so', 'packer_pyx.pyx') lenv.Depends(parser, libdbc) diff --git a/can/parser_pyx.pyx b/can/parser_pyx.pyx index cd1c1216f1..724b5e4875 100644 --- a/can/parser_pyx.pyx +++ b/can/parser_pyx.pyx @@ -5,19 +5,15 @@ from cython.operator cimport dereference as deref, preincrement as preinc from libcpp.pair cimport pair from libcpp.string cimport string from libcpp.vector cimport vector -from libcpp cimport bool from libcpp.unordered_set cimport unordered_set from libc.stdint cimport uint32_t from .common cimport CANParser as cpp_CANParser -from .common cimport dbc_lookup, SignalValue, DBC, CanData +from .common cimport dbc_lookup, SignalValue, DBC, CanData, CanFrame import numbers from collections import defaultdict -cdef extern from "strings_to_candata.cc": - void strings_to_candata(const vector[string] &strings, vector[CanData] &candata, bool sendcan) except + - cdef class CANParser: cdef: cpp_CANParser *can @@ -74,16 +70,38 @@ cdef class CANParser: del self.can def update_strings(self, strings, sendcan=False): + # input format: + # [nanos, [[address, 0, data, src], ...]] + # [[nanos, [[address, 0, data, src], ...], ...]] + for v in self.vl_all.values(): for l in v.values(): # no-cython-lint l.clear() cdef vector[SignalValue] new_vals cdef unordered_set[uint32_t] updated_addrs - cdef vector[CanData] can_data - - strings_to_candata(strings, can_data, sendcan) - self.can.update(can_data, new_vals) + cdef CanFrame* frame + cdef CanData* can_data + cdef vector[CanData] can_data_array + + try: + if len(strings) and not isinstance(strings[0], (list, tuple)): + strings = [strings] + + can_data_array.reserve(len(strings)) + for s in strings: + can_data = &(can_data_array.emplace_back()) + can_data.nanos = s[0] + can_data.frames.reserve(len(s[1])) + for f in s[1]: + frame = &(can_data.frames.emplace_back()) + frame.address = f[0] + frame.dat = f[2] + frame.src = f[3] + except TypeError: + raise RuntimeError("invalid parameter") + + self.can.update(can_data_array, new_vals) cdef vector[SignalValue].iterator it = new_vals.begin() cdef SignalValue* cv diff --git a/can/strings_to_candata.cc b/can/strings_to_candata.cc deleted file mode 100644 index 46ce918602..0000000000 --- a/can/strings_to_candata.cc +++ /dev/null @@ -1,37 +0,0 @@ -#include - -#include -#include - -#include "cereal/gen/cpp/log.capnp.h" -#include "opendbc/can/common.h" - -void strings_to_candata(const std::vector &strings, std::vector &can_data, bool sendcan) { - kj::Array aligned_buf; - can_data.reserve(strings.size()); - for (const auto &s : strings) { - const size_t buf_size = (s.length() / sizeof(capnp::word)) + 1; - if (aligned_buf.size() < buf_size) { - aligned_buf = kj::heapArray(buf_size); - } - memcpy(aligned_buf.begin(), s.data(), s.length()); - - // extract the messages - capnp::FlatArrayMessageReader cmsg(aligned_buf.slice(0, buf_size)); - cereal::Event::Reader event = cmsg.getRoot(); - - auto &can = can_data.emplace_back(); - can.nanos = event.getLogMonoTime(); - - auto cans = sendcan ? event.getSendcan() : event.getCan(); - can.frames.reserve(cans.size()); - for (const auto cmsg : cans) { - auto &frame = can.frames.emplace_back(); - frame.src = cmsg.getSrc(); - frame.address = cmsg.getAddress(); - auto dat = cmsg.getDat(); - frame.dat.resize(dat.size()); - memcpy(frame.dat.data(), dat.begin(), dat.size()); - } - } -} diff --git a/can/tests/test_checksums.py b/can/tests/test_checksums.py index 93ddbe0074..1e61262a03 100755 --- a/can/tests/test_checksums.py +++ b/can/tests/test_checksums.py @@ -3,7 +3,6 @@ from opendbc.can.parser import CANParser from opendbc.can.packer import CANPacker -from opendbc.can.tests.test_packer_parser import can_list_to_can_capnp class TestCanChecksums(unittest.TestCase): @@ -31,8 +30,7 @@ def test_honda_checksum(self): packer.make_can_msg("LKAS_HUD", 0, values), packer.make_can_msg("LKAS_HUD_A", 0, values), ] - can_strings = [can_list_to_can_capnp(msgs), ] - parser.update_strings(can_strings) + parser.update_strings([0, msgs]) self.assertEqual(parser.vl['LKAS_HUD']['CHECKSUM'], std) self.assertEqual(parser.vl['LKAS_HUD_A']['CHECKSUM'], ext) diff --git a/can/tests/test_packer_parser.py b/can/tests/test_packer_parser.py index 8c3653a220..662573ab3f 100755 --- a/can/tests/test_packer_parser.py +++ b/can/tests/test_packer_parser.py @@ -2,7 +2,6 @@ import unittest import random -import cereal.messaging as messaging from opendbc.can.parser import CANParser from opendbc.can.packer import CANPacker from opendbc.can.tests import TEST_DBC @@ -10,27 +9,6 @@ MAX_BAD_COUNTER = 5 -# Python implementation so we don't have to depend on boardd -def can_list_to_can_capnp(can_msgs, msgtype='can', logMonoTime=None): - dat = messaging.new_message(msgtype, len(can_msgs)) - - if logMonoTime is not None: - dat.logMonoTime = logMonoTime - - for i, can_msg in enumerate(can_msgs): - if msgtype == 'sendcan': - cc = dat.sendcan[i] - else: - cc = dat.can[i] - - cc.address = can_msg[0] - cc.busTime = can_msg[1] - cc.dat = bytes(can_msg[2]) - cc.src = can_msg[3] - - return dat.to_bytes() - - class TestCanParserPacker(unittest.TestCase): def test_packer(self): packer = CANPacker(TEST_DBC) @@ -51,8 +29,7 @@ def test_packer_counter(self): # packer should increment the counter for i in range(1000): msg = packer.make_can_msg("CAN_FD_MESSAGE", 0, {}) - dat = can_list_to_can_capnp([msg, ]) - parser.update_strings([dat]) + parser.update_strings([0, [msg]]) self.assertEqual(parser.vl["CAN_FD_MESSAGE"]["COUNTER"], i % 256) # setting COUNTER should override @@ -61,16 +38,14 @@ def test_packer_counter(self): msg = packer.make_can_msg("CAN_FD_MESSAGE", 0, { "COUNTER": cnt, }) - dat = can_list_to_can_capnp([msg, ]) - parser.update_strings([dat]) + parser.update_strings([0, [msg]]) self.assertEqual(parser.vl["CAN_FD_MESSAGE"]["COUNTER"], cnt) # then, should resume counting from the override value cnt = parser.vl["CAN_FD_MESSAGE"]["COUNTER"] for i in range(100): msg = packer.make_can_msg("CAN_FD_MESSAGE", 0, {}) - dat = can_list_to_can_capnp([msg, ]) - parser.update_strings([dat]) + parser.update_strings([0, [msg]]) self.assertEqual(parser.vl["CAN_FD_MESSAGE"]["COUNTER"], (cnt + i) % 256) def test_parser_can_valid(self): @@ -83,16 +58,14 @@ def test_parser_can_valid(self): # not valid until the message is seen for _ in range(100): - dat = can_list_to_can_capnp([]) - parser.update_strings([dat]) + parser.update_strings([0, []]) self.assertFalse(parser.can_valid) # valid once seen for i in range(1, 100): t = int(0.01 * i * 1e9) msg = packer.make_can_msg("CAN_FD_MESSAGE", 0, {}) - dat = can_list_to_can_capnp([msg, ], logMonoTime=t) - parser.update_strings([dat]) + parser.update_strings([t, [msg]]) self.assertTrue(parser.can_valid) def test_parser_counter_can_valid(self): @@ -107,17 +80,15 @@ def test_parser_counter_can_valid(self): parser = CANParser("honda_civic_touring_2016_can_generated", msgs, 0) msg = packer.make_can_msg("STEERING_CONTROL", 0, {"COUNTER": 0}) - bts = can_list_to_can_capnp([msg]) # bad static counter, invalid once it's seen MAX_BAD_COUNTER messages for idx in range(0x1000): - parser.update_strings([bts]) + parser.update_strings([0, [msg]]) self.assertEqual((idx + 1) < MAX_BAD_COUNTER, parser.can_valid) # one to recover msg = packer.make_can_msg("STEERING_CONTROL", 0, {"COUNTER": 1}) - bts = can_list_to_can_capnp([msg]) - parser.update_strings([bts]) + parser.update_strings([0, [msg]]) self.assertTrue(parser.can_valid) def test_parser_no_partial_update(self): @@ -139,8 +110,7 @@ def rx_steering_msg(values, bad_checksum=False): msg[2] = bytearray(msg[2]) msg[2][4] = (msg[2][4] & 0xF0) | ((msg[2][4] & 0x0F) + 1) - bts = can_list_to_can_capnp([msg]) - parser.update_strings([bts]) + parser.update_strings([0, [msg]]) rx_steering_msg({"STEER_TORQUE": 100}, bad_checksum=False) self.assertEqual(parser.vl["STEERING_CONTROL"]["STEER_TORQUE"], 100) @@ -183,8 +153,7 @@ def test_packer_parser(self): } msgs = [packer.make_can_msg(k, 0, v) for k, v in values.items()] - bts = can_list_to_can_capnp(msgs) - parser.update_strings([bts]) + parser.update_strings([0, msgs]) for k, v in values.items(): for key, val in v.items(): @@ -204,9 +173,7 @@ def test_scale_offset(self): for brake in range(100): values = {"USER_BRAKE": brake} msgs = packer.make_can_msg("VSA_STATUS", 0, values) - bts = can_list_to_can_capnp([msgs]) - - parser.update_strings([bts]) + parser.update_strings([0, [msgs]]) self.assertAlmostEqual(parser.vl["VSA_STATUS"]["USER_BRAKE"], brake) @@ -230,8 +197,7 @@ def test_subaru(self): } msgs = packer.make_can_msg("ES_LKAS", 0, values) - bts = can_list_to_can_capnp([msgs]) - parser.update_strings([bts]) + parser.update_strings([0, [msgs]]) self.assertAlmostEqual(parser.vl["ES_LKAS"]["LKAS_Output"], steer) self.assertAlmostEqual(parser.vl["ES_LKAS"]["LKAS_Request"], active) @@ -260,8 +226,7 @@ def send_msg(blank=False): else: msgs = [packer.make_can_msg("VSA_STATUS", 0, {}), ] - can = can_list_to_can_capnp(msgs, logMonoTime=t) - parser.update_strings([can, ]) + parser.update_strings([t, msgs]) # all good, no timeout for _ in range(1000): @@ -299,8 +264,7 @@ def test_updated(self): can_msgs[frame].append(packer.make_can_msg("VSA_STATUS", 0, values)) idx += 1 - can_strings = [can_list_to_can_capnp(msgs) for msgs in can_msgs] - parser.update_strings(can_strings) + parser.update_strings([[0, can_msgs[0]], [0, can_msgs[1]]]) vl_all = parser.vl_all["VSA_STATUS"]["USER_BRAKE"] self.assertEqual(vl_all, user_brake_vals) @@ -334,7 +298,7 @@ def test_timestamp_nanos(self): for i in range(10): log_mono_time = int(0.01 * i * 1e+9) can_msg = packer.make_can_msg("VSA_STATUS", 0, {}) - can_strings.append(can_list_to_can_capnp([can_msg], logMonoTime=log_mono_time)) + can_strings.append((log_mono_time, [can_msg])) parser.update_strings(can_strings) ts_nanos = parser.ts_nanos["VSA_STATUS"].values() diff --git a/can/tests/test_parser_performance.py b/can/tests/test_parser_performance.py index 2010fa4bf7..ac1bbdf93e 100755 --- a/can/tests/test_parser_performance.py +++ b/can/tests/test_parser_performance.py @@ -4,7 +4,6 @@ from opendbc.can.parser import CANParser from opendbc.can.packer import CANPacker -from opendbc.can.tests.test_packer_parser import can_list_to_can_capnp @unittest.skip("TODO: varies too much between machines") @@ -17,8 +16,7 @@ def _benchmark(self, checks, thresholds, n): for i in range(50000): values = {"ACC_CONTROL": {"ACC_TYPE": 1, "ALLOW_LONG_PRESS": 3}} msgs = [packer.make_can_msg(k, 0, v) for k, v in values.items()] - bts = can_list_to_can_capnp(msgs, logMonoTime=int(0.01 * i * 1e9)) - can_msgs.append(bts) + can_msgs.append([int(0.01 * i * 1e9), msgs]) ets = [] for _ in range(25):