remove dependence on cereal

SConstruct

@@ -6,8 +6,6 @@ import numpy as np
 zmq = 'zmq'
 arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
 
-cereal_dir = Dir('.')
-
 python_path = sysconfig.get_paths()['include']
 cpppath = [
  '#',
@@ -55,11 +53,6 @@ env = Environment(
 common = ''
 Export('env', 'zmq', 'arch', 'common')
 
-cereal = [File('#cereal/libcereal.a')]
-messaging = [File('#cereal/libmessaging.a')]
-Export('cereal', 'messaging')
-
-
 envCython = env.Clone()
 envCython["CPPPATH"] += [np.get_include()]
 envCython["CCFLAGS"] += ["-Wno-#warnings", "-Wno-shadow", "-Wno-deprecated-declarations"]
@@ -79,6 +72,4 @@ envCython["LIBS"] = python_libs
 
 Export('envCython')
 
-
-SConscript(['cereal/SConscript'])
 SConscript(['opendbc/can/SConscript'])

can/SConscript

@@ -1,12 +1,12 @@
-Import('env', 'envCython', 'cereal', 'common')
+Import('env', 'envCython', 'common')
 
 import os
 
 envDBC = env.Clone()
 dbc_file_path = '-DDBC_FILE_PATH=\'"%s"\'' % (envDBC.Dir("..").abspath)
 envDBC['CXXFLAGS'] += [dbc_file_path]
 src = ["dbc.cc", "parser.cc", "packer.cc", "common.cc"]
-libs = [common, "capnp", "kj", "zmq"]
+libs = [common, "zmq"]
 
 # shared library for openpilot
 libdbc = envDBC.SharedLibrary('libdbc', src, LIBS=libs)

can/common.h

@@ -6,13 +6,6 @@
 #include <unordered_map>
 #include <vector>
 
-#include <capnp/dynamic.h>
-#include <capnp/serialize.h>
-
-#ifndef DYNAMIC_CAPNP
-#include "cereal/gen/cpp/log.capnp.h"
-#endif
-
 #include "opendbc/can/common_dbc.h"
 
 #define INFO printf
@@ -35,6 +28,17 @@ unsigned int xor_checksum(uint32_t address, const Signal &sig, const std::vector
 unsigned int hkg_can_fd_checksum(uint32_t address, const Signal &sig, const std::vector<uint8_t> &d);
 unsigned int pedal_checksum(uint32_t address, const Signal &sig, const std::vector<uint8_t> &d);
 
+struct CanFrame {
+ long src;
+ long address;
+ std::vector<uint8_t> dat;
+};
+
+struct CanData {
+ uint64_t nanos;
+ std::vector<CanFrame> frames;
+};
+
 class MessageState {
 public:
  std::string name;
@@ -61,8 +65,6 @@ class MessageState {
 class CANParser {
 private:
  const int bus;
- kj::Array<capnp::word> aligned_buf;
-
  const DBC *dbc = NULL;
  std::unordered_map<uint32_t, MessageState> message_states;
 
@@ -78,14 +80,12 @@ class CANParser {
  CANParser(int abus, const std::string& dbc_name,
  const std::vector<std::pair<uint32_t, int>> &messages);
  CANParser(int abus, const std::string& dbc_name, bool ignore_checksum, bool ignore_counter);
- #ifndef DYNAMIC_CAPNP
- void update_string(const std::string &data, bool sendcan);
- void update_strings(const std::vector<std::string> &data, std::vector<SignalValue> &vals, bool sendcan);
- void UpdateCans(uint64_t nanos, const capnp::List<cereal::CanData>::Reader& cans);
- #endif
- void UpdateCans(uint64_t nanos, const capnp::DynamicStruct::Reader& cans);
- void UpdateValid(uint64_t nanos);
+ void update(const std::vector<CanData> &can_data, std::vector<SignalValue> &vals);
  void query_latest(std::vector<SignalValue> &vals, uint64_t last_ts = 0);
+
+protected:
+ void update_can(const CanData &can);
+ void UpdateValid(uint64_t nanos);
 };
 
 class CANPacker {

can/common.pxd

@@ -64,11 +64,20 @@ cdef extern from "common_dbc.h":
 cdef extern from "common.h":
  cdef const DBC* dbc_lookup(const string) except +
 
+ cdef struct CanFrame:
+ long src
+ long address
+ vector[uint8_t] dat
+
+ cdef struct CanData:
+ uint64_t nanos
+ vector[CanFrame] frames
+
  cdef cppclass CANParser:
  bool can_valid
  bool bus_timeout
  CANParser(int, string, vector[pair[uint32_t, int]]) except +
- void update_strings(vector[string]&, vector[SignalValue]&, bool) except +
+ void update(vector[CanData]&, vector[SignalValue]&) except +
 
  cdef cppclass CANPacker:
  CANPacker(string)

can/parser.cc

@@ -94,7 +94,7 @@ bool MessageState::update_counter_generic(int64_t v, int cnt_size) {
 
 
 CANParser::CANParser(int abus, const std::string& dbc_name, const std::vector<std::pair<uint32_t, int>> &messages)
- : bus(abus), aligned_buf(kj::heapArray<capnp::word>(1024)) {
+ : bus(abus) {
  dbc = dbc_lookup(dbc_name);
  assert(dbc);
  init_crc_lookup_tables();
@@ -171,106 +171,57 @@ CANParser::CANParser(int abus, const std::string& dbc_name, bool ignore_checksum
  }
 }
 
-#ifndef DYNAMIC_CAPNP
-void CANParser::update_string(const std::string &data, bool sendcan) {
- // format for board, make copy due to alignment issues.
- const size_t buf_size = (data.length() / sizeof(capnp::word)) + 1;
- if (aligned_buf.size() < buf_size) {
- aligned_buf = kj::heapArray<capnp::word>(buf_size);
- }
- memcpy(aligned_buf.begin(), data.data(), data.length());
-
- // extract the messages
- capnp::FlatArrayMessageReader cmsg(aligned_buf.slice(0, buf_size));
- cereal::Event::Reader event = cmsg.getRoot<cereal::Event>();
-
- if (first_nanos == 0) {
- first_nanos = event.getLogMonoTime();
- }
- last_nanos = event.getLogMonoTime();
-
- auto cans = sendcan ? event.getSendcan() : event.getCan();
- UpdateCans(last_nanos, cans);
-
- UpdateValid(last_nanos);
-}
-
-void CANParser::update_strings(const std::vector<std::string> &data, std::vector<SignalValue> &vals, bool sendcan) {
+void CANParser::update(const std::vector<CanData> &can_data, std::vector<SignalValue> &vals) {
  uint64_t current_nanos = 0;
- for (const auto &d : data) {
- update_string(d, sendcan);
+ for (const auto &c : can_data) {
+ if (first_nanos == 0) {
+ first_nanos = c.nanos;
+ }
  if (current_nanos == 0) {
- current_nanos = last_nanos;
+ current_nanos = c.nanos;
  }
+ last_nanos = c.nanos;
+
+ update_can(c);
+ UpdateValid(last_nanos);
  }
+
  query_latest(vals, current_nanos);
 }
 
-void CANParser::UpdateCans(uint64_t nanos, const capnp::List<cereal::CanData>::Reader& cans) {
- //DEBUG("got %d messages\n", cans.size());
-
+void CANParser::update_can(const CanData &can) {
  bool bus_empty = true;
-
- // parse the messages
- for (const auto cmsg : cans) {
- if (cmsg.getSrc() != bus) {
+ for (const auto &frame : can.frames) {
+ if (frame.src != bus) {
  // DEBUG("skip %d: wrong bus\n", cmsg.getAddress());
  continue;
  }
  bus_empty = false;
 
- auto state_it = message_states.find(cmsg.getAddress());
+ auto state_it = message_states.find(frame.address);
  if (state_it == message_states.end()) {
  // DEBUG("skip %d: not specified\n", cmsg.getAddress());
  continue;
  }
-
- auto dat = cmsg.getDat();
-
- if (dat.size() > 64) {
- DEBUG("got message longer than 64 bytes: 0x%X %zu\n", cmsg.getAddress(), dat.size());
+ if (frame.dat.size() > 64) {
+ DEBUG("got message longer than 64 bytes: 0x%X %zu\n", frame.address, frame.dat.size());
  continue;
  }
 
  // TODO: this actually triggers for some cars. fix and enable this
- //if (dat.size() != state_it->second.size) {
+ // if (dat.size() != state_it->second.size) {
  // DEBUG("got message with unexpected length: expected %d, got %zu for %d", state_it->second.size, dat.size(), cmsg.getAddress());
  // continue;
  //}
 
- std::vector<uint8_t> data(dat.size(), 0);
- memcpy(data.data(), dat.begin(), dat.size());
- state_it->second.parse(nanos, data);
+ state_it->second.parse(can.nanos, frame.dat);
  }
 
  // update bus timeout
  if (!bus_empty) {
- last_nonempty_nanos = nanos;
+ last_nonempty_nanos = can.nanos;
  }
- bus_timeout = (nanos - last_nonempty_nanos) > bus_timeout_threshold;
-}
-#endif
-
-void CANParser::UpdateCans(uint64_t nanos, const capnp::DynamicStruct::Reader& cmsg) {
- // assume message struct is `cereal::CanData` and parse
- assert(cmsg.has("address") && cmsg.has("src") && cmsg.has("dat") && cmsg.has("busTime"));
-
- if (cmsg.get("src").as<uint8_t>() != bus) {
- DEBUG("skip %d: wrong bus\n", cmsg.get("address").as<uint32_t>());
- return;
- }
-
- auto state_it = message_states.find(cmsg.get("address").as<uint32_t>());
- if (state_it == message_states.end()) {
- DEBUG("skip %d: not specified\n", cmsg.get("address").as<uint32_t>());
- return;
- }
-
- auto dat = cmsg.get("dat").as<capnp::Data>();
- if (dat.size() > 64) return; // shouldn't ever happen
- std::vector<uint8_t> data(dat.size(), 0);
- memcpy(data.data(), dat.begin(), dat.size());
- state_it->second.parse(nanos, data);
+ bus_timeout = (can.nanos - last_nonempty_nanos) > bus_timeout_threshold;
 }
 
 void CANParser::UpdateValid(uint64_t nanos) {

can/parser_pyx.pyx

@@ -9,9 +9,11 @@ from libcpp.unordered_set cimport unordered_set
 from libc.stdint cimport uint32_t
 
 from .common cimport CANParser as cpp_CANParser
-from .common cimport dbc_lookup, SignalValue, DBC
+from .common cimport dbc_lookup, SignalValue, DBC, CanData, CanFrame
 
 import numbers
+import capnp
+from cereal import log
 from collections import defaultdict
 
 
@@ -75,10 +77,30 @@ cdef class CANParser:
  for l in v.values(): # no-cython-lint
  l.clear()
 
+ cdef CanFrame* frame
+ cdef CanData* can_data
+ cdef vector[CanData] can_data_array
+
+ can_data_array.reserve(len(strings))
+ try:
+ for s in strings:
+ with log.Event.from_bytes(s) as msg:
+ can_event = msg.sendCan if sendcan else msg.can
+ can_data = &(can_data_array.emplace_back())
+ can_data.nanos = msg.logMonoTime
+ can_data.frames.reserve(len(can_event))
+ for c in can_event:
+ frame = &(can_data.frames.emplace_back())
+ frame.src = c.src
+ frame.address = c.address
+ frame.dat = c.dat
+ except capnp.lib.capnp.KjException as ex:
+ raise RuntimeError(str(ex))
+
  cdef vector[SignalValue] new_vals
  cdef unordered_set[uint32_t] updated_addrs
 
- self.can.update_strings(strings, new_vals, sendcan)
+ self.can.update(can_data_array, new_vals)
  cdef vector[SignalValue].iterator it = new_vals.begin()
  cdef SignalValue* cv
  while it != new_vals.end():