From 9eadf1787332bc9ec67c5272e257b65b7ebb1f33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Harald=20Sch=C3=A4fer?= Date: Tue, 4 Jun 2024 17:09:18 -0700 Subject: [PATCH 01/10] Rm duplicate enums? (#575) Duplicate enums --- log.capnp | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/log.capnp b/log.capnp index 692c598f7..36ca692dc 100644 --- a/log.capnp +++ b/log.capnp @@ -1140,29 +1140,6 @@ struct LateralPlan @0xe1e9318e2ae8b51e { u @1 :List(Float32); } - enum Desire { - none @0; - turnLeft @1; - turnRight @2; - laneChangeLeft @3; - laneChangeRight @4; - keepLeft @5; - keepRight @6; - } - - enum LaneChangeState { - off @0; - preLaneChange @1; - laneChangeStarting @2; - laneChangeFinishing @3; - } - - enum LaneChangeDirection { - none @0; - left @1; - right @2; - } - # deprecated curvatureDEPRECATED @22 :Float32; curvatureRateDEPRECATED @23 :Float32; From 4ef85977a490274264fee5a9309634f1348a819b Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Tue, 4 Jun 2024 22:21:46 -0700 Subject: [PATCH 02/10] remove generate_javascript.sh --- generate_javascript.sh | 26 -------------------------- 1 file changed, 26 deletions(-) delete mode 100755 generate_javascript.sh diff --git a/generate_javascript.sh b/generate_javascript.sh deleted file mode 100755 index d6525a64d..000000000 --- a/generate_javascript.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -rm -r gen/ts -rm -r gen/js - -mkdir gen/ts -mkdir gen/js - -echo "Installing needed npm modules" -npm i capnpc-ts capnp-ts - -capnpc -o node_modules/.bin/capnpc-ts:gen/ts log.capnp car.capnp -capnpc -o node_modules/.bin/capnpc-ts:gen/ts car.capnp - -cat log.capnp | egrep '\([a-zA-Z]*\.[^\s]+\.[^s]+\)' | sed 's/^.*([a-zA-Z]*\.\([a-zA-Z.]*\)).*/\1/' | while read line -do - TOKEN=`echo $line | sed 's/\./_/g'` - ROOT=`echo $line | sed 's/\..*$//g'` - cat gen/ts/log.capnp.ts | grep '^import.*'${TOKEN} - if [[ "$?" == "1" ]] - then - sed -i 's/^\(import {.*\)'${ROOT}'\(,*\) \(.*\)$/\1'${ROOT}', '${TOKEN}'\2 \3/' ./gen/ts/log.capnp.ts - fi -done - -tsc ./gen/ts/* --lib es2015 --outDir ./gen/js From 6d2cc6e22229a9c855d8474e5643b26fbf2b5976 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Harald=20Sch=C3=A4fer?= Date: Thu, 6 Jun 2024 13:58:15 -0700 Subject: [PATCH 03/10] Restructure cereal (#620) * Dont want those here * Move visionipc * doesnt make sense for now * Compiles * Move rest over * Somewhat works * Add .sp * Delete messaging/messaging_pyx.so * Delete messaging/messaging_pyx.cpp * ignore this too * Not needed * Add pyproj * Fix zmq port --- .github/workflows/repo.yml | 28 - .github/workflows/tests.yml | 61 - .gitignore | 1 + Dockerfile | 54 - LICENSE | 7 - README.md | 60 - SConscript | 49 +- SConstruct | 89 - __init__.py | 9 - car.capnp | 706 ------- codecov.yml | 8 - custom.capnp | 39 - include/c++.capnp | 26 - legacy.capnp | 574 ------ log.capnp | 2369 ------------------------ maptile.capnp | 49 - messaging/.gitignore | 9 - messaging/__init__.py | 306 --- messaging/bridge.cc | 92 - messaging/demo.cc | 50 - messaging/demo.py | 29 - messaging/event.cc | 2 +- messaging/impl_fake.cc | 2 +- messaging/impl_fake.h | 4 +- messaging/impl_msgq.cc | 18 +- messaging/impl_msgq.h | 4 +- messaging/impl_zmq.cc | 12 +- messaging/impl_zmq.h | 2 +- messaging/messaging.cc | 8 +- messaging/messaging.h | 96 +- messaging/messaging.pxd | 4 +- messaging/msgq.cc | 2 +- messaging/msgq.md | 54 - messaging/msgq_tests.cc | 130 +- messaging/socketmaster.cc | 210 --- messaging/stress.py | 14 - messaging/tests/__init__.py | 0 messaging/tests/test_fake.py | 193 -- messaging/tests/test_messaging.py | 247 --- messaging/tests/test_poller.py | 142 -- messaging/tests/test_pub_sub_master.py | 163 -- messaging/tests/test_services.py | 33 - pyproject.toml | 21 - services.py | 127 -- site_scons/site_tools/cython.py | 72 - visionipc/__init__.py | 2 +- visionipc/ipc.cc | 2 +- visionipc/tests/test_visionipc.py | 2 +- visionipc/visionbuf.cc | 2 +- visionipc/visionbuf.h | 2 +- visionipc/visionbuf_cl.cc | 2 +- visionipc/visionbuf_ion.cc | 2 +- visionipc/visionipc.pxd | 8 +- visionipc/visionipc_client.cc | 9 +- visionipc/visionipc_client.h | 5 +- visionipc/visionipc_server.cc | 8 +- visionipc/visionipc_server.h | 4 +- visionipc/visionipc_tests.cc | 6 +- 58 files changed, 151 insertions(+), 6078 deletions(-) delete mode 100644 .github/workflows/repo.yml delete mode 100644 .github/workflows/tests.yml delete mode 100644 Dockerfile delete mode 100644 LICENSE delete mode 100644 README.md delete mode 100644 SConstruct delete mode 100644 __init__.py delete mode 100644 car.capnp delete mode 100644 codecov.yml delete mode 100644 custom.capnp delete mode 100644 include/c++.capnp delete mode 100644 legacy.capnp delete mode 100644 log.capnp delete mode 100644 maptile.capnp delete mode 100644 messaging/bridge.cc delete mode 100644 messaging/demo.cc delete mode 100644 messaging/demo.py delete mode 100644 messaging/msgq.md delete mode 100644 messaging/socketmaster.cc delete mode 100644 messaging/stress.py delete mode 100644 messaging/tests/__init__.py delete mode 100644 messaging/tests/test_fake.py delete mode 100755 messaging/tests/test_messaging.py delete mode 100644 messaging/tests/test_poller.py delete mode 100755 messaging/tests/test_pub_sub_master.py delete mode 100755 messaging/tests/test_services.py delete mode 100644 pyproject.toml delete mode 100755 services.py delete mode 100644 site_scons/site_tools/cython.py diff --git a/.github/workflows/repo.yml b/.github/workflows/repo.yml deleted file mode 100644 index 4c4a7c4bb..000000000 --- a/.github/workflows/repo.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: repo - -on: - schedule: - - cron: "0 15 1 * *" - workflow_dispatch: - -jobs: - pre-commit-autoupdate: - name: pre-commit autoupdate - runs-on: ubuntu-latest - container: - image: ghcr.io/commaai/cereal:latest - steps: - - uses: actions/checkout@v3 - - name: pre-commit autoupdate - run: | - git config --global --add safe.directory '*' - pre-commit autoupdate - - name: Create Pull Request - uses: peter-evans/create-pull-request@5b4a9f6a9e2af26e5f02351490b90d01eb8ec1e5 - with: - token: ${{ secrets.ACTIONS_CREATE_PR_PAT }} - commit-message: Update pre-commit hook versions - title: 'pre-commit: autoupdate hooks' - branch: pre-commit-updates - base: master - delete-branch: true diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml deleted file mode 100644 index ddc52acd4..000000000 --- a/.github/workflows/tests.yml +++ /dev/null @@ -1,61 +0,0 @@ -name: tests - -on: [push, pull_request] - -env: - DOCKER_REGISTRY: ghcr.io/commaai - RUN: docker run -e PYTHONWARNINGS=error --shm-size 1G --name cereal cereal /bin/sh -c - RUN_NAMED: docker run -e PYTHONWARNINGS=error --shm-size 1G --rm cereal /bin/sh -c - CI_RUN: docker run -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID --rm cerealci /bin/bash -c - BUILD: docker buildx build --pull --load --cache-to type=inline --cache-from $DOCKER_REGISTRY/cereal:latest -t cereal -f Dockerfile . - PYTHONWARNINGS: error - -jobs: - build: - name: build - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Build docker image - run: eval "$BUILD" - - name: Push to dockerhub - if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/cereal' - run: | - docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} - docker tag cereal $DOCKER_REGISTRY/cereal:latest - docker push $DOCKER_REGISTRY/cereal:latest - - unit_tests: - name: unit tests - runs-on: ubuntu-latest - strategy: - matrix: - flags: ['', '--asan', '--ubsan'] - backend: ['MSGQ', 'ZMQ'] - steps: - - uses: actions/checkout@v3 - - name: Build docker image - run: eval "$BUILD" - - name: C++ tests - run: | - $RUN "export ${{ matrix.backend }}=1 && \ - scons ${{ matrix.flags }} -j$(nproc) && \ - messaging/test_runner && \ - visionipc/test_runner" - - name: python tests - run: $RUN_NAMED "${{ matrix.backend }}=1 coverage run -m unittest discover ." - - name: Upload coverage - run: | - docker commit cereal cerealci - $CI_RUN "cd /project/cereal && bash <(curl -s https://codecov.io/bash) -v -F unit_tests_${{ matrix.backend }}" - - static_analysis: - name: static analysis - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Build docker image - run: eval "$BUILD" - - name: Static analysis - # TODO: a package pre-commit installs has a warning, remove the unset once that's fixed - run: $RUN "git init && git add -A && unset PYTHONWARNINGS && pre-commit run --all" diff --git a/.gitignore b/.gitignore index d6ec0036c..6db3c7cd4 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ __pycache__ .*.swp .*.swo *.os +*.so *.o *.a diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index d2e3ab591..000000000 --- a/Dockerfile +++ /dev/null @@ -1,54 +0,0 @@ -FROM ubuntu:24.04 - -ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get install -y --no-install-recommends \ - autoconf \ - build-essential \ - ca-certificates \ - capnproto \ - clang \ - cppcheck \ - curl \ - git \ - libbz2-dev \ - libcapnp-dev \ - libclang-rt-dev \ - libffi-dev \ - liblzma-dev \ - libncurses5-dev \ - libncursesw5-dev \ - libreadline-dev \ - libsqlite3-dev \ - libssl-dev \ - libtool \ - libzmq3-dev \ - llvm \ - make \ - cmake \ - ocl-icd-opencl-dev \ - opencl-headers \ - python3-dev \ - python3-pip \ - tk-dev \ - wget \ - xz-utils \ - zlib1g-dev \ - && rm -rf /var/lib/apt/lists/* - -RUN pip3 install --break-system-packages --no-cache-dir pyyaml Cython scons pycapnp pre-commit ruff parameterized coverage numpy - -WORKDIR /project/ -RUN cd /tmp/ && \ - git clone -b v2.x --depth 1 https://github.com/catchorg/Catch2.git && \ - cd Catch2 && \ - mv single_include/catch2/ /project/ && \ - cd .. \ - rm -rf Catch2 - -WORKDIR /project/cereal - -ENV PYTHONPATH=/project - -COPY . . -RUN rm -rf .git && \ - scons -c && scons -j$(nproc) diff --git a/LICENSE b/LICENSE deleted file mode 100644 index f1fd199c6..000000000 --- a/LICENSE +++ /dev/null @@ -1,7 +0,0 @@ -Copyright (c) 2020, Comma.ai, Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md deleted file mode 100644 index e3326aab0..000000000 --- a/README.md +++ /dev/null @@ -1,60 +0,0 @@ -# What is cereal? [![cereal tests](https://github.com/commaai/cereal/workflows/tests/badge.svg?event=push)](https://github.com/commaai/cereal/actions) [![codecov](https://codecov.io/gh/commaai/cereal/branch/master/graph/badge.svg)](https://codecov.io/gh/commaai/cereal) - -cereal is both a messaging spec for robotics systems as well as generic high performance IPC pub sub messaging with a single publisher and multiple subscribers. - -Imagine this use case: -* A sensor process reads gyro measurements directly from an IMU and publishes a `sensorEvents` packet -* A calibration process subscribes to the `sensorEvents` packet to use the IMU -* A localization process subscribes to the `sensorEvents` packet to use the IMU also - - -## Messaging Spec - -You'll find the message types in [log.capnp](log.capnp). It uses [Cap'n proto](https://capnproto.org/capnp-tool.html) and defines one struct called `Event`. - -All `Events` have a `logMonoTime` and a `valid`. Then a big union defines the packet type. - -### Best Practices - -- **All fields must describe quantities in SI units**, unless otherwise specified in the field name. -- In the context of the message they are in, field names should be completely unambiguous. -- All values should be easy to plot and be human-readable with minimal parsing. - -### Maintaining backwards-compatibility - -When making changes to the messaging spec you want to maintain backwards-compatibility, such that old logs can -be parsed with a new version of cereal. Adding structs and adding members to structs is generally safe, most other -things are not. Read more details [here](https://capnproto.org/language.html). - -### Custom forks - -Forks of [openpilot](https://github.com/commaai/openpilot) might want to add things to the messaging -spec, however this could conflict with future changes made in mainline cereal/openpilot. Rebasing against mainline openpilot -then means breaking backwards-compatibility with all old logs of your fork. So we added reserved events in -[custom.capnp](custom.capnp) that we will leave empty in mainline cereal/openpilot. **If you only modify those, you can ensure your -fork will remain backwards-compatible with all versions of mainline cereal/openpilot and your fork.** - -## Pub Sub Backends - -cereal supports two backends, one based on [zmq](https://zeromq.org/) and another called [msgq](messaging/msgq.cc), a custom pub sub based on shared memory that doesn't require the bytes to pass through the kernel. - -Example ---- -```python -import cereal.messaging as messaging - -# in subscriber -sm = messaging.SubMaster(['sensorEvents']) -while 1: - sm.update() - print(sm['sensorEvents']) - -``` - -```python -# in publisher -pm = messaging.PubMaster(['sensorEvents']) -dat = messaging.new_message('sensorEvents', size=1) -dat.sensorEvents[0] = {"gyro": {"v": [0.1, -0.1, 0.1]}} -pm.send('sensorEvents', dat) -``` diff --git a/SConscript b/SConscript index b120cd207..f8eb6a5e6 100644 --- a/SConscript +++ b/SConscript @@ -1,28 +1,12 @@ Import('env', 'envCython', 'arch', 'common') -import shutil -cereal_dir = Dir('.') +visionipc_dir = Dir('visionipc') gen_dir = Dir('gen') -messaging_dir = Dir('messaging') -# Build cereal - -schema_files = ['log.capnp', 'car.capnp', 'legacy.capnp', 'custom.capnp'] -env.Command(["gen/c/include/c++.capnp.h"], [], "mkdir -p " + gen_dir.path + "/c/include && touch $TARGETS") -env.Command([f'gen/cpp/{s}.c++' for s in schema_files] + [f'gen/cpp/{s}.h' for s in schema_files], - schema_files, - f"capnpc --src-prefix={cereal_dir.path} $SOURCES -o c++:{gen_dir.path}/cpp/") - -# TODO: remove non shared cereal and messaging -cereal_objects = env.SharedObject([f'gen/cpp/{s}.c++' for s in schema_files]) - -cereal = env.Library('cereal', cereal_objects) -env.SharedLibrary('cereal_shared', cereal_objects) # Build messaging -services_h = env.Command(['services.h'], ['services.py'], 'python3 ' + cereal_dir.path + '/services.py > $TARGET') messaging_objects = env.SharedObject([ 'messaging/messaging.cc', @@ -31,29 +15,22 @@ messaging_objects = env.SharedObject([ 'messaging/impl_msgq.cc', 'messaging/impl_fake.cc', 'messaging/msgq.cc', - 'messaging/socketmaster.cc', ]) - messaging = env.Library('messaging', messaging_objects) -Depends('messaging/impl_zmq.cc', services_h) +messaging_python = envCython.Program('messaging/messaging_pyx.so', 'messaging/messaging_pyx.pyx', LIBS=envCython["LIBS"]+[messaging, "zmq", common]) -env.Program('messaging/bridge', ['messaging/bridge.cc'], LIBS=[messaging, 'zmq', common]) -Depends('messaging/bridge.cc', services_h) +if GetOption('extras'): + env.Program('messaging/test_runner', ['messaging/test_runner.cc', 'messaging/msgq_tests.cc'], LIBS=[messaging, common]) -messaging_python = envCython.Program('messaging/messaging_pyx.so', 'messaging/messaging_pyx.pyx', LIBS=envCython["LIBS"]+[messaging, "zmq", common]) # Build Vision IPC -vipc_sources = [ - 'visionipc/ipc.cc', - 'visionipc/visionipc_server.cc', - 'visionipc/visionipc_client.cc', - 'visionipc/visionbuf.cc', -] +vipc_files = ['ipc.cc', 'visionipc_server.cc', 'visionipc_client.cc', 'visionbuf.cc'] +vipc_sources = [f'{visionipc_dir.abspath}/{f}' for f in vipc_files] if arch == "larch64": - vipc_sources += ['visionipc/visionbuf_ion.cc'] + vipc_sources += [f'{visionipc_dir.abspath}/visionbuf_ion.cc'] else: - vipc_sources += ['visionipc/visionbuf_cl.cc'] + vipc_sources += [f'{visionipc_dir.abspath}/visionbuf_cl.cc'] vipc_objects = env.SharedObject(vipc_sources) visionipc = env.Library('visionipc', vipc_objects) @@ -65,14 +42,12 @@ if arch == "Darwin": vipc_frameworks.append('OpenCL') else: vipc_libs.append('OpenCL') -envCython.Program('visionipc/visionipc_pyx.so', 'visionipc/visionipc_pyx.pyx', +envCython.Program(f'{visionipc_dir.abspath}/visionipc_pyx.so', f'{visionipc_dir.abspath}/visionipc_pyx.pyx', LIBS=vipc_libs, FRAMEWORKS=vipc_frameworks) if GetOption('extras'): - env.Program('messaging/test_runner', ['messaging/test_runner.cc', 'messaging/msgq_tests.cc'], LIBS=[messaging, common]) - - env.Program('visionipc/test_runner', ['visionipc/test_runner.cc', 'visionipc/visionipc_tests.cc'], + env.Program('visionipc/test_runner', + ['visionipc/test_runner.cc', 'visionipc/visionipc_tests.cc'], LIBS=['pthread'] + vipc_libs, FRAMEWORKS=vipc_frameworks) - -Export('cereal', 'messaging', 'messaging_python', 'visionipc') \ No newline at end of file +Export('visionipc', 'messaging', 'messaging_python') diff --git a/SConstruct b/SConstruct deleted file mode 100644 index 408c7bfa9..000000000 --- a/SConstruct +++ /dev/null @@ -1,89 +0,0 @@ -import os -import platform -import subprocess -import sysconfig -import numpy as np - -arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip() -if platform.system() == "Darwin": - arch = "Darwin" - -common = '' - -cpppath = [ - f"#/../", - '/usr/lib/include', - '/opt/homebrew/include', - sysconfig.get_paths()['include'], -] - -libpath = [ - '/opt/homebrew/lib', -] - -AddOption('--minimal', - action='store_false', - dest='extras', - default=True, - help='the minimum build. no tests, tools, etc.') - -AddOption('--asan', - action='store_true', - help='turn on ASAN') - -AddOption('--ubsan', - action='store_true', - help='turn on UBSan') - -ccflags = [] -ldflags = [] -if GetOption('ubsan'): - flags = [ - "-fsanitize=undefined", - "-fno-sanitize-recover=undefined", - ] - ccflags += flags - ldflags += flags -elif GetOption('asan'): - ccflags += ["-fsanitize=address", "-fno-omit-frame-pointer"] - ldflags += ["-fsanitize=address"] - -env = Environment( - ENV=os.environ, - CC='clang', - CXX='clang++', - CCFLAGS=[ - "-g", - "-fPIC", - "-O2", - "-Wunused", - "-Werror", - "-Wshadow", - "-Wno-vla-cxx-extension", - ] + ccflags, - LDFLAGS=ldflags, - LINKFLAGS=ldflags, - - CFLAGS="-std=gnu11", - CXXFLAGS="-std=c++1z", - CPPPATH=cpppath, - LIBPATH=libpath, - CYTHONCFILESUFFIX=".cpp", - tools=["default", "cython"] -) - -Export('env', 'arch', 'common') - -envCython = env.Clone(LIBS=[]) -envCython["CPPPATH"] += [np.get_include()] -envCython["CCFLAGS"] += ["-Wno-#warnings", "-Wno-shadow", "-Wno-deprecated-declarations"] -envCython["CCFLAGS"].remove('-Werror') -if arch == "Darwin": - envCython["LINKFLAGS"] = ["-bundle", "-undefined", "dynamic_lookup"] -else: - envCython["LINKFLAGS"] = ["-pthread", "-shared"] - -Export('envCython') - - -SConscript(['SConscript']) diff --git a/__init__.py b/__init__.py deleted file mode 100644 index 89c5cf38e..000000000 --- a/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -import os -import capnp - -CEREAL_PATH = os.path.dirname(os.path.abspath(__file__)) -capnp.remove_import_hook() - -log = capnp.load(os.path.join(CEREAL_PATH, "log.capnp")) -car = capnp.load(os.path.join(CEREAL_PATH, "car.capnp")) -custom = capnp.load(os.path.join(CEREAL_PATH, "custom.capnp")) diff --git a/car.capnp b/car.capnp deleted file mode 100644 index d7ec1b248..000000000 --- a/car.capnp +++ /dev/null @@ -1,706 +0,0 @@ -using Cxx = import "./include/c++.capnp"; -$Cxx.namespace("cereal"); - -@0x8e2af1e708af8b8d; - -# ******* events causing controls state machine transition ******* - -struct CarEvent @0x9b1657f34caf3ad3 { - name @0 :EventName; - - # event types - enable @1 :Bool; - noEntry @2 :Bool; - warning @3 :Bool; # alerts presented only when enabled or soft disabling - userDisable @4 :Bool; - softDisable @5 :Bool; - immediateDisable @6 :Bool; - preEnable @7 :Bool; - permanent @8 :Bool; # alerts presented regardless of openpilot state - overrideLateral @10 :Bool; - overrideLongitudinal @9 :Bool; - - enum EventName @0xbaa8c5d505f727de { - canError @0; - steerUnavailable @1; - wrongGear @4; - doorOpen @5; - seatbeltNotLatched @6; - espDisabled @7; - wrongCarMode @8; - steerTempUnavailable @9; - reverseGear @10; - buttonCancel @11; - buttonEnable @12; - pedalPressed @13; # exits active state - preEnableStandstill @73; # added during pre-enable state with brake - gasPressedOverride @108; # added when user is pressing gas with no disengage on gas - steerOverride @114; - cruiseDisabled @14; - speedTooLow @17; - outOfSpace @18; - overheat @19; - calibrationIncomplete @20; - calibrationInvalid @21; - calibrationRecalibrating @117; - controlsMismatch @22; - pcmEnable @23; - pcmDisable @24; - radarFault @26; - brakeHold @28; - parkBrake @29; - manualRestart @30; - lowSpeedLockout @31; - joystickDebug @34; - steerTempUnavailableSilent @35; - resumeRequired @36; - preDriverDistracted @37; - promptDriverDistracted @38; - driverDistracted @39; - preDriverUnresponsive @43; - promptDriverUnresponsive @44; - driverUnresponsive @45; - belowSteerSpeed @46; - lowBattery @48; - accFaulted @51; - sensorDataInvalid @52; - commIssue @53; - commIssueAvgFreq @109; - tooDistracted @54; - posenetInvalid @55; - soundsUnavailable @56; - preLaneChangeLeft @57; - preLaneChangeRight @58; - laneChange @59; - lowMemory @63; - stockAeb @64; - ldw @65; - carUnrecognized @66; - invalidLkasSetting @69; - speedTooHigh @70; - laneChangeBlocked @71; - relayMalfunction @72; - stockFcw @74; - startup @75; - startupNoCar @76; - startupNoControl @77; - startupMaster @78; - startupNoFw @104; - fcw @79; - steerSaturated @80; - belowEngageSpeed @84; - noGps @85; - wrongCruiseMode @87; - modeldLagging @89; - deviceFalling @90; - fanMalfunction @91; - cameraMalfunction @92; - cameraFrameRate @110; - processNotRunning @95; - dashcamMode @96; - controlsInitializing @98; - usbError @99; - roadCameraError @100; - driverCameraError @101; - wideRoadCameraError @102; - highCpuUsage @105; - cruiseMismatch @106; - lkasDisabled @107; - canBusMissing @111; - controlsdLagging @112; - resumeBlocked @113; - steerTimeLimit @115; - vehicleSensorsInvalid @116; - locationdTemporaryError @103; - locationdPermanentError @118; - paramsdTemporaryError @50; - paramsdPermanentError @119; - actuatorsApiUnavailable @120; - - radarCanErrorDEPRECATED @15; - communityFeatureDisallowedDEPRECATED @62; - radarCommIssueDEPRECATED @67; - driverMonitorLowAccDEPRECATED @68; - gasUnavailableDEPRECATED @3; - dataNeededDEPRECATED @16; - modelCommIssueDEPRECATED @27; - ipasOverrideDEPRECATED @33; - geofenceDEPRECATED @40; - driverMonitorOnDEPRECATED @41; - driverMonitorOffDEPRECATED @42; - calibrationProgressDEPRECATED @47; - invalidGiraffeHondaDEPRECATED @49; - invalidGiraffeToyotaDEPRECATED @60; - internetConnectivityNeededDEPRECATED @61; - whitePandaUnsupportedDEPRECATED @81; - commIssueWarningDEPRECATED @83; - focusRecoverActiveDEPRECATED @86; - neosUpdateRequiredDEPRECATED @88; - modelLagWarningDEPRECATED @93; - startupOneplusDEPRECATED @82; - startupFuzzyFingerprintDEPRECATED @97; - noTargetDEPRECATED @25; - brakeUnavailableDEPRECATED @2; - plannerErrorDEPRECATED @32; - gpsMalfunctionDEPRECATED @94; - } -} - -# ******* main car state @ 100hz ******* -# all speeds in m/s - -struct CarState { - events @13 :List(CarEvent); - - # CAN health - canValid @26 :Bool; # invalid counter/checksums - canTimeout @40 :Bool; # CAN bus dropped out - canErrorCounter @48 :UInt32; - canRcvTimeout @49 :Bool; - - # car speed - vEgo @1 :Float32; # best estimate of speed - aEgo @16 :Float32; # best estimate of acceleration - vEgoRaw @17 :Float32; # unfiltered speed from CAN sensors - vEgoCluster @44 :Float32; # best estimate of speed shown on car's instrument cluster, used for UI - - yawRate @22 :Float32; # best estimate of yaw rate - standstill @18 :Bool; - wheelSpeeds @2 :WheelSpeeds; - - # gas pedal, 0.0-1.0 - gas @3 :Float32; # this is user pedal only - gasPressed @4 :Bool; # this is user pedal only - - engineRpm @46 :Float32; - - # brake pedal, 0.0-1.0 - brake @5 :Float32; # this is user pedal only - brakePressed @6 :Bool; # this is user pedal only - regenBraking @45 :Bool; # this is user pedal only - parkingBrake @39 :Bool; - brakeHoldActive @38 :Bool; - - # steering wheel - steeringAngleDeg @7 :Float32; - steeringAngleOffsetDeg @37 :Float32; # Offset betweens sensors in case there multiple - steeringRateDeg @15 :Float32; - steeringTorque @8 :Float32; # TODO: standardize units - steeringTorqueEps @27 :Float32; # TODO: standardize units - steeringPressed @9 :Bool; # if the user is using the steering wheel - steerFaultTemporary @35 :Bool; # temporary EPS fault - steerFaultPermanent @36 :Bool; # permanent EPS fault - stockAeb @30 :Bool; - stockFcw @31 :Bool; - espDisabled @32 :Bool; - accFaulted @42 :Bool; - carFaultedNonCritical @47 :Bool; # some ECU is faulted, but car remains controllable - - # cruise state - cruiseState @10 :CruiseState; - - # gear - gearShifter @14 :GearShifter; - - # button presses - buttonEvents @11 :List(ButtonEvent); - leftBlinker @20 :Bool; - rightBlinker @21 :Bool; - genericToggle @23 :Bool; - - # lock info - doorOpen @24 :Bool; - seatbeltUnlatched @25 :Bool; - - # clutch (manual transmission only) - clutchPressed @28 :Bool; - - # blindspot sensors - leftBlindspot @33 :Bool; # Is there something blocking the left lane change - rightBlindspot @34 :Bool; # Is there something blocking the right lane change - - fuelGauge @41 :Float32; # battery or fuel tank level from 0.0 to 1.0 - charging @43 :Bool; - - # process meta - cumLagMs @50 :Float32; - - struct WheelSpeeds { - # optional wheel speeds - fl @0 :Float32; - fr @1 :Float32; - rl @2 :Float32; - rr @3 :Float32; - } - - struct CruiseState { - enabled @0 :Bool; - speed @1 :Float32; - speedCluster @6 :Float32; # Set speed as shown on instrument cluster - available @2 :Bool; - speedOffset @3 :Float32; - standstill @4 :Bool; - nonAdaptive @5 :Bool; - } - - enum GearShifter { - unknown @0; - park @1; - drive @2; - neutral @3; - reverse @4; - sport @5; - low @6; - brake @7; - eco @8; - manumatic @9; - } - - # send on change - struct ButtonEvent { - pressed @0 :Bool; - type @1 :Type; - - enum Type { - unknown @0; - leftBlinker @1; - rightBlinker @2; - accelCruise @3; - decelCruise @4; - cancel @5; - altButton1 @6; - altButton2 @7; - altButton3 @8; - setCruise @9; - resumeCruise @10; - gapAdjustCruise @11; - } - } - - # deprecated - errorsDEPRECATED @0 :List(CarEvent.EventName); - brakeLightsDEPRECATED @19 :Bool; - steeringRateLimitedDEPRECATED @29 :Bool; - canMonoTimesDEPRECATED @12: List(UInt64); -} - -# ******* radar state @ 20hz ******* - -struct RadarData @0x888ad6581cf0aacb { - errors @0 :List(Error); - points @1 :List(RadarPoint); - - enum Error { - canError @0; - fault @1; - wrongConfig @2; - } - - # similar to LiveTracks - # is one timestamp valid for all? I think so - struct RadarPoint { - trackId @0 :UInt64; # no trackId reuse - - # these 3 are the minimum required - dRel @1 :Float32; # m from the front bumper of the car - yRel @2 :Float32; # m - vRel @3 :Float32; # m/s - - # these are optional and valid if they are not NaN - aRel @4 :Float32; # m/s^2 - yvRel @5 :Float32; # m/s - - # some radars flag measurements VS estimates - measured @6 :Bool; - } - - # deprecated - canMonoTimesDEPRECATED @2 :List(UInt64); -} - -# ******* car controls @ 100hz ******* - -struct CarControl { - # must be true for any actuator commands to work - enabled @0 :Bool; - latActive @11: Bool; - longActive @12: Bool; - - # Actuator commands as computed by controlsd - actuators @6 :Actuators; - - # moved to CarOutput - actuatorsOutputDEPRECATED @10 :Actuators; - - leftBlinker @15: Bool; - rightBlinker @16: Bool; - - orientationNED @13 :List(Float32); - angularVelocity @14 :List(Float32); - - cruiseControl @4 :CruiseControl; - hudControl @5 :HUDControl; - - struct Actuators { - # range from 0.0 - 1.0 - gas @0: Float32; - brake @1: Float32; - # range from -1.0 - 1.0 - steer @2: Float32; - # value sent over can to the car - steerOutputCan @8: Float32; - steeringAngleDeg @3: Float32; - - curvature @7: Float32; - - speed @6: Float32; # m/s - accel @4: Float32; # m/s^2 - longControlState @5: LongControlState; - - enum LongControlState @0xe40f3a917d908282{ - off @0; - pid @1; - stopping @2; - starting @3; - } - } - - struct CruiseControl { - cancel @0: Bool; - resume @1: Bool; - override @4: Bool; - speedOverrideDEPRECATED @2: Float32; - accelOverrideDEPRECATED @3: Float32; - } - - struct HUDControl { - speedVisible @0: Bool; - setSpeed @1: Float32; - lanesVisible @2: Bool; - leadVisible @3: Bool; - visualAlert @4: VisualAlert; - audibleAlert @5: AudibleAlert; - rightLaneVisible @6: Bool; - leftLaneVisible @7: Bool; - rightLaneDepart @8: Bool; - leftLaneDepart @9: Bool; - leadDistanceBars @10: Int8; # 1-3: 1 is closest, 3 is farthest. some ports may utilize 2-4 bars instead - - enum VisualAlert { - # these are the choices from the Honda - # map as good as you can for your car - none @0; - fcw @1; - steerRequired @2; - brakePressed @3; - wrongGear @4; - seatbeltUnbuckled @5; - speedTooHigh @6; - ldw @7; - } - - enum AudibleAlert { - none @0; - - engage @1; - disengage @2; - refuse @3; - - warningSoft @4; - warningImmediate @5; - - prompt @6; - promptRepeat @7; - promptDistracted @8; - } - } - - gasDEPRECATED @1 :Float32; - brakeDEPRECATED @2 :Float32; - steeringTorqueDEPRECATED @3 :Float32; - activeDEPRECATED @7 :Bool; - rollDEPRECATED @8 :Float32; - pitchDEPRECATED @9 :Float32; -} - -struct CarOutput { - # Any car specific rate limits or quirks applied by - # the CarController are reflected in actuatorsOutput - # and matches what is sent to the car - actuatorsOutput @0 :CarControl.Actuators; -} - -# ****** car param ****** - -struct CarParams { - carName @0 :Text; - carFingerprint @1 :Text; - fuzzyFingerprint @55 :Bool; - - notCar @66 :Bool; # flag for non-car robotics platforms - - pcmCruise @3 :Bool; # is openpilot's state tied to the PCM's cruise state? - enableDsu @5 :Bool; # driving support unit - enableBsm @56 :Bool; # blind spot monitoring - flags @64 :UInt32; # flags for car specific quirks - experimentalLongitudinalAvailable @71 :Bool; - - minEnableSpeed @7 :Float32; - minSteerSpeed @8 :Float32; - safetyConfigs @62 :List(SafetyConfig); - alternativeExperience @65 :Int16; # panda flag for features like no disengage on gas - - # Car docs fields - maxLateralAccel @68 :Float32; - autoResumeSng @69 :Bool; # describes whether car can resume from a stop automatically - - # things about the car in the manual - mass @17 :Float32; # [kg] curb weight: all fluids no cargo - wheelbase @18 :Float32; # [m] distance from rear axle to front axle - centerToFront @19 :Float32; # [m] distance from center of mass to front axle - steerRatio @20 :Float32; # [] ratio of steering wheel angle to front wheel angle - steerRatioRear @21 :Float32; # [] ratio of steering wheel angle to rear wheel angle (usually 0) - - # things we can derive - rotationalInertia @22 :Float32; # [kg*m2] body rotational inertia - tireStiffnessFactor @72 :Float32; # scaling factor used in calculating tireStiffness[Front,Rear] - tireStiffnessFront @23 :Float32; # [N/rad] front tire coeff of stiff - tireStiffnessRear @24 :Float32; # [N/rad] rear tire coeff of stiff - - longitudinalTuning @25 :LongitudinalPIDTuning; - lateralParams @48 :LateralParams; - lateralTuning :union { - pid @26 :LateralPIDTuning; - indiDEPRECATED @27 :LateralINDITuning; - lqrDEPRECATED @40 :LateralLQRTuning; - torque @67 :LateralTorqueTuning; - } - - steerLimitAlert @28 :Bool; - steerLimitTimer @47 :Float32; # time before steerLimitAlert is issued - - vEgoStopping @29 :Float32; # Speed at which the car goes into stopping state - vEgoStarting @59 :Float32; # Speed at which the car goes into starting state - stoppingControl @31 :Bool; # Does the car allow full control even at lows speeds when stopping - steerControlType @34 :SteerControlType; - radarUnavailable @35 :Bool; # True when radar objects aren't visible on CAN or aren't parsed out - stopAccel @60 :Float32; # Required acceleration to keep vehicle stationary - stoppingDecelRate @52 :Float32; # m/s^2/s while trying to stop - startAccel @32 :Float32; # Required acceleration to get car moving - startingState @70 :Bool; # Does this car make use of special starting state - - steerActuatorDelay @36 :Float32; # Steering wheel actuator delay in seconds - longitudinalActuatorDelayLowerBound @61 :Float32; # Gas/Brake actuator delay in seconds, lower bound - longitudinalActuatorDelayUpperBound @58 :Float32; # Gas/Brake actuator delay in seconds, upper bound - openpilotLongitudinalControl @37 :Bool; # is openpilot doing the longitudinal control? - carVin @38 :Text; # VIN number queried during fingerprinting - dashcamOnly @41: Bool; - passive @73: Bool; # is openpilot in control? - transmissionType @43 :TransmissionType; - carFw @44 :List(CarFw); - - radarTimeStep @45: Float32 = 0.05; # time delta between radar updates, 20Hz is very standard - fingerprintSource @49: FingerprintSource; - networkLocation @50 :NetworkLocation; # Where Panda/C2 is integrated into the car's CAN network - - wheelSpeedFactor @63 :Float32; # Multiplier on wheels speeds to computer actual speeds - - struct SafetyConfig { - safetyModel @0 :SafetyModel; - safetyParam @3 :UInt16; - safetyParamDEPRECATED @1 :Int16; - safetyParam2DEPRECATED @2 :UInt32; - } - - struct LateralParams { - torqueBP @0 :List(Int32); - torqueV @1 :List(Int32); - } - - struct LateralPIDTuning { - kpBP @0 :List(Float32); - kpV @1 :List(Float32); - kiBP @2 :List(Float32); - kiV @3 :List(Float32); - kf @4 :Float32; - } - - struct LateralTorqueTuning { - useSteeringAngle @0 :Bool; - kp @1 :Float32; - ki @2 :Float32; - friction @3 :Float32; - kf @4 :Float32; - steeringAngleDeadzoneDeg @5 :Float32; - latAccelFactor @6 :Float32; - latAccelOffset @7 :Float32; - } - - struct LongitudinalPIDTuning { - kpBP @0 :List(Float32); - kpV @1 :List(Float32); - kiBP @2 :List(Float32); - kiV @3 :List(Float32); - kf @6 :Float32; - deadzoneBP @4 :List(Float32); - deadzoneV @5 :List(Float32); - } - - struct LateralINDITuning { - outerLoopGainBP @4 :List(Float32); - outerLoopGainV @5 :List(Float32); - innerLoopGainBP @6 :List(Float32); - innerLoopGainV @7 :List(Float32); - timeConstantBP @8 :List(Float32); - timeConstantV @9 :List(Float32); - actuatorEffectivenessBP @10 :List(Float32); - actuatorEffectivenessV @11 :List(Float32); - - outerLoopGainDEPRECATED @0 :Float32; - innerLoopGainDEPRECATED @1 :Float32; - timeConstantDEPRECATED @2 :Float32; - actuatorEffectivenessDEPRECATED @3 :Float32; - } - - struct LateralLQRTuning { - scale @0 :Float32; - ki @1 :Float32; - dcGain @2 :Float32; - - # State space system - a @3 :List(Float32); - b @4 :List(Float32); - c @5 :List(Float32); - - k @6 :List(Float32); # LQR gain - l @7 :List(Float32); # Kalman gain - } - - enum SafetyModel { - silent @0; - hondaNidec @1; - toyota @2; - elm327 @3; - gm @4; - hondaBoschGiraffe @5; - ford @6; - cadillac @7; - hyundai @8; - chrysler @9; - tesla @10; - subaru @11; - gmPassive @12; - mazda @13; - nissan @14; - volkswagen @15; - toyotaIpas @16; - allOutput @17; - gmAscm @18; - noOutput @19; # like silent but without silent CAN TXs - hondaBosch @20; - volkswagenPq @21; - subaruPreglobal @22; # pre-Global platform - hyundaiLegacy @23; - hyundaiCommunity @24; - volkswagenMlb @25; - hongqi @26; - body @27; - hyundaiCanfd @28; - volkswagenMqbEvo @29; - chryslerCusw @30; - psa @31; - } - - enum SteerControlType { - torque @0; - angle @1; - - curvatureDEPRECATED @2; - } - - enum TransmissionType { - unknown @0; - automatic @1; # Traditional auto, including DSG - manual @2; # True "stick shift" only - direct @3; # Electric vehicle or other direct drive - cvt @4; - } - - struct CarFw { - ecu @0 :Ecu; - fwVersion @1 :Data; - address @2 :UInt32; - subAddress @3 :UInt8; - responseAddress @4 :UInt32; - request @5 :List(Data); - brand @6 :Text; - bus @7 :UInt8; - logging @8 :Bool; - obdMultiplexing @9 :Bool; - } - - enum Ecu { - eps @0; - abs @1; - fwdRadar @2; - fwdCamera @3; - engine @4; - unknown @5; - transmission @8; # Transmission Control Module - hybrid @18; # hybrid control unit, e.g. Chrysler's HCP, Honda's IMA Control Unit, Toyota's hybrid control computer - srs @9; # airbag - gateway @10; # can gateway - hud @11; # heads up display - combinationMeter @12; # instrument cluster - electricBrakeBooster @15; - shiftByWire @16; - adas @19; - cornerRadar @21; - hvac @20; - parkingAdas @7; # parking assist system ECU, e.g. Toyota's IPAS, Hyundai's RSPA, etc. - epb @22; # electronic parking brake - telematics @23; - body @24; # body control module - - # Toyota only - dsu @6; - - # Honda only - vsa @13; # Vehicle Stability Assist - programmedFuelInjection @14; - - debug @17; - } - - enum FingerprintSource { - can @0; - fw @1; - fixed @2; - } - - enum NetworkLocation { - fwdCamera @0; # Standard/default integration at LKAS camera - gateway @1; # Integration at vehicle's CAN gateway - } - - enableGasInterceptorDEPRECATED @2 :Bool; - enableCameraDEPRECATED @4 :Bool; - enableApgsDEPRECATED @6 :Bool; - steerRateCostDEPRECATED @33 :Float32; - isPandaBlackDEPRECATED @39 :Bool; - hasStockCameraDEPRECATED @57 :Bool; - safetyParamDEPRECATED @10 :Int16; - safetyModelDEPRECATED @9 :SafetyModel; - safetyModelPassiveDEPRECATED @42 :SafetyModel = silent; - minSpeedCanDEPRECATED @51 :Float32; - communityFeatureDEPRECATED @46: Bool; - startingAccelRateDEPRECATED @53 :Float32; - steerMaxBPDEPRECATED @11 :List(Float32); - steerMaxVDEPRECATED @12 :List(Float32); - gasMaxBPDEPRECATED @13 :List(Float32); - gasMaxVDEPRECATED @14 :List(Float32); - brakeMaxBPDEPRECATED @15 :List(Float32); - brakeMaxVDEPRECATED @16 :List(Float32); - directAccelControlDEPRECATED @30 :Bool; - maxSteeringAngleDegDEPRECATED @54 :Float32; -} diff --git a/codecov.yml b/codecov.yml deleted file mode 100644 index 83427c3ee..000000000 --- a/codecov.yml +++ /dev/null @@ -1,8 +0,0 @@ -comment: false -coverage: - status: - project: - default: - informational: true - patch: off - diff --git a/custom.capnp b/custom.capnp deleted file mode 100644 index 369222add..000000000 --- a/custom.capnp +++ /dev/null @@ -1,39 +0,0 @@ -using Cxx = import "./include/c++.capnp"; -$Cxx.namespace("cereal"); - -@0xb526ba661d550a59; - -# custom.capnp: a home for empty structs reserved for custom forks -# These structs are guaranteed to remain reserved and empty in mainline -# cereal, so use these if you want custom events in your fork. - -# you can rename the struct, but don't change the identifier -struct CustomReserved0 @0x81c2f05a394cf4af { -} - -struct CustomReserved1 @0xaedffd8f31e7b55d { -} - -struct CustomReserved2 @0xf35cc4560bbf6ec2 { -} - -struct CustomReserved3 @0xda96579883444c35 { -} - -struct CustomReserved4 @0x80ae746ee2596b11 { -} - -struct CustomReserved5 @0xa5cd762cd951a455 { -} - -struct CustomReserved6 @0xf98d843bfd7004a3 { -} - -struct CustomReserved7 @0xb86e6369214c01c8 { -} - -struct CustomReserved8 @0xf416ec09499d9d19 { -} - -struct CustomReserved9 @0xa1680744031fdb2d { -} diff --git a/include/c++.capnp b/include/c++.capnp deleted file mode 100644 index 2bda54717..000000000 --- a/include/c++.capnp +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright (c) 2013-2014 Sandstorm Development Group, Inc. and contributors -# Licensed under the MIT License: -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. - -@0xbdf87d7bb8304e81; -$namespace("capnp::annotations"); - -annotation namespace(file): Text; -annotation name(field, enumerant, struct, enum, interface, method, param, group, union): Text; diff --git a/legacy.capnp b/legacy.capnp deleted file mode 100644 index a8fa5e4a1..000000000 --- a/legacy.capnp +++ /dev/null @@ -1,574 +0,0 @@ -using Cxx = import "./include/c++.capnp"; -$Cxx.namespace("cereal"); - -@0x80ef1ec4889c2a63; - -# legacy.capnp: a home for deprecated structs - -struct LogRotate @0x9811e1f38f62f2d1 { - segmentNum @0 :Int32; - path @1 :Text; -} - -struct LiveUI @0xc08240f996aefced { - rearViewCam @0 :Bool; - alertText1 @1 :Text; - alertText2 @2 :Text; - awarenessStatus @3 :Float32; -} - -struct UiLayoutState @0x88dcce08ad29dda0 { - activeApp @0 :App; - sidebarCollapsed @1 :Bool; - mapEnabled @2 :Bool; - mockEngaged @3 :Bool; - - enum App @0x9917470acf94d285 { - home @0; - music @1; - nav @2; - settings @3; - none @4; - } -} - -struct OrbslamCorrection @0x8afd33dc9b35e1aa { - correctionMonoTime @0 :UInt64; - prePositionECEF @1 :List(Float64); - postPositionECEF @2 :List(Float64); - prePoseQuatECEF @3 :List(Float32); - postPoseQuatECEF @4 :List(Float32); - numInliers @5 :UInt32; -} - -struct EthernetPacket @0xa99a9d5b33cf5859 { - pkt @0 :Data; - ts @1 :Float32; -} - -struct CellInfo @0xcff7566681c277ce { - timestamp @0 :UInt64; - repr @1 :Text; # android toString() for now -} - -struct WifiScan @0xd4df5a192382ba0b { - bssid @0 :Text; - ssid @1 :Text; - capabilities @2 :Text; - frequency @3 :Int32; - level @4 :Int32; - timestamp @5 :Int64; - - centerFreq0 @6 :Int32; - centerFreq1 @7 :Int32; - channelWidth @8 :ChannelWidth; - operatorFriendlyName @9 :Text; - venueName @10 :Text; - is80211mcResponder @11 :Bool; - passpoint @12 :Bool; - - distanceCm @13 :Int32; - distanceSdCm @14 :Int32; - - enum ChannelWidth @0xcb6a279f015f6b51 { - w20Mhz @0; - w40Mhz @1; - w80Mhz @2; - w160Mhz @3; - w80Plus80Mhz @4; - } -} - -struct LiveEventData @0x94b7baa90c5c321e { - name @0 :Text; - value @1 :Int32; -} - -struct ModelData @0xb8aad62cffef28a9 { - frameId @0 :UInt32; - frameAge @12 :UInt32; - frameDropPerc @13 :Float32; - timestampEof @9 :UInt64; - modelExecutionTime @14 :Float32; - gpuExecutionTime @16 :Float32; - rawPred @15 :Data; - - path @1 :PathData; - leftLane @2 :PathData; - rightLane @3 :PathData; - lead @4 :LeadData; - freePath @6 :List(Float32); - - settings @5 :ModelSettings; - leadFuture @7 :LeadData; - speed @8 :List(Float32); - meta @10 :MetaData; - longitudinal @11 :LongitudinalData; - - struct PathData @0x8817eeea389e9f08 { - points @0 :List(Float32); - prob @1 :Float32; - std @2 :Float32; - stds @3 :List(Float32); - poly @4 :List(Float32); - validLen @5 :Float32; - } - - struct LeadData @0xd1c9bef96d26fa91 { - dist @0 :Float32; - prob @1 :Float32; - std @2 :Float32; - relVel @3 :Float32; - relVelStd @4 :Float32; - relY @5 :Float32; - relYStd @6 :Float32; - relA @7 :Float32; - relAStd @8 :Float32; - } - - struct ModelSettings @0xa26e3710efd3e914 { - bigBoxX @0 :UInt16; - bigBoxY @1 :UInt16; - bigBoxWidth @2 :UInt16; - bigBoxHeight @3 :UInt16; - boxProjection @4 :List(Float32); - yuvCorrection @5 :List(Float32); - inputTransform @6 :List(Float32); - } - - struct MetaData @0x9744f25fb60f2bf8 { - engagedProb @0 :Float32; - desirePrediction @1 :List(Float32); - brakeDisengageProb @2 :Float32; - gasDisengageProb @3 :Float32; - steerOverrideProb @4 :Float32; - desireState @5 :List(Float32); - } - - struct LongitudinalData @0xf98f999c6a071122 { - distances @2 :List(Float32); - speeds @0 :List(Float32); - accelerations @1 :List(Float32); - } -} - -struct ECEFPoint @0xc25bbbd524983447 { - x @0 :Float64; - y @1 :Float64; - z @2 :Float64; -} - -struct ECEFPointDEPRECATED @0xe10e21168db0c7f7 { - x @0 :Float32; - y @1 :Float32; - z @2 :Float32; -} - -struct GPSPlannerPoints @0xab54c59699f8f9f3 { - curPosDEPRECATED @0 :ECEFPointDEPRECATED; - pointsDEPRECATED @1 :List(ECEFPointDEPRECATED); - curPos @6 :ECEFPoint; - points @7 :List(ECEFPoint); - valid @2 :Bool; - trackName @3 :Text; - speedLimit @4 :Float32; - accelTarget @5 :Float32; -} - -struct GPSPlannerPlan @0xf5ad1d90cdc1dd6b { - valid @0 :Bool; - poly @1 :List(Float32); - trackName @2 :Text; - speed @3 :Float32; - acceleration @4 :Float32; - pointsDEPRECATED @5 :List(ECEFPointDEPRECATED); - points @6 :List(ECEFPoint); - xLookahead @7 :Float32; -} - -struct UiNavigationEvent @0x90c8426c3eaddd3b { - type @0: Type; - status @1: Status; - distanceTo @2: Float32; - endRoadPointDEPRECATED @3: ECEFPointDEPRECATED; - endRoadPoint @4: ECEFPoint; - - enum Type @0xe8db07dcf8fcea05 { - none @0; - laneChangeLeft @1; - laneChangeRight @2; - mergeLeft @3; - mergeRight @4; - turnLeft @5; - turnRight @6; - } - - enum Status @0xb9aa88c75ef99a1f { - none @0; - passive @1; - approaching @2; - active @3; - } -} - -struct LiveLocationData @0xb99b2bc7a57e8128 { - status @0 :UInt8; - - # 3D fix - lat @1 :Float64; - lon @2 :Float64; - alt @3 :Float32; # m - - # speed - speed @4 :Float32; # m/s - - # NED velocity components - vNED @5 :List(Float32); - - # roll, pitch, heading (x,y,z) - roll @6 :Float32; # WRT to center of earth? - pitch @7 :Float32; # WRT to center of earth? - heading @8 :Float32; # WRT to north? - - # what are these? - wanderAngle @9 :Float32; - trackAngle @10 :Float32; - - # car frame -- https://upload.wikimedia.org/wikipedia/commons/f/f5/RPY_angles_of_cars.png - - # gyro, in car frame, deg/s - gyro @11 :List(Float32); - - # accel, in car frame, m/s^2 - accel @12 :List(Float32); - - accuracy @13 :Accuracy; - - source @14 :SensorSource; - # if we are fixing a location in the past - fixMonoTime @15 :UInt64; - - gpsWeek @16 :Int32; - timeOfWeek @17 :Float64; - - positionECEF @18 :List(Float64); - poseQuatECEF @19 :List(Float32); - pitchCalibration @20 :Float32; - yawCalibration @21 :Float32; - imuFrame @22 :List(Float32); - - struct Accuracy @0x943dc4625473b03f { - pNEDError @0 :List(Float32); - vNEDError @1 :List(Float32); - rollError @2 :Float32; - pitchError @3 :Float32; - headingError @4 :Float32; - ellipsoidSemiMajorError @5 :Float32; - ellipsoidSemiMinorError @6 :Float32; - ellipsoidOrientationError @7 :Float32; - } - - enum SensorSource @0xc871d3cc252af657 { - applanix @0; - kalman @1; - orbslam @2; - timing @3; - dummy @4; - } -} - -struct OrbOdometry @0xd7700859ed1f5b76 { - # timing first - startMonoTime @0 :UInt64; - endMonoTime @1 :UInt64; - - # fundamental matrix and error - f @2: List(Float64); - err @3: Float64; - - # number of inlier points - inliers @4: Int32; - - # for debug only - # indexed by endMonoTime features - # value is startMonoTime feature match - # -1 if no match - matches @5: List(Int16); -} - -struct OrbFeatures @0xcd60164a8a0159ef { - timestampEof @0 :UInt64; - # transposed arrays of normalized image coordinates - # len(xs) == len(ys) == len(descriptors) * 32 - xs @1 :List(Float32); - ys @2 :List(Float32); - descriptors @3 :Data; - octaves @4 :List(Int8); - - # match index to last OrbFeatures - # -1 if no match - timestampLastEof @5 :UInt64; - matches @6: List(Int16); -} - -struct OrbFeaturesSummary @0xd500d30c5803fa4f { - timestampEof @0 :UInt64; - timestampLastEof @1 :UInt64; - - featureCount @2 :UInt16; - matchCount @3 :UInt16; - computeNs @4 :UInt64; -} - -struct OrbKeyFrame @0xc8233c0345e27e24 { - # this is a globally unique id for the KeyFrame - id @0: UInt64; - - # this is the location of the KeyFrame - pos @1: ECEFPoint; - - # these are the features in the world - # len(dpos) == len(descriptors) * 32 - dpos @2 :List(ECEFPoint); - descriptors @3 :Data; -} - -struct KalmanOdometry @0x92e21bb7ea38793a { - trans @0 :List(Float32); # m/s in device frame - rot @1 :List(Float32); # rad/s in device frame - transStd @2 :List(Float32); # std m/s in device frame - rotStd @3 :List(Float32); # std rad/s in device frame -} - -struct OrbObservation @0x9b326d4e436afec7 { - observationMonoTime @0 :UInt64; - normalizedCoordinates @1 :List(Float32); - locationECEF @2 :List(Float64); - matchDistance @3: UInt32; -} - -struct CalibrationFeatures @0x8fdfadb254ea867a { - frameId @0 :UInt32; - - p0 @1 :List(Float32); - p1 @2 :List(Float32); - status @3 :List(Int8); -} - -struct NavStatus @0xbd8822120928120c { - isNavigating @0 :Bool; - currentAddress @1 :Address; - - struct Address @0xce7cd672cacc7814 { - title @0 :Text; - lat @1 :Float64; - lng @2 :Float64; - house @3 :Text; - address @4 :Text; - street @5 :Text; - city @6 :Text; - state @7 :Text; - country @8 :Text; - } -} - -struct NavUpdate @0xdb98be6565516acb { - isNavigating @0 :Bool; - curSegment @1 :Int32; - segments @2 :List(Segment); - - struct LatLng @0x9eaef9187cadbb9b { - lat @0 :Float64; - lng @1 :Float64; - } - - struct Segment @0xa5b39b4fc4d7da3f { - from @0 :LatLng; - to @1 :LatLng; - updateTime @2 :Int32; - distance @3 :Int32; - crossTime @4 :Int32; - exitNo @5 :Int32; - instruction @6 :Instruction; - - parts @7 :List(LatLng); - - enum Instruction @0xc5417a637451246f { - turnLeft @0; - turnRight @1; - keepLeft @2; - keepRight @3; - straight @4; - roundaboutExitNumber @5; - roundaboutExit @6; - roundaboutTurnLeft @7; - unkn8 @8; - roundaboutStraight @9; - unkn10 @10; - roundaboutTurnRight @11; - unkn12 @12; - roundaboutUturn @13; - unkn14 @14; - arrive @15; - exitLeft @16; - exitRight @17; - unkn18 @18; - uturn @19; - # ... - } - } -} - -struct TrafficEvent @0xacfa74a094e62626 { - type @0 :Type; - distance @1 :Float32; - action @2 :Action; - resuming @3 :Bool; - - enum Type @0xd85d75253435bf4b { - stopSign @0; - lightRed @1; - lightYellow @2; - lightGreen @3; - stopLight @4; - } - - enum Action @0xa6f6ce72165ccb49 { - none @0; - yield @1; - stop @2; - resumeReady @3; - } - -} - - -struct AndroidGnss @0xdfdf30d03fc485bd { - union { - measurements @0 :Measurements; - navigationMessage @1 :NavigationMessage; - } - - struct Measurements @0xa20710d4f428d6cd { - clock @0 :Clock; - measurements @1 :List(Measurement); - - struct Clock @0xa0e27b453a38f450 { - timeNanos @0 :Int64; - hardwareClockDiscontinuityCount @1 :Int32; - - hasTimeUncertaintyNanos @2 :Bool; - timeUncertaintyNanos @3 :Float64; - - hasLeapSecond @4 :Bool; - leapSecond @5 :Int32; - - hasFullBiasNanos @6 :Bool; - fullBiasNanos @7 :Int64; - - hasBiasNanos @8 :Bool; - biasNanos @9 :Float64; - - hasBiasUncertaintyNanos @10 :Bool; - biasUncertaintyNanos @11 :Float64; - - hasDriftNanosPerSecond @12 :Bool; - driftNanosPerSecond @13 :Float64; - - hasDriftUncertaintyNanosPerSecond @14 :Bool; - driftUncertaintyNanosPerSecond @15 :Float64; - } - - struct Measurement @0xd949bf717d77614d { - svId @0 :Int32; - constellation @1 :Constellation; - - timeOffsetNanos @2 :Float64; - state @3 :Int32; - receivedSvTimeNanos @4 :Int64; - receivedSvTimeUncertaintyNanos @5 :Int64; - cn0DbHz @6 :Float64; - pseudorangeRateMetersPerSecond @7 :Float64; - pseudorangeRateUncertaintyMetersPerSecond @8 :Float64; - accumulatedDeltaRangeState @9 :Int32; - accumulatedDeltaRangeMeters @10 :Float64; - accumulatedDeltaRangeUncertaintyMeters @11 :Float64; - - hasCarrierFrequencyHz @12 :Bool; - carrierFrequencyHz @13 :Float32; - hasCarrierCycles @14 :Bool; - carrierCycles @15 :Int64; - hasCarrierPhase @16 :Bool; - carrierPhase @17 :Float64; - hasCarrierPhaseUncertainty @18 :Bool; - carrierPhaseUncertainty @19 :Float64; - hasSnrInDb @20 :Bool; - snrInDb @21 :Float64; - - multipathIndicator @22 :MultipathIndicator; - - enum Constellation @0x9ef1f3ff0deb5ffb { - unknown @0; - gps @1; - sbas @2; - glonass @3; - qzss @4; - beidou @5; - galileo @6; - } - - enum State @0xcbb9490adce12d72 { - unknown @0; - codeLock @1; - bitSync @2; - subframeSync @3; - towDecoded @4; - msecAmbiguous @5; - symbolSync @6; - gloStringSync @7; - gloTodDecoded @8; - bdsD2BitSync @9; - bdsD2SubframeSync @10; - galE1bcCodeLock @11; - galE1c2ndCodeLock @12; - galE1bPageSync @13; - sbasSync @14; - } - - enum MultipathIndicator @0xc04e7b6231d4caa8 { - unknown @0; - detected @1; - notDetected @2; - } - } - } - - struct NavigationMessage @0xe2517b083095fd4e { - type @0 :Int32; - svId @1 :Int32; - messageId @2 :Int32; - submessageId @3 :Int32; - data @4 :Data; - status @5 :Status; - - enum Status @0xec1ff7996b35366f { - unknown @0; - parityPassed @1; - parityRebuilt @2; - } - } -} - -struct LidarPts @0xe3d6685d4e9d8f7a { - r @0 :List(UInt16); # uint16 m*500.0 - theta @1 :List(UInt16); # uint16 deg*100.0 - reflect @2 :List(UInt8); # uint8 0-255 - - # For storing out of file. - idx @3 :UInt64; - - # For storing in file - pkt @4 :Data; -} - - diff --git a/log.capnp b/log.capnp deleted file mode 100644 index 36ca692dc..000000000 --- a/log.capnp +++ /dev/null @@ -1,2369 +0,0 @@ -using Cxx = import "./include/c++.capnp"; -$Cxx.namespace("cereal"); - -using Car = import "car.capnp"; -using Legacy = import "legacy.capnp"; -using Custom = import "custom.capnp"; - -@0xf3b1f17e25a4285b; - -const logVersion :Int32 = 1; - -struct Map(Key, Value) { - entries @0 :List(Entry); - struct Entry { - key @0 :Key; - value @1 :Value; - } -} - -enum LongitudinalPersonality { - aggressive @0; - standard @1; - relaxed @2; -} - -struct InitData { - kernelArgs @0 :List(Text); - kernelVersion @15 :Text; - osVersion @18 :Text; - - dongleId @2 :Text; - bootlogId @22 :Text; - - deviceType @3 :DeviceType; - version @4 :Text; - gitCommit @10 :Text; - gitCommitDate @21 :Text; - gitBranch @11 :Text; - gitRemote @13 :Text; - - androidProperties @16 :Map(Text, Text); - - pandaInfo @8 :PandaInfo; - - dirty @9 :Bool; - passive @12 :Bool; - params @17 :Map(Text, Data); - - commands @19 :Map(Text, Data); - - wallTimeNanos @20 :UInt64; - - enum DeviceType { - unknown @0; - neo @1; - chffrAndroid @2; - chffrIos @3; - tici @4; - pc @5; - tizi @6; - mici @7; - } - - struct PandaInfo { - hasPanda @0 :Bool; - dongleId @1 :Text; - stVersion @2 :Text; - espVersion @3 :Text; - } - - # ***** deprecated stuff ***** - gctxDEPRECATED @1 :Text; - androidBuildInfo @5 :AndroidBuildInfo; - androidSensorsDEPRECATED @6 :List(AndroidSensor); - chffrAndroidExtraDEPRECATED @7 :ChffrAndroidExtra; - iosBuildInfoDEPRECATED @14 :IosBuildInfo; - - struct AndroidBuildInfo { - board @0 :Text; - bootloader @1 :Text; - brand @2 :Text; - device @3 :Text; - display @4 :Text; - fingerprint @5 :Text; - hardware @6 :Text; - host @7 :Text; - id @8 :Text; - manufacturer @9 :Text; - model @10 :Text; - product @11 :Text; - radioVersion @12 :Text; - serial @13 :Text; - supportedAbis @14 :List(Text); - tags @15 :Text; - time @16 :Int64; - type @17 :Text; - user @18 :Text; - - versionCodename @19 :Text; - versionRelease @20 :Text; - versionSdk @21 :Int32; - versionSecurityPatch @22 :Text; - } - - struct AndroidSensor { - id @0 :Int32; - name @1 :Text; - vendor @2 :Text; - version @3 :Int32; - handle @4 :Int32; - type @5 :Int32; - maxRange @6 :Float32; - resolution @7 :Float32; - power @8 :Float32; - minDelay @9 :Int32; - fifoReservedEventCount @10 :UInt32; - fifoMaxEventCount @11 :UInt32; - stringType @12 :Text; - maxDelay @13 :Int32; - } - - struct ChffrAndroidExtra { - allCameraCharacteristics @0 :Map(Text, Text); - } - - struct IosBuildInfo { - appVersion @0 :Text; - appBuild @1 :UInt32; - osVersion @2 :Text; - deviceModel @3 :Text; - } -} - -struct FrameData { - frameId @0 :UInt32; - frameIdSensor @25 :UInt32; - requestId @28 :UInt32; - encodeId @1 :UInt32; - - frameType @7 :FrameType; - - # Timestamps - timestampEof @2 :UInt64; - timestampSof @8 :UInt64; - processingTime @23 :Float32; - - # Exposure - integLines @4 :Int32; - highConversionGain @20 :Bool; - gain @15 :Float32; # This includes highConversionGain if enabled - measuredGreyFraction @21 :Float32; - targetGreyFraction @22 :Float32; - exposureValPercent @27 :Float32; - - transform @10 :List(Float32); - - image @6 :Data; - - temperaturesC @24 :List(Float32); - - enum FrameType { - unknown @0; - neo @1; - chffrAndroid @2; - front @3; - } - - sensor @26 :ImageSensor; - enum ImageSensor { - unknown @0; - ar0231 @1; - ox03c10 @2; - os04c10 @3; - } - - frameLengthDEPRECATED @3 :Int32; - globalGainDEPRECATED @5 :Int32; - androidCaptureResultDEPRECATED @9 :AndroidCaptureResult; - lensPosDEPRECATED @11 :Int32; - lensSagDEPRECATED @12 :Float32; - lensErrDEPRECATED @13 :Float32; - lensTruePosDEPRECATED @14 :Float32; - focusValDEPRECATED @16 :List(Int16); - focusConfDEPRECATED @17 :List(UInt8); - sharpnessScoreDEPRECATED @18 :List(UInt16); - recoverStateDEPRECATED @19 :Int32; - struct AndroidCaptureResult { - sensitivity @0 :Int32; - frameDuration @1 :Int64; - exposureTime @2 :Int64; - rollingShutterSkew @3 :UInt64; - colorCorrectionTransform @4 :List(Int32); - colorCorrectionGains @5 :List(Float32); - displayRotation @6 :Int8; - } -} - -struct Thumbnail { - frameId @0 :UInt32; - timestampEof @1 :UInt64; - thumbnail @2 :Data; - encoding @3 :Encoding; - - enum Encoding { - unknown @0; - jpeg @1; - keyframe @2; - } -} - -struct GPSNMEAData { - timestamp @0 :Int64; - localWallTime @1 :UInt64; - nmea @2 :Text; -} - -# android sensor_event_t -struct SensorEventData { - version @0 :Int32; - sensor @1 :Int32; - type @2 :Int32; - timestamp @3 :Int64; - uncalibratedDEPRECATED @10 :Bool; - - union { - acceleration @4 :SensorVec; - magnetic @5 :SensorVec; - orientation @6 :SensorVec; - gyro @7 :SensorVec; - pressure @9 :SensorVec; - magneticUncalibrated @11 :SensorVec; - gyroUncalibrated @12 :SensorVec; - proximity @13: Float32; - light @14: Float32; - temperature @15: Float32; - } - source @8 :SensorSource; - - struct SensorVec { - v @0 :List(Float32); - status @1 :Int8; - } - - enum SensorSource { - android @0; - iOS @1; - fiber @2; - velodyne @3; # Velodyne IMU - bno055 @4; # Bosch accelerometer - lsm6ds3 @5; # includes LSM6DS3 and LSM6DS3TR, TR = tape reel - bmp280 @6; # barometer - mmc3416x @7; # magnetometer - bmx055 @8; - rpr0521 @9; - lsm6ds3trc @10; - mmc5603nj @11; - } -} - -# android struct GpsLocation -struct GpsLocationData { - # Contains module-specific flags. - flags @0 :UInt16; - - # Represents latitude in degrees. - latitude @1 :Float64; - - # Represents longitude in degrees. - longitude @2 :Float64; - - # Represents altitude in meters above the WGS 84 reference ellipsoid. - altitude @3 :Float64; - - # Represents speed in meters per second. - speed @4 :Float32; - - # Represents heading in degrees. - bearingDeg @5 :Float32; - - # Represents expected horizontal accuracy in meters. - horizontalAccuracy @6 :Float32; - - unixTimestampMillis @7 :Int64; - - source @8 :SensorSource; - - # Represents NED velocity in m/s. - vNED @9 :List(Float32); - - # Represents expected vertical accuracy in meters. (presumably 1 sigma?) - verticalAccuracy @10 :Float32; - - # Represents bearing accuracy in degrees. (presumably 1 sigma?) - bearingAccuracyDeg @11 :Float32; - - # Represents velocity accuracy in m/s. (presumably 1 sigma?) - speedAccuracy @12 :Float32; - - hasFix @13 :Bool; - - enum SensorSource { - android @0; - iOS @1; - car @2; - velodyne @3; # Velodyne IMU - fusion @4; - external @5; - ublox @6; - trimble @7; - qcomdiag @8; - unicore @9; - } -} - -enum Desire { - none @0; - turnLeft @1; - turnRight @2; - laneChangeLeft @3; - laneChangeRight @4; - keepLeft @5; - keepRight @6; -} - -enum LaneChangeState { - off @0; - preLaneChange @1; - laneChangeStarting @2; - laneChangeFinishing @3; -} - -enum LaneChangeDirection { - none @0; - left @1; - right @2; -} - -struct CanData { - address @0 :UInt32; - busTime @1 :UInt16; - dat @2 :Data; - src @3 :UInt8; -} - -struct DeviceState @0xa4d8b5af2aa492eb { - deviceType @45 :InitData.DeviceType; - - networkType @22 :NetworkType; - networkInfo @31 :NetworkInfo; - networkStrength @24 :NetworkStrength; - networkStats @43 :NetworkStats; - networkMetered @41 :Bool; - lastAthenaPingTime @32 :UInt64; - - started @11 :Bool; - startedMonoTime @13 :UInt64; - - # system utilization - freeSpacePercent @7 :Float32; - memoryUsagePercent @19 :Int8; - gpuUsagePercent @33 :Int8; - cpuUsagePercent @34 :List(Int8); # per-core cpu usage - - # power - offroadPowerUsageUwh @23 :UInt32; - carBatteryCapacityUwh @25 :UInt32; - powerDrawW @40 :Float32; - somPowerDrawW @42 :Float32; - - # device thermals - cpuTempC @26 :List(Float32); - gpuTempC @27 :List(Float32); - memoryTempC @28 :Float32; - nvmeTempC @35 :List(Float32); - modemTempC @36 :List(Float32); - pmicTempC @39 :List(Float32); - maxTempC @44 :Float32; # max of other temps, used to control fan - thermalZones @38 :List(ThermalZone); - thermalStatus @14 :ThermalStatus; - - fanSpeedPercentDesired @10 :UInt16; - screenBrightnessPercent @37 :Int8; - - struct ThermalZone { - name @0 :Text; - temp @1 :Float32; - } - - enum ThermalStatus { - green @0; - yellow @1; - red @2; - danger @3; - } - - enum NetworkType { - none @0; - wifi @1; - cell2G @2; - cell3G @3; - cell4G @4; - cell5G @5; - ethernet @6; - } - - enum NetworkStrength { - unknown @0; - poor @1; - moderate @2; - good @3; - great @4; - } - - struct NetworkInfo { - technology @0 :Text; - operator @1 :Text; - band @2 :Text; - channel @3 :UInt16; - extra @4 :Text; - state @5 :Text; - } - - struct NetworkStats { - wwanTx @0 :Int64; - wwanRx @1 :Int64; - } - - # deprecated - cpu0DEPRECATED @0 :UInt16; - cpu1DEPRECATED @1 :UInt16; - cpu2DEPRECATED @2 :UInt16; - cpu3DEPRECATED @3 :UInt16; - memDEPRECATED @4 :UInt16; - gpuDEPRECATED @5 :UInt16; - batDEPRECATED @6 :UInt32; - pa0DEPRECATED @21 :UInt16; - cpuUsagePercentDEPRECATED @20 :Int8; - batteryStatusDEPRECATED @9 :Text; - batteryVoltageDEPRECATED @16 :Int32; - batteryTempCDEPRECATED @29 :Float32; - batteryPercentDEPRECATED @8 :Int16; - batteryCurrentDEPRECATED @15 :Int32; - chargingErrorDEPRECATED @17 :Bool; - chargingDisabledDEPRECATED @18 :Bool; - usbOnlineDEPRECATED @12 :Bool; - ambientTempCDEPRECATED @30 :Float32; -} - -struct PandaState @0xa7649e2575e4591e { - ignitionLine @2 :Bool; - rxBufferOverflow @7 :UInt32; - txBufferOverflow @8 :UInt32; - pandaType @10 :PandaType; - ignitionCan @13 :Bool; - faultStatus @15 :FaultStatus; - powerSaveEnabled @16 :Bool; - uptime @17 :UInt32; - faults @18 :List(FaultType); - heartbeatLost @22 :Bool; - interruptLoad @25 :Float32; - fanPower @28 :UInt8; - fanStallCount @34 :UInt8; - - spiChecksumErrorCount @33 :UInt16; - - harnessStatus @21 :HarnessStatus; - sbu1Voltage @35 :Float32; - sbu2Voltage @36 :Float32; - - # can health - canState0 @29 :PandaCanState; - canState1 @30 :PandaCanState; - canState2 @31 :PandaCanState; - - # safety stuff - controlsAllowed @3 :Bool; - safetyRxInvalid @19 :UInt32; - safetyTxBlocked @24 :UInt32; - safetyModel @14 :Car.CarParams.SafetyModel; - safetyParam @27 :UInt16; - alternativeExperience @23 :Int16; - safetyRxChecksInvalid @32 :Bool; - - voltage @0 :UInt32; - current @1 :UInt32; - - enum FaultStatus { - none @0; - faultTemp @1; - faultPerm @2; - } - - enum FaultType { - relayMalfunction @0; - unusedInterruptHandled @1; - interruptRateCan1 @2; - interruptRateCan2 @3; - interruptRateCan3 @4; - interruptRateTach @5; - interruptRateGmlanDEPRECATED @6; - interruptRateInterrupts @7; - interruptRateSpiDma @8; - interruptRateSpiCs @9; - interruptRateUart1 @10; - interruptRateUart2 @11; - interruptRateUart3 @12; - interruptRateUart5 @13; - interruptRateUartDma @14; - interruptRateUsb @15; - interruptRateTim1 @16; - interruptRateTim3 @17; - registerDivergent @18; - interruptRateKlineInit @19; - interruptRateClockSource @20; - interruptRateTick @21; - interruptRateExti @22; - interruptRateSpi @23; - interruptRateUart7 @24; - sirenMalfunction @25; - heartbeatLoopWatchdog @26; - # Update max fault type in boardd when adding faults - } - - enum PandaType @0x8a58adf93e5b3751 { - unknown @0; - whitePanda @1; - greyPanda @2; - blackPanda @3; - pedal @4; - uno @5; - dos @6; - redPanda @7; - redPandaV2 @8; - tres @9; - cuatro @10; - } - - enum HarnessStatus { - notConnected @0; - normal @1; - flipped @2; - } - - struct PandaCanState { - busOff @0 :Bool; - busOffCnt @1 :UInt32; - errorWarning @2 :Bool; - errorPassive @3 :Bool; - lastError @4 :LecErrorCode; - lastStoredError @5 :LecErrorCode; - lastDataError @6 :LecErrorCode; - lastDataStoredError @7 :LecErrorCode; - receiveErrorCnt @8 :UInt8; - transmitErrorCnt @9 :UInt8; - totalErrorCnt @10 :UInt32; - totalTxLostCnt @11 :UInt32; - totalRxLostCnt @12 :UInt32; - totalTxCnt @13 :UInt32; - totalRxCnt @14 :UInt32; - totalFwdCnt @15 :UInt32; - canSpeed @16 :UInt16; - canDataSpeed @17 :UInt16; - canfdEnabled @18 :Bool; - brsEnabled @19 :Bool; - canfdNonIso @20 :Bool; - irq0CallRate @21 :UInt32; - irq1CallRate @22 :UInt32; - irq2CallRate @23 :UInt32; - canCoreResetCnt @24 :UInt32; - - enum LecErrorCode { - noError @0; - stuffError @1; - formError @2; - ackError @3; - bit1Error @4; - bit0Error @5; - crcError @6; - noChange @7; - } - } - - gasInterceptorDetectedDEPRECATED @4 :Bool; - startedSignalDetectedDEPRECATED @5 :Bool; - hasGpsDEPRECATED @6 :Bool; - gmlanSendErrsDEPRECATED @9 :UInt32; - fanSpeedRpmDEPRECATED @11 :UInt16; - usbPowerModeDEPRECATED @12 :PeripheralState.UsbPowerModeDEPRECATED; - safetyParamDEPRECATED @20 :Int16; - safetyParam2DEPRECATED @26 :UInt32; -} - -struct PeripheralState { - pandaType @0 :PandaState.PandaType; - voltage @1 :UInt32; - current @2 :UInt32; - fanSpeedRpm @3 :UInt16; - - usbPowerModeDEPRECATED @4 :UsbPowerModeDEPRECATED; - enum UsbPowerModeDEPRECATED @0xa8883583b32c9877 { - none @0; - client @1; - cdp @2; - dcp @3; - } -} - -struct RadarState @0x9a185389d6fdd05f { - mdMonoTime @6 :UInt64; - carStateMonoTime @11 :UInt64; - radarErrors @12 :List(Car.RadarData.Error); - - leadOne @3 :LeadData; - leadTwo @4 :LeadData; - cumLagMs @5 :Float32; - - struct LeadData { - dRel @0 :Float32; - yRel @1 :Float32; - vRel @2 :Float32; - aRel @3 :Float32; - vLead @4 :Float32; - dPath @6 :Float32; - vLat @7 :Float32; - vLeadK @8 :Float32; - aLeadK @9 :Float32; - fcw @10 :Bool; - status @11 :Bool; - aLeadTau @12 :Float32; - modelProb @13 :Float32; - radar @14 :Bool; - radarTrackId @15 :Int32 = -1; - - aLeadDEPRECATED @5 :Float32; - } - - # deprecated - ftMonoTimeDEPRECATED @7 :UInt64; - warpMatrixDEPRECATED @0 :List(Float32); - angleOffsetDEPRECATED @1 :Float32; - calStatusDEPRECATED @2 :Int8; - calCycleDEPRECATED @8 :Int32; - calPercDEPRECATED @9 :Int8; - canMonoTimesDEPRECATED @10 :List(UInt64); -} - -struct LiveCalibrationData { - calStatus @11 :Status; - calCycle @2 :Int32; - calPerc @3 :Int8; - validBlocks @9 :Int32; - - # view_frame_from_road_frame - # ui's is inversed needs new - extrinsicMatrix @4 :List(Float32); - # the direction of travel vector in device frame - rpyCalib @7 :List(Float32); - rpyCalibSpread @8 :List(Float32); - wideFromDeviceEuler @10 :List(Float32); - height @12 :List(Float32); - - warpMatrixDEPRECATED @0 :List(Float32); - calStatusDEPRECATED @1 :Int8; - warpMatrix2DEPRECATED @5 :List(Float32); - warpMatrixBigDEPRECATED @6 :List(Float32); - - enum Status { - uncalibrated @0; - calibrated @1; - invalid @2; - recalibrating @3; - } -} - -struct LiveTracks { - trackId @0 :Int32; - dRel @1 :Float32; - yRel @2 :Float32; - vRel @3 :Float32; - aRel @4 :Float32; - timeStamp @5 :Float32; - status @6 :Float32; - currentTime @7 :Float32; - stationary @8 :Bool; - oncoming @9 :Bool; -} - -struct ControlsState @0x97ff69c53601abf1 { - startMonoTime @48 :UInt64; - longitudinalPlanMonoTime @28 :UInt64; - lateralPlanMonoTime @50 :UInt64; - - state @31 :OpenpilotState; - enabled @19 :Bool; - active @36 :Bool; - - experimentalMode @64 :Bool; - personality @66 :LongitudinalPersonality; - - longControlState @30 :Car.CarControl.Actuators.LongControlState; - vPid @2 :Float32; - vTargetLead @3 :Float32; - vCruise @22 :Float32; # actual set speed - vCruiseCluster @63 :Float32; # set speed to display in the UI - upAccelCmd @4 :Float32; - uiAccelCmd @5 :Float32; - ufAccelCmd @33 :Float32; - aTarget @35 :Float32; - curvature @37 :Float32; # path curvature from vehicle model - desiredCurvature @61 :Float32; # lag adjusted curvatures used by lateral controllers - forceDecel @51 :Bool; - - # UI alerts - alertText1 @24 :Text; - alertText2 @25 :Text; - alertStatus @38 :AlertStatus; - alertSize @39 :AlertSize; - alertBlinkingRate @42 :Float32; - alertType @44 :Text; - alertSound @56 :Car.CarControl.HUDControl.AudibleAlert; - engageable @41 :Bool; # can OP be engaged? - - cumLagMs @15 :Float32; - - lateralControlState :union { - indiState @52 :LateralINDIState; - pidState @53 :LateralPIDState; - angleState @58 :LateralAngleState; - debugState @59 :LateralDebugState; - torqueState @60 :LateralTorqueState; - - curvatureStateDEPRECATED @65 :LateralCurvatureState; - lqrStateDEPRECATED @55 :LateralLQRState; - } - - enum OpenpilotState @0xdbe58b96d2d1ac61 { - disabled @0; - preEnabled @1; - enabled @2; - softDisabling @3; - overriding @4; # superset of overriding with steering or accelerator - } - - enum AlertStatus { - normal @0; # low priority alert for user's convenience - userPrompt @1; # mid priority alert that might require user intervention - critical @2; # high priority alert that needs immediate user intervention - } - - enum AlertSize { - none @0; # don't display the alert - small @1; # small box - mid @2; # mid screen - full @3; # full screen - } - - struct LateralINDIState { - active @0 :Bool; - steeringAngleDeg @1 :Float32; - steeringRateDeg @2 :Float32; - steeringAccelDeg @3 :Float32; - rateSetPoint @4 :Float32; - accelSetPoint @5 :Float32; - accelError @6 :Float32; - delayedOutput @7 :Float32; - delta @8 :Float32; - output @9 :Float32; - saturated @10 :Bool; - steeringAngleDesiredDeg @11 :Float32; - steeringRateDesiredDeg @12 :Float32; - } - - struct LateralPIDState { - active @0 :Bool; - steeringAngleDeg @1 :Float32; - steeringRateDeg @2 :Float32; - angleError @3 :Float32; - p @4 :Float32; - i @5 :Float32; - f @6 :Float32; - output @7 :Float32; - saturated @8 :Bool; - steeringAngleDesiredDeg @9 :Float32; - } - - struct LateralTorqueState { - active @0 :Bool; - error @1 :Float32; - errorRate @8 :Float32; - p @2 :Float32; - i @3 :Float32; - d @4 :Float32; - f @5 :Float32; - output @6 :Float32; - saturated @7 :Bool; - actualLateralAccel @9 :Float32; - desiredLateralAccel @10 :Float32; - } - - struct LateralLQRState { - active @0 :Bool; - steeringAngleDeg @1 :Float32; - i @2 :Float32; - output @3 :Float32; - lqrOutput @4 :Float32; - saturated @5 :Bool; - steeringAngleDesiredDeg @6 :Float32; - } - - struct LateralAngleState { - active @0 :Bool; - steeringAngleDeg @1 :Float32; - output @2 :Float32; - saturated @3 :Bool; - steeringAngleDesiredDeg @4 :Float32; - } - - struct LateralCurvatureState { - active @0 :Bool; - actualCurvature @1 :Float32; - desiredCurvature @2 :Float32; - error @3 :Float32; - p @4 :Float32; - i @5 :Float32; - f @6 :Float32; - output @7 :Float32; - saturated @8 :Bool; - } - - struct LateralDebugState { - active @0 :Bool; - steeringAngleDeg @1 :Float32; - output @2 :Float32; - saturated @3 :Bool; - } - - # deprecated - vEgoDEPRECATED @0 :Float32; - vEgoRawDEPRECATED @32 :Float32; - aEgoDEPRECATED @1 :Float32; - canMonoTimeDEPRECATED @16 :UInt64; - radarStateMonoTimeDEPRECATED @17 :UInt64; - mdMonoTimeDEPRECATED @18 :UInt64; - yActualDEPRECATED @6 :Float32; - yDesDEPRECATED @7 :Float32; - upSteerDEPRECATED @8 :Float32; - uiSteerDEPRECATED @9 :Float32; - ufSteerDEPRECATED @34 :Float32; - aTargetMinDEPRECATED @10 :Float32; - aTargetMaxDEPRECATED @11 :Float32; - rearViewCamDEPRECATED @23 :Bool; - driverMonitoringOnDEPRECATED @43 :Bool; - hudLeadDEPRECATED @14 :Int32; - alertSoundDEPRECATED @45 :Text; - angleModelBiasDEPRECATED @27 :Float32; - gpsPlannerActiveDEPRECATED @40 :Bool; - decelForTurnDEPRECATED @47 :Bool; - decelForModelDEPRECATED @54 :Bool; - awarenessStatusDEPRECATED @26 :Float32; - angleSteersDEPRECATED @13 :Float32; - vCurvatureDEPRECATED @46 :Float32; - mapValidDEPRECATED @49 :Bool; - jerkFactorDEPRECATED @12 :Float32; - steerOverrideDEPRECATED @20 :Bool; - steeringAngleDesiredDegDEPRECATED @29 :Float32; - canMonoTimesDEPRECATED @21 :List(UInt64); - desiredCurvatureRateDEPRECATED @62 :Float32; - canErrorCounterDEPRECATED @57 :UInt32; -} - -# All SI units and in device frame -struct XYZTData @0xc3cbae1fd505ae80 { - x @0 :List(Float32); - y @1 :List(Float32); - z @2 :List(Float32); - t @3 :List(Float32); - xStd @4 :List(Float32); - yStd @5 :List(Float32); - zStd @6 :List(Float32); -} - -struct ModelDataV2 { - frameId @0 :UInt32; - frameIdExtra @20 :UInt32; - frameAge @1 :UInt32; - frameDropPerc @2 :Float32; - timestampEof @3 :UInt64; - modelExecutionTime @15 :Float32; - gpuExecutionTime @17 :Float32; - rawPredictions @16 :Data; - - # predicted future position, orientation, etc.. - position @4 :XYZTData; - orientation @5 :XYZTData; - velocity @6 :XYZTData; - orientationRate @7 :XYZTData; - acceleration @19 :XYZTData; - - # prediction lanelines and road edges - laneLines @8 :List(XYZTData); - laneLineProbs @9 :List(Float32); - laneLineStds @13 :List(Float32); - roadEdges @10 :List(XYZTData); - roadEdgeStds @14 :List(Float32); - - # predicted lead cars - leads @11 :List(LeadDataV2); - leadsV3 @18 :List(LeadDataV3); - - meta @12 :MetaData; - confidence @23: ConfidenceClass; - - # Model perceived motion - temporalPose @21 :Pose; - - navEnabledDEPRECATED @22 :Bool; - locationMonoTimeDEPRECATED @24 :UInt64; - - # e2e lateral planner - lateralPlannerSolutionDEPRECATED @25: LateralPlannerSolution; - action @26: Action; - - struct LeadDataV2 { - prob @0 :Float32; # probability that car is your lead at time t - t @1 :Float32; - - # x and y are relative position in device frame - # v is norm relative speed - # a is norm relative acceleration - xyva @2 :List(Float32); - xyvaStd @3 :List(Float32); - } - - struct LeadDataV3 { - prob @0 :Float32; # probability that car is your lead at time t - probTime @1 :Float32; - t @2 :List(Float32); - - # x and y are relative position in device frame - # v absolute norm speed - # a is derivative of v - x @3 :List(Float32); - xStd @4 :List(Float32); - y @5 :List(Float32); - yStd @6 :List(Float32); - v @7 :List(Float32); - vStd @8 :List(Float32); - a @9 :List(Float32); - aStd @10 :List(Float32); - } - - - struct MetaData { - engagedProb @0 :Float32; - desirePrediction @1 :List(Float32); - desireState @5 :List(Float32); - disengagePredictions @6 :DisengagePredictions; - hardBrakePredicted @7 :Bool; - laneChangeState @8 :LaneChangeState; - laneChangeDirection @9 :LaneChangeDirection; - - - # deprecated - brakeDisengageProbDEPRECATED @2 :Float32; - gasDisengageProbDEPRECATED @3 :Float32; - steerOverrideProbDEPRECATED @4 :Float32; - } - - enum ConfidenceClass { - red @0; - yellow @1; - green @2; - } - - struct DisengagePredictions { - t @0 :List(Float32); - brakeDisengageProbs @1 :List(Float32); - gasDisengageProbs @2 :List(Float32); - steerOverrideProbs @3 :List(Float32); - brake3MetersPerSecondSquaredProbs @4 :List(Float32); - brake4MetersPerSecondSquaredProbs @5 :List(Float32); - brake5MetersPerSecondSquaredProbs @6 :List(Float32); - } - - struct Pose { - trans @0 :List(Float32); # m/s in device frame - rot @1 :List(Float32); # rad/s in device frame - transStd @2 :List(Float32); # std m/s in device frame - rotStd @3 :List(Float32); # std rad/s in device frame - } - - struct LateralPlannerSolution { - x @0 :List(Float32); - y @1 :List(Float32); - yaw @2 :List(Float32); - yawRate @3 :List(Float32); - xStd @4 :List(Float32); - yStd @5 :List(Float32); - yawStd @6 :List(Float32); - yawRateStd @7 :List(Float32); - } - - struct Action { - desiredCurvature @0 :Float32; - } -} - -struct EncodeIndex { - # picture from camera - frameId @0 :UInt32; - type @1 :Type; - # index of encoder from start of route - encodeId @2 :UInt32; - # minute long segment this frame is in - segmentNum @3 :Int32; - # index into camera file in segment in presentation order - segmentId @4 :UInt32; - # index into camera file in segment in encode order - segmentIdEncode @5 :UInt32; - timestampSof @6 :UInt64; - timestampEof @7 :UInt64; - - # encoder metadata - flags @8 :UInt32; - len @9 :UInt32; - - enum Type { - bigBoxLossless @0; - fullHEVC @1; - qcameraH264 @6; - livestreamH264 @7; - - # deprecated - bigBoxHEVCDEPRECATED @2; - chffrAndroidH264DEPRECATED @3; - fullLosslessClipDEPRECATED @4; - frontDEPRECATED @5; - - } -} - -struct AndroidLogEntry { - id @0 :UInt8; - ts @1 :UInt64; - priority @2 :UInt8; - pid @3 :Int32; - tid @4 :Int32; - tag @5 :Text; - message @6 :Text; -} - -struct LongitudinalPlan @0xe00b5b3eba12876c { - modelMonoTime @9 :UInt64; - hasLead @7 :Bool; - fcw @8 :Bool; - longitudinalPlanSource @15 :LongitudinalPlanSource; - processingDelay @29 :Float32; - - # desired speed/accel/jerk over next 2.5s - accels @32 :List(Float32); - speeds @33 :List(Float32); - jerks @34 :List(Float32); - - solverExecutionTime @35 :Float32; - - enum LongitudinalPlanSource { - cruise @0; - lead0 @1; - lead1 @2; - lead2 @3; - e2e @4; - } - - # deprecated - vCruiseDEPRECATED @16 :Float32; - aCruiseDEPRECATED @17 :Float32; - vTargetDEPRECATED @3 :Float32; - vTargetFutureDEPRECATED @14 :Float32; - aTargetDEPRECATED @18 :Float32; - vStartDEPRECATED @26 :Float32; - aStartDEPRECATED @27 :Float32; - vMaxDEPRECATED @20 :Float32; - radarStateMonoTimeDEPRECATED @10 :UInt64; - jerkFactorDEPRECATED @6 :Float32; - hasLeftLaneDEPRECATED @23 :Bool; - hasRightLaneDEPRECATED @24 :Bool; - aTargetMinDEPRECATED @4 :Float32; - aTargetMaxDEPRECATED @5 :Float32; - lateralValidDEPRECATED @0 :Bool; - longitudinalValidDEPRECATED @2 :Bool; - dPolyDEPRECATED @1 :List(Float32); - laneWidthDEPRECATED @11 :Float32; - vCurvatureDEPRECATED @21 :Float32; - decelForTurnDEPRECATED @22 :Bool; - mapValidDEPRECATED @25 :Bool; - radarValidDEPRECATED @28 :Bool; - radarCanErrorDEPRECATED @30 :Bool; - commIssueDEPRECATED @31 :Bool; - eventsDEPRECATED @13 :List(Car.CarEvent); - gpsTrajectoryDEPRECATED @12 :GpsTrajectory; - gpsPlannerActiveDEPRECATED @19 :Bool; - personalityDEPRECATED @36 :LongitudinalPersonality; - - struct GpsTrajectory { - x @0 :List(Float32); - y @1 :List(Float32); - } -} -struct UiPlan { - frameId @2 :UInt32; - position @0 :XYZTData; - accel @1 :List(Float32); -} - -struct LateralPlan @0xe1e9318e2ae8b51e { - modelMonoTime @31 :UInt64; - laneWidthDEPRECATED @0 :Float32; - lProbDEPRECATED @5 :Float32; - rProbDEPRECATED @7 :Float32; - dPathPoints @20 :List(Float32); - dProbDEPRECATED @21 :Float32; - - mpcSolutionValid @9 :Bool; - desire @17 :Desire; - laneChangeState @18 :LaneChangeState; - laneChangeDirection @19 :LaneChangeDirection; - useLaneLines @29 :Bool; - - # desired curvatures over next 2.5s in rad/m - psis @26 :List(Float32); - curvatures @27 :List(Float32); - curvatureRates @28 :List(Float32); - - solverExecutionTime @30 :Float32; - solverCost @32 :Float32; - solverState @33 :SolverState; - - struct SolverState { - x @0 :List(List(Float32)); - u @1 :List(Float32); - } - - # deprecated - curvatureDEPRECATED @22 :Float32; - curvatureRateDEPRECATED @23 :Float32; - rawCurvatureDEPRECATED @24 :Float32; - rawCurvatureRateDEPRECATED @25 :Float32; - cProbDEPRECATED @3 :Float32; - dPolyDEPRECATED @1 :List(Float32); - cPolyDEPRECATED @2 :List(Float32); - lPolyDEPRECATED @4 :List(Float32); - rPolyDEPRECATED @6 :List(Float32); - modelValidDEPRECATED @12 :Bool; - commIssueDEPRECATED @15 :Bool; - posenetValidDEPRECATED @16 :Bool; - sensorValidDEPRECATED @14 :Bool; - paramsValidDEPRECATED @10 :Bool; - steeringAngleDegDEPRECATED @8 :Float32; # deg - steeringRateDegDEPRECATED @13 :Float32; # deg/s - angleOffsetDegDEPRECATED @11 :Float32; -} - -struct LiveLocationKalman { - - # More info on reference frames: - # https://github.com/commaai/openpilot/tree/master/common/transformations - - positionECEF @0 : Measurement; - positionGeodetic @1 : Measurement; - velocityECEF @2 : Measurement; - velocityNED @3 : Measurement; - velocityDevice @4 : Measurement; - accelerationDevice @5: Measurement; - - - # These angles are all eulers and roll, pitch, yaw - # orientationECEF transforms to rot matrix: ecef_from_device - orientationECEF @6 : Measurement; - calibratedOrientationECEF @20 : Measurement; - orientationNED @7 : Measurement; - angularVelocityDevice @8 : Measurement; - - # orientationNEDCalibrated transforms to rot matrix: NED_from_calibrated - calibratedOrientationNED @9 : Measurement; - - # Calibrated frame is simply device frame - # aligned with the vehicle - velocityCalibrated @10 : Measurement; - accelerationCalibrated @11 : Measurement; - angularVelocityCalibrated @12 : Measurement; - - gpsWeek @13 :Int32; - gpsTimeOfWeek @14 :Float64; - status @15 :Status; - unixTimestampMillis @16 :Int64; - inputsOK @17 :Bool = true; - posenetOK @18 :Bool = true; - gpsOK @19 :Bool = true; - sensorsOK @21 :Bool = true; - deviceStable @22 :Bool = true; - timeSinceReset @23 :Float64; - excessiveResets @24 :Bool; - timeToFirstFix @25 :Float32; - - filterState @26 : Measurement; - - enum Status { - uninitialized @0; - uncalibrated @1; - valid @2; - } - - struct Measurement { - value @0 : List(Float64); - std @1 : List(Float64); - valid @2 : Bool; - } -} - -struct ProcLog { - cpuTimes @0 :List(CPUTimes); - mem @1 :Mem; - procs @2 :List(Process); - - struct Process { - pid @0 :Int32; - name @1 :Text; - state @2 :UInt8; - ppid @3 :Int32; - - cpuUser @4 :Float32; - cpuSystem @5 :Float32; - cpuChildrenUser @6 :Float32; - cpuChildrenSystem @7 :Float32; - priority @8 :Int64; - nice @9 :Int32; - numThreads @10 :Int32; - startTime @11 :Float64; - - memVms @12 :UInt64; - memRss @13 :UInt64; - - processor @14 :Int32; - - cmdline @15 :List(Text); - exe @16 :Text; - } - - struct CPUTimes { - cpuNum @0 :Int64; - user @1 :Float32; - nice @2 :Float32; - system @3 :Float32; - idle @4 :Float32; - iowait @5 :Float32; - irq @6 :Float32; - softirq @7 :Float32; - } - - struct Mem { - total @0 :UInt64; - free @1 :UInt64; - available @2 :UInt64; - buffers @3 :UInt64; - cached @4 :UInt64; - active @5 :UInt64; - inactive @6 :UInt64; - shared @7 :UInt64; - } -} - -struct GnssMeasurements { - measTime @0 :UInt64; - gpsWeek @1 :Int16; - gpsTimeOfWeek @2 :Float64; - - correctedMeasurements @3 :List(CorrectedMeasurement); - ephemerisStatuses @9 :List(EphemerisStatus); - - kalmanPositionECEF @4 :LiveLocationKalman.Measurement; - kalmanVelocityECEF @5 :LiveLocationKalman.Measurement; - positionECEF @6 :LiveLocationKalman.Measurement; - velocityECEF @7 :LiveLocationKalman.Measurement; - timeToFirstFix @8 :Float32; - # Todo sync this with timing pulse of ublox - - struct EphemerisStatus { - constellationId @0 :ConstellationId; - svId @1 :UInt8; - type @2 :EphemerisType; - source @3 :EphemerisSource; - gpsWeek @4 : UInt16; - tow @5 :Float64; - } - - struct CorrectedMeasurement { - constellationId @0 :ConstellationId; - svId @1 :UInt8; - # Is 0 when not Glonass constellation. - glonassFrequency @2 :Int8; - pseudorange @3 :Float64; - pseudorangeStd @4 :Float64; - pseudorangeRate @5 :Float64; - pseudorangeRateStd @6 :Float64; - # Satellite position and velocity [x,y,z] - satPos @7 :List(Float64); - satVel @8 :List(Float64); - ephemerisSourceDEPRECATED @9 :EphemerisSourceDEPRECATED; - } - - struct EphemerisSourceDEPRECATED { - type @0 :EphemerisType; - # first epoch in file: - gpsWeek @1 :Int16; # -1 if Nav - gpsTimeOfWeek @2 :Int32; # -1 if Nav. Integer for seconds is good enough for logs. - } - - enum ConstellationId { - # Satellite Constellation using the Ublox gnssid as index - gps @0; - sbas @1; - galileo @2; - beidou @3; - imes @4; - qznss @5; - glonass @6; - } - - enum EphemerisType { - nav @0; - # Different ultra-rapid files: - nasaUltraRapid @1; - glonassIacUltraRapid @2; - qcom @3; - } - - enum EphemerisSource { - gnssChip @0; - internet @1; - cache @2; - unknown @3; - } -} - -struct UbloxGnss { - union { - measurementReport @0 :MeasurementReport; - ephemeris @1 :Ephemeris; - ionoData @2 :IonoData; - hwStatus @3 :HwStatus; - hwStatus2 @4 :HwStatus2; - glonassEphemeris @5 :GlonassEphemeris; - satReport @6 :SatReport; - } - - struct SatReport { - #received time of week in gps time in seconds and gps week - iTow @0 :UInt32; - svs @1 :List(SatInfo); - - struct SatInfo { - svId @0 :UInt8; - gnssId @1 :UInt8; - flagsBitfield @2 :UInt32; - } - } - - struct MeasurementReport { - #received time of week in gps time in seconds and gps week - rcvTow @0 :Float64; - gpsWeek @1 :UInt16; - # leap seconds in seconds - leapSeconds @2 :UInt16; - # receiver status - receiverStatus @3 :ReceiverStatus; - # num of measurements to follow - numMeas @4 :UInt8; - measurements @5 :List(Measurement); - - struct ReceiverStatus { - # leap seconds have been determined - leapSecValid @0 :Bool; - # Clock reset applied - clkReset @1 :Bool; - } - - struct Measurement { - svId @0 :UInt8; - trackingStatus @1 :TrackingStatus; - # pseudorange in meters - pseudorange @2 :Float64; - # carrier phase measurement in cycles - carrierCycles @3 :Float64; - # doppler measurement in Hz - doppler @4 :Float32; - # GNSS id, 0 is gps - gnssId @5 :UInt8; - glonassFrequencyIndex @6 :UInt8; - # carrier phase locktime counter in ms - locktime @7 :UInt16; - # Carrier-to-noise density ratio (signal strength) in dBHz - cno @8 :UInt8; - # pseudorange standard deviation in meters - pseudorangeStdev @9 :Float32; - # carrier phase standard deviation in cycles - carrierPhaseStdev @10 :Float32; - # doppler standard deviation in Hz - dopplerStdev @11 :Float32; - sigId @12 :UInt8; - - struct TrackingStatus { - # pseudorange valid - pseudorangeValid @0 :Bool; - # carrier phase valid - carrierPhaseValid @1 :Bool; - # half cycle valid - halfCycleValid @2 :Bool; - # half cycle subtracted from phase - halfCycleSubtracted @3 :Bool; - } - } - } - - struct Ephemeris { - # This is according to the rinex (2?) format - svId @0 :UInt16; - year @1 :UInt16; - month @2 :UInt16; - day @3 :UInt16; - hour @4 :UInt16; - minute @5 :UInt16; - second @6 :Float32; - af0 @7 :Float64; - af1 @8 :Float64; - af2 @9 :Float64; - - iode @10 :Float64; - crs @11 :Float64; - deltaN @12 :Float64; - m0 @13 :Float64; - - cuc @14 :Float64; - ecc @15 :Float64; - cus @16 :Float64; - a @17 :Float64; # note that this is not the root!! - - toe @18 :Float64; - cic @19 :Float64; - omega0 @20 :Float64; - cis @21 :Float64; - - i0 @22 :Float64; - crc @23 :Float64; - omega @24 :Float64; - omegaDot @25 :Float64; - - iDot @26 :Float64; - codesL2 @27 :Float64; - gpsWeekDEPRECATED @28 :Float64; - l2 @29 :Float64; - - svAcc @30 :Float64; - svHealth @31 :Float64; - tgd @32 :Float64; - iodc @33 :Float64; - - transmissionTime @34 :Float64; - fitInterval @35 :Float64; - - toc @36 :Float64; - - ionoCoeffsValid @37 :Bool; - ionoAlpha @38 :List(Float64); - ionoBeta @39 :List(Float64); - - towCount @40 :UInt32; - toeWeek @41 :UInt16; - tocWeek @42 :UInt16; - } - - struct IonoData { - svHealth @0 :UInt32; - tow @1 :Float64; - gpsWeek @2 :Float64; - - ionoAlpha @3 :List(Float64); - ionoBeta @4 :List(Float64); - - healthValid @5 :Bool; - ionoCoeffsValid @6 :Bool; - } - - struct HwStatus { - noisePerMS @0 :UInt16; - agcCnt @1 :UInt16; - aStatus @2 :AntennaSupervisorState; - aPower @3 :AntennaPowerStatus; - jamInd @4 :UInt8; - flags @5 :UInt8; - - enum AntennaSupervisorState { - init @0; - dontknow @1; - ok @2; - short @3; - open @4; - } - - enum AntennaPowerStatus { - off @0; - on @1; - dontknow @2; - } - } - - struct HwStatus2 { - ofsI @0 :Int8; - magI @1 :UInt8; - ofsQ @2 :Int8; - magQ @3 :UInt8; - cfgSource @4 :ConfigSource; - lowLevCfg @5 :UInt32; - postStatus @6 :UInt32; - - enum ConfigSource { - undefined @0; - rom @1; - otp @2; - configpins @3; - flash @4; - } - } - - struct GlonassEphemeris { - svId @0 :UInt16; - year @1 :UInt16; - dayInYear @2 :UInt16; - hour @3 :UInt16; - minute @4 :UInt16; - second @5 :Float32; - - x @6 :Float64; - xVel @7 :Float64; - xAccel @8 :Float64; - y @9 :Float64; - yVel @10 :Float64; - yAccel @11 :Float64; - z @12 :Float64; - zVel @13 :Float64; - zAccel @14 :Float64; - - svType @15 :UInt8; - svURA @16 :Float32; - age @17 :UInt8; - - svHealth @18 :UInt8; - tkDEPRECATED @19 :UInt16; - tb @20 :UInt16; - - tauN @21 :Float64; - deltaTauN @22 :Float64; - gammaN @23 :Float64; - - p1 @24 :UInt8; - p2 @25 :UInt8; - p3 @26 :UInt8; - p4 @27 :UInt8; - - freqNumDEPRECATED @28 :UInt32; - - n4 @29 :UInt8; - nt @30 :UInt16; - freqNum @31 :Int16; - tkSeconds @32 :UInt32; - } -} - -struct QcomGnss @0xde94674b07ae51c1 { - logTs @0 :UInt64; - union { - measurementReport @1 :MeasurementReport; - clockReport @2 :ClockReport; - drMeasurementReport @3 :DrMeasurementReport; - drSvPoly @4 :DrSvPolyReport; - rawLog @5 :Data; - } - - enum MeasurementSource @0xd71a12b6faada7ee { - gps @0; - glonass @1; - beidou @2; - unknown3 @3; - unknown4 @4; - unknown5 @5; - sbas @6; - } - - enum SVObservationState @0xe81e829a0d6c83e9 { - idle @0; - search @1; - searchVerify @2; - bitEdge @3; - trackVerify @4; - track @5; - restart @6; - dpo @7; - glo10msBe @8; - glo10msAt @9; - } - - struct MeasurementStatus @0xe501010e1bcae83b { - subMillisecondIsValid @0 :Bool; - subBitTimeIsKnown @1 :Bool; - satelliteTimeIsKnown @2 :Bool; - bitEdgeConfirmedFromSignal @3 :Bool; - measuredVelocity @4 :Bool; - fineOrCoarseVelocity @5 :Bool; - lockPointValid @6 :Bool; - lockPointPositive @7 :Bool; - lastUpdateFromDifference @8 :Bool; - lastUpdateFromVelocityDifference @9 :Bool; - strongIndicationOfCrossCorelation @10 :Bool; - tentativeMeasurement @11 :Bool; - measurementNotUsable @12 :Bool; - sirCheckIsNeeded @13 :Bool; - probationMode @14 :Bool; - - glonassMeanderBitEdgeValid @15 :Bool; - glonassTimeMarkValid @16 :Bool; - - gpsRoundRobinRxDiversity @17 :Bool; - gpsRxDiversity @18 :Bool; - gpsLowBandwidthRxDiversityCombined @19 :Bool; - gpsHighBandwidthNu4 @20 :Bool; - gpsHighBandwidthNu8 @21 :Bool; - gpsHighBandwidthUniform @22 :Bool; - multipathIndicator @23 :Bool; - - imdJammingIndicator @24 :Bool; - lteB13TxJammingIndicator @25 :Bool; - freshMeasurementIndicator @26 :Bool; - - multipathEstimateIsValid @27 :Bool; - directionIsValid @28 :Bool; - } - - struct MeasurementReport @0xf580d7d86b7b8692 { - source @0 :MeasurementSource; - - fCount @1 :UInt32; - - gpsWeek @2 :UInt16; - glonassCycleNumber @3 :UInt8; - glonassNumberOfDays @4 :UInt16; - - milliseconds @5 :UInt32; - timeBias @6 :Float32; - clockTimeUncertainty @7 :Float32; - clockFrequencyBias @8 :Float32; - clockFrequencyUncertainty @9 :Float32; - - sv @10 :List(SV); - - struct SV @0xf10c595ae7bb2c27 { - svId @0 :UInt8; - observationState @2 :SVObservationState; - observations @3 :UInt8; - goodObservations @4 :UInt8; - gpsParityErrorCount @5 :UInt16; - glonassFrequencyIndex @1 :Int8; - glonassHemmingErrorCount @6 :UInt8; - filterStages @7 :UInt8; - carrierNoise @8 :UInt16; - latency @9 :Int16; - predetectInterval @10 :UInt8; - postdetections @11 :UInt16; - - unfilteredMeasurementIntegral @12 :UInt32; - unfilteredMeasurementFraction @13 :Float32; - unfilteredTimeUncertainty @14 :Float32; - unfilteredSpeed @15 :Float32; - unfilteredSpeedUncertainty @16 :Float32; - measurementStatus @17 :MeasurementStatus; - multipathEstimate @18 :UInt32; - azimuth @19 :Float32; - elevation @20 :Float32; - carrierPhaseCyclesIntegral @21 :Int32; - carrierPhaseCyclesFraction @22 :UInt16; - fineSpeed @23 :Float32; - fineSpeedUncertainty @24 :Float32; - cycleSlipCount @25 :UInt8; - } - - } - - struct ClockReport @0xca965e4add8f4f0b { - hasFCount @0 :Bool; - fCount @1 :UInt32; - - hasGpsWeek @2 :Bool; - gpsWeek @3 :UInt16; - hasGpsMilliseconds @4 :Bool; - gpsMilliseconds @5 :UInt32; - gpsTimeBias @6 :Float32; - gpsClockTimeUncertainty @7 :Float32; - gpsClockSource @8 :UInt8; - - hasGlonassYear @9 :Bool; - glonassYear @10 :UInt8; - hasGlonassDay @11 :Bool; - glonassDay @12 :UInt16; - hasGlonassMilliseconds @13 :Bool; - glonassMilliseconds @14 :UInt32; - glonassTimeBias @15 :Float32; - glonassClockTimeUncertainty @16 :Float32; - glonassClockSource @17 :UInt8; - - bdsWeek @18 :UInt16; - bdsMilliseconds @19 :UInt32; - bdsTimeBias @20 :Float32; - bdsClockTimeUncertainty @21 :Float32; - bdsClockSource @22 :UInt8; - - galWeek @23 :UInt16; - galMilliseconds @24 :UInt32; - galTimeBias @25 :Float32; - galClockTimeUncertainty @26 :Float32; - galClockSource @27 :UInt8; - - clockFrequencyBias @28 :Float32; - clockFrequencyUncertainty @29 :Float32; - frequencySource @30 :UInt8; - gpsLeapSeconds @31 :UInt8; - gpsLeapSecondsUncertainty @32 :UInt8; - gpsLeapSecondsSource @33 :UInt8; - - gpsToGlonassTimeBiasMilliseconds @34 :Float32; - gpsToGlonassTimeBiasMillisecondsUncertainty @35 :Float32; - gpsToBdsTimeBiasMilliseconds @36 :Float32; - gpsToBdsTimeBiasMillisecondsUncertainty @37 :Float32; - bdsToGloTimeBiasMilliseconds @38 :Float32; - bdsToGloTimeBiasMillisecondsUncertainty @39 :Float32; - gpsToGalTimeBiasMilliseconds @40 :Float32; - gpsToGalTimeBiasMillisecondsUncertainty @41 :Float32; - galToGloTimeBiasMilliseconds @42 :Float32; - galToGloTimeBiasMillisecondsUncertainty @43 :Float32; - galToBdsTimeBiasMilliseconds @44 :Float32; - galToBdsTimeBiasMillisecondsUncertainty @45 :Float32; - - hasRtcTime @46 :Bool; - systemRtcTime @47 :UInt32; - fCountOffset @48 :UInt32; - lpmRtcCount @49 :UInt32; - clockResets @50 :UInt32; - } - - struct DrMeasurementReport @0x8053c39445c6c75c { - - reason @0 :UInt8; - seqNum @1 :UInt8; - seqMax @2 :UInt8; - rfLoss @3 :UInt16; - - systemRtcValid @4 :Bool; - fCount @5 :UInt32; - clockResets @6 :UInt32; - systemRtcTime @7 :UInt64; - - gpsLeapSeconds @8 :UInt8; - gpsLeapSecondsUncertainty @9 :UInt8; - gpsToGlonassTimeBiasMilliseconds @10 :Float32; - gpsToGlonassTimeBiasMillisecondsUncertainty @11 :Float32; - - gpsWeek @12 :UInt16; - gpsMilliseconds @13 :UInt32; - gpsTimeBiasMs @14 :UInt32; - gpsClockTimeUncertaintyMs @15 :UInt32; - gpsClockSource @16 :UInt8; - - glonassClockSource @17 :UInt8; - glonassYear @18 :UInt8; - glonassDay @19 :UInt16; - glonassMilliseconds @20 :UInt32; - glonassTimeBias @21 :Float32; - glonassClockTimeUncertainty @22 :Float32; - - clockFrequencyBias @23 :Float32; - clockFrequencyUncertainty @24 :Float32; - frequencySource @25 :UInt8; - - source @26 :MeasurementSource; - - sv @27 :List(SV); - - struct SV @0xf08b81df8cbf459c { - svId @0 :UInt8; - glonassFrequencyIndex @1 :Int8; - observationState @2 :SVObservationState; - observations @3 :UInt8; - goodObservations @4 :UInt8; - filterStages @5 :UInt8; - predetectInterval @6 :UInt8; - cycleSlipCount @7 :UInt8; - postdetections @8 :UInt16; - - measurementStatus @9 :MeasurementStatus; - - carrierNoise @10 :UInt16; - rfLoss @11 :UInt16; - latency @12 :Int16; - - filteredMeasurementFraction @13 :Float32; - filteredMeasurementIntegral @14 :UInt32; - filteredTimeUncertainty @15 :Float32; - filteredSpeed @16 :Float32; - filteredSpeedUncertainty @17 :Float32; - - unfilteredMeasurementFraction @18 :Float32; - unfilteredMeasurementIntegral @19 :UInt32; - unfilteredTimeUncertainty @20 :Float32; - unfilteredSpeed @21 :Float32; - unfilteredSpeedUncertainty @22 :Float32; - - multipathEstimate @23 :UInt32; - azimuth @24 :Float32; - elevation @25 :Float32; - dopplerAcceleration @26 :Float32; - fineSpeed @27 :Float32; - fineSpeedUncertainty @28 :Float32; - - carrierPhase @29 :Float64; - fCount @30 :UInt32; - - parityErrorCount @31 :UInt16; - goodParity @32 :Bool; - } - } - - struct DrSvPolyReport @0xb1fb80811a673270 { - svId @0 :UInt16; - frequencyIndex @1 :Int8; - - hasPosition @2 :Bool; - hasIono @3 :Bool; - hasTropo @4 :Bool; - hasElevation @5 :Bool; - polyFromXtra @6 :Bool; - hasSbasIono @7 :Bool; - - iode @8 :UInt16; - t0 @9 :Float64; - xyz0 @10 :List(Float64); - xyzN @11 :List(Float64); - other @12 :List(Float32); - - positionUncertainty @13 :Float32; - ionoDelay @14 :Float32; - ionoDot @15 :Float32; - sbasIonoDelay @16 :Float32; - sbasIonoDot @17 :Float32; - tropoDelay @18 :Float32; - elevation @19 :Float32; - elevationDot @20 :Float32; - elevationUncertainty @21 :Float32; - velocityCoeff @22 :List(Float64); - - gpsWeek @23 :UInt16; - gpsTow @24 :Float64; - } -} - -struct Clocks { - wallTimeNanos @3 :UInt64; # unix epoch time - - bootTimeNanosDEPRECATED @0 :UInt64; - monotonicNanosDEPRECATED @1 :UInt64; - monotonicRawNanosDEPRECATD @2 :UInt64; - modemUptimeMillisDEPRECATED @4 :UInt64; -} - -struct LiveMpcData { - x @0 :List(Float32); - y @1 :List(Float32); - psi @2 :List(Float32); - curvature @3 :List(Float32); - qpIterations @4 :UInt32; - calculationTime @5 :UInt64; - cost @6 :Float64; -} - -struct LiveLongitudinalMpcData { - xEgo @0 :List(Float32); - vEgo @1 :List(Float32); - aEgo @2 :List(Float32); - xLead @3 :List(Float32); - vLead @4 :List(Float32); - aLead @5 :List(Float32); - aLeadTau @6 :Float32; # lead accel time constant - qpIterations @7 :UInt32; - mpcId @8 :UInt32; - calculationTime @9 :UInt64; - cost @10 :Float64; -} - -struct Joystick { - # convenient for debug and live tuning - axes @0: List(Float32); - buttons @1: List(Bool); -} - -struct DriverStateV2 { - frameId @0 :UInt32; - modelExecutionTime @1 :Float32; - dspExecutionTime @2 :Float32; - rawPredictions @3 :Data; - - poorVisionProb @4 :Float32; - wheelOnRightProb @5 :Float32; - - leftDriverData @6 :DriverData; - rightDriverData @7 :DriverData; - - struct DriverData { - faceOrientation @0 :List(Float32); - faceOrientationStd @1 :List(Float32); - facePosition @2 :List(Float32); - facePositionStd @3 :List(Float32); - faceProb @4 :Float32; - leftEyeProb @5 :Float32; - rightEyeProb @6 :Float32; - leftBlinkProb @7 :Float32; - rightBlinkProb @8 :Float32; - sunglassesProb @9 :Float32; - occludedProb @10 :Float32; - readyProb @11 :List(Float32); - notReadyProb @12 :List(Float32); - } -} - -struct DriverStateDEPRECATED @0xb83c6cc593ed0a00 { - frameId @0 :UInt32; - modelExecutionTime @14 :Float32; - dspExecutionTime @16 :Float32; - rawPredictions @15 :Data; - - faceOrientation @3 :List(Float32); - facePosition @4 :List(Float32); - faceProb @5 :Float32; - leftEyeProb @6 :Float32; - rightEyeProb @7 :Float32; - leftBlinkProb @8 :Float32; - rightBlinkProb @9 :Float32; - faceOrientationStd @11 :List(Float32); - facePositionStd @12 :List(Float32); - sunglassesProb @13 :Float32; - poorVision @17 :Float32; - partialFace @18 :Float32; - distractedPose @19 :Float32; - distractedEyes @20 :Float32; - eyesOnRoad @21 :Float32; - phoneUse @22 :Float32; - occludedProb @23 :Float32; - - readyProb @24 :List(Float32); - notReadyProb @25 :List(Float32); - - irPwrDEPRECATED @10 :Float32; - descriptorDEPRECATED @1 :List(Float32); - stdDEPRECATED @2 :Float32; -} - -struct DriverMonitoringState @0xb83cda094a1da284 { - events @0 :List(Car.CarEvent); - faceDetected @1 :Bool; - isDistracted @2 :Bool; - distractedType @17 :UInt32; - awarenessStatus @3 :Float32; - posePitchOffset @6 :Float32; - posePitchValidCount @7 :UInt32; - poseYawOffset @8 :Float32; - poseYawValidCount @9 :UInt32; - stepChange @10 :Float32; - awarenessActive @11 :Float32; - awarenessPassive @12 :Float32; - isLowStd @13 :Bool; - hiStdCount @14 :UInt32; - isActiveMode @16 :Bool; - isRHD @4 :Bool; - - isPreviewDEPRECATED @15 :Bool; - rhdCheckedDEPRECATED @5 :Bool; -} - -struct Boot { - wallTimeNanos @0 :UInt64; - pstore @4 :Map(Text, Data); - commands @5 :Map(Text, Data); - launchLog @3 :Text; - - lastKmsgDEPRECATED @1 :Data; - lastPmsgDEPRECATED @2 :Data; -} - -struct LiveParametersData { - valid @0 :Bool; - gyroBias @1 :Float32; - angleOffsetDeg @2 :Float32; - angleOffsetAverageDeg @3 :Float32; - stiffnessFactor @4 :Float32; - steerRatio @5 :Float32; - sensorValid @6 :Bool; - posenetSpeed @8 :Float32; - posenetValid @9 :Bool; - angleOffsetFastStd @10 :Float32; - angleOffsetAverageStd @11 :Float32; - stiffnessFactorStd @12 :Float32; - steerRatioStd @13 :Float32; - roll @14 :Float32; - filterState @15 :LiveLocationKalman.Measurement; - - yawRateDEPRECATED @7 :Float32; -} - -struct LiveTorqueParametersData { - liveValid @0 :Bool; - latAccelFactorRaw @1 :Float32; - latAccelOffsetRaw @2 :Float32; - frictionCoefficientRaw @3 :Float32; - latAccelFactorFiltered @4 :Float32; - latAccelOffsetFiltered @5 :Float32; - frictionCoefficientFiltered @6 :Float32; - totalBucketPoints @7 :Float32; - decay @8 :Float32; - maxResets @9 :Float32; - points @10 :List(List(Float32)); - version @11 :Int32; - useParams @12 :Bool; -} - -struct LiveMapDataDEPRECATED { - speedLimitValid @0 :Bool; - speedLimit @1 :Float32; - speedAdvisoryValid @12 :Bool; - speedAdvisory @13 :Float32; - speedLimitAheadValid @14 :Bool; - speedLimitAhead @15 :Float32; - speedLimitAheadDistance @16 :Float32; - curvatureValid @2 :Bool; - curvature @3 :Float32; - wayId @4 :UInt64; - roadX @5 :List(Float32); - roadY @6 :List(Float32); - lastGps @7: GpsLocationData; - roadCurvatureX @8 :List(Float32); - roadCurvature @9 :List(Float32); - distToTurn @10 :Float32; - mapValid @11 :Bool; -} - -struct CameraOdometry { - frameId @4 :UInt32; - timestampEof @5 :UInt64; - trans @0 :List(Float32); # m/s in device frame - rot @1 :List(Float32); # rad/s in device frame - transStd @2 :List(Float32); # std m/s in device frame - rotStd @3 :List(Float32); # std rad/s in device frame - wideFromDeviceEuler @6 :List(Float32); - wideFromDeviceEulerStd @7 :List(Float32); - roadTransformTrans @8 :List(Float32); - roadTransformTransStd @9 :List(Float32); -} - -struct Sentinel { - enum SentinelType { - endOfSegment @0; - endOfRoute @1; - startOfSegment @2; - startOfRoute @3; - } - type @0 :SentinelType; - signal @1 :Int32; -} - -struct UIDebug { - drawTimeMillis @0 :Float32; -} - -struct ManagerState { - processes @0 :List(ProcessState); - - struct ProcessState { - name @0 :Text; - pid @1 :Int32; - running @2 :Bool; - shouldBeRunning @4 :Bool; - exitCode @3 :Int32; - } -} - -struct UploaderState { - immediateQueueSize @0 :UInt32; - immediateQueueCount @1 :UInt32; - rawQueueSize @2 :UInt32; - rawQueueCount @3 :UInt32; - - # stats for last successfully uploaded file - lastTime @4 :Float32; # s - lastSpeed @5 :Float32; # MB/s - lastFilename @6 :Text; -} - -struct NavInstruction { - maneuverPrimaryText @0 :Text; - maneuverSecondaryText @1 :Text; - maneuverDistance @2 :Float32; # m - maneuverType @3 :Text; # TODO: Make Enum - maneuverModifier @4 :Text; # TODO: Make Enum - - distanceRemaining @5 :Float32; # m - timeRemaining @6 :Float32; # s - timeRemainingTypical @7 :Float32; # s - - lanes @8 :List(Lane); - showFull @9 :Bool; - - speedLimit @10 :Float32; # m/s - speedLimitSign @11 :SpeedLimitSign; - - allManeuvers @12 :List(Maneuver); - - struct Lane { - directions @0 :List(Direction); - active @1 :Bool; - activeDirection @2 :Direction; - } - - enum Direction { - none @0; - left @1; - right @2; - straight @3; - slightLeft @4; - slightRight @5; - } - - enum SpeedLimitSign { - mutcd @0; # US Style - vienna @1; # EU Style - } - - struct Maneuver { - distance @0 :Float32; - type @1 :Text; - modifier @2 :Text; - } -} - -struct NavRoute { - coordinates @0 :List(Coordinate); - - struct Coordinate { - latitude @0 :Float32; - longitude @1 :Float32; - } -} - -struct MapRenderState { - locationMonoTime @0 :UInt64; - renderTime @1 :Float32; - frameId @2: UInt32; -} - -struct NavModelData { - frameId @0 :UInt32; - locationMonoTime @6 :UInt64; - modelExecutionTime @1 :Float32; - dspExecutionTime @2 :Float32; - features @3 :List(Float32); - # predicted future position - position @4 :XYData; - desirePrediction @5 :List(Float32); - - # All SI units and in device frame - struct XYData { - x @0 :List(Float32); - y @1 :List(Float32); - xStd @2 :List(Float32); - yStd @3 :List(Float32); - } -} - -struct EncodeData { - idx @0 :EncodeIndex; - data @1 :Data; - header @2 :Data; - unixTimestampNanos @3 :UInt64; - width @4 :UInt32; - height @5 :UInt32; -} - -struct UserFlag { -} - -struct Microphone { - soundPressure @0 :Float32; - - # uncalibrated, A-weighted - soundPressureWeighted @3 :Float32; - soundPressureWeightedDb @1 :Float32; - filteredSoundPressureWeightedDb @2 :Float32; -} - -struct Event { - logMonoTime @0 :UInt64; # nanoseconds - valid @67 :Bool = true; - - union { - # *********** log metadata *********** - initData @1 :InitData; - sentinel @73 :Sentinel; - - # *********** bootlog *********** - boot @60 :Boot; - - # ********** openpilot daemon msgs ********** - gpsNMEA @3 :GPSNMEAData; - can @5 :List(CanData); - controlsState @7 :ControlsState; - gyroscope @99 :SensorEventData; - gyroscope2 @100 :SensorEventData; - accelerometer @98 :SensorEventData; - accelerometer2 @101 :SensorEventData; - magnetometer @95 :SensorEventData; - lightSensor @96 :SensorEventData; - temperatureSensor @97 :SensorEventData; - temperatureSensor2 @123 :SensorEventData; - pandaStates @81 :List(PandaState); - peripheralState @80 :PeripheralState; - radarState @13 :RadarState; - liveTracks @16 :List(LiveTracks); - sendcan @17 :List(CanData); - liveCalibration @19 :LiveCalibrationData; - carState @22 :Car.CarState; - carControl @23 :Car.CarControl; - carOutput @127 :Car.CarOutput; - longitudinalPlan @24 :LongitudinalPlan; - uiPlan @106 :UiPlan; - ubloxGnss @34 :UbloxGnss; - ubloxRaw @39 :Data; - qcomGnss @31 :QcomGnss; - gpsLocationExternal @48 :GpsLocationData; - gpsLocation @21 :GpsLocationData; - gnssMeasurements @91 :GnssMeasurements; - liveParameters @61 :LiveParametersData; - liveTorqueParameters @94 :LiveTorqueParametersData; - cameraOdometry @63 :CameraOdometry; - thumbnail @66: Thumbnail; - onroadEvents @68: List(Car.CarEvent); - carParams @69: Car.CarParams; - driverMonitoringState @71: DriverMonitoringState; - liveLocationKalman @72 :LiveLocationKalman; - modelV2 @75 :ModelDataV2; - driverStateV2 @92 :DriverStateV2; - - # camera stuff, each camera state has a matching encode idx - roadCameraState @2 :FrameData; - driverCameraState @70: FrameData; - wideRoadCameraState @74: FrameData; - roadEncodeIdx @15 :EncodeIndex; - driverEncodeIdx @76 :EncodeIndex; - wideRoadEncodeIdx @77 :EncodeIndex; - qRoadEncodeIdx @90 :EncodeIndex; - - livestreamRoadEncodeIdx @117 :EncodeIndex; - livestreamWideRoadEncodeIdx @118 :EncodeIndex; - livestreamDriverEncodeIdx @119 :EncodeIndex; - - # microphone data - microphone @103 :Microphone; - - # systems stuff - androidLog @20 :AndroidLogEntry; - managerState @78 :ManagerState; - uploaderState @79 :UploaderState; - procLog @33 :ProcLog; - clocks @35 :Clocks; - deviceState @6 :DeviceState; - logMessage @18 :Text; - errorLogMessage @85 :Text; - - # navigation - navInstruction @82 :NavInstruction; - navRoute @83 :NavRoute; - navThumbnail @84: Thumbnail; - mapRenderState @105: MapRenderState; - - # UI services - userFlag @93 :UserFlag; - uiDebug @102 :UIDebug; - - # *********** debug *********** - testJoystick @52 :Joystick; - roadEncodeData @86 :EncodeData; - driverEncodeData @87 :EncodeData; - wideRoadEncodeData @88 :EncodeData; - qRoadEncodeData @89 :EncodeData; - - livestreamRoadEncodeData @120 :EncodeData; - livestreamWideRoadEncodeData @121 :EncodeData; - livestreamDriverEncodeData @122 :EncodeData; - - customReservedRawData0 @124 :Data; - customReservedRawData1 @125 :Data; - customReservedRawData2 @126 :Data; - - # *********** Custom: reserved for forks *********** - customReserved0 @107 :Custom.CustomReserved0; - customReserved1 @108 :Custom.CustomReserved1; - customReserved2 @109 :Custom.CustomReserved2; - customReserved3 @110 :Custom.CustomReserved3; - customReserved4 @111 :Custom.CustomReserved4; - customReserved5 @112 :Custom.CustomReserved5; - customReserved6 @113 :Custom.CustomReserved6; - customReserved7 @114 :Custom.CustomReserved7; - customReserved8 @115 :Custom.CustomReserved8; - customReserved9 @116 :Custom.CustomReserved9; - - # *********** legacy + deprecated *********** - model @9 :Legacy.ModelData; # TODO: rename modelV2 and mark this as deprecated - liveMpcDEPRECATED @36 :LiveMpcData; - liveLongitudinalMpcDEPRECATED @37 :LiveLongitudinalMpcData; - liveLocationKalmanDEPRECATED @51 :Legacy.LiveLocationData; - orbslamCorrectionDEPRECATED @45 :Legacy.OrbslamCorrection; - liveUIDEPRECATED @14 :Legacy.LiveUI; - sensorEventDEPRECATED @4 :SensorEventData; - liveEventDEPRECATED @8 :List(Legacy.LiveEventData); - liveLocationDEPRECATED @25 :Legacy.LiveLocationData; - ethernetDataDEPRECATED @26 :List(Legacy.EthernetPacket); - cellInfoDEPRECATED @28 :List(Legacy.CellInfo); - wifiScanDEPRECATED @29 :List(Legacy.WifiScan); - uiNavigationEventDEPRECATED @50 :Legacy.UiNavigationEvent; - liveMapDataDEPRECATED @62 :LiveMapDataDEPRECATED; - gpsPlannerPointsDEPRECATED @40 :Legacy.GPSPlannerPoints; - gpsPlannerPlanDEPRECATED @41 :Legacy.GPSPlannerPlan; - applanixRawDEPRECATED @42 :Data; - androidGnssDEPRECATED @30 :Legacy.AndroidGnss; - lidarPtsDEPRECATED @32 :Legacy.LidarPts; - navStatusDEPRECATED @38 :Legacy.NavStatus; - trafficEventsDEPRECATED @43 :List(Legacy.TrafficEvent); - liveLocationTimingDEPRECATED @44 :Legacy.LiveLocationData; - liveLocationCorrectedDEPRECATED @46 :Legacy.LiveLocationData; - navUpdateDEPRECATED @27 :Legacy.NavUpdate; - orbObservationDEPRECATED @47 :List(Legacy.OrbObservation); - locationDEPRECATED @49 :Legacy.LiveLocationData; - orbOdometryDEPRECATED @53 :Legacy.OrbOdometry; - orbFeaturesDEPRECATED @54 :Legacy.OrbFeatures; - applanixLocationDEPRECATED @55 :Legacy.LiveLocationData; - orbKeyFrameDEPRECATED @56 :Legacy.OrbKeyFrame; - orbFeaturesSummaryDEPRECATED @58 :Legacy.OrbFeaturesSummary; - featuresDEPRECATED @10 :Legacy.CalibrationFeatures; - kalmanOdometryDEPRECATED @65 :Legacy.KalmanOdometry; - uiLayoutStateDEPRECATED @57 :Legacy.UiLayoutState; - pandaStateDEPRECATED @12 :PandaState; - driverStateDEPRECATED @59 :DriverStateDEPRECATED; - sensorEventsDEPRECATED @11 :List(SensorEventData); - lateralPlanDEPRECATED @64 :LateralPlan; - navModelDEPRECATED @104 :NavModelData; - } -} diff --git a/maptile.capnp b/maptile.capnp deleted file mode 100644 index c8a23a182..000000000 --- a/maptile.capnp +++ /dev/null @@ -1,49 +0,0 @@ -using Cxx = import "./include/c++.capnp"; -$Cxx.namespace("cereal"); - -@0xa086df597ef5d7a0; - -# Geometry -struct Point { - x @0: Float64; - y @1: Float64; - z @2: Float64; -} - -struct PolyLine { - points @0: List(Point); -} - -# Map features -struct Lane { - id @0 :Text; - - leftBoundary @1 :LaneBoundary; - rightBoundary @2 :LaneBoundary; - - leftAdjacentId @3 :Text; - rightAdjacentId @4 :Text; - - inboundIds @5 :List(Text); - outboundIds @6 :List(Text); - - struct LaneBoundary { - polyLine @0 :PolyLine; - startHeading @1 :Float32; # WRT north - } -} - -# Map tiles -struct TileSummary { - version @0 :Text; - updatedAt @1 :UInt64; # Millis since epoch - - level @2 :UInt8; - x @3 :UInt16; - y @4 :UInt16; -} - -struct MapTile { - summary @0 :TileSummary; - lanes @1 :List(Lane); -} diff --git a/messaging/.gitignore b/messaging/.gitignore index dbbe8e22a..9f0c6fe7e 100644 --- a/messaging/.gitignore +++ b/messaging/.gitignore @@ -1,10 +1 @@ -demo -bridge -test_runner -*.o -*.os -*.d -*.a -*.so messaging_pyx.cpp -build/ diff --git a/messaging/__init__.py b/messaging/__init__.py index a2695bda2..e69de29bb 100644 --- a/messaging/__init__.py +++ b/messaging/__init__.py @@ -1,306 +0,0 @@ -# must be built with scons -from .messaging_pyx import Context, Poller, SubSocket, PubSocket, SocketEventHandle, toggle_fake_events, \ - set_fake_prefix, get_fake_prefix, delete_fake_prefix, wait_for_one_event -from .messaging_pyx import MultiplePublishersError, MessagingError - -import os -import capnp -import time - -from typing import Optional, List, Union, Dict, Deque -from collections import deque - -from cereal import log -from cereal.services import SERVICE_LIST - -assert MultiplePublishersError -assert MessagingError -assert toggle_fake_events -assert set_fake_prefix -assert get_fake_prefix -assert delete_fake_prefix -assert wait_for_one_event - -NO_TRAVERSAL_LIMIT = 2**64-1 - -context = Context() - - -def fake_event_handle(endpoint: str, identifier: Optional[str] = None, override: bool = True, enable: bool = False) -> SocketEventHandle: - identifier = identifier or get_fake_prefix() - handle = SocketEventHandle(endpoint, identifier, override) - if override: - handle.enabled = enable - - return handle - - -def log_from_bytes(dat: bytes) -> capnp.lib.capnp._DynamicStructReader: - with log.Event.from_bytes(dat, traversal_limit_in_words=NO_TRAVERSAL_LIMIT) as msg: - return msg - - -def new_message(service: Optional[str], size: Optional[int] = None, **kwargs) -> capnp.lib.capnp._DynamicStructBuilder: - args = { - 'valid': False, - 'logMonoTime': int(time.monotonic() * 1e9), - **kwargs - } - dat = log.Event.new_message(**args) - if service is not None: - if size is None: - dat.init(service) - else: - dat.init(service, size) - return dat - - -def pub_sock(endpoint: str) -> PubSocket: - sock = PubSocket() - sock.connect(context, endpoint) - return sock - - -def sub_sock(endpoint: str, poller: Optional[Poller] = None, addr: str = "127.0.0.1", - conflate: bool = False, timeout: Optional[int] = None) -> SubSocket: - sock = SubSocket() - sock.connect(context, endpoint, addr.encode('utf8'), conflate) - - if timeout is not None: - sock.setTimeout(timeout) - - if poller is not None: - poller.registerSocket(sock) - return sock - - -def drain_sock_raw(sock: SubSocket, wait_for_one: bool = False) -> List[bytes]: - """Receive all message currently available on the queue""" - ret: List[bytes] = [] - while 1: - if wait_for_one and len(ret) == 0: - dat = sock.receive() - else: - dat = sock.receive(non_blocking=True) - - if dat is None: - break - - ret.append(dat) - - return ret - - -def drain_sock(sock: SubSocket, wait_for_one: bool = False) -> List[capnp.lib.capnp._DynamicStructReader]: - """Receive all message currently available on the queue""" - msgs = drain_sock_raw(sock, wait_for_one=wait_for_one) - return [log_from_bytes(m) for m in msgs] - - -# TODO: print when we drop packets? -def recv_sock(sock: SubSocket, wait: bool = False) -> Optional[capnp.lib.capnp._DynamicStructReader]: - """Same as drain sock, but only returns latest message. Consider using conflate instead.""" - dat = None - - while 1: - if wait and dat is None: - recv = sock.receive() - else: - recv = sock.receive(non_blocking=True) - - if recv is None: # Timeout hit - break - - dat = recv - - if dat is not None: - dat = log_from_bytes(dat) - - return dat - - -def recv_one(sock: SubSocket) -> Optional[capnp.lib.capnp._DynamicStructReader]: - dat = sock.receive() - if dat is not None: - dat = log_from_bytes(dat) - return dat - - -def recv_one_or_none(sock: SubSocket) -> Optional[capnp.lib.capnp._DynamicStructReader]: - dat = sock.receive(non_blocking=True) - if dat is not None: - dat = log_from_bytes(dat) - return dat - - -def recv_one_retry(sock: SubSocket) -> capnp.lib.capnp._DynamicStructReader: - """Keep receiving until we get a message""" - while True: - dat = sock.receive() - if dat is not None: - return log_from_bytes(dat) - - -class SubMaster: - def __init__(self, services: List[str], poll: Optional[str] = None, - ignore_alive: Optional[List[str]] = None, ignore_avg_freq: Optional[List[str]] = None, - ignore_valid: Optional[List[str]] = None, addr: str = "127.0.0.1", frequency: Optional[float] = None): - self.frame = -1 - self.seen = {s: False for s in services} - self.updated = {s: False for s in services} - self.recv_time = {s: 0. for s in services} - self.recv_frame = {s: 0 for s in services} - self.alive = {s: False for s in services} - self.freq_ok = {s: False for s in services} - self.recv_dts: Dict[str, Deque[float]] = {} - self.sock = {} - self.data = {} - self.valid = {} - self.logMonoTime = {} - - self.max_freq = {} - self.min_freq = {} - - self.poller = Poller() - polled_services = set([poll, ] if poll is not None else services) - self.non_polled_services = set(services) - polled_services - - self.ignore_average_freq = [] if ignore_avg_freq is None else ignore_avg_freq - self.ignore_alive = [] if ignore_alive is None else ignore_alive - self.ignore_valid = [] if ignore_valid is None else ignore_valid - - self.simulation = bool(int(os.getenv("SIMULATION", "0"))) - - # if freq and poll aren't specified, assume the max to be conservative - assert frequency is None or poll is None, "Do not specify 'frequency' - frequency of the polled service will be used." - self.update_freq = frequency or max([SERVICE_LIST[s].frequency for s in polled_services]) - - for s in services: - p = self.poller if s not in self.non_polled_services else None - self.sock[s] = sub_sock(s, poller=p, addr=addr, conflate=True) - - try: - data = new_message(s) - except capnp.lib.capnp.KjException: - data = new_message(s, 0) # lists - - self.data[s] = getattr(data.as_reader(), s) - self.logMonoTime[s] = 0 - self.valid[s] = True # FIXME: this should default to False - - freq = max(min([SERVICE_LIST[s].frequency, self.update_freq]), 1.) - if s == poll: - max_freq = freq - min_freq = freq - else: - max_freq = min(freq, self.update_freq) - if SERVICE_LIST[s].frequency >= 2*self.update_freq: - min_freq = self.update_freq - elif self.update_freq >= 2*SERVICE_LIST[s].frequency: - min_freq = freq - else: - min_freq = min(freq, freq / 2.) - self.max_freq[s] = max_freq*1.2 - self.min_freq[s] = min_freq*0.8 - self.recv_dts[s] = deque(maxlen=int(10*freq)) - - def __getitem__(self, s: str) -> capnp.lib.capnp._DynamicStructReader: - return self.data[s] - - def _check_avg_freq(self, s: str) -> bool: - return SERVICE_LIST[s].frequency > 0.99 and (s not in self.ignore_average_freq) and (s not in self.ignore_alive) - - def update(self, timeout: int = 100) -> None: - msgs = [] - for sock in self.poller.poll(timeout): - msgs.append(recv_one_or_none(sock)) - - # non-blocking receive for non-polled sockets - for s in self.non_polled_services: - msgs.append(recv_one_or_none(self.sock[s])) - self.update_msgs(time.monotonic(), msgs) - - def update_msgs(self, cur_time: float, msgs: List[capnp.lib.capnp._DynamicStructReader]) -> None: - self.frame += 1 - self.updated = dict.fromkeys(self.updated, False) - for msg in msgs: - if msg is None: - continue - - s = msg.which() - self.seen[s] = True - self.updated[s] = True - - if self.recv_time[s] > 1e-5: - self.recv_dts[s].append(cur_time - self.recv_time[s]) - self.recv_time[s] = cur_time - self.recv_frame[s] = self.frame - self.data[s] = getattr(msg, s) - self.logMonoTime[s] = msg.logMonoTime - self.valid[s] = msg.valid - - for s in self.data: - if SERVICE_LIST[s].frequency > 1e-5 and not self.simulation: - # alive if delay is within 10x the expected frequency - self.alive[s] = (cur_time - self.recv_time[s]) < (10. / SERVICE_LIST[s].frequency) - - # check average frequency; slow to fall, quick to recover - dts = self.recv_dts[s] - assert dts.maxlen is not None - recent_dts = list(dts)[-int(dts.maxlen / 10):] - try: - avg_freq = 1 / (sum(dts) / len(dts)) - avg_freq_recent = 1 / (sum(recent_dts) / len(recent_dts)) - except ZeroDivisionError: - avg_freq = 0 - avg_freq_recent = 0 - - avg_freq_ok = self.min_freq[s] <= avg_freq <= self.max_freq[s] - recent_freq_ok = self.min_freq[s] <= avg_freq_recent <= self.max_freq[s] - self.freq_ok[s] = avg_freq_ok or recent_freq_ok - else: - self.freq_ok[s] = True - if self.simulation: - self.alive[s] = self.seen[s] # alive is defined as seen when simulation flag set - else: - self.alive[s] = True - - def all_alive(self, service_list: Optional[List[str]] = None) -> bool: - if service_list is None: - service_list = list(self.sock.keys()) - return all(self.alive[s] for s in service_list if s not in self.ignore_alive) - - def all_freq_ok(self, service_list: Optional[List[str]] = None) -> bool: - if service_list is None: - service_list = list(self.sock.keys()) - return all(self.freq_ok[s] for s in service_list if self._check_avg_freq(s)) - - def all_valid(self, service_list: Optional[List[str]] = None) -> bool: - if service_list is None: - service_list = list(self.sock.keys()) - return all(self.valid[s] for s in service_list if s not in self.ignore_valid) - - def all_checks(self, service_list: Optional[List[str]] = None) -> bool: - return self.all_alive(service_list) and self.all_freq_ok(service_list) and self.all_valid(service_list) - - -class PubMaster: - def __init__(self, services: List[str]): - self.sock = {} - for s in services: - self.sock[s] = pub_sock(s) - - def send(self, s: str, dat: Union[bytes, capnp.lib.capnp._DynamicStructBuilder]) -> None: - if not isinstance(dat, bytes): - dat = dat.to_bytes() - self.sock[s].send(dat) - - def wait_for_readers_to_update(self, s: str, timeout: int, dt: float = 0.05) -> bool: - for _ in range(int(timeout*(1./dt))): - if self.sock[s].all_readers_updated(): - return True - time.sleep(dt) - return False - - def all_readers_updated(self, s: str) -> bool: - return self.sock[s].all_readers_updated() # type: ignore diff --git a/messaging/bridge.cc b/messaging/bridge.cc deleted file mode 100644 index 4a5390caf..000000000 --- a/messaging/bridge.cc +++ /dev/null @@ -1,92 +0,0 @@ -#include -#include -#include -#include -#include -#include - -typedef void (*sighandler_t)(int sig); - -#include "cereal/services.h" -#include "cereal/messaging/impl_msgq.h" -#include "cereal/messaging/impl_zmq.h" - -std::atomic do_exit = false; -static void set_do_exit(int sig) { - do_exit = true; -} - -void sigpipe_handler(int sig) { - assert(sig == SIGPIPE); - std::cout << "SIGPIPE received" << std::endl; -} - -static std::vector get_services(std::string whitelist_str, bool zmq_to_msgq) { - std::vector service_list; - for (const auto& it : services) { - std::string name = it.second.name; - bool in_whitelist = whitelist_str.find(name) != std::string::npos; - if (name == "plusFrame" || name == "uiLayoutState" || (zmq_to_msgq && !in_whitelist)) { - continue; - } - service_list.push_back(name); - } - return service_list; -} - -int main(int argc, char** argv) { - signal(SIGPIPE, (sighandler_t)sigpipe_handler); - signal(SIGINT, (sighandler_t)set_do_exit); - signal(SIGTERM, (sighandler_t)set_do_exit); - - bool zmq_to_msgq = argc > 2; - std::string ip = zmq_to_msgq ? argv[1] : "127.0.0.1"; - std::string whitelist_str = zmq_to_msgq ? std::string(argv[2]) : ""; - - Poller *poller; - Context *pub_context; - Context *sub_context; - if (zmq_to_msgq) { // republishes zmq debugging messages as msgq - poller = new ZMQPoller(); - pub_context = new MSGQContext(); - sub_context = new ZMQContext(); - } else { - poller = new MSGQPoller(); - pub_context = new ZMQContext(); - sub_context = new MSGQContext(); - } - - std::map sub2pub; - for (auto endpoint : get_services(whitelist_str, zmq_to_msgq)) { - PubSocket * pub_sock; - SubSocket * sub_sock; - if (zmq_to_msgq) { - pub_sock = new MSGQPubSocket(); - sub_sock = new ZMQSubSocket(); - } else { - pub_sock = new ZMQPubSocket(); - sub_sock = new MSGQSubSocket(); - } - pub_sock->connect(pub_context, endpoint); - sub_sock->connect(sub_context, endpoint, ip, false); - - poller->registerSocket(sub_sock); - sub2pub[sub_sock] = pub_sock; - } - - while (!do_exit) { - for (auto sub_sock : poller->poll(100)) { - Message * msg = sub_sock->receive(); - if (msg == NULL) continue; - int ret; - do { - ret = sub2pub[sub_sock]->sendMessage(msg); - } while (ret == -1 && errno == EINTR && !do_exit); - assert(ret >= 0 || do_exit); - delete msg; - - if (do_exit) break; - } - } - return 0; -} diff --git a/messaging/demo.cc b/messaging/demo.cc deleted file mode 100644 index 5b4d24468..000000000 --- a/messaging/demo.cc +++ /dev/null @@ -1,50 +0,0 @@ -#include -#include -#include -#include -#include - -#include "cereal/messaging/messaging.h" -#include "cereal/messaging/impl_zmq.h" - -#define MSGS 1e5 - -int main() { - Context * c = Context::create(); - SubSocket * sub_sock = SubSocket::create(c, "controlsState"); - PubSocket * pub_sock = PubSocket::create(c, "controlsState"); - - char data[8]; - - Poller * poller = Poller::create({sub_sock}); - - auto start = std::chrono::steady_clock::now(); - - for (uint64_t i = 0; i < MSGS; i++){ - *(uint64_t*)data = i; - pub_sock->send(data, 8); - - auto r = poller->poll(100); - - for (auto p : r){ - Message * m = p->receive(); - uint64_t ii = *(uint64_t*)m->getData(); - assert(i == ii); - delete m; - } - } - - - auto end = std::chrono::steady_clock::now(); - double elapsed = std::chrono::duration_cast(end - start).count() / 1e9; - double throughput = ((double) MSGS / (double) elapsed); - std::cout << throughput << " msg/s" << std::endl; - - delete poller; - delete sub_sock; - delete pub_sock; - delete c; - - - return 0; -} diff --git a/messaging/demo.py b/messaging/demo.py deleted file mode 100644 index e4850e32b..000000000 --- a/messaging/demo.py +++ /dev/null @@ -1,29 +0,0 @@ -import time - -from messaging_pyx import Context, Poller, SubSocket, PubSocket - -MSGS = 1e5 - -if __name__ == "__main__": - c = Context() - sub_sock = SubSocket() - pub_sock = PubSocket() - - sub_sock.connect(c, "controlsState") - pub_sock.connect(c, "controlsState") - - poller = Poller() - poller.registerSocket(sub_sock) - - t = time.time() - for i in range(int(MSGS)): - bts = i.to_bytes(4, 'little') - pub_sock.send(bts) - - for s in poller.poll(100): - dat = s.receive() - ii = int.from_bytes(dat, 'little') - assert(i == ii) - - dt = time.time() - t - print("%.1f msg/s" % (MSGS / dt)) diff --git a/messaging/event.cc b/messaging/event.cc index a708de915..0c22e7868 100644 --- a/messaging/event.cc +++ b/messaging/event.cc @@ -13,7 +13,7 @@ #include #include -#include "cereal/messaging/event.h" +#include "msgq/messaging/event.h" #ifndef __APPLE__ #include diff --git a/messaging/impl_fake.cc b/messaging/impl_fake.cc index 178b8b7a0..8399ba5f8 100644 --- a/messaging/impl_fake.cc +++ b/messaging/impl_fake.cc @@ -1,4 +1,4 @@ -#include "cereal/messaging/impl_fake.h" +#include "msgq/messaging/impl_fake.h" void FakePoller::registerSocket(SubSocket *socket) { this->sockets.push_back(socket); diff --git a/messaging/impl_fake.h b/messaging/impl_fake.h index 0ec8486a0..ae40a037f 100644 --- a/messaging/impl_fake.h +++ b/messaging/impl_fake.h @@ -11,8 +11,8 @@ #include #include -#include "cereal/messaging/messaging.h" -#include "cereal/messaging/event.h" +#include "msgq/messaging/messaging.h" +#include "msgq/messaging/event.h" template class FakeSubSocket: public TSubSocket { diff --git a/messaging/impl_msgq.cc b/messaging/impl_msgq.cc index 8f2c10a08..429642b87 100644 --- a/messaging/impl_msgq.cc +++ b/messaging/impl_msgq.cc @@ -5,8 +5,7 @@ #include #include -#include "cereal/services.h" -#include "cereal/messaging/impl_msgq.h" +#include "msgq/messaging/impl_msgq.h" volatile sig_atomic_t msgq_do_exit = 0; @@ -16,10 +15,6 @@ void sig_handler(int signal) { msgq_do_exit = 1; } -static bool service_exists(std::string path){ - return services.count(path) > 0; -} - MSGQContext::MSGQContext() { } @@ -58,10 +53,6 @@ int MSGQSubSocket::connect(Context *context, std::string endpoint, std::string a assert(context); assert(address == "127.0.0.1"); - if (check_endpoint && !service_exists(std::string(endpoint))){ - std::cout << "Warning, " << std::string(endpoint) << " is not in service list." << std::endl; - } - q = new msgq_queue_t; int r = msgq_new_queue(q, endpoint.c_str(), DEFAULT_SEGMENT_SIZE); if (r != 0){ @@ -150,9 +141,10 @@ MSGQSubSocket::~MSGQSubSocket(){ int MSGQPubSocket::connect(Context *context, std::string endpoint, bool check_endpoint){ assert(context); - if (check_endpoint && !service_exists(std::string(endpoint))){ - std::cout << "Warning, " << std::string(endpoint) << " is not in service list." << std::endl; - } + // TODO + //if (check_endpoint && !service_exists(std::string(endpoint))){ + // std::cout << "Warning, " << std::string(endpoint) << " is not in service list." << std::endl; + //} q = new msgq_queue_t; int r = msgq_new_queue(q, endpoint.c_str(), DEFAULT_SEGMENT_SIZE); diff --git a/messaging/impl_msgq.h b/messaging/impl_msgq.h index 68235f0c6..95f13799b 100644 --- a/messaging/impl_msgq.h +++ b/messaging/impl_msgq.h @@ -3,8 +3,8 @@ #include #include -#include "cereal/messaging/messaging.h" -#include "cereal/messaging/msgq.h" +#include "msgq/messaging/messaging.h" +#include "msgq/messaging/msgq.h" #define MAX_POLLERS 128 diff --git a/messaging/impl_zmq.cc b/messaging/impl_zmq.cc index 7da9df1b8..7e45f2bd9 100644 --- a/messaging/impl_zmq.cc +++ b/messaging/impl_zmq.cc @@ -5,11 +5,16 @@ #include #include -#include "cereal/services.h" -#include "cereal/messaging/impl_zmq.h" +#include "msgq/messaging/impl_zmq.h" +//FIXME: This is a hack to get the port number from the socket name, might have collisions static int get_port(std::string endpoint) { - return services.at(endpoint).port; + std::hash hasher; + size_t hash_value = hasher(endpoint); + int start_port = 8023; + int max_port = 65535; + int port = start_port + (hash_value % (max_port - start_port)); + return port; } ZMQContext::ZMQContext() { @@ -59,6 +64,7 @@ int ZMQSubSocket::connect(Context *context, std::string endpoint, std::string ad int reconnect_ivl = 500; zmq_setsockopt(sock, ZMQ_RECONNECT_IVL_MAX, &reconnect_ivl, sizeof(reconnect_ivl)); + full_endpoint = "tcp://" + address + ":"; if (check_endpoint){ full_endpoint += std::to_string(get_port(endpoint)); diff --git a/messaging/impl_zmq.h b/messaging/impl_zmq.h index 903875f63..93ebfb5d4 100644 --- a/messaging/impl_zmq.h +++ b/messaging/impl_zmq.h @@ -4,7 +4,7 @@ #include #include -#include "cereal/messaging/messaging.h" +#include "msgq/messaging/messaging.h" #define MAX_POLLERS 128 diff --git a/messaging/messaging.cc b/messaging/messaging.cc index 6b7fe8f90..5119816b8 100644 --- a/messaging/messaging.cc +++ b/messaging/messaging.cc @@ -1,10 +1,10 @@ #include #include -#include "cereal/messaging/messaging.h" -#include "cereal/messaging/impl_zmq.h" -#include "cereal/messaging/impl_msgq.h" -#include "cereal/messaging/impl_fake.h" +#include "msgq/messaging/messaging.h" +#include "msgq/messaging/impl_zmq.h" +#include "msgq/messaging/impl_msgq.h" +#include "msgq/messaging/impl_fake.h" #ifdef __APPLE__ const bool MUST_USE_ZMQ = true; diff --git a/messaging/messaging.h b/messaging/messaging.h index 91106514a..1f0e3e06b 100644 --- a/messaging/messaging.h +++ b/messaging/messaging.h @@ -7,9 +7,7 @@ #include #include -#include -#include "cereal/gen/cpp/log.capnp.h" #ifdef __APPLE__ #define CLOCK_BOOTTIME CLOCK_MONOTONIC @@ -67,96 +65,4 @@ class Poller { static Poller * create(); static Poller * create(std::vector sockets); virtual ~Poller(){} -}; - -class SubMaster { -public: - SubMaster(const std::vector &service_list, const std::vector &poll = {}, - const char *address = nullptr, const std::vector &ignore_alive = {}); - void update(int timeout = 1000); - void update_msgs(uint64_t current_time, const std::vector> &messages); - inline bool allAlive(const std::vector &service_list = {}) { return all_(service_list, false, true); } - inline bool allValid(const std::vector &service_list = {}) { return all_(service_list, true, false); } - inline bool allAliveAndValid(const std::vector &service_list = {}) { return all_(service_list, true, true); } - void drain(); - ~SubMaster(); - - uint64_t frame = 0; - bool updated(const char *name) const; - bool alive(const char *name) const; - bool valid(const char *name) const; - uint64_t rcv_frame(const char *name) const; - uint64_t rcv_time(const char *name) const; - cereal::Event::Reader &operator[](const char *name) const; - -private: - bool all_(const std::vector &service_list, bool valid, bool alive); - Poller *poller_ = nullptr; - struct SubMessage; - std::map messages_; - std::map services_; -}; - -class MessageBuilder : public capnp::MallocMessageBuilder { -public: - MessageBuilder() = default; - - cereal::Event::Builder initEvent(bool valid = true) { - cereal::Event::Builder event = initRoot(); - struct timespec t; - clock_gettime(CLOCK_BOOTTIME, &t); - uint64_t current_time = t.tv_sec * 1000000000ULL + t.tv_nsec; - event.setLogMonoTime(current_time); - event.setValid(valid); - return event; - } - - kj::ArrayPtr toBytes() { - heapArray_ = capnp::messageToFlatArray(*this); - return heapArray_.asBytes(); - } - - size_t getSerializedSize() { - return capnp::computeSerializedSizeInWords(*this) * sizeof(capnp::word); - } - - int serializeToBuffer(unsigned char *buffer, size_t buffer_size) { - size_t serialized_size = getSerializedSize(); - if (serialized_size > buffer_size) { return -1; } - kj::ArrayOutputStream out(kj::ArrayPtr(buffer, buffer_size)); - capnp::writeMessage(out, *this); - return serialized_size; - } - -private: - kj::Array heapArray_; -}; - -class PubMaster { -public: - PubMaster(const std::vector &service_list); - inline int send(const char *name, capnp::byte *data, size_t size) { return sockets_.at(name)->send((char *)data, size); } - int send(const char *name, MessageBuilder &msg); - ~PubMaster(); - -private: - std::map sockets_; -}; - -class AlignedBuffer { -public: - kj::ArrayPtr align(const char *data, const size_t size) { - words_size = size / sizeof(capnp::word) + 1; - if (aligned_buf.size() < words_size) { - aligned_buf = kj::heapArray(words_size < 512 ? 512 : words_size); - } - memcpy(aligned_buf.begin(), data, size); - return aligned_buf.slice(0, words_size); - } - inline kj::ArrayPtr align(Message *m) { - return align(m->getData(), m->getSize()); - } -private: - kj::Array aligned_buf; - size_t words_size; -}; +}; \ No newline at end of file diff --git a/messaging/messaging.pxd b/messaging/messaging.pxd index 97b9a22b0..420fc09b5 100644 --- a/messaging/messaging.pxd +++ b/messaging/messaging.pxd @@ -6,7 +6,7 @@ from libcpp.vector cimport vector from libcpp cimport bool -cdef extern from "cereal/messaging/impl_fake.h": +cdef extern from "msgq/messaging/impl_fake.h": cdef cppclass Event: @staticmethod int wait_for_one(vector[Event], int) except + @@ -34,7 +34,7 @@ cdef extern from "cereal/messaging/impl_fake.h": Event recv_ready() -cdef extern from "cereal/messaging/messaging.h": +cdef extern from "msgq/messaging/messaging.h": cdef cppclass Context: @staticmethod Context * create() diff --git a/messaging/msgq.cc b/messaging/msgq.cc index af93bbf60..9344652f0 100644 --- a/messaging/msgq.cc +++ b/messaging/msgq.cc @@ -23,7 +23,7 @@ #include -#include "cereal/messaging/msgq.h" +#include "msgq/messaging/msgq.h" void sigusr2_handler(int signal) { assert(signal == SIGUSR2); diff --git a/messaging/msgq.md b/messaging/msgq.md deleted file mode 100644 index 34fe35682..000000000 --- a/messaging/msgq.md +++ /dev/null @@ -1,54 +0,0 @@ -# MSGQ: A lock free single producer multi consumer message queue - -## What is MSGQ? -MSGQ is a system to pass messages from a single producer to multiple consumers. All the consumers need to be able to receive all the messages. It is designed to be a high performance replacement for ZMQ-like SUB/PUB patterns. It uses a ring buffer in shared memory to efficiently read and write data. Each read requires a copy. Writing can be done without a copy, as long as the size of the data is known in advance. - -## Storage -The storage for the queue consists of an area of metadata, and the actual buffer. The metadata contains: - -1. A counter to the number of readers that are active -2. A pointer to the head of the queue for writing. From now on referred to as *write pointer* -3. A cycle counter for the writer. This counter is incremented when the writer wraps around -4. N pointers, pointing to the current read position for all the readers. From now on referred to as *read pointer* -5. N counters, counting the number of cycles for all the readers -6. N booleans, indicating validity for all the readers. From now on referred to as *validity flag* - -The counter and the pointer are both 32 bit values, packed into 64 bit so they can be read and written atomically. - -The data buffer is a ring buffer. All messages are prefixed by an 8 byte size field, followed by the data. A size of -1 indicates a wrap-around, and means the next message is stored at the beginning of the buffer. - - -## Writing -Writing involves the following steps: - -1. Check if the area that is to be written overlaps with any of the read pointers, mark those readers as invalid by clearing the validity flag. -2. Write the message -3. Increase the write pointer by the size of the message - -In case there is not enough space at the end of the buffer, a special empty message with a prefix of -1 is written. The cycle counter is incremented by one. In this case step 1 will check there are no read pointers pointing to the remainder of the buffer. Then another write cycle will start with the actual message. - -There always needs to be 8 bytes of empty space at the end of the buffer. By doing this there is always space to write the -1. - -## Reset reader -When the reader is lagging too much behind the read pointer becomes invalid and no longer points to the beginning of a valid message. To reset a reader to the current write pointer, the following steps are performed: - -1. Set valid flag -2. Set read cycle counter to that of the writer -3. Set read pointer to write pointer - -## Reading -Reading involves the following steps: - -1. Read the size field at the current read pointer -2. Read the validity flag -3. Copy the data out of the buffer -4. Increase the read pointer by the size of the message -5. Check the validity flag again - -Before starting the copy, the valid flag is checked. This is to prevent a race condition where the size prefix was invalid, and the read could read outside of the buffer. Make sure that step 1 and 2 are not reordered by your compiler or CPU. - -If a writer overwrites the data while it's being copied out, the data will be invalid. Therefore the validity flag is also checked after reading it. The order of step 4 and 5 does not matter. - -If at steps 2 or 5 the validity flag is not set, the reader is reset. Any data that was already read is discarded. After the reader is reset, the reading starts from the beginning. - -If a message with size -1 is encountered, step 3 and 4 are replaced by increasing the cycle counter and setting the read pointer to the beginning of the buffer. After that another read is performed. diff --git a/messaging/msgq_tests.cc b/messaging/msgq_tests.cc index d33facbc0..bc3984e62 100644 --- a/messaging/msgq_tests.cc +++ b/messaging/msgq_tests.cc @@ -1,7 +1,8 @@ #include "catch2/catch.hpp" -#include "cereal/messaging/msgq.h" +#include "msgq/messaging/msgq.h" -TEST_CASE("ALIGN"){ +TEST_CASE("ALIGN") +{ REQUIRE(ALIGN(0) == 0); REQUIRE(ALIGN(1) == 8); REQUIRE(ALIGN(7) == 8); @@ -9,7 +10,8 @@ TEST_CASE("ALIGN"){ REQUIRE(ALIGN(99999) == 100000); } -TEST_CASE("msgq_msg_init_size"){ +TEST_CASE("msgq_msg_init_size") +{ const size_t msg_size = 30; msgq_msg_t msg; @@ -19,11 +21,13 @@ TEST_CASE("msgq_msg_init_size"){ msgq_msg_close(&msg); } -TEST_CASE("msgq_msg_init_data"){ +TEST_CASE("msgq_msg_init_data") +{ const size_t msg_size = 30; - char * data = new char[msg_size]; + char *data = new char[msg_size]; - for (size_t i = 0; i < msg_size; i++){ + for (size_t i = 0; i < msg_size; i++) + { data[i] = i; } @@ -37,8 +41,8 @@ TEST_CASE("msgq_msg_init_data"){ msgq_msg_close(&msg); } - -TEST_CASE("msgq_init_subscriber"){ +TEST_CASE("msgq_init_subscriber") +{ remove("/dev/shm/test_queue"); msgq_queue_t q; msgq_new_queue(&q, "test_queue", 1024); @@ -54,10 +58,11 @@ TEST_CASE("msgq_init_subscriber"){ REQUIRE(q.read_conflate == false); REQUIRE(*q.read_valids[0] == true); REQUIRE((*q.read_pointers[0] >> 32) == 0); - REQUIRE((*q.read_pointers[0] & 0xFFFFFFFF) == 255); + REQUIRE((*q.read_pointers[0] & 0xFFFFFFFF) == 255); } -TEST_CASE("msgq_msg_send first message"){ +TEST_CASE("msgq_msg_send first message") +{ remove("/dev/shm/test_queue"); msgq_queue_t q; msgq_new_queue(&q, "test_queue", 1024); @@ -67,24 +72,25 @@ TEST_CASE("msgq_msg_send first message"){ size_t msg_size = 128; - SECTION("Aligned message size"){ + SECTION("Aligned message size") + { } - SECTION("Unaligned message size"){ + SECTION("Unaligned message size") + { msg_size--; } + char *data = new char[msg_size]; - char * data = new char[msg_size]; - - for (size_t i = 0; i < msg_size; i++){ + for (size_t i = 0; i < msg_size; i++) + { data[i] = i; } msgq_msg_t msg; msgq_msg_init_data(&msg, data, msg_size); - msgq_msg_send(&msg, &q); - REQUIRE(*(int64_t*)q.data == msg_size); // Check size tag + REQUIRE(*(int64_t *)q.data == msg_size); // Check size tag REQUIRE(*q.write_pointer == 128 + sizeof(int64_t)); REQUIRE(memcmp(q.data + sizeof(int64_t), data, msg_size) == 0); @@ -92,7 +98,8 @@ TEST_CASE("msgq_msg_send first message"){ msgq_msg_close(&msg); } -TEST_CASE("msgq_msg_send test wraparound"){ +TEST_CASE("msgq_msg_send test wraparound") +{ remove("/dev/shm/test_queue"); msgq_queue_t q; msgq_new_queue(&q, "test_queue", 1024); @@ -105,7 +112,8 @@ TEST_CASE("msgq_msg_send test wraparound"){ msgq_msg_t msg; msgq_msg_init_size(&msg, msg_size); - for (int i = 0; i < 8; i++) { + for (int i = 0; i < 8; i++) + { msgq_msg_send(&msg, &q); } // Check 8th message was written at the beginning @@ -115,14 +123,15 @@ TEST_CASE("msgq_msg_send test wraparound"){ REQUIRE((*q.write_pointer >> 32) == 1); // Check wraparound tag - char * tag_location = q.data; + char *tag_location = q.data; tag_location += 7 * (msg_size + sizeof(int64_t)); - REQUIRE(*(int64_t*)tag_location == -1); + REQUIRE(*(int64_t *)tag_location == -1); msgq_msg_close(&msg); } -TEST_CASE("msgq_msg_recv test wraparound"){ +TEST_CASE("msgq_msg_recv test wraparound") +{ remove("/dev/shm/test_queue"); msgq_queue_t q_pub, q_sub; msgq_new_queue(&q_pub, "test_queue", 1024); @@ -138,9 +147,10 @@ TEST_CASE("msgq_msg_recv test wraparound"){ msgq_msg_t msg1; msgq_msg_init_size(&msg1, msg_size); - - SECTION("Check cycle counter after reset") { - for (int i = 0; i < 8; i++) { + SECTION("Check cycle counter after reset") + { + for (int i = 0; i < 8; i++) + { msgq_msg_send(&msg1, &q_pub); } @@ -149,8 +159,10 @@ TEST_CASE("msgq_msg_recv test wraparound"){ REQUIRE(msg2.size == 0); // Reader had to reset msgq_msg_close(&msg2); } - SECTION("Check cycle counter while keeping up with writer") { - for (int i = 0; i < 8; i++) { + SECTION("Check cycle counter while keeping up with writer") + { + for (int i = 0; i < 8; i++) + { msgq_msg_send(&msg1, &q_pub); msgq_msg_t msg2; @@ -164,7 +176,8 @@ TEST_CASE("msgq_msg_recv test wraparound"){ msgq_msg_close(&msg1); } -TEST_CASE("msgq_msg_send test invalidation"){ +TEST_CASE("msgq_msg_send test invalidation") +{ remove("/dev/shm/test_queue"); msgq_queue_t q_pub, q_sub; msgq_new_queue(&q_pub, "test_queue", 1024); @@ -176,13 +189,16 @@ TEST_CASE("msgq_msg_send test invalidation"){ REQUIRE(*q_sub.read_valids[0] == true); - SECTION("read pointer in tag"){ + SECTION("read pointer in tag") + { *q_sub.read_pointers[0] = 0; } - SECTION("read pointer in data section"){ + SECTION("read pointer in data section") + { *q_sub.read_pointers[0] = 64; } - SECTION("read pointer in wraparound section"){ + SECTION("read pointer in wraparound section") + { *q_pub.write_pointer = ((uint64_t)1 << 32) | 1000; // Writer is one cycle ahead *q_sub.read_pointers[0] = 1020; } @@ -196,7 +212,8 @@ TEST_CASE("msgq_msg_send test invalidation"){ msgq_msg_close(&msg); } -TEST_CASE("msgq_init_subscriber init 2 subscribers"){ +TEST_CASE("msgq_init_subscriber init 2 subscribers") +{ remove("/dev/shm/test_queue"); msgq_queue_t q1, q2; msgq_new_queue(&q1, "test_queue", 1024); @@ -218,8 +235,8 @@ TEST_CASE("msgq_init_subscriber init 2 subscribers"){ REQUIRE(q2.reader_id == 1); } - -TEST_CASE("Write 1 msg, read 1 msg", "[integration]"){ +TEST_CASE("Write 1 msg, read 1 msg", "[integration]") +{ remove("/dev/shm/test_queue"); const size_t msg_size = 128; msgq_queue_t writer, reader; @@ -234,7 +251,8 @@ TEST_CASE("Write 1 msg, read 1 msg", "[integration]"){ msgq_msg_t outgoing_msg; msgq_msg_init_size(&outgoing_msg, msg_size); - for (size_t i = 0; i < msg_size; i++){ + for (size_t i = 0; i < msg_size; i++) + { outgoing_msg.data[i] = i; } @@ -253,7 +271,8 @@ TEST_CASE("Write 1 msg, read 1 msg", "[integration]"){ msgq_msg_close(&incoming_msg2); } -TEST_CASE("Write 2 msg, read 2 msg - conflate = false", "[integration]"){ +TEST_CASE("Write 2 msg, read 2 msg - conflate = false", "[integration]") +{ remove("/dev/shm/test_queue"); const size_t msg_size = 128; msgq_queue_t writer, reader; @@ -268,7 +287,8 @@ TEST_CASE("Write 2 msg, read 2 msg - conflate = false", "[integration]"){ msgq_msg_t outgoing_msg; msgq_msg_init_size(&outgoing_msg, msg_size); - for (size_t i = 0; i < msg_size; i++){ + for (size_t i = 0; i < msg_size; i++) + { outgoing_msg.data[i] = i; } @@ -288,7 +308,8 @@ TEST_CASE("Write 2 msg, read 2 msg - conflate = false", "[integration]"){ msgq_msg_close(&incoming_msg2); } -TEST_CASE("Write 2 msg, read 2 msg - conflate = true", "[integration]"){ +TEST_CASE("Write 2 msg, read 2 msg - conflate = true", "[integration]") +{ remove("/dev/shm/test_queue"); const size_t msg_size = 128; msgq_queue_t writer, reader; @@ -304,7 +325,8 @@ TEST_CASE("Write 2 msg, read 2 msg - conflate = true", "[integration]"){ msgq_msg_t outgoing_msg; msgq_msg_init_size(&outgoing_msg, msg_size); - for (size_t i = 0; i < msg_size; i++){ + for (size_t i = 0; i < msg_size; i++) + { outgoing_msg.data[i] = i; } @@ -324,7 +346,8 @@ TEST_CASE("Write 2 msg, read 2 msg - conflate = true", "[integration]"){ msgq_msg_close(&incoming_msg2); } -TEST_CASE("1 publisher, 1 slow subscriber", "[integration]"){ +TEST_CASE("1 publisher, 1 slow subscriber", "[integration]") +{ remove("/dev/shm/test_queue"); msgq_queue_t writer, reader; @@ -337,19 +360,24 @@ TEST_CASE("1 publisher, 1 slow subscriber", "[integration]"){ int n_received = 0; int n_skipped = 0; - for (uint64_t i = 0; i < 1e5; i++) { + for (uint64_t i = 0; i < 1e5; i++) + { msgq_msg_t outgoing_msg; - msgq_msg_init_data(&outgoing_msg, (char*)&i, sizeof(uint64_t)); + msgq_msg_init_data(&outgoing_msg, (char *)&i, sizeof(uint64_t)); msgq_msg_send(&outgoing_msg, &writer); msgq_msg_close(&outgoing_msg); - if (i % 10 == 0){ + if (i % 10 == 0) + { msgq_msg_t msg1; msgq_msg_recv(&msg1, &reader); - if (msg1.size == 0){ + if (msg1.size == 0) + { n_skipped++; - } else { + } + else + { n_received++; } msgq_msg_close(&msg1); @@ -361,7 +389,8 @@ TEST_CASE("1 publisher, 1 slow subscriber", "[integration]"){ REQUIRE(n_skipped == 1428); } -TEST_CASE("1 publisher, 2 subscribers", "[integration]"){ +TEST_CASE("1 publisher, 2 subscribers", "[integration]") +{ remove("/dev/shm/test_queue"); msgq_queue_t writer, reader1, reader2; @@ -373,9 +402,10 @@ TEST_CASE("1 publisher, 2 subscribers", "[integration]"){ msgq_init_subscriber(&reader1); msgq_init_subscriber(&reader2); - for (uint64_t i = 0; i < 1024 * 3; i++) { + for (uint64_t i = 0; i < 1024 * 3; i++) + { msgq_msg_t outgoing_msg; - msgq_msg_init_data(&outgoing_msg, (char*)&i, sizeof(uint64_t)); + msgq_msg_init_data(&outgoing_msg, (char *)&i, sizeof(uint64_t)); msgq_msg_send(&outgoing_msg, &writer); msgq_msg_t msg1, msg2; @@ -384,8 +414,8 @@ TEST_CASE("1 publisher, 2 subscribers", "[integration]"){ REQUIRE(msg1.size == sizeof(uint64_t)); REQUIRE(msg2.size == sizeof(uint64_t)); - REQUIRE(*(uint64_t*)msg1.data == i); - REQUIRE(*(uint64_t*)msg2.data == i); + REQUIRE(*(uint64_t *)msg1.data == i); + REQUIRE(*(uint64_t *)msg2.data == i); msgq_msg_close(&outgoing_msg); msgq_msg_close(&msg1); diff --git a/messaging/socketmaster.cc b/messaging/socketmaster.cc deleted file mode 100644 index 3054b4f7d..000000000 --- a/messaging/socketmaster.cc +++ /dev/null @@ -1,210 +0,0 @@ -#include -#include -#include -#include -#include - -#include "cereal/services.h" -#include "cereal/messaging/messaging.h" - -const bool SIMULATION = (getenv("SIMULATION") != nullptr) && (std::string(getenv("SIMULATION")) == "1"); - -static inline uint64_t nanos_since_boot() { - struct timespec t; - clock_gettime(CLOCK_BOOTTIME, &t); - return t.tv_sec * 1000000000ULL + t.tv_nsec; -} - -static inline bool inList(const std::vector &list, const char *value) { - for (auto &v : list) { - if (strcmp(value, v) == 0) return true; - } - return false; -} - -class MessageContext { -public: - MessageContext() : ctx_(nullptr) {} - ~MessageContext() { delete ctx_; } - inline Context *context() { - std::call_once(init_flag, [=]() { ctx_ = Context::create(); }); - return ctx_; - } -private: - Context *ctx_; - std::once_flag init_flag; -}; - -MessageContext message_context; - -struct SubMaster::SubMessage { - std::string name; - SubSocket *socket = nullptr; - int freq = 0; - bool updated = false, alive = false, valid = true, ignore_alive; - uint64_t rcv_time = 0, rcv_frame = 0; - void *allocated_msg_reader = nullptr; - bool is_polled = false; - capnp::FlatArrayMessageReader *msg_reader = nullptr; - AlignedBuffer aligned_buf; - cereal::Event::Reader event; -}; - -SubMaster::SubMaster(const std::vector &service_list, const std::vector &poll, - const char *address, const std::vector &ignore_alive) { - poller_ = Poller::create(); - for (auto name : service_list) { - assert(services.count(std::string(name)) > 0); - - service serv = services.at(std::string(name)); - SubSocket *socket = SubSocket::create(message_context.context(), name, address ? address : "127.0.0.1", true); - assert(socket != 0); - bool is_polled = inList(poll, name) || poll.empty(); - if (is_polled) poller_->registerSocket(socket); - SubMessage *m = new SubMessage{ - .name = name, - .socket = socket, - .freq = serv.frequency, - .ignore_alive = inList(ignore_alive, name), - .allocated_msg_reader = malloc(sizeof(capnp::FlatArrayMessageReader)), - .is_polled = is_polled}; - m->msg_reader = new (m->allocated_msg_reader) capnp::FlatArrayMessageReader({}); - messages_[socket] = m; - services_[name] = m; - } -} - -void SubMaster::update(int timeout) { - for (auto &kv : messages_) kv.second->updated = false; - - auto sockets = poller_->poll(timeout); - - // add non-polled sockets for non-blocking receive - for (auto &kv : messages_) { - SubMessage *m = kv.second; - SubSocket *s = kv.first; - if (!m->is_polled) sockets.push_back(s); - } - - uint64_t current_time = nanos_since_boot(); - - std::vector> messages; - - for (auto s : sockets) { - Message *msg = s->receive(true); - if (msg == nullptr) continue; - - SubMessage *m = messages_.at(s); - - m->msg_reader->~FlatArrayMessageReader(); - capnp::ReaderOptions options; - options.traversalLimitInWords = kj::maxValue; // Don't limit - m->msg_reader = new (m->allocated_msg_reader) capnp::FlatArrayMessageReader(m->aligned_buf.align(msg), options); - delete msg; - messages.push_back({m->name, m->msg_reader->getRoot()}); - } - - update_msgs(current_time, messages); -} - -void SubMaster::update_msgs(uint64_t current_time, const std::vector> &messages){ - if (++frame == UINT64_MAX) frame = 1; - - for (auto &kv : messages) { - auto m_find = services_.find(kv.first); - if (m_find == services_.end()){ - continue; - } - SubMessage *m = m_find->second; - m->event = kv.second; - m->updated = true; - m->rcv_time = current_time; - m->rcv_frame = frame; - m->valid = m->event.getValid(); - if (SIMULATION) m->alive = true; - } - - if (!SIMULATION) { - for (auto &kv : messages_) { - SubMessage *m = kv.second; - m->alive = (m->freq <= (1e-5) || ((current_time - m->rcv_time) * (1e-9)) < (10.0 / m->freq)); - } - } -} - -bool SubMaster::all_(const std::vector &service_list, bool valid, bool alive) { - int found = 0; - for (auto &kv : messages_) { - SubMessage *m = kv.second; - if (service_list.size() == 0 || inList(service_list, m->name.c_str())) { - found += (!valid || m->valid) && (!alive || (m->alive || m->ignore_alive)); - } - } - return service_list.size() == 0 ? found == messages_.size() : found == service_list.size(); -} - -void SubMaster::drain() { - while (true) { - auto polls = poller_->poll(0); - if (polls.size() == 0) - break; - - for (auto sock : polls) { - Message *msg = sock->receive(true); - delete msg; - } - } -} - -bool SubMaster::updated(const char *name) const { - return services_.at(name)->updated; -} - -bool SubMaster::alive(const char *name) const { - return services_.at(name)->alive; -} - -bool SubMaster::valid(const char *name) const { - return services_.at(name)->valid; -} - -uint64_t SubMaster::rcv_frame(const char *name) const { - return services_.at(name)->rcv_frame; -} - -uint64_t SubMaster::rcv_time(const char *name) const { - return services_.at(name)->rcv_time; -} - -cereal::Event::Reader &SubMaster::operator[](const char *name) const { - return services_.at(name)->event; -} - -SubMaster::~SubMaster() { - delete poller_; - for (auto &kv : messages_) { - SubMessage *m = kv.second; - m->msg_reader->~FlatArrayMessageReader(); - free(m->allocated_msg_reader); - delete m->socket; - delete m; - } -} - -PubMaster::PubMaster(const std::vector &service_list) { - for (auto name : service_list) { - assert(services.count(name) > 0); - PubSocket *socket = PubSocket::create(message_context.context(), name); - assert(socket); - sockets_[name] = socket; - } -} - -int PubMaster::send(const char *name, MessageBuilder &msg) { - auto bytes = msg.toBytes(); - return send(name, bytes.begin(), bytes.size()); -} - -PubMaster::~PubMaster() { - for (auto s : sockets_) delete s.second; -} diff --git a/messaging/stress.py b/messaging/stress.py deleted file mode 100644 index 1a27e520e..000000000 --- a/messaging/stress.py +++ /dev/null @@ -1,14 +0,0 @@ -from messaging_pyx import Context, SubSocket, PubSocket - -if __name__ == "__main__": - c = Context() - pub_sock = PubSocket() - pub_sock.connect(c, "controlsState") - - for i in range(int(1e10)): - print(i) - sub_sock = SubSocket() - sub_sock.connect(c, "controlsState") - - pub_sock.send(b'a') - print(sub_sock.receive()) diff --git a/messaging/tests/__init__.py b/messaging/tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/messaging/tests/test_fake.py b/messaging/tests/test_fake.py deleted file mode 100644 index 1d3521745..000000000 --- a/messaging/tests/test_fake.py +++ /dev/null @@ -1,193 +0,0 @@ -import os -import unittest -import multiprocessing -import platform -from parameterized import parameterized_class -from typing import Optional - -import cereal.messaging as messaging - -WAIT_TIMEOUT = 5 - - -@unittest.skipIf(platform.system() == "Darwin", "Events not supported on macOS") -class TestEvents(unittest.TestCase): - - def test_mutation(self): - handle = messaging.fake_event_handle("carState") - event = handle.recv_called_event - - self.assertFalse(event.peek()) - event.set() - self.assertTrue(event.peek()) - event.clear() - self.assertFalse(event.peek()) - - del event - - def test_wait(self): - handle = messaging.fake_event_handle("carState") - event = handle.recv_called_event - - event.set() - try: - event.wait(WAIT_TIMEOUT) - self.assertTrue(event.peek()) - except RuntimeError: - self.fail("event.wait() timed out") - - def test_wait_multiprocess(self): - handle = messaging.fake_event_handle("carState") - event = handle.recv_called_event - - def set_event_run(): - event.set() - - try: - p = multiprocessing.Process(target=set_event_run) - p.start() - event.wait(WAIT_TIMEOUT) - self.assertTrue(event.peek()) - except RuntimeError: - self.fail("event.wait() timed out") - - p.kill() - - def test_wait_zero_timeout(self): - handle = messaging.fake_event_handle("carState") - event = handle.recv_called_event - - try: - event.wait(0) - self.fail("event.wait() did not time out") - except RuntimeError: - self.assertFalse(event.peek()) - - -@unittest.skipIf(platform.system() == "Darwin", "FakeSockets not supported on macOS") -@unittest.skipIf("ZMQ" in os.environ, "FakeSockets not supported on ZMQ") -@parameterized_class([{"prefix": None}, {"prefix": "test"}]) -class TestFakeSockets(unittest.TestCase): - prefix: Optional[str] = None - - def setUp(self): - messaging.toggle_fake_events(True) - if self.prefix is not None: - messaging.set_fake_prefix(self.prefix) - else: - messaging.delete_fake_prefix() - - def tearDown(self): - messaging.toggle_fake_events(False) - messaging.delete_fake_prefix() - - def test_event_handle_init(self): - handle = messaging.fake_event_handle("controlsState", override=True) - - self.assertFalse(handle.enabled) - self.assertGreaterEqual(handle.recv_called_event.fd, 0) - self.assertGreaterEqual(handle.recv_ready_event.fd, 0) - - def test_non_managed_socket_state(self): - # non managed socket should have zero state - _ = messaging.pub_sock("ubloxGnss") - - handle = messaging.fake_event_handle("ubloxGnss", override=False) - - self.assertFalse(handle.enabled) - self.assertEqual(handle.recv_called_event.fd, 0) - self.assertEqual(handle.recv_ready_event.fd, 0) - - def test_managed_socket_state(self): - # managed socket should not change anything about the state - handle = messaging.fake_event_handle("ubloxGnss") - handle.enabled = True - - expected_enabled = handle.enabled - expected_recv_called_fd = handle.recv_called_event.fd - expected_recv_ready_fd = handle.recv_ready_event.fd - - _ = messaging.pub_sock("ubloxGnss") - - self.assertEqual(handle.enabled, expected_enabled) - self.assertEqual(handle.recv_called_event.fd, expected_recv_called_fd) - self.assertEqual(handle.recv_ready_event.fd, expected_recv_ready_fd) - - def test_sockets_enable_disable(self): - carState_handle = messaging.fake_event_handle("ubloxGnss", enable=True) - recv_called = carState_handle.recv_called_event - recv_ready = carState_handle.recv_ready_event - - pub_sock = messaging.pub_sock("ubloxGnss") - sub_sock = messaging.sub_sock("ubloxGnss") - - try: - carState_handle.enabled = True - recv_ready.set() - pub_sock.send(b"test") - _ = sub_sock.receive() - self.assertTrue(recv_called.peek()) - recv_called.clear() - - carState_handle.enabled = False - recv_ready.set() - pub_sock.send(b"test") - _ = sub_sock.receive() - self.assertFalse(recv_called.peek()) - except RuntimeError: - self.fail("event.wait() timed out") - - def test_synced_pub_sub(self): - def daemon_repub_process_run(): - pub_sock = messaging.pub_sock("ubloxGnss") - sub_sock = messaging.sub_sock("carState") - - frame = -1 - while True: - frame += 1 - msg = sub_sock.receive(non_blocking=True) - if msg is None: - print("none received") - continue - - bts = frame.to_bytes(8, 'little') - pub_sock.send(bts) - - carState_handle = messaging.fake_event_handle("carState", enable=True) - recv_called = carState_handle.recv_called_event - recv_ready = carState_handle.recv_ready_event - - p = multiprocessing.Process(target=daemon_repub_process_run) - p.start() - - pub_sock = messaging.pub_sock("carState") - sub_sock = messaging.sub_sock("ubloxGnss") - - try: - for i in range(10): - recv_called.wait(WAIT_TIMEOUT) - recv_called.clear() - - if i == 0: - sub_sock.receive(non_blocking=True) - - bts = i.to_bytes(8, 'little') - pub_sock.send(bts) - - recv_ready.set() - recv_called.wait(WAIT_TIMEOUT) - - msg = sub_sock.receive(non_blocking=True) - self.assertIsNotNone(msg) - self.assertEqual(len(msg), 8) - - frame = int.from_bytes(msg, 'little') - self.assertEqual(frame, i) - except RuntimeError: - self.fail("event.wait() timed out") - finally: - p.kill() - - -if __name__ == "__main__": - unittest.main() diff --git a/messaging/tests/test_messaging.py b/messaging/tests/test_messaging.py deleted file mode 100755 index fdebc5b26..000000000 --- a/messaging/tests/test_messaging.py +++ /dev/null @@ -1,247 +0,0 @@ -#!/usr/bin/env python3 -import os -import capnp -import multiprocessing -import numbers -import random -import threading -import time -import unittest -from parameterized import parameterized - -from cereal import log, car -import cereal.messaging as messaging -from cereal.services import SERVICE_LIST - -events = [evt for evt in log.Event.schema.union_fields if evt in SERVICE_LIST.keys()] - -def random_sock(): - return random.choice(events) - -def random_socks(num_socks=10): - return list({random_sock() for _ in range(num_socks)}) - -def random_bytes(length=1000): - return bytes([random.randrange(0xFF) for _ in range(length)]) - -def zmq_sleep(t=1): - if "ZMQ" in os.environ: - time.sleep(t) - -def zmq_expected_failure(func): - if "ZMQ" in os.environ: - return unittest.expectedFailure(func) - else: - return func - -# TODO: this should take any capnp struct and returrn a msg with random populated data -def random_carstate(): - fields = ["vEgo", "aEgo", "gas", "steeringAngleDeg"] - msg = messaging.new_message("carState") - cs = msg.carState - for f in fields: - setattr(cs, f, random.random() * 10) - return msg - -# TODO: this should compare any capnp structs -def assert_carstate(cs1, cs2): - for f in car.CarState.schema.non_union_fields: - # TODO: check all types - val1, val2 = getattr(cs1, f), getattr(cs2, f) - if isinstance(val1, numbers.Number): - assert val1 == val2, f"{f}: sent '{val1}' vs recvd '{val2}'" - -def delayed_send(delay, sock, dat): - def send_func(): - sock.send(dat) - threading.Timer(delay, send_func).start() - -class TestPubSubSockets(unittest.TestCase): - - def setUp(self): - # ZMQ pub socket takes too long to die - # sleep to prevent multiple publishers error between tests - zmq_sleep() - - def test_pub_sub(self): - sock = random_sock() - pub_sock = messaging.pub_sock(sock) - sub_sock = messaging.sub_sock(sock, conflate=False, timeout=None) - zmq_sleep(3) - - for _ in range(1000): - msg = random_bytes() - pub_sock.send(msg) - recvd = sub_sock.receive() - self.assertEqual(msg, recvd) - - def test_conflate(self): - sock = random_sock() - pub_sock = messaging.pub_sock(sock) - for conflate in [True, False]: - for _ in range(10): - num_msgs = random.randint(3, 10) - sub_sock = messaging.sub_sock(sock, conflate=conflate, timeout=None) - zmq_sleep() - - sent_msgs = [] - for __ in range(num_msgs): - msg = random_bytes() - pub_sock.send(msg) - sent_msgs.append(msg) - time.sleep(0.1) - recvd_msgs = messaging.drain_sock_raw(sub_sock) - if conflate: - self.assertEqual(len(recvd_msgs), 1) - else: - # TODO: compare actual data - self.assertEqual(len(recvd_msgs), len(sent_msgs)) - - def test_receive_timeout(self): - sock = random_sock() - for _ in range(10): - timeout = random.randrange(200) - sub_sock = messaging.sub_sock(sock, timeout=timeout) - zmq_sleep() - - start_time = time.monotonic() - recvd = sub_sock.receive() - self.assertLess(time.monotonic() - start_time, 0.2) - assert recvd is None - -class TestMessaging(unittest.TestCase): - - def setUp(self): - # TODO: ZMQ tests are too slow; all sleeps will need to be - # replaced with logic to block on the necessary condition - if "ZMQ" in os.environ: - raise unittest.SkipTest - - # ZMQ pub socket takes too long to die - # sleep to prevent multiple publishers error between tests - zmq_sleep() - - @parameterized.expand(events) - def test_new_message(self, evt): - try: - msg = messaging.new_message(evt) - except capnp.lib.capnp.KjException: - msg = messaging.new_message(evt, random.randrange(200)) - self.assertLess(time.monotonic() - msg.logMonoTime, 0.1) - self.assertFalse(msg.valid) - self.assertEqual(evt, msg.which()) - - @parameterized.expand(events) - def test_pub_sock(self, evt): - messaging.pub_sock(evt) - - @parameterized.expand(events) - def test_sub_sock(self, evt): - messaging.sub_sock(evt) - - @parameterized.expand([ - (messaging.drain_sock, capnp._DynamicStructReader), - (messaging.drain_sock_raw, bytes), - ]) - def test_drain_sock(self, func, expected_type): - sock = "carState" - pub_sock = messaging.pub_sock(sock) - sub_sock = messaging.sub_sock(sock, timeout=1000) - zmq_sleep() - - # no wait and no msgs in queue - msgs = func(sub_sock) - self.assertIsInstance(msgs, list) - self.assertEqual(len(msgs), 0) - - # no wait but msgs are queued up - num_msgs = random.randrange(3, 10) - for _ in range(num_msgs): - pub_sock.send(messaging.new_message(sock).to_bytes()) - time.sleep(0.1) - msgs = func(sub_sock) - self.assertIsInstance(msgs, list) - self.assertTrue(all(isinstance(msg, expected_type) for msg in msgs)) - self.assertEqual(len(msgs), num_msgs) - - def test_recv_sock(self): - sock = "carState" - pub_sock = messaging.pub_sock(sock) - sub_sock = messaging.sub_sock(sock, timeout=100) - zmq_sleep() - - # no wait and no msg in queue, socket should timeout - recvd = messaging.recv_sock(sub_sock) - self.assertTrue(recvd is None) - - # no wait and one msg in queue - msg = random_carstate() - pub_sock.send(msg.to_bytes()) - time.sleep(0.01) - recvd = messaging.recv_sock(sub_sock) - self.assertIsInstance(recvd, capnp._DynamicStructReader) - # https://github.com/python/mypy/issues/13038 - assert_carstate(msg.carState, recvd.carState) # type: ignore[union-attr] - - def test_recv_one(self): - sock = "carState" - pub_sock = messaging.pub_sock(sock) - sub_sock = messaging.sub_sock(sock, timeout=1000) - zmq_sleep() - - # no msg in queue, socket should timeout - recvd = messaging.recv_one(sub_sock) - self.assertTrue(recvd is None) - - # one msg in queue - msg = random_carstate() - pub_sock.send(msg.to_bytes()) - recvd = messaging.recv_one(sub_sock) - self.assertIsInstance(recvd, capnp._DynamicStructReader) - assert_carstate(msg.carState, recvd.carState) # type: ignore[union-attr] - - @zmq_expected_failure - def test_recv_one_or_none(self): - sock = "carState" - pub_sock = messaging.pub_sock(sock) - sub_sock = messaging.sub_sock(sock) - zmq_sleep() - - # no msg in queue, socket shouldn't block - recvd = messaging.recv_one_or_none(sub_sock) - self.assertTrue(recvd is None) - - # one msg in queue - msg = random_carstate() - pub_sock.send(msg.to_bytes()) - recvd = messaging.recv_one_or_none(sub_sock) - self.assertIsInstance(recvd, capnp._DynamicStructReader) - assert_carstate(msg.carState, recvd.carState) # type: ignore[union-attr] - - def test_recv_one_retry(self): - sock = "carState" - sock_timeout = 0.1 - pub_sock = messaging.pub_sock(sock) - sub_sock = messaging.sub_sock(sock, timeout=round(sock_timeout*1000)) - zmq_sleep() - - # this test doesn't work with ZMQ since multiprocessing interrupts it - if "ZMQ" not in os.environ: - # wait 15 socket timeouts and make sure it's still retrying - p = multiprocessing.Process(target=messaging.recv_one_retry, args=(sub_sock,)) - p.start() - time.sleep(sock_timeout*15) - self.assertTrue(p.is_alive()) - p.terminate() - - # wait 15 socket timeouts before sending - msg = random_carstate() - delayed_send(sock_timeout*15, pub_sock, msg.to_bytes()) - start_time = time.monotonic() - recvd = messaging.recv_one_retry(sub_sock) - self.assertGreaterEqual(time.monotonic() - start_time, sock_timeout*15) - self.assertIsInstance(recvd, capnp._DynamicStructReader) - assert_carstate(msg.carState, recvd.carState) - -if __name__ == "__main__": - unittest.main() diff --git a/messaging/tests/test_poller.py b/messaging/tests/test_poller.py deleted file mode 100644 index bcff5e40c..000000000 --- a/messaging/tests/test_poller.py +++ /dev/null @@ -1,142 +0,0 @@ -import unittest -import time -import cereal.messaging as messaging - -import concurrent.futures - - -def poller(): - context = messaging.Context() - - p = messaging.Poller() - - sub = messaging.SubSocket() - sub.connect(context, 'controlsState') - p.registerSocket(sub) - - socks = p.poll(10000) - r = [s.receive(non_blocking=True) for s in socks] - - return r - - -class TestPoller(unittest.TestCase): - def test_poll_once(self): - context = messaging.Context() - - pub = messaging.PubSocket() - pub.connect(context, 'controlsState') - - with concurrent.futures.ThreadPoolExecutor() as e: - poll = e.submit(poller) - - time.sleep(0.1) # Slow joiner syndrome - - # Send message - pub.send(b"a") - - # Wait for poll result - result = poll.result() - - del pub - context.term() - - self.assertEqual(result, [b"a"]) - - def test_poll_and_create_many_subscribers(self): - context = messaging.Context() - - pub = messaging.PubSocket() - pub.connect(context, 'controlsState') - - with concurrent.futures.ThreadPoolExecutor() as e: - poll = e.submit(poller) - - time.sleep(0.1) # Slow joiner syndrome - c = messaging.Context() - for _ in range(10): - messaging.SubSocket().connect(c, 'controlsState') - - time.sleep(0.1) - - # Send message - pub.send(b"a") - - # Wait for poll result - result = poll.result() - - del pub - context.term() - - self.assertEqual(result, [b"a"]) - - def test_multiple_publishers_exception(self): - context = messaging.Context() - - with self.assertRaises(messaging.MultiplePublishersError): - pub1 = messaging.PubSocket() - pub1.connect(context, 'controlsState') - - pub2 = messaging.PubSocket() - pub2.connect(context, 'controlsState') - - pub1.send(b"a") - - del pub1 - del pub2 - context.term() - - def test_multiple_messages(self): - context = messaging.Context() - - pub = messaging.PubSocket() - pub.connect(context, 'controlsState') - - sub = messaging.SubSocket() - sub.connect(context, 'controlsState') - - time.sleep(0.1) # Slow joiner - - for i in range(1, 100): - pub.send(b'a'*i) - - msg_seen = False - i = 1 - while True: - r = sub.receive(non_blocking=True) - - if r is not None: - self.assertEqual(b'a'*i, r) - - msg_seen = True - i += 1 - - if r is None and msg_seen: # ZMQ sometimes receives nothing on the first receive - break - - del pub - del sub - context.term() - - def test_conflate(self): - context = messaging.Context() - - pub = messaging.PubSocket() - pub.connect(context, 'controlsState') - - sub = messaging.SubSocket() - sub.connect(context, 'controlsState', conflate=True) - - time.sleep(0.1) # Slow joiner - pub.send(b'a') - pub.send(b'b') - - self.assertEqual(b'b', sub.receive()) - - del pub - del sub - context.term() - - -if __name__ == "__main__": - unittest.main() diff --git a/messaging/tests/test_pub_sub_master.py b/messaging/tests/test_pub_sub_master.py deleted file mode 100755 index 81a1cf2d5..000000000 --- a/messaging/tests/test_pub_sub_master.py +++ /dev/null @@ -1,163 +0,0 @@ -#!/usr/bin/env python3 -import random -import time -from typing import Sized, cast -import unittest - -import cereal.messaging as messaging -from cereal.messaging.tests.test_messaging import events, random_sock, random_socks, \ - random_bytes, random_carstate, assert_carstate, \ - zmq_sleep - - -class TestSubMaster(unittest.TestCase): - - def setUp(self): - # ZMQ pub socket takes too long to die - # sleep to prevent multiple publishers error between tests - zmq_sleep(3) - - def test_init(self): - sm = messaging.SubMaster(events) - for p in [sm.updated, sm.recv_time, sm.recv_frame, sm.alive, - sm.sock, sm.data, sm.logMonoTime, sm.valid]: - self.assertEqual(len(cast(Sized, p)), len(events)) - - def test_init_state(self): - socks = random_socks() - sm = messaging.SubMaster(socks) - self.assertEqual(sm.frame, -1) - self.assertFalse(any(sm.updated.values())) - self.assertFalse(any(sm.alive.values())) - self.assertTrue(all(t == 0. for t in sm.recv_time.values())) - self.assertTrue(all(f == 0 for f in sm.recv_frame.values())) - self.assertTrue(all(t == 0 for t in sm.logMonoTime.values())) - - for p in [sm.updated, sm.recv_time, sm.recv_frame, sm.alive, - sm.sock, sm.data, sm.logMonoTime, sm.valid]: - self.assertEqual(len(cast(Sized, p)), len(socks)) - - def test_getitem(self): - sock = "carState" - pub_sock = messaging.pub_sock(sock) - sm = messaging.SubMaster([sock,]) - zmq_sleep() - - msg = random_carstate() - pub_sock.send(msg.to_bytes()) - sm.update(1000) - assert_carstate(msg.carState, sm[sock]) - - # TODO: break this test up to individually test SubMaster.update and SubMaster.update_msgs - def test_update(self): - sock = "carState" - pub_sock = messaging.pub_sock(sock) - sm = messaging.SubMaster([sock,]) - zmq_sleep() - - for i in range(10): - msg = messaging.new_message(sock) - pub_sock.send(msg.to_bytes()) - sm.update(1000) - self.assertEqual(sm.frame, i) - self.assertTrue(all(sm.updated.values())) - - def test_update_timeout(self): - sock = random_sock() - sm = messaging.SubMaster([sock,]) - for _ in range(5): - timeout = random.randrange(1000, 5000) - start_time = time.monotonic() - sm.update(timeout) - t = time.monotonic() - start_time - self.assertGreaterEqual(t, timeout/1000.) - self.assertLess(t, 5) - self.assertFalse(any(sm.updated.values())) - - def test_avg_frequency_checks(self): - for poll in (True, False): - sm = messaging.SubMaster(["modelV2", "carParams", "carState", "cameraOdometry", "liveCalibration"], - poll=("modelV2" if poll else None), - frequency=(20. if not poll else None)) - - checks = { - "carState": (20, 20), - "modelV2": (20, 20 if poll else 10), - "cameraOdometry": (20, 10), - "liveCalibration": (4, 4), - "carParams": (None, None), - } - - for service, (max_freq, min_freq) in checks.items(): - if max_freq is not None: - assert sm._check_avg_freq(service) - assert sm.max_freq[service] == max_freq*1.2 - assert sm.min_freq[service] == min_freq*0.8 - else: - assert not sm._check_avg_freq(service) - - def test_alive(self): - pass - - def test_ignore_alive(self): - pass - - def test_valid(self): - pass - - # SubMaster should always conflate - def test_conflate(self): - sock = "carState" - pub_sock = messaging.pub_sock(sock) - sm = messaging.SubMaster([sock,]) - - n = 10 - for i in range(n+1): - msg = messaging.new_message(sock) - msg.carState.vEgo = i - pub_sock.send(msg.to_bytes()) - time.sleep(0.01) - sm.update(1000) - self.assertEqual(sm[sock].vEgo, n) - - -class TestPubMaster(unittest.TestCase): - - def setUp(self): - # ZMQ pub socket takes too long to die - # sleep to prevent multiple publishers error between tests - zmq_sleep(3) - - def test_init(self): - messaging.PubMaster(events) - - def test_send(self): - socks = random_socks() - pm = messaging.PubMaster(socks) - sub_socks = {s: messaging.sub_sock(s, conflate=True, timeout=1000) for s in socks} - zmq_sleep() - - # PubMaster accepts either a capnp msg builder or bytes - for capnp in [True, False]: - for i in range(100): - sock = socks[i % len(socks)] - - if capnp: - try: - msg = messaging.new_message(sock) - except Exception: - msg = messaging.new_message(sock, random.randrange(50)) - else: - msg = random_bytes() - - pm.send(sock, msg) - recvd = sub_socks[sock].receive() - - if capnp: - msg.clear_write_flag() - msg = msg.to_bytes() - self.assertEqual(msg, recvd, i) - - -if __name__ == "__main__": - unittest.main() diff --git a/messaging/tests/test_services.py b/messaging/tests/test_services.py deleted file mode 100755 index f26bdbc5f..000000000 --- a/messaging/tests/test_services.py +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env python3 -import os -import tempfile -from typing import Dict -import unittest -from parameterized import parameterized - -import cereal.services as services -from cereal.services import SERVICE_LIST, RESERVED_PORT, STARTING_PORT - - -class TestServices(unittest.TestCase): - - @parameterized.expand(SERVICE_LIST.keys()) - def test_services(self, s): - service = SERVICE_LIST[s] - self.assertTrue(service.port != RESERVED_PORT) - self.assertTrue(service.port >= STARTING_PORT) - self.assertTrue(service.frequency <= 104) - - def test_no_duplicate_port(self): - ports: Dict[int, str] = {} - for name, service in SERVICE_LIST.items(): - self.assertFalse(service.port in ports.keys(), f"duplicate port {service.port}") - ports[service.port] = name - - def test_generated_header(self): - with tempfile.NamedTemporaryFile(suffix=".h") as f: - ret = os.system(f"python3 {services.__file__} > {f.name} && clang++ {f.name}") - self.assertEqual(ret, 0, "generated services header is not valid C") - -if __name__ == "__main__": - unittest.main() diff --git a/pyproject.toml b/pyproject.toml deleted file mode 100644 index 73928614e..000000000 --- a/pyproject.toml +++ /dev/null @@ -1,21 +0,0 @@ -# https://beta.ruff.rs/docs/configuration/#using-pyprojecttoml -[tool.ruff] -lint.select = ["E", "F", "W", "PIE", "C4", "ISC", "RUF100", "A"] -lint.ignore = ["W292", "E741", "E402", "C408", "ISC003"] -lint.flake8-implicit-str-concat.allow-multiline=false - -line-length = 160 -target-version="py311" - -[mypy.tool] -# third-party packages -ignore_missing_imports=true - -# helpful warnings -warn_redundant_casts=true -warn_unreachable=true -warn_unused_ignores=true - -# restrict dynamic typing -warn_return_any=true -check_untyped_defs=true diff --git a/services.py b/services.py deleted file mode 100755 index 22c035262..000000000 --- a/services.py +++ /dev/null @@ -1,127 +0,0 @@ -#!/usr/bin/env python3 -from typing import Optional - -RESERVED_PORT = 8022 # sshd -STARTING_PORT = 8001 - - -def new_port(port: int): - port += STARTING_PORT - return port + 1 if port >= RESERVED_PORT else port - - -class Service: - def __init__(self, port: int, should_log: bool, frequency: float, decimation: Optional[int] = None): - self.port = port - self.should_log = should_log - self.frequency = frequency - self.decimation = decimation - - -_services: dict[str, tuple] = { - # service: (should_log, frequency, qlog decimation (optional)) - # note: the "EncodeIdx" packets will still be in the log - "gyroscope": (True, 104., 104), - "gyroscope2": (True, 100., 100), - "accelerometer": (True, 104., 104), - "accelerometer2": (True, 100., 100), - "magnetometer": (True, 25., 25), - "lightSensor": (True, 100., 100), - "temperatureSensor": (True, 2., 200), - "temperatureSensor2": (True, 2., 200), - "gpsNMEA": (True, 9.), - "deviceState": (True, 2., 1), - "can": (True, 100., 1223), # decimation gives ~5 msgs in a full segment - "controlsState": (True, 100., 10), - "pandaStates": (True, 10., 1), - "peripheralState": (True, 2., 1), - "radarState": (True, 20., 5), - "roadEncodeIdx": (False, 20., 1), - "liveTracks": (True, 20.), - "sendcan": (True, 100., 139), - "logMessage": (True, 0.), - "errorLogMessage": (True, 0., 1), - "liveCalibration": (True, 4., 4), - "liveTorqueParameters": (True, 4., 1), - "androidLog": (True, 0.), - "carState": (True, 100., 10), - "carControl": (True, 100., 10), - "carOutput": (True, 100., 10), - "longitudinalPlan": (True, 20., 5), - "procLog": (True, 0.5, 15), - "gpsLocationExternal": (True, 10., 10), - "gpsLocation": (True, 1., 1), - "ubloxGnss": (True, 10.), - "qcomGnss": (True, 2.), - "gnssMeasurements": (True, 10., 10), - "clocks": (True, 0.1, 1), - "ubloxRaw": (True, 20.), - "liveLocationKalman": (True, 20., 5), - "liveParameters": (True, 20., 5), - "cameraOdometry": (True, 20., 5), - "thumbnail": (True, 0.2, 1), - "onroadEvents": (True, 1., 1), - "carParams": (True, 0.02, 1), - "roadCameraState": (True, 20., 20), - "driverCameraState": (True, 20., 20), - "driverEncodeIdx": (False, 20., 1), - "driverStateV2": (True, 20., 10), - "driverMonitoringState": (True, 20., 10), - "wideRoadEncodeIdx": (False, 20., 1), - "wideRoadCameraState": (True, 20., 20), - "modelV2": (True, 20., 40), - "managerState": (True, 2., 1), - "uploaderState": (True, 0., 1), - "navInstruction": (True, 1., 10), - "navRoute": (True, 0.), - "navThumbnail": (True, 0.), - "uiPlan": (True, 20., 40.), - "qRoadEncodeIdx": (False, 20.), - "userFlag": (True, 0., 1), - "microphone": (True, 10., 10), - - # debug - "uiDebug": (True, 0., 1), - "testJoystick": (True, 0.), - "roadEncodeData": (False, 20.), - "driverEncodeData": (False, 20.), - "wideRoadEncodeData": (False, 20.), - "qRoadEncodeData": (False, 20.), - "livestreamWideRoadEncodeIdx": (False, 20.), - "livestreamRoadEncodeIdx": (False, 20.), - "livestreamDriverEncodeIdx": (False, 20.), - "livestreamWideRoadEncodeData": (False, 20.), - "livestreamRoadEncodeData": (False, 20.), - "livestreamDriverEncodeData": (False, 20.), - "customReservedRawData0": (True, 0.), - "customReservedRawData1": (True, 0.), - "customReservedRawData2": (True, 0.), -} -SERVICE_LIST = {name: Service(new_port(idx), *vals) for - idx, (name, vals) in enumerate(_services.items())} - - -def build_header(): - h = "" - h += "/* THIS IS AN AUTOGENERATED FILE, PLEASE EDIT services.py */\n" - h += "#ifndef __SERVICES_H\n" - h += "#define __SERVICES_H\n" - - h += "#include \n" - h += "#include \n" - - h += "struct service { std::string name; int port; bool should_log; int frequency; int decimation; };\n" - h += "static std::map services = {\n" - for k, v in SERVICE_LIST.items(): - should_log = "true" if v.should_log else "false" - decimation = -1 if v.decimation is None else v.decimation - h += ' { "%s", {"%s", %d, %s, %d, %d}},\n' % \ - (k, k, v.port, should_log, v.frequency, decimation) - h += "};\n" - - h += "#endif\n" - return h - - -if __name__ == "__main__": - print(build_header()) diff --git a/site_scons/site_tools/cython.py b/site_scons/site_tools/cython.py deleted file mode 100644 index c29147553..000000000 --- a/site_scons/site_tools/cython.py +++ /dev/null @@ -1,72 +0,0 @@ -import re -import SCons -from SCons.Action import Action -from SCons.Scanner import Scanner - -pyx_from_import_re = re.compile(r'^from\s+(\S+)\s+cimport', re.M) -pyx_import_re = re.compile(r'^cimport\s+(\S+)', re.M) -cdef_import_re = re.compile(r'^cdef extern from\s+.(\S+).:', re.M) - - -def pyx_scan(node, env, path, arg=None): - contents = node.get_text_contents() - - # from cimport ... - matches = pyx_from_import_re.findall(contents) - # cimport - matches += pyx_import_re.findall(contents) - - # Modules can be either .pxd or .pyx files - files = [m.replace('.', '/') + '.pxd' for m in matches] - files += [m.replace('.', '/') + '.pyx' for m in matches] - - # cdef extern from - files += cdef_import_re.findall(contents) - - # Handle relative imports - cur_dir = str(node.get_dir()) - files = [cur_dir + f if f.startswith('/') else f for f in files] - - # Filter out non-existing files (probably system imports) - files = [f for f in files if env.File(f).exists()] - return env.File(files) - - -pyxscanner = Scanner(function=pyx_scan, skeys=['.pyx', '.pxd'], recursive=True) -cythonAction = Action("$CYTHONCOM") - - -def create_builder(env): - try: - cython = env['BUILDERS']['Cython'] - except KeyError: - cython = SCons.Builder.Builder( - action=cythonAction, - emitter={}, - suffix=cython_suffix_emitter, - single_source=1 - ) - env.Append(SCANNERS=pyxscanner) - env['BUILDERS']['Cython'] = cython - return cython - -def cython_suffix_emitter(env, source): - return "$CYTHONCFILESUFFIX" - -def generate(env): - env["CYTHON"] = "cythonize" - env["CYTHONCOM"] = "$CYTHON $CYTHONFLAGS $SOURCE" - env["CYTHONCFILESUFFIX"] = ".cpp" - - c_file, _ = SCons.Tool.createCFileBuilders(env) - - c_file.suffix['.pyx'] = cython_suffix_emitter - c_file.add_action('.pyx', cythonAction) - - c_file.suffix['.py'] = cython_suffix_emitter - c_file.add_action('.py', cythonAction) - - create_builder(env) - -def exists(env): - return True diff --git a/visionipc/__init__.py b/visionipc/__init__.py index 7adfc58d1..57011537e 100644 --- a/visionipc/__init__.py +++ b/visionipc/__init__.py @@ -1,4 +1,4 @@ -from cereal.visionipc.visionipc_pyx import VisionBuf, VisionIpcClient, VisionIpcServer, VisionStreamType, get_endpoint_name +from msgq.visionipc.visionipc_pyx import VisionBuf, VisionIpcClient, VisionIpcServer, VisionStreamType, get_endpoint_name assert VisionBuf assert VisionIpcClient assert VisionIpcServer diff --git a/visionipc/ipc.cc b/visionipc/ipc.cc index c4ab9a475..30f7d21d4 100644 --- a/visionipc/ipc.cc +++ b/visionipc/ipc.cc @@ -15,7 +15,7 @@ #define getsocket() socket(AF_UNIX, SOCK_SEQPACKET, 0) #endif -#include "cereal/visionipc/ipc.h" +#include "msgq/visionipc/ipc.h" int ipc_connect(const char* socket_path) { int err; diff --git a/visionipc/tests/test_visionipc.py b/visionipc/tests/test_visionipc.py index 7bf075d7f..1c34613dd 100755 --- a/visionipc/tests/test_visionipc.py +++ b/visionipc/tests/test_visionipc.py @@ -4,7 +4,7 @@ import random import unittest import numpy as np -from cereal.visionipc import VisionIpcServer, VisionIpcClient, VisionStreamType +from msgq.visionipc import VisionIpcServer, VisionIpcClient, VisionStreamType def zmq_sleep(t=1): if "ZMQ" in os.environ: diff --git a/visionipc/visionbuf.cc b/visionipc/visionbuf.cc index e9e0ff397..fe7e5db9c 100644 --- a/visionipc/visionbuf.cc +++ b/visionipc/visionbuf.cc @@ -1,4 +1,4 @@ -#include "cereal/visionipc/visionbuf.h" +#include "msgq/visionipc/visionbuf.h" #define ALIGN(x, align) (((x) + (align)-1) & ~((align)-1)) diff --git a/visionipc/visionbuf.h b/visionipc/visionbuf.h index e0e78f453..6b678a004 100644 --- a/visionipc/visionbuf.h +++ b/visionipc/visionbuf.h @@ -1,6 +1,6 @@ #pragma once -#include "cereal/visionipc/visionipc.h" +#include "msgq/visionipc/visionipc.h" #define CL_USE_DEPRECATED_OPENCL_1_2_APIS #ifdef __APPLE__ diff --git a/visionipc/visionbuf_cl.cc b/visionipc/visionbuf_cl.cc index 0315d8db9..db7df00dc 100644 --- a/visionipc/visionbuf_cl.cc +++ b/visionipc/visionbuf_cl.cc @@ -1,4 +1,4 @@ -#include "cereal/visionipc/visionbuf.h" +#include "msgq/visionipc/visionbuf.h" #include #include diff --git a/visionipc/visionbuf_ion.cc b/visionipc/visionbuf_ion.cc index f72e76cf7..13be15491 100644 --- a/visionipc/visionbuf_ion.cc +++ b/visionipc/visionbuf_ion.cc @@ -14,7 +14,7 @@ #include -#include "cereal/visionipc/visionbuf.h" +#include "msgq/visionipc/visionbuf.h" // keep trying if x gets interrupted by a signal #define HANDLE_EINTR(x) \ diff --git a/visionipc/visionipc.pxd b/visionipc/visionipc.pxd index 3151dfc7d..645ab2208 100644 --- a/visionipc/visionipc.pxd +++ b/visionipc/visionipc.pxd @@ -7,7 +7,7 @@ from libcpp.set cimport set from libc.stdint cimport uint32_t, uint64_t from libcpp cimport bool, int -cdef extern from "cereal/visionipc/visionbuf.h": +cdef extern from "msgq/visionipc/visionbuf.h": struct _cl_device_id struct _cl_context struct _cl_mem @@ -30,14 +30,14 @@ cdef extern from "cereal/visionipc/visionbuf.h": cl_mem buf_cl void set_frame_id(uint64_t id) -cdef extern from "cereal/visionipc/visionipc.h": +cdef extern from "msgq/visionipc/visionipc.h": struct VisionIpcBufExtra: uint32_t frame_id uint64_t timestamp_sof uint64_t timestamp_eof bool valid -cdef extern from "cereal/visionipc/visionipc_server.h": +cdef extern from "msgq/visionipc/visionipc_server.h": string get_endpoint_name(string, VisionStreamType) cdef cppclass VisionIpcServer: @@ -48,7 +48,7 @@ cdef extern from "cereal/visionipc/visionipc_server.h": void send(VisionBuf *, VisionIpcBufExtra *, bool) void start_listener() -cdef extern from "cereal/visionipc/visionipc_client.h": +cdef extern from "msgq/visionipc/visionipc_client.h": cdef cppclass VisionIpcClient: int num_buffers VisionBuf buffers[1] diff --git a/visionipc/visionipc_client.cc b/visionipc/visionipc_client.cc index e3c6d0d1c..ab1bfa3ae 100644 --- a/visionipc/visionipc_client.cc +++ b/visionipc/visionipc_client.cc @@ -4,10 +4,11 @@ #include #include -#include "cereal/visionipc/ipc.h" -#include "cereal/visionipc/visionipc_client.h" -#include "cereal/visionipc/visionipc_server.h" -#include "cereal/logger/logger.h" +#include "msgq/visionipc/ipc.h" +#include "msgq/visionipc/visionipc_client.h" +#include "msgq/visionipc/visionipc_server.h" +#include "msgq/logger/logger.h" +#include "msgq/logger/logger.h" static int connect_to_vipc_server(const std::string &name, bool blocking) { const std::string ipc_path = get_ipc_path(name); diff --git a/visionipc/visionipc_client.h b/visionipc/visionipc_client.h index 970bac3a7..e03677867 100644 --- a/visionipc/visionipc_client.h +++ b/visionipc/visionipc_client.h @@ -3,8 +3,9 @@ #include #include -#include "cereal/messaging/messaging.h" -#include "cereal/visionipc/visionbuf.h" +#include "msgq/messaging/messaging.h" +#include "msgq/visionipc/visionbuf.h" + class VisionIpcClient { private: diff --git a/visionipc/visionipc_server.cc b/visionipc/visionipc_server.cc index da9d11f91..e2ac55e64 100644 --- a/visionipc/visionipc_server.cc +++ b/visionipc/visionipc_server.cc @@ -8,10 +8,10 @@ #include #include -#include "cereal/messaging/messaging.h" -#include "cereal/visionipc/ipc.h" -#include "cereal/visionipc/visionipc_server.h" -#include "cereal/logger/logger.h" +#include "msgq/messaging/messaging.h" +#include "msgq/visionipc/ipc.h" +#include "msgq/visionipc/visionipc_server.h" +#include "msgq/logger/logger.h" std::string get_endpoint_name(std::string name, VisionStreamType type){ if (messaging_use_zmq()){ diff --git a/visionipc/visionipc_server.h b/visionipc/visionipc_server.h index c494b1fcf..4207f41cd 100644 --- a/visionipc/visionipc_server.h +++ b/visionipc/visionipc_server.h @@ -5,8 +5,8 @@ #include #include -#include "cereal/messaging/messaging.h" -#include "cereal/visionipc/visionbuf.h" +#include "msgq/messaging/messaging.h" +#include "msgq/visionipc/visionbuf.h" std::string get_endpoint_name(std::string name, VisionStreamType type); std::string get_ipc_path(const std::string &name); diff --git a/visionipc/visionipc_tests.cc b/visionipc/visionipc_tests.cc index 4a081df11..8c4e7fa3f 100644 --- a/visionipc/visionipc_tests.cc +++ b/visionipc/visionipc_tests.cc @@ -2,8 +2,10 @@ #include #include "catch2/catch.hpp" -#include "cereal/visionipc/visionipc_server.h" -#include "cereal/visionipc/visionipc_client.h" + +#include "msgq/visionipc/visionipc_server.h" +#include "msgq/visionipc/visionipc_client.h" + static void zmq_sleep(int milliseconds=1000){ if (messaging_use_zmq()){ From 615aea9b5519d2a3631fce4753bed29287fc4f9b Mon Sep 17 00:00:00 2001 From: Bruce Wayne Date: Thu, 6 Jun 2024 14:44:57 -0700 Subject: [PATCH 04/10] add readme --- README.md | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 000000000..34fe35682 --- /dev/null +++ b/README.md @@ -0,0 +1,54 @@ +# MSGQ: A lock free single producer multi consumer message queue + +## What is MSGQ? +MSGQ is a system to pass messages from a single producer to multiple consumers. All the consumers need to be able to receive all the messages. It is designed to be a high performance replacement for ZMQ-like SUB/PUB patterns. It uses a ring buffer in shared memory to efficiently read and write data. Each read requires a copy. Writing can be done without a copy, as long as the size of the data is known in advance. + +## Storage +The storage for the queue consists of an area of metadata, and the actual buffer. The metadata contains: + +1. A counter to the number of readers that are active +2. A pointer to the head of the queue for writing. From now on referred to as *write pointer* +3. A cycle counter for the writer. This counter is incremented when the writer wraps around +4. N pointers, pointing to the current read position for all the readers. From now on referred to as *read pointer* +5. N counters, counting the number of cycles for all the readers +6. N booleans, indicating validity for all the readers. From now on referred to as *validity flag* + +The counter and the pointer are both 32 bit values, packed into 64 bit so they can be read and written atomically. + +The data buffer is a ring buffer. All messages are prefixed by an 8 byte size field, followed by the data. A size of -1 indicates a wrap-around, and means the next message is stored at the beginning of the buffer. + + +## Writing +Writing involves the following steps: + +1. Check if the area that is to be written overlaps with any of the read pointers, mark those readers as invalid by clearing the validity flag. +2. Write the message +3. Increase the write pointer by the size of the message + +In case there is not enough space at the end of the buffer, a special empty message with a prefix of -1 is written. The cycle counter is incremented by one. In this case step 1 will check there are no read pointers pointing to the remainder of the buffer. Then another write cycle will start with the actual message. + +There always needs to be 8 bytes of empty space at the end of the buffer. By doing this there is always space to write the -1. + +## Reset reader +When the reader is lagging too much behind the read pointer becomes invalid and no longer points to the beginning of a valid message. To reset a reader to the current write pointer, the following steps are performed: + +1. Set valid flag +2. Set read cycle counter to that of the writer +3. Set read pointer to write pointer + +## Reading +Reading involves the following steps: + +1. Read the size field at the current read pointer +2. Read the validity flag +3. Copy the data out of the buffer +4. Increase the read pointer by the size of the message +5. Check the validity flag again + +Before starting the copy, the valid flag is checked. This is to prevent a race condition where the size prefix was invalid, and the read could read outside of the buffer. Make sure that step 1 and 2 are not reordered by your compiler or CPU. + +If a writer overwrites the data while it's being copied out, the data will be invalid. Therefore the validity flag is also checked after reading it. The order of step 4 and 5 does not matter. + +If at steps 2 or 5 the validity flag is not set, the reader is reset. Any data that was already read is discarded. After the reader is reset, the reading starts from the beginning. + +If a message with size -1 is encountered, step 3 and 4 are replaced by increasing the cycle counter and setting the read pointer to the beginning of the buffer. After that another read is performed. From 7c00db60e5564bb4cf283485c07b84d84a5a326c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Harald=20Sch=C3=A4fer?= Date: Fri, 7 Jun 2024 10:51:15 -0700 Subject: [PATCH 05/10] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 34fe35682..a4378654b 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # MSGQ: A lock free single producer multi consumer message queue -## What is MSGQ? -MSGQ is a system to pass messages from a single producer to multiple consumers. All the consumers need to be able to receive all the messages. It is designed to be a high performance replacement for ZMQ-like SUB/PUB patterns. It uses a ring buffer in shared memory to efficiently read and write data. Each read requires a copy. Writing can be done without a copy, as long as the size of the data is known in advance. +## What is this library? +MSGQ is a generic high performance IPC pub sub system with a single publisher and multiple subscribers. MSGQ is designed to be a high performance replacement for ZMQ-like SUB/PUB patterns. It uses a ring buffer in shared memory to efficiently read and write data. Each read requires a copy. Writing can be done without a copy, as long as the size of the data is known in advance. While MSGQ is the core of this library, this library also allows replacing the MSGQ backend with ZMQ or a spoofed implementation that can be used for deterministic testing. This library also contains visionipc, an IPC system specifically for large continiguous buffers (like images/video). ## Storage The storage for the queue consists of an area of metadata, and the actual buffer. The metadata contains: From 2ac05eae17b4f2b5cd519abd00db2a2217b04744 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Harald=20Sch=C3=A4fer?= Date: Fri, 7 Jun 2024 13:32:22 -0700 Subject: [PATCH 06/10] Get tests running again (#621) * Get tests running again * rn * I don't understand what any of this means * More updates * Try fixing test_fake * test fake passes * All python tests pass --- .github/workflows/repo.yml | 27 +++++ .github/workflows/tests.yml | 61 ++++++++++ Dockerfile | 54 +++++++++ SConscript | 4 +- SConstruct | 89 ++++++++++++++ codecov.yml | 8 ++ messaging/__init__.py | 61 ++++++++++ messaging/tests/__init__.py | 0 messaging/tests/test_fake.py | 193 ++++++++++++++++++++++++++++++ messaging/tests/test_messaging.py | 88 ++++++++++++++ messaging/tests/test_poller.py | 142 ++++++++++++++++++++++ pyproject.toml | 21 ++++ site_scons/site_tools/cython.py | 72 +++++++++++ 13 files changed, 817 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/repo.yml create mode 100644 .github/workflows/tests.yml create mode 100644 Dockerfile create mode 100644 SConstruct create mode 100644 codecov.yml create mode 100644 messaging/tests/__init__.py create mode 100644 messaging/tests/test_fake.py create mode 100755 messaging/tests/test_messaging.py create mode 100644 messaging/tests/test_poller.py create mode 100644 pyproject.toml create mode 100644 site_scons/site_tools/cython.py diff --git a/.github/workflows/repo.yml b/.github/workflows/repo.yml new file mode 100644 index 000000000..5f12686f8 --- /dev/null +++ b/.github/workflows/repo.yml @@ -0,0 +1,27 @@ +name: repo + +on: + schedule: + - cron: "0 15 1 * *" + workflow_dispatch: + +jobs: + pre-commit-autoupdate: + name: pre-commit autoupdate + runs-on: ubuntu-latest + container: + steps: + - uses: actions/checkout@v3 + - name: pre-commit autoupdate + run: | + git config --global --add safe.directory '*' + pre-commit autoupdate + - name: Create Pull Request + uses: peter-evans/create-pull-request@5b4a9f6a9e2af26e5f02351490b90d01eb8ec1e5 + with: + token: ${{ secrets.ACTIONS_CREATE_PR_PAT }} + commit-message: Update pre-commit hook versions + title: 'pre-commit: autoupdate hooks' + branch: pre-commit-updates + base: master + delete-branch: true diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 000000000..2f42fa7de --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,61 @@ +name: tests + +on: [push, pull_request] + +env: + DOCKER_REGISTRY: ghcr.io/commaai + RUN: docker run -e PYTHONWARNINGS=error --shm-size 1G --name msgq msgq /bin/sh -c + RUN_NAMED: docker run -e PYTHONWARNINGS=error --shm-size 1G --rm msgq /bin/sh -c + CI_RUN: docker run -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID --rm msgqci /bin/bash -c + BUILD: docker buildx build --pull --load --cache-to type=inline --cache-from $DOCKER_REGISTRY/msgq:latest -t msgq -f Dockerfile . + PYTHONWARNINGS: error + +jobs: + build: + name: build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Build docker image + run: eval "$BUILD" + - name: Push to dockerhub + if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/msgq' + run: | + docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} + docker tag msgq $DOCKER_REGISTRY/msgq:latest + docker push $DOCKER_REGISTRY/msgq:latest + + unit_tests: + name: unit tests + runs-on: ubuntu-latest + strategy: + matrix: + flags: ['', '--asan', '--ubsan'] + backend: ['MSGQ', 'ZMQ'] + steps: + - uses: actions/checkout@v3 + - name: Build docker image + run: eval "$BUILD" + - name: C++ tests + run: | + $RUN "export ${{ matrix.backend }}=1 && \ + scons ${{ matrix.flags }} -j$(nproc) && \ + messaging/test_runner && \ + visionipc/test_runner" + - name: python tests + run: $RUN_NAMED "${{ matrix.backend }}=1 coverage run -m unittest discover ." + - name: Upload coverage + run: | + docker commit msgq msgqci + $CI_RUN "cd /project/msgq && bash <(curl -s https://codecov.io/bash) -v -F unit_tests_${{ matrix.backend }}" + + static_analysis: + name: static analysis + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Build docker image + run: eval "$BUILD" + - name: Static analysis + # TODO: a package pre-commit installs has a warning, remove the unset once that's fixed + run: $RUN "git init && git add -A && unset PYTHONWARNINGS && pre-commit run --all" diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..4c1b68256 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,54 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y --no-install-recommends \ + autoconf \ + build-essential \ + ca-certificates \ + capnproto \ + clang \ + cppcheck \ + curl \ + git \ + libbz2-dev \ + libcapnp-dev \ + libclang-rt-dev \ + libffi-dev \ + liblzma-dev \ + libncurses5-dev \ + libncursesw5-dev \ + libreadline-dev \ + libsqlite3-dev \ + libssl-dev \ + libtool \ + libzmq3-dev \ + llvm \ + make \ + cmake \ + ocl-icd-opencl-dev \ + opencl-headers \ + python3-dev \ + python3-pip \ + tk-dev \ + wget \ + xz-utils \ + zlib1g-dev \ + && rm -rf /var/lib/apt/lists/* + +RUN pip3 install --break-system-packages --no-cache-dir pyyaml Cython scons pycapnp pre-commit ruff parameterized coverage numpy + +WORKDIR /project/ +RUN cd /tmp/ && \ + git clone -b v2.x --depth 1 https://github.com/catchorg/Catch2.git && \ + cd Catch2 && \ + mv single_include/catch2/ /project/ && \ + cd .. \ + rm -rf Catch2 + +WORKDIR /project/msgq + +ENV PYTHONPATH=/project + +COPY . . +RUN rm -rf .git && \ + scons -c && scons -j$(nproc) diff --git a/SConscript b/SConscript index f8eb6a5e6..5c5267cbc 100644 --- a/SConscript +++ b/SConscript @@ -19,9 +19,6 @@ messaging_objects = env.SharedObject([ messaging = env.Library('messaging', messaging_objects) messaging_python = envCython.Program('messaging/messaging_pyx.so', 'messaging/messaging_pyx.pyx', LIBS=envCython["LIBS"]+[messaging, "zmq", common]) -if GetOption('extras'): - env.Program('messaging/test_runner', ['messaging/test_runner.cc', 'messaging/msgq_tests.cc'], LIBS=[messaging, common]) - # Build Vision IPC vipc_files = ['ipc.cc', 'visionipc_server.cc', 'visionipc_client.cc', 'visionbuf.cc'] @@ -46,6 +43,7 @@ envCython.Program(f'{visionipc_dir.abspath}/visionipc_pyx.so', f'{visionipc_dir. LIBS=vipc_libs, FRAMEWORKS=vipc_frameworks) if GetOption('extras'): + env.Program('messaging/test_runner', ['messaging/test_runner.cc', 'messaging/msgq_tests.cc'], LIBS=[messaging, common]) env.Program('visionipc/test_runner', ['visionipc/test_runner.cc', 'visionipc/visionipc_tests.cc'], LIBS=['pthread'] + vipc_libs, FRAMEWORKS=vipc_frameworks) diff --git a/SConstruct b/SConstruct new file mode 100644 index 000000000..408c7bfa9 --- /dev/null +++ b/SConstruct @@ -0,0 +1,89 @@ +import os +import platform +import subprocess +import sysconfig +import numpy as np + +arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip() +if platform.system() == "Darwin": + arch = "Darwin" + +common = '' + +cpppath = [ + f"#/../", + '/usr/lib/include', + '/opt/homebrew/include', + sysconfig.get_paths()['include'], +] + +libpath = [ + '/opt/homebrew/lib', +] + +AddOption('--minimal', + action='store_false', + dest='extras', + default=True, + help='the minimum build. no tests, tools, etc.') + +AddOption('--asan', + action='store_true', + help='turn on ASAN') + +AddOption('--ubsan', + action='store_true', + help='turn on UBSan') + +ccflags = [] +ldflags = [] +if GetOption('ubsan'): + flags = [ + "-fsanitize=undefined", + "-fno-sanitize-recover=undefined", + ] + ccflags += flags + ldflags += flags +elif GetOption('asan'): + ccflags += ["-fsanitize=address", "-fno-omit-frame-pointer"] + ldflags += ["-fsanitize=address"] + +env = Environment( + ENV=os.environ, + CC='clang', + CXX='clang++', + CCFLAGS=[ + "-g", + "-fPIC", + "-O2", + "-Wunused", + "-Werror", + "-Wshadow", + "-Wno-vla-cxx-extension", + ] + ccflags, + LDFLAGS=ldflags, + LINKFLAGS=ldflags, + + CFLAGS="-std=gnu11", + CXXFLAGS="-std=c++1z", + CPPPATH=cpppath, + LIBPATH=libpath, + CYTHONCFILESUFFIX=".cpp", + tools=["default", "cython"] +) + +Export('env', 'arch', 'common') + +envCython = env.Clone(LIBS=[]) +envCython["CPPPATH"] += [np.get_include()] +envCython["CCFLAGS"] += ["-Wno-#warnings", "-Wno-shadow", "-Wno-deprecated-declarations"] +envCython["CCFLAGS"].remove('-Werror') +if arch == "Darwin": + envCython["LINKFLAGS"] = ["-bundle", "-undefined", "dynamic_lookup"] +else: + envCython["LINKFLAGS"] = ["-pthread", "-shared"] + +Export('envCython') + + +SConscript(['SConscript']) diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 000000000..83427c3ee --- /dev/null +++ b/codecov.yml @@ -0,0 +1,8 @@ +comment: false +coverage: + status: + project: + default: + informational: true + patch: off + diff --git a/messaging/__init__.py b/messaging/__init__.py index e69de29bb..bf6b1e882 100644 --- a/messaging/__init__.py +++ b/messaging/__init__.py @@ -0,0 +1,61 @@ +# must be built with scons +from msgq.messaging.messaging_pyx import Context, Poller, SubSocket, PubSocket, SocketEventHandle, toggle_fake_events, \ + set_fake_prefix, get_fake_prefix, delete_fake_prefix, wait_for_one_event +from msgq.messaging.messaging_pyx import MultiplePublishersError, MessagingError + +from typing import Optional, List + +assert MultiplePublishersError +assert MessagingError +assert toggle_fake_events +assert set_fake_prefix +assert get_fake_prefix +assert delete_fake_prefix +assert wait_for_one_event + +NO_TRAVERSAL_LIMIT = 2**64-1 + +context = Context() + + +def fake_event_handle(endpoint: str, identifier: Optional[str] = None, override: bool = True, enable: bool = False) -> SocketEventHandle: + identifier = identifier or get_fake_prefix() + handle = SocketEventHandle(endpoint, identifier, override) + if override: + handle.enabled = enable + + return handle + +def pub_sock(endpoint: str) -> PubSocket: + sock = PubSocket() + sock.connect(context, endpoint) + return sock + + +def sub_sock(endpoint: str, poller: Optional[Poller] = None, addr: str = "127.0.0.1", + conflate: bool = False, timeout: Optional[int] = None) -> SubSocket: + sock = SubSocket() + sock.connect(context, endpoint, addr.encode('utf8'), conflate) + + if timeout is not None: + sock.setTimeout(timeout) + + if poller is not None: + poller.registerSocket(sock) + return sock + +def drain_sock_raw(sock: SubSocket, wait_for_one: bool = False) -> List[bytes]: + """Receive all message currently available on the queue""" + ret: List[bytes] = [] + while 1: + if wait_for_one and len(ret) == 0: + dat = sock.receive() + else: + dat = sock.receive(non_blocking=True) + + if dat is None: + break + + ret.append(dat) + + return ret diff --git a/messaging/tests/__init__.py b/messaging/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/messaging/tests/test_fake.py b/messaging/tests/test_fake.py new file mode 100644 index 000000000..d17065565 --- /dev/null +++ b/messaging/tests/test_fake.py @@ -0,0 +1,193 @@ +import os +import unittest +import multiprocessing +import platform +from parameterized import parameterized_class +from typing import Optional + +import msgq.messaging as messaging + +WAIT_TIMEOUT = 5 + + +@unittest.skipIf(platform.system() == "Darwin", "Events not supported on macOS") +class TestEvents(unittest.TestCase): + + def test_mutation(self): + handle = messaging.fake_event_handle("carState") + event = handle.recv_called_event + + self.assertFalse(event.peek()) + event.set() + self.assertTrue(event.peek()) + event.clear() + self.assertFalse(event.peek()) + + del event + + def test_wait(self): + handle = messaging.fake_event_handle("carState") + event = handle.recv_called_event + + event.set() + try: + event.wait(WAIT_TIMEOUT) + self.assertTrue(event.peek()) + except RuntimeError: + self.fail("event.wait() timed out") + + def test_wait_multiprocess(self): + handle = messaging.fake_event_handle("carState") + event = handle.recv_called_event + + def set_event_run(): + event.set() + + try: + p = multiprocessing.Process(target=set_event_run) + p.start() + event.wait(WAIT_TIMEOUT) + self.assertTrue(event.peek()) + except RuntimeError: + self.fail("event.wait() timed out") + + p.kill() + + def test_wait_zero_timeout(self): + handle = messaging.fake_event_handle("carState") + event = handle.recv_called_event + + try: + event.wait(0) + self.fail("event.wait() did not time out") + except RuntimeError: + self.assertFalse(event.peek()) + + +@unittest.skipIf(platform.system() == "Darwin", "FakeSockets not supported on macOS") +@unittest.skipIf("ZMQ" in os.environ, "FakeSockets not supported on ZMQ") +@parameterized_class([{"prefix": None}, {"prefix": "test"}]) +class TestFakeSockets(unittest.TestCase): + prefix: Optional[str] = None + + def setUp(self): + messaging.toggle_fake_events(True) + if self.prefix is not None: + messaging.set_fake_prefix(self.prefix) + else: + messaging.delete_fake_prefix() + + def tearDown(self): + messaging.toggle_fake_events(False) + messaging.delete_fake_prefix() + + def test_event_handle_init(self): + handle = messaging.fake_event_handle("controlsState", override=True) + + self.assertFalse(handle.enabled) + self.assertGreaterEqual(handle.recv_called_event.fd, 0) + self.assertGreaterEqual(handle.recv_ready_event.fd, 0) + + def test_non_managed_socket_state(self): + # non managed socket should have zero state + _ = messaging.pub_sock("ubloxGnss") + + handle = messaging.fake_event_handle("ubloxGnss", override=False) + + self.assertFalse(handle.enabled) + self.assertEqual(handle.recv_called_event.fd, 0) + self.assertEqual(handle.recv_ready_event.fd, 0) + + def test_managed_socket_state(self): + # managed socket should not change anything about the state + handle = messaging.fake_event_handle("ubloxGnss") + handle.enabled = True + + expected_enabled = handle.enabled + expected_recv_called_fd = handle.recv_called_event.fd + expected_recv_ready_fd = handle.recv_ready_event.fd + + _ = messaging.pub_sock("ubloxGnss") + + self.assertEqual(handle.enabled, expected_enabled) + self.assertEqual(handle.recv_called_event.fd, expected_recv_called_fd) + self.assertEqual(handle.recv_ready_event.fd, expected_recv_ready_fd) + + def test_sockets_enable_disable(self): + carState_handle = messaging.fake_event_handle("ubloxGnss", enable=True) + recv_called = carState_handle.recv_called_event + recv_ready = carState_handle.recv_ready_event + + pub_sock = messaging.pub_sock("ubloxGnss") + sub_sock = messaging.sub_sock("ubloxGnss") + + try: + carState_handle.enabled = True + recv_ready.set() + pub_sock.send(b"test") + _ = sub_sock.receive() + self.assertTrue(recv_called.peek()) + recv_called.clear() + + carState_handle.enabled = False + recv_ready.set() + pub_sock.send(b"test") + _ = sub_sock.receive() + self.assertFalse(recv_called.peek()) + except RuntimeError: + self.fail("event.wait() timed out") + + def test_synced_pub_sub(self): + def daemon_repub_process_run(): + pub_sock = messaging.pub_sock("ubloxGnss") + sub_sock = messaging.sub_sock("carState") + + frame = -1 + while True: + frame += 1 + msg = sub_sock.receive(non_blocking=True) + if msg is None: + print("none received") + continue + + bts = frame.to_bytes(8, 'little') + pub_sock.send(bts) + + carState_handle = messaging.fake_event_handle("carState", enable=True) + recv_called = carState_handle.recv_called_event + recv_ready = carState_handle.recv_ready_event + + p = multiprocessing.Process(target=daemon_repub_process_run) + p.start() + + pub_sock = messaging.pub_sock("carState") + sub_sock = messaging.sub_sock("ubloxGnss") + + try: + for i in range(10): + recv_called.wait(WAIT_TIMEOUT) + recv_called.clear() + + if i == 0: + sub_sock.receive(non_blocking=True) + + bts = i.to_bytes(8, 'little') + pub_sock.send(bts) + + recv_ready.set() + recv_called.wait(WAIT_TIMEOUT) + + msg = sub_sock.receive(non_blocking=True) + self.assertIsNotNone(msg) + self.assertEqual(len(msg), 8) + + frame = int.from_bytes(msg, 'little') + self.assertEqual(frame, i) + except RuntimeError: + self.fail("event.wait() timed out") + finally: + p.kill() + + +if __name__ == "__main__": + unittest.main() diff --git a/messaging/tests/test_messaging.py b/messaging/tests/test_messaging.py new file mode 100755 index 000000000..d2e30b61f --- /dev/null +++ b/messaging/tests/test_messaging.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +import os +import random +import threading +import time +import string +import unittest + +import msgq.messaging as messaging + + +def random_sock(): + return ''.join(random.choices(string.ascii_uppercase + string.digits, k=10)) + +def random_bytes(length=1000): + return bytes([random.randrange(0xFF) for _ in range(length)]) + +def zmq_sleep(t=1): + if "ZMQ" in os.environ: + time.sleep(t) + +def zmq_expected_failure(func): + if "ZMQ" in os.environ: + return unittest.expectedFailure(func) + else: + return func + +def delayed_send(delay, sock, dat): + def send_func(): + sock.send(dat) + threading.Timer(delay, send_func).start() + +class TestPubSubSockets(unittest.TestCase): + + def setUp(self): + # ZMQ pub socket takes too long to die + # sleep to prevent multiple publishers error between tests + zmq_sleep() + + def test_pub_sub(self): + sock = random_sock() + pub_sock = messaging.pub_sock(sock) + sub_sock = messaging.sub_sock(sock, conflate=False, timeout=None) + zmq_sleep(3) + + for _ in range(1000): + msg = random_bytes() + pub_sock.send(msg) + recvd = sub_sock.receive() + self.assertEqual(msg, recvd) + + def test_conflate(self): + sock = random_sock() + pub_sock = messaging.pub_sock(sock) + for conflate in [True, False]: + for _ in range(10): + num_msgs = random.randint(3, 10) + sub_sock = messaging.sub_sock(sock, conflate=conflate, timeout=None) + zmq_sleep() + + sent_msgs = [] + for __ in range(num_msgs): + msg = random_bytes() + pub_sock.send(msg) + sent_msgs.append(msg) + time.sleep(0.1) + recvd_msgs = messaging.drain_sock_raw(sub_sock) + if conflate: + self.assertEqual(len(recvd_msgs), 1) + else: + # TODO: compare actual data + self.assertEqual(len(recvd_msgs), len(sent_msgs)) + + def test_receive_timeout(self): + sock = random_sock() + for _ in range(10): + timeout = random.randrange(200) + sub_sock = messaging.sub_sock(sock, timeout=timeout) + zmq_sleep() + + start_time = time.monotonic() + recvd = sub_sock.receive() + self.assertLess(time.monotonic() - start_time, 0.2) + assert recvd is None + + +if __name__ == "__main__": + unittest.main() diff --git a/messaging/tests/test_poller.py b/messaging/tests/test_poller.py new file mode 100644 index 000000000..5ac990e31 --- /dev/null +++ b/messaging/tests/test_poller.py @@ -0,0 +1,142 @@ +import unittest +import time +import msgq.messaging as messaging +import concurrent.futures + +SERVICE_NAME = 'myService' + +def poller(): + context = messaging.Context() + + p = messaging.Poller() + + sub = messaging.SubSocket() + sub.connect(context, SERVICE_NAME) + p.registerSocket(sub) + + socks = p.poll(10000) + r = [s.receive(non_blocking=True) for s in socks] + + return r + + +class TestPoller(unittest.TestCase): + def test_poll_once(self): + context = messaging.Context() + + pub = messaging.PubSocket() + pub.connect(context, SERVICE_NAME) + + with concurrent.futures.ThreadPoolExecutor() as e: + poll = e.submit(poller) + + time.sleep(0.1) # Slow joiner syndrome + + # Send message + pub.send(b"a") + + # Wait for poll result + result = poll.result() + + del pub + context.term() + + self.assertEqual(result, [b"a"]) + + def test_poll_and_create_many_subscribers(self): + context = messaging.Context() + + pub = messaging.PubSocket() + pub.connect(context, SERVICE_NAME) + + with concurrent.futures.ThreadPoolExecutor() as e: + poll = e.submit(poller) + + time.sleep(0.1) # Slow joiner syndrome + c = messaging.Context() + for _ in range(10): + messaging.SubSocket().connect(c, SERVICE_NAME) + + time.sleep(0.1) + + # Send message + pub.send(b"a") + + # Wait for poll result + result = poll.result() + + del pub + context.term() + + self.assertEqual(result, [b"a"]) + + def test_multiple_publishers_exception(self): + context = messaging.Context() + + with self.assertRaises(messaging.MultiplePublishersError): + pub1 = messaging.PubSocket() + pub1.connect(context, SERVICE_NAME) + + pub2 = messaging.PubSocket() + pub2.connect(context, SERVICE_NAME) + + pub1.send(b"a") + + del pub1 + del pub2 + context.term() + + def test_multiple_messages(self): + context = messaging.Context() + + pub = messaging.PubSocket() + pub.connect(context, SERVICE_NAME) + + sub = messaging.SubSocket() + sub.connect(context, SERVICE_NAME) + + time.sleep(0.1) # Slow joiner + + for i in range(1, 100): + pub.send(b'a'*i) + + msg_seen = False + i = 1 + while True: + r = sub.receive(non_blocking=True) + + if r is not None: + self.assertEqual(b'a'*i, r) + + msg_seen = True + i += 1 + + if r is None and msg_seen: # ZMQ sometimes receives nothing on the first receive + break + + del pub + del sub + context.term() + + def test_conflate(self): + context = messaging.Context() + + pub = messaging.PubSocket() + pub.connect(context, SERVICE_NAME) + + sub = messaging.SubSocket() + sub.connect(context, SERVICE_NAME, conflate=True) + + time.sleep(0.1) # Slow joiner + pub.send(b'a') + pub.send(b'b') + + self.assertEqual(b'b', sub.receive()) + + del pub + del sub + context.term() + + +if __name__ == "__main__": + unittest.main() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..73928614e --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,21 @@ +# https://beta.ruff.rs/docs/configuration/#using-pyprojecttoml +[tool.ruff] +lint.select = ["E", "F", "W", "PIE", "C4", "ISC", "RUF100", "A"] +lint.ignore = ["W292", "E741", "E402", "C408", "ISC003"] +lint.flake8-implicit-str-concat.allow-multiline=false + +line-length = 160 +target-version="py311" + +[mypy.tool] +# third-party packages +ignore_missing_imports=true + +# helpful warnings +warn_redundant_casts=true +warn_unreachable=true +warn_unused_ignores=true + +# restrict dynamic typing +warn_return_any=true +check_untyped_defs=true diff --git a/site_scons/site_tools/cython.py b/site_scons/site_tools/cython.py new file mode 100644 index 000000000..c29147553 --- /dev/null +++ b/site_scons/site_tools/cython.py @@ -0,0 +1,72 @@ +import re +import SCons +from SCons.Action import Action +from SCons.Scanner import Scanner + +pyx_from_import_re = re.compile(r'^from\s+(\S+)\s+cimport', re.M) +pyx_import_re = re.compile(r'^cimport\s+(\S+)', re.M) +cdef_import_re = re.compile(r'^cdef extern from\s+.(\S+).:', re.M) + + +def pyx_scan(node, env, path, arg=None): + contents = node.get_text_contents() + + # from cimport ... + matches = pyx_from_import_re.findall(contents) + # cimport + matches += pyx_import_re.findall(contents) + + # Modules can be either .pxd or .pyx files + files = [m.replace('.', '/') + '.pxd' for m in matches] + files += [m.replace('.', '/') + '.pyx' for m in matches] + + # cdef extern from + files += cdef_import_re.findall(contents) + + # Handle relative imports + cur_dir = str(node.get_dir()) + files = [cur_dir + f if f.startswith('/') else f for f in files] + + # Filter out non-existing files (probably system imports) + files = [f for f in files if env.File(f).exists()] + return env.File(files) + + +pyxscanner = Scanner(function=pyx_scan, skeys=['.pyx', '.pxd'], recursive=True) +cythonAction = Action("$CYTHONCOM") + + +def create_builder(env): + try: + cython = env['BUILDERS']['Cython'] + except KeyError: + cython = SCons.Builder.Builder( + action=cythonAction, + emitter={}, + suffix=cython_suffix_emitter, + single_source=1 + ) + env.Append(SCANNERS=pyxscanner) + env['BUILDERS']['Cython'] = cython + return cython + +def cython_suffix_emitter(env, source): + return "$CYTHONCFILESUFFIX" + +def generate(env): + env["CYTHON"] = "cythonize" + env["CYTHONCOM"] = "$CYTHON $CYTHONFLAGS $SOURCE" + env["CYTHONCFILESUFFIX"] = ".cpp" + + c_file, _ = SCons.Tool.createCFileBuilders(env) + + c_file.suffix['.pyx'] = cython_suffix_emitter + c_file.add_action('.pyx', cythonAction) + + c_file.suffix['.py'] = cython_suffix_emitter + c_file.add_action('.py', cythonAction) + + create_builder(env) + +def exists(env): + return True From 3f82842fbf3ce835496314d87fdf47f7b268f07d Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Fri, 7 Jun 2024 15:37:22 -0700 Subject: [PATCH 07/10] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a4378654b..dc10530e0 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # MSGQ: A lock free single producer multi consumer message queue ## What is this library? -MSGQ is a generic high performance IPC pub sub system with a single publisher and multiple subscribers. MSGQ is designed to be a high performance replacement for ZMQ-like SUB/PUB patterns. It uses a ring buffer in shared memory to efficiently read and write data. Each read requires a copy. Writing can be done without a copy, as long as the size of the data is known in advance. While MSGQ is the core of this library, this library also allows replacing the MSGQ backend with ZMQ or a spoofed implementation that can be used for deterministic testing. This library also contains visionipc, an IPC system specifically for large continiguous buffers (like images/video). +MSGQ is a generic high performance IPC pub sub system with a single publisher and multiple subscribers. MSGQ is designed to be a high performance replacement for ZMQ-like SUB/PUB patterns. It uses a ring buffer in shared memory to efficiently read and write data. Each read requires a copy. Writing can be done without a copy, as long as the size of the data is known in advance. While MSGQ is the core of this library, this library also allows replacing the MSGQ backend with ZMQ or a spoofed implementation that can be used for deterministic testing. This library also contains visionipc, an IPC system specifically for large contiguous buffers (like images/video). ## Storage The storage for the queue consists of an area of metadata, and the actual buffer. The metadata contains: From 381fc3d9dfe7d2ff40a075ff8c1f980ae2a62d19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Harald=20Sch=C3=A4fer?= Date: Sun, 9 Jun 2024 17:17:12 -0700 Subject: [PATCH 08/10] Restructure package (#622) * Move around * rename messaging to ipc * More renames * refactor visionipc ipc * more movement * compiles * works well * update workflow * Update * test fake * fix names * Fix test * exclude library * exclude from lint too * Rm dir * rm this wayu * Try again * mv logger * delete old * HAX * Move logger down * add warning abck --- .github/workflows/tests.yml | 4 +- .pre-commit-config.yaml | 3 +- Dockerfile | 6 +-- SConscript | 40 ++++++++-------- SConstruct | 3 +- messaging/.gitignore | 1 - msgq/.gitignore | 1 + {messaging => msgq}/__init__.py | 6 +-- {messaging => msgq}/event.cc | 2 +- {messaging => msgq}/event.h | 0 {messaging => msgq}/impl_fake.cc | 2 +- {messaging => msgq}/impl_fake.h | 4 +- {messaging => msgq}/impl_msgq.cc | 2 +- {messaging => msgq}/impl_msgq.h | 4 +- {messaging => msgq}/impl_zmq.cc | 2 +- {messaging => msgq}/impl_zmq.h | 2 +- messaging/messaging.cc => msgq/ipc.cc | 8 ++-- messaging/messaging.h => msgq/ipc.h | 0 messaging/messaging.pxd => msgq/ipc.pxd | 4 +- .../messaging_pyx.pyx => msgq/ipc_pyx.pyx | 26 +++++----- {logger => msgq/logger}/logger.h | 0 {messaging => msgq}/msgq.cc | 2 +- {messaging => msgq}/msgq.h | 0 {messaging => msgq}/msgq_tests.cc | 2 +- {messaging => msgq}/test_runner.cc | 0 {messaging => msgq}/tests/__init__.py | 0 {messaging => msgq}/tests/test_fake.py | 47 +++++++++---------- {messaging => msgq}/tests/test_messaging.py | 15 +++--- {messaging => msgq}/tests/test_poller.py | 40 ++++++++-------- {visionipc => msgq/visionipc}/.gitignore | 0 {visionipc => msgq/visionipc}/__init__.py | 0 {visionipc => msgq/visionipc}/test_runner.cc | 0 .../visionipc}/tests/__init__.py | 0 .../visionipc}/tests/test_visionipc.py | 0 {visionipc => msgq/visionipc}/visionbuf.cc | 0 {visionipc => msgq/visionipc}/visionbuf.h | 0 {visionipc => msgq/visionipc}/visionbuf_cl.cc | 0 .../visionipc}/visionbuf_ion.cc | 0 .../ipc.cc => msgq/visionipc/visionipc.cc | 2 +- {visionipc => msgq/visionipc}/visionipc.h | 6 +++ {visionipc => msgq/visionipc}/visionipc.pxd | 0 .../visionipc}/visionipc_client.cc | 6 +-- .../visionipc}/visionipc_client.h | 2 +- .../visionipc}/visionipc_pyx.pxd | 0 .../visionipc}/visionipc_pyx.pyx | 0 .../visionipc}/visionipc_server.cc | 6 +-- .../visionipc}/visionipc_server.h | 2 +- .../visionipc}/visionipc_tests.cc | 0 visionipc/ipc.h | 7 --- 49 files changed, 128 insertions(+), 129 deletions(-) delete mode 100644 messaging/.gitignore create mode 100644 msgq/.gitignore rename {messaging => msgq}/__init__.py (87%) rename {messaging => msgq}/event.cc (99%) rename {messaging => msgq}/event.h (100%) rename {messaging => msgq}/impl_fake.cc (82%) rename {messaging => msgq}/impl_fake.h (95%) rename {messaging => msgq}/impl_msgq.cc (99%) rename {messaging => msgq}/impl_msgq.h (95%) rename {messaging => msgq}/impl_zmq.cc (98%) rename {messaging => msgq}/impl_zmq.h (97%) rename messaging/messaging.cc => msgq/ipc.cc (94%) rename messaging/messaging.h => msgq/ipc.h (100%) rename messaging/messaging.pxd => msgq/ipc.pxd (93%) rename messaging/messaging_pyx.pyx => msgq/ipc_pyx.pyx (89%) rename {logger => msgq/logger}/logger.h (100%) rename {messaging => msgq}/msgq.cc (99%) rename {messaging => msgq}/msgq.h (100%) rename {messaging => msgq}/msgq_tests.cc (99%) rename {messaging => msgq}/test_runner.cc (100%) rename {messaging => msgq}/tests/__init__.py (100%) rename {messaging => msgq}/tests/test_fake.py (78%) rename {messaging => msgq}/tests/test_messaging.py (83%) rename {messaging => msgq}/tests/test_poller.py (75%) rename {visionipc => msgq/visionipc}/.gitignore (100%) rename {visionipc => msgq/visionipc}/__init__.py (100%) rename {visionipc => msgq/visionipc}/test_runner.cc (100%) rename {visionipc => msgq/visionipc}/tests/__init__.py (100%) rename {visionipc => msgq/visionipc}/tests/test_visionipc.py (100%) rename {visionipc => msgq/visionipc}/visionbuf.cc (100%) rename {visionipc => msgq/visionipc}/visionbuf.h (100%) rename {visionipc => msgq/visionipc}/visionbuf_cl.cc (100%) rename {visionipc => msgq/visionipc}/visionbuf_ion.cc (100%) rename visionipc/ipc.cc => msgq/visionipc/visionipc.cc (98%) rename {visionipc => msgq/visionipc}/visionipc.h (57%) rename {visionipc => msgq/visionipc}/visionipc.pxd (100%) rename {visionipc => msgq/visionipc}/visionipc_client.cc (97%) rename {visionipc => msgq/visionipc}/visionipc_client.h (95%) rename {visionipc => msgq/visionipc}/visionipc_pyx.pxd (100%) rename {visionipc => msgq/visionipc}/visionipc_pyx.pyx (100%) rename {visionipc => msgq/visionipc}/visionipc_server.cc (98%) rename {visionipc => msgq/visionipc}/visionipc_server.h (97%) rename {visionipc => msgq/visionipc}/visionipc_tests.cc (100%) delete mode 100644 visionipc/ipc.h diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2f42fa7de..16219b0bc 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -40,8 +40,8 @@ jobs: run: | $RUN "export ${{ matrix.backend }}=1 && \ scons ${{ matrix.flags }} -j$(nproc) && \ - messaging/test_runner && \ - visionipc/test_runner" + msgq/test_runner && \ + msgq/visionipc/test_runner" - name: python tests run: $RUN_NAMED "${{ matrix.backend }}=1 coverage run -m unittest discover ." - name: Upload coverage diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 03360e0f9..b32abe20c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,3 +1,4 @@ +files: ^msgq/ repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.6.0 @@ -21,7 +22,7 @@ repos: entry: cppcheck language: system types: [c++] - exclude: '^(messaging/msgq_tests.cc|messaging/test_runner.cc)' + exclude: '^(msgq/msgq_tests.cc|msgq/test_runner.cc)' args: - --error-exitcode=1 - --inline-suppr diff --git a/Dockerfile b/Dockerfile index 4c1b68256..982d8fa5b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -37,11 +37,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ RUN pip3 install --break-system-packages --no-cache-dir pyyaml Cython scons pycapnp pre-commit ruff parameterized coverage numpy -WORKDIR /project/ +WORKDIR /project/msgq/ RUN cd /tmp/ && \ git clone -b v2.x --depth 1 https://github.com/catchorg/Catch2.git && \ cd Catch2 && \ - mv single_include/catch2/ /project/ && \ + mv single_include/* /project/msgq/ && \ cd .. \ rm -rf Catch2 @@ -50,5 +50,5 @@ WORKDIR /project/msgq ENV PYTHONPATH=/project COPY . . -RUN rm -rf .git && \ +RUN ls && rm -rf .git && \ scons -c && scons -j$(nproc) diff --git a/SConscript b/SConscript index 5c5267cbc..7c0eb5a43 100644 --- a/SConscript +++ b/SConscript @@ -1,27 +1,24 @@ Import('env', 'envCython', 'arch', 'common') -visionipc_dir = Dir('visionipc') +visionipc_dir = Dir('msgq/visionipc') gen_dir = Dir('gen') -# Build messaging - - -messaging_objects = env.SharedObject([ - 'messaging/messaging.cc', - 'messaging/event.cc', - 'messaging/impl_zmq.cc', - 'messaging/impl_msgq.cc', - 'messaging/impl_fake.cc', - 'messaging/msgq.cc', +# Build msgq +msgq_objects = env.SharedObject([ + 'msgq/ipc.cc', + 'msgq/event.cc', + 'msgq/impl_zmq.cc', + 'msgq/impl_msgq.cc', + 'msgq/impl_fake.cc', + 'msgq/msgq.cc', ]) -messaging = env.Library('messaging', messaging_objects) -messaging_python = envCython.Program('messaging/messaging_pyx.so', 'messaging/messaging_pyx.pyx', LIBS=envCython["LIBS"]+[messaging, "zmq", common]) - +msgq = env.Library('msgq', msgq_objects) +msgq_python = envCython.Program('msgq/ipc_pyx.so', 'msgq/ipc_pyx.pyx', LIBS=envCython["LIBS"]+[msgq, "zmq", common]) # Build Vision IPC -vipc_files = ['ipc.cc', 'visionipc_server.cc', 'visionipc_client.cc', 'visionbuf.cc'] +vipc_files = ['visionipc.cc', 'visionipc_server.cc', 'visionipc_client.cc', 'visionbuf.cc'] vipc_sources = [f'{visionipc_dir.abspath}/{f}' for f in vipc_files] if arch == "larch64": @@ -29,12 +26,15 @@ if arch == "larch64": else: vipc_sources += [f'{visionipc_dir.abspath}/visionbuf_cl.cc'] +print(f'Building Vision IPC with {vipc_sources}') vipc_objects = env.SharedObject(vipc_sources) +print(f'Building Vision IPC with {vipc_objects}') + visionipc = env.Library('visionipc', vipc_objects) vipc_frameworks = [] -vipc_libs = envCython["LIBS"] + [visionipc, messaging, common, "zmq"] +vipc_libs = envCython["LIBS"] + [visionipc, msgq, common, "zmq"] if arch == "Darwin": vipc_frameworks.append('OpenCL') else: @@ -43,9 +43,9 @@ envCython.Program(f'{visionipc_dir.abspath}/visionipc_pyx.so', f'{visionipc_dir. LIBS=vipc_libs, FRAMEWORKS=vipc_frameworks) if GetOption('extras'): - env.Program('messaging/test_runner', ['messaging/test_runner.cc', 'messaging/msgq_tests.cc'], LIBS=[messaging, common]) - env.Program('visionipc/test_runner', - ['visionipc/test_runner.cc', 'visionipc/visionipc_tests.cc'], + env.Program('msgq/test_runner', ['msgq/test_runner.cc', 'msgq/msgq_tests.cc'], LIBS=[msgq, common]) + env.Program(f'{visionipc_dir.abspath}/test_runner', + [f'{visionipc_dir.abspath}/test_runner.cc', f'{visionipc_dir.abspath}/visionipc_tests.cc'], LIBS=['pthread'] + vipc_libs, FRAMEWORKS=vipc_frameworks) -Export('visionipc', 'messaging', 'messaging_python') +Export('visionipc', 'msgq', 'msgq_python') diff --git a/SConstruct b/SConstruct index 408c7bfa9..c1e7fc521 100644 --- a/SConstruct +++ b/SConstruct @@ -11,7 +11,8 @@ if platform.system() == "Darwin": common = '' cpppath = [ - f"#/../", + f"#/", + '#msgq/', '/usr/lib/include', '/opt/homebrew/include', sysconfig.get_paths()['include'], diff --git a/messaging/.gitignore b/messaging/.gitignore deleted file mode 100644 index 9f0c6fe7e..000000000 --- a/messaging/.gitignore +++ /dev/null @@ -1 +0,0 @@ -messaging_pyx.cpp diff --git a/msgq/.gitignore b/msgq/.gitignore new file mode 100644 index 000000000..6bd751773 --- /dev/null +++ b/msgq/.gitignore @@ -0,0 +1 @@ +ipc_pyx.cpp diff --git a/messaging/__init__.py b/msgq/__init__.py similarity index 87% rename from messaging/__init__.py rename to msgq/__init__.py index bf6b1e882..340a1254e 100644 --- a/messaging/__init__.py +++ b/msgq/__init__.py @@ -1,12 +1,12 @@ # must be built with scons -from msgq.messaging.messaging_pyx import Context, Poller, SubSocket, PubSocket, SocketEventHandle, toggle_fake_events, \ +from msgq.ipc_pyx import Context, Poller, SubSocket, PubSocket, SocketEventHandle, toggle_fake_events, \ set_fake_prefix, get_fake_prefix, delete_fake_prefix, wait_for_one_event -from msgq.messaging.messaging_pyx import MultiplePublishersError, MessagingError +from msgq.ipc_pyx import MultiplePublishersError, IpcError from typing import Optional, List assert MultiplePublishersError -assert MessagingError +assert IpcError assert toggle_fake_events assert set_fake_prefix assert get_fake_prefix diff --git a/messaging/event.cc b/msgq/event.cc similarity index 99% rename from messaging/event.cc rename to msgq/event.cc index 0c22e7868..be782e3b8 100644 --- a/messaging/event.cc +++ b/msgq/event.cc @@ -13,7 +13,7 @@ #include #include -#include "msgq/messaging/event.h" +#include "msgq/event.h" #ifndef __APPLE__ #include diff --git a/messaging/event.h b/msgq/event.h similarity index 100% rename from messaging/event.h rename to msgq/event.h diff --git a/messaging/impl_fake.cc b/msgq/impl_fake.cc similarity index 82% rename from messaging/impl_fake.cc rename to msgq/impl_fake.cc index 8399ba5f8..a7607206f 100644 --- a/messaging/impl_fake.cc +++ b/msgq/impl_fake.cc @@ -1,4 +1,4 @@ -#include "msgq/messaging/impl_fake.h" +#include "msgq/impl_fake.h" void FakePoller::registerSocket(SubSocket *socket) { this->sockets.push_back(socket); diff --git a/messaging/impl_fake.h b/msgq/impl_fake.h similarity index 95% rename from messaging/impl_fake.h rename to msgq/impl_fake.h index ae40a037f..2c3b74dc9 100644 --- a/messaging/impl_fake.h +++ b/msgq/impl_fake.h @@ -11,8 +11,8 @@ #include #include -#include "msgq/messaging/messaging.h" -#include "msgq/messaging/event.h" +#include "msgq/ipc.h" +#include "msgq/event.h" template class FakeSubSocket: public TSubSocket { diff --git a/messaging/impl_msgq.cc b/msgq/impl_msgq.cc similarity index 99% rename from messaging/impl_msgq.cc rename to msgq/impl_msgq.cc index 429642b87..b23991351 100644 --- a/messaging/impl_msgq.cc +++ b/msgq/impl_msgq.cc @@ -5,7 +5,7 @@ #include #include -#include "msgq/messaging/impl_msgq.h" +#include "msgq/impl_msgq.h" volatile sig_atomic_t msgq_do_exit = 0; diff --git a/messaging/impl_msgq.h b/msgq/impl_msgq.h similarity index 95% rename from messaging/impl_msgq.h rename to msgq/impl_msgq.h index 95f13799b..4d9db1850 100644 --- a/messaging/impl_msgq.h +++ b/msgq/impl_msgq.h @@ -3,8 +3,8 @@ #include #include -#include "msgq/messaging/messaging.h" -#include "msgq/messaging/msgq.h" +#include "msgq/ipc.h" +#include "msgq/msgq.h" #define MAX_POLLERS 128 diff --git a/messaging/impl_zmq.cc b/msgq/impl_zmq.cc similarity index 98% rename from messaging/impl_zmq.cc rename to msgq/impl_zmq.cc index 7e45f2bd9..434888eb8 100644 --- a/messaging/impl_zmq.cc +++ b/msgq/impl_zmq.cc @@ -5,7 +5,7 @@ #include #include -#include "msgq/messaging/impl_zmq.h" +#include "msgq/impl_zmq.h" //FIXME: This is a hack to get the port number from the socket name, might have collisions static int get_port(std::string endpoint) { diff --git a/messaging/impl_zmq.h b/msgq/impl_zmq.h similarity index 97% rename from messaging/impl_zmq.h rename to msgq/impl_zmq.h index 93ebfb5d4..718196e30 100644 --- a/messaging/impl_zmq.h +++ b/msgq/impl_zmq.h @@ -4,7 +4,7 @@ #include #include -#include "msgq/messaging/messaging.h" +#include "msgq/ipc.h" #define MAX_POLLERS 128 diff --git a/messaging/messaging.cc b/msgq/ipc.cc similarity index 94% rename from messaging/messaging.cc rename to msgq/ipc.cc index 5119816b8..f620f46fa 100644 --- a/messaging/messaging.cc +++ b/msgq/ipc.cc @@ -1,10 +1,10 @@ #include #include -#include "msgq/messaging/messaging.h" -#include "msgq/messaging/impl_zmq.h" -#include "msgq/messaging/impl_msgq.h" -#include "msgq/messaging/impl_fake.h" +#include "msgq/ipc.h" +#include "msgq/impl_zmq.h" +#include "msgq/impl_msgq.h" +#include "msgq/impl_fake.h" #ifdef __APPLE__ const bool MUST_USE_ZMQ = true; diff --git a/messaging/messaging.h b/msgq/ipc.h similarity index 100% rename from messaging/messaging.h rename to msgq/ipc.h diff --git a/messaging/messaging.pxd b/msgq/ipc.pxd similarity index 93% rename from messaging/messaging.pxd rename to msgq/ipc.pxd index 420fc09b5..2c7ac963e 100644 --- a/messaging/messaging.pxd +++ b/msgq/ipc.pxd @@ -6,7 +6,7 @@ from libcpp.vector cimport vector from libcpp cimport bool -cdef extern from "msgq/messaging/impl_fake.h": +cdef extern from "msgq/impl_fake.h": cdef cppclass Event: @staticmethod int wait_for_one(vector[Event], int) except + @@ -34,7 +34,7 @@ cdef extern from "msgq/messaging/impl_fake.h": Event recv_ready() -cdef extern from "msgq/messaging/messaging.h": +cdef extern from "msgq/ipc.h": cdef cppclass Context: @staticmethod Context * create() diff --git a/messaging/messaging_pyx.pyx b/msgq/ipc_pyx.pyx similarity index 89% rename from messaging/messaging_pyx.pyx rename to msgq/ipc_pyx.pyx index 8216aeea9..d8797f395 100644 --- a/messaging/messaging_pyx.pyx +++ b/msgq/ipc_pyx.pyx @@ -10,22 +10,22 @@ from libc.string cimport strerror from cython.operator import dereference -from .messaging cimport Context as cppContext -from .messaging cimport SubSocket as cppSubSocket -from .messaging cimport PubSocket as cppPubSocket -from .messaging cimport Poller as cppPoller -from .messaging cimport Message as cppMessage -from .messaging cimport Event as cppEvent, SocketEventHandle as cppSocketEventHandle +from .ipc cimport Context as cppContext +from .ipc cimport SubSocket as cppSubSocket +from .ipc cimport PubSocket as cppPubSocket +from .ipc cimport Poller as cppPoller +from .ipc cimport Message as cppMessage +from .ipc cimport Event as cppEvent, SocketEventHandle as cppSocketEventHandle -class MessagingError(Exception): +class IpcError(Exception): def __init__(self, endpoint=None): suffix = f"with {endpoint.decode('utf-8')}" if endpoint else "" message = f"Messaging failure {suffix}: {strerror(errno.errno).decode('utf-8')}" super().__init__(message) -class MultiplePublishersError(MessagingError): +class MultiplePublishersError(IpcError): pass @@ -170,7 +170,7 @@ cdef class SubSocket: self.is_owner = True if self.socket == NULL: - raise MessagingError + raise IpcError def __dealloc__(self): if self.is_owner: @@ -190,7 +190,7 @@ cdef class SubSocket: if errno.errno == errno.EADDRINUSE: raise MultiplePublishersError(endpoint) else: - raise MessagingError(endpoint) + raise IpcError(endpoint) def setTimeout(self, int timeout): self.socket.setTimeout(timeout) @@ -219,7 +219,7 @@ cdef class PubSocket: def __cinit__(self): self.socket = cppPubSocket.create() if self.socket == NULL: - raise MessagingError + raise IpcError def __dealloc__(self): del self.socket @@ -231,7 +231,7 @@ cdef class PubSocket: if errno.errno == errno.EADDRINUSE: raise MultiplePublishersError(endpoint) else: - raise MessagingError(endpoint) + raise IpcError(endpoint) def send(self, bytes data): length = len(data) @@ -241,7 +241,7 @@ cdef class PubSocket: if errno.errno == errno.EADDRINUSE: raise MultiplePublishersError else: - raise MessagingError + raise IpcError def all_readers_updated(self): return self.socket.all_readers_updated() diff --git a/logger/logger.h b/msgq/logger/logger.h similarity index 100% rename from logger/logger.h rename to msgq/logger/logger.h diff --git a/messaging/msgq.cc b/msgq/msgq.cc similarity index 99% rename from messaging/msgq.cc rename to msgq/msgq.cc index 9344652f0..fed2959bd 100644 --- a/messaging/msgq.cc +++ b/msgq/msgq.cc @@ -23,7 +23,7 @@ #include -#include "msgq/messaging/msgq.h" +#include "msgq/msgq.h" void sigusr2_handler(int signal) { assert(signal == SIGUSR2); diff --git a/messaging/msgq.h b/msgq/msgq.h similarity index 100% rename from messaging/msgq.h rename to msgq/msgq.h diff --git a/messaging/msgq_tests.cc b/msgq/msgq_tests.cc similarity index 99% rename from messaging/msgq_tests.cc rename to msgq/msgq_tests.cc index bc3984e62..02f17917a 100644 --- a/messaging/msgq_tests.cc +++ b/msgq/msgq_tests.cc @@ -1,5 +1,5 @@ #include "catch2/catch.hpp" -#include "msgq/messaging/msgq.h" +#include "msgq/msgq.h" TEST_CASE("ALIGN") { diff --git a/messaging/test_runner.cc b/msgq/test_runner.cc similarity index 100% rename from messaging/test_runner.cc rename to msgq/test_runner.cc diff --git a/messaging/tests/__init__.py b/msgq/tests/__init__.py similarity index 100% rename from messaging/tests/__init__.py rename to msgq/tests/__init__.py diff --git a/messaging/tests/test_fake.py b/msgq/tests/test_fake.py similarity index 78% rename from messaging/tests/test_fake.py rename to msgq/tests/test_fake.py index d17065565..b5ed297ab 100644 --- a/messaging/tests/test_fake.py +++ b/msgq/tests/test_fake.py @@ -2,11 +2,10 @@ import unittest import multiprocessing import platform +import msgq from parameterized import parameterized_class from typing import Optional -import msgq.messaging as messaging - WAIT_TIMEOUT = 5 @@ -14,7 +13,7 @@ class TestEvents(unittest.TestCase): def test_mutation(self): - handle = messaging.fake_event_handle("carState") + handle = msgq.fake_event_handle("carState") event = handle.recv_called_event self.assertFalse(event.peek()) @@ -26,7 +25,7 @@ def test_mutation(self): del event def test_wait(self): - handle = messaging.fake_event_handle("carState") + handle = msgq.fake_event_handle("carState") event = handle.recv_called_event event.set() @@ -37,7 +36,7 @@ def test_wait(self): self.fail("event.wait() timed out") def test_wait_multiprocess(self): - handle = messaging.fake_event_handle("carState") + handle = msgq.fake_event_handle("carState") event = handle.recv_called_event def set_event_run(): @@ -54,7 +53,7 @@ def set_event_run(): p.kill() def test_wait_zero_timeout(self): - handle = messaging.fake_event_handle("carState") + handle = msgq.fake_event_handle("carState") event = handle.recv_called_event try: @@ -71,18 +70,18 @@ class TestFakeSockets(unittest.TestCase): prefix: Optional[str] = None def setUp(self): - messaging.toggle_fake_events(True) + msgq.toggle_fake_events(True) if self.prefix is not None: - messaging.set_fake_prefix(self.prefix) + msgq.set_fake_prefix(self.prefix) else: - messaging.delete_fake_prefix() + msgq.delete_fake_prefix() def tearDown(self): - messaging.toggle_fake_events(False) - messaging.delete_fake_prefix() + msgq.toggle_fake_events(False) + msgq.delete_fake_prefix() def test_event_handle_init(self): - handle = messaging.fake_event_handle("controlsState", override=True) + handle = msgq.fake_event_handle("controlsState", override=True) self.assertFalse(handle.enabled) self.assertGreaterEqual(handle.recv_called_event.fd, 0) @@ -90,9 +89,9 @@ def test_event_handle_init(self): def test_non_managed_socket_state(self): # non managed socket should have zero state - _ = messaging.pub_sock("ubloxGnss") + _ = msgq.pub_sock("ubloxGnss") - handle = messaging.fake_event_handle("ubloxGnss", override=False) + handle = msgq.fake_event_handle("ubloxGnss", override=False) self.assertFalse(handle.enabled) self.assertEqual(handle.recv_called_event.fd, 0) @@ -100,26 +99,26 @@ def test_non_managed_socket_state(self): def test_managed_socket_state(self): # managed socket should not change anything about the state - handle = messaging.fake_event_handle("ubloxGnss") + handle = msgq.fake_event_handle("ubloxGnss") handle.enabled = True expected_enabled = handle.enabled expected_recv_called_fd = handle.recv_called_event.fd expected_recv_ready_fd = handle.recv_ready_event.fd - _ = messaging.pub_sock("ubloxGnss") + _ = msgq.pub_sock("ubloxGnss") self.assertEqual(handle.enabled, expected_enabled) self.assertEqual(handle.recv_called_event.fd, expected_recv_called_fd) self.assertEqual(handle.recv_ready_event.fd, expected_recv_ready_fd) def test_sockets_enable_disable(self): - carState_handle = messaging.fake_event_handle("ubloxGnss", enable=True) + carState_handle = msgq.fake_event_handle("ubloxGnss", enable=True) recv_called = carState_handle.recv_called_event recv_ready = carState_handle.recv_ready_event - pub_sock = messaging.pub_sock("ubloxGnss") - sub_sock = messaging.sub_sock("ubloxGnss") + pub_sock = msgq.pub_sock("ubloxGnss") + sub_sock = msgq.sub_sock("ubloxGnss") try: carState_handle.enabled = True @@ -139,8 +138,8 @@ def test_sockets_enable_disable(self): def test_synced_pub_sub(self): def daemon_repub_process_run(): - pub_sock = messaging.pub_sock("ubloxGnss") - sub_sock = messaging.sub_sock("carState") + pub_sock = msgq.pub_sock("ubloxGnss") + sub_sock = msgq.sub_sock("carState") frame = -1 while True: @@ -153,15 +152,15 @@ def daemon_repub_process_run(): bts = frame.to_bytes(8, 'little') pub_sock.send(bts) - carState_handle = messaging.fake_event_handle("carState", enable=True) + carState_handle = msgq.fake_event_handle("carState", enable=True) recv_called = carState_handle.recv_called_event recv_ready = carState_handle.recv_ready_event p = multiprocessing.Process(target=daemon_repub_process_run) p.start() - pub_sock = messaging.pub_sock("carState") - sub_sock = messaging.sub_sock("ubloxGnss") + pub_sock = msgq.pub_sock("carState") + sub_sock = msgq.sub_sock("ubloxGnss") try: for i in range(10): diff --git a/messaging/tests/test_messaging.py b/msgq/tests/test_messaging.py similarity index 83% rename from messaging/tests/test_messaging.py rename to msgq/tests/test_messaging.py index d2e30b61f..bbeeb3d84 100755 --- a/messaging/tests/test_messaging.py +++ b/msgq/tests/test_messaging.py @@ -5,8 +5,7 @@ import time import string import unittest - -import msgq.messaging as messaging +import msgq def random_sock(): @@ -39,8 +38,8 @@ def setUp(self): def test_pub_sub(self): sock = random_sock() - pub_sock = messaging.pub_sock(sock) - sub_sock = messaging.sub_sock(sock, conflate=False, timeout=None) + pub_sock = msgq.pub_sock(sock) + sub_sock = msgq.sub_sock(sock, conflate=False, timeout=None) zmq_sleep(3) for _ in range(1000): @@ -51,11 +50,11 @@ def test_pub_sub(self): def test_conflate(self): sock = random_sock() - pub_sock = messaging.pub_sock(sock) + pub_sock = msgq.pub_sock(sock) for conflate in [True, False]: for _ in range(10): num_msgs = random.randint(3, 10) - sub_sock = messaging.sub_sock(sock, conflate=conflate, timeout=None) + sub_sock = msgq.sub_sock(sock, conflate=conflate, timeout=None) zmq_sleep() sent_msgs = [] @@ -64,7 +63,7 @@ def test_conflate(self): pub_sock.send(msg) sent_msgs.append(msg) time.sleep(0.1) - recvd_msgs = messaging.drain_sock_raw(sub_sock) + recvd_msgs = msgq.drain_sock_raw(sub_sock) if conflate: self.assertEqual(len(recvd_msgs), 1) else: @@ -75,7 +74,7 @@ def test_receive_timeout(self): sock = random_sock() for _ in range(10): timeout = random.randrange(200) - sub_sock = messaging.sub_sock(sock, timeout=timeout) + sub_sock = msgq.sub_sock(sock, timeout=timeout) zmq_sleep() start_time = time.monotonic() diff --git a/messaging/tests/test_poller.py b/msgq/tests/test_poller.py similarity index 75% rename from messaging/tests/test_poller.py rename to msgq/tests/test_poller.py index 5ac990e31..a68ff4fe7 100644 --- a/messaging/tests/test_poller.py +++ b/msgq/tests/test_poller.py @@ -1,16 +1,16 @@ import unittest import time -import msgq.messaging as messaging +import msgq import concurrent.futures SERVICE_NAME = 'myService' def poller(): - context = messaging.Context() + context = msgq.Context() - p = messaging.Poller() + p = msgq.Poller() - sub = messaging.SubSocket() + sub = msgq.SubSocket() sub.connect(context, SERVICE_NAME) p.registerSocket(sub) @@ -22,9 +22,9 @@ def poller(): class TestPoller(unittest.TestCase): def test_poll_once(self): - context = messaging.Context() + context = msgq.Context() - pub = messaging.PubSocket() + pub = msgq.PubSocket() pub.connect(context, SERVICE_NAME) with concurrent.futures.ThreadPoolExecutor() as e: @@ -44,18 +44,18 @@ def test_poll_once(self): self.assertEqual(result, [b"a"]) def test_poll_and_create_many_subscribers(self): - context = messaging.Context() + context = msgq.Context() - pub = messaging.PubSocket() + pub = msgq.PubSocket() pub.connect(context, SERVICE_NAME) with concurrent.futures.ThreadPoolExecutor() as e: poll = e.submit(poller) time.sleep(0.1) # Slow joiner syndrome - c = messaging.Context() + c = msgq.Context() for _ in range(10): - messaging.SubSocket().connect(c, SERVICE_NAME) + msgq.SubSocket().connect(c, SERVICE_NAME) time.sleep(0.1) @@ -71,13 +71,13 @@ def test_poll_and_create_many_subscribers(self): self.assertEqual(result, [b"a"]) def test_multiple_publishers_exception(self): - context = messaging.Context() + context = msgq.Context() - with self.assertRaises(messaging.MultiplePublishersError): - pub1 = messaging.PubSocket() + with self.assertRaises(msgq.MultiplePublishersError): + pub1 = msgq.PubSocket() pub1.connect(context, SERVICE_NAME) - pub2 = messaging.PubSocket() + pub2 = msgq.PubSocket() pub2.connect(context, SERVICE_NAME) pub1.send(b"a") @@ -87,12 +87,12 @@ def test_multiple_publishers_exception(self): context.term() def test_multiple_messages(self): - context = messaging.Context() + context = msgq.Context() - pub = messaging.PubSocket() + pub = msgq.PubSocket() pub.connect(context, SERVICE_NAME) - sub = messaging.SubSocket() + sub = msgq.SubSocket() sub.connect(context, SERVICE_NAME) time.sleep(0.1) # Slow joiner @@ -119,12 +119,12 @@ def test_multiple_messages(self): context.term() def test_conflate(self): - context = messaging.Context() + context = msgq.Context() - pub = messaging.PubSocket() + pub = msgq.PubSocket() pub.connect(context, SERVICE_NAME) - sub = messaging.SubSocket() + sub = msgq.SubSocket() sub.connect(context, SERVICE_NAME, conflate=True) time.sleep(0.1) # Slow joiner diff --git a/visionipc/.gitignore b/msgq/visionipc/.gitignore similarity index 100% rename from visionipc/.gitignore rename to msgq/visionipc/.gitignore diff --git a/visionipc/__init__.py b/msgq/visionipc/__init__.py similarity index 100% rename from visionipc/__init__.py rename to msgq/visionipc/__init__.py diff --git a/visionipc/test_runner.cc b/msgq/visionipc/test_runner.cc similarity index 100% rename from visionipc/test_runner.cc rename to msgq/visionipc/test_runner.cc diff --git a/visionipc/tests/__init__.py b/msgq/visionipc/tests/__init__.py similarity index 100% rename from visionipc/tests/__init__.py rename to msgq/visionipc/tests/__init__.py diff --git a/visionipc/tests/test_visionipc.py b/msgq/visionipc/tests/test_visionipc.py similarity index 100% rename from visionipc/tests/test_visionipc.py rename to msgq/visionipc/tests/test_visionipc.py diff --git a/visionipc/visionbuf.cc b/msgq/visionipc/visionbuf.cc similarity index 100% rename from visionipc/visionbuf.cc rename to msgq/visionipc/visionbuf.cc diff --git a/visionipc/visionbuf.h b/msgq/visionipc/visionbuf.h similarity index 100% rename from visionipc/visionbuf.h rename to msgq/visionipc/visionbuf.h diff --git a/visionipc/visionbuf_cl.cc b/msgq/visionipc/visionbuf_cl.cc similarity index 100% rename from visionipc/visionbuf_cl.cc rename to msgq/visionipc/visionbuf_cl.cc diff --git a/visionipc/visionbuf_ion.cc b/msgq/visionipc/visionbuf_ion.cc similarity index 100% rename from visionipc/visionbuf_ion.cc rename to msgq/visionipc/visionbuf_ion.cc diff --git a/visionipc/ipc.cc b/msgq/visionipc/visionipc.cc similarity index 98% rename from visionipc/ipc.cc rename to msgq/visionipc/visionipc.cc index 30f7d21d4..48e13c27d 100644 --- a/visionipc/ipc.cc +++ b/msgq/visionipc/visionipc.cc @@ -15,7 +15,7 @@ #define getsocket() socket(AF_UNIX, SOCK_SEQPACKET, 0) #endif -#include "msgq/visionipc/ipc.h" +#include "msgq/visionipc/visionipc.h" int ipc_connect(const char* socket_path) { int err; diff --git a/visionipc/visionipc.h b/msgq/visionipc/visionipc.h similarity index 57% rename from visionipc/visionipc.h rename to msgq/visionipc/visionipc.h index 7489bc958..224f129c9 100644 --- a/visionipc/visionipc.h +++ b/msgq/visionipc/visionipc.h @@ -3,6 +3,12 @@ #include #include + +int ipc_connect(const char* socket_path); +int ipc_bind(const char* socket_path); +int ipc_sendrecv_with_fds(bool send, int fd, void *buf, size_t buf_size, int* fds, int num_fds, + int *out_num_fds); + constexpr int VISIONIPC_MAX_FDS = 128; struct VisionIpcBufExtra { diff --git a/visionipc/visionipc.pxd b/msgq/visionipc/visionipc.pxd similarity index 100% rename from visionipc/visionipc.pxd rename to msgq/visionipc/visionipc.pxd diff --git a/visionipc/visionipc_client.cc b/msgq/visionipc/visionipc_client.cc similarity index 97% rename from visionipc/visionipc_client.cc rename to msgq/visionipc/visionipc_client.cc index ab1bfa3ae..9b24da296 100644 --- a/visionipc/visionipc_client.cc +++ b/msgq/visionipc/visionipc_client.cc @@ -4,11 +4,11 @@ #include #include -#include "msgq/visionipc/ipc.h" +#include "msgq/visionipc/visionipc.h" #include "msgq/visionipc/visionipc_client.h" #include "msgq/visionipc/visionipc_server.h" -#include "msgq/logger/logger.h" -#include "msgq/logger/logger.h" +#include "logger/logger.h" +#include "logger/logger.h" static int connect_to_vipc_server(const std::string &name, bool blocking) { const std::string ipc_path = get_ipc_path(name); diff --git a/visionipc/visionipc_client.h b/msgq/visionipc/visionipc_client.h similarity index 95% rename from visionipc/visionipc_client.h rename to msgq/visionipc/visionipc_client.h index e03677867..e4abdc5de 100644 --- a/visionipc/visionipc_client.h +++ b/msgq/visionipc/visionipc_client.h @@ -3,7 +3,7 @@ #include #include -#include "msgq/messaging/messaging.h" +#include "msgq/ipc.h" #include "msgq/visionipc/visionbuf.h" diff --git a/visionipc/visionipc_pyx.pxd b/msgq/visionipc/visionipc_pyx.pxd similarity index 100% rename from visionipc/visionipc_pyx.pxd rename to msgq/visionipc/visionipc_pyx.pxd diff --git a/visionipc/visionipc_pyx.pyx b/msgq/visionipc/visionipc_pyx.pyx similarity index 100% rename from visionipc/visionipc_pyx.pyx rename to msgq/visionipc/visionipc_pyx.pyx diff --git a/visionipc/visionipc_server.cc b/msgq/visionipc/visionipc_server.cc similarity index 98% rename from visionipc/visionipc_server.cc rename to msgq/visionipc/visionipc_server.cc index e2ac55e64..611d10b20 100644 --- a/visionipc/visionipc_server.cc +++ b/msgq/visionipc/visionipc_server.cc @@ -8,10 +8,10 @@ #include #include -#include "msgq/messaging/messaging.h" -#include "msgq/visionipc/ipc.h" +#include "msgq/ipc.h" +#include "msgq/visionipc/visionipc.h" #include "msgq/visionipc/visionipc_server.h" -#include "msgq/logger/logger.h" +#include "logger/logger.h" std::string get_endpoint_name(std::string name, VisionStreamType type){ if (messaging_use_zmq()){ diff --git a/visionipc/visionipc_server.h b/msgq/visionipc/visionipc_server.h similarity index 97% rename from visionipc/visionipc_server.h rename to msgq/visionipc/visionipc_server.h index 4207f41cd..feacc4d10 100644 --- a/visionipc/visionipc_server.h +++ b/msgq/visionipc/visionipc_server.h @@ -5,7 +5,7 @@ #include #include -#include "msgq/messaging/messaging.h" +#include "msgq/ipc.h" #include "msgq/visionipc/visionbuf.h" std::string get_endpoint_name(std::string name, VisionStreamType type); diff --git a/visionipc/visionipc_tests.cc b/msgq/visionipc/visionipc_tests.cc similarity index 100% rename from visionipc/visionipc_tests.cc rename to msgq/visionipc/visionipc_tests.cc diff --git a/visionipc/ipc.h b/visionipc/ipc.h deleted file mode 100644 index 14bb61a52..000000000 --- a/visionipc/ipc.h +++ /dev/null @@ -1,7 +0,0 @@ -#pragma once -#include - -int ipc_connect(const char* socket_path); -int ipc_bind(const char* socket_path); -int ipc_sendrecv_with_fds(bool send, int fd, void *buf, size_t buf_size, int* fds, int num_fds, - int *out_num_fds); From eabd20c91986a25569d8e89057710e1efab4cc8a Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Tue, 11 Jun 2024 11:21:08 -0700 Subject: [PATCH 09/10] remove prints in sconscript[ --- SConscript | 3 --- 1 file changed, 3 deletions(-) diff --git a/SConscript b/SConscript index 7c0eb5a43..147eb3042 100644 --- a/SConscript +++ b/SConscript @@ -26,10 +26,7 @@ if arch == "larch64": else: vipc_sources += [f'{visionipc_dir.abspath}/visionbuf_cl.cc'] -print(f'Building Vision IPC with {vipc_sources}') vipc_objects = env.SharedObject(vipc_sources) -print(f'Building Vision IPC with {vipc_objects}') - visionipc = env.Library('visionipc', vipc_objects) From 74074d650f5d516a33962c1681a2a15b1d603537 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Tue, 11 Jun 2024 16:04:35 -0700 Subject: [PATCH 10/10] msgq: bump max readers (#623) --- msgq/msgq.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgq/msgq.h b/msgq/msgq.h index 0a72a3864..94e184944 100644 --- a/msgq/msgq.h +++ b/msgq/msgq.h @@ -6,7 +6,7 @@ #include #define DEFAULT_SEGMENT_SIZE (10 * 1024 * 1024) -#define NUM_READERS 12 +#define NUM_READERS 15 #define ALIGN(n) ((n + (8 - 1)) & -8) #define UNUSED(x) (void)x