Merge branch 'commaai:master' into PA-testing

.github/labeler.yaml

@@ -44,7 +44,7 @@ subaru:
 
 tesla:
  - changed-files:
- - any-glob-to-all-files: 'selfdrive/car/telsa/*'
+ - any-glob-to-all-files: 'selfdrive/car/tesla/*'
 
 toyota:
  - changed-files:

.github/workflows/ci_weekly_report.yaml

@@ -10,38 +10,30 @@ concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true
 
+env:
+ CI_RUNS: ${{ github.event.inputs.ci_runs || '50' }}
+
 jobs:
  setup:
  if: github.repository == 'commaai/openpilot'
  runs-on: ubuntu-latest
  outputs:
- ci_runs: ${{ steps.ci_runs_setup.outputs.value }}
+ ci_runs: ${{ steps.ci_runs_setup.outputs.matrix }}
  steps:
  - id: ci_runs_setup
+ name: CI_RUNS=${{ env.CI_RUNS }}
  run: |
- CI_RUNS=${{ inputs.ci_runs || '50' }}
- mylist="value=["
-
- for i in $(seq 1 $CI_RUNS);
- do
- if [ $i != $CI_RUNS ]; then
- mylist+="\"$i\", "
- else
- mylist+="\"$i\"]"
- fi
- done
+ matrix=$(python3 -c "import json; print(json.dumps({ 'run_number' : list(range(${{ env.CI_RUNS }})) }))")
+ echo "matrix=$matrix" >> $GITHUB_OUTPUT
 
- echo "$mylist" >> $GITHUB_OUTPUT
- echo "Number of CI runs for report: $CI_RUNS"
  ci_matrix_run:
  needs: [ setup ]
  strategy:
  fail-fast: false
- matrix:
- value: ${{fromJSON(needs.setup.outputs.ci_runs)}}
+ matrix: ${{fromJSON(needs.setup.outputs.ci_runs)}}
  uses: commaai/openpilot/.github/workflows/ci_weekly_run.yaml@master
  with:
- run_number: ${{ matrix.value }}
+ run_number: ${{ matrix.run_number }}
 
  report:
  needs: [ci_matrix_run]
@@ -62,62 +54,48 @@ jobs:
  })
  var report = {}
  jobs.slice(1, jobs.length-1).forEach(job => {
- const jobName = job.name.split('/')[2].trim();
+ if (job.conclusion === "skipped") return;
+ const jobName = job.name.split(" / ")[2];
+ const runRegex = /\((.*?)\)/;
+ const run = job.name.match(runRegex)[1];
  report[jobName] = report[jobName] || { successes: [], failures: [], cancelled: [] };
  switch (job.conclusion) {
  case "success":
- report[jobName].successes.push(job.html_url); break;
+ report[jobName].successes.push({ "run_number": run, "link": job.html_url}); break;
  case "failure":
- report[jobName].failures.push(job.html_url); break;
+ report[jobName].failures.push({ "run_number": run, "link": job.html_url }); break;
  case "cancelled":
- report[jobName].cancelled.push(job.html_url); break;
+ report[jobName].cancelled.push({ "run_number": run, "link": job.html_url }); break;
  }
  });
- return JSON.stringify(report);
+ return JSON.stringify({"jobs": report});
 
  - name: Add job results to summary
  env:
  JOB_RESULTS: ${{ fromJSON(steps.get-job-results.outputs.result) }}
  run: |
- echo $JOB_RESULTS > job_results.json
- generate_html_table() {
- echo "<table>"
- echo "<thead>"
- echo " <tr>"
- echo " <th>Job</th>"
- echo " <th>Succeeded ✅</th>"
- echo " <th>Failed ❌</th>"
- echo " <th>Cancelled (timed out) ⏰</th>"
- echo " </tr>"
- echo "</thead>"
- jq -r '
- "<tbody>",
- keys[] as $job |
- "<tr>",
- " <td>\($job)</td>",
- " <td>",
- " <details>",
- " <summary>(\(.[$job].successes | length))</summary>",
- " \(.[$job].successes[])<br>",
- " </details>",
- " </td>",
- " <td>",
- " <details>",
- " <summary>(\(.[$job].failures | length))</summary>",
- " \(.[$job].failures[])<br>",
- " </details>",
- " </td>",
- " <td>",
- " <details>",
- " <summary>(\(.[$job].cancelled | length))</summary>",
- " \(.[$job].cancelled[])<br>",
- " </details>",
- " </td>",
- "</tr>"
- ' job_results.json
- echo "</tbody>"
- echo "</table>"
- }
- echo "# CI Job Summary" >> $GITHUB_STEP_SUMMARY
- generate_html_table >> $GITHUB_STEP_SUMMARY
+ cat <<EOF >> template.html
+ <table>
+ <thead>
+ <tr>
+ <th></th>
+ <th>Job</th>
+ <th>✅ Passing</th>
+ <th>❌ Failure Details</th>
+ </tr>
+ </thead>
+ <tbody>
+ {% for key in jobs.keys() %}<tr>
+ <td>{% for i in range(5) %}{% if i+1 <= (5 * jobs[key]["successes"]|length // ${{ env.CI_RUNS }}) %}🟩{% else %}🟥{% endif %}{% endfor%}</td>
+ <td>{{ key }}</td>
+ <td>{{ 100 * jobs[key]["successes"]|length // ${{ env.CI_RUNS }} }}%</td>
+ <td>{% if jobs[key]["failures"]|length > 0 %}<details>{% for failure in jobs[key]["failures"] %}<a href="{{ failure['link'] }}">Log for run #{{ failure['run_number'] }}</a><br>{% endfor %}</details>{% else %}{% endif %}</td>
+ </td>
+ </tr>{% endfor %}
+ </table>
+ EOF
 
+ pip install jinja2-cli
+ echo $JOB_RESULTS | jinja2 template.html > report.html
+ echo "# CI Test Report - ${{ env.CI_RUNS }} Runs" >> $GITHUB_STEP_SUMMARY
+ cat report.html >> $GITHUB_STEP_SUMMARY

.github/workflows/docs.yaml

@@ -15,30 +15,25 @@ concurrency:
  group: docs-tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
  cancel-in-progress: true
 
-env:
- BASE_IMAGE: openpilot-base
-
- BUILD: selfdrive/test/docker_build.sh base
-
- RUN: docker run --shm-size 1G -v $GITHUB_WORKSPACE:/tmp/openpilot -w /tmp/openpilot -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
-
 jobs:
  docs:
  name: build docs
  runs-on: ubuntu-latest
- timeout-minutes: 45
+ timeout-minutes: 1
  steps:
  - uses: actions/checkout@v4
  with:
  submodules: true
- - uses: ./.github/workflows/setup-with-retry
- - name: Build openpilot
- run: |
- ${{ env.RUN }} "scons -j$(nproc)"
+
+ # Build
  - name: Build docs
  run: |
- ${{ env.RUN }} "apt update && apt install -y doxygen && cd docs && make -j$(nproc) html"
+ # TODO: can we install just the "docs" dependency group without the normal deps?
+ pip install mkdocs mkdocs-terminal
+ cd docs
+ mkdocs build
 
+ # Push to docs.comma.ai
  - uses: actions/checkout@v4
  if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
  with:
@@ -53,16 +48,17 @@ jobs:
  source release/identity.sh
 
  cd openpilot-docs
-
  git checkout --orphan tmp
  git rm -rf .
 
- cp -r ../build/docs/html/ docs/
- cp -r ../docs/README.md .
+ # copy over docs
+ cp -r ../docs/site/ docs/
+
+ # GitHub pages config
  touch docs/.nojekyll
  echo -n docs.comma.ai > docs/CNAME
- git add -f .
 
+ git add -f .
  git commit -m "build docs"
 
  # docs live in different repo to not bloat openpilot's full clone size

.github/workflows/selfdrive_tests.yaml

@@ -106,8 +106,17 @@ jobs:
  SKIP_PROMPT: 1
  # package install has DeprecationWarnings
  PYTHONWARNINGS: default
- - name: Test openpilot environment
- run: . .venv/bin/activate && scons -h
+ - run: echo "CACHE_COMMIT_DATE=$(git log -1 --pretty='format:%cd' --date=format:'%Y-%m-%d-%H:%M')" >> $GITHUB_ENV
+ - name: Getting scons cache
+ uses: 'actions/cache@v4'
+ with:
+ path: /tmp/scons_cache
+ key: scons-${{ runner.arch }}-macos-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
+ restore-keys: |
+ scons-${{ runner.arch }}-macos-${{ env.CACHE_COMMIT_DATE }}
+ scons-${{ runner.arch }}-macos
+ - name: Building openpilot
+ run: . .venv/bin/activate && scons -j$(nproc)
 
  static_analysis:
  name: static analysis
@@ -328,3 +337,53 @@ jobs:
  with:
  name: report-${{ inputs.run_number }}
  path: selfdrive/ui/tests/test_ui/report_${{ inputs.run_number }}
+ - name: Get changes to selfdrive/ui
+ if: ${{ github.event_name == 'pull_request' }}
+ id: changed-files
+ uses: tj-actions/changed-files@v44
+ with:
+ files: |
+ selfdrive/ui/**
+ - name: Checkout ci-artifacts
+ if: ${{ github.event_name == 'pull_request' && steps.changed-files.outputs.any_changed == 'true' }}
+ uses: actions/checkout@v4
+ with:
+ repository: commaai/ci-artifacts
+ ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
+ path: ${{ github.workspace }}/ci-artifacts
+ ref: master
+ - name: Push Screenshots
+ if: ${{ github.event_name == 'pull_request' && steps.changed-files.outputs.any_changed == 'true' }}
+ working-directory: ${{ github.workspace }}/ci-artifacts
+ run: |
+ git checkout -b openpilot/pr-${{ github.event.pull_request.number }}
+ git config user.name "GitHub Actions Bot"
+ git config user.email "<>"
+ sudo mv ${{ github.workspace }}/selfdrive/ui/tests/test_ui/report/screenshots/* .
+ git add .
+ git commit -m "screenshots for PR #${{ github.event.pull_request.number }}"
+ git push origin openpilot/pr-${{ github.event.pull_request.number }} --force
+ - name: Comment Screenshots on PR
+ if: ${{ github.event_name == 'pull_request' && steps.changed-files.outputs.any_changed == 'true' }}
+ uses: thollander/actions-comment-pull-request@v2
+ with:
+ message: |
+ <!-- _(run_id_screenshots **${{ github.run_id }}**)_ -->
+ ## UI Screenshots
+ <table>
+ <tr>
+ <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.pull_request.number }}/homescreen.png"></td>
+ <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.pull_request.number }}/onroad.png"></td>
+ </tr>
+ <tr>
+ <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.pull_request.number }}/onroad_map.png"></td>
+ <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.pull_request.number }}/onroad_sidebar.png"></td>
+ </tr>
+ <tr>
+ <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.pull_request.number }}/settings_network.png"></td>
+ <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.pull_request.number }}/settings_device.png"></td>
+ </tr>
+ </table>
+ comment_tag: run_id_screenshots
+ pr_number: ${{ github.event.pull_request.number }}
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.pre-commit-config.yaml

@@ -33,7 +33,7 @@ repos:
  - -L bu,ro,te,ue,alo,hda,ois,nam,nams,ned,som,parm,setts,inout,warmup,bumb,nd,sie,preints,whit,indexIn
  - --builtins clear,rare,informal,usage,code,names,en-GB_to_en-US
 - repo: https://github.com/astral-sh/ruff-pre-commit
- rev: v0.5.0
+ rev: v0.5.1
  hooks:
  - id: ruff
  exclude: '^(third_party/)|(msgq/)|(panda/)|(rednose/)|(rednose_repo/)|(tinygrad/)|(tinygrad_repo/)|(teleoprtc/)|(teleoprtc_repo/)'

cereal/messaging/__init__.py

@@ -17,8 +17,8 @@
 NO_TRAVERSAL_LIMIT = 2**64-1
 
 
-def log_from_bytes(dat: bytes) -> capnp.lib.capnp._DynamicStructReader:
- with log.Event.from_bytes(dat, traversal_limit_in_words=NO_TRAVERSAL_LIMIT) as msg:
+def log_from_bytes(dat: bytes, struct: capnp.lib.capnp._StructModule = log.Event) -> capnp.lib.capnp._DynamicStructReader:
+ with struct.from_bytes(dat, traversal_limit_in_words=NO_TRAVERSAL_LIMIT) as msg:
  return msg