protobuf: automate derived file generation

* Use the bufbuild/buf tool to:
  * Lint the protobuf schema.
  * Check for breaking changes.
  * Configure the generation of derived Python files.
* Adds a GitHub action which automates the checking and generation of
  derived files and commits the results back on pull request branches.

.codacy.yml

@@ -1,4 +1,4 @@
 exclude_paths:
   - 'etc/**'
   - 'tests/**'
-  - 'cylc/flow/**_pb2.py'
+  - 'cylc/flow/network/protobuf/**_pb2.py'

.codecov.yml

@@ -28,7 +28,7 @@ coverage:
 # files to ignore
 ignore:
   - "tests/**"
-  - "ws_messages_pb2.py"
+  - "cylc/flow/network/protobuf/cylc/v5/schema_pb2.py"
   - "cylc/flow/scripts/report_timings.py"
 
 flag_management:

.coveragerc

@@ -14,7 +14,7 @@ disable_warnings =
     module-not-measured
 omit =
     tests/*
-    */cylc/flow/*_pb2.py
+    */cylc/flow/network/protobuf/cylc/v5/*_pb2.py
     cylc/flow/etc/*
     cylc/flow/scripts/report_timings.py
 parallel = True
@@ -43,7 +43,7 @@ fail_under=0
 ignore_errors = False
 omit =
     tests/*
-    */cylc/flow/*_pb2.py
+    */cylc/flow/network/protobuf/cylc/v5/*_pb2.py
     cylc/flow/etc/*
 precision = 2
 show_missing = False

.github/workflows/protobuf.yml

@@ -0,0 +1,83 @@
+name: protobuf
+# CI tests to run against the protobuf schema on change:
+
+on:
+  # run for any PRs raising changes to the protobuf files or setup
+  pull_request:
+    paths:
+      - 'cylc/flow/network/protobuf/**'
+
+jobs:
+  protobuf:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          # we need all of the commits on the PR branch in order to be able to add new ones
+          fetch-depth: 100
+
+      - name: Configure git
+        uses: cylc/release-actions/configure-git@v1
+
+      - name: Install Protobuf
+        uses: mamba-org/setup-micromamba@v1
+        with:
+          # install protobuf into a mamba env (note use shell = "bash -el {0}"
+          # to access this envionment)
+          environment-name: protobuf
+          create-args: protobuf
+          init-shell: bash
+
+      - name: Install bufbuild/buf
+        run: |
+          eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"  # activate homebrew
+
+          # NOTE: bufbuild does exist on conda-forge but hasn't been updated for a while
+          brew install bufbuild/buf/buf
+
+      - name: Lint
+        run: |
+          # lint .proto files
+          eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"  # activate homebrew
+          cd cylc/flow/network/protobuf
+          buf lint
+
+      - name: Compatibility
+        shell: bash -el {0}
+        run: |
+          eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"  # activate homebrew
+          cd cylc/flow/network/protobuf
+          # NOTE: there is currently no process for committing a breaking change.
+          # If a breaking change is needed:
+          # - Increment the Cylc API version number.
+          # - Increment the protobuf schema version number to match.
+          # - Increment the API number filter in cylc-uiserver.
+          # - Bypass this step of the workflow.
+          buf breaking \
+            --against 'https://github.com/cylc/cylc-flow.git#tag=${{ github.base_ref }},subdir=cylc/flow/network/protobuf'
+
+      - name: Build
+        shell: bash -el {0}
+        run: |
+          # generate .py and .pyi files from the .proto files
+          eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"  # activate homebrew
+          micromamba activate protobuf
+          cd cylc/flow/network/protobuf
+          buf generate
+
+      - name: Commit & Push
+        run: |
+          if [[ -z $(git diff --stat) ]]; then
+            echo '::error:: No functional changes made to the protobuf schema'
+            exit 0
+          else
+            echo '::info:: pushing update commit'
+              git add -u
+              git commit -m 'protobuf: updating generated files'
+              git remote add pr https://github.com/${{ github.event.pull_request.head.repo.owner.login }}/cylc-flow
+              git push pr HEAD:${{ github.head_ref }}
+            exit 0
+          fi

cylc/flow/data_store_mgr.py

@@ -73,7 +73,7 @@
 
 from cylc.flow import __version__ as CYLC_VERSION, LOG
 from cylc.flow.cycling.loader import get_point
-from cylc.flow.data_messages_pb2 import (
+from cylc.flow.network.protobuf.cylc.v5.schema_pb2 import (
     PbEdge, PbEntireWorkflow, PbFamily, PbFamilyProxy, PbJob, PbTask,
     PbTaskProxy, PbWorkflow, PbRuntime, AllDeltas, EDeltas, FDeltas,
     FPDeltas, JDeltas, TDeltas, TPDeltas, WDeltas)
@@ -382,7 +382,7 @@ def create_delta_store(delta=None, workflow_id=None):
     """Create a mini data-store out of the all deltas message.
 
     Args:
-        delta (cylc.flow.data_messages_pb2.AllDeltas):
+        delta (cylc.flow.v5.schema_pb2.AllDeltas):
             The message of accumulated deltas for publish/push.
         workflow_id (str):
             The workflow ID.
@@ -430,18 +430,18 @@ class DataStoreMgr:
             Local store of config.get_first_parent_ancestors()
         .data (dict):
             .edges (dict):
-                cylc.flow.data_messages_pb2.PbEdge by internal ID.
+                cylc.flow.v5.schema_pb2.PbEdge by internal ID.
             .families (dict):
-                cylc.flow.data_messages_pb2.PbFamily by name (internal ID).
+                cylc.flow.v5.schema_pb2.PbFamily by name (internal ID).
             .family_proxies (dict):
-                cylc.flow.data_messages_pb2.PbFamilyProxy by internal ID.
+                cylc.flow.v5.schema_pb2.PbFamilyProxy by internal ID.
             .jobs (dict):
-                cylc.flow.data_messages_pb2.PbJob by internal ID.
+                cylc.flow.v5.schema_pb2.PbJob by internal ID.
             .tasks (dict):
-                cylc.flow.data_messages_pb2.PbTask by name (internal ID).
+                cylc.flow.v5.schema_pb2.PbTask by name (internal ID).
             .task_proxies (dict):
-                cylc.flow.data_messages_pb2.PbTaskProxy by internal ID.
-            .workflow (cylc.flow.data_messages_pb2.PbWorkflow)
+                cylc.flow.v5.schema_pb2.PbTaskProxy by internal ID.
+            .workflow (cylc.flow.v5.schema_pb2.PbWorkflow)
                 Message containing the global information of the workflow.
         .descendants (dict):
             Local store of config.get_first_parent_descendants()
@@ -2688,7 +2688,7 @@ def get_entire_workflow(self):
         """Gather data elements into single Protobuf message.
 
         Returns:
-            cylc.flow.data_messages_pb2.PbEntireWorkflow
+            cylc.flow.v5.schema_pb2.PbEntireWorkflow
 
         """
 

cylc/flow/network/protobuf/buf.gen.yaml

@@ -0,0 +1,10 @@
+# file containing protobuf configuration
+
+version: v2
+
+plugins:
+  - protoc_builtin: python
+    out: .
+
+  - protoc_builtin: pyi
+    out: .

cylc/flow/network/protobuf/buf.yaml

@@ -0,0 +1,11 @@
+# file containing protobuf configuration
+
+version: v2
+
+lint:
+  use:
+    - DEFAULT
+
+breaking:
+  use:
+    - FILE

cylc/flow/data_messages.proto → cylc/flow/network/protobuf/cylc/v5/schema.proto

@@ -1,5 +1,7 @@
 syntax = "proto3";
 
+package cylc.v5;
+
 /* THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
  Copyright (C) NIWA & British Crown (Met Office) & Contributors.
 
@@ -16,6 +18,7 @@ syntax = "proto3";
  You should have received a copy of the GNU General Public License
  along with this program.  If not, see <http://www.gnu.org/licenses/>.*/
 
+
 /* Protobuf message definitions
  *
  * The original intention of these messages is for use as data elements sent
@@ -24,15 +27,21 @@ syntax = "proto3";
  * This file is not needed at runtime. It is used to generate python protobuf
  * message modules.
  *
- * Command:
- * $ protoc -I=./ --python_out=./ --pyi_out=./ data_messages.proto
+ * Hand edit this file to make changes to the schema, use "bufbuild" to lint
+ * changes and regenerate Python files e.g:
  *
- * Pre-compiled protoc binary may be download from:
- * https://github.com/protocolbuffers/protobuf/releases
+ *   # cd into the directory which contains the "buf.yaml" file, this is the
+ *   # project's root directory as far as protobuf is concerned
+ *   $ cd cylc/flow/network/protobuf
  *
- * If merge/rebase conflicts arise, then regenerate the module.
- * (DO NOT manually resolve conflicts)
+ *   # install bufbuild
+ *   $ npm install @bufbuild/buf
  *
+ *   # lint protobuf schema source files
+ *   node_modules/@bufbuild/buf/bin/buf lint
+ *
+ *   # generate Python files
+ *   node_modules/@bufbuild/buf/bin/buf generate
  *
  * WARNING: Avoid re-indexing existing fields!
  *    - Field numbers do not need to be continuous/sequential (gaps are fine).
@@ -43,14 +52,14 @@ syntax = "proto3";
  *
  * https://developers.google.com/protocol-buffers/docs/proto3#assigning_field_numbers
  *
- *
  * */
 
 
 // Query type messages
 message PbMeta {
     optional string title = 1;
     optional string description = 2;
+    // buf:lint:ignore FIELD_LOWER_SNAKE_CASE
     optional string URL = 3;
     optional string user_defined = 4;
 }

cylc/flow/network/server.py

@@ -36,7 +36,7 @@
 from cylc.flow.network.resolvers import Resolvers
 from cylc.flow.network.schema import schema
 from cylc.flow.data_store_mgr import DELTAS_MAP
-from cylc.flow.data_messages_pb2 import PbEntireWorkflow
+from cylc.flow.network.protobuf.cylc.v5.schema_pb2 import PbEntireWorkflow
 
 if TYPE_CHECKING:
     from cylc.flow.scheduler import Scheduler

cylc/flow/prerequisite.py

@@ -22,7 +22,7 @@
 
 from cylc.flow.cycling.loader import get_point
 from cylc.flow.exceptions import TriggerExpressionError
-from cylc.flow.data_messages_pb2 import (
+from cylc.flow.network.protobuf.cylc.v5.schema_pb2 import (
     PbPrerequisite,
     PbCondition,
 )

pytest.ini

@@ -22,7 +22,7 @@ addopts = --verbose
     # group tests by module or class
     --dist=loadscope
     # ignore files which cause issues with test collection
-    --ignore=cylc/flow/data_messages_pb2.py
+    --ignore=cylc/flow/v5/*
     --ignore=cylc/flow/parsec/empysupport.py
     --ignore=cylc/flow/parsec/example
     # disable pytest-tornasync because it conflicts with pytest-asyncio's auto mode

tests/unit/network/test_graphql.py

@@ -20,7 +20,7 @@
 from pytest import param
 from graphql import parse
 
-from cylc.flow.data_messages_pb2 import PbTaskProxy, PbPrerequisite
+from cylc.flow.network.protobuf.cylc.v5.schema_pb2 import PbTaskProxy, PbPrerequisite
 from cylc.flow.network.graphql import (
     AstDocArguments, null_setter, NULL_VALUE, grow_tree
 )

tox.ini

@@ -25,7 +25,7 @@ exclude=
     .git,
     __pycache__,
     .tox,
-    **data_messages_pb2.py
+    cylc/flow/network/protobuf/cylc/**
 paths =
     ./cylc/flow
     ./tests