fix: the include paths for openapiv2

build: simplify the include path discovery build: export the proto dependencies under proto-include This means consumers do not need to replicate the tricky go list based include path discovery ci: add a job to check the dockerall task works For people who can't or wont install protoc and go
datatrails · Jan 13, 2024 · f4f1a44 · f4f1a44
1 parent c7be1e4
commit f4f1a44
Show file tree

Hide file tree

Showing 11 changed files with 179 additions and 69 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -30,10 +30,7 @@ jobs:
       - name: Generate, Test and Export
         run: |
           # Note: it is by design that we don't use the builder
-          task apis:bootstrap
-          task apis:generate
-          task apis:test
-          task apis:export
+          task all
 
       - name: Show exports
         working-directory: exported/

diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml
@@ -0,0 +1,21 @@
+name: Developer Workflow Tests
+
+on:
+  # we ony run these on pull request as they significantly hinder developer feed
+  # back if they are triggered on push 
+  pull_request:
+    branches:
+      - main
+jobs:
+  docker-all:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup go-task
+        uses: arduino/setup-task@v1
+        with:
+          version: 3.x
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: docker all
+        run: |
+          task dockerall
diff --git a/README.md b/README.md
@@ -2,6 +2,26 @@
 
 Common public api definitions for the DataTrails platform
 
+## Finding and including proto files for depdendecnies
+
+tools/go.mod is the source of truth for all proto providing dependencies. That file alone specifies both the upstream version we are using *and* is used, via go install, to make the .proto files available locally
+
+This corresponds to the practice recommended by grpc-gateway and elsewhere
+
+1. *** ALWAYS *** Use tools/go.mod to specify the dependency version.
+2. Add the package to the `go install` command in the apis:preflight task
+3. If necessary, add a var for the new path in any-api **and** then add a reference to that var in the PROTO_INC var.
+
+Following this practice removes the need for dual maintenance of dependency versions in the builder image. It also produces a build cycle that is significantly faster.
+
+Cross repository builds in docker while using go.work to refer to locally modified sources don't work. And this setup is essential for an efficient workflow.
+
+## bootstrap proto files
+
+The proto's for protoc itself, the googleapis, and the grpc_health proxy are needed by almost everything and are also don't apear to be compatible with the tools/go.mod approach
+
+For this reason we curl the proto's and make them available in our aggregate proto-includes archive
+
 ## Workflow for updating common apis
 
 ### Ensure the go tool chain is setup on your host
@@ -46,12 +66,11 @@ If you want to iterate on *just* the helper go code and there tests, do one roun
 
 `apis:bootstrap` -> `apis:clean:generated`
 
-Then just iterate using `apis:build`
-
+Then just iterate using `task apis:generate` and `task apis:test`
 
 #### For avid
 
-* task avid:xxx
+See the README.md in avid/src/api/README.md
 
 ##### Build one api against locally cloned go-datatrails-common-api
 
@@ -60,8 +79,4 @@ The protos can be included exactly as they are found from a clone of go-datatrai
     task apis:assetsv2-api \
         DATATRAILS_COMMON_API="../../go-datatrails-common-api"
 
-It is necessary however to run `task apis:bootsrap` after cloning go-datatrails-common
-
-#### For forestrie
-
-* task:xxx
+It is necessary however to run `task apis:bootsrap` after cloning go-datatrails-common
diff --git a/Taskfile.yml b/Taskfile.yml
@@ -67,22 +67,42 @@ tasks:
   ### -------------------------
   # Primary workflow tasks 
   ### -------------------------
-  all:
-    desc: "do everything necessary after clone (or rebase)"
+  dockerall:
+    desc: "do everything necessary after clone (or rebase) in the builder"
     cmds:
       - task: builder-start
       - defer: {task: builder-cleanup}
-      # These steps should exactly mirror the steps in .github/workflows/ci.yml
-      - docker exec -t {{.BUILD_CONTAINER}} task apis:bootstrap
-      - docker exec -t {{.BUILD_CONTAINER}} task apis:generate
-      - docker exec -t {{.BUILD_CONTAINER}} task apis:test
-      - docker exec -t {{.BUILD_CONTAINER}} task apis:export
+      - task: all
+
+  all:
+    desc: |
+      do everything necessary after clone (or rebase)
+
+      Note: This requires go and protoc installed on your host. If you are not
+      comfortable doing this then use dockerall.
+
+      ** The CI (ci.yml) uses this task and provides a clear illustration of the
+      pre-requisites. It does not use docker **
+
+    cmds:
+      - task: clean
+      - task: bootstrap
+      - task: generate
+      - task: apis:test
+      - task: apis:export
+
+  # 0. clean out generated and imported files
+  clean:
+    - rm -rf proto-include
+    - rm -rf exported
+    - task: apis:clean:generated
+
+  # 1. bootstrap (only needed post clone or rebase)
+  bootstrap:
+    - task: apis:bootstrap
 
   # 2. generate
   generate:
     desc: generates all the artifacts we need pre-build
     cmds:
-      - task: builder-start
-      - defer: {task: builder-cleanup}
-      # DO NOT add bootstrap here, use all instead
-      - docker exec -t {{.BUILD_CONTAINER}} task apis:generate
+      - task: apis:generate
diff --git a/datatrails-common-api/.gitignore b/datatrails-common-api/.gitignore
@@ -0,0 +1 @@
+go.work
diff --git a/datatrails-common-api/assets/v2/assets/enums.proto b/datatrails-common-api/assets/v2/assets/enums.proto
@@ -7,6 +7,13 @@ enum ConfirmationStatus {
     PENDING = 1;        // not yet committed
     CONFIRMED = 2;      // committed
     FAILED = 3;         // permanent failure
+
+	// Regarding the new statuses for forestrie,  See
+	// https://github.com/datatrails/epic-8120-scalable-proof-mechanisms/blob/main/event-trust-levels.md
+    STORED = 4; // forestrie, "its in the db"
+    COMMITTED = 5; // forestrie, "you can know if its changed"
+    // CONFIRMED = 6; // forestrie, "You can easily prove it changed"
+    UNEQUIVOCAL = 7; // forestrie, "You easily prove it was publicly available to all" 
 }
 
 enum TrackedStatus {

diff --git a/datatrails-common-api/assets/v2/assets/eventresponse.proto b/datatrails-common-api/assets/v2/assets/eventresponse.proto
@@ -9,6 +9,45 @@ import "datatrails-common-api/assets/v2/assets/enums.proto";
 import "datatrails-common-api/assets/v2/assets/principal.proto";
 import "datatrails-common-api/attribute/v2/attribute/attribute.proto";
 
+message MerklLogEntry {
+    option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_schema) = {
+        json_schema: {
+            description: "The verifiable recording of an event for proof mechanism MERKLE_LOG"
+        }
+    };
+
+    // Event trust level commited fields
+
+    // versiona and epoch of the tenants log data.
+    uint32 log_version = 1;
+    uint32 log_epoch = 2;
+    uint64 log_leaf_index = 3; // TBD: this may be redundant.
+    uint64 log_index = 4; // TBD: this will very likely be included in the trie key
+
+    // triekey_committed includes a time ordered sub key for verifiable, roughly
+    // time aligned, correlation against other indepdendently logged events. Per
+    // tenant log the id is guaranteed unique.  For all practical purposes it is
+    // system wide unique, but there are operational assumptions to make that
+    // hold.
+    bytes triekey_committed = 5 [
+    (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = {
+        description: "256 bit radix 2 merkle trie key for the event. this key includes a unique, time ordered 64 bit, sub key"
+        read_only: true
+    }];
+
+    // upsert & db query friendly copy of the idtimestamp field from triekey_committed
+
+    fixed64 idtimestamp_committed = 6 [
+    (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = {
+        description: "unique and time ordered verifiable log entry id"
+        read_only: true
+    }];
+
+    // TODO: Event trust level confirmed fields
+
+    // TODO: Event trust level uniquivocal fields
+}
+
 message EventResponse {
     option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_schema) = {
         json_schema: {
@@ -102,9 +141,9 @@ message EventResponse {
         read_only: true
     }];
 
-    // timestamp operation has been committed on the blockchain
+    // timestamp for when the event was committed to a verifiable log
     google.protobuf.Timestamp timestamp_committed = 7 [ (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = {
-        description: "time of event as recorded on blockchain"
+        description: "time of event as recorded in verifiable storage"
         read_only: true
     }];
 
@@ -164,4 +203,10 @@ message EventResponse {
         "Identity of the tenant the that created this event"
         max_length: 1024
     }];
+
+    MerklLogEntry log_entry = 20 [ (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = {
+        description:
+        "verifiable log entry details"
+        max_length: 1024
+    }];
 }
diff --git a/datatrails-common-api/assets/v2/assets/service.proto b/datatrails-common-api/assets/v2/assets/service.proto
@@ -5,7 +5,6 @@ package archivist.v2;
 option go_package="github.com/datatrails/go-datatrails-common-api-gen/assets/v2/assets;assets";
 
 import "google/api/annotations.proto";
-import "validate/validate.proto";
 import "protoc-gen-openapiv2/options/annotations.proto";
 
 import "datatrails-common-api/assets/v2/assets/assetresponse.proto";

diff --git a/datatrails-common-api/go.mod b/datatrails-common-api/go.mod
@@ -8,7 +8,7 @@ go 1.21
 // This allows this module to operate as tho it were the generated module. This
 // allows us to manage the proto tool dependencies via this go.mod. This go.mod
 // is also used as the go.mod for the generated package.
-replace github.com/datatrails/go-datatrails-common-api-gen => ./
+// replace github.com/datatrails/go-datatrails-common-api-gen => ./
 
 require (
 	github.com/datatrails/go-datatrails-common v0.10.2

diff --git a/getproto.sh b/getproto.sh
diff --git a/taskfiles/Taskfile_apis.yml b/taskfiles/Taskfile_apis.yml
@@ -17,6 +17,8 @@ env_build: &env_build
   # The GOCACHE in a linux container on Docker for windows MUST be on a linuxy
   # file system
   GOCACHE: /tmp/datatrails/go-datatrails-common-api
+  GOBIN:
+    sh: echo ${GOBIN:-$HOME/go/bin}
 
 
 vars:
@@ -43,7 +45,9 @@ vars:
   # imports
   GOPKG_OVERRIDES: ",Mgoogle/protobuf/field_mask.proto=google.golang.org/genproto/protobuf/field_mask"
 
-  # Upstream proto definition files are placed here by the bootstrap tasks
+  # proto definition files whose versions can not be controlled by tools/go.mod
+  # are collected in proto-include by the bootstrap tasks. Note the grpc_health
+  # proto's are not currently versioned and are always taken from master
   GOOGLE_COMMON_PROTOS_VERSION: '{{.GOOGLE_COMMON_PROTOS_VERSION | default "1.50.0"}}'
   GOOGLE_COMMON_INC_ARCHIVE_BASEDIR: '{{.GOOGLE_COMMON_INC_ARCHIVE_BASEDIR | default "api-common-protos"}}'
 
@@ -96,11 +100,20 @@ tasks:
       - for ff in $(find . -name "*.swagger.json"); do rm -f $ff; done
 
   preflight:
-    summary: run prior to api generation tasks
+    summary: |
+      installs the tools/go.mod specified protoc plugins and .proto definitions
+      the common apis depend upon
+
+      And prepares the .proto's under proto-include so that consumers do not
+      need to replicate the arrangements for finding the proto include paths
+
     dir: '../datatrails-common-api'
+    vars:
+      API_PROTO_DEPS: >-
+        github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-openapiv2
+        github.com/envoyproxy/protoc-gen-validate
     cmds:
       - |
-        go mod download
         go get github.com/datatrails/go-datatrails-common-api-gen/tools
 
         go install \
@@ -111,6 +124,32 @@ tasks:
           github.com/envoyproxy/protoc-gen-validate \
           github.com/lyft/protoc-gen-star/v2 \
           github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc
+      - for: { var: API_PROTO_DEPS }
+        cmd: |
+          depdir=$(GOCACHE=/tmp/datatrails/go-datatrails-common-api/api \
+            go list -f {{"{{"}}.Dir{{"}}"}} {{.ITEM}})
+
+          basedir=$(basename $depdir)
+          basedir=${basedir%%@*} # strip the go module version
+          rm -rf ../proto-include/${basedir}
+          for proto in $(find $depdir -type f -name "*.proto"); do
+            target=../proto-include/${basedir}/${proto#$depdir/}
+            mkdir -p $(dirname $target)
+            cp -v $proto $target
+          done
+          # find ../proto-include/${basedir}
+          # find $d -type f -name "*.proto"
+          # echo $(dirname $d)
+
+      # move things around so that consumers only need to -I proto-include,
+      # and do any general pruning or tidy up.
+      - |
+        # consumers of protoc-gen-validate expect a -I which allows them to do
+        # import "validate/validate.proto". And we don't need the java bindings or the example bindings
+        rm -rf ../proto-include/validate
+        mv -v ../proto-include/protoc-gen-validate/validate ../proto-include/
+        rm -rf ../proto-include/protoc-gen-validate
+
 
   postflight:
     summary: run after all api generation tasks
@@ -134,7 +173,6 @@ tasks:
         go build ./...
         go test ./...
 
-
   export:
     desc: export the sources to the desired DESTDIR
     dir: '../datatrails-common-api'
@@ -163,7 +201,9 @@ tasks:
         mkdir -p {{.DESTDIR}}/swagger
 
         # The source protos are all gathered under a 'google' common top dir
-        cp -vr ../proto-include/* {{.DESTDIR}}proto-include/
+        for d in $(ls -d ../proto-include/*);
+          do cp -vr $d {{.DESTDIR}}proto-include/
+        done
 
         for api in {{.APIS}}; do
           mkdir -vp {{.DESTDIR}}{{.COMMON_GODIR}}/${api}
@@ -220,7 +260,7 @@ tasks:
       # Generates the swagger definitions for the gRPC service.
       # Generates the html docs for the gRPC service.
       - |
-        protoc -I {{.SELF_INC}} {{.PROTOC_INC}} \
+        protoc -I . -I proto-include \
          --go_out=paths=source_relative:{{.OUTPATH}} \
          --go-grpc_out=paths=source_relative:{{.OUTPATH}} \
          --validate_out=lang=go,paths=source_relative:{{.OUTPATH}} \
@@ -238,36 +278,4 @@ tasks:
       # OUTPATH: datatrails-common-api
       OUTPATH: . 
       GOBIN:
-        sh: echo ${GOBIN:-$HOME/go/bin}
-      # These variables capture the locations that .proto files for third party
-      # packages can be found at. The locations can change as a consequence of go
-      # mod module updates, so we require a dynamic approach.  The particularly
-      # grotty bit is the -f  argment - we need to pass -f {{.Dir}} to the go list
-      # command. To do so we have to prevent go template from evaluating it.
-      #
-      PROTO_GEN_GO_DIR:
-        # Notice; the tricksy 'cd datatrails-common-api' rather than just
-        # running in that directory via tasks 'dir:' directive is so that we can
-        # have the same import statements in the proto files *everywhere*. All
-        # consumers of datatrails-common-api get to do things very simply in
-        # return for this. AND it makes iteratively generating proto changes
-        # across repositories a lot more efficient
-        sh: |
-          echo $(./getproto.sh google.golang.org/protobuf/cmd/protoc-gen-go)
-
-      GRPC_GATEWAY_DIR:
-        sh: |
-          echo $(dirname $(./getproto.sh github.com/grpc-ecosystem/grpc-gateway/v2/runtime))
-          
-      ENVOY_VALIDATE:
-        sh: |
-          echo $(./getproto.sh github.com/envoyproxy/protoc-gen-validate)
-      
-      SELF_INC: .
-
-      # Global includes used for all protoc invocations.
-      # task sets cwd initially to the directory of the Taskfile.yml
-      PROTOC_INC: "-I {{.ENVOY_VALIDATE}} \
-        -I {{.PROTO_GEN_GO_DIR}} \
-        -I {{.GRPC_GATEWAY_DIR}} \
-        -I proto-include"
+        sh: echo ${GOBIN:-$HOME/go/bin}