From f4f1a440f0dcb93f93b497495dd6f665e00c8652 Mon Sep 17 00:00:00 2001 From: Robin Bryce Date: Sat, 13 Jan 2024 11:42:47 +0100 Subject: [PATCH] fix: the include paths for openapiv2 build: simplify the include path discovery build: export the proto dependencies under proto-include This means consumers do not need to replicate the tricky go list based include path discovery ci: add a job to check the dockerall task works For people who can't or wont install protoc and go --- .github/workflows/ci.yml | 5 +- .github/workflows/workflow.yml | 21 +++++ README.md | 31 +++++-- Taskfile.yml | 42 ++++++--- datatrails-common-api/.gitignore | 1 + .../assets/v2/assets/enums.proto | 7 ++ .../assets/v2/assets/eventresponse.proto | 49 ++++++++++- .../assets/v2/assets/service.proto | 1 - datatrails-common-api/go.mod | 2 +- getproto.sh | 3 - taskfiles/Taskfile_apis.yml | 86 ++++++++++--------- 11 files changed, 179 insertions(+), 69 deletions(-) create mode 100644 .github/workflows/workflow.yml create mode 100644 datatrails-common-api/.gitignore delete mode 100755 getproto.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 91c3b09..a750f58 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,10 +30,7 @@ jobs: - name: Generate, Test and Export run: | # Note: it is by design that we don't use the builder - task apis:bootstrap - task apis:generate - task apis:test - task apis:export + task all - name: Show exports working-directory: exported/ diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml new file mode 100644 index 0000000..8f4a796 --- /dev/null +++ b/.github/workflows/workflow.yml @@ -0,0 +1,21 @@ +name: Developer Workflow Tests + +on: + # we ony run these on pull request as they significantly hinder developer feed + # back if they are triggered on push + pull_request: + branches: + - main +jobs: + docker-all: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Setup go-task + uses: arduino/setup-task@v1 + with: + version: 3.x + repo-token: ${{ secrets.GITHUB_TOKEN }} + - name: docker all + run: | + task dockerall diff --git a/README.md b/README.md index 4e72e75..ea400df 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,26 @@ Common public api definitions for the DataTrails platform +## Finding and including proto files for depdendecnies + +tools/go.mod is the source of truth for all proto providing dependencies. That file alone specifies both the upstream version we are using *and* is used, via go install, to make the .proto files available locally + +This corresponds to the practice recommended by grpc-gateway and elsewhere + +1. *** ALWAYS *** Use tools/go.mod to specify the dependency version. +2. Add the package to the `go install` command in the apis:preflight task +3. If necessary, add a var for the new path in any-api **and** then add a reference to that var in the PROTO_INC var. + +Following this practice removes the need for dual maintenance of dependency versions in the builder image. It also produces a build cycle that is significantly faster. + +Cross repository builds in docker while using go.work to refer to locally modified sources don't work. And this setup is essential for an efficient workflow. + +## bootstrap proto files + +The proto's for protoc itself, the googleapis, and the grpc_health proxy are needed by almost everything and are also don't apear to be compatible with the tools/go.mod approach + +For this reason we curl the proto's and make them available in our aggregate proto-includes archive + ## Workflow for updating common apis ### Ensure the go tool chain is setup on your host @@ -46,12 +66,11 @@ If you want to iterate on *just* the helper go code and there tests, do one roun `apis:bootstrap` -> `apis:clean:generated` -Then just iterate using `apis:build` - +Then just iterate using `task apis:generate` and `task apis:test` #### For avid -* task avid:xxx +See the README.md in avid/src/api/README.md ##### Build one api against locally cloned go-datatrails-common-api @@ -60,8 +79,4 @@ The protos can be included exactly as they are found from a clone of go-datatrai task apis:assetsv2-api \ DATATRAILS_COMMON_API="../../go-datatrails-common-api" -It is necessary however to run `task apis:bootsrap` after cloning go-datatrails-common - -#### For forestrie - -* task:xxx +It is necessary however to run `task apis:bootsrap` after cloning go-datatrails-common \ No newline at end of file diff --git a/Taskfile.yml b/Taskfile.yml index 6683c71..ad3d363 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -67,22 +67,42 @@ tasks: ### ------------------------- # Primary workflow tasks ### ------------------------- - all: - desc: "do everything necessary after clone (or rebase)" + dockerall: + desc: "do everything necessary after clone (or rebase) in the builder" cmds: - task: builder-start - defer: {task: builder-cleanup} - # These steps should exactly mirror the steps in .github/workflows/ci.yml - - docker exec -t {{.BUILD_CONTAINER}} task apis:bootstrap - - docker exec -t {{.BUILD_CONTAINER}} task apis:generate - - docker exec -t {{.BUILD_CONTAINER}} task apis:test - - docker exec -t {{.BUILD_CONTAINER}} task apis:export + - task: all + + all: + desc: | + do everything necessary after clone (or rebase) + + Note: This requires go and protoc installed on your host. If you are not + comfortable doing this then use dockerall. + + ** The CI (ci.yml) uses this task and provides a clear illustration of the + pre-requisites. It does not use docker ** + + cmds: + - task: clean + - task: bootstrap + - task: generate + - task: apis:test + - task: apis:export + + # 0. clean out generated and imported files + clean: + - rm -rf proto-include + - rm -rf exported + - task: apis:clean:generated + + # 1. bootstrap (only needed post clone or rebase) + bootstrap: + - task: apis:bootstrap # 2. generate generate: desc: generates all the artifacts we need pre-build cmds: - - task: builder-start - - defer: {task: builder-cleanup} - # DO NOT add bootstrap here, use all instead - - docker exec -t {{.BUILD_CONTAINER}} task apis:generate \ No newline at end of file + - task: apis:generate \ No newline at end of file diff --git a/datatrails-common-api/.gitignore b/datatrails-common-api/.gitignore new file mode 100644 index 0000000..b9014e6 --- /dev/null +++ b/datatrails-common-api/.gitignore @@ -0,0 +1 @@ +go.work diff --git a/datatrails-common-api/assets/v2/assets/enums.proto b/datatrails-common-api/assets/v2/assets/enums.proto index 03488ce..7085cdd 100644 --- a/datatrails-common-api/assets/v2/assets/enums.proto +++ b/datatrails-common-api/assets/v2/assets/enums.proto @@ -7,6 +7,13 @@ enum ConfirmationStatus { PENDING = 1; // not yet committed CONFIRMED = 2; // committed FAILED = 3; // permanent failure + + // Regarding the new statuses for forestrie, See + // https://github.com/datatrails/epic-8120-scalable-proof-mechanisms/blob/main/event-trust-levels.md + STORED = 4; // forestrie, "its in the db" + COMMITTED = 5; // forestrie, "you can know if its changed" + // CONFIRMED = 6; // forestrie, "You can easily prove it changed" + UNEQUIVOCAL = 7; // forestrie, "You easily prove it was publicly available to all" } enum TrackedStatus { diff --git a/datatrails-common-api/assets/v2/assets/eventresponse.proto b/datatrails-common-api/assets/v2/assets/eventresponse.proto index 8e90fd9..e91221a 100644 --- a/datatrails-common-api/assets/v2/assets/eventresponse.proto +++ b/datatrails-common-api/assets/v2/assets/eventresponse.proto @@ -9,6 +9,45 @@ import "datatrails-common-api/assets/v2/assets/enums.proto"; import "datatrails-common-api/assets/v2/assets/principal.proto"; import "datatrails-common-api/attribute/v2/attribute/attribute.proto"; +message MerklLogEntry { + option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_schema) = { + json_schema: { + description: "The verifiable recording of an event for proof mechanism MERKLE_LOG" + } + }; + + // Event trust level commited fields + + // versiona and epoch of the tenants log data. + uint32 log_version = 1; + uint32 log_epoch = 2; + uint64 log_leaf_index = 3; // TBD: this may be redundant. + uint64 log_index = 4; // TBD: this will very likely be included in the trie key + + // triekey_committed includes a time ordered sub key for verifiable, roughly + // time aligned, correlation against other indepdendently logged events. Per + // tenant log the id is guaranteed unique. For all practical purposes it is + // system wide unique, but there are operational assumptions to make that + // hold. + bytes triekey_committed = 5 [ + (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = { + description: "256 bit radix 2 merkle trie key for the event. this key includes a unique, time ordered 64 bit, sub key" + read_only: true + }]; + + // upsert & db query friendly copy of the idtimestamp field from triekey_committed + + fixed64 idtimestamp_committed = 6 [ + (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = { + description: "unique and time ordered verifiable log entry id" + read_only: true + }]; + + // TODO: Event trust level confirmed fields + + // TODO: Event trust level uniquivocal fields +} + message EventResponse { option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_schema) = { json_schema: { @@ -102,9 +141,9 @@ message EventResponse { read_only: true }]; - // timestamp operation has been committed on the blockchain + // timestamp for when the event was committed to a verifiable log google.protobuf.Timestamp timestamp_committed = 7 [ (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = { - description: "time of event as recorded on blockchain" + description: "time of event as recorded in verifiable storage" read_only: true }]; @@ -164,4 +203,10 @@ message EventResponse { "Identity of the tenant the that created this event" max_length: 1024 }]; + + MerklLogEntry log_entry = 20 [ (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = { + description: + "verifiable log entry details" + max_length: 1024 + }]; } \ No newline at end of file diff --git a/datatrails-common-api/assets/v2/assets/service.proto b/datatrails-common-api/assets/v2/assets/service.proto index 5f88803..d44925c 100644 --- a/datatrails-common-api/assets/v2/assets/service.proto +++ b/datatrails-common-api/assets/v2/assets/service.proto @@ -5,7 +5,6 @@ package archivist.v2; option go_package="github.com/datatrails/go-datatrails-common-api-gen/assets/v2/assets;assets"; import "google/api/annotations.proto"; -import "validate/validate.proto"; import "protoc-gen-openapiv2/options/annotations.proto"; import "datatrails-common-api/assets/v2/assets/assetresponse.proto"; diff --git a/datatrails-common-api/go.mod b/datatrails-common-api/go.mod index d5408d6..e62713f 100644 --- a/datatrails-common-api/go.mod +++ b/datatrails-common-api/go.mod @@ -8,7 +8,7 @@ go 1.21 // This allows this module to operate as tho it were the generated module. This // allows us to manage the proto tool dependencies via this go.mod. This go.mod // is also used as the go.mod for the generated package. -replace github.com/datatrails/go-datatrails-common-api-gen => ./ +// replace github.com/datatrails/go-datatrails-common-api-gen => ./ require ( github.com/datatrails/go-datatrails-common v0.10.2 diff --git a/getproto.sh b/getproto.sh deleted file mode 100755 index 9376fc3..0000000 --- a/getproto.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/sh -cd datatrails-common-api/ -echo $(GOCACHE=/tmp/datatrails/go-datatrails-common-api/api go list -f {{.Dir}} $1) \ No newline at end of file diff --git a/taskfiles/Taskfile_apis.yml b/taskfiles/Taskfile_apis.yml index 4219e1c..3410b4c 100644 --- a/taskfiles/Taskfile_apis.yml +++ b/taskfiles/Taskfile_apis.yml @@ -17,6 +17,8 @@ env_build: &env_build # The GOCACHE in a linux container on Docker for windows MUST be on a linuxy # file system GOCACHE: /tmp/datatrails/go-datatrails-common-api + GOBIN: + sh: echo ${GOBIN:-$HOME/go/bin} vars: @@ -43,7 +45,9 @@ vars: # imports GOPKG_OVERRIDES: ",Mgoogle/protobuf/field_mask.proto=google.golang.org/genproto/protobuf/field_mask" - # Upstream proto definition files are placed here by the bootstrap tasks + # proto definition files whose versions can not be controlled by tools/go.mod + # are collected in proto-include by the bootstrap tasks. Note the grpc_health + # proto's are not currently versioned and are always taken from master GOOGLE_COMMON_PROTOS_VERSION: '{{.GOOGLE_COMMON_PROTOS_VERSION | default "1.50.0"}}' GOOGLE_COMMON_INC_ARCHIVE_BASEDIR: '{{.GOOGLE_COMMON_INC_ARCHIVE_BASEDIR | default "api-common-protos"}}' @@ -96,11 +100,20 @@ tasks: - for ff in $(find . -name "*.swagger.json"); do rm -f $ff; done preflight: - summary: run prior to api generation tasks + summary: | + installs the tools/go.mod specified protoc plugins and .proto definitions + the common apis depend upon + + And prepares the .proto's under proto-include so that consumers do not + need to replicate the arrangements for finding the proto include paths + dir: '../datatrails-common-api' + vars: + API_PROTO_DEPS: >- + github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-openapiv2 + github.com/envoyproxy/protoc-gen-validate cmds: - | - go mod download go get github.com/datatrails/go-datatrails-common-api-gen/tools go install \ @@ -111,6 +124,32 @@ tasks: github.com/envoyproxy/protoc-gen-validate \ github.com/lyft/protoc-gen-star/v2 \ github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc + - for: { var: API_PROTO_DEPS } + cmd: | + depdir=$(GOCACHE=/tmp/datatrails/go-datatrails-common-api/api \ + go list -f {{"{{"}}.Dir{{"}}"}} {{.ITEM}}) + + basedir=$(basename $depdir) + basedir=${basedir%%@*} # strip the go module version + rm -rf ../proto-include/${basedir} + for proto in $(find $depdir -type f -name "*.proto"); do + target=../proto-include/${basedir}/${proto#$depdir/} + mkdir -p $(dirname $target) + cp -v $proto $target + done + # find ../proto-include/${basedir} + # find $d -type f -name "*.proto" + # echo $(dirname $d) + + # move things around so that consumers only need to -I proto-include, + # and do any general pruning or tidy up. + - | + # consumers of protoc-gen-validate expect a -I which allows them to do + # import "validate/validate.proto". And we don't need the java bindings or the example bindings + rm -rf ../proto-include/validate + mv -v ../proto-include/protoc-gen-validate/validate ../proto-include/ + rm -rf ../proto-include/protoc-gen-validate + postflight: summary: run after all api generation tasks @@ -134,7 +173,6 @@ tasks: go build ./... go test ./... - export: desc: export the sources to the desired DESTDIR dir: '../datatrails-common-api' @@ -163,7 +201,9 @@ tasks: mkdir -p {{.DESTDIR}}/swagger # The source protos are all gathered under a 'google' common top dir - cp -vr ../proto-include/* {{.DESTDIR}}proto-include/ + for d in $(ls -d ../proto-include/*); + do cp -vr $d {{.DESTDIR}}proto-include/ + done for api in {{.APIS}}; do mkdir -vp {{.DESTDIR}}{{.COMMON_GODIR}}/${api} @@ -220,7 +260,7 @@ tasks: # Generates the swagger definitions for the gRPC service. # Generates the html docs for the gRPC service. - | - protoc -I {{.SELF_INC}} {{.PROTOC_INC}} \ + protoc -I . -I proto-include \ --go_out=paths=source_relative:{{.OUTPATH}} \ --go-grpc_out=paths=source_relative:{{.OUTPATH}} \ --validate_out=lang=go,paths=source_relative:{{.OUTPATH}} \ @@ -238,36 +278,4 @@ tasks: # OUTPATH: datatrails-common-api OUTPATH: . GOBIN: - sh: echo ${GOBIN:-$HOME/go/bin} - # These variables capture the locations that .proto files for third party - # packages can be found at. The locations can change as a consequence of go - # mod module updates, so we require a dynamic approach. The particularly - # grotty bit is the -f argment - we need to pass -f {{.Dir}} to the go list - # command. To do so we have to prevent go template from evaluating it. - # - PROTO_GEN_GO_DIR: - # Notice; the tricksy 'cd datatrails-common-api' rather than just - # running in that directory via tasks 'dir:' directive is so that we can - # have the same import statements in the proto files *everywhere*. All - # consumers of datatrails-common-api get to do things very simply in - # return for this. AND it makes iteratively generating proto changes - # across repositories a lot more efficient - sh: | - echo $(./getproto.sh google.golang.org/protobuf/cmd/protoc-gen-go) - - GRPC_GATEWAY_DIR: - sh: | - echo $(dirname $(./getproto.sh github.com/grpc-ecosystem/grpc-gateway/v2/runtime)) - - ENVOY_VALIDATE: - sh: | - echo $(./getproto.sh github.com/envoyproxy/protoc-gen-validate) - - SELF_INC: . - - # Global includes used for all protoc invocations. - # task sets cwd initially to the directory of the Taskfile.yml - PROTOC_INC: "-I {{.ENVOY_VALIDATE}} \ - -I {{.PROTO_GEN_GO_DIR}} \ - -I {{.GRPC_GATEWAY_DIR}} \ - -I proto-include" \ No newline at end of file + sh: echo ${GOBIN:-$HOME/go/bin} \ No newline at end of file