From db8ea99ed4ce96d01d3bb25d3c6afb8b54f622ea Mon Sep 17 00:00:00 2001 From: Robin Bryce Date: Sat, 13 Jan 2024 11:42:47 +0100 Subject: [PATCH 1/2] feat: forestrie event message additions * Add the confirmation states representing the forestrie trust levels as described here: https://github.com/datatrails/\ epic-8120-scalable-proof-mechanisms/blob/main/event-trust-levels.md * Add the provisional extra details we need in the event for the MERKLE_LOG proof mechanism * Use a single oneof per proof mech details fix: the include paths for openapiv2 build: simplify the include path discovery build: export the proto dependencies under proto-include This means consumers do not need to replicate the tricky go list based include path discovery ci: add a job to check the dockerall task works But note this is disabled due to acr permissions issues For people who can't or wont install protoc and go AB#8859 remove fields that are not settled yet use oneof for proof_details --- .github/workflows/ci.yml | 5 +- .github/workflows/workflow.yml | 24 ++++++ README.md | 31 +++++-- Taskfile.yml | 42 ++++++--- datatrails-common-api/.gitignore | 1 + .../assets/v2/assets/enums.proto | 7 ++ .../assets/v2/assets/eventresponse.proto | 20 ++++- .../assets/v2/assets/merklelogentry.proto | 53 ++++++++++++ .../assets/v2/assets/service.proto | 1 - datatrails-common-api/go.mod | 2 +- getproto.sh | 3 - taskfiles/Taskfile_apis.yml | 86 ++++++++++--------- 12 files changed, 206 insertions(+), 69 deletions(-) create mode 100644 .github/workflows/workflow.yml create mode 100644 datatrails-common-api/.gitignore create mode 100644 datatrails-common-api/assets/v2/assets/merklelogentry.proto delete mode 100755 getproto.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 91c3b09..a750f58 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,10 +30,7 @@ jobs: - name: Generate, Test and Export run: | # Note: it is by design that we don't use the builder - task apis:bootstrap - task apis:generate - task apis:test - task apis:export + task all - name: Show exports working-directory: exported/ diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml new file mode 100644 index 0000000..6e5fc8f --- /dev/null +++ b/.github/workflows/workflow.yml @@ -0,0 +1,24 @@ +name: Developer Workflow Tests + +on: + # we ony run these on pull request as they significantly hinder developer feed + # back if they are triggered on push + pull_request: + branches: + - main +jobs: + docker-all: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Setup go-task + uses: arduino/setup-task@v1 + with: + version: 3.x + repo-token: ${{ secrets.GITHUB_TOKEN }} + - name: docker all + run: | + + echo "TODO: permissions on accessing our acr prevent this from working" + exit 0 + # task dockerall diff --git a/README.md b/README.md index 4e72e75..ea400df 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,26 @@ Common public api definitions for the DataTrails platform +## Finding and including proto files for depdendecnies + +tools/go.mod is the source of truth for all proto providing dependencies. That file alone specifies both the upstream version we are using *and* is used, via go install, to make the .proto files available locally + +This corresponds to the practice recommended by grpc-gateway and elsewhere + +1. *** ALWAYS *** Use tools/go.mod to specify the dependency version. +2. Add the package to the `go install` command in the apis:preflight task +3. If necessary, add a var for the new path in any-api **and** then add a reference to that var in the PROTO_INC var. + +Following this practice removes the need for dual maintenance of dependency versions in the builder image. It also produces a build cycle that is significantly faster. + +Cross repository builds in docker while using go.work to refer to locally modified sources don't work. And this setup is essential for an efficient workflow. + +## bootstrap proto files + +The proto's for protoc itself, the googleapis, and the grpc_health proxy are needed by almost everything and are also don't apear to be compatible with the tools/go.mod approach + +For this reason we curl the proto's and make them available in our aggregate proto-includes archive + ## Workflow for updating common apis ### Ensure the go tool chain is setup on your host @@ -46,12 +66,11 @@ If you want to iterate on *just* the helper go code and there tests, do one roun `apis:bootstrap` -> `apis:clean:generated` -Then just iterate using `apis:build` - +Then just iterate using `task apis:generate` and `task apis:test` #### For avid -* task avid:xxx +See the README.md in avid/src/api/README.md ##### Build one api against locally cloned go-datatrails-common-api @@ -60,8 +79,4 @@ The protos can be included exactly as they are found from a clone of go-datatrai task apis:assetsv2-api \ DATATRAILS_COMMON_API="../../go-datatrails-common-api" -It is necessary however to run `task apis:bootsrap` after cloning go-datatrails-common - -#### For forestrie - -* task:xxx +It is necessary however to run `task apis:bootsrap` after cloning go-datatrails-common \ No newline at end of file diff --git a/Taskfile.yml b/Taskfile.yml index 6683c71..ad3d363 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -67,22 +67,42 @@ tasks: ### ------------------------- # Primary workflow tasks ### ------------------------- - all: - desc: "do everything necessary after clone (or rebase)" + dockerall: + desc: "do everything necessary after clone (or rebase) in the builder" cmds: - task: builder-start - defer: {task: builder-cleanup} - # These steps should exactly mirror the steps in .github/workflows/ci.yml - - docker exec -t {{.BUILD_CONTAINER}} task apis:bootstrap - - docker exec -t {{.BUILD_CONTAINER}} task apis:generate - - docker exec -t {{.BUILD_CONTAINER}} task apis:test - - docker exec -t {{.BUILD_CONTAINER}} task apis:export + - task: all + + all: + desc: | + do everything necessary after clone (or rebase) + + Note: This requires go and protoc installed on your host. If you are not + comfortable doing this then use dockerall. + + ** The CI (ci.yml) uses this task and provides a clear illustration of the + pre-requisites. It does not use docker ** + + cmds: + - task: clean + - task: bootstrap + - task: generate + - task: apis:test + - task: apis:export + + # 0. clean out generated and imported files + clean: + - rm -rf proto-include + - rm -rf exported + - task: apis:clean:generated + + # 1. bootstrap (only needed post clone or rebase) + bootstrap: + - task: apis:bootstrap # 2. generate generate: desc: generates all the artifacts we need pre-build cmds: - - task: builder-start - - defer: {task: builder-cleanup} - # DO NOT add bootstrap here, use all instead - - docker exec -t {{.BUILD_CONTAINER}} task apis:generate \ No newline at end of file + - task: apis:generate \ No newline at end of file diff --git a/datatrails-common-api/.gitignore b/datatrails-common-api/.gitignore new file mode 100644 index 0000000..b9014e6 --- /dev/null +++ b/datatrails-common-api/.gitignore @@ -0,0 +1 @@ +go.work diff --git a/datatrails-common-api/assets/v2/assets/enums.proto b/datatrails-common-api/assets/v2/assets/enums.proto index 03488ce..7085cdd 100644 --- a/datatrails-common-api/assets/v2/assets/enums.proto +++ b/datatrails-common-api/assets/v2/assets/enums.proto @@ -7,6 +7,13 @@ enum ConfirmationStatus { PENDING = 1; // not yet committed CONFIRMED = 2; // committed FAILED = 3; // permanent failure + + // Regarding the new statuses for forestrie, See + // https://github.com/datatrails/epic-8120-scalable-proof-mechanisms/blob/main/event-trust-levels.md + STORED = 4; // forestrie, "its in the db" + COMMITTED = 5; // forestrie, "you can know if its changed" + // CONFIRMED = 6; // forestrie, "You can easily prove it changed" + UNEQUIVOCAL = 7; // forestrie, "You easily prove it was publicly available to all" } enum TrackedStatus { diff --git a/datatrails-common-api/assets/v2/assets/eventresponse.proto b/datatrails-common-api/assets/v2/assets/eventresponse.proto index 8e90fd9..5bf2f82 100644 --- a/datatrails-common-api/assets/v2/assets/eventresponse.proto +++ b/datatrails-common-api/assets/v2/assets/eventresponse.proto @@ -7,6 +7,7 @@ import "protoc-gen-openapiv2/options/annotations.proto"; import "google/protobuf/timestamp.proto"; import "datatrails-common-api/assets/v2/assets/enums.proto"; import "datatrails-common-api/assets/v2/assets/principal.proto"; +import "datatrails-common-api/assets/v2/assets/merklelogentry.proto"; import "datatrails-common-api/attribute/v2/attribute/attribute.proto"; message EventResponse { @@ -102,9 +103,9 @@ message EventResponse { read_only: true }]; - // timestamp operation has been committed on the blockchain + // timestamp for when the event was committed to a verifiable log google.protobuf.Timestamp timestamp_committed = 7 [ (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = { - description: "time of event as recorded on blockchain" + description: "time of event as recorded in verifiable storage" read_only: true }]; @@ -135,6 +136,8 @@ message EventResponse { type: STRING }]; + // NOTICE: We expect to retire simple hash and then remove all the top level dlt fields. + // hash of transaction committing this operation on blockchain string transaction_id = 11 [ (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = { description: "hash of the transaction as a hex string `0x11bf5b37e0b842e08dcfdc8c4aefc000`" @@ -164,4 +167,17 @@ message EventResponse { "Identity of the tenant the that created this event" max_length: 1024 }]; + + // An event has exactly one proof mechanism. This field caputures the proof + // mechanism specific details supporting the trustworthyness of the event + // record. We anticipate at least two proof mechs: merkle_log and + // verkle_log. We use oneof to avoid repeating the scattering of randomly + // re-purposed fields we currently have for simple hash vs khipu. + oneof proof_details { + MerklLogEntry merklelog_entry = 19 [ (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = { + description: + "verifiable merkle mmr log entry details" + max_length: 1024 + }]; + }; } \ No newline at end of file diff --git a/datatrails-common-api/assets/v2/assets/merklelogentry.proto b/datatrails-common-api/assets/v2/assets/merklelogentry.proto new file mode 100644 index 0000000..b65bd05 --- /dev/null +++ b/datatrails-common-api/assets/v2/assets/merklelogentry.proto @@ -0,0 +1,53 @@ +// Maintainers, please refer to the style guide here: +// https://developers.google.com/protocol-buffers/docs/style +syntax = "proto3"; +package archivist.v2; +option go_package="github.com/datatrails/go-datatrails-common-api-gen/assets/v2/assets;assets"; +import "google/protobuf/timestamp.proto"; + +// MerkeLogCommit provides the log entry details for a single mmr leaf. +message MerkleLogCommit { + /* The mmr index */ + uint64 index = 3; + /* The mmr *leaf* index */ + uint64 leaf_index = 4; // TBD: this may be redundant. + /* time ordered and strictly unique per tenant. system wide + * unique with very reasonable operational assumptions. */ + fixed64 idtimestamp = 5; +} + +// The message sent from forestrie to avid notifying that the corresponding +// event is commited to the tenants log. +message MerkleLogCommitMessage { + + // The tenant identity and the event identity for the committed event. + string tenant_identity = 1; + string event_identity = 2; + /* The time portion of idtimestamp that contributed to the hash of the event + * (the idtimestamp is _also_ included. + * This must be copied into event.timestamp_committed when the saas db is updated */ + google.protobuf.Timestamp timestamp = 6; + + uint32 log_version = 3; + uint32 log_epoch = 4; + MerkleLogCommit committed = 5; +} + + + +// The details stored in the SaaS db for a proof mech MERKLE_LOG commitment +message MerklLogEntry { + + // The tenant log version and epoch when the log entry was created. + uint32 log_version = 1; + uint32 log_epoch = 2; + + // Event trust level commited fields + MerkleLogCommit committed = 3; + + // TODO: Event trust level confirmed fields + + // signature over tenant mmr root + + // TODO: Event trust level uniquivocal fields +} \ No newline at end of file diff --git a/datatrails-common-api/assets/v2/assets/service.proto b/datatrails-common-api/assets/v2/assets/service.proto index 5f88803..d44925c 100644 --- a/datatrails-common-api/assets/v2/assets/service.proto +++ b/datatrails-common-api/assets/v2/assets/service.proto @@ -5,7 +5,6 @@ package archivist.v2; option go_package="github.com/datatrails/go-datatrails-common-api-gen/assets/v2/assets;assets"; import "google/api/annotations.proto"; -import "validate/validate.proto"; import "protoc-gen-openapiv2/options/annotations.proto"; import "datatrails-common-api/assets/v2/assets/assetresponse.proto"; diff --git a/datatrails-common-api/go.mod b/datatrails-common-api/go.mod index d5408d6..e62713f 100644 --- a/datatrails-common-api/go.mod +++ b/datatrails-common-api/go.mod @@ -8,7 +8,7 @@ go 1.21 // This allows this module to operate as tho it were the generated module. This // allows us to manage the proto tool dependencies via this go.mod. This go.mod // is also used as the go.mod for the generated package. -replace github.com/datatrails/go-datatrails-common-api-gen => ./ +// replace github.com/datatrails/go-datatrails-common-api-gen => ./ require ( github.com/datatrails/go-datatrails-common v0.10.2 diff --git a/getproto.sh b/getproto.sh deleted file mode 100755 index 9376fc3..0000000 --- a/getproto.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/sh -cd datatrails-common-api/ -echo $(GOCACHE=/tmp/datatrails/go-datatrails-common-api/api go list -f {{.Dir}} $1) \ No newline at end of file diff --git a/taskfiles/Taskfile_apis.yml b/taskfiles/Taskfile_apis.yml index 4219e1c..3410b4c 100644 --- a/taskfiles/Taskfile_apis.yml +++ b/taskfiles/Taskfile_apis.yml @@ -17,6 +17,8 @@ env_build: &env_build # The GOCACHE in a linux container on Docker for windows MUST be on a linuxy # file system GOCACHE: /tmp/datatrails/go-datatrails-common-api + GOBIN: + sh: echo ${GOBIN:-$HOME/go/bin} vars: @@ -43,7 +45,9 @@ vars: # imports GOPKG_OVERRIDES: ",Mgoogle/protobuf/field_mask.proto=google.golang.org/genproto/protobuf/field_mask" - # Upstream proto definition files are placed here by the bootstrap tasks + # proto definition files whose versions can not be controlled by tools/go.mod + # are collected in proto-include by the bootstrap tasks. Note the grpc_health + # proto's are not currently versioned and are always taken from master GOOGLE_COMMON_PROTOS_VERSION: '{{.GOOGLE_COMMON_PROTOS_VERSION | default "1.50.0"}}' GOOGLE_COMMON_INC_ARCHIVE_BASEDIR: '{{.GOOGLE_COMMON_INC_ARCHIVE_BASEDIR | default "api-common-protos"}}' @@ -96,11 +100,20 @@ tasks: - for ff in $(find . -name "*.swagger.json"); do rm -f $ff; done preflight: - summary: run prior to api generation tasks + summary: | + installs the tools/go.mod specified protoc plugins and .proto definitions + the common apis depend upon + + And prepares the .proto's under proto-include so that consumers do not + need to replicate the arrangements for finding the proto include paths + dir: '../datatrails-common-api' + vars: + API_PROTO_DEPS: >- + github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-openapiv2 + github.com/envoyproxy/protoc-gen-validate cmds: - | - go mod download go get github.com/datatrails/go-datatrails-common-api-gen/tools go install \ @@ -111,6 +124,32 @@ tasks: github.com/envoyproxy/protoc-gen-validate \ github.com/lyft/protoc-gen-star/v2 \ github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc + - for: { var: API_PROTO_DEPS } + cmd: | + depdir=$(GOCACHE=/tmp/datatrails/go-datatrails-common-api/api \ + go list -f {{"{{"}}.Dir{{"}}"}} {{.ITEM}}) + + basedir=$(basename $depdir) + basedir=${basedir%%@*} # strip the go module version + rm -rf ../proto-include/${basedir} + for proto in $(find $depdir -type f -name "*.proto"); do + target=../proto-include/${basedir}/${proto#$depdir/} + mkdir -p $(dirname $target) + cp -v $proto $target + done + # find ../proto-include/${basedir} + # find $d -type f -name "*.proto" + # echo $(dirname $d) + + # move things around so that consumers only need to -I proto-include, + # and do any general pruning or tidy up. + - | + # consumers of protoc-gen-validate expect a -I which allows them to do + # import "validate/validate.proto". And we don't need the java bindings or the example bindings + rm -rf ../proto-include/validate + mv -v ../proto-include/protoc-gen-validate/validate ../proto-include/ + rm -rf ../proto-include/protoc-gen-validate + postflight: summary: run after all api generation tasks @@ -134,7 +173,6 @@ tasks: go build ./... go test ./... - export: desc: export the sources to the desired DESTDIR dir: '../datatrails-common-api' @@ -163,7 +201,9 @@ tasks: mkdir -p {{.DESTDIR}}/swagger # The source protos are all gathered under a 'google' common top dir - cp -vr ../proto-include/* {{.DESTDIR}}proto-include/ + for d in $(ls -d ../proto-include/*); + do cp -vr $d {{.DESTDIR}}proto-include/ + done for api in {{.APIS}}; do mkdir -vp {{.DESTDIR}}{{.COMMON_GODIR}}/${api} @@ -220,7 +260,7 @@ tasks: # Generates the swagger definitions for the gRPC service. # Generates the html docs for the gRPC service. - | - protoc -I {{.SELF_INC}} {{.PROTOC_INC}} \ + protoc -I . -I proto-include \ --go_out=paths=source_relative:{{.OUTPATH}} \ --go-grpc_out=paths=source_relative:{{.OUTPATH}} \ --validate_out=lang=go,paths=source_relative:{{.OUTPATH}} \ @@ -238,36 +278,4 @@ tasks: # OUTPATH: datatrails-common-api OUTPATH: . GOBIN: - sh: echo ${GOBIN:-$HOME/go/bin} - # These variables capture the locations that .proto files for third party - # packages can be found at. The locations can change as a consequence of go - # mod module updates, so we require a dynamic approach. The particularly - # grotty bit is the -f argment - we need to pass -f {{.Dir}} to the go list - # command. To do so we have to prevent go template from evaluating it. - # - PROTO_GEN_GO_DIR: - # Notice; the tricksy 'cd datatrails-common-api' rather than just - # running in that directory via tasks 'dir:' directive is so that we can - # have the same import statements in the proto files *everywhere*. All - # consumers of datatrails-common-api get to do things very simply in - # return for this. AND it makes iteratively generating proto changes - # across repositories a lot more efficient - sh: | - echo $(./getproto.sh google.golang.org/protobuf/cmd/protoc-gen-go) - - GRPC_GATEWAY_DIR: - sh: | - echo $(dirname $(./getproto.sh github.com/grpc-ecosystem/grpc-gateway/v2/runtime)) - - ENVOY_VALIDATE: - sh: | - echo $(./getproto.sh github.com/envoyproxy/protoc-gen-validate) - - SELF_INC: . - - # Global includes used for all protoc invocations. - # task sets cwd initially to the directory of the Taskfile.yml - PROTOC_INC: "-I {{.ENVOY_VALIDATE}} \ - -I {{.PROTO_GEN_GO_DIR}} \ - -I {{.GRPC_GATEWAY_DIR}} \ - -I proto-include" \ No newline at end of file + sh: echo ${GOBIN:-$HOME/go/bin} \ No newline at end of file From 31d4d62bef8ab52c55bf561463b965ea155bad5f Mon Sep 17 00:00:00 2001 From: Robin Bryce Date: Mon, 15 Jan 2024 10:30:18 +0100 Subject: [PATCH 2/2] review changes build: use 'local-all' as the native host target and reserve 'all' for people who want no surprises fix: committed -> commit for the field carrying the log commit info move the forestrie CONFIRMED comment to the original CONFIRMED constant and remove the confusing duplicate --- .github/workflows/ci.yml | 2 +- .github/workflows/workflow.yml | 3 ++- Taskfile.yml | 8 ++++---- datatrails-common-api/assets/v2/assets/enums.proto | 6 +++--- .../assets/v2/assets/merklelogentry.proto | 2 +- 5 files changed, 11 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a750f58..5d9cd09 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,7 +30,7 @@ jobs: - name: Generate, Test and Export run: | # Note: it is by design that we don't use the builder - task all + task local-all - name: Show exports working-directory: exported/ diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 6e5fc8f..53eec60 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -21,4 +21,5 @@ jobs: echo "TODO: permissions on accessing our acr prevent this from working" exit 0 - # task dockerall + # ensure the all target (which runs everything in docker) works + # task all diff --git a/Taskfile.yml b/Taskfile.yml index ad3d363..064a705 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -67,16 +67,16 @@ tasks: ### ------------------------- # Primary workflow tasks ### ------------------------- - dockerall: + all: desc: "do everything necessary after clone (or rebase) in the builder" cmds: - task: builder-start - defer: {task: builder-cleanup} - - task: all + - task: local-all - all: + local-all: desc: | - do everything necessary after clone (or rebase) + do everything necessary after clone (or rebase) (faster as it RUNS ON NATIVE HOST) Note: This requires go and protoc installed on your host. If you are not comfortable doing this then use dockerall. diff --git a/datatrails-common-api/assets/v2/assets/enums.proto b/datatrails-common-api/assets/v2/assets/enums.proto index 7085cdd..335e707 100644 --- a/datatrails-common-api/assets/v2/assets/enums.proto +++ b/datatrails-common-api/assets/v2/assets/enums.proto @@ -5,15 +5,15 @@ option go_package="github.com/datatrails/go-datatrails-common-api-gen/assets/v2/ enum ConfirmationStatus { CONFIRMATION_STATUS_UNSPECIFIED = 0; PENDING = 1; // not yet committed - CONFIRMED = 2; // committed + CONFIRMED = 2; // committed. forestrie: "You can easily prove it changed" FAILED = 3; // permanent failure // Regarding the new statuses for forestrie, See // https://github.com/datatrails/epic-8120-scalable-proof-mechanisms/blob/main/event-trust-levels.md STORED = 4; // forestrie, "its in the db" COMMITTED = 5; // forestrie, "you can know if its changed" - // CONFIRMED = 6; // forestrie, "You can easily prove it changed" - UNEQUIVOCAL = 7; // forestrie, "You easily prove it was publicly available to all" + // We re-use the constant for CONFIRMED (above) + UNEQUIVOCAL = 6; // forestrie, "You easily prove it was publicly available to all" } enum TrackedStatus { diff --git a/datatrails-common-api/assets/v2/assets/merklelogentry.proto b/datatrails-common-api/assets/v2/assets/merklelogentry.proto index b65bd05..3f56d56 100644 --- a/datatrails-common-api/assets/v2/assets/merklelogentry.proto +++ b/datatrails-common-api/assets/v2/assets/merklelogentry.proto @@ -30,7 +30,7 @@ message MerkleLogCommitMessage { uint32 log_version = 3; uint32 log_epoch = 4; - MerkleLogCommit committed = 5; + MerkleLogCommit commit = 5; }