Skip to content

Commit

Permalink
update protoc and related (#552)
Browse files Browse the repository at this point in the history
* update protoc and related

updated protoc to v25.2
updated protoc-gen-go to v1.32.0
added protoc-gen-go-grpc at v1.3.0

Modified the makefile to use "go install" to install tools.
Use extended regular expressions in "sed" command.

Fixes build on arm OSX host

* fix github actions

Always run "make build_go" on github actions

Always update the timesptamp of spec.md/csi.proto to make sure lib/go is built on CI

* remove github.com/golang/protobuf usage

* simplify Makefile

- use git to check for out-of-date generated files, avoids all the temp files and manual diff in the Makefile.
- not specify version for protoc-gen-go, it will use the version from go.mod
- do not copy over the protoc binary. Call it directly from unzipped path. So it can find the includes automatically, and we don't need the -I or --go_opts=Mxxx flags.
- use paths=source_relative to generate go files directly into destination, avoid the copy.
- simplify the sed command used to extract "csi.proto" file.
  • Loading branch information
huww98 authored Mar 5, 2024
1 parent 886cd48 commit abf6346
Show file tree
Hide file tree
Showing 9 changed files with 7,712 additions and 4,615 deletions.
5 changes: 2 additions & 3 deletions .github/workflows/build.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ jobs:

- name: Build Test
run: |
touch spec.md # Ensure its timestamp is newer than any generated files
make
make -C lib/go protoc
make -C lib/go protoc-gen-go
make build_go
git diff --exit-code || (echo "Generated files are out of date. Please run 'make' and commit the changes." && exit 1)
46 changes: 9 additions & 37 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -4,50 +4,22 @@ CSI_SPEC := spec.md
CSI_PROTO := csi.proto
## Build go language bindings
CSI_A := csi.a
CSI_GO := lib/go/csi/csi.pb.go
CSI_PKG := lib/go/csi

# This is the target for building the temporary CSI protobuf file.
#
# The temporary file is not versioned, and thus will always be
# built on GitHub Actions.
$(CSI_PROTO).tmp: $(CSI_SPEC) Makefile
echo "// Code generated by make; DO NOT EDIT." > "$@"
cat $< | sed -n -e '/```protobuf$$/,/^```$$/ p' | sed '/^```/d' >> "$@"

# This is the target for building the CSI protobuf file.
#
# This target depends on its temp file, which is not versioned.
# Therefore when built on GitHub Actions the temp file will always
# be built and trigger this target. On GitHub Actions the temp file
# is compared with the real file, and if they differ the build
# will fail.
#
# Locally the temp file is simply copied over the real file.
$(CSI_PROTO): $(CSI_PROTO).tmp
ifeq (true,$(GITHUB_ACTIONS))
diff "$@" "$?"
else
diff "$@" "$?" > /dev/null 2>&1 || cp -f "$?" "$@"
endif

build: check

# If this is not running on GitHub Actions then for sake of convenience
# go ahead and update the language bindings as well.
ifneq (true,$(GITHUB_ACTIONS))
build: build_cpp build_go
endif
$(CSI_PROTO): $(CSI_SPEC) Makefile
echo "// Code generated by make; DO NOT EDIT." > "$@"
sed -n -e '/```protobuf$$/,/^```$$/ {//!p;}' $< >> "$@"

build: check build_cpp build_go

build_cpp:
$(MAKE) -C lib/cxx

# The file exists, but could be out-of-date.
$(CSI_GO): $(CSI_PROTO)
$(MAKE) -C lib/go csi/csi.pb.go
$(CSI_PKG)/%.pb.go: $(CSI_PROTO)
$(MAKE) -C lib/go

$(CSI_A): $(CSI_GO)
$(CSI_A): $(CSI_PKG)/*.go
go mod download
go install ./$(CSI_PKG)
go build -o "$@" ./$(CSI_PKG)
Expand All @@ -60,10 +32,10 @@ clean:

clobber: clean
$(MAKE) -C lib/go $@
rm -f $(CSI_PROTO) $(CSI_PROTO).tmp
rm -f $(CSI_PROTO)

# check generated files for violation of standards
check: $(CSI_PROTO)
awk '{ if (length > 72) print NR, $$0 }' $? | diff - /dev/null

.PHONY: clean clobber check
.PHONY: clean clobber check build_go build_cpp
3 changes: 2 additions & 1 deletion csi.proto
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@ import "google/protobuf/descriptor.proto";
import "google/protobuf/timestamp.proto";
import "google/protobuf/wrappers.proto";

option go_package = "csi";
option go_package =
"github.com/container-storage-interface/spec/lib/go/csi";

extend google.protobuf.EnumOptions {
// Indicates that this enum is OPTIONAL and part of an experimental
Expand Down
4 changes: 2 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,14 @@ module github.com/container-storage-interface/spec
go 1.18

require (
github.com/golang/protobuf v1.5.3
google.golang.org/grpc v1.57.0
google.golang.org/protobuf v1.32.0
)

require (
github.com/golang/protobuf v1.5.3 // indirect
golang.org/x/net v0.10.0 // indirect
golang.org/x/sys v0.8.0 // indirect
golang.org/x/text v0.9.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5 // indirect
google.golang.org/protobuf v1.31.0 // indirect
)
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -16,5 +16,5 @@ google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw=
google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I=
google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
92 changes: 31 additions & 61 deletions lib/go/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -14,14 +14,19 @@ GOPATH := $(word 1,$(subst :, ,$(GOPATH)))
endif
export GOPATH

GOBIN := $(shell go env GOBIN)
ifeq (,$(strip $(GOBIN)))
GOBIN := $(GOPATH)/bin
endif


########################################################################
## PROTOC ##
########################################################################

# Only set PROTOC_VER if it has an empty value.
ifeq (,$(strip $(PROTOC_VER)))
PROTOC_VER := 3.9.1
PROTOC_VER := 25.2
endif

PROTOC_OS := $(shell uname -s)
Expand All @@ -32,95 +37,60 @@ endif
PROTOC_ARCH := $(shell uname -m)
ifeq (i386,$(PROTOC_ARCH))
PROTOC_ARCH := x86_32
else ifeq (arm64,$(PROTOC_ARCH))
PROTOC_ARCH := aarch_64
endif

PROTOC := ./protoc
PROTOC_ZIP := protoc-$(PROTOC_VER)-$(PROTOC_OS)-$(PROTOC_ARCH).zip
PROTOC_URL := https://github.com/google/protobuf/releases/download/v$(PROTOC_VER)/$(PROTOC_ZIP)
PROTOC_URL := https://github.com/protocolbuffers/protobuf/releases/download/v$(PROTOC_VER)/$(PROTOC_ZIP)
PROTOC_TMP_DIR := .protoc
PROTOC_TMP_BIN := $(PROTOC_TMP_DIR)/bin/protoc
PROTOC := $(PROTOC_TMP_DIR)/bin/protoc

$(GOBIN)/protoc-gen-go: ../../go.mod
go install google.golang.org/protobuf/cmd/protoc-gen-go
$(GOBIN)/protoc-gen-go-grpc:
go install google.golang.org/grpc/cmd/[email protected]

$(PROTOC):
-mkdir -p "$(PROTOC_TMP_DIR)" && \
curl -L $(PROTOC_URL) -o "$(PROTOC_TMP_DIR)/$(PROTOC_ZIP)" && \
unzip "$(PROTOC_TMP_DIR)/$(PROTOC_ZIP)" -d "$(PROTOC_TMP_DIR)" && \
chmod 0755 "$(PROTOC_TMP_BIN)" && \
cp -f "$(PROTOC_TMP_BIN)" "$@"
chmod 0755 "$@"
stat "$@" > /dev/null 2>&1


########################################################################
## PROTOC-GEN-GO ##
########################################################################

# This is the recipe for getting and installing the go plug-in
# for protoc
PROTOC_GEN_GO_PKG := github.com/golang/protobuf/protoc-gen-go
PROTOC_GEN_GO := protoc-gen-go
$(PROTOC_GEN_GO): PROTOBUF_PKG := $(dir $(PROTOC_GEN_GO_PKG))
$(PROTOC_GEN_GO): PROTOBUF_VERSION := v1.3.2
$(PROTOC_GEN_GO):
mkdir -p $(dir $(GOPATH)/src/$(PROTOBUF_PKG))
test -d $(GOPATH)/src/$(PROTOBUF_PKG)/.git || git clone https://$(PROTOBUF_PKG) $(GOPATH)/src/$(PROTOBUF_PKG)
(cd $(GOPATH)/src/$(PROTOBUF_PKG) && \
(test "$$(git describe --tags | head -1)" = "$(PROTOBUF_VERSION)" || \
(git fetch && git checkout tags/$(PROTOBUF_VERSION))))
(cd $(GOPATH)/src/$(PROTOBUF_PKG) && go get -v -d $$(go list -f '{{ .ImportPath }}' ./...)) && \
go build -o "$@" $(PROTOC_GEN_GO_PKG)

PROTOC_ALL := $(GOBIN)/protoc-gen-go $(GOBIN)/protoc-gen-go-grpc $(PROTOC)

########################################################################
## PATH ##
########################################################################

# Update PATH with the current directory. This enables the protoc
# binary to discover the protoc-gen-go binary, built inside this
# directory.
export PATH := $(shell pwd):$(PATH)
# Update PATH with GOBIN. This enables the protoc binary to discover
# the protoc-gen-go binary
export PATH := $(GOBIN):$(PATH)


########################################################################
## BUILD ##
########################################################################
CSI_PROTO := ../../csi.proto
CSI_PKG_ROOT := github.com/container-storage-interface/spec
CSI_PKG_SUB := $(shell cat $(CSI_PROTO) | sed -n -e 's/^package.\([^;]*\).v[0-9]\+;$$/\1/p'|tr '.' '/')
CSI_BUILD := $(CSI_PKG_SUB)/.build
CSI_PKG_SUB := csi
CSI_GO := $(CSI_PKG_SUB)/csi.pb.go
CSI_GO_TMP := $(CSI_BUILD)/$(CSI_PKG_ROOT)/csi.pb.go

# This recipe generates the go language bindings to a temp area.
$(CSI_GO_TMP): HERE := $(shell pwd)
$(CSI_GO_TMP): PTYPES_PKG := github.com/golang/protobuf/ptypes
$(CSI_GO_TMP): GO_OUT := plugins=grpc
$(CSI_GO_TMP): GO_OUT := $(GO_OUT),Mgoogle/protobuf/descriptor.proto=github.com/golang/protobuf/protoc-gen-go/descriptor
$(CSI_GO_TMP): GO_OUT := $(GO_OUT),Mgoogle/protobuf/wrappers.proto=$(PTYPES_PKG)/wrappers
$(CSI_GO_TMP): GO_OUT := $(GO_OUT):"$(HERE)/$(CSI_BUILD)"
$(CSI_GO_TMP): INCLUDE := -I$(GOPATH)/src -I$(HERE)/$(PROTOC_TMP_DIR)/include
$(CSI_GO_TMP): $(CSI_PROTO) | $(PROTOC) $(PROTOC_GEN_GO)
@mkdir -p "$(@D)"
(cd "$(GOPATH)/src" && \
$(HERE)/$(PROTOC) $(INCLUDE) --go_out=$(GO_OUT) "$(CSI_PKG_ROOT)/$(<F)")

# The temp language bindings are compared to the ones that are
# versioned. If they are different then it means the language
# bindings were not updated prior to being committed.
$(CSI_GO): $(CSI_GO_TMP)
ifeq (true,$(GITHUB_ACTIONS))
diff "$@" "$?"
else
@mkdir -p "$(@D)"
diff "$@" "$?" > /dev/null 2>&1 || cp -f "$?" "$@"
endif
CSI_GRPC := $(CSI_PKG_SUB)/csi_grpc.pb.go

# This recipe generates the go language bindings
$(CSI_GO) $(CSI_GRPC): $(CSI_PROTO) $(PROTOC_ALL)
@mkdir -p "$(@D)"
$(PROTOC) -I../.. --go-grpc_out=$(CSI_PKG_SUB) --go_out=$(CSI_PKG_SUB) \
--go_opt=paths=source_relative --go-grpc_opt=paths=source_relative \
"$(<F)"

build: $(CSI_GO)
build: $(CSI_GO) $(CSI_GRPC)

clean:
go clean -i ./...
rm -rf "$(CSI_GO)" "$(CSI_BUILD)"
rm -rf "$(CSI_PKG_SUB)"

clobber: clean
rm -fr "$(PROTOC)" "$(PROTOC_GEN_GO)" "$(CSI_PKG_SUB)"
rm -fr "$(PROTOC_TMP_DIR)"

.PHONY: clean clobber
Loading

0 comments on commit abf6346

Please sign in to comment.