Merge pull request #16 from colinmcintosh/release

Testing and Building Improvements

.github/workflows/go.yml

@@ -1,4 +1,4 @@
-name: Testing
+name: Build and Test Commits
 
 on:
   push:
@@ -8,11 +8,11 @@ on:
 
 jobs:
   build:
-    name: Test
+    name: Build and Test
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        go: [ '1.13', '1.14' ]
+        go: [ '1.13', '1.14', '1.15' ]
     steps:
       - name: Set up timezone
         uses: zcong1993/[email protected]

.github/workflows/release.yml

@@ -0,0 +1,67 @@
+on:
+  release:
+    types: [published]
+
+name: Build Assets for Release Tag
+
+jobs:
+  build:
+    name: Upload Asset to Release
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        platform: [ 'linux', 'freebsd', 'windows', 'darwin' ]
+        architecture: [ 'amd64' ]
+        include:
+          - platform: linux
+            architecture: arm
+          - platform: linux
+            architecture: arm64
+    steps:
+      - name: Set up timezone
+        uses: zcong1993/[email protected]
+        with:
+          timezone: America/Los_Angeles
+
+      - name: Set up Golang
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.15
+        id: go
+
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Build asset
+        env:
+          GOOS: ${{ matrix.platform }}
+          GOARCH: ${{ matrix.architecture }}
+        run: make release
+
+      - name: Get release
+        id: get-release
+        uses: bruceadams/[email protected]
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Upload zip asset
+        id: upload-release-zip-asset
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.get-release.outputs.upload_url }}
+          asset_path: ./release/gnmi-gateway-${{ steps.get-release.outputs.tag_name }}-${{ matrix.platform }}-${{ matrix.architecture }}.zip
+          asset_name: gnmi-gateway-${{ steps.get-release.outputs.tag_name }}-${{ matrix.platform }}-${{ matrix.architecture }}.zip
+          asset_content_type: application/zip
+
+      - name: Upload sha256 asset
+        id: upload-release-sha256-asset
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.get-release.outputs.upload_url }}
+          asset_path: ./release/gnmi-gateway-${{ steps.get-release.outputs.tag_name }}-${{ matrix.platform }}-${{ matrix.architecture }}.zip.sha256
+          asset_name: gnmi-gateway-${{ steps.get-release.outputs.tag_name }}-${{ matrix.platform }}-${{ matrix.architecture }}.zip.sha256
+          asset_content_type: text/plain

.gitignore

@@ -2,6 +2,19 @@
 scratch.txt
 targets.json
 
+# Certificates
+server.crt
+server.key
+
+# Emitted binary and releases
+gnmi-gateway
+release/
+
+# OpenConfig models download
+oc-models/
+
+# pprof debugging web server
+cpu.pprof
 
 # GoLang-specific ignores
 # Binaries for programs and plugins
@@ -172,16 +185,3 @@ com_crashlytics_export_strings.xml
 crashlytics.properties
 crashlytics-build.properties
 fabric.properties
-
-# Certificates
-server.crt
-server.key
-
-# Emitted binary
-gnmi-gateway
-
-# OpenConfig models download
-oc-models/
-
-# pprof debugging web server
-cpu.pprof

Makefile

@@ -5,6 +5,8 @@ GOLDFLAGS += -X github.com/openconfig/gnmi-gateway/gateway.Version=$(VERSION)
 GOLDFLAGS += -X github.com/openconfig/gnmi-gateway/gateway.Buildtime=$(BUILDTIME)
 GOFLAGS = -ldflags "$(GOLDFLAGS)"
 
+.PHONY: build release
+
 build: clean
 	go build -o gnmi-gateway $(GOFLAGS) .
 	./gnmi-gateway -version
@@ -26,10 +28,29 @@ download:
 godoc:
 	godoc -http=":6060"
 
+imports:
+	goimports -local github.com/openconfig/gnmi-gateway -w -l $(shell find . -type f -name '*.go' -not -path './vendor/*' -not -path '*.pb.go')
+
 integration:
 	echo "Integration Test Note: Make sure you have Zookeeper running on 127.0.0.1:2181 (see README)."
 	go test -tags=integration -count=1 -cover ./...
 
+lint: imports
+	go fmt ./ ./gateway/...
+	go vet
+
+release:
+	mkdir -p release
+	rm -f release/gnmi-gateway release/gnmi-gateway.exe
+ifeq ($(shell go env GOOS), windows)
+	go build -o release/gnmi-gateway.exe $(GOFLAGS) .
+	cd release; zip -m "gnmi-gateway-$(shell git describe --tags --abbrev=0)-$(shell go env GOOS)-$(shell go env GOARCH).zip" gnmi-gateway.exe
+else
+	go build -o release/gnmi-gateway $(GOFLAGS) .
+	cd release; zip -m "gnmi-gateway-$(shell git describe --tags --abbrev=0)-$(shell go env GOOS)-$(shell go env GOARCH).zip" gnmi-gateway
+endif
+	cd release; sha256sum "gnmi-gateway-$(shell git describe --tags --abbrev=0)-$(shell go env GOOS)-$(shell go env GOARCH).zip" > "gnmi-gateway-$(shell git describe --tags --abbrev=0)-$(shell go env GOOS)-$(shell go env GOARCH).zip.sha256"
+
 run: build
 	./gnmi-gateway -EnableGNMIServer -ServerTLSCert=server.crt -ServerTLSKey=server.key -TargetLoaders=json -TargetJSONFile=targets.json
 

README.md

@@ -42,7 +42,26 @@ gnmi-gateway cache, gNMI clients with relevant subscriptions, and
 Target Loaders are components that are used to generate target connection
 configurations that are sent to the connection manager. Target Loaders
 and the connection manager communicate using the [target.proto][6] model
-found in the github.com/openconfig/gnmi repository. See
+found in the github.com/openconfig/gnmi repository. gnmi-gateway accepts a few
+additional parameters in the Target.meta field:
+
+    NoTLSVerify: inlcude this field to disable TLS verification. This enables
+                 the use of self-signed certificates. Note that connections
+                 without TLS are not supported per the gNMI specification.
+
+    NoLock: include this field to disable locking for the associated target even
+            if clustering is enabled. Only include this field if you are
+            handling de-duplication outside of gnmi-gateway.
+               
+There are a few Target Loaders included with gnmi-gateway that you can use
+right away using the `-TargetLoaders` flag from the command-line. The Target
+Loaders included are:
+
+- [json](./gateway/loaders/json/json.go)
+- [netbox](./gateway/loaders/netbox/netbox.go)
+- [simple](./gateway/loaders/simple/simple.go)
+
+If you'd like to build your own Target Loader see
 [loaders/loader.go](./gateway/loaders/loader.go) for details on how to
 implement the TargetLoader interface.
 
@@ -60,8 +79,17 @@ connections or you can run exporters on a server acting as clients to another
 gnmi-gateway cluster. This allows for some flexibility in your deployment
 design.
 
-See [exporters/exporter.go](./gateway/exporters/exporter.go) for details on
-how to implement the Exporter interface.
+Some Exporters have been included with gnmi-gateway and you can start using them
+by providing a comma-separated list of Exporters from the command-line with the
+`-Exporters` flag. The included Exporters are:
+
+- [debug](./gateway/exporters/debug/debug.go) (log to stdout)
+- [kafka](./gateway/exporters/kafka/kafka.go)
+- [prometheus](./gateway/exporters/prometheus/prometheus.go)
+
+To build a custom Exporter see
+[exporters/exporter.go](./gateway/exporters/exporter.go) for details on how to
+implement the Exporter interface.
 
 
 ## Documentation

gateway/clustering/zookeeper.go

@@ -17,10 +17,12 @@ package clustering
 
 import (
 	"fmt"
-	"github.com/go-zookeeper/zk"
-	"github.com/openconfig/gnmi-gateway/gateway/configuration"
 	"strings"
 	"time"
+
+	"github.com/go-zookeeper/zk"
+
+	"github.com/openconfig/gnmi-gateway/gateway/configuration"
 )
 
 const ClusterMemberPath = "/members"

gateway/clustering/zookeeper_integration_test.go

@@ -19,13 +19,15 @@ package clustering_test
 
 import (
 	"fmt"
-	"github.com/go-zookeeper/zk"
-	"github.com/openconfig/gnmi-gateway/gateway/clustering"
-	"github.com/openconfig/gnmi-gateway/gateway/configuration"
-	"github.com/stretchr/testify/assert"
 	"sync"
 	"testing"
 	"time"
+
+	"github.com/go-zookeeper/zk"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/openconfig/gnmi-gateway/gateway/clustering"
+	"github.com/openconfig/gnmi-gateway/gateway/configuration"
 )
 
 const ZookeeperIntegrationServerAddress = "127.0.0.1:2181"

gateway/configuration/config.go

@@ -22,14 +22,15 @@ import (
 	"crypto/tls"
 	"encoding/json"
 	"fmt"
-	"github.com/Netflix/spectator-go"
-	gnmipb "github.com/openconfig/gnmi/proto/gnmi"
-	"github.com/rs/zerolog"
-	"google.golang.org/grpc/credentials"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"time"
+
+	"github.com/Netflix/spectator-go"
+	gnmipb "github.com/openconfig/gnmi/proto/gnmi"
+	"github.com/rs/zerolog"
+	"google.golang.org/grpc/credentials"
 )
 
 // GatewayConfig contains all of the configurables and tunables for various components of the gateway.

gateway/connections/state.go

@@ -33,21 +33,23 @@ import (
 	"context"
 	"crypto/tls"
 	"fmt"
+	"sync"
+	"time"
+
 	"github.com/Netflix/spectator-go"
 	"github.com/Netflix/spectator-go/histogram"
 	"github.com/golang/protobuf/proto"
-	"github.com/openconfig/gnmi-gateway/gateway/configuration"
-	"github.com/openconfig/gnmi-gateway/gateway/locking"
-	"github.com/openconfig/gnmi-gateway/gateway/stats"
-	"github.com/openconfig/gnmi-gateway/gateway/utils"
 	"github.com/openconfig/gnmi/cache"
 	"github.com/openconfig/gnmi/client"
 	gnmiclient "github.com/openconfig/gnmi/client/gnmi"
 	gnmipb "github.com/openconfig/gnmi/proto/gnmi"
 	targetpb "github.com/openconfig/gnmi/proto/target"
 	"golang.org/x/sync/semaphore"
-	"sync"
-	"time"
+
+	"github.com/openconfig/gnmi-gateway/gateway/configuration"
+	"github.com/openconfig/gnmi-gateway/gateway/locking"
+	"github.com/openconfig/gnmi-gateway/gateway/stats"
+	"github.com/openconfig/gnmi-gateway/gateway/utils"
 )
 
 // ConnectionState makes the calls to connect a target, tracks any associated connection state, and is the container for
@@ -69,6 +71,9 @@ type ConnectionState struct {
 	lock locking.DistributedLocker
 	// The unique name of the target that is being connected to
 	name        string
+	// noTLSWarning indicates if the warning about the NoTLS flag deprecation
+	// has been displayed yet.
+	noTLSWarning bool
 	queryTarget string
 	request     *gnmipb.SubscribeRequest
 	// seen is the list of targets that have been seen on this connection
@@ -159,9 +164,15 @@ func (t *ConnectionState) doConnect() {
 
 	// TODO (cmcintosh): make PR for targetpb to include TLS config and remove this
 	_, NoTLS := t.target.Meta["NoTLS"]
+	if NoTLS  && !t.noTLSWarning {
+		t.noTLSWarning = true
+		t.config.Log.Warn().Msg("DEPRECATED: The 'NoTLS' target flag has been deprecated and will be removed in a future release. Please use 'NoTLSVerify' instead.")
+	}
+
+	_, NoTLSVerify := t.target.Meta["NoTLSVerify"]
 
 	// TLS is always enabled for localTargets but we won't verify certs if no client TLS config exists.
-	if t.config.ClientTLSConfig != nil && !NoTLS {
+	if t.config.ClientTLSConfig != nil && (!NoTLS || !NoTLSVerify) {
 		query.TLS = t.config.ClientTLSConfig
 	} else {
 		query.TLS = &tls.Config{

gateway/connections/zookeeper.go

@@ -21,12 +21,13 @@ import (
 	"sync"
 
 	"github.com/go-zookeeper/zk"
-	"github.com/openconfig/gnmi-gateway/gateway/configuration"
-	"github.com/openconfig/gnmi-gateway/gateway/locking"
 	"github.com/openconfig/gnmi/cache"
 	targetlib "github.com/openconfig/gnmi/target"
 	"github.com/rs/zerolog/log"
 	"golang.org/x/sync/semaphore"
+
+	"github.com/openconfig/gnmi-gateway/gateway/configuration"
+	"github.com/openconfig/gnmi-gateway/gateway/locking"
 )
 
 var _ ConnectionManager = new(ZookeeperConnectionManager)

gateway/connections/zookeeper_test.go

@@ -16,10 +16,12 @@
 package connections
 
 import (
-	"github.com/openconfig/gnmi-gateway/gateway/configuration"
+	"testing"
+
 	"github.com/openconfig/gnmi/client"
 	"github.com/stretchr/testify/assert"
-	"testing"
+
+	"github.com/openconfig/gnmi-gateway/gateway/configuration"
 )
 
 func TestNewZookeeperConnectionManagerDefault(t *testing.T) {

gateway/exporters/all/all.go

@@ -15,6 +15,8 @@
 
 package all
 
-import _ "github.com/openconfig/gnmi-gateway/gateway/exporters/debug"
-import _ "github.com/openconfig/gnmi-gateway/gateway/exporters/kafka"
-import _ "github.com/openconfig/gnmi-gateway/gateway/exporters/prometheus"
+import (
+	_ "github.com/openconfig/gnmi-gateway/gateway/exporters/debug"
+	_ "github.com/openconfig/gnmi-gateway/gateway/exporters/kafka"
+	_ "github.com/openconfig/gnmi-gateway/gateway/exporters/prometheus"
+)

gateway/exporters/debug/debug.go

@@ -18,12 +18,13 @@
 package debug
 
 import (
-	"github.com/openconfig/gnmi-gateway/gateway/configuration"
-	"github.com/openconfig/gnmi-gateway/gateway/exporters"
-	"github.com/openconfig/gnmi-gateway/gateway/utils"
 	"github.com/openconfig/gnmi/cache"
 	"github.com/openconfig/gnmi/ctree"
 	gnmipb "github.com/openconfig/gnmi/proto/gnmi"
+
+	"github.com/openconfig/gnmi-gateway/gateway/configuration"
+	"github.com/openconfig/gnmi-gateway/gateway/exporters"
+	"github.com/openconfig/gnmi-gateway/gateway/utils"
 )
 
 const Name = "debug"