diff --git a/.travis.yml b/.travis.yml index 83efafcbc..9a987ab7c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,7 @@ +version: ~> 1.0 language: go go: -- 1.x +- "1.13.x" dist: bionic go_import_path: github.com/google/keytransparency diff --git a/README.md b/README.md index 7d04fe30b..ae93265ba 100644 --- a/README.md +++ b/README.md @@ -32,8 +32,8 @@ development. ## Key Transparency Client ### Setup -1. Install [Go 1.10](https://golang.org/doc/install). -2. `go get -u github.com/google/keytransparency/cmd/keytransparency-client ` +1. Install [Go 1.13](https://golang.org/doc/install). +2. `go get github.com/google/keytransparency/cmd/keytransparency-client ` ### Client operations @@ -96,8 +96,8 @@ NB A default for the Key Transparency server URL is being used here. The default - Docker Compose 1.11.0+ `docker-compose --version` ```sh -go get -u github.com/google/keytransparency/... -go get -u github.com/google/trillian/... +go get github.com/google/keytransparency/... +go get github.com/google/trillian/... cd $(go env GOPATH)/src/github.com/google/keytransparency ./scripts/prepare_server.sh -f docker-compose -f docker-compose.yml docker-compose.prod.yml up diff --git a/cmd/keytransparency-monitor/main.go b/cmd/keytransparency-monitor/main.go index b34a518d5..6cec18375 100644 --- a/cmd/keytransparency-monitor/main.go +++ b/cmd/keytransparency-monitor/main.go @@ -16,13 +16,14 @@ package main import ( "context" + "crypto" "crypto/tls" "flag" "net" "net/http" "github.com/golang/glog" - "github.com/google/trillian/crypto" + tcrypto "github.com/google/trillian/crypto" "github.com/google/trillian/crypto/keys/pem" "github.com/prometheus/client_golang/prometheus/promhttp" "google.golang.org/grpc" @@ -75,7 +76,7 @@ func main() { if err != nil { glog.Exitf("Could not create signer from %v: %v", *signingKey, err) } - signer := crypto.NewSHA256Signer(key) + signer := tcrypto.NewSigner(0, key, crypto.SHA256) store := fake.NewMonitorStorage() // Create monitoring background process. diff --git a/cmd/keytransparency-sequencer/main.go b/cmd/keytransparency-sequencer/main.go index b2931f228..71f0d8035 100644 --- a/cmd/keytransparency-sequencer/main.go +++ b/cmd/keytransparency-sequencer/main.go @@ -16,7 +16,6 @@ package main import ( "context" - "database/sql" "flag" "fmt" "net" @@ -44,10 +43,10 @@ import ( pb "github.com/google/keytransparency/core/api/v1/keytransparency_go_proto" dir "github.com/google/keytransparency/core/directory" spb "github.com/google/keytransparency/core/sequencer/sequencer_go_proto" + ktsql "github.com/google/keytransparency/impl/sql" etcdelect "github.com/google/trillian/util/election2/etcd" grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" - _ "github.com/go-sql-driver/mysql" // Set database engine. _ "github.com/google/trillian/crypto/keys/der/proto" _ "github.com/google/trillian/merkle/coniks" // Register hasher _ "github.com/google/trillian/merkle/rfc6962" // Register hasher @@ -74,17 +73,6 @@ var ( batchSize = flag.Int("batch-size", 100, "Maximum number of mutations to process per map revision") ) -func openDB() *sql.DB { - db, err := sql.Open("mysql", *serverDBPath) - if err != nil { - glog.Exitf("sql.Open(): %v", err) - } - if err := db.Ping(); err != nil { - glog.Exitf("db.Ping(): %v", err) - } - return db -} - // getElectionFactory returns an election factory based on flags, and a // function which releases the resources associated with the factory. func getElectionFactory() (election2.Factory, func()) { @@ -130,7 +118,10 @@ func main() { } // Database tables - sqldb := openDB() + sqldb, err := ktsql.Open(*serverDBPath) + if err != nil { + glog.Exit(err) + } defer sqldb.Close() mutations, err := mutationstorage.New(sqldb) diff --git a/cmd/keytransparency-server/main.go b/cmd/keytransparency-server/main.go index 60e86222f..d65711153 100644 --- a/cmd/keytransparency-server/main.go +++ b/cmd/keytransparency-server/main.go @@ -16,7 +16,6 @@ package main import ( "context" - "database/sql" "flag" "net/http" @@ -37,11 +36,11 @@ import ( "github.com/google/keytransparency/impl/sql/mutationstorage" pb "github.com/google/keytransparency/core/api/v1/keytransparency_go_proto" + ktsql "github.com/google/keytransparency/impl/sql" grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" grpc_auth "github.com/grpc-ecosystem/go-grpc-middleware/auth" grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" - _ "github.com/go-sql-driver/mysql" // Set database engine. _ "github.com/google/trillian/crypto/keys/der/proto" ) @@ -58,23 +57,15 @@ var ( revisionPageSize = flag.Int("revision-page-size", 10, "Max number of revisions to return at once") ) -func openDB() *sql.DB { - db, err := sql.Open("mysql", *serverDBPath) - if err != nil { - glog.Exitf("sql.Open(): %v", err) - } - if err := db.Ping(); err != nil { - glog.Exitf("db.Ping(): %v", err) - } - return db -} - func main() { flag.Parse() ctx := context.Background() // Open Resources. - sqldb := openDB() + sqldb, err := ktsql.Open(*serverDBPath) + if err != nil { + glog.Exit(err) + } defer sqldb.Close() creds, err := credentials.NewServerTLSFromFile(*certFile, *keyFile) diff --git a/core/integration/monitor_tests.go b/core/integration/monitor_tests.go index ea24b43cb..350cfa2e6 100644 --- a/core/integration/monitor_tests.go +++ b/core/integration/monitor_tests.go @@ -16,6 +16,7 @@ package integration import ( "context" + "crypto" "sync" "testing" "time" @@ -28,7 +29,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "github.com/google/trillian/crypto" + tcrypto "github.com/google/trillian/crypto" "github.com/google/trillian/crypto/keys/pem" tpb "github.com/google/keytransparency/core/testdata/transcript_go_proto" @@ -49,7 +50,7 @@ func TestMonitor(ctx context.Context, env *Env, t *testing.T) []*tpb.Action { if err != nil { t.Fatalf("Couldn't create signer: %v", err) } - signer := crypto.NewSHA256Signer(privKey) + signer := tcrypto.NewSigner(0, privKey, crypto.SHA256) store := fake.NewMonitorStorage() mon, err := monitor.NewFromDirectory(env.Cli, env.Directory, signer, store) if err != nil { diff --git a/core/integration/storagetest/batch.go b/core/integration/storagetest/batch.go index 556730470..636a86a78 100644 --- a/core/integration/storagetest/batch.go +++ b/core/integration/storagetest/batch.go @@ -27,15 +27,13 @@ import ( spb "github.com/google/keytransparency/core/sequencer/sequencer_go_proto" ) -// Batcher writes batch definitions to storage. -type Batcher = sequencer.Batcher +// batchStorageFactory returns a new database object, and a function for cleaning it up. +type batchStorageFactory func(ctx context.Context, t *testing.T, dirID string) (sequencer.Batcher, func(context.Context)) -type BatchStorageFactory func(ctx context.Context, t *testing.T, dirID string) Batcher - -type BatchStorageTest func(ctx context.Context, t *testing.T, f BatchStorageFactory) +type BatchStorageTest func(ctx context.Context, t *testing.T, f batchStorageFactory) // RunBatchStorageTests runs all the batch storage tests against the provided map storage implementation. -func RunBatchStorageTests(t *testing.T, factory BatchStorageFactory) { +func RunBatchStorageTests(t *testing.T, factory batchStorageFactory) { ctx := context.Background() b := &BatchTests{} for name, f := range map[string]BatchStorageTest{ @@ -52,9 +50,10 @@ func RunBatchStorageTests(t *testing.T, factory BatchStorageFactory) { // BatchTests is a suite of tests to run against type BatchTests struct{} -func (*BatchTests) TestNotFound(ctx context.Context, t *testing.T, f BatchStorageFactory) { +func (*BatchTests) TestNotFound(ctx context.Context, t *testing.T, f batchStorageFactory) { domainID := "testnotfounddir" - b := f(ctx, t, domainID) + b, done := f(ctx, t, domainID) + defer done(ctx) _, err := b.ReadBatch(ctx, domainID, 0) st := status.Convert(err) if got, want := st.Code(), codes.NotFound; got != want { @@ -62,9 +61,10 @@ func (*BatchTests) TestNotFound(ctx context.Context, t *testing.T, f BatchStorag } } -func (*BatchTests) TestWriteBatch(ctx context.Context, t *testing.T, f BatchStorageFactory) { +func (*BatchTests) TestWriteBatch(ctx context.Context, t *testing.T, f batchStorageFactory) { domainID := "writebatchtest" - b := f(ctx, t, domainID) + b, done := f(ctx, t, domainID) + defer done(ctx) for _, tc := range []struct { rev int64 wantErr bool @@ -86,9 +86,10 @@ func (*BatchTests) TestWriteBatch(ctx context.Context, t *testing.T, f BatchStor } } -func (*BatchTests) TestReadBatch(ctx context.Context, t *testing.T, f BatchStorageFactory) { +func (*BatchTests) TestReadBatch(ctx context.Context, t *testing.T, f batchStorageFactory) { domainID := "readbatchtest" - b := f(ctx, t, domainID) + b, done := f(ctx, t, domainID) + defer done(ctx) for _, tc := range []struct { rev int64 want *spb.MapMetadata @@ -115,9 +116,10 @@ func (*BatchTests) TestReadBatch(ctx context.Context, t *testing.T, f BatchStora } } -func (*BatchTests) TestHighestRev(ctx context.Context, t *testing.T, f BatchStorageFactory) { +func (*BatchTests) TestHighestRev(ctx context.Context, t *testing.T, f batchStorageFactory) { domainID := "writebatchtest" - b := f(ctx, t, domainID) + b, done := f(ctx, t, domainID) + defer done(ctx) for _, tc := range []struct { rev int64 sources []*spb.MapMetadata_SourceSlice diff --git a/core/integration/storagetest/mutation_logs.go b/core/integration/storagetest/mutation_logs.go index 5fdbd1c99..35c15cef8 100644 --- a/core/integration/storagetest/mutation_logs.go +++ b/core/integration/storagetest/mutation_logs.go @@ -16,24 +16,28 @@ package storagetest import ( "context" + "fmt" "testing" "time" "github.com/golang/protobuf/proto" + "github.com/google/go-cmp/cmp" "github.com/google/keytransparency/core/keyserver" pb "github.com/google/keytransparency/core/api/v1/keytransparency_go_proto" ) -type MutationLogsFactory func(ctx context.Context, t *testing.T, dirID string, logIDs ...int64) keyserver.MutationLogs +// mutationLogsFactory returns a new database object, and a function for cleaning it up. +type mutationLogsFactory func(ctx context.Context, t *testing.T, dirID string, logIDs ...int64) (keyserver.MutationLogs, func(context.Context)) // RunMutationLogsTests runs all the tests against the provided storage implementation. -func RunMutationLogsTests(t *testing.T, factory MutationLogsFactory) { +func RunMutationLogsTests(t *testing.T, factory mutationLogsFactory) { ctx := context.Background() b := &mutationLogsTests{} - for name, f := range map[string]func(ctx context.Context, t *testing.T, f MutationLogsFactory){ + for name, f := range map[string]func(ctx context.Context, t *testing.T, f mutationLogsFactory){ // TODO(gbelvin): Discover test methods via reflection. - "TestReadLog": b.TestReadLog, + "TestReadLog": b.TestReadLog, + "TestReadLogExact": b.TestReadLogExact, } { t.Run(name, func(t *testing.T) { f(ctx, t, factory) }) } @@ -50,35 +54,87 @@ func mustMarshal(t *testing.T, p proto.Message) []byte { return b } +// https://dev.mysql.com/doc/refman/8.0/en/datetime.html +var minWatermark = time.Date(1000, 1, 1, 0, 0, 0, 0, time.UTC) + // TestReadLog ensures that reads happen in atomic units of batch size. -func (mutationLogsTests) TestReadLog(ctx context.Context, t *testing.T, newForTest MutationLogsFactory) { +func (mutationLogsTests) TestReadLog(ctx context.Context, t *testing.T, newForTest mutationLogsFactory) { directoryID := "TestReadLog" logID := int64(5) // Any log ID. - m := newForTest(ctx, t, directoryID, logID) - // Write ten batches, three entries each. + m, done := newForTest(ctx, t, directoryID, logID) + defer done(ctx) + // Write ten batches. for i := byte(0); i < 10; i++ { entry := &pb.EntryUpdate{Mutation: &pb.SignedEntry{Entry: mustMarshal(t, &pb.Entry{Index: []byte{i}})}} - if _, err := m.Send(ctx, directoryID, entry, entry, entry); err != nil { + if _, err := m.Send(ctx, directoryID, logID, entry, entry, entry); err != nil { t.Fatalf("Send(): %v", err) } } - for _, tc := range []struct { + for i, tc := range []struct { limit int32 - count int + want int }{ - {limit: 0, count: 0}, - {limit: 1, count: 3}, // We asked for 1 item, which gets us into the first batch, so we return 3 items. - {limit: 3, count: 3}, // We asked for 3 items, which gets us through the first batch, so we return 3 items. - {limit: 4, count: 6}, // Reading 4 items gets us into the second batch of size 3. - {limit: 100, count: 30}, // Reading all the items gets us the 30 items we wrote. + {limit: 0, want: 0}, + {limit: 1, want: 3}, // We asked for 1 item, which gets us into the first batch, so we return 3 items. + {limit: 3, want: 3}, // We asked for 3 items, which gets us through the first batch, so we return 3 items. + {limit: 4, want: 6}, // Reading 4 items gets us into the second batch of size 3. + {limit: 100, want: 30}, // Reading all the items gets us the 30 items we wrote. } { - rows, err := m.ReadLog(ctx, directoryID, logID, 0, time.Now().UnixNano(), tc.limit) + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + rows, err := m.ReadLog(ctx, directoryID, logID, minWatermark, time.Now(), tc.limit) + if err != nil { + t.Fatalf("ReadLog(%v): %v", tc.limit, err) + } + if got := len(rows); got != tc.want { + t.Fatalf("ReadLog(%v): len: %v, want %v", tc.limit, got, tc.want) + } + }) + } +} + +// TestReadLogExact ensures that reads respect the low inclusive, high exclusive API. +func (mutationLogsTests) TestReadLogExact(ctx context.Context, t *testing.T, newForTest mutationLogsFactory) { + directoryID := "TestReadLogExact" + logID := int64(5) // Any log ID. + m, done := newForTest(ctx, t, directoryID, logID) + defer done(ctx) + // Write ten batches. + idx := make([]time.Time, 0, 10) + for i := byte(0); i < 10; i++ { + entry := &pb.EntryUpdate{Mutation: &pb.SignedEntry{Entry: []byte{i}}} + ts, err := m.Send(ctx, directoryID, logID, entry) if err != nil { - t.Fatalf("ReadLog(%v): %v", tc.limit, err) - } - if got, want := len(rows), tc.count; got != want { - t.Fatalf("ReadLog(%v): len: %v, want %v", tc.limit, got, want) + t.Fatalf("Send(): %v", err) } + idx = append(idx, ts) + } + + for i, tc := range []struct { + low, high time.Time + want []byte + }{ + {low: idx[0], high: idx[0], want: []byte{}}, + {low: idx[0], high: idx[1], want: []byte{0}}, + {low: idx[0], high: idx[9], want: []byte{0, 1, 2, 3, 4, 5, 6, 7, 8}}, + {low: idx[1], high: idx[9], want: []byte{1, 2, 3, 4, 5, 6, 7, 8}}, + // Ensure that adding 1 correctly modifies the range semantics. + {low: idx[0].Add(1), high: idx[9], want: []byte{1, 2, 3, 4, 5, 6, 7, 8}}, + {low: idx[0].Add(1), high: idx[9].Add(1), want: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9}}, + } { + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + rows, err := m.ReadLog(ctx, directoryID, logID, tc.low, tc.high, 100) + if err != nil { + t.Fatalf("ReadLog(): %v", err) + } + got := make([]byte, 0, len(rows)) + for _, r := range rows { + i := r.Mutation.Entry[0] + got = append(got, i) + } + if !cmp.Equal(got, tc.want) { + t.Fatalf("ReadLog(%v,%v): got %v, want %v", tc.low, tc.high, got, tc.want) + } + }) } } diff --git a/core/integration/storagetest/mutation_logs_admin.go b/core/integration/storagetest/mutation_logs_admin.go index a09089f90..49effebcf 100644 --- a/core/integration/storagetest/mutation_logs_admin.go +++ b/core/integration/storagetest/mutation_logs_admin.go @@ -24,13 +24,14 @@ import ( "google.golang.org/grpc/status" ) -type LogsAdminFactory func(ctx context.Context, t *testing.T, dirID string, logIDs ...int64) adminserver.LogsAdmin +// logAdminFactory returns a new database object, and a function for cleaning it up. +type logAdminFactory func(ctx context.Context, t *testing.T, dirID string, logIDs ...int64) (adminserver.LogsAdmin, func(context.Context)) // RunLogsAdminTests runs all the admin tests against the provided storage implementation. -func RunLogsAdminTests(t *testing.T, factory LogsAdminFactory) { +func RunLogsAdminTests(t *testing.T, factory logAdminFactory) { ctx := context.Background() b := &logsAdminTests{} - for name, f := range map[string]func(ctx context.Context, t *testing.T, f LogsAdminFactory){ + for name, f := range map[string]func(ctx context.Context, t *testing.T, f logAdminFactory){ // TODO(gbelvin): Discover test methods via reflection. "TestSetWritable": b.TestSetWritable, "TestListLogs": b.TestListLogs, @@ -41,15 +42,16 @@ func RunLogsAdminTests(t *testing.T, factory LogsAdminFactory) { type logsAdminTests struct{} -func (logsAdminTests) TestSetWritable(ctx context.Context, t *testing.T, f LogsAdminFactory) { +func (logsAdminTests) TestSetWritable(ctx context.Context, t *testing.T, f logAdminFactory) { directoryID := "TestSetWritable" - m := f(ctx, t, directoryID, 1) + m, done := f(ctx, t, directoryID, 1) + defer done(ctx) if st := status.Convert(m.SetWritable(ctx, directoryID, 2, true)); st.Code() != codes.NotFound { t.Errorf("SetWritable(non-existent logid): %v, want %v", st, codes.NotFound) } } -func (logsAdminTests) TestListLogs(ctx context.Context, t *testing.T, f LogsAdminFactory) { +func (logsAdminTests) TestListLogs(ctx context.Context, t *testing.T, f logAdminFactory) { directoryID := "TestListLogs" for _, tc := range []struct { desc string @@ -64,7 +66,8 @@ func (logsAdminTests) TestListLogs(ctx context.Context, t *testing.T, f LogsAdmi {desc: "multi", logIDs: []int64{1, 2, 3}, setWritable: map[int64]bool{1: true, 2: false}, wantLogIDs: []int64{1, 3}}, } { t.Run(tc.desc, func(t *testing.T) { - m := f(ctx, t, directoryID, tc.logIDs...) + m, done := f(ctx, t, directoryID, tc.logIDs...) + defer done(ctx) wantLogs := make(map[int64]bool) for _, logID := range tc.wantLogIDs { wantLogs[logID] = true diff --git a/core/integration/storagetest/mutation_logs_reader.go b/core/integration/storagetest/mutation_logs_reader.go new file mode 100644 index 000000000..0993bba6c --- /dev/null +++ b/core/integration/storagetest/mutation_logs_reader.go @@ -0,0 +1,199 @@ +// Copyright 2019 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storagetest + +import ( + "context" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/keytransparency/core/sequencer" + + pb "github.com/google/keytransparency/core/api/v1/keytransparency_go_proto" +) + +// LogsReadWriter supports test's ability to write to and read from the mutation logs. +type LogsReadWriter interface { + sequencer.LogsReader + // Send submits the whole group of mutations atomically to a log. + // TODO(gbelvin): Create a batch level object to make it clear that this a batch of updates. + // Returns the timestamp that the mutation batch got written at. + // This timestamp can be used as an argument to the lower bound of ReadLog [low, high). + // To acquire a value to use for the upper bound of ReadLog [low, high), use HighWatermark. + Send(ctx context.Context, directoryID string, logID int64, mutation ...*pb.EntryUpdate) (time.Time, error) +} + +// logsRWFactory returns a new database object, and a function for cleaning it up. +type logsRWFactory func(ctx context.Context, t *testing.T, dirID string, logIDs ...int64) (LogsReadWriter, func(context.Context)) + +// RunMutationLogsReaderTests runs all the tests against the provided storage implementation. +func RunMutationLogsReaderTests(t *testing.T, factory logsRWFactory) { + ctx := context.Background() + b := &mutationLogsReaderTests{} + type TestFunc func(ctx context.Context, t *testing.T, f logsRWFactory) + for name, f := range map[string]TestFunc{ + // TODO(gbelvin): Discover test methods via reflection. + "TestHighWatermarkPreserve": b.TestHighWatermarkPreserve, + "TestHighWatermarkRead": b.TestHighWatermarkRead, + "TestHighWatermarkBatch": b.TestHighWatermarkBatch, + } { + t.Run(name, func(t *testing.T) { f(ctx, t, factory) }) + } +} + +type mutationLogsReaderTests struct{} + +func setupWatermarks(ctx context.Context, t *testing.T, m LogsReadWriter, dirID string, logID int64, maxIndex int) ([]time.Time, []time.Time) { + t.Helper() + // Setup the test by writing 10 items to the mutation log and + // collecting the reported high water mark after each write. + sent := []time.Time{} // Timestamps that Send reported. + hwm := []time.Time{} // High water marks collected after each Send. + for i := 0; i <= maxIndex; i++ { + ts, err := m.Send(ctx, dirID, logID, &pb.EntryUpdate{Mutation: &pb.SignedEntry{Entry: []byte{byte(i)}}}) + if err != nil { + t.Fatalf("Send(%v): %v", logID, err) + } + count, wm, err := m.HighWatermark(ctx, dirID, logID, minWatermark, 100 /*batchSize*/) + if err != nil { + t.Fatalf("HighWatermark(): %v", err) + } + if want := int32(i) + 1; count != want { + t.Fatalf("HighWatermark(): count %v, want %v", count, want) + } + sent = append(sent, ts) + hwm = append(hwm, wm) + } + return sent, hwm +} + +// Tests that query HighWatermarks with varying parameters and validate results directly. +func (mutationLogsReaderTests) TestHighWatermarkPreserve(ctx context.Context, t *testing.T, newForTest logsRWFactory) { + directoryID := "TestHighWatermarkPreserve" + logID := int64(1) + m, done := newForTest(ctx, t, directoryID, logID) + defer done(ctx) + maxIndex := 9 + sent, _ := setupWatermarks(ctx, t, m, directoryID, logID, maxIndex) + + arbitraryTime := time.Date(1, 2, 3, 4, 5, 6, 7, time.UTC) + for _, tc := range []struct { + desc string + start time.Time + batch int32 + want time.Time + }{ + // Verify that high watermarks preserves the starting time when batch size is 0. + {desc: "batch 0", start: arbitraryTime, batch: 0, want: arbitraryTime}, + // Verify that high watermarks preserves the starting time when there are no rows in the result. + {desc: "rows 0", start: sent[maxIndex].Add(time.Second), batch: 1, want: sent[maxIndex].Add(time.Second)}, + } { + t.Run(tc.desc, func(t *testing.T) { + count, got, err := m.HighWatermark(ctx, directoryID, logID, tc.start, tc.batch) + if err != nil { + t.Errorf("HighWatermark(): %v", err) + } + if !got.Equal(tc.want) { + t.Errorf("HighWatermark(%v, %v) high: %v, want %v", tc.start, tc.batch, got, tc.want) + } + if count != 0 { + t.Errorf("HighWatermark(): count %v, want 0", count) + } + }) + } +} + +// Tests that use the watermarks defined during setup. +func (mutationLogsReaderTests) TestHighWatermarkRead(ctx context.Context, t *testing.T, newForTest logsRWFactory) { + directoryID := "TestHighWatermarkRead" + logID := int64(1) + m, done := newForTest(ctx, t, directoryID, logID) + defer done(ctx) + maxIndex := 9 + _, hwm := setupWatermarks(ctx, t, m, directoryID, logID, maxIndex) + for _, tc := range []struct { + desc string + readHigh time.Time + want []byte + }{ + // Verify that highwatermark can retrieve all the data written so far. + {desc: "all", readHigh: hwm[maxIndex], want: []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}}, + // Verify that data retrieved at highwatermark doesn't change when more data is written. + {desc: "stable", readHigh: hwm[2], want: []byte{0, 1, 2}}, + } { + t.Run(tc.desc, func(t *testing.T) { + low := minWatermark + rows, err := m.ReadLog(ctx, directoryID, logID, low, tc.readHigh, 100) + if err != nil { + t.Fatalf("ReadLog(): %v", err) + } + got := make([]byte, 0, len(rows)) + for _, r := range rows { + i := r.Mutation.Entry[0] + got = append(got, i) + } + if !cmp.Equal(got, tc.want) { + t.Fatalf("ReadLog(%v,%v): got %v, want %v", low, tc.readHigh, got, tc.want) + } + }) + } +} + +// Tests that query HighWatermarks with varying parameters and validate results using ReadLog. +func (mutationLogsReaderTests) TestHighWatermarkBatch(ctx context.Context, t *testing.T, newForTest logsRWFactory) { + directoryID := "TestHighWatermarkBatch" + logID := int64(1) + m, done := newForTest(ctx, t, directoryID, logID) + defer done(ctx) + maxIndex := 9 + sent, _ := setupWatermarks(ctx, t, m, directoryID, logID, maxIndex) + for _, tc := range []struct { + desc string + start time.Time + batch int32 + want []byte + }{ + // Verify that limiting batch size controls the number of items returned. + {desc: "limit batch", start: sent[0], batch: 2, want: []byte{0, 1}}, + // Verify that advancing start by 1 with the same batch size advances the results by one. + {desc: "start 1", start: sent[1], batch: 2, want: []byte{1, 2}}, + // Verify that watermarks in between primary keys resolve correctly. + {desc: "start 0.1", start: sent[0].Add(1), batch: 2, want: []byte{1, 2}}, + } { + t.Run(tc.desc, func(t *testing.T) { + count, wm, err := m.HighWatermark(ctx, directoryID, logID, tc.start, tc.batch) + if err != nil { + t.Errorf("HighWatermark(): %v", err) + } + if want := int32(len(tc.want)); count != want { + t.Errorf("HighWatermark() count: %v, want %v", count, want) + } + + rows, err := m.ReadLog(ctx, directoryID, logID, tc.start, wm, tc.batch) + if err != nil { + t.Fatalf("ReadLog(): %v", err) + } + got := make([]byte, 0, len(rows)) + for _, r := range rows { + i := r.Mutation.Entry[0] + got = append(got, i) + } + if !cmp.Equal(got, tc.want) { + t.Fatalf("ReadLog(%v,%v): got %v, want %v", tc.start, wm, got, tc.want) + } + }) + } +} diff --git a/core/keyserver/keyserver.go b/core/keyserver/keyserver.go index e58f6ddc8..32e7b47f5 100644 --- a/core/keyserver/keyserver.go +++ b/core/keyserver/keyserver.go @@ -18,8 +18,10 @@ package keyserver import ( "context" "fmt" + "math/rand" "runtime" "sync" + "time" "github.com/golang/glog" "github.com/golang/protobuf/proto" @@ -64,23 +66,20 @@ func createMetrics(mf monitoring.MetricFactory) { directoryIDLabel, logIDLabel) } -// WriteWatermark is the metadata that Send creates. -type WriteWatermark struct { - LogID int64 - Watermark int64 -} - // MutationLogs provides sets of time ordered message logs. type MutationLogs interface { - // Send submits the whole group of mutations atomically to a random log. + // Send submits the whole group of mutations atomically to a given log. // TODO(gbelvin): Create a batch level object to make it clear that this a batch of updates. - Send(ctx context.Context, directoryID string, mutation ...*pb.EntryUpdate) (*WriteWatermark, error) + // Returns the timestamp that the mutation batch got written at. + Send(ctx context.Context, directoryID string, logID int64, mutation ...*pb.EntryUpdate) (time.Time, error) // ReadLog returns the messages in the (low, high] range stored in the // specified log. ReadLog always returns complete units of the original // batches sent via Send, and will return more items than limit if // needed to do so. - ReadLog(ctx context.Context, directoryID string, logID, low, high int64, + ReadLog(ctx context.Context, directoryID string, logID int64, low, high time.Time, limit int32) ([]*mutator.LogMessage, error) + // ListLogs returns a list of logs, optionally filtered by the writable bit. + ListLogs(ctx context.Context, directoryID string, writable bool) ([]int64, error) } // BatchReader reads batch definitions. @@ -659,18 +658,32 @@ func (s *Server) BatchQueueUserUpdate(ctx context.Context, in *pb.BatchQueueUser } tdone() - // Save mutation to the database. - wm, err := s.logs.Send(ctx, directory.DirectoryID, in.Updates...) + // Pick a random logID. Note, this effectively picks a random QoS. See issue #1377. + // TODO(gbelvin): Define an explicit QoS / Load ballancing API. + wmLogID, err := s.randLog(ctx, directory.DirectoryID) + if st := status.Convert(err); st.Code() != codes.OK { + return nil, status.Errorf(st.Code(), "Could not pick a log to write to: %v", err) + } + wmTime, err := s.logs.Send(ctx, directory.DirectoryID, wmLogID, in.Updates...) if st := status.Convert(err); st.Code() != codes.OK { glog.Errorf("mutations.Write failed: %v", err) return nil, status.Errorf(st.Code(), "Mutation write error") } - if wm != nil { - watermarkWritten.Set(float64(wm.Watermark), directory.DirectoryID, fmt.Sprintf("%v", wm.LogID)) - sequencerQueueWritten.Add(float64(len(in.Updates)), directory.DirectoryID, fmt.Sprintf("%v", wm.LogID)) + watermarkWritten.Set(float64(wmTime.UnixNano()), directory.DirectoryID, fmt.Sprintf("%v", wmLogID)) + sequencerQueueWritten.Add(float64(len(in.Updates)), directory.DirectoryID, fmt.Sprintf("%v", wmLogID)) + return &empty.Empty{}, nil +} + +func (s *Server) randLog(ctx context.Context, directoryID string) (int64, error) { + // TODO(gbelvin): Cache these results. + writable := true + logIDs, err := s.logs.ListLogs(ctx, directoryID, writable) + if err != nil { + return 0, err } - return &empty.Empty{}, nil + // Return a random log. + return logIDs[rand.Intn(len(logIDs))], nil } // GetDirectory returns all info tied to the specified directory. diff --git a/core/keyserver/keyserver_test.go b/core/keyserver/keyserver_test.go index 9de0a9b3d..e22830c7d 100644 --- a/core/keyserver/keyserver_test.go +++ b/core/keyserver/keyserver_test.go @@ -27,6 +27,7 @@ import ( "github.com/google/keytransparency/core/directory" "github.com/google/keytransparency/core/fake" "github.com/google/trillian/testonly" + "github.com/google/trillian/testonly/matchers" "github.com/google/trillian/types" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -123,11 +124,12 @@ func TestLatestRevision(t *testing.T) { }, err) if tc.wantErr == codes.OK { e.s.Map.EXPECT().GetLeafByRevision(gomock.Any(), - &tpb.GetMapLeafByRevisionRequest{ - MapId: mapID, - Index: make([]byte, 32), - Revision: tc.treeSize - 1, - }). + matchers.ProtoEqual( + &tpb.GetMapLeafByRevisionRequest{ + MapId: mapID, + Index: make([]byte, 32), + Revision: tc.treeSize - 1, + })). Return(&tpb.GetMapLeafResponse{ MapLeafInclusion: &tpb.MapLeafInclusion{ Leaf: &tpb.MapLeaf{ @@ -156,11 +158,11 @@ func TestLatestRevision(t *testing.T) { }, err).Times(2) for i := int64(0); i < tc.treeSize; i++ { e.s.Map.EXPECT().GetLeafByRevision(gomock.Any(), - &tpb.GetMapLeafByRevisionRequest{ + matchers.ProtoEqual(&tpb.GetMapLeafByRevisionRequest{ MapId: mapID, Index: make([]byte, 32), Revision: i, - }). + })). Return(&tpb.GetMapLeafResponse{ MapLeafInclusion: &tpb.MapLeafInclusion{ Leaf: &tpb.MapLeaf{ diff --git a/core/keyserver/paginator.go b/core/keyserver/paginator.go index 8f200ff24..7bf9cd3be 100644 --- a/core/keyserver/paginator.go +++ b/core/keyserver/paginator.go @@ -18,7 +18,9 @@ import ( "encoding/base64" "github.com/golang/protobuf/proto" + "github.com/golang/protobuf/ptypes" "github.com/google/keytransparency/core/mutator" + "github.com/google/keytransparency/core/sequencer/metadata" rtpb "github.com/google/keytransparency/core/keyserver/readtoken_go_proto" spb "github.com/google/keytransparency/core/sequencer/sequencer_go_proto" @@ -63,9 +65,13 @@ func (s SourceList) First() *rtpb.ReadToken { // Empty struct means there is nothing else to page through. return &rtpb.ReadToken{} } + st, err := ptypes.TimestampProto(metadata.FromProto(s[0]).StartTime()) + if err != nil { + panic("invalid timestamp") + } return &rtpb.ReadToken{ - SliceIndex: 0, - LowWatermark: s[0].LowestInclusive, + SliceIndex: 0, + StartTime: st, } } @@ -75,9 +81,13 @@ func (s SourceList) First() *rtpb.ReadToken { func (s SourceList) Next(rt *rtpb.ReadToken, lastRow *mutator.LogMessage) *rtpb.ReadToken { if lastRow != nil { // There are more items in this source slice. + st, err := ptypes.TimestampProto(lastRow.ID) + if err != nil { + panic("invalid timestamp") + } return &rtpb.ReadToken{ - SliceIndex: rt.SliceIndex, - LowWatermark: lastRow.ID, + SliceIndex: rt.SliceIndex, + StartTime: st, } } @@ -86,8 +96,13 @@ func (s SourceList) Next(rt *rtpb.ReadToken, lastRow *mutator.LogMessage) *rtpb. // There are no more source slices to iterate over. return &rtpb.ReadToken{} // Encodes to "" } + + st, err := ptypes.TimestampProto(metadata.FromProto(s[rt.SliceIndex+1]).StartTime()) + if err != nil { + panic("invalid timestamp") + } return &rtpb.ReadToken{ - SliceIndex: rt.SliceIndex + 1, - LowWatermark: s[rt.SliceIndex+1].LowestInclusive, + SliceIndex: rt.SliceIndex + 1, + StartTime: st, } } diff --git a/core/keyserver/paginator_test.go b/core/keyserver/paginator_test.go index e4d10e575..76843b027 100644 --- a/core/keyserver/paginator_test.go +++ b/core/keyserver/paginator_test.go @@ -16,14 +16,26 @@ package keyserver import ( "testing" + "time" "github.com/golang/protobuf/proto" - "github.com/google/keytransparency/core/mutator" + "github.com/google/keytransparency/core/sequencer/metadata" + spb "github.com/google/keytransparency/core/sequencer/sequencer_go_proto" + tpb "github.com/golang/protobuf/ptypes/timestamp" rtpb "github.com/google/keytransparency/core/keyserver/readtoken_go_proto" ) +func newSourceSlice(t *testing.T, logID int64, low, high time.Time) *spb.MapMetadata_SourceSlice { + t.Helper() + s, err := metadata.New(logID, low, high) + if err != nil { + t.Fatalf("Invalid source: %v", err) + } + return s.Proto() +} + func TestEncodeToken(t *testing.T) { for _, tc := range []struct { rt *rtpb.ReadToken @@ -43,7 +55,7 @@ func TestEncodeToken(t *testing.T) { } func TestTokenEncodeDecode(t *testing.T) { - rt1 := &rtpb.ReadToken{SliceIndex: 2, LowWatermark: 5} + rt1 := &rtpb.ReadToken{SliceIndex: 2, StartTime: &tpb.Timestamp{Nanos: 5}} rt1Token, err := EncodeToken(rt1) if err != nil { t.Fatalf("EncodeToken(%v): %v", rt1, err) @@ -69,16 +81,16 @@ func TestTokenEncodeDecode(t *testing.T) { } func TestFirst(t *testing.T) { + start := time.Unix(0, 0) for _, tc := range []struct { s SourceList want *rtpb.ReadToken }{ { s: SourceList{ - {LogId: 2, LowestInclusive: 2, HighestExclusive: 11}, - {LogId: 3, LowestInclusive: 11, HighestExclusive: 21}, - }, - want: &rtpb.ReadToken{SliceIndex: 0, LowWatermark: 2}, + newSourceSlice(t, 2, start.Add(2*time.Microsecond), start.Add(11*time.Microsecond)), + newSourceSlice(t, 3, start.Add(11*time.Microsecond), start.Add(21*time.Microsecond))}, + want: &rtpb.ReadToken{SliceIndex: 0, StartTime: timestamp(t, start.Add(2*time.Microsecond))}, }, {s: SourceList{}, want: &rtpb.ReadToken{}}, } { @@ -89,9 +101,10 @@ func TestFirst(t *testing.T) { } func TestNext(t *testing.T) { + start := time.Unix(1, 0) a := SourceList{ - {LogId: 2, LowestInclusive: 2, HighestExclusive: 11}, - {LogId: 3, LowestInclusive: 11, HighestExclusive: 21}, + newSourceSlice(t, 2, start.Add(2*time.Microsecond), start.Add(11*time.Microsecond)), + newSourceSlice(t, 3, start.Add(11*time.Microsecond), start.Add(21*time.Microsecond)), } for _, tc := range []struct { s SourceList @@ -103,16 +116,16 @@ func TestNext(t *testing.T) { { desc: "first page", s: a, - rt: &rtpb.ReadToken{SliceIndex: 0, LowWatermark: 2}, - lastRow: &mutator.LogMessage{ID: 6}, - want: &rtpb.ReadToken{SliceIndex: 0, LowWatermark: 6}, + rt: &rtpb.ReadToken{SliceIndex: 0, StartTime: timestamp(t, start.Add(2*time.Microsecond))}, + lastRow: &mutator.LogMessage{ID: start.Add(6 * time.Microsecond)}, + want: &rtpb.ReadToken{SliceIndex: 0, StartTime: timestamp(t, start.Add(6*time.Microsecond))}, }, { desc: "next source", s: a, rt: &rtpb.ReadToken{}, lastRow: nil, - want: &rtpb.ReadToken{SliceIndex: 1, LowWatermark: 11}, + want: &rtpb.ReadToken{SliceIndex: 1, StartTime: timestamp(t, start.Add(11*time.Microsecond))}, }, { desc: "last page", @@ -124,7 +137,7 @@ func TestNext(t *testing.T) { { desc: "empty", s: SourceList{}, - rt: &rtpb.ReadToken{SliceIndex: 1, LowWatermark: 2}, + rt: &rtpb.ReadToken{SliceIndex: 1, StartTime: timestamp(t, start.Add(2*time.Microsecond))}, lastRow: nil, want: &rtpb.ReadToken{}, }, diff --git a/core/keyserver/readtoken.proto b/core/keyserver/readtoken.proto index ff6dd1aec..95344382d 100644 --- a/core/keyserver/readtoken.proto +++ b/core/keyserver/readtoken.proto @@ -19,13 +19,16 @@ package google.keytransparency.v1; option go_package = "github.com/google/keytransparency/core/keyserver/readtoken_go_proto"; import "v1/keytransparency.proto"; +import "google/protobuf/timestamp.proto"; + // ReadToken can be serialized and handed to users for pagination. message ReadToken { // slice_index identifies the source for reading. int64 slice_index = 1; - // low_watemark identifies the lowest (exclusive) row to return. - int64 low_watermark = 2; + reserved 2; + // start_time identifies the lowest (exclusive) row to return. + google.protobuf.Timestamp start_time = 3; } // ListUserRevisions token can be serialized and handed to users for pagination diff --git a/core/keyserver/readtoken_go_proto/readtoken.pb.go b/core/keyserver/readtoken_go_proto/readtoken.pb.go index 8f8dc3c08..418bc740e 100644 --- a/core/keyserver/readtoken_go_proto/readtoken.pb.go +++ b/core/keyserver/readtoken_go_proto/readtoken.pb.go @@ -6,6 +6,7 @@ package readtoken_go_proto import ( fmt "fmt" proto "github.com/golang/protobuf/proto" + timestamp "github.com/golang/protobuf/ptypes/timestamp" keytransparency_go_proto "github.com/google/keytransparency/core/api/v1/keytransparency_go_proto" math "math" ) @@ -25,11 +26,11 @@ const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package type ReadToken struct { // slice_index identifies the source for reading. SliceIndex int64 `protobuf:"varint,1,opt,name=slice_index,json=sliceIndex,proto3" json:"slice_index,omitempty"` - // low_watemark identifies the lowest (exclusive) row to return. - LowWatermark int64 `protobuf:"varint,2,opt,name=low_watermark,json=lowWatermark,proto3" json:"low_watermark,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + // start_time identifies the lowest (exclusive) row to return. + StartTime *timestamp.Timestamp `protobuf:"bytes,3,opt,name=start_time,json=startTime,proto3" json:"start_time,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *ReadToken) Reset() { *m = ReadToken{} } @@ -64,11 +65,11 @@ func (m *ReadToken) GetSliceIndex() int64 { return 0 } -func (m *ReadToken) GetLowWatermark() int64 { +func (m *ReadToken) GetStartTime() *timestamp.Timestamp { if m != nil { - return m.LowWatermark + return m.StartTime } - return 0 + return nil } // ListUserRevisions token can be serialized and handed to users for pagination @@ -134,22 +135,23 @@ func init() { func init() { proto.RegisterFile("readtoken.proto", fileDescriptor_735a2ae6888918c9) } var fileDescriptor_735a2ae6888918c9 = []byte{ - // 257 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x74, 0x90, 0xc1, 0x4a, 0xc4, 0x30, - 0x10, 0x86, 0xa9, 0x82, 0x62, 0x56, 0x11, 0x73, 0x90, 0xea, 0x45, 0x59, 0x2f, 0x5e, 0x4c, 0x59, - 0xf7, 0x0d, 0x14, 0x0f, 0x82, 0x1e, 0x0c, 0x8a, 0xe0, 0x25, 0x64, 0xdb, 0xa1, 0x86, 0x76, 0x33, - 0xeb, 0x64, 0xda, 0xba, 0x2f, 0xe2, 0xf3, 0xca, 0xa6, 0x56, 0x61, 0x65, 0x4f, 0x81, 0x6f, 0xf2, - 0x7f, 0x33, 0xfc, 0xe2, 0x90, 0xc0, 0x16, 0x8c, 0x15, 0x78, 0xb5, 0x20, 0x64, 0x94, 0x27, 0x25, - 0x62, 0x59, 0x83, 0xaa, 0x60, 0xc9, 0x64, 0x7d, 0x58, 0x58, 0x02, 0x9f, 0x2f, 0x55, 0x3b, 0x39, - 0x4d, 0xdb, 0x49, 0xb6, 0x8e, 0x63, 0x68, 0xfc, 0x24, 0xf6, 0x34, 0xd8, 0xe2, 0x79, 0xe5, 0x91, - 0x67, 0x62, 0x14, 0x6a, 0x97, 0x83, 0x71, 0xbe, 0x80, 0xcf, 0x34, 0x39, 0x4f, 0x2e, 0xb7, 0xb5, - 0x88, 0xe8, 0x7e, 0x45, 0xe4, 0x85, 0x38, 0xa8, 0xb1, 0x33, 0x9d, 0x65, 0xa0, 0xb9, 0xa5, 0x2a, - 0xdd, 0x8a, 0x5f, 0xf6, 0x6b, 0xec, 0x5e, 0x07, 0x36, 0xfe, 0x4a, 0xc4, 0xf1, 0x83, 0x0b, 0xfc, - 0x12, 0x80, 0x34, 0xb4, 0x2e, 0x38, 0xf4, 0xa1, 0x5f, 0xf0, 0x28, 0x76, 0x09, 0x3e, 0x1a, 0x08, - 0x1c, 0xe5, 0xa3, 0xeb, 0xa9, 0xda, 0x78, 0xb4, 0xfa, 0xe7, 0xd0, 0x7d, 0x54, 0x0f, 0x0e, 0x79, - 0x25, 0x24, 0x0d, 0x43, 0x43, 0xc0, 0x0d, 0x79, 0x28, 0x7e, 0x6e, 0x3a, 0xa2, 0xbf, 0x58, 0x3f, - 0xb8, 0xb9, 0x7b, 0xbb, 0x2d, 0x1d, 0xbf, 0x37, 0x33, 0x95, 0xe3, 0x3c, 0xeb, 0x17, 0xaf, 0xd7, - 0x92, 0xe5, 0x48, 0x11, 0x06, 0xa0, 0x16, 0x28, 0xfb, 0x2d, 0xd9, 0x94, 0x68, 0x62, 0x65, 0xb3, - 0x9d, 0xf8, 0x4c, 0xbf, 0x03, 0x00, 0x00, 0xff, 0xff, 0x42, 0xa4, 0x18, 0x01, 0x81, 0x01, 0x00, - 0x00, + // 287 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x74, 0x91, 0x31, 0x4f, 0xc3, 0x30, + 0x10, 0x85, 0x95, 0x16, 0x01, 0x75, 0x07, 0x20, 0x03, 0x0a, 0x5d, 0x5a, 0x75, 0xea, 0x82, 0xad, + 0xb6, 0x13, 0x2b, 0x88, 0x01, 0x04, 0x8b, 0x55, 0x16, 0x96, 0xc8, 0x4d, 0x8e, 0x60, 0xb5, 0xb1, + 0xc3, 0xf9, 0x12, 0xd1, 0x3f, 0xc2, 0xef, 0x45, 0xb1, 0x1b, 0x90, 0x8a, 0x98, 0xac, 0x7b, 0x77, + 0xef, 0x3e, 0x3f, 0x9b, 0x9d, 0x21, 0xa8, 0x9c, 0xec, 0x06, 0x0c, 0xaf, 0xd0, 0x92, 0x8d, 0xaf, + 0x0a, 0x6b, 0x8b, 0x2d, 0xf0, 0x0d, 0xec, 0x08, 0x95, 0x71, 0x95, 0x42, 0x30, 0xd9, 0x8e, 0x37, + 0xf3, 0x51, 0xd2, 0xcc, 0xc5, 0xa1, 0xec, 0x4d, 0xa3, 0x71, 0x30, 0x09, 0x5f, 0xad, 0xeb, 0x37, + 0x41, 0xba, 0x04, 0x47, 0xaa, 0xac, 0xc2, 0xc0, 0xb4, 0x64, 0x03, 0x09, 0x2a, 0x5f, 0xb5, 0xa0, + 0x78, 0xcc, 0x86, 0x6e, 0xab, 0x33, 0x48, 0xb5, 0xc9, 0xe1, 0x33, 0x89, 0x26, 0xd1, 0xac, 0x2f, + 0x99, 0x97, 0x1e, 0x5a, 0x25, 0xbe, 0x61, 0xcc, 0x91, 0x42, 0x4a, 0xdb, 0x35, 0x49, 0x7f, 0x12, + 0xcd, 0x86, 0x8b, 0x11, 0xdf, 0x5f, 0xac, 0x63, 0xf0, 0x55, 0xc7, 0x90, 0x03, 0x3f, 0xdd, 0xd6, + 0x8f, 0x47, 0xa7, 0xbd, 0xf3, 0xfe, 0xf4, 0x2b, 0x62, 0x97, 0x4f, 0xda, 0xd1, 0x8b, 0x03, 0x94, + 0xd0, 0x68, 0xa7, 0xad, 0x71, 0x01, 0xfe, 0xcc, 0x4e, 0x10, 0x3e, 0x6a, 0x70, 0xe4, 0xc1, 0xc3, + 0xc5, 0x92, 0xff, 0x9b, 0x98, 0xff, 0xd9, 0x21, 0x83, 0x55, 0x76, 0x3b, 0xe2, 0x6b, 0x16, 0x63, + 0xd7, 0x4c, 0x11, 0xa8, 0x46, 0x03, 0x79, 0xd2, 0xf3, 0x91, 0x2e, 0xf0, 0xd7, 0x16, 0x1a, 0xb7, + 0xf7, 0xaf, 0x77, 0x85, 0xa6, 0xf7, 0x7a, 0xcd, 0x33, 0x5b, 0x8a, 0xfd, 0xab, 0x1d, 0x80, 0x45, + 0x66, 0xd1, 0x8b, 0x0e, 0xb0, 0x01, 0x14, 0x3f, 0x3f, 0x94, 0x16, 0x36, 0x0d, 0xe9, 0x8f, 0xfd, + 0xb1, 0xfc, 0x0e, 0x00, 0x00, 0xff, 0xff, 0x1f, 0xd9, 0xb2, 0x4d, 0xbe, 0x01, 0x00, 0x00, } diff --git a/core/keyserver/revisions.go b/core/keyserver/revisions.go index 941395c4a..dc67f6ea1 100644 --- a/core/keyserver/revisions.go +++ b/core/keyserver/revisions.go @@ -19,12 +19,14 @@ import ( "github.com/golang/glog" "github.com/golang/protobuf/proto" + "github.com/golang/protobuf/ptypes" "github.com/google/trillian/types" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "github.com/google/keytransparency/core/directory" "github.com/google/keytransparency/core/mutator" + "github.com/google/keytransparency/core/sequencer/metadata" pb "github.com/google/keytransparency/core/api/v1/keytransparency_go_proto" tpb "github.com/google/trillian" @@ -139,12 +141,16 @@ func (s *Server) ListMutations(ctx context.Context, in *pb.ListMutationsRequest) } // Read PageSize + 1 messages from the log to see if there is another page. - high := meta.Sources[rt.SliceIndex].HighestExclusive + high := metadata.FromProto(meta.Sources[rt.SliceIndex]).EndTime() logID := meta.Sources[rt.SliceIndex].LogId - msgs, err := s.logs.ReadLog(ctx, d.DirectoryID, logID, rt.LowWatermark, high, in.PageSize+1) + low, err := ptypes.Timestamp(rt.StartTime) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "Invalid start timestamp: %v", err) + } + msgs, err := s.logs.ReadLog(ctx, d.DirectoryID, logID, low, high, in.PageSize+1) if st := status.Convert(err); st.Code() != codes.OK { glog.Errorf("ListMutations(): ReadLog(%v, log: %v/(%v, %v], batchSize: %v): %v", - d.DirectoryID, logID, rt.LowWatermark, high, in.PageSize, err) + d.DirectoryID, logID, low, high, in.PageSize, err) return nil, status.Errorf(st.Code(), "Reading mutations range failed: %v", st.Message()) } moreInLogID := len(msgs) == int(in.PageSize+1) diff --git a/core/keyserver/revisions_test.go b/core/keyserver/revisions_test.go index bf7c57b90..1035ad56e 100644 --- a/core/keyserver/revisions_test.go +++ b/core/keyserver/revisions_test.go @@ -16,20 +16,24 @@ package keyserver import ( "context" - "errors" "fmt" "testing" "time" "github.com/golang/mock/gomock" "github.com/golang/protobuf/proto" + "github.com/golang/protobuf/ptypes" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "github.com/google/keytransparency/core/mutator" + "github.com/google/go-cmp/cmp" + "github.com/google/keytransparency/impl/memory" + "github.com/google/trillian/testonly/matchers" + protopb "github.com/golang/protobuf/ptypes/timestamp" pb "github.com/google/keytransparency/core/api/v1/keytransparency_go_proto" rtpb "github.com/google/keytransparency/core/keyserver/readtoken_go_proto" + "github.com/google/keytransparency/core/sequencer/metadata" spb "github.com/google/keytransparency/core/sequencer/sequencer_go_proto" tpb "github.com/google/trillian" ) @@ -53,6 +57,30 @@ func genIndexes(start, end int64) [][]byte { return indexes } +func genEntryUpdates(t *testing.T, start, end int64) []*pb.EntryUpdate { + t.Helper() + entries := make([]*pb.EntryUpdate, 0) + for i := start; i < end; i++ { + entries = append(entries, &pb.EntryUpdate{ + Mutation: &pb.SignedEntry{ + Entry: mustMarshal(t, &pb.Entry{ + Index: []byte(fmt.Sprintf("key_%v", i)), + }), + }, + }) + } + return entries +} + +func timestamp(t *testing.T, ts time.Time) *protopb.Timestamp { + t.Helper() + ret, err := ptypes.TimestampProto(ts) + if err != nil { + t.Fatal(err) + } + return ret +} + func TestGetRevisionStream(t *testing.T) { srv := &Server{} err := srv.GetRevisionStream(nil, nil) @@ -61,39 +89,31 @@ func TestGetRevisionStream(t *testing.T) { } } +func newSource(t *testing.T, logID int64, low, high time.Time) *spb.MapMetadata_SourceSlice { + t.Helper() + s, err := metadata.New(logID, low, high) + if err != nil { + t.Fatalf("Invalid source: %v", err) + } + return s.Proto() +} + type batchStorage map[int64]SourceList // Map of Revision to Sources func (b batchStorage) ReadBatch(ctx context.Context, dirID string, rev int64) (*spb.MapMetadata, error) { return &spb.MapMetadata{Sources: b[rev]}, nil } -type mutations map[int64][]*mutator.LogMessage // Map of logID to Slice of LogMessages - -func (m *mutations) Send(ctx context.Context, dirID string, mutation ...*pb.EntryUpdate) (*WriteWatermark, error) { - return nil, errors.New("unimplemented") -} +func MustEncodeToken(t *testing.T, low time.Time) string { + t.Helper() -func (m *mutations) ReadLog(ctx context.Context, dirID string, - logID, low, high int64, batchSize int32) ([]*mutator.LogMessage, error) { - logShard := (*m)[logID] - if low > int64(len(logShard)) { - return nil, fmt.Errorf("invalid argument: low: %v, want <= max watermark: %v", low, len(logShard)) - } - count := high - low - if count > int64(batchSize) { - count = int64(batchSize) - } - if low+count > int64(len(logShard)) { - count = int64(len(logShard)) - low + 1 + st, err := ptypes.TimestampProto(low) + if err != nil { + t.Fatal(err) } - return logShard[low : low+count], nil -} - -func MustEncodeToken(t *testing.T, low int64) string { - t.Helper() rt := &rtpb.ReadToken{ - SliceIndex: 0, - LowWatermark: low, + SliceIndex: 0, + StartTime: st, } token, err := EncodeToken(rt) if err != nil { @@ -104,30 +124,22 @@ func MustEncodeToken(t *testing.T, low int64) string { func TestListMutations(t *testing.T) { ctx := context.Background() - fakeBatches := batchStorage{ - 1: SourceList{{LogId: 0, LowestInclusive: 2, HighestExclusive: 7}}, - 2: SourceList{{LogId: 0, LowestInclusive: 7, HighestExclusive: 11}}, + dirID := "TestListMutations" + logID := int64(0) + fakeLogs := memory.NewMutationLogs() + idx := make([]time.Time, 0, 12) + for i := int64(0); i < 12; i++ { + // Send one entry. + ts, err := fakeLogs.Send(ctx, dirID, logID, genEntryUpdates(t, i, i+1)...) + if err != nil { + t.Fatal(err) + } + idx = append(idx, ts) } - fakeLogs := make(mutations) - for _, sources := range fakeBatches { - for _, source := range sources { - extendBy := source.HighestExclusive - int64(len(fakeLogs[source.LogId])) - if extendBy > 0 { - fakeLogs[source.LogId] = append(fakeLogs[source.LogId], make([]*mutator.LogMessage, extendBy)...) - } - for i := source.LowestInclusive; i < source.HighestExclusive; i++ { - fakeLogs[source.LogId][i] = &mutator.LogMessage{ - ID: i, - Mutation: &pb.SignedEntry{ - Entry: mustMarshal(t, &pb.Entry{ - Index: []byte(fmt.Sprintf("key_%v", i)), - Commitment: []byte(fmt.Sprintf("value_%v", i)), - }), - }, - } - } - } + fakeBatches := batchStorage{ + 1: SourceList{newSource(t, 0, idx[2], idx[7])}, + 2: SourceList{newSource(t, 0, idx[7], idx[11])}, } for _, tc := range []struct { @@ -138,12 +150,12 @@ func TestListMutations(t *testing.T) { wantNext *rtpb.ReadToken wantErr bool }{ - {desc: "exact page", pageSize: 6, start: 2, end: 7, wantNext: &rtpb.ReadToken{}}, + {desc: "first page", pageSize: 6, start: 2, end: 7, wantNext: &rtpb.ReadToken{}}, {desc: "large page", pageSize: 10, start: 2, end: 7, wantNext: &rtpb.ReadToken{}}, - {desc: "partial", pageSize: 4, start: 2, end: 6, wantNext: &rtpb.ReadToken{LowWatermark: 6}}, - {desc: "large page with token", token: MustEncodeToken(t, 3), pageSize: 10, start: 3, end: 7, wantNext: &rtpb.ReadToken{}}, - {desc: "small page with token", token: MustEncodeToken(t, 3), pageSize: 2, start: 3, end: 5, - wantNext: &rtpb.ReadToken{LowWatermark: 5}}, + {desc: "partial", pageSize: 4, start: 2, end: 6, wantNext: &rtpb.ReadToken{StartTime: timestamp(t, idx[6])}}, + {desc: "large page with token", token: MustEncodeToken(t, idx[3]), pageSize: 10, start: 3, end: 7, wantNext: &rtpb.ReadToken{}}, + {desc: "small page with token", token: MustEncodeToken(t, idx[3]), pageSize: 2, start: 3, end: 5, + wantNext: &rtpb.ReadToken{StartTime: timestamp(t, idx[5])}}, {desc: "invalid page token", token: "some_token", pageSize: 0, wantErr: true}, } { t.Run(tc.desc, func(t *testing.T) { @@ -160,10 +172,11 @@ func TestListMutations(t *testing.T) { if !tc.wantErr { e.s.Map.EXPECT().GetLeavesByRevision(gomock.Any(), - &tpb.GetMapLeavesByRevisionRequest{ - MapId: mapID, - Index: genIndexes(tc.start, tc.end), - }).Return(&tpb.GetMapLeavesResponse{ + matchers.ProtoEqual( + &tpb.GetMapLeavesByRevisionRequest{ + MapId: mapID, + Index: genIndexes(tc.start, tc.end), + })).Return(&tpb.GetMapLeavesResponse{ MapLeafInclusion: genInclusions(tc.start, tc.end), }, nil) } @@ -180,14 +193,15 @@ func TestListMutations(t *testing.T) { if err != nil { return } - mtns := fakeLogs[0][tc.start:tc.end] - if got, want := len(resp.Mutations), len(mtns); got != want { - t.Fatalf("len(resp.Mutations):%v, want %v", got, want) + + got := []*pb.EntryUpdate{} + for _, m := range resp.Mutations { + got = append(got, &pb.EntryUpdate{Mutation: m.Mutation}) } - for i, mut := range resp.Mutations { - if got, want := mut.Mutation, mtns[i].Mutation; !proto.Equal(got, want) { - t.Errorf("resp.Mutations[i].Update:%v, want %v", got, want) - } + + if want := genEntryUpdates(t, tc.start, tc.end); !cmp.Equal( + got, want, cmp.Comparer(proto.Equal)) { + t.Errorf("got: %v, want: %v, diff: \n%v", got, want, cmp.Diff(got, want)) } var npt rtpb.ReadToken diff --git a/core/mutator/mutator.go b/core/mutator/mutator.go index b6bc9dad8..4c845d28e 100644 --- a/core/mutator/mutator.go +++ b/core/mutator/mutator.go @@ -18,6 +18,8 @@ package mutator import ( + "time" + "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -50,7 +52,7 @@ type VerifyMutationFn func(mutation *pb.SignedEntry) error // LogMessage represents a change to a user, and associated data. type LogMessage struct { - ID int64 + ID time.Time LocalID int64 Mutation *pb.SignedEntry ExtraData *pb.Committed diff --git a/core/sequencer/metadata/sourceslice.go b/core/sequencer/metadata/sourceslice.go new file mode 100644 index 000000000..6a40bf042 --- /dev/null +++ b/core/sequencer/metadata/sourceslice.go @@ -0,0 +1,86 @@ +// Copyright 2019 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package metadata helps enforce a consistent standard of meaning around the map metadata object. +package metadata + +import ( + "fmt" + "math" + "time" + + spb "github.com/google/keytransparency/core/sequencer/sequencer_go_proto" +) + +var minTime = time.Unix(0, math.MinInt64) // 1677-09-21 00:12:43.145224192 +var maxTime = time.Unix(0, math.MaxInt64) // 2262-04-11 23:47:16.854775807 +const quantum = time.Microsecond +const nanosPerQuantum = int64(quantum / time.Nanosecond) + +// validateTime determines whether a timestamp is valid. +// Valid timestamps can be represented by an int64 number of microseconds from the epoch. +// +// Every valid timestamp can be represented by a time.Time, but the converse is not true. +func validateTime(ts time.Time) error { + if ts.Before(minTime) { + return fmt.Errorf("timestamp %v before %v", ts, minTime) + } + if ts.After(maxTime) { + return fmt.Errorf("timestamp %v after %v", ts, maxTime) + } + return nil +} + +// New creates a new source slice from time objects. +// Returns an error if the time objects cannot be represented correctly. +func New(logID int64, low, high time.Time) (*SourceSlice, error) { + if err := validateTime(low); err != nil { + return nil, err + } + if err := validateTime(high); err != nil { + return nil, err + } + return &SourceSlice{s: &spb.MapMetadata_SourceSlice{ + LogId: logID, + LowestInclusive: low.Add(quantum-1).Truncate(quantum).UnixNano() / nanosPerQuantum, + HighestExclusive: high.Add(quantum-1).Truncate(quantum).UnixNano() / nanosPerQuantum, + }}, nil +} + +// FromProto returns a wrapper for SourceSlice +func FromProto(s *spb.MapMetadata_SourceSlice) *SourceSlice { + return &SourceSlice{s: s} +} + +// SourceSlice defines accessor and conversion methods for MapMetadata_SourceSlice +// TODO(gbelvin): fully migrate to source slices that encode time directly. +// For now, we need to have a wrapper to do the conversions between time and int64 consistently. +type SourceSlice struct { + s *spb.MapMetadata_SourceSlice +} + +// StartTime returns LowestInclusive as a time.Time +func (s SourceSlice) StartTime() time.Time { + return time.Unix(0, s.s.GetLowestInclusive()*nanosPerQuantum) +} + +// EndTime returns HighestExclusive as a time.Time +func (s SourceSlice) EndTime() time.Time { + return time.Unix(0, s.s.GetHighestExclusive()*nanosPerQuantum) +} + +// Proto returns the proto representation of SourceSlice +func (s *SourceSlice) Proto() *spb.MapMetadata_SourceSlice { + return s.s +} diff --git a/core/sequencer/metadata/sourceslice_test.go b/core/sequencer/metadata/sourceslice_test.go new file mode 100644 index 000000000..ddbec2d22 --- /dev/null +++ b/core/sequencer/metadata/sourceslice_test.go @@ -0,0 +1,63 @@ +// Copyright 2019 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package metadata + +import ( + "testing" + "time" + + "github.com/golang/protobuf/proto" + + spb "github.com/google/keytransparency/core/sequencer/sequencer_go_proto" +) + +func TestValidateTime(t *testing.T) { + for _, tc := range []struct { + ts time.Time + valid bool + }{ + {ts: time.Time{}, valid: false}, + {ts: time.Date(1, 0, 0, 0, 0, 0, 0, time.UTC), valid: false}, + {ts: time.Date(1000, 0, 0, 0, 0, 0, 0, time.UTC), valid: false}, + {ts: time.Date(2000, 0, 0, 0, 0, 0, 0, time.UTC), valid: true}, + {ts: time.Date(200000, 0, 0, 0, 0, 0, 0, time.UTC), valid: false}, + } { + err := validateTime(tc.ts) + if got := err == nil; got != tc.valid { + t.Errorf("validateTime(%v): %v, want valid: %v", tc.ts, err, tc.valid) + } + } +} + +func TestTimeToProto(t *testing.T) { + logID := int64(1) + for _, tc := range []struct { + low, high time.Time + want *spb.MapMetadata_SourceSlice + }{ + {low: time.Unix(0, 0), high: time.Unix(0, 0), want: &spb.MapMetadata_SourceSlice{LogId: logID, LowestInclusive: 0}}, + {low: time.Unix(0, 1), high: time.Unix(0, 0), want: &spb.MapMetadata_SourceSlice{LogId: logID, LowestInclusive: 1}}, + {low: time.Unix(0, 1000), high: time.Unix(0, 0), want: &spb.MapMetadata_SourceSlice{LogId: logID, LowestInclusive: 1}}, + {low: time.Unix(1, 0), high: time.Unix(0, 0), want: &spb.MapMetadata_SourceSlice{LogId: logID, LowestInclusive: 1000000}}, + {low: time.Unix(1, 0), high: time.Unix(1, 0), want: &spb.MapMetadata_SourceSlice{LogId: logID, LowestInclusive: 1000000, HighestExclusive: 1000000}}, + } { + ss, err := New(logID, tc.low, tc.high) + if err != nil { + t.Fatal(err) + } + if got := ss.Proto(); !proto.Equal(got, tc.want) { + t.Errorf("New().Proto(): %v, want %v", got, tc.want) + } + } +} diff --git a/core/sequencer/runner/native_test.go b/core/sequencer/runner/native_test.go index 2534a4c66..e31e7abdf 100644 --- a/core/sequencer/runner/native_test.go +++ b/core/sequencer/runner/native_test.go @@ -17,6 +17,7 @@ package runner import ( "testing" + "github.com/golang/protobuf/proto" "github.com/google/go-cmp/cmp" "github.com/google/keytransparency/core/mutator/entry" @@ -52,7 +53,7 @@ func TestJoin(t *testing.T) { for g := range Join(tc.leaves, tc.msgs, func(label string) { metrics[label]++ }) { got = append(got, g) } - if !cmp.Equal(got, tc.want) { + if !cmp.Equal(got, tc.want, cmp.Comparer(proto.Equal)) { t.Errorf("Join(): %v, want %v\n diff: %v", got, tc.want, cmp.Diff(got, tc.want)) } diff --git a/core/sequencer/sequencer.go b/core/sequencer/sequencer.go index 8d0175c32..4501d71d4 100644 --- a/core/sequencer/sequencer.go +++ b/core/sequencer/sequencer.go @@ -125,7 +125,6 @@ func (s *Sequencer) ForAllMasterships(ctx context.Context, f func(ctx context.Co // DefineRevisions method on all directories that this sequencer is currently // master for. func (s *Sequencer) DefineRevisionsForAllMasterships(ctx context.Context, batchSize int32) error { - glog.Infof("DefineRevisionsForAllMasterships") return s.ForAllMasterships(ctx, func(ctx context.Context, dirID string) error { // TODO(pavelkalinnikov): Make these parameters configurable. req := &spb.DefineRevisionsRequest{ @@ -146,7 +145,6 @@ func (s *Sequencer) DefineRevisionsForAllMasterships(ctx context.Context, batchS // ApplyRevisions method on all directories that this sequencer is currently // master for. func (s *Sequencer) ApplyRevisionsForAllMasterships(ctx context.Context) error { - glog.Infof("ApplyRevisionsForAllMasterships") return s.ForAllMasterships(ctx, func(ctx context.Context, dirID string) error { req := &spb.ApplyRevisionsRequest{DirectoryId: dirID} if _, err := s.sequencerClient.ApplyRevisions(ctx, req); err != nil { @@ -160,7 +158,6 @@ func (s *Sequencer) ApplyRevisionsForAllMasterships(ctx context.Context) error { // PublishLogForAllMasterships runs KeyTransparencySequencer.PublishRevisions on // all directories this sequencer is currently master for. func (s *Sequencer) PublishLogForAllMasterships(ctx context.Context) error { - glog.Infof("PublishLogForAllMasterships") return s.ForAllMasterships(ctx, func(ctx context.Context, dirID string) error { publishReq := &spb.PublishRevisionsRequest{DirectoryId: dirID} _, err := s.sequencerClient.PublishRevisions(ctx, publishReq) diff --git a/core/sequencer/sequencer_api.proto b/core/sequencer/sequencer_api.proto index fb64255e5..a71659db9 100644 --- a/core/sequencer/sequencer_api.proto +++ b/core/sequencer/sequencer_api.proto @@ -31,10 +31,12 @@ message MapMetadata { // lowest_inclusive is the lowest primary key (inclusive) of the source // log that has been incorporated into this map revision. The primary // keys of logged items MUST be monotonically increasing. + // Defined in terms of microseconds from the Unix epoch. int64 lowest_inclusive = 1; // highest_exclusive is the highest primary key (exclusive) of the source // log that has been incorporated into this map revision. The primary keys // of logged items MUST be monotonically increasing. + // Defined in terms of microseconds from the Unix epoch. int64 highest_exclusive = 2; // log_id is the ID of the source log. int64 log_id = 3; diff --git a/core/sequencer/sequencer_go_proto/sequencer_api.pb.go b/core/sequencer/sequencer_go_proto/sequencer_api.pb.go index 3e2cc1522..c49d2cf0e 100644 --- a/core/sequencer/sequencer_go_proto/sequencer_api.pb.go +++ b/core/sequencer/sequencer_go_proto/sequencer_api.pb.go @@ -76,10 +76,12 @@ type MapMetadata_SourceSlice struct { // lowest_inclusive is the lowest primary key (inclusive) of the source // log that has been incorporated into this map revision. The primary // keys of logged items MUST be monotonically increasing. + // Defined in terms of microseconds from the Unix epoch. LowestInclusive int64 `protobuf:"varint,1,opt,name=lowest_inclusive,json=lowestInclusive,proto3" json:"lowest_inclusive,omitempty"` // highest_exclusive is the highest primary key (exclusive) of the source // log that has been incorporated into this map revision. The primary keys // of logged items MUST be monotonically increasing. + // Defined in terms of microseconds from the Unix epoch. HighestExclusive int64 `protobuf:"varint,2,opt,name=highest_exclusive,json=highestExclusive,proto3" json:"highest_exclusive,omitempty"` // log_id is the ID of the source log. LogId int64 `protobuf:"varint,3,opt,name=log_id,json=logId,proto3" json:"log_id,omitempty"` diff --git a/core/sequencer/server.go b/core/sequencer/server.go index cbc80d189..4e938a75d 100644 --- a/core/sequencer/server.go +++ b/core/sequencer/server.go @@ -32,6 +32,7 @@ import ( "github.com/google/keytransparency/core/mutator" "github.com/google/keytransparency/core/mutator/entry" "github.com/google/keytransparency/core/sequencer/mapper" + "github.com/google/keytransparency/core/sequencer/metadata" "github.com/google/keytransparency/core/sequencer/runner" spb "github.com/google/keytransparency/core/sequencer/sequencer_go_proto" @@ -62,6 +63,13 @@ var ( unappliedRevisions monitoring.Gauge ) +// zero is the time to use for the beginning edge of new watermarks. +// When getting the watermark for a new log that is not in the lastMeta set, +// we need to pick a lower bound starting point for reading. time.Time{} is +// not a valid choice here because it is too low to represent with metadata.New() +// so we need to pick a different, low sentinel value that the tests know about. +var zero = time.Unix(0, 0) + func createMetrics(mf monitoring.MetricFactory) { knownDirectories = mf.NewGauge( "known_directories", @@ -124,17 +132,15 @@ type Watermarks map[int64]int64 type LogsReader interface { // HighWatermark returns the number of items and the highest primary // key up to batchSize items after start (exclusive). - HighWatermark(ctx context.Context, directoryID string, logID, start int64, - batchSize int32) (count int32, watermark int64, err error) + HighWatermark(ctx context.Context, directoryID string, logID int64, start time.Time, + batchSize int32) (count int32, watermark time.Time, err error) // ListLogs returns the logIDs associated with directoryID that have their write bits set, // or all logIDs associated with directoryID if writable is false. ListLogs(ctx context.Context, directoryID string, writable bool) ([]int64, error) - // ReadLog returns the lowest messages in the (low, high] range stored in the - // specified log, up to batchSize. Paginate by setting low to the - // highest LogMessage returned in the previous page. - ReadLog(ctx context.Context, directoryID string, logID, low, high int64, + // ReadLog returns the lowest messages in the [low, high) range stored in the specified log, up to batchSize. + ReadLog(ctx context.Context, directoryID string, logID int64, low, high time.Time, batchSize int32) ([]*mutator.LogMessage, error) } @@ -347,23 +353,25 @@ func (s *Server) ApplyRevisions(ctx context.Context, in *spb.ApplyRevisionsReque func (s *Server) readMessages(ctx context.Context, source *spb.MapMetadata_SourceSlice, directoryID string, chunkSize int32, emit func(*mutator.LogMessage)) error { - low := source.LowestInclusive - high := source.HighestExclusive - // Loop until less than chunkSize items are returned. - for count := chunkSize; count == chunkSize; { - batch, err := s.logs.ReadLog(ctx, directoryID, source.LogId, low, high, chunkSize) + ss := metadata.FromProto(source) + low := ss.StartTime() + high := ss.EndTime() + for moreToRead := true; moreToRead; { + // Request one more item than chunkSize so we can find the next page token. + batch, err := s.logs.ReadLog(ctx, directoryID, source.LogId, low, high, chunkSize+1) if err != nil { return fmt.Errorf("logs.ReadLog(): %v", err) } - count = int32(len(batch)) glog.Infof("ReadLog(dir: %v log: %v, (%v, %v], %v) count: %v", - directoryID, source.LogId, low, high, chunkSize, count) + directoryID, source.LogId, low, high, chunkSize, len(batch)) + moreToRead = int32(len(batch)) == (chunkSize + 1) + if moreToRead { + low = batch[chunkSize].ID // Use the last row as the start of the next read. + batch = batch[:chunkSize] // Don't emit the next page token. + } logEntryCount.Add(float64(len(batch)), directoryID, fmt.Sprintf("%v", source.LogId)) for _, m := range batch { emit(m) - if m.ID > low { - low = m.ID - } } } return nil @@ -385,7 +393,6 @@ func (s *Server) ApplyRevision(ctx context.Context, in *spb.ApplyRevisionRequest logSlices := runner.DoMapMetaFn(mapper.MapMetaFn, meta, incMetricFn) logItems, err := runner.DoReadFn(ctx, s.readMessages, logSlices, in.DirectoryId, s.BatchSize, incMetricFn) if err != nil { - mutationFailures.Inc(err.Error()) return nil, err } @@ -509,6 +516,7 @@ func (s *Server) PublishRevisions(ctx context.Context, leaves[int64(mapRoot.Revision)] = rawMapRoot.GetMapRoot() revs = append(revs, int64(mapRoot.Revision)) } + glog.Infof("Publishing revisions %d-%d", logRoot.TreeSize-1, end) if err := logClient.AddSequencedLeaves(ctx, leaves); err != nil { glog.Errorf("AddSequencedLeaves(revs: %v): %v", revs, err) return nil, err @@ -535,12 +543,13 @@ func (s *Server) HighWatermarks(ctx context.Context, directoryID string, lastMet // revision. // TODO(gbelvin): Separate end watermarks for the sequencer's needs // from ranges of watermarks for the verifier's needs. - ends := map[int64]int64{} - starts := map[int64]int64{} + ends := make(map[int64]time.Time) + starts := make(map[int64]time.Time) for _, source := range lastMeta.GetSources() { - if ends[source.LogId] < source.HighestExclusive { - ends[source.LogId] = source.HighestExclusive - starts[source.LogId] = source.HighestExclusive + highest := metadata.FromProto(source).EndTime() + if ends[source.LogId].Before(highest) { + ends[source.LogId] = highest + starts[source.LogId] = highest } } @@ -551,7 +560,10 @@ func (s *Server) HighWatermarks(ctx context.Context, directoryID string, lastMet } // TODO(gbelvin): Get HighWatermarks in parallel. for _, logID := range logIDs { - low := ends[logID] + low, ok := ends[logID] + if !ok { + low = zero // Here be dragons. See comment on zero. + } count, high, err := s.logs.HighWatermark(ctx, directoryID, logID, low, batchSize) if err != nil { return 0, nil, status.Errorf(codes.Internal, @@ -565,11 +577,11 @@ func (s *Server) HighWatermarks(ctx context.Context, directoryID string, lastMet meta := &spb.MapMetadata{} for logID, end := range ends { - meta.Sources = append(meta.Sources, &spb.MapMetadata_SourceSlice{ - LogId: logID, - LowestInclusive: starts[logID], - HighestExclusive: end, - }) + src, err := metadata.New(logID, starts[logID], end) + if err != nil { + return 0, nil, err + } + meta.Sources = append(meta.Sources, src.Proto()) } // Deterministic results are nice. sort.Slice(meta.Sources, func(a, b int) bool { diff --git a/core/sequencer/server_test.go b/core/sequencer/server_test.go index 770766255..cbbaec730 100644 --- a/core/sequencer/server_test.go +++ b/core/sequencer/server_test.go @@ -17,8 +17,8 @@ package sequencer import ( "context" "fmt" - "sort" "testing" + "time" "github.com/golang/protobuf/proto" "github.com/google/go-cmp/cmp" @@ -26,10 +26,12 @@ import ( "github.com/google/trillian/types" "google.golang.org/grpc" - "github.com/google/keytransparency/core/mutator" "github.com/google/keytransparency/core/sequencer/mapper" + "github.com/google/keytransparency/core/sequencer/metadata" "github.com/google/keytransparency/core/sequencer/runner" + "github.com/google/keytransparency/impl/memory" + pb "github.com/google/keytransparency/core/api/v1/keytransparency_go_proto" spb "github.com/google/keytransparency/core/sequencer/sequencer_go_proto" tpb "github.com/google/trillian" ) @@ -38,36 +40,8 @@ const directoryID = "directoryID" func fakeMetric(_ string) {} -type fakeLogs map[int64][]mutator.LogMessage - -func (l fakeLogs) ReadLog(ctx context.Context, directoryID string, logID, low, high int64, - batchSize int32) ([]*mutator.LogMessage, error) { - refs := make([]*mutator.LogMessage, 0, int(high-low)) - for i := low; i < high; i++ { - l[logID][i].ID = i - refs = append(refs, &l[logID][i]) - } - return refs, nil -} - -func (l fakeLogs) ListLogs(ctx context.Context, directoryID string, writable bool) ([]int64, error) { - logIDs := make([]int64, 0, len(l)) - for logID := range l { - logIDs = append(logIDs, logID) - } - // sort logsIDs for test repeatability. - sort.Slice(logIDs, func(i, j int) bool { return logIDs[i] < logIDs[j] }) - return logIDs, nil -} - -func (l fakeLogs) HighWatermark(ctx context.Context, directoryID string, logID, start int64, - batchSize int32) (int32, int64, error) { - high := start + int64(batchSize) - if high > int64(len(l[logID])) { - high = int64(len(l[logID])) - } - count := int32(high - start) - return count, high, nil +func init() { + initMetrics.Do(func() { createMetrics(monitoring.InertMetricFactory{}) }) } type fakeTrillianFactory struct { @@ -124,16 +98,42 @@ func (b *fakeBatcher) ReadBatch(_ context.Context, _ string, rev int64) (*spb.Ma return meta, nil } +func setupLogs(ctx context.Context, t *testing.T, dirID string, logLengths map[int64]int) (memory.MutationLogs, map[int64][]time.Time) { + t.Helper() + fakeLogs := memory.NewMutationLogs() + idx := make(map[int64][]time.Time) + for logID, msgs := range logLengths { + if err := fakeLogs.AddLogs(ctx, dirID, logID); err != nil { + t.Fatal(err) + } + for i := 0; i < msgs; i++ { + ts, err := fakeLogs.Send(ctx, dirID, logID, &pb.EntryUpdate{}) + if err != nil { + t.Fatal(err) + } + idx[logID] = append(idx[logID], ts) + } + } + return fakeLogs, idx +} + +func newSource(t *testing.T, logID int64, low, high time.Time) *spb.MapMetadata_SourceSlice { + t.Helper() + s, err := metadata.New(logID, low, high) + if err != nil { + t.Fatalf("Invalid source: %v", err) + } + return s.Proto() +} + func TestDefiningRevisions(t *testing.T) { // Verify that outstanding revisions prevent future revisions from being created. ctx := context.Background() mapRev := int64(2) - initMetrics.Do(func() { createMetrics(monitoring.InertMetricFactory{}) }) + dirID := "foobar" + fakeLogs, idx := setupLogs(ctx, t, dirID, map[int64]int{0: 10, 1: 20}) s := Server{ - logs: fakeLogs{ - 0: make([]mutator.LogMessage, 10), - 1: make([]mutator.LogMessage, 20), - }, + logs: fakeLogs, trillian: &fakeTrillianFactory{ tmap: &fakeMap{latestMapRoot: &types.MapRootV1{Revision: uint64(mapRev)}}, }, @@ -153,14 +153,14 @@ func TestDefiningRevisions(t *testing.T) { {desc: "skewed", highestRev: mapRev - 1, wantNew: mapRev - 1}, {desc: "almost_drained", highestRev: mapRev, meta: spb.MapMetadata{Sources: []*spb.MapMetadata_SourceSlice{ - {LogId: 0, LowestInclusive: 0, HighestExclusive: 9}, - {LogId: 1, LowestInclusive: 0, HighestExclusive: 20}, + newSource(t, 0, zero.Add(0*time.Microsecond), zero.Add(9*time.Microsecond)), + newSource(t, 1, zero.Add(0*time.Microsecond), zero.Add(20*time.Microsecond)), }}, wantNew: mapRev + 1}, {desc: "drained", highestRev: mapRev, meta: spb.MapMetadata{Sources: []*spb.MapMetadata_SourceSlice{ - {LogId: 0, LowestInclusive: 0, HighestExclusive: 10}, - {LogId: 1, LowestInclusive: 0, HighestExclusive: 20}, + newSource(t, 0, zero, idx[0][9].Add(1)), + newSource(t, 1, zero, idx[1][19].Add(1)), }}, wantNew: mapRev}, } { @@ -202,10 +202,9 @@ func TestDefiningRevisions(t *testing.T) { func TestReadMessages(t *testing.T) { ctx := context.Background() - s := Server{logs: fakeLogs{ - 0: make([]mutator.LogMessage, 10), - 1: make([]mutator.LogMessage, 20), - }} + dirID := "TestReadMessages" + fakeLogs, idx := setupLogs(ctx, t, dirID, map[int64]int{0: 10, 1: 20}) + s := Server{logs: fakeLogs} for _, tc := range []struct { meta *spb.MapMetadata @@ -213,11 +212,14 @@ func TestReadMessages(t *testing.T) { want int }{ {batchSize: 1, want: 9, meta: &spb.MapMetadata{Sources: []*spb.MapMetadata_SourceSlice{ - {LogId: 0, LowestInclusive: 1, HighestExclusive: 10}, + newSource(t, 0, idx[0][1], idx[0][9].Add(1)), + }}}, + {batchSize: 10000, want: 9, meta: &spb.MapMetadata{Sources: []*spb.MapMetadata_SourceSlice{ + newSource(t, 0, idx[0][1], idx[0][9].Add(1)), }}}, {batchSize: 1, want: 19, meta: &spb.MapMetadata{Sources: []*spb.MapMetadata_SourceSlice{ - {LogId: 0, LowestInclusive: 1, HighestExclusive: 10}, - {LogId: 1, LowestInclusive: 1, HighestExclusive: 11}, + newSource(t, 0, idx[0][1], idx[0][9].Add(1)), + newSource(t, 1, idx[1][1], idx[1][10].Add(1)), }}}, } { logSlices := runner.DoMapMetaFn(mapper.MapMetaFn, tc.meta, fakeMetric) @@ -226,17 +228,16 @@ func TestReadMessages(t *testing.T) { t.Errorf("readMessages(): %v", err) } if got := len(logItems); got != tc.want { - t.Errorf("readMessages(): len: %v, want %v", got, tc.want) + t.Errorf("readMessages(%v): len: %v, want %v", tc.meta, got, tc.want) } } } func TestHighWatermarks(t *testing.T) { ctx := context.Background() - s := Server{logs: fakeLogs{ - 0: make([]mutator.LogMessage, 10), - 1: make([]mutator.LogMessage, 20), - }} + dirID := "TestHighWatermark" + fakeLogs, idx := setupLogs(ctx, t, dirID, map[int64]int{0: 10, 1: 20}) + s := Server{logs: fakeLogs} for _, tc := range []struct { desc string @@ -247,32 +248,42 @@ func TestHighWatermarks(t *testing.T) { }{ {desc: "nobatch", batchSize: 30, count: 30, next: &spb.MapMetadata{Sources: []*spb.MapMetadata_SourceSlice{ - {LogId: 0, HighestExclusive: 10}, - {LogId: 1, HighestExclusive: 20}}}}, + newSource(t, 0, zero, idx[0][9].Add(1)), + newSource(t, 1, zero, idx[1][19].Add(1)), + }}}, {desc: "exactbatch", batchSize: 20, count: 20, next: &spb.MapMetadata{Sources: []*spb.MapMetadata_SourceSlice{ - {LogId: 0, HighestExclusive: 10}, - {LogId: 1, HighestExclusive: 10}}}}, + newSource(t, 0, zero, idx[0][9].Add(1)), + newSource(t, 1, zero, idx[1][9].Add(1)), + }}}, {desc: "batchwprev", batchSize: 20, count: 20, last: &spb.MapMetadata{Sources: []*spb.MapMetadata_SourceSlice{ - {LogId: 0, HighestExclusive: 10}}}, + newSource(t, 0, zero, idx[0][9].Add(2)), + }}, next: &spb.MapMetadata{Sources: []*spb.MapMetadata_SourceSlice{ - {LogId: 0, LowestInclusive: 10, HighestExclusive: 10}, - {LogId: 1, HighestExclusive: 20}}}}, + // Nothing to read from log 1, preserve watermark of log 1. + newSource(t, 0, idx[0][9].Add(2), idx[0][9].Add(2)), + newSource(t, 1, zero, idx[1][19].Add(1)), + }}}, // Don't drop existing watermarks. {desc: "keep existing", batchSize: 1, count: 1, last: &spb.MapMetadata{Sources: []*spb.MapMetadata_SourceSlice{ - {LogId: 1, HighestExclusive: 10}}}, + newSource(t, 1, zero, zero.Add(10)), + }}, next: &spb.MapMetadata{Sources: []*spb.MapMetadata_SourceSlice{ - {LogId: 0, HighestExclusive: 1}, - {LogId: 1, LowestInclusive: 10, HighestExclusive: 10}}}}, + newSource(t, 0, zero, idx[0][0].Add(1)), + // No reads from log 1, but don't drop the watermark. + newSource(t, 1, zero.Add(10), zero.Add(10)), + }}}, {desc: "logs that dont move", batchSize: 0, count: 0, last: &spb.MapMetadata{Sources: []*spb.MapMetadata_SourceSlice{ - {LogId: 3, HighestExclusive: 10}}}, + newSource(t, 3, zero, zero.Add(10*time.Microsecond)), + }}, next: &spb.MapMetadata{Sources: []*spb.MapMetadata_SourceSlice{ - {LogId: 0}, - {LogId: 1}, - {LogId: 3, LowestInclusive: 10, HighestExclusive: 10}}}}, + newSource(t, 0, zero, zero), + newSource(t, 1, zero, zero), + newSource(t, 3, zero.Add(10*time.Microsecond), zero.Add(10*time.Microsecond)), + }}}, } { t.Run(tc.desc, func(t *testing.T) { count, next, err := s.HighWatermarks(ctx, directoryID, tc.last, tc.batchSize) @@ -283,7 +294,7 @@ func TestHighWatermarks(t *testing.T) { t.Errorf("HighWatermarks(): count: %v, want %v", count, tc.count) } if !proto.Equal(next, tc.next) { - t.Errorf("HighWatermarks(): diff(-got, +want): %v", cmp.Diff(next, &tc.next)) + t.Errorf("HighWatermarks(): diff(-got, +want): %v", cmp.Diff(next, tc.next)) } }) } diff --git a/core/sequencer/trillian_client.go b/core/sequencer/trillian_client.go index adbcc878e..69a77b25f 100644 --- a/core/sequencer/trillian_client.go +++ b/core/sequencer/trillian_client.go @@ -127,27 +127,6 @@ type MapClient struct { *tclient.MapClient } -// SetLeavesAtRevision creates a new map revision and returns its verified root. -// TODO(gbelvin): Move to Trillian Map client. -func (c *MapClient) SetLeavesAtRevision(ctx context.Context, rev int64, - leaves []*tpb.MapLeaf, metadata []byte) (*types.MapRootV1, error) { - // Set new leaf values. - setResp, err := c.Conn.SetLeaves(ctx, &tpb.SetMapLeavesRequest{ - MapId: c.MapID, - Revision: rev, - Leaves: leaves, - Metadata: metadata, - }) - if err != nil { - return nil, err - } - mapRoot, err := c.VerifySignedMapRoot(setResp.GetMapRoot()) - if err != nil { - return nil, status.Errorf(codes.Internal, "VerifySignedMapRoot(): %v", err) - } - return mapRoot, nil -} - // GetAndVerifyLatestMapRoot verifies and returns the latest map root. func (c *MapClient) GetAndVerifyLatestMapRoot(ctx context.Context) (*tpb.SignedMapRoot, *types.MapRootV1, error) { rootResp, err := c.Conn.GetSignedMapRoot(ctx, &tpb.GetSignedMapRootRequest{MapId: c.MapID}) diff --git a/deploy/kubernetes/db-deployment.yaml b/deploy/kubernetes/db-deployment.yaml index 755f3ea63..3027b890a 100644 --- a/deploy/kubernetes/db-deployment.yaml +++ b/deploy/kubernetes/db-deployment.yaml @@ -27,7 +27,7 @@ spec: value: "yes" - name: MYSQL_USER value: test - image: gcr.io/key-transparency/db + image: gcr.io/trillian-opensource-ci/db_server:latest name: db ports: - containerPort: 3306 diff --git a/deploy/kubernetes/log-server-deployment.yaml b/deploy/kubernetes/log-server-deployment.yaml index 3ba607e76..7c0031514 100644 --- a/deploy/kubernetes/log-server-deployment.yaml +++ b/deploy/kubernetes/log-server-deployment.yaml @@ -25,7 +25,7 @@ spec: "--http_endpoint=0.0.0.0:8091", "--alsologtostderr" ] - image: gcr.io/key-transparency/log-server:latest + image: gcr.io/trillian-opensource-ci/log_server:latest livenessProbe: httpGet: path: /metrics diff --git a/deploy/kubernetes/log-signer-deployment.yaml b/deploy/kubernetes/log-signer-deployment.yaml index 6ee760dfc..74f0ea68f 100644 --- a/deploy/kubernetes/log-signer-deployment.yaml +++ b/deploy/kubernetes/log-signer-deployment.yaml @@ -29,7 +29,7 @@ spec: "--force_master=true", "--alsologtostderr" ] - image: gcr.io/key-transparency/log-signer:latest + image: gcr.io/trillian-opensource-ci/log_signer:latest livenessProbe: httpGet: path: /metrics diff --git a/deploy/kubernetes/map-server-deployment.yaml b/deploy/kubernetes/map-server-deployment.yaml index 0617cc3ac..505f25084 100644 --- a/deploy/kubernetes/map-server-deployment.yaml +++ b/deploy/kubernetes/map-server-deployment.yaml @@ -25,7 +25,7 @@ spec: "--http_endpoint=0.0.0.0:8091", "--alsologtostderr" ] - image: gcr.io/key-transparency/map-server:latest + image: gcr.io/trillian-opensource-ci/map_server:latest livenessProbe: httpGet: path: /metrics diff --git a/deploy/kubernetes/monitor-deployment.yaml b/deploy/kubernetes/monitor-deployment.yaml index f8322e2ba..49db8b948 100644 --- a/deploy/kubernetes/monitor-deployment.yaml +++ b/deploy/kubernetes/monitor-deployment.yaml @@ -17,16 +17,20 @@ spec: labels: io.kompose.service: monitor spec: + volumes: + - name: secrets + secret: + secretName: kt-secrets containers: - command: - - /go/bin/keytransparency-monitor + - /keytransparency-monitor - --addr=0.0.0.0:8099 - --kt-url=server:8080 - --insecure - --directoryid=default - - --tls-key=/kt/server.key - - --tls-cert=/kt/server.crt - - --sign-key=/kt/monitor_sign-key.pem + - --tls-key=/run/secrets/server.key + - --tls-cert=/run/secrets/server.crt + - --sign-key=/run/secrets/monitor_sign-key.pem - --password=towel - --alsologtostderr - --v=3 @@ -35,5 +39,9 @@ spec: ports: - containerPort: 8099 resources: {} + volumeMounts: + - name: secrets + mountPath: "/run/secrets" + readOnly: true restartPolicy: Always status: {} diff --git a/deploy/kubernetes/sequencer-deployment.yaml b/deploy/kubernetes/sequencer-deployment.yaml index 904673fc5..414a85d87 100644 --- a/deploy/kubernetes/sequencer-deployment.yaml +++ b/deploy/kubernetes/sequencer-deployment.yaml @@ -17,14 +17,20 @@ spec: labels: io.kompose.service: sequencer spec: + volumes: + - name: secrets + secret: + secretName: kt-secrets containers: - command: - - /go/bin/keytransparency-sequencer + - /keytransparency-sequencer - --force_master - --db=test:zaphod@tcp(db:3306)/test - --addr=0.0.0.0:8080 - --log-url=log-server:8090 - --map-url=map-server:8090 + - --tls-key=/run/secrets/server.key + - --tls-cert=/run/secrets/server.crt - --alsologtostderr - --v=5 image: gcr.io/key-transparency/keytransparency-sequencer:latest @@ -37,6 +43,10 @@ spec: - containerPort: 8080 - containerPort: 8081 resources: {} + volumeMounts: + - name: secrets + mountPath: "/run/secrets" + readOnly: true - name: prometheus-to-sd image: gcr.io/google-containers/prometheus-to-sd:v0.2.6 command: diff --git a/deploy/kubernetes/server-deployment.yaml b/deploy/kubernetes/server-deployment.yaml index b4131de8c..88f7b36bb 100644 --- a/deploy/kubernetes/server-deployment.yaml +++ b/deploy/kubernetes/server-deployment.yaml @@ -18,15 +18,19 @@ spec: labels: io.kompose.service: server spec: + volumes: + - name: secrets + secret: + secretName: kt-secrets containers: - command: - - /go/bin/keytransparency-server + - /keytransparency-server - --addr=0.0.0.0:8080 - --db=test:zaphod@tcp(db:3306)/test - --log-url=log-server:8090 - --map-url=map-server:8090 - - --tls-key=/kt/server.key - - --tls-cert=/kt/server.crt + - --tls-key=/run/secrets/server.key + - --tls-cert=/run/secrets/server.crt - --auth-type=insecure-fake - --alsologtostderr - --v=5 @@ -40,6 +44,10 @@ spec: - containerPort: 8080 - containerPort: 8081 resources: {} + volumeMounts: + - name: secrets + mountPath: "/run/secrets" + readOnly: true - name: prometheus-to-sd image: gcr.io/google-containers/prometheus-to-sd:v0.2.6 command: diff --git a/go.mod b/go.mod index bbb38f4a0..207ad7157 100644 --- a/go.mod +++ b/go.mod @@ -3,56 +3,73 @@ module github.com/google/keytransparency go 1.12 require ( - github.com/Masterminds/sprig v2.20.0+incompatible // indirect + cloud.google.com/go v0.49.0 // indirect + cloud.google.com/go/spanner v1.1.0 // indirect + github.com/Masterminds/semver v1.5.0 // indirect + github.com/Masterminds/sprig v2.22.0+incompatible // indirect github.com/benlaurie/objecthash v0.0.0-20180202135721-d1e3d6079fc1 - github.com/coreos/etcd v3.3.15+incompatible // indirect + github.com/bombsimon/wsl v1.2.8 // indirect + github.com/cespare/xxhash/v2 v2.1.1 // indirect + github.com/coreos/etcd v3.3.17+incompatible // indirect github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect - github.com/emicklei/proto v1.6.15 // indirect - github.com/fullstorydev/grpcurl v1.3.2 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect + github.com/emicklei/proto v1.8.0 // indirect github.com/go-sql-driver/mysql v1.4.1 - github.com/gobuffalo/flect v0.1.6 // indirect - github.com/gogo/protobuf v1.3.0 // indirect + github.com/gogo/protobuf v1.3.1 // indirect github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b + github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9 // indirect github.com/golang/mock v1.3.1 github.com/golang/protobuf v1.3.2 - github.com/golangci/golangci-lint v1.18.0 // indirect - github.com/google/go-cmp v0.3.0 + github.com/golangci/golangci-lint v1.21.0 // indirect + github.com/google/certificate-transparency-go v1.1.0 // indirect + github.com/google/go-cmp v0.3.1 github.com/google/tink v1.2.1-0.20190523150020-6495d823d968 - github.com/google/trillian v1.3.2-0.20190910105707-fa759d0a2aee + github.com/google/trillian v1.3.3 github.com/gorilla/websocket v1.4.1 // indirect github.com/gostaticanalysis/analysisutil v0.0.3 // indirect - github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 + github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 - github.com/grpc-ecosystem/grpc-gateway v1.11.1 + github.com/grpc-ecosystem/grpc-gateway v1.9.4 + github.com/imdario/mergo v0.3.8 // indirect + github.com/json-iterator/go v1.1.8 // indirect + github.com/jstemmer/go-junit-report v0.9.1 // indirect github.com/kr/pretty v0.1.0 github.com/kylelemons/godebug v1.1.0 github.com/lyft/protoc-gen-validate v0.1.0 // indirect - github.com/mattn/go-isatty v0.0.9 // indirect - github.com/mattn/go-sqlite3 v1.10.0 - github.com/mwitkow/go-proto-validators v0.1.0 // indirect - github.com/prometheus/client_golang v1.1.0 - github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 // indirect - github.com/prometheus/procfs v0.0.4 // indirect + github.com/mattn/go-isatty v0.0.10 // indirect + github.com/mattn/go-runewidth v0.0.6 // indirect + github.com/mitchellh/copystructure v1.0.0 // indirect + github.com/mitchellh/reflectwalk v1.0.1 // indirect + github.com/mwitkow/go-proto-validators v0.2.0 // indirect + github.com/olekukonko/tablewriter v0.0.2 // indirect + github.com/pelletier/go-toml v1.6.0 // indirect + github.com/prometheus/client_golang v1.2.1 + github.com/prometheus/procfs v0.0.7 // indirect github.com/russross/blackfriday v2.0.0+incompatible // indirect - github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect + github.com/securego/gosec v0.0.0-20191119104125-df484bfa9e9f // indirect github.com/spf13/cobra v0.0.5 - github.com/spf13/pflag v1.0.3 - github.com/spf13/viper v1.4.0 - github.com/ultraware/funlen v0.0.2 // indirect - github.com/urfave/cli v1.22.0 // indirect - go.etcd.io/etcd v3.3.15+incompatible // indirect - golang.org/x/crypto v0.0.0-20190909091759-094676da4a83 - golang.org/x/net v0.0.0-20190909003024-a7b16738d86b // indirect + github.com/spf13/pflag v1.0.5 + github.com/spf13/viper v1.5.0 + github.com/uber/prototool v1.9.0 // indirect + github.com/urfave/cli v1.22.1 // indirect + github.com/uudashr/gocognit v1.0.0 // indirect + go.etcd.io/etcd v3.3.17+incompatible // indirect + go.opencensus.io v0.22.2 // indirect + go.uber.org/atomic v1.5.1 // indirect + go.uber.org/multierr v1.4.0 // indirect + go.uber.org/zap v1.13.0 // indirect + golang.org/x/crypto v0.0.0-20191119213627-4f8c1d86b1ba + golang.org/x/net v0.0.0-20191119073136-fc4aabc6c914 // indirect golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 - golang.org/x/sync v0.0.0-20190423024810-112230192c58 - golang.org/x/sys v0.0.0-20190910064555-bbd175535a8b // indirect - golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 - golang.org/x/tools v0.0.0-20190910044552-dd2b5c81c578 // indirect - google.golang.org/api v0.7.0 - google.golang.org/genproto v0.0.0-20190905072037-92dd089d5514 - google.golang.org/grpc v1.23.0 - mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f // indirect + golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e + golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e // indirect + golang.org/x/time v0.0.0-20191024005414-555d28b269f0 + golang.org/x/tools v0.0.0-20191120221951-8fd459516a27 // indirect + google.golang.org/api v0.14.0 + google.golang.org/appengine v1.6.5 // indirect + google.golang.org/genproto v0.0.0-20191115221424-83cc0476cb11 + google.golang.org/grpc v1.25.1 + gopkg.in/yaml.v2 v2.2.7 // indirect + mvdan.cc/unparam v0.0.0-20191111180625-960b1ec0f2c2 // indirect sigs.k8s.io/yaml v1.1.0 // indirect ) - -replace google.golang.org/genproto v0.0.0-20170818100345-ee236bd376b0 => google.golang.org/genproto v0.0.0-20170818010345-ee236bd376b0 diff --git a/go.sum b/go.sum index cd497d197..3072b8afd 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,25 @@ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.41.0 h1:NFvqUTDnSNYPX5oReekmB+D+90jrJIcVImxQ3qrBVgM= cloud.google.com/go v0.41.0/go.mod h1:OauMR7DV8fzvZIl2qg6rkaIhD/vmgk4iwEw/h6ercmg= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.49.0 h1:CH+lkubJzcPYB1Ggupcq0+k8Ni2ILdG2lYjDIgavDBQ= +cloud.google.com/go v0.49.0/go.mod h1:hGvAdzcWNbyuxS3nWhD7H2cIJxjRRTRLQVB0bdputVY= +cloud.google.com/go/bigquery v1.0.1 h1:hL+ycaJpVE9M7nLoiXb/Pn10ENE2u+oddxbD8uu0ZVU= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/datastore v1.0.0 h1:Kt+gOPPp2LEPWp8CSfxhsM8ik9CcyE/gYu+0r+RnZvM= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/pubsub v1.0.1 h1:W9tAK3E57P75u0XLLR82LZyw8VpAnhmyTOxW9qzmyj8= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/spanner v1.1.0 h1:hIjiz2Pf6Hy3BWz+Oaw7XUqP+EzWDkj0/DtTkKazxzk= +cloud.google.com/go/spanner v1.1.0/go.mod h1:TzTaF9l2ZY2CIetNvVpUu6ZQy8YEOtzB6ICa5EwYjL0= +cloud.google.com/go/storage v1.0.0 h1:VV2nUM3wwLLGh9lSABFgZMjInyUbJeaRSE64WuAIQ+4= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= contrib.go.opencensus.io/exporter/stackdriver v0.12.1/go.mod h1:iwB6wGarfphGGe/e5CWqyUk/cLzKnWsOKPVW3no6OTw= contrib.go.opencensus.io/resource v0.1.1/go.mod h1:F361eGI91LCmW1I/Saf+rX0+OFcigGlFvXwEGEnkRLA= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= @@ -15,18 +32,24 @@ github.com/Masterminds/goutils v1.1.0 h1:zukEsf/1JZwCMgHiK3GZftabmxiCw4apj3a28RP github.com/Masterminds/goutils v1.1.0/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Masterminds/semver v1.4.2 h1:WBLTQ37jOCzSLtXNdoo8bNM8876KhNqOKvrlGITgsTc= github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= +github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= +github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/sprig v2.18.0+incompatible h1:QoGhlbC6pter1jxKnjMFxT8EqsLuDE6FEcNbWEpw+lI= github.com/Masterminds/sprig v2.18.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= -github.com/Masterminds/sprig v2.20.0+incompatible h1:dJTKKuUkYW3RMFdQFXPU/s6hg10RgctmTjRcbZ98Ap8= -github.com/Masterminds/sprig v2.20.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= +github.com/Masterminds/sprig v2.22.0+incompatible h1:z4yfnGrZ7netVz+0EDJ0Wi+5VZCSYp4Z0m2dk6cEM60= +github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OpenPeeDeeP/depguard v1.0.0 h1:k9QF73nrHT3nPLz3lu6G5s+3Hi8Je36ODr1F5gjAXXM= github.com/OpenPeeDeeP/depguard v1.0.0/go.mod h1:7/4sitnI9YlQgTLLk734QlzXT8DuHVnAyztLplQjk+o= +github.com/OpenPeeDeeP/depguard v1.0.1 h1:VlW4R6jmBIv3/u1JNlawEvJMM4J+dPORPaZasQee8Us= +github.com/OpenPeeDeeP/depguard v1.0.1/go.mod h1:xsIw86fROiiwelg+jB2uM9PiKihMMmUx/1V+TNhjQvM= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/aws/aws-sdk-go v1.19.18/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= @@ -40,8 +63,18 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bombsimon/wsl v1.2.5 h1:9gTOkIwVtoDZywvX802SDHokeX4kW1cKnV8ZTVAPkRs= +github.com/bombsimon/wsl v1.2.5/go.mod h1:43lEF/i0kpXbLCeDXL9LMT8c92HyBywXb0AsgMHYngM= +github.com/bombsimon/wsl v1.2.8 h1:b+E/W7koicKBZDU+vEsw/hnQTN8026Gv1eMZDLUU/Wc= +github.com/bombsimon/wsl v1.2.8/go.mod h1:43lEF/i0kpXbLCeDXL9LMT8c92HyBywXb0AsgMHYngM= github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.0 h1:yTUvW7Vhb89inJ+8irsUqiWjh8iT6sQPZiQzI6ReGkA= +github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/coreos/bbolt v1.3.2 h1:wZwiHHUieZCquLkDL0B8UhzreNWsPHooDAG3q34zk0s= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= @@ -51,8 +84,8 @@ github.com/coreos/etcd v3.3.10+incompatible h1:jFneRYjIvLMLhDLCzuTuU4rSJUjRplcJQ github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible h1:8F3hqu9fGYLBifCmRCJsicFqDx/D68Rt3q1JMazcgBQ= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/etcd v3.3.15+incompatible h1:+9RjdC18gMxNQVvSiXvObLu29mOFmkgdsB4cRTlV+EE= -github.com/coreos/etcd v3.3.15+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/etcd v3.3.17+incompatible h1:f/Z3EoDSx1yjaIjLQGo1diYUlQYSBrrAQ5vP8NjwXwo= +github.com/coreos/etcd v3.3.17+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0 h1:3Jm3tLmsgAYcjC+4Up7hJrFBPr+n7rAqYeSw/SZazuY= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -68,6 +101,11 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbp github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -79,29 +117,34 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn github.com/emicklei/proto v1.6.12/go.mod h1:rn1FgRS/FANiZdD2djyH7TMA9jdRDcYQ9IEN9yvjX0A= github.com/emicklei/proto v1.6.13 h1:8iuAuKbFmFhkmstObb0EV/Hrn9W+x6EuV1y5Da8Ye9E= github.com/emicklei/proto v1.6.13/go.mod h1:rn1FgRS/FANiZdD2djyH7TMA9jdRDcYQ9IEN9yvjX0A= -github.com/emicklei/proto v1.6.15 h1:XbpwxmuOPrdES97FrSfpyy67SSCV/wBIKXqgJzh6hNw= -github.com/emicklei/proto v1.6.15/go.mod h1:rn1FgRS/FANiZdD2djyH7TMA9jdRDcYQ9IEN9yvjX0A= +github.com/emicklei/proto v1.7.0/go.mod h1:rn1FgRS/FANiZdD2djyH7TMA9jdRDcYQ9IEN9yvjX0A= +github.com/emicklei/proto v1.8.0 h1:/MjKUvy7nh0ryszHc/PIIFiv/BU5XYX4NfvpM19C5+U= +github.com/emicklei/proto v1.8.0/go.mod h1:rn1FgRS/FANiZdD2djyH7TMA9jdRDcYQ9IEN9yvjX0A= github.com/emirpasic/gods v1.9.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.6.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fullstorydev/grpcurl v1.3.0 h1:XwiEVudma689HpecmJX6sTmOQCd1G5Cyqyj+zSqdk30= -github.com/fullstorydev/grpcurl v1.3.0/go.mod h1:MtIe/E5RoUPHdbcijFZe/WoJKQtal5JNj0KGIbVqCy8= -github.com/fullstorydev/grpcurl v1.3.1 h1:mSFlRqdcfDkDujpG96nGLAZcSDpG0UnxtoCFQx83AiI= -github.com/fullstorydev/grpcurl v1.3.1/go.mod h1:Hsh6mJe5uN/Oqxv/QDmmUClKE+wMs4PUNh/Xz16H0R8= github.com/fullstorydev/grpcurl v1.3.2 h1:cJKWsBYMocdxXQvgbnhtLG810SL5MhKT4K7BagxRih8= github.com/fullstorydev/grpcurl v1.3.2/go.mod h1:kvk8xPCXOrwVd9zYdjy+xSOT4YWm6kyth4Y9NMfBns4= +github.com/fullstorydev/grpcurl v1.4.0 h1:rKQyAaegPtCj4mpItnCHd+PIEHspIZl14VWhHYIHhls= +github.com/fullstorydev/grpcurl v1.4.0/go.mod h1:kvk8xPCXOrwVd9zYdjy+xSOT4YWm6kyth4Y9NMfBns4= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/gliderlabs/ssh v0.1.4/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/go-critic/go-critic v0.3.5-0.20190526074819-1df300866540 h1:djv/qAomOVj8voCHt0M0OYwR/4vfDq1zNKSPKjJCexs= github.com/go-critic/go-critic v0.3.5-0.20190526074819-1df300866540/go.mod h1:+sE8vrLDS2M0pZkBk0wy6+nLdKexVDrl/jBqQOTDThA= +github.com/go-critic/go-critic v0.3.5-0.20190904082202-d79a9f0c64db h1:GYXWx7Vr3+zv833u+8IoXbNnQY0AdXsxAgI0kX7xcwA= +github.com/go-critic/go-critic v0.3.5-0.20190904082202-d79a9f0c64db/go.mod h1:+sE8vrLDS2M0pZkBk0wy6+nLdKexVDrl/jBqQOTDThA= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-lintpack/lintpack v0.5.2 h1:DI5mA3+eKdWeJ40nU4d6Wc26qmdG8RCi/btYq0TuRN0= github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= @@ -139,6 +182,7 @@ github.com/gobuffalo/flect v0.1.6 h1:D7KWNRFiCknJKA495/e1BO7oxqf8tbieaLv/ehoZ/+g github.com/gobuffalo/flect v0.1.6/go.mod h1:W3K3X9ksuZfir8f/LrfVtWmCDQFfayuylOJ7sz/Fj80= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/gofrs/flock v0.0.0-20190320160742-5135e617513b/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gofrs/flock v0.7.1 h1:DP+LD/t0njgoPBvT5MJLeliUIVQR03hiKR6vezdwHlc= github.com/gofrs/flock v0.7.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/protobuf v1.1.1 h1:72R+M5VuhED/KujmZVcIquuo8mBgX4oVda//DQb3PXo= @@ -147,12 +191,16 @@ github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.0 h1:G8O7TerXerS4F6sx9OV7/nRfJdnXgHZu/S/7F2SN+UE= github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9 h1:uHTyIjqVhYRhLbJ8nIiOJHkEZZ+5YoOsAbD3sk82NiE= +github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.0.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.1.1 h1:G5FRp8JnTd7RQH5kemVNlMeyXQAztQ3mOWV95KxsXH8= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -183,16 +231,20 @@ github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d h1:pXTK/gkVNs7Zyy github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU= github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98 h1:0OkFarm1Zy2CjCiDKfK9XHgmc2wbDlRMD2hD8anAJHU= github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU= +github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a h1:iR3fYXUjHCR97qWS8ch1y9zPNsgXThGwjKPrYfqMPks= +github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU= github.com/golangci/golangci-lint v1.17.2-0.20190910081718-bad04bb7378f h1:Ocb3mZ76SbwTM6VKfiMPEppkwzitinZFRW9E6zAD5qc= github.com/golangci/golangci-lint v1.17.2-0.20190910081718-bad04bb7378f/go.mod h1:kaqo8l0OZKYPtjNmG4z4HrWLgcYNIJ9B9q3LWri9uLg= -github.com/golangci/golangci-lint v1.18.0 h1:XmQgfcLofSG/6AsQuQqmLizB+3GggD+o6ObBG9L+VMM= -github.com/golangci/golangci-lint v1.18.0/go.mod h1:kaqo8l0OZKYPtjNmG4z4HrWLgcYNIJ9B9q3LWri9uLg= +github.com/golangci/golangci-lint v1.21.0 h1:HxAxpR8Z0M8omihvQdsD3PF0qPjlqYqp2vMJzstoKeI= +github.com/golangci/golangci-lint v1.21.0/go.mod h1:phxpHK52q7SE+5KpPnti4oZTdFCEsn/tKN+nFvCKXfk= github.com/golangci/gosec v0.0.0-20190211064107-66fb7fc33547 h1:fUdgm/BdKvwOHxg5AhNbkNRp2mSy8sxTXyBVs/laQHo= github.com/golangci/gosec v0.0.0-20190211064107-66fb7fc33547/go.mod h1:0qUabqiIQgfmlAmulqxyiGkkyF6/tOGSnY2cnPVwrzU= github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc h1:gLLhTLMk2/SutryVJ6D4VZCU3CUqr8YloG7FPIBWFpI= github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU= github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217 h1:En/tZdwhAn0JNwLuXzP3k2RVtMqMmOEK7Yu/g3tmtJE= github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg= +github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0 h1:MfyDlzVjl1hoaPzPD4Gpb/QgoRfSBR0jdhwGyAWwMSA= +github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg= github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca h1:kNY3/svz5T29MYHubXix4aDDuE3RWHkPvopM/EDv/MA= github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o= github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770 h1:EL/O5HGrF7Jaq0yNhBLucz9hTuRzj2LdwGBOaENgxIk= @@ -209,21 +261,27 @@ github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/certificate-transparency-go v1.0.22-0.20190910093103-496c2e82955b h1:cagua7VIVtUFRYOfOeRvLPEQ6KJut//9bo6WFsOl8cc= github.com/google/certificate-transparency-go v1.0.22-0.20190910093103-496c2e82955b/go.mod h1:i+Q7XY+ArBveOUT36jiHGfuSK1fHICIg6sUkRxPAbCs= +github.com/google/certificate-transparency-go v1.1.0 h1:10MlrYzh5wfkToxWI4yJzffsxLfxcEDlOATMx/V9Kzw= +github.com/google/certificate-transparency-go v1.1.0/go.mod h1:i+Q7XY+ArBveOUT36jiHGfuSK1fHICIg6sUkRxPAbCs= github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/licenseclassifier v0.0.0-20190501212618-47b603fe1b8c/go.mod h1:qsqn2hxC+vURpyBRygGUuinTO42MFRLcsmQ/P8v94+M= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/monologue v0.0.0-20190606152607-4b11a32b5934/go.mod h1:6NTfaQoUpg5QmPsCUWLR3ig33FHrKXhTtWzF0DVdmuk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/tink v1.2.1-0.20190523150020-6495d823d968 h1:3P4TOKsvMmO5ZyE1CPGDiechVe/uxSSjydlSctvcpRg= github.com/google/tink v1.2.1-0.20190523150020-6495d823d968/go.mod h1:eu7D8x3z2rMO7fyvHVhMx8yoFH+vH8EZR1uO3hjEIhQ= github.com/google/trillian v1.2.2-0.20190612132142-05461f4df60a/go.mod h1:YPmUVn5NGwgnDUgqlVyFGMTgaWlnSvH7W5p+NdOG8UA= -github.com/google/trillian v1.3.2-0.20190910105707-fa759d0a2aee h1:rXa4uOLiHeFeL1EAgArf9eooF53zQ4vp5U5lOL0jd4o= -github.com/google/trillian v1.3.2-0.20190910105707-fa759d0a2aee/go.mod h1:zsKY3ZxWBwNhFsCLPRG/Aw9Zl8shNDA175tRx30lzYM= +github.com/google/trillian v1.3.3 h1:xwIDDPBdK5I8hvCTZAna9dNCdZc1OUvkPnSSdiwjT8Y= +github.com/google/trillian v1.3.3/go.mod h1:FFIfFeCKosf1D9mKBxMySFmBI96xHCatmA7NZOXo+qI= github.com/google/trillian-examples v0.0.0-20190603134952-4e75ba15216c/go.mod h1:WgL3XZ3pA8/9cm7yxqWrZE6iZkESB2ItGxy5Fo6k2lk= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -242,14 +300,14 @@ github.com/gostaticanalysis/analysisutil v0.0.3 h1:iwp+5/UAyzQSFgQ4uR2sni99sJ8Eo github.com/gostaticanalysis/analysisutil v0.0.3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 h1:Iju5GlWwrvL6UBg4zJJt3btmonfrMlCDdsejg4CZE7c= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 h1:THDBEeQ9xZ8JEaCLyLQqXMMdRqNr0QAUJTIkQAUtFjg= +github.com/grpc-ecosystem/go-grpc-middleware v1.1.0/go.mod h1:f5nM7jw/oeRSadq3xCzHAvxcr8HZnzsqU6ILg/0NiiE= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0 h1:bM6ZAFZmc/wPFaRDi0d5L7hGEZEx/2u+Tmr2evNHDiI= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.4 h1:5xLhQjsk4zqPf9EHCrja2qFZMx+yBqkO3XgJ14bNnU0= github.com/grpc-ecosystem/grpc-gateway v1.9.4/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.11.1 h1:/dBYI+n4xIL+Y9SKXQrjlKTmJJDwCSlNLRwZ5nBhIek= -github.com/grpc-ecosystem/grpc-gateway v1.11.1/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= @@ -265,13 +323,12 @@ github.com/huandu/xstrings v1.2.0 h1:yPeWdRnmynF7p+lLYz0H2tthW9lqhMJrQV/U7yy4wX0 github.com/huandu/xstrings v1.2.0/go.mod h1:DvyZB1rfVYsBIigL8HwpZgxHwXozlTgGqn63UyNX5k4= github.com/imdario/mergo v0.3.7 h1:Y+UAYTZ7gDEuOfhxKWy+dvb5dRQ6rJjFSdX2HZY1/gI= github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.8 h1:CGgOkSJeqMRmt0D9XLWExdT4m4F1vd3FV3VPt+0VxkQ= +github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jhump/protoreflect v1.4.1/go.mod h1:gZ3i/BeD62fjlaIL0VW4UDMT70CTX+3m4pOnAlJ0BX8= -github.com/jhump/protoreflect v1.4.4 h1:kySdALZUh7xRtW6UoZjjHtlR8k7rLzx5EXJFRvsO5UY= -github.com/jhump/protoreflect v1.4.4/go.mod h1:gZ3i/BeD62fjlaIL0VW4UDMT70CTX+3m4pOnAlJ0BX8= github.com/jhump/protoreflect v1.5.0 h1:NgpVT+dX71c8hZnxHof2M7QDK7QtohIJ7DYycjnkyfc= github.com/jhump/protoreflect v1.5.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -281,7 +338,12 @@ github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwK github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok= +github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024 h1:rBMNdlhTLzJjJSDIjNEXX1Pz3Hmwmz91v+zycvx9PJc= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/juju/ratelimit v1.0.1/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kevinburke/ssh_config v0.0.0-20180830205328-81db2a75821e/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= @@ -302,14 +364,16 @@ github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/letsencrypt/pkcs11key v2.0.1-0.20170608213348-396559074696+incompatible/go.mod h1:iGYXKqDXt0cpBthCHdr9ZdsQwyGlYFh/+8xa4WzIQ34= -github.com/letsencrypt/pkcs11key/v3 v3.0.0/go.mod h1:sfoiYd3g5fIel4ZANw27sbgXzN43FY5hNP6uxEUywkM= github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4= github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/lyft/protoc-gen-validate v0.0.14 h1:xbdDVIHd0Xq5Bfzu+8JR9s7mFmJPMvNLmfGhgcHJdFU= github.com/lyft/protoc-gen-validate v0.0.14/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= @@ -320,31 +384,44 @@ github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDe github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb h1:RHba4YImhrUVQDHUCe2BNSOz4tVy2yGyXhvYDvxGgeE= +github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg= -github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/go-isatty v0.0.10 h1:qxFzApOv4WsAL965uUPIsXzAKCZxN2p9UqdhFS4ZW10= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.6 h1:V2iyH+aX9C5fsYCpK60U8BYIvmhqxuOL3JZcqc1NB7k= +github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-sqlite3 v1.10.0 h1:jbhqpg7tQe4SupckyijYiy0mJJ/pRyHvXf7JdWK860o= github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/pkcs11 v1.0.2/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= +github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ= +github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk= +github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk= github.com/mitchellh/go-wordwrap v1.0.0 h1:6GlHJ/LTGMrIJbwgdqdl2eEH8o+Exx/0m8ir9Gns0u4= github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/reflectwalk v1.0.0 h1:9D+8oIskB4VJBN5SFlmc27fSlIBZaov1Wpk/IfikLNY= +github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/mitchellh/reflectwalk v1.0.1 h1:FVzMWA5RllMAKIdUSC8mdWo3XtwoecrH79BY70sEEpE= +github.com/mitchellh/reflectwalk v1.0.1/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -353,11 +430,12 @@ github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= github.com/mozilla/tls-observatory v0.0.0-20180409132520-8791a200eb40/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk= +github.com/mozilla/tls-observatory v0.0.0-20190404164649-a3c1b6cfecfd/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-proto-validators v0.0.0-20190212092829-1f388280e944 h1:RY9I2vt95pyT12MKfJo1Ck3O+NPQqlukBahURwDtJBc= github.com/mwitkow/go-proto-validators v0.0.0-20190212092829-1f388280e944/go.mod h1:m2XC9Qq0AlmmVksL6FktJCdTYyLk7V3fKyp0sl1yWQo= -github.com/mwitkow/go-proto-validators v0.1.0 h1:2Org0/cGKUUUDzoLSRSsGJDqyLWrb5lG57o5+QdRr8M= -github.com/mwitkow/go-proto-validators v0.1.0/go.mod h1:bA3eoTMLQkf/A7h7JOC3ddMBLXwS19KK7DEeSPL1O+4= +github.com/mwitkow/go-proto-validators v0.2.0 h1:F6LFfmgVnfULfaRsQWBbe7F7ocuHCr9+7m+GAeDzNbQ= +github.com/mwitkow/go-proto-validators v0.2.0/go.mod h1:ZfA1hW+UH/2ZHOWvQ3HnQaU0DtnpXu850MZiy+YUgcc= github.com/nbutton23/zxcvbn-go v0.0.0-20160627004424-a22cb81b2ecd/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU= github.com/nbutton23/zxcvbn-go v0.0.0-20171102151520-eafdab6b0663/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU= github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d h1:AREM5mwr4u1ORQBMvzfzBgpsctsbQikCVpvC+tX285E= @@ -365,14 +443,23 @@ github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d/go.mod h1:o96d github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.1 h1:b3iUnf1v+ppJiOfNX4yxxqfWKMQPZR5yoh8urCTFX88= github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/olekukonko/tablewriter v0.0.2 h1:sq53g+DWf0J6/ceFUHpQ0nAEb6WgM++fq16MZ91cS6o= +github.com/olekukonko/tablewriter v0.0.2/go.mod h1:rSAaSIOAGT9odnlyGlUfAJaoc5w2fSBUmeGDbRWPxyQ= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.3 h1:OoxbjfXVZyod1fmWYhI7SEyaD8B00ynP3T+D5GiyHOY= +github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.1 h1:K0jcRCwNQM3vFGh1ppMtDh/+7ApJrjldlX8fA0jDTLQ= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/otiai10/copy v1.0.1/go.mod h1:8bMCJrAqOtN/d9oyh5HR7HhLQMvcGMpGdwRDYsfOCHc= github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= github.com/otiai10/curr v0.0.0-20190513014714-f5a3d24e5776/go.mod h1:3HNVkVOU7vZeFXocWuvtcS0XSFLcf2XUSDHkq9t1jU4= @@ -384,6 +471,8 @@ github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.4.0 h1:u3Z1r+oOXJIkxqw34zVhyPgjBsm6X2wn21NWs/HfSeg= github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= +github.com/pelletier/go-toml v1.6.0 h1:aetoXYr0Tv7xRU/V4B4IZJ2QcbtMUFoNb3ORp7TzIK4= +github.com/pelletier/go-toml v1.6.0/go.mod h1:5N711Q9dKgbdkxHL+MEfF31hpT7l0S0s/t2kKREewys= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -396,8 +485,8 @@ github.com/prometheus/client_golang v0.9.4 h1:Y8E/JaaPbmFSW2V81Ab/d8yZFYQQGbni1b github.com/prometheus/client_golang v0.9.4/go.mod h1:oCXIBxdI62A4cR6aTRJCgetEjecSIYzOEaeAn4iYEpM= github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.1.0 h1:BQ53HtBmfOitExawJ6LokA4x8ov/z0SYYb0+HxJfRI8= -github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= +github.com/prometheus/client_golang v1.2.1 h1:JnMpQc6ppsNgw9QPAGF6Dod479itz7lvlsMzzNayLOI= +github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -410,6 +499,8 @@ github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.6.0 h1:kRhiuYSXR3+uv2IbVbZhUxK5zVD/2pp3Gd2PpvPkpEo= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= +github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY= +github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 h1:sofwID9zm4tzrgykg80hfFph1mryUeLRsUfoocVVmRY= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= @@ -417,8 +508,9 @@ github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNG github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.3 h1:CTwfnzjQ+8dS6MhHHu4YswVAD99sL2wjPqP+VkURmKE= github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= -github.com/prometheus/procfs v0.0.4 h1:w8DjqFMJDjuVwdZBQoOozr4MVWOnwF7RcL/7uxBjY78= -github.com/prometheus/procfs v0.0.4/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= +github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= +github.com/prometheus/procfs v0.0.7 h1:RS5GAlMbnkWkhs4+bPocMTmGjYkuCY5djjqEDdXOhcQ= +github.com/prometheus/procfs v0.0.7/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/pseudomuto/protoc-gen-doc v1.3.0 h1:wpwmaSCWY2lGwkzAxAaqYcGyoklZjZmeXrJ/X7IskJM= github.com/pseudomuto/protoc-gen-doc v1.3.0/go.mod h1:fwtQAY9erXp3mC92O8OTECnDlJT2r0Ff4KSEKbGEmy0= @@ -429,13 +521,21 @@ github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6So github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.2.1/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.3.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rs/cors v1.6.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday v2.0.0+incompatible h1:cBXrhZNUf9C+La9/YpS+UHpUT8YD6Td9ZMSU9APFcsk= github.com/russross/blackfriday v2.0.0+incompatible/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/go-glob v0.0.0-20170128012129-256dc444b735/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= +github.com/securego/gosec v0.0.0-20191002120514-e680875ea14d h1:BzRvVq1EHuIjxpijCEKpAxzKUUMurOQ4sknehIATRh8= +github.com/securego/gosec v0.0.0-20191002120514-e680875ea14d/go.mod h1:w5+eXa0mYznDkHaMCXA4XYffjlH+cy1oyKbfzJXa2Do= +github.com/securego/gosec v0.0.0-20191119104125-df484bfa9e9f h1:1egnuKwFhkqg8hXU5huGkxz9iPFu4dLINWRSIPjAM+M= +github.com/securego/gosec v0.0.0-20191119104125-df484bfa9e9f/go.mod h1:H5UrtKXL5BGF4FgRa7p2fyqU/lddaTSCLjexYIfS0Bk= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shirou/gopsutil v0.0.0-20180427012116-c95755e4bcd7/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc= github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= github.com/shurcooL/go v0.0.0-20190330031554-6713ea532688 h1:p7kt1rmLjMUkAmuyY/GUZeCoOud+nq297UKCCgEqYmA= @@ -474,38 +574,59 @@ github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0 github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.0.2/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM= github.com/spf13/viper v1.3.2 h1:VUFqw5KcqRf7i70GOzW7N+Q7+gxVBkSSqiXB12+JQ4M= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/spf13/viper v1.5.0 h1:GpsTwfsQ27oS/Aha/6d1oD7tpKIqWnOA6tgOX9HHkt4= +github.com/spf13/viper v1.5.0/go.mod h1:AkYRkVJF8TkSG/xet6PzXX+l39KhhXa2pdqVSxnTcn4= github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/timakin/bodyclose v0.0.0-20190721030226-87058b9bfcec h1:AmoEvWAO3nDx1MEcMzPh+GzOOIA5Znpv6++c7bePPY0= github.com/timakin/bodyclose v0.0.0-20190721030226-87058b9bfcec/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk= +github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e h1:RumXZ56IrCj4CL+g1b9OL/oH0QnsF976bC8xQFYUD5Q= +github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce/go.mod h1:o8v6yHRoik09Xen7gje4m9ERNah1d1PPsVq1VEx9vE4= -github.com/uber/prototool v1.8.1 h1:wlba8JLLuyTn0a7W6AqLrecGypFkrGOb+TwciSrW/rg= -github.com/uber/prototool v1.8.1/go.mod h1:E+xJFE/vf87XeqHbKM4uGQMrJ7NyKQb0+G7NtqHBgYs= +github.com/uber/prototool v1.8.2-0.20190910022025-7df3b957ffe3 h1:MAmouJ3XiYdtio1FxhxW0FX1lUsVYQsB+GflGB2ltrM= +github.com/uber/prototool v1.8.2-0.20190910022025-7df3b957ffe3/go.mod h1:fQyN8ZUYd9QoSiV0T/Da6NUSZY4ARW+EfJaL/4V1DD8= +github.com/uber/prototool v1.9.0 h1:s5O7ZQUqmV8jfaB9J69HD0DlU4S0CL4ecZlemkhF3TQ= +github.com/uber/prototool v1.9.0/go.mod h1:srCJMbeTuTT/1xkYIuUc1iCKsjjn8RBw9xxQl0R4aZg= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ultraware/funlen v0.0.1 h1:UeC9tpM4wNWzUJfan8z9sFE4QCzjjzlCZmuJN+aOkH0= github.com/ultraware/funlen v0.0.1/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA= github.com/ultraware/funlen v0.0.2 h1:Av96YVBwwNSe4MLR7iI/BIa3VyI7/djnto/pK3Uxbdo= github.com/ultraware/funlen v0.0.2/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA= +github.com/ultraware/whitespace v0.0.4 h1:If7Va4cM03mpgrNH9k49/VOicWpGoG70XPBFFODYDsg= +github.com/ultraware/whitespace v0.0.4/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA= github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/urfave/cli v1.22.0 h1:8nz/RUUotroXnOpYzT/Fy3sBp+2XEbXaY641/s3nbFI= -github.com/urfave/cli v1.22.0/go.mod h1:b3D7uWrF2GilkNgYpgcg6J+JMUw7ehmNkE8sZdliGLc= +github.com/urfave/cli v1.22.1 h1:+mkCCcOFKPnCmVYVcURKps1Xe+3zP90gSYGNfRkjoIY= +github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/uudashr/gocognit v0.0.0-20190926065955-1655d0de0517 h1:ChMKTho2hWKpks/nD/FL2KqM1wuVt62oJeiE8+eFpGs= +github.com/uudashr/gocognit v0.0.0-20190926065955-1655d0de0517/go.mod h1:j44Ayx2KW4+oB6SWMv8KsmHzZrOInQav7D3cQMJ5JUM= +github.com/uudashr/gocognit v1.0.0 h1:NST9SQhYHpFiyKdXn8UAiogdB1xkt5RnA1/rR/oBcGw= +github.com/uudashr/gocognit v1.0.0/go.mod h1:j44Ayx2KW4+oB6SWMv8KsmHzZrOInQav7D3cQMJ5JUM= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s= github.com/valyala/quicktemplate v1.1.1/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4= +github.com/valyala/quicktemplate v1.2.0/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/xanzy/ssh-agent v0.2.0/go.mod h1:0NyE30eGUDliuLEHJgYte/zncp2zdTStcOnWhgSqHD8= github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= @@ -518,39 +639,65 @@ go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd v3.3.13+incompatible h1:jCejD5EMnlGxFvcGRyEV4VGlENZc7oPQX6o0t7n3xbw= go.etcd.io/etcd v3.3.13+incompatible/go.mod h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSFZfzsF9A6aI= -go.etcd.io/etcd v3.3.15+incompatible h1:0VpOVCF6EFnJptt8Jh0EWEHO4j2fepyV1fpu9xz/UoQ= -go.etcd.io/etcd v3.3.15+incompatible/go.mod h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSFZfzsF9A6aI= +go.etcd.io/etcd v3.3.17+incompatible h1:g8iRku1SID8QAW8cDlV0L/PkZlw63LSiYEHYHoE6j/s= +go.etcd.io/etcd v3.3.17+incompatible/go.mod h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSFZfzsF9A6aI= go.opencensus.io v0.21.0 h1:mU6zScU4U1YAFPHEHYk+3JC4SY7JxgkqS10ZOSyksNg= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0 h1:C9hSCOW830chIVkdja34wa6Ky+IzWllkUinR+BtRZd4= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2 h1:75k/FF0Q2YM8QYo07VPddOLBslDt1MZOdEslOHvmzAs= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.5.1 h1:rsqfU5vBkVknbhUGbAUwQKR2H4ItV8tjJ+6kJX4cxHM= +go.uber.org/atomic v1.5.1/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.2.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.4.0 h1:f3WCSC2KzAcBXGATIxAB1E2XuCpNU255wNKZ505qi3E= +go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.13.0 h1:nR6NoDBgAf67s68NhaXbsojM+2gxp3S1hWkHDl27pVU= +go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5 h1:58fnuSXlxZmFdJyvtTFVmVhcMLU6v5fEb/ok4wyqtNU= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190909091759-094676da4a83 h1:mgAKeshyNqWKdENOnQsg+8dRTwZFIwFaO3HNl52sweA= -golang.org/x/crypto v0.0.0-20190909091759-094676da4a83/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= +golang.org/x/crypto v0.0.0-20191119213627-4f8c1d86b1ba h1:9bFeDpN3gTqNanMVqNcoR/pJQuP5uroC3t1D7eXozTE= +golang.org/x/crypto v0.0.0-20191119213627-4f8c1d86b1ba/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136 h1:A1gGSx58LAGVHUUsOf7IiR0u8Xb6W51gRwfDBhkdcaw= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190409202823-959b441ac422 h1:QzoH/1pFpZguR8NrRHLcO6jKqfv2zpuSqZLgdm7ZmjI= golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/net v0.0.0-20170915142106-8351a756f30f/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180530234432-1e491301e022/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -572,8 +719,10 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7 h1:rTIdg5QFRR7XCaK4LCjBiPbx8j4DQRpdYMnGn/bJUEU= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190909003024-a7b16738d86b h1:XfVGCX+0T4WOStkaOsJRllbsiImhB2jgVBGc9L0lPGc= -golang.org/x/net v0.0.0-20190909003024-a7b16738d86b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191009170851-d66e71096ffb/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191119073136-fc4aabc6c914 h1:MlY3mEfbnWGmUi4rtHOtNnnnN4UJRGSyLPx+DXA5Sq4= +golang.org/x/net v0.0.0-20191119073136-fc4aabc6c914/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -585,6 +734,8 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20171026204733-164713f0dfce/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180903190138-2b024373dcd9/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -606,9 +757,11 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3 h1:4y9KwBHBgBNwDbtu44R5o1fdOCQUEXhbk/P4A9WmJq0= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190910064555-bbd175535a8b h1:3S2h5FadpNr0zUUCVZjlKIEYF+KaX/OBplTGo89CYHI= -golang.org/x/sys v0.0.0-20190910064555-bbd175535a8b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e h1:N7DeIrjYszNmSW409R3frPPwglRwMkXSBzwVbkOjLLA= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20170915090833-1cbadb444a80/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -618,8 +771,11 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20170915040203-e531a2a1c15f/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -639,12 +795,27 @@ golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190624190245-7f2218787638/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190719005602-e377ae9d6386/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190909030654-5b82db07426d h1:PhtdWYteEBebOX7KXm4qkIAVSUTHQ883/2hRB92r9lk= golang.org/x/tools v0.0.0-20190909030654-5b82db07426d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190910044552-dd2b5c81c578 h1:f0Gfd654rnnfXT1+BK1YHPTS1qQdKrPIaGQwWxNE44k= golang.org/x/tools v0.0.0-20190910044552-dd2b5c81c578/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911151314-feee8acb394c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190930201159-7c411dea38b0/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191010075000-0337d82405ff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191101200257-8dbcdeb83d3f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191105231337-689d0f08e67a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191120221951-8fd459516a27 h1:Hnjbvuhm7GOIJHtxOlsKcq7CEDvPLMHawSA1AzKVlxA= +golang.org/x/tools v0.0.0-20191120221951-8fd459516a27/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.5.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= @@ -652,12 +823,19 @@ google.golang.org/api v0.6.0 h1:2tJEkRfnZL5g1GeBUlITh/rqT5HG3sFcoVCUUxmgJ2g= google.golang.org/api v0.6.0/go.mod h1:btoxGiFvQNVUZQ8W08zLtrVS08CNpINPEfxXxgJL1Q4= google.golang.org/api v0.7.0 h1:9sdfJOzWlkqPltHAuzT2Cp+yrBeY1KRVYgms8soxMwM= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0 h1:uMf5uLi4eQMRrMKhCplNik4U4H8Z6C1br3zOtAa/aDE= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20170818010345-ee236bd376b0/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -668,8 +846,14 @@ google.golang.org/genproto v0.0.0-20190605220351-eb0b1bdb6ae6 h1:XRqWpmQ5ACYxWuY google.golang.org/genproto v0.0.0-20190605220351-eb0b1bdb6ae6/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190626174449-989357319d63/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190708153700-3bdd9d9f5532/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= -google.golang.org/genproto v0.0.0-20190905072037-92dd089d5514 h1:oFSK4421fpCKRrpzIpybyBVWyht05NegY9+L/3TLAZs= -google.golang.org/genproto v0.0.0-20190905072037-92dd089d5514/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191028173616-919d9bdd9fe6/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115221424-83cc0476cb11 h1:51D++eCgOHufw5VfDE9Uzqyyc+OyQIjb9hkYy9LN5Fk= +google.golang.org/genproto v0.0.0-20191115221424-83cc0476cb11/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -681,11 +865,16 @@ google.golang.org/grpc v1.22.0 h1:J0UbZOIrCAl+fpTOf8YLs4dJo8L/owV4LYVtAXQoPkw= google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0 h1:AzbTB6ux+okLTzP8Ru1Xs41C303zdcfEht7MQnYJt5A= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= +google.golang.org/grpc v1.25.1 h1:wdKvqQk7IttEw92GoRyKG2IDrUIpgpj6H6m81yfeMW0= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.28 h1:n1tBJnnK2r7g9OW2btFH91V92STTUevLXYFb8gy9EMk= gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= @@ -705,11 +894,16 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo= +gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a h1:LJwr7TCTghdatWv40WobzlKXc9c4s8oGa7QKJUtHhWA= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed h1:WX1yoOaKQfddO/mLzdV4wptyWgoH/6hwLs7QHTixo0I= mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc= mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b h1:DxJ5nJdkhDlLok9K6qO+5290kphDJbHOQO1DFFFTeBo= @@ -719,6 +913,8 @@ mvdan.cc/unparam v0.0.0-20190310220240-1b9ccfa71afe h1:Ekmnp+NcP2joadI9pbK4Bva87 mvdan.cc/unparam v0.0.0-20190310220240-1b9ccfa71afe/go.mod h1:BnhuWBAqxH3+J5bDybdxgw5ZfS+DsVd4iylsKQePN8o= mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f h1:Cq7MalBHYACRd6EesksG1Q8EoIAKOsiZviGKbOLIej4= mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f/go.mod h1:4G1h5nDURzA3bwVMZIVpwbkw+04kSxk3rAtzlimaUJw= +mvdan.cc/unparam v0.0.0-20191111180625-960b1ec0f2c2 h1:K7wru2CfJGumS5hkiguQ0Rb9ebKM2Jo8s5d4Jm9lFaM= +mvdan.cc/unparam v0.0.0-20191111180625-960b1ec0f2c2/go.mod h1:rCqoQrfAmpTX/h2APczwM7UymU/uvaOluiVPIYCSY/k= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= diff --git a/impl/integration/env.go b/impl/integration/env.go index a55c8fa4a..7529de3b8 100644 --- a/impl/integration/env.go +++ b/impl/integration/env.go @@ -20,6 +20,7 @@ import ( "encoding/pem" "fmt" "net" + "testing" "time" "github.com/golang/glog" @@ -41,10 +42,10 @@ import ( "github.com/google/keytransparency/impl/authorization" "github.com/google/keytransparency/impl/sql/directory" "github.com/google/keytransparency/impl/sql/mutationstorage" + "github.com/google/keytransparency/impl/sql/testdb" "github.com/google/trillian/crypto/keys/der" "github.com/google/trillian/crypto/keyspb" "github.com/google/trillian/monitoring" - "github.com/google/trillian/storage/testdb" pb "github.com/google/keytransparency/core/api/v1/keytransparency_go_proto" spb "github.com/google/keytransparency/core/sequencer/sequencer_go_proto" @@ -53,7 +54,6 @@ import ( _ "github.com/google/trillian/merkle/coniks" // Register hasher _ "github.com/google/trillian/merkle/rfc6962" // Register hasher - _ "github.com/mattn/go-sqlite3" // Use sqlite database for testing. ) var ( @@ -116,19 +116,17 @@ func keyFromPEM(p string) *any.Any { } // NewEnv sets up common resources for tests. -func NewEnv(ctx context.Context) (*Env, error) { +func NewEnv(ctx context.Context, t testing.TB) *Env { + t.Helper() timeout := 6 * time.Second directoryID := "integration" - db, dbDone, err := testdb.NewTrillianDB(ctx) - if err != nil { - return nil, fmt.Errorf("env: failed to open database: %v", err) - } + db, dbDone := testdb.NewForTest(ctx, t) // Map server mapEnv, err := ttest.NewMapEnv(ctx, false) if err != nil { - return nil, fmt.Errorf("env: failed to create trillian map server: %v", err) + t.Fatalf("env: failed to create trillian map server: %v", err) } // Log server @@ -136,17 +134,17 @@ func NewEnv(ctx context.Context) (*Env, error) { unused := "" logEnv, err := ttest.NewLogEnv(ctx, numSequencers, unused) if err != nil { - return nil, fmt.Errorf("env: failed to create trillian log server: %v", err) + t.Fatalf("env: failed to create trillian log server: %v", err) } // Configure directory, which creates new map and log trees. directoryStorage, err := directory.NewStorage(db) if err != nil { - return nil, fmt.Errorf("env: failed to create directory storage: %v", err) + t.Fatalf("env: failed to create directory storage: %v", err) } mutations, err := mutationstorage.New(db) if err != nil { - return nil, fmt.Errorf("env: Failed to create mutations object: %v", err) + t.Fatalf("env: Failed to create mutations object: %v", err) } adminSvr := adminserver.New(logEnv.Log, mapEnv.Map, logEnv.Admin, mapEnv.Admin, directoryStorage, mutations, mutations, vrfKeyGen) cctx, cancel := context.WithTimeout(ctx, timeout) @@ -160,7 +158,7 @@ func NewEnv(ctx context.Context) (*Env, error) { MapPrivateKey: keyFromPEM(mapPriv), }) if err != nil { - return nil, fmt.Errorf("env: CreateDirectory(): %v", err) + t.Fatalf("env: CreateDirectory(): %v", err) } glog.V(5).Infof("Directory: %# v", pretty.Formatter(directoryPB)) @@ -169,7 +167,7 @@ func NewEnv(ctx context.Context) (*Env, error) { lis, cc, err := Listen() if err != nil { - return nil, fmt.Errorf("env: Listen(): %v", err) + t.Fatalf("env: Listen(): %v", err) } gsvr := grpc.NewServer( @@ -206,7 +204,7 @@ func NewEnv(ctx context.Context) (*Env, error) { func(lv *tclient.LogVerifier) verifier.LogTracker { return tracker.NewSynchronous(lv) }, ) if err != nil { - return nil, fmt.Errorf("error reading config: %v", err) + t.Fatalf("error reading config: %v", err) } // Integration tests manually create revisions immediately, so retry fairly quickly. client.RetryDelay = 10 * time.Millisecond @@ -228,7 +226,7 @@ func NewEnv(ctx context.Context) (*Env, error) { grpcCC: cc, dbDone: dbDone, db: db, - }, nil + } } // Close releases resources allocated by NewEnv. diff --git a/impl/integration/main_test.go b/impl/integration/main_test.go index 23e2ad60f..831266c0f 100644 --- a/impl/integration/main_test.go +++ b/impl/integration/main_test.go @@ -36,11 +36,7 @@ func TestIntegration(t *testing.T) { t.Run(test.Name, func(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - env, err := NewEnv(ctx) - if err != nil { - t.Fatalf("Could not create Env: %v", err) - } - + env := NewEnv(ctx, t) defer env.Close() cctx, cancel := context.WithCancel(ctx) actions := test.Fn(cctx, env.Env, t) diff --git a/impl/memory/mutation_logs.go b/impl/memory/mutation_logs.go new file mode 100644 index 000000000..c652f0463 --- /dev/null +++ b/impl/memory/mutation_logs.go @@ -0,0 +1,126 @@ +// Copyright 2019 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package memory supplies fake in-memory implementations for testing purposes. +package memory + +import ( + "context" + "fmt" + "sort" + "time" + + "github.com/google/keytransparency/core/mutator" + + pb "github.com/google/keytransparency/core/api/v1/keytransparency_go_proto" +) + +var clock = time.Unix(10, 0) // Start our clock at an arbitrary, non-zero place. + +// NewMutationLogs creates a new fake MutationLogs. +func NewMutationLogs() MutationLogs { + return make(MutationLogs) +} + +type batch struct { + time time.Time + msgs []*mutator.LogMessage +} + +// MutationLogs is a fake, in-memory implementation of a keyserver.MutationLogss. +// All requests are presumed to be for the same domainID. +// TODO(gbelvin): Support multiple domainIDs, if tests call for it. +// MutationLogs is NOT threadsafe. +type MutationLogs map[int64][]batch // Map of logID to Slice of LogMessages + +// AddLogs adds logIDs to the mutation database. +func (m MutationLogs) AddLogs(_ context.Context, _ string, logIDs ...int64) error { + for _, logID := range logIDs { + m[logID] = make([]batch, 0) + } + return nil +} + +// ListLogs returns a sorted list of logIDs. +func (m MutationLogs) ListLogs(_ context.Context, _ string, writable bool) ([]int64, error) { + logIDs := []int64{} + for logID := range m { + logIDs = append(logIDs, logID) + } + sort.Slice(logIDs, func(a, b int) bool { return logIDs[a] < logIDs[b] }) + return logIDs, nil +} + +// Send stores a batch of mutations in a given logID. +func (m MutationLogs) Send(_ context.Context, _ string, logID int64, mutation ...*pb.EntryUpdate) (time.Time, error) { + clock = clock.Add(time.Second) + ts := clock + // Only save the Merkle tree bits. + entries := make([]*pb.SignedEntry, 0, len(mutation)) + for _, i := range mutation { + entries = append(entries, i.Mutation) + } + + logShard := m[logID] + if len(logShard) > 0 && logShard[len(logShard)-1].time.After(ts) { + return time.Time{}, fmt.Errorf("inserting mutation entry %v out of order", ts) + } + + // Convert []SignedEntry into []LogMessage for storage. + msgs := make([]*mutator.LogMessage, 0, len(entries)) + for _, e := range entries { + msgs = append(msgs, &mutator.LogMessage{ID: ts, Mutation: e}) + } + m[logID] = append(logShard, batch{time: ts, msgs: msgs}) + return ts, nil +} + +// ReadLog returns mutations between [low, high). Always returns complete batches. +// ReadLog will return more items than batchSize if necessary to return a complete batch. +func (m MutationLogs) ReadLog(_ context.Context, _ string, + logID int64, low, high time.Time, batchSize int32) ([]*mutator.LogMessage, error) { + logShard := m[logID] + if len(logShard) == 0 || batchSize == 0 { + return nil, nil + } + start := sort.Search(len(logShard), func(i int) bool { return !logShard[i].time.Before(low) }) + end := sort.Search(len(logShard), func(i int) bool { return !logShard[i].time.Before(high) }) + // If the search is unsuccessful, i will be equal to len(logShard). + if start == len(logShard) && logShard[start].time.Before(low) { + return nil, fmt.Errorf("invalid argument: low: %v, want <= max watermark: %v", low, logShard[start].time) + } + out := make([]*mutator.LogMessage, 0, batchSize) + for i := start; i < end; i++ { + out = append(out, logShard[i].msgs...) + if int32(len(out)) >= batchSize { + break + } + } + return out, nil +} + +// HighWatermark returns the highest timestamp batchSize items beyond start. +func (m MutationLogs) HighWatermark(_ context.Context, _ string, logID int64, start time.Time, + batchSize int32) (int32, time.Time, error) { + logShard := m[logID] + i := sort.Search(len(logShard), func(i int) bool { return !logShard[i].time.Before(start) }) + + count := int32(0) + high := start // Preserve start time if there are no rows to process. + for ; i < len(logShard) && count < batchSize; i++ { + high = logShard[i].time.Add(time.Nanosecond) // Returns exclusive n + 1 + count += int32(len(logShard[i].msgs)) + } + return count, high, nil +} diff --git a/impl/memory/mutation_logs_test.go b/impl/memory/mutation_logs_test.go new file mode 100644 index 000000000..c5ba1ccec --- /dev/null +++ b/impl/memory/mutation_logs_test.go @@ -0,0 +1,46 @@ +// Copyright 2019 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package memory + +import ( + "context" + "testing" + + "github.com/google/keytransparency/core/integration/storagetest" + "github.com/google/keytransparency/core/keyserver" +) + +// Tests for the tests! +func TestMutationLogsIntegration(t *testing.T) { + storagetest.RunMutationLogsTests(t, + func(ctx context.Context, t *testing.T, dirID string, logIDs ...int64) (keyserver.MutationLogs, func(context.Context)) { + m := NewMutationLogs() + if err := m.AddLogs(ctx, dirID, logIDs...); err != nil { + t.Fatal(err) + } + return m, func(context.Context) {} + }) +} + +func TestMutationLogsReaderIntegration(t *testing.T) { + storagetest.RunMutationLogsReaderTests(t, + func(ctx context.Context, t *testing.T, dirID string, logIDs ...int64) (storagetest.LogsReadWriter, func(context.Context)) { + m := NewMutationLogs() + if err := m.AddLogs(ctx, dirID, logIDs...); err != nil { + t.Fatal(err) + } + return m, func(context.Context) {} + }) +} diff --git a/impl/sql/directory/storage_test.go b/impl/sql/directory/storage_test.go index 4fc6baf5c..f2d4d5c43 100644 --- a/impl/sql/directory/storage_test.go +++ b/impl/sql/directory/storage_test.go @@ -16,40 +16,35 @@ package directory import ( "context" - "database/sql" "testing" "time" "github.com/golang/protobuf/proto" "github.com/google/go-cmp/cmp" "github.com/google/keytransparency/core/directory" - tpb "github.com/google/trillian" + "github.com/google/keytransparency/impl/sql/testdb" "github.com/google/trillian/crypto/keyspb" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - _ "github.com/mattn/go-sqlite3" + tpb "github.com/google/trillian" ) -func newStorage(t *testing.T) (s directory.Storage, close func()) { +func newStorage(ctx context.Context, t *testing.T) (directory.Storage, func(context.Context)) { t.Helper() - db, err := sql.Open("sqlite3", ":memory:") - if err != nil { - t.Fatalf("sql.Open(): %v", err) - } - closeFunc := func() { db.Close() } - s, err = NewStorage(db) + db, done := testdb.NewForTest(ctx, t) + s, err := NewStorage(db) if err != nil { - closeFunc() + done(ctx) t.Fatalf("Failed to create adminstorage: %v", err) } - return s, closeFunc + return s, done } func TestList(t *testing.T) { ctx := context.Background() - s, closeF := newStorage(t) - defer closeF() + s, done := newStorage(ctx, t) + defer done(ctx) for _, tc := range []struct { directories []*directory.Directory readDeleted bool @@ -105,8 +100,8 @@ func TestList(t *testing.T) { func TestWriteReadDelete(t *testing.T) { ctx := context.Background() - s, closeF := newStorage(t) - defer closeF() + s, done := newStorage(ctx, t) + defer done(ctx) for _, tc := range []struct { desc string d directory.Directory @@ -249,8 +244,8 @@ func TestWriteReadDelete(t *testing.T) { func TestDelete(t *testing.T) { ctx := context.Background() - s, closeF := newStorage(t) - defer closeF() + s, done := newStorage(ctx, t) + defer done(ctx) for _, tc := range []struct { directoryID string }{ diff --git a/impl/sql/mutationstorage/queue.go b/impl/sql/mutationstorage/mutation_logs.go similarity index 83% rename from impl/sql/mutationstorage/queue.go rename to impl/sql/mutationstorage/mutation_logs.go index 520d407f5..fa5f61d7c 100644 --- a/impl/sql/mutationstorage/queue.go +++ b/impl/sql/mutationstorage/mutation_logs.go @@ -17,12 +17,10 @@ package mutationstorage import ( "context" "database/sql" - "math/rand" "time" "github.com/golang/glog" "github.com/golang/protobuf/proto" - "github.com/google/keytransparency/core/keyserver" "github.com/google/keytransparency/core/mutator" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -30,9 +28,12 @@ import ( pb "github.com/google/keytransparency/core/api/v1/keytransparency_go_proto" ) +// quantum is the fidelity of the Timestamp column. +// See https://dev.mysql.com/doc/refman/8.0/en/fractional-seconds.html +const quantum = time.Microsecond + // SetWritable enables or disables new writes from going to logID. func (m *Mutations) SetWritable(ctx context.Context, directoryID string, logID int64, enabled bool) error { - glog.Errorf("mutationstorage: SetWritable(%v, %v, enabled: %v)", directoryID, logID, enabled) result, err := m.db.ExecContext(ctx, `UPDATE Logs SET Enabled = ? WHERE DirectoryID = ? AND LogID = ?;`, enabled, directoryID, logID) @@ -68,30 +69,26 @@ func (m *Mutations) AddLogs(ctx context.Context, directoryID string, logIDs ...i // Send writes mutations to the leading edge (by sequence number) of the mutations table. // Returns the logID/watermark pair that was written, or nil if nothing was written. // TODO(gbelvin): Make updates a slice. -func (m *Mutations) Send(ctx context.Context, directoryID string, updates ...*pb.EntryUpdate) (*keyserver.WriteWatermark, error) { +func (m *Mutations) Send(ctx context.Context, directoryID string, logID int64, updates ...*pb.EntryUpdate) (time.Time, error) { glog.Infof("mutationstorage: Send(%v, )", directoryID) if len(updates) == 0 { - return nil, nil - } - logID, err := m.randLog(ctx, directoryID) - if err != nil { - return nil, err + return time.Time{}, nil } updateData := make([][]byte, 0, len(updates)) for _, u := range updates { data, err := proto.Marshal(u) if err != nil { - return nil, err + return time.Time{}, err } updateData = append(updateData, data) } // TODO(gbelvin): Implement retry with backoff for retryable errors if // we get timestamp contention. - ts := time.Now() + ts := time.Now().Truncate(quantum) if err := m.send(ctx, ts, directoryID, logID, updateData...); err != nil { - return nil, err + return time.Time{}, err } - return &keyserver.WriteWatermark{LogID: logID, Watermark: ts.UnixNano()}, nil + return ts, nil } // ListLogs returns a list of all logs for directoryID, optionally filtered for writable logs. @@ -125,19 +122,6 @@ func (m *Mutations) ListLogs(ctx context.Context, directoryID string, writable b return logIDs, nil } -// randLog returns a random, enabled log for directoryID. -func (m *Mutations) randLog(ctx context.Context, directoryID string) (int64, error) { - // TODO(gbelvin): Cache these results. - writable := true - logIDs, err := m.ListLogs(ctx, directoryID, writable) - if err != nil { - return 0, err - } - - // Return a random log. - return logIDs[rand.Intn(len(logIDs))], nil -} - // ts must be greater than all other timestamps currently recorded for directoryID. func (m *Mutations) send(ctx context.Context, ts time.Time, directoryID string, logID int64, mData ...[]byte) (ret error) { @@ -154,23 +138,22 @@ func (m *Mutations) send(ctx context.Context, ts time.Time, directoryID string, } }() - var maxTime int64 + var maxTime sql.NullTime if err := tx.QueryRowContext(ctx, - `SELECT COALESCE(MAX(Time), 0) FROM Queue WHERE DirectoryID = ? AND LogID = ?;`, + `SELECT MAX(Time) FROM Queue WHERE DirectoryID = ? AND LogID = ?;`, directoryID, logID).Scan(&maxTime); err != nil { return status.Errorf(codes.Internal, "could not find max timestamp: %v", err) } - tsTime := ts.UnixNano() - if tsTime <= maxTime { + + if !ts.After(maxTime.Time) { return status.Errorf(codes.Aborted, - "current timestamp: %v, want > max-timestamp of queued mutations: %v", - tsTime, maxTime) + "current timestamp: %v, want > max-timestamp of queued mutations: %v", ts, maxTime) } for i, data := range mData { if _, err = tx.ExecContext(ctx, `INSERT INTO Queue (DirectoryID, LogID, Time, LocalID, Mutation) VALUES (?, ?, ?, ?, ?);`, - directoryID, logID, tsTime, i, data); err != nil { + directoryID, logID, ts, i, data); err != nil { return status.Errorf(codes.Internal, "failed inserting into queue: %v", err) } } @@ -179,29 +162,38 @@ func (m *Mutations) send(ctx context.Context, ts time.Time, directoryID string, // HighWatermark returns the highest watermark +1 in logID that is less than or // equal to batchSize items greater than start. -func (m *Mutations) HighWatermark(ctx context.Context, directoryID string, logID, - start int64, batchSize int32) (int32, int64, error) { +func (m *Mutations) HighWatermark(ctx context.Context, directoryID string, logID int64, + start time.Time, batchSize int32) (int32, time.Time, error) { + startQuery := start.Add(quantum - 1).Truncate(quantum) var count int32 - var high int64 + var high sql.NullTime if err := m.db.QueryRowContext(ctx, - `SELECT COUNT(*), COALESCE(MAX(T1.Time)+1, ?) FROM + `SELECT COUNT(*), MAX(T1.Time) FROM ( SELECT Q.Time FROM Queue as Q WHERE Q.DirectoryID = ? AND Q.LogID = ? AND Q.Time >= ? ORDER BY Q.Time ASC LIMIT ? ) AS T1`, - start, directoryID, logID, start, batchSize). + directoryID, logID, startQuery, batchSize). Scan(&count, &high); err != nil { - return 0, 0, err + return 0, start, err + } + if count == 0 { + // When there are no rows, return the start time as the highest timestamp. + return 0, start, nil } - return count, high, nil + return count, high.Time.Add(1 * time.Microsecond), nil } // ReadLog reads all mutations in logID between [low, high). // ReadLog may return more rows than batchSize in order to fetch all the rows at a particular timestamp. func (m *Mutations) ReadLog(ctx context.Context, directoryID string, - logID, low, high int64, batchSize int32) ([]*mutator.LogMessage, error) { + logID int64, low, high time.Time, batchSize int32) ([]*mutator.LogMessage, error) { + // Advance the low and high marks to the next highest quantum to preserve read semantics. + low = low.Add(quantum - 1).Truncate(quantum) + high = high.Add(quantum - 1).Truncate(quantum) + rows, err := m.db.QueryContext(ctx, `SELECT Time, LocalID, Mutation FROM Queue WHERE DirectoryID = ? AND LogID = ? AND Time >= ? AND Time < ? @@ -242,7 +234,8 @@ func (m *Mutations) ReadLog(ctx context.Context, directoryID string, func readQueueMessages(rows *sql.Rows) ([]*mutator.LogMessage, error) { results := make([]*mutator.LogMessage, 0) for rows.Next() { - var timestamp, localID int64 + var timestamp time.Time + var localID int64 var mData []byte if err := rows.Scan(×tamp, &localID, &mData); err != nil { return nil, err diff --git a/impl/sql/mutationstorage/mutation_logs_test.go b/impl/sql/mutationstorage/mutation_logs_test.go index 1727af74c..9cda05d5c 100644 --- a/impl/sql/mutationstorage/mutation_logs_test.go +++ b/impl/sql/mutationstorage/mutation_logs_test.go @@ -20,85 +20,55 @@ import ( "testing" "time" - "github.com/google/go-cmp/cmp" "github.com/google/keytransparency/core/adminserver" "github.com/google/keytransparency/core/integration/storagetest" "github.com/google/keytransparency/core/keyserver" + "github.com/google/keytransparency/impl/sql/testdb" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" pb "github.com/google/keytransparency/core/api/v1/keytransparency_go_proto" - _ "github.com/mattn/go-sqlite3" ) -func newForTest(ctx context.Context, t testing.TB, dirID string, logIDs ...int64) *Mutations { - m, err := New(newDB(t)) +func newForTest(ctx context.Context, t testing.TB, dirID string, logIDs ...int64) (*Mutations, func(context.Context)) { + db, done := testdb.NewForTest(ctx, t) + m, err := New(db) if err != nil { t.Fatalf("Failed to create mutation storage: %v", err) } if err := m.AddLogs(ctx, dirID, logIDs...); err != nil { t.Fatalf("AddLogs(): %v", err) } - return m + return m, done } func TestMutationLogsIntegration(t *testing.T) { storagetest.RunMutationLogsTests(t, - func(ctx context.Context, t *testing.T, dirID string, logIDs ...int64) keyserver.MutationLogs { + func(ctx context.Context, t *testing.T, dirID string, logIDs ...int64) (keyserver.MutationLogs, func(context.Context)) { return newForTest(ctx, t, dirID, logIDs...) }) } func TestLogsAdminIntegration(t *testing.T) { storagetest.RunLogsAdminTests(t, - func(ctx context.Context, t *testing.T, dirID string, logIDs ...int64) adminserver.LogsAdmin { + func(ctx context.Context, t *testing.T, dirID string, logIDs ...int64) (adminserver.LogsAdmin, func(context.Context)) { return newForTest(ctx, t, dirID, logIDs...) }) } -func TestRandLog(t *testing.T) { - ctx := context.Background() - directoryID := "TestRandLog" - - for _, tc := range []struct { - desc string - send []int64 - wantCode codes.Code - wantLogs map[int64]bool - }{ - {desc: "no rows", wantCode: codes.NotFound, wantLogs: map[int64]bool{}}, - {desc: "one row", send: []int64{10}, wantLogs: map[int64]bool{10: true}}, - {desc: "second", send: []int64{1, 2, 3}, wantLogs: map[int64]bool{ - 1: true, - 2: true, - 3: true, - }}, - } { - t.Run(tc.desc, func(t *testing.T) { - m := newForTest(ctx, t, directoryID, tc.send...) - logs := make(map[int64]bool) - for i := 0; i < 10*len(tc.wantLogs); i++ { - logID, err := m.randLog(ctx, directoryID) - if got, want := status.Code(err), tc.wantCode; got != want { - t.Errorf("randLog(): %v, want %v", got, want) - } - if err != nil { - break - } - logs[logID] = true - } - if got, want := logs, tc.wantLogs; !cmp.Equal(got, want) { - t.Errorf("logs: %v, want %v", got, want) - } +func TestMutationLogsReaderIntegration(t *testing.T) { + storagetest.RunMutationLogsReaderTests(t, + func(ctx context.Context, t *testing.T, dirID string, logIDs ...int64) (storagetest.LogsReadWriter, func(context.Context)) { + return newForTest(ctx, t, dirID, logIDs...) }) - } } func BenchmarkSend(b *testing.B) { ctx := context.Background() directoryID := "BenchmarkSend" logID := int64(1) - m := newForTest(ctx, b, directoryID, logID) + m, done := newForTest(ctx, b, directoryID, logID) + defer done(ctx) update := &pb.EntryUpdate{Mutation: &pb.SignedEntry{Entry: []byte("xxxxxxxxxxxxxxxxxx")}} for _, tc := range []struct { @@ -120,7 +90,7 @@ func BenchmarkSend(b *testing.B) { updates = append(updates, update) } for n := 0; n < b.N; n++ { - if _, err := m.Send(ctx, directoryID, updates...); err != nil { + if _, err := m.Send(ctx, directoryID, logID, updates...); err != nil { b.Errorf("Send(): %v", err) } } @@ -132,11 +102,12 @@ func TestSend(t *testing.T) { ctx := context.Background() directoryID := "TestSend" - m := newForTest(ctx, t, directoryID, 1, 2) + m, done := newForTest(ctx, t, directoryID, 1, 2) + defer done(ctx) update := []byte("bar") - ts1 := time.Now() - ts2 := ts1.Add(time.Duration(1)) - ts3 := ts2.Add(time.Duration(1)) + ts1 := time.Now().Truncate(time.Microsecond) + ts2 := ts1.Add(1 * time.Microsecond) + ts3 := ts2.Add(1 * time.Microsecond) // Test cases are cumulative. Earlier test caes setup later test cases. for _, tc := range []struct { @@ -157,51 +128,3 @@ func TestSend(t *testing.T) { } } } - -func TestWatermark(t *testing.T) { - ctx := context.Background() - directoryID := "TestWatermark" - logIDs := []int64{1, 2} - m := newForTest(ctx, t, directoryID, logIDs...) - update := []byte("bar") - - startTS := time.Now() - for ts := startTS; ts.Before(startTS.Add(10)); ts = ts.Add(1) { - for _, logID := range logIDs { - if err := m.send(ctx, ts, directoryID, logID, update); err != nil { - t.Fatalf("m.send(%v): %v", logID, err) - } - } - } - - start := startTS.UnixNano() - for _, tc := range []struct { - desc string - logID int64 - start int64 - batchSize int32 - count int32 - want int64 - }{ - {desc: "log1 max", logID: 1, batchSize: 100, want: start + 10, count: 10}, - {desc: "log2 max", logID: 2, batchSize: 100, want: start + 10, count: 10}, - {desc: "batch0", logID: 1, batchSize: 0}, - {desc: "batch0start55", logID: 1, start: 55, batchSize: 0, want: 55}, - {desc: "batch5", logID: 1, batchSize: 5, want: start + 5, count: 5}, - {desc: "start1", logID: 1, start: start + 2, batchSize: 5, want: start + 7, count: 5}, - {desc: "start8", logID: 1, start: start + 8, batchSize: 5, want: start + 10, count: 2}, - } { - t.Run(tc.desc, func(t *testing.T) { - count, got, err := m.HighWatermark(ctx, directoryID, tc.logID, tc.start, tc.batchSize) - if err != nil { - t.Errorf("highWatermark(): %v", err) - } - if got != tc.want { - t.Errorf("highWatermark(%v) high: %v, want %v", tc.start, got, tc.want) - } - if count != tc.count { - t.Errorf("highWatermark(%v) count: %v, want %v", tc.start, count, tc.count) - } - }) - } -} diff --git a/impl/sql/mutationstorage/mutations.go b/impl/sql/mutationstorage/mutations.go index e10adb732..f244bea6f 100644 --- a/impl/sql/mutationstorage/mutations.go +++ b/impl/sql/mutationstorage/mutations.go @@ -39,7 +39,7 @@ var ( `CREATE TABLE IF NOT EXISTS Queue ( DirectoryID VARCHAR(30) NOT NULL, LogID BIGINT NOT NULL, - Time BIGINT NOT NULL, + Time DATETIME(6) NOT NULL, LocalID BIGINT NOT NULL, Mutation BLOB NOT NULL, PRIMARY KEY(DirectoryID, LogID, Time, LocalID ) diff --git a/impl/sql/mutationstorage/mutations_test.go b/impl/sql/mutationstorage/mutations_test.go index fdd486f82..2e95ff3a7 100644 --- a/impl/sql/mutationstorage/mutations_test.go +++ b/impl/sql/mutationstorage/mutations_test.go @@ -15,29 +15,21 @@ package mutationstorage import ( "context" - "database/sql" "testing" "github.com/google/keytransparency/core/integration/storagetest" - - _ "github.com/mattn/go-sqlite3" + "github.com/google/keytransparency/core/sequencer" + "github.com/google/keytransparency/impl/sql/testdb" ) -func newDB(t testing.TB) *sql.DB { - db, err := sql.Open("sqlite3", ":memory:") - if err != nil { - t.Fatalf("sql.Open(): %v", err) - } - return db -} - func TestBatchIntegration(t *testing.T) { - storageFactory := func(context.Context, *testing.T, string) storagetest.Batcher { - m, err := New(newDB(t)) + storageFactory := func(ctx context.Context, t *testing.T, _ string) (sequencer.Batcher, func(context.Context)) { + db, done := testdb.NewForTest(ctx, t) + m, err := New(db) if err != nil { t.Fatalf("Failed to create mutations: %v", err) } - return m + return m, done } storagetest.RunBatchStorageTests(t, storageFactory) diff --git a/impl/sql/open.go b/impl/sql/open.go new file mode 100644 index 000000000..fe7d790a6 --- /dev/null +++ b/impl/sql/open.go @@ -0,0 +1,42 @@ +// Copyright 2019 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package sql provides functions for interacting with MySQL. +package sql + +import ( + "database/sql" + "time" + + "github.com/go-sql-driver/mysql" +) + +// Open the MySQL database specified by the dsn string. +func Open(dsn string) (*sql.DB, error) { + cfg, err := mysql.ParseDSN(dsn) + if err != nil { + return nil, err + } + + // MySQL flags that affect storage logic. + cfg.ClientFoundRows = true // Return number of matching rows instead of rows changed. + cfg.ParseTime = true // Parse time values to time.Time + cfg.Loc = time.UTC + + db, err := sql.Open("mysql", cfg.FormatDSN()) + if err != nil { + return nil, err + } + return db, db.Ping() +} diff --git a/impl/sql/testdb/testdb.go b/impl/sql/testdb/testdb.go new file mode 100644 index 000000000..d5d46f24d --- /dev/null +++ b/impl/sql/testdb/testdb.go @@ -0,0 +1,60 @@ +// Copyright 2019 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package testdb supports opening ephemeral databases for testing. +package testdb + +import ( + "context" + "database/sql" + "flag" + "fmt" + "log" + "testing" + "time" + + ktsql "github.com/google/keytransparency/impl/sql" +) + +var dataSourceURI = flag.String("kt_test_mysql_uri", "root@tcp(127.0.0.1)/", "The MySQL URI to use when running tests") + +// NewForTest creates a temporary database. +// Returns a function for deleting the database. +func NewForTest(ctx context.Context, t testing.TB) (*sql.DB, func(context.Context)) { + t.Helper() + db, err := ktsql.Open(*dataSourceURI) + if err != nil { + t.Fatal(err) + } + + dbName := fmt.Sprintf("test_%v", time.Now().UnixNano()) + if _, err := db.ExecContext(ctx, fmt.Sprintf("CREATE DATABASE `%s`", dbName)); err != nil { + t.Fatalf("Failed to create test database: %v", err) + } + + // Open test database + db.Close() + db, err = ktsql.Open(*dataSourceURI + dbName) + if err != nil { + t.Fatal(err) + } + + done := func(ctx context.Context) { + defer db.Close() + if _, err := db.ExecContext(ctx, fmt.Sprintf("DROP DATABASE `%s`", dbName)); err != nil { + log.Printf("Failed to drop test database %q: %v", dbName, err) + } + } + return db, done +} diff --git a/impl/sql/testdb/testdb_test.go b/impl/sql/testdb/testdb_test.go new file mode 100644 index 000000000..60b57ef75 --- /dev/null +++ b/impl/sql/testdb/testdb_test.go @@ -0,0 +1,32 @@ +// Copyright 2019 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testdb + +import ( + "context" + "testing" +) + +func TestNewForTest(t *testing.T) { + ctx := context.Background() + db, done := NewForTest(ctx, t) + if err := db.Ping(); err != nil { + t.Error(err) + } + done(ctx) + // Verify that the databse was really closed by looking for the + // error in the logs when running the test with -v + done(ctx) +} diff --git a/scripts/deploy.sh b/scripts/deploy.sh index e4d2613f9..70b349e2d 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -36,13 +36,12 @@ gcloud --quiet auth configure-docker # Test current directory before deleting anything test $(basename $(pwd)) == "keytransparency" || exit 1 -echo "Generating keys..." -rm -f ./genfiles/* -./scripts/prepare_server.sh -f # kubectl exits with 1 if kt-secret does not exist -kubectl get secret kt-secrets -if [ $? -ne 0 ]; then - kubectl create secret generic kt-secrets --from-file=genfiles/server.crt --from-file=genfiles/server.key +if ! kubectl get secret kt-secrets; then + echo "Generating keys..." + rm -f ./genfiles/* + ./scripts/prepare_server.sh -f + kubectl create secret generic kt-secrets --from-file=genfiles/server.crt --from-file=genfiles/server.key --from-file=genfiles/monitor_sign-key.pem fi echo "Building docker images..."