From 716c08aba1c6aabb207779184a061cbfeaf4be99 Mon Sep 17 00:00:00 2001
From: Pavel Kalinnikov <pkalinnikov@google.com>
Date: Wed, 16 Oct 2019 20:56:50 +0100
Subject: [PATCH 01/22] client: Remove unused SetLeavesAtRevision (#1367)

KT only uses the WriteLeaves RPC, so SetLeaves can be removed.
---
 core/sequencer/trillian_client.go | 21 ---------------------
 1 file changed, 21 deletions(-)

diff --git a/core/sequencer/trillian_client.go b/core/sequencer/trillian_client.go
index adbcc878e..69a77b25f 100644
--- a/core/sequencer/trillian_client.go
+++ b/core/sequencer/trillian_client.go
@@ -127,27 +127,6 @@ type MapClient struct {
 	*tclient.MapClient
 }
 
-// SetLeavesAtRevision creates a new map revision and returns its verified root.
-// TODO(gbelvin): Move to Trillian Map client.
-func (c *MapClient) SetLeavesAtRevision(ctx context.Context, rev int64,
-	leaves []*tpb.MapLeaf, metadata []byte) (*types.MapRootV1, error) {
-	// Set new leaf values.
-	setResp, err := c.Conn.SetLeaves(ctx, &tpb.SetMapLeavesRequest{
-		MapId:    c.MapID,
-		Revision: rev,
-		Leaves:   leaves,
-		Metadata: metadata,
-	})
-	if err != nil {
-		return nil, err
-	}
-	mapRoot, err := c.VerifySignedMapRoot(setResp.GetMapRoot())
-	if err != nil {
-		return nil, status.Errorf(codes.Internal, "VerifySignedMapRoot(): %v", err)
-	}
-	return mapRoot, nil
-}
-
 // GetAndVerifyLatestMapRoot verifies and returns the latest map root.
 func (c *MapClient) GetAndVerifyLatestMapRoot(ctx context.Context) (*tpb.SignedMapRoot, *types.MapRootV1, error) {
 	rootResp, err := c.Conn.GetSignedMapRoot(ctx, &tpb.GetSignedMapRootRequest{MapId: c.MapID})

From 575344f256a6ca2e5fea3a9c3025aa69d57f63e3 Mon Sep 17 00:00:00 2001
From: Gary Belvin <gdb@google.com>
Date: Fri, 25 Oct 2019 14:19:01 +0100
Subject: [PATCH 02/22] Define ReadLog API in terms of timestamps (#1368)

* Define ReadLogs in terms of time.Time
* Use time.Time for return API
---
 core/integration/storagetest/mutation_logs.go | 18 +++----
 core/keyserver/keyserver.go                   |  7 +--
 core/keyserver/paginator.go                   | 27 +++++++---
 core/keyserver/paginator_test.go              | 16 +++---
 core/keyserver/readtoken.proto                |  7 ++-
 .../readtoken_go_proto/readtoken.pb.go        | 54 ++++++++++---------
 core/keyserver/revisions.go                   | 12 +++--
 core/keyserver/revisions_test.go              | 35 +++++++-----
 core/mutator/mutator.go                       |  4 +-
 core/sequencer/metadata/sourceslice.go        | 41 ++++++++++++++
 core/sequencer/server.go                      | 22 ++++----
 core/sequencer/server_test.go                 | 22 ++++----
 .../sql/mutationstorage/mutation_logs_test.go | 27 +++++-----
 impl/sql/mutationstorage/queue.go             | 21 ++++----
 14 files changed, 197 insertions(+), 116 deletions(-)
 create mode 100644 core/sequencer/metadata/sourceslice.go

diff --git a/core/integration/storagetest/mutation_logs.go b/core/integration/storagetest/mutation_logs.go
index 5fdbd1c99..f955892e4 100644
--- a/core/integration/storagetest/mutation_logs.go
+++ b/core/integration/storagetest/mutation_logs.go
@@ -65,20 +65,20 @@ func (mutationLogsTests) TestReadLog(ctx context.Context, t *testing.T, newForTe
 
 	for _, tc := range []struct {
 		limit int32
-		count int
+		want  int
 	}{
-		{limit: 0, count: 0},
-		{limit: 1, count: 3},    // We asked for 1 item, which gets us into the first batch, so we return 3 items.
-		{limit: 3, count: 3},    // We asked for 3 items, which gets us through the first batch, so we return 3 items.
-		{limit: 4, count: 6},    // Reading 4 items gets us into the second batch of size 3.
-		{limit: 100, count: 30}, // Reading all the items gets us the 30 items we wrote.
+		{limit: 0, want: 0},
+		{limit: 1, want: 3},    // We asked for 1 item, which gets us into the first batch, so we return 3 items.
+		{limit: 3, want: 3},    // We asked for 3 items, which gets us through the first batch, so we return 3 items.
+		{limit: 4, want: 6},    // Reading 4 items gets us into the second batch of size 3.
+		{limit: 100, want: 30}, // Reading all the items gets us the 30 items we wrote.
 	} {
-		rows, err := m.ReadLog(ctx, directoryID, logID, 0, time.Now().UnixNano(), tc.limit)
+		rows, err := m.ReadLog(ctx, directoryID, logID, time.Time{}, time.Now(), tc.limit)
 		if err != nil {
 			t.Fatalf("ReadLog(%v): %v", tc.limit, err)
 		}
-		if got, want := len(rows), tc.count; got != want {
-			t.Fatalf("ReadLog(%v): len: %v, want %v", tc.limit, got, want)
+		if got := len(rows); got != tc.want {
+			t.Fatalf("ReadLog(%v): len: %v, want %v", tc.limit, got, tc.want)
 		}
 	}
 }
diff --git a/core/keyserver/keyserver.go b/core/keyserver/keyserver.go
index e58f6ddc8..d4a5af9fd 100644
--- a/core/keyserver/keyserver.go
+++ b/core/keyserver/keyserver.go
@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"runtime"
 	"sync"
+	"time"
 
 	"github.com/golang/glog"
 	"github.com/golang/protobuf/proto"
@@ -67,7 +68,7 @@ func createMetrics(mf monitoring.MetricFactory) {
 // WriteWatermark is the metadata that Send creates.
 type WriteWatermark struct {
 	LogID     int64
-	Watermark int64
+	Watermark time.Time
 }
 
 // MutationLogs provides sets of time ordered message logs.
@@ -79,7 +80,7 @@ type MutationLogs interface {
 	// specified log. ReadLog always returns complete units of the original
 	// batches sent via Send, and will return  more items than limit if
 	// needed to do so.
-	ReadLog(ctx context.Context, directoryID string, logID, low, high int64,
+	ReadLog(ctx context.Context, directoryID string, logID int64, low, high time.Time,
 		limit int32) ([]*mutator.LogMessage, error)
 }
 
@@ -666,7 +667,7 @@ func (s *Server) BatchQueueUserUpdate(ctx context.Context, in *pb.BatchQueueUser
 		return nil, status.Errorf(st.Code(), "Mutation write error")
 	}
 	if wm != nil {
-		watermarkWritten.Set(float64(wm.Watermark), directory.DirectoryID, fmt.Sprintf("%v", wm.LogID))
+		watermarkWritten.Set(float64(wm.Watermark.UnixNano()), directory.DirectoryID, fmt.Sprintf("%v", wm.LogID))
 		sequencerQueueWritten.Add(float64(len(in.Updates)), directory.DirectoryID, fmt.Sprintf("%v", wm.LogID))
 	}
 
diff --git a/core/keyserver/paginator.go b/core/keyserver/paginator.go
index 8f200ff24..132f3996b 100644
--- a/core/keyserver/paginator.go
+++ b/core/keyserver/paginator.go
@@ -18,7 +18,9 @@ import (
 	"encoding/base64"
 
 	"github.com/golang/protobuf/proto"
+	"github.com/golang/protobuf/ptypes"
 	"github.com/google/keytransparency/core/mutator"
+	"github.com/google/keytransparency/core/sequencer/metadata"
 
 	rtpb "github.com/google/keytransparency/core/keyserver/readtoken_go_proto"
 	spb "github.com/google/keytransparency/core/sequencer/sequencer_go_proto"
@@ -63,9 +65,13 @@ func (s SourceList) First() *rtpb.ReadToken {
 		// Empty struct means there is nothing else to page through.
 		return &rtpb.ReadToken{}
 	}
+	st, err := ptypes.TimestampProto(metadata.Source(s[0]).StartTime())
+	if err != nil {
+		panic("invalid timestamp")
+	}
 	return &rtpb.ReadToken{
-		SliceIndex:   0,
-		LowWatermark: s[0].LowestInclusive,
+		SliceIndex: 0,
+		StartTime:  st,
 	}
 }
 
@@ -75,9 +81,13 @@ func (s SourceList) First() *rtpb.ReadToken {
 func (s SourceList) Next(rt *rtpb.ReadToken, lastRow *mutator.LogMessage) *rtpb.ReadToken {
 	if lastRow != nil {
 		// There are more items in this source slice.
+		st, err := ptypes.TimestampProto(lastRow.ID)
+		if err != nil {
+			panic("invalid timestamp")
+		}
 		return &rtpb.ReadToken{
-			SliceIndex:   rt.SliceIndex,
-			LowWatermark: lastRow.ID,
+			SliceIndex: rt.SliceIndex,
+			StartTime:  st,
 		}
 	}
 
@@ -86,8 +96,13 @@ func (s SourceList) Next(rt *rtpb.ReadToken, lastRow *mutator.LogMessage) *rtpb.
 		// There are no more source slices to iterate over.
 		return &rtpb.ReadToken{} // Encodes to ""
 	}
+
+	st, err := ptypes.TimestampProto(metadata.Source(s[rt.SliceIndex+1]).StartTime())
+	if err != nil {
+		panic("invalid timestamp")
+	}
 	return &rtpb.ReadToken{
-		SliceIndex:   rt.SliceIndex + 1,
-		LowWatermark: s[rt.SliceIndex+1].LowestInclusive,
+		SliceIndex: rt.SliceIndex + 1,
+		StartTime:  st,
 	}
 }
diff --git a/core/keyserver/paginator_test.go b/core/keyserver/paginator_test.go
index e4d10e575..9340182e7 100644
--- a/core/keyserver/paginator_test.go
+++ b/core/keyserver/paginator_test.go
@@ -16,11 +16,13 @@ package keyserver
 
 import (
 	"testing"
+	"time"
 
 	"github.com/golang/protobuf/proto"
 
 	"github.com/google/keytransparency/core/mutator"
 
+	tpb "github.com/golang/protobuf/ptypes/timestamp"
 	rtpb "github.com/google/keytransparency/core/keyserver/readtoken_go_proto"
 )
 
@@ -43,7 +45,7 @@ func TestEncodeToken(t *testing.T) {
 }
 
 func TestTokenEncodeDecode(t *testing.T) {
-	rt1 := &rtpb.ReadToken{SliceIndex: 2, LowWatermark: 5}
+	rt1 := &rtpb.ReadToken{SliceIndex: 2, StartTime: &tpb.Timestamp{Nanos: 5}}
 	rt1Token, err := EncodeToken(rt1)
 	if err != nil {
 		t.Fatalf("EncodeToken(%v): %v", rt1, err)
@@ -78,7 +80,7 @@ func TestFirst(t *testing.T) {
 				{LogId: 2, LowestInclusive: 2, HighestExclusive: 11},
 				{LogId: 3, LowestInclusive: 11, HighestExclusive: 21},
 			},
-			want: &rtpb.ReadToken{SliceIndex: 0, LowWatermark: 2},
+			want: &rtpb.ReadToken{SliceIndex: 0, StartTime: &tpb.Timestamp{Nanos: 2}},
 		},
 		{s: SourceList{}, want: &rtpb.ReadToken{}},
 	} {
@@ -103,16 +105,16 @@ func TestNext(t *testing.T) {
 		{
 			desc:    "first page",
 			s:       a,
-			rt:      &rtpb.ReadToken{SliceIndex: 0, LowWatermark: 2},
-			lastRow: &mutator.LogMessage{ID: 6},
-			want:    &rtpb.ReadToken{SliceIndex: 0, LowWatermark: 6},
+			rt:      &rtpb.ReadToken{SliceIndex: 0, StartTime: &tpb.Timestamp{Nanos: 2}},
+			lastRow: &mutator.LogMessage{ID: time.Unix(0, 6)},
+			want:    &rtpb.ReadToken{SliceIndex: 0, StartTime: &tpb.Timestamp{Nanos: 6}},
 		},
 		{
 			desc:    "next source",
 			s:       a,
 			rt:      &rtpb.ReadToken{},
 			lastRow: nil,
-			want:    &rtpb.ReadToken{SliceIndex: 1, LowWatermark: 11},
+			want:    &rtpb.ReadToken{SliceIndex: 1, StartTime: &tpb.Timestamp{Nanos: 11}},
 		},
 		{
 			desc:    "last page",
@@ -124,7 +126,7 @@ func TestNext(t *testing.T) {
 		{
 			desc:    "empty",
 			s:       SourceList{},
-			rt:      &rtpb.ReadToken{SliceIndex: 1, LowWatermark: 2},
+			rt:      &rtpb.ReadToken{SliceIndex: 1, StartTime: &tpb.Timestamp{Nanos: 2}},
 			lastRow: nil,
 			want:    &rtpb.ReadToken{},
 		},
diff --git a/core/keyserver/readtoken.proto b/core/keyserver/readtoken.proto
index ff6dd1aec..95344382d 100644
--- a/core/keyserver/readtoken.proto
+++ b/core/keyserver/readtoken.proto
@@ -19,13 +19,16 @@ package google.keytransparency.v1;
 option go_package = "github.com/google/keytransparency/core/keyserver/readtoken_go_proto";
 
 import "v1/keytransparency.proto";
+import "google/protobuf/timestamp.proto";
+
 
 // ReadToken can be serialized and handed to users for pagination.
 message ReadToken {
   // slice_index identifies the source for reading.
   int64 slice_index = 1;
-  // low_watemark identifies the lowest (exclusive) row to return.
-  int64 low_watermark = 2;
+  reserved 2;
+  // start_time identifies the lowest (exclusive) row to return.
+  google.protobuf.Timestamp start_time = 3;
 }
 
 // ListUserRevisions token can be serialized and handed to users for pagination
diff --git a/core/keyserver/readtoken_go_proto/readtoken.pb.go b/core/keyserver/readtoken_go_proto/readtoken.pb.go
index 8f8dc3c08..418bc740e 100644
--- a/core/keyserver/readtoken_go_proto/readtoken.pb.go
+++ b/core/keyserver/readtoken_go_proto/readtoken.pb.go
@@ -6,6 +6,7 @@ package readtoken_go_proto
 import (
 	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
+	timestamp "github.com/golang/protobuf/ptypes/timestamp"
 	keytransparency_go_proto "github.com/google/keytransparency/core/api/v1/keytransparency_go_proto"
 	math "math"
 )
@@ -25,11 +26,11 @@ const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
 type ReadToken struct {
 	// slice_index identifies the source for reading.
 	SliceIndex int64 `protobuf:"varint,1,opt,name=slice_index,json=sliceIndex,proto3" json:"slice_index,omitempty"`
-	// low_watemark identifies the lowest (exclusive) row to return.
-	LowWatermark         int64    `protobuf:"varint,2,opt,name=low_watermark,json=lowWatermark,proto3" json:"low_watermark,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	// start_time identifies the lowest (exclusive) row to return.
+	StartTime            *timestamp.Timestamp `protobuf:"bytes,3,opt,name=start_time,json=startTime,proto3" json:"start_time,omitempty"`
+	XXX_NoUnkeyedLiteral struct{}             `json:"-"`
+	XXX_unrecognized     []byte               `json:"-"`
+	XXX_sizecache        int32                `json:"-"`
 }
 
 func (m *ReadToken) Reset()         { *m = ReadToken{} }
@@ -64,11 +65,11 @@ func (m *ReadToken) GetSliceIndex() int64 {
 	return 0
 }
 
-func (m *ReadToken) GetLowWatermark() int64 {
+func (m *ReadToken) GetStartTime() *timestamp.Timestamp {
 	if m != nil {
-		return m.LowWatermark
+		return m.StartTime
 	}
-	return 0
+	return nil
 }
 
 // ListUserRevisions token can be serialized and handed to users for pagination
@@ -134,22 +135,23 @@ func init() {
 func init() { proto.RegisterFile("readtoken.proto", fileDescriptor_735a2ae6888918c9) }
 
 var fileDescriptor_735a2ae6888918c9 = []byte{
-	// 257 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x74, 0x90, 0xc1, 0x4a, 0xc4, 0x30,
-	0x10, 0x86, 0xa9, 0x82, 0x62, 0x56, 0x11, 0x73, 0x90, 0xea, 0x45, 0x59, 0x2f, 0x5e, 0x4c, 0x59,
-	0xf7, 0x0d, 0x14, 0x0f, 0x82, 0x1e, 0x0c, 0x8a, 0xe0, 0x25, 0x64, 0xdb, 0xa1, 0x86, 0x76, 0x33,
-	0xeb, 0x64, 0xda, 0xba, 0x2f, 0xe2, 0xf3, 0xca, 0xa6, 0x56, 0x61, 0x65, 0x4f, 0x81, 0x6f, 0xf2,
-	0x7f, 0x33, 0xfc, 0xe2, 0x90, 0xc0, 0x16, 0x8c, 0x15, 0x78, 0xb5, 0x20, 0x64, 0x94, 0x27, 0x25,
-	0x62, 0x59, 0x83, 0xaa, 0x60, 0xc9, 0x64, 0x7d, 0x58, 0x58, 0x02, 0x9f, 0x2f, 0x55, 0x3b, 0x39,
-	0x4d, 0xdb, 0x49, 0xb6, 0x8e, 0x63, 0x68, 0xfc, 0x24, 0xf6, 0x34, 0xd8, 0xe2, 0x79, 0xe5, 0x91,
-	0x67, 0x62, 0x14, 0x6a, 0x97, 0x83, 0x71, 0xbe, 0x80, 0xcf, 0x34, 0x39, 0x4f, 0x2e, 0xb7, 0xb5,
-	0x88, 0xe8, 0x7e, 0x45, 0xe4, 0x85, 0x38, 0xa8, 0xb1, 0x33, 0x9d, 0x65, 0xa0, 0xb9, 0xa5, 0x2a,
-	0xdd, 0x8a, 0x5f, 0xf6, 0x6b, 0xec, 0x5e, 0x07, 0x36, 0xfe, 0x4a, 0xc4, 0xf1, 0x83, 0x0b, 0xfc,
-	0x12, 0x80, 0x34, 0xb4, 0x2e, 0x38, 0xf4, 0xa1, 0x5f, 0xf0, 0x28, 0x76, 0x09, 0x3e, 0x1a, 0x08,
-	0x1c, 0xe5, 0xa3, 0xeb, 0xa9, 0xda, 0x78, 0xb4, 0xfa, 0xe7, 0xd0, 0x7d, 0x54, 0x0f, 0x0e, 0x79,
-	0x25, 0x24, 0x0d, 0x43, 0x43, 0xc0, 0x0d, 0x79, 0x28, 0x7e, 0x6e, 0x3a, 0xa2, 0xbf, 0x58, 0x3f,
-	0xb8, 0xb9, 0x7b, 0xbb, 0x2d, 0x1d, 0xbf, 0x37, 0x33, 0x95, 0xe3, 0x3c, 0xeb, 0x17, 0xaf, 0xd7,
-	0x92, 0xe5, 0x48, 0x11, 0x06, 0xa0, 0x16, 0x28, 0xfb, 0x2d, 0xd9, 0x94, 0x68, 0x62, 0x65, 0xb3,
-	0x9d, 0xf8, 0x4c, 0xbf, 0x03, 0x00, 0x00, 0xff, 0xff, 0x42, 0xa4, 0x18, 0x01, 0x81, 0x01, 0x00,
-	0x00,
+	// 287 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x74, 0x91, 0x31, 0x4f, 0xc3, 0x30,
+	0x10, 0x85, 0x95, 0x16, 0x01, 0x75, 0x07, 0x20, 0x03, 0x0a, 0x5d, 0x5a, 0x75, 0xea, 0x82, 0xad,
+	0xb6, 0x13, 0x2b, 0x88, 0x01, 0x04, 0x8b, 0x55, 0x16, 0x96, 0xc8, 0x4d, 0x8e, 0x60, 0xb5, 0xb1,
+	0xc3, 0xf9, 0x12, 0xd1, 0x3f, 0xc2, 0xef, 0x45, 0xb1, 0x1b, 0x90, 0x8a, 0x98, 0xac, 0x7b, 0x77,
+	0xef, 0x3e, 0x3f, 0x9b, 0x9d, 0x21, 0xa8, 0x9c, 0xec, 0x06, 0x0c, 0xaf, 0xd0, 0x92, 0x8d, 0xaf,
+	0x0a, 0x6b, 0x8b, 0x2d, 0xf0, 0x0d, 0xec, 0x08, 0x95, 0x71, 0x95, 0x42, 0x30, 0xd9, 0x8e, 0x37,
+	0xf3, 0x51, 0xd2, 0xcc, 0xc5, 0xa1, 0xec, 0x4d, 0xa3, 0x71, 0x30, 0x09, 0x5f, 0xad, 0xeb, 0x37,
+	0x41, 0xba, 0x04, 0x47, 0xaa, 0xac, 0xc2, 0xc0, 0xb4, 0x64, 0x03, 0x09, 0x2a, 0x5f, 0xb5, 0xa0,
+	0x78, 0xcc, 0x86, 0x6e, 0xab, 0x33, 0x48, 0xb5, 0xc9, 0xe1, 0x33, 0x89, 0x26, 0xd1, 0xac, 0x2f,
+	0x99, 0x97, 0x1e, 0x5a, 0x25, 0xbe, 0x61, 0xcc, 0x91, 0x42, 0x4a, 0xdb, 0x35, 0x49, 0x7f, 0x12,
+	0xcd, 0x86, 0x8b, 0x11, 0xdf, 0x5f, 0xac, 0x63, 0xf0, 0x55, 0xc7, 0x90, 0x03, 0x3f, 0xdd, 0xd6,
+	0x8f, 0x47, 0xa7, 0xbd, 0xf3, 0xfe, 0xf4, 0x2b, 0x62, 0x97, 0x4f, 0xda, 0xd1, 0x8b, 0x03, 0x94,
+	0xd0, 0x68, 0xa7, 0xad, 0x71, 0x01, 0xfe, 0xcc, 0x4e, 0x10, 0x3e, 0x6a, 0x70, 0xe4, 0xc1, 0xc3,
+	0xc5, 0x92, 0xff, 0x9b, 0x98, 0xff, 0xd9, 0x21, 0x83, 0x55, 0x76, 0x3b, 0xe2, 0x6b, 0x16, 0x63,
+	0xd7, 0x4c, 0x11, 0xa8, 0x46, 0x03, 0x79, 0xd2, 0xf3, 0x91, 0x2e, 0xf0, 0xd7, 0x16, 0x1a, 0xb7,
+	0xf7, 0xaf, 0x77, 0x85, 0xa6, 0xf7, 0x7a, 0xcd, 0x33, 0x5b, 0x8a, 0xfd, 0xab, 0x1d, 0x80, 0x45,
+	0x66, 0xd1, 0x8b, 0x0e, 0xb0, 0x01, 0x14, 0x3f, 0x3f, 0x94, 0x16, 0x36, 0x0d, 0xe9, 0x8f, 0xfd,
+	0xb1, 0xfc, 0x0e, 0x00, 0x00, 0xff, 0xff, 0x1f, 0xd9, 0xb2, 0x4d, 0xbe, 0x01, 0x00, 0x00,
 }
diff --git a/core/keyserver/revisions.go b/core/keyserver/revisions.go
index 941395c4a..fd12872ec 100644
--- a/core/keyserver/revisions.go
+++ b/core/keyserver/revisions.go
@@ -19,12 +19,14 @@ import (
 
 	"github.com/golang/glog"
 	"github.com/golang/protobuf/proto"
+	"github.com/golang/protobuf/ptypes"
 	"github.com/google/trillian/types"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 
 	"github.com/google/keytransparency/core/directory"
 	"github.com/google/keytransparency/core/mutator"
+	"github.com/google/keytransparency/core/sequencer/metadata"
 
 	pb "github.com/google/keytransparency/core/api/v1/keytransparency_go_proto"
 	tpb "github.com/google/trillian"
@@ -139,12 +141,16 @@ func (s *Server) ListMutations(ctx context.Context, in *pb.ListMutationsRequest)
 	}
 
 	// Read PageSize + 1 messages from the log to see if there is another page.
-	high := meta.Sources[rt.SliceIndex].HighestExclusive
+	high := metadata.Source(meta.Sources[rt.SliceIndex]).EndTime()
 	logID := meta.Sources[rt.SliceIndex].LogId
-	msgs, err := s.logs.ReadLog(ctx, d.DirectoryID, logID, rt.LowWatermark, high, in.PageSize+1)
+	low, err := ptypes.Timestamp(rt.StartTime)
+	if err != nil {
+		return nil, status.Errorf(codes.InvalidArgument, "Invalid start timestamp: %v", err)
+	}
+	msgs, err := s.logs.ReadLog(ctx, d.DirectoryID, logID, low, high, in.PageSize+1)
 	if st := status.Convert(err); st.Code() != codes.OK {
 		glog.Errorf("ListMutations(): ReadLog(%v, log: %v/(%v, %v], batchSize: %v): %v",
-			d.DirectoryID, logID, rt.LowWatermark, high, in.PageSize, err)
+			d.DirectoryID, logID, low, high, in.PageSize, err)
 		return nil, status.Errorf(st.Code(), "Reading mutations range failed: %v", st.Message())
 	}
 	moreInLogID := len(msgs) == int(in.PageSize+1)
diff --git a/core/keyserver/revisions_test.go b/core/keyserver/revisions_test.go
index bf7c57b90..5edfa26b6 100644
--- a/core/keyserver/revisions_test.go
+++ b/core/keyserver/revisions_test.go
@@ -23,11 +23,13 @@ import (
 
 	"github.com/golang/mock/gomock"
 	"github.com/golang/protobuf/proto"
+	"github.com/golang/protobuf/ptypes"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 
 	"github.com/google/keytransparency/core/mutator"
 
+	protopb "github.com/golang/protobuf/ptypes/timestamp"
 	pb "github.com/google/keytransparency/core/api/v1/keytransparency_go_proto"
 	rtpb "github.com/google/keytransparency/core/keyserver/readtoken_go_proto"
 	spb "github.com/google/keytransparency/core/sequencer/sequencer_go_proto"
@@ -74,26 +76,31 @@ func (m *mutations) Send(ctx context.Context, dirID string, mutation ...*pb.Entr
 }
 
 func (m *mutations) ReadLog(ctx context.Context, dirID string,
-	logID, low, high int64, batchSize int32) ([]*mutator.LogMessage, error) {
+	logID int64, low, high time.Time, batchSize int32) ([]*mutator.LogMessage, error) {
 	logShard := (*m)[logID]
-	if low > int64(len(logShard)) {
+	if low.UnixNano() > int64(len(logShard)) {
 		return nil, fmt.Errorf("invalid argument: low: %v, want <= max watermark: %v", low, len(logShard))
 	}
-	count := high - low
+	count := high.UnixNano() - low.UnixNano()
 	if count > int64(batchSize) {
 		count = int64(batchSize)
 	}
-	if low+count > int64(len(logShard)) {
-		count = int64(len(logShard)) - low + 1
+	if low.UnixNano()+count > int64(len(logShard)) {
+		count = int64(len(logShard)) - low.UnixNano() + 1
 	}
-	return logShard[low : low+count], nil
+	return logShard[low.UnixNano() : low.UnixNano()+count], nil
 }
 
-func MustEncodeToken(t *testing.T, low int64) string {
+func MustEncodeToken(t *testing.T, low time.Time) string {
 	t.Helper()
+
+	st, err := ptypes.TimestampProto(low)
+	if err != nil {
+		t.Fatal(err)
+	}
 	rt := &rtpb.ReadToken{
-		SliceIndex:   0,
-		LowWatermark: low,
+		SliceIndex: 0,
+		StartTime:  st,
 	}
 	token, err := EncodeToken(rt)
 	if err != nil {
@@ -118,7 +125,7 @@ func TestListMutations(t *testing.T) {
 			}
 			for i := source.LowestInclusive; i < source.HighestExclusive; i++ {
 				fakeLogs[source.LogId][i] = &mutator.LogMessage{
-					ID: i,
+					ID: time.Unix(0, i*int64(time.Nanosecond)),
 					Mutation: &pb.SignedEntry{
 						Entry: mustMarshal(t, &pb.Entry{
 							Index:      []byte(fmt.Sprintf("key_%v", i)),
@@ -140,10 +147,10 @@ func TestListMutations(t *testing.T) {
 	}{
 		{desc: "exact page", pageSize: 6, start: 2, end: 7, wantNext: &rtpb.ReadToken{}},
 		{desc: "large page", pageSize: 10, start: 2, end: 7, wantNext: &rtpb.ReadToken{}},
-		{desc: "partial", pageSize: 4, start: 2, end: 6, wantNext: &rtpb.ReadToken{LowWatermark: 6}},
-		{desc: "large page with token", token: MustEncodeToken(t, 3), pageSize: 10, start: 3, end: 7, wantNext: &rtpb.ReadToken{}},
-		{desc: "small page with token", token: MustEncodeToken(t, 3), pageSize: 2, start: 3, end: 5,
-			wantNext: &rtpb.ReadToken{LowWatermark: 5}},
+		{desc: "partial", pageSize: 4, start: 2, end: 6, wantNext: &rtpb.ReadToken{StartTime: &protopb.Timestamp{Nanos: 6}}},
+		{desc: "large page with token", token: MustEncodeToken(t, time.Unix(0, 3)), pageSize: 10, start: 3, end: 7, wantNext: &rtpb.ReadToken{}},
+		{desc: "small page with token", token: MustEncodeToken(t, time.Unix(0, 3)), pageSize: 2, start: 3, end: 5,
+			wantNext: &rtpb.ReadToken{StartTime: &protopb.Timestamp{Nanos: 5}}},
 		{desc: "invalid page token", token: "some_token", pageSize: 0, wantErr: true},
 	} {
 		t.Run(tc.desc, func(t *testing.T) {
diff --git a/core/mutator/mutator.go b/core/mutator/mutator.go
index b6bc9dad8..4c845d28e 100644
--- a/core/mutator/mutator.go
+++ b/core/mutator/mutator.go
@@ -18,6 +18,8 @@
 package mutator
 
 import (
+	"time"
+
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 
@@ -50,7 +52,7 @@ type VerifyMutationFn func(mutation *pb.SignedEntry) error
 
 // LogMessage represents a change to a user, and associated data.
 type LogMessage struct {
-	ID        int64
+	ID        time.Time
 	LocalID   int64
 	Mutation  *pb.SignedEntry
 	ExtraData *pb.Committed
diff --git a/core/sequencer/metadata/sourceslice.go b/core/sequencer/metadata/sourceslice.go
new file mode 100644
index 000000000..9900cad52
--- /dev/null
+++ b/core/sequencer/metadata/sourceslice.go
@@ -0,0 +1,41 @@
+// Copyright 2019 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package metadata
+
+import (
+	"time"
+
+	spb "github.com/google/keytransparency/core/sequencer/sequencer_go_proto"
+)
+
+// Source returns a wrapper for SourceSlice
+func Source(s *spb.MapMetadata_SourceSlice) SourceSlice {
+	return SourceSlice{s: s}
+}
+
+// SourceSlice defines accessor and conversion methods for MapMetadata_SourceSlice
+type SourceSlice struct {
+	s *spb.MapMetadata_SourceSlice
+}
+
+// StartTime returns LowestInclusive as a time.Time
+func (s SourceSlice) StartTime() time.Time {
+	return time.Unix(0, s.s.GetLowestInclusive())
+}
+
+// EndTime returns HighestExclusive as a time.Time
+func (s SourceSlice) EndTime() time.Time {
+	return time.Unix(0, s.s.GetHighestExclusive())
+}
diff --git a/core/sequencer/server.go b/core/sequencer/server.go
index 16fb4a121..f85b643bb 100644
--- a/core/sequencer/server.go
+++ b/core/sequencer/server.go
@@ -32,6 +32,7 @@ import (
 	"github.com/google/keytransparency/core/mutator"
 	"github.com/google/keytransparency/core/mutator/entry"
 	"github.com/google/keytransparency/core/sequencer/mapper"
+	"github.com/google/keytransparency/core/sequencer/metadata"
 	"github.com/google/keytransparency/core/sequencer/runner"
 
 	spb "github.com/google/keytransparency/core/sequencer/sequencer_go_proto"
@@ -124,8 +125,8 @@ type Watermarks map[int64]int64
 type LogsReader interface {
 	// HighWatermark returns the number of items and the highest primary
 	// key up to batchSize items after start (exclusive).
-	HighWatermark(ctx context.Context, directoryID string, logID, start int64,
-		batchSize int32) (count int32, watermark int64, err error)
+	HighWatermark(ctx context.Context, directoryID string, logID int64, start time.Time,
+		batchSize int32) (count int32, watermark time.Time, err error)
 
 	// ListLogs returns the logIDs associated with directoryID that have their write bits set,
 	// or all logIDs associated with directoryID if writable is false.
@@ -134,7 +135,7 @@ type LogsReader interface {
 	// ReadLog returns the lowest messages in the (low, high] range stored in the
 	// specified log, up to batchSize.  Paginate by setting low to the
 	// highest LogMessage returned in the previous page.
-	ReadLog(ctx context.Context, directoryID string, logID, low, high int64,
+	ReadLog(ctx context.Context, directoryID string, logID int64, low, high time.Time,
 		batchSize int32) ([]*mutator.LogMessage, error)
 }
 
@@ -339,8 +340,9 @@ func (s *Server) ApplyRevisions(ctx context.Context, in *spb.ApplyRevisionsReque
 func (s *Server) readMessages(ctx context.Context, source *spb.MapMetadata_SourceSlice,
 	directoryID string, chunkSize int32,
 	emit func(*mutator.LogMessage)) error {
-	low := source.LowestInclusive
-	high := source.HighestExclusive
+	ss := metadata.Source(source)
+	low := ss.StartTime()
+	high := ss.EndTime()
 	// Loop until less than chunkSize items are returned.
 	for count := chunkSize; count == chunkSize; {
 		batch, err := s.logs.ReadLog(ctx, directoryID, source.LogId, low, high, chunkSize)
@@ -353,7 +355,7 @@ func (s *Server) readMessages(ctx context.Context, source *spb.MapMetadata_Sourc
 		logEntryCount.Add(float64(len(batch)), directoryID, fmt.Sprintf("%v", source.LogId))
 		for _, m := range batch {
 			emit(m)
-			if m.ID > low {
+			if m.ID.After(low) {
 				low = m.ID
 			}
 		}
@@ -527,8 +529,8 @@ func (s *Server) HighWatermarks(ctx context.Context, directoryID string, lastMet
 	// revision.
 	// TODO(gbelvin): Separate end watermarks for the sequencer's needs
 	// from ranges of watermarks for the verifier's needs.
-	ends := map[int64]int64{}
-	starts := map[int64]int64{}
+	ends := make(map[int64]int64)
+	starts := make(map[int64]int64)
 	for _, source := range lastMeta.GetSources() {
 		if ends[source.LogId] < source.HighestExclusive {
 			ends[source.LogId] = source.HighestExclusive
@@ -543,14 +545,14 @@ func (s *Server) HighWatermarks(ctx context.Context, directoryID string, lastMet
 	}
 	// TODO(gbelvin): Get HighWatermarks in parallel.
 	for _, logID := range logIDs {
-		low := ends[logID]
+		low := time.Unix(0, ends[logID])
 		count, high, err := s.logs.HighWatermark(ctx, directoryID, logID, low, batchSize)
 		if err != nil {
 			return 0, nil, status.Errorf(codes.Internal,
 				"HighWatermark(%v/%v, start: %v, batch: %v): %v",
 				directoryID, logID, low, batchSize, err)
 		}
-		starts[logID], ends[logID] = low, high
+		starts[logID], ends[logID] = low.UnixNano(), high.UnixNano()
 		total += count
 		batchSize -= count
 	}
diff --git a/core/sequencer/server_test.go b/core/sequencer/server_test.go
index 770766255..1cdb21986 100644
--- a/core/sequencer/server_test.go
+++ b/core/sequencer/server_test.go
@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"sort"
 	"testing"
+	"time"
 
 	"github.com/golang/protobuf/proto"
 	"github.com/google/go-cmp/cmp"
@@ -38,14 +39,15 @@ const directoryID = "directoryID"
 
 func fakeMetric(_ string) {}
 
+// fakeLogs are indexed by logID, and nanoseconds from time 0
 type fakeLogs map[int64][]mutator.LogMessage
 
-func (l fakeLogs) ReadLog(ctx context.Context, directoryID string, logID, low, high int64,
+func (l fakeLogs) ReadLog(ctx context.Context, directoryID string, logID int64, low, high time.Time,
 	batchSize int32) ([]*mutator.LogMessage, error) {
-	refs := make([]*mutator.LogMessage, 0, int(high-low))
-	for i := low; i < high; i++ {
-		l[logID][i].ID = i
-		refs = append(refs, &l[logID][i])
+	refs := make([]*mutator.LogMessage, 0)
+	for i := low; i.Before(high); i = i.Add(time.Nanosecond) {
+		l[logID][i.UnixNano()].ID = i
+		refs = append(refs, &l[logID][i.UnixNano()])
 	}
 	return refs, nil
 }
@@ -60,14 +62,14 @@ func (l fakeLogs) ListLogs(ctx context.Context, directoryID string, writable boo
 	return logIDs, nil
 }
 
-func (l fakeLogs) HighWatermark(ctx context.Context, directoryID string, logID, start int64,
-	batchSize int32) (int32, int64, error) {
-	high := start + int64(batchSize)
+func (l fakeLogs) HighWatermark(ctx context.Context, directoryID string, logID int64, start time.Time,
+	batchSize int32) (int32, time.Time, error) {
+	high := start.UnixNano() + int64(batchSize)
 	if high > int64(len(l[logID])) {
 		high = int64(len(l[logID]))
 	}
-	count := int32(high - start)
-	return count, high, nil
+	count := int32(high - start.UnixNano())
+	return count, time.Unix(0, high), nil
 }
 
 type fakeTrillianFactory struct {
diff --git a/impl/sql/mutationstorage/mutation_logs_test.go b/impl/sql/mutationstorage/mutation_logs_test.go
index 1727af74c..18dd0f2dd 100644
--- a/impl/sql/mutationstorage/mutation_logs_test.go
+++ b/impl/sql/mutationstorage/mutation_logs_test.go
@@ -165,8 +165,8 @@ func TestWatermark(t *testing.T) {
 	m := newForTest(ctx, t, directoryID, logIDs...)
 	update := []byte("bar")
 
-	startTS := time.Now()
-	for ts := startTS; ts.Before(startTS.Add(10)); ts = ts.Add(1) {
+	start := time.Now()
+	for ts := start; ts.Before(start.Add(10)); ts = ts.Add(1) {
 		for _, logID := range logIDs {
 			if err := m.send(ctx, ts, directoryID, logID, update); err != nil {
 				t.Fatalf("m.send(%v): %v", logID, err)
@@ -174,30 +174,29 @@ func TestWatermark(t *testing.T) {
 		}
 	}
 
-	start := startTS.UnixNano()
 	for _, tc := range []struct {
 		desc      string
 		logID     int64
-		start     int64
+		start     time.Time
 		batchSize int32
 		count     int32
-		want      int64
+		want      time.Time
 	}{
-		{desc: "log1 max", logID: 1, batchSize: 100, want: start + 10, count: 10},
-		{desc: "log2 max", logID: 2, batchSize: 100, want: start + 10, count: 10},
-		{desc: "batch0", logID: 1, batchSize: 0},
-		{desc: "batch0start55", logID: 1, start: 55, batchSize: 0, want: 55},
-		{desc: "batch5", logID: 1, batchSize: 5, want: start + 5, count: 5},
-		{desc: "start1", logID: 1, start: start + 2, batchSize: 5, want: start + 7, count: 5},
-		{desc: "start8", logID: 1, start: start + 8, batchSize: 5, want: start + 10, count: 2},
+		{desc: "log1 max", logID: 1, batchSize: 100, want: start.Add(10), count: 10},
+		{desc: "log2 max", logID: 2, batchSize: 100, want: start.Add(10), count: 10},
+		{desc: "batch0", logID: 1, batchSize: 0, start: time.Unix(0, 0), want: time.Unix(0, 0)},
+		{desc: "batch0start55", logID: 1, start: time.Unix(0, 55), batchSize: 0, want: time.Unix(0, 55)},
+		{desc: "batch5", logID: 1, batchSize: 5, want: start.Add(5), count: 5},
+		{desc: "start1", logID: 1, start: start.Add(2), batchSize: 5, want: start.Add(7), count: 5},
+		{desc: "start8", logID: 1, start: start.Add(8), batchSize: 5, want: start.Add(10), count: 2},
 	} {
 		t.Run(tc.desc, func(t *testing.T) {
 			count, got, err := m.HighWatermark(ctx, directoryID, tc.logID, tc.start, tc.batchSize)
 			if err != nil {
 				t.Errorf("highWatermark(): %v", err)
 			}
-			if got != tc.want {
-				t.Errorf("highWatermark(%v) high: %v, want %v", tc.start, got, tc.want)
+			if !got.Equal(tc.want) {
+				t.Errorf("highWatermark(%v) high: %v, want %v", tc.start, got.UnixNano(), tc.want)
 			}
 			if count != tc.count {
 				t.Errorf("highWatermark(%v) count: %v, want %v", tc.start, count, tc.count)
diff --git a/impl/sql/mutationstorage/queue.go b/impl/sql/mutationstorage/queue.go
index 520d407f5..1c8daec63 100644
--- a/impl/sql/mutationstorage/queue.go
+++ b/impl/sql/mutationstorage/queue.go
@@ -32,7 +32,6 @@ import (
 
 // SetWritable enables or disables new writes from going to logID.
 func (m *Mutations) SetWritable(ctx context.Context, directoryID string, logID int64, enabled bool) error {
-	glog.Errorf("mutationstorage: SetWritable(%v, %v, enabled: %v)", directoryID, logID, enabled)
 	result, err := m.db.ExecContext(ctx,
 		`UPDATE Logs SET Enabled = ? WHERE DirectoryID = ? AND LogID = ?;`,
 		enabled, directoryID, logID)
@@ -91,7 +90,7 @@ func (m *Mutations) Send(ctx context.Context, directoryID string, updates ...*pb
 	if err := m.send(ctx, ts, directoryID, logID, updateData...); err != nil {
 		return nil, err
 	}
-	return &keyserver.WriteWatermark{LogID: logID, Watermark: ts.UnixNano()}, nil
+	return &keyserver.WriteWatermark{LogID: logID, Watermark: ts}, nil
 }
 
 // ListLogs returns a list of all logs for directoryID, optionally filtered for writable logs.
@@ -179,8 +178,8 @@ func (m *Mutations) send(ctx context.Context, ts time.Time, directoryID string,
 
 // HighWatermark returns the highest watermark +1 in logID that is less than or
 // equal to batchSize items greater than start.
-func (m *Mutations) HighWatermark(ctx context.Context, directoryID string, logID,
-	start int64, batchSize int32) (int32, int64, error) {
+func (m *Mutations) HighWatermark(ctx context.Context, directoryID string, logID int64,
+	start time.Time, batchSize int32) (int32, time.Time, error) {
 	var count int32
 	var high int64
 	if err := m.db.QueryRowContext(ctx,
@@ -191,23 +190,23 @@ func (m *Mutations) HighWatermark(ctx context.Context, directoryID string, logID
 			ORDER BY Q.Time ASC
 			LIMIT ?
 		) AS T1`,
-		start, directoryID, logID, start, batchSize).
+		start.UnixNano(), directoryID, logID, start.UnixNano(), batchSize).
 		Scan(&count, &high); err != nil {
-		return 0, 0, err
+		return 0, time.Time{}, err
 	}
-	return count, high, nil
+	return count, time.Unix(0, high), nil
 }
 
 // ReadLog reads all mutations in logID between [low, high).
 // ReadLog may return more rows than batchSize in order to fetch all the rows at a particular timestamp.
 func (m *Mutations) ReadLog(ctx context.Context, directoryID string,
-	logID, low, high int64, batchSize int32) ([]*mutator.LogMessage, error) {
+	logID int64, low, high time.Time, batchSize int32) ([]*mutator.LogMessage, error) {
 	rows, err := m.db.QueryContext(ctx,
 		`SELECT Time, LocalID, Mutation FROM Queue
 		WHERE DirectoryID = ? AND LogID = ? AND Time >= ? AND Time < ?
 		ORDER BY Time, LocalID ASC
 		LIMIT ?;`,
-		directoryID, logID, low, high, batchSize)
+		directoryID, logID, low.UnixNano(), high.UnixNano(), batchSize)
 	if err != nil {
 		return nil, err
 	}
@@ -224,7 +223,7 @@ func (m *Mutations) ReadLog(ctx context.Context, directoryID string,
 			`SELECT Time, LocalID, Mutation FROM Queue
 			WHERE DirectoryID = ? AND LogID = ? AND Time = ? AND LocalID > ?
 			ORDER BY LocalID ASC;`,
-			directoryID, logID, last.ID, last.LocalID)
+			directoryID, logID, last.ID.UnixNano(), last.LocalID)
 		if err != nil {
 			return nil, err
 		}
@@ -252,7 +251,7 @@ func readQueueMessages(rows *sql.Rows) ([]*mutator.LogMessage, error) {
 			return nil, err
 		}
 		results = append(results, &mutator.LogMessage{
-			ID:        timestamp,
+			ID:        time.Unix(0, timestamp),
 			LocalID:   localID,
 			Mutation:  entryUpdate.Mutation,
 			ExtraData: entryUpdate.Committed,

From 90e1d58555c2e91a2f4207ab388c5f99af2abff8 Mon Sep 17 00:00:00 2001
From: Gary Belvin <gdb@google.com>
Date: Fri, 25 Oct 2019 17:19:37 +0100
Subject: [PATCH 03/22] Use and test the same MySQL connection that main.go
 uses  (#1370)

* Use mysql for tests

Test with the same database we use in prod.

- Each test gets its own database
- Drop test databases after they are done.

* Move sql.Open to a central place
* Set MySQL db flags
* Use testdb in directory storage
---
 cmd/keytransparency-sequencer/main.go         | 19 ++----
 cmd/keytransparency-server/main.go            | 19 ++----
 core/integration/storagetest/batch.go         | 30 +++++-----
 core/integration/storagetest/mutation_logs.go | 12 ++--
 .../storagetest/mutation_logs_admin.go        | 17 +++---
 go.mod                                        |  1 -
 impl/integration/env.go                       |  1 -
 impl/sql/directory/storage_test.go            | 31 ++++------
 .../sql/mutationstorage/mutation_logs_test.go | 25 ++++----
 impl/sql/mutationstorage/mutations_test.go    | 20 ++-----
 impl/sql/open.go                              | 39 ++++++++++++
 impl/sql/testdb/testdb.go                     | 59 +++++++++++++++++++
 impl/sql/testdb/testdb_test.go                | 32 ++++++++++
 13 files changed, 207 insertions(+), 98 deletions(-)
 create mode 100644 impl/sql/open.go
 create mode 100644 impl/sql/testdb/testdb.go
 create mode 100644 impl/sql/testdb/testdb_test.go

diff --git a/cmd/keytransparency-sequencer/main.go b/cmd/keytransparency-sequencer/main.go
index b2931f228..71f0d8035 100644
--- a/cmd/keytransparency-sequencer/main.go
+++ b/cmd/keytransparency-sequencer/main.go
@@ -16,7 +16,6 @@ package main
 
 import (
 	"context"
-	"database/sql"
 	"flag"
 	"fmt"
 	"net"
@@ -44,10 +43,10 @@ import (
 	pb "github.com/google/keytransparency/core/api/v1/keytransparency_go_proto"
 	dir "github.com/google/keytransparency/core/directory"
 	spb "github.com/google/keytransparency/core/sequencer/sequencer_go_proto"
+	ktsql "github.com/google/keytransparency/impl/sql"
 	etcdelect "github.com/google/trillian/util/election2/etcd"
 	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
 
-	_ "github.com/go-sql-driver/mysql" // Set database engine.
 	_ "github.com/google/trillian/crypto/keys/der/proto"
 	_ "github.com/google/trillian/merkle/coniks"  // Register hasher
 	_ "github.com/google/trillian/merkle/rfc6962" // Register hasher
@@ -74,17 +73,6 @@ var (
 	batchSize  = flag.Int("batch-size", 100, "Maximum number of mutations to process per map revision")
 )
 
-func openDB() *sql.DB {
-	db, err := sql.Open("mysql", *serverDBPath)
-	if err != nil {
-		glog.Exitf("sql.Open(): %v", err)
-	}
-	if err := db.Ping(); err != nil {
-		glog.Exitf("db.Ping(): %v", err)
-	}
-	return db
-}
-
 // getElectionFactory returns an election factory based on flags, and a
 // function which releases the resources associated with the factory.
 func getElectionFactory() (election2.Factory, func()) {
@@ -130,7 +118,10 @@ func main() {
 	}
 
 	// Database tables
-	sqldb := openDB()
+	sqldb, err := ktsql.Open(*serverDBPath)
+	if err != nil {
+		glog.Exit(err)
+	}
 	defer sqldb.Close()
 
 	mutations, err := mutationstorage.New(sqldb)
diff --git a/cmd/keytransparency-server/main.go b/cmd/keytransparency-server/main.go
index 60e86222f..d65711153 100644
--- a/cmd/keytransparency-server/main.go
+++ b/cmd/keytransparency-server/main.go
@@ -16,7 +16,6 @@ package main
 
 import (
 	"context"
-	"database/sql"
 	"flag"
 	"net/http"
 
@@ -37,11 +36,11 @@ import (
 	"github.com/google/keytransparency/impl/sql/mutationstorage"
 
 	pb "github.com/google/keytransparency/core/api/v1/keytransparency_go_proto"
+	ktsql "github.com/google/keytransparency/impl/sql"
 	grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
 	grpc_auth "github.com/grpc-ecosystem/go-grpc-middleware/auth"
 	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
 
-	_ "github.com/go-sql-driver/mysql" // Set database engine.
 	_ "github.com/google/trillian/crypto/keys/der/proto"
 )
 
@@ -58,23 +57,15 @@ var (
 	revisionPageSize = flag.Int("revision-page-size", 10, "Max number of revisions to return at once")
 )
 
-func openDB() *sql.DB {
-	db, err := sql.Open("mysql", *serverDBPath)
-	if err != nil {
-		glog.Exitf("sql.Open(): %v", err)
-	}
-	if err := db.Ping(); err != nil {
-		glog.Exitf("db.Ping(): %v", err)
-	}
-	return db
-}
-
 func main() {
 	flag.Parse()
 	ctx := context.Background()
 
 	// Open Resources.
-	sqldb := openDB()
+	sqldb, err := ktsql.Open(*serverDBPath)
+	if err != nil {
+		glog.Exit(err)
+	}
 	defer sqldb.Close()
 
 	creds, err := credentials.NewServerTLSFromFile(*certFile, *keyFile)
diff --git a/core/integration/storagetest/batch.go b/core/integration/storagetest/batch.go
index 556730470..636a86a78 100644
--- a/core/integration/storagetest/batch.go
+++ b/core/integration/storagetest/batch.go
@@ -27,15 +27,13 @@ import (
 	spb "github.com/google/keytransparency/core/sequencer/sequencer_go_proto"
 )
 
-// Batcher writes batch definitions to storage.
-type Batcher = sequencer.Batcher
+// batchStorageFactory returns a new database object, and a function for cleaning it up.
+type batchStorageFactory func(ctx context.Context, t *testing.T, dirID string) (sequencer.Batcher, func(context.Context))
 
-type BatchStorageFactory func(ctx context.Context, t *testing.T, dirID string) Batcher
-
-type BatchStorageTest func(ctx context.Context, t *testing.T, f BatchStorageFactory)
+type BatchStorageTest func(ctx context.Context, t *testing.T, f batchStorageFactory)
 
 // RunBatchStorageTests runs all the batch storage tests against the provided map storage implementation.
-func RunBatchStorageTests(t *testing.T, factory BatchStorageFactory) {
+func RunBatchStorageTests(t *testing.T, factory batchStorageFactory) {
 	ctx := context.Background()
 	b := &BatchTests{}
 	for name, f := range map[string]BatchStorageTest{
@@ -52,9 +50,10 @@ func RunBatchStorageTests(t *testing.T, factory BatchStorageFactory) {
 // BatchTests is a suite of tests to run against
 type BatchTests struct{}
 
-func (*BatchTests) TestNotFound(ctx context.Context, t *testing.T, f BatchStorageFactory) {
+func (*BatchTests) TestNotFound(ctx context.Context, t *testing.T, f batchStorageFactory) {
 	domainID := "testnotfounddir"
-	b := f(ctx, t, domainID)
+	b, done := f(ctx, t, domainID)
+	defer done(ctx)
 	_, err := b.ReadBatch(ctx, domainID, 0)
 	st := status.Convert(err)
 	if got, want := st.Code(), codes.NotFound; got != want {
@@ -62,9 +61,10 @@ func (*BatchTests) TestNotFound(ctx context.Context, t *testing.T, f BatchStorag
 	}
 }
 
-func (*BatchTests) TestWriteBatch(ctx context.Context, t *testing.T, f BatchStorageFactory) {
+func (*BatchTests) TestWriteBatch(ctx context.Context, t *testing.T, f batchStorageFactory) {
 	domainID := "writebatchtest"
-	b := f(ctx, t, domainID)
+	b, done := f(ctx, t, domainID)
+	defer done(ctx)
 	for _, tc := range []struct {
 		rev     int64
 		wantErr bool
@@ -86,9 +86,10 @@ func (*BatchTests) TestWriteBatch(ctx context.Context, t *testing.T, f BatchStor
 	}
 }
 
-func (*BatchTests) TestReadBatch(ctx context.Context, t *testing.T, f BatchStorageFactory) {
+func (*BatchTests) TestReadBatch(ctx context.Context, t *testing.T, f batchStorageFactory) {
 	domainID := "readbatchtest"
-	b := f(ctx, t, domainID)
+	b, done := f(ctx, t, domainID)
+	defer done(ctx)
 	for _, tc := range []struct {
 		rev  int64
 		want *spb.MapMetadata
@@ -115,9 +116,10 @@ func (*BatchTests) TestReadBatch(ctx context.Context, t *testing.T, f BatchStora
 	}
 }
 
-func (*BatchTests) TestHighestRev(ctx context.Context, t *testing.T, f BatchStorageFactory) {
+func (*BatchTests) TestHighestRev(ctx context.Context, t *testing.T, f batchStorageFactory) {
 	domainID := "writebatchtest"
-	b := f(ctx, t, domainID)
+	b, done := f(ctx, t, domainID)
+	defer done(ctx)
 	for _, tc := range []struct {
 		rev     int64
 		sources []*spb.MapMetadata_SourceSlice
diff --git a/core/integration/storagetest/mutation_logs.go b/core/integration/storagetest/mutation_logs.go
index f955892e4..389473054 100644
--- a/core/integration/storagetest/mutation_logs.go
+++ b/core/integration/storagetest/mutation_logs.go
@@ -25,13 +25,14 @@ import (
 	pb "github.com/google/keytransparency/core/api/v1/keytransparency_go_proto"
 )
 
-type MutationLogsFactory func(ctx context.Context, t *testing.T, dirID string, logIDs ...int64) keyserver.MutationLogs
+// mutationLogsFactory returns a new database object, and a function for cleaning it up.
+type mutationLogsFactory func(ctx context.Context, t *testing.T, dirID string, logIDs ...int64) (keyserver.MutationLogs, func(context.Context))
 
 // RunMutationLogsTests runs all the tests against the provided storage implementation.
-func RunMutationLogsTests(t *testing.T, factory MutationLogsFactory) {
+func RunMutationLogsTests(t *testing.T, factory mutationLogsFactory) {
 	ctx := context.Background()
 	b := &mutationLogsTests{}
-	for name, f := range map[string]func(ctx context.Context, t *testing.T, f MutationLogsFactory){
+	for name, f := range map[string]func(ctx context.Context, t *testing.T, f mutationLogsFactory){
 		// TODO(gbelvin): Discover test methods via reflection.
 		"TestReadLog": b.TestReadLog,
 	} {
@@ -51,10 +52,11 @@ func mustMarshal(t *testing.T, p proto.Message) []byte {
 }
 
 // TestReadLog ensures that reads happen in atomic units of batch size.
-func (mutationLogsTests) TestReadLog(ctx context.Context, t *testing.T, newForTest MutationLogsFactory) {
+func (mutationLogsTests) TestReadLog(ctx context.Context, t *testing.T, newForTest mutationLogsFactory) {
 	directoryID := "TestReadLog"
 	logID := int64(5) // Any log ID.
-	m := newForTest(ctx, t, directoryID, logID)
+	m, done := newForTest(ctx, t, directoryID, logID)
+	defer done(ctx)
 	// Write ten batches, three entries each.
 	for i := byte(0); i < 10; i++ {
 		entry := &pb.EntryUpdate{Mutation: &pb.SignedEntry{Entry: mustMarshal(t, &pb.Entry{Index: []byte{i}})}}
diff --git a/core/integration/storagetest/mutation_logs_admin.go b/core/integration/storagetest/mutation_logs_admin.go
index a09089f90..49effebcf 100644
--- a/core/integration/storagetest/mutation_logs_admin.go
+++ b/core/integration/storagetest/mutation_logs_admin.go
@@ -24,13 +24,14 @@ import (
 	"google.golang.org/grpc/status"
 )
 
-type LogsAdminFactory func(ctx context.Context, t *testing.T, dirID string, logIDs ...int64) adminserver.LogsAdmin
+// logAdminFactory returns a new database object, and a function for cleaning it up.
+type logAdminFactory func(ctx context.Context, t *testing.T, dirID string, logIDs ...int64) (adminserver.LogsAdmin, func(context.Context))
 
 // RunLogsAdminTests runs all the admin tests against the provided storage implementation.
-func RunLogsAdminTests(t *testing.T, factory LogsAdminFactory) {
+func RunLogsAdminTests(t *testing.T, factory logAdminFactory) {
 	ctx := context.Background()
 	b := &logsAdminTests{}
-	for name, f := range map[string]func(ctx context.Context, t *testing.T, f LogsAdminFactory){
+	for name, f := range map[string]func(ctx context.Context, t *testing.T, f logAdminFactory){
 		// TODO(gbelvin): Discover test methods via reflection.
 		"TestSetWritable": b.TestSetWritable,
 		"TestListLogs":    b.TestListLogs,
@@ -41,15 +42,16 @@ func RunLogsAdminTests(t *testing.T, factory LogsAdminFactory) {
 
 type logsAdminTests struct{}
 
-func (logsAdminTests) TestSetWritable(ctx context.Context, t *testing.T, f LogsAdminFactory) {
+func (logsAdminTests) TestSetWritable(ctx context.Context, t *testing.T, f logAdminFactory) {
 	directoryID := "TestSetWritable"
-	m := f(ctx, t, directoryID, 1)
+	m, done := f(ctx, t, directoryID, 1)
+	defer done(ctx)
 	if st := status.Convert(m.SetWritable(ctx, directoryID, 2, true)); st.Code() != codes.NotFound {
 		t.Errorf("SetWritable(non-existent logid): %v, want %v", st, codes.NotFound)
 	}
 }
 
-func (logsAdminTests) TestListLogs(ctx context.Context, t *testing.T, f LogsAdminFactory) {
+func (logsAdminTests) TestListLogs(ctx context.Context, t *testing.T, f logAdminFactory) {
 	directoryID := "TestListLogs"
 	for _, tc := range []struct {
 		desc        string
@@ -64,7 +66,8 @@ func (logsAdminTests) TestListLogs(ctx context.Context, t *testing.T, f LogsAdmi
 		{desc: "multi", logIDs: []int64{1, 2, 3}, setWritable: map[int64]bool{1: true, 2: false}, wantLogIDs: []int64{1, 3}},
 	} {
 		t.Run(tc.desc, func(t *testing.T) {
-			m := f(ctx, t, directoryID, tc.logIDs...)
+			m, done := f(ctx, t, directoryID, tc.logIDs...)
+			defer done(ctx)
 			wantLogs := make(map[int64]bool)
 			for _, logID := range tc.wantLogIDs {
 				wantLogs[logID] = true
diff --git a/go.mod b/go.mod
index bbb38f4a0..0435b0bcc 100644
--- a/go.mod
+++ b/go.mod
@@ -28,7 +28,6 @@ require (
 	github.com/kylelemons/godebug v1.1.0
 	github.com/lyft/protoc-gen-validate v0.1.0 // indirect
 	github.com/mattn/go-isatty v0.0.9 // indirect
-	github.com/mattn/go-sqlite3 v1.10.0
 	github.com/mwitkow/go-proto-validators v0.1.0 // indirect
 	github.com/prometheus/client_golang v1.1.0
 	github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 // indirect
diff --git a/impl/integration/env.go b/impl/integration/env.go
index a55c8fa4a..5c0f329b2 100644
--- a/impl/integration/env.go
+++ b/impl/integration/env.go
@@ -53,7 +53,6 @@ import (
 
 	_ "github.com/google/trillian/merkle/coniks"  // Register hasher
 	_ "github.com/google/trillian/merkle/rfc6962" // Register hasher
-	_ "github.com/mattn/go-sqlite3"               // Use sqlite database for testing.
 )
 
 var (
diff --git a/impl/sql/directory/storage_test.go b/impl/sql/directory/storage_test.go
index 4fc6baf5c..f2d4d5c43 100644
--- a/impl/sql/directory/storage_test.go
+++ b/impl/sql/directory/storage_test.go
@@ -16,40 +16,35 @@ package directory
 
 import (
 	"context"
-	"database/sql"
 	"testing"
 	"time"
 
 	"github.com/golang/protobuf/proto"
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/keytransparency/core/directory"
-	tpb "github.com/google/trillian"
+	"github.com/google/keytransparency/impl/sql/testdb"
 	"github.com/google/trillian/crypto/keyspb"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 
-	_ "github.com/mattn/go-sqlite3"
+	tpb "github.com/google/trillian"
 )
 
-func newStorage(t *testing.T) (s directory.Storage, close func()) {
+func newStorage(ctx context.Context, t *testing.T) (directory.Storage, func(context.Context)) {
 	t.Helper()
-	db, err := sql.Open("sqlite3", ":memory:")
-	if err != nil {
-		t.Fatalf("sql.Open(): %v", err)
-	}
-	closeFunc := func() { db.Close() }
-	s, err = NewStorage(db)
+	db, done := testdb.NewForTest(ctx, t)
+	s, err := NewStorage(db)
 	if err != nil {
-		closeFunc()
+		done(ctx)
 		t.Fatalf("Failed to create adminstorage: %v", err)
 	}
-	return s, closeFunc
+	return s, done
 }
 
 func TestList(t *testing.T) {
 	ctx := context.Background()
-	s, closeF := newStorage(t)
-	defer closeF()
+	s, done := newStorage(ctx, t)
+	defer done(ctx)
 	for _, tc := range []struct {
 		directories []*directory.Directory
 		readDeleted bool
@@ -105,8 +100,8 @@ func TestList(t *testing.T) {
 
 func TestWriteReadDelete(t *testing.T) {
 	ctx := context.Background()
-	s, closeF := newStorage(t)
-	defer closeF()
+	s, done := newStorage(ctx, t)
+	defer done(ctx)
 	for _, tc := range []struct {
 		desc                 string
 		d                    directory.Directory
@@ -249,8 +244,8 @@ func TestWriteReadDelete(t *testing.T) {
 
 func TestDelete(t *testing.T) {
 	ctx := context.Background()
-	s, closeF := newStorage(t)
-	defer closeF()
+	s, done := newStorage(ctx, t)
+	defer done(ctx)
 	for _, tc := range []struct {
 		directoryID string
 	}{
diff --git a/impl/sql/mutationstorage/mutation_logs_test.go b/impl/sql/mutationstorage/mutation_logs_test.go
index 18dd0f2dd..7fd2dacc1 100644
--- a/impl/sql/mutationstorage/mutation_logs_test.go
+++ b/impl/sql/mutationstorage/mutation_logs_test.go
@@ -24,34 +24,35 @@ import (
 	"github.com/google/keytransparency/core/adminserver"
 	"github.com/google/keytransparency/core/integration/storagetest"
 	"github.com/google/keytransparency/core/keyserver"
+	"github.com/google/keytransparency/impl/sql/testdb"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 
 	pb "github.com/google/keytransparency/core/api/v1/keytransparency_go_proto"
-	_ "github.com/mattn/go-sqlite3"
 )
 
-func newForTest(ctx context.Context, t testing.TB, dirID string, logIDs ...int64) *Mutations {
-	m, err := New(newDB(t))
+func newForTest(ctx context.Context, t testing.TB, dirID string, logIDs ...int64) (*Mutations, func(context.Context)) {
+	db, done := testdb.NewForTest(ctx, t)
+	m, err := New(db)
 	if err != nil {
 		t.Fatalf("Failed to create mutation storage: %v", err)
 	}
 	if err := m.AddLogs(ctx, dirID, logIDs...); err != nil {
 		t.Fatalf("AddLogs(): %v", err)
 	}
-	return m
+	return m, done
 }
 
 func TestMutationLogsIntegration(t *testing.T) {
 	storagetest.RunMutationLogsTests(t,
-		func(ctx context.Context, t *testing.T, dirID string, logIDs ...int64) keyserver.MutationLogs {
+		func(ctx context.Context, t *testing.T, dirID string, logIDs ...int64) (keyserver.MutationLogs, func(context.Context)) {
 			return newForTest(ctx, t, dirID, logIDs...)
 		})
 }
 
 func TestLogsAdminIntegration(t *testing.T) {
 	storagetest.RunLogsAdminTests(t,
-		func(ctx context.Context, t *testing.T, dirID string, logIDs ...int64) adminserver.LogsAdmin {
+		func(ctx context.Context, t *testing.T, dirID string, logIDs ...int64) (adminserver.LogsAdmin, func(context.Context)) {
 			return newForTest(ctx, t, dirID, logIDs...)
 		})
 }
@@ -75,7 +76,8 @@ func TestRandLog(t *testing.T) {
 		}},
 	} {
 		t.Run(tc.desc, func(t *testing.T) {
-			m := newForTest(ctx, t, directoryID, tc.send...)
+			m, done := newForTest(ctx, t, directoryID, tc.send...)
+			defer done(ctx)
 			logs := make(map[int64]bool)
 			for i := 0; i < 10*len(tc.wantLogs); i++ {
 				logID, err := m.randLog(ctx, directoryID)
@@ -98,7 +100,8 @@ func BenchmarkSend(b *testing.B) {
 	ctx := context.Background()
 	directoryID := "BenchmarkSend"
 	logID := int64(1)
-	m := newForTest(ctx, b, directoryID, logID)
+	m, done := newForTest(ctx, b, directoryID, logID)
+	defer done(ctx)
 
 	update := &pb.EntryUpdate{Mutation: &pb.SignedEntry{Entry: []byte("xxxxxxxxxxxxxxxxxx")}}
 	for _, tc := range []struct {
@@ -132,7 +135,8 @@ func TestSend(t *testing.T) {
 	ctx := context.Background()
 
 	directoryID := "TestSend"
-	m := newForTest(ctx, t, directoryID, 1, 2)
+	m, done := newForTest(ctx, t, directoryID, 1, 2)
+	defer done(ctx)
 	update := []byte("bar")
 	ts1 := time.Now()
 	ts2 := ts1.Add(time.Duration(1))
@@ -162,7 +166,8 @@ func TestWatermark(t *testing.T) {
 	ctx := context.Background()
 	directoryID := "TestWatermark"
 	logIDs := []int64{1, 2}
-	m := newForTest(ctx, t, directoryID, logIDs...)
+	m, done := newForTest(ctx, t, directoryID, logIDs...)
+	defer done(ctx)
 	update := []byte("bar")
 
 	start := time.Now()
diff --git a/impl/sql/mutationstorage/mutations_test.go b/impl/sql/mutationstorage/mutations_test.go
index fdd486f82..2e95ff3a7 100644
--- a/impl/sql/mutationstorage/mutations_test.go
+++ b/impl/sql/mutationstorage/mutations_test.go
@@ -15,29 +15,21 @@ package mutationstorage
 
 import (
 	"context"
-	"database/sql"
 	"testing"
 
 	"github.com/google/keytransparency/core/integration/storagetest"
-
-	_ "github.com/mattn/go-sqlite3"
+	"github.com/google/keytransparency/core/sequencer"
+	"github.com/google/keytransparency/impl/sql/testdb"
 )
 
-func newDB(t testing.TB) *sql.DB {
-	db, err := sql.Open("sqlite3", ":memory:")
-	if err != nil {
-		t.Fatalf("sql.Open(): %v", err)
-	}
-	return db
-}
-
 func TestBatchIntegration(t *testing.T) {
-	storageFactory := func(context.Context, *testing.T, string) storagetest.Batcher {
-		m, err := New(newDB(t))
+	storageFactory := func(ctx context.Context, t *testing.T, _ string) (sequencer.Batcher, func(context.Context)) {
+		db, done := testdb.NewForTest(ctx, t)
+		m, err := New(db)
 		if err != nil {
 			t.Fatalf("Failed to create mutations: %v", err)
 		}
-		return m
+		return m, done
 	}
 
 	storagetest.RunBatchStorageTests(t, storageFactory)
diff --git a/impl/sql/open.go b/impl/sql/open.go
new file mode 100644
index 000000000..36ed7e814
--- /dev/null
+++ b/impl/sql/open.go
@@ -0,0 +1,39 @@
+// Copyright 2019 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package sql provides functions for interacting with MySQL.
+package sql
+
+import (
+	"database/sql"
+
+	"github.com/go-sql-driver/mysql"
+)
+
+// Open the MySQL database specified by the dsn string.
+func Open(dsn string) (*sql.DB, error) {
+	cfg, err := mysql.ParseDSN(dsn)
+	if err != nil {
+		return nil, err
+	}
+
+	// MySQL flags that affect storage logic.
+	cfg.ClientFoundRows = true // Return number of matching rows instead of rows changed.
+
+	db, err := sql.Open("mysql", cfg.FormatDSN())
+	if err != nil {
+		return nil, err
+	}
+	return db, db.Ping()
+}
diff --git a/impl/sql/testdb/testdb.go b/impl/sql/testdb/testdb.go
new file mode 100644
index 000000000..12c8f33d2
--- /dev/null
+++ b/impl/sql/testdb/testdb.go
@@ -0,0 +1,59 @@
+// Copyright 2019 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package testdb supports opening ephemeral databases for testing.
+package testdb
+
+import (
+	"context"
+	"database/sql"
+	"flag"
+	"fmt"
+	"log"
+	"testing"
+	"time"
+
+	ktsql "github.com/google/keytransparency/impl/sql"
+)
+
+var dataSourceURI = flag.String("test_mysql_uri", "root@tcp(127.0.0.1)/", "The MySQL URI to use when running tests")
+
+// NewForTest creates a temporary database.
+// Returns a function for deleting the database.
+func NewForTest(ctx context.Context, t testing.TB) (*sql.DB, func(context.Context)) {
+	db, err := ktsql.Open(*dataSourceURI)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	dbName := fmt.Sprintf("test_%v", time.Now().UnixNano())
+	if _, err := db.ExecContext(ctx, fmt.Sprintf("CREATE DATABASE `%s`", dbName)); err != nil {
+		t.Fatalf("Failed to create test database: %v", err)
+	}
+
+	// Open test database
+	db.Close()
+	db, err = ktsql.Open(*dataSourceURI + dbName)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	done := func(ctx context.Context) {
+		defer db.Close()
+		if _, err := db.ExecContext(ctx, fmt.Sprintf("DROP DATABASE `%s`", dbName)); err != nil {
+			log.Printf("Failed to drop test database %q: %v", dbName, err)
+		}
+	}
+	return db, done
+}
diff --git a/impl/sql/testdb/testdb_test.go b/impl/sql/testdb/testdb_test.go
new file mode 100644
index 000000000..60b57ef75
--- /dev/null
+++ b/impl/sql/testdb/testdb_test.go
@@ -0,0 +1,32 @@
+// Copyright 2019 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package testdb
+
+import (
+	"context"
+	"testing"
+)
+
+func TestNewForTest(t *testing.T) {
+	ctx := context.Background()
+	db, done := NewForTest(ctx, t)
+	if err := db.Ping(); err != nil {
+		t.Error(err)
+	}
+	done(ctx)
+	// Verify that the databse was really closed by looking for the
+	// error in the logs when running the test with -v
+	done(ctx)
+}

From b19838ef64ad54f9783af233e9d6b5bddc98729c Mon Sep 17 00:00:00 2001
From: Gary Belvin <gdb@google.com>
Date: Mon, 28 Oct 2019 16:08:42 +0000
Subject: [PATCH 04/22] Use testdb in integration tests (#1371)

* Use testdb in integration test
---
 impl/integration/env.go                       | 27 +++++++++----------
 impl/integration/main_test.go                 |  6 +----
 .../{queue.go => mutation_logs.go}            |  0
 impl/sql/testdb/testdb.go                     |  3 ++-
 4 files changed, 16 insertions(+), 20 deletions(-)
 rename impl/sql/mutationstorage/{queue.go => mutation_logs.go} (100%)

diff --git a/impl/integration/env.go b/impl/integration/env.go
index 5c0f329b2..7529de3b8 100644
--- a/impl/integration/env.go
+++ b/impl/integration/env.go
@@ -20,6 +20,7 @@ import (
 	"encoding/pem"
 	"fmt"
 	"net"
+	"testing"
 	"time"
 
 	"github.com/golang/glog"
@@ -41,10 +42,10 @@ import (
 	"github.com/google/keytransparency/impl/authorization"
 	"github.com/google/keytransparency/impl/sql/directory"
 	"github.com/google/keytransparency/impl/sql/mutationstorage"
+	"github.com/google/keytransparency/impl/sql/testdb"
 	"github.com/google/trillian/crypto/keys/der"
 	"github.com/google/trillian/crypto/keyspb"
 	"github.com/google/trillian/monitoring"
-	"github.com/google/trillian/storage/testdb"
 
 	pb "github.com/google/keytransparency/core/api/v1/keytransparency_go_proto"
 	spb "github.com/google/keytransparency/core/sequencer/sequencer_go_proto"
@@ -115,19 +116,17 @@ func keyFromPEM(p string) *any.Any {
 }
 
 // NewEnv sets up common resources for tests.
-func NewEnv(ctx context.Context) (*Env, error) {
+func NewEnv(ctx context.Context, t testing.TB) *Env {
+	t.Helper()
 	timeout := 6 * time.Second
 	directoryID := "integration"
 
-	db, dbDone, err := testdb.NewTrillianDB(ctx)
-	if err != nil {
-		return nil, fmt.Errorf("env: failed to open database: %v", err)
-	}
+	db, dbDone := testdb.NewForTest(ctx, t)
 
 	// Map server
 	mapEnv, err := ttest.NewMapEnv(ctx, false)
 	if err != nil {
-		return nil, fmt.Errorf("env: failed to create trillian map server: %v", err)
+		t.Fatalf("env: failed to create trillian map server: %v", err)
 	}
 
 	// Log server
@@ -135,17 +134,17 @@ func NewEnv(ctx context.Context) (*Env, error) {
 	unused := ""
 	logEnv, err := ttest.NewLogEnv(ctx, numSequencers, unused)
 	if err != nil {
-		return nil, fmt.Errorf("env: failed to create trillian log server: %v", err)
+		t.Fatalf("env: failed to create trillian log server: %v", err)
 	}
 
 	// Configure directory, which creates new map and log trees.
 	directoryStorage, err := directory.NewStorage(db)
 	if err != nil {
-		return nil, fmt.Errorf("env: failed to create directory storage: %v", err)
+		t.Fatalf("env: failed to create directory storage: %v", err)
 	}
 	mutations, err := mutationstorage.New(db)
 	if err != nil {
-		return nil, fmt.Errorf("env: Failed to create mutations object: %v", err)
+		t.Fatalf("env: Failed to create mutations object: %v", err)
 	}
 	adminSvr := adminserver.New(logEnv.Log, mapEnv.Map, logEnv.Admin, mapEnv.Admin, directoryStorage, mutations, mutations, vrfKeyGen)
 	cctx, cancel := context.WithTimeout(ctx, timeout)
@@ -159,7 +158,7 @@ func NewEnv(ctx context.Context) (*Env, error) {
 		MapPrivateKey: keyFromPEM(mapPriv),
 	})
 	if err != nil {
-		return nil, fmt.Errorf("env: CreateDirectory(): %v", err)
+		t.Fatalf("env: CreateDirectory(): %v", err)
 	}
 	glog.V(5).Infof("Directory: %# v", pretty.Formatter(directoryPB))
 
@@ -168,7 +167,7 @@ func NewEnv(ctx context.Context) (*Env, error) {
 
 	lis, cc, err := Listen()
 	if err != nil {
-		return nil, fmt.Errorf("env: Listen(): %v", err)
+		t.Fatalf("env: Listen(): %v", err)
 	}
 
 	gsvr := grpc.NewServer(
@@ -205,7 +204,7 @@ func NewEnv(ctx context.Context) (*Env, error) {
 		func(lv *tclient.LogVerifier) verifier.LogTracker { return tracker.NewSynchronous(lv) },
 	)
 	if err != nil {
-		return nil, fmt.Errorf("error reading config: %v", err)
+		t.Fatalf("error reading config: %v", err)
 	}
 	// Integration tests manually create revisions immediately, so retry fairly quickly.
 	client.RetryDelay = 10 * time.Millisecond
@@ -227,7 +226,7 @@ func NewEnv(ctx context.Context) (*Env, error) {
 		grpcCC:     cc,
 		dbDone:     dbDone,
 		db:         db,
-	}, nil
+	}
 }
 
 // Close releases resources allocated by NewEnv.
diff --git a/impl/integration/main_test.go b/impl/integration/main_test.go
index 23e2ad60f..831266c0f 100644
--- a/impl/integration/main_test.go
+++ b/impl/integration/main_test.go
@@ -36,11 +36,7 @@ func TestIntegration(t *testing.T) {
 		t.Run(test.Name, func(t *testing.T) {
 			ctx, cancel := context.WithCancel(context.Background())
 			defer cancel()
-			env, err := NewEnv(ctx)
-			if err != nil {
-				t.Fatalf("Could not create Env: %v", err)
-			}
-
+			env := NewEnv(ctx, t)
 			defer env.Close()
 			cctx, cancel := context.WithCancel(ctx)
 			actions := test.Fn(cctx, env.Env, t)
diff --git a/impl/sql/mutationstorage/queue.go b/impl/sql/mutationstorage/mutation_logs.go
similarity index 100%
rename from impl/sql/mutationstorage/queue.go
rename to impl/sql/mutationstorage/mutation_logs.go
diff --git a/impl/sql/testdb/testdb.go b/impl/sql/testdb/testdb.go
index 12c8f33d2..d5d46f24d 100644
--- a/impl/sql/testdb/testdb.go
+++ b/impl/sql/testdb/testdb.go
@@ -27,11 +27,12 @@ import (
 	ktsql "github.com/google/keytransparency/impl/sql"
 )
 
-var dataSourceURI = flag.String("test_mysql_uri", "root@tcp(127.0.0.1)/", "The MySQL URI to use when running tests")
+var dataSourceURI = flag.String("kt_test_mysql_uri", "root@tcp(127.0.0.1)/", "The MySQL URI to use when running tests")
 
 // NewForTest creates a temporary database.
 // Returns a function for deleting the database.
 func NewForTest(ctx context.Context, t testing.TB) (*sql.DB, func(context.Context)) {
+	t.Helper()
 	db, err := ktsql.Open(*dataSourceURI)
 	if err != nil {
 		t.Fatal(err)

From ae9e7990c5eb3edecbef79b14f1d22f01f4a828a Mon Sep 17 00:00:00 2001
From: Gary Belvin <gdb@google.com>
Date: Wed, 30 Oct 2019 17:02:50 +0000
Subject: [PATCH 05/22] Switch Timestamp storage to mysql DATETIME (#1369)

* Scan to NullTime
* Store Microseconds -- timesout
* Store DATETIME with microsecond precision
* Adjust tests for MySQL min timestamp
* Remove unnecessary conversions
* Truncate input times to Micros
---
 core/integration/storagetest/mutation_logs.go |  5 ++-
 impl/sql/mutationstorage/mutation_logs.go     | 39 +++++++++++--------
 .../sql/mutationstorage/mutation_logs_test.go | 36 +++++++++--------
 impl/sql/mutationstorage/mutations.go         |  2 +-
 impl/sql/open.go                              |  3 ++
 5 files changed, 50 insertions(+), 35 deletions(-)

diff --git a/core/integration/storagetest/mutation_logs.go b/core/integration/storagetest/mutation_logs.go
index 389473054..a6351a113 100644
--- a/core/integration/storagetest/mutation_logs.go
+++ b/core/integration/storagetest/mutation_logs.go
@@ -51,6 +51,9 @@ func mustMarshal(t *testing.T, p proto.Message) []byte {
 	return b
 }
 
+// https://dev.mysql.com/doc/refman/8.0/en/datetime.html
+var minWatermark = time.Date(1000, 1, 1, 0, 0, 0, 0, time.UTC)
+
 // TestReadLog ensures that reads happen in atomic units of batch size.
 func (mutationLogsTests) TestReadLog(ctx context.Context, t *testing.T, newForTest mutationLogsFactory) {
 	directoryID := "TestReadLog"
@@ -75,7 +78,7 @@ func (mutationLogsTests) TestReadLog(ctx context.Context, t *testing.T, newForTe
 		{limit: 4, want: 6},    // Reading 4 items gets us into the second batch of size 3.
 		{limit: 100, want: 30}, // Reading all the items gets us the 30 items we wrote.
 	} {
-		rows, err := m.ReadLog(ctx, directoryID, logID, time.Time{}, time.Now(), tc.limit)
+		rows, err := m.ReadLog(ctx, directoryID, logID, minWatermark, time.Now(), tc.limit)
 		if err != nil {
 			t.Fatalf("ReadLog(%v): %v", tc.limit, err)
 		}
diff --git a/impl/sql/mutationstorage/mutation_logs.go b/impl/sql/mutationstorage/mutation_logs.go
index 1c8daec63..24ba9a575 100644
--- a/impl/sql/mutationstorage/mutation_logs.go
+++ b/impl/sql/mutationstorage/mutation_logs.go
@@ -153,23 +153,25 @@ func (m *Mutations) send(ctx context.Context, ts time.Time, directoryID string,
 		}
 	}()
 
-	var maxTime int64
+	var maxTime sql.NullTime
 	if err := tx.QueryRowContext(ctx,
-		`SELECT COALESCE(MAX(Time), 0) FROM Queue WHERE DirectoryID = ? AND LogID = ?;`,
+		`SELECT MAX(Time) FROM Queue WHERE DirectoryID = ? AND LogID = ?;`,
 		directoryID, logID).Scan(&maxTime); err != nil {
 		return status.Errorf(codes.Internal, "could not find max timestamp: %v", err)
 	}
-	tsTime := ts.UnixNano()
-	if tsTime <= maxTime {
+
+	// The Timestamp column has a maximum fidelity of microseconds.
+	// See https://dev.mysql.com/doc/refman/8.0/en/fractional-seconds.html
+	ts = ts.Truncate(time.Microsecond)
+	if !ts.After(maxTime.Time) {
 		return status.Errorf(codes.Aborted,
-			"current timestamp: %v, want > max-timestamp of queued mutations: %v",
-			tsTime, maxTime)
+			"current timestamp: %v, want > max-timestamp of queued mutations: %v", ts, maxTime)
 	}
 
 	for i, data := range mData {
 		if _, err = tx.ExecContext(ctx,
 			`INSERT INTO Queue (DirectoryID, LogID, Time, LocalID, Mutation) VALUES (?, ?, ?, ?, ?);`,
-			directoryID, logID, tsTime, i, data); err != nil {
+			directoryID, logID, ts, i, data); err != nil {
 			return status.Errorf(codes.Internal, "failed inserting into queue: %v", err)
 		}
 	}
@@ -181,20 +183,24 @@ func (m *Mutations) send(ctx context.Context, ts time.Time, directoryID string,
 func (m *Mutations) HighWatermark(ctx context.Context, directoryID string, logID int64,
 	start time.Time, batchSize int32) (int32, time.Time, error) {
 	var count int32
-	var high int64
+	var high sql.NullTime
 	if err := m.db.QueryRowContext(ctx,
-		`SELECT COUNT(*), COALESCE(MAX(T1.Time)+1, ?) FROM 
+		`SELECT COUNT(*), MAX(T1.Time) FROM
 		(
 			SELECT Q.Time FROM Queue as Q
 			WHERE Q.DirectoryID = ? AND Q.LogID = ? AND Q.Time >= ?
 			ORDER BY Q.Time ASC
 			LIMIT ?
 		) AS T1`,
-		start.UnixNano(), directoryID, logID, start.UnixNano(), batchSize).
+		directoryID, logID, start, batchSize).
 		Scan(&count, &high); err != nil {
-		return 0, time.Time{}, err
+		return 0, start, err
+	}
+	if count == 0 {
+		// When there are no rows, return the start time as the highest timestamp.
+		return 0, start, nil
 	}
-	return count, time.Unix(0, high), nil
+	return count, high.Time.Add(1 * time.Microsecond), nil
 }
 
 // ReadLog reads all mutations in logID between [low, high).
@@ -206,7 +212,7 @@ func (m *Mutations) ReadLog(ctx context.Context, directoryID string,
 		WHERE DirectoryID = ? AND LogID = ? AND Time >= ? AND Time < ?
 		ORDER BY Time, LocalID ASC
 		LIMIT ?;`,
-		directoryID, logID, low.UnixNano(), high.UnixNano(), batchSize)
+		directoryID, logID, low, high, batchSize)
 	if err != nil {
 		return nil, err
 	}
@@ -223,7 +229,7 @@ func (m *Mutations) ReadLog(ctx context.Context, directoryID string,
 			`SELECT Time, LocalID, Mutation FROM Queue
 			WHERE DirectoryID = ? AND LogID = ? AND Time = ? AND LocalID > ?
 			ORDER BY LocalID ASC;`,
-			directoryID, logID, last.ID.UnixNano(), last.LocalID)
+			directoryID, logID, last.ID, last.LocalID)
 		if err != nil {
 			return nil, err
 		}
@@ -241,7 +247,8 @@ func (m *Mutations) ReadLog(ctx context.Context, directoryID string,
 func readQueueMessages(rows *sql.Rows) ([]*mutator.LogMessage, error) {
 	results := make([]*mutator.LogMessage, 0)
 	for rows.Next() {
-		var timestamp, localID int64
+		var timestamp time.Time
+		var localID int64
 		var mData []byte
 		if err := rows.Scan(&timestamp, &localID, &mData); err != nil {
 			return nil, err
@@ -251,7 +258,7 @@ func readQueueMessages(rows *sql.Rows) ([]*mutator.LogMessage, error) {
 			return nil, err
 		}
 		results = append(results, &mutator.LogMessage{
-			ID:        time.Unix(0, timestamp),
+			ID:        timestamp,
 			LocalID:   localID,
 			Mutation:  entryUpdate.Mutation,
 			ExtraData: entryUpdate.Committed,
diff --git a/impl/sql/mutationstorage/mutation_logs_test.go b/impl/sql/mutationstorage/mutation_logs_test.go
index 7fd2dacc1..d05a85991 100644
--- a/impl/sql/mutationstorage/mutation_logs_test.go
+++ b/impl/sql/mutationstorage/mutation_logs_test.go
@@ -138,9 +138,9 @@ func TestSend(t *testing.T) {
 	m, done := newForTest(ctx, t, directoryID, 1, 2)
 	defer done(ctx)
 	update := []byte("bar")
-	ts1 := time.Now()
-	ts2 := ts1.Add(time.Duration(1))
-	ts3 := ts2.Add(time.Duration(1))
+	ts1 := time.Now().Truncate(time.Microsecond)
+	ts2 := ts1.Add(1 * time.Microsecond)
+	ts3 := ts2.Add(1 * time.Microsecond)
 
 	// Test cases are cumulative. Earlier test caes setup later test cases.
 	for _, tc := range []struct {
@@ -162,6 +162,9 @@ func TestSend(t *testing.T) {
 	}
 }
 
+// https://dev.mysql.com/doc/refman/8.0/en/datetime.html
+var minWatermark = time.Date(1000, 1, 1, 0, 0, 0, 0, time.UTC)
+
 func TestWatermark(t *testing.T) {
 	ctx := context.Background()
 	directoryID := "TestWatermark"
@@ -170,12 +173,11 @@ func TestWatermark(t *testing.T) {
 	defer done(ctx)
 	update := []byte("bar")
 
-	start := time.Now()
-	for ts := start; ts.Before(start.Add(10)); ts = ts.Add(1) {
-		for _, logID := range logIDs {
-			if err := m.send(ctx, ts, directoryID, logID, update); err != nil {
-				t.Fatalf("m.send(%v): %v", logID, err)
-			}
+	start := time.Now().Truncate(time.Microsecond)
+	for ts := start; ts.Before(start.Add(10 * time.Microsecond)); ts = ts.Add(1 * time.Microsecond) {
+		logID := int64(1)
+		if err := m.send(ctx, ts, directoryID, logID, update); err != nil {
+			t.Fatalf("m.send(%v): %v", logID, err)
 		}
 	}
 
@@ -187,13 +189,13 @@ func TestWatermark(t *testing.T) {
 		count     int32
 		want      time.Time
 	}{
-		{desc: "log1 max", logID: 1, batchSize: 100, want: start.Add(10), count: 10},
-		{desc: "log2 max", logID: 2, batchSize: 100, want: start.Add(10), count: 10},
-		{desc: "batch0", logID: 1, batchSize: 0, start: time.Unix(0, 0), want: time.Unix(0, 0)},
-		{desc: "batch0start55", logID: 1, start: time.Unix(0, 55), batchSize: 0, want: time.Unix(0, 55)},
-		{desc: "batch5", logID: 1, batchSize: 5, want: start.Add(5), count: 5},
-		{desc: "start1", logID: 1, start: start.Add(2), batchSize: 5, want: start.Add(7), count: 5},
-		{desc: "start8", logID: 1, start: start.Add(8), batchSize: 5, want: start.Add(10), count: 2},
+		{desc: "log1 max", logID: 1, batchSize: 100, start: start, want: start.Add(10 * time.Microsecond), count: 10},
+		{desc: "log2 empty", logID: 2, batchSize: 100, start: start, want: start},
+		{desc: "batch0", logID: 1, batchSize: 0, start: minWatermark, want: minWatermark},
+		{desc: "batch0start55", logID: 1, start: minWatermark.Add(55 * time.Microsecond), batchSize: 0, want: minWatermark.Add(55 * time.Microsecond)},
+		{desc: "batch5", logID: 1, start: start, batchSize: 5, want: start.Add(5 * time.Microsecond), count: 5},
+		{desc: "start1", logID: 1, start: start.Add(2 * time.Microsecond), batchSize: 5, want: start.Add(7 * time.Microsecond), count: 5},
+		{desc: "start8", logID: 1, start: start.Add(8 * time.Microsecond), batchSize: 5, want: start.Add(10 * time.Microsecond), count: 2},
 	} {
 		t.Run(tc.desc, func(t *testing.T) {
 			count, got, err := m.HighWatermark(ctx, directoryID, tc.logID, tc.start, tc.batchSize)
@@ -201,7 +203,7 @@ func TestWatermark(t *testing.T) {
 				t.Errorf("highWatermark(): %v", err)
 			}
 			if !got.Equal(tc.want) {
-				t.Errorf("highWatermark(%v) high: %v, want %v", tc.start, got.UnixNano(), tc.want)
+				t.Errorf("highWatermark(%v) high: %v, want %v", tc.start, got, tc.want)
 			}
 			if count != tc.count {
 				t.Errorf("highWatermark(%v) count: %v, want %v", tc.start, count, tc.count)
diff --git a/impl/sql/mutationstorage/mutations.go b/impl/sql/mutationstorage/mutations.go
index e10adb732..f244bea6f 100644
--- a/impl/sql/mutationstorage/mutations.go
+++ b/impl/sql/mutationstorage/mutations.go
@@ -39,7 +39,7 @@ var (
 		`CREATE TABLE IF NOT EXISTS Queue (
 		DirectoryID VARCHAR(30)   NOT NULL,
 		LogID    BIGINT           NOT NULL,
-		Time     BIGINT           NOT NULL,
+		Time     DATETIME(6)      NOT NULL,
 		LocalID  BIGINT           NOT NULL,
 		Mutation BLOB             NOT NULL,
 		PRIMARY KEY(DirectoryID, LogID, Time, LocalID )
diff --git a/impl/sql/open.go b/impl/sql/open.go
index 36ed7e814..fe7d790a6 100644
--- a/impl/sql/open.go
+++ b/impl/sql/open.go
@@ -17,6 +17,7 @@ package sql
 
 import (
 	"database/sql"
+	"time"
 
 	"github.com/go-sql-driver/mysql"
 )
@@ -30,6 +31,8 @@ func Open(dsn string) (*sql.DB, error) {
 
 	// MySQL flags that affect storage logic.
 	cfg.ClientFoundRows = true // Return number of matching rows instead of rows changed.
+	cfg.ParseTime = true       // Parse time values to time.Time
+	cfg.Loc = time.UTC
 
 	db, err := sql.Open("mysql", cfg.FormatDSN())
 	if err != nil {

From b4814e14ed162ce476b89faa66f4013929c8bdd4 Mon Sep 17 00:00:00 2001
From: Gary Belvin <gdb@google.com>
Date: Tue, 5 Nov 2019 10:20:45 +0000
Subject: [PATCH 06/22] Break layering violation by using native types (#1374)

The storage layer should not have a dependency on the frontend library.

By breakinig this dependency, we can avoid import cycles in later PRs.
---
 core/integration/storagetest/mutation_logs.go  |  2 +-
 core/keyserver/keyserver.go                    | 18 +++++-------------
 core/keyserver/revisions_test.go               |  4 ++--
 impl/sql/mutationstorage/mutation_logs.go      | 13 ++++++-------
 impl/sql/mutationstorage/mutation_logs_test.go |  2 +-
 5 files changed, 15 insertions(+), 24 deletions(-)

diff --git a/core/integration/storagetest/mutation_logs.go b/core/integration/storagetest/mutation_logs.go
index a6351a113..ca57576bd 100644
--- a/core/integration/storagetest/mutation_logs.go
+++ b/core/integration/storagetest/mutation_logs.go
@@ -63,7 +63,7 @@ func (mutationLogsTests) TestReadLog(ctx context.Context, t *testing.T, newForTe
 	// Write ten batches, three entries each.
 	for i := byte(0); i < 10; i++ {
 		entry := &pb.EntryUpdate{Mutation: &pb.SignedEntry{Entry: mustMarshal(t, &pb.Entry{Index: []byte{i}})}}
-		if _, err := m.Send(ctx, directoryID, entry, entry, entry); err != nil {
+		if _, _, err := m.Send(ctx, directoryID, entry, entry, entry); err != nil {
 			t.Fatalf("Send(): %v", err)
 		}
 	}
diff --git a/core/keyserver/keyserver.go b/core/keyserver/keyserver.go
index d4a5af9fd..8097a92c4 100644
--- a/core/keyserver/keyserver.go
+++ b/core/keyserver/keyserver.go
@@ -65,17 +65,12 @@ func createMetrics(mf monitoring.MetricFactory) {
 		directoryIDLabel, logIDLabel)
 }
 
-// WriteWatermark is the metadata that Send creates.
-type WriteWatermark struct {
-	LogID     int64
-	Watermark time.Time
-}
-
 // MutationLogs provides sets of time ordered message logs.
 type MutationLogs interface {
 	// Send submits the whole group of mutations atomically to a random log.
 	// TODO(gbelvin): Create a batch level object to make it clear that this a batch of updates.
-	Send(ctx context.Context, directoryID string, mutation ...*pb.EntryUpdate) (*WriteWatermark, error)
+	// Returns the logID and timestamp that the mutation batch got written at.
+	Send(ctx context.Context, directoryID string, mutation ...*pb.EntryUpdate) (int64, time.Time, error)
 	// ReadLog returns the messages in the (low, high] range stored in the
 	// specified log. ReadLog always returns complete units of the original
 	// batches sent via Send, and will return  more items than limit if
@@ -661,16 +656,13 @@ func (s *Server) BatchQueueUserUpdate(ctx context.Context, in *pb.BatchQueueUser
 	tdone()
 
 	// Save mutation to the database.
-	wm, err := s.logs.Send(ctx, directory.DirectoryID, in.Updates...)
+	wmLogID, wmTime, err := s.logs.Send(ctx, directory.DirectoryID, in.Updates...)
 	if st := status.Convert(err); st.Code() != codes.OK {
 		glog.Errorf("mutations.Write failed: %v", err)
 		return nil, status.Errorf(st.Code(), "Mutation write error")
 	}
-	if wm != nil {
-		watermarkWritten.Set(float64(wm.Watermark.UnixNano()), directory.DirectoryID, fmt.Sprintf("%v", wm.LogID))
-		sequencerQueueWritten.Add(float64(len(in.Updates)), directory.DirectoryID, fmt.Sprintf("%v", wm.LogID))
-	}
-
+	watermarkWritten.Set(float64(wmTime.UnixNano()), directory.DirectoryID, fmt.Sprintf("%v", wmLogID))
+	sequencerQueueWritten.Add(float64(len(in.Updates)), directory.DirectoryID, fmt.Sprintf("%v", wmLogID))
 	return &empty.Empty{}, nil
 }
 
diff --git a/core/keyserver/revisions_test.go b/core/keyserver/revisions_test.go
index 5edfa26b6..117f12296 100644
--- a/core/keyserver/revisions_test.go
+++ b/core/keyserver/revisions_test.go
@@ -71,8 +71,8 @@ func (b batchStorage) ReadBatch(ctx context.Context, dirID string, rev int64) (*
 
 type mutations map[int64][]*mutator.LogMessage // Map of logID to Slice of LogMessages
 
-func (m *mutations) Send(ctx context.Context, dirID string, mutation ...*pb.EntryUpdate) (*WriteWatermark, error) {
-	return nil, errors.New("unimplemented")
+func (m *mutations) Send(ctx context.Context, dirID string, mutation ...*pb.EntryUpdate) (int64, time.Time, error) {
+	return 0, time.Time{}, errors.New("unimplemented")
 }
 
 func (m *mutations) ReadLog(ctx context.Context, dirID string,
diff --git a/impl/sql/mutationstorage/mutation_logs.go b/impl/sql/mutationstorage/mutation_logs.go
index 24ba9a575..2f22c3e05 100644
--- a/impl/sql/mutationstorage/mutation_logs.go
+++ b/impl/sql/mutationstorage/mutation_logs.go
@@ -22,7 +22,6 @@ import (
 
 	"github.com/golang/glog"
 	"github.com/golang/protobuf/proto"
-	"github.com/google/keytransparency/core/keyserver"
 	"github.com/google/keytransparency/core/mutator"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
@@ -67,20 +66,20 @@ func (m *Mutations) AddLogs(ctx context.Context, directoryID string, logIDs ...i
 // Send writes mutations to the leading edge (by sequence number) of the mutations table.
 // Returns the logID/watermark pair that was written, or nil if nothing was written.
 // TODO(gbelvin): Make updates a slice.
-func (m *Mutations) Send(ctx context.Context, directoryID string, updates ...*pb.EntryUpdate) (*keyserver.WriteWatermark, error) {
+func (m *Mutations) Send(ctx context.Context, directoryID string, updates ...*pb.EntryUpdate) (int64, time.Time, error) {
 	glog.Infof("mutationstorage: Send(%v, <mutation>)", directoryID)
 	if len(updates) == 0 {
-		return nil, nil
+		return 0, time.Time{}, nil
 	}
 	logID, err := m.randLog(ctx, directoryID)
 	if err != nil {
-		return nil, err
+		return 0, time.Time{}, err
 	}
 	updateData := make([][]byte, 0, len(updates))
 	for _, u := range updates {
 		data, err := proto.Marshal(u)
 		if err != nil {
-			return nil, err
+			return 0, time.Time{}, err
 		}
 		updateData = append(updateData, data)
 	}
@@ -88,9 +87,9 @@ func (m *Mutations) Send(ctx context.Context, directoryID string, updates ...*pb
 	// we get timestamp contention.
 	ts := time.Now()
 	if err := m.send(ctx, ts, directoryID, logID, updateData...); err != nil {
-		return nil, err
+		return 0, time.Time{}, err
 	}
-	return &keyserver.WriteWatermark{LogID: logID, Watermark: ts}, nil
+	return logID, ts, nil
 }
 
 // ListLogs returns a list of all logs for directoryID, optionally filtered for writable logs.
diff --git a/impl/sql/mutationstorage/mutation_logs_test.go b/impl/sql/mutationstorage/mutation_logs_test.go
index d05a85991..ffa62f95d 100644
--- a/impl/sql/mutationstorage/mutation_logs_test.go
+++ b/impl/sql/mutationstorage/mutation_logs_test.go
@@ -123,7 +123,7 @@ func BenchmarkSend(b *testing.B) {
 				updates = append(updates, update)
 			}
 			for n := 0; n < b.N; n++ {
-				if _, err := m.Send(ctx, directoryID, updates...); err != nil {
+				if _, _, err := m.Send(ctx, directoryID, updates...); err != nil {
 					b.Errorf("Send(): %v", err)
 				}
 			}

From d55590d9383b0834260c912edc204503bcb5f66b Mon Sep 17 00:00:00 2001
From: Gary Belvin <gdb@google.com>
Date: Tue, 5 Nov 2019 11:28:37 +0000
Subject: [PATCH 07/22] Init metrics for whole test file (#1373)

---
 core/sequencer/server_test.go | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/core/sequencer/server_test.go b/core/sequencer/server_test.go
index 1cdb21986..f01ad7bca 100644
--- a/core/sequencer/server_test.go
+++ b/core/sequencer/server_test.go
@@ -39,6 +39,10 @@ const directoryID = "directoryID"
 
 func fakeMetric(_ string) {}
 
+func init() {
+	initMetrics.Do(func() { createMetrics(monitoring.InertMetricFactory{}) })
+}
+
 // fakeLogs are indexed by logID, and nanoseconds from time 0
 type fakeLogs map[int64][]mutator.LogMessage
 
@@ -130,7 +134,6 @@ func TestDefiningRevisions(t *testing.T) {
 	// Verify that outstanding revisions prevent future revisions from being created.
 	ctx := context.Background()
 	mapRev := int64(2)
-	initMetrics.Do(func() { createMetrics(monitoring.InertMetricFactory{}) })
 	s := Server{
 		logs: fakeLogs{
 			0: make([]mutator.LogMessage, 10),

From 8cf3790cf3dfa4d74bb603a1f203c90dfb506048 Mon Sep 17 00:00:00 2001
From: Gary Belvin <gdb@google.com>
Date: Wed, 6 Nov 2019 14:30:45 +0000
Subject: [PATCH 08/22] Move the responsibility to pick an input log from
 storage to the frontend (#1376)

* Remove rand log impl
* Use passed in logID
* Remove logID from return values
* Implement randLog in keyserver
---
 core/integration/storagetest/mutation_logs.go |  2 +-
 core/keyserver/keyserver.go                   | 30 ++++++++++---
 core/keyserver/revisions_test.go              | 12 +++++-
 impl/sql/mutationstorage/mutation_logs.go     | 28 +++----------
 .../sql/mutationstorage/mutation_logs_test.go | 42 +------------------
 5 files changed, 42 insertions(+), 72 deletions(-)

diff --git a/core/integration/storagetest/mutation_logs.go b/core/integration/storagetest/mutation_logs.go
index ca57576bd..66825a0f0 100644
--- a/core/integration/storagetest/mutation_logs.go
+++ b/core/integration/storagetest/mutation_logs.go
@@ -63,7 +63,7 @@ func (mutationLogsTests) TestReadLog(ctx context.Context, t *testing.T, newForTe
 	// Write ten batches, three entries each.
 	for i := byte(0); i < 10; i++ {
 		entry := &pb.EntryUpdate{Mutation: &pb.SignedEntry{Entry: mustMarshal(t, &pb.Entry{Index: []byte{i}})}}
-		if _, _, err := m.Send(ctx, directoryID, entry, entry, entry); err != nil {
+		if _, err := m.Send(ctx, directoryID, logID, entry, entry, entry); err != nil {
 			t.Fatalf("Send(): %v", err)
 		}
 	}
diff --git a/core/keyserver/keyserver.go b/core/keyserver/keyserver.go
index 8097a92c4..32e7b47f5 100644
--- a/core/keyserver/keyserver.go
+++ b/core/keyserver/keyserver.go
@@ -18,6 +18,7 @@ package keyserver
 import (
 	"context"
 	"fmt"
+	"math/rand"
 	"runtime"
 	"sync"
 	"time"
@@ -67,16 +68,18 @@ func createMetrics(mf monitoring.MetricFactory) {
 
 // MutationLogs provides sets of time ordered message logs.
 type MutationLogs interface {
-	// Send submits the whole group of mutations atomically to a random log.
+	// Send submits the whole group of mutations atomically to a given log.
 	// TODO(gbelvin): Create a batch level object to make it clear that this a batch of updates.
-	// Returns the logID and timestamp that the mutation batch got written at.
-	Send(ctx context.Context, directoryID string, mutation ...*pb.EntryUpdate) (int64, time.Time, error)
+	// Returns the timestamp that the mutation batch got written at.
+	Send(ctx context.Context, directoryID string, logID int64, mutation ...*pb.EntryUpdate) (time.Time, error)
 	// ReadLog returns the messages in the (low, high] range stored in the
 	// specified log. ReadLog always returns complete units of the original
 	// batches sent via Send, and will return  more items than limit if
 	// needed to do so.
 	ReadLog(ctx context.Context, directoryID string, logID int64, low, high time.Time,
 		limit int32) ([]*mutator.LogMessage, error)
+	// ListLogs returns a list of logs, optionally filtered by the writable bit.
+	ListLogs(ctx context.Context, directoryID string, writable bool) ([]int64, error)
 }
 
 // BatchReader reads batch definitions.
@@ -655,8 +658,13 @@ func (s *Server) BatchQueueUserUpdate(ctx context.Context, in *pb.BatchQueueUser
 	}
 	tdone()
 
-	// Save mutation to the database.
-	wmLogID, wmTime, err := s.logs.Send(ctx, directory.DirectoryID, in.Updates...)
+	// Pick a random logID.  Note, this effectively picks a random QoS. See issue #1377.
+	// TODO(gbelvin): Define an explicit QoS / Load ballancing API.
+	wmLogID, err := s.randLog(ctx, directory.DirectoryID)
+	if st := status.Convert(err); st.Code() != codes.OK {
+		return nil, status.Errorf(st.Code(), "Could not pick a log to write to: %v", err)
+	}
+	wmTime, err := s.logs.Send(ctx, directory.DirectoryID, wmLogID, in.Updates...)
 	if st := status.Convert(err); st.Code() != codes.OK {
 		glog.Errorf("mutations.Write failed: %v", err)
 		return nil, status.Errorf(st.Code(), "Mutation write error")
@@ -666,6 +674,18 @@ func (s *Server) BatchQueueUserUpdate(ctx context.Context, in *pb.BatchQueueUser
 	return &empty.Empty{}, nil
 }
 
+func (s *Server) randLog(ctx context.Context, directoryID string) (int64, error) {
+	// TODO(gbelvin): Cache these results.
+	writable := true
+	logIDs, err := s.logs.ListLogs(ctx, directoryID, writable)
+	if err != nil {
+		return 0, err
+	}
+
+	// Return a random log.
+	return logIDs[rand.Intn(len(logIDs))], nil
+}
+
 // GetDirectory returns all info tied to the specified directory.
 //
 // This API to get all necessary data needed to verify a particular
diff --git a/core/keyserver/revisions_test.go b/core/keyserver/revisions_test.go
index 117f12296..ea74b1d84 100644
--- a/core/keyserver/revisions_test.go
+++ b/core/keyserver/revisions_test.go
@@ -71,8 +71,16 @@ func (b batchStorage) ReadBatch(ctx context.Context, dirID string, rev int64) (*
 
 type mutations map[int64][]*mutator.LogMessage // Map of logID to Slice of LogMessages
 
-func (m *mutations) Send(ctx context.Context, dirID string, mutation ...*pb.EntryUpdate) (int64, time.Time, error) {
-	return 0, time.Time{}, errors.New("unimplemented")
+func (m *mutations) Send(ctx context.Context, dirID string, _ int64, mutation ...*pb.EntryUpdate) (time.Time, error) {
+	return time.Time{}, errors.New("unimplemented")
+}
+
+func (m *mutations) ListLogs(ctx context.Context, dirID string, _ bool) ([]int64, error) {
+	logIDs := []int64{}
+	for id := range *m {
+		logIDs = append(logIDs, id)
+	}
+	return logIDs, nil
 }
 
 func (m *mutations) ReadLog(ctx context.Context, dirID string,
diff --git a/impl/sql/mutationstorage/mutation_logs.go b/impl/sql/mutationstorage/mutation_logs.go
index 2f22c3e05..e8c75b40f 100644
--- a/impl/sql/mutationstorage/mutation_logs.go
+++ b/impl/sql/mutationstorage/mutation_logs.go
@@ -17,7 +17,6 @@ package mutationstorage
 import (
 	"context"
 	"database/sql"
-	"math/rand"
 	"time"
 
 	"github.com/golang/glog"
@@ -66,20 +65,16 @@ func (m *Mutations) AddLogs(ctx context.Context, directoryID string, logIDs ...i
 // Send writes mutations to the leading edge (by sequence number) of the mutations table.
 // Returns the logID/watermark pair that was written, or nil if nothing was written.
 // TODO(gbelvin): Make updates a slice.
-func (m *Mutations) Send(ctx context.Context, directoryID string, updates ...*pb.EntryUpdate) (int64, time.Time, error) {
+func (m *Mutations) Send(ctx context.Context, directoryID string, logID int64, updates ...*pb.EntryUpdate) (time.Time, error) {
 	glog.Infof("mutationstorage: Send(%v, <mutation>)", directoryID)
 	if len(updates) == 0 {
-		return 0, time.Time{}, nil
-	}
-	logID, err := m.randLog(ctx, directoryID)
-	if err != nil {
-		return 0, time.Time{}, err
+		return time.Time{}, nil
 	}
 	updateData := make([][]byte, 0, len(updates))
 	for _, u := range updates {
 		data, err := proto.Marshal(u)
 		if err != nil {
-			return 0, time.Time{}, err
+			return time.Time{}, err
 		}
 		updateData = append(updateData, data)
 	}
@@ -87,9 +82,9 @@ func (m *Mutations) Send(ctx context.Context, directoryID string, updates ...*pb
 	// we get timestamp contention.
 	ts := time.Now()
 	if err := m.send(ctx, ts, directoryID, logID, updateData...); err != nil {
-		return 0, time.Time{}, err
+		return time.Time{}, err
 	}
-	return logID, ts, nil
+	return ts, nil
 }
 
 // ListLogs returns a list of all logs for directoryID, optionally filtered for writable logs.
@@ -123,19 +118,6 @@ func (m *Mutations) ListLogs(ctx context.Context, directoryID string, writable b
 	return logIDs, nil
 }
 
-// randLog returns a random, enabled log for directoryID.
-func (m *Mutations) randLog(ctx context.Context, directoryID string) (int64, error) {
-	// TODO(gbelvin): Cache these results.
-	writable := true
-	logIDs, err := m.ListLogs(ctx, directoryID, writable)
-	if err != nil {
-		return 0, err
-	}
-
-	// Return a random log.
-	return logIDs[rand.Intn(len(logIDs))], nil
-}
-
 // ts must be greater than all other timestamps currently recorded for directoryID.
 func (m *Mutations) send(ctx context.Context, ts time.Time, directoryID string,
 	logID int64, mData ...[]byte) (ret error) {
diff --git a/impl/sql/mutationstorage/mutation_logs_test.go b/impl/sql/mutationstorage/mutation_logs_test.go
index ffa62f95d..94c0e2af7 100644
--- a/impl/sql/mutationstorage/mutation_logs_test.go
+++ b/impl/sql/mutationstorage/mutation_logs_test.go
@@ -20,7 +20,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/google/go-cmp/cmp"
 	"github.com/google/keytransparency/core/adminserver"
 	"github.com/google/keytransparency/core/integration/storagetest"
 	"github.com/google/keytransparency/core/keyserver"
@@ -57,45 +56,6 @@ func TestLogsAdminIntegration(t *testing.T) {
 		})
 }
 
-func TestRandLog(t *testing.T) {
-	ctx := context.Background()
-	directoryID := "TestRandLog"
-
-	for _, tc := range []struct {
-		desc     string
-		send     []int64
-		wantCode codes.Code
-		wantLogs map[int64]bool
-	}{
-		{desc: "no rows", wantCode: codes.NotFound, wantLogs: map[int64]bool{}},
-		{desc: "one row", send: []int64{10}, wantLogs: map[int64]bool{10: true}},
-		{desc: "second", send: []int64{1, 2, 3}, wantLogs: map[int64]bool{
-			1: true,
-			2: true,
-			3: true,
-		}},
-	} {
-		t.Run(tc.desc, func(t *testing.T) {
-			m, done := newForTest(ctx, t, directoryID, tc.send...)
-			defer done(ctx)
-			logs := make(map[int64]bool)
-			for i := 0; i < 10*len(tc.wantLogs); i++ {
-				logID, err := m.randLog(ctx, directoryID)
-				if got, want := status.Code(err), tc.wantCode; got != want {
-					t.Errorf("randLog(): %v, want %v", got, want)
-				}
-				if err != nil {
-					break
-				}
-				logs[logID] = true
-			}
-			if got, want := logs, tc.wantLogs; !cmp.Equal(got, want) {
-				t.Errorf("logs: %v, want %v", got, want)
-			}
-		})
-	}
-}
-
 func BenchmarkSend(b *testing.B) {
 	ctx := context.Background()
 	directoryID := "BenchmarkSend"
@@ -123,7 +83,7 @@ func BenchmarkSend(b *testing.B) {
 				updates = append(updates, update)
 			}
 			for n := 0; n < b.N; n++ {
-				if _, _, err := m.Send(ctx, directoryID, updates...); err != nil {
+				if _, err := m.Send(ctx, directoryID, logID, updates...); err != nil {
 					b.Errorf("Send(): %v", err)
 				}
 			}

From 8d7e472a846d3567711e9699992e0d2b4b037770 Mon Sep 17 00:00:00 2001
From: Gary Belvin <gdb@google.com>
Date: Thu, 7 Nov 2019 11:16:01 +0000
Subject: [PATCH 09/22] Do pagination the right way (#1378)

* Do pagination the right way
---
 core/sequencer/server.go      | 21 ++++++++++-----------
 core/sequencer/server_test.go |  5 ++++-
 2 files changed, 14 insertions(+), 12 deletions(-)

diff --git a/core/sequencer/server.go b/core/sequencer/server.go
index f85b643bb..fe1875183 100644
--- a/core/sequencer/server.go
+++ b/core/sequencer/server.go
@@ -132,9 +132,7 @@ type LogsReader interface {
 	// or all logIDs associated with directoryID if writable is false.
 	ListLogs(ctx context.Context, directoryID string, writable bool) ([]int64, error)
 
-	// ReadLog returns the lowest messages in the (low, high] range stored in the
-	// specified log, up to batchSize.  Paginate by setting low to the
-	// highest LogMessage returned in the previous page.
+	// ReadLog returns the lowest messages in the [low, high) range stored in the specified log, up to batchSize.
 	ReadLog(ctx context.Context, directoryID string, logID int64, low, high time.Time,
 		batchSize int32) ([]*mutator.LogMessage, error)
 }
@@ -343,21 +341,22 @@ func (s *Server) readMessages(ctx context.Context, source *spb.MapMetadata_Sourc
 	ss := metadata.Source(source)
 	low := ss.StartTime()
 	high := ss.EndTime()
-	// Loop until less than chunkSize items are returned.
-	for count := chunkSize; count == chunkSize; {
-		batch, err := s.logs.ReadLog(ctx, directoryID, source.LogId, low, high, chunkSize)
+	for moreToRead := true; moreToRead; {
+		// Request one more item than chunkSize so we can find the next page token.
+		batch, err := s.logs.ReadLog(ctx, directoryID, source.LogId, low, high, chunkSize+1)
 		if err != nil {
 			return fmt.Errorf("logs.ReadLog(): %v", err)
 		}
-		count = int32(len(batch))
 		glog.Infof("ReadLog(dir: %v log: %v, (%v, %v], %v) count: %v",
-			directoryID, source.LogId, low, high, chunkSize, count)
+			directoryID, source.LogId, low, high, chunkSize, len(batch))
+		moreToRead = int32(len(batch)) == (chunkSize + 1)
+		if moreToRead {
+			low = batch[chunkSize].ID // Use the last row as the start of the next read.
+			batch = batch[:chunkSize] // Don't emit the next page token.
+		}
 		logEntryCount.Add(float64(len(batch)), directoryID, fmt.Sprintf("%v", source.LogId))
 		for _, m := range batch {
 			emit(m)
-			if m.ID.After(low) {
-				low = m.ID
-			}
 		}
 	}
 	return nil
diff --git a/core/sequencer/server_test.go b/core/sequencer/server_test.go
index f01ad7bca..780f7ef0e 100644
--- a/core/sequencer/server_test.go
+++ b/core/sequencer/server_test.go
@@ -220,6 +220,9 @@ func TestReadMessages(t *testing.T) {
 		{batchSize: 1, want: 9, meta: &spb.MapMetadata{Sources: []*spb.MapMetadata_SourceSlice{
 			{LogId: 0, LowestInclusive: 1, HighestExclusive: 10},
 		}}},
+		{batchSize: 10000, want: 9, meta: &spb.MapMetadata{Sources: []*spb.MapMetadata_SourceSlice{
+			{LogId: 0, LowestInclusive: 1, HighestExclusive: 10},
+		}}},
 		{batchSize: 1, want: 19, meta: &spb.MapMetadata{Sources: []*spb.MapMetadata_SourceSlice{
 			{LogId: 0, LowestInclusive: 1, HighestExclusive: 10},
 			{LogId: 1, LowestInclusive: 1, HighestExclusive: 11},
@@ -231,7 +234,7 @@ func TestReadMessages(t *testing.T) {
 			t.Errorf("readMessages(): %v", err)
 		}
 		if got := len(logItems); got != tc.want {
-			t.Errorf("readMessages(): len: %v, want %v", got, tc.want)
+			t.Errorf("readMessages(%v): len: %v, want %v", tc.meta, got, tc.want)
 		}
 	}
 }

From f1ff340c3c158718e04c20f06131d543697c5e8f Mon Sep 17 00:00:00 2001
From: Joe Tsai <joetsai@digital-static.net>
Date: Thu, 7 Nov 2019 03:32:09 -0800
Subject: [PATCH 10/22] Fix generic comparisons on protobuf messages (#1379)

Generated protobuf messages contain internal data structures
that general purpose comparison functions (e.g., reflect.DeepEqual,
pretty.Compare, etc) do not properly compare. It is already the case
today that these functions may report a difference when two messages
are actually semantically equivalent.

Fix all usages by either calling proto.Equal directly if
the top-level types are themselves proto.Message, or by calling
cmp.Equal with the cmp.Comparer(proto.Equal) option specified.
This option teaches cmp to use proto.Equal anytime it encounters
proto.Message types.
---
 core/sequencer/runner/native_test.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/core/sequencer/runner/native_test.go b/core/sequencer/runner/native_test.go
index 2534a4c66..e31e7abdf 100644
--- a/core/sequencer/runner/native_test.go
+++ b/core/sequencer/runner/native_test.go
@@ -17,6 +17,7 @@ package runner
 import (
 	"testing"
 
+	"github.com/golang/protobuf/proto"
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/keytransparency/core/mutator/entry"
 
@@ -52,7 +53,7 @@ func TestJoin(t *testing.T) {
 			for g := range Join(tc.leaves, tc.msgs, func(label string) { metrics[label]++ }) {
 				got = append(got, g)
 			}
-			if !cmp.Equal(got, tc.want) {
+			if !cmp.Equal(got, tc.want, cmp.Comparer(proto.Equal)) {
 				t.Errorf("Join(): %v, want %v\n diff: %v",
 					got, tc.want, cmp.Diff(got, tc.want))
 			}

From 36d41a9df1567365884fd7cf7cc416fd8aad5a9c Mon Sep 17 00:00:00 2001
From: Gary Belvin <gdb@google.com>
Date: Thu, 7 Nov 2019 12:07:49 +0000
Subject: [PATCH 11/22] In memory logs implementation (#1375)

* Memory impl of mutation logs
* Use memory impl in sequencer
* ReadLogExact test
* keyserver uses Send API
* Send uses a static clock - no Now() in tests
* Finish watermark tests using idx
---
 core/integration/storagetest/mutation_logs.go |  47 ++++++-
 core/keyserver/revisions_test.go              | 119 ++++++++---------
 core/sequencer/server_test.go                 | 111 +++++++--------
 impl/memory/mutation_logs.go                  | 126 ++++++++++++++++++
 impl/memory/mutation_logs_test.go             |  35 +++++
 5 files changed, 307 insertions(+), 131 deletions(-)
 create mode 100644 impl/memory/mutation_logs.go
 create mode 100644 impl/memory/mutation_logs_test.go

diff --git a/core/integration/storagetest/mutation_logs.go b/core/integration/storagetest/mutation_logs.go
index 66825a0f0..3fd1af560 100644
--- a/core/integration/storagetest/mutation_logs.go
+++ b/core/integration/storagetest/mutation_logs.go
@@ -20,6 +20,7 @@ import (
 	"time"
 
 	"github.com/golang/protobuf/proto"
+	"github.com/google/go-cmp/cmp"
 	"github.com/google/keytransparency/core/keyserver"
 
 	pb "github.com/google/keytransparency/core/api/v1/keytransparency_go_proto"
@@ -34,7 +35,8 @@ func RunMutationLogsTests(t *testing.T, factory mutationLogsFactory) {
 	b := &mutationLogsTests{}
 	for name, f := range map[string]func(ctx context.Context, t *testing.T, f mutationLogsFactory){
 		// TODO(gbelvin): Discover test methods via reflection.
-		"TestReadLog": b.TestReadLog,
+		"TestReadLog":      b.TestReadLog,
+		"TestReadLogExact": b.TestReadLogExact,
 	} {
 		t.Run(name, func(t *testing.T) { f(ctx, t, factory) })
 	}
@@ -60,7 +62,7 @@ func (mutationLogsTests) TestReadLog(ctx context.Context, t *testing.T, newForTe
 	logID := int64(5) // Any log ID.
 	m, done := newForTest(ctx, t, directoryID, logID)
 	defer done(ctx)
-	// Write ten batches, three entries each.
+	// Write ten batches.
 	for i := byte(0); i < 10; i++ {
 		entry := &pb.EntryUpdate{Mutation: &pb.SignedEntry{Entry: mustMarshal(t, &pb.Entry{Index: []byte{i}})}}
 		if _, err := m.Send(ctx, directoryID, logID, entry, entry, entry); err != nil {
@@ -87,3 +89,44 @@ func (mutationLogsTests) TestReadLog(ctx context.Context, t *testing.T, newForTe
 		}
 	}
 }
+
+// TestReadLogExact ensures that reads respect the low inclusive, high exclusive API.
+func (mutationLogsTests) TestReadLogExact(ctx context.Context, t *testing.T, newForTest mutationLogsFactory) {
+	directoryID := "TestReadLogExact"
+	logID := int64(5) // Any log ID.
+	m, done := newForTest(ctx, t, directoryID, logID)
+	defer done(ctx)
+	// Write ten batches.
+	idx := make([]time.Time, 0, 10)
+	for i := byte(0); i < 10; i++ {
+		entry := &pb.EntryUpdate{Mutation: &pb.SignedEntry{Entry: []byte{i}}}
+		ts, err := m.Send(ctx, directoryID, logID, entry)
+		if err != nil {
+			t.Fatalf("Send(): %v", err)
+		}
+		idx = append(idx, ts)
+	}
+
+	for _, tc := range []struct {
+		low, high int
+		want      []byte
+	}{
+		{low: 0, high: 0, want: []byte{}},
+		{low: 0, high: 1, want: []byte{0}},
+		{low: 0, high: 9, want: []byte{0, 1, 2, 3, 4, 5, 6, 7, 8}},
+		{low: 1, high: 9, want: []byte{1, 2, 3, 4, 5, 6, 7, 8}},
+	} {
+		rows, err := m.ReadLog(ctx, directoryID, logID, idx[tc.low], idx[tc.high], 100)
+		if err != nil {
+			t.Fatalf("ReadLog(): %v", err)
+		}
+		got := make([]byte, 0, len(rows))
+		for _, r := range rows {
+			i := r.Mutation.Entry[0]
+			got = append(got, i)
+		}
+		if !cmp.Equal(got, tc.want) {
+			t.Fatalf("ReadLog(%v,%v): got %v, want %v", tc.low, tc.high, got, tc.want)
+		}
+	}
+}
diff --git a/core/keyserver/revisions_test.go b/core/keyserver/revisions_test.go
index ea74b1d84..e09079c67 100644
--- a/core/keyserver/revisions_test.go
+++ b/core/keyserver/revisions_test.go
@@ -16,7 +16,6 @@ package keyserver
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"testing"
 	"time"
@@ -27,7 +26,8 @@ import (
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 
-	"github.com/google/keytransparency/core/mutator"
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/keytransparency/impl/memory"
 
 	protopb "github.com/golang/protobuf/ptypes/timestamp"
 	pb "github.com/google/keytransparency/core/api/v1/keytransparency_go_proto"
@@ -55,6 +55,30 @@ func genIndexes(start, end int64) [][]byte {
 	return indexes
 }
 
+func genEntryUpdates(t *testing.T, start, end int64) []*pb.EntryUpdate {
+	t.Helper()
+	entries := make([]*pb.EntryUpdate, 0)
+	for i := start; i < end; i++ {
+		entries = append(entries, &pb.EntryUpdate{
+			Mutation: &pb.SignedEntry{
+				Entry: mustMarshal(t, &pb.Entry{
+					Index: []byte(fmt.Sprintf("key_%v", i)),
+				}),
+			},
+		})
+	}
+	return entries
+}
+
+func timestamp(t *testing.T, ts time.Time) *protopb.Timestamp {
+	t.Helper()
+	ret, err := ptypes.TimestampProto(ts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return ret
+}
+
 func TestGetRevisionStream(t *testing.T) {
 	srv := &Server{}
 	err := srv.GetRevisionStream(nil, nil)
@@ -69,36 +93,6 @@ func (b batchStorage) ReadBatch(ctx context.Context, dirID string, rev int64) (*
 	return &spb.MapMetadata{Sources: b[rev]}, nil
 }
 
-type mutations map[int64][]*mutator.LogMessage // Map of logID to Slice of LogMessages
-
-func (m *mutations) Send(ctx context.Context, dirID string, _ int64, mutation ...*pb.EntryUpdate) (time.Time, error) {
-	return time.Time{}, errors.New("unimplemented")
-}
-
-func (m *mutations) ListLogs(ctx context.Context, dirID string, _ bool) ([]int64, error) {
-	logIDs := []int64{}
-	for id := range *m {
-		logIDs = append(logIDs, id)
-	}
-	return logIDs, nil
-}
-
-func (m *mutations) ReadLog(ctx context.Context, dirID string,
-	logID int64, low, high time.Time, batchSize int32) ([]*mutator.LogMessage, error) {
-	logShard := (*m)[logID]
-	if low.UnixNano() > int64(len(logShard)) {
-		return nil, fmt.Errorf("invalid argument: low: %v, want <= max watermark: %v", low, len(logShard))
-	}
-	count := high.UnixNano() - low.UnixNano()
-	if count > int64(batchSize) {
-		count = int64(batchSize)
-	}
-	if low.UnixNano()+count > int64(len(logShard)) {
-		count = int64(len(logShard)) - low.UnixNano() + 1
-	}
-	return logShard[low.UnixNano() : low.UnixNano()+count], nil
-}
-
 func MustEncodeToken(t *testing.T, low time.Time) string {
 	t.Helper()
 
@@ -119,30 +113,22 @@ func MustEncodeToken(t *testing.T, low time.Time) string {
 
 func TestListMutations(t *testing.T) {
 	ctx := context.Background()
-	fakeBatches := batchStorage{
-		1: SourceList{{LogId: 0, LowestInclusive: 2, HighestExclusive: 7}},
-		2: SourceList{{LogId: 0, LowestInclusive: 7, HighestExclusive: 11}},
+	dirID := "TestListMutations"
+	logID := int64(0)
+	fakeLogs := memory.NewMutationLogs()
+	idx := make([]time.Time, 0, 12)
+	for i := int64(0); i < 12; i++ {
+		// Send one entry.
+		ts, err := fakeLogs.Send(ctx, dirID, logID, genEntryUpdates(t, i, i+1)...)
+		if err != nil {
+			t.Fatal(err)
+		}
+		idx = append(idx, ts)
 	}
 
-	fakeLogs := make(mutations)
-	for _, sources := range fakeBatches {
-		for _, source := range sources {
-			extendBy := source.HighestExclusive - int64(len(fakeLogs[source.LogId]))
-			if extendBy > 0 {
-				fakeLogs[source.LogId] = append(fakeLogs[source.LogId], make([]*mutator.LogMessage, extendBy)...)
-			}
-			for i := source.LowestInclusive; i < source.HighestExclusive; i++ {
-				fakeLogs[source.LogId][i] = &mutator.LogMessage{
-					ID: time.Unix(0, i*int64(time.Nanosecond)),
-					Mutation: &pb.SignedEntry{
-						Entry: mustMarshal(t, &pb.Entry{
-							Index:      []byte(fmt.Sprintf("key_%v", i)),
-							Commitment: []byte(fmt.Sprintf("value_%v", i)),
-						}),
-					},
-				}
-			}
-		}
+	fakeBatches := batchStorage{
+		1: SourceList{{LogId: 0, LowestInclusive: idx[2].UnixNano(), HighestExclusive: idx[7].UnixNano()}},
+		2: SourceList{{LogId: 0, LowestInclusive: idx[7].UnixNano(), HighestExclusive: idx[11].UnixNano()}},
 	}
 
 	for _, tc := range []struct {
@@ -153,12 +139,12 @@ func TestListMutations(t *testing.T) {
 		wantNext   *rtpb.ReadToken
 		wantErr    bool
 	}{
-		{desc: "exact page", pageSize: 6, start: 2, end: 7, wantNext: &rtpb.ReadToken{}},
+		{desc: "first page", pageSize: 6, start: 2, end: 7, wantNext: &rtpb.ReadToken{}},
 		{desc: "large page", pageSize: 10, start: 2, end: 7, wantNext: &rtpb.ReadToken{}},
-		{desc: "partial", pageSize: 4, start: 2, end: 6, wantNext: &rtpb.ReadToken{StartTime: &protopb.Timestamp{Nanos: 6}}},
-		{desc: "large page with token", token: MustEncodeToken(t, time.Unix(0, 3)), pageSize: 10, start: 3, end: 7, wantNext: &rtpb.ReadToken{}},
-		{desc: "small page with token", token: MustEncodeToken(t, time.Unix(0, 3)), pageSize: 2, start: 3, end: 5,
-			wantNext: &rtpb.ReadToken{StartTime: &protopb.Timestamp{Nanos: 5}}},
+		{desc: "partial", pageSize: 4, start: 2, end: 6, wantNext: &rtpb.ReadToken{StartTime: timestamp(t, idx[6])}},
+		{desc: "large page with token", token: MustEncodeToken(t, idx[3]), pageSize: 10, start: 3, end: 7, wantNext: &rtpb.ReadToken{}},
+		{desc: "small page with token", token: MustEncodeToken(t, idx[3]), pageSize: 2, start: 3, end: 5,
+			wantNext: &rtpb.ReadToken{StartTime: timestamp(t, idx[5])}},
 		{desc: "invalid page token", token: "some_token", pageSize: 0, wantErr: true},
 	} {
 		t.Run(tc.desc, func(t *testing.T) {
@@ -195,14 +181,15 @@ func TestListMutations(t *testing.T) {
 			if err != nil {
 				return
 			}
-			mtns := fakeLogs[0][tc.start:tc.end]
-			if got, want := len(resp.Mutations), len(mtns); got != want {
-				t.Fatalf("len(resp.Mutations):%v, want %v", got, want)
+
+			got := []*pb.EntryUpdate{}
+			for _, m := range resp.Mutations {
+				got = append(got, &pb.EntryUpdate{Mutation: m.Mutation})
 			}
-			for i, mut := range resp.Mutations {
-				if got, want := mut.Mutation, mtns[i].Mutation; !proto.Equal(got, want) {
-					t.Errorf("resp.Mutations[i].Update:%v, want %v", got, want)
-				}
+
+			if want := genEntryUpdates(t, tc.start, tc.end); !cmp.Equal(
+				got, want, cmp.Comparer(proto.Equal)) {
+				t.Errorf("got: %v, want: %v, diff: \n%v", got, want, cmp.Diff(got, want))
 			}
 
 			var npt rtpb.ReadToken
diff --git a/core/sequencer/server_test.go b/core/sequencer/server_test.go
index 780f7ef0e..02f13677a 100644
--- a/core/sequencer/server_test.go
+++ b/core/sequencer/server_test.go
@@ -17,7 +17,6 @@ package sequencer
 import (
 	"context"
 	"fmt"
-	"sort"
 	"testing"
 	"time"
 
@@ -27,10 +26,11 @@ import (
 	"github.com/google/trillian/types"
 	"google.golang.org/grpc"
 
-	"github.com/google/keytransparency/core/mutator"
+	"github.com/google/keytransparency/impl/memory"
+
+	pb "github.com/google/keytransparency/core/api/v1/keytransparency_go_proto"
 	"github.com/google/keytransparency/core/sequencer/mapper"
 	"github.com/google/keytransparency/core/sequencer/runner"
-
 	spb "github.com/google/keytransparency/core/sequencer/sequencer_go_proto"
 	tpb "github.com/google/trillian"
 )
@@ -43,39 +43,6 @@ func init() {
 	initMetrics.Do(func() { createMetrics(monitoring.InertMetricFactory{}) })
 }
 
-// fakeLogs are indexed by logID, and nanoseconds from time 0
-type fakeLogs map[int64][]mutator.LogMessage
-
-func (l fakeLogs) ReadLog(ctx context.Context, directoryID string, logID int64, low, high time.Time,
-	batchSize int32) ([]*mutator.LogMessage, error) {
-	refs := make([]*mutator.LogMessage, 0)
-	for i := low; i.Before(high); i = i.Add(time.Nanosecond) {
-		l[logID][i.UnixNano()].ID = i
-		refs = append(refs, &l[logID][i.UnixNano()])
-	}
-	return refs, nil
-}
-
-func (l fakeLogs) ListLogs(ctx context.Context, directoryID string, writable bool) ([]int64, error) {
-	logIDs := make([]int64, 0, len(l))
-	for logID := range l {
-		logIDs = append(logIDs, logID)
-	}
-	// sort logsIDs for test repeatability.
-	sort.Slice(logIDs, func(i, j int) bool { return logIDs[i] < logIDs[j] })
-	return logIDs, nil
-}
-
-func (l fakeLogs) HighWatermark(ctx context.Context, directoryID string, logID int64, start time.Time,
-	batchSize int32) (int32, time.Time, error) {
-	high := start.UnixNano() + int64(batchSize)
-	if high > int64(len(l[logID])) {
-		high = int64(len(l[logID]))
-	}
-	count := int32(high - start.UnixNano())
-	return count, time.Unix(0, high), nil
-}
-
 type fakeTrillianFactory struct {
 	tmap   trillianMap
 	tlog   trillianLog
@@ -130,15 +97,33 @@ func (b *fakeBatcher) ReadBatch(_ context.Context, _ string, rev int64) (*spb.Ma
 	return meta, nil
 }
 
+func setupLogs(ctx context.Context, t *testing.T, dirID string, logLengths map[int64]int) (memory.MutationLogs, map[int64][]time.Time) {
+	t.Helper()
+	fakeLogs := memory.NewMutationLogs()
+	idx := make(map[int64][]time.Time)
+	for logID, msgs := range logLengths {
+		if err := fakeLogs.AddLogs(ctx, dirID, logID); err != nil {
+			t.Fatal(err)
+		}
+		for i := 0; i < msgs; i++ {
+			ts, err := fakeLogs.Send(ctx, dirID, logID, &pb.EntryUpdate{})
+			if err != nil {
+				t.Fatal(err)
+			}
+			idx[logID] = append(idx[logID], ts)
+		}
+	}
+	return fakeLogs, idx
+}
+
 func TestDefiningRevisions(t *testing.T) {
 	// Verify that outstanding revisions prevent future revisions from being created.
 	ctx := context.Background()
 	mapRev := int64(2)
+	dirID := "foobar"
+	fakeLogs, idx := setupLogs(ctx, t, dirID, map[int64]int{0: 10, 1: 20})
 	s := Server{
-		logs: fakeLogs{
-			0: make([]mutator.LogMessage, 10),
-			1: make([]mutator.LogMessage, 20),
-		},
+		logs: fakeLogs,
 		trillian: &fakeTrillianFactory{
 			tmap: &fakeMap{latestMapRoot: &types.MapRootV1{Revision: uint64(mapRev)}},
 		},
@@ -164,8 +149,8 @@ func TestDefiningRevisions(t *testing.T) {
 			wantNew: mapRev + 1},
 		{desc: "drained", highestRev: mapRev,
 			meta: spb.MapMetadata{Sources: []*spb.MapMetadata_SourceSlice{
-				{LogId: 0, LowestInclusive: 0, HighestExclusive: 10},
-				{LogId: 1, LowestInclusive: 0, HighestExclusive: 20},
+				{LogId: 0, LowestInclusive: 0, HighestExclusive: idx[0][9].Add(1).UnixNano()},
+				{LogId: 1, LowestInclusive: 0, HighestExclusive: idx[1][19].Add(1).UnixNano()},
 			}},
 			wantNew: mapRev},
 	} {
@@ -207,10 +192,9 @@ func TestDefiningRevisions(t *testing.T) {
 
 func TestReadMessages(t *testing.T) {
 	ctx := context.Background()
-	s := Server{logs: fakeLogs{
-		0: make([]mutator.LogMessage, 10),
-		1: make([]mutator.LogMessage, 20),
-	}}
+	dirID := "TestReadMessages"
+	fakeLogs, idx := setupLogs(ctx, t, dirID, map[int64]int{0: 10, 1: 20})
+	s := Server{logs: fakeLogs}
 
 	for _, tc := range []struct {
 		meta      *spb.MapMetadata
@@ -218,14 +202,14 @@ func TestReadMessages(t *testing.T) {
 		want      int
 	}{
 		{batchSize: 1, want: 9, meta: &spb.MapMetadata{Sources: []*spb.MapMetadata_SourceSlice{
-			{LogId: 0, LowestInclusive: 1, HighestExclusive: 10},
+			{LogId: 0, LowestInclusive: idx[0][1].UnixNano(), HighestExclusive: idx[0][9].Add(1).UnixNano()},
 		}}},
 		{batchSize: 10000, want: 9, meta: &spb.MapMetadata{Sources: []*spb.MapMetadata_SourceSlice{
-			{LogId: 0, LowestInclusive: 1, HighestExclusive: 10},
+			{LogId: 0, LowestInclusive: idx[0][1].UnixNano(), HighestExclusive: idx[0][9].Add(1).UnixNano()},
 		}}},
 		{batchSize: 1, want: 19, meta: &spb.MapMetadata{Sources: []*spb.MapMetadata_SourceSlice{
-			{LogId: 0, LowestInclusive: 1, HighestExclusive: 10},
-			{LogId: 1, LowestInclusive: 1, HighestExclusive: 11},
+			{LogId: 0, LowestInclusive: idx[0][1].UnixNano(), HighestExclusive: idx[0][9].Add(1).UnixNano()},
+			{LogId: 1, LowestInclusive: idx[1][1].UnixNano(), HighestExclusive: idx[1][10].Add(1).UnixNano()},
 		}}},
 	} {
 		logSlices := runner.DoMapMetaFn(mapper.MapMetaFn, tc.meta, fakeMetric)
@@ -241,10 +225,9 @@ func TestReadMessages(t *testing.T) {
 
 func TestHighWatermarks(t *testing.T) {
 	ctx := context.Background()
-	s := Server{logs: fakeLogs{
-		0: make([]mutator.LogMessage, 10),
-		1: make([]mutator.LogMessage, 20),
-	}}
+	dirID := "TestHighWatermark"
+	fakeLogs, idx := setupLogs(ctx, t, dirID, map[int64]int{0: 10, 1: 20})
+	s := Server{logs: fakeLogs}
 
 	for _, tc := range []struct {
 		desc      string
@@ -255,24 +238,26 @@ func TestHighWatermarks(t *testing.T) {
 	}{
 		{desc: "nobatch", batchSize: 30, count: 30,
 			next: &spb.MapMetadata{Sources: []*spb.MapMetadata_SourceSlice{
-				{LogId: 0, HighestExclusive: 10},
-				{LogId: 1, HighestExclusive: 20}}}},
+				{LogId: 0, HighestExclusive: idx[0][9].Add(1).UnixNano()},
+				{LogId: 1, HighestExclusive: idx[1][19].Add(1).UnixNano()}}}},
 		{desc: "exactbatch", batchSize: 20, count: 20,
 			next: &spb.MapMetadata{Sources: []*spb.MapMetadata_SourceSlice{
-				{LogId: 0, HighestExclusive: 10},
-				{LogId: 1, HighestExclusive: 10}}}},
+				{LogId: 0, HighestExclusive: idx[0][9].Add(1).UnixNano()},
+				{LogId: 1, HighestExclusive: idx[1][9].Add(1).UnixNano()}}}},
 		{desc: "batchwprev", batchSize: 20, count: 20,
 			last: &spb.MapMetadata{Sources: []*spb.MapMetadata_SourceSlice{
-				{LogId: 0, HighestExclusive: 10}}},
+				{LogId: 0, HighestExclusive: idx[0][9].Add(2).UnixNano()}}},
 			next: &spb.MapMetadata{Sources: []*spb.MapMetadata_SourceSlice{
-				{LogId: 0, LowestInclusive: 10, HighestExclusive: 10},
-				{LogId: 1, HighestExclusive: 20}}}},
+				// Nothing to read from log 1, preserve watermark of log 1.
+				{LogId: 0, LowestInclusive: idx[0][9].Add(2).UnixNano(), HighestExclusive: idx[0][9].Add(2).UnixNano()},
+				{LogId: 1, HighestExclusive: idx[1][19].Add(1).UnixNano()}}}},
 		// Don't drop existing watermarks.
 		{desc: "keep existing", batchSize: 1, count: 1,
 			last: &spb.MapMetadata{Sources: []*spb.MapMetadata_SourceSlice{
 				{LogId: 1, HighestExclusive: 10}}},
 			next: &spb.MapMetadata{Sources: []*spb.MapMetadata_SourceSlice{
-				{LogId: 0, HighestExclusive: 1},
+				{LogId: 0, HighestExclusive: idx[0][0].Add(1).UnixNano()},
+				// No reads from log 1, but don't drop the watermark.
 				{LogId: 1, LowestInclusive: 10, HighestExclusive: 10}}}},
 		{desc: "logs that dont move", batchSize: 0, count: 0,
 			last: &spb.MapMetadata{Sources: []*spb.MapMetadata_SourceSlice{
@@ -291,7 +276,7 @@ func TestHighWatermarks(t *testing.T) {
 				t.Errorf("HighWatermarks(): count: %v, want %v", count, tc.count)
 			}
 			if !proto.Equal(next, tc.next) {
-				t.Errorf("HighWatermarks(): diff(-got, +want): %v", cmp.Diff(next, &tc.next))
+				t.Errorf("HighWatermarks(): diff(-got, +want): %v", cmp.Diff(next, tc.next))
 			}
 		})
 	}
diff --git a/impl/memory/mutation_logs.go b/impl/memory/mutation_logs.go
new file mode 100644
index 000000000..c652f0463
--- /dev/null
+++ b/impl/memory/mutation_logs.go
@@ -0,0 +1,126 @@
+// Copyright 2019 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package memory supplies fake in-memory implementations for testing purposes.
+package memory
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"time"
+
+	"github.com/google/keytransparency/core/mutator"
+
+	pb "github.com/google/keytransparency/core/api/v1/keytransparency_go_proto"
+)
+
+var clock = time.Unix(10, 0) // Start our clock at an arbitrary, non-zero place.
+
+// NewMutationLogs creates a new fake MutationLogs.
+func NewMutationLogs() MutationLogs {
+	return make(MutationLogs)
+}
+
+type batch struct {
+	time time.Time
+	msgs []*mutator.LogMessage
+}
+
+// MutationLogs is a fake, in-memory implementation of a keyserver.MutationLogss.
+// All requests are presumed to be for the same domainID.
+// TODO(gbelvin): Support multiple domainIDs, if tests call for it.
+// MutationLogs is NOT threadsafe.
+type MutationLogs map[int64][]batch // Map of logID to Slice of LogMessages
+
+// AddLogs adds logIDs to the mutation database.
+func (m MutationLogs) AddLogs(_ context.Context, _ string, logIDs ...int64) error {
+	for _, logID := range logIDs {
+		m[logID] = make([]batch, 0)
+	}
+	return nil
+}
+
+// ListLogs returns a sorted list of logIDs.
+func (m MutationLogs) ListLogs(_ context.Context, _ string, writable bool) ([]int64, error) {
+	logIDs := []int64{}
+	for logID := range m {
+		logIDs = append(logIDs, logID)
+	}
+	sort.Slice(logIDs, func(a, b int) bool { return logIDs[a] < logIDs[b] })
+	return logIDs, nil
+}
+
+// Send stores a batch of mutations in a given logID.
+func (m MutationLogs) Send(_ context.Context, _ string, logID int64, mutation ...*pb.EntryUpdate) (time.Time, error) {
+	clock = clock.Add(time.Second)
+	ts := clock
+	// Only save the Merkle tree bits.
+	entries := make([]*pb.SignedEntry, 0, len(mutation))
+	for _, i := range mutation {
+		entries = append(entries, i.Mutation)
+	}
+
+	logShard := m[logID]
+	if len(logShard) > 0 && logShard[len(logShard)-1].time.After(ts) {
+		return time.Time{}, fmt.Errorf("inserting mutation entry %v out of order", ts)
+	}
+
+	// Convert []SignedEntry into []LogMessage for storage.
+	msgs := make([]*mutator.LogMessage, 0, len(entries))
+	for _, e := range entries {
+		msgs = append(msgs, &mutator.LogMessage{ID: ts, Mutation: e})
+	}
+	m[logID] = append(logShard, batch{time: ts, msgs: msgs})
+	return ts, nil
+}
+
+// ReadLog returns mutations between [low, high).  Always returns complete batches.
+// ReadLog will return more items than batchSize if necessary to return a complete batch.
+func (m MutationLogs) ReadLog(_ context.Context, _ string,
+	logID int64, low, high time.Time, batchSize int32) ([]*mutator.LogMessage, error) {
+	logShard := m[logID]
+	if len(logShard) == 0 || batchSize == 0 {
+		return nil, nil
+	}
+	start := sort.Search(len(logShard), func(i int) bool { return !logShard[i].time.Before(low) })
+	end := sort.Search(len(logShard), func(i int) bool { return !logShard[i].time.Before(high) })
+	// If the search is unsuccessful, i will be equal to len(logShard).
+	if start == len(logShard) && logShard[start].time.Before(low) {
+		return nil, fmt.Errorf("invalid argument: low: %v, want <= max watermark: %v", low, logShard[start].time)
+	}
+	out := make([]*mutator.LogMessage, 0, batchSize)
+	for i := start; i < end; i++ {
+		out = append(out, logShard[i].msgs...)
+		if int32(len(out)) >= batchSize {
+			break
+		}
+	}
+	return out, nil
+}
+
+// HighWatermark returns the highest timestamp batchSize items beyond start.
+func (m MutationLogs) HighWatermark(_ context.Context, _ string, logID int64, start time.Time,
+	batchSize int32) (int32, time.Time, error) {
+	logShard := m[logID]
+	i := sort.Search(len(logShard), func(i int) bool { return !logShard[i].time.Before(start) })
+
+	count := int32(0)
+	high := start // Preserve start time if there are no rows to process.
+	for ; i < len(logShard) && count < batchSize; i++ {
+		high = logShard[i].time.Add(time.Nanosecond) // Returns exclusive n + 1
+		count += int32(len(logShard[i].msgs))
+	}
+	return count, high, nil
+}
diff --git a/impl/memory/mutation_logs_test.go b/impl/memory/mutation_logs_test.go
new file mode 100644
index 000000000..be50687ed
--- /dev/null
+++ b/impl/memory/mutation_logs_test.go
@@ -0,0 +1,35 @@
+// Copyright 2019 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package memory
+
+import (
+	"context"
+	"testing"
+
+	"github.com/google/keytransparency/core/integration/storagetest"
+	"github.com/google/keytransparency/core/keyserver"
+)
+
+// Tests for the tests!
+func TestMutationLogsIntegration(t *testing.T) {
+	storagetest.RunMutationLogsTests(t,
+		func(ctx context.Context, t *testing.T, dirID string, logIDs ...int64) (keyserver.MutationLogs, func(context.Context)) {
+			m := NewMutationLogs()
+			if err := m.AddLogs(ctx, dirID, logIDs...); err != nil {
+				t.Fatal(err)
+			}
+			return m, func(context.Context) {}
+		})
+}

From ea1b47d7bb96b2c759d220c51d54aae50ce018d0 Mon Sep 17 00:00:00 2001
From: Gary Belvin <gdb@google.com>
Date: Tue, 12 Nov 2019 18:01:45 +0000
Subject: [PATCH 12/22] Support and test mutation log queries at intermediate
 timestamps (#1380)

* Move HighWatermarks tests to integration/storagetest

* Test inter-quantum reads

* Don't return timestamps with a finer granularity than we actually have.

* New HighWatermark tests that respect interface.

Replace tests that wanted to predict the exact value of HighWatermark
with tests that rely on the behavior of ReadLog to assert that the value
returned is indeed correct.

This allows different storage layers to use their own strategies
for returning high watermarks.

It also removes the requirement that HighWatermark values themselves
be stored with nanosecond precision.
---
 core/integration/storagetest/mutation_logs.go |  60 +++---
 .../storagetest/mutation_logs_reader.go       | 199 ++++++++++++++++++
 impl/memory/mutation_logs_test.go             |  11 +
 impl/sql/mutationstorage/mutation_logs.go     |  16 +-
 .../sql/mutationstorage/mutation_logs_test.go |  57 +----
 5 files changed, 262 insertions(+), 81 deletions(-)
 create mode 100644 core/integration/storagetest/mutation_logs_reader.go

diff --git a/core/integration/storagetest/mutation_logs.go b/core/integration/storagetest/mutation_logs.go
index 3fd1af560..35c15cef8 100644
--- a/core/integration/storagetest/mutation_logs.go
+++ b/core/integration/storagetest/mutation_logs.go
@@ -16,6 +16,7 @@ package storagetest
 
 import (
 	"context"
+	"fmt"
 	"testing"
 	"time"
 
@@ -70,7 +71,7 @@ func (mutationLogsTests) TestReadLog(ctx context.Context, t *testing.T, newForTe
 		}
 	}
 
-	for _, tc := range []struct {
+	for i, tc := range []struct {
 		limit int32
 		want  int
 	}{
@@ -80,13 +81,15 @@ func (mutationLogsTests) TestReadLog(ctx context.Context, t *testing.T, newForTe
 		{limit: 4, want: 6},    // Reading 4 items gets us into the second batch of size 3.
 		{limit: 100, want: 30}, // Reading all the items gets us the 30 items we wrote.
 	} {
-		rows, err := m.ReadLog(ctx, directoryID, logID, minWatermark, time.Now(), tc.limit)
-		if err != nil {
-			t.Fatalf("ReadLog(%v): %v", tc.limit, err)
-		}
-		if got := len(rows); got != tc.want {
-			t.Fatalf("ReadLog(%v): len: %v, want %v", tc.limit, got, tc.want)
-		}
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			rows, err := m.ReadLog(ctx, directoryID, logID, minWatermark, time.Now(), tc.limit)
+			if err != nil {
+				t.Fatalf("ReadLog(%v): %v", tc.limit, err)
+			}
+			if got := len(rows); got != tc.want {
+				t.Fatalf("ReadLog(%v): len: %v, want %v", tc.limit, got, tc.want)
+			}
+		})
 	}
 }
 
@@ -107,26 +110,31 @@ func (mutationLogsTests) TestReadLogExact(ctx context.Context, t *testing.T, new
 		idx = append(idx, ts)
 	}
 
-	for _, tc := range []struct {
-		low, high int
+	for i, tc := range []struct {
+		low, high time.Time
 		want      []byte
 	}{
-		{low: 0, high: 0, want: []byte{}},
-		{low: 0, high: 1, want: []byte{0}},
-		{low: 0, high: 9, want: []byte{0, 1, 2, 3, 4, 5, 6, 7, 8}},
-		{low: 1, high: 9, want: []byte{1, 2, 3, 4, 5, 6, 7, 8}},
+		{low: idx[0], high: idx[0], want: []byte{}},
+		{low: idx[0], high: idx[1], want: []byte{0}},
+		{low: idx[0], high: idx[9], want: []byte{0, 1, 2, 3, 4, 5, 6, 7, 8}},
+		{low: idx[1], high: idx[9], want: []byte{1, 2, 3, 4, 5, 6, 7, 8}},
+		// Ensure that adding 1 correctly modifies the range semantics.
+		{low: idx[0].Add(1), high: idx[9], want: []byte{1, 2, 3, 4, 5, 6, 7, 8}},
+		{low: idx[0].Add(1), high: idx[9].Add(1), want: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9}},
 	} {
-		rows, err := m.ReadLog(ctx, directoryID, logID, idx[tc.low], idx[tc.high], 100)
-		if err != nil {
-			t.Fatalf("ReadLog(): %v", err)
-		}
-		got := make([]byte, 0, len(rows))
-		for _, r := range rows {
-			i := r.Mutation.Entry[0]
-			got = append(got, i)
-		}
-		if !cmp.Equal(got, tc.want) {
-			t.Fatalf("ReadLog(%v,%v): got %v, want %v", tc.low, tc.high, got, tc.want)
-		}
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			rows, err := m.ReadLog(ctx, directoryID, logID, tc.low, tc.high, 100)
+			if err != nil {
+				t.Fatalf("ReadLog(): %v", err)
+			}
+			got := make([]byte, 0, len(rows))
+			for _, r := range rows {
+				i := r.Mutation.Entry[0]
+				got = append(got, i)
+			}
+			if !cmp.Equal(got, tc.want) {
+				t.Fatalf("ReadLog(%v,%v): got %v, want %v", tc.low, tc.high, got, tc.want)
+			}
+		})
 	}
 }
diff --git a/core/integration/storagetest/mutation_logs_reader.go b/core/integration/storagetest/mutation_logs_reader.go
new file mode 100644
index 000000000..0993bba6c
--- /dev/null
+++ b/core/integration/storagetest/mutation_logs_reader.go
@@ -0,0 +1,199 @@
+// Copyright 2019 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package storagetest
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/keytransparency/core/sequencer"
+
+	pb "github.com/google/keytransparency/core/api/v1/keytransparency_go_proto"
+)
+
+// LogsReadWriter supports test's ability to write to and read from the mutation logs.
+type LogsReadWriter interface {
+	sequencer.LogsReader
+	// Send submits the whole group of mutations atomically to a log.
+	// TODO(gbelvin): Create a batch level object to make it clear that this a batch of updates.
+	// Returns the timestamp that the mutation batch got written at.
+	// This timestamp can be used as an argument to the lower bound of ReadLog [low, high).
+	// To acquire a value to  use  for the  upper bound of ReadLog [low, high), use HighWatermark.
+	Send(ctx context.Context, directoryID string, logID int64, mutation ...*pb.EntryUpdate) (time.Time, error)
+}
+
+// logsRWFactory returns a new database object, and a function for cleaning it up.
+type logsRWFactory func(ctx context.Context, t *testing.T, dirID string, logIDs ...int64) (LogsReadWriter, func(context.Context))
+
+// RunMutationLogsReaderTests runs all the tests against the provided storage implementation.
+func RunMutationLogsReaderTests(t *testing.T, factory logsRWFactory) {
+	ctx := context.Background()
+	b := &mutationLogsReaderTests{}
+	type TestFunc func(ctx context.Context, t *testing.T, f logsRWFactory)
+	for name, f := range map[string]TestFunc{
+		// TODO(gbelvin): Discover test methods via reflection.
+		"TestHighWatermarkPreserve": b.TestHighWatermarkPreserve,
+		"TestHighWatermarkRead":     b.TestHighWatermarkRead,
+		"TestHighWatermarkBatch":    b.TestHighWatermarkBatch,
+	} {
+		t.Run(name, func(t *testing.T) { f(ctx, t, factory) })
+	}
+}
+
+type mutationLogsReaderTests struct{}
+
+func setupWatermarks(ctx context.Context, t *testing.T, m LogsReadWriter, dirID string, logID int64, maxIndex int) ([]time.Time, []time.Time) {
+	t.Helper()
+	// Setup the test by writing 10 items to the mutation log and
+	// collecting the reported high water mark after each write.
+	sent := []time.Time{} // Timestamps that Send reported.
+	hwm := []time.Time{}  // High water marks collected after each Send.
+	for i := 0; i <= maxIndex; i++ {
+		ts, err := m.Send(ctx, dirID, logID, &pb.EntryUpdate{Mutation: &pb.SignedEntry{Entry: []byte{byte(i)}}})
+		if err != nil {
+			t.Fatalf("Send(%v): %v", logID, err)
+		}
+		count, wm, err := m.HighWatermark(ctx, dirID, logID, minWatermark, 100 /*batchSize*/)
+		if err != nil {
+			t.Fatalf("HighWatermark(): %v", err)
+		}
+		if want := int32(i) + 1; count != want {
+			t.Fatalf("HighWatermark(): count %v, want %v", count, want)
+		}
+		sent = append(sent, ts)
+		hwm = append(hwm, wm)
+	}
+	return sent, hwm
+}
+
+// Tests that query HighWatermarks with varying parameters and validate results directly.
+func (mutationLogsReaderTests) TestHighWatermarkPreserve(ctx context.Context, t *testing.T, newForTest logsRWFactory) {
+	directoryID := "TestHighWatermarkPreserve"
+	logID := int64(1)
+	m, done := newForTest(ctx, t, directoryID, logID)
+	defer done(ctx)
+	maxIndex := 9
+	sent, _ := setupWatermarks(ctx, t, m, directoryID, logID, maxIndex)
+
+	arbitraryTime := time.Date(1, 2, 3, 4, 5, 6, 7, time.UTC)
+	for _, tc := range []struct {
+		desc  string
+		start time.Time
+		batch int32
+		want  time.Time
+	}{
+		// Verify that high watermarks preserves the starting time when batch size is 0.
+		{desc: "batch 0", start: arbitraryTime, batch: 0, want: arbitraryTime},
+		// Verify that high watermarks preserves the starting time when there are no rows in the result.
+		{desc: "rows 0", start: sent[maxIndex].Add(time.Second), batch: 1, want: sent[maxIndex].Add(time.Second)},
+	} {
+		t.Run(tc.desc, func(t *testing.T) {
+			count, got, err := m.HighWatermark(ctx, directoryID, logID, tc.start, tc.batch)
+			if err != nil {
+				t.Errorf("HighWatermark(): %v", err)
+			}
+			if !got.Equal(tc.want) {
+				t.Errorf("HighWatermark(%v, %v) high: %v, want %v", tc.start, tc.batch, got, tc.want)
+			}
+			if count != 0 {
+				t.Errorf("HighWatermark(): count %v, want 0", count)
+			}
+		})
+	}
+}
+
+// Tests that use the watermarks defined during setup.
+func (mutationLogsReaderTests) TestHighWatermarkRead(ctx context.Context, t *testing.T, newForTest logsRWFactory) {
+	directoryID := "TestHighWatermarkRead"
+	logID := int64(1)
+	m, done := newForTest(ctx, t, directoryID, logID)
+	defer done(ctx)
+	maxIndex := 9
+	_, hwm := setupWatermarks(ctx, t, m, directoryID, logID, maxIndex)
+	for _, tc := range []struct {
+		desc     string
+		readHigh time.Time
+		want     []byte
+	}{
+		// Verify that highwatermark can retrieve all the data written so far.
+		{desc: "all", readHigh: hwm[maxIndex], want: []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}},
+		// Verify that data retrieved at highwatermark doesn't change when more data is written.
+		{desc: "stable", readHigh: hwm[2], want: []byte{0, 1, 2}},
+	} {
+		t.Run(tc.desc, func(t *testing.T) {
+			low := minWatermark
+			rows, err := m.ReadLog(ctx, directoryID, logID, low, tc.readHigh, 100)
+			if err != nil {
+				t.Fatalf("ReadLog(): %v", err)
+			}
+			got := make([]byte, 0, len(rows))
+			for _, r := range rows {
+				i := r.Mutation.Entry[0]
+				got = append(got, i)
+			}
+			if !cmp.Equal(got, tc.want) {
+				t.Fatalf("ReadLog(%v,%v): got %v, want %v", low, tc.readHigh, got, tc.want)
+			}
+		})
+	}
+}
+
+// Tests that query HighWatermarks with varying parameters and validate results using ReadLog.
+func (mutationLogsReaderTests) TestHighWatermarkBatch(ctx context.Context, t *testing.T, newForTest logsRWFactory) {
+	directoryID := "TestHighWatermarkBatch"
+	logID := int64(1)
+	m, done := newForTest(ctx, t, directoryID, logID)
+	defer done(ctx)
+	maxIndex := 9
+	sent, _ := setupWatermarks(ctx, t, m, directoryID, logID, maxIndex)
+	for _, tc := range []struct {
+		desc  string
+		start time.Time
+		batch int32
+		want  []byte
+	}{
+		// Verify that limiting batch size controls the number of items returned.
+		{desc: "limit batch", start: sent[0], batch: 2, want: []byte{0, 1}},
+		// Verify that advancing start by 1 with the same batch size advances the results by one.
+		{desc: "start 1", start: sent[1], batch: 2, want: []byte{1, 2}},
+		// Verify that watermarks in between primary keys resolve correctly.
+		{desc: "start 0.1", start: sent[0].Add(1), batch: 2, want: []byte{1, 2}},
+	} {
+		t.Run(tc.desc, func(t *testing.T) {
+			count, wm, err := m.HighWatermark(ctx, directoryID, logID, tc.start, tc.batch)
+			if err != nil {
+				t.Errorf("HighWatermark(): %v", err)
+			}
+			if want := int32(len(tc.want)); count != want {
+				t.Errorf("HighWatermark() count: %v, want %v", count, want)
+			}
+
+			rows, err := m.ReadLog(ctx, directoryID, logID, tc.start, wm, tc.batch)
+			if err != nil {
+				t.Fatalf("ReadLog(): %v", err)
+			}
+			got := make([]byte, 0, len(rows))
+			for _, r := range rows {
+				i := r.Mutation.Entry[0]
+				got = append(got, i)
+			}
+			if !cmp.Equal(got, tc.want) {
+				t.Fatalf("ReadLog(%v,%v): got %v, want %v", tc.start, wm, got, tc.want)
+			}
+		})
+	}
+}
diff --git a/impl/memory/mutation_logs_test.go b/impl/memory/mutation_logs_test.go
index be50687ed..c5ba1ccec 100644
--- a/impl/memory/mutation_logs_test.go
+++ b/impl/memory/mutation_logs_test.go
@@ -33,3 +33,14 @@ func TestMutationLogsIntegration(t *testing.T) {
 			return m, func(context.Context) {}
 		})
 }
+
+func TestMutationLogsReaderIntegration(t *testing.T) {
+	storagetest.RunMutationLogsReaderTests(t,
+		func(ctx context.Context, t *testing.T, dirID string, logIDs ...int64) (storagetest.LogsReadWriter, func(context.Context)) {
+			m := NewMutationLogs()
+			if err := m.AddLogs(ctx, dirID, logIDs...); err != nil {
+				t.Fatal(err)
+			}
+			return m, func(context.Context) {}
+		})
+}
diff --git a/impl/sql/mutationstorage/mutation_logs.go b/impl/sql/mutationstorage/mutation_logs.go
index e8c75b40f..fa5f61d7c 100644
--- a/impl/sql/mutationstorage/mutation_logs.go
+++ b/impl/sql/mutationstorage/mutation_logs.go
@@ -28,6 +28,10 @@ import (
 	pb "github.com/google/keytransparency/core/api/v1/keytransparency_go_proto"
 )
 
+// quantum is the fidelity of the Timestamp column.
+// See https://dev.mysql.com/doc/refman/8.0/en/fractional-seconds.html
+const quantum = time.Microsecond
+
 // SetWritable enables or disables new writes from going to logID.
 func (m *Mutations) SetWritable(ctx context.Context, directoryID string, logID int64, enabled bool) error {
 	result, err := m.db.ExecContext(ctx,
@@ -80,7 +84,7 @@ func (m *Mutations) Send(ctx context.Context, directoryID string, logID int64, u
 	}
 	// TODO(gbelvin): Implement retry with backoff for retryable errors if
 	// we get timestamp contention.
-	ts := time.Now()
+	ts := time.Now().Truncate(quantum)
 	if err := m.send(ctx, ts, directoryID, logID, updateData...); err != nil {
 		return time.Time{}, err
 	}
@@ -141,9 +145,6 @@ func (m *Mutations) send(ctx context.Context, ts time.Time, directoryID string,
 		return status.Errorf(codes.Internal, "could not find max timestamp: %v", err)
 	}
 
-	// The Timestamp column has a maximum fidelity of microseconds.
-	// See https://dev.mysql.com/doc/refman/8.0/en/fractional-seconds.html
-	ts = ts.Truncate(time.Microsecond)
 	if !ts.After(maxTime.Time) {
 		return status.Errorf(codes.Aborted,
 			"current timestamp: %v, want > max-timestamp of queued mutations: %v", ts, maxTime)
@@ -163,6 +164,7 @@ func (m *Mutations) send(ctx context.Context, ts time.Time, directoryID string,
 // equal to batchSize items greater than start.
 func (m *Mutations) HighWatermark(ctx context.Context, directoryID string, logID int64,
 	start time.Time, batchSize int32) (int32, time.Time, error) {
+	startQuery := start.Add(quantum - 1).Truncate(quantum)
 	var count int32
 	var high sql.NullTime
 	if err := m.db.QueryRowContext(ctx,
@@ -173,7 +175,7 @@ func (m *Mutations) HighWatermark(ctx context.Context, directoryID string, logID
 			ORDER BY Q.Time ASC
 			LIMIT ?
 		) AS T1`,
-		directoryID, logID, start, batchSize).
+		directoryID, logID, startQuery, batchSize).
 		Scan(&count, &high); err != nil {
 		return 0, start, err
 	}
@@ -188,6 +190,10 @@ func (m *Mutations) HighWatermark(ctx context.Context, directoryID string, logID
 // ReadLog may return more rows than batchSize in order to fetch all the rows at a particular timestamp.
 func (m *Mutations) ReadLog(ctx context.Context, directoryID string,
 	logID int64, low, high time.Time, batchSize int32) ([]*mutator.LogMessage, error) {
+	// Advance the low and high marks to the next highest quantum to preserve read semantics.
+	low = low.Add(quantum - 1).Truncate(quantum)
+	high = high.Add(quantum - 1).Truncate(quantum)
+
 	rows, err := m.db.QueryContext(ctx,
 		`SELECT Time, LocalID, Mutation FROM Queue
 		WHERE DirectoryID = ? AND LogID = ? AND Time >= ? AND Time < ?
diff --git a/impl/sql/mutationstorage/mutation_logs_test.go b/impl/sql/mutationstorage/mutation_logs_test.go
index 94c0e2af7..9cda05d5c 100644
--- a/impl/sql/mutationstorage/mutation_logs_test.go
+++ b/impl/sql/mutationstorage/mutation_logs_test.go
@@ -56,6 +56,13 @@ func TestLogsAdminIntegration(t *testing.T) {
 		})
 }
 
+func TestMutationLogsReaderIntegration(t *testing.T) {
+	storagetest.RunMutationLogsReaderTests(t,
+		func(ctx context.Context, t *testing.T, dirID string, logIDs ...int64) (storagetest.LogsReadWriter, func(context.Context)) {
+			return newForTest(ctx, t, dirID, logIDs...)
+		})
+}
+
 func BenchmarkSend(b *testing.B) {
 	ctx := context.Background()
 	directoryID := "BenchmarkSend"
@@ -121,53 +128,3 @@ func TestSend(t *testing.T) {
 		}
 	}
 }
-
-// https://dev.mysql.com/doc/refman/8.0/en/datetime.html
-var minWatermark = time.Date(1000, 1, 1, 0, 0, 0, 0, time.UTC)
-
-func TestWatermark(t *testing.T) {
-	ctx := context.Background()
-	directoryID := "TestWatermark"
-	logIDs := []int64{1, 2}
-	m, done := newForTest(ctx, t, directoryID, logIDs...)
-	defer done(ctx)
-	update := []byte("bar")
-
-	start := time.Now().Truncate(time.Microsecond)
-	for ts := start; ts.Before(start.Add(10 * time.Microsecond)); ts = ts.Add(1 * time.Microsecond) {
-		logID := int64(1)
-		if err := m.send(ctx, ts, directoryID, logID, update); err != nil {
-			t.Fatalf("m.send(%v): %v", logID, err)
-		}
-	}
-
-	for _, tc := range []struct {
-		desc      string
-		logID     int64
-		start     time.Time
-		batchSize int32
-		count     int32
-		want      time.Time
-	}{
-		{desc: "log1 max", logID: 1, batchSize: 100, start: start, want: start.Add(10 * time.Microsecond), count: 10},
-		{desc: "log2 empty", logID: 2, batchSize: 100, start: start, want: start},
-		{desc: "batch0", logID: 1, batchSize: 0, start: minWatermark, want: minWatermark},
-		{desc: "batch0start55", logID: 1, start: minWatermark.Add(55 * time.Microsecond), batchSize: 0, want: minWatermark.Add(55 * time.Microsecond)},
-		{desc: "batch5", logID: 1, start: start, batchSize: 5, want: start.Add(5 * time.Microsecond), count: 5},
-		{desc: "start1", logID: 1, start: start.Add(2 * time.Microsecond), batchSize: 5, want: start.Add(7 * time.Microsecond), count: 5},
-		{desc: "start8", logID: 1, start: start.Add(8 * time.Microsecond), batchSize: 5, want: start.Add(10 * time.Microsecond), count: 2},
-	} {
-		t.Run(tc.desc, func(t *testing.T) {
-			count, got, err := m.HighWatermark(ctx, directoryID, tc.logID, tc.start, tc.batchSize)
-			if err != nil {
-				t.Errorf("highWatermark(): %v", err)
-			}
-			if !got.Equal(tc.want) {
-				t.Errorf("highWatermark(%v) high: %v, want %v", tc.start, got, tc.want)
-			}
-			if count != tc.count {
-				t.Errorf("highWatermark(%v) count: %v, want %v", tc.start, count, tc.count)
-			}
-		})
-	}
-}

From 8bcfcd7754f1fab34a1c757133625e6606e4fbee Mon Sep 17 00:00:00 2001
From: Gary Belvin <gdb@google.com>
Date: Tue, 12 Nov 2019 20:23:19 +0000
Subject: [PATCH 13/22] Reduce log spam (#1382)

---
 core/sequencer/sequencer.go | 3 ---
 core/sequencer/server.go    | 2 ++
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/core/sequencer/sequencer.go b/core/sequencer/sequencer.go
index 8d0175c32..4501d71d4 100644
--- a/core/sequencer/sequencer.go
+++ b/core/sequencer/sequencer.go
@@ -125,7 +125,6 @@ func (s *Sequencer) ForAllMasterships(ctx context.Context, f func(ctx context.Co
 // DefineRevisions method on all directories that this sequencer is currently
 // master for.
 func (s *Sequencer) DefineRevisionsForAllMasterships(ctx context.Context, batchSize int32) error {
-	glog.Infof("DefineRevisionsForAllMasterships")
 	return s.ForAllMasterships(ctx, func(ctx context.Context, dirID string) error {
 		// TODO(pavelkalinnikov): Make these parameters configurable.
 		req := &spb.DefineRevisionsRequest{
@@ -146,7 +145,6 @@ func (s *Sequencer) DefineRevisionsForAllMasterships(ctx context.Context, batchS
 // ApplyRevisions method on all directories that this sequencer is currently
 // master for.
 func (s *Sequencer) ApplyRevisionsForAllMasterships(ctx context.Context) error {
-	glog.Infof("ApplyRevisionsForAllMasterships")
 	return s.ForAllMasterships(ctx, func(ctx context.Context, dirID string) error {
 		req := &spb.ApplyRevisionsRequest{DirectoryId: dirID}
 		if _, err := s.sequencerClient.ApplyRevisions(ctx, req); err != nil {
@@ -160,7 +158,6 @@ func (s *Sequencer) ApplyRevisionsForAllMasterships(ctx context.Context) error {
 // PublishLogForAllMasterships runs KeyTransparencySequencer.PublishRevisions on
 // all directories this sequencer is currently master for.
 func (s *Sequencer) PublishLogForAllMasterships(ctx context.Context) error {
-	glog.Infof("PublishLogForAllMasterships")
 	return s.ForAllMasterships(ctx, func(ctx context.Context, dirID string) error {
 		publishReq := &spb.PublishRevisionsRequest{DirectoryId: dirID}
 		_, err := s.sequencerClient.PublishRevisions(ctx, publishReq)
diff --git a/core/sequencer/server.go b/core/sequencer/server.go
index fe1875183..3d79e1224 100644
--- a/core/sequencer/server.go
+++ b/core/sequencer/server.go
@@ -323,6 +323,7 @@ func (s *Server) ApplyRevisions(ctx context.Context, in *spb.ApplyRevisionsReque
 		glog.Warningf("ApplyRevisions: too many outstanding revisions: %d; applying %d", cnt, mx)
 		cnt = mx
 	}
+	glog.Infof("Applying revisions %d-%d", resp.HighestApplied+1, resp.HighestApplied+cnt)
 	for i := int64(1); i <= cnt; i++ {
 		req := &spb.ApplyRevisionRequest{
 			DirectoryId: in.DirectoryId, Revision: resp.HighestApplied + i}
@@ -502,6 +503,7 @@ func (s *Server) PublishRevisions(ctx context.Context,
 		leaves[int64(mapRoot.Revision)] = rawMapRoot.GetMapRoot()
 		revs = append(revs, int64(mapRoot.Revision))
 	}
+	glog.Infof("Publishing revisions %d-%d", logRoot.TreeSize-1, end)
 	if err := logClient.AddSequencedLeaves(ctx, leaves); err != nil {
 		glog.Errorf("AddSequencedLeaves(revs: %v): %v", revs, err)
 		return nil, err

From 64e70194d189b8b75e2c360f8136006c6135f785 Mon Sep 17 00:00:00 2001
From: Gary Belvin <gdb@google.com>
Date: Wed, 13 Nov 2019 12:01:00 +0000
Subject: [PATCH 14/22] Library for converting time.Time to sequencer
 watermarks (#1381)

* Use metadata API in tests

* Cleaner metadata API

* Use new metadata API in servers

* Use metadata API in sequencer
---
 core/keyserver/paginator.go                 |  4 +-
 core/keyserver/paginator_test.go            | 35 +++++++----
 core/keyserver/revisions.go                 |  2 +-
 core/sequencer/metadata/sourceslice.go      | 48 ++++++++++++++-
 core/sequencer/metadata/sourceslice_test.go | 37 ++++++++++++
 core/sequencer/server.go                    | 37 ++++++++----
 core/sequencer/server_test.go               | 66 +++++++++++++--------
 7 files changed, 174 insertions(+), 55 deletions(-)
 create mode 100644 core/sequencer/metadata/sourceslice_test.go

diff --git a/core/keyserver/paginator.go b/core/keyserver/paginator.go
index 132f3996b..7bf9cd3be 100644
--- a/core/keyserver/paginator.go
+++ b/core/keyserver/paginator.go
@@ -65,7 +65,7 @@ func (s SourceList) First() *rtpb.ReadToken {
 		// Empty struct means there is nothing else to page through.
 		return &rtpb.ReadToken{}
 	}
-	st, err := ptypes.TimestampProto(metadata.Source(s[0]).StartTime())
+	st, err := ptypes.TimestampProto(metadata.FromProto(s[0]).StartTime())
 	if err != nil {
 		panic("invalid timestamp")
 	}
@@ -97,7 +97,7 @@ func (s SourceList) Next(rt *rtpb.ReadToken, lastRow *mutator.LogMessage) *rtpb.
 		return &rtpb.ReadToken{} // Encodes to ""
 	}
 
-	st, err := ptypes.TimestampProto(metadata.Source(s[rt.SliceIndex+1]).StartTime())
+	st, err := ptypes.TimestampProto(metadata.FromProto(s[rt.SliceIndex+1]).StartTime())
 	if err != nil {
 		panic("invalid timestamp")
 	}
diff --git a/core/keyserver/paginator_test.go b/core/keyserver/paginator_test.go
index 9340182e7..76843b027 100644
--- a/core/keyserver/paginator_test.go
+++ b/core/keyserver/paginator_test.go
@@ -19,13 +19,23 @@ import (
 	"time"
 
 	"github.com/golang/protobuf/proto"
-
 	"github.com/google/keytransparency/core/mutator"
+	"github.com/google/keytransparency/core/sequencer/metadata"
+	spb "github.com/google/keytransparency/core/sequencer/sequencer_go_proto"
 
 	tpb "github.com/golang/protobuf/ptypes/timestamp"
 	rtpb "github.com/google/keytransparency/core/keyserver/readtoken_go_proto"
 )
 
+func newSourceSlice(t *testing.T, logID int64, low, high time.Time) *spb.MapMetadata_SourceSlice {
+	t.Helper()
+	s, err := metadata.New(logID, low, high)
+	if err != nil {
+		t.Fatalf("Invalid source: %v", err)
+	}
+	return s.Proto()
+}
+
 func TestEncodeToken(t *testing.T) {
 	for _, tc := range []struct {
 		rt   *rtpb.ReadToken
@@ -71,16 +81,16 @@ func TestTokenEncodeDecode(t *testing.T) {
 }
 
 func TestFirst(t *testing.T) {
+	start := time.Unix(0, 0)
 	for _, tc := range []struct {
 		s    SourceList
 		want *rtpb.ReadToken
 	}{
 		{
 			s: SourceList{
-				{LogId: 2, LowestInclusive: 2, HighestExclusive: 11},
-				{LogId: 3, LowestInclusive: 11, HighestExclusive: 21},
-			},
-			want: &rtpb.ReadToken{SliceIndex: 0, StartTime: &tpb.Timestamp{Nanos: 2}},
+				newSourceSlice(t, 2, start.Add(2*time.Microsecond), start.Add(11*time.Microsecond)),
+				newSourceSlice(t, 3, start.Add(11*time.Microsecond), start.Add(21*time.Microsecond))},
+			want: &rtpb.ReadToken{SliceIndex: 0, StartTime: timestamp(t, start.Add(2*time.Microsecond))},
 		},
 		{s: SourceList{}, want: &rtpb.ReadToken{}},
 	} {
@@ -91,9 +101,10 @@ func TestFirst(t *testing.T) {
 }
 
 func TestNext(t *testing.T) {
+	start := time.Unix(1, 0)
 	a := SourceList{
-		{LogId: 2, LowestInclusive: 2, HighestExclusive: 11},
-		{LogId: 3, LowestInclusive: 11, HighestExclusive: 21},
+		newSourceSlice(t, 2, start.Add(2*time.Microsecond), start.Add(11*time.Microsecond)),
+		newSourceSlice(t, 3, start.Add(11*time.Microsecond), start.Add(21*time.Microsecond)),
 	}
 	for _, tc := range []struct {
 		s       SourceList
@@ -105,16 +116,16 @@ func TestNext(t *testing.T) {
 		{
 			desc:    "first page",
 			s:       a,
-			rt:      &rtpb.ReadToken{SliceIndex: 0, StartTime: &tpb.Timestamp{Nanos: 2}},
-			lastRow: &mutator.LogMessage{ID: time.Unix(0, 6)},
-			want:    &rtpb.ReadToken{SliceIndex: 0, StartTime: &tpb.Timestamp{Nanos: 6}},
+			rt:      &rtpb.ReadToken{SliceIndex: 0, StartTime: timestamp(t, start.Add(2*time.Microsecond))},
+			lastRow: &mutator.LogMessage{ID: start.Add(6 * time.Microsecond)},
+			want:    &rtpb.ReadToken{SliceIndex: 0, StartTime: timestamp(t, start.Add(6*time.Microsecond))},
 		},
 		{
 			desc:    "next source",
 			s:       a,
 			rt:      &rtpb.ReadToken{},
 			lastRow: nil,
-			want:    &rtpb.ReadToken{SliceIndex: 1, StartTime: &tpb.Timestamp{Nanos: 11}},
+			want:    &rtpb.ReadToken{SliceIndex: 1, StartTime: timestamp(t, start.Add(11*time.Microsecond))},
 		},
 		{
 			desc:    "last page",
@@ -126,7 +137,7 @@ func TestNext(t *testing.T) {
 		{
 			desc:    "empty",
 			s:       SourceList{},
-			rt:      &rtpb.ReadToken{SliceIndex: 1, StartTime: &tpb.Timestamp{Nanos: 2}},
+			rt:      &rtpb.ReadToken{SliceIndex: 1, StartTime: timestamp(t, start.Add(2*time.Microsecond))},
 			lastRow: nil,
 			want:    &rtpb.ReadToken{},
 		},
diff --git a/core/keyserver/revisions.go b/core/keyserver/revisions.go
index fd12872ec..dc67f6ea1 100644
--- a/core/keyserver/revisions.go
+++ b/core/keyserver/revisions.go
@@ -141,7 +141,7 @@ func (s *Server) ListMutations(ctx context.Context, in *pb.ListMutationsRequest)
 	}
 
 	// Read PageSize + 1 messages from the log to see if there is another page.
-	high := metadata.Source(meta.Sources[rt.SliceIndex]).EndTime()
+	high := metadata.FromProto(meta.Sources[rt.SliceIndex]).EndTime()
 	logID := meta.Sources[rt.SliceIndex].LogId
 	low, err := ptypes.Timestamp(rt.StartTime)
 	if err != nil {
diff --git a/core/sequencer/metadata/sourceslice.go b/core/sequencer/metadata/sourceslice.go
index 9900cad52..01301eb06 100644
--- a/core/sequencer/metadata/sourceslice.go
+++ b/core/sequencer/metadata/sourceslice.go
@@ -15,17 +15,54 @@
 package metadata
 
 import (
+	"fmt"
+	"math"
 	"time"
 
 	spb "github.com/google/keytransparency/core/sequencer/sequencer_go_proto"
 )
 
-// Source returns a wrapper for SourceSlice
-func Source(s *spb.MapMetadata_SourceSlice) SourceSlice {
-	return SourceSlice{s: s}
+var minTime = time.Unix(0, math.MinInt64) // 1677-09-21 00:12:43.145224192
+var maxTime = time.Unix(0, math.MaxInt64) // 2262-04-11 23:47:16.854775807
+
+// validateTime determines whether a timestamp is valid.
+// Valid timestamps can be represented by an int64 number of microseconds from the epoch.
+//
+// Every valid timestamp can be represented by a time.Time, but the converse is not true.
+func validateTime(ts time.Time) error {
+	if ts.Before(minTime) {
+		return fmt.Errorf("timestamp %v before %v", ts, minTime)
+	}
+	if ts.After(maxTime) {
+		return fmt.Errorf("timestamp %v after %v", ts, maxTime)
+	}
+	return nil
+}
+
+// New creates a new source slice from time objects.
+// Returns an error if the time objects cannot be represented correctly.
+func New(logID int64, low, high time.Time) (*SourceSlice, error) {
+	if err := validateTime(low); err != nil {
+		return nil, err
+	}
+	if err := validateTime(high); err != nil {
+		return nil, err
+	}
+	return &SourceSlice{s: &spb.MapMetadata_SourceSlice{
+		LogId:            logID,
+		LowestInclusive:  low.UnixNano(),
+		HighestExclusive: high.UnixNano(),
+	}}, nil
+}
+
+// FromProto returns a wrapper for SourceSlice
+func FromProto(s *spb.MapMetadata_SourceSlice) *SourceSlice {
+	return &SourceSlice{s: s}
 }
 
 // SourceSlice defines accessor and conversion methods for MapMetadata_SourceSlice
+// TODO(gbelvin): fully migrate to source slices that encode time directly.
+// For now, we need to have a wrapper to do the conversions between time and int64 consistently.
 type SourceSlice struct {
 	s *spb.MapMetadata_SourceSlice
 }
@@ -39,3 +76,8 @@ func (s SourceSlice) StartTime() time.Time {
 func (s SourceSlice) EndTime() time.Time {
 	return time.Unix(0, s.s.GetHighestExclusive())
 }
+
+// Proto returns the proto representation of SourceSlice
+func (s *SourceSlice) Proto() *spb.MapMetadata_SourceSlice {
+	return s.s
+}
diff --git a/core/sequencer/metadata/sourceslice_test.go b/core/sequencer/metadata/sourceslice_test.go
new file mode 100644
index 000000000..46bc7ec9c
--- /dev/null
+++ b/core/sequencer/metadata/sourceslice_test.go
@@ -0,0 +1,37 @@
+// Copyright 2019 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package metadata
+
+import (
+	"testing"
+	"time"
+)
+
+func TestValidateTime(t *testing.T) {
+	for _, tc := range []struct {
+		ts    time.Time
+		valid bool
+	}{
+		{ts: time.Time{}, valid: false},
+		{ts: time.Date(1, 0, 0, 0, 0, 0, 0, time.UTC), valid: false},
+		{ts: time.Date(1000, 0, 0, 0, 0, 0, 0, time.UTC), valid: false},
+		{ts: time.Date(2000, 0, 0, 0, 0, 0, 0, time.UTC), valid: true},
+		{ts: time.Date(200000, 0, 0, 0, 0, 0, 0, time.UTC), valid: false},
+	} {
+		err := validateTime(tc.ts)
+		if got := err == nil; got != tc.valid {
+			t.Errorf("validateTime(%v): %v, want valid: %v", tc.ts, err, tc.valid)
+		}
+	}
+}
diff --git a/core/sequencer/server.go b/core/sequencer/server.go
index 3d79e1224..8c317cb44 100644
--- a/core/sequencer/server.go
+++ b/core/sequencer/server.go
@@ -63,6 +63,13 @@ var (
 	unappliedRevisions monitoring.Gauge
 )
 
+// zero is the time to use for the beginning edge of new watermarks.
+// When getting the watermark for a new log that is not in the lastMeta set,
+// we need to pick a lower bound starting point for reading.  time.Time{} is
+// not a valid choice here because it is too low to represent with metadata.New()
+// so we need to pick a different, low sentinel value that the tests know about.
+var zero = time.Unix(0, 0)
+
 func createMetrics(mf monitoring.MetricFactory) {
 	knownDirectories = mf.NewGauge(
 		"known_directories",
@@ -339,7 +346,7 @@ func (s *Server) ApplyRevisions(ctx context.Context, in *spb.ApplyRevisionsReque
 func (s *Server) readMessages(ctx context.Context, source *spb.MapMetadata_SourceSlice,
 	directoryID string, chunkSize int32,
 	emit func(*mutator.LogMessage)) error {
-	ss := metadata.Source(source)
+	ss := metadata.FromProto(source)
 	low := ss.StartTime()
 	high := ss.EndTime()
 	for moreToRead := true; moreToRead; {
@@ -530,12 +537,13 @@ func (s *Server) HighWatermarks(ctx context.Context, directoryID string, lastMet
 	// revision.
 	// TODO(gbelvin): Separate end watermarks for the sequencer's needs
 	// from ranges of watermarks for the verifier's needs.
-	ends := make(map[int64]int64)
-	starts := make(map[int64]int64)
+	ends := make(map[int64]time.Time)
+	starts := make(map[int64]time.Time)
 	for _, source := range lastMeta.GetSources() {
-		if ends[source.LogId] < source.HighestExclusive {
-			ends[source.LogId] = source.HighestExclusive
-			starts[source.LogId] = source.HighestExclusive
+		highest := metadata.FromProto(source).EndTime()
+		if ends[source.LogId].Before(highest) {
+			ends[source.LogId] = highest
+			starts[source.LogId] = highest
 		}
 	}
 
@@ -546,25 +554,28 @@ func (s *Server) HighWatermarks(ctx context.Context, directoryID string, lastMet
 	}
 	// TODO(gbelvin): Get HighWatermarks in parallel.
 	for _, logID := range logIDs {
-		low := time.Unix(0, ends[logID])
+		low, ok := ends[logID]
+		if !ok {
+			low = zero // Here be dragons. See comment on zero.
+		}
 		count, high, err := s.logs.HighWatermark(ctx, directoryID, logID, low, batchSize)
 		if err != nil {
 			return 0, nil, status.Errorf(codes.Internal,
 				"HighWatermark(%v/%v, start: %v, batch: %v): %v",
 				directoryID, logID, low, batchSize, err)
 		}
-		starts[logID], ends[logID] = low.UnixNano(), high.UnixNano()
+		starts[logID], ends[logID] = low, high
 		total += count
 		batchSize -= count
 	}
 
 	meta := &spb.MapMetadata{}
 	for logID, end := range ends {
-		meta.Sources = append(meta.Sources, &spb.MapMetadata_SourceSlice{
-			LogId:            logID,
-			LowestInclusive:  starts[logID],
-			HighestExclusive: end,
-		})
+		src, err := metadata.New(logID, starts[logID], end)
+		if err != nil {
+			return 0, nil, err
+		}
+		meta.Sources = append(meta.Sources, src.Proto())
 	}
 	// Deterministic results are nice.
 	sort.Slice(meta.Sources, func(a, b int) bool {
diff --git a/core/sequencer/server_test.go b/core/sequencer/server_test.go
index 02f13677a..cbbaec730 100644
--- a/core/sequencer/server_test.go
+++ b/core/sequencer/server_test.go
@@ -26,11 +26,12 @@ import (
 	"github.com/google/trillian/types"
 	"google.golang.org/grpc"
 
+	"github.com/google/keytransparency/core/sequencer/mapper"
+	"github.com/google/keytransparency/core/sequencer/metadata"
+	"github.com/google/keytransparency/core/sequencer/runner"
 	"github.com/google/keytransparency/impl/memory"
 
 	pb "github.com/google/keytransparency/core/api/v1/keytransparency_go_proto"
-	"github.com/google/keytransparency/core/sequencer/mapper"
-	"github.com/google/keytransparency/core/sequencer/runner"
 	spb "github.com/google/keytransparency/core/sequencer/sequencer_go_proto"
 	tpb "github.com/google/trillian"
 )
@@ -116,6 +117,15 @@ func setupLogs(ctx context.Context, t *testing.T, dirID string, logLengths map[i
 	return fakeLogs, idx
 }
 
+func newSource(t *testing.T, logID int64, low, high time.Time) *spb.MapMetadata_SourceSlice {
+	t.Helper()
+	s, err := metadata.New(logID, low, high)
+	if err != nil {
+		t.Fatalf("Invalid source: %v", err)
+	}
+	return s.Proto()
+}
+
 func TestDefiningRevisions(t *testing.T) {
 	// Verify that outstanding revisions prevent future revisions from being created.
 	ctx := context.Background()
@@ -143,14 +153,14 @@ func TestDefiningRevisions(t *testing.T) {
 		{desc: "skewed", highestRev: mapRev - 1, wantNew: mapRev - 1},
 		{desc: "almost_drained", highestRev: mapRev,
 			meta: spb.MapMetadata{Sources: []*spb.MapMetadata_SourceSlice{
-				{LogId: 0, LowestInclusive: 0, HighestExclusive: 9},
-				{LogId: 1, LowestInclusive: 0, HighestExclusive: 20},
+				newSource(t, 0, zero.Add(0*time.Microsecond), zero.Add(9*time.Microsecond)),
+				newSource(t, 1, zero.Add(0*time.Microsecond), zero.Add(20*time.Microsecond)),
 			}},
 			wantNew: mapRev + 1},
 		{desc: "drained", highestRev: mapRev,
 			meta: spb.MapMetadata{Sources: []*spb.MapMetadata_SourceSlice{
-				{LogId: 0, LowestInclusive: 0, HighestExclusive: idx[0][9].Add(1).UnixNano()},
-				{LogId: 1, LowestInclusive: 0, HighestExclusive: idx[1][19].Add(1).UnixNano()},
+				newSource(t, 0, zero, idx[0][9].Add(1)),
+				newSource(t, 1, zero, idx[1][19].Add(1)),
 			}},
 			wantNew: mapRev},
 	} {
@@ -202,14 +212,14 @@ func TestReadMessages(t *testing.T) {
 		want      int
 	}{
 		{batchSize: 1, want: 9, meta: &spb.MapMetadata{Sources: []*spb.MapMetadata_SourceSlice{
-			{LogId: 0, LowestInclusive: idx[0][1].UnixNano(), HighestExclusive: idx[0][9].Add(1).UnixNano()},
+			newSource(t, 0, idx[0][1], idx[0][9].Add(1)),
 		}}},
 		{batchSize: 10000, want: 9, meta: &spb.MapMetadata{Sources: []*spb.MapMetadata_SourceSlice{
-			{LogId: 0, LowestInclusive: idx[0][1].UnixNano(), HighestExclusive: idx[0][9].Add(1).UnixNano()},
+			newSource(t, 0, idx[0][1], idx[0][9].Add(1)),
 		}}},
 		{batchSize: 1, want: 19, meta: &spb.MapMetadata{Sources: []*spb.MapMetadata_SourceSlice{
-			{LogId: 0, LowestInclusive: idx[0][1].UnixNano(), HighestExclusive: idx[0][9].Add(1).UnixNano()},
-			{LogId: 1, LowestInclusive: idx[1][1].UnixNano(), HighestExclusive: idx[1][10].Add(1).UnixNano()},
+			newSource(t, 0, idx[0][1], idx[0][9].Add(1)),
+			newSource(t, 1, idx[1][1], idx[1][10].Add(1)),
 		}}},
 	} {
 		logSlices := runner.DoMapMetaFn(mapper.MapMetaFn, tc.meta, fakeMetric)
@@ -238,34 +248,42 @@ func TestHighWatermarks(t *testing.T) {
 	}{
 		{desc: "nobatch", batchSize: 30, count: 30,
 			next: &spb.MapMetadata{Sources: []*spb.MapMetadata_SourceSlice{
-				{LogId: 0, HighestExclusive: idx[0][9].Add(1).UnixNano()},
-				{LogId: 1, HighestExclusive: idx[1][19].Add(1).UnixNano()}}}},
+				newSource(t, 0, zero, idx[0][9].Add(1)),
+				newSource(t, 1, zero, idx[1][19].Add(1)),
+			}}},
 		{desc: "exactbatch", batchSize: 20, count: 20,
 			next: &spb.MapMetadata{Sources: []*spb.MapMetadata_SourceSlice{
-				{LogId: 0, HighestExclusive: idx[0][9].Add(1).UnixNano()},
-				{LogId: 1, HighestExclusive: idx[1][9].Add(1).UnixNano()}}}},
+				newSource(t, 0, zero, idx[0][9].Add(1)),
+				newSource(t, 1, zero, idx[1][9].Add(1)),
+			}}},
 		{desc: "batchwprev", batchSize: 20, count: 20,
 			last: &spb.MapMetadata{Sources: []*spb.MapMetadata_SourceSlice{
-				{LogId: 0, HighestExclusive: idx[0][9].Add(2).UnixNano()}}},
+				newSource(t, 0, zero, idx[0][9].Add(2)),
+			}},
 			next: &spb.MapMetadata{Sources: []*spb.MapMetadata_SourceSlice{
 				// Nothing to read from log 1, preserve watermark of log 1.
-				{LogId: 0, LowestInclusive: idx[0][9].Add(2).UnixNano(), HighestExclusive: idx[0][9].Add(2).UnixNano()},
-				{LogId: 1, HighestExclusive: idx[1][19].Add(1).UnixNano()}}}},
+				newSource(t, 0, idx[0][9].Add(2), idx[0][9].Add(2)),
+				newSource(t, 1, zero, idx[1][19].Add(1)),
+			}}},
 		// Don't drop existing watermarks.
 		{desc: "keep existing", batchSize: 1, count: 1,
 			last: &spb.MapMetadata{Sources: []*spb.MapMetadata_SourceSlice{
-				{LogId: 1, HighestExclusive: 10}}},
+				newSource(t, 1, zero, zero.Add(10)),
+			}},
 			next: &spb.MapMetadata{Sources: []*spb.MapMetadata_SourceSlice{
-				{LogId: 0, HighestExclusive: idx[0][0].Add(1).UnixNano()},
+				newSource(t, 0, zero, idx[0][0].Add(1)),
 				// No reads from log 1, but don't drop the watermark.
-				{LogId: 1, LowestInclusive: 10, HighestExclusive: 10}}}},
+				newSource(t, 1, zero.Add(10), zero.Add(10)),
+			}}},
 		{desc: "logs that dont move", batchSize: 0, count: 0,
 			last: &spb.MapMetadata{Sources: []*spb.MapMetadata_SourceSlice{
-				{LogId: 3, HighestExclusive: 10}}},
+				newSource(t, 3, zero, zero.Add(10*time.Microsecond)),
+			}},
 			next: &spb.MapMetadata{Sources: []*spb.MapMetadata_SourceSlice{
-				{LogId: 0},
-				{LogId: 1},
-				{LogId: 3, LowestInclusive: 10, HighestExclusive: 10}}}},
+				newSource(t, 0, zero, zero),
+				newSource(t, 1, zero, zero),
+				newSource(t, 3, zero.Add(10*time.Microsecond), zero.Add(10*time.Microsecond)),
+			}}},
 	} {
 		t.Run(tc.desc, func(t *testing.T) {
 			count, next, err := s.HighWatermarks(ctx, directoryID, tc.last, tc.batchSize)

From f6260488e36d46a00fc73e4218b12fb3eecb33cb Mon Sep 17 00:00:00 2001
From: Gary Belvin <gdb@google.com>
Date: Wed, 13 Nov 2019 15:07:47 +0000
Subject: [PATCH 15/22] Define watermarks as micros (#1384)

* Add microseconds comment to proto
* metadata package uses microseconds
* Test inter-quanta timestamp conversion
* Use metadata helper in keyserver  test
---
 core/keyserver/revisions_test.go              | 14 ++++++++--
 core/sequencer/metadata/sourceslice.go        | 11 +++++---
 core/sequencer/metadata/sourceslice_test.go   | 26 +++++++++++++++++++
 core/sequencer/sequencer_api.proto            |  2 ++
 .../sequencer_go_proto/sequencer_api.pb.go    |  2 ++
 5 files changed, 49 insertions(+), 6 deletions(-)

diff --git a/core/keyserver/revisions_test.go b/core/keyserver/revisions_test.go
index e09079c67..85b3a9227 100644
--- a/core/keyserver/revisions_test.go
+++ b/core/keyserver/revisions_test.go
@@ -32,6 +32,7 @@ import (
 	protopb "github.com/golang/protobuf/ptypes/timestamp"
 	pb "github.com/google/keytransparency/core/api/v1/keytransparency_go_proto"
 	rtpb "github.com/google/keytransparency/core/keyserver/readtoken_go_proto"
+	"github.com/google/keytransparency/core/sequencer/metadata"
 	spb "github.com/google/keytransparency/core/sequencer/sequencer_go_proto"
 	tpb "github.com/google/trillian"
 )
@@ -87,6 +88,15 @@ func TestGetRevisionStream(t *testing.T) {
 	}
 }
 
+func newSource(t *testing.T, logID int64, low, high time.Time) *spb.MapMetadata_SourceSlice {
+	t.Helper()
+	s, err := metadata.New(logID, low, high)
+	if err != nil {
+		t.Fatalf("Invalid source: %v", err)
+	}
+	return s.Proto()
+}
+
 type batchStorage map[int64]SourceList // Map of Revision to Sources
 
 func (b batchStorage) ReadBatch(ctx context.Context, dirID string, rev int64) (*spb.MapMetadata, error) {
@@ -127,8 +137,8 @@ func TestListMutations(t *testing.T) {
 	}
 
 	fakeBatches := batchStorage{
-		1: SourceList{{LogId: 0, LowestInclusive: idx[2].UnixNano(), HighestExclusive: idx[7].UnixNano()}},
-		2: SourceList{{LogId: 0, LowestInclusive: idx[7].UnixNano(), HighestExclusive: idx[11].UnixNano()}},
+		1: SourceList{newSource(t, 0, idx[2], idx[7])},
+		2: SourceList{newSource(t, 0, idx[7], idx[11])},
 	}
 
 	for _, tc := range []struct {
diff --git a/core/sequencer/metadata/sourceslice.go b/core/sequencer/metadata/sourceslice.go
index 01301eb06..6a40bf042 100644
--- a/core/sequencer/metadata/sourceslice.go
+++ b/core/sequencer/metadata/sourceslice.go
@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+// Package metadata helps enforce a consistent standard of meaning around the map metadata object.
 package metadata
 
 import (
@@ -24,6 +25,8 @@ import (
 
 var minTime = time.Unix(0, math.MinInt64) // 1677-09-21 00:12:43.145224192
 var maxTime = time.Unix(0, math.MaxInt64) // 2262-04-11 23:47:16.854775807
+const quantum = time.Microsecond
+const nanosPerQuantum = int64(quantum / time.Nanosecond)
 
 // validateTime determines whether a timestamp is valid.
 // Valid timestamps can be represented by an int64 number of microseconds from the epoch.
@@ -50,8 +53,8 @@ func New(logID int64, low, high time.Time) (*SourceSlice, error) {
 	}
 	return &SourceSlice{s: &spb.MapMetadata_SourceSlice{
 		LogId:            logID,
-		LowestInclusive:  low.UnixNano(),
-		HighestExclusive: high.UnixNano(),
+		LowestInclusive:  low.Add(quantum-1).Truncate(quantum).UnixNano() / nanosPerQuantum,
+		HighestExclusive: high.Add(quantum-1).Truncate(quantum).UnixNano() / nanosPerQuantum,
 	}}, nil
 }
 
@@ -69,12 +72,12 @@ type SourceSlice struct {
 
 // StartTime returns LowestInclusive as a time.Time
 func (s SourceSlice) StartTime() time.Time {
-	return time.Unix(0, s.s.GetLowestInclusive())
+	return time.Unix(0, s.s.GetLowestInclusive()*nanosPerQuantum)
 }
 
 // EndTime returns HighestExclusive as a time.Time
 func (s SourceSlice) EndTime() time.Time {
-	return time.Unix(0, s.s.GetHighestExclusive())
+	return time.Unix(0, s.s.GetHighestExclusive()*nanosPerQuantum)
 }
 
 // Proto returns the proto representation of SourceSlice
diff --git a/core/sequencer/metadata/sourceslice_test.go b/core/sequencer/metadata/sourceslice_test.go
index 46bc7ec9c..ddbec2d22 100644
--- a/core/sequencer/metadata/sourceslice_test.go
+++ b/core/sequencer/metadata/sourceslice_test.go
@@ -16,6 +16,10 @@ package metadata
 import (
 	"testing"
 	"time"
+
+	"github.com/golang/protobuf/proto"
+
+	spb "github.com/google/keytransparency/core/sequencer/sequencer_go_proto"
 )
 
 func TestValidateTime(t *testing.T) {
@@ -35,3 +39,25 @@ func TestValidateTime(t *testing.T) {
 		}
 	}
 }
+
+func TestTimeToProto(t *testing.T) {
+	logID := int64(1)
+	for _, tc := range []struct {
+		low, high time.Time
+		want      *spb.MapMetadata_SourceSlice
+	}{
+		{low: time.Unix(0, 0), high: time.Unix(0, 0), want: &spb.MapMetadata_SourceSlice{LogId: logID, LowestInclusive: 0}},
+		{low: time.Unix(0, 1), high: time.Unix(0, 0), want: &spb.MapMetadata_SourceSlice{LogId: logID, LowestInclusive: 1}},
+		{low: time.Unix(0, 1000), high: time.Unix(0, 0), want: &spb.MapMetadata_SourceSlice{LogId: logID, LowestInclusive: 1}},
+		{low: time.Unix(1, 0), high: time.Unix(0, 0), want: &spb.MapMetadata_SourceSlice{LogId: logID, LowestInclusive: 1000000}},
+		{low: time.Unix(1, 0), high: time.Unix(1, 0), want: &spb.MapMetadata_SourceSlice{LogId: logID, LowestInclusive: 1000000, HighestExclusive: 1000000}},
+	} {
+		ss, err := New(logID, tc.low, tc.high)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if got := ss.Proto(); !proto.Equal(got, tc.want) {
+			t.Errorf("New().Proto(): %v, want %v", got, tc.want)
+		}
+	}
+}
diff --git a/core/sequencer/sequencer_api.proto b/core/sequencer/sequencer_api.proto
index 135acdffe..509cf00ba 100644
--- a/core/sequencer/sequencer_api.proto
+++ b/core/sequencer/sequencer_api.proto
@@ -31,10 +31,12 @@ message MapMetadata {
     // lowest_inclusive is the lowest primary key (inclusive) of the source
     // log that has been incorporated into this map revision. The primary
     // keys of logged items MUST be monotonically increasing.
+    // Defined in terms of microseconds from the Unix epoch.
     int64 lowest_inclusive = 1;
     // highest_exclusive is the highest primary key (exclusive) of the source
     // log that has been incorporated into this map revision. The primary keys
     // of logged items MUST be monotonically increasing.
+    // Defined in terms of microseconds from the Unix epoch.
     int64 highest_exclusive = 2;
     // log_id is the ID of the source log.
     int64 log_id = 3;
diff --git a/core/sequencer/sequencer_go_proto/sequencer_api.pb.go b/core/sequencer/sequencer_go_proto/sequencer_api.pb.go
index 970d0dee6..c99276234 100644
--- a/core/sequencer/sequencer_go_proto/sequencer_api.pb.go
+++ b/core/sequencer/sequencer_go_proto/sequencer_api.pb.go
@@ -76,10 +76,12 @@ type MapMetadata_SourceSlice struct {
 	// lowest_inclusive is the lowest primary key (inclusive) of the source
 	// log that has been incorporated into this map revision. The primary
 	// keys of logged items MUST be monotonically increasing.
+	// Defined in terms of microseconds from the Unix epoch.
 	LowestInclusive int64 `protobuf:"varint,1,opt,name=lowest_inclusive,json=lowestInclusive,proto3" json:"lowest_inclusive,omitempty"`
 	// highest_exclusive is the highest primary key (exclusive) of the source
 	// log that has been incorporated into this map revision. The primary keys
 	// of logged items MUST be monotonically increasing.
+	// Defined in terms of microseconds from the Unix epoch.
 	HighestExclusive int64 `protobuf:"varint,2,opt,name=highest_exclusive,json=highestExclusive,proto3" json:"highest_exclusive,omitempty"`
 	// log_id is the ID of the source log.
 	LogId                int64    `protobuf:"varint,3,opt,name=log_id,json=logId,proto3" json:"log_id,omitempty"`

From b74adc4c58e2e2c0394e226e27552589f26956d5 Mon Sep 17 00:00:00 2001
From: Gary Belvin <gdb@google.com>
Date: Wed, 13 Nov 2019 15:29:01 +0000
Subject: [PATCH 16/22] Emit metric with the right dimensions (#1383)

Emitting the wrong number of dimensions will cause an error.
---
 core/sequencer/server.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/core/sequencer/server.go b/core/sequencer/server.go
index 8c317cb44..1ad78a6e9 100644
--- a/core/sequencer/server.go
+++ b/core/sequencer/server.go
@@ -386,7 +386,6 @@ func (s *Server) ApplyRevision(ctx context.Context, in *spb.ApplyRevisionRequest
 	logSlices := runner.DoMapMetaFn(mapper.MapMetaFn, meta, incMetricFn)
 	logItems, err := runner.DoReadFn(ctx, s.readMessages, logSlices, in.DirectoryId, s.BatchSize, incMetricFn)
 	if err != nil {
-		mutationFailures.Inc(err.Error())
 		return nil, err
 	}
 

From addbb266ef589fede434ff1cf7fabe01f2c7b6b7 Mon Sep 17 00:00:00 2001
From: Gary Belvin <gdb@google.com>
Date: Sat, 23 Nov 2019 19:11:27 -0800
Subject: [PATCH 17/22] Update install instructions for go 1.13 (#1388)

* Remove -u from instructions

-u is no longer needed now that we have go modules

* Document that go 1.13 is required
---
 README.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 7d04fe30b..ae93265ba 100644
--- a/README.md
+++ b/README.md
@@ -32,8 +32,8 @@ development.
 ## Key Transparency Client
 
 ### Setup
-1. Install [Go 1.10](https://golang.org/doc/install).
-2. `go get -u github.com/google/keytransparency/cmd/keytransparency-client `
+1. Install [Go 1.13](https://golang.org/doc/install).
+2. `go get github.com/google/keytransparency/cmd/keytransparency-client `
 
 ### Client operations
 
@@ -96,8 +96,8 @@ NB A default for the Key Transparency server URL is being used here. The default
    - Docker Compose 1.11.0+ `docker-compose --version`
 
 ```sh
-go get -u github.com/google/keytransparency/...
-go get -u github.com/google/trillian/...
+go get github.com/google/keytransparency/...
+go get github.com/google/trillian/...
 cd $(go env GOPATH)/src/github.com/google/keytransparency
 ./scripts/prepare_server.sh -f
 docker-compose -f docker-compose.yml docker-compose.prod.yml up

From 56dbae323252aa56b0f3c39d879a8b2b25ab5652 Mon Sep 17 00:00:00 2001
From: Gary Belvin <gdb@google.com>
Date: Sat, 23 Nov 2019 19:25:30 -0800
Subject: [PATCH 18/22] Upgrade dependencies (#1387)

```
go get github.com/google/trillian v1.3.3
go get -u ./...
go mod tidy
```
---
 go.mod |  86 +++++++++++-------
 go.sum | 282 ++++++++++++++++++++++++++++++++++++++++++++++++---------
 2 files changed, 291 insertions(+), 77 deletions(-)

diff --git a/go.mod b/go.mod
index 0435b0bcc..207ad7157 100644
--- a/go.mod
+++ b/go.mod
@@ -3,55 +3,73 @@ module github.com/google/keytransparency
 go 1.12
 
 require (
-	github.com/Masterminds/sprig v2.20.0+incompatible // indirect
+	cloud.google.com/go v0.49.0 // indirect
+	cloud.google.com/go/spanner v1.1.0 // indirect
+	github.com/Masterminds/semver v1.5.0 // indirect
+	github.com/Masterminds/sprig v2.22.0+incompatible // indirect
 	github.com/benlaurie/objecthash v0.0.0-20180202135721-d1e3d6079fc1
-	github.com/coreos/etcd v3.3.15+incompatible // indirect
+	github.com/bombsimon/wsl v1.2.8 // indirect
+	github.com/cespare/xxhash/v2 v2.1.1 // indirect
+	github.com/coreos/etcd v3.3.17+incompatible // indirect
 	github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect
-	github.com/emicklei/proto v1.6.15 // indirect
-	github.com/fullstorydev/grpcurl v1.3.2 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
+	github.com/emicklei/proto v1.8.0 // indirect
 	github.com/go-sql-driver/mysql v1.4.1
-	github.com/gobuffalo/flect v0.1.6 // indirect
-	github.com/gogo/protobuf v1.3.0 // indirect
+	github.com/gogo/protobuf v1.3.1 // indirect
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
+	github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9 // indirect
 	github.com/golang/mock v1.3.1
 	github.com/golang/protobuf v1.3.2
-	github.com/golangci/golangci-lint v1.18.0 // indirect
-	github.com/google/go-cmp v0.3.0
+	github.com/golangci/golangci-lint v1.21.0 // indirect
+	github.com/google/certificate-transparency-go v1.1.0 // indirect
+	github.com/google/go-cmp v0.3.1
 	github.com/google/tink v1.2.1-0.20190523150020-6495d823d968
-	github.com/google/trillian v1.3.2-0.20190910105707-fa759d0a2aee
+	github.com/google/trillian v1.3.3
 	github.com/gorilla/websocket v1.4.1 // indirect
 	github.com/gostaticanalysis/analysisutil v0.0.3 // indirect
-	github.com/grpc-ecosystem/go-grpc-middleware v1.0.0
+	github.com/grpc-ecosystem/go-grpc-middleware v1.1.0
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
-	github.com/grpc-ecosystem/grpc-gateway v1.11.1
+	github.com/grpc-ecosystem/grpc-gateway v1.9.4
+	github.com/imdario/mergo v0.3.8 // indirect
+	github.com/json-iterator/go v1.1.8 // indirect
+	github.com/jstemmer/go-junit-report v0.9.1 // indirect
 	github.com/kr/pretty v0.1.0
 	github.com/kylelemons/godebug v1.1.0
 	github.com/lyft/protoc-gen-validate v0.1.0 // indirect
-	github.com/mattn/go-isatty v0.0.9 // indirect
-	github.com/mwitkow/go-proto-validators v0.1.0 // indirect
-	github.com/prometheus/client_golang v1.1.0
-	github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 // indirect
-	github.com/prometheus/procfs v0.0.4 // indirect
+	github.com/mattn/go-isatty v0.0.10 // indirect
+	github.com/mattn/go-runewidth v0.0.6 // indirect
+	github.com/mitchellh/copystructure v1.0.0 // indirect
+	github.com/mitchellh/reflectwalk v1.0.1 // indirect
+	github.com/mwitkow/go-proto-validators v0.2.0 // indirect
+	github.com/olekukonko/tablewriter v0.0.2 // indirect
+	github.com/pelletier/go-toml v1.6.0 // indirect
+	github.com/prometheus/client_golang v1.2.1
+	github.com/prometheus/procfs v0.0.7 // indirect
 	github.com/russross/blackfriday v2.0.0+incompatible // indirect
-	github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
+	github.com/securego/gosec v0.0.0-20191119104125-df484bfa9e9f // indirect
 	github.com/spf13/cobra v0.0.5
-	github.com/spf13/pflag v1.0.3
-	github.com/spf13/viper v1.4.0
-	github.com/ultraware/funlen v0.0.2 // indirect
-	github.com/urfave/cli v1.22.0 // indirect
-	go.etcd.io/etcd v3.3.15+incompatible // indirect
-	golang.org/x/crypto v0.0.0-20190909091759-094676da4a83
-	golang.org/x/net v0.0.0-20190909003024-a7b16738d86b // indirect
+	github.com/spf13/pflag v1.0.5
+	github.com/spf13/viper v1.5.0
+	github.com/uber/prototool v1.9.0 // indirect
+	github.com/urfave/cli v1.22.1 // indirect
+	github.com/uudashr/gocognit v1.0.0 // indirect
+	go.etcd.io/etcd v3.3.17+incompatible // indirect
+	go.opencensus.io v0.22.2 // indirect
+	go.uber.org/atomic v1.5.1 // indirect
+	go.uber.org/multierr v1.4.0 // indirect
+	go.uber.org/zap v1.13.0 // indirect
+	golang.org/x/crypto v0.0.0-20191119213627-4f8c1d86b1ba
+	golang.org/x/net v0.0.0-20191119073136-fc4aabc6c914 // indirect
 	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
-	golang.org/x/sync v0.0.0-20190423024810-112230192c58
-	golang.org/x/sys v0.0.0-20190910064555-bbd175535a8b // indirect
-	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4
-	golang.org/x/tools v0.0.0-20190910044552-dd2b5c81c578 // indirect
-	google.golang.org/api v0.7.0
-	google.golang.org/genproto v0.0.0-20190905072037-92dd089d5514
-	google.golang.org/grpc v1.23.0
-	mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f // indirect
+	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
+	golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e // indirect
+	golang.org/x/time v0.0.0-20191024005414-555d28b269f0
+	golang.org/x/tools v0.0.0-20191120221951-8fd459516a27 // indirect
+	google.golang.org/api v0.14.0
+	google.golang.org/appengine v1.6.5 // indirect
+	google.golang.org/genproto v0.0.0-20191115221424-83cc0476cb11
+	google.golang.org/grpc v1.25.1
+	gopkg.in/yaml.v2 v2.2.7 // indirect
+	mvdan.cc/unparam v0.0.0-20191111180625-960b1ec0f2c2 // indirect
 	sigs.k8s.io/yaml v1.1.0 // indirect
 )
-
-replace google.golang.org/genproto v0.0.0-20170818100345-ee236bd376b0 => google.golang.org/genproto v0.0.0-20170818010345-ee236bd376b0
diff --git a/go.sum b/go.sum
index cd497d197..3072b8afd 100644
--- a/go.sum
+++ b/go.sum
@@ -6,8 +6,25 @@ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
 cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
 cloud.google.com/go v0.41.0 h1:NFvqUTDnSNYPX5oReekmB+D+90jrJIcVImxQ3qrBVgM=
 cloud.google.com/go v0.41.0/go.mod h1:OauMR7DV8fzvZIl2qg6rkaIhD/vmgk4iwEw/h6ercmg=
+cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
+cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
+cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
+cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
+cloud.google.com/go v0.49.0 h1:CH+lkubJzcPYB1Ggupcq0+k8Ni2ILdG2lYjDIgavDBQ=
+cloud.google.com/go v0.49.0/go.mod h1:hGvAdzcWNbyuxS3nWhD7H2cIJxjRRTRLQVB0bdputVY=
+cloud.google.com/go/bigquery v1.0.1 h1:hL+ycaJpVE9M7nLoiXb/Pn10ENE2u+oddxbD8uu0ZVU=
+cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
+cloud.google.com/go/datastore v1.0.0 h1:Kt+gOPPp2LEPWp8CSfxhsM8ik9CcyE/gYu+0r+RnZvM=
+cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
+cloud.google.com/go/pubsub v1.0.1 h1:W9tAK3E57P75u0XLLR82LZyw8VpAnhmyTOxW9qzmyj8=
+cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go/spanner v1.1.0 h1:hIjiz2Pf6Hy3BWz+Oaw7XUqP+EzWDkj0/DtTkKazxzk=
+cloud.google.com/go/spanner v1.1.0/go.mod h1:TzTaF9l2ZY2CIetNvVpUu6ZQy8YEOtzB6ICa5EwYjL0=
+cloud.google.com/go/storage v1.0.0 h1:VV2nUM3wwLLGh9lSABFgZMjInyUbJeaRSE64WuAIQ+4=
+cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
 contrib.go.opencensus.io/exporter/stackdriver v0.12.1/go.mod h1:iwB6wGarfphGGe/e5CWqyUk/cLzKnWsOKPVW3no6OTw=
 contrib.go.opencensus.io/resource v0.1.1/go.mod h1:F361eGI91LCmW1I/Saf+rX0+OFcigGlFvXwEGEnkRLA=
+dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
@@ -15,18 +32,24 @@ github.com/Masterminds/goutils v1.1.0 h1:zukEsf/1JZwCMgHiK3GZftabmxiCw4apj3a28RP
 github.com/Masterminds/goutils v1.1.0/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
 github.com/Masterminds/semver v1.4.2 h1:WBLTQ37jOCzSLtXNdoo8bNM8876KhNqOKvrlGITgsTc=
 github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
+github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww=
+github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
 github.com/Masterminds/sprig v2.18.0+incompatible h1:QoGhlbC6pter1jxKnjMFxT8EqsLuDE6FEcNbWEpw+lI=
 github.com/Masterminds/sprig v2.18.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o=
-github.com/Masterminds/sprig v2.20.0+incompatible h1:dJTKKuUkYW3RMFdQFXPU/s6hg10RgctmTjRcbZ98Ap8=
-github.com/Masterminds/sprig v2.20.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o=
+github.com/Masterminds/sprig v2.22.0+incompatible h1:z4yfnGrZ7netVz+0EDJ0Wi+5VZCSYp4Z0m2dk6cEM60=
+github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/OpenPeeDeeP/depguard v1.0.0 h1:k9QF73nrHT3nPLz3lu6G5s+3Hi8Je36ODr1F5gjAXXM=
 github.com/OpenPeeDeeP/depguard v1.0.0/go.mod h1:7/4sitnI9YlQgTLLk734QlzXT8DuHVnAyztLplQjk+o=
+github.com/OpenPeeDeeP/depguard v1.0.1 h1:VlW4R6jmBIv3/u1JNlawEvJMM4J+dPORPaZasQee8Us=
+github.com/OpenPeeDeeP/depguard v1.0.1/go.mod h1:xsIw86fROiiwelg+jB2uM9PiKihMMmUx/1V+TNhjQvM=
 github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
 github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/aws/aws-sdk-go v1.19.18/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
@@ -40,8 +63,18 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/bombsimon/wsl v1.2.5 h1:9gTOkIwVtoDZywvX802SDHokeX4kW1cKnV8ZTVAPkRs=
+github.com/bombsimon/wsl v1.2.5/go.mod h1:43lEF/i0kpXbLCeDXL9LMT8c92HyBywXb0AsgMHYngM=
+github.com/bombsimon/wsl v1.2.8 h1:b+E/W7koicKBZDU+vEsw/hnQTN8026Gv1eMZDLUU/Wc=
+github.com/bombsimon/wsl v1.2.8/go.mod h1:43lEF/i0kpXbLCeDXL9LMT8c92HyBywXb0AsgMHYngM=
 github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/cespare/xxhash/v2 v2.1.0 h1:yTUvW7Vhb89inJ+8irsUqiWjh8iT6sQPZiQzI6ReGkA=
+github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM=
+github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
+github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/coreos/bbolt v1.3.2 h1:wZwiHHUieZCquLkDL0B8UhzreNWsPHooDAG3q34zk0s=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
@@ -51,8 +84,8 @@ github.com/coreos/etcd v3.3.10+incompatible h1:jFneRYjIvLMLhDLCzuTuU4rSJUjRplcJQ
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/etcd v3.3.13+incompatible h1:8F3hqu9fGYLBifCmRCJsicFqDx/D68Rt3q1JMazcgBQ=
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
-github.com/coreos/etcd v3.3.15+incompatible h1:+9RjdC18gMxNQVvSiXvObLu29mOFmkgdsB4cRTlV+EE=
-github.com/coreos/etcd v3.3.15+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/etcd v3.3.17+incompatible h1:f/Z3EoDSx1yjaIjLQGo1diYUlQYSBrrAQ5vP8NjwXwo=
+github.com/coreos/etcd v3.3.17+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
 github.com/coreos/go-semver v0.2.0 h1:3Jm3tLmsgAYcjC+4Up7hJrFBPr+n7rAqYeSw/SZazuY=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
@@ -68,6 +101,11 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbp
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk=
 github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
+github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY=
+github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM=
+github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -79,29 +117,34 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn
 github.com/emicklei/proto v1.6.12/go.mod h1:rn1FgRS/FANiZdD2djyH7TMA9jdRDcYQ9IEN9yvjX0A=
 github.com/emicklei/proto v1.6.13 h1:8iuAuKbFmFhkmstObb0EV/Hrn9W+x6EuV1y5Da8Ye9E=
 github.com/emicklei/proto v1.6.13/go.mod h1:rn1FgRS/FANiZdD2djyH7TMA9jdRDcYQ9IEN9yvjX0A=
-github.com/emicklei/proto v1.6.15 h1:XbpwxmuOPrdES97FrSfpyy67SSCV/wBIKXqgJzh6hNw=
-github.com/emicklei/proto v1.6.15/go.mod h1:rn1FgRS/FANiZdD2djyH7TMA9jdRDcYQ9IEN9yvjX0A=
+github.com/emicklei/proto v1.7.0/go.mod h1:rn1FgRS/FANiZdD2djyH7TMA9jdRDcYQ9IEN9yvjX0A=
+github.com/emicklei/proto v1.8.0 h1:/MjKUvy7nh0ryszHc/PIIFiv/BU5XYX4NfvpM19C5+U=
+github.com/emicklei/proto v1.8.0/go.mod h1:rn1FgRS/FANiZdD2djyH7TMA9jdRDcYQ9IEN9yvjX0A=
 github.com/emirpasic/gods v1.9.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
 github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/fatih/color v1.6.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/fullstorydev/grpcurl v1.3.0 h1:XwiEVudma689HpecmJX6sTmOQCd1G5Cyqyj+zSqdk30=
-github.com/fullstorydev/grpcurl v1.3.0/go.mod h1:MtIe/E5RoUPHdbcijFZe/WoJKQtal5JNj0KGIbVqCy8=
-github.com/fullstorydev/grpcurl v1.3.1 h1:mSFlRqdcfDkDujpG96nGLAZcSDpG0UnxtoCFQx83AiI=
-github.com/fullstorydev/grpcurl v1.3.1/go.mod h1:Hsh6mJe5uN/Oqxv/QDmmUClKE+wMs4PUNh/Xz16H0R8=
 github.com/fullstorydev/grpcurl v1.3.2 h1:cJKWsBYMocdxXQvgbnhtLG810SL5MhKT4K7BagxRih8=
 github.com/fullstorydev/grpcurl v1.3.2/go.mod h1:kvk8xPCXOrwVd9zYdjy+xSOT4YWm6kyth4Y9NMfBns4=
+github.com/fullstorydev/grpcurl v1.4.0 h1:rKQyAaegPtCj4mpItnCHd+PIEHspIZl14VWhHYIHhls=
+github.com/fullstorydev/grpcurl v1.4.0/go.mod h1:kvk8xPCXOrwVd9zYdjy+xSOT4YWm6kyth4Y9NMfBns4=
 github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
 github.com/gliderlabs/ssh v0.1.4/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
 github.com/go-critic/go-critic v0.3.5-0.20190526074819-1df300866540 h1:djv/qAomOVj8voCHt0M0OYwR/4vfDq1zNKSPKjJCexs=
 github.com/go-critic/go-critic v0.3.5-0.20190526074819-1df300866540/go.mod h1:+sE8vrLDS2M0pZkBk0wy6+nLdKexVDrl/jBqQOTDThA=
+github.com/go-critic/go-critic v0.3.5-0.20190904082202-d79a9f0c64db h1:GYXWx7Vr3+zv833u+8IoXbNnQY0AdXsxAgI0kX7xcwA=
+github.com/go-critic/go-critic v0.3.5-0.20190904082202-d79a9f0c64db/go.mod h1:+sE8vrLDS2M0pZkBk0wy6+nLdKexVDrl/jBqQOTDThA=
+github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-lintpack/lintpack v0.5.2 h1:DI5mA3+eKdWeJ40nU4d6Wc26qmdG8RCi/btYq0TuRN0=
 github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
@@ -139,6 +182,7 @@ github.com/gobuffalo/flect v0.1.6 h1:D7KWNRFiCknJKA495/e1BO7oxqf8tbieaLv/ehoZ/+g
 github.com/gobuffalo/flect v0.1.6/go.mod h1:W3K3X9ksuZfir8f/LrfVtWmCDQFfayuylOJ7sz/Fj80=
 github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
 github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
+github.com/gofrs/flock v0.0.0-20190320160742-5135e617513b/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
 github.com/gofrs/flock v0.7.1 h1:DP+LD/t0njgoPBvT5MJLeliUIVQR03hiKR6vezdwHlc=
 github.com/gofrs/flock v0.7.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
 github.com/gogo/protobuf v1.1.1 h1:72R+M5VuhED/KujmZVcIquuo8mBgX4oVda//DQb3PXo=
@@ -147,12 +191,16 @@ github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/gogo/protobuf v1.3.0 h1:G8O7TerXerS4F6sx9OV7/nRfJdnXgHZu/S/7F2SN+UE=
 github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
+github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
+github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9 h1:uHTyIjqVhYRhLbJ8nIiOJHkEZZ+5YoOsAbD3sk82NiE=
+github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.0.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.1.1 h1:G5FRp8JnTd7RQH5kemVNlMeyXQAztQ3mOWV95KxsXH8=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
@@ -183,16 +231,20 @@ github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d h1:pXTK/gkVNs7Zyy
 github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU=
 github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98 h1:0OkFarm1Zy2CjCiDKfK9XHgmc2wbDlRMD2hD8anAJHU=
 github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
+github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a h1:iR3fYXUjHCR97qWS8ch1y9zPNsgXThGwjKPrYfqMPks=
+github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
 github.com/golangci/golangci-lint v1.17.2-0.20190910081718-bad04bb7378f h1:Ocb3mZ76SbwTM6VKfiMPEppkwzitinZFRW9E6zAD5qc=
 github.com/golangci/golangci-lint v1.17.2-0.20190910081718-bad04bb7378f/go.mod h1:kaqo8l0OZKYPtjNmG4z4HrWLgcYNIJ9B9q3LWri9uLg=
-github.com/golangci/golangci-lint v1.18.0 h1:XmQgfcLofSG/6AsQuQqmLizB+3GggD+o6ObBG9L+VMM=
-github.com/golangci/golangci-lint v1.18.0/go.mod h1:kaqo8l0OZKYPtjNmG4z4HrWLgcYNIJ9B9q3LWri9uLg=
+github.com/golangci/golangci-lint v1.21.0 h1:HxAxpR8Z0M8omihvQdsD3PF0qPjlqYqp2vMJzstoKeI=
+github.com/golangci/golangci-lint v1.21.0/go.mod h1:phxpHK52q7SE+5KpPnti4oZTdFCEsn/tKN+nFvCKXfk=
 github.com/golangci/gosec v0.0.0-20190211064107-66fb7fc33547 h1:fUdgm/BdKvwOHxg5AhNbkNRp2mSy8sxTXyBVs/laQHo=
 github.com/golangci/gosec v0.0.0-20190211064107-66fb7fc33547/go.mod h1:0qUabqiIQgfmlAmulqxyiGkkyF6/tOGSnY2cnPVwrzU=
 github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc h1:gLLhTLMk2/SutryVJ6D4VZCU3CUqr8YloG7FPIBWFpI=
 github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU=
 github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217 h1:En/tZdwhAn0JNwLuXzP3k2RVtMqMmOEK7Yu/g3tmtJE=
 github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg=
+github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0 h1:MfyDlzVjl1hoaPzPD4Gpb/QgoRfSBR0jdhwGyAWwMSA=
+github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg=
 github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca h1:kNY3/svz5T29MYHubXix4aDDuE3RWHkPvopM/EDv/MA=
 github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o=
 github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770 h1:EL/O5HGrF7Jaq0yNhBLucz9hTuRzj2LdwGBOaENgxIk=
@@ -209,21 +261,27 @@ github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/certificate-transparency-go v1.0.22-0.20190910093103-496c2e82955b h1:cagua7VIVtUFRYOfOeRvLPEQ6KJut//9bo6WFsOl8cc=
 github.com/google/certificate-transparency-go v1.0.22-0.20190910093103-496c2e82955b/go.mod h1:i+Q7XY+ArBveOUT36jiHGfuSK1fHICIg6sUkRxPAbCs=
+github.com/google/certificate-transparency-go v1.1.0 h1:10MlrYzh5wfkToxWI4yJzffsxLfxcEDlOATMx/V9Kzw=
+github.com/google/certificate-transparency-go v1.1.0/go.mod h1:i+Q7XY+ArBveOUT36jiHGfuSK1fHICIg6sUkRxPAbCs=
 github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/licenseclassifier v0.0.0-20190501212618-47b603fe1b8c/go.mod h1:qsqn2hxC+vURpyBRygGUuinTO42MFRLcsmQ/P8v94+M=
+github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/monologue v0.0.0-20190606152607-4b11a32b5934/go.mod h1:6NTfaQoUpg5QmPsCUWLR3ig33FHrKXhTtWzF0DVdmuk=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/tink v1.2.1-0.20190523150020-6495d823d968 h1:3P4TOKsvMmO5ZyE1CPGDiechVe/uxSSjydlSctvcpRg=
 github.com/google/tink v1.2.1-0.20190523150020-6495d823d968/go.mod h1:eu7D8x3z2rMO7fyvHVhMx8yoFH+vH8EZR1uO3hjEIhQ=
 github.com/google/trillian v1.2.2-0.20190612132142-05461f4df60a/go.mod h1:YPmUVn5NGwgnDUgqlVyFGMTgaWlnSvH7W5p+NdOG8UA=
-github.com/google/trillian v1.3.2-0.20190910105707-fa759d0a2aee h1:rXa4uOLiHeFeL1EAgArf9eooF53zQ4vp5U5lOL0jd4o=
-github.com/google/trillian v1.3.2-0.20190910105707-fa759d0a2aee/go.mod h1:zsKY3ZxWBwNhFsCLPRG/Aw9Zl8shNDA175tRx30lzYM=
+github.com/google/trillian v1.3.3 h1:xwIDDPBdK5I8hvCTZAna9dNCdZc1OUvkPnSSdiwjT8Y=
+github.com/google/trillian v1.3.3/go.mod h1:FFIfFeCKosf1D9mKBxMySFmBI96xHCatmA7NZOXo+qI=
 github.com/google/trillian-examples v0.0.0-20190603134952-4e75ba15216c/go.mod h1:WgL3XZ3pA8/9cm7yxqWrZE6iZkESB2ItGxy5Fo6k2lk=
 github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -242,14 +300,14 @@ github.com/gostaticanalysis/analysisutil v0.0.3 h1:iwp+5/UAyzQSFgQ4uR2sni99sJ8Eo
 github.com/gostaticanalysis/analysisutil v0.0.3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 h1:Iju5GlWwrvL6UBg4zJJt3btmonfrMlCDdsejg4CZE7c=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
+github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 h1:THDBEeQ9xZ8JEaCLyLQqXMMdRqNr0QAUJTIkQAUtFjg=
+github.com/grpc-ecosystem/go-grpc-middleware v1.1.0/go.mod h1:f5nM7jw/oeRSadq3xCzHAvxcr8HZnzsqU6ILg/0NiiE=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.9.0 h1:bM6ZAFZmc/wPFaRDi0d5L7hGEZEx/2u+Tmr2evNHDiI=
 github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
 github.com/grpc-ecosystem/grpc-gateway v1.9.4 h1:5xLhQjsk4zqPf9EHCrja2qFZMx+yBqkO3XgJ14bNnU0=
 github.com/grpc-ecosystem/grpc-gateway v1.9.4/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
-github.com/grpc-ecosystem/grpc-gateway v1.11.1 h1:/dBYI+n4xIL+Y9SKXQrjlKTmJJDwCSlNLRwZ5nBhIek=
-github.com/grpc-ecosystem/grpc-gateway v1.11.1/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
 github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
@@ -265,13 +323,12 @@ github.com/huandu/xstrings v1.2.0 h1:yPeWdRnmynF7p+lLYz0H2tthW9lqhMJrQV/U7yy4wX0
 github.com/huandu/xstrings v1.2.0/go.mod h1:DvyZB1rfVYsBIigL8HwpZgxHwXozlTgGqn63UyNX5k4=
 github.com/imdario/mergo v0.3.7 h1:Y+UAYTZ7gDEuOfhxKWy+dvb5dRQ6rJjFSdX2HZY1/gI=
 github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/imdario/mergo v0.3.8 h1:CGgOkSJeqMRmt0D9XLWExdT4m4F1vd3FV3VPt+0VxkQ=
+github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
-github.com/jhump/protoreflect v1.4.1/go.mod h1:gZ3i/BeD62fjlaIL0VW4UDMT70CTX+3m4pOnAlJ0BX8=
-github.com/jhump/protoreflect v1.4.4 h1:kySdALZUh7xRtW6UoZjjHtlR8k7rLzx5EXJFRvsO5UY=
-github.com/jhump/protoreflect v1.4.4/go.mod h1:gZ3i/BeD62fjlaIL0VW4UDMT70CTX+3m4pOnAlJ0BX8=
 github.com/jhump/protoreflect v1.5.0 h1:NgpVT+dX71c8hZnxHof2M7QDK7QtohIJ7DYycjnkyfc=
 github.com/jhump/protoreflect v1.5.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
@@ -281,7 +338,12 @@ github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwK
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo=
 github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok=
+github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024 h1:rBMNdlhTLzJjJSDIjNEXX1Pz3Hmwmz91v+zycvx9PJc=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o=
+github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/juju/ratelimit v1.0.1/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kevinburke/ssh_config v0.0.0-20180830205328-81db2a75821e/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
@@ -302,14 +364,16 @@ github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/letsencrypt/pkcs11key v2.0.1-0.20170608213348-396559074696+incompatible/go.mod h1:iGYXKqDXt0cpBthCHdr9ZdsQwyGlYFh/+8xa4WzIQ34=
-github.com/letsencrypt/pkcs11key/v3 v3.0.0/go.mod h1:sfoiYd3g5fIel4ZANw27sbgXzN43FY5hNP6uxEUywkM=
 github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4=
 github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0=
+github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
 github.com/lyft/protoc-gen-validate v0.0.14 h1:xbdDVIHd0Xq5Bfzu+8JR9s7mFmJPMvNLmfGhgcHJdFU=
 github.com/lyft/protoc-gen-validate v0.0.14/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
@@ -320,31 +384,44 @@ github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDe
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb h1:RHba4YImhrUVQDHUCe2BNSOz4tVy2yGyXhvYDvxGgeE=
+github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
 github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
+github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
-github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg=
-github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
+github.com/mattn/go-isatty v0.0.10 h1:qxFzApOv4WsAL965uUPIsXzAKCZxN2p9UqdhFS4ZW10=
+github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
 github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y=
 github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
+github.com/mattn/go-runewidth v0.0.6 h1:V2iyH+aX9C5fsYCpK60U8BYIvmhqxuOL3JZcqc1NB7k=
+github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/mattn/go-sqlite3 v1.10.0 h1:jbhqpg7tQe4SupckyijYiy0mJJ/pRyHvXf7JdWK860o=
 github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/miekg/pkcs11 v1.0.2/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
+github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ=
+github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
 github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk=
+github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk=
 github.com/mitchellh/go-wordwrap v1.0.0 h1:6GlHJ/LTGMrIJbwgdqdl2eEH8o+Exx/0m8ir9Gns0u4=
 github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo=
 github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mitchellh/reflectwalk v1.0.0 h1:9D+8oIskB4VJBN5SFlmc27fSlIBZaov1Wpk/IfikLNY=
+github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
+github.com/mitchellh/reflectwalk v1.0.1 h1:FVzMWA5RllMAKIdUSC8mdWo3XtwoecrH79BY70sEEpE=
+github.com/mitchellh/reflectwalk v1.0.1/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -353,11 +430,12 @@ github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
 github.com/mozilla/tls-observatory v0.0.0-20180409132520-8791a200eb40/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk=
+github.com/mozilla/tls-observatory v0.0.0-20190404164649-a3c1b6cfecfd/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mwitkow/go-proto-validators v0.0.0-20190212092829-1f388280e944 h1:RY9I2vt95pyT12MKfJo1Ck3O+NPQqlukBahURwDtJBc=
 github.com/mwitkow/go-proto-validators v0.0.0-20190212092829-1f388280e944/go.mod h1:m2XC9Qq0AlmmVksL6FktJCdTYyLk7V3fKyp0sl1yWQo=
-github.com/mwitkow/go-proto-validators v0.1.0 h1:2Org0/cGKUUUDzoLSRSsGJDqyLWrb5lG57o5+QdRr8M=
-github.com/mwitkow/go-proto-validators v0.1.0/go.mod h1:bA3eoTMLQkf/A7h7JOC3ddMBLXwS19KK7DEeSPL1O+4=
+github.com/mwitkow/go-proto-validators v0.2.0 h1:F6LFfmgVnfULfaRsQWBbe7F7ocuHCr9+7m+GAeDzNbQ=
+github.com/mwitkow/go-proto-validators v0.2.0/go.mod h1:ZfA1hW+UH/2ZHOWvQ3HnQaU0DtnpXu850MZiy+YUgcc=
 github.com/nbutton23/zxcvbn-go v0.0.0-20160627004424-a22cb81b2ecd/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU=
 github.com/nbutton23/zxcvbn-go v0.0.0-20171102151520-eafdab6b0663/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU=
 github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d h1:AREM5mwr4u1ORQBMvzfzBgpsctsbQikCVpvC+tX285E=
@@ -365,14 +443,23 @@ github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d/go.mod h1:o96d
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/olekukonko/tablewriter v0.0.1 h1:b3iUnf1v+ppJiOfNX4yxxqfWKMQPZR5yoh8urCTFX88=
 github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
+github.com/olekukonko/tablewriter v0.0.2 h1:sq53g+DWf0J6/ceFUHpQ0nAEb6WgM++fq16MZ91cS6o=
+github.com/olekukonko/tablewriter v0.0.2/go.mod h1:rSAaSIOAGT9odnlyGlUfAJaoc5w2fSBUmeGDbRWPxyQ=
 github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w=
 github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.10.3 h1:OoxbjfXVZyod1fmWYhI7SEyaD8B00ynP3T+D5GiyHOY=
+github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo=
 github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.7.1 h1:K0jcRCwNQM3vFGh1ppMtDh/+7ApJrjldlX8fA0jDTLQ=
+github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
+github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/otiai10/copy v1.0.1/go.mod h1:8bMCJrAqOtN/d9oyh5HR7HhLQMvcGMpGdwRDYsfOCHc=
 github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE=
 github.com/otiai10/curr v0.0.0-20190513014714-f5a3d24e5776/go.mod h1:3HNVkVOU7vZeFXocWuvtcS0XSFLcf2XUSDHkq9t1jU4=
@@ -384,6 +471,8 @@ github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml v1.4.0 h1:u3Z1r+oOXJIkxqw34zVhyPgjBsm6X2wn21NWs/HfSeg=
 github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo=
+github.com/pelletier/go-toml v1.6.0 h1:aetoXYr0Tv7xRU/V4B4IZJ2QcbtMUFoNb3ORp7TzIK4=
+github.com/pelletier/go-toml v1.6.0/go.mod h1:5N711Q9dKgbdkxHL+MEfF31hpT7l0S0s/t2kKREewys=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -396,8 +485,8 @@ github.com/prometheus/client_golang v0.9.4 h1:Y8E/JaaPbmFSW2V81Ab/d8yZFYQQGbni1b
 github.com/prometheus/client_golang v0.9.4/go.mod h1:oCXIBxdI62A4cR6aTRJCgetEjecSIYzOEaeAn4iYEpM=
 github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
-github.com/prometheus/client_golang v1.1.0 h1:BQ53HtBmfOitExawJ6LokA4x8ov/z0SYYb0+HxJfRI8=
-github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g=
+github.com/prometheus/client_golang v1.2.1 h1:JnMpQc6ppsNgw9QPAGF6Dod479itz7lvlsMzzNayLOI=
+github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@@ -410,6 +499,8 @@ github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.6.0 h1:kRhiuYSXR3+uv2IbVbZhUxK5zVD/2pp3Gd2PpvPkpEo=
 github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc=
+github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY=
+github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 h1:sofwID9zm4tzrgykg80hfFph1mryUeLRsUfoocVVmRY=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
@@ -417,8 +508,9 @@ github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNG
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.3 h1:CTwfnzjQ+8dS6MhHHu4YswVAD99sL2wjPqP+VkURmKE=
 github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
-github.com/prometheus/procfs v0.0.4 h1:w8DjqFMJDjuVwdZBQoOozr4MVWOnwF7RcL/7uxBjY78=
-github.com/prometheus/procfs v0.0.4/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
+github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
+github.com/prometheus/procfs v0.0.7 h1:RS5GAlMbnkWkhs4+bPocMTmGjYkuCY5djjqEDdXOhcQ=
+github.com/prometheus/procfs v0.0.7/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/pseudomuto/protoc-gen-doc v1.3.0 h1:wpwmaSCWY2lGwkzAxAaqYcGyoklZjZmeXrJ/X7IskJM=
 github.com/pseudomuto/protoc-gen-doc v1.3.0/go.mod h1:fwtQAY9erXp3mC92O8OTECnDlJT2r0Ff4KSEKbGEmy0=
@@ -429,13 +521,21 @@ github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6So
 github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.2.1/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rogpeppe/go-internal v1.3.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
 github.com/rs/cors v1.6.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/russross/blackfriday v2.0.0+incompatible h1:cBXrhZNUf9C+La9/YpS+UHpUT8YD6Td9ZMSU9APFcsk=
 github.com/russross/blackfriday v2.0.0+incompatible/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
+github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
+github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/go-glob v0.0.0-20170128012129-256dc444b735/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
+github.com/securego/gosec v0.0.0-20191002120514-e680875ea14d h1:BzRvVq1EHuIjxpijCEKpAxzKUUMurOQ4sknehIATRh8=
+github.com/securego/gosec v0.0.0-20191002120514-e680875ea14d/go.mod h1:w5+eXa0mYznDkHaMCXA4XYffjlH+cy1oyKbfzJXa2Do=
+github.com/securego/gosec v0.0.0-20191119104125-df484bfa9e9f h1:1egnuKwFhkqg8hXU5huGkxz9iPFu4dLINWRSIPjAM+M=
+github.com/securego/gosec v0.0.0-20191119104125-df484bfa9e9f/go.mod h1:H5UrtKXL5BGF4FgRa7p2fyqU/lddaTSCLjexYIfS0Bk=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/shirou/gopsutil v0.0.0-20180427012116-c95755e4bcd7/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
+github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc=
 github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
 github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
 github.com/shurcooL/go v0.0.0-20190330031554-6713ea532688 h1:p7kt1rmLjMUkAmuyY/GUZeCoOud+nq297UKCCgEqYmA=
@@ -474,38 +574,59 @@ github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0
 github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.0.2/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM=
 github.com/spf13/viper v1.3.2 h1:VUFqw5KcqRf7i70GOzW7N+Q7+gxVBkSSqiXB12+JQ4M=
 github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
 github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU=
 github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
+github.com/spf13/viper v1.5.0 h1:GpsTwfsQ27oS/Aha/6d1oD7tpKIqWnOA6tgOX9HHkt4=
+github.com/spf13/viper v1.5.0/go.mod h1:AkYRkVJF8TkSG/xet6PzXX+l39KhhXa2pdqVSxnTcn4=
 github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
+github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
+github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/timakin/bodyclose v0.0.0-20190721030226-87058b9bfcec h1:AmoEvWAO3nDx1MEcMzPh+GzOOIA5Znpv6++c7bePPY0=
 github.com/timakin/bodyclose v0.0.0-20190721030226-87058b9bfcec/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
+github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e h1:RumXZ56IrCj4CL+g1b9OL/oH0QnsF976bC8xQFYUD5Q=
+github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce/go.mod h1:o8v6yHRoik09Xen7gje4m9ERNah1d1PPsVq1VEx9vE4=
-github.com/uber/prototool v1.8.1 h1:wlba8JLLuyTn0a7W6AqLrecGypFkrGOb+TwciSrW/rg=
-github.com/uber/prototool v1.8.1/go.mod h1:E+xJFE/vf87XeqHbKM4uGQMrJ7NyKQb0+G7NtqHBgYs=
+github.com/uber/prototool v1.8.2-0.20190910022025-7df3b957ffe3 h1:MAmouJ3XiYdtio1FxhxW0FX1lUsVYQsB+GflGB2ltrM=
+github.com/uber/prototool v1.8.2-0.20190910022025-7df3b957ffe3/go.mod h1:fQyN8ZUYd9QoSiV0T/Da6NUSZY4ARW+EfJaL/4V1DD8=
+github.com/uber/prototool v1.9.0 h1:s5O7ZQUqmV8jfaB9J69HD0DlU4S0CL4ecZlemkhF3TQ=
+github.com/uber/prototool v1.9.0/go.mod h1:srCJMbeTuTT/1xkYIuUc1iCKsjjn8RBw9xxQl0R4aZg=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/ultraware/funlen v0.0.1 h1:UeC9tpM4wNWzUJfan8z9sFE4QCzjjzlCZmuJN+aOkH0=
 github.com/ultraware/funlen v0.0.1/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA=
 github.com/ultraware/funlen v0.0.2 h1:Av96YVBwwNSe4MLR7iI/BIa3VyI7/djnto/pK3Uxbdo=
 github.com/ultraware/funlen v0.0.2/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA=
+github.com/ultraware/whitespace v0.0.4 h1:If7Va4cM03mpgrNH9k49/VOicWpGoG70XPBFFODYDsg=
+github.com/ultraware/whitespace v0.0.4/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA=
 github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw=
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
-github.com/urfave/cli v1.22.0 h1:8nz/RUUotroXnOpYzT/Fy3sBp+2XEbXaY641/s3nbFI=
-github.com/urfave/cli v1.22.0/go.mod h1:b3D7uWrF2GilkNgYpgcg6J+JMUw7ehmNkE8sZdliGLc=
+github.com/urfave/cli v1.22.1 h1:+mkCCcOFKPnCmVYVcURKps1Xe+3zP90gSYGNfRkjoIY=
+github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
+github.com/uudashr/gocognit v0.0.0-20190926065955-1655d0de0517 h1:ChMKTho2hWKpks/nD/FL2KqM1wuVt62oJeiE8+eFpGs=
+github.com/uudashr/gocognit v0.0.0-20190926065955-1655d0de0517/go.mod h1:j44Ayx2KW4+oB6SWMv8KsmHzZrOInQav7D3cQMJ5JUM=
+github.com/uudashr/gocognit v1.0.0 h1:NST9SQhYHpFiyKdXn8UAiogdB1xkt5RnA1/rR/oBcGw=
+github.com/uudashr/gocognit v1.0.0/go.mod h1:j44Ayx2KW4+oB6SWMv8KsmHzZrOInQav7D3cQMJ5JUM=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s=
 github.com/valyala/quicktemplate v1.1.1/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4=
+github.com/valyala/quicktemplate v1.2.0/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4=
 github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
 github.com/xanzy/ssh-agent v0.2.0/go.mod h1:0NyE30eGUDliuLEHJgYte/zncp2zdTStcOnWhgSqHD8=
 github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4=
@@ -518,39 +639,65 @@ go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk=
 go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/etcd v3.3.13+incompatible h1:jCejD5EMnlGxFvcGRyEV4VGlENZc7oPQX6o0t7n3xbw=
 go.etcd.io/etcd v3.3.13+incompatible/go.mod h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSFZfzsF9A6aI=
-go.etcd.io/etcd v3.3.15+incompatible h1:0VpOVCF6EFnJptt8Jh0EWEHO4j2fepyV1fpu9xz/UoQ=
-go.etcd.io/etcd v3.3.15+incompatible/go.mod h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSFZfzsF9A6aI=
+go.etcd.io/etcd v3.3.17+incompatible h1:g8iRku1SID8QAW8cDlV0L/PkZlw63LSiYEHYHoE6j/s=
+go.etcd.io/etcd v3.3.17+incompatible/go.mod h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSFZfzsF9A6aI=
 go.opencensus.io v0.21.0 h1:mU6zScU4U1YAFPHEHYk+3JC4SY7JxgkqS10ZOSyksNg=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0 h1:C9hSCOW830chIVkdja34wa6Ky+IzWllkUinR+BtRZd4=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
+go.opencensus.io v0.22.2 h1:75k/FF0Q2YM8QYo07VPddOLBslDt1MZOdEslOHvmzAs=
+go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
+go.uber.org/atomic v1.5.1 h1:rsqfU5vBkVknbhUGbAUwQKR2H4ItV8tjJ+6kJX4cxHM=
+go.uber.org/atomic v1.5.1/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
 go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/multierr v1.2.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
+go.uber.org/multierr v1.4.0 h1:f3WCSC2KzAcBXGATIxAB1E2XuCpNU255wNKZ505qi3E=
+go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
+go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4=
+go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
 go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+go.uber.org/zap v1.13.0 h1:nR6NoDBgAf67s68NhaXbsojM+2gxp3S1hWkHDl27pVU=
+go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5 h1:58fnuSXlxZmFdJyvtTFVmVhcMLU6v5fEb/ok4wyqtNU=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc=
 golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190909091759-094676da4a83 h1:mgAKeshyNqWKdENOnQsg+8dRTwZFIwFaO3HNl52sweA=
-golang.org/x/crypto v0.0.0-20190909091759-094676da4a83/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY=
+golang.org/x/crypto v0.0.0-20191119213627-4f8c1d86b1ba h1:9bFeDpN3gTqNanMVqNcoR/pJQuP5uroC3t1D7eXozTE=
+golang.org/x/crypto v0.0.0-20191119213627-4f8c1d86b1ba/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
+golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
+golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136 h1:A1gGSx58LAGVHUUsOf7IiR0u8Xb6W51gRwfDBhkdcaw=
+golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20190409202823-959b441ac422 h1:QzoH/1pFpZguR8NrRHLcO6jKqfv2zpuSqZLgdm7ZmjI=
 golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
 golang.org/x/net v0.0.0-20170915142106-8351a756f30f/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180530234432-1e491301e022/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -572,8 +719,10 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190628185345-da137c7871d7 h1:rTIdg5QFRR7XCaK4LCjBiPbx8j4DQRpdYMnGn/bJUEU=
 golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190909003024-a7b16738d86b h1:XfVGCX+0T4WOStkaOsJRllbsiImhB2jgVBGc9L0lPGc=
-golang.org/x/net v0.0.0-20190909003024-a7b16738d86b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191009170851-d66e71096ffb/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191119073136-fc4aabc6c914 h1:MlY3mEfbnWGmUi4rtHOtNnnnN4UJRGSyLPx+DXA5Sq4=
+golang.org/x/net v0.0.0-20191119073136-fc4aabc6c914/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -585,6 +734,8 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20171026204733-164713f0dfce/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180903190138-2b024373dcd9/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -606,9 +757,11 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3 h1:4y9KwBHBgBNwDbtu44R5o1fdOCQUEXhbk/P4A9WmJq0=
 golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190910064555-bbd175535a8b h1:3S2h5FadpNr0zUUCVZjlKIEYF+KaX/OBplTGo89CYHI=
-golang.org/x/sys v0.0.0-20190910064555-bbd175535a8b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e h1:N7DeIrjYszNmSW409R3frPPwglRwMkXSBzwVbkOjLLA=
+golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.0.0-20170915090833-1cbadb444a80/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -618,8 +771,11 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20170915040203-e531a2a1c15f/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -639,12 +795,27 @@ golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBn
 golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190624190245-7f2218787638/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190719005602-e377ae9d6386/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI=
+golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190909030654-5b82db07426d h1:PhtdWYteEBebOX7KXm4qkIAVSUTHQ883/2hRB92r9lk=
 golang.org/x/tools v0.0.0-20190909030654-5b82db07426d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190910044552-dd2b5c81c578 h1:f0Gfd654rnnfXT1+BK1YHPTS1qQdKrPIaGQwWxNE44k=
 golang.org/x/tools v0.0.0-20190910044552-dd2b5c81c578/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190911151314-feee8acb394c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190930201159-7c411dea38b0/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191010075000-0337d82405ff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191101200257-8dbcdeb83d3f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191105231337-689d0f08e67a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191120221951-8fd459516a27 h1:Hnjbvuhm7GOIJHtxOlsKcq7CEDvPLMHawSA1AzKVlxA=
+golang.org/x/tools v0.0.0-20191120221951-8fd459516a27/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/api v0.5.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
@@ -652,12 +823,19 @@ google.golang.org/api v0.6.0 h1:2tJEkRfnZL5g1GeBUlITh/rqT5HG3sFcoVCUUxmgJ2g=
 google.golang.org/api v0.6.0/go.mod h1:btoxGiFvQNVUZQ8W08zLtrVS08CNpINPEfxXxgJL1Q4=
 google.golang.org/api v0.7.0 h1:9sdfJOzWlkqPltHAuzT2Cp+yrBeY1KRVYgms8soxMwM=
 google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
+google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.14.0 h1:uMf5uLi4eQMRrMKhCplNik4U4H8Z6C1br3zOtAa/aDE=
+google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I=
 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM=
+google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/genproto v0.0.0-20170818010345-ee236bd376b0/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
@@ -668,8 +846,14 @@ google.golang.org/genproto v0.0.0-20190605220351-eb0b1bdb6ae6 h1:XRqWpmQ5ACYxWuY
 google.golang.org/genproto v0.0.0-20190605220351-eb0b1bdb6ae6/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s=
 google.golang.org/genproto v0.0.0-20190626174449-989357319d63/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s=
 google.golang.org/genproto v0.0.0-20190708153700-3bdd9d9f5532/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s=
-google.golang.org/genproto v0.0.0-20190905072037-92dd089d5514 h1:oFSK4421fpCKRrpzIpybyBVWyht05NegY9+L/3TLAZs=
-google.golang.org/genproto v0.0.0-20190905072037-92dd089d5514/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
+google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191028173616-919d9bdd9fe6/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191115221424-83cc0476cb11 h1:51D++eCgOHufw5VfDE9Uzqyyc+OyQIjb9hkYy9LN5Fk=
+google.golang.org/genproto v0.0.0-20191115221424-83cc0476cb11/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
 google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
@@ -681,11 +865,16 @@ google.golang.org/grpc v1.22.0 h1:J0UbZOIrCAl+fpTOf8YLs4dJo8L/owV4LYVtAXQoPkw=
 google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.23.0 h1:AzbTB6ux+okLTzP8Ru1Xs41C303zdcfEht7MQnYJt5A=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA=
+google.golang.org/grpc v1.25.1 h1:wdKvqQk7IttEw92GoRyKG2IDrUIpgpj6H6m81yfeMW0=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
 gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/cheggaaa/pb.v1 v1.0.28 h1:n1tBJnnK2r7g9OW2btFH91V92STTUevLXYFb8gy9EMk=
 gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
@@ -705,11 +894,16 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo=
+gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a h1:LJwr7TCTghdatWv40WobzlKXc9c4s8oGa7QKJUtHhWA=
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed h1:WX1yoOaKQfddO/mLzdV4wptyWgoH/6hwLs7QHTixo0I=
 mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc=
 mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b h1:DxJ5nJdkhDlLok9K6qO+5290kphDJbHOQO1DFFFTeBo=
@@ -719,6 +913,8 @@ mvdan.cc/unparam v0.0.0-20190310220240-1b9ccfa71afe h1:Ekmnp+NcP2joadI9pbK4Bva87
 mvdan.cc/unparam v0.0.0-20190310220240-1b9ccfa71afe/go.mod h1:BnhuWBAqxH3+J5bDybdxgw5ZfS+DsVd4iylsKQePN8o=
 mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f h1:Cq7MalBHYACRd6EesksG1Q8EoIAKOsiZviGKbOLIej4=
 mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f/go.mod h1:4G1h5nDURzA3bwVMZIVpwbkw+04kSxk3rAtzlimaUJw=
+mvdan.cc/unparam v0.0.0-20191111180625-960b1ec0f2c2 h1:K7wru2CfJGumS5hkiguQ0Rb9ebKM2Jo8s5d4Jm9lFaM=
+mvdan.cc/unparam v0.0.0-20191111180625-960b1ec0f2c2/go.mod h1:rCqoQrfAmpTX/h2APczwM7UymU/uvaOluiVPIYCSY/k=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=

From 1b6164b714f8e889e298e001164f73ba5256e306 Mon Sep 17 00:00:00 2001
From: Gary Belvin <gdb@google.com>
Date: Tue, 3 Dec 2019 15:59:23 +0000
Subject: [PATCH 19/22] Fix the kubernetes configs (#1397)

* Update Trillian Docker URLs

* Fix KT binary paths

* Fetch keys from secrets

* include monitor_sign-key in kt-secrets
---
 deploy/kubernetes/db-deployment.yaml         |  2 +-
 deploy/kubernetes/log-server-deployment.yaml |  2 +-
 deploy/kubernetes/log-signer-deployment.yaml |  2 +-
 deploy/kubernetes/map-server-deployment.yaml |  2 +-
 deploy/kubernetes/monitor-deployment.yaml    | 16 ++++++++++++----
 deploy/kubernetes/sequencer-deployment.yaml  | 12 +++++++++++-
 deploy/kubernetes/server-deployment.yaml     | 14 +++++++++++---
 scripts/deploy.sh                            | 11 +++++------
 8 files changed, 43 insertions(+), 18 deletions(-)

diff --git a/deploy/kubernetes/db-deployment.yaml b/deploy/kubernetes/db-deployment.yaml
index 755f3ea63..3027b890a 100644
--- a/deploy/kubernetes/db-deployment.yaml
+++ b/deploy/kubernetes/db-deployment.yaml
@@ -27,7 +27,7 @@ spec:
           value: "yes"
         - name: MYSQL_USER
           value: test
-        image: gcr.io/key-transparency/db
+        image: gcr.io/trillian-opensource-ci/db_server:latest
         name: db
         ports:
         - containerPort: 3306
diff --git a/deploy/kubernetes/log-server-deployment.yaml b/deploy/kubernetes/log-server-deployment.yaml
index 3ba607e76..7c0031514 100644
--- a/deploy/kubernetes/log-server-deployment.yaml
+++ b/deploy/kubernetes/log-server-deployment.yaml
@@ -25,7 +25,7 @@ spec:
         "--http_endpoint=0.0.0.0:8091",
         "--alsologtostderr"
         ]
-        image: gcr.io/key-transparency/log-server:latest
+        image: gcr.io/trillian-opensource-ci/log_server:latest
         livenessProbe:
          httpGet:
            path: /metrics
diff --git a/deploy/kubernetes/log-signer-deployment.yaml b/deploy/kubernetes/log-signer-deployment.yaml
index 6ee760dfc..74f0ea68f 100644
--- a/deploy/kubernetes/log-signer-deployment.yaml
+++ b/deploy/kubernetes/log-signer-deployment.yaml
@@ -29,7 +29,7 @@ spec:
         "--force_master=true",
         "--alsologtostderr"
         ]
-        image: gcr.io/key-transparency/log-signer:latest
+        image: gcr.io/trillian-opensource-ci/log_signer:latest
         livenessProbe:
          httpGet:
            path: /metrics
diff --git a/deploy/kubernetes/map-server-deployment.yaml b/deploy/kubernetes/map-server-deployment.yaml
index 0617cc3ac..505f25084 100644
--- a/deploy/kubernetes/map-server-deployment.yaml
+++ b/deploy/kubernetes/map-server-deployment.yaml
@@ -25,7 +25,7 @@ spec:
         "--http_endpoint=0.0.0.0:8091",
         "--alsologtostderr"
         ]
-        image: gcr.io/key-transparency/map-server:latest
+        image: gcr.io/trillian-opensource-ci/map_server:latest
         livenessProbe:
          httpGet:
            path: /metrics
diff --git a/deploy/kubernetes/monitor-deployment.yaml b/deploy/kubernetes/monitor-deployment.yaml
index f8322e2ba..49db8b948 100644
--- a/deploy/kubernetes/monitor-deployment.yaml
+++ b/deploy/kubernetes/monitor-deployment.yaml
@@ -17,16 +17,20 @@ spec:
       labels:
         io.kompose.service: monitor
     spec:
+      volumes:
+       - name: secrets
+         secret:
+           secretName: kt-secrets
       containers:
       - command:
-        - /go/bin/keytransparency-monitor
+        - /keytransparency-monitor
         - --addr=0.0.0.0:8099
         - --kt-url=server:8080
         - --insecure
         - --directoryid=default
-        - --tls-key=/kt/server.key
-        - --tls-cert=/kt/server.crt
-        - --sign-key=/kt/monitor_sign-key.pem
+        - --tls-key=/run/secrets/server.key
+        - --tls-cert=/run/secrets/server.crt
+        - --sign-key=/run/secrets/monitor_sign-key.pem
         - --password=towel
         - --alsologtostderr
         - --v=3
@@ -35,5 +39,9 @@ spec:
         ports:
         - containerPort: 8099
         resources: {}
+        volumeMounts:
+        - name: secrets
+          mountPath: "/run/secrets"
+          readOnly: true
       restartPolicy: Always
 status: {}
diff --git a/deploy/kubernetes/sequencer-deployment.yaml b/deploy/kubernetes/sequencer-deployment.yaml
index 904673fc5..414a85d87 100644
--- a/deploy/kubernetes/sequencer-deployment.yaml
+++ b/deploy/kubernetes/sequencer-deployment.yaml
@@ -17,14 +17,20 @@ spec:
       labels:
         io.kompose.service: sequencer
     spec:
+      volumes:
+       - name: secrets
+         secret:
+           secretName: kt-secrets
       containers:
       - command:
-        - /go/bin/keytransparency-sequencer
+        - /keytransparency-sequencer
         - --force_master
         - --db=test:zaphod@tcp(db:3306)/test
         - --addr=0.0.0.0:8080
         - --log-url=log-server:8090
         - --map-url=map-server:8090
+        - --tls-key=/run/secrets/server.key
+        - --tls-cert=/run/secrets/server.crt
         - --alsologtostderr
         - --v=5
         image: gcr.io/key-transparency/keytransparency-sequencer:latest
@@ -37,6 +43,10 @@ spec:
         - containerPort: 8080
         - containerPort: 8081
         resources: {}
+        volumeMounts:
+        - name: secrets
+          mountPath: "/run/secrets"
+          readOnly: true
       - name: prometheus-to-sd
         image: gcr.io/google-containers/prometheus-to-sd:v0.2.6
         command:
diff --git a/deploy/kubernetes/server-deployment.yaml b/deploy/kubernetes/server-deployment.yaml
index b4131de8c..88f7b36bb 100644
--- a/deploy/kubernetes/server-deployment.yaml
+++ b/deploy/kubernetes/server-deployment.yaml
@@ -18,15 +18,19 @@ spec:
       labels:
         io.kompose.service: server
     spec:
+      volumes:
+       - name: secrets
+         secret:
+           secretName: kt-secrets
       containers:
       - command:
-        - /go/bin/keytransparency-server
+        - /keytransparency-server
         - --addr=0.0.0.0:8080
         - --db=test:zaphod@tcp(db:3306)/test
         - --log-url=log-server:8090
         - --map-url=map-server:8090
-        - --tls-key=/kt/server.key
-        - --tls-cert=/kt/server.crt
+        - --tls-key=/run/secrets/server.key
+        - --tls-cert=/run/secrets/server.crt
         - --auth-type=insecure-fake
         - --alsologtostderr
         - --v=5
@@ -40,6 +44,10 @@ spec:
         - containerPort: 8080
         - containerPort: 8081
         resources: {}
+        volumeMounts:
+        - name: secrets
+          mountPath: "/run/secrets"
+          readOnly: true
       - name: prometheus-to-sd
         image: gcr.io/google-containers/prometheus-to-sd:v0.2.6
         command:
diff --git a/scripts/deploy.sh b/scripts/deploy.sh
index e4d2613f9..70b349e2d 100755
--- a/scripts/deploy.sh
+++ b/scripts/deploy.sh
@@ -36,13 +36,12 @@ gcloud --quiet auth configure-docker
 # Test current directory before deleting anything
 test $(basename $(pwd)) == "keytransparency" || exit 1
 
-echo "Generating keys..."
-rm -f ./genfiles/*
-./scripts/prepare_server.sh -f
 # kubectl exits with 1 if kt-secret does not exist
-kubectl get secret kt-secrets
-if [ $? -ne 0 ]; then
-  kubectl create secret generic kt-secrets --from-file=genfiles/server.crt --from-file=genfiles/server.key
+if ! kubectl get secret kt-secrets; then
+  echo "Generating keys..."
+  rm -f ./genfiles/*
+  ./scripts/prepare_server.sh -f
+  kubectl create secret generic kt-secrets --from-file=genfiles/server.crt --from-file=genfiles/server.key --from-file=genfiles/monitor_sign-key.pem
 fi
 
 echo "Building docker images..."

From 04a6a4ec3a24eff47d6b60df0c596f44e717a44a Mon Sep 17 00:00:00 2001
From: Martin <Martin2112@users.noreply.github.com>
Date: Fri, 6 Dec 2019 15:34:26 +0000
Subject: [PATCH 20/22] Fixups for deleted test constructor in trillian repo
 (#1398)

---
 cmd/keytransparency-monitor/main.go | 5 +++--
 core/integration/monitor_tests.go   | 5 +++--
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/cmd/keytransparency-monitor/main.go b/cmd/keytransparency-monitor/main.go
index b34a518d5..6cec18375 100644
--- a/cmd/keytransparency-monitor/main.go
+++ b/cmd/keytransparency-monitor/main.go
@@ -16,13 +16,14 @@ package main
 
 import (
 	"context"
+	"crypto"
 	"crypto/tls"
 	"flag"
 	"net"
 	"net/http"
 
 	"github.com/golang/glog"
-	"github.com/google/trillian/crypto"
+	tcrypto "github.com/google/trillian/crypto"
 	"github.com/google/trillian/crypto/keys/pem"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"google.golang.org/grpc"
@@ -75,7 +76,7 @@ func main() {
 	if err != nil {
 		glog.Exitf("Could not create signer from %v: %v", *signingKey, err)
 	}
-	signer := crypto.NewSHA256Signer(key)
+	signer := tcrypto.NewSigner(0, key, crypto.SHA256)
 	store := fake.NewMonitorStorage()
 
 	// Create monitoring background process.
diff --git a/core/integration/monitor_tests.go b/core/integration/monitor_tests.go
index ea24b43cb..350cfa2e6 100644
--- a/core/integration/monitor_tests.go
+++ b/core/integration/monitor_tests.go
@@ -16,6 +16,7 @@ package integration
 
 import (
 	"context"
+	"crypto"
 	"sync"
 	"testing"
 	"time"
@@ -28,7 +29,7 @@ import (
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 
-	"github.com/google/trillian/crypto"
+	tcrypto "github.com/google/trillian/crypto"
 	"github.com/google/trillian/crypto/keys/pem"
 
 	tpb "github.com/google/keytransparency/core/testdata/transcript_go_proto"
@@ -49,7 +50,7 @@ func TestMonitor(ctx context.Context, env *Env, t *testing.T) []*tpb.Action {
 	if err != nil {
 		t.Fatalf("Couldn't create signer: %v", err)
 	}
-	signer := crypto.NewSHA256Signer(privKey)
+	signer := tcrypto.NewSigner(0, privKey, crypto.SHA256)
 	store := fake.NewMonitorStorage()
 	mon, err := monitor.NewFromDirectory(env.Cli, env.Directory, signer, store)
 	if err != nil {

From edc57c2eefe5b0ad70ea8259ee31d3adefed472d Mon Sep 17 00:00:00 2001
From: Gary Belvin <gdb@google.com>
Date: Mon, 9 Dec 2019 10:14:45 +0000
Subject: [PATCH 21/22] Use ProtoEqual for gomock reflection equality (#1401)

---
 core/keyserver/keyserver_test.go | 16 +++++++++-------
 core/keyserver/revisions_test.go | 10 ++++++----
 2 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/core/keyserver/keyserver_test.go b/core/keyserver/keyserver_test.go
index 9de0a9b3d..e22830c7d 100644
--- a/core/keyserver/keyserver_test.go
+++ b/core/keyserver/keyserver_test.go
@@ -27,6 +27,7 @@ import (
 	"github.com/google/keytransparency/core/directory"
 	"github.com/google/keytransparency/core/fake"
 	"github.com/google/trillian/testonly"
+	"github.com/google/trillian/testonly/matchers"
 	"github.com/google/trillian/types"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
@@ -123,11 +124,12 @@ func TestLatestRevision(t *testing.T) {
 				}, err)
 			if tc.wantErr == codes.OK {
 				e.s.Map.EXPECT().GetLeafByRevision(gomock.Any(),
-					&tpb.GetMapLeafByRevisionRequest{
-						MapId:    mapID,
-						Index:    make([]byte, 32),
-						Revision: tc.treeSize - 1,
-					}).
+					matchers.ProtoEqual(
+						&tpb.GetMapLeafByRevisionRequest{
+							MapId:    mapID,
+							Index:    make([]byte, 32),
+							Revision: tc.treeSize - 1,
+						})).
 					Return(&tpb.GetMapLeafResponse{
 						MapLeafInclusion: &tpb.MapLeafInclusion{
 							Leaf: &tpb.MapLeaf{
@@ -156,11 +158,11 @@ func TestLatestRevision(t *testing.T) {
 				}, err).Times(2)
 			for i := int64(0); i < tc.treeSize; i++ {
 				e.s.Map.EXPECT().GetLeafByRevision(gomock.Any(),
-					&tpb.GetMapLeafByRevisionRequest{
+					matchers.ProtoEqual(&tpb.GetMapLeafByRevisionRequest{
 						MapId:    mapID,
 						Index:    make([]byte, 32),
 						Revision: i,
-					}).
+					})).
 					Return(&tpb.GetMapLeafResponse{
 						MapLeafInclusion: &tpb.MapLeafInclusion{
 							Leaf: &tpb.MapLeaf{
diff --git a/core/keyserver/revisions_test.go b/core/keyserver/revisions_test.go
index 85b3a9227..1035ad56e 100644
--- a/core/keyserver/revisions_test.go
+++ b/core/keyserver/revisions_test.go
@@ -28,6 +28,7 @@ import (
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/keytransparency/impl/memory"
+	"github.com/google/trillian/testonly/matchers"
 
 	protopb "github.com/golang/protobuf/ptypes/timestamp"
 	pb "github.com/google/keytransparency/core/api/v1/keytransparency_go_proto"
@@ -171,10 +172,11 @@ func TestListMutations(t *testing.T) {
 
 			if !tc.wantErr {
 				e.s.Map.EXPECT().GetLeavesByRevision(gomock.Any(),
-					&tpb.GetMapLeavesByRevisionRequest{
-						MapId: mapID,
-						Index: genIndexes(tc.start, tc.end),
-					}).Return(&tpb.GetMapLeavesResponse{
+					matchers.ProtoEqual(
+						&tpb.GetMapLeavesByRevisionRequest{
+							MapId: mapID,
+							Index: genIndexes(tc.start, tc.end),
+						})).Return(&tpb.GetMapLeavesResponse{
 					MapLeafInclusion: genInclusions(tc.start, tc.end),
 				}, nil)
 			}

From 647159f4ee9ca859404a828de34ee614a64d1308 Mon Sep 17 00:00:00 2001
From: Gary Belvin <gdb@google.com>
Date: Tue, 10 Dec 2019 08:41:36 +0000
Subject: [PATCH 22/22] Enable travis build validation (#1400)

---
 .travis.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 83efafcbc..9a987ab7c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,6 +1,7 @@
+version: ~> 1.0
 language: go
 go:
-- 1.x
+- "1.13.x"
 dist: bionic
 
 go_import_path: github.com/google/keytransparency