Add support for Static CT API entry bundle paths (#178)

api/layout/paths.go

@@ -37,25 +37,24 @@ func EntriesPathForLogIndex(seq, logSize uint64) string {
 	return EntriesPath(seq/256, logSize)
 }
 
-// EntriesPath returns the local path for the nth entry bundle. p denotes the partial
-// tile size, or 0 if the tile is complete.
-func EntriesPath(n, logSize uint64) string {
+// NWithSuffix returns a tiles-spec "N" path, with a partial suffix if applicable.
+func NWithSuffix(l, n, logSize uint64) string {
 	suffix := ""
-	if p := partialTileSize(0, n, logSize); p > 0 {
+	if p := partialTileSize(l, n, logSize); p > 0 {
 		suffix = fmt.Sprintf(".p/%d", p)
 	}
-	return fmt.Sprintf("tile/entries%s%s", fmtN(n), suffix)
+	return fmt.Sprintf("%s%s", fmtN(n), suffix)
+}
+
+// EntriesPath returns the local path for the nth entry bundle. p denotes the partial
+// tile size, or 0 if the tile is complete.
+func EntriesPath(n, logSize uint64) string {
+	return fmt.Sprintf("tile/entries/%s", NWithSuffix(0, n, logSize))
 }
 
 // TilePath builds the path to the subtree tile with the given level and index in tile space.
 func TilePath(tileLevel, tileIndex, logSize uint64) string {
-	suffix := ""
-	p := partialTileSize(tileLevel, tileIndex, logSize)
-	if p > 0 {
-		suffix = fmt.Sprintf(".p/%d", p)
-	}
-
-	return fmt.Sprintf("tile/%d%s%s", tileLevel, fmtN(tileIndex), suffix)
+	return fmt.Sprintf("tile/%d/%s", tileLevel, NWithSuffix(tileLevel, tileIndex, logSize))
 }
 
 // fmtN returns the "N" part of a Tiles-spec path.
@@ -67,10 +66,10 @@ func TilePath(tileLevel, tileIndex, logSize uint64) string {
 //
 // See https://github.com/C2SP/C2SP/blob/main/tlog-tiles.md#:~:text=index%201234067%20will%20be%20encoded%20as%20x001/x234/067
 func fmtN(N uint64) string {
-	n := fmt.Sprintf("/%03d", N%1000)
+	n := fmt.Sprintf("%03d", N%1000)
 	N /= 1000
 	for N > 0 {
-		n = fmt.Sprintf("/x%03d%s", N%1000, n)
+		n = fmt.Sprintf("x%03d/%s", N%1000, n)
 		N /= 1000
 	}
 	return n

api/layout/paths_test.go

@@ -169,6 +169,70 @@ func TestTilePath(t *testing.T) {
 	}
 }
 
+func TestNWithSuffix(t *testing.T) {
+	for _, test := range []struct {
+		level    uint64
+		index    uint64
+		logSize  uint64
+		wantPath string
+	}{
+		{
+			level:    0,
+			index:    0,
+			logSize:  256,
+			wantPath: "000",
+		}, {
+			level:    0,
+			index:    0,
+			logSize:  0,
+			wantPath: "000",
+		}, {
+			level:    0,
+			index:    0,
+			logSize:  255,
+			wantPath: "000.p/255",
+		}, {
+			level:    1,
+			index:    0,
+			logSize:  math.MaxUint64,
+			wantPath: "000",
+		}, {
+			level:    1,
+			index:    0,
+			logSize:  256,
+			wantPath: "000.p/1",
+		}, {
+			level:    1,
+			index:    0,
+			logSize:  1024,
+			wantPath: "000.p/4",
+		}, {
+			level:    15,
+			index:    455667,
+			logSize:  math.MaxUint64,
+			wantPath: "x455/667",
+		}, {
+			level:    3,
+			index:    1234567,
+			logSize:  math.MaxUint64,
+			wantPath: "x001/x234/567",
+		}, {
+			level:    15,
+			index:    123456789,
+			logSize:  math.MaxUint64,
+			wantPath: "x123/x456/789",
+		},
+	} {
+		desc := fmt.Sprintf("level %x index %x", test.level, test.index)
+		t.Run(desc, func(t *testing.T) {
+			gotPath := NWithSuffix(test.level, test.index, test.logSize)
+			if gotPath != test.wantPath {
+				t.Errorf("Got path %q want %q", gotPath, test.wantPath)
+			}
+		})
+	}
+}
+
 func TestParseTileLevelIndexWidth(t *testing.T) {
 	for _, test := range []struct {
 		pathLevel string

ct_only.go

@@ -16,7 +16,9 @@ package tessera
 
 import (
 	"context"
+	"fmt"
 
+	"github.com/transparency-dev/trillian-tessera/api/layout"
 	"github.com/transparency-dev/trillian-tessera/ctonly"
 )
 
@@ -56,3 +58,14 @@ func convertCTEntry(e *ctonly.Entry) *Entry {
 
 	return r
 }
+
+// WithCTLayout instructs the underlying storage to use a Static CT API compatible scheme for layout.
+func WithCTLayout() func(*StorageOptions) {
+	return func(opts *StorageOptions) {
+		opts.EntriesPath = ctEntriesPath
+	}
+}
+
+func ctEntriesPath(n, logSize uint64) string {
+	return fmt.Sprintf("tile/data/%s", layout.NWithSuffix(0, n, logSize))
+}

ct_only_test.go

@@ -0,0 +1,64 @@
+// Copyright 2024 Google LLC. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tessera
+
+import (
+	"fmt"
+	"math"
+	"testing"
+)
+
+func TestCTEntriesPath(t *testing.T) {
+	for _, test := range []struct {
+		N        uint64
+		logSize  uint64
+		wantPath string
+	}{
+		{
+			N:        0,
+			logSize:  289,
+			wantPath: "tile/data/000",
+		},
+		{
+			N:        0,
+			logSize:  8,
+			wantPath: "tile/data/000.p/8",
+		}, {
+			N:        255,
+			logSize:  256 * 256,
+			wantPath: "tile/data/255",
+		}, {
+			N:        255,
+			logSize:  255*256 - 3,
+			wantPath: "tile/data/255.p/253",
+		}, {
+			N:        256,
+			logSize:  257 * 256,
+			wantPath: "tile/data/256",
+		}, {
+			N:        123456789000,
+			logSize:  math.MaxUint64,
+			wantPath: "tile/data/x123/x456/x789/000",
+		},
+	} {
+		desc := fmt.Sprintf("N %d", test.N)
+		t.Run(desc, func(t *testing.T) {
+			gotPath := ctEntriesPath(test.N, test.logSize)
+			if gotPath != test.wantPath {
+				t.Errorf("got file %q want %q", gotPath, test.wantPath)
+			}
+		})
+	}
+}

log.go

@@ -20,6 +20,7 @@ import (
 	"time"
 
 	f_log "github.com/transparency-dev/formats/log"
+	"github.com/transparency-dev/trillian-tessera/api/layout"
 	"golang.org/x/mod/sumdb/note"
 )
 
@@ -41,6 +42,9 @@ type NewCPFunc func(size uint64, hash []byte) ([]byte, error)
 // ParseCPFunc is the signature of a function which knows how to verify and parse checkpoints.
 type ParseCPFunc func(raw []byte) (*f_log.Checkpoint, error)
 
+// EntriesPathFunc is the signature of a function which knows how to format entry bundle paths.
+type EntriesPathFunc func(n, logSize uint64) string
+
 // StorageOptions holds optional settings for all storage implementations.
 type StorageOptions struct {
 	NewCP   NewCPFunc
@@ -50,13 +54,16 @@ type StorageOptions struct {
 	BatchMaxSize uint
 
 	PushbackMaxOutstanding uint
+
+	EntriesPath EntriesPathFunc
 }
 
 // ResolveStorageOptions turns a variadic array of storage options into a StorageOptions instance.
 func ResolveStorageOptions(opts ...func(*StorageOptions)) *StorageOptions {
 	defaults := &StorageOptions{
 		BatchMaxSize: DefaultBatchMaxSize,
 		BatchMaxAge:  DefaultBatchMaxAge,
+		EntriesPath:  layout.EntriesPath,
 	}
 	for _, opt := range opts {
 		opt(defaults)

storage/gcp/gcp.go

@@ -70,7 +70,8 @@ type Storage struct {
 	projectID string
 	bucket    string
 
-	newCP tessera.NewCPFunc
+	newCP       tessera.NewCPFunc
+	entriesPath tessera.EntriesPathFunc
 
 	sequencer sequencer
 	objStore  objStore
@@ -129,12 +130,13 @@ func New(ctx context.Context, cfg Config, opts ...func(*tessera.StorageOptions))
 	}
 
 	r := &Storage{
-		gcsClient: c,
-		projectID: cfg.ProjectID,
-		bucket:    cfg.Bucket,
-		objStore:  gcsStorage,
-		sequencer: seq,
-		newCP:     opt.NewCP,
+		gcsClient:   c,
+		projectID:   cfg.ProjectID,
+		bucket:      cfg.Bucket,
+		objStore:    gcsStorage,
+		sequencer:   seq,
+		newCP:       opt.NewCP,
+		entriesPath: opt.EntriesPath,
 	}
 	r.queue = storage.NewQueue(ctx, opt.BatchMaxAge, opt.BatchMaxSize, r.sequencer.assignEntries)
 
@@ -229,7 +231,7 @@ func (s *Storage) getTiles(ctx context.Context, tileIDs []storage.TileID, logSiz
 //
 // Returns a wrapped os.ErrNotExist if the bundle does not exist.
 func (s *Storage) getEntryBundle(ctx context.Context, bundleIndex uint64, logSize uint64) ([]byte, error) {
-	objName := layout.EntriesPath(bundleIndex, logSize)
+	objName := s.entriesPath(bundleIndex, logSize)
 	data, _, err := s.objStore.getObject(ctx, objName)
 	if err != nil {
 		if errors.Is(err, gcs.ErrObjectNotExist) {
@@ -245,7 +247,7 @@ func (s *Storage) getEntryBundle(ctx context.Context, bundleIndex uint64, logSiz
 
 // setEntryBundle idempotently stores the serialised entry bundle at the location implied by the bundleIndex and treeSize.
 func (s *Storage) setEntryBundle(ctx context.Context, bundleIndex uint64, logSize uint64, bundleRaw []byte) error {
-	objName := layout.EntriesPath(bundleIndex, logSize)
+	objName := s.entriesPath(bundleIndex, logSize)
 	// Note that setObject does an idempotent interpretation of DoesNotExist - it only
 	// returns an error if the named object exists _and_ contains different data to what's
 	// passed in here.

storage/gcp/gcp_test.go

@@ -264,7 +264,8 @@ func TestBundleRoundtrip(t *testing.T) {
 	ctx := context.Background()
 	m := newMemObjStore()
 	s := &Storage{
-		objStore: m,
+		objStore:    m,
+		entriesPath: layout.EntriesPath,
 	}
 
 	for _, test := range []struct {

storage/posix/files.go

@@ -49,6 +49,8 @@ type Storage struct {
 
 	curSize uint64
 	newCP   tessera.NewCPFunc
+
+	entriesPath tessera.EntriesPathFunc
 }
 
 // NewTreeFunc is the signature of a function which receives information about newly integrated trees.
@@ -66,10 +68,11 @@ func New(ctx context.Context, path string, curTree func() (uint64, []byte, error
 	opt := tessera.ResolveStorageOptions(opts...)
 
 	r := &Storage{
-		path:    path,
-		curSize: curSize,
-		curTree: curTree,
-		newCP:   opt.NewCP,
+		path:        path,
+		curSize:     curSize,
+		curTree:     curTree,
+		newCP:       opt.NewCP,
+		entriesPath: opt.EntriesPath,
 	}
 	r.queue = storage.NewQueue(ctx, opt.BatchMaxAge, opt.BatchMaxSize, r.sequenceBatch)
 
@@ -124,7 +127,7 @@ func (s *Storage) Add(ctx context.Context, e *tessera.Entry) (uint64, error) {
 
 // GetEntryBundle retrieves the Nth entries bundle for a log of the given size.
 func (s *Storage) GetEntryBundle(ctx context.Context, index, logSize uint64) ([]byte, error) {
-	return os.ReadFile(filepath.Join(s.path, layout.EntriesPath(index, logSize)))
+	return os.ReadFile(filepath.Join(s.path, s.entriesPath(index, logSize)))
 }
 
 // sequenceBatch writes the entries from the provided batch into the entry bundle files of the log.
@@ -173,7 +176,7 @@ func (s *Storage) sequenceBatch(ctx context.Context, entries []*tessera.Entry) e
 		}
 	}
 	writeBundle := func(bundleIndex uint64) error {
-		bf := filepath.Join(s.path, layout.EntriesPath(bundleIndex, newSize))
+		bf := filepath.Join(s.path, s.entriesPath(bundleIndex, newSize))
 		if err := os.MkdirAll(filepath.Dir(bf), dirPerm); err != nil {
 			return fmt.Errorf("failed to make entries directory structure: %w", err)
 		}