diff --git a/api/layout/paths.go b/api/layout/paths.go index 7c42a07d..5a5bc2e3 100644 --- a/api/layout/paths.go +++ b/api/layout/paths.go @@ -37,25 +37,24 @@ func EntriesPathForLogIndex(seq, logSize uint64) string { return EntriesPath(seq/256, logSize) } -// EntriesPath returns the local path for the nth entry bundle. p denotes the partial -// tile size, or 0 if the tile is complete. -func EntriesPath(n, logSize uint64) string { +// NWithSuffix returns a tiles-spec "N" path, with a partial suffix if applicable. +func NWithSuffix(l, n, logSize uint64) string { suffix := "" - if p := partialTileSize(0, n, logSize); p > 0 { + if p := partialTileSize(l, n, logSize); p > 0 { suffix = fmt.Sprintf(".p/%d", p) } - return fmt.Sprintf("tile/entries%s%s", fmtN(n), suffix) + return fmt.Sprintf("%s%s", fmtN(n), suffix) +} + +// EntriesPath returns the local path for the nth entry bundle. p denotes the partial +// tile size, or 0 if the tile is complete. +func EntriesPath(n, logSize uint64) string { + return fmt.Sprintf("tile/entries/%s", NWithSuffix(0, n, logSize)) } // TilePath builds the path to the subtree tile with the given level and index in tile space. func TilePath(tileLevel, tileIndex, logSize uint64) string { - suffix := "" - p := partialTileSize(tileLevel, tileIndex, logSize) - if p > 0 { - suffix = fmt.Sprintf(".p/%d", p) - } - - return fmt.Sprintf("tile/%d%s%s", tileLevel, fmtN(tileIndex), suffix) + return fmt.Sprintf("tile/%d/%s", tileLevel, NWithSuffix(tileLevel, tileIndex, logSize)) } // fmtN returns the "N" part of a Tiles-spec path. @@ -67,10 +66,10 @@ func TilePath(tileLevel, tileIndex, logSize uint64) string { // // See https://github.com/C2SP/C2SP/blob/main/tlog-tiles.md#:~:text=index%201234067%20will%20be%20encoded%20as%20x001/x234/067 func fmtN(N uint64) string { - n := fmt.Sprintf("/%03d", N%1000) + n := fmt.Sprintf("%03d", N%1000) N /= 1000 for N > 0 { - n = fmt.Sprintf("/x%03d%s", N%1000, n) + n = fmt.Sprintf("x%03d/%s", N%1000, n) N /= 1000 } return n diff --git a/api/layout/paths_test.go b/api/layout/paths_test.go index 70234d95..a97dc9be 100644 --- a/api/layout/paths_test.go +++ b/api/layout/paths_test.go @@ -169,6 +169,70 @@ func TestTilePath(t *testing.T) { } } +func TestNWithSuffix(t *testing.T) { + for _, test := range []struct { + level uint64 + index uint64 + logSize uint64 + wantPath string + }{ + { + level: 0, + index: 0, + logSize: 256, + wantPath: "000", + }, { + level: 0, + index: 0, + logSize: 0, + wantPath: "000", + }, { + level: 0, + index: 0, + logSize: 255, + wantPath: "000.p/255", + }, { + level: 1, + index: 0, + logSize: math.MaxUint64, + wantPath: "000", + }, { + level: 1, + index: 0, + logSize: 256, + wantPath: "000.p/1", + }, { + level: 1, + index: 0, + logSize: 1024, + wantPath: "000.p/4", + }, { + level: 15, + index: 455667, + logSize: math.MaxUint64, + wantPath: "x455/667", + }, { + level: 3, + index: 1234567, + logSize: math.MaxUint64, + wantPath: "x001/x234/567", + }, { + level: 15, + index: 123456789, + logSize: math.MaxUint64, + wantPath: "x123/x456/789", + }, + } { + desc := fmt.Sprintf("level %x index %x", test.level, test.index) + t.Run(desc, func(t *testing.T) { + gotPath := NWithSuffix(test.level, test.index, test.logSize) + if gotPath != test.wantPath { + t.Errorf("Got path %q want %q", gotPath, test.wantPath) + } + }) + } +} + func TestParseTileLevelIndexWidth(t *testing.T) { for _, test := range []struct { pathLevel string diff --git a/ct_only.go b/ct_only.go index d6b60ac0..9fa8eb53 100644 --- a/ct_only.go +++ b/ct_only.go @@ -16,7 +16,9 @@ package tessera import ( "context" + "fmt" + "github.com/transparency-dev/trillian-tessera/api/layout" "github.com/transparency-dev/trillian-tessera/ctonly" ) @@ -56,3 +58,14 @@ func convertCTEntry(e *ctonly.Entry) *Entry { return r } + +// WithCTLayout instructs the underlying storage to use a Static CT API compatible scheme for layout. +func WithCTLayout() func(*StorageOptions) { + return func(opts *StorageOptions) { + opts.EntriesPath = ctEntriesPath + } +} + +func ctEntriesPath(n, logSize uint64) string { + return fmt.Sprintf("tile/data/%s", layout.NWithSuffix(0, n, logSize)) +} diff --git a/ct_only_test.go b/ct_only_test.go new file mode 100644 index 00000000..7048a466 --- /dev/null +++ b/ct_only_test.go @@ -0,0 +1,64 @@ +// Copyright 2024 Google LLC. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tessera + +import ( + "fmt" + "math" + "testing" +) + +func TestCTEntriesPath(t *testing.T) { + for _, test := range []struct { + N uint64 + logSize uint64 + wantPath string + }{ + { + N: 0, + logSize: 289, + wantPath: "tile/data/000", + }, + { + N: 0, + logSize: 8, + wantPath: "tile/data/000.p/8", + }, { + N: 255, + logSize: 256 * 256, + wantPath: "tile/data/255", + }, { + N: 255, + logSize: 255*256 - 3, + wantPath: "tile/data/255.p/253", + }, { + N: 256, + logSize: 257 * 256, + wantPath: "tile/data/256", + }, { + N: 123456789000, + logSize: math.MaxUint64, + wantPath: "tile/data/x123/x456/x789/000", + }, + } { + desc := fmt.Sprintf("N %d", test.N) + t.Run(desc, func(t *testing.T) { + gotPath := ctEntriesPath(test.N, test.logSize) + if gotPath != test.wantPath { + t.Errorf("got file %q want %q", gotPath, test.wantPath) + } + }) + } +} diff --git a/log.go b/log.go index da6431f1..bb2c3088 100644 --- a/log.go +++ b/log.go @@ -20,6 +20,7 @@ import ( "time" f_log "github.com/transparency-dev/formats/log" + "github.com/transparency-dev/trillian-tessera/api/layout" "golang.org/x/mod/sumdb/note" ) @@ -41,6 +42,9 @@ type NewCPFunc func(size uint64, hash []byte) ([]byte, error) // ParseCPFunc is the signature of a function which knows how to verify and parse checkpoints. type ParseCPFunc func(raw []byte) (*f_log.Checkpoint, error) +// EntriesPathFunc is the signature of a function which knows how to format entry bundle paths. +type EntriesPathFunc func(n, logSize uint64) string + // StorageOptions holds optional settings for all storage implementations. type StorageOptions struct { NewCP NewCPFunc @@ -50,6 +54,8 @@ type StorageOptions struct { BatchMaxSize uint PushbackMaxOutstanding uint + + EntriesPath EntriesPathFunc } // ResolveStorageOptions turns a variadic array of storage options into a StorageOptions instance. @@ -57,6 +63,7 @@ func ResolveStorageOptions(opts ...func(*StorageOptions)) *StorageOptions { defaults := &StorageOptions{ BatchMaxSize: DefaultBatchMaxSize, BatchMaxAge: DefaultBatchMaxAge, + EntriesPath: layout.EntriesPath, } for _, opt := range opts { opt(defaults) diff --git a/storage/gcp/gcp.go b/storage/gcp/gcp.go index f575a282..635c6ae7 100644 --- a/storage/gcp/gcp.go +++ b/storage/gcp/gcp.go @@ -70,7 +70,8 @@ type Storage struct { projectID string bucket string - newCP tessera.NewCPFunc + newCP tessera.NewCPFunc + entriesPath tessera.EntriesPathFunc sequencer sequencer objStore objStore @@ -129,12 +130,13 @@ func New(ctx context.Context, cfg Config, opts ...func(*tessera.StorageOptions)) } r := &Storage{ - gcsClient: c, - projectID: cfg.ProjectID, - bucket: cfg.Bucket, - objStore: gcsStorage, - sequencer: seq, - newCP: opt.NewCP, + gcsClient: c, + projectID: cfg.ProjectID, + bucket: cfg.Bucket, + objStore: gcsStorage, + sequencer: seq, + newCP: opt.NewCP, + entriesPath: opt.EntriesPath, } r.queue = storage.NewQueue(ctx, opt.BatchMaxAge, opt.BatchMaxSize, r.sequencer.assignEntries) @@ -229,7 +231,7 @@ func (s *Storage) getTiles(ctx context.Context, tileIDs []storage.TileID, logSiz // // Returns a wrapped os.ErrNotExist if the bundle does not exist. func (s *Storage) getEntryBundle(ctx context.Context, bundleIndex uint64, logSize uint64) ([]byte, error) { - objName := layout.EntriesPath(bundleIndex, logSize) + objName := s.entriesPath(bundleIndex, logSize) data, _, err := s.objStore.getObject(ctx, objName) if err != nil { if errors.Is(err, gcs.ErrObjectNotExist) { @@ -245,7 +247,7 @@ func (s *Storage) getEntryBundle(ctx context.Context, bundleIndex uint64, logSiz // setEntryBundle idempotently stores the serialised entry bundle at the location implied by the bundleIndex and treeSize. func (s *Storage) setEntryBundle(ctx context.Context, bundleIndex uint64, logSize uint64, bundleRaw []byte) error { - objName := layout.EntriesPath(bundleIndex, logSize) + objName := s.entriesPath(bundleIndex, logSize) // Note that setObject does an idempotent interpretation of DoesNotExist - it only // returns an error if the named object exists _and_ contains different data to what's // passed in here. diff --git a/storage/gcp/gcp_test.go b/storage/gcp/gcp_test.go index 2fda4b72..47d3ef46 100644 --- a/storage/gcp/gcp_test.go +++ b/storage/gcp/gcp_test.go @@ -264,7 +264,8 @@ func TestBundleRoundtrip(t *testing.T) { ctx := context.Background() m := newMemObjStore() s := &Storage{ - objStore: m, + objStore: m, + entriesPath: layout.EntriesPath, } for _, test := range []struct { diff --git a/storage/posix/files.go b/storage/posix/files.go index 658a99fb..0ac639a7 100644 --- a/storage/posix/files.go +++ b/storage/posix/files.go @@ -49,6 +49,8 @@ type Storage struct { curSize uint64 newCP tessera.NewCPFunc + + entriesPath tessera.EntriesPathFunc } // NewTreeFunc is the signature of a function which receives information about newly integrated trees. @@ -66,10 +68,11 @@ func New(ctx context.Context, path string, curTree func() (uint64, []byte, error opt := tessera.ResolveStorageOptions(opts...) r := &Storage{ - path: path, - curSize: curSize, - curTree: curTree, - newCP: opt.NewCP, + path: path, + curSize: curSize, + curTree: curTree, + newCP: opt.NewCP, + entriesPath: opt.EntriesPath, } r.queue = storage.NewQueue(ctx, opt.BatchMaxAge, opt.BatchMaxSize, r.sequenceBatch) @@ -124,7 +127,7 @@ func (s *Storage) Add(ctx context.Context, e *tessera.Entry) (uint64, error) { // GetEntryBundle retrieves the Nth entries bundle for a log of the given size. func (s *Storage) GetEntryBundle(ctx context.Context, index, logSize uint64) ([]byte, error) { - return os.ReadFile(filepath.Join(s.path, layout.EntriesPath(index, logSize))) + return os.ReadFile(filepath.Join(s.path, s.entriesPath(index, logSize))) } // sequenceBatch writes the entries from the provided batch into the entry bundle files of the log. @@ -173,7 +176,7 @@ func (s *Storage) sequenceBatch(ctx context.Context, entries []*tessera.Entry) e } } writeBundle := func(bundleIndex uint64) error { - bf := filepath.Join(s.path, layout.EntriesPath(bundleIndex, newSize)) + bf := filepath.Join(s.path, s.entriesPath(bundleIndex, newSize)) if err := os.MkdirAll(filepath.Dir(bf), dirPerm); err != nil { return fmt.Errorf("failed to make entries directory structure: %w", err) }