POSIX: inline log initialization

Both personalities had duplicate code for creating directories and writing out an empty checkpoint. This logic is going to be the same for any POSIX log usage, so it should be part of the log implementation.

For now, this is behind an extra parameter that is passed to New. There may be better ways to set this up, but I tried a few and this was the best. This is a pragmatic step forward and better than what we had, so let's get this in and refactor further from here if needed.

Also cleaned up the `New` method by moving an inline function that was assigned to a field is instead defined as a method.

cmd/example-posix/main.go

@@ -27,16 +27,9 @@ import (
 
 	"golang.org/x/mod/sumdb/note"
 
-	"github.com/transparency-dev/merkle/rfc6962"
 	tessera "github.com/transparency-dev/trillian-tessera"
 	"github.com/transparency-dev/trillian-tessera/storage/posix"
 	"k8s.io/klog/v2"
-
-	fmtlog "github.com/transparency-dev/formats/log"
-)
-
-const (
-	dirPerm = 0o755
 )
 
 var (
@@ -85,40 +78,17 @@ func main() {
 	if err != nil {
 		klog.Exitf("Failed to instantiate signer: %q", err)
 	}
-	origin := s.Name()
-
-	writeCP := func(size uint64, root []byte) error {
-		cp := &fmtlog.Checkpoint{
-			Origin: origin,
-			Size:   size,
-			Hash:   root,
-		}
-		n, err := note.Sign(&note.Note{Text: string(cp.Marshal())}, s)
-		if err != nil {
-			return err
-		}
-		return posix.WriteCheckpoint(*storageDir, n)
-	}
-	if *initialise {
-		if files, err := os.ReadDir(*storageDir); err == nil && len(files) > 0 {
-			klog.Exitf("Cannot initialize a log at a non-empty directory")
-		}
-		if err := os.MkdirAll(*storageDir, dirPerm); err != nil {
-			klog.Exitf("Failed to create log directory %q: %v", *storageDir, err)
-		}
-		if err := writeCP(0, rfc6962.DefaultHasher.EmptyRoot()); err != nil {
-			klog.Exitf("Failed to write empty checkpoint")
-		}
-		klog.Infof("Initialized the log at %q", *storageDir)
-	}
 
 	// Check signatures
 	v, err := note.NewVerifier(pubKey)
 	if err != nil {
-		klog.Exitf("Failed to instantiate Verifier: %q", err)
+		klog.Exitf("Failed to instantiate Verifier: %v", err)
 	}
 
-	storage := posix.New(ctx, *storageDir, tessera.WithCheckpointSignerVerifier(s, v), tessera.WithBatching(256, time.Second))
+	storage, err := posix.New(ctx, *storageDir, *initialise, tessera.WithCheckpointSignerVerifier(s, v), tessera.WithBatching(256, time.Second))
+	if err != nil {
+		klog.Exitf("Failed to construct storage: %v", err)
+	}
 
 	http.HandleFunc("POST /add", func(w http.ResponseWriter, r *http.Request) {
 		b, err := io.ReadAll(r.Body)

cmd/posix-oneshot/main.go

@@ -29,16 +29,9 @@ import (
 
 	"golang.org/x/mod/sumdb/note"
 
-	"github.com/transparency-dev/merkle/rfc6962"
 	tessera "github.com/transparency-dev/trillian-tessera"
 	"github.com/transparency-dev/trillian-tessera/storage/posix"
 	"k8s.io/klog/v2"
-
-	fmtlog "github.com/transparency-dev/formats/log"
-)
-
-const (
-	dirPerm = 0o755
 )
 
 var (
@@ -57,33 +50,23 @@ func main() {
 	// Gather the info needed for reading/writing checkpoints
 	v := getVerifierOrDie()
 	s := getSignerOrDie()
-	origin := s.Name()
 
-	if *initialise {
-		// Create the directory structure and write out an empty checkpoint
-		if err := os.MkdirAll(*storageDir, dirPerm); err != nil {
-			klog.Exitf("Failed to create log directory: %q", err)
-		}
-		// TODO(mhutchinson): This empty checkpoint initialization should live in Tessera
-		emptyCP := &fmtlog.Checkpoint{
-			Origin: origin,
-			Size:   0,
-			Hash:   rfc6962.DefaultHasher.EmptyRoot(),
-		}
-		n, err := note.Sign(&note.Note{Text: string(emptyCP.Marshal())}, s)
-		if err != nil {
-			klog.Exitf("Failed to sign empty checkpoint: %s", err)
-		}
-		if err := posix.WriteCheckpoint(*storageDir, n); err != nil {
-			klog.Exitf("Failed to write empty checkpoint: %s", err)
+	if len(*entries) == 0 {
+		if *initialise {
+			_, err := posix.New(ctx, *storageDir, *initialise, tessera.WithCheckpointSignerVerifier(s, v))
+			if err != nil {
+				klog.Exitf("Failed to initialise storage: %v", err)
+			}
 		}
-		// TODO(mhutchinson): This should continue if *entries is provided
+		klog.Info("No entries provided to integrate; exiting")
 		os.Exit(0)
 	}
 
 	filesToAdd := readEntriesOrDie()
-
-	st := posix.New(ctx, *storageDir, tessera.WithCheckpointSignerVerifier(s, v), tessera.WithBatching(uint(len(filesToAdd)), time.Second))
+	st, err := posix.New(ctx, *storageDir, *initialise, tessera.WithCheckpointSignerVerifier(s, v), tessera.WithBatching(uint(len(filesToAdd)), time.Second))
+	if err != nil {
+		klog.Exitf("Failed to construct storage: %v", err)
+	}
 
 	// sequence entries
 

storage/posix/files.go

@@ -24,6 +24,7 @@ import (
 	"sync"
 	"syscall"
 
+	"github.com/transparency-dev/merkle/rfc6962"
 	tessera "github.com/transparency-dev/trillian-tessera"
 	"github.com/transparency-dev/trillian-tessera/api"
 	"github.com/transparency-dev/trillian-tessera/api/layout"
@@ -45,9 +46,9 @@ type Storage struct {
 
 	cpFile *os.File
 
-	curTree func() (uint64, []byte, error)
 	curSize uint64
 	newCP   tessera.NewCPFunc
+	parseCP tessera.ParseCPFunc
 
 	entriesPath tessera.EntriesPathFunc
 }
@@ -56,34 +57,35 @@ type Storage struct {
 type NewTreeFunc func(size uint64, root []byte) error
 
 // New creates a new POSIX storage.
-func New(ctx context.Context, path string, opts ...func(*tessera.StorageOptions)) *Storage {
+// - path is a directory in which the log should be stored
+// - create must only be set when first creating the log, and will create the directory structure and an empty checkpoint
+func New(ctx context.Context, path string, create bool, opts ...func(*tessera.StorageOptions)) (*Storage, error) {
 	opt := tessera.ResolveStorageOptions(opts...)
-	curTree := func() (uint64, []byte, error) {
-		cpRaw, err := readCheckpoint(path)
-		if err != nil {
-			return 0, nil, fmt.Errorf("failed to read log checkpoint: %q", err)
-		}
-		cp, err := opt.ParseCP(cpRaw)
-		if err != nil {
-			return 0, nil, fmt.Errorf("failed to parse Checkpoint: %q", err)
-		}
-		return cp.Size, cp.Hash, nil
-	}
-	curSize, _, err := curTree()
-	if err != nil {
-		panic(err)
-	}
 
 	r := &Storage{
 		path:        path,
-		curSize:     curSize,
-		curTree:     curTree,
 		newCP:       opt.NewCP,
+		parseCP:     opt.ParseCP,
 		entriesPath: opt.EntriesPath,
 	}
+	if err := r.initialise(create); err != nil {
+		return nil, err
+	}
 	r.queue = storage.NewQueue(ctx, opt.BatchMaxAge, opt.BatchMaxSize, r.sequenceBatch)
 
-	return r
+	return r, nil
+}
+
+func (s *Storage) curTree() (uint64, []byte, error) {
+	cpRaw, err := readCheckpoint(s.path)
+	if err != nil {
+		return 0, nil, fmt.Errorf("failed to read log checkpoint: %q", err)
+	}
+	cp, err := s.parseCP(cpRaw)
+	if err != nil {
+		return 0, nil, fmt.Errorf("failed to parse Checkpoint: %q", err)
+	}
+	return cp.Size, cp.Hash, nil
 }
 
 // lockCP places a POSIX advisory lock for the checkpoint.
@@ -362,6 +364,33 @@ func (s *Storage) StoreTile(_ context.Context, level, index, logSize uint64, til
 	return nil
 }
 
+// initialise ensures that the storage location is valid by loading the checkpoint from this location.
+// If `create` is set to true, then this will first ensure that the directory path is created, and
+// an empty checkpoint is created in this directory.
+func (s *Storage) initialise(create bool) error {
+	if create {
+		// Create the directory structure and write out an empty checkpoint
+		klog.Infof("Initializing directory for POSIX log at %q (this should only happen ONCE per log!)", s.path)
+		if err := os.MkdirAll(s.path, dirPerm); err != nil {
+			return fmt.Errorf("Failed to create log directory: %q", err)
+		}
+		n, err := s.newCP(0, rfc6962.DefaultHasher.EmptyRoot())
+		if err != nil {
+			return fmt.Errorf("failed to sign empty checkpoint: %v", err)
+		}
+		if err := WriteCheckpoint(s.path, n); err != nil {
+			return fmt.Errorf("failed to write empty checkpoint: %v", err)
+		}
+	}
+	curSize, _, err := s.curTree()
+	if err != nil {
+		return fmt.Errorf("failed to load checkpoint for log: %v", err)
+	}
+	s.curSize = curSize
+
+	return nil
+}
+
 // WriteCheckpoint stores a raw log checkpoint on disk.
 func WriteCheckpoint(path string, newCPRaw []byte) error {
 	if err := createExclusive(filepath.Join(path, layout.CheckpointPath), newCPRaw); err != nil {