From d32f6610668e6e76eb9ac120ddce686187bacd77 Mon Sep 17 00:00:00 2001 From: Martin Hutchinson Date: Tue, 3 Sep 2024 14:00:13 +0100 Subject: [PATCH] POSIX oneshot: refactor code out of main method The goal here is that the main method should contain only the interesting code for putting the entries in the log. This PR moves code into methods to aid readability: - initializing the verifier - initializing the signer - evaluating the glob This fixes a bug where it would deadlock due to filling entryChan if more than 100 leaves were added at a time. --- cmd/posix-oneshot/main.go | 160 +++++++++++++++++++++----------------- 1 file changed, 87 insertions(+), 73 deletions(-) diff --git a/cmd/posix-oneshot/main.go b/cmd/posix-oneshot/main.go index ee865e39..4fce0a93 100644 --- a/cmd/posix-oneshot/main.go +++ b/cmd/posix-oneshot/main.go @@ -52,93 +52,50 @@ var ( func main() { klog.InitFlags(nil) flag.Parse() - ctx := context.Background() - // Read log public key from file or environment variable - var pubKey string - var err error - if len(*pubKeyFile) > 0 { - pubKey, err = getKeyFile(*pubKeyFile) - if err != nil { - klog.Exitf("Unable to get public key: %q", err) - } - } else { - pubKey = os.Getenv("LOG_PUBLIC_KEY") - if len(pubKey) == 0 { - klog.Exit("Supply public key file path using --public_key or set LOG_PUBLIC_KEY environment variable") - } - } - // Read log private key from file or environment variable - var privKey string - if len(*privKeyFile) > 0 { - privKey, err = getKeyFile(*privKeyFile) - if err != nil { - klog.Exitf("Unable to get private key: %q", err) - } - } else { - privKey = os.Getenv("LOG_PRIVATE_KEY") - if len(privKey) == 0 { - klog.Exit("Supply private key file path using --private_key or set LOG_PRIVATE_KEY environment variable") - } - } - - s, err := note.NewSigner(privKey) - if err != nil { - klog.Exitf("Failed to instantiate signer: %q", err) - } + // Gather the info needed for reading/writing checkpoints + v := getVerifierOrDie() + s := getSignerOrDie() origin := s.Name() - writeCP := func(size uint64, root []byte) error { - cp := &fmtlog.Checkpoint{ - Origin: origin, - Size: size, - Hash: root, - } - n, err := note.Sign(¬e.Note{Text: string(cp.Marshal())}, s) - if err != nil { - return err - } - return posix.WriteCheckpoint(*storageDir, n) - } if *initialise { + // Create the directory structure and write out an empty checkpoint if err := os.MkdirAll(*storageDir, dirPerm); err != nil { klog.Exitf("Failed to create log directory: %q", err) } - if err := writeCP(0, rfc6962.DefaultHasher.EmptyRoot()); err != nil { - klog.Exitf("Failed to write empty checkpoint") + // TODO(mhutchinson): This empty checkpoint initialization should live in Tessera + emptyCP := &fmtlog.Checkpoint{ + Origin: origin, + Size: 0, + Hash: rfc6962.DefaultHasher.EmptyRoot(), + } + n, err := note.Sign(¬e.Note{Text: string(emptyCP.Marshal())}, s) + if err != nil { + klog.Exitf("Failed to sign empty checkpoint: %s", err) } + if err := posix.WriteCheckpoint(*storageDir, n); err != nil { + klog.Exitf("Failed to write empty checkpoint: %s", err) + } + // TODO(mhutchinson): This should continue if *entries is provided os.Exit(0) } - toAdd, err := filepath.Glob(*entries) - if err != nil { - klog.Exitf("Failed to glob entries %q: %q", *entries, err) - } - klog.V(1).Infof("toAdd: %v", toAdd) - if len(toAdd) == 0 { - klog.Exit("Sequence must be run with at least one valid entry") - } - - cpRaw, err := posix.ReadCheckpoint(*storageDir) - if err != nil { - klog.Exitf("Failed to read log checkpoint: %q", err) - } - - // Check signatures - v, err := note.NewVerifier(pubKey) - if err != nil { - klog.Exitf("Failed to instantiate Verifier: %q", err) - } + filesToAdd := readEntriesOrDie() + // TODO(mhutchinson): This function should be implemented inside Tessera (it has the verifier now) readCP := func() (uint64, []byte, error) { + cpRaw, err := posix.ReadCheckpoint(*storageDir) + if err != nil { + klog.Exitf("Failed to read log checkpoint: %q", err) + } cp, _, _, err := fmtlog.ParseCheckpoint(cpRaw, origin, v) if err != nil { return 0, []byte{}, fmt.Errorf("Failed to parse Checkpoint: %q", err) } return cp.Size, cp.Hash, nil } - st := posix.New(ctx, *storageDir, readCP, tessera.WithCheckpointSignerVerifier(s, v), tessera.WithBatching(uint(len(toAdd)), time.Second)) + st := posix.New(ctx, *storageDir, readCP, tessera.WithCheckpointSignerVerifier(s, v), tessera.WithBatching(uint(len(filesToAdd)), time.Second)) // sequence entries @@ -150,20 +107,19 @@ func main() { name string f tessera.IndexFuture } - entryChan := make(chan entryInfo, 100) - for _, fp := range toAdd { + indexFutures := make([]entryInfo, len(filesToAdd)) + for _, fp := range filesToAdd { b, err := os.ReadFile(fp) if err != nil { klog.Exitf("Failed to read entry file %q: %q", fp, err) } - // ask storage to sequence, we'll put the future we get back into the entryChan for later... + // ask storage to sequence and we'll store the future for later f := st.Add(ctx, tessera.NewEntry(b)) - entryChan <- entryInfo{name: fp, f: f} + indexFutures = append(indexFutures, entryInfo{name: fp, f: f}) } - close(entryChan) - for entry := range entryChan { + for _, entry := range indexFutures { seq, err := entry.f() if err != nil { klog.Exitf("failed to sequence %q: %q", entry.name, err) @@ -172,6 +128,52 @@ func main() { } } +// Read log public key from file or environment variable +func getVerifierOrDie() note.Verifier { + var pubKey string + var err error + if len(*pubKeyFile) > 0 { + pubKey, err = getKeyFile(*pubKeyFile) + if err != nil { + klog.Exitf("Unable to get public key: %q", err) + } + } else { + pubKey = os.Getenv("LOG_PUBLIC_KEY") + if len(pubKey) == 0 { + klog.Exit("Supply public key file path using --public_key or set LOG_PUBLIC_KEY environment variable") + } + } + // Check signatures + v, err := note.NewVerifier(pubKey) + if err != nil { + klog.Exitf("Failed to instantiate Verifier: %q", err) + } + + return v +} + +// Read log private key from file or environment variable +func getSignerOrDie() note.Signer { + var privKey string + var err error + if len(*privKeyFile) > 0 { + privKey, err = getKeyFile(*privKeyFile) + if err != nil { + klog.Exitf("Unable to get private key: %q", err) + } + } else { + privKey = os.Getenv("LOG_PRIVATE_KEY") + if len(privKey) == 0 { + klog.Exit("Supply private key file path using --private_key or set LOG_PRIVATE_KEY environment variable") + } + } + s, err := note.NewSigner(privKey) + if err != nil { + klog.Exitf("Failed to instantiate signer: %q", err) + } + return s +} + func getKeyFile(path string) (string, error) { k, err := os.ReadFile(path) if err != nil { @@ -179,3 +181,15 @@ func getKeyFile(path string) (string, error) { } return string(k), nil } + +func readEntriesOrDie() []string { + toAdd, err := filepath.Glob(*entries) + if err != nil { + klog.Exitf("Failed to glob entries %q: %q", *entries, err) + } + klog.V(1).Infof("toAdd: %v", toAdd) + if len(toAdd) == 0 { + klog.Exit("Sequence must be run with at least one valid entry") + } + return toAdd +}