Add MySQL tlog tiles API integration test

.github/workflows/integration_test.yml

@@ -0,0 +1,21 @@
+name: Integration Test
+
+on: [push, pull_request]
+
+permissions:
+ contents: read
+
+jobs:
+ mysql-tlog-tiles-api:
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+ - name: Start Docker services (tessera-example-mysql-db and tessera-example-mysql)
+ run: docker compose -f ./cmd/example-mysql/docker/compose.yaml up --build --detach
+ - name: Run integration test 
+ run: go test -v -race ./integration/example-mysql/... --run_mysql_integration_test=true
+ - name: Stop Docker services (tessera-example-mysql-db and tessera-example-mysql)
+ if: ${{ always() }}
+ run: docker compose -f ./cmd/example-mysql/docker/compose.yaml down

cmd/example-mysql/main.go

@@ -135,7 +135,7 @@ func main() {
  return
  }
 
- entryBundle, err := storage.ReadEntryBundle(r.Context(), index)
+ entryBundle, err := storage.ReadEntryBundle(r.Context(), index/256)
  if err != nil {
  // TODO: Move this error back into storage implementation.
  if err == sql.ErrNoRows {
@@ -167,7 +167,6 @@ func main() {
  klog.Warningf("/add: %v", err)
  }
  }()
-
  idx, err := storage.Add(r.Context(), tessera.NewEntry(b))
  if err != nil {
  w.WriteHeader(http.StatusInternalServerError)

integration/example-mysql/integration_test.go

@@ -0,0 +1,232 @@
+// Copyright 2024 The Tessera authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package integration_test contains some integration tests which are intended to
+// serve as a way of checking that example-mysql binary works as intended,
+// as well as providing a simple example of how to run and use it.
+package integration_test
+
+import (
+ "bytes"
+ "context"
+ "flag"
+ "fmt"
+ "io"
+ "net/http"
+ "net/url"
+ "os"
+ "strconv"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/transparency-dev/formats/log"
+ "github.com/transparency-dev/merkle/proof"
+ "github.com/transparency-dev/merkle/rfc6962"
+ "github.com/transparency-dev/trillian-tessera/client"
+ "golang.org/x/mod/sumdb/note"
+ "golang.org/x/sync/errgroup"
+ "k8s.io/klog/v2"
+)
+
+var (
+ runMySQLIntegrationTest = flag.Bool("run_mysql_integration_test", false, "If true, the MySQL integration tests in this package will not be skipped")
+ logURL = flag.String("log_url", "http://localhost:2024", "Log storage root URL, e.g. https://log.server/and/path/")
+ testEntrySize = flag.Int("test_entry_size", 1024, "The number of entries to be tested in the live log integration")
+
+ noteVerifier note.Verifier
+
+ hc = &http.Client{
+ Transport: &http.Transport{
+ MaxIdleConns: 256,
+ MaxIdleConnsPerHost: 256,
+ },
+ Timeout: 5 * time.Second,
+ }
+)
+
+const testPublicKey = "Test-Betty+df84580a+AQQASqPUZoIHcJAF5mBOryctwFdTV1E0GRY4kEAtTzwB"
+
+func TestMain(m *testing.M) {
+ klog.InitFlags(nil)
+ flag.Parse()
+
+ if !*runMySQLIntegrationTest {
+ klog.Warning("example-mysql integration tests are skipped")
+ return
+ }
+
+ var err error
+ noteVerifier, err = note.NewVerifier(testPublicKey)
+ if err != nil {
+ klog.Fatalf("Failed to create new verifier: %v", err)
+ }
+
+ os.Exit(m.Run())
+}
+
+func TestLiveLogIntegration(t *testing.T) {
+ ctx := context.Background()
+ checkpoints := []log.Checkpoint{}
+ var entryIndexMap sync.Map
+
+ // Step 1 - Get checkpoint initial size for increment validation.
+ var checkpointInitSize uint64
+ checkpoint, _, _, err := client.FetchCheckpoint(ctx, httpRead, noteVerifier, noteVerifier.Name())
+ if err != nil {
+ t.Errorf("client.FetchCheckpoint: %v", err)
+ }
+ checkpointInitSize = checkpoint.Size
+ t.Logf("checkpoint initial size: %d", checkpointInitSize)
+ checkpoints = append(checkpoints, *checkpoint)
+
+ // Step 2 - Add entries and get new checkpoints. The entry data comes from the int loop ranging from 0 to the test entry size - 1.
+ addEntriesURL, err := url.JoinPath(*logURL, "add")
+ if err != nil {
+ t.Errorf("url.JoinPath: %v", err)
+ }
+ entryWriter := entryWriter{
+ addURL: addEntriesURL,
+ }
+ errG := errgroup.Group{}
+ for i := 0; i < *testEntrySize; i++ {
+ errG.Go(func() error {
+ index, err := entryWriter.add(ctx, []byte(fmt.Sprint(i)))
+ if err != nil {
+ t.Errorf("entryWriter.add: %v", err)
+ }
+ entryIndexMap.Store(i, index)
+ checkpoint, _, _, err := client.FetchCheckpoint(ctx, httpRead, noteVerifier, noteVerifier.Name())
+ if err != nil {
+ t.Errorf("client.FetchCheckpoint: %v", err)
+ }
+ checkpoints = append(checkpoints, *checkpoint)
+ return err
+ })
+ }
+ if err := errG.Wait(); err != nil {
+ t.Errorf("addEntry: %v", err)
+ }
+
+ // Step 3 - Validate checkpoint size increment.
+ checkpoint, _, _, err = client.FetchCheckpoint(ctx, httpRead, noteVerifier, noteVerifier.Name())
+ if err != nil {
+ t.Errorf("client.FetchCheckpoint: %v", err)
+ }
+ t.Logf("checkpoint final size: %d", checkpoint.Size)
+ gotIncrease := checkpoint.Size - checkpointInitSize
+ if gotIncrease != uint64(*testEntrySize) {
+ t.Errorf("checkpoint size increase got: %d, want: %d", gotIncrease, *testEntrySize)
+ }
+
+ // Step 4 - Loop through the entry data index map to verify leaves and inclusion proofs.
+ entryIndexMap.Range(func(k, v any) bool {
+ data := k.(int)
+ index := v.(uint64)
+
+ // Step 4.1 - Get entry bundles to read back what was written, check leaves are correct.
+ entryBundle, err := client.GetEntryBundle(ctx, httpRead, index, checkpoint.Size)
+ if err != nil {
+ t.Errorf("client.GetEntryBundle: %v", err)
+ }
+
+ got, want := entryBundle.Entries[index%256], []byte(fmt.Sprint(data))
+ if !bytes.Equal(got, want) {
+ t.Errorf("Entry bundle got %v want %v", got, want)
+ }
+
+ // Step 4.2 - Test inclusion proofs.
+ pb, err := client.NewProofBuilder(ctx, *checkpoint, httpRead)
+ if err != nil {
+ t.Errorf("client.NewProofBuilder: %v", err)
+ }
+ ip, err := pb.InclusionProof(ctx, index)
+ if err != nil {
+ t.Errorf("pb.InclusionProof: %v", err)
+ }
+ leafHash := rfc6962.DefaultHasher.HashLeaf([]byte(fmt.Sprint(data)))
+ if err := proof.VerifyInclusion(rfc6962.DefaultHasher, index, checkpoint.Size, leafHash, ip, checkpoint.Hash); err != nil {
+ t.Errorf("proof.VerifyInclusion: %v", err)
+ }
+
+ return true
+ })
+
+ // Step 5 - Test consistency proofs.
+ if err := client.CheckConsistency(ctx, httpRead, checkpoints); err != nil {
+ t.Errorf("log consistency for %v: unexpected proof returned", err)
+ }
+}
+
+func httpRead(ctx context.Context, path string) ([]byte, error) {
+ url, err := url.JoinPath(*logURL, path)
+ if err != nil {
+ return nil, err
+ }
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+ if err != nil {
+ return nil, err
+ }
+ resp, err := hc.Do(req)
+ if err != nil {
+ return nil, err
+ }
+ body, err := io.ReadAll(resp.Body)
+ defer func() {
+ if err := resp.Body.Close(); err != nil {
+ klog.Warningf("resp.Body.Close(): %v", err)
+ }
+ }()
+ if err != nil {
+ return nil, fmt.Errorf("failed to read response from %s: %w", path, err)
+ }
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("code: %s, path: %s, body: %s", resp.Status, path, strings.TrimSpace(string(body)))
+ }
+ return body, nil
+}
+
+type entryWriter struct {
+ addURL string
+}
+
+func (w *entryWriter) add(ctx context.Context, entry []byte) (uint64, error) {
+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, w.addURL, bytes.NewReader(entry))
+ if err != nil {
+ return 0, err
+ }
+ resp, err := hc.Do(req)
+ if err != nil {
+ return 0, err
+ }
+ body, err := io.ReadAll(resp.Body)
+ defer func() {
+ if err := resp.Body.Close(); err != nil {
+ klog.Warningf("resp.Body.Close(): %v", err)
+ }
+ }()
+ if err != nil {
+ return 0, fmt.Errorf("failed to read response from %s: %w", w.addURL, err)
+ }
+ if resp.StatusCode != http.StatusOK {
+ return 0, fmt.Errorf("code: %s, path: %s, body: %s", resp.Status, w.addURL, strings.TrimSpace(string(body)))
+ }
+ index, err := strconv.ParseUint(string(body), 10, 64)
+ if err != nil {
+ return 0, err
+ }
+
+ return index, nil
+}