From 0d88f1368fb351dff6c16b3e4d306a019c2e27ab Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Sat, 10 Aug 2024 22:13:20 +0000 Subject: [PATCH] Add MySQL tlog tiles API integration test --- .github/workflows/integration_test.yml | 21 ++ cmd/example-mysql/main.go | 3 +- integration/example-mysql/integration_test.go | 232 ++++++++++++++++++ 3 files changed, 254 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/integration_test.yml create mode 100644 integration/example-mysql/integration_test.go diff --git a/.github/workflows/integration_test.yml b/.github/workflows/integration_test.yml new file mode 100644 index 00000000..7fd5121f --- /dev/null +++ b/.github/workflows/integration_test.yml @@ -0,0 +1,21 @@ +name: Integration Test + +on: [push, pull_request] + +permissions: + contents: read + +jobs: + mysql-tlog-tiles-api: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - name: Start Docker services (tessera-example-mysql-db and tessera-example-mysql) + run: docker compose -f ./cmd/example-mysql/docker/compose.yaml up --build --detach + - name: Run integration test + run: go test -v -race ./integration/example-mysql/... --run_mysql_integration_test=true + - name: Stop Docker services (tessera-example-mysql-db and tessera-example-mysql) + if: ${{ always() }} + run: docker compose -f ./cmd/example-mysql/docker/compose.yaml down diff --git a/cmd/example-mysql/main.go b/cmd/example-mysql/main.go index 7ed6a1cc..23d696ce 100644 --- a/cmd/example-mysql/main.go +++ b/cmd/example-mysql/main.go @@ -135,7 +135,7 @@ func main() { return } - entryBundle, err := storage.ReadEntryBundle(r.Context(), index) + entryBundle, err := storage.ReadEntryBundle(r.Context(), index/256) if err != nil { // TODO: Move this error back into storage implementation. if err == sql.ErrNoRows { @@ -167,7 +167,6 @@ func main() { klog.Warningf("/add: %v", err) } }() - idx, err := storage.Add(r.Context(), tessera.NewEntry(b)) if err != nil { w.WriteHeader(http.StatusInternalServerError) diff --git a/integration/example-mysql/integration_test.go b/integration/example-mysql/integration_test.go new file mode 100644 index 00000000..711994c3 --- /dev/null +++ b/integration/example-mysql/integration_test.go @@ -0,0 +1,232 @@ +// Copyright 2024 The Tessera authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package integration_test contains some integration tests which are intended to +// serve as a way of checking that example-mysql binary works as intended, +// as well as providing a simple example of how to run and use it. +package integration_test + +import ( + "bytes" + "context" + "flag" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/transparency-dev/formats/log" + "github.com/transparency-dev/merkle/proof" + "github.com/transparency-dev/merkle/rfc6962" + "github.com/transparency-dev/trillian-tessera/client" + "golang.org/x/mod/sumdb/note" + "golang.org/x/sync/errgroup" + "k8s.io/klog/v2" +) + +var ( + runMySQLIntegrationTest = flag.Bool("run_mysql_integration_test", false, "If true, the MySQL integration tests in this package will not be skipped") + logURL = flag.String("log_url", "http://localhost:2024", "Log storage root URL, e.g. https://log.server/and/path/") + testEntrySize = flag.Int("test_entry_size", 1024, "The number of entries to be tested in the live log integration") + + noteVerifier note.Verifier + + hc = &http.Client{ + Transport: &http.Transport{ + MaxIdleConns: 256, + MaxIdleConnsPerHost: 256, + }, + Timeout: 5 * time.Second, + } +) + +const testPublicKey = "Test-Betty+df84580a+AQQASqPUZoIHcJAF5mBOryctwFdTV1E0GRY4kEAtTzwB" + +func TestMain(m *testing.M) { + klog.InitFlags(nil) + flag.Parse() + + if !*runMySQLIntegrationTest { + klog.Warning("example-mysql integration tests are skipped") + return + } + + var err error + noteVerifier, err = note.NewVerifier(testPublicKey) + if err != nil { + klog.Fatalf("Failed to create new verifier: %v", err) + } + + os.Exit(m.Run()) +} + +func TestLiveLogIntegration(t *testing.T) { + ctx := context.Background() + checkpoints := []log.Checkpoint{} + var entryIndexMap sync.Map + + // Step 1 - Get checkpoint initial size for increment validation. + var checkpointInitSize uint64 + checkpoint, _, _, err := client.FetchCheckpoint(ctx, httpRead, noteVerifier, noteVerifier.Name()) + if err != nil { + t.Errorf("client.FetchCheckpoint: %v", err) + } + checkpointInitSize = checkpoint.Size + t.Logf("checkpoint initial size: %d", checkpointInitSize) + checkpoints = append(checkpoints, *checkpoint) + + // Step 2 - Add entries and get new checkpoints. The entry data comes from the int loop ranging from 0 to the test entry size - 1. + addEntriesURL, err := url.JoinPath(*logURL, "add") + if err != nil { + t.Errorf("url.JoinPath: %v", err) + } + entryWriter := entryWriter{ + addURL: addEntriesURL, + } + errG := errgroup.Group{} + for i := 0; i < *testEntrySize; i++ { + errG.Go(func() error { + index, err := entryWriter.add(ctx, []byte(fmt.Sprint(i))) + if err != nil { + t.Errorf("entryWriter.add: %v", err) + } + entryIndexMap.Store(i, index) + checkpoint, _, _, err := client.FetchCheckpoint(ctx, httpRead, noteVerifier, noteVerifier.Name()) + if err != nil { + t.Errorf("client.FetchCheckpoint: %v", err) + } + checkpoints = append(checkpoints, *checkpoint) + return err + }) + } + if err := errG.Wait(); err != nil { + t.Errorf("addEntry: %v", err) + } + + // Step 3 - Validate checkpoint size increment. + checkpoint, _, _, err = client.FetchCheckpoint(ctx, httpRead, noteVerifier, noteVerifier.Name()) + if err != nil { + t.Errorf("client.FetchCheckpoint: %v", err) + } + t.Logf("checkpoint final size: %d", checkpoint.Size) + gotIncrease := checkpoint.Size - checkpointInitSize + if gotIncrease != uint64(*testEntrySize) { + t.Errorf("checkpoint size increase got: %d, want: %d", gotIncrease, *testEntrySize) + } + + // Step 4 - Loop through the entry data index map to verify leaves and inclusion proofs. + entryIndexMap.Range(func(k, v any) bool { + data := k.(int) + index := v.(uint64) + + // Step 4.1 - Get entry bundles to read back what was written, check leaves are correct. + entryBundle, err := client.GetEntryBundle(ctx, httpRead, index, checkpoint.Size) + if err != nil { + t.Errorf("client.GetEntryBundle: %v", err) + } + + got, want := entryBundle.Entries[index%256], []byte(fmt.Sprint(data)) + if !bytes.Equal(got, want) { + t.Errorf("Entry bundle got %v want %v", got, want) + } + + // Step 4.2 - Test inclusion proofs. + pb, err := client.NewProofBuilder(ctx, *checkpoint, httpRead) + if err != nil { + t.Errorf("client.NewProofBuilder: %v", err) + } + ip, err := pb.InclusionProof(ctx, index) + if err != nil { + t.Errorf("pb.InclusionProof: %v", err) + } + leafHash := rfc6962.DefaultHasher.HashLeaf([]byte(fmt.Sprint(data))) + if err := proof.VerifyInclusion(rfc6962.DefaultHasher, index, checkpoint.Size, leafHash, ip, checkpoint.Hash); err != nil { + t.Errorf("proof.VerifyInclusion: %v", err) + } + + return true + }) + + // Step 5 - Test consistency proofs. + if err := client.CheckConsistency(ctx, httpRead, checkpoints); err != nil { + t.Errorf("log consistency for %v: unexpected proof returned", err) + } +} + +func httpRead(ctx context.Context, path string) ([]byte, error) { + url, err := url.JoinPath(*logURL, path) + if err != nil { + return nil, err + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + resp, err := hc.Do(req) + if err != nil { + return nil, err + } + body, err := io.ReadAll(resp.Body) + defer func() { + if err := resp.Body.Close(); err != nil { + klog.Warningf("resp.Body.Close(): %v", err) + } + }() + if err != nil { + return nil, fmt.Errorf("failed to read response from %s: %w", path, err) + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("code: %s, path: %s, body: %s", resp.Status, path, strings.TrimSpace(string(body))) + } + return body, nil +} + +type entryWriter struct { + addURL string +} + +func (w *entryWriter) add(ctx context.Context, entry []byte) (uint64, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, w.addURL, bytes.NewReader(entry)) + if err != nil { + return 0, err + } + resp, err := hc.Do(req) + if err != nil { + return 0, err + } + body, err := io.ReadAll(resp.Body) + defer func() { + if err := resp.Body.Close(); err != nil { + klog.Warningf("resp.Body.Close(): %v", err) + } + }() + if err != nil { + return 0, fmt.Errorf("failed to read response from %s: %w", w.addURL, err) + } + if resp.StatusCode != http.StatusOK { + return 0, fmt.Errorf("code: %s, path: %s, body: %s", resp.Status, w.addURL, strings.TrimSpace(string(body))) + } + index, err := strconv.ParseUint(string(body), 10, 64) + if err != nil { + return 0, err + } + + return index, nil +}