From b9c0e49c6fba97fb4a73b72dee39c0b8547016a1 Mon Sep 17 00:00:00 2001 From: Al Cutter Date: Thu, 28 Mar 2024 18:16:01 +0000 Subject: [PATCH 1/4] Add support for RFC6962 logs via sunlight checkpoints --- internal/feeder/rfc6962/rfc6962_feeder.go | 150 ++++++++++++++++++++++ omniwitness/omniwitness.go | 5 + 2 files changed, 155 insertions(+) create mode 100644 internal/feeder/rfc6962/rfc6962_feeder.go diff --git a/internal/feeder/rfc6962/rfc6962_feeder.go b/internal/feeder/rfc6962/rfc6962_feeder.go new file mode 100644 index 0000000..e388aa3 --- /dev/null +++ b/internal/feeder/rfc6962/rfc6962_feeder.go @@ -0,0 +1,150 @@ +// Copyright 2021 Google LLC. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package rfc6962 is an implementation of a witness feeder for RFC6962 logs. +// +// This package uses the sunlight checkpoint representation of RFC6962 Signed Tree Head +// structures in order to be able to feed them natively into the omniwitness. +// +// Note that Signed Tree Heads and sunlight checkpoints are convertible between +// these formats without needing the log private key. +package rfc6962 + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "time" + + "github.com/transparency-dev/formats/log" + "github.com/transparency-dev/formats/note" + "github.com/transparency-dev/witness/internal/config" + "github.com/transparency-dev/witness/internal/feeder" + "k8s.io/klog/v2" +) + +// signedTreeHead represents the structure returned by the get-sth CT method +// after base64 decoding; see sections 3.5 and 4.3. +type signedTreeHead struct { + Version int `json:"sth_version"` // The version of the protocol to which the STH conforms + TreeSize uint64 `json:"tree_size"` // The number of entries in the new tree + Timestamp uint64 `json:"timestamp"` // The time at which the STH was created + SHA256RootHash []byte `json:"sha256_root_hash"` // The root hash of the log's Merkle tree + TreeHeadSignature []byte `json:"tree_head_signature"` // Log's signature over a TLS-encoded TreeHeadSignature + LogID []byte `json:"log_id"` // The SHA256 hash of the log's public key +} + +// proof is a partial representation of the JSON struct returned by the CT +// get-sth-consistency request. +type proof struct { + Consistency [][]byte `json:"consistency"` +} + +// FeedLog feeds checkpoints from the source log to the witness. +// If interval is non-zero, this function will return when the context is done, otherwise it will perform +// one feed cycle and return. +// +// Note that this feeder expects the configured URL to contain a "treeID" query parameter which contains the +// correct Rekor log tree ID. +func FeedLog(ctx context.Context, l config.Log, w feeder.Witness, c *http.Client, interval time.Duration) error { + lURL, err := url.Parse(l.URL) + if err != nil { + return fmt.Errorf("invalid LogURL %q: %v", l.URL, err) + } + + fetchCP := func(ctx context.Context) ([]byte, error) { + sth, err := get(ctx, c, lURL, "ct/v1/get-sth") + if err != nil { + return nil, fmt.Errorf("failed to fetch STH: %v", err) + } + + cp, err := note.RFC6962STHToCheckpoint(sth, l.Verifier) + if err != nil { + return nil, fmt.Errorf("unable to convert STH to checkpoint: %v", err) + } + return cp, nil + } + fetchProof := func(ctx context.Context, from, to log.Checkpoint) ([][]byte, error) { + if from.Size == 0 { + return [][]byte{}, nil + } + cp := proof{} + if err := getJSON(ctx, c, lURL, fmt.Sprintf("ct/v1/get-sth-consistency?first=%d&second=%d", from.Size, to.Size), &cp); err != nil { + return nil, fmt.Errorf("failed to fetch consistency proof: %v", err) + } + return cp.Consistency, nil + } + + opts := feeder.FeedOpts{ + LogID: l.ID, + LogOrigin: l.Origin, + FetchCheckpoint: fetchCP, + FetchProof: fetchProof, + LogSigVerifier: l.Verifier, + Witness: w, + } + if interval > 0 { + return feeder.Run(ctx, interval, opts) + } + _, err = feeder.FeedOnce(ctx, opts) + return err +} + +func getJSON(ctx context.Context, c *http.Client, base *url.URL, path string, s interface{}) error { + raw, err := get(ctx, c, base, path) + if err != nil { + return fmt.Errorf("failed to get: %v", err) + } + if err := json.Unmarshal(raw, s); err != nil { + klog.Infof("Got body:\n%s", string(raw)) + return fmt.Errorf("failed to unmarshal JSON: %v", err) + } + return nil +} + +func get(ctx context.Context, c *http.Client, base *url.URL, path string) ([]byte, error) { + u, err := base.Parse(path) + if err != nil { + return nil, fmt.Errorf("failed to parse URL: %v", err) + } + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %v", err) + } + req = req.WithContext(ctx) + req.Header.Set("Accept", "application/json") + + rsp, err := c.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to make request to %q: %v", u.String(), err) + } + defer rsp.Body.Close() + + if rsp.StatusCode == 404 { + return nil, os.ErrNotExist + } + if rsp.StatusCode != 200 { + return nil, fmt.Errorf("unexpected status fetching %q: %s", u.String(), rsp.Status) + } + + raw, err := io.ReadAll(rsp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read body from %q: %v", u.String(), err) + } + return raw, nil +} diff --git a/omniwitness/omniwitness.go b/omniwitness/omniwitness.go index 5e90542..daec91d 100644 --- a/omniwitness/omniwitness.go +++ b/omniwitness/omniwitness.go @@ -43,6 +43,7 @@ import ( "github.com/transparency-dev/witness/internal/feeder" "github.com/transparency-dev/witness/internal/feeder/pixelbt" "github.com/transparency-dev/witness/internal/feeder/rekor" + "github.com/transparency-dev/witness/internal/feeder/rfc6962" "github.com/transparency-dev/witness/internal/feeder/serverless" "github.com/transparency-dev/witness/internal/feeder/sumdb" ) @@ -211,6 +212,7 @@ const ( SumDB Pixel Rekor + RFC6962 ) var ( @@ -219,6 +221,7 @@ var ( "sumdb": SumDB, "pixel": Pixel, "rekor": Rekor, + "rfc6962": RFC6962, } ) @@ -244,6 +247,8 @@ func (f Feeder) FeedFunc() logFeeder { return pixelbt.FeedLog case Rekor: return rekor.FeedLog + case RFC6962: + return rfc6962.FeedLog } panic(fmt.Sprintf("unknown feeder enum: %q", f)) } From b08992b351f3bd624fb01d39e4bace9ea6201aeb Mon Sep 17 00:00:00 2001 From: Al Cutter Date: Thu, 28 Mar 2024 18:16:07 +0000 Subject: [PATCH 2/4] Example config --- omniwitness/logs.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/omniwitness/logs.yaml b/omniwitness/logs.yaml index 09b7efb..129dcf3 100644 --- a/omniwitness/logs.yaml +++ b/omniwitness/logs.yaml @@ -42,3 +42,8 @@ Logs: PublicKey: transparency.dev-aw-ftlog-ci-3+3f689522+Aa1Eifq6rRC8qiK+bya07yV1fXyP156pEMsX7CFBC6gg Feeder: serverless UseCompact: false + - Origin: ct.googleapis.com/logs/us1/argon2024 + URL: https://ct.googleapis.com/logs/us1/argon2024/ + PublicKey: ct.googleapis.com/logs/us1/argon2024+7deb49d0+BTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABB25bKnLaZTFXOa2pgO70rjcVEMXKJkMBgFQHZ1kwFlGK9zIAx0FtC2oCfeZQe0E++VXuiYE9hFSzhRlOy92K8A= + Feeder: rfc6962 + UseCompact: false From 1bb652d7a892cb5cbe4bbd0a9cde188cd0d07800 Mon Sep 17 00:00:00 2001 From: Al Cutter Date: Mon, 15 Apr 2024 16:22:55 +0100 Subject: [PATCH 3/4] Bump formats to 3372d765e9e4d07c2d910ca3e35712d160c874ff --- go.mod | 6 ++++-- go.sum | 16 ++++++++++------ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index c7134d5..dd1d522 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( require ( github.com/prometheus/client_golang v1.19.0 - github.com/transparency-dev/formats v0.0.0-20231205184308-949529efd6b3 + github.com/transparency-dev/formats v0.0.0-20240415152152-3372d765e9e4 github.com/transparency-dev/serverless-log v0.0.0-20240408141044-5d483a81bdb7 k8s.io/klog/v2 v2.120.1 ) @@ -26,10 +26,12 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/go-logr/logr v1.4.1 // indirect + github.com/google/certificate-transparency-go v1.1.8 // indirect github.com/kr/text v0.1.0 // indirect github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.48.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect - golang.org/x/sys v0.17.0 // indirect + golang.org/x/crypto v0.21.0 // indirect + golang.org/x/sys v0.18.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de // indirect ) diff --git a/go.sum b/go.sum index 1cf5cf9..7c6e6c8 100644 --- a/go.sum +++ b/go.sum @@ -8,6 +8,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/google/certificate-transparency-go v1.1.8 h1:LGYKkgZF7satzgTak9R4yzfJXEeYVAjV6/EAEJOf1to= +github.com/google/certificate-transparency-go v1.1.8/go.mod h1:bV/o8r0TBKRf1X//iiiSgWrvII4d7/8OiA+3vG26gI8= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= @@ -29,20 +31,22 @@ github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= -github.com/transparency-dev/formats v0.0.0-20231205184308-949529efd6b3 h1:Mpx9pqc7bKrx2QQxKL3SPbLIGH4gTBR1ZFrNuKq3CcY= -github.com/transparency-dev/formats v0.0.0-20231205184308-949529efd6b3/go.mod h1:tY9Z9oBaYdQt4NWIhsFAtv0altwLk+K9Gg/2tbS0eBQ= +github.com/transparency-dev/formats v0.0.0-20240415152152-3372d765e9e4 h1:hpLmCbRJ9rtPsaYuEpVCbdBqdtrmfYAqxjMNGkaP/+s= +github.com/transparency-dev/formats v0.0.0-20240415152152-3372d765e9e4/go.mod h1:tyFqZYwVHIv/ap0obDip//541gI2TL7SwuWkhlg5WXc= github.com/transparency-dev/merkle v0.0.2 h1:Q9nBoQcZcgPamMkGn7ghV8XiTZ/kRxn1yCG81+twTK4= github.com/transparency-dev/merkle v0.0.2/go.mod h1:pqSy+OXefQ1EDUVmAJ8MUhHB9TXGuzVAT58PqBoHz1A= github.com/transparency-dev/serverless-log v0.0.0-20240408141044-5d483a81bdb7 h1:Caqvx+/b2hpuK5dHLMtKxoNsNhSf6JsT9m+7Xgk1z6Y= github.com/transparency-dev/serverless-log v0.0.0-20240408141044-5d483a81bdb7/go.mod h1:A+cQ9EQeah/Ua7JaMOAAKkCfyDZPsq74o+UgwqQEPsQ= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= +golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= -golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de h1:cZGRis4/ot9uVm639a+rHCUaG0JJHEsdyzSQTMX+suY= From 483b318bb650a86f7aae79034942bb105f302757 Mon Sep 17 00:00:00 2001 From: Al Cutter Date: Mon, 15 Apr 2024 16:29:03 +0100 Subject: [PATCH 4/4] lint --- internal/feeder/rfc6962/rfc6962_feeder.go | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/internal/feeder/rfc6962/rfc6962_feeder.go b/internal/feeder/rfc6962/rfc6962_feeder.go index e388aa3..dc08f4f 100644 --- a/internal/feeder/rfc6962/rfc6962_feeder.go +++ b/internal/feeder/rfc6962/rfc6962_feeder.go @@ -38,17 +38,6 @@ import ( "k8s.io/klog/v2" ) -// signedTreeHead represents the structure returned by the get-sth CT method -// after base64 decoding; see sections 3.5 and 4.3. -type signedTreeHead struct { - Version int `json:"sth_version"` // The version of the protocol to which the STH conforms - TreeSize uint64 `json:"tree_size"` // The number of entries in the new tree - Timestamp uint64 `json:"timestamp"` // The time at which the STH was created - SHA256RootHash []byte `json:"sha256_root_hash"` // The root hash of the log's Merkle tree - TreeHeadSignature []byte `json:"tree_head_signature"` // Log's signature over a TLS-encoded TreeHeadSignature - LogID []byte `json:"log_id"` // The SHA256 hash of the log's public key -} - // proof is a partial representation of the JSON struct returned by the CT // get-sth-consistency request. type proof struct { @@ -111,7 +100,6 @@ func getJSON(ctx context.Context, c *http.Client, base *url.URL, path string, s return fmt.Errorf("failed to get: %v", err) } if err := json.Unmarshal(raw, s); err != nil { - klog.Infof("Got body:\n%s", string(raw)) return fmt.Errorf("failed to unmarshal JSON: %v", err) } return nil @@ -133,7 +121,11 @@ func get(ctx context.Context, c *http.Client, base *url.URL, path string) ([]byt if err != nil { return nil, fmt.Errorf("failed to make request to %q: %v", u.String(), err) } - defer rsp.Body.Close() + defer func() { + if err := rsp.Body.Close(); err != nil { + klog.Infof("Close: %v", err) + } + }() if rsp.StatusCode == 404 { return nil, os.ErrNotExist