Adds an RFC6962-via-sunlight feeder

go.mod

@@ -17,7 +17,7 @@ require (
 
 require (
  github.com/prometheus/client_golang v1.19.0
- github.com/transparency-dev/formats v0.0.0-20231205184308-949529efd6b3
+ github.com/transparency-dev/formats v0.0.0-20240415152152-3372d765e9e4
  github.com/transparency-dev/serverless-log v0.0.0-20240408141044-5d483a81bdb7
  k8s.io/klog/v2 v2.120.1
 )
@@ -26,10 +26,12 @@ require (
  github.com/beorn7/perks v1.0.1 // indirect
  github.com/cespare/xxhash/v2 v2.2.0 // indirect
  github.com/go-logr/logr v1.4.1 // indirect
+ github.com/google/certificate-transparency-go v1.1.8 // indirect
  github.com/kr/text v0.1.0 // indirect
  github.com/prometheus/client_model v0.5.0 // indirect
  github.com/prometheus/common v0.48.0 // indirect
  github.com/prometheus/procfs v0.12.0 // indirect
- golang.org/x/sys v0.17.0 // indirect
+ golang.org/x/crypto v0.21.0 // indirect
+ golang.org/x/sys v0.18.0 // indirect
  google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de // indirect
 )

go.sum

@@ -8,6 +8,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ=
 github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/google/certificate-transparency-go v1.1.8 h1:LGYKkgZF7satzgTak9R4yzfJXEeYVAjV6/EAEJOf1to=
+github.com/google/certificate-transparency-go v1.1.8/go.mod h1:bV/o8r0TBKRf1X//iiiSgWrvII4d7/8OiA+3vG26gI8=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
@@ -29,20 +31,22 @@ github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k
 github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
 github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
-github.com/transparency-dev/formats v0.0.0-20231205184308-949529efd6b3 h1:Mpx9pqc7bKrx2QQxKL3SPbLIGH4gTBR1ZFrNuKq3CcY=
-github.com/transparency-dev/formats v0.0.0-20231205184308-949529efd6b3/go.mod h1:tY9Z9oBaYdQt4NWIhsFAtv0altwLk+K9Gg/2tbS0eBQ=
+github.com/transparency-dev/formats v0.0.0-20240415152152-3372d765e9e4 h1:hpLmCbRJ9rtPsaYuEpVCbdBqdtrmfYAqxjMNGkaP/+s=
+github.com/transparency-dev/formats v0.0.0-20240415152152-3372d765e9e4/go.mod h1:tyFqZYwVHIv/ap0obDip//541gI2TL7SwuWkhlg5WXc=
 github.com/transparency-dev/merkle v0.0.2 h1:Q9nBoQcZcgPamMkGn7ghV8XiTZ/kRxn1yCG81+twTK4=
 github.com/transparency-dev/merkle v0.0.2/go.mod h1:pqSy+OXefQ1EDUVmAJ8MUhHB9TXGuzVAT58PqBoHz1A=
 github.com/transparency-dev/serverless-log v0.0.0-20240408141044-5d483a81bdb7 h1:Caqvx+/b2hpuK5dHLMtKxoNsNhSf6JsT9m+7Xgk1z6Y=
 github.com/transparency-dev/serverless-log v0.0.0-20240408141044-5d483a81bdb7/go.mod h1:A+cQ9EQeah/Ua7JaMOAAKkCfyDZPsq74o+UgwqQEPsQ=
+golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
+golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
 golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA=
 golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
-golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
-golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc=
+golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
 golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
 golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
-golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
+golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de h1:cZGRis4/ot9uVm639a+rHCUaG0JJHEsdyzSQTMX+suY=

internal/feeder/rfc6962/rfc6962_feeder.go

@@ -0,0 +1,142 @@
+// Copyright 2021 Google LLC. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package rfc6962 is an implementation of a witness feeder for RFC6962 logs.
+//
+// This package uses the sunlight checkpoint representation of RFC6962 Signed Tree Head
+// structures in order to be able to feed them natively into the omniwitness.
+//
+// Note that Signed Tree Heads and sunlight checkpoints are convertible between
+// these formats without needing the log private key.
+package rfc6962
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
+ "net/url"
+ "os"
+ "time"
+
+ "github.com/transparency-dev/formats/log"
+ "github.com/transparency-dev/formats/note"
+ "github.com/transparency-dev/witness/internal/config"
+ "github.com/transparency-dev/witness/internal/feeder"
+ "k8s.io/klog/v2"
+)
+
+// proof is a partial representation of the JSON struct returned by the CT
+// get-sth-consistency request.
+type proof struct {
+ Consistency [][]byte `json:"consistency"`
+}
+
+// FeedLog feeds checkpoints from the source log to the witness.
+// If interval is non-zero, this function will return when the context is done, otherwise it will perform
+// one feed cycle and return.
+//
+// Note that this feeder expects the configured URL to contain a "treeID" query parameter which contains the
+// correct Rekor log tree ID.
+func FeedLog(ctx context.Context, l config.Log, w feeder.Witness, c *http.Client, interval time.Duration) error {
+ lURL, err := url.Parse(l.URL)
+ if err != nil {
+ return fmt.Errorf("invalid LogURL %q: %v", l.URL, err)
+ }
+
+ fetchCP := func(ctx context.Context) ([]byte, error) {
+ sth, err := get(ctx, c, lURL, "ct/v1/get-sth")
+ if err != nil {
+ return nil, fmt.Errorf("failed to fetch STH: %v", err)
+ }
+
+ cp, err := note.RFC6962STHToCheckpoint(sth, l.Verifier)
+ if err != nil {
+ return nil, fmt.Errorf("unable to convert STH to checkpoint: %v", err)
+ }
+ return cp, nil
+ }
+ fetchProof := func(ctx context.Context, from, to log.Checkpoint) ([][]byte, error) {
+ if from.Size == 0 {
+ return [][]byte{}, nil
+ }
+ cp := proof{}
+ if err := getJSON(ctx, c, lURL, fmt.Sprintf("ct/v1/get-sth-consistency?first=%d&second=%d", from.Size, to.Size), &cp); err != nil {
+ return nil, fmt.Errorf("failed to fetch consistency proof: %v", err)
+ }
+ return cp.Consistency, nil
+ }
+
+ opts := feeder.FeedOpts{
+ LogID: l.ID,
+ LogOrigin: l.Origin,
+ FetchCheckpoint: fetchCP,
+ FetchProof: fetchProof,
+ LogSigVerifier: l.Verifier,
+ Witness: w,
+ }
+ if interval > 0 {
+ return feeder.Run(ctx, interval, opts)
+ }
+ _, err = feeder.FeedOnce(ctx, opts)
+ return err
+}
+
+func getJSON(ctx context.Context, c *http.Client, base *url.URL, path string, s interface{}) error {
+ raw, err := get(ctx, c, base, path)
+ if err != nil {
+ return fmt.Errorf("failed to get: %v", err)
+ }
+ if err := json.Unmarshal(raw, s); err != nil {
+ return fmt.Errorf("failed to unmarshal JSON: %v", err)
+ }
+ return nil
+}
+
+func get(ctx context.Context, c *http.Client, base *url.URL, path string) ([]byte, error) {
+ u, err := base.Parse(path)
+ if err != nil {
+ return nil, fmt.Errorf("failed to parse URL: %v", err)
+ }
+ req, err := http.NewRequest("GET", u.String(), nil)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create request: %v", err)
+ }
+ req = req.WithContext(ctx)
+ req.Header.Set("Accept", "application/json")
+
+ rsp, err := c.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to make request to %q: %v", u.String(), err)
+ }
+ defer func() {
+ if err := rsp.Body.Close(); err != nil {
+ klog.Infof("Close: %v", err)
+ }
+ }()
+
+ if rsp.StatusCode == 404 {
+ return nil, os.ErrNotExist
+ }
+ if rsp.StatusCode != 200 {
+ return nil, fmt.Errorf("unexpected status fetching %q: %s", u.String(), rsp.Status)
+ }
+
+ raw, err := io.ReadAll(rsp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read body from %q: %v", u.String(), err)
+ }
+ return raw, nil
+}

omniwitness/logs.yaml

@@ -42,3 +42,8 @@ Logs:
  PublicKey: transparency.dev-aw-ftlog-ci-3+3f689522+Aa1Eifq6rRC8qiK+bya07yV1fXyP156pEMsX7CFBC6gg
  Feeder: serverless
  UseCompact: false
+ - Origin: ct.googleapis.com/logs/us1/argon2024
+ URL: https://ct.googleapis.com/logs/us1/argon2024/
+ PublicKey: ct.googleapis.com/logs/us1/argon2024+7deb49d0+BTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABB25bKnLaZTFXOa2pgO70rjcVEMXKJkMBgFQHZ1kwFlGK9zIAx0FtC2oCfeZQe0E++VXuiYE9hFSzhRlOy92K8A=
+ Feeder: rfc6962
+ UseCompact: false

omniwitness/omniwitness.go

@@ -43,6 +43,7 @@ import (
  "github.com/transparency-dev/witness/internal/feeder"
  "github.com/transparency-dev/witness/internal/feeder/pixelbt"
  "github.com/transparency-dev/witness/internal/feeder/rekor"
+ "github.com/transparency-dev/witness/internal/feeder/rfc6962"
  "github.com/transparency-dev/witness/internal/feeder/serverless"
  "github.com/transparency-dev/witness/internal/feeder/sumdb"
 )
@@ -211,6 +212,7 @@ const (
  SumDB
  Pixel
  Rekor
+ RFC6962
 )
 
 var (
@@ -219,6 +221,7 @@ var (
  "sumdb": SumDB,
  "pixel": Pixel,
  "rekor": Rekor,
+ "rfc6962": RFC6962,
  }
 )
 
@@ -244,6 +247,8 @@ func (f Feeder) FeedFunc() logFeeder {
  return pixelbt.FeedLog
  case Rekor:
  return rekor.FeedLog
+ case RFC6962:
+ return rfc6962.FeedLog
  }
  panic(fmt.Sprintf("unknown feeder enum: %q", f))
 }