feat: init (#1)

This PR adds an initial implementation for a storage node. It implements the "storage node" in the following sequence diagram: ```mermaid sequenceDiagram participant alice participant service participant storagenode alice->>service: blob/add with: alicespace nb: blob=car size=3 service->>storagenode: blob/allocate with: storagenode nb: space=alicespace blob=car size=3 storagenode->>service: receipt ran: blob/allocate out: url service->>alice: receipt ran: blob/add fx: blob/allocate, http/put, blob/accept <br> (included in response: receipt ran: blob/allocate out: url) alice->>storagenode: HTTP PUT alice->>service: receipt ran: http/put service->>storagenode: blob/accept with: storagenode nb: space=alicespace blob=car storagenode->>service: receipt ran: blob/accept out: locationcommitment alice->>service: poll to get receipt for blob/accept ``` * Filesystem blob storage and HTTP read/write API (done ✅) * Delegation storage for location commitments and HTTP retrieval API (done ✅) * Allocation tracking store to keep track of pending uploads (done ✅) * Signed URL support (done ✅) * Ucanto endpoint for `blob/allocate` (done ✅) * Ucanto endpoint for `blob/accept` (done ✅) * IPNI publisher for location commitments (done ✅) * Build out the CLI (done ✅) Next PR: * Indexing service caching for location commitments Outside of planned scope here be needed eventually: * Receipt storage and retrieval (for future use when submitting logs for verification)
storacha · Oct 21, 2024 · daa353b · daa353b
1 parent b60da10
commit daa353b
Show file tree

Hide file tree

Showing 64 changed files with 4,895 additions and 0 deletions.
diff --git a/.github/workflows/go-check.yml b/.github/workflows/go-check.yml
@@ -0,0 +1,18 @@
+name: Go Checks
+
+on:
+  pull_request:
+  push:
+    branches: ["main"]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  go-check:
+    uses: ipdxco/unified-github-workflows/.github/workflows/[email protected]
diff --git a/.github/workflows/go-test-config.json b/.github/workflows/go-test-config.json
@@ -0,0 +1,3 @@
+{
+  "skip32bit": true
+}
diff --git a/.github/workflows/go-test.yml b/.github/workflows/go-test.yml
@@ -0,0 +1,20 @@
+name: Go Test
+
+on:
+  pull_request:
+  push:
+    branches: ["main"]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  go-test:
+    uses: ipdxco/unified-github-workflows/.github/workflows/[email protected]
+    with:
+      go-versions: '["this"]' 
diff --git a/cmd/main.go b/cmd/main.go
@@ -0,0 +1,223 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/url"
+	"os"
+	"path"
+	"time"
+
+	leveldb "github.com/ipfs/go-ds-leveldb"
+	logging "github.com/ipfs/go-log/v2"
+	"github.com/storacha/go-ucanto/did"
+	"github.com/storacha/go-ucanto/principal"
+	ed25519 "github.com/storacha/go-ucanto/principal/ed25519/signer"
+	"github.com/storacha/storage/pkg/server"
+	"github.com/storacha/storage/pkg/service/storage"
+	"github.com/storacha/storage/pkg/store/blobstore"
+	"github.com/urfave/cli/v2"
+)
+
+var log = logging.Logger("cmd")
+
+func main() {
+	app := &cli.App{
+		Name:  "storage",
+		Usage: "Manage running a storage node.",
+		Commands: []*cli.Command{
+			{
+				Name:  "start",
+				Usage: "Start the storage node daemon.",
+				Flags: []cli.Flag{
+					&cli.IntFlag{
+						Name:    "port",
+						Aliases: []string{"p"},
+						Value:   3000,
+						Usage:   "Port to bind the server to.",
+					},
+					&cli.StringFlag{
+						Name:    "private-key",
+						Aliases: []string{"s"},
+						Usage:   "Multibase base64 encoded private key identity for the node.",
+					},
+					&cli.StringFlag{
+						Name:    "data-dir",
+						Aliases: []string{"d"},
+						Usage:   "Root directory to store data in.",
+					},
+					&cli.StringFlag{
+						Name:    "public-url",
+						Aliases: []string{"u"},
+						Usage:   "URL the node is publically accessible at.",
+					},
+				},
+				Action: func(cCtx *cli.Context) error {
+					var id principal.Signer
+					var err error
+					if cCtx.String("private-key") == "" {
+						id, err = ed25519.Generate()
+						if err != nil {
+							return fmt.Errorf("generating ed25519 key: %w", err)
+						}
+						log.Errorf("Server ID is not configured, generated one for you: %s", id.DID().String())
+					} else {
+						id, err = ed25519.Parse(cCtx.String("private-key"))
+						if err != nil {
+							return fmt.Errorf("parsing private key: %w", err)
+						}
+					}
+
+					homeDir, err := os.UserHomeDir()
+					if err != nil {
+						return fmt.Errorf("getting user home directory: %w", err)
+					}
+
+					dataDir := cCtx.String("data-dir")
+					if dataDir == "" {
+						dir, err := mkdirp(homeDir, ".storacha")
+						if err != nil {
+							return err
+						}
+						log.Errorf("Data directory is not configured, using default: %s", dir)
+						dataDir = dir
+					}
+
+					blobStore, err := blobstore.NewFsBlobstore(path.Join(dataDir, "blobs"))
+					if err != nil {
+						return fmt.Errorf("creating blob storage: %w", err)
+					}
+
+					allocsDir, err := mkdirp(dataDir, "allocation")
+					if err != nil {
+						return err
+					}
+					allocDs, err := leveldb.NewDatastore(allocsDir, nil)
+					if err != nil {
+						return err
+					}
+					claimsDir, err := mkdirp(dataDir, "claim")
+					if err != nil {
+						return err
+					}
+					claimDs, err := leveldb.NewDatastore(claimsDir, nil)
+					if err != nil {
+						return err
+					}
+					publisherDir, err := mkdirp(dataDir, "publisher")
+					if err != nil {
+						return err
+					}
+					publisherDs, err := leveldb.NewDatastore(publisherDir, nil)
+					if err != nil {
+						return err
+					}
+
+					pubURLstr := cCtx.String("public-url")
+					if pubURLstr == "" {
+						pubURLstr = fmt.Sprintf("http://localhost:%d", cCtx.Int("port"))
+						log.Errorf("Public URL is not configured, using: %s", pubURLstr)
+					}
+					pubURL, err := url.Parse(pubURLstr)
+					if err != nil {
+						return fmt.Errorf("parsing public URL: %w", err)
+					}
+
+					svc, err := storage.New(
+						storage.WithIdentity(id),
+						storage.WithBlobstore(blobStore),
+						storage.WithAllocationDatastore(allocDs),
+						storage.WithClaimDatastore(claimDs),
+						storage.WithPublisherDatastore(publisherDs),
+						storage.WithPublicURL(*pubURL),
+					)
+					if err != nil {
+						return fmt.Errorf("creating service instance: %w", err)
+					}
+					defer svc.Close()
+
+					go func() {
+						time.Sleep(time.Millisecond * 50)
+						if err == nil {
+							printHero(id.DID())
+						}
+					}()
+					err = server.ListenAndServe(fmt.Sprintf(":%d", cCtx.Int("port")), svc)
+					return err
+				},
+			},
+			{
+				Name:    "identity",
+				Aliases: []string{"id"},
+				Usage:   "Identity tools.",
+				Subcommands: []*cli.Command{
+					{
+						Name:    "generate",
+						Aliases: []string{"gen"},
+						Usage:   "Generate a new decentralized identity.",
+						Flags: []cli.Flag{
+							&cli.BoolFlag{
+								Name:  "json",
+								Usage: "output JSON",
+							},
+						},
+						Action: func(cCtx *cli.Context) error {
+							signer, err := ed25519.Generate()
+							if err != nil {
+								return fmt.Errorf("generating ed25519 key: %w", err)
+							}
+							did := signer.DID().String()
+							key, err := ed25519.Format(signer)
+							if err != nil {
+								return fmt.Errorf("formatting ed25519 key: %w", err)
+							}
+							if cCtx.Bool("json") {
+								out, err := json.Marshal(struct {
+									DID string `json:"did"`
+									Key string `json:"key"`
+								}{did, key})
+								if err != nil {
+									return fmt.Errorf("marshaling JSON: %w", err)
+								}
+								fmt.Println(string(out))
+							} else {
+								fmt.Printf("# %s\n", did)
+								fmt.Println(key)
+							}
+							return nil
+						},
+					},
+				},
+			},
+		},
+	}
+
+	if err := app.Run(os.Args); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func printHero(id did.DID) {
+	fmt.Printf(`
+ 00000000                                                   00                  
+00      00    00                                            00                  
+ 000        000000   00000000   00000  0000000    0000000   00000000    0000000 
+    00000     00    00     000  00           00  00     0   00    00         00 
+        000   00    00      00  00     00000000  00         00    00    0000000 
+000     000   00    00     000  00    000    00  000    00  00    00   00    00 
+ 000000000    0000   0000000    00     000000000   000000   00    00   000000000
+
+🔥 Storage Node %s
+🆔 %s
+🚀 Ready!
+`, "v0.0.0", id.String())
+}
+
+func mkdirp(dirpath ...string) (string, error) {
+	dir := path.Join(dirpath...)
+	err := os.MkdirAll(dir, 0777)
+	if err != nil {
+		return "", fmt.Errorf("creating directory: %s: %w", dir, err)
+	}
+	return dir, nil
+}
diff --git a/go.mod b/go.mod
@@ -0,0 +1,108 @@
+module github.com/storacha/storage
+
+go 1.23
+
+toolchain go1.23.2
+
+require (
+	github.com/aws/aws-sdk-go-v2 v1.32.2
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.65.3
+	github.com/ipfs/go-cid v0.4.1
+	github.com/ipfs/go-datastore v0.6.0
+	github.com/ipfs/go-ds-leveldb v0.5.0
+	github.com/ipfs/go-log/v2 v2.5.1
+	github.com/ipld/go-ipld-prime v0.21.1-0.20240917223228-6148356a4c2e
+	github.com/ipni/go-libipni v0.6.13
+	github.com/libp2p/go-libp2p v0.36.5
+	github.com/multiformats/go-multiaddr v0.13.0
+	github.com/multiformats/go-multibase v0.2.0
+	github.com/multiformats/go-multicodec v0.9.0
+	github.com/multiformats/go-multihash v0.2.3
+	github.com/multiformats/go-varint v0.0.7
+	github.com/storacha/go-ucanto v0.1.1-0.20241018155815-175193fb3b33
+	github.com/storacha/ipni-publisher v0.0.0-20241018055706-032286a2dc3f
+	github.com/stretchr/testify v1.9.0
+	github.com/urfave/cli/v2 v2.27.4
+)
+
+require (
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.21 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.2 // indirect
+	github.com/aws/smithy-go v1.22.0 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/gobwas/glob v0.2.3 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/hashicorp/golang-lru v1.0.2 // indirect
+	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
+	github.com/ipfs/bbloom v0.0.4 // indirect
+	github.com/ipfs/go-block-format v0.2.0 // indirect
+	github.com/ipfs/go-blockservice v0.5.2 // indirect
+	github.com/ipfs/go-ipfs-blockstore v1.3.1 // indirect
+	github.com/ipfs/go-ipfs-ds-help v1.1.1 // indirect
+	github.com/ipfs/go-ipfs-exchange-interface v0.2.1 // indirect
+	github.com/ipfs/go-ipfs-util v0.0.3 // indirect
+	github.com/ipfs/go-ipld-cbor v0.1.0 // indirect
+	github.com/ipfs/go-ipld-format v0.6.0 // indirect
+	github.com/ipfs/go-ipld-legacy v0.2.1 // indirect
+	github.com/ipfs/go-log v1.0.5 // indirect
+	github.com/ipfs/go-merkledag v0.11.0 // indirect
+	github.com/ipfs/go-metrics-interface v0.0.1 // indirect
+	github.com/ipfs/go-verifcid v0.0.3 // indirect
+	github.com/ipld/go-car v0.6.2 // indirect
+	github.com/ipld/go-codec-dagpb v1.6.0 // indirect
+	github.com/jbenet/goprocess v0.1.4 // indirect
+	github.com/klauspost/compress v1.17.11 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.8 // indirect
+	github.com/libp2p/go-buffer-pool v0.1.0 // indirect
+	github.com/libp2p/go-libp2p-pubsub v0.12.0 // indirect
+	github.com/libp2p/go-msgio v0.3.0 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/minio/sha256-simd v1.0.1 // indirect
+	github.com/mr-tron/base58 v1.2.0 // indirect
+	github.com/multiformats/go-base32 v0.1.0 // indirect
+	github.com/multiformats/go-base36 v0.2.0 // indirect
+	github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect
+	github.com/multiformats/go-multistream v0.5.0 // indirect
+	github.com/onsi/ginkgo/v2 v2.20.2 // indirect
+	github.com/onsi/gomega v1.34.1 // indirect
+	github.com/opentracing/opentracing-go v1.2.0 // indirect
+	github.com/pion/ice/v2 v2.3.36 // indirect
+	github.com/pion/interceptor v0.1.37 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/polydawn/refmt v0.89.1-0.20231129105047-37766d95467a // indirect
+	github.com/prometheus/common v0.60.0 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/spaolacci/murmur3 v1.1.0 // indirect
+	github.com/syndtr/goleveldb v1.0.0 // indirect
+	github.com/ucan-wg/go-ucan v0.0.0-20240916120445-37f52863156c // indirect
+	github.com/whyrusleeping/cbor-gen v0.1.2 // indirect
+	github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
+	go.opentelemetry.io/otel v1.30.0 // indirect
+	go.opentelemetry.io/otel/metric v1.30.0 // indirect
+	go.opentelemetry.io/otel/trace v1.30.0 // indirect
+	go.uber.org/atomic v1.11.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	go.uber.org/zap v1.27.0 // indirect
+	golang.org/x/crypto v0.27.0 // indirect
+	golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect
+	golang.org/x/sys v0.25.0 // indirect
+	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	lukechampine.com/blake3 v1.3.0 // indirect
+)