-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
This PR adds an initial implementation for a storage node. It implements the "storage node" in the following sequence diagram: ```mermaid sequenceDiagram participant alice participant service participant storagenode alice->>service: blob/add with: alicespace nb: blob=car size=3 service->>storagenode: blob/allocate with: storagenode nb: space=alicespace blob=car size=3 storagenode->>service: receipt ran: blob/allocate out: url service->>alice: receipt ran: blob/add fx: blob/allocate, http/put, blob/accept <br> (included in response: receipt ran: blob/allocate out: url) alice->>storagenode: HTTP PUT alice->>service: receipt ran: http/put service->>storagenode: blob/accept with: storagenode nb: space=alicespace blob=car storagenode->>service: receipt ran: blob/accept out: locationcommitment alice->>service: poll to get receipt for blob/accept ``` * Filesystem blob storage and HTTP read/write API (done ✅) * Delegation storage for location commitments and HTTP retrieval API (done ✅) * Allocation tracking store to keep track of pending uploads (done ✅) * Signed URL support (done ✅) * Ucanto endpoint for `blob/allocate` (done ✅) * Ucanto endpoint for `blob/accept` (done ✅) * IPNI publisher for location commitments (done ✅) * Build out the CLI (done ✅) Next PR: * Indexing service caching for location commitments Outside of planned scope here be needed eventually: * Receipt storage and retrieval (for future use when submitting logs for verification)
- Loading branch information
Showing
64 changed files
with
4,895 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,18 @@ | ||
name: Go Checks | ||
|
||
on: | ||
pull_request: | ||
push: | ||
branches: ["main"] | ||
workflow_dispatch: | ||
|
||
permissions: | ||
contents: read | ||
|
||
concurrency: | ||
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} | ||
cancel-in-progress: true | ||
|
||
jobs: | ||
go-check: | ||
uses: ipdxco/unified-github-workflows/.github/workflows/[email protected] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
{ | ||
"skip32bit": true | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,20 @@ | ||
name: Go Test | ||
|
||
on: | ||
pull_request: | ||
push: | ||
branches: ["main"] | ||
workflow_dispatch: | ||
|
||
permissions: | ||
contents: read | ||
|
||
concurrency: | ||
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} | ||
cancel-in-progress: true | ||
|
||
jobs: | ||
go-test: | ||
uses: ipdxco/unified-github-workflows/.github/workflows/[email protected] | ||
with: | ||
go-versions: '["this"]' |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,223 @@ | ||
package main | ||
|
||
import ( | ||
"encoding/json" | ||
"fmt" | ||
"net/url" | ||
"os" | ||
"path" | ||
"time" | ||
|
||
leveldb "github.com/ipfs/go-ds-leveldb" | ||
logging "github.com/ipfs/go-log/v2" | ||
"github.com/storacha/go-ucanto/did" | ||
"github.com/storacha/go-ucanto/principal" | ||
ed25519 "github.com/storacha/go-ucanto/principal/ed25519/signer" | ||
"github.com/storacha/storage/pkg/server" | ||
"github.com/storacha/storage/pkg/service/storage" | ||
"github.com/storacha/storage/pkg/store/blobstore" | ||
"github.com/urfave/cli/v2" | ||
) | ||
|
||
var log = logging.Logger("cmd") | ||
|
||
func main() { | ||
app := &cli.App{ | ||
Name: "storage", | ||
Usage: "Manage running a storage node.", | ||
Commands: []*cli.Command{ | ||
{ | ||
Name: "start", | ||
Usage: "Start the storage node daemon.", | ||
Flags: []cli.Flag{ | ||
&cli.IntFlag{ | ||
Name: "port", | ||
Aliases: []string{"p"}, | ||
Value: 3000, | ||
Usage: "Port to bind the server to.", | ||
}, | ||
&cli.StringFlag{ | ||
Name: "private-key", | ||
Aliases: []string{"s"}, | ||
Usage: "Multibase base64 encoded private key identity for the node.", | ||
}, | ||
&cli.StringFlag{ | ||
Name: "data-dir", | ||
Aliases: []string{"d"}, | ||
Usage: "Root directory to store data in.", | ||
}, | ||
&cli.StringFlag{ | ||
Name: "public-url", | ||
Aliases: []string{"u"}, | ||
Usage: "URL the node is publically accessible at.", | ||
}, | ||
}, | ||
Action: func(cCtx *cli.Context) error { | ||
var id principal.Signer | ||
var err error | ||
if cCtx.String("private-key") == "" { | ||
id, err = ed25519.Generate() | ||
if err != nil { | ||
return fmt.Errorf("generating ed25519 key: %w", err) | ||
} | ||
log.Errorf("Server ID is not configured, generated one for you: %s", id.DID().String()) | ||
} else { | ||
id, err = ed25519.Parse(cCtx.String("private-key")) | ||
if err != nil { | ||
return fmt.Errorf("parsing private key: %w", err) | ||
} | ||
} | ||
|
||
homeDir, err := os.UserHomeDir() | ||
if err != nil { | ||
return fmt.Errorf("getting user home directory: %w", err) | ||
} | ||
|
||
dataDir := cCtx.String("data-dir") | ||
if dataDir == "" { | ||
dir, err := mkdirp(homeDir, ".storacha") | ||
if err != nil { | ||
return err | ||
} | ||
log.Errorf("Data directory is not configured, using default: %s", dir) | ||
dataDir = dir | ||
} | ||
|
||
blobStore, err := blobstore.NewFsBlobstore(path.Join(dataDir, "blobs")) | ||
if err != nil { | ||
return fmt.Errorf("creating blob storage: %w", err) | ||
} | ||
|
||
allocsDir, err := mkdirp(dataDir, "allocation") | ||
if err != nil { | ||
return err | ||
} | ||
allocDs, err := leveldb.NewDatastore(allocsDir, nil) | ||
if err != nil { | ||
return err | ||
} | ||
claimsDir, err := mkdirp(dataDir, "claim") | ||
if err != nil { | ||
return err | ||
} | ||
claimDs, err := leveldb.NewDatastore(claimsDir, nil) | ||
if err != nil { | ||
return err | ||
} | ||
publisherDir, err := mkdirp(dataDir, "publisher") | ||
if err != nil { | ||
return err | ||
} | ||
publisherDs, err := leveldb.NewDatastore(publisherDir, nil) | ||
if err != nil { | ||
return err | ||
} | ||
|
||
pubURLstr := cCtx.String("public-url") | ||
if pubURLstr == "" { | ||
pubURLstr = fmt.Sprintf("http://localhost:%d", cCtx.Int("port")) | ||
log.Errorf("Public URL is not configured, using: %s", pubURLstr) | ||
} | ||
pubURL, err := url.Parse(pubURLstr) | ||
if err != nil { | ||
return fmt.Errorf("parsing public URL: %w", err) | ||
} | ||
|
||
svc, err := storage.New( | ||
storage.WithIdentity(id), | ||
storage.WithBlobstore(blobStore), | ||
storage.WithAllocationDatastore(allocDs), | ||
storage.WithClaimDatastore(claimDs), | ||
storage.WithPublisherDatastore(publisherDs), | ||
storage.WithPublicURL(*pubURL), | ||
) | ||
if err != nil { | ||
return fmt.Errorf("creating service instance: %w", err) | ||
} | ||
defer svc.Close() | ||
|
||
go func() { | ||
time.Sleep(time.Millisecond * 50) | ||
if err == nil { | ||
printHero(id.DID()) | ||
} | ||
}() | ||
err = server.ListenAndServe(fmt.Sprintf(":%d", cCtx.Int("port")), svc) | ||
return err | ||
}, | ||
}, | ||
{ | ||
Name: "identity", | ||
Aliases: []string{"id"}, | ||
Usage: "Identity tools.", | ||
Subcommands: []*cli.Command{ | ||
{ | ||
Name: "generate", | ||
Aliases: []string{"gen"}, | ||
Usage: "Generate a new decentralized identity.", | ||
Flags: []cli.Flag{ | ||
&cli.BoolFlag{ | ||
Name: "json", | ||
Usage: "output JSON", | ||
}, | ||
}, | ||
Action: func(cCtx *cli.Context) error { | ||
signer, err := ed25519.Generate() | ||
if err != nil { | ||
return fmt.Errorf("generating ed25519 key: %w", err) | ||
} | ||
did := signer.DID().String() | ||
key, err := ed25519.Format(signer) | ||
if err != nil { | ||
return fmt.Errorf("formatting ed25519 key: %w", err) | ||
} | ||
if cCtx.Bool("json") { | ||
out, err := json.Marshal(struct { | ||
DID string `json:"did"` | ||
Key string `json:"key"` | ||
}{did, key}) | ||
if err != nil { | ||
return fmt.Errorf("marshaling JSON: %w", err) | ||
} | ||
fmt.Println(string(out)) | ||
} else { | ||
fmt.Printf("# %s\n", did) | ||
fmt.Println(key) | ||
} | ||
return nil | ||
}, | ||
}, | ||
}, | ||
}, | ||
}, | ||
} | ||
|
||
if err := app.Run(os.Args); err != nil { | ||
log.Fatal(err) | ||
} | ||
} | ||
|
||
func printHero(id did.DID) { | ||
fmt.Printf(` | ||
00000000 00 | ||
00 00 00 00 | ||
000 000000 00000000 00000 0000000 0000000 00000000 0000000 | ||
00000 00 00 000 00 00 00 0 00 00 00 | ||
000 00 00 00 00 00000000 00 00 00 0000000 | ||
000 000 00 00 000 00 000 00 000 00 00 00 00 00 | ||
000000000 0000 0000000 00 000000000 000000 00 00 000000000 | ||
🔥 Storage Node %s | ||
🆔 %s | ||
🚀 Ready! | ||
`, "v0.0.0", id.String()) | ||
} | ||
|
||
func mkdirp(dirpath ...string) (string, error) { | ||
dir := path.Join(dirpath...) | ||
err := os.MkdirAll(dir, 0777) | ||
if err != nil { | ||
return "", fmt.Errorf("creating directory: %s: %w", dir, err) | ||
} | ||
return dir, nil | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,108 @@ | ||
module github.com/storacha/storage | ||
|
||
go 1.23 | ||
|
||
toolchain go1.23.2 | ||
|
||
require ( | ||
github.com/aws/aws-sdk-go-v2 v1.32.2 | ||
github.com/aws/aws-sdk-go-v2/service/s3 v1.65.3 | ||
github.com/ipfs/go-cid v0.4.1 | ||
github.com/ipfs/go-datastore v0.6.0 | ||
github.com/ipfs/go-ds-leveldb v0.5.0 | ||
github.com/ipfs/go-log/v2 v2.5.1 | ||
github.com/ipld/go-ipld-prime v0.21.1-0.20240917223228-6148356a4c2e | ||
github.com/ipni/go-libipni v0.6.13 | ||
github.com/libp2p/go-libp2p v0.36.5 | ||
github.com/multiformats/go-multiaddr v0.13.0 | ||
github.com/multiformats/go-multibase v0.2.0 | ||
github.com/multiformats/go-multicodec v0.9.0 | ||
github.com/multiformats/go-multihash v0.2.3 | ||
github.com/multiformats/go-varint v0.0.7 | ||
github.com/storacha/go-ucanto v0.1.1-0.20241018155815-175193fb3b33 | ||
github.com/storacha/ipni-publisher v0.0.0-20241018055706-032286a2dc3f | ||
github.com/stretchr/testify v1.9.0 | ||
github.com/urfave/cli/v2 v2.27.4 | ||
) | ||
|
||
require ( | ||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 // indirect | ||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21 // indirect | ||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21 // indirect | ||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.21 // indirect | ||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 // indirect | ||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.2 // indirect | ||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2 // indirect | ||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.2 // indirect | ||
github.com/aws/smithy-go v1.22.0 // indirect | ||
github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect | ||
github.com/davecgh/go-spew v1.1.1 // indirect | ||
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect | ||
github.com/go-logr/logr v1.4.2 // indirect | ||
github.com/go-logr/stdr v1.2.2 // indirect | ||
github.com/gobwas/glob v0.2.3 // indirect | ||
github.com/gogo/protobuf v1.3.2 // indirect | ||
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db // indirect | ||
github.com/google/uuid v1.6.0 // indirect | ||
github.com/hashicorp/errwrap v1.1.0 // indirect | ||
github.com/hashicorp/go-multierror v1.1.1 // indirect | ||
github.com/hashicorp/golang-lru v1.0.2 // indirect | ||
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect | ||
github.com/ipfs/bbloom v0.0.4 // indirect | ||
github.com/ipfs/go-block-format v0.2.0 // indirect | ||
github.com/ipfs/go-blockservice v0.5.2 // indirect | ||
github.com/ipfs/go-ipfs-blockstore v1.3.1 // indirect | ||
github.com/ipfs/go-ipfs-ds-help v1.1.1 // indirect | ||
github.com/ipfs/go-ipfs-exchange-interface v0.2.1 // indirect | ||
github.com/ipfs/go-ipfs-util v0.0.3 // indirect | ||
github.com/ipfs/go-ipld-cbor v0.1.0 // indirect | ||
github.com/ipfs/go-ipld-format v0.6.0 // indirect | ||
github.com/ipfs/go-ipld-legacy v0.2.1 // indirect | ||
github.com/ipfs/go-log v1.0.5 // indirect | ||
github.com/ipfs/go-merkledag v0.11.0 // indirect | ||
github.com/ipfs/go-metrics-interface v0.0.1 // indirect | ||
github.com/ipfs/go-verifcid v0.0.3 // indirect | ||
github.com/ipld/go-car v0.6.2 // indirect | ||
github.com/ipld/go-codec-dagpb v1.6.0 // indirect | ||
github.com/jbenet/goprocess v0.1.4 // indirect | ||
github.com/klauspost/compress v1.17.11 // indirect | ||
github.com/klauspost/cpuid/v2 v2.2.8 // indirect | ||
github.com/libp2p/go-buffer-pool v0.1.0 // indirect | ||
github.com/libp2p/go-libp2p-pubsub v0.12.0 // indirect | ||
github.com/libp2p/go-msgio v0.3.0 // indirect | ||
github.com/mattn/go-isatty v0.0.20 // indirect | ||
github.com/minio/sha256-simd v1.0.1 // indirect | ||
github.com/mr-tron/base58 v1.2.0 // indirect | ||
github.com/multiformats/go-base32 v0.1.0 // indirect | ||
github.com/multiformats/go-base36 v0.2.0 // indirect | ||
github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect | ||
github.com/multiformats/go-multistream v0.5.0 // indirect | ||
github.com/onsi/ginkgo/v2 v2.20.2 // indirect | ||
github.com/onsi/gomega v1.34.1 // indirect | ||
github.com/opentracing/opentracing-go v1.2.0 // indirect | ||
github.com/pion/ice/v2 v2.3.36 // indirect | ||
github.com/pion/interceptor v0.1.37 // indirect | ||
github.com/pkg/errors v0.9.1 // indirect | ||
github.com/pmezard/go-difflib v1.0.0 // indirect | ||
github.com/polydawn/refmt v0.89.1-0.20231129105047-37766d95467a // indirect | ||
github.com/prometheus/common v0.60.0 // indirect | ||
github.com/russross/blackfriday/v2 v2.1.0 // indirect | ||
github.com/spaolacci/murmur3 v1.1.0 // indirect | ||
github.com/syndtr/goleveldb v1.0.0 // indirect | ||
github.com/ucan-wg/go-ucan v0.0.0-20240916120445-37f52863156c // indirect | ||
github.com/whyrusleeping/cbor-gen v0.1.2 // indirect | ||
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect | ||
go.opentelemetry.io/otel v1.30.0 // indirect | ||
go.opentelemetry.io/otel/metric v1.30.0 // indirect | ||
go.opentelemetry.io/otel/trace v1.30.0 // indirect | ||
go.uber.org/atomic v1.11.0 // indirect | ||
go.uber.org/multierr v1.11.0 // indirect | ||
go.uber.org/zap v1.27.0 // indirect | ||
golang.org/x/crypto v0.27.0 // indirect | ||
golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect | ||
golang.org/x/sys v0.25.0 // indirect | ||
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect | ||
google.golang.org/protobuf v1.34.2 // indirect | ||
gopkg.in/yaml.v3 v3.0.1 // indirect | ||
lukechampine.com/blake3 v1.3.0 // indirect | ||
) |
Oops, something went wrong.