Merge branch 'main' of github.com:storacha/storage

cmd/client.go

@@ -0,0 +1,195 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+
+	"github.com/ipfs/go-cid"
+	cidlink "github.com/ipld/go-ipld-prime/linking/cid"
+	"github.com/storacha/go-piece/pkg/piece"
+	"github.com/storacha/go-ucanto/core/delegation"
+	"github.com/storacha/go-ucanto/core/ipld/hash/sha256"
+	"github.com/storacha/go-ucanto/did"
+	ed25519 "github.com/storacha/go-ucanto/principal/ed25519/signer"
+	"github.com/storacha/storage/pkg/client"
+	"github.com/urfave/cli/v2"
+)
+
+var ErrMustBePieceLinkOrHaveSize = errors.New("passing pieceCID v1 reqiures a size to be present")
+var clientCmd = &cli.Command{
+	Name:    "client",
+	Aliases: []string{"c"},
+	Usage:   "test a running storage node as a client",
+	Subcommands: []*cli.Command{
+		{
+			Name:    "upload",
+			Aliases: []string{"ba"},
+			Usage:   "invoke a blob allocation",
+			Flags: append([]cli.Flag{
+				&cli.StringFlag{
+					Name:     "space-did",
+					Aliases:  []string{"sd"},
+					Usage:    "did for the space to use",
+					EnvVars:  []string{"STORAGE_CLIENT_SPACE_DID"},
+					Required: true,
+				},
+				&cli.StringFlag{
+					Name:     "blob",
+					Aliases:  []string{"b"},
+					Usage:    "blob to upload",
+					EnvVars:  []string{"STORAGE_CLIENT_BLOB_FILE"},
+					Required: true,
+				},
+			}, ClientSetupFlags...),
+			Action: func(cCtx *cli.Context) error {
+				client, err := getClient(cCtx)
+				if err != nil {
+					return err
+				}
+				spaceDid, err := did.Parse(cCtx.String("space-did"))
+				if err != nil {
+					return fmt.Errorf("parsing space did: %w", err)
+				}
+				blobFile, err := os.Open(cCtx.String("blob"))
+				if err != nil {
+					return fmt.Errorf("opening blob file: %w", err)
+				}
+				blobData, err := io.ReadAll(blobFile)
+				if err != nil {
+					return fmt.Errorf("reading blob file: %w", err)
+				}
+				digest, err := sha256.Hasher.Sum(blobData)
+				if err != nil {
+					return fmt.Errorf("calculating blob digest: %w", err)
+				}
+				address, err := client.BlobAllocate(spaceDid, digest.Bytes(), uint64(len(blobData)), cidlink.Link{Cid: cid.NewCidV1(cid.Raw, digest.Bytes())})
+				if err != nil {
+					return fmt.Errorf("invocing blob allocation: %w", err)
+				}
+				if address != nil {
+					fmt.Printf("now uploading to: %s\n", address.URL.String())
+
+					req, err := http.NewRequest(http.MethodPut, address.URL.String(), bytes.NewReader(blobData))
+					if err != nil {
+						return fmt.Errorf("uploading blob: %w", err)
+					}
+					req.Header = address.Headers
+					res, err := http.DefaultClient.Do(req)
+					if err != nil {
+						return fmt.Errorf("sending blob: %w", err)
+					}
+					if res.StatusCode >= 300 || res.StatusCode < 200 {
+						resData, err := io.ReadAll(res.Body)
+						if err != nil {
+							return fmt.Errorf("reading response body: %w", err)
+						}
+						return fmt.Errorf("unsuccessful put, status: %s, message: %s", res.Status, string(resData))
+					}
+				}
+				blobResult, err := client.BlobAccept(spaceDid, digest.Bytes(), uint64(len(blobData)), cidlink.Link{Cid: cid.NewCidV1(cid.Raw, digest.Bytes())})
+				if err != nil {
+					return fmt.Errorf("accepting blob: %w", err)
+				}
+				fmt.Printf("uploaded blob available at: %s\n", blobResult.LocationCommitment.Location[0].String())
+				if blobResult.PDPAccept != nil {
+					fmt.Printf("submitted for PDP aggregation: %s\n", blobResult.PDPAccept.Piece.Link().String())
+				}
+				return nil
+			},
+		},
+		{
+			Name:    "pdp-info",
+			Aliases: []string{"pi"},
+			Usage:   "get piece information",
+			Flags: append([]cli.Flag{
+				&cli.StringFlag{
+					Name:     "piece",
+					Aliases:  []string{"pc"},
+					Usage:    "pieceCID to get information on",
+					EnvVars:  []string{"STORAGE_PIECE_CID"},
+					Required: true,
+				},
+				&cli.Uint64Flag{
+					Name:    "size",
+					Aliases: []string{"ps"},
+					Usage:   "optional size if passing a piece cid v1 of data",
+					EnvVars: []string{"STORAGE_PIECE_SIZE"},
+				},
+			}, ClientSetupFlags...),
+			Action: func(cCtx *cli.Context) error {
+				client, err := getClient(cCtx)
+				if err != nil {
+					return err
+				}
+				pieceStr := cCtx.String("piece")
+				pieceCid, err := cid.Decode(pieceStr)
+				if err != nil {
+					return fmt.Errorf("decoding cid: %w", err)
+				}
+				pieceLink, err := piece.FromLink(cidlink.Link{Cid: pieceCid})
+				if err != nil {
+					if !cCtx.IsSet("size") {
+						return ErrMustBePieceLinkOrHaveSize
+					}
+					pieceLink, err = piece.FromV1LinkAndSize(cidlink.Link{Cid: pieceCid}, cCtx.Uint64("size"))
+					if err != nil {
+						return fmt.Errorf("parsing as pieceCID v1: %w", err)
+					}
+				}
+				ok, err := client.PDPInfo(pieceLink)
+				if err != nil {
+					return fmt.Errorf("getting pdp info: %w", err)
+				}
+				asJSON, err := json.MarshalIndent(ok, "", "  ")
+				if err != nil {
+					return fmt.Errorf("marshaling info to json: %w", err)
+				}
+				fmt.Print(string(asJSON))
+				return nil
+			},
+		},
+	},
+}
+
+func getClient(cCtx *cli.Context) (*client.Client, error) {
+	id, err := ed25519.Parse(cCtx.String("client-key"))
+	if err != nil {
+		return nil, fmt.Errorf("parsing private key: %w", err)
+	}
+	proofFile, err := os.Open(cCtx.String("proof"))
+	if err != nil {
+		return nil, fmt.Errorf("opening delegation car file: %w", err)
+	}
+	proofData, err := io.ReadAll(proofFile)
+	if err != nil {
+		return nil, fmt.Errorf("reading delegation car file: %w", err)
+	}
+	proof, err := delegation.Extract(proofData)
+	if err != nil {
+		return nil, fmt.Errorf("extracting storage proof: %w", err)
+	}
+	nodeDID, err := did.Parse(cCtx.String("node-did"))
+	if err != nil {
+		return nil, fmt.Errorf("parsing node did: %w", err)
+	}
+	nodeURL, err := url.Parse(cCtx.String("node-url"))
+	if err != nil {
+		return nil, fmt.Errorf("parsing node url: %w", err)
+	}
+	client, err := client.NewClient(client.Config{
+		ID:             id,
+		StorageNodeID:  nodeDID,
+		StorageNodeURL: *nodeURL,
+		StorageProof:   delegation.FromDelegation(proof),
+	})
+	if err != nil {
+		return nil, fmt.Errorf("setting up client: %w", err)
+	}
+	return client, nil
+}

cmd/delegation.go

@@ -0,0 +1,75 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/storacha/go-capabilities/pkg/blob"
+	"github.com/storacha/go-capabilities/pkg/pdp"
+	"github.com/storacha/go-ucanto/core/delegation"
+	"github.com/storacha/go-ucanto/did"
+	ed25519 "github.com/storacha/go-ucanto/principal/ed25519/signer"
+	"github.com/storacha/go-ucanto/ucan"
+	"github.com/urfave/cli/v2"
+)
+
+var delegationCmd = &cli.Command{
+	Name:    "delegation",
+	Aliases: []string{"dg"},
+	Usage:   "Delegation tools.",
+	Subcommands: []*cli.Command{
+		{
+			Name:    "generate",
+			Aliases: []string{"gen"},
+			Usage:   "Generate a new storage delegation",
+			Flags: []cli.Flag{
+				PrivateKeyFlag,
+				&cli.StringFlag{
+					Name:     "client-did",
+					Aliases:  []string{"d"},
+					Usage:    "did for a client",
+					EnvVars:  []string{"STORAGE_CLIENT_DID"},
+					Required: true,
+				},
+			},
+			Action: func(cCtx *cli.Context) error {
+				id, err := ed25519.Parse(cCtx.String("private-key"))
+				if err != nil {
+					return fmt.Errorf("parsing private key: %w", err)
+				}
+				clientDid, err := did.Parse(cCtx.String("client-did"))
+				if err != nil {
+					return fmt.Errorf("parsing client-did: %w", err)
+				}
+				delegation, err := delegation.Delegate(
+					id,
+					clientDid,
+					[]ucan.Capability[ucan.NoCaveats]{
+						ucan.NewCapability(
+							blob.AllocateAbility,
+							id.DID().String(),
+							ucan.NoCaveats{},
+						),
+						ucan.NewCapability(
+							blob.AcceptAbility,
+							id.DID().String(),
+							ucan.NoCaveats{},
+						),
+						ucan.NewCapability(
+							pdp.InfoAbility,
+							id.DID().String(),
+							ucan.NoCaveats{},
+						),
+					},
+					delegation.WithNoExpiration(),
+				)
+				if err != nil {
+					return fmt.Errorf("generating delegation: %w", err)
+				}
+				_, err = io.Copy(os.Stdout, delegation.Archive())
+				return err
+			},
+		},
+	},
+}

cmd/flags.go

@@ -0,0 +1,73 @@
+package main
+
+import "github.com/urfave/cli/v2"
+
+func RequiredStringFlag(strFlag *cli.StringFlag) *cli.StringFlag {
+	copy := *strFlag
+	copy.Required = true
+	return &copy
+}
+
+func RequiredIntFlag(strFlag *cli.Int64Flag) *cli.Int64Flag {
+	copy := *strFlag
+	copy.Required = true
+	return &copy
+}
+
+var CurioURLFlag = &cli.StringFlag{
+	Name:    "curio-url",
+	Aliases: []string{"c"},
+	Usage:   "URL of a running instance of curio",
+	EnvVars: []string{"STORAGE_CURIO_URL"},
+}
+
+var PrivateKeyFlag = &cli.StringFlag{
+	Name:     "private-key",
+	Aliases:  []string{"s"},
+	Usage:    "Multibase base64 encoded private key identity for the node.",
+	EnvVars:  []string{"STORAGE_PRIVATE_KEY"},
+	Required: true,
+}
+
+var ClientKeyFlag = &cli.StringFlag{
+	Name:     "client-key",
+	Aliases:  []string{"s"},
+	Usage:    "Multibase base64 encoded private key identity for the client",
+	EnvVars:  []string{"STORAGE_CLIENT_KEY"},
+	Required: true,
+}
+var NodeDIDFlag = &cli.StringFlag{
+	Name:     "node-did",
+	Aliases:  []string{"nd"},
+	Usage:    "did for the storage node",
+	EnvVars:  []string{"STORAGE_NODE_DID"},
+	Required: true,
+}
+var NodeURLFlag = &cli.StringFlag{
+	Name:     "node-url",
+	Aliases:  []string{"nu"},
+	Usage:    "url for the storage node",
+	EnvVars:  []string{"STORAGE_NODE_URL"},
+	Required: true,
+}
+var ProofFlag = &cli.StringFlag{
+	Name:     "proof",
+	Aliases:  []string{"p"},
+	Usage:    "CAR file containing a storage proof delegation",
+	EnvVars:  []string{"STORAGE_CLIENT_PROOF"},
+	Required: true,
+}
+
+var ClientSetupFlags = []cli.Flag{
+	ClientKeyFlag,
+	NodeDIDFlag,
+	NodeURLFlag,
+	ProofFlag,
+}
+
+var ProofSetFlag = &cli.Int64Flag{
+	Name:    "pdp-proofset",
+	Aliases: []string{"pdp"},
+	Usage:   "Proofset to use with PDP",
+	EnvVars: []string{"STORAGE_PDP_PROOFSET"},
+}

cmd/identity.go

@@ -0,0 +1,53 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+
+	ed25519 "github.com/storacha/go-ucanto/principal/ed25519/signer"
+	"github.com/urfave/cli/v2"
+)
+
+var identityCmd = &cli.Command{
+	Name:    "identity",
+	Aliases: []string{"id"},
+	Usage:   "Identity tools.",
+	Subcommands: []*cli.Command{
+		{
+			Name:    "generate",
+			Aliases: []string{"gen"},
+			Usage:   "Generate a new decentralized identity.",
+			Flags: []cli.Flag{
+				&cli.BoolFlag{
+					Name:  "json",
+					Usage: "output JSON",
+				},
+			},
+			Action: func(cCtx *cli.Context) error {
+				signer, err := ed25519.Generate()
+				if err != nil {
+					return fmt.Errorf("generating ed25519 key: %w", err)
+				}
+				did := signer.DID().String()
+				key, err := ed25519.Format(signer)
+				if err != nil {
+					return fmt.Errorf("formatting ed25519 key: %w", err)
+				}
+				if cCtx.Bool("json") {
+					out, err := json.Marshal(struct {
+						DID string `json:"did"`
+						Key string `json:"key"`
+					}{did, key})
+					if err != nil {
+						return fmt.Errorf("marshaling JSON: %w", err)
+					}
+					fmt.Println(string(out))
+				} else {
+					fmt.Printf("# %s\n", did)
+					fmt.Println(key)
+				}
+				return nil
+			},
+		},
+	},
+}