Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: map blobstore implements filesystemer #14

Merged
merged 6 commits into from
Oct 28, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ require (
github.com/multiformats/go-multihash v0.2.3
github.com/storacha/go-capabilities v0.0.0-20241021134022-7144600f5aeb
github.com/storacha/go-metadata v0.0.0-20241021141939-f94d93dcda78
github.com/storacha/go-ucanto v0.1.1-0.20241022122125-fc561a4d642c
github.com/storacha/go-ucanto v0.1.1-0.20241028093931-42d9e1db4bfb
github.com/storacha/ipni-publisher v0.0.0-20241018055706-032286a2dc3f
github.com/stretchr/testify v1.9.0
github.com/urfave/cli/v2 v2.27.4
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -540,8 +540,8 @@ github.com/storacha/go-capabilities v0.0.0-20241021134022-7144600f5aeb h1:A9/44D
github.com/storacha/go-capabilities v0.0.0-20241021134022-7144600f5aeb/go.mod h1:1x2qMcvOUvF6prSiWXsEeEfV3OcX/L2fk6iNgPMNfZU=
github.com/storacha/go-metadata v0.0.0-20241021141939-f94d93dcda78 h1:NAti9hMLMo8F0Iyz5ldS41CY6MyukRH0OrTPV4u2340=
github.com/storacha/go-metadata v0.0.0-20241021141939-f94d93dcda78/go.mod h1:DcwwQnyFuTk531cKD9sUkQg/gzpwKaIqH9I7oZYyeRc=
github.com/storacha/go-ucanto v0.1.1-0.20241022122125-fc561a4d642c h1:Z8MZyBenGPhGE+w43UBhPq5uF2cri5o9zzyTmWM337Q=
github.com/storacha/go-ucanto v0.1.1-0.20241022122125-fc561a4d642c/go.mod h1:vzifzvheQKs507Yr7qj2x4MYNrdBwjdKXtHsD3xpDm4=
github.com/storacha/go-ucanto v0.1.1-0.20241028093931-42d9e1db4bfb h1:bjubXTmLyLl7eap6HhIoFmwIH6H4CCtnJlcE4hbc2Qo=
github.com/storacha/go-ucanto v0.1.1-0.20241028093931-42d9e1db4bfb/go.mod h1:7ba9jAgqmwlF/JfyFUQcGV07uiYNlmJNu8qH4hHtrJk=
github.com/storacha/ipni-publisher v0.0.0-20241018055706-032286a2dc3f h1:62fTASO3wRPCWCkl6we2DftsFy/DfbmVpwJyqK7gmUc=
github.com/storacha/ipni-publisher v0.0.0-20241018055706-032286a2dc3f/go.mod h1:fEuGSF5WMaOSAyDQCYAvow6Y+YKzpXczEk3A+H+s1fQ=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
Expand Down
14 changes: 1 addition & 13 deletions pkg/service/blobs/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,19 +36,7 @@ func NewBlobGetHandler(blobs blobstore.Blobstore) func(http.ResponseWriter, *htt
if fsblobs, ok := blobs.(blobstore.FileSystemer); ok {
serveHTTP := http.FileServer(fsblobs.FileSystem()).ServeHTTP
return func(w http.ResponseWriter, r *http.Request) {
_, bytes, err := multibase.Decode(r.PathValue("blob"))
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this might be simpler with ServeFile instead of FileServer.

also go errata: if you just want to use system utils, you may find ServeFileFS and FileServerFS easer cause you can just use some system utils (if this is actually helpful)

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah yes - looks more appropriate. I didn't realize it existed! If I have a moment I'll look into replacing it.

if err != nil {
http.Error(w, fmt.Sprintf("decoding multibase encoded digest: %s", err), http.StatusBadRequest)
return
}

digest, err := multihash.Cast(bytes)
if err != nil {
http.Error(w, fmt.Sprintf("invalid multihash digest: %s", err), http.StatusBadRequest)
return
}

r.URL.Path = fsblobs.EncodePath(digest)
r.URL.Path = r.URL.Path[len("/blob"):]
serveHTTP(w, r)
}
}
Expand Down
1 change: 1 addition & 0 deletions pkg/service/blobs/server_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,7 @@ func requireRetrievableBlob(t *testing.T, endpoint url.URL, digest multihash.Mul
res, err := http.Get(bloburl.String())
require.NoError(t, err)

require.Equal(t, http.StatusOK, res.StatusCode)
body, err := io.ReadAll(res.Body)
require.NoError(t, err)
require.Equal(t, data, body)
Expand Down
9 changes: 7 additions & 2 deletions pkg/service/storage/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,12 @@ func (srv *Server) Serve(mux *http.ServeMux) {

func NewHandler(server server.ServerView) func(http.ResponseWriter, *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
res, _ := server.Request(ucanhttp.NewHTTPRequest(r.Body, r.Header))
res, err := server.Request(ucanhttp.NewHTTPRequest(r.Body, r.Header))
if err != nil {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

+1

log.Errorf("handling UCAN request: %w", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}

for key, vals := range res.Headers() {
for _, v := range vals {
Expand All @@ -40,7 +45,7 @@ func NewHandler(server server.ServerView) func(http.ResponseWriter, *http.Reques
w.WriteHeader(res.Status())
}

_, err := io.Copy(w, res.Body())
_, err = io.Copy(w, res.Body())
if err != nil {
log.Errorf("sending UCAN response: %w", err)
}
Expand Down
20 changes: 20 additions & 0 deletions pkg/store/blobstore/blobstore_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"time"

"github.com/multiformats/go-multihash"
"github.com/storacha/storage/pkg/internal/digestutil"
"github.com/storacha/storage/pkg/internal/testutil"
"github.com/storacha/storage/pkg/store"
"github.com/stretchr/testify/require"
Expand Down Expand Up @@ -59,5 +60,24 @@ func TestBlobstore(t *testing.T) {
err := s.Put(context.Background(), digest, uint64(len(data)), bytes.NewBuffer(baddata))
require.Equal(t, ErrDataInconsistent, err)
})

t.Run("filesystemer "+k, func(t *testing.T) {
data := testutil.RandomBytes(10)
digest := testutil.Must(multihash.Sum(data, multihash.SHA2_256, -1))(t)

err := s.Put(context.Background(), digest, uint64(len(data)), bytes.NewBuffer(data))
require.NoError(t, err)

fsr, ok := s.(FileSystemer)
require.True(t, ok)

f, err := fsr.FileSystem().Open(fmt.Sprintf("/%s", digestutil.Format(digest)))
require.NoError(t, err)

b, err := io.ReadAll(f)
require.NoError(t, err)

require.Equal(t, data, b)
})
}
}
25 changes: 20 additions & 5 deletions pkg/store/blobstore/fs.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"os"
"path"

"github.com/multiformats/go-multibase"
"github.com/multiformats/go-multihash"
"github.com/storacha/storage/pkg/internal/digestutil"
"github.com/storacha/storage/pkg/store"
Expand Down Expand Up @@ -71,13 +72,9 @@ type FsBlobstore struct {
tmpdir string
}

func (b *FsBlobstore) EncodePath(digest multihash.Multihash) string {
return encodePath(digest)
}

// FileSystem returns a filesystem interface for reading blobs.
func (b *FsBlobstore) FileSystem() http.FileSystem {
return http.Dir(b.rootdir)
return &fsDir{http.Dir(b.rootdir)}
}

func (b *FsBlobstore) Get(ctx context.Context, digest multihash.Multihash, opts ...GetOption) (Object, error) {
Expand Down Expand Up @@ -194,3 +191,21 @@ func NewFsBlobstore(rootdir string, tmpdir string) (*FsBlobstore, error) {
}
return &FsBlobstore{rootdir, tmpdir}, nil
}

type fsDir struct {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

again you may want to look at fs.FS

fs http.FileSystem
}

var _ http.FileSystem = (*fsDir)(nil)

func (d *fsDir) Open(path string) (http.File, error) {
_, bytes, err := multibase.Decode(path[1:])
if err != nil {
return nil, fs.ErrNotExist
}
digest, err := multihash.Cast(bytes)
if err != nil {
return nil, fs.ErrNotExist
}
return d.fs.Open(encodePath(digest))
}
2 changes: 0 additions & 2 deletions pkg/store/blobstore/interface.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,4 @@ type Blobstore interface {
type FileSystemer interface {
// FileSystem returns a filesystem interface for reading blobs.
FileSystem() http.FileSystem
// EncodePath converts a digest to a filesystem path.
EncodePath(digest multihash.Multihash) string
}
59 changes: 59 additions & 0 deletions pkg/store/blobstore/map.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,10 @@ import (
"crypto/sha256"
"fmt"
"io"
"io/fs"
"net/http"
"os"
"time"

"github.com/multiformats/go-multihash"
"github.com/storacha/storage/pkg/internal/digestutil"
Expand Down Expand Up @@ -86,10 +90,65 @@ func (mb *MapBlobstore) Put(ctx context.Context, digest multihash.Multihash, siz
return nil
}

func (mb *MapBlobstore) FileSystem() http.FileSystem {
return &mapDir{mb.data}
}

var _ Blobstore = (*MapBlobstore)(nil)

// NewMapBlobstore creates a [Blobstore] backed by an in-memory map.
func NewMapBlobstore() *MapBlobstore {
data := map[string][]byte{}
return &MapBlobstore{data}
}

type mapDir struct {
data map[string][]byte
}

var _ http.FileSystem = (*mapDir)(nil)

func (d *mapDir) Open(path string) (http.File, error) {
name := path[1:]
data, ok := d.data[name]
if !ok {
return nil, fs.ErrNotExist
}
return &mapFile{
Reader: bytes.NewReader(data),
info: mapFileInfo{name, int64(len(data))},
}, nil
}

type mapFile struct {
*bytes.Reader
info fs.FileInfo
}

func (m *mapFile) Close() error {
return nil
}

func (m *mapFile) Readdir(count int) ([]fs.FileInfo, error) {
panic("unimplemented") // should not be called - there are no directories
}

func (m *mapFile) Stat() (fs.FileInfo, error) {
return m.info, nil
}

var _ http.File = (*mapFile)(nil)

type mapFileInfo struct {
name string
size int64
}

func (mfi mapFileInfo) Name() string { return mfi.name }
func (mfi mapFileInfo) Size() int64 { return mfi.size }
func (mfi mapFileInfo) Mode() os.FileMode { return 0444 }
func (mfi mapFileInfo) ModTime() time.Time { return time.Time{} }
func (mfi mapFileInfo) IsDir() bool { return false }
func (mfi mapFileInfo) Sys() interface{} { return nil }

var _ fs.FileInfo = (*mapFileInfo)(nil)
Loading