Merge pull request #1 from imstevez/master

merge tag 'v0.2.0' of 'ipfs/go-ipfs-files'

file.go

@@ -67,7 +67,9 @@ type DirIterator interface {
 type Directory interface {
 	Node
 
-	// Entries returns a stateful iterator over directory entries.
+	// Entries returns a stateful iterator over directory entries. The iterator
+	// may consume the Directory state so it must be called only once (this
+	// applies specifically to the multipartIterator).
 	//
 	// Example usage:
 	//

filewriter.go

@@ -1,19 +1,28 @@
 package files
 
 import (
+	"errors"
 	"fmt"
 	"io"
 	"os"
 	"path/filepath"
 )
 
+var ErrInvalidDirectoryEntry = errors.New("invalid directory entry name")
+var ErrPathExistsOverwrite = errors.New("path already exists and overwriting is not allowed")
+
 // WriteTo writes the given node to the local filesystem at fpath.
 func WriteTo(nd Node, fpath string) error {
+	if _, err := os.Lstat(fpath); err == nil {
+		return ErrPathExistsOverwrite
+	} else if !os.IsNotExist(err) {
+		return err
+	}
 	switch nd := nd.(type) {
 	case *Symlink:
 		return os.Symlink(nd.Target, fpath)
 	case File:
-		f, err := os.Create(fpath)
+		f, err := createNewFile(fpath)
 		defer f.Close()
 		if err != nil {
 			return err
@@ -31,7 +40,14 @@ func WriteTo(nd Node, fpath string) error {
 
 		entries := nd.Entries()
 		for entries.Next() {
-			child := filepath.Join(fpath, entries.Name())
+			entryName := entries.Name()
+			if entryName == "" ||
+				entryName == "." ||
+				entryName == ".." ||
+				!isValidFilename(entryName) {
+				return ErrInvalidDirectoryEntry
+			}
+			child := filepath.Join(fpath, entryName)
 			if err := WriteTo(entries.Node(), child); err != nil {
 				return err
 			}

filewriter_test.go

@@ -2,10 +2,11 @@ package files
 
 import (
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
 )
 
 func TestWriteTo(t *testing.T) {
@@ -18,26 +19,26 @@ func TestWriteTo(t *testing.T) {
 			"a": NewBytesFile([]byte("foobar")),
 		}),
 	})
-	tmppath, err := ioutil.TempDir("", "files-test")
+	tmppath, err := os.MkdirTemp("", "files-test")
 	if err != nil {
 		t.Fatal(err)
 	}
 	defer os.RemoveAll(tmppath)
 
-	path := tmppath + "/output"
+	path := filepath.Join(tmppath, "output")
 
 	err = WriteTo(sf, path)
 	if err != nil {
 		t.Fatal(err)
 	}
 	expected := map[string]string{
-		".":   "",
-		"1":   "Some text!\n",
-		"2":   "beep",
-		"3":   "",
-		"4":   "boop",
-		"5":   "",
-		"5/a": "foobar",
+		".":                       "",
+		"1":                       "Some text!\n",
+		"2":                       "beep",
+		"3":                       "",
+		"4":                       "boop",
+		"5":                       "",
+		filepath.FromSlash("5/a"): "foobar",
 	}
 	err = filepath.Walk(path, func(cpath string, info os.FileInfo, err error) error {
 		if err != nil {
@@ -58,7 +59,7 @@ func TestWriteTo(t *testing.T) {
 				return fmt.Errorf("expected a directory at %q", rpath)
 			}
 		} else {
-			actual, err := ioutil.ReadFile(cpath)
+			actual, err := os.ReadFile(cpath)
 			if err != nil {
 				return err
 			}
@@ -75,3 +76,25 @@ func TestWriteTo(t *testing.T) {
 		t.Fatalf("failed to find: %#v", expected)
 	}
 }
+
+func TestDontAllowOverwrite(t *testing.T) {
+	tmppath, err := os.MkdirTemp("", "files-test")
+	assert.NoError(t, err)
+	defer os.RemoveAll(tmppath)
+
+	path := filepath.Join(tmppath, "output")
+
+	// Check we can actually write to the output path before trying invalid entries
+	// and leave an existing entry to test overwrite protection.
+	assert.NoError(t, WriteTo(NewMapDirectory(map[string]Node{
+		"exisiting-entry": NewBytesFile(nil),
+	}), path))
+
+	assert.Equal(t, ErrPathExistsOverwrite, WriteTo(NewBytesFile(nil), filepath.Join(path)))
+	assert.Equal(t, ErrPathExistsOverwrite, WriteTo(NewBytesFile(nil), filepath.Join(path, "exisiting-entry")))
+	// The directory in `path` has already been created so this should fail too:
+	assert.Equal(t, ErrPathExistsOverwrite, WriteTo(NewMapDirectory(map[string]Node{
+		"any-name": NewBytesFile(nil),
+	}), filepath.Join(path)))
+	os.RemoveAll(path)
+}

filewriter_unix.go

@@ -0,0 +1,19 @@
+//go:build darwin || linux || netbsd || openbsd || freebsd || dragonfly
+
+package files
+
+import (
+	"os"
+	"strings"
+	"syscall"
+)
+
+var invalidChars = `/` + "\x00"
+
+func isValidFilename(filename string) bool {
+	return !strings.ContainsAny(filename, invalidChars)
+}
+
+func createNewFile(path string) (*os.File, error) {
+	return os.OpenFile(path, os.O_EXCL|os.O_CREATE|os.O_WRONLY|syscall.O_NOFOLLOW, 0666)
+}

filewriter_unix_test.go

@@ -0,0 +1,33 @@
+//go:build darwin || linux || netbsd || openbsd
+
+package files
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestWriteToInvalidPaths(t *testing.T) {
+	tmppath, err := os.MkdirTemp("", "files-test")
+	assert.NoError(t, err)
+	defer os.RemoveAll(tmppath)
+
+	path := filepath.Join(tmppath, "output")
+
+	// Check we can actually write to the output path before trying invalid entries.
+	assert.NoError(t, WriteTo(NewMapDirectory(map[string]Node{
+		"valid-entry": NewBytesFile(nil),
+	}), path))
+	os.RemoveAll(path)
+
+	// Now try all invalid entry names
+	for _, entryName := range []string{"", ".", "..", "/", "", "not/a/base/path"} {
+		assert.Equal(t, ErrInvalidDirectoryEntry, WriteTo(NewMapDirectory(map[string]Node{
+			entryName: NewBytesFile(nil),
+		}), filepath.Join(path)))
+		os.RemoveAll(path)
+	}
+}

filewriter_windows.go

@@ -0,0 +1,45 @@
+//go:build windows
+
+package files
+
+import (
+	"os"
+	"strings"
+)
+
+var invalidChars = `<>:"/\|?*` + "\x00"
+
+var reservedNames = map[string]struct{}{
+	"CON":  {},
+	"PRN":  {},
+	"AUX":  {},
+	"NUL":  {},
+	"COM1": {},
+	"COM2": {},
+	"COM3": {},
+	"COM4": {},
+	"COM5": {},
+	"COM6": {},
+	"COM7": {},
+	"COM8": {},
+	"COM9": {},
+	"LPT1": {},
+	"LPT2": {},
+	"LPT3": {},
+	"LPT4": {},
+	"LPT5": {},
+	"LPT6": {},
+	"LPT7": {},
+	"LPT8": {},
+	"LPT9": {},
+}
+
+func isValidFilename(filename string) bool {
+	_, isReservedName := reservedNames[filename]
+	return !strings.ContainsAny(filename, invalidChars) &&
+		!isReservedName
+}
+
+func createNewFile(path string) (*os.File, error) {
+	return os.OpenFile(path, os.O_EXCL|os.O_CREATE|os.O_WRONLY, 0666)
+}

filewriter_windows_test.go

@@ -0,0 +1,35 @@
+//go:build windows
+
+package files
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestWriteToInvalidPaths(t *testing.T) {
+	tmppath, err := os.MkdirTemp("", "files-test")
+	assert.NoError(t, err)
+	defer os.RemoveAll(tmppath)
+
+	path := filepath.Join(tmppath, "output")
+
+	// Check we can actually write to the output path before trying invalid entries.
+	assert.NoError(t, WriteTo(NewMapDirectory(map[string]Node{
+		"valid-entry": NewBytesFile(nil),
+	}), path))
+	os.RemoveAll(path)
+
+	// Now try all invalid entry names
+	for _, entryName := range []string{"", ".", "..", "/", "", "not/a/base/path",
+		"<", ">", ":", "\"", "\\", "|", "?", "*", "\x00",
+		"CON", "PRN", "AUX", "NUL", "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", "COM9", "LPT1", "LPT2", "LPT3", "LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9"} {
+		assert.Equal(t, ErrInvalidDirectoryEntry, WriteTo(NewMapDirectory(map[string]Node{
+			entryName: NewBytesFile(nil),
+		}), filepath.Join(path)))
+		os.RemoveAll(path)
+	}
+}

filter_test.go

@@ -1,7 +1,6 @@
 package files
 
 import (
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"testing"
@@ -16,6 +15,10 @@ func (m *mockFileInfo) Name() string {
 	return m.name
 }
 
+func (m *mockFileInfo) Sys() interface{} {
+	return nil
+}
+
 var _ os.FileInfo = &mockFileInfo{}
 
 func TestFileFilter(t *testing.T) {
@@ -31,13 +34,13 @@ func TestFileFilter(t *testing.T) {
 	if err == nil {
 		t.Errorf("creating a filter without an invalid ignore file path should have failed")
 	}
-	tmppath, err := ioutil.TempDir("", "filter-test")
+	tmppath, err := os.MkdirTemp("", "filter-test")
 	if err != nil {
 		t.Fatal(err)
 	}
 	ignoreFilePath := filepath.Join(tmppath, "ignoreFile")
 	ignoreFileContents := []byte("a.txt")
-	if err := ioutil.WriteFile(ignoreFilePath, ignoreFileContents, 0666); err != nil {
+	if err := os.WriteFile(ignoreFilePath, ignoreFileContents, 0666); err != nil {
 		t.Fatal(err)
 	}
 	filterWithIgnoreFile, err := NewFilter(ignoreFilePath, nil, false)

go.mod

@@ -2,8 +2,14 @@ module github.com/TRON-US/go-btfs-files
 
 require (
 	github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3
-	github.com/stretchr/testify v1.5.1 // indirect
+	github.com/stretchr/testify v1.7.0
 	golang.org/x/sys v0.0.0-20190302025703-b6889370fb10
 )
 
-go 1.12
+require (
+	github.com/davecgh/go-spew v1.1.0 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
+)
+
+go 1.18

go.sum

@@ -5,11 +5,11 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
-github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 golang.org/x/sys v0.0.0-20190302025703-b6889370fb10 h1:xQJI9OEiErEQ++DoXOHqEpzsGMrAv2Q2jyCpi7DmfpQ=
 golang.org/x/sys v0.0.0-20190302025703-b6889370fb10/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
-gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

helpers_test.go

@@ -1,7 +1,7 @@
 package files
 
 import (
-	"io/ioutil"
+	"io"
 	"testing"
 )
 
@@ -56,7 +56,7 @@ func CheckDir(t *testing.T, dir Directory, expected []Event) {
 				if !ok {
 					t.Fatalf("[%d] expected file to be a normal file: %T", i, it.Node())
 				}
-				out, err := ioutil.ReadAll(mf)
+				out, err := io.ReadAll(mf)
 				if err != nil {
 					t.Errorf("[%d] failed to read file", i)
 					continue

is_hidden.go

@@ -1,4 +1,4 @@
-//+build !windows
+//go:build !windows
 
 package files