Merge pull request #10 from TRON-US/ipfs-0.5.1

Upgrade to upstream ipfs 0.5.1

.travis.yml

@@ -4,7 +4,7 @@ os:
 language: go
 
 go:
-  - 1.11.x
+  - 1.14.x
 
 env:
   global:

filter.go

@@ -0,0 +1,49 @@
+package files
+
+import (
+	"os"
+
+	ignore "github.com/crackcomm/go-gitignore"
+)
+
+// Filter represents a set of rules for determining if a file should be included or excluded.
+// A rule follows the syntax for patterns used in .gitgnore files for specifying untracked files.
+// Examples:
+// foo.txt
+// *.app
+// bar/
+// **/baz
+// fizz/**
+type Filter struct {
+	// IncludeHidden - Include hidden files
+	IncludeHidden bool
+	// Rules - File filter rules
+	Rules *ignore.GitIgnore
+}
+
+// NewFilter creates a new file filter from a .gitignore file and/or a list of ignore rules.
+// An ignoreFile is a path to a file with .gitignore-style patterns to exclude, one per line
+// rules is an array of strings representing .gitignore-style patterns
+// For reference on ignore rule syntax, see https://git-scm.com/docs/gitignore
+func NewFilter(ignoreFile string, rules []string, includeHidden bool) (*Filter, error) {
+	var ignoreRules *ignore.GitIgnore
+	var err error
+	if ignoreFile == "" {
+		ignoreRules, err = ignore.CompileIgnoreLines(rules...)
+	} else {
+		ignoreRules, err = ignore.CompileIgnoreFileAndLines(ignoreFile, rules...)
+	}
+	if err != nil {
+		return nil, err
+	}
+	return &Filter{IncludeHidden: includeHidden, Rules: ignoreRules}, nil
+}
+
+// ShouldExclude takes an os.FileInfo object and applies rules to determine if its target should be excluded.
+func (filter *Filter) ShouldExclude(fileInfo os.FileInfo) (result bool) {
+	path := fileInfo.Name()
+	if !filter.IncludeHidden && isHidden(fileInfo) {
+		return true
+	}
+	return filter.Rules.MatchesPath(path)
+}

filter_test.go

@@ -0,0 +1,50 @@
+package files
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+type mockFileInfo struct {
+	os.FileInfo
+	name string
+}
+
+func (m *mockFileInfo) Name() string {
+	return m.name
+}
+
+var _ os.FileInfo = &mockFileInfo{}
+
+func TestFileFilter(t *testing.T) {
+	includeHidden := true
+	filter, err := NewFilter("", nil, includeHidden)
+	if err != nil {
+		t.Errorf("failed to create filter with empty rules")
+	}
+	if filter.IncludeHidden != includeHidden {
+		t.Errorf("new filter should include hidden files")
+	}
+	_, err = NewFilter("ignoreFileThatDoesNotExist", nil, false)
+	if err == nil {
+		t.Errorf("creating a filter without an invalid ignore file path should have failed")
+	}
+	tmppath, err := ioutil.TempDir("", "filter-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	ignoreFilePath := filepath.Join(tmppath, "ignoreFile")
+	ignoreFileContents := []byte("a.txt")
+	if err := ioutil.WriteFile(ignoreFilePath, ignoreFileContents, 0666); err != nil {
+		t.Fatal(err)
+	}
+	filterWithIgnoreFile, err := NewFilter(ignoreFilePath, nil, false)
+	if err != nil {
+		t.Errorf("failed to create filter with ignore file")
+	}
+	if !filterWithIgnoreFile.ShouldExclude(&mockFileInfo{name: "a.txt"}) {
+		t.Errorf("filter should've excluded expected file from ignoreFile: %s", "a.txt")
+	}
+}

go.mod

@@ -1,5 +1,9 @@
 module github.com/TRON-US/go-btfs-files
 
-require golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae
+require (
+	github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3
+	github.com/stretchr/testify v1.5.1 // indirect
+	golang.org/x/sys v0.0.0-20190302025703-b6889370fb10
+)
 
 go 1.12

go.sum

@@ -1,2 +1,15 @@
-golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae h1:xiXzMMEQdQcric9hXtr1QU98MHunKK7OTtsoU6bYWs4=
-golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 h1:HVTnpeuvF6Owjd5mniCL8DEXo7uYXdQEmOP4FJbV5tg=
+github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE=
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+golang.org/x/sys v0.0.0-20190302025703-b6889370fb10 h1:xQJI9OEiErEQ++DoXOHqEpzsGMrAv2Q2jyCpi7DmfpQ=
+golang.org/x/sys v0.0.0-20190302025703-b6889370fb10/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

serialfile.go

@@ -3,37 +3,46 @@ package files
 import (
 	"errors"
 	"fmt"
-	"io"
 	"io/ioutil"
 	"os"
 	"path/filepath"
-	"strings"
 )
 
 // serialFile implements Node, and reads from a path on the OS filesystem.
-// No more than one file will be opened at a time (directories will advance
-// to the next file when NextFile() is called).
+// No more than one file will be opened at a time.
 type serialFile struct {
-	path              string
-	files             []os.FileInfo
-	stat              os.FileInfo
-	handleHiddenFiles bool
-	size              int64
+	path   string
+	files  []os.FileInfo
+	stat   os.FileInfo
+	filter *Filter
+	size   int64
 }
 
 type serialIterator struct {
-	files             []os.FileInfo
-	handleHiddenFiles bool
-	path              string
+	files  []os.FileInfo
+	path   string
+	filter *Filter
 
 	curName string
 	curFile Node
 
 	err error
 }
 
-// TODO: test/document limitations
-func NewSerialFile(path string, hidden bool, stat os.FileInfo) (Node, error) {
+// NewSerialFile takes a filepath, a bool specifying if hidden files should be included,
+// and a fileInfo and returns a Node representing file, directory or special file.
+func NewSerialFile(path string, includeHidden bool, stat os.FileInfo) (Node, error) {
+	filter, err := NewFilter("", nil, includeHidden)
+	if err != nil {
+		return nil, err
+	}
+	return NewSerialFileWithFilter(path, filter, stat)
+}
+
+// NewSerialFileWith takes a filepath, a filter for determining which files should be
+// operated upon if the filepath is a directory, and a fileInfo and returns a
+// Node representing file, directory or special file.
+func NewSerialFileWithFilter(path string, filter *Filter, stat os.FileInfo) (Node, error) {
 	switch mode := stat.Mode(); {
 	case mode.IsRegular():
 		file, err := os.Open(path)
@@ -43,12 +52,12 @@ func NewSerialFile(path string, hidden bool, stat os.FileInfo) (Node, error) {
 		return NewReaderPathFile(path, file, stat)
 	case mode.IsDir():
 		// for directories, stat all of the contents first, so we know what files to
-		// open when NextFile() is called
+		// open when Entries() is called
 		contents, err := ioutil.ReadDir(path)
 		if err != nil {
 			return nil, err
 		}
-		return &serialFile{path, contents, stat, hidden, 0}, nil
+		return &serialFile{path, contents, stat, filter, 0}, nil
 	case mode&os.ModeSymlink != 0:
 		target, err := os.Readlink(path)
 		if err != nil {
@@ -76,7 +85,7 @@ func (it *serialIterator) Next() bool {
 
 	stat := it.files[0]
 	it.files = it.files[1:]
-	for !it.handleHiddenFiles && isHidden(stat) {
+	for it.filter.ShouldExclude(stat) {
 		if len(it.files) == 0 {
 			return false
 		}
@@ -91,7 +100,7 @@ func (it *serialIterator) Next() bool {
 	// recursively call the constructor on the next file
 	// if it's a regular file, we will open it as a ReaderFile
 	// if it's a directory, files in it will be opened serially
-	sf, err := NewSerialFile(filePath, it.handleHiddenFiles, stat)
+	sf, err := NewSerialFileWithFilter(filePath, it.filter, stat)
 	if err != nil {
 		it.err = err
 		return false
@@ -111,9 +120,9 @@ func (it *serialIterator) BreadthFirstTraversal() {
 
 func (f *serialFile) Entries() DirIterator {
 	return &serialIterator{
-		path:              f.path,
-		files:             f.files,
-		handleHiddenFiles: f.handleHiddenFiles,
+		path:   f.path,
+		files:  f.files,
+		filter: f.filter,
 	}
 }
 
@@ -125,38 +134,6 @@ func IsSerialFileDirectory(d Directory) bool {
 	}
 }
 
-func (f *serialFile) NextFile() (string, Node, error) {
-	// if there aren't any files left in the root directory, we're done
-	if len(f.files) == 0 {
-		return "", nil, io.EOF
-	}
-
-	stat := f.files[0]
-	f.files = f.files[1:]
-
-	for !f.handleHiddenFiles && strings.HasPrefix(stat.Name(), ".") {
-		if len(f.files) == 0 {
-			return "", nil, io.EOF
-		}
-
-		stat = f.files[0]
-		f.files = f.files[1:]
-	}
-
-	// open the next file
-	filePath := filepath.ToSlash(filepath.Join(f.path, stat.Name()))
-
-	// recursively call the constructor on the next file
-	// if it's a regular file, we will open it as a ReaderFile
-	// if it's a directory, files in it will be opened serially
-	sf, err := NewSerialFile(filePath, f.handleHiddenFiles, stat)
-	if err != nil {
-		return "", nil, err
-	}
-
-	return stat.Name(), sf, nil
-}
-
 func (f *serialFile) Close() error {
 	return nil
 }
@@ -173,13 +150,18 @@ func (f *serialFile) Size() (int64, error) {
 
 	var du int64
 	err := filepath.Walk(f.path, func(p string, fi os.FileInfo, err error) error {
-		if err != nil {
+		if err != nil || fi == nil {
 			return err
 		}
 
-		if fi != nil && fi.Mode().IsRegular() {
+		if f.filter.ShouldExclude(fi) {
+			if fi.Mode().IsDir() {
+				return filepath.SkipDir
+			}
+		} else if fi.Mode().IsRegular() {
 			du += fi.Size()
 		}
+
 		return nil
 	})