From 2b34e278cd80cdb924b4b28bf434c4c084ab61fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 7 Nov 2018 18:53:12 +0100 Subject: [PATCH] Directory iterators --- file.go | 75 +++++++++++++++++++++++++------ file_test.go | 37 ++++++++-------- multifilereader.go | 46 +++++++++++-------- multifilereader_test.go | 77 ++++++++++++++++---------------- multipartfile.go | 69 ++++++++++++++++++++++------- serialfile.go | 98 ++++++++++++++++++++++++++++++++++++----- slicefile.go | 80 +++++++++++++++++++++++++-------- webfile.go | 5 +-- 8 files changed, 348 insertions(+), 139 deletions(-) diff --git a/file.go b/file.go index b2d9712..693baab 100644 --- a/file.go +++ b/file.go @@ -28,7 +28,7 @@ type File interface { Size() (int64, error) } -// Regular represents the regular Unix file +// Regular represents a regular Unix file type Regular interface { File @@ -36,24 +36,71 @@ type Regular interface { io.Seeker } -// Directory is a special file which can link to any number of files +// DirEntry exposes information about a directory entry +type DirEntry interface { + // Name returns the base name of this entry, which is the base name of + // the referenced file + Name() string + + // File returns the file referenced by this DirEntry + File() File + + // Regular is an alias for ent.File().(Regular). If the file isn't a regular + // file, nil value will be returned + Regular() Regular + + // Dir is an alias for ent.File().(directory). If the file isn't a directory, + // nil value will be returned + Dir() Directory +} + +// DirIterator is a iterator over directory entries. +// See Directory.Entries for more +type DirIterator interface { + // DirEntry holds information about current directory entry. + // Note that after creating new iterator you MUST call Next() at least once + // before accessing these methods. Calling these methods without prior calls + // to Next() and after Next() returned false may result in undefined behavior + DirEntry + + // Next advances the iterator to the next file. + Next() bool + + // Err may return an error after the previous call to Next() returned `false`. + // If the previous call to Next() returned `true`, Err() is guaranteed to + // return nil + Err() error +} + +// Directory is a special file which can link to any number of files. type Directory interface { File - // NextFile returns the next child file available (if the File is a - // directory). It will return io.EOF if no more files are - // available. + // Entries returns a stateful iterator over directory entries. + // + // Example usage: + // + // it := dir.Entries() + // for it.Next() { + // name := it.Name() + // file := it.File() + // [...] + // } + // if it.Err() != nil { + // return err + // } // // Note: - // - Some implementations may only allow reading in order - if a - // child directory is returned, you need to read all it's children - // first before calling NextFile on parent again. Before doing parallel - // reading or reading entire level at once, make sure the implementation - // you are using allows that - // - Returned files may not be sorted + // - Below limitations aren't applicable to all implementations, consult + // your implementations manual before using this interface in a way that + // doesn't meet these constraints + // - Some implementations may only allow reading in order - so if the iterator + // returns a directory you must iterate over it's entries first before + // calling Next again + // - Order is not guaranteed // - Depending on implementation it may not be safe to iterate multiple - // children in parallel - NextFile() (string, File, error) + // 'branches' in parallel + Entries() (DirIterator, error) } // FileInfo exposes information on files in local filesystem @@ -63,6 +110,6 @@ type FileInfo interface { // AbsPath returns full real file path. AbsPath() string - // Stat returns os.Stat of this file + // Stat returns os.Stat of this file, may be nil for some files Stat() os.FileInfo } diff --git a/file_test.go b/file_test.go index 9b13082..cbf227c 100644 --- a/file_test.go +++ b/file_test.go @@ -9,21 +9,24 @@ import ( ) func TestSliceFiles(t *testing.T) { - files := []FileEntry{ - {NewReaderFile(ioutil.NopCloser(strings.NewReader("Some text!\n")), nil), ""}, - {NewReaderFile(ioutil.NopCloser(strings.NewReader("beep")), nil), ""}, - {NewReaderFile(ioutil.NopCloser(strings.NewReader("boop")), nil), ""}, + files := []DirEntry{ + FileEntry("", NewReaderFile(ioutil.NopCloser(strings.NewReader("Some text!\n")), nil)), + FileEntry("", NewReaderFile(ioutil.NopCloser(strings.NewReader("beep")), nil)), + FileEntry("", NewReaderFile(ioutil.NopCloser(strings.NewReader("boop")), nil)), } buf := make([]byte, 20) sf := NewSliceFile(files) + it, err := sf.Entries() + if err != nil { + t.Fatal(err) + } - _, file, err := sf.NextFile() - if file == nil || err != nil { - t.Fatal("Expected a file and nil error") + if !it.Next() { + t.Fatal("Expected a file") } - rf, ok := file.(Regular) - if !ok { + rf := it.Regular() + if rf == nil { t.Fatal("Expected a regular file") } read, err := rf.Read(buf) @@ -31,18 +34,14 @@ func TestSliceFiles(t *testing.T) { t.Fatal("NextFile got a file in the wrong order") } - _, file, err = sf.NextFile() - if file == nil || err != nil { - t.Fatal("Expected a file and nil error") + if !it.Next() { + t.Fatal("Expected a file") } - _, file, err = sf.NextFile() - if file == nil || err != nil { - t.Fatal("Expected a file and nil error") + if !it.Next() { + t.Fatal("Expected a file") } - - _, file, err = sf.NextFile() - if file != nil || err != io.EOF { - t.Fatal("Expected a nil file and io.EOF") + if it.Next() { + t.Fatal("Wild file appeared!") } if err := sf.Close(); err != nil { diff --git a/multifilereader.go b/multifilereader.go index b0ce1d9..8817ad8 100644 --- a/multifilereader.go +++ b/multifilereader.go @@ -17,7 +17,7 @@ type MultiFileReader struct { io.Reader // directory stack for NextFile - files []Directory + files []DirIterator path []string currentFile File @@ -34,16 +34,21 @@ type MultiFileReader struct { // NewMultiFileReader constructs a MultiFileReader. `file` can be any `commands.Directory`. // If `form` is set to true, the multipart data will have a Content-Type of 'multipart/form-data', // if `form` is false, the Content-Type will be 'multipart/mixed'. -func NewMultiFileReader(file Directory, form bool) *MultiFileReader { +func NewMultiFileReader(file Directory, form bool) (*MultiFileReader, error) { + it, err := file.Entries() + if err != nil { + return nil, err + } + mfr := &MultiFileReader{ - files: []Directory{file}, + files: []DirIterator{it}, path: []string{""}, form: form, mutex: &sync.Mutex{}, } mfr.mpWriter = multipart.NewWriter(&mfr.buf) - return mfr + return mfr, nil } func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) { @@ -57,47 +62,50 @@ func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) { // if the current file isn't set, advance to the next file if mfr.currentFile == nil { - var file File - var name string + var entry DirEntry - for file == nil { + for entry == nil { if len(mfr.files) == 0 { mfr.mpWriter.Close() mfr.closed = true return mfr.buf.Read(buf) } - nextName, nextFile, err := mfr.files[len(mfr.files)-1].NextFile() - if err == io.EOF { + if !mfr.files[len(mfr.files)-1].Next() { mfr.files = mfr.files[:len(mfr.files)-1] mfr.path = mfr.path[:len(mfr.path)-1] continue - } else if err != nil { - return 0, err + } + if mfr.files[len(mfr.files)-1].Err() != nil { + return 0, mfr.files[len(mfr.files)-1].Err() } - file = nextFile - name = nextName + entry = mfr.files[len(mfr.files)-1] } // handle starting a new file part if !mfr.closed { - mfr.currentFile = file + mfr.currentFile = entry.File() // write the boundary and headers header := make(textproto.MIMEHeader) - filename := url.QueryEscape(path.Join(path.Join(mfr.path...), name)) + filename := url.QueryEscape(path.Join(path.Join(mfr.path...), entry.Name())) header.Set("Content-Disposition", fmt.Sprintf("file; filename=\"%s\"", filename)) var contentType string - switch f := file.(type) { + switch f := entry.File().(type) { case *Symlink: contentType = "application/symlink" case Directory: - mfr.files = append(mfr.files, f) - mfr.path = append(mfr.path, name) + newIt, err := f.Entries() + if err != nil { + return 0, err + } + + mfr.files = append(mfr.files, newIt) + mfr.path = append(mfr.path, entry.Name()) contentType = "application/x-directory" case Regular: // otherwise, use the file as a reader to read its contents @@ -107,7 +115,7 @@ func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) { } header.Set("Content-Type", contentType) - if rf, ok := file.(FileInfo); ok { + if rf, ok := entry.File().(FileInfo); ok { header.Set("abspath", rf.AbsPath()) } diff --git a/multifilereader_test.go b/multifilereader_test.go index 5f0cac5..dd7dac9 100644 --- a/multifilereader_test.go +++ b/multifilereader_test.go @@ -10,23 +10,27 @@ import ( var text = "Some text! :)" -func getTestMultiFileReader() *MultiFileReader { - fileset := []FileEntry{ - {NewReaderFile(ioutil.NopCloser(strings.NewReader(text)), nil), "file.txt"}, - {NewSliceFile([]FileEntry{ - {NewReaderFile(ioutil.NopCloser(strings.NewReader("bleep")), nil), "a.txt"}, - {NewReaderFile(ioutil.NopCloser(strings.NewReader("bloop")), nil), "b.txt"}, - }), "boop"}, - {NewReaderFile(ioutil.NopCloser(strings.NewReader("beep")), nil), "beep.txt"}, +func getTestMultiFileReader(t *testing.T) *MultiFileReader { + fileset := []DirEntry{ + FileEntry("file.txt", NewReaderFile(ioutil.NopCloser(strings.NewReader(text)), nil)), + FileEntry("boop", NewSliceFile([]DirEntry{ + FileEntry("a.txt", NewReaderFile(ioutil.NopCloser(strings.NewReader("bleep")), nil)), + FileEntry("b.txt", NewReaderFile(ioutil.NopCloser(strings.NewReader("bloop")), nil)), + })), + FileEntry("beep.txt", NewReaderFile(ioutil.NopCloser(strings.NewReader("beep")), nil)), } sf := NewSliceFile(fileset) // testing output by reading it with the go stdlib "mime/multipart" Reader - return NewMultiFileReader(sf, true) + r, err := NewMultiFileReader(sf, true) + if err != nil { + t.Fatal(err) + } + return r } func TestMultiFileReaderToMultiFile(t *testing.T) { - mfr := getTestMultiFileReader() + mfr := getTestMultiFileReader(t) mpReader := multipart.NewReader(mfr, mfr.Boundary()) mf, err := NewFileFromPartReader(mpReader, multipartFormdataType) if err != nil { @@ -37,51 +41,48 @@ func TestMultiFileReaderToMultiFile(t *testing.T) { if !ok { t.Fatal("Expected a directory") } + it, err := md.Entries() + if err != nil { + t.Fatal(err) + } - fn, f, err := md.NextFile() - if fn != "file.txt" || f == nil || err != nil { - t.Fatal("NextFile returned unexpected data") + if !it.Next() || it.Name() != "file.txt" { + t.Fatal("iterator didn't work as expected") } - dn, d, err := md.NextFile() - if dn != "boop" || d == nil || err != nil { - t.Fatal("NextFile returned unexpected data") + if !it.Next() || it.Name() != "boop" || it.Dir() == nil { + t.Fatal("iterator didn't work as expected") } - df, ok := d.(Directory) - if !ok { - t.Fatal("Expected a directory") + subIt, err := it.Dir().Entries() + if err != nil { + t.Fatal(err) } - cfn, cf, err := df.NextFile() - if cfn != "a.txt" || cf == nil || err != nil { - t.Fatal("NextFile returned unexpected data") + if !subIt.Next() || subIt.Name() != "a.txt" || subIt.Dir() != nil { + t.Fatal("iterator didn't work as expected") } - cfn, cf, err = df.NextFile() - if cfn != "b.txt" || cf == nil || err != nil { - t.Fatal("NextFile returned unexpected data") + if !subIt.Next() || subIt.Name() != "b.txt" || subIt.Dir() != nil { + t.Fatal("iterator didn't work as expected") } - cfn, cf, err = df.NextFile() - if cfn != "" || cf != nil || err != io.EOF { - t.Fatal("NextFile returned unexpected data") + if subIt.Next() { + t.Fatal("iterator didn't work as expected") } // try to break internal state - cfn, cf, err = df.NextFile() - if cfn != "" || cf != nil || err != io.EOF { - t.Fatal("NextFile returned unexpected data") + if subIt.Next() { + t.Fatal("iterator didn't work as expected") } - fn, f, err = md.NextFile() - if fn != "beep.txt" || f == nil || err != nil { - t.Fatal("NextFile returned unexpected data") + if !it.Next() || it.Name() != "beep.txt" || it.Dir() != nil { + t.Fatal("iterator didn't work as expected") } } func TestOutput(t *testing.T) { - mfr := getTestMultiFileReader() + mfr := getTestMultiFileReader(t) mpReader := &peekReader{r: multipart.NewReader(mfr, mfr.Boundary())} buf := make([]byte, 20) @@ -153,9 +154,9 @@ func TestOutput(t *testing.T) { t.Fatal("Expected filename to be \"b.txt\"") } - cname, child, err = mpd.NextFile() - if child != nil || err != io.EOF { - t.Fatal("Expected to get (nil, io.EOF)") + it, err := mpd.Entries() + if it.Next() { + t.Fatal("Expected to get false") } part, err = mpReader.NextPart() diff --git a/multipartfile.go b/multipartfile.go index 5c62c2c..be7e221 100644 --- a/multipartfile.go +++ b/multipartfile.go @@ -2,7 +2,6 @@ package files import ( "errors" - "io" "io/ioutil" "mime" "mime/multipart" @@ -39,7 +38,7 @@ func NewFileFromPartReader(reader *multipart.Reader, mediatype string) (File, er } f := &MultipartFile{ - Reader: &peekReader{r: reader}, + Reader: &peekReader{r: reader}, Mediatype: mediatype, } @@ -95,30 +94,65 @@ func isDirectory(mediatype string) bool { return mediatype == multipartFormdataType || mediatype == applicationDirectory } -func (f *MultipartFile) NextFile() (string, File, error) { - if f.Reader == nil { - return "", nil, io.EOF +type multipartIterator struct { + f *MultipartFile + + curFile File + curName string + err error +} + +func (it *multipartIterator) Name() string { + return it.curName +} + +func (it *multipartIterator) File() File { + return it.curFile +} + +func (it *multipartIterator) Regular() Regular { + return castRegular(it.File()) +} + +func (it *multipartIterator) Dir() Directory { + return castDir(it.File()) +} + +func (it *multipartIterator) Next() bool { + if it.f.Reader == nil { + return false } - part, err := f.Reader.NextPart() + part, err := it.f.Reader.NextPart() if err != nil { - return "", nil, err + it.err = err + return false } - name, cf, err := newFileFromPart(f.fileName(), part, f.Reader) + name, cf, err := newFileFromPart(it.f.fileName(), part, it.f.Reader) if err != ErrPartOutsideParent { - return name, cf, err + it.curFile = cf + it.curName = name + it.err = err + return err == nil } // we read too much, try to fix this - pr, ok := f.Reader.(*peekReader) + pr, ok := it.f.Reader.(*peekReader) if !ok { - return "", nil, errors.New("cannot undo NextPart") + it.err = errors.New("cannot undo NextPart") + return false } - if err := pr.put(part); err != nil { - return "", nil, err - } - return "", nil, io.EOF + it.err = pr.put(part) + return false +} + +func (it *multipartIterator) Err() error { + panic("implement me") +} + +func (f *MultipartFile) Entries() (DirIterator, error) { + return &multipartIterator{f: f}, nil } func (f *MultipartFile) fileName() string { @@ -135,7 +169,10 @@ func (f *MultipartFile) fileName() string { } func (f *MultipartFile) Close() error { - return f.Part.Close() + if f.Part != nil { + return f.Part.Close() + } + return nil } func (f *MultipartFile) Size() (int64, error) { diff --git a/serialfile.go b/serialfile.go index 666399d..77afafa 100644 --- a/serialfile.go +++ b/serialfile.go @@ -1,6 +1,7 @@ package files import ( + "errors" "fmt" "io" "io/ioutil" @@ -19,6 +20,18 @@ type serialFile struct { handleHiddenFiles bool } +type serialIterator struct { + files []os.FileInfo + handleHiddenFiles bool + path string + + curName string + curFile File + + err error +} + +// TODO: test/document limitations func NewSerialFile(path string, hidden bool, stat os.FileInfo) (File, error) { switch mode := stat.Mode(); { case mode.IsRegular(): @@ -46,20 +59,69 @@ func NewSerialFile(path string, hidden bool, stat os.FileInfo) (File, error) { } } -func (f *serialFile) NextFile() (string, File, error) { - // if a file was opened previously, close it - err := f.Close() - if err != nil { - switch err2 := err.(type) { - case *os.PathError: - if err2.Err != os.ErrClosed { - return "", nil, err - } - default: - return "", nil, err +func (it *serialIterator) Name() string { + return it.curName +} + +func (it *serialIterator) File() File { + return it.curFile +} + +func (it *serialIterator) Regular() Regular { + return castRegular(it.File()) +} + +func (it *serialIterator) Dir() Directory { + return castDir(it.File()) +} + +func (it *serialIterator) Next() bool { + // if there aren't any files left in the root directory, we're done + if len(it.files) == 0 { + return false + } + + stat := it.files[0] + it.files = it.files[1:] + for !it.handleHiddenFiles && strings.HasPrefix(stat.Name(), ".") { + if len(it.files) == 0 { + return false } + + stat = it.files[0] + it.files = it.files[1:] + } + + // open the next file + filePath := filepath.ToSlash(filepath.Join(it.path, stat.Name())) + + // recursively call the constructor on the next file + // if it's a regular file, we will open it as a ReaderFile + // if it's a directory, files in it will be opened serially + sf, err := NewSerialFile(filePath, it.handleHiddenFiles, stat) + if err != nil { + it.err = err + return false } + it.curName = stat.Name() + it.curFile = sf + return true +} + +func (it *serialIterator) Err() error { + return it.err +} + +func (f *serialFile) Entries() (DirIterator, error) { + return &serialIterator{ + path: f.path, + files: f.files, + handleHiddenFiles: f.handleHiddenFiles, + }, nil +} + +func (f *serialFile) NextFile() (string, File, error) { // if there aren't any files left in the root directory, we're done if len(f.files) == 0 { return "", nil, io.EOF @@ -101,7 +163,8 @@ func (f *serialFile) Stat() os.FileInfo { func (f *serialFile) Size() (int64, error) { if !f.stat.IsDir() { - return f.stat.Size(), nil + //something went terribly, terribly wrong + return 0, errors.New("serialFile is not a directory") } var du int64 @@ -119,4 +182,15 @@ func (f *serialFile) Size() (int64, error) { return du, err } +func castRegular(f File) Regular { + r, _ := f.(Regular) + return r +} + +func castDir(f File) Directory { + d, _ := f.(Directory) + return d +} + var _ Directory = &serialFile{} +var _ DirIterator = &serialIterator{} diff --git a/slicefile.go b/slicefile.go index 0d78e16..62c299f 100644 --- a/slicefile.go +++ b/slicefile.go @@ -1,33 +1,76 @@ package files -import ( - "io" -) +type fileEntry struct { + name string + file File +} + +func (e fileEntry) Name() string { + return e.name +} + +func (e fileEntry) File() File { + return e.file +} + +func (e fileEntry) Regular() Regular { + return castRegular(e.file) +} + +func (e fileEntry) Dir() Directory { + return castDir(e.file) +} + +func FileEntry(name string, file File) DirEntry { + return fileEntry{ + name: name, + file: file, + } +} + +type sliceIterator struct { + files []DirEntry + n int +} + +func (it *sliceIterator) Name() string { + return it.files[it.n].Name() +} + +func (it *sliceIterator) File() File { + return it.files[it.n].File() +} + +func (it *sliceIterator) Regular() Regular { + return it.files[it.n].Regular() +} -type FileEntry struct { - File File - Name string +func (it *sliceIterator) Dir() Directory { + return it.files[it.n].Dir() +} + +func (it *sliceIterator) Next() bool { + it.n++ + return it.n < len(it.files) +} + +func (it *sliceIterator) Err() error { + return nil } // SliceFile implements File, and provides simple directory handling. // It contains children files, and is created from a `[]File`. // SliceFiles are always directories, and can't be read from or closed. type SliceFile struct { - files []FileEntry - n int + files []DirEntry } -func NewSliceFile(files []FileEntry) Directory { - return &SliceFile{files, 0} +func NewSliceFile(files []DirEntry) Directory { + return &SliceFile{files} } -func (f *SliceFile) NextFile() (string, File, error) { - if f.n >= len(f.files) { - return "", nil, io.EOF - } - file := f.files[f.n] - f.n++ - return file.Name, file.File, nil +func (f *SliceFile) Entries() (DirIterator, error) { + return &sliceIterator{files: f.files, n: -1}, nil } func (f *SliceFile) Close() error { @@ -42,7 +85,7 @@ func (f *SliceFile) Size() (int64, error) { var size int64 for _, file := range f.files { - s, err := file.File.Size() + s, err := file.File().Size() if err != nil { return 0, err } @@ -53,3 +96,4 @@ func (f *SliceFile) Size() (int64, error) { } var _ Directory = &SliceFile{} +var _ DirEntry = fileEntry{} diff --git a/webfile.go b/webfile.go index 5638f77..c8e930a 100644 --- a/webfile.go +++ b/webfile.go @@ -11,8 +11,8 @@ import ( // from a Web URL (http). A GET request will be performed // against the source when calling Read(). type WebFile struct { - body io.ReadCloser - url *url.URL + body io.ReadCloser + url *url.URL contentLength int64 } @@ -61,5 +61,4 @@ func (wf *WebFile) Size() (int64, error) { return wf.contentLength, nil } - var _ Regular = &WebFile{}