From 0af35d7149e0e25af863379e77b054803f07ad4e Mon Sep 17 00:00:00 2001 From: PThorpe92 Date: Fri, 22 Sep 2023 21:15:27 -0400 Subject: [PATCH] feat: add support to respect .gitignore files --- go.mod | 1 + go.sum | 2 + src/cli/cli.go | 8 ++- src/croc/croc.go | 125 ++++++++++++++++++++++++++++++++++++++---- src/croc/croc_test.go | 40 ++++++++++++-- 5 files changed, 158 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index 5e9a79bc2..76160f739 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/chzyer/readline v1.5.1 github.com/denisbrodbeck/machineid v1.0.1 github.com/kalafut/imohash v1.0.2 + github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 github.com/schollz/cli/v2 v2.2.1 github.com/schollz/logger v1.2.0 github.com/schollz/mnemonicode v1.0.2-0.20190421205639-63fa713ece0d diff --git a/go.sum b/go.sum index afae9c75c..81efadc46 100644 --- a/go.sum +++ b/go.sum @@ -41,6 +41,8 @@ github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUc github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 h1:OkMGxebDjyw0ULyrTYWeN0UNCCkmCWfjPnIA2W6oviI= +github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06/go.mod h1:+ePHsJ1keEjQtpvf9HHw0f4ZeJ0TLRsxhunSI2hYJSs= github.com/schollz/cli/v2 v2.2.1 h1:ou22Mj7ZPjrKz+8k2iDTWaHskEEV5NiAxGrdsCL36VU= github.com/schollz/cli/v2 v2.2.1/go.mod h1:My6bfphRLZUhZdlFUK8scAxMWHydE7k4s2ed2Dtnn+s= github.com/schollz/logger v1.2.0 h1:5WXfINRs3lEUTCZ7YXhj0uN+qukjizvITLm3Ca2m0Ho= diff --git a/src/cli/cli.go b/src/cli/cli.go index 64502fe40..70f56d690 100644 --- a/src/cli/cli.go +++ b/src/cli/cli.go @@ -44,6 +44,7 @@ func Run() (err error) { app.UsageText = `Send a file: croc send file.txt + -git to respect your .gitignore Send multiple files: croc send file1.txt file2.txt file3.txt or @@ -70,6 +71,7 @@ func Run() (err error) { &cli.StringFlag{Name: "text", Aliases: []string{"t"}, Usage: "send some text"}, &cli.BoolFlag{Name: "no-local", Usage: "disable local relay when sending"}, &cli.BoolFlag{Name: "no-multi", Usage: "disable multiplexing"}, + &cli.BoolFlag{Name: "git", Usage: "enable .gitignore respect / don't send ignored files"}, &cli.StringFlag{Name: "ports", Value: "9009,9010,9011,9012,9013", Usage: "ports of the local relay (optional)"}, }, HelpName: "croc send", @@ -198,6 +200,7 @@ func send(c *cli.Context) (err error) { HashAlgorithm: c.String("hash"), ThrottleUpload: c.String("throttleUpload"), ZipFolder: c.Bool("zip"), + GitIgnore: c.Bool("git"), } if crocOptions.RelayAddress != models.DEFAULT_RELAY { crocOptions.RelayAddress6 = "" @@ -243,6 +246,9 @@ func send(c *cli.Context) (err error) { if !c.IsSet("hash") { crocOptions.HashAlgorithm = rememberedOptions.HashAlgorithm } + if !c.IsSet("git") { + crocOptions.GitIgnore = rememberedOptions.GitIgnore + } } var fnames []string @@ -281,7 +287,7 @@ func send(c *cli.Context) (err error) { // generate code phrase crocOptions.SharedSecret = utils.GetRandomName() } - minimalFileInfos, emptyFoldersToTransfer, totalNumberFolders, err := croc.GetFilesInfo(fnames, crocOptions.ZipFolder) + minimalFileInfos, emptyFoldersToTransfer, totalNumberFolders, err := croc.GetFilesInfo(fnames, crocOptions.ZipFolder, crocOptions.GitIgnore) if err != nil { return } diff --git a/src/croc/croc.go b/src/croc/croc.go index 5ec7866a9..a4e23a195 100644 --- a/src/croc/croc.go +++ b/src/croc/croc.go @@ -20,6 +20,7 @@ import ( "golang.org/x/time/rate" "github.com/denisbrodbeck/machineid" + ignore "github.com/sabhiram/go-gitignore" log "github.com/schollz/logger" "github.com/schollz/pake/v3" "github.com/schollz/peerdiscovery" @@ -77,6 +78,7 @@ type Options struct { ThrottleUpload string ZipFolder bool TestFlag bool + GitIgnore bool } // Client holds the state of the croc transfer @@ -101,6 +103,7 @@ type Client struct { TotalNumberFolders int FilesToTransferCurrentNum int FilesHasFinished map[int]struct{} + TotalFilesIgnored int // send / receive information of current file CurrentFile *os.File @@ -149,6 +152,7 @@ type FileInfo struct { Symlink string `json:"sy,omitempty"` Mode os.FileMode `json:"md,omitempty"` TempFile bool `json:"tf,omitempty"` + IsIgnored bool `json:"ig,omitempty"` } // RemoteFileRequest requests specific bytes @@ -251,9 +255,51 @@ func isEmptyFolder(folderPath string) (bool, error) { return false, nil } +// helper function to walk each subfolder and parses against an ignore file. +// returns a hashmap Key: Absolute filepath, Value: boolean (true=ignore) +func gitWalk(dir string, gitObj *ignore.GitIgnore, files map[string]bool) { + var ignoredDir bool + var current string + err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if isChild(current, path) && ignoredDir { + files[path] = true + return nil + } + if info.IsDir() && filepath.Base(path) == filepath.Base(dir) { + ignoredDir = false // Skip applying ignore rules for root directory + return nil + } + if gitObj.MatchesPath(info.Name()) { + files[path] = true + ignoredDir = true + current = path + return nil + } else { + files[path] = false + ignoredDir = false + return nil + } + }) + if err != nil { + log.Errorf("filepath error") + } +} + +func isChild(parentPath, childPath string) bool { + relPath, err := filepath.Rel(parentPath, childPath) + if err != nil { + return false + } + return !strings.HasPrefix(relPath, "..") + +} + // This function retrieves the important file information // for every file that will be transferred -func GetFilesInfo(fnames []string, zipfolder bool) (filesInfo []FileInfo, emptyFolders []FileInfo, totalNumberFolders int, err error) { +func GetFilesInfo(fnames []string, zipfolder bool, ignoreGit bool) (filesInfo []FileInfo, emptyFolders []FileInfo, totalNumberFolders int, err error) { // fnames: the relative/absolute paths of files/folders that will be transferred totalNumberFolders = 0 var paths []string @@ -271,7 +317,44 @@ func GetFilesInfo(fnames []string, zipfolder bool) (filesInfo []FileInfo, emptyF paths = append(paths, fname) } } - + var ignoredPaths = make(map[string]bool) + if ignoreGit { + wd, wdErr := os.Stat(".gitignore") + if wdErr == nil { + gitIgnore, gitErr := ignore.CompileIgnoreFile(wd.Name()) + if gitErr == nil { + for _, path := range paths { + abs, absErr := filepath.Abs(path) + if absErr != nil { + err = absErr + return + } + if gitIgnore.MatchesPath(path) { + ignoredPaths[abs] = true + } + } + } + } + for _, path := range paths { + abs, absErr := filepath.Abs(path) + if absErr != nil { + err = absErr + return + } + file, fileErr := os.Stat(path) + if fileErr == nil && file.IsDir() { + _, subErr := os.Stat(filepath.Join(path, ".gitignore")) + if subErr == nil { + gitObj, gitObjErr := ignore.CompileIgnoreFile(filepath.Join(path, ".gitignore")) + if gitObjErr != nil { + err = gitObjErr + return + } + gitWalk(abs, gitObj, ignoredPaths) + } + } + } + } for _, fpath := range paths { stat, errStat := os.Lstat(fpath) @@ -286,7 +369,6 @@ func GetFilesInfo(fnames []string, zipfolder bool) (filesInfo []FileInfo, emptyF err = errAbs return } - if stat.IsDir() && zipfolder { if fpath[len(fpath)-1:] != "/" { fpath += "/" @@ -304,7 +386,8 @@ func GetFilesInfo(fnames []string, zipfolder bool) (filesInfo []FileInfo, emptyF err = errAbs return } - filesInfo = append(filesInfo, FileInfo{ + + fInfo := FileInfo{ Name: stat.Name(), FolderRemote: "./", FolderSource: filepath.Dir(absPath), @@ -312,7 +395,12 @@ func GetFilesInfo(fnames []string, zipfolder bool) (filesInfo []FileInfo, emptyF ModTime: stat.ModTime(), Mode: stat.Mode(), TempFile: true, - }) + IsIgnored: ignoredPaths[absPath], + } + if fInfo.IsIgnored { + continue + } + filesInfo = append(filesInfo, fInfo) continue } @@ -325,7 +413,7 @@ func GetFilesInfo(fnames []string, zipfolder bool) (filesInfo []FileInfo, emptyF remoteFolder := strings.TrimPrefix(filepath.Dir(pathName), filepath.Dir(absPath)+string(os.PathSeparator)) if !info.IsDir() { - filesInfo = append(filesInfo, FileInfo{ + fInfo := FileInfo{ Name: info.Name(), FolderRemote: strings.ReplaceAll(remoteFolder, string(os.PathSeparator), "/") + "/", FolderSource: filepath.Dir(pathName), @@ -333,10 +421,19 @@ func GetFilesInfo(fnames []string, zipfolder bool) (filesInfo []FileInfo, emptyF ModTime: info.ModTime(), Mode: info.Mode(), TempFile: false, - }) + IsIgnored: ignoredPaths[pathName], + } + if fInfo.IsIgnored && ignoreGit { + return nil + } else { + filesInfo = append(filesInfo, fInfo) + } } else { - totalNumberFolders++ + if ignoredPaths[pathName] { + return filepath.SkipDir + } isEmptyFolder, _ := isEmptyFolder(pathName) + totalNumberFolders++ if isEmptyFolder { emptyFolders = append(emptyFolders, FileInfo{ // Name: info.Name(), @@ -352,7 +449,7 @@ func GetFilesInfo(fnames []string, zipfolder bool) (filesInfo []FileInfo, emptyF } } else { - filesInfo = append(filesInfo, FileInfo{ + fInfo := FileInfo{ Name: stat.Name(), FolderRemote: "./", FolderSource: filepath.Dir(absPath), @@ -360,9 +457,14 @@ func GetFilesInfo(fnames []string, zipfolder bool) (filesInfo []FileInfo, emptyF ModTime: stat.ModTime(), Mode: stat.Mode(), TempFile: false, - }) + IsIgnored: ignoredPaths[absPath], + } + if fInfo.IsIgnored && ignoreGit { + continue + } else { + filesInfo = append(filesInfo, fInfo) + } } - } return } @@ -517,7 +619,6 @@ func (c *Client) Send(filesInfo []FileInfo, emptyFoldersToTransfer []FileInfo, t c.TotalNumberFolders = totalNumberFolders c.TotalNumberOfContents = len(filesInfo) err = c.sendCollectFiles(filesInfo) - if err != nil { return } diff --git a/src/croc/croc_test.go b/src/croc/croc_test.go index dec48abc9..445ec5afe 100644 --- a/src/croc/croc_test.go +++ b/src/croc/croc_test.go @@ -5,6 +5,7 @@ import ( "path" "path/filepath" "runtime" + "strings" "sync" "testing" "time" @@ -41,6 +42,7 @@ func TestCrocReadme(t *testing.T) { DisableLocal: true, Curve: "siec", Overwrite: true, + GitIgnore: false, }) if err != nil { panic(err) @@ -66,7 +68,7 @@ func TestCrocReadme(t *testing.T) { var wg sync.WaitGroup wg.Add(2) go func() { - filesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{"../../README.md"}, false) + filesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{"../../README.md"}, false, false) if errGet != nil { t.Errorf("failed to get minimal info: %v", errGet) } @@ -132,7 +134,7 @@ func TestCrocEmptyFolder(t *testing.T) { var wg sync.WaitGroup wg.Add(2) go func() { - filesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{pathName}, false) + filesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{pathName}, false, false) if errGet != nil { t.Errorf("failed to get minimal info: %v", errGet) } @@ -174,6 +176,7 @@ func TestCrocSymlink(t *testing.T) { DisableLocal: true, Curve: "siec", Overwrite: true, + GitIgnore: false, }) if err != nil { panic(err) @@ -199,7 +202,7 @@ func TestCrocSymlink(t *testing.T) { var wg sync.WaitGroup wg.Add(2) go func() { - filesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{pathName}, false) + filesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{pathName}, false, false) if errGet != nil { t.Errorf("failed to get minimal info: %v", errGet) } @@ -229,6 +232,32 @@ func TestCrocSymlink(t *testing.T) { t.Errorf("symlink transfer failed: %s", err.Error()) } } +func testCrocIgnoreGit(t *testing.T) { + log.SetLevel("trace") + defer os.Remove(".gitignore") + time.Sleep(300 * time.Millisecond) + + time.Sleep(1 * time.Second) + file, err := os.Create(".gitignore") + if err != nil { + log.Errorf("error creating file") + } + _, err = file.WriteString("LICENSE") + if err != nil { + log.Errorf("error writing to file") + } + time.Sleep(1 * time.Second) + // due to how files are ignored in this function, all we have to do to test is make sure LICENSE doesn't get included in FilesInfo. + filesInfo, _, _, errGet := GetFilesInfo([]string{"../../LICENSE", ".gitignore", "croc.go"}, false, true) + if errGet != nil { + t.Errorf("failed to get minimal info: %v", errGet) + } + for _, file := range filesInfo { + if strings.Contains(file.Name, "LICENSE") { + t.Errorf("test failed, should ignore LICENSE") + } + } +} func TestCrocLocal(t *testing.T) { log.SetLevel("trace") @@ -249,6 +278,7 @@ func TestCrocLocal(t *testing.T) { DisableLocal: false, Curve: "siec", Overwrite: true, + GitIgnore: false, }) if err != nil { panic(err) @@ -276,7 +306,7 @@ func TestCrocLocal(t *testing.T) { os.Create("touched") wg.Add(2) go func() { - filesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{"../../LICENSE", "touched"}, false) + filesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{"../../LICENSE", "touched"}, false, false) if errGet != nil { t.Errorf("failed to get minimal info: %v", errGet) } @@ -329,7 +359,7 @@ func TestCrocError(t *testing.T) { Curve: "siec", Overwrite: true, }) - filesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{tmpfile.Name()}, false) + filesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{tmpfile.Name()}, false, false) if errGet != nil { t.Errorf("failed to get minimal info: %v", errGet) }