From ba267cb676e2ab06fc57e1612f7534d1d8501957 Mon Sep 17 00:00:00 2001 From: Alexandr Krylovskiy Date: Sun, 21 Jan 2018 14:42:39 +0100 Subject: [PATCH 001/120] references: sort: compare author timestamps when commit timestamps are equal. Fixes #725 Signed-off-by: Alexandr Krylovskiy --- references.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/references.go b/references.go index a1872a5a1..5673ac135 100644 --- a/references.go +++ b/references.go @@ -47,7 +47,9 @@ func (s commitSorterer) Len() int { } func (s commitSorterer) Less(i, j int) bool { - return s.l[i].Committer.When.Before(s.l[j].Committer.When) + return s.l[i].Committer.When.Before(s.l[j].Committer.When) || + s.l[i].Committer.When.Equal(s.l[j].Committer.When) && + s.l[i].Author.When.Before(s.l[j].Author.When) } func (s commitSorterer) Swap(i, j int) { From 2eb97fb44707bc122c53bf2ad8eaf24ad8d679f4 Mon Sep 17 00:00:00 2001 From: Dustin Frisch Date: Wed, 9 May 2018 12:50:49 +0200 Subject: [PATCH 002/120] Use remote name in fetch while clone Fixes #827 Signed-off-by: Dustin Frisch --- repository.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/repository.go b/repository.go index 717381bdb..247966611 100644 --- a/repository.go +++ b/repository.go @@ -520,10 +520,11 @@ func (r *Repository) clone(ctx context.Context, o *CloneOptions) error { ref, err := r.fetchAndUpdateReferences(ctx, &FetchOptions{ RefSpecs: r.cloneRefSpec(o, c), - Depth: o.Depth, - Auth: o.Auth, - Progress: o.Progress, - Tags: o.Tags, + Depth: o.Depth, + Auth: o.Auth, + Progress: o.Progress, + Tags: o.Tags, + RemoteName: o.RemoteName, }, o.ReferenceName) if err != nil { return err From 939793b42974c12abf2d2b65facee489004a9e06 Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Mon, 14 May 2018 17:16:16 +0200 Subject: [PATCH 003/120] git: remote, Do not iterate all references on update. The current code iterates all the references in the remote to check if they match the refspec. This is OK when the refspec is a wildcard but is a waste of time when they are not. A hash with references is generated for fast access before starting the update and used only when the refspec is not a wildcard. In a repository with 7800 references this meant 7800 * 7800 checks. With the current code it took 8m30s to update the references. With the new code it takes less than 0.5s. References are already extensively tested in remote_test.go. Signed-off-by: Javi Fontan --- remote.go | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/remote.go b/remote.go index 4b86955e8..60461d611 100644 --- a/remote.go +++ b/remote.go @@ -371,14 +371,22 @@ func (r *Remote) addReferencesToUpdate( refspecs []config.RefSpec, localRefs []*plumbing.Reference, remoteRefs storer.ReferenceStorer, - req *packp.ReferenceUpdateRequest) error { + req *packp.ReferenceUpdateRequest, +) error { + // This references dictionary will be used to search references by name. + refsDict := make(map[string]*plumbing.Reference) + for _, ref := range localRefs { + refsDict[ref.Name().String()] = ref + } + for _, rs := range refspecs { if rs.IsDelete() { if err := r.deleteReferences(rs, remoteRefs, req); err != nil { return err } } else { - if err := r.addOrUpdateReferences(rs, localRefs, remoteRefs, req); err != nil { + err := r.addOrUpdateReferences(rs, localRefs, refsDict, remoteRefs, req) + if err != nil { return err } } @@ -390,9 +398,21 @@ func (r *Remote) addReferencesToUpdate( func (r *Remote) addOrUpdateReferences( rs config.RefSpec, localRefs []*plumbing.Reference, + refsDict map[string]*plumbing.Reference, remoteRefs storer.ReferenceStorer, req *packp.ReferenceUpdateRequest, ) error { + // If it is not a wilcard refspec we can directly search for the reference + // in the references dictionary. + if !rs.IsWildcard() { + ref, ok := refsDict[rs.Src()] + if !ok { + return nil + } + + return r.addReferenceIfRefSpecMatches(rs, remoteRefs, ref, req) + } + for _, ref := range localRefs { err := r.addReferenceIfRefSpecMatches(rs, remoteRefs, ref, req) if err != nil { From 689e334b51565dda54fcd44b2bf14da99eed61bb Mon Sep 17 00:00:00 2001 From: David Symonds Date: Wed, 30 May 2018 10:34:28 +1000 Subject: [PATCH 004/120] idxfile: optimise allocations in readObjectNames This makes all the required Entry allocations in one go, instead of huge amounts of small individual allocations. Signed-off-by: David Symonds --- plumbing/format/idxfile/decoder.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/plumbing/format/idxfile/decoder.go b/plumbing/format/idxfile/decoder.go index f36121322..45afb1ec0 100644 --- a/plumbing/format/idxfile/decoder.go +++ b/plumbing/format/idxfile/decoder.go @@ -6,7 +6,6 @@ import ( "errors" "io" - "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/utils/binary" ) @@ -98,13 +97,14 @@ func readFanout(idx *Idxfile, r io.Reader) error { func readObjectNames(idx *Idxfile, r io.Reader) error { c := int(idx.ObjectCount) + new := make([]Entry, c) for i := 0; i < c; i++ { - var ref plumbing.Hash - if _, err := io.ReadFull(r, ref[:]); err != nil { + e := &new[i] + if _, err := io.ReadFull(r, e.Hash[:]); err != nil { return err } - idx.Entries = append(idx.Entries, &Entry{Hash: ref}) + idx.Entries = append(idx.Entries, e) } return nil From cf532f99e3e7632bc1d813245a4c79ae38b4d320 Mon Sep 17 00:00:00 2001 From: David Symonds Date: Wed, 30 May 2018 11:06:44 +1000 Subject: [PATCH 005/120] packfile: improve Index memory representation to be more compact Instead of using a map for offset indexing, use a sorted slice. Binary searching is fast, and a slice is much more compact. This has a negligible hit on speed, but has a significant impact on memory usage, especially for larger repos. benchmark old ns/op new ns/op delta BenchmarkIndexConstruction-12 15506506 14056098 -9.35% benchmark old allocs new allocs delta BenchmarkIndexConstruction-12 60764 60385 -0.62% benchmark old bytes new bytes delta BenchmarkIndexConstruction-12 4318145 3913169 -9.38% Signed-off-by: David Symonds --- plumbing/format/packfile/index.go | 53 +++++++++++++++++++++----- plumbing/format/packfile/index_test.go | 37 +++++++++++------- 2 files changed, 67 insertions(+), 23 deletions(-) diff --git a/plumbing/format/packfile/index.go b/plumbing/format/packfile/index.go index 2c5f98f8f..7d8f2ad10 100644 --- a/plumbing/format/packfile/index.go +++ b/plumbing/format/packfile/index.go @@ -1,6 +1,8 @@ package packfile import ( + "sort" + "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/format/idxfile" ) @@ -10,7 +12,7 @@ import ( // or to store them. type Index struct { byHash map[plumbing.Hash]*idxfile.Entry - byOffset map[uint64]*idxfile.Entry + byOffset []*idxfile.Entry // sorted by their offset } // NewIndex creates a new empty index with the given size. Size is a hint and @@ -19,7 +21,7 @@ type Index struct { func NewIndex(size int) *Index { return &Index{ byHash: make(map[plumbing.Hash]*idxfile.Entry, size), - byOffset: make(map[uint64]*idxfile.Entry, size), + byOffset: make([]*idxfile.Entry, 0, size), } } @@ -27,28 +29,54 @@ func NewIndex(size int) *Index { func NewIndexFromIdxFile(idxf *idxfile.Idxfile) *Index { idx := &Index{ byHash: make(map[plumbing.Hash]*idxfile.Entry, idxf.ObjectCount), - byOffset: make(map[uint64]*idxfile.Entry, idxf.ObjectCount), + byOffset: make([]*idxfile.Entry, 0, idxf.ObjectCount), } for _, e := range idxf.Entries { - idx.add(e) + idx.addUnsorted(e) } + sort.Sort(orderByOffset(idx.byOffset)) return idx } +// orderByOffset is a sort.Interface adapter that arranges +// a slice of entries by their offset. +type orderByOffset []*idxfile.Entry + +func (o orderByOffset) Len() int { return len(o) } +func (o orderByOffset) Less(i, j int) bool { return o[i].Offset < o[j].Offset } +func (o orderByOffset) Swap(i, j int) { o[i], o[j] = o[j], o[i] } + // Add adds a new Entry with the given values to the index. func (idx *Index) Add(h plumbing.Hash, offset uint64, crc32 uint32) { - e := idxfile.Entry{ + e := &idxfile.Entry{ Hash: h, Offset: offset, CRC32: crc32, } - idx.add(&e) + idx.byHash[e.Hash] = e + + // Find the right position in byOffset. + // Look for the first position whose offset is *greater* than e.Offset. + i := sort.Search(len(idx.byOffset), func(i int) bool { + return idx.byOffset[i].Offset > offset + }) + if i == len(idx.byOffset) { + // Simple case: add it to the end. + idx.byOffset = append(idx.byOffset, e) + return + } + // Harder case: shift existing entries down by one to make room. + // Append a nil entry first so we can use existing capacity in case + // the index was carefully preallocated. + idx.byOffset = append(idx.byOffset, nil) + copy(idx.byOffset[i+1:], idx.byOffset[i:len(idx.byOffset)-1]) + idx.byOffset[i] = e } -func (idx *Index) add(e *idxfile.Entry) { +func (idx *Index) addUnsorted(e *idxfile.Entry) { idx.byHash[e.Hash] = e - idx.byOffset[e.Offset] = e + idx.byOffset = append(idx.byOffset, e) } // LookupHash looks an entry up by its hash. An idxfile.Entry is returned and @@ -61,8 +89,13 @@ func (idx *Index) LookupHash(h plumbing.Hash) (*idxfile.Entry, bool) { // LookupHash looks an entry up by its offset in the packfile. An idxfile.Entry // is returned and a bool, which is true if it was found or false if it wasn't. func (idx *Index) LookupOffset(offset uint64) (*idxfile.Entry, bool) { - e, ok := idx.byOffset[offset] - return e, ok + i := sort.Search(len(idx.byOffset), func(i int) bool { + return idx.byOffset[i].Offset >= offset + }) + if i >= len(idx.byOffset) || idx.byOffset[i].Offset != offset { + return nil, false // not present + } + return idx.byOffset[i], true } // Size returns the number of entries in the index. diff --git a/plumbing/format/packfile/index_test.go b/plumbing/format/packfile/index_test.go index 67147046f..8de886dac 100644 --- a/plumbing/format/packfile/index_test.go +++ b/plumbing/format/packfile/index_test.go @@ -3,6 +3,7 @@ package packfile import ( "strconv" "strings" + "testing" "gopkg.in/src-d/go-git.v4/plumbing" @@ -26,12 +27,12 @@ func (s *IndexSuite) TestLookupOffset(c *C) { e, ok := idx.LookupOffset(uint64(o2)) c.Assert(ok, Equals, true) c.Assert(e, NotNil) - c.Assert(e.Hash, Equals, s.toHash(o2)) + c.Assert(e.Hash, Equals, toHash(o2)) c.Assert(e.Offset, Equals, uint64(o2)) } } - h1 := s.toHash(o1) + h1 := toHash(o1) idx.Add(h1, uint64(o1), 0) for o2 := 0; o2 < 10000; o2 += 100 { @@ -43,7 +44,7 @@ func (s *IndexSuite) TestLookupOffset(c *C) { e, ok := idx.LookupOffset(uint64(o2)) c.Assert(ok, Equals, true) c.Assert(e, NotNil) - c.Assert(e.Hash, Equals, s.toHash(o2)) + c.Assert(e.Hash, Equals, toHash(o2)) c.Assert(e.Offset, Equals, uint64(o2)) } } @@ -56,31 +57,31 @@ func (s *IndexSuite) TestLookupHash(c *C) { for o1 := 0; o1 < 10000; o1 += 100 { for o2 := 0; o2 < 10000; o2 += 100 { if o2 >= o1 { - e, ok := idx.LookupHash(s.toHash(o2)) + e, ok := idx.LookupHash(toHash(o2)) c.Assert(ok, Equals, false) c.Assert(e, IsNil) } else { - e, ok := idx.LookupHash(s.toHash(o2)) + e, ok := idx.LookupHash(toHash(o2)) c.Assert(ok, Equals, true) c.Assert(e, NotNil) - c.Assert(e.Hash, Equals, s.toHash(o2)) + c.Assert(e.Hash, Equals, toHash(o2)) c.Assert(e.Offset, Equals, uint64(o2)) } } - h1 := s.toHash(o1) + h1 := toHash(o1) idx.Add(h1, uint64(o1), 0) for o2 := 0; o2 < 10000; o2 += 100 { if o2 > o1 { - e, ok := idx.LookupHash(s.toHash(o2)) + e, ok := idx.LookupHash(toHash(o2)) c.Assert(ok, Equals, false) c.Assert(e, IsNil) } else { - e, ok := idx.LookupHash(s.toHash(o2)) + e, ok := idx.LookupHash(toHash(o2)) c.Assert(ok, Equals, true) c.Assert(e, NotNil) - c.Assert(e.Hash, Equals, s.toHash(o2)) + c.Assert(e.Hash, Equals, toHash(o2)) c.Assert(e.Offset, Equals, uint64(o2)) } } @@ -92,7 +93,7 @@ func (s *IndexSuite) TestSize(c *C) { for o1 := 0; o1 < 1000; o1++ { c.Assert(idx.Size(), Equals, o1) - h1 := s.toHash(o1) + h1 := toHash(o1) idx.Add(h1, uint64(o1), 0) } } @@ -107,7 +108,7 @@ func (s *IndexSuite) TestIdxFileEmpty(c *C) { func (s *IndexSuite) TestIdxFile(c *C) { idx := NewIndex(0) for o1 := 0; o1 < 1000; o1++ { - h1 := s.toHash(o1) + h1 := toHash(o1) idx.Add(h1, uint64(o1), 0) } @@ -115,8 +116,18 @@ func (s *IndexSuite) TestIdxFile(c *C) { c.Assert(idx, DeepEquals, idx2) } -func (s *IndexSuite) toHash(i int) plumbing.Hash { +func toHash(i int) plumbing.Hash { is := strconv.Itoa(i) padding := strings.Repeat("a", 40-len(is)) return plumbing.NewHash(padding + is) } + +func BenchmarkIndexConstruction(b *testing.B) { + b.ReportAllocs() + + idx := NewIndex(0) + for o := 0; o < 1e6*b.N; o += 100 { + h1 := toHash(o) + idx.Add(h1, uint64(o), 0) + } +} From 79b7f24160029966238b04dd41f69add0741a1d2 Mon Sep 17 00:00:00 2001 From: Joseph Vusich Date: Wed, 30 May 2018 02:42:46 +0000 Subject: [PATCH 006/120] config: modules, Ignore submodules with dotdot '..' path components. Fixes CVE-2018-11235 References: * https://blogs.msdn.microsoft.com/devops/2018/05/29/announcing-the-may-2018-git-security-vulnerability/ * https://security-tracker.debian.org/tracker/CVE-2018-11235 * https://github.com/git/git/commit/0383bbb9015898cbc79abd7b64316484d7713b44 Signed-off-by: Joseph Vusich --- config/config.go | 12 ++++++++---- config/modules.go | 20 ++++++++++++-------- config/modules_test.go | 26 ++++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 12 deletions(-) diff --git a/config/config.go b/config/config.go index c730015ed..ce6506dae 100644 --- a/config/config.go +++ b/config/config.go @@ -135,7 +135,7 @@ func (c *Config) Unmarshal(b []byte) error { if err := c.unmarshalPack(); err != nil { return err } - c.unmarshalSubmodules() + unmarshalSubmodules(c.Raw, c.Submodules) if err := c.unmarshalBranches(); err != nil { return err @@ -182,13 +182,17 @@ func (c *Config) unmarshalRemotes() error { return nil } -func (c *Config) unmarshalSubmodules() { - s := c.Raw.Section(submoduleSection) +func unmarshalSubmodules(fc *format.Config, submodules map[string]*Submodule) { + s := fc.Section(submoduleSection) for _, sub := range s.Subsections { m := &Submodule{} m.unmarshal(sub) - c.Submodules[m.Name] = m + if m.Validate() == ErrModuleBadPath { + continue + } + + submodules[m.Name] = m } } diff --git a/config/modules.go b/config/modules.go index b20898405..90758d932 100644 --- a/config/modules.go +++ b/config/modules.go @@ -3,6 +3,7 @@ package config import ( "bytes" "errors" + "regexp" format "gopkg.in/src-d/go-git.v4/plumbing/format/config" ) @@ -10,6 +11,12 @@ import ( var ( ErrModuleEmptyURL = errors.New("module config: empty URL") ErrModuleEmptyPath = errors.New("module config: empty path") + ErrModuleBadPath = errors.New("submodule has an invalid path") +) + +var ( + // Matches module paths with dotdot ".." components. + dotdotPath = regexp.MustCompile(`(^|[/\\])\.\.([/\\]|$)`) ) // Modules defines the submodules properties, represents a .gitmodules file @@ -44,14 +51,7 @@ func (m *Modules) Unmarshal(b []byte) error { return err } - s := m.raw.Section(submoduleSection) - for _, sub := range s.Subsections { - mod := &Submodule{} - mod.unmarshal(sub) - - m.Submodules[mod.Path] = mod - } - + unmarshalSubmodules(m.raw, m.Submodules) return nil } @@ -102,6 +102,10 @@ func (m *Submodule) Validate() error { return ErrModuleEmptyURL } + if dotdotPath.MatchString(m.Path) { + return ErrModuleBadPath + } + return nil } diff --git a/config/modules_test.go b/config/modules_test.go index 36cd93f73..8e10d70f1 100644 --- a/config/modules_test.go +++ b/config/modules_test.go @@ -11,6 +11,29 @@ func (s *ModulesSuite) TestValidateMissingURL(c *C) { c.Assert(m.Validate(), Equals, ErrModuleEmptyURL) } +func (s *ModulesSuite) TestValidateBadPath(c *C) { + input := []string{ + `..`, + `../`, + `../bar`, + + `/..`, + `/../bar`, + + `foo/..`, + `foo/../`, + `foo/../bar`, + } + + for _, p := range input { + m := &Submodule{ + Path: p, + URL: "https://example.com/", + } + c.Assert(m.Validate(), Equals, ErrModuleBadPath) + } +} + func (s *ModulesSuite) TestValidateMissingName(c *C) { m := &Submodule{URL: "bar"} c.Assert(m.Validate(), Equals, ErrModuleEmptyPath) @@ -39,6 +62,9 @@ func (s *ModulesSuite) TestUnmarshall(c *C) { path = foo/bar url = https://github.com/foo/bar.git branch = dev +[submodule "suspicious"] + path = ../../foo/bar + url = https://github.com/foo/bar.git `) cfg := NewModules() From d87faeca21e6f416e88ae3d24dae58845d7487d4 Mon Sep 17 00:00:00 2001 From: Joseph Vusich Date: Wed, 30 May 2018 03:47:57 +0000 Subject: [PATCH 007/120] worktree: Don't allow .gitmodules to be a symlink. Fixes CVE-2018-11235 References: * https://blogs.msdn.microsoft.com/devops/2018/05/29/announcing-the-may-2018-git-security-vulnerability/ * https://security-tracker.debian.org/tracker/CVE-2018-11235 * https://github.com/git/git/commit/10ecfa76491e4923988337b2e2243b05376b40de Signed-off-by: Joseph Vusich --- submodule_test.go | 15 +++++++++++++++ worktree.go | 12 ++++++++++++ 2 files changed, 27 insertions(+) diff --git a/submodule_test.go b/submodule_test.go index 7c97179e4..2c0a2edeb 100644 --- a/submodule_test.go +++ b/submodule_test.go @@ -196,6 +196,21 @@ func (s *SubmoduleSuite) TestSubmodulesInit(c *C) { } } +func (s *SubmoduleSuite) TestGitSubmodulesSymlink(c *C) { + f, err := s.Worktree.Filesystem.Create("badfile") + c.Assert(err, IsNil) + defer f.Close() + + err = s.Worktree.Filesystem.Remove(gitmodulesFile) + c.Assert(err, IsNil) + + err = s.Worktree.Filesystem.Symlink("badfile", gitmodulesFile) + c.Assert(err, IsNil) + + _, err = s.Worktree.Submodules() + c.Assert(err, Equals, ErrGitModulesSymlink) +} + func (s *SubmoduleSuite) TestSubmodulesStatus(c *C) { sm, err := s.Worktree.Submodules() c.Assert(err, IsNil) diff --git a/worktree.go b/worktree.go index ddf6fffd0..99b2cd124 100644 --- a/worktree.go +++ b/worktree.go @@ -28,6 +28,7 @@ var ( ErrWorktreeNotClean = errors.New("worktree is not clean") ErrSubmoduleNotFound = errors.New("submodule not found") ErrUnstagedChanges = errors.New("worktree contains unstaged changes") + ErrGitModulesSymlink = errors.New(gitmodulesFile + " is a symlink") ) // Worktree represents a git worktree. @@ -680,7 +681,18 @@ func (w *Worktree) newSubmodule(fromModules, fromConfig *config.Submodule) *Subm return m } +func (w *Worktree) isSymlink(path string) bool { + if s, err := w.Filesystem.Lstat(path); err == nil { + return s.Mode()&os.ModeSymlink != 0 + } + return false +} + func (w *Worktree) readGitmodulesFile() (*config.Modules, error) { + if w.isSymlink(gitmodulesFile) { + return nil, ErrGitModulesSymlink + } + f, err := w.Filesystem.Open(gitmodulesFile) if err != nil { if os.IsNotExist(err) { From b0d807a1ae0687ef3a01d78c1dc5e55f7217268f Mon Sep 17 00:00:00 2001 From: Antonio Jesus Navarro Perez Date: Tue, 5 Jun 2018 18:33:27 +0200 Subject: [PATCH 008/120] dotgit: Move package outside internal. Signed-off-by: Antonio Jesus Navarro Perez --- storage/filesystem/config.go | 2 +- storage/filesystem/config_test.go | 2 +- storage/filesystem/{internal => }/dotgit/dotgit.go | 0 .../{internal => }/dotgit/dotgit_rewrite_packed_refs_nix.go | 0 .../{internal => }/dotgit/dotgit_rewrite_packed_refs_norwfs.go | 0 .../{internal => }/dotgit/dotgit_rewrite_packed_refs_windows.go | 0 storage/filesystem/{internal => }/dotgit/dotgit_setref.go | 0 .../filesystem/{internal => }/dotgit/dotgit_setref_norwfs.go | 0 storage/filesystem/{internal => }/dotgit/dotgit_test.go | 0 storage/filesystem/{internal => }/dotgit/writers.go | 0 storage/filesystem/{internal => }/dotgit/writers_test.go | 0 storage/filesystem/index.go | 2 +- storage/filesystem/module.go | 2 +- storage/filesystem/object.go | 2 +- storage/filesystem/object_test.go | 2 +- storage/filesystem/reference.go | 2 +- storage/filesystem/shallow.go | 2 +- storage/filesystem/storage.go | 2 +- 18 files changed, 9 insertions(+), 9 deletions(-) rename storage/filesystem/{internal => }/dotgit/dotgit.go (100%) rename storage/filesystem/{internal => }/dotgit/dotgit_rewrite_packed_refs_nix.go (100%) rename storage/filesystem/{internal => }/dotgit/dotgit_rewrite_packed_refs_norwfs.go (100%) rename storage/filesystem/{internal => }/dotgit/dotgit_rewrite_packed_refs_windows.go (100%) rename storage/filesystem/{internal => }/dotgit/dotgit_setref.go (100%) rename storage/filesystem/{internal => }/dotgit/dotgit_setref_norwfs.go (100%) rename storage/filesystem/{internal => }/dotgit/dotgit_test.go (100%) rename storage/filesystem/{internal => }/dotgit/writers.go (100%) rename storage/filesystem/{internal => }/dotgit/writers_test.go (100%) diff --git a/storage/filesystem/config.go b/storage/filesystem/config.go index 85feaf0a6..be812e424 100644 --- a/storage/filesystem/config.go +++ b/storage/filesystem/config.go @@ -5,7 +5,7 @@ import ( "os" "gopkg.in/src-d/go-git.v4/config" - "gopkg.in/src-d/go-git.v4/storage/filesystem/internal/dotgit" + "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit" "gopkg.in/src-d/go-git.v4/utils/ioutil" ) diff --git a/storage/filesystem/config_test.go b/storage/filesystem/config_test.go index cc03119d3..71c947da6 100644 --- a/storage/filesystem/config_test.go +++ b/storage/filesystem/config_test.go @@ -5,7 +5,7 @@ import ( "os" "gopkg.in/src-d/go-git.v4/config" - "gopkg.in/src-d/go-git.v4/storage/filesystem/internal/dotgit" + "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit" . "gopkg.in/check.v1" "gopkg.in/src-d/go-billy.v4/osfs" diff --git a/storage/filesystem/internal/dotgit/dotgit.go b/storage/filesystem/dotgit/dotgit.go similarity index 100% rename from storage/filesystem/internal/dotgit/dotgit.go rename to storage/filesystem/dotgit/dotgit.go diff --git a/storage/filesystem/internal/dotgit/dotgit_rewrite_packed_refs_nix.go b/storage/filesystem/dotgit/dotgit_rewrite_packed_refs_nix.go similarity index 100% rename from storage/filesystem/internal/dotgit/dotgit_rewrite_packed_refs_nix.go rename to storage/filesystem/dotgit/dotgit_rewrite_packed_refs_nix.go diff --git a/storage/filesystem/internal/dotgit/dotgit_rewrite_packed_refs_norwfs.go b/storage/filesystem/dotgit/dotgit_rewrite_packed_refs_norwfs.go similarity index 100% rename from storage/filesystem/internal/dotgit/dotgit_rewrite_packed_refs_norwfs.go rename to storage/filesystem/dotgit/dotgit_rewrite_packed_refs_norwfs.go diff --git a/storage/filesystem/internal/dotgit/dotgit_rewrite_packed_refs_windows.go b/storage/filesystem/dotgit/dotgit_rewrite_packed_refs_windows.go similarity index 100% rename from storage/filesystem/internal/dotgit/dotgit_rewrite_packed_refs_windows.go rename to storage/filesystem/dotgit/dotgit_rewrite_packed_refs_windows.go diff --git a/storage/filesystem/internal/dotgit/dotgit_setref.go b/storage/filesystem/dotgit/dotgit_setref.go similarity index 100% rename from storage/filesystem/internal/dotgit/dotgit_setref.go rename to storage/filesystem/dotgit/dotgit_setref.go diff --git a/storage/filesystem/internal/dotgit/dotgit_setref_norwfs.go b/storage/filesystem/dotgit/dotgit_setref_norwfs.go similarity index 100% rename from storage/filesystem/internal/dotgit/dotgit_setref_norwfs.go rename to storage/filesystem/dotgit/dotgit_setref_norwfs.go diff --git a/storage/filesystem/internal/dotgit/dotgit_test.go b/storage/filesystem/dotgit/dotgit_test.go similarity index 100% rename from storage/filesystem/internal/dotgit/dotgit_test.go rename to storage/filesystem/dotgit/dotgit_test.go diff --git a/storage/filesystem/internal/dotgit/writers.go b/storage/filesystem/dotgit/writers.go similarity index 100% rename from storage/filesystem/internal/dotgit/writers.go rename to storage/filesystem/dotgit/writers.go diff --git a/storage/filesystem/internal/dotgit/writers_test.go b/storage/filesystem/dotgit/writers_test.go similarity index 100% rename from storage/filesystem/internal/dotgit/writers_test.go rename to storage/filesystem/dotgit/writers_test.go diff --git a/storage/filesystem/index.go b/storage/filesystem/index.go index 092edecf6..2ebf57e61 100644 --- a/storage/filesystem/index.go +++ b/storage/filesystem/index.go @@ -4,7 +4,7 @@ import ( "os" "gopkg.in/src-d/go-git.v4/plumbing/format/index" - "gopkg.in/src-d/go-git.v4/storage/filesystem/internal/dotgit" + "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit" "gopkg.in/src-d/go-git.v4/utils/ioutil" ) diff --git a/storage/filesystem/module.go b/storage/filesystem/module.go index 6f3de3f28..7c8c8d866 100644 --- a/storage/filesystem/module.go +++ b/storage/filesystem/module.go @@ -2,7 +2,7 @@ package filesystem import ( "gopkg.in/src-d/go-git.v4/storage" - "gopkg.in/src-d/go-git.v4/storage/filesystem/internal/dotgit" + "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit" ) type ModuleStorage struct { diff --git a/storage/filesystem/object.go b/storage/filesystem/object.go index 26190fda3..54f268aac 100644 --- a/storage/filesystem/object.go +++ b/storage/filesystem/object.go @@ -11,7 +11,7 @@ import ( "gopkg.in/src-d/go-git.v4/plumbing/format/objfile" "gopkg.in/src-d/go-git.v4/plumbing/format/packfile" "gopkg.in/src-d/go-git.v4/plumbing/storer" - "gopkg.in/src-d/go-git.v4/storage/filesystem/internal/dotgit" + "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit" "gopkg.in/src-d/go-git.v4/storage/memory" "gopkg.in/src-d/go-git.v4/utils/ioutil" diff --git a/storage/filesystem/object_test.go b/storage/filesystem/object_test.go index de8f2b2b9..4b57a6776 100644 --- a/storage/filesystem/object_test.go +++ b/storage/filesystem/object_test.go @@ -2,7 +2,7 @@ package filesystem import ( "gopkg.in/src-d/go-git.v4/plumbing" - "gopkg.in/src-d/go-git.v4/storage/filesystem/internal/dotgit" + "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit" . "gopkg.in/check.v1" "gopkg.in/src-d/go-git-fixtures.v3" diff --git a/storage/filesystem/reference.go b/storage/filesystem/reference.go index 7313f05e8..a891b837b 100644 --- a/storage/filesystem/reference.go +++ b/storage/filesystem/reference.go @@ -3,7 +3,7 @@ package filesystem import ( "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/storer" - "gopkg.in/src-d/go-git.v4/storage/filesystem/internal/dotgit" + "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit" ) type ReferenceStorage struct { diff --git a/storage/filesystem/shallow.go b/storage/filesystem/shallow.go index 173767cb2..502d406da 100644 --- a/storage/filesystem/shallow.go +++ b/storage/filesystem/shallow.go @@ -5,7 +5,7 @@ import ( "fmt" "gopkg.in/src-d/go-git.v4/plumbing" - "gopkg.in/src-d/go-git.v4/storage/filesystem/internal/dotgit" + "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit" "gopkg.in/src-d/go-git.v4/utils/ioutil" ) diff --git a/storage/filesystem/storage.go b/storage/filesystem/storage.go index 82b137c7c..d7aa18bcc 100644 --- a/storage/filesystem/storage.go +++ b/storage/filesystem/storage.go @@ -2,7 +2,7 @@ package filesystem import ( - "gopkg.in/src-d/go-git.v4/storage/filesystem/internal/dotgit" + "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit" "gopkg.in/src-d/go-billy.v4" ) From c39bd4d4a6ba0b0e75a9902c3bbb2064f27a3f6e Mon Sep 17 00:00:00 2001 From: Antonio Jesus Navarro Perez Date: Tue, 5 Jun 2018 18:45:15 +0200 Subject: [PATCH 009/120] Remove println Signed-off-by: Antonio Jesus Navarro Perez --- plumbing/transport/test/receive_pack.go | 1 - 1 file changed, 1 deletion(-) diff --git a/plumbing/transport/test/receive_pack.go b/plumbing/transport/test/receive_pack.go index 6179850ab..57f602dd8 100644 --- a/plumbing/transport/test/receive_pack.go +++ b/plumbing/transport/test/receive_pack.go @@ -231,7 +231,6 @@ func (s *ReceivePackSuite) receivePackNoCheck(c *C, ep *transport.Endpoint, // fixtures are generated with read only permissions, this casuses // errors deleting or modifying files. rootPath := ep.Path - println("STAT", rootPath) stat, err := os.Stat(ep.Path) if rootPath != "" && err == nil && stat.IsDir() { From f01958913fab6e1967c1317b7222d1160212371c Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Wed, 6 Jun 2018 15:18:57 +0200 Subject: [PATCH 010/120] plumbing: object, adds tree path cache to trees. Fixes #793 The cache is used in Tree.FindEntry for faster path search. Signed-off-by: Javi Fontan --- plumbing/object/tree.go | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/plumbing/object/tree.go b/plumbing/object/tree.go index c2399f8cd..30bbcb038 100644 --- a/plumbing/object/tree.go +++ b/plumbing/object/tree.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "path" + "path/filepath" "strings" "gopkg.in/src-d/go-git.v4/plumbing" @@ -34,6 +35,7 @@ type Tree struct { s storer.EncodedObjectStorer m map[string]*TreeEntry + t map[string]*Tree // tree path cache } // GetTree gets a tree from an object storer and decodes it. @@ -111,14 +113,37 @@ func (t *Tree) TreeEntryFile(e *TreeEntry) (*File, error) { // FindEntry search a TreeEntry in this tree or any subtree. func (t *Tree) FindEntry(path string) (*TreeEntry, error) { + if t.t == nil { + t.t = make(map[string]*Tree) + } + pathParts := strings.Split(path, "/") + startingTree := t + pathCurrent := "" + + // search for the longest path in the tree path cache + for i := len(pathParts); i > 1; i-- { + path := filepath.Join(pathParts[:i]...) + + tree, ok := t.t[path] + if ok { + startingTree = tree + pathParts = pathParts[i:] + pathCurrent = path + + break + } + } var tree *Tree var err error - for tree = t; len(pathParts) > 1; pathParts = pathParts[1:] { + for tree = startingTree; len(pathParts) > 1; pathParts = pathParts[1:] { if tree, err = tree.dir(pathParts[0]); err != nil { return nil, err } + + pathCurrent = filepath.Join(pathCurrent, pathParts[0]) + t.t[pathCurrent] = tree } return tree.entry(pathParts[0]) From 88f0dc3d89a0391f9f52d913207556d15a4c2a77 Mon Sep 17 00:00:00 2001 From: kuba-- Date: Thu, 7 Jun 2018 22:23:58 +0200 Subject: [PATCH 011/120] plumbing: packfile, Don't push empty objects. Fixes #840 Signed-off-by: kuba-- --- plumbing/format/packfile/delta_test.go | 21 ++++++++++++++++++--- plumbing/format/packfile/diff_delta.go | 2 +- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/plumbing/format/packfile/delta_test.go b/plumbing/format/packfile/delta_test.go index 42b777ae8..98f53f6d6 100644 --- a/plumbing/format/packfile/delta_test.go +++ b/plumbing/format/packfile/delta_test.go @@ -62,7 +62,7 @@ func (s *DeltaSuite) SetUpSuite(c *C) { target: []piece{{"1", 30}, {"2", 20}, {"7", 40}, {"4", 400}, {"5", 10}}, }, { - description: "A copy operation bigger tan 64kb", + description: "A copy operation bigger than 64kb", base: []piece{{bigRandStr, 1}, {"1", 200}}, target: []piece{{bigRandStr, 1}}, }} @@ -72,12 +72,16 @@ var bigRandStr = randStringBytes(100 * 1024) const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" -func randStringBytes(n int) string { +func randBytes(n int) []byte { b := make([]byte, n) for i := range b { b[i] = letterBytes[rand.Intn(len(letterBytes))] } - return string(b) + return b +} + +func randStringBytes(n int) string { + return string(randBytes(n)) } func (s *DeltaSuite) TestAddDelta(c *C) { @@ -110,3 +114,14 @@ func (s *DeltaSuite) TestIncompleteDelta(c *C) { c.Assert(err, NotNil) c.Assert(result, IsNil) } + +func (s *DeltaSuite) TestMaxCopySizeDelta(c *C) { + baseBuf := randBytes(maxCopySize) + targetBuf := baseBuf[0:] + targetBuf = append(targetBuf, byte(1)) + + delta := DiffDelta(baseBuf, targetBuf) + result, err := PatchDelta(baseBuf, delta) + c.Assert(err, IsNil) + c.Assert(result, DeepEquals, targetBuf) +} diff --git a/plumbing/format/packfile/diff_delta.go b/plumbing/format/packfile/diff_delta.go index 4d56dc103..d35e78aea 100644 --- a/plumbing/format/packfile/diff_delta.go +++ b/plumbing/format/packfile/diff_delta.go @@ -111,7 +111,7 @@ func diffDelta(index *deltaIndex, src []byte, tgt []byte) []byte { rl := l aOffset := offset - for { + for rl > 0 { if rl < maxCopySize { buf.Write(encodeCopyOperation(aOffset, rl)) break From ecd2bd553ce223252d9784572fd47bd9f597618e Mon Sep 17 00:00:00 2001 From: Miguel Molina Date: Fri, 8 Jun 2018 12:31:15 +0200 Subject: [PATCH 012/120] storage: filesystem, make ObjectStorage constructor public Signed-off-by: Miguel Molina --- storage/filesystem/object.go | 5 +++-- storage/filesystem/object_test.go | 10 +++++----- storage/filesystem/storage.go | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/storage/filesystem/object.go b/storage/filesystem/object.go index 54f268aac..9ffe4dcf5 100644 --- a/storage/filesystem/object.go +++ b/storage/filesystem/object.go @@ -26,7 +26,8 @@ type ObjectStorage struct { index map[plumbing.Hash]*packfile.Index } -func newObjectStorage(dir *dotgit.DotGit) (ObjectStorage, error) { +// NewObjectStorage creates a new ObjectStorage with the given .git directory. +func NewObjectStorage(dir *dotgit.DotGit) (ObjectStorage, error) { s := ObjectStorage{ deltaBaseCache: cache.NewObjectLRUDefault(), dir: dir, @@ -166,7 +167,7 @@ func (s *ObjectStorage) EncodedObject(t plumbing.ObjectType, h plumbing.Hash) (p // Create a new object storage with the DotGit(s) and check for the // required hash object. Skip when not found. for _, dg := range dotgits { - o, oe := newObjectStorage(dg) + o, oe := NewObjectStorage(dg) if oe != nil { continue } diff --git a/storage/filesystem/object_test.go b/storage/filesystem/object_test.go index 4b57a6776..ecd6bebc3 100644 --- a/storage/filesystem/object_test.go +++ b/storage/filesystem/object_test.go @@ -24,7 +24,7 @@ var _ = Suite(&FsSuite{ func (s *FsSuite) TestGetFromObjectFile(c *C) { fs := fixtures.ByTag(".git").ByTag("unpacked").One().DotGit() - o, err := newObjectStorage(dotgit.New(fs)) + o, err := NewObjectStorage(dotgit.New(fs)) c.Assert(err, IsNil) expected := plumbing.NewHash("f3dfe29d268303fc6e1bbce268605fc99573406e") @@ -36,7 +36,7 @@ func (s *FsSuite) TestGetFromObjectFile(c *C) { func (s *FsSuite) TestGetFromPackfile(c *C) { fixtures.Basic().ByTag(".git").Test(c, func(f *fixtures.Fixture) { fs := f.DotGit() - o, err := newObjectStorage(dotgit.New(fs)) + o, err := NewObjectStorage(dotgit.New(fs)) c.Assert(err, IsNil) expected := plumbing.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5") @@ -48,7 +48,7 @@ func (s *FsSuite) TestGetFromPackfile(c *C) { func (s *FsSuite) TestGetFromPackfileMultiplePackfiles(c *C) { fs := fixtures.ByTag(".git").ByTag("multi-packfile").One().DotGit() - o, err := newObjectStorage(dotgit.New(fs)) + o, err := NewObjectStorage(dotgit.New(fs)) c.Assert(err, IsNil) expected := plumbing.NewHash("8d45a34641d73851e01d3754320b33bb5be3c4d3") @@ -65,7 +65,7 @@ func (s *FsSuite) TestGetFromPackfileMultiplePackfiles(c *C) { func (s *FsSuite) TestIter(c *C) { fixtures.ByTag(".git").ByTag("packfile").Test(c, func(f *fixtures.Fixture) { fs := f.DotGit() - o, err := newObjectStorage(dotgit.New(fs)) + o, err := NewObjectStorage(dotgit.New(fs)) c.Assert(err, IsNil) iter, err := o.IterEncodedObjects(plumbing.AnyObject) @@ -86,7 +86,7 @@ func (s *FsSuite) TestIterWithType(c *C) { fixtures.ByTag(".git").Test(c, func(f *fixtures.Fixture) { for _, t := range s.Types { fs := f.DotGit() - o, err := newObjectStorage(dotgit.New(fs)) + o, err := NewObjectStorage(dotgit.New(fs)) c.Assert(err, IsNil) iter, err := o.IterEncodedObjects(t) diff --git a/storage/filesystem/storage.go b/storage/filesystem/storage.go index d7aa18bcc..622bb4a8d 100644 --- a/storage/filesystem/storage.go +++ b/storage/filesystem/storage.go @@ -25,7 +25,7 @@ type Storage struct { // NewStorage returns a new Storage backed by a given `fs.Filesystem` func NewStorage(fs billy.Filesystem) (*Storage, error) { dir := dotgit.New(fs) - o, err := newObjectStorage(dir) + o, err := NewObjectStorage(dir) if err != nil { return nil, err } From bf6190841e8b6cd3a216bc056e5b71c73e18c410 Mon Sep 17 00:00:00 2001 From: Eric Billingsley Date: Fri, 8 Jun 2018 15:19:12 -0700 Subject: [PATCH 013/120] plumbing/transport: http, Adds token authentication support [Fixes #858] Signed-off-by: Eric Billingsley --- _examples/branch/main.go | 2 +- plumbing/transport/http/common.go | 25 +++++++++++++++++++++++++ plumbing/transport/http/common_test.go | 13 +++++++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/_examples/branch/main.go b/_examples/branch/main.go index fa1ad0182..ff33ead54 100644 --- a/_examples/branch/main.go +++ b/_examples/branch/main.go @@ -28,7 +28,7 @@ func main() { // Create a new plumbing.HashReference object with the name of the branch // and the hash from the HEAD. The reference name should be a full reference - // name and now a abbreviated one, as is used on the git cli. + // name and not an abbreviated one, as is used on the git cli. // // For tags we should use `refs/tags/%s` instead of `refs/heads/%s` used // for branches. diff --git a/plumbing/transport/http/common.go b/plumbing/transport/http/common.go index 2c337b77a..c03484646 100644 --- a/plumbing/transport/http/common.go +++ b/plumbing/transport/http/common.go @@ -201,6 +201,31 @@ func (a *BasicAuth) String() string { return fmt.Sprintf("%s - %s:%s", a.Name(), a.Username, masked) } +// TokenAuth implements the go-git http.AuthMethod and transport.AuthMethod interfaces +type TokenAuth struct { + Token string +} + +func (a *TokenAuth) setAuth(r *http.Request) { + if a == nil { + return + } + r.Header.Add("Authorization", fmt.Sprintf("Bearer %s", a.Token)) +} + +// Name is name of the auth +func (a *TokenAuth) Name() string { + return "http-token-auth" +} + +func (a *TokenAuth) String() string { + masked := "*******" + if a.Token == "" { + masked = "" + } + return fmt.Sprintf("%s - %s", a.Name(), masked) +} + // Err is a dedicated error to return errors based on status code type Err struct { Response *http.Response diff --git a/plumbing/transport/http/common_test.go b/plumbing/transport/http/common_test.go index 8d57996c5..71eede48f 100644 --- a/plumbing/transport/http/common_test.go +++ b/plumbing/transport/http/common_test.go @@ -54,6 +54,19 @@ func (s *ClientSuite) TestNewBasicAuth(c *C) { c.Assert(a.String(), Equals, "http-basic-auth - foo:*******") } +func (s *ClientSuite) TestNewTokenAuth(c *C) { + a := &TokenAuth{"OAUTH-TOKEN-TEXT"} + + c.Assert(a.Name(), Equals, "http-token-auth") + c.Assert(a.String(), Equals, "http-token-auth - *******") + + // Check header is set correctly + req, err := http.NewRequest("GET", "https://github.com/git-fixtures/basic", nil) + c.Assert(err, Equals, nil) + a.setAuth(req) + c.Assert(req.Header.Get("Authorization"), Equals, "Bearer OAUTH-TOKEN-TEXT") +} + func (s *ClientSuite) TestNewErrOK(c *C) { res := &http.Response{StatusCode: http.StatusOK} err := NewErr(res) From 960fe7b668c7a0ebbd75f8898c95d5c4bdedf1f3 Mon Sep 17 00:00:00 2001 From: Morgan Date: Wed, 13 Jun 2018 10:19:36 +0200 Subject: [PATCH 014/120] Fix documentation for Notes It previously said that it returned all references that are branches, but that's not true. Signed-off-by: Morgan Bazalgette --- repository.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/repository.go b/repository.go index 717381bdb..680522c6b 100644 --- a/repository.go +++ b/repository.go @@ -872,7 +872,8 @@ func (r *Repository) Branches() (storer.ReferenceIter, error) { }, refIter), nil } -// Notes returns all the References that are Branches. +// Notes returns all the References that are notes. For more information: +// https://git-scm.com/docs/git-notes func (r *Repository) Notes() (storer.ReferenceIter, error) { refIter, err := r.Storer.IterReferences() if err != nil { From 2d9816a5e7daea58a1419fef70bfc8d220ffd6a2 Mon Sep 17 00:00:00 2001 From: David Symonds Date: Thu, 21 Jun 2018 13:24:03 +1000 Subject: [PATCH 015/120] packfile: optimise NewIndexFromIdxFile for a very common case Loading from an on-disk idxfile will usually already have the idxfile entries in order, so check that before wasting time on sorting. Signed-off-by: David Symonds --- plumbing/format/packfile/index.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/plumbing/format/packfile/index.go b/plumbing/format/packfile/index.go index 7d8f2ad10..021b2d102 100644 --- a/plumbing/format/packfile/index.go +++ b/plumbing/format/packfile/index.go @@ -31,10 +31,20 @@ func NewIndexFromIdxFile(idxf *idxfile.Idxfile) *Index { byHash: make(map[plumbing.Hash]*idxfile.Entry, idxf.ObjectCount), byOffset: make([]*idxfile.Entry, 0, idxf.ObjectCount), } - for _, e := range idxf.Entries { + sorted := true + for i, e := range idxf.Entries { idx.addUnsorted(e) + if i > 0 && idx.byOffset[i-1].Offset >= e.Offset { + sorted = false + } + } + + // If the idxfile was loaded from a regular packfile index + // then it will already be in offset order, in which case we + // can avoid doing a relatively expensive idempotent sort. + if !sorted { + sort.Sort(orderByOffset(idx.byOffset)) } - sort.Sort(orderByOffset(idx.byOffset)) return idx } From b53ba8dcffd24e25815feff78e8246597e949f21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1ximo=20Cuadros?= Date: Thu, 21 Jun 2018 12:40:21 +0200 Subject: [PATCH 016/120] Remote.Fetch: error on missing remote reference MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Máximo Cuadros --- remote.go | 40 +++++++++++++++++++++++++++++++++++----- remote_test.go | 14 ++++++++++++++ 2 files changed, 49 insertions(+), 5 deletions(-) diff --git a/remote.go b/remote.go index 60461d611..bf4519c71 100644 --- a/remote.go +++ b/remote.go @@ -630,14 +630,29 @@ func calculateRefs( spec = append(spec, refspecTag) } + refs := make(memory.ReferenceStorage) + for _, s := range spec { + if err := doCalculateRefs(s, remoteRefs, refs); err != nil { + return nil, err + } + } + + return refs, nil +} + +func doCalculateRefs( + s config.RefSpec, + remoteRefs storer.ReferenceStorer, + refs memory.ReferenceStorage, +) error { iter, err := remoteRefs.IterReferences() if err != nil { - return nil, err + return err } - refs := make(memory.ReferenceStorage) - return refs, iter.ForEach(func(ref *plumbing.Reference) error { - if !config.MatchAny(spec, ref.Name()) { + var matched bool + err = iter.ForEach(func(ref *plumbing.Reference) error { + if !s.Match(ref.Name()) { return nil } @@ -654,8 +669,23 @@ func calculateRefs( return nil } - return refs.SetReference(ref) + matched = true + if err := refs.SetReference(ref); err != nil { + return err + } + + if !s.IsWildcard() { + return storer.ErrStop + } + + return nil }) + + if !matched && !s.IsWildcard() { + return fmt.Errorf("couldn't find remote ref %q", s.Src()) + } + + return err } func getWants(localStorer storage.Storer, refs memory.ReferenceStorage) ([]plumbing.Hash, error) { diff --git a/remote_test.go b/remote_test.go index 82ec1fc29..dd386b083 100644 --- a/remote_test.go +++ b/remote_test.go @@ -100,6 +100,20 @@ func (s *RemoteSuite) TestFetch(c *C) { }) } +func (s *RemoteSuite) TestFetchNonExistantReference(c *C) { + r := newRemote(memory.NewStorage(), &config.RemoteConfig{ + URLs: []string{s.GetLocalRepositoryURL(fixtures.ByTag("tags").One())}, + }) + + err := r.Fetch(&FetchOptions{ + RefSpecs: []config.RefSpec{ + config.RefSpec("+refs/heads/foo:refs/remotes/origin/foo"), + }, + }) + + c.Assert(err, ErrorMatches, "couldn't find remote ref.*") +} + func (s *RemoteSuite) TestFetchContext(c *C) { r := newRemote(memory.NewStorage(), &config.RemoteConfig{ URLs: []string{s.GetLocalRepositoryURL(fixtures.ByTag("tags").One())}, From da5d474fb43dffd1b28cd5662b3a5bf7e446cd5c Mon Sep 17 00:00:00 2001 From: "Santiago M. Mola" Date: Fri, 15 Jun 2018 17:20:51 +0200 Subject: [PATCH 017/120] storage/filesystem: avoid norwfs build flag norwfs build flag was used to work on filesystems that do not support neither opening a file in read/write mode or renaming a file (e.f. sivafs). This had two problems: - go-git could not be compiled to work properly both with regular filesystems and limited filesystems at the same time. - the norwfs trick was not available on Windows. This PR removes the norwfs build flag, as well as the windows conditional flag on the dotgit package. For the file open mode, we use the new billy capabilities, to check at runtime if the filesystem supports opening a file in read/write mode or not. For the renaming, we just try and fallback to alternative methods if billy.ErrNotSupported is returned. Signed-off-by: Santiago M. Mola --- storage/filesystem/dotgit/dotgit.go | 2 +- .../dotgit/dotgit_rewrite_packed_refs.go | 81 +++++++++++++++++++ .../dotgit/dotgit_rewrite_packed_refs_nix.go | 17 ---- .../dotgit_rewrite_packed_refs_norwfs.go | 34 -------- .../dotgit_rewrite_packed_refs_windows.go | 42 ---------- 5 files changed, 82 insertions(+), 94 deletions(-) create mode 100644 storage/filesystem/dotgit/dotgit_rewrite_packed_refs.go delete mode 100644 storage/filesystem/dotgit/dotgit_rewrite_packed_refs_nix.go delete mode 100644 storage/filesystem/dotgit/dotgit_rewrite_packed_refs_norwfs.go delete mode 100644 storage/filesystem/dotgit/dotgit_rewrite_packed_refs_windows.go diff --git a/storage/filesystem/dotgit/dotgit.go b/storage/filesystem/dotgit/dotgit.go index 52b621c58..dc12f23cf 100644 --- a/storage/filesystem/dotgit/dotgit.go +++ b/storage/filesystem/dotgit/dotgit.go @@ -469,7 +469,7 @@ func (d *DotGit) openAndLockPackedRefs(doCreate bool) ( // File mode is retrieved from a constant defined in the target specific // files (dotgit_rewrite_packed_refs_*). Some modes are not available // in all filesystems. - openFlags := openAndLockPackedRefsMode + openFlags := d.openAndLockPackedRefsMode() if doCreate { openFlags |= os.O_CREATE } diff --git a/storage/filesystem/dotgit/dotgit_rewrite_packed_refs.go b/storage/filesystem/dotgit/dotgit_rewrite_packed_refs.go new file mode 100644 index 000000000..7f1c02c15 --- /dev/null +++ b/storage/filesystem/dotgit/dotgit_rewrite_packed_refs.go @@ -0,0 +1,81 @@ +package dotgit + +import ( + "io" + "os" + "runtime" + + "gopkg.in/src-d/go-billy.v4" + "gopkg.in/src-d/go-git.v4/utils/ioutil" +) + +func (d *DotGit) openAndLockPackedRefsMode() int { + if billy.CapabilityCheck(d.fs, billy.ReadAndWriteCapability) { + return os.O_RDWR + } + + return os.O_RDONLY +} + +func (d *DotGit) rewritePackedRefsWhileLocked( + tmp billy.File, pr billy.File) error { + // Try plain rename. If we aren't using the bare Windows filesystem as the + // storage layer, we might be able to get away with a rename over a locked + // file. + err := d.fs.Rename(tmp.Name(), pr.Name()) + if err == nil { + return nil + } + + // If we are in a filesystem that does not support rename (e.g. sivafs) + // a full copy is done. + if err == billy.ErrNotSupported { + return d.copyNewFile(tmp, pr) + } + + if runtime.GOOS != "windows" { + return err + } + + // Otherwise, Windows doesn't let us rename over a locked file, so + // we have to do a straight copy. Unfortunately this could result + // in a partially-written file if the process fails before the + // copy completes. + return d.copyToExistingFile(tmp, pr) +} + +func (d *DotGit) copyToExistingFile(tmp, pr billy.File) error { + _, err := pr.Seek(0, io.SeekStart) + if err != nil { + return err + } + err = pr.Truncate(0) + if err != nil { + return err + } + _, err = tmp.Seek(0, io.SeekStart) + if err != nil { + return err + } + _, err = io.Copy(pr, tmp) + + return err +} + +func (d *DotGit) copyNewFile(tmp billy.File, pr billy.File) (err error) { + prWrite, err := d.fs.Create(pr.Name()) + if err != nil { + return err + } + + defer ioutil.CheckClose(prWrite, &err) + + _, err = tmp.Seek(0, io.SeekStart) + if err != nil { + return err + } + + _, err = io.Copy(prWrite, tmp) + + return err +} diff --git a/storage/filesystem/dotgit/dotgit_rewrite_packed_refs_nix.go b/storage/filesystem/dotgit/dotgit_rewrite_packed_refs_nix.go deleted file mode 100644 index c76079321..000000000 --- a/storage/filesystem/dotgit/dotgit_rewrite_packed_refs_nix.go +++ /dev/null @@ -1,17 +0,0 @@ -// +build !windows,!norwfs - -package dotgit - -import ( - "os" - - "gopkg.in/src-d/go-billy.v4" -) - -const openAndLockPackedRefsMode = os.O_RDWR - -func (d *DotGit) rewritePackedRefsWhileLocked( - tmp billy.File, pr billy.File) error { - // On non-Windows platforms, we can have atomic rename. - return d.fs.Rename(tmp.Name(), pr.Name()) -} diff --git a/storage/filesystem/dotgit/dotgit_rewrite_packed_refs_norwfs.go b/storage/filesystem/dotgit/dotgit_rewrite_packed_refs_norwfs.go deleted file mode 100644 index 6e43b4285..000000000 --- a/storage/filesystem/dotgit/dotgit_rewrite_packed_refs_norwfs.go +++ /dev/null @@ -1,34 +0,0 @@ -// +build norwfs - -package dotgit - -import ( - "io" - "os" - - "gopkg.in/src-d/go-billy.v4" -) - -const openAndLockPackedRefsMode = os.O_RDONLY - -// Instead of renaming that can not be supported in simpler filesystems -// a full copy is done. -func (d *DotGit) rewritePackedRefsWhileLocked( - tmp billy.File, pr billy.File) error { - - prWrite, err := d.fs.Create(pr.Name()) - if err != nil { - return err - } - - defer prWrite.Close() - - _, err = tmp.Seek(0, io.SeekStart) - if err != nil { - return err - } - - _, err = io.Copy(prWrite, tmp) - - return err -} diff --git a/storage/filesystem/dotgit/dotgit_rewrite_packed_refs_windows.go b/storage/filesystem/dotgit/dotgit_rewrite_packed_refs_windows.go deleted file mode 100644 index 897d2c94c..000000000 --- a/storage/filesystem/dotgit/dotgit_rewrite_packed_refs_windows.go +++ /dev/null @@ -1,42 +0,0 @@ -// +build windows,!norwfs - -package dotgit - -import ( - "io" - "os" - - "gopkg.in/src-d/go-billy.v4" -) - -const openAndLockPackedRefsMode = os.O_RDWR - -func (d *DotGit) rewritePackedRefsWhileLocked( - tmp billy.File, pr billy.File) error { - // If we aren't using the bare Windows filesystem as the storage - // layer, we might be able to get away with a rename over a locked - // file. - err := d.fs.Rename(tmp.Name(), pr.Name()) - if err == nil { - return nil - } - - // Otherwise, Windows doesn't let us rename over a locked file, so - // we have to do a straight copy. Unfortunately this could result - // in a partially-written file if the process fails before the - // copy completes. - _, err = pr.Seek(0, io.SeekStart) - if err != nil { - return err - } - err = pr.Truncate(0) - if err != nil { - return err - } - _, err = tmp.Seek(0, io.SeekStart) - if err != nil { - return err - } - _, err = io.Copy(pr, tmp) - return err -} From 7d9b66fd9f707b31ac80c39d8027772de86c291d Mon Sep 17 00:00:00 2001 From: Marc Barussaud Date: Tue, 26 Jun 2018 11:59:22 +0200 Subject: [PATCH 018/120] utils: diff, skip useless rune->string conversion According to library documentation : https://github.com/sergi/go-diff/blob/master/diffmatchpatch/diff.go#L391 Signed-off-by: Marc Barussaud --- utils/diff/diff.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/diff/diff.go b/utils/diff/diff.go index b840ad6d7..f49ae55ba 100644 --- a/utils/diff/diff.go +++ b/utils/diff/diff.go @@ -16,8 +16,8 @@ import ( // string into the dst string. func Do(src, dst string) (diffs []diffmatchpatch.Diff) { dmp := diffmatchpatch.New() - wSrc, wDst, warray := dmp.DiffLinesToChars(src, dst) - diffs = dmp.DiffMain(wSrc, wDst, false) + wSrc, wDst, warray := dmp.DiffLinesToRunes(src, dst) + diffs = dmp.DiffMainRunes(wSrc, wDst, false) diffs = dmp.DiffCharsToLines(diffs, warray) return diffs } From 9251ea764df3de13518f974635e76315b2b89e3e Mon Sep 17 00:00:00 2001 From: Marc Barussaud Date: Tue, 26 Jun 2018 15:23:19 +0200 Subject: [PATCH 019/120] plumbing: add context to allow cancel on diff/patch computing Signed-off-by: Marc Barussaud --- plumbing/object/change.go | 21 +++++++++- plumbing/object/change_test.go | 61 ++++++++++++++++++++++++++++ plumbing/object/commit.go | 11 +++++- plumbing/object/commit_test.go | 66 +++++++++++++++++++++++++++++++ plumbing/object/difftree.go | 13 +++++- plumbing/object/patch.go | 32 ++++++++++++++- plumbing/object/tree.go | 20 +++++++++- utils/merkletrie/difftree.go | 21 ++++++++++ utils/merkletrie/difftree_test.go | 61 ++++++++++++++++++++++++++++ 9 files changed, 297 insertions(+), 9 deletions(-) diff --git a/plumbing/object/change.go b/plumbing/object/change.go index 729ff5a3c..a1b4c2749 100644 --- a/plumbing/object/change.go +++ b/plumbing/object/change.go @@ -2,6 +2,7 @@ package object import ( "bytes" + "context" "fmt" "strings" @@ -81,7 +82,15 @@ func (c *Change) String() string { // Patch returns a Patch with all the file changes in chunks. This // representation can be used to create several diff outputs. func (c *Change) Patch() (*Patch, error) { - return getPatch("", c) + return c.PatchContext(context.Background()) +} + +// Patch returns a Patch with all the file changes in chunks. This +// representation can be used to create several diff outputs. +// If context expires, an non-nil error will be returned +// Provided context must be non-nil +func (c *Change) PatchContext(ctx context.Context) (*Patch, error) { + return getPatchContext(ctx, "", c) } func (c *Change) name() string { @@ -136,5 +145,13 @@ func (c Changes) String() string { // Patch returns a Patch with all the changes in chunks. This // representation can be used to create several diff outputs. func (c Changes) Patch() (*Patch, error) { - return getPatch("", c...) + return c.PatchContext(context.Background()) +} + +// Patch returns a Patch with all the changes in chunks. This +// representation can be used to create several diff outputs. +// If context expires, an non-nil error will be returned +// Provided context must be non-nil +func (c Changes) PatchContext(ctx context.Context) (*Patch, error) { + return getPatchContext(ctx, "", c...) } diff --git a/plumbing/object/change_test.go b/plumbing/object/change_test.go index 7036fa3b2..b0e89c711 100644 --- a/plumbing/object/change_test.go +++ b/plumbing/object/change_test.go @@ -1,6 +1,7 @@ package object import ( + "context" "sort" "gopkg.in/src-d/go-git.v4/plumbing" @@ -82,6 +83,12 @@ func (s *ChangeSuite) TestInsert(c *C) { c.Assert(len(p.FilePatches()[0].Chunks()), Equals, 1) c.Assert(p.FilePatches()[0].Chunks()[0].Type(), Equals, diff.Add) + p, err = change.PatchContext(context.Background()) + c.Assert(err, IsNil) + c.Assert(len(p.FilePatches()), Equals, 1) + c.Assert(len(p.FilePatches()[0].Chunks()), Equals, 1) + c.Assert(p.FilePatches()[0].Chunks()[0].Type(), Equals, diff.Add) + str := change.String() c.Assert(str, Equals, "") } @@ -134,6 +141,12 @@ func (s *ChangeSuite) TestDelete(c *C) { c.Assert(len(p.FilePatches()[0].Chunks()), Equals, 1) c.Assert(p.FilePatches()[0].Chunks()[0].Type(), Equals, diff.Delete) + p, err = change.PatchContext(context.Background()) + c.Assert(err, IsNil) + c.Assert(len(p.FilePatches()), Equals, 1) + c.Assert(len(p.FilePatches()[0].Chunks()), Equals, 1) + c.Assert(p.FilePatches()[0].Chunks()[0].Type(), Equals, diff.Delete) + str := change.String() c.Assert(str, Equals, "") } @@ -206,6 +219,18 @@ func (s *ChangeSuite) TestModify(c *C) { c.Assert(p.FilePatches()[0].Chunks()[5].Type(), Equals, diff.Add) c.Assert(p.FilePatches()[0].Chunks()[6].Type(), Equals, diff.Equal) + p, err = change.PatchContext(context.Background()) + c.Assert(err, IsNil) + c.Assert(len(p.FilePatches()), Equals, 1) + c.Assert(len(p.FilePatches()[0].Chunks()), Equals, 7) + c.Assert(p.FilePatches()[0].Chunks()[0].Type(), Equals, diff.Equal) + c.Assert(p.FilePatches()[0].Chunks()[1].Type(), Equals, diff.Delete) + c.Assert(p.FilePatches()[0].Chunks()[2].Type(), Equals, diff.Add) + c.Assert(p.FilePatches()[0].Chunks()[3].Type(), Equals, diff.Equal) + c.Assert(p.FilePatches()[0].Chunks()[4].Type(), Equals, diff.Delete) + c.Assert(p.FilePatches()[0].Chunks()[5].Type(), Equals, diff.Add) + c.Assert(p.FilePatches()[0].Chunks()[6].Type(), Equals, diff.Equal) + str := change.String() c.Assert(str, Equals, "") } @@ -367,3 +392,39 @@ func (s *ChangeSuite) TestChangesSort(c *C) { sort.Sort(changes) c.Assert(changes.String(), Equals, expected) } + +func (s *ChangeSuite) TestCancel(c *C) { + // Commit a5078b19f08f63e7948abd0a5e2fb7d319d3a565 of the go-git + // fixture inserted "examples/clone/main.go". + // + // On that commit, the "examples/clone" tree is + // 6efca3ff41cab651332f9ebc0c96bb26be809615 + // + // and the "examples/colone/main.go" is + // f95dc8f7923add1a8b9f72ecb1e8db1402de601a + + path := "examples/clone/main.go" + name := "main.go" + mode := filemode.Regular + blob := plumbing.NewHash("f95dc8f7923add1a8b9f72ecb1e8db1402de601a") + tree := plumbing.NewHash("6efca3ff41cab651332f9ebc0c96bb26be809615") + + change := &Change{ + From: empty, + To: ChangeEntry{ + Name: path, + Tree: s.tree(c, tree), + TreeEntry: TreeEntry{ + Name: name, + Mode: mode, + Hash: blob, + }, + }, + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + p, err := change.PatchContext(ctx) + c.Assert(p, IsNil) + c.Assert(err, ErrorMatches, "operation canceled") +} diff --git a/plumbing/object/commit.go b/plumbing/object/commit.go index c9a4c0ee8..3ed85ba3b 100644 --- a/plumbing/object/commit.go +++ b/plumbing/object/commit.go @@ -3,6 +3,7 @@ package object import ( "bufio" "bytes" + "context" "errors" "fmt" "io" @@ -75,7 +76,8 @@ func (c *Commit) Tree() (*Tree, error) { } // Patch returns the Patch between the actual commit and the provided one. -func (c *Commit) Patch(to *Commit) (*Patch, error) { +// Error will be return if context expires. Provided context must be non-nil +func (c *Commit) PatchContext(ctx context.Context, to *Commit) (*Patch, error) { fromTree, err := c.Tree() if err != nil { return nil, err @@ -86,7 +88,12 @@ func (c *Commit) Patch(to *Commit) (*Patch, error) { return nil, err } - return fromTree.Patch(toTree) + return fromTree.PatchContext(ctx, toTree) +} + +// Patch returns the Patch between the actual commit and the provided one. +func (c *Commit) Patch(to *Commit) (*Patch, error) { + return c.PatchContext(context.Background(), to) } // Parents return a CommitIter to the parent Commits. diff --git a/plumbing/object/commit_test.go b/plumbing/object/commit_test.go index 191b14d8a..996d4818a 100644 --- a/plumbing/object/commit_test.go +++ b/plumbing/object/commit_test.go @@ -2,6 +2,7 @@ package object import ( "bytes" + "context" "io" "strings" "time" @@ -132,6 +133,59 @@ Binary files /dev/null and b/binary.jpg differ c.Assert(buf.String(), Equals, patch.String()) } +func (s *SuiteCommit) TestPatchContext(c *C) { + from := s.commit(c, plumbing.NewHash("918c48b83bd081e863dbe1b80f8998f058cd8294")) + to := s.commit(c, plumbing.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5")) + + patch, err := from.PatchContext(context.Background(), to) + c.Assert(err, IsNil) + + buf := bytes.NewBuffer(nil) + err = patch.Encode(buf) + c.Assert(err, IsNil) + + c.Assert(buf.String(), Equals, `diff --git a/vendor/foo.go b/vendor/foo.go +new file mode 100644 +index 0000000000000000000000000000000000000000..9dea2395f5403188298c1dabe8bdafe562c491e3 +--- /dev/null ++++ b/vendor/foo.go +@@ -0,0 +1,7 @@ ++package main ++ ++import "fmt" ++ ++func main() { ++ fmt.Println("Hello, playground") ++} +`) + c.Assert(buf.String(), Equals, patch.String()) + + from = s.commit(c, plumbing.NewHash("b8e471f58bcbca63b07bda20e428190409c2db47")) + to = s.commit(c, plumbing.NewHash("35e85108805c84807bc66a02d91535e1e24b38b9")) + + patch, err = from.PatchContext(context.Background(), to) + c.Assert(err, IsNil) + + buf.Reset() + err = patch.Encode(buf) + c.Assert(err, IsNil) + + c.Assert(buf.String(), Equals, `diff --git a/CHANGELOG b/CHANGELOG +deleted file mode 100644 +index d3ff53e0564a9f87d8e84b6e28e5060e517008aa..0000000000000000000000000000000000000000 +--- a/CHANGELOG ++++ /dev/null +@@ -1 +0,0 @@ +-Initial changelog +diff --git a/binary.jpg b/binary.jpg +new file mode 100644 +index 0000000000000000000000000000000000000000..d5c0f4ab811897cadf03aec358ae60d21f91c50d +Binary files /dev/null and b/binary.jpg differ +`) + + c.Assert(buf.String(), Equals, patch.String()) +} + func (s *SuiteCommit) TestCommitEncodeDecodeIdempotent(c *C) { ts, err := time.Parse(time.RFC3339, "2006-01-02T15:04:05-07:00") c.Assert(err, IsNil) @@ -363,3 +417,15 @@ sYyf9RfOnw/KUFAQbdtvLx3ikODQC+D3KBtuKI9ISHQfgw== _, ok := e.Identities["Sunny "] c.Assert(ok, Equals, true) } + +func (s *SuiteCommit) TestPatchCancel(c *C) { + from := s.commit(c, plumbing.NewHash("918c48b83bd081e863dbe1b80f8998f058cd8294")) + to := s.commit(c, plumbing.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5")) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + patch, err := from.PatchContext(ctx, to) + c.Assert(patch, IsNil) + c.Assert(err, ErrorMatches, "operation canceled") + +} diff --git a/plumbing/object/difftree.go b/plumbing/object/difftree.go index ac58c4dcb..a30a29e37 100644 --- a/plumbing/object/difftree.go +++ b/plumbing/object/difftree.go @@ -2,6 +2,7 @@ package object import ( "bytes" + "context" "gopkg.in/src-d/go-git.v4/utils/merkletrie" "gopkg.in/src-d/go-git.v4/utils/merkletrie/noder" @@ -10,6 +11,13 @@ import ( // DiffTree compares the content and mode of the blobs found via two // tree objects. func DiffTree(a, b *Tree) (Changes, error) { + return DiffTreeContext(context.Background(), a, b) +} + +// DiffTree compares the content and mode of the blobs found via two +// tree objects. Provided context must be non-nil. +// An error will be return if context expires +func DiffTreeContext(ctx context.Context, a, b *Tree) (Changes, error) { from := NewTreeRootNode(a) to := NewTreeRootNode(b) @@ -17,8 +25,11 @@ func DiffTree(a, b *Tree) (Changes, error) { return bytes.Equal(a.Hash(), b.Hash()) } - merkletrieChanges, err := merkletrie.DiffTree(from, to, hashEqual) + merkletrieChanges, err := merkletrie.DiffTreeContext(ctx, from, to, hashEqual) if err != nil { + if err == merkletrie.ErrCanceled { + return nil, ErrCanceled + } return nil, err } diff --git a/plumbing/object/patch.go b/plumbing/object/patch.go index aa96a96c8..adeaccb0a 100644 --- a/plumbing/object/patch.go +++ b/plumbing/object/patch.go @@ -2,6 +2,8 @@ package object import ( "bytes" + "context" + "errors" "fmt" "io" "math" @@ -15,10 +17,25 @@ import ( dmp "github.com/sergi/go-diff/diffmatchpatch" ) +var ( + ErrCanceled = errors.New("operation canceled") +) + func getPatch(message string, changes ...*Change) (*Patch, error) { + ctx := context.Background() + return getPatchContext(ctx, message, changes...) +} + +func getPatchContext(ctx context.Context, message string, changes ...*Change) (*Patch, error) { var filePatches []fdiff.FilePatch for _, c := range changes { - fp, err := filePatch(c) + select { + case <-ctx.Done(): + return nil, ErrCanceled + default: + } + + fp, err := filePatchWithContext(ctx, c) if err != nil { return nil, err } @@ -29,7 +46,7 @@ func getPatch(message string, changes ...*Change) (*Patch, error) { return &Patch{message, filePatches}, nil } -func filePatch(c *Change) (fdiff.FilePatch, error) { +func filePatchWithContext(ctx context.Context, c *Change) (fdiff.FilePatch, error) { from, to, err := c.Files() if err != nil { return nil, err @@ -52,6 +69,12 @@ func filePatch(c *Change) (fdiff.FilePatch, error) { var chunks []fdiff.Chunk for _, d := range diffs { + select { + case <-ctx.Done(): + return nil, ErrCanceled + default: + } + var op fdiff.Operation switch d.Type { case dmp.DiffEqual: @@ -70,6 +93,11 @@ func filePatch(c *Change) (fdiff.FilePatch, error) { from: c.From, to: c.To, }, nil + +} + +func filePatch(c *Change) (fdiff.FilePatch, error) { + return filePatchWithContext(context.Background(), c) } func fileContent(f *File) (content string, isBinary bool, err error) { diff --git a/plumbing/object/tree.go b/plumbing/object/tree.go index 30bbcb038..86d19c01d 100644 --- a/plumbing/object/tree.go +++ b/plumbing/object/tree.go @@ -2,6 +2,7 @@ package object import ( "bufio" + "context" "errors" "fmt" "io" @@ -295,15 +296,30 @@ func (from *Tree) Diff(to *Tree) (Changes, error) { return DiffTree(from, to) } +// Diff returns a list of changes between this tree and the provided one +// Error will be returned if context expires +// Provided context must be non nil +func (from *Tree) DiffContext(ctx context.Context, to *Tree) (Changes, error) { + return DiffTreeContext(ctx, from, to) +} + // Patch returns a slice of Patch objects with all the changes between trees // in chunks. This representation can be used to create several diff outputs. func (from *Tree) Patch(to *Tree) (*Patch, error) { - changes, err := DiffTree(from, to) + return from.PatchContext(context.Background(), to) +} + +// Patch returns a slice of Patch objects with all the changes between trees +// in chunks. This representation can be used to create several diff outputs. +// If context expires, an error will be returned +// Provided context must be non-nil +func (from *Tree) PatchContext(ctx context.Context, to *Tree) (*Patch, error) { + changes, err := DiffTreeContext(ctx, from, to) if err != nil { return nil, err } - return changes.Patch() + return changes.PatchContext(ctx) } // treeEntryIter facilitates iterating through the TreeEntry objects in a Tree. diff --git a/utils/merkletrie/difftree.go b/utils/merkletrie/difftree.go index 229409688..d57ed1333 100644 --- a/utils/merkletrie/difftree.go +++ b/utils/merkletrie/difftree.go @@ -248,14 +248,29 @@ package merkletrie // h: else of i import ( + "context" + "errors" "fmt" "gopkg.in/src-d/go-git.v4/utils/merkletrie/noder" ) +var ( + ErrCanceled = errors.New("operation canceled") +) + // DiffTree calculates the list of changes between two merkletries. It // uses the provided hashEqual callback to compare noders. func DiffTree(fromTree, toTree noder.Noder, + hashEqual noder.Equal) (Changes, error) { + return DiffTreeContext(context.Background(), fromTree, toTree, hashEqual) +} + +// DiffTree calculates the list of changes between two merkletries. It +// uses the provided hashEqual callback to compare noders. +// Error will be returned if context expires +// Provided context must be non nil +func DiffTreeContext(ctx context.Context, fromTree, toTree noder.Noder, hashEqual noder.Equal) (Changes, error) { ret := NewChanges() @@ -265,6 +280,12 @@ func DiffTree(fromTree, toTree noder.Noder, } for { + select { + case <-ctx.Done(): + return nil, ErrCanceled + default: + } + from := ii.from.current to := ii.to.current diff --git a/utils/merkletrie/difftree_test.go b/utils/merkletrie/difftree_test.go index 9f033b132..ab0eb5772 100644 --- a/utils/merkletrie/difftree_test.go +++ b/utils/merkletrie/difftree_test.go @@ -2,6 +2,7 @@ package merkletrie_test import ( "bytes" + ctx "context" "fmt" "reflect" "sort" @@ -61,9 +62,45 @@ func (t diffTreeTest) innerRun(c *C, context string, reverse bool) { c.Assert(obtained, changesEquals, expected, comment) } +func (t diffTreeTest) innerRunCtx(c *C, context string, reverse bool) { + comment := Commentf("\n%s", context) + if reverse { + comment = Commentf("%s [REVERSED]", comment.CheckCommentString()) + } + + a, err := fsnoder.New(t.from) + c.Assert(err, IsNil, comment) + comment = Commentf("%s\n\t from = %s", comment.CheckCommentString(), a) + + b, err := fsnoder.New(t.to) + c.Assert(err, IsNil, comment) + comment = Commentf("%s\n\t to = %s", comment.CheckCommentString(), b) + + expected, err := newChangesFromString(t.expected) + c.Assert(err, IsNil, comment) + + if reverse { + a, b = b, a + expected = expected.reverse() + } + comment = Commentf("%s\n\texpected = %s", comment.CheckCommentString(), expected) + + results, err := merkletrie.DiffTreeContext(ctx.Background(), a, b, fsnoder.HashEqual) + c.Assert(err, IsNil, comment) + + obtained, err := newChanges(results) + c.Assert(err, IsNil, comment) + + comment = Commentf("%s\n\tobtained = %s", comment.CheckCommentString(), obtained) + + c.Assert(obtained, changesEquals, expected, comment) +} + func (t diffTreeTest) run(c *C, context string) { t.innerRun(c, context, false) t.innerRun(c, context, true) + t.innerRunCtx(c, context, false) + t.innerRunCtx(c, context, true) } type change struct { @@ -437,3 +474,27 @@ func (s *DiffTreeSuite) TestIssue275(c *C) { }, }) } + +func (s *DiffTreeSuite) TestCancel(c *C) { + t := diffTreeTest{"()", "(a<> b<1> c() d<> e<2> f())", "+a +b +d +e"} + comment := Commentf("\n%s", "test cancel:") + + a, err := fsnoder.New(t.from) + c.Assert(err, IsNil, comment) + comment = Commentf("%s\n\t from = %s", comment.CheckCommentString(), a) + + b, err := fsnoder.New(t.to) + c.Assert(err, IsNil, comment) + comment = Commentf("%s\n\t to = %s", comment.CheckCommentString(), b) + + expected, err := newChangesFromString(t.expected) + c.Assert(err, IsNil, comment) + + comment = Commentf("%s\n\texpected = %s", comment.CheckCommentString(), expected) + context, cancel := ctx.WithCancel(ctx.Background()) + cancel() + results, err := merkletrie.DiffTreeContext(context, a, b, fsnoder.HashEqual) + c.Assert(results, IsNil, comment) + c.Assert(err, ErrorMatches, "operation canceled") + +} From e1c269422ab209443b80f05095627c2795e32fd5 Mon Sep 17 00:00:00 2001 From: Mark Bartel Date: Mon, 2 Jul 2018 14:34:22 -0400 Subject: [PATCH 020/120] worktree: add test for correct tree sorting (issue #881) Signed-off-by: Mark Bartel --- worktree_commit_test.go | 54 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/worktree_commit_test.go b/worktree_commit_test.go index 5575bca1a..2100626a4 100644 --- a/worktree_commit_test.go +++ b/worktree_commit_test.go @@ -8,9 +8,15 @@ import ( "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/storage/memory" + "bytes" . "gopkg.in/check.v1" "gopkg.in/src-d/go-billy.v4/memfs" + "gopkg.in/src-d/go-billy.v4/osfs" "gopkg.in/src-d/go-billy.v4/util" + "gopkg.in/src-d/go-git.v4/storage/filesystem" + "io/ioutil" + "os" + "os/exec" ) func (s *WorktreeSuite) TestCommitInvalidOptions(c *C) { @@ -135,6 +141,54 @@ func (s *WorktreeSuite) TestRemoveAndCommitAll(c *C) { assertStorageStatus(c, s.Repository, 13, 11, 11, expected) } +func (s *WorktreeSuite) TestCommitTreeSort(c *C) { + path, err := ioutil.TempDir(os.TempDir(), "test-commit-tree-sort") + c.Assert(err, IsNil) + fs := osfs.New(path) + st, err := filesystem.NewStorage(fs) + c.Assert(err, IsNil) + r, err := Init(st, nil) + c.Assert(err, IsNil) + + r, err = Clone(memory.NewStorage(), memfs.New(), &CloneOptions{ + URL: path, + }) + + w, err := r.Worktree() + c.Assert(err, IsNil) + + mfs := w.Filesystem + + err = mfs.MkdirAll("delta", 0755) + c.Assert(err, IsNil) + + for _, p := range []string{"delta_last", "Gamma", "delta/middle", "Beta", "delta-first", "alpha"} { + util.WriteFile(mfs, p, []byte("foo"), 0644) + _, err = w.Add(p) + c.Assert(err, IsNil) + } + + _, err = w.Commit("foo\n", &CommitOptions{ + All: true, + Author: defaultSignature(), + }) + c.Assert(err, IsNil) + + err = r.Push(&PushOptions{}) + c.Assert(err, IsNil) + + cmd := exec.Command("git", "fsck") + cmd.Dir = path + cmd.Env = os.Environ() + buf := &bytes.Buffer{} + cmd.Stderr = buf + cmd.Stdout = buf + + err = cmd.Run() + + c.Assert(err, IsNil, Commentf("%s", buf.Bytes())) +} + func assertStorageStatus( c *C, r *Repository, treesCount, blobCount, commitCount int, head plumbing.Hash, From 7ff71b5a66eea98983c610d33523041a5a829e2f Mon Sep 17 00:00:00 2001 From: Mark Bartel Date: Tue, 3 Jul 2018 17:20:57 -0400 Subject: [PATCH 021/120] worktree: sort the tree object. Fixes #881 Signed-off-by: Mark Bartel --- worktree_commit.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/worktree_commit.go b/worktree_commit.go index 5fa63ab0a..ee5cd8240 100644 --- a/worktree_commit.go +++ b/worktree_commit.go @@ -11,6 +11,7 @@ import ( "gopkg.in/src-d/go-git.v4/storage" "gopkg.in/src-d/go-billy.v4" + "sort" ) // Commit stores the current contents of the index in a new commit along with @@ -162,7 +163,20 @@ func (h *buildTreeHelper) doBuildTree(e *index.Entry, parent, fullpath string) { h.trees[parent].Entries = append(h.trees[parent].Entries, te) } +type sortableEntries []object.TreeEntry + +func (sortableEntries) sortName(te object.TreeEntry) string { + if te.Mode == filemode.Dir { + return te.Name + "/" + } + return te.Name +} +func (se sortableEntries) Len() int { return len(se) } +func (se sortableEntries) Less(i int, j int) bool { return se.sortName(se[i]) < se.sortName(se[j]) } +func (se sortableEntries) Swap(i int, j int) { se[i], se[j] = se[j], se[i] } + func (h *buildTreeHelper) copyTreeToStorageRecursive(parent string, t *object.Tree) (plumbing.Hash, error) { + sort.Sort(sortableEntries(t.Entries)) for i, e := range t.Entries { if e.Mode != filemode.Dir && !e.Hash.IsZero() { continue From f0c4318e7b8d5cbf91723c71a4ba20d1bd0cfaf5 Mon Sep 17 00:00:00 2001 From: Mark Bartel Date: Sat, 7 Jul 2018 23:20:05 -0400 Subject: [PATCH 022/120] worktree: address PR comments: sort imports appropriately Signed-off-by: Mark Bartel --- worktree_commit.go | 2 +- worktree_commit_test.go | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/worktree_commit.go b/worktree_commit.go index ee5cd8240..f0e0b4244 100644 --- a/worktree_commit.go +++ b/worktree_commit.go @@ -2,6 +2,7 @@ package git import ( "path" + "sort" "strings" "gopkg.in/src-d/go-git.v4/plumbing" @@ -11,7 +12,6 @@ import ( "gopkg.in/src-d/go-git.v4/storage" "gopkg.in/src-d/go-billy.v4" - "sort" ) // Commit stores the current contents of the index in a new commit along with diff --git a/worktree_commit_test.go b/worktree_commit_test.go index 2100626a4..5ca9b511a 100644 --- a/worktree_commit_test.go +++ b/worktree_commit_test.go @@ -1,22 +1,22 @@ package git import ( + "bytes" + "io/ioutil" + "os" + "os/exec" "time" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/object" "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/storage/memory" + "gopkg.in/src-d/go-git.v4/storage/filesystem" - "bytes" . "gopkg.in/check.v1" "gopkg.in/src-d/go-billy.v4/memfs" "gopkg.in/src-d/go-billy.v4/osfs" "gopkg.in/src-d/go-billy.v4/util" - "gopkg.in/src-d/go-git.v4/storage/filesystem" - "io/ioutil" - "os" - "os/exec" ) func (s *WorktreeSuite) TestCommitInvalidOptions(c *C) { From b304997a387a5106321fe87069a6f136d9fbd1f6 Mon Sep 17 00:00:00 2001 From: James Ravn Date: Thu, 5 Jul 2018 10:50:09 +0100 Subject: [PATCH 023/120] plumbing: object, expose ErrEntryNotFound in FindEntry. Fixes #883 FindEntry will return ErrDirNotFound if the directory doesn't exist. But it doesn't return a public error if the entry itself is missing. This exposes the internal error ErrEntryNotFound, so users can programmatically check for this condition. Signed-off-by: James Ravn --- plumbing/object/tree.go | 5 ++--- plumbing/object/tree_test.go | 6 ++++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/plumbing/object/tree.go b/plumbing/object/tree.go index 30bbcb038..7d9e90b3f 100644 --- a/plumbing/object/tree.go +++ b/plumbing/object/tree.go @@ -25,6 +25,7 @@ var ( ErrMaxTreeDepth = errors.New("maximum tree depth exceeded") ErrFileNotFound = errors.New("file not found") ErrDirectoryNotFound = errors.New("directory not found") + ErrEntryNotFound = errors.New("entry not found") ) // Tree is basically like a directory - it references a bunch of other trees @@ -166,8 +167,6 @@ func (t *Tree) dir(baseName string) (*Tree, error) { return tree, err } -var errEntryNotFound = errors.New("entry not found") - func (t *Tree) entry(baseName string) (*TreeEntry, error) { if t.m == nil { t.buildMap() @@ -175,7 +174,7 @@ func (t *Tree) entry(baseName string) (*TreeEntry, error) { entry, ok := t.m[baseName] if !ok { - return nil, errEntryNotFound + return nil, ErrEntryNotFound } return entry, nil diff --git a/plumbing/object/tree_test.go b/plumbing/object/tree_test.go index 3a687dd5c..59d5d215f 100644 --- a/plumbing/object/tree_test.go +++ b/plumbing/object/tree_test.go @@ -114,6 +114,12 @@ func (s *TreeSuite) TestFindEntry(c *C) { c.Assert(e.Name, Equals, "foo.go") } +func (s *TreeSuite) TestFindEntryNotFound(c *C) { + e, err := s.Tree.FindEntry("not-found") + c.Assert(e, IsNil) + c.Assert(err, Equals, ErrEntryNotFound) +} + // Overrides returned plumbing.EncodedObject for given hash. // Otherwise, delegates to actual storer to get real object type fakeStorer struct { From 54d8c38fd63feefe6c25c1ac2945a6fc0bc7f16a Mon Sep 17 00:00:00 2001 From: Jerome Doucet Date: Sat, 14 Jul 2018 15:19:56 +0200 Subject: [PATCH 024/120] plumbing/transport/internal: common, add support of Gogs for ErrRepositoryNotFound, avoiding to get an 'unknown error: '. Add some tests for existing supported services (github, gitlab, etc...) too. Signed-off-by: Jerome Doucet --- plumbing/transport/internal/common/common.go | 5 ++ .../transport/internal/common/common_test.go | 78 +++++++++++++++++++ 2 files changed, 83 insertions(+) create mode 100644 plumbing/transport/internal/common/common_test.go diff --git a/plumbing/transport/internal/common/common.go b/plumbing/transport/internal/common/common.go index 8ec1ea52b..00497f3c1 100644 --- a/plumbing/transport/internal/common/common.go +++ b/plumbing/transport/internal/common/common.go @@ -382,6 +382,7 @@ var ( gitProtocolNotFoundErr = "ERR \n Repository not found." gitProtocolNoSuchErr = "ERR no such repository" gitProtocolAccessDeniedErr = "ERR access denied" + gogsAccessDeniedErr = "Gogs: Repository does not exist or you do not have access" ) func isRepoNotFoundError(s string) bool { @@ -409,6 +410,10 @@ func isRepoNotFoundError(s string) bool { return true } + if strings.HasPrefix(s, gogsAccessDeniedErr) { + return true + } + return false } diff --git a/plumbing/transport/internal/common/common_test.go b/plumbing/transport/internal/common/common_test.go new file mode 100644 index 000000000..b2f035d71 --- /dev/null +++ b/plumbing/transport/internal/common/common_test.go @@ -0,0 +1,78 @@ +package common + +import ( + "fmt" + "testing" + + . "gopkg.in/check.v1" +) + +func Test(t *testing.T) { TestingT(t) } + +type CommonSuite struct{} + +var _ = Suite(&CommonSuite{}) + +func (s *CommonSuite) TestIsRepoNotFoundErrorForUnknowSource(c *C) { + msg := "unknown system is complaining of something very sad :(" + + isRepoNotFound := isRepoNotFoundError(msg) + + c.Assert(isRepoNotFound, Equals, false) +} + +func (s *CommonSuite) TestIsRepoNotFoundErrorForGithub(c *C) { + msg := fmt.Sprintf("%s : some error stuf", githubRepoNotFoundErr) + + isRepoNotFound := isRepoNotFoundError(msg) + + c.Assert(isRepoNotFound, Equals, true) +} + +func (s *CommonSuite) TestIsRepoNotFoundErrorForBitBucket(c *C) { + msg := fmt.Sprintf("%s : some error stuf", bitbucketRepoNotFoundErr) + + isRepoNotFound := isRepoNotFoundError(msg) + + c.Assert(isRepoNotFound, Equals, true) +} + +func (s *CommonSuite) TestIsRepoNotFoundErrorForLocal(c *C) { + msg := fmt.Sprintf("some error stuf : %s", localRepoNotFoundErr) + + isRepoNotFound := isRepoNotFoundError(msg) + + c.Assert(isRepoNotFound, Equals, true) +} + +func (s *CommonSuite) TestIsRepoNotFoundErrorForGitProtocolNotFound(c *C) { + msg := fmt.Sprintf("%s : some error stuf", gitProtocolNotFoundErr) + + isRepoNotFound := isRepoNotFoundError(msg) + + c.Assert(isRepoNotFound, Equals, true) +} + +func (s *CommonSuite) TestIsRepoNotFoundErrorForGitProtocolNoSuch(c *C) { + msg := fmt.Sprintf("%s : some error stuf", gitProtocolNoSuchErr) + + isRepoNotFound := isRepoNotFoundError(msg) + + c.Assert(isRepoNotFound, Equals, true) +} + +func (s *CommonSuite) TestIsRepoNotFoundErrorForGitProtocolAccessDenied(c *C) { + msg := fmt.Sprintf("%s : some error stuf", gitProtocolAccessDeniedErr) + + isRepoNotFound := isRepoNotFoundError(msg) + + c.Assert(isRepoNotFound, Equals, true) +} + +func (s *CommonSuite) TestIsRepoNotFoundErrorForGogsAccessDenied(c *C) { + msg := fmt.Sprintf("%s : some error stuf", gogsAccessDeniedErr) + + isRepoNotFound := isRepoNotFoundError(msg) + + c.Assert(isRepoNotFound, Equals, true) +} From 8df413fe09e6cb2069a76b6df6715d0e610c8458 Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Wed, 18 Jul 2018 11:14:49 +0200 Subject: [PATCH 025/120] plumbing/object: fix pgp signature encoder/decoder The way of reading pgp signatures was searching for pgp begin line in the header. This caused problems when this string appeared and was not part of the signature. For example if it appears in the message as an example or is part of the author name the decoder starts treating it as a signature. In this state the code was not able to notice then the header ended so it entered in an infinite loop searching for pgp end string. Now it uses the same method as original git. Searches for gpgsig section in header and starts getting all lines until the next part. In encoder the string used to add signatures was incorrect. It is now changed to the proper "gpgsig" string instead of "pgpsig". Signed-off-by: Javi Fontan --- plumbing/object/commit.go | 31 +++++++++++++------------------ plumbing/object/commit_test.go | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 18 deletions(-) diff --git a/plumbing/object/commit.go b/plumbing/object/commit.go index 3ed85ba3b..b1c0e0180 100644 --- a/plumbing/object/commit.go +++ b/plumbing/object/commit.go @@ -17,8 +17,9 @@ import ( ) const ( - beginpgp string = "-----BEGIN PGP SIGNATURE-----" - endpgp string = "-----END PGP SIGNATURE-----" + beginpgp string = "-----BEGIN PGP SIGNATURE-----" + endpgp string = "-----END PGP SIGNATURE-----" + headerpgp string = "gpgsig" ) // Hash represents the hash of an object @@ -181,23 +182,13 @@ func (c *Commit) Decode(o plumbing.EncodedObject) (err error) { } if pgpsig { - // Check if it's the end of a PGP signature. - if bytes.Contains(line, []byte(endpgp)) { - c.PGPSignature += endpgp + "\n" - pgpsig = false - } else { - // Trim the left padding. + if len(line) > 0 && line[0] == ' ' { line = bytes.TrimLeft(line, " ") c.PGPSignature += string(line) + continue + } else { + pgpsig = false } - continue - } - - // Check if it's the beginning of a PGP signature. - if bytes.Contains(line, []byte(beginpgp)) { - c.PGPSignature += beginpgp + "\n" - pgpsig = true - continue } if !message { @@ -217,6 +208,9 @@ func (c *Commit) Decode(o plumbing.EncodedObject) (err error) { c.Author.Decode(split[1]) case "committer": c.Committer.Decode(split[1]) + case headerpgp: + c.PGPSignature += string(split[1]) + "\n" + pgpsig = true } } else { c.Message += string(line) @@ -269,13 +263,14 @@ func (b *Commit) encode(o plumbing.EncodedObject, includeSig bool) (err error) { } if b.PGPSignature != "" && includeSig { - if _, err = fmt.Fprint(w, "pgpsig"); err != nil { + if _, err = fmt.Fprint(w, "\n"+headerpgp); err != nil { return err } // Split all the signature lines and write with a left padding and // newline at the end. - lines := strings.Split(b.PGPSignature, "\n") + signature := strings.TrimSuffix(b.PGPSignature, "\n") + lines := strings.Split(signature, "\n") for _, line := range lines { if _, err = fmt.Fprintf(w, " %s\n", line); err != nil { return err diff --git a/plumbing/object/commit_test.go b/plumbing/object/commit_test.go index 996d4818a..b5dfbe325 100644 --- a/plumbing/object/commit_test.go +++ b/plumbing/object/commit_test.go @@ -324,6 +324,38 @@ RUysgqjcpT8+iQM1PblGfHR4XAhuOqN5Fx06PSaFZhqvWFezJ28/CLyX5q+oIVk= err = decoded.Decode(encoded) c.Assert(err, IsNil) c.Assert(decoded.PGPSignature, Equals, pgpsignature) + + // signature in author name + + commit.PGPSignature = "" + commit.Author.Name = beginpgp + encoded = &plumbing.MemoryObject{} + decoded = &Commit{} + + err = commit.Encode(encoded) + c.Assert(err, IsNil) + + err = decoded.Decode(encoded) + c.Assert(err, IsNil) + c.Assert(decoded.PGPSignature, Equals, "") + c.Assert(decoded.Author.Name, Equals, beginpgp) + + // broken signature + + commit.PGPSignature = beginpgp + "\n" + + "some\n" + + "trash\n" + + endpgp + + "text\n" + encoded = &plumbing.MemoryObject{} + decoded = &Commit{} + + err = commit.Encode(encoded) + c.Assert(err, IsNil) + + err = decoded.Decode(encoded) + c.Assert(err, IsNil) + c.Assert(decoded.PGPSignature, Equals, commit.PGPSignature) } func (s *SuiteCommit) TestStat(c *C) { From 009f1069a1248c1e9189a9e4c342f6d017156ec4 Mon Sep 17 00:00:00 2001 From: Miguel Molina Date: Thu, 19 Jul 2018 15:20:10 +0200 Subject: [PATCH 026/120] plumbing/format/idxfile: add new Index and MemoryIndex Signed-off-by: Miguel Molina --- plumbing/format/idxfile/decoder.go | 109 +++++++----- plumbing/format/idxfile/decoder_test.go | 106 +++++------ plumbing/format/idxfile/encoder.go | 101 +++++------ plumbing/format/idxfile/encoder_test.go | 21 +-- plumbing/format/idxfile/idxfile.go | 224 ++++++++++++++++++++---- plumbing/format/idxfile/idxfile_test.go | 109 ++++++++++++ plumbing/format/packfile/decoder.go | 27 +-- plumbing/format/packfile/index.go | 125 ------------- storage/filesystem/index.go | 47 ----- storage/filesystem/object.go | 2 +- 10 files changed, 484 insertions(+), 387 deletions(-) create mode 100644 plumbing/format/idxfile/idxfile_test.go delete mode 100644 plumbing/format/packfile/index.go delete mode 100644 storage/filesystem/index.go diff --git a/plumbing/format/idxfile/decoder.go b/plumbing/format/idxfile/decoder.go index 45afb1ec0..25ff88e03 100644 --- a/plumbing/format/idxfile/decoder.go +++ b/plumbing/format/idxfile/decoder.go @@ -17,6 +17,11 @@ var ( ErrMalformedIdxFile = errors.New("Malformed IDX file") ) +const ( + fanout = 256 + objectIDLength = 20 +) + // Decoder reads and decodes idx files from an input stream. type Decoder struct { *bufio.Reader @@ -27,13 +32,13 @@ func NewDecoder(r io.Reader) *Decoder { return &Decoder{bufio.NewReader(r)} } -// Decode reads from the stream and decode the content into the Idxfile struct. -func (d *Decoder) Decode(idx *Idxfile) error { +// Decode reads from the stream and decode the content into the MemoryIndex struct. +func (d *Decoder) Decode(idx *MemoryIndex) error { if err := validateHeader(d); err != nil { return err } - flow := []func(*Idxfile, io.Reader) error{ + flow := []func(*MemoryIndex, io.Reader) error{ readVersion, readFanout, readObjectNames, @@ -48,10 +53,6 @@ func (d *Decoder) Decode(idx *Idxfile) error { } } - if !idx.isValid() { - return ErrMalformedIdxFile - } - return nil } @@ -68,7 +69,7 @@ func validateHeader(r io.Reader) error { return nil } -func readVersion(idx *Idxfile, r io.Reader) error { +func readVersion(idx *MemoryIndex, r io.Reader) error { v, err := binary.ReadUint32(r) if err != nil { return err @@ -82,74 +83,92 @@ func readVersion(idx *Idxfile, r io.Reader) error { return nil } -func readFanout(idx *Idxfile, r io.Reader) error { - var err error - for i := 0; i < 255; i++ { - idx.Fanout[i], err = binary.ReadUint32(r) +func readFanout(idx *MemoryIndex, r io.Reader) error { + for k := 0; k < fanout; k++ { + n, err := binary.ReadUint32(r) if err != nil { return err } + + idx.Fanout[k] = n + idx.FanoutMapping[k] = noMapping } - idx.ObjectCount, err = binary.ReadUint32(r) - return err + return nil } -func readObjectNames(idx *Idxfile, r io.Reader) error { - c := int(idx.ObjectCount) - new := make([]Entry, c) - for i := 0; i < c; i++ { - e := &new[i] - if _, err := io.ReadFull(r, e.Hash[:]); err != nil { +func readObjectNames(idx *MemoryIndex, r io.Reader) error { + for k := 0; k < fanout; k++ { + var buckets uint32 + if k == 0 { + buckets = idx.Fanout[k] + } else { + buckets = idx.Fanout[k] - idx.Fanout[k-1] + } + + if buckets == 0 { + continue + } + + if buckets < 0 { + return ErrMalformedIdxFile + } + + idx.FanoutMapping[k] = len(idx.Names) + + nameLen := int(buckets * objectIDLength) + bin := make([]byte, nameLen) + if _, err := io.ReadFull(r, bin); err != nil { return err } - idx.Entries = append(idx.Entries, e) + idx.Names = append(idx.Names, bin) + idx.Offset32 = append(idx.Offset32, make([]byte, buckets*4)) + idx.Crc32 = append(idx.Crc32, make([]byte, buckets*4)) } return nil } -func readCRC32(idx *Idxfile, r io.Reader) error { - c := int(idx.ObjectCount) - for i := 0; i < c; i++ { - if err := binary.Read(r, &idx.Entries[i].CRC32); err != nil { - return err +func readCRC32(idx *MemoryIndex, r io.Reader) error { + for k := 0; k < fanout; k++ { + if pos := idx.FanoutMapping[k]; pos != noMapping { + if _, err := io.ReadFull(r, idx.Crc32[pos]); err != nil { + return err + } } } return nil } -func readOffsets(idx *Idxfile, r io.Reader) error { - c := int(idx.ObjectCount) - - for i := 0; i < c; i++ { - o, err := binary.ReadUint32(r) - if err != nil { - return err +func readOffsets(idx *MemoryIndex, r io.Reader) error { + var o64cnt int + for k := 0; k < fanout; k++ { + if pos := idx.FanoutMapping[k]; pos != noMapping { + if _, err := io.ReadFull(r, idx.Offset32[pos]); err != nil { + return err + } + + for p := 0; p < len(idx.Offset32[pos]); p += 4 { + if idx.Offset32[pos][p]&(byte(1)<<7) > 0 { + o64cnt++ + } + } } - - idx.Entries[i].Offset = uint64(o) } - for i := 0; i < c; i++ { - if idx.Entries[i].Offset <= offsetLimit { - continue - } - - o, err := binary.ReadUint64(r) - if err != nil { + if o64cnt > 0 { + idx.Offset64 = make([]byte, o64cnt*8) + if _, err := io.ReadFull(r, idx.Offset64); err != nil { return err } - - idx.Entries[i].Offset = o } return nil } -func readChecksums(idx *Idxfile, r io.Reader) error { +func readChecksums(idx *MemoryIndex, r io.Reader) error { if _, err := io.ReadFull(r, idx.PackfileChecksum[:]); err != nil { return err } diff --git a/plumbing/format/idxfile/decoder_test.go b/plumbing/format/idxfile/decoder_test.go index 20d6859a7..b43d7c5d5 100644 --- a/plumbing/format/idxfile/decoder_test.go +++ b/plumbing/format/idxfile/decoder_test.go @@ -4,11 +4,12 @@ import ( "bytes" "encoding/base64" "fmt" + "io" + "io/ioutil" "testing" + "gopkg.in/src-d/go-git.v4/plumbing" . "gopkg.in/src-d/go-git.v4/plumbing/format/idxfile" - "gopkg.in/src-d/go-git.v4/plumbing/format/packfile" - "gopkg.in/src-d/go-git.v4/storage/memory" . "gopkg.in/check.v1" "gopkg.in/src-d/go-git-fixtures.v3" @@ -26,51 +27,34 @@ func (s *IdxfileSuite) TestDecode(c *C) { f := fixtures.Basic().One() d := NewDecoder(f.Idx()) - idx := &Idxfile{} + idx := new(MemoryIndex) err := d.Decode(idx) c.Assert(err, IsNil) - c.Assert(idx.Entries, HasLen, 31) - c.Assert(idx.Entries[0].Hash.String(), Equals, "1669dce138d9b841a518c64b10914d88f5e488ea") - c.Assert(idx.Entries[0].Offset, Equals, uint64(615)) - c.Assert(idx.Entries[0].CRC32, Equals, uint32(3645019190)) + count, _ := idx.Count() + c.Assert(count, Equals, int64(31)) - c.Assert(fmt.Sprintf("%x", idx.IdxChecksum), Equals, "fb794f1ec720b9bc8e43257451bd99c4be6fa1c9") - c.Assert(fmt.Sprintf("%x", idx.PackfileChecksum), Equals, f.PackfileHash.String()) -} - -func (s *IdxfileSuite) TestDecodeCRCs(c *C) { - f := fixtures.Basic().ByTag("ofs-delta").One() - - scanner := packfile.NewScanner(f.Packfile()) - storage := memory.NewStorage() - - pd, err := packfile.NewDecoder(scanner, storage) + hash := plumbing.NewHash("1669dce138d9b841a518c64b10914d88f5e488ea") + ok, err := idx.Contains(hash) c.Assert(err, IsNil) - _, err = pd.Decode() - c.Assert(err, IsNil) - - i := pd.Index().ToIdxFile() - i.Version = VersionSupported + c.Assert(ok, Equals, true) - buf := bytes.NewBuffer(nil) - e := NewEncoder(buf) - _, err = e.Encode(i) + offset, err := idx.FindOffset(hash) c.Assert(err, IsNil) + c.Assert(offset, Equals, int64(615)) - idx := &Idxfile{} - - d := NewDecoder(buf) - err = d.Decode(idx) + crc32, err := idx.FindCRC32(hash) c.Assert(err, IsNil) + c.Assert(crc32, Equals, uint32(3645019190)) - c.Assert(idx.Entries, DeepEquals, i.Entries) + c.Assert(fmt.Sprintf("%x", idx.IdxChecksum), Equals, "fb794f1ec720b9bc8e43257451bd99c4be6fa1c9") + c.Assert(fmt.Sprintf("%x", idx.PackfileChecksum), Equals, f.PackfileHash.String()) } func (s *IdxfileSuite) TestDecode64bitsOffsets(c *C) { f := bytes.NewBufferString(fixtureLarge4GB) - idx := &Idxfile{} + idx := new(MemoryIndex) d := NewDecoder(base64.NewDecoder(base64.StdEncoding, f)) err := d.Decode(idx) @@ -88,29 +72,22 @@ func (s *IdxfileSuite) TestDecode64bitsOffsets(c *C) { "35858be9c6f5914cbe6768489c41eb6809a2bceb": 5924278919, } - for _, e := range idx.Entries { - c.Assert(expected[e.Hash.String()], Equals, e.Offset) - } -} - -func (s *IdxfileSuite) TestDecode64bitsOffsetsIdempotent(c *C) { - f := bytes.NewBufferString(fixtureLarge4GB) - - expected := &Idxfile{} - - d := NewDecoder(base64.NewDecoder(base64.StdEncoding, f)) - err := d.Decode(expected) + iter, err := idx.Entries() c.Assert(err, IsNil) - buf := bytes.NewBuffer(nil) - _, err = NewEncoder(buf).Encode(expected) - c.Assert(err, IsNil) + var entries int + for { + e, err := iter.Next() + if err == io.EOF { + break + } + c.Assert(err, IsNil) + entries++ - idx := &Idxfile{} - err = NewDecoder(buf).Decode(idx) - c.Assert(err, IsNil) + c.Assert(expected[e.Hash.String()], Equals, e.Offset) + } - c.Assert(idx.Entries, DeepEquals, expected.Entries) + c.Assert(entries, Equals, len(expected)) } const fixtureLarge4GB = `/3RPYwAAAAIAAAAAAAAAAAAAAAAAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEA @@ -139,3 +116,30 @@ AAAAAAAMgAAAAQAAAI6AAAACgAAAA4AAAASAAAAFAAAAAV9Qam8AAAABYR1ShwAAAACdxfYxAAAA ANz1Di4AAAABPUnxJAAAAADNxzlGr6vCJpIFz4XaG/fi/f9C9zgQ8ptKSQpfQ1NMJBGTDTxxYGGp ch2xUA== ` + +func BenchmarkDecode(b *testing.B) { + if err := fixtures.Init(); err != nil { + b.Errorf("unexpected error initializing fixtures: %s", err) + } + + f := fixtures.Basic().One() + fixture, err := ioutil.ReadAll(f.Idx()) + if err != nil { + b.Errorf("unexpected error reading idx file: %s", err) + } + + defer func() { + if err := fixtures.Clean(); err != nil { + b.Errorf("unexpected error cleaning fixtures: %s", err) + } + }() + + for i := 0; i < b.N; i++ { + f := bytes.NewBuffer(fixture) + idx := new(MemoryIndex) + d := NewDecoder(f) + if err := d.Decode(idx); err != nil { + b.Errorf("unexpected error decoding: %s", err) + } + } +} diff --git a/plumbing/format/idxfile/encoder.go b/plumbing/format/idxfile/encoder.go index 40abfb830..55df4667f 100644 --- a/plumbing/format/idxfile/encoder.go +++ b/plumbing/format/idxfile/encoder.go @@ -4,12 +4,11 @@ import ( "crypto/sha1" "hash" "io" - "sort" "gopkg.in/src-d/go-git.v4/utils/binary" ) -// Encoder writes Idxfile structs to an output stream. +// Encoder writes MemoryIndex structs to an output stream. type Encoder struct { io.Writer hash hash.Hash @@ -22,11 +21,9 @@ func NewEncoder(w io.Writer) *Encoder { return &Encoder{mw, h} } -// Encode encodes an Idxfile to the encoder writer. -func (e *Encoder) Encode(idx *Idxfile) (int, error) { - idx.Entries.Sort() - - flow := []func(*Idxfile) (int, error){ +// Encode encodes an MemoryIndex to the encoder writer. +func (e *Encoder) Encode(idx *MemoryIndex) (int, error) { + flow := []func(*MemoryIndex) (int, error){ e.encodeHeader, e.encodeFanout, e.encodeHashes, @@ -48,7 +45,7 @@ func (e *Encoder) Encode(idx *Idxfile) (int, error) { return sz, nil } -func (e *Encoder) encodeHeader(idx *Idxfile) (int, error) { +func (e *Encoder) encodeHeader(idx *MemoryIndex) (int, error) { c, err := e.Write(idxHeader) if err != nil { return c, err @@ -57,75 +54,81 @@ func (e *Encoder) encodeHeader(idx *Idxfile) (int, error) { return c + 4, binary.WriteUint32(e, idx.Version) } -func (e *Encoder) encodeFanout(idx *Idxfile) (int, error) { - fanout := idx.calculateFanout() - for _, c := range fanout { +func (e *Encoder) encodeFanout(idx *MemoryIndex) (int, error) { + for _, c := range idx.Fanout { if err := binary.WriteUint32(e, c); err != nil { return 0, err } } - return 1024, nil + return fanout * 4, nil } -func (e *Encoder) encodeHashes(idx *Idxfile) (int, error) { - sz := 0 - for _, ent := range idx.Entries { - i, err := e.Write(ent.Hash[:]) - sz += i +func (e *Encoder) encodeHashes(idx *MemoryIndex) (int, error) { + var size int + for k := 0; k < fanout; k++ { + pos := idx.FanoutMapping[k] + if pos == noMapping { + continue + } + n, err := e.Write(idx.Names[pos]) if err != nil { - return sz, err + return size, err } + size += n } - - return sz, nil + return size, nil } -func (e *Encoder) encodeCRC32(idx *Idxfile) (int, error) { - sz := 0 - for _, ent := range idx.Entries { - err := binary.Write(e, ent.CRC32) - sz += 4 +func (e *Encoder) encodeCRC32(idx *MemoryIndex) (int, error) { + var size int + for k := 0; k < fanout; k++ { + pos := idx.FanoutMapping[k] + if pos == noMapping { + continue + } + n, err := e.Write(idx.Crc32[pos]) if err != nil { - return sz, err + return size, err } + + size += n } - return sz, nil + return size, nil } -func (e *Encoder) encodeOffsets(idx *Idxfile) (int, error) { - sz := 0 - - var o64bits []uint64 - for _, ent := range idx.Entries { - o := ent.Offset - if o > offsetLimit { - o64bits = append(o64bits, o) - o = offsetLimit + uint64(len(o64bits)) +func (e *Encoder) encodeOffsets(idx *MemoryIndex) (int, error) { + var size int + for k := 0; k < fanout; k++ { + pos := idx.FanoutMapping[k] + if pos == noMapping { + continue } - if err := binary.WriteUint32(e, uint32(o)); err != nil { - return sz, err + n, err := e.Write(idx.Offset32[pos]) + if err != nil { + return size, err } - sz += 4 + size += n } - for _, o := range o64bits { - if err := binary.WriteUint64(e, o); err != nil { - return sz, err + if len(idx.Offset64) > 0 { + n, err := e.Write(idx.Offset64) + if err != nil { + return size, err } - sz += 8 + size += n } - return sz, nil + return size, nil } -func (e *Encoder) encodeChecksums(idx *Idxfile) (int, error) { +func (e *Encoder) encodeChecksums(idx *MemoryIndex) (int, error) { if _, err := e.Write(idx.PackfileChecksum[:]); err != nil { return 0, err } @@ -137,11 +140,3 @@ func (e *Encoder) encodeChecksums(idx *Idxfile) (int, error) { return 40, nil } - -// EntryList implements sort.Interface allowing sorting in increasing order. -type EntryList []*Entry - -func (p EntryList) Len() int { return len(p) } -func (p EntryList) Less(i, j int) bool { return p[i].Hash.String() < p[j].Hash.String() } -func (p EntryList) Swap(i, j int) { p[i], p[j] = p[j], p[i] } -func (p EntryList) Sort() { sort.Sort(p) } diff --git a/plumbing/format/idxfile/encoder_test.go b/plumbing/format/idxfile/encoder_test.go index e5b96b743..e8deeea1c 100644 --- a/plumbing/format/idxfile/encoder_test.go +++ b/plumbing/format/idxfile/encoder_test.go @@ -4,37 +4,18 @@ import ( "bytes" "io/ioutil" - "gopkg.in/src-d/go-git.v4/plumbing" . "gopkg.in/src-d/go-git.v4/plumbing/format/idxfile" . "gopkg.in/check.v1" "gopkg.in/src-d/go-git-fixtures.v3" ) -func (s *IdxfileSuite) TestEncode(c *C) { - expected := &Idxfile{} - expected.Add(plumbing.NewHash("4bfc730165c370df4a012afbb45ba3f9c332c0d4"), 82, 82) - expected.Add(plumbing.NewHash("8fa2238efdae08d83c12ee176fae65ff7c99af46"), 42, 42) - - buf := bytes.NewBuffer(nil) - e := NewEncoder(buf) - _, err := e.Encode(expected) - c.Assert(err, IsNil) - - idx := &Idxfile{} - d := NewDecoder(buf) - err = d.Decode(idx) - c.Assert(err, IsNil) - - c.Assert(idx.Entries, DeepEquals, expected.Entries) -} - func (s *IdxfileSuite) TestDecodeEncode(c *C) { fixtures.ByTag("packfile").Test(c, func(f *fixtures.Fixture) { expected, err := ioutil.ReadAll(f.Idx()) c.Assert(err, IsNil) - idx := &Idxfile{} + idx := new(MemoryIndex) d := NewDecoder(bytes.NewBuffer(expected)) err = d.Decode(idx) c.Assert(err, IsNil) diff --git a/plumbing/format/idxfile/idxfile.go b/plumbing/format/idxfile/idxfile.go index 6b05eaace..b1966086c 100644 --- a/plumbing/format/idxfile/idxfile.go +++ b/plumbing/format/idxfile/idxfile.go @@ -1,68 +1,222 @@ package idxfile -import "gopkg.in/src-d/go-git.v4/plumbing" +import ( + "bytes" + "io" + + "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/utils/binary" +) const ( // VersionSupported is the only idx version supported. VersionSupported = 2 - offsetLimit = 0x7fffffff + noMapping = -1 ) var ( idxHeader = []byte{255, 't', 'O', 'c'} ) -// Idxfile is the in memory representation of an idx file. -type Idxfile struct { - Version uint32 - Fanout [255]uint32 - ObjectCount uint32 - Entries EntryList +// Index represents an index of a packfile. +type Index interface { + // Contains checks whether the given hash is in the index. + Contains(h plumbing.Hash) (bool, error) + // FindOffset finds the offset in the packfile for the object with + // the given hash. + FindOffset(h plumbing.Hash) (int64, error) + // FindCRC32 finds the CRC32 of the object with the given hash. + FindCRC32(h plumbing.Hash) (uint32, error) + // Count returns the number of entries in the index. + Count() (int64, error) + // Entries returns an iterator to retrieve all index entries. + Entries() (EntryIter, error) +} + +// MemoryIndex is the in memory representation of an idx file. +type MemoryIndex struct { + Version uint32 + Fanout [256]uint32 + // FanoutMapping maps the position in the fanout table to the position + // in the Names, Offset32 and Crc32 slices. This improves the memory + // usage by not needing an array with unnecessary empty slots. + FanoutMapping [256]int + Names [][]byte + Offset32 [][]byte + Crc32 [][]byte + Offset64 []byte PackfileChecksum [20]byte IdxChecksum [20]byte } -func NewIdxfile() *Idxfile { - return &Idxfile{} +var _ Index = (*MemoryIndex)(nil) + +// NewMemoryIndex returns an instance of a new MemoryIndex. +func NewMemoryIndex() *MemoryIndex { + return &MemoryIndex{} } -// Entry is the in memory representation of an object entry in the idx file. -type Entry struct { - Hash plumbing.Hash - CRC32 uint32 - Offset uint64 +func (idx *MemoryIndex) findHashIndex(h plumbing.Hash) int { + k := idx.FanoutMapping[h[0]] + if k == noMapping { + return -1 + } + + data := idx.Names[k] + high := uint64(len(idx.Offset32[k])) >> 2 + if high == 0 { + return -1 + } + + low := uint64(0) + for { + mid := (low + high) >> 1 + offset := mid + (mid << 2) + + cmp := bytes.Compare(h[:], data[offset:offset+objectIDLength]) + if cmp < 0 { + high = mid + } else if cmp == 0 { + return int(mid) + } else { + low = mid + 1 + } + + if low < high { + break + } + } + + return -1 } -// Add adds a new Entry with the given values to the Idxfile. -func (idx *Idxfile) Add(h plumbing.Hash, offset uint64, crc32 uint32) { - idx.Entries = append(idx.Entries, &Entry{ - Hash: h, - Offset: offset, - CRC32: crc32, - }) +// Contains implements the Index interface. +func (idx *MemoryIndex) Contains(h plumbing.Hash) (bool, error) { + i := idx.findHashIndex(h) + return i >= 0, nil } -func (idx *Idxfile) isValid() bool { - fanout := idx.calculateFanout() - for k, c := range idx.Fanout { - if fanout[k] != c { - return false +// FindOffset implements the Index interface. +func (idx *MemoryIndex) FindOffset(h plumbing.Hash) (int64, error) { + k := idx.FanoutMapping[h[0]] + i := idx.findHashIndex(h) + if i < 0 { + return 0, plumbing.ErrObjectNotFound + } + + return idx.getOffset(k, i) +} + +const isO64Mask = uint64(1) << 31 + +func (idx *MemoryIndex) getOffset(firstLevel, secondLevel int) (int64, error) { + offset := secondLevel << 2 + buf := bytes.NewBuffer(idx.Offset32[firstLevel][offset : offset+4]) + ofs, err := binary.ReadUint32(buf) + if err != nil { + return -1, err + } + + if (uint64(ofs) & isO64Mask) != 0 { + offset := 8 * (uint64(ofs) & ^isO64Mask) + buf := bytes.NewBuffer(idx.Offset64[offset : offset+8]) + n, err := binary.ReadUint64(buf) + if err != nil { + return -1, err } + + return int64(n), nil } - return true + return int64(ofs), nil } -func (idx *Idxfile) calculateFanout() [256]uint32 { - fanout := [256]uint32{} - for _, e := range idx.Entries { - fanout[e.Hash[0]]++ +// FindCRC32 implements the Index interface. +func (idx *MemoryIndex) FindCRC32(h plumbing.Hash) (uint32, error) { + k := idx.FanoutMapping[h[0]] + i := idx.findHashIndex(h) + if i < 0 { + return 0, plumbing.ErrObjectNotFound } - for i := 1; i < 256; i++ { - fanout[i] += fanout[i-1] + return idx.getCrc32(k, i) +} + +func (idx *MemoryIndex) getCrc32(firstLevel, secondLevel int) (uint32, error) { + offset := secondLevel << 2 + buf := bytes.NewBuffer(idx.Crc32[firstLevel][offset : offset+4]) + return binary.ReadUint32(buf) +} + +// Count implements the Index interface. +func (idx *MemoryIndex) Count() (int64, error) { + return int64(idx.Fanout[fanout-1]), nil +} + +// Entries implements the Index interface. +func (idx *MemoryIndex) Entries() (EntryIter, error) { + return &idxfileEntryIter{idx, 0, 0, 0}, nil +} + +// EntryIter is an iterator that will return the entries in a packfile index. +type EntryIter interface { + // Next returns the next entry in the packfile index. + Next() (*Entry, error) + // Close closes the iterator. + Close() error +} + +type idxfileEntryIter struct { + idx *MemoryIndex + total int + firstLevel, secondLevel int +} + +func (i *idxfileEntryIter) Next() (*Entry, error) { + for { + if i.firstLevel >= fanout { + return nil, io.EOF + } + + if i.total >= int(i.idx.Fanout[i.firstLevel]) { + i.firstLevel++ + i.secondLevel = 0 + continue + } + + entry := new(Entry) + ofs := i.secondLevel * objectIDLength + copy(entry.Hash[:], i.idx.Names[i.idx.FanoutMapping[i.firstLevel]][ofs:]) + + pos := i.idx.FanoutMapping[entry.Hash[0]] + + offset, err := i.idx.getOffset(pos, i.secondLevel) + if err != nil { + return nil, err + } + entry.Offset = uint64(offset) + + entry.CRC32, err = i.idx.getCrc32(pos, i.secondLevel) + if err != nil { + return nil, err + } + + i.secondLevel++ + i.total++ + + return entry, nil } +} - return fanout +func (i *idxfileEntryIter) Close() error { + i.firstLevel = fanout + return nil +} + +// Entry is the in memory representation of an object entry in the idx file. +type Entry struct { + Hash plumbing.Hash + CRC32 uint32 + Offset uint64 } diff --git a/plumbing/format/idxfile/idxfile_test.go b/plumbing/format/idxfile/idxfile_test.go new file mode 100644 index 000000000..f42a41998 --- /dev/null +++ b/plumbing/format/idxfile/idxfile_test.go @@ -0,0 +1,109 @@ +package idxfile_test + +import ( + "bytes" + "encoding/base64" + "io" + "testing" + + "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/format/idxfile" +) + +func BenchmarkFindOffset(b *testing.B) { + idx := fixtureIndex(b) + + for i := 0; i < b.N; i++ { + for _, h := range fixtureHashes { + _, err := idx.FindOffset(h) + if err != nil { + b.Fatalf("error getting offset: %s", err) + } + } + } +} + +func BenchmarkFindCRC32(b *testing.B) { + idx := fixtureIndex(b) + + for i := 0; i < b.N; i++ { + for _, h := range fixtureHashes { + _, err := idx.FindCRC32(h) + if err != nil { + b.Fatalf("error getting crc32: %s", err) + } + } + } +} + +func BenchmarkContains(b *testing.B) { + idx := fixtureIndex(b) + + for i := 0; i < b.N; i++ { + for _, h := range fixtureHashes { + ok, err := idx.Contains(h) + if err != nil { + b.Fatalf("error checking if hash is in index: %s", err) + } + + if !ok { + b.Error("expected hash to be in index") + } + } + } +} + +func BenchmarkEntries(b *testing.B) { + idx := fixtureIndex(b) + + for i := 0; i < b.N; i++ { + iter, err := idx.Entries() + if err != nil { + b.Fatalf("unexpected error getting entries: %s", err) + } + + var entries int + for { + _, err := iter.Next() + if err != nil { + if err == io.EOF { + break + } + + b.Errorf("unexpected error getting entry: %s", err) + } + + entries++ + } + + if entries != len(fixtureHashes) { + b.Errorf("expecting entries to be %d, got %d", len(fixtureHashes), entries) + } + } +} + +var fixtureHashes = []plumbing.Hash{ + plumbing.NewHash("303953e5aa461c203a324821bc1717f9b4fff895"), + plumbing.NewHash("5296768e3d9f661387ccbff18c4dea6c997fd78c"), + plumbing.NewHash("03fc8d58d44267274edef4585eaeeb445879d33f"), + plumbing.NewHash("8f3ceb4ea4cb9e4a0f751795eb41c9a4f07be772"), + plumbing.NewHash("e0d1d625010087f79c9e01ad9d8f95e1628dda02"), + plumbing.NewHash("90eba326cdc4d1d61c5ad25224ccbf08731dd041"), + plumbing.NewHash("bab53055add7bc35882758a922c54a874d6b1272"), + plumbing.NewHash("1b8995f51987d8a449ca5ea4356595102dc2fbd4"), + plumbing.NewHash("35858be9c6f5914cbe6768489c41eb6809a2bceb"), +} + +func fixtureIndex(t testing.TB) *idxfile.MemoryIndex { + f := bytes.NewBufferString(fixtureLarge4GB) + + idx := new(idxfile.MemoryIndex) + + d := idxfile.NewDecoder(base64.NewDecoder(base64.StdEncoding, f)) + err := d.Decode(idx) + if err != nil { + t.Fatalf("unexpected error decoding index: %s", err) + } + + return idx +} diff --git a/plumbing/format/packfile/decoder.go b/plumbing/format/packfile/decoder.go index f706e5d84..765401f5e 100644 --- a/plumbing/format/packfile/decoder.go +++ b/plumbing/format/packfile/decoder.go @@ -5,6 +5,7 @@ import ( "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/cache" + "gopkg.in/src-d/go-git.v4/plumbing/format/idxfile" "gopkg.in/src-d/go-git.v4/plumbing/storer" ) @@ -63,7 +64,7 @@ type Decoder struct { // hasBuiltIndex indicates if the index is fully built or not. If it is not, // will be built incrementally while decoding. hasBuiltIndex bool - idx *Index + idx idxfile.Index offsetToType map[int64]plumbing.ObjectType decoderType plumbing.ObjectType @@ -117,7 +118,7 @@ func NewDecoderForType(s *Scanner, o storer.EncodedObjectStorer, o: o, deltaBaseCache: cacheObject, - idx: NewIndex(0), + idx: idxfile.NewMemoryIndex(), offsetToType: make(map[int64]plumbing.ObjectType), decoderType: t, }, nil @@ -150,7 +151,8 @@ func (d *Decoder) doDecode() error { } if !d.hasBuiltIndex { - d.idx = NewIndex(int(count)) + // TODO: MemoryIndex is not writable, change to something else + d.idx = idxfile.NewMemoryIndex() } defer func() { d.hasBuiltIndex = true }() @@ -284,12 +286,12 @@ func (d *Decoder) ofsDeltaType(offset int64) (plumbing.ObjectType, error) { } func (d *Decoder) refDeltaType(ref plumbing.Hash) (plumbing.ObjectType, error) { - e, ok := d.idx.LookupHash(ref) - if !ok { + offset, err := d.idx.FindOffset(ref) + if err != nil { return plumbing.InvalidObject, plumbing.ErrObjectNotFound } - return d.ofsDeltaType(int64(e.Offset)) + return d.ofsDeltaType(offset) } func (d *Decoder) decodeByHeader(h *ObjectHeader) (plumbing.EncodedObject, error) { @@ -314,9 +316,14 @@ func (d *Decoder) decodeByHeader(h *ObjectHeader) (plumbing.EncodedObject, error return obj, err } + // TODO: remove this + _ = crc + + /* Add is no longer available if !d.hasBuiltIndex { d.idx.Add(obj.Hash(), uint64(h.Offset), crc) } + */ return obj, nil } @@ -448,8 +455,8 @@ func (d *Decoder) recallByOffset(o int64) (plumbing.EncodedObject, error) { func (d *Decoder) recallByHash(h plumbing.Hash) (plumbing.EncodedObject, error) { if d.s.IsSeekable { - if e, ok := d.idx.LookupHash(h); ok { - return d.DecodeObjectAt(int64(e.Offset)) + if offset, err := d.idx.FindOffset(h); err != nil { + return d.DecodeObjectAt(offset) } } @@ -475,7 +482,7 @@ func (d *Decoder) recallByHashNonSeekable(h plumbing.Hash) (obj plumbing.Encoded // SetIndex sets an index for the packfile. It is recommended to set this. // The index might be read from a file or reused from a previous Decoder usage // (see Index function). -func (d *Decoder) SetIndex(idx *Index) { +func (d *Decoder) SetIndex(idx idxfile.Index) { d.hasBuiltIndex = true d.idx = idx } @@ -484,7 +491,7 @@ func (d *Decoder) SetIndex(idx *Index) { // Index will return it. Otherwise, it will return an index that is built while // decoding. If neither SetIndex was called with a full index or Decode called // for the whole packfile, then the returned index will be incomplete. -func (d *Decoder) Index() *Index { +func (d *Decoder) Index() idxfile.Index { return d.idx } diff --git a/plumbing/format/packfile/index.go b/plumbing/format/packfile/index.go deleted file mode 100644 index 021b2d102..000000000 --- a/plumbing/format/packfile/index.go +++ /dev/null @@ -1,125 +0,0 @@ -package packfile - -import ( - "sort" - - "gopkg.in/src-d/go-git.v4/plumbing" - "gopkg.in/src-d/go-git.v4/plumbing/format/idxfile" -) - -// Index is an in-memory representation of a packfile index. -// This uses idxfile.Idxfile under the hood to obtain indexes from .idx files -// or to store them. -type Index struct { - byHash map[plumbing.Hash]*idxfile.Entry - byOffset []*idxfile.Entry // sorted by their offset -} - -// NewIndex creates a new empty index with the given size. Size is a hint and -// can be 0. It is recommended to set it to the number of objects to be indexed -// if it is known beforehand (e.g. reading from a packfile). -func NewIndex(size int) *Index { - return &Index{ - byHash: make(map[plumbing.Hash]*idxfile.Entry, size), - byOffset: make([]*idxfile.Entry, 0, size), - } -} - -// NewIndexFromIdxFile creates a new Index from an idxfile.IdxFile. -func NewIndexFromIdxFile(idxf *idxfile.Idxfile) *Index { - idx := &Index{ - byHash: make(map[plumbing.Hash]*idxfile.Entry, idxf.ObjectCount), - byOffset: make([]*idxfile.Entry, 0, idxf.ObjectCount), - } - sorted := true - for i, e := range idxf.Entries { - idx.addUnsorted(e) - if i > 0 && idx.byOffset[i-1].Offset >= e.Offset { - sorted = false - } - } - - // If the idxfile was loaded from a regular packfile index - // then it will already be in offset order, in which case we - // can avoid doing a relatively expensive idempotent sort. - if !sorted { - sort.Sort(orderByOffset(idx.byOffset)) - } - - return idx -} - -// orderByOffset is a sort.Interface adapter that arranges -// a slice of entries by their offset. -type orderByOffset []*idxfile.Entry - -func (o orderByOffset) Len() int { return len(o) } -func (o orderByOffset) Less(i, j int) bool { return o[i].Offset < o[j].Offset } -func (o orderByOffset) Swap(i, j int) { o[i], o[j] = o[j], o[i] } - -// Add adds a new Entry with the given values to the index. -func (idx *Index) Add(h plumbing.Hash, offset uint64, crc32 uint32) { - e := &idxfile.Entry{ - Hash: h, - Offset: offset, - CRC32: crc32, - } - idx.byHash[e.Hash] = e - - // Find the right position in byOffset. - // Look for the first position whose offset is *greater* than e.Offset. - i := sort.Search(len(idx.byOffset), func(i int) bool { - return idx.byOffset[i].Offset > offset - }) - if i == len(idx.byOffset) { - // Simple case: add it to the end. - idx.byOffset = append(idx.byOffset, e) - return - } - // Harder case: shift existing entries down by one to make room. - // Append a nil entry first so we can use existing capacity in case - // the index was carefully preallocated. - idx.byOffset = append(idx.byOffset, nil) - copy(idx.byOffset[i+1:], idx.byOffset[i:len(idx.byOffset)-1]) - idx.byOffset[i] = e -} - -func (idx *Index) addUnsorted(e *idxfile.Entry) { - idx.byHash[e.Hash] = e - idx.byOffset = append(idx.byOffset, e) -} - -// LookupHash looks an entry up by its hash. An idxfile.Entry is returned and -// a bool, which is true if it was found or false if it wasn't. -func (idx *Index) LookupHash(h plumbing.Hash) (*idxfile.Entry, bool) { - e, ok := idx.byHash[h] - return e, ok -} - -// LookupHash looks an entry up by its offset in the packfile. An idxfile.Entry -// is returned and a bool, which is true if it was found or false if it wasn't. -func (idx *Index) LookupOffset(offset uint64) (*idxfile.Entry, bool) { - i := sort.Search(len(idx.byOffset), func(i int) bool { - return idx.byOffset[i].Offset >= offset - }) - if i >= len(idx.byOffset) || idx.byOffset[i].Offset != offset { - return nil, false // not present - } - return idx.byOffset[i], true -} - -// Size returns the number of entries in the index. -func (idx *Index) Size() int { - return len(idx.byHash) -} - -// ToIdxFile converts the index to an idxfile.Idxfile, which can then be used -// to serialize. -func (idx *Index) ToIdxFile() *idxfile.Idxfile { - idxf := idxfile.NewIdxfile() - for _, e := range idx.byHash { - idxf.Entries = append(idxf.Entries, e) - } - - return idxf -} diff --git a/storage/filesystem/index.go b/storage/filesystem/index.go deleted file mode 100644 index 2ebf57e61..000000000 --- a/storage/filesystem/index.go +++ /dev/null @@ -1,47 +0,0 @@ -package filesystem - -import ( - "os" - - "gopkg.in/src-d/go-git.v4/plumbing/format/index" - "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit" - "gopkg.in/src-d/go-git.v4/utils/ioutil" -) - -type IndexStorage struct { - dir *dotgit.DotGit -} - -func (s *IndexStorage) SetIndex(idx *index.Index) (err error) { - f, err := s.dir.IndexWriter() - if err != nil { - return err - } - - defer ioutil.CheckClose(f, &err) - - e := index.NewEncoder(f) - err = e.Encode(idx) - return err -} - -func (s *IndexStorage) Index() (i *index.Index, err error) { - idx := &index.Index{ - Version: 2, - } - - f, err := s.dir.Index() - if err != nil { - if os.IsNotExist(err) { - return idx, nil - } - - return nil, err - } - - defer ioutil.CheckClose(f, &err) - - d := index.NewDecoder(f) - err = d.Decode(idx) - return idx, err -} diff --git a/storage/filesystem/object.go b/storage/filesystem/object.go index 9ffe4dcf5..ef67f5011 100644 --- a/storage/filesystem/object.go +++ b/storage/filesystem/object.go @@ -63,7 +63,7 @@ func (s *ObjectStorage) loadIdxFile(h plumbing.Hash) (err error) { } defer ioutil.CheckClose(f, &err) - idxf := idxfile.NewIdxfile() + idxf := idxfile.NewMemoryIndex() d := idxfile.NewDecoder(f) if err = d.Decode(idxf); err != nil { return err From da5677f5ba3970d585d5955b15a6a1c3c262c07b Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Thu, 19 Jul 2018 17:05:45 +0200 Subject: [PATCH 027/120] plumbing/packfile: add new packfile parser Signed-off-by: Javi Fontan --- plumbing/format/packfile/parser.go | 359 ++++++++++++++++++++++++ plumbing/format/packfile/parser_test.go | 139 +++++++++ 2 files changed, 498 insertions(+) create mode 100644 plumbing/format/packfile/parser.go create mode 100644 plumbing/format/packfile/parser_test.go diff --git a/plumbing/format/packfile/parser.go b/plumbing/format/packfile/parser.go new file mode 100644 index 000000000..460fc3f5a --- /dev/null +++ b/plumbing/format/packfile/parser.go @@ -0,0 +1,359 @@ +package packfile + +import ( + "bytes" + "errors" + "io" + + "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" +) + +// Observer interface is implemented by index encoders. +type Observer interface { + // OnHeader is called when a new packfile is opened. + OnHeader(count uint32) error + // OnInflatedObjectHeader is called for each object header read. + OnInflatedObjectHeader(t plumbing.ObjectType, objSize int64, pos int64) error + // OnInflatedObjectContent is called for each decoded object. + OnInflatedObjectContent(h plumbing.Hash, pos int64, crc uint32) error + // OnFooter is called when decoding is done. + OnFooter(h plumbing.Hash) error +} + +// Parser decodes a packfile and calls any observer associated to it. Is used +// to generate indexes. +type Parser struct { + scanner *Scanner + count uint32 + oi []*objectInfo + oiByHash map[plumbing.Hash]*objectInfo + oiByOffset map[int64]*objectInfo + hashOffset map[plumbing.Hash]int64 + checksum plumbing.Hash + + cache *cache.ObjectLRU + + ob []Observer +} + +// NewParser creates a new Parser struct. +func NewParser(scanner *Scanner, ob ...Observer) *Parser { + return &Parser{ + scanner: scanner, + ob: ob, + count: 0, + cache: cache.NewObjectLRUDefault(), + } +} + +// Parse start decoding phase of the packfile. +func (p *Parser) Parse() (plumbing.Hash, error) { + err := p.init() + if err != nil { + return plumbing.ZeroHash, err + } + + err = p.firstPass() + if err != nil { + return plumbing.ZeroHash, err + } + + err = p.resolveDeltas() + if err != nil { + return plumbing.ZeroHash, err + } + + for _, o := range p.ob { + err := o.OnFooter(p.checksum) + if err != nil { + return plumbing.ZeroHash, err + } + } + + return p.checksum, nil +} + +func (p *Parser) init() error { + _, c, err := p.scanner.Header() + if err != nil { + return err + } + + for _, o := range p.ob { + err := o.OnHeader(c) + if err != nil { + return err + } + } + + p.count = c + p.oiByHash = make(map[plumbing.Hash]*objectInfo, p.count) + p.oiByOffset = make(map[int64]*objectInfo, p.count) + p.oi = make([]*objectInfo, p.count) + + return nil +} + +func (p *Parser) firstPass() error { + buf := new(bytes.Buffer) + + for i := uint32(0); i < p.count; i++ { + buf.Truncate(0) + + oh, err := p.scanner.NextObjectHeader() + if err != nil { + return err + } + + delta := false + var ota *objectInfo + switch t := oh.Type; t { + case plumbing.OFSDeltaObject, plumbing.REFDeltaObject: + delta = true + + var parent *objectInfo + var ok bool + + if t == plumbing.OFSDeltaObject { + parent, ok = p.oiByOffset[oh.OffsetReference] + } else { + parent, ok = p.oiByHash[oh.Reference] + } + + if !ok { + // TODO improve error + return errors.New("Reference delta not found") + } + + ota = newDeltaObject(oh.Offset, oh.Length, t, parent) + + parent.Children = append(parent.Children, ota) + default: + ota = newBaseObject(oh.Offset, oh.Length, t) + } + + size, crc, err := p.scanner.NextObject(buf) + if err != nil { + return err + } + + ota.Crc32 = crc + ota.PackSize = size + ota.Length = oh.Length + + if !delta { + ota.Write(buf.Bytes()) + ota.SHA1 = ota.Sum() + } + + p.oiByOffset[oh.Offset] = ota + p.oiByHash[oh.Reference] = ota + + p.oi[i] = ota + } + + checksum, err := p.scanner.Checksum() + p.checksum = checksum + + if err == io.EOF { + return nil + } + + return err +} + +func (p *Parser) resolveDeltas() error { + for _, obj := range p.oi { + for _, o := range p.ob { + err := o.OnInflatedObjectHeader(obj.Type, obj.Length, obj.Offset) + if err != nil { + return err + } + + err = o.OnInflatedObjectContent(obj.SHA1, obj.Offset, obj.Crc32) + if err != nil { + return err + } + } + + if !obj.IsDelta() && len(obj.Children) > 0 { + var err error + base, err := p.get(obj) + if err != nil { + return err + } + + for _, child := range obj.Children { + _, err = p.resolveObject(child, base) + if err != nil { + return err + } + } + } + } + + return nil +} + +func (p *Parser) get(o *objectInfo) ([]byte, error) { + e, ok := p.cache.Get(o.SHA1) + if ok { + r, err := e.Reader() + if err != nil { + return nil, err + } + + buf := make([]byte, e.Size()) + _, err = r.Read(buf) + if err != nil { + return nil, err + } + + return buf, nil + } + + // Read from disk + if o.DiskType.IsDelta() { + base, err := p.get(o.Parent) + if err != nil { + return nil, err + } + + data, err := p.resolveObject(o, base) + if err != nil { + return nil, err + } + + if len(o.Children) > 0 { + m := &plumbing.MemoryObject{} + m.Write(data) + m.SetType(o.Type) + m.SetSize(o.Size()) + p.cache.Put(m) + } + + return data, nil + } + + data, err := p.readData(o) + if err != nil { + return nil, err + } + + if len(o.Children) > 0 { + m := &plumbing.MemoryObject{} + m.Write(data) + m.SetType(o.Type) + m.SetSize(o.Size()) + p.cache.Put(m) + } + + return data, nil +} + +func (p *Parser) resolveObject( + o *objectInfo, + base []byte) ([]byte, error) { + + if !o.DiskType.IsDelta() { + return nil, nil + } + + data, err := p.readData(o) + if err != nil { + return nil, err + } + + data, err = applyPatchBase(o, data, base) + if err != nil { + return nil, err + } + + return data, nil +} + +func (p *Parser) readData(o *objectInfo) ([]byte, error) { + buf := new(bytes.Buffer) + + // TODO: skip header. Header size can be calculated with the offset of the + // next offset in the first pass. + p.scanner.SeekFromStart(o.Offset) + _, err := p.scanner.NextObjectHeader() + if err != nil { + return nil, err + } + + buf.Truncate(0) + + _, _, err = p.scanner.NextObject(buf) + if err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +func applyPatchBase(ota *objectInfo, data, base []byte) ([]byte, error) { + patched, err := PatchDelta(base, data) + if err != nil { + return nil, err + } + + ota.Type = ota.Parent.Type + hash := plumbing.ComputeHash(ota.Type, patched) + + ota.SHA1 = hash + + return patched, nil +} + +type objectInfo struct { + plumbing.Hasher + + Offset int64 + Length int64 + PackSize int64 + Type plumbing.ObjectType + DiskType plumbing.ObjectType + + Crc32 uint32 + + Parent *objectInfo + Children []*objectInfo + SHA1 plumbing.Hash +} + +func newBaseObject(offset, length int64, t plumbing.ObjectType) *objectInfo { + return newDeltaObject(offset, length, t, nil) +} + +func newDeltaObject( + offset, length int64, + t plumbing.ObjectType, + parent *objectInfo, +) *objectInfo { + children := make([]*objectInfo, 0) + + obj := &objectInfo{ + Hasher: plumbing.NewHasher(t, length), + Offset: offset, + Length: length, + PackSize: 0, + Type: t, + DiskType: t, + Crc32: 0, + Parent: parent, + Children: children, + } + + return obj +} + +func (o *objectInfo) IsDelta() bool { + return o.Type.IsDelta() +} + +func (o *objectInfo) Size() int64 { + return o.Length +} diff --git a/plumbing/format/packfile/parser_test.go b/plumbing/format/packfile/parser_test.go new file mode 100644 index 000000000..87a880436 --- /dev/null +++ b/plumbing/format/packfile/parser_test.go @@ -0,0 +1,139 @@ +package packfile_test + +import ( + "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/format/packfile" + + . "gopkg.in/check.v1" + "gopkg.in/src-d/go-git-fixtures.v3" +) + +type ParserSuite struct { + fixtures.Suite +} + +var _ = Suite(&ParserSuite{}) + +func (s *ParserSuite) TestParserHashes(c *C) { + f := fixtures.Basic().One() + scanner := packfile.NewScanner(f.Packfile()) + + obs := new(testObserver) + parser := packfile.NewParser(scanner, obs) + + ch, err := parser.Parse() + c.Assert(err, IsNil) + + checksum := "a3fed42da1e8189a077c0e6846c040dcf73fc9dd" + c.Assert(ch.String(), Equals, checksum) + + c.Assert(obs.checksum, Equals, checksum) + c.Assert(int(obs.count), Equals, int(31)) + + commit := plumbing.CommitObject + blob := plumbing.BlobObject + tree := plumbing.TreeObject + + objs := []observerObject{ + {"e8d3ffab552895c19b9fcf7aa264d277cde33881", commit, 254, 12, 0xaa07ba4b}, + {"6ecf0ef2c2dffb796033e5a02219af86ec6584e5", commit, 93, 186, 0xf706df58}, + {"918c48b83bd081e863dbe1b80f8998f058cd8294", commit, 242, 286, 0x12438846}, + {"af2d6a6954d532f8ffb47615169c8fdf9d383a1a", commit, 242, 449, 0x2905a38c}, + {"1669dce138d9b841a518c64b10914d88f5e488ea", commit, 333, 615, 0xd9429436}, + {"a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69", commit, 332, 838, 0xbecfde4e}, + {"35e85108805c84807bc66a02d91535e1e24b38b9", commit, 244, 1063, 0x780e4b3e}, + {"b8e471f58bcbca63b07bda20e428190409c2db47", commit, 243, 1230, 0xdc18344f}, + {"b029517f6300c2da0f4b651b8642506cd6aaf45d", commit, 187, 1392, 0xcf4e4280}, + {"32858aad3c383ed1ff0a0f9bdf231d54a00c9e88", blob, 189, 1524, 0x1f08118a}, + {"d3ff53e0564a9f87d8e84b6e28e5060e517008aa", blob, 18, 1685, 0xafded7b8}, + {"c192bd6a24ea1ab01d78686e417c8bdc7c3d197f", blob, 1072, 1713, 0xcc1428ed}, + {"d5c0f4ab811897cadf03aec358ae60d21f91c50d", blob, 76110, 2351, 0x1631d22f}, + {"880cd14280f4b9b6ed3986d6671f907d7cc2a198", blob, 2780, 78050, 0xbfff5850}, + {"49c6bb89b17060d7b4deacb7b338fcc6ea2352a9", blob, 217848, 78882, 0xd108e1d8}, + {"c8f1d8c61f9da76f4cb49fd86322b6e685dba956", blob, 706, 80725, 0x8e97ba25}, + {"9a48f23120e880dfbe41f7c9b7b708e9ee62a492", blob, 11488, 80998, 0x7316ff70}, + {"9dea2395f5403188298c1dabe8bdafe562c491e3", blob, 78, 84032, 0xdb4fce56}, + {"dbd3641b371024f44d0e469a9c8f5457b0660de1", tree, 272, 84115, 0x901cce2c}, + {"a8d315b2b1c615d43042c3a62402b8a54288cf5c", tree, 43, 84375, 0xec4552b0}, + {"a39771a7651f97faf5c72e08224d857fc35133db", tree, 38, 84430, 0x847905bf}, + {"5a877e6a906a2743ad6e45d99c1793642aaf8eda", tree, 75, 84479, 0x3689459a}, + {"586af567d0bb5e771e49bdd9434f5e0fb76d25fa", tree, 38, 84559, 0xe67af94a}, + {"cf4aa3b38974fb7d81f367c0830f7d78d65ab86b", tree, 34, 84608, 0xc2314a2e}, + {"7e59600739c96546163833214c36459e324bad0a", blob, 9, 84653, 0xcd987848}, + {"fb72698cab7617ac416264415f13224dfd7a165e", tree, 6, 84671, 0x8a853a6d}, + {"4d081c50e250fa32ea8b1313cf8bb7c2ad7627fd", tree, 9, 84688, 0x70c6518}, + {"eba74343e2f15d62adedfd8c883ee0262b5c8021", tree, 6, 84708, 0x4f4108e2}, + {"c2d30fa8ef288618f65f6eed6e168e0d514886f4", tree, 5, 84725, 0xd6fe09e9}, + {"8dcef98b1d52143e1e2dbc458ffe38f925786bf2", tree, 8, 84741, 0xf07a2804}, + {"aa9b383c260e1d05fbbf6b30a02914555e20c725", tree, 4, 84760, 0x1d75d6be}, + } + + c.Assert(obs.objects, DeepEquals, objs) +} + +type observerObject struct { + hash string + otype plumbing.ObjectType + size int64 + offset int64 + crc uint32 +} + +type testObserver struct { + count uint32 + checksum string + objects []observerObject + pos map[int64]int +} + +func (t *testObserver) OnHeader(count uint32) error { + t.count = count + t.pos = make(map[int64]int, count) + return nil +} + +func (t *testObserver) OnInflatedObjectHeader(otype plumbing.ObjectType, objSize int64, pos int64) error { + o := t.get(pos) + o.otype = otype + o.size = objSize + o.offset = pos + + t.put(pos, o) + + return nil +} + +func (t *testObserver) OnInflatedObjectContent(h plumbing.Hash, pos int64, crc uint32) error { + o := t.get(pos) + o.hash = h.String() + o.crc = crc + + t.put(pos, o) + + return nil +} + +func (t *testObserver) OnFooter(h plumbing.Hash) error { + t.checksum = h.String() + return nil +} + +func (t *testObserver) get(pos int64) observerObject { + i, ok := t.pos[pos] + if ok { + return t.objects[i] + } + + return observerObject{} +} + +func (t *testObserver) put(pos int64, o observerObject) { + i, ok := t.pos[pos] + if ok { + t.objects[i] = o + return + } + + t.pos[pos] = len(t.objects) + t.objects = append(t.objects, o) +} From ce91d71f96097ede2bb77d2af444aee6fff73183 Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Thu, 19 Jul 2018 23:25:14 +0200 Subject: [PATCH 028/120] plumbing/packfile: disable lookup by offset In one case it disables the cache and the other disables lookup when the scanner is not seekable. Could be added back later. Signed-off-by: Javi Fontan --- plumbing/format/packfile/decoder.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/plumbing/format/packfile/decoder.go b/plumbing/format/packfile/decoder.go index 765401f5e..9bfd69ba8 100644 --- a/plumbing/format/packfile/decoder.go +++ b/plumbing/format/packfile/decoder.go @@ -403,12 +403,13 @@ func (d *Decoder) fillOFSDeltaObjectContent(obj plumbing.EncodedObject, offset i return 0, err } - e, ok := d.idx.LookupOffset(uint64(offset)) - var base plumbing.EncodedObject - if ok { - base, ok = d.cacheGet(e.Hash) - } + // e, ok := d.idx.LookupOffset(uint64(offset)) + // if ok { + // base, ok = d.cacheGet(e.Hash) + // } + var base plumbing.EncodedObject + ok := false if !ok { base, err = d.recallByOffset(offset) if err != nil { @@ -446,9 +447,9 @@ func (d *Decoder) recallByOffset(o int64) (plumbing.EncodedObject, error) { return d.DecodeObjectAt(o) } - if e, ok := d.idx.LookupOffset(uint64(o)); ok { - return d.recallByHashNonSeekable(e.Hash) - } + // if e, ok := d.idx.LookupOffset(uint64(o)); ok { + // return d.recallByHashNonSeekable(e.Hash) + // } return nil, plumbing.ErrObjectNotFound } From 355cfc3df3a64d1bd438e0e17e1c4ba21350badf Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Thu, 19 Jul 2018 23:27:16 +0200 Subject: [PATCH 029/120] plumbing: idxfile, add idxfile.Writer with Observer interface It's still not complete: * 64 bit offsets * IdxChecksum Signed-off-by: Javi Fontan --- plumbing/format/idxfile/writer.go | 132 ++++++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 plumbing/format/idxfile/writer.go diff --git a/plumbing/format/idxfile/writer.go b/plumbing/format/idxfile/writer.go new file mode 100644 index 000000000..aac68b503 --- /dev/null +++ b/plumbing/format/idxfile/writer.go @@ -0,0 +1,132 @@ +package idxfile + +import ( + "bytes" + "math" + "sort" + + "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/utils/binary" +) + +type object struct { + hash plumbing.Hash + offset int64 + crc uint32 +} + +type objects []object + +// Writer implements a packfile Observer interface and is used to generate +// indexes. +type Writer struct { + count uint32 + checksum plumbing.Hash + objects objects +} + +// Create index returns a filled MemoryIndex with the information filled by +// the observer callbacks. +func (w *Writer) CreateIndex() (*MemoryIndex, error) { + idx := new(MemoryIndex) + sort.Sort(w.objects) + + // unmap all fans by default + for i := range idx.FanoutMapping { + idx.FanoutMapping[i] = noMapping + } + + buf := new(bytes.Buffer) + + last := -1 + bucket := -1 + for i, o := range w.objects { + fan := o.hash[0] + + // fill the gaps between fans + for j := last + 1; j < int(fan); j++ { + idx.Fanout[j] = uint32(i) + } + + // update the number of objects for this position + idx.Fanout[fan] = uint32(i + 1) + + // we move from one bucket to another, update counters and allocate + // memory + if last != int(fan) { + bucket++ + idx.FanoutMapping[fan] = bucket + last = int(fan) + + idx.Names = append(idx.Names, make([]byte, 0)) + idx.Offset32 = append(idx.Offset32, make([]byte, 0)) + idx.Crc32 = append(idx.Crc32, make([]byte, 0)) + } + + idx.Names[bucket] = append(idx.Names[bucket], o.hash[:]...) + + // TODO: implement 64 bit offsets + if o.offset > math.MaxInt32 { + panic("64 bit offsets not implemented") + } + + buf.Truncate(0) + binary.WriteUint32(buf, uint32(o.offset)) + idx.Offset32[bucket] = append(idx.Offset32[bucket], buf.Bytes()...) + + buf.Truncate(0) + binary.WriteUint32(buf, uint32(o.crc)) + idx.Crc32[bucket] = append(idx.Crc32[bucket], buf.Bytes()...) + } + + for j := last + 1; j < 256; j++ { + idx.Fanout[j] = uint32(len(w.objects)) + } + + idx.PackfileChecksum = w.checksum + // TODO: fill IdxChecksum + + return idx, nil +} + +// Add appends new object data. +func (w *Writer) Add(h plumbing.Hash, pos int64, crc uint32) { + w.objects = append(w.objects, object{h, pos, crc}) +} + +// OnHeader implements packfile.Observer interface. +func (w *Writer) OnHeader(count uint32) error { + w.count = count + w.objects = make(objects, 0, count) + return nil +} + +// OnInflatedObjectHeader implements packfile.Observer interface. +func (w *Writer) OnInflatedObjectHeader(t plumbing.ObjectType, objSize int64, pos int64) error { + return nil +} + +// OnInflatedObjectContent implements packfile.Observer interface. +func (w *Writer) OnInflatedObjectContent(h plumbing.Hash, pos int64, crc uint32) error { + w.Add(h, pos, crc) + return nil +} + +// OnFooter implements packfile.Observer interface. +func (w *Writer) OnFooter(h plumbing.Hash) error { + w.checksum = h + return nil +} + +func (o objects) Len() int { + return len(o) +} + +func (o objects) Less(i int, j int) bool { + cmp := bytes.Compare(o[i].hash[:], o[j].hash[:]) + return cmp < 0 +} + +func (o objects) Swap(i int, j int) { + o[i], o[j] = o[j], o[i] +} From 4e3765aef344eae2fbcd977fefd66b6571638d59 Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Fri, 20 Jul 2018 12:22:55 +0200 Subject: [PATCH 030/120] plumbing/idxfile: use Entry to hold object data Signed-off-by: Javi Fontan --- plumbing/format/idxfile/writer.go | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/plumbing/format/idxfile/writer.go b/plumbing/format/idxfile/writer.go index aac68b503..3c5a00e3c 100644 --- a/plumbing/format/idxfile/writer.go +++ b/plumbing/format/idxfile/writer.go @@ -9,13 +9,8 @@ import ( "gopkg.in/src-d/go-git.v4/utils/binary" ) -type object struct { - hash plumbing.Hash - offset int64 - crc uint32 -} - -type objects []object +// objects implements sort.Interface and uses hash as sorting key. +type objects []Entry // Writer implements a packfile Observer interface and is used to generate // indexes. @@ -41,7 +36,7 @@ func (w *Writer) CreateIndex() (*MemoryIndex, error) { last := -1 bucket := -1 for i, o := range w.objects { - fan := o.hash[0] + fan := o.Hash[0] // fill the gaps between fans for j := last + 1; j < int(fan); j++ { @@ -63,19 +58,19 @@ func (w *Writer) CreateIndex() (*MemoryIndex, error) { idx.Crc32 = append(idx.Crc32, make([]byte, 0)) } - idx.Names[bucket] = append(idx.Names[bucket], o.hash[:]...) + idx.Names[bucket] = append(idx.Names[bucket], o.Hash[:]...) // TODO: implement 64 bit offsets - if o.offset > math.MaxInt32 { + if o.Offset > math.MaxInt32 { panic("64 bit offsets not implemented") } buf.Truncate(0) - binary.WriteUint32(buf, uint32(o.offset)) + binary.WriteUint32(buf, uint32(o.Offset)) idx.Offset32[bucket] = append(idx.Offset32[bucket], buf.Bytes()...) buf.Truncate(0) - binary.WriteUint32(buf, uint32(o.crc)) + binary.WriteUint32(buf, uint32(o.CRC32)) idx.Crc32[bucket] = append(idx.Crc32[bucket], buf.Bytes()...) } @@ -90,8 +85,8 @@ func (w *Writer) CreateIndex() (*MemoryIndex, error) { } // Add appends new object data. -func (w *Writer) Add(h plumbing.Hash, pos int64, crc uint32) { - w.objects = append(w.objects, object{h, pos, crc}) +func (w *Writer) Add(h plumbing.Hash, pos uint64, crc uint32) { + w.objects = append(w.objects, Entry{h, crc, pos}) } // OnHeader implements packfile.Observer interface. @@ -108,7 +103,7 @@ func (w *Writer) OnInflatedObjectHeader(t plumbing.ObjectType, objSize int64, po // OnInflatedObjectContent implements packfile.Observer interface. func (w *Writer) OnInflatedObjectContent(h plumbing.Hash, pos int64, crc uint32) error { - w.Add(h, pos, crc) + w.Add(h, uint64(pos), crc) return nil } @@ -123,7 +118,7 @@ func (o objects) Len() int { } func (o objects) Less(i int, j int) bool { - cmp := bytes.Compare(o[i].hash[:], o[j].hash[:]) + cmp := bytes.Compare(o[i].Hash[:], o[j].Hash[:]) return cmp < 0 } From 65e8359db00ae79838d19e19f69594f6a262c3d4 Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Fri, 20 Jul 2018 13:01:27 +0200 Subject: [PATCH 031/120] plumbing/idxfile: support offset64 generating indexes Signed-off-by: Javi Fontan --- plumbing/format/idxfile/writer.go | 25 +++++++++++--- plumbing/format/idxfile/writer_test.go | 45 ++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 5 deletions(-) create mode 100644 plumbing/format/idxfile/writer_test.go diff --git a/plumbing/format/idxfile/writer.go b/plumbing/format/idxfile/writer.go index 3c5a00e3c..ea5408186 100644 --- a/plumbing/format/idxfile/writer.go +++ b/plumbing/format/idxfile/writer.go @@ -18,12 +18,16 @@ type Writer struct { count uint32 checksum plumbing.Hash objects objects + offset64 uint32 + idx *MemoryIndex } // Create index returns a filled MemoryIndex with the information filled by // the observer callbacks. func (w *Writer) CreateIndex() (*MemoryIndex, error) { idx := new(MemoryIndex) + w.idx = idx + sort.Sort(w.objects) // unmap all fans by default @@ -60,13 +64,13 @@ func (w *Writer) CreateIndex() (*MemoryIndex, error) { idx.Names[bucket] = append(idx.Names[bucket], o.Hash[:]...) - // TODO: implement 64 bit offsets - if o.Offset > math.MaxInt32 { - panic("64 bit offsets not implemented") + offset := o.Offset + if offset > math.MaxInt32 { + offset = w.addOffset64(offset) } buf.Truncate(0) - binary.WriteUint32(buf, uint32(o.Offset)) + binary.WriteUint32(buf, uint32(offset)) idx.Offset32[bucket] = append(idx.Offset32[bucket], buf.Bytes()...) buf.Truncate(0) @@ -78,12 +82,23 @@ func (w *Writer) CreateIndex() (*MemoryIndex, error) { idx.Fanout[j] = uint32(len(w.objects)) } + idx.Version = VersionSupported idx.PackfileChecksum = w.checksum - // TODO: fill IdxChecksum return idx, nil } +func (w *Writer) addOffset64(pos uint64) uint64 { + buf := new(bytes.Buffer) + binary.WriteUint64(buf, pos) + w.idx.Offset64 = append(w.idx.Offset64, buf.Bytes()...) + + index := uint64(w.offset64 | (1 << 31)) + w.offset64++ + + return index +} + // Add appends new object data. func (w *Writer) Add(h plumbing.Hash, pos uint64, crc uint32) { w.objects = append(w.objects, Entry{h, crc, pos}) diff --git a/plumbing/format/idxfile/writer_test.go b/plumbing/format/idxfile/writer_test.go new file mode 100644 index 000000000..92d2046cf --- /dev/null +++ b/plumbing/format/idxfile/writer_test.go @@ -0,0 +1,45 @@ +package idxfile_test + +import ( + "bytes" + "io/ioutil" + + "gopkg.in/src-d/go-git.v4/plumbing/format/idxfile" + "gopkg.in/src-d/go-git.v4/plumbing/format/packfile" + + . "gopkg.in/check.v1" + "gopkg.in/src-d/go-git-fixtures.v3" +) + +type IndexSuite struct { + fixtures.Suite +} + +var _ = Suite(&IndexSuite{}) + +func (s *IndexSuite) TestIndexWriter(c *C) { + f := fixtures.Basic().One() + scanner := packfile.NewScanner(f.Packfile()) + + obs := new(idxfile.Writer) + parser := packfile.NewParser(scanner, obs) + + _, err := parser.Parse() + c.Assert(err, IsNil) + + idx, err := obs.CreateIndex() + c.Assert(err, IsNil) + + idxFile := f.Idx() + expected, err := ioutil.ReadAll(idxFile) + c.Assert(err, IsNil) + idxFile.Close() + + buf := new(bytes.Buffer) + encoder := idxfile.NewEncoder(buf) + n, err := encoder.Encode(idx) + c.Assert(err, IsNil) + c.Assert(n, Equals, len(expected)) + + c.Assert(buf.Bytes(), DeepEquals, expected) +} From a716126aa7f9b77030d2e697db24d206d944f05d Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Tue, 24 Jul 2018 17:36:21 +0200 Subject: [PATCH 032/120] plumbing/packfile: preallocate memory in PatchDelta Signed-off-by: Javi Fontan --- plumbing/format/packfile/patch_delta.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plumbing/format/packfile/patch_delta.go b/plumbing/format/packfile/patch_delta.go index c60485179..a972f1c42 100644 --- a/plumbing/format/packfile/patch_delta.go +++ b/plumbing/format/packfile/patch_delta.go @@ -63,8 +63,8 @@ func PatchDelta(src, delta []byte) ([]byte, error) { targetSz, delta := decodeLEB128(delta) remainingTargetSz := targetSz - var dest []byte var cmd byte + dest := make([]byte, 0, targetSz) for { if len(delta) == 0 { return nil, ErrInvalidDelta From 7418b411660aaa3d8d54eb602fda8accaed2833f Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Thu, 26 Jul 2018 12:24:26 +0200 Subject: [PATCH 033/120] plumbing/idxfile: fix bug searching in MemoryIndex Signed-off-by: Javi Fontan --- plumbing/format/idxfile/idxfile.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plumbing/format/idxfile/idxfile.go b/plumbing/format/idxfile/idxfile.go index b1966086c..adeba448c 100644 --- a/plumbing/format/idxfile/idxfile.go +++ b/plumbing/format/idxfile/idxfile.go @@ -72,7 +72,7 @@ func (idx *MemoryIndex) findHashIndex(h plumbing.Hash) int { low := uint64(0) for { mid := (low + high) >> 1 - offset := mid + (mid << 2) + offset := mid * objectIDLength cmp := bytes.Compare(h[:], data[offset:offset+objectIDLength]) if cmp < 0 { @@ -83,7 +83,7 @@ func (idx *MemoryIndex) findHashIndex(h plumbing.Hash) int { low = mid + 1 } - if low < high { + if low > high { break } } From 4ddd6783cf9707f8b72ebb00e5bb4705f5fd436a Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Thu, 26 Jul 2018 12:27:53 +0200 Subject: [PATCH 034/120] plumbing/idxfile: add offset/hash mapping to index This functionality may be moved elsewhere in the future but is needed now to fit filesystem.ObjectStorage and the new index. Signed-off-by: Javi Fontan --- plumbing/format/idxfile/idxfile.go | 51 ++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/plumbing/format/idxfile/idxfile.go b/plumbing/format/idxfile/idxfile.go index adeba448c..f8debb1a9 100644 --- a/plumbing/format/idxfile/idxfile.go +++ b/plumbing/format/idxfile/idxfile.go @@ -28,6 +28,8 @@ type Index interface { FindOffset(h plumbing.Hash) (int64, error) // FindCRC32 finds the CRC32 of the object with the given hash. FindCRC32(h plumbing.Hash) (uint32, error) + // FindHash finds the hash for the object with the given offset. + FindHash(o int64) (plumbing.Hash, error) // Count returns the number of entries in the index. Count() (int64, error) // Entries returns an iterator to retrieve all index entries. @@ -48,6 +50,8 @@ type MemoryIndex struct { Offset64 []byte PackfileChecksum [20]byte IdxChecksum [20]byte + + offsetHash map[int64]plumbing.Hash } var _ Index = (*MemoryIndex)(nil) @@ -149,6 +153,53 @@ func (idx *MemoryIndex) getCrc32(firstLevel, secondLevel int) (uint32, error) { return binary.ReadUint32(buf) } +// FindHash implements the Index interface. +func (idx *MemoryIndex) FindHash(o int64) (plumbing.Hash, error) { + // Lazily generate the reverse offset/hash map if required. + if idx.offsetHash == nil { + err := idx.genOffsetHash() + if err != nil { + return plumbing.ZeroHash, nil + } + } + + hash, ok := idx.offsetHash[o] + if !ok { + return plumbing.ZeroHash, plumbing.ErrObjectNotFound + } + + return hash, nil +} + +// genOffsetHash generates the offset/hash mapping for reverse search. +func (idx *MemoryIndex) genOffsetHash() error { + count, err := idx.Count() + if err != nil { + return err + } + + idx.offsetHash = make(map[int64]plumbing.Hash, count) + + iter, err := idx.Entries() + if err != nil { + return err + } + + var entry *Entry + for err != nil { + entry, err = iter.Next() + if err == nil { + idx.offsetHash[int64(entry.Offset)] = entry.Hash + } + } + + if err == io.EOF { + return nil + } + + return err +} + // Count implements the Index interface. func (idx *MemoryIndex) Count() (int64, error) { return int64(idx.Fanout[fanout-1]), nil From 74f56f388bbe8072bfcd976add2373f9a7e20341 Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Thu, 26 Jul 2018 13:14:02 +0200 Subject: [PATCH 035/120] plumbing/idxfile: index is created only once and retrieved with Index Index is also automatically generated when OnFooter is called. Signed-off-by: Javi Fontan --- plumbing/format/idxfile/writer.go | 103 +++++++++++++++++-------- plumbing/format/idxfile/writer_test.go | 2 +- 2 files changed, 70 insertions(+), 35 deletions(-) diff --git a/plumbing/format/idxfile/writer.go b/plumbing/format/idxfile/writer.go index ea5408186..efcdcc6c1 100644 --- a/plumbing/format/idxfile/writer.go +++ b/plumbing/format/idxfile/writer.go @@ -2,8 +2,10 @@ package idxfile import ( "bytes" + "fmt" "math" "sort" + "sync" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/utils/binary" @@ -15,18 +17,80 @@ type objects []Entry // Writer implements a packfile Observer interface and is used to generate // indexes. type Writer struct { + m sync.Mutex + count uint32 checksum plumbing.Hash objects objects offset64 uint32 - idx *MemoryIndex + finished bool + index *MemoryIndex +} + +// Index returns a previously created MemoryIndex or creates a new one if +// needed. +func (w *Writer) Index() (*MemoryIndex, error) { + w.m.Lock() + defer w.m.Unlock() + + if w.index == nil { + return w.createIndex() + } + + return w.index, nil +} + +// Add appends new object data. +func (w *Writer) Add(h plumbing.Hash, pos uint64, crc uint32) { + w.m.Lock() + defer w.m.Unlock() + + w.objects = append(w.objects, Entry{h, crc, pos}) +} + +func (w *Writer) Finished() bool { + return w.finished +} + +// OnHeader implements packfile.Observer interface. +func (w *Writer) OnHeader(count uint32) error { + w.count = count + w.objects = make(objects, 0, count) + return nil +} + +// OnInflatedObjectHeader implements packfile.Observer interface. +func (w *Writer) OnInflatedObjectHeader(t plumbing.ObjectType, objSize int64, pos int64) error { + return nil +} + +// OnInflatedObjectContent implements packfile.Observer interface. +func (w *Writer) OnInflatedObjectContent(h plumbing.Hash, pos int64, crc uint32) error { + w.Add(h, uint64(pos), crc) + return nil } -// Create index returns a filled MemoryIndex with the information filled by +// OnFooter implements packfile.Observer interface. +func (w *Writer) OnFooter(h plumbing.Hash) error { + w.checksum = h + w.finished = true + _, err := w.createIndex() + if err != nil { + return err + } + + return nil +} + +// creatIndex returns a filled MemoryIndex with the information filled by // the observer callbacks. -func (w *Writer) CreateIndex() (*MemoryIndex, error) { +func (w *Writer) createIndex() (*MemoryIndex, error) { + if !w.finished { + return nil, fmt.Errorf("the index still hasn't finished building") + } + idx := new(MemoryIndex) - w.idx = idx + w.index = idx sort.Sort(w.objects) @@ -91,7 +155,7 @@ func (w *Writer) CreateIndex() (*MemoryIndex, error) { func (w *Writer) addOffset64(pos uint64) uint64 { buf := new(bytes.Buffer) binary.WriteUint64(buf, pos) - w.idx.Offset64 = append(w.idx.Offset64, buf.Bytes()...) + w.index.Offset64 = append(w.index.Offset64, buf.Bytes()...) index := uint64(w.offset64 | (1 << 31)) w.offset64++ @@ -99,35 +163,6 @@ func (w *Writer) addOffset64(pos uint64) uint64 { return index } -// Add appends new object data. -func (w *Writer) Add(h plumbing.Hash, pos uint64, crc uint32) { - w.objects = append(w.objects, Entry{h, crc, pos}) -} - -// OnHeader implements packfile.Observer interface. -func (w *Writer) OnHeader(count uint32) error { - w.count = count - w.objects = make(objects, 0, count) - return nil -} - -// OnInflatedObjectHeader implements packfile.Observer interface. -func (w *Writer) OnInflatedObjectHeader(t plumbing.ObjectType, objSize int64, pos int64) error { - return nil -} - -// OnInflatedObjectContent implements packfile.Observer interface. -func (w *Writer) OnInflatedObjectContent(h plumbing.Hash, pos int64, crc uint32) error { - w.Add(h, uint64(pos), crc) - return nil -} - -// OnFooter implements packfile.Observer interface. -func (w *Writer) OnFooter(h plumbing.Hash) error { - w.checksum = h - return nil -} - func (o objects) Len() int { return len(o) } diff --git a/plumbing/format/idxfile/writer_test.go b/plumbing/format/idxfile/writer_test.go index 92d2046cf..51273a365 100644 --- a/plumbing/format/idxfile/writer_test.go +++ b/plumbing/format/idxfile/writer_test.go @@ -27,7 +27,7 @@ func (s *IndexSuite) TestIndexWriter(c *C) { _, err := parser.Parse() c.Assert(err, IsNil) - idx, err := obs.CreateIndex() + idx, err := obs.Index() c.Assert(err, IsNil) idxFile := f.Idx() From 79f249465b24104b73c9dc220d9098cecdab4d77 Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Thu, 26 Jul 2018 13:42:51 +0200 Subject: [PATCH 036/120] plumbing, storage: integrate new index Now dotgit.PackWriter uses the new packfile.Parser and index. Signed-off-by: Javi Fontan --- plumbing/format/packfile/decoder.go | 9 +++-- plumbing/format/packfile/parser.go | 11 +++--- storage/filesystem/dotgit/writers.go | 33 ++++++++-------- storage/filesystem/dotgit/writers_test.go | 3 +- storage/filesystem/object.go | 46 ++++++++++++++--------- 5 files changed, 57 insertions(+), 45 deletions(-) diff --git a/plumbing/format/packfile/decoder.go b/plumbing/format/packfile/decoder.go index 9bfd69ba8..69aef2d7d 100644 --- a/plumbing/format/packfile/decoder.go +++ b/plumbing/format/packfile/decoder.go @@ -447,11 +447,12 @@ func (d *Decoder) recallByOffset(o int64) (plumbing.EncodedObject, error) { return d.DecodeObjectAt(o) } - // if e, ok := d.idx.LookupOffset(uint64(o)); ok { - // return d.recallByHashNonSeekable(e.Hash) - // } + hash, err := d.idx.FindHash(o) + if err != nil { + return nil, err + } - return nil, plumbing.ErrObjectNotFound + return d.recallByHashNonSeekable(hash) } func (d *Decoder) recallByHash(h plumbing.Hash) (plumbing.EncodedObject, error) { diff --git a/plumbing/format/packfile/parser.go b/plumbing/format/packfile/parser.go index 460fc3f5a..696f5ba96 100644 --- a/plumbing/format/packfile/parser.go +++ b/plumbing/format/packfile/parser.go @@ -311,11 +311,12 @@ func applyPatchBase(ota *objectInfo, data, base []byte) ([]byte, error) { type objectInfo struct { plumbing.Hasher - Offset int64 - Length int64 - PackSize int64 - Type plumbing.ObjectType - DiskType plumbing.ObjectType + Offset int64 + Length int64 + HeaderLength int64 + PackSize int64 + Type plumbing.ObjectType + DiskType plumbing.ObjectType Crc32 uint32 diff --git a/storage/filesystem/dotgit/writers.go b/storage/filesystem/dotgit/writers.go index c2b420f8a..e1ede3cb9 100644 --- a/storage/filesystem/dotgit/writers.go +++ b/storage/filesystem/dotgit/writers.go @@ -20,13 +20,14 @@ import ( // is renamed/moved (depends on the Filesystem implementation) to the final // location, if the PackWriter is not used, nothing is written type PackWriter struct { - Notify func(plumbing.Hash, *packfile.Index) + Notify func(plumbing.Hash, *idxfile.Writer) fs billy.Filesystem fr, fw billy.File synced *syncedReader checksum plumbing.Hash - index *packfile.Index + parser *packfile.Parser + writer *idxfile.Writer result chan error } @@ -55,20 +56,16 @@ func newPackWrite(fs billy.Filesystem) (*PackWriter, error) { func (w *PackWriter) buildIndex() { s := packfile.NewScanner(w.synced) - d, err := packfile.NewDecoder(s, nil) - if err != nil { - w.result <- err - return - } + w.writer = new(idxfile.Writer) + w.parser = packfile.NewParser(s, w.writer) - checksum, err := d.Decode() + checksum, err := w.parser.Parse() if err != nil { w.result <- err return } w.checksum = checksum - w.index = d.Index() w.result <- err } @@ -92,8 +89,8 @@ func (w *PackWriter) Write(p []byte) (int, error) { // was written, the tempfiles are deleted without writing a packfile. func (w *PackWriter) Close() error { defer func() { - if w.Notify != nil && w.index != nil && w.index.Size() > 0 { - w.Notify(w.checksum, w.index) + if w.Notify != nil && w.writer != nil && w.writer.Finished() { + w.Notify(w.checksum, w.writer) } close(w.result) @@ -115,7 +112,7 @@ func (w *PackWriter) Close() error { return err } - if w.index == nil || w.index.Size() == 0 { + if w.writer == nil || !w.writer.Finished() { return w.clean() } @@ -145,11 +142,13 @@ func (w *PackWriter) save() error { } func (w *PackWriter) encodeIdx(writer io.Writer) error { - idx := w.index.ToIdxFile() - idx.PackfileChecksum = w.checksum - idx.Version = idxfile.VersionSupported + idx, err := w.writer.Index() + if err != nil { + return err + } + e := idxfile.NewEncoder(writer) - _, err := e.Encode(idx) + _, err = e.Encode(idx) return err } @@ -209,7 +208,6 @@ func (s *syncedReader) isBlocked() bool { func (s *syncedReader) wake() { if s.isBlocked() { - // fmt.Println("wake") atomic.StoreUint32(&s.blocked, 0) s.news <- true } @@ -220,7 +218,6 @@ func (s *syncedReader) sleep() { written := atomic.LoadUint64(&s.written) if read >= written { atomic.StoreUint32(&s.blocked, 1) - // fmt.Println("sleep", read, written) <-s.news } diff --git a/storage/filesystem/dotgit/writers_test.go b/storage/filesystem/dotgit/writers_test.go index bf0076203..5a5f7b402 100644 --- a/storage/filesystem/dotgit/writers_test.go +++ b/storage/filesystem/dotgit/writers_test.go @@ -9,6 +9,7 @@ import ( "strconv" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/format/idxfile" "gopkg.in/src-d/go-git.v4/plumbing/format/packfile" . "gopkg.in/check.v1" @@ -148,7 +149,7 @@ func (s *SuiteDotGit) TestPackWriterUnusedNotify(c *C) { w, err := newPackWrite(fs) c.Assert(err, IsNil) - w.Notify = func(h plumbing.Hash, idx *packfile.Index) { + w.Notify = func(h plumbing.Hash, idx *idxfile.Writer) { c.Fatal("unexpected call to PackWriter.Notify") } diff --git a/storage/filesystem/object.go b/storage/filesystem/object.go index ef67f5011..b73b3093e 100644 --- a/storage/filesystem/object.go +++ b/storage/filesystem/object.go @@ -23,7 +23,7 @@ type ObjectStorage struct { deltaBaseCache cache.Object dir *dotgit.DotGit - index map[plumbing.Hash]*packfile.Index + index map[plumbing.Hash]idxfile.Index } // NewObjectStorage creates a new ObjectStorage with the given .git directory. @@ -41,7 +41,7 @@ func (s *ObjectStorage) requireIndex() error { return nil } - s.index = make(map[plumbing.Hash]*packfile.Index) + s.index = make(map[plumbing.Hash]idxfile.Index) packs, err := s.dir.ObjectPacks() if err != nil { return err @@ -69,7 +69,7 @@ func (s *ObjectStorage) loadIdxFile(h plumbing.Hash) (err error) { return err } - s.index[h] = packfile.NewIndexFromIdxFile(idxf) + s.index[h] = idxf return err } @@ -87,8 +87,11 @@ func (s *ObjectStorage) PackfileWriter() (io.WriteCloser, error) { return nil, err } - w.Notify = func(h plumbing.Hash, idx *packfile.Index) { - s.index[h] = idx + w.Notify = func(h plumbing.Hash, writer *idxfile.Writer) { + index, err := writer.Index() + if err == nil { + s.index[h] = index + } } return w, nil @@ -278,7 +281,7 @@ func (s *ObjectStorage) getFromPackfile(h plumbing.Hash, canBeDelta bool) ( func (s *ObjectStorage) decodeObjectAt( f billy.File, - idx *packfile.Index, + idx idxfile.Index, offset int64) (plumbing.EncodedObject, error) { if _, err := f.Seek(0, io.SeekStart); err != nil { return nil, err @@ -299,7 +302,7 @@ func (s *ObjectStorage) decodeObjectAt( func (s *ObjectStorage) decodeDeltaObjectAt( f billy.File, - idx *packfile.Index, + idx idxfile.Index, offset int64, hash plumbing.Hash) (plumbing.EncodedObject, error) { if _, err := f.Seek(0, io.SeekStart); err != nil { @@ -324,12 +327,10 @@ func (s *ObjectStorage) decodeDeltaObjectAt( case plumbing.REFDeltaObject: base = header.Reference case plumbing.OFSDeltaObject: - e, ok := idx.LookupOffset(uint64(header.OffsetReference)) - if !ok { - return nil, plumbing.ErrObjectNotFound + base, err = idx.FindHash(header.OffsetReference) + if err != nil { + return nil, err } - - base = e.Hash default: return s.decodeObjectAt(f, idx, offset) } @@ -350,8 +351,9 @@ func (s *ObjectStorage) decodeDeltaObjectAt( func (s *ObjectStorage) findObjectInPackfile(h plumbing.Hash) (plumbing.Hash, plumbing.Hash, int64) { for packfile, index := range s.index { - if e, ok := index.LookupHash(h); ok { - return packfile, e.Hash, int64(e.Offset) + offset, err := index.FindOffset(h) + if err == nil { + return packfile, h, offset } } @@ -460,12 +462,22 @@ type packfileIter struct { total uint32 } -func NewPackfileIter(f billy.File, t plumbing.ObjectType) (storer.EncodedObjectIter, error) { +// NewPackfileIter returns a new EncodedObjectIter for the provided packfile +// and object type. +func NewPackfileIter( + f billy.File, + t plumbing.ObjectType, +) (storer.EncodedObjectIter, error) { return newPackfileIter(f, t, make(map[plumbing.Hash]struct{}), nil, nil) } -func newPackfileIter(f billy.File, t plumbing.ObjectType, seen map[plumbing.Hash]struct{}, - index *packfile.Index, cache cache.Object) (storer.EncodedObjectIter, error) { +func newPackfileIter( + f billy.File, + t plumbing.ObjectType, + seen map[plumbing.Hash]struct{}, + index idxfile.Index, + cache cache.Object, +) (storer.EncodedObjectIter, error) { s := packfile.NewScanner(f) _, total, err := s.Header() if err != nil { From ffdfb7dbabb78090b27ca29b762b803969c89fd7 Mon Sep 17 00:00:00 2001 From: Miguel Molina Date: Fri, 20 Jul 2018 15:51:15 +0200 Subject: [PATCH 037/120] plumbing: packfile, new Packfile representation Signed-off-by: Miguel Molina --- plumbing/format/packfile/decoder.go | 57 +++-- plumbing/format/packfile/decoder_test.go | 12 +- plumbing/format/packfile/index_test.go | 133 ------------ plumbing/format/packfile/packfile.go | 249 ++++++++++++++++++++++ plumbing/format/packfile/packfile_test.go | 121 +++++++++++ plumbing/memory.go | 8 +- storage/filesystem/storage.go | 15 ++ 7 files changed, 437 insertions(+), 158 deletions(-) delete mode 100644 plumbing/format/packfile/index_test.go create mode 100644 plumbing/format/packfile/packfile.go create mode 100644 plumbing/format/packfile/packfile_test.go diff --git a/plumbing/format/packfile/decoder.go b/plumbing/format/packfile/decoder.go index 69aef2d7d..b1a0a2695 100644 --- a/plumbing/format/packfile/decoder.go +++ b/plumbing/format/packfile/decoder.go @@ -2,6 +2,7 @@ package packfile import ( "bytes" + "io" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/cache" @@ -68,6 +69,7 @@ type Decoder struct { offsetToType map[int64]plumbing.ObjectType decoderType plumbing.ObjectType + offsetToHash map[int64]plumbing.Hash } // NewDecoder returns a new Decoder that decodes a Packfile using the given @@ -120,6 +122,7 @@ func NewDecoderForType(s *Scanner, o storer.EncodedObjectStorer, idx: idxfile.NewMemoryIndex(), offsetToType: make(map[int64]plumbing.ObjectType), + offsetToHash: make(map[int64]plumbing.Hash), decoderType: t, }, nil } @@ -144,6 +147,27 @@ func (d *Decoder) Decode() (checksum plumbing.Hash, err error) { return d.s.Checksum() } +func (d *Decoder) fillOffsetsToHashes() error { + entries, err := d.idx.Entries() + if err != nil { + return err + } + + for { + e, err := entries.Next() + if err != nil { + if err == io.EOF { + break + } + return err + } + + d.offsetToHash[int64(e.Offset)] = e.Hash + } + + return entries.Close() +} + func (d *Decoder) doDecode() error { _, count, err := d.s.Header() if err != nil { @@ -156,6 +180,12 @@ func (d *Decoder) doDecode() error { } defer func() { d.hasBuiltIndex = true }() + if d.hasBuiltIndex && !d.s.IsSeekable { + if err := d.fillOffsetsToHashes(); err != nil { + return err + } + } + _, isTxStorer := d.o.(storer.Transactioner) switch { case d.o == nil: @@ -299,15 +329,14 @@ func (d *Decoder) decodeByHeader(h *ObjectHeader) (plumbing.EncodedObject, error obj.SetSize(h.Length) obj.SetType(h.Type) - var crc uint32 var err error switch h.Type { case plumbing.CommitObject, plumbing.TreeObject, plumbing.BlobObject, plumbing.TagObject: - crc, err = d.fillRegularObjectContent(obj) + _, err = d.fillRegularObjectContent(obj) case plumbing.REFDeltaObject: - crc, err = d.fillREFDeltaObjectContent(obj, h.Reference) + _, err = d.fillREFDeltaObjectContent(obj, h.Reference) case plumbing.OFSDeltaObject: - crc, err = d.fillOFSDeltaObjectContent(obj, h.OffsetReference) + _, err = d.fillOFSDeltaObjectContent(obj, h.OffsetReference) default: err = ErrInvalidObject.AddDetails("type %q", h.Type) } @@ -316,14 +345,7 @@ func (d *Decoder) decodeByHeader(h *ObjectHeader) (plumbing.EncodedObject, error return obj, err } - // TODO: remove this - _ = crc - - /* Add is no longer available - if !d.hasBuiltIndex { - d.idx.Add(obj.Hash(), uint64(h.Offset), crc) - } - */ + d.offsetToHash[h.Offset] = obj.Hash() return obj, nil } @@ -403,13 +425,12 @@ func (d *Decoder) fillOFSDeltaObjectContent(obj plumbing.EncodedObject, offset i return 0, err } - // e, ok := d.idx.LookupOffset(uint64(offset)) - // if ok { - // base, ok = d.cacheGet(e.Hash) - // } - + h, ok := d.offsetToHash[offset] var base plumbing.EncodedObject - ok := false + if ok { + base, ok = d.cacheGet(h) + } + if !ok { base, err = d.recallByOffset(offset) if err != nil { diff --git a/plumbing/format/packfile/decoder_test.go b/plumbing/format/packfile/decoder_test.go index b5bc7b7c5..4fe9b5e58 100644 --- a/plumbing/format/packfile/decoder_test.go +++ b/plumbing/format/packfile/decoder_test.go @@ -5,7 +5,6 @@ import ( "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/cache" - "gopkg.in/src-d/go-git.v4/plumbing/format/idxfile" "gopkg.in/src-d/go-git.v4/plumbing/format/packfile" "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/storage/filesystem" @@ -47,6 +46,7 @@ func (s *ReaderSuite) TestDecode(c *C) { }) } +/* func (s *ReaderSuite) TestDecodeByTypeRefDelta(c *C) { f := fixtures.Basic().ByTag("ref-delta").One() @@ -101,7 +101,9 @@ func (s *ReaderSuite) TestDecodeByTypeRefDeltaError(c *C) { }) } +*/ +/* func (s *ReaderSuite) TestDecodeByType(c *C) { ts := []plumbing.ObjectType{ plumbing.CommitObject, @@ -140,6 +142,8 @@ func (s *ReaderSuite) TestDecodeByType(c *C) { } }) } +*/ + func (s *ReaderSuite) TestDecodeByTypeConstructor(c *C) { f := fixtures.Basic().ByTag("packfile").One() storage := memory.NewStorage() @@ -280,6 +284,7 @@ var expectedHashes = []string{ "7e59600739c96546163833214c36459e324bad0a", } +/* func (s *ReaderSuite) TestDecodeCRCs(c *C) { f := fixtures.Basic().ByTag("ofs-delta").One() @@ -366,7 +371,7 @@ func (s *ReaderSuite) TestSetIndex(c *C) { idxf := d.Index().ToIdxFile() c.Assert(idxf.Entries, HasLen, 1) c.Assert(idxf.Entries[0].Offset, Equals, uint64(42)) -} +}*/ func assertObjects(c *C, s storer.EncodedObjectStorer, expects []string) { @@ -385,6 +390,7 @@ func assertObjects(c *C, s storer.EncodedObjectStorer, expects []string) { } } +/* func getIndexFromIdxFile(r io.Reader) *packfile.Index { idxf := idxfile.NewIdxfile() d := idxfile.NewDecoder(r) @@ -393,4 +399,4 @@ func getIndexFromIdxFile(r io.Reader) *packfile.Index { } return packfile.NewIndexFromIdxFile(idxf) -} +}*/ diff --git a/plumbing/format/packfile/index_test.go b/plumbing/format/packfile/index_test.go deleted file mode 100644 index 8de886dac..000000000 --- a/plumbing/format/packfile/index_test.go +++ /dev/null @@ -1,133 +0,0 @@ -package packfile - -import ( - "strconv" - "strings" - "testing" - - "gopkg.in/src-d/go-git.v4/plumbing" - - . "gopkg.in/check.v1" -) - -type IndexSuite struct{} - -var _ = Suite(&IndexSuite{}) - -func (s *IndexSuite) TestLookupOffset(c *C) { - idx := NewIndex(0) - - for o1 := 0; o1 < 10000; o1 += 100 { - for o2 := 0; o2 < 10000; o2 += 100 { - if o2 >= o1 { - e, ok := idx.LookupOffset(uint64(o2)) - c.Assert(ok, Equals, false) - c.Assert(e, IsNil) - } else { - e, ok := idx.LookupOffset(uint64(o2)) - c.Assert(ok, Equals, true) - c.Assert(e, NotNil) - c.Assert(e.Hash, Equals, toHash(o2)) - c.Assert(e.Offset, Equals, uint64(o2)) - } - } - - h1 := toHash(o1) - idx.Add(h1, uint64(o1), 0) - - for o2 := 0; o2 < 10000; o2 += 100 { - if o2 > o1 { - e, ok := idx.LookupOffset(uint64(o2)) - c.Assert(ok, Equals, false) - c.Assert(e, IsNil) - } else { - e, ok := idx.LookupOffset(uint64(o2)) - c.Assert(ok, Equals, true) - c.Assert(e, NotNil) - c.Assert(e.Hash, Equals, toHash(o2)) - c.Assert(e.Offset, Equals, uint64(o2)) - } - } - } -} - -func (s *IndexSuite) TestLookupHash(c *C) { - idx := NewIndex(0) - - for o1 := 0; o1 < 10000; o1 += 100 { - for o2 := 0; o2 < 10000; o2 += 100 { - if o2 >= o1 { - e, ok := idx.LookupHash(toHash(o2)) - c.Assert(ok, Equals, false) - c.Assert(e, IsNil) - } else { - e, ok := idx.LookupHash(toHash(o2)) - c.Assert(ok, Equals, true) - c.Assert(e, NotNil) - c.Assert(e.Hash, Equals, toHash(o2)) - c.Assert(e.Offset, Equals, uint64(o2)) - } - } - - h1 := toHash(o1) - idx.Add(h1, uint64(o1), 0) - - for o2 := 0; o2 < 10000; o2 += 100 { - if o2 > o1 { - e, ok := idx.LookupHash(toHash(o2)) - c.Assert(ok, Equals, false) - c.Assert(e, IsNil) - } else { - e, ok := idx.LookupHash(toHash(o2)) - c.Assert(ok, Equals, true) - c.Assert(e, NotNil) - c.Assert(e.Hash, Equals, toHash(o2)) - c.Assert(e.Offset, Equals, uint64(o2)) - } - } - } -} - -func (s *IndexSuite) TestSize(c *C) { - idx := NewIndex(0) - - for o1 := 0; o1 < 1000; o1++ { - c.Assert(idx.Size(), Equals, o1) - h1 := toHash(o1) - idx.Add(h1, uint64(o1), 0) - } -} - -func (s *IndexSuite) TestIdxFileEmpty(c *C) { - idx := NewIndex(0) - idxf := idx.ToIdxFile() - idx2 := NewIndexFromIdxFile(idxf) - c.Assert(idx, DeepEquals, idx2) -} - -func (s *IndexSuite) TestIdxFile(c *C) { - idx := NewIndex(0) - for o1 := 0; o1 < 1000; o1++ { - h1 := toHash(o1) - idx.Add(h1, uint64(o1), 0) - } - - idx2 := NewIndexFromIdxFile(idx.ToIdxFile()) - c.Assert(idx, DeepEquals, idx2) -} - -func toHash(i int) plumbing.Hash { - is := strconv.Itoa(i) - padding := strings.Repeat("a", 40-len(is)) - return plumbing.NewHash(padding + is) -} - -func BenchmarkIndexConstruction(b *testing.B) { - b.ReportAllocs() - - idx := NewIndex(0) - for o := 0; o < 1e6*b.N; o += 100 { - h1 := toHash(o) - idx.Add(h1, uint64(o), 0) - } -} diff --git a/plumbing/format/packfile/packfile.go b/plumbing/format/packfile/packfile.go new file mode 100644 index 000000000..cee6031f1 --- /dev/null +++ b/plumbing/format/packfile/packfile.go @@ -0,0 +1,249 @@ +package packfile + +import ( + "bytes" + "io" + + billy "gopkg.in/src-d/go-billy.v4" + "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" + "gopkg.in/src-d/go-git.v4/plumbing/format/idxfile" + "gopkg.in/src-d/go-git.v4/plumbing/storer" +) + +// Packfile allows retrieving information from inside a packfile. +type Packfile struct { + idxfile.Index + billy.File + s *Scanner + deltaBaseCache cache.Object + offsetToHash map[int64]plumbing.Hash +} + +// NewPackfile returns a packfile representation for the given packfile file +// and packfile idx. +func NewPackfile(index idxfile.Index, file billy.File) *Packfile { + s := NewScanner(file) + + return &Packfile{ + index, + file, + s, + cache.NewObjectLRUDefault(), + make(map[int64]plumbing.Hash), + } +} + +// Get retrieves the encoded object in the packfile with the given hash. +func (p *Packfile) Get(h plumbing.Hash) (plumbing.EncodedObject, error) { + offset, err := p.FindOffset(h) + if err != nil { + return nil, err + } + + return p.GetByOffset(offset) +} + +// GetByOffset retrieves the encoded object from the packfile with the given +// offset. +func (p *Packfile) GetByOffset(o int64) (plumbing.EncodedObject, error) { + if h, ok := p.offsetToHash[o]; ok { + if obj, ok := p.deltaBaseCache.Get(h); ok { + return obj, nil + } + } + + if _, err := p.s.SeekFromStart(o); err != nil { + return nil, err + } + + return p.nextObject() +} + +func (p *Packfile) nextObject() (plumbing.EncodedObject, error) { + h, err := p.s.NextObjectHeader() + if err != nil { + return nil, err + } + + obj := new(plumbing.MemoryObject) + obj.SetSize(h.Length) + obj.SetType(h.Type) + + switch h.Type { + case plumbing.CommitObject, plumbing.TreeObject, plumbing.BlobObject, plumbing.TagObject: + err = p.fillRegularObjectContent(obj) + case plumbing.REFDeltaObject: + err = p.fillREFDeltaObjectContent(obj, h.Reference) + case plumbing.OFSDeltaObject: + err = p.fillOFSDeltaObjectContent(obj, h.OffsetReference) + default: + err = ErrInvalidObject.AddDetails("type %q", h.Type) + } + + if err != nil { + return obj, err + } + + p.offsetToHash[h.Offset] = obj.Hash() + + return obj, nil +} + +func (p *Packfile) fillRegularObjectContent(obj plumbing.EncodedObject) error { + w, err := obj.Writer() + if err != nil { + return err + } + + _, _, err = p.s.NextObject(w) + return err +} + +func (p *Packfile) fillREFDeltaObjectContent(obj plumbing.EncodedObject, ref plumbing.Hash) error { + buf := bufPool.Get().(*bytes.Buffer) + buf.Reset() + _, _, err := p.s.NextObject(buf) + if err != nil { + return err + } + + base, ok := p.cacheGet(ref) + if !ok { + base, err = p.Get(ref) + if err != nil { + return err + } + } + + obj.SetType(base.Type()) + err = ApplyDelta(obj, base, buf.Bytes()) + p.cachePut(obj) + bufPool.Put(buf) + + return err +} + +func (p *Packfile) fillOFSDeltaObjectContent(obj plumbing.EncodedObject, offset int64) error { + buf := bytes.NewBuffer(nil) + _, _, err := p.s.NextObject(buf) + if err != nil { + return err + } + + var base plumbing.EncodedObject + h, ok := p.offsetToHash[offset] + if ok { + base, ok = p.cacheGet(h) + } + + if !ok { + base, err = p.GetByOffset(offset) + if err != nil { + return err + } + + p.cachePut(base) + } + + obj.SetType(base.Type()) + err = ApplyDelta(obj, base, buf.Bytes()) + p.cachePut(obj) + + return err +} + +func (p *Packfile) cacheGet(h plumbing.Hash) (plumbing.EncodedObject, bool) { + if p.deltaBaseCache == nil { + return nil, false + } + + return p.deltaBaseCache.Get(h) +} + +func (p *Packfile) cachePut(obj plumbing.EncodedObject) { + if p.deltaBaseCache == nil { + return + } + + p.deltaBaseCache.Put(obj) +} + +// GetAll returns an iterator with all encoded objects in the packfile. +// The iterator returned is not thread-safe, it should be used in the same +// thread as the Packfile instance. +func (p *Packfile) GetAll() (storer.EncodedObjectIter, error) { + s := NewScanner(p.File) + + _, count, err := s.Header() + if err != nil { + return nil, err + } + + return &objectIter{ + // Easiest way to provide an object decoder is just to pass a Packfile + // instance. To not mess with the seeks, it's a new instance with a + // different scanner but the same cache and offset to hash map for + // reusing as much cache as possible. + d: &Packfile{p.Index, nil, s, p.deltaBaseCache, p.offsetToHash}, + count: int(count), + }, nil +} + +// ID returns the ID of the packfile, which is the checksum at the end of it. +func (p *Packfile) ID() (plumbing.Hash, error) { + if _, err := p.File.Seek(-20, io.SeekEnd); err != nil { + return plumbing.ZeroHash, err + } + + var hash plumbing.Hash + if _, err := io.ReadFull(p.File, hash[:]); err != nil { + return plumbing.ZeroHash, err + } + + return hash, nil +} + +// Close the packfile and its resources. +func (p *Packfile) Close() error { + return p.File.Close() +} + +type objectDecoder interface { + nextObject() (plumbing.EncodedObject, error) +} + +type objectIter struct { + d objectDecoder + count int + pos int +} + +func (i *objectIter) Next() (plumbing.EncodedObject, error) { + if i.pos >= i.count { + return nil, io.EOF + } + + i.pos++ + return i.d.nextObject() +} + +func (i *objectIter) ForEach(f func(plumbing.EncodedObject) error) error { + for { + o, err := i.Next() + if err != nil { + if err == io.EOF { + return nil + } + return err + } + + if err := f(o); err != nil { + return err + } + } +} + +func (i *objectIter) Close() { + i.pos = i.count +} diff --git a/plumbing/format/packfile/packfile_test.go b/plumbing/format/packfile/packfile_test.go new file mode 100644 index 000000000..10e408008 --- /dev/null +++ b/plumbing/format/packfile/packfile_test.go @@ -0,0 +1,121 @@ +package packfile + +import ( + "io" + "math" + + . "gopkg.in/check.v1" + "gopkg.in/src-d/go-billy.v4/osfs" + fixtures "gopkg.in/src-d/go-git-fixtures.v3" + "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/format/idxfile" +) + +type PackfileSuite struct { + fixtures.Suite + p *Packfile + idx *idxfile.MemoryIndex + f *fixtures.Fixture +} + +var _ = Suite(&PackfileSuite{}) + +func (s *PackfileSuite) TestGet(c *C) { + for h := range expectedEntries { + obj, err := s.p.Get(h) + c.Assert(err, IsNil) + c.Assert(obj, Not(IsNil)) + c.Assert(obj.Hash(), Equals, h) + } + + _, err := s.p.Get(plumbing.ZeroHash) + c.Assert(err, Equals, plumbing.ErrObjectNotFound) +} + +func (s *PackfileSuite) TestGetByOffset(c *C) { + for h, o := range expectedEntries { + obj, err := s.p.GetByOffset(o) + c.Assert(err, IsNil) + c.Assert(obj, Not(IsNil)) + c.Assert(obj.Hash(), Equals, h) + } + + _, err := s.p.GetByOffset(math.MaxInt64) + c.Assert(err, Equals, io.EOF) +} + +func (s *PackfileSuite) TestID(c *C) { + id, err := s.p.ID() + c.Assert(err, IsNil) + c.Assert(id, Equals, s.f.PackfileHash) +} + +func (s *PackfileSuite) TestGetAll(c *C) { + iter, err := s.p.GetAll() + c.Assert(err, IsNil) + + var objects int + for { + o, err := iter.Next() + if err == io.EOF { + break + } + c.Assert(err, IsNil) + + objects++ + _, ok := expectedEntries[o.Hash()] + c.Assert(ok, Equals, true) + } + + c.Assert(objects, Equals, len(expectedEntries)) +} + +var expectedEntries = map[plumbing.Hash]int64{ + plumbing.NewHash("1669dce138d9b841a518c64b10914d88f5e488ea"): 615, + plumbing.NewHash("32858aad3c383ed1ff0a0f9bdf231d54a00c9e88"): 1524, + plumbing.NewHash("35e85108805c84807bc66a02d91535e1e24b38b9"): 1063, + plumbing.NewHash("49c6bb89b17060d7b4deacb7b338fcc6ea2352a9"): 78882, + plumbing.NewHash("4d081c50e250fa32ea8b1313cf8bb7c2ad7627fd"): 84688, + plumbing.NewHash("586af567d0bb5e771e49bdd9434f5e0fb76d25fa"): 84559, + plumbing.NewHash("5a877e6a906a2743ad6e45d99c1793642aaf8eda"): 84479, + plumbing.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5"): 186, + plumbing.NewHash("7e59600739c96546163833214c36459e324bad0a"): 84653, + plumbing.NewHash("880cd14280f4b9b6ed3986d6671f907d7cc2a198"): 78050, + plumbing.NewHash("8dcef98b1d52143e1e2dbc458ffe38f925786bf2"): 84741, + plumbing.NewHash("918c48b83bd081e863dbe1b80f8998f058cd8294"): 286, + plumbing.NewHash("9a48f23120e880dfbe41f7c9b7b708e9ee62a492"): 80998, + plumbing.NewHash("9dea2395f5403188298c1dabe8bdafe562c491e3"): 84032, + plumbing.NewHash("a39771a7651f97faf5c72e08224d857fc35133db"): 84430, + plumbing.NewHash("a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69"): 838, + plumbing.NewHash("a8d315b2b1c615d43042c3a62402b8a54288cf5c"): 84375, + plumbing.NewHash("aa9b383c260e1d05fbbf6b30a02914555e20c725"): 84760, + plumbing.NewHash("af2d6a6954d532f8ffb47615169c8fdf9d383a1a"): 449, + plumbing.NewHash("b029517f6300c2da0f4b651b8642506cd6aaf45d"): 1392, + plumbing.NewHash("b8e471f58bcbca63b07bda20e428190409c2db47"): 1230, + plumbing.NewHash("c192bd6a24ea1ab01d78686e417c8bdc7c3d197f"): 1713, + plumbing.NewHash("c2d30fa8ef288618f65f6eed6e168e0d514886f4"): 84725, + plumbing.NewHash("c8f1d8c61f9da76f4cb49fd86322b6e685dba956"): 80725, + plumbing.NewHash("cf4aa3b38974fb7d81f367c0830f7d78d65ab86b"): 84608, + plumbing.NewHash("d3ff53e0564a9f87d8e84b6e28e5060e517008aa"): 1685, + plumbing.NewHash("d5c0f4ab811897cadf03aec358ae60d21f91c50d"): 2351, + plumbing.NewHash("dbd3641b371024f44d0e469a9c8f5457b0660de1"): 84115, + plumbing.NewHash("e8d3ffab552895c19b9fcf7aa264d277cde33881"): 12, + plumbing.NewHash("eba74343e2f15d62adedfd8c883ee0262b5c8021"): 84708, + plumbing.NewHash("fb72698cab7617ac416264415f13224dfd7a165e"): 84671, +} + +func (s *PackfileSuite) SetUpTest(c *C) { + s.f = fixtures.Basic().One() + + f, err := osfs.New("/").Open(s.f.Packfile().Name()) + c.Assert(err, IsNil) + + s.idx = idxfile.NewMemoryIndex() + c.Assert(idxfile.NewDecoder(s.f.Idx()).Decode(s.idx), IsNil) + + s.p = NewPackfile(s.idx, f) +} + +func (s *PackfileSuite) TearDownTest(c *C) { + c.Assert(s.p.Close(), IsNil) +} diff --git a/plumbing/memory.go b/plumbing/memory.go index 51cbb54d8..b8e1e1b81 100644 --- a/plumbing/memory.go +++ b/plumbing/memory.go @@ -14,10 +14,10 @@ type MemoryObject struct { sz int64 } -// Hash return the object Hash, the hash is calculated on-the-fly the first -// time is called, the subsequent calls the same Hash is returned even if the -// type or the content has changed. The Hash is only generated if the size of -// the content is exactly the Object.Size +// Hash returns the object Hash, the hash is calculated on-the-fly the first +// time it's called, in all subsequent calls the same Hash is returned even +// if the type or the content have changed. The Hash is only generated if the +// size of the content is exactly the object size. func (o *MemoryObject) Hash() Hash { if o.h == ZeroHash && int64(len(o.cont)) == o.sz { o.h = ComputeHash(o.t, o.cont) diff --git a/storage/filesystem/storage.go b/storage/filesystem/storage.go index 622bb4a8d..6af906d26 100644 --- a/storage/filesystem/storage.go +++ b/storage/filesystem/storage.go @@ -2,6 +2,9 @@ package filesystem import ( + "fmt" + + "gopkg.in/src-d/go-git.v4/plumbing/format/index" "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit" "gopkg.in/src-d/go-billy.v4" @@ -51,3 +54,15 @@ func (s *Storage) Filesystem() billy.Filesystem { func (s *Storage) Init() error { return s.dir.Initialize() } + +type IndexStorage struct { + dir *dotgit.DotGit +} + +func (IndexStorage) SetIndex(*index.Index) error { + return fmt.Errorf("not implemented") +} + +func (IndexStorage) Index() (*index.Index, error) { + return nil, fmt.Errorf("not implemented") +} From bc565c1ba0516677d9227e19de544a9126db0a55 Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Thu, 26 Jul 2018 19:27:35 +0200 Subject: [PATCH 038/120] plumbing, packfile: delete index_test as is no longer used Signed-off-by: Javi Fontan --- plumbing/format/packfile/index_test.go | 133 ------------------------- 1 file changed, 133 deletions(-) delete mode 100644 plumbing/format/packfile/index_test.go diff --git a/plumbing/format/packfile/index_test.go b/plumbing/format/packfile/index_test.go deleted file mode 100644 index 8de886dac..000000000 --- a/plumbing/format/packfile/index_test.go +++ /dev/null @@ -1,133 +0,0 @@ -package packfile - -import ( - "strconv" - "strings" - "testing" - - "gopkg.in/src-d/go-git.v4/plumbing" - - . "gopkg.in/check.v1" -) - -type IndexSuite struct{} - -var _ = Suite(&IndexSuite{}) - -func (s *IndexSuite) TestLookupOffset(c *C) { - idx := NewIndex(0) - - for o1 := 0; o1 < 10000; o1 += 100 { - for o2 := 0; o2 < 10000; o2 += 100 { - if o2 >= o1 { - e, ok := idx.LookupOffset(uint64(o2)) - c.Assert(ok, Equals, false) - c.Assert(e, IsNil) - } else { - e, ok := idx.LookupOffset(uint64(o2)) - c.Assert(ok, Equals, true) - c.Assert(e, NotNil) - c.Assert(e.Hash, Equals, toHash(o2)) - c.Assert(e.Offset, Equals, uint64(o2)) - } - } - - h1 := toHash(o1) - idx.Add(h1, uint64(o1), 0) - - for o2 := 0; o2 < 10000; o2 += 100 { - if o2 > o1 { - e, ok := idx.LookupOffset(uint64(o2)) - c.Assert(ok, Equals, false) - c.Assert(e, IsNil) - } else { - e, ok := idx.LookupOffset(uint64(o2)) - c.Assert(ok, Equals, true) - c.Assert(e, NotNil) - c.Assert(e.Hash, Equals, toHash(o2)) - c.Assert(e.Offset, Equals, uint64(o2)) - } - } - } -} - -func (s *IndexSuite) TestLookupHash(c *C) { - idx := NewIndex(0) - - for o1 := 0; o1 < 10000; o1 += 100 { - for o2 := 0; o2 < 10000; o2 += 100 { - if o2 >= o1 { - e, ok := idx.LookupHash(toHash(o2)) - c.Assert(ok, Equals, false) - c.Assert(e, IsNil) - } else { - e, ok := idx.LookupHash(toHash(o2)) - c.Assert(ok, Equals, true) - c.Assert(e, NotNil) - c.Assert(e.Hash, Equals, toHash(o2)) - c.Assert(e.Offset, Equals, uint64(o2)) - } - } - - h1 := toHash(o1) - idx.Add(h1, uint64(o1), 0) - - for o2 := 0; o2 < 10000; o2 += 100 { - if o2 > o1 { - e, ok := idx.LookupHash(toHash(o2)) - c.Assert(ok, Equals, false) - c.Assert(e, IsNil) - } else { - e, ok := idx.LookupHash(toHash(o2)) - c.Assert(ok, Equals, true) - c.Assert(e, NotNil) - c.Assert(e.Hash, Equals, toHash(o2)) - c.Assert(e.Offset, Equals, uint64(o2)) - } - } - } -} - -func (s *IndexSuite) TestSize(c *C) { - idx := NewIndex(0) - - for o1 := 0; o1 < 1000; o1++ { - c.Assert(idx.Size(), Equals, o1) - h1 := toHash(o1) - idx.Add(h1, uint64(o1), 0) - } -} - -func (s *IndexSuite) TestIdxFileEmpty(c *C) { - idx := NewIndex(0) - idxf := idx.ToIdxFile() - idx2 := NewIndexFromIdxFile(idxf) - c.Assert(idx, DeepEquals, idx2) -} - -func (s *IndexSuite) TestIdxFile(c *C) { - idx := NewIndex(0) - for o1 := 0; o1 < 1000; o1++ { - h1 := toHash(o1) - idx.Add(h1, uint64(o1), 0) - } - - idx2 := NewIndexFromIdxFile(idx.ToIdxFile()) - c.Assert(idx, DeepEquals, idx2) -} - -func toHash(i int) plumbing.Hash { - is := strconv.Itoa(i) - padding := strings.Repeat("a", 40-len(is)) - return plumbing.NewHash(padding + is) -} - -func BenchmarkIndexConstruction(b *testing.B) { - b.ReportAllocs() - - idx := NewIndex(0) - for o := 0; o < 1e6*b.N; o += 100 { - h1 := toHash(o) - idx.Add(h1, uint64(o), 0) - } -} From 4b366ac48de72f63905c6e92e387677e83e97d5c Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Thu, 26 Jul 2018 19:28:34 +0200 Subject: [PATCH 039/120] plumbing: fix two errors in idxfile and packfile decoder Signed-off-by: Javi Fontan --- plumbing/format/idxfile/idxfile.go | 2 +- plumbing/format/packfile/decoder.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plumbing/format/idxfile/idxfile.go b/plumbing/format/idxfile/idxfile.go index f8debb1a9..f57df2e6e 100644 --- a/plumbing/format/idxfile/idxfile.go +++ b/plumbing/format/idxfile/idxfile.go @@ -87,7 +87,7 @@ func (idx *MemoryIndex) findHashIndex(h plumbing.Hash) int { low = mid + 1 } - if low > high { + if low >= high { break } } diff --git a/plumbing/format/packfile/decoder.go b/plumbing/format/packfile/decoder.go index 69aef2d7d..87c347fd5 100644 --- a/plumbing/format/packfile/decoder.go +++ b/plumbing/format/packfile/decoder.go @@ -457,7 +457,7 @@ func (d *Decoder) recallByOffset(o int64) (plumbing.EncodedObject, error) { func (d *Decoder) recallByHash(h plumbing.Hash) (plumbing.EncodedObject, error) { if d.s.IsSeekable { - if offset, err := d.idx.FindOffset(h); err != nil { + if offset, err := d.idx.FindOffset(h); err == nil { return d.DecodeObjectAt(offset) } } From 3657a32e0ead55601a2af578abecd65dd2d8b64b Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Fri, 27 Jul 2018 12:24:09 +0200 Subject: [PATCH 040/120] storage/filesystem: add back IndexStorage Signed-off-by: Javi Fontan --- storage/filesystem/index.go | 47 +++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 storage/filesystem/index.go diff --git a/storage/filesystem/index.go b/storage/filesystem/index.go new file mode 100644 index 000000000..2ebf57e61 --- /dev/null +++ b/storage/filesystem/index.go @@ -0,0 +1,47 @@ +package filesystem + +import ( + "os" + + "gopkg.in/src-d/go-git.v4/plumbing/format/index" + "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit" + "gopkg.in/src-d/go-git.v4/utils/ioutil" +) + +type IndexStorage struct { + dir *dotgit.DotGit +} + +func (s *IndexStorage) SetIndex(idx *index.Index) (err error) { + f, err := s.dir.IndexWriter() + if err != nil { + return err + } + + defer ioutil.CheckClose(f, &err) + + e := index.NewEncoder(f) + err = e.Encode(idx) + return err +} + +func (s *IndexStorage) Index() (i *index.Index, err error) { + idx := &index.Index{ + Version: 2, + } + + f, err := s.dir.Index() + if err != nil { + if os.IsNotExist(err) { + return idx, nil + } + + return nil, err + } + + defer ioutil.CheckClose(f, &err) + + d := index.NewDecoder(f) + err = d.Decode(idx) + return idx, err +} From ccd0fa0bc17f0680038529b00f5c5a44f8e77b41 Mon Sep 17 00:00:00 2001 From: Miguel Molina Date: Fri, 27 Jul 2018 15:07:25 +0200 Subject: [PATCH 041/120] plumbing: packfile, lazy object reads with DiskObjects Signed-off-by: Miguel Molina --- plumbing/format/idxfile/idxfile.go | 25 ++- plumbing/format/packfile/decoder.go | 2 +- plumbing/format/packfile/disk_object.go | 64 +++++++ plumbing/format/packfile/packfile.go | 208 +++++++++++++++++++--- plumbing/format/packfile/packfile_test.go | 46 +++++ storage/memory/storage.go | 10 ++ 6 files changed, 314 insertions(+), 41 deletions(-) create mode 100644 plumbing/format/packfile/disk_object.go diff --git a/plumbing/format/idxfile/idxfile.go b/plumbing/format/idxfile/idxfile.go index f8debb1a9..d4a936535 100644 --- a/plumbing/format/idxfile/idxfile.go +++ b/plumbing/format/idxfile/idxfile.go @@ -87,7 +87,7 @@ func (idx *MemoryIndex) findHashIndex(h plumbing.Hash) int { low = mid + 1 } - if low > high { + if low >= high { break } } @@ -157,9 +157,8 @@ func (idx *MemoryIndex) getCrc32(firstLevel, secondLevel int) (uint32, error) { func (idx *MemoryIndex) FindHash(o int64) (plumbing.Hash, error) { // Lazily generate the reverse offset/hash map if required. if idx.offsetHash == nil { - err := idx.genOffsetHash() - if err != nil { - return plumbing.ZeroHash, nil + if err := idx.genOffsetHash(); err != nil { + return plumbing.ZeroHash, err } } @@ -185,19 +184,17 @@ func (idx *MemoryIndex) genOffsetHash() error { return err } - var entry *Entry - for err != nil { - entry, err = iter.Next() - if err == nil { - idx.offsetHash[int64(entry.Offset)] = entry.Hash + for { + entry, err := iter.Next() + if err != nil { + if err == io.EOF { + return nil + } + return err } - } - if err == io.EOF { - return nil + idx.offsetHash[int64(entry.Offset)] = entry.Hash } - - return err } // Count implements the Index interface. diff --git a/plumbing/format/packfile/decoder.go b/plumbing/format/packfile/decoder.go index b1a0a2695..edf386b13 100644 --- a/plumbing/format/packfile/decoder.go +++ b/plumbing/format/packfile/decoder.go @@ -478,7 +478,7 @@ func (d *Decoder) recallByOffset(o int64) (plumbing.EncodedObject, error) { func (d *Decoder) recallByHash(h plumbing.Hash) (plumbing.EncodedObject, error) { if d.s.IsSeekable { - if offset, err := d.idx.FindOffset(h); err != nil { + if offset, err := d.idx.FindOffset(h); err == nil { return d.DecodeObjectAt(offset) } } diff --git a/plumbing/format/packfile/disk_object.go b/plumbing/format/packfile/disk_object.go new file mode 100644 index 000000000..d3e852024 --- /dev/null +++ b/plumbing/format/packfile/disk_object.go @@ -0,0 +1,64 @@ +package packfile + +import ( + "io" + + "gopkg.in/src-d/go-git.v4/plumbing" +) + +// DiskObject is an object from the packfile on disk. +type DiskObject struct { + hash plumbing.Hash + h *ObjectHeader + offset int64 + size int64 + typ plumbing.ObjectType + packfile *Packfile +} + +// NewDiskObject creates a new disk object. +func NewDiskObject( + hash plumbing.Hash, + finalType plumbing.ObjectType, + offset int64, + contentSize int64, + packfile *Packfile, +) *DiskObject { + return &DiskObject{ + hash: hash, + offset: offset, + size: contentSize, + typ: finalType, + packfile: packfile, + } +} + +// Reader implements the plumbing.EncodedObject interface. +func (o *DiskObject) Reader() (io.ReadCloser, error) { + return o.packfile.getObjectContent(o.offset) +} + +// SetSize implements the plumbing.EncodedObject interface. This method +// is a noop. +func (o *DiskObject) SetSize(int64) {} + +// SetType implements the plumbing.EncodedObject interface. This method is +// a noop. +func (o *DiskObject) SetType(plumbing.ObjectType) {} + +// Hash implements the plumbing.EncodedObject interface. +func (o *DiskObject) Hash() plumbing.Hash { return o.hash } + +// Size implements the plumbing.EncodedObject interface. +func (o *DiskObject) Size() int64 { return o.size } + +// Type implements the plumbing.EncodedObject interface. +func (o *DiskObject) Type() plumbing.ObjectType { + return o.typ +} + +// Writer implements the plumbing.EncodedObject interface. This method always +// returns a nil writer. +func (o *DiskObject) Writer() (io.WriteCloser, error) { + return nil, nil +} diff --git a/plumbing/format/packfile/packfile.go b/plumbing/format/packfile/packfile.go index cee6031f1..00014f6a5 100644 --- a/plumbing/format/packfile/packfile.go +++ b/plumbing/format/packfile/packfile.go @@ -17,7 +17,7 @@ type Packfile struct { billy.File s *Scanner deltaBaseCache cache.Object - offsetToHash map[int64]plumbing.Hash + offsetToType map[int64]plumbing.ObjectType } // NewPackfile returns a packfile representation for the given packfile file @@ -30,7 +30,7 @@ func NewPackfile(index idxfile.Index, file billy.File) *Packfile { file, s, cache.NewObjectLRUDefault(), - make(map[int64]plumbing.Hash), + make(map[int64]plumbing.ObjectType), } } @@ -47,8 +47,9 @@ func (p *Packfile) Get(h plumbing.Hash) (plumbing.EncodedObject, error) { // GetByOffset retrieves the encoded object from the packfile with the given // offset. func (p *Packfile) GetByOffset(o int64) (plumbing.EncodedObject, error) { - if h, ok := p.offsetToHash[o]; ok { - if obj, ok := p.deltaBaseCache.Get(h); ok { + hash, err := p.FindHash(o) + if err == nil { + if obj, ok := p.deltaBaseCache.Get(hash); ok { return obj, nil } } @@ -60,13 +61,166 @@ func (p *Packfile) GetByOffset(o int64) (plumbing.EncodedObject, error) { return p.nextObject() } -func (p *Packfile) nextObject() (plumbing.EncodedObject, error) { +func (p *Packfile) nextObjectHeader() (*ObjectHeader, error) { h, err := p.s.NextObjectHeader() + p.s.pendingObject = nil + return h, err +} + +func (p *Packfile) getObjectData( + h *ObjectHeader, +) (typ plumbing.ObjectType, size int64, err error) { + switch h.Type { + case plumbing.CommitObject, plumbing.TreeObject, plumbing.BlobObject, plumbing.TagObject: + typ = h.Type + size = h.Length + case plumbing.REFDeltaObject, plumbing.OFSDeltaObject: + buf := bufPool.Get().(*bytes.Buffer) + buf.Reset() + defer bufPool.Put(buf) + + _, _, err = p.s.NextObject(buf) + if err != nil { + return + } + + delta := buf.Bytes() + _, delta = decodeLEB128(delta) // skip src size + sz, _ := decodeLEB128(delta) + size = int64(sz) + + var offset int64 + if h.Type == plumbing.REFDeltaObject { + offset, err = p.FindOffset(h.Reference) + if err != nil { + return + } + } else { + offset = h.OffsetReference + } + + if baseType, ok := p.offsetToType[offset]; ok { + typ = baseType + } else { + if _, err = p.s.SeekFromStart(offset); err != nil { + return + } + + h, err = p.nextObjectHeader() + if err != nil { + return + } + + typ, _, err = p.getObjectData(h) + if err != nil { + return + } + } + default: + err = ErrInvalidObject.AddDetails("type %q", h.Type) + } + + return +} + +func (p *Packfile) getObjectSize(h *ObjectHeader) (int64, error) { + switch h.Type { + case plumbing.CommitObject, plumbing.TreeObject, plumbing.BlobObject, plumbing.TagObject: + return h.Length, nil + case plumbing.REFDeltaObject, plumbing.OFSDeltaObject: + buf := bufPool.Get().(*bytes.Buffer) + buf.Reset() + defer bufPool.Put(buf) + + if _, _, err := p.s.NextObject(buf); err != nil { + return 0, err + } + + delta := buf.Bytes() + _, delta = decodeLEB128(delta) // skip src size + sz, _ := decodeLEB128(delta) + return int64(sz), nil + default: + return 0, ErrInvalidObject.AddDetails("type %q", h.Type) + } +} + +func (p *Packfile) getObjectType(h *ObjectHeader) (typ plumbing.ObjectType, err error) { + switch h.Type { + case plumbing.CommitObject, plumbing.TreeObject, plumbing.BlobObject, plumbing.TagObject: + return h.Type, nil + case plumbing.REFDeltaObject, plumbing.OFSDeltaObject: + var offset int64 + if h.Type == plumbing.REFDeltaObject { + offset, err = p.FindOffset(h.Reference) + if err != nil { + return + } + } else { + offset = h.OffsetReference + } + + if baseType, ok := p.offsetToType[offset]; ok { + typ = baseType + } else { + if _, err = p.s.SeekFromStart(offset); err != nil { + return + } + + h, err = p.nextObjectHeader() + if err != nil { + return + } + + typ, err = p.getObjectType(h) + if err != nil { + return + } + } + default: + err = ErrInvalidObject.AddDetails("type %q", h.Type) + } + + return +} + +func (p *Packfile) nextObject() (plumbing.EncodedObject, error) { + h, err := p.nextObjectHeader() + if err != nil { + return nil, err + } + + hash, err := p.FindHash(h.Offset) + if err != nil { + return nil, err + } + + size, err := p.getObjectSize(h) if err != nil { return nil, err } - obj := new(plumbing.MemoryObject) + typ, err := p.getObjectType(h) + if err != nil { + return nil, err + } + + p.offsetToType[h.Offset] = typ + + return NewDiskObject(hash, typ, h.Offset, size, p), nil +} + +func (p *Packfile) getObjectContent(offset int64) (io.ReadCloser, error) { + if _, err := p.s.SeekFromStart(offset); err != nil { + return nil, err + } + + h, err := p.nextObjectHeader() + if err != nil { + return nil, err + } + + var obj = new(plumbing.MemoryObject) obj.SetSize(h.Length) obj.SetType(h.Type) @@ -82,12 +236,10 @@ func (p *Packfile) nextObject() (plumbing.EncodedObject, error) { } if err != nil { - return obj, err + return nil, err } - p.offsetToHash[h.Offset] = obj.Hash() - - return obj, nil + return obj.Reader() } func (p *Packfile) fillRegularObjectContent(obj plumbing.EncodedObject) error { @@ -132,9 +284,10 @@ func (p *Packfile) fillOFSDeltaObjectContent(obj plumbing.EncodedObject, offset } var base plumbing.EncodedObject - h, ok := p.offsetToHash[offset] - if ok { - base, ok = p.cacheGet(h) + var ok bool + hash, err := p.FindHash(offset) + if err == nil { + base, ok = p.cacheGet(hash) } if !ok { @@ -173,9 +326,7 @@ func (p *Packfile) cachePut(obj plumbing.EncodedObject) { // The iterator returned is not thread-safe, it should be used in the same // thread as the Packfile instance. func (p *Packfile) GetAll() (storer.EncodedObjectIter, error) { - s := NewScanner(p.File) - - _, count, err := s.Header() + entries, err := p.Entries() if err != nil { return nil, err } @@ -185,8 +336,14 @@ func (p *Packfile) GetAll() (storer.EncodedObjectIter, error) { // instance. To not mess with the seeks, it's a new instance with a // different scanner but the same cache and offset to hash map for // reusing as much cache as possible. - d: &Packfile{p.Index, nil, s, p.deltaBaseCache, p.offsetToHash}, - count: int(count), + p: &Packfile{ + p.Index, + p.File, + NewScanner(p.File), + p.deltaBaseCache, + p.offsetToType, + }, + iter: entries, }, nil } @@ -214,18 +371,17 @@ type objectDecoder interface { } type objectIter struct { - d objectDecoder - count int - pos int + p *Packfile + iter idxfile.EntryIter } func (i *objectIter) Next() (plumbing.EncodedObject, error) { - if i.pos >= i.count { - return nil, io.EOF + e, err := i.iter.Next() + if err != nil { + return nil, err } - i.pos++ - return i.d.nextObject() + return i.p.GetByOffset(int64(e.Offset)) } func (i *objectIter) ForEach(f func(plumbing.EncodedObject) error) error { @@ -245,5 +401,5 @@ func (i *objectIter) ForEach(f func(plumbing.EncodedObject) error) error { } func (i *objectIter) Close() { - i.pos = i.count + i.iter.Close() } diff --git a/plumbing/format/packfile/packfile_test.go b/plumbing/format/packfile/packfile_test.go index 10e408008..0d7a80653 100644 --- a/plumbing/format/packfile/packfile_test.go +++ b/plumbing/format/packfile/packfile_test.go @@ -1,14 +1,18 @@ package packfile import ( + "bytes" "io" "math" + "io/ioutil" + . "gopkg.in/check.v1" "gopkg.in/src-d/go-billy.v4/osfs" fixtures "gopkg.in/src-d/go-git-fixtures.v3" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/format/idxfile" + "gopkg.in/src-d/go-git.v4/storage/memory" ) type PackfileSuite struct { @@ -104,6 +108,48 @@ var expectedEntries = map[plumbing.Hash]int64{ plumbing.NewHash("fb72698cab7617ac416264415f13224dfd7a165e"): 84671, } +func (s *PackfileSuite) TestContent(c *C) { + storer := memory.NewObjectStorage() + decoder, err := NewDecoder(NewScanner(s.f.Packfile()), storer) + c.Assert(err, IsNil) + + _, err = decoder.Decode() + c.Assert(err, IsNil) + + iter, err := s.p.GetAll() + c.Assert(err, IsNil) + + for { + o, err := iter.Next() + if err == io.EOF { + break + } + c.Assert(err, IsNil) + + o2, err := storer.EncodedObject(plumbing.AnyObject, o.Hash()) + c.Assert(err, IsNil) + + c.Assert(o.Type(), Equals, o2.Type()) + c.Assert(o.Size(), Equals, o2.Size()) + + r, err := o.Reader() + c.Assert(err, IsNil) + + c1, err := ioutil.ReadAll(r) + c.Assert(err, IsNil) + c.Assert(r.Close(), IsNil) + + r, err = o2.Reader() + c.Assert(err, IsNil) + + c2, err := ioutil.ReadAll(r) + c.Assert(err, IsNil) + c.Assert(r.Close(), IsNil) + + c.Assert(bytes.Compare(c1, c2), Equals, 0) + } +} + func (s *PackfileSuite) SetUpTest(c *C) { s.f = fixtures.Basic().One() diff --git a/storage/memory/storage.go b/storage/memory/storage.go index 2e3250905..a950a6224 100644 --- a/storage/memory/storage.go +++ b/storage/memory/storage.go @@ -91,6 +91,16 @@ type ObjectStorage struct { Tags map[plumbing.Hash]plumbing.EncodedObject } +func NewObjectStorage() *ObjectStorage { + return &ObjectStorage{ + Objects: make(map[plumbing.Hash]plumbing.EncodedObject), + Commits: make(map[plumbing.Hash]plumbing.EncodedObject), + Trees: make(map[plumbing.Hash]plumbing.EncodedObject), + Blobs: make(map[plumbing.Hash]plumbing.EncodedObject), + Tags: make(map[plumbing.Hash]plumbing.EncodedObject), + } +} + func (o *ObjectStorage) NewEncodedObject() plumbing.EncodedObject { return &plumbing.MemoryObject{} } From 823abfeb3d677a74e5bb50b20cbe8cc0306e9075 Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Fri, 27 Jul 2018 18:08:55 +0200 Subject: [PATCH 042/120] plumbing/idxfile: test FindHash and writer with 64 bit offsets Signed-off-by: Javi Fontan --- plumbing/format/idxfile/idxfile_test.go | 59 ++++++++++++++++++++++--- plumbing/format/idxfile/writer_test.go | 58 ++++++++++++++++++++++-- 2 files changed, 107 insertions(+), 10 deletions(-) diff --git a/plumbing/format/idxfile/idxfile_test.go b/plumbing/format/idxfile/idxfile_test.go index f42a41998..d15accf3a 100644 --- a/plumbing/format/idxfile/idxfile_test.go +++ b/plumbing/format/idxfile/idxfile_test.go @@ -3,15 +3,22 @@ package idxfile_test import ( "bytes" "encoding/base64" + "fmt" "io" "testing" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/format/idxfile" + + . "gopkg.in/check.v1" + "gopkg.in/src-d/go-git-fixtures.v3" ) func BenchmarkFindOffset(b *testing.B) { - idx := fixtureIndex(b) + idx, err := fixtureIndex() + if err != nil { + b.Fatalf(err.Error()) + } for i := 0; i < b.N; i++ { for _, h := range fixtureHashes { @@ -24,7 +31,10 @@ func BenchmarkFindOffset(b *testing.B) { } func BenchmarkFindCRC32(b *testing.B) { - idx := fixtureIndex(b) + idx, err := fixtureIndex() + if err != nil { + b.Fatalf(err.Error()) + } for i := 0; i < b.N; i++ { for _, h := range fixtureHashes { @@ -37,7 +47,10 @@ func BenchmarkFindCRC32(b *testing.B) { } func BenchmarkContains(b *testing.B) { - idx := fixtureIndex(b) + idx, err := fixtureIndex() + if err != nil { + b.Fatalf(err.Error()) + } for i := 0; i < b.N; i++ { for _, h := range fixtureHashes { @@ -54,7 +67,10 @@ func BenchmarkContains(b *testing.B) { } func BenchmarkEntries(b *testing.B) { - idx := fixtureIndex(b) + idx, err := fixtureIndex() + if err != nil { + b.Fatalf(err.Error()) + } for i := 0; i < b.N; i++ { iter, err := idx.Entries() @@ -82,6 +98,23 @@ func BenchmarkEntries(b *testing.B) { } } +type IndexSuite struct { + fixtures.Suite +} + +var _ = Suite(&IndexSuite{}) + +func (s *IndexSuite) TestFindHash(c *C) { + idx, err := fixtureIndex() + c.Assert(err, IsNil) + + for i, pos := range fixtureOffsets { + hash, err := idx.FindHash(pos) + c.Assert(err, IsNil) + c.Assert(hash, Equals, fixtureHashes[i]) + } +} + var fixtureHashes = []plumbing.Hash{ plumbing.NewHash("303953e5aa461c203a324821bc1717f9b4fff895"), plumbing.NewHash("5296768e3d9f661387ccbff18c4dea6c997fd78c"), @@ -94,7 +127,19 @@ var fixtureHashes = []plumbing.Hash{ plumbing.NewHash("35858be9c6f5914cbe6768489c41eb6809a2bceb"), } -func fixtureIndex(t testing.TB) *idxfile.MemoryIndex { +var fixtureOffsets = []int64{ + 12, + 142, + 1601322837, + 2646996529, + 3452385606, + 3707047470, + 5323223332, + 5894072943, + 5924278919, +} + +func fixtureIndex() (*idxfile.MemoryIndex, error) { f := bytes.NewBufferString(fixtureLarge4GB) idx := new(idxfile.MemoryIndex) @@ -102,8 +147,8 @@ func fixtureIndex(t testing.TB) *idxfile.MemoryIndex { d := idxfile.NewDecoder(base64.NewDecoder(base64.StdEncoding, f)) err := d.Decode(idx) if err != nil { - t.Fatalf("unexpected error decoding index: %s", err) + return nil, fmt.Errorf("unexpected error decoding index: %s", err) } - return idx + return idx, nil } diff --git a/plumbing/format/idxfile/writer_test.go b/plumbing/format/idxfile/writer_test.go index 51273a365..780acd978 100644 --- a/plumbing/format/idxfile/writer_test.go +++ b/plumbing/format/idxfile/writer_test.go @@ -2,8 +2,10 @@ package idxfile_test import ( "bytes" + "encoding/base64" "io/ioutil" + "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/format/idxfile" "gopkg.in/src-d/go-git.v4/plumbing/format/packfile" @@ -11,13 +13,13 @@ import ( "gopkg.in/src-d/go-git-fixtures.v3" ) -type IndexSuite struct { +type WriterSuite struct { fixtures.Suite } -var _ = Suite(&IndexSuite{}) +var _ = Suite(&WriterSuite{}) -func (s *IndexSuite) TestIndexWriter(c *C) { +func (s *WriterSuite) TestWriter(c *C) { f := fixtures.Basic().One() scanner := packfile.NewScanner(f.Packfile()) @@ -43,3 +45,53 @@ func (s *IndexSuite) TestIndexWriter(c *C) { c.Assert(buf.Bytes(), DeepEquals, expected) } + +func (s *WriterSuite) TestWriterLarge(c *C) { + writer := new(idxfile.Writer) + err := writer.OnHeader(uint32(len(fixture4GbEntries))) + c.Assert(err, IsNil) + + for _, o := range fixture4GbEntries { + err = writer.OnInflatedObjectContent(plumbing.NewHash(o.hash), o.offset, o.crc) + c.Assert(err, IsNil) + } + + err = writer.OnFooter(fixture4GbChecksum) + c.Assert(err, IsNil) + + idx, err := writer.Index() + c.Assert(err, IsNil) + + // load fixture index + f := bytes.NewBufferString(fixtureLarge4GB) + expected, err := ioutil.ReadAll(base64.NewDecoder(base64.StdEncoding, f)) + c.Assert(err, IsNil) + + buf := new(bytes.Buffer) + encoder := idxfile.NewEncoder(buf) + n, err := encoder.Encode(idx) + c.Assert(err, IsNil) + c.Assert(n, Equals, len(expected)) + + c.Assert(buf.Bytes(), DeepEquals, expected) +} + +var ( + fixture4GbChecksum = plumbing.NewHash("afabc2269205cf85da1bf7e2fdff42f73810f29b") + + fixture4GbEntries = []struct { + offset int64 + hash string + crc uint32 + }{ + {12, "303953e5aa461c203a324821bc1717f9b4fff895", 0xbc347c4c}, + {142, "5296768e3d9f661387ccbff18c4dea6c997fd78c", 0xcdc22842}, + {1601322837, "03fc8d58d44267274edef4585eaeeb445879d33f", 0x929dfaaa}, + {2646996529, "8f3ceb4ea4cb9e4a0f751795eb41c9a4f07be772", 0xa61def8a}, + {3452385606, "e0d1d625010087f79c9e01ad9d8f95e1628dda02", 0x06bea180}, + {3707047470, "90eba326cdc4d1d61c5ad25224ccbf08731dd041", 0x7193f3ba}, + {5323223332, "bab53055add7bc35882758a922c54a874d6b1272", 0xac269b8e}, + {5894072943, "1b8995f51987d8a449ca5ea4356595102dc2fbd4", 0x2187c056}, + {5924278919, "35858be9c6f5914cbe6768489c41eb6809a2bceb", 0x9c89d9d2}, + } +) From 6f8f2ed229cc88a175d6ea47a53135b6dcef6912 Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Fri, 27 Jul 2018 18:17:43 +0200 Subject: [PATCH 043/120] storage/filesystem: remove duplicated IndexStorage Signed-off-by: Javi Fontan --- storage/filesystem/storage.go | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/storage/filesystem/storage.go b/storage/filesystem/storage.go index 6af906d26..622bb4a8d 100644 --- a/storage/filesystem/storage.go +++ b/storage/filesystem/storage.go @@ -2,9 +2,6 @@ package filesystem import ( - "fmt" - - "gopkg.in/src-d/go-git.v4/plumbing/format/index" "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit" "gopkg.in/src-d/go-billy.v4" @@ -54,15 +51,3 @@ func (s *Storage) Filesystem() billy.Filesystem { func (s *Storage) Init() error { return s.dir.Initialize() } - -type IndexStorage struct { - dir *dotgit.DotGit -} - -func (IndexStorage) SetIndex(*index.Index) error { - return fmt.Errorf("not implemented") -} - -func (IndexStorage) Index() (*index.Index, error) { - return nil, fmt.Errorf("not implemented") -} From b4cd0899e24e0e8c7910bcdc33c96dc463dcb1e4 Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Fri, 27 Jul 2018 18:31:40 +0200 Subject: [PATCH 044/120] plumbing/packfile: add index generation to decoder Signed-off-by: Javi Fontan --- plumbing/format/packfile/decoder.go | 32 ++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/plumbing/format/packfile/decoder.go b/plumbing/format/packfile/decoder.go index edf386b13..d6bc0efb8 100644 --- a/plumbing/format/packfile/decoder.go +++ b/plumbing/format/packfile/decoder.go @@ -66,6 +66,7 @@ type Decoder struct { // will be built incrementally while decoding. hasBuiltIndex bool idx idxfile.Index + writer *idxfile.Writer offsetToType map[int64]plumbing.ObjectType decoderType plumbing.ObjectType @@ -144,7 +145,17 @@ func (d *Decoder) Decode() (checksum plumbing.Hash, err error) { return plumbing.ZeroHash, err } - return d.s.Checksum() + checksum, err = d.s.Checksum() + if err != nil { + return plumbing.ZeroHash, err + } + + if !d.hasBuiltIndex { + d.writer.OnFooter(checksum) + d.idx = d.Index() + } + + return checksum, err } func (d *Decoder) fillOffsetsToHashes() error { @@ -177,6 +188,8 @@ func (d *Decoder) doDecode() error { if !d.hasBuiltIndex { // TODO: MemoryIndex is not writable, change to something else d.idx = idxfile.NewMemoryIndex() + d.writer = new(idxfile.Writer) + d.writer.OnHeader(count) } defer func() { d.hasBuiltIndex = true }() @@ -329,14 +342,15 @@ func (d *Decoder) decodeByHeader(h *ObjectHeader) (plumbing.EncodedObject, error obj.SetSize(h.Length) obj.SetType(h.Type) + var crc uint32 var err error switch h.Type { case plumbing.CommitObject, plumbing.TreeObject, plumbing.BlobObject, plumbing.TagObject: - _, err = d.fillRegularObjectContent(obj) + crc, err = d.fillRegularObjectContent(obj) case plumbing.REFDeltaObject: - _, err = d.fillREFDeltaObjectContent(obj, h.Reference) + crc, err = d.fillREFDeltaObjectContent(obj, h.Reference) case plumbing.OFSDeltaObject: - _, err = d.fillOFSDeltaObjectContent(obj, h.OffsetReference) + crc, err = d.fillOFSDeltaObjectContent(obj, h.OffsetReference) default: err = ErrInvalidObject.AddDetails("type %q", h.Type) } @@ -345,6 +359,10 @@ func (d *Decoder) decodeByHeader(h *ObjectHeader) (plumbing.EncodedObject, error return obj, err } + if !d.hasBuiltIndex { + d.writer.Add(obj.Hash(), uint64(h.Offset), crc) + } + d.offsetToHash[h.Offset] = obj.Hash() return obj, nil @@ -468,9 +486,9 @@ func (d *Decoder) recallByOffset(o int64) (plumbing.EncodedObject, error) { return d.DecodeObjectAt(o) } - hash, err := d.idx.FindHash(o) - if err != nil { - return nil, err + hash, ok := d.offsetToHash[o] + if !ok { + return nil, plumbing.ErrObjectNotFound } return d.recallByHashNonSeekable(hash) From 3d2bcdabdeda200d04eeaa3aeca7cc653317a961 Mon Sep 17 00:00:00 2001 From: Antonio Jesus Navarro Perez Date: Mon, 30 Jul 2018 10:38:28 +0200 Subject: [PATCH 045/120] Fix wrong godoc on Tags() method. Reword Tags() method documentation. Point to TagObjects() method to get all the tags on a repository. Signed-off-by: Antonio Jesus Navarro Perez --- repository.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/repository.go b/repository.go index 680522c6b..54572bc2b 100644 --- a/repository.go +++ b/repository.go @@ -845,8 +845,9 @@ func (r *Repository) Log(o *LogOptions) (object.CommitIter, error) { return nil, fmt.Errorf("invalid Order=%v", o.Order) } -// Tags returns all the References from Tags. This method returns all the tag -// types, lightweight, and annotated ones. +// Tags returns all the References from Tags. This method returns only lightweight +// tags. Note that not all the tags are lightweight ones. To return annotated tags +// too, you need to call TagObjects() method. func (r *Repository) Tags() (storer.ReferenceIter, error) { refIter, err := r.Storer.IterReferences() if err != nil { From 6f7fc05543861ee074aa17f75e1d1b5c1b948d48 Mon Sep 17 00:00:00 2001 From: Miguel Molina Date: Mon, 30 Jul 2018 17:11:01 +0200 Subject: [PATCH 046/120] plumbing: packfile, fix package tests Signed-off-by: Miguel Molina --- plumbing/format/idxfile/idxfile.go | 8 +++ plumbing/format/idxfile/writer.go | 11 +++- plumbing/format/packfile/decoder.go | 28 ++++++--- plumbing/format/packfile/decoder_test.go | 72 ++++++++++++++++------- plumbing/format/packfile/packfile_test.go | 2 +- storage/memory/storage.go | 10 ---- 6 files changed, 88 insertions(+), 43 deletions(-) diff --git a/plumbing/format/idxfile/idxfile.go b/plumbing/format/idxfile/idxfile.go index d4a936535..71c763015 100644 --- a/plumbing/format/idxfile/idxfile.go +++ b/plumbing/format/idxfile/idxfile.go @@ -67,6 +67,10 @@ func (idx *MemoryIndex) findHashIndex(h plumbing.Hash) int { return -1 } + if len(idx.Names) <= k { + return -1 + } + data := idx.Names[k] high := uint64(len(idx.Offset32[k])) >> 2 if high == 0 { @@ -103,6 +107,10 @@ func (idx *MemoryIndex) Contains(h plumbing.Hash) (bool, error) { // FindOffset implements the Index interface. func (idx *MemoryIndex) FindOffset(h plumbing.Hash) (int64, error) { + if len(idx.FanoutMapping) <= int(h[0]) { + return 0, plumbing.ErrObjectNotFound + } + k := idx.FanoutMapping[h[0]] i := idx.findHashIndex(h) if i < 0 { diff --git a/plumbing/format/idxfile/writer.go b/plumbing/format/idxfile/writer.go index efcdcc6c1..a22cf1616 100644 --- a/plumbing/format/idxfile/writer.go +++ b/plumbing/format/idxfile/writer.go @@ -25,6 +25,7 @@ type Writer struct { offset64 uint32 finished bool index *MemoryIndex + added map[plumbing.Hash]struct{} } // Index returns a previously created MemoryIndex or creates a new one if @@ -45,7 +46,15 @@ func (w *Writer) Add(h plumbing.Hash, pos uint64, crc uint32) { w.m.Lock() defer w.m.Unlock() - w.objects = append(w.objects, Entry{h, crc, pos}) + if w.added == nil { + w.added = make(map[plumbing.Hash]struct{}) + } + + if _, ok := w.added[h]; !ok { + w.added[h] = struct{}{} + w.objects = append(w.objects, Entry{h, crc, pos}) + } + } func (w *Writer) Finished() bool { diff --git a/plumbing/format/packfile/decoder.go b/plumbing/format/packfile/decoder.go index d6bc0efb8..6bb067759 100644 --- a/plumbing/format/packfile/decoder.go +++ b/plumbing/format/packfile/decoder.go @@ -122,6 +122,7 @@ func NewDecoderForType(s *Scanner, o storer.EncodedObjectStorer, deltaBaseCache: cacheObject, idx: idxfile.NewMemoryIndex(), + writer: new(idxfile.Writer), offsetToType: make(map[int64]plumbing.ObjectType), offsetToHash: make(map[int64]plumbing.Hash), decoderType: t, @@ -152,7 +153,12 @@ func (d *Decoder) Decode() (checksum plumbing.Hash, err error) { if !d.hasBuiltIndex { d.writer.OnFooter(checksum) - d.idx = d.Index() + + idx, err := d.writer.Index() + if err != nil { + return plumbing.ZeroHash, err + } + d.SetIndex(idx) } return checksum, err @@ -186,12 +192,8 @@ func (d *Decoder) doDecode() error { } if !d.hasBuiltIndex { - // TODO: MemoryIndex is not writable, change to something else - d.idx = idxfile.NewMemoryIndex() - d.writer = new(idxfile.Writer) d.writer.OnHeader(count) } - defer func() { d.hasBuiltIndex = true }() if d.hasBuiltIndex && !d.s.IsSeekable { if err := d.fillOffsetsToHashes(); err != nil { @@ -202,12 +204,18 @@ func (d *Decoder) doDecode() error { _, isTxStorer := d.o.(storer.Transactioner) switch { case d.o == nil: - return d.decodeObjects(int(count)) + err = d.decodeObjects(int(count)) case isTxStorer: - return d.decodeObjectsWithObjectStorerTx(int(count)) + err = d.decodeObjectsWithObjectStorerTx(int(count)) default: - return d.decodeObjectsWithObjectStorer(int(count)) + err = d.decodeObjectsWithObjectStorer(int(count)) + } + + if err != nil { + return err } + + return nil } func (d *Decoder) decodeObjects(count int) error { @@ -509,8 +517,10 @@ func (d *Decoder) recallByHash(h plumbing.Hash) (plumbing.EncodedObject, error) func (d *Decoder) recallByHashNonSeekable(h plumbing.Hash) (obj plumbing.EncodedObject, err error) { if d.tx != nil { obj, err = d.tx.EncodedObject(plumbing.AnyObject, h) - } else { + } else if d.o != nil { obj, err = d.o.EncodedObject(plumbing.AnyObject, h) + } else { + return nil, plumbing.ErrObjectNotFound } if err != plumbing.ErrObjectNotFound { diff --git a/plumbing/format/packfile/decoder_test.go b/plumbing/format/packfile/decoder_test.go index 4fe9b5e58..d4f714504 100644 --- a/plumbing/format/packfile/decoder_test.go +++ b/plumbing/format/packfile/decoder_test.go @@ -5,6 +5,7 @@ import ( "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/cache" + "gopkg.in/src-d/go-git.v4/plumbing/format/idxfile" "gopkg.in/src-d/go-git.v4/plumbing/format/packfile" "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/storage/filesystem" @@ -46,7 +47,6 @@ func (s *ReaderSuite) TestDecode(c *C) { }) } -/* func (s *ReaderSuite) TestDecodeByTypeRefDelta(c *C) { f := fixtures.Basic().ByTag("ref-delta").One() @@ -101,9 +101,7 @@ func (s *ReaderSuite) TestDecodeByTypeRefDeltaError(c *C) { }) } -*/ -/* func (s *ReaderSuite) TestDecodeByType(c *C) { ts := []plumbing.ObjectType{ plumbing.CommitObject, @@ -142,7 +140,6 @@ func (s *ReaderSuite) TestDecodeByType(c *C) { } }) } -*/ func (s *ReaderSuite) TestDecodeByTypeConstructor(c *C) { f := fixtures.Basic().ByTag("packfile").One() @@ -184,7 +181,7 @@ func (s *ReaderSuite) TestDecodeMultipleTimes(c *C) { func (s *ReaderSuite) TestDecodeInMemory(c *C) { fixtures.Basic().ByTag("packfile").Test(c, func(f *fixtures.Fixture) { scanner := packfile.NewScanner(f.Packfile()) - d, err := packfile.NewDecoder(scanner, nil) + d, err := packfile.NewDecoder(scanner, memory.NewStorage()) c.Assert(err, IsNil) ch, err := d.Decode() @@ -284,7 +281,6 @@ var expectedHashes = []string{ "7e59600739c96546163833214c36459e324bad0a", } -/* func (s *ReaderSuite) TestDecodeCRCs(c *C) { f := fixtures.Basic().ByTag("ofs-delta").One() @@ -297,8 +293,16 @@ func (s *ReaderSuite) TestDecodeCRCs(c *C) { c.Assert(err, IsNil) var sum uint64 - idx := d.Index().ToIdxFile() - for _, e := range idx.Entries { + iter, err := d.Index().Entries() + c.Assert(err, IsNil) + + for { + e, err := iter.Next() + if err == io.EOF { + break + } + + c.Assert(err, IsNil) sum += uint64(e.CRC32) } @@ -349,12 +353,30 @@ func (s *ReaderSuite) TestIndex(c *C) { d, err := packfile.NewDecoder(scanner, nil) c.Assert(err, IsNil) - c.Assert(d.Index().ToIdxFile().Entries, HasLen, 0) + c.Assert(indexEntries(c, d), Equals, 0) _, err = d.Decode() c.Assert(err, IsNil) - c.Assert(len(d.Index().ToIdxFile().Entries), Equals, 31) + c.Assert(indexEntries(c, d), Equals, 31) +} + +func indexEntries(c *C, d *packfile.Decoder) int { + var count int + entries, err := d.Index().Entries() + c.Assert(err, IsNil) + + for { + _, err := entries.Next() + if err == io.EOF { + break + } + + c.Assert(err, IsNil) + count++ + } + + return count } func (s *ReaderSuite) TestSetIndex(c *C) { @@ -363,18 +385,25 @@ func (s *ReaderSuite) TestSetIndex(c *C) { d, err := packfile.NewDecoder(scanner, nil) c.Assert(err, IsNil) - idx := packfile.NewIndex(1) + w := new(idxfile.Writer) h := plumbing.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5") - idx.Add(h, uint64(42), 0) + w.Add(h, uint64(42), 0) + w.OnFooter(plumbing.ZeroHash) + + var idx idxfile.Index + idx, err = w.Index() + c.Assert(err, IsNil) d.SetIndex(idx) - idxf := d.Index().ToIdxFile() - c.Assert(idxf.Entries, HasLen, 1) - c.Assert(idxf.Entries[0].Offset, Equals, uint64(42)) -}*/ + idx = d.Index() + c.Assert(indexEntries(c, d), Equals, 1) -func assertObjects(c *C, s storer.EncodedObjectStorer, expects []string) { + offset, err := idx.FindOffset(h) + c.Assert(err, IsNil) + c.Assert(offset, Equals, int64(42)) +} +func assertObjects(c *C, s storer.EncodedObjectStorer, expects []string) { i, err := s.IterEncodedObjects(plumbing.AnyObject) c.Assert(err, IsNil) @@ -390,13 +419,12 @@ func assertObjects(c *C, s storer.EncodedObjectStorer, expects []string) { } } -/* -func getIndexFromIdxFile(r io.Reader) *packfile.Index { - idxf := idxfile.NewIdxfile() +func getIndexFromIdxFile(r io.Reader) idxfile.Index { + idxf := idxfile.NewMemoryIndex() d := idxfile.NewDecoder(r) if err := d.Decode(idxf); err != nil { panic(err) } - return packfile.NewIndexFromIdxFile(idxf) -}*/ + return idxf +} diff --git a/plumbing/format/packfile/packfile_test.go b/plumbing/format/packfile/packfile_test.go index 0d7a80653..a17a48377 100644 --- a/plumbing/format/packfile/packfile_test.go +++ b/plumbing/format/packfile/packfile_test.go @@ -109,7 +109,7 @@ var expectedEntries = map[plumbing.Hash]int64{ } func (s *PackfileSuite) TestContent(c *C) { - storer := memory.NewObjectStorage() + storer := memory.NewStorage() decoder, err := NewDecoder(NewScanner(s.f.Packfile()), storer) c.Assert(err, IsNil) diff --git a/storage/memory/storage.go b/storage/memory/storage.go index a950a6224..2e3250905 100644 --- a/storage/memory/storage.go +++ b/storage/memory/storage.go @@ -91,16 +91,6 @@ type ObjectStorage struct { Tags map[plumbing.Hash]plumbing.EncodedObject } -func NewObjectStorage() *ObjectStorage { - return &ObjectStorage{ - Objects: make(map[plumbing.Hash]plumbing.EncodedObject), - Commits: make(map[plumbing.Hash]plumbing.EncodedObject), - Trees: make(map[plumbing.Hash]plumbing.EncodedObject), - Blobs: make(map[plumbing.Hash]plumbing.EncodedObject), - Tags: make(map[plumbing.Hash]plumbing.EncodedObject), - } -} - func (o *ObjectStorage) NewEncodedObject() plumbing.EncodedObject { return &plumbing.MemoryObject{} } From 6a24b4c1f0cb9e5daf30fa7979f2643a967af1ad Mon Sep 17 00:00:00 2001 From: Miguel Molina Date: Tue, 7 Aug 2018 18:41:19 +0200 Subject: [PATCH 047/120] *: use parser to populate non writable storages and bug fixes Signed-off-by: Miguel Molina --- common_test.go | 9 +- plumbing/format/idxfile/writer.go | 2 +- plumbing/format/idxfile/writer_test.go | 2 +- plumbing/format/packfile/common.go | 79 ++- plumbing/format/packfile/decoder.go | 553 ------------------ plumbing/format/packfile/decoder_test.go | 430 -------------- .../format/packfile/encoder_advanced_test.go | 37 +- plumbing/format/packfile/encoder_test.go | 110 ++-- plumbing/format/packfile/packfile.go | 135 +++-- plumbing/format/packfile/packfile_test.go | 169 ++++-- plumbing/format/packfile/parser.go | 130 ++-- plumbing/format/packfile/parser_test.go | 2 +- plumbing/object/blob_test.go | 23 +- plumbing/object/difftree_test.go | 16 +- plumbing/object/object_test.go | 5 +- plumbing/transport/test/receive_pack.go | 8 +- plumbing/transport/test/upload_pack.go | 5 +- storage/filesystem/object.go | 77 +-- storage/filesystem/object_test.go | 6 +- 19 files changed, 561 insertions(+), 1237 deletions(-) delete mode 100644 plumbing/format/packfile/decoder.go delete mode 100644 plumbing/format/packfile/decoder_test.go diff --git a/common_test.go b/common_test.go index f8f4e6124..efe1ecc92 100644 --- a/common_test.go +++ b/common_test.go @@ -113,14 +113,7 @@ func (s *BaseSuite) NewRepositoryFromPackfile(f *fixtures.Fixture) *Repository { p := f.Packfile() defer p.Close() - n := packfile.NewScanner(p) - d, err := packfile.NewDecoder(n, storer) - if err != nil { - panic(err) - } - - _, err = d.Decode() - if err != nil { + if err := packfile.UpdateObjectStorage(storer, p); err != nil { panic(err) } diff --git a/plumbing/format/idxfile/writer.go b/plumbing/format/idxfile/writer.go index a22cf1616..89b79cd1d 100644 --- a/plumbing/format/idxfile/writer.go +++ b/plumbing/format/idxfile/writer.go @@ -74,7 +74,7 @@ func (w *Writer) OnInflatedObjectHeader(t plumbing.ObjectType, objSize int64, po } // OnInflatedObjectContent implements packfile.Observer interface. -func (w *Writer) OnInflatedObjectContent(h plumbing.Hash, pos int64, crc uint32) error { +func (w *Writer) OnInflatedObjectContent(h plumbing.Hash, pos int64, crc uint32, _ []byte) error { w.Add(h, uint64(pos), crc) return nil } diff --git a/plumbing/format/idxfile/writer_test.go b/plumbing/format/idxfile/writer_test.go index 780acd978..7c3cceb89 100644 --- a/plumbing/format/idxfile/writer_test.go +++ b/plumbing/format/idxfile/writer_test.go @@ -52,7 +52,7 @@ func (s *WriterSuite) TestWriterLarge(c *C) { c.Assert(err, IsNil) for _, o := range fixture4GbEntries { - err = writer.OnInflatedObjectContent(plumbing.NewHash(o.hash), o.offset, o.crc) + err = writer.OnInflatedObjectContent(plumbing.NewHash(o.hash), o.offset, o.crc, nil) c.Assert(err, IsNil) } diff --git a/plumbing/format/packfile/common.go b/plumbing/format/packfile/common.go index beb015d3e..76254f036 100644 --- a/plumbing/format/packfile/common.go +++ b/plumbing/format/packfile/common.go @@ -2,9 +2,11 @@ package packfile import ( "bytes" + "errors" "io" "sync" + "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/utils/ioutil" ) @@ -23,24 +25,24 @@ const ( maskType = uint8(112) // 0111 0000 ) -// UpdateObjectStorage updates the given storer.EncodedObjectStorer with the contents of the +// UpdateObjectStorage updates the storer with the objects in the given // packfile. -func UpdateObjectStorage(s storer.EncodedObjectStorer, packfile io.Reader) error { - if sw, ok := s.(storer.PackfileWriter); ok { - return writePackfileToObjectStorage(sw, packfile) +func UpdateObjectStorage(s storer.Storer, packfile io.Reader) error { + if pw, ok := s.(storer.PackfileWriter); ok { + return WritePackfileToObjectStorage(pw, packfile) } - stream := NewScanner(packfile) - d, err := NewDecoder(stream, s) - if err != nil { - return err - } - - _, err = d.Decode() + updater := newPackfileStorageUpdater(s) + _, err := NewParser(NewScanner(packfile), updater).Parse() return err } -func writePackfileToObjectStorage(sw storer.PackfileWriter, packfile io.Reader) (err error) { +// WritePackfileToObjectStorage writes all the packfile objects into the given +// object storage. +func WritePackfileToObjectStorage( + sw storer.PackfileWriter, + packfile io.Reader, +) (err error) { w, err := sw.PackfileWriter() if err != nil { return err @@ -56,3 +58,56 @@ var bufPool = sync.Pool{ return bytes.NewBuffer(nil) }, } + +var errMissingObjectContent = errors.New("missing object content") + +type packfileStorageUpdater struct { + storer.Storer + lastSize int64 + lastType plumbing.ObjectType +} + +func newPackfileStorageUpdater(s storer.Storer) *packfileStorageUpdater { + return &packfileStorageUpdater{Storer: s} +} + +func (p *packfileStorageUpdater) OnHeader(count uint32) error { + return nil +} + +func (p *packfileStorageUpdater) OnInflatedObjectHeader( + t plumbing.ObjectType, + objSize int64, + pos int64, +) error { + if p.lastSize > 0 || p.lastType != plumbing.InvalidObject { + return errMissingObjectContent + } + + p.lastType = t + p.lastSize = objSize + return nil +} + +func (p *packfileStorageUpdater) OnInflatedObjectContent( + h plumbing.Hash, + pos int64, + crc uint32, + content []byte, +) error { + obj := new(plumbing.MemoryObject) + obj.SetSize(p.lastSize) + obj.SetType(p.lastType) + if _, err := obj.Write(content); err != nil { + return err + } + + _, err := p.SetEncodedObject(obj) + p.lastSize = 0 + p.lastType = plumbing.InvalidObject + return err +} + +func (p *packfileStorageUpdater) OnFooter(h plumbing.Hash) error { + return nil +} diff --git a/plumbing/format/packfile/decoder.go b/plumbing/format/packfile/decoder.go deleted file mode 100644 index 6bb067759..000000000 --- a/plumbing/format/packfile/decoder.go +++ /dev/null @@ -1,553 +0,0 @@ -package packfile - -import ( - "bytes" - "io" - - "gopkg.in/src-d/go-git.v4/plumbing" - "gopkg.in/src-d/go-git.v4/plumbing/cache" - "gopkg.in/src-d/go-git.v4/plumbing/format/idxfile" - "gopkg.in/src-d/go-git.v4/plumbing/storer" -) - -// Format specifies if the packfile uses ref-deltas or ofs-deltas. -type Format int - -// Possible values of the Format type. -const ( - UnknownFormat Format = iota - OFSDeltaFormat - REFDeltaFormat -) - -var ( - // ErrMaxObjectsLimitReached is returned by Decode when the number - // of objects in the packfile is higher than - // Decoder.MaxObjectsLimit. - ErrMaxObjectsLimitReached = NewError("max. objects limit reached") - // ErrInvalidObject is returned by Decode when an invalid object is - // found in the packfile. - ErrInvalidObject = NewError("invalid git object") - // ErrPackEntryNotFound is returned by Decode when a reference in - // the packfile references and unknown object. - ErrPackEntryNotFound = NewError("can't find a pack entry") - // ErrZLib is returned by Decode when there was an error unzipping - // the packfile contents. - ErrZLib = NewError("zlib reading error") - // ErrCannotRecall is returned by RecallByOffset or RecallByHash if the object - // to recall cannot be returned. - ErrCannotRecall = NewError("cannot recall object") - // ErrResolveDeltasNotSupported is returned if a NewDecoder is used with a - // non-seekable scanner and without a plumbing.ObjectStorage - ErrResolveDeltasNotSupported = NewError("resolve delta is not supported") - // ErrNonSeekable is returned if a ReadObjectAt method is called without a - // seekable scanner - ErrNonSeekable = NewError("non-seekable scanner") - // ErrRollback error making Rollback over a transaction after an error - ErrRollback = NewError("rollback error, during set error") - // ErrAlreadyDecoded is returned if NewDecoder is called for a second time - ErrAlreadyDecoded = NewError("packfile was already decoded") -) - -// Decoder reads and decodes packfiles from an input Scanner, if an ObjectStorer -// was provided the decoded objects are store there. If not the decode object -// is destroyed. The Offsets and CRCs are calculated whether an -// ObjectStorer was provided or not. -type Decoder struct { - deltaBaseCache cache.Object - - s *Scanner - o storer.EncodedObjectStorer - tx storer.Transaction - - isDecoded bool - - // hasBuiltIndex indicates if the index is fully built or not. If it is not, - // will be built incrementally while decoding. - hasBuiltIndex bool - idx idxfile.Index - writer *idxfile.Writer - - offsetToType map[int64]plumbing.ObjectType - decoderType plumbing.ObjectType - offsetToHash map[int64]plumbing.Hash -} - -// NewDecoder returns a new Decoder that decodes a Packfile using the given -// Scanner and stores the objects in the provided EncodedObjectStorer. ObjectStorer can be nil, in this -// If the passed EncodedObjectStorer is nil, objects are not stored, but -// offsets on the Packfile and CRCs are calculated. -// -// If EncodedObjectStorer is nil and the Scanner is not Seekable, ErrNonSeekable is -// returned. -// -// If the ObjectStorer implements storer.Transactioner, a transaction is created -// during the Decode execution. If anything fails, Rollback is called -func NewDecoder(s *Scanner, o storer.EncodedObjectStorer) (*Decoder, error) { - return NewDecoderForType(s, o, plumbing.AnyObject, - cache.NewObjectLRUDefault()) -} - -// NewDecoderWithCache is a version of NewDecoder where cache can be specified. -func NewDecoderWithCache(s *Scanner, o storer.EncodedObjectStorer, - cacheObject cache.Object) (*Decoder, error) { - - return NewDecoderForType(s, o, plumbing.AnyObject, cacheObject) -} - -// NewDecoderForType returns a new Decoder but in this case for a specific object type. -// When an object is read using this Decoder instance and it is not of the same type of -// the specified one, nil will be returned. This is intended to avoid the content -// deserialization of all the objects. -// -// cacheObject is a cache.Object implementation that is used to speed up the -// process. If cache is not needed you can pass nil. To create an LRU cache -// object with the default size you can use the helper cache.ObjectLRUDefault(). -func NewDecoderForType(s *Scanner, o storer.EncodedObjectStorer, - t plumbing.ObjectType, cacheObject cache.Object) (*Decoder, error) { - - if t == plumbing.OFSDeltaObject || - t == plumbing.REFDeltaObject || - t == plumbing.InvalidObject { - return nil, plumbing.ErrInvalidType - } - - if !canResolveDeltas(s, o) { - return nil, ErrResolveDeltasNotSupported - } - - return &Decoder{ - s: s, - o: o, - deltaBaseCache: cacheObject, - - idx: idxfile.NewMemoryIndex(), - writer: new(idxfile.Writer), - offsetToType: make(map[int64]plumbing.ObjectType), - offsetToHash: make(map[int64]plumbing.Hash), - decoderType: t, - }, nil -} - -func canResolveDeltas(s *Scanner, o storer.EncodedObjectStorer) bool { - return s.IsSeekable || o != nil -} - -// Decode reads a packfile and stores it in the value pointed to by s. The -// offsets and the CRCs are calculated by this method -func (d *Decoder) Decode() (checksum plumbing.Hash, err error) { - defer func() { d.isDecoded = true }() - - if d.isDecoded { - return plumbing.ZeroHash, ErrAlreadyDecoded - } - - if err := d.doDecode(); err != nil { - return plumbing.ZeroHash, err - } - - checksum, err = d.s.Checksum() - if err != nil { - return plumbing.ZeroHash, err - } - - if !d.hasBuiltIndex { - d.writer.OnFooter(checksum) - - idx, err := d.writer.Index() - if err != nil { - return plumbing.ZeroHash, err - } - d.SetIndex(idx) - } - - return checksum, err -} - -func (d *Decoder) fillOffsetsToHashes() error { - entries, err := d.idx.Entries() - if err != nil { - return err - } - - for { - e, err := entries.Next() - if err != nil { - if err == io.EOF { - break - } - return err - } - - d.offsetToHash[int64(e.Offset)] = e.Hash - } - - return entries.Close() -} - -func (d *Decoder) doDecode() error { - _, count, err := d.s.Header() - if err != nil { - return err - } - - if !d.hasBuiltIndex { - d.writer.OnHeader(count) - } - - if d.hasBuiltIndex && !d.s.IsSeekable { - if err := d.fillOffsetsToHashes(); err != nil { - return err - } - } - - _, isTxStorer := d.o.(storer.Transactioner) - switch { - case d.o == nil: - err = d.decodeObjects(int(count)) - case isTxStorer: - err = d.decodeObjectsWithObjectStorerTx(int(count)) - default: - err = d.decodeObjectsWithObjectStorer(int(count)) - } - - if err != nil { - return err - } - - return nil -} - -func (d *Decoder) decodeObjects(count int) error { - for i := 0; i < count; i++ { - if _, err := d.DecodeObject(); err != nil { - return err - } - } - - return nil -} - -func (d *Decoder) decodeObjectsWithObjectStorer(count int) error { - for i := 0; i < count; i++ { - obj, err := d.DecodeObject() - if err != nil { - return err - } - - if _, err := d.o.SetEncodedObject(obj); err != nil { - return err - } - } - - return nil -} - -func (d *Decoder) decodeObjectsWithObjectStorerTx(count int) error { - d.tx = d.o.(storer.Transactioner).Begin() - - for i := 0; i < count; i++ { - obj, err := d.DecodeObject() - if err != nil { - return err - } - - if _, err := d.tx.SetEncodedObject(obj); err != nil { - if rerr := d.tx.Rollback(); rerr != nil { - return ErrRollback.AddDetails( - "error: %s, during tx.Set error: %s", rerr, err, - ) - } - - return err - } - - } - - return d.tx.Commit() -} - -// DecodeObject reads the next object from the scanner and returns it. This -// method can be used in replacement of the Decode method, to work in a -// interactive way. If you created a new decoder instance using NewDecoderForType -// constructor, if the object decoded is not equals to the specified one, nil will -// be returned -func (d *Decoder) DecodeObject() (plumbing.EncodedObject, error) { - return d.doDecodeObject(d.decoderType) -} - -func (d *Decoder) doDecodeObject(t plumbing.ObjectType) (plumbing.EncodedObject, error) { - h, err := d.s.NextObjectHeader() - if err != nil { - return nil, err - } - - if t == plumbing.AnyObject { - return d.decodeByHeader(h) - } - - return d.decodeIfSpecificType(h) -} - -func (d *Decoder) decodeIfSpecificType(h *ObjectHeader) (plumbing.EncodedObject, error) { - var ( - obj plumbing.EncodedObject - realType plumbing.ObjectType - err error - ) - switch h.Type { - case plumbing.OFSDeltaObject: - realType, err = d.ofsDeltaType(h.OffsetReference) - case plumbing.REFDeltaObject: - realType, err = d.refDeltaType(h.Reference) - if err == plumbing.ErrObjectNotFound { - obj, err = d.decodeByHeader(h) - if err != nil { - realType = obj.Type() - } - } - default: - realType = h.Type - } - - if err != nil { - return nil, err - } - - d.offsetToType[h.Offset] = realType - - if d.decoderType == realType { - if obj != nil { - return obj, nil - } - - return d.decodeByHeader(h) - } - - return nil, nil -} - -func (d *Decoder) ofsDeltaType(offset int64) (plumbing.ObjectType, error) { - t, ok := d.offsetToType[offset] - if !ok { - return plumbing.InvalidObject, plumbing.ErrObjectNotFound - } - - return t, nil -} - -func (d *Decoder) refDeltaType(ref plumbing.Hash) (plumbing.ObjectType, error) { - offset, err := d.idx.FindOffset(ref) - if err != nil { - return plumbing.InvalidObject, plumbing.ErrObjectNotFound - } - - return d.ofsDeltaType(offset) -} - -func (d *Decoder) decodeByHeader(h *ObjectHeader) (plumbing.EncodedObject, error) { - obj := d.newObject() - obj.SetSize(h.Length) - obj.SetType(h.Type) - - var crc uint32 - var err error - switch h.Type { - case plumbing.CommitObject, plumbing.TreeObject, plumbing.BlobObject, plumbing.TagObject: - crc, err = d.fillRegularObjectContent(obj) - case plumbing.REFDeltaObject: - crc, err = d.fillREFDeltaObjectContent(obj, h.Reference) - case plumbing.OFSDeltaObject: - crc, err = d.fillOFSDeltaObjectContent(obj, h.OffsetReference) - default: - err = ErrInvalidObject.AddDetails("type %q", h.Type) - } - - if err != nil { - return obj, err - } - - if !d.hasBuiltIndex { - d.writer.Add(obj.Hash(), uint64(h.Offset), crc) - } - - d.offsetToHash[h.Offset] = obj.Hash() - - return obj, nil -} - -func (d *Decoder) newObject() plumbing.EncodedObject { - if d.o == nil { - return &plumbing.MemoryObject{} - } - - return d.o.NewEncodedObject() -} - -// DecodeObjectAt reads an object at the given location. Every EncodedObject -// returned is added into a internal index. This is intended to be able to regenerate -// objects from deltas (offset deltas or reference deltas) without an package index -// (.idx file). If Decode wasn't called previously objects offset should provided -// using the SetOffsets method. It decodes the object regardless of the Decoder -// type. -func (d *Decoder) DecodeObjectAt(offset int64) (plumbing.EncodedObject, error) { - if !d.s.IsSeekable { - return nil, ErrNonSeekable - } - - beforeJump, err := d.s.SeekFromStart(offset) - if err != nil { - return nil, err - } - - defer func() { - _, seekErr := d.s.SeekFromStart(beforeJump) - if err == nil { - err = seekErr - } - }() - - return d.doDecodeObject(plumbing.AnyObject) -} - -func (d *Decoder) fillRegularObjectContent(obj plumbing.EncodedObject) (uint32, error) { - w, err := obj.Writer() - if err != nil { - return 0, err - } - - _, crc, err := d.s.NextObject(w) - return crc, err -} - -func (d *Decoder) fillREFDeltaObjectContent(obj plumbing.EncodedObject, ref plumbing.Hash) (uint32, error) { - buf := bufPool.Get().(*bytes.Buffer) - buf.Reset() - _, crc, err := d.s.NextObject(buf) - if err != nil { - return 0, err - } - - base, ok := d.cacheGet(ref) - if !ok { - base, err = d.recallByHash(ref) - if err != nil { - return 0, err - } - } - - obj.SetType(base.Type()) - err = ApplyDelta(obj, base, buf.Bytes()) - d.cachePut(obj) - bufPool.Put(buf) - - return crc, err -} - -func (d *Decoder) fillOFSDeltaObjectContent(obj plumbing.EncodedObject, offset int64) (uint32, error) { - buf := bytes.NewBuffer(nil) - _, crc, err := d.s.NextObject(buf) - if err != nil { - return 0, err - } - - h, ok := d.offsetToHash[offset] - var base plumbing.EncodedObject - if ok { - base, ok = d.cacheGet(h) - } - - if !ok { - base, err = d.recallByOffset(offset) - if err != nil { - return 0, err - } - - d.cachePut(base) - } - - obj.SetType(base.Type()) - err = ApplyDelta(obj, base, buf.Bytes()) - d.cachePut(obj) - - return crc, err -} - -func (d *Decoder) cacheGet(h plumbing.Hash) (plumbing.EncodedObject, bool) { - if d.deltaBaseCache == nil { - return nil, false - } - - return d.deltaBaseCache.Get(h) -} - -func (d *Decoder) cachePut(obj plumbing.EncodedObject) { - if d.deltaBaseCache == nil { - return - } - - d.deltaBaseCache.Put(obj) -} - -func (d *Decoder) recallByOffset(o int64) (plumbing.EncodedObject, error) { - if d.s.IsSeekable { - return d.DecodeObjectAt(o) - } - - hash, ok := d.offsetToHash[o] - if !ok { - return nil, plumbing.ErrObjectNotFound - } - - return d.recallByHashNonSeekable(hash) -} - -func (d *Decoder) recallByHash(h plumbing.Hash) (plumbing.EncodedObject, error) { - if d.s.IsSeekable { - if offset, err := d.idx.FindOffset(h); err == nil { - return d.DecodeObjectAt(offset) - } - } - - return d.recallByHashNonSeekable(h) -} - -// recallByHashNonSeekable if we are in a transaction the objects are read from -// the transaction, if not are directly read from the ObjectStorer -func (d *Decoder) recallByHashNonSeekable(h plumbing.Hash) (obj plumbing.EncodedObject, err error) { - if d.tx != nil { - obj, err = d.tx.EncodedObject(plumbing.AnyObject, h) - } else if d.o != nil { - obj, err = d.o.EncodedObject(plumbing.AnyObject, h) - } else { - return nil, plumbing.ErrObjectNotFound - } - - if err != plumbing.ErrObjectNotFound { - return obj, err - } - - return nil, plumbing.ErrObjectNotFound -} - -// SetIndex sets an index for the packfile. It is recommended to set this. -// The index might be read from a file or reused from a previous Decoder usage -// (see Index function). -func (d *Decoder) SetIndex(idx idxfile.Index) { - d.hasBuiltIndex = true - d.idx = idx -} - -// Index returns the index for the packfile. If index was set with SetIndex, -// Index will return it. Otherwise, it will return an index that is built while -// decoding. If neither SetIndex was called with a full index or Decode called -// for the whole packfile, then the returned index will be incomplete. -func (d *Decoder) Index() idxfile.Index { - return d.idx -} - -// Close closes the Scanner. usually this mean that the whole reader is read and -// discarded -func (d *Decoder) Close() error { - return d.s.Close() -} diff --git a/plumbing/format/packfile/decoder_test.go b/plumbing/format/packfile/decoder_test.go deleted file mode 100644 index d4f714504..000000000 --- a/plumbing/format/packfile/decoder_test.go +++ /dev/null @@ -1,430 +0,0 @@ -package packfile_test - -import ( - "io" - - "gopkg.in/src-d/go-git.v4/plumbing" - "gopkg.in/src-d/go-git.v4/plumbing/cache" - "gopkg.in/src-d/go-git.v4/plumbing/format/idxfile" - "gopkg.in/src-d/go-git.v4/plumbing/format/packfile" - "gopkg.in/src-d/go-git.v4/plumbing/storer" - "gopkg.in/src-d/go-git.v4/storage/filesystem" - "gopkg.in/src-d/go-git.v4/storage/memory" - - . "gopkg.in/check.v1" - "gopkg.in/src-d/go-billy.v4/memfs" - "gopkg.in/src-d/go-git-fixtures.v3" -) - -type ReaderSuite struct { - fixtures.Suite -} - -var _ = Suite(&ReaderSuite{}) - -func (s *ReaderSuite) TestNewDecodeNonSeekable(c *C) { - scanner := packfile.NewScanner(nil) - d, err := packfile.NewDecoder(scanner, nil) - - c.Assert(d, IsNil) - c.Assert(err, NotNil) -} - -func (s *ReaderSuite) TestDecode(c *C) { - fixtures.Basic().ByTag("packfile").Test(c, func(f *fixtures.Fixture) { - scanner := packfile.NewScanner(f.Packfile()) - storage := memory.NewStorage() - - d, err := packfile.NewDecoder(scanner, storage) - c.Assert(err, IsNil) - defer d.Close() - - ch, err := d.Decode() - c.Assert(err, IsNil) - c.Assert(ch, Equals, f.PackfileHash) - - assertObjects(c, storage, expectedHashes) - }) -} - -func (s *ReaderSuite) TestDecodeByTypeRefDelta(c *C) { - f := fixtures.Basic().ByTag("ref-delta").One() - - storage := memory.NewStorage() - scanner := packfile.NewScanner(f.Packfile()) - d, err := packfile.NewDecoderForType(scanner, storage, plumbing.CommitObject, - cache.NewObjectLRUDefault()) - c.Assert(err, IsNil) - - // Index required to decode by ref-delta. - d.SetIndex(getIndexFromIdxFile(f.Idx())) - - defer d.Close() - - _, count, err := scanner.Header() - c.Assert(err, IsNil) - - var i uint32 - for i = 0; i < count; i++ { - obj, err := d.DecodeObject() - c.Assert(err, IsNil) - - if obj != nil { - c.Assert(obj.Type(), Equals, plumbing.CommitObject) - } - } -} - -func (s *ReaderSuite) TestDecodeByTypeRefDeltaError(c *C) { - fixtures.Basic().ByTag("ref-delta").Test(c, func(f *fixtures.Fixture) { - storage := memory.NewStorage() - scanner := packfile.NewScanner(f.Packfile()) - d, err := packfile.NewDecoderForType(scanner, storage, - plumbing.CommitObject, cache.NewObjectLRUDefault()) - c.Assert(err, IsNil) - - defer d.Close() - - _, count, err := scanner.Header() - c.Assert(err, IsNil) - - isError := false - var i uint32 - for i = 0; i < count; i++ { - _, err := d.DecodeObject() - if err != nil { - isError = true - break - } - } - c.Assert(isError, Equals, true) - }) - -} - -func (s *ReaderSuite) TestDecodeByType(c *C) { - ts := []plumbing.ObjectType{ - plumbing.CommitObject, - plumbing.TagObject, - plumbing.TreeObject, - plumbing.BlobObject, - } - - fixtures.Basic().ByTag("packfile").Test(c, func(f *fixtures.Fixture) { - for _, t := range ts { - storage := memory.NewStorage() - scanner := packfile.NewScanner(f.Packfile()) - d, err := packfile.NewDecoderForType(scanner, storage, t, - cache.NewObjectLRUDefault()) - c.Assert(err, IsNil) - - // when the packfile is ref-delta based, the offsets are required - if f.Is("ref-delta") { - d.SetIndex(getIndexFromIdxFile(f.Idx())) - } - - defer d.Close() - - _, count, err := scanner.Header() - c.Assert(err, IsNil) - - var i uint32 - for i = 0; i < count; i++ { - obj, err := d.DecodeObject() - c.Assert(err, IsNil) - - if obj != nil { - c.Assert(obj.Type(), Equals, t) - } - } - } - }) -} - -func (s *ReaderSuite) TestDecodeByTypeConstructor(c *C) { - f := fixtures.Basic().ByTag("packfile").One() - storage := memory.NewStorage() - scanner := packfile.NewScanner(f.Packfile()) - - _, err := packfile.NewDecoderForType(scanner, storage, - plumbing.OFSDeltaObject, cache.NewObjectLRUDefault()) - c.Assert(err, Equals, plumbing.ErrInvalidType) - - _, err = packfile.NewDecoderForType(scanner, storage, - plumbing.REFDeltaObject, cache.NewObjectLRUDefault()) - - c.Assert(err, Equals, plumbing.ErrInvalidType) - - _, err = packfile.NewDecoderForType(scanner, storage, plumbing.InvalidObject, - cache.NewObjectLRUDefault()) - c.Assert(err, Equals, plumbing.ErrInvalidType) -} - -func (s *ReaderSuite) TestDecodeMultipleTimes(c *C) { - f := fixtures.Basic().ByTag("packfile").One() - scanner := packfile.NewScanner(f.Packfile()) - storage := memory.NewStorage() - - d, err := packfile.NewDecoder(scanner, storage) - c.Assert(err, IsNil) - defer d.Close() - - ch, err := d.Decode() - c.Assert(err, IsNil) - c.Assert(ch, Equals, f.PackfileHash) - - ch, err = d.Decode() - c.Assert(err, Equals, packfile.ErrAlreadyDecoded) - c.Assert(ch, Equals, plumbing.ZeroHash) -} - -func (s *ReaderSuite) TestDecodeInMemory(c *C) { - fixtures.Basic().ByTag("packfile").Test(c, func(f *fixtures.Fixture) { - scanner := packfile.NewScanner(f.Packfile()) - d, err := packfile.NewDecoder(scanner, memory.NewStorage()) - c.Assert(err, IsNil) - - ch, err := d.Decode() - c.Assert(err, IsNil) - c.Assert(ch, Equals, f.PackfileHash) - }) -} - -type nonSeekableReader struct { - r io.Reader -} - -func (nsr nonSeekableReader) Read(b []byte) (int, error) { - return nsr.r.Read(b) -} - -func (s *ReaderSuite) TestDecodeNoSeekableWithTxStorer(c *C) { - fixtures.Basic().ByTag("packfile").Test(c, func(f *fixtures.Fixture) { - reader := nonSeekableReader{ - r: f.Packfile(), - } - - scanner := packfile.NewScanner(reader) - - var storage storer.EncodedObjectStorer = memory.NewStorage() - _, isTxStorer := storage.(storer.Transactioner) - c.Assert(isTxStorer, Equals, true) - - d, err := packfile.NewDecoder(scanner, storage) - c.Assert(err, IsNil) - defer d.Close() - - ch, err := d.Decode() - c.Assert(err, IsNil) - c.Assert(ch, Equals, f.PackfileHash) - - assertObjects(c, storage, expectedHashes) - }) -} - -func (s *ReaderSuite) TestDecodeNoSeekableWithoutTxStorer(c *C) { - fixtures.Basic().ByTag("packfile").Test(c, func(f *fixtures.Fixture) { - reader := nonSeekableReader{ - r: f.Packfile(), - } - - scanner := packfile.NewScanner(reader) - - var storage storer.EncodedObjectStorer - storage, _ = filesystem.NewStorage(memfs.New()) - _, isTxStorer := storage.(storer.Transactioner) - c.Assert(isTxStorer, Equals, false) - - d, err := packfile.NewDecoder(scanner, storage) - c.Assert(err, IsNil) - defer d.Close() - - ch, err := d.Decode() - c.Assert(err, IsNil) - c.Assert(ch, Equals, f.PackfileHash) - - assertObjects(c, storage, expectedHashes) - }) -} - -var expectedHashes = []string{ - "918c48b83bd081e863dbe1b80f8998f058cd8294", - "af2d6a6954d532f8ffb47615169c8fdf9d383a1a", - "1669dce138d9b841a518c64b10914d88f5e488ea", - "a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69", - "b8e471f58bcbca63b07bda20e428190409c2db47", - "35e85108805c84807bc66a02d91535e1e24b38b9", - "b029517f6300c2da0f4b651b8642506cd6aaf45d", - "32858aad3c383ed1ff0a0f9bdf231d54a00c9e88", - "d3ff53e0564a9f87d8e84b6e28e5060e517008aa", - "c192bd6a24ea1ab01d78686e417c8bdc7c3d197f", - "d5c0f4ab811897cadf03aec358ae60d21f91c50d", - "49c6bb89b17060d7b4deacb7b338fcc6ea2352a9", - "cf4aa3b38974fb7d81f367c0830f7d78d65ab86b", - "9dea2395f5403188298c1dabe8bdafe562c491e3", - "586af567d0bb5e771e49bdd9434f5e0fb76d25fa", - "9a48f23120e880dfbe41f7c9b7b708e9ee62a492", - "5a877e6a906a2743ad6e45d99c1793642aaf8eda", - "c8f1d8c61f9da76f4cb49fd86322b6e685dba956", - "a8d315b2b1c615d43042c3a62402b8a54288cf5c", - "a39771a7651f97faf5c72e08224d857fc35133db", - "880cd14280f4b9b6ed3986d6671f907d7cc2a198", - "fb72698cab7617ac416264415f13224dfd7a165e", - "4d081c50e250fa32ea8b1313cf8bb7c2ad7627fd", - "eba74343e2f15d62adedfd8c883ee0262b5c8021", - "c2d30fa8ef288618f65f6eed6e168e0d514886f4", - "8dcef98b1d52143e1e2dbc458ffe38f925786bf2", - "aa9b383c260e1d05fbbf6b30a02914555e20c725", - "6ecf0ef2c2dffb796033e5a02219af86ec6584e5", - "dbd3641b371024f44d0e469a9c8f5457b0660de1", - "e8d3ffab552895c19b9fcf7aa264d277cde33881", - "7e59600739c96546163833214c36459e324bad0a", -} - -func (s *ReaderSuite) TestDecodeCRCs(c *C) { - f := fixtures.Basic().ByTag("ofs-delta").One() - - scanner := packfile.NewScanner(f.Packfile()) - storage := memory.NewStorage() - - d, err := packfile.NewDecoder(scanner, storage) - c.Assert(err, IsNil) - _, err = d.Decode() - c.Assert(err, IsNil) - - var sum uint64 - iter, err := d.Index().Entries() - c.Assert(err, IsNil) - - for { - e, err := iter.Next() - if err == io.EOF { - break - } - - c.Assert(err, IsNil) - sum += uint64(e.CRC32) - } - - c.Assert(int(sum), Equals, 78022211966) -} - -func (s *ReaderSuite) TestDecodeObjectAt(c *C) { - f := fixtures.Basic().One() - scanner := packfile.NewScanner(f.Packfile()) - d, err := packfile.NewDecoder(scanner, nil) - c.Assert(err, IsNil) - - // when the packfile is ref-delta based, the offsets are required - if f.Is("ref-delta") { - d.SetIndex(getIndexFromIdxFile(f.Idx())) - } - - // the objects at reference 186, is a delta, so should be recall, - // without being read before. - obj, err := d.DecodeObjectAt(186) - c.Assert(err, IsNil) - c.Assert(obj.Hash().String(), Equals, "6ecf0ef2c2dffb796033e5a02219af86ec6584e5") -} - -func (s *ReaderSuite) TestDecodeObjectAtForType(c *C) { - f := fixtures.Basic().One() - scanner := packfile.NewScanner(f.Packfile()) - d, err := packfile.NewDecoderForType(scanner, nil, plumbing.TreeObject, - cache.NewObjectLRUDefault()) - c.Assert(err, IsNil) - - // when the packfile is ref-delta based, the offsets are required - if f.Is("ref-delta") { - d.SetIndex(getIndexFromIdxFile(f.Idx())) - } - - // the objects at reference 186, is a delta, so should be recall, - // without being read before. - obj, err := d.DecodeObjectAt(186) - c.Assert(err, IsNil) - c.Assert(obj.Type(), Equals, plumbing.CommitObject) - c.Assert(obj.Hash().String(), Equals, "6ecf0ef2c2dffb796033e5a02219af86ec6584e5") -} - -func (s *ReaderSuite) TestIndex(c *C) { - f := fixtures.Basic().One() - scanner := packfile.NewScanner(f.Packfile()) - d, err := packfile.NewDecoder(scanner, nil) - c.Assert(err, IsNil) - - c.Assert(indexEntries(c, d), Equals, 0) - - _, err = d.Decode() - c.Assert(err, IsNil) - - c.Assert(indexEntries(c, d), Equals, 31) -} - -func indexEntries(c *C, d *packfile.Decoder) int { - var count int - entries, err := d.Index().Entries() - c.Assert(err, IsNil) - - for { - _, err := entries.Next() - if err == io.EOF { - break - } - - c.Assert(err, IsNil) - count++ - } - - return count -} - -func (s *ReaderSuite) TestSetIndex(c *C) { - f := fixtures.Basic().One() - scanner := packfile.NewScanner(f.Packfile()) - d, err := packfile.NewDecoder(scanner, nil) - c.Assert(err, IsNil) - - w := new(idxfile.Writer) - h := plumbing.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5") - w.Add(h, uint64(42), 0) - w.OnFooter(plumbing.ZeroHash) - - var idx idxfile.Index - idx, err = w.Index() - c.Assert(err, IsNil) - d.SetIndex(idx) - - idx = d.Index() - c.Assert(indexEntries(c, d), Equals, 1) - - offset, err := idx.FindOffset(h) - c.Assert(err, IsNil) - c.Assert(offset, Equals, int64(42)) -} - -func assertObjects(c *C, s storer.EncodedObjectStorer, expects []string) { - i, err := s.IterEncodedObjects(plumbing.AnyObject) - c.Assert(err, IsNil) - - var count int - err = i.ForEach(func(plumbing.EncodedObject) error { count++; return nil }) - c.Assert(err, IsNil) - c.Assert(count, Equals, len(expects)) - - for _, exp := range expects { - obt, err := s.EncodedObject(plumbing.AnyObject, plumbing.NewHash(exp)) - c.Assert(err, IsNil) - c.Assert(obt.Hash().String(), Equals, exp) - } -} - -func getIndexFromIdxFile(r io.Reader) idxfile.Index { - idxf := idxfile.NewMemoryIndex() - d := idxfile.NewDecoder(r) - if err := d.Decode(idxf); err != nil { - panic(err) - } - - return idxf -} diff --git a/plumbing/format/packfile/encoder_advanced_test.go b/plumbing/format/packfile/encoder_advanced_test.go index 8cc7180da..6ffebc29b 100644 --- a/plumbing/format/packfile/encoder_advanced_test.go +++ b/plumbing/format/packfile/encoder_advanced_test.go @@ -2,14 +2,16 @@ package packfile_test import ( "bytes" + "io" "math/rand" "testing" + "gopkg.in/src-d/go-billy.v3/memfs" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/format/idxfile" . "gopkg.in/src-d/go-git.v4/plumbing/format/packfile" "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/storage/filesystem" - "gopkg.in/src-d/go-git.v4/storage/memory" . "gopkg.in/check.v1" "gopkg.in/src-d/go-git-fixtures.v3" @@ -34,7 +36,6 @@ func (s *EncoderAdvancedSuite) TestEncodeDecode(c *C) { c.Assert(err, IsNil) s.testEncodeDecode(c, storage, 10) }) - } func (s *EncoderAdvancedSuite) TestEncodeDecodeNoDeltaCompression(c *C) { @@ -52,8 +53,11 @@ func (s *EncoderAdvancedSuite) TestEncodeDecodeNoDeltaCompression(c *C) { }) } -func (s *EncoderAdvancedSuite) testEncodeDecode(c *C, storage storer.Storer, packWindow uint) { - +func (s *EncoderAdvancedSuite) testEncodeDecode( + c *C, + storage storer.Storer, + packWindow uint, +) { objIter, err := storage.IterEncodedObjects(plumbing.AnyObject) c.Assert(err, IsNil) @@ -80,16 +84,31 @@ func (s *EncoderAdvancedSuite) testEncodeDecode(c *C, storage storer.Storer, pac encodeHash, err := enc.Encode(hashes, packWindow) c.Assert(err, IsNil) - scanner := NewScanner(buf) - storage = memory.NewStorage() - d, err := NewDecoder(scanner, storage) + f, err := memfs.New().Create("packfile") + c.Assert(err, IsNil) + + _, err = f.Write(buf.Bytes()) + c.Assert(err, IsNil) + + _, err = f.Seek(0, io.SeekStart) c.Assert(err, IsNil) - decodeHash, err := d.Decode() + + w := new(idxfile.Writer) + _, err = NewParser(NewScanner(f), w).Parse() + c.Assert(err, IsNil) + index, err := w.Index() + c.Assert(err, IsNil) + + _, err = f.Seek(0, io.SeekStart) c.Assert(err, IsNil) + p := NewPackfile(index, f) + + decodeHash, err := p.ID() + c.Assert(err, IsNil) c.Assert(encodeHash, Equals, decodeHash) - objIter, err = storage.IterEncodedObjects(plumbing.AnyObject) + objIter, err = p.GetAll() c.Assert(err, IsNil) obtainedObjects := map[plumbing.Hash]bool{} err = objIter.ForEach(func(o plumbing.EncodedObject) error { diff --git a/plumbing/format/packfile/encoder_test.go b/plumbing/format/packfile/encoder_test.go index 84d03fb32..7b6dde2d9 100644 --- a/plumbing/format/packfile/encoder_test.go +++ b/plumbing/format/packfile/encoder_test.go @@ -2,8 +2,12 @@ package packfile import ( "bytes" + "io" + stdioutil "io/ioutil" + "gopkg.in/src-d/go-billy.v3/memfs" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/format/idxfile" "gopkg.in/src-d/go-git.v4/storage/memory" . "gopkg.in/check.v1" @@ -130,24 +134,20 @@ func (s *EncoderSuite) simpleDeltaTest(c *C) { }) c.Assert(err, IsNil) - scanner := NewScanner(s.buf) - - storage := memory.NewStorage() - d, err := NewDecoder(scanner, storage) - c.Assert(err, IsNil) - - decHash, err := d.Decode() + p, cleanup := packfileFromReader(c, s.buf) + defer cleanup() + decHash, err := p.ID() c.Assert(err, IsNil) c.Assert(encHash, Equals, decHash) - decSrc, err := storage.EncodedObject(srcObject.Type(), srcObject.Hash()) + decSrc, err := p.Get(srcObject.Hash()) c.Assert(err, IsNil) - c.Assert(decSrc, DeepEquals, srcObject) + objectsEqual(c, decSrc, srcObject) - decTarget, err := storage.EncodedObject(targetObject.Type(), targetObject.Hash()) + decTarget, err := p.Get(targetObject.Hash()) c.Assert(err, IsNil) - c.Assert(decTarget, DeepEquals, targetObject) + objectsEqual(c, decTarget, targetObject) } func (s *EncoderSuite) deltaOverDeltaTest(c *C) { @@ -173,27 +173,24 @@ func (s *EncoderSuite) deltaOverDeltaTest(c *C) { }) c.Assert(err, IsNil) - scanner := NewScanner(s.buf) - storage := memory.NewStorage() - d, err := NewDecoder(scanner, storage) - c.Assert(err, IsNil) - - decHash, err := d.Decode() + p, cleanup := packfileFromReader(c, s.buf) + defer cleanup() + decHash, err := p.ID() c.Assert(err, IsNil) c.Assert(encHash, Equals, decHash) - decSrc, err := storage.EncodedObject(srcObject.Type(), srcObject.Hash()) + decSrc, err := p.Get(srcObject.Hash()) c.Assert(err, IsNil) - c.Assert(decSrc, DeepEquals, srcObject) + objectsEqual(c, decSrc, srcObject) - decTarget, err := storage.EncodedObject(targetObject.Type(), targetObject.Hash()) + decTarget, err := p.Get(targetObject.Hash()) c.Assert(err, IsNil) - c.Assert(decTarget, DeepEquals, targetObject) + objectsEqual(c, decTarget, targetObject) - decOtherTarget, err := storage.EncodedObject(otherTargetObject.Type(), otherTargetObject.Hash()) + decOtherTarget, err := p.Get(otherTargetObject.Hash()) c.Assert(err, IsNil) - c.Assert(decOtherTarget, DeepEquals, otherTargetObject) + objectsEqual(c, decOtherTarget, otherTargetObject) } func (s *EncoderSuite) deltaOverDeltaCyclicTest(c *C) { @@ -248,29 +245,70 @@ func (s *EncoderSuite) deltaOverDeltaCyclicTest(c *C) { }) c.Assert(err, IsNil) - scanner := NewScanner(s.buf) - storage := memory.NewStorage() - d, err := NewDecoder(scanner, storage) + p, cleanup := packfileFromReader(c, s.buf) + defer cleanup() + decHash, err := p.ID() c.Assert(err, IsNil) - decHash, err := d.Decode() + c.Assert(encHash, Equals, decHash) + + decSrc, err := p.Get(o1.Hash()) c.Assert(err, IsNil) + objectsEqual(c, decSrc, o1) - c.Assert(encHash, Equals, decHash) + decTarget, err := p.Get(o2.Hash()) + c.Assert(err, IsNil) + objectsEqual(c, decTarget, o2) + + decOtherTarget, err := p.Get(o3.Hash()) + c.Assert(err, IsNil) + objectsEqual(c, decOtherTarget, o3) + + decAnotherTarget, err := p.Get(o4.Hash()) + c.Assert(err, IsNil) + objectsEqual(c, decAnotherTarget, o4) +} + +func objectsEqual(c *C, o1, o2 plumbing.EncodedObject) { + c.Assert(o1.Type(), Equals, o2.Type()) + c.Assert(o1.Hash(), Equals, o2.Hash()) + c.Assert(o1.Size(), Equals, o2.Size()) - decSrc, err := storage.EncodedObject(o1.Type(), o1.Hash()) + r1, err := o1.Reader() c.Assert(err, IsNil) - c.Assert(decSrc, DeepEquals, o1) - decTarget, err := storage.EncodedObject(o2.Type(), o2.Hash()) + b1, err := stdioutil.ReadAll(r1) c.Assert(err, IsNil) - c.Assert(decTarget, DeepEquals, o2) - decOtherTarget, err := storage.EncodedObject(o3.Type(), o3.Hash()) + r2, err := o2.Reader() c.Assert(err, IsNil) - c.Assert(decOtherTarget, DeepEquals, o3) - decAnotherTarget, err := storage.EncodedObject(o4.Type(), o4.Hash()) + b2, err := stdioutil.ReadAll(r2) c.Assert(err, IsNil) - c.Assert(decAnotherTarget, DeepEquals, o4) + + c.Assert(bytes.Compare(b1, b2), Equals, 0) +} + +func packfileFromReader(c *C, buf *bytes.Buffer) (*Packfile, func()) { + file, err := memfs.New().Create("packfile") + c.Assert(err, IsNil) + + _, err = file.Write(buf.Bytes()) + c.Assert(err, IsNil) + + _, err = file.Seek(0, io.SeekStart) + c.Assert(err, IsNil) + + scanner := NewScanner(file) + + w := new(idxfile.Writer) + _, err = NewParser(scanner, w).Parse() + c.Assert(err, IsNil) + + index, err := w.Index() + c.Assert(err, IsNil) + + return NewPackfile(index, file), func() { + c.Assert(file.Close(), IsNil) + } } diff --git a/plumbing/format/packfile/packfile.go b/plumbing/format/packfile/packfile.go index 2e831f255..37743ba70 100644 --- a/plumbing/format/packfile/packfile.go +++ b/plumbing/format/packfile/packfile.go @@ -3,38 +3,55 @@ package packfile import ( "bytes" "io" + stdioutil "io/ioutil" "os" - billy "gopkg.in/src-d/go-billy.v4" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/format/idxfile" "gopkg.in/src-d/go-git.v4/plumbing/storer" ) +var ( + // ErrInvalidObject is returned by Decode when an invalid object is + // found in the packfile. + ErrInvalidObject = NewError("invalid git object") + // ErrZLib is returned by Decode when there was an error unzipping + // the packfile contents. + ErrZLib = NewError("zlib reading error") +) + // Packfile allows retrieving information from inside a packfile. type Packfile struct { idxfile.Index - billy.File + file io.ReadSeeker s *Scanner deltaBaseCache cache.Object offsetToType map[int64]plumbing.ObjectType } -// NewPackfile returns a packfile representation for the given packfile file -// and packfile idx. -func NewPackfile(index idxfile.Index, file billy.File) *Packfile { +// NewPackfileWithCache creates a new Packfile with the given object cache. +func NewPackfileWithCache( + index idxfile.Index, + file io.ReadSeeker, + cache cache.Object, +) *Packfile { s := NewScanner(file) - return &Packfile{ index, file, s, - cache.NewObjectLRUDefault(), + cache, make(map[int64]plumbing.ObjectType), } } +// NewPackfile returns a packfile representation for the given packfile file +// and packfile idx. +func NewPackfile(index idxfile.Index, file io.ReadSeeker) *Packfile { + return NewPackfileWithCache(index, file, cache.NewObjectLRUDefault()) +} + // Get retrieves the encoded object in the packfile with the given hash. func (p *Packfile) Get(h plumbing.Hash) (plumbing.EncodedObject, error) { offset, err := p.FindOffset(h) @@ -334,35 +351,49 @@ func (p *Packfile) cachePut(obj plumbing.EncodedObject) { // The iterator returned is not thread-safe, it should be used in the same // thread as the Packfile instance. func (p *Packfile) GetAll() (storer.EncodedObjectIter, error) { - entries, err := p.Entries() - if err != nil { - return nil, err - } + return p.GetByType(plumbing.AnyObject) +} - return &objectIter{ - // Easiest way to provide an object decoder is just to pass a Packfile - // instance. To not mess with the seeks, it's a new instance with a - // different scanner but the same cache and offset to hash map for - // reusing as much cache as possible. - p: &Packfile{ - p.Index, - p.File, - NewScanner(p.File), - p.deltaBaseCache, - p.offsetToType, - }, - iter: entries, - }, nil +// GetByType returns all the objects of the given type. +func (p *Packfile) GetByType(typ plumbing.ObjectType) (storer.EncodedObjectIter, error) { + switch typ { + case plumbing.AnyObject, + plumbing.BlobObject, + plumbing.TreeObject, + plumbing.CommitObject, + plumbing.TagObject: + entries, err := p.Entries() + if err != nil { + return nil, err + } + + return &objectIter{ + // Easiest way to provide an object decoder is just to pass a Packfile + // instance. To not mess with the seeks, it's a new instance with a + // different scanner but the same cache and offset to hash map for + // reusing as much cache as possible. + p: p, + iter: entries, + typ: typ, + }, nil + default: + return nil, plumbing.ErrInvalidType + } } // ID returns the ID of the packfile, which is the checksum at the end of it. func (p *Packfile) ID() (plumbing.Hash, error) { - if _, err := p.File.Seek(-20, io.SeekEnd); err != nil { + prev, err := p.file.Seek(-20, io.SeekEnd) + if err != nil { return plumbing.ZeroHash, err } var hash plumbing.Hash - if _, err := io.ReadFull(p.File, hash[:]); err != nil { + if _, err := io.ReadFull(p.file, hash[:]); err != nil { + return plumbing.ZeroHash, err + } + + if _, err := p.file.Seek(prev, io.SeekStart); err != nil { return plumbing.ZeroHash, err } @@ -371,25 +402,59 @@ func (p *Packfile) ID() (plumbing.Hash, error) { // Close the packfile and its resources. func (p *Packfile) Close() error { - return p.File.Close() + closer, ok := p.file.(io.Closer) + if !ok { + return nil + } + + return closer.Close() } -type objectDecoder interface { - nextObject() (plumbing.EncodedObject, error) +// MemoryObjectFromDisk converts a DiskObject to a MemoryObject. +func MemoryObjectFromDisk(obj plumbing.EncodedObject) (plumbing.EncodedObject, error) { + o2 := new(plumbing.MemoryObject) + o2.SetType(obj.Type()) + o2.SetSize(obj.Size()) + + r, err := obj.Reader() + if err != nil { + return nil, err + } + + data, err := stdioutil.ReadAll(r) + if err != nil { + return nil, err + } + + if _, err := o2.Write(data); err != nil { + return nil, err + } + + return o2, nil } type objectIter struct { p *Packfile + typ plumbing.ObjectType iter idxfile.EntryIter } func (i *objectIter) Next() (plumbing.EncodedObject, error) { - e, err := i.iter.Next() - if err != nil { - return nil, err - } + for { + e, err := i.iter.Next() + if err != nil { + return nil, err + } - return i.p.GetByOffset(int64(e.Offset)) + obj, err := i.p.GetByOffset(int64(e.Offset)) + if err != nil { + return nil, err + } + + if i.typ == plumbing.AnyObject || obj.Type() == i.typ { + return obj, nil + } + } } func (i *objectIter) ForEach(f func(plumbing.EncodedObject) error) error { diff --git a/plumbing/format/packfile/packfile_test.go b/plumbing/format/packfile/packfile_test.go index e2347942c..3193bed04 100644 --- a/plumbing/format/packfile/packfile_test.go +++ b/plumbing/format/packfile/packfile_test.go @@ -1,23 +1,21 @@ -package packfile +package packfile_test import ( - "bytes" "io" "math" - "io/ioutil" - . "gopkg.in/check.v1" "gopkg.in/src-d/go-billy.v4/osfs" fixtures "gopkg.in/src-d/go-git-fixtures.v3" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/format/idxfile" - "gopkg.in/src-d/go-git.v4/storage/memory" + "gopkg.in/src-d/go-git.v4/plumbing/format/packfile" + "gopkg.in/src-d/go-git.v4/plumbing/storer" ) type PackfileSuite struct { fixtures.Suite - p *Packfile + p *packfile.Packfile idx *idxfile.MemoryIndex f *fixtures.Fixture } @@ -108,60 +106,157 @@ var expectedEntries = map[plumbing.Hash]int64{ plumbing.NewHash("fb72698cab7617ac416264415f13224dfd7a165e"): 84671, } -func (s *PackfileSuite) TestContent(c *C) { - storer := memory.NewStorage() - decoder, err := NewDecoder(NewScanner(s.f.Packfile()), storer) - c.Assert(err, IsNil) +func (s *PackfileSuite) SetUpTest(c *C) { + s.f = fixtures.Basic().One() - _, err = decoder.Decode() + f, err := osfs.New("").Open(s.f.Packfile().Name()) c.Assert(err, IsNil) - iter, err := s.p.GetAll() + s.idx = idxfile.NewMemoryIndex() + c.Assert(idxfile.NewDecoder(s.f.Idx()).Decode(s.idx), IsNil) + + s.p = packfile.NewPackfile(s.idx, f) +} + +func (s *PackfileSuite) TearDownTest(c *C) { + c.Assert(s.p.Close(), IsNil) +} + +func (s *PackfileSuite) TestDecode(c *C) { + fixtures.Basic().ByTag("packfile").Test(c, func(f *fixtures.Fixture) { + index := getIndexFromIdxFile(f.Idx()) + p := packfile.NewPackfile(index, f.Packfile()) + defer p.Close() + + for _, h := range expectedHashes { + obj, err := p.Get(plumbing.NewHash(h)) + c.Assert(err, IsNil) + c.Assert(obj.Hash().String(), Equals, h) + } + }) +} + +func (s *PackfileSuite) TestDecodeByTypeRefDelta(c *C) { + f := fixtures.Basic().ByTag("ref-delta").One() + + index := getIndexFromIdxFile(f.Idx()) + packfile := packfile.NewPackfile(index, f.Packfile()) + defer packfile.Close() + + iter, err := packfile.GetByType(plumbing.CommitObject) c.Assert(err, IsNil) + var count int for { - o, err := iter.Next() + obj, err := iter.Next() if err == io.EOF { break } + count++ c.Assert(err, IsNil) + c.Assert(obj.Type(), Equals, plumbing.CommitObject) + } - o2, err := storer.EncodedObject(plumbing.AnyObject, o.Hash()) - c.Assert(err, IsNil) + c.Assert(count > 0, Equals, true) +} - c.Assert(o.Type(), Equals, o2.Type()) - c.Assert(o.Size(), Equals, o2.Size()) +func (s *PackfileSuite) TestDecodeByType(c *C) { + ts := []plumbing.ObjectType{ + plumbing.CommitObject, + plumbing.TagObject, + plumbing.TreeObject, + plumbing.BlobObject, + } - r, err := o.Reader() - c.Assert(err, IsNil) + fixtures.Basic().ByTag("packfile").Test(c, func(f *fixtures.Fixture) { + for _, t := range ts { + index := getIndexFromIdxFile(f.Idx()) + packfile := packfile.NewPackfile(index, f.Packfile()) + defer packfile.Close() - c1, err := ioutil.ReadAll(r) - c.Assert(err, IsNil) - c.Assert(r.Close(), IsNil) + iter, err := packfile.GetByType(t) + c.Assert(err, IsNil) - r, err = o2.Reader() - c.Assert(err, IsNil) + c.Assert(iter.ForEach(func(obj plumbing.EncodedObject) error { + c.Assert(obj.Type(), Equals, t) + return nil + }), IsNil) + } + }) +} - c2, err := ioutil.ReadAll(r) - c.Assert(err, IsNil) - c.Assert(r.Close(), IsNil) +func (s *PackfileSuite) TestDecodeByTypeConstructor(c *C) { + f := fixtures.Basic().ByTag("packfile").One() + index := getIndexFromIdxFile(f.Idx()) + packfile := packfile.NewPackfile(index, f.Packfile()) + defer packfile.Close() - c.Assert(bytes.Compare(c1, c2), Equals, 0) - } + _, err := packfile.GetByType(plumbing.OFSDeltaObject) + c.Assert(err, Equals, plumbing.ErrInvalidType) + + _, err = packfile.GetByType(plumbing.REFDeltaObject) + c.Assert(err, Equals, plumbing.ErrInvalidType) + + _, err = packfile.GetByType(plumbing.InvalidObject) + c.Assert(err, Equals, plumbing.ErrInvalidType) } -func (s *PackfileSuite) SetUpTest(c *C) { - s.f = fixtures.Basic().One() +var expectedHashes = []string{ + "918c48b83bd081e863dbe1b80f8998f058cd8294", + "af2d6a6954d532f8ffb47615169c8fdf9d383a1a", + "1669dce138d9b841a518c64b10914d88f5e488ea", + "a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69", + "b8e471f58bcbca63b07bda20e428190409c2db47", + "35e85108805c84807bc66a02d91535e1e24b38b9", + "b029517f6300c2da0f4b651b8642506cd6aaf45d", + "32858aad3c383ed1ff0a0f9bdf231d54a00c9e88", + "d3ff53e0564a9f87d8e84b6e28e5060e517008aa", + "c192bd6a24ea1ab01d78686e417c8bdc7c3d197f", + "d5c0f4ab811897cadf03aec358ae60d21f91c50d", + "49c6bb89b17060d7b4deacb7b338fcc6ea2352a9", + "cf4aa3b38974fb7d81f367c0830f7d78d65ab86b", + "9dea2395f5403188298c1dabe8bdafe562c491e3", + "586af567d0bb5e771e49bdd9434f5e0fb76d25fa", + "9a48f23120e880dfbe41f7c9b7b708e9ee62a492", + "5a877e6a906a2743ad6e45d99c1793642aaf8eda", + "c8f1d8c61f9da76f4cb49fd86322b6e685dba956", + "a8d315b2b1c615d43042c3a62402b8a54288cf5c", + "a39771a7651f97faf5c72e08224d857fc35133db", + "880cd14280f4b9b6ed3986d6671f907d7cc2a198", + "fb72698cab7617ac416264415f13224dfd7a165e", + "4d081c50e250fa32ea8b1313cf8bb7c2ad7627fd", + "eba74343e2f15d62adedfd8c883ee0262b5c8021", + "c2d30fa8ef288618f65f6eed6e168e0d514886f4", + "8dcef98b1d52143e1e2dbc458ffe38f925786bf2", + "aa9b383c260e1d05fbbf6b30a02914555e20c725", + "6ecf0ef2c2dffb796033e5a02219af86ec6584e5", + "dbd3641b371024f44d0e469a9c8f5457b0660de1", + "e8d3ffab552895c19b9fcf7aa264d277cde33881", + "7e59600739c96546163833214c36459e324bad0a", +} - f, err := osfs.New("").Open(s.f.Packfile().Name()) +func assertObjects(c *C, s storer.EncodedObjectStorer, expects []string) { + i, err := s.IterEncodedObjects(plumbing.AnyObject) c.Assert(err, IsNil) - s.idx = idxfile.NewMemoryIndex() - c.Assert(idxfile.NewDecoder(s.f.Idx()).Decode(s.idx), IsNil) + var count int + err = i.ForEach(func(plumbing.EncodedObject) error { count++; return nil }) + c.Assert(err, IsNil) + c.Assert(count, Equals, len(expects)) - s.p = NewPackfile(s.idx, f) + for _, exp := range expects { + obt, err := s.EncodedObject(plumbing.AnyObject, plumbing.NewHash(exp)) + c.Assert(err, IsNil) + c.Assert(obt.Hash().String(), Equals, exp) + } } -func (s *PackfileSuite) TearDownTest(c *C) { - c.Assert(s.p.Close(), IsNil) +func getIndexFromIdxFile(r io.Reader) idxfile.Index { + idxf := idxfile.NewMemoryIndex() + d := idxfile.NewDecoder(r) + if err := d.Decode(idxf); err != nil { + panic(err) + } + + return idxf } diff --git a/plumbing/format/packfile/parser.go b/plumbing/format/packfile/parser.go index 696f5ba96..f0a76747c 100644 --- a/plumbing/format/packfile/parser.go +++ b/plumbing/format/packfile/parser.go @@ -9,6 +9,16 @@ import ( "gopkg.in/src-d/go-git.v4/plumbing/cache" ) +var ( + // ErrObjectContentAlreadyRead is returned when the content of the object + // was already read, since the content can only be read once. + ErrObjectContentAlreadyRead = errors.New("object content was already read") + + // ErrReferenceDeltaNotFound is returned when the reference delta is not + // found. + ErrReferenceDeltaNotFound = errors.New("reference delta not found") +) + // Observer interface is implemented by index encoders. type Observer interface { // OnHeader is called when a new packfile is opened. @@ -16,7 +26,7 @@ type Observer interface { // OnInflatedObjectHeader is called for each object header read. OnInflatedObjectHeader(t plumbing.ObjectType, objSize int64, pos int64) error // OnInflatedObjectContent is called for each decoded object. - OnInflatedObjectContent(h plumbing.Hash, pos int64, crc uint32) error + OnInflatedObjectContent(h plumbing.Hash, pos int64, crc uint32, content []byte) error // OnFooter is called when decoding is done. OnFooter(h plumbing.Hash) error } @@ -32,41 +42,44 @@ type Parser struct { hashOffset map[plumbing.Hash]int64 checksum plumbing.Hash - cache *cache.ObjectLRU + cache *cache.ObjectLRU + contentCache map[int64][]byte ob []Observer } // NewParser creates a new Parser struct. func NewParser(scanner *Scanner, ob ...Observer) *Parser { + var contentCache map[int64][]byte + if !scanner.IsSeekable { + contentCache = make(map[int64][]byte) + } + return &Parser{ - scanner: scanner, - ob: ob, - count: 0, - cache: cache.NewObjectLRUDefault(), + scanner: scanner, + ob: ob, + count: 0, + cache: cache.NewObjectLRUDefault(), + contentCache: contentCache, } } // Parse start decoding phase of the packfile. func (p *Parser) Parse() (plumbing.Hash, error) { - err := p.init() - if err != nil { + if err := p.init(); err != nil { return plumbing.ZeroHash, err } - err = p.firstPass() - if err != nil { + if err := p.firstPass(); err != nil { return plumbing.ZeroHash, err } - err = p.resolveDeltas() - if err != nil { + if err := p.resolveDeltas(); err != nil { return plumbing.ZeroHash, err } for _, o := range p.ob { - err := o.OnFooter(p.checksum) - if err != nil { + if err := o.OnFooter(p.checksum); err != nil { return plumbing.ZeroHash, err } } @@ -81,8 +94,7 @@ func (p *Parser) init() error { } for _, o := range p.ob { - err := o.OnHeader(c) - if err != nil { + if err := o.OnHeader(c); err != nil { return err } } @@ -99,7 +111,7 @@ func (p *Parser) firstPass() error { buf := new(bytes.Buffer) for i := uint32(0); i < p.count; i++ { - buf.Truncate(0) + buf.Reset() oh, err := p.scanner.NextObjectHeader() if err != nil { @@ -122,8 +134,7 @@ func (p *Parser) firstPass() error { } if !ok { - // TODO improve error - return errors.New("Reference delta not found") + return ErrReferenceDeltaNotFound } ota = newDeltaObject(oh.Offset, oh.Length, t, parent) @@ -143,35 +154,41 @@ func (p *Parser) firstPass() error { ota.Length = oh.Length if !delta { - ota.Write(buf.Bytes()) + if _, err := ota.Write(buf.Bytes()); err != nil { + return err + } ota.SHA1 = ota.Sum() + p.oiByHash[ota.SHA1] = ota } p.oiByOffset[oh.Offset] = ota - p.oiByHash[oh.Reference] = ota p.oi[i] = ota } - checksum, err := p.scanner.Checksum() - p.checksum = checksum - - if err == io.EOF { - return nil + var err error + p.checksum, err = p.scanner.Checksum() + if err != nil && err != io.EOF { + return err } - return err + return nil } func (p *Parser) resolveDeltas() error { for _, obj := range p.oi { + content, err := obj.Content() + if err != nil { + return err + } + for _, o := range p.ob { err := o.OnInflatedObjectHeader(obj.Type, obj.Length, obj.Offset) if err != nil { return err } - err = o.OnInflatedObjectContent(obj.SHA1, obj.Offset, obj.Crc32) + err = o.OnInflatedObjectContent(obj.SHA1, obj.Offset, obj.Crc32, content) if err != nil { return err } @@ -185,8 +202,7 @@ func (p *Parser) resolveDeltas() error { } for _, child := range obj.Children { - _, err = p.resolveObject(child, base) - if err != nil { + if _, err := p.resolveObject(child, base); err != nil { return err } } @@ -205,8 +221,7 @@ func (p *Parser) get(o *objectInfo) ([]byte, error) { } buf := make([]byte, e.Size()) - _, err = r.Read(buf) - if err != nil { + if _, err = r.Read(buf); err != nil { return nil, err } @@ -254,8 +269,8 @@ func (p *Parser) get(o *objectInfo) ([]byte, error) { func (p *Parser) resolveObject( o *objectInfo, - base []byte) ([]byte, error) { - + base []byte, +) ([]byte, error) { if !o.DiskType.IsDelta() { return nil, nil } @@ -278,16 +293,17 @@ func (p *Parser) readData(o *objectInfo) ([]byte, error) { // TODO: skip header. Header size can be calculated with the offset of the // next offset in the first pass. - p.scanner.SeekFromStart(o.Offset) - _, err := p.scanner.NextObjectHeader() - if err != nil { + if _, err := p.scanner.SeekFromStart(o.Offset); err != nil { return nil, err } - buf.Truncate(0) + if _, err := p.scanner.NextObjectHeader(); err != nil { + return nil, err + } - _, _, err = p.scanner.NextObject(buf) - if err != nil { + buf.Reset() + + if _, _, err := p.scanner.NextObject(buf); err != nil { return nil, err } @@ -301,9 +317,11 @@ func applyPatchBase(ota *objectInfo, data, base []byte) ([]byte, error) { } ota.Type = ota.Parent.Type - hash := plumbing.ComputeHash(ota.Type, patched) - - ota.SHA1 = hash + ota.Hasher = plumbing.NewHasher(ota.Type, int64(len(patched))) + if _, err := ota.Write(patched); err != nil { + return nil, err + } + ota.SHA1 = ota.Sum() return patched, nil } @@ -323,6 +341,8 @@ type objectInfo struct { Parent *objectInfo Children []*objectInfo SHA1 plumbing.Hash + + content *bytes.Buffer } func newBaseObject(offset, length int64, t plumbing.ObjectType) *objectInfo { @@ -351,6 +371,30 @@ func newDeltaObject( return obj } +func (o *objectInfo) Write(bs []byte) (int, error) { + n, err := o.Hasher.Write(bs) + if err != nil { + return 0, err + } + + o.content = bytes.NewBuffer(nil) + + _, _ = o.content.Write(bs) + return n, nil +} + +// Content returns the content of the object. This operation can only be done +// once. +func (o *objectInfo) Content() ([]byte, error) { + if o.content == nil { + return nil, ErrObjectContentAlreadyRead + } + + r := o.content + o.content = nil + return r.Bytes(), nil +} + func (o *objectInfo) IsDelta() bool { return o.Type.IsDelta() } diff --git a/plumbing/format/packfile/parser_test.go b/plumbing/format/packfile/parser_test.go index 87a880436..b18f20fd7 100644 --- a/plumbing/format/packfile/parser_test.go +++ b/plumbing/format/packfile/parser_test.go @@ -103,7 +103,7 @@ func (t *testObserver) OnInflatedObjectHeader(otype plumbing.ObjectType, objSize return nil } -func (t *testObserver) OnInflatedObjectContent(h plumbing.Hash, pos int64, crc uint32) error { +func (t *testObserver) OnInflatedObjectContent(h plumbing.Hash, pos int64, crc uint32, _ []byte) error { o := t.get(pos) o.hash = h.String() o.crc = crc diff --git a/plumbing/object/blob_test.go b/plumbing/object/blob_test.go index 5ed9de0dd..181436d5a 100644 --- a/plumbing/object/blob_test.go +++ b/plumbing/object/blob_test.go @@ -1,6 +1,7 @@ package object import ( + "bytes" "io" "io/ioutil" @@ -88,8 +89,26 @@ func (s *BlobsSuite) TestBlobIter(c *C) { } c.Assert(err, IsNil) - c.Assert(b, DeepEquals, blobs[i]) - i += 1 + c.Assert(b.ID(), Equals, blobs[i].ID()) + c.Assert(b.Size, Equals, blobs[i].Size) + c.Assert(b.Type(), Equals, blobs[i].Type()) + + r1, err := b.Reader() + c.Assert(err, IsNil) + + b1, err := ioutil.ReadAll(r1) + c.Assert(err, IsNil) + c.Assert(r1.Close(), IsNil) + + r2, err := blobs[i].Reader() + c.Assert(err, IsNil) + + b2, err := ioutil.ReadAll(r2) + c.Assert(err, IsNil) + c.Assert(r2.Close(), IsNil) + + c.Assert(bytes.Compare(b1, b2), Equals, 0) + i++ } iter.Close() diff --git a/plumbing/object/difftree_test.go b/plumbing/object/difftree_test.go index 40af8f2de..ff9ecbc3f 100644 --- a/plumbing/object/difftree_test.go +++ b/plumbing/object/difftree_test.go @@ -45,25 +45,17 @@ func (s *DiffTreeSuite) storageFromPackfile(f *fixtures.Fixture) storer.EncodedO return sto } - sto = memory.NewStorage() + storer := memory.NewStorage() pf := f.Packfile() - defer pf.Close() - n := packfile.NewScanner(pf) - d, err := packfile.NewDecoder(n, sto) - if err != nil { - panic(err) - } - - _, err = d.Decode() - if err != nil { + if err := packfile.UpdateObjectStorage(storer, pf); err != nil { panic(err) } - s.cache[f.URL] = sto - return sto + s.cache[f.URL] = storer + return storer } var _ = Suite(&DiffTreeSuite{}) diff --git a/plumbing/object/object_test.go b/plumbing/object/object_test.go index 4f0fcb34c..68aa1a13e 100644 --- a/plumbing/object/object_test.go +++ b/plumbing/object/object_test.go @@ -197,8 +197,9 @@ func (s *ObjectsSuite) TestObjectIter(c *C) { } c.Assert(err, IsNil) - c.Assert(o, DeepEquals, objects[i]) - i += 1 + c.Assert(o.ID(), Equals, objects[i].ID()) + c.Assert(o.Type(), Equals, objects[i].Type()) + i++ } iter.Close() diff --git a/plumbing/transport/test/receive_pack.go b/plumbing/transport/test/receive_pack.go index 57f602dd8..5aea1c019 100644 --- a/plumbing/transport/test/receive_pack.go +++ b/plumbing/transport/test/receive_pack.go @@ -262,13 +262,16 @@ func (s *ReceivePackSuite) receivePackNoCheck(c *C, ep *transport.Endpoint, req.Packfile = s.emptyPackfile() } - return r.ReceivePack(context.Background(), req) + if s, err := r.ReceivePack(context.Background(), req); err != nil { + return s, err + } else { + return s, err + } } func (s *ReceivePackSuite) receivePack(c *C, ep *transport.Endpoint, req *packp.ReferenceUpdateRequest, fixture *fixtures.Fixture, callAdvertisedReferences bool) { - url := "" if fixture != nil { url = fixture.URL @@ -279,7 +282,6 @@ func (s *ReceivePackSuite) receivePack(c *C, ep *transport.Endpoint, ep.String(), url, callAdvertisedReferences, ) report, err := s.receivePackNoCheck(c, ep, req, fixture, callAdvertisedReferences) - c.Assert(err, IsNil, comment) if req.Capabilities.Supports(capability.ReportStatus) { c.Assert(report, NotNil, comment) diff --git a/plumbing/transport/test/upload_pack.go b/plumbing/transport/test/upload_pack.go index 70e4e561c..8709ac2c0 100644 --- a/plumbing/transport/test/upload_pack.go +++ b/plumbing/transport/test/upload_pack.go @@ -258,11 +258,8 @@ func (s *UploadPackSuite) checkObjectNumber(c *C, r io.Reader, n int) { b, err := ioutil.ReadAll(r) c.Assert(err, IsNil) buf := bytes.NewBuffer(b) - scanner := packfile.NewScanner(buf) storage := memory.NewStorage() - d, err := packfile.NewDecoder(scanner, storage) - c.Assert(err, IsNil) - _, err = d.Decode() + err = packfile.UpdateObjectStorage(storage, buf) c.Assert(err, IsNil) c.Assert(len(storage.Objects), Equals, n) } diff --git a/storage/filesystem/object.go b/storage/filesystem/object.go index b73b3093e..2032eac04 100644 --- a/storage/filesystem/object.go +++ b/storage/filesystem/object.go @@ -12,7 +12,6 @@ import ( "gopkg.in/src-d/go-git.v4/plumbing/format/packfile" "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit" - "gopkg.in/src-d/go-git.v4/storage/memory" "gopkg.in/src-d/go-git.v4/utils/ioutil" "gopkg.in/src-d/go-billy.v4" @@ -282,29 +281,34 @@ func (s *ObjectStorage) getFromPackfile(h plumbing.Hash, canBeDelta bool) ( func (s *ObjectStorage) decodeObjectAt( f billy.File, idx idxfile.Index, - offset int64) (plumbing.EncodedObject, error) { - if _, err := f.Seek(0, io.SeekStart); err != nil { - return nil, err + offset int64, +) (plumbing.EncodedObject, error) { + hash, err := idx.FindHash(offset) + if err == nil { + obj, ok := s.deltaBaseCache.Get(hash) + if ok { + return obj, nil + } } - p := packfile.NewScanner(f) + if err != nil && err != plumbing.ErrObjectNotFound { + return nil, err + } - d, err := packfile.NewDecoderWithCache(p, memory.NewStorage(), - s.deltaBaseCache) + obj, err := packfile.NewPackfile(idx, f).GetByOffset(offset) if err != nil { return nil, err } - d.SetIndex(idx) - obj, err := d.DecodeObjectAt(offset) - return obj, err + return packfile.MemoryObjectFromDisk(obj) } func (s *ObjectStorage) decodeDeltaObjectAt( f billy.File, idx idxfile.Index, offset int64, - hash plumbing.Hash) (plumbing.EncodedObject, error) { + hash plumbing.Hash, +) (plumbing.EncodedObject, error) { if _, err := f.Seek(0, io.SeekStart); err != nil { return nil, err } @@ -453,22 +457,23 @@ func (it *lazyPackfilesIter) Close() { } type packfileIter struct { - f billy.File - d *packfile.Decoder - t plumbing.ObjectType - - seen map[plumbing.Hash]struct{} - position uint32 - total uint32 + iter storer.EncodedObjectIter + seen map[plumbing.Hash]struct{} } // NewPackfileIter returns a new EncodedObjectIter for the provided packfile // and object type. func NewPackfileIter( f billy.File, + idxFile billy.File, t plumbing.ObjectType, ) (storer.EncodedObjectIter, error) { - return newPackfileIter(f, t, make(map[plumbing.Hash]struct{}), nil, nil) + idx := idxfile.NewMemoryIndex() + if err := idxfile.NewDecoder(idxFile).Decode(idx); err != nil { + return nil, err + } + + return newPackfileIter(f, t, make(map[plumbing.Hash]struct{}), idx, nil) } func newPackfileIter( @@ -478,47 +483,26 @@ func newPackfileIter( index idxfile.Index, cache cache.Object, ) (storer.EncodedObjectIter, error) { - s := packfile.NewScanner(f) - _, total, err := s.Header() + iter, err := packfile.NewPackfile(index, f).GetByType(t) if err != nil { return nil, err } - d, err := packfile.NewDecoderForType(s, memory.NewStorage(), t, cache) - if err != nil { - return nil, err - } - - d.SetIndex(index) - return &packfileIter{ - f: f, - d: d, - t: t, - - total: total, - seen: seen, + iter: iter, + seen: seen, }, nil } func (iter *packfileIter) Next() (plumbing.EncodedObject, error) { for { - if iter.position >= iter.total { - return nil, io.EOF - } - - obj, err := iter.d.DecodeObject() + obj, err := iter.iter.Next() if err != nil { return nil, err } - iter.position++ - if obj == nil { - continue - } - if _, ok := iter.seen[obj.Hash()]; ok { - return iter.Next() + continue } return obj, nil @@ -531,8 +515,7 @@ func (iter *packfileIter) ForEach(cb func(plumbing.EncodedObject) error) error { } func (iter *packfileIter) Close() { - iter.f.Close() - iter.d.Close() + iter.iter.Close() } type objectsIter struct { diff --git a/storage/filesystem/object_test.go b/storage/filesystem/object_test.go index ecd6bebc3..ae11c3bdc 100644 --- a/storage/filesystem/object_test.go +++ b/storage/filesystem/object_test.go @@ -115,7 +115,11 @@ func (s *FsSuite) TestPackfileIter(c *C) { for _, h := range ph { f, err := dg.ObjectPack(h) c.Assert(err, IsNil) - iter, err := NewPackfileIter(f, t) + + idxf, err := dg.ObjectPackIdx(h) + c.Assert(err, IsNil) + + iter, err := NewPackfileIter(f, idxf, t) c.Assert(err, IsNil) err = iter.ForEach(func(o plumbing.EncodedObject) error { c.Assert(o.Type(), Equals, t) From 7b1248177ad3533c63590c7c47495f28f0f3da71 Mon Sep 17 00:00:00 2001 From: Fedor Korotkov Date: Wed, 8 Aug 2018 09:09:12 -0400 Subject: [PATCH 048/120] Fixed cloning of a single tag Relates to #870 Signed-off-by: Fedor Korotkov --- remote.go | 4 ++-- repository.go | 6 +++--- repository_test.go | 28 +++++++++++++++++++++++++++- 3 files changed, 32 insertions(+), 6 deletions(-) diff --git a/remote.go b/remote.go index bf4519c71..0556b9894 100644 --- a/remote.go +++ b/remote.go @@ -619,7 +619,7 @@ func getHaves( return result, nil } -const refspecTag = "+refs/tags/*:refs/tags/*" +const refspecAllTags = "+refs/tags/*:refs/tags/*" func calculateRefs( spec []config.RefSpec, @@ -627,7 +627,7 @@ func calculateRefs( tagMode TagMode, ) (memory.ReferenceStorage, error) { if tagMode == AllTags { - spec = append(spec, refspecTag) + spec = append(spec, refspecAllTags) } refs := make(memory.ReferenceStorage) diff --git a/repository.go b/repository.go index 54572bc2b..818cfb3c3 100644 --- a/repository.go +++ b/repository.go @@ -583,7 +583,7 @@ func (r *Repository) clone(ctx context.Context, o *CloneOptions) error { } const ( - refspecTagWithDepth = "+refs/tags/%s:refs/tags/%[1]s" + refspecTag = "+refs/tags/%s:refs/tags/%[1]s" refspecSingleBranch = "+refs/heads/%s:refs/remotes/%s/%[1]s" refspecSingleBranchHEAD = "+HEAD:refs/remotes/%s/HEAD" ) @@ -592,8 +592,8 @@ func (r *Repository) cloneRefSpec(o *CloneOptions, c *config.RemoteConfig) []con var rs string switch { - case o.ReferenceName.IsTag() && o.Depth > 0: - rs = fmt.Sprintf(refspecTagWithDepth, o.ReferenceName.Short()) + case o.ReferenceName.IsTag(): + rs = fmt.Sprintf(refspecTag, o.ReferenceName.Short()) case o.SingleBranch && o.ReferenceName == plumbing.HEAD: rs = fmt.Sprintf(refspecSingleBranchHEAD, c.Name) case o.SingleBranch: diff --git a/repository_test.go b/repository_test.go index b78fbb70b..75808784a 100644 --- a/repository_test.go +++ b/repository_test.go @@ -846,7 +846,33 @@ func (s *RepositorySuite) TestCloneDetachedHEAD(c *C) { objects, err := r.Objects() c.Assert(err, IsNil) objects.ForEach(func(object.Object) error { count++; return nil }) - c.Assert(count, Equals, 31) + c.Assert(count, Equals, 28) +} + +func (s *RepositorySuite) TestCloneDetachedHEADAndSingle(c *C) { + r, _ := Init(memory.NewStorage(), nil) + err := r.clone(context.Background(), &CloneOptions{ + URL: s.GetBasicLocalRepositoryURL(), + ReferenceName: plumbing.ReferenceName("refs/tags/v1.0.0"), + SingleBranch: true, + }) + c.Assert(err, IsNil) + + cfg, err := r.Config() + c.Assert(err, IsNil) + c.Assert(cfg.Branches, HasLen, 0) + + head, err := r.Reference(plumbing.HEAD, false) + c.Assert(err, IsNil) + c.Assert(head, NotNil) + c.Assert(head.Type(), Equals, plumbing.HashReference) + c.Assert(head.Hash().String(), Equals, "6ecf0ef2c2dffb796033e5a02219af86ec6584e5") + + count := 0 + objects, err := r.Objects() + c.Assert(err, IsNil) + objects.ForEach(func(object.Object) error { count++; return nil }) + c.Assert(count, Equals, 28) } func (s *RepositorySuite) TestCloneDetachedHEADAndShallow(c *C) { From 5889a3b669f0f515ff445aa040afc1e7eeb2bbd1 Mon Sep 17 00:00:00 2001 From: Miguel Molina Date: Wed, 8 Aug 2018 16:56:20 +0200 Subject: [PATCH 049/120] plumbing: packfile, allow non-seekable sources on Parser Signed-off-by: Miguel Molina --- plumbing/format/idxfile/writer_test.go | 5 +- plumbing/format/packfile/common.go | 63 +--- .../format/packfile/encoder_advanced_test.go | 5 +- plumbing/format/packfile/encoder_test.go | 5 +- plumbing/format/packfile/parser.go | 311 ++++++++++++------ plumbing/format/packfile/parser_test.go | 19 +- storage/filesystem/dotgit/writers.go | 7 +- 7 files changed, 235 insertions(+), 180 deletions(-) diff --git a/plumbing/format/idxfile/writer_test.go b/plumbing/format/idxfile/writer_test.go index 7c3cceb89..912211d3a 100644 --- a/plumbing/format/idxfile/writer_test.go +++ b/plumbing/format/idxfile/writer_test.go @@ -24,9 +24,10 @@ func (s *WriterSuite) TestWriter(c *C) { scanner := packfile.NewScanner(f.Packfile()) obs := new(idxfile.Writer) - parser := packfile.NewParser(scanner, obs) + parser, err := packfile.NewParser(scanner, obs) + c.Assert(err, IsNil) - _, err := parser.Parse() + _, err = parser.Parse() c.Assert(err, IsNil) idx, err := obs.Index() diff --git a/plumbing/format/packfile/common.go b/plumbing/format/packfile/common.go index 76254f036..2b4acebde 100644 --- a/plumbing/format/packfile/common.go +++ b/plumbing/format/packfile/common.go @@ -2,11 +2,9 @@ package packfile import ( "bytes" - "errors" "io" "sync" - "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/utils/ioutil" ) @@ -32,8 +30,12 @@ func UpdateObjectStorage(s storer.Storer, packfile io.Reader) error { return WritePackfileToObjectStorage(pw, packfile) } - updater := newPackfileStorageUpdater(s) - _, err := NewParser(NewScanner(packfile), updater).Parse() + p, err := NewParserWithStorage(NewScanner(packfile), s) + if err != nil { + return err + } + + _, err = p.Parse() return err } @@ -58,56 +60,3 @@ var bufPool = sync.Pool{ return bytes.NewBuffer(nil) }, } - -var errMissingObjectContent = errors.New("missing object content") - -type packfileStorageUpdater struct { - storer.Storer - lastSize int64 - lastType plumbing.ObjectType -} - -func newPackfileStorageUpdater(s storer.Storer) *packfileStorageUpdater { - return &packfileStorageUpdater{Storer: s} -} - -func (p *packfileStorageUpdater) OnHeader(count uint32) error { - return nil -} - -func (p *packfileStorageUpdater) OnInflatedObjectHeader( - t plumbing.ObjectType, - objSize int64, - pos int64, -) error { - if p.lastSize > 0 || p.lastType != plumbing.InvalidObject { - return errMissingObjectContent - } - - p.lastType = t - p.lastSize = objSize - return nil -} - -func (p *packfileStorageUpdater) OnInflatedObjectContent( - h plumbing.Hash, - pos int64, - crc uint32, - content []byte, -) error { - obj := new(plumbing.MemoryObject) - obj.SetSize(p.lastSize) - obj.SetType(p.lastType) - if _, err := obj.Write(content); err != nil { - return err - } - - _, err := p.SetEncodedObject(obj) - p.lastSize = 0 - p.lastType = plumbing.InvalidObject - return err -} - -func (p *packfileStorageUpdater) OnFooter(h plumbing.Hash) error { - return nil -} diff --git a/plumbing/format/packfile/encoder_advanced_test.go b/plumbing/format/packfile/encoder_advanced_test.go index 6ffebc29b..78ddc45fe 100644 --- a/plumbing/format/packfile/encoder_advanced_test.go +++ b/plumbing/format/packfile/encoder_advanced_test.go @@ -94,7 +94,10 @@ func (s *EncoderAdvancedSuite) testEncodeDecode( c.Assert(err, IsNil) w := new(idxfile.Writer) - _, err = NewParser(NewScanner(f), w).Parse() + parser, err := NewParser(NewScanner(f), w) + c.Assert(err, IsNil) + + _, err = parser.Parse() c.Assert(err, IsNil) index, err := w.Index() c.Assert(err, IsNil) diff --git a/plumbing/format/packfile/encoder_test.go b/plumbing/format/packfile/encoder_test.go index 7b6dde2d9..24e20820c 100644 --- a/plumbing/format/packfile/encoder_test.go +++ b/plumbing/format/packfile/encoder_test.go @@ -302,7 +302,10 @@ func packfileFromReader(c *C, buf *bytes.Buffer) (*Packfile, func()) { scanner := NewScanner(file) w := new(idxfile.Writer) - _, err = NewParser(scanner, w).Parse() + p, err := NewParser(scanner, w) + c.Assert(err, IsNil) + + _, err = p.Parse() c.Assert(err, IsNil) index, err := w.Index() diff --git a/plumbing/format/packfile/parser.go b/plumbing/format/packfile/parser.go index f0a76747c..beb3e27ac 100644 --- a/plumbing/format/packfile/parser.go +++ b/plumbing/format/packfile/parser.go @@ -7,16 +7,20 @@ import ( "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/cache" + "gopkg.in/src-d/go-git.v4/plumbing/storer" ) var ( - // ErrObjectContentAlreadyRead is returned when the content of the object - // was already read, since the content can only be read once. - ErrObjectContentAlreadyRead = errors.New("object content was already read") - // ErrReferenceDeltaNotFound is returned when the reference delta is not // found. ErrReferenceDeltaNotFound = errors.New("reference delta not found") + + // ErrNotSeekableSource is returned when the source for the parser is not + // seekable and a storage was not provided, so it can't be parsed. + ErrNotSeekableSource = errors.New("parser source is not seekable and storage was not provided") + + // ErrDeltaNotCached is returned when the delta could not be found in cache. + ErrDeltaNotCached = errors.New("delta could not be found in cache") ) // Observer interface is implemented by index encoders. @@ -34,34 +38,96 @@ type Observer interface { // Parser decodes a packfile and calls any observer associated to it. Is used // to generate indexes. type Parser struct { - scanner *Scanner - count uint32 - oi []*objectInfo - oiByHash map[plumbing.Hash]*objectInfo - oiByOffset map[int64]*objectInfo - hashOffset map[plumbing.Hash]int64 - checksum plumbing.Hash - - cache *cache.ObjectLRU - contentCache map[int64][]byte + storage storer.EncodedObjectStorer + scanner *Scanner + count uint32 + oi []*objectInfo + oiByHash map[plumbing.Hash]*objectInfo + oiByOffset map[int64]*objectInfo + hashOffset map[plumbing.Hash]int64 + pendingRefDeltas map[plumbing.Hash][]*objectInfo + checksum plumbing.Hash + + cache *cache.ObjectLRU + // delta content by offset, only used if source is not seekable + deltas map[int64][]byte ob []Observer } -// NewParser creates a new Parser struct. -func NewParser(scanner *Scanner, ob ...Observer) *Parser { - var contentCache map[int64][]byte +// NewParser creates a new Parser. The Scanner source must be seekable. +// If it's not, NewParserWithStorage should be used instead. +func NewParser(scanner *Scanner, ob ...Observer) (*Parser, error) { + return NewParserWithStorage(scanner, nil, ob...) +} + +// NewParserWithStorage creates a new Parser. The scanner source must either +// be seekable or a storage must be provided. +func NewParserWithStorage( + scanner *Scanner, + storage storer.EncodedObjectStorer, + ob ...Observer, +) (*Parser, error) { + if !scanner.IsSeekable && storage == nil { + return nil, ErrNotSeekableSource + } + + var deltas map[int64][]byte if !scanner.IsSeekable { - contentCache = make(map[int64][]byte) + deltas = make(map[int64][]byte) } return &Parser{ - scanner: scanner, - ob: ob, - count: 0, - cache: cache.NewObjectLRUDefault(), - contentCache: contentCache, + storage: storage, + scanner: scanner, + ob: ob, + count: 0, + cache: cache.NewObjectLRUDefault(), + pendingRefDeltas: make(map[plumbing.Hash][]*objectInfo), + deltas: deltas, + }, nil +} + +func (p *Parser) forEachObserver(f func(o Observer) error) error { + for _, o := range p.ob { + if err := f(o); err != nil { + return err + } } + return nil +} + +func (p *Parser) onHeader(count uint32) error { + return p.forEachObserver(func(o Observer) error { + return o.OnHeader(count) + }) +} + +func (p *Parser) onInflatedObjectHeader( + t plumbing.ObjectType, + objSize int64, + pos int64, +) error { + return p.forEachObserver(func(o Observer) error { + return o.OnInflatedObjectHeader(t, objSize, pos) + }) +} + +func (p *Parser) onInflatedObjectContent( + h plumbing.Hash, + pos int64, + crc uint32, + content []byte, +) error { + return p.forEachObserver(func(o Observer) error { + return o.OnInflatedObjectContent(h, pos, crc, content) + }) +} + +func (p *Parser) onFooter(h plumbing.Hash) error { + return p.forEachObserver(func(o Observer) error { + return o.OnFooter(h) + }) } // Parse start decoding phase of the packfile. @@ -70,7 +136,13 @@ func (p *Parser) Parse() (plumbing.Hash, error) { return plumbing.ZeroHash, err } - if err := p.firstPass(); err != nil { + if err := p.indexObjects(); err != nil { + return plumbing.ZeroHash, err + } + + var err error + p.checksum, err = p.scanner.Checksum() + if err != nil && err != io.EOF { return plumbing.ZeroHash, err } @@ -78,10 +150,12 @@ func (p *Parser) Parse() (plumbing.Hash, error) { return plumbing.ZeroHash, err } - for _, o := range p.ob { - if err := o.OnFooter(p.checksum); err != nil { - return plumbing.ZeroHash, err - } + if len(p.pendingRefDeltas) > 0 { + return plumbing.ZeroHash, ErrReferenceDeltaNotFound + } + + if err := p.onFooter(p.checksum); err != nil { + return plumbing.ZeroHash, err } return p.checksum, nil @@ -93,10 +167,8 @@ func (p *Parser) init() error { return err } - for _, o := range p.ob { - if err := o.OnHeader(c); err != nil { - return err - } + if err := p.onHeader(c); err != nil { + return err } p.count = c @@ -107,7 +179,7 @@ func (p *Parser) init() error { return nil } -func (p *Parser) firstPass() error { +func (p *Parser) indexObjects() error { buf := new(bytes.Buffer) for i := uint32(0); i < p.count; i++ { @@ -121,25 +193,30 @@ func (p *Parser) firstPass() error { delta := false var ota *objectInfo switch t := oh.Type; t { - case plumbing.OFSDeltaObject, plumbing.REFDeltaObject: + case plumbing.OFSDeltaObject: delta = true - var parent *objectInfo - var ok bool - - if t == plumbing.OFSDeltaObject { - parent, ok = p.oiByOffset[oh.OffsetReference] - } else { - parent, ok = p.oiByHash[oh.Reference] - } - + parent, ok := p.oiByOffset[oh.OffsetReference] if !ok { - return ErrReferenceDeltaNotFound + return plumbing.ErrObjectNotFound } ota = newDeltaObject(oh.Offset, oh.Length, t, parent) - parent.Children = append(parent.Children, ota) + case plumbing.REFDeltaObject: + delta = true + + parent, ok := p.oiByHash[oh.Reference] + if ok { + ota = newDeltaObject(oh.Offset, oh.Length, t, parent) + parent.Children = append(parent.Children, ota) + } else { + ota = newBaseObject(oh.Offset, oh.Length, t) + p.pendingRefDeltas[oh.Reference] = append( + p.pendingRefDeltas[oh.Reference], + ota, + ) + } default: ota = newBaseObject(oh.Offset, oh.Length, t) } @@ -153,23 +230,35 @@ func (p *Parser) firstPass() error { ota.PackSize = size ota.Length = oh.Length + data := buf.Bytes() if !delta { - if _, err := ota.Write(buf.Bytes()); err != nil { + if _, err := ota.Write(data); err != nil { return err } ota.SHA1 = ota.Sum() p.oiByHash[ota.SHA1] = ota } - p.oiByOffset[oh.Offset] = ota + if p.storage != nil && !delta { + obj := new(plumbing.MemoryObject) + obj.SetSize(oh.Length) + obj.SetType(oh.Type) + if _, err := obj.Write(data); err != nil { + return err + } - p.oi[i] = ota - } + if _, err := p.storage.SetEncodedObject(obj); err != nil { + return err + } + } - var err error - p.checksum, err = p.scanner.Checksum() - if err != nil && err != io.EOF { - return err + if delta && !p.scanner.IsSeekable { + p.deltas[oh.Offset] = make([]byte, len(data)) + copy(p.deltas[oh.Offset], data) + } + + p.oiByOffset[oh.Offset] = ota + p.oi[i] = ota } return nil @@ -177,21 +266,17 @@ func (p *Parser) firstPass() error { func (p *Parser) resolveDeltas() error { for _, obj := range p.oi { - content, err := obj.Content() + content, err := p.get(obj) if err != nil { return err } - for _, o := range p.ob { - err := o.OnInflatedObjectHeader(obj.Type, obj.Length, obj.Offset) - if err != nil { - return err - } + if err := p.onInflatedObjectHeader(obj.Type, obj.Length, obj.Offset); err != nil { + return err + } - err = o.OnInflatedObjectContent(obj.SHA1, obj.Offset, obj.Crc32, content) - if err != nil { - return err - } + if err := p.onInflatedObjectContent(obj.SHA1, obj.Offset, obj.Crc32, content); err != nil { + return err } if !obj.IsDelta() && len(obj.Children) > 0 { @@ -206,6 +291,11 @@ func (p *Parser) resolveDeltas() error { return err } } + + // Remove the delta from the cache. + if obj.DiskType.IsDelta() && !p.scanner.IsSeekable { + delete(p.deltas, obj.Offset) + } } } @@ -214,7 +304,17 @@ func (p *Parser) resolveDeltas() error { func (p *Parser) get(o *objectInfo) ([]byte, error) { e, ok := p.cache.Get(o.SHA1) - if ok { + // If it's not on the cache and is not a delta we can try to find it in the + // storage, if there's one. + if !ok && p.storage != nil && !o.Type.IsDelta() { + var err error + e, err = p.storage.EncodedObject(plumbing.AnyObject, o.SHA1) + if err != nil { + return nil, err + } + } + + if e != nil { r, err := e.Reader() if err != nil { return nil, err @@ -228,32 +328,23 @@ func (p *Parser) get(o *objectInfo) ([]byte, error) { return buf, nil } - // Read from disk + var data []byte if o.DiskType.IsDelta() { base, err := p.get(o.Parent) if err != nil { return nil, err } - data, err := p.resolveObject(o, base) + data, err = p.resolveObject(o, base) if err != nil { return nil, err } - - if len(o.Children) > 0 { - m := &plumbing.MemoryObject{} - m.Write(data) - m.SetType(o.Type) - m.SetSize(o.Size()) - p.cache.Put(m) + } else { + var err error + data, err = p.readData(o) + if err != nil { + return nil, err } - - return data, nil - } - - data, err := p.readData(o) - if err != nil { - return nil, err } if len(o.Children) > 0 { @@ -285,11 +376,39 @@ func (p *Parser) resolveObject( return nil, err } + if pending, ok := p.pendingRefDeltas[o.SHA1]; ok { + for _, po := range pending { + po.Parent = o + o.Children = append(o.Children, po) + } + delete(p.pendingRefDeltas, o.SHA1) + } + + if p.storage != nil { + obj := new(plumbing.MemoryObject) + obj.SetSize(o.Size()) + obj.SetType(o.Type) + if _, err := obj.Write(data); err != nil { + return nil, err + } + + if _, err := p.storage.SetEncodedObject(obj); err != nil { + return nil, err + } + } + return data, nil } func (p *Parser) readData(o *objectInfo) ([]byte, error) { - buf := new(bytes.Buffer) + if !p.scanner.IsSeekable && o.DiskType.IsDelta() { + data, ok := p.deltas[o.Offset] + if !ok { + return nil, ErrDeltaNotCached + } + + return data, nil + } // TODO: skip header. Header size can be calculated with the offset of the // next offset in the first pass. @@ -301,8 +420,7 @@ func (p *Parser) readData(o *objectInfo) ([]byte, error) { return nil, err } - buf.Reset() - + buf := new(bytes.Buffer) if _, _, err := p.scanner.NextObject(buf); err != nil { return nil, err } @@ -322,6 +440,7 @@ func applyPatchBase(ota *objectInfo, data, base []byte) ([]byte, error) { return nil, err } ota.SHA1 = ota.Sum() + ota.Length = int64(len(patched)) return patched, nil } @@ -341,8 +460,6 @@ type objectInfo struct { Parent *objectInfo Children []*objectInfo SHA1 plumbing.Hash - - content *bytes.Buffer } func newBaseObject(offset, length int64, t plumbing.ObjectType) *objectInfo { @@ -371,30 +488,6 @@ func newDeltaObject( return obj } -func (o *objectInfo) Write(bs []byte) (int, error) { - n, err := o.Hasher.Write(bs) - if err != nil { - return 0, err - } - - o.content = bytes.NewBuffer(nil) - - _, _ = o.content.Write(bs) - return n, nil -} - -// Content returns the content of the object. This operation can only be done -// once. -func (o *objectInfo) Content() ([]byte, error) { - if o.content == nil { - return nil, ErrObjectContentAlreadyRead - } - - r := o.content - o.content = nil - return r.Bytes(), nil -} - func (o *objectInfo) IsDelta() bool { return o.Type.IsDelta() } diff --git a/plumbing/format/packfile/parser_test.go b/plumbing/format/packfile/parser_test.go index b18f20fd7..7bce7379a 100644 --- a/plumbing/format/packfile/parser_test.go +++ b/plumbing/format/packfile/parser_test.go @@ -19,7 +19,8 @@ func (s *ParserSuite) TestParserHashes(c *C) { scanner := packfile.NewScanner(f.Packfile()) obs := new(testObserver) - parser := packfile.NewParser(scanner, obs) + parser, err := packfile.NewParser(scanner, obs) + c.Assert(err, IsNil) ch, err := parser.Parse() c.Assert(err, IsNil) @@ -36,7 +37,7 @@ func (s *ParserSuite) TestParserHashes(c *C) { objs := []observerObject{ {"e8d3ffab552895c19b9fcf7aa264d277cde33881", commit, 254, 12, 0xaa07ba4b}, - {"6ecf0ef2c2dffb796033e5a02219af86ec6584e5", commit, 93, 186, 0xf706df58}, + {"6ecf0ef2c2dffb796033e5a02219af86ec6584e5", commit, 245, 186, 0xf706df58}, {"918c48b83bd081e863dbe1b80f8998f058cd8294", commit, 242, 286, 0x12438846}, {"af2d6a6954d532f8ffb47615169c8fdf9d383a1a", commit, 242, 449, 0x2905a38c}, {"1669dce138d9b841a518c64b10914d88f5e488ea", commit, 333, 615, 0xd9429436}, @@ -54,18 +55,18 @@ func (s *ParserSuite) TestParserHashes(c *C) { {"9a48f23120e880dfbe41f7c9b7b708e9ee62a492", blob, 11488, 80998, 0x7316ff70}, {"9dea2395f5403188298c1dabe8bdafe562c491e3", blob, 78, 84032, 0xdb4fce56}, {"dbd3641b371024f44d0e469a9c8f5457b0660de1", tree, 272, 84115, 0x901cce2c}, - {"a8d315b2b1c615d43042c3a62402b8a54288cf5c", tree, 43, 84375, 0xec4552b0}, + {"a8d315b2b1c615d43042c3a62402b8a54288cf5c", tree, 271, 84375, 0xec4552b0}, {"a39771a7651f97faf5c72e08224d857fc35133db", tree, 38, 84430, 0x847905bf}, {"5a877e6a906a2743ad6e45d99c1793642aaf8eda", tree, 75, 84479, 0x3689459a}, {"586af567d0bb5e771e49bdd9434f5e0fb76d25fa", tree, 38, 84559, 0xe67af94a}, {"cf4aa3b38974fb7d81f367c0830f7d78d65ab86b", tree, 34, 84608, 0xc2314a2e}, {"7e59600739c96546163833214c36459e324bad0a", blob, 9, 84653, 0xcd987848}, - {"fb72698cab7617ac416264415f13224dfd7a165e", tree, 6, 84671, 0x8a853a6d}, - {"4d081c50e250fa32ea8b1313cf8bb7c2ad7627fd", tree, 9, 84688, 0x70c6518}, - {"eba74343e2f15d62adedfd8c883ee0262b5c8021", tree, 6, 84708, 0x4f4108e2}, - {"c2d30fa8ef288618f65f6eed6e168e0d514886f4", tree, 5, 84725, 0xd6fe09e9}, - {"8dcef98b1d52143e1e2dbc458ffe38f925786bf2", tree, 8, 84741, 0xf07a2804}, - {"aa9b383c260e1d05fbbf6b30a02914555e20c725", tree, 4, 84760, 0x1d75d6be}, + {"fb72698cab7617ac416264415f13224dfd7a165e", tree, 238, 84671, 0x8a853a6d}, + {"4d081c50e250fa32ea8b1313cf8bb7c2ad7627fd", tree, 179, 84688, 0x70c6518}, + {"eba74343e2f15d62adedfd8c883ee0262b5c8021", tree, 148, 84708, 0x4f4108e2}, + {"c2d30fa8ef288618f65f6eed6e168e0d514886f4", tree, 110, 84725, 0xd6fe09e9}, + {"8dcef98b1d52143e1e2dbc458ffe38f925786bf2", tree, 111, 84741, 0xf07a2804}, + {"aa9b383c260e1d05fbbf6b30a02914555e20c725", tree, 73, 84760, 0x1d75d6be}, } c.Assert(obs.objects, DeepEquals, objs) diff --git a/storage/filesystem/dotgit/writers.go b/storage/filesystem/dotgit/writers.go index e1ede3cb9..93d2d8cc7 100644 --- a/storage/filesystem/dotgit/writers.go +++ b/storage/filesystem/dotgit/writers.go @@ -57,7 +57,12 @@ func newPackWrite(fs billy.Filesystem) (*PackWriter, error) { func (w *PackWriter) buildIndex() { s := packfile.NewScanner(w.synced) w.writer = new(idxfile.Writer) - w.parser = packfile.NewParser(s, w.writer) + var err error + w.parser, err = packfile.NewParser(s, w.writer) + if err != nil { + w.result <- err + return + } checksum, err := w.parser.Parse() if err != nil { From b3d995f5ca6b544ed8a48fced85ffa94600af302 Mon Sep 17 00:00:00 2001 From: Miguel Molina Date: Thu, 9 Aug 2018 09:23:44 +0200 Subject: [PATCH 050/120] plumbing: packfile, add Parse benchmark Signed-off-by: Miguel Molina --- plumbing/format/packfile/parser_test.go | 30 +++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/plumbing/format/packfile/parser_test.go b/plumbing/format/packfile/parser_test.go index 7bce7379a..b5d482e83 100644 --- a/plumbing/format/packfile/parser_test.go +++ b/plumbing/format/packfile/parser_test.go @@ -1,6 +1,8 @@ package packfile_test import ( + "testing" + "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/format/packfile" @@ -138,3 +140,31 @@ func (t *testObserver) put(pos int64, o observerObject) { t.pos[pos] = len(t.objects) t.objects = append(t.objects, o) } + +func BenchmarkParse(b *testing.B) { + if err := fixtures.Init(); err != nil { + b.Fatal(err) + } + + defer func() { + if err := fixtures.Clean(); err != nil { + b.Fatal(err) + } + }() + + for _, f := range fixtures.ByTag("packfile") { + b.Run(f.URL, func(b *testing.B) { + for i := 0; i < b.N; i++ { + parser, err := packfile.NewParser(packfile.NewScanner(f.Packfile())) + if err != nil { + b.Fatal(err) + } + + _, err = parser.Parse() + if err != nil { + b.Fatal(err) + } + } + }) + } +} From 71a3c9161d4d8d2baf16440a86a02e8f5678aef2 Mon Sep 17 00:00:00 2001 From: Miguel Molina Date: Thu, 9 Aug 2018 10:55:51 +0200 Subject: [PATCH 051/120] plumbing: packfile, read object content only once Signed-off-by: Miguel Molina --- plumbing/format/packfile/parser.go | 22 +++++++++++++++------- plumbing/format/packfile/parser_test.go | 25 +++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 7 deletions(-) diff --git a/plumbing/format/packfile/parser.go b/plumbing/format/packfile/parser.go index beb3e27ac..581c3340b 100644 --- a/plumbing/format/packfile/parser.go +++ b/plumbing/format/packfile/parser.go @@ -280,14 +280,8 @@ func (p *Parser) resolveDeltas() error { } if !obj.IsDelta() && len(obj.Children) > 0 { - var err error - base, err := p.get(obj) - if err != nil { - return err - } - for _, child := range obj.Children { - if _, err := p.resolveObject(child, base); err != nil { + if _, err := p.resolveObject(child, content); err != nil { return err } } @@ -297,12 +291,18 @@ func (p *Parser) resolveDeltas() error { delete(p.deltas, obj.Offset) } } + + obj.Content = nil } return nil } func (p *Parser) get(o *objectInfo) ([]byte, error) { + if len(o.Content) > 0 { + return o.Content, nil + } + e, ok := p.cache.Get(o.SHA1) // If it's not on the cache and is not a delta we can try to find it in the // storage, if there's one. @@ -460,6 +460,8 @@ type objectInfo struct { Parent *objectInfo Children []*objectInfo SHA1 plumbing.Hash + + Content []byte } func newBaseObject(offset, length int64, t plumbing.ObjectType) *objectInfo { @@ -488,6 +490,12 @@ func newDeltaObject( return obj } +func (o *objectInfo) Write(b []byte) (int, error) { + o.Content = make([]byte, len(b)) + copy(o.Content, b) + return o.Hasher.Write(b) +} + func (o *objectInfo) IsDelta() bool { return o.Type.IsDelta() } diff --git a/plumbing/format/packfile/parser_test.go b/plumbing/format/packfile/parser_test.go index b5d482e83..012a1402e 100644 --- a/plumbing/format/packfile/parser_test.go +++ b/plumbing/format/packfile/parser_test.go @@ -168,3 +168,28 @@ func BenchmarkParse(b *testing.B) { }) } } + +func BenchmarkParseBasic(b *testing.B) { + if err := fixtures.Init(); err != nil { + b.Fatal(err) + } + + defer func() { + if err := fixtures.Clean(); err != nil { + b.Fatal(err) + } + }() + + f := fixtures.Basic().One() + for i := 0; i < b.N; i++ { + parser, err := packfile.NewParser(packfile.NewScanner(f.Packfile())) + if err != nil { + b.Fatal(err) + } + + _, err = parser.Parse() + if err != nil { + b.Fatal(err) + } + } +} From 34cc506735ee0cd29362da51592b49a217df8159 Mon Sep 17 00:00:00 2001 From: Miguel Molina Date: Thu, 9 Aug 2018 12:16:57 +0200 Subject: [PATCH 052/120] storage: filesystem, benchmark PackfileIter Signed-off-by: Miguel Molina --- storage/filesystem/object.go | 30 ++++++++++-- storage/filesystem/object_test.go | 79 ++++++++++++++++++++++++++----- 2 files changed, 94 insertions(+), 15 deletions(-) diff --git a/storage/filesystem/object.go b/storage/filesystem/object.go index 2032eac04..4757938c9 100644 --- a/storage/filesystem/object.go +++ b/storage/filesystem/object.go @@ -509,9 +509,20 @@ func (iter *packfileIter) Next() (plumbing.EncodedObject, error) { } } -// ForEach is never called since is used inside of a MultiObjectIterator func (iter *packfileIter) ForEach(cb func(plumbing.EncodedObject) error) error { - return nil + for { + o, err := iter.Next() + if err != nil { + if err == io.EOF { + return nil + } + return err + } + + if err := cb(o); err != nil { + return err + } + } } func (iter *packfileIter) Close() { @@ -543,9 +554,20 @@ func (iter *objectsIter) Next() (plumbing.EncodedObject, error) { return obj, err } -// ForEach is never called since is used inside of a MultiObjectIterator func (iter *objectsIter) ForEach(cb func(plumbing.EncodedObject) error) error { - return nil + for { + o, err := iter.Next() + if err != nil { + if err == io.EOF { + return nil + } + return err + } + + if err := cb(o); err != nil { + return err + } + } } func (iter *objectsIter) Close() { diff --git a/storage/filesystem/object_test.go b/storage/filesystem/object_test.go index ae11c3bdc..0dc19fea3 100644 --- a/storage/filesystem/object_test.go +++ b/storage/filesystem/object_test.go @@ -1,6 +1,8 @@ package filesystem import ( + "testing" + "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit" @@ -10,17 +12,16 @@ import ( type FsSuite struct { fixtures.Suite - Types []plumbing.ObjectType } -var _ = Suite(&FsSuite{ - Types: []plumbing.ObjectType{ - plumbing.CommitObject, - plumbing.TagObject, - plumbing.TreeObject, - plumbing.BlobObject, - }, -}) +var objectTypes = []plumbing.ObjectType{ + plumbing.CommitObject, + plumbing.TagObject, + plumbing.TreeObject, + plumbing.BlobObject, +} + +var _ = Suite(&FsSuite{}) func (s *FsSuite) TestGetFromObjectFile(c *C) { fs := fixtures.ByTag(".git").ByTag("unpacked").One().DotGit() @@ -84,7 +85,7 @@ func (s *FsSuite) TestIter(c *C) { func (s *FsSuite) TestIterWithType(c *C) { fixtures.ByTag(".git").Test(c, func(f *fixtures.Fixture) { - for _, t := range s.Types { + for _, t := range objectTypes { fs := f.DotGit() o, err := NewObjectStorage(dotgit.New(fs)) c.Assert(err, IsNil) @@ -108,7 +109,7 @@ func (s *FsSuite) TestPackfileIter(c *C) { fs := f.DotGit() dg := dotgit.New(fs) - for _, t := range s.Types { + for _, t := range objectTypes { ph, err := dg.ObjectPacks() c.Assert(err, IsNil) @@ -132,3 +133,59 @@ func (s *FsSuite) TestPackfileIter(c *C) { }) } + +func BenchmarkPackfileIter(b *testing.B) { + if err := fixtures.Init(); err != nil { + b.Fatal(err) + } + + defer func() { + if err := fixtures.Clean(); err != nil { + b.Fatal(err) + } + }() + + for _, f := range fixtures.ByTag(".git") { + b.Run(f.URL, func(b *testing.B) { + fs := f.DotGit() + dg := dotgit.New(fs) + + for i := 0; i < b.N; i++ { + for _, t := range objectTypes { + ph, err := dg.ObjectPacks() + if err != nil { + b.Fatal(err) + } + + for _, h := range ph { + f, err := dg.ObjectPack(h) + if err != nil { + b.Fatal(err) + } + + idxf, err := dg.ObjectPackIdx(h) + if err != nil { + b.Fatal(err) + } + + iter, err := NewPackfileIter(f, idxf, t) + if err != nil { + b.Fatal(err) + } + + err = iter.ForEach(func(o plumbing.EncodedObject) error { + if o.Type() != t { + b.Errorf("expecting %s, got %s", t, o.Type()) + } + return nil + }) + + if err != nil { + b.Fatal(err) + } + } + } + } + }) + } +} From 65dc4f9f192cc013e4765fb1162ce6ebda16573d Mon Sep 17 00:00:00 2001 From: Miguel Molina Date: Thu, 9 Aug 2018 12:18:49 +0200 Subject: [PATCH 053/120] plumbing: packfile, rename DiskObject to FSObject Signed-off-by: Miguel Molina --- .../packfile/{disk_object.go => fsobject.go} | 26 +++++++++---------- plumbing/format/packfile/packfile.go | 4 +-- 2 files changed, 15 insertions(+), 15 deletions(-) rename plumbing/format/packfile/{disk_object.go => fsobject.go} (65%) diff --git a/plumbing/format/packfile/disk_object.go b/plumbing/format/packfile/fsobject.go similarity index 65% rename from plumbing/format/packfile/disk_object.go rename to plumbing/format/packfile/fsobject.go index d3e852024..d63127e1b 100644 --- a/plumbing/format/packfile/disk_object.go +++ b/plumbing/format/packfile/fsobject.go @@ -6,8 +6,8 @@ import ( "gopkg.in/src-d/go-git.v4/plumbing" ) -// DiskObject is an object from the packfile on disk. -type DiskObject struct { +// FSObject is an object from the packfile on the filesystem. +type FSObject struct { hash plumbing.Hash h *ObjectHeader offset int64 @@ -16,15 +16,15 @@ type DiskObject struct { packfile *Packfile } -// NewDiskObject creates a new disk object. -func NewDiskObject( +// NewFSObject creates a new filesystem object. +func NewFSObject( hash plumbing.Hash, finalType plumbing.ObjectType, offset int64, contentSize int64, packfile *Packfile, -) *DiskObject { - return &DiskObject{ +) *FSObject { + return &FSObject{ hash: hash, offset: offset, size: contentSize, @@ -34,31 +34,31 @@ func NewDiskObject( } // Reader implements the plumbing.EncodedObject interface. -func (o *DiskObject) Reader() (io.ReadCloser, error) { +func (o *FSObject) Reader() (io.ReadCloser, error) { return o.packfile.getObjectContent(o.offset) } // SetSize implements the plumbing.EncodedObject interface. This method // is a noop. -func (o *DiskObject) SetSize(int64) {} +func (o *FSObject) SetSize(int64) {} // SetType implements the plumbing.EncodedObject interface. This method is // a noop. -func (o *DiskObject) SetType(plumbing.ObjectType) {} +func (o *FSObject) SetType(plumbing.ObjectType) {} // Hash implements the plumbing.EncodedObject interface. -func (o *DiskObject) Hash() plumbing.Hash { return o.hash } +func (o *FSObject) Hash() plumbing.Hash { return o.hash } // Size implements the plumbing.EncodedObject interface. -func (o *DiskObject) Size() int64 { return o.size } +func (o *FSObject) Size() int64 { return o.size } // Type implements the plumbing.EncodedObject interface. -func (o *DiskObject) Type() plumbing.ObjectType { +func (o *FSObject) Type() plumbing.ObjectType { return o.typ } // Writer implements the plumbing.EncodedObject interface. This method always // returns a nil writer. -func (o *DiskObject) Writer() (io.WriteCloser, error) { +func (o *FSObject) Writer() (io.WriteCloser, error) { return nil, nil } diff --git a/plumbing/format/packfile/packfile.go b/plumbing/format/packfile/packfile.go index 37743ba70..df8a3d4a0 100644 --- a/plumbing/format/packfile/packfile.go +++ b/plumbing/format/packfile/packfile.go @@ -232,7 +232,7 @@ func (p *Packfile) nextObject() (plumbing.EncodedObject, error) { p.offsetToType[h.Offset] = typ - return NewDiskObject(hash, typ, h.Offset, size, p), nil + return NewFSObject(hash, typ, h.Offset, size, p), nil } func (p *Packfile) getObjectContent(offset int64) (io.ReadCloser, error) { @@ -410,7 +410,7 @@ func (p *Packfile) Close() error { return closer.Close() } -// MemoryObjectFromDisk converts a DiskObject to a MemoryObject. +// MemoryObjectFromDisk converts a FSObject to a MemoryObject. func MemoryObjectFromDisk(obj plumbing.EncodedObject) (plumbing.EncodedObject, error) { o2 := new(plumbing.MemoryObject) o2.SetType(obj.Type()) From 038cf238e6250094c7aeb387fd7ea92438719699 Mon Sep 17 00:00:00 2001 From: Miguel Molina Date: Thu, 9 Aug 2018 12:36:37 +0200 Subject: [PATCH 054/120] storage: filesystem, close Packfile after iterating objects Signed-off-by: Miguel Molina --- plumbing/object/blob_test.go | 7 +++++++ storage/filesystem/object.go | 11 ++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/plumbing/object/blob_test.go b/plumbing/object/blob_test.go index 181436d5a..e08ff2520 100644 --- a/plumbing/object/blob_test.go +++ b/plumbing/object/blob_test.go @@ -6,6 +6,7 @@ import ( "io/ioutil" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/format/packfile" . "gopkg.in/check.v1" ) @@ -70,6 +71,12 @@ func (s *BlobsSuite) TestBlobIter(c *C) { blobs := []*Blob{} iter.ForEach(func(b *Blob) error { + var err error + b.obj, err = packfile.MemoryObjectFromDisk(b.obj) + if err != nil { + return err + } + blobs = append(blobs, b) return nil }) diff --git a/storage/filesystem/object.go b/storage/filesystem/object.go index 4757938c9..86d0da989 100644 --- a/storage/filesystem/object.go +++ b/storage/filesystem/object.go @@ -457,12 +457,14 @@ func (it *lazyPackfilesIter) Close() { } type packfileIter struct { + pack billy.File iter storer.EncodedObjectIter seen map[plumbing.Hash]struct{} } // NewPackfileIter returns a new EncodedObjectIter for the provided packfile -// and object type. +// and object type. Packfile and index file will be closed after they're +// used. func NewPackfileIter( f billy.File, idxFile billy.File, @@ -473,6 +475,10 @@ func NewPackfileIter( return nil, err } + if err := idxFile.Close(); err != nil { + return nil, err + } + return newPackfileIter(f, t, make(map[plumbing.Hash]struct{}), idx, nil) } @@ -489,6 +495,7 @@ func newPackfileIter( } return &packfileIter{ + pack: f, iter: iter, seen: seen, }, nil @@ -514,6 +521,7 @@ func (iter *packfileIter) ForEach(cb func(plumbing.EncodedObject) error) error { o, err := iter.Next() if err != nil { if err == io.EOF { + iter.Close() return nil } return err @@ -527,6 +535,7 @@ func (iter *packfileIter) ForEach(cb func(plumbing.EncodedObject) error) error { func (iter *packfileIter) Close() { iter.iter.Close() + _ = iter.pack.Close() } type objectsIter struct { From d93b3869f366df7488286614b0205968bc6263df Mon Sep 17 00:00:00 2001 From: Miguel Molina Date: Thu, 9 Aug 2018 13:11:37 +0200 Subject: [PATCH 055/120] storage: filesystem, add PackfileIter benchmark reading object content Signed-off-by: Miguel Molina --- storage/filesystem/object_test.go | 67 +++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/storage/filesystem/object_test.go b/storage/filesystem/object_test.go index 0dc19fea3..88f22bfbe 100644 --- a/storage/filesystem/object_test.go +++ b/storage/filesystem/object_test.go @@ -1,6 +1,7 @@ package filesystem import ( + "io/ioutil" "testing" "gopkg.in/src-d/go-git.v4/plumbing" @@ -189,3 +190,69 @@ func BenchmarkPackfileIter(b *testing.B) { }) } } + +func BenchmarkPackfileIterReadContent(b *testing.B) { + if err := fixtures.Init(); err != nil { + b.Fatal(err) + } + + defer func() { + if err := fixtures.Clean(); err != nil { + b.Fatal(err) + } + }() + + for _, f := range fixtures.ByTag(".git") { + b.Run(f.URL, func(b *testing.B) { + fs := f.DotGit() + dg := dotgit.New(fs) + + for i := 0; i < b.N; i++ { + for _, t := range objectTypes { + ph, err := dg.ObjectPacks() + if err != nil { + b.Fatal(err) + } + + for _, h := range ph { + f, err := dg.ObjectPack(h) + if err != nil { + b.Fatal(err) + } + + idxf, err := dg.ObjectPackIdx(h) + if err != nil { + b.Fatal(err) + } + + iter, err := NewPackfileIter(f, idxf, t) + if err != nil { + b.Fatal(err) + } + + err = iter.ForEach(func(o plumbing.EncodedObject) error { + if o.Type() != t { + b.Errorf("expecting %s, got %s", t, o.Type()) + } + + r, err := o.Reader() + if err != nil { + b.Fatal(err) + } + + if _, err := ioutil.ReadAll(r); err != nil { + b.Fatal(err) + } + + return r.Close() + }) + + if err != nil { + b.Fatal(err) + } + } + } + } + }) + } +} From 56c5e91b158bc4569b38bfd5d27d4b4be5e06a27 Mon Sep 17 00:00:00 2001 From: Miguel Molina Date: Thu, 9 Aug 2018 16:53:00 +0200 Subject: [PATCH 056/120] plumbing: packfile, open and close packfile on FSObject reads Signed-off-by: Miguel Molina --- .../format/packfile/encoder_advanced_test.go | 7 +- plumbing/format/packfile/encoder_test.go | 7 +- plumbing/format/packfile/fsobject.go | 68 ++++++++++++++---- plumbing/format/packfile/packfile.go | 69 +++++++++++-------- plumbing/format/packfile/packfile_test.go | 31 +++++++-- plumbing/object/blob_test.go | 7 -- storage/filesystem/dotgit/dotgit.go | 5 ++ storage/filesystem/object.go | 15 ++-- storage/filesystem/object_test.go | 40 ++++++++++- 9 files changed, 174 insertions(+), 75 deletions(-) diff --git a/plumbing/format/packfile/encoder_advanced_test.go b/plumbing/format/packfile/encoder_advanced_test.go index 78ddc45fe..fc1419eea 100644 --- a/plumbing/format/packfile/encoder_advanced_test.go +++ b/plumbing/format/packfile/encoder_advanced_test.go @@ -6,7 +6,7 @@ import ( "math/rand" "testing" - "gopkg.in/src-d/go-billy.v3/memfs" + "gopkg.in/src-d/go-billy.v4/memfs" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/format/idxfile" . "gopkg.in/src-d/go-git.v4/plumbing/format/packfile" @@ -84,7 +84,8 @@ func (s *EncoderAdvancedSuite) testEncodeDecode( encodeHash, err := enc.Encode(hashes, packWindow) c.Assert(err, IsNil) - f, err := memfs.New().Create("packfile") + fs := memfs.New() + f, err := fs.Create("packfile") c.Assert(err, IsNil) _, err = f.Write(buf.Bytes()) @@ -105,7 +106,7 @@ func (s *EncoderAdvancedSuite) testEncodeDecode( _, err = f.Seek(0, io.SeekStart) c.Assert(err, IsNil) - p := NewPackfile(index, f) + p := NewPackfile(index, fs, f) decodeHash, err := p.ID() c.Assert(err, IsNil) diff --git a/plumbing/format/packfile/encoder_test.go b/plumbing/format/packfile/encoder_test.go index 24e20820c..80b916ded 100644 --- a/plumbing/format/packfile/encoder_test.go +++ b/plumbing/format/packfile/encoder_test.go @@ -5,7 +5,7 @@ import ( "io" stdioutil "io/ioutil" - "gopkg.in/src-d/go-billy.v3/memfs" + "gopkg.in/src-d/go-billy.v4/memfs" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/format/idxfile" "gopkg.in/src-d/go-git.v4/storage/memory" @@ -290,7 +290,8 @@ func objectsEqual(c *C, o1, o2 plumbing.EncodedObject) { } func packfileFromReader(c *C, buf *bytes.Buffer) (*Packfile, func()) { - file, err := memfs.New().Create("packfile") + fs := memfs.New() + file, err := fs.Create("packfile") c.Assert(err, IsNil) _, err = file.Write(buf.Bytes()) @@ -311,7 +312,7 @@ func packfileFromReader(c *C, buf *bytes.Buffer) (*Packfile, func()) { index, err := w.Index() c.Assert(err, IsNil) - return NewPackfile(index, file), func() { + return NewPackfile(index, fs, file), func() { c.Assert(file.Close(), IsNil) } } diff --git a/plumbing/format/packfile/fsobject.go b/plumbing/format/packfile/fsobject.go index d63127e1b..6fd3ca54d 100644 --- a/plumbing/format/packfile/fsobject.go +++ b/plumbing/format/packfile/fsobject.go @@ -3,17 +3,23 @@ package packfile import ( "io" + billy "gopkg.in/src-d/go-billy.v4" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" + "gopkg.in/src-d/go-git.v4/plumbing/format/idxfile" ) // FSObject is an object from the packfile on the filesystem. type FSObject struct { - hash plumbing.Hash - h *ObjectHeader - offset int64 - size int64 - typ plumbing.ObjectType - packfile *Packfile + hash plumbing.Hash + h *ObjectHeader + offset int64 + size int64 + typ plumbing.ObjectType + index idxfile.Index + fs billy.Filesystem + path string + cache cache.Object } // NewFSObject creates a new filesystem object. @@ -22,20 +28,42 @@ func NewFSObject( finalType plumbing.ObjectType, offset int64, contentSize int64, - packfile *Packfile, + index idxfile.Index, + fs billy.Filesystem, + path string, + cache cache.Object, ) *FSObject { return &FSObject{ - hash: hash, - offset: offset, - size: contentSize, - typ: finalType, - packfile: packfile, + hash: hash, + offset: offset, + size: contentSize, + typ: finalType, + index: index, + fs: fs, + path: path, + cache: cache, } } // Reader implements the plumbing.EncodedObject interface. func (o *FSObject) Reader() (io.ReadCloser, error) { - return o.packfile.getObjectContent(o.offset) + f, err := o.fs.Open(o.path) + if err != nil { + return nil, err + } + + p := NewPackfileWithCache(o.index, nil, f, o.cache) + r, err := p.getObjectContent(o.offset) + if err != nil { + _ = f.Close() + return nil, err + } + + if err := f.Close(); err != nil { + return nil, err + } + + return r, nil } // SetSize implements the plumbing.EncodedObject interface. This method @@ -62,3 +90,17 @@ func (o *FSObject) Type() plumbing.ObjectType { func (o *FSObject) Writer() (io.WriteCloser, error) { return nil, nil } + +type objectReader struct { + io.ReadCloser + f billy.File +} + +func (r *objectReader) Close() error { + if err := r.ReadCloser.Close(); err != nil { + _ = r.f.Close() + return err + } + + return r.f.Close() +} diff --git a/plumbing/format/packfile/packfile.go b/plumbing/format/packfile/packfile.go index df8a3d4a0..5feb78142 100644 --- a/plumbing/format/packfile/packfile.go +++ b/plumbing/format/packfile/packfile.go @@ -3,9 +3,9 @@ package packfile import ( "bytes" "io" - stdioutil "io/ioutil" "os" + billy "gopkg.in/src-d/go-billy.v4" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/format/idxfile" @@ -24,21 +24,26 @@ var ( // Packfile allows retrieving information from inside a packfile. type Packfile struct { idxfile.Index - file io.ReadSeeker + fs billy.Filesystem + file billy.File s *Scanner deltaBaseCache cache.Object offsetToType map[int64]plumbing.ObjectType } // NewPackfileWithCache creates a new Packfile with the given object cache. +// If the filesystem is provided, the packfile will return FSObjects, otherwise +// it will return MemoryObjects. func NewPackfileWithCache( index idxfile.Index, - file io.ReadSeeker, + fs billy.Filesystem, + file billy.File, cache cache.Object, ) *Packfile { s := NewScanner(file) return &Packfile{ index, + fs, file, s, cache, @@ -48,8 +53,10 @@ func NewPackfileWithCache( // NewPackfile returns a packfile representation for the given packfile file // and packfile idx. -func NewPackfile(index idxfile.Index, file io.ReadSeeker) *Packfile { - return NewPackfileWithCache(index, file, cache.NewObjectLRUDefault()) +// If the filesystem is provided, the packfile will return FSObjects, otherwise +// it will return MemoryObjects. +func NewPackfile(index idxfile.Index, fs billy.Filesystem, file billy.File) *Packfile { + return NewPackfileWithCache(index, fs, file, cache.NewObjectLRUDefault()) } // Get retrieves the encoded object in the packfile with the given hash. @@ -215,6 +222,12 @@ func (p *Packfile) nextObject() (plumbing.EncodedObject, error) { return nil, err } + // If we have no filesystem, we will return a MemoryObject instead + // of an FSObject. + if p.fs == nil { + return p.getNextObject(h) + } + hash, err := p.FindHash(h.Offset) if err != nil { return nil, err @@ -232,7 +245,16 @@ func (p *Packfile) nextObject() (plumbing.EncodedObject, error) { p.offsetToType[h.Offset] = typ - return NewFSObject(hash, typ, h.Offset, size, p), nil + return NewFSObject( + hash, + typ, + h.Offset, + size, + p.Index, + p.fs, + p.file.Name(), + p.deltaBaseCache, + ), nil } func (p *Packfile) getObjectContent(offset int64) (io.ReadCloser, error) { @@ -245,10 +267,20 @@ func (p *Packfile) getObjectContent(offset int64) (io.ReadCloser, error) { return nil, err } + obj, err := p.getNextObject(h) + if err != nil { + return nil, err + } + + return obj.Reader() +} + +func (p *Packfile) getNextObject(h *ObjectHeader) (plumbing.EncodedObject, error) { var obj = new(plumbing.MemoryObject) obj.SetSize(h.Length) obj.SetType(h.Type) + var err error switch h.Type { case plumbing.CommitObject, plumbing.TreeObject, plumbing.BlobObject, plumbing.TagObject: err = p.fillRegularObjectContent(obj) @@ -264,7 +296,7 @@ func (p *Packfile) getObjectContent(offset int64) (io.ReadCloser, error) { return nil, err } - return obj.Reader() + return obj, nil } func (p *Packfile) fillRegularObjectContent(obj plumbing.EncodedObject) error { @@ -410,29 +442,6 @@ func (p *Packfile) Close() error { return closer.Close() } -// MemoryObjectFromDisk converts a FSObject to a MemoryObject. -func MemoryObjectFromDisk(obj plumbing.EncodedObject) (plumbing.EncodedObject, error) { - o2 := new(plumbing.MemoryObject) - o2.SetType(obj.Type()) - o2.SetSize(obj.Size()) - - r, err := obj.Reader() - if err != nil { - return nil, err - } - - data, err := stdioutil.ReadAll(r) - if err != nil { - return nil, err - } - - if _, err := o2.Write(data); err != nil { - return nil, err - } - - return o2, nil -} - type objectIter struct { p *Packfile typ plumbing.ObjectType diff --git a/plumbing/format/packfile/packfile_test.go b/plumbing/format/packfile/packfile_test.go index 3193bed04..05dc8a7ac 100644 --- a/plumbing/format/packfile/packfile_test.go +++ b/plumbing/format/packfile/packfile_test.go @@ -109,13 +109,14 @@ var expectedEntries = map[plumbing.Hash]int64{ func (s *PackfileSuite) SetUpTest(c *C) { s.f = fixtures.Basic().One() - f, err := osfs.New("").Open(s.f.Packfile().Name()) + fs := osfs.New("") + f, err := fs.Open(s.f.Packfile().Name()) c.Assert(err, IsNil) s.idx = idxfile.NewMemoryIndex() c.Assert(idxfile.NewDecoder(s.f.Idx()).Decode(s.idx), IsNil) - s.p = packfile.NewPackfile(s.idx, f) + s.p = packfile.NewPackfile(s.idx, fs, f) } func (s *PackfileSuite) TearDownTest(c *C) { @@ -125,7 +126,11 @@ func (s *PackfileSuite) TearDownTest(c *C) { func (s *PackfileSuite) TestDecode(c *C) { fixtures.Basic().ByTag("packfile").Test(c, func(f *fixtures.Fixture) { index := getIndexFromIdxFile(f.Idx()) - p := packfile.NewPackfile(index, f.Packfile()) + fs := osfs.New("") + pf, err := fs.Open(f.Packfile().Name()) + c.Assert(err, IsNil) + + p := packfile.NewPackfile(index, fs, pf) defer p.Close() for _, h := range expectedHashes { @@ -140,7 +145,11 @@ func (s *PackfileSuite) TestDecodeByTypeRefDelta(c *C) { f := fixtures.Basic().ByTag("ref-delta").One() index := getIndexFromIdxFile(f.Idx()) - packfile := packfile.NewPackfile(index, f.Packfile()) + fs := osfs.New("") + pf, err := fs.Open(f.Packfile().Name()) + c.Assert(err, IsNil) + + packfile := packfile.NewPackfile(index, fs, pf) defer packfile.Close() iter, err := packfile.GetByType(plumbing.CommitObject) @@ -171,7 +180,11 @@ func (s *PackfileSuite) TestDecodeByType(c *C) { fixtures.Basic().ByTag("packfile").Test(c, func(f *fixtures.Fixture) { for _, t := range ts { index := getIndexFromIdxFile(f.Idx()) - packfile := packfile.NewPackfile(index, f.Packfile()) + fs := osfs.New("") + pf, err := fs.Open(f.Packfile().Name()) + c.Assert(err, IsNil) + + packfile := packfile.NewPackfile(index, fs, pf) defer packfile.Close() iter, err := packfile.GetByType(t) @@ -188,10 +201,14 @@ func (s *PackfileSuite) TestDecodeByType(c *C) { func (s *PackfileSuite) TestDecodeByTypeConstructor(c *C) { f := fixtures.Basic().ByTag("packfile").One() index := getIndexFromIdxFile(f.Idx()) - packfile := packfile.NewPackfile(index, f.Packfile()) + fs := osfs.New("") + pf, err := fs.Open(f.Packfile().Name()) + c.Assert(err, IsNil) + + packfile := packfile.NewPackfile(index, fs, pf) defer packfile.Close() - _, err := packfile.GetByType(plumbing.OFSDeltaObject) + _, err = packfile.GetByType(plumbing.OFSDeltaObject) c.Assert(err, Equals, plumbing.ErrInvalidType) _, err = packfile.GetByType(plumbing.REFDeltaObject) diff --git a/plumbing/object/blob_test.go b/plumbing/object/blob_test.go index e08ff2520..181436d5a 100644 --- a/plumbing/object/blob_test.go +++ b/plumbing/object/blob_test.go @@ -6,7 +6,6 @@ import ( "io/ioutil" "gopkg.in/src-d/go-git.v4/plumbing" - "gopkg.in/src-d/go-git.v4/plumbing/format/packfile" . "gopkg.in/check.v1" ) @@ -71,12 +70,6 @@ func (s *BlobsSuite) TestBlobIter(c *C) { blobs := []*Blob{} iter.ForEach(func(b *Blob) error { - var err error - b.obj, err = packfile.MemoryObjectFromDisk(b.obj) - if err != nil { - return err - } - blobs = append(blobs, b) return nil }) diff --git a/storage/filesystem/dotgit/dotgit.go b/storage/filesystem/dotgit/dotgit.go index dc12f23cf..af07eb527 100644 --- a/storage/filesystem/dotgit/dotgit.go +++ b/storage/filesystem/dotgit/dotgit.go @@ -784,6 +784,11 @@ func (d *DotGit) Alternates() ([]*DotGit, error) { return alternates, nil } +// Fs returns the underlying filesystem of the DotGit folder. +func (d *DotGit) Fs() billy.Filesystem { + return d.fs +} + func isHex(s string) bool { for _, b := range []byte(s) { if isNum(b) { diff --git a/storage/filesystem/object.go b/storage/filesystem/object.go index 86d0da989..6958e3217 100644 --- a/storage/filesystem/object.go +++ b/storage/filesystem/object.go @@ -295,12 +295,7 @@ func (s *ObjectStorage) decodeObjectAt( return nil, err } - obj, err := packfile.NewPackfile(idx, f).GetByOffset(offset) - if err != nil { - return nil, err - } - - return packfile.MemoryObjectFromDisk(obj) + return packfile.NewPackfile(idx, s.dir.Fs(), f).GetByOffset(offset) } func (s *ObjectStorage) decodeDeltaObjectAt( @@ -404,7 +399,7 @@ func (s *ObjectStorage) buildPackfileIters(t plumbing.ObjectType, seen map[plumb if err != nil { return nil, err } - return newPackfileIter(pack, t, seen, s.index[h], s.deltaBaseCache) + return newPackfileIter(s.dir.Fs(), pack, t, seen, s.index[h], s.deltaBaseCache) }, }, nil } @@ -466,6 +461,7 @@ type packfileIter struct { // and object type. Packfile and index file will be closed after they're // used. func NewPackfileIter( + fs billy.Filesystem, f billy.File, idxFile billy.File, t plumbing.ObjectType, @@ -479,17 +475,18 @@ func NewPackfileIter( return nil, err } - return newPackfileIter(f, t, make(map[plumbing.Hash]struct{}), idx, nil) + return newPackfileIter(fs, f, t, make(map[plumbing.Hash]struct{}), idx, nil) } func newPackfileIter( + fs billy.Filesystem, f billy.File, t plumbing.ObjectType, seen map[plumbing.Hash]struct{}, index idxfile.Index, cache cache.Object, ) (storer.EncodedObjectIter, error) { - iter, err := packfile.NewPackfile(index, f).GetByType(t) + iter, err := packfile.NewPackfile(index, fs, f).GetByType(t) if err != nil { return nil, err } diff --git a/storage/filesystem/object_test.go b/storage/filesystem/object_test.go index 88f22bfbe..b1408b7c2 100644 --- a/storage/filesystem/object_test.go +++ b/storage/filesystem/object_test.go @@ -121,7 +121,7 @@ func (s *FsSuite) TestPackfileIter(c *C) { idxf, err := dg.ObjectPackIdx(h) c.Assert(err, IsNil) - iter, err := NewPackfileIter(f, idxf, t) + iter, err := NewPackfileIter(fs, f, idxf, t) c.Assert(err, IsNil) err = iter.ForEach(func(o plumbing.EncodedObject) error { c.Assert(o.Type(), Equals, t) @@ -169,7 +169,7 @@ func BenchmarkPackfileIter(b *testing.B) { b.Fatal(err) } - iter, err := NewPackfileIter(f, idxf, t) + iter, err := NewPackfileIter(fs, f, idxf, t) if err != nil { b.Fatal(err) } @@ -225,7 +225,7 @@ func BenchmarkPackfileIterReadContent(b *testing.B) { b.Fatal(err) } - iter, err := NewPackfileIter(f, idxf, t) + iter, err := NewPackfileIter(fs, f, idxf, t) if err != nil { b.Fatal(err) } @@ -256,3 +256,37 @@ func BenchmarkPackfileIterReadContent(b *testing.B) { }) } } + +func BenchmarkGetObjectFromPackfile(b *testing.B) { + if err := fixtures.Init(); err != nil { + b.Fatal(err) + } + + defer func() { + if err := fixtures.Clean(); err != nil { + b.Fatal(err) + } + }() + + for _, f := range fixtures.Basic() { + b.Run(f.URL, func(b *testing.B) { + fs := f.DotGit() + o, err := NewObjectStorage(dotgit.New(fs)) + if err != nil { + b.Fatal(err) + } + + for i := 0; i < b.N; i++ { + expected := plumbing.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5") + obj, err := o.EncodedObject(plumbing.AnyObject, expected) + if err != nil { + b.Fatal(err) + } + + if obj.Hash() != expected { + b.Errorf("expecting %s, got %s", expected, obj.Hash()) + } + } + }) + } +} From b944bc45af20b7362786f014fba1bbd72ba7fc76 Mon Sep 17 00:00:00 2001 From: Miguel Molina Date: Fri, 10 Aug 2018 10:37:26 +0200 Subject: [PATCH 057/120] git: add benchmark for iterating repository objects Signed-off-by: Miguel Molina --- repository_test.go | 56 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/repository_test.go b/repository_test.go index b78fbb70b..e34627e0f 100644 --- a/repository_test.go +++ b/repository_test.go @@ -1688,3 +1688,59 @@ func (s *RepositorySuite) TestBrokenMultipleShallowFetch(c *C) { }) c.Assert(err, IsNil) } + +func BenchmarkObjects(b *testing.B) { + if err := fixtures.Init(); err != nil { + b.Fatal(err) + } + + defer func() { + if err := fixtures.Clean(); err != nil { + b.Fatal(err) + } + }() + + for _, f := range fixtures.ByTag("packfile") { + if f.DotGitHash == plumbing.ZeroHash { + continue + } + + b.Run(f.URL, func(b *testing.B) { + fs := f.DotGit() + storer, err := filesystem.NewStorage(fs) + if err != nil { + b.Fatal(err) + } + + worktree, err := fs.Chroot(filepath.Dir(fs.Root())) + if err != nil { + b.Fatal(err) + } + + repo, err := Open(storer, worktree) + if err != nil { + b.Fatal(err) + } + + for i := 0; i < b.N; i++ { + iter, err := repo.Objects() + if err != nil { + b.Fatal(err) + } + + for { + _, err := iter.Next() + if err == io.EOF { + break + } + + if err != nil { + b.Fatal(err) + } + } + + iter.Close() + } + }) + } +} From 8d75d239e93474e4287870e4e5143da14e2c360d Mon Sep 17 00:00:00 2001 From: Miguel Molina Date: Fri, 10 Aug 2018 12:33:56 +0200 Subject: [PATCH 058/120] plumbing: idxfile, Crc32 to CRC32 and return ok from findHashIndex Signed-off-by: Miguel Molina --- plumbing/format/idxfile/decoder.go | 4 ++-- plumbing/format/idxfile/encoder.go | 2 +- plumbing/format/idxfile/idxfile.go | 36 +++++++++++++++--------------- plumbing/format/idxfile/writer.go | 4 ++-- 4 files changed, 23 insertions(+), 23 deletions(-) diff --git a/plumbing/format/idxfile/decoder.go b/plumbing/format/idxfile/decoder.go index 25ff88e03..5b927826a 100644 --- a/plumbing/format/idxfile/decoder.go +++ b/plumbing/format/idxfile/decoder.go @@ -124,7 +124,7 @@ func readObjectNames(idx *MemoryIndex, r io.Reader) error { idx.Names = append(idx.Names, bin) idx.Offset32 = append(idx.Offset32, make([]byte, buckets*4)) - idx.Crc32 = append(idx.Crc32, make([]byte, buckets*4)) + idx.CRC32 = append(idx.CRC32, make([]byte, buckets*4)) } return nil @@ -133,7 +133,7 @@ func readObjectNames(idx *MemoryIndex, r io.Reader) error { func readCRC32(idx *MemoryIndex, r io.Reader) error { for k := 0; k < fanout; k++ { if pos := idx.FanoutMapping[k]; pos != noMapping { - if _, err := io.ReadFull(r, idx.Crc32[pos]); err != nil { + if _, err := io.ReadFull(r, idx.CRC32[pos]); err != nil { return err } } diff --git a/plumbing/format/idxfile/encoder.go b/plumbing/format/idxfile/encoder.go index 55df4667f..e47951102 100644 --- a/plumbing/format/idxfile/encoder.go +++ b/plumbing/format/idxfile/encoder.go @@ -89,7 +89,7 @@ func (e *Encoder) encodeCRC32(idx *MemoryIndex) (int, error) { continue } - n, err := e.Write(idx.Crc32[pos]) + n, err := e.Write(idx.CRC32[pos]) if err != nil { return size, err } diff --git a/plumbing/format/idxfile/idxfile.go b/plumbing/format/idxfile/idxfile.go index 71c763015..c977beedb 100644 --- a/plumbing/format/idxfile/idxfile.go +++ b/plumbing/format/idxfile/idxfile.go @@ -41,12 +41,12 @@ type MemoryIndex struct { Version uint32 Fanout [256]uint32 // FanoutMapping maps the position in the fanout table to the position - // in the Names, Offset32 and Crc32 slices. This improves the memory + // in the Names, Offset32 and CRC32 slices. This improves the memory // usage by not needing an array with unnecessary empty slots. FanoutMapping [256]int Names [][]byte Offset32 [][]byte - Crc32 [][]byte + CRC32 [][]byte Offset64 []byte PackfileChecksum [20]byte IdxChecksum [20]byte @@ -61,20 +61,20 @@ func NewMemoryIndex() *MemoryIndex { return &MemoryIndex{} } -func (idx *MemoryIndex) findHashIndex(h plumbing.Hash) int { +func (idx *MemoryIndex) findHashIndex(h plumbing.Hash) (int, bool) { k := idx.FanoutMapping[h[0]] if k == noMapping { - return -1 + return 0, false } if len(idx.Names) <= k { - return -1 + return 0, false } data := idx.Names[k] high := uint64(len(idx.Offset32[k])) >> 2 if high == 0 { - return -1 + return 0, false } low := uint64(0) @@ -86,7 +86,7 @@ func (idx *MemoryIndex) findHashIndex(h plumbing.Hash) int { if cmp < 0 { high = mid } else if cmp == 0 { - return int(mid) + return int(mid), true } else { low = mid + 1 } @@ -96,13 +96,13 @@ func (idx *MemoryIndex) findHashIndex(h plumbing.Hash) int { } } - return -1 + return 0, false } // Contains implements the Index interface. func (idx *MemoryIndex) Contains(h plumbing.Hash) (bool, error) { - i := idx.findHashIndex(h) - return i >= 0, nil + _, ok := idx.findHashIndex(h) + return ok, nil } // FindOffset implements the Index interface. @@ -112,8 +112,8 @@ func (idx *MemoryIndex) FindOffset(h plumbing.Hash) (int64, error) { } k := idx.FanoutMapping[h[0]] - i := idx.findHashIndex(h) - if i < 0 { + i, ok := idx.findHashIndex(h) + if !ok { return 0, plumbing.ErrObjectNotFound } @@ -147,17 +147,17 @@ func (idx *MemoryIndex) getOffset(firstLevel, secondLevel int) (int64, error) { // FindCRC32 implements the Index interface. func (idx *MemoryIndex) FindCRC32(h plumbing.Hash) (uint32, error) { k := idx.FanoutMapping[h[0]] - i := idx.findHashIndex(h) - if i < 0 { + i, ok := idx.findHashIndex(h) + if !ok { return 0, plumbing.ErrObjectNotFound } - return idx.getCrc32(k, i) + return idx.getCRC32(k, i) } -func (idx *MemoryIndex) getCrc32(firstLevel, secondLevel int) (uint32, error) { +func (idx *MemoryIndex) getCRC32(firstLevel, secondLevel int) (uint32, error) { offset := secondLevel << 2 - buf := bytes.NewBuffer(idx.Crc32[firstLevel][offset : offset+4]) + buf := bytes.NewBuffer(idx.CRC32[firstLevel][offset : offset+4]) return binary.ReadUint32(buf) } @@ -253,7 +253,7 @@ func (i *idxfileEntryIter) Next() (*Entry, error) { } entry.Offset = uint64(offset) - entry.CRC32, err = i.idx.getCrc32(pos, i.secondLevel) + entry.CRC32, err = i.idx.getCRC32(pos, i.secondLevel) if err != nil { return nil, err } diff --git a/plumbing/format/idxfile/writer.go b/plumbing/format/idxfile/writer.go index 89b79cd1d..aa919e783 100644 --- a/plumbing/format/idxfile/writer.go +++ b/plumbing/format/idxfile/writer.go @@ -132,7 +132,7 @@ func (w *Writer) createIndex() (*MemoryIndex, error) { idx.Names = append(idx.Names, make([]byte, 0)) idx.Offset32 = append(idx.Offset32, make([]byte, 0)) - idx.Crc32 = append(idx.Crc32, make([]byte, 0)) + idx.CRC32 = append(idx.CRC32, make([]byte, 0)) } idx.Names[bucket] = append(idx.Names[bucket], o.Hash[:]...) @@ -148,7 +148,7 @@ func (w *Writer) createIndex() (*MemoryIndex, error) { buf.Truncate(0) binary.WriteUint32(buf, uint32(o.CRC32)) - idx.Crc32[bucket] = append(idx.Crc32[bucket], buf.Bytes()...) + idx.CRC32[bucket] = append(idx.CRC32[bucket], buf.Bytes()...) } for j := last + 1; j < 256; j++ { From a8c4426d204f42e683e902dcb277494004d5e59d Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Tue, 14 Aug 2018 11:59:11 +0200 Subject: [PATCH 059/120] plumbing: add buffer cache and use it in packfile parser It uses less memory and is faster as slices don't have to be converted from/to MemoryObject and they are indexed by offset. Signed-off-by: Javi Fontan --- plumbing/cache/buffer_lru.go | 98 ++++++++++++++++++++++ plumbing/cache/buffer_test.go | 128 +++++++++++++++++++++++++++++ plumbing/cache/common.go | 13 +++ plumbing/format/packfile/parser.go | 24 +++--- 4 files changed, 249 insertions(+), 14 deletions(-) create mode 100644 plumbing/cache/buffer_lru.go create mode 100644 plumbing/cache/buffer_test.go diff --git a/plumbing/cache/buffer_lru.go b/plumbing/cache/buffer_lru.go new file mode 100644 index 000000000..f2c0f907f --- /dev/null +++ b/plumbing/cache/buffer_lru.go @@ -0,0 +1,98 @@ +package cache + +import ( + "container/list" + "sync" +) + +// BufferLRU implements an object cache with an LRU eviction policy and a +// maximum size (measured in object size). +type BufferLRU struct { + MaxSize FileSize + + actualSize FileSize + ll *list.List + cache map[int64]*list.Element + mut sync.Mutex +} + +// NewBufferLRU creates a new BufferLRU with the given maximum size. The maximum +// size will never be exceeded. +func NewBufferLRU(maxSize FileSize) *BufferLRU { + return &BufferLRU{MaxSize: maxSize} +} + +// NewBufferLRUDefault creates a new BufferLRU with the default cache size. +func NewBufferLRUDefault() *BufferLRU { + return &BufferLRU{MaxSize: DefaultMaxSize} +} + +type buffer struct { + Key int64 + Slice []byte +} + +// Put puts a buffer into the cache. If the buffer is already in the cache, it +// will be marked as used. Otherwise, it will be inserted. A buffers might +// be evicted to make room for the new one. +func (c *BufferLRU) Put(key int64, slice []byte) { + c.mut.Lock() + defer c.mut.Unlock() + + if c.cache == nil { + c.actualSize = 0 + c.cache = make(map[int64]*list.Element, 1000) + c.ll = list.New() + } + + if ee, ok := c.cache[key]; ok { + c.ll.MoveToFront(ee) + ee.Value = buffer{key, slice} + return + } + + objSize := FileSize(len(slice)) + + if objSize > c.MaxSize { + return + } + + for c.actualSize+objSize > c.MaxSize { + last := c.ll.Back() + lastObj := last.Value.(buffer) + lastSize := FileSize(len(lastObj.Slice)) + + c.ll.Remove(last) + delete(c.cache, lastObj.Key) + c.actualSize -= lastSize + } + + ee := c.ll.PushFront(buffer{key, slice}) + c.cache[key] = ee + c.actualSize += objSize +} + +// Get returns a buffer by its key. It marks the buffer as used. If the buffer +// is not in the cache, (nil, false) will be returned. +func (c *BufferLRU) Get(key int64) ([]byte, bool) { + c.mut.Lock() + defer c.mut.Unlock() + + ee, ok := c.cache[key] + if !ok { + return nil, false + } + + c.ll.MoveToFront(ee) + return ee.Value.(buffer).Slice, true +} + +// Clear the content of this buffer cache. +func (c *BufferLRU) Clear() { + c.mut.Lock() + defer c.mut.Unlock() + + c.ll = nil + c.cache = nil + c.actualSize = 0 +} diff --git a/plumbing/cache/buffer_test.go b/plumbing/cache/buffer_test.go new file mode 100644 index 000000000..262138a62 --- /dev/null +++ b/plumbing/cache/buffer_test.go @@ -0,0 +1,128 @@ +package cache + +import ( + "sync" + + . "gopkg.in/check.v1" +) + +type BufferSuite struct { + c map[string]Buffer + aBuffer []byte + bBuffer []byte + cBuffer []byte + dBuffer []byte + eBuffer []byte +} + +var _ = Suite(&BufferSuite{}) + +func (s *BufferSuite) SetUpTest(c *C) { + s.aBuffer = []byte("a") + s.bBuffer = []byte("bbb") + s.cBuffer = []byte("c") + s.dBuffer = []byte("d") + s.eBuffer = []byte("ee") + + s.c = make(map[string]Buffer) + s.c["two_bytes"] = NewBufferLRU(2 * Byte) + s.c["default_lru"] = NewBufferLRUDefault() +} + +func (s *BufferSuite) TestPutSameBuffer(c *C) { + for _, o := range s.c { + o.Put(1, s.aBuffer) + o.Put(1, s.aBuffer) + _, ok := o.Get(1) + c.Assert(ok, Equals, true) + } +} + +func (s *BufferSuite) TestPutBigBuffer(c *C) { + for _, o := range s.c { + o.Put(1, s.bBuffer) + _, ok := o.Get(2) + c.Assert(ok, Equals, false) + } +} + +func (s *BufferSuite) TestPutCacheOverflow(c *C) { + // this test only works with an specific size + o := s.c["two_bytes"] + + o.Put(1, s.aBuffer) + o.Put(2, s.cBuffer) + o.Put(3, s.dBuffer) + + obj, ok := o.Get(1) + c.Assert(ok, Equals, false) + c.Assert(obj, IsNil) + obj, ok = o.Get(2) + c.Assert(ok, Equals, true) + c.Assert(obj, NotNil) + obj, ok = o.Get(3) + c.Assert(ok, Equals, true) + c.Assert(obj, NotNil) +} + +func (s *BufferSuite) TestEvictMultipleBuffers(c *C) { + o := s.c["two_bytes"] + + o.Put(1, s.cBuffer) + o.Put(2, s.dBuffer) // now cache is full with two objects + o.Put(3, s.eBuffer) // this put should evict all previous objects + + obj, ok := o.Get(1) + c.Assert(ok, Equals, false) + c.Assert(obj, IsNil) + obj, ok = o.Get(2) + c.Assert(ok, Equals, false) + c.Assert(obj, IsNil) + obj, ok = o.Get(3) + c.Assert(ok, Equals, true) + c.Assert(obj, NotNil) +} + +func (s *BufferSuite) TestClear(c *C) { + for _, o := range s.c { + o.Put(1, s.aBuffer) + o.Clear() + obj, ok := o.Get(1) + c.Assert(ok, Equals, false) + c.Assert(obj, IsNil) + } +} + +func (s *BufferSuite) TestConcurrentAccess(c *C) { + for _, o := range s.c { + var wg sync.WaitGroup + + for i := 0; i < 1000; i++ { + wg.Add(3) + go func(i int) { + o.Put(int64(i), []byte{00}) + wg.Done() + }(i) + + go func(i int) { + if i%30 == 0 { + o.Clear() + } + wg.Done() + }(i) + + go func(i int) { + o.Get(int64(i)) + wg.Done() + }(i) + } + + wg.Wait() + } +} + +func (s *BufferSuite) TestDefaultLRU(c *C) { + defaultLRU := s.c["default_lru"].(*BufferLRU) + + c.Assert(defaultLRU.MaxSize, Equals, DefaultMaxSize) +} diff --git a/plumbing/cache/common.go b/plumbing/cache/common.go index e77baf01c..2b7f36a56 100644 --- a/plumbing/cache/common.go +++ b/plumbing/cache/common.go @@ -24,3 +24,16 @@ type Object interface { // Clear clears every object from the cache. Clear() } + +// Buffer is an interface to a buffer cache. +type Buffer interface { + // Put puts a buffer into the cache. If the buffer is already in the cache, + // it will be marked as used. Otherwise, it will be inserted. Buffer might + // be evicted to make room for the new one. + Put(key int64, slice []byte) + // Get returns a buffer by its key. It marks the buffer as used. If the + // buffer is not in the cache, (nil, false) will be returned. + Get(key int64) ([]byte, bool) + // Clear clears every object from the cache. + Clear() +} diff --git a/plumbing/format/packfile/parser.go b/plumbing/format/packfile/parser.go index 581c3340b..88f33dc36 100644 --- a/plumbing/format/packfile/parser.go +++ b/plumbing/format/packfile/parser.go @@ -48,7 +48,7 @@ type Parser struct { pendingRefDeltas map[plumbing.Hash][]*objectInfo checksum plumbing.Hash - cache *cache.ObjectLRU + cache *cache.BufferLRU // delta content by offset, only used if source is not seekable deltas map[int64][]byte @@ -82,7 +82,7 @@ func NewParserWithStorage( scanner: scanner, ob: ob, count: 0, - cache: cache.NewObjectLRUDefault(), + cache: cache.NewBufferLRUDefault(), pendingRefDeltas: make(map[plumbing.Hash][]*objectInfo), deltas: deltas, }, nil @@ -303,29 +303,29 @@ func (p *Parser) get(o *objectInfo) ([]byte, error) { return o.Content, nil } - e, ok := p.cache.Get(o.SHA1) + b, ok := p.cache.Get(o.Offset) // If it's not on the cache and is not a delta we can try to find it in the // storage, if there's one. if !ok && p.storage != nil && !o.Type.IsDelta() { var err error - e, err = p.storage.EncodedObject(plumbing.AnyObject, o.SHA1) + e, err := p.storage.EncodedObject(plumbing.AnyObject, o.SHA1) if err != nil { return nil, err } - } - if e != nil { r, err := e.Reader() if err != nil { return nil, err } - buf := make([]byte, e.Size()) - if _, err = r.Read(buf); err != nil { + b = make([]byte, e.Size()) + if _, err = r.Read(b); err != nil { return nil, err } + } - return buf, nil + if b != nil { + return b, nil } var data []byte @@ -348,11 +348,7 @@ func (p *Parser) get(o *objectInfo) ([]byte, error) { } if len(o.Children) > 0 { - m := &plumbing.MemoryObject{} - m.Write(data) - m.SetType(o.Type) - m.SetSize(o.Size()) - p.cache.Put(m) + p.cache.Put(o.Offset, data) } return data, nil From 555a6ca02e88279cef421df88a108c2955fcde77 Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Tue, 14 Aug 2018 12:21:12 +0200 Subject: [PATCH 060/120] plumbing/pacfile: tidy up objectInfo struct * a new hasher is created when needed * delete unused fields * base content is no longer kept in memory Signed-off-by: Javi Fontan --- plumbing/format/packfile/parser.go | 58 ++++++++++++------------------ 1 file changed, 22 insertions(+), 36 deletions(-) diff --git a/plumbing/format/packfile/parser.go b/plumbing/format/packfile/parser.go index 88f33dc36..3a9c4d75b 100644 --- a/plumbing/format/packfile/parser.go +++ b/plumbing/format/packfile/parser.go @@ -221,21 +221,22 @@ func (p *Parser) indexObjects() error { ota = newBaseObject(oh.Offset, oh.Length, t) } - size, crc, err := p.scanner.NextObject(buf) + _, crc, err := p.scanner.NextObject(buf) if err != nil { return err } ota.Crc32 = crc - ota.PackSize = size ota.Length = oh.Length data := buf.Bytes() if !delta { - if _, err := ota.Write(data); err != nil { + sha1, err := getSHA1(ota.Type, data) + if err != nil { return err } - ota.SHA1 = ota.Sum() + + ota.SHA1 = sha1 p.oiByHash[ota.SHA1] = ota } @@ -291,18 +292,12 @@ func (p *Parser) resolveDeltas() error { delete(p.deltas, obj.Offset) } } - - obj.Content = nil } return nil } func (p *Parser) get(o *objectInfo) ([]byte, error) { - if len(o.Content) > 0 { - return o.Content, nil - } - b, ok := p.cache.Get(o.Offset) // If it's not on the cache and is not a delta we can try to find it in the // storage, if there's one. @@ -406,8 +401,6 @@ func (p *Parser) readData(o *objectInfo) ([]byte, error) { return data, nil } - // TODO: skip header. Header size can be calculated with the offset of the - // next offset in the first pass. if _, err := p.scanner.SeekFromStart(o.Offset); err != nil { return nil, err } @@ -431,33 +424,37 @@ func applyPatchBase(ota *objectInfo, data, base []byte) ([]byte, error) { } ota.Type = ota.Parent.Type - ota.Hasher = plumbing.NewHasher(ota.Type, int64(len(patched))) - if _, err := ota.Write(patched); err != nil { + sha1, err := getSHA1(ota.Type, patched) + if err != nil { return nil, err } - ota.SHA1 = ota.Sum() + + ota.SHA1 = sha1 ota.Length = int64(len(patched)) return patched, nil } -type objectInfo struct { - plumbing.Hasher +func getSHA1(t plumbing.ObjectType, data []byte) (plumbing.Hash, error) { + hasher := plumbing.NewHasher(t, int64(len(data))) + if _, err := hasher.Write(data); err != nil { + return plumbing.ZeroHash, err + } + + return hasher.Sum(), nil +} - Offset int64 - Length int64 - HeaderLength int64 - PackSize int64 - Type plumbing.ObjectType - DiskType plumbing.ObjectType +type objectInfo struct { + Offset int64 + Length int64 + Type plumbing.ObjectType + DiskType plumbing.ObjectType Crc32 uint32 Parent *objectInfo Children []*objectInfo SHA1 plumbing.Hash - - Content []byte } func newBaseObject(offset, length int64, t plumbing.ObjectType) *objectInfo { @@ -469,29 +466,18 @@ func newDeltaObject( t plumbing.ObjectType, parent *objectInfo, ) *objectInfo { - children := make([]*objectInfo, 0) - obj := &objectInfo{ - Hasher: plumbing.NewHasher(t, length), Offset: offset, Length: length, - PackSize: 0, Type: t, DiskType: t, Crc32: 0, Parent: parent, - Children: children, } return obj } -func (o *objectInfo) Write(b []byte) (int, error) { - o.Content = make([]byte, len(b)) - copy(o.Content, b) - return o.Hasher.Write(b) -} - func (o *objectInfo) IsDelta() bool { return o.Type.IsDelta() } From eb2aa9b2c3bf7af93fd261228be1b96e61c52bcf Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Tue, 14 Aug 2018 16:56:29 +0200 Subject: [PATCH 061/120] plumbing/packfile: do not compute sha1 for already undeltified objects Signed-off-by: Javi Fontan --- plumbing/format/packfile/parser.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/plumbing/format/packfile/parser.go b/plumbing/format/packfile/parser.go index 3a9c4d75b..28582b5fa 100644 --- a/plumbing/format/packfile/parser.go +++ b/plumbing/format/packfile/parser.go @@ -423,14 +423,16 @@ func applyPatchBase(ota *objectInfo, data, base []byte) ([]byte, error) { return nil, err } - ota.Type = ota.Parent.Type - sha1, err := getSHA1(ota.Type, patched) - if err != nil { - return nil, err - } + if ota.SHA1 == plumbing.ZeroHash { + ota.Type = ota.Parent.Type + sha1, err := getSHA1(ota.Type, patched) + if err != nil { + return nil, err + } - ota.SHA1 = sha1 - ota.Length = int64(len(patched)) + ota.SHA1 = sha1 + ota.Length = int64(len(patched)) + } return patched, nil } From ae5501623169ec981091a5f1cfb56ab8e7688031 Mon Sep 17 00:00:00 2001 From: noxora Date: Tue, 14 Aug 2018 14:02:26 -0500 Subject: [PATCH 062/120] added hook support Signed-off-by: noxora trying a possible fix to the delete test Signed-off-by: noxora still trying to fix this test Signed-off-by: noxora fixes did not work, seems to be a windows env problem Signed-off-by: noxora --- storage/filesystem/dotgit/dotgit.go | 53 ++++++++++++++++++++++-- storage/filesystem/dotgit/dotgit_test.go | 49 +++++++++++++++++++++- 2 files changed, 98 insertions(+), 4 deletions(-) diff --git a/storage/filesystem/dotgit/dotgit.go b/storage/filesystem/dotgit/dotgit.go index dc12f23cf..d16a77daf 100644 --- a/storage/filesystem/dotgit/dotgit.go +++ b/storage/filesystem/dotgit/dotgit.go @@ -279,19 +279,66 @@ func (d *DotGit) objectPath(h plumbing.Hash) string { return d.fs.Join(objectsPath, hash[0:2], hash[2:40]) } +//incomingObjectPath is intended to add support for a git pre-recieve hook to be written +//it adds support for go-git to find objects in an "incoming" directory, so that the library +//can be used to write a pre-recieve hook that deals with the incoming objects. +//More on git hooks found here : https://git-scm.com/docs/githooks +//More on 'quarantine'/incoming directory here : https://git-scm.com/docs/git-receive-pack +func (d *DotGit) incomingObjectPath(h plumbing.Hash) string { + hString := h.String() + directoryContents, err := d.fs.ReadDir(objectsPath) + if err != nil { + return d.fs.Join(objectsPath, hString[0:2], hString[2:40]) + } + var incomingDirName string + for _, file := range directoryContents { + if strings.Split(file.Name(), "-")[0] == "incoming" && file.IsDir() { + incomingDirName = file.Name() + } + } + if incomingDirName == "" { + return d.fs.Join(objectsPath, hString[0:2], hString[2:40]) + } + return d.fs.Join(objectsPath, incomingDirName, hString[0:2], hString[2:40]) +} + // Object returns a fs.File pointing the object file, if exists func (d *DotGit) Object(h plumbing.Hash) (billy.File, error) { - return d.fs.Open(d.objectPath(h)) + obj1, err1 := d.fs.Open(d.objectPath(h)) + if os.IsNotExist(err1) { + obj2, err2 := d.fs.Open(d.incomingObjectPath(h)) + if err2 != nil { + return obj1, err1 + } + return obj2, err2 + } + return obj1, err1 } // ObjectStat returns a os.FileInfo pointing the object file, if exists func (d *DotGit) ObjectStat(h plumbing.Hash) (os.FileInfo, error) { - return d.fs.Stat(d.objectPath(h)) + obj1, err1 := d.fs.Stat(d.objectPath(h)) + if os.IsNotExist(err1) { + obj2, err2 := d.fs.Stat(d.incomingObjectPath(h)) + if err2 != nil { + return obj1, err1 + } + return obj2, err2 + } + return obj1, err1 } // ObjectDelete removes the object file, if exists func (d *DotGit) ObjectDelete(h plumbing.Hash) error { - return d.fs.Remove(d.objectPath(h)) + err1 := d.fs.Remove(d.objectPath(h)) + if os.IsNotExist(err1) { + err2 := d.fs.Remove(d.incomingObjectPath(h)) + if err2 != nil { + return err1 + } + return err2 + } + return err1 } func (d *DotGit) readReferenceFrom(rd io.Reader, name string) (ref *plumbing.Reference, err error) { diff --git a/storage/filesystem/dotgit/dotgit_test.go b/storage/filesystem/dotgit/dotgit_test.go index 7733eef78..ce3050039 100644 --- a/storage/filesystem/dotgit/dotgit_test.go +++ b/storage/filesystem/dotgit/dotgit_test.go @@ -9,11 +9,11 @@ import ( "strings" "testing" + fixtures "gopkg.in/src-d/go-git-fixtures.v3" "gopkg.in/src-d/go-git.v4/plumbing" . "gopkg.in/check.v1" "gopkg.in/src-d/go-billy.v4/osfs" - "gopkg.in/src-d/go-git-fixtures.v3" ) func Test(t *testing.T) { TestingT(t) } @@ -537,6 +537,53 @@ func (s *SuiteDotGit) TestObject(c *C) { file.Name(), fs.Join("objects", "03", "db8e1fbe133a480f2867aac478fd866686d69e")), Equals, true, ) + incomingHash := "9d25e0f9bde9f82882b49fe29117b9411cb157b7" //made up hash + incomingDirPath := fs.Join("objects", "incoming-123456") + incomingFilePath := fs.Join(incomingDirPath, incomingHash[0:2], incomingHash[2:40]) + fs.MkdirAll(incomingDirPath, os.FileMode(0755)) + fs.Create(incomingFilePath) + + file, err = dir.Object(plumbing.NewHash(incomingHash)) + c.Assert(err, IsNil) +} + +func (s *SuiteDotGit) TestObjectStat(c *C) { + fs := fixtures.ByTag(".git").ByTag("unpacked").One().DotGit() + dir := New(fs) + + hash := plumbing.NewHash("03db8e1fbe133a480f2867aac478fd866686d69e") + _, err := dir.ObjectStat(hash) + c.Assert(err, IsNil) + incomingHash := "9d25e0f9bde9f82882b49fe29117b9411cb157b7" //made up hash + incomingDirPath := fs.Join("objects", "incoming-123456") + incomingFilePath := fs.Join(incomingDirPath, incomingHash[0:2], incomingHash[2:40]) + fs.MkdirAll(incomingDirPath, os.FileMode(0755)) + fs.Create(incomingFilePath) + + _, err = dir.ObjectStat(plumbing.NewHash(incomingHash)) + c.Assert(err, IsNil) +} + +func (s *SuiteDotGit) TestObjectDelete(c *C) { + fs := fixtures.ByTag(".git").ByTag("unpacked").One().DotGit() + dir := New(fs) + + hash := plumbing.NewHash("03db8e1fbe133a480f2867aac478fd866686d69e") + err := dir.ObjectDelete(hash) + c.Assert(err, IsNil) + //incomingHash := "9d25e0f9bde9f82882b49fe29117b9411cb157b7" //made up hash + //incomingDirPath := fs.Join("objects", "incoming-123456") + //incomingSubDirPath := fs.Join(incomingDirPath, incomingHash[0:2]) + //incomingFilePath := fs.Join(incomingDirPath, incomingHash[2:40]) + //err = fs.MkdirAll(incomingDirPath, os.FileMode(0755)) + //c.Assert(err, IsNil) + //err = fs.MkdirAll(incomingSubDirPath, os.FileMode(0755)) + //c.Assert(err, IsNil) + //_, err = fs.Create(incomingFilePath) + //c.Assert(err, IsNil) + + //err = dir.ObjectDelete(plumbing.NewHash(incomingHash)) + //c.Assert(err, IsNil) } func (s *SuiteDotGit) TestObjectNotFound(c *C) { From ec3d2a817d7cf43696a42d8460c7a8957a12a57b Mon Sep 17 00:00:00 2001 From: Chris Marchesi Date: Thu, 16 Aug 2018 17:41:03 -0700 Subject: [PATCH 063/120] plumbing: object, Don't add new line at end of commit signature The way that commit signatures were being written out was causing an extra newline to be written at the end of the commit when the message encoding was already taking care of this. Ultimately, this results in a corrupt object, rendering the object unverifiable with the signature in the commit. Signed-off-by: Chris Marchesi --- plumbing/object/commit.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/plumbing/object/commit.go b/plumbing/object/commit.go index b1c0e0180..00ae3f144 100644 --- a/plumbing/object/commit.go +++ b/plumbing/object/commit.go @@ -263,18 +263,18 @@ func (b *Commit) encode(o plumbing.EncodedObject, includeSig bool) (err error) { } if b.PGPSignature != "" && includeSig { - if _, err = fmt.Fprint(w, "\n"+headerpgp); err != nil { + if _, err = fmt.Fprint(w, "\n"+headerpgp+" "); err != nil { return err } - // Split all the signature lines and write with a left padding and - // newline at the end. + // Split all the signature lines and re-write with a left padding and + // newline. Use join for this so it's clear that a newline should not be + // added after this section, as it will be added when the message is + // printed. signature := strings.TrimSuffix(b.PGPSignature, "\n") lines := strings.Split(signature, "\n") - for _, line := range lines { - if _, err = fmt.Fprintf(w, " %s\n", line); err != nil { - return err - } + if _, err = fmt.Fprint(w, strings.Join(lines, "\n ")); err != nil { + return err } } From 0ef699d06cd038b73ea22a6d1eb19aff2761156f Mon Sep 17 00:00:00 2001 From: Chris Marchesi Date: Thu, 16 Aug 2018 17:46:07 -0700 Subject: [PATCH 064/120] git: Add ability to PGP sign commits This adds the ability to sign commits by adding the SignKey field to CommitOptions. If present, the commit will be signed during the WorkTree.Commit call. The supplied SignKey must already be decrypted by the caller. Signed-off-by: Chris Marchesi --- options.go | 4 ++ worktree_commit.go | 25 +++++++++ worktree_commit_test.go | 115 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 144 insertions(+) diff --git a/options.go b/options.go index 885980ef0..7b1570f7b 100644 --- a/options.go +++ b/options.go @@ -4,6 +4,7 @@ import ( "errors" "regexp" + "golang.org/x/crypto/openpgp" "gopkg.in/src-d/go-git.v4/config" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/object" @@ -347,6 +348,9 @@ type CommitOptions struct { // Parents are the parents commits for the new commit, by default when // len(Parents) is zero, the hash of HEAD reference is used. Parents []plumbing.Hash + // A key to sign the commit with. A nil value here means the commit will not + // be signed. The private key must be present and already decrypted. + SignKey *openpgp.Entity } // Validate validates the fields and sets the default values. diff --git a/worktree_commit.go b/worktree_commit.go index 5fa63ab0a..ad7880ae4 100644 --- a/worktree_commit.go +++ b/worktree_commit.go @@ -4,6 +4,7 @@ import ( "path" "strings" + "golang.org/x/crypto/openpgp" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/filemode" "gopkg.in/src-d/go-git.v4/plumbing/format/index" @@ -92,6 +93,14 @@ func (w *Worktree) buildCommitObject(msg string, opts *CommitOptions, tree plumb ParentHashes: opts.Parents, } + if opts.SignKey != nil { + sig, err := w.buildCommitSignature(commit, opts.SignKey) + if err != nil { + return plumbing.ZeroHash, err + } + commit.PGPSignature = sig + } + obj := w.r.Storer.NewEncodedObject() if err := commit.Encode(obj); err != nil { return plumbing.ZeroHash, err @@ -99,6 +108,22 @@ func (w *Worktree) buildCommitObject(msg string, opts *CommitOptions, tree plumb return w.r.Storer.SetEncodedObject(obj) } +func (w *Worktree) buildCommitSignature(commit *object.Commit, signKey *openpgp.Entity) (string, error) { + encoded := &plumbing.MemoryObject{} + if err := commit.Encode(encoded); err != nil { + return "", err + } + r, err := encoded.Reader() + if err != nil { + return "", err + } + var b strings.Builder + if err := openpgp.ArmoredDetachSign(&b, signKey, r, nil); err != nil { + return "", err + } + return b.String(), nil +} + // buildTreeHelper converts a given index.Index file into multiple git objects // reading the blobs from the given filesystem and creating the trees from the // index structure. The created objects are pushed to a given Storer. diff --git a/worktree_commit_test.go b/worktree_commit_test.go index 5575bca1a..242d2fdab 100644 --- a/worktree_commit_test.go +++ b/worktree_commit_test.go @@ -1,8 +1,11 @@ package git import ( + "strings" "time" + "golang.org/x/crypto/openpgp" + "golang.org/x/crypto/openpgp/armor" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/object" "gopkg.in/src-d/go-git.v4/plumbing/storer" @@ -135,6 +138,47 @@ func (s *WorktreeSuite) TestRemoveAndCommitAll(c *C) { assertStorageStatus(c, s.Repository, 13, 11, 11, expected) } +func (s *WorktreeSuite) TestCommitSign(c *C) { + // expectedHash := plumbing.NewHash("98c4ac7c29c913f7461eae06e024dc18e80d23a4") + + fs := memfs.New() + storage := memory.NewStorage() + + r, err := Init(storage, fs) + c.Assert(err, IsNil) + + w, err := r.Worktree() + c.Assert(err, IsNil) + + util.WriteFile(fs, "foo", []byte("foo"), 0644) + + _, err = w.Add("foo") + c.Assert(err, IsNil) + + key := commitSignKey(c) + hash, err := w.Commit("foo\n", &CommitOptions{Author: defaultSignature(), SignKey: key}) + // c.Assert(hash, Equals, expectedHash) + c.Assert(err, IsNil) + + // assertStorageStatus(c, r, 1, 1, 1, expectedHash) + + // Verify the commit. + pks := new(strings.Builder) + pkw, err := armor.Encode(pks, openpgp.PublicKeyType, nil) + c.Assert(err, IsNil) + + err = key.Serialize(pkw) + c.Assert(err, IsNil) + err = pkw.Close() + c.Assert(err, IsNil) + + expectedCommit, err := r.CommitObject(hash) + c.Assert(err, IsNil) + actual, err := expectedCommit.Verify(pks.String()) + c.Assert(err, IsNil) + c.Assert(actual.PrimaryKey, DeepEquals, key.PrimaryKey) +} + func assertStorageStatus( c *C, r *Repository, treesCount, blobCount, commitCount int, head plumbing.Hash, @@ -173,3 +217,74 @@ func defaultSignature() *object.Signature { When: when, } } + +func commitSignKey(c *C) *openpgp.Entity { + s := strings.NewReader(armoredKeyRing) + es, err := openpgp.ReadArmoredKeyRing(s) + c.Assert(err, IsNil) + + c.Assert(es, HasLen, 1) + c.Assert(es[0].Identities, HasLen, 1) + _, ok := es[0].Identities["foo bar "] + c.Assert(ok, Equals, true) + + return es[0] +} + +const armoredKeyRing = ` +-----BEGIN PGP PRIVATE KEY BLOCK----- + +lQcYBFt1xEwBEACcJKn7ZVm465OXXDM3gvIMft4GKD82VmzujjAT1XQQf3srIrNR +lgFZiSE3fYFvc7SbqIXRVn1Fg9N95XSF6S5C8PwRwtpEPBfKOwGSuFWdlL68vRXX +yDs0u9Ih3TYOtnJ9qBJ/QUt6dOZd/CC88qaEEn0tvNU1A2uujjWuJCbcnfjZisM8 +8fF2vTkgN7eSojhK07WFGwbuYxJmkfZ1zyvaeeSH61WCE29vXKObZtc0dqA/AOW4 +x08hFyzAdLSqSdMURHpMjHGO1/0xtnZXaf346IJDwOyJKoIEVuSLppIKB8uUCuMZ +ux+cYXfwBkzTKD7SmQO5Q0lU13QRz+aLCOjWij5PR5KqqYpLgS2iiEE12gxx7uuj +a5pHjRdCrMNlvn8fFwgRjhciZWftHy3cda0wMLEchezYBK4xF2bWqfO8jIrsXHxr +m9/9T7t7f9wIJN4fFNVQDeNOsONMv63eZKudAfkElIBwfnBDGHLD14uID9KVnPUY +EJ/pm1lptDd5nBsR1sfaLIj05vM29yByk/jE7HCvqlHi/6H4igCjhGm7+9ypHmPO +iviXKNCJpllCPNMaJRaOYcTYfX9PMMNIrp6txuzNwFNj5bwoyaegaa6R2/9hm7Pu +ljXd6Clck2XNEX6Uig3ZBM+LrAn4fn5IEH7XfXGlK0SNvjZMRk6kgpiInwARAQAB +AA/7B02YRk10PrfOBY/m2VtnjfnHY9RVytb5/+uNLPlz3iI/J0JXhaLw68q64f/5 +NEjeJYyHBKH5ZkjQUIpswqc6r2ja8V25nJshOAL1FgWQTLhlmuBO5xdAk2GPQc1d +7gw8iKaeztwO0rPfmesuSfXYDWh+Qqc/rIBhvfnqz2i+5JOVOxZs2nQszg7OzMaJ +y/L2JjZiLBhRYNFMluIiRRE1aXQkohdnmgV4HviWIB8MROjqWPpT6OFNB78Lmc1k +v5CIG5i2oD0Xr8SMb5HQMS4yqKptDnnsjSizZGDaV2nNxx8ugI+yXrInvDQEk6i1 +fb7bbfA4ORTJTkRs84IKj23s0oXk25bHrglt8A/dwYg2U4vAhp1lmWUNM1XJGKdu +OW5113Uny2bE6G9euWwLBebVLETWLBWPDVkcohxtWSoPIgk5B2JAtF35k6JpmoNW +QIeaFf6E3juduwRaPixariIks0o0ofFusVKYJHcAMzT8NNZEzixjn0KNVPEtsRmm +CvF2kJmp6sNXw2pipBli3P6rKrm/Hd+fLDHAfsdX49IrKjJEt9Yg9iXoNHv8EPAi +KAjIGaG4rMbmNPsgeCviz3/OSQM0G4Uba1CzW/oi8XPxYnRmvvkPw1LilNZ1mPYg +zAfAfBkp7IOMk7eVBNZ7Y87aaf8n+6X1IhS74nNBscBSN+kIAMMJiDXel+QGmUb3 +Esz051k43FQjcrPE1d9JLmxVvBDbwU9dEWfBvo4lJGWllmVBipW4A1MswqOzGApG +TpD5YDfJ24+KMTz1BrROUApxzl3LI2NIl2DuqeVaeiiZf3EzunlZM/J21Dljl9Bi +aSLA18DxkEMJ4xiKZCaqCUabRB1czOJbagA8F2xV2OOrQkweSLI82l6PLL5bWtw9 +NhpuZ8/rQotXAk2AYNqQVOjtEdCfyaAnDm3lsTTYpEg4opZ2iFoOsVv4Q4oGWqOA +9eiDLy3RtRkA6HgSGcbkafGEkw5LNQPR3z2K42MCmcvkZnbbzVAUi2RstYXoE6gq +HOK3Qn0IAMzy6jMuyQ2+8ILEkJfjIfV3451pqvFi2hMuXCyrTA84tiPWJMo8yJLW +DUZf3ll3f2bq8XF7tCcVAUSgc9EhJtz2j4Gkr3mQZ7NQ41FeHXdNBk8n1895gBnN +jE2ubcu6h7lYKGol6z1mFfSBIZDeiFkM0WBOkocxWbjytwSC7QJAD0VRtpdMvf41 +BxL1tijwZ7Ocp0niM2pwdfZqAQR2o6qbnwIVVEAvJpnMcpKeanirO49NPA3Slp5M ++4Hdxkq82vvWNkCIYW5TWuQ6Vqb3z6B3DZWAIWvIFr9jXz/vk/8sOiaJARaZxgoX +gEny2WIj0mOB7HTZTAUT0PZ3b7/npksH/ivxTJGH3rcgnu5CsaC8khzbjlQ7T68s +qj95QfgpaXFprx3kHlZXUdAb+SMWgFpxQQ08hplKhroKqkZ5KpOm1YpTHYs3JMK4 +L0oCIr3JFD+JPkg99sVRJ+wRHknxMyJlSTyo91JS1OO1QcleJkY67Fp3dE4U9BNW +1Sn9nQ5vKz4Ihfuwp1c3awqseGUxo66VkOJtxz39DSsouRwTJZN79MTDxCKxlFh8 +7VjWnNN6w6YPLa7m3IezEokIHVbrGMnCt/i4/IoBh9jvwVOzs9LDAPAAEiRtLBD/ +oVpB92IDhAfNSbjATly1uBqeRwlbxLa83wjoRi1CVh/Gq0q8KbEMXxdyNLQVZm9v +IGJhciA8Zm9vQGZvby5mb28+iQJUBBMBCAA+FiEESwOtDG37jLruovFg4daDgE6b +E5AFAlt1xEwCGwMFCQPCZwAFCwkIBwIGFQgJCgsCBBYCAwECHgECF4AACgkQ4daD +gE6bE5BaTQ/+JnqAMhRVVxD+uordvXkO3xxsdPTGrc4c+JZjwL6bitFg/Teqqk0M +jnBqVuNSE4OiAKEcFNB6GxFR+PJPJXoR9OBFH5gKUGL4YfBshgExZv0G56eMlIKx +GRjJwxxH3gnM2edCb00TYX/IMzi1jrLE178kVsqT8kG2eBPMExiKdkci2qs/FHd4 +ic92NmySgPiA7WGWwA9MkhCoeULNUi42Kmp3voM9xxlltom5lICGB/sRFwY1Frye +Br2WesD5Si1ravw+/QiOCaici6zfd6b/L/o9gRJgBtbrCLYlhJptvYbeKEnMI9dD +wgBBRth/KQHqUJ3aBtpSuTtIRWrkpPEajPHHaLp2RZPTB/sEsOornWUYq0gphbER +u2N7Wzea9jLb92AVwTS78J9GxrBQrh8H8+0lFoJPSZGL3c4UMh7C9NkfQ3O9UYqg +RCjYhp7FxVqSzUBPDymcky/t2DOLuEMdfrRJCCJPz8kUa75Hzd2tEWmSgbx8hnd6 +fuVWv/l43kdYimBKYtw2WzqSbD9JE54AMnHwZBWiaxtMGYh4ue/CcvMQyxBLjkvz +6/J0a+0pElbaFVwkJYtbY1Xe/wv/TRk2aHfJSa5cdS+HbFBC+Jc08t2c1WX9fzUH +YgOocz31q0WXTiXnQTRQuUqMDRHCRt+kI+ndLEjQqhrHFE8O5Smbj/s= +=bnr6 +-----END PGP PRIVATE KEY BLOCK----- +` From c9b2eac59cf97c9a20ea3e9e5ad9bdef6f1dc82b Mon Sep 17 00:00:00 2001 From: Chris Marchesi Date: Thu, 16 Aug 2018 18:15:49 -0700 Subject: [PATCH 065/120] git: Remove use of strings.Builder This was added in Go 1.10 and is not supported on Go 1.9. Switched to bytes.Buffer to ensure compatibility. Signed-off-by: Chris Marchesi --- worktree_commit.go | 3 ++- worktree_commit_test.go | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/worktree_commit.go b/worktree_commit.go index ad7880ae4..b83bf0c7a 100644 --- a/worktree_commit.go +++ b/worktree_commit.go @@ -1,6 +1,7 @@ package git import ( + "bytes" "path" "strings" @@ -117,7 +118,7 @@ func (w *Worktree) buildCommitSignature(commit *object.Commit, signKey *openpgp. if err != nil { return "", err } - var b strings.Builder + var b bytes.Buffer if err := openpgp.ArmoredDetachSign(&b, signKey, r, nil); err != nil { return "", err } diff --git a/worktree_commit_test.go b/worktree_commit_test.go index 242d2fdab..17a129365 100644 --- a/worktree_commit_test.go +++ b/worktree_commit_test.go @@ -1,6 +1,7 @@ package git import ( + "bytes" "strings" "time" @@ -163,7 +164,7 @@ func (s *WorktreeSuite) TestCommitSign(c *C) { // assertStorageStatus(c, r, 1, 1, 1, expectedHash) // Verify the commit. - pks := new(strings.Builder) + pks := new(bytes.Buffer) pkw, err := armor.Encode(pks, openpgp.PublicKeyType, nil) c.Assert(err, IsNil) From b2edb6dc79ea19b881bb5104395186f9d8bf2966 Mon Sep 17 00:00:00 2001 From: Chris Marchesi Date: Thu, 16 Aug 2018 19:39:38 -0700 Subject: [PATCH 066/120] git: Remove old hash validation code This will not work for a signed commit as with the GPG signature being a part of the commit, the hash is now non-deterministic. Verification of the commit is done through the validation of the signature. Signed-off-by: Chris Marchesi --- worktree_commit_test.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/worktree_commit_test.go b/worktree_commit_test.go index 17a129365..823a886fc 100644 --- a/worktree_commit_test.go +++ b/worktree_commit_test.go @@ -140,8 +140,6 @@ func (s *WorktreeSuite) TestRemoveAndCommitAll(c *C) { } func (s *WorktreeSuite) TestCommitSign(c *C) { - // expectedHash := plumbing.NewHash("98c4ac7c29c913f7461eae06e024dc18e80d23a4") - fs := memfs.New() storage := memory.NewStorage() @@ -158,11 +156,8 @@ func (s *WorktreeSuite) TestCommitSign(c *C) { key := commitSignKey(c) hash, err := w.Commit("foo\n", &CommitOptions{Author: defaultSignature(), SignKey: key}) - // c.Assert(hash, Equals, expectedHash) c.Assert(err, IsNil) - // assertStorageStatus(c, r, 1, 1, 1, expectedHash) - // Verify the commit. pks := new(bytes.Buffer) pkw, err := armor.Encode(pks, openpgp.PublicKeyType, nil) From 39954f2d9310204cc586715def7f075296a21d4c Mon Sep 17 00:00:00 2001 From: Chris Marchesi Date: Thu, 16 Aug 2018 20:02:00 -0700 Subject: [PATCH 067/120] git: Add extra test for testing bad key error case I'm hoping this helps get codecov to a tolerable delta. :) Signed-off-by: Chris Marchesi --- worktree_commit_test.go | 140 ++++++++++++++++++++++++---------------- 1 file changed, 85 insertions(+), 55 deletions(-) diff --git a/worktree_commit_test.go b/worktree_commit_test.go index 823a886fc..004926fc5 100644 --- a/worktree_commit_test.go +++ b/worktree_commit_test.go @@ -7,6 +7,7 @@ import ( "golang.org/x/crypto/openpgp" "golang.org/x/crypto/openpgp/armor" + "golang.org/x/crypto/openpgp/errors" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/object" "gopkg.in/src-d/go-git.v4/plumbing/storer" @@ -154,7 +155,7 @@ func (s *WorktreeSuite) TestCommitSign(c *C) { _, err = w.Add("foo") c.Assert(err, IsNil) - key := commitSignKey(c) + key := commitSignKey(c, true) hash, err := w.Commit("foo\n", &CommitOptions{Author: defaultSignature(), SignKey: key}) c.Assert(err, IsNil) @@ -175,6 +176,26 @@ func (s *WorktreeSuite) TestCommitSign(c *C) { c.Assert(actual.PrimaryKey, DeepEquals, key.PrimaryKey) } +func (s *WorktreeSuite) TestCommitSignBadKey(c *C) { + fs := memfs.New() + storage := memory.NewStorage() + + r, err := Init(storage, fs) + c.Assert(err, IsNil) + + w, err := r.Worktree() + c.Assert(err, IsNil) + + util.WriteFile(fs, "foo", []byte("foo"), 0644) + + _, err = w.Add("foo") + c.Assert(err, IsNil) + + key := commitSignKey(c, false) + _, err = w.Commit("foo\n", &CommitOptions{Author: defaultSignature(), SignKey: key}) + c.Assert(err, Equals, errors.InvalidArgumentError("signing key is encrypted")) +} + func assertStorageStatus( c *C, r *Repository, treesCount, blobCount, commitCount int, head plumbing.Hash, @@ -214,7 +235,7 @@ func defaultSignature() *object.Signature { } } -func commitSignKey(c *C) *openpgp.Entity { +func commitSignKey(c *C, decrypt bool) *openpgp.Entity { s := strings.NewReader(armoredKeyRing) es, err := openpgp.ReadArmoredKeyRing(s) c.Assert(err, IsNil) @@ -224,63 +245,72 @@ func commitSignKey(c *C) *openpgp.Entity { _, ok := es[0].Identities["foo bar "] c.Assert(ok, Equals, true) - return es[0] + key := es[0] + if decrypt { + err = key.PrivateKey.Decrypt([]byte(keyPassphrase)) + c.Assert(err, IsNil) + } + + return key } const armoredKeyRing = ` -----BEGIN PGP PRIVATE KEY BLOCK----- -lQcYBFt1xEwBEACcJKn7ZVm465OXXDM3gvIMft4GKD82VmzujjAT1XQQf3srIrNR -lgFZiSE3fYFvc7SbqIXRVn1Fg9N95XSF6S5C8PwRwtpEPBfKOwGSuFWdlL68vRXX -yDs0u9Ih3TYOtnJ9qBJ/QUt6dOZd/CC88qaEEn0tvNU1A2uujjWuJCbcnfjZisM8 -8fF2vTkgN7eSojhK07WFGwbuYxJmkfZ1zyvaeeSH61WCE29vXKObZtc0dqA/AOW4 -x08hFyzAdLSqSdMURHpMjHGO1/0xtnZXaf346IJDwOyJKoIEVuSLppIKB8uUCuMZ -ux+cYXfwBkzTKD7SmQO5Q0lU13QRz+aLCOjWij5PR5KqqYpLgS2iiEE12gxx7uuj -a5pHjRdCrMNlvn8fFwgRjhciZWftHy3cda0wMLEchezYBK4xF2bWqfO8jIrsXHxr -m9/9T7t7f9wIJN4fFNVQDeNOsONMv63eZKudAfkElIBwfnBDGHLD14uID9KVnPUY -EJ/pm1lptDd5nBsR1sfaLIj05vM29yByk/jE7HCvqlHi/6H4igCjhGm7+9ypHmPO -iviXKNCJpllCPNMaJRaOYcTYfX9PMMNIrp6txuzNwFNj5bwoyaegaa6R2/9hm7Pu -ljXd6Clck2XNEX6Uig3ZBM+LrAn4fn5IEH7XfXGlK0SNvjZMRk6kgpiInwARAQAB -AA/7B02YRk10PrfOBY/m2VtnjfnHY9RVytb5/+uNLPlz3iI/J0JXhaLw68q64f/5 -NEjeJYyHBKH5ZkjQUIpswqc6r2ja8V25nJshOAL1FgWQTLhlmuBO5xdAk2GPQc1d -7gw8iKaeztwO0rPfmesuSfXYDWh+Qqc/rIBhvfnqz2i+5JOVOxZs2nQszg7OzMaJ -y/L2JjZiLBhRYNFMluIiRRE1aXQkohdnmgV4HviWIB8MROjqWPpT6OFNB78Lmc1k -v5CIG5i2oD0Xr8SMb5HQMS4yqKptDnnsjSizZGDaV2nNxx8ugI+yXrInvDQEk6i1 -fb7bbfA4ORTJTkRs84IKj23s0oXk25bHrglt8A/dwYg2U4vAhp1lmWUNM1XJGKdu -OW5113Uny2bE6G9euWwLBebVLETWLBWPDVkcohxtWSoPIgk5B2JAtF35k6JpmoNW -QIeaFf6E3juduwRaPixariIks0o0ofFusVKYJHcAMzT8NNZEzixjn0KNVPEtsRmm -CvF2kJmp6sNXw2pipBli3P6rKrm/Hd+fLDHAfsdX49IrKjJEt9Yg9iXoNHv8EPAi -KAjIGaG4rMbmNPsgeCviz3/OSQM0G4Uba1CzW/oi8XPxYnRmvvkPw1LilNZ1mPYg -zAfAfBkp7IOMk7eVBNZ7Y87aaf8n+6X1IhS74nNBscBSN+kIAMMJiDXel+QGmUb3 -Esz051k43FQjcrPE1d9JLmxVvBDbwU9dEWfBvo4lJGWllmVBipW4A1MswqOzGApG -TpD5YDfJ24+KMTz1BrROUApxzl3LI2NIl2DuqeVaeiiZf3EzunlZM/J21Dljl9Bi -aSLA18DxkEMJ4xiKZCaqCUabRB1czOJbagA8F2xV2OOrQkweSLI82l6PLL5bWtw9 -NhpuZ8/rQotXAk2AYNqQVOjtEdCfyaAnDm3lsTTYpEg4opZ2iFoOsVv4Q4oGWqOA -9eiDLy3RtRkA6HgSGcbkafGEkw5LNQPR3z2K42MCmcvkZnbbzVAUi2RstYXoE6gq -HOK3Qn0IAMzy6jMuyQ2+8ILEkJfjIfV3451pqvFi2hMuXCyrTA84tiPWJMo8yJLW -DUZf3ll3f2bq8XF7tCcVAUSgc9EhJtz2j4Gkr3mQZ7NQ41FeHXdNBk8n1895gBnN -jE2ubcu6h7lYKGol6z1mFfSBIZDeiFkM0WBOkocxWbjytwSC7QJAD0VRtpdMvf41 -BxL1tijwZ7Ocp0niM2pwdfZqAQR2o6qbnwIVVEAvJpnMcpKeanirO49NPA3Slp5M -+4Hdxkq82vvWNkCIYW5TWuQ6Vqb3z6B3DZWAIWvIFr9jXz/vk/8sOiaJARaZxgoX -gEny2WIj0mOB7HTZTAUT0PZ3b7/npksH/ivxTJGH3rcgnu5CsaC8khzbjlQ7T68s -qj95QfgpaXFprx3kHlZXUdAb+SMWgFpxQQ08hplKhroKqkZ5KpOm1YpTHYs3JMK4 -L0oCIr3JFD+JPkg99sVRJ+wRHknxMyJlSTyo91JS1OO1QcleJkY67Fp3dE4U9BNW -1Sn9nQ5vKz4Ihfuwp1c3awqseGUxo66VkOJtxz39DSsouRwTJZN79MTDxCKxlFh8 -7VjWnNN6w6YPLa7m3IezEokIHVbrGMnCt/i4/IoBh9jvwVOzs9LDAPAAEiRtLBD/ -oVpB92IDhAfNSbjATly1uBqeRwlbxLa83wjoRi1CVh/Gq0q8KbEMXxdyNLQVZm9v -IGJhciA8Zm9vQGZvby5mb28+iQJUBBMBCAA+FiEESwOtDG37jLruovFg4daDgE6b -E5AFAlt1xEwCGwMFCQPCZwAFCwkIBwIGFQgJCgsCBBYCAwECHgECF4AACgkQ4daD -gE6bE5BaTQ/+JnqAMhRVVxD+uordvXkO3xxsdPTGrc4c+JZjwL6bitFg/Teqqk0M -jnBqVuNSE4OiAKEcFNB6GxFR+PJPJXoR9OBFH5gKUGL4YfBshgExZv0G56eMlIKx -GRjJwxxH3gnM2edCb00TYX/IMzi1jrLE178kVsqT8kG2eBPMExiKdkci2qs/FHd4 -ic92NmySgPiA7WGWwA9MkhCoeULNUi42Kmp3voM9xxlltom5lICGB/sRFwY1Frye -Br2WesD5Si1ravw+/QiOCaici6zfd6b/L/o9gRJgBtbrCLYlhJptvYbeKEnMI9dD -wgBBRth/KQHqUJ3aBtpSuTtIRWrkpPEajPHHaLp2RZPTB/sEsOornWUYq0gphbER -u2N7Wzea9jLb92AVwTS78J9GxrBQrh8H8+0lFoJPSZGL3c4UMh7C9NkfQ3O9UYqg -RCjYhp7FxVqSzUBPDymcky/t2DOLuEMdfrRJCCJPz8kUa75Hzd2tEWmSgbx8hnd6 -fuVWv/l43kdYimBKYtw2WzqSbD9JE54AMnHwZBWiaxtMGYh4ue/CcvMQyxBLjkvz -6/J0a+0pElbaFVwkJYtbY1Xe/wv/TRk2aHfJSa5cdS+HbFBC+Jc08t2c1WX9fzUH -YgOocz31q0WXTiXnQTRQuUqMDRHCRt+kI+ndLEjQqhrHFE8O5Smbj/s= -=bnr6 +lQdGBFt2OHgBEADQpRmFm9X9xBfUljVs1B24MXWRHcEP5tx2k6Cp90sSz/ZOJcxH +RjzYuXjpkE7g/PaZxAMVS1PptJip/w1/+5l2gZ7RmzU/e3hKe4vALHzKMVp8t7Ta +0e2K3STxapCr9FNITjQRGOhnFwqiYoPCf9u5Iy8uszDH7HHnBZx+Nvbl95dDvmMs +aFUKMeaoFD19iwEdRu6gJo7YIWF/8zwHi49neKigisGKh5PI0KUYeRPydXeCZIKQ +ofdk+CPUS4r3dVhxTMYeHn/Vrep3blEA45E7KJ+TESmKkwliEgdjJwaVkUfJhBkb +p2pMPKwbxLma9GCJBimOkehFv8/S+xn/xrLSsTxeOCIzMp3I5vgjR5QfONq5kuB1 +qbr8rDpSCHmTd7tzixFA0tVPBsvToA5Cz2MahJ+vmouusiWq/2YzGNE4zlzezNZ1 +3dgsVJm67xUSs0qY5ipKzButCFSKnaj1hLNR1NsUd0NPrVBTGblxULLuD99GhoXk +/pcM5dCGTUX7XIarSFTEgBNQytpmfgt1Xbw2ErmlAdiFb4/5uBdbsVFAjglBvRI5 +VhFXr7mUd+XR/23aRczdAnp+Zg7VvyaJQi0ZwEj7VvLzpSAneVrxEcnuc2MBkUgT +TN/Z5LYqC93nr6vB7+HMwoBZ8hBAkO4rTKYQl3eMUSkIsE45CqI7Hz0eXQARAQAB +/gcDAqG5KzRnSp/38h4JKzJhSBRyyBPrgpYqR6ivFABzPUPJjO0gqRYzx/C+HJyl +z+QED0WH+sW8Ns4PkAgNWZ+225fzSssavLcPwjncy9pzcV+7bc76cFb77fSve+1D +LxhpzN58q03cSXPoamcDD7yY8GYYkAquLDZw+eRQ57BbBrNjXyfpGkBmtULymLqZ +SgkuV5we7//lRPDIuPk+9lszJXBUW3k5e32CR47B/hI6Pu0DTlN9VesAEmXRNsi9 +YlRiO74nGPQPEWGjnEUQ++W8ip0CzoSrmPhrdGQlSR+SBEbBCuXz1lsj7D9cBxwH +qHgwhYKvWz/gaY702+i/S1Cu/PjEpY3WPC5oSSNSSgypD8uSpcb4s2LffIegJNck +e1AuiovG6u/3QXPot0jHhdy+Qwe+oaJfSEBGQ4fD4W6GbPxwOIQGgXV0bRaeHYgL +iUWbN3rTLLVfDJKVo2ahvqZ7i4whfMuu1gGWQ4OEizrCDqp0x48HchEOC+T1eP3T +Zjth2YMtzZdXlpt5HNKeaY6ZP+NWILwvOQBd3UtNpeaCNhUa0YyB7GD/k7tZnCIZ +aNyF/DpnRrSQ9mAOffVn2NDGUv+01LnhIfa2tJes9XPmTc6ASrn/RGE9xH0X7wBD +HfAdGhHgbkzwNeYkQvSh1WyWj5C0Sq7X70dIYdcO81i5MMtlJrzrlB5/YCFVWSxt +7/EqwMBT3g9mkjAqo6beHxI1Hukn9rt9A6+MU64r0/cB+mVZuiBDoU/+KIiXBWiE +F/C1n/BO115WoWG35vj5oH+syuv3lRuPaz8GxoffcT+FUkmevZO1/BjEAABAwMS1 +nlB4y6xMJ0i2aCB2kp7ThDOOeTIQpdvtDLqRtQsVTpk73AEuDeKmULJnE2+Shi7v +yrNj1CPiBdYzz8jBDJYQH87iFQrro7VQNZzMMxpMWXQOZYWidHuBz4TgJJ0ll0JN +KwLtqv5wdf2zG8zNli0Dz+JwiwQ1kXDcA03rxHBCFALvkdIX0KUvTaTSV7OJ65VI +rcIwB5fSZgRE7m/9RjBGq/U+n4Kw+vlfpL7UeECJM0N7l8ekgTqqKv2Czu29eTjF +QOnpQtjgsWVpOnHKpQUfCN1Nxg8H1ytH9HQwLn+cGjm/yK55yIK+03X/vSs2m2qz +2zDhWlgvHLsDOEQkNsuOIvLkNM6Hv3MLTldknC+vMla34fYqpHfV1phL4npVByMW +CFOOzLa3qCoBXIGWvtnDx06r/8apHnt256G2X0iuRWWK+XpatMjmriZnj8vyGdIg +TZ1sNXnuFKMcXYMIvLANZXz4Rabbe6tTJ+BUVkbCGja4Z9iwmYvga77Mr2vjhtwi +CesRpcz6gR6U5fLddRZXyzKGxC3uQzokc9RtTuRNgSBZQ0oki++d6sr0+jOb54Mr +wfcMbMgpkQK0IJsMoOxzPLU8s6rISJvFi4IQ2dPYog17GS7Kjb1IGjGUxNKVHiIE +Is9wB+6bB51ZUUwc0zDSkuS6EaXLLVzmS7a3TOkVzu6J760TDVLL2+PDYkkBUP6O +SA2yeHirpyMma9QII1sw3xcKH/kDeyWigiB1VDKQpuq1PP98lYjQwAbe3Xrpy2FO +L/v6dSOJ+imgxD4osT0SanGkZEwPqJFvs6BI9Af8q9ia0xfK3Iu6F2F8JxmG1YiR +tUm9kCu3X/fNyE08G2sxD8QzGP9VS529nEDRBqkAgY6EHTpRKhPer9QrkUnqEyDZ +4s7RPcJW+cII/FPW8mSMgTqxFtTZgqNaqPPLevrTnTYTdrW/RkEs1mm0FWZvbyBi +YXIgPGZvb0Bmb28uZm9vPokCVAQTAQgAPhYhBJICM5a3zdmD+nRGF3grx+nZaj4C +BQJbdjh4AhsDBQkDwmcABQsJCAcCBhUICQoLAgQWAgMBAh4BAheAAAoJEHgrx+nZ +aj4CTyUP/2+4k4hXkkBrEeD0yDpmR/FrAgCOZ3iRWca9bJwKtV0hW0HSztlPEfng +wkwBmmyrnDevA+Ur4/hsBoTzfL4Fzo4OQDg2PZpSpIAHC1m/SQMN/s188RM8eK+Q +JBtinAo2IDoZyBi5Ar4rVNXrRpgvzwOLm15kpuPp15wxO+4gYOkNIT06yUrDNh3J +ccXmgZoVD54JmvKrEXscqX71/1NkaUhwZfFALN3+TVXUUdv1icQUJtxNBc29arwM +LuPuj9XAm5XJaVXDfsJyGu4aj4g6AJDXjVW1d2MgXv1rMRud7CGuX2PmO3CUUua9 +cUaavop5AmtF/+IsHae9qRt8PiMGTebV8IZ3Z6DZeOYDnfJVOXoIUcrAvX3LoImc +ephBdZ0KmYvaxlDrjtWAvmD6sPgwSvjLiXTmbmAkjRBXCVve4THf05kVUMcr8tmz +Il8LB+Dri2TfanBKykf6ulH0p2MHgSGQbYA5MuSp+soOitD5YvCxM7o/O0frrfit +p/O8mPerMEaYF1+3QbF5ApJkXCmjFCj71EPwXEDcl3VIGc+zA49oNjZMMmCcX2Gc +JyKTWizfuRBGeG5VhCCmTQQjZHPMVO255mdzsPkb6ZHEnolDapY6QXccV5x05XqD +sObFTy6iwEITdGmxN40pNE3WbhYGqOoXb8iRIG2hURv0gfG1/iI0 +=8g3t -----END PGP PRIVATE KEY BLOCK----- ` + +const keyPassphrase = "abcdef0123456789" From 8cf7edbc99282245bc8803a322dbf499ab77575d Mon Sep 17 00:00:00 2001 From: "Santiago M. Mola" Date: Fri, 17 Aug 2018 12:18:06 +0200 Subject: [PATCH 068/120] dotgit: fix object delete test Signed-off-by: Santiago M. Mola --- storage/filesystem/dotgit/dotgit_test.go | 32 +++++++++++++----------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/storage/filesystem/dotgit/dotgit_test.go b/storage/filesystem/dotgit/dotgit_test.go index ce3050039..64c2aeead 100644 --- a/storage/filesystem/dotgit/dotgit_test.go +++ b/storage/filesystem/dotgit/dotgit_test.go @@ -9,11 +9,11 @@ import ( "strings" "testing" - fixtures "gopkg.in/src-d/go-git-fixtures.v3" "gopkg.in/src-d/go-git.v4/plumbing" . "gopkg.in/check.v1" "gopkg.in/src-d/go-billy.v4/osfs" + "gopkg.in/src-d/go-git-fixtures.v3" ) func Test(t *testing.T) { TestingT(t) } @@ -571,19 +571,23 @@ func (s *SuiteDotGit) TestObjectDelete(c *C) { hash := plumbing.NewHash("03db8e1fbe133a480f2867aac478fd866686d69e") err := dir.ObjectDelete(hash) c.Assert(err, IsNil) - //incomingHash := "9d25e0f9bde9f82882b49fe29117b9411cb157b7" //made up hash - //incomingDirPath := fs.Join("objects", "incoming-123456") - //incomingSubDirPath := fs.Join(incomingDirPath, incomingHash[0:2]) - //incomingFilePath := fs.Join(incomingDirPath, incomingHash[2:40]) - //err = fs.MkdirAll(incomingDirPath, os.FileMode(0755)) - //c.Assert(err, IsNil) - //err = fs.MkdirAll(incomingSubDirPath, os.FileMode(0755)) - //c.Assert(err, IsNil) - //_, err = fs.Create(incomingFilePath) - //c.Assert(err, IsNil) - - //err = dir.ObjectDelete(plumbing.NewHash(incomingHash)) - //c.Assert(err, IsNil) + + incomingHash := "9d25e0f9bde9f82882b49fe29117b9411cb157b7" //made up hash + incomingDirPath := fs.Join("objects", "incoming-123456") + incomingSubDirPath := fs.Join(incomingDirPath, incomingHash[0:2]) + incomingFilePath := fs.Join(incomingSubDirPath, incomingHash[2:40]) + + err = fs.MkdirAll(incomingSubDirPath, os.FileMode(0755)) + c.Assert(err, IsNil) + + f, err := fs.Create(incomingFilePath) + c.Assert(err, IsNil) + + err = f.Close() + c.Assert(err, IsNil) + + err = dir.ObjectDelete(plumbing.NewHash(incomingHash)) + c.Assert(err, IsNil) } func (s *SuiteDotGit) TestObjectNotFound(c *C) { From 166623633e285e17b0582443c9d03b842b6370fa Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Fri, 17 Aug 2018 18:52:18 +0200 Subject: [PATCH 069/120] object: fix panic when reading object header When the first line of the pgp signature is an empty line or some header is malformed it crashes as there's no data for the header element. For example, if author name is "\n". Signed-off-by: Javi Fontan --- plumbing/object/commit.go | 16 +++++++++++----- plumbing/object/commit_test.go | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 5 deletions(-) diff --git a/plumbing/object/commit.go b/plumbing/object/commit.go index 00ae3f144..e2543426a 100644 --- a/plumbing/object/commit.go +++ b/plumbing/object/commit.go @@ -199,17 +199,23 @@ func (c *Commit) Decode(o plumbing.EncodedObject) (err error) { } split := bytes.SplitN(line, []byte{' '}, 2) + + var data []byte + if len(split) == 2 { + data = split[1] + } + switch string(split[0]) { case "tree": - c.TreeHash = plumbing.NewHash(string(split[1])) + c.TreeHash = plumbing.NewHash(string(data)) case "parent": - c.ParentHashes = append(c.ParentHashes, plumbing.NewHash(string(split[1]))) + c.ParentHashes = append(c.ParentHashes, plumbing.NewHash(string(data))) case "author": - c.Author.Decode(split[1]) + c.Author.Decode(data) case "committer": - c.Committer.Decode(split[1]) + c.Committer.Decode(data) case headerpgp: - c.PGPSignature += string(split[1]) + "\n" + c.PGPSignature += string(data) + "\n" pgpsig = true } } else { diff --git a/plumbing/object/commit_test.go b/plumbing/object/commit_test.go index b5dfbe325..e72b703d1 100644 --- a/plumbing/object/commit_test.go +++ b/plumbing/object/commit_test.go @@ -325,6 +325,22 @@ RUysgqjcpT8+iQM1PblGfHR4XAhuOqN5Fx06PSaFZhqvWFezJ28/CLyX5q+oIVk= c.Assert(err, IsNil) c.Assert(decoded.PGPSignature, Equals, pgpsignature) + // signature with extra empty line, it caused "index out of range" when + // parsing it + + pgpsignature2 := "\n" + pgpsignature + + commit.PGPSignature = pgpsignature2 + encoded = &plumbing.MemoryObject{} + decoded = &Commit{} + + err = commit.Encode(encoded) + c.Assert(err, IsNil) + + err = decoded.Decode(encoded) + c.Assert(err, IsNil) + c.Assert(decoded.PGPSignature, Equals, pgpsignature2) + // signature in author name commit.PGPSignature = "" @@ -461,3 +477,21 @@ func (s *SuiteCommit) TestPatchCancel(c *C) { c.Assert(err, ErrorMatches, "operation canceled") } + +func (s *SuiteCommit) TestMalformedHeader(c *C) { + encoded := &plumbing.MemoryObject{} + decoded := &Commit{} + commit := *s.Commit + + commit.PGPSignature = "\n" + commit.Author.Name = "\n" + commit.Author.Email = "\n" + commit.Committer.Name = "\n" + commit.Committer.Email = "\n" + + err := commit.Encode(encoded) + c.Assert(err, IsNil) + + err = decoded.Decode(encoded) + c.Assert(err, IsNil) +} From 7b4a8379327653167e87e9e46a2d397f8fd9cfc8 Mon Sep 17 00:00:00 2001 From: Fedor Korotkov Date: Sun, 19 Aug 2018 11:51:21 -0400 Subject: [PATCH 070/120] Fixed an edge case for .gitignore Fixes #923 Signed-off-by: Fedor Korotkov --- plumbing/format/gitignore/pattern.go | 3 +++ plumbing/format/gitignore/pattern_test.go | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/plumbing/format/gitignore/pattern.go b/plumbing/format/gitignore/pattern.go index 26033522b..098cb5021 100644 --- a/plumbing/format/gitignore/pattern.go +++ b/plumbing/format/gitignore/pattern.go @@ -133,6 +133,9 @@ func (p *pattern) globMatch(path []string, isDir bool) bool { } else if match { matched = true break + } else if len(path) == 0 { + // if nothing left then fail + matched = false } } } else { diff --git a/plumbing/format/gitignore/pattern_test.go b/plumbing/format/gitignore/pattern_test.go index f94cef37a..c410442b6 100644 --- a/plumbing/format/gitignore/pattern_test.go +++ b/plumbing/format/gitignore/pattern_test.go @@ -281,3 +281,9 @@ func (s *PatternSuite) TestGlobMatch_wrongPattern_onTraversal_mismatch(c *C) { r := p.Match([]string{"value", "head", "vol["}, false) c.Assert(r, Equals, NoMatch) } + +func (s *PatternSuite) TestGlobMatch_issue_923(c *C) { + p := ParsePattern("**/android/**/GeneratedPluginRegistrant.java", nil) + r := p.Match([]string{"packages", "flutter_tools", "lib", "src", "android", "gradle.dart"}, false) + c.Assert(r, Equals, NoMatch) +} From f84c6b194f2eced0c068b9cdc9264d30e6d2021b Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Tue, 21 Aug 2018 17:35:52 +0200 Subject: [PATCH 071/120] plumbing/idxfile: object iterators returns entries in offset order In the latest change the order was changed from offset order in packfiles to hash order. This makes reading all the objects not as efficient as before. It also created problems when the previous order was expected. Also added EntriesByOffset to indexes. Signed-off-by: Javi Fontan --- plumbing/format/idxfile/idxfile.go | 69 +++++++++++++++++++++++++ plumbing/format/idxfile/idxfile_test.go | 15 ++++++ plumbing/format/packfile/packfile.go | 2 +- 3 files changed, 85 insertions(+), 1 deletion(-) diff --git a/plumbing/format/idxfile/idxfile.go b/plumbing/format/idxfile/idxfile.go index c977beedb..5fed278b1 100644 --- a/plumbing/format/idxfile/idxfile.go +++ b/plumbing/format/idxfile/idxfile.go @@ -3,6 +3,7 @@ package idxfile import ( "bytes" "io" + "sort" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/utils/binary" @@ -34,6 +35,9 @@ type Index interface { Count() (int64, error) // Entries returns an iterator to retrieve all index entries. Entries() (EntryIter, error) + // EntriesByOffset returns an iterator to retrieve all index entries ordered + // by offset. + EntriesByOffset() (EntryIter, error) } // MemoryIndex is the in memory representation of an idx file. @@ -215,6 +219,36 @@ func (idx *MemoryIndex) Entries() (EntryIter, error) { return &idxfileEntryIter{idx, 0, 0, 0}, nil } +// EntriesByOffset implements the Index interface. +func (idx *MemoryIndex) EntriesByOffset() (EntryIter, error) { + count, err := idx.Count() + if err != nil { + return nil, err + } + + iter := &idxfileEntryOffsetIter{ + entries: make(entriesByOffset, count), + } + + entries, err := idx.Entries() + if err != nil { + return nil, err + } + + for pos := 0; int64(pos) < count; pos++ { + entry, err := entries.Next() + if err != nil { + return nil, err + } + + iter.entries[pos] = entry + } + + sort.Sort(iter.entries) + + return iter, nil +} + // EntryIter is an iterator that will return the entries in a packfile index. type EntryIter interface { // Next returns the next entry in the packfile index. @@ -276,3 +310,38 @@ type Entry struct { CRC32 uint32 Offset uint64 } + +type idxfileEntryOffsetIter struct { + entries entriesByOffset + pos int +} + +func (i *idxfileEntryOffsetIter) Next() (*Entry, error) { + if i.pos >= len(i.entries) { + return nil, io.EOF + } + + entry := i.entries[i.pos] + i.pos++ + + return entry, nil +} + +func (i *idxfileEntryOffsetIter) Close() error { + i.pos = len(i.entries) + 1 + return nil +} + +type entriesByOffset []*Entry + +func (o entriesByOffset) Len() int { + return len(o) +} + +func (o entriesByOffset) Less(i int, j int) bool { + return o[i].Offset < o[j].Offset +} + +func (o entriesByOffset) Swap(i int, j int) { + o[i], o[j] = o[j], o[i] +} diff --git a/plumbing/format/idxfile/idxfile_test.go b/plumbing/format/idxfile/idxfile_test.go index d15accf3a..0e0ca2a22 100644 --- a/plumbing/format/idxfile/idxfile_test.go +++ b/plumbing/format/idxfile/idxfile_test.go @@ -115,6 +115,21 @@ func (s *IndexSuite) TestFindHash(c *C) { } } +func (s *IndexSuite) TestEntriesByOffset(c *C) { + idx, err := fixtureIndex() + c.Assert(err, IsNil) + + entries, err := idx.EntriesByOffset() + c.Assert(err, IsNil) + + for _, pos := range fixtureOffsets { + e, err := entries.Next() + c.Assert(err, IsNil) + + c.Assert(e.Offset, Equals, uint64(pos)) + } +} + var fixtureHashes = []plumbing.Hash{ plumbing.NewHash("303953e5aa461c203a324821bc1717f9b4fff895"), plumbing.NewHash("5296768e3d9f661387ccbff18c4dea6c997fd78c"), diff --git a/plumbing/format/packfile/packfile.go b/plumbing/format/packfile/packfile.go index 5feb78142..18fcca7e8 100644 --- a/plumbing/format/packfile/packfile.go +++ b/plumbing/format/packfile/packfile.go @@ -394,7 +394,7 @@ func (p *Packfile) GetByType(typ plumbing.ObjectType) (storer.EncodedObjectIter, plumbing.TreeObject, plumbing.CommitObject, plumbing.TagObject: - entries, err := p.Entries() + entries, err := p.EntriesByOffset() if err != nil { return nil, err } From b9f5efe3dbee0cd15a553bcc03f7a37f3dcaa676 Mon Sep 17 00:00:00 2001 From: Chris Marchesi Date: Sat, 18 Aug 2018 11:03:33 -0700 Subject: [PATCH 072/120] git: Add tagging support This adds a few methods: * CreateTag, which can be used to create both lightweight and annotated tags with a supplied TagObjectOptions struct. PGP signing is possible as well. * Tag, to fetch a single tag ref. As opposed to Tags or TagObjects, this will also fetch the tag object if it exists and return it along with the output. Lightweight tags just return the object as nil. * DeleteTag, to delete a tag. This simply deletes the ref. The object is left orphaned to be GCed later. I'm not 100% sure if DeleteTag is the correct behavior - looking for details on exactly *what* happens to a tag object if you delete the ref and not the tag were sparse, and groking the Git source did not really produce much insight to the untrained eye. This may be something that comes up in review. If deletion of the object is necessary, the in-memory storer may require some updates to allow DeleteLooseObject to be supported. Signed-off-by: Chris Marchesi --- options.go | 47 +++++++- repository.go | 134 ++++++++++++++++++++++- repository_test.go | 265 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 443 insertions(+), 3 deletions(-) diff --git a/options.go b/options.go index 7b1570f7b..6b00b0d02 100644 --- a/options.go +++ b/options.go @@ -348,8 +348,9 @@ type CommitOptions struct { // Parents are the parents commits for the new commit, by default when // len(Parents) is zero, the hash of HEAD reference is used. Parents []plumbing.Hash - // A key to sign the commit with. A nil value here means the commit will not - // be signed. The private key must be present and already decrypted. + // SignKey denotes a key to sign the commit with. A nil value here means the + // commit will not be signed. The private key must be present and already + // decrypted. SignKey *openpgp.Entity } @@ -377,6 +378,48 @@ func (o *CommitOptions) Validate(r *Repository) error { return nil } +var ( + ErrMissingName = errors.New("name field is required") + ErrMissingTagger = errors.New("tagger field is required") + ErrMissingMessage = errors.New("message field is required") + ErrBadObjectType = errors.New("bad object type for tagging") +) + +// TagObjectOptions describes how a tag object should be created. +type TagObjectOptions struct { + // Tagger defines the signature of the tag creator. + Tagger *object.Signature + // Message defines the annotation of the tag. + Message string + // TargetType is the object type of the target. The object specified by + // Target must be of this type. + TargetType plumbing.ObjectType + // SignKey denotes a key to sign the tag with. A nil value here means the tag + // will not be signed. The private key must be present and already decrypted. + SignKey *openpgp.Entity +} + +// Validate validates the fields and sets the default values. +func (o *TagObjectOptions) Validate(r *Repository, hash plumbing.Hash) error { + if o.Tagger == nil { + return ErrMissingTagger + } + + if o.Message == "" { + return ErrMissingMessage + } + + if o.TargetType == plumbing.InvalidObject || o.TargetType == plumbing.AnyObject { + return ErrBadObjectType + } + + if _, err := r.Object(o.TargetType, hash); err != nil { + return err + } + + return nil +} + // ListOptions describes how a remote list should be performed. type ListOptions struct { // Auth credentials, if required, to use with the remote repository. diff --git a/repository.go b/repository.go index 818cfb3c3..ab14eba0b 100644 --- a/repository.go +++ b/repository.go @@ -1,15 +1,18 @@ package git import ( + "bytes" "context" "errors" "fmt" stdioutil "io/ioutil" "os" + "path" "path/filepath" "strings" "time" + "golang.org/x/crypto/openpgp" "gopkg.in/src-d/go-git.v4/config" "gopkg.in/src-d/go-git.v4/internal/revision" "gopkg.in/src-d/go-git.v4/plumbing" @@ -31,7 +34,12 @@ var ( // ErrBranchExists an error stating the specified branch already exists ErrBranchExists = errors.New("branch already exists") // ErrBranchNotFound an error stating the specified branch does not exist - ErrBranchNotFound = errors.New("branch not found") + ErrBranchNotFound = errors.New("branch not found") + // ErrTagExists an error stating the specified tag already exists + ErrTagExists = errors.New("tag already exists") + // ErrTagNotFound an error stating the specified tag does not exist + ErrTagNotFound = errors.New("tag not found") + ErrInvalidReference = errors.New("invalid reference, should be a tag or a branch") ErrRepositoryNotExists = errors.New("repository does not exist") ErrRepositoryAlreadyExists = errors.New("repository already exists") @@ -484,6 +492,130 @@ func (r *Repository) DeleteBranch(name string) error { return r.Storer.SetConfig(cfg) } +// CreateTag creates a tag. If opts is included, the tag is an annotated tag, +// otherwise a lightweight tag is created. +func (r *Repository) CreateTag(name string, hash plumbing.Hash, opts *TagObjectOptions) (*plumbing.Reference, error) { + rname := plumbing.ReferenceName(path.Join("refs", "tags", name)) + + _, err := r.Storer.Reference(rname) + switch err { + case nil: + // Tag exists, this is an error + return nil, ErrTagExists + case plumbing.ErrReferenceNotFound: + // Tag missing, available for creation, pass this + default: + // Some other error + return nil, err + } + + var target plumbing.Hash + if opts != nil { + target, err = r.createTagObject(name, hash, opts) + if err != nil { + return nil, err + } + } else { + target = hash + } + + ref := plumbing.NewHashReference(rname, target) + if err = r.Storer.SetReference(ref); err != nil { + return nil, err + } + + return ref, nil +} + +func (r *Repository) createTagObject(name string, hash plumbing.Hash, opts *TagObjectOptions) (plumbing.Hash, error) { + if err := opts.Validate(r, hash); err != nil { + return plumbing.ZeroHash, err + } + + tag := &object.Tag{ + Name: name, + Tagger: *opts.Tagger, + Message: opts.Message, + TargetType: opts.TargetType, + Target: hash, + } + + if opts.SignKey != nil { + sig, err := r.buildTagSignature(tag, opts.SignKey) + if err != nil { + return plumbing.ZeroHash, err + } + + tag.PGPSignature = sig + } + + obj := r.Storer.NewEncodedObject() + if err := tag.Encode(obj); err != nil { + return plumbing.ZeroHash, err + } + + return r.Storer.SetEncodedObject(obj) +} + +func (r *Repository) buildTagSignature(tag *object.Tag, signKey *openpgp.Entity) (string, error) { + encoded := &plumbing.MemoryObject{} + if err := tag.Encode(encoded); err != nil { + return "", err + } + + rdr, err := encoded.Reader() + if err != nil { + return "", err + } + + var b bytes.Buffer + if err := openpgp.ArmoredDetachSign(&b, signKey, rdr, nil); err != nil { + return "", err + } + + return b.String(), nil +} + +// Tag fetches a tag from the repository. The tag is returned as a raw +// reference. If the tag is annotated, a non-nil tag object is returned. +func (r *Repository) Tag(name string) (*plumbing.Reference, *object.Tag, error) { + ref, err := r.Reference(plumbing.ReferenceName(path.Join("refs", "tags", name)), false) + if err != nil { + if err == plumbing.ErrReferenceNotFound { + // Return a friendly error for this one, versus just ReferenceNotFound. + return nil, nil, ErrTagNotFound + } + + return nil, nil, err + } + + obj, err := r.TagObject(ref.Hash()) + if err != nil && err != plumbing.ErrObjectNotFound { + return nil, nil, err + } + + return ref, obj, nil +} + +// DeleteTag deletes a tag from the repository. +func (r *Repository) DeleteTag(name string) error { + _, obj, err := r.Tag(name) + if err != nil { + return err + } + + if err = r.Storer.RemoveReference(plumbing.ReferenceName(path.Join("refs", "tags", name))); err != nil { + return err + } + + // Delete the tag object if this was an annotated tag. + if obj != nil { + return r.DeleteObject(obj.Hash) + } + + return nil +} + func (r *Repository) resolveToCommitHash(h plumbing.Hash) (plumbing.Hash, error) { obj, err := r.Storer.EncodedObject(plumbing.AnyObject, h) if err != nil { diff --git a/repository_test.go b/repository_test.go index 261af7a7b..4fd26a189 100644 --- a/repository_test.go +++ b/repository_test.go @@ -13,6 +13,9 @@ import ( "testing" "time" + "golang.org/x/crypto/openpgp" + "golang.org/x/crypto/openpgp/armor" + openpgperr "golang.org/x/crypto/openpgp/errors" "gopkg.in/src-d/go-git.v4/config" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/object" @@ -1275,6 +1278,268 @@ func (s *RepositorySuite) TestTags(c *C) { c.Assert(count, Equals, 5) } +func (s *RepositorySuite) TestCreateTagLightweight(c *C) { + url := s.GetLocalRepositoryURL( + fixtures.ByURL("https://github.com/git-fixtures/tags.git").One(), + ) + + r, _ := Init(memory.NewStorage(), nil) + err := r.clone(context.Background(), &CloneOptions{URL: url}) + c.Assert(err, IsNil) + + expected, err := r.Head() + c.Assert(err, IsNil) + + ref, err := r.CreateTag("foobar", expected.Hash(), nil) + c.Assert(err, IsNil) + c.Assert(ref, NotNil) + + actual, obj, err := r.Tag("foobar") + c.Assert(err, IsNil) + c.Assert(obj, IsNil) + + c.Assert(expected.Hash(), Equals, actual.Hash()) +} + +func (s *RepositorySuite) TestCreateTagLightweightExists(c *C) { + url := s.GetLocalRepositoryURL( + fixtures.ByURL("https://github.com/git-fixtures/tags.git").One(), + ) + + r, _ := Init(memory.NewStorage(), nil) + err := r.clone(context.Background(), &CloneOptions{URL: url}) + c.Assert(err, IsNil) + + expected, err := r.Head() + c.Assert(err, IsNil) + + ref, err := r.CreateTag("lightweight-tag", expected.Hash(), nil) + c.Assert(ref, IsNil) + c.Assert(err, Equals, ErrTagExists) +} + +func (s *RepositorySuite) TestCreateTagAnnotated(c *C) { + url := s.GetLocalRepositoryURL( + fixtures.ByURL("https://github.com/git-fixtures/tags.git").One(), + ) + + r, _ := Init(memory.NewStorage(), nil) + err := r.clone(context.Background(), &CloneOptions{URL: url}) + c.Assert(err, IsNil) + + h, err := r.Head() + c.Assert(err, IsNil) + + expectedHash := h.Hash() + + ref, err := r.CreateTag("foobar", expectedHash, &TagObjectOptions{ + Tagger: defaultSignature(), + Message: "foo bar baz qux", + TargetType: plumbing.CommitObject, + }) + c.Assert(err, IsNil) + + tag, obj, err := r.Tag("foobar") + c.Assert(err, IsNil) + c.Assert(obj, NotNil) + + c.Assert(ref, DeepEquals, tag) + c.Assert(obj.Hash, Equals, ref.Hash()) + c.Assert(obj.Type(), Equals, plumbing.TagObject) + c.Assert(obj.Target, Equals, expectedHash) +} + +func (s *RepositorySuite) TestCreateTagAnnotatedBadOpts(c *C) { + url := s.GetLocalRepositoryURL( + fixtures.ByURL("https://github.com/git-fixtures/tags.git").One(), + ) + + r, _ := Init(memory.NewStorage(), nil) + err := r.clone(context.Background(), &CloneOptions{URL: url}) + c.Assert(err, IsNil) + + h, err := r.Head() + c.Assert(err, IsNil) + + expectedHash := h.Hash() + + ref, err := r.CreateTag("foobar", expectedHash, &TagObjectOptions{ + Message: "foo bar baz qux", + TargetType: plumbing.CommitObject, + }) + c.Assert(ref, IsNil) + c.Assert(err, Equals, ErrMissingTagger) + + ref, err = r.CreateTag("foobar", expectedHash, &TagObjectOptions{ + Tagger: defaultSignature(), + TargetType: plumbing.CommitObject, + }) + c.Assert(ref, IsNil) + c.Assert(err, Equals, ErrMissingMessage) + + ref, err = r.CreateTag("foobar", expectedHash, &TagObjectOptions{ + Tagger: defaultSignature(), + Message: "foo bar baz qux", + }) + c.Assert(ref, IsNil) + c.Assert(err, Equals, ErrBadObjectType) + + ref, err = r.CreateTag("foobar", expectedHash, &TagObjectOptions{ + Tagger: defaultSignature(), + Message: "foo bar baz qux", + TargetType: plumbing.TagObject, + }) + c.Assert(ref, IsNil) + c.Assert(err, Equals, plumbing.ErrObjectNotFound) +} + +func (s *RepositorySuite) TestCreateTagSigned(c *C) { + url := s.GetLocalRepositoryURL( + fixtures.ByURL("https://github.com/git-fixtures/tags.git").One(), + ) + + r, _ := Init(memory.NewStorage(), nil) + err := r.clone(context.Background(), &CloneOptions{URL: url}) + c.Assert(err, IsNil) + + h, err := r.Head() + c.Assert(err, IsNil) + + key := commitSignKey(c, true) + _, err = r.CreateTag("foobar", h.Hash(), &TagObjectOptions{ + Tagger: defaultSignature(), + Message: "foo bar baz qux", + TargetType: plumbing.CommitObject, + SignKey: key, + }) + c.Assert(err, IsNil) + + _, obj, err := r.Tag("foobar") + c.Assert(err, IsNil) + c.Assert(obj, NotNil) + + // Verify the tag. + pks := new(bytes.Buffer) + pkw, err := armor.Encode(pks, openpgp.PublicKeyType, nil) + c.Assert(err, IsNil) + + err = key.Serialize(pkw) + c.Assert(err, IsNil) + err = pkw.Close() + c.Assert(err, IsNil) + + actual, err := obj.Verify(pks.String()) + c.Assert(err, IsNil) + c.Assert(actual.PrimaryKey, DeepEquals, key.PrimaryKey) +} + +func (s *RepositorySuite) TestCreateTagSignedBadKey(c *C) { + url := s.GetLocalRepositoryURL( + fixtures.ByURL("https://github.com/git-fixtures/tags.git").One(), + ) + + r, _ := Init(memory.NewStorage(), nil) + err := r.clone(context.Background(), &CloneOptions{URL: url}) + c.Assert(err, IsNil) + + h, err := r.Head() + c.Assert(err, IsNil) + + key := commitSignKey(c, false) + _, err = r.CreateTag("foobar", h.Hash(), &TagObjectOptions{ + Tagger: defaultSignature(), + Message: "foo bar baz qux", + TargetType: plumbing.CommitObject, + SignKey: key, + }) + c.Assert(err, Equals, openpgperr.InvalidArgumentError("signing key is encrypted")) +} + +func (s *RepositorySuite) TestTagLightweight(c *C) { + url := s.GetLocalRepositoryURL( + fixtures.ByURL("https://github.com/git-fixtures/tags.git").One(), + ) + + r, _ := Init(memory.NewStorage(), nil) + err := r.clone(context.Background(), &CloneOptions{URL: url}) + c.Assert(err, IsNil) + + expected := plumbing.NewHash("f7b877701fbf855b44c0a9e86f3fdce2c298b07f") + + tag, obj, err := r.Tag("lightweight-tag") + c.Assert(err, IsNil) + c.Assert(obj, IsNil) + + actual := tag.Hash() + c.Assert(expected, Equals, actual) +} + +func (s *RepositorySuite) TestTagLightweightMissingTag(c *C) { + url := s.GetLocalRepositoryURL( + fixtures.ByURL("https://github.com/git-fixtures/tags.git").One(), + ) + + r, _ := Init(memory.NewStorage(), nil) + err := r.clone(context.Background(), &CloneOptions{URL: url}) + c.Assert(err, IsNil) + + tag, obj, err := r.Tag("lightweight-tag-tag") + c.Assert(tag, IsNil) + c.Assert(obj, IsNil) + c.Assert(err, Equals, ErrTagNotFound) +} + +func (s *RepositorySuite) TestTagAnnotated(c *C) { + url := s.GetLocalRepositoryURL( + fixtures.ByURL("https://github.com/git-fixtures/tags.git").One(), + ) + + r, _ := Init(memory.NewStorage(), nil) + err := r.clone(context.Background(), &CloneOptions{URL: url}) + c.Assert(err, IsNil) + + tag, obj, err := r.Tag("annotated-tag") + c.Assert(err, IsNil) + c.Assert(obj, NotNil) + + expectedHash := plumbing.NewHash("b742a2a9fa0afcfa9a6fad080980fbc26b007c69") + expectedTarget := plumbing.NewHash("f7b877701fbf855b44c0a9e86f3fdce2c298b07f") + actualHash := tag.Hash() + c.Assert(expectedHash, Equals, actualHash) + c.Assert(obj.Hash, Equals, expectedHash) + c.Assert(obj.Type(), Equals, plumbing.TagObject) + c.Assert(obj.Target, Equals, expectedTarget) +} + +func (s *RepositorySuite) TestDeleteTag(c *C) { + url := s.GetLocalRepositoryURL( + fixtures.ByURL("https://github.com/git-fixtures/tags.git").One(), + ) + + r, _ := Init(memory.NewStorage(), nil) + err := r.clone(context.Background(), &CloneOptions{URL: url}) + c.Assert(err, IsNil) + + err = r.DeleteTag("lightweight-tag") + c.Assert(err, IsNil) + + _, _, err = r.Tag("lightweight-tag") + c.Assert(err, Equals, ErrTagNotFound) +} + +func (s *RepositorySuite) TestDeleteTagMissingTag(c *C) { + url := s.GetLocalRepositoryURL( + fixtures.ByURL("https://github.com/git-fixtures/tags.git").One(), + ) + + r, _ := Init(memory.NewStorage(), nil) + err := r.clone(context.Background(), &CloneOptions{URL: url}) + c.Assert(err, IsNil) + + err = r.DeleteTag("lightweight-tag-tag") + c.Assert(err, Equals, ErrTagNotFound) +} + func (s *RepositorySuite) TestBranches(c *C) { f := fixtures.ByURL("https://github.com/git-fixtures/root-references.git").One() sto, err := filesystem.NewStorage(f.DotGit()) From 9b73a3ead6559576cb017b09c41c23c251b5af1c Mon Sep 17 00:00:00 2001 From: Chris Marchesi Date: Tue, 21 Aug 2018 17:38:09 -0700 Subject: [PATCH 073/120] plumbing: object, correct tag PGP encoding As with the update in ec3d2a8, tag encoding needed to be corrected to ensure extra newlines were not being added in during tag object encoding, so that it did not corrupt the object for verification. Signed-off-by: Chris Marchesi --- plumbing/object/tag.go | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/plumbing/object/tag.go b/plumbing/object/tag.go index 905206bda..63549730f 100644 --- a/plumbing/object/tag.go +++ b/plumbing/object/tag.go @@ -195,13 +195,9 @@ func (t *Tag) encode(o plumbing.EncodedObject, includeSig bool) (err error) { return err } - if t.PGPSignature != "" && includeSig { - // Split all the signature lines and write with a newline at the end. - lines := strings.Split(t.PGPSignature, "\n") - for _, line := range lines { - if _, err = fmt.Fprintf(w, "%s\n", line); err != nil { - return err - } + if includeSig { + if _, err = fmt.Fprint(w, "\n"+t.PGPSignature); err != nil { + return err } } From 8c3c8b30b3394677d8eb16b159bfe9b4f61726a8 Mon Sep 17 00:00:00 2001 From: Chris Marchesi Date: Tue, 21 Aug 2018 22:20:09 -0700 Subject: [PATCH 074/120] plumbing: object, don't add extra newline on PGP signature Tag encoding/decoding seems to be a lot more sensitive to requiring the exact expected format in the object, which generally includes messages canonicalized so that they have a newline on the end (even if they didn't before). As such, the message should be written with the newline (no need for an extra), and the PGP signature right after that, which will be newline split already, so there's no need to split it again. All of this means it's very important for the caller to send the message in the correct format - which I'm correcting in the next commit. Signed-off-by: Chris Marchesi --- plumbing/object/tag.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/plumbing/object/tag.go b/plumbing/object/tag.go index 63549730f..03749f9a4 100644 --- a/plumbing/object/tag.go +++ b/plumbing/object/tag.go @@ -195,8 +195,13 @@ func (t *Tag) encode(o plumbing.EncodedObject, includeSig bool) (err error) { return err } + // Note that this is highly sensitive to what it sent along in the message. + // Message *always* needs to end with a newline, or else the message and the + // signature will be concatenated into a corrupt object. Since this is a + // lower-level method, we assume you know what you are doing and have already + // done the needful on the message in the caller. if includeSig { - if _, err = fmt.Fprint(w, "\n"+t.PGPSignature); err != nil { + if _, err = fmt.Fprint(w, t.PGPSignature); err != nil { return err } } From 2c2b532a525ab2c05f6e9675efd529a052121713 Mon Sep 17 00:00:00 2001 From: Chris Marchesi Date: Tue, 21 Aug 2018 22:24:41 -0700 Subject: [PATCH 075/120] git: Canonicalize incoming annotated tag messages Tag messages are highly sensitive to being in the expected format, especially when encoding/decoding for PGP verification. As such, we do a simple trimming of whitespace on the incoming message and add a newline on the end, to ensure there are no surprises here. Signed-off-by: Chris Marchesi --- options.go | 8 +++++++- repository_test.go | 43 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/options.go b/options.go index 6b00b0d02..ed0baa3ad 100644 --- a/options.go +++ b/options.go @@ -3,6 +3,7 @@ package git import ( "errors" "regexp" + "strings" "golang.org/x/crypto/openpgp" "gopkg.in/src-d/go-git.v4/config" @@ -389,7 +390,9 @@ var ( type TagObjectOptions struct { // Tagger defines the signature of the tag creator. Tagger *object.Signature - // Message defines the annotation of the tag. + // Message defines the annotation of the tag. It is canonicalized during + // validation into the format expected by git - no leading whitespace and + // ending in a newline. Message string // TargetType is the object type of the target. The object specified by // Target must be of this type. @@ -409,6 +412,9 @@ func (o *TagObjectOptions) Validate(r *Repository, hash plumbing.Hash) error { return ErrMissingMessage } + // Canonicalize the message into the expected message format. + o.Message = strings.TrimSpace(o.Message) + "\n" + if o.TargetType == plumbing.InvalidObject || o.TargetType == plumbing.AnyObject { return ErrBadObjectType } diff --git a/repository_test.go b/repository_test.go index 4fd26a189..fd80152e7 100644 --- a/repository_test.go +++ b/repository_test.go @@ -1455,6 +1455,49 @@ func (s *RepositorySuite) TestCreateTagSignedBadKey(c *C) { c.Assert(err, Equals, openpgperr.InvalidArgumentError("signing key is encrypted")) } +func (s *RepositorySuite) TestCreateTagCanonicalize(c *C) { + url := s.GetLocalRepositoryURL( + fixtures.ByURL("https://github.com/git-fixtures/tags.git").One(), + ) + + r, _ := Init(memory.NewStorage(), nil) + err := r.clone(context.Background(), &CloneOptions{URL: url}) + c.Assert(err, IsNil) + + h, err := r.Head() + c.Assert(err, IsNil) + + key := commitSignKey(c, true) + _, err = r.CreateTag("foobar", h.Hash(), &TagObjectOptions{ + Tagger: defaultSignature(), + Message: "\n\nfoo bar baz qux\n\nsome message here", + TargetType: plumbing.CommitObject, + SignKey: key, + }) + c.Assert(err, IsNil) + + _, obj, err := r.Tag("foobar") + c.Assert(err, IsNil) + c.Assert(obj, NotNil) + + // Assert the new canonicalized message. + c.Assert(obj.Message, Equals, "foo bar baz qux\n\nsome message here\n") + + // Verify the tag. + pks := new(bytes.Buffer) + pkw, err := armor.Encode(pks, openpgp.PublicKeyType, nil) + c.Assert(err, IsNil) + + err = key.Serialize(pkw) + c.Assert(err, IsNil) + err = pkw.Close() + c.Assert(err, IsNil) + + actual, err := obj.Verify(pks.String()) + c.Assert(err, IsNil) + c.Assert(actual.PrimaryKey, DeepEquals, key.PrimaryKey) +} + func (s *RepositorySuite) TestTagLightweight(c *C) { url := s.GetLocalRepositoryURL( fixtures.ByURL("https://github.com/git-fixtures/tags.git").One(), From 8d8d4c52f42d5e5e7eccd8e51671b7d73d331ea0 Mon Sep 17 00:00:00 2001 From: Chris Marchesi Date: Tue, 21 Aug 2018 22:33:29 -0700 Subject: [PATCH 076/120] git: Replace test signing key with one with longer expiry The old one was created with defaults, which would have caused CI failures in 2 years. The new one is valid for 10 years: > gpg --list-secret-keys /root/.gnupg/pubring.kbx ------------------------ sec rsa4096 2018-08-22 [SC] [expires: 2028-08-19] 93A17FF01E54328546087C8E029395402EFCCD53 uid [ unknown] foo bar Signed-off-by: Chris Marchesi --- worktree_commit_test.go | 106 ++++++++++++++++++++-------------------- 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/worktree_commit_test.go b/worktree_commit_test.go index 004926fc5..fdeaec0c5 100644 --- a/worktree_commit_test.go +++ b/worktree_commit_test.go @@ -257,59 +257,59 @@ func commitSignKey(c *C, decrypt bool) *openpgp.Entity { const armoredKeyRing = ` -----BEGIN PGP PRIVATE KEY BLOCK----- -lQdGBFt2OHgBEADQpRmFm9X9xBfUljVs1B24MXWRHcEP5tx2k6Cp90sSz/ZOJcxH -RjzYuXjpkE7g/PaZxAMVS1PptJip/w1/+5l2gZ7RmzU/e3hKe4vALHzKMVp8t7Ta -0e2K3STxapCr9FNITjQRGOhnFwqiYoPCf9u5Iy8uszDH7HHnBZx+Nvbl95dDvmMs -aFUKMeaoFD19iwEdRu6gJo7YIWF/8zwHi49neKigisGKh5PI0KUYeRPydXeCZIKQ -ofdk+CPUS4r3dVhxTMYeHn/Vrep3blEA45E7KJ+TESmKkwliEgdjJwaVkUfJhBkb -p2pMPKwbxLma9GCJBimOkehFv8/S+xn/xrLSsTxeOCIzMp3I5vgjR5QfONq5kuB1 -qbr8rDpSCHmTd7tzixFA0tVPBsvToA5Cz2MahJ+vmouusiWq/2YzGNE4zlzezNZ1 -3dgsVJm67xUSs0qY5ipKzButCFSKnaj1hLNR1NsUd0NPrVBTGblxULLuD99GhoXk -/pcM5dCGTUX7XIarSFTEgBNQytpmfgt1Xbw2ErmlAdiFb4/5uBdbsVFAjglBvRI5 -VhFXr7mUd+XR/23aRczdAnp+Zg7VvyaJQi0ZwEj7VvLzpSAneVrxEcnuc2MBkUgT -TN/Z5LYqC93nr6vB7+HMwoBZ8hBAkO4rTKYQl3eMUSkIsE45CqI7Hz0eXQARAQAB -/gcDAqG5KzRnSp/38h4JKzJhSBRyyBPrgpYqR6ivFABzPUPJjO0gqRYzx/C+HJyl -z+QED0WH+sW8Ns4PkAgNWZ+225fzSssavLcPwjncy9pzcV+7bc76cFb77fSve+1D -LxhpzN58q03cSXPoamcDD7yY8GYYkAquLDZw+eRQ57BbBrNjXyfpGkBmtULymLqZ -SgkuV5we7//lRPDIuPk+9lszJXBUW3k5e32CR47B/hI6Pu0DTlN9VesAEmXRNsi9 -YlRiO74nGPQPEWGjnEUQ++W8ip0CzoSrmPhrdGQlSR+SBEbBCuXz1lsj7D9cBxwH -qHgwhYKvWz/gaY702+i/S1Cu/PjEpY3WPC5oSSNSSgypD8uSpcb4s2LffIegJNck -e1AuiovG6u/3QXPot0jHhdy+Qwe+oaJfSEBGQ4fD4W6GbPxwOIQGgXV0bRaeHYgL -iUWbN3rTLLVfDJKVo2ahvqZ7i4whfMuu1gGWQ4OEizrCDqp0x48HchEOC+T1eP3T -Zjth2YMtzZdXlpt5HNKeaY6ZP+NWILwvOQBd3UtNpeaCNhUa0YyB7GD/k7tZnCIZ -aNyF/DpnRrSQ9mAOffVn2NDGUv+01LnhIfa2tJes9XPmTc6ASrn/RGE9xH0X7wBD -HfAdGhHgbkzwNeYkQvSh1WyWj5C0Sq7X70dIYdcO81i5MMtlJrzrlB5/YCFVWSxt -7/EqwMBT3g9mkjAqo6beHxI1Hukn9rt9A6+MU64r0/cB+mVZuiBDoU/+KIiXBWiE -F/C1n/BO115WoWG35vj5oH+syuv3lRuPaz8GxoffcT+FUkmevZO1/BjEAABAwMS1 -nlB4y6xMJ0i2aCB2kp7ThDOOeTIQpdvtDLqRtQsVTpk73AEuDeKmULJnE2+Shi7v -yrNj1CPiBdYzz8jBDJYQH87iFQrro7VQNZzMMxpMWXQOZYWidHuBz4TgJJ0ll0JN -KwLtqv5wdf2zG8zNli0Dz+JwiwQ1kXDcA03rxHBCFALvkdIX0KUvTaTSV7OJ65VI -rcIwB5fSZgRE7m/9RjBGq/U+n4Kw+vlfpL7UeECJM0N7l8ekgTqqKv2Czu29eTjF -QOnpQtjgsWVpOnHKpQUfCN1Nxg8H1ytH9HQwLn+cGjm/yK55yIK+03X/vSs2m2qz -2zDhWlgvHLsDOEQkNsuOIvLkNM6Hv3MLTldknC+vMla34fYqpHfV1phL4npVByMW -CFOOzLa3qCoBXIGWvtnDx06r/8apHnt256G2X0iuRWWK+XpatMjmriZnj8vyGdIg -TZ1sNXnuFKMcXYMIvLANZXz4Rabbe6tTJ+BUVkbCGja4Z9iwmYvga77Mr2vjhtwi -CesRpcz6gR6U5fLddRZXyzKGxC3uQzokc9RtTuRNgSBZQ0oki++d6sr0+jOb54Mr -wfcMbMgpkQK0IJsMoOxzPLU8s6rISJvFi4IQ2dPYog17GS7Kjb1IGjGUxNKVHiIE -Is9wB+6bB51ZUUwc0zDSkuS6EaXLLVzmS7a3TOkVzu6J760TDVLL2+PDYkkBUP6O -SA2yeHirpyMma9QII1sw3xcKH/kDeyWigiB1VDKQpuq1PP98lYjQwAbe3Xrpy2FO -L/v6dSOJ+imgxD4osT0SanGkZEwPqJFvs6BI9Af8q9ia0xfK3Iu6F2F8JxmG1YiR -tUm9kCu3X/fNyE08G2sxD8QzGP9VS529nEDRBqkAgY6EHTpRKhPer9QrkUnqEyDZ -4s7RPcJW+cII/FPW8mSMgTqxFtTZgqNaqPPLevrTnTYTdrW/RkEs1mm0FWZvbyBi -YXIgPGZvb0Bmb28uZm9vPokCVAQTAQgAPhYhBJICM5a3zdmD+nRGF3grx+nZaj4C -BQJbdjh4AhsDBQkDwmcABQsJCAcCBhUICQoLAgQWAgMBAh4BAheAAAoJEHgrx+nZ -aj4CTyUP/2+4k4hXkkBrEeD0yDpmR/FrAgCOZ3iRWca9bJwKtV0hW0HSztlPEfng -wkwBmmyrnDevA+Ur4/hsBoTzfL4Fzo4OQDg2PZpSpIAHC1m/SQMN/s188RM8eK+Q -JBtinAo2IDoZyBi5Ar4rVNXrRpgvzwOLm15kpuPp15wxO+4gYOkNIT06yUrDNh3J -ccXmgZoVD54JmvKrEXscqX71/1NkaUhwZfFALN3+TVXUUdv1icQUJtxNBc29arwM -LuPuj9XAm5XJaVXDfsJyGu4aj4g6AJDXjVW1d2MgXv1rMRud7CGuX2PmO3CUUua9 -cUaavop5AmtF/+IsHae9qRt8PiMGTebV8IZ3Z6DZeOYDnfJVOXoIUcrAvX3LoImc -ephBdZ0KmYvaxlDrjtWAvmD6sPgwSvjLiXTmbmAkjRBXCVve4THf05kVUMcr8tmz -Il8LB+Dri2TfanBKykf6ulH0p2MHgSGQbYA5MuSp+soOitD5YvCxM7o/O0frrfit -p/O8mPerMEaYF1+3QbF5ApJkXCmjFCj71EPwXEDcl3VIGc+zA49oNjZMMmCcX2Gc -JyKTWizfuRBGeG5VhCCmTQQjZHPMVO255mdzsPkb6ZHEnolDapY6QXccV5x05XqD -sObFTy6iwEITdGmxN40pNE3WbhYGqOoXb8iRIG2hURv0gfG1/iI0 -=8g3t +lQdGBFt89QIBEAC8du0Purt9yeFuLlBYHcexnZvcbaci2pY+Ejn1VnxM7caFxRX/ +b2weZi9E6+I0F+K/hKIaidPdcbK92UCL0Vp6F3izjqategZ7o44vlK/HfWFME4wv +sou6lnig9ovA73HRyzngi3CmqWxSdg8lL0kIJLNzlvCFEd4Z34BnEkagklQJRymo +0WnmLJjSnZFT5Nk7q5jrcR7ApbD98cakvgivDlUBPJCk2JFPWheCkouWPHMvLXQz +bZXW5RFz4lJsMUWa/S3ofvIOnjG5Etnil3IA4uksS8fSDkGus998mBvUwzqX7xBh +dK17ZEbxDdO4PuVJDkjvq618rMu8FVk5yVd59rUketSnGrehd/+vdh6qtgQC4tu1 +RldbUVAuKZGg79H61nWnvrDZmbw4eoqCEuv1+aZsM9ElSC5Ps2J0rtpHRyBndKn+ +8Jlc/KTH04/O+FAhEv0IgMTFEm3iAq8udBhRBgu6Y4gJyn4tqy6+6ZjPUNos8GOG ++ZJPdrgHHHfQged1ygeceN6W2AwQRet/B3/rieHf2V93uHJy/DjYUEuBhPm9nxqi +R6ILUr97Sj2EsvLyfQO9pFpIctoNKEJmDx/C9tkFMNNlQhpsBitSdR2/wancw9ND +iWV/J9roUdC0qns7eNSbiFe3Len8Xir7srnjAFgbGvOu9jDBUuiKGT5F3wARAQAB +/gcDAl+0SktmjrUW8uwpvru6GeIeo5kc4rXuD7iIxH6nDl3nmjZMX7qWvp+pRTHH +0hEDH44899PDvzclBN3ouehfFUbJ+DBy8umBiLqF8Mu2PrKjdmyv3BvnbTkqPM3m +2Su7WmUDBhG00X07lfl8fTpZJG80onEGzGynryP/xVm4ymzoHyYGksntXLYr2HJ5 +aV6L7sL2/STsaaOVHoa/oEmVBo1+NRsTxRRUcFVLs3g0OIi6ZCeSevBdavMwf9Iv +b5Bs/e0+GLpP71XzFpdrGcL6oGjZH/dgdeypzbGA+FHtQJqynN3qEE9eCc9cfTGL +2zN2OtnMA28NtPVN4SnSxQIDvycWx68NZjfwLOK+gswfKpimp+6xMWSnNIRDyU9M +w0hdNPMK9JAxm/MlnkR7x6ysX/8vrVVFl9gWOmxzJ5L4kvfMsHcV5ZFRP8OnVA6a +NFBWIBGXF1uQC4qrXup/xKyWJOoH++cMo2cjPT3+3oifZgdBydVfHXjS9aQ/S3Sa +A6henWyx/qeBGPVRuXWdXIOKDboOPK8JwQaGd6yazKkH9c5tDohmQHzZ6ho0gyAt +dh+g9ZyiZVpjc6excfK/DP/RdUOYKw3Ur9652hKephvYZzHvPjTbqVkhS7JjZkVY +rukQ64d5T0pE1B4y+If4hLFXMNQtfo0TIsATNA69jop+KFnJpLzAB+Ee33EA/HUl +YC5EJCJaXt6kdtYFac0HvVWiz5ZuMhdtzpJfvOe+Olp/xR9nIPW3XZojQoHIZKwu +gXeZeVMvfeoq+ymKAKNH5Np4WaUDF7Wh9VLl045jGyF5viyy61ivC0eyAzp5W1uy +gJBZwafVma5MhmZUS2dFs0hBwBrKRzZZhN65VvfSYw6CnXp83ryUjReDvrLmqZDM +FNpSMDKRk1+k9Wwi3m+fzLAvlxoHscJ5Any7ApsvBRbyehP8MAAG7UV3jImugTLi +yN6FKVwziQXiC4/97oKbA1YYNjTT7Qw9gWTXvLRspn4f9997brcA9dm0M0seTjLa +lc5hTJwJQdvPPI2klf+YgPvsD6nrP1moeWBb8irICqG1/BoE0JHPS+bqJ1J+m1iV +kRV/+4pV2bLlXKqg1LEvqANW+1P1eM2nbbVB7EQn8ZOPIKMoCLoC1QWUPNfnemsW +U5ynAbhsbm16PDJql0ApEgUCEDfsXTu1ui6SIO3bs/gWyD9HEmnfaYMYDKF+j+0r +jXd4GnCxb+Yu3wV5WyewOHouzC+++h/3WcDLkOYZ9pcIbA86qT+v6b9MuTAU0D3c +wlDv8r5J59zOcXl4HpMb2BY5F9dZn8hjgeVJRhJdij9x1TQ8qlVasSi4Eq8SiPmZ +PZz33Pk6yn2caQ6wd47A79LXCbFQqJqA5aA6oS4DOpENGS5fh7WUZq/MTcmm9GsG +w2gHxocASK9RCUYgZFWVYgLDuviMMWvc/2TJcTMxdF0Amu3erYAD90smFs0g/6fZ +4pRLnKFuifwAMGMOx7jbW5tmOaSPx6XkuYvkDJeLMHoN3z/8bZEG5VpayypwFGyV +bk/YIUWg/KM/43juDPdTvab9tZzYIjxC6on7dtYIAGjZis97XZou3KYKTaMe1VY6 +IhrnVzJ0JAHpd1prf9NUz96e1vjGdn3I61JgjNp5sWklIJEZzvaD28Eovf/LH1BO +gYFFCvsWXaRoPHNQ5a9m7CROkLeHUFgRu5uriqHxxQHgogDznc8/3fnvDAHNpNb6 +Jnk4zaeVR3tTyIjiNM+wxUFPDNFpJWmQbSDCcPVYTbpznzVRnhqrw7q0FWZvbyBi +YXIgPGZvb0Bmb28uZm9vPokCVAQTAQgAPgIbAwULCQgHAgYVCAkKCwIEFgIDAQIe +AQIXgBYhBJOhf/AeVDKFRgh8jgKTlUAu/M1TBQJbfPU4BQkSzAM2AAoJEAKTlUAu +/M1TVTIQALA6ocNc2fXz1loLykMxlfnX/XxiyNDOUPDZkrZtscqqWPYaWvJK3OiD +32bdVEbftnAiFvJYkinrCXLEmwwf5wyOxKFmCHwwKhH0UYt60yF4WwlOVNstGSAy +RkPMEEmVfMXS9K1nzKv/9A5YsqMQob7sN5CMN66Vrm0RKSvOF/NhhM9v8fC0QSU2 +GZNO0tnRfaS4wMnFr5L4FuDST+14F5sJT7ZEJz7HfbxXKLvvWbvqLlCYHJOdz56s +X/eKde8eT9/LSzcmgsd7rGS2np5901kubww5jllUl1CFnk3Mdg9FTJl5u9Epuhnn +823Jpdy1ZNbyLqZ266Z/q2HepDA7P/GqIXgWdHjwG2y1YAC4JIkA4RBbesQwqAXs +6cX5gqRFRl5iDGEP5zclS0y5mWi/J8bLYxMYfqxs9EZtHd9DumWISi87804TEzYa +WDijMlW7PR8QRW0vdmtYOhJZOlTnomLQx2v27iqpVXRh12J1aYVBFC+IvG1vhCf9 +FL3LzAHHEGlIoDaKJMd+Wg/Lm/f1PqqQx3lWIh9hhKh5Qx6hcuJH669JOWuEdxfo +1so50aItG+tdDKqXflmOi7grrUURchYYKteaW2fC2SQgzDClprALI7aj9s/lDrEN +CgLH6twOqdSFWqB/4ASDMsNeLeKX3WOYKYYMlE01cj3T1m6dpRUO +=gIM9 -----END PGP PRIVATE KEY BLOCK----- ` From 790191ef92ec6382ce65cc30286c901863b3b7a3 Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Wed, 22 Aug 2018 16:46:50 +0200 Subject: [PATCH 077/120] plumbing, storage: add bases to the common cache After clone only resolved deltas were added to the cache. This caused slowdowns in small repositories where most objects can be held in cache. It also makes packfiles reuse delta cache from the store. Previously it created a new delta cache each time a packfile object was created. This also slowed down a bit accessing objects and had an impact on memory consumption when bases are added to the cache. Signed-off-by: Javi Fontan --- plumbing/format/packfile/fsobject.go | 10 ++++++++++ plumbing/format/packfile/packfile.go | 15 +++++++++++++++ storage/filesystem/object.go | 18 ++++++++++++++++-- 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/plumbing/format/packfile/fsobject.go b/plumbing/format/packfile/fsobject.go index 6fd3ca54d..330cb73c9 100644 --- a/plumbing/format/packfile/fsobject.go +++ b/plumbing/format/packfile/fsobject.go @@ -47,6 +47,16 @@ func NewFSObject( // Reader implements the plumbing.EncodedObject interface. func (o *FSObject) Reader() (io.ReadCloser, error) { + obj, ok := o.cache.Get(o.hash) + if ok { + reader, err := obj.Reader() + if err != nil { + return nil, err + } + + return reader, nil + } + f, err := o.fs.Open(o.path) if err != nil { return nil, err diff --git a/plumbing/format/packfile/packfile.go b/plumbing/format/packfile/packfile.go index 18fcca7e8..852a8344b 100644 --- a/plumbing/format/packfile/packfile.go +++ b/plumbing/format/packfile/packfile.go @@ -258,6 +258,19 @@ func (p *Packfile) nextObject() (plumbing.EncodedObject, error) { } func (p *Packfile) getObjectContent(offset int64) (io.ReadCloser, error) { + ref, err := p.FindHash(offset) + if err == nil { + obj, ok := p.cacheGet(ref) + if ok { + reader, err := obj.Reader() + if err != nil { + return nil, err + } + + return reader, nil + } + } + if _, err := p.s.SeekFromStart(offset); err != nil { return nil, err } @@ -306,6 +319,8 @@ func (p *Packfile) fillRegularObjectContent(obj plumbing.EncodedObject) error { } _, _, err = p.s.NextObject(w) + p.cachePut(obj) + return err } diff --git a/storage/filesystem/object.go b/storage/filesystem/object.go index 6958e3217..3a3a2bd97 100644 --- a/storage/filesystem/object.go +++ b/storage/filesystem/object.go @@ -295,7 +295,14 @@ func (s *ObjectStorage) decodeObjectAt( return nil, err } - return packfile.NewPackfile(idx, s.dir.Fs(), f).GetByOffset(offset) + var p *packfile.Packfile + if s.deltaBaseCache != nil { + p = packfile.NewPackfileWithCache(idx, s.dir.Fs(), f, s.deltaBaseCache) + } else { + p = packfile.NewPackfile(idx, s.dir.Fs(), f) + } + + return p.GetByOffset(offset) } func (s *ObjectStorage) decodeDeltaObjectAt( @@ -486,7 +493,14 @@ func newPackfileIter( index idxfile.Index, cache cache.Object, ) (storer.EncodedObjectIter, error) { - iter, err := packfile.NewPackfile(index, fs, f).GetByType(t) + var p *packfile.Packfile + if cache != nil { + p = packfile.NewPackfileWithCache(index, fs, f, cache) + } else { + p = packfile.NewPackfile(index, fs, f) + } + + iter, err := p.GetByType(t) if err != nil { return nil, err } From 119459a6b9ddaa244f76f67b182bf2c627434d02 Mon Sep 17 00:00:00 2001 From: Chris Marchesi Date: Thu, 23 Aug 2018 10:14:21 -0700 Subject: [PATCH 078/120] git: Discern tag target type from supplied hash I figured there was a way to do this without having to have TagObjectOptions supply this in - there is. Added support for this in and removed the object type from TagObjectOptions. Signed-off-by: Chris Marchesi --- options.go | 12 ----------- repository.go | 7 ++++++- repository_test.go | 52 +++++++++++++++++++++------------------------- 3 files changed, 30 insertions(+), 41 deletions(-) diff --git a/options.go b/options.go index ed0baa3ad..856bd5e24 100644 --- a/options.go +++ b/options.go @@ -383,7 +383,6 @@ var ( ErrMissingName = errors.New("name field is required") ErrMissingTagger = errors.New("tagger field is required") ErrMissingMessage = errors.New("message field is required") - ErrBadObjectType = errors.New("bad object type for tagging") ) // TagObjectOptions describes how a tag object should be created. @@ -394,9 +393,6 @@ type TagObjectOptions struct { // validation into the format expected by git - no leading whitespace and // ending in a newline. Message string - // TargetType is the object type of the target. The object specified by - // Target must be of this type. - TargetType plumbing.ObjectType // SignKey denotes a key to sign the tag with. A nil value here means the tag // will not be signed. The private key must be present and already decrypted. SignKey *openpgp.Entity @@ -415,14 +411,6 @@ func (o *TagObjectOptions) Validate(r *Repository, hash plumbing.Hash) error { // Canonicalize the message into the expected message format. o.Message = strings.TrimSpace(o.Message) + "\n" - if o.TargetType == plumbing.InvalidObject || o.TargetType == plumbing.AnyObject { - return ErrBadObjectType - } - - if _, err := r.Object(o.TargetType, hash); err != nil { - return err - } - return nil } diff --git a/repository.go b/repository.go index ab14eba0b..6da15a13a 100644 --- a/repository.go +++ b/repository.go @@ -532,11 +532,16 @@ func (r *Repository) createTagObject(name string, hash plumbing.Hash, opts *TagO return plumbing.ZeroHash, err } + rawobj, err := object.GetObject(r.Storer, hash) + if err != nil { + return plumbing.ZeroHash, err + } + tag := &object.Tag{ Name: name, Tagger: *opts.Tagger, Message: opts.Message, - TargetType: opts.TargetType, + TargetType: rawobj.Type(), Target: hash, } diff --git a/repository_test.go b/repository_test.go index fd80152e7..795ee55c0 100644 --- a/repository_test.go +++ b/repository_test.go @@ -1333,9 +1333,8 @@ func (s *RepositorySuite) TestCreateTagAnnotated(c *C) { expectedHash := h.Hash() ref, err := r.CreateTag("foobar", expectedHash, &TagObjectOptions{ - Tagger: defaultSignature(), - Message: "foo bar baz qux", - TargetType: plumbing.CommitObject, + Tagger: defaultSignature(), + Message: "foo bar baz qux", }) c.Assert(err, IsNil) @@ -1364,32 +1363,32 @@ func (s *RepositorySuite) TestCreateTagAnnotatedBadOpts(c *C) { expectedHash := h.Hash() ref, err := r.CreateTag("foobar", expectedHash, &TagObjectOptions{ - Message: "foo bar baz qux", - TargetType: plumbing.CommitObject, + Message: "foo bar baz qux", }) c.Assert(ref, IsNil) c.Assert(err, Equals, ErrMissingTagger) ref, err = r.CreateTag("foobar", expectedHash, &TagObjectOptions{ - Tagger: defaultSignature(), - TargetType: plumbing.CommitObject, + Tagger: defaultSignature(), }) c.Assert(ref, IsNil) c.Assert(err, Equals, ErrMissingMessage) +} - ref, err = r.CreateTag("foobar", expectedHash, &TagObjectOptions{ +func (s *RepositorySuite) TestCreateTagAnnotatedBadHash(c *C) { + url := s.GetLocalRepositoryURL( + fixtures.ByURL("https://github.com/git-fixtures/tags.git").One(), + ) + + r, _ := Init(memory.NewStorage(), nil) + err := r.clone(context.Background(), &CloneOptions{URL: url}) + c.Assert(err, IsNil) + + ref, err := r.CreateTag("foobar", plumbing.ZeroHash, &TagObjectOptions{ Tagger: defaultSignature(), Message: "foo bar baz qux", }) c.Assert(ref, IsNil) - c.Assert(err, Equals, ErrBadObjectType) - - ref, err = r.CreateTag("foobar", expectedHash, &TagObjectOptions{ - Tagger: defaultSignature(), - Message: "foo bar baz qux", - TargetType: plumbing.TagObject, - }) - c.Assert(ref, IsNil) c.Assert(err, Equals, plumbing.ErrObjectNotFound) } @@ -1407,10 +1406,9 @@ func (s *RepositorySuite) TestCreateTagSigned(c *C) { key := commitSignKey(c, true) _, err = r.CreateTag("foobar", h.Hash(), &TagObjectOptions{ - Tagger: defaultSignature(), - Message: "foo bar baz qux", - TargetType: plumbing.CommitObject, - SignKey: key, + Tagger: defaultSignature(), + Message: "foo bar baz qux", + SignKey: key, }) c.Assert(err, IsNil) @@ -1447,10 +1445,9 @@ func (s *RepositorySuite) TestCreateTagSignedBadKey(c *C) { key := commitSignKey(c, false) _, err = r.CreateTag("foobar", h.Hash(), &TagObjectOptions{ - Tagger: defaultSignature(), - Message: "foo bar baz qux", - TargetType: plumbing.CommitObject, - SignKey: key, + Tagger: defaultSignature(), + Message: "foo bar baz qux", + SignKey: key, }) c.Assert(err, Equals, openpgperr.InvalidArgumentError("signing key is encrypted")) } @@ -1469,10 +1466,9 @@ func (s *RepositorySuite) TestCreateTagCanonicalize(c *C) { key := commitSignKey(c, true) _, err = r.CreateTag("foobar", h.Hash(), &TagObjectOptions{ - Tagger: defaultSignature(), - Message: "\n\nfoo bar baz qux\n\nsome message here", - TargetType: plumbing.CommitObject, - SignKey: key, + Tagger: defaultSignature(), + Message: "\n\nfoo bar baz qux\n\nsome message here", + SignKey: key, }) c.Assert(err, IsNil) From 01631f0e5c4be73cefaa8b2cc8a4811005871656 Mon Sep 17 00:00:00 2001 From: Chris Marchesi Date: Thu, 23 Aug 2018 11:57:51 -0700 Subject: [PATCH 079/120] git: Don't return tag object with Tag, adjust docs for Tag and Tags I've mainly noticed that in using the current Tag function, that there were lots of times that I was ignoring the ref or the object, depending on what I needed. This was evident in the tests as well. As such, I think it just makes more sense for a singular tag fetcher to return just a ref, through which the caller may grab the annotation if they need it, and if it exists. Also, contrary to the docs on Tags, all tags have a ref, even if they are annotated. The difference between a lightweight tag and an annotated tag is the presence of the tag object, which the ref will point to if the tag is annotated. As such I've adjusted the docs with an example as to how one can get the annotation for a tag ref through the iterator. Source: https://git-scm.com/book/en/v2/Git-Internals-Git-References, tags section. Signed-off-by: Chris Marchesi --- repository.go | 70 ++++++++++++++++++++++++++++++++++++---------- repository_test.go | 51 +++++++++++---------------------- 2 files changed, 71 insertions(+), 50 deletions(-) diff --git a/repository.go b/repository.go index 6da15a13a..68cc5cc28 100644 --- a/repository.go +++ b/repository.go @@ -581,34 +581,52 @@ func (r *Repository) buildTagSignature(tag *object.Tag, signKey *openpgp.Entity) return b.String(), nil } -// Tag fetches a tag from the repository. The tag is returned as a raw -// reference. If the tag is annotated, a non-nil tag object is returned. -func (r *Repository) Tag(name string) (*plumbing.Reference, *object.Tag, error) { +// Tag fetches a tag from the repository. +// +// If you want to check to see if the tag is an annotated tag, you can call +// TagObject on the hash of the reference in ForEach: +// +// ref, err := r.Tag("v0.1.0") +// if err != nil { +// // Handle error +// } +// +// obj, err := r.TagObject(ref.Hash()) +// switch err { +// case nil: +// // Tag object present +// case plumbing.ErrObjectNotFound: +// // Not a tag object +// default: +// // Some other error +// } +// +func (r *Repository) Tag(name string) (*plumbing.Reference, error) { ref, err := r.Reference(plumbing.ReferenceName(path.Join("refs", "tags", name)), false) if err != nil { if err == plumbing.ErrReferenceNotFound { // Return a friendly error for this one, versus just ReferenceNotFound. - return nil, nil, ErrTagNotFound + return nil, ErrTagNotFound } - return nil, nil, err - } - - obj, err := r.TagObject(ref.Hash()) - if err != nil && err != plumbing.ErrObjectNotFound { - return nil, nil, err + return nil, err } - return ref, obj, nil + return ref, nil } // DeleteTag deletes a tag from the repository. func (r *Repository) DeleteTag(name string) error { - _, obj, err := r.Tag(name) + ref, err := r.Tag(name) if err != nil { return err } + obj, err := r.TagObject(ref.Hash()) + if err != nil && err != plumbing.ErrObjectNotFound { + return err + } + if err = r.Storer.RemoveReference(plumbing.ReferenceName(path.Join("refs", "tags", name))); err != nil { return err } @@ -982,9 +1000,31 @@ func (r *Repository) Log(o *LogOptions) (object.CommitIter, error) { return nil, fmt.Errorf("invalid Order=%v", o.Order) } -// Tags returns all the References from Tags. This method returns only lightweight -// tags. Note that not all the tags are lightweight ones. To return annotated tags -// too, you need to call TagObjects() method. +// Tags returns all the tag References in a repository. +// +// If you want to check to see if the tag is an annotated tag, you can call +// TagObject on the hash Reference passed in through ForEach: +// +// iter, err := r.Tags() +// if err != nil { +// // Handle error +// } +// +// if err := iter.ForEach(func (ref *plumbing.Reference) error { +// obj, err := r.TagObject(ref.Hash()) +// switch err { +// case nil: +// // Tag object present +// case plumbing.ErrObjectNotFound: +// // Not a tag object +// default: +// // Some other error +// return err +// } +// }); err != nil { +// // Handle outer iterator error +// } +// func (r *Repository) Tags() (storer.ReferenceIter, error) { refIter, err := r.Storer.IterReferences() if err != nil { diff --git a/repository_test.go b/repository_test.go index 795ee55c0..0415cc41a 100644 --- a/repository_test.go +++ b/repository_test.go @@ -1294,9 +1294,8 @@ func (s *RepositorySuite) TestCreateTagLightweight(c *C) { c.Assert(err, IsNil) c.Assert(ref, NotNil) - actual, obj, err := r.Tag("foobar") + actual, err := r.Tag("foobar") c.Assert(err, IsNil) - c.Assert(obj, IsNil) c.Assert(expected.Hash(), Equals, actual.Hash()) } @@ -1338,9 +1337,11 @@ func (s *RepositorySuite) TestCreateTagAnnotated(c *C) { }) c.Assert(err, IsNil) - tag, obj, err := r.Tag("foobar") + tag, err := r.Tag("foobar") + c.Assert(err, IsNil) + + obj, err := r.TagObject(tag.Hash()) c.Assert(err, IsNil) - c.Assert(obj, NotNil) c.Assert(ref, DeepEquals, tag) c.Assert(obj.Hash, Equals, ref.Hash()) @@ -1412,9 +1413,11 @@ func (s *RepositorySuite) TestCreateTagSigned(c *C) { }) c.Assert(err, IsNil) - _, obj, err := r.Tag("foobar") + tag, err := r.Tag("foobar") + c.Assert(err, IsNil) + + obj, err := r.TagObject(tag.Hash()) c.Assert(err, IsNil) - c.Assert(obj, NotNil) // Verify the tag. pks := new(bytes.Buffer) @@ -1472,9 +1475,11 @@ func (s *RepositorySuite) TestCreateTagCanonicalize(c *C) { }) c.Assert(err, IsNil) - _, obj, err := r.Tag("foobar") + tag, err := r.Tag("foobar") + c.Assert(err, IsNil) + + obj, err := r.TagObject(tag.Hash()) c.Assert(err, IsNil) - c.Assert(obj, NotNil) // Assert the new canonicalized message. c.Assert(obj.Message, Equals, "foo bar baz qux\n\nsome message here\n") @@ -1505,9 +1510,8 @@ func (s *RepositorySuite) TestTagLightweight(c *C) { expected := plumbing.NewHash("f7b877701fbf855b44c0a9e86f3fdce2c298b07f") - tag, obj, err := r.Tag("lightweight-tag") + tag, err := r.Tag("lightweight-tag") c.Assert(err, IsNil) - c.Assert(obj, IsNil) actual := tag.Hash() c.Assert(expected, Equals, actual) @@ -1522,34 +1526,11 @@ func (s *RepositorySuite) TestTagLightweightMissingTag(c *C) { err := r.clone(context.Background(), &CloneOptions{URL: url}) c.Assert(err, IsNil) - tag, obj, err := r.Tag("lightweight-tag-tag") + tag, err := r.Tag("lightweight-tag-tag") c.Assert(tag, IsNil) - c.Assert(obj, IsNil) c.Assert(err, Equals, ErrTagNotFound) } -func (s *RepositorySuite) TestTagAnnotated(c *C) { - url := s.GetLocalRepositoryURL( - fixtures.ByURL("https://github.com/git-fixtures/tags.git").One(), - ) - - r, _ := Init(memory.NewStorage(), nil) - err := r.clone(context.Background(), &CloneOptions{URL: url}) - c.Assert(err, IsNil) - - tag, obj, err := r.Tag("annotated-tag") - c.Assert(err, IsNil) - c.Assert(obj, NotNil) - - expectedHash := plumbing.NewHash("b742a2a9fa0afcfa9a6fad080980fbc26b007c69") - expectedTarget := plumbing.NewHash("f7b877701fbf855b44c0a9e86f3fdce2c298b07f") - actualHash := tag.Hash() - c.Assert(expectedHash, Equals, actualHash) - c.Assert(obj.Hash, Equals, expectedHash) - c.Assert(obj.Type(), Equals, plumbing.TagObject) - c.Assert(obj.Target, Equals, expectedTarget) -} - func (s *RepositorySuite) TestDeleteTag(c *C) { url := s.GetLocalRepositoryURL( fixtures.ByURL("https://github.com/git-fixtures/tags.git").One(), @@ -1562,7 +1543,7 @@ func (s *RepositorySuite) TestDeleteTag(c *C) { err = r.DeleteTag("lightweight-tag") c.Assert(err, IsNil) - _, _, err = r.Tag("lightweight-tag") + _, err = r.Tag("lightweight-tag") c.Assert(err, Equals, ErrTagNotFound) } From c7a4011d78a00bd93a2f82a39bb67c2dda5453f5 Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Sat, 25 Aug 2018 19:17:45 +0200 Subject: [PATCH 080/120] storage/dotgit: search for incoming dir only once Search for incoming object directory was done once each time objects were accessed. This means a ReadDir of the objects path that is expensive. Now incoming directory is searched the first time an object is accessed and its name kept in DotGit to be reused. Signed-off-by: Javi Fontan --- storage/filesystem/dotgit/dotgit.go | 41 ++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/storage/filesystem/dotgit/dotgit.go b/storage/filesystem/dotgit/dotgit.go index d0a14aebe..addb64c91 100644 --- a/storage/filesystem/dotgit/dotgit.go +++ b/storage/filesystem/dotgit/dotgit.go @@ -61,6 +61,10 @@ var ( // type is not zero-value-safe, use the New function to initialize it. type DotGit struct { fs billy.Filesystem + + // incoming object directory information + incomingChecked bool + incomingDirName string } // New returns a DotGit value ready to be used. The path argument must @@ -286,26 +290,37 @@ func (d *DotGit) objectPath(h plumbing.Hash) string { //More on 'quarantine'/incoming directory here : https://git-scm.com/docs/git-receive-pack func (d *DotGit) incomingObjectPath(h plumbing.Hash) string { hString := h.String() - directoryContents, err := d.fs.ReadDir(objectsPath) - if err != nil { + + if d.incomingDirName == "" { return d.fs.Join(objectsPath, hString[0:2], hString[2:40]) } - var incomingDirName string - for _, file := range directoryContents { - if strings.Split(file.Name(), "-")[0] == "incoming" && file.IsDir() { - incomingDirName = file.Name() + + return d.fs.Join(objectsPath, d.incomingDirName, hString[0:2], hString[2:40]) +} + +// hasIncomingObjects searches for an incoming directory and keeps its name +// so it doesn't have to be found each time an object is accessed. +func (d *DotGit) hasIncomingObjects() bool { + if !d.incomingChecked { + directoryContents, err := d.fs.ReadDir(objectsPath) + if err == nil { + for _, file := range directoryContents { + if strings.Split(file.Name(), "-")[0] == "incoming" && file.IsDir() { + d.incomingDirName = file.Name() + } + } } + + d.incomingChecked = true } - if incomingDirName == "" { - return d.fs.Join(objectsPath, hString[0:2], hString[2:40]) - } - return d.fs.Join(objectsPath, incomingDirName, hString[0:2], hString[2:40]) + + return d.incomingDirName != "" } // Object returns a fs.File pointing the object file, if exists func (d *DotGit) Object(h plumbing.Hash) (billy.File, error) { obj1, err1 := d.fs.Open(d.objectPath(h)) - if os.IsNotExist(err1) { + if os.IsNotExist(err1) && d.hasIncomingObjects() { obj2, err2 := d.fs.Open(d.incomingObjectPath(h)) if err2 != nil { return obj1, err1 @@ -318,7 +333,7 @@ func (d *DotGit) Object(h plumbing.Hash) (billy.File, error) { // ObjectStat returns a os.FileInfo pointing the object file, if exists func (d *DotGit) ObjectStat(h plumbing.Hash) (os.FileInfo, error) { obj1, err1 := d.fs.Stat(d.objectPath(h)) - if os.IsNotExist(err1) { + if os.IsNotExist(err1) && d.hasIncomingObjects() { obj2, err2 := d.fs.Stat(d.incomingObjectPath(h)) if err2 != nil { return obj1, err1 @@ -331,7 +346,7 @@ func (d *DotGit) ObjectStat(h plumbing.Hash) (os.FileInfo, error) { // ObjectDelete removes the object file, if exists func (d *DotGit) ObjectDelete(h plumbing.Hash) error { err1 := d.fs.Remove(d.objectPath(h)) - if os.IsNotExist(err1) { + if os.IsNotExist(err1) && d.hasIncomingObjects() { err2 := d.fs.Remove(d.incomingObjectPath(h)) if err2 != nil { return err1 From b1da90b0dde34b521cb252bc28c59e4ffd840d1d Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Mon, 27 Aug 2018 14:45:24 +0200 Subject: [PATCH 081/120] storage/dotgit: use HasPrefix instead of Split Also reformatted function comment and fixed some typos. Signed-off-by: Javi Fontan --- storage/filesystem/dotgit/dotgit.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/storage/filesystem/dotgit/dotgit.go b/storage/filesystem/dotgit/dotgit.go index addb64c91..df4f75691 100644 --- a/storage/filesystem/dotgit/dotgit.go +++ b/storage/filesystem/dotgit/dotgit.go @@ -283,11 +283,14 @@ func (d *DotGit) objectPath(h plumbing.Hash) string { return d.fs.Join(objectsPath, hash[0:2], hash[2:40]) } -//incomingObjectPath is intended to add support for a git pre-recieve hook to be written -//it adds support for go-git to find objects in an "incoming" directory, so that the library -//can be used to write a pre-recieve hook that deals with the incoming objects. -//More on git hooks found here : https://git-scm.com/docs/githooks -//More on 'quarantine'/incoming directory here : https://git-scm.com/docs/git-receive-pack +// incomingObjectPath is intended to add support for a git pre-receive hook +// to be written it adds support for go-git to find objects in an "incoming" +// directory, so that the library can be used to write a pre-receive hook +// that deals with the incoming objects. +// +// More on git hooks found here : https://git-scm.com/docs/githooks +// More on 'quarantine'/incoming directory here: +// https://git-scm.com/docs/git-receive-pack func (d *DotGit) incomingObjectPath(h plumbing.Hash) string { hString := h.String() @@ -305,7 +308,7 @@ func (d *DotGit) hasIncomingObjects() bool { directoryContents, err := d.fs.ReadDir(objectsPath) if err == nil { for _, file := range directoryContents { - if strings.Split(file.Name(), "-")[0] == "incoming" && file.IsDir() { + if strings.HasPrefix(file.Name(), "incoming-") && file.IsDir() { d.incomingDirName = file.Name() } } From 0167dabb78412ed5fb76cb4b174a6708c3be52b8 Mon Sep 17 00:00:00 2001 From: kuba-- Date: Fri, 24 Aug 2018 23:44:44 +0200 Subject: [PATCH 082/120] Remove empty dirs when cleaning with Dir opt. Signed-off-by: kuba-- --- storage/filesystem/dotgit/dotgit.go | 54 ++++++++++++++++++--------- worktree.go | 58 ++++++++++++++++++++++------- worktree_test.go | 9 +++++ 3 files changed, 89 insertions(+), 32 deletions(-) diff --git a/storage/filesystem/dotgit/dotgit.go b/storage/filesystem/dotgit/dotgit.go index d0a14aebe..df4f75691 100644 --- a/storage/filesystem/dotgit/dotgit.go +++ b/storage/filesystem/dotgit/dotgit.go @@ -61,6 +61,10 @@ var ( // type is not zero-value-safe, use the New function to initialize it. type DotGit struct { fs billy.Filesystem + + // incoming object directory information + incomingChecked bool + incomingDirName string } // New returns a DotGit value ready to be used. The path argument must @@ -279,33 +283,47 @@ func (d *DotGit) objectPath(h plumbing.Hash) string { return d.fs.Join(objectsPath, hash[0:2], hash[2:40]) } -//incomingObjectPath is intended to add support for a git pre-recieve hook to be written -//it adds support for go-git to find objects in an "incoming" directory, so that the library -//can be used to write a pre-recieve hook that deals with the incoming objects. -//More on git hooks found here : https://git-scm.com/docs/githooks -//More on 'quarantine'/incoming directory here : https://git-scm.com/docs/git-receive-pack +// incomingObjectPath is intended to add support for a git pre-receive hook +// to be written it adds support for go-git to find objects in an "incoming" +// directory, so that the library can be used to write a pre-receive hook +// that deals with the incoming objects. +// +// More on git hooks found here : https://git-scm.com/docs/githooks +// More on 'quarantine'/incoming directory here: +// https://git-scm.com/docs/git-receive-pack func (d *DotGit) incomingObjectPath(h plumbing.Hash) string { hString := h.String() - directoryContents, err := d.fs.ReadDir(objectsPath) - if err != nil { + + if d.incomingDirName == "" { return d.fs.Join(objectsPath, hString[0:2], hString[2:40]) } - var incomingDirName string - for _, file := range directoryContents { - if strings.Split(file.Name(), "-")[0] == "incoming" && file.IsDir() { - incomingDirName = file.Name() + + return d.fs.Join(objectsPath, d.incomingDirName, hString[0:2], hString[2:40]) +} + +// hasIncomingObjects searches for an incoming directory and keeps its name +// so it doesn't have to be found each time an object is accessed. +func (d *DotGit) hasIncomingObjects() bool { + if !d.incomingChecked { + directoryContents, err := d.fs.ReadDir(objectsPath) + if err == nil { + for _, file := range directoryContents { + if strings.HasPrefix(file.Name(), "incoming-") && file.IsDir() { + d.incomingDirName = file.Name() + } + } } + + d.incomingChecked = true } - if incomingDirName == "" { - return d.fs.Join(objectsPath, hString[0:2], hString[2:40]) - } - return d.fs.Join(objectsPath, incomingDirName, hString[0:2], hString[2:40]) + + return d.incomingDirName != "" } // Object returns a fs.File pointing the object file, if exists func (d *DotGit) Object(h plumbing.Hash) (billy.File, error) { obj1, err1 := d.fs.Open(d.objectPath(h)) - if os.IsNotExist(err1) { + if os.IsNotExist(err1) && d.hasIncomingObjects() { obj2, err2 := d.fs.Open(d.incomingObjectPath(h)) if err2 != nil { return obj1, err1 @@ -318,7 +336,7 @@ func (d *DotGit) Object(h plumbing.Hash) (billy.File, error) { // ObjectStat returns a os.FileInfo pointing the object file, if exists func (d *DotGit) ObjectStat(h plumbing.Hash) (os.FileInfo, error) { obj1, err1 := d.fs.Stat(d.objectPath(h)) - if os.IsNotExist(err1) { + if os.IsNotExist(err1) && d.hasIncomingObjects() { obj2, err2 := d.fs.Stat(d.incomingObjectPath(h)) if err2 != nil { return obj1, err1 @@ -331,7 +349,7 @@ func (d *DotGit) ObjectStat(h plumbing.Hash) (os.FileInfo, error) { // ObjectDelete removes the object file, if exists func (d *DotGit) ObjectDelete(h plumbing.Hash) error { err1 := d.fs.Remove(d.objectPath(h)) - if os.IsNotExist(err1) { + if os.IsNotExist(err1) && d.hasIncomingObjects() { err2 := d.fs.Remove(d.incomingObjectPath(h)) if err2 != nil { return err1 diff --git a/worktree.go b/worktree.go index 99b2cd124..921e600a0 100644 --- a/worktree.go +++ b/worktree.go @@ -713,29 +713,56 @@ func (w *Worktree) readGitmodulesFile() (*config.Modules, error) { } // Clean the worktree by removing untracked files. +// An empty dir could be removed - this is what `git clean -f -d .` does. func (w *Worktree) Clean(opts *CleanOptions) error { s, err := w.Status() if err != nil { return err } - // Check Worktree status to be Untracked, obtain absolute path and delete. - for relativePath, status := range s { - // Check if the path contains a directory and if Dir options is false, - // skip the path. - if relativePath != filepath.Base(relativePath) && !opts.Dir { + root := "" + files, err := w.Filesystem.ReadDir(root) + if err != nil { + return err + } + return w.doClean(s, opts, root, files) +} + +func (w *Worktree) doClean(status Status, opts *CleanOptions, dir string, files []os.FileInfo) error { + for _, fi := range files { + if fi.Name() == ".git" { continue } - // Remove the file only if it's an untracked file. - if status.Worktree == Untracked { - absPath := filepath.Join(w.Filesystem.Root(), relativePath) - if err := os.Remove(absPath); err != nil { + // relative path under the root + path := filepath.Join(dir, fi.Name()) + if fi.IsDir() { + if !opts.Dir { + continue + } + + subfiles, err := w.Filesystem.ReadDir(path) + if err != nil { + return err + } + err = w.doClean(status, opts, path, subfiles) + if err != nil { return err } + } else { + // check if file is 'Untracked' + s, ok := (status)[filepath.ToSlash(path)] + if ok && s.Worktree == Untracked { + if err := w.Filesystem.Remove(path); err != nil { + return err + } + } } } + if opts.Dir { + return doCleanDirectories(w.Filesystem, dir) + } return nil } @@ -881,15 +908,18 @@ func rmFileAndDirIfEmpty(fs billy.Filesystem, name string) error { return err } - path := filepath.Dir(name) - files, err := fs.ReadDir(path) + dir := filepath.Dir(name) + return doCleanDirectories(fs, dir) +} + +// doCleanDirectories removes empty subdirs (without files) +func doCleanDirectories(fs billy.Filesystem, dir string) error { + files, err := fs.ReadDir(dir) if err != nil { return err } - if len(files) == 0 { - fs.Remove(path) + return fs.Remove(dir) } - return nil } diff --git a/worktree_test.go b/worktree_test.go index df191b0a0..c714011c0 100644 --- a/worktree_test.go +++ b/worktree_test.go @@ -1591,6 +1591,10 @@ func (s *WorktreeSuite) TestClean(c *C) { c.Assert(len(status), Equals, 1) + fi, err := fs.Lstat("pkgA") + c.Assert(err, IsNil) + c.Assert(fi.IsDir(), Equals, true) + // Clean with Dir: true. err = wt.Clean(&CleanOptions{Dir: true}) c.Assert(err, IsNil) @@ -1599,6 +1603,11 @@ func (s *WorktreeSuite) TestClean(c *C) { c.Assert(err, IsNil) c.Assert(len(status), Equals, 0) + + // An empty dir should be deleted, as well. + _, err = fs.Lstat("pkgA") + c.Assert(err, ErrorMatches, ".*(no such file or directory.*|.*file does not exist)*.") + } func (s *WorktreeSuite) TestAlternatesRepo(c *C) { From 75fa41d21c8d27ee0d5d7c7cb7ceeb2b765be330 Mon Sep 17 00:00:00 2001 From: kuba-- Date: Wed, 29 Aug 2018 14:56:25 +0200 Subject: [PATCH 083/120] Add Status.IsUntracked function Signed-off-by: kuba-- --- status.go | 13 +++++++++++-- worktree.go | 4 +--- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/status.go b/status.go index ef8a500f2..ecbf79350 100644 --- a/status.go +++ b/status.go @@ -1,7 +1,10 @@ package git -import "fmt" -import "bytes" +import ( + "bytes" + "fmt" + "path/filepath" +) // Status represents the current status of a Worktree. // The key of the map is the path of the file. @@ -17,6 +20,12 @@ func (s Status) File(path string) *FileStatus { return s[path] } +// IsUntracked checks if file for given path is 'Untracked' +func (s Status) IsUntracked(path string) bool { + stat, ok := (s)[filepath.ToSlash(path)] + return ok && stat.Worktree == Untracked +} + // IsClean returns true if all the files aren't in Unmodified status. func (s Status) IsClean() bool { for _, status := range s { diff --git a/worktree.go b/worktree.go index 921e600a0..e45d81548 100644 --- a/worktree.go +++ b/worktree.go @@ -750,9 +750,7 @@ func (w *Worktree) doClean(status Status, opts *CleanOptions, dir string, files return err } } else { - // check if file is 'Untracked' - s, ok := (status)[filepath.ToSlash(path)] - if ok && s.Worktree == Untracked { + if status.IsUntracked(path) { if err := w.Filesystem.Remove(path); err != nil { return err } From ba3ee05efbdeb11364d585ec4dfa84fe07e64430 Mon Sep 17 00:00:00 2001 From: Taru Karttunen Date: Wed, 29 Aug 2018 12:43:23 +0000 Subject: [PATCH 084/120] plumbing: object: Clamp object timestamps before unix epoch to unix epoch Signed-off-by: Taru Karttunen --- plumbing/object/object.go | 6 +++++- plumbing/object/tag_test.go | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/plumbing/object/object.go b/plumbing/object/object.go index 4b59aba7a..e960e50c9 100644 --- a/plumbing/object/object.go +++ b/plumbing/object/object.go @@ -152,7 +152,11 @@ func (s *Signature) decodeTimeAndTimeZone(b []byte) { } func (s *Signature) encodeTimeAndTimeZone(w io.Writer) error { - _, err := fmt.Fprintf(w, "%d %s", s.When.Unix(), s.When.Format("-0700")) + u := s.When.Unix() + if u < 0 { + u = 0 + } + _, err := fmt.Fprintf(w, "%d %s", u, s.When.Format("-0700")) return err } diff --git a/plumbing/object/tag_test.go b/plumbing/object/tag_test.go index 9900093e7..e7dd06e21 100644 --- a/plumbing/object/tag_test.go +++ b/plumbing/object/tag_test.go @@ -265,7 +265,7 @@ func (s *TagSuite) TestStringNonCommit(c *C) { c.Assert(tag.String(), Equals, "tag TAG TWO\n"+ "Tagger: <>\n"+ - "Date: Mon Jan 01 00:00:00 0001 +0000\n"+ + "Date: Thu Jan 01 00:00:00 1970 +0000\n"+ "\n"+ "tag two\n") } From 14d9faa61e2d38d31593c8536121d814947240fc Mon Sep 17 00:00:00 2001 From: Zaq? Wiedmann Date: Tue, 28 Aug 2018 17:32:29 -0700 Subject: [PATCH 085/120] config: add commentChar to core config struct Signed-off-by: Zaq? Wiedmann --- config/config.go | 5 +++++ config/config_test.go | 2 ++ 2 files changed, 7 insertions(+) diff --git a/config/config.go b/config/config.go index ce6506dae..a637f6d70 100644 --- a/config/config.go +++ b/config/config.go @@ -40,6 +40,9 @@ type Config struct { IsBare bool // Worktree is the path to the root of the working tree. Worktree string + // CommentChar is the character indicating the start of a + // comment for commands like commit and tag + CommentChar string } Pack struct { @@ -113,6 +116,7 @@ const ( urlKey = "url" bareKey = "bare" worktreeKey = "worktree" + commentCharKey = "commentChar" windowKey = "window" mergeKey = "merge" @@ -151,6 +155,7 @@ func (c *Config) unmarshalCore() { } c.Core.Worktree = s.Options.Get(worktreeKey) + c.Core.CommentChar = s.Options.Get(commentCharKey) } func (c *Config) unmarshalPack() error { diff --git a/config/config_test.go b/config/config_test.go index 5cd713e45..fe73de872 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -13,6 +13,7 @@ func (s *ConfigSuite) TestUnmarshall(c *C) { input := []byte(`[core] bare = true worktree = foo + commentchar = bar [pack] window = 20 [remote "origin"] @@ -38,6 +39,7 @@ func (s *ConfigSuite) TestUnmarshall(c *C) { c.Assert(cfg.Core.IsBare, Equals, true) c.Assert(cfg.Core.Worktree, Equals, "foo") + c.Assert(cfg.Core.CommentChar, Equals, "bar") c.Assert(cfg.Pack.Window, Equals, uint(20)) c.Assert(cfg.Remotes, HasLen, 2) c.Assert(cfg.Remotes["origin"].Name, Equals, "origin") From 1e1a7d0623459807d6f1e871492147f971f7540c Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Thu, 30 Aug 2018 15:29:51 +0200 Subject: [PATCH 086/120] git: add Static option to PlainOpen Also adds Static configuration to Storage and DotGit. This option means that the git repository is not expected to be modified while open and enables some optimizations. Each time a file is accessed the storer tries to open an object file for the requested hash. When this is done for a lot of objects it is expensive. With Static option a list of object files is generated the first time an object is accessed and used to check if exists instead of using system calls. A similar optimization is done for packfiles. Signed-off-by: Javi Fontan --- options.go | 2 + repository.go | 11 +- repository_test.go | 19 +++ storage/filesystem/dotgit/dotgit.go | 182 +++++++++++++++++++++++++++- storage/filesystem/storage.go | 27 ++++- storage/filesystem/storage_test.go | 18 +++ 6 files changed, 249 insertions(+), 10 deletions(-) diff --git a/options.go b/options.go index 7b1570f7b..7b551460c 100644 --- a/options.go +++ b/options.go @@ -431,6 +431,8 @@ type PlainOpenOptions struct { // DetectDotGit defines whether parent directories should be // walked until a .git directory or file is found. DetectDotGit bool + // Static means that the repository won't be modified while open. + Static bool } // Validate validates the fields and sets the default values. diff --git a/repository.go b/repository.go index 818cfb3c3..d99d6eb21 100644 --- a/repository.go +++ b/repository.go @@ -235,9 +235,8 @@ func PlainOpen(path string) (*Repository, error) { return PlainOpenWithOptions(path, &PlainOpenOptions{}) } -// PlainOpen opens a git repository from the given path. It detects if the -// repository is bare or a normal one. If the path doesn't contain a valid -// repository ErrRepositoryNotExists is returned +// PlainOpenWithOptions opens a git repository from the given path with specific +// options. See PlainOpen for more info. func PlainOpenWithOptions(path string, o *PlainOpenOptions) (*Repository, error) { dot, wt, err := dotGitToOSFilesystems(path, o.DetectDotGit) if err != nil { @@ -252,7 +251,11 @@ func PlainOpenWithOptions(path string, o *PlainOpenOptions) (*Repository, error) return nil, err } - s, err := filesystem.NewStorage(dot) + so := filesystem.StorageOptions{ + Static: o.Static, + } + + s, err := filesystem.NewStorageWithOptions(dot, so) if err != nil { return nil, err } diff --git a/repository_test.go b/repository_test.go index 261af7a7b..b891413f4 100644 --- a/repository_test.go +++ b/repository_test.go @@ -550,6 +550,25 @@ func (s *RepositorySuite) TestPlainOpenNotExistsDetectDotGit(c *C) { c.Assert(r, IsNil) } +func (s *RepositorySuite) TestPlainOpenStatic(c *C) { + dir, err := ioutil.TempDir("", "plain-open") + c.Assert(err, IsNil) + defer os.RemoveAll(dir) + + r, err := PlainInit(dir, true) + c.Assert(err, IsNil) + c.Assert(r, NotNil) + + op := &PlainOpenOptions{Static: true} + r, err = PlainOpenWithOptions(dir, op) + c.Assert(err, IsNil) + c.Assert(r, NotNil) + + sto, ok := r.Storer.(*filesystem.Storage) + c.Assert(ok, Equals, true) + c.Assert(sto.StorageOptions.Static, Equals, true) +} + func (s *RepositorySuite) TestPlainClone(c *C) { r, err := PlainClone(c.MkDir(), false, &CloneOptions{ URL: s.GetBasicLocalRepositoryURL(), diff --git a/storage/filesystem/dotgit/dotgit.go b/storage/filesystem/dotgit/dotgit.go index df4f75691..2048ddc29 100644 --- a/storage/filesystem/dotgit/dotgit.go +++ b/storage/filesystem/dotgit/dotgit.go @@ -60,18 +60,39 @@ var ( // The DotGit type represents a local git repository on disk. This // type is not zero-value-safe, use the New function to initialize it. type DotGit struct { + DotGitOptions fs billy.Filesystem // incoming object directory information incomingChecked bool incomingDirName string + + objectList []plumbing.Hash + objectMap map[plumbing.Hash]struct{} + packList []plumbing.Hash + packMap map[plumbing.Hash]struct{} +} + +// DotGitOptions holds configuration options for new DotGit objects. +type DotGitOptions struct { + // Static means that the filesystem won't be changed while the repo is open. + Static bool } // New returns a DotGit value ready to be used. The path argument must // be the absolute path of a git repository directory (e.g. // "/foo/bar/.git"). func New(fs billy.Filesystem) *DotGit { - return &DotGit{fs: fs} + return NewWithOptions(fs, DotGitOptions{}) +} + +// NewWithOptions creates a new DotGit and sets non default configuration +// options. See New for complete help. +func NewWithOptions(fs billy.Filesystem, o DotGitOptions) *DotGit { + return &DotGit{ + DotGitOptions: o, + fs: fs, + } } // Initialize creates all the folder scaffolding. @@ -143,11 +164,25 @@ func (d *DotGit) Shallow() (billy.File, error) { // NewObjectPack return a writer for a new packfile, it saves the packfile to // disk and also generates and save the index for the given packfile. func (d *DotGit) NewObjectPack() (*PackWriter, error) { + d.cleanPackList() return newPackWrite(d.fs) } // ObjectPacks returns the list of availables packfiles func (d *DotGit) ObjectPacks() ([]plumbing.Hash, error) { + if !d.Static { + return d.objectPacks() + } + + err := d.genPackList() + if err != nil { + return nil, err + } + + return d.packList, nil +} + +func (d *DotGit) objectPacks() ([]plumbing.Hash, error) { packDir := d.fs.Join(objectsPath, packPath) files, err := d.fs.ReadDir(packDir) if err != nil { @@ -181,6 +216,11 @@ func (d *DotGit) objectPackPath(hash plumbing.Hash, extension string) string { } func (d *DotGit) objectPackOpen(hash plumbing.Hash, extension string) (billy.File, error) { + err := d.hasPack(hash) + if err != nil { + return nil, err + } + pack, err := d.fs.Open(d.objectPackPath(hash, extension)) if err != nil { if os.IsNotExist(err) { @@ -195,15 +235,27 @@ func (d *DotGit) objectPackOpen(hash plumbing.Hash, extension string) (billy.Fil // ObjectPack returns a fs.File of the given packfile func (d *DotGit) ObjectPack(hash plumbing.Hash) (billy.File, error) { + err := d.hasPack(hash) + if err != nil { + return nil, err + } + return d.objectPackOpen(hash, `pack`) } // ObjectPackIdx returns a fs.File of the index file for a given packfile func (d *DotGit) ObjectPackIdx(hash plumbing.Hash) (billy.File, error) { + err := d.hasPack(hash) + if err != nil { + return nil, err + } + return d.objectPackOpen(hash, `idx`) } func (d *DotGit) DeleteOldObjectPackAndIndex(hash plumbing.Hash, t time.Time) error { + d.cleanPackList() + path := d.objectPackPath(hash, `pack`) if !t.IsZero() { fi, err := d.fs.Stat(path) @@ -224,12 +276,23 @@ func (d *DotGit) DeleteOldObjectPackAndIndex(hash plumbing.Hash, t time.Time) er // NewObject return a writer for a new object file. func (d *DotGit) NewObject() (*ObjectWriter, error) { + d.cleanObjectList() + return newObjectWriter(d.fs) } // Objects returns a slice with the hashes of objects found under the // .git/objects/ directory. func (d *DotGit) Objects() ([]plumbing.Hash, error) { + if d.Static { + err := d.genObjectList() + if err != nil { + return nil, err + } + + return d.objectList, nil + } + var objects []plumbing.Hash err := d.ForEachObjectHash(func(hash plumbing.Hash) error { objects = append(objects, hash) @@ -241,9 +304,29 @@ func (d *DotGit) Objects() ([]plumbing.Hash, error) { return objects, nil } -// Objects returns a slice with the hashes of objects found under the -// .git/objects/ directory. +// ForEachObjectHash iterates over the hashes of objects found under the +// .git/objects/ directory and executes the provided . func (d *DotGit) ForEachObjectHash(fun func(plumbing.Hash) error) error { + if !d.Static { + return d.forEachObjectHash(fun) + } + + err := d.genObjectList() + if err != nil { + return err + } + + for _, h := range d.objectList { + err := fun(h) + if err != nil { + return err + } + } + + return nil +} + +func (d *DotGit) forEachObjectHash(fun func(plumbing.Hash) error) error { files, err := d.fs.ReadDir(objectsPath) if err != nil { if os.IsNotExist(err) { @@ -278,6 +361,87 @@ func (d *DotGit) ForEachObjectHash(fun func(plumbing.Hash) error) error { return nil } +func (d *DotGit) cleanObjectList() { + d.objectMap = nil + d.objectList = nil +} + +func (d *DotGit) genObjectList() error { + if d.objectMap != nil { + return nil + } + + d.objectMap = make(map[plumbing.Hash]struct{}) + return d.forEachObjectHash(func(h plumbing.Hash) error { + d.objectList = append(d.objectList, h) + d.objectMap[h] = struct{}{} + + return nil + }) +} + +func (d *DotGit) hasObject(h plumbing.Hash) error { + if !d.Static { + return nil + } + + err := d.genObjectList() + if err != nil { + return err + } + + _, ok := d.objectMap[h] + if !ok { + return plumbing.ErrObjectNotFound + } + + return nil +} + +func (d *DotGit) cleanPackList() { + d.packMap = nil + d.packList = nil +} + +func (d *DotGit) genPackList() error { + if d.packMap != nil { + return nil + } + + op, err := d.objectPacks() + if err != nil { + return err + } + + d.packMap = make(map[plumbing.Hash]struct{}) + d.packList = nil + + for _, h := range op { + d.packList = append(d.packList, h) + d.packMap[h] = struct{}{} + } + + return nil +} + +func (d *DotGit) hasPack(h plumbing.Hash) error { + if !d.Static { + return nil + } + + err := d.genPackList() + if err != nil { + return err + } + + _, ok := d.packMap[h] + if !ok { + return ErrPackfileNotFound + } + + return nil +} + func (d *DotGit) objectPath(h plumbing.Hash) string { hash := h.String() return d.fs.Join(objectsPath, hash[0:2], hash[2:40]) @@ -322,6 +486,11 @@ func (d *DotGit) hasIncomingObjects() bool { // Object returns a fs.File pointing the object file, if exists func (d *DotGit) Object(h plumbing.Hash) (billy.File, error) { + err := d.hasObject(h) + if err != nil { + return nil, err + } + obj1, err1 := d.fs.Open(d.objectPath(h)) if os.IsNotExist(err1) && d.hasIncomingObjects() { obj2, err2 := d.fs.Open(d.incomingObjectPath(h)) @@ -335,6 +504,11 @@ func (d *DotGit) Object(h plumbing.Hash) (billy.File, error) { // ObjectStat returns a os.FileInfo pointing the object file, if exists func (d *DotGit) ObjectStat(h plumbing.Hash) (os.FileInfo, error) { + err := d.hasObject(h) + if err != nil { + return nil, err + } + obj1, err1 := d.fs.Stat(d.objectPath(h)) if os.IsNotExist(err1) && d.hasIncomingObjects() { obj2, err2 := d.fs.Stat(d.incomingObjectPath(h)) @@ -348,6 +522,8 @@ func (d *DotGit) ObjectStat(h plumbing.Hash) (os.FileInfo, error) { // ObjectDelete removes the object file, if exists func (d *DotGit) ObjectDelete(h plumbing.Hash) error { + d.cleanObjectList() + err1 := d.fs.Remove(d.objectPath(h)) if os.IsNotExist(err1) && d.hasIncomingObjects() { err2 := d.fs.Remove(d.incomingObjectPath(h)) diff --git a/storage/filesystem/storage.go b/storage/filesystem/storage.go index 622bb4a8d..a969a1fbc 100644 --- a/storage/filesystem/storage.go +++ b/storage/filesystem/storage.go @@ -11,6 +11,8 @@ import ( // standard git format (this is, the .git directory). Zero values of this type // are not safe to use, see the NewStorage function below. type Storage struct { + StorageOptions + fs billy.Filesystem dir *dotgit.DotGit @@ -22,17 +24,36 @@ type Storage struct { ModuleStorage } +// StorageOptions holds configuration for the storage. +type StorageOptions struct { + // Static means that the filesystem is not modified while the repo is open. + Static bool +} + // NewStorage returns a new Storage backed by a given `fs.Filesystem` func NewStorage(fs billy.Filesystem) (*Storage, error) { - dir := dotgit.New(fs) + return NewStorageWithOptions(fs, StorageOptions{}) +} + +// NewStorageWithOptions returns a new Storage backed by a given `fs.Filesystem` +func NewStorageWithOptions( + fs billy.Filesystem, + ops StorageOptions, +) (*Storage, error) { + dOps := dotgit.DotGitOptions{ + Static: ops.Static, + } + + dir := dotgit.NewWithOptions(fs, dOps) o, err := NewObjectStorage(dir) if err != nil { return nil, err } return &Storage{ - fs: fs, - dir: dir, + StorageOptions: ops, + fs: fs, + dir: dir, ObjectStorage: o, ReferenceStorage: ReferenceStorage{dir: dir}, diff --git a/storage/filesystem/storage_test.go b/storage/filesystem/storage_test.go index 4d9ba6fec..d7ebf7122 100644 --- a/storage/filesystem/storage_test.go +++ b/storage/filesystem/storage_test.go @@ -26,6 +26,10 @@ func (s *StorageSuite) SetUpTest(c *C) { storage, err := NewStorage(osfs.New(s.dir)) c.Assert(err, IsNil) + setUpTest(s, c, storage) +} + +func setUpTest(s *StorageSuite, c *C, storage *Storage) { // ensure that right interfaces are implemented var _ storer.EncodedObjectStorer = storage var _ storer.IndexStorer = storage @@ -51,3 +55,17 @@ func (s *StorageSuite) TestNewStorageShouldNotAddAnyContentsToDir(c *C) { c.Assert(err, IsNil) c.Assert(fis, HasLen, 0) } + +type StorageStaticSuite struct { + StorageSuite +} + +var _ = Suite(&StorageStaticSuite{}) + +func (s *StorageStaticSuite) SetUpTest(c *C) { + s.dir = c.MkDir() + storage, err := NewStorageWithOptions(osfs.New(s.dir), StorageOptions{Static: true}) + c.Assert(err, IsNil) + + setUpTest(&s.StorageSuite, c, storage) +} From 82945e31dd8bce5fc51d4fd16d696a6d326e5f44 Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Thu, 30 Aug 2018 18:33:37 +0200 Subject: [PATCH 087/120] git, storer: use a common storer.Options for storer and PlainOpen Signed-off-by: Javi Fontan --- options.go | 6 ++++-- plumbing/storer/storer.go | 6 ++++++ repository.go | 6 +----- repository_test.go | 7 +++++-- storage/filesystem/dotgit/dotgit.go | 17 ++++++----------- storage/filesystem/storage.go | 25 ++++++++----------------- storage/filesystem/storage_test.go | 4 +++- 7 files changed, 33 insertions(+), 38 deletions(-) diff --git a/options.go b/options.go index 7b551460c..f67a454d5 100644 --- a/options.go +++ b/options.go @@ -9,6 +9,7 @@ import ( "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/object" "gopkg.in/src-d/go-git.v4/plumbing/protocol/packp/sideband" + "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/plumbing/transport" ) @@ -428,11 +429,12 @@ func (o *GrepOptions) Validate(w *Worktree) error { // PlainOpenOptions describes how opening a plain repository should be // performed. type PlainOpenOptions struct { + // Storage layer options. + Storage storer.Options + // DetectDotGit defines whether parent directories should be // walked until a .git directory or file is found. DetectDotGit bool - // Static means that the repository won't be modified while open. - Static bool } // Validate validates the fields and sets the default values. diff --git a/plumbing/storer/storer.go b/plumbing/storer/storer.go index c7bc65a0c..1b7d2266f 100644 --- a/plumbing/storer/storer.go +++ b/plumbing/storer/storer.go @@ -13,3 +13,9 @@ type Initializer interface { // any. Init() error } + +// Options holds configuration for the storage. +type Options struct { + // Static means that the filesystem is not modified while the repo is open. + Static bool +} diff --git a/repository.go b/repository.go index d99d6eb21..4ad525279 100644 --- a/repository.go +++ b/repository.go @@ -251,11 +251,7 @@ func PlainOpenWithOptions(path string, o *PlainOpenOptions) (*Repository, error) return nil, err } - so := filesystem.StorageOptions{ - Static: o.Static, - } - - s, err := filesystem.NewStorageWithOptions(dot, so) + s, err := filesystem.NewStorageWithOptions(dot, o.Storage) if err != nil { return nil, err } diff --git a/repository_test.go b/repository_test.go index b891413f4..8956a9d3e 100644 --- a/repository_test.go +++ b/repository_test.go @@ -559,14 +559,17 @@ func (s *RepositorySuite) TestPlainOpenStatic(c *C) { c.Assert(err, IsNil) c.Assert(r, NotNil) - op := &PlainOpenOptions{Static: true} + op := &PlainOpenOptions{ + Storage: storer.Options{Static: true}, + } + r, err = PlainOpenWithOptions(dir, op) c.Assert(err, IsNil) c.Assert(r, NotNil) sto, ok := r.Storer.(*filesystem.Storage) c.Assert(ok, Equals, true) - c.Assert(sto.StorageOptions.Static, Equals, true) + c.Assert(sto.Options.Static, Equals, true) } func (s *RepositorySuite) TestPlainClone(c *C) { diff --git a/storage/filesystem/dotgit/dotgit.go b/storage/filesystem/dotgit/dotgit.go index 2048ddc29..41e5c7543 100644 --- a/storage/filesystem/dotgit/dotgit.go +++ b/storage/filesystem/dotgit/dotgit.go @@ -14,6 +14,7 @@ import ( "gopkg.in/src-d/go-billy.v4/osfs" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/utils/ioutil" "gopkg.in/src-d/go-billy.v4" @@ -60,7 +61,7 @@ var ( // The DotGit type represents a local git repository on disk. This // type is not zero-value-safe, use the New function to initialize it. type DotGit struct { - DotGitOptions + storer.Options fs billy.Filesystem // incoming object directory information @@ -73,25 +74,19 @@ type DotGit struct { packMap map[plumbing.Hash]struct{} } -// DotGitOptions holds configuration options for new DotGit objects. -type DotGitOptions struct { - // Static means that the filesystem won't be changed while the repo is open. - Static bool -} - // New returns a DotGit value ready to be used. The path argument must // be the absolute path of a git repository directory (e.g. // "/foo/bar/.git"). func New(fs billy.Filesystem) *DotGit { - return NewWithOptions(fs, DotGitOptions{}) + return NewWithOptions(fs, storer.Options{}) } // NewWithOptions creates a new DotGit and sets non default configuration // options. See New for complete help. -func NewWithOptions(fs billy.Filesystem, o DotGitOptions) *DotGit { +func NewWithOptions(fs billy.Filesystem, o storer.Options) *DotGit { return &DotGit{ - DotGitOptions: o, - fs: fs, + Options: o, + fs: fs, } } diff --git a/storage/filesystem/storage.go b/storage/filesystem/storage.go index a969a1fbc..24e64545e 100644 --- a/storage/filesystem/storage.go +++ b/storage/filesystem/storage.go @@ -2,6 +2,7 @@ package filesystem import ( + "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit" "gopkg.in/src-d/go-billy.v4" @@ -11,7 +12,7 @@ import ( // standard git format (this is, the .git directory). Zero values of this type // are not safe to use, see the NewStorage function below. type Storage struct { - StorageOptions + storer.Options fs billy.Filesystem dir *dotgit.DotGit @@ -24,36 +25,26 @@ type Storage struct { ModuleStorage } -// StorageOptions holds configuration for the storage. -type StorageOptions struct { - // Static means that the filesystem is not modified while the repo is open. - Static bool -} - // NewStorage returns a new Storage backed by a given `fs.Filesystem` func NewStorage(fs billy.Filesystem) (*Storage, error) { - return NewStorageWithOptions(fs, StorageOptions{}) + return NewStorageWithOptions(fs, storer.Options{}) } // NewStorageWithOptions returns a new Storage backed by a given `fs.Filesystem` func NewStorageWithOptions( fs billy.Filesystem, - ops StorageOptions, + ops storer.Options, ) (*Storage, error) { - dOps := dotgit.DotGitOptions{ - Static: ops.Static, - } - - dir := dotgit.NewWithOptions(fs, dOps) + dir := dotgit.NewWithOptions(fs, ops) o, err := NewObjectStorage(dir) if err != nil { return nil, err } return &Storage{ - StorageOptions: ops, - fs: fs, - dir: dir, + Options: ops, + fs: fs, + dir: dir, ObjectStorage: o, ReferenceStorage: ReferenceStorage{dir: dir}, diff --git a/storage/filesystem/storage_test.go b/storage/filesystem/storage_test.go index d7ebf7122..23628c7a7 100644 --- a/storage/filesystem/storage_test.go +++ b/storage/filesystem/storage_test.go @@ -64,7 +64,9 @@ var _ = Suite(&StorageStaticSuite{}) func (s *StorageStaticSuite) SetUpTest(c *C) { s.dir = c.MkDir() - storage, err := NewStorageWithOptions(osfs.New(s.dir), StorageOptions{Static: true}) + storage, err := NewStorageWithOptions( + osfs.New(s.dir), + storer.Options{Static: true}) c.Assert(err, IsNil) setUpTest(&s.StorageSuite, c, storage) From d7e6cf5b73947108d0c16b9c04b38891de47ef5d Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Thu, 30 Aug 2018 18:35:39 +0200 Subject: [PATCH 088/120] dotgit: fix typo in comment Signed-off-by: Javi Fontan --- storage/filesystem/dotgit/dotgit.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/filesystem/dotgit/dotgit.go b/storage/filesystem/dotgit/dotgit.go index 41e5c7543..c42ed88cd 100644 --- a/storage/filesystem/dotgit/dotgit.go +++ b/storage/filesystem/dotgit/dotgit.go @@ -300,7 +300,7 @@ func (d *DotGit) Objects() ([]plumbing.Hash, error) { } // ForEachObjectHash iterates over the hashes of objects found under the -// .git/objects/ directory and executes the provided . +// .git/objects/ directory and executes the provided function. func (d *DotGit) ForEachObjectHash(fun func(plumbing.Hash) error) error { if !d.Static { return d.forEachObjectHash(fun) From 2a7c664b62dd0d87f7ab67b30b1952727788cffa Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Fri, 31 Aug 2018 12:29:27 +0200 Subject: [PATCH 089/120] git: do not expose storage options in PlainOpen Signed-off-by: Javi Fontan --- options.go | 4 ---- repository.go | 2 +- repository_test.go | 22 ---------------------- 3 files changed, 1 insertion(+), 27 deletions(-) diff --git a/options.go b/options.go index f67a454d5..7b1570f7b 100644 --- a/options.go +++ b/options.go @@ -9,7 +9,6 @@ import ( "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/object" "gopkg.in/src-d/go-git.v4/plumbing/protocol/packp/sideband" - "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/plumbing/transport" ) @@ -429,9 +428,6 @@ func (o *GrepOptions) Validate(w *Worktree) error { // PlainOpenOptions describes how opening a plain repository should be // performed. type PlainOpenOptions struct { - // Storage layer options. - Storage storer.Options - // DetectDotGit defines whether parent directories should be // walked until a .git directory or file is found. DetectDotGit bool diff --git a/repository.go b/repository.go index 4ad525279..f619934a4 100644 --- a/repository.go +++ b/repository.go @@ -251,7 +251,7 @@ func PlainOpenWithOptions(path string, o *PlainOpenOptions) (*Repository, error) return nil, err } - s, err := filesystem.NewStorageWithOptions(dot, o.Storage) + s, err := filesystem.NewStorage(dot) if err != nil { return nil, err } diff --git a/repository_test.go b/repository_test.go index 8956a9d3e..261af7a7b 100644 --- a/repository_test.go +++ b/repository_test.go @@ -550,28 +550,6 @@ func (s *RepositorySuite) TestPlainOpenNotExistsDetectDotGit(c *C) { c.Assert(r, IsNil) } -func (s *RepositorySuite) TestPlainOpenStatic(c *C) { - dir, err := ioutil.TempDir("", "plain-open") - c.Assert(err, IsNil) - defer os.RemoveAll(dir) - - r, err := PlainInit(dir, true) - c.Assert(err, IsNil) - c.Assert(r, NotNil) - - op := &PlainOpenOptions{ - Storage: storer.Options{Static: true}, - } - - r, err = PlainOpenWithOptions(dir, op) - c.Assert(err, IsNil) - c.Assert(r, NotNil) - - sto, ok := r.Storer.(*filesystem.Storage) - c.Assert(ok, Equals, true) - c.Assert(sto.Options.Static, Equals, true) -} - func (s *RepositorySuite) TestPlainClone(c *C) { r, err := PlainClone(c.MkDir(), false, &CloneOptions{ URL: s.GetBasicLocalRepositoryURL(), From cf626677508238893c7c88c3c786a02f17afcc4c Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Fri, 31 Aug 2018 14:56:23 +0200 Subject: [PATCH 090/120] plumbing/storer: rename Static option to ExclusiveAccess Signed-off-by: Javi Fontan --- plumbing/storer/storer.go | 5 +++-- storage/filesystem/dotgit/dotgit.go | 10 +++++----- storage/filesystem/storage_test.go | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/plumbing/storer/storer.go b/plumbing/storer/storer.go index 1b7d2266f..9bbb44fe4 100644 --- a/plumbing/storer/storer.go +++ b/plumbing/storer/storer.go @@ -16,6 +16,7 @@ type Initializer interface { // Options holds configuration for the storage. type Options struct { - // Static means that the filesystem is not modified while the repo is open. - Static bool + // ExclusiveAccess means that the filesystem is not modified externally + // while the repo is open. + ExclusiveAccess bool } diff --git a/storage/filesystem/dotgit/dotgit.go b/storage/filesystem/dotgit/dotgit.go index c42ed88cd..7626078df 100644 --- a/storage/filesystem/dotgit/dotgit.go +++ b/storage/filesystem/dotgit/dotgit.go @@ -165,7 +165,7 @@ func (d *DotGit) NewObjectPack() (*PackWriter, error) { // ObjectPacks returns the list of availables packfiles func (d *DotGit) ObjectPacks() ([]plumbing.Hash, error) { - if !d.Static { + if !d.ExclusiveAccess { return d.objectPacks() } @@ -279,7 +279,7 @@ func (d *DotGit) NewObject() (*ObjectWriter, error) { // Objects returns a slice with the hashes of objects found under the // .git/objects/ directory. func (d *DotGit) Objects() ([]plumbing.Hash, error) { - if d.Static { + if d.ExclusiveAccess { err := d.genObjectList() if err != nil { return nil, err @@ -302,7 +302,7 @@ func (d *DotGit) Objects() ([]plumbing.Hash, error) { // ForEachObjectHash iterates over the hashes of objects found under the // .git/objects/ directory and executes the provided function. func (d *DotGit) ForEachObjectHash(fun func(plumbing.Hash) error) error { - if !d.Static { + if !d.ExclusiveAccess { return d.forEachObjectHash(fun) } @@ -376,7 +376,7 @@ func (d *DotGit) genObjectList() error { } func (d *DotGit) hasObject(h plumbing.Hash) error { - if !d.Static { + if !d.ExclusiveAccess { return nil } @@ -420,7 +420,7 @@ func (d *DotGit) genPackList() error { } func (d *DotGit) hasPack(h plumbing.Hash) error { - if !d.Static { + if !d.ExclusiveAccess { return nil } diff --git a/storage/filesystem/storage_test.go b/storage/filesystem/storage_test.go index 23628c7a7..11bf4fc3c 100644 --- a/storage/filesystem/storage_test.go +++ b/storage/filesystem/storage_test.go @@ -66,7 +66,7 @@ func (s *StorageStaticSuite) SetUpTest(c *C) { s.dir = c.MkDir() storage, err := NewStorageWithOptions( osfs.New(s.dir), - storer.Options{Static: true}) + storer.Options{ExclusiveAccess: true}) c.Assert(err, IsNil) setUpTest(&s.StorageSuite, c, storage) From 95acbf6c3958b7540a8549aa049051325fcecd8b Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Mon, 3 Sep 2018 11:17:22 +0200 Subject: [PATCH 091/120] storage/filesystem: make Storage options private Signed-off-by: Javi Fontan --- storage/filesystem/storage.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storage/filesystem/storage.go b/storage/filesystem/storage.go index 24e64545e..d2c528747 100644 --- a/storage/filesystem/storage.go +++ b/storage/filesystem/storage.go @@ -12,7 +12,7 @@ import ( // standard git format (this is, the .git directory). Zero values of this type // are not safe to use, see the NewStorage function below. type Storage struct { - storer.Options + options storer.Options fs billy.Filesystem dir *dotgit.DotGit @@ -42,7 +42,7 @@ func NewStorageWithOptions( } return &Storage{ - Options: ops, + options: ops, fs: fs, dir: dir, From 874f669becc25489081306bbbcbbc27b970f6295 Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Mon, 3 Sep 2018 19:40:22 +0200 Subject: [PATCH 092/120] storage/filesystem: move Options to filesytem and dotgit Signed-off-by: Javi Fontan --- plumbing/storer/storer.go | 7 ------- storage/filesystem/dotgit/dotgit.go | 28 +++++++++++++++++----------- storage/filesystem/object.go | 12 ++++++++++++ storage/filesystem/storage.go | 27 +++++++++++++++++---------- storage/filesystem/storage_test.go | 8 ++++---- 5 files changed, 50 insertions(+), 32 deletions(-) diff --git a/plumbing/storer/storer.go b/plumbing/storer/storer.go index 9bbb44fe4..c7bc65a0c 100644 --- a/plumbing/storer/storer.go +++ b/plumbing/storer/storer.go @@ -13,10 +13,3 @@ type Initializer interface { // any. Init() error } - -// Options holds configuration for the storage. -type Options struct { - // ExclusiveAccess means that the filesystem is not modified externally - // while the repo is open. - ExclusiveAccess bool -} diff --git a/storage/filesystem/dotgit/dotgit.go b/storage/filesystem/dotgit/dotgit.go index 7626078df..00dd2a459 100644 --- a/storage/filesystem/dotgit/dotgit.go +++ b/storage/filesystem/dotgit/dotgit.go @@ -14,7 +14,6 @@ import ( "gopkg.in/src-d/go-billy.v4/osfs" "gopkg.in/src-d/go-git.v4/plumbing" - "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/utils/ioutil" "gopkg.in/src-d/go-billy.v4" @@ -58,11 +57,18 @@ var ( ErrSymRefTargetNotFound = errors.New("symbolic reference target not found") ) +// Options holds configuration for the storage. +type Options struct { + // ExclusiveAccess means that the filesystem is not modified externally + // while the repo is open. + ExclusiveAccess bool +} + // The DotGit type represents a local git repository on disk. This // type is not zero-value-safe, use the New function to initialize it. type DotGit struct { - storer.Options - fs billy.Filesystem + options Options + fs billy.Filesystem // incoming object directory information incomingChecked bool @@ -78,14 +84,14 @@ type DotGit struct { // be the absolute path of a git repository directory (e.g. // "/foo/bar/.git"). func New(fs billy.Filesystem) *DotGit { - return NewWithOptions(fs, storer.Options{}) + return NewWithOptions(fs, Options{}) } // NewWithOptions creates a new DotGit and sets non default configuration // options. See New for complete help. -func NewWithOptions(fs billy.Filesystem, o storer.Options) *DotGit { +func NewWithOptions(fs billy.Filesystem, o Options) *DotGit { return &DotGit{ - Options: o, + options: o, fs: fs, } } @@ -165,7 +171,7 @@ func (d *DotGit) NewObjectPack() (*PackWriter, error) { // ObjectPacks returns the list of availables packfiles func (d *DotGit) ObjectPacks() ([]plumbing.Hash, error) { - if !d.ExclusiveAccess { + if !d.options.ExclusiveAccess { return d.objectPacks() } @@ -279,7 +285,7 @@ func (d *DotGit) NewObject() (*ObjectWriter, error) { // Objects returns a slice with the hashes of objects found under the // .git/objects/ directory. func (d *DotGit) Objects() ([]plumbing.Hash, error) { - if d.ExclusiveAccess { + if d.options.ExclusiveAccess { err := d.genObjectList() if err != nil { return nil, err @@ -302,7 +308,7 @@ func (d *DotGit) Objects() ([]plumbing.Hash, error) { // ForEachObjectHash iterates over the hashes of objects found under the // .git/objects/ directory and executes the provided function. func (d *DotGit) ForEachObjectHash(fun func(plumbing.Hash) error) error { - if !d.ExclusiveAccess { + if !d.options.ExclusiveAccess { return d.forEachObjectHash(fun) } @@ -376,7 +382,7 @@ func (d *DotGit) genObjectList() error { } func (d *DotGit) hasObject(h plumbing.Hash) error { - if !d.ExclusiveAccess { + if !d.options.ExclusiveAccess { return nil } @@ -420,7 +426,7 @@ func (d *DotGit) genPackList() error { } func (d *DotGit) hasPack(h plumbing.Hash) error { - if !d.ExclusiveAccess { + if !d.options.ExclusiveAccess { return nil } diff --git a/storage/filesystem/object.go b/storage/filesystem/object.go index 3a3a2bd97..3519385f6 100644 --- a/storage/filesystem/object.go +++ b/storage/filesystem/object.go @@ -18,6 +18,8 @@ import ( ) type ObjectStorage struct { + options Options + // deltaBaseCache is an object cache uses to cache delta's bases when deltaBaseCache cache.Object @@ -27,7 +29,17 @@ type ObjectStorage struct { // NewObjectStorage creates a new ObjectStorage with the given .git directory. func NewObjectStorage(dir *dotgit.DotGit) (ObjectStorage, error) { + return NewObjectStorageWithOptions(dir, Options{}) +} + +// NewObjectStorageWithOptions creates a new ObjectStorage with the given .git +// directory and sets its options. +func NewObjectStorageWithOptions( + dir *dotgit.DotGit, + ops Options, +) (ObjectStorage, error) { s := ObjectStorage{ + options: ops, deltaBaseCache: cache.NewObjectLRUDefault(), dir: dir, } diff --git a/storage/filesystem/storage.go b/storage/filesystem/storage.go index d2c528747..25b3653c0 100644 --- a/storage/filesystem/storage.go +++ b/storage/filesystem/storage.go @@ -2,7 +2,6 @@ package filesystem import ( - "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit" "gopkg.in/src-d/go-billy.v4" @@ -12,8 +11,6 @@ import ( // standard git format (this is, the .git directory). Zero values of this type // are not safe to use, see the NewStorage function below. type Storage struct { - options storer.Options - fs billy.Filesystem dir *dotgit.DotGit @@ -25,26 +22,36 @@ type Storage struct { ModuleStorage } +// Options holds configuration for the storage. +type Options struct { + // ExclusiveAccess means that the filesystem is not modified externally + // while the repo is open. + ExclusiveAccess bool +} + // NewStorage returns a new Storage backed by a given `fs.Filesystem` func NewStorage(fs billy.Filesystem) (*Storage, error) { - return NewStorageWithOptions(fs, storer.Options{}) + return NewStorageWithOptions(fs, Options{}) } // NewStorageWithOptions returns a new Storage backed by a given `fs.Filesystem` func NewStorageWithOptions( fs billy.Filesystem, - ops storer.Options, + ops Options, ) (*Storage, error) { - dir := dotgit.NewWithOptions(fs, ops) - o, err := NewObjectStorage(dir) + dirOps := dotgit.Options{ + ExclusiveAccess: ops.ExclusiveAccess, + } + + dir := dotgit.NewWithOptions(fs, dirOps) + o, err := NewObjectStorageWithOptions(dir, ops) if err != nil { return nil, err } return &Storage{ - options: ops, - fs: fs, - dir: dir, + fs: fs, + dir: dir, ObjectStorage: o, ReferenceStorage: ReferenceStorage{dir: dir}, diff --git a/storage/filesystem/storage_test.go b/storage/filesystem/storage_test.go index 11bf4fc3c..7f85ef548 100644 --- a/storage/filesystem/storage_test.go +++ b/storage/filesystem/storage_test.go @@ -56,17 +56,17 @@ func (s *StorageSuite) TestNewStorageShouldNotAddAnyContentsToDir(c *C) { c.Assert(fis, HasLen, 0) } -type StorageStaticSuite struct { +type StorageExclusiveSuite struct { StorageSuite } -var _ = Suite(&StorageStaticSuite{}) +var _ = Suite(&StorageExclusiveSuite{}) -func (s *StorageStaticSuite) SetUpTest(c *C) { +func (s *StorageExclusiveSuite) SetUpTest(c *C) { s.dir = c.MkDir() storage, err := NewStorageWithOptions( osfs.New(s.dir), - storer.Options{ExclusiveAccess: true}) + Options{ExclusiveAccess: true}) c.Assert(err, IsNil) setUpTest(&s.StorageSuite, c, storage) From 659ec443b4a975e3adf78f24e59ad69d210d2c0b Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Mon, 3 Sep 2018 20:02:25 +0200 Subject: [PATCH 093/120] storage/dotgit: add ExclusiveAccess tests in dotgit This functionality was already tested in storage/filesystem. The coverage tool only takes into account files from the same package of the test. Signed-off-by: Javi Fontan --- storage/filesystem/dotgit/dotgit_test.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/storage/filesystem/dotgit/dotgit_test.go b/storage/filesystem/dotgit/dotgit_test.go index 64c2aeead..c34543e1d 100644 --- a/storage/filesystem/dotgit/dotgit_test.go +++ b/storage/filesystem/dotgit/dotgit_test.go @@ -9,6 +9,7 @@ import ( "strings" "testing" + "gopkg.in/src-d/go-billy.v4" "gopkg.in/src-d/go-git.v4/plumbing" . "gopkg.in/check.v1" @@ -424,6 +425,18 @@ func (s *SuiteDotGit) TestObjectPacks(c *C) { fs := f.DotGit() dir := New(fs) + testObjectPacks(c, fs, dir, f) +} + +func (s *SuiteDotGit) TestObjectPacksExclusive(c *C) { + f := fixtures.Basic().ByTag(".git").One() + fs := f.DotGit() + dir := NewWithOptions(fs, Options{ExclusiveAccess: true}) + + testObjectPacks(c, fs, dir, f) +} + +func testObjectPacks(c *C, fs billy.Filesystem, dir *DotGit, f *fixtures.Fixture) { hashes, err := dir.ObjectPacks() c.Assert(err, IsNil) c.Assert(hashes, HasLen, 1) @@ -506,6 +519,17 @@ func (s *SuiteDotGit) TestObjects(c *C) { fs := fixtures.ByTag(".git").ByTag("unpacked").One().DotGit() dir := New(fs) + testObjects(c, fs, dir) +} + +func (s *SuiteDotGit) TestObjectsExclusive(c *C) { + fs := fixtures.ByTag(".git").ByTag("unpacked").One().DotGit() + dir := NewWithOptions(fs, Options{ExclusiveAccess: true}) + + testObjects(c, fs, dir) +} + +func testObjects(c *C, fs billy.Filesystem, dir *DotGit) { hashes, err := dir.Objects() c.Assert(err, IsNil) c.Assert(hashes, HasLen, 187) From 6384ab93a2dbac9045ee19099455cbcfe82d0201 Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Thu, 30 Aug 2018 20:28:40 +0200 Subject: [PATCH 094/120] storage/dotgit: add KeepDescriptors option This option maintains packfile file descriptors opened after reading objects from them. It improves performance as it does not have to be opening packfiles each time an object is needed. Also adds Close to EncodedObjectStorer to close all the files manualy. Signed-off-by: Javi Fontan --- plumbing/storer/object.go | 2 ++ plumbing/storer/object_test.go | 4 +++ storage/filesystem/dotgit/dotgit.go | 43 +++++++++++++++++++++++- storage/filesystem/dotgit/dotgit_test.go | 28 +++++++++++++++ storage/filesystem/object.go | 10 +++++- storage/filesystem/storage.go | 4 +++ storage/memory/storage.go | 3 ++ 7 files changed, 92 insertions(+), 2 deletions(-) diff --git a/plumbing/storer/object.go b/plumbing/storer/object.go index 92aa62918..806b6bb78 100644 --- a/plumbing/storer/object.go +++ b/plumbing/storer/object.go @@ -40,6 +40,8 @@ type EncodedObjectStorer interface { // HasEncodedObject returns ErrObjNotFound if the object doesn't // exist. If the object does exist, it returns nil. HasEncodedObject(plumbing.Hash) error + // Close any opened files. + Close() error } // DeltaObjectStorer is an EncodedObjectStorer that can return delta diff --git a/plumbing/storer/object_test.go b/plumbing/storer/object_test.go index 6b4fe0fb6..adddec45f 100644 --- a/plumbing/storer/object_test.go +++ b/plumbing/storer/object_test.go @@ -157,3 +157,7 @@ func (o *MockObjectStorage) IterEncodedObjects(t plumbing.ObjectType) (EncodedOb func (o *MockObjectStorage) Begin() Transaction { return nil } + +func (o *MockObjectStorage) Close() error { + return nil +} diff --git a/storage/filesystem/dotgit/dotgit.go b/storage/filesystem/dotgit/dotgit.go index 00dd2a459..df5cd10d7 100644 --- a/storage/filesystem/dotgit/dotgit.go +++ b/storage/filesystem/dotgit/dotgit.go @@ -62,6 +62,9 @@ type Options struct { // ExclusiveAccess means that the filesystem is not modified externally // while the repo is open. ExclusiveAccess bool + // KeepDescriptors makes the file descriptors to be reused but they will + // need to be manually closed calling Close(). + KeepDescriptors bool } // The DotGit type represents a local git repository on disk. This @@ -78,6 +81,8 @@ type DotGit struct { objectMap map[plumbing.Hash]struct{} packList []plumbing.Hash packMap map[plumbing.Hash]struct{} + + files map[string]billy.File } // New returns a DotGit value ready to be used. The path argument must @@ -123,6 +128,28 @@ func (d *DotGit) Initialize() error { return nil } +// Close closes all opened files. +func (d *DotGit) Close() error { + var firstError error + if d.files != nil { + for _, f := range d.files { + err := f.Close() + if err != nil && firstError == nil { + firstError = err + continue + } + } + + d.files = nil + } + + if firstError != nil { + return firstError + } + + return nil +} + // ConfigWriter returns a file pointer for write to the config file func (d *DotGit) ConfigWriter() (billy.File, error) { return d.fs.Create(configPath) @@ -217,12 +244,22 @@ func (d *DotGit) objectPackPath(hash plumbing.Hash, extension string) string { } func (d *DotGit) objectPackOpen(hash plumbing.Hash, extension string) (billy.File, error) { + if d.files == nil { + d.files = make(map[string]billy.File) + } + err := d.hasPack(hash) if err != nil { return nil, err } - pack, err := d.fs.Open(d.objectPackPath(hash, extension)) + path := d.objectPackPath(hash, extension) + f, ok := d.files[path] + if ok { + return f, nil + } + + pack, err := d.fs.Open(path) if err != nil { if os.IsNotExist(err) { return nil, ErrPackfileNotFound @@ -231,6 +268,10 @@ func (d *DotGit) objectPackOpen(hash plumbing.Hash, extension string) (billy.Fil return nil, err } + if d.options.KeepDescriptors && extension == "pack" { + d.files[path] = pack + } + return pack, nil } diff --git a/storage/filesystem/dotgit/dotgit_test.go b/storage/filesystem/dotgit/dotgit_test.go index c34543e1d..50f8e649a 100644 --- a/storage/filesystem/dotgit/dotgit_test.go +++ b/storage/filesystem/dotgit/dotgit_test.go @@ -465,6 +465,34 @@ func (s *SuiteDotGit) TestObjectPack(c *C) { c.Assert(filepath.Ext(pack.Name()), Equals, ".pack") } +func (s *SuiteDotGit) TestObjectPackWithKeepDescriptors(c *C) { + f := fixtures.Basic().ByTag(".git").One() + fs := f.DotGit() + dir := NewWithOptions(fs, Options{KeepDescriptors: true}) + + pack, err := dir.ObjectPack(f.PackfileHash) + c.Assert(err, IsNil) + c.Assert(filepath.Ext(pack.Name()), Equals, ".pack") + + pack2, err := dir.ObjectPack(f.PackfileHash) + c.Assert(err, IsNil) + c.Assert(pack, Equals, pack2) + + err = dir.Close() + c.Assert(err, IsNil) + + pack2, err = dir.ObjectPack(f.PackfileHash) + c.Assert(err, IsNil) + c.Assert(pack, Not(Equals), pack2) + + err = pack2.Close() + c.Assert(err, IsNil) + + err = dir.Close() + c.Assert(err, NotNil) + +} + func (s *SuiteDotGit) TestObjectPackIdx(c *C) { f := fixtures.Basic().ByTag(".git").One() fs := f.DotGit() diff --git a/storage/filesystem/object.go b/storage/filesystem/object.go index 3519385f6..3545e2751 100644 --- a/storage/filesystem/object.go +++ b/storage/filesystem/object.go @@ -74,6 +74,7 @@ func (s *ObjectStorage) loadIdxFile(h plumbing.Hash) (err error) { } defer ioutil.CheckClose(f, &err) + idxf := idxfile.NewMemoryIndex() d := idxfile.NewDecoder(f) if err = d.Decode(idxf); err != nil { @@ -280,7 +281,9 @@ func (s *ObjectStorage) getFromPackfile(h plumbing.Hash, canBeDelta bool) ( return nil, err } - defer ioutil.CheckClose(f, &err) + if !s.options.KeepDescriptors { + defer ioutil.CheckClose(f, &err) + } idx := s.index[pack] if canBeDelta { @@ -423,6 +426,11 @@ func (s *ObjectStorage) buildPackfileIters(t plumbing.ObjectType, seen map[plumb }, nil } +// Close closes all opened files. +func (s *ObjectStorage) Close() error { + return s.dir.Close() +} + type lazyPackfilesIter struct { hashes []plumbing.Hash open func(h plumbing.Hash) (storer.EncodedObjectIter, error) diff --git a/storage/filesystem/storage.go b/storage/filesystem/storage.go index 25b3653c0..7fae7897c 100644 --- a/storage/filesystem/storage.go +++ b/storage/filesystem/storage.go @@ -27,6 +27,9 @@ type Options struct { // ExclusiveAccess means that the filesystem is not modified externally // while the repo is open. ExclusiveAccess bool + // KeepDescriptors makes the file descriptors to be reused but they will + // need to be manually closed calling Close(). + KeepDescriptors bool } // NewStorage returns a new Storage backed by a given `fs.Filesystem` @@ -41,6 +44,7 @@ func NewStorageWithOptions( ) (*Storage, error) { dirOps := dotgit.Options{ ExclusiveAccess: ops.ExclusiveAccess, + KeepDescriptors: ops.KeepDescriptors, } dir := dotgit.NewWithOptions(fs, dirOps) diff --git a/storage/memory/storage.go b/storage/memory/storage.go index 2e3250905..3d3b348bf 100644 --- a/storage/memory/storage.go +++ b/storage/memory/storage.go @@ -183,6 +183,9 @@ func (o *ObjectStorage) ObjectPacks() ([]plumbing.Hash, error) { func (o *ObjectStorage) DeleteOldObjectPackAndIndex(plumbing.Hash, time.Time) error { return nil } +func (s *ObjectStorage) Close() error { + return nil +} var errNotSupported = fmt.Errorf("Not supported") From 5260b87fd08f70df6f1eb297ccb04d74d32dd6c8 Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Tue, 4 Sep 2018 18:26:59 +0200 Subject: [PATCH 095/120] plumbing/storer: do not expose Close in EncodedObjectStorer interface Signed-off-by: Javi Fontan --- plumbing/storer/object.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/plumbing/storer/object.go b/plumbing/storer/object.go index 806b6bb78..92aa62918 100644 --- a/plumbing/storer/object.go +++ b/plumbing/storer/object.go @@ -40,8 +40,6 @@ type EncodedObjectStorer interface { // HasEncodedObject returns ErrObjNotFound if the object doesn't // exist. If the object does exist, it returns nil. HasEncodedObject(plumbing.Hash) error - // Close any opened files. - Close() error } // DeltaObjectStorer is an EncodedObjectStorer that can return delta From 9013dde72d0387a74b728ee336019728ba159d1c Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Wed, 5 Sep 2018 11:01:06 +0200 Subject: [PATCH 096/120] storage/filesystem: add KeepDescriptors test Also delete Close from MockObjectStorage and memory storer. Signed-off-by: Javi Fontan --- plumbing/storer/object_test.go | 4 ---- storage/filesystem/object_test.go | 31 +++++++++++++++++++++++++++++++ storage/memory/storage.go | 3 --- 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/plumbing/storer/object_test.go b/plumbing/storer/object_test.go index adddec45f..6b4fe0fb6 100644 --- a/plumbing/storer/object_test.go +++ b/plumbing/storer/object_test.go @@ -157,7 +157,3 @@ func (o *MockObjectStorage) IterEncodedObjects(t plumbing.ObjectType) (EncodedOb func (o *MockObjectStorage) Begin() Transaction { return nil } - -func (o *MockObjectStorage) Close() error { - return nil -} diff --git a/storage/filesystem/object_test.go b/storage/filesystem/object_test.go index b1408b7c2..6feb6ae14 100644 --- a/storage/filesystem/object_test.go +++ b/storage/filesystem/object_test.go @@ -48,6 +48,37 @@ func (s *FsSuite) TestGetFromPackfile(c *C) { }) } +func (s *FsSuite) TestGetFromPackfileKeepDescriptors(c *C) { + fixtures.Basic().ByTag(".git").Test(c, func(f *fixtures.Fixture) { + fs := f.DotGit() + dg := dotgit.NewWithOptions(fs, dotgit.Options{KeepDescriptors: true}) + o, err := NewObjectStorageWithOptions(dg, Options{KeepDescriptors: true}) + c.Assert(err, IsNil) + + expected := plumbing.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5") + obj, err := o.EncodedObject(plumbing.AnyObject, expected) + c.Assert(err, IsNil) + c.Assert(obj.Hash(), Equals, expected) + + packfiles, err := dg.ObjectPacks() + c.Assert(err, IsNil) + + pack1, err := dg.ObjectPack(packfiles[0]) + c.Assert(err, IsNil) + + err = o.Close() + c.Assert(err, IsNil) + + pack2, err := dg.ObjectPack(packfiles[0]) + c.Assert(err, IsNil) + c.Assert(pack1, Not(Equals), pack2) + + err = o.Close() + c.Assert(err, IsNil) + + }) +} + func (s *FsSuite) TestGetFromPackfileMultiplePackfiles(c *C) { fs := fixtures.ByTag(".git").ByTag("multi-packfile").One().DotGit() o, err := NewObjectStorage(dotgit.New(fs)) diff --git a/storage/memory/storage.go b/storage/memory/storage.go index 3d3b348bf..2e3250905 100644 --- a/storage/memory/storage.go +++ b/storage/memory/storage.go @@ -183,9 +183,6 @@ func (o *ObjectStorage) ObjectPacks() ([]plumbing.Hash, error) { func (o *ObjectStorage) DeleteOldObjectPackAndIndex(plumbing.Hash, time.Time) error { return nil } -func (s *ObjectStorage) Close() error { - return nil -} var errNotSupported = fmt.Errorf("Not supported") From 8176f084d861891d1846a2d46bf669d0d3463ebd Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Thu, 6 Sep 2018 19:09:02 +0200 Subject: [PATCH 097/120] storage/filesystem: compare files using offset in test Using equals to compare files it uses diff to do so. This can potentially consume lots of ram. Changed the comparison to use file offsets. If the descriptor is reused the offset is maintained. Signed-off-by: Javi Fontan --- storage/filesystem/dotgit/dotgit_test.go | 15 +++++++++++++-- storage/filesystem/object_test.go | 8 +++++++- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/storage/filesystem/dotgit/dotgit_test.go b/storage/filesystem/dotgit/dotgit_test.go index 50f8e649a..308c6b70f 100644 --- a/storage/filesystem/dotgit/dotgit_test.go +++ b/storage/filesystem/dotgit/dotgit_test.go @@ -474,16 +474,27 @@ func (s *SuiteDotGit) TestObjectPackWithKeepDescriptors(c *C) { c.Assert(err, IsNil) c.Assert(filepath.Ext(pack.Name()), Equals, ".pack") + // Move to an specific offset + pack.Seek(42, os.SEEK_SET) + pack2, err := dir.ObjectPack(f.PackfileHash) c.Assert(err, IsNil) - c.Assert(pack, Equals, pack2) + + // If the file is the same the offset should be the same + offset, err := pack2.Seek(0, os.SEEK_CUR) + c.Assert(err, IsNil) + c.Assert(offset, Equals, int64(42)) err = dir.Close() c.Assert(err, IsNil) pack2, err = dir.ObjectPack(f.PackfileHash) c.Assert(err, IsNil) - c.Assert(pack, Not(Equals), pack2) + + // If the file is opened again its offset should be 0 + offset, err = pack2.Seek(0, os.SEEK_CUR) + c.Assert(err, IsNil) + c.Assert(offset, Equals, int64(0)) err = pack2.Close() c.Assert(err, IsNil) diff --git a/storage/filesystem/object_test.go b/storage/filesystem/object_test.go index 6feb6ae14..4a921a9e6 100644 --- a/storage/filesystem/object_test.go +++ b/storage/filesystem/object_test.go @@ -2,6 +2,7 @@ package filesystem import ( "io/ioutil" + "os" "testing" "gopkg.in/src-d/go-git.v4/plumbing" @@ -66,12 +67,17 @@ func (s *FsSuite) TestGetFromPackfileKeepDescriptors(c *C) { pack1, err := dg.ObjectPack(packfiles[0]) c.Assert(err, IsNil) + pack1.Seek(42, os.SEEK_SET) + err = o.Close() c.Assert(err, IsNil) pack2, err := dg.ObjectPack(packfiles[0]) c.Assert(err, IsNil) - c.Assert(pack1, Not(Equals), pack2) + + offset, err := pack2.Seek(0, os.SEEK_CUR) + c.Assert(err, IsNil) + c.Assert(offset, Equals, int64(0)) err = o.Close() c.Assert(err, IsNil) From a4b12e4161738af6f724776c0c8c55f90542f06f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kuba=20Podg=C3=B3rski?= Date: Fri, 7 Sep 2018 10:25:23 +0200 Subject: [PATCH 098/120] plumbing/transport: ssh check if list of known_hosts files is empty Signed-off-by: kuba-- --- plumbing/transport/ssh/auth_method.go | 14 ++--- plumbing/transport/ssh/auth_method_test.go | 62 +++++++++++++++++++++- 2 files changed, 69 insertions(+), 7 deletions(-) diff --git a/plumbing/transport/ssh/auth_method.go b/plumbing/transport/ssh/auth_method.go index 84cfab2a6..dbb47c56b 100644 --- a/plumbing/transport/ssh/auth_method.go +++ b/plumbing/transport/ssh/auth_method.go @@ -236,7 +236,7 @@ func (a *PublicKeysCallback) ClientConfig() (*ssh.ClientConfig, error) { // NewKnownHostsCallback returns ssh.HostKeyCallback based on a file based on a // known_hosts file. http://man.openbsd.org/sshd#SSH_KNOWN_HOSTS_FILE_FORMAT // -// If files is empty, the list of files will be read from the SSH_KNOWN_HOSTS +// If list of files is empty, then it will be read from the SSH_KNOWN_HOSTS // environment variable, example: // /home/foo/custom_known_hosts_file:/etc/custom_known/hosts_file // @@ -244,13 +244,15 @@ func (a *PublicKeysCallback) ClientConfig() (*ssh.ClientConfig, error) { // ~/.ssh/known_hosts // /etc/ssh/ssh_known_hosts func NewKnownHostsCallback(files ...string) (ssh.HostKeyCallback, error) { - files, err := getDefaultKnownHostsFiles() - if err != nil { - return nil, err + var err error + + if len(files) == 0 { + if files, err = getDefaultKnownHostsFiles(); err != nil { + return nil, err + } } - files, err = filterKnownHostsFiles(files...) - if err != nil { + if files, err = filterKnownHostsFiles(files...); err != nil { return nil, err } diff --git a/plumbing/transport/ssh/auth_method_test.go b/plumbing/transport/ssh/auth_method_test.go index 00256694f..0cde61eea 100644 --- a/plumbing/transport/ssh/auth_method_test.go +++ b/plumbing/transport/ssh/auth_method_test.go @@ -1,16 +1,30 @@ package ssh import ( + "bufio" "fmt" "io/ioutil" "os" + "strings" + "golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh/testdata" . "gopkg.in/check.v1" ) -type SuiteCommon struct{} +type ( + SuiteCommon struct{} + + mockKnownHosts struct{} +) + +func (mockKnownHosts) host() string { return "github.com" } +func (mockKnownHosts) knownHosts() []byte { + return []byte(`github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==`) +} +func (mockKnownHosts) Network() string { return "tcp" } +func (mockKnownHosts) String() string { return "github.com:22" } var _ = Suite(&SuiteCommon{}) @@ -149,3 +163,49 @@ func (*SuiteCommon) TestNewPublicKeysWithInvalidPEM(c *C) { c.Assert(err, NotNil) c.Assert(auth, IsNil) } + +func (*SuiteCommon) TestNewKnownHostsCallback(c *C) { + var mock = mockKnownHosts{} + + f, err := ioutil.TempFile("", "known-hosts") + c.Assert(err, IsNil) + + _, err = f.Write(mock.knownHosts()) + c.Assert(err, IsNil) + + err = f.Close() + c.Assert(err, IsNil) + + defer os.RemoveAll(f.Name()) + + f, err = os.Open(f.Name()) + c.Assert(err, IsNil) + + defer f.Close() + + var hostKey ssh.PublicKey + scanner := bufio.NewScanner(f) + for scanner.Scan() { + fields := strings.Split(scanner.Text(), " ") + if len(fields) != 3 { + continue + } + if strings.Contains(fields[0], mock.host()) { + var err error + hostKey, _, _, _, err = ssh.ParseAuthorizedKey(scanner.Bytes()) + if err != nil { + c.Fatalf("error parsing %q: %v", fields[2], err) + } + break + } + } + if hostKey == nil { + c.Fatalf("no hostkey for %s", mock.host()) + } + + clb, err := NewKnownHostsCallback(f.Name()) + c.Assert(err, IsNil) + + err = clb(mock.String(), mock, hostKey) + c.Assert(err, IsNil) +} From 80170bd73d5d6298ea6d40c66987fcde8148f1e8 Mon Sep 17 00:00:00 2001 From: Antonio Jesus Navarro Perez Date: Fri, 7 Sep 2018 10:50:31 +0200 Subject: [PATCH 099/120] Fix fatal corrupt patch in unified diff format Signed-off-by: Antonio Jesus Navarro Perez --- plumbing/format/diff/unified_encoder.go | 8 +++-- plumbing/format/diff/unified_encoder_test.go | 37 ++++++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/plumbing/format/diff/unified_encoder.go b/plumbing/format/diff/unified_encoder.go index 58edd9516..8bd6d8abc 100644 --- a/plumbing/format/diff/unified_encoder.go +++ b/plumbing/format/diff/unified_encoder.go @@ -237,9 +237,13 @@ func (c *hunksGenerator) addLineNumbers(la, lb int, linesBefore int, i int, op O // we need to search for a reference for the next diff switch { case linesBefore != 0 && c.ctxLines != 0: - clb = lb - c.ctxLines + 1 + if lb > c.ctxLines { + clb = lb - c.ctxLines + 1 + } else { + clb = 1 + } case c.ctxLines == 0: - clb = lb - c.ctxLines + clb = lb case i != len(c.chunks)-1: next := c.chunks[i+1] if next.Type() == op || next.Type() == Equal { diff --git a/plumbing/format/diff/unified_encoder_test.go b/plumbing/format/diff/unified_encoder_test.go index 0e419ca0d..7736af19f 100644 --- a/plumbing/format/diff/unified_encoder_test.go +++ b/plumbing/format/diff/unified_encoder_test.go @@ -150,6 +150,43 @@ var oneChunkPatchInverted Patch = testPatch{ } var fixtures []*fixture = []*fixture{{ + patch: testPatch{ + message: "", + filePatches: []testFilePatch{{ + from: &testFile{ + mode: filemode.Regular, + path: "README.md", + seed: "hello\nworld\n", + }, + to: &testFile{ + mode: filemode.Regular, + path: "README.md", + seed: "hello\nbug\n", + }, + chunks: []testChunk{{ + content: "hello", + op: Equal, + }, { + content: "world", + op: Delete, + }, { + content: "bug", + op: Add, + }}, + }}, + }, + desc: "positive negative number", + context: 2, + diff: `diff --git a/README.md b/README.md +index 94954abda49de8615a048f8d2e64b5de848e27a1..f3dad9514629b9ff9136283ae331ad1fc95748a8 100644 +--- a/README.md ++++ b/README.md +@@ -1,2 +1,2 @@ + hello +-world ++bug +`, +}, { patch: testPatch{ message: "", filePatches: []testFilePatch{{ From 8f6b3127c1ff7661113fff2662416c328971a285 Mon Sep 17 00:00:00 2001 From: kuba-- Date: Fri, 7 Sep 2018 09:27:35 +0200 Subject: [PATCH 100/120] Expose Storage cache. Signed-off-by: kuba-- --- common_test.go | 11 ++---- .../format/packfile/encoder_advanced_test.go | 7 ++-- plumbing/object/change_adaptor_test.go | 4 +-- plumbing/object/change_test.go | 7 ++-- plumbing/object/commit_test.go | 4 +-- plumbing/object/difftree_test.go | 4 +-- plumbing/object/file_test.go | 17 ++++----- plumbing/object/object_test.go | 4 +-- plumbing/object/patch_test.go | 5 +-- plumbing/object/tag_test.go | 5 ++- plumbing/object/tree_test.go | 4 +-- plumbing/revlist/revlist_test.go | 12 +++---- plumbing/transport/server/loader.go | 3 +- plumbing/transport/server/server_test.go | 4 +-- prune_test.go | 4 +-- remote_test.go | 35 +++++++------------ repository.go | 11 ++---- repository_test.go | 30 ++++++---------- storage/filesystem/dotgit/dotgit.go | 4 +-- storage/filesystem/module.go | 3 +- storage/filesystem/object.go | 29 ++++++--------- storage/filesystem/object_test.go | 25 +++++-------- storage/filesystem/storage.go | 29 ++++++++------- storage/filesystem/storage_test.go | 11 +++--- worktree_commit_test.go | 4 +-- 25 files changed, 111 insertions(+), 165 deletions(-) diff --git a/common_test.go b/common_test.go index efe1ecc92..dad0a3777 100644 --- a/common_test.go +++ b/common_test.go @@ -4,6 +4,7 @@ import ( "testing" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/format/packfile" "gopkg.in/src-d/go-git.v4/plumbing/transport" "gopkg.in/src-d/go-git.v4/storage/filesystem" @@ -59,10 +60,7 @@ func (s *BaseSuite) NewRepository(f *fixtures.Fixture) *Repository { dotgit = f.DotGit() worktree = memfs.New() - st, err := filesystem.NewStorage(dotgit) - if err != nil { - panic(err) - } + st := filesystem.NewStorage(dotgit, cache.NewObjectLRUDefault()) r, err := Open(st, worktree) if err != nil { @@ -89,10 +87,7 @@ func (s *BaseSuite) NewRepositoryWithEmptyWorktree(f *fixtures.Fixture) *Reposit worktree := memfs.New() - st, err := filesystem.NewStorage(dotgit) - if err != nil { - panic(err) - } + st := filesystem.NewStorage(dotgit, cache.NewObjectLRUDefault()) r, err := Open(st, worktree) if err != nil { diff --git a/plumbing/format/packfile/encoder_advanced_test.go b/plumbing/format/packfile/encoder_advanced_test.go index fc1419eea..e15126e66 100644 --- a/plumbing/format/packfile/encoder_advanced_test.go +++ b/plumbing/format/packfile/encoder_advanced_test.go @@ -8,6 +8,7 @@ import ( "gopkg.in/src-d/go-billy.v4/memfs" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/format/idxfile" . "gopkg.in/src-d/go-git.v4/plumbing/format/packfile" "gopkg.in/src-d/go-git.v4/plumbing/storer" @@ -32,8 +33,7 @@ func (s *EncoderAdvancedSuite) TestEncodeDecode(c *C) { fixs = append(fixs, fixtures.ByURL("https://github.com/src-d/go-git.git"). ByTag("packfile").ByTag(".git").One()) fixs.Test(c, func(f *fixtures.Fixture) { - storage, err := filesystem.NewStorage(f.DotGit()) - c.Assert(err, IsNil) + storage := filesystem.NewStorage(f.DotGit(), cache.NewObjectLRUDefault()) s.testEncodeDecode(c, storage, 10) }) } @@ -47,8 +47,7 @@ func (s *EncoderAdvancedSuite) TestEncodeDecodeNoDeltaCompression(c *C) { fixs = append(fixs, fixtures.ByURL("https://github.com/src-d/go-git.git"). ByTag("packfile").ByTag(".git").One()) fixs.Test(c, func(f *fixtures.Fixture) { - storage, err := filesystem.NewStorage(f.DotGit()) - c.Assert(err, IsNil) + storage := filesystem.NewStorage(f.DotGit(), cache.NewObjectLRUDefault()) s.testEncodeDecode(c, storage, 0) }) } diff --git a/plumbing/object/change_adaptor_test.go b/plumbing/object/change_adaptor_test.go index 803c3b89a..c7c003b96 100644 --- a/plumbing/object/change_adaptor_test.go +++ b/plumbing/object/change_adaptor_test.go @@ -4,6 +4,7 @@ import ( "sort" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/filemode" "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/storage/filesystem" @@ -23,8 +24,7 @@ type ChangeAdaptorSuite struct { func (s *ChangeAdaptorSuite) SetUpSuite(c *C) { s.Suite.SetUpSuite(c) s.Fixture = fixtures.Basic().One() - sto, err := filesystem.NewStorage(s.Fixture.DotGit()) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(s.Fixture.DotGit(), cache.NewObjectLRUDefault()) s.Storer = sto } diff --git a/plumbing/object/change_test.go b/plumbing/object/change_test.go index b0e89c711..e2f0a2346 100644 --- a/plumbing/object/change_test.go +++ b/plumbing/object/change_test.go @@ -5,6 +5,7 @@ import ( "sort" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/filemode" "gopkg.in/src-d/go-git.v4/plumbing/format/diff" "gopkg.in/src-d/go-git.v4/plumbing/storer" @@ -25,8 +26,7 @@ func (s *ChangeSuite) SetUpSuite(c *C) { s.Suite.SetUpSuite(c) s.Fixture = fixtures.ByURL("https://github.com/src-d/go-git.git"). ByTag(".git").One() - sto, err := filesystem.NewStorage(s.Fixture.DotGit()) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(s.Fixture.DotGit(), cache.NewObjectLRUDefault()) s.Storer = sto } @@ -253,8 +253,7 @@ func (s *ChangeSuite) TestNoFileFilemodes(c *C) { s.Suite.SetUpSuite(c) f := fixtures.ByURL("https://github.com/git-fixtures/submodule.git").One() - sto, err := filesystem.NewStorage(f.DotGit()) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(f.DotGit(), cache.NewObjectLRUDefault()) iter, err := sto.IterEncodedObjects(plumbing.AnyObject) c.Assert(err, IsNil) diff --git a/plumbing/object/commit_test.go b/plumbing/object/commit_test.go index e72b703d1..c9acf42f3 100644 --- a/plumbing/object/commit_test.go +++ b/plumbing/object/commit_test.go @@ -8,6 +8,7 @@ import ( "time" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" . "gopkg.in/check.v1" "gopkg.in/src-d/go-git-fixtures.v3" @@ -247,8 +248,7 @@ func (s *SuiteCommit) TestStringMultiLine(c *C) { hash := plumbing.NewHash("e7d896db87294e33ca3202e536d4d9bb16023db3") f := fixtures.ByURL("https://github.com/src-d/go-git.git").One() - sto, err := filesystem.NewStorage(f.DotGit()) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(f.DotGit(), cache.NewObjectLRUDefault()) o, err := sto.EncodedObject(plumbing.CommitObject, hash) c.Assert(err, IsNil) diff --git a/plumbing/object/difftree_test.go b/plumbing/object/difftree_test.go index ff9ecbc3f..4af86840a 100644 --- a/plumbing/object/difftree_test.go +++ b/plumbing/object/difftree_test.go @@ -4,6 +4,7 @@ import ( "sort" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/filemode" "gopkg.in/src-d/go-git.v4/plumbing/format/packfile" "gopkg.in/src-d/go-git.v4/plumbing/storer" @@ -25,8 +26,7 @@ type DiffTreeSuite struct { func (s *DiffTreeSuite) SetUpSuite(c *C) { s.Suite.SetUpSuite(c) s.Fixture = fixtures.Basic().One() - sto, err := filesystem.NewStorage(s.Fixture.DotGit()) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(s.Fixture.DotGit(), cache.NewObjectLRUDefault()) s.Storer = sto s.cache = make(map[string]storer.EncodedObjectStorer) } diff --git a/plumbing/object/file_test.go b/plumbing/object/file_test.go index edb82d01c..4b92749cb 100644 --- a/plumbing/object/file_test.go +++ b/plumbing/object/file_test.go @@ -4,6 +4,7 @@ import ( "io" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/filemode" "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/storage/filesystem" @@ -44,8 +45,7 @@ var fileIterTests = []struct { func (s *FileSuite) TestIter(c *C) { for i, t := range fileIterTests { f := fixtures.ByURL(t.repo).One() - sto, err := filesystem.NewStorage(f.DotGit()) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(f.DotGit(), cache.NewObjectLRUDefault()) h := plumbing.NewHash(t.commit) commit, err := GetCommit(sto, h) @@ -106,8 +106,7 @@ hs_err_pid* func (s *FileSuite) TestContents(c *C) { for i, t := range contentsTests { f := fixtures.ByURL(t.repo).One() - sto, err := filesystem.NewStorage(f.DotGit()) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(f.DotGit(), cache.NewObjectLRUDefault()) h := plumbing.NewHash(t.commit) commit, err := GetCommit(sto, h) @@ -160,8 +159,7 @@ var linesTests = []struct { func (s *FileSuite) TestLines(c *C) { for i, t := range linesTests { f := fixtures.ByURL(t.repo).One() - sto, err := filesystem.NewStorage(f.DotGit()) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(f.DotGit(), cache.NewObjectLRUDefault()) h := plumbing.NewHash(t.commit) commit, err := GetCommit(sto, h) @@ -195,8 +193,7 @@ var ignoreEmptyDirEntriesTests = []struct { func (s *FileSuite) TestIgnoreEmptyDirEntries(c *C) { for i, t := range ignoreEmptyDirEntriesTests { f := fixtures.ByURL(t.repo).One() - sto, err := filesystem.NewStorage(f.DotGit()) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(f.DotGit(), cache.NewObjectLRUDefault()) h := plumbing.NewHash(t.commit) commit, err := GetCommit(sto, h) @@ -251,9 +248,7 @@ func (s *FileSuite) TestFileIter(c *C) { func (s *FileSuite) TestFileIterSubmodule(c *C) { dotgit := fixtures.ByURL("https://github.com/git-fixtures/submodule.git").One().DotGit() - st, err := filesystem.NewStorage(dotgit) - - c.Assert(err, IsNil) + st := filesystem.NewStorage(dotgit, cache.NewObjectLRUDefault()) hash := plumbing.NewHash("b685400c1f9316f350965a5993d350bc746b0bf4") commit, err := GetCommit(st, hash) diff --git a/plumbing/object/object_test.go b/plumbing/object/object_test.go index 68aa1a13e..8f0eededd 100644 --- a/plumbing/object/object_test.go +++ b/plumbing/object/object_test.go @@ -7,6 +7,7 @@ import ( "time" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/filemode" "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/storage/filesystem" @@ -26,8 +27,7 @@ type BaseObjectsSuite struct { func (s *BaseObjectsSuite) SetUpSuite(c *C) { s.Suite.SetUpSuite(c) s.Fixture = fixtures.Basic().One() - storer, err := filesystem.NewStorage(s.Fixture.DotGit()) - c.Assert(err, IsNil) + storer := filesystem.NewStorage(s.Fixture.DotGit(), cache.NewObjectLRUDefault()) s.Storer = storer } diff --git a/plumbing/object/patch_test.go b/plumbing/object/patch_test.go index 8eb65ec30..47057fba7 100644 --- a/plumbing/object/patch_test.go +++ b/plumbing/object/patch_test.go @@ -4,6 +4,7 @@ import ( . "gopkg.in/check.v1" fixtures "gopkg.in/src-d/go-git-fixtures.v3" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/storage/filesystem" ) @@ -14,8 +15,8 @@ type PatchSuite struct { var _ = Suite(&PatchSuite{}) func (s *PatchSuite) TestStatsWithSubmodules(c *C) { - storer, err := filesystem.NewStorage( - fixtures.ByURL("https://github.com/git-fixtures/submodule.git").One().DotGit()) + storer := filesystem.NewStorage( + fixtures.ByURL("https://github.com/git-fixtures/submodule.git").One().DotGit(), cache.NewObjectLRUDefault()) commit, err := GetCommit(storer, plumbing.NewHash("b685400c1f9316f350965a5993d350bc746b0bf4")) diff --git a/plumbing/object/tag_test.go b/plumbing/object/tag_test.go index e7dd06e21..59c28b022 100644 --- a/plumbing/object/tag_test.go +++ b/plumbing/object/tag_test.go @@ -7,6 +7,7 @@ import ( "time" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/storage/filesystem" "gopkg.in/src-d/go-git.v4/storage/memory" @@ -22,9 +23,7 @@ var _ = Suite(&TagSuite{}) func (s *TagSuite) SetUpSuite(c *C) { s.BaseObjectsSuite.SetUpSuite(c) - storer, err := filesystem.NewStorage( - fixtures.ByURL("https://github.com/git-fixtures/tags.git").One().DotGit()) - c.Assert(err, IsNil) + storer := filesystem.NewStorage(fixtures.ByURL("https://github.com/git-fixtures/tags.git").One().DotGit(), cache.NewObjectLRUDefault()) s.Storer = storer } diff --git a/plumbing/object/tree_test.go b/plumbing/object/tree_test.go index 59d5d215f..736642186 100644 --- a/plumbing/object/tree_test.go +++ b/plumbing/object/tree_test.go @@ -5,6 +5,7 @@ import ( "io" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/filemode" "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/storage/filesystem" @@ -341,8 +342,7 @@ func (s *TreeSuite) TestTreeWalkerNextNonRecursive(c *C) { func (s *TreeSuite) TestTreeWalkerNextSubmodule(c *C) { dotgit := fixtures.ByURL("https://github.com/git-fixtures/submodule.git").One().DotGit() - st, err := filesystem.NewStorage(dotgit) - c.Assert(err, IsNil) + st := filesystem.NewStorage(dotgit, cache.NewObjectLRUDefault()) hash := plumbing.NewHash("b685400c1f9316f350965a5993d350bc746b0bf4") commit, err := GetCommit(st, hash) diff --git a/plumbing/revlist/revlist_test.go b/plumbing/revlist/revlist_test.go index 55d9bca2a..dea1c73d9 100644 --- a/plumbing/revlist/revlist_test.go +++ b/plumbing/revlist/revlist_test.go @@ -4,6 +4,7 @@ import ( "testing" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/object" "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/storage/filesystem" @@ -51,8 +52,7 @@ const ( func (s *RevListSuite) SetUpTest(c *C) { s.Suite.SetUpSuite(c) - sto, err := filesystem.NewStorage(fixtures.Basic().One().DotGit()) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(fixtures.Basic().One().DotGit(), cache.NewObjectLRUDefault()) s.Storer = sto } @@ -67,8 +67,7 @@ func (s *RevListSuite) TestRevListObjects_Submodules(c *C) { "6ecf0ef2c2dffb796033e5a02219af86ec6584e5": true, } - sto, err := filesystem.NewStorage(fixtures.ByTag("submodule").One().DotGit()) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(fixtures.ByTag("submodule").One().DotGit(), cache.NewObjectLRUDefault()) ref, err := storer.ResolveReference(sto, plumbing.HEAD) c.Assert(err, IsNil) @@ -109,10 +108,9 @@ func (s *RevListSuite) TestRevListObjects(c *C) { } func (s *RevListSuite) TestRevListObjectsTagObject(c *C) { - sto, err := filesystem.NewStorage( + sto := filesystem.NewStorage( fixtures.ByTag("tags"). - ByURL("https://github.com/git-fixtures/tags.git").One().DotGit()) - c.Assert(err, IsNil) + ByURL("https://github.com/git-fixtures/tags.git").One().DotGit(), cache.NewObjectLRUDefault()) expected := map[string]bool{ "70846e9a10ef7b41064b40f07713d5b8b9a8fc73": true, diff --git a/plumbing/transport/server/loader.go b/plumbing/transport/server/loader.go index c83752c25..13b35262d 100644 --- a/plumbing/transport/server/loader.go +++ b/plumbing/transport/server/loader.go @@ -1,6 +1,7 @@ package server import ( + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/plumbing/transport" "gopkg.in/src-d/go-git.v4/storage/filesystem" @@ -43,7 +44,7 @@ func (l *fsLoader) Load(ep *transport.Endpoint) (storer.Storer, error) { return nil, transport.ErrRepositoryNotFound } - return filesystem.NewStorage(fs) + return filesystem.NewStorage(fs, cache.NewObjectLRUDefault()), nil } // MapLoader is a Loader that uses a lookup map of storer.Storer by diff --git a/plumbing/transport/server/server_test.go b/plumbing/transport/server/server_test.go index 33d74d19f..302ff486e 100644 --- a/plumbing/transport/server/server_test.go +++ b/plumbing/transport/server/server_test.go @@ -3,6 +3,7 @@ package server_test import ( "testing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/transport" "gopkg.in/src-d/go-git.v4/plumbing/transport/client" "gopkg.in/src-d/go-git.v4/plumbing/transport/server" @@ -53,8 +54,7 @@ func (s *BaseSuite) prepareRepositories(c *C) { fs := fixtures.Basic().One().DotGit() s.Endpoint, err = transport.NewEndpoint(fs.Root()) c.Assert(err, IsNil) - s.loader[s.Endpoint.String()], err = filesystem.NewStorage(fs) - c.Assert(err, IsNil) + s.loader[s.Endpoint.String()] = filesystem.NewStorage(fs, cache.NewObjectLRUDefault()) s.EmptyEndpoint, err = transport.NewEndpoint("/empty.git") c.Assert(err, IsNil) diff --git a/prune_test.go b/prune_test.go index 60652ec9f..670cd07bd 100644 --- a/prune_test.go +++ b/prune_test.go @@ -4,6 +4,7 @@ import ( "time" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/storage" "gopkg.in/src-d/go-git.v4/storage/filesystem" @@ -22,8 +23,7 @@ func (s *PruneSuite) testPrune(c *C, deleteTime time.Time) { srcFs := fixtures.ByTag("unpacked").One().DotGit() var sto storage.Storer var err error - sto, err = filesystem.NewStorage(srcFs) - c.Assert(err, IsNil) + sto = filesystem.NewStorage(srcFs, cache.NewObjectLRUDefault()) los := sto.(storer.LooseObjectStorer) c.Assert(los, NotNil) diff --git a/remote_test.go b/remote_test.go index dd386b083..175faed36 100644 --- a/remote_test.go +++ b/remote_test.go @@ -9,6 +9,7 @@ import ( "gopkg.in/src-d/go-git.v4/config" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/protocol/packp" "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/storage" @@ -238,7 +239,7 @@ func (s *RemoteSuite) TestFetchWithPackfileWriter(c *C) { defer os.RemoveAll(dir) // clean up - fss, err := filesystem.NewStorage(osfs.New(dir)) + fss := filesystem.NewStorage(osfs.New(dir), cache.NewObjectLRUDefault()) c.Assert(err, IsNil) mock := &mockPackfileWriter{Storer: fss} @@ -375,8 +376,7 @@ func (s *RemoteSuite) TestFetchFastForwardFS(c *C) { defer os.RemoveAll(dir) // clean up - fss, err := filesystem.NewStorage(osfs.New(dir)) - c.Assert(err, IsNil) + fss := filesystem.NewStorage(osfs.New(dir), cache.NewObjectLRUDefault()) // This exercises `storage.filesystem.Storage.CheckAndSetReference()`. s.testFetchFastForward(c, fss) @@ -400,8 +400,7 @@ func (s *RemoteSuite) TestPushToEmptyRepository(c *C) { c.Assert(err, IsNil) srcFs := fixtures.Basic().One().DotGit() - sto, err := filesystem.NewStorage(srcFs) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(srcFs, cache.NewObjectLRUDefault()) r := newRemote(sto, &config.RemoteConfig{ Name: DefaultRemoteName, @@ -438,8 +437,7 @@ func (s *RemoteSuite) TestPushContext(c *C) { c.Assert(err, IsNil) fs := fixtures.ByURL("https://github.com/git-fixtures/tags.git").One().DotGit() - sto, err := filesystem.NewStorage(fs) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(fs, cache.NewObjectLRUDefault()) r := newRemote(sto, &config.RemoteConfig{ Name: DefaultRemoteName, @@ -461,8 +459,7 @@ func (s *RemoteSuite) TestPushTags(c *C) { c.Assert(err, IsNil) fs := fixtures.ByURL("https://github.com/git-fixtures/tags.git").One().DotGit() - sto, err := filesystem.NewStorage(fs) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(fs, cache.NewObjectLRUDefault()) r := newRemote(sto, &config.RemoteConfig{ Name: DefaultRemoteName, @@ -485,15 +482,14 @@ func (s *RemoteSuite) TestPushTags(c *C) { func (s *RemoteSuite) TestPushNoErrAlreadyUpToDate(c *C) { fs := fixtures.Basic().One().DotGit() - sto, err := filesystem.NewStorage(fs) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(fs, cache.NewObjectLRUDefault()) r := newRemote(sto, &config.RemoteConfig{ Name: DefaultRemoteName, URLs: []string{fs.Root()}, }) - err = r.Push(&PushOptions{ + err := r.Push(&PushOptions{ RefSpecs: []config.RefSpec{"refs/heads/*:refs/heads/*"}, }) c.Assert(err, Equals, NoErrAlreadyUpToDate) @@ -501,8 +497,7 @@ func (s *RemoteSuite) TestPushNoErrAlreadyUpToDate(c *C) { func (s *RemoteSuite) TestPushDeleteReference(c *C) { fs := fixtures.Basic().One().DotGit() - sto, err := filesystem.NewStorage(fs) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(fs, cache.NewObjectLRUDefault()) r, err := PlainClone(c.MkDir(), true, &CloneOptions{ URL: fs.Root(), @@ -526,8 +521,7 @@ func (s *RemoteSuite) TestPushDeleteReference(c *C) { func (s *RemoteSuite) TestPushRejectNonFastForward(c *C) { fs := fixtures.Basic().One().DotGit() - server, err := filesystem.NewStorage(fs) - c.Assert(err, IsNil) + server := filesystem.NewStorage(fs, cache.NewObjectLRUDefault()) r, err := PlainClone(c.MkDir(), true, &CloneOptions{ URL: fs.Root(), @@ -554,12 +548,10 @@ func (s *RemoteSuite) TestPushRejectNonFastForward(c *C) { func (s *RemoteSuite) TestPushForce(c *C) { f := fixtures.Basic().One() - sto, err := filesystem.NewStorage(f.DotGit()) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(f.DotGit(), cache.NewObjectLRUDefault()) dstFs := f.DotGit() - dstSto, err := filesystem.NewStorage(dstFs) - c.Assert(err, IsNil) + dstSto := filesystem.NewStorage(dstFs, cache.NewObjectLRUDefault()) url := dstFs.Root() r := newRemote(sto, &config.RemoteConfig{ @@ -703,8 +695,7 @@ func (s *RemoteSuite) TestPushWrongRemoteName(c *C) { func (s *RemoteSuite) TestGetHaves(c *C) { f := fixtures.Basic().One() - sto, err := filesystem.NewStorage(f.DotGit()) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(f.DotGit(), cache.NewObjectLRUDefault()) var localRefs = []*plumbing.Reference{ plumbing.NewReferenceFromStrings( diff --git a/repository.go b/repository.go index f619934a4..bfe06a369 100644 --- a/repository.go +++ b/repository.go @@ -13,6 +13,7 @@ import ( "gopkg.in/src-d/go-git.v4/config" "gopkg.in/src-d/go-git.v4/internal/revision" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/format/packfile" "gopkg.in/src-d/go-git.v4/plumbing/object" "gopkg.in/src-d/go-git.v4/plumbing/storer" @@ -220,10 +221,7 @@ func PlainInit(path string, isBare bool) (*Repository, error) { dot, _ = wt.Chroot(GitDirName) } - s, err := filesystem.NewStorage(dot) - if err != nil { - return nil, err - } + s := filesystem.NewStorage(dot, cache.NewObjectLRUDefault()) return Init(s, wt) } @@ -251,10 +249,7 @@ func PlainOpenWithOptions(path string, o *PlainOpenOptions) (*Repository, error) return nil, err } - s, err := filesystem.NewStorage(dot) - if err != nil { - return nil, err - } + s := filesystem.NewStorage(dot, cache.NewObjectLRUDefault()) return Open(s, wt) } diff --git a/repository_test.go b/repository_test.go index 261af7a7b..88071cfd5 100644 --- a/repository_test.go +++ b/repository_test.go @@ -15,6 +15,7 @@ import ( "gopkg.in/src-d/go-git.v4/config" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/object" "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/storage" @@ -51,8 +52,7 @@ func (s *RepositorySuite) TestInitNonStandardDotGit(c *C) { fs := osfs.New(dir) dot, _ := fs.Chroot("storage") - storage, err := filesystem.NewStorage(dot) - c.Assert(err, IsNil) + storage := filesystem.NewStorage(dot, cache.NewObjectLRUDefault()) wt, _ := fs.Chroot("worktree") r, err := Init(storage, wt) @@ -78,8 +78,7 @@ func (s *RepositorySuite) TestInitStandardDotGit(c *C) { fs := osfs.New(dir) dot, _ := fs.Chroot(".git") - storage, err := filesystem.NewStorage(dot) - c.Assert(err, IsNil) + storage := filesystem.NewStorage(dot, cache.NewObjectLRUDefault()) r, err := Init(storage, fs) c.Assert(err, IsNil) @@ -1058,8 +1057,7 @@ func (s *RepositorySuite) TestPushDepth(c *C) { func (s *RepositorySuite) TestPushNonExistentRemote(c *C) { srcFs := fixtures.Basic().One().DotGit() - sto, err := filesystem.NewStorage(srcFs) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(srcFs, cache.NewObjectLRUDefault()) r, err := Open(sto, srcFs) c.Assert(err, IsNil) @@ -1277,8 +1275,7 @@ func (s *RepositorySuite) TestTags(c *C) { func (s *RepositorySuite) TestBranches(c *C) { f := fixtures.ByURL("https://github.com/git-fixtures/root-references.git").One() - sto, err := filesystem.NewStorage(f.DotGit()) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(f.DotGit(), cache.NewObjectLRUDefault()) r, err := Open(sto, f.DotGit()) c.Assert(err, IsNil) @@ -1495,8 +1492,7 @@ func (s *RepositorySuite) TestWorktreeBare(c *C) { func (s *RepositorySuite) TestResolveRevision(c *C) { f := fixtures.ByURL("https://github.com/git-fixtures/basic.git").One() - sto, err := filesystem.NewStorage(f.DotGit()) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(f.DotGit(), cache.NewObjectLRUDefault()) r, err := Open(sto, f.DotGit()) c.Assert(err, IsNil) @@ -1548,9 +1544,9 @@ func (s *RepositorySuite) TestResolveRevisionWithErrors(c *C) { c.Assert(err, IsNil) datas := map[string]string{ - "efs/heads/master~": "reference not found", - "HEAD^3": `Revision invalid : "3" found must be 0, 1 or 2 after "^"`, - "HEAD^{/whatever}": `No commit message match regexp : "whatever"`, + "efs/heads/master~": "reference not found", + "HEAD^3": `Revision invalid : "3" found must be 0, 1 or 2 after "^"`, + "HEAD^{/whatever}": `No commit message match regexp : "whatever"`, "4e1243bd22c66e76c2ba9eddc1f91394e57f9f83": "reference not found", "918c48b83bd081e863dbe1b80f8998f058cd8294": `refname "918c48b83bd081e863dbe1b80f8998f058cd8294" is ambiguous`, } @@ -1567,8 +1563,7 @@ func (s *RepositorySuite) testRepackObjects( srcFs := fixtures.ByTag("unpacked").One().DotGit() var sto storage.Storer var err error - sto, err = filesystem.NewStorage(srcFs) - c.Assert(err, IsNil) + sto = filesystem.NewStorage(srcFs, cache.NewObjectLRUDefault()) los := sto.(storer.LooseObjectStorer) c.Assert(los, NotNil) @@ -1733,10 +1728,7 @@ func BenchmarkObjects(b *testing.B) { b.Run(f.URL, func(b *testing.B) { fs := f.DotGit() - storer, err := filesystem.NewStorage(fs) - if err != nil { - b.Fatal(err) - } + storer := filesystem.NewStorage(fs, cache.NewObjectLRUDefault()) worktree, err := fs.Chroot(filepath.Dir(fs.Root())) if err != nil { diff --git a/storage/filesystem/dotgit/dotgit.go b/storage/filesystem/dotgit/dotgit.go index df5cd10d7..a58c2482a 100644 --- a/storage/filesystem/dotgit/dotgit.go +++ b/storage/filesystem/dotgit/dotgit.go @@ -92,8 +92,8 @@ func New(fs billy.Filesystem) *DotGit { return NewWithOptions(fs, Options{}) } -// NewWithOptions creates a new DotGit and sets non default configuration -// options. See New for complete help. +// NewWithOptions sets non default configuration options. +// See New for complete help. func NewWithOptions(fs billy.Filesystem, o Options) *DotGit { return &DotGit{ options: o, diff --git a/storage/filesystem/module.go b/storage/filesystem/module.go index 7c8c8d866..927220674 100644 --- a/storage/filesystem/module.go +++ b/storage/filesystem/module.go @@ -1,6 +1,7 @@ package filesystem import ( + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/storage" "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit" ) @@ -15,5 +16,5 @@ func (s *ModuleStorage) Module(name string) (storage.Storer, error) { return nil, err } - return NewStorage(fs) + return NewStorage(fs, cache.NewObjectLRUDefault()), nil } diff --git a/storage/filesystem/object.go b/storage/filesystem/object.go index 3545e2751..9eb085fd3 100644 --- a/storage/filesystem/object.go +++ b/storage/filesystem/object.go @@ -27,24 +27,18 @@ type ObjectStorage struct { index map[plumbing.Hash]idxfile.Index } -// NewObjectStorage creates a new ObjectStorage with the given .git directory. -func NewObjectStorage(dir *dotgit.DotGit) (ObjectStorage, error) { - return NewObjectStorageWithOptions(dir, Options{}) -} - -// NewObjectStorageWithOptions creates a new ObjectStorage with the given .git -// directory and sets its options. -func NewObjectStorageWithOptions( - dir *dotgit.DotGit, - ops Options, -) (ObjectStorage, error) { - s := ObjectStorage{ +// NewObjectStorage creates a new ObjectStorage with the given .git directory and cache. +func NewObjectStorage(dir *dotgit.DotGit, cache cache.Object) *ObjectStorage { + return NewObjectStorageWithOptions(dir, cache, Options{}) +} + +// NewObjectStorageWithOptions creates a new ObjectStorage with the given .git directory, cache and extra options +func NewObjectStorageWithOptions(dir *dotgit.DotGit, cache cache.Object, ops Options) *ObjectStorage { + return &ObjectStorage{ options: ops, - deltaBaseCache: cache.NewObjectLRUDefault(), + deltaBaseCache: cache, dir: dir, } - - return s, nil } func (s *ObjectStorage) requireIndex() error { @@ -182,10 +176,7 @@ func (s *ObjectStorage) EncodedObject(t plumbing.ObjectType, h plumbing.Hash) (p // Create a new object storage with the DotGit(s) and check for the // required hash object. Skip when not found. for _, dg := range dotgits { - o, oe := NewObjectStorage(dg) - if oe != nil { - continue - } + o := NewObjectStorage(dg, s.deltaBaseCache) enobj, enerr := o.EncodedObject(t, h) if enerr != nil { continue diff --git a/storage/filesystem/object_test.go b/storage/filesystem/object_test.go index 4a921a9e6..bd4a94b40 100644 --- a/storage/filesystem/object_test.go +++ b/storage/filesystem/object_test.go @@ -6,6 +6,7 @@ import ( "testing" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit" . "gopkg.in/check.v1" @@ -27,8 +28,7 @@ var _ = Suite(&FsSuite{}) func (s *FsSuite) TestGetFromObjectFile(c *C) { fs := fixtures.ByTag(".git").ByTag("unpacked").One().DotGit() - o, err := NewObjectStorage(dotgit.New(fs)) - c.Assert(err, IsNil) + o := NewObjectStorage(dotgit.New(fs), cache.NewObjectLRUDefault()) expected := plumbing.NewHash("f3dfe29d268303fc6e1bbce268605fc99573406e") obj, err := o.EncodedObject(plumbing.AnyObject, expected) @@ -39,8 +39,7 @@ func (s *FsSuite) TestGetFromObjectFile(c *C) { func (s *FsSuite) TestGetFromPackfile(c *C) { fixtures.Basic().ByTag(".git").Test(c, func(f *fixtures.Fixture) { fs := f.DotGit() - o, err := NewObjectStorage(dotgit.New(fs)) - c.Assert(err, IsNil) + o := NewObjectStorage(dotgit.New(fs), cache.NewObjectLRUDefault()) expected := plumbing.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5") obj, err := o.EncodedObject(plumbing.AnyObject, expected) @@ -53,8 +52,7 @@ func (s *FsSuite) TestGetFromPackfileKeepDescriptors(c *C) { fixtures.Basic().ByTag(".git").Test(c, func(f *fixtures.Fixture) { fs := f.DotGit() dg := dotgit.NewWithOptions(fs, dotgit.Options{KeepDescriptors: true}) - o, err := NewObjectStorageWithOptions(dg, Options{KeepDescriptors: true}) - c.Assert(err, IsNil) + o := NewObjectStorageWithOptions(dg, cache.NewObjectLRUDefault(), Options{KeepDescriptors: true}) expected := plumbing.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5") obj, err := o.EncodedObject(plumbing.AnyObject, expected) @@ -87,8 +85,7 @@ func (s *FsSuite) TestGetFromPackfileKeepDescriptors(c *C) { func (s *FsSuite) TestGetFromPackfileMultiplePackfiles(c *C) { fs := fixtures.ByTag(".git").ByTag("multi-packfile").One().DotGit() - o, err := NewObjectStorage(dotgit.New(fs)) - c.Assert(err, IsNil) + o := NewObjectStorage(dotgit.New(fs), cache.NewObjectLRUDefault()) expected := plumbing.NewHash("8d45a34641d73851e01d3754320b33bb5be3c4d3") obj, err := o.getFromPackfile(expected, false) @@ -104,8 +101,7 @@ func (s *FsSuite) TestGetFromPackfileMultiplePackfiles(c *C) { func (s *FsSuite) TestIter(c *C) { fixtures.ByTag(".git").ByTag("packfile").Test(c, func(f *fixtures.Fixture) { fs := f.DotGit() - o, err := NewObjectStorage(dotgit.New(fs)) - c.Assert(err, IsNil) + o := NewObjectStorage(dotgit.New(fs), cache.NewObjectLRUDefault()) iter, err := o.IterEncodedObjects(plumbing.AnyObject) c.Assert(err, IsNil) @@ -125,8 +121,7 @@ func (s *FsSuite) TestIterWithType(c *C) { fixtures.ByTag(".git").Test(c, func(f *fixtures.Fixture) { for _, t := range objectTypes { fs := f.DotGit() - o, err := NewObjectStorage(dotgit.New(fs)) - c.Assert(err, IsNil) + o := NewObjectStorage(dotgit.New(fs), cache.NewObjectLRUDefault()) iter, err := o.IterEncodedObjects(t) c.Assert(err, IsNil) @@ -308,11 +303,7 @@ func BenchmarkGetObjectFromPackfile(b *testing.B) { for _, f := range fixtures.Basic() { b.Run(f.URL, func(b *testing.B) { fs := f.DotGit() - o, err := NewObjectStorage(dotgit.New(fs)) - if err != nil { - b.Fatal(err) - } - + o := NewObjectStorage(dotgit.New(fs), cache.NewObjectLRUDefault()) for i := 0; i < b.N; i++ { expected := plumbing.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5") obj, err := o.EncodedObject(plumbing.AnyObject, expected) diff --git a/storage/filesystem/storage.go b/storage/filesystem/storage.go index 7fae7897c..14a772abe 100644 --- a/storage/filesystem/storage.go +++ b/storage/filesystem/storage.go @@ -2,6 +2,7 @@ package filesystem import ( + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit" "gopkg.in/src-d/go-billy.v4" @@ -32,38 +33,35 @@ type Options struct { KeepDescriptors bool } -// NewStorage returns a new Storage backed by a given `fs.Filesystem` -func NewStorage(fs billy.Filesystem) (*Storage, error) { - return NewStorageWithOptions(fs, Options{}) +// NewStorage returns a new Storage backed by a given `fs.Filesystem` and cache. +func NewStorage(fs billy.Filesystem, cache cache.Object) *Storage { + return NewStorageWithOptions(fs, cache, Options{}) } -// NewStorageWithOptions returns a new Storage backed by a given `fs.Filesystem` -func NewStorageWithOptions( - fs billy.Filesystem, - ops Options, -) (*Storage, error) { +// NewStorageWithOptions returns a new Storage with extra options, +// backed by a given `fs.Filesystem` and cache. +func NewStorageWithOptions(fs billy.Filesystem, cache cache.Object, ops Options) *Storage { dirOps := dotgit.Options{ ExclusiveAccess: ops.ExclusiveAccess, KeepDescriptors: ops.KeepDescriptors, } - dir := dotgit.NewWithOptions(fs, dirOps) - o, err := NewObjectStorageWithOptions(dir, ops) - if err != nil { - return nil, err - } return &Storage{ fs: fs, dir: dir, - ObjectStorage: o, + ObjectStorage: ObjectStorage{ + options: ops, + deltaBaseCache: cache, + dir: dir, + }, ReferenceStorage: ReferenceStorage{dir: dir}, IndexStorage: IndexStorage{dir: dir}, ShallowStorage: ShallowStorage{dir: dir}, ConfigStorage: ConfigStorage{dir: dir}, ModuleStorage: ModuleStorage{dir: dir}, - }, nil + } } // Filesystem returns the underlying filesystem @@ -71,6 +69,7 @@ func (s *Storage) Filesystem() billy.Filesystem { return s.fs } +// Init initializes .git directory func (s *Storage) Init() error { return s.dir.Initialize() } diff --git a/storage/filesystem/storage_test.go b/storage/filesystem/storage_test.go index 7f85ef548..6fa0d908a 100644 --- a/storage/filesystem/storage_test.go +++ b/storage/filesystem/storage_test.go @@ -4,6 +4,7 @@ import ( "io/ioutil" "testing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/storage/test" @@ -23,8 +24,7 @@ var _ = Suite(&StorageSuite{}) func (s *StorageSuite) SetUpTest(c *C) { s.dir = c.MkDir() - storage, err := NewStorage(osfs.New(s.dir)) - c.Assert(err, IsNil) + storage := NewStorage(osfs.New(s.dir), cache.NewObjectLRUDefault()) setUpTest(s, c, storage) } @@ -44,8 +44,7 @@ func setUpTest(s *StorageSuite, c *C, storage *Storage) { func (s *StorageSuite) TestFilesystem(c *C) { fs := memfs.New() - storage, err := NewStorage(fs) - c.Assert(err, IsNil) + storage := NewStorage(fs, cache.NewObjectLRUDefault()) c.Assert(storage.Filesystem(), Equals, fs) } @@ -64,10 +63,10 @@ var _ = Suite(&StorageExclusiveSuite{}) func (s *StorageExclusiveSuite) SetUpTest(c *C) { s.dir = c.MkDir() - storage, err := NewStorageWithOptions( + storage := NewStorageWithOptions( osfs.New(s.dir), + cache.NewObjectLRUDefault(), Options{ExclusiveAccess: true}) - c.Assert(err, IsNil) setUpTest(&s.StorageSuite, c, storage) } diff --git a/worktree_commit_test.go b/worktree_commit_test.go index 6979bd559..62aae8a2f 100644 --- a/worktree_commit_test.go +++ b/worktree_commit_test.go @@ -9,6 +9,7 @@ import ( "time" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/object" "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/storage/filesystem" @@ -205,8 +206,7 @@ func (s *WorktreeSuite) TestCommitTreeSort(c *C) { path, err := ioutil.TempDir(os.TempDir(), "test-commit-tree-sort") c.Assert(err, IsNil) fs := osfs.New(path) - st, err := filesystem.NewStorage(fs) - c.Assert(err, IsNil) + st := filesystem.NewStorage(fs, cache.NewObjectLRUDefault()) r, err := Init(st, nil) c.Assert(err, IsNil) From 6b3f46b3da8924c058abd7159b8c1212b7c78d07 Mon Sep 17 00:00:00 2001 From: Chris Marchesi Date: Fri, 7 Sep 2018 07:32:12 -0700 Subject: [PATCH 101/120] git: s/fetch/returns/ on Tag function doc This is to avoid any ambiguity with the act of "fetching" in git in general. Signed-off-by: Chris Marchesi --- repository.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository.go b/repository.go index 68cc5cc28..67bc96a89 100644 --- a/repository.go +++ b/repository.go @@ -581,7 +581,7 @@ func (r *Repository) buildTagSignature(tag *object.Tag, signKey *openpgp.Entity) return b.String(), nil } -// Tag fetches a tag from the repository. +// Tag returns a tag from the repository. // // If you want to check to see if the tag is an annotated tag, you can call // TagObject on the hash of the reference in ForEach: From 1000bc0ef82a87049c6f01cebacd7aa9d06824c6 Mon Sep 17 00:00:00 2001 From: Chris Marchesi Date: Fri, 7 Sep 2018 08:55:05 -0700 Subject: [PATCH 102/120] git: Add Tag objects to the list of supported objects for walking This is necessary to support pruning on Tag objects. Signed-off-by: Chris Marchesi --- object_walker.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/object_walker.go b/object_walker.go index 4cbbcca65..f8b19cdb0 100644 --- a/object_walker.go +++ b/object_walker.go @@ -94,6 +94,8 @@ func (p *objectWalker) walkObjectTree(hash plumbing.Hash) error { return err } } + case *object.Tag: + return p.walkObjectTree(obj.Target) default: // Error out on unhandled object types. return fmt.Errorf("Unknown object %X %s %T\n", obj.ID(), obj.Type(), obj) From b19b3b84745ea2e2af13a859ff7943c6de20fe4e Mon Sep 17 00:00:00 2001 From: Chris Marchesi Date: Fri, 7 Sep 2018 08:55:43 -0700 Subject: [PATCH 103/120] git: Don't touch tag objects orphaned by tag deletion Deleting a tag ref for an annotated tag in normal git behavior does not delete the tag object right away. This is handled by the normal GC process. Signed-off-by: Chris Marchesi --- repository.go | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/repository.go b/repository.go index 67bc96a89..096ec1321 100644 --- a/repository.go +++ b/repository.go @@ -617,26 +617,12 @@ func (r *Repository) Tag(name string) (*plumbing.Reference, error) { // DeleteTag deletes a tag from the repository. func (r *Repository) DeleteTag(name string) error { - ref, err := r.Tag(name) + _, err := r.Tag(name) if err != nil { return err } - obj, err := r.TagObject(ref.Hash()) - if err != nil && err != plumbing.ErrObjectNotFound { - return err - } - - if err = r.Storer.RemoveReference(plumbing.ReferenceName(path.Join("refs", "tags", name))); err != nil { - return err - } - - // Delete the tag object if this was an annotated tag. - if obj != nil { - return r.DeleteObject(obj.Hash) - } - - return nil + return r.Storer.RemoveReference(plumbing.ReferenceName(path.Join("refs", "tags", name))) } func (r *Repository) resolveToCommitHash(h plumbing.Hash) (plumbing.Hash, error) { From 9cc654ccb69324c7ab9cd67d8a92f9641b3f77b4 Mon Sep 17 00:00:00 2001 From: Chris Marchesi Date: Fri, 7 Sep 2018 08:57:11 -0700 Subject: [PATCH 104/120] git: Add some tests for annotated tag deletion Added a couple of tests for annotated tag deletion: * The first one is a general test and should work regardless of the fixture used - the tag object could possibly be packed, so we do a prune *and* a repack operation before testing to see if the object was GCed correctly. * The second one actually creates the tag to be deleted, so that the tag object gets created as a loose, unpacked object. This is so we can effectively test that purning unpacked objects is now working 100% correctly (this was failing before because tag objects were not supported for walking). Signed-off-by: Chris Marchesi --- repository_test.go | 113 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 110 insertions(+), 3 deletions(-) diff --git a/repository_test.go b/repository_test.go index 0415cc41a..88ad7154e 100644 --- a/repository_test.go +++ b/repository_test.go @@ -1560,6 +1560,113 @@ func (s *RepositorySuite) TestDeleteTagMissingTag(c *C) { c.Assert(err, Equals, ErrTagNotFound) } +func (s *RepositorySuite) TestDeleteTagAnnotated(c *C) { + url := s.GetLocalRepositoryURL( + fixtures.ByURL("https://github.com/git-fixtures/tags.git").One(), + ) + + dir, err := ioutil.TempDir("", "go-git-test-deletetag-annotated") + c.Assert(err, IsNil) + + defer os.RemoveAll(dir) // clean up + + fss, err := filesystem.NewStorage(osfs.New(dir)) + c.Assert(err, IsNil) + + r, _ := Init(fss, nil) + err = r.clone(context.Background(), &CloneOptions{URL: url}) + c.Assert(err, IsNil) + + ref, err := r.Tag("annotated-tag") + c.Assert(ref, NotNil) + c.Assert(err, IsNil) + + obj, err := r.TagObject(ref.Hash()) + c.Assert(obj, NotNil) + c.Assert(err, IsNil) + + err = r.DeleteTag("annotated-tag") + c.Assert(err, IsNil) + + _, err = r.Tag("annotated-tag") + c.Assert(err, Equals, ErrTagNotFound) + + // Run a prune (and repack, to ensure that we are GCing everything regardless + // of the fixture in use) and try to get the tag object again. + // + // The repo needs to be re-opened after the repack. + err = r.Prune(PruneOptions{Handler: r.DeleteObject}) + c.Assert(err, IsNil) + + err = r.RepackObjects(&RepackConfig{}) + c.Assert(err, IsNil) + + r, err = PlainOpen(dir) + c.Assert(r, NotNil) + c.Assert(err, IsNil) + + // Now check to see if the GC was effective in removing the tag object. + obj, err = r.TagObject(ref.Hash()) + c.Assert(obj, IsNil) + c.Assert(err, Equals, plumbing.ErrObjectNotFound) +} + +func (s *RepositorySuite) TestDeleteTagAnnotatedUnpacked(c *C) { + url := s.GetLocalRepositoryURL( + fixtures.ByURL("https://github.com/git-fixtures/tags.git").One(), + ) + + dir, err := ioutil.TempDir("", "go-git-test-deletetag-annotated-unpacked") + c.Assert(err, IsNil) + + defer os.RemoveAll(dir) // clean up + + fss, err := filesystem.NewStorage(osfs.New(dir)) + c.Assert(err, IsNil) + + r, _ := Init(fss, nil) + err = r.clone(context.Background(), &CloneOptions{URL: url}) + c.Assert(err, IsNil) + + // Create a tag for the deletion test. This ensures that the ultimate loose + // object will be unpacked (as we aren't doing anything that should pack it), + // so that we can effectively test that a prune deletes it, without having to + // resort to a repack. + h, err := r.Head() + c.Assert(err, IsNil) + + expectedHash := h.Hash() + + ref, err := r.CreateTag("foobar", expectedHash, &TagObjectOptions{ + Tagger: defaultSignature(), + Message: "foo bar baz qux", + }) + c.Assert(err, IsNil) + + tag, err := r.Tag("foobar") + c.Assert(err, IsNil) + + obj, err := r.TagObject(tag.Hash()) + c.Assert(obj, NotNil) + c.Assert(err, IsNil) + + err = r.DeleteTag("foobar") + c.Assert(err, IsNil) + + _, err = r.Tag("foobar") + c.Assert(err, Equals, ErrTagNotFound) + + // As mentioned, only run a prune. We are not testing for packed objects + // here. + err = r.Prune(PruneOptions{Handler: r.DeleteObject}) + c.Assert(err, IsNil) + + // Now check to see if the GC was effective in removing the tag object. + obj, err = r.TagObject(ref.Hash()) + c.Assert(obj, IsNil) + c.Assert(err, Equals, plumbing.ErrObjectNotFound) +} + func (s *RepositorySuite) TestBranches(c *C) { f := fixtures.ByURL("https://github.com/git-fixtures/root-references.git").One() sto, err := filesystem.NewStorage(f.DotGit()) @@ -1833,9 +1940,9 @@ func (s *RepositorySuite) TestResolveRevisionWithErrors(c *C) { c.Assert(err, IsNil) datas := map[string]string{ - "efs/heads/master~": "reference not found", - "HEAD^3": `Revision invalid : "3" found must be 0, 1 or 2 after "^"`, - "HEAD^{/whatever}": `No commit message match regexp : "whatever"`, + "efs/heads/master~": "reference not found", + "HEAD^3": `Revision invalid : "3" found must be 0, 1 or 2 after "^"`, + "HEAD^{/whatever}": `No commit message match regexp : "whatever"`, "4e1243bd22c66e76c2ba9eddc1f91394e57f9f83": "reference not found", "918c48b83bd081e863dbe1b80f8998f058cd8294": `refname "918c48b83bd081e863dbe1b80f8998f058cd8294" is ambiguous`, } From f8adfff71d844df7efa1367b7958e8f26411aaf9 Mon Sep 17 00:00:00 2001 From: Chris Marchesi Date: Fri, 7 Sep 2018 10:11:44 -0700 Subject: [PATCH 105/120] git: s/TagObjectOptions/CreateTagOptions/ Just renaming the TagObjectOptions type to CreateTagOptions so that it's consistent with the other option types. Signed-off-by: Chris Marchesi --- options.go | 6 +++--- repository.go | 4 ++-- repository_test.go | 16 ++++++++-------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/options.go b/options.go index 856bd5e24..b5727703c 100644 --- a/options.go +++ b/options.go @@ -385,8 +385,8 @@ var ( ErrMissingMessage = errors.New("message field is required") ) -// TagObjectOptions describes how a tag object should be created. -type TagObjectOptions struct { +// CreateTagOptions describes how a tag object should be created. +type CreateTagOptions struct { // Tagger defines the signature of the tag creator. Tagger *object.Signature // Message defines the annotation of the tag. It is canonicalized during @@ -399,7 +399,7 @@ type TagObjectOptions struct { } // Validate validates the fields and sets the default values. -func (o *TagObjectOptions) Validate(r *Repository, hash plumbing.Hash) error { +func (o *CreateTagOptions) Validate(r *Repository, hash plumbing.Hash) error { if o.Tagger == nil { return ErrMissingTagger } diff --git a/repository.go b/repository.go index 096ec1321..a132c39f8 100644 --- a/repository.go +++ b/repository.go @@ -494,7 +494,7 @@ func (r *Repository) DeleteBranch(name string) error { // CreateTag creates a tag. If opts is included, the tag is an annotated tag, // otherwise a lightweight tag is created. -func (r *Repository) CreateTag(name string, hash plumbing.Hash, opts *TagObjectOptions) (*plumbing.Reference, error) { +func (r *Repository) CreateTag(name string, hash plumbing.Hash, opts *CreateTagOptions) (*plumbing.Reference, error) { rname := plumbing.ReferenceName(path.Join("refs", "tags", name)) _, err := r.Storer.Reference(rname) @@ -527,7 +527,7 @@ func (r *Repository) CreateTag(name string, hash plumbing.Hash, opts *TagObjectO return ref, nil } -func (r *Repository) createTagObject(name string, hash plumbing.Hash, opts *TagObjectOptions) (plumbing.Hash, error) { +func (r *Repository) createTagObject(name string, hash plumbing.Hash, opts *CreateTagOptions) (plumbing.Hash, error) { if err := opts.Validate(r, hash); err != nil { return plumbing.ZeroHash, err } diff --git a/repository_test.go b/repository_test.go index 88ad7154e..89911c0da 100644 --- a/repository_test.go +++ b/repository_test.go @@ -1331,7 +1331,7 @@ func (s *RepositorySuite) TestCreateTagAnnotated(c *C) { expectedHash := h.Hash() - ref, err := r.CreateTag("foobar", expectedHash, &TagObjectOptions{ + ref, err := r.CreateTag("foobar", expectedHash, &CreateTagOptions{ Tagger: defaultSignature(), Message: "foo bar baz qux", }) @@ -1363,13 +1363,13 @@ func (s *RepositorySuite) TestCreateTagAnnotatedBadOpts(c *C) { expectedHash := h.Hash() - ref, err := r.CreateTag("foobar", expectedHash, &TagObjectOptions{ + ref, err := r.CreateTag("foobar", expectedHash, &CreateTagOptions{ Message: "foo bar baz qux", }) c.Assert(ref, IsNil) c.Assert(err, Equals, ErrMissingTagger) - ref, err = r.CreateTag("foobar", expectedHash, &TagObjectOptions{ + ref, err = r.CreateTag("foobar", expectedHash, &CreateTagOptions{ Tagger: defaultSignature(), }) c.Assert(ref, IsNil) @@ -1385,7 +1385,7 @@ func (s *RepositorySuite) TestCreateTagAnnotatedBadHash(c *C) { err := r.clone(context.Background(), &CloneOptions{URL: url}) c.Assert(err, IsNil) - ref, err := r.CreateTag("foobar", plumbing.ZeroHash, &TagObjectOptions{ + ref, err := r.CreateTag("foobar", plumbing.ZeroHash, &CreateTagOptions{ Tagger: defaultSignature(), Message: "foo bar baz qux", }) @@ -1406,7 +1406,7 @@ func (s *RepositorySuite) TestCreateTagSigned(c *C) { c.Assert(err, IsNil) key := commitSignKey(c, true) - _, err = r.CreateTag("foobar", h.Hash(), &TagObjectOptions{ + _, err = r.CreateTag("foobar", h.Hash(), &CreateTagOptions{ Tagger: defaultSignature(), Message: "foo bar baz qux", SignKey: key, @@ -1447,7 +1447,7 @@ func (s *RepositorySuite) TestCreateTagSignedBadKey(c *C) { c.Assert(err, IsNil) key := commitSignKey(c, false) - _, err = r.CreateTag("foobar", h.Hash(), &TagObjectOptions{ + _, err = r.CreateTag("foobar", h.Hash(), &CreateTagOptions{ Tagger: defaultSignature(), Message: "foo bar baz qux", SignKey: key, @@ -1468,7 +1468,7 @@ func (s *RepositorySuite) TestCreateTagCanonicalize(c *C) { c.Assert(err, IsNil) key := commitSignKey(c, true) - _, err = r.CreateTag("foobar", h.Hash(), &TagObjectOptions{ + _, err = r.CreateTag("foobar", h.Hash(), &CreateTagOptions{ Tagger: defaultSignature(), Message: "\n\nfoo bar baz qux\n\nsome message here", SignKey: key, @@ -1637,7 +1637,7 @@ func (s *RepositorySuite) TestDeleteTagAnnotatedUnpacked(c *C) { expectedHash := h.Hash() - ref, err := r.CreateTag("foobar", expectedHash, &TagObjectOptions{ + ref, err := r.CreateTag("foobar", expectedHash, &CreateTagOptions{ Tagger: defaultSignature(), Message: "foo bar baz qux", }) From 9ce4eeab99708a2ac35124585103f9e4a04126df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1ximo=20Cuadros?= Date: Mon, 10 Sep 2018 12:36:46 +0200 Subject: [PATCH 106/120] repository: fix test for new Storage constructor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Máximo Cuadros --- repository_test.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/repository_test.go b/repository_test.go index e48f42d3c..2710d9d0d 100644 --- a/repository_test.go +++ b/repository_test.go @@ -1568,8 +1568,7 @@ func (s *RepositorySuite) TestDeleteTagAnnotated(c *C) { defer os.RemoveAll(dir) // clean up - fss, err := filesystem.NewStorage(osfs.New(dir)) - c.Assert(err, IsNil) + fss := filesystem.NewStorage(osfs.New(dir), cache.NewObjectLRUDefault()) r, _ := Init(fss, nil) err = r.clone(context.Background(), &CloneOptions{URL: url}) @@ -1619,8 +1618,7 @@ func (s *RepositorySuite) TestDeleteTagAnnotatedUnpacked(c *C) { defer os.RemoveAll(dir) // clean up - fss, err := filesystem.NewStorage(osfs.New(dir)) - c.Assert(err, IsNil) + fss := filesystem.NewStorage(osfs.New(dir), cache.NewObjectLRUDefault()) r, _ := Init(fss, nil) err = r.clone(context.Background(), &CloneOptions{URL: url}) @@ -1936,9 +1934,9 @@ func (s *RepositorySuite) TestResolveRevisionWithErrors(c *C) { c.Assert(err, IsNil) datas := map[string]string{ - "efs/heads/master~": "reference not found", - "HEAD^3": `Revision invalid : "3" found must be 0, 1 or 2 after "^"`, - "HEAD^{/whatever}": `No commit message match regexp : "whatever"`, + "efs/heads/master~": "reference not found", + "HEAD^3": `Revision invalid : "3" found must be 0, 1 or 2 after "^"`, + "HEAD^{/whatever}": `No commit message match regexp : "whatever"`, "4e1243bd22c66e76c2ba9eddc1f91394e57f9f83": "reference not found", "918c48b83bd081e863dbe1b80f8998f058cd8294": `refname "918c48b83bd081e863dbe1b80f8998f058cd8294" is ambiguous`, } From 83649a1c08e0dcc103de54652ed5b9c33d301326 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1ximo=20Cuadros?= Date: Mon, 10 Sep 2018 13:10:14 +0200 Subject: [PATCH 107/120] *: go modules support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Máximo Cuadros --- go.mod | 29 +++++++++++++++++++++++++++++ go.sum | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+) create mode 100644 go.mod create mode 100644 go.sum diff --git a/go.mod b/go.mod new file mode 100644 index 000000000..e2693508a --- /dev/null +++ b/go.mod @@ -0,0 +1,29 @@ +module gopkg.in/src-d/go-git.v4 + +require ( + github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 // indirect + github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/emirpasic/gods v1.9.0 + github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 // indirect + github.com/gliderlabs/ssh v0.1.1 + github.com/google/go-cmp v0.2.0 + github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 + github.com/jessevdk/go-flags v1.4.0 + github.com/kevinburke/ssh_config v0.0.0-20180830205328-81db2a75821e + github.com/mitchellh/go-homedir v1.0.0 + github.com/pelletier/go-buffruneio v0.2.0 // indirect + github.com/pkg/errors v0.8.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/sergi/go-diff v1.0.0 + github.com/src-d/gcfg v1.3.0 + github.com/stretchr/testify v1.2.2 // indirect + github.com/xanzy/ssh-agent v0.2.0 + golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 + golang.org/x/net v0.0.0-20180906233101-161cd47e91fd // indirect + golang.org/x/text v0.3.0 + gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 + gopkg.in/src-d/go-billy.v4 v4.2.1 + gopkg.in/src-d/go-git-fixtures.v3 v3.1.1 + gopkg.in/warnings.v0 v0.1.2 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 000000000..e262a66da --- /dev/null +++ b/go.sum @@ -0,0 +1,57 @@ +github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs= +github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs= +github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA= +github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emirpasic/gods v1.9.0 h1:rUF4PuzEjMChMiNsVjdI+SyLu7rEqpQ5reNFnhC7oFo= +github.com/emirpasic/gods v1.9.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= +github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ= +github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= +github.com/gliderlabs/ssh v0.1.1 h1:j3L6gSLQalDETeEg/Jg0mGY0/y/N6zI2xX1978P0Uqw= +github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= +github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/kevinburke/ssh_config v0.0.0-20180830205328-81db2a75821e h1:RgQk53JHp/Cjunrr1WlsXSZpqXn+uREuHvUVcK82CV8= +github.com/kevinburke/ssh_config v0.0.0-20180830205328-81db2a75821e/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/mitchellh/go-homedir v1.0.0 h1:vKb8ShqSby24Yrqr/yDYkuFz8d0WUjys40rvnGC8aR0= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/pelletier/go-buffruneio v0.2.0 h1:U4t4R6YkofJ5xHm3dJzuRpPZ0mr5MMCoAWooScCR7aA= +github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo= +github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/src-d/gcfg v1.3.0 h1:2BEDr8r0I0b8h/fOqwtxCEiq2HJu8n2JGZJQFGXWLjg= +github.com/src-d/gcfg v1.3.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI= +github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/xanzy/ssh-agent v0.2.0 h1:Adglfbi5p9Z0BmK2oKU9nTG+zKfniSfnaMYB+ULd+Ro= +github.com/xanzy/ssh-agent v0.2.0/go.mod h1:0NyE30eGUDliuLEHJgYte/zncp2zdTStcOnWhgSqHD8= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 h1:u+LnwYTOOW7Ukr/fppxEb1Nwz0AtPflrblfvUudpo+I= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/sys v0.0.0-20180903190138-2b024373dcd9 h1:lkiLiLBHGoH3XnqSLUIaBsilGMUjI+Uy2Xu2JLUtTas= +golang.org/x/sys v0.0.0-20180903190138-2b024373dcd9/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/src-d/go-billy.v4 v4.2.1 h1:omN5CrMrMcQ+4I8bJ0wEhOBPanIRWzFC953IiXKdYzo= +gopkg.in/src-d/go-billy.v4 v4.2.1/go.mod h1:tm33zBoOwxjYHZIE+OV8bxTWFMJLrconzFMd38aARFk= +gopkg.in/src-d/go-git-fixtures.v3 v3.1.1 h1:XWW/s5W18RaJpmo1l0IYGqXKuJITWRFuA45iOf1dKJs= +gopkg.in/src-d/go-git-fixtures.v3 v3.1.1/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= From 2fb32d2a8601213b6db109d3e9028c6b64af1874 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1ximo=20Cuadros?= Date: Mon, 10 Sep 2018 13:13:07 +0200 Subject: [PATCH 108/120] travis: drop go1.9 add go1.11 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6484425e3..c68b5f473 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,8 @@ language: go go: - - 1.9.x - "1.10" + - "1.11" go_import_path: gopkg.in/src-d/go-git.v4 From 4896974b4daf86f53d782c868d408f830f84c294 Mon Sep 17 00:00:00 2001 From: kuba-- Date: Mon, 17 Sep 2018 22:20:50 +0200 Subject: [PATCH 109/120] Fix potential LRU cache size issue. Signed-off-by: kuba-- --- plumbing/cache/buffer_lru.go | 24 +++++++++++++----------- plumbing/cache/buffer_test.go | 23 +++++++++++++++++++++++ plumbing/cache/object_lru.go | 24 +++++++++++++----------- plumbing/cache/object_test.go | 19 +++++++++++++++++++ 4 files changed, 68 insertions(+), 22 deletions(-) diff --git a/plumbing/cache/buffer_lru.go b/plumbing/cache/buffer_lru.go index f2c0f907f..e86ccb25f 100644 --- a/plumbing/cache/buffer_lru.go +++ b/plumbing/cache/buffer_lru.go @@ -45,19 +45,25 @@ func (c *BufferLRU) Put(key int64, slice []byte) { c.ll = list.New() } + bufSize := FileSize(len(slice)) if ee, ok := c.cache[key]; ok { + oldBuf := ee.Value.(buffer) + // in this case bufSize is a delta: new size - old size + bufSize -= FileSize(len(oldBuf.Slice)) + c.ll.MoveToFront(ee) ee.Value = buffer{key, slice} - return - } + } else { + if bufSize > c.MaxSize { + return + } - objSize := FileSize(len(slice)) - - if objSize > c.MaxSize { - return + ee := c.ll.PushFront(buffer{key, slice}) + c.cache[key] = ee } - for c.actualSize+objSize > c.MaxSize { + c.actualSize += bufSize + for c.actualSize > c.MaxSize { last := c.ll.Back() lastObj := last.Value.(buffer) lastSize := FileSize(len(lastObj.Slice)) @@ -66,10 +72,6 @@ func (c *BufferLRU) Put(key int64, slice []byte) { delete(c.cache, lastObj.Key) c.actualSize -= lastSize } - - ee := c.ll.PushFront(buffer{key, slice}) - c.cache[key] = ee - c.actualSize += objSize } // Get returns a buffer by its key. It marks the buffer as used. If the buffer diff --git a/plumbing/cache/buffer_test.go b/plumbing/cache/buffer_test.go index 262138a62..3e3adc25e 100644 --- a/plumbing/cache/buffer_test.go +++ b/plumbing/cache/buffer_test.go @@ -1,6 +1,7 @@ package cache import ( + "bytes" "sync" . "gopkg.in/check.v1" @@ -38,6 +39,28 @@ func (s *BufferSuite) TestPutSameBuffer(c *C) { } } +func (s *ObjectSuite) TestPutSameBufferWithDifferentSize(c *C) { + aBuffer := []byte("a") + bBuffer := []byte("bbb") + cBuffer := []byte("ccccc") + dBuffer := []byte("ddddddd") + + cache := NewBufferLRU(7 * Byte) + cache.Put(1, aBuffer) + cache.Put(1, bBuffer) + cache.Put(1, cBuffer) + cache.Put(1, dBuffer) + + c.Assert(cache.MaxSize, Equals, 7*Byte) + c.Assert(cache.actualSize, Equals, 7*Byte) + c.Assert(cache.ll.Len(), Equals, 1) + + buf, ok := cache.Get(1) + c.Assert(bytes.Equal(buf, dBuffer), Equals, true) + c.Assert(FileSize(len(buf)), Equals, 7*Byte) + c.Assert(ok, Equals, true) +} + func (s *BufferSuite) TestPutBigBuffer(c *C) { for _, o := range s.c { o.Put(1, s.bBuffer) diff --git a/plumbing/cache/object_lru.go b/plumbing/cache/object_lru.go index 049453950..31c020253 100644 --- a/plumbing/cache/object_lru.go +++ b/plumbing/cache/object_lru.go @@ -42,20 +42,26 @@ func (c *ObjectLRU) Put(obj plumbing.EncodedObject) { c.ll = list.New() } + objSize := FileSize(obj.Size()) key := obj.Hash() if ee, ok := c.cache[key]; ok { + oldObj := ee.Value.(plumbing.EncodedObject) + // in this case objSize is a delta: new size - old size + objSize -= FileSize(oldObj.Size()) + c.ll.MoveToFront(ee) ee.Value = obj - return - } - - objSize := FileSize(obj.Size()) + } else { + if objSize > c.MaxSize { + return + } - if objSize > c.MaxSize { - return + ee := c.ll.PushFront(obj) + c.cache[key] = ee } - for c.actualSize+objSize > c.MaxSize { + c.actualSize += objSize + for c.actualSize > c.MaxSize { last := c.ll.Back() lastObj := last.Value.(plumbing.EncodedObject) lastSize := FileSize(lastObj.Size()) @@ -64,10 +70,6 @@ func (c *ObjectLRU) Put(obj plumbing.EncodedObject) { delete(c.cache, lastObj.Hash()) c.actualSize -= lastSize } - - ee := c.ll.PushFront(obj) - c.cache[key] = ee - c.actualSize += objSize } // Get returns an object by its hash. It marks the object as used. If the object diff --git a/plumbing/cache/object_test.go b/plumbing/cache/object_test.go index ac3f0a3f6..b3e5f7972 100644 --- a/plumbing/cache/object_test.go +++ b/plumbing/cache/object_test.go @@ -45,6 +45,25 @@ func (s *ObjectSuite) TestPutSameObject(c *C) { } } +func (s *ObjectSuite) TestPutSameObjectWithDifferentSize(c *C) { + const hash = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + + cache := NewObjectLRU(7 * Byte) + cache.Put(newObject(hash, 1*Byte)) + cache.Put(newObject(hash, 3*Byte)) + cache.Put(newObject(hash, 5*Byte)) + cache.Put(newObject(hash, 7*Byte)) + + c.Assert(cache.MaxSize, Equals, 7*Byte) + c.Assert(cache.actualSize, Equals, 7*Byte) + c.Assert(cache.ll.Len(), Equals, 1) + + obj, ok := cache.Get(plumbing.NewHash(hash)) + c.Assert(obj.Hash(), Equals, plumbing.NewHash(hash)) + c.Assert(FileSize(obj.Size()), Equals, 7*Byte) + c.Assert(ok, Equals, true) +} + func (s *ObjectSuite) TestPutBigObject(c *C) { for _, o := range s.c { o.Put(s.bObject) From edfc16e3ea6b0ce2533bacb5f370d042042b4784 Mon Sep 17 00:00:00 2001 From: kuba-- Date: Mon, 17 Sep 2018 23:26:45 +0200 Subject: [PATCH 110/120] Remove empty space to trigger windows build. Signed-off-by: kuba-- --- plumbing/cache/buffer_lru.go | 2 -- plumbing/cache/object_lru.go | 2 -- 2 files changed, 4 deletions(-) diff --git a/plumbing/cache/buffer_lru.go b/plumbing/cache/buffer_lru.go index e86ccb25f..acaf19520 100644 --- a/plumbing/cache/buffer_lru.go +++ b/plumbing/cache/buffer_lru.go @@ -50,14 +50,12 @@ func (c *BufferLRU) Put(key int64, slice []byte) { oldBuf := ee.Value.(buffer) // in this case bufSize is a delta: new size - old size bufSize -= FileSize(len(oldBuf.Slice)) - c.ll.MoveToFront(ee) ee.Value = buffer{key, slice} } else { if bufSize > c.MaxSize { return } - ee := c.ll.PushFront(buffer{key, slice}) c.cache[key] = ee } diff --git a/plumbing/cache/object_lru.go b/plumbing/cache/object_lru.go index 31c020253..53d8b02d9 100644 --- a/plumbing/cache/object_lru.go +++ b/plumbing/cache/object_lru.go @@ -48,14 +48,12 @@ func (c *ObjectLRU) Put(obj plumbing.EncodedObject) { oldObj := ee.Value.(plumbing.EncodedObject) // in this case objSize is a delta: new size - old size objSize -= FileSize(oldObj.Size()) - c.ll.MoveToFront(ee) ee.Value = obj } else { if objSize > c.MaxSize { return } - ee := c.ll.PushFront(obj) c.cache[key] = ee } From 82c7a306db75d083db50dbd41aebebd8bd55081b Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Thu, 20 Sep 2018 17:10:39 +0200 Subject: [PATCH 111/120] storage/filesystem: keep packs open in PackfileIter PackfileIter was not taking into account the option KeepDescriptors and was always closing the file. This caused "file already closed" errors when iterating packfiles in with KeepDescriptors active. Signed-off-by: Javi Fontan --- storage/filesystem/object.go | 33 ++++++++++++++++------- storage/filesystem/object_test.go | 44 ++++++++++++++++++++++++++++--- 2 files changed, 63 insertions(+), 14 deletions(-) diff --git a/storage/filesystem/object.go b/storage/filesystem/object.go index 9eb085fd3..4aedacca9 100644 --- a/storage/filesystem/object.go +++ b/storage/filesystem/object.go @@ -396,7 +396,10 @@ func (s *ObjectStorage) IterEncodedObjects(t plumbing.ObjectType) (storer.Encode return storer.NewMultiEncodedObjectIter(iters), nil } -func (s *ObjectStorage) buildPackfileIters(t plumbing.ObjectType, seen map[plumbing.Hash]struct{}) (storer.EncodedObjectIter, error) { +func (s *ObjectStorage) buildPackfileIters( + t plumbing.ObjectType, + seen map[plumbing.Hash]struct{}, +) (storer.EncodedObjectIter, error) { if err := s.requireIndex(); err != nil { return nil, err } @@ -412,7 +415,10 @@ func (s *ObjectStorage) buildPackfileIters(t plumbing.ObjectType, seen map[plumb if err != nil { return nil, err } - return newPackfileIter(s.dir.Fs(), pack, t, seen, s.index[h], s.deltaBaseCache) + return newPackfileIter( + s.dir.Fs(), pack, t, seen, s.index[h], + s.deltaBaseCache, s.options.KeepDescriptors, + ) }, }, nil } @@ -470,9 +476,10 @@ func (it *lazyPackfilesIter) Close() { } type packfileIter struct { - pack billy.File - iter storer.EncodedObjectIter - seen map[plumbing.Hash]struct{} + pack billy.File + iter storer.EncodedObjectIter + seen map[plumbing.Hash]struct{} + keepPack bool } // NewPackfileIter returns a new EncodedObjectIter for the provided packfile @@ -483,6 +490,7 @@ func NewPackfileIter( f billy.File, idxFile billy.File, t plumbing.ObjectType, + keepPack bool, ) (storer.EncodedObjectIter, error) { idx := idxfile.NewMemoryIndex() if err := idxfile.NewDecoder(idxFile).Decode(idx); err != nil { @@ -493,7 +501,8 @@ func NewPackfileIter( return nil, err } - return newPackfileIter(fs, f, t, make(map[plumbing.Hash]struct{}), idx, nil) + seen := make(map[plumbing.Hash]struct{}) + return newPackfileIter(fs, f, t, seen, idx, nil, keepPack) } func newPackfileIter( @@ -503,6 +512,7 @@ func newPackfileIter( seen map[plumbing.Hash]struct{}, index idxfile.Index, cache cache.Object, + keepPack bool, ) (storer.EncodedObjectIter, error) { var p *packfile.Packfile if cache != nil { @@ -517,9 +527,10 @@ func newPackfileIter( } return &packfileIter{ - pack: f, - iter: iter, - seen: seen, + pack: f, + iter: iter, + seen: seen, + keepPack: keepPack, }, nil } @@ -557,7 +568,9 @@ func (iter *packfileIter) ForEach(cb func(plumbing.EncodedObject) error) error { func (iter *packfileIter) Close() { iter.iter.Close() - _ = iter.pack.Close() + if !iter.keepPack { + _ = iter.pack.Close() + } } type objectsIter struct { diff --git a/storage/filesystem/object_test.go b/storage/filesystem/object_test.go index bd4a94b40..407abf29c 100644 --- a/storage/filesystem/object_test.go +++ b/storage/filesystem/object_test.go @@ -153,18 +153,54 @@ func (s *FsSuite) TestPackfileIter(c *C) { idxf, err := dg.ObjectPackIdx(h) c.Assert(err, IsNil) - iter, err := NewPackfileIter(fs, f, idxf, t) + iter, err := NewPackfileIter(fs, f, idxf, t, false) c.Assert(err, IsNil) + err = iter.ForEach(func(o plumbing.EncodedObject) error { c.Assert(o.Type(), Equals, t) return nil }) - c.Assert(err, IsNil) } } }) +} + +func (s *FsSuite) TestPackfileIterKeepDescriptors(c *C) { + fixtures.ByTag(".git").Test(c, func(f *fixtures.Fixture) { + fs := f.DotGit() + ops := dotgit.Options{KeepDescriptors: true} + dg := dotgit.NewWithOptions(fs, ops) + + for _, t := range objectTypes { + ph, err := dg.ObjectPacks() + c.Assert(err, IsNil) + + for _, h := range ph { + f, err := dg.ObjectPack(h) + c.Assert(err, IsNil) + + idxf, err := dg.ObjectPackIdx(h) + c.Assert(err, IsNil) + iter, err := NewPackfileIter(fs, f, idxf, t, true) + c.Assert(err, IsNil) + + err = iter.ForEach(func(o plumbing.EncodedObject) error { + c.Assert(o.Type(), Equals, t) + return nil + }) + c.Assert(err, IsNil) + + // test twice to check that packfiles are not closed + err = iter.ForEach(func(o plumbing.EncodedObject) error { + c.Assert(o.Type(), Equals, t) + return nil + }) + c.Assert(err, IsNil) + } + } + }) } func BenchmarkPackfileIter(b *testing.B) { @@ -201,7 +237,7 @@ func BenchmarkPackfileIter(b *testing.B) { b.Fatal(err) } - iter, err := NewPackfileIter(fs, f, idxf, t) + iter, err := NewPackfileIter(fs, f, idxf, t, false) if err != nil { b.Fatal(err) } @@ -257,7 +293,7 @@ func BenchmarkPackfileIterReadContent(b *testing.B) { b.Fatal(err) } - iter, err := NewPackfileIter(fs, f, idxf, t) + iter, err := NewPackfileIter(fs, f, idxf, t, false) if err != nil { b.Fatal(err) } From a7b0102b83aa86fd299623d35666bfee93fee0c6 Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Fri, 21 Sep 2018 10:43:43 +0200 Subject: [PATCH 112/120] storage/filesystem: add more doc to NewPackfileIter Signed-off-by: Javi Fontan --- storage/filesystem/object.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/storage/filesystem/object.go b/storage/filesystem/object.go index 4aedacca9..68bd140fb 100644 --- a/storage/filesystem/object.go +++ b/storage/filesystem/object.go @@ -476,15 +476,18 @@ func (it *lazyPackfilesIter) Close() { } type packfileIter struct { - pack billy.File - iter storer.EncodedObjectIter - seen map[plumbing.Hash]struct{} + pack billy.File + iter storer.EncodedObjectIter + seen map[plumbing.Hash]struct{} + + // tells whether the pack file should be left open after iteration or not keepPack bool } // NewPackfileIter returns a new EncodedObjectIter for the provided packfile // and object type. Packfile and index file will be closed after they're -// used. +// used. If keepPack is true the packfile won't be closed after the iteration +// finished. func NewPackfileIter( fs billy.Filesystem, f billy.File, From 156d632a533263091491e9b4a3d9770245fa4af9 Mon Sep 17 00:00:00 2001 From: Jongmin Kim Date: Wed, 26 Sep 2018 23:59:23 +0900 Subject: [PATCH 113/120] all: remove extra 's' in "mismatch" Signed-off-by: Jongmin Kim --- config/refspec.go | 2 +- plumbing/format/index/decoder.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/refspec.go b/config/refspec.go index c9b9d524f..391705ccc 100644 --- a/config/refspec.go +++ b/config/refspec.go @@ -15,7 +15,7 @@ const ( var ( ErrRefSpecMalformedSeparator = errors.New("malformed refspec, separators are wrong") - ErrRefSpecMalformedWildcard = errors.New("malformed refspec, missmatched number of wildcards") + ErrRefSpecMalformedWildcard = errors.New("malformed refspec, mismatched number of wildcards") ) // RefSpec is a mapping from local branches to remote references diff --git a/plumbing/format/index/decoder.go b/plumbing/format/index/decoder.go index 1a58128f4..df25530c6 100644 --- a/plumbing/format/index/decoder.go +++ b/plumbing/format/index/decoder.go @@ -21,7 +21,7 @@ var ( // ErrMalformedSignature is returned by Decode when the index header file is // malformed ErrMalformedSignature = errors.New("malformed index signature file") - // ErrInvalidChecksum is returned by Decode if the SHA1 hash missmatch with + // ErrInvalidChecksum is returned by Decode if the SHA1 hash mismatch with // the read content ErrInvalidChecksum = errors.New("invalid checksum") From 37f80c63cc19d8f224d7c5eb0338594b9cef3838 Mon Sep 17 00:00:00 2001 From: "Santiago M. Mola" Date: Thu, 27 Sep 2018 09:27:48 +0200 Subject: [PATCH 114/120] test: improve test for urlencoded user:pass Signed-off-by: Santiago M. Mola --- plumbing/transport/common_test.go | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/plumbing/transport/common_test.go b/plumbing/transport/common_test.go index 17f62a64e..65ed5b9f6 100644 --- a/plumbing/transport/common_test.go +++ b/plumbing/transport/common_test.go @@ -1,6 +1,7 @@ package transport import ( + "fmt" "net/url" "testing" @@ -155,12 +156,21 @@ func (s *SuiteCommon) TestNewEndpointFileURL(c *C) { } func (s *SuiteCommon) TestValidEndpoint(c *C) { - e, err := NewEndpoint("http://github.com/user/repository.git") - e.User = "person@mail.com" - e.Password = " !\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~" - url, err := url.Parse(e.String()) + user := "person@mail.com" + pass := " !\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~" + e, err := NewEndpoint(fmt.Sprintf( + "http://%s:%s@github.com/user/repository.git", + url.PathEscape(user), + url.PathEscape(pass), + )) c.Assert(err, IsNil) - c.Assert(url, NotNil) + c.Assert(e, NotNil) + c.Assert(e.User, Equals, user) + c.Assert(e.Password, Equals, pass) + c.Assert(e.Host, Equals, "github.com") + c.Assert(e.Path, Equals, "/user/repository.git") + + c.Assert(e.String(), Equals, "http://person@mail.com:%20%21%22%23$%25&%27%28%29%2A+%2C-.%2F:%3B%3C=%3E%3F@%5B%5C%5D%5E_%60%7B%7C%7D~@github.com/user/repository.git") } func (s *SuiteCommon) TestNewEndpointInvalidURL(c *C) { From 41af4294c1347c2f35e255d4ac01ecd25b9d5d55 Mon Sep 17 00:00:00 2001 From: u5surf Date: Tue, 2 Oct 2018 20:04:47 +0900 Subject: [PATCH 115/120] use time.IsZero in Prune Signed-off-by: u5surf --- prune.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prune.go b/prune.go index 04913d656..c840325f1 100644 --- a/prune.go +++ b/prune.go @@ -49,7 +49,7 @@ func (r *Repository) Prune(opt PruneOptions) error { } // Otherwise it is a candidate for pruning. // Check out for too new objects next. - if opt.OnlyObjectsOlderThan != (time.Time{}) { + if !opt.OnlyObjectsOlderThan.IsZero() { // Errors here are non-fatal. The object may be e.g. packed. // Or concurrently deleted. Skip such objects. t, err := los.LooseObjectTime(hash) From c2ba07bbd7dea849049184a888d67b016a20b6ae Mon Sep 17 00:00:00 2001 From: Filip Navara Date: Fri, 5 Oct 2018 10:15:15 +0200 Subject: [PATCH 116/120] Add test for Windows local paths. Signed-off-by: Filip Navara --- config/config_test.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/config/config_test.go b/config/config_test.go index fe73de872..db0932caa 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -24,6 +24,8 @@ func (s *ConfigSuite) TestUnmarshall(c *C) { url = git@github.com:src-d/go-git.git fetch = +refs/heads/*:refs/remotes/origin/* fetch = +refs/pull/*:refs/remotes/origin/pull/* +[remote "win-local"] + url = X:\\Git\\ [submodule "qux"] path = qux url = https://github.com/foo/qux.git @@ -41,13 +43,15 @@ func (s *ConfigSuite) TestUnmarshall(c *C) { c.Assert(cfg.Core.Worktree, Equals, "foo") c.Assert(cfg.Core.CommentChar, Equals, "bar") c.Assert(cfg.Pack.Window, Equals, uint(20)) - c.Assert(cfg.Remotes, HasLen, 2) + c.Assert(cfg.Remotes, HasLen, 3) c.Assert(cfg.Remotes["origin"].Name, Equals, "origin") c.Assert(cfg.Remotes["origin"].URLs, DeepEquals, []string{"git@github.com:mcuadros/go-git.git"}) c.Assert(cfg.Remotes["origin"].Fetch, DeepEquals, []RefSpec{"+refs/heads/*:refs/remotes/origin/*"}) c.Assert(cfg.Remotes["alt"].Name, Equals, "alt") c.Assert(cfg.Remotes["alt"].URLs, DeepEquals, []string{"git@github.com:mcuadros/go-git.git", "git@github.com:src-d/go-git.git"}) c.Assert(cfg.Remotes["alt"].Fetch, DeepEquals, []RefSpec{"+refs/heads/*:refs/remotes/origin/*", "+refs/pull/*:refs/remotes/origin/pull/*"}) + c.Assert(cfg.Remotes["win-local"].Name, Equals, "win-local") + c.Assert(cfg.Remotes["win-local"].URLs, DeepEquals, []string{"X:\\Git\\"}) c.Assert(cfg.Submodules, HasLen, 1) c.Assert(cfg.Submodules["qux"].Name, Equals, "qux") c.Assert(cfg.Submodules["qux"].URL, Equals, "https://github.com/foo/qux.git") @@ -69,6 +73,8 @@ func (s *ConfigSuite) TestMarshall(c *C) { fetch = +refs/pull/*:refs/remotes/origin/pull/* [remote "origin"] url = git@github.com:mcuadros/go-git.git +[remote "win-local"] + url = "X:\\Git\\" [submodule "qux"] url = https://github.com/foo/qux.git [branch "master"] @@ -91,6 +97,11 @@ func (s *ConfigSuite) TestMarshall(c *C) { Fetch: []RefSpec{"+refs/heads/*:refs/remotes/origin/*", "+refs/pull/*:refs/remotes/origin/pull/*"}, } + cfg.Remotes["win-local"] = &RemoteConfig{ + Name: "win-local", + URLs: []string{"X:\\Git\\"}, + } + cfg.Submodules["qux"] = &Submodule{ Name: "qux", URL: "https://github.com/foo/qux.git", @@ -119,6 +130,8 @@ func (s *ConfigSuite) TestUnmarshallMarshall(c *C) { url = git@github.com:mcuadros/go-git.git fetch = +refs/heads/*:refs/remotes/origin/* mirror = true +[remote "win-local"] + url = "X:\\Git\\" [branch "master"] remote = origin merge = refs/heads/master From da56abd1530ce85640479a6b6cf292009891a0f5 Mon Sep 17 00:00:00 2001 From: David Url Date: Sat, 6 Oct 2018 13:00:11 +0200 Subject: [PATCH 117/120] git: Fix Status.IsClean() documentation The documentation of the IsClean Method contained a negation, so it was describing the opposite of its actual behavior. Fixes #838 Signed-off-by: David Url --- status.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/status.go b/status.go index ecbf79350..7f18e0227 100644 --- a/status.go +++ b/status.go @@ -26,7 +26,7 @@ func (s Status) IsUntracked(path string) bool { return ok && stat.Worktree == Untracked } -// IsClean returns true if all the files aren't in Unmodified status. +// IsClean returns true if all the files are in Unmodified status. func (s Status) IsClean() bool { for _, status := range s { if status.Worktree != Unmodified || status.Staging != Unmodified { From 0bfe038a16551ede1d22bfb54f52c31b646a9e1a Mon Sep 17 00:00:00 2001 From: Nithin Gangadharan Date: Thu, 11 Oct 2018 14:17:08 +0530 Subject: [PATCH 118/120] Plumbing: object, Add support for Log with filenames. Fixes #826 (#979) plumbing: object, Add support for Log with filenames. Fixes #826 --- options.go | 4 + plumbing/object/commit_walker_file.go | 115 +++++++++++++++++++++ repository.go | 19 ++-- repository_test.go | 139 ++++++++++++++++++++++++++ 4 files changed, 271 insertions(+), 6 deletions(-) create mode 100644 plumbing/object/commit_walker_file.go diff --git a/options.go b/options.go index b5727703c..b8bc1e979 100644 --- a/options.go +++ b/options.go @@ -330,6 +330,10 @@ type LogOptions struct { // set Order=LogOrderCommitterTime for ordering by committer time (more compatible with `git log`) // set Order=LogOrderBSF for Breadth-first search Order LogOrder + + // Show only those commits in which the specified file was inserted/updated. + // It is equivalent to running `git log -- `. + FileName *string } var ( diff --git a/plumbing/object/commit_walker_file.go b/plumbing/object/commit_walker_file.go new file mode 100644 index 000000000..84e738ac6 --- /dev/null +++ b/plumbing/object/commit_walker_file.go @@ -0,0 +1,115 @@ +package object + +import ( + "gopkg.in/src-d/go-git.v4/plumbing/storer" + "io" +) + +type commitFileIter struct { + fileName string + sourceIter CommitIter + currentCommit *Commit +} + +// NewCommitFileIterFromIter returns a commit iterator which performs diffTree between +// successive trees returned from the commit iterator from the argument. The purpose of this is +// to find the commits that explain how the files that match the path came to be. +func NewCommitFileIterFromIter(fileName string, commitIter CommitIter) CommitIter { + iterator := new(commitFileIter) + iterator.sourceIter = commitIter + iterator.fileName = fileName + return iterator +} + +func (c *commitFileIter) Next() (*Commit, error) { + if c.currentCommit == nil { + var err error + c.currentCommit, err = c.sourceIter.Next() + if err != nil { + return nil, err + } + } + commit, commitErr := c.getNextFileCommit() + + // Setting current-commit to nil to prevent unwanted states when errors are raised + if commitErr != nil { + c.currentCommit = nil + } + return commit, commitErr +} + +func (c *commitFileIter) getNextFileCommit() (*Commit, error) { + for { + // Parent-commit can be nil if the current-commit is the initial commit + parentCommit, parentCommitErr := c.sourceIter.Next() + if parentCommitErr != nil { + // If the parent-commit is beyond the initial commit, keep it nil + if parentCommitErr != io.EOF { + return nil, parentCommitErr + } + parentCommit = nil + } + + // Fetch the trees of the current and parent commits + currentTree, currTreeErr := c.currentCommit.Tree() + if currTreeErr != nil { + return nil, currTreeErr + } + + var parentTree *Tree + if parentCommit != nil { + var parentTreeErr error + parentTree, parentTreeErr = parentCommit.Tree() + if parentTreeErr != nil { + return nil, parentTreeErr + } + } + + // Find diff between current and parent trees + changes, diffErr := DiffTree(currentTree, parentTree) + if diffErr != nil { + return nil, diffErr + } + + foundChangeForFile := false + for _, change := range changes { + if change.name() == c.fileName { + foundChangeForFile = true + break + } + } + + // Storing the current-commit in-case a change is found, and + // Updating the current-commit for the next-iteration + prevCommit := c.currentCommit + c.currentCommit = parentCommit + + if foundChangeForFile == true { + return prevCommit, nil + } + + // If not matches found and if parent-commit is beyond the initial commit, then return with EOF + if parentCommit == nil { + return nil, io.EOF + } + } +} + +func (c *commitFileIter) ForEach(cb func(*Commit) error) error { + for { + commit, nextErr := c.Next() + if nextErr != nil { + return nextErr + } + err := cb(commit) + if err == storer.ErrStop { + return nil + } else if err != nil { + return err + } + } +} + +func (c *commitFileIter) Close() { + c.sourceIter.Close() +} diff --git a/repository.go b/repository.go index be1f0574f..62e22a6b6 100644 --- a/repository.go +++ b/repository.go @@ -965,19 +965,26 @@ func (r *Repository) Log(o *LogOptions) (object.CommitIter, error) { return nil, err } + var commitIter object.CommitIter switch o.Order { case LogOrderDefault: - return object.NewCommitPreorderIter(commit, nil, nil), nil + commitIter = object.NewCommitPreorderIter(commit, nil, nil) case LogOrderDFS: - return object.NewCommitPreorderIter(commit, nil, nil), nil + commitIter = object.NewCommitPreorderIter(commit, nil, nil) case LogOrderDFSPost: - return object.NewCommitPostorderIter(commit, nil), nil + commitIter = object.NewCommitPostorderIter(commit, nil) case LogOrderBSF: - return object.NewCommitIterBSF(commit, nil, nil), nil + commitIter = object.NewCommitIterBSF(commit, nil, nil) case LogOrderCommitterTime: - return object.NewCommitIterCTime(commit, nil, nil), nil + commitIter = object.NewCommitIterCTime(commit, nil, nil) + default: + return nil, fmt.Errorf("invalid Order=%v", o.Order) + } + + if o.FileName == nil { + return commitIter, nil } - return nil, fmt.Errorf("invalid Order=%v", o.Order) + return object.NewCommitFileIterFromIter(*o.FileName, commitIter), nil } // Tags returns all the tag References in a repository. diff --git a/repository_test.go b/repository_test.go index 2710d9d0d..bf02933c7 100644 --- a/repository_test.go +++ b/repository_test.go @@ -1143,6 +1143,145 @@ func (s *RepositorySuite) TestLogError(c *C) { c.Assert(err, NotNil) } +func (s *RepositorySuite) TestLogFileNext(c *C) { + r, _ := Init(memory.NewStorage(), nil) + err := r.clone(context.Background(), &CloneOptions{ + URL: s.GetBasicLocalRepositoryURL(), + }) + + c.Assert(err, IsNil) + + fileName := "vendor/foo.go" + cIter, err := r.Log(&LogOptions{FileName: &fileName}) + + c.Assert(err, IsNil) + + commitOrder := []plumbing.Hash{ + plumbing.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5"), + } + + for _, o := range commitOrder { + commit, err := cIter.Next() + c.Assert(err, IsNil) + c.Assert(commit.Hash, Equals, o) + } + _, err = cIter.Next() + c.Assert(err, Equals, io.EOF) +} + +func (s *RepositorySuite) TestLogFileForEach(c *C) { + r, _ := Init(memory.NewStorage(), nil) + err := r.clone(context.Background(), &CloneOptions{ + URL: s.GetBasicLocalRepositoryURL(), + }) + + c.Assert(err, IsNil) + + fileName := "php/crappy.php" + cIter, err := r.Log(&LogOptions{FileName: &fileName}) + + c.Assert(err, IsNil) + + commitOrder := []plumbing.Hash{ + plumbing.NewHash("918c48b83bd081e863dbe1b80f8998f058cd8294"), + } + + expectedIndex := 0 + cIter.ForEach(func(commit *object.Commit) error { + expectedCommitHash := commitOrder[expectedIndex] + c.Assert(commit.Hash.String(), Equals, expectedCommitHash.String()) + expectedIndex += 1 + return nil + }) + c.Assert(expectedIndex, Equals, 1) +} + +func (s *RepositorySuite) TestLogInvalidFile(c *C) { + r, _ := Init(memory.NewStorage(), nil) + err := r.clone(context.Background(), &CloneOptions{ + URL: s.GetBasicLocalRepositoryURL(), + }) + c.Assert(err, IsNil) + + // Throwing in a file that does not exist + fileName := "vendor/foo12.go" + cIter, err := r.Log(&LogOptions{FileName: &fileName}) + // Not raising an error since `git log -- vendor/foo12.go` responds silently + c.Assert(err, IsNil) + + _, err = cIter.Next() + c.Assert(err, Equals, io.EOF) +} + +func (s *RepositorySuite) TestLogFileInitialCommit(c *C) { + r, _ := Init(memory.NewStorage(), nil) + err := r.clone(context.Background(), &CloneOptions{ + URL: s.GetBasicLocalRepositoryURL(), + }) + c.Assert(err, IsNil) + + fileName := "LICENSE" + cIter, err := r.Log(&LogOptions{ + Order: LogOrderCommitterTime, + FileName: &fileName, + }) + + c.Assert(err, IsNil) + + commitOrder := []plumbing.Hash{ + plumbing.NewHash("b029517f6300c2da0f4b651b8642506cd6aaf45d"), + } + + expectedIndex := 0 + cIter.ForEach(func(commit *object.Commit) error { + expectedCommitHash := commitOrder[expectedIndex] + c.Assert(commit.Hash.String(), Equals, expectedCommitHash.String()) + expectedIndex += 1 + return nil + }) + c.Assert(expectedIndex, Equals, 1) +} + +func (s *RepositorySuite) TestLogFileWithOtherParamsFail(c *C) { + r, _ := Init(memory.NewStorage(), nil) + err := r.clone(context.Background(), &CloneOptions{ + URL: s.GetBasicLocalRepositoryURL(), + }) + c.Assert(err, IsNil) + + fileName := "vendor/foo.go" + cIter, err := r.Log(&LogOptions{ + Order: LogOrderCommitterTime, + FileName: &fileName, + From: plumbing.NewHash("35e85108805c84807bc66a02d91535e1e24b38b9"), + }) + c.Assert(err, IsNil) + _, iterErr := cIter.Next() + c.Assert(iterErr, Equals, io.EOF) +} + +func (s *RepositorySuite) TestLogFileWithOtherParamsPass(c *C) { + r, _ := Init(memory.NewStorage(), nil) + err := r.clone(context.Background(), &CloneOptions{ + URL: s.GetBasicLocalRepositoryURL(), + }) + c.Assert(err, IsNil) + + fileName := "LICENSE" + cIter, err := r.Log(&LogOptions{ + Order: LogOrderCommitterTime, + FileName: &fileName, + From: plumbing.NewHash("35e85108805c84807bc66a02d91535e1e24b38b9"), + }) + c.Assert(err, IsNil) + commitVal, iterErr := cIter.Next() + c.Assert(iterErr, Equals, nil) + c.Assert(commitVal.Hash.String(), Equals, "b029517f6300c2da0f4b651b8642506cd6aaf45d") + + _, iterErr = cIter.Next() + c.Assert(iterErr, Equals, io.EOF) +} + func (s *RepositorySuite) TestCommit(c *C) { r, _ := Init(memory.NewStorage(), nil) err := r.clone(context.Background(), &CloneOptions{ From 529f8438980a29999ca44f3a3e74eb317d0cf3c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1ximo=20Cuadros?= Date: Mon, 15 Oct 2018 11:28:12 +0200 Subject: [PATCH 119/120] use remote name in fetch while clone, test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Máximo Cuadros --- repository.go | 2 +- repository_test.go | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/repository.go b/repository.go index 247966611..2d20f49fc 100644 --- a/repository.go +++ b/repository.go @@ -519,7 +519,7 @@ func (r *Repository) clone(ctx context.Context, o *CloneOptions) error { } ref, err := r.fetchAndUpdateReferences(ctx, &FetchOptions{ - RefSpecs: r.cloneRefSpec(o, c), + RefSpecs: r.cloneRefSpec(o, c), Depth: o.Depth, Auth: o.Auth, Progress: o.Progress, diff --git a/repository_test.go b/repository_test.go index b78fbb70b..9303bef02 100644 --- a/repository_test.go +++ b/repository_test.go @@ -566,6 +566,19 @@ func (s *RepositorySuite) TestPlainClone(c *C) { c.Assert(cfg.Branches["master"].Name, Equals, "master") } +func (s *RepositorySuite) TestPlainCloneWithRemoteName(c *C) { + r, err := PlainClone(c.MkDir(), false, &CloneOptions{ + URL: s.GetBasicLocalRepositoryURL(), + RemoteName: "test", + }) + + c.Assert(err, IsNil) + + remote, err := r.Remote("test") + c.Assert(err, IsNil) + c.Assert(remote, NotNil) +} + func (s *RepositorySuite) TestPlainCloneContext(c *C) { ctx, cancel := context.WithCancel(context.Background()) cancel() From e250584026a05e5b4145349cd6462596dbe0f37a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1ximo=20Cuadros?= Date: Mon, 15 Oct 2018 12:06:14 +0200 Subject: [PATCH 120/120] references: sort: compare author timestamps when commit timestamps are equal, test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Máximo Cuadros --- references_test.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/references_test.go b/references_test.go index cefc7a274..6e755631b 100644 --- a/references_test.go +++ b/references_test.go @@ -163,7 +163,19 @@ var referencesTests = [...]struct { "1e14f94bcf82694fdc7e2dcbbfdbbed58db0f4d9", "1e3d328a2cabda5d0aaddc5dec65271343e0dc37", }}, - + {"https://github.com/spinnaker/spinnaker.git", "f39d86f59a0781f130e8de6b2115329c1fbe9545", "README.adoc", []string{ + "638f61b3331695f46f1a88095e26dea0f09f176b", + "bd42370d3fe8d410e78acb96f81cb3d838ad1c21", + "d6905eab6fec1841c7cf8e4484499f5c8d7d423e", + "c0a70a0f5aa494f0ae01c55ba191f2325556489a", + "811795c8a185e88f5d269195cb68b29c8d0fe170", + "d6e6fe0194447cc280f942d6a2e0521b68ea7796", + "174bdbf9edfb0ca88415dd4a673852d5b22e7036", + "9944d6cf72b8f82d622d85dad7434472bc8f397d", + "e805183c72f0426fb073728c01901c2fd2db1da6", + "8ef83dd443a05e9122681950399edaa58a38d466", + "d73f9cee49a5ad27a42a6e18af7c49a8f28ad8a8", + }}, // FAILS /* // this contains an empty move