From cda466f02298eb9320fdaebf1719b85db44aab39 Mon Sep 17 00:00:00 2001 From: coffeegoddd Date: Tue, 21 Jan 2025 15:32:35 -0800 Subject: [PATCH 1/2] /go/libraries/doltcore/env/actions: make iter resolved tags paginated, sort in lexicographical order --- go/libraries/doltcore/env/actions/tag.go | 107 +++++++++---- go/libraries/doltcore/env/actions/tag_test.go | 150 ++++++++++++++++++ 2 files changed, 223 insertions(+), 34 deletions(-) create mode 100644 go/libraries/doltcore/env/actions/tag_test.go diff --git a/go/libraries/doltcore/env/actions/tag.go b/go/libraries/doltcore/env/actions/tag.go index 1580ad43808..d092348dd83 100644 --- a/go/libraries/doltcore/env/actions/tag.go +++ b/go/libraries/doltcore/env/actions/tag.go @@ -17,7 +17,6 @@ package actions import ( "context" "fmt" - "sort" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" "github.com/dolthub/dolt/go/libraries/doltcore/env" @@ -25,6 +24,8 @@ import ( "github.com/dolthub/dolt/go/store/datas" ) +const DefaultPageSize = 100 + type TagProps struct { TaggerName string TaggerEmail string @@ -97,67 +98,105 @@ func DeleteTagsOnDB(ctx context.Context, ddb *doltdb.DoltDB, tagNames ...string) return nil } -// IterResolvedTags iterates over tags in dEnv.DoltDB from newest to oldest, resolving the tag to a commit and calling cb(). -func IterResolvedTags(ctx context.Context, ddb *doltdb.DoltDB, cb func(tag *doltdb.Tag) (stop bool, err error)) error { +// IterUnresolvedTags iterates over tags in dEnv.DoltDB, and calls cb() for each with an unresolved Tag. +func IterUnresolvedTags(ctx context.Context, ddb *doltdb.DoltDB, cb func(tag *doltdb.TagResolver) (stop bool, err error)) error { tagRefs, err := ddb.GetTags(ctx) - if err != nil { return err } - var resolved []*doltdb.Tag - for _, r := range tagRefs { - tr, ok := r.(ref.TagRef) - if !ok { - return fmt.Errorf("DoltDB.GetTags() returned non-tag DoltRef") - } + tagResolvers, err := ddb.GetTagResolvers(ctx, tagRefs) + if err != nil { + return err + } - tag, err := ddb.ResolveTag(ctx, tr) + for _, tagResolver := range tagResolvers { + stop, err := cb(&tagResolver) if err != nil { return err } + if stop { + break + } + } + return nil +} + +// IterResolvedTagsPaginated iterates over tags in dEnv.DoltDB in their default lexicographical order, resolving the tag to a commit and calling cb(). +// Returns the next tag name if there are more results available. +func IterResolvedTagsPaginated(ctx context.Context, ddb *doltdb.DoltDB, startTag string, cb func(tag *doltdb.Tag) (stop bool, err error)) (string, error) { + // tags returned here are sorted lexicographically + tagRefs, err := ddb.GetTags(ctx) + if err != nil { + return "", err + } - resolved = append(resolved, tag) + // find starting index based on start tag + startIdx := 0 + if startTag != "" { + for i, tr := range tagRefs { + if tr.GetPath() == startTag { + startIdx = i + 1 // start after the given tag + break + } + } } - // iterate newest to oldest - sort.Slice(resolved, func(i, j int) bool { - return resolved[i].Meta.Timestamp > resolved[j].Meta.Timestamp - }) + // get page of results + endIdx := startIdx + DefaultPageSize + if endIdx > len(tagRefs) { + endIdx = len(tagRefs) + } - for _, tag := range resolved { - stop, err := cb(tag) + pageTagRefs := tagRefs[startIdx:endIdx] + // resolve tags for this page + for _, tr := range pageTagRefs { + tag, err := ddb.ResolveTag(ctx, tr.(ref.TagRef)) if err != nil { - return err + return "", err } + + stop, err := cb(tag) + if err != nil { + return "", err + } + if stop { break } } - return nil -} -// IterUnresolvedTags iterates over tags in dEnv.DoltDB, and calls cb() for each with an unresovled Tag. -func IterUnresolvedTags(ctx context.Context, ddb *doltdb.DoltDB, cb func(tag *doltdb.TagResolver) (stop bool, err error)) error { - tagRefs, err := ddb.GetTags(ctx) - if err != nil { - return err + // return next tag name if there are more results + if endIdx < len(tagRefs) { + lastTag := pageTagRefs[len(pageTagRefs)-1] + return lastTag.GetPath(), nil } - tagResolvers, err := ddb.GetTagResolvers(ctx, tagRefs) + return "", nil +} + +// VisitResolvedTag iterates over tags in ddb until the given tag name is found, then calls cb() with the resolved tag. +func VisitResolvedTag(ctx context.Context, ddb *doltdb.DoltDB, tagName string, cb func(tag *doltdb.Tag) error) error { + tagRefs, err := ddb.GetTags(ctx) if err != nil { return err } - for _, tagResolver := range tagResolvers { - stop, err := cb(&tagResolver) - if err != nil { - return err + for _, r := range tagRefs { + tr, ok := r.(ref.TagRef) + if !ok { + return fmt.Errorf("DoltDB.GetTags() returned non-tag DoltRef") } - if stop { - break + + if tr.GetPath() == tagName { + tag, err := ddb.ResolveTag(ctx, tr) + if err != nil { + return err + } + return cb(tag) } } - return nil + + return doltdb.ErrTagNotFound } diff --git a/go/libraries/doltcore/env/actions/tag_test.go b/go/libraries/doltcore/env/actions/tag_test.go new file mode 100644 index 00000000000..6ad86189abb --- /dev/null +++ b/go/libraries/doltcore/env/actions/tag_test.go @@ -0,0 +1,150 @@ +// Copyright 2025 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package actions + +import ( + "context" + "fmt" + "sort" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" + "github.com/dolthub/dolt/go/libraries/doltcore/env" + "github.com/dolthub/dolt/go/libraries/utils/filesys" + "github.com/dolthub/dolt/go/store/types" +) + +const ( + testHomeDir = "/user/bheni" + workingDir = "/user/bheni/datasets/addresses" + credsDir = "creds" + + configFile = "config.json" + GlobalConfigFile = "config_global.json" +) + +func testHomeDirFunc() (string, error) { + return testHomeDir, nil +} + +func createTestEnv() (*env.DoltEnv, *filesys.InMemFS) { + initialDirs := []string{testHomeDir, workingDir} + initialFiles := map[string][]byte{} + + fs := filesys.NewInMemFS(initialDirs, initialFiles, workingDir) + dEnv := env.Load(context.Background(), testHomeDirFunc, fs, doltdb.InMemDoltDB, "test") + + return dEnv, fs +} + +func TestVisitResolvedTag(t *testing.T) { + dEnv, _ := createTestEnv() + ctx := context.Background() + + // Initialize repo + err := dEnv.InitRepo(ctx, types.Format_Default, "test user", "test@test.com", "main") + require.NoError(t, err) + + // Create a tag + tagName := "test-tag" + tagMsg := "test tag message" + err = CreateTag(ctx, dEnv, tagName, "main", TagProps{TaggerName: "test user", TaggerEmail: "test@test.com", Description: tagMsg}) + require.NoError(t, err) + + // Visit the tag and verify its properties + var foundTag *doltdb.Tag + err = VisitResolvedTag(ctx, dEnv.DoltDB, tagName, func(tag *doltdb.Tag) error { + foundTag = tag + return nil + }) + require.NoError(t, err) + require.NotNil(t, foundTag) + require.Equal(t, tagName, foundTag.Name) + require.Equal(t, tagMsg, foundTag.Meta.Description) + + // Test visiting non-existent tag + err = VisitResolvedTag(ctx, dEnv.DoltDB, "non-existent-tag", func(tag *doltdb.Tag) error { + return nil + }) + require.Equal(t, doltdb.ErrTagNotFound, err) +} + +func TestIterResolvedTagsPaginated(t *testing.T) { + dEnv, _ := createTestEnv() + ctx := context.Background() + + // Initialize repo + err := dEnv.InitRepo(ctx, types.Format_Default, "test user", "test@test.com", "main") + require.NoError(t, err) + + expectedTagNames := make([]string, DefaultPageSize*2) + // Create multiple tags with different timestamps + tagNames := make([]string, DefaultPageSize*2) + for i := range tagNames { + tagName := fmt.Sprintf("tag-%d", i) + err = CreateTag(ctx, dEnv, tagName, "main", TagProps{ + TaggerName: "test user", + TaggerEmail: "test@test.com", + Description: fmt.Sprintf("test tag %s", tagName), + }) + time.Sleep(2 * time.Millisecond) + require.NoError(t, err) + tagNames[i] = tagName + expectedTagNames[i] = tagName + } + + // Sort expected tag names to ensure they are in the correct order + sort.Strings(expectedTagNames) + + // Test first page + var foundTags []string + pageToken, err := IterResolvedTagsPaginated(ctx, dEnv.DoltDB, "", func(tag *doltdb.Tag) (bool, error) { + foundTags = append(foundTags, tag.Name) + return false, nil + }) + require.NoError(t, err) + require.NotEmpty(t, pageToken) // Should have next page + require.Equal(t, DefaultPageSize, len(foundTags)) // Default page size tags returned + require.Equal(t, expectedTagNames[:DefaultPageSize], foundTags) + + // Test second page + var secondPageTags []string + nextPageToken, err := IterResolvedTagsPaginated(ctx, dEnv.DoltDB, pageToken, func(tag *doltdb.Tag) (bool, error) { + secondPageTags = append(secondPageTags, tag.Name) + return false, nil + }) + + require.NoError(t, err) + require.Empty(t, nextPageToken) // Should be no more pages + require.Equal(t, DefaultPageSize, len(secondPageTags)) // Remaining tags + require.Equal(t, expectedTagNames[DefaultPageSize:], secondPageTags) + + // Verify all tags were found + allFoundTags := append(foundTags, secondPageTags...) + require.Equal(t, len(tagNames), len(allFoundTags)) + require.Equal(t, expectedTagNames, allFoundTags) + + // Test early termination + var earlyTermTags []string + _, err = IterResolvedTagsPaginated(ctx, dEnv.DoltDB, "", func(tag *doltdb.Tag) (bool, error) { + earlyTermTags = append(earlyTermTags, tag.Name) + return true, nil // Stop after first tag + }) + require.NoError(t, err) + require.Equal(t, 1, len(earlyTermTags)) +} From 09dd814fd8a575d8b932d12c678a46aa74877e76 Mon Sep 17 00:00:00 2001 From: coffeegoddd Date: Thu, 30 Jan 2025 09:29:28 -0800 Subject: [PATCH 2/2] /go/libraries/doltcore/env/actions/tag.go: reinstage iterresolved tags for better deprecation process --- go/libraries/doltcore/env/actions/tag.go | 42 ++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/go/libraries/doltcore/env/actions/tag.go b/go/libraries/doltcore/env/actions/tag.go index d092348dd83..a6bb57e8c5e 100644 --- a/go/libraries/doltcore/env/actions/tag.go +++ b/go/libraries/doltcore/env/actions/tag.go @@ -17,6 +17,7 @@ package actions import ( "context" "fmt" + "sort" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" "github.com/dolthub/dolt/go/libraries/doltcore/env" @@ -122,6 +123,47 @@ func IterUnresolvedTags(ctx context.Context, ddb *doltdb.DoltDB, cb func(tag *do return nil } +// IterResolvedTags iterates over tags in dEnv.DoltDB from newest to oldest, resolving the tag to a commit and calling cb(). +func IterResolvedTags(ctx context.Context, ddb *doltdb.DoltDB, cb func(tag *doltdb.Tag) (stop bool, err error)) error { + tagRefs, err := ddb.GetTags(ctx) + + if err != nil { + return err + } + + var resolved []*doltdb.Tag + for _, r := range tagRefs { + tr, ok := r.(ref.TagRef) + if !ok { + return fmt.Errorf("DoltDB.GetTags() returned non-tag DoltRef") + } + + tag, err := ddb.ResolveTag(ctx, tr) + if err != nil { + return err + } + + resolved = append(resolved, tag) + } + + // iterate newest to oldest + sort.Slice(resolved, func(i, j int) bool { + return resolved[i].Meta.Timestamp > resolved[j].Meta.Timestamp + }) + + for _, tag := range resolved { + stop, err := cb(tag) + + if err != nil { + return err + } + if stop { + break + } + } + return nil +} + // IterResolvedTagsPaginated iterates over tags in dEnv.DoltDB in their default lexicographical order, resolving the tag to a commit and calling cb(). // Returns the next tag name if there are more results available. func IterResolvedTagsPaginated(ctx context.Context, ddb *doltdb.DoltDB, startTag string, cb func(tag *doltdb.Tag) (stop bool, err error)) (string, error) {