Skip to content

Commit

Permalink
/go/libraries/doltcore/env/actions: make iter resolved tags paginated…
Browse files Browse the repository at this point in the history
…, sort in lexicographical order
  • Loading branch information
coffeegoddd committed Jan 21, 2025
1 parent 97f8fed commit 03793b7
Show file tree
Hide file tree
Showing 2 changed files with 223 additions and 34 deletions.
107 changes: 73 additions & 34 deletions go/libraries/doltcore/env/actions/tag.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,15 @@ package actions
import (
"context"
"fmt"
"sort"

"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
"github.com/dolthub/dolt/go/libraries/doltcore/env"
"github.com/dolthub/dolt/go/libraries/doltcore/ref"
"github.com/dolthub/dolt/go/store/datas"
)

const DefaultPageSize = 100

type TagProps struct {
TaggerName string
TaggerEmail string
Expand Down Expand Up @@ -97,67 +98,105 @@ func DeleteTagsOnDB(ctx context.Context, ddb *doltdb.DoltDB, tagNames ...string)
return nil
}

// IterResolvedTags iterates over tags in dEnv.DoltDB from newest to oldest, resolving the tag to a commit and calling cb().
func IterResolvedTags(ctx context.Context, ddb *doltdb.DoltDB, cb func(tag *doltdb.Tag) (stop bool, err error)) error {
// IterUnresolvedTags iterates over tags in dEnv.DoltDB, and calls cb() for each with an unresolved Tag.
func IterUnresolvedTags(ctx context.Context, ddb *doltdb.DoltDB, cb func(tag *doltdb.TagResolver) (stop bool, err error)) error {
tagRefs, err := ddb.GetTags(ctx)

if err != nil {
return err
}

var resolved []*doltdb.Tag
for _, r := range tagRefs {
tr, ok := r.(ref.TagRef)
if !ok {
return fmt.Errorf("DoltDB.GetTags() returned non-tag DoltRef")
}
tagResolvers, err := ddb.GetTagResolvers(ctx, tagRefs)
if err != nil {
return err
}

tag, err := ddb.ResolveTag(ctx, tr)
for _, tagResolver := range tagResolvers {
stop, err := cb(&tagResolver)
if err != nil {
return err
}
if stop {
break
}
}
return nil
}

// IterResolvedTagsPaginated iterates over tags in dEnv.DoltDB in their default lexicographical order, resolving the tag to a commit and calling cb().
// Returns the next tag name if there are more results available.
func IterResolvedTagsPaginated(ctx context.Context, ddb *doltdb.DoltDB, startTag string, cb func(tag *doltdb.Tag) (stop bool, err error)) (string, error) {
// tags returned here are sorted lexicographically
tagRefs, err := ddb.GetTags(ctx)
if err != nil {
return "", err
}

resolved = append(resolved, tag)
// find starting index based on start tag
startIdx := 0
if startTag != "" {
for i, tr := range tagRefs {
if tr.GetPath() == startTag {
startIdx = i + 1 // start after the given tag
break
}
}
}

// iterate newest to oldest
sort.Slice(resolved, func(i, j int) bool {
return resolved[i].Meta.Timestamp > resolved[j].Meta.Timestamp
})
// get page of results
endIdx := startIdx + DefaultPageSize
if endIdx > len(tagRefs) {
endIdx = len(tagRefs)
}

for _, tag := range resolved {
stop, err := cb(tag)
pageTagRefs := tagRefs[startIdx:endIdx]

// resolve tags for this page
for _, tr := range pageTagRefs {
tag, err := ddb.ResolveTag(ctx, tr.(ref.TagRef))
if err != nil {
return err
return "", err
}

stop, err := cb(tag)
if err != nil {
return "", err
}

if stop {
break
}
}
return nil
}

// IterUnresolvedTags iterates over tags in dEnv.DoltDB, and calls cb() for each with an unresovled Tag.
func IterUnresolvedTags(ctx context.Context, ddb *doltdb.DoltDB, cb func(tag *doltdb.TagResolver) (stop bool, err error)) error {
tagRefs, err := ddb.GetTags(ctx)
if err != nil {
return err
// return next tag name if there are more results
if endIdx < len(tagRefs) {
lastTag := pageTagRefs[len(pageTagRefs)-1]
return lastTag.GetPath(), nil
}

tagResolvers, err := ddb.GetTagResolvers(ctx, tagRefs)
return "", nil
}

// VisitResolvedTag iterates over tags in ddb until the given tag name is found, then calls cb() with the resolved tag.
func VisitResolvedTag(ctx context.Context, ddb *doltdb.DoltDB, tagName string, cb func(tag *doltdb.Tag) error) error {
tagRefs, err := ddb.GetTags(ctx)
if err != nil {
return err
}

for _, tagResolver := range tagResolvers {
stop, err := cb(&tagResolver)
if err != nil {
return err
for _, r := range tagRefs {
tr, ok := r.(ref.TagRef)
if !ok {
return fmt.Errorf("DoltDB.GetTags() returned non-tag DoltRef")
}
if stop {
break

if tr.GetPath() == tagName {
tag, err := ddb.ResolveTag(ctx, tr)
if err != nil {
return err
}
return cb(tag)
}
}
return nil

return doltdb.ErrTagNotFound
}
150 changes: 150 additions & 0 deletions go/libraries/doltcore/env/actions/tag_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,150 @@
// Copyright 2025 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package actions

import (
"context"
"fmt"
"sort"
"testing"
"time"

"github.com/stretchr/testify/require"

"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
"github.com/dolthub/dolt/go/libraries/doltcore/env"
"github.com/dolthub/dolt/go/libraries/utils/filesys"
"github.com/dolthub/dolt/go/store/types"
)

const (
testHomeDir = "/user/bheni"
workingDir = "/user/bheni/datasets/addresses"
credsDir = "creds"

configFile = "config.json"
GlobalConfigFile = "config_global.json"
)

func testHomeDirFunc() (string, error) {
return testHomeDir, nil
}

func createTestEnv() (*env.DoltEnv, *filesys.InMemFS) {
initialDirs := []string{testHomeDir, workingDir}
initialFiles := map[string][]byte{}

fs := filesys.NewInMemFS(initialDirs, initialFiles, workingDir)
dEnv := env.Load(context.Background(), testHomeDirFunc, fs, doltdb.InMemDoltDB, "test")

return dEnv, fs
}

func TestVisitResolvedTag(t *testing.T) {
dEnv, _ := createTestEnv()
ctx := context.Background()

// Initialize repo
err := dEnv.InitRepo(ctx, types.Format_Default, "test user", "[email protected]", "main")
require.NoError(t, err)

// Create a tag
tagName := "test-tag"
tagMsg := "test tag message"
err = CreateTag(ctx, dEnv, tagName, "main", TagProps{TaggerName: "test user", TaggerEmail: "[email protected]", Description: tagMsg})
require.NoError(t, err)

// Visit the tag and verify its properties
var foundTag *doltdb.Tag
err = VisitResolvedTag(ctx, dEnv.DoltDB, tagName, func(tag *doltdb.Tag) error {
foundTag = tag
return nil
})
require.NoError(t, err)
require.NotNil(t, foundTag)
require.Equal(t, tagName, foundTag.Name)
require.Equal(t, tagMsg, foundTag.Meta.Description)

// Test visiting non-existent tag
err = VisitResolvedTag(ctx, dEnv.DoltDB, "non-existent-tag", func(tag *doltdb.Tag) error {
return nil
})
require.Equal(t, doltdb.ErrTagNotFound, err)
}

func TestIterResolvedTagsPaginated(t *testing.T) {
dEnv, _ := createTestEnv()
ctx := context.Background()

// Initialize repo
err := dEnv.InitRepo(ctx, types.Format_Default, "test user", "[email protected]", "main")
require.NoError(t, err)

expectedTagNames := make([]string, DefaultPageSize*2)
// Create multiple tags with different timestamps
tagNames := make([]string, DefaultPageSize*2)
for i := range tagNames {
tagName := fmt.Sprintf("tag-%d", i)
err = CreateTag(ctx, dEnv, tagName, "main", TagProps{
TaggerName: "test user",
TaggerEmail: "[email protected]",
Description: fmt.Sprintf("test tag %s", tagName),
})
time.Sleep(2 * time.Millisecond)
require.NoError(t, err)
tagNames[i] = tagName
expectedTagNames[i] = tagName
}

// Sort expected tag names to ensure they are in the correct order
sort.Strings(expectedTagNames)

// Test first page
var foundTags []string
pageToken, err := IterResolvedTagsPaginated(ctx, dEnv.DoltDB, "", func(tag *doltdb.Tag) (bool, error) {
foundTags = append(foundTags, tag.Name)
return false, nil
})
require.NoError(t, err)
require.NotEmpty(t, pageToken) // Should have next page
require.Equal(t, DefaultPageSize, len(foundTags)) // Default page size tags returned
require.Equal(t, expectedTagNames[:DefaultPageSize], foundTags)

// Test second page
var secondPageTags []string
nextPageToken, err := IterResolvedTagsPaginated(ctx, dEnv.DoltDB, pageToken, func(tag *doltdb.Tag) (bool, error) {
secondPageTags = append(secondPageTags, tag.Name)
return false, nil
})

require.NoError(t, err)
require.Empty(t, nextPageToken) // Should be no more pages
require.Equal(t, DefaultPageSize, len(secondPageTags)) // Remaining tags
require.Equal(t, expectedTagNames[DefaultPageSize:], secondPageTags)

// Verify all tags were found
allFoundTags := append(foundTags, secondPageTags...)
require.Equal(t, len(tagNames), len(allFoundTags))
require.Equal(t, expectedTagNames, allFoundTags)

// Test early termination
var earlyTermTags []string
_, err = IterResolvedTagsPaginated(ctx, dEnv.DoltDB, "", func(tag *doltdb.Tag) (bool, error) {
earlyTermTags = append(earlyTermTags, tag.Name)
return true, nil // Stop after first tag
})
require.NoError(t, err)
require.Equal(t, 1, len(earlyTermTags))
}

0 comments on commit 03793b7

Please sign in to comment.