Skip to content

Commit

Permalink
Merge pull request #8783 from dolthub/nicktobey/lazy-load
Browse files Browse the repository at this point in the history
Avoid loading DB for commands where it's not necessary.
  • Loading branch information
nicktobey authored Feb 3, 2025
2 parents d1e90c4 + 2911085 commit b4b6d71
Show file tree
Hide file tree
Showing 112 changed files with 846 additions and 687 deletions.
4 changes: 2 additions & 2 deletions go/cmd/dolt/cli/command.go
Original file line number Diff line number Diff line change
Expand Up @@ -236,8 +236,8 @@ func (hc SubCommandHandler) handleCommand(ctx context.Context, commandStr string
}

fgc, ok := cmd.(FormatGatedCommand)
if ok && dEnv.DoltDB != nil && fgc.GatedForNBF(dEnv.DoltDB.Format()) {
vs := dEnv.DoltDB.Format().VersionString()
if ok && dEnv.DoltDB(ctx) != nil && fgc.GatedForNBF(dEnv.DoltDB(ctx).Format()) {
vs := dEnv.DoltDB(ctx).Format().VersionString()
err := fmt.Sprintf("Dolt command '%s' is not supported in format %s", cmd.Name(), vs)
PrintErrln(color.YellowString(err))
return 1
Expand Down
2 changes: 1 addition & 1 deletion go/cmd/dolt/commands/admin/setref.go
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ func (cmd SetRefCmd) Exec(ctx context.Context, commandStr string, args []string,

h := hash.Parse(apr.MustGetValue("to"))

err := dEnv.DoltDB.SetHead(ctx, r, h)
err := dEnv.DoltDB(ctx).SetHead(ctx, r, h)
if err != nil {
verr := errhand.BuildDError("error setting %s to %s", r.String(), h.String()).AddCause(err).Build()
commands.HandleVErrAndExitCode(verr, usage)
Expand Down
2 changes: 1 addition & 1 deletion go/cmd/dolt/commands/admin/showroot.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ func (cmd ShowRootCmd) Exec(ctx context.Context, commandStr string, args []strin

cli.ParseArgsOrDie(ap, args, usage)

db := doltdb.HackDatasDatabaseFromDoltDB(dEnv.DoltDB)
db := doltdb.HackDatasDatabaseFromDoltDB(dEnv.DoltDB(ctx))
dss, err := db.Datasets(ctx)
if err != nil {
verr := errhand.BuildDError("failed to get database datasets").AddCause(err).Build()
Expand Down
4 changes: 2 additions & 2 deletions go/cmd/dolt/commands/archive.go
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ func (cmd ArchiveCmd) Exec(ctx context.Context, commandStr string, args []string
help, _ := cli.HelpAndUsagePrinters(cli.CommandDocsForCommandString(commandStr, docs, ap))
apr := cli.ParseArgsOrDie(ap, args, help)

db := doltdb.HackDatasDatabaseFromDoltDB(dEnv.DoltDB)
db := doltdb.HackDatasDatabaseFromDoltDB(dEnv.DoltDB(ctx))
cs := datas.ChunkStoreFromDatabase(db)
if _, ok := cs.(*nbs.GenerationalNBS); !ok {
cli.PrintErrln("archive command requires a GenerationalNBS")
Expand Down Expand Up @@ -130,7 +130,7 @@ func (cmd ArchiveCmd) Exec(ctx context.Context, commandStr string, args []string

groupings := nbs.NewChunkRelations()
if apr.Contains(groupChunksFlag) {
err = historicalFuzzyMatching(ctx, hs, &groupings, dEnv.DoltDB)
err = historicalFuzzyMatching(ctx, hs, &groupings, dEnv.DoltDB(ctx))
if err != nil {
cli.PrintErrln(err)
return 1
Expand Down
8 changes: 4 additions & 4 deletions go/cmd/dolt/commands/backup.go
Original file line number Diff line number Diff line change
Expand Up @@ -275,15 +275,15 @@ func backup(ctx context.Context, dEnv *env.DoltEnv, b env.Remote) errhand.Verbos
return errhand.BuildDError("error: archive files present. Please revert them with the --revert flag before running this command.").Build()
}

destDb, err := b.GetRemoteDB(ctx, dEnv.DoltDB.ValueReadWriter().Format(), dEnv)
destDb, err := b.GetRemoteDB(ctx, dEnv.DoltDB(ctx).ValueReadWriter().Format(), dEnv)
if err != nil {
return errhand.BuildDError("error: unable to open destination.").AddCause(err).Build()
}
tmpDir, err := dEnv.TempTableFilesDir()
if err != nil {
return errhand.BuildDError("error: ").AddCause(err).Build()
}
err = actions.SyncRoots(ctx, dEnv.DoltDB, destDb, tmpDir, buildProgStarter(defaultLanguage), stopProgFuncs)
err = actions.SyncRoots(ctx, dEnv.DoltDB(ctx), destDb, tmpDir, buildProgStarter(defaultLanguage), stopProgFuncs)

switch err {
case nil:
Expand Down Expand Up @@ -361,7 +361,7 @@ func restoreBackup(ctx context.Context, dEnv *env.DoltEnv, apr *argparser.ArgPar
return errhand.VerboseErrorFromError(err)
}

err = actions.SyncRoots(ctx, srcDb, existingDEnv.DoltDB, tmpDir, buildProgStarter(downloadLanguage), stopProgFuncs)
err = actions.SyncRoots(ctx, srcDb, existingDEnv.DoltDB(ctx), tmpDir, buildProgStarter(downloadLanguage), stopProgFuncs)
if err != nil {
return errhand.VerboseErrorFromError(err)
}
Expand All @@ -384,7 +384,7 @@ func restoreBackup(ctx context.Context, dEnv *env.DoltEnv, apr *argparser.ArgPar
if err != nil {
return errhand.VerboseErrorFromError(err)
}
err = actions.SyncRoots(ctx, srcDb, clonedEnv.DoltDB, tmpDir, buildProgStarter(downloadLanguage), stopProgFuncs)
err = actions.SyncRoots(ctx, srcDb, clonedEnv.DoltDB(ctx), tmpDir, buildProgStarter(downloadLanguage), stopProgFuncs)
if err != nil {
// If we're cloning into a directory that already exists do not erase it. Otherwise
// make best effort to delete the directory we created.
Expand Down
9 changes: 5 additions & 4 deletions go/cmd/dolt/commands/branch.go
Original file line number Diff line number Diff line change
Expand Up @@ -110,9 +110,10 @@ func (cmd BranchCmd) Exec(ctx context.Context, commandStr string, args []string,
return status
}

errorBuilder := errhand.BuildDError("error: failed to create query engine")
queryEngine, sqlCtx, closeFunc, err := cliCtx.QueryEngine(ctx)
if err != nil {
return HandleVErrAndExitCode(errhand.BuildDError("error: failed to create query engine").AddCause(err).Build(), nil)
return HandleVErrAndExitCode(errorBuilder.AddCause(err).Build(), nil)
}

if closeFunc != nil {
Expand Down Expand Up @@ -259,7 +260,7 @@ func printCurrentBranch(sqlCtx *sql.Context, queryEngine cli.Queryist) int {
}

func printAllDatasets(ctx context.Context, dEnv *env.DoltEnv) int {
refs, err := dEnv.DoltDB.GetHeadRefs(ctx)
refs, err := dEnv.DoltDB(ctx).GetHeadRefs(ctx)
if err != nil {
return HandleVErrAndExitCode(errhand.VerboseErrorFromError(err), nil)
}
Expand All @@ -270,7 +271,7 @@ func printAllDatasets(ctx context.Context, dEnv *env.DoltEnv) int {
cli.Println(" " + r.String())
}

branches, err := dEnv.DoltDB.GetBranches(ctx)
branches, err := dEnv.DoltDB(ctx).GetBranches(ctx)
if err != nil {
return HandleVErrAndExitCode(errhand.VerboseErrorFromError(err), nil)
}
Expand All @@ -284,7 +285,7 @@ func printAllDatasets(ctx context.Context, dEnv *env.DoltEnv) int {
return HandleVErrAndExitCode(errhand.VerboseErrorFromError(err), nil)
}

_, err = dEnv.DoltDB.ResolveWorkingSet(ctx, w)
_, err = dEnv.DoltDB(ctx).ResolveWorkingSet(ctx, w)
if errors.Is(err, doltdb.ErrWorkingSetNotFound) {
continue
} else if err != nil {
Expand Down
6 changes: 3 additions & 3 deletions go/cmd/dolt/commands/ci/init.go
Original file line number Diff line number Diff line change
Expand Up @@ -115,9 +115,9 @@ func (cmd InitCmd) Exec(ctx context.Context, commandStr string, args []string, d
}

func newDatabase(ctx context.Context, name string, dEnv *env.DoltEnv, useBulkEditor bool) (sqle.Database, error) {
deaf := dEnv.DbEaFactory()
deaf := dEnv.DbEaFactory(ctx)
if useBulkEditor {
deaf = dEnv.BulkDbEaFactory()
deaf = dEnv.BulkDbEaFactory(ctx)
}
tmpDir, err := dEnv.TempTableFilesDir()
if err != nil {
Expand All @@ -127,5 +127,5 @@ func newDatabase(ctx context.Context, name string, dEnv *env.DoltEnv, useBulkEdi
Deaf: deaf,
Tempdir: tmpDir,
}
return sqle.NewDatabase(ctx, name, dEnv.DbData(), opts)
return sqle.NewDatabase(ctx, name, dEnv.DbData(ctx), opts)
}
2 changes: 1 addition & 1 deletion go/cmd/dolt/commands/dump.go
Original file line number Diff line number Diff line change
Expand Up @@ -595,7 +595,7 @@ func getTableWriter(ctx context.Context, dEnv *env.DoltEnv, tblOpts *tableOption
if err != nil {
return nil, errhand.BuildDError("error: ").AddCause(err).Build()
}
opts := editor.Options{Deaf: dEnv.DbEaFactory(), Tempdir: tmpDir}
opts := editor.Options{Deaf: dEnv.DbEaFactory(ctx), Tempdir: tmpDir}

writer, err := dEnv.FS.OpenForWriteAppend(filePath, os.ModePerm)
if err != nil {
Expand Down
6 changes: 3 additions & 3 deletions go/cmd/dolt/commands/engine/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,9 @@ func CollectDBs(ctx context.Context, mrEnv *env.MultiRepoEnv, useBulkEditor bool
}

func newDatabase(ctx context.Context, name string, dEnv *env.DoltEnv, useBulkEditor bool) (sqle.Database, error) {
deaf := dEnv.DbEaFactory()
deaf := dEnv.DbEaFactory(ctx)
if useBulkEditor {
deaf = dEnv.BulkDbEaFactory()
deaf = dEnv.BulkDbEaFactory(ctx)
}
tmpDir, err := dEnv.TempTableFilesDir()
if err != nil {
Expand All @@ -63,5 +63,5 @@ func newDatabase(ctx context.Context, name string, dEnv *env.DoltEnv, useBulkEdi
Deaf: deaf,
Tempdir: tmpDir,
}
return sqle.NewDatabase(ctx, name, dEnv.DbData(), opts)
return sqle.NewDatabase(ctx, name, dEnv.DbData(ctx), opts)
}
6 changes: 3 additions & 3 deletions go/cmd/dolt/commands/filter-branch.go
Original file line number Diff line number Diff line change
Expand Up @@ -263,7 +263,7 @@ func getNerf(ctx context.Context, dEnv *env.DoltEnv, apr *argparser.ArgParseResu
return nil, err
}

optCmt, err := dEnv.DoltDB.Resolve(ctx, cs, headRef)
optCmt, err := dEnv.DoltDB(ctx).Resolve(ctx, cs, headRef)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -339,8 +339,8 @@ func rebaseSqlEngine(ctx context.Context, dEnv *env.DoltEnv, root doltdb.RootVal
if err != nil {
return nil, nil, err
}
opts := editor.Options{Deaf: dEnv.DbEaFactory(), Tempdir: tmpDir}
db, err := dsqle.NewDatabase(ctx, filterDbName, dEnv.DbData(), opts)
opts := editor.Options{Deaf: dEnv.DbEaFactory(ctx), Tempdir: tmpDir}
db, err := dsqle.NewDatabase(ctx, filterDbName, dEnv.DbData(ctx), opts)
if err != nil {
return nil, nil, err
}
Expand Down
2 changes: 1 addition & 1 deletion go/cmd/dolt/commands/fsck.go
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ func (cmd FsckCmd) Exec(ctx context.Context, commandStr string, args []string, d
terminate = func() bool {
defer close(progress)
var err error
report, err = dEnv.DoltDB.FSCK(ctx, progress)
report, err = dEnv.DoltDB(ctx).FSCK(ctx, progress)
if err != nil {
cli.PrintErrln(err.Error())
return true
Expand Down
2 changes: 1 addition & 1 deletion go/cmd/dolt/commands/indexcmds/rebuild.go
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ func (cmd RebuildCmd) Exec(ctx context.Context, commandStr string, args []string
if err != nil {
return HandleErr(errhand.BuildDError("error: ").AddCause(err).Build(), nil)
}
opts := editor.Options{Deaf: dEnv.DbEaFactory(), Tempdir: tmpDir}
opts := editor.Options{Deaf: dEnv.DbEaFactory(ctx), Tempdir: tmpDir}
sch, err := table.GetSchema(ctx)
if err != nil {
return HandleErr(errhand.BuildDError("could not get table schema").AddCause(err).Build(), nil)
Expand Down
12 changes: 7 additions & 5 deletions go/cmd/dolt/commands/init_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -67,15 +67,16 @@ func TestInit(t *testing.T) {

for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
ctx := context.Background()
dEnv := createUninitializedEnv()
gCfg, _ := dEnv.Config.GetConfig(env.GlobalConfig)
gCfg.SetStrings(test.GlobalConfig)
apr := argparser.ArgParseResults{}
latebind := func(ctx context.Context) (cli.Queryist, *sql.Context, func(), error) { return nil, nil, func() {}, nil }
cliCtx, _ := cli.NewCliContext(&apr, dEnv.Config, dEnv.FS, latebind)

result := InitCmd{}.Exec(context.Background(), "dolt init", test.Args, dEnv, cliCtx)
defer dEnv.DoltDB.Close()
result := InitCmd{}.Exec(ctx, "dolt init", test.Args, dEnv, cliCtx)
defer dEnv.DoltDB(ctx).Close()

require.Equalf(t, test.ExpectSuccess, result == 0, "- Expected success: %t; result: %t;", test.ExpectSuccess, result == 0)

Expand All @@ -92,12 +93,13 @@ func TestInit(t *testing.T) {
}

func TestInitTwice(t *testing.T) {
ctx := context.Background()
dEnv := createUninitializedEnv()
result := InitCmd{}.Exec(context.Background(), "dolt init", []string{"-name", "Bill Billerson", "-email", "[email protected]"}, dEnv, nil)
result := InitCmd{}.Exec(ctx, "dolt init", []string{"-name", "Bill Billerson", "-email", "[email protected]"}, dEnv, nil)
require.True(t, result == 0, "First init should succeed")
defer dEnv.DoltDB.Close()
defer dEnv.DoltDB(ctx).Close()

result = InitCmd{}.Exec(context.Background(), "dolt init", []string{"-name", "Bill Billerson", "-email", "[email protected]"}, dEnv, nil)
result = InitCmd{}.Exec(ctx, "dolt init", []string{"-name", "Bill Billerson", "-email", "[email protected]"}, dEnv, nil)
require.True(t, result != 0, "Second init should fail")
}

Expand Down
14 changes: 8 additions & 6 deletions go/cmd/dolt/commands/log_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,16 +32,17 @@ import (
)

func TestLog(t *testing.T) {
ctx := context.Background()
dEnv := createUninitializedEnv()
err := dEnv.InitRepo(context.Background(), types.Format_Default, "Bill Billerson", "[email protected]", env.DefaultInitBranch)
defer dEnv.DoltDB.Close()
err := dEnv.InitRepo(ctx, types.Format_Default, "Bill Billerson", "[email protected]", env.DefaultInitBranch)
defer dEnv.DoltDB(ctx).Close()

if err != nil {
t.Error("Failed to init repo")
}

cs, _ := doltdb.NewCommitSpec(env.DefaultInitBranch)
opt, _ := dEnv.DoltDB.Resolve(context.Background(), cs, nil)
opt, _ := dEnv.DoltDB(ctx).Resolve(context.Background(), cs, nil)
commit, _ := opt.ToCommit()

meta, _ := commit.GetCommitMeta(context.Background())
Expand All @@ -53,16 +54,17 @@ func TestLogSigterm(t *testing.T) {
t.Skip("Skipping test as function used is not supported on Windows")
}

ctx := context.Background()
dEnv := createUninitializedEnv()
err := dEnv.InitRepo(context.Background(), types.Format_Default, "Bill Billerson", "[email protected]", env.DefaultInitBranch)
defer dEnv.DoltDB.Close()
err := dEnv.InitRepo(ctx, types.Format_Default, "Bill Billerson", "[email protected]", env.DefaultInitBranch)
defer dEnv.DoltDB(ctx).Close()

if err != nil {
t.Error("Failed to init repo")
}

cs, _ := doltdb.NewCommitSpec(env.DefaultInitBranch)
optCmt, _ := dEnv.DoltDB.Resolve(context.Background(), cs, nil)
optCmt, _ := dEnv.DoltDB(ctx).Resolve(context.Background(), cs, nil)
commit, _ := optCmt.ToCommit()
cMeta, _ := commit.GetCommitMeta(context.Background())
cHash, _ := commit.HashOf()
Expand Down
6 changes: 3 additions & 3 deletions go/cmd/dolt/commands/migrate.go
Original file line number Diff line number Diff line change
Expand Up @@ -89,15 +89,15 @@ func (cmd MigrateCmd) Exec(ctx context.Context, commandStr string, args []string
return 0 // unreachable
}

// MigrateDatabase migrates the NomsBinFormat of |dEnv.DoltDB|.
// MigrateDatabase migrates the NomsBinFormat of |dEnv.DoltDB(ctx)|.
func MigrateDatabase(ctx context.Context, dEnv *env.DoltEnv, dropConflicts bool) error {
menv, err := migrate.NewEnvironment(ctx, dEnv)
if err != nil {
return err
}
menv.DropConflicts = dropConflicts

if curr := menv.Existing.DoltDB.Format(); types.IsFormat_DOLT(curr) {
if curr := menv.Existing.DoltDB(ctx).Format(); types.IsFormat_DOLT(curr) {
cli.Println("database is already migrated")
return nil
}
Expand All @@ -108,7 +108,7 @@ func MigrateDatabase(ctx context.Context, dEnv *env.DoltEnv, dropConflicts bool)
}
cli.Println("migrating database at tmp dir: ", p)

err = migrate.TraverseDAG(ctx, menv, menv.Existing.DoltDB, menv.Migration.DoltDB)
err = migrate.TraverseDAG(ctx, menv, menv.Existing.DoltDB(ctx), menv.Migration.DoltDB(ctx))
if err != nil {
return err
}
Expand Down
2 changes: 1 addition & 1 deletion go/cmd/dolt/commands/read_tables.go
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,7 @@ func pullTableValue(ctx context.Context, dEnv *env.DoltEnv, srcDB *doltdb.DoltDB
cli.Println("Retrieving", tblName)
runProgFunc := buildProgStarter(language)
wg, pullerEventCh := runProgFunc(newCtx)
err = dEnv.DoltDB.PullChunks(ctx, tmpDir, srcDB, []hash.Hash{tblHash}, pullerEventCh, nil)
err = dEnv.DoltDB(ctx).PullChunks(ctx, tmpDir, srcDB, []hash.Hash{tblHash}, pullerEventCh, nil)
stopProgFuncs(cancelFunc, wg, pullerEventCh)
if err != nil {
return nil, errhand.BuildDError("Failed reading chunks for remote table '%s' at '%s'", tblName, commitStr).AddCause(err).Build()
Expand Down
2 changes: 1 addition & 1 deletion go/cmd/dolt/commands/schcmds/export.go
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ func exportSchemas(ctx context.Context, apr *argparser.ArgParseResults, root dol
if err != nil {
return errhand.BuildDError("error: ").AddCause(err).Build()
}
opts := editor.Options{Deaf: dEnv.DbEaFactory(), Tempdir: tmpDir}
opts := editor.Options{Deaf: dEnv.DbEaFactory(ctx), Tempdir: tmpDir}
verr := exportTblSchema(ctx, tn, root, wr, opts)
if verr != nil {
return verr
Expand Down
2 changes: 1 addition & 1 deletion go/cmd/dolt/commands/schcmds/import.go
Original file line number Diff line number Diff line change
Expand Up @@ -292,7 +292,7 @@ func importSchema(ctx context.Context, dEnv *env.DoltEnv, apr *argparser.ArgPars
return verr
}

sch, verr := inferSchemaFromFile(ctx, dEnv.DoltDB.ValueReadWriter().Format(), impArgs, root)
sch, verr := inferSchemaFromFile(ctx, dEnv.DoltDB(ctx).ValueReadWriter().Format(), impArgs, root)
if verr != nil {
return verr
}
Expand Down
2 changes: 1 addition & 1 deletion go/cmd/dolt/commands/schcmds/show.go
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ func printSchemas(ctx context.Context, apr *argparser.ArgParseResults, dEnv *env
if err != nil {
return errhand.BuildDError("error: ").AddCause(err).Build()
}
opts := editor.Options{Deaf: dEnv.DbEaFactory(), Tempdir: tmpDir}
opts := editor.Options{Deaf: dEnv.DbEaFactory(ctx), Tempdir: tmpDir}
sqlCtx, engine, _ := dsqle.PrepareCreateTableStmt(ctx, dsqle.NewUserSpaceDatabase(root, opts))

var notFound []string
Expand Down
2 changes: 1 addition & 1 deletion go/cmd/dolt/commands/schcmds/update-tag.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ func (cmd UpdateTagCmd) Exec(ctx context.Context, commandStr string, args []stri
help, usage := cli.HelpAndUsagePrinters(cli.CommandDocsForCommandString(commandStr, updateTagDocs, ap))
apr := cli.ParseArgsOrDie(ap, args, help)

if !types.IsFormat_DOLT(dEnv.DoltDB.Format()) {
if !types.IsFormat_DOLT(dEnv.DoltDB(ctx).Format()) {
verr := errhand.BuildDError("update-tag is only available in storage format __DOLT__").Build()
return commands.HandleVErrAndExitCode(verr, usage)
}
Expand Down
4 changes: 2 additions & 2 deletions go/cmd/dolt/commands/send_metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -82,8 +82,8 @@ func (cmd SendMetricsCmd) ArgParser() *argparser.ArgParser {

// Exec is the implementation of the command that flushes the events to the grpc service
func (cmd SendMetricsCmd) Exec(ctx context.Context, commandStr string, args []string, dEnv *env.DoltEnv, cliCtx cli.CliContext) int {
if dEnv.DoltDB != nil { // see go/cmd/dolt/dolt.go:interceptSendMetrics()
cli.PrintErrln("expected DoltEnv without DoltDB")
if dEnv.DoltDB(ctx) != nil { // see go/cmd/dolt/dolt.go:interceptSendMetrics()
cli.PrintErrln("expected DoltEnv without doltDB")
return 1
}

Expand Down
Loading

0 comments on commit b4b6d71

Please sign in to comment.