From 81e21a6051a85bfe641c9890c614aaf09b3aca1c Mon Sep 17 00:00:00 2001 From: milogreg Date: Wed, 4 Dec 2024 17:41:53 -0800 Subject: [PATCH 01/31] Generate config.yaml with current sql server config if one doesn't already exist --- go/cmd/dolt/commands/sqlserver/sqlserver.go | 54 +++++++++++++++ .../doltcore/servercfg/yaml_config.go | 69 ++++++++++++++++++- .../doltcore/servercfg/yaml_config_test.go | 32 +++++++++ 3 files changed, 154 insertions(+), 1 deletion(-) diff --git a/go/cmd/dolt/commands/sqlserver/sqlserver.go b/go/cmd/dolt/commands/sqlserver/sqlserver.go index 8baaff2b956..53a876fd38d 100644 --- a/go/cmd/dolt/commands/sqlserver/sqlserver.go +++ b/go/cmd/dolt/commands/sqlserver/sqlserver.go @@ -17,6 +17,7 @@ package sqlserver import ( "context" "fmt" + "os" "path/filepath" "strings" @@ -245,6 +246,11 @@ func StartServer(ctx context.Context, versionStr, commandStr string, args []stri return err } + err = generateYamlConfigIfNone(ap, help, args, dEnv, serverConfig) + if err != nil { + return err + } + err = servercfg.ApplySystemVariables(serverConfig, sql.SystemVariables) if err != nil { return err @@ -432,3 +438,51 @@ func setupDoltConfig(dEnv *env.DoltEnv, apr *argparser.ArgParseResults, config s return nil } + +func generateYamlConfigIfNone( + ap *argparser.ArgParser, + help cli.UsagePrinter, + args []string, + dEnv *env.DoltEnv, + serverConfig servercfg.ServerConfig) error { + const yamlConfigName = "config.yaml" + + specifiesConfigFile, err := argsSpecifyServerConfigFile(ap, help, args) + if err != nil { + return err + } + + if specifiesConfigFile { + return nil + } + + path := filepath.Join(serverConfig.DataDir(), yamlConfigName) + exists, isDir := dEnv.FS.Exists(path) + if exists { + if isDir { + cli.PrintErrf("Couldn't generate YAML config at %s: directory with same name already exists", path) + } + + return nil + } + + yamlConfig := servercfg.ServerConfigAsYAMLConfig(serverConfig) + err = dEnv.FS.WriteFile(path, []byte(yamlConfig.VerboseString()), os.ModePerm) + if err != nil { + return err + } + + return nil +} + +func argsSpecifyServerConfigFile(ap *argparser.ArgParser, help cli.UsagePrinter, args []string) (bool, error) { + apr := cli.ParseArgsOrDie(ap, args, help) + if err := validateSqlServerArgs(apr); err != nil { + cli.PrintErrln(color.RedString(err.Error())) + return false, err + } + + _, hasConfigFlag := apr.GetValue(configFileFlag) + + return hasConfigFlag, nil +} diff --git a/go/libraries/doltcore/servercfg/yaml_config.go b/go/libraries/doltcore/servercfg/yaml_config.go index b872580a980..c02200f137c 100644 --- a/go/libraries/doltcore/servercfg/yaml_config.go +++ b/go/libraries/doltcore/servercfg/yaml_config.go @@ -17,6 +17,7 @@ package servercfg import ( "fmt" "path/filepath" + "reflect" "strings" "unicode" "unicode/utf8" @@ -249,7 +250,73 @@ func clusterConfigAsYAMLConfig(config ClusterConfig) *ClusterYAMLConfig { // String returns the YAML representation of the config func (cfg YAMLConfig) String() string { - data, err := yaml.Marshal(cfg) + return formattedYAMLMarshal(cfg) +} + +// Same as String, but includes nil values for empty fields rather than omitting them. +func (cfg YAMLConfig) VerboseString() string { + return formattedYAMLMarshal(removeOmitemptyTags(cfg)) +} + +// Assumes 'in' has no circular references. +func removeOmitemptyTags(in any) any { + val := reflect.ValueOf(in) + typ := reflect.TypeOf(in) + + newType := removeOmitemptyTagsType(typ) + newVal := deepConvert(val, newType) + + return newVal.Interface() +} + +func removeOmitemptyTagsType(typ reflect.Type) reflect.Type { + switch typ.Kind() { + case reflect.Pointer: + return reflect.PointerTo(removeOmitemptyTagsType(typ.Elem())) + case reflect.Struct: + fields := []reflect.StructField{} + for i := 0; i < typ.NumField(); i++ { + field := typ.Field(i) + if field.IsExported() { + field.Tag = reflect.StructTag(strings.Replace(string(field.Tag), ",omitempty", "", -1)) + field.Type = removeOmitemptyTagsType(field.Type) + fields = append(fields, field) + } + } + + return reflect.StructOf(fields) + default: + return typ + } +} + +func deepConvert(val reflect.Value, typ reflect.Type) reflect.Value { + switch val.Kind() { + case reflect.Pointer: + if val.IsNil() { + return reflect.Zero(typ) + } + elemType := typ.Elem() + convertedPtr := reflect.New(elemType) + convertedPtr.Elem().Set(deepConvert(val.Elem(), elemType)) + + return convertedPtr + case reflect.Struct: + convertedStruct := reflect.New(typ).Elem() + for i := 0; i < convertedStruct.NumField(); i++ { + fieldName := typ.Field(i).Name + field := convertedStruct.Field(i) + field.Set(deepConvert(val.FieldByName(fieldName), field.Type())) + } + + return convertedStruct + default: + return val.Convert(typ) + } +} + +func formattedYAMLMarshal(toMarshal any) string { + data, err := yaml.Marshal(toMarshal) if err != nil { return "Failed to marshal as yaml: " + err.Error() diff --git a/go/libraries/doltcore/servercfg/yaml_config_test.go b/go/libraries/doltcore/servercfg/yaml_config_test.go index 20e3fefaa65..12d618fd9fe 100644 --- a/go/libraries/doltcore/servercfg/yaml_config_test.go +++ b/go/libraries/doltcore/servercfg/yaml_config_test.go @@ -15,6 +15,7 @@ package servercfg import ( + "reflect" "testing" "github.com/stretchr/testify/assert" @@ -415,3 +416,34 @@ listener: err = ValidateConfig(cfg) assert.Error(t, err) } + +// Tests if a circular reference exists in YAMLConfig. +// +// This test can be safely removed if: +// - YAMLConfig.VerboseString is updated to properly handle circular references. +// - yaml.Marshal is checked to make sure it properly handles circular references. +func TestNoCircularReferenceInYAMLConfig(t *testing.T) { + assert.False(t, hasCircularReference(reflect.TypeOf(YAMLConfig{}), map[reflect.Type]bool{})) +} + +// Only checks for struct circular references, not slice, map, etc. +func hasCircularReference(t reflect.Type, visited map[reflect.Type]bool) bool { + if t.Kind() == reflect.Ptr { + t = t.Elem() + } + if t.Kind() != reflect.Struct { + return false + } + if visited[t] { + return true + } + visited[t] = true + for i := 0; i < t.NumField(); i++ { + field := t.Field(i) + if hasCircularReference(field.Type, visited) { + return true + } + } + visited[t] = false + return false +} From 0df8e9a418982c856fa369bc59163326a4bb0398 Mon Sep 17 00:00:00 2001 From: milogreg Date: Thu, 5 Dec 2024 13:50:08 -0800 Subject: [PATCH 02/31] Fill default values in YAML config if they exist rather than leaving them null --- .../doltcore/servercfg/serverconfig.go | 4 ++ .../doltcore/servercfg/yaml_config.go | 37 ++++++++++++++++++- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/go/libraries/doltcore/servercfg/serverconfig.go b/go/libraries/doltcore/servercfg/serverconfig.go index 70e4380dd02..92b7a3b6868 100644 --- a/go/libraries/doltcore/servercfg/serverconfig.go +++ b/go/libraries/doltcore/servercfg/serverconfig.go @@ -211,6 +211,10 @@ type ServerConfig interface { // DefaultServerConfig creates a `*ServerConfig` that has all of the options set to their default values. func DefaultServerConfig() ServerConfig { + return defaultServerConfigYAML() +} + +func defaultServerConfigYAML() *YAMLConfig { return &YAMLConfig{ LogLevelStr: ptr(string(DefaultLogLevel)), MaxQueryLenInLogs: ptr(DefaultMaxLoggedQueryLen), diff --git a/go/libraries/doltcore/servercfg/yaml_config.go b/go/libraries/doltcore/servercfg/yaml_config.go index c02200f137c..285f4b808a5 100644 --- a/go/libraries/doltcore/servercfg/yaml_config.go +++ b/go/libraries/doltcore/servercfg/yaml_config.go @@ -255,7 +255,42 @@ func (cfg YAMLConfig) String() string { // Same as String, but includes nil values for empty fields rather than omitting them. func (cfg YAMLConfig) VerboseString() string { - return formattedYAMLMarshal(removeOmitemptyTags(cfg)) + withDefaults := cfg + withDefaults.fillDefaults() + + return formattedYAMLMarshal(removeOmitemptyTags(withDefaults)) +} + +// Assumes YAMLConfig has no circular references. +func (cfg *YAMLConfig) fillDefaults() { + defaults := defaultServerConfigYAML() + recursiveFillDefaults(reflect.ValueOf(cfg), reflect.ValueOf(defaults)) +} + +func recursiveFillDefaults(cfgValue, defaultsValue reflect.Value) { + cfgValue = cfgValue.Elem() + defaultsValue = defaultsValue.Elem() + + if cfgValue.Kind() != reflect.Struct { + return + } + + for i := 0; i < cfgValue.NumField(); i++ { + field := cfgValue.Field(i) + defaultField := defaultsValue.Field(i) + + if field.Kind() == reflect.Pointer { + if !defaultField.IsNil() { + if field.IsNil() { + field.Set(defaultField) + } else { + recursiveFillDefaults(field, defaultField) + } + } + } else { + recursiveFillDefaults(field.Addr(), defaultField.Addr()) + } + } } // Assumes 'in' has no circular references. From 3b24a0c08ba04a37c8e581ae0a3c0c3fde6d380c Mon Sep 17 00:00:00 2001 From: milogreg Date: Thu, 5 Dec 2024 14:32:37 -0800 Subject: [PATCH 03/31] Comment out null values in generated YAML config --- .../doltcore/servercfg/yaml_config.go | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/go/libraries/doltcore/servercfg/yaml_config.go b/go/libraries/doltcore/servercfg/yaml_config.go index 285f4b808a5..f665987841f 100644 --- a/go/libraries/doltcore/servercfg/yaml_config.go +++ b/go/libraries/doltcore/servercfg/yaml_config.go @@ -253,12 +253,17 @@ func (cfg YAMLConfig) String() string { return formattedYAMLMarshal(cfg) } -// Same as String, but includes nil values for empty fields rather than omitting them. +// Behaves like String, but includes empty fields instead of omitting them. +// If an empty field has a default value, the default will be used. +// If an empty field has no default value, a commented-out placeholder will be used. func (cfg YAMLConfig) VerboseString() string { withDefaults := cfg withDefaults.fillDefaults() - return formattedYAMLMarshal(removeOmitemptyTags(withDefaults)) + formatted := formattedYAMLMarshal(removeOmitemptyTags(withDefaults)) + formatted = commentNullYAMLValues(formatted) + + return formatted } // Assumes YAMLConfig has no circular references. @@ -381,6 +386,19 @@ func formattedYAMLMarshal(toMarshal any) string { return result } +func commentNullYAMLValues(needsComments string) string { + lines := strings.Split(needsComments, "\n") + for i := 0; i < len(lines); i++ { + if strings.HasSuffix(lines[i], "null") { + withoutSpace := strings.TrimSpace(lines[i]) + space := lines[i][:len(lines[i])-len(withoutSpace)] + lines[i] = space + "# " + withoutSpace + } + } + + return strings.Join(lines, "\n") +} + // Host returns the domain that the server will run on. Accepts an IPv4 or IPv6 address, in addition to localhost. func (cfg YAMLConfig) Host() string { if cfg.ListenerConfig.HostStr == nil { From 1b407e076593b9e6957c140fbb36224a80a65374 Mon Sep 17 00:00:00 2001 From: milogreg Date: Sun, 8 Dec 2024 16:40:45 -0800 Subject: [PATCH 04/31] Add bats tests for config file generation --- .../bats/helper/query-server-common.bash | 12 +- .../sql-server-config-file-generation.bats | 142 ++++++++++++++++++ 2 files changed, 151 insertions(+), 3 deletions(-) create mode 100644 integration-tests/bats/sql-server-config-file-generation.bats diff --git a/integration-tests/bats/helper/query-server-common.bash b/integration-tests/bats/helper/query-server-common.bash index 456be9d53ea..cf9e05aef89 100644 --- a/integration-tests/bats/helper/query-server-common.bash +++ b/integration-tests/bats/helper/query-server-common.bash @@ -60,12 +60,18 @@ start_sql_server() { # arguments to dolt-sql-server (excluding --port, which is defined in # this func) start_sql_server_with_args() { - DEFAULT_DB="" PORT=$( definePORT ) + start_sql_server_with_args_no_port "$@" --port=$PORT +} + +# behaves like start_sql_server_with_args, but doesn't define --port. +# caller must set variable PORT to proper value before calling. +start_sql_server_with_args_no_port() { + DEFAULT_DB="" if [ "$IS_WINDOWS" == true ]; then - dolt sql-server "$@" --port=$PORT & + dolt sql-server "$@" & else - dolt sql-server "$@" --port=$PORT --socket "dolt.$PORT.sock" & + dolt sql-server "$@" --socket "dolt.$PORT.sock" & fi SERVER_PID=$! wait_for_connection $PORT 8500 diff --git a/integration-tests/bats/sql-server-config-file-generation.bats b/integration-tests/bats/sql-server-config-file-generation.bats new file mode 100644 index 00000000000..63daf9e0ad9 --- /dev/null +++ b/integration-tests/bats/sql-server-config-file-generation.bats @@ -0,0 +1,142 @@ +#!/usr/bin/env bats +load "$BATS_TEST_DIRNAME/helper/common.bash" +load "$BATS_TEST_DIRNAME/helper/query-server-common.bash" + +CONFIG_FILE_NAME=config.yaml + +DATABASE_DIRS=( + . + mydir + nest1/nest2/nest3 +) + +setup() { + setup_common +} + +teardown() { + stop_sql_server + teardown_common +} + +@test "sql-server-config-file-generation: config file is generated if one doesn't exist" { + for data_dir in "${DATABASE_DIRS[@]}"; do + if [[ "$data_dir" != "." ]]; then + mkdir -p "$data_dir" + fi + + start_sql_server_with_args --data-dir "$data_dir" --host 0.0.0.0 --user dolt + + [[ -f "$data_dir/$CONFIG_FILE_NAME" ]] || false + + rm "$data_dir/$CONFIG_FILE_NAME" + stop_sql_server + done +} + +@test "sql-server-config-file-generation: config file isn't generated if one already exists" { + for data_dir in "${DATABASE_DIRS[@]}"; do + if [[ "$data_dir" != "." ]]; then + mkdir -p "$data_dir" + fi + + echo "Don't overwrite me!" >"$data_dir/$CONFIG_FILE_NAME" + + start_sql_server_with_args --data-dir "$data_dir" --host 0.0.0.0 --user dolt + + run cat "$data_dir/$CONFIG_FILE_NAME" + [ $status -eq 0 ] + [[ "$output" =~ "Don't overwrite me!" ]] || false + + rm "$data_dir/$CONFIG_FILE_NAME" + stop_sql_server + done +} + +@test "sql-server-config-file-generation: config file isn't generated if a config is specified in args" { + for data_dir in "${DATABASE_DIRS[@]}"; do + if [[ "$data_dir" != "." ]]; then + mkdir -p "$data_dir" + fi + + NOT_CONFIG_FILE_NAME="not-$CONFIG_FILE_NAME" + PORT=$(definePORT) + + cat >"$data_dir/$NOT_CONFIG_FILE_NAME" < Date: Mon, 9 Dec 2024 11:49:31 -0800 Subject: [PATCH 05/31] Skip bats tests when SQL_ENGINE is set to remote-engine --- integration-tests/bats/sql-server-config-file-generation.bats | 3 +++ 1 file changed, 3 insertions(+) diff --git a/integration-tests/bats/sql-server-config-file-generation.bats b/integration-tests/bats/sql-server-config-file-generation.bats index 63daf9e0ad9..339ce763ada 100644 --- a/integration-tests/bats/sql-server-config-file-generation.bats +++ b/integration-tests/bats/sql-server-config-file-generation.bats @@ -11,6 +11,9 @@ DATABASE_DIRS=( ) setup() { + if [ "$SQL_ENGINE" = "remote-engine" ]; then + skip "This test tests remote connections directly, SQL_ENGINE is not needed." + fi setup_common } From 698cc6356c604312b43445556e4bfecae780c645 Mon Sep 17 00:00:00 2001 From: milogreg Date: Mon, 9 Dec 2024 15:13:38 -0800 Subject: [PATCH 06/31] Merge main --- .github/workflows/cd-release-pgo.yaml | 4 +- .../commands/sqlserver/command_line_config.go | 34 ------- go/cmd/dolt/commands/sqlserver/server.go | 15 +-- go/cmd/dolt/commands/sqlserver/sqlserver.go | 2 - go/cmd/dolt/doltversion/version.go | 2 +- go/go.mod | 6 +- go/go.sum | 8 +- go/go.work | 4 +- .../doltcore/branch_control/branch_control.go | 16 +++- go/libraries/doltcore/diff/table_deltas.go | 3 - go/libraries/doltcore/doltdb/table.go | 46 --------- .../doltcore/merge/violations_prolly.go | 21 ++++- .../doltcore/servercfg/serverconfig.go | 13 --- .../servercfg/testdata/minver_validation.txt | 6 +- .../doltcore/servercfg/yaml_config.go | 88 +++++++----------- .../doltcore/servercfg/yaml_config_test.go | 1 - go/libraries/doltcore/sqle/alterschema.go | 5 +- .../doltcore/sqle/alterschema_test.go | 2 +- go/libraries/doltcore/sqle/database.go | 17 +++- .../sqle/dtables/constraint_violations.go | 2 +- .../dtables/constraint_violations_prolly.go | 62 +++++++++++- .../sqle/dtables/merge_status_table.go | 20 ++-- .../doltcore/sqle/dtables/status_table.go | 16 +++- .../doltcore/sqle/dtables/workspace_table.go | 3 + .../sqle/enginetest/dolt_engine_tests.go | 1 + .../doltcore/sqle/enginetest/dolt_harness.go | 16 ++-- .../doltcore/sqle/enginetest/dolt_queries.go | 8 +- .../sqle/enginetest/dolt_queries_merge.go | 4 +- .../doltcore/sqle/read_replica_database.go | 14 +-- .../doltcore/table/typed/parquet/reader.go | 4 +- .../continuous_integration/SysbenchDockerfile | 2 +- integration-tests/DataDumpLoadDockerfile | 2 +- integration-tests/MySQLDockerfile | 2 +- integration-tests/ORMDockerfile | 2 +- integration-tests/bats/branch-control.bats | 5 + .../bats/helper/index-on-writes-common.bash | 2 +- .../bats/helper/parquet/strings.parquet | Bin 0 -> 282 bytes .../bats/helper/parquet/strings.sql | 1 + .../bats/import-create-tables.bats | 15 +++ integration-tests/bats/replication.bats | 38 ++++++++ integration-tests/bats/sql-server.bats | 2 +- integration-tests/go-sql-server-driver/go.mod | 4 +- .../tests/sql-server-config.yaml | 44 +-------- 43 files changed, 283 insertions(+), 279 deletions(-) create mode 100644 integration-tests/bats/helper/parquet/strings.parquet create mode 100644 integration-tests/bats/helper/parquet/strings.sql diff --git a/.github/workflows/cd-release-pgo.yaml b/.github/workflows/cd-release-pgo.yaml index fd694cc6fdb..27d8a853931 100644 --- a/.github/workflows/cd-release-pgo.yaml +++ b/.github/workflows/cd-release-pgo.yaml @@ -84,9 +84,9 @@ jobs: run: | latest=$(git rev-parse HEAD) echo "commitish=$latest" >> $GITHUB_OUTPUT - GO_BUILD_VERSION=1.22 go/utils/publishrelease/buildpgobinaries.sh + GO_BUILD_VERSION=1.23.3 go/utils/publishrelease/buildpgobinaries.sh env: - GO_BUILD_VERSION: "1.22" + GO_BUILD_VERSION: "1.23.3" PROFILE: ${{ format('{0}/dolt-cpu-profile.pprof', github.workspace) }} - name: Create Release id: create_release diff --git a/go/cmd/dolt/commands/sqlserver/command_line_config.go b/go/cmd/dolt/commands/sqlserver/command_line_config.go index 592e25747af..6c05e48744e 100755 --- a/go/cmd/dolt/commands/sqlserver/command_line_config.go +++ b/go/cmd/dolt/commands/sqlserver/command_line_config.go @@ -41,13 +41,11 @@ type commandLineServerConfig struct { autoCommit bool doltTransactionCommit bool maxConnections uint64 - queryParallelism int tlsKey string tlsCert string requireSecureTransport bool maxLoggedQueryLen int shouldEncodeLoggedQuery bool - persistenceBehavior string privilegeFilePath string branchControlFilePath string allowCleartextPasswords bool @@ -72,8 +70,6 @@ func DefaultCommandLineServerConfig() *commandLineServerConfig { logLevel: servercfg.DefaultLogLevel, autoCommit: servercfg.DefaultAutoCommit, maxConnections: servercfg.DefaultMaxConnections, - queryParallelism: servercfg.DefaultQueryParallelism, - persistenceBehavior: servercfg.DefaultPersistenceBahavior, dataDir: servercfg.DefaultDataDir, cfgDir: filepath.Join(servercfg.DefaultDataDir, servercfg.DefaultCfgDir), privilegeFilePath: filepath.Join(servercfg.DefaultDataDir, servercfg.DefaultCfgDir, servercfg.DefaultPrivilegeFilePath), @@ -124,10 +120,6 @@ func NewCommandLineConfig(creds *cli.UserPassword, apr *argparser.ArgParseResult config.WithRemotesapiReadOnly(&val) } - if persistenceBehavior, ok := apr.GetValue(persistenceBehaviorFlag); ok { - config.withPersistenceBehavior(persistenceBehavior) - } - if timeoutStr, ok := apr.GetValue(timeoutFlag); ok { timeout, err := strconv.ParseUint(timeoutStr, 10, 64) @@ -156,10 +148,6 @@ func NewCommandLineConfig(creds *cli.UserPassword, apr *argparser.ArgParseResult config.withDataDir(dataDir) } - if queryParallelism, ok := apr.GetInt(queryParallelismFlag); ok { - config.withQueryParallelism(queryParallelism) - } - if maxConnections, ok := apr.GetInt(maxConnectionsFlag); ok { config.withMaxConnections(uint64(maxConnections)) } @@ -236,16 +224,6 @@ func (cfg *commandLineServerConfig) MaxConnections() uint64 { return cfg.maxConnections } -// QueryParallelism returns the parallelism that should be used by the go-mysql-server analyzer -func (cfg *commandLineServerConfig) QueryParallelism() int { - return cfg.queryParallelism -} - -// PersistenceBehavior returns whether to autoload persisted server configuration -func (cfg *commandLineServerConfig) PersistenceBehavior() string { - return cfg.persistenceBehavior -} - // TLSKey returns a path to the servers PEM-encoded private TLS key. "" if there is none. func (cfg *commandLineServerConfig) TLSKey() string { return cfg.tlsKey @@ -402,12 +380,6 @@ func (cfg *commandLineServerConfig) withMaxConnections(maxConnections uint64) *c return cfg } -// withQueryParallelism updates the query parallelism and returns the called `*commandLineServerConfig`, which is useful for chaining calls. -func (cfg *commandLineServerConfig) withQueryParallelism(queryParallelism int) *commandLineServerConfig { - cfg.queryParallelism = queryParallelism - return cfg -} - // withDataDir updates the path to a directory to use as the data dir. func (cfg *commandLineServerConfig) withDataDir(dataDir string) *commandLineServerConfig { cfg.dataDir = dataDir @@ -420,12 +392,6 @@ func (cfg *commandLineServerConfig) withCfgDir(cfgDir string) *commandLineServer return cfg } -// withPersistenceBehavior updates persistence behavior of system globals on server init -func (cfg *commandLineServerConfig) withPersistenceBehavior(persistenceBehavior string) *commandLineServerConfig { - cfg.persistenceBehavior = persistenceBehavior - return cfg -} - // withPrivilegeFilePath updates the path to the file which contains all needed privilege information in the form of a JSON string func (cfg *commandLineServerConfig) withPrivilegeFilePath(privFilePath string) *commandLineServerConfig { cfg.privilegeFilePath = privFilePath diff --git a/go/cmd/dolt/commands/sqlserver/server.go b/go/cmd/dolt/commands/sqlserver/server.go index 76487a5e9a5..3915b1973f0 100644 --- a/go/cmd/dolt/commands/sqlserver/server.go +++ b/go/cmd/dolt/commands/sqlserver/server.go @@ -956,18 +956,9 @@ func getConfigFromServerConfig(serverConfig servercfg.ServerConfig) (server.Conf return server.Config{}, err } - // if persist is 'load' we use currently set persisted global variable, - // else if 'ignore' we set persisted global variable to current value from serverConfig - if serverConfig.PersistenceBehavior() == servercfg.LoadPerisistentGlobals { - serverConf, err = serverConf.NewConfig() - if err != nil { - return server.Config{}, err - } - } else { - err = sql.SystemVariables.SetGlobal("max_connections", serverConfig.MaxConnections()) - if err != nil { - return server.Config{}, err - } + serverConf, err = serverConf.NewConfig() + if err != nil { + return server.Config{}, err } // Do not set the value of Version. Let it default to what go-mysql-server uses. This should be equivalent diff --git a/go/cmd/dolt/commands/sqlserver/sqlserver.go b/go/cmd/dolt/commands/sqlserver/sqlserver.go index 53a876fd38d..32cce773b76 100644 --- a/go/cmd/dolt/commands/sqlserver/sqlserver.go +++ b/go/cmd/dolt/commands/sqlserver/sqlserver.go @@ -45,7 +45,6 @@ const ( configFileFlag = "config" queryParallelismFlag = "query-parallelism" maxConnectionsFlag = "max-connections" - persistenceBehaviorFlag = "persistence-behavior" allowCleartextPasswordsFlag = "allow-cleartext-passwords" socketFlag = "socket" remotesapiPortFlag = "remotesapi-port" @@ -173,7 +172,6 @@ func (cmd SqlServerCmd) ArgParserWithName(name string) *argparser.ArgParser { ap.SupportsFlag(noAutoCommitFlag, "", "Set @@autocommit = off for the server.") ap.SupportsInt(queryParallelismFlag, "", "num-go-routines", "Deprecated, no effect in current versions of Dolt") ap.SupportsInt(maxConnectionsFlag, "", "max-connections", fmt.Sprintf("Set the number of connections handled by the server. Defaults to `%d`.", serverConfig.MaxConnections())) - ap.SupportsString(persistenceBehaviorFlag, "", "persistence-behavior", fmt.Sprintf("Indicate whether to `load` or `ignore` persisted global variables. Defaults to `%s`.", serverConfig.PersistenceBehavior())) ap.SupportsString(commands.PrivsFilePathFlag, "", "privilege file", "Path to a file to load and store users and grants. Defaults to `$doltcfg-dir/privileges.db`. Will be created as needed.") ap.SupportsString(commands.BranchCtrlPathFlag, "", "branch control file", "Path to a file to load and store branch control permissions. Defaults to `$doltcfg-dir/branch_control.db`. Will be created as needed.") ap.SupportsString(allowCleartextPasswordsFlag, "", "allow-cleartext-passwords", "Allows use of cleartext passwords. Defaults to false.") diff --git a/go/cmd/dolt/doltversion/version.go b/go/cmd/dolt/doltversion/version.go index 9b6d6414973..be05ffa14ab 100644 --- a/go/cmd/dolt/doltversion/version.go +++ b/go/cmd/dolt/doltversion/version.go @@ -16,5 +16,5 @@ package doltversion const ( - Version = "1.43.20" + Version = "1.44.1" ) diff --git a/go/go.mod b/go/go.mod index 88d98360995..58b655bc87f 100644 --- a/go/go.mod +++ b/go/go.mod @@ -15,7 +15,7 @@ require ( github.com/dolthub/fslock v0.0.3 github.com/dolthub/ishell v0.0.0-20240701202509-2b217167d718 github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81 - github.com/dolthub/vitess v0.0.0-20241126223332-cd8f828f26ac + github.com/dolthub/vitess v0.0.0-20241209181212-588631aba4be github.com/dustin/go-humanize v1.0.1 github.com/fatih/color v1.13.0 github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 @@ -57,7 +57,7 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 github.com/creasty/defaults v1.6.0 github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2 - github.com/dolthub/go-mysql-server v0.18.2-0.20241203001635-29d50967b518 + github.com/dolthub/go-mysql-server v0.18.2-0.20241209182739-d644619dc9ed github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63 github.com/dolthub/swiss v0.1.0 github.com/goccy/go-json v0.10.2 @@ -170,4 +170,4 @@ require ( replace github.com/dolthub/dolt/go/gen/proto/dolt/services/eventsapi => ./gen/proto/dolt/services/eventsapi -go 1.22.2 +go 1.23.3 diff --git a/go/go.sum b/go/go.sum index 8bab33443e9..4012820204c 100644 --- a/go/go.sum +++ b/go/go.sum @@ -183,8 +183,8 @@ github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U= github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0= github.com/dolthub/go-icu-regex v0.0.0-20240916130659-0118adc6b662 h1:aC17hZD6iwzBwwfO5M+3oBT5E5gGRiQPdn+vzpDXqIA= github.com/dolthub/go-icu-regex v0.0.0-20240916130659-0118adc6b662/go.mod h1:KPUcpx070QOfJK1gNe0zx4pA5sicIK1GMikIGLKC168= -github.com/dolthub/go-mysql-server v0.18.2-0.20241203001635-29d50967b518 h1:aqDEdL6NI1A2RBEoVr+d41yGLiH6ww1YhIsoO1ydV70= -github.com/dolthub/go-mysql-server v0.18.2-0.20241203001635-29d50967b518/go.mod h1:QdaXQKE8XFwM4P1yN14m2eydx4V2xyuqpQp4tmNoXzQ= +github.com/dolthub/go-mysql-server v0.18.2-0.20241209182739-d644619dc9ed h1:YlXOo9xfRRglBrg+OvAFpBNohzWtK3YQ7SnTZNTbYZ0= +github.com/dolthub/go-mysql-server v0.18.2-0.20241209182739-d644619dc9ed/go.mod h1:Ra4lA9OjCy9J5tWPn05IBBlODnwaXZc8j+cChvfN/9Q= github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63 h1:OAsXLAPL4du6tfbBgK0xXHZkOlos63RdKYS3Sgw/dfI= github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63/go.mod h1:lV7lUeuDhH5thVGDCKXbatwKy2KW80L4rMT46n+Y2/Q= github.com/dolthub/ishell v0.0.0-20240701202509-2b217167d718 h1:lT7hE5k+0nkBdj/1UOSFwjWpNxf+LCApbRHgnCA17XE= @@ -197,8 +197,8 @@ github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81 h1:7/v8q9X github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81/go.mod h1:siLfyv2c92W1eN/R4QqG/+RjjX5W2+gCTRjZxBjI3TY= github.com/dolthub/swiss v0.1.0 h1:EaGQct3AqeP/MjASHLiH6i4TAmgbG/c4rA6a1bzCOPc= github.com/dolthub/swiss v0.1.0/go.mod h1:BeucyB08Vb1G9tumVN3Vp/pyY4AMUnr9p7Rz7wJ7kAQ= -github.com/dolthub/vitess v0.0.0-20241126223332-cd8f828f26ac h1:A0U/OdIqdCkAV0by7MVBbnSyZBsa94ZjIZxx7PhjBW4= -github.com/dolthub/vitess v0.0.0-20241126223332-cd8f828f26ac/go.mod h1:alcJgfdyIhFaAiYyEmuDCFSLCzedz3KCaIclLoCUtJg= +github.com/dolthub/vitess v0.0.0-20241209181212-588631aba4be h1:YF+vUXEAMqai036iZPJS2JRKd3pXcCQ1TYcLWBR4boo= +github.com/dolthub/vitess v0.0.0-20241209181212-588631aba4be/go.mod h1:1gQZs/byeHLMSul3Lvl3MzioMtOW1je79QYGyi2fd70= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= diff --git a/go/go.work b/go/go.work index 0535a0ff63f..36a9d704cb5 100644 --- a/go/go.work +++ b/go/go.work @@ -1,6 +1,6 @@ -go 1.22.5 +go 1.23.3 -toolchain go1.22.7 +toolchain go1.23.3 use ( . diff --git a/go/libraries/doltcore/branch_control/branch_control.go b/go/libraries/doltcore/branch_control/branch_control.go index 4225f5390fc..efbb7c9f5ca 100644 --- a/go/libraries/doltcore/branch_control/branch_control.go +++ b/go/libraries/doltcore/branch_control/branch_control.go @@ -19,6 +19,7 @@ import ( goerrors "errors" "fmt" "os" + "strings" "sync/atomic" flatbuffers "github.com/dolthub/flatbuffers/v23/go" @@ -243,7 +244,7 @@ func CheckAccess(ctx context.Context, flags Permissions) error { user := branchAwareSession.GetUser() host := branchAwareSession.GetHost() - database := branchAwareSession.GetCurrentDatabase() + database := getDatabaseNameOnly(branchAwareSession.GetCurrentDatabase()) branch, err := branchAwareSession.GetBranch() if err != nil { return err @@ -277,7 +278,7 @@ func CanCreateBranch(ctx context.Context, branchName string) error { user := branchAwareSession.GetUser() host := branchAwareSession.GetHost() - database := branchAwareSession.GetCurrentDatabase() + database := getDatabaseNameOnly(branchAwareSession.GetCurrentDatabase()) if controller.Namespace.CanCreate(database, branchName, user, host) { return nil } @@ -304,7 +305,7 @@ func CanDeleteBranch(ctx context.Context, branchName string) error { user := branchAwareSession.GetUser() host := branchAwareSession.GetHost() - database := branchAwareSession.GetCurrentDatabase() + database := getDatabaseNameOnly(branchAwareSession.GetCurrentDatabase()) // Get the permissions for the branch, user, and host combination _, perms := controller.Access.Match(database, branchName, user, host) // If the user has the write or admin flags, then we allow access @@ -329,7 +330,7 @@ func AddAdminForContext(ctx context.Context, branchName string) error { user := branchAwareSession.GetUser() host := branchAwareSession.GetHost() - database := branchAwareSession.GetCurrentDatabase() + database := getDatabaseNameOnly(branchAwareSession.GetCurrentDatabase()) // Check if we already have admin permissions for the given branch, as there's no need to do another insertion if so controller.Access.RWMutex.RLock() _, modPerms := controller.Access.Match(database, branchName, user, host) @@ -381,3 +382,10 @@ func HasDatabasePrivileges(ctx Context, database string) bool { sql.PrivilegeType_Insert, sql.PrivilegeType_Update, sql.PrivilegeType_Delete, sql.PrivilegeType_Execute, sql.PrivilegeType_GrantOption) return hasSuper || isGlobalAdmin || isDatabaseAdmin } + +// getDatabaseNameOnly gets the database name only, which is useful for when the database name includes a revision. +// This is a direct reimplementation of the logic in dsess.SplitRevisionDbName, however we cannot use that function due +// to import cycles. +func getDatabaseNameOnly(dbName string) string { + return strings.SplitN(dbName, "/", 2)[0] +} diff --git a/go/libraries/doltcore/diff/table_deltas.go b/go/libraries/doltcore/diff/table_deltas.go index e91b7d3023a..e1cc60ccf21 100644 --- a/go/libraries/doltcore/diff/table_deltas.go +++ b/go/libraries/doltcore/diff/table_deltas.go @@ -520,9 +520,6 @@ func (td TableDelta) HasPrimaryKeySetChanged() bool { } func (td TableDelta) HasChanges() (bool, error) { - fromString := td.FromTable.DebugString(context.Background(), td.FromTable.NodeStore()) - toString := td.ToTable.DebugString(context.Background(), td.ToTable.NodeStore()) - _, _ = fromString, toString hashChanged, err := td.HasHashChanged() if err != nil { return false, err diff --git a/go/libraries/doltcore/doltdb/table.go b/go/libraries/doltcore/doltdb/table.go index bafce8b00f3..fcfccaed75c 100644 --- a/go/libraries/doltcore/doltdb/table.go +++ b/go/libraries/doltcore/doltdb/table.go @@ -20,13 +20,9 @@ import ( "fmt" "unicode" - "github.com/dolthub/go-mysql-server/sql" - gmstypes "github.com/dolthub/go-mysql-server/sql/types" - "github.com/dolthub/dolt/go/libraries/doltcore/conflict" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable" "github.com/dolthub/dolt/go/libraries/doltcore/schema" - "github.com/dolthub/dolt/go/libraries/doltcore/schema/typeinfo" "github.com/dolthub/dolt/go/store/hash" "github.com/dolthub/dolt/go/store/prolly/tree" "github.com/dolthub/dolt/go/store/types" @@ -35,8 +31,6 @@ import ( var ErrNoConflictsResolved = errors.New("no conflicts resolved") -const dolt_row_hash_tag = 0 - // IsValidTableName checks if name is a valid identifier, and doesn't end with space characters func IsValidTableName(name string) bool { if len(name) == 0 || unicode.IsSpace(rune(name[len(name)-1])) { @@ -351,46 +345,6 @@ func (t *Table) getNomsConflictSchemas(ctx context.Context) (base, sch, mergeSch return cs.Base, cs.Schema, cs.MergeSchema, nil } -// GetConstraintViolationsSchema returns this table's dolt_constraint_violations system table schema. -func (t *Table) GetConstraintViolationsSchema(ctx context.Context) (schema.Schema, error) { - sch, err := t.GetSchema(ctx) - if err != nil { - return nil, err - } - - typeType, err := typeinfo.FromSqlType( - gmstypes.MustCreateEnumType([]string{"foreign key", "unique index", "check constraint", "not null"}, sql.Collation_Default)) - if err != nil { - return nil, err - } - typeCol, err := schema.NewColumnWithTypeInfo("violation_type", schema.DoltConstraintViolationsTypeTag, typeType, true, "", false, "") - if err != nil { - return nil, err - } - infoCol, err := schema.NewColumnWithTypeInfo("violation_info", schema.DoltConstraintViolationsInfoTag, typeinfo.JSONType, false, "", false, "") - if err != nil { - return nil, err - } - - colColl := schema.NewColCollection() - - // the commit hash or working set hash of the right side during merge - colColl = colColl.Append(schema.NewColumn("from_root_ish", 0, types.StringKind, false)) - colColl = colColl.Append(typeCol) - if schema.IsKeyless(sch) { - // If this is a keyless table, we need to add a new column for the keyless table's generated row hash. - // We need to add this internal row hash value, in order to guarantee a unique primary key in the - // constraint violations table. - colColl = colColl.Append(schema.NewColumn("dolt_row_hash", dolt_row_hash_tag, types.BlobKind, true)) - } else { - colColl = colColl.Append(sch.GetPKCols().GetColumns()...) - } - colColl = colColl.Append(sch.GetNonPKCols().GetColumns()...) - colColl = colColl.Append(infoCol) - - return schema.SchemaFromCols(colColl) -} - // GetConstraintViolations returns a map of all constraint violations for this table, along with a bool indicating // whether the table has any violations. func (t *Table) GetConstraintViolations(ctx context.Context) (types.Map, error) { diff --git a/go/libraries/doltcore/merge/violations_prolly.go b/go/libraries/doltcore/merge/violations_prolly.go index 977bb042638..2c7d1cf8e80 100644 --- a/go/libraries/doltcore/merge/violations_prolly.go +++ b/go/libraries/doltcore/merge/violations_prolly.go @@ -25,7 +25,7 @@ import ( "github.com/dolthub/dolt/go/store/val" ) -func NextConstraintViolation(ctx context.Context, itr prolly.ArtifactIter, kd, vd val.TupleDesc, ns tree.NodeStore) (violationType uint64, key sql.Row, value sql.Row, err error) { +func NextConstraintViolation(ctx context.Context, itr prolly.ArtifactIter, kd, vd val.TupleDesc, ns tree.NodeStore) (violationType any, key sql.Row, value sql.Row, err error) { art, err := itr.Next(ctx) if err != nil { return @@ -56,7 +56,13 @@ func NextConstraintViolation(ctx context.Context, itr prolly.ArtifactIter, kd, v return MapCVType(art.ArtType), key, value, nil } -func MapCVType(artifactType prolly.ArtifactType) (outType uint64) { +// MapCVType maps an ArtifactType to value for a sql.Row in the dolt_constraint_violations_* table. +// This is used by Doltgres to convert the ArtifactType to the correct type. +var MapCVType = func(artifactType prolly.ArtifactType) any { + return mapCVType(artifactType) +} + +func mapCVType(artifactType prolly.ArtifactType) (outType uint64) { switch artifactType { case prolly.ArtifactTypeForeignKeyViol: outType = uint64(CvType_ForeignKey) @@ -72,7 +78,16 @@ func MapCVType(artifactType prolly.ArtifactType) (outType uint64) { return } -func UnmapCVType(in CvType) (out prolly.ArtifactType) { +// UnmapCVType unmaps a sql.Row value from the dolt_constraint_violations_* table to an ArtifactType. +// This is used by Doltgres to convert a value of a different type to an ArtifactType. +var UnmapCVType = func(in any) (out prolly.ArtifactType) { + if cv, ok := in.(uint64); ok { + return unmapCVType(CvType(cv)) + } + panic("invalid type") +} + +func unmapCVType(in CvType) (out prolly.ArtifactType) { switch in { case CvType_ForeignKey: out = prolly.ArtifactTypeForeignKeyViol diff --git a/go/libraries/doltcore/servercfg/serverconfig.go b/go/libraries/doltcore/servercfg/serverconfig.go index 92b7a3b6868..0e0b9dca168 100644 --- a/go/libraries/doltcore/servercfg/serverconfig.go +++ b/go/libraries/doltcore/servercfg/serverconfig.go @@ -50,8 +50,6 @@ const ( DefaultAutoCommit = true DefaultDoltTransactionCommit = false DefaultMaxConnections = 100 - DefaultQueryParallelism = 0 - DefaultPersistenceBahavior = LoadPerisistentGlobals DefaultDataDir = "." DefaultCfgDir = ".doltcfg" DefaultPrivilegeFilePath = "privileges.db" @@ -64,11 +62,6 @@ const ( DefaultEncodeLoggedQuery = false ) -const ( - IgnorePeristentGlobals = "ignore" - LoadPerisistentGlobals = "load" -) - func ptr[T any](t T) *T { return &t } @@ -153,8 +146,6 @@ type ServerConfig interface { CfgDir() string // MaxConnections returns the maximum number of simultaneous connections the server will allow. The default is 1 MaxConnections() uint64 - // QueryParallelism returns the parallelism that should be used by the go-mysql-server analyzer - QueryParallelism() int // TLSKey returns a path to the servers PEM-encoded private TLS key. "" if there is none. TLSKey() string // TLSCert returns a path to the servers PEM-encoded TLS certificate chain. "" if there is none. @@ -169,8 +160,6 @@ type ServerConfig interface { // If true, queries will be logged as base64 encoded strings. // If false (default behavior), queries will be logged as strings, but newlines and tabs will be replaced with spaces. ShouldEncodeLoggedQuery() bool - // PersistenceBehavior is "load" if we include persisted system globals on server init - PersistenceBehavior() string // DisableClientMultiStatements is true if we want the server to not // process incoming ComQuery packets as if they had multiple queries in // them, even if the client advertises support for MULTI_STATEMENTS. @@ -222,7 +211,6 @@ func defaultServerConfigYAML() *YAMLConfig { BehaviorConfig: BehaviorYAMLConfig{ ReadOnly: ptr(DefaultReadOnly), AutoCommit: ptr(DefaultAutoCommit), - PersistenceBehavior: ptr(DefaultPersistenceBahavior), DoltTransactionCommit: ptr(DefaultDoltTransactionCommit), }, UserConfig: UserYAMLConfig{ @@ -237,7 +225,6 @@ func defaultServerConfigYAML() *YAMLConfig { WriteTimeoutMillis: ptr(uint64(DefaultTimeout)), AllowCleartextPasswords: ptr(DefaultAllowCleartextPasswords), }, - PerformanceConfig: PerformanceYAMLConfig{QueryParallelism: ptr(DefaultQueryParallelism)}, DataDirStr: ptr(DefaultDataDir), CfgDirStr: ptr(filepath.Join(DefaultDataDir, DefaultCfgDir)), PrivilegeFile: ptr(filepath.Join(DefaultDataDir, DefaultCfgDir, DefaultPrivilegeFilePath)), diff --git a/go/libraries/doltcore/servercfg/testdata/minver_validation.txt b/go/libraries/doltcore/servercfg/testdata/minver_validation.txt index 7f85e181327..dcf4d8d1202 100644 --- a/go/libraries/doltcore/servercfg/testdata/minver_validation.txt +++ b/go/libraries/doltcore/servercfg/testdata/minver_validation.txt @@ -7,7 +7,7 @@ EncodeLoggedQuery *bool 0.0.0 encode_logged_query,omitempty BehaviorConfig servercfg.BehaviorYAMLConfig 0.0.0 behavior -ReadOnly *bool 0.0.0 read_only -AutoCommit *bool 0.0.0 autocommit --PersistenceBehavior *string 0.0.0 persistence_behavior +-PersistenceBehavior *string 0.0.0 persistence_behavior,omitempty -DisableClientMultiStatements *bool 0.0.0 disable_client_multi_statements -DoltTransactionCommit *bool 0.0.0 dolt_transaction_commit -EventSchedulerStatus *string 1.17.0 event_scheduler,omitempty @@ -25,8 +25,8 @@ ListenerConfig servercfg.ListenerYAMLConfig 0.0.0 listener -RequireSecureTransport *bool 0.0.0 require_secure_transport -AllowCleartextPasswords *bool 0.0.0 allow_cleartext_passwords -Socket *string 0.0.0 socket,omitempty -PerformanceConfig servercfg.PerformanceYAMLConfig 0.0.0 performance --QueryParallelism *int 0.0.0 query_parallelism +PerformanceConfig *servercfg.PerformanceYAMLConfig 0.0.0 performance,omitempty +-QueryParallelism *int 0.0.0 query_parallelism,omitempty DataDirStr *string 0.0.0 data_dir,omitempty CfgDirStr *string 0.0.0 cfg_dir,omitempty MetricsConfig servercfg.MetricsYAMLConfig 0.0.0 metrics diff --git a/go/libraries/doltcore/servercfg/yaml_config.go b/go/libraries/doltcore/servercfg/yaml_config.go index f665987841f..2511cde9575 100644 --- a/go/libraries/doltcore/servercfg/yaml_config.go +++ b/go/libraries/doltcore/servercfg/yaml_config.go @@ -52,8 +52,8 @@ func nillableIntPtr(n int) *int { type BehaviorYAMLConfig struct { ReadOnly *bool `yaml:"read_only"` AutoCommit *bool `yaml:"autocommit"` - // PersistenceBehavior regulates loading persisted system variable configuration. - PersistenceBehavior *string `yaml:"persistence_behavior"` + // PersistenceBehavior is unused, but still present to prevent breaking any YAML configs that still use it. + PersistenceBehavior *string `yaml:"persistence_behavior,omitempty"` // Disable processing CLIENT_MULTI_STATEMENTS support on the // sql server. Dolt's handling of CLIENT_MULTI_STATEMENTS is currently // broken. If a client advertises to support it (mysql cli client @@ -95,7 +95,8 @@ type ListenerYAMLConfig struct { // PerformanceYAMLConfig contains configuration parameters for performance tweaking type PerformanceYAMLConfig struct { - QueryParallelism *int `yaml:"query_parallelism"` + // QueryParallelism is deprecated but still present to prevent breaking YAML config that still uses it + QueryParallelism *int `yaml:"query_parallelism,omitempty"` } type MetricsYAMLConfig struct { @@ -124,20 +125,20 @@ type UserSessionVars struct { // YAMLConfig is a ServerConfig implementation which is read from a yaml file type YAMLConfig struct { - LogLevelStr *string `yaml:"log_level,omitempty"` - MaxQueryLenInLogs *int `yaml:"max_logged_query_len,omitempty"` - EncodeLoggedQuery *bool `yaml:"encode_logged_query,omitempty"` - BehaviorConfig BehaviorYAMLConfig `yaml:"behavior"` - UserConfig UserYAMLConfig `yaml:"user"` - ListenerConfig ListenerYAMLConfig `yaml:"listener"` - PerformanceConfig PerformanceYAMLConfig `yaml:"performance"` - DataDirStr *string `yaml:"data_dir,omitempty"` - CfgDirStr *string `yaml:"cfg_dir,omitempty"` - MetricsConfig MetricsYAMLConfig `yaml:"metrics"` - RemotesapiConfig RemotesapiYAMLConfig `yaml:"remotesapi"` - ClusterCfg *ClusterYAMLConfig `yaml:"cluster,omitempty"` - PrivilegeFile *string `yaml:"privilege_file,omitempty"` - BranchControlFile *string `yaml:"branch_control_file,omitempty"` + LogLevelStr *string `yaml:"log_level,omitempty"` + MaxQueryLenInLogs *int `yaml:"max_logged_query_len,omitempty"` + EncodeLoggedQuery *bool `yaml:"encode_logged_query,omitempty"` + BehaviorConfig BehaviorYAMLConfig `yaml:"behavior"` + UserConfig UserYAMLConfig `yaml:"user"` + ListenerConfig ListenerYAMLConfig `yaml:"listener"` + PerformanceConfig *PerformanceYAMLConfig `yaml:"performance,omitempty"` + DataDirStr *string `yaml:"data_dir,omitempty"` + CfgDirStr *string `yaml:"cfg_dir,omitempty"` + MetricsConfig MetricsYAMLConfig `yaml:"metrics"` + RemotesapiConfig RemotesapiYAMLConfig `yaml:"remotesapi"` + ClusterCfg *ClusterYAMLConfig `yaml:"cluster,omitempty"` + PrivilegeFile *string `yaml:"privilege_file,omitempty"` + BranchControlFile *string `yaml:"branch_control_file,omitempty"` // TODO: Rename to UserVars_ Vars []UserSessionVars `yaml:"user_session_vars"` SystemVars_ map[string]interface{} `yaml:"system_variables,omitempty" minver:"1.11.1"` @@ -175,37 +176,33 @@ func YamlConfigFromFile(fs filesys.Filesys, path string) (ServerConfig, error) { } func ServerConfigAsYAMLConfig(cfg ServerConfig) *YAMLConfig { - systemVars := map[string]interface{}(cfg.SystemVars()) + systemVars := cfg.SystemVars() return &YAMLConfig{ LogLevelStr: ptr(string(cfg.LogLevel())), MaxQueryLenInLogs: nillableIntPtr(cfg.MaxLoggedQueryLen()), EncodeLoggedQuery: nillableBoolPtr(cfg.ShouldEncodeLoggedQuery()), BehaviorConfig: BehaviorYAMLConfig{ - ptr(cfg.ReadOnly()), - ptr(cfg.AutoCommit()), - ptr(cfg.PersistenceBehavior()), - ptr(cfg.DisableClientMultiStatements()), - ptr(cfg.DoltTransactionCommit()), - ptr(cfg.EventSchedulerStatus()), + ReadOnly: ptr(cfg.ReadOnly()), + AutoCommit: ptr(cfg.AutoCommit()), + DisableClientMultiStatements: ptr(cfg.DisableClientMultiStatements()), + DoltTransactionCommit: ptr(cfg.DoltTransactionCommit()), + EventSchedulerStatus: ptr(cfg.EventSchedulerStatus()), }, UserConfig: UserYAMLConfig{ Name: ptr(cfg.User()), Password: ptr(cfg.Password()), }, ListenerConfig: ListenerYAMLConfig{ - ptr(cfg.Host()), - ptr(cfg.Port()), - ptr(cfg.MaxConnections()), - ptr(cfg.ReadTimeout()), - ptr(cfg.WriteTimeout()), - nillableStrPtr(cfg.TLSKey()), - nillableStrPtr(cfg.TLSCert()), - nillableBoolPtr(cfg.RequireSecureTransport()), - nillableBoolPtr(cfg.AllowCleartextPasswords()), - nillableStrPtr(cfg.Socket()), - }, - PerformanceConfig: PerformanceYAMLConfig{ - QueryParallelism: nillableIntPtr(cfg.QueryParallelism()), + HostStr: ptr(cfg.Host()), + PortNumber: ptr(cfg.Port()), + MaxConnections: ptr(cfg.MaxConnections()), + ReadTimeoutMillis: ptr(cfg.ReadTimeout()), + WriteTimeoutMillis: ptr(cfg.WriteTimeout()), + TLSKey: nillableStrPtr(cfg.TLSKey()), + TLSCert: nillableStrPtr(cfg.TLSCert()), + RequireSecureTransport: nillableBoolPtr(cfg.RequireSecureTransport()), + AllowCleartextPasswords: nillableBoolPtr(cfg.AllowCleartextPasswords()), + Socket: nillableStrPtr(cfg.Socket()), }, DataDirStr: ptr(cfg.DataDir()), CfgDirStr: ptr(cfg.CfgDir()), @@ -596,15 +593,6 @@ func (cfg YAMLConfig) AllowCleartextPasswords() bool { return *cfg.ListenerConfig.AllowCleartextPasswords } -// QueryParallelism returns the parallelism that should be used by the go-mysql-server analyzer -func (cfg YAMLConfig) QueryParallelism() int { - if cfg.PerformanceConfig.QueryParallelism == nil { - return DefaultQueryParallelism - } - - return *cfg.PerformanceConfig.QueryParallelism -} - // TLSKey returns a path to the servers PEM-encoded private TLS key. "" if there is none. func (cfg YAMLConfig) TLSKey() string { if cfg.ListenerConfig.TLSKey == nil { @@ -648,14 +636,6 @@ func (cfg YAMLConfig) ShouldEncodeLoggedQuery() bool { return *cfg.EncodeLoggedQuery } -// PersistenceBehavior is "load" if we include persisted system globals on server init -func (cfg YAMLConfig) PersistenceBehavior() string { - if cfg.BehaviorConfig.PersistenceBehavior == nil { - return LoadPerisistentGlobals - } - return *cfg.BehaviorConfig.PersistenceBehavior -} - // DataDir is the path to a directory to use as the data dir, both to create new databases and locate existing ones. func (cfg YAMLConfig) DataDir() string { if cfg.DataDirStr != nil { diff --git a/go/libraries/doltcore/servercfg/yaml_config_test.go b/go/libraries/doltcore/servercfg/yaml_config_test.go index 12d618fd9fe..6fa44ea211a 100644 --- a/go/libraries/doltcore/servercfg/yaml_config_test.go +++ b/go/libraries/doltcore/servercfg/yaml_config_test.go @@ -33,7 +33,6 @@ behavior: read_only: false autocommit: true dolt_transaction_commit: true - persistence_behavior: load disable_client_multi_statements: false event_scheduler: ON diff --git a/go/libraries/doltcore/sqle/alterschema.go b/go/libraries/doltcore/sqle/alterschema.go index 7cc95274cf1..7defa7011e4 100755 --- a/go/libraries/doltcore/sqle/alterschema.go +++ b/go/libraries/doltcore/sqle/alterschema.go @@ -28,15 +28,14 @@ import ( ) // renameTable renames a table with in a RootValue and returns the updated root. -func renameTable(ctx context.Context, root doltdb.RootValue, oldName, newName string) (doltdb.RootValue, error) { +func renameTable(ctx context.Context, root doltdb.RootValue, oldName, newName doltdb.TableName) (doltdb.RootValue, error) { if newName == oldName { return root, nil } else if root == nil { panic("invalid parameters") } - // TODO: schema name - return root.RenameTable(ctx, doltdb.TableName{Name: oldName}, doltdb.TableName{Name: newName}) + return root.RenameTable(ctx, oldName, newName) } // Nullable represents whether a column can have a null value. diff --git a/go/libraries/doltcore/sqle/alterschema_test.go b/go/libraries/doltcore/sqle/alterschema_test.go index b8e5ffd121c..656b633ac1d 100644 --- a/go/libraries/doltcore/sqle/alterschema_test.go +++ b/go/libraries/doltcore/sqle/alterschema_test.go @@ -96,7 +96,7 @@ func TestRenameTable(t *testing.T) { require.NoError(t, err) beforeSch := schemas[doltdb.TableName{Name: tt.oldName}] - updatedRoot, err := renameTable(ctx, root, tt.oldName, tt.newName) + updatedRoot, err := renameTable(ctx, root, doltdb.TableName{Name: tt.oldName}, doltdb.TableName{Name: tt.newName}) if len(tt.expectedErr) > 0 { assert.Error(t, err) assert.Contains(t, err.Error(), tt.expectedErr) diff --git a/go/libraries/doltcore/sqle/database.go b/go/libraries/doltcore/sqle/database.go index 61edb64dc29..390f2e0c384 100644 --- a/go/libraries/doltcore/sqle/database.go +++ b/go/libraries/doltcore/sqle/database.go @@ -1726,12 +1726,23 @@ func (db Database) RenameTable(ctx *sql.Context, oldName, newName string) error return ErrInvalidTableName.New(newName) } - if _, ok, _ := db.GetTableInsensitive(ctx, newName); ok { - return sql.ErrTableAlreadyExists.New(newName) + oldNameWithSchema, _, exists, err := resolve.Table(ctx, root, oldName) + if err != nil { + return err } - newRoot, err := renameTable(ctx, root, oldName, newName) + if !exists { + return sql.ErrTableNotFound.New(oldName) + } + + // TODO: we have no way to rename a table to a different schema, need to change the GMS interface for that + newNameWithSchema := doltdb.TableName{Schema: oldNameWithSchema.Schema, Name: newName} + _, exists, err = root.ResolveTableName(ctx, newNameWithSchema) + if exists { + return sql.ErrTableAlreadyExists.New(newName) + } + newRoot, err := renameTable(ctx, root, oldNameWithSchema, newNameWithSchema) if err != nil { return err } diff --git a/go/libraries/doltcore/sqle/dtables/constraint_violations.go b/go/libraries/doltcore/sqle/dtables/constraint_violations.go index 6796e5c78da..ec0f834e0cd 100644 --- a/go/libraries/doltcore/sqle/dtables/constraint_violations.go +++ b/go/libraries/doltcore/sqle/dtables/constraint_violations.go @@ -42,7 +42,7 @@ func newNomsCVTable(ctx *sql.Context, tblName doltdb.TableName, root doltdb.Root if err != nil { return nil, err } - cvSch, err := tbl.GetConstraintViolationsSchema(ctx) + cvSch, err := getConstraintViolationsSchema(ctx, tbl, tblName, root) if err != nil { return nil, err } diff --git a/go/libraries/doltcore/sqle/dtables/constraint_violations_prolly.go b/go/libraries/doltcore/sqle/dtables/constraint_violations_prolly.go index 8f70890b261..0ab8ccdf0ae 100644 --- a/go/libraries/doltcore/sqle/dtables/constraint_violations_prolly.go +++ b/go/libraries/doltcore/sqle/dtables/constraint_violations_prolly.go @@ -15,23 +15,81 @@ package dtables import ( + "context" "encoding/json" "github.com/dolthub/go-mysql-server/sql" + gmstypes "github.com/dolthub/go-mysql-server/sql/types" + "github.com/dolthub/vitess/go/sqltypes" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable" "github.com/dolthub/dolt/go/libraries/doltcore/merge" "github.com/dolthub/dolt/go/libraries/doltcore/schema" + "github.com/dolthub/dolt/go/libraries/doltcore/schema/typeinfo" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/index" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/sqlutil" "github.com/dolthub/dolt/go/store/hash" "github.com/dolthub/dolt/go/store/pool" "github.com/dolthub/dolt/go/store/prolly" "github.com/dolthub/dolt/go/store/prolly/tree" + "github.com/dolthub/dolt/go/store/types" "github.com/dolthub/dolt/go/store/val" ) +func getDoltConstraintViolationsBaseSqlSchema() sql.Schema { + return []*sql.Column{ + {Name: "from_root_ish", Type: gmstypes.MustCreateStringWithDefaults(sqltypes.VarChar, 1023), PrimaryKey: false, Nullable: true}, + {Name: "violation_type", Type: gmstypes.MustCreateEnumType([]string{"foreign key", "unique index", "check constraint", "not null"}, sql.Collation_Default), PrimaryKey: true}, + } +} + +// GetDoltConstraintViolationsBaseSqlSchema returns the base schema for the dolt_constraint_violations_* system table. +// This is used by Doltgres to update the dolt_constraint_violations_* schema using Doltgres types. +var GetDoltConstraintViolationsBaseSqlSchema = getDoltConstraintViolationsBaseSqlSchema + +// getConstraintViolationsSchema returns a table's dolt_constraint_violations system table schema. +func getConstraintViolationsSchema(ctx context.Context, t *doltdb.Table, tn doltdb.TableName, root doltdb.RootValue) (schema.Schema, error) { + sch, err := t.GetSchema(ctx) + if err != nil { + return nil, err + } + + baseSch := sql.NewPrimaryKeySchema(GetDoltConstraintViolationsBaseSqlSchema()) + baseDoltSch, err := sqlutil.ToDoltSchema(ctx, root, tn, baseSch, root, sql.Collation_Default) + if err != nil { + return nil, err + } + baseColColl := baseDoltSch.GetAllCols() + baseCols := baseColColl.GetColumns() + + schSize := sch.GetAllCols().Size() + if schema.IsKeyless(sch) { + // Keyless tables have an additional dolt_row_hash column + schSize += 1 + } + + cols := make([]schema.Column, 0, baseColColl.Size()+schSize) + cols = append(cols, baseCols[0:2]...) + infoCol, err := schema.NewColumnWithTypeInfo("violation_info", schema.DoltConstraintViolationsInfoTag, typeinfo.JSONType, false, "", false, "") + if err != nil { + return nil, err + } + + if schema.IsKeyless(sch) { + // If this is a keyless table, we need to add a new column for the keyless table's generated row hash. + // We need to add this internal row hash value, in order to guarantee a unique primary key in the + // constraint violations table. + cols = append(cols, schema.NewColumn("dolt_row_hash", 0, types.BlobKind, true)) + } else { + cols = append(cols, sch.GetPKCols().GetColumns()...) + } + cols = append(cols, sch.GetNonPKCols().GetColumns()...) + cols = append(cols, infoCol) + + return schema.NewSchema(schema.NewColCollection(cols...), nil, schema.Collation_Default, nil, nil) +} + func newProllyCVTable(ctx *sql.Context, tblName doltdb.TableName, root doltdb.RootValue, rs RootSetter) (sql.Table, error) { var tbl *doltdb.Table var err error @@ -39,7 +97,7 @@ func newProllyCVTable(ctx *sql.Context, tblName doltdb.TableName, root doltdb.Ro if err != nil { return nil, err } - cvSch, err := tbl.GetConstraintViolationsSchema(ctx) + cvSch, err := getConstraintViolationsSchema(ctx, tbl, tblName, root) if err != nil { return nil, err } @@ -278,7 +336,7 @@ func (d *prollyCVDeleter) Delete(ctx *sql.Context, r sql.Row) error { d.kb.PutCommitAddr(d.kd.Count()-2, h) // Finally the artifact type - artType := merge.UnmapCVType(merge.CvType(r[1].(uint64))) + artType := merge.UnmapCVType(r[1]) d.kb.PutUint8(d.kd.Count()-1, uint8(artType)) key := d.kb.Build(d.pool) diff --git a/go/libraries/doltcore/sqle/dtables/merge_status_table.go b/go/libraries/doltcore/sqle/dtables/merge_status_table.go index c0d1bd9982f..32e99b931e7 100644 --- a/go/libraries/doltcore/sqle/dtables/merge_status_table.go +++ b/go/libraries/doltcore/sqle/dtables/merge_status_table.go @@ -41,16 +41,24 @@ func (mst MergeStatusTable) String() string { return mst.tableName } -func (mst MergeStatusTable) Schema() sql.Schema { +func getDoltMergeStatusSchema(dbName, tableName string) sql.Schema { return []*sql.Column{ - {Name: "is_merging", Type: types.Boolean, Source: mst.tableName, PrimaryKey: false, Nullable: false, DatabaseSource: mst.dbName}, - {Name: "source", Type: types.Text, Source: mst.tableName, PrimaryKey: false, Nullable: true, DatabaseSource: mst.dbName}, - {Name: "source_commit", Type: types.Text, Source: mst.tableName, PrimaryKey: false, Nullable: true, DatabaseSource: mst.dbName}, - {Name: "target", Type: types.Text, Source: mst.tableName, PrimaryKey: false, Nullable: true, DatabaseSource: mst.dbName}, - {Name: "unmerged_tables", Type: types.Text, Source: mst.tableName, PrimaryKey: false, Nullable: true, DatabaseSource: mst.dbName}, + {Name: "is_merging", Type: types.Boolean, Source: tableName, PrimaryKey: false, Nullable: false, DatabaseSource: dbName}, + {Name: "source", Type: types.Text, Source: tableName, PrimaryKey: false, Nullable: true, DatabaseSource: dbName}, + {Name: "source_commit", Type: types.Text, Source: tableName, PrimaryKey: false, Nullable: true, DatabaseSource: dbName}, + {Name: "target", Type: types.Text, Source: tableName, PrimaryKey: false, Nullable: true, DatabaseSource: dbName}, + {Name: "unmerged_tables", Type: types.Text, Source: tableName, PrimaryKey: false, Nullable: true, DatabaseSource: dbName}, } } +// GetDoltMergeStatusSchema returns the schema of the dolt_merge_status system table. This is used +// by Doltgres to update the dolt_merge_status schema using Doltgres types. +var GetDoltMergeStatusSchema = getDoltMergeStatusSchema + +func (mst MergeStatusTable) Schema() sql.Schema { + return GetDoltMergeStatusSchema(mst.dbName, mst.tableName) +} + func (mst MergeStatusTable) Collation() sql.CollationID { return sql.Collation_Default } diff --git a/go/libraries/doltcore/sqle/dtables/status_table.go b/go/libraries/doltcore/sqle/dtables/status_table.go index 8550bdfb2b3..e16fbacd5c1 100644 --- a/go/libraries/doltcore/sqle/dtables/status_table.go +++ b/go/libraries/doltcore/sqle/dtables/status_table.go @@ -61,14 +61,22 @@ func (st StatusTable) String() string { return st.tableName } -func (st StatusTable) Schema() sql.Schema { +func getDoltStatusSchema(tableName string) sql.Schema { return []*sql.Column{ - {Name: "table_name", Type: types.Text, Source: st.tableName, PrimaryKey: true, Nullable: false}, - {Name: "staged", Type: types.Boolean, Source: st.tableName, PrimaryKey: true, Nullable: false}, - {Name: "status", Type: types.Text, Source: st.tableName, PrimaryKey: true, Nullable: false}, + {Name: "table_name", Type: types.Text, Source: tableName, PrimaryKey: true, Nullable: false}, + {Name: "staged", Type: types.Boolean, Source: tableName, PrimaryKey: true, Nullable: false}, + {Name: "status", Type: types.Text, Source: tableName, PrimaryKey: true, Nullable: false}, } } +// GetDoltStatusSchema returns the schema of the dolt_status system table. This is used +// by Doltgres to update the dolt_status schema using Doltgres types. +var GetDoltStatusSchema = getDoltStatusSchema + +func (st StatusTable) Schema() sql.Schema { + return GetDoltStatusSchema(st.tableName) +} + func (st StatusTable) Collation() sql.CollationID { return sql.Collation_Default } diff --git a/go/libraries/doltcore/sqle/dtables/workspace_table.go b/go/libraries/doltcore/sqle/dtables/workspace_table.go index 287a2c4ad65..7a409356e4e 100644 --- a/go/libraries/doltcore/sqle/dtables/workspace_table.go +++ b/go/libraries/doltcore/sqle/dtables/workspace_table.go @@ -409,6 +409,9 @@ func NewWorkspaceTable(ctx *sql.Context, workspaceTableName string, tableName do sch := sql.NewPrimaryKeySchema(GetDoltWorkspaceBaseSqlSchema()) baseDoltSch, err := sqlutil.ToDoltSchema(ctx, head, tableName, sch, head, sql.Collation_Default) + if err != nil { + return nil, err + } totalSch, err := workspaceSchema(fromSch, toSch, baseDoltSch) if err != nil { diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_engine_tests.go b/go/libraries/doltcore/sqle/enginetest/dolt_engine_tests.go index b82a05ad88e..e0f25e34a57 100755 --- a/go/libraries/doltcore/sqle/enginetest/dolt_engine_tests.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_engine_tests.go @@ -683,6 +683,7 @@ func RunDoltRevisionDbScriptsTest(t *testing.T, h DoltEnginetestHarness) { require.NoError(t, err) defer e.Close() ctx := h.NewContext() + ctx.SetCurrentDatabase("mydb") setupScripts := []setup.SetupScript{ {"create table t01 (pk int primary key, c1 int)"}, diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_harness.go b/go/libraries/doltcore/sqle/enginetest/dolt_harness.go index 22d4c8074c0..e8539db4da3 100644 --- a/go/libraries/doltcore/sqle/enginetest/dolt_harness.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_harness.go @@ -270,6 +270,10 @@ func (d *DoltHarness) NewEngine(t *testing.T) (enginetest.QueryEngine, error) { if err != nil { return nil, err } + + // Get a fresh session after running setup scripts, since some setup scripts can change the session state + d.session, err = dsess.NewDoltSession(enginetest.NewBaseSession(), d.provider, d.multiRepoEnv.Config(), d.branchControl, d.statsPro, writer.NewWriteSession) + require.NoError(t, err) } if d.configureStats { @@ -304,16 +308,14 @@ func (d *DoltHarness) NewEngine(t *testing.T) (enginetest.QueryEngine, error) { d.engine.Analyzer.Catalog.MySQLDb.AddRootAccount() d.engine.Analyzer.Catalog.StatsProvider = statspro.NewProvider(d.provider.(*sqle.DoltDatabaseProvider), statsnoms.NewNomsStatsFactory(d.multiRepoEnv.RemoteDialProvider())) - // Get a fresh session if we are reusing the engine - if !initializeEngine { - var err error - d.session, err = dsess.NewDoltSession(enginetest.NewBaseSession(), d.provider, d.multiRepoEnv.Config(), d.branchControl, d.statsPro, writer.NewWriteSession) - require.NoError(t, err) - } - + var err error ctx := enginetest.NewContext(d) e, err := enginetest.RunSetupScripts(ctx, d.engine, d.resetScripts(), d.SupportsNativeIndexCreation()) + // Get a fresh session after running setup scripts, since some setup scripts can change the session state + d.session, err = dsess.NewDoltSession(enginetest.NewBaseSession(), d.provider, d.multiRepoEnv.Config(), d.branchControl, d.statsPro, writer.NewWriteSession) + require.NoError(t, err) + return e, err } diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_queries.go b/go/libraries/doltcore/sqle/enginetest/dolt_queries.go index 6c6102c2227..949e220be09 100644 --- a/go/libraries/doltcore/sqle/enginetest/dolt_queries.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_queries.go @@ -2076,7 +2076,7 @@ var HistorySystemTableScriptTests = []queries.ScriptTest{ }, }, { - Query: "explain select pk, c from dolt_history_t1 where pk = 3", + Query: "explain plan select pk, c from dolt_history_t1 where pk = 3", Expected: []sql.Row{ {"Filter"}, {" ├─ (dolt_history_t1.pk = 3)"}, @@ -2087,7 +2087,7 @@ var HistorySystemTableScriptTests = []queries.ScriptTest{ }, }, { - Query: "explain select pk, c from dolt_history_t1 where pk = 3 and committer = 'someguy'", + Query: "explain plan select pk, c from dolt_history_t1 where pk = 3 and committer = 'someguy'", Expected: []sql.Row{ {"Project"}, {" ├─ columns: [dolt_history_t1.pk, dolt_history_t1.c]"}, @@ -2151,7 +2151,7 @@ var HistorySystemTableScriptTests = []queries.ScriptTest{ }, }, { - Query: "explain select pk, c from dolt_history_t1 where c = 4", + Query: "explain plan select pk, c from dolt_history_t1 where c = 4", Expected: []sql.Row{ {"Filter"}, {" ├─ (dolt_history_t1.c = 4)"}, @@ -2162,7 +2162,7 @@ var HistorySystemTableScriptTests = []queries.ScriptTest{ }, }, { - Query: "explain select pk, c from dolt_history_t1 where c = 10 and committer = 'someguy'", + Query: "explain plan select pk, c from dolt_history_t1 where c = 10 and committer = 'someguy'", Expected: []sql.Row{ {"Project"}, {" ├─ columns: [dolt_history_t1.pk, dolt_history_t1.c]"}, diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_queries_merge.go b/go/libraries/doltcore/sqle/enginetest/dolt_queries_merge.go index e615f7691ff..c72472b3cfc 100644 --- a/go/libraries/doltcore/sqle/enginetest/dolt_queries_merge.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_queries_merge.go @@ -1661,7 +1661,7 @@ var MergeScripts = []queries.ScriptTest{ Expected: []sql.Row{{doltCommit, 1, 0, "merge successful"}}, }, { - Query: "INSERT INTO t VALUES (NULL,5),(6,6),(NULL,7);", + Query: "INSERT INTO t(c0) VALUES (5),(6),(7);", Expected: []sql.Row{{types.OkResult{RowsAffected: 3, InsertID: 5}}}, }, { @@ -1698,7 +1698,7 @@ var MergeScripts = []queries.ScriptTest{ Expected: []sql.Row{{doltCommit, 0, 0, "merge successful"}}, }, { - Query: "INSERT INTO t VALUES (NULL,6),(7,7),(NULL,8);", + Query: "INSERT INTO t(c0) VALUES (6),(7),(8);", Expected: []sql.Row{{types.OkResult{RowsAffected: 3, InsertID: 6}}}, }, { diff --git a/go/libraries/doltcore/sqle/read_replica_database.go b/go/libraries/doltcore/sqle/read_replica_database.go index 22323d94ea2..fba8ca14cd4 100644 --- a/go/libraries/doltcore/sqle/read_replica_database.go +++ b/go/libraries/doltcore/sqle/read_replica_database.go @@ -275,7 +275,7 @@ type pullBehavior bool const pullBehaviorFastForward pullBehavior = false const pullBehaviorForcePull pullBehavior = true -// pullBranches pulls the remote branches named and returns the map of their hashes keyed by branch path. +// pullBranches pulls the named remote branches and tags and returns the map of their hashes keyed by ref ID. func pullBranches( ctx *sql.Context, rrd ReadReplicaDatabase, @@ -283,17 +283,17 @@ func pullBranches( localRefs []doltdb.RefWithHash, behavior pullBehavior, ) (map[string]doltdb.RefWithHash, error) { - localRefsByPath := make(map[string]doltdb.RefWithHash) - remoteRefsByPath := make(map[string]doltdb.RefWithHash) + localRefsById := make(map[string]doltdb.RefWithHash) + remoteRefsById := make(map[string]doltdb.RefWithHash) remoteHashes := make([]hash.Hash, len(remoteRefs)) for i, b := range remoteRefs { - remoteRefsByPath[b.Ref.GetPath()] = b + remoteRefsById[b.Ref.String()] = b remoteHashes[i] = b.Hash } for _, b := range localRefs { - localRefsByPath[b.Ref.GetPath()] = b + localRefsById[b.Ref.String()] = b } // XXX: Our view of which remote branches to pull and what to set the @@ -311,7 +311,7 @@ func pullBranches( REFS: // every successful pass through the loop below must end with `continue REFS` to get out of the retry loop for _, remoteRef := range remoteRefs { trackingRef := ref.NewRemoteRef(rrd.remote.Name, remoteRef.Ref.GetPath()) - localRef, localRefExists := localRefsByPath[remoteRef.Ref.GetPath()] + localRef, localRefExists := localRefsById[remoteRef.Ref.String()] // loop on optimistic lock failures OPTIMISTIC_RETRY: @@ -378,7 +378,7 @@ func pullBranches( return nil, err } - return remoteRefsByPath, nil + return remoteRefsById, nil } // expandWildcardBranchPattern evaluates |pattern| and returns a list of branch names from the source database that diff --git a/go/libraries/doltcore/table/typed/parquet/reader.go b/go/libraries/doltcore/table/typed/parquet/reader.go index 81a66f7915e..d883fe65707 100644 --- a/go/libraries/doltcore/table/typed/parquet/reader.go +++ b/go/libraries/doltcore/table/typed/parquet/reader.go @@ -68,6 +68,8 @@ func NewParquetReader(vrw types.ValueReadWriter, fr source.ParquetFile, sche sch return nil, err } + rootName := pr.SchemaHandler.GetRootExName() + columns := sche.GetAllCols().GetColumns() num := pr.GetNumRows() @@ -75,7 +77,7 @@ func NewParquetReader(vrw types.ValueReadWriter, fr source.ParquetFile, sche sch data := make(map[string][]interface{}) var colName []string for _, col := range columns { - colData, _, _, cErr := pr.ReadColumnByPath(common.ReformPathStr(fmt.Sprintf("parquet_go_root.%s", col.Name)), num) + colData, _, _, cErr := pr.ReadColumnByPath(common.ReformPathStr(fmt.Sprintf("%s.%s", rootName, col.Name)), num) if cErr != nil { return nil, fmt.Errorf("cannot read column: %s", cErr.Error()) } diff --git a/go/performance/continuous_integration/SysbenchDockerfile b/go/performance/continuous_integration/SysbenchDockerfile index a5acf7e8710..c625469b823 100644 --- a/go/performance/continuous_integration/SysbenchDockerfile +++ b/go/performance/continuous_integration/SysbenchDockerfile @@ -1,5 +1,5 @@ # syntax=docker/dockerfile:1.3-labs -FROM --platform=linux/amd64 golang:1.22-alpine as gobin +FROM --platform=linux/amd64 golang:1.23.3-alpine as gobin FROM --platform=linux/amd64 ubuntu:22.04 COPY --from=gobin /usr/local/go/ /go/ diff --git a/integration-tests/DataDumpLoadDockerfile b/integration-tests/DataDumpLoadDockerfile index 2ce2c9776ac..fa069bcff73 100644 --- a/integration-tests/DataDumpLoadDockerfile +++ b/integration-tests/DataDumpLoadDockerfile @@ -20,7 +20,7 @@ RUN apt update -y && \ # install go WORKDIR /root -ENV GO_VERSION=1.22.1 +ENV GO_VERSION=1.23.3 ENV GOPATH=$HOME/go ENV PATH=$PATH:$GOPATH/bin ENV PATH=$PATH:$GOPATH/bin:/usr/local/go/bin diff --git a/integration-tests/MySQLDockerfile b/integration-tests/MySQLDockerfile index 89f4b609873..a6e3efe0232 100644 --- a/integration-tests/MySQLDockerfile +++ b/integration-tests/MySQLDockerfile @@ -59,7 +59,7 @@ RUN apt update -y && \ # install go WORKDIR /root -ENV GO_VERSION=1.22.1 +ENV GO_VERSION=1.23.3 ENV GOPATH=$HOME/go ENV PATH=$PATH:$GOPATH/bin ENV PATH=$PATH:$GOPATH/bin:/usr/local/go/bin diff --git a/integration-tests/ORMDockerfile b/integration-tests/ORMDockerfile index ba0851db303..b389ee7e063 100644 --- a/integration-tests/ORMDockerfile +++ b/integration-tests/ORMDockerfile @@ -25,7 +25,7 @@ RUN apt update -y && \ # install go WORKDIR /root -ENV GO_VERSION=1.22.1 +ENV GO_VERSION=1.23.3 ENV GOPATH=$HOME/go ENV PATH=$PATH:$GOPATH/bin ENV PATH=$PATH:$GOPATH/bin:/usr/local/go/bin diff --git a/integration-tests/bats/branch-control.bats b/integration-tests/bats/branch-control.bats index c38a5d93a0b..bb75b8f26c0 100644 --- a/integration-tests/bats/branch-control.bats +++ b/integration-tests/bats/branch-control.bats @@ -333,6 +333,11 @@ SQL @test "branch-control: Issue #8623" { # https://github.com/dolthub/dolt/issues/8623 dolt sql <>z4cV2M@tWzWK#nh*KXFfB7SWDq$^fW zGw(fS-kVuHy)q2Az(tHb*C31lh5)dUO6llZ8&%)kV52o2koXyOhu)fY`+*05M`H*) zD}OBb9tlY}K_MQIF#>X-vE%)_qT#r1r3rbU}@*w5e%K zx1u}atBqrtR;s$Ei?`!dJX<>9fAo@vp59LoEO)|u?xlU`Of)U`t*WDp&67OkB^SJ` f1fR>SNm Date: Mon, 9 Dec 2024 15:20:43 -0800 Subject: [PATCH 07/31] Don't use removed persistence_behavior config option in tests --- integration-tests/bats/sql-server-config-file-generation.bats | 3 --- 1 file changed, 3 deletions(-) diff --git a/integration-tests/bats/sql-server-config-file-generation.bats b/integration-tests/bats/sql-server-config-file-generation.bats index 339ce763ada..06c7fe90272 100644 --- a/integration-tests/bats/sql-server-config-file-generation.bats +++ b/integration-tests/bats/sql-server-config-file-generation.bats @@ -95,7 +95,6 @@ EOF start_sql_server_with_args \ --readonly \ --no-auto-commit \ - --persistence-behavior ignore \ --max-connections 77 \ --timeout 7777777 \ --allow-cleartext-passwords true \ @@ -106,7 +105,6 @@ EOF [ $status -eq 0 ] [[ "$output" =~ "read_only: true" ]] || false [[ "$output" =~ "autocommit: false" ]] || false - [[ "$output" =~ "persistence_behavior: ignore" ]] || false [[ "$output" =~ "max_connections: 77" ]] || false [[ "$output" =~ "read_timeout_millis: 7777777" ]] || false [[ "$output" =~ "write_timeout_millis: 7777777" ]] || false @@ -131,7 +129,6 @@ EOF # default (not set by args) [[ "$output" =~ "read_only: false" ]] || false [[ "$output" =~ "autocommit: true" ]] || false - [[ "$output" =~ "persistence_behavior: load" ]] || false [[ "$output" =~ "allow_cleartext_passwords: false" ]] || false } From 6f8911524b1af36ce934fbad00fd038caf400482 Mon Sep 17 00:00:00 2001 From: milogreg Date: Mon, 9 Dec 2024 17:40:29 -0800 Subject: [PATCH 08/31] Make default filler not use reflection --- .../doltcore/servercfg/yaml_config.go | 83 +++++++++++++------ 1 file changed, 58 insertions(+), 25 deletions(-) diff --git a/go/libraries/doltcore/servercfg/yaml_config.go b/go/libraries/doltcore/servercfg/yaml_config.go index 2511cde9575..04ca882b7d4 100644 --- a/go/libraries/doltcore/servercfg/yaml_config.go +++ b/go/libraries/doltcore/servercfg/yaml_config.go @@ -254,8 +254,7 @@ func (cfg YAMLConfig) String() string { // If an empty field has a default value, the default will be used. // If an empty field has no default value, a commented-out placeholder will be used. func (cfg YAMLConfig) VerboseString() string { - withDefaults := cfg - withDefaults.fillDefaults() + withDefaults := cfg.withDefaultsFilledIn() formatted := formattedYAMLMarshal(removeOmitemptyTags(withDefaults)) formatted = commentNullYAMLValues(formatted) @@ -263,36 +262,70 @@ func (cfg YAMLConfig) VerboseString() string { return formatted } -// Assumes YAMLConfig has no circular references. -func (cfg *YAMLConfig) fillDefaults() { +func (cfg YAMLConfig) withDefaultsFilledIn() YAMLConfig { defaults := defaultServerConfigYAML() - recursiveFillDefaults(reflect.ValueOf(cfg), reflect.ValueOf(defaults)) -} + withDefaults := cfg -func recursiveFillDefaults(cfgValue, defaultsValue reflect.Value) { - cfgValue = cfgValue.Elem() - defaultsValue = defaultsValue.Elem() + if withDefaults.LogLevelStr == nil { + withDefaults.LogLevelStr = defaults.LogLevelStr + } + if withDefaults.MaxQueryLenInLogs == nil { + withDefaults.MaxQueryLenInLogs = defaults.MaxQueryLenInLogs + } + if withDefaults.EncodeLoggedQuery == nil { + withDefaults.EncodeLoggedQuery = defaults.EncodeLoggedQuery + } - if cfgValue.Kind() != reflect.Struct { - return + if withDefaults.BehaviorConfig.ReadOnly == nil { + withDefaults.BehaviorConfig.ReadOnly = defaults.BehaviorConfig.ReadOnly + } + if withDefaults.BehaviorConfig.AutoCommit == nil { + withDefaults.BehaviorConfig.AutoCommit = defaults.BehaviorConfig.AutoCommit + } + if withDefaults.BehaviorConfig.DoltTransactionCommit == nil { + withDefaults.BehaviorConfig.DoltTransactionCommit = defaults.BehaviorConfig.DoltTransactionCommit } - for i := 0; i < cfgValue.NumField(); i++ { - field := cfgValue.Field(i) - defaultField := defaultsValue.Field(i) + if withDefaults.UserConfig.Name == nil { + withDefaults.UserConfig.Name = defaults.UserConfig.Name + } + if withDefaults.UserConfig.Password == nil { + withDefaults.UserConfig.Password = defaults.UserConfig.Password + } - if field.Kind() == reflect.Pointer { - if !defaultField.IsNil() { - if field.IsNil() { - field.Set(defaultField) - } else { - recursiveFillDefaults(field, defaultField) - } - } - } else { - recursiveFillDefaults(field.Addr(), defaultField.Addr()) - } + if withDefaults.ListenerConfig.HostStr == nil { + withDefaults.ListenerConfig.HostStr = defaults.ListenerConfig.HostStr } + if withDefaults.ListenerConfig.PortNumber == nil { + withDefaults.ListenerConfig.PortNumber = defaults.ListenerConfig.PortNumber + } + if withDefaults.ListenerConfig.MaxConnections == nil { + withDefaults.ListenerConfig.MaxConnections = defaults.ListenerConfig.MaxConnections + } + if withDefaults.ListenerConfig.ReadTimeoutMillis == nil { + withDefaults.ListenerConfig.ReadTimeoutMillis = defaults.ListenerConfig.ReadTimeoutMillis + } + if withDefaults.ListenerConfig.WriteTimeoutMillis == nil { + withDefaults.ListenerConfig.WriteTimeoutMillis = defaults.ListenerConfig.WriteTimeoutMillis + } + if withDefaults.ListenerConfig.AllowCleartextPasswords == nil { + withDefaults.ListenerConfig.AllowCleartextPasswords = defaults.ListenerConfig.AllowCleartextPasswords + } + + if withDefaults.DataDirStr == nil { + withDefaults.DataDirStr = defaults.DataDirStr + } + if withDefaults.CfgDirStr == nil { + withDefaults.CfgDirStr = defaults.CfgDirStr + } + if withDefaults.PrivilegeFile == nil { + withDefaults.PrivilegeFile = defaults.PrivilegeFile + } + if withDefaults.BranchControlFile == nil { + withDefaults.BranchControlFile = defaults.BranchControlFile + } + + return withDefaults } // Assumes 'in' has no circular references. From 2654f847a9f68cca3dadcaf937eb427f9f291b5e Mon Sep 17 00:00:00 2001 From: milogreg Date: Mon, 9 Dec 2024 17:42:54 -0800 Subject: [PATCH 09/31] Remove circular reference test --- .../doltcore/servercfg/yaml_config_test.go | 34 +------------------ 1 file changed, 1 insertion(+), 33 deletions(-) diff --git a/go/libraries/doltcore/servercfg/yaml_config_test.go b/go/libraries/doltcore/servercfg/yaml_config_test.go index 6fa44ea211a..9a7e2838ce2 100644 --- a/go/libraries/doltcore/servercfg/yaml_config_test.go +++ b/go/libraries/doltcore/servercfg/yaml_config_test.go @@ -15,7 +15,6 @@ package servercfg import ( - "reflect" "testing" "github.com/stretchr/testify/assert" @@ -414,35 +413,4 @@ listener: require.NoError(t, err) err = ValidateConfig(cfg) assert.Error(t, err) -} - -// Tests if a circular reference exists in YAMLConfig. -// -// This test can be safely removed if: -// - YAMLConfig.VerboseString is updated to properly handle circular references. -// - yaml.Marshal is checked to make sure it properly handles circular references. -func TestNoCircularReferenceInYAMLConfig(t *testing.T) { - assert.False(t, hasCircularReference(reflect.TypeOf(YAMLConfig{}), map[reflect.Type]bool{})) -} - -// Only checks for struct circular references, not slice, map, etc. -func hasCircularReference(t reflect.Type, visited map[reflect.Type]bool) bool { - if t.Kind() == reflect.Ptr { - t = t.Elem() - } - if t.Kind() != reflect.Struct { - return false - } - if visited[t] { - return true - } - visited[t] = true - for i := 0; i < t.NumField(); i++ { - field := t.Field(i) - if hasCircularReference(field.Type, visited) { - return true - } - } - visited[t] = false - return false -} +} \ No newline at end of file From 211a52e187d677171e99ff455e1db47a0f38d55e Mon Sep 17 00:00:00 2001 From: milogreg Date: Mon, 9 Dec 2024 18:01:56 -0800 Subject: [PATCH 10/31] Add test for commentNullYAMLValues --- .../doltcore/servercfg/yaml_config.go | 2 ++ .../doltcore/servercfg/yaml_config_test.go | 32 ++++++++++++++++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/go/libraries/doltcore/servercfg/yaml_config.go b/go/libraries/doltcore/servercfg/yaml_config.go index 04ca882b7d4..c2f9d150f3c 100644 --- a/go/libraries/doltcore/servercfg/yaml_config.go +++ b/go/libraries/doltcore/servercfg/yaml_config.go @@ -416,6 +416,8 @@ func formattedYAMLMarshal(toMarshal any) string { return result } +// Assumes no non-null fields will be set to unquoted strings ending in null. +// For example, `field: 123-null` will be falsely commented but `field: '123-null'` is fine. func commentNullYAMLValues(needsComments string) string { lines := strings.Split(needsComments, "\n") for i := 0; i < len(lines); i++ { diff --git a/go/libraries/doltcore/servercfg/yaml_config_test.go b/go/libraries/doltcore/servercfg/yaml_config_test.go index 9a7e2838ce2..a9577f376ab 100644 --- a/go/libraries/doltcore/servercfg/yaml_config_test.go +++ b/go/libraries/doltcore/servercfg/yaml_config_test.go @@ -413,4 +413,34 @@ listener: require.NoError(t, err) err = ValidateConfig(cfg) assert.Error(t, err) -} \ No newline at end of file +} + +func TestCommentNullYAMLValues(t *testing.T) { + toComment := ` +value1: value +value2: null +null: value +nest1: + value1: null + value2: value + nest2: + value1: "null" + nest3: + with_many_spaces: null +` + + withComments := ` +value1: value +# value2: null +null: value +nest1: + # value1: null + value2: value + nest2: + value1: "null" + nest3: + # with_many_spaces: null +` + + assert.Equal(t, withComments, commentNullYAMLValues(toComment)) +} From bc409a6570774a70aa127a39f39b78187f296dd9 Mon Sep 17 00:00:00 2001 From: milogreg Date: Mon, 9 Dec 2024 18:04:58 -0800 Subject: [PATCH 11/31] Remove CLI warning for when directory named config.yaml exists in database directory --- go/cmd/dolt/commands/sqlserver/sqlserver.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/go/cmd/dolt/commands/sqlserver/sqlserver.go b/go/cmd/dolt/commands/sqlserver/sqlserver.go index 32cce773b76..6b138ff06e5 100644 --- a/go/cmd/dolt/commands/sqlserver/sqlserver.go +++ b/go/cmd/dolt/commands/sqlserver/sqlserver.go @@ -455,12 +455,8 @@ func generateYamlConfigIfNone( } path := filepath.Join(serverConfig.DataDir(), yamlConfigName) - exists, isDir := dEnv.FS.Exists(path) + exists, _ := dEnv.FS.Exists(path) if exists { - if isDir { - cli.PrintErrf("Couldn't generate YAML config at %s: directory with same name already exists", path) - } - return nil } From b31647378b6d56a6f7c0aca7711bcae67c4dbc22 Mon Sep 17 00:00:00 2001 From: milogreg Date: Mon, 9 Dec 2024 18:31:46 -0800 Subject: [PATCH 12/31] Add additional doc comments for functions --- go/cmd/dolt/commands/sqlserver/sqlserver.go | 5 +++++ go/libraries/doltcore/servercfg/yaml_config.go | 7 ++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/go/cmd/dolt/commands/sqlserver/sqlserver.go b/go/cmd/dolt/commands/sqlserver/sqlserver.go index 6b138ff06e5..fa1de0232e6 100644 --- a/go/cmd/dolt/commands/sqlserver/sqlserver.go +++ b/go/cmd/dolt/commands/sqlserver/sqlserver.go @@ -437,6 +437,10 @@ func setupDoltConfig(dEnv *env.DoltEnv, apr *argparser.ArgParseResults, config s return nil } +// generateYamlConfigIfNone creates a YAML config file in the database directory if one is not specified in the args +// and one doesn't already exist in the database directory. The fields of the generated YAML config file are set +// using serverConfig if serverConfig specifies a value for the field, otherwise the field is set to a default value +// or is replaced with a commented-out placeholder. func generateYamlConfigIfNone( ap *argparser.ArgParser, help cli.UsagePrinter, @@ -469,6 +473,7 @@ func generateYamlConfigIfNone( return nil } +// argsSpecifyServerConfigFile returns true if the args specify a config file, false otherwise. func argsSpecifyServerConfigFile(ap *argparser.ArgParser, help cli.UsagePrinter, args []string) (bool, error) { apr := cli.ParseArgsOrDie(ap, args, help) if err := validateSqlServerArgs(apr); err != nil { diff --git a/go/libraries/doltcore/servercfg/yaml_config.go b/go/libraries/doltcore/servercfg/yaml_config.go index c2f9d150f3c..70b8d7d92d7 100644 --- a/go/libraries/doltcore/servercfg/yaml_config.go +++ b/go/libraries/doltcore/servercfg/yaml_config.go @@ -250,7 +250,7 @@ func (cfg YAMLConfig) String() string { return formattedYAMLMarshal(cfg) } -// Behaves like String, but includes empty fields instead of omitting them. +// VerboseString behaves like String, but includes empty fields instead of omitting them. // If an empty field has a default value, the default will be used. // If an empty field has no default value, a commented-out placeholder will be used. func (cfg YAMLConfig) VerboseString() string { @@ -262,6 +262,7 @@ func (cfg YAMLConfig) VerboseString() string { return formatted } +// withDefaultsFilledIn returns the config with default values in place of nil values. func (cfg YAMLConfig) withDefaultsFilledIn() YAMLConfig { defaults := defaultServerConfigYAML() withDefaults := cfg @@ -385,6 +386,8 @@ func deepConvert(val reflect.Value, typ reflect.Type) reflect.Value { } } +// formattedYAMLMarshal returns the same result as yaml.Marshal, +// but with top-level fields separated by an additional newline. func formattedYAMLMarshal(toMarshal any) string { data, err := yaml.Marshal(toMarshal) @@ -416,6 +419,8 @@ func formattedYAMLMarshal(toMarshal any) string { return result } +// commentNullYAMLValues returns the given YAML-formatted string with null fields commented out. +// // Assumes no non-null fields will be set to unquoted strings ending in null. // For example, `field: 123-null` will be falsely commented but `field: '123-null'` is fine. func commentNullYAMLValues(needsComments string) string { From bd4c41031c6aad0edf622a288a2e3e03ed680850 Mon Sep 17 00:00:00 2001 From: milogreg Date: Wed, 11 Dec 2024 10:06:31 -0800 Subject: [PATCH 13/31] Merge main --- go/cmd/dolt/commands/query_diff.go | 8 ++++---- go/cmd/dolt/doltversion/version.go | 2 +- go/go.mod | 4 ++-- go/go.sum | 8 ++++---- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/go/cmd/dolt/commands/query_diff.go b/go/cmd/dolt/commands/query_diff.go index 69c894669fc..146a4a32246 100644 --- a/go/cmd/dolt/commands/query_diff.go +++ b/go/cmd/dolt/commands/query_diff.go @@ -70,8 +70,8 @@ func (q QueryDiff) ArgParser() *argparser.ArgParser { func (q QueryDiff) compareRows(pkOrds []int, row1, row2 sql.Row) (int, bool) { var cmp int for _, pkOrd := range pkOrds { - pk1, _ := gmstypes.ConvertToString(row1[pkOrd], gmstypes.Text) - pk2, _ := gmstypes.ConvertToString(row2[pkOrd], gmstypes.Text) + pk1, _ := gmstypes.ConvertToString(row1[pkOrd], gmstypes.Text, nil) + pk2, _ := gmstypes.ConvertToString(row2[pkOrd], gmstypes.Text, nil) if pk1 < pk2 { cmp = -1 } else if pk1 > pk2 { @@ -82,8 +82,8 @@ func (q QueryDiff) compareRows(pkOrds []int, row1, row2 sql.Row) (int, bool) { } var diff bool for i := 0; i < len(row1); i++ { - a, _ := gmstypes.ConvertToString(row1[i], gmstypes.Text) - b, _ := gmstypes.ConvertToString(row2[i], gmstypes.Text) + a, _ := gmstypes.ConvertToString(row1[i], gmstypes.Text, nil) + b, _ := gmstypes.ConvertToString(row2[i], gmstypes.Text, nil) if a != b { diff = true break diff --git a/go/cmd/dolt/doltversion/version.go b/go/cmd/dolt/doltversion/version.go index be05ffa14ab..a70bf8ad8d9 100644 --- a/go/cmd/dolt/doltversion/version.go +++ b/go/cmd/dolt/doltversion/version.go @@ -16,5 +16,5 @@ package doltversion const ( - Version = "1.44.1" + Version = "1.44.2" ) diff --git a/go/go.mod b/go/go.mod index 58b655bc87f..efe0d1cb90d 100644 --- a/go/go.mod +++ b/go/go.mod @@ -15,7 +15,7 @@ require ( github.com/dolthub/fslock v0.0.3 github.com/dolthub/ishell v0.0.0-20240701202509-2b217167d718 github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81 - github.com/dolthub/vitess v0.0.0-20241209181212-588631aba4be + github.com/dolthub/vitess v0.0.0-20241211024425-b00987f7ba54 github.com/dustin/go-humanize v1.0.1 github.com/fatih/color v1.13.0 github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 @@ -57,7 +57,7 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 github.com/creasty/defaults v1.6.0 github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2 - github.com/dolthub/go-mysql-server v0.18.2-0.20241209182739-d644619dc9ed + github.com/dolthub/go-mysql-server v0.18.2-0.20241211025720-09a7e80b835c github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63 github.com/dolthub/swiss v0.1.0 github.com/goccy/go-json v0.10.2 diff --git a/go/go.sum b/go/go.sum index 4012820204c..dbaf8f15525 100644 --- a/go/go.sum +++ b/go/go.sum @@ -183,8 +183,8 @@ github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U= github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0= github.com/dolthub/go-icu-regex v0.0.0-20240916130659-0118adc6b662 h1:aC17hZD6iwzBwwfO5M+3oBT5E5gGRiQPdn+vzpDXqIA= github.com/dolthub/go-icu-regex v0.0.0-20240916130659-0118adc6b662/go.mod h1:KPUcpx070QOfJK1gNe0zx4pA5sicIK1GMikIGLKC168= -github.com/dolthub/go-mysql-server v0.18.2-0.20241209182739-d644619dc9ed h1:YlXOo9xfRRglBrg+OvAFpBNohzWtK3YQ7SnTZNTbYZ0= -github.com/dolthub/go-mysql-server v0.18.2-0.20241209182739-d644619dc9ed/go.mod h1:Ra4lA9OjCy9J5tWPn05IBBlODnwaXZc8j+cChvfN/9Q= +github.com/dolthub/go-mysql-server v0.18.2-0.20241211025720-09a7e80b835c h1:U5IPpq8pPDELi2SCj/SjTXGvBWVpYJ26Q7O7oC5NQd4= +github.com/dolthub/go-mysql-server v0.18.2-0.20241211025720-09a7e80b835c/go.mod h1:QedjnglQ9+7yeRz4jAseFVZZgTxIZhEdt9G4RGqv4Jc= github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63 h1:OAsXLAPL4du6tfbBgK0xXHZkOlos63RdKYS3Sgw/dfI= github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63/go.mod h1:lV7lUeuDhH5thVGDCKXbatwKy2KW80L4rMT46n+Y2/Q= github.com/dolthub/ishell v0.0.0-20240701202509-2b217167d718 h1:lT7hE5k+0nkBdj/1UOSFwjWpNxf+LCApbRHgnCA17XE= @@ -197,8 +197,8 @@ github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81 h1:7/v8q9X github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81/go.mod h1:siLfyv2c92W1eN/R4QqG/+RjjX5W2+gCTRjZxBjI3TY= github.com/dolthub/swiss v0.1.0 h1:EaGQct3AqeP/MjASHLiH6i4TAmgbG/c4rA6a1bzCOPc= github.com/dolthub/swiss v0.1.0/go.mod h1:BeucyB08Vb1G9tumVN3Vp/pyY4AMUnr9p7Rz7wJ7kAQ= -github.com/dolthub/vitess v0.0.0-20241209181212-588631aba4be h1:YF+vUXEAMqai036iZPJS2JRKd3pXcCQ1TYcLWBR4boo= -github.com/dolthub/vitess v0.0.0-20241209181212-588631aba4be/go.mod h1:1gQZs/byeHLMSul3Lvl3MzioMtOW1je79QYGyi2fd70= +github.com/dolthub/vitess v0.0.0-20241211024425-b00987f7ba54 h1:nzBnC0Rt1gFtscJEz4veYd/mazZEdbdmed+tujdaKOo= +github.com/dolthub/vitess v0.0.0-20241211024425-b00987f7ba54/go.mod h1:1gQZs/byeHLMSul3Lvl3MzioMtOW1je79QYGyi2fd70= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= From 7627f6b25d18c9b1f24cad8fefba93468db893e4 Mon Sep 17 00:00:00 2001 From: milogreg Date: Mon, 6 Jan 2025 11:22:25 -0800 Subject: [PATCH 14/31] Merge main --- .../get-mysql-dolt-job-json.sh | 2 +- README.md | 2 +- go/Godeps/LICENSES | 24 +- go/cmd/dolt/cli/cli_context.go | 20 +- go/cmd/dolt/commands/filter-branch.go | 2 +- go/cmd/dolt/commands/init_test.go | 2 +- go/cmd/dolt/commands/schcmds/import.go | 2 +- go/cmd/dolt/commands/signed_commits_test.go | 2 +- go/cmd/dolt/commands/sql.go | 2 +- go/cmd/dolt/commands/sql_statement_scanner.go | 29 +- .../commands/sql_statement_scanner_test.go | 2 +- go/cmd/dolt/commands/sql_test.go | 22 +- .../commands/sqlserver/command_line_config.go | 24 +- go/cmd/dolt/commands/sqlserver/server.go | 23 +- go/cmd/dolt/commands/sqlserver/server_test.go | 8 +- go/cmd/dolt/commands/sqlserver/sqlserver.go | 90 ++++- go/cmd/dolt/commands/utils.go | 4 +- go/cmd/dolt/dolt.go | 338 +++++++++++------- go/cmd/dolt/doltversion/version.go | 2 +- go/cmd/dolt/system_checks.go | 84 +++-- go/gen/fb/serial/fileidentifiers.go | 1 + go/gen/fb/serial/tuple.go | 102 ++++++ go/go.mod | 19 +- go/go.sum | 42 +-- go/go.work.sum | 2 + .../doltcore/doltdb/commit_hooks_test.go | 2 +- go/libraries/doltcore/doltdb/doltdb.go | 38 ++ go/libraries/doltcore/doltdb/doltdb_test.go | 4 +- go/libraries/doltcore/doltdb/durable/index.go | 37 +- go/libraries/doltcore/doltdb/errors.go | 1 + .../doltcore/doltdb/feature_version_test.go | 2 +- .../doltcore/doltdb/foreign_key_test.go | 4 +- go/libraries/doltcore/doltdb/gc_test.go | 2 +- go/libraries/doltcore/doltdb/hooksdatabase.go | 8 + go/libraries/doltcore/doltdb/root_val.go | 2 +- go/libraries/doltcore/doltdb/table.go | 2 +- .../dtestutils/testcommands/multienv.go | 4 +- go/libraries/doltcore/env/environment.go | 16 +- .../doltcore/merge/fulltext_rebuild.go | 2 +- .../merge/keyless_integration_test.go | 8 +- .../doltcore/merge/schema_integration_test.go | 8 +- go/libraries/doltcore/merge/violations_fk.go | 8 +- .../doltcore/migrate/integration_test.go | 2 +- .../doltcore/rebase/filter_branch_test.go | 4 +- go/libraries/doltcore/ref/ref.go | 7 + go/libraries/doltcore/ref/tuple_ref.go | 41 +++ .../doltcore/schema/integration_test.go | 2 +- go/libraries/doltcore/schema/schema.go | 19 +- go/libraries/doltcore/schema/schema_test.go | 13 +- .../servercfg/testdata/minver_validation.txt | 2 +- .../doltcore/servercfg/yaml_config.go | 7 +- .../doltcore/servercfg/yaml_config_test.go | 17 +- .../doltcore/sqle/alterschema_test.go | 2 +- go/libraries/doltcore/sqle/common_test.go | 2 +- .../doltcore/sqle/database_provider.go | 7 +- .../dolt_log_table_function.go | 4 +- .../sqle/dtables/commit_diff_table.go | 2 +- .../sqle/dtables/conflicts_tables_prolly.go | 2 +- .../doltcore/sqle/dtables/diff_table.go | 41 ++- .../doltcore/sqle/dtables/prolly_row_conv.go | 20 +- .../sqle/enginetest/dolt_engine_test.go | 4 +- .../sqle/enginetest/dolt_engine_tests.go | 51 +-- .../doltcore/sqle/enginetest/dolt_harness.go | 1 + .../sqle/enginetest/dolt_procedure_queries.go | 40 +++ .../sqle/enginetest/dolt_queries_diff.go | 103 +++++- .../doltcore/sqle/enginetest/stats_queries.go | 79 +++- .../doltcore/sqle/index/dolt_index.go | 25 +- .../database_revision_test.go | 2 +- .../integration_test/history_table_test.go | 4 +- .../sqle/integration_test/json_value_test.go | 2 +- go/libraries/doltcore/sqle/kvexec/builder.go | 2 +- .../doltcore/sqle/procedures_table.go | 17 +- .../doltcore/sqle/statsnoms/database.go | 130 ++++++- go/libraries/doltcore/sqle/statsnoms/load.go | 99 +++-- .../doltcore/sqle/statspro/analyze.go | 20 +- .../doltcore/sqle/statspro/auto_refresh.go | 15 +- .../doltcore/sqle/statspro/configure.go | 8 +- .../doltcore/sqle/statspro/initdbhook.go | 27 +- .../doltcore/sqle/statspro/interface.go | 13 +- .../doltcore/sqle/statspro/stats_provider.go | 9 + .../doltcore/sqle/system_variables.go | 2 +- go/libraries/doltcore/sqle/tables.go | 6 +- go/libraries/doltcore/sqle/temp_table.go | 2 +- go/libraries/doltcore/sqle/testutil.go | 2 +- .../editor/creation/external_build_index.go | 2 +- .../doltcore/table/editor/creation/index.go | 2 +- .../table/untyped/sqlexport/sqlwriter_test.go | 2 +- go/libraries/utils/argparser/parser_test.go | 13 + go/libraries/utils/argparser/results.go | 27 ++ go/performance/import_benchmarker/testdef.go | 2 +- go/performance/microsysbench/sysbench_test.go | 28 +- go/serial/fileidentifiers.go | 1 + go/serial/generate.sh | 1 + go/serial/tuple.fbs | 22 ++ go/store/datas/database.go | 4 + go/store/datas/database_common.go | 20 ++ go/store/datas/dataset.go | 38 ++ go/store/datas/tuple.go | 106 ++++++ go/store/nbs/archive_build.go | 23 +- go/store/nbs/store_test.go | 30 +- go/store/prolly/message/address_map.go | 6 +- go/store/prolly/message/blob.go | 4 +- go/store/prolly/message/commit_closure.go | 4 +- go/store/prolly/message/item_access.go | 26 +- go/store/prolly/message/merge_artifacts.go | 12 +- go/store/prolly/message/message.go | 4 +- go/store/prolly/message/prolly_map.go | 12 +- go/store/prolly/message/prolly_map_test.go | 2 +- go/store/prolly/message/serialize.go | 11 + go/store/prolly/tree/node_test.go | 2 +- go/store/types/value_store.go | 1 + go/store/val/codec.go | 6 +- go/store/val/codec_test.go | 2 +- go/store/val/tuple.go | 2 - go/store/val/tuple_compare.go | 2 +- go/store/val/tuple_descriptor.go | 4 +- integration-tests/bats/archive.bats | 153 +++----- .../bats/caching_sha2_password.bats | 152 ++++++++ integration-tests/bats/fsck.bats | 41 ++- integration-tests/bats/profile.bats | 2 +- integration-tests/bats/sql-diff.bats | 73 ++-- integration-tests/bats/sql-server.bats | 40 ++- integration-tests/bats/stats.bats | 59 +++ integration-tests/go-sql-server-driver/go.mod | 6 +- integration-tests/go-sql-server-driver/go.sum | 12 +- 125 files changed, 2090 insertions(+), 724 deletions(-) create mode 100644 go/gen/fb/serial/tuple.go create mode 100644 go/libraries/doltcore/ref/tuple_ref.go create mode 100644 go/serial/tuple.fbs create mode 100644 go/store/datas/tuple.go create mode 100644 integration-tests/bats/caching_sha2_password.bats diff --git a/.github/scripts/performance-benchmarking/get-mysql-dolt-job-json.sh b/.github/scripts/performance-benchmarking/get-mysql-dolt-job-json.sh index 950bb2c1c5d..4ed73ccdfaa 100755 --- a/.github/scripts/performance-benchmarking/get-mysql-dolt-job-json.sh +++ b/.github/scripts/performance-benchmarking/get-mysql-dolt-job-json.sh @@ -19,7 +19,7 @@ issueNumber="$9" initBigRepo="${10}" nomsBinFormat="${11}" withTpcc="${12}" -precision="1" +precision="2" tpccRegex="tpcc%" toProfileKey="" diff --git a/README.md b/README.md index f1549ab801e..f8a20b5ed40 100644 --- a/README.md +++ b/README.md @@ -254,7 +254,7 @@ You are connected! While we're here let's grab a copy of MySQL so we can connect with that client. Head over to the [MySQL Getting Started](https://dev.mysql.com/doc/mysql-getting-started/en/) documentation and install MySQL on your machine. I used [Homebrew](https://brew.sh/) to install MySQL on my Mac: `brew install mysql@8.4`. Alternatively, you can install only the client component by running `brew install mysql-client@8.4`. -NOTE: Make sure you install a MySQL 8.4 release. MySQL 8.4 is the current Long Term Support (LTS) release, meaning this is the stable and supported version of MySQL. MySQL 9.0 is also available, but is an "innovation" release, meaning it has more recent changes and features, but may not be as stable as the LTS release. The 9.0 release changes authentication support and isn't able to connect to a Dolt SQL server by default. You can install MySQL 8.4 with Homebrew by running `brew install mysql@8.4`. We are currently [working on support for the 9.0 auth changes](https://github.com/dolthub/dolt/issues/8496) and expect to support MySQL 9.0 clients without `mysql_native_password` support before the end of 2024. +NOTE: Make sure you install a MySQL 8.4 release. MySQL 8.4 is the current Long Term Support (LTS) release, meaning this is the stable and supported version of MySQL. MySQL 9.0 is also available, but is an "innovation" release, meaning it has more recent changes and features, but may not be as stable as the LTS release. The 9.0 release changes authentication support and isn't able to connect to a Dolt SQL server by default. You can install MySQL 8.4 with Homebrew by running `brew install mysql@8.4`. If you do want to use MySQL-9.0, read [our post on how to configure Dolt for `caching_sha2_password` authentication](https://www.dolthub.com/blog/2024-12-11-mysql9-and-caching-sha2-auth-support/). MySQL comes with a MySQL server called `mysqld` and a MySQL client called `mysql`. You're only interested in the client. After following the instructions from MySQL's documentation, make sure you have a copy of the `mysql` client on your path: diff --git a/go/Godeps/LICENSES b/go/Godeps/LICENSES index d333363a42e..f5221763117 100644 --- a/go/Godeps/LICENSES +++ b/go/Godeps/LICENSES @@ -11278,7 +11278,7 @@ THE SOFTWARE. ================================================================================ = golang.org/x/crypto licensed under: = -Copyright (c) 2009 The Go Authors. All rights reserved. +Copyright 2009 The Go Authors. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -11290,7 +11290,7 @@ notice, this list of conditions and the following disclaimer. copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - * Neither the name of Google Inc. nor the names of its + * Neither the name of Google LLC nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -11306,7 +11306,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -= LICENSE ed6066ae50f153e2965216c6d4b9335900f1f8b2b526527f49a619d7 = += LICENSE 38c969f398439fdd936e8cb41f8255378028fdd77ecb756c2e3401ae = ================================================================================ ================================================================================ @@ -11414,7 +11414,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ = golang.org/x/sync licensed under: = -Copyright (c) 2009 The Go Authors. All rights reserved. +Copyright 2009 The Go Authors. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -11426,7 +11426,7 @@ notice, this list of conditions and the following disclaimer. copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - * Neither the name of Google Inc. nor the names of its + * Neither the name of Google LLC nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -11442,7 +11442,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -= LICENSE ed6066ae50f153e2965216c6d4b9335900f1f8b2b526527f49a619d7 = += LICENSE 38c969f398439fdd936e8cb41f8255378028fdd77ecb756c2e3401ae = ================================================================================ ================================================================================ @@ -11482,7 +11482,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ = golang.org/x/term licensed under: = -Copyright (c) 2009 The Go Authors. All rights reserved. +Copyright 2009 The Go Authors. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -11494,7 +11494,7 @@ notice, this list of conditions and the following disclaimer. copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - * Neither the name of Google Inc. nor the names of its + * Neither the name of Google LLC nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -11510,13 +11510,13 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -= LICENSE ed6066ae50f153e2965216c6d4b9335900f1f8b2b526527f49a619d7 = += LICENSE 38c969f398439fdd936e8cb41f8255378028fdd77ecb756c2e3401ae = ================================================================================ ================================================================================ = golang.org/x/text licensed under: = -Copyright (c) 2009 The Go Authors. All rights reserved. +Copyright 2009 The Go Authors. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -11528,7 +11528,7 @@ notice, this list of conditions and the following disclaimer. copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - * Neither the name of Google Inc. nor the names of its + * Neither the name of Google LLC nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -11544,7 +11544,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -= LICENSE ed6066ae50f153e2965216c6d4b9335900f1f8b2b526527f49a619d7 = += LICENSE 38c969f398439fdd936e8cb41f8255378028fdd77ecb756c2e3401ae = ================================================================================ ================================================================================ diff --git a/go/cmd/dolt/cli/cli_context.go b/go/cmd/dolt/cli/cli_context.go index 659608836e0..06487f275ec 100644 --- a/go/cmd/dolt/cli/cli_context.go +++ b/go/cmd/dolt/cli/cli_context.go @@ -23,6 +23,7 @@ import ( "github.com/dolthub/dolt/go/cmd/dolt/errhand" "github.com/dolthub/dolt/go/libraries/doltcore/env" "github.com/dolthub/dolt/go/libraries/utils/argparser" + "github.com/dolthub/dolt/go/libraries/utils/filesys" ) // LateBindQueryist is a function that will be called the first time Queryist is needed for use. Input is a context which @@ -39,17 +40,23 @@ type LateBindQueryist func(ctx context.Context) (Queryist, *sql.Context, func(), type CliContext interface { // GlobalArgs returns the arguments passed before the subcommand. GlobalArgs() *argparser.ArgParseResults + WorkingDir() filesys.Filesys Config() *env.DoltCliConfig QueryEngine(ctx context.Context) (Queryist, *sql.Context, func(), error) } // NewCliContext creates a new CliContext instance. Arguments must not be nil. -func NewCliContext(args *argparser.ArgParseResults, config *env.DoltCliConfig, latebind LateBindQueryist) (CliContext, errhand.VerboseError) { - if args == nil || config == nil || latebind == nil { - return nil, errhand.VerboseErrorFromError(errors.New("Invariant violated. args, config, and latebind must be non nil.")) +func NewCliContext(args *argparser.ArgParseResults, config *env.DoltCliConfig, cwd filesys.Filesys, latebind LateBindQueryist) (CliContext, errhand.VerboseError) { + if args == nil || config == nil || cwd == nil || latebind == nil { + return nil, errhand.VerboseErrorFromError(errors.New("Invariant violated. args, config, cwd, and latebind must be non nil.")) } - return LateBindCliContext{globalArgs: args, config: config, activeContext: &QueryistContext{}, bind: latebind}, nil + return LateBindCliContext{ + globalArgs: args, + config: config, + cwd: cwd, + activeContext: &QueryistContext{}, + bind: latebind}, nil } type QueryistContext struct { @@ -62,6 +69,7 @@ type QueryistContext struct { // created once. type LateBindCliContext struct { globalArgs *argparser.ArgParseResults + cwd filesys.Filesys config *env.DoltCliConfig activeContext *QueryistContext @@ -92,6 +100,10 @@ func (lbc LateBindCliContext) QueryEngine(ctx context.Context) (Queryist, *sql.C return qryist, sqlCtx, closer, nil } +func (lbc LateBindCliContext) WorkingDir() filesys.Filesys { + return lbc.cwd +} + // Config returns the dolt config stored in CliContext func (lbc LateBindCliContext) Config() *env.DoltCliConfig { return lbc.config diff --git a/go/cmd/dolt/commands/filter-branch.go b/go/cmd/dolt/commands/filter-branch.go index 81c5c702c29..2161119f516 100644 --- a/go/cmd/dolt/commands/filter-branch.go +++ b/go/cmd/dolt/commands/filter-branch.go @@ -281,7 +281,7 @@ func processFilterQuery(ctx context.Context, dEnv *env.DoltEnv, root doltdb.Root return nil, err } - scanner := newStreamScanner(strings.NewReader(query)) + scanner := NewStreamScanner(strings.NewReader(query)) if err != nil { return nil, err } diff --git a/go/cmd/dolt/commands/init_test.go b/go/cmd/dolt/commands/init_test.go index ab5a411d1ea..a287a414e36 100644 --- a/go/cmd/dolt/commands/init_test.go +++ b/go/cmd/dolt/commands/init_test.go @@ -72,7 +72,7 @@ func TestInit(t *testing.T) { gCfg.SetStrings(test.GlobalConfig) apr := argparser.ArgParseResults{} latebind := func(ctx context.Context) (cli.Queryist, *sql.Context, func(), error) { return nil, nil, func() {}, nil } - cliCtx, _ := cli.NewCliContext(&apr, dEnv.Config, latebind) + cliCtx, _ := cli.NewCliContext(&apr, dEnv.Config, dEnv.FS, latebind) result := InitCmd{}.Exec(context.Background(), "dolt init", test.Args, dEnv, cliCtx) defer dEnv.DoltDB.Close() diff --git a/go/cmd/dolt/commands/schcmds/import.go b/go/cmd/dolt/commands/schcmds/import.go index c4cb9da28f7..56c86d5fdfa 100644 --- a/go/cmd/dolt/commands/schcmds/import.go +++ b/go/cmd/dolt/commands/schcmds/import.go @@ -330,7 +330,7 @@ func putEmptyTableWithSchema(ctx context.Context, tblName string, root doltdb.Ro return nil, errhand.BuildDError("error: failed to get table.").AddCause(err).Build() } - empty, err := durable.NewEmptyIndex(ctx, root.VRW(), root.NodeStore(), sch, false) + empty, err := durable.NewEmptyPrimaryIndex(ctx, root.VRW(), root.NodeStore(), sch) if err != nil { return nil, errhand.BuildDError("error: failed to get table.").AddCause(err).Build() } diff --git a/go/cmd/dolt/commands/signed_commits_test.go b/go/cmd/dolt/commands/signed_commits_test.go index 8fbfec98b22..e83d1132bf3 100644 --- a/go/cmd/dolt/commands/signed_commits_test.go +++ b/go/cmd/dolt/commands/signed_commits_test.go @@ -149,7 +149,7 @@ func execCommand(ctx context.Context, wd string, cmd cli.Command, args []string, return } - cliCtx, err := cli.NewCliContext(apr, cfg, latebind) + cliCtx, err := cli.NewCliContext(apr, cfg, dEnv.FS, latebind) if err != nil { err = fmt.Errorf("error creating cli context: %w", err) return diff --git a/go/cmd/dolt/commands/sql.go b/go/cmd/dolt/commands/sql.go index aa44b4d7b56..914141da163 100644 --- a/go/cmd/dolt/commands/sql.go +++ b/go/cmd/dolt/commands/sql.go @@ -612,7 +612,7 @@ func saveQuery(ctx *sql.Context, root doltdb.RootValue, query string, name strin // execBatchMode runs all the queries in the input reader func execBatchMode(ctx *sql.Context, qryist cli.Queryist, input io.Reader, continueOnErr bool, format engine.PrintResultFormat) error { - scanner := newStreamScanner(input) + scanner := NewStreamScanner(input) var query string for scanner.Scan() { if fileReadProg != nil { diff --git a/go/cmd/dolt/commands/sql_statement_scanner.go b/go/cmd/dolt/commands/sql_statement_scanner.go index 77a49b2c822..5789e4779f4 100755 --- a/go/cmd/dolt/commands/sql_statement_scanner.go +++ b/go/cmd/dolt/commands/sql_statement_scanner.go @@ -44,11 +44,11 @@ const delimPrefixLen = 10 var delimPrefix = []byte("delimiter ") -// streamScanner is an iterator that reads bytes from |inp| until either +// StreamScanner is an iterator that reads bytes from |inp| until either // (1) we match a DELIMITER statement, (2) we match the |delimiter| token, // or (3) we EOF the file. After each Scan() call, the valid token will // span from the buffer beginning to |state.end|. -type streamScanner struct { +type StreamScanner struct { inp io.Reader buf []byte maxSize int @@ -61,8 +61,9 @@ type streamScanner struct { state *qState } -func newStreamScanner(r io.Reader) *streamScanner { - return &streamScanner{inp: r, buf: make([]byte, pageSize), maxSize: maxStatementBufferBytes, delimiter: []byte(";"), state: new(qState)} +// NewStreamScanner returns a new StreamScanner +func NewStreamScanner(r io.Reader) *StreamScanner { + return &StreamScanner{inp: r, buf: make([]byte, pageSize), maxSize: maxStatementBufferBytes, delimiter: []byte(";"), state: new(qState)} } type qState struct { @@ -77,7 +78,7 @@ type qState struct { statementStartLine int } -func (s *streamScanner) Scan() bool { +func (s *StreamScanner) Scan() bool { // truncate last query s.truncate() s.resetState() @@ -131,7 +132,7 @@ func (s *streamScanner) Scan() bool { } } -func (s *streamScanner) skipWhitespace() bool { +func (s *StreamScanner) skipWhitespace() bool { for { if s.i >= s.fill { if err := s.read(); err != nil { @@ -153,17 +154,17 @@ func (s *streamScanner) skipWhitespace() bool { return true } -func (s *streamScanner) truncate() { +func (s *StreamScanner) truncate() { // copy size should be 4k or less s.state.start = s.i s.state.end = s.i } -func (s *streamScanner) resetState() { +func (s *StreamScanner) resetState() { s.state = &qState{} } -func (s *streamScanner) read() error { +func (s *StreamScanner) read() error { if s.fill >= s.maxSize { // if script exceeds buffer that's OK, if // a single query exceeds buffer that's not OK @@ -195,21 +196,21 @@ func (s *streamScanner) read() error { return nil } -func (s *streamScanner) Err() error { +func (s *StreamScanner) Err() error { return s.err } -func (s *streamScanner) Bytes() []byte { +func (s *StreamScanner) Bytes() []byte { return s.buf[s.state.start:s.state.end] } // Text returns the most recent token generated by a call to [Scanner.Scan] // as a newly allocated string holding its bytes. -func (s *streamScanner) Text() string { +func (s *StreamScanner) Text() string { return string(s.Bytes()) } -func (s *streamScanner) isDelimiterExpr() (error, bool) { +func (s *StreamScanner) isDelimiterExpr() (error, bool) { if s.i == 0 && s.fill-s.i < delimPrefixLen { // need to see first |delimPrefixLen| characters if err := s.read(); err != nil { @@ -251,7 +252,7 @@ func (s *streamScanner) isDelimiterExpr() (error, bool) { return nil, false } -func (s *streamScanner) seekDelimiter() (error, bool) { +func (s *StreamScanner) seekDelimiter() (error, bool) { if s.i >= s.fill { return nil, false } diff --git a/go/cmd/dolt/commands/sql_statement_scanner_test.go b/go/cmd/dolt/commands/sql_statement_scanner_test.go index f99b8563750..46bd62fb046 100755 --- a/go/cmd/dolt/commands/sql_statement_scanner_test.go +++ b/go/cmd/dolt/commands/sql_statement_scanner_test.go @@ -209,7 +209,7 @@ insert into foo values (1,2,3)|`, for _, tt := range testcases { t.Run(tt.input, func(t *testing.T) { reader := strings.NewReader(tt.input) - scanner := newStreamScanner(reader) + scanner := NewStreamScanner(reader) var i int for scanner.Scan() { require.True(t, i < len(tt.statements)) diff --git a/go/cmd/dolt/commands/sql_test.go b/go/cmd/dolt/commands/sql_test.go index 4a064d2104f..240b3d32d0e 100644 --- a/go/cmd/dolt/commands/sql_test.go +++ b/go/cmd/dolt/commands/sql_test.go @@ -43,7 +43,7 @@ func TestSqlConsole(t *testing.T) { require.NoError(t, err) defer dEnv.DoltDB.Close() - cliCtx, err := NewArgFreeCliContext(ctx, dEnv) + cliCtx, err := NewArgFreeCliContext(ctx, dEnv, dEnv.FS) require.NoError(t, err) args := []string{} @@ -76,7 +76,7 @@ func TestSqlBatchMode(t *testing.T) { require.NoError(t, err) defer dEnv.DoltDB.Close() - cliCtx, err := NewArgFreeCliContext(ctx, dEnv) + cliCtx, err := NewArgFreeCliContext(ctx, dEnv, dEnv.FS) require.NoError(t, err) args := []string{"-b", "-q", test.query} @@ -119,7 +119,7 @@ func TestSqlSelect(t *testing.T) { require.NoError(t, err) defer dEnv.DoltDB.Close() - cliCtx, err := NewArgFreeCliContext(ctx, dEnv) + cliCtx, err := NewArgFreeCliContext(ctx, dEnv, dEnv.FS) require.NoError(t, err) args := []string{"-q", test.query} @@ -149,7 +149,7 @@ func TestSqlShow(t *testing.T) { require.NoError(t, err) defer dEnv.DoltDB.Close() - cliCtx, err := NewArgFreeCliContext(ctx, dEnv) + cliCtx, err := NewArgFreeCliContext(ctx, dEnv, dEnv.FS) require.NoError(t, err) args := []string{"-q", test.query} @@ -188,7 +188,7 @@ func TestCreateTable(t *testing.T) { assert.NoError(t, err) assert.False(t, has, "table exists before creating it") - cliCtx, err := NewArgFreeCliContext(ctx, dEnv) + cliCtx, err := NewArgFreeCliContext(ctx, dEnv, dEnv.FS) require.NoError(t, err) args := []string{"-q", test.query} @@ -232,7 +232,7 @@ func TestShowTables(t *testing.T) { require.NoError(t, err) defer dEnv.DoltDB.Close() - cliCtx, err := NewArgFreeCliContext(ctx, dEnv) + cliCtx, err := NewArgFreeCliContext(ctx, dEnv, dEnv.FS) require.NoError(t, err) args := []string{"-q", test.query} @@ -268,7 +268,7 @@ func TestAlterTable(t *testing.T) { require.NoError(t, err) defer dEnv.DoltDB.Close() - cliCtx, err := NewArgFreeCliContext(ctx, dEnv) + cliCtx, err := NewArgFreeCliContext(ctx, dEnv, dEnv.FS) require.NoError(t, err) args := []string{"-q", test.query} @@ -299,7 +299,7 @@ func TestDropTable(t *testing.T) { require.NoError(t, err) defer dEnv.DoltDB.Close() - cliCtx, err := NewArgFreeCliContext(ctx, dEnv) + cliCtx, err := NewArgFreeCliContext(ctx, dEnv, dEnv.FS) require.NoError(t, err) args := []string{"-q", test.query} @@ -420,7 +420,7 @@ func TestInsert(t *testing.T) { require.NoError(t, err) defer dEnv.DoltDB.Close() - cliCtx, err := NewArgFreeCliContext(ctx, dEnv) + cliCtx, err := NewArgFreeCliContext(ctx, dEnv, dEnv.FS) require.NoError(t, err) args := []string{"-q", test.query} @@ -504,7 +504,7 @@ func TestUpdate(t *testing.T) { require.NoError(t, err) defer dEnv.DoltDB.Close() - cliCtx, err := NewArgFreeCliContext(ctx, dEnv) + cliCtx, err := NewArgFreeCliContext(ctx, dEnv, dEnv.FS) require.NoError(t, err) args := []string{"-q", test.query} @@ -582,7 +582,7 @@ func TestDelete(t *testing.T) { require.NoError(t, err) defer dEnv.DoltDB.Close() - cliCtx, err := NewArgFreeCliContext(ctx, dEnv) + cliCtx, err := NewArgFreeCliContext(ctx, dEnv, dEnv.FS) require.NoError(t, err) args := []string{"-q", test.query} diff --git a/go/cmd/dolt/commands/sqlserver/command_line_config.go b/go/cmd/dolt/commands/sqlserver/command_line_config.go index 6c05e48744e..41d37130e1c 100755 --- a/go/cmd/dolt/commands/sqlserver/command_line_config.go +++ b/go/cmd/dolt/commands/sqlserver/command_line_config.go @@ -80,8 +80,10 @@ func DefaultCommandLineServerConfig() *commandLineServerConfig { } } -// NewCommandLineConfig returns server config based on the credentials and command line arguments given. -func NewCommandLineConfig(creds *cli.UserPassword, apr *argparser.ArgParseResults) (servercfg.ServerConfig, error) { +// NewCommandLineConfig returns server config based on the credentials and command line arguments given. The dataDirOverride +// parameter is used to override the data dir specified in the command line arguments. This comes up when there are +// situations where there are multiple ways to specify the data dir. +func NewCommandLineConfig(creds *cli.UserPassword, apr *argparser.ArgParseResults, dataDirOverride string) (servercfg.ServerConfig, error) { config := DefaultCommandLineServerConfig() if sock, ok := apr.GetValue(socketFlag); ok { @@ -144,8 +146,16 @@ func NewCommandLineConfig(creds *cli.UserPassword, apr *argparser.ArgParseResult config.withDataDir(dataDir) } - if dataDir, ok := apr.GetValue(commands.DataDirFlag); ok { - config.withDataDir(dataDir) + // We explicitly don't use the dataDir flag from the APR here. The data dir flag is pulled out early and converted + // to an absolute path. It is read in the GetDataDirPreStart function, which is called early in dolt.go to get the + // data dir for any dolt process. This complexity exists because the server's config.yaml config file can contain the + // dataDir, but we don't execute any server specific logic until after the database environment is initialized. + if dataDirOverride != "" { + config.withDataDir(dataDirOverride) + } else { + if dd, ok := apr.GetValue(commands.DataDirFlag); ok { + config.withDataDir(dd) + } } if maxConnections, ok := apr.GetInt(maxConnectionsFlag); ok { @@ -466,7 +476,7 @@ type ServerConfigReader interface { // ReadConfigFile reads a config file and returns a ServerConfig for it ReadConfigFile(cwdFS filesys.Filesys, file string) (servercfg.ServerConfig, error) // ReadConfigArgs reads command line arguments and returns a ServerConfig for them - ReadConfigArgs(args *argparser.ArgParseResults) (servercfg.ServerConfig, error) + ReadConfigArgs(args *argparser.ArgParseResults, dataDirOverride string) (servercfg.ServerConfig, error) } var _ ServerConfigReader = DoltServerConfigReader{} @@ -475,6 +485,6 @@ func (d DoltServerConfigReader) ReadConfigFile(cwdFS filesys.Filesys, file strin return servercfg.YamlConfigFromFile(cwdFS, file) } -func (d DoltServerConfigReader) ReadConfigArgs(args *argparser.ArgParseResults) (servercfg.ServerConfig, error) { - return NewCommandLineConfig(nil, args) +func (d DoltServerConfigReader) ReadConfigArgs(args *argparser.ArgParseResults, dataDirOverride string) (servercfg.ServerConfig, error) { + return NewCommandLineConfig(nil, args, dataDirOverride) } diff --git a/go/cmd/dolt/commands/sqlserver/server.go b/go/cmd/dolt/commands/sqlserver/server.go index 3915b1973f0..b46ac8e2361 100644 --- a/go/cmd/dolt/commands/sqlserver/server.go +++ b/go/cmd/dolt/commands/sqlserver/server.go @@ -154,30 +154,13 @@ func ConfigureServices( controller.Register(newHeartbeatService(version, dEnv)) fs := dEnv.FS - InitDataDir := &svcs.AnonService{ + InitFailsafes := &svcs.AnonService{ InitF: func(ctx context.Context) (err error) { - if len(serverConfig.DataDir()) > 0 && serverConfig.DataDir() != "." { - fs, err = dEnv.FS.WithWorkingDir(serverConfig.DataDir()) - if err != nil { - return err - } - // If datadir has changed, then reload the DoltEnv to ensure its local - // configuration store gets configured correctly - dEnv.FS = fs - dEnv = env.Load(ctx, dEnv.GetUserHomeDir, fs, doltdb.LocalDirDoltDB, dEnv.Version) - - // If the datadir has changed, then we need to load any persisted global variables - // from the new datadir's local configuration store - err = dsess.InitPersistedSystemVars(dEnv) - if err != nil { - logrus.Errorf("failed to load persisted global variables: %s\n", err.Error()) - } - } dEnv.Config.SetFailsafes(env.DefaultFailsafeConfig) return nil }, } - controller.Register(InitDataDir) + controller.Register(InitFailsafes) var mrEnv *env.MultiRepoEnv InitMultiEnv := &svcs.AnonService{ @@ -903,7 +886,7 @@ func portInUse(hostPort string) bool { } func newSessionBuilder(se *engine.SqlEngine, config servercfg.ServerConfig) server.SessionBuilder { - userToSessionVars := make(map[string]map[string]string) + userToSessionVars := make(map[string]map[string]interface{}) userVars := config.UserVars() for _, curr := range userVars { userToSessionVars[curr.Name] = curr.Vars diff --git a/go/cmd/dolt/commands/sqlserver/server_test.go b/go/cmd/dolt/commands/sqlserver/server_test.go index c86f0284a03..c165b477a8f 100644 --- a/go/cmd/dolt/commands/sqlserver/server_test.go +++ b/go/cmd/dolt/commands/sqlserver/server_test.go @@ -77,7 +77,7 @@ func TestServerArgs(t *testing.T) { "-t", "5", "-l", "info", "-r", - }, dEnv, controller) + }, dEnv, dEnv.FS, controller) }() err = controller.WaitForStart() require.NoError(t, err) @@ -118,7 +118,7 @@ listener: dEnv.FS.WriteFile("config.yaml", []byte(yamlConfig), os.ModePerm) StartServer(context.Background(), "0.0.0", "dolt sql-server", []string{ "--config", "config.yaml", - }, dEnv, controller) + }, dEnv, dEnv.FS, controller) }() err = controller.WaitForStart() require.NoError(t, err) @@ -151,7 +151,7 @@ func TestServerBadArgs(t *testing.T) { t.Run(strings.Join(test, " "), func(t *testing.T) { controller := svcs.NewController() go func() { - StartServer(context.Background(), "test", "dolt sql-server", test, env, controller) + StartServer(context.Background(), "test", "dolt sql-server", test, env, env.FS, controller) }() if !assert.Error(t, controller.WaitForStart()) { controller.Stop() @@ -286,7 +286,7 @@ func TestServerFailsIfPortInUse(t *testing.T) { "-t", "5", "-l", "info", "-r", - }, dEnv, controller) + }, dEnv, dEnv.FS, controller) }() err = controller.WaitForStart() diff --git a/go/cmd/dolt/commands/sqlserver/sqlserver.go b/go/cmd/dolt/commands/sqlserver/sqlserver.go index fa1de0232e6..b9cb9dc20ab 100644 --- a/go/cmd/dolt/commands/sqlserver/sqlserver.go +++ b/go/cmd/dolt/commands/sqlserver/sqlserver.go @@ -16,6 +16,7 @@ package sqlserver import ( "context" + "errors" "fmt" "os" "path/filepath" @@ -214,7 +215,7 @@ func (cmd SqlServerCmd) Exec(ctx context.Context, commandStr string, args []stri } }() - err := StartServer(newCtx, cmd.VersionStr, commandStr, args, dEnv, controller) + err := StartServer(newCtx, cmd.VersionStr, commandStr, args, dEnv, cliCtx.WorkingDir(), controller) if err != nil { cli.Println(color.RedString(err.Error())) return 1 @@ -236,10 +237,10 @@ func validateSqlServerArgs(apr *argparser.ArgParseResults) error { } // StartServer starts the sql server with the controller provided and blocks until the server is stopped. -func StartServer(ctx context.Context, versionStr, commandStr string, args []string, dEnv *env.DoltEnv, controller *svcs.Controller) error { +func StartServer(ctx context.Context, versionStr, commandStr string, args []string, dEnv *env.DoltEnv, cwd filesys.Filesys, controller *svcs.Controller) error { ap := SqlServerCmd{}.ArgParser() help, _ := cli.HelpAndUsagePrinters(cli.CommandDocsForCommandString(commandStr, sqlServerDocs, ap)) - serverConfig, err := ServerConfigFromArgs(ap, help, args, dEnv) + serverConfig, err := ServerConfigFromArgs(ap, help, args, dEnv, cwd) if err != nil { return err } @@ -267,9 +268,63 @@ func StartServer(ctx context.Context, versionStr, commandStr string, args []stri return nil } +// GetDataDirPreStart returns the data dir to use for the process. This is called early in the bootstrapping of the process +// to ensure that we know the data dir early. This function first parses the args for the --data-dir flag, +// then attempts to find it in the server's yaml config file if it was specified. +// +// The returned value is non-empty only if we found a data dir. The string will be an absolute path to the data dir. An +// empty string indicates that there was no data dir specified, and the caller should determine the data dir. +// +// If the --data-dir flag is specified in the command line, and the config file, an error is returned. +func GetDataDirPreStart(fs filesys.Filesys, args []string) (string, error) { + ap := SqlServerCmd{}.ArgParser() + apr, err := cli.ParseArgs(ap, args, nil) + if err != nil { + // Parse failure at this stage is ignored. We'll handle it during command execution, to be more consistent with + // other commands. + return "", nil + } + + cliDataDir, hasDataDirCliArg := apr.GetValue(commands.DataDirFlag) + if hasDataDirCliArg { + cliDataDir, err = fs.Abs(cliDataDir) + if err != nil { + return "", err + } + } + + var cfgDataDir string + confArg, hasConfArg := apr.GetValue(configFileFlag) + if hasConfArg { + reader := DoltServerConfigReader{} + cfg, err := reader.ReadConfigFile(fs, confArg) + if err != nil { + return "", err + } + + if cfg.DataDir() != "" { + cfgDataDir, err = fs.Abs(cfg.DataDir()) + if err != nil { + return "", err + } + } + } + + if cfgDataDir != "" && cliDataDir != "" { + return "", errors.New("--data-dir specified in both config file and command line. Please specify only one.") + } + if cfgDataDir != "" { + return cfgDataDir, nil + } + if cliDataDir != "" { + return cliDataDir, nil + } + return "", nil +} + // ServerConfigFromArgs returns a ServerConfig from the given args -func ServerConfigFromArgs(ap *argparser.ArgParser, help cli.UsagePrinter, args []string, dEnv *env.DoltEnv) (servercfg.ServerConfig, error) { - return ServerConfigFromArgsWithReader(ap, help, args, dEnv, DoltServerConfigReader{}) +func ServerConfigFromArgs(ap *argparser.ArgParser, help cli.UsagePrinter, args []string, dEnv *env.DoltEnv, cwd filesys.Filesys) (servercfg.ServerConfig, error) { + return ServerConfigFromArgsWithReader(ap, help, args, dEnv, cwd, DoltServerConfigReader{}) } // ServerConfigFromArgsWithReader returns a ServerConfig from the given args, using the provided ServerConfigReader @@ -278,6 +333,7 @@ func ServerConfigFromArgsWithReader( help cli.UsagePrinter, args []string, dEnv *env.DoltEnv, + cwd filesys.Filesys, reader ServerConfigReader, ) (servercfg.ServerConfig, error) { apr := cli.ParseArgsOrDie(ap, args, help) @@ -286,12 +342,17 @@ func ServerConfigFromArgsWithReader( return nil, err } - serverConfig, err := getServerConfig(dEnv.FS, apr, reader) + dataDir, err := dEnv.FS.Abs("") + if err != nil { + return nil, err + } + + serverConfig, err := getServerConfig(cwd, apr, dataDir, reader) if err != nil { return nil, fmt.Errorf("bad configuration: %w", err) } - if err = setupDoltConfig(dEnv, apr, serverConfig); err != nil { + if err = setupDoltConfig(dEnv, cwd, apr, serverConfig); err != nil { return nil, fmt.Errorf("bad configuration: %w", err) } @@ -300,10 +361,10 @@ func ServerConfigFromArgsWithReader( // getServerConfig returns ServerConfig that is set either from yaml file if given, if not it is set with values defined // on command line. Server config variables not defined are set to default values. -func getServerConfig(cwdFS filesys.Filesys, apr *argparser.ArgParseResults, reader ServerConfigReader) (servercfg.ServerConfig, error) { +func getServerConfig(cwdFS filesys.Filesys, apr *argparser.ArgParseResults, dataDirOverride string, reader ServerConfigReader) (servercfg.ServerConfig, error) { cfgFile, ok := apr.GetValue(configFileFlag) if !ok { - return reader.ReadConfigArgs(apr) + return reader.ReadConfigArgs(apr, dataDirOverride) } cfg, err := reader.ReadConfigFile(cwdFS, cfgFile) @@ -338,7 +399,7 @@ func GetClientConfig(cwdFS filesys.Filesys, creds *cli.UserPassword, apr *argpar cfgFile, hasCfgFile := apr.GetValue(configFileFlag) if !hasCfgFile { - return NewCommandLineConfig(creds, apr) + return NewCommandLineConfig(creds, apr, "") } var yamlCfg servercfg.YAMLConfig @@ -363,7 +424,7 @@ func GetClientConfig(cwdFS filesys.Filesys, creds *cli.UserPassword, apr *argpar } // setupDoltConfig updates the given server config with where to create .doltcfg directory -func setupDoltConfig(dEnv *env.DoltEnv, apr *argparser.ArgParseResults, config servercfg.ServerConfig) error { +func setupDoltConfig(dEnv *env.DoltEnv, cwd filesys.Filesys, apr *argparser.ArgParseResults, config servercfg.ServerConfig) error { if _, ok := apr.GetValue(configFileFlag); ok { return nil } @@ -378,6 +439,13 @@ func setupDoltConfig(dEnv *env.DoltEnv, apr *argparser.ArgParseResults, config s dataDir := serverConfig.DataDir() cfgDir, cfgDirSpecified := apr.GetValue(commands.CfgDirFlag) if cfgDirSpecified { + if !filepath.IsAbs(cfgDir) { + var err error + cfgDir, err = cwd.Abs(cfgDir) + if err != nil { + return err + } + } cfgDirPath = cfgDir } else if dataDirSpecified { cfgDirPath = filepath.Join(dataDir, commands.DefaultCfgDirName) diff --git a/go/cmd/dolt/commands/utils.go b/go/cmd/dolt/commands/utils.go index 448d9a3a844..f1b9a55eed1 100644 --- a/go/cmd/dolt/commands/utils.go +++ b/go/cmd/dolt/commands/utils.go @@ -106,7 +106,7 @@ func MaybeGetCommitWithVErr(dEnv *env.DoltEnv, maybeCommit string) (*doltdb.Comm } // NewArgFreeCliContext creates a new CliContext instance with no arguments using a local SqlEngine. This is useful for testing primarily -func NewArgFreeCliContext(ctx context.Context, dEnv *env.DoltEnv) (cli.CliContext, errhand.VerboseError) { +func NewArgFreeCliContext(ctx context.Context, dEnv *env.DoltEnv, cwd filesys.Filesys) (cli.CliContext, errhand.VerboseError) { mrEnv, err := env.MultiEnvForSingleEnv(ctx, dEnv) if err != nil { return nil, errhand.VerboseErrorFromError(err) @@ -119,7 +119,7 @@ func NewArgFreeCliContext(ctx context.Context, dEnv *env.DoltEnv) (cli.CliContex if err != nil { return nil, verr } - return cli.NewCliContext(argparser.NewEmptyResults(), dEnv.Config, lateBind) + return cli.NewCliContext(argparser.NewEmptyResults(), dEnv.Config, cwd, lateBind) } // BuildSqlEngineQueryist Utility function to build a local SQLEngine for use interacting with data on disk using diff --git a/go/cmd/dolt/dolt.go b/go/cmd/dolt/dolt.go index 81a0d7460ea..9c89eb4fdbd 100644 --- a/go/cmd/dolt/dolt.go +++ b/go/cmd/dolt/dolt.go @@ -132,7 +132,6 @@ var doltSubCommands = []cli.Command{ var commandsWithoutCliCtx = []cli.Command{ admin.Commands, - sqlserver.SqlServerCmd{VersionStr: doltversion.Version}, commands.CloneCmd{}, commands.BackupCmd{}, commands.LoginCmd{}, @@ -431,7 +430,7 @@ func runMain() int { case featureVersionFlag: var err error if len(args) == 0 { - err = fmt.Errorf("missing argument for the --feature-version flag") + err = errors.New("missing argument for the --feature-version flag") } else { if featureVersion, err := strconv.Atoi(args[1]); err == nil { doltdb.DoltFeatureVersion = doltdb.FeatureVersion(featureVersion) @@ -468,53 +467,14 @@ func runMain() int { return exit } - _, usage := cli.HelpAndUsagePrinters(globalDocs) - - var fs filesys.Filesys - fs = filesys.LocalFS - apr, _, err := globalArgParser.ParseGlobalArgs(args) - if err == argparser.ErrHelp { - doltCommand.PrintUsage("dolt") - cli.Println(globalSpecialMsg) - usage() - return 0 - } else if err != nil { - cli.PrintErrln(color.RedString("Failure to parse global arguments: %v", err)) - return 1 - } - - dataDir, hasDataDir := apr.GetValue(commands.DataDirFlag) - if hasDataDir { - // If a relative path was provided, this ensures we have an absolute path everywhere. - dataDir, err = fs.Abs(dataDir) - if err != nil { - cli.PrintErrln(color.RedString("Failed to get absolute path for %s: %v", dataDir, err)) - return 1 - } - if ok, dir := fs.Exists(dataDir); !ok || !dir { - cli.Println(color.RedString("Provided data directory does not exist: %s", dataDir)) - return 1 - } - } - - // Current working directory is preserved to ensure that user provided path arguments are always calculated - // relative to this directory. The root environment's FS will be updated to be the --data-dir path if the user - // specified one. - cwdFS := fs - dataDirFS, err := fs.WithWorkingDir(dataDir) - if err != nil { - cli.PrintErrln(color.RedString("Failed to set the data directory. %v", err)) - return 1 - } - - dEnv := env.Load(ctx, env.GetCurrentUserHomeDir, dataDirFS, doltdb.LocalDirDoltDB, doltversion.Version) - - homeDir, err := env.GetCurrentUserHomeDir() - if err != nil { - cli.PrintErrln(color.RedString("Failed to load the HOME directory: %v", err)) - return 1 + cfg, terminate, status := createBootstrapConfig(ctx, args) + if terminate { + return status } + args = nil + // This is the dEnv passed to sub-commands, and is used to create the multi-repo environment. + dEnv := env.Load(ctx, env.GetCurrentUserHomeDir, cfg.dataDirFS, doltdb.LocalDirDoltDB, doltversion.Version) if dEnv.CfgLoadErr != nil { cli.PrintErrln(color.RedString("Failed to load the global config. %v", dEnv.CfgLoadErr)) return 1 @@ -522,7 +482,7 @@ func runMain() int { strMetricsDisabled := dEnv.Config.GetStringOrDefault(config.MetricsDisabled, "false") var metricsEmitter events.Emitter - metricsEmitter = events.NewFileEmitter(homeDir, dbfactory.DoltDir) + metricsEmitter = events.NewFileEmitter(cfg.homeDir, dbfactory.DoltDir) metricsDisabled, err := strconv.ParseBool(strMetricsDisabled) if err != nil || metricsDisabled { metricsEmitter = events.NullEmitter{} @@ -530,20 +490,6 @@ func runMain() int { events.SetGlobalCollector(events.NewCollector(doltversion.Version, metricsEmitter)) - globalConfig, ok := dEnv.Config.GetConfig(env.GlobalConfig) - if !ok { - cli.PrintErrln(color.RedString("Failed to get global config")) - return 1 - } - - globalConfig.Iter(func(name, val string) (stop bool) { - option := strings.ToLower(name) - if _, ok := config.ConfigOptions[option]; !ok && !strings.HasPrefix(option, env.SqlServerGlobalsPrefix) { - cli.PrintErrf("Warning: Unknown global config option '%s'. Use `dolt config --global --unset %s` to remove.\n", name, name) - } - return false - }) - // try verifying contents of local config localConfig, ok := dEnv.Config.GetConfig(env.LocalConfig) if ok { @@ -556,29 +502,16 @@ func runMain() int { }) } - apr, remainingArgs, subcommandName, err := parseGlobalArgsAndSubCommandName(globalConfig, args) - if err == argparser.ErrHelp { - doltCommand.PrintUsage("dolt") - cli.Println(globalSpecialMsg) - usage() + defer emitUsageEvents(metricsEmitter, cfg.subCommand) - return 0 - } else if err != nil { - cli.PrintErrln(color.RedString("Failure to parse arguments: %v", err)) - return 1 - } - - defer emitUsageEvents(metricsEmitter, args) - - if needsWriteAccess(subcommandName) { - err = reconfigIfTempFileMoveFails(dEnv) + if needsWriteAccess(cfg.subCommand) { + err = reconfigIfTempFileMoveFails(cfg.dataDirFS) if err != nil { cli.PrintErrln(color.RedString("Failed to setup the temporary directory. %v`", err)) return 1 } } - defer tempfiles.MovableTempFileProvider.Clean() // Find all database names and add global variables for them. This needs to @@ -598,7 +531,7 @@ func runMain() int { // variables like `${db_name}_default_branch` (maybe these should not be // part of Dolt config in the first place!). - mrEnv, err := env.MultiEnvForDirectory(ctx, dEnv.Config.WriteableConfig(), dataDirFS, dEnv.Version, dEnv) + mrEnv, err := env.MultiEnvForDirectory(ctx, dEnv.Config.WriteableConfig(), cfg.dataDirFS, dEnv.Version, dEnv) if err != nil { cli.PrintErrln("failed to load database names") return 1 @@ -615,30 +548,30 @@ func runMain() int { } var cliCtx cli.CliContext = nil - if initCliContext(subcommandName) { + if initCliContext(cfg.subCommand) { // validate that --user and --password are set appropriately. - aprAlt, creds, err := cli.BuildUserPasswordPrompt(apr) - apr = aprAlt + aprAlt, creds, err := cli.BuildUserPasswordPrompt(cfg.apr) if err != nil { cli.PrintErrln(color.RedString("Failed to parse credentials: %v", err)) return 1 } + cfg.apr = aprAlt - lateBind, err := buildLateBinder(ctx, cwdFS, dEnv, mrEnv, creds, apr, subcommandName, verboseEngineSetup) + lateBind, err := buildLateBinder(ctx, cfg.cwdFs, dEnv, mrEnv, creds, cfg.apr, cfg.subCommand, verboseEngineSetup) if err != nil { cli.PrintErrln(color.RedString("%v", err)) return 1 } - cliCtx, err = cli.NewCliContext(apr, dEnv.Config, lateBind) + cliCtx, err = cli.NewCliContext(cfg.apr, dEnv.Config, cfg.cwdFs, lateBind) if err != nil { cli.PrintErrln(color.RedString("Unexpected Error: %v", err)) return 1 } } else { - if args[0] != subcommandName { - if supportsGlobalArgs(subcommandName) { + if cfg.hasGlobalArgs { + if supportsGlobalArgs(cfg.subCommand) { cli.PrintErrln( `Global arguments are not supported for this command as it has not yet been migrated to function in a remote context. If you're interested in running this command against a remote host, hit us up on discord (https://discord.gg/gqr7K4VNKe).`) @@ -652,7 +585,7 @@ or check the docs for questions about usage.`) } ctx, stop := context.WithCancel(ctx) - res := doltCommand.Exec(ctx, "dolt", remainingArgs, dEnv, cliCtx) + res := doltCommand.Exec(ctx, "dolt", cfg.remainingArgs, dEnv, cliCtx) stop() if err = dbfactory.CloseAllLocalDatabases(); err != nil { @@ -671,6 +604,46 @@ or check the docs for questions about usage.`) return res } +// resolveDataDirDeeply goes through three levels of resolution for the data directory. The simple case is to look at +// the --data-dir flag which was provide before the sub-command. When runing sql-server, the data directory can also +// be specificed in the arguments after the sub-command, and the config file. This method will ensure there is only +// one of these three options specified, and return it as an absolute path. If there is an error, or if there are multiple +// options specified, an error is returned. +func resolveDataDirDeeply(gArgs *argparser.ArgParseResults, subCmd string, remainingArgs []string, cwdFs filesys.Filesys) (dataDir string, err error) { + // global config is the dolt --data-dir sub-command version. Applies to most CLI commands. + globalDir, hasGlobalDataDir := gArgs.GetValue(commands.DataDirFlag) + if hasGlobalDataDir { + // If a relative path was provided, this ensures we have an absolute path everywhere. + dd, err := cwdFs.Abs(globalDir) + if err != nil { + return "", errors.New(fmt.Sprintf("Failed to get absolute path for %s: %v", dataDir, err)) + } + dataDir = dd + } + + if subCmd == (sqlserver.SqlServerCmd{}).Name() { + // GetDataDirPreStart always returns an absolute path. + dd, err := sqlserver.GetDataDirPreStart(cwdFs, remainingArgs) + if err != nil { + return "", err + } + + if dd != "" { + if hasGlobalDataDir { + return "", errors.New("cannot specify both global --data-dir argument and --data-dir in sql-server config. Please specify only one.") + } + dataDir = dd + } + } + + if dataDir == "" { + // No data dir specified, so we default to the current directory. + return cwdFs.Abs("") + } + + return dataDir, nil +} + // buildLateBinder builds a LateBindQueryist for which is used to obtain the Queryist used for the length of the // command execution. func buildLateBinder(ctx context.Context, cwdFS filesys.Filesys, rootEnv *env.DoltEnv, mrEnv *env.MultiRepoEnv, creds *cli.UserPassword, apr *argparser.ArgParseResults, subcommandName string, verbose bool) (cli.LateBindQueryist, error) { @@ -742,12 +715,12 @@ If you're interested in running this command against a remote host, hit us up on if noValidRepository && isValidRepositoryRequired { return func(ctx context.Context) (cli.Queryist, *sql.Context, func(), error) { - err := fmt.Errorf("The current directory is not a valid dolt repository.") + err := errors.New("The current directory is not a valid dolt repository.") if errors.Is(rootEnv.DBLoadError, nbs.ErrUnsupportedTableFileFormat) { // This is fairly targeted and specific to allow for better error messaging. We should consider // breaking this out into its own function if we add more conditions. - err = fmt.Errorf("The data in this database is in an unsupported format. Please upgrade to the latest version of Dolt.") + err = errors.New("The data in this database is in an unsupported format. Please upgrade to the latest version of Dolt.") } return nil, nil, nil, err @@ -827,14 +800,14 @@ func seedGlobalRand() { // 1. The config key |metrics.disabled|, when set to |true|, disables all metrics emission // 2. The environment key |DOLT_DISABLE_EVENT_FLUSH| allows writing events to disk but not sending them to the server. // This is mostly used for testing. -func emitUsageEvents(emitter events.Emitter, args []string) { +func emitUsageEvents(emitter events.Emitter, subCmd string) { // write events collector := events.GlobalCollector() ctx := context.Background() _ = emitter.LogEvents(ctx, doltversion.Version, collector.Close()) // flush events - if !eventFlushDisabled && len(args) > 0 && shouldFlushEvents(args[0]) { + if !eventFlushDisabled && shouldFlushEvents(subCmd) { _ = flushEventsDir() } } @@ -879,66 +852,170 @@ func interceptSendMetrics(ctx context.Context, args []string) (bool, int) { return true, doltCommand.Exec(ctx, "dolt", args, dEnv, nil) } -// parseGlobalArgsAndSubCommandName parses the global arguments, including a profile if given or a default profile if exists. Also returns the subcommand name. -func parseGlobalArgsAndSubCommandName(globalConfig config.ReadWriteConfig, args []string) (apr *argparser.ArgParseResults, remaining []string, subcommandName string, err error) { - apr, remaining, err = globalArgParser.ParseGlobalArgs(args) +// bootstrapConfig is the struct that holds the parsed arguments and other configurations. Most importantly, is holds +// the data dir information for the process. There are multiple ways for users to specify the data directory, and constructing +// the bootstrap config early in the process start up allows us to simplify the startup process. +type bootstrapConfig struct { + // apr is the parsed global arguments. This will not include hidden administrative arguments, which are pulled out first. + // This APR will include profile injected arguments. + apr *argparser.ArgParseResults + // hasGlobalArgs is true if the global arguments were provided. + hasGlobalArgs bool + + // dataDir is the absolute path to the data directory. This should be the final word - uses of dataDir downstream should + // only use this path. + dataDir string + // dataDirFS is the filesys.Filesys for the data directory. + dataDirFS filesys.Filesys + // cwdFs is the filesys.Filesys where the process started. Used when there are relative path arguments provided by the user. + cwdFs filesys.Filesys + + // remainingArgs is the remaining arguments after parsing global arguments. This includes the subCommand at location 0 + // always. + remainingArgs []string + subCommand string + + // homeDir is the absolute path to the user's home directory. This is resolved using env.GetCurrentUserHomeDir, and saved + homeDir string +} + +// createBootstrapConfig parses the global arguments, inspects current working directory, loads the profile, and +// even digs into server config to build the bootstrapConfig struct. If all goes well, |cfg| is set to a struct that +// contains all the parsed arguments and other configurations. If there is an error, |cfg| will be nil. +// |terminate| is set to true if the process should end for any reason. Errors or messages to the user will be printed already. +// |status| is the exit code to terminate with, and can be ignored if |terminate| is false. +func createBootstrapConfig(ctx context.Context, args []string) (cfg *bootstrapConfig, terminate bool, status int) { + lfs := filesys.LocalFS + cwd, err := lfs.Abs("") + cwdFs, err := lfs.WithWorkingDir(cwd) if err != nil { - return nil, nil, "", err + cli.PrintErrln(color.RedString("Failed to load the current working directory: %v", err)) + return nil, true, 1 + } + + tmpEnv := env.LoadWithoutDB(ctx, env.GetCurrentUserHomeDir, cwdFs, doltversion.Version) + var globalConfig config.ReadWriteConfig + + homeDir, err := env.GetCurrentUserHomeDir() + if err != nil { + cli.PrintErrln(color.RedString("Failed to load the HOME directory: %v", err)) + return nil, true, 1 + } + + if tmpEnv.CfgLoadErr != nil { + cli.PrintErrln(color.RedString("Failed to load the global config: %v", tmpEnv.CfgLoadErr)) + return nil, true, 1 + } + + if tmpEnv.Config != nil { + var ok bool + globalConfig, ok = tmpEnv.Config.GetConfig(env.GlobalConfig) + if !ok { + cli.PrintErrln(color.RedString("Failed to load the global config")) + return nil, true, 1 + } + } else { + panic("runtime error. tmpEnv.Config is nil by no tmpEnv.CfgLoadErr set") + } + + globalConfig.Iter(func(name, val string) (stop bool) { + option := strings.ToLower(name) + if _, ok := config.ConfigOptions[option]; !ok && !strings.HasPrefix(option, env.SqlServerGlobalsPrefix) { + cli.PrintErrf("Warning: Unknown global config option '%s'. Use `dolt config --global --unset %s` to remove.\n", name, name) + } + return false + }) + + _, usage := cli.HelpAndUsagePrinters(globalDocs) + apr, remainingArgs, err := globalArgParser.ParseGlobalArgs(args) + if errors.Is(err, argparser.ErrHelp) { + doltCommand.PrintUsage("dolt") + cli.Println(globalSpecialMsg) + usage() + return nil, true, 0 + } else if err != nil { + cli.PrintErrln(color.RedString("Failed to parse global arguments: %v", err)) + return nil, true, 1 } - subcommandName = remaining[0] + hasGlobalArgs := false + if len(remainingArgs) != len(args) { + hasGlobalArgs = true + } + + subCommand := remainingArgs[0] + // If there is a profile flag, we want to load the profile and inject it's args into the global args. useDefaultProfile := false profileName, hasProfile := apr.GetValue(commands.ProfileFlag) encodedProfiles, err := globalConfig.GetString(commands.GlobalCfgProfileKey) if err != nil { if err == config.ErrConfigParamNotFound { if hasProfile { - return nil, nil, "", fmt.Errorf("no profiles found") + cli.PrintErrln(color.RedString("Unable to load profile: %s. Not found.", profileName)) + return nil, true, 1 } else { - return apr, remaining, subcommandName, nil + // We done. Jump to returning what we have. } } else { - return nil, nil, "", err + cli.Println(color.RedString("Failed to retrieve config key: %v", err)) + return nil, true, 1 } } - profiles, err := commands.DecodeProfile(encodedProfiles) + profilesJson, err := commands.DecodeProfile(encodedProfiles) if err != nil { - return nil, nil, "", err + cli.PrintErrln(color.RedString("Failed to decode profiles: %v", err)) + return nil, true, 1 } - if !hasProfile && supportsGlobalArgs(subcommandName) { - defaultProfile := gjson.Get(profiles, commands.DefaultProfileName) + if !hasProfile && supportsGlobalArgs(subCommand) { + defaultProfile := gjson.Get(profilesJson, commands.DefaultProfileName) if defaultProfile.Exists() { - args = append([]string{"--profile", commands.DefaultProfileName}, args...) - apr, remaining, err = globalArgParser.ParseGlobalArgs(args) - if err != nil { - return nil, nil, "", err - } - profileName, _ = apr.GetValue(commands.ProfileFlag) + profileName = commands.DefaultProfileName useDefaultProfile = true } } if hasProfile || useDefaultProfile { - profileArgs, err := getProfile(apr, profileName, profiles) - if err != nil { - return nil, nil, "", err - } - args = append(profileArgs, args...) - apr, remaining, err = globalArgParser.ParseGlobalArgs(args) + apr, err = injectProfileArgs(apr, profileName, profilesJson) if err != nil { - return nil, nil, "", err + cli.PrintErrln(color.RedString("Failed to inject profile arguments: %v", err)) + return nil, true, 1 } } - return + dataDir, err := resolveDataDirDeeply(apr, subCommand, remainingArgs[1:], cwdFs) + if err != nil { + cli.PrintErrln(color.RedString("Failed to resolve the data directory: %v", err)) + return nil, true, 1 + } + + dataDirFS, err := cwdFs.WithWorkingDir(dataDir) + if err != nil { + cli.PrintErrln(color.RedString("Failed to set the data directory to: %s. %v", dataDir, err)) + return nil, true, 1 + } + + cfg = &bootstrapConfig{ + apr: apr, + hasGlobalArgs: hasGlobalArgs, + remainingArgs: remainingArgs, + dataDirFS: dataDirFS, + dataDir: dataDir, + cwdFs: cwdFs, + subCommand: subCommand, + homeDir: homeDir, + } + + return cfg, false, 0 } -// getProfile retrieves the given profile from the provided list of profiles and returns the args (as flags) and values -// for that profile in a []string. If the profile is not found, an error is returned. -func getProfile(apr *argparser.ArgParseResults, profileName, profiles string) (result []string, err error) { - prof := gjson.Get(profiles, profileName) +// injectProfileArgs retrieves the given |profileName| from the provided |profilesJson| and inject the profile details +// in the provided |apr|. A new ArgParseResults is returned which contains the profile details. If the profile is not +// found, an error is returned. +func injectProfileArgs(apr *argparser.ArgParseResults, profileName, profilesJson string) (aprUpdated *argparser.ArgParseResults, err error) { + prof := gjson.Get(profilesJson, profileName) + aprUpdated = apr if prof.Exists() { hasPassword := false password := "" @@ -950,20 +1027,29 @@ func getProfile(apr *argparser.ArgParseResults, profileName, profiles string) (r hasPassword = value.Bool() } else if flag == cli.NoTLSFlag { if value.Bool() { - result = append(result, "--"+flag) - continue + // There is currently no way to unset a flag, but setting is to the empty string at least sets it to true. + aprUpdated, err = aprUpdated.SetArgument(flag, "") + if err != nil { + return nil, err + } } } else { if value.Str != "" { - result = append(result, "--"+flag, value.Str) + aprUpdated, err = aprUpdated.SetArgument(flag, value.Str) + if err != nil { + return nil, err + } } } } } if !apr.Contains(cli.PasswordFlag) && hasPassword { - result = append(result, "--"+cli.PasswordFlag, password) + aprUpdated, err = aprUpdated.SetArgument(cli.PasswordFlag, password) + if err != nil { + return nil, err + } } - return result, nil + return aprUpdated, nil } else { return nil, fmt.Errorf("profile %s not found", profileName) } diff --git a/go/cmd/dolt/doltversion/version.go b/go/cmd/dolt/doltversion/version.go index a70bf8ad8d9..ce8d9e91efa 100644 --- a/go/cmd/dolt/doltversion/version.go +++ b/go/cmd/dolt/doltversion/version.go @@ -16,5 +16,5 @@ package doltversion const ( - Version = "1.44.2" + Version = "1.45.2" ) diff --git a/go/cmd/dolt/system_checks.go b/go/cmd/dolt/system_checks.go index 99fb1a7137a..bffeff96d1b 100644 --- a/go/cmd/dolt/system_checks.go +++ b/go/cmd/dolt/system_checks.go @@ -17,63 +17,83 @@ package main import ( "fmt" "os" + "path/filepath" + "github.com/dolthub/dolt/go/libraries/doltcore/dbfactory" "github.com/dolthub/dolt/go/libraries/doltcore/env" "github.com/dolthub/dolt/go/libraries/utils/file" + "github.com/dolthub/dolt/go/libraries/utils/filesys" "github.com/dolthub/dolt/go/store/util/tempfiles" ) -// returns false if it fails to verify that it can move files from the default temp directory to the local directory. -func canMoveTempFile() bool { - const testfile = "./testfile" +// reconfigIfTempFileMoveFails checks to see if the file system used for the data directory supports moves from TMPDIR. +// If this is not possible, we can't perform atomic moves of storage files, so we force the temp dir to be in the datadir +// to assure they are on the same file system. +func reconfigIfTempFileMoveFails(dataDir filesys.Filesys) error { + absP, err := dataDir.Abs("") + if err != nil { + return err + } - f, err := os.CreateTemp("", "") + dotDoltCreated := false + tmpDirCreated := false + doltDir := filepath.Join(absP, dbfactory.DoltDir) + stat, err := os.Stat(doltDir) if err != nil { - return false - } + err := os.MkdirAll(doltDir, os.ModePerm) + if err != nil { + return fmt.Errorf("failed to create dolt dir '%s': %s", doltDir, err.Error()) + } - name := f.Name() - err = f.Close() + dotDoltCreated = true + } + doltTmpDir := filepath.Join(doltDir, env.TmpDirName) + stat, err = os.Stat(doltTmpDir) if err != nil { - return false + err := os.MkdirAll(doltTmpDir, os.ModePerm) + if err != nil { + return fmt.Errorf("failed to create temp dir '%s': %s", doltTmpDir, err.Error()) + } + tmpDirCreated = true + + } else if !stat.IsDir() { + return fmt.Errorf("attempting to use '%s' as a temp directory, but there exists a file with that name", doltTmpDir) } - err = file.Rename(name, testfile) + tmpF, err := os.CreateTemp("", "") + if err != nil { + return err + } + name := tmpF.Name() + err = tmpF.Close() if err != nil { - _ = file.Remove(name) - return false + return err } - _ = file.Remove(testfile) - return true -} + movedName := filepath.Join(doltTmpDir, "testfile") -// If we cannot verify that we can move files for any reason, use a ./.dolt/tmp as the temp dir. -func reconfigIfTempFileMoveFails(dEnv *env.DoltEnv) error { - if !canMoveTempFile() { - tmpDir := "./.dolt/tmp" + err = file.Rename(name, movedName) + if err == nil { + // If rename was successful, then the tmp dir is fine, so no need to change it. Clean up the things we created. + _ = file.Remove(movedName) - if !dEnv.HasDoltDir() { - tmpDir = "./.tmp" + if tmpDirCreated { + _ = file.Remove(doltTmpDir) } - stat, err := os.Stat(tmpDir) - - if err != nil { - err := os.MkdirAll(tmpDir, os.ModePerm) - - if err != nil { - return fmt.Errorf("failed to create temp dir '%s': %s", tmpDir, err.Error()) - } - } else if !stat.IsDir() { - return fmt.Errorf("attempting to use '%s' as a temp directory, but there exists a file with that name", tmpDir) + if dotDoltCreated { + _ = file.Remove(doltDir) } - tempfiles.MovableTempFileProvider = tempfiles.NewTempFileProviderAt(tmpDir) + return nil } + _ = file.Remove(name) + + // Rename failed. So we force the tmp dir to be the data dir. + tempfiles.MovableTempFileProvider = tempfiles.NewTempFileProviderAt(doltTmpDir) return nil } diff --git a/go/gen/fb/serial/fileidentifiers.go b/go/gen/fb/serial/fileidentifiers.go index 64cb36d9f47..628cc4f99b8 100644 --- a/go/gen/fb/serial/fileidentifiers.go +++ b/go/gen/fb/serial/fileidentifiers.go @@ -41,6 +41,7 @@ const StashListFileID = "SLST" const StashFileID = "STSH" const StatisticFileID = "STAT" const DoltgresRootValueFileID = "DGRV" +const TupleFileID = "TUPL" const MessageTypesKind int = 27 diff --git a/go/gen/fb/serial/tuple.go b/go/gen/fb/serial/tuple.go new file mode 100644 index 00000000000..6ca885ecdff --- /dev/null +++ b/go/gen/fb/serial/tuple.go @@ -0,0 +1,102 @@ +// Copyright 2022-2023 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package serial + +import ( + flatbuffers "github.com/dolthub/flatbuffers/v23/go" +) + +type Tuple struct { + _tab flatbuffers.Table +} + +func InitTupleRoot(o *Tuple, buf []byte, offset flatbuffers.UOffsetT) error { + n := flatbuffers.GetUOffsetT(buf[offset:]) + return o.Init(buf, n+offset) +} + +func TryGetRootAsTuple(buf []byte, offset flatbuffers.UOffsetT) (*Tuple, error) { + x := &Tuple{} + return x, InitTupleRoot(x, buf, offset) +} + +func TryGetSizePrefixedRootAsTuple(buf []byte, offset flatbuffers.UOffsetT) (*Tuple, error) { + x := &Tuple{} + return x, InitTupleRoot(x, buf, offset+flatbuffers.SizeUint32) +} + +func (rcv *Tuple) Init(buf []byte, i flatbuffers.UOffsetT) error { + rcv._tab.Bytes = buf + rcv._tab.Pos = i + if TupleNumFields < rcv.Table().NumFields() { + return flatbuffers.ErrTableHasUnknownFields + } + return nil +} + +func (rcv *Tuple) Table() flatbuffers.Table { + return rcv._tab +} + +func (rcv *Tuple) Value(j int) byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + a := rcv._tab.Vector(o) + return rcv._tab.GetByte(a + flatbuffers.UOffsetT(j*1)) + } + return 0 +} + +func (rcv *Tuple) ValueLength() int { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + return rcv._tab.VectorLen(o) + } + return 0 +} + +func (rcv *Tuple) ValueBytes() []byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + return rcv._tab.ByteVector(o + rcv._tab.Pos) + } + return nil +} + +func (rcv *Tuple) MutateValue(j int, n byte) bool { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + a := rcv._tab.Vector(o) + return rcv._tab.MutateByte(a+flatbuffers.UOffsetT(j*1), n) + } + return false +} + +const TupleNumFields = 1 + +func TupleStart(builder *flatbuffers.Builder) { + builder.StartObject(TupleNumFields) +} +func TupleAddValue(builder *flatbuffers.Builder, value flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(0, flatbuffers.UOffsetT(value), 0) +} +func TupleStartValueVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { + return builder.StartVector(1, numElems, 1) +} +func TupleEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/go/go.mod b/go/go.mod index efe0d1cb90d..22588da0edb 100644 --- a/go/go.mod +++ b/go/go.mod @@ -15,7 +15,7 @@ require ( github.com/dolthub/fslock v0.0.3 github.com/dolthub/ishell v0.0.0-20240701202509-2b217167d718 github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81 - github.com/dolthub/vitess v0.0.0-20241211024425-b00987f7ba54 + github.com/dolthub/vitess v0.0.0-20241231200706-18992bb25fdc github.com/dustin/go-humanize v1.0.1 github.com/fatih/color v1.13.0 github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 @@ -38,10 +38,10 @@ require ( github.com/stretchr/testify v1.9.0 github.com/tealeg/xlsx v1.0.5 go.uber.org/zap v1.24.0 - golang.org/x/crypto v0.23.0 + golang.org/x/crypto v0.31.0 golang.org/x/net v0.25.0 - golang.org/x/sync v0.7.0 - golang.org/x/sys v0.27.0 + golang.org/x/sync v0.10.0 + golang.org/x/sys v0.28.0 google.golang.org/api v0.126.0 google.golang.org/grpc v1.57.1 google.golang.org/protobuf v1.31.0 @@ -53,11 +53,10 @@ require ( github.com/Shopify/toxiproxy/v2 v2.5.0 github.com/aliyun/aliyun-oss-go-sdk v2.2.5+incompatible github.com/cenkalti/backoff/v4 v4.1.3 - github.com/cespare/xxhash v1.1.0 github.com/cespare/xxhash/v2 v2.2.0 github.com/creasty/defaults v1.6.0 github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2 - github.com/dolthub/go-mysql-server v0.18.2-0.20241211025720-09a7e80b835c + github.com/dolthub/go-mysql-server v0.19.1-0.20241231201953-4af32d139747 github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63 github.com/dolthub/swiss v0.1.0 github.com/goccy/go-json v0.10.2 @@ -88,7 +87,7 @@ require ( go.opentelemetry.io/otel/sdk v1.32.0 go.opentelemetry.io/otel/trace v1.32.0 golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 - golang.org/x/text v0.16.0 + golang.org/x/text v0.21.0 gonum.org/v1/plot v0.11.0 gopkg.in/errgo.v2 v2.1.0 gopkg.in/go-jose/go-jose.v2 v2.6.3 @@ -108,7 +107,7 @@ require ( github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/dolthub/go-icu-regex v0.0.0-20240916130659-0118adc6b662 // indirect + github.com/dolthub/go-icu-regex v0.0.0-20241215010122-db690dd53c90 // indirect github.com/dolthub/jsonpath v0.0.2-0.20240227200619-19675ab05c71 // indirect github.com/dolthub/maphash v0.0.0-20221220182448-74e1e1ea1577 // indirect github.com/go-fonts/liberation v0.2.0 // indirect @@ -144,7 +143,7 @@ require ( github.com/prometheus/procfs v0.8.0 // indirect github.com/rs/xid v1.4.0 // indirect github.com/sony/gobreaker v0.5.0 // indirect - github.com/tetratelabs/wazero v1.1.0 // indirect + github.com/tetratelabs/wazero v1.8.2 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect github.com/tklauser/go-sysconf v0.3.9 // indirect @@ -157,7 +156,7 @@ require ( golang.org/x/image v0.18.0 // indirect golang.org/x/mod v0.17.0 // indirect golang.org/x/oauth2 v0.8.0 // indirect - golang.org/x/term v0.20.0 // indirect + golang.org/x/term v0.27.0 // indirect golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect diff --git a/go/go.sum b/go/go.sum index dbaf8f15525..55f9e7af7fa 100644 --- a/go/go.sum +++ b/go/go.sum @@ -62,8 +62,6 @@ github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q github.com/HdrHistogram/hdrhistogram-go v1.1.2 h1:5IcZpTvzydCQeHzK4Ef/D5rrSqwxob0t8PQPMybUNFM= github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= -github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/Shopify/toxiproxy/v2 v2.5.0 h1:i4LPT+qrSlKNtQf5QliVjdP08GyAH8+BUIc9gT0eahc= @@ -138,8 +136,6 @@ github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QH github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= @@ -181,10 +177,10 @@ github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2 h1:u3PMzfF8RkKd3lB9pZ2bfn0qEG+1G github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2/go.mod h1:mIEZOHnFx4ZMQeawhw9rhsj+0zwQj7adVsnBX7t+eKY= github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U= github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0= -github.com/dolthub/go-icu-regex v0.0.0-20240916130659-0118adc6b662 h1:aC17hZD6iwzBwwfO5M+3oBT5E5gGRiQPdn+vzpDXqIA= -github.com/dolthub/go-icu-regex v0.0.0-20240916130659-0118adc6b662/go.mod h1:KPUcpx070QOfJK1gNe0zx4pA5sicIK1GMikIGLKC168= -github.com/dolthub/go-mysql-server v0.18.2-0.20241211025720-09a7e80b835c h1:U5IPpq8pPDELi2SCj/SjTXGvBWVpYJ26Q7O7oC5NQd4= -github.com/dolthub/go-mysql-server v0.18.2-0.20241211025720-09a7e80b835c/go.mod h1:QedjnglQ9+7yeRz4jAseFVZZgTxIZhEdt9G4RGqv4Jc= +github.com/dolthub/go-icu-regex v0.0.0-20241215010122-db690dd53c90 h1:Sni8jrP0sy/w9ZYXoff4g/ixe+7bFCZlfCqXKJSU+zM= +github.com/dolthub/go-icu-regex v0.0.0-20241215010122-db690dd53c90/go.mod h1:ylU4XjUpsMcvl/BKeRRMXSH7e7WBrPXdSLvnRJYrxEA= +github.com/dolthub/go-mysql-server v0.19.1-0.20241231201953-4af32d139747 h1:1ilNzgumvHqWf0YhOoUNaycJH5RvtSgHLPpOzkQbhQk= +github.com/dolthub/go-mysql-server v0.19.1-0.20241231201953-4af32d139747/go.mod h1:ycL4Q6pJ5b+D8fnOa6GRmoB6PmvHAYn2w/IHYPbcQR0= github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63 h1:OAsXLAPL4du6tfbBgK0xXHZkOlos63RdKYS3Sgw/dfI= github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63/go.mod h1:lV7lUeuDhH5thVGDCKXbatwKy2KW80L4rMT46n+Y2/Q= github.com/dolthub/ishell v0.0.0-20240701202509-2b217167d718 h1:lT7hE5k+0nkBdj/1UOSFwjWpNxf+LCApbRHgnCA17XE= @@ -197,8 +193,8 @@ github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81 h1:7/v8q9X github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81/go.mod h1:siLfyv2c92W1eN/R4QqG/+RjjX5W2+gCTRjZxBjI3TY= github.com/dolthub/swiss v0.1.0 h1:EaGQct3AqeP/MjASHLiH6i4TAmgbG/c4rA6a1bzCOPc= github.com/dolthub/swiss v0.1.0/go.mod h1:BeucyB08Vb1G9tumVN3Vp/pyY4AMUnr9p7Rz7wJ7kAQ= -github.com/dolthub/vitess v0.0.0-20241211024425-b00987f7ba54 h1:nzBnC0Rt1gFtscJEz4veYd/mazZEdbdmed+tujdaKOo= -github.com/dolthub/vitess v0.0.0-20241211024425-b00987f7ba54/go.mod h1:1gQZs/byeHLMSul3Lvl3MzioMtOW1je79QYGyi2fd70= +github.com/dolthub/vitess v0.0.0-20241231200706-18992bb25fdc h1:3FuwEDwyue/JuHdnwGSbQhE9xKAFM+k1y3uXi58h7Gk= +github.com/dolthub/vitess v0.0.0-20241231200706-18992bb25fdc/go.mod h1:1gQZs/byeHLMSul3Lvl3MzioMtOW1je79QYGyi2fd70= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= @@ -641,8 +637,6 @@ github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4k github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/sony/gobreaker v0.5.0 h1:dRCvqm0P490vZPmy7ppEk2qCnCieBooFJ+YoXGYB+yg= github.com/sony/gobreaker v0.5.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= @@ -671,8 +665,8 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tealeg/xlsx v1.0.5 h1:+f8oFmvY8Gw1iUXzPk+kz+4GpbDZPK1FhPiQRd+ypgE= github.com/tealeg/xlsx v1.0.5/go.mod h1:btRS8dz54TDnvKNosuAqxrM1QgN1udgk9O34bDCnORM= -github.com/tetratelabs/wazero v1.1.0 h1:EByoAhC+QcYpwSZJSs/aV0uokxPwBgKxfiokSUwAknQ= -github.com/tetratelabs/wazero v1.1.0/go.mod h1:wYx2gNRg8/WihJfSDxA1TIL8H+GkfLYm+bIfbblu9VQ= +github.com/tetratelabs/wazero v1.8.2 h1:yIgLR/b2bN31bjxwXHD8a3d+BogigR952csSDdLYEv4= +github.com/tetratelabs/wazero v1.8.2/go.mod h1:yAI0XTsMBhREkM/YDAK/zNou3GoiAce1P6+rp/wQhjs= github.com/thepudds/swisstable v0.0.0-20221011152303-9c77dc657777 h1:5u+6YWU2faS+Sr/x8j9yalMpSDUkatNOZWXV3wMUCGQ= github.com/thepudds/swisstable v0.0.0-20221011152303-9c77dc657777/go.mod h1:4af3KxEsswy6aTzsTcwa8QZUSh4V+80oHdp1QX9uJHA= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= @@ -775,8 +769,8 @@ golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWP golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= -golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -890,8 +884,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -955,13 +949,13 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= -golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= -golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -970,8 +964,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/go/go.work.sum b/go/go.work.sum index 529930d78a6..7203a8e7f6f 100644 --- a/go/go.work.sum +++ b/go/go.work.sum @@ -679,6 +679,7 @@ go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v8 go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/exp/typeparams v0.0.0-20221208152030-732eee02a75a/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= @@ -697,6 +698,7 @@ golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2 h1:IRJeR9r1pYWsHKTRe/IInb7lYvbBVIqOgsX/u0mbOWY= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= diff --git a/go/libraries/doltcore/doltdb/commit_hooks_test.go b/go/libraries/doltcore/doltdb/commit_hooks_test.go index e94894acd3c..fa20ac9c52e 100644 --- a/go/libraries/doltcore/doltdb/commit_hooks_test.go +++ b/go/libraries/doltcore/doltdb/commit_hooks_test.go @@ -251,7 +251,7 @@ func TestAsyncPushOnWrite(t *testing.T) { assert.NoError(t, err) tSchema := createTestSchema(t) - rowData, err := durable.NewEmptyIndex(ctx, ddb.vrw, ddb.ns, tSchema, false) + rowData, err := durable.NewEmptyPrimaryIndex(ctx, ddb.vrw, ddb.ns, tSchema) require.NoError(t, err) tbl, err := CreateTestTable(ddb.vrw, ddb.ns, tSchema, rowData) require.NoError(t, err) diff --git a/go/libraries/doltcore/doltdb/doltdb.go b/go/libraries/doltcore/doltdb/doltdb.go index 5ac905c0fc4..133d74ea7da 100644 --- a/go/libraries/doltcore/doltdb/doltdb.go +++ b/go/libraries/doltcore/doltdb/doltdb.go @@ -1066,6 +1066,7 @@ func (ddb *DoltDB) GetRefsWithHashes(ctx context.Context) ([]RefWithHash, error) } var tagsRefFilter = map[ref.RefType]struct{}{ref.TagRefType: {}} +var tuplesRefFilter = map[ref.RefType]struct{}{ref.TupleRefType: {}} // GetTags returns a list of all tags in the database. func (ddb *DoltDB) GetTags(ctx context.Context) ([]ref.DoltRef, error) { @@ -1113,6 +1114,35 @@ func (ddb *DoltDB) GetTagsWithHashes(ctx context.Context) ([]TagWithHash, error) return refs, err } +// SetTuple sets a key ref value +func (ddb *DoltDB) SetTuple(ctx context.Context, key string, value []byte) error { + ds, err := ddb.db.GetDataset(ctx, ref.NewTupleRef(key).String()) + if err != nil { + return err + } + _, err = ddb.db.SetTuple(ctx, ds, value) + return err +} + +// GetTuple returns a key's value, whether the key was valid, and an optional error. +func (ddb *DoltDB) GetTuple(ctx context.Context, key string) ([]byte, bool, error) { + ds, err := ddb.db.GetDataset(ctx, ref.NewTupleRef(key).String()) + if err != nil { + return nil, false, err + } + + if !ds.HasHead() { + return nil, false, nil + } + + tup, err := datas.LoadTuple(ctx, ddb.Format(), ddb.NodeStore(), ddb.ValueReadWriter(), ds) + if err != nil { + return nil, false, err + } + + return tup.Bytes(), true, nil +} + var workspacesRefFilter = map[ref.RefType]struct{}{ref.WorkspaceRefType: {}} // GetWorkspaces returns a list of all workspaces in the database. @@ -1587,6 +1617,14 @@ func (ddb *DoltDB) DeleteWorkingSet(ctx context.Context, workingSetRef ref.Worki return err } +func (ddb *DoltDB) DeleteTuple(ctx context.Context, key string) error { + err := ddb.deleteRef(ctx, ref.NewTupleRef(key), nil, "") + if err == ErrBranchNotFound { + return ErrTupleNotFound + } + return err +} + func (ddb *DoltDB) DeleteTag(ctx context.Context, tag ref.DoltRef) error { err := ddb.deleteRef(ctx, tag, nil, "") diff --git a/go/libraries/doltcore/doltdb/doltdb_test.go b/go/libraries/doltcore/doltdb/doltdb_test.go index 5332ac8f987..ef1f09de50e 100644 --- a/go/libraries/doltcore/doltdb/doltdb_test.go +++ b/go/libraries/doltcore/doltdb/doltdb_test.go @@ -83,7 +83,7 @@ func CreateTestTable(vrw types.ValueReadWriter, ns tree.NodeStore, tSchema schem func createTestRowData(t *testing.T, vrw types.ValueReadWriter, ns tree.NodeStore, sch schema.Schema) durable.Index { if types.Format_Default == types.Format_DOLT { - idx, err := durable.NewEmptyIndex(context.Background(), vrw, ns, sch, false) + idx, err := durable.NewEmptyPrimaryIndex(context.Background(), vrw, ns, sch) require.NoError(t, err) return idx } @@ -303,7 +303,7 @@ func TestLDNoms(t *testing.T) { ctx := context.Background() tSchema := createTestSchema(t) - rowData, err := durable.NewEmptyIndex(ctx, ddb.vrw, ddb.ns, tSchema, false) + rowData, err := durable.NewEmptyPrimaryIndex(ctx, ddb.vrw, ddb.ns, tSchema) if err != nil { t.Fatal("Failed to create new empty index") } diff --git a/go/libraries/doltcore/doltdb/durable/index.go b/go/libraries/doltcore/doltdb/durable/index.go index aa765fd8be8..9340d6608bf 100644 --- a/go/libraries/doltcore/doltdb/durable/index.go +++ b/go/libraries/doltcore/doltdb/durable/index.go @@ -123,8 +123,25 @@ func indexFromAddr(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeS } } -// NewEmptyIndex returns an index with no rows. -func NewEmptyIndex(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, sch schema.Schema, isKeylessSecondary bool) (Index, error) { +// NewEmptyPrimaryIndex creates a new empty Index for use as the primary index in a table. +func NewEmptyPrimaryIndex(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, indexSchema schema.Schema) (Index, error) { + return newEmptyIndex(ctx, vrw, ns, indexSchema, false) +} + +// NewEmptyForeignKeyIndex creates a new empty Index for use as a foreign key index. +// Foreign keys cannot appear on keyless tables. +func NewEmptyForeignKeyIndex(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, indexSchema schema.Schema) (Index, error) { + return newEmptyIndex(ctx, vrw, ns, indexSchema, false) +} + +// NewEmptyIndexFromTableSchema creates a new empty Index described by a schema.Index. +func NewEmptyIndexFromTableSchema(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, idx schema.Index, tableSchema schema.Schema) (Index, error) { + indexSchema := idx.Schema() + return newEmptyIndex(ctx, vrw, ns, indexSchema, schema.IsKeyless(tableSchema)) +} + +// newEmptyIndex returns an index with no rows. +func newEmptyIndex(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, sch schema.Schema, isKeylessSecondary bool) (Index, error) { switch vrw.Format() { case types.Format_LD_1: m, err := types.NewMap(ctx, vrw) @@ -138,17 +155,21 @@ func NewEmptyIndex(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeS if isKeylessSecondary { kd = prolly.AddHashToSchema(kd) } - m, err := prolly.NewMapFromTuples(ctx, ns, kd, vd) - if err != nil { - return nil, err - } - return IndexFromProllyMap(m), nil + return NewEmptyProllyIndex(ctx, ns, kd, vd) default: return nil, errNbfUnknown } } +func NewEmptyProllyIndex(ctx context.Context, ns tree.NodeStore, kd, vd val.TupleDesc) (Index, error) { + m, err := prolly.NewMapFromTuples(ctx, ns, kd, vd) + if err != nil { + return nil, err + } + return IndexFromProllyMap(m), nil +} + type nomsIndex struct { index types.Map vrw types.ValueReadWriter @@ -393,7 +414,7 @@ func NewIndexSetWithEmptyIndexes(ctx context.Context, vrw types.ValueReadWriter, return nil, err } for _, index := range sch.Indexes().AllIndexes() { - empty, err := NewEmptyIndex(ctx, vrw, ns, index.Schema(), false) + empty, err := NewEmptyIndexFromTableSchema(ctx, vrw, ns, index, sch) if err != nil { return nil, err } diff --git a/go/libraries/doltcore/doltdb/errors.go b/go/libraries/doltcore/doltdb/errors.go index d2cbf7a3e57..2301df7cc57 100644 --- a/go/libraries/doltcore/doltdb/errors.go +++ b/go/libraries/doltcore/doltdb/errors.go @@ -33,6 +33,7 @@ var ErrFoundHashNotACommit = errors.New("the value retrieved for this hash is no var ErrHashNotFound = errors.New("could not find a value for this hash") var ErrBranchNotFound = errors.New("branch not found") var ErrTagNotFound = errors.New("tag not found") +var ErrTupleNotFound = errors.New("tuple not found") var ErrWorkingSetNotFound = errors.New("working set not found") var ErrWorkspaceNotFound = errors.New("workspace not found") var ErrTableNotFound = errors.New("table not found") diff --git a/go/libraries/doltcore/doltdb/feature_version_test.go b/go/libraries/doltcore/doltdb/feature_version_test.go index 2197875387f..4069db86da0 100644 --- a/go/libraries/doltcore/doltdb/feature_version_test.go +++ b/go/libraries/doltcore/doltdb/feature_version_test.go @@ -59,7 +59,7 @@ func (cmd fvCommand) exec(ctx context.Context, dEnv *env.DoltEnv) int { doltdb.DoltFeatureVersion = cmd.user.vers defer func() { doltdb.DoltFeatureVersion = DoltFeatureVersionCopy }() - cliCtx, _ := commands.NewArgFreeCliContext(ctx, dEnv) + cliCtx, _ := commands.NewArgFreeCliContext(ctx, dEnv, dEnv.FS) return cmd.cmd.Exec(ctx, cmd.cmd.Name(), cmd.args, dEnv, cliCtx) } diff --git a/go/libraries/doltcore/doltdb/foreign_key_test.go b/go/libraries/doltcore/doltdb/foreign_key_test.go index 8f8d8044b8f..809745beb73 100644 --- a/go/libraries/doltcore/doltdb/foreign_key_test.go +++ b/go/libraries/doltcore/doltdb/foreign_key_test.go @@ -109,7 +109,7 @@ func TestForeignKeyErrors(t *testing.T) { ctx := context.Background() dEnv := dtestutils.CreateTestEnv() - cliCtx, err := commands.NewArgFreeCliContext(ctx, dEnv) + cliCtx, err := commands.NewArgFreeCliContext(ctx, dEnv, dEnv.FS) require.NoError(t, err) for _, c := range cmds { @@ -154,7 +154,7 @@ func testForeignKeys(t *testing.T, test foreignKeyTest) { ctx := context.Background() dEnv := dtestutils.CreateTestEnv() - cliCtx, verr := commands.NewArgFreeCliContext(ctx, dEnv) + cliCtx, verr := commands.NewArgFreeCliContext(ctx, dEnv, dEnv.FS) require.NoError(t, verr) for _, c := range fkSetupCommon { diff --git a/go/libraries/doltcore/doltdb/gc_test.go b/go/libraries/doltcore/doltdb/gc_test.go index 343c3213fbe..414832ead9f 100644 --- a/go/libraries/doltcore/doltdb/gc_test.go +++ b/go/libraries/doltcore/doltdb/gc_test.go @@ -122,7 +122,7 @@ func testGarbageCollection(t *testing.T, test gcTest) { dEnv := dtestutils.CreateTestEnv() defer dEnv.DoltDB.Close() - cliCtx, verr := commands.NewArgFreeCliContext(ctx, dEnv) + cliCtx, verr := commands.NewArgFreeCliContext(ctx, dEnv, dEnv.FS) require.NoError(t, verr) for _, c := range gcSetupCommon { diff --git a/go/libraries/doltcore/doltdb/hooksdatabase.go b/go/libraries/doltcore/doltdb/hooksdatabase.go index 9f0ab06353d..f774dede3bd 100644 --- a/go/libraries/doltcore/doltdb/hooksdatabase.go +++ b/go/libraries/doltcore/doltdb/hooksdatabase.go @@ -195,3 +195,11 @@ func (db hooksDatabase) Tag(ctx context.Context, ds datas.Dataset, commitAddr ha } return ds, err } + +func (db hooksDatabase) SetTuple(ctx context.Context, ds datas.Dataset, val []byte) (datas.Dataset, error) { + ds, err := db.Database.SetTuple(ctx, ds, val) + if err == nil { + db.ExecuteCommitHooks(ctx, ds, false) + } + return ds, err +} diff --git a/go/libraries/doltcore/doltdb/root_val.go b/go/libraries/doltcore/doltdb/root_val.go index d079e7651f8..2f4ff1c3510 100644 --- a/go/libraries/doltcore/doltdb/root_val.go +++ b/go/libraries/doltcore/doltdb/root_val.go @@ -924,7 +924,7 @@ func (root *rootValue) putTable(ctx context.Context, tName TableName, ref types. func CreateEmptyTable(ctx context.Context, root RootValue, tName TableName, sch schema.Schema) (RootValue, error) { ns := root.NodeStore() vrw := root.VRW() - empty, err := durable.NewEmptyIndex(ctx, vrw, ns, sch, false) + empty, err := durable.NewEmptyPrimaryIndex(ctx, vrw, ns, sch) if err != nil { return nil, err } diff --git a/go/libraries/doltcore/doltdb/table.go b/go/libraries/doltcore/doltdb/table.go index fcfccaed75c..9fda2989c5e 100644 --- a/go/libraries/doltcore/doltdb/table.go +++ b/go/libraries/doltcore/doltdb/table.go @@ -86,7 +86,7 @@ func NewTableFromDurable(table durable.Table) *Table { } func NewEmptyTable(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, sch schema.Schema) (*Table, error) { - rows, err := durable.NewEmptyIndex(ctx, vrw, ns, sch, false) + rows, err := durable.NewEmptyPrimaryIndex(ctx, vrw, ns, sch) if err != nil { return nil, err } diff --git a/go/libraries/doltcore/dtestutils/testcommands/multienv.go b/go/libraries/doltcore/dtestutils/testcommands/multienv.go index 93fef0de294..e573fd9ed39 100644 --- a/go/libraries/doltcore/dtestutils/testcommands/multienv.go +++ b/go/libraries/doltcore/dtestutils/testcommands/multienv.go @@ -168,7 +168,7 @@ func (mr *MultiRepoTestSetup) NewBranch(dbName, branchName string) { func (mr *MultiRepoTestSetup) CheckoutBranch(dbName, branchName string) { dEnv := mr.envs[dbName] - cliCtx, _ := cmd.NewArgFreeCliContext(context.Background(), dEnv) + cliCtx, _ := cmd.NewArgFreeCliContext(context.Background(), dEnv, dEnv.FS) _, sqlCtx, closeFunc, err := cliCtx.QueryEngine(context.Background()) if err != nil { mr.Errhand(err) @@ -390,7 +390,7 @@ func createTestTable(dEnv *env.DoltEnv, tableName string, sch schema.Schema) err vrw := dEnv.DoltDB.ValueReadWriter() ns := dEnv.DoltDB.NodeStore() - idx, err := durable.NewEmptyIndex(ctx, vrw, ns, sch, false) + idx, err := durable.NewEmptyPrimaryIndex(ctx, vrw, ns, sch) if err != nil { return err } diff --git a/go/libraries/doltcore/env/environment.go b/go/libraries/doltcore/env/environment.go index c8eb0a84937..69a2bb547e1 100644 --- a/go/libraries/doltcore/env/environment.go +++ b/go/libraries/doltcore/env/environment.go @@ -51,6 +51,8 @@ const ( DefaultRemotesApiPort = "443" tempTablesDir = "temptf" + + TmpDirName = "tmp" ) var zeroHashStr = (hash.Hash{}).String() @@ -149,7 +151,7 @@ func (dEnv *DoltEnv) ReloadRepoState() error { return nil } -func LoadWithoutDB(ctx context.Context, hdp HomeDirProvider, fs filesys.Filesys, version string) *DoltEnv { +func LoadWithoutDB(_ context.Context, hdp HomeDirProvider, fs filesys.Filesys, version string) *DoltEnv { cfg, cfgErr := LoadDoltCliConfig(hdp, fs) repoState, rsErr := createRepoState(fs) @@ -450,13 +452,21 @@ func (dEnv *DoltEnv) createDirectories(dir string) (string, error) { } if dEnv.hasDoltDir(dir) { - // Special case a completely empty directory. We can allow that. + // Special case a completely empty directory, or one which has a "tmp" directory but nothing else. + // The `.dolt/tmp` directory is created while verifying that we can rename table files which is early + // in the startup process. It will only exist if we need it because the TMPDIR environment variable is set to + // a path which is on a different partition than the .dolt directory. dotDolt := mustAbs(dEnv, dbfactory.DoltDir) entries, err := os.ReadDir(dotDolt) if err != nil { return "", err } - if len(entries) != 0 { + + if len(entries) == 1 { + if !entries[0].IsDir() || entries[0].Name() != TmpDirName { + return "", fmt.Errorf(".dolt directory already exists at '%s'", dir) + } + } else if len(entries) != 0 { return "", fmt.Errorf(".dolt directory already exists at '%s'", dir) } } diff --git a/go/libraries/doltcore/merge/fulltext_rebuild.go b/go/libraries/doltcore/merge/fulltext_rebuild.go index e439e663f1e..e1cf674a19a 100644 --- a/go/libraries/doltcore/merge/fulltext_rebuild.go +++ b/go/libraries/doltcore/merge/fulltext_rebuild.go @@ -330,7 +330,7 @@ func purgeFulltextTableData(ctx *sql.Context, root doltdb.RootValue, tableNames if err != nil { return nil, err } - rows, err := durable.NewEmptyIndex(ctx, tbl.ValueReadWriter(), tbl.NodeStore(), sch, false) + rows, err := durable.NewEmptyPrimaryIndex(ctx, tbl.ValueReadWriter(), tbl.NodeStore(), sch) if err != nil { return nil, err } diff --git a/go/libraries/doltcore/merge/keyless_integration_test.go b/go/libraries/doltcore/merge/keyless_integration_test.go index 822fa75f0d1..e7652f80a20 100644 --- a/go/libraries/doltcore/merge/keyless_integration_test.go +++ b/go/libraries/doltcore/merge/keyless_integration_test.go @@ -114,7 +114,7 @@ func TestKeylessMerge(t *testing.T) { require.NoError(t, err) err = dEnv.UpdateWorkingRoot(ctx, root) require.NoError(t, err) - cliCtx, err := cmd.NewArgFreeCliContext(ctx, dEnv) + cliCtx, err := cmd.NewArgFreeCliContext(ctx, dEnv, dEnv.FS) require.NoError(t, err) for _, c := range test.setup { @@ -249,7 +249,7 @@ func TestKeylessMergeConflicts(t *testing.T) { require.NoError(t, err) err = dEnv.UpdateWorkingRoot(ctx, root) require.NoError(t, err) - cliCtx, err := cmd.NewArgFreeCliContext(ctx, dEnv) + cliCtx, err := cmd.NewArgFreeCliContext(ctx, dEnv, dEnv.FS) require.NoError(t, err) for _, c := range cc { @@ -282,7 +282,7 @@ func TestKeylessMergeConflicts(t *testing.T) { defer dEnv.DoltDB.Close() setupTest(t, ctx, dEnv, test.setup) - cliCtx, verr := cmd.NewArgFreeCliContext(ctx, dEnv) + cliCtx, verr := cmd.NewArgFreeCliContext(ctx, dEnv, dEnv.FS) require.NoError(t, verr) resolve := cnfcmds.ResolveCmd{} @@ -302,7 +302,7 @@ func TestKeylessMergeConflicts(t *testing.T) { defer dEnv.DoltDB.Close() setupTest(t, ctx, dEnv, test.setup) - cliCtx, verr := cmd.NewArgFreeCliContext(ctx, dEnv) + cliCtx, verr := cmd.NewArgFreeCliContext(ctx, dEnv, dEnv.FS) require.NoError(t, verr) resolve := cnfcmds.ResolveCmd{} diff --git a/go/libraries/doltcore/merge/schema_integration_test.go b/go/libraries/doltcore/merge/schema_integration_test.go index 42b0dc22d9e..04db04f971f 100644 --- a/go/libraries/doltcore/merge/schema_integration_test.go +++ b/go/libraries/doltcore/merge/schema_integration_test.go @@ -40,7 +40,7 @@ type testCommand struct { } func (tc testCommand) exec(t *testing.T, ctx context.Context, dEnv *env.DoltEnv) int { - cliCtx, err := commands.NewArgFreeCliContext(ctx, dEnv) + cliCtx, err := commands.NewArgFreeCliContext(ctx, dEnv, dEnv.FS) require.NoError(t, err) return tc.cmd.Exec(ctx, tc.cmd.Name(), tc.args, dEnv, cliCtx) } @@ -559,7 +559,7 @@ func testMergeSchemas(t *testing.T, test mergeSchemaTest) { defer dEnv.DoltDB.Close() ctx := context.Background() - cliCtx, _ := commands.NewArgFreeCliContext(ctx, dEnv) + cliCtx, _ := commands.NewArgFreeCliContext(ctx, dEnv, dEnv.FS) for _, c := range setupCommon { exit := c.exec(t, ctx, dEnv) @@ -618,7 +618,7 @@ func testMergeSchemasWithConflicts(t *testing.T, test mergeSchemaConflictTest) { require.Equal(t, 0, exit) } - cliCtx, _ := commands.NewArgFreeCliContext(ctx, dEnv) + cliCtx, _ := commands.NewArgFreeCliContext(ctx, dEnv, dEnv.FS) // assert that we're on main exitCode := commands.CheckoutCmd{}.Exec(ctx, "checkout", []string{env.DefaultInitBranch}, dEnv, cliCtx) @@ -680,7 +680,7 @@ func testMergeForeignKeys(t *testing.T, test mergeForeignKeyTest) { require.Equal(t, 0, exit) } - cliCtx, _ := commands.NewArgFreeCliContext(ctx, dEnv) + cliCtx, _ := commands.NewArgFreeCliContext(ctx, dEnv, dEnv.FS) // assert that we're on main exitCode := commands.CheckoutCmd{}.Exec(ctx, "checkout", []string{env.DefaultInitBranch}, dEnv, cliCtx) diff --git a/go/libraries/doltcore/merge/violations_fk.go b/go/libraries/doltcore/merge/violations_fk.go index a2f3233329b..0fbd8e58222 100644 --- a/go/libraries/doltcore/merge/violations_fk.go +++ b/go/libraries/doltcore/merge/violations_fk.go @@ -107,7 +107,7 @@ func GetForeignKeyViolations(ctx context.Context, newRoot, baseRoot doltdb.RootV return err } // Parent does not exist in the ancestor so we use an empty map - emptyIdx, err := durable.NewEmptyIndex(ctx, postParent.Table.ValueReadWriter(), postParent.Table.NodeStore(), postParent.Schema, false) + emptyIdx, err := durable.NewEmptyPrimaryIndex(ctx, postParent.Table.ValueReadWriter(), postParent.Table.NodeStore(), postParent.Schema) if err != nil { return err } @@ -129,7 +129,7 @@ func GetForeignKeyViolations(ctx context.Context, newRoot, baseRoot doltdb.RootV return err } // Child does not exist in the ancestor so we use an empty map - emptyIdx, err := durable.NewEmptyIndex(ctx, postChild.Table.ValueReadWriter(), postChild.Table.NodeStore(), postChild.Schema, false) + emptyIdx, err := durable.NewEmptyPrimaryIndex(ctx, postChild.Table.ValueReadWriter(), postChild.Table.NodeStore(), postChild.Schema) if err != nil { return err } @@ -370,7 +370,7 @@ func parentFkConstraintViolations( } var idx durable.Index if empty { - idx, err = durable.NewEmptyIndex(ctx, postChild.Table.ValueReadWriter(), postParent.Table.NodeStore(), postParent.Schema, false) + idx, err = durable.NewEmptyForeignKeyIndex(ctx, postChild.Table.ValueReadWriter(), postParent.Table.NodeStore(), postParent.Index.Schema()) if err != nil { return err } @@ -405,7 +405,7 @@ func childFkConstraintViolations( } var idx durable.Index if empty { - idx, err = durable.NewEmptyIndex(ctx, postChild.Table.ValueReadWriter(), postChild.Table.NodeStore(), postChild.Schema, false) + idx, err = durable.NewEmptyForeignKeyIndex(ctx, postChild.Table.ValueReadWriter(), postChild.Table.NodeStore(), postChild.Index.Schema()) if err != nil { return err } diff --git a/go/libraries/doltcore/migrate/integration_test.go b/go/libraries/doltcore/migrate/integration_test.go index 317d783eb3b..ea385447889 100644 --- a/go/libraries/doltcore/migrate/integration_test.go +++ b/go/libraries/doltcore/migrate/integration_test.go @@ -170,7 +170,7 @@ func setupMigrationTest(t *testing.T, ctx context.Context, test migrationTest) * dEnv, err = test.hook(ctx, dEnv) require.NoError(t, err) } - cliCtx, err := commands.NewArgFreeCliContext(ctx, dEnv) + cliCtx, err := commands.NewArgFreeCliContext(ctx, dEnv, dEnv.FS) require.NoError(t, err) cmd := commands.SqlCmd{} diff --git a/go/libraries/doltcore/rebase/filter_branch_test.go b/go/libraries/doltcore/rebase/filter_branch_test.go index 1f0d1c76183..d8f6d5a0585 100644 --- a/go/libraries/doltcore/rebase/filter_branch_test.go +++ b/go/libraries/doltcore/rebase/filter_branch_test.go @@ -202,7 +202,7 @@ func filterBranchTests() []filterBranchTest { func setupFilterBranchTests(t *testing.T) *env.DoltEnv { ctx := context.Background() dEnv := dtestutils.CreateTestEnv() - cliCtx, err := cmd.NewArgFreeCliContext(ctx, dEnv) + cliCtx, err := cmd.NewArgFreeCliContext(ctx, dEnv, dEnv.FS) require.NoError(t, err) for _, c := range setupCommon { @@ -217,7 +217,7 @@ func testFilterBranch(t *testing.T, test filterBranchTest) { ctx := context.Background() dEnv := setupFilterBranchTests(t) defer dEnv.DoltDB.Close() - cliCtx, err := cmd.NewArgFreeCliContext(ctx, dEnv) + cliCtx, err := cmd.NewArgFreeCliContext(ctx, dEnv, dEnv.FS) require.NoError(t, err) for _, c := range test.setup { diff --git a/go/libraries/doltcore/ref/ref.go b/go/libraries/doltcore/ref/ref.go index 27f9bc8ec22..be856926f55 100644 --- a/go/libraries/doltcore/ref/ref.go +++ b/go/libraries/doltcore/ref/ref.go @@ -57,6 +57,9 @@ const ( // StatsRefType is a reference to a statistics table StatsRefType RefType = "statistics" + + // TupleRefType is a reference to a statistics table + TupleRefType RefType = "tuples" ) // HeadRefTypes are the ref types that point to a HEAD and contain a Commit struct. These are the types that are @@ -205,5 +208,9 @@ func Parse(str string) (DoltRef, error) { return NewStatsRef(str[len(prefix):]), nil } + if prefix := PrefixForType(TupleRefType); strings.HasPrefix(str, prefix) { + return NewTupleRef(str[len(prefix):]), nil + } + return nil, ErrUnknownRefType } diff --git a/go/libraries/doltcore/ref/tuple_ref.go b/go/libraries/doltcore/ref/tuple_ref.go new file mode 100644 index 00000000000..d4db039ef58 --- /dev/null +++ b/go/libraries/doltcore/ref/tuple_ref.go @@ -0,0 +1,41 @@ +// Copyright 2023 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ref + +type TupleRef struct { + key string +} + +var _ DoltRef = TupleRef{} + +// NewTupleRef creates a reference to a statistic dataset head. +func NewTupleRef(key string) TupleRef { + return TupleRef{key} +} + +// GetType will return TupleRefType +func (br TupleRef) GetType() RefType { + return TupleRefType +} + +// GetPath returns the name of the tag +func (br TupleRef) GetPath() string { + return br.key +} + +// String returns the fully qualified reference name e.g. refs/heads/main +func (br TupleRef) String() string { + return String(br) +} diff --git a/go/libraries/doltcore/schema/integration_test.go b/go/libraries/doltcore/schema/integration_test.go index b3968d42f90..cd3f5d4bb19 100644 --- a/go/libraries/doltcore/schema/integration_test.go +++ b/go/libraries/doltcore/schema/integration_test.go @@ -176,7 +176,7 @@ func TestGetKeyTags(t *testing.T) { func runTestSql(t *testing.T, ctx context.Context, setup []string) (*doltdb.DoltDB, doltdb.RootValue) { dEnv := dtestutils.CreateTestEnv() cmd := commands.SqlCmd{} - cliCtx, verr := commands.NewArgFreeCliContext(ctx, dEnv) + cliCtx, verr := commands.NewArgFreeCliContext(ctx, dEnv, dEnv.FS) require.NoError(t, verr) for _, query := range setup { diff --git a/go/libraries/doltcore/schema/schema.go b/go/libraries/doltcore/schema/schema.go index f15b775f6a1..bbdbdfd6e14 100644 --- a/go/libraries/doltcore/schema/schema.go +++ b/go/libraries/doltcore/schema/schema.go @@ -307,13 +307,12 @@ func ArePrimaryKeySetsDiffable(format *types.NomsBinFormat, fromSch, toSch Schem // use to map key, value val.Tuple's of schema |inSch| to |outSch|. The first // ordinal map is for keys, and the second is for values. If a column of |inSch| // is missing in |outSch| then that column's index in the ordinal map holds -1. -func MapSchemaBasedOnTagAndName(inSch, outSch Schema) ([]int, []int, error) { +func MapSchemaBasedOnTagAndName(inSch, outSch Schema) (val.OrdinalMapping, val.OrdinalMapping, error) { keyMapping := make([]int, inSch.GetPKCols().Size()) - valMapping := make([]int, inSch.GetNonPKCols().Size()) // if inSch or outSch is empty schema. This can be from added or dropped table. if len(inSch.GetAllCols().cols) == 0 || len(outSch.GetAllCols().cols) == 0 { - return keyMapping, valMapping, nil + return keyMapping, make([]int, inSch.GetNonPKCols().Size()), nil } err := inSch.GetPKCols().Iter(func(tag uint64, col Column) (stop bool, err error) { @@ -330,14 +329,20 @@ func MapSchemaBasedOnTagAndName(inSch, outSch Schema) ([]int, []int, error) { return nil, nil, err } - err = inSch.GetNonPKCols().Iter(func(tag uint64, col Column) (stop bool, err error) { - i := inSch.GetNonPKCols().TagToIdx[col.Tag] - if col, ok := outSch.GetNonPKCols().GetByName(col.Name); ok { - j := outSch.GetNonPKCols().TagToIdx[col.Tag] + inNonPKCols := inSch.GetNonPKCols() + outNonPKCols := outSch.GetNonPKCols() + valMapping := make([]int, inSch.GetNonPKCols().Size()) + err = inNonPKCols.Iter(func(tag uint64, col Column) (stop bool, err error) { + i := inNonPKCols.TagToIdx[col.Tag] + if col.Virtual { + valMapping[i] = -1 + } else if col, ok := outNonPKCols.GetByName(col.Name); ok { + j := outNonPKCols.TagToIdx[col.Tag] valMapping[i] = j } else { valMapping[i] = -1 } + return false, nil }) if err != nil { diff --git a/go/libraries/doltcore/schema/schema_test.go b/go/libraries/doltcore/schema/schema_test.go index 2182fd8efae..5705e4ca624 100644 --- a/go/libraries/doltcore/schema/schema_test.go +++ b/go/libraries/doltcore/schema/schema_test.go @@ -25,6 +25,7 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/schema/typeinfo" "github.com/dolthub/dolt/go/store/types" + "github.com/dolthub/dolt/go/store/val" ) const ( @@ -231,7 +232,7 @@ func TestArePrimaryKeySetsDiffable(t *testing.T) { From Schema To Schema Diffable bool - KeyMap []int + KeyMap val.OrdinalMapping }{ { Name: "Basic", @@ -240,7 +241,7 @@ func TestArePrimaryKeySetsDiffable(t *testing.T) { To: MustSchemaFromCols(NewColCollection( NewColumn("pk", 0, types.IntKind, true))), Diffable: true, - KeyMap: []int{0}, + KeyMap: val.OrdinalMapping{0}, }, { Name: "PK-Column renames", @@ -249,7 +250,7 @@ func TestArePrimaryKeySetsDiffable(t *testing.T) { To: MustSchemaFromCols(NewColCollection( NewColumn("pk2", 1, types.IntKind, true))), Diffable: true, - KeyMap: []int{0}, + KeyMap: val.OrdinalMapping{0}, }, { Name: "Only pk ordering should matter for diffability", @@ -259,7 +260,7 @@ func TestArePrimaryKeySetsDiffable(t *testing.T) { To: MustSchemaFromCols(NewColCollection( NewColumn("pk", 1, types.IntKind, true))), Diffable: true, - KeyMap: []int{0}, + KeyMap: val.OrdinalMapping{0}, }, { Name: "Only pk ordering should matter for diffability - inverse", @@ -269,7 +270,7 @@ func TestArePrimaryKeySetsDiffable(t *testing.T) { NewColumn("col1", 2, types.IntKind, false), NewColumn("pk", 1, types.IntKind, true))), Diffable: true, - KeyMap: []int{0}, + KeyMap: val.OrdinalMapping{0}, }, { Name: "Only pk ordering should matter for diffability - compound", @@ -281,7 +282,7 @@ func TestArePrimaryKeySetsDiffable(t *testing.T) { NewColumn("pk1", 0, types.IntKind, true), NewColumn("pk2", 2, types.IntKind, true))), Diffable: true, - KeyMap: []int{0, 1}, + KeyMap: val.OrdinalMapping{0, 1}, }, { Name: "Tag mismatches", diff --git a/go/libraries/doltcore/servercfg/testdata/minver_validation.txt b/go/libraries/doltcore/servercfg/testdata/minver_validation.txt index dcf4d8d1202..fc9d38d5f1f 100644 --- a/go/libraries/doltcore/servercfg/testdata/minver_validation.txt +++ b/go/libraries/doltcore/servercfg/testdata/minver_validation.txt @@ -54,7 +54,7 @@ PrivilegeFile *string 0.0.0 privilege_file,omitempty BranchControlFile *string 0.0.0 branch_control_file,omitempty Vars []servercfg.UserSessionVars 0.0.0 user_session_vars -Name string 0.0.0 name --Vars map[string]string 0.0.0 vars +-Vars map[string]interface{} 0.0.0 vars SystemVars_ map[string]interface{} 1.11.1 system_variables,omitempty Jwks []servercfg.JwksConfig 0.0.0 jwks -Name string 0.0.0 name diff --git a/go/libraries/doltcore/servercfg/yaml_config.go b/go/libraries/doltcore/servercfg/yaml_config.go index 70b8d7d92d7..ea711da0de7 100644 --- a/go/libraries/doltcore/servercfg/yaml_config.go +++ b/go/libraries/doltcore/servercfg/yaml_config.go @@ -119,8 +119,8 @@ func (r RemotesapiYAMLConfig) ReadOnly() bool { } type UserSessionVars struct { - Name string `yaml:"name"` - Vars map[string]string `yaml:"vars"` + Name string `yaml:"name"` + Vars map[string]interface{} `yaml:"vars"` } // YAMLConfig is a ServerConfig implementation which is read from a yaml file @@ -572,6 +572,9 @@ func (cfg YAMLConfig) MetricsPort() int { if cfg.MetricsConfig.Host == nil { return DefaultMetricsPort } + if cfg.MetricsConfig.Port == nil { + return DefaultMetricsPort + } return *cfg.MetricsConfig.Port } diff --git a/go/libraries/doltcore/servercfg/yaml_config_test.go b/go/libraries/doltcore/servercfg/yaml_config_test.go index a9577f376ab..53166a58efc 100644 --- a/go/libraries/doltcore/servercfg/yaml_config_test.go +++ b/go/libraries/doltcore/servercfg/yaml_config_test.go @@ -106,7 +106,7 @@ jwks: expected.Vars = []UserSessionVars{ { Name: "user0", - Vars: map[string]string{ + Vars: map[string]interface{}{ "var1": "val0_1", "var2": "val0_2", "var3": "val0_3", @@ -114,7 +114,7 @@ jwks: }, { Name: "user1", - Vars: map[string]string{ + Vars: map[string]interface{}{ "var1": "val1_1", "var2": "val1_2", "var4": "val1_4", @@ -415,6 +415,19 @@ listener: assert.Error(t, err) } +func TestYAMLConfigMetrics(t *testing.T) { + var cfg YAMLConfig + err := yaml.Unmarshal([]byte(` +metrics: + host: localhost + port: null +`), &cfg) + require.NoError(t, err) + + assert.Equal(t, "localhost", cfg.MetricsHost()) + assert.Equal(t, -1, cfg.MetricsPort()) +} + func TestCommentNullYAMLValues(t *testing.T) { toComment := ` value1: value diff --git a/go/libraries/doltcore/sqle/alterschema_test.go b/go/libraries/doltcore/sqle/alterschema_test.go index 656b633ac1d..fe9807c4570 100644 --- a/go/libraries/doltcore/sqle/alterschema_test.go +++ b/go/libraries/doltcore/sqle/alterschema_test.go @@ -269,7 +269,7 @@ func makePeopleTable(ctx context.Context, dEnv *env.DoltEnv) (*env.DoltEnv, erro if err != nil { return nil, err } - rows, err := durable.NewEmptyIndex(ctx, root.VRW(), root.NodeStore(), sch, false) + rows, err := durable.NewEmptyPrimaryIndex(ctx, root.VRW(), root.NodeStore(), sch) if err != nil { return nil, err } diff --git a/go/libraries/doltcore/sqle/common_test.go b/go/libraries/doltcore/sqle/common_test.go index 1c5e3a45137..3e889ece5fc 100644 --- a/go/libraries/doltcore/sqle/common_test.go +++ b/go/libraries/doltcore/sqle/common_test.go @@ -181,7 +181,7 @@ func CreateTestTable(t *testing.T, dEnv *env.DoltEnv, tableName string, sch sche vrw := dEnv.DoltDB.ValueReadWriter() ns := dEnv.DoltDB.NodeStore() - rows, err := durable.NewEmptyIndex(ctx, vrw, ns, sch, false) + rows, err := durable.NewEmptyPrimaryIndex(ctx, vrw, ns, sch) require.NoError(t, err) tbl, err := doltdb.NewTable(ctx, vrw, ns, sch, rows, nil, nil) require.NoError(t, err) diff --git a/go/libraries/doltcore/sqle/database_provider.go b/go/libraries/doltcore/sqle/database_provider.go index 37b64d8f2fb..bda772aad43 100644 --- a/go/libraries/doltcore/sqle/database_provider.go +++ b/go/libraries/doltcore/sqle/database_provider.go @@ -1414,12 +1414,11 @@ func (p *DoltDatabaseProvider) ExternalStoredProcedures(_ *sql.Context, name str } // TableFunction implements the sql.TableFunctionProvider interface -func (p *DoltDatabaseProvider) TableFunction(_ *sql.Context, name string) (sql.TableFunction, error) { +func (p *DoltDatabaseProvider) TableFunction(_ *sql.Context, name string) (sql.TableFunction, bool) { if fun, ok := p.tableFunctions[strings.ToLower(name)]; ok { - return fun, nil + return fun, true } - - return nil, sql.ErrTableFunctionNotFound.New(name) + return nil, false } // ensureReplicaHeadExists tries to pull the latest version of a remote branch. Will fail if the branch diff --git a/go/libraries/doltcore/sqle/dtablefunctions/dolt_log_table_function.go b/go/libraries/doltcore/sqle/dtablefunctions/dolt_log_table_function.go index d06592b0f0e..daf43c8b206 100644 --- a/go/libraries/doltcore/sqle/dtablefunctions/dolt_log_table_function.go +++ b/go/libraries/doltcore/sqle/dtablefunctions/dolt_log_table_function.go @@ -110,7 +110,9 @@ func (ltf *LogTableFunction) Name() string { // Resolved implements the sql.Resolvable interface func (ltf *LogTableFunction) Resolved() bool { for _, expr := range ltf.revisionExprs { - return expr.Resolved() + if !expr.Resolved() { + return false + } } return true } diff --git a/go/libraries/doltcore/sqle/dtables/commit_diff_table.go b/go/libraries/doltcore/sqle/dtables/commit_diff_table.go index 4eeee2a8126..dcc50a12d56 100644 --- a/go/libraries/doltcore/sqle/dtables/commit_diff_table.go +++ b/go/libraries/doltcore/sqle/dtables/commit_diff_table.go @@ -216,7 +216,7 @@ func (dt *CommitDiffTable) LookupPartitions(ctx *sql.Context, i sql.IndexLookup) fromSch: dt.targetSchema, } - isDiffable, err := dp.isDiffablePartition(ctx) + isDiffable, _, err := dp.isDiffablePartition(ctx) if err != nil { return nil, err } diff --git a/go/libraries/doltcore/sqle/dtables/conflicts_tables_prolly.go b/go/libraries/doltcore/sqle/dtables/conflicts_tables_prolly.go index 3201c0d0bff..a9b323ac747 100644 --- a/go/libraries/doltcore/sqle/dtables/conflicts_tables_prolly.go +++ b/go/libraries/doltcore/sqle/dtables/conflicts_tables_prolly.go @@ -415,7 +415,7 @@ func (itr *prollyConflictRowIter) loadTableMaps(ctx *sql.Context, baseHash, thei var idx durable.Index if !ok { - idx, err = durable.NewEmptyIndex(ctx, itr.vrw, itr.ns, itr.ourSch, false) + idx, err = durable.NewEmptyPrimaryIndex(ctx, itr.vrw, itr.ns, itr.ourSch) } else { idx, err = baseTbl.GetRowData(ctx) } diff --git a/go/libraries/doltcore/sqle/dtables/diff_table.go b/go/libraries/doltcore/sqle/dtables/diff_table.go index c69b8f6cd36..0956f1c8232 100644 --- a/go/libraries/doltcore/sqle/dtables/diff_table.go +++ b/go/libraries/doltcore/sqle/dtables/diff_table.go @@ -599,7 +599,7 @@ func tableData(ctx *sql.Context, tbl *doltdb.Table, ddb *doltdb.DoltDB) (durable var err error if tbl == nil { - data, err = durable.NewEmptyIndex(ctx, ddb.ValueReadWriter(), ddb.NodeStore(), schema.EmptySchema, false) + data, err = durable.NewEmptyPrimaryIndex(ctx, ddb.ValueReadWriter(), ddb.NodeStore(), schema.EmptySchema) if err != nil { return nil, nil, err } @@ -684,31 +684,42 @@ func (dp DiffPartition) GetRowIter(ctx *sql.Context, ddb *doltdb.DoltDB, joiner // isDiffablePartition checks if the commit pair for this partition is "diffable". // If the primary key sets changed between the two commits, it may not be -// possible to diff them. -func (dp *DiffPartition) isDiffablePartition(ctx *sql.Context) (bool, error) { +// possible to diff them. We return two bools: simpleDiff is returned if the primary key sets are close enough that we +// can confidently merge the diff (using schema.ArePrimaryKeySetsDiffable). fuzzyDiff is returned if the primary key +// sets are not close enough to merge the diff, but we can still make an approximate comparison (using schema.MapSchemaBasedOnTagAndName). +func (dp *DiffPartition) isDiffablePartition(ctx *sql.Context) (simpleDiff bool, fuzzyDiff bool, err error) { // dp.to is nil when a table has been deleted previously. In this case, we return // false, to stop processing diffs, since that previously deleted table is considered // a logically different table and we don't want to mix the diffs together. if dp.to == nil { - return false, nil + return false, false, nil } // dp.from is nil when the to commit created a new table if dp.from == nil { - return true, nil + return true, false, nil } fromSch, err := dp.from.GetSchema(ctx) if err != nil { - return false, err + return false, false, err } toSch, err := dp.to.GetSchema(ctx) if err != nil { - return false, err + return false, false, err + } + + easyDiff := schema.ArePrimaryKeySetsDiffable(dp.from.Format(), fromSch, toSch) + if easyDiff { + return true, false, nil } - return schema.ArePrimaryKeySetsDiffable(dp.from.Format(), fromSch, toSch), nil + _, _, err = schema.MapSchemaBasedOnTagAndName(fromSch, toSch) + if err == nil { + return false, true, nil + } + return false, false, nil } type partitionSelectFunc func(*sql.Context, DiffPartition) (bool, error) @@ -762,6 +773,7 @@ type DiffPartitions struct { selectFunc partitionSelectFunc toSch schema.Schema fromSch schema.Schema + stopNext bool } // processCommit is called in a commit iteration loop. Adds partitions when it finds a commit and its parent that have @@ -821,6 +833,10 @@ func (dps *DiffPartitions) processCommit(ctx *sql.Context, cmHash hash.Hash, cm } func (dps *DiffPartitions) Next(ctx *sql.Context) (sql.Partition, error) { + if dps.stopNext { + return nil, io.EOF + } + for { cmHash, optCmt, err := dps.cmItr.Next(ctx) if err != nil { @@ -852,16 +868,21 @@ func (dps *DiffPartitions) Next(ctx *sql.Context) (sql.Partition, error) { if next != nil { // If we can't diff this commit with its parent, don't traverse any lower - canDiff, err := next.isDiffablePartition(ctx) + simpleDiff, fuzzyDiff, err := next.isDiffablePartition(ctx) if err != nil { return nil, err } - if !canDiff { + if !simpleDiff && !fuzzyDiff { ctx.Warn(PrimaryKeyChangeWarningCode, fmt.Sprintf(PrimaryKeyChangeWarning, next.fromName, next.toName)) return nil, io.EOF } + if !simpleDiff && fuzzyDiff { + ctx.Warn(PrimaryKeyChangeWarningCode, fmt.Sprintf(PrimaryKeyChangeWarning, next.fromName, next.toName)) + dps.stopNext = true + } + return *next, nil } } diff --git a/go/libraries/doltcore/sqle/dtables/prolly_row_conv.go b/go/libraries/doltcore/sqle/dtables/prolly_row_conv.go index d0f5714ccb2..687fcc21b2c 100644 --- a/go/libraries/doltcore/sqle/dtables/prolly_row_conv.go +++ b/go/libraries/doltcore/sqle/dtables/prolly_row_conv.go @@ -102,21 +102,31 @@ func NewProllyRowConverter(inSch, outSch schema.Schema, warnFn rowconv.WarnFunct // PutConverted converts the |key| and |value| val.Tuple from |inSchema| to |outSchema| // and places the converted row in |dstRow|. -func (c ProllyRowConverter) PutConverted(ctx context.Context, key, value val.Tuple, dstRow []interface{}) error { - err := c.putFields(ctx, key, c.keyProj, c.keyDesc, c.pkTargetTypes, dstRow) +func (c ProllyRowConverter) PutConverted(ctx context.Context, key, value val.Tuple, dstRow sql.Row) error { + err := c.putFields(ctx, key, c.keyProj, c.keyDesc, c.pkTargetTypes, dstRow, true) if err != nil { return err } - return c.putFields(ctx, value, c.valProj, c.valDesc, c.nonPkTargetTypes, dstRow) + return c.putFields(ctx, value, c.valProj, c.valDesc, c.nonPkTargetTypes, dstRow, false) } -func (c ProllyRowConverter) putFields(ctx context.Context, tup val.Tuple, proj val.OrdinalMapping, desc val.TupleDesc, targetTypes []sql.Type, dstRow []interface{}) error { +func (c ProllyRowConverter) putFields(ctx context.Context, tup val.Tuple, proj val.OrdinalMapping, desc val.TupleDesc, targetTypes []sql.Type, dstRow sql.Row, isPk bool) error { + virtualOffset := 0 for i, j := range proj { if j == -1 { + nonPkCols := c.inSchema.GetNonPKCols() + if len(nonPkCols.GetColumns()) > i { + // Skip over virtual columns in non-pk cols as they are not stored + if !isPk && nonPkCols.GetByIndex(i).Virtual { + virtualOffset++ + } + } + continue } - f, err := tree.GetField(ctx, desc, i, tup, c.ns) + + f, err := tree.GetField(ctx, desc, i-virtualOffset, tup, c.ns) if err != nil { return err } diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go b/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go index 78805998839..20043743b26 100644 --- a/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go @@ -1663,9 +1663,9 @@ func TestStatsHistograms(t *testing.T) { // TestStatsIO force a provider reload in-between setup and assertions that // forces a round trip of the statistics table before inspecting values. -func TestStatsIO(t *testing.T) { +func TestStatsStorage(t *testing.T) { h := newDoltEnginetestHarness(t) - RunStatsIOTests(t, h) + RunStatsStorageTests(t, h) } func TestStatsIOWithoutReload(t *testing.T) { diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_engine_tests.go b/go/libraries/doltcore/sqle/enginetest/dolt_engine_tests.go index e0f25e34a57..0e83de5cca9 100755 --- a/go/libraries/doltcore/sqle/enginetest/dolt_engine_tests.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_engine_tests.go @@ -1410,7 +1410,7 @@ func RunSystemTableIndexesTests(t *testing.T, harness DoltEnginetestHarness) { } for _, stt := range SystemTableIndexTests { - harness = harness.NewHarness(t).WithParallelism(2) + harness = harness.NewHarness(t).WithParallelism(1) defer harness.Close() harness.SkipSetupCommit() e := mustNewEngine(t, harness) @@ -1422,27 +1422,30 @@ func RunSystemTableIndexesTests(t *testing.T, harness DoltEnginetestHarness) { enginetest.RunQueryWithContext(t, e, harness, ctx, q) } - for i, c := range []string{"inner", "lookup", "hash", "merge"} { - e.EngineAnalyzer().Coster = biasedCosters[i] - for _, tt := range stt.queries { - if tt.query == "select count(*) from dolt_blame_xy" && c == "inner" { - // todo we either need join hints to work inside the blame view - // and force the window relation to be primary, or we need the - // blame view's timestamp columns to be specific enough to not - // overlap during testing. - t.Skip("the blame table is unstable as secondary table in join with exchange node") - } - t.Run(fmt.Sprintf("%s(%s): %s", stt.name, c, tt.query), func(t *testing.T) { - if tt.skip { - t.Skip() - } - - ctx = ctx.WithQuery(tt.query) - if tt.exp != nil { - enginetest.TestQueryWithContext(t, ctx, e, harness, tt.query, tt.exp, nil, nil, nil) + costers := []string{"inner", "lookup", "hash", "merge"} + for i, c := range costers { + t.Run(c, func(t *testing.T) { + e.EngineAnalyzer().Coster = biasedCosters[i] + for _, tt := range stt.queries { + if tt.query == "select count(*) from dolt_blame_xy" && c == "inner" { + // todo we either need join hints to work inside the blame view + // and force the window relation to be primary, or we need the + // blame view's timestamp columns to be specific enough to not + // overlap during testing. + t.Skip("the blame table is unstable as secondary table in join with exchange node") } - }) - } + t.Run(fmt.Sprintf("%s(%s): %s", stt.name, c, tt.query), func(t *testing.T) { + if tt.skip { + t.Skip() + } + + ctx = ctx.WithQuery(tt.query) + if tt.exp != nil { + enginetest.TestQueryWithContext(t, ctx, e, harness, tt.query, tt.exp, nil, nil, nil) + } + }) + } + }) } } } @@ -1553,8 +1556,8 @@ func RunStatsHistogramTests(t *testing.T, h DoltEnginetestHarness) { } } -func RunStatsIOTests(t *testing.T, h DoltEnginetestHarness) { - for _, script := range append(DoltStatsIOTests, DoltHistogramTests...) { +func RunStatsStorageTests(t *testing.T, h DoltEnginetestHarness) { + for _, script := range append(DoltStatsStorageTests, DoltHistogramTests...) { func() { h = h.NewHarness(t).WithConfigureStats(true) defer h.Close() @@ -1569,7 +1572,7 @@ func RunStatsIOTests(t *testing.T, h DoltEnginetestHarness) { } func RunStatsIOTestsWithoutReload(t *testing.T, h DoltEnginetestHarness) { - for _, script := range append(DoltStatsIOTests, DoltHistogramTests...) { + for _, script := range append(DoltStatsStorageTests, DoltHistogramTests...) { func() { h = h.NewHarness(t).WithConfigureStats(true) defer h.Close() diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_harness.go b/go/libraries/doltcore/sqle/enginetest/dolt_harness.go index e8539db4da3..be5b92cb257 100644 --- a/go/libraries/doltcore/sqle/enginetest/dolt_harness.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_harness.go @@ -98,6 +98,7 @@ type DoltEnginetestHarness interface { WithSkippedQueries(skipped []string) DoltEnginetestHarness // WithParallelism returns a copy of the harness with parallelism set to the given number of threads + // Deprecated: parallelism currently no-ops WithParallelism(parallelism int) DoltEnginetestHarness // WithConfigureStats returns a copy of the harness with the given configureStats value diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_procedure_queries.go b/go/libraries/doltcore/sqle/enginetest/dolt_procedure_queries.go index c63c437c0c7..143ebaed886 100755 --- a/go/libraries/doltcore/sqle/enginetest/dolt_procedure_queries.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_procedure_queries.go @@ -409,4 +409,44 @@ end }, }, }, + + { + Name: "create and call procedure which exceeds 1024 bytes", + SetUpScript: []string{ + `CREATE PROCEDURE long_proc() +BEGIN + DECLARE long_text TEXT; + SET long_text = CONCAT( + 'Lorem ipsum dolor sit amet, consectetur adipiscing elit. ', + 'Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. ', + 'Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris ', + 'nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in ', + 'reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. ', + 'Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia ', + 'deserunt mollit anim id est laborum.', + 'Lorem ipsum dolor sit amet, consectetur adipiscing elit. ', + 'Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. ', + 'Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris ', + 'nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in ', + 'reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. ', + 'Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia ', + 'deserunt mollit anim id est laborum.', + 'Lorem ipsum dolor sit amet, consectetur adipiscing elit. ', + 'Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. ', + 'Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris ', + 'nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in ', + 'reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. ', + 'Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia ', + 'deserunt mollit anim id est laborum.'); + SELECT SHA2(long_text,256) AS checksum, LENGTH(long_text) AS length; +END +`, + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "call long_proc();", + Expected: []sql.Row{{"a702e99e5ee2dc03095bb2efd58e28330b6ea085d036249de82977a5c0dbb4be", 1335}}, + }, + }, + }, } diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_queries_diff.go b/go/libraries/doltcore/sqle/enginetest/dolt_queries_diff.go index e38dd4eefcf..2cd42880b82 100644 --- a/go/libraries/doltcore/sqle/enginetest/dolt_queries_diff.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_queries_diff.go @@ -528,7 +528,7 @@ var DiffSystemTableScriptTests = []queries.ScriptTest{ }, { Query: "SELECT COUNT(*) FROM DOLT_DIFF_t;", - Expected: []sql.Row{{1}}, + Expected: []sql.Row{{7}}, }, { Query: "SELECT to_pk, to_c1, from_pk, from_c1, diff_type FROM DOLT_DIFF_t where to_commit=@Commit4;", @@ -536,6 +536,36 @@ var DiffSystemTableScriptTests = []queries.ScriptTest{ }, }, }, + { + // Similar to previous test, but with one row to avoid ordering issues. + Name: "altered keyless table add pk", // https://github.com/dolthub/dolt/issues/8625 + SetUpScript: []string{ + "create table tbl (i int, j int);", + "insert into tbl values (42, 23);", + "call dolt_commit('-Am', 'commit1');", + "alter table tbl add primary key(i);", + "call dolt_commit('-am', 'commit2');", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "SELECT to_i,to_j,from_i,from_j,diff_type FROM dolt_diff_tbl;", + // Output in the situation is admittedly wonky. Updating the PK leaves in a place where we can't really render + // the diff, but we want to show something. In this case, the 'pk' column tag changes, so in the last two rows + // of the output you see we add "nil,23" and remove "nil,23" when in fact those columns were "42" with a different + // tag. + // + // In the past we just returned an empty set in this case. The + // warning is kind of essential to understand what is happening. + Expected: []sql.Row{ + {42, 23, nil, nil, "added"}, + {nil, nil, nil, 23, "removed"}, + }, + ExpectedWarningsCount: 1, + ExpectedWarning: 1105, + ExpectedWarningMessageSubstring: "due to primary key set change", + }, + }, + }, { Name: "table with commit column should maintain its data in diff", SetUpScript: []string{ @@ -713,8 +743,10 @@ var Dolt1DiffSystemTableScripts = []queries.ScriptTest{ }, Assertions: []queries.ScriptTestAssertion{ { - Query: "SELECT to_pk1, to_pk2, from_pk1, from_pk2, diff_type from dolt_diff_t;", - Expected: []sql.Row{{"2", "2", nil, nil, "added"}}, + Query: "SELECT to_pk1, to_pk2, from_pk1, from_pk2, diff_type from dolt_diff_t;", + Expected: []sql.Row{ + {"2", "2", nil, nil, "added"}, + }, }, }, }, @@ -1496,6 +1528,62 @@ on a.to_pk = b.to_pk;`, }, }, }, + { + Name: "diff table function works with virtual generated columns", + SetUpScript: []string{ + "create table t (i int primary key, j int, k int, jk int generated always as (10 * j + k), l int);", + "call dolt_commit('-Am', 'created table')", + "insert into t(i, j, k, l) values (1, 2, 3, 4);", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "select * from t;", + Expected: []sql.Row{ + {1, 2, 3, 23, 4}, + }, + }, + { + Query: "select to_i, to_jk, from_i, from_jk from dolt_diff_t;", + Expected: []sql.Row{ + {1, nil, nil, nil}, + }, + }, + { + Query: "select to_i, to_jk, from_i, from_jk from dolt_diff('HEAD', 'WORKING', 't');", + Expected: []sql.Row{ + {1, nil, nil, nil}, + }, + }, + }, + }, + { + Name: "diff table function works with stored generated columns", + SetUpScript: []string{ + "create table t (i int primary key, j int, k int, jk int generated always as (10 * j + k) stored, l int);", + "call dolt_commit('-Am', 'created table')", + "insert into t(i, j, k, l) values (1, 2, 3, 4);", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "select * from t;", + Expected: []sql.Row{ + {1, 2, 3, 23, 4}, + }, + }, + { + Query: "select to_i, to_jk, from_i, from_jk from dolt_diff_t;", + Expected: []sql.Row{ + {1, 23, nil, nil}, + }, + }, + { + Query: "select to_i, to_jk, from_i, from_jk from dolt_diff('HEAD', 'WORKING', 't');", + Expected: []sql.Row{ + {1, 23, nil, nil}, + }, + }, + }, + }, } var DiffStatTableFunctionScriptTests = []queries.ScriptTest{ @@ -5242,6 +5330,7 @@ var CommitDiffSystemTableScriptTests = []queries.ScriptTest{ }, }, }, + { Name: "added and dropped table", SetUpScript: []string{ @@ -6151,6 +6240,14 @@ var SystemTableIndexTests = []systabScript{ ORDER BY cm.date, cm.message asc`, exp: []sql.Row{{5}}, }, + { + query: "select count(*) /*+ JOIN_ORDER(a,b) */ from dolt_diff_xy a join xy b on x = to_x", + exp: []sql.Row{{45}}, + }, + { + query: "select count(*) /*+ JOIN_ORDER(b,a) */ from dolt_diff_xy a join xy b on x = to_x", + exp: []sql.Row{{45}}, + }, }, }, { diff --git a/go/libraries/doltcore/sqle/enginetest/stats_queries.go b/go/libraries/doltcore/sqle/enginetest/stats_queries.go index a844607e71b..fedb7297d5f 100644 --- a/go/libraries/doltcore/sqle/enginetest/stats_queries.go +++ b/go/libraries/doltcore/sqle/enginetest/stats_queries.go @@ -295,7 +295,7 @@ var DoltHistogramTests = []queries.ScriptTest{ }, } -var DoltStatsIOTests = []queries.ScriptTest{ +var DoltStatsStorageTests = []queries.ScriptTest{ { Name: "single-table", SetUpScript: []string{ @@ -569,6 +569,73 @@ var DoltStatsIOTests = []queries.ScriptTest{ }, }, }, + { + Name: "differentiate table cases", + SetUpScript: []string{ + "set @@PERSIST.dolt_stats_auto_refresh_interval = 0;", + "set @@PERSIST.dolt_stats_auto_refresh_threshold = 0;", + "set @@PERSIST.dolt_stats_branches ='main'", + "CREATE table XY (x bigint primary key, y varchar(16))", + "insert into XY values (0,'0'), (1,'1'), (2,'2')", + "analyze table XY", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "select table_name, upper_bound from dolt_statistics", + Expected: []sql.Row{{"xy", "2"}}, + }, + }, + }, + { + Name: "deleted table loads OK", + SetUpScript: []string{ + "set @@PERSIST.dolt_stats_auto_refresh_interval = 0;", + "set @@PERSIST.dolt_stats_auto_refresh_threshold = 0;", + "set @@PERSIST.dolt_stats_branches ='main'", + "CREATE table xy (x bigint primary key, y varchar(16))", + "insert into xy values (0,'0'), (1,'1'), (2,'2')", + "analyze table xy", + "CREATE table uv (u bigint primary key, v varchar(16))", + "insert into uv values (0,'0'), (1,'1'), (2,'2')", + "analyze table uv", + "drop table uv", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "select table_name, upper_bound from dolt_statistics", + Expected: []sql.Row{{"xy", "2"}}, + }, + }, + }, + { + Name: "differentiate branch names", + SetUpScript: []string{ + "set @@PERSIST.dolt_stats_auto_refresh_interval = 0;", + "set @@PERSIST.dolt_stats_auto_refresh_threshold = 0;", + "set @@PERSIST.dolt_stats_branches ='main,feat'", + "CREATE table xy (x bigint primary key, y varchar(16))", + "insert into xy values (0,'0'), (1,'1'), (2,'2')", + "analyze table xy", + "call dolt_checkout('-b', 'feat')", + "CREATE table xy (x varchar(16) primary key, y bigint, z bigint)", + "insert into xy values (3,'3',3)", + "analyze table xy", + "call dolt_checkout('main')", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "select table_name, upper_bound from dolt_statistics", + Expected: []sql.Row{{"xy", "2"}}, + }, + { + Query: "call dolt_checkout('feat')", + }, + { + Query: "select table_name, upper_bound from dolt_statistics", + Expected: []sql.Row{{"xy", "3"}}, + }, + }, + }, { Name: "drop primary key", SetUpScript: []string{ @@ -963,11 +1030,15 @@ func TestProviderReloadScriptWithEngine(t *testing.T, e enginetest.QueryEngine, t.Errorf("expected *gms.Engine but found: %T", e) } + branches := eng.Analyzer.Catalog.StatsProvider.(*statspro.Provider).TrackedBranches("mydb") + brCopy := make([]string, len(branches)) + copy(brCopy, branches) err := eng.Analyzer.Catalog.StatsProvider.DropDbStats(ctx, "mydb", false) require.NoError(t, err) - - err = eng.Analyzer.Catalog.StatsProvider.(*statspro.Provider).LoadStats(ctx, "mydb", "main") - require.NoError(t, err) + for _, branch := range brCopy { + err = eng.Analyzer.Catalog.StatsProvider.(*statspro.Provider).LoadStats(ctx, "mydb", branch) + require.NoError(t, err) + } } for _, assertion := range assertions { diff --git a/go/libraries/doltcore/sqle/index/dolt_index.go b/go/libraries/doltcore/sqle/index/dolt_index.go index a519656c9e7..8362facb818 100644 --- a/go/libraries/doltcore/sqle/index/dolt_index.go +++ b/go/libraries/doltcore/sqle/index/dolt_index.go @@ -141,18 +141,19 @@ func DoltDiffIndexesFromTable(ctx context.Context, db, tbl string, t *doltdb.Tab // to_ columns toIndex := doltIndex{ - id: "PRIMARY", - tblName: doltdb.DoltDiffTablePrefix + tbl, - dbName: db, - columns: toCols, - indexSch: sch, - tableSch: sch, - unique: true, - comment: "", - vrw: t.ValueReadWriter(), - ns: t.NodeStore(), - keyBld: keyBld, - order: sql.IndexOrderAsc, + id: "PRIMARY", + tblName: doltdb.DoltDiffTablePrefix + tbl, + dbName: db, + columns: toCols, + indexSch: sch, + tableSch: sch, + unique: true, + comment: "", + vrw: t.ValueReadWriter(), + ns: t.NodeStore(), + keyBld: keyBld, + // only ordered on PK within a diff partition + order: sql.IndexOrderNone, constrainedToLookupExpression: false, } diff --git a/go/libraries/doltcore/sqle/integration_test/database_revision_test.go b/go/libraries/doltcore/sqle/integration_test/database_revision_test.go index 42a08ac38eb..83e1ed0664d 100644 --- a/go/libraries/doltcore/sqle/integration_test/database_revision_test.go +++ b/go/libraries/doltcore/sqle/integration_test/database_revision_test.go @@ -152,7 +152,7 @@ func TestDbRevision(t *testing.T) { dEnv := dtestutils.CreateTestEnv() defer dEnv.DoltDB.Close() - cliCtx, _ := cmd.NewArgFreeCliContext(ctx, dEnv) + cliCtx, _ := cmd.NewArgFreeCliContext(ctx, dEnv, dEnv.FS) setup := append(setupCommon, test.setup...) for _, c := range setup { diff --git a/go/libraries/doltcore/sqle/integration_test/history_table_test.go b/go/libraries/doltcore/sqle/integration_test/history_table_test.go index 54282e8e73c..ec40abf4bbd 100644 --- a/go/libraries/doltcore/sqle/integration_test/history_table_test.go +++ b/go/libraries/doltcore/sqle/integration_test/history_table_test.go @@ -212,7 +212,7 @@ var INIT = "" // HEAD~4 func setupHistoryTests(t *testing.T) *env.DoltEnv { ctx := context.Background() dEnv := dtestutils.CreateTestEnv() - cliCtx, verr := cmd.NewArgFreeCliContext(ctx, dEnv) + cliCtx, verr := cmd.NewArgFreeCliContext(ctx, dEnv, dEnv.FS) require.NoError(t, verr) for _, c := range setupCommon { @@ -239,7 +239,7 @@ func setupHistoryTests(t *testing.T) *env.DoltEnv { func testHistoryTable(t *testing.T, test historyTableTest, dEnv *env.DoltEnv) { ctx := context.Background() - cliCtx, verr := cmd.NewArgFreeCliContext(ctx, dEnv) + cliCtx, verr := cmd.NewArgFreeCliContext(ctx, dEnv, dEnv.FS) require.NoError(t, verr) for _, c := range test.setup { diff --git a/go/libraries/doltcore/sqle/integration_test/json_value_test.go b/go/libraries/doltcore/sqle/integration_test/json_value_test.go index 1a693773719..4999d3cf2f8 100644 --- a/go/libraries/doltcore/sqle/integration_test/json_value_test.go +++ b/go/libraries/doltcore/sqle/integration_test/json_value_test.go @@ -127,7 +127,7 @@ func TestJsonValues(t *testing.T) { func testJsonValue(t *testing.T, test jsonValueTest, setupCommon []testCommand) { ctx := context.Background() dEnv := dtestutils.CreateTestEnv() - cliCtx, verr := cmd.NewArgFreeCliContext(ctx, dEnv) + cliCtx, verr := cmd.NewArgFreeCliContext(ctx, dEnv, dEnv.FS) require.NoError(t, verr) setup := append(setupCommon, test.setup...) diff --git a/go/libraries/doltcore/sqle/kvexec/builder.go b/go/libraries/doltcore/sqle/kvexec/builder.go index e649e0e24cf..7380c6f115f 100644 --- a/go/libraries/doltcore/sqle/kvexec/builder.go +++ b/go/libraries/doltcore/sqle/kvexec/builder.go @@ -524,7 +524,7 @@ func getMergeKv(ctx *sql.Context, n sql.Node) (mergeState, error) { //case *dtables.DiffTable: // TODO: add interface to include system tables default: - return ms, nil + return ms, fmt.Errorf("non-standard indexed table not supported") } if idx.Format() != types.Format_DOLT { diff --git a/go/libraries/doltcore/sqle/procedures_table.go b/go/libraries/doltcore/sqle/procedures_table.go index d90c138b7ee..28288e09674 100644 --- a/go/libraries/doltcore/sqle/procedures_table.go +++ b/go/libraries/doltcore/sqle/procedures_table.go @@ -21,10 +21,13 @@ import ( "time" "github.com/dolthub/go-mysql-server/sql" + gmstypes "github.com/dolthub/go-mysql-server/sql/types" + "github.com/dolthub/vitess/go/sqltypes" "gopkg.in/src-d/go-errors.v1" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" "github.com/dolthub/dolt/go/libraries/doltcore/schema" + "github.com/dolthub/dolt/go/libraries/doltcore/schema/typeinfo" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dtables" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/index" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/sqlutil" @@ -130,9 +133,21 @@ func ProceduresTableSqlSchema() sql.PrimaryKeySchema { // The fixed dolt schema for the `dolt_procedures` table. func ProceduresTableSchema() schema.Schema { + // Max len is 8K - big enough to be useful, but not too big to exceed max total row len of 64K. + t, err := gmstypes.CreateStringWithDefaults(sqltypes.VarChar, typeinfo.MaxVarcharLength/2) + if err != nil { + panic(err) // should never happen. All constants. + } + ti := typeinfo.CreateVarStringTypeFromSqlType(t) + + stmtCol, err := schema.NewColumnWithTypeInfo(doltdb.ProceduresTableCreateStmtCol, schema.DoltProceduresCreateStmtTag, ti, false, "", false, "") + if err != nil { + panic(err) // should never happen. + } + colColl := schema.NewColCollection( schema.NewColumn(doltdb.ProceduresTableNameCol, schema.DoltProceduresNameTag, types.StringKind, true, schema.NotNullConstraint{}), - schema.NewColumn(doltdb.ProceduresTableCreateStmtCol, schema.DoltProceduresCreateStmtTag, types.StringKind, false), + stmtCol, schema.NewColumn(doltdb.ProceduresTableCreatedAtCol, schema.DoltProceduresCreatedAtTag, types.TimestampKind, false), schema.NewColumn(doltdb.ProceduresTableModifiedAtCol, schema.DoltProceduresModifiedAtTag, types.TimestampKind, false), schema.NewColumn(doltdb.ProceduresTableSqlModeCol, schema.DoltProceduresSqlModeTag, types.StringKind, false), diff --git a/go/libraries/doltcore/sqle/statsnoms/database.go b/go/libraries/doltcore/sqle/statsnoms/database.go index 2c1491e18fa..e7fee45b158 100644 --- a/go/libraries/doltcore/sqle/statsnoms/database.go +++ b/go/libraries/doltcore/sqle/statsnoms/database.go @@ -87,12 +87,14 @@ func (sf NomsStatsFactory) Init(ctx *sql.Context, sourceDb dsess.SqlDatabase, pr dEnv = env.LoadWithoutDB(ctx, hdp, statsFs, "") } - ddb, err := doltdb.LoadDoltDBWithParams(ctx, types.Format_Default, urlPath, statsFs, params) - if err != nil { - return nil, err - } + if dEnv.DoltDB == nil { + ddb, err := doltdb.LoadDoltDBWithParams(ctx, types.Format_Default, urlPath, statsFs, params) + if err != nil { + return nil, err + } - dEnv.DoltDB = ddb + dEnv.DoltDB = ddb + } deaf := dEnv.DbEaFactory() @@ -139,6 +141,28 @@ func (n *NomsStatsDatabase) Branches() []string { } func (n *NomsStatsDatabase) LoadBranchStats(ctx *sql.Context, branch string) error { + branchQDbName := statspro.BranchQualifiedDatabase(n.sourceDb.Name(), branch) + + dSess := dsess.DSessFromSess(ctx.Session) + sqlDb, err := dSess.Provider().Database(ctx, branchQDbName) + if err != nil { + ctx.GetLogger().Debugf("statistics load: branch not found: %s; `call dolt_stats_prune()` to delete stale statistics", branch) + return nil + } + branchQDb, ok := sqlDb.(dsess.SqlDatabase) + if !ok { + return fmt.Errorf("branch/database not found: %s", branchQDbName) + } + + if ok, err := n.SchemaChange(ctx, branch, branchQDb); err != nil { + return err + } else if ok { + ctx.GetLogger().Debugf("statistics load: detected schema change incompatility, purging %s/%s", branch, n.sourceDb.Name()) + if err := n.DeleteBranchStats(ctx, branch, true); err != nil { + return err + } + } + statsMap, err := n.destDb.DbData().Ddb.GetStatistics(ctx, branch) if errors.Is(err, doltdb.ErrNoStatistics) { return n.trackBranch(ctx, branch) @@ -152,7 +176,8 @@ func (n *NomsStatsDatabase) LoadBranchStats(ctx *sql.Context, branch string) err } else if cnt == 0 { return n.trackBranch(ctx, branch) } - doltStats, err := loadStats(ctx, n.sourceDb, statsMap) + + doltStats, err := loadStats(ctx, branchQDb, statsMap) if err != nil { return err } @@ -164,6 +189,58 @@ func (n *NomsStatsDatabase) LoadBranchStats(ctx *sql.Context, branch string) err return nil } +func (n *NomsStatsDatabase) SchemaChange(ctx *sql.Context, branch string, branchQDb dsess.SqlDatabase) (bool, error) { + root, err := branchQDb.GetRoot(ctx) + if err != nil { + return false, err + } + tables, err := branchQDb.GetTableNames(ctx) + if err != nil { + return false, err + } + + var keys []string + var schHashes []hash.Hash + for _, tableName := range tables { + table, ok, err := root.GetTable(ctx, doltdb.TableName{Name: tableName}) + if err != nil { + return false, err + } + if !ok { + return false, nil + } + curHash, err := table.GetSchemaHash(ctx) + if err != nil { + return false, err + } + + keys = append(keys, n.schemaTupleKey(branch, tableName)) + schHashes = append(schHashes, curHash) + } + + ddb := n.destDb.DbData().Ddb + var schemaChange bool + for i, key := range keys { + curHash := schHashes[i] + if val, ok, err := ddb.GetTuple(ctx, key); err != nil { + return false, err + } else if ok { + oldHash := hash.Parse(string(val)) + if !ok || !oldHash.Equal(curHash) { + schemaChange = true + break + } + } + } + if schemaChange { + for _, key := range keys { + ddb.DeleteTuple(ctx, key) + } + return true, nil + } + return false, nil +} + func (n *NomsStatsDatabase) getBranchStats(branch string) dbStats { for i, b := range n.branches { if strings.EqualFold(b, branch) { @@ -272,6 +349,7 @@ func (n *NomsStatsDatabase) DeleteBranchStats(ctx *sql.Context, branch string, f n.dirty = append(n.dirty[:i], n.dirty[i+1:]...) n.stats = append(n.stats[:i], n.stats[i+1:]...) n.tableHashes = append(n.tableHashes[:i], n.tableHashes[i+1:]...) + n.schemaHashes = append(n.schemaHashes[:i], n.schemaHashes[i+1:]...) } } if flush { @@ -364,24 +442,54 @@ func (n *NomsStatsDatabase) SetTableHash(branch, tableName string, h hash.Hash) } } -func (n *NomsStatsDatabase) GetSchemaHash(branch, tableName string) hash.Hash { +func (n *NomsStatsDatabase) GetSchemaHash(ctx context.Context, branch, tableName string) (hash.Hash, error) { n.mu.Lock() defer n.mu.Unlock() for i, b := range n.branches { if strings.EqualFold(branch, b) { - return n.schemaHashes[i][tableName] + return n.schemaHashes[i][tableName], nil + } + if val, ok, err := n.destDb.DbData().Ddb.GetTuple(ctx, n.schemaTupleKey(branch, tableName)); ok { + if err != nil { + return hash.Hash{}, err + } + h := hash.Parse(string(val)) + n.schemaHashes[i][tableName] = h + return h, nil + } else if err != nil { + return hash.Hash{}, err } + break } - return hash.Hash{} + return hash.Hash{}, nil +} + +func (n *NomsStatsDatabase) schemaTupleKey(branch, tableName string) string { + return n.sourceDb.Name() + "/" + branch + "/" + tableName } -func (n *NomsStatsDatabase) SetSchemaHash(branch, tableName string, h hash.Hash) { +func (n *NomsStatsDatabase) SetSchemaHash(ctx context.Context, branch, tableName string, h hash.Hash) error { n.mu.Lock() defer n.mu.Unlock() + branchIdx := -1 for i, b := range n.branches { if strings.EqualFold(branch, b) { - n.schemaHashes[i][tableName] = h + branchIdx = i break } } + if branchIdx < 0 { + branchIdx = len(n.branches) + if err := n.trackBranch(ctx, branch); err != nil { + return err + } + } + + n.schemaHashes[branchIdx][tableName] = h + key := n.schemaTupleKey(branch, tableName) + if err := n.destDb.DbData().Ddb.DeleteTuple(ctx, key); err != doltdb.ErrTupleNotFound { + return err + } + + return n.destDb.DbData().Ddb.SetTuple(ctx, key, []byte(h.String())) } diff --git a/go/libraries/doltcore/sqle/statsnoms/load.go b/go/libraries/doltcore/sqle/statsnoms/load.go index 55b438b1cab..72051260260 100644 --- a/go/libraries/doltcore/sqle/statsnoms/load.go +++ b/go/libraries/doltcore/sqle/statsnoms/load.go @@ -45,6 +45,7 @@ func loadStats(ctx *sql.Context, db dsess.SqlDatabase, m prolly.Map) (map[sql.St return nil, err } currentStat := statspro.NewDoltStats() + invalidTables := make(map[string]bool) for { row, err := iter.Next(ctx) if errors.Is(err, io.EOF) { @@ -74,27 +75,31 @@ func loadStats(ctx *sql.Context, db dsess.SqlDatabase, m prolly.Map) (map[sql.St } qual := sql.NewStatQualifier(dbName, schemaName, tableName, indexName) + if _, ok := invalidTables[tableName]; ok { + continue + } + if currentStat.Statistic.Qual.String() != qual.String() { if !currentStat.Statistic.Qual.Empty() { - currentStat.Statistic.LowerBnd, currentStat.Tb, err = loadLowerBound(ctx, db, currentStat.Statistic.Qual, len(currentStat.Columns())) - if err != nil { - return nil, err - } - fds, colSet, err := loadFuncDeps(ctx, db, currentStat.Statistic.Qual) - if err != nil { - return nil, err - } - currentStat.Statistic.Fds = fds - currentStat.Statistic.Colset = colSet currentStat.UpdateActive() qualToStats[currentStat.Statistic.Qual] = currentStat } currentStat = statspro.NewDoltStats() - currentStat.Statistic.Qual = qual - currentStat.Statistic.Cols = columns - currentStat.Statistic.LowerBnd, currentStat.Tb, err = loadLowerBound(ctx, db, currentStat.Statistic.Qual, len(currentStat.Columns())) - if err != nil { + + tab, ok, err := db.GetTableInsensitive(ctx, qual.Table()) + if ok { + currentStat.Statistic.Qual = qual + currentStat.Statistic.Cols = columns + currentStat.Statistic.LowerBnd, currentStat.Tb, currentStat.Statistic.Fds, currentStat.Statistic.Colset, err = loadRefdProps(ctx, db, tab, currentStat.Statistic.Qual, len(currentStat.Columns())) + if err != nil { + return nil, err + } + } else if !ok { + ctx.GetLogger().Debugf("stats load: table previously collected is missing from root: %s", tableName) + invalidTables[qual.Table()] = true + continue + } else if err != nil { return nil, err } } @@ -168,18 +173,10 @@ func loadStats(ctx *sql.Context, db dsess.SqlDatabase, m prolly.Map) (map[sql.St currentStat.Statistic.Created = createdAt } } - currentStat.Statistic.LowerBnd, currentStat.Tb, err = loadLowerBound(ctx, db, currentStat.Statistic.Qual, len(currentStat.Columns())) - if err != nil { - return nil, err - } - fds, colSet, err := loadFuncDeps(ctx, db, currentStat.Statistic.Qual) - if err != nil { - return nil, err + if !currentStat.Qualifier().Empty() { + currentStat.UpdateActive() + qualToStats[currentStat.Statistic.Qual] = currentStat } - currentStat.Statistic.Fds = fds - currentStat.Statistic.Colset = colSet - currentStat.UpdateActive() - qualToStats[currentStat.Statistic.Qual] = currentStat return qualToStats, nil } @@ -195,14 +192,44 @@ func parseTypeStrings(typs []string) ([]sql.Type, error) { return ret, nil } -func loadLowerBound(ctx *sql.Context, db dsess.SqlDatabase, qual sql.StatQualifier, cols int) (sql.Row, *val.TupleBuilder, error) { +func loadRefdProps(ctx *sql.Context, db dsess.SqlDatabase, sqlTable sql.Table, qual sql.StatQualifier, cols int) (sql.Row, *val.TupleBuilder, *sql.FuncDepSet, sql.ColSet, error) { root, err := db.GetRoot(ctx) - table, ok, err := root.GetTable(ctx, doltdb.TableName{Name: qual.Table()}) + if err != nil { + return nil, nil, nil, sql.ColSet{}, err + } + + iat, ok := sqlTable.(sql.IndexAddressable) + if !ok { + return nil, nil, nil, sql.ColSet{}, nil + } + + indexes, err := iat.GetIndexes(ctx) + if err != nil { + return nil, nil, nil, sql.ColSet{}, err + } + + var sqlIdx sql.Index + for _, i := range indexes { + if strings.EqualFold(i.ID(), qual.Index()) { + sqlIdx = i + break + } + } + + if sqlIdx == nil { + return nil, nil, nil, sql.ColSet{}, fmt.Errorf("%w: index not found: '%s'", statspro.ErrFailedToLoad, qual.Index()) + } + + fds, colset, err := stats.IndexFds(qual.Table(), sqlTable.Schema(), sqlIdx) + if err != nil { + return nil, nil, nil, sql.ColSet{}, err + } + table, ok, err := root.GetTable(ctx, doltdb.TableName{Name: sqlTable.Name()}) if !ok { - return nil, nil, sql.ErrTableNotFound.New(qual.Table()) + return nil, nil, nil, sql.ColSet{}, sql.ErrTableNotFound.New(qual.Table()) } if err != nil { - return nil, nil, err + return nil, nil, nil, sql.ColSet{}, err } var idx durable.Index @@ -212,7 +239,7 @@ func loadLowerBound(ctx *sql.Context, db dsess.SqlDatabase, qual sql.StatQualifi idx, err = table.GetIndexRowData(ctx, qual.Index()) } if err != nil { - return nil, nil, err + return nil, nil, nil, sql.ColSet{}, err } prollyMap := durable.ProllyMapFromIndex(idx) @@ -220,17 +247,17 @@ func loadLowerBound(ctx *sql.Context, db dsess.SqlDatabase, qual sql.StatQualifi buffPool := prollyMap.NodeStore().Pool() if cnt, err := prollyMap.Count(); err != nil { - return nil, nil, err + return nil, nil, nil, sql.ColSet{}, err } else if cnt == 0 { - return nil, keyBuilder, nil + return nil, keyBuilder, nil, sql.ColSet{}, nil } firstIter, err := prollyMap.IterOrdinalRange(ctx, 0, 1) if err != nil { - return nil, nil, err + return nil, nil, nil, sql.ColSet{}, err } keyBytes, _, err := firstIter.Next(ctx) if err != nil { - return nil, nil, err + return nil, nil, nil, sql.ColSet{}, err } for i := range keyBuilder.Desc.Types { keyBuilder.PutRaw(i, keyBytes.GetField(i)) @@ -241,10 +268,10 @@ func loadLowerBound(ctx *sql.Context, db dsess.SqlDatabase, qual sql.StatQualifi for i := 0; i < keyBuilder.Desc.Count(); i++ { firstRow[i], err = tree.GetField(ctx, prollyMap.KeyDesc(), i, firstKey, prollyMap.NodeStore()) if err != nil { - return nil, nil, err + return nil, nil, nil, sql.ColSet{}, err } } - return firstRow, keyBuilder, nil + return firstRow, keyBuilder, fds, colset, nil } func loadFuncDeps(ctx *sql.Context, db dsess.SqlDatabase, qual sql.StatQualifier) (*sql.FuncDepSet, sql.ColSet, error) { diff --git a/go/libraries/doltcore/sqle/statspro/analyze.go b/go/libraries/doltcore/sqle/statspro/analyze.go index 0672b6f6f8c..faa1869315c 100644 --- a/go/libraries/doltcore/sqle/statspro/analyze.go +++ b/go/libraries/doltcore/sqle/statspro/analyze.go @@ -47,7 +47,7 @@ func (p *Provider) BootstrapDatabaseStats(ctx *sql.Context, db string) error { branches := p.getStatsBranches(ctx) var rows uint64 for _, branch := range branches { - sqlDb, err := dSess.Provider().Database(ctx, p.branchQualifiedDatabase(db, branch)) + sqlDb, err := dSess.Provider().Database(ctx, BranchQualifiedDatabase(db, branch)) if err != nil { if sql.ErrDatabaseNotFound.Is(err) { // default branch is not valid @@ -91,7 +91,7 @@ func (p *Provider) RefreshTableStatsWithBranch(ctx *sql.Context, table sql.Table dSess := dsess.DSessFromSess(ctx.Session) - sqlDb, err := dSess.Provider().Database(ctx, p.branchQualifiedDatabase(db, branch)) + sqlDb, err := dSess.Provider().Database(ctx, BranchQualifiedDatabase(db, branch)) if err != nil { return err } @@ -144,11 +144,15 @@ func (p *Provider) RefreshTableStatsWithBranch(ctx *sql.Context, table sql.Table return err } - if oldSchHash := statDb.GetSchemaHash(branch, tableName); oldSchHash.IsEmpty() { - statDb.SetSchemaHash(branch, tableName, schHash) + if oldSchHash, err := statDb.GetSchemaHash(ctx, branch, tableName); oldSchHash.IsEmpty() { + if err := statDb.SetSchemaHash(ctx, branch, tableName, schHash); err != nil { + return fmt.Errorf("set schema hash error: %w", err) + } } else if oldSchHash != schHash { ctx.GetLogger().Debugf("statistics refresh: detected table schema change: %s,%s/%s", dbName, table, branch) - statDb.SetSchemaHash(branch, tableName, schHash) + if err := statDb.SetSchemaHash(ctx, branch, tableName, schHash); err != nil { + return err + } stats, err := p.GetTableDoltStats(ctx, branch, dbName, schemaName, tableName) if err != nil { @@ -157,6 +161,8 @@ func (p *Provider) RefreshTableStatsWithBranch(ctx *sql.Context, table sql.Table for _, stat := range stats { statDb.DeleteStats(ctx, branch, stat.Qualifier()) } + } else if err != nil { + return err } tablePrefix := fmt.Sprintf("%s.", tableName) @@ -208,9 +214,9 @@ func (p *Provider) RefreshTableStatsWithBranch(ctx *sql.Context, table sql.Table return statDb.Flush(ctx, branch) } -// branchQualifiedDatabase returns a branch qualified database. If the database +// BranchQualifiedDatabase returns a branch qualified database. If the database // is already branch suffixed no duplication is applied. -func (p *Provider) branchQualifiedDatabase(db, branch string) string { +func BranchQualifiedDatabase(db, branch string) string { suffix := fmt.Sprintf("/%s", branch) if !strings.HasSuffix(db, suffix) { return fmt.Sprintf("%s%s", db, suffix) diff --git a/go/libraries/doltcore/sqle/statspro/auto_refresh.go b/go/libraries/doltcore/sqle/statspro/auto_refresh.go index a5640479f20..3322065f809 100644 --- a/go/libraries/doltcore/sqle/statspro/auto_refresh.go +++ b/go/libraries/doltcore/sqle/statspro/auto_refresh.go @@ -85,7 +85,7 @@ func (p *Provider) InitAutoRefreshWithParams(ctxFactory func(ctx context.Context if br, ok, err := ddb.HasBranch(ctx, branch); ok { sqlCtx.GetLogger().Debugf("starting statistics refresh check for '%s': %s", dbName, time.Now().String()) // update WORKING session references - sqlDb, err := dSess.Provider().Database(sqlCtx, p.branchQualifiedDatabase(dbName, branch)) + sqlDb, err := dSess.Provider().Database(sqlCtx, BranchQualifiedDatabase(dbName, branch)) if err != nil { sqlCtx.GetLogger().Debugf("statistics refresh error: %s", err.Error()) return @@ -168,12 +168,15 @@ func (p *Provider) checkRefresh(ctx *sql.Context, sqlDb sql.Database, dbName, br schemaName = strings.ToLower(schTab.DatabaseSchema().SchemaName()) } - if oldSchHash := statDb.GetSchemaHash(branch, table); oldSchHash.IsEmpty() { - statDb.SetSchemaHash(branch, table, schHash) + if oldSchHash, err := statDb.GetSchemaHash(ctx, branch, table); oldSchHash.IsEmpty() { + if err := statDb.SetSchemaHash(ctx, branch, table, schHash); err != nil { + return err + } } else if oldSchHash != schHash { ctx.GetLogger().Debugf("statistics refresh: detected table schema change: %s,%s/%s", dbName, table, branch) - statDb.SetSchemaHash(branch, table, schHash) - + if err := statDb.SetSchemaHash(ctx, branch, table, schHash); err != nil { + return err + } stats, err := p.GetTableDoltStats(ctx, branch, dbName, schemaName, table) if err != nil { return err @@ -181,6 +184,8 @@ func (p *Provider) checkRefresh(ctx *sql.Context, sqlDb sql.Database, dbName, br for _, stat := range stats { statDb.DeleteStats(ctx, branch, stat.Qualifier()) } + } else if err != nil { + return err } iat, ok := sqlTable.(sql.IndexAddressableTable) diff --git a/go/libraries/doltcore/sqle/statspro/configure.go b/go/libraries/doltcore/sqle/statspro/configure.go index 1931f4f919a..f8492a08b61 100644 --- a/go/libraries/doltcore/sqle/statspro/configure.go +++ b/go/libraries/doltcore/sqle/statspro/configure.go @@ -29,6 +29,8 @@ import ( "github.com/dolthub/dolt/go/libraries/utils/filesys" ) +var helpMsg = "call dolt_stats_purge() to reset statistics" + func (p *Provider) Configure(ctx context.Context, ctxFactory func(ctx context.Context) (*sql.Context, error), bThreads *sql.BackgroundThreads, dbs []dsess.SqlDatabase) error { p.SetStarter(NewStatsInitDatabaseHook(p, ctxFactory, bThreads)) @@ -134,7 +136,7 @@ func (p *Provider) Load(ctx *sql.Context, fs filesys.Filesys, db dsess.SqlDataba // |statPath| is either file://./stat or mem://stat statsDb, err := p.sf.Init(ctx, db, p.pro, fs, env.GetCurrentUserHomeDir) if err != nil { - ctx.GetLogger().Errorf("initialize stats failure: %s\n", err.Error()) + ctx.GetLogger().Errorf("initialize stats failure for %s: %s; %s\n", db.Name(), err.Error(), helpMsg) return } @@ -142,11 +144,11 @@ func (p *Provider) Load(ctx *sql.Context, fs filesys.Filesys, db dsess.SqlDataba if err = statsDb.LoadBranchStats(ctx, branch); err != nil { // if branch name is invalid, continue loading rest // TODO: differentiate bad branch name from other errors - ctx.GetLogger().Errorf("load stats init failure: %s\n", err.Error()) + ctx.GetLogger().Errorf("load stats init failure for %s: %s; %s\n", db.Name(), err.Error(), helpMsg) continue } if err := statsDb.Flush(ctx, branch); err != nil { - ctx.GetLogger().Errorf("load stats flush failure: %s\n", err.Error()) + ctx.GetLogger().Errorf("load stats flush failure for %s: %s; %s\n", db.Name(), err.Error(), helpMsg) continue } } diff --git a/go/libraries/doltcore/sqle/statspro/initdbhook.go b/go/libraries/doltcore/sqle/statspro/initdbhook.go index 17d2cf8a406..8e11408ea59 100644 --- a/go/libraries/doltcore/sqle/statspro/initdbhook.go +++ b/go/libraries/doltcore/sqle/statspro/initdbhook.go @@ -16,6 +16,7 @@ package statspro import ( "context" + "fmt" "strings" "github.com/dolthub/go-mysql-server/sql" @@ -38,13 +39,36 @@ func NewStatsInitDatabaseHook( db dsess.SqlDatabase, ) error { dbName := strings.ToLower(db.Name()) - if _, ok := statsProv.getStatDb(dbName); !ok { + if statsDb, ok := statsProv.getStatDb(dbName); !ok { statsDb, err := statsProv.sf.Init(ctx, db, statsProv.pro, denv.FS, env.GetCurrentUserHomeDir) if err != nil { ctx.GetLogger().Debugf("statistics load error: %s", err.Error()) return nil } statsProv.setStatDb(dbName, statsDb) + } else { + dSess := dsess.DSessFromSess(ctx.Session) + for _, br := range statsDb.Branches() { + branchQDbName := BranchQualifiedDatabase(dbName, br) + sqlDb, err := dSess.Provider().Database(ctx, branchQDbName) + if err != nil { + ctx.GetLogger().Logger.Errorf("branch not found: %s", br) + continue + } + branchQDb, ok := sqlDb.(dsess.SqlDatabase) + if !ok { + return fmt.Errorf("branch/database not found: %s", branchQDbName) + } + + if ok, err := statsDb.SchemaChange(ctx, br, branchQDb); err != nil { + return err + } else if ok { + if err := statsDb.DeleteBranchStats(ctx, br, true); err != nil { + return err + } + } + } + ctx.GetLogger().Debugf("statistics init error: preexisting stats db: %s", dbName) } ctx.GetLogger().Debugf("statistics refresh: initialize %s", name) return statsProv.InitAutoRefresh(ctxFactory, name, bThreads) @@ -62,6 +86,7 @@ func NewStatsDropDatabaseHook(statsProv *Provider) sqle.DropDatabaseHook { if err := db.Close(); err != nil { ctx.GetLogger().Debugf("failed to close stats database: %s", err) } + delete(statsProv.statDbs, name) } } } diff --git a/go/libraries/doltcore/sqle/statspro/interface.go b/go/libraries/doltcore/sqle/statspro/interface.go index 987de042069..5a423466f91 100644 --- a/go/libraries/doltcore/sqle/statspro/interface.go +++ b/go/libraries/doltcore/sqle/statspro/interface.go @@ -50,11 +50,20 @@ type Database interface { Flush(ctx context.Context, branch string) error // Close finalizes any file references. Close() error + // SetTableHash updates the most recently tracked table stats table hash SetTableHash(branch, tableName string, h hash.Hash) + // GetTableHash returns the most recently tracked table stats table hash GetTableHash(branch, tableName string) hash.Hash - SetSchemaHash(branch, tableName string, h hash.Hash) - GetSchemaHash(branch, tableName string) hash.Hash + // SetSchemaHash updates the most recently stored table stat's schema hash + SetSchemaHash(ctx context.Context, branch, tableName string, h hash.Hash) error + // GetSchemaHash returns the schema hash for the latest stored statistics + GetSchemaHash(ctx context.Context, branch, tableName string) (hash.Hash, error) + // Branches returns the set of branches with tracked statistics databases Branches() []string + // SchemaChange returns false if any table schema in the session + // root is incompatible with the latest schema used to create a stored + // set of statistics. + SchemaChange(ctx *sql.Context, branch string, branchQdb dsess.SqlDatabase) (bool, error) } // StatsFactory instances construct statistic databases. diff --git a/go/libraries/doltcore/sqle/statspro/stats_provider.go b/go/libraries/doltcore/sqle/statspro/stats_provider.go index 4e05e60e26a..573e20b638a 100644 --- a/go/libraries/doltcore/sqle/statspro/stats_provider.go +++ b/go/libraries/doltcore/sqle/statspro/stats_provider.go @@ -162,6 +162,15 @@ func (p *Provider) ThreadStatus(dbName string) string { return "no active stats thread" } +func (p *Provider) TrackedBranches(dbName string) []string { + db, ok := p.getStatDb(dbName) + if !ok { + return nil + } + return db.Branches() + +} + func (p *Provider) GetTableStats(ctx *sql.Context, db string, table sql.Table) ([]sql.Statistic, error) { dSess := dsess.DSessFromSess(ctx.Session) branch, err := dSess.GetBranch() diff --git a/go/libraries/doltcore/sqle/system_variables.go b/go/libraries/doltcore/sqle/system_variables.go index afc7a5dd943..ae5cd34e8b3 100644 --- a/go/libraries/doltcore/sqle/system_variables.go +++ b/go/libraries/doltcore/sqle/system_variables.go @@ -244,7 +244,7 @@ var DoltSystemVariables = []sql.SystemVariable{ Dynamic: true, Scope: sql.GetMysqlScope(sql.SystemVariableScope_Global), Type: types.NewSystemIntType(dsess.DoltStatsAutoRefreshInterval, 0, math.MaxInt, false), - Default: 120, + Default: 600, }, &sql.MysqlSystemVariable{ Name: dsess.DoltStatsBranches, diff --git a/go/libraries/doltcore/sqle/tables.go b/go/libraries/doltcore/sqle/tables.go index c7bbe80f495..7eac3dadf61 100644 --- a/go/libraries/doltcore/sqle/tables.go +++ b/go/libraries/doltcore/sqle/tables.go @@ -937,7 +937,7 @@ func emptyFulltextTable( return nil, nil, err } - empty, err := durable.NewEmptyIndex(ctx, dt.ValueReadWriter(), dt.NodeStore(), doltSchema, false) + empty, err := durable.NewEmptyPrimaryIndex(ctx, dt.ValueReadWriter(), dt.NodeStore(), doltSchema) if err != nil { return nil, nil, err } @@ -1041,7 +1041,7 @@ func (t *WritableDoltTable) truncate( } for _, idx := range sch.Indexes().AllIndexes() { - empty, err := durable.NewEmptyIndex(ctx, table.ValueReadWriter(), table.NodeStore(), idx.Schema(), false) + empty, err := durable.NewEmptyIndexFromTableSchema(ctx, table.ValueReadWriter(), table.NodeStore(), idx, sch) if err != nil { return nil, err } @@ -1065,7 +1065,7 @@ func (t *WritableDoltTable) truncate( } } - empty, err := durable.NewEmptyIndex(ctx, table.ValueReadWriter(), table.NodeStore(), sch, false) + empty, err := durable.NewEmptyPrimaryIndex(ctx, table.ValueReadWriter(), table.NodeStore(), sch) if err != nil { return nil, err } diff --git a/go/libraries/doltcore/sqle/temp_table.go b/go/libraries/doltcore/sqle/temp_table.go index daabc958452..6a07a804d02 100644 --- a/go/libraries/doltcore/sqle/temp_table.go +++ b/go/libraries/doltcore/sqle/temp_table.go @@ -109,7 +109,7 @@ func NewTempTable( vrw := ddb.ValueReadWriter() ns := ddb.NodeStore() - idx, err := durable.NewEmptyIndex(ctx, vrw, ns, sch, false) + idx, err := durable.NewEmptyPrimaryIndex(ctx, vrw, ns, sch) if err != nil { return nil, err } diff --git a/go/libraries/doltcore/sqle/testutil.go b/go/libraries/doltcore/sqle/testutil.go index aa6276acbf2..9af4a26bcba 100644 --- a/go/libraries/doltcore/sqle/testutil.go +++ b/go/libraries/doltcore/sqle/testutil.go @@ -425,7 +425,7 @@ func CreateEmptyTestTable(dEnv *env.DoltEnv, tableName string, sch schema.Schema vrw := dEnv.DoltDB.ValueReadWriter() ns := dEnv.DoltDB.NodeStore() - rows, err := durable.NewEmptyIndex(ctx, vrw, ns, sch, false) + rows, err := durable.NewEmptyPrimaryIndex(ctx, vrw, ns, sch) if err != nil { return err } diff --git a/go/libraries/doltcore/table/editor/creation/external_build_index.go b/go/libraries/doltcore/table/editor/creation/external_build_index.go index 1b7f2e20846..878cb456602 100644 --- a/go/libraries/doltcore/table/editor/creation/external_build_index.go +++ b/go/libraries/doltcore/table/editor/creation/external_build_index.go @@ -41,7 +41,7 @@ const ( // single prolly tree materialization by presorting the index keys in an // intermediate file format. func BuildProllyIndexExternal(ctx *sql.Context, vrw types.ValueReadWriter, ns tree.NodeStore, sch schema.Schema, tableName string, idx schema.Index, primary prolly.Map, uniqCb DupEntryCb) (durable.Index, error) { - empty, err := durable.NewEmptyIndex(ctx, vrw, ns, idx.Schema(), schema.IsKeyless(sch)) + empty, err := durable.NewEmptyIndexFromTableSchema(ctx, vrw, ns, idx, sch) if err != nil { return nil, err } diff --git a/go/libraries/doltcore/table/editor/creation/index.go b/go/libraries/doltcore/table/editor/creation/index.go index 7c9ea3a6dc2..b52a56f0342 100644 --- a/go/libraries/doltcore/table/editor/creation/index.go +++ b/go/libraries/doltcore/table/editor/creation/index.go @@ -214,7 +214,7 @@ func BuildUniqueProllyIndex( primary prolly.Map, cb DupEntryCb, ) (durable.Index, error) { - empty, err := durable.NewEmptyIndex(ctx, vrw, ns, idx.Schema(), schema.IsKeyless(sch)) + empty, err := durable.NewEmptyIndexFromTableSchema(ctx, vrw, ns, idx, sch) if err != nil { return nil, err } diff --git a/go/libraries/doltcore/table/untyped/sqlexport/sqlwriter_test.go b/go/libraries/doltcore/table/untyped/sqlexport/sqlwriter_test.go index 0332d7576ef..a1580c15318 100644 --- a/go/libraries/doltcore/table/untyped/sqlexport/sqlwriter_test.go +++ b/go/libraries/doltcore/table/untyped/sqlexport/sqlwriter_test.go @@ -92,7 +92,7 @@ func TestEndToEnd(t *testing.T) { root, err := dEnv.WorkingRoot(ctx) require.NoError(t, err) - empty, err := durable.NewEmptyIndex(ctx, root.VRW(), root.NodeStore(), tt.sch, false) + empty, err := durable.NewEmptyPrimaryIndex(ctx, root.VRW(), root.NodeStore(), tt.sch) require.NoError(t, err) indexes, err := durable.NewIndexSet(ctx, root.VRW(), root.NodeStore()) diff --git a/go/libraries/utils/argparser/parser_test.go b/go/libraries/utils/argparser/parser_test.go index 225a5cdffdd..452844343ca 100644 --- a/go/libraries/utils/argparser/parser_test.go +++ b/go/libraries/utils/argparser/parser_test.go @@ -169,3 +169,16 @@ func TestArgParser(t *testing.T) { } } } + +func TestArgParserSet(t *testing.T) { + ap := createParserWithOptionalArgs() + apr, err := ap.Parse([]string{"-o", "optional value", "-f", "foo", "bar"}) + require.NoError(t, err) + + apr, err = apr.SetArgument("param", "abcdefg") + require.NoError(t, err) + assert.Equal(t, map[string]string{"flag": "", "optional": "optional value", "param": "abcdefg"}, apr.options) + + apr, err = apr.SetArgument("garbage", "garbage value") + require.Error(t, err) +} diff --git a/go/libraries/utils/argparser/results.go b/go/libraries/utils/argparser/results.go index 535761ee562..7c2704b5be5 100644 --- a/go/libraries/utils/argparser/results.go +++ b/go/libraries/utils/argparser/results.go @@ -144,6 +144,33 @@ func (res *ArgParseResults) DropValue(name string) *ArgParseResults { return &ArgParseResults{newNamedArgs, res.Args, res.parser, NO_POSITIONAL_ARGS} } +// SetArgument inserts or replaces an argument. A new ArgParseResults object is returned with the new +// argument added. The parser of the original ArgParseResults is used to verify that the option is supported. +// +// If the option is not supported, this is considered a runtime error, and an error is returned to that effect. +func (res *ArgParseResults) SetArgument(name, val string) (*ArgParseResults, error) { + newNamedArgs := make(map[string]string, len(res.options)+1) + for flag, origVal := range res.options { + newNamedArgs[flag] = origVal + } + + found := false + // Verify that the options is supported - using the long name + for _, opt := range res.parser.Supported { + if opt.Name == name { + found = true + break + } + } + + if !found { + return nil, UnknownArgumentParam{name: name} + } + newNamedArgs[name] = val + + return &ArgParseResults{newNamedArgs, res.Args, res.parser, res.PositionalArgsSeparatorIndex}, nil +} + func (res *ArgParseResults) MustGetValue(name string) string { val, ok := res.options[name] diff --git a/go/performance/import_benchmarker/testdef.go b/go/performance/import_benchmarker/testdef.go index 486aff5a5b8..3bc352cfc10 100644 --- a/go/performance/import_benchmarker/testdef.go +++ b/go/performance/import_benchmarker/testdef.go @@ -27,7 +27,7 @@ import ( "testing" "time" - "github.com/cespare/xxhash" + "github.com/cespare/xxhash/v2" "github.com/creasty/defaults" sql2 "github.com/dolthub/go-mysql-server/sql" gmstypes "github.com/dolthub/go-mysql-server/sql/types" diff --git a/go/performance/microsysbench/sysbench_test.go b/go/performance/microsysbench/sysbench_test.go index 0a2795016f2..ffe312213c7 100644 --- a/go/performance/microsysbench/sysbench_test.go +++ b/go/performance/microsysbench/sysbench_test.go @@ -24,6 +24,7 @@ import ( "strings" "testing" + "github.com/dolthub/go-mysql-server/server" "github.com/dolthub/go-mysql-server/sql" "github.com/stretchr/testify/require" @@ -60,6 +61,18 @@ func BenchmarkOltpPointSelect(b *testing.B) { }) } +func BenchmarkTableScan(b *testing.B) { + benchmarkSysbenchQuery(b, func(int) string { + return "SELECT * FROM sbtest1" + }) +} + +func BenchmarkOltpIndexScan(b *testing.B) { + benchmarkSysbenchQuery(b, func(int) string { + return "SELECT * FROM sbtest1 WHERE k > 0" + }) +} + func BenchmarkOltpJoinScan(b *testing.B) { benchmarkSysbenchQuery(b, func(int) string { return `select a.id, a.k @@ -109,12 +122,21 @@ func BenchmarkSelectRandomRanges(b *testing.B) { func benchmarkSysbenchQuery(b *testing.B, getQuery func(int) string) { ctx, eng := setupBenchmark(b, dEnv) for i := 0; i < b.N; i++ { - _, iter, _, err := eng.Query(ctx, getQuery(i)) + schema, iter, _, err := eng.Query(ctx, getQuery(i)) require.NoError(b, err) + i := 0 + buf := sql.NewByteBuffer(16000) for { - if _, err = iter.Next(ctx); err != nil { + i++ + row, err := iter.Next(ctx) + if err != nil { break } + outputRow, err := server.RowToSQL(ctx, schema, row, nil, buf) + _ = outputRow + if i%128 == 0 { + buf.Reset() + } } require.Error(b, io.EOF) err = iter.Close(ctx) @@ -156,7 +178,7 @@ func populateRepo(dEnv *env.DoltEnv, insertData string) { execSql := func(dEnv *env.DoltEnv, q string) int { ctx := context.Background() args := []string{"-r", "null", "-q", q} - cliCtx, err := commands.NewArgFreeCliContext(ctx, dEnv) + cliCtx, err := commands.NewArgFreeCliContext(ctx, dEnv, dEnv.FS) if err != nil { panic(err) } diff --git a/go/serial/fileidentifiers.go b/go/serial/fileidentifiers.go index 64cb36d9f47..628cc4f99b8 100644 --- a/go/serial/fileidentifiers.go +++ b/go/serial/fileidentifiers.go @@ -41,6 +41,7 @@ const StashListFileID = "SLST" const StashFileID = "STSH" const StatisticFileID = "STAT" const DoltgresRootValueFileID = "DGRV" +const TupleFileID = "TUPL" const MessageTypesKind int = 27 diff --git a/go/serial/generate.sh b/go/serial/generate.sh index 267d926bcb5..91ae999a07f 100755 --- a/go/serial/generate.sh +++ b/go/serial/generate.sh @@ -37,6 +37,7 @@ fi stat.fbs \ table.fbs \ tag.fbs \ + tuple.fbs \ workingset.fbs # prefix files with copyright header diff --git a/go/serial/tuple.fbs b/go/serial/tuple.fbs new file mode 100644 index 00000000000..6fbcb079ad6 --- /dev/null +++ b/go/serial/tuple.fbs @@ -0,0 +1,22 @@ +// Copyright 2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +table Tuple { + value:[ubyte] (required); +} + +// KEEP THIS IN SYNC WITH fileidentifiers.go +file_identifier "TUPL"; + +root_type Tuple; diff --git a/go/store/datas/database.go b/go/store/datas/database.go index 5b3913d0a2e..fff64fbe735 100644 --- a/go/store/datas/database.go +++ b/go/store/datas/database.go @@ -97,6 +97,10 @@ type Database interface { // `opts.Meta`. Tag(ctx context.Context, ds Dataset, commitAddr hash.Hash, opts TagOptions) (Dataset, error) + // SetTuple puts an arbitrary byte array into the chunkstore. + // The dataset reference keys access to the value. + SetTuple(ctx context.Context, ds Dataset, val []byte) (Dataset, error) + // UpdateStashList updates the stash list dataset only with given address hash to the updated stash list. // The new/updated stash list address should be obtained before calling this function depending on // whether add or remove a stash actions have been performed. This function does not perform any actions diff --git a/go/store/datas/database_common.go b/go/store/datas/database_common.go index a63e97521b4..29dc88f4be4 100644 --- a/go/store/datas/database_common.go +++ b/go/store/datas/database_common.go @@ -744,6 +744,26 @@ func (db *database) doTag(ctx context.Context, datasetID string, tagAddr hash.Ha }) } +func (db *database) SetTuple(ctx context.Context, ds Dataset, val []byte) (Dataset, error) { + tupleAddr, _, err := newTuple(ctx, db, val) + if err != nil { + return Dataset{}, err + } + return db.doHeadUpdate(ctx, ds, func(ds Dataset) error { + return db.update(ctx, func(_ context.Context, datasets types.Map) (types.Map, error) { + // this is for old format, so this should not happen + return datasets, errors.New("WriteTuple is not supported for old storage format") + }, func(ctx context.Context, am prolly.AddressMap) (prolly.AddressMap, error) { + ae := am.Editor() + err := ae.Update(ctx, ds.ID(), tupleAddr) + if err != nil { + return prolly.AddressMap{}, err + } + return ae.Flush(ctx) + }) + }) +} + func (db *database) SetStatsRef(ctx context.Context, ds Dataset, mapAddr hash.Hash) (Dataset, error) { statAddr, _, err := newStat(ctx, db, mapAddr) if err != nil { diff --git a/go/store/datas/dataset.go b/go/store/datas/dataset.go index 4867c2c37ec..a41ffb94aa2 100644 --- a/go/store/datas/dataset.go +++ b/go/store/datas/dataset.go @@ -558,6 +558,42 @@ func (s statisticsHead) value() types.Value { return s.msg } +func newTupleHead(sm types.SerialMessage, addr hash.Hash) serialStashListHead { + return serialStashListHead{sm, addr} +} + +type tupleHead struct { + msg types.SerialMessage + addr hash.Hash +} + +var _ dsHead = tupleHead{} + +// TypeName implements dsHead +func (s tupleHead) TypeName() string { + return "Tuple" +} + +// Addr implements dsHead +func (s tupleHead) Addr() hash.Hash { + return s.addr +} + +// HeadTag implements dsHead +func (s tupleHead) HeadTag() (*TagMeta, hash.Hash, error) { + return nil, hash.Hash{}, errors.New("HeadTag called on tuple") +} + +// HeadWorkingSet implements dsHead +func (s tupleHead) HeadWorkingSet() (*WorkingSetHead, error) { + return nil, errors.New("HeadWorkingSet called on statistic") +} + +// value implements dsHead +func (s tupleHead) value() types.Value { + return s.msg +} + // Dataset is a named value within a Database. Different head values may be stored in a dataset. Most commonly, this is // a commit, but other values are also supported in some cases. type Dataset struct { @@ -615,6 +651,8 @@ func newHead(ctx context.Context, head types.Value, addr hash.Hash) (dsHead, err return newSerialStashListHead(sm, addr), nil case serial.StatisticFileID: return newStatisticHead(sm, addr), nil + case serial.TupleFileID: + return newTupleHead(sm, addr), nil } } diff --git a/go/store/datas/tuple.go b/go/store/datas/tuple.go new file mode 100644 index 00000000000..09732b5c549 --- /dev/null +++ b/go/store/datas/tuple.go @@ -0,0 +1,106 @@ +// Copyright 2023 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package datas + +import ( + "context" + "errors" + "fmt" + + flatbuffers "github.com/dolthub/flatbuffers/v23/go" + + "github.com/dolthub/dolt/go/gen/fb/serial" + "github.com/dolthub/dolt/go/store/hash" + "github.com/dolthub/dolt/go/store/prolly/tree" + "github.com/dolthub/dolt/go/store/types" +) + +type Tuple struct { + val []byte +} + +func (t Tuple) Bytes() []byte { + return t.val +} + +// IsTuple determines whether the types.Value is a tuple +func IsTuple(v types.Value) (bool, error) { + if _, ok := v.(types.Struct); ok { + // this should not return true as stash is not supported for old format + return false, nil + } else if sm, ok := v.(types.SerialMessage); ok { + return serial.GetFileID(sm) == serial.StatisticFileID, nil + } else { + return false, nil + } +} + +// LoadTuple attempts to dereference a database's Tuple Dataset into a typed Tuple object. +func LoadTuple(ctx context.Context, nbf *types.NomsBinFormat, ns tree.NodeStore, vr types.ValueReader, ds Dataset) (*Tuple, error) { + if !nbf.UsesFlatbuffers() { + return nil, errors.New("loadTuple: Tuple are not supported for old storage format") + } + + rootHash, hasHead := ds.MaybeHeadAddr() + if !hasHead { + return &Tuple{}, nil + } + + val, err := vr.ReadValue(ctx, rootHash) + if err != nil { + return nil, err + } + + if val == nil { + return nil, ErrNoBranchStats + } + + return parse_Tuple(ctx, []byte(val.(types.SerialMessage)), ns, vr) +} + +// newStat writes an address to a Tuple map as a Tuple message +// in the provided database. +func newTuple(ctx context.Context, db *database, addr []byte) (hash.Hash, types.Ref, error) { + data := Tuple_flatbuffer(addr) + r, err := db.WriteValue(ctx, types.SerialMessage(data)) + if err != nil { + return hash.Hash{}, types.Ref{}, err + } + return r.TargetHash(), r, nil +} + +// Tuple_flatbuffer encodes a prolly map address in a Tuple message. +func Tuple_flatbuffer(val []byte) serial.Message { + builder := flatbuffers.NewBuilder(1024) + valOff := builder.CreateByteVector(val[:]) + + serial.TupleStart(builder) + serial.TupleAddValue(builder, valOff) + return serial.FinishMessage(builder, serial.CommitEnd(builder), []byte(serial.TupleFileID)) +} + +// parse_Tuple converts a Tuple serial massage (STAT) into a Tuple object +// embedding the stats table and address. +func parse_Tuple(ctx context.Context, bs []byte, ns tree.NodeStore, vr types.ValueReader) (*Tuple, error) { + if serial.GetFileID(bs) != serial.TupleFileID { + return nil, fmt.Errorf("expected Tuple file id, got: " + serial.GetFileID(bs)) + } + tup, err := serial.TryGetRootAsTuple(bs, serial.MessagePrefixSz) + if err != nil { + return nil, err + } + + return &Tuple{val: tup.ValueBytes()}, nil +} diff --git a/go/store/nbs/archive_build.go b/go/store/nbs/archive_build.go index 5e497ca7c28..72f15d94f04 100644 --- a/go/store/nbs/archive_build.go +++ b/go/store/nbs/archive_build.go @@ -225,10 +225,12 @@ func convertTableFileToArchive( // cg.print(n, p) //} + const fourMb = 1 << 22 + // Allocate buffer used to compress chunks. - cmpBuff := make([]byte, 0, maxChunkSize) + cmpBuff := make([]byte, 0, fourMb) - cmpDefDict := gozstd.Compress(cmpBuff, defaultDict) + cmpBuff = gozstd.Compress(cmpBuff[:0], defaultDict) // p("Default Dict Raw vs Compressed: %d , %d\n", len(defaultDict), len(cmpDefDict)) arcW, err := newArchiveWriter() @@ -236,12 +238,12 @@ func convertTableFileToArchive( return "", hash.Hash{}, err } var defaultDictByteSpanId uint32 - defaultDictByteSpanId, err = arcW.writeByteSpan(cmpDefDict) + defaultDictByteSpanId, err = arcW.writeByteSpan(cmpBuff) if err != nil { return "", hash.Hash{}, err } - _, grouped, singles, err := writeDataToArchive(ctx, cmpBuff, allChunks, cgList, defaultDictByteSpanId, defaultCDict, arcW, progress, stats) + _, grouped, singles, err := writeDataToArchive(ctx, cmpBuff[:0], allChunks, cgList, defaultDictByteSpanId, defaultCDict, arcW, progress, stats) if err != nil { return "", hash.Hash{}, err } @@ -337,9 +339,9 @@ func writeDataToArchive( if cg.totalBytesSavedWDict > cg.totalBytesSavedDefaultDict { groupCount++ - cmpDict := gozstd.Compress(cmpBuff, cg.dict) + cmpBuff = gozstd.Compress(cmpBuff[:0], cg.dict) - dictId, err := arcW.writeByteSpan(cmpDict) + dictId, err := arcW.writeByteSpan(cmpBuff) if err != nil { return 0, 0, 0, err } @@ -351,9 +353,9 @@ func writeDataToArchive( } if !arcW.chunkSeen(cs.chunkId) { - compressed := gozstd.CompressDict(cmpBuff, c.Data(), cg.cDict) + cmpBuff = gozstd.CompressDict(cmpBuff[:0], c.Data(), cg.cDict) - dataId, err := arcW.writeByteSpan(compressed) + dataId, err := arcW.writeByteSpan(cmpBuff) if err != nil { return 0, 0, 0, err } @@ -380,7 +382,6 @@ func writeDataToArchive( case <-ctx.Done(): return 0, 0, 0, ctx.Err() default: - var compressed []byte dictId := uint32(0) c, e2 := chunkCache.get(ctx, h, stats) @@ -388,10 +389,10 @@ func writeDataToArchive( return 0, 0, 0, e2 } - compressed = gozstd.CompressDict(cmpBuff, c.Data(), defaultDict) + cmpBuff = gozstd.CompressDict(cmpBuff[:0], c.Data(), defaultDict) dictId = defaultSpanId - id, err := arcW.writeByteSpan(compressed) + id, err := arcW.writeByteSpan(cmpBuff) if err != nil { return 0, 0, 0, err } diff --git a/go/store/nbs/store_test.go b/go/store/nbs/store_test.go index b12a41b71a6..a1eb211c1d1 100644 --- a/go/store/nbs/store_test.go +++ b/go/store/nbs/store_test.go @@ -23,7 +23,6 @@ import ( "math/rand" "os" "path/filepath" - "sync" "testing" "time" @@ -297,8 +296,10 @@ func TestNBSCopyGC(t *testing.T) { st, _, _ := makeTestLocalStore(t, 8) defer st.Close() - keepers := makeChunkSet(64, 64) - tossers := makeChunkSet(64, 64) + const numChunks = 64 + + keepers := makeChunkSet(numChunks, 64) + tossers := makeChunkSet(numChunks, 64) for _, c := range keepers { err := st.Put(ctx, c, noopGetAddrs) @@ -333,26 +334,17 @@ func TestNBSCopyGC(t *testing.T) { require.NoError(t, err) require.True(t, ok) - keepChan := make(chan []hash.Hash, 16) - var msErr error - wg := &sync.WaitGroup{} - wg.Add(1) - go func() { - require.NoError(t, st.BeginGC(nil)) - var finalizer chunks.GCFinalizer - finalizer, msErr = st.MarkAndSweepChunks(ctx, keepChan, nil, chunks.GCMode_Full) - if msErr == nil { - msErr = finalizer.SwapChunksInStore(ctx) - } - st.EndGC() - wg.Done() - }() + keepChan := make(chan []hash.Hash, numChunks) for h := range keepers { keepChan <- []hash.Hash{h} } close(keepChan) - wg.Wait() - require.NoError(t, msErr) + + require.NoError(t, st.BeginGC(nil)) + finalizer, err := st.MarkAndSweepChunks(ctx, keepChan, nil, chunks.GCMode_Full) + require.NoError(t, err) + require.NoError(t, finalizer.SwapChunksInStore(ctx)) + st.EndGC() for h, c := range keepers { out, err := st.Get(ctx, h) diff --git a/go/store/prolly/message/address_map.go b/go/store/prolly/message/address_map.go index add5560b4d0..16d8bd49895 100644 --- a/go/store/prolly/message/address_map.go +++ b/go/store/prolly/message/address_map.go @@ -90,9 +90,9 @@ func getAddressMapKeys(msg serial.Message) (keys ItemAccess, err error) { return keys, err } keys.bufStart = lookupVectorOffset(addressMapKeyItemsBytesVOffset, am.Table()) - keys.bufLen = uint16(am.KeyItemsLength()) + keys.bufLen = uint32(am.KeyItemsLength()) keys.offStart = lookupVectorOffset(addressMapKeyItemsOffsetsVOffset, am.Table()) - keys.offLen = uint16(am.KeyOffsetsLength() * uint16Size) + keys.offLen = uint32(am.KeyOffsetsLength() * uint16Size) return } @@ -103,7 +103,7 @@ func getAddressMapValues(msg serial.Message) (values ItemAccess, err error) { return values, err } values.bufStart = lookupVectorOffset(addressMapAddressArrayVOffset, am.Table()) - values.bufLen = uint16(am.AddressArrayLength()) + values.bufLen = uint32(am.AddressArrayLength()) values.itemWidth = hash.ByteLen return } diff --git a/go/store/prolly/message/blob.go b/go/store/prolly/message/blob.go index c65299243ed..c768bc409fa 100644 --- a/go/store/prolly/message/blob.go +++ b/go/store/prolly/message/blob.go @@ -78,11 +78,11 @@ func getBlobValues(msg serial.Message) (values ItemAccess, err error) { } if b.TreeLevel() > 0 { values.bufStart = lookupVectorOffset(blobAddressArrayVOffset, b.Table()) - values.bufLen = uint16(b.AddressArrayLength()) + values.bufLen = uint32(b.AddressArrayLength()) values.itemWidth = hash.ByteLen } else { values.bufStart = lookupVectorOffset(blobPayloadBytesVOffset, b.Table()) - values.bufLen = uint16(b.PayloadLength()) + values.bufLen = uint32(b.PayloadLength()) values.itemWidth = uint16(b.PayloadLength()) } return diff --git a/go/store/prolly/message/commit_closure.go b/go/store/prolly/message/commit_closure.go index cbd6e418550..317ee109212 100644 --- a/go/store/prolly/message/commit_closure.go +++ b/go/store/prolly/message/commit_closure.go @@ -40,7 +40,7 @@ func getCommitClosureKeys(msg serial.Message) (ItemAccess, error) { return ret, err } ret.bufStart = lookupVectorOffset(commitClosureKeyItemBytesVOffset, m.Table()) - ret.bufLen = uint16(m.KeyItemsLength()) + ret.bufLen = uint32(m.KeyItemsLength()) ret.itemWidth = uint16(commitClosureKeyLength) return ret, nil } @@ -58,7 +58,7 @@ func getCommitClosureValues(msg serial.Message) (ItemAccess, error) { ret.itemWidth = 0 } else { ret.bufStart = lookupVectorOffset(commitClosureAddressArrayVOffset, m.Table()) - ret.bufLen = uint16(m.AddressArrayLength()) + ret.bufLen = uint32(m.AddressArrayLength()) ret.itemWidth = hash.ByteLen } return ret, nil diff --git a/go/store/prolly/message/item_access.go b/go/store/prolly/message/item_access.go index ea6c6676869..26e03cfc4b0 100644 --- a/go/store/prolly/message/item_access.go +++ b/go/store/prolly/message/item_access.go @@ -19,24 +19,32 @@ import ( "github.com/dolthub/dolt/go/store/val" ) +type offsetSize uint8 + +const ( + OFFSET_SIZE_16 offsetSize = iota + OFFSET_SIZE_32 +) + // ItemAccess accesses items in a serial.Message. type ItemAccess struct { // bufStart is the offset to the start of the // Item buffer within a serial.Message. // bufLen is the length of the Item buffer. - bufStart, bufLen uint16 + bufStart, bufLen uint32 // offStart, if nonzero, is the offset to the // start of the uin16 offset buffer within a // serial.Message. A zero value for offStart // indicates an empty offset buffer. // bufLen is the length of the Item buffer. - offStart, offLen uint16 + offStart, offLen uint32 // If the serial.Message does not contain an // offset buffer (offStart is zero), then // Items have a fixed width equal to itemWidth. - itemWidth uint16 + itemWidth uint16 + offsetSize offsetSize } // GetItem returns the ith Item from the buffer. @@ -44,8 +52,16 @@ func (acc ItemAccess) GetItem(i int, msg serial.Message) []byte { buf := msg[acc.bufStart : acc.bufStart+acc.bufLen] off := msg[acc.offStart : acc.offStart+acc.offLen] if acc.offStart != 0 { - stop := val.ReadUint16(off[(i*2)+2 : (i*2)+4]) - start := val.ReadUint16(off[(i * 2) : (i*2)+2]) + var stop, start uint32 + switch acc.offsetSize { + case OFFSET_SIZE_16: + stop = uint32(val.ReadUint16(off[(i*2)+2 : (i*2)+4])) + start = uint32(val.ReadUint16(off[(i * 2) : (i*2)+2])) + case OFFSET_SIZE_32: + stop = val.ReadUint32(off[(i*4)+4 : (i*4)+8]) + start = val.ReadUint32(off[(i * 4) : (i*4)+4]) + } + return buf[start:stop] } else { stop := int(acc.itemWidth) * (i + 1) diff --git a/go/store/prolly/message/merge_artifacts.go b/go/store/prolly/message/merge_artifacts.go index 55586c6eef4..781072308ef 100644 --- a/go/store/prolly/message/merge_artifacts.go +++ b/go/store/prolly/message/merge_artifacts.go @@ -113,22 +113,22 @@ func getArtifactMapKeysAndValues(msg serial.Message) (keys, values ItemAccess, l return } keys.bufStart = lookupVectorOffset(mergeArtifactKeyItemBytesVOffset, ma.Table()) - keys.bufLen = uint16(ma.KeyItemsLength()) + keys.bufLen = uint32(ma.KeyItemsLength()) keys.offStart = lookupVectorOffset(mergeArtifactKeyOffsetsVOffset, ma.Table()) - keys.offLen = uint16(ma.KeyOffsetsLength() * uint16Size) + keys.offLen = uint32(ma.KeyOffsetsLength() * uint16Size) - count = (keys.offLen / 2) - 1 + count = uint16(keys.offLen/2) - 1 level = uint16(ma.TreeLevel()) vv := ma.ValueItemsBytes() if vv != nil { values.bufStart = lookupVectorOffset(mergeArtifactValueItemBytesVOffset, ma.Table()) - values.bufLen = uint16(ma.ValueItemsLength()) + values.bufLen = uint32(ma.ValueItemsLength()) values.offStart = lookupVectorOffset(mergeArtifactValueOffsetsVOffset, ma.Table()) - values.offLen = uint16(ma.ValueOffsetsLength() * uint16Size) + values.offLen = uint32(ma.ValueOffsetsLength() * uint16Size) } else { values.bufStart = lookupVectorOffset(mergeArtifactAddressArrayVOffset, ma.Table()) - values.bufLen = uint16(ma.AddressArrayLength()) + values.bufLen = uint32(ma.AddressArrayLength()) values.itemWidth = hash.ByteLen } return diff --git a/go/store/prolly/message/message.go b/go/store/prolly/message/message.go index 5947e65ca41..8b9202cd963 100644 --- a/go/store/prolly/message/message.go +++ b/go/store/prolly/message/message.go @@ -124,11 +124,11 @@ func GetSubtrees(msg serial.Message) ([]uint64, error) { } } -func lookupVectorOffset(vo fb.VOffsetT, tab fb.Table) uint16 { +func lookupVectorOffset(vo fb.VOffsetT, tab fb.Table) uint32 { off := fb.UOffsetT(tab.Offset(vo)) + tab.Pos off += fb.GetUOffsetT(tab.Bytes[off:]) // data starts after metadata containing the vector length - return uint16(off + fb.UOffsetT(fb.SizeUOffsetT)) + return uint32(off + fb.UOffsetT(fb.SizeUOffsetT)) } func assertTrue(b bool, msg string) { diff --git a/go/store/prolly/message/prolly_map.go b/go/store/prolly/message/prolly_map.go index f5e914abe2f..a74300268e6 100644 --- a/go/store/prolly/message/prolly_map.go +++ b/go/store/prolly/message/prolly_map.go @@ -111,22 +111,22 @@ func getProllyMapKeysAndValues(msg serial.Message) (keys, values ItemAccess, lev return } keys.bufStart = lookupVectorOffset(prollyMapKeyItemBytesVOffset, pm.Table()) - keys.bufLen = uint16(pm.KeyItemsLength()) + keys.bufLen = uint32(pm.KeyItemsLength()) keys.offStart = lookupVectorOffset(prollyMapKeyOffsetsVOffset, pm.Table()) - keys.offLen = uint16(pm.KeyOffsetsLength() * uint16Size) + keys.offLen = uint32(pm.KeyOffsetsLength() * uint16Size) - count = (keys.offLen / 2) - 1 + count = uint16(keys.offLen/2) - 1 level = uint16(pm.TreeLevel()) vv := pm.ValueItemsBytes() if vv != nil { values.bufStart = lookupVectorOffset(prollyMapValueItemBytesVOffset, pm.Table()) - values.bufLen = uint16(pm.ValueItemsLength()) + values.bufLen = uint32(pm.ValueItemsLength()) values.offStart = lookupVectorOffset(prollyMapValueOffsetsVOffset, pm.Table()) - values.offLen = uint16(pm.ValueOffsetsLength() * uint16Size) + values.offLen = uint32(pm.ValueOffsetsLength() * uint16Size) } else { values.bufStart = lookupVectorOffset(prollyMapAddressArrayBytesVOffset, pm.Table()) - values.bufLen = uint16(pm.AddressArrayLength()) + values.bufLen = uint32(pm.AddressArrayLength()) values.itemWidth = hash.ByteLen } return diff --git a/go/store/prolly/message/prolly_map_test.go b/go/store/prolly/message/prolly_map_test.go index 9cb3264a685..76b8db4e9a5 100644 --- a/go/store/prolly/message/prolly_map_test.go +++ b/go/store/prolly/message/prolly_map_test.go @@ -48,7 +48,7 @@ func TestGetKeyValueOffsetsVectors(t *testing.T) { func TestItemAccessSize(t *testing.T) { sz := unsafe.Sizeof(ItemAccess{}) - assert.Equal(t, 10, int(sz)) + assert.Equal(t, 20, int(sz)) } func randomByteSlices(t *testing.T, count int) (keys, values [][]byte) { diff --git a/go/store/prolly/message/serialize.go b/go/store/prolly/message/serialize.go index f9515dcf216..cb7424fe3c8 100644 --- a/go/store/prolly/message/serialize.go +++ b/go/store/prolly/message/serialize.go @@ -62,6 +62,17 @@ func writeItemOffsets(b *fb.Builder, items [][]byte, sumSz int) fb.UOffsetT { return b.EndVector(len(items) + 1) } +func writeItemOffsets32(b *fb.Builder, items [][]byte, sumSz int) fb.UOffsetT { + var off = sumSz + for i := len(items) - 1; i >= 0; i-- { + b.PrependUint32(uint32(off)) + off -= len(items[i]) + } + assertTrue(off == 0, "incorrect final value after serializing offStart") + b.PrependUint32(uint32(off)) + return b.EndVector(len(items) + 1) +} + // countAddresses returns the number of chunk addresses stored within |items|. func countAddresses(items [][]byte, td val.TupleDesc) (cnt int) { for i := len(items) - 1; i >= 0; i-- { diff --git a/go/store/prolly/tree/node_test.go b/go/store/prolly/tree/node_test.go index 9dec1d92ab3..040cd7919ce 100644 --- a/go/store/prolly/tree/node_test.go +++ b/go/store/prolly/tree/node_test.go @@ -68,7 +68,7 @@ func TestRoundTripNodeItems(t *testing.T) { func TestNodeSize(t *testing.T) { sz := unsafe.Sizeof(Node{}) - assert.Equal(t, 56, int(sz)) + assert.Equal(t, 80, int(sz)) } func BenchmarkNodeGet(b *testing.B) { diff --git a/go/store/types/value_store.go b/go/store/types/value_store.go index ac2163f3dfa..8d9cc8266c6 100644 --- a/go/store/types/value_store.go +++ b/go/store/types/value_store.go @@ -675,6 +675,7 @@ func (lvs *ValueStore) GC(ctx context.Context, mode GCMode, oldGenRefs, newGenRe if err != nil { return err } + defer collector.EndGC() root, err := lvs.Root(ctx) if err != nil { diff --git a/go/store/val/codec.go b/go/store/val/codec.go index 2fa2df923ec..36b23366bce 100644 --- a/go/store/val/codec.go +++ b/go/store/val/codec.go @@ -306,7 +306,7 @@ func compareInt32(l, r int32) int { } } -func readUint32(val []byte) uint32 { +func ReadUint32(val []byte) uint32 { expectSize(val, uint32Size) return binary.LittleEndian.Uint32(val) } @@ -368,7 +368,7 @@ func compareUint64(l, r uint64) int { func readFloat32(val []byte) float32 { expectSize(val, float32Size) - return math.Float32frombits(readUint32(val)) + return math.Float32frombits(ReadUint32(val)) } func writeFloat32(buf []byte, val float32) { @@ -490,7 +490,7 @@ const ( func readDate(val []byte) (date time.Time) { expectSize(val, dateSize) - t := readUint32(val) + t := ReadUint32(val) y := t >> yearShift m := (t & monthMask) >> monthShift d := (t & dayMask) diff --git a/go/store/val/codec_test.go b/go/store/val/codec_test.go index c9078b269d1..52df79cde48 100644 --- a/go/store/val/codec_test.go +++ b/go/store/val/codec_test.go @@ -457,7 +457,7 @@ func roundTripUints(t *testing.T) { for _, value := range uintegers { exp := uint32(value) writeUint32(buf, exp) - assert.Equal(t, exp, readUint32(buf)) + assert.Equal(t, exp, ReadUint32(buf)) zero(buf) } diff --git a/go/store/val/tuple.go b/go/store/val/tuple.go index 90c17ddcee9..2d48991689b 100644 --- a/go/store/val/tuple.go +++ b/go/store/val/tuple.go @@ -18,7 +18,6 @@ import ( "math" "github.com/dolthub/dolt/go/store/hash" - "github.com/dolthub/dolt/go/store/pool" ) @@ -124,7 +123,6 @@ func cloneTuple(pool pool.BuffPool, tup Tuple) Tuple { func allocateTuple(pool pool.BuffPool, bufSz ByteSize, fields int) (tup Tuple, offs offsets) { offSz := offsetsSize(fields) tup = pool.Get(uint64(bufSz + offSz + countSize)) - writeFieldCount(tup, fields) offs = offsets(tup[bufSz : bufSz+offSz]) diff --git a/go/store/val/tuple_compare.go b/go/store/val/tuple_compare.go index 6417f2827ef..82f1d5bdefb 100644 --- a/go/store/val/tuple_compare.go +++ b/go/store/val/tuple_compare.go @@ -104,7 +104,7 @@ func compare(typ Type, left, right []byte) int { case Int32Enc: return compareInt32(readInt32(left), readInt32(right)) case Uint32Enc: - return compareUint32(readUint32(left), readUint32(right)) + return compareUint32(ReadUint32(left), ReadUint32(right)) case Int64Enc: return compareInt64(readInt64(left), readInt64(right)) case Uint64Enc: diff --git a/go/store/val/tuple_descriptor.go b/go/store/val/tuple_descriptor.go index 358b6fcbe67..bd55519ab35 100644 --- a/go/store/val/tuple_descriptor.go +++ b/go/store/val/tuple_descriptor.go @@ -282,7 +282,7 @@ func (td TupleDesc) GetUint32(i int, tup Tuple) (v uint32, ok bool) { td.expectEncoding(i, Uint32Enc) b := td.GetField(i, tup) if b != nil { - v, ok = readUint32(b), true + v, ok = ReadUint32(b), true } return } @@ -590,7 +590,7 @@ func (td TupleDesc) formatValue(enc Encoding, i int, value []byte) string { v := readInt32(value) return strconv.Itoa(int(v)) case Uint32Enc: - v := readUint32(value) + v := ReadUint32(value) return strconv.Itoa(int(v)) case Int64Enc: v := readInt64(value) diff --git a/integration-tests/bats/archive.bats b/integration-tests/bats/archive.bats index d00f4ab20c1..0e31e7b0aea 100755 --- a/integration-tests/bats/archive.bats +++ b/integration-tests/bats/archive.bats @@ -7,7 +7,7 @@ setup() { dolt sql -q "create table tbl (i int auto_increment primary key, guid char(36))" dolt commit -A -m "create tbl" - make_inserts + dolt sql -q "$(insert_statement)" } teardown() { @@ -15,29 +15,50 @@ teardown() { teardown_common } -# Insert 25 new rows, then commit. -make_inserts() { - for ((i=1; i<=25; i++)) +# Inserts 25 new rows and commits them. +insert_statement() { + res="INSERT INTO tbl (guid) VALUES (UUID());" + for ((i=1; i<=24; i++)) do - dolt sql -q "INSERT INTO tbl (guid) VALUES (UUID())" + res="$res INSERT INTO tbl (guid) VALUES (UUID());" done - dolt commit -a -m "Add 25 values" + res="$res call dolt_commit(\"-A\", \"--allow-empty\", \"-m\", \"Add 25 values\");" + echo "$res" } -# Randomly update 10 rows, then commit. -make_updates() { - for ((i=1; i<=10; i++)) +# Updates 10 random rows and commits the changes. +update_statement() { + res="SET @max_id = (SELECT MAX(i) FROM tbl); +SET @random_id = FLOOR(1 + RAND() * @max_id); +UPDATE tbl SET guid = UUID() WHERE i >= @random_id LIMIT 1;" + for ((i=1; i<=9; i++)) do - dolt sql -q " - SET @max_id = (SELECT MAX(i) FROM tbl); - SET @random_id = FLOOR(1 + RAND() * @max_id); - UPDATE tbl SET guid = UUID() WHERE i >= @random_id LIMIT 1;" + res="$res +SET @max_id = (SELECT MAX(i) FROM tbl); +SET @random_id = FLOOR(1 + RAND() * @max_id); +UPDATE tbl SET guid = UUID() WHERE i >= @random_id LIMIT 1;" done - dolt commit -a -m "Update 10 values." + res="$res call dolt_commit(\"-A\", \"--allow-empty\", \"-m\", \"Update 10 values\");" + echo "$res" +} + +# A series of 10 update-and-commit-then-insert-and-commit pairs, followed by a dolt_gc call +# +# This is useful because we need at least 25 retained chunks to create a commit. +mutations_and_gc_statement() { + query=`update_statement` + for ((j=1; j<=9; j++)) + do + query="$query $(insert_statement)" + query="$query $(update_statement)" + done + query="$query $(insert_statement)" + query="$query call dolt_gc();" + echo "$query" } @test "archive: too few chunks" { - make_updates + dolt sql -q "$(update_statement)" dolt gc run dolt archive @@ -51,51 +72,21 @@ make_updates() { [[ "$output" =~ "Run 'dolt gc' first" ]] || false } -# This test runs over 45 seconds, resulting in a timeout in lambdabats -# bats test_tags=no_lambda @test "archive: single archive" { - # We need at least 25 chunks to create an archive. - for ((j=1; j<=10; j++)) - do - make_updates - make_inserts - done - - dolt gc + dolt sql -q "$(mutations_and_gc_statement)" dolt archive files=$(find . -name "*darc" | wc -l | sed 's/[ \t]//g') [ "$files" -eq "1" ] # Ensure updates continue to work. - make_updates + dolt sql -q "$(update_statement)" } - -# This test runs over 45 seconds, resulting in a timeout in lambdabats -# bats test_tags=no_lambda @test "archive: multiple archives" { - # We need at least 25 chunks to create an archive. - for ((j=1; j<=10; j++)) - do - make_updates - make_inserts - done - dolt gc - - for ((j=1; j<=10; j++)) - do - make_updates - make_inserts - done - dolt gc - - for ((j=1; j<=10; j++)) - do - make_updates - make_inserts - done - dolt gc + dolt sql -q "$(mutations_and_gc_statement)" + dolt sql -q "$(mutations_and_gc_statement)" + dolt sql -q "$(mutations_and_gc_statement)" dolt archive @@ -107,42 +98,19 @@ make_updates() { [ "$commits" -eq "186" ] } -# This test runs over 45 seconds, resulting in a timeout in lambdabats -# bats test_tags=no_lambda @test "archive: archive multiple times" { - # We need at least 25 chunks to create an archive. - for ((j=1; j<=10; j++)) - do - make_updates - make_inserts - done - dolt gc + dolt sql -q "$(mutations_and_gc_statement)" dolt archive - for ((j=1; j<=10; j++)) - do - make_updates - make_inserts - done - - dolt gc + dolt sql -q "$(mutations_and_gc_statement)" dolt archive files=$(find . -name "*darc" | wc -l | sed 's/[ \t]//g') [ "$files" -eq "2" ] } -# This test runs over 45 seconds, resulting in a timeout in lambdabats -# bats test_tags=no_lambda @test "archive: archive with remotesrv no go" { - # We need at least 25 chunks to create an archive. - for ((j=1; j<=10; j++)) - do - make_updates - make_inserts - done - dolt gc - + dolt sql -q "$(mutations_and_gc_statement)" dolt archive run dolt sql-server --remotesapi-port=12321 @@ -154,16 +122,8 @@ make_updates() { [[ "$output" =~ "archive files present" ]] || false } -# This test runs over 45 seconds, resulting in a timeout in lambdabats -# bats test_tags=no_lambda @test "archive: archive --revert (fast)" { - # We need at least 25 chunks to create an archive. - for ((j=1; j<=10; j++)) - do - make_updates - make_inserts - done - dolt gc + dolt sql -q "$(mutations_and_gc_statement)" dolt archive dolt archive --revert @@ -172,16 +132,8 @@ make_updates() { [ "$commits" -eq "66" ] } -# This test runs over 45 seconds, resulting in a timeout in lambdabats -# bats test_tags=no_lambda @test "archive: archive --revert (rebuild)" { - # We need at least 25 chunks to create an archive. - for ((j=1; j<=10; j++)) - do - make_updates - make_inserts - done - dolt gc + dolt sql -q "$(mutations_and_gc_statement)" dolt archive dolt gc # This will delete the unused table files. dolt archive --revert @@ -191,17 +143,8 @@ make_updates() { [ "$commits" -eq "66" ] } -# This test runs over 45 seconds, resulting in a timeout in lambdabats -# bats test_tags=no_lambda @test "archive: archive backup no go" { - # We need at least 25 chunks to create an archive. - for ((j=1; j<=10; j++)) - do - make_updates - make_inserts - done - - dolt gc + dolt sql -q "$(mutations_and_gc_statement)" dolt archive dolt backup add bac1 file://../bac1 @@ -215,4 +158,4 @@ make_updates() { [ "$status" -eq 1 ] # NM4 - TODO. This message is cryptic, but plumbing the error through is awkward. [[ "$output" =~ "Archive chunk source" ]] || false -} \ No newline at end of file +} diff --git a/integration-tests/bats/caching_sha2_password.bats b/integration-tests/bats/caching_sha2_password.bats new file mode 100644 index 00000000000..fb852afa5aa --- /dev/null +++ b/integration-tests/bats/caching_sha2_password.bats @@ -0,0 +1,152 @@ +#!/usr/bin/env bats +load $BATS_TEST_DIRNAME/helper/common.bash +load $BATS_TEST_DIRNAME/helper/query-server-common.bash + +setup() { + skiponwindows "sql-server tests are missing dependencies on Windows" + if [ "$SQL_ENGINE" = "remote-engine" ]; then + skip "This test tests remote connections directly, SQL_ENGINE is not needed." + fi + + BATS_TEST_DIR=$PWD + PROJECT_ROOT=$BATS_TEST_DIR/../.. + CERTS_DIR=$PROJECT_ROOT/go/libraries/doltcore/servercfg/testdata + setup_no_dolt_init + dolt init + cp $CERTS_DIR/chain_cert.pem $CERTS_DIR/chain_key.pem . +} + +teardown() { + stop_sql_server 1 && sleep 0.5 + rm -rf $BATS_TMPDIR/sql-server-test$$ + teardown_common +} + +start_sql_server() { + PORT=$( definePORT ) + cat >config.yml <= @random_id LIMIT 1;" + res="$res +SET @max_id = (SELECT MAX(i) FROM tbl); +SET @random_id = FLOOR(1 + RAND() * @max_id); +UPDATE tbl SET guid = UUID() WHERE i >= @random_id LIMIT 1;" done - dolt commit -a -m "Update 10 values." + res="$res call dolt_commit(\"-A\", \"--allow-empty\", \"-m\", \"Update 10 values\");" + echo "$res" } @test "fsck: bad commit" { @@ -42,18 +49,18 @@ make_updates() { [[ "$output" =~ "hacky@hackypants.com" ]] || false } -# This test runs over 45 seconds, resulting in a timeout in lambdabats -# bats test_tags=no_lambda @test "fsck: good archive" { dolt init dolt sql -q "create table tbl (i int auto_increment primary key, guid char(36))" dolt commit -A -m "create tbl" + stmt="" for ((j=1; j<=10; j++)) do - make_inserts - make_updates + stmt="$stmt $(insert_statement)" + stmt="$stmt $(update_statement)" done + dolt sql -q "$stmt" dolt gc dolt archive @@ -66,7 +73,7 @@ make_updates() { dolt sql -q "create table tbl (i int auto_increment primary key, guid char(36))" dolt commit -Am "Create table tbl" - make_inserts + dolt sql -q "$(insert_statement)" # Objects are in the journal. Don't gc. dolt fsck diff --git a/integration-tests/bats/profile.bats b/integration-tests/bats/profile.bats index 405c6522a9d..53d7d6716f7 100755 --- a/integration-tests/bats/profile.bats +++ b/integration-tests/bats/profile.bats @@ -48,7 +48,7 @@ teardown() { run dolt --profile nonExistentProfile sql -q "select * from altDB_tbl" [ "$status" -eq 1 ] - [[ "$output" =~ "Failure to parse arguments: profile nonExistentProfile not found" ]] || false + [[ "$output" =~ "Failed to inject profile arguments: profile nonExistentProfile not found" ]] || false } @test "profile: additional flag gets used" { diff --git a/integration-tests/bats/sql-diff.bats b/integration-tests/bats/sql-diff.bats index 92b7842aa35..6ab73563844 100644 --- a/integration-tests/bats/sql-diff.bats +++ b/integration-tests/bats/sql-diff.bats @@ -591,7 +591,10 @@ SQL done } -@test "sql-diff: supports multiple primary keys" { +run_2pk5col_ints() { + local query_name=$1 + + # Initial setup dolt checkout -b firstbranch dolt sql < patch.sql newbranch - dolt checkout firstbranch - dolt sql < patch.sql - rm patch.sql - dolt add . - dolt commit -m "Reconciled with newbranch" + # Generate patch, apply on firstbranch, and verify no differences + dolt diff -r sql firstbranch > patch.sql newbranch + dolt checkout firstbranch + dolt sql < patch.sql + rm patch.sql + dolt add . + dolt commit -m "Reconciled with newbranch" - # confirm that both branches have the same content - run dolt diff -r sql firstbranch newbranch - [ "$status" -eq 0 ] - [ "$output" = "" ] - done + # Confirm branches are identical + run dolt diff -r sql firstbranch newbranch + [ "$status" -eq 0 ] + [ "$output" = "" ] +} + +@test "sql-diff: supports multiple primary keys (delete)" { + run_2pk5col_ints "delete" +} +@test "sql-diff: supports multiple primary keys (add)" { + run_2pk5col_ints "add" } +@test "sql-diff: supports multiple primary keys (update)" { + run_2pk5col_ints "update" +} +@test "sql-diff: supports multiple primary keys (single_pk_update)" { + run_2pk5col_ints "single_pk_update" +} +@test "sql-diff: supports multiple primary keys (all_pk_update)" { + run_2pk5col_ints "all_pk_update" +} +@test "sql-diff: supports multiple primary keys (create_table)" { + run_2pk5col_ints "create_table" +} + @test "sql-diff: escapes values for MySQL string literals" { # https://dev.mysql.com/doc/refman/8.0/en/string-literals.html diff --git a/integration-tests/bats/sql-server.bats b/integration-tests/bats/sql-server.bats index a563932230e..e1d27107eb4 100644 --- a/integration-tests/bats/sql-server.bats +++ b/integration-tests/bats/sql-server.bats @@ -193,12 +193,16 @@ user_session_vars: - name: user1 vars: aws_credentials_file: /Users/user1/.aws/config - aws_credentials_profile: lddev" > server.yaml + aws_credentials_profile: lddev +- name: user3 + vars: + autocommit: 0" > server.yaml dolt --privilege-file=privs.json sql -q "CREATE USER dolt@'127.0.0.1'" dolt --privilege-file=privs.json sql -q "CREATE USER user0@'127.0.0.1' IDENTIFIED BY 'pass0'" dolt --privilege-file=privs.json sql -q "CREATE USER user1@'127.0.0.1' IDENTIFIED BY 'pass1'" dolt --privilege-file=privs.json sql -q "CREATE USER user2@'127.0.0.1' IDENTIFIED BY 'pass2'" + dolt --privilege-file=privs.json sql -q "CREATE USER user3@'127.0.0.1' IDENTIFIED BY 'pass3'" start_sql_server_with_config "" server.yaml @@ -213,6 +217,9 @@ user_session_vars: run dolt --host=127.0.0.1 --port=$PORT --no-tls --user=user2 --password=pass2 sql -q "SET @@aws_credentials_file='/Users/should_fail';" [[ "$output" =~ "Variable 'aws_credentials_file' is a read only variable" ]] || false + + run dolt --host=127.0.0.1 --port=$PORT --no-tls --user=user3 --password=pass3 sql -q "SELECT @@autocommit;" + [[ "$output" =~ "0" ]] || false } @test "sql-server: read-only mode" { @@ -1355,7 +1362,7 @@ END""") [[ $output =~ "test2" ]] || false } -@test "sql-server: fetch uses database tempdir from different working directory" { +@test "sql-server: fetch uses database data dir from different working directory" { skiponwindows "Missing dependencies" mkdir remote1 @@ -1961,3 +1968,32 @@ behavior: [[ "$output" =~ "Detected that a Dolt sql-server is running from this directory." ]] || false [[ "$output" =~ "Stop the sql-server before initializing this directory as a Dolt database." ]] || false } + + +@test "sql-server: fail to start when multiple data dirs found" { + skiponwindows "Missing dependencies" + + mkdir datadir1 + mkdir datadir2 + + # This file is legit, and would work if there was no --data-dir on the cli. + cat > config.yml < ../../go/ diff --git a/integration-tests/go-sql-server-driver/go.sum b/integration-tests/go-sql-server-driver/go.sum index bf7dc00d9ff..152910313e8 100644 --- a/integration-tests/go-sql-server-driver/go.sum +++ b/integration-tests/go-sql-server-driver/go.sum @@ -14,12 +14,12 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= -golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= -golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/square/go-jose.v2 v2.5.1 h1:7odma5RETjNHWJnR32wx8t+Io4djHE1PqxCFx3iiZ2w= From 290d0d2aa9e673d3ab245e18bb4adf7065a4ed75 Mon Sep 17 00:00:00 2001 From: milogreg Date: Mon, 6 Jan 2025 14:51:33 -0800 Subject: [PATCH 15/31] Rework config file generation to only include fields explicitly set and to always include a commented-out placeholder for unset fields, even if no default value exists --- .../commands/sqlserver/command_line_config.go | 17 + go/cmd/dolt/commands/sqlserver/sqlserver.go | 42 +- .../doltcore/servercfg/serverconfig.go | 37 +- .../doltcore/servercfg/yaml_config.go | 393 ++++++++++++------ .../doltcore/servercfg/yaml_config_test.go | 30 -- 5 files changed, 332 insertions(+), 187 deletions(-) diff --git a/go/cmd/dolt/commands/sqlserver/command_line_config.go b/go/cmd/dolt/commands/sqlserver/command_line_config.go index 41d37130e1c..378732003d8 100755 --- a/go/cmd/dolt/commands/sqlserver/command_line_config.go +++ b/go/cmd/dolt/commands/sqlserver/command_line_config.go @@ -105,9 +105,11 @@ func NewCommandLineConfig(creds *cli.UserPassword, apr *argparser.ArgParseResult if creds == nil { if user, ok := apr.GetValue(cli.UserFlag); ok { config.withUser(user) + config.valuesSet[servercfg.UserKey] = struct{}{} } if password, ok := apr.GetValue(cli.PasswordFlag); ok { config.withPassword(password) + config.valuesSet[servercfg.PasswordKey] = struct{}{} } } else { config.withUser(creds.Username) @@ -163,7 +165,14 @@ func NewCommandLineConfig(creds *cli.UserPassword, apr *argparser.ArgParseResult } config.autoCommit = !apr.Contains(noAutoCommitFlag) + if apr.Contains(noAutoCommitFlag) { + config.valuesSet[servercfg.AutoCommitKey] = struct{}{} + } + config.allowCleartextPasswords = apr.Contains(allowCleartextPasswordsFlag) + if apr.Contains(allowCleartextPasswordsFlag) { + config.valuesSet[servercfg.AllowCleartextPasswordsKey] = struct{}{} + } if connStr, ok := apr.GetValue(goldenMysqlConn); ok { cli.Println(connStr) @@ -341,12 +350,14 @@ func (cfg *commandLineServerConfig) Socket() string { // WithHost updates the host and returns the called `*commandLineServerConfig`, which is useful for chaining calls. func (cfg *commandLineServerConfig) WithHost(host string) *commandLineServerConfig { cfg.host = host + cfg.valuesSet[servercfg.HostKey] = struct{}{} return cfg } // WithPort updates the port and returns the called `*commandLineServerConfig`, which is useful for chaining calls. func (cfg *commandLineServerConfig) WithPort(port int) *commandLineServerConfig { cfg.port = port + cfg.valuesSet[servercfg.PortKey] = struct{}{} return cfg } @@ -373,12 +384,14 @@ func (cfg *commandLineServerConfig) withTimeout(timeout uint64) *commandLineServ // withReadOnly updates the read only flag and returns the called `*commandLineServerConfig`, which is useful for chaining calls. func (cfg *commandLineServerConfig) withReadOnly(readonly bool) *commandLineServerConfig { cfg.readOnly = readonly + cfg.valuesSet[servercfg.ReadOnlyKey] = struct{}{} return cfg } // withLogLevel updates the log level and returns the called `*commandLineServerConfig`, which is useful for chaining calls. func (cfg *commandLineServerConfig) withLogLevel(loglevel servercfg.LogLevel) *commandLineServerConfig { cfg.logLevel = loglevel + cfg.valuesSet[servercfg.LogLevelKey] = struct{}{} return cfg } @@ -416,23 +429,27 @@ func (cfg *commandLineServerConfig) withBranchControlFilePath(branchControlFileP func (cfg *commandLineServerConfig) withAllowCleartextPasswords(allow bool) *commandLineServerConfig { cfg.allowCleartextPasswords = allow + cfg.valuesSet[servercfg.AllowCleartextPasswordsKey] = struct{}{} return cfg } // WithSocket updates the path to the unix socket file func (cfg *commandLineServerConfig) WithSocket(sockFilePath string) *commandLineServerConfig { cfg.socket = sockFilePath + cfg.valuesSet[servercfg.SocketKey] = struct{}{} return cfg } // WithRemotesapiPort sets the remotesapi port to use. func (cfg *commandLineServerConfig) WithRemotesapiPort(port *int) *commandLineServerConfig { cfg.remotesapiPort = port + cfg.valuesSet[servercfg.RemotesapiPortKey] = struct{}{} return cfg } func (cfg *commandLineServerConfig) WithRemotesapiReadOnly(readonly *bool) *commandLineServerConfig { cfg.remotesapiReadOnly = readonly + cfg.valuesSet[servercfg.RemotesapiReadOnlyKey] = struct{}{} return cfg } diff --git a/go/cmd/dolt/commands/sqlserver/sqlserver.go b/go/cmd/dolt/commands/sqlserver/sqlserver.go index b9cb9dc20ab..1f63a0c7965 100644 --- a/go/cmd/dolt/commands/sqlserver/sqlserver.go +++ b/go/cmd/dolt/commands/sqlserver/sqlserver.go @@ -482,8 +482,17 @@ func setupDoltConfig(dEnv *env.DoltEnv, cwd filesys.Filesys, apr *argparser.ArgP } serverConfig.withCfgDir(cfgDirPath) + if cfgDirSpecified { + serverConfig.valuesSet[servercfg.CfgDirKey] = struct{}{} + } + + if dataDirSpecified { + serverConfig.valuesSet[servercfg.DataDirKey] = struct{}{} + } + if privsFp, ok := apr.GetValue(commands.PrivsFilePathFlag); ok { serverConfig.withPrivilegeFilePath(privsFp) + serverConfig.valuesSet[servercfg.PrivilegeFilePathKey] = struct{}{} } else { path, err := dEnv.FS.Abs(filepath.Join(cfgDirPath, commands.DefaultPrivsName)) if err != nil { @@ -494,6 +503,7 @@ func setupDoltConfig(dEnv *env.DoltEnv, cwd filesys.Filesys, apr *argparser.ArgP if branchControlFilePath, ok := apr.GetValue(commands.BranchCtrlPathFlag); ok { serverConfig.withBranchControlFilePath(branchControlFilePath) + serverConfig.valuesSet[servercfg.BranchControlFilePathKey] = struct{}{} } else { path, err := dEnv.FS.Abs(filepath.Join(cfgDirPath, commands.DefaultBranchCtrlName)) if err != nil { @@ -517,12 +527,13 @@ func generateYamlConfigIfNone( serverConfig servercfg.ServerConfig) error { const yamlConfigName = "config.yaml" - specifiesConfigFile, err := argsSpecifyServerConfigFile(ap, help, args) - if err != nil { + apr := cli.ParseArgsOrDie(ap, args, help) + if err := validateSqlServerArgs(apr); err != nil { + cli.PrintErrln(color.RedString(err.Error())) return err } - if specifiesConfigFile { + if apr.Contains(configFileFlag) { return nil } @@ -532,24 +543,19 @@ func generateYamlConfigIfNone( return nil } - yamlConfig := servercfg.ServerConfigAsYAMLConfig(serverConfig) - err = dEnv.FS.WriteFile(path, []byte(yamlConfig.VerboseString()), os.ModePerm) - if err != nil { - return err + yamlConfig := servercfg.ServerConfigSetValuesAsYAMLConfig(serverConfig) + + if connStr, ok := apr.GetValue(goldenMysqlConn); ok { + yamlConfig.GoldenMysqlConn = &connStr } - return nil -} + generatedYaml := `# This file was generated using your configuration. +# Uncomment and edit lines as necessary to modify your configuration.` + "\n\n" + yamlConfig.VerboseString() -// argsSpecifyServerConfigFile returns true if the args specify a config file, false otherwise. -func argsSpecifyServerConfigFile(ap *argparser.ArgParser, help cli.UsagePrinter, args []string) (bool, error) { - apr := cli.ParseArgsOrDie(ap, args, help) - if err := validateSqlServerArgs(apr); err != nil { - cli.PrintErrln(color.RedString(err.Error())) - return false, err + err := dEnv.FS.WriteFile(path, []byte(generatedYaml), os.ModePerm) + if err != nil { + return err } - _, hasConfigFlag := apr.GetValue(configFileFlag) - - return hasConfigFlag, nil + return nil } diff --git a/go/libraries/doltcore/servercfg/serverconfig.go b/go/libraries/doltcore/servercfg/serverconfig.go index 0e0b9dca168..2a5c1a3e2ca 100644 --- a/go/libraries/doltcore/servercfg/serverconfig.go +++ b/go/libraries/doltcore/servercfg/serverconfig.go @@ -269,10 +269,39 @@ func ValidateConfig(config ServerConfig) error { } const ( - MaxConnectionsKey = "max_connections" - ReadTimeoutKey = "net_read_timeout" - WriteTimeoutKey = "net_write_timeout" - EventSchedulerKey = "event_scheduler" + HostKey = "host" + PortKey = "port" + UserKey = "user" + PasswordKey = "password" + ReadTimeoutKey = "net_read_timeout" + WriteTimeoutKey = "net_write_timeout" + ReadOnlyKey = "read_only" + LogLevelKey = "log_level" + AutoCommitKey = "autocommit" + DoltTransactionCommitKey = "dolt_transaction_commit" + DataDirKey = "data_dir" + CfgDirKey = "cfg_dir" + MaxConnectionsKey = "max_connections" + TLSKeyKey = "tls_key" + TLSCertKey = "tls_cert" + RequireSecureTransportKey = "require_secure_transport" + MaxLoggedQueryLenKey = "max_logged_query_len" + ShouldEncodeLoggedQueryKey = "should_encode_logged_query" + DisableClientMultiStatementsKey = "disable_client_multi_statements" + MetricsLabelsKey = "metrics_labels" + MetricsHostKey = "metrics_host" + MetricsPortKey = "metrics_port" + PrivilegeFilePathKey = "privilege_file_path" + BranchControlFilePathKey = "branch_control_file_path" + UserVarsKey = "user_vars" + SystemVarsKey = "system_vars" + JwksConfigKey = "jwks_config" + AllowCleartextPasswordsKey = "allow_cleartext_passwords" + SocketKey = "socket" + RemotesapiPortKey = "remotesapi_port" + RemotesapiReadOnlyKey = "remotesapi_read_only" + ClusterConfigKey = "cluster_config" + EventSchedulerKey = "event_scheduler" ) type SystemVariableTarget interface { diff --git a/go/libraries/doltcore/servercfg/yaml_config.go b/go/libraries/doltcore/servercfg/yaml_config.go index ea711da0de7..81cb88a95c8 100644 --- a/go/libraries/doltcore/servercfg/yaml_config.go +++ b/go/libraries/doltcore/servercfg/yaml_config.go @@ -17,7 +17,6 @@ package servercfg import ( "fmt" "path/filepath" - "reflect" "strings" "unicode" "unicode/utf8" @@ -50,8 +49,8 @@ func nillableIntPtr(n int) *int { // BehaviorYAMLConfig contains server configuration regarding how the server should behave type BehaviorYAMLConfig struct { - ReadOnly *bool `yaml:"read_only"` - AutoCommit *bool `yaml:"autocommit"` + ReadOnly *bool `yaml:"read_only,omitempty"` + AutoCommit *bool `yaml:"autocommit,omitempty"` // PersistenceBehavior is unused, but still present to prevent breaking any YAML configs that still use it. PersistenceBehavior *string `yaml:"persistence_behavior,omitempty"` // Disable processing CLIENT_MULTI_STATEMENTS support on the @@ -60,35 +59,35 @@ type BehaviorYAMLConfig struct { // does), and then sends statements that contain embedded unquoted ';'s // (such as a CREATE TRIGGER), then those incoming queries will be // misprocessed. - DisableClientMultiStatements *bool `yaml:"disable_client_multi_statements"` + DisableClientMultiStatements *bool `yaml:"disable_client_multi_statements,omitempty"` // DoltTransactionCommit enables the @@dolt_transaction_commit system variable, which // automatically creates a Dolt commit when any SQL transaction is committed. - DoltTransactionCommit *bool `yaml:"dolt_transaction_commit"` + DoltTransactionCommit *bool `yaml:"dolt_transaction_commit,omitempty"` EventSchedulerStatus *string `yaml:"event_scheduler,omitempty" minver:"1.17.0"` } // UserYAMLConfig contains server configuration regarding the user account clients must use to connect type UserYAMLConfig struct { - Name *string `yaml:"name"` - Password *string `yaml:"password"` + Name *string `yaml:"name,omitempty"` + Password *string `yaml:"password,omitempty"` } // ListenerYAMLConfig contains information on the network connection that the server will open type ListenerYAMLConfig struct { - HostStr *string `yaml:"host"` - PortNumber *int `yaml:"port"` - MaxConnections *uint64 `yaml:"max_connections"` - ReadTimeoutMillis *uint64 `yaml:"read_timeout_millis"` - WriteTimeoutMillis *uint64 `yaml:"write_timeout_millis"` + HostStr *string `yaml:"host,omitempty"` + PortNumber *int `yaml:"port,omitempty"` + MaxConnections *uint64 `yaml:"max_connections,omitempty"` + ReadTimeoutMillis *uint64 `yaml:"read_timeout_millis,omitempty"` + WriteTimeoutMillis *uint64 `yaml:"write_timeout_millis,omitempty"` // TLSKey is a file system path to an unencrypted private TLS key in PEM format. - TLSKey *string `yaml:"tls_key"` + TLSKey *string `yaml:"tls_key,omitempty"` // TLSCert is a file system path to a TLS certificate chain in PEM format. - TLSCert *string `yaml:"tls_cert"` + TLSCert *string `yaml:"tls_cert,omitempty"` // RequireSecureTransport can enable a mode where non-TLS connections are turned away. - RequireSecureTransport *bool `yaml:"require_secure_transport"` + RequireSecureTransport *bool `yaml:"require_secure_transport,omitempty"` // AllowCleartextPasswords enables use of cleartext passwords. - AllowCleartextPasswords *bool `yaml:"allow_cleartext_passwords"` + AllowCleartextPasswords *bool `yaml:"allow_cleartext_passwords,omitempty"` // Socket is unix socket file path Socket *string `yaml:"socket,omitempty"` } @@ -100,9 +99,9 @@ type PerformanceYAMLConfig struct { } type MetricsYAMLConfig struct { - Labels map[string]string `yaml:"labels"` - Host *string `yaml:"host"` - Port *int `yaml:"port"` + Labels map[string]string `yaml:"labels,omitempty"` + Host *string `yaml:"host,omitempty"` + Port *int `yaml:"port,omitempty"` } type RemotesapiYAMLConfig struct { @@ -128,21 +127,21 @@ type YAMLConfig struct { LogLevelStr *string `yaml:"log_level,omitempty"` MaxQueryLenInLogs *int `yaml:"max_logged_query_len,omitempty"` EncodeLoggedQuery *bool `yaml:"encode_logged_query,omitempty"` - BehaviorConfig BehaviorYAMLConfig `yaml:"behavior"` - UserConfig UserYAMLConfig `yaml:"user"` - ListenerConfig ListenerYAMLConfig `yaml:"listener"` + BehaviorConfig BehaviorYAMLConfig `yaml:"behavior,omitempty"` + UserConfig UserYAMLConfig `yaml:"user,omitempty"` + ListenerConfig ListenerYAMLConfig `yaml:"listener,omitempty"` PerformanceConfig *PerformanceYAMLConfig `yaml:"performance,omitempty"` DataDirStr *string `yaml:"data_dir,omitempty"` CfgDirStr *string `yaml:"cfg_dir,omitempty"` - MetricsConfig MetricsYAMLConfig `yaml:"metrics"` - RemotesapiConfig RemotesapiYAMLConfig `yaml:"remotesapi"` + MetricsConfig MetricsYAMLConfig `yaml:"metrics,omitempty"` + RemotesapiConfig RemotesapiYAMLConfig `yaml:"remotesapi,omitempty"` ClusterCfg *ClusterYAMLConfig `yaml:"cluster,omitempty"` PrivilegeFile *string `yaml:"privilege_file,omitempty"` BranchControlFile *string `yaml:"branch_control_file,omitempty"` // TODO: Rename to UserVars_ - Vars []UserSessionVars `yaml:"user_session_vars"` + Vars []UserSessionVars `yaml:"user_session_vars,omitempty"` SystemVars_ map[string]interface{} `yaml:"system_variables,omitempty" minver:"1.11.1"` - Jwks []JwksConfig `yaml:"jwks"` + Jwks []JwksConfig `yaml:"jwks,omitempty"` GoldenMysqlConn *string `yaml:"golden_mysql_conn,omitempty"` } @@ -245,21 +244,228 @@ func clusterConfigAsYAMLConfig(config ClusterConfig) *ClusterYAMLConfig { } } +// ServerConfigSetValuesAsYAMLConfig returns a YAMLConfig containing only values +// that were explicitly set in the given ServerConfig. +func ServerConfigSetValuesAsYAMLConfig(cfg ServerConfig) *YAMLConfig { + systemVars := cfg.SystemVars() + + return &YAMLConfig{ + LogLevelStr: zeroIf(ptr(string(cfg.LogLevel())), !cfg.ValueSet(LogLevelKey)), + MaxQueryLenInLogs: zeroIf(ptr(cfg.MaxLoggedQueryLen()), !cfg.ValueSet(MaxLoggedQueryLenKey)), + EncodeLoggedQuery: zeroIf(ptr(cfg.ShouldEncodeLoggedQuery()), !cfg.ValueSet(ShouldEncodeLoggedQueryKey)), + BehaviorConfig: BehaviorYAMLConfig{ + ReadOnly: zeroIf(ptr(cfg.ReadOnly()), !cfg.ValueSet(ReadOnlyKey)), + AutoCommit: zeroIf(ptr(cfg.AutoCommit()), !cfg.ValueSet(AutoCommitKey)), + DisableClientMultiStatements: zeroIf(ptr(cfg.DisableClientMultiStatements()), !cfg.ValueSet(DisableClientMultiStatementsKey)), + DoltTransactionCommit: zeroIf(ptr(cfg.DoltTransactionCommit()), !cfg.ValueSet(DoltTransactionCommitKey)), + EventSchedulerStatus: zeroIf(ptr(cfg.EventSchedulerStatus()), !cfg.ValueSet(EventSchedulerKey)), + }, + UserConfig: UserYAMLConfig{ + Name: zeroIf(ptr(cfg.User()), !cfg.ValueSet(UserKey)), + Password: zeroIf(ptr(cfg.Password()), !cfg.ValueSet(PasswordKey)), + }, + ListenerConfig: ListenerYAMLConfig{ + HostStr: zeroIf(ptr(cfg.Host()), !cfg.ValueSet(HostKey)), + PortNumber: zeroIf(ptr(cfg.Port()), !cfg.ValueSet(PortKey)), + MaxConnections: zeroIf(ptr(cfg.MaxConnections()), !cfg.ValueSet(MaxConnectionsKey)), + ReadTimeoutMillis: zeroIf(ptr(cfg.ReadTimeout()), !cfg.ValueSet(ReadTimeoutKey)), + WriteTimeoutMillis: zeroIf(ptr(cfg.WriteTimeout()), !cfg.ValueSet(WriteTimeoutKey)), + TLSKey: zeroIf(ptr(cfg.TLSKey()), !cfg.ValueSet(TLSKeyKey)), + TLSCert: zeroIf(ptr(cfg.TLSCert()), !cfg.ValueSet(TLSCertKey)), + RequireSecureTransport: zeroIf(ptr(cfg.RequireSecureTransport()), !cfg.ValueSet(RequireSecureTransportKey)), + AllowCleartextPasswords: zeroIf(ptr(cfg.AllowCleartextPasswords()), !cfg.ValueSet(AllowCleartextPasswordsKey)), + Socket: zeroIf(ptr(cfg.Socket()), !cfg.ValueSet(SocketKey)), + }, + DataDirStr: zeroIf(ptr(cfg.DataDir()), !cfg.ValueSet(DataDirKey)), + CfgDirStr: zeroIf(ptr(cfg.CfgDir()), !cfg.ValueSet(CfgDirKey)), + MetricsConfig: MetricsYAMLConfig{ + Labels: zeroIf(cfg.MetricsLabels(), !cfg.ValueSet(MetricsLabelsKey)), + Host: zeroIf(ptr(cfg.MetricsHost()), !cfg.ValueSet(MetricsHostKey)), + Port: zeroIf(ptr(cfg.MetricsPort()), !cfg.ValueSet(MetricsPortKey)), + }, + RemotesapiConfig: RemotesapiYAMLConfig{ + Port_: zeroIf(cfg.RemotesapiPort(), !cfg.ValueSet(RemotesapiPortKey)), + ReadOnly_: zeroIf(cfg.RemotesapiReadOnly(), !cfg.ValueSet(RemotesapiReadOnlyKey)), + }, + ClusterCfg: zeroIf(clusterConfigAsYAMLConfig(cfg.ClusterConfig()), !cfg.ValueSet(ClusterConfigKey)), + PrivilegeFile: zeroIf(ptr(cfg.PrivilegeFilePath()), !cfg.ValueSet(PrivilegeFilePathKey)), + BranchControlFile: zeroIf(ptr(cfg.BranchControlFilePath()), !cfg.ValueSet(BranchControlFilePathKey)), + SystemVars_: zeroIf(systemVars, !cfg.ValueSet(SystemVarsKey)), + Vars: zeroIf(cfg.UserVars(), !cfg.ValueSet(UserVarsKey)), + Jwks: zeroIf(cfg.JwksConfig(), !cfg.ValueSet(JwksConfigKey)), + } +} + +func zeroIf[T any](val T, condition bool) T { + if condition { + var zero T + return zero + } + return val +} + // String returns the YAML representation of the config func (cfg YAMLConfig) String() string { - return formattedYAMLMarshal(cfg) + data, err := yaml.Marshal(cfg) + + if err != nil { + return "Failed to marshal as yaml: " + err.Error() + } + + unformatted := string(data) + + // format the yaml to be easier to read. + lines := strings.Split(unformatted, "\n") + + var formatted []string + formatted = append(formatted, lines[0]) + for i := 1; i < len(lines); i++ { + if len(lines[i]) == 0 { + continue + } + + r, _ := utf8.DecodeRuneInString(lines[i]) + if !unicode.IsSpace(r) { + formatted = append(formatted, "") + } + + formatted = append(formatted, lines[i]) + } + + result := strings.Join(formatted, "\n") + return result } -// VerboseString behaves like String, but includes empty fields instead of omitting them. -// If an empty field has a default value, the default will be used. -// If an empty field has no default value, a commented-out placeholder will be used. +// VerboseString behaves like String, but includes commented-out placeholders for empty fields instead of omitting them. func (cfg YAMLConfig) VerboseString() string { - withDefaults := cfg.withDefaultsFilledIn() + withPlaceholders := cfg.withPlaceholdersFilledIn() + + return commentYAMLDiffs(cfg.String(), withPlaceholders.String()) +} + +// withPlaceholdersFilledIn returns the config with placeholder values in place of nil values. +// +// The placeholder value for a field will be its default value if one exists, or an arbitrary +// example value if no default exists. Deprecated/unused fields won't be given an example value. +// +// The config generated by this function should only be used to produce example values for +// commented-out YAML fields, and shouldn't be used to actually configure anything. +func (cfg YAMLConfig) withPlaceholdersFilledIn() YAMLConfig { + withPlaceholders := cfg.withDefaultsFilledIn() + + if withPlaceholders.BehaviorConfig.DisableClientMultiStatements == nil { + withPlaceholders.BehaviorConfig.DisableClientMultiStatements = ptr(false) + } + if withPlaceholders.BehaviorConfig.EventSchedulerStatus == nil { + withPlaceholders.BehaviorConfig.EventSchedulerStatus = ptr("OFF") + } + + if withPlaceholders.ListenerConfig.TLSKey == nil { + withPlaceholders.ListenerConfig.TLSKey = ptr("key.pem") + } + if withPlaceholders.ListenerConfig.TLSCert == nil { + withPlaceholders.ListenerConfig.TLSCert = ptr("cert.pem") + } + if withPlaceholders.ListenerConfig.RequireSecureTransport == nil { + withPlaceholders.ListenerConfig.RequireSecureTransport = ptr(false) + } + if withPlaceholders.ListenerConfig.Socket == nil { + withPlaceholders.ListenerConfig.Socket = ptr(DefaultUnixSocketFilePath) + } + + if withPlaceholders.MetricsConfig.Labels == nil { + withPlaceholders.MetricsConfig.Labels = map[string]string{ + "label1": "value1", + "label2": "2", + "label3": "true", + } + } + if withPlaceholders.MetricsConfig.Host == nil { + withPlaceholders.MetricsConfig.Host = ptr("123.45.67.89") + } + if withPlaceholders.MetricsConfig.Port == nil { + withPlaceholders.MetricsConfig.Port = ptr(9091) + } + + if withPlaceholders.RemotesapiConfig.Port_ == nil { + withPlaceholders.RemotesapiConfig.Port_ = ptr(8000) + } + if withPlaceholders.RemotesapiConfig.ReadOnly_ == nil { + withPlaceholders.RemotesapiConfig.ReadOnly_ = ptr(false) + } + + if withPlaceholders.ClusterCfg == nil { + withPlaceholders.ClusterCfg = &ClusterYAMLConfig{ + StandbyRemotes_: []StandbyRemoteYAMLConfig{ + StandbyRemoteYAMLConfig{ + Name_: "standby_replica_one", + RemoteURLTemplate_: "https://standby_replica_one.svc.cluster.local:50051/{database}", + }, + StandbyRemoteYAMLConfig{ + Name_: "standby_replica_two", + RemoteURLTemplate_: "https://standby_replica_two.svc.cluster.local:50051/{database}", + }, + }, + BootstrapRole_: "primary", + BootstrapEpoch_: 1, + RemotesAPI: ClusterRemotesAPIYAMLConfig{ + Addr_: "127.0.0.1", + Port_: 50051, + TLSKey_: "remotesapi_key.pem", + TLSCert_: "remotesapi_chain.pem", + TLSCA_: "standby_cas.pem", + URLMatches: []string{ + "https://standby_replica_one.svc.cluster.local", + "https://standby_replica_two.svc.cluster.local", + }, + DNSMatches: []string{ + "standby_replica_one.svc.cluster.local", + "standby_replica_two.svc.cluster.local", + }, + }, + } + } + + if withPlaceholders.Vars == nil { + withPlaceholders.Vars = []UserSessionVars{ + UserSessionVars{ + Name: "root", + Vars: map[string]interface{}{ + "dolt_show_system_tables": 1, + "dolt_log_level": "warn", + }, + }, + } + } + + if withPlaceholders.SystemVars_ == nil { + withPlaceholders.SystemVars_ = map[string]interface{}{ + "dolt_transaction_commit": 1, + "dolt_log_level": "info", + } + } + + if withPlaceholders.Jwks == nil { + withPlaceholders.Jwks = []JwksConfig{ + JwksConfig{ + Name: "name1", + LocationUrl: "https://example.com", + Claims: map[string]string{ + "field1": "a", + "field2": "b", + }, + FieldsToLog: []string{ + "field1", "field2", + }, + }, + } + } - formatted := formattedYAMLMarshal(removeOmitemptyTags(withDefaults)) - formatted = commentNullYAMLValues(formatted) + if withPlaceholders.GoldenMysqlConn == nil { + withPlaceholders.GoldenMysqlConn = ptr("username:password@tcp(127.0.0.1:3306)/db") + } - return formatted + return withPlaceholders } // withDefaultsFilledIn returns the config with default values in place of nil values. @@ -329,111 +535,28 @@ func (cfg YAMLConfig) withDefaultsFilledIn() YAMLConfig { return withDefaults } -// Assumes 'in' has no circular references. -func removeOmitemptyTags(in any) any { - val := reflect.ValueOf(in) - typ := reflect.TypeOf(in) - - newType := removeOmitemptyTagsType(typ) - newVal := deepConvert(val, newType) - - return newVal.Interface() -} - -func removeOmitemptyTagsType(typ reflect.Type) reflect.Type { - switch typ.Kind() { - case reflect.Pointer: - return reflect.PointerTo(removeOmitemptyTagsType(typ.Elem())) - case reflect.Struct: - fields := []reflect.StructField{} - for i := 0; i < typ.NumField(); i++ { - field := typ.Field(i) - if field.IsExported() { - field.Tag = reflect.StructTag(strings.Replace(string(field.Tag), ",omitempty", "", -1)) - field.Type = removeOmitemptyTagsType(field.Type) - fields = append(fields, field) - } - } - - return reflect.StructOf(fields) - default: - return typ - } -} - -func deepConvert(val reflect.Value, typ reflect.Type) reflect.Value { - switch val.Kind() { - case reflect.Pointer: - if val.IsNil() { - return reflect.Zero(typ) - } - elemType := typ.Elem() - convertedPtr := reflect.New(elemType) - convertedPtr.Elem().Set(deepConvert(val.Elem(), elemType)) - - return convertedPtr - case reflect.Struct: - convertedStruct := reflect.New(typ).Elem() - for i := 0; i < convertedStruct.NumField(); i++ { - fieldName := typ.Field(i).Name - field := convertedStruct.Field(i) - field.Set(deepConvert(val.FieldByName(fieldName), field.Type())) - } - - return convertedStruct - default: - return val.Convert(typ) - } -} - -// formattedYAMLMarshal returns the same result as yaml.Marshal, -// but with top-level fields separated by an additional newline. -func formattedYAMLMarshal(toMarshal any) string { - data, err := yaml.Marshal(toMarshal) - - if err != nil { - return "Failed to marshal as yaml: " + err.Error() - } - - unformatted := string(data) - - // format the yaml to be easier to read. - lines := strings.Split(unformatted, "\n") - - var formatted []string - formatted = append(formatted, lines[0]) - for i := 1; i < len(lines); i++ { - if len(lines[i]) == 0 { - continue - } - - r, _ := utf8.DecodeRuneInString(lines[i]) - if !unicode.IsSpace(r) { - formatted = append(formatted, "") - } - - formatted = append(formatted, lines[i]) - } - - result := strings.Join(formatted, "\n") - return result -} - -// commentNullYAMLValues returns the given YAML-formatted string with null fields commented out. +// commentYAMLDiffs takes YAML-formatted strings |a| and |b| and returns a YAML-formatted string +// containing all of the lines in |a|, along with comments containing all of the lines in |b| that are not in |a|. // -// Assumes no non-null fields will be set to unquoted strings ending in null. -// For example, `field: 123-null` will be falsely commented but `field: '123-null'` is fine. -func commentNullYAMLValues(needsComments string) string { - lines := strings.Split(needsComments, "\n") - for i := 0; i < len(lines); i++ { - if strings.HasSuffix(lines[i], "null") { - withoutSpace := strings.TrimSpace(lines[i]) - space := lines[i][:len(lines[i])-len(withoutSpace)] - lines[i] = space + "# " + withoutSpace +// Assumes all lines in |a| appear in |b|, with the same relative ordering. +func commentYAMLDiffs(a, b string) string { + linesA := strings.Split(a, "\n") + linesB := strings.Split(b, "\n") + + aIdx := 0 + for bIdx := range linesB { + if aIdx >= len(linesA) || linesA[aIdx] != linesB[bIdx] { + withoutSpace := strings.TrimSpace(linesB[bIdx]) + if len(withoutSpace) > 0 { + space := linesB[bIdx][:len(linesB[bIdx])-len(withoutSpace)] + linesB[bIdx] = space + "# " + withoutSpace + } + } else { + aIdx++ } } - return strings.Join(lines, "\n") + return strings.Join(linesB, "\n") } // Host returns the domain that the server will run on. Accepts an IPv4 or IPv6 address, in addition to localhost. diff --git a/go/libraries/doltcore/servercfg/yaml_config_test.go b/go/libraries/doltcore/servercfg/yaml_config_test.go index 53166a58efc..67a46e1da07 100644 --- a/go/libraries/doltcore/servercfg/yaml_config_test.go +++ b/go/libraries/doltcore/servercfg/yaml_config_test.go @@ -427,33 +427,3 @@ metrics: assert.Equal(t, "localhost", cfg.MetricsHost()) assert.Equal(t, -1, cfg.MetricsPort()) } - -func TestCommentNullYAMLValues(t *testing.T) { - toComment := ` -value1: value -value2: null -null: value -nest1: - value1: null - value2: value - nest2: - value1: "null" - nest3: - with_many_spaces: null -` - - withComments := ` -value1: value -# value2: null -null: value -nest1: - # value1: null - value2: value - nest2: - value1: "null" - nest3: - # with_many_spaces: null -` - - assert.Equal(t, withComments, commentNullYAMLValues(toComment)) -} From 96e22009db00a0076a71d44875f661e5079d931f Mon Sep 17 00:00:00 2001 From: milogreg Date: Mon, 6 Jan 2025 15:00:53 -0800 Subject: [PATCH 16/31] Make YAML formatting not treat list items in a top-level field as top-level fields themselves (fixing awkward newlines in top-level fields containing lists --- go/libraries/doltcore/servercfg/yaml_config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/libraries/doltcore/servercfg/yaml_config.go b/go/libraries/doltcore/servercfg/yaml_config.go index 81cb88a95c8..fc10514ce1e 100644 --- a/go/libraries/doltcore/servercfg/yaml_config.go +++ b/go/libraries/doltcore/servercfg/yaml_config.go @@ -325,7 +325,7 @@ func (cfg YAMLConfig) String() string { } r, _ := utf8.DecodeRuneInString(lines[i]) - if !unicode.IsSpace(r) { + if !unicode.IsSpace(r) && r != '-' { formatted = append(formatted, "") } From 6eb99593e902781a6fe601fffa49574e3c448e3c Mon Sep 17 00:00:00 2001 From: milogreg Date: Mon, 6 Jan 2025 15:14:52 -0800 Subject: [PATCH 17/31] Update minver_validation.txt --- .../servercfg/testdata/minver_validation.txt | 50 +++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/go/libraries/doltcore/servercfg/testdata/minver_validation.txt b/go/libraries/doltcore/servercfg/testdata/minver_validation.txt index fc9d38d5f1f..b0ed893c19e 100644 --- a/go/libraries/doltcore/servercfg/testdata/minver_validation.txt +++ b/go/libraries/doltcore/servercfg/testdata/minver_validation.txt @@ -4,36 +4,36 @@ LogLevelStr *string 0.0.0 log_level,omitempty MaxQueryLenInLogs *int 0.0.0 max_logged_query_len,omitempty EncodeLoggedQuery *bool 0.0.0 encode_logged_query,omitempty -BehaviorConfig servercfg.BehaviorYAMLConfig 0.0.0 behavior --ReadOnly *bool 0.0.0 read_only --AutoCommit *bool 0.0.0 autocommit +BehaviorConfig servercfg.BehaviorYAMLConfig 0.0.0 behavior,omitempty +-ReadOnly *bool 0.0.0 read_only,omitempty +-AutoCommit *bool 0.0.0 autocommit,omitempty -PersistenceBehavior *string 0.0.0 persistence_behavior,omitempty --DisableClientMultiStatements *bool 0.0.0 disable_client_multi_statements --DoltTransactionCommit *bool 0.0.0 dolt_transaction_commit +-DisableClientMultiStatements *bool 0.0.0 disable_client_multi_statements,omitempty +-DoltTransactionCommit *bool 0.0.0 dolt_transaction_commit,omitempty -EventSchedulerStatus *string 1.17.0 event_scheduler,omitempty -UserConfig servercfg.UserYAMLConfig 0.0.0 user --Name *string 0.0.0 name --Password *string 0.0.0 password -ListenerConfig servercfg.ListenerYAMLConfig 0.0.0 listener --HostStr *string 0.0.0 host --PortNumber *int 0.0.0 port --MaxConnections *uint64 0.0.0 max_connections --ReadTimeoutMillis *uint64 0.0.0 read_timeout_millis --WriteTimeoutMillis *uint64 0.0.0 write_timeout_millis --TLSKey *string 0.0.0 tls_key --TLSCert *string 0.0.0 tls_cert --RequireSecureTransport *bool 0.0.0 require_secure_transport --AllowCleartextPasswords *bool 0.0.0 allow_cleartext_passwords +UserConfig servercfg.UserYAMLConfig 0.0.0 user,omitempty +-Name *string 0.0.0 name,omitempty +-Password *string 0.0.0 password,omitempty +ListenerConfig servercfg.ListenerYAMLConfig 0.0.0 listener,omitempty +-HostStr *string 0.0.0 host,omitempty +-PortNumber *int 0.0.0 port,omitempty +-MaxConnections *uint64 0.0.0 max_connections,omitempty +-ReadTimeoutMillis *uint64 0.0.0 read_timeout_millis,omitempty +-WriteTimeoutMillis *uint64 0.0.0 write_timeout_millis,omitempty +-TLSKey *string 0.0.0 tls_key,omitempty +-TLSCert *string 0.0.0 tls_cert,omitempty +-RequireSecureTransport *bool 0.0.0 require_secure_transport,omitempty +-AllowCleartextPasswords *bool 0.0.0 allow_cleartext_passwords,omitempty -Socket *string 0.0.0 socket,omitempty PerformanceConfig *servercfg.PerformanceYAMLConfig 0.0.0 performance,omitempty -QueryParallelism *int 0.0.0 query_parallelism,omitempty DataDirStr *string 0.0.0 data_dir,omitempty CfgDirStr *string 0.0.0 cfg_dir,omitempty -MetricsConfig servercfg.MetricsYAMLConfig 0.0.0 metrics --Labels map[string]string 0.0.0 labels --Host *string 0.0.0 host --Port *int 0.0.0 port -RemotesapiConfig servercfg.RemotesapiYAMLConfig 0.0.0 remotesapi +MetricsConfig servercfg.MetricsYAMLConfig 0.0.0 metrics,omitempty +-Labels map[string]string 0.0.0 labels,omitempty +-Host *string 0.0.0 host,omitempty +-Port *int 0.0.0 port,omitempty +RemotesapiConfig servercfg.RemotesapiYAMLConfig 0.0.0 remotesapi,omitempty -Port_ *int 0.0.0 port,omitempty -ReadOnly_ *bool 1.30.5 read_only,omitempty ClusterCfg *servercfg.ClusterYAMLConfig 0.0.0 cluster,omitempty @@ -52,11 +52,11 @@ ClusterCfg *servercfg.ClusterYAMLConfig 0.0.0 cluster,omitempty --DNSMatches []string 0.0.0 server_name_dns PrivilegeFile *string 0.0.0 privilege_file,omitempty BranchControlFile *string 0.0.0 branch_control_file,omitempty -Vars []servercfg.UserSessionVars 0.0.0 user_session_vars +Vars []servercfg.UserSessionVars 0.0.0 user_session_vars,omitempty -Name string 0.0.0 name -Vars map[string]interface{} 0.0.0 vars SystemVars_ map[string]interface{} 1.11.1 system_variables,omitempty -Jwks []servercfg.JwksConfig 0.0.0 jwks +Jwks []servercfg.JwksConfig 0.0.0 jwks,omitempty -Name string 0.0.0 name -LocationUrl string 0.0.0 location_url -Claims map[string]string 0.0.0 claims From 86446e6c18c954f0964e14041b6048c7be16b2b6 Mon Sep 17 00:00:00 2001 From: milogreg Date: Mon, 6 Jan 2025 15:22:01 -0800 Subject: [PATCH 18/31] Update config file gen bats test --- .../bats/sql-server-config-file-generation.bats | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/integration-tests/bats/sql-server-config-file-generation.bats b/integration-tests/bats/sql-server-config-file-generation.bats index 06c7fe90272..151c92787ef 100644 --- a/integration-tests/bats/sql-server-config-file-generation.bats +++ b/integration-tests/bats/sql-server-config-file-generation.bats @@ -111,7 +111,7 @@ EOF [[ "$output" =~ "allow_cleartext_passwords: true" ]] || false } -@test "sql-server-config-file-generation: generated config file has unset fields set to default values" { +@test "sql-server-config-file-generation: generated config file uses default values as placeholders for unset fields" { start_sql_server_with_args \ --max-connections 77 \ --timeout 7777777 \ @@ -127,9 +127,9 @@ EOF [[ "$output" =~ "write_timeout_millis: 7777777" ]] || false # default (not set by args) - [[ "$output" =~ "read_only: false" ]] || false - [[ "$output" =~ "autocommit: true" ]] || false - [[ "$output" =~ "allow_cleartext_passwords: false" ]] || false + [[ "$output" =~ "# read_only: false" ]] || false + [[ "$output" =~ "# autocommit: true" ]] || false + [[ "$output" =~ "# allow_cleartext_passwords: false" ]] || false } @test "sql-server-config-file-generation: generated config file has placeholders for unset fields with no default values" { From 7a3b62e05bb6a126a7b78626a45d72976aabfa59 Mon Sep 17 00:00:00 2001 From: milogreg Date: Mon, 6 Jan 2025 15:56:38 -0800 Subject: [PATCH 19/31] Add test to ensure YAMLConfig's String and VerboseString functions produce equivalent YAML --- .../doltcore/servercfg/yaml_config_test.go | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/go/libraries/doltcore/servercfg/yaml_config_test.go b/go/libraries/doltcore/servercfg/yaml_config_test.go index 67a46e1da07..eb28ebc18c6 100644 --- a/go/libraries/doltcore/servercfg/yaml_config_test.go +++ b/go/libraries/doltcore/servercfg/yaml_config_test.go @@ -427,3 +427,59 @@ metrics: assert.Equal(t, "localhost", cfg.MetricsHost()) assert.Equal(t, -1, cfg.MetricsPort()) } + +// Tests that YAMLConfig.String() and YAMLConfig.VerboseString() produce equivalent YAML. +func TestYAMLConfigVerboseStringEquivalent(t *testing.T) { + yamlEquivalent := func(a, b string) bool { + var unmarshaled1 any + err := yaml.Unmarshal([]byte(a), &unmarshaled1) + require.NoError(t, err) + + var unmarshaled2 any + err = yaml.Unmarshal([]byte(a), &unmarshaled2) + require.NoError(t, err) + + remarshaled1, err := yaml.Marshal(unmarshaled1) + require.NoError(t, err) + + remarshaled2, err := yaml.Marshal(unmarshaled1) + require.NoError(t, err) + + return string(remarshaled1) == string(remarshaled2) + } + + configs := []YAMLConfig{ + YAMLConfig{ + LogLevelStr: ptr("warn"), + MaxQueryLenInLogs: ptr(1234), + ListenerConfig: ListenerYAMLConfig{ + HostStr: ptr("XXYYZZ"), + PortNumber: ptr(33333), + }, + DataDirStr: ptr("abcdef"), + GoldenMysqlConn: ptr("abc123"), + }, + YAMLConfig{ + MetricsConfig: MetricsYAMLConfig{ + Labels: map[string]string{ + "xyz": "123", + "0": "AAABBB", + }, + Host: ptr("!!!!!!!!"), + }, + }, + YAMLConfig{ + MetricsConfig: MetricsYAMLConfig{ + Port: ptr(0), + }, + RemotesapiConfig: RemotesapiYAMLConfig{ + Port_: ptr(111), + ReadOnly_: ptr(false), + }, + }, + } + + for _, config := range configs { + assert.True(t, yamlEquivalent(config.String(), config.VerboseString())) + } +} From a3e3b116b536a8c87296427c451059c5ab65c75e Mon Sep 17 00:00:00 2001 From: milogreg Date: Mon, 6 Jan 2025 16:11:03 -0800 Subject: [PATCH 20/31] Add test for commentYAMLDiffs --- .../doltcore/servercfg/yaml_config_test.go | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/go/libraries/doltcore/servercfg/yaml_config_test.go b/go/libraries/doltcore/servercfg/yaml_config_test.go index eb28ebc18c6..fbee7fd38e7 100644 --- a/go/libraries/doltcore/servercfg/yaml_config_test.go +++ b/go/libraries/doltcore/servercfg/yaml_config_test.go @@ -483,3 +483,72 @@ func TestYAMLConfigVerboseStringEquivalent(t *testing.T) { assert.True(t, yamlEquivalent(config.String(), config.VerboseString())) } } + +func TestCommentYAMLDiffs(t *testing.T) { + a := `abc: 100 +dddddd: "1234" +fire: water + +a: + b: + c: 1001011 + + t: g + +x: +- we +- se +- ll` + + b := `abc: 100 +dddddd: "1234" + +fire: water +extra1: 12345 + +a: + b: + c: 1001011 + extra2: iiiii + + t: g + +x: +- we +- extra3 +- extra4 +- se +- extra5 +- ll + +extra6: + extra7: + extra8: 999` + + expected := `abc: 100 +dddddd: "1234" + +fire: water +# extra1: 12345 + +a: + b: + c: 1001011 + # extra2: iiiii + + t: g + +x: +- we +# - extra3 +# - extra4 +- se +# - extra5 +- ll + +# extra6: + # extra7: + # extra8: 999` + + assert.Equal(t, expected, commentYAMLDiffs(a, b)) +} From 3a2e99974de74342fe9d10b3256e75e482d913a4 Mon Sep 17 00:00:00 2001 From: milogreg Date: Tue, 7 Jan 2025 13:35:04 -0800 Subject: [PATCH 21/31] Fix docs for generateYamlConfigIfNone --- go/cmd/dolt/commands/sqlserver/sqlserver.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/go/cmd/dolt/commands/sqlserver/sqlserver.go b/go/cmd/dolt/commands/sqlserver/sqlserver.go index 1f63a0c7965..4f53c403d4e 100644 --- a/go/cmd/dolt/commands/sqlserver/sqlserver.go +++ b/go/cmd/dolt/commands/sqlserver/sqlserver.go @@ -516,9 +516,8 @@ func setupDoltConfig(dEnv *env.DoltEnv, cwd filesys.Filesys, apr *argparser.ArgP } // generateYamlConfigIfNone creates a YAML config file in the database directory if one is not specified in the args -// and one doesn't already exist in the database directory. The fields of the generated YAML config file are set -// using serverConfig if serverConfig specifies a value for the field, otherwise the field is set to a default value -// or is replaced with a commented-out placeholder. +// and one doesn't already exist in the database directory. The fields of the YAML file are generated using the values +// in serverConfig that were explicitly set by the command line args. func generateYamlConfigIfNone( ap *argparser.ArgParser, help cli.UsagePrinter, From c72b0616f9a94f418bc9756155cad67635970fc3 Mon Sep 17 00:00:00 2001 From: milogreg Date: Tue, 7 Jan 2025 13:46:54 -0800 Subject: [PATCH 22/31] Make logic for checking mysql connection string more consistent with other fields --- go/cmd/dolt/commands/sqlserver/command_line_config.go | 1 + go/cmd/dolt/commands/sqlserver/sqlserver.go | 4 ---- go/libraries/doltcore/servercfg/serverconfig.go | 1 + go/libraries/doltcore/servercfg/yaml_config.go | 8 +++++++- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/go/cmd/dolt/commands/sqlserver/command_line_config.go b/go/cmd/dolt/commands/sqlserver/command_line_config.go index 378732003d8..65e8cca5f4d 100755 --- a/go/cmd/dolt/commands/sqlserver/command_line_config.go +++ b/go/cmd/dolt/commands/sqlserver/command_line_config.go @@ -177,6 +177,7 @@ func NewCommandLineConfig(creds *cli.UserPassword, apr *argparser.ArgParseResult if connStr, ok := apr.GetValue(goldenMysqlConn); ok { cli.Println(connStr) config.withGoldenMysqlConnectionString(connStr) + config.valuesSet[servercfg.GoldenMysqlConnectionStringKey] = struct{}{} } if esStatus, ok := apr.GetValue(eventSchedulerStatus); ok { diff --git a/go/cmd/dolt/commands/sqlserver/sqlserver.go b/go/cmd/dolt/commands/sqlserver/sqlserver.go index 4f53c403d4e..949d1872cc8 100644 --- a/go/cmd/dolt/commands/sqlserver/sqlserver.go +++ b/go/cmd/dolt/commands/sqlserver/sqlserver.go @@ -544,10 +544,6 @@ func generateYamlConfigIfNone( yamlConfig := servercfg.ServerConfigSetValuesAsYAMLConfig(serverConfig) - if connStr, ok := apr.GetValue(goldenMysqlConn); ok { - yamlConfig.GoldenMysqlConn = &connStr - } - generatedYaml := `# This file was generated using your configuration. # Uncomment and edit lines as necessary to modify your configuration.` + "\n\n" + yamlConfig.VerboseString() diff --git a/go/libraries/doltcore/servercfg/serverconfig.go b/go/libraries/doltcore/servercfg/serverconfig.go index 2a5c1a3e2ca..5f1b59cd04a 100644 --- a/go/libraries/doltcore/servercfg/serverconfig.go +++ b/go/libraries/doltcore/servercfg/serverconfig.go @@ -302,6 +302,7 @@ const ( RemotesapiReadOnlyKey = "remotesapi_read_only" ClusterConfigKey = "cluster_config" EventSchedulerKey = "event_scheduler" + GoldenMysqlConnectionStringKey = "golden_mysql_connection_string" ) type SystemVariableTarget interface { diff --git a/go/libraries/doltcore/servercfg/yaml_config.go b/go/libraries/doltcore/servercfg/yaml_config.go index fc10514ce1e..b31ac4816fa 100644 --- a/go/libraries/doltcore/servercfg/yaml_config.go +++ b/go/libraries/doltcore/servercfg/yaml_config.go @@ -249,7 +249,7 @@ func clusterConfigAsYAMLConfig(config ClusterConfig) *ClusterYAMLConfig { func ServerConfigSetValuesAsYAMLConfig(cfg ServerConfig) *YAMLConfig { systemVars := cfg.SystemVars() - return &YAMLConfig{ + res := &YAMLConfig{ LogLevelStr: zeroIf(ptr(string(cfg.LogLevel())), !cfg.ValueSet(LogLevelKey)), MaxQueryLenInLogs: zeroIf(ptr(cfg.MaxLoggedQueryLen()), !cfg.ValueSet(MaxLoggedQueryLenKey)), EncodeLoggedQuery: zeroIf(ptr(cfg.ShouldEncodeLoggedQuery()), !cfg.ValueSet(ShouldEncodeLoggedQueryKey)), @@ -294,6 +294,12 @@ func ServerConfigSetValuesAsYAMLConfig(cfg ServerConfig) *YAMLConfig { Vars: zeroIf(cfg.UserVars(), !cfg.ValueSet(UserVarsKey)), Jwks: zeroIf(cfg.JwksConfig(), !cfg.ValueSet(JwksConfigKey)), } + + if validatingCfg, ok := cfg.(ValidatingServerConfig); ok { + res.GoldenMysqlConn = zeroIf(ptr(validatingCfg.GoldenMysqlConnectionString()), !cfg.ValueSet(GoldenMysqlConnectionStringKey)) + } + + return res } func zeroIf[T any](val T, condition bool) T { From 70d537218437452d4185a4e6974618b3023e46b2 Mon Sep 17 00:00:00 2001 From: milogreg Date: Tue, 7 Jan 2025 14:58:19 -0800 Subject: [PATCH 23/31] Add test to ensure that an expected YAML config is generated verbatim for a given set of command line args --- go/cmd/dolt/commands/sqlserver/server_test.go | 119 ++++++++++++++++++ go/cmd/dolt/commands/sqlserver/sqlserver.go | 15 ++- 2 files changed, 130 insertions(+), 4 deletions(-) diff --git a/go/cmd/dolt/commands/sqlserver/server_test.go b/go/cmd/dolt/commands/sqlserver/server_test.go index c165b477a8f..7c0daeaaec1 100644 --- a/go/cmd/dolt/commands/sqlserver/server_test.go +++ b/go/cmd/dolt/commands/sqlserver/server_test.go @@ -33,6 +33,7 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/sqle" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess" "github.com/dolthub/dolt/go/libraries/utils/config" + "github.com/dolthub/dolt/go/libraries/utils/filesys" "github.com/dolthub/dolt/go/libraries/utils/svcs" ) @@ -510,3 +511,121 @@ func TestReadReplica(t *testing.T) { assert.ElementsMatch(t, res, []int{0}) }) } + +func TestGenerateYamlConfig(t *testing.T) { + args := []string{ + "--user", "my_name", + "--timeout", "11", + "--branch-control-file", "dir1/dir2/abc.db", + "--golden", "abc:xyz@tcp(127.0.0.1:3306)/db123", + } + + expected := `# This file was generated using your configuration. +# Uncomment and edit lines as necessary to modify your configuration. + +# log_level: info + +# max_logged_query_len: 0 + +# encode_logged_query: false + +# behavior: + # read_only: false + # autocommit: true + # disable_client_multi_statements: false + # dolt_transaction_commit: false + # event_scheduler: "OFF" + +user: + name: my_name + # password: "" + +listener: + # host: localhost + # port: 3306 + # max_connections: 100 + read_timeout_millis: 11000 + write_timeout_millis: 11000 + # tls_key: key.pem + # tls_cert: cert.pem + # require_secure_transport: false + # allow_cleartext_passwords: false + # socket: /tmp/mysql.sock + +# data_dir: . + +# cfg_dir: .doltcfg + +# metrics: + # labels: + # label1: value1 + # label2: "2" + # label3: "true" + # host: 123.45.67.89 + # port: 9091 + +# remotesapi: + # port: 8000 + # read_only: false + +# cluster: + # standby_remotes: + # - name: standby_replica_one + # remote_url_template: https://standby_replica_one.svc.cluster.local:50051/{database} + # - name: standby_replica_two + # remote_url_template: https://standby_replica_two.svc.cluster.local:50051/{database} + # bootstrap_role: primary + # bootstrap_epoch: 1 + # remotesapi: + # address: 127.0.0.1 + # port: 50051 + # tls_key: remotesapi_key.pem + # tls_cert: remotesapi_chain.pem + # tls_ca: standby_cas.pem + # server_name_urls: + # - https://standby_replica_one.svc.cluster.local + # - https://standby_replica_two.svc.cluster.local + # server_name_dns: + # - standby_replica_one.svc.cluster.local + # - standby_replica_two.svc.cluster.local + +# privilege_file: .doltcfg/privileges.db + +branch_control_file: dir1/dir2/abc.db + +# user_session_vars: +# - name: root + # vars: + # dolt_log_level: warn + # dolt_show_system_tables: 1 + +# system_variables: + # dolt_log_level: info + # dolt_transaction_commit: 1 + +# jwks: +# - name: name1 + # location_url: https://example.com + # claims: + # field1: a + # field2: b + # fields_to_log: + # - field1 + # - field2 + +golden_mysql_conn: abc:xyz@tcp(127.0.0.1:3306)/db123` + + ap := SqlServerCmd{}.ArgParser() + + dEnv := sqle.CreateTestEnv() + + cwd, err := os.Getwd() + require.NoError(t, err) + cwdFs, err := filesys.LocalFilesysWithWorkingDir(cwd) + require.NoError(t, err) + + serverConfig, err := ServerConfigFromArgs(ap, nil, args, dEnv, cwdFs) + require.NoError(t, err) + + assert.Equal(t, expected, generateYamlConfig(serverConfig)) +} diff --git a/go/cmd/dolt/commands/sqlserver/sqlserver.go b/go/cmd/dolt/commands/sqlserver/sqlserver.go index 949d1872cc8..ab792e2a4d2 100644 --- a/go/cmd/dolt/commands/sqlserver/sqlserver.go +++ b/go/cmd/dolt/commands/sqlserver/sqlserver.go @@ -542,10 +542,7 @@ func generateYamlConfigIfNone( return nil } - yamlConfig := servercfg.ServerConfigSetValuesAsYAMLConfig(serverConfig) - - generatedYaml := `# This file was generated using your configuration. -# Uncomment and edit lines as necessary to modify your configuration.` + "\n\n" + yamlConfig.VerboseString() + generatedYaml := generateYamlConfig(serverConfig) err := dEnv.FS.WriteFile(path, []byte(generatedYaml), os.ModePerm) if err != nil { @@ -554,3 +551,13 @@ func generateYamlConfigIfNone( return nil } + +// generateYamlConfig returns a YAML string containing the fields in serverConfig that +// were explicitly set by the command line args, along with commented-out placeholders for any +// fields that were not explicitly set by the command line args. +func generateYamlConfig(serverConfig servercfg.ServerConfig) string { + yamlConfig := servercfg.ServerConfigSetValuesAsYAMLConfig(serverConfig) + + return `# This file was generated using your configuration. +# Uncomment and edit lines as necessary to modify your configuration.` + "\n\n" + yamlConfig.VerboseString() +} From faf11c86c3bc4f5c34a34971a0bf005541ceb885 Mon Sep 17 00:00:00 2001 From: milogreg Date: Wed, 8 Jan 2025 12:09:35 -0800 Subject: [PATCH 24/31] Remove golden mysql connection string from config file generation --- .../dolt/commands/sqlserver/command_line_config.go | 1 - go/cmd/dolt/commands/sqlserver/server_test.go | 5 +---- go/libraries/doltcore/servercfg/serverconfig.go | 1 - go/libraries/doltcore/servercfg/yaml_config.go | 12 +----------- 4 files changed, 2 insertions(+), 17 deletions(-) diff --git a/go/cmd/dolt/commands/sqlserver/command_line_config.go b/go/cmd/dolt/commands/sqlserver/command_line_config.go index 65e8cca5f4d..378732003d8 100755 --- a/go/cmd/dolt/commands/sqlserver/command_line_config.go +++ b/go/cmd/dolt/commands/sqlserver/command_line_config.go @@ -177,7 +177,6 @@ func NewCommandLineConfig(creds *cli.UserPassword, apr *argparser.ArgParseResult if connStr, ok := apr.GetValue(goldenMysqlConn); ok { cli.Println(connStr) config.withGoldenMysqlConnectionString(connStr) - config.valuesSet[servercfg.GoldenMysqlConnectionStringKey] = struct{}{} } if esStatus, ok := apr.GetValue(eventSchedulerStatus); ok { diff --git a/go/cmd/dolt/commands/sqlserver/server_test.go b/go/cmd/dolt/commands/sqlserver/server_test.go index 7c0daeaaec1..4bcf233916f 100644 --- a/go/cmd/dolt/commands/sqlserver/server_test.go +++ b/go/cmd/dolt/commands/sqlserver/server_test.go @@ -517,7 +517,6 @@ func TestGenerateYamlConfig(t *testing.T) { "--user", "my_name", "--timeout", "11", "--branch-control-file", "dir1/dir2/abc.db", - "--golden", "abc:xyz@tcp(127.0.0.1:3306)/db123", } expected := `# This file was generated using your configuration. @@ -611,9 +610,7 @@ branch_control_file: dir1/dir2/abc.db # field2: b # fields_to_log: # - field1 - # - field2 - -golden_mysql_conn: abc:xyz@tcp(127.0.0.1:3306)/db123` + # - field2` ap := SqlServerCmd{}.ArgParser() diff --git a/go/libraries/doltcore/servercfg/serverconfig.go b/go/libraries/doltcore/servercfg/serverconfig.go index 5f1b59cd04a..2a5c1a3e2ca 100644 --- a/go/libraries/doltcore/servercfg/serverconfig.go +++ b/go/libraries/doltcore/servercfg/serverconfig.go @@ -302,7 +302,6 @@ const ( RemotesapiReadOnlyKey = "remotesapi_read_only" ClusterConfigKey = "cluster_config" EventSchedulerKey = "event_scheduler" - GoldenMysqlConnectionStringKey = "golden_mysql_connection_string" ) type SystemVariableTarget interface { diff --git a/go/libraries/doltcore/servercfg/yaml_config.go b/go/libraries/doltcore/servercfg/yaml_config.go index b31ac4816fa..1270837c704 100644 --- a/go/libraries/doltcore/servercfg/yaml_config.go +++ b/go/libraries/doltcore/servercfg/yaml_config.go @@ -249,7 +249,7 @@ func clusterConfigAsYAMLConfig(config ClusterConfig) *ClusterYAMLConfig { func ServerConfigSetValuesAsYAMLConfig(cfg ServerConfig) *YAMLConfig { systemVars := cfg.SystemVars() - res := &YAMLConfig{ + return &YAMLConfig{ LogLevelStr: zeroIf(ptr(string(cfg.LogLevel())), !cfg.ValueSet(LogLevelKey)), MaxQueryLenInLogs: zeroIf(ptr(cfg.MaxLoggedQueryLen()), !cfg.ValueSet(MaxLoggedQueryLenKey)), EncodeLoggedQuery: zeroIf(ptr(cfg.ShouldEncodeLoggedQuery()), !cfg.ValueSet(ShouldEncodeLoggedQueryKey)), @@ -294,12 +294,6 @@ func ServerConfigSetValuesAsYAMLConfig(cfg ServerConfig) *YAMLConfig { Vars: zeroIf(cfg.UserVars(), !cfg.ValueSet(UserVarsKey)), Jwks: zeroIf(cfg.JwksConfig(), !cfg.ValueSet(JwksConfigKey)), } - - if validatingCfg, ok := cfg.(ValidatingServerConfig); ok { - res.GoldenMysqlConn = zeroIf(ptr(validatingCfg.GoldenMysqlConnectionString()), !cfg.ValueSet(GoldenMysqlConnectionStringKey)) - } - - return res } func zeroIf[T any](val T, condition bool) T { @@ -467,10 +461,6 @@ func (cfg YAMLConfig) withPlaceholdersFilledIn() YAMLConfig { } } - if withPlaceholders.GoldenMysqlConn == nil { - withPlaceholders.GoldenMysqlConn = ptr("username:password@tcp(127.0.0.1:3306)/db") - } - return withPlaceholders } From 538b011884df72639afcf993a91ddd2dbb23dbb0 Mon Sep 17 00:00:00 2001 From: milogreg Date: Wed, 8 Jan 2025 12:23:42 -0800 Subject: [PATCH 25/31] Fix TestGenerateYamlConfig for windows --- go/cmd/dolt/commands/sqlserver/server_test.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/go/cmd/dolt/commands/sqlserver/server_test.go b/go/cmd/dolt/commands/sqlserver/server_test.go index 4bcf233916f..2dd6f663daf 100644 --- a/go/cmd/dolt/commands/sqlserver/server_test.go +++ b/go/cmd/dolt/commands/sqlserver/server_test.go @@ -17,6 +17,7 @@ package sqlserver import ( "net/http" "os" + "path/filepath" "strings" "sync" "testing" @@ -519,6 +520,9 @@ func TestGenerateYamlConfig(t *testing.T) { "--branch-control-file", "dir1/dir2/abc.db", } + privilegeFilePath, err := filepath.Localize(".doltcfg/privileges.db") + require.NoError(t, err) + expected := `# This file was generated using your configuration. # Uncomment and edit lines as necessary to modify your configuration. @@ -588,7 +592,8 @@ listener: # - standby_replica_one.svc.cluster.local # - standby_replica_two.svc.cluster.local -# privilege_file: .doltcfg/privileges.db +# privilege_file: ` + privilegeFilePath + + ` branch_control_file: dir1/dir2/abc.db From dad928c204791a6874b45ce6c295086a4133bc2f Mon Sep 17 00:00:00 2001 From: milogreg Date: Wed, 8 Jan 2025 12:28:39 -0800 Subject: [PATCH 26/31] Change comment on top of generated YAML config file --- go/cmd/dolt/commands/sqlserver/sqlserver.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/go/cmd/dolt/commands/sqlserver/sqlserver.go b/go/cmd/dolt/commands/sqlserver/sqlserver.go index ab792e2a4d2..22cc03dd39b 100644 --- a/go/cmd/dolt/commands/sqlserver/sqlserver.go +++ b/go/cmd/dolt/commands/sqlserver/sqlserver.go @@ -558,6 +558,9 @@ func generateYamlConfigIfNone( func generateYamlConfig(serverConfig servercfg.ServerConfig) string { yamlConfig := servercfg.ServerConfigSetValuesAsYAMLConfig(serverConfig) - return `# This file was generated using your configuration. -# Uncomment and edit lines as necessary to modify your configuration.` + "\n\n" + yamlConfig.VerboseString() + return `# Dolt SQL server configuration +# +# Uncomment and edit lines as necessary to modify your configuration. +# Full documentation: https://docs.dolthub.com/sql-reference/server/configuration +#` + "\n\n" + yamlConfig.VerboseString() } From dcebf8c95b06d4e768bfeff6f9abc58c5856ed97 Mon Sep 17 00:00:00 2001 From: milogreg Date: Wed, 8 Jan 2025 12:46:06 -0800 Subject: [PATCH 27/31] Change YAML config placeholder values --- .../doltcore/servercfg/yaml_config.go | 32 ++++--------------- 1 file changed, 7 insertions(+), 25 deletions(-) diff --git a/go/libraries/doltcore/servercfg/yaml_config.go b/go/libraries/doltcore/servercfg/yaml_config.go index 1270837c704..93fc4aead3f 100644 --- a/go/libraries/doltcore/servercfg/yaml_config.go +++ b/go/libraries/doltcore/servercfg/yaml_config.go @@ -346,7 +346,12 @@ func (cfg YAMLConfig) VerboseString() string { // withPlaceholdersFilledIn returns the config with placeholder values in place of nil values. // // The placeholder value for a field will be its default value if one exists, or an arbitrary -// example value if no default exists. Deprecated/unused fields won't be given an example value. +// example value if no default exists. +// +// The following will not be given placeholder values: +// - Deprecated or unused fields. +// - The field |Labels| in |MetricsConfig|. +// - The field |Jwks|. // // The config generated by this function should only be used to produce example values for // commented-out YAML fields, and shouldn't be used to actually configure anything. @@ -373,15 +378,8 @@ func (cfg YAMLConfig) withPlaceholdersFilledIn() YAMLConfig { withPlaceholders.ListenerConfig.Socket = ptr(DefaultUnixSocketFilePath) } - if withPlaceholders.MetricsConfig.Labels == nil { - withPlaceholders.MetricsConfig.Labels = map[string]string{ - "label1": "value1", - "label2": "2", - "label3": "true", - } - } if withPlaceholders.MetricsConfig.Host == nil { - withPlaceholders.MetricsConfig.Host = ptr("123.45.67.89") + withPlaceholders.MetricsConfig.Host = ptr("localhost") } if withPlaceholders.MetricsConfig.Port == nil { withPlaceholders.MetricsConfig.Port = ptr(9091) @@ -445,22 +443,6 @@ func (cfg YAMLConfig) withPlaceholdersFilledIn() YAMLConfig { } } - if withPlaceholders.Jwks == nil { - withPlaceholders.Jwks = []JwksConfig{ - JwksConfig{ - Name: "name1", - LocationUrl: "https://example.com", - Claims: map[string]string{ - "field1": "a", - "field2": "b", - }, - FieldsToLog: []string{ - "field1", "field2", - }, - }, - } - } - return withPlaceholders } From 54df51d7f9de0b1ae20281a9bd00956a41f2e11b Mon Sep 17 00:00:00 2001 From: milogreg Date: Wed, 8 Jan 2025 12:53:45 -0800 Subject: [PATCH 28/31] Reorder YAMLConfig struct to put MetricsConfig and ClusterConfig at the bottom --- .../servercfg/testdata/minver_validation.txt | 32 +++++++++---------- .../doltcore/servercfg/yaml_config.go | 4 +-- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/go/libraries/doltcore/servercfg/testdata/minver_validation.txt b/go/libraries/doltcore/servercfg/testdata/minver_validation.txt index b0ed893c19e..55f31a70b6b 100644 --- a/go/libraries/doltcore/servercfg/testdata/minver_validation.txt +++ b/go/libraries/doltcore/servercfg/testdata/minver_validation.txt @@ -29,13 +29,25 @@ PerformanceConfig *servercfg.PerformanceYAMLConfig 0.0.0 performance,omitempty -QueryParallelism *int 0.0.0 query_parallelism,omitempty DataDirStr *string 0.0.0 data_dir,omitempty CfgDirStr *string 0.0.0 cfg_dir,omitempty +RemotesapiConfig servercfg.RemotesapiYAMLConfig 0.0.0 remotesapi,omitempty +-Port_ *int 0.0.0 port,omitempty +-ReadOnly_ *bool 1.30.5 read_only,omitempty +PrivilegeFile *string 0.0.0 privilege_file,omitempty +BranchControlFile *string 0.0.0 branch_control_file,omitempty +Vars []servercfg.UserSessionVars 0.0.0 user_session_vars,omitempty +-Name string 0.0.0 name +-Vars map[string]interface{} 0.0.0 vars +SystemVars_ map[string]interface{} 1.11.1 system_variables,omitempty +Jwks []servercfg.JwksConfig 0.0.0 jwks,omitempty +-Name string 0.0.0 name +-LocationUrl string 0.0.0 location_url +-Claims map[string]string 0.0.0 claims +-FieldsToLog []string 0.0.0 fields_to_log +GoldenMysqlConn *string 0.0.0 golden_mysql_conn,omitempty MetricsConfig servercfg.MetricsYAMLConfig 0.0.0 metrics,omitempty -Labels map[string]string 0.0.0 labels,omitempty -Host *string 0.0.0 host,omitempty -Port *int 0.0.0 port,omitempty -RemotesapiConfig servercfg.RemotesapiYAMLConfig 0.0.0 remotesapi,omitempty --Port_ *int 0.0.0 port,omitempty --ReadOnly_ *bool 1.30.5 read_only,omitempty ClusterCfg *servercfg.ClusterYAMLConfig 0.0.0 cluster,omitempty -StandbyRemotes_ []servercfg.StandbyRemoteYAMLConfig 0.0.0 standby_remotes --Name_ string 0.0.0 name @@ -49,16 +61,4 @@ ClusterCfg *servercfg.ClusterYAMLConfig 0.0.0 cluster,omitempty --TLSCert_ string 0.0.0 tls_cert --TLSCA_ string 0.0.0 tls_ca --URLMatches []string 0.0.0 server_name_urls ---DNSMatches []string 0.0.0 server_name_dns -PrivilegeFile *string 0.0.0 privilege_file,omitempty -BranchControlFile *string 0.0.0 branch_control_file,omitempty -Vars []servercfg.UserSessionVars 0.0.0 user_session_vars,omitempty --Name string 0.0.0 name --Vars map[string]interface{} 0.0.0 vars -SystemVars_ map[string]interface{} 1.11.1 system_variables,omitempty -Jwks []servercfg.JwksConfig 0.0.0 jwks,omitempty --Name string 0.0.0 name --LocationUrl string 0.0.0 location_url --Claims map[string]string 0.0.0 claims --FieldsToLog []string 0.0.0 fields_to_log -GoldenMysqlConn *string 0.0.0 golden_mysql_conn,omitempty \ No newline at end of file +--DNSMatches []string 0.0.0 server_name_dns \ No newline at end of file diff --git a/go/libraries/doltcore/servercfg/yaml_config.go b/go/libraries/doltcore/servercfg/yaml_config.go index 93fc4aead3f..c1f48588d44 100644 --- a/go/libraries/doltcore/servercfg/yaml_config.go +++ b/go/libraries/doltcore/servercfg/yaml_config.go @@ -133,9 +133,7 @@ type YAMLConfig struct { PerformanceConfig *PerformanceYAMLConfig `yaml:"performance,omitempty"` DataDirStr *string `yaml:"data_dir,omitempty"` CfgDirStr *string `yaml:"cfg_dir,omitempty"` - MetricsConfig MetricsYAMLConfig `yaml:"metrics,omitempty"` RemotesapiConfig RemotesapiYAMLConfig `yaml:"remotesapi,omitempty"` - ClusterCfg *ClusterYAMLConfig `yaml:"cluster,omitempty"` PrivilegeFile *string `yaml:"privilege_file,omitempty"` BranchControlFile *string `yaml:"branch_control_file,omitempty"` // TODO: Rename to UserVars_ @@ -143,6 +141,8 @@ type YAMLConfig struct { SystemVars_ map[string]interface{} `yaml:"system_variables,omitempty" minver:"1.11.1"` Jwks []JwksConfig `yaml:"jwks,omitempty"` GoldenMysqlConn *string `yaml:"golden_mysql_conn,omitempty"` + MetricsConfig MetricsYAMLConfig `yaml:"metrics,omitempty"` + ClusterCfg *ClusterYAMLConfig `yaml:"cluster,omitempty"` } var _ ServerConfig = YAMLConfig{} From a0d9c0f556e55e89fe4d0f90af393f6d261d6f38 Mon Sep 17 00:00:00 2001 From: milogreg Date: Wed, 8 Jan 2025 12:59:05 -0800 Subject: [PATCH 29/31] Update TestGenerateYamlConfig --- go/cmd/dolt/commands/sqlserver/server_test.go | 59 ++++++++----------- 1 file changed, 24 insertions(+), 35 deletions(-) diff --git a/go/cmd/dolt/commands/sqlserver/server_test.go b/go/cmd/dolt/commands/sqlserver/server_test.go index 2dd6f663daf..38886fedde7 100644 --- a/go/cmd/dolt/commands/sqlserver/server_test.go +++ b/go/cmd/dolt/commands/sqlserver/server_test.go @@ -523,8 +523,11 @@ func TestGenerateYamlConfig(t *testing.T) { privilegeFilePath, err := filepath.Localize(".doltcfg/privileges.db") require.NoError(t, err) - expected := `# This file was generated using your configuration. + expected := `# Dolt SQL server configuration +# # Uncomment and edit lines as necessary to modify your configuration. +# Full documentation: https://docs.dolthub.com/sql-reference/server/configuration +# # log_level: info @@ -559,18 +562,29 @@ listener: # cfg_dir: .doltcfg -# metrics: - # labels: - # label1: value1 - # label2: "2" - # label3: "true" - # host: 123.45.67.89 - # port: 9091 - # remotesapi: # port: 8000 # read_only: false +# privilege_file: ` + privilegeFilePath + + ` + +branch_control_file: dir1/dir2/abc.db + +# user_session_vars: +# - name: root + # vars: + # dolt_log_level: warn + # dolt_show_system_tables: 1 + +# system_variables: + # dolt_log_level: info + # dolt_transaction_commit: 1 + +# metrics: + # host: localhost + # port: 9091 + # cluster: # standby_remotes: # - name: standby_replica_one @@ -590,32 +604,7 @@ listener: # - https://standby_replica_two.svc.cluster.local # server_name_dns: # - standby_replica_one.svc.cluster.local - # - standby_replica_two.svc.cluster.local - -# privilege_file: ` + privilegeFilePath + - ` - -branch_control_file: dir1/dir2/abc.db - -# user_session_vars: -# - name: root - # vars: - # dolt_log_level: warn - # dolt_show_system_tables: 1 - -# system_variables: - # dolt_log_level: info - # dolt_transaction_commit: 1 - -# jwks: -# - name: name1 - # location_url: https://example.com - # claims: - # field1: a - # field2: b - # fields_to_log: - # - field1 - # - field2` + # - standby_replica_two.svc.cluster.local` ap := SqlServerCmd{}.ArgParser() From f2c7416331900b1e38c5f62cc877aadac6cbdacd Mon Sep 17 00:00:00 2001 From: milogreg Date: Wed, 8 Jan 2025 13:30:16 -0800 Subject: [PATCH 30/31] Change the YAMLConfig fields MetricsConfig.Labels and Jwks to pointers so that omitempty can differentiate between an empty slice/map and a nil slice/map, allowing MetricsConfig.Labels and Jwks to have empty placeholder values in generated YAML --- go/cmd/dolt/commands/sqlserver/server_test.go | 3 ++ .../servercfg/testdata/minver_validation.txt | 8 +-- .../doltcore/servercfg/yaml_config.go | 51 +++++++++++++------ .../doltcore/servercfg/yaml_config_test.go | 6 +-- 4 files changed, 43 insertions(+), 25 deletions(-) diff --git a/go/cmd/dolt/commands/sqlserver/server_test.go b/go/cmd/dolt/commands/sqlserver/server_test.go index 38886fedde7..496cb324337 100644 --- a/go/cmd/dolt/commands/sqlserver/server_test.go +++ b/go/cmd/dolt/commands/sqlserver/server_test.go @@ -581,7 +581,10 @@ branch_control_file: dir1/dir2/abc.db # dolt_log_level: info # dolt_transaction_commit: 1 +# jwks: [] + # metrics: + # labels: {} # host: localhost # port: 9091 diff --git a/go/libraries/doltcore/servercfg/testdata/minver_validation.txt b/go/libraries/doltcore/servercfg/testdata/minver_validation.txt index 55f31a70b6b..3c6e17e97ff 100644 --- a/go/libraries/doltcore/servercfg/testdata/minver_validation.txt +++ b/go/libraries/doltcore/servercfg/testdata/minver_validation.txt @@ -38,14 +38,10 @@ Vars []servercfg.UserSessionVars 0.0.0 user_session_vars,omitempty -Name string 0.0.0 name -Vars map[string]interface{} 0.0.0 vars SystemVars_ map[string]interface{} 1.11.1 system_variables,omitempty -Jwks []servercfg.JwksConfig 0.0.0 jwks,omitempty --Name string 0.0.0 name --LocationUrl string 0.0.0 location_url --Claims map[string]string 0.0.0 claims --FieldsToLog []string 0.0.0 fields_to_log +Jwks *[]servercfg.JwksConfig 0.0.0 jwks,omitempty GoldenMysqlConn *string 0.0.0 golden_mysql_conn,omitempty MetricsConfig servercfg.MetricsYAMLConfig 0.0.0 metrics,omitempty --Labels map[string]string 0.0.0 labels,omitempty +-Labels *map[string]string 0.0.0 labels,omitempty -Host *string 0.0.0 host,omitempty -Port *int 0.0.0 port,omitempty ClusterCfg *servercfg.ClusterYAMLConfig 0.0.0 cluster,omitempty diff --git a/go/libraries/doltcore/servercfg/yaml_config.go b/go/libraries/doltcore/servercfg/yaml_config.go index c1f48588d44..739ab45163a 100644 --- a/go/libraries/doltcore/servercfg/yaml_config.go +++ b/go/libraries/doltcore/servercfg/yaml_config.go @@ -47,6 +47,20 @@ func nillableIntPtr(n int) *int { return &n } +func nillableSlicePtr[T any](s []T) *[]T { + if len(s) == 0 { + return nil + } + return &s +} + +func nillableMapPtr[K comparable, V any](m map[K]V) *map[K]V { + if len(m) == 0 { + return nil + } + return &m +} + // BehaviorYAMLConfig contains server configuration regarding how the server should behave type BehaviorYAMLConfig struct { ReadOnly *bool `yaml:"read_only,omitempty"` @@ -99,9 +113,9 @@ type PerformanceYAMLConfig struct { } type MetricsYAMLConfig struct { - Labels map[string]string `yaml:"labels,omitempty"` - Host *string `yaml:"host,omitempty"` - Port *int `yaml:"port,omitempty"` + Labels *map[string]string `yaml:"labels,omitempty"` + Host *string `yaml:"host,omitempty"` + Port *int `yaml:"port,omitempty"` } type RemotesapiYAMLConfig struct { @@ -139,7 +153,7 @@ type YAMLConfig struct { // TODO: Rename to UserVars_ Vars []UserSessionVars `yaml:"user_session_vars,omitempty"` SystemVars_ map[string]interface{} `yaml:"system_variables,omitempty" minver:"1.11.1"` - Jwks []JwksConfig `yaml:"jwks,omitempty"` + Jwks *[]JwksConfig `yaml:"jwks,omitempty"` GoldenMysqlConn *string `yaml:"golden_mysql_conn,omitempty"` MetricsConfig MetricsYAMLConfig `yaml:"metrics,omitempty"` ClusterCfg *ClusterYAMLConfig `yaml:"cluster,omitempty"` @@ -206,7 +220,7 @@ func ServerConfigAsYAMLConfig(cfg ServerConfig) *YAMLConfig { DataDirStr: ptr(cfg.DataDir()), CfgDirStr: ptr(cfg.CfgDir()), MetricsConfig: MetricsYAMLConfig{ - Labels: cfg.MetricsLabels(), + Labels: nillableMapPtr(cfg.MetricsLabels()), Host: nillableStrPtr(cfg.MetricsHost()), Port: ptr(cfg.MetricsPort()), }, @@ -219,7 +233,7 @@ func ServerConfigAsYAMLConfig(cfg ServerConfig) *YAMLConfig { BranchControlFile: ptr(cfg.BranchControlFilePath()), SystemVars_: systemVars, Vars: cfg.UserVars(), - Jwks: cfg.JwksConfig(), + Jwks: nillableSlicePtr(cfg.JwksConfig()), } } @@ -279,7 +293,7 @@ func ServerConfigSetValuesAsYAMLConfig(cfg ServerConfig) *YAMLConfig { DataDirStr: zeroIf(ptr(cfg.DataDir()), !cfg.ValueSet(DataDirKey)), CfgDirStr: zeroIf(ptr(cfg.CfgDir()), !cfg.ValueSet(CfgDirKey)), MetricsConfig: MetricsYAMLConfig{ - Labels: zeroIf(cfg.MetricsLabels(), !cfg.ValueSet(MetricsLabelsKey)), + Labels: zeroIf(ptr(cfg.MetricsLabels()), !cfg.ValueSet(MetricsLabelsKey)), Host: zeroIf(ptr(cfg.MetricsHost()), !cfg.ValueSet(MetricsHostKey)), Port: zeroIf(ptr(cfg.MetricsPort()), !cfg.ValueSet(MetricsPortKey)), }, @@ -292,7 +306,7 @@ func ServerConfigSetValuesAsYAMLConfig(cfg ServerConfig) *YAMLConfig { BranchControlFile: zeroIf(ptr(cfg.BranchControlFilePath()), !cfg.ValueSet(BranchControlFilePathKey)), SystemVars_: zeroIf(systemVars, !cfg.ValueSet(SystemVarsKey)), Vars: zeroIf(cfg.UserVars(), !cfg.ValueSet(UserVarsKey)), - Jwks: zeroIf(cfg.JwksConfig(), !cfg.ValueSet(JwksConfigKey)), + Jwks: zeroIf(ptr(cfg.JwksConfig()), !cfg.ValueSet(JwksConfigKey)), } } @@ -346,12 +360,7 @@ func (cfg YAMLConfig) VerboseString() string { // withPlaceholdersFilledIn returns the config with placeholder values in place of nil values. // // The placeholder value for a field will be its default value if one exists, or an arbitrary -// example value if no default exists. -// -// The following will not be given placeholder values: -// - Deprecated or unused fields. -// - The field |Labels| in |MetricsConfig|. -// - The field |Jwks|. +// example value if no default exists. Deprecated or unused fields will not be given placeholder values. // // The config generated by this function should only be used to produce example values for // commented-out YAML fields, and shouldn't be used to actually configure anything. @@ -378,6 +387,9 @@ func (cfg YAMLConfig) withPlaceholdersFilledIn() YAMLConfig { withPlaceholders.ListenerConfig.Socket = ptr(DefaultUnixSocketFilePath) } + if withPlaceholders.MetricsConfig.Labels == nil { + withPlaceholders.MetricsConfig.Labels = &map[string]string{} + } if withPlaceholders.MetricsConfig.Host == nil { withPlaceholders.MetricsConfig.Host = ptr("localhost") } @@ -443,6 +455,10 @@ func (cfg YAMLConfig) withPlaceholdersFilledIn() YAMLConfig { } } + if withPlaceholders.Jwks == nil { + withPlaceholders.Jwks = &[]JwksConfig{} + } + return withPlaceholders } @@ -658,7 +674,10 @@ func (cfg YAMLConfig) DisableClientMultiStatements() bool { // MetricsLabels returns labels that are applied to all prometheus metrics func (cfg YAMLConfig) MetricsLabels() map[string]string { - return cfg.MetricsConfig.Labels + if cfg.MetricsConfig.Labels != nil { + return *cfg.MetricsConfig.Labels + } + return nil } func (cfg YAMLConfig) MetricsHost() string { @@ -725,7 +744,7 @@ func (cfg YAMLConfig) SystemVars() map[string]interface{} { // wksConfig is JSON Web Key Set config, and used to validate a user authed with a jwt (JSON Web Token). func (cfg YAMLConfig) JwksConfig() []JwksConfig { if cfg.Jwks != nil { - return cfg.Jwks + return *cfg.Jwks } return nil } diff --git a/go/libraries/doltcore/servercfg/yaml_config_test.go b/go/libraries/doltcore/servercfg/yaml_config_test.go index fbee7fd38e7..4125ae72837 100644 --- a/go/libraries/doltcore/servercfg/yaml_config_test.go +++ b/go/libraries/doltcore/servercfg/yaml_config_test.go @@ -95,7 +95,7 @@ jwks: expected.MetricsConfig = MetricsYAMLConfig{ Host: ptr("123.45.67.89"), Port: ptr(9091), - Labels: map[string]string{ + Labels: &map[string]string{ "label1": "value1", "label2": "2", "label3": "true", @@ -121,7 +121,7 @@ jwks: }, }, } - expected.Jwks = []JwksConfig{ + expected.Jwks = &[]JwksConfig{ { Name: "jwks_name", LocationUrl: "https://website.com", @@ -461,7 +461,7 @@ func TestYAMLConfigVerboseStringEquivalent(t *testing.T) { }, YAMLConfig{ MetricsConfig: MetricsYAMLConfig{ - Labels: map[string]string{ + Labels: &map[string]string{ "xyz": "123", "0": "AAABBB", }, From 00815e5a675022d248633f2d06e48882a3e97571 Mon Sep 17 00:00:00 2001 From: milogreg Date: Wed, 8 Jan 2025 13:34:59 -0800 Subject: [PATCH 31/31] Remove redundant arg validation in generateYamlConfigIfNone --- go/cmd/dolt/commands/sqlserver/sqlserver.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/go/cmd/dolt/commands/sqlserver/sqlserver.go b/go/cmd/dolt/commands/sqlserver/sqlserver.go index 22cc03dd39b..f6d36563ca5 100644 --- a/go/cmd/dolt/commands/sqlserver/sqlserver.go +++ b/go/cmd/dolt/commands/sqlserver/sqlserver.go @@ -527,10 +527,6 @@ func generateYamlConfigIfNone( const yamlConfigName = "config.yaml" apr := cli.ParseArgsOrDie(ap, args, help) - if err := validateSqlServerArgs(apr); err != nil { - cli.PrintErrln(color.RedString(err.Error())) - return err - } if apr.Contains(configFileFlag) { return nil