Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Generate config.yaml when running sql-server #8644

Merged
merged 32 commits into from
Jan 10, 2025
Merged
Show file tree
Hide file tree
Changes from 23 commits
Commits
Show all changes
32 commits
Select commit Hold shift + click to select a range
81e21a6
Generate config.yaml with current sql server config if one doesn't al…
milogreg Dec 5, 2024
0df8e9a
Fill default values in YAML config if they exist rather than leaving …
milogreg Dec 5, 2024
3b24a0c
Comment out null values in generated YAML config
milogreg Dec 5, 2024
1b407e0
Add bats tests for config file generation
milogreg Dec 9, 2024
5f4f2d1
Skip bats tests when SQL_ENGINE is set to remote-engine
milogreg Dec 9, 2024
698cc63
Merge main
milogreg Dec 9, 2024
27dcc30
Don't use removed persistence_behavior config option in tests
milogreg Dec 9, 2024
6f89115
Make default filler not use reflection
milogreg Dec 10, 2024
2654f84
Remove circular reference test
milogreg Dec 10, 2024
211a52e
Add test for commentNullYAMLValues
milogreg Dec 10, 2024
bc409a6
Remove CLI warning for when directory named config.yaml exists in dat…
milogreg Dec 10, 2024
b316473
Add additional doc comments for functions
milogreg Dec 10, 2024
bd4c410
Merge main
milogreg Dec 11, 2024
7627f6b
Merge main
milogreg Jan 6, 2025
290d0d2
Rework config file generation to only include fields explicitly set a…
milogreg Jan 6, 2025
96e2200
Make YAML formatting not treat list items in a top-level field as top…
milogreg Jan 6, 2025
6eb9959
Update minver_validation.txt
milogreg Jan 6, 2025
86446e6
Update config file gen bats test
milogreg Jan 6, 2025
7a3b62e
Add test to ensure YAMLConfig's String and VerboseString functions pr…
milogreg Jan 6, 2025
a3e3b11
Add test for commentYAMLDiffs
milogreg Jan 7, 2025
c70c87f
Merge branch 'main' into config-file-gen
milogreg Jan 7, 2025
3a2e999
Fix docs for generateYamlConfigIfNone
milogreg Jan 7, 2025
c72b061
Make logic for checking mysql connection string more consistent with …
milogreg Jan 7, 2025
70d5372
Add test to ensure that an expected YAML config is generated verbatim…
milogreg Jan 7, 2025
faf11c8
Remove golden mysql connection string from config file generation
milogreg Jan 8, 2025
538b011
Fix TestGenerateYamlConfig for windows
milogreg Jan 8, 2025
dad928c
Change comment on top of generated YAML config file
milogreg Jan 8, 2025
dcebf8c
Change YAML config placeholder values
milogreg Jan 8, 2025
54df51d
Reorder YAMLConfig struct to put MetricsConfig and ClusterConfig at t…
milogreg Jan 8, 2025
a0d9c0f
Update TestGenerateYamlConfig
milogreg Jan 8, 2025
f2c7416
Change the YAMLConfig fields MetricsConfig.Labels and Jwks to pointer…
milogreg Jan 8, 2025
00815e5
Remove redundant arg validation in generateYamlConfigIfNone
milogreg Jan 8, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions go/cmd/dolt/commands/sqlserver/command_line_config.go
Original file line number Diff line number Diff line change
Expand Up @@ -105,9 +105,11 @@ func NewCommandLineConfig(creds *cli.UserPassword, apr *argparser.ArgParseResult
if creds == nil {
if user, ok := apr.GetValue(cli.UserFlag); ok {
config.withUser(user)
config.valuesSet[servercfg.UserKey] = struct{}{}
}
if password, ok := apr.GetValue(cli.PasswordFlag); ok {
config.withPassword(password)
config.valuesSet[servercfg.PasswordKey] = struct{}{}
}
} else {
config.withUser(creds.Username)
Expand Down Expand Up @@ -163,11 +165,19 @@ func NewCommandLineConfig(creds *cli.UserPassword, apr *argparser.ArgParseResult
}

config.autoCommit = !apr.Contains(noAutoCommitFlag)
if apr.Contains(noAutoCommitFlag) {
config.valuesSet[servercfg.AutoCommitKey] = struct{}{}
}

config.allowCleartextPasswords = apr.Contains(allowCleartextPasswordsFlag)
if apr.Contains(allowCleartextPasswordsFlag) {
config.valuesSet[servercfg.AllowCleartextPasswordsKey] = struct{}{}
}

if connStr, ok := apr.GetValue(goldenMysqlConn); ok {
cli.Println(connStr)
config.withGoldenMysqlConnectionString(connStr)
config.valuesSet[servercfg.GoldenMysqlConnectionStringKey] = struct{}{}
}

if esStatus, ok := apr.GetValue(eventSchedulerStatus); ok {
Expand Down Expand Up @@ -341,12 +351,14 @@ func (cfg *commandLineServerConfig) Socket() string {
// WithHost updates the host and returns the called `*commandLineServerConfig`, which is useful for chaining calls.
func (cfg *commandLineServerConfig) WithHost(host string) *commandLineServerConfig {
cfg.host = host
cfg.valuesSet[servercfg.HostKey] = struct{}{}
return cfg
}

// WithPort updates the port and returns the called `*commandLineServerConfig`, which is useful for chaining calls.
func (cfg *commandLineServerConfig) WithPort(port int) *commandLineServerConfig {
cfg.port = port
cfg.valuesSet[servercfg.PortKey] = struct{}{}
return cfg
}

Expand All @@ -373,12 +385,14 @@ func (cfg *commandLineServerConfig) withTimeout(timeout uint64) *commandLineServ
// withReadOnly updates the read only flag and returns the called `*commandLineServerConfig`, which is useful for chaining calls.
func (cfg *commandLineServerConfig) withReadOnly(readonly bool) *commandLineServerConfig {
cfg.readOnly = readonly
cfg.valuesSet[servercfg.ReadOnlyKey] = struct{}{}
return cfg
}

// withLogLevel updates the log level and returns the called `*commandLineServerConfig`, which is useful for chaining calls.
func (cfg *commandLineServerConfig) withLogLevel(loglevel servercfg.LogLevel) *commandLineServerConfig {
cfg.logLevel = loglevel
cfg.valuesSet[servercfg.LogLevelKey] = struct{}{}
return cfg
}

Expand Down Expand Up @@ -416,23 +430,27 @@ func (cfg *commandLineServerConfig) withBranchControlFilePath(branchControlFileP

func (cfg *commandLineServerConfig) withAllowCleartextPasswords(allow bool) *commandLineServerConfig {
cfg.allowCleartextPasswords = allow
cfg.valuesSet[servercfg.AllowCleartextPasswordsKey] = struct{}{}
return cfg
}

// WithSocket updates the path to the unix socket file
func (cfg *commandLineServerConfig) WithSocket(sockFilePath string) *commandLineServerConfig {
cfg.socket = sockFilePath
cfg.valuesSet[servercfg.SocketKey] = struct{}{}
return cfg
}

// WithRemotesapiPort sets the remotesapi port to use.
func (cfg *commandLineServerConfig) WithRemotesapiPort(port *int) *commandLineServerConfig {
cfg.remotesapiPort = port
cfg.valuesSet[servercfg.RemotesapiPortKey] = struct{}{}
return cfg
}

func (cfg *commandLineServerConfig) WithRemotesapiReadOnly(readonly *bool) *commandLineServerConfig {
cfg.remotesapiReadOnly = readonly
cfg.valuesSet[servercfg.RemotesapiReadOnlyKey] = struct{}{}
return cfg
}

Expand Down
56 changes: 56 additions & 0 deletions go/cmd/dolt/commands/sqlserver/sqlserver.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"strings"

Expand Down Expand Up @@ -244,6 +245,11 @@ func StartServer(ctx context.Context, versionStr, commandStr string, args []stri
return err
}

err = generateYamlConfigIfNone(ap, help, args, dEnv, serverConfig)
zachmu marked this conversation as resolved.
Show resolved Hide resolved
if err != nil {
return err
}

err = servercfg.ApplySystemVariables(serverConfig, sql.SystemVariables)
if err != nil {
return err
Expand Down Expand Up @@ -476,8 +482,17 @@ func setupDoltConfig(dEnv *env.DoltEnv, cwd filesys.Filesys, apr *argparser.ArgP
}
serverConfig.withCfgDir(cfgDirPath)

if cfgDirSpecified {
serverConfig.valuesSet[servercfg.CfgDirKey] = struct{}{}
}

if dataDirSpecified {
serverConfig.valuesSet[servercfg.DataDirKey] = struct{}{}
}

if privsFp, ok := apr.GetValue(commands.PrivsFilePathFlag); ok {
serverConfig.withPrivilegeFilePath(privsFp)
serverConfig.valuesSet[servercfg.PrivilegeFilePathKey] = struct{}{}
} else {
path, err := dEnv.FS.Abs(filepath.Join(cfgDirPath, commands.DefaultPrivsName))
if err != nil {
Expand All @@ -488,6 +503,7 @@ func setupDoltConfig(dEnv *env.DoltEnv, cwd filesys.Filesys, apr *argparser.ArgP

if branchControlFilePath, ok := apr.GetValue(commands.BranchCtrlPathFlag); ok {
serverConfig.withBranchControlFilePath(branchControlFilePath)
serverConfig.valuesSet[servercfg.BranchControlFilePathKey] = struct{}{}
} else {
path, err := dEnv.FS.Abs(filepath.Join(cfgDirPath, commands.DefaultBranchCtrlName))
if err != nil {
Expand All @@ -498,3 +514,43 @@ func setupDoltConfig(dEnv *env.DoltEnv, cwd filesys.Filesys, apr *argparser.ArgP

return nil
}

// generateYamlConfigIfNone creates a YAML config file in the database directory if one is not specified in the args
// and one doesn't already exist in the database directory. The fields of the YAML file are generated using the values
// in serverConfig that were explicitly set by the command line args.
func generateYamlConfigIfNone(
zachmu marked this conversation as resolved.
Show resolved Hide resolved
ap *argparser.ArgParser,
help cli.UsagePrinter,
args []string,
dEnv *env.DoltEnv,
serverConfig servercfg.ServerConfig) error {
const yamlConfigName = "config.yaml"

apr := cli.ParseArgsOrDie(ap, args, help)
if err := validateSqlServerArgs(apr); err != nil {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is redundant, already happens in ServerConfigFromArgs

cli.PrintErrln(color.RedString(err.Error()))
return err
}

if apr.Contains(configFileFlag) {
return nil
}

path := filepath.Join(serverConfig.DataDir(), yamlConfigName)
exists, _ := dEnv.FS.Exists(path)
if exists {
return nil
}

yamlConfig := servercfg.ServerConfigSetValuesAsYAMLConfig(serverConfig)

generatedYaml := `# This file was generated using your configuration.
# Uncomment and edit lines as necessary to modify your configuration.` + "\n\n" + yamlConfig.VerboseString()
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
generatedYaml := `# This file was generated using your configuration.
# Uncomment and edit lines as necessary to modify your configuration.` + "\n\n" + yamlConfig.VerboseString()
generatedYaml := `# Dolt SQL server configuration
#
# Uncomment and edit lines as necessary to modify your configuration.
# Full documentation: https://docs.dolthub.com/sql-reference/server/configuration
#` + "\n\n" + yamlConfig.VerboseString()


err := dEnv.FS.WriteFile(path, []byte(generatedYaml), os.ModePerm)
if err != nil {
return err
}

return nil
}
42 changes: 38 additions & 4 deletions go/libraries/doltcore/servercfg/serverconfig.go
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,10 @@ type ServerConfig interface {

// DefaultServerConfig creates a `*ServerConfig` that has all of the options set to their default values.
func DefaultServerConfig() ServerConfig {
return defaultServerConfigYAML()
}

func defaultServerConfigYAML() *YAMLConfig {
return &YAMLConfig{
LogLevelStr: ptr(string(DefaultLogLevel)),
MaxQueryLenInLogs: ptr(DefaultMaxLoggedQueryLen),
Expand Down Expand Up @@ -265,10 +269,40 @@ func ValidateConfig(config ServerConfig) error {
}

const (
MaxConnectionsKey = "max_connections"
ReadTimeoutKey = "net_read_timeout"
WriteTimeoutKey = "net_write_timeout"
EventSchedulerKey = "event_scheduler"
HostKey = "host"
PortKey = "port"
UserKey = "user"
PasswordKey = "password"
ReadTimeoutKey = "net_read_timeout"
WriteTimeoutKey = "net_write_timeout"
ReadOnlyKey = "read_only"
LogLevelKey = "log_level"
AutoCommitKey = "autocommit"
DoltTransactionCommitKey = "dolt_transaction_commit"
DataDirKey = "data_dir"
CfgDirKey = "cfg_dir"
MaxConnectionsKey = "max_connections"
TLSKeyKey = "tls_key"
TLSCertKey = "tls_cert"
RequireSecureTransportKey = "require_secure_transport"
MaxLoggedQueryLenKey = "max_logged_query_len"
ShouldEncodeLoggedQueryKey = "should_encode_logged_query"
DisableClientMultiStatementsKey = "disable_client_multi_statements"
MetricsLabelsKey = "metrics_labels"
MetricsHostKey = "metrics_host"
MetricsPortKey = "metrics_port"
PrivilegeFilePathKey = "privilege_file_path"
BranchControlFilePathKey = "branch_control_file_path"
UserVarsKey = "user_vars"
SystemVarsKey = "system_vars"
JwksConfigKey = "jwks_config"
AllowCleartextPasswordsKey = "allow_cleartext_passwords"
SocketKey = "socket"
RemotesapiPortKey = "remotesapi_port"
RemotesapiReadOnlyKey = "remotesapi_read_only"
ClusterConfigKey = "cluster_config"
EventSchedulerKey = "event_scheduler"
GoldenMysqlConnectionStringKey = "golden_mysql_connection_string"
)

type SystemVariableTarget interface {
Expand Down
50 changes: 25 additions & 25 deletions go/libraries/doltcore/servercfg/testdata/minver_validation.txt
Original file line number Diff line number Diff line change
Expand Up @@ -4,36 +4,36 @@
LogLevelStr *string 0.0.0 log_level,omitempty
MaxQueryLenInLogs *int 0.0.0 max_logged_query_len,omitempty
EncodeLoggedQuery *bool 0.0.0 encode_logged_query,omitempty
BehaviorConfig servercfg.BehaviorYAMLConfig 0.0.0 behavior
-ReadOnly *bool 0.0.0 read_only
-AutoCommit *bool 0.0.0 autocommit
BehaviorConfig servercfg.BehaviorYAMLConfig 0.0.0 behavior,omitempty
-ReadOnly *bool 0.0.0 read_only,omitempty
-AutoCommit *bool 0.0.0 autocommit,omitempty
-PersistenceBehavior *string 0.0.0 persistence_behavior,omitempty
-DisableClientMultiStatements *bool 0.0.0 disable_client_multi_statements
-DoltTransactionCommit *bool 0.0.0 dolt_transaction_commit
-DisableClientMultiStatements *bool 0.0.0 disable_client_multi_statements,omitempty
-DoltTransactionCommit *bool 0.0.0 dolt_transaction_commit,omitempty
-EventSchedulerStatus *string 1.17.0 event_scheduler,omitempty
UserConfig servercfg.UserYAMLConfig 0.0.0 user
-Name *string 0.0.0 name
-Password *string 0.0.0 password
ListenerConfig servercfg.ListenerYAMLConfig 0.0.0 listener
-HostStr *string 0.0.0 host
-PortNumber *int 0.0.0 port
-MaxConnections *uint64 0.0.0 max_connections
-ReadTimeoutMillis *uint64 0.0.0 read_timeout_millis
-WriteTimeoutMillis *uint64 0.0.0 write_timeout_millis
-TLSKey *string 0.0.0 tls_key
-TLSCert *string 0.0.0 tls_cert
-RequireSecureTransport *bool 0.0.0 require_secure_transport
-AllowCleartextPasswords *bool 0.0.0 allow_cleartext_passwords
UserConfig servercfg.UserYAMLConfig 0.0.0 user,omitempty
-Name *string 0.0.0 name,omitempty
-Password *string 0.0.0 password,omitempty
ListenerConfig servercfg.ListenerYAMLConfig 0.0.0 listener,omitempty
-HostStr *string 0.0.0 host,omitempty
-PortNumber *int 0.0.0 port,omitempty
-MaxConnections *uint64 0.0.0 max_connections,omitempty
-ReadTimeoutMillis *uint64 0.0.0 read_timeout_millis,omitempty
-WriteTimeoutMillis *uint64 0.0.0 write_timeout_millis,omitempty
-TLSKey *string 0.0.0 tls_key,omitempty
-TLSCert *string 0.0.0 tls_cert,omitempty
-RequireSecureTransport *bool 0.0.0 require_secure_transport,omitempty
-AllowCleartextPasswords *bool 0.0.0 allow_cleartext_passwords,omitempty
-Socket *string 0.0.0 socket,omitempty
PerformanceConfig *servercfg.PerformanceYAMLConfig 0.0.0 performance,omitempty
-QueryParallelism *int 0.0.0 query_parallelism,omitempty
DataDirStr *string 0.0.0 data_dir,omitempty
CfgDirStr *string 0.0.0 cfg_dir,omitempty
MetricsConfig servercfg.MetricsYAMLConfig 0.0.0 metrics
-Labels map[string]string 0.0.0 labels
-Host *string 0.0.0 host
-Port *int 0.0.0 port
RemotesapiConfig servercfg.RemotesapiYAMLConfig 0.0.0 remotesapi
MetricsConfig servercfg.MetricsYAMLConfig 0.0.0 metrics,omitempty
-Labels map[string]string 0.0.0 labels,omitempty
-Host *string 0.0.0 host,omitempty
-Port *int 0.0.0 port,omitempty
RemotesapiConfig servercfg.RemotesapiYAMLConfig 0.0.0 remotesapi,omitempty
-Port_ *int 0.0.0 port,omitempty
-ReadOnly_ *bool 1.30.5 read_only,omitempty
ClusterCfg *servercfg.ClusterYAMLConfig 0.0.0 cluster,omitempty
Expand All @@ -52,11 +52,11 @@ ClusterCfg *servercfg.ClusterYAMLConfig 0.0.0 cluster,omitempty
--DNSMatches []string 0.0.0 server_name_dns
PrivilegeFile *string 0.0.0 privilege_file,omitempty
BranchControlFile *string 0.0.0 branch_control_file,omitempty
Vars []servercfg.UserSessionVars 0.0.0 user_session_vars
Vars []servercfg.UserSessionVars 0.0.0 user_session_vars,omitempty
-Name string 0.0.0 name
-Vars map[string]interface{} 0.0.0 vars
SystemVars_ map[string]interface{} 1.11.1 system_variables,omitempty
Jwks []servercfg.JwksConfig 0.0.0 jwks
Jwks []servercfg.JwksConfig 0.0.0 jwks,omitempty
-Name string 0.0.0 name
-LocationUrl string 0.0.0 location_url
-Claims map[string]string 0.0.0 claims
Expand Down
Loading
Loading