From 59995b68e86f39bd77f4f3e6682d2a46121c60d9 Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Mon, 6 May 2024 10:56:16 -0700 Subject: [PATCH 1/6] Bug fix for Decimal type serialization for binlog events --- .../binlog_type_serialization.go | 15 ++++--- .../binlog_type_serialization_test.go | 42 +++++++++++++++++++ 2 files changed, 51 insertions(+), 6 deletions(-) diff --git a/go/libraries/doltcore/sqle/binlogreplication/binlog_type_serialization.go b/go/libraries/doltcore/sqle/binlogreplication/binlog_type_serialization.go index d0a1c351a6f..8b53b436eab 100644 --- a/go/libraries/doltcore/sqle/binlogreplication/binlog_type_serialization.go +++ b/go/libraries/doltcore/sqle/binlogreplication/binlog_type_serialization.go @@ -19,15 +19,15 @@ import ( "fmt" "math" "strconv" + "strings" + "github.com/dolthub/dolt/go/store/hash" + "github.com/dolthub/dolt/go/store/prolly/tree" + "github.com/dolthub/dolt/go/store/val" "github.com/dolthub/go-mysql-server/sql" gmstypes "github.com/dolthub/go-mysql-server/sql/types" "github.com/dolthub/vitess/go/mysql" "github.com/dolthub/vitess/go/vt/proto/query" - - "github.com/dolthub/dolt/go/store/hash" - "github.com/dolthub/dolt/go/store/prolly/tree" - "github.com/dolthub/dolt/go/store/val" ) // typeSerializer defines the serialization interface for serializing a value in Dolt's @@ -291,9 +291,12 @@ func (d decimalSerializer) serialize(_ *sql.Context, typ sql.Type, descriptor va decimalValue.Exponent(), decimalValue.String()) } - absStringVal := decimalValue.Abs().String() - firstFractionalDigitIdx := len(absStringVal) + int(decimalValue.Exponent()) + absStringVal := decimalValue.Abs().StringFixed(int32(scale)) + firstFractionalDigitIdx := strings.Index(absStringVal, ".") + 1 stringIntegerVal := absStringVal[:firstFractionalDigitIdx-1] + for len(stringIntegerVal) < int(numFullDigits) { + stringIntegerVal = "0" + stringIntegerVal + } stringFractionalVal := absStringVal[firstFractionalDigitIdx:] buffer := make([]byte, length) diff --git a/go/libraries/doltcore/sqle/binlogreplication/binlog_type_serialization_test.go b/go/libraries/doltcore/sqle/binlogreplication/binlog_type_serialization_test.go index 229d7e6a93d..26679f791f4 100644 --- a/go/libraries/doltcore/sqle/binlogreplication/binlog_type_serialization_test.go +++ b/go/libraries/doltcore/sqle/binlogreplication/binlog_type_serialization_test.go @@ -652,6 +652,48 @@ func TestIntegerSerializer(t *testing.T) { func TestDecimalSerializer(t *testing.T) { s := decimalSerializer{} + t.Run("0", func(t *testing.T) { + typ := gmstypes.MustCreateDecimalType(14, 4) + tupleDesc, tupleBuilder := newTupleBuilderForEncoding(val.DecimalEnc) + dec, err := decimal.NewFromString("0") + require.NoError(t, err) + tupleBuilder.PutDecimal(0, dec) + tuple := tupleBuilder.Build(buffPool) + bytes, err := s.serialize(nil, typ, tupleDesc, tuple, 0, nil) + require.NoError(t, err) + require.Equal(t, []byte{0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, bytes) + typeId, metadata := s.metadata(nil, typ) + require.EqualValues(t, mysql.TypeNewDecimal, typeId) + require.EqualValues(t, 14<<8|4, metadata) + }) + t.Run("100", func(t *testing.T) { + typ := gmstypes.MustCreateDecimalType(14, 4) + tupleDesc, tupleBuilder := newTupleBuilderForEncoding(val.DecimalEnc) + dec, err := decimal.NewFromString("100") + require.NoError(t, err) + tupleBuilder.PutDecimal(0, dec) + tuple := tupleBuilder.Build(buffPool) + bytes, err := s.serialize(nil, typ, tupleDesc, tuple, 0, nil) + require.NoError(t, err) + require.Equal(t, []byte{0x80, 0x0, 0x0, 0x0, 0x64, 0x0, 0x0}, bytes) + typeId, metadata := s.metadata(nil, typ) + require.EqualValues(t, mysql.TypeNewDecimal, typeId) + require.EqualValues(t, 14<<8|4, metadata) + }) + t.Run("1.1", func(t *testing.T) { + typ := gmstypes.MustCreateDecimalType(14, 4) + tupleDesc, tupleBuilder := newTupleBuilderForEncoding(val.DecimalEnc) + dec, err := decimal.NewFromString("1.1") + require.NoError(t, err) + tupleBuilder.PutDecimal(0, dec) + tuple := tupleBuilder.Build(buffPool) + bytes, err := s.serialize(nil, typ, tupleDesc, tuple, 0, nil) + require.NoError(t, err) + require.Equal(t, []byte{0x80, 0x0, 0x0, 0x0, 0x1, 0x3, 0xe8}, bytes) + typeId, metadata := s.metadata(nil, typ) + require.EqualValues(t, mysql.TypeNewDecimal, typeId) + require.EqualValues(t, 14<<8|4, metadata) + }) t.Run("1234567890.1234", func(t *testing.T) { typ := gmstypes.MustCreateDecimalType(14, 4) tupleDesc, tupleBuilder := newTupleBuilderForEncoding(val.DecimalEnc) From 108bfb5acc2a5808e1217f084fac63a29ba95a29 Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Mon, 6 May 2024 11:43:30 -0700 Subject: [PATCH 2/6] Bug fix for Decimal type binlog serialization when scale is 0 --- .../binlog_type_serialization.go | 43 +++++++++++-------- .../binlog_type_serialization_test.go | 14 ++++++ 2 files changed, 39 insertions(+), 18 deletions(-) diff --git a/go/libraries/doltcore/sqle/binlogreplication/binlog_type_serialization.go b/go/libraries/doltcore/sqle/binlogreplication/binlog_type_serialization.go index 8b53b436eab..dd83fe18919 100644 --- a/go/libraries/doltcore/sqle/binlogreplication/binlog_type_serialization.go +++ b/go/libraries/doltcore/sqle/binlogreplication/binlog_type_serialization.go @@ -292,12 +292,16 @@ func (d decimalSerializer) serialize(_ *sql.Context, typ sql.Type, descriptor va } absStringVal := decimalValue.Abs().StringFixed(int32(scale)) - firstFractionalDigitIdx := strings.Index(absStringVal, ".") + 1 - stringIntegerVal := absStringVal[:firstFractionalDigitIdx-1] + stringIntegerVal := absStringVal + stringFractionalVal := "" + if scale > 0 { + firstFractionalDigitIdx := strings.Index(absStringVal, ".") + 1 + stringIntegerVal = absStringVal[:firstFractionalDigitIdx-1] + stringFractionalVal = absStringVal[firstFractionalDigitIdx:] + } for len(stringIntegerVal) < int(numFullDigits) { stringIntegerVal = "0" + stringIntegerVal } - stringFractionalVal := absStringVal[firstFractionalDigitIdx:] buffer := make([]byte, length) bufferPos := 0 @@ -322,23 +326,26 @@ func (d decimalSerializer) serialize(_ *sql.Context, typ sql.Type, descriptor va remainingString) } - // Fill in full fractional digits - writtenBytes, remainingString, err = encodeDecimalBits(stringFractionalVal, buffer[bufferPos:]) - if err != nil { - return nil, err - } - bufferPos += int(writtenBytes) + // If there is a scale, then encode the fractional digits of the number + if scale > 0 { + // Fill in full fractional digits + writtenBytes, remainingString, err = encodeDecimalBits(stringFractionalVal, buffer[bufferPos:]) + if err != nil { + return nil, err + } + bufferPos += int(writtenBytes) - // Fill in partial fractional digits – these are at the end of the fractional component - writtenBytes, err = encodePartialDecimalBits(remainingString, buffer[bufferPos:]) - if err != nil { - return nil, err - } - bufferPos += int(writtenBytes) + // Fill in partial fractional digits – these are at the end of the fractional component + writtenBytes, err = encodePartialDecimalBits(remainingString, buffer[bufferPos:]) + if err != nil { + return nil, err + } + bufferPos += int(writtenBytes) - if bufferPos != len(buffer) { - return nil, fmt.Errorf( - "unexpected position; bufferPos: %d, len(buffer): %d", bufferPos, len(buffer)) + if bufferPos != len(buffer) { + return nil, fmt.Errorf( + "unexpected position; bufferPos: %d, len(buffer): %d", bufferPos, len(buffer)) + } } // We always xor the first bit in the first byte to indicate a positive value. If the value is diff --git a/go/libraries/doltcore/sqle/binlogreplication/binlog_type_serialization_test.go b/go/libraries/doltcore/sqle/binlogreplication/binlog_type_serialization_test.go index 26679f791f4..7befa3217a1 100644 --- a/go/libraries/doltcore/sqle/binlogreplication/binlog_type_serialization_test.go +++ b/go/libraries/doltcore/sqle/binlogreplication/binlog_type_serialization_test.go @@ -694,6 +694,20 @@ func TestDecimalSerializer(t *testing.T) { require.EqualValues(t, mysql.TypeNewDecimal, typeId) require.EqualValues(t, 14<<8|4, metadata) }) + t.Run("10", func(t *testing.T) { + typ := gmstypes.MustCreateDecimalType(19, 0) + tupleDesc, tupleBuilder := newTupleBuilderForEncoding(val.DecimalEnc) + dec, err := decimal.NewFromString("100") + require.NoError(t, err) + tupleBuilder.PutDecimal(0, dec) + tuple := tupleBuilder.Build(buffPool) + bytes, err := s.serialize(nil, typ, tupleDesc, tuple, 0, nil) + require.NoError(t, err) + require.Equal(t, []byte{0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x64}, bytes) + typeId, metadata := s.metadata(nil, typ) + require.EqualValues(t, mysql.TypeNewDecimal, typeId) + require.EqualValues(t, 19<<8|0, metadata) + }) t.Run("1234567890.1234", func(t *testing.T) { typ := gmstypes.MustCreateDecimalType(14, 4) tupleDesc, tupleBuilder := newTupleBuilderForEncoding(val.DecimalEnc) From e897fdf847de2e36a851e71e1ff0b1ae57b8add1 Mon Sep 17 00:00:00 2001 From: fulghum Date: Mon, 6 May 2024 18:58:38 +0000 Subject: [PATCH 3/6] [ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/update.sh --- .../sqle/binlogreplication/binlog_type_serialization.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/go/libraries/doltcore/sqle/binlogreplication/binlog_type_serialization.go b/go/libraries/doltcore/sqle/binlogreplication/binlog_type_serialization.go index dd83fe18919..314908e905c 100644 --- a/go/libraries/doltcore/sqle/binlogreplication/binlog_type_serialization.go +++ b/go/libraries/doltcore/sqle/binlogreplication/binlog_type_serialization.go @@ -21,13 +21,14 @@ import ( "strconv" "strings" - "github.com/dolthub/dolt/go/store/hash" - "github.com/dolthub/dolt/go/store/prolly/tree" - "github.com/dolthub/dolt/go/store/val" "github.com/dolthub/go-mysql-server/sql" gmstypes "github.com/dolthub/go-mysql-server/sql/types" "github.com/dolthub/vitess/go/mysql" "github.com/dolthub/vitess/go/vt/proto/query" + + "github.com/dolthub/dolt/go/store/hash" + "github.com/dolthub/dolt/go/store/prolly/tree" + "github.com/dolthub/dolt/go/store/val" ) // typeSerializer defines the serialization interface for serializing a value in Dolt's From 83d5aac1fb3544c2b00ee7fec541980434ce02e8 Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Mon, 6 May 2024 13:26:56 -0700 Subject: [PATCH 4/6] Update go/libraries/doltcore/sqle/binlogreplication/binlog_type_serialization.go --- .../sqle/binlogreplication/binlog_type_serialization.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go/libraries/doltcore/sqle/binlogreplication/binlog_type_serialization.go b/go/libraries/doltcore/sqle/binlogreplication/binlog_type_serialization.go index 314908e905c..e8984ce027f 100644 --- a/go/libraries/doltcore/sqle/binlogreplication/binlog_type_serialization.go +++ b/go/libraries/doltcore/sqle/binlogreplication/binlog_type_serialization.go @@ -292,6 +292,8 @@ func (d decimalSerializer) serialize(_ *sql.Context, typ sql.Type, descriptor va decimalValue.Exponent(), decimalValue.String()) } + // Load the value into a fully padded (to precision and scale) string format, + // so that we can process the digit groups for the binary encoding. absStringVal := decimalValue.Abs().StringFixed(int32(scale)) stringIntegerVal := absStringVal stringFractionalVal := "" From 9d9eaf26c5db165c9caeb13fa15bf90d90089d37 Mon Sep 17 00:00:00 2001 From: fulghum Date: Mon, 6 May 2024 20:33:50 +0000 Subject: [PATCH 5/6] [ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/update.sh --- .../sqle/binlogreplication/binlog_type_serialization.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/libraries/doltcore/sqle/binlogreplication/binlog_type_serialization.go b/go/libraries/doltcore/sqle/binlogreplication/binlog_type_serialization.go index e8984ce027f..db76d42e8eb 100644 --- a/go/libraries/doltcore/sqle/binlogreplication/binlog_type_serialization.go +++ b/go/libraries/doltcore/sqle/binlogreplication/binlog_type_serialization.go @@ -292,7 +292,7 @@ func (d decimalSerializer) serialize(_ *sql.Context, typ sql.Type, descriptor va decimalValue.Exponent(), decimalValue.String()) } - // Load the value into a fully padded (to precision and scale) string format, + // Load the value into a fully padded (to precision and scale) string format, // so that we can process the digit groups for the binary encoding. absStringVal := decimalValue.Abs().StringFixed(int32(scale)) stringIntegerVal := absStringVal From a9e8fc94eded694c8d1004c249993b8dc6fc7b11 Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Mon, 6 May 2024 09:15:13 -0700 Subject: [PATCH 6/6] Making `dolt init` work with the --data-dir param --- go/cmd/dolt/commands/config.go | 7 ++++++- go/cmd/dolt/commands/config_test.go | 6 +++++- go/libraries/doltcore/env/config.go | 7 ++++--- go/libraries/doltcore/env/environment.go | 6 +++++- go/libraries/doltcore/env/environment_test.go | 5 ++++- integration-tests/bats/init.bats | 21 +++++++++++++++++++ 6 files changed, 45 insertions(+), 7 deletions(-) diff --git a/go/cmd/dolt/commands/config.go b/go/cmd/dolt/commands/config.go index 192d54dae09..3aa87d268ba 100644 --- a/go/cmd/dolt/commands/config.go +++ b/go/cmd/dolt/commands/config.go @@ -233,7 +233,12 @@ func addOperation(dEnv *env.DoltEnv, setCfgTypes *set.StrSet, args []string, usa case globalParamName: panic("Should not have been able to get this far without a global config.") case localParamName: - err := dEnv.Config.CreateLocalConfig(updates) + configDir, err := dEnv.FS.Abs(".") + if err != nil { + cli.PrintErrln(color.RedString("Unable to resolve current path to create repo local config file")) + return 1 + } + err = dEnv.Config.CreateLocalConfig(configDir, updates) if err != nil { cli.PrintErrln(color.RedString("Unable to create repo local config file")) return 1 diff --git a/go/cmd/dolt/commands/config_test.go b/go/cmd/dolt/commands/config_test.go index 03c97428fc2..0bef1fa8464 100644 --- a/go/cmd/dolt/commands/config_test.go +++ b/go/cmd/dolt/commands/config_test.go @@ -36,7 +36,11 @@ func initializeConfigs(dEnv *env.DoltEnv, element env.ConfigScope) { globalCfg, _ := dEnv.Config.GetConfig(env.GlobalConfig) globalCfg.SetStrings(map[string]string{"title": "senior dufus"}) case env.LocalConfig: - dEnv.Config.CreateLocalConfig(map[string]string{"title": "senior dufus"}) + configDir, err := dEnv.FS.Abs(".") + if err != nil { + panic("Unable to resolve current path to create repo local config file: " + err.Error()) + } + dEnv.Config.CreateLocalConfig(configDir, map[string]string{"title": "senior dufus"}) } } func TestConfigAdd(t *testing.T) { diff --git a/go/libraries/doltcore/env/config.go b/go/libraries/doltcore/env/config.go index 7f7645c22a1..d26d5c01614 100644 --- a/go/libraries/doltcore/env/config.go +++ b/go/libraries/doltcore/env/config.go @@ -111,10 +111,11 @@ func ensureGlobalConfig(path string, fs filesys.ReadWriteFS) (config.ReadWriteCo return config.NewFileConfig(path, fs, map[string]string{}) } -// CreateLocalConfig creates a new repository local config file. The current directory must have already been initialized +// CreateLocalConfig creates a new repository local config file with the values from |val| +// at the directory |dir|. The |dir| directory must have already been initialized // as a data repository before a local config can be created. -func (dcc *DoltCliConfig) CreateLocalConfig(vals map[string]string) error { - return dcc.createLocalConfigAt(".", vals) +func (dcc *DoltCliConfig) CreateLocalConfig(dir string, vals map[string]string) error { + return dcc.createLocalConfigAt(dir, vals) } func (dcc *DoltCliConfig) createLocalConfigAt(dir string, vals map[string]string) error { diff --git a/go/libraries/doltcore/env/environment.go b/go/libraries/doltcore/env/environment.go index 62ef6aef95e..44eecc1e454 100644 --- a/go/libraries/doltcore/env/environment.go +++ b/go/libraries/doltcore/env/environment.go @@ -457,8 +457,12 @@ func (dEnv *DoltEnv) createDirectories(dir string) (string, error) { } func (dEnv *DoltEnv) configureRepo(doltDir string) error { - err := dEnv.Config.CreateLocalConfig(map[string]string{}) + configDir, err := dEnv.FS.Abs(".") + if err != nil { + return fmt.Errorf("unable to resolve current path to create repo local config file: %s", err.Error()) + } + err = dEnv.Config.CreateLocalConfig(configDir, map[string]string{}) if err != nil { return fmt.Errorf("failed creating file %s", getLocalConfigPath()) } diff --git a/go/libraries/doltcore/env/environment_test.go b/go/libraries/doltcore/env/environment_test.go index c62b0c07266..fcc942607bc 100644 --- a/go/libraries/doltcore/env/environment_test.go +++ b/go/libraries/doltcore/env/environment_test.go @@ -129,7 +129,10 @@ func TestRepoDirNoLocal(t *testing.T) { require.NoError(t, dEnv.CfgLoadErr) // RSLoadErr will be set because the above method of creating the repo doesn't initialize a valid working or staged - err := dEnv.Config.CreateLocalConfig(map[string]string{"user.name": "bheni"}) + configDir, err := dEnv.FS.Abs(".") + require.NoError(t, err) + + err = dEnv.Config.CreateLocalConfig(configDir, map[string]string{"user.name": "bheni"}) require.NoError(t, err) if !dEnv.HasLocalConfig() { diff --git a/integration-tests/bats/init.bats b/integration-tests/bats/init.bats index e9ce1dc5068..0ffc53e7286 100644 --- a/integration-tests/bats/init.bats +++ b/integration-tests/bats/init.bats @@ -277,6 +277,27 @@ teardown() { [[ $output =~ "commit dolt" ]] || [[ $output =~ "commit do1t" ]] || [[ $output =~ "commit d0lt" ]] || [[ $output =~ "commit d01t" ]] || false } +@test "init: initialize a non-working directory with --data-dir" { + baseDir=$(pwd) + mkdir not_a_repo + mkdir repo_dir + cd not_a_repo + dolt --data-dir $baseDir/repo_dir init + + # Assert that the current directory has NOT been initialized + run dolt status + [ $status -eq 1 ] + [[ $output =~ "The current directory is not a valid dolt repository" ]] || false + [ ! -d "$baseDir/not_a_repo/.dolt" ] + + # Assert that ../repo_dir HAS been initialized + cd $baseDir/repo_dir + run dolt status + [ $status -eq 0 ] + [[ $output =~ "On branch main" ]] || false + [ -d "$baseDir/repo_dir/.dolt" ] +} + assert_valid_repository () { run dolt log [ "$status" -eq 0 ]