Add go mod

build.sh

@@ -23,11 +23,10 @@ function build {
     exit 1
   fi
 
-  # TODO: remove GO111MODULE once gh-ost uses Go modules
   echo "Building ${osname} binary"
   export GOOS
   export GOARCH
-  GO111MODULE=off go build -ldflags "$ldflags" -o $buildpath/$target go/cmd/gh-ost/main.go
+  go build -ldflags "$ldflags" -o $buildpath/$target go/cmd/gh-ost/main.go
 
   if [ $? -ne 0 ]; then
       echo "Build failed for ${osname}"

go.mod

@@ -0,0 +1,18 @@
+module github.com/github/gh-ost
+
+go 1.16
+
+require (
+	github.com/go-ini/ini v1.62.0
+	github.com/go-sql-driver/mysql v1.5.0
+	github.com/openark/golib v0.0.0-20210531070646-355f37940af8
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/satori/go.uuid v1.2.0
+	github.com/siddontang/go-mysql v1.1.0
+	github.com/smartystreets/goconvey v1.6.4 // indirect
+	golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83
+	golang.org/x/net v0.0.0-20210224082022-3d97a244fca7
+	golang.org/x/text v0.3.5
+	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
+	gopkg.in/ini.v1 v1.62.0 // indirect
+)

go.sum

@@ -0,0 +1,72 @@
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/go-ini/ini v1.62.0 h1:7VJT/ZXjzqSrvtraFp4ONq80hTcRQth1c9ZnQ3uNQvU=
+github.com/go-ini/ini v1.62.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
+github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
+github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
+github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/openark/golib v0.0.0-20210531070646-355f37940af8 h1:9ciIHNuyFqRWi9NpMNw9sVLB6z1ItpP5ZhTY9Q1xVu4=
+github.com/openark/golib v0.0.0-20210531070646-355f37940af8/go.mod h1:1jj8x1eDVZxgc/Z4VyamX4qTbAdHPUQA6NeVtCd8Sl8=
+github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8 h1:USx2/E1bX46VG32FIw034Au6seQ2fY9NEILmNh/UlQg=
+github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ=
+github.com/pingcap/errors v0.11.0 h1:DCJQB8jrHbQ1VVlMFIrbj2ApScNNotVmkSNplu2yUt4=
+github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
+github.com/pingcap/parser v0.0.0-20190506092653-e336082eb825/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA=
+github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
+github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
+github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 h1:pntxY8Ary0t43dCZ5dqY4YTJCObLY1kIXl0uzMv+7DE=
+github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
+github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 h1:xT+JlYxNGqyT+XcU8iUrN18JYed2TvG9yN5ULG2jATM=
+github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726/go.mod h1:3yhqj7WBBfRhbBlzyOC3gUxftwsU0u8gqevxwIHQpMw=
+github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 h1:oI+RNwuC9jF2g2lP0u0cVEEZrc/AYBCuFdvwrLWM/6Q=
+github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07/go.mod h1:yFdBgwXP24JziuRl2NMUahT7nGLNOKi1SIiFxMttVD4=
+github.com/siddontang/go-mysql v1.1.0 h1:NfkS1skrPwUd3hsUqhc6jrv24dKTNMANxKRmDsf1fMc=
+github.com/siddontang/go-mysql v1.1.0/go.mod h1:+W4RCzesQDI11HvIkaDjS8yM36SpAnGNQ7jmTLn5BnU=
+github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
+github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g=
+golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 h1:OgUuv8lsRpBibGNbSizVwKWlysjaNzmC9gYMhPVfqFM=
+golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ=
+golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU=
+gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=

go/base/context.go

@@ -19,10 +19,9 @@ import (
 
 	"github.com/github/gh-ost/go/mysql"
 	"github.com/github/gh-ost/go/sql"
-	"github.com/outbrain/golib/log"
+	"github.com/openark/golib/log"
 
-	"gopkg.in/gcfg.v1"
-	gcfgscanner "gopkg.in/gcfg.v1/scanner"
+	"github.com/go-ini/ini"
 )
 
 // RowsEstimateMethod is the type of row number estimation
@@ -807,10 +806,39 @@ func (this *MigrationContext) ReadConfigFile() error {
 	if this.ConfigFile == "" {
 		return nil
 	}
-	gcfg.RelaxedParserMode = true
-	gcfgscanner.RelaxedScannerMode = true
-	if err := gcfg.ReadFileInto(&this.config, this.ConfigFile); err != nil {
-		return fmt.Errorf("Error reading config file %s. Details: %s", this.ConfigFile, err.Error())
+	cfg, err := ini.Load(this.ConfigFile)
+	if err != nil {
+		return err
+	}
+
+	if cfg.Section("client").Haskey("user") {
+		this.config.Client.User = cfg.Section("client").Key("user").String()
+	}
+
+	if cfg.Section("client").Haskey("password") {
+		this.config.Client.Password = cfg.Section("client").Key("password").String()
+	}
+
+	if cfg.Section("osc").Haskey("chunk_size") {
+		this.config.Osc.Chunk_Size, err = cfg.Section("osc").Key("chunk_size").Int64()
+		if err != nil {
+			return fmt.Errorf("Unable to read osc chunk size: %s", err.Error())
+		}
+	}
+
+	if cfg.Section("osc").Haskey("max_load") {
+		this.config.Osc.Max_Load = cfg.Section("osc").Key("max_load").String()
+	}
+
+	if cfg.Section("osc").Haskey("replication_lag_query") {
+		this.config.Osc.Replication_Lag_Query = cfg.Section("osc").Key("replication_lag_query").String()
+	}
+
+	if cfg.Section("osc").Haskey("max_lag_millis") {
+		this.config.Osc.Max_Lag_Millis, err = cfg.Section("osc").Key("max_lag_millis").Int64()
+		if err != nil {
+			return fmt.Errorf("Unable to read max lag millis: %s", err.Error())
+		}
 	}
 
 	// We accept user & password in the form "${SOME_ENV_VARIABLE}" in which case we pull

go/base/context_test.go

@@ -1,16 +1,19 @@
 /*
-   Copyright 2016 GitHub Inc.
+   Copyright 2021 GitHub Inc.
 	 See https://github.com/github/gh-ost/blob/master/LICENSE
 */
 
 package base
 
 import (
+	"fmt"
+	"io/ioutil"
+	"os"
 	"testing"
 	"time"
 
-	"github.com/outbrain/golib/log"
-	test "github.com/outbrain/golib/tests"
+	"github.com/openark/golib/log"
+	test "github.com/openark/golib/tests"
 )
 
 func init() {
@@ -56,3 +59,65 @@ func TestGetTableNames(t *testing.T) {
 		test.S(t).ExpectEquals(context.GetChangelogTableName(), "_tmp_ghc")
 	}
 }
+
+func TestReadConfigFile(t *testing.T) {
+	{
+		context := NewMigrationContext()
+		context.ConfigFile = "/does/not/exist"
+		if err := context.ReadConfigFile(); err == nil {
+			t.Fatal("Expected .ReadConfigFile() to return an error, got nil")
+		}
+	}
+	{
+		f, err := ioutil.TempFile("", t.Name())
+		if err != nil {
+			t.Fatalf("Failed to create tmp file: %v", err)
+		}
+		defer os.Remove(f.Name())
+
+		f.Write([]byte("[client]"))
+		context := NewMigrationContext()
+		context.ConfigFile = f.Name()
+		if err := context.ReadConfigFile(); err != nil {
+			t.Fatalf(".ReadConfigFile() failed: %v", err)
+		}
+	}
+	{
+		f, err := ioutil.TempFile("", t.Name())
+		if err != nil {
+			t.Fatalf("Failed to create tmp file: %v", err)
+		}
+		defer os.Remove(f.Name())
+
+		f.Write([]byte(fmt.Sprintf("[client]\nuser=test\npassword=123456")))
+		context := NewMigrationContext()
+		context.ConfigFile = f.Name()
+		if err := context.ReadConfigFile(); err != nil {
+			t.Fatalf(".ReadConfigFile() failed: %v", err)
+		}
+
+		if context.config.Client.User != "test" {
+			t.Fatalf("Expected client user %q, got %q", "test", context.config.Client.User)
+		} else if context.config.Client.Password != "123456" {
+			t.Fatalf("Expected client password %q, got %q", "123456", context.config.Client.Password)
+		}
+	}
+	{
+		f, err := ioutil.TempFile("", t.Name())
+		if err != nil {
+			t.Fatalf("Failed to create tmp file: %v", err)
+		}
+		defer os.Remove(f.Name())
+
+		f.Write([]byte(fmt.Sprintf("[osc]\nmax_load=10")))
+		context := NewMigrationContext()
+		context.ConfigFile = f.Name()
+		if err := context.ReadConfigFile(); err != nil {
+			t.Fatalf(".ReadConfigFile() failed: %v", err)
+		}
+
+		if context.config.Osc.Max_Load != "10" {
+			t.Fatalf("Expected osc 'max_load' %q, got %q", "10", context.config.Osc.Max_Load)
+		}
+	}
+}

go/base/default_logger.go

@@ -1,7 +1,7 @@
 package base
 
 import (
-	"github.com/outbrain/golib/log"
+	"github.com/openark/golib/log"
 )
 
 type simpleLogger struct{}

go/base/load_map_test.go

@@ -8,8 +8,8 @@ package base
 import (
 	"testing"
 
-	"github.com/outbrain/golib/log"
-	test "github.com/outbrain/golib/tests"
+	"github.com/openark/golib/log"
+	test "github.com/openark/golib/tests"
 )
 
 func init() {

go/base/utils_test.go

@@ -8,8 +8,8 @@ package base
 import (
 	"testing"
 
-	"github.com/outbrain/golib/log"
-	test "github.com/outbrain/golib/tests"
+	"github.com/openark/golib/log"
+	test "github.com/openark/golib/tests"
 )
 
 func init() {

go/cmd/gh-ost/main.go

@@ -16,7 +16,7 @@ import (
 	"github.com/github/gh-ost/go/logic"
 	"github.com/github/gh-ost/go/sql"
 	_ "github.com/go-sql-driver/mysql"
-	"github.com/outbrain/golib/log"
+	"github.com/openark/golib/log"
 
 	"golang.org/x/crypto/ssh/terminal"
 )

go/logic/applier.go

@@ -16,7 +16,7 @@ import (
 	"github.com/github/gh-ost/go/mysql"
 	"github.com/github/gh-ost/go/sql"
 
-	"github.com/outbrain/golib/sqlutils"
+	"github.com/openark/golib/sqlutils"
 	"sync"
 )
 

go/logic/hooks.go

@@ -14,7 +14,7 @@ import (
 	"sync/atomic"
 
 	"github.com/github/gh-ost/go/base"
-	"github.com/outbrain/golib/log"
+	"github.com/openark/golib/log"
 )
 
 const (

go/logic/inspect.go

@@ -17,7 +17,7 @@ import (
 	"github.com/github/gh-ost/go/mysql"
 	"github.com/github/gh-ost/go/sql"
 
-	"github.com/outbrain/golib/sqlutils"
+	"github.com/openark/golib/sqlutils"
 )
 
 const startSlavePostWaitMilliseconds = 500 * time.Millisecond

go/logic/streamer.go

@@ -16,7 +16,7 @@ import (
 	"github.com/github/gh-ost/go/binlog"
 	"github.com/github/gh-ost/go/mysql"
 
-	"github.com/outbrain/golib/sqlutils"
+	"github.com/openark/golib/sqlutils"
 )
 
 type BinlogEventListener struct {

go/mysql/binlog_test.go

@@ -8,8 +8,8 @@ package mysql
 import (
 	"testing"
 
-	"github.com/outbrain/golib/log"
-	test "github.com/outbrain/golib/tests"
+	"github.com/openark/golib/log"
+	test "github.com/openark/golib/tests"
 )
 
 func init() {

go/mysql/connection_test.go

@@ -9,8 +9,8 @@ import (
 	"crypto/tls"
 	"testing"
 
-	"github.com/outbrain/golib/log"
-	test "github.com/outbrain/golib/tests"
+	"github.com/openark/golib/log"
+	test "github.com/openark/golib/tests"
 )
 
 func init() {

go/mysql/instance_key_test.go

@@ -8,8 +8,8 @@ package mysql
 import (
 	"testing"
 
-	"github.com/outbrain/golib/log"
-	test "github.com/outbrain/golib/tests"
+	"github.com/openark/golib/log"
+	test "github.com/openark/golib/tests"
 )
 
 func init() {

go/mysql/utils.go

@@ -14,8 +14,8 @@ import (
 
 	"github.com/github/gh-ost/go/sql"
 
-	"github.com/outbrain/golib/log"
-	"github.com/outbrain/golib/sqlutils"
+	"github.com/openark/golib/log"
+	"github.com/openark/golib/sqlutils"
 )
 
 const (

go/os/process.go

@@ -17,7 +17,7 @@
 package os
 
 import (
-	"github.com/outbrain/golib/log"
+	"github.com/openark/golib/log"
 	"io/ioutil"
 	"os"
 	"os/exec"

go/sql/builder_test.go

@@ -12,8 +12,8 @@ import (
 	"regexp"
 	"strings"
 
-	"github.com/outbrain/golib/log"
-	test "github.com/outbrain/golib/tests"
+	"github.com/openark/golib/log"
+	test "github.com/openark/golib/tests"
 )
 
 var (