From 82398b526b2ab192afbcd3b438218bdf88bfbc8e Mon Sep 17 00:00:00 2001 From: WangXiangUSTC Date: Mon, 9 Sep 2019 14:30:20 +0800 Subject: [PATCH] remove binogctl (#266) --- Makefile | 9 +-- README.md | 6 -- tidb-binlog/binlogctl/README.md | 100 ------------------------ tidb-binlog/binlogctl/config.go | 121 ----------------------------- tidb-binlog/binlogctl/main.go | 66 ---------------- tidb-binlog/binlogctl/meta.go | 126 ------------------------------ tidb-binlog/binlogctl/nodes.go | 132 -------------------------------- 7 files changed, 3 insertions(+), 557 deletions(-) delete mode 100644 tidb-binlog/binlogctl/README.md delete mode 100644 tidb-binlog/binlogctl/config.go delete mode 100644 tidb-binlog/binlogctl/main.go delete mode 100644 tidb-binlog/binlogctl/meta.go delete mode 100644 tidb-binlog/binlogctl/nodes.go diff --git a/Makefile b/Makefile index 2ecde9627..86c17625c 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build importer dump_region binlogctl sync_diff_inspector ddl_checker test check deps +.PHONY: build importer dump_region sync_diff_inspector ddl_checker test check deps # Ensure GOPATH is set before running build process. ifeq "$(GOPATH)" "" @@ -25,9 +25,9 @@ PACKAGE_LIST := go list ./... PACKAGES := $$($(PACKAGE_LIST)) FAIL_ON_STDOUT := awk '{ print } END { if (NR > 0) { exit 1 } }' -build: prepare check importer binlogctl sync_diff_inspector ddl_checker finish +build: prepare check importer sync_diff_inspector ddl_checker finish -prepare: +prepare: cp go.mod1 go.mod cp go.sum1 go.sum @@ -37,9 +37,6 @@ importer: dump_region: $(GO) build -ldflags '$(LDFLAGS)' -o bin/dump_region ./dump_region -binlogctl: - $(GO) build -ldflags '$(LDFLAGS)' -o bin/binlogctl ./tidb-binlog/binlogctl - sync_diff_inspector: $(GO) build -ldflags '$(LDFLAGS)' -o bin/sync_diff_inspector ./sync_diff_inspector diff --git a/README.md b/README.md index 3d6238c63..ee8a0562d 100644 --- a/README.md +++ b/README.md @@ -11,8 +11,6 @@ make importer # build importer make sync_diff_inspector # build sync_diff_inspector -make binlogctl # build binlogctl - make ddl_checker # build ddl_checker ``` @@ -28,10 +26,6 @@ When tidb-tools are built successfully, you can find the binary in the `bin` dir A tool for comparing two databases' data and outputting a brief report about the differences. -- [binlogctl](./tidb-binlog/binlogctl) - - A tool for performing some tidb-binlog related operations, like querying the status of Pump/Drainer and pause/offline some Pump/Drainer. - - [ddl_checker](./ddl_checker) A tool for checking if DDL SQL can be successfully executed by TiDB. diff --git a/tidb-binlog/binlogctl/README.md b/tidb-binlog/binlogctl/README.md deleted file mode 100644 index 05baebdcc..000000000 --- a/tidb-binlog/binlogctl/README.md +++ /dev/null @@ -1,100 +0,0 @@ -# binlogctl - -binlogctl is a tool for performing some tidb-binlog related operations, like querying the status of Pump/Drainer and unregistering some Pump/Drainer. - -## How to use - -``` -Usage of binlogctl: - -V prints version and exit - -cmd string - operator: "generate_meta", "pumps", "drainers", "update-pump", "update-drainer", "pause-pump", "pause-drainer", "offline-pump", "offline-drainer" (default "pumps") - -data-dir string - meta directory path (default "binlog_position") - -node-id string - id of node, used to delete some node with operations delete-pump and delete-drainer - -pd-urls string - a comma separated list of PD endpoints (default "http://127.0.0.1:2379") - -ssl-ca string - Path of file that contains the list of trusted SSL CAs for connection with cluster components - -ssl-cert string - Path of file that contains X509 certificate in PEM format for connection with cluster components - -ssl-key string - Path of file that contains X509 key in PEM format for connection with cluster components - -time-zone Asia/Shanghai - set time zone if you want to save time info in the savepoint file, for example `Asia/Shanghai` for CST time and `Local` for the local time -``` - -## Download Binary (CentOS 7+ platform) - -```bash -# Download the tool package. -wget http://download.pingcap.org/tidb-tools-latest-linux-amd64.tar.gz -wget http://download.pingcap.org/tidb-tools-latest-linux-amd64.sha256 - -# Check the file integrity. If the result is OK, the file is correct. -sha256sum -c tidb-tools-latest-linux-amd64.sha256 - -# Extract the package. -tar -xzf tidb-tools-latest-linux-amd64.tar.gz -cd tidb-tools-latest-linux-amd64 -``` - -## Example - -### Query Pump/Drainer - -Run the following command: - -``` -bin/binlogctl -pd-urls=http://127.0.0.1:2379 -cmd pumps -bin/binlogctl -pd-urls=http://127.0.0.1:2379 -cmd drainers -``` - -Then the result will be like this (the output will be formatted later): - -``` -2018/06/21 11:24:10 nodes.go:53: [info] pump: &{NodeID:ip-192-168-199-118:8250 Host:127.0.0.1:8250 IsAlive:true IsOffline:false LatestFilePos:{Suffix:0 Offset:15320} LatestKafkaPos:{Suffix:0 Offset:382} OfflineTS:0} -``` - -### Unregister Pump/Drainer - -### update pump/drainer's state -pump/drainer's state can be online, pausing, paused, closing and offline. In most cases, we only need update pump/drainer's state to paused or offline. -``` -bin/binlogctl -pd-urls=http://127.0.0.1:2379 -cmd update-pump/update-drainer -node-id ip-127-0-0-1:8250/{nodeID} -state {state} -``` -This cmd will update pump/drainer's state. - -### pause/offline pump/drainer -``` -bin/binlogctl -pd-urls=http://127.0.0.1:2379 -cmd pause-pump/pause-drainer/offline-pump/offline-drainer -node-id ip-127-0-0-1:8250/{nodeID} -``` -binlogctl will send http request to pump/drainer, and finally pump/drainer will exit by itself with paused or offline state. - -### Generate `meta` - -`meta` contains commit TS that can be used to specify the location of the synchronized data. - -Run the following command: - -``` -bin/binlogctl -pd-urls=http://127.0.0.1:2379 -cmd generate_meta -``` - -Then the result will be like this: - -``` -INFO[0000] [pd] create pd client with endpoints [http://192.168.199.118:32379] -INFO[0000] [pd] leader switches to: http://192.168.199.118:32379, previous: -INFO[0000] [pd] init cluster id 6569368151110378289 -2018/06/21 11:24:47 meta.go:117: [info] meta: &{CommitTS:400962745252184065} -``` - -It will also generate a `{data-dir}/savepoint` meta file - -TODO: improve `meta` later, like adding offset of the Kafka topic that corresponds to each Pump node - -## License - -Apache 2.0 license. See the [LICENSE](../LICENSE) file for details. diff --git a/tidb-binlog/binlogctl/config.go b/tidb-binlog/binlogctl/config.go deleted file mode 100644 index 45899cd18..000000000 --- a/tidb-binlog/binlogctl/config.go +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright 2018 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "crypto/tls" - "flag" - "fmt" - - "github.com/pingcap/errors" - "github.com/pingcap/tidb-tools/pkg/utils" -) - -const ( - defaultEtcdURLs = "http://127.0.0.1:2379" - defaultDataDir = "binlog_position" -) - -const ( - generateMeta = "generate_meta" - queryPumps = "pumps" - queryDrainers = "drainers" - updatePump = "update-pump" - updateDrainer = "update-drainer" - pausePump = "pause-pump" - offlinePump = "offline-pump" - pauseDrainer = "pause-drainer" - offlineDrainer = "offline-drainer" -) - -// Config holds the configuration of drainer -type Config struct { - *flag.FlagSet - - Command string `toml:"cmd" json:"cmd"` - NodeID string `toml:"node-id" json:"node-id"` - DataDir string `toml:"data-dir" json:"data-dir"` - TimeZone string `toml:"time-zone" json:"time-zone"` - EtcdURLs string `toml:"pd-urls" json:"pd-urls"` - SSLCA string `toml:"ssl-ca" json:"ssl-ca"` - SSLCert string `toml:"ssl-cert" json:"ssl-cert"` - SSLKey string `toml:"ssl-key" json:"ssl-key"` - State string `toml:"state" json:"state"` - tls *tls.Config - printVersion bool -} - -// NewConfig returns an instance of configuration -func NewConfig() *Config { - cfg := &Config{} - cfg.FlagSet = flag.NewFlagSet("binlogctl", flag.ContinueOnError) - - cfg.FlagSet.StringVar(&cfg.Command, "cmd", "pumps", "operator: \"generate_meta\", \"pumps\", \"drainers\", \"update-pump\", \"update-drainer\", \"pause-pump\", \"pause-drainer\", \"offline-pump\", \"offline-drainer\"") - cfg.FlagSet.StringVar(&cfg.NodeID, "node-id", "", "id of node, use to update some node with operation update-pump, update-drainer, pause-pump, pause-drainer, offline-pump and offline-drainer") - cfg.FlagSet.StringVar(&cfg.DataDir, "data-dir", defaultDataDir, "meta directory path") - cfg.FlagSet.StringVar(&cfg.EtcdURLs, "pd-urls", defaultEtcdURLs, "a comma separated list of PD endpoints") - cfg.FlagSet.StringVar(&cfg.SSLCA, "ssl-ca", "", "Path of file that contains list of trusted SSL CAs for connection with cluster components.") - cfg.FlagSet.StringVar(&cfg.SSLCert, "ssl-cert", "", "Path of file that contains X509 certificate in PEM format for connection with cluster components.") - cfg.FlagSet.StringVar(&cfg.SSLKey, "ssl-key", "", "Path of file that contains X509 key in PEM format for connection with cluster components.") - cfg.FlagSet.StringVar(&cfg.TimeZone, "time-zone", "", "set time zone if you want save time info in savepoint file, for example `Asia/Shanghai` for CST time, `Local` for local time") - cfg.FlagSet.StringVar(&cfg.State, "state", "", "set node's state, can set to online, pausing, paused, closing or offline.") - cfg.FlagSet.BoolVar(&cfg.printVersion, "V", false, "prints version and exit") - - return cfg -} - -// Parse parses all config from command-line flags, environment vars or the configuration file -func (cfg *Config) Parse(args []string) error { - // parse first to get config file - err := cfg.FlagSet.Parse(args) - if err != nil { - return errors.Trace(err) - } - // parse command line options - if len(cfg.FlagSet.Args()) > 0 { - return errors.Errorf("'%s' is not a valid flag", cfg.FlagSet.Arg(0)) - } - - if cfg.printVersion { - fmt.Printf(utils.GetRawInfo("binlogctl")) - return flag.ErrHelp - } - - // adjust configuration - adjustString(&cfg.DataDir, defaultDataDir) - - // transfore tls config - cfg.tls, err = utils.ToTLSConfig(cfg.SSLCA, cfg.SSLCert, cfg.SSLKey) - if err != nil { - return errors.Errorf("tls config error %v", err) - } - - return cfg.validate() -} - -func adjustString(v *string, defValue string) { - if len(*v) == 0 { - *v = defValue - } -} - -// validate checks whether the configuration is valid -func (cfg *Config) validate() error { - // check EtcdEndpoints - _, err := utils.ParseHostPortAddr(cfg.EtcdURLs) - if err != nil { - return errors.Errorf("parse EtcdURLs error: %s, %v", cfg.EtcdURLs, err) - } - return nil -} diff --git a/tidb-binlog/binlogctl/main.go b/tidb-binlog/binlogctl/main.go deleted file mode 100644 index 3c849017c..000000000 --- a/tidb-binlog/binlogctl/main.go +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright 2018 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "flag" - "os" - - "github.com/pingcap/log" - "github.com/pingcap/tidb-tools/tidb-binlog/node" - "go.uber.org/zap" -) - -const ( - pause = "pause" - close = "close" -) - -func main() { - cfg := NewConfig() - err := cfg.Parse(os.Args[1:]) - switch err { - case nil: - case flag.ErrHelp: - os.Exit(0) - default: - log.Error("parse cmd flags", zap.Error(err)) - os.Exit(2) - } - - switch cfg.Command { - case generateMeta: - err = generateMetaInfo(cfg) - case queryPumps: - err = queryNodesByKind(cfg.EtcdURLs, node.PumpNode) - case queryDrainers: - err = queryNodesByKind(cfg.EtcdURLs, node.DrainerNode) - case updatePump: - err = updateNodeState(cfg.EtcdURLs, node.PumpNode, cfg.NodeID, cfg.State) - case updateDrainer: - err = updateNodeState(cfg.EtcdURLs, node.DrainerNode, cfg.NodeID, cfg.State) - case pausePump: - err = applyAction(cfg.EtcdURLs, node.PumpNode, cfg.NodeID, pause) - case pauseDrainer: - err = applyAction(cfg.EtcdURLs, node.DrainerNode, cfg.NodeID, pause) - case offlinePump: - err = applyAction(cfg.EtcdURLs, node.PumpNode, cfg.NodeID, close) - case offlineDrainer: - err = applyAction(cfg.EtcdURLs, node.DrainerNode, cfg.NodeID, close) - } - - if err != nil { - log.Fatal("fail to execute command", zap.String("command", cfg.Command), zap.Error(err)) - } -} diff --git a/tidb-binlog/binlogctl/meta.go b/tidb-binlog/binlogctl/meta.go deleted file mode 100644 index fea557199..000000000 --- a/tidb-binlog/binlogctl/meta.go +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright 2018 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "bytes" - "context" - "fmt" - "os" - "path" - "time" - - "github.com/BurntSushi/toml" - "github.com/pingcap/errors" - "github.com/pingcap/log" - pd "github.com/pingcap/pd/client" - "github.com/pingcap/tidb-tools/pkg/utils" - "github.com/siddontang/go/ioutil2" - "go.uber.org/zap" -) - -const physicalShiftBits = 18 -const slowDist = 30 * time.Millisecond - -// generateMeta generates Meta from pd -func generateMetaInfo(cfg *Config) error { - if err := os.MkdirAll(cfg.DataDir, 0700); err != nil { - return errors.Trace(err) - } - - // get newest ts from pd - commitTS, err := GetTSO(cfg) - if err != nil { - log.Error("get tso failed", zap.Error(err)) - return errors.Trace(err) - } - - // generate meta file - metaFileName := path.Join(cfg.DataDir, "savepoint") - err = saveMeta(metaFileName, commitTS, cfg.TimeZone) - return errors.Trace(err) -} - -// GetTSO gets ts from pd -func GetTSO(cfg *Config) (int64, error) { - now := time.Now() - - ectdEndpoints, err := utils.ParseHostPortAddr(cfg.EtcdURLs) - if err != nil { - return 0, errors.Trace(err) - } - - pdCli, err := pd.NewClient(ectdEndpoints, pd.SecurityOption{ - CAPath: cfg.SSLCA, - CertPath: cfg.SSLCert, - KeyPath: cfg.SSLKey, - }) - physical, logical, err := pdCli.GetTS(context.Background()) - if err != nil { - return 0, errors.Trace(err) - } - dist := time.Since(now) - if dist > slowDist { - log.Warn("get timestamp too slow", zap.Duration("dist", dist)) - } - - return int64(composeTS(physical, logical)), nil -} - -func composeTS(physical, logical int64) uint64 { - return uint64((physical << physicalShiftBits) + logical) -} - -// Meta contains commit TS that can be used to specifies the location of the synchronized data -// TODO: improve meta later, like adding offset of kafka topic that corresponds to each pump node -type Meta struct { - CommitTS int64 `toml:"commitTS" json:"commitTS"` -} - -// String returns the string of Meta -func (m *Meta) String() string { - return fmt.Sprintf("commitTS: %d", m.CommitTS) -} - -// saveMeta saves current tso in meta file. -func saveMeta(metaFileName string, ts int64, timeZone string) error { - meta := &Meta{CommitTS: ts} - - var buf bytes.Buffer - e := toml.NewEncoder(&buf) - err := e.Encode(meta) - if err != nil { - return errors.Annotatef(err, "save meta %+v into %s", meta, metaFileName) - } - - if timeZone != "" { - t := utils.TSOToRoughTime(ts) - location, err1 := time.LoadLocation(timeZone) - if err1 != nil { - log.Warn("fail to load location", zap.String("time zone", timeZone), zap.Error(err1)) - } else { - buf.WriteString(t.UTC().String()) - buf.WriteByte('\n') - buf.WriteString(t.In(location).String()) - } - } - - err = ioutil2.WriteFileAtomic(metaFileName, buf.Bytes(), 0644) - if err != nil { - return errors.Annotatef(err, "save meta %+v into %s", meta, metaFileName) - } - - log.Info("save meta", zap.Stringer("meta", meta)) - return nil -} diff --git a/tidb-binlog/binlogctl/nodes.go b/tidb-binlog/binlogctl/nodes.go deleted file mode 100644 index bfb4fb00b..000000000 --- a/tidb-binlog/binlogctl/nodes.go +++ /dev/null @@ -1,132 +0,0 @@ -// Copyright 2018 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "context" - "fmt" - "net/http" - "time" - - "github.com/pingcap/errors" - "github.com/pingcap/log" - "github.com/pingcap/tidb-tools/pkg/etcd" - "github.com/pingcap/tidb-tools/pkg/utils" - "github.com/pingcap/tidb-tools/tidb-binlog/node" - "go.uber.org/zap" -) - -var ( - etcdDialTimeout = 5 * time.Second -) - -// queryNodesByKind returns specified nodes, like pumps/drainers -func queryNodesByKind(urls string, kind string) error { - registry, err := createRegistry(urls) - if err != nil { - return errors.Trace(err) - } - - nodes, _, err := registry.Nodes(context.Background(), node.NodePrefix[kind]) - if err != nil { - return errors.Trace(err) - } - - for _, n := range nodes { - log.Info("query node", zap.String("type", kind), zap.Stringer("node", n)) - } - - return nil -} - -// updateNodeState update pump or drainer's state. -func updateNodeState(urls, kind, nodeID, state string) error { - /* - node's state can be online, pausing, paused, closing and offline. - if the state is one of them, will update the node's state saved in etcd directly. - */ - registry, err := createRegistry(urls) - if err != nil { - return errors.Trace(err) - } - - nodes, _, err := registry.Nodes(context.Background(), node.NodePrefix[kind]) - if err != nil { - return errors.Trace(err) - } - - for _, n := range nodes { - if n.NodeID != nodeID { - continue - } - switch state { - case node.Online, node.Pausing, node.Paused, node.Closing, node.Offline: - n.State = state - return registry.UpdateNode(context.Background(), node.NodePrefix[kind], n) - default: - return errors.Errorf("state %s is illegal", state) - } - } - - return errors.NotFoundf("node %s, id %s from etcd %s", kind, nodeID, urls) -} - -// createRegistry returns an ectd registry -func createRegistry(urls string) (*node.EtcdRegistry, error) { - ectdEndpoints, err := utils.ParseHostPortAddr(urls) - if err != nil { - return nil, errors.Trace(err) - } - cli, err := etcd.NewClientFromCfg(ectdEndpoints, etcdDialTimeout, node.DefaultRootPath, nil) - if err != nil { - return nil, errors.Trace(err) - } - - return node.NewEtcdRegistry(cli, etcdDialTimeout), nil -} - -func applyAction(urls, kind, nodeID string, action string) error { - registry, err := createRegistry(urls) - if err != nil { - return errors.Trace(err) - } - - nodes, _, err := registry.Nodes(context.Background(), node.NodePrefix[kind]) - if err != nil { - return errors.Trace(err) - } - - for _, n := range nodes { - if n.NodeID != nodeID { - continue - } - - client := &http.Client{} - url := fmt.Sprintf("http://%s/state/%s/%s", n.Addr, n.NodeID, action) - log.Debug("send put http request", zap.String("url", url)) - req, err := http.NewRequest("PUT", url, nil) - if err != nil { - return errors.Trace(err) - } - _, err = client.Do(req) - if err == nil { - log.Info("apply action on node success", zap.String("action", action), zap.String("NodeID", n.NodeID)) - return nil - } - - return errors.Trace(err) - } - - return errors.NotFoundf("nodeID %s", nodeID) -}