From 8412fe81d4a5652dbe705d291ff72f6981c9e4a9 Mon Sep 17 00:00:00 2001 From: Florent Poinsard <35779988+frouioui@users.noreply.github.com> Date: Fri, 24 Jan 2025 11:49:50 -0600 Subject: [PATCH 01/11] Update the how to release java docs (#17603) Signed-off-by: Florent Poinsard --- doc/internal/release/how-to-release.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/internal/release/how-to-release.md b/doc/internal/release/how-to-release.md index 08411f9c0ac..97e58a074d7 100644 --- a/doc/internal/release/how-to-release.md +++ b/doc/internal/release/how-to-release.md @@ -364,7 +364,8 @@ You will need administrator privileges on the vitess repository to be able to ma ```bash cd ./java/ - mvn clean deploy -P release -DskipTests + # For <= v21.0, we must use -DskipTests in the mvn command below + mvn clean deploy -P release cd .. ``` From 5265e510f2bc34c4236729f45f843486dba9cfe7 Mon Sep 17 00:00:00 2001 From: Dirkjan Bussink Date: Mon, 27 Jan 2025 09:59:24 +0100 Subject: [PATCH 02/11] Remove deprecated syntax (#17631) Signed-off-by: Dirkjan Bussink --- .../src/main/java/io/vitess/example/MysqlJDBCExample.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/example/src/main/java/io/vitess/example/MysqlJDBCExample.java b/java/example/src/main/java/io/vitess/example/MysqlJDBCExample.java index 05513eaa2bd..85717fb20f0 100644 --- a/java/example/src/main/java/io/vitess/example/MysqlJDBCExample.java +++ b/java/example/src/main/java/io/vitess/example/MysqlJDBCExample.java @@ -109,7 +109,7 @@ private static void readData(Connection conn) throws SQLException { } private static void validateReplica(Connection conn) throws SQLException { - String sql = "show slave status"; + String sql = "show replica status"; try (Statement stmt = conn.createStatement(); ResultSet rs = stmt.executeQuery(sql)) { if (!rs.next()) { throw new RuntimeException("connected to wrong tablet"); From 44e46edf207aa7df53bea859d343aaaba86048b0 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Mon, 27 Jan 2025 11:25:51 +0200 Subject: [PATCH 03/11] `schemadiff`: validate uniqueness of `CHECK` and of `FOREIGN KEY` constraints (#17627) Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/vt/schemadiff/errors.go | 18 +++++++++ go/vt/schemadiff/schema.go | 21 +++++++++++ go/vt/schemadiff/schema_diff_test.go | 4 ++ go/vt/schemadiff/schema_test.go | 56 ++++++++++++++++++++++++++-- 4 files changed, 95 insertions(+), 4 deletions(-) diff --git a/go/vt/schemadiff/errors.go b/go/vt/schemadiff/errors.go index c938e736206..74d1c231a19 100644 --- a/go/vt/schemadiff/errors.go +++ b/go/vt/schemadiff/errors.go @@ -497,3 +497,21 @@ type NonDeterministicDefaultError struct { func (e *NonDeterministicDefaultError) Error() string { return fmt.Sprintf("column %s.%s default value uses non-deterministic function: %s", sqlescape.EscapeID(e.Table), sqlescape.EscapeID(e.Column), e.Function) } + +type DuplicateCheckConstraintNameError struct { + Table string + Constraint string +} + +func (e *DuplicateCheckConstraintNameError) Error() string { + return fmt.Sprintf("duplicate check constraint name %s in table %s", sqlescape.EscapeID(e.Constraint), sqlescape.EscapeID(e.Table)) +} + +type DuplicateForeignKeyConstraintNameError struct { + Table string + Constraint string +} + +func (e *DuplicateForeignKeyConstraintNameError) Error() string { + return fmt.Sprintf("duplicate foreign key constraint name %s in table %s", sqlescape.EscapeID(e.Constraint), sqlescape.EscapeID(e.Table)) +} diff --git a/go/vt/schemadiff/schema.go b/go/vt/schemadiff/schema.go index 3b42d6cf42d..bbd1258070e 100644 --- a/go/vt/schemadiff/schema.go +++ b/go/vt/schemadiff/schema.go @@ -415,6 +415,27 @@ func (s *Schema) normalize(hints *DiffHints) error { } } } + + // Validate uniqueness of check constraint and of foreign key constraint names + fkConstraintNames := map[string]bool{} + checkConstraintNames := map[string]bool{} + for _, t := range s.tables { + for _, cs := range t.TableSpec.Constraints { + if _, ok := cs.Details.(*sqlparser.ForeignKeyDefinition); ok { + if _, ok := fkConstraintNames[cs.Name.String()]; ok { + errs = errors.Join(errs, &DuplicateForeignKeyConstraintNameError{Table: t.Name(), Constraint: cs.Name.String()}) + } + fkConstraintNames[cs.Name.String()] = true + } + if _, ok := cs.Details.(*sqlparser.CheckConstraintDefinition); ok { + if _, ok := checkConstraintNames[cs.Name.String()]; ok { + errs = errors.Join(errs, &DuplicateCheckConstraintNameError{Table: t.Name(), Constraint: cs.Name.String()}) + } + checkConstraintNames[cs.Name.String()] = true + } + } + } + return errs } diff --git a/go/vt/schemadiff/schema_diff_test.go b/go/vt/schemadiff/schema_diff_test.go index b39166451b2..70bdd9ed54b 100644 --- a/go/vt/schemadiff/schema_diff_test.go +++ b/go/vt/schemadiff/schema_diff_test.go @@ -1327,6 +1327,10 @@ func TestSchemaDiff(t *testing.T) { // TestDiffFiles diffs two schema files on the local file system. It requires the $TEST_SCHEMADIFF_DIFF_FILES // environment variable to be set to a comma-separated list of two file paths, e.g. "/tmp/from.sql,/tmp/to.sql". // If the variable is unspecified, the test is skipped. It is useful for ad-hoc testing of schema diffs. +// The way to run this test is: +// ```sh +// $ TEST_SCHEMADIFF_DIFF_FILES=/tmp/1.sql,/tmp/2.sql go test -v -count=1 -run TestDiffFiles ./go/vt/schemadiff/ +// ``` func TestDiffFiles(t *testing.T) { ctx := context.Background() diff --git a/go/vt/schemadiff/schema_test.go b/go/vt/schemadiff/schema_test.go index 8a4f54269cd..1710cec12c7 100644 --- a/go/vt/schemadiff/schema_test.go +++ b/go/vt/schemadiff/schema_test.go @@ -287,11 +287,11 @@ func TestTableForeignKeyOrdering(t *testing.T) { "create table t11 (id int primary key, i int, key ix (i), constraint f12 foreign key (i) references t12(id) on delete restrict, constraint f20 foreign key (i) references t20(id) on delete restrict)", "create table t15(id int, primary key(id))", "create view v09 as select * from v13, t17", - "create table t20 (id int primary key, i int, key ix (i), constraint f15 foreign key (i) references t15(id) on delete restrict)", + "create table t20 (id int primary key, i int, key ix (i), constraint f2015 foreign key (i) references t15(id) on delete restrict)", "create view v13 as select * from t20", - "create table t12 (id int primary key, i int, key ix (i), constraint f15 foreign key (i) references t15(id) on delete restrict)", - "create table t17 (id int primary key, i int, key ix (i), constraint f11 foreign key (i) references t11(id) on delete restrict, constraint f15 foreign key (i) references t15(id) on delete restrict)", - "create table t16 (id int primary key, i int, key ix (i), constraint f11 foreign key (i) references t11(id) on delete restrict, constraint f15 foreign key (i) references t15(id) on delete restrict)", + "create table t12 (id int primary key, i int, key ix (i), constraint f1215 foreign key (i) references t15(id) on delete restrict)", + "create table t17 (id int primary key, i int, key ix (i), constraint f1711 foreign key (i) references t11(id) on delete restrict, constraint f1715 foreign key (i) references t15(id) on delete restrict)", + "create table t16 (id int primary key, i int, key ix (i), constraint f1611 foreign key (i) references t11(id) on delete restrict, constraint f1615 foreign key (i) references t15(id) on delete restrict)", "create table t14 (id int primary key, i int, key ix (i), constraint f14 foreign key (i) references t14(id) on delete restrict)", } expectSortedTableNames := []string{ @@ -522,6 +522,54 @@ func TestInvalidSchema(t *testing.T) { { schema: "create table post (id varchar(191) charset utf8mb4 not null, `title` text, primary key (`id`)); create table post_fks (id varchar(191) not null, `post_id` varchar(191) collate utf8mb4_0900_ai_ci, primary key (id), constraint post_fk foreign key (post_id) references post (id)) charset utf8mb4, collate utf8mb4_0900_as_ci;", }, + // constaint names + { + schema: `create table t1 (id int primary key, CONSTRAINT const_id CHECK (id > 0))`, + }, + { + schema: `create table t1 (id int primary key, CONSTRAINT const_id1 CHECK (id > 0), CONSTRAINT const_id2 CHECK (id < 10));`, + }, + { + schema: ` + create table t1 (id int primary key, CONSTRAINT const_id CHECK (id > 0), CONSTRAINT const_id CHECK (id < 10)); + `, + expectErr: &DuplicateCheckConstraintNameError{Table: "t1", Constraint: "const_id"}, + }, + { + schema: ` + create table t1 (id int primary key, CONSTRAINT const_id1 CHECK (id > 0)); + create table t2 (id int primary key, CONSTRAINT const_id2 CHECK (id > 0)); + `, + }, + { + schema: ` + create table t1 (id int primary key, CONSTRAINT const_id CHECK (id > 0)); + create table t2 (id int primary key, CONSTRAINT const_id CHECK (id > 0)); + `, + expectErr: &DuplicateCheckConstraintNameError{Table: "t2", Constraint: "const_id"}, + }, + { + // OK for foreign key constraint and check constraint to have same name + schema: ` + create table t1 (id int primary key, CONSTRAINT const_id CHECK (id > 0)); + create table t2 (id int primary key, CONSTRAINT const_id FOREIGN KEY (id) REFERENCES t1 (id) ON DELETE CASCADE); + `, + }, + { + schema: ` + create table t1 (id int primary key, CONSTRAINT const_id CHECK (id > 0)); + create table t2 (id int primary key, CONSTRAINT const_id FOREIGN KEY (id) REFERENCES t1 (id) ON DELETE CASCADE); + create table t3 (id int primary key, CONSTRAINT const_id FOREIGN KEY (id) REFERENCES t1 (id) ON DELETE CASCADE); + `, + expectErr: &DuplicateForeignKeyConstraintNameError{Table: "t3", Constraint: "const_id"}, + }, + { + schema: ` + create table t1 (id int primary key, CONSTRAINT const_id CHECK (id > 0)); + create table t2 (id int primary key, CONSTRAINT const_id FOREIGN KEY (id) REFERENCES t1 (id) ON DELETE CASCADE, CONSTRAINT const_id FOREIGN KEY (id) REFERENCES t1 (id) ON DELETE CASCADE); + `, + expectErr: &DuplicateForeignKeyConstraintNameError{Table: "t2", Constraint: "const_id"}, + }, } for _, ts := range tt { t.Run(ts.schema, func(t *testing.T) { From de33a39a9db727aee46e913afba6066d87377895 Mon Sep 17 00:00:00 2001 From: Noble Mittal <62551163+beingnoble03@users.noreply.github.com> Date: Tue, 28 Jan 2025 02:43:47 +0530 Subject: [PATCH 04/11] test: Add unit tests for `vtctl/workflow` (#17618) Signed-off-by: Noble Mittal --- go/vt/vtctl/workflow/framework_test.go | 31 +++-- go/vt/vtctl/workflow/server_test.go | 82 ++++++++++++ go/vt/vtctl/workflow/traffic_switcher_test.go | 121 ++++++++++++++++++ go/vt/vtctl/workflow/utils_test.go | 76 +++++++++++ 4 files changed, 302 insertions(+), 8 deletions(-) diff --git a/go/vt/vtctl/workflow/framework_test.go b/go/vt/vtctl/workflow/framework_test.go index a2c1b2ef8e3..fad48e31e0c 100644 --- a/go/vt/vtctl/workflow/framework_test.go +++ b/go/vt/vtctl/workflow/framework_test.go @@ -272,7 +272,7 @@ type testTMClient struct { createVReplicationWorkflowRequests map[uint32]*createVReplicationWorkflowRequestResponse readVReplicationWorkflowRequests map[uint32]*readVReplicationWorkflowRequestResponse updateVReplicationWorklowsRequests map[uint32]*tabletmanagerdatapb.UpdateVReplicationWorkflowsRequest - applySchemaRequests map[uint32]*applySchemaRequestResponse + applySchemaRequests map[uint32][]*applySchemaRequestResponse primaryPositions map[uint32]string vdiffRequests map[uint32]*vdiffRequestResponse refreshStateErrors map[uint32]error @@ -296,7 +296,7 @@ func newTestTMClient(env *testEnv) *testTMClient { createVReplicationWorkflowRequests: make(map[uint32]*createVReplicationWorkflowRequestResponse), readVReplicationWorkflowRequests: make(map[uint32]*readVReplicationWorkflowRequestResponse), updateVReplicationWorklowsRequests: make(map[uint32]*tabletmanagerdatapb.UpdateVReplicationWorkflowsRequest), - applySchemaRequests: make(map[uint32]*applySchemaRequestResponse), + applySchemaRequests: make(map[uint32][]*applySchemaRequestResponse), readVReplicationWorkflowsResponses: make(map[string][]*tabletmanagerdatapb.ReadVReplicationWorkflowsResponse), primaryPositions: make(map[uint32]string), vdiffRequests: make(map[uint32]*vdiffRequestResponse), @@ -398,10 +398,9 @@ func (tmc *testTMClient) GetSchema(ctx context.Context, tablet *topodatapb.Table schemaDefn := &tabletmanagerdatapb.SchemaDefinition{} for _, table := range req.Tables { if table == "/.*/" { - // Special case of all tables in keyspace. - for key, tableDefn := range tmc.schema { + for key, schemaDefinition := range tmc.schema { if strings.HasPrefix(key, tablet.Keyspace+".") { - schemaDefn.TableDefinitions = append(schemaDefn.TableDefinitions, tableDefn.TableDefinitions...) + schemaDefn.TableDefinitions = append(schemaDefn.TableDefinitions, schemaDefinition.TableDefinitions...) } } break @@ -414,6 +413,12 @@ func (tmc *testTMClient) GetSchema(ctx context.Context, tablet *topodatapb.Table } schemaDefn.TableDefinitions = append(schemaDefn.TableDefinitions, tableDefn.TableDefinitions...) } + for key, schemaDefinition := range tmc.schema { + if strings.HasPrefix(key, tablet.Keyspace) { + schemaDefn.DatabaseSchema = schemaDefinition.DatabaseSchema + break + } + } return schemaDefn, nil } @@ -508,10 +513,10 @@ func (tmc *testTMClient) expectApplySchemaRequest(tabletID uint32, req *applySch defer tmc.mu.Unlock() if tmc.applySchemaRequests == nil { - tmc.applySchemaRequests = make(map[uint32]*applySchemaRequestResponse) + tmc.applySchemaRequests = make(map[uint32][]*applySchemaRequestResponse) } - tmc.applySchemaRequests[tabletID] = req + tmc.applySchemaRequests[tabletID] = append(tmc.applySchemaRequests[tabletID], req) } // Note: ONLY breaks up change.SQL into individual statements and executes it. Does NOT fully implement ApplySchema. @@ -519,11 +524,17 @@ func (tmc *testTMClient) ApplySchema(ctx context.Context, tablet *topodatapb.Tab tmc.mu.Lock() defer tmc.mu.Unlock() - if expect, ok := tmc.applySchemaRequests[tablet.Alias.Uid]; ok { + if requests, ok := tmc.applySchemaRequests[tablet.Alias.Uid]; ok { + if len(requests) == 0 { + return nil, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "unexpected ApplySchema request on tablet %s: got %+v", + topoproto.TabletAliasString(tablet.Alias), change) + } + expect := requests[0] if !reflect.DeepEqual(change, expect.change) { return nil, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "unexpected ApplySchema request on tablet %s: got %+v, want %+v", topoproto.TabletAliasString(tablet.Alias), change, expect.change) } + tmc.applySchemaRequests[tablet.Alias.Uid] = tmc.applySchemaRequests[tablet.Alias.Uid][1:] return expect.res, expect.err } @@ -779,6 +790,10 @@ func (tmc *testTMClient) getVReplicationWorkflowsResponse(key string) *tabletman return resp } +func (tmc *testTMClient) ReloadSchema(ctx context.Context, tablet *topodatapb.Tablet, waitPosition string) error { + return nil +} + // // Utility / helper functions. // diff --git a/go/vt/vtctl/workflow/server_test.go b/go/vt/vtctl/workflow/server_test.go index ae34dabfc19..4676b732245 100644 --- a/go/vt/vtctl/workflow/server_test.go +++ b/go/vt/vtctl/workflow/server_test.go @@ -34,10 +34,12 @@ import ( "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/test/utils" "vitess.io/vitess/go/vt/logutil" + "vitess.io/vitess/go/vt/mysqlctl/tmutils" "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/topo/topoproto" "vitess.io/vitess/go/vt/topotools" "vitess.io/vitess/go/vt/vtenv" + "vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication" "vitess.io/vitess/go/vt/vttablet/tmclient" binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" @@ -2315,3 +2317,83 @@ func TestWorkflowStatus(t *testing.T) { assert.Equal(t, float32(50), stateTable1.RowsPercentage) assert.Equal(t, float32(50), stateTable2.RowsPercentage) } + +func TestDeleteShard(t *testing.T) { + ctx := context.Background() + + sourceKeyspace := &testKeyspace{"source_keyspace", []string{"-"}} + targetKeyspace := &testKeyspace{"target_keyspace", []string{"-"}} + + te := newTestEnv(t, ctx, defaultCellName, sourceKeyspace, targetKeyspace) + defer te.close() + + // Verify that shard exists. + si, err := te.ts.GetShard(ctx, targetKeyspace.KeyspaceName, targetKeyspace.ShardNames[0]) + require.NoError(t, err) + require.NotNil(t, si) + + // Expect to fail if recursive is false. + err = te.ws.DeleteShard(ctx, targetKeyspace.KeyspaceName, targetKeyspace.ShardNames[0], false, true) + assert.ErrorContains(t, err, "shard target_keyspace/- still has 1 tablets in cell") + + // Should not throw error if given keyspace or shard is invalid. + err = te.ws.DeleteShard(ctx, "invalid_keyspace", "-", false, true) + assert.NoError(t, err) + + // Successful shard delete. + err = te.ws.DeleteShard(ctx, targetKeyspace.KeyspaceName, targetKeyspace.ShardNames[0], true, true) + assert.NoError(t, err) + + // Check if the shard was deleted. + _, err = te.ts.GetShard(ctx, targetKeyspace.KeyspaceName, targetKeyspace.ShardNames[0]) + assert.ErrorContains(t, err, "node doesn't exist") +} + +func TestCopySchemaShard(t *testing.T) { + ctx := context.Background() + + sourceKeyspace := &testKeyspace{"source_keyspace", []string{"-"}} + targetKeyspace := &testKeyspace{"target_keyspace", []string{"-"}} + + te := newTestEnv(t, ctx, defaultCellName, sourceKeyspace, targetKeyspace) + defer te.close() + + sqlSchema := `create table t1(id bigint(20) unsigned auto_increment, msg varchar(64), primary key (id)) Engine=InnoDB;` + te.tmc.schema[fmt.Sprintf("%s.t1", sourceKeyspace.KeyspaceName)] = &tabletmanagerdatapb.SchemaDefinition{ + DatabaseSchema: "CREATE DATABASE {{.DatabaseName}}", + TableDefinitions: []*tabletmanagerdatapb.TableDefinition{ + { + Name: "t1", + Schema: sqlSchema, + Columns: []string{ + "id", + "msg", + }, + Type: tmutils.TableBaseTable, + }, + }, + } + + // Expect queries on target shards + te.tmc.expectApplySchemaRequest(200, &applySchemaRequestResponse{ + change: &tmutils.SchemaChange{ + SQL: "CREATE DATABASE `vt_target_keyspace`", + Force: false, + AllowReplication: true, + SQLMode: vreplication.SQLMode, + }, + }) + te.tmc.expectApplySchemaRequest(200, &applySchemaRequestResponse{ + change: &tmutils.SchemaChange{ + SQL: sqlSchema, + Force: false, + AllowReplication: true, + SQLMode: vreplication.SQLMode, + }, + }) + + sourceTablet := te.tablets[sourceKeyspace.KeyspaceName][100] + err := te.ws.CopySchemaShard(ctx, sourceTablet.Alias, []string{"/.*/"}, nil, false, targetKeyspace.KeyspaceName, "-", 1*time.Second, true) + assert.NoError(t, err) + assert.Empty(t, te.tmc.applySchemaRequests[200]) +} diff --git a/go/vt/vtctl/workflow/traffic_switcher_test.go b/go/vt/vtctl/workflow/traffic_switcher_test.go index 2cf998eb8e4..a7da91174b9 100644 --- a/go/vt/vtctl/workflow/traffic_switcher_test.go +++ b/go/vt/vtctl/workflow/traffic_switcher_test.go @@ -30,6 +30,7 @@ import ( "vitess.io/vitess/go/sqlescape" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/mysqlctl/tmutils" + "vitess.io/vitess/go/vt/proto/binlogdata" "vitess.io/vitess/go/vt/proto/vschema" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/topo" @@ -912,3 +913,123 @@ func TestAddParticipatingTablesToKeyspace(t *testing.T) { assert.Len(t, vs.Tables["t1"].ColumnVindexes, 2) assert.Len(t, vs.Tables["t2"].ColumnVindexes, 1) } + +func TestCancelMigration_TABLES(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + workflowName := "wf1" + tableName := "t1" + + sourceKeyspace := &testKeyspace{ + KeyspaceName: "sourceks", + ShardNames: []string{"0"}, + } + targetKeyspace := &testKeyspace{ + KeyspaceName: "targetks", + ShardNames: []string{"0"}, + } + + schema := map[string]*tabletmanagerdatapb.SchemaDefinition{ + tableName: { + TableDefinitions: []*tabletmanagerdatapb.TableDefinition{ + { + Name: tableName, + Schema: fmt.Sprintf("CREATE TABLE %s (id BIGINT, name VARCHAR(64), PRIMARY KEY (id))", tableName), + }, + }, + }, + } + + env := newTestEnv(t, ctx, defaultCellName, sourceKeyspace, targetKeyspace) + defer env.close() + env.tmc.schema = schema + + ts, _, err := env.ws.getWorkflowState(ctx, targetKeyspace.KeyspaceName, workflowName) + require.NoError(t, err) + + sm, err := BuildStreamMigrator(ctx, ts, false, sqlparser.NewTestParser()) + require.NoError(t, err) + + env.tmc.expectVRQuery(200, "update _vt.vreplication set state='Running', message='' where db_name='vt_targetks' and workflow='wf1'", &sqltypes.Result{}) + env.tmc.expectVRQuery(100, "delete from _vt.vreplication where db_name = 'vt_sourceks' and workflow = 'wf1_reverse'", &sqltypes.Result{}) + + ctx, _, err = env.ts.LockKeyspace(ctx, targetKeyspace.KeyspaceName, "test") + require.NoError(t, err) + + ctx, _, err = env.ts.LockKeyspace(ctx, sourceKeyspace.KeyspaceName, "test") + require.NoError(t, err) + + err = topo.CheckKeyspaceLocked(ctx, ts.targetKeyspace) + require.NoError(t, err) + + err = topo.CheckKeyspaceLocked(ctx, ts.sourceKeyspace) + require.NoError(t, err) + + ts.cancelMigration(ctx, sm) + + // Expect the queries to be cleared + assert.Empty(t, env.tmc.vrQueries[100]) + assert.Empty(t, env.tmc.vrQueries[200]) +} + +func TestCancelMigration_SHARDS(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + workflowName := "wf1" + tableName := "t1" + + sourceKeyspace := &testKeyspace{ + KeyspaceName: "sourceks", + ShardNames: []string{"0"}, + } + targetKeyspace := &testKeyspace{ + KeyspaceName: "targetks", + ShardNames: []string{"0"}, + } + + schema := map[string]*tabletmanagerdatapb.SchemaDefinition{ + tableName: { + TableDefinitions: []*tabletmanagerdatapb.TableDefinition{ + { + Name: tableName, + Schema: fmt.Sprintf("CREATE TABLE %s (id BIGINT, name VARCHAR(64), PRIMARY KEY (id))", tableName), + }, + }, + }, + } + + env := newTestEnv(t, ctx, defaultCellName, sourceKeyspace, targetKeyspace) + defer env.close() + env.tmc.schema = schema + + ts, _, err := env.ws.getWorkflowState(ctx, targetKeyspace.KeyspaceName, workflowName) + require.NoError(t, err) + ts.migrationType = binlogdata.MigrationType_SHARDS + + sm, err := BuildStreamMigrator(ctx, ts, false, sqlparser.NewTestParser()) + require.NoError(t, err) + + env.tmc.expectVRQuery(100, "update /*vt+ ALLOW_UNSAFE_VREPLICATION_WRITE */ _vt.vreplication set state='Running', stop_pos=null, message='' where db_name='vt_sourceks' and workflow != 'wf1_reverse'", &sqltypes.Result{}) + env.tmc.expectVRQuery(200, "update _vt.vreplication set state='Running', message='' where db_name='vt_targetks' and workflow='wf1'", &sqltypes.Result{}) + env.tmc.expectVRQuery(100, "delete from _vt.vreplication where db_name = 'vt_sourceks' and workflow = 'wf1_reverse'", &sqltypes.Result{}) + + ctx, _, err = env.ts.LockKeyspace(ctx, targetKeyspace.KeyspaceName, "test") + require.NoError(t, err) + + ctx, _, err = env.ts.LockKeyspace(ctx, sourceKeyspace.KeyspaceName, "test") + require.NoError(t, err) + + err = topo.CheckKeyspaceLocked(ctx, ts.targetKeyspace) + require.NoError(t, err) + + err = topo.CheckKeyspaceLocked(ctx, ts.sourceKeyspace) + require.NoError(t, err) + + ts.cancelMigration(ctx, sm) + + // Expect the queries to be cleared + assert.Empty(t, env.tmc.vrQueries[100]) + assert.Empty(t, env.tmc.vrQueries[200]) +} diff --git a/go/vt/vtctl/workflow/utils_test.go b/go/vt/vtctl/workflow/utils_test.go index eecbfd6269b..99850639ac5 100644 --- a/go/vt/vtctl/workflow/utils_test.go +++ b/go/vt/vtctl/workflow/utils_test.go @@ -15,6 +15,7 @@ import ( "github.com/stretchr/testify/require" clientv3 "go.etcd.io/etcd/client/v3" + "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/testfiles" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/topo" @@ -22,6 +23,7 @@ import ( "vitess.io/vitess/go/vt/topo/memorytopo" "vitess.io/vitess/go/vt/topotools" + tabletmanagerdatapb "vitess.io/vitess/go/vt/proto/tabletmanagerdata" "vitess.io/vitess/go/vt/proto/vtctldata" ) @@ -280,3 +282,77 @@ func TestValidateSourceTablesExist(t *testing.T) { }) } } + +func TestLegacyBuildTargets(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + workflowName := "wf1" + tableName := "t1" + + sourceKeyspace := &testKeyspace{ + KeyspaceName: "sourceks", + ShardNames: []string{"0"}, + } + targetKeyspace := &testKeyspace{ + KeyspaceName: "targetks", + ShardNames: []string{"-80", "80-"}, + } + + schema := map[string]*tabletmanagerdatapb.SchemaDefinition{ + tableName: { + TableDefinitions: []*tabletmanagerdatapb.TableDefinition{ + { + Name: tableName, + Schema: fmt.Sprintf("CREATE TABLE %s (id BIGINT, name VARCHAR(64), PRIMARY KEY (id))", tableName), + }, + }, + }, + } + + env := newTestEnv(t, ctx, defaultCellName, sourceKeyspace, targetKeyspace) + defer env.close() + env.tmc.schema = schema + + result1 := sqltypes.MakeTestResult(sqltypes.MakeTestFields( + "id|source|message|cell|tablet_types|workflow_type|workflow_sub_type|defer_secondary_keys", + "int64|varchar|varchar|varchar|varchar|int64|int64|int64"), + "1|keyspace:\"source\" shard:\"-80\" filter:{rules:{match:\"t1\"} rules:{match:\"t2\"}}||||0|0|0", + ) + result2 := sqltypes.MakeTestResult(sqltypes.MakeTestFields( + "id|source|message|cell|tablet_types|workflow_type|workflow_sub_type|defer_secondary_keys", + "int64|varchar|varchar|varchar|varchar|int64|int64|int64"), + "1|keyspace:\"source\" shard:\"80-\" filter:{rules:{match:\"t1\"} rules:{match:\"t2\"}}||||0|0|0", + "2|keyspace:\"source\" shard:\"80-\" filter:{rules:{match:\"t3\"} rules:{match:\"t4\"}}||||0|0|0", + ) + env.tmc.expectVRQuery(200, "select id, source, message, cell, tablet_types, workflow_type, workflow_sub_type, defer_secondary_keys from _vt.vreplication where workflow='wf1' and db_name='vt_targetks'", result1) + env.tmc.expectVRQuery(210, "select id, source, message, cell, tablet_types, workflow_type, workflow_sub_type, defer_secondary_keys from _vt.vreplication where workflow='wf1' and db_name='vt_targetks'", result2) + + ti, err := LegacyBuildTargets(ctx, env.ts, env.tmc, targetKeyspace.KeyspaceName, workflowName, targetKeyspace.ShardNames) + require.NoError(t, err) + // Expect 2 targets as there are 2 target shards. + assert.Len(t, ti.Targets, 2) + + assert.NotNil(t, ti.Targets["-80"]) + assert.NotNil(t, ti.Targets["80-"]) + + t1 := ti.Targets["-80"] + t2 := ti.Targets["80-"] + assert.Len(t, t1.Sources, 1) + assert.Len(t, t2.Sources, 2) + assert.Len(t, t1.Sources[1].Filter.Rules, 2) + + assert.Equal(t, t1.Sources[1].Filter.Rules[0].Match, "t1") + assert.Equal(t, t1.Sources[1].Filter.Rules[1].Match, "t2") + assert.Equal(t, t1.Sources[1].Shard, "-80") + + assert.Len(t, t2.Sources[1].Filter.Rules, 2) + assert.Len(t, t2.Sources[2].Filter.Rules, 2) + + assert.Equal(t, t2.Sources[1].Shard, "80-") + assert.Equal(t, t2.Sources[2].Shard, "80-") + assert.Equal(t, t2.Sources[1].Filter.Rules[0].Match, "t1") + assert.Equal(t, t2.Sources[1].Filter.Rules[1].Match, "t2") + assert.Equal(t, t2.Sources[2].Filter.Rules[0].Match, "t3") + assert.Equal(t, t2.Sources[2].Filter.Rules[1].Match, "t4") +} From 489fd05eda9e90f8583a2e81e8fa5c7417759fe8 Mon Sep 17 00:00:00 2001 From: jwang <121262788+jwangace@users.noreply.github.com> Date: Mon, 27 Jan 2025 15:36:39 -0800 Subject: [PATCH 05/11] --consolidator-query-waiter-cap to set the max number of waiter for consolidated query (#17244) Signed-off-by: Jun Wang Co-authored-by: Jun Wang --- changelog/22.0/22.0.0/summary.md | 2 ++ go/flags/endtoend/vtcombo.txt | 1 + go/flags/endtoend/vttablet.txt | 1 + go/sync2/consolidator.go | 10 +++++- go/sync2/consolidator_test.go | 31 +++++++++++++++++++ go/sync2/fake_consolidator.go | 5 +++ go/vt/vttablet/tabletserver/query_executor.go | 12 ++++--- .../vttablet/tabletserver/tabletenv/config.go | 2 ++ 8 files changed, 59 insertions(+), 5 deletions(-) diff --git a/changelog/22.0/22.0.0/summary.md b/changelog/22.0/22.0.0/summary.md index b2c5c029851..7375077dced 100644 --- a/changelog/22.0/22.0.0/summary.md +++ b/changelog/22.0/22.0.0/summary.md @@ -147,6 +147,8 @@ The `querylog-mode` setting can be configured to `error` to log only queries tha While the flag will continue to accept float values (interpreted as seconds) for backward compatibility, **float inputs are deprecated** and will be removed in a future release. +- `--consolidator-query-waiter-cap` flag to set the maximum number of clients allowed to wait on the consolidator. The default value is set to 0 for unlimited wait. Users can adjust this value based on the performance of VTTablet to avoid excessive memory usage and the risk of being OOMKilled, particularly in Kubernetes deployments. + ### `--topo_read_concurrency` behaviour changes The `--topo_read_concurrency` flag was added to all components that access the topology and the provided limit is now applied separately for each global or local cell _(default `32`)_. diff --git a/go/flags/endtoend/vtcombo.txt b/go/flags/endtoend/vtcombo.txt index 561f6048ce6..3ae0cbc9b77 100644 --- a/go/flags/endtoend/vtcombo.txt +++ b/go/flags/endtoend/vtcombo.txt @@ -44,6 +44,7 @@ Flags: --config-path strings Paths to search for config files in. (default [{{ .Workdir }}]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) --config-type string Config file type (omit to infer config type from file extension). + --consolidator-query-waiter-cap int Configure the maximum number of clients allowed to wait on the consolidator. --consolidator-stream-query-size int Configure the stream consolidator query size in bytes. Setting to 0 disables the stream consolidator. (default 2097152) --consolidator-stream-total-size int Configure the stream consolidator total size in bytes. Setting to 0 disables the stream consolidator. (default 134217728) --consul_auth_static_file string JSON File to read the topos/tokens from. diff --git a/go/flags/endtoend/vttablet.txt b/go/flags/endtoend/vttablet.txt index 398d10afd7c..0a9a0ef99ce 100644 --- a/go/flags/endtoend/vttablet.txt +++ b/go/flags/endtoend/vttablet.txt @@ -78,6 +78,7 @@ Flags: --config-path strings Paths to search for config files in. (default [{{ .Workdir }}]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) --config-type string Config file type (omit to infer config type from file extension). + --consolidator-query-waiter-cap int Configure the maximum number of clients allowed to wait on the consolidator. --consolidator-stream-query-size int Configure the stream consolidator query size in bytes. Setting to 0 disables the stream consolidator. (default 2097152) --consolidator-stream-total-size int Configure the stream consolidator total size in bytes. Setting to 0 disables the stream consolidator. (default 134217728) --consul_auth_static_file string JSON File to read the topos/tokens from. diff --git a/go/sync2/consolidator.go b/go/sync2/consolidator.go index 401daaef1f1..df900625abb 100644 --- a/go/sync2/consolidator.go +++ b/go/sync2/consolidator.go @@ -40,6 +40,7 @@ type PendingResult interface { SetResult(*sqltypes.Result) Result() *sqltypes.Result Wait() + AddWaiterCounter(int64) *int64 } type consolidator struct { @@ -77,6 +78,7 @@ func (co *consolidator) Create(query string) (PendingResult, bool) { defer co.mu.Unlock() var r *pendingResult if r, ok := co.queries[query]; ok { + r.AddWaiterCounter(1) return r, false } r = &pendingResult{consolidator: co, query: query} @@ -122,17 +124,23 @@ func (rs *pendingResult) Wait() { rs.executing.RLock() } +func (rs *pendingResult) AddWaiterCounter(c int64) *int64 { + atomic.AddInt64(rs.consolidator.totalWaiterCount, c) + return rs.consolidator.totalWaiterCount +} + // ConsolidatorCache is a thread-safe object used for counting how often recent // queries have been consolidated. // It is also used by the txserializer package to count how often transactions // have been queued and had to wait because they targeted the same row (range). type ConsolidatorCache struct { *cache.LRUCache[*ccount] + totalWaiterCount *int64 } // NewConsolidatorCache creates a new cache with the given capacity. func NewConsolidatorCache(capacity int64) *ConsolidatorCache { - return &ConsolidatorCache{cache.NewLRUCache[*ccount](capacity)} + return &ConsolidatorCache{cache.NewLRUCache[*ccount](capacity), new(int64)} } // Record increments the count for "query" by 1. diff --git a/go/sync2/consolidator_test.go b/go/sync2/consolidator_test.go index 132a253ba29..5437bb335a6 100644 --- a/go/sync2/consolidator_test.go +++ b/go/sync2/consolidator_test.go @@ -18,11 +18,42 @@ package sync2 import ( "reflect" + "sync" "testing" "vitess.io/vitess/go/sqltypes" ) +func TestAddWaiterCount(t *testing.T) { + con := NewConsolidator() + sql := "select * from SomeTable" + pr, _ := con.Create(sql) + var wgAdd sync.WaitGroup + var wgSub sync.WaitGroup + + var concurrent = 1000 + + for i := 0; i < concurrent; i++ { + wgAdd.Add(1) + wgSub.Add(1) + go func() { + defer wgAdd.Done() + pr.AddWaiterCounter(1) + }() + go func() { + defer wgSub.Done() + pr.AddWaiterCounter(-1) + }() + } + + wgAdd.Wait() + wgSub.Wait() + + if *pr.AddWaiterCounter(0) != 0 { + t.Fatalf("Expect 0 totalWaiterCount but got: %v", *pr.AddWaiterCounter(0)) + } +} + func TestConsolidator(t *testing.T) { con := NewConsolidator() sql := "select * from SomeTable" diff --git a/go/sync2/fake_consolidator.go b/go/sync2/fake_consolidator.go index 64c59e78a5a..aadee1d37ce 100644 --- a/go/sync2/fake_consolidator.go +++ b/go/sync2/fake_consolidator.go @@ -112,3 +112,8 @@ func (fr *FakePendingResult) SetResult(result *sqltypes.Result) { func (fr *FakePendingResult) Wait() { fr.WaitCalls++ } + +// AddWaiterCounter is currently a no-op. +func (fr *FakePendingResult) AddWaiterCounter(int64) *int64 { + return new(int64) +} diff --git a/go/vt/vttablet/tabletserver/query_executor.go b/go/vt/vttablet/tabletserver/query_executor.go index e4a165960fd..755394723f3 100644 --- a/go/vt/vttablet/tabletserver/query_executor.go +++ b/go/vt/vttablet/tabletserver/query_executor.go @@ -718,10 +718,14 @@ func (qre *QueryExecutor) execSelect() (*sqltypes.Result, error) { q.SetErr(err) } } else { - qre.logStats.QuerySources |= tabletenv.QuerySourceConsolidator - startTime := time.Now() - q.Wait() - qre.tsv.stats.WaitTimings.Record("Consolidations", startTime) + waiterCap := qre.tsv.config.ConsolidatorQueryWaiterCap + if waiterCap == 0 || *q.AddWaiterCounter(0) <= waiterCap { + qre.logStats.QuerySources |= tabletenv.QuerySourceConsolidator + startTime := time.Now() + q.Wait() + qre.tsv.stats.WaitTimings.Record("Consolidations", startTime) + } + q.AddWaiterCounter(-1) } if q.Err() != nil { return nil, q.Err() diff --git a/go/vt/vttablet/tabletserver/tabletenv/config.go b/go/vt/vttablet/tabletserver/tabletenv/config.go index 08bf0b1d7c8..ddab935d393 100644 --- a/go/vt/vttablet/tabletserver/tabletenv/config.go +++ b/go/vt/vttablet/tabletserver/tabletenv/config.go @@ -198,6 +198,7 @@ func registerTabletEnvFlags(fs *pflag.FlagSet) { fs.Int64Var(¤tConfig.ConsolidatorStreamQuerySize, "consolidator-stream-query-size", defaultConfig.ConsolidatorStreamQuerySize, "Configure the stream consolidator query size in bytes. Setting to 0 disables the stream consolidator.") fs.Int64Var(¤tConfig.ConsolidatorStreamTotalSize, "consolidator-stream-total-size", defaultConfig.ConsolidatorStreamTotalSize, "Configure the stream consolidator total size in bytes. Setting to 0 disables the stream consolidator.") + fs.Int64Var(¤tConfig.ConsolidatorQueryWaiterCap, "consolidator-query-waiter-cap", 0, "Configure the maximum number of clients allowed to wait on the consolidator.") fs.DurationVar(&healthCheckInterval, "health_check_interval", defaultConfig.Healthcheck.Interval, "Interval between health checks") fs.DurationVar(°radedThreshold, "degraded_threshold", defaultConfig.Healthcheck.DegradedThreshold, "replication lag after which a replica is considered degraded") fs.DurationVar(&unhealthyThreshold, "unhealthy_threshold", defaultConfig.Healthcheck.UnhealthyThreshold, "replication lag after which a replica is considered unhealthy") @@ -324,6 +325,7 @@ type TabletConfig struct { StreamBufferSize int `json:"streamBufferSize,omitempty"` ConsolidatorStreamTotalSize int64 `json:"consolidatorStreamTotalSize,omitempty"` ConsolidatorStreamQuerySize int64 `json:"consolidatorStreamQuerySize,omitempty"` + ConsolidatorQueryWaiterCap int64 `json:"consolidatorMaxQueryWait,omitempty"` QueryCacheMemory int64 `json:"queryCacheMemory,omitempty"` QueryCacheDoorkeeper bool `json:"queryCacheDoorkeeper,omitempty"` SchemaReloadInterval time.Duration `json:"schemaReloadIntervalSeconds,omitempty"` From 8921bce64b732c48ca6bb52037fa9b2796e36d1d Mon Sep 17 00:00:00 2001 From: Manan Gupta <35839558+GuptaManan100@users.noreply.github.com> Date: Tue, 28 Jan 2025 11:59:34 +0530 Subject: [PATCH 06/11] Support KeyRange in `--clusters_to_watch` flag (#17604) Signed-off-by: Manan Gupta --- changelog/22.0/22.0.0/summary.md | 6 + go/flags/endtoend/vtorc.txt | 2 +- go/vt/key/key.go | 8 + go/vt/topo/shard_test.go | 8 + go/vt/vtorc/logic/keyspace_shard_discovery.go | 26 +-- .../logic/keyspace_shard_discovery_test.go | 5 +- go/vt/vtorc/logic/tablet_discovery.go | 90 ++++---- go/vt/vtorc/logic/tablet_discovery_test.go | 200 +++++++++++++++--- go/vt/vtorc/logic/vtorc.go | 6 - 9 files changed, 258 insertions(+), 93 deletions(-) diff --git a/changelog/22.0/22.0.0/summary.md b/changelog/22.0/22.0.0/summary.md index 7375077dced..f773df2b10c 100644 --- a/changelog/22.0/22.0.0/summary.md +++ b/changelog/22.0/22.0.0/summary.md @@ -15,6 +15,7 @@ - **[Stalled Disk Recovery in VTOrc](#stall-disk-recovery)** - **[Update default MySQL version to 8.0.40](#mysql-8-0-40)** - **[Update lite images to Debian Bookworm](#debian-bookworm)** + - **[KeyRanges in `--clusters_to_watch` in VTOrc](#key-range-vtorc)** - **[Support for Filtering Query logs on Error](#query-logs)** - **[Minor Changes](#minor-changes)** - **[VTTablet Flags](#flags-vttablet)** @@ -135,6 +136,11 @@ This is the last time this will be needed in the `8.0.x` series, as starting wit The base system now uses Debian Bookworm instead of Debian Bullseye for the `vitess/lite` images. This change was brought by [Pull Request #17552]. +### KeyRanges in `--clusters_to_watch` in VTOrc +VTOrc now supports specifying keyranges in the `--clusters_to_watch` flag. This means that there is no need to restart a VTOrc instance with a different flag value when you reshard a keyspace. +For example, if a VTOrc is configured to watch `ks/-80`, then it would watch all the shards that fall under the keyrange `-80`. If a reshard is performed and `-80` is split into new shards `-40` and `40-80`, the VTOrc instance will automatically start watching the new shards without needing a restart. In the previous logic, specifying `ks/-80` for the flag would mean that VTOrc would watch only 1 (or no) shard. In the new system, since we interpret `-80` as a key range, it can watch multiple shards as described in the example. +Users can continue to specify exact keyranges. The new feature is backward compatible. + ### Support for Filtering Query logs on Error The `querylog-mode` setting can be configured to `error` to log only queries that result in errors. This option is supported in both VTGate and VTTablet. diff --git a/go/flags/endtoend/vtorc.txt b/go/flags/endtoend/vtorc.txt index ca8083709e5..57eb907cf4d 100644 --- a/go/flags/endtoend/vtorc.txt +++ b/go/flags/endtoend/vtorc.txt @@ -24,7 +24,7 @@ Flags: --bind-address string Bind address for the server. If empty, the server will listen on all available unicast and anycast IP addresses of the local system. --catch-sigpipe catch and ignore SIGPIPE on stdout and stderr if specified --change-tablets-with-errant-gtid-to-drained Whether VTOrc should be changing the type of tablets with errant GTIDs to DRAINED - --clusters_to_watch strings Comma-separated list of keyspaces or keyspace/shards that this instance will monitor and repair. Defaults to all clusters in the topology. Example: "ks1,ks2/-80" + --clusters_to_watch strings Comma-separated list of keyspaces or keyspace/keyranges that this instance will monitor and repair. Defaults to all clusters in the topology. Example: "ks1,ks2/-80" --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default warn) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") diff --git a/go/vt/key/key.go b/go/vt/key/key.go index 89d956bd433..82852daa16e 100644 --- a/go/vt/key/key.go +++ b/go/vt/key/key.go @@ -95,6 +95,14 @@ func NewKeyRange(start []byte, end []byte) *topodatapb.KeyRange { return &topodatapb.KeyRange{Start: start, End: end} } +// NewCompleteKeyRange returns a complete key range. +func NewCompleteKeyRange() *topodatapb.KeyRange { + return &topodatapb.KeyRange{ + Start: nil, + End: nil, + } +} + // KeyRangeAdd adds two adjacent KeyRange values (in any order) into a single value. If the values are not adjacent, // it returns false. func KeyRangeAdd(a, b *topodatapb.KeyRange) (*topodatapb.KeyRange, bool) { diff --git a/go/vt/topo/shard_test.go b/go/vt/topo/shard_test.go index 6bd4aae5b62..915bcd18e3c 100644 --- a/go/vt/topo/shard_test.go +++ b/go/vt/topo/shard_test.go @@ -323,6 +323,14 @@ func TestValidateShardName(t *testing.T) { }, valid: true, }, + { + name: "-", + expectedRange: &topodatapb.KeyRange{ + Start: []byte{}, + End: []byte{}, + }, + valid: true, + }, { name: "40-80", expectedRange: &topodatapb.KeyRange{ diff --git a/go/vt/vtorc/logic/keyspace_shard_discovery.go b/go/vt/vtorc/logic/keyspace_shard_discovery.go index 0dd17cb65fd..8115e614418 100644 --- a/go/vt/vtorc/logic/keyspace_shard_discovery.go +++ b/go/vt/vtorc/logic/keyspace_shard_discovery.go @@ -18,10 +18,10 @@ package logic import ( "context" - "sort" - "strings" "sync" + "golang.org/x/exp/maps" + "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/topo" @@ -31,7 +31,7 @@ import ( // RefreshAllKeyspacesAndShards reloads the keyspace and shard information for the keyspaces that vtorc is concerned with. func RefreshAllKeyspacesAndShards(ctx context.Context) error { var keyspaces []string - if len(clustersToWatch) == 0 { // all known keyspaces + if len(shardsToWatch) == 0 { // all known keyspaces ctx, cancel := context.WithTimeout(ctx, topo.RemoteOperationTimeout) defer cancel() var err error @@ -41,26 +41,10 @@ func RefreshAllKeyspacesAndShards(ctx context.Context) error { return err } } else { - // Parse input and build list of keyspaces - for _, ks := range clustersToWatch { - if strings.Contains(ks, "/") { - // This is a keyspace/shard specification - input := strings.Split(ks, "/") - keyspaces = append(keyspaces, input[0]) - } else { - // Assume this is a keyspace - keyspaces = append(keyspaces, ks) - } - } - if len(keyspaces) == 0 { - log.Errorf("Found no keyspaces for input: %+v", clustersToWatch) - return nil - } + // Get keyspaces to watch from the list of known keyspaces. + keyspaces = maps.Keys(shardsToWatch) } - // Sort the list of keyspaces. - // The list can have duplicates because the input to clusters to watch may have multiple shards of the same keyspace - sort.Strings(keyspaces) refreshCtx, refreshCancel := context.WithTimeout(ctx, topo.RemoteOperationTimeout) defer refreshCancel() var wg sync.WaitGroup diff --git a/go/vt/vtorc/logic/keyspace_shard_discovery_test.go b/go/vt/vtorc/logic/keyspace_shard_discovery_test.go index 8218af45db6..f05295416d0 100644 --- a/go/vt/vtorc/logic/keyspace_shard_discovery_test.go +++ b/go/vt/vtorc/logic/keyspace_shard_discovery_test.go @@ -93,6 +93,8 @@ func TestRefreshAllKeyspaces(t *testing.T) { // Set clusters to watch to only watch ks1 and ks3 onlyKs1and3 := []string{"ks1/-80", "ks3/-80", "ks3/80-"} clustersToWatch = onlyKs1and3 + err := initializeShardsToWatch() + require.NoError(t, err) require.NoError(t, RefreshAllKeyspacesAndShards(context.Background())) // Verify that we only have ks1 and ks3 in vtorc's db. @@ -106,6 +108,8 @@ func TestRefreshAllKeyspaces(t *testing.T) { // Set clusters to watch to watch all keyspaces clustersToWatch = nil + err = initializeShardsToWatch() + require.NoError(t, err) // Change the durability policy of ks1 reparenttestutil.SetKeyspaceDurability(ctx, t, ts, "ks1", policy.DurabilitySemiSync) require.NoError(t, RefreshAllKeyspacesAndShards(context.Background())) @@ -119,7 +123,6 @@ func TestRefreshAllKeyspaces(t *testing.T) { verifyPrimaryAlias(t, "ks3", "80-", "zone_ks3-0000000101", "") verifyKeyspaceInfo(t, "ks4", keyspaceDurabilityTest, "") verifyPrimaryAlias(t, "ks4", "80-", "zone_ks4-0000000101", "") - } func TestRefreshKeyspace(t *testing.T) { diff --git a/go/vt/vtorc/logic/tablet_discovery.go b/go/vt/vtorc/logic/tablet_discovery.go index eb10bb2a667..c5c23df0cd0 100644 --- a/go/vt/vtorc/logic/tablet_discovery.go +++ b/go/vt/vtorc/logic/tablet_discovery.go @@ -32,6 +32,7 @@ import ( "google.golang.org/protobuf/proto" "vitess.io/vitess/go/vt/external/golib/sqlutils" + "vitess.io/vitess/go/vt/key" "vitess.io/vitess/go/vt/log" topodatapb "vitess.io/vitess/go/vt/proto/topodata" "vitess.io/vitess/go/vt/topo" @@ -48,8 +49,10 @@ var ( clustersToWatch []string shutdownWaitTime = 30 * time.Second shardsLockCounter int32 - shardsToWatch map[string][]string - shardsToWatchMu sync.Mutex + // shardsToWatch is a map storing the shards for a given keyspace that need to be watched. + // We store the key range for all the shards that we want to watch. + // This is populated by parsing `--clusters_to_watch` flag. + shardsToWatch map[string][]*topodatapb.KeyRange // ErrNoPrimaryTablet is a fixed error message. ErrNoPrimaryTablet = errors.New("no primary tablet found") @@ -57,18 +60,18 @@ var ( // RegisterFlags registers the flags required by VTOrc func RegisterFlags(fs *pflag.FlagSet) { - fs.StringSliceVar(&clustersToWatch, "clusters_to_watch", clustersToWatch, "Comma-separated list of keyspaces or keyspace/shards that this instance will monitor and repair. Defaults to all clusters in the topology. Example: \"ks1,ks2/-80\"") + fs.StringSliceVar(&clustersToWatch, "clusters_to_watch", clustersToWatch, "Comma-separated list of keyspaces or keyspace/keyranges that this instance will monitor and repair. Defaults to all clusters in the topology. Example: \"ks1,ks2/-80\"") fs.DurationVar(&shutdownWaitTime, "shutdown_wait_time", shutdownWaitTime, "Maximum time to wait for VTOrc to release all the locks that it is holding before shutting down on SIGTERM") } -// updateShardsToWatch parses the --clusters_to_watch flag-value +// initializeShardsToWatch parses the --clusters_to_watch flag-value // into a map of keyspace/shards. -func updateShardsToWatch() { +func initializeShardsToWatch() error { + shardsToWatch = make(map[string][]*topodatapb.KeyRange) if len(clustersToWatch) == 0 { - return + return nil } - newShardsToWatch := make(map[string][]string, 0) for _, ks := range clustersToWatch { if strings.Contains(ks, "/") && !strings.HasSuffix(ks, "/") { // Validate keyspace/shard parses. @@ -77,34 +80,50 @@ func updateShardsToWatch() { log.Errorf("Could not parse keyspace/shard %q: %+v", ks, err) continue } - newShardsToWatch[k] = append(newShardsToWatch[k], s) + if !key.IsValidKeyRange(s) { + return fmt.Errorf("invalid key range %q while parsing clusters to watch", s) + } + // Parse the shard name into key range value. + keyRanges, err := key.ParseShardingSpec(s) + if err != nil { + return fmt.Errorf("could not parse shard name %q: %+v", s, err) + } + shardsToWatch[k] = append(shardsToWatch[k], keyRanges...) } else { - ctx, cancel := context.WithTimeout(context.Background(), topo.RemoteOperationTimeout) - defer cancel() - // Assume this is a keyspace and find all shards in keyspace. // Remove trailing slash if exists. ks = strings.TrimSuffix(ks, "/") - shards, err := ts.GetShardNames(ctx, ks) - if err != nil { - // Log the err and continue. - log.Errorf("Error fetching shards for keyspace: %v", ks) - continue - } - if len(shards) == 0 { - log.Errorf("Topo has no shards for ks: %v", ks) - continue - } - newShardsToWatch[ks] = shards + // We store the entire range of key range if nothing is specified. + shardsToWatch[ks] = []*topodatapb.KeyRange{key.NewCompleteKeyRange()} } } - if len(newShardsToWatch) == 0 { - log.Error("No keyspace/shards to watch") - return + + if len(shardsToWatch) == 0 { + log.Error("No keyspace/shards to watch, watching all keyspaces") } + return nil +} - shardsToWatchMu.Lock() - defer shardsToWatchMu.Unlock() - shardsToWatch = newShardsToWatch +// shouldWatchTablet checks if the given tablet is part of the watch list. +func shouldWatchTablet(tablet *topodatapb.Tablet) bool { + // If we are watching all keyspaces, then we want to watch this tablet too. + if len(shardsToWatch) == 0 { + return true + } + shardRanges, ok := shardsToWatch[tablet.GetKeyspace()] + // If we don't have the keyspace in our map, then this tablet + // doesn't need to be watched. + if !ok { + return false + } + // Get the tablet's key range, and check if + // it is part of the shard ranges we are watching. + kr := tablet.GetKeyRange() + for _, shardRange := range shardRanges { + if key.KeyRangeContainsKeyRange(shardRange, kr) { + return true + } + } + return false } // OpenTabletDiscovery opens the vitess topo if enables and returns a ticker @@ -117,7 +136,10 @@ func OpenTabletDiscovery() <-chan time.Time { log.Error(err) } // Parse --clusters_to_watch into a filter. - updateShardsToWatch() + err := initializeShardsToWatch() + if err != nil { + log.Fatalf("Error parsing --clusters-to-watch: %v", err) + } // We refresh all information from the topo once before we start the ticks to do // it on a timer. ctx, cancel := context.WithTimeout(context.Background(), topo.RemoteOperationTimeout) @@ -179,16 +201,10 @@ func refreshTabletsUsing(ctx context.Context, loader func(tabletAlias string), f // Filter tablets that should not be watched using shardsToWatch map. matchedTablets := make([]*topo.TabletInfo, 0, len(tablets)) func() { - shardsToWatchMu.Lock() - defer shardsToWatchMu.Unlock() for _, t := range tablets { - if len(shardsToWatch) > 0 { - _, ok := shardsToWatch[t.Tablet.Keyspace] - if !ok || !slices.Contains(shardsToWatch[t.Tablet.Keyspace], t.Tablet.Shard) { - continue // filter - } + if shouldWatchTablet(t.Tablet) { + matchedTablets = append(matchedTablets, t) } - matchedTablets = append(matchedTablets, t) } }() diff --git a/go/vt/vtorc/logic/tablet_discovery_test.go b/go/vt/vtorc/logic/tablet_discovery_test.go index 54284e8a017..4514ef81724 100644 --- a/go/vt/vtorc/logic/tablet_discovery_test.go +++ b/go/vt/vtorc/logic/tablet_discovery_test.go @@ -30,6 +30,7 @@ import ( "google.golang.org/protobuf/proto" "vitess.io/vitess/go/vt/external/golib/sqlutils" + "vitess.io/vitess/go/vt/key" topodatapb "vitess.io/vitess/go/vt/proto/topodata" "vitess.io/vitess/go/vt/proto/vttime" "vitess.io/vitess/go/vt/topo" @@ -102,60 +103,200 @@ var ( } ) -func TestUpdateShardsToWatch(t *testing.T) { +func TestShouldWatchTablet(t *testing.T) { oldClustersToWatch := clustersToWatch - oldTs := ts defer func() { clustersToWatch = oldClustersToWatch shardsToWatch = nil - ts = oldTs }() - // Create a memory topo-server and create the keyspace and shard records - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + testCases := []struct { + in []string + tablet *topodatapb.Tablet + expectedShouldWatch bool + }{ + { + in: []string{}, + tablet: &topodatapb.Tablet{ + Keyspace: keyspace, + Shard: shard, + }, + expectedShouldWatch: true, + }, + { + in: []string{keyspace}, + tablet: &topodatapb.Tablet{ + Keyspace: keyspace, + Shard: shard, + }, + expectedShouldWatch: true, + }, + { + in: []string{keyspace + "/-"}, + tablet: &topodatapb.Tablet{ + Keyspace: keyspace, + Shard: shard, + }, + expectedShouldWatch: true, + }, + { + in: []string{keyspace + "/" + shard}, + tablet: &topodatapb.Tablet{ + Keyspace: keyspace, + Shard: shard, + }, + expectedShouldWatch: true, + }, + { + in: []string{"ks/-70", "ks/70-"}, + tablet: &topodatapb.Tablet{ + Keyspace: "ks", + KeyRange: key.NewKeyRange([]byte{0x50}, []byte{0x70}), + }, + expectedShouldWatch: true, + }, + { + in: []string{"ks/-70", "ks/70-"}, + tablet: &topodatapb.Tablet{ + Keyspace: "ks", + KeyRange: key.NewKeyRange([]byte{0x40}, []byte{0x50}), + }, + expectedShouldWatch: true, + }, + { + in: []string{"ks/-70", "ks/70-"}, + tablet: &topodatapb.Tablet{ + Keyspace: "ks", + KeyRange: key.NewKeyRange([]byte{0x70}, []byte{0x90}), + }, + expectedShouldWatch: true, + }, + { + in: []string{"ks/-70", "ks/70-"}, + tablet: &topodatapb.Tablet{ + Keyspace: "ks", + KeyRange: key.NewKeyRange([]byte{0x60}, []byte{0x90}), + }, + expectedShouldWatch: false, + }, + { + in: []string{"ks/50-70"}, + tablet: &topodatapb.Tablet{ + Keyspace: "ks", + KeyRange: key.NewKeyRange([]byte{0x50}, []byte{0x70}), + }, + expectedShouldWatch: true, + }, + { + in: []string{"ks2/-70", "ks2/70-", "unknownKs/-", "ks/-80"}, + tablet: &topodatapb.Tablet{ + Keyspace: "ks", + KeyRange: key.NewKeyRange([]byte{0x60}, []byte{0x80}), + }, + expectedShouldWatch: true, + }, + { + in: []string{"ks2/-70", "ks2/70-", "unknownKs/-", "ks/-80"}, + tablet: &topodatapb.Tablet{ + Keyspace: "ks", + KeyRange: key.NewKeyRange([]byte{0x80}, []byte{0x90}), + }, + expectedShouldWatch: false, + }, + { + in: []string{"ks2/-70", "ks2/70-", "unknownKs/-", "ks/-80"}, + tablet: &topodatapb.Tablet{ + Keyspace: "ks", + KeyRange: key.NewKeyRange([]byte{0x90}, []byte{0xa0}), + }, + expectedShouldWatch: false, + }, + } - ts = memorytopo.NewServer(ctx, cell1) - _, err := ts.GetOrCreateShard(context.Background(), keyspace, shard) - require.NoError(t, err) + for _, tt := range testCases { + t.Run(fmt.Sprintf("%v-Tablet-%v-%v", strings.Join(tt.in, ","), tt.tablet.GetKeyspace(), tt.tablet.GetShard()), func(t *testing.T) { + clustersToWatch = tt.in + err := initializeShardsToWatch() + require.NoError(t, err) + assert.Equal(t, tt.expectedShouldWatch, shouldWatchTablet(tt.tablet)) + }) + } +} + +// TestInitializeShardsToWatch tests that we initialize the shardsToWatch map correctly +// using the `--clusters_to_watch` flag. +func TestInitializeShardsToWatch(t *testing.T) { + oldClustersToWatch := clustersToWatch + defer func() { + clustersToWatch = oldClustersToWatch + shardsToWatch = nil + }() testCases := []struct { - in []string - expected map[string][]string + in []string + expected map[string][]*topodatapb.KeyRange + expectedErr string }{ { in: []string{}, - expected: nil, + expected: map[string][]*topodatapb.KeyRange{}, }, { - in: []string{""}, - expected: map[string][]string{}, + in: []string{"unknownKs"}, + expected: map[string][]*topodatapb.KeyRange{ + "unknownKs": { + key.NewCompleteKeyRange(), + }, + }, }, { in: []string{"test/-"}, - expected: map[string][]string{ - "test": {"-"}, + expected: map[string][]*topodatapb.KeyRange{ + "test": { + key.NewCompleteKeyRange(), + }, + }, + }, + { + in: []string{"test/324"}, + expectedErr: `invalid key range "324" while parsing clusters to watch`, + }, + { + in: []string{"test/0"}, + expected: map[string][]*topodatapb.KeyRange{ + "test": { + key.NewCompleteKeyRange(), + }, }, }, { in: []string{"test/-", "test2/-80", "test2/80-"}, - expected: map[string][]string{ - "test": {"-"}, - "test2": {"-80", "80-"}, + expected: map[string][]*topodatapb.KeyRange{ + "test": { + key.NewCompleteKeyRange(), + }, + "test2": { + key.NewKeyRange(nil, []byte{0x80}), + key.NewKeyRange([]byte{0x80}, nil), + }, }, }, { - // confirm shards fetch from topo + // known keyspace in: []string{keyspace}, - expected: map[string][]string{ - keyspace: {shard}, + expected: map[string][]*topodatapb.KeyRange{ + keyspace: { + key.NewCompleteKeyRange(), + }, }, }, { - // confirm shards fetch from topo when keyspace has trailing-slash + // keyspace with trailing-slash in: []string{keyspace + "/"}, - expected: map[string][]string{ - keyspace: {shard}, + expected: map[string][]*topodatapb.KeyRange{ + keyspace: { + key.NewCompleteKeyRange(), + }, }, }, } @@ -163,10 +304,15 @@ func TestUpdateShardsToWatch(t *testing.T) { for _, testCase := range testCases { t.Run(strings.Join(testCase.in, ","), func(t *testing.T) { defer func() { - shardsToWatch = make(map[string][]string, 0) + shardsToWatch = make(map[string][]*topodatapb.KeyRange, 0) }() clustersToWatch = testCase.in - updateShardsToWatch() + err := initializeShardsToWatch() + if testCase.expectedErr != "" { + require.EqualError(t, err, testCase.expectedErr) + return + } + require.NoError(t, err) require.Equal(t, testCase.expected, shardsToWatch) }) } diff --git a/go/vt/vtorc/logic/vtorc.go b/go/vt/vtorc/logic/vtorc.go index 1fde6e31c0d..5ac5af50d47 100644 --- a/go/vt/vtorc/logic/vtorc.go +++ b/go/vt/vtorc/logic/vtorc.go @@ -326,12 +326,6 @@ func refreshAllInformation(ctx context.Context) error { return RefreshAllKeyspacesAndShards(ctx) }) - // Refresh shards to watch. - eg.Go(func() error { - updateShardsToWatch() - return nil - }) - // Refresh all tablets. eg.Go(func() error { return refreshAllTablets(ctx) From c2f4dcf80a86e092ca7ce43bd3eb5d5eb66cc999 Mon Sep 17 00:00:00 2001 From: Manan Gupta <35839558+GuptaManan100@users.noreply.github.com> Date: Tue, 28 Jan 2025 11:59:45 +0530 Subject: [PATCH 07/11] Refactor Disk Stall implementation and mark tablet not serving if disk is stalled (#17624) Signed-off-by: Manan Gupta --- .../replicationdata/replicationdata.pb.go | 30 ++++++--- .../replicationdata_vtproto.pb.go | 36 +++++++++++ go/vt/vtorc/inst/instance_dao.go | 7 ++- .../vttablet/tabletmanager/rpc_replication.go | 12 ++-- go/vt/vttablet/tabletmanager/tm_init.go | 12 +--- go/vt/vttablet/tabletserver/controller.go | 3 + .../disk_health_monitor.go | 39 +++++++++++- .../disk_health_monitor_test.go | 18 +++++- go/vt/vttablet/tabletserver/state_manager.go | 3 +- .../tabletserver/state_manager_test.go | 62 +++++++++++++------ go/vt/vttablet/tabletserver/tabletserver.go | 40 +++++++----- go/vt/vttablet/tabletservermock/controller.go | 6 ++ proto/replicationdata.proto | 1 + web/vtadmin/src/proto/vtadmin.d.ts | 6 ++ web/vtadmin/src/proto/vtadmin.js | 23 +++++++ 15 files changed, 232 insertions(+), 66 deletions(-) rename go/vt/vttablet/{tabletmanager => tabletserver}/disk_health_monitor.go (67%) rename go/vt/vttablet/{tabletmanager => tabletserver}/disk_health_monitor_test.go (85%) diff --git a/go/vt/proto/replicationdata/replicationdata.pb.go b/go/vt/proto/replicationdata/replicationdata.pb.go index d5462e1ea2b..bb973577d55 100644 --- a/go/vt/proto/replicationdata/replicationdata.pb.go +++ b/go/vt/proto/replicationdata/replicationdata.pb.go @@ -509,6 +509,7 @@ type FullStatus struct { SemiSyncWaitForReplicaCount uint32 `protobuf:"varint,20,opt,name=semi_sync_wait_for_replica_count,json=semiSyncWaitForReplicaCount,proto3" json:"semi_sync_wait_for_replica_count,omitempty"` SuperReadOnly bool `protobuf:"varint,21,opt,name=super_read_only,json=superReadOnly,proto3" json:"super_read_only,omitempty"` ReplicationConfiguration *Configuration `protobuf:"bytes,22,opt,name=replication_configuration,json=replicationConfiguration,proto3" json:"replication_configuration,omitempty"` + DiskStalled bool `protobuf:"varint,23,opt,name=disk_stalled,json=diskStalled,proto3" json:"disk_stalled,omitempty"` } func (x *FullStatus) Reset() { @@ -695,6 +696,13 @@ func (x *FullStatus) GetReplicationConfiguration() *Configuration { return nil } +func (x *FullStatus) GetDiskStalled() bool { + if x != nil { + return x.DiskStalled + } + return false +} + var File_replicationdata_proto protoreflect.FileDescriptor var file_replicationdata_proto_rawDesc = []byte{ @@ -782,7 +790,7 @@ var file_replicationdata_proto_rawDesc = []byte{ 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x66, 0x69, 0x6c, 0x65, 0x50, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x75, 0x75, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, - 0x55, 0x75, 0x69, 0x64, 0x22, 0xc8, 0x08, 0x0a, 0x0a, 0x46, 0x75, 0x6c, 0x6c, 0x53, 0x74, 0x61, + 0x55, 0x75, 0x69, 0x64, 0x22, 0xeb, 0x08, 0x0a, 0x0a, 0x46, 0x75, 0x6c, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x49, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x75, 0x75, 0x69, 0x64, 0x18, @@ -850,15 +858,17 @@ var file_replicationdata_proto_rawDesc = []byte{ 0x16, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x18, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2a, - 0x3b, 0x0a, 0x13, 0x53, 0x74, 0x6f, 0x70, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x12, 0x0a, 0x0e, 0x49, 0x4f, 0x41, 0x4e, 0x44, 0x53, - 0x51, 0x4c, 0x54, 0x48, 0x52, 0x45, 0x41, 0x44, 0x10, 0x00, 0x12, 0x10, 0x0a, 0x0c, 0x49, 0x4f, - 0x54, 0x48, 0x52, 0x45, 0x41, 0x44, 0x4f, 0x4e, 0x4c, 0x59, 0x10, 0x01, 0x42, 0x2e, 0x5a, 0x2c, - 0x76, 0x69, 0x74, 0x65, 0x73, 0x73, 0x2e, 0x69, 0x6f, 0x2f, 0x76, 0x69, 0x74, 0x65, 0x73, 0x73, - 0x2f, 0x67, 0x6f, 0x2f, 0x76, 0x74, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x72, 0x65, 0x70, - 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x64, 0x61, 0x74, 0x61, 0x62, 0x06, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x33, + 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, + 0x21, 0x0a, 0x0c, 0x64, 0x69, 0x73, 0x6b, 0x5f, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x65, 0x64, 0x18, + 0x17, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0b, 0x64, 0x69, 0x73, 0x6b, 0x53, 0x74, 0x61, 0x6c, 0x6c, + 0x65, 0x64, 0x2a, 0x3b, 0x0a, 0x13, 0x53, 0x74, 0x6f, 0x70, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x12, 0x0a, 0x0e, 0x49, 0x4f, 0x41, + 0x4e, 0x44, 0x53, 0x51, 0x4c, 0x54, 0x48, 0x52, 0x45, 0x41, 0x44, 0x10, 0x00, 0x12, 0x10, 0x0a, + 0x0c, 0x49, 0x4f, 0x54, 0x48, 0x52, 0x45, 0x41, 0x44, 0x4f, 0x4e, 0x4c, 0x59, 0x10, 0x01, 0x42, + 0x2e, 0x5a, 0x2c, 0x76, 0x69, 0x74, 0x65, 0x73, 0x73, 0x2e, 0x69, 0x6f, 0x2f, 0x76, 0x69, 0x74, + 0x65, 0x73, 0x73, 0x2f, 0x67, 0x6f, 0x2f, 0x76, 0x74, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, + 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x64, 0x61, 0x74, 0x61, 0x62, + 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/go/vt/proto/replicationdata/replicationdata_vtproto.pb.go b/go/vt/proto/replicationdata/replicationdata_vtproto.pb.go index b3d638a1327..a515397c065 100644 --- a/go/vt/proto/replicationdata/replicationdata_vtproto.pb.go +++ b/go/vt/proto/replicationdata/replicationdata_vtproto.pb.go @@ -142,6 +142,7 @@ func (m *FullStatus) CloneVT() *FullStatus { r.SemiSyncWaitForReplicaCount = m.SemiSyncWaitForReplicaCount r.SuperReadOnly = m.SuperReadOnly r.ReplicationConfiguration = m.ReplicationConfiguration.CloneVT() + r.DiskStalled = m.DiskStalled if len(m.unknownFields) > 0 { r.unknownFields = make([]byte, len(m.unknownFields)) copy(r.unknownFields, m.unknownFields) @@ -552,6 +553,18 @@ func (m *FullStatus) MarshalToSizedBufferVT(dAtA []byte) (int, error) { i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } + if m.DiskStalled { + i-- + if m.DiskStalled { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0xb8 + } if m.ReplicationConfiguration != nil { size, err := m.ReplicationConfiguration.MarshalToSizedBufferVT(dAtA[:i]) if err != nil { @@ -975,6 +988,9 @@ func (m *FullStatus) SizeVT() (n int) { l = m.ReplicationConfiguration.SizeVT() n += 2 + l + protohelpers.SizeOfVarint(uint64(l)) } + if m.DiskStalled { + n += 3 + } n += len(m.unknownFields) return n } @@ -2551,6 +2567,26 @@ func (m *FullStatus) UnmarshalVT(dAtA []byte) error { return err } iNdEx = postIndex + case 23: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field DiskStalled", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.DiskStalled = bool(v != 0) default: iNdEx = preIndex skippy, err := protohelpers.Skip(dAtA[iNdEx:]) diff --git a/go/vt/vtorc/inst/instance_dao.go b/go/vt/vtorc/inst/instance_dao.go index f92a15079dd..9e35e6e3e0b 100644 --- a/go/vt/vtorc/inst/instance_dao.go +++ b/go/vt/vtorc/inst/instance_dao.go @@ -206,9 +206,10 @@ func ReadTopologyInstanceBufferable(tabletAlias string, latency *stopwatch.Named fs, err = fullStatus(tabletAlias) if err != nil { - if config.GetStalledDiskPrimaryRecovery() && strings.Contains(err.Error(), "stalled disk") { - stalledDisk = true - } + goto Cleanup + } + if config.GetStalledDiskPrimaryRecovery() && fs.DiskStalled { + stalledDisk = true goto Cleanup } partialSuccess = true // We at least managed to read something from the server. diff --git a/go/vt/vttablet/tabletmanager/rpc_replication.go b/go/vt/vttablet/tabletmanager/rpc_replication.go index b27b25d87c6..dec94ee6f16 100644 --- a/go/vt/vttablet/tabletmanager/rpc_replication.go +++ b/go/vt/vttablet/tabletmanager/rpc_replication.go @@ -18,7 +18,6 @@ package tabletmanager import ( "context" - "errors" "fmt" "runtime" "strings" @@ -62,10 +61,13 @@ func (tm *TabletManager) FullStatus(ctx context.Context) (*replicationdatapb.Ful return nil, err } - // Return error if the disk is stalled or rejecting writes. - // Noop by default, must be enabled with the flag "disk-write-dir". - if tm.dhMonitor.IsDiskStalled() { - return nil, errors.New("stalled disk") + // Return if the disk is stalled or rejecting writes. + // If the disk is stalled, we can't be sure if reads will go through + // or not, so we should not run any reads either. + if tm.QueryServiceControl.IsDiskStalled() { + return &replicationdatapb.FullStatus{ + DiskStalled: true, + }, nil } // Server ID - "select @@global.server_id" diff --git a/go/vt/vttablet/tabletmanager/tm_init.go b/go/vt/vttablet/tabletmanager/tm_init.go index c22ea0a6e51..fbef04de357 100644 --- a/go/vt/vttablet/tabletmanager/tm_init.go +++ b/go/vt/vttablet/tabletmanager/tm_init.go @@ -95,11 +95,8 @@ var ( skipBuildInfoTags = "/.*/" initTags flagutil.StringMapValue - initTimeout = 1 * time.Minute - mysqlShutdownTimeout = mysqlctl.DefaultShutdownTimeout - stalledDiskWriteDir = "" - stalledDiskWriteTimeout = 30 * time.Second - stalledDiskWriteInterval = 5 * time.Second + initTimeout = 1 * time.Minute + mysqlShutdownTimeout = mysqlctl.DefaultShutdownTimeout ) func registerInitFlags(fs *pflag.FlagSet) { @@ -112,9 +109,6 @@ func registerInitFlags(fs *pflag.FlagSet) { fs.Var(&initTags, "init_tags", "(init parameter) comma separated list of key:value pairs used to tag the tablet") fs.DurationVar(&initTimeout, "init_timeout", initTimeout, "(init parameter) timeout to use for the init phase.") fs.DurationVar(&mysqlShutdownTimeout, "mysql-shutdown-timeout", mysqlShutdownTimeout, "timeout to use when MySQL is being shut down.") - fs.StringVar(&stalledDiskWriteDir, "disk-write-dir", stalledDiskWriteDir, "if provided, tablet will attempt to write a file to this directory to check if the disk is stalled") - fs.DurationVar(&stalledDiskWriteTimeout, "disk-write-timeout", stalledDiskWriteTimeout, "if writes exceed this duration, the disk is considered stalled") - fs.DurationVar(&stalledDiskWriteInterval, "disk-write-interval", stalledDiskWriteInterval, "how often to write to the disk to check whether it is stalled") } var ( @@ -170,7 +164,6 @@ type TabletManager struct { VREngine *vreplication.Engine VDiffEngine *vdiff.Engine Env *vtenv.Environment - dhMonitor DiskHealthMonitor // tmc is used to run an RPC against other vttablets. tmc tmclient.TabletManagerClient @@ -379,7 +372,6 @@ func (tm *TabletManager) Start(tablet *topodatapb.Tablet, config *tabletenv.Tabl tm.tmc = tmclient.NewTabletManagerClient() tm.tmState = newTMState(tm, tablet) tm.actionSema = semaphore.NewWeighted(1) - tm.dhMonitor = newDiskHealthMonitor(tm.BatchCtx) tm._waitForGrantsComplete = make(chan struct{}) tm.baseTabletType = tablet.Type diff --git a/go/vt/vttablet/tabletserver/controller.go b/go/vt/vttablet/tabletserver/controller.go index c4a4bef99fc..ab2875ae27b 100644 --- a/go/vt/vttablet/tabletserver/controller.go +++ b/go/vt/vttablet/tabletserver/controller.go @@ -122,6 +122,9 @@ type Controller interface { // SetDemotePrimaryStalled marks that demote primary is stalled in the state manager. SetDemotePrimaryStalled() + + // IsDiskStalled returns if the disk is stalled. + IsDiskStalled() bool } // Ensure TabletServer satisfies Controller interface. diff --git a/go/vt/vttablet/tabletmanager/disk_health_monitor.go b/go/vt/vttablet/tabletserver/disk_health_monitor.go similarity index 67% rename from go/vt/vttablet/tabletmanager/disk_health_monitor.go rename to go/vt/vttablet/tabletserver/disk_health_monitor.go index e35bc662a12..f477f7fd30c 100644 --- a/go/vt/vttablet/tabletmanager/disk_health_monitor.go +++ b/go/vt/vttablet/tabletserver/disk_health_monitor.go @@ -1,4 +1,20 @@ -package tabletmanager +/* +Copyright 2024 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tabletserver import ( "context" @@ -7,8 +23,29 @@ import ( "strconv" "sync" "time" + + "github.com/spf13/pflag" + + "vitess.io/vitess/go/vt/servenv" ) +var ( + stalledDiskWriteDir = "" + stalledDiskWriteTimeout = 30 * time.Second + stalledDiskWriteInterval = 5 * time.Second +) + +func init() { + servenv.OnParseFor("vtcombo", registerInitFlags) + servenv.OnParseFor("vttablet", registerInitFlags) +} + +func registerInitFlags(fs *pflag.FlagSet) { + fs.StringVar(&stalledDiskWriteDir, "disk-write-dir", stalledDiskWriteDir, "if provided, tablet will attempt to write a file to this directory to check if the disk is stalled") + fs.DurationVar(&stalledDiskWriteTimeout, "disk-write-timeout", stalledDiskWriteTimeout, "if writes exceed this duration, the disk is considered stalled") + fs.DurationVar(&stalledDiskWriteInterval, "disk-write-interval", stalledDiskWriteInterval, "how often to write to the disk to check whether it is stalled") +} + type DiskHealthMonitor interface { // IsDiskStalled returns true if the disk is stalled or rejecting writes. IsDiskStalled() bool diff --git a/go/vt/vttablet/tabletmanager/disk_health_monitor_test.go b/go/vt/vttablet/tabletserver/disk_health_monitor_test.go similarity index 85% rename from go/vt/vttablet/tabletmanager/disk_health_monitor_test.go rename to go/vt/vttablet/tabletserver/disk_health_monitor_test.go index 68930f3061d..8b47e40ee79 100644 --- a/go/vt/vttablet/tabletmanager/disk_health_monitor_test.go +++ b/go/vt/vttablet/tabletserver/disk_health_monitor_test.go @@ -1,4 +1,20 @@ -package tabletmanager +/* +Copyright 2024 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tabletserver import ( "context" diff --git a/go/vt/vttablet/tabletserver/state_manager.go b/go/vt/vttablet/tabletserver/state_manager.go index 4512b26f177..0ccd0e42735 100644 --- a/go/vt/vttablet/tabletserver/state_manager.go +++ b/go/vt/vttablet/tabletserver/state_manager.go @@ -97,6 +97,7 @@ type stateManager struct { replHealthy bool demotePrimaryStalled bool lameduck bool + diskHealthMonitor DiskHealthMonitor alsoAllow []topodatapb.TabletType reason string transitionErr error @@ -777,7 +778,7 @@ func (sm *stateManager) IsServing() bool { } func (sm *stateManager) isServingLocked() bool { - return sm.state == StateServing && sm.wantState == StateServing && sm.replHealthy && !sm.demotePrimaryStalled && !sm.lameduck + return sm.state == StateServing && sm.wantState == StateServing && sm.replHealthy && !sm.demotePrimaryStalled && !sm.lameduck && !sm.diskHealthMonitor.IsDiskStalled() } func (sm *stateManager) AppendDetails(details []*kv) []*kv { diff --git a/go/vt/vttablet/tabletserver/state_manager_test.go b/go/vt/vttablet/tabletserver/state_manager_test.go index f8059d6edea..99a3e7e681d 100644 --- a/go/vt/vttablet/tabletserver/state_manager_test.go +++ b/go/vt/vttablet/tabletserver/state_manager_test.go @@ -41,7 +41,9 @@ import ( var testNow = time.Now() func TestStateManagerStateByName(t *testing.T) { - sm := &stateManager{} + sm := &stateManager{ + diskHealthMonitor: newNoopDiskHealthMonitor(), + } sm.replHealthy = true sm.wantState = StateServing @@ -147,6 +149,29 @@ func TestStateManagerUnservePrimary(t *testing.T) { assert.Equal(t, StateNotServing, sm.state) } +type testDiskMonitor struct { + isDiskStalled bool +} + +func (t *testDiskMonitor) IsDiskStalled() bool { + return t.isDiskStalled +} + +// TestIsServingLocked tests isServingLocked() functionality. +func TestIsServingLocked(t *testing.T) { + sm := newTestStateManager() + defer sm.StopService() + tdm := &testDiskMonitor{isDiskStalled: false} + sm.diskHealthMonitor = tdm + + err := sm.SetServingType(topodatapb.TabletType_REPLICA, testNow, StateServing, "") + require.NoError(t, err) + require.True(t, sm.isServingLocked()) + + tdm.isDiskStalled = true + require.False(t, sm.isServingLocked()) +} + func TestStateManagerUnserveNonPrimary(t *testing.T) { sm := newTestStateManager() defer sm.StopService() @@ -778,23 +803,24 @@ func newTestStateManager() *stateManager { parser := sqlparser.NewTestParser() env := tabletenv.NewEnv(vtenv.NewTestEnv(), cfg, "StateManagerTest") sm := &stateManager{ - statelessql: NewQueryList("stateless", parser), - statefulql: NewQueryList("stateful", parser), - olapql: NewQueryList("olap", parser), - hs: newHealthStreamer(env, &topodatapb.TabletAlias{}, schema.NewEngine(env)), - se: &testSchemaEngine{}, - rt: &testReplTracker{lag: 1 * time.Second}, - vstreamer: &testSubcomponent{}, - tracker: &testSubcomponent{}, - watcher: &testSubcomponent{}, - qe: &testQueryEngine{}, - txThrottler: &testTxThrottler{}, - te: &testTxEngine{}, - messager: &testSubcomponent{}, - ddle: &testOnlineDDLExecutor{}, - throttler: &testLagThrottler{}, - tableGC: &testTableGC{}, - rw: newRequestsWaiter(), + statelessql: NewQueryList("stateless", parser), + statefulql: NewQueryList("stateful", parser), + olapql: NewQueryList("olap", parser), + hs: newHealthStreamer(env, &topodatapb.TabletAlias{}, schema.NewEngine(env)), + se: &testSchemaEngine{}, + rt: &testReplTracker{lag: 1 * time.Second}, + vstreamer: &testSubcomponent{}, + tracker: &testSubcomponent{}, + watcher: &testSubcomponent{}, + qe: &testQueryEngine{}, + txThrottler: &testTxThrottler{}, + te: &testTxEngine{}, + messager: &testSubcomponent{}, + ddle: &testOnlineDDLExecutor{}, + diskHealthMonitor: newNoopDiskHealthMonitor(), + throttler: &testLagThrottler{}, + tableGC: &testTableGC{}, + rw: newRequestsWaiter(), } sm.Init(env, &querypb.Target{}) sm.hs.InitDBConfig(&querypb.Target{}) diff --git a/go/vt/vttablet/tabletserver/tabletserver.go b/go/vt/vttablet/tabletserver/tabletserver.go index 30f73d2d818..b65b3949354 100644 --- a/go/vt/vttablet/tabletserver/tabletserver.go +++ b/go/vt/vttablet/tabletserver/tabletserver.go @@ -190,23 +190,24 @@ func NewTabletServer(ctx context.Context, env *vtenv.Environment, name string, c tsv.onlineDDLExecutor = onlineddl.NewExecutor(tsv, alias, topoServer, tsv.lagThrottler, tabletTypeFunc, tsv.onlineDDLExecutorToggleTableBuffer, tsv.tableGC.RequestChecks, tsv.te.preparedPool.IsEmptyForTable) tsv.sm = &stateManager{ - statelessql: tsv.statelessql, - statefulql: tsv.statefulql, - olapql: tsv.olapql, - hs: tsv.hs, - se: tsv.se, - rt: tsv.rt, - vstreamer: tsv.vstreamer, - tracker: tsv.tracker, - watcher: tsv.watcher, - qe: tsv.qe, - txThrottler: tsv.txThrottler, - te: tsv.te, - messager: tsv.messager, - ddle: tsv.onlineDDLExecutor, - throttler: tsv.lagThrottler, - tableGC: tsv.tableGC, - rw: newRequestsWaiter(), + statelessql: tsv.statelessql, + statefulql: tsv.statefulql, + olapql: tsv.olapql, + hs: tsv.hs, + se: tsv.se, + rt: tsv.rt, + vstreamer: tsv.vstreamer, + tracker: tsv.tracker, + watcher: tsv.watcher, + qe: tsv.qe, + txThrottler: tsv.txThrottler, + te: tsv.te, + messager: tsv.messager, + ddle: tsv.onlineDDLExecutor, + throttler: tsv.lagThrottler, + tableGC: tsv.tableGC, + rw: newRequestsWaiter(), + diskHealthMonitor: newDiskHealthMonitor(ctx), } tsv.exporter.NewGaugeFunc("TabletState", "Tablet server state", func() int64 { return int64(tsv.sm.State()) }) @@ -767,6 +768,11 @@ func (tsv *TabletServer) SetDemotePrimaryStalled() { tsv.BroadcastHealth() } +// IsDiskStalled returns if the disk is stalled or not. +func (tsv *TabletServer) IsDiskStalled() bool { + return tsv.sm.diskHealthMonitor.IsDiskStalled() +} + // CreateTransaction creates the metadata for a 2PC transaction. func (tsv *TabletServer) CreateTransaction(ctx context.Context, target *querypb.Target, dtid string, participants []*querypb.Target) (err error) { return tsv.execRequest( diff --git a/go/vt/vttablet/tabletservermock/controller.go b/go/vt/vttablet/tabletservermock/controller.go index a5242751454..21b38755302 100644 --- a/go/vt/vttablet/tabletservermock/controller.go +++ b/go/vt/vttablet/tabletservermock/controller.go @@ -279,6 +279,12 @@ func (tqsc *Controller) SetDemotePrimaryStalled() { tqsc.MethodCalled["SetDemotePrimaryStalled"] = true } +// IsDiskStalled is part of the tabletserver.Controller interface +func (tqsc *Controller) IsDiskStalled() bool { + tqsc.MethodCalled["IsDiskStalled"] = true + return false +} + // EnterLameduck implements tabletserver.Controller. func (tqsc *Controller) EnterLameduck() { tqsc.mu.Lock() diff --git a/proto/replicationdata.proto b/proto/replicationdata.proto index e788fc64bc3..eba4d323ee6 100644 --- a/proto/replicationdata.proto +++ b/proto/replicationdata.proto @@ -105,4 +105,5 @@ message FullStatus { uint32 semi_sync_wait_for_replica_count = 20; bool super_read_only = 21; replicationdata.Configuration replication_configuration = 22; + bool disk_stalled = 23; } diff --git a/web/vtadmin/src/proto/vtadmin.d.ts b/web/vtadmin/src/proto/vtadmin.d.ts index 4411c436083..5a8859b90e7 100644 --- a/web/vtadmin/src/proto/vtadmin.d.ts +++ b/web/vtadmin/src/proto/vtadmin.d.ts @@ -48595,6 +48595,9 @@ export namespace replicationdata { /** FullStatus replication_configuration */ replication_configuration?: (replicationdata.IConfiguration|null); + + /** FullStatus disk_stalled */ + disk_stalled?: (boolean|null); } /** Represents a FullStatus. */ @@ -48672,6 +48675,9 @@ export namespace replicationdata { /** FullStatus replication_configuration. */ public replication_configuration?: (replicationdata.IConfiguration|null); + /** FullStatus disk_stalled. */ + public disk_stalled: boolean; + /** * Creates a new FullStatus instance using the specified properties. * @param [properties] Properties to set diff --git a/web/vtadmin/src/proto/vtadmin.js b/web/vtadmin/src/proto/vtadmin.js index 457ae6e4214..cafe51c0058 100644 --- a/web/vtadmin/src/proto/vtadmin.js +++ b/web/vtadmin/src/proto/vtadmin.js @@ -118108,6 +118108,7 @@ export const replicationdata = $root.replicationdata = (() => { * @property {number|null} [semi_sync_wait_for_replica_count] FullStatus semi_sync_wait_for_replica_count * @property {boolean|null} [super_read_only] FullStatus super_read_only * @property {replicationdata.IConfiguration|null} [replication_configuration] FullStatus replication_configuration + * @property {boolean|null} [disk_stalled] FullStatus disk_stalled */ /** @@ -118301,6 +118302,14 @@ export const replicationdata = $root.replicationdata = (() => { */ FullStatus.prototype.replication_configuration = null; + /** + * FullStatus disk_stalled. + * @member {boolean} disk_stalled + * @memberof replicationdata.FullStatus + * @instance + */ + FullStatus.prototype.disk_stalled = false; + /** * Creates a new FullStatus instance using the specified properties. * @function create @@ -118369,6 +118378,8 @@ export const replicationdata = $root.replicationdata = (() => { writer.uint32(/* id 21, wireType 0 =*/168).bool(message.super_read_only); if (message.replication_configuration != null && Object.hasOwnProperty.call(message, "replication_configuration")) $root.replicationdata.Configuration.encode(message.replication_configuration, writer.uint32(/* id 22, wireType 2 =*/178).fork()).ldelim(); + if (message.disk_stalled != null && Object.hasOwnProperty.call(message, "disk_stalled")) + writer.uint32(/* id 23, wireType 0 =*/184).bool(message.disk_stalled); return writer; }; @@ -118491,6 +118502,10 @@ export const replicationdata = $root.replicationdata = (() => { message.replication_configuration = $root.replicationdata.Configuration.decode(reader, reader.uint32()); break; } + case 23: { + message.disk_stalled = reader.bool(); + break; + } default: reader.skipType(tag & 7); break; @@ -118598,6 +118613,9 @@ export const replicationdata = $root.replicationdata = (() => { if (error) return "replication_configuration." + error; } + if (message.disk_stalled != null && message.hasOwnProperty("disk_stalled")) + if (typeof message.disk_stalled !== "boolean") + return "disk_stalled: boolean expected"; return null; }; @@ -118673,6 +118691,8 @@ export const replicationdata = $root.replicationdata = (() => { throw TypeError(".replicationdata.FullStatus.replication_configuration: object expected"); message.replication_configuration = $root.replicationdata.Configuration.fromObject(object.replication_configuration); } + if (object.disk_stalled != null) + message.disk_stalled = Boolean(object.disk_stalled); return message; }; @@ -118716,6 +118736,7 @@ export const replicationdata = $root.replicationdata = (() => { object.semi_sync_wait_for_replica_count = 0; object.super_read_only = false; object.replication_configuration = null; + object.disk_stalled = false; } if (message.server_id != null && message.hasOwnProperty("server_id")) object.server_id = message.server_id; @@ -118764,6 +118785,8 @@ export const replicationdata = $root.replicationdata = (() => { object.super_read_only = message.super_read_only; if (message.replication_configuration != null && message.hasOwnProperty("replication_configuration")) object.replication_configuration = $root.replicationdata.Configuration.toObject(message.replication_configuration, options); + if (message.disk_stalled != null && message.hasOwnProperty("disk_stalled")) + object.disk_stalled = message.disk_stalled; return object; }; From 19890cfb8ef7bd486e475b58a713d89fc2c29282 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Taylor?= Date: Tue, 28 Jan 2025 11:09:53 +0100 Subject: [PATCH 08/11] Optimise AST rewriting (#17623) Signed-off-by: Andres Taylor Signed-off-by: Harshit Gangal Co-authored-by: Harshit Gangal --- go/vt/sqlparser/ast_rewriting.go | 557 -------------- go/vt/sqlparser/ast_rewriting_test.go | 565 -------------- go/vt/sqlparser/bind_var_needs.go | 18 +- go/vt/sqlparser/normalizer.go | 786 ++++++++++++++++---- go/vt/sqlparser/normalizer_test.go | 612 ++++++++++++++- go/vt/sqlparser/redact_query.go | 4 +- go/vt/sqlparser/utils.go | 5 +- go/vt/vtgate/executor_test.go | 24 + go/vt/vtgate/planbuilder/builder.go | 6 +- go/vt/vtgate/planbuilder/simplifier_test.go | 12 +- go/vt/vtgate/semantics/typer_test.go | 17 +- 11 files changed, 1279 insertions(+), 1327 deletions(-) delete mode 100644 go/vt/sqlparser/ast_rewriting.go delete mode 100644 go/vt/sqlparser/ast_rewriting_test.go diff --git a/go/vt/sqlparser/ast_rewriting.go b/go/vt/sqlparser/ast_rewriting.go deleted file mode 100644 index 05e7e290fc1..00000000000 --- a/go/vt/sqlparser/ast_rewriting.go +++ /dev/null @@ -1,557 +0,0 @@ -/* -Copyright 2020 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package sqlparser - -import ( - "strconv" - "strings" - - querypb "vitess.io/vitess/go/vt/proto/query" - vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" - "vitess.io/vitess/go/vt/sysvars" - "vitess.io/vitess/go/vt/vterrors" -) - -var HasValueSubQueryBaseName = []byte("__sq_has_values") - -// SQLSelectLimitUnset default value for sql_select_limit not set. -const SQLSelectLimitUnset = -1 - -// RewriteASTResult contains the rewritten ast and meta information about it -type RewriteASTResult struct { - *BindVarNeeds - AST Statement // The rewritten AST -} - -type VSchemaViews interface { - FindView(name TableName) TableStatement -} - -// PrepareAST will normalize the query -func PrepareAST( - in Statement, - reservedVars *ReservedVars, - bindVars map[string]*querypb.BindVariable, - parameterize bool, - keyspace string, - selectLimit int, - setVarComment string, - sysVars map[string]string, - fkChecksState *bool, - views VSchemaViews, -) (*RewriteASTResult, error) { - if parameterize { - err := Normalize(in, reservedVars, bindVars) - if err != nil { - return nil, err - } - } - return RewriteAST(in, keyspace, selectLimit, setVarComment, sysVars, fkChecksState, views) -} - -// RewriteAST rewrites the whole AST, replacing function calls and adding column aliases to queries. -// SET_VAR comments are also added to the AST if required. -func RewriteAST( - in Statement, - keyspace string, - selectLimit int, - setVarComment string, - sysVars map[string]string, - fkChecksState *bool, - views VSchemaViews, -) (*RewriteASTResult, error) { - er := newASTRewriter(keyspace, selectLimit, setVarComment, sysVars, fkChecksState, views) - er.shouldRewriteDatabaseFunc = shouldRewriteDatabaseFunc(in) - result := SafeRewrite(in, er.rewriteDown, er.rewriteUp) - if er.err != nil { - return nil, er.err - } - - out, ok := result.(Statement) - if !ok { - return nil, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "statement rewriting returned a non statement: %s", String(out)) - } - - r := &RewriteASTResult{ - AST: out, - BindVarNeeds: er.bindVars, - } - return r, nil -} - -func shouldRewriteDatabaseFunc(in Statement) bool { - selct, ok := in.(*Select) - if !ok { - return false - } - if len(selct.From) != 1 { - return false - } - aliasedTable, ok := selct.From[0].(*AliasedTableExpr) - if !ok { - return false - } - tableName, ok := aliasedTable.Expr.(TableName) - if !ok { - return false - } - return tableName.Name.String() == "dual" -} - -type astRewriter struct { - bindVars *BindVarNeeds - shouldRewriteDatabaseFunc bool - err error - - // we need to know this to make a decision if we can safely rewrite JOIN USING => JOIN ON - hasStarInSelect bool - - keyspace string - selectLimit int - setVarComment string - fkChecksState *bool - sysVars map[string]string - views VSchemaViews -} - -func newASTRewriter(keyspace string, selectLimit int, setVarComment string, sysVars map[string]string, fkChecksState *bool, views VSchemaViews) *astRewriter { - return &astRewriter{ - bindVars: &BindVarNeeds{}, - keyspace: keyspace, - selectLimit: selectLimit, - setVarComment: setVarComment, - fkChecksState: fkChecksState, - sysVars: sysVars, - views: views, - } -} - -const ( - // LastInsertIDName is a reserved bind var name for last_insert_id() - LastInsertIDName = "__lastInsertId" - - // DBVarName is a reserved bind var name for database() - DBVarName = "__vtdbname" - - // FoundRowsName is a reserved bind var name for found_rows() - FoundRowsName = "__vtfrows" - - // RowCountName is a reserved bind var name for row_count() - RowCountName = "__vtrcount" - - // UserDefinedVariableName is what we prepend bind var names for user defined variables - UserDefinedVariableName = "__vtudv" -) - -func (er *astRewriter) rewriteAliasedExpr(node *AliasedExpr) (*BindVarNeeds, error) { - inner := newASTRewriter(er.keyspace, er.selectLimit, er.setVarComment, er.sysVars, nil, er.views) - inner.shouldRewriteDatabaseFunc = er.shouldRewriteDatabaseFunc - tmp := SafeRewrite(node.Expr, inner.rewriteDown, inner.rewriteUp) - newExpr, ok := tmp.(Expr) - if !ok { - return nil, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "failed to rewrite AST. function expected to return Expr returned a %s", String(tmp)) - } - node.Expr = newExpr - return inner.bindVars, nil -} - -func (er *astRewriter) rewriteDown(node SQLNode, _ SQLNode) bool { - switch node := node.(type) { - case *Select: - er.visitSelect(node) - case *PrepareStmt, *ExecuteStmt: - return false // nothing to rewrite here. - } - return true -} - -func (er *astRewriter) rewriteUp(cursor *Cursor) bool { - // Add SET_VAR comment to this node if it supports it and is needed - if supportOptimizerHint, supportsOptimizerHint := cursor.Node().(SupportOptimizerHint); supportsOptimizerHint { - if er.setVarComment != "" { - newComments, err := supportOptimizerHint.GetParsedComments().AddQueryHint(er.setVarComment) - if err != nil { - er.err = err - return false - } - supportOptimizerHint.SetComments(newComments) - } - if er.fkChecksState != nil { - newComments := supportOptimizerHint.GetParsedComments().SetMySQLSetVarValue(sysvars.ForeignKeyChecks, FkChecksStateString(er.fkChecksState)) - supportOptimizerHint.SetComments(newComments) - } - } - - switch node := cursor.Node().(type) { - case *Union: - er.rewriteUnion(node) - case *FuncExpr: - er.funcRewrite(cursor, node) - case *Variable: - er.rewriteVariable(cursor, node) - case *Subquery: - er.unnestSubQueries(cursor, node) - case *NotExpr: - er.rewriteNotExpr(cursor, node) - case *AliasedTableExpr: - er.rewriteAliasedTable(cursor, node) - case *ShowBasic: - er.rewriteShowBasic(node) - case *ExistsExpr: - er.existsRewrite(cursor, node) - case DistinctableAggr: - er.rewriteDistinctableAggr(cursor, node) - } - return true -} - -func (er *astRewriter) rewriteUnion(node *Union) { - // set select limit if explicitly not set when sql_select_limit is set on the connection. - if er.selectLimit > 0 && node.Limit == nil { - node.Limit = &Limit{Rowcount: NewIntLiteral(strconv.Itoa(er.selectLimit))} - } -} - -func (er *astRewriter) rewriteAliasedTable(cursor *Cursor, node *AliasedTableExpr) { - aliasTableName, ok := node.Expr.(TableName) - if !ok { - return - } - - // Qualifier should not be added to dual table - tblName := aliasTableName.Name.String() - if tblName == "dual" { - return - } - - if SystemSchema(er.keyspace) { - if aliasTableName.Qualifier.IsEmpty() { - aliasTableName.Qualifier = NewIdentifierCS(er.keyspace) - node.Expr = aliasTableName - cursor.Replace(node) - } - return - } - - // Could we be dealing with a view? - if er.views == nil { - return - } - view := er.views.FindView(aliasTableName) - if view == nil { - return - } - - // Aha! It's a view. Let's replace it with a derived table - node.Expr = &DerivedTable{Select: Clone(view)} // TODO: this is a bit hacky. We want to update the schema def so it contains new types - if node.As.IsEmpty() { - node.As = NewIdentifierCS(tblName) - } -} - -func (er *astRewriter) rewriteShowBasic(node *ShowBasic) { - if node.Command == VariableGlobal || node.Command == VariableSession { - varsToAdd := sysvars.GetInterestingVariables() - for _, sysVar := range varsToAdd { - er.bindVars.AddSysVar(sysVar) - } - } -} - -func (er *astRewriter) rewriteNotExpr(cursor *Cursor, node *NotExpr) { - switch inner := node.Expr.(type) { - case *ComparisonExpr: - // not col = 42 => col != 42 - // not col > 42 => col <= 42 - // etc - canChange, inverse := inverseOp(inner.Operator) - if canChange { - inner.Operator = inverse - cursor.Replace(inner) - } - case *NotExpr: - // not not true => true - cursor.Replace(inner.Expr) - case BoolVal: - // not true => false - inner = !inner - cursor.Replace(inner) - } -} - -func (er *astRewriter) rewriteVariable(cursor *Cursor, node *Variable) { - // Iff we are in SET, we want to change the scope of variables if a modifier has been set - // and only on the lhs of the assignment: - // set session sql_mode = @someElse - // here we need to change the scope of `sql_mode` and not of `@someElse` - if v, isSet := cursor.Parent().(*SetExpr); isSet && v.Var == node { - return - } - // no rewriting for global scope variable. - // this should be returned from the underlying database. - switch node.Scope { - case VariableScope: - er.udvRewrite(cursor, node) - case SessionScope, NextTxScope: - er.sysVarRewrite(cursor, node) - } -} - -func (er *astRewriter) visitSelect(node *Select) { - for _, col := range node.SelectExprs { - if _, hasStar := col.(*StarExpr); hasStar { - er.hasStarInSelect = true - continue - } - - aliasedExpr, ok := col.(*AliasedExpr) - if !ok || aliasedExpr.As.NotEmpty() { - continue - } - buf := NewTrackedBuffer(nil) - aliasedExpr.Expr.Format(buf) - // select last_insert_id() -> select :__lastInsertId as `last_insert_id()` - innerBindVarNeeds, err := er.rewriteAliasedExpr(aliasedExpr) - if err != nil { - er.err = err - return - } - if innerBindVarNeeds.HasRewrites() { - aliasedExpr.As = NewIdentifierCI(buf.String()) - } - er.bindVars.MergeWith(innerBindVarNeeds) - - } - // set select limit if explicitly not set when sql_select_limit is set on the connection. - if er.selectLimit > 0 && node.Limit == nil { - node.Limit = &Limit{Rowcount: NewIntLiteral(strconv.Itoa(er.selectLimit))} - } -} - -func inverseOp(i ComparisonExprOperator) (bool, ComparisonExprOperator) { - switch i { - case EqualOp: - return true, NotEqualOp - case LessThanOp: - return true, GreaterEqualOp - case GreaterThanOp: - return true, LessEqualOp - case LessEqualOp: - return true, GreaterThanOp - case GreaterEqualOp: - return true, LessThanOp - case NotEqualOp: - return true, EqualOp - case InOp: - return true, NotInOp - case NotInOp: - return true, InOp - case LikeOp: - return true, NotLikeOp - case NotLikeOp: - return true, LikeOp - case RegexpOp: - return true, NotRegexpOp - case NotRegexpOp: - return true, RegexpOp - } - - return false, i -} - -func (er *astRewriter) sysVarRewrite(cursor *Cursor, node *Variable) { - lowered := node.Name.Lowered() - - var found bool - if er.sysVars != nil { - _, found = er.sysVars[lowered] - } - - switch lowered { - case sysvars.Autocommit.Name, - sysvars.Charset.Name, - sysvars.ClientFoundRows.Name, - sysvars.DDLStrategy.Name, - sysvars.MigrationContext.Name, - sysvars.Names.Name, - sysvars.TransactionMode.Name, - sysvars.ReadAfterWriteGTID.Name, - sysvars.ReadAfterWriteTimeOut.Name, - sysvars.SessionEnableSystemSettings.Name, - sysvars.SessionTrackGTIDs.Name, - sysvars.SessionUUID.Name, - sysvars.SkipQueryPlanCache.Name, - sysvars.Socket.Name, - sysvars.SQLSelectLimit.Name, - sysvars.Version.Name, - sysvars.VersionComment.Name, - sysvars.QueryTimeout.Name, - sysvars.Workload.Name: - found = true - } - - if found { - cursor.Replace(bindVarExpression("__vt" + lowered)) - er.bindVars.AddSysVar(lowered) - } -} - -func (er *astRewriter) udvRewrite(cursor *Cursor, node *Variable) { - udv := strings.ToLower(node.Name.CompliantName()) - cursor.Replace(bindVarExpression(UserDefinedVariableName + udv)) - er.bindVars.AddUserDefVar(udv) -} - -var funcRewrites = map[string]string{ - "last_insert_id": LastInsertIDName, - "database": DBVarName, - "schema": DBVarName, - "found_rows": FoundRowsName, - "row_count": RowCountName, -} - -func (er *astRewriter) funcRewrite(cursor *Cursor, node *FuncExpr) { - lowered := node.Name.Lowered() - if lowered == "last_insert_id" && len(node.Exprs) > 0 { - // if we are dealing with is LAST_INSERT_ID() with an argument, we don't need to rewrite it. - // with an argument, this is an identity function that will update the session state and - // sets the correct fields in the OK TCP packet that we send back - return - } - bindVar, found := funcRewrites[lowered] - if !found || (bindVar == DBVarName && !er.shouldRewriteDatabaseFunc) { - return - } - if len(node.Exprs) > 0 { - er.err = vterrors.Errorf(vtrpcpb.Code_UNIMPLEMENTED, "Argument to %s() not supported", lowered) - return - } - cursor.Replace(bindVarExpression(bindVar)) - er.bindVars.AddFuncResult(bindVar) -} - -func (er *astRewriter) unnestSubQueries(cursor *Cursor, subquery *Subquery) { - if _, isExists := cursor.Parent().(*ExistsExpr); isExists { - return - } - sel, isSimpleSelect := subquery.Select.(*Select) - if !isSimpleSelect { - return - } - - if len(sel.SelectExprs) != 1 || - len(sel.OrderBy) != 0 || - sel.GroupBy != nil || - len(sel.From) != 1 || - sel.Where != nil || - sel.Having != nil || - sel.Limit != nil || sel.Lock != NoLock { - return - } - - aliasedTable, ok := sel.From[0].(*AliasedTableExpr) - if !ok { - return - } - table, ok := aliasedTable.Expr.(TableName) - if !ok || table.Name.String() != "dual" { - return - } - expr, ok := sel.SelectExprs[0].(*AliasedExpr) - if !ok { - return - } - _, isColName := expr.Expr.(*ColName) - if isColName { - // If we find a single col-name in a `dual` subquery, we can be pretty sure the user is returning a column - // already projected. - // `select 1 as x, (select x)` - // is perfectly valid - any aliased columns to the left are available inside subquery scopes - return - } - er.bindVars.NoteRewrite() - // we need to make sure that the inner expression also gets rewritten, - // so we fire off another rewriter traversal here - rewritten := SafeRewrite(expr.Expr, er.rewriteDown, er.rewriteUp) - - // Here we need to handle the subquery rewrite in case in occurs in an IN clause - // For example, SELECT id FROM user WHERE id IN (SELECT 1 FROM DUAL) - // Here we cannot rewrite the query to SELECT id FROM user WHERE id IN 1, since that is syntactically wrong - // We must rewrite it to SELECT id FROM user WHERE id IN (1) - // Find more cases in the test file - rewrittenExpr, isExpr := rewritten.(Expr) - _, isColTuple := rewritten.(ColTuple) - comparisonExpr, isCompExpr := cursor.Parent().(*ComparisonExpr) - // Check that the parent is a comparison operator with IN or NOT IN operation. - // Also, if rewritten is already a ColTuple (like a subquery), then we do not need this - // We also need to check that rewritten is an Expr, if it is then we can rewrite it as a ValTuple - if isCompExpr && (comparisonExpr.Operator == InOp || comparisonExpr.Operator == NotInOp) && !isColTuple && isExpr { - cursor.Replace(ValTuple{rewrittenExpr}) - return - } - - cursor.Replace(rewritten) -} - -func (er *astRewriter) existsRewrite(cursor *Cursor, node *ExistsExpr) { - sel, ok := node.Subquery.Select.(*Select) - if !ok { - return - } - - if sel.Having != nil { - // If the query has HAVING, we can't take any shortcuts - return - } - - if sel.GroupBy == nil && sel.SelectExprs.AllAggregation() { - // in these situations, we are guaranteed to always get a non-empty result, - // so we can replace the EXISTS with a literal true - cursor.Replace(BoolVal(true)) - } - - // If we are not doing HAVING, we can safely replace all select expressions with a - // single `1` and remove any grouping - sel.SelectExprs = SelectExprs{ - &AliasedExpr{Expr: NewIntLiteral("1")}, - } - sel.GroupBy = nil -} - -// rewriteDistinctableAggr removed Distinct from Max and Min Aggregations as it does not impact the result. But, makes the plan simpler. -func (er *astRewriter) rewriteDistinctableAggr(cursor *Cursor, node DistinctableAggr) { - if !node.IsDistinct() { - return - } - switch aggr := node.(type) { - case *Max, *Min: - aggr.SetDistinct(false) - er.bindVars.NoteRewrite() - } -} - -func bindVarExpression(name string) Expr { - return NewArgument(name) -} - -// SystemSchema returns true if the schema passed is system schema -func SystemSchema(schema string) bool { - return strings.EqualFold(schema, "information_schema") || - strings.EqualFold(schema, "performance_schema") || - strings.EqualFold(schema, "sys") || - strings.EqualFold(schema, "mysql") -} diff --git a/go/vt/sqlparser/ast_rewriting_test.go b/go/vt/sqlparser/ast_rewriting_test.go deleted file mode 100644 index 8b3e3d44c54..00000000000 --- a/go/vt/sqlparser/ast_rewriting_test.go +++ /dev/null @@ -1,565 +0,0 @@ -/* -Copyright 2019 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package sqlparser - -import ( - "fmt" - "testing" - - "github.com/stretchr/testify/assert" - - "vitess.io/vitess/go/vt/sysvars" - - "github.com/stretchr/testify/require" -) - -type testCaseSetVar struct { - in, expected, setVarComment string -} - -type testCaseSysVar struct { - in, expected string - sysVar map[string]string -} - -type myTestCase struct { - in, expected string - liid, db, foundRows, rowCount, rawGTID, rawTimeout, sessTrackGTID bool - ddlStrategy, migrationContext, sessionUUID, sessionEnableSystemSettings bool - udv int - autocommit, foreignKeyChecks, clientFoundRows, skipQueryPlanCache, socket, queryTimeout bool - sqlSelectLimit, transactionMode, workload, version, versionComment bool -} - -func TestRewrites(in *testing.T) { - tests := []myTestCase{{ - in: "SELECT 42", - expected: "SELECT 42", - // no bindvar needs - }, { - in: "SELECT @@version", - expected: "SELECT :__vtversion as `@@version`", - version: true, - }, { - in: "SELECT @@query_timeout", - expected: "SELECT :__vtquery_timeout as `@@query_timeout`", - queryTimeout: true, - }, { - in: "SELECT @@version_comment", - expected: "SELECT :__vtversion_comment as `@@version_comment`", - versionComment: true, - }, { - in: "SELECT @@enable_system_settings", - expected: "SELECT :__vtenable_system_settings as `@@enable_system_settings`", - sessionEnableSystemSettings: true, - }, { - in: "SELECT last_insert_id()", - expected: "SELECT :__lastInsertId as `last_insert_id()`", - liid: true, - }, { - in: "SELECT database()", - expected: "SELECT :__vtdbname as `database()`", - db: true, - }, { - in: "SELECT database() from test", - expected: "SELECT database() from test", - // no bindvar needs - }, { - in: "SELECT last_insert_id() as test", - expected: "SELECT :__lastInsertId as test", - liid: true, - }, { - in: "SELECT last_insert_id() + database()", - expected: "SELECT :__lastInsertId + :__vtdbname as `last_insert_id() + database()`", - db: true, liid: true, - }, { - // unnest database() call - in: "select (select database()) from test", - expected: "select database() as `(select database() from dual)` from test", - // no bindvar needs - }, { - // unnest database() call - in: "select (select database() from dual) from test", - expected: "select database() as `(select database() from dual)` from test", - // no bindvar needs - }, { - in: "select (select database() from dual) from dual", - expected: "select :__vtdbname as `(select database() from dual)` from dual", - db: true, - }, { - // don't unnest solo columns - in: "select 1 as foobar, (select foobar)", - expected: "select 1 as foobar, (select foobar from dual) from dual", - }, { - in: "select id from user where database()", - expected: "select id from user where database()", - // no bindvar needs - }, { - in: "select table_name from information_schema.tables where table_schema = database()", - expected: "select table_name from information_schema.tables where table_schema = database()", - // no bindvar needs - }, { - in: "select schema()", - expected: "select :__vtdbname as `schema()`", - db: true, - }, { - in: "select found_rows()", - expected: "select :__vtfrows as `found_rows()`", - foundRows: true, - }, { - in: "select @`x y`", - expected: "select :__vtudvx_y as `@``x y``` from dual", - udv: 1, - }, { - in: "select id from t where id = @x and val = @y", - expected: "select id from t where id = :__vtudvx and val = :__vtudvy", - db: false, udv: 2, - }, { - in: "insert into t(id) values(@xyx)", - expected: "insert into t(id) values(:__vtudvxyx)", - db: false, udv: 1, - }, { - in: "select row_count()", - expected: "select :__vtrcount as `row_count()`", - rowCount: true, - }, { - in: "SELECT lower(database())", - expected: "SELECT lower(:__vtdbname) as `lower(database())`", - db: true, - }, { - in: "SELECT @@autocommit", - expected: "SELECT :__vtautocommit as `@@autocommit`", - autocommit: true, - }, { - in: "SELECT @@client_found_rows", - expected: "SELECT :__vtclient_found_rows as `@@client_found_rows`", - clientFoundRows: true, - }, { - in: "SELECT @@skip_query_plan_cache", - expected: "SELECT :__vtskip_query_plan_cache as `@@skip_query_plan_cache`", - skipQueryPlanCache: true, - }, { - in: "SELECT @@sql_select_limit", - expected: "SELECT :__vtsql_select_limit as `@@sql_select_limit`", - sqlSelectLimit: true, - }, { - in: "SELECT @@transaction_mode", - expected: "SELECT :__vttransaction_mode as `@@transaction_mode`", - transactionMode: true, - }, { - in: "SELECT @@workload", - expected: "SELECT :__vtworkload as `@@workload`", - workload: true, - }, { - in: "SELECT @@socket", - expected: "SELECT :__vtsocket as `@@socket`", - socket: true, - }, { - in: "select (select 42) from dual", - expected: "select 42 as `(select 42 from dual)` from dual", - }, { - in: "select * from user where col = (select 42)", - expected: "select * from user where col = 42", - }, { - in: "select * from (select 42) as t", // this is not an expression, and should not be rewritten - expected: "select * from (select 42) as t", - }, { - in: `select (select (select (select (select (select last_insert_id()))))) as x`, - expected: "select :__lastInsertId as x from dual", - liid: true, - }, { - in: `select * from user where col = @@ddl_strategy`, - expected: "select * from user where col = :__vtddl_strategy", - ddlStrategy: true, - }, { - in: `select * from user where col = @@migration_context`, - expected: "select * from user where col = :__vtmigration_context", - migrationContext: true, - }, { - in: `select * from user where col = @@read_after_write_gtid OR col = @@read_after_write_timeout OR col = @@session_track_gtids`, - expected: "select * from user where col = :__vtread_after_write_gtid or col = :__vtread_after_write_timeout or col = :__vtsession_track_gtids", - rawGTID: true, rawTimeout: true, sessTrackGTID: true, - }, { - in: "SELECT * FROM tbl WHERE id IN (SELECT 1 FROM dual)", - expected: "SELECT * FROM tbl WHERE id IN (1)", - }, { - in: "SELECT * FROM tbl WHERE id IN (SELECT last_insert_id() FROM dual)", - expected: "SELECT * FROM tbl WHERE id IN (:__lastInsertId)", - liid: true, - }, { - in: "SELECT * FROM tbl WHERE id IN (SELECT (SELECT 1 FROM dual WHERE 1 = 0) FROM dual)", - expected: "SELECT * FROM tbl WHERE id IN (SELECT 1 FROM dual WHERE 1 = 0)", - }, { - in: "SELECT * FROM tbl WHERE id IN (SELECT 1 FROM dual WHERE 1 = 0)", - expected: "SELECT * FROM tbl WHERE id IN (SELECT 1 FROM dual WHERE 1 = 0)", - }, { - in: "SELECT * FROM tbl WHERE id IN (SELECT 1,2 FROM dual)", - expected: "SELECT * FROM tbl WHERE id IN (SELECT 1,2 FROM dual)", - }, { - in: "SELECT * FROM tbl WHERE id IN (SELECT 1 FROM dual ORDER BY 1)", - expected: "SELECT * FROM tbl WHERE id IN (SELECT 1 FROM dual ORDER BY 1)", - }, { - in: "SELECT * FROM tbl WHERE id IN (SELECT id FROM user GROUP BY id)", - expected: "SELECT * FROM tbl WHERE id IN (SELECT id FROM user GROUP BY id)", - }, { - in: "SELECT * FROM tbl WHERE id IN (SELECT 1 FROM dual, user)", - expected: "SELECT * FROM tbl WHERE id IN (SELECT 1 FROM dual, user)", - }, { - in: "SELECT * FROM tbl WHERE id IN (SELECT 1 FROM dual limit 1)", - expected: "SELECT * FROM tbl WHERE id IN (SELECT 1 FROM dual limit 1)", - }, { - // SELECT * behaves different depending the join type used, so if that has been used, we won't rewrite - in: "SELECT * FROM A JOIN B USING (id1,id2,id3)", - expected: "SELECT * FROM A JOIN B USING (id1,id2,id3)", - }, { - in: "CALL proc(@foo)", - expected: "CALL proc(:__vtudvfoo)", - udv: 1, - }, { - in: "SELECT * FROM tbl WHERE NOT id = 42", - expected: "SELECT * FROM tbl WHERE id != 42", - }, { - in: "SELECT * FROM tbl WHERE not id < 12", - expected: "SELECT * FROM tbl WHERE id >= 12", - }, { - in: "SELECT * FROM tbl WHERE not id > 12", - expected: "SELECT * FROM tbl WHERE id <= 12", - }, { - in: "SELECT * FROM tbl WHERE not id <= 33", - expected: "SELECT * FROM tbl WHERE id > 33", - }, { - in: "SELECT * FROM tbl WHERE not id >= 33", - expected: "SELECT * FROM tbl WHERE id < 33", - }, { - in: "SELECT * FROM tbl WHERE not id != 33", - expected: "SELECT * FROM tbl WHERE id = 33", - }, { - in: "SELECT * FROM tbl WHERE not id in (1,2,3)", - expected: "SELECT * FROM tbl WHERE id not in (1,2,3)", - }, { - in: "SELECT * FROM tbl WHERE not id not in (1,2,3)", - expected: "SELECT * FROM tbl WHERE id in (1,2,3)", - }, { - in: "SELECT * FROM tbl WHERE not id not in (1,2,3)", - expected: "SELECT * FROM tbl WHERE id in (1,2,3)", - }, { - in: "SELECT * FROM tbl WHERE not id like '%foobar'", - expected: "SELECT * FROM tbl WHERE id not like '%foobar'", - }, { - in: "SELECT * FROM tbl WHERE not id not like '%foobar'", - expected: "SELECT * FROM tbl WHERE id like '%foobar'", - }, { - in: "SELECT * FROM tbl WHERE not id regexp '%foobar'", - expected: "SELECT * FROM tbl WHERE id not regexp '%foobar'", - }, { - in: "SELECT * FROM tbl WHERE not id not regexp '%foobar'", - expected: "select * from tbl where id regexp '%foobar'", - }, { - in: "SELECT * FROM tbl WHERE exists(select col1, col2 from other_table where foo > bar)", - expected: "SELECT * FROM tbl WHERE exists(select 1 from other_table where foo > bar)", - }, { - in: "SELECT * FROM tbl WHERE exists(select col1, col2 from other_table where foo > bar limit 100 offset 34)", - expected: "SELECT * FROM tbl WHERE exists(select 1 from other_table where foo > bar limit 100 offset 34)", - }, { - in: "SELECT * FROM tbl WHERE exists(select col1, col2, count(*) from other_table where foo > bar group by col1, col2)", - expected: "SELECT * FROM tbl WHERE exists(select 1 from other_table where foo > bar)", - }, { - in: "SELECT * FROM tbl WHERE exists(select col1, col2 from other_table where foo > bar group by col1, col2)", - expected: "SELECT * FROM tbl WHERE exists(select 1 from other_table where foo > bar)", - }, { - in: "SELECT * FROM tbl WHERE exists(select count(*) from other_table where foo > bar)", - expected: "SELECT * FROM tbl WHERE true", - }, { - in: "SELECT * FROM tbl WHERE exists(select col1, col2, count(*) from other_table where foo > bar group by col1, col2 having count(*) > 3)", - expected: "SELECT * FROM tbl WHERE exists(select col1, col2, count(*) from other_table where foo > bar group by col1, col2 having count(*) > 3)", - }, { - in: "SELECT id, name, salary FROM user_details", - expected: "SELECT id, name, salary FROM (select user.id, user.name, user_extra.salary from user join user_extra where user.id = user_extra.user_id) as user_details", - }, { - in: "select max(distinct c1), min(distinct c2), avg(distinct c3), sum(distinct c4), count(distinct c5), group_concat(distinct c6) from tbl", - expected: "select max(c1) as `max(distinct c1)`, min(c2) as `min(distinct c2)`, avg(distinct c3), sum(distinct c4), count(distinct c5), group_concat(distinct c6) from tbl", - }, { - in: "SHOW VARIABLES", - expected: "SHOW VARIABLES", - autocommit: true, - foreignKeyChecks: true, - clientFoundRows: true, - skipQueryPlanCache: true, - sqlSelectLimit: true, - transactionMode: true, - workload: true, - version: true, - versionComment: true, - ddlStrategy: true, - migrationContext: true, - sessionUUID: true, - sessionEnableSystemSettings: true, - rawGTID: true, - rawTimeout: true, - sessTrackGTID: true, - socket: true, - queryTimeout: true, - }, { - in: "SHOW GLOBAL VARIABLES", - expected: "SHOW GLOBAL VARIABLES", - autocommit: true, - foreignKeyChecks: true, - clientFoundRows: true, - skipQueryPlanCache: true, - sqlSelectLimit: true, - transactionMode: true, - workload: true, - version: true, - versionComment: true, - ddlStrategy: true, - migrationContext: true, - sessionUUID: true, - sessionEnableSystemSettings: true, - rawGTID: true, - rawTimeout: true, - sessTrackGTID: true, - socket: true, - queryTimeout: true, - }} - parser := NewTestParser() - for _, tc := range tests { - in.Run(tc.in, func(t *testing.T) { - require := require.New(t) - stmt, err := parser.Parse(tc.in) - require.NoError(err) - - result, err := RewriteAST( - stmt, - "ks", // passing `ks` just to test that no rewriting happens as it is not system schema - SQLSelectLimitUnset, - "", - nil, - nil, - &fakeViews{}, - ) - require.NoError(err) - - expected, err := parser.Parse(tc.expected) - require.NoError(err, "test expectation does not parse [%s]", tc.expected) - - s := String(expected) - assert := assert.New(t) - assert.Equal(s, String(result.AST)) - assert.Equal(tc.liid, result.NeedsFuncResult(LastInsertIDName), "should need last insert id") - assert.Equal(tc.db, result.NeedsFuncResult(DBVarName), "should need database name") - assert.Equal(tc.foundRows, result.NeedsFuncResult(FoundRowsName), "should need found rows") - assert.Equal(tc.rowCount, result.NeedsFuncResult(RowCountName), "should need row count") - assert.Equal(tc.udv, len(result.NeedUserDefinedVariables), "count of user defined variables") - assert.Equal(tc.autocommit, result.NeedsSysVar(sysvars.Autocommit.Name), "should need :__vtautocommit") - assert.Equal(tc.foreignKeyChecks, result.NeedsSysVar(sysvars.ForeignKeyChecks), "should need :__vtforeignKeyChecks") - assert.Equal(tc.clientFoundRows, result.NeedsSysVar(sysvars.ClientFoundRows.Name), "should need :__vtclientFoundRows") - assert.Equal(tc.skipQueryPlanCache, result.NeedsSysVar(sysvars.SkipQueryPlanCache.Name), "should need :__vtskipQueryPlanCache") - assert.Equal(tc.sqlSelectLimit, result.NeedsSysVar(sysvars.SQLSelectLimit.Name), "should need :__vtsqlSelectLimit") - assert.Equal(tc.transactionMode, result.NeedsSysVar(sysvars.TransactionMode.Name), "should need :__vttransactionMode") - assert.Equal(tc.workload, result.NeedsSysVar(sysvars.Workload.Name), "should need :__vtworkload") - assert.Equal(tc.queryTimeout, result.NeedsSysVar(sysvars.QueryTimeout.Name), "should need :__vtquery_timeout") - assert.Equal(tc.ddlStrategy, result.NeedsSysVar(sysvars.DDLStrategy.Name), "should need ddlStrategy") - assert.Equal(tc.migrationContext, result.NeedsSysVar(sysvars.MigrationContext.Name), "should need migrationContext") - assert.Equal(tc.sessionUUID, result.NeedsSysVar(sysvars.SessionUUID.Name), "should need sessionUUID") - assert.Equal(tc.sessionEnableSystemSettings, result.NeedsSysVar(sysvars.SessionEnableSystemSettings.Name), "should need sessionEnableSystemSettings") - assert.Equal(tc.rawGTID, result.NeedsSysVar(sysvars.ReadAfterWriteGTID.Name), "should need rawGTID") - assert.Equal(tc.rawTimeout, result.NeedsSysVar(sysvars.ReadAfterWriteTimeOut.Name), "should need rawTimeout") - assert.Equal(tc.sessTrackGTID, result.NeedsSysVar(sysvars.SessionTrackGTIDs.Name), "should need sessTrackGTID") - assert.Equal(tc.version, result.NeedsSysVar(sysvars.Version.Name), "should need Vitess version") - assert.Equal(tc.versionComment, result.NeedsSysVar(sysvars.VersionComment.Name), "should need Vitess version") - assert.Equal(tc.socket, result.NeedsSysVar(sysvars.Socket.Name), "should need :__vtsocket") - }) - } -} - -type fakeViews struct{} - -func (*fakeViews) FindView(name TableName) TableStatement { - if name.Name.String() != "user_details" { - return nil - } - parser := NewTestParser() - statement, err := parser.Parse("select user.id, user.name, user_extra.salary from user join user_extra where user.id = user_extra.user_id") - if err != nil { - return nil - } - return statement.(TableStatement) -} - -func TestRewritesWithSetVarComment(in *testing.T) { - tests := []testCaseSetVar{{ - in: "select 1", - expected: "select 1", - setVarComment: "", - }, { - in: "select 1", - expected: "select /*+ AA(a) */ 1", - setVarComment: "AA(a)", - }, { - in: "insert /* toto */ into t(id) values(1)", - expected: "insert /*+ AA(a) */ /* toto */ into t(id) values(1)", - setVarComment: "AA(a)", - }, { - in: "select /* toto */ * from t union select * from s", - expected: "select /*+ AA(a) */ /* toto */ * from t union select /*+ AA(a) */ * from s", - setVarComment: "AA(a)", - }, { - in: "vstream /* toto */ * from t1", - expected: "vstream /*+ AA(a) */ /* toto */ * from t1", - setVarComment: "AA(a)", - }, { - in: "stream /* toto */ t from t1", - expected: "stream /*+ AA(a) */ /* toto */ t from t1", - setVarComment: "AA(a)", - }, { - in: "update /* toto */ t set id = 1", - expected: "update /*+ AA(a) */ /* toto */ t set id = 1", - setVarComment: "AA(a)", - }, { - in: "delete /* toto */ from t", - expected: "delete /*+ AA(a) */ /* toto */ from t", - setVarComment: "AA(a)", - }} - - parser := NewTestParser() - for _, tc := range tests { - in.Run(tc.in, func(t *testing.T) { - require := require.New(t) - stmt, err := parser.Parse(tc.in) - require.NoError(err) - - result, err := RewriteAST(stmt, "ks", SQLSelectLimitUnset, tc.setVarComment, nil, nil, &fakeViews{}) - require.NoError(err) - - expected, err := parser.Parse(tc.expected) - require.NoError(err, "test expectation does not parse [%s]", tc.expected) - - assert.Equal(t, String(expected), String(result.AST)) - }) - } -} - -func TestRewritesSysVar(in *testing.T) { - tests := []testCaseSysVar{{ - in: "select @x = @@sql_mode", - expected: "select :__vtudvx = @@sql_mode as `@x = @@sql_mode` from dual", - }, { - in: "select @x = @@sql_mode", - expected: "select :__vtudvx = :__vtsql_mode as `@x = @@sql_mode` from dual", - sysVar: map[string]string{"sql_mode": "' '"}, - }, { - in: "SELECT @@tx_isolation", - expected: "select @@tx_isolation from dual", - }, { - in: "SELECT @@transaction_isolation", - expected: "select @@transaction_isolation from dual", - }, { - in: "SELECT @@session.transaction_isolation", - expected: "select @@session.transaction_isolation from dual", - }, { - in: "SELECT @@tx_isolation", - sysVar: map[string]string{"tx_isolation": "'READ-COMMITTED'"}, - expected: "select :__vttx_isolation as `@@tx_isolation` from dual", - }, { - in: "SELECT @@transaction_isolation", - sysVar: map[string]string{"transaction_isolation": "'READ-COMMITTED'"}, - expected: "select :__vttransaction_isolation as `@@transaction_isolation` from dual", - }, { - in: "SELECT @@session.transaction_isolation", - sysVar: map[string]string{"transaction_isolation": "'READ-COMMITTED'"}, - expected: "select :__vttransaction_isolation as `@@session.transaction_isolation` from dual", - }} - - parser := NewTestParser() - for _, tc := range tests { - in.Run(tc.in, func(t *testing.T) { - require := require.New(t) - stmt, err := parser.Parse(tc.in) - require.NoError(err) - - result, err := RewriteAST(stmt, "ks", SQLSelectLimitUnset, "", tc.sysVar, nil, &fakeViews{}) - require.NoError(err) - - expected, err := parser.Parse(tc.expected) - require.NoError(err, "test expectation does not parse [%s]", tc.expected) - - assert.Equal(t, String(expected), String(result.AST)) - }) - } -} - -func TestRewritesWithDefaultKeyspace(in *testing.T) { - tests := []myTestCase{{ - in: "SELECT 1 from x.test", - expected: "SELECT 1 from x.test", // no change - }, { - in: "SELECT x.col as c from x.test", - expected: "SELECT x.col as c from x.test", // no change - }, { - in: "SELECT 1 from test", - expected: "SELECT 1 from sys.test", - }, { - in: "SELECT 1 from test as t", - expected: "SELECT 1 from sys.test as t", - }, { - in: "SELECT 1 from `test 24` as t", - expected: "SELECT 1 from sys.`test 24` as t", - }, { - in: "SELECT 1, (select 1 from test) from x.y", - expected: "SELECT 1, (select 1 from sys.test) from x.y", - }, { - in: "SELECT 1 from (select 2 from test) t", - expected: "SELECT 1 from (select 2 from sys.test) t", - }, { - in: "SELECT 1 from test where exists(select 2 from test)", - expected: "SELECT 1 from sys.test where exists(select 1 from sys.test)", - }, { - in: "SELECT 1 from dual", - expected: "SELECT 1 from dual", - }, { - in: "SELECT (select 2 from dual) from DUAL", - expected: "SELECT 2 as `(select 2 from dual)` from DUAL", - }} - - parser := NewTestParser() - for _, tc := range tests { - in.Run(tc.in, func(t *testing.T) { - require := require.New(t) - stmt, err := parser.Parse(tc.in) - require.NoError(err) - - result, err := RewriteAST(stmt, "sys", SQLSelectLimitUnset, "", nil, nil, &fakeViews{}) - require.NoError(err) - - expected, err := parser.Parse(tc.expected) - require.NoError(err, "test expectation does not parse [%s]", tc.expected) - - assert.Equal(t, String(expected), String(result.AST)) - }) - } -} - -func TestReservedVars(t *testing.T) { - for _, prefix := range []string{"vtg", "bv"} { - t.Run("prefix_"+prefix, func(t *testing.T) { - reserved := NewReservedVars(prefix, make(BindVars)) - for i := 1; i < 1000; i++ { - require.Equal(t, fmt.Sprintf("%s%d", prefix, i), reserved.nextUnusedVar()) - } - }) - } -} diff --git a/go/vt/sqlparser/bind_var_needs.go b/go/vt/sqlparser/bind_var_needs.go index 1b26919ca03..64e5c528e97 100644 --- a/go/vt/sqlparser/bind_var_needs.go +++ b/go/vt/sqlparser/bind_var_needs.go @@ -22,14 +22,7 @@ type BindVarNeeds struct { NeedSystemVariable, // NeedUserDefinedVariables keeps track of all user defined variables a query is using NeedUserDefinedVariables []string - otherRewrites bool -} - -// MergeWith adds bind vars needs coming from sub scopes -func (bvn *BindVarNeeds) MergeWith(other *BindVarNeeds) { - bvn.NeedFunctionResult = append(bvn.NeedFunctionResult, other.NeedFunctionResult...) - bvn.NeedSystemVariable = append(bvn.NeedSystemVariable, other.NeedSystemVariable...) - bvn.NeedUserDefinedVariables = append(bvn.NeedUserDefinedVariables, other.NeedUserDefinedVariables...) + otherRewrites int } // AddFuncResult adds a function bindvar need @@ -58,14 +51,11 @@ func (bvn *BindVarNeeds) NeedsSysVar(name string) bool { } func (bvn *BindVarNeeds) NoteRewrite() { - bvn.otherRewrites = true + bvn.otherRewrites++ } -func (bvn *BindVarNeeds) HasRewrites() bool { - return bvn.otherRewrites || - len(bvn.NeedFunctionResult) > 0 || - len(bvn.NeedUserDefinedVariables) > 0 || - len(bvn.NeedSystemVariable) > 0 +func (bvn *BindVarNeeds) NumberOfRewrites() int { + return len(bvn.NeedFunctionResult) + len(bvn.NeedUserDefinedVariables) + len(bvn.NeedSystemVariable) + bvn.otherRewrites } func contains(strings []string, name string) bool { diff --git a/go/vt/sqlparser/normalizer.go b/go/vt/sqlparser/normalizer.go index 02cb11e2a97..fb3813b7019 100644 --- a/go/vt/sqlparser/normalizer.go +++ b/go/vt/sqlparser/normalizer.go @@ -18,107 +18,277 @@ package sqlparser import ( "bytes" + "fmt" + "strconv" + "strings" "vitess.io/vitess/go/mysql/datetime" "vitess.io/vitess/go/sqltypes" + querypb "vitess.io/vitess/go/vt/proto/query" vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" + "vitess.io/vitess/go/vt/sysvars" "vitess.io/vitess/go/vt/vterrors" +) - querypb "vitess.io/vitess/go/vt/proto/query" +type ( + // BindVars represents a set of reserved bind variables extracted from a SQL statement. + BindVars map[string]struct{} + // normalizer transforms SQL statements to support parameterization and streamline query planning. + // + // It serves two primary purposes: + // 1. **Parameterization:** Allows multiple invocations of the same query with different literals by converting literals + // to bind variables. This enables efficient reuse of execution plans with varying parameters. + // 2. **Simplified Planning:** Reduces the complexity for the query planner by standardizing SQL patterns. For example, + // it ensures that table columns are consistently placed on the left side of comparison expressions. This uniformity + // minimizes the number of distinct patterns the planner must handle, enhancing planning efficiency. + normalizer struct { + bindVars map[string]*querypb.BindVariable + reserved *ReservedVars + vals map[Literal]string + err error + inDerived int + inSelect int + + bindVarNeeds *BindVarNeeds + shouldRewriteDatabaseFunc bool + hasStarInSelect bool + + keyspace string + selectLimit int + setVarComment string + fkChecksState *bool + sysVars map[string]string + views VSchemaViews + + onLeave map[*AliasedExpr]func(*AliasedExpr) + parameterize bool + } + // RewriteASTResult holds the result of rewriting the AST, including bind variable needs. + RewriteASTResult struct { + *BindVarNeeds + AST Statement // The rewritten AST + } + // VSchemaViews provides access to view definitions within the VSchema. + VSchemaViews interface { + FindView(name TableName) TableStatement + } ) -// BindVars is a set of reserved bind variables from a SQL statement -type BindVars map[string]struct{} +const ( + // SQLSelectLimitUnset indicates that sql_select_limit is not set. + SQLSelectLimitUnset = -1 + // LastInsertIDName is the bind variable name for LAST_INSERT_ID(). + LastInsertIDName = "__lastInsertId" + // DBVarName is the bind variable name for DATABASE(). + DBVarName = "__vtdbname" + // FoundRowsName is the bind variable name for FOUND_ROWS(). + FoundRowsName = "__vtfrows" + // RowCountName is the bind variable name for ROW_COUNT(). + RowCountName = "__vtrcount" + // UserDefinedVariableName is the prefix for user-defined variable bind names. + UserDefinedVariableName = "__vtudv" +) -// Normalize changes the statement to use bind values, and -// updates the bind vars to those values. The supplied prefix -// is used to generate the bind var names. The function ensures -// that there are no collisions with existing bind vars. -// Within Select constructs, bind vars are deduped. This allows -// us to identify vindex equality. Otherwise, every value is -// treated as distinct. -func Normalize(stmt Statement, reserved *ReservedVars, bindVars map[string]*querypb.BindVariable) error { - nz := newNormalizer(reserved, bindVars) - _ = SafeRewrite(stmt, nz.walkStatementDown, nz.walkStatementUp) - return nz.err +// funcRewrites lists all functions that must be rewritten. we don't want these to make it down to mysql, +// we need to handle these in the vtgate +var funcRewrites = map[string]string{ + "last_insert_id": LastInsertIDName, + "database": DBVarName, + "schema": DBVarName, + "found_rows": FoundRowsName, + "row_count": RowCountName, } -type normalizer struct { - bindVars map[string]*querypb.BindVariable - reserved *ReservedVars - vals map[Literal]string - err error - inDerived int - inSelect int -} +// PrepareAST normalizes the input SQL statement and returns the rewritten AST along with bind variable information. +func PrepareAST( + in Statement, + reservedVars *ReservedVars, + bindVars map[string]*querypb.BindVariable, + parameterize bool, + keyspace string, + selectLimit int, + setVarComment string, + sysVars map[string]string, + fkChecksState *bool, + views VSchemaViews, +) (*RewriteASTResult, error) { + nz := newNormalizer(reservedVars, bindVars, keyspace, selectLimit, setVarComment, sysVars, fkChecksState, views, parameterize) + nz.shouldRewriteDatabaseFunc = shouldRewriteDatabaseFunc(in) + out := SafeRewrite(in, nz.walkDown, nz.walkUp) + if nz.err != nil { + return nil, nz.err + } -func newNormalizer(reserved *ReservedVars, bindVars map[string]*querypb.BindVariable) *normalizer { - return &normalizer{ - bindVars: bindVars, - reserved: reserved, - vals: make(map[Literal]string), + r := &RewriteASTResult{ + AST: out.(Statement), + BindVarNeeds: nz.bindVarNeeds, } + return r, nil } -// walkStatementUp is one half of the top level walk function. -func (nz *normalizer) walkStatementUp(cursor *Cursor) bool { - if nz.err != nil { - return false - } - switch node := cursor.node.(type) { - case *DerivedTable: - nz.inDerived-- - case *Select: - nz.inSelect-- - case *Literal: - if nz.inSelect == 0 { - nz.convertLiteral(node, cursor) - return nz.err == nil - } - parent := cursor.Parent() - switch parent.(type) { - case *Order, *GroupBy: - return true - case *Limit: - nz.convertLiteral(node, cursor) - default: - nz.convertLiteralDedup(node, cursor) - } +func newNormalizer( + reserved *ReservedVars, + bindVars map[string]*querypb.BindVariable, + keyspace string, + selectLimit int, + setVarComment string, + sysVars map[string]string, + fkChecksState *bool, + views VSchemaViews, + parameterize bool, +) *normalizer { + return &normalizer{ + bindVars: bindVars, + reserved: reserved, + vals: make(map[Literal]string), + bindVarNeeds: &BindVarNeeds{}, + keyspace: keyspace, + selectLimit: selectLimit, + setVarComment: setVarComment, + fkChecksState: fkChecksState, + sysVars: sysVars, + views: views, + onLeave: make(map[*AliasedExpr]func(*AliasedExpr)), + parameterize: parameterize, } - return nz.err == nil // only continue if we haven't found any errors } -// walkStatementDown is the top level walk function. -// If it encounters a Select, it switches to a mode -// where variables are deduped. -func (nz *normalizer) walkStatementDown(node, _ SQLNode) bool { +// walkDown processes nodes when traversing down the AST. +// It handles normalization logic based on node types. +func (nz *normalizer) walkDown(node, _ SQLNode) bool { switch node := node.(type) { - // no need to normalize the statement types - case *Set, *Show, *Begin, *Commit, *Rollback, *Savepoint, DDLStatement, *SRollback, *Release, *OtherAdmin, *Analyze: + case *Begin, *Commit, *Rollback, *Savepoint, *SRollback, *Release, *OtherAdmin, *Analyze, *AssignmentExpr, + *PrepareStmt, *ExecuteStmt, *FramePoint, *ColName, TableName, *ConvertType: + // These statement don't need normalizing return false + case *Set: + // Disable parameterization within SET statements. + nz.parameterize = false case *DerivedTable: nz.inDerived++ case *Select: nz.inSelect++ - case SelectExprs: - return nz.inDerived == 0 + if nz.selectLimit > 0 && node.Limit == nil { + node.Limit = &Limit{Rowcount: NewIntLiteral(strconv.Itoa(nz.selectLimit))} + } + case *AliasedExpr: + nz.noteAliasedExprName(node) case *ComparisonExpr: nz.convertComparison(node) case *UpdateExpr: nz.convertUpdateExpr(node) - case *ColName, TableName: - // Common node types that never contain Literal or ListArgs but create a lot of object - // allocations. - return false - case *ConvertType: // we should not rewrite the type description + case *StarExpr: + nz.hasStarInSelect = true + // No rewriting needed for prepare or execute statements. return false - case *FramePoint: - // do not make a bind var for rows and range + case *ShowBasic: + if node.Command != VariableGlobal && node.Command != VariableSession { + break + } + varsToAdd := sysvars.GetInterestingVariables() + for _, sysVar := range varsToAdd { + nz.bindVarNeeds.AddSysVar(sysVar) + } + } + b := nz.err == nil + if !b { + fmt.Println(1) + } + return b +} + +// noteAliasedExprName tracks expressions without aliases to add alias if expression is rewritten +func (nz *normalizer) noteAliasedExprName(node *AliasedExpr) { + if node.As.NotEmpty() { + return + } + buf := NewTrackedBuffer(nil) + node.Expr.Format(buf) + rewrites := nz.bindVarNeeds.NumberOfRewrites() + nz.onLeave[node] = func(newAliasedExpr *AliasedExpr) { + if nz.bindVarNeeds.NumberOfRewrites() > rewrites { + newAliasedExpr.As = NewIdentifierCI(buf.String()) + } + } +} + +// walkUp processes nodes when traversing up the AST. +// It finalizes normalization logic based on node types. +func (nz *normalizer) walkUp(cursor *Cursor) bool { + // Add SET_VAR comments if applicable. + if supportOptimizerHint, supports := cursor.Node().(SupportOptimizerHint); supports { + if nz.setVarComment != "" { + newComments, err := supportOptimizerHint.GetParsedComments().AddQueryHint(nz.setVarComment) + if err != nil { + nz.err = err + return false + } + supportOptimizerHint.SetComments(newComments) + } + if nz.fkChecksState != nil { + newComments := supportOptimizerHint.GetParsedComments().SetMySQLSetVarValue(sysvars.ForeignKeyChecks, FkChecksStateString(nz.fkChecksState)) + supportOptimizerHint.SetComments(newComments) + } + } + + if nz.err != nil { return false } - return nz.err == nil // only continue if we haven't found any errors + + switch node := cursor.node.(type) { + case *DerivedTable: + nz.inDerived-- + case *Select: + nz.inSelect-- + case *AliasedExpr: + // if we are tracking this node for changes, this is the time to add the alias if needed + if onLeave, ok := nz.onLeave[node]; ok { + onLeave(node) + delete(nz.onLeave, node) + } + case *Union: + nz.rewriteUnion(node) + case *FuncExpr: + nz.funcRewrite(cursor, node) + case *Variable: + nz.rewriteVariable(cursor, node) + case *Subquery: + nz.unnestSubQueries(cursor, node) + case *NotExpr: + nz.rewriteNotExpr(cursor, node) + case *AliasedTableExpr: + nz.rewriteAliasedTable(cursor, node) + case *ShowBasic: + nz.rewriteShowBasic(node) + case *ExistsExpr: + nz.existsRewrite(cursor, node) + case DistinctableAggr: + nz.rewriteDistinctableAggr(node) + case *Literal: + nz.visitLiteral(cursor, node) + } + return nz.err == nil } +func (nz *normalizer) visitLiteral(cursor *Cursor, node *Literal) { + if !nz.shouldParameterize() { + return + } + if nz.inSelect == 0 { + nz.convertLiteral(node, cursor) + return + } + switch cursor.Parent().(type) { + case *Order, *GroupBy: + return + case *Limit: + nz.convertLiteral(node, cursor) + default: + nz.convertLiteralDedup(node, cursor) + } +} + +// validateLiteral ensures that a Literal node has a valid value based on its type. func validateLiteral(node *Literal) error { switch node.Type { case DateVal: @@ -137,37 +307,31 @@ func validateLiteral(node *Literal) error { return nil } +// convertLiteralDedup converts a Literal node to a bind variable with deduplication. func (nz *normalizer) convertLiteralDedup(node *Literal, cursor *Cursor) { - err := validateLiteral(node) - if err != nil { + if err := validateLiteral(node); err != nil { nz.err = err + return } - // If value is too long, don't dedup. - // Such values are most likely not for vindexes. - // We save a lot of CPU because we avoid building - // the key for them. + // Skip deduplication for long values. if len(node.Val) > 256 { nz.convertLiteral(node, cursor) return } - // Make the bindvar - bval := SQLToBindvar(node) + bval := literalToBindvar(node) if bval == nil { return } - // Check if there's a bindvar for that value already. - bvname, ok := nz.vals[*node] - if !ok { - // If there's no such bindvar, make a new one. + bvname, exists := nz.vals[*node] + if !exists { bvname = nz.reserved.nextUnusedVar() nz.vals[*node] = bvname nz.bindVars[bvname] = bval } - // Modify the AST node to a bindvar. arg, err := NewTypedArgumentFromLiteral(bvname, node) if err != nil { nz.err = err @@ -176,14 +340,14 @@ func (nz *normalizer) convertLiteralDedup(node *Literal, cursor *Cursor) { cursor.Replace(arg) } -// convertLiteral converts an Literal without the dedup. +// convertLiteral converts a Literal node to a bind variable without deduplication. func (nz *normalizer) convertLiteral(node *Literal, cursor *Cursor) { - err := validateLiteral(node) - if err != nil { + if err := validateLiteral(node); err != nil { nz.err = err + return } - bval := SQLToBindvar(node) + bval := literalToBindvar(node) if bval == nil { return } @@ -198,11 +362,7 @@ func (nz *normalizer) convertLiteral(node *Literal, cursor *Cursor) { cursor.Replace(arg) } -// convertComparison attempts to convert IN clauses to -// use the list bind var construct. If it fails, it returns -// with no change made. The walk function will then continue -// and iterate on converting each individual value into separate -// bind vars. +// convertComparison handles the conversion of comparison expressions to use bind variables. func (nz *normalizer) convertComparison(node *ComparisonExpr) { switch node.Operator { case InOp, NotInOp: @@ -212,14 +372,19 @@ func (nz *normalizer) convertComparison(node *ComparisonExpr) { } } +// rewriteOtherComparisons parameterizes non-IN comparison expressions. func (nz *normalizer) rewriteOtherComparisons(node *ComparisonExpr) { - newR := nz.parameterize(node.Left, node.Right) + newR := nz.normalizeComparisonWithBindVar(node.Left, node.Right) if newR != nil { node.Right = newR } } -func (nz *normalizer) parameterize(left, right Expr) Expr { +// normalizeComparisonWithBindVar attempts to replace a literal in a comparison with a bind variable. +func (nz *normalizer) normalizeComparisonWithBindVar(left, right Expr) Expr { + if !nz.shouldParameterize() { + return nil + } col, ok := left.(*ColName) if !ok { return nil @@ -228,13 +393,12 @@ func (nz *normalizer) parameterize(left, right Expr) Expr { if !ok { return nil } - err := validateLiteral(lit) - if err != nil { + if err := validateLiteral(lit); err != nil { nz.err = err return nil } - bval := SQLToBindvar(lit) + bval := literalToBindvar(lit) if bval == nil { return nil } @@ -247,18 +411,14 @@ func (nz *normalizer) parameterize(left, right Expr) Expr { return arg } +// decideBindVarName determines the appropriate bind variable name for a given literal and column. func (nz *normalizer) decideBindVarName(lit *Literal, col *ColName, bval *querypb.BindVariable) string { if len(lit.Val) <= 256 { - // first we check if we already have a bindvar for this value. if we do, we re-use that bindvar name - bvname, ok := nz.vals[*lit] - if ok { + if bvname, ok := nz.vals[*lit]; ok { return bvname } } - // If there's no such bindvar, or we have a big value, make a new one. - // Big values are most likely not for vindexes. - // We save a lot of CPU because we avoid building bvname := nz.reserved.ReserveColName(col) nz.vals[*lit] = bvname nz.bindVars[bvname] = bval @@ -266,19 +426,22 @@ func (nz *normalizer) decideBindVarName(lit *Literal, col *ColName, bval *queryp return bvname } +// rewriteInComparisons converts IN and NOT IN expressions to use list bind variables. func (nz *normalizer) rewriteInComparisons(node *ComparisonExpr) { + if !nz.shouldParameterize() { + return + } tupleVals, ok := node.Right.(ValTuple) if !ok { return } - // The RHS is a tuple of values. - // Make a list bindvar. + // Create a list bind variable for the tuple. bvals := &querypb.BindVariable{ Type: querypb.Type_TUPLE, } for _, val := range tupleVals { - bval := SQLToBindvar(val) + bval := literalToBindvar(val) if bval == nil { return } @@ -289,76 +452,74 @@ func (nz *normalizer) rewriteInComparisons(node *ComparisonExpr) { } bvname := nz.reserved.nextUnusedVar() nz.bindVars[bvname] = bvals - // Modify RHS to be a list bindvar. node.Right = ListArg(bvname) } +// convertUpdateExpr parameterizes expressions in UPDATE statements. func (nz *normalizer) convertUpdateExpr(node *UpdateExpr) { - newR := nz.parameterize(node.Name, node.Expr) + newR := nz.normalizeComparisonWithBindVar(node.Name, node.Expr) if newR != nil { node.Expr = newR } } -func SQLToBindvar(node SQLNode) *querypb.BindVariable { - if node, ok := node.(*Literal); ok { - var v sqltypes.Value - var err error - switch node.Type { - case StrVal: - v, err = sqltypes.NewValue(sqltypes.VarChar, node.Bytes()) - case IntVal: - v, err = sqltypes.NewValue(sqltypes.Int64, node.Bytes()) - case FloatVal: - v, err = sqltypes.NewValue(sqltypes.Float64, node.Bytes()) - case DecimalVal: - v, err = sqltypes.NewValue(sqltypes.Decimal, node.Bytes()) - case HexNum: - buf := make([]byte, 0, len(node.Bytes())) - buf = append(buf, "0x"...) - buf = append(buf, bytes.ToUpper(node.Bytes()[2:])...) - v, err = sqltypes.NewValue(sqltypes.HexNum, buf) - case HexVal: - // We parse the `x'7b7d'` string literal into a hex encoded string of `7b7d` in the parser - // We need to re-encode it back to the original MySQL query format before passing it on as a bindvar value to MySQL - buf := make([]byte, 0, len(node.Bytes())+3) - buf = append(buf, 'x', '\'') - buf = append(buf, bytes.ToUpper(node.Bytes())...) - buf = append(buf, '\'') - v, err = sqltypes.NewValue(sqltypes.HexVal, buf) - case BitNum: - out := make([]byte, 0, len(node.Bytes())+2) - out = append(out, '0', 'b') - out = append(out, node.Bytes()[2:]...) - v, err = sqltypes.NewValue(sqltypes.BitNum, out) - case DateVal: - v, err = sqltypes.NewValue(sqltypes.Date, node.Bytes()) - case TimeVal: - v, err = sqltypes.NewValue(sqltypes.Time, node.Bytes()) - case TimestampVal: - // This is actually a DATETIME MySQL type. The timestamp literal - // syntax is part of the SQL standard and MySQL DATETIME matches - // the type best. - v, err = sqltypes.NewValue(sqltypes.Datetime, node.Bytes()) - default: - return nil - } - if err != nil { - return nil - } - return sqltypes.ValueBindVariable(v) +// literalToBindvar converts a SQLNode to a BindVariable if possible. +func literalToBindvar(node SQLNode) *querypb.BindVariable { + lit, ok := node.(*Literal) + if !ok { + return nil } - return nil + var v sqltypes.Value + var err error + switch lit.Type { + case StrVal: + v, err = sqltypes.NewValue(sqltypes.VarChar, lit.Bytes()) + case IntVal: + v, err = sqltypes.NewValue(sqltypes.Int64, lit.Bytes()) + case FloatVal: + v, err = sqltypes.NewValue(sqltypes.Float64, lit.Bytes()) + case DecimalVal: + v, err = sqltypes.NewValue(sqltypes.Decimal, lit.Bytes()) + case HexNum: + buf := make([]byte, 0, len(lit.Bytes())) + buf = append(buf, "0x"...) + buf = append(buf, bytes.ToUpper(lit.Bytes()[2:])...) + v, err = sqltypes.NewValue(sqltypes.HexNum, buf) + case HexVal: + // Re-encode hex string literals to original MySQL format. + buf := make([]byte, 0, len(lit.Bytes())+3) + buf = append(buf, 'x', '\'') + buf = append(buf, bytes.ToUpper(lit.Bytes())...) + buf = append(buf, '\'') + v, err = sqltypes.NewValue(sqltypes.HexVal, buf) + case BitNum: + out := make([]byte, 0, len(lit.Bytes())+2) + out = append(out, '0', 'b') + out = append(out, lit.Bytes()[2:]...) + v, err = sqltypes.NewValue(sqltypes.BitNum, out) + case DateVal: + v, err = sqltypes.NewValue(sqltypes.Date, lit.Bytes()) + case TimeVal: + v, err = sqltypes.NewValue(sqltypes.Time, lit.Bytes()) + case TimestampVal: + // Use DATETIME type for TIMESTAMP literals. + v, err = sqltypes.NewValue(sqltypes.Datetime, lit.Bytes()) + default: + return nil + } + if err != nil { + return nil + } + return sqltypes.ValueBindVariable(v) } -// GetBindvars returns a map of the bind vars referenced in the statement. -func GetBindvars(stmt Statement) map[string]struct{} { +// getBindvars extracts bind variables from a SQL statement. +func getBindvars(stmt Statement) map[string]struct{} { bindvars := make(map[string]struct{}) _ = Walk(func(node SQLNode) (kontinue bool, err error) { switch node := node.(type) { case *ColName, TableName: - // Common node types that never contain expressions but create a lot of object - // allocations. + // These node types do not contain expressions. return false, nil case *Argument: bindvars[node.Name] = struct{}{} @@ -369,3 +530,312 @@ func GetBindvars(stmt Statement) map[string]struct{} { }, stmt) return bindvars } + +var HasValueSubQueryBaseName = []byte("__sq_has_values") + +// shouldRewriteDatabaseFunc determines if the database function should be rewritten based on the statement. +func shouldRewriteDatabaseFunc(in Statement) bool { + selct, ok := in.(*Select) + if !ok { + return false + } + if len(selct.From) != 1 { + return false + } + aliasedTable, ok := selct.From[0].(*AliasedTableExpr) + if !ok { + return false + } + tableName, ok := aliasedTable.Expr.(TableName) + if !ok { + return false + } + return tableName.Name.String() == "dual" +} + +// rewriteUnion sets the SELECT limit for UNION statements if not already set. +func (nz *normalizer) rewriteUnion(node *Union) { + if nz.selectLimit > 0 && node.Limit == nil { + node.Limit = &Limit{Rowcount: NewIntLiteral(strconv.Itoa(nz.selectLimit))} + } +} + +// rewriteAliasedTable handles the rewriting of aliased tables, including view substitutions. +func (nz *normalizer) rewriteAliasedTable(cursor *Cursor, node *AliasedTableExpr) { + aliasTableName, ok := node.Expr.(TableName) + if !ok { + return + } + + // Do not add qualifiers to the dual table. + tblName := aliasTableName.Name.String() + if tblName == "dual" { + return + } + + if SystemSchema(nz.keyspace) { + if aliasTableName.Qualifier.IsEmpty() { + aliasTableName.Qualifier = NewIdentifierCS(nz.keyspace) + node.Expr = aliasTableName + cursor.Replace(node) + } + return + } + + // Replace views with their underlying definitions. + if nz.views == nil { + return + } + view := nz.views.FindView(aliasTableName) + if view == nil { + return + } + + // Substitute the view with a derived table. + node.Expr = &DerivedTable{Select: Clone(view)} + if node.As.IsEmpty() { + node.As = NewIdentifierCS(tblName) + } +} + +// rewriteShowBasic handles the rewriting of SHOW statements, particularly for system variables. +func (nz *normalizer) rewriteShowBasic(node *ShowBasic) { + if node.Command == VariableGlobal || node.Command == VariableSession { + varsToAdd := sysvars.GetInterestingVariables() + for _, sysVar := range varsToAdd { + nz.bindVarNeeds.AddSysVar(sysVar) + } + } +} + +// rewriteNotExpr simplifies NOT expressions where possible. +func (nz *normalizer) rewriteNotExpr(cursor *Cursor, node *NotExpr) { + switch inner := node.Expr.(type) { + case *ComparisonExpr: + // Invert comparison operators. + if canChange, inverse := inverseOp(inner.Operator); canChange { + inner.Operator = inverse + cursor.Replace(inner) + } + case *NotExpr: + // Simplify double negation. + cursor.Replace(inner.Expr) + case BoolVal: + // Negate boolean values. + cursor.Replace(!inner) + } +} + +// rewriteVariable handles the rewriting of variable expressions to bind variables. +func (nz *normalizer) rewriteVariable(cursor *Cursor, node *Variable) { + // Do not rewrite variables on the left side of SET assignments. + if v, isSet := cursor.Parent().(*SetExpr); isSet && v.Var == node { + return + } + switch node.Scope { + case VariableScope: + nz.udvRewrite(cursor, node) + case SessionScope, NextTxScope: + nz.sysVarRewrite(cursor, node) + } +} + +// inverseOp returns the inverse operator for a given comparison operator. +func inverseOp(i ComparisonExprOperator) (bool, ComparisonExprOperator) { + switch i { + case EqualOp: + return true, NotEqualOp + case LessThanOp: + return true, GreaterEqualOp + case GreaterThanOp: + return true, LessEqualOp + case LessEqualOp: + return true, GreaterThanOp + case GreaterEqualOp: + return true, LessThanOp + case NotEqualOp: + return true, EqualOp + case InOp: + return true, NotInOp + case NotInOp: + return true, InOp + case LikeOp: + return true, NotLikeOp + case NotLikeOp: + return true, LikeOp + case RegexpOp: + return true, NotRegexpOp + case NotRegexpOp: + return true, RegexpOp + } + return false, i +} + +// sysVarRewrite replaces system variables with corresponding bind variables. +func (nz *normalizer) sysVarRewrite(cursor *Cursor, node *Variable) { + lowered := node.Name.Lowered() + + var found bool + if nz.sysVars != nil { + _, found = nz.sysVars[lowered] + } + + switch lowered { + case sysvars.Autocommit.Name, + sysvars.Charset.Name, + sysvars.ClientFoundRows.Name, + sysvars.DDLStrategy.Name, + sysvars.MigrationContext.Name, + sysvars.Names.Name, + sysvars.TransactionMode.Name, + sysvars.ReadAfterWriteGTID.Name, + sysvars.ReadAfterWriteTimeOut.Name, + sysvars.SessionEnableSystemSettings.Name, + sysvars.SessionTrackGTIDs.Name, + sysvars.SessionUUID.Name, + sysvars.SkipQueryPlanCache.Name, + sysvars.Socket.Name, + sysvars.SQLSelectLimit.Name, + sysvars.Version.Name, + sysvars.VersionComment.Name, + sysvars.QueryTimeout.Name, + sysvars.Workload.Name: + found = true + } + + if found { + cursor.Replace(NewArgument("__vt" + lowered)) + nz.bindVarNeeds.AddSysVar(lowered) + } +} + +// udvRewrite replaces user-defined variables with corresponding bind variables. +func (nz *normalizer) udvRewrite(cursor *Cursor, node *Variable) { + udv := strings.ToLower(node.Name.CompliantName()) + cursor.Replace(NewArgument(UserDefinedVariableName + udv)) + nz.bindVarNeeds.AddUserDefVar(udv) +} + +// funcRewrite replaces certain function expressions with bind variables. +func (nz *normalizer) funcRewrite(cursor *Cursor, node *FuncExpr) { + lowered := node.Name.Lowered() + if lowered == "last_insert_id" && len(node.Exprs) > 0 { + // Do not rewrite LAST_INSERT_ID() when it has arguments. + return + } + bindVar, found := funcRewrites[lowered] + if !found || (bindVar == DBVarName && !nz.shouldRewriteDatabaseFunc) { + return + } + if len(node.Exprs) > 0 { + nz.err = vterrors.Errorf(vtrpcpb.Code_UNIMPLEMENTED, "Argument to %s() not supported", lowered) + return + } + cursor.Replace(NewArgument(bindVar)) + nz.bindVarNeeds.AddFuncResult(bindVar) +} + +// unnestSubQueries attempts to simplify dual subqueries where possible. +// select (select database() from dual) from test +// => +// select database() from test +func (nz *normalizer) unnestSubQueries(cursor *Cursor, subquery *Subquery) { + if _, isExists := cursor.Parent().(*ExistsExpr); isExists { + return + } + sel, isSimpleSelect := subquery.Select.(*Select) + if !isSimpleSelect { + return + } + + if len(sel.SelectExprs) != 1 || + len(sel.OrderBy) != 0 || + sel.GroupBy != nil || + len(sel.From) != 1 || + sel.Where != nil || + sel.Having != nil || + sel.Limit != nil || sel.Lock != NoLock { + return + } + + aliasedTable, ok := sel.From[0].(*AliasedTableExpr) + if !ok { + return + } + table, ok := aliasedTable.Expr.(TableName) + if !ok || table.Name.String() != "dual" { + return + } + expr, ok := sel.SelectExprs[0].(*AliasedExpr) + if !ok { + return + } + _, isColName := expr.Expr.(*ColName) + if isColName { + // Skip if the subquery already returns a column name. + return + } + nz.bindVarNeeds.NoteRewrite() + rewritten := SafeRewrite(expr.Expr, nz.walkDown, nz.walkUp) + + // Handle special cases for IN clauses. + rewrittenExpr, isExpr := rewritten.(Expr) + _, isColTuple := rewritten.(ColTuple) + comparisonExpr, isCompExpr := cursor.Parent().(*ComparisonExpr) + if isCompExpr && (comparisonExpr.Operator == InOp || comparisonExpr.Operator == NotInOp) && !isColTuple && isExpr { + cursor.Replace(ValTuple{rewrittenExpr}) + return + } + + cursor.Replace(rewritten) +} + +// existsRewrite optimizes EXISTS expressions where possible. +func (nz *normalizer) existsRewrite(cursor *Cursor, node *ExistsExpr) { + sel, ok := node.Subquery.Select.(*Select) + if !ok { + return + } + + if sel.Having != nil { + // Cannot optimize if HAVING clause is present. + return + } + + if sel.GroupBy == nil && sel.SelectExprs.AllAggregation() { + // Replace EXISTS with a boolean true if guaranteed to be non-empty. + cursor.Replace(BoolVal(true)) + return + } + + // Simplify the subquery by selecting a constant. + // WHERE EXISTS(SELECT 1 FROM ...) + sel.SelectExprs = SelectExprs{ + &AliasedExpr{Expr: NewIntLiteral("1")}, + } + sel.GroupBy = nil +} + +// rewriteDistinctableAggr removes DISTINCT from certain aggregations to simplify the plan. +func (nz *normalizer) rewriteDistinctableAggr(node DistinctableAggr) { + if !node.IsDistinct() { + return + } + switch aggr := node.(type) { + case *Max, *Min: + aggr.SetDistinct(false) + nz.bindVarNeeds.NoteRewrite() + } +} + +func (nz *normalizer) shouldParameterize() bool { + return !(nz.inDerived > 0 && len(nz.onLeave) > 0) && nz.parameterize +} + +// SystemSchema checks if the given schema is a system schema. +func SystemSchema(schema string) bool { + return strings.EqualFold(schema, "information_schema") || + strings.EqualFold(schema, "performance_schema") || + strings.EqualFold(schema, "sys") || + strings.EqualFold(schema, "mysql") +} diff --git a/go/vt/sqlparser/normalizer_test.go b/go/vt/sqlparser/normalizer_test.go index 7919a321c91..7c3e660ac9d 100644 --- a/go/vt/sqlparser/normalizer_test.go +++ b/go/vt/sqlparser/normalizer_test.go @@ -25,8 +25,9 @@ import ( "strings" "testing" - "github.com/stretchr/testify/assert" + "vitess.io/vitess/go/vt/sysvars" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "vitess.io/vitess/go/sqltypes" @@ -448,10 +449,11 @@ func TestNormalize(t *testing.T) { t.Run(tc.in, func(t *testing.T) { stmt, err := parser.Parse(tc.in) require.NoError(t, err) - known := GetBindvars(stmt) + known := getBindvars(stmt) bv := make(map[string]*querypb.BindVariable) - require.NoError(t, Normalize(stmt, NewReservedVars(prefix, known), bv)) - assert.Equal(t, tc.outstmt, String(stmt)) + out, err := PrepareAST(stmt, NewReservedVars(prefix, known), bv, true, "ks", 0, "", map[string]string{}, nil, nil) + require.NoError(t, err) + assert.Equal(t, tc.outstmt, String(out.AST)) assert.Equal(t, tc.outbv, bv) }) } @@ -476,9 +478,10 @@ func TestNormalizeInvalidDates(t *testing.T) { t.Run(tc.in, func(t *testing.T) { stmt, err := parser.Parse(tc.in) require.NoError(t, err) - known := GetBindvars(stmt) + known := getBindvars(stmt) bv := make(map[string]*querypb.BindVariable) - require.EqualError(t, Normalize(stmt, NewReservedVars("bv", known), bv), tc.err.Error()) + _, err = PrepareAST(stmt, NewReservedVars("bv", known), bv, true, "ks", 0, "", map[string]string{}, nil, nil) + require.EqualError(t, err, tc.err.Error()) }) } } @@ -498,9 +501,10 @@ func TestNormalizeValidSQL(t *testing.T) { } bv := make(map[string]*querypb.BindVariable) known := make(BindVars) - err = Normalize(tree, NewReservedVars("vtg", known), bv) + + out, err := PrepareAST(tree, NewReservedVars("vtg", known), bv, true, "ks", 0, "", map[string]string{}, nil, nil) require.NoError(t, err) - normalizerOutput := String(tree) + normalizerOutput := String(out.AST) if normalizerOutput == "otheradmin" || normalizerOutput == "otherread" { return } @@ -529,9 +533,9 @@ func TestNormalizeOneCasae(t *testing.T) { } bv := make(map[string]*querypb.BindVariable) known := make(BindVars) - err = Normalize(tree, NewReservedVars("vtg", known), bv) + out, err := PrepareAST(tree, NewReservedVars("vtg", known), bv, true, "ks", 0, "", map[string]string{}, nil, nil) require.NoError(t, err) - normalizerOutput := String(tree) + normalizerOutput := String(out.AST) require.EqualValues(t, testOne.output, normalizerOutput) if normalizerOutput == "otheradmin" || normalizerOutput == "otherread" { return @@ -546,7 +550,7 @@ func TestGetBindVars(t *testing.T) { if err != nil { t.Fatal(err) } - got := GetBindvars(stmt) + got := getBindvars(stmt) want := map[string]struct{}{ "v1": {}, "v2": {}, @@ -559,6 +563,586 @@ func TestGetBindVars(t *testing.T) { } } +type testCaseSetVar struct { + in, expected, setVarComment string +} + +type testCaseSysVar struct { + in, expected string + sysVar map[string]string +} + +type myTestCase struct { + in, expected string + liid, db, foundRows, rowCount, rawGTID, rawTimeout, sessTrackGTID bool + ddlStrategy, migrationContext, sessionUUID, sessionEnableSystemSettings bool + udv int + autocommit, foreignKeyChecks, clientFoundRows, skipQueryPlanCache, socket, queryTimeout bool + sqlSelectLimit, transactionMode, workload, version, versionComment bool +} + +func TestRewrites(in *testing.T) { + tests := []myTestCase{{ + in: "SELECT 42", + expected: "SELECT 42", + // no bindvar needs + }, { + in: "SELECT @@version", + expected: "SELECT :__vtversion as `@@version`", + version: true, + }, { + in: "SELECT @@query_timeout", + expected: "SELECT :__vtquery_timeout as `@@query_timeout`", + queryTimeout: true, + }, { + in: "SELECT @@version_comment", + expected: "SELECT :__vtversion_comment as `@@version_comment`", + versionComment: true, + }, { + in: "SELECT @@enable_system_settings", + expected: "SELECT :__vtenable_system_settings as `@@enable_system_settings`", + sessionEnableSystemSettings: true, + }, { + in: "SELECT last_insert_id()", + expected: "SELECT :__lastInsertId as `last_insert_id()`", + liid: true, + }, { + in: "SELECT database()", + expected: "SELECT :__vtdbname as `database()`", + db: true, + }, { + in: "SELECT database() from test", + expected: "SELECT database() from test", + // no bindvar needs + }, { + in: "SELECT last_insert_id() as test", + expected: "SELECT :__lastInsertId as test", + liid: true, + }, { + in: "SELECT last_insert_id() + database()", + expected: "SELECT :__lastInsertId + :__vtdbname as `last_insert_id() + database()`", + db: true, liid: true, + }, { + // unnest database() call + in: "select (select database()) from test", + expected: "select database() as `(select database() from dual)` from test", + // no bindvar needs + }, { + // unnest database() call + in: "select (select database() from dual) from test", + expected: "select database() as `(select database() from dual)` from test", + // no bindvar needs + }, { + in: "select (select database() from dual) from dual", + expected: "select :__vtdbname as `(select database() from dual)` from dual", + db: true, + }, { + // don't unnest solo columns + in: "select 1 as foobar, (select foobar)", + expected: "select 1 as foobar, (select foobar from dual) from dual", + }, { + in: "select id from user where database()", + expected: "select id from user where database()", + // no bindvar needs + }, { + in: "select table_name from information_schema.tables where table_schema = database()", + expected: "select table_name from information_schema.tables where table_schema = database()", + // no bindvar needs + }, { + in: "select schema()", + expected: "select :__vtdbname as `schema()`", + db: true, + }, { + in: "select found_rows()", + expected: "select :__vtfrows as `found_rows()`", + foundRows: true, + }, { + in: "select @`x y`", + expected: "select :__vtudvx_y as `@``x y``` from dual", + udv: 1, + }, { + in: "select id from t where id = @x and val = @y", + expected: "select id from t where id = :__vtudvx and val = :__vtudvy", + db: false, udv: 2, + }, { + in: "insert into t(id) values(@xyx)", + expected: "insert into t(id) values(:__vtudvxyx)", + db: false, udv: 1, + }, { + in: "select row_count()", + expected: "select :__vtrcount as `row_count()`", + rowCount: true, + }, { + in: "SELECT lower(database())", + expected: "SELECT lower(:__vtdbname) as `lower(database())`", + db: true, + }, { + in: "SELECT @@autocommit", + expected: "SELECT :__vtautocommit as `@@autocommit`", + autocommit: true, + }, { + in: "SELECT @@client_found_rows", + expected: "SELECT :__vtclient_found_rows as `@@client_found_rows`", + clientFoundRows: true, + }, { + in: "SELECT @@skip_query_plan_cache", + expected: "SELECT :__vtskip_query_plan_cache as `@@skip_query_plan_cache`", + skipQueryPlanCache: true, + }, { + in: "SELECT @@sql_select_limit", + expected: "SELECT :__vtsql_select_limit as `@@sql_select_limit`", + sqlSelectLimit: true, + }, { + in: "SELECT @@transaction_mode", + expected: "SELECT :__vttransaction_mode as `@@transaction_mode`", + transactionMode: true, + }, { + in: "SELECT @@workload", + expected: "SELECT :__vtworkload as `@@workload`", + workload: true, + }, { + in: "SELECT @@socket", + expected: "SELECT :__vtsocket as `@@socket`", + socket: true, + }, { + in: "select (select 42) from dual", + expected: "select 42 as `(select 42 from dual)` from dual", + }, { + in: "select * from user where col = (select 42)", + expected: "select * from user where col = 42", + }, { + in: "select * from (select 42) as t", // this is not an expression, and should not be rewritten + expected: "select * from (select 42) as t", + }, { + in: `select (select (select (select (select (select last_insert_id()))))) as x`, + expected: "select :__lastInsertId as x from dual", + liid: true, + }, { + in: `select * from (select last_insert_id()) as t`, + expected: "select * from (select :__lastInsertId as `last_insert_id()` from dual) as t", + liid: true, + }, { + in: `select * from user where col = @@ddl_strategy`, + expected: "select * from user where col = :__vtddl_strategy", + ddlStrategy: true, + }, { + in: `select * from user where col = @@migration_context`, + expected: "select * from user where col = :__vtmigration_context", + migrationContext: true, + }, { + in: `select * from user where col = @@read_after_write_gtid OR col = @@read_after_write_timeout OR col = @@session_track_gtids`, + expected: "select * from user where col = :__vtread_after_write_gtid or col = :__vtread_after_write_timeout or col = :__vtsession_track_gtids", + rawGTID: true, rawTimeout: true, sessTrackGTID: true, + }, { + in: "SELECT * FROM tbl WHERE id IN (SELECT 1 FROM dual)", + expected: "SELECT * FROM tbl WHERE id IN (1)", + }, { + in: "SELECT * FROM tbl WHERE id IN (SELECT last_insert_id() FROM dual)", + expected: "SELECT * FROM tbl WHERE id IN (:__lastInsertId)", + liid: true, + }, { + in: "SELECT * FROM tbl WHERE id IN (SELECT (SELECT 1 FROM dual WHERE 1 = 0) FROM dual)", + expected: "SELECT * FROM tbl WHERE id IN (SELECT 1 FROM dual WHERE 1 = 0)", + }, { + in: "SELECT * FROM tbl WHERE id IN (SELECT 1 FROM dual WHERE 1 = 0)", + expected: "SELECT * FROM tbl WHERE id IN (SELECT 1 FROM dual WHERE 1 = 0)", + }, { + in: "SELECT * FROM tbl WHERE id IN (SELECT 1,2 FROM dual)", + expected: "SELECT * FROM tbl WHERE id IN (SELECT 1,2 FROM dual)", + }, { + in: "SELECT * FROM tbl WHERE id IN (SELECT 1 FROM dual ORDER BY 1)", + expected: "SELECT * FROM tbl WHERE id IN (SELECT 1 FROM dual ORDER BY 1)", + }, { + in: "SELECT * FROM tbl WHERE id IN (SELECT id FROM user GROUP BY id)", + expected: "SELECT * FROM tbl WHERE id IN (SELECT id FROM user GROUP BY id)", + }, { + in: "SELECT * FROM tbl WHERE id IN (SELECT 1 FROM dual, user)", + expected: "SELECT * FROM tbl WHERE id IN (SELECT 1 FROM dual, user)", + }, { + in: "SELECT * FROM tbl WHERE id IN (SELECT 1 FROM dual limit 1)", + expected: "SELECT * FROM tbl WHERE id IN (SELECT 1 FROM dual limit 1)", + }, { + // SELECT * behaves different depending the join type used, so if that has been used, we won't rewrite + in: "SELECT * FROM A JOIN B USING (id1,id2,id3)", + expected: "SELECT * FROM A JOIN B USING (id1,id2,id3)", + }, { + in: "CALL proc(@foo)", + expected: "CALL proc(:__vtudvfoo)", + udv: 1, + }, { + in: "SELECT * FROM tbl WHERE NOT id = 42", + expected: "SELECT * FROM tbl WHERE id != 42", + }, { + in: "SELECT * FROM tbl WHERE not id < 12", + expected: "SELECT * FROM tbl WHERE id >= 12", + }, { + in: "SELECT * FROM tbl WHERE not id > 12", + expected: "SELECT * FROM tbl WHERE id <= 12", + }, { + in: "SELECT * FROM tbl WHERE not id <= 33", + expected: "SELECT * FROM tbl WHERE id > 33", + }, { + in: "SELECT * FROM tbl WHERE not id >= 33", + expected: "SELECT * FROM tbl WHERE id < 33", + }, { + in: "SELECT * FROM tbl WHERE not id != 33", + expected: "SELECT * FROM tbl WHERE id = 33", + }, { + in: "SELECT * FROM tbl WHERE not id in (1,2,3)", + expected: "SELECT * FROM tbl WHERE id not in (1,2,3)", + }, { + in: "SELECT * FROM tbl WHERE not id not in (1,2,3)", + expected: "SELECT * FROM tbl WHERE id in (1,2,3)", + }, { + in: "SELECT * FROM tbl WHERE not id not in (1,2,3)", + expected: "SELECT * FROM tbl WHERE id in (1,2,3)", + }, { + in: "SELECT * FROM tbl WHERE not id like '%foobar'", + expected: "SELECT * FROM tbl WHERE id not like '%foobar'", + }, { + in: "SELECT * FROM tbl WHERE not id not like '%foobar'", + expected: "SELECT * FROM tbl WHERE id like '%foobar'", + }, { + in: "SELECT * FROM tbl WHERE not id regexp '%foobar'", + expected: "SELECT * FROM tbl WHERE id not regexp '%foobar'", + }, { + in: "SELECT * FROM tbl WHERE not id not regexp '%foobar'", + expected: "select * from tbl where id regexp '%foobar'", + }, { + in: "SELECT * FROM tbl WHERE exists(select col1, col2 from other_table where foo > bar)", + expected: "SELECT * FROM tbl WHERE exists(select 1 from other_table where foo > bar)", + }, { + in: "SELECT * FROM tbl WHERE exists(select col1, col2 from other_table where foo > bar limit 100 offset 34)", + expected: "SELECT * FROM tbl WHERE exists(select 1 from other_table where foo > bar limit 100 offset 34)", + }, { + in: "SELECT * FROM tbl WHERE exists(select col1, col2, count(*) from other_table where foo > bar group by col1, col2)", + expected: "SELECT * FROM tbl WHERE exists(select 1 from other_table where foo > bar)", + }, { + in: "SELECT * FROM tbl WHERE exists(select col1, col2 from other_table where foo > bar group by col1, col2)", + expected: "SELECT * FROM tbl WHERE exists(select 1 from other_table where foo > bar)", + }, { + in: "SELECT * FROM tbl WHERE exists(select count(*) from other_table where foo > bar)", + expected: "SELECT * FROM tbl WHERE true", + }, { + in: "SELECT * FROM tbl WHERE exists(select col1, col2, count(*) from other_table where foo > bar group by col1, col2 having count(*) > 3)", + expected: "SELECT * FROM tbl WHERE exists(select col1, col2, count(*) from other_table where foo > bar group by col1, col2 having count(*) > 3)", + }, { + in: "SELECT id, name, salary FROM user_details", + expected: "SELECT id, name, salary FROM (select user.id, user.name, user_extra.salary from user join user_extra where user.id = user_extra.user_id) as user_details", + }, { + in: "select max(distinct c1), min(distinct c2), avg(distinct c3), sum(distinct c4), count(distinct c5), group_concat(distinct c6) from tbl", + expected: "select max(c1) as `max(distinct c1)`, min(c2) as `min(distinct c2)`, avg(distinct c3), sum(distinct c4), count(distinct c5), group_concat(distinct c6) from tbl", + }, { + in: "SHOW VARIABLES", + expected: "SHOW VARIABLES", + autocommit: true, + foreignKeyChecks: true, + clientFoundRows: true, + skipQueryPlanCache: true, + sqlSelectLimit: true, + transactionMode: true, + workload: true, + version: true, + versionComment: true, + ddlStrategy: true, + migrationContext: true, + sessionUUID: true, + sessionEnableSystemSettings: true, + rawGTID: true, + rawTimeout: true, + sessTrackGTID: true, + socket: true, + queryTimeout: true, + }, { + in: "SHOW GLOBAL VARIABLES", + expected: "SHOW GLOBAL VARIABLES", + autocommit: true, + foreignKeyChecks: true, + clientFoundRows: true, + skipQueryPlanCache: true, + sqlSelectLimit: true, + transactionMode: true, + workload: true, + version: true, + versionComment: true, + ddlStrategy: true, + migrationContext: true, + sessionUUID: true, + sessionEnableSystemSettings: true, + rawGTID: true, + rawTimeout: true, + sessTrackGTID: true, + socket: true, + queryTimeout: true, + }} + parser := NewTestParser() + for _, tc := range tests { + in.Run(tc.in, func(t *testing.T) { + require := require.New(t) + stmt, known, err := parser.Parse2(tc.in) + require.NoError(err) + vars := NewReservedVars("v", known) + result, err := PrepareAST( + stmt, + vars, + map[string]*querypb.BindVariable{}, + false, + "ks", + 0, + "", + map[string]string{}, + nil, + &fakeViews{}, + ) + require.NoError(err) + + expected, err := parser.Parse(tc.expected) + require.NoError(err, "test expectation does not parse [%s]", tc.expected) + + s := String(expected) + assert := assert.New(t) + assert.Equal(s, String(result.AST)) + assert.Equal(tc.liid, result.NeedsFuncResult(LastInsertIDName), "should need last insert id") + assert.Equal(tc.db, result.NeedsFuncResult(DBVarName), "should need database name") + assert.Equal(tc.foundRows, result.NeedsFuncResult(FoundRowsName), "should need found rows") + assert.Equal(tc.rowCount, result.NeedsFuncResult(RowCountName), "should need row count") + assert.Equal(tc.udv, len(result.NeedUserDefinedVariables), "count of user defined variables") + assert.Equal(tc.autocommit, result.NeedsSysVar(sysvars.Autocommit.Name), "should need :__vtautocommit") + assert.Equal(tc.foreignKeyChecks, result.NeedsSysVar(sysvars.ForeignKeyChecks), "should need :__vtforeignKeyChecks") + assert.Equal(tc.clientFoundRows, result.NeedsSysVar(sysvars.ClientFoundRows.Name), "should need :__vtclientFoundRows") + assert.Equal(tc.skipQueryPlanCache, result.NeedsSysVar(sysvars.SkipQueryPlanCache.Name), "should need :__vtskipQueryPlanCache") + assert.Equal(tc.sqlSelectLimit, result.NeedsSysVar(sysvars.SQLSelectLimit.Name), "should need :__vtsqlSelectLimit") + assert.Equal(tc.transactionMode, result.NeedsSysVar(sysvars.TransactionMode.Name), "should need :__vttransactionMode") + assert.Equal(tc.workload, result.NeedsSysVar(sysvars.Workload.Name), "should need :__vtworkload") + assert.Equal(tc.queryTimeout, result.NeedsSysVar(sysvars.QueryTimeout.Name), "should need :__vtquery_timeout") + assert.Equal(tc.ddlStrategy, result.NeedsSysVar(sysvars.DDLStrategy.Name), "should need ddlStrategy") + assert.Equal(tc.migrationContext, result.NeedsSysVar(sysvars.MigrationContext.Name), "should need migrationContext") + assert.Equal(tc.sessionUUID, result.NeedsSysVar(sysvars.SessionUUID.Name), "should need sessionUUID") + assert.Equal(tc.sessionEnableSystemSettings, result.NeedsSysVar(sysvars.SessionEnableSystemSettings.Name), "should need sessionEnableSystemSettings") + assert.Equal(tc.rawGTID, result.NeedsSysVar(sysvars.ReadAfterWriteGTID.Name), "should need rawGTID") + assert.Equal(tc.rawTimeout, result.NeedsSysVar(sysvars.ReadAfterWriteTimeOut.Name), "should need rawTimeout") + assert.Equal(tc.sessTrackGTID, result.NeedsSysVar(sysvars.SessionTrackGTIDs.Name), "should need sessTrackGTID") + assert.Equal(tc.version, result.NeedsSysVar(sysvars.Version.Name), "should need Vitess version") + assert.Equal(tc.versionComment, result.NeedsSysVar(sysvars.VersionComment.Name), "should need Vitess version") + assert.Equal(tc.socket, result.NeedsSysVar(sysvars.Socket.Name), "should need :__vtsocket") + }) + } +} + +type fakeViews struct{} + +func (*fakeViews) FindView(name TableName) TableStatement { + if name.Name.String() != "user_details" { + return nil + } + parser := NewTestParser() + statement, err := parser.Parse("select user.id, user.name, user_extra.salary from user join user_extra where user.id = user_extra.user_id") + if err != nil { + return nil + } + return statement.(TableStatement) +} + +func TestRewritesWithSetVarComment(in *testing.T) { + tests := []testCaseSetVar{{ + in: "select 1", + expected: "select 1", + setVarComment: "", + }, { + in: "select 1", + expected: "select /*+ AA(a) */ 1", + setVarComment: "AA(a)", + }, { + in: "insert /* toto */ into t(id) values(1)", + expected: "insert /*+ AA(a) */ /* toto */ into t(id) values(1)", + setVarComment: "AA(a)", + }, { + in: "select /* toto */ * from t union select * from s", + expected: "select /*+ AA(a) */ /* toto */ * from t union select /*+ AA(a) */ * from s", + setVarComment: "AA(a)", + }, { + in: "vstream /* toto */ * from t1", + expected: "vstream /*+ AA(a) */ /* toto */ * from t1", + setVarComment: "AA(a)", + }, { + in: "stream /* toto */ t from t1", + expected: "stream /*+ AA(a) */ /* toto */ t from t1", + setVarComment: "AA(a)", + }, { + in: "update /* toto */ t set id = 1", + expected: "update /*+ AA(a) */ /* toto */ t set id = 1", + setVarComment: "AA(a)", + }, { + in: "delete /* toto */ from t", + expected: "delete /*+ AA(a) */ /* toto */ from t", + setVarComment: "AA(a)", + }} + + parser := NewTestParser() + for _, tc := range tests { + in.Run(tc.in, func(t *testing.T) { + require := require.New(t) + stmt, err := parser.Parse(tc.in) + require.NoError(err) + vars := NewReservedVars("v", nil) + result, err := PrepareAST( + stmt, + vars, + map[string]*querypb.BindVariable{}, + false, + "ks", + 0, + tc.setVarComment, + map[string]string{}, + nil, + &fakeViews{}, + ) + + require.NoError(err) + + expected, err := parser.Parse(tc.expected) + require.NoError(err, "test expectation does not parse [%s]", tc.expected) + + assert.Equal(t, String(expected), String(result.AST)) + }) + } +} + +func TestRewritesSysVar(in *testing.T) { + tests := []testCaseSysVar{{ + in: "select @x = @@sql_mode", + expected: "select :__vtudvx = @@sql_mode as `@x = @@sql_mode` from dual", + }, { + in: "select @x = @@sql_mode", + expected: "select :__vtudvx = :__vtsql_mode as `@x = @@sql_mode` from dual", + sysVar: map[string]string{"sql_mode": "' '"}, + }, { + in: "SELECT @@tx_isolation", + expected: "select @@tx_isolation from dual", + }, { + in: "SELECT @@transaction_isolation", + expected: "select @@transaction_isolation from dual", + }, { + in: "SELECT @@session.transaction_isolation", + expected: "select @@session.transaction_isolation from dual", + }, { + in: "SELECT @@tx_isolation", + sysVar: map[string]string{"tx_isolation": "'READ-COMMITTED'"}, + expected: "select :__vttx_isolation as `@@tx_isolation` from dual", + }, { + in: "SELECT @@transaction_isolation", + sysVar: map[string]string{"transaction_isolation": "'READ-COMMITTED'"}, + expected: "select :__vttransaction_isolation as `@@transaction_isolation` from dual", + }, { + in: "SELECT @@session.transaction_isolation", + sysVar: map[string]string{"transaction_isolation": "'READ-COMMITTED'"}, + expected: "select :__vttransaction_isolation as `@@session.transaction_isolation` from dual", + }} + + parser := NewTestParser() + for _, tc := range tests { + in.Run(tc.in, func(t *testing.T) { + require := require.New(t) + stmt, err := parser.Parse(tc.in) + require.NoError(err) + vars := NewReservedVars("v", nil) + result, err := PrepareAST( + stmt, + vars, + map[string]*querypb.BindVariable{}, + false, + "ks", + 0, + "", + tc.sysVar, + nil, + &fakeViews{}, + ) + + require.NoError(err) + + expected, err := parser.Parse(tc.expected) + require.NoError(err, "test expectation does not parse [%s]", tc.expected) + + assert.Equal(t, String(expected), String(result.AST)) + }) + } +} + +func TestRewritesWithDefaultKeyspace(in *testing.T) { + tests := []myTestCase{{ + in: "SELECT 1 from x.test", + expected: "SELECT 1 from x.test", // no change + }, { + in: "SELECT x.col as c from x.test", + expected: "SELECT x.col as c from x.test", // no change + }, { + in: "SELECT 1 from test", + expected: "SELECT 1 from sys.test", + }, { + in: "SELECT 1 from test as t", + expected: "SELECT 1 from sys.test as t", + }, { + in: "SELECT 1 from `test 24` as t", + expected: "SELECT 1 from sys.`test 24` as t", + }, { + in: "SELECT 1, (select 1 from test) from x.y", + expected: "SELECT 1, (select 1 from sys.test) from x.y", + }, { + in: "SELECT 1 from (select 2 from test) t", + expected: "SELECT 1 from (select 2 from sys.test) t", + }, { + in: "SELECT 1 from test where exists(select 2 from test)", + expected: "SELECT 1 from sys.test where exists(select 1 from sys.test)", + }, { + in: "SELECT 1 from dual", + expected: "SELECT 1 from dual", + }, { + in: "SELECT (select 2 from dual) from DUAL", + expected: "SELECT 2 as `(select 2 from dual)` from DUAL", + }} + + parser := NewTestParser() + for _, tc := range tests { + in.Run(tc.in, func(t *testing.T) { + require := require.New(t) + stmt, err := parser.Parse(tc.in) + require.NoError(err) + vars := NewReservedVars("v", nil) + result, err := PrepareAST( + stmt, + vars, + map[string]*querypb.BindVariable{}, + false, + "sys", + 0, + "", + map[string]string{}, + nil, + &fakeViews{}, + ) + + require.NoError(err) + + expected, err := parser.Parse(tc.expected) + require.NoError(err, "test expectation does not parse [%s]", tc.expected) + + assert.Equal(t, String(expected), String(result.AST)) + }) + } +} + +func TestReservedVars(t *testing.T) { + for _, prefix := range []string{"vtg", "bv"} { + t.Run("prefix_"+prefix, func(t *testing.T) { + reserved := NewReservedVars(prefix, make(BindVars)) + for i := 1; i < 1000; i++ { + require.Equal(t, fmt.Sprintf("%s%d", prefix, i), reserved.nextUnusedVar()) + } + }) + } +} + /* Skipping ColName, TableName: BenchmarkNormalize-8 1000000 2205 ns/op 821 B/op 27 allocs/op @@ -573,7 +1157,8 @@ func BenchmarkNormalize(b *testing.B) { b.Fatal(err) } for i := 0; i < b.N; i++ { - require.NoError(b, Normalize(ast, NewReservedVars("", reservedVars), map[string]*querypb.BindVariable{})) + _, err := PrepareAST(ast, NewReservedVars("", reservedVars), map[string]*querypb.BindVariable{}, true, "ks", 0, "", map[string]string{}, nil, nil) + require.NoError(b, err) } } @@ -602,7 +1187,8 @@ func BenchmarkNormalizeTraces(b *testing.B) { for i := 0; i < b.N; i++ { for i, query := range parsed { - _ = Normalize(query, NewReservedVars("", reservedVars[i]), map[string]*querypb.BindVariable{}) + _, err := PrepareAST(query, NewReservedVars("", reservedVars[i]), map[string]*querypb.BindVariable{}, true, "ks", 0, "", map[string]string{}, nil, nil) + require.NoError(b, err) } } }) diff --git a/go/vt/sqlparser/redact_query.go b/go/vt/sqlparser/redact_query.go index e6b8c009c68..2d018d7c0eb 100644 --- a/go/vt/sqlparser/redact_query.go +++ b/go/vt/sqlparser/redact_query.go @@ -28,10 +28,10 @@ func (p *Parser) RedactSQLQuery(sql string) (string, error) { return "", err } - err = Normalize(stmt, NewReservedVars("redacted", reservedVars), bv) + out, err := PrepareAST(stmt, NewReservedVars("redacted", reservedVars), bv, true, "ks", 0, "", map[string]string{}, nil, nil) if err != nil { return "", err } - return comments.Leading + String(stmt) + comments.Trailing, nil + return comments.Leading + String(out.AST) + comments.Trailing, nil } diff --git a/go/vt/sqlparser/utils.go b/go/vt/sqlparser/utils.go index b785128917f..c56e7740fc5 100644 --- a/go/vt/sqlparser/utils.go +++ b/go/vt/sqlparser/utils.go @@ -41,11 +41,12 @@ func (p *Parser) QueryMatchesTemplates(query string, queryTemplates []string) (m if err != nil { return "", err } - err = Normalize(stmt, NewReservedVars("", reservedVars), bv) + + out, err := PrepareAST(stmt, NewReservedVars("", reservedVars), bv, true, "ks", 0, "", map[string]string{}, nil, nil) if err != nil { return "", err } - normalized := CanonicalString(stmt) + normalized := CanonicalString(out.AST) return normalized, nil } diff --git a/go/vt/vtgate/executor_test.go b/go/vt/vtgate/executor_test.go index 904805e789b..5e7e5c2a07d 100644 --- a/go/vt/vtgate/executor_test.go +++ b/go/vt/vtgate/executor_test.go @@ -1860,6 +1860,30 @@ func TestPassthroughDDL(t *testing.T) { sbc2.Queries = nil } +func TestShowStatus(t *testing.T) { + executor, sbc1, _, _, ctx := createExecutorEnvWithConfig(t, createExecutorConfigWithNormalizer()) + session := &vtgatepb.Session{ + TargetString: "TestExecutor", + } + + sql1 := "show slave status" + _, err := executorExec(ctx, executor, session, sql1, nil) + require.NoError(t, err) + + sql2 := "show replica status" + _, err = executorExec(ctx, executor, session, sql2, nil) + require.NoError(t, err) + + wantQueries := []*querypb.BoundQuery{{ + Sql: sql1, + BindVariables: map[string]*querypb.BindVariable{}, + }, { + Sql: sql2, + BindVariables: map[string]*querypb.BindVariable{}, + }} + assert.Equal(t, wantQueries, sbc1.Queries) +} + func TestParseEmptyTargetSingleKeyspace(t *testing.T) { r, _, _, _, _ := createExecutorEnv(t) diff --git a/go/vt/vtgate/planbuilder/builder.go b/go/vt/vtgate/planbuilder/builder.go index 065c50a6dfa..85d9f5f94ea 100644 --- a/go/vt/vtgate/planbuilder/builder.go +++ b/go/vt/vtgate/planbuilder/builder.go @@ -73,7 +73,7 @@ func (staticConfig) DirectEnabled() bool { // TestBuilder builds a plan for a query based on the specified vschema. // This method is only used from tests func TestBuilder(query string, vschema plancontext.VSchema, keyspace string) (*engine.Plan, error) { - stmt, reserved, err := vschema.Environment().Parser().Parse2(query) + stmt, known, err := vschema.Environment().Parser().Parse2(query) if err != nil { return nil, err } @@ -93,12 +93,12 @@ func TestBuilder(query string, vschema plancontext.VSchema, keyspace string) (*e }() } } - result, err := sqlparser.RewriteAST(stmt, keyspace, sqlparser.SQLSelectLimitUnset, "", nil, vschema.GetForeignKeyChecksState(), vschema) + reservedVars := sqlparser.NewReservedVars("vtg", known) + result, err := sqlparser.PrepareAST(stmt, reservedVars, map[string]*querypb.BindVariable{}, false, keyspace, sqlparser.SQLSelectLimitUnset, "", nil, vschema.GetForeignKeyChecksState(), vschema) if err != nil { return nil, err } - reservedVars := sqlparser.NewReservedVars("vtg", reserved) return BuildFromStmt(context.Background(), query, result.AST, reservedVars, vschema, result.BindVarNeeds, staticConfig{}) } diff --git a/go/vt/vtgate/planbuilder/simplifier_test.go b/go/vt/vtgate/planbuilder/simplifier_test.go index 5aeb0565f9b..012475ba021 100644 --- a/go/vt/vtgate/planbuilder/simplifier_test.go +++ b/go/vt/vtgate/planbuilder/simplifier_test.go @@ -21,6 +21,8 @@ import ( "fmt" "testing" + querypb "vitess.io/vitess/go/vt/proto/query" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -45,8 +47,8 @@ func TestSimplifyBuggyQuery(t *testing.T) { stmt, reserved, err := sqlparser.NewTestParser().Parse2(query) require.NoError(t, err) - rewritten, _ := sqlparser.RewriteAST(sqlparser.Clone(stmt), vw.CurrentDb(), sqlparser.SQLSelectLimitUnset, "", nil, nil, nil) reservedVars := sqlparser.NewReservedVars("vtg", reserved) + rewritten, _ := sqlparser.PrepareAST(sqlparser.Clone(stmt), reservedVars, map[string]*querypb.BindVariable{}, false, vw.CurrentDb(), sqlparser.SQLSelectLimitUnset, "", nil, nil, nil) simplified := simplifier.SimplifyStatement( stmt.(sqlparser.TableStatement), @@ -69,8 +71,8 @@ func TestSimplifyPanic(t *testing.T) { stmt, reserved, err := sqlparser.NewTestParser().Parse2(query) require.NoError(t, err) - rewritten, _ := sqlparser.RewriteAST(sqlparser.Clone(stmt), vw.CurrentDb(), sqlparser.SQLSelectLimitUnset, "", nil, nil, nil) reservedVars := sqlparser.NewReservedVars("vtg", reserved) + rewritten, _ := sqlparser.PrepareAST(sqlparser.Clone(stmt), reservedVars, map[string]*querypb.BindVariable{}, false, vw.CurrentDb(), sqlparser.SQLSelectLimitUnset, "", nil, nil, nil) simplified := simplifier.SimplifyStatement( stmt.(sqlparser.TableStatement), @@ -100,12 +102,12 @@ func TestUnsupportedFile(t *testing.T) { t.Skip() return } - rewritten, err := sqlparser.RewriteAST(stmt, vw.CurrentDb(), sqlparser.SQLSelectLimitUnset, "", nil, nil, nil) + reservedVars := sqlparser.NewReservedVars("vtg", reserved) + rewritten, err := sqlparser.PrepareAST(stmt, reservedVars, map[string]*querypb.BindVariable{}, false, vw.CurrentDb(), sqlparser.SQLSelectLimitUnset, "", nil, nil, nil) if err != nil { t.Skip() } - reservedVars := sqlparser.NewReservedVars("vtg", reserved) ast := rewritten.AST origQuery := sqlparser.String(ast) stmt, _, _ = sqlparser.NewTestParser().Parse2(tcase.Query) @@ -133,7 +135,7 @@ func keepSameError(query string, reservedVars *sqlparser.ReservedVars, vschema * if err != nil { panic(err) } - rewritten, _ := sqlparser.RewriteAST(stmt, vschema.CurrentDb(), sqlparser.SQLSelectLimitUnset, "", nil, nil, nil) + rewritten, _ := sqlparser.PrepareAST(stmt, reservedVars, map[string]*querypb.BindVariable{}, false, vschema.CurrentDb(), sqlparser.SQLSelectLimitUnset, "", nil, nil, nil) ast := rewritten.AST _, expected := BuildFromStmt(context.Background(), query, ast, reservedVars, vschema, rewritten.BindVarNeeds, staticConfig{}) if expected == nil { diff --git a/go/vt/vtgate/semantics/typer_test.go b/go/vt/vtgate/semantics/typer_test.go index 7de5ecf1340..1ec642b8168 100644 --- a/go/vt/vtgate/semantics/typer_test.go +++ b/go/vt/vtgate/semantics/typer_test.go @@ -41,15 +41,16 @@ func TestNormalizerAndSemanticAnalysisIntegration(t *testing.T) { for _, test := range tests { t.Run(test.query, func(t *testing.T) { - parse, err := sqlparser.NewTestParser().Parse(test.query) + parse, known, err := sqlparser.NewTestParser().Parse2(test.query) require.NoError(t, err) - err = sqlparser.Normalize(parse, sqlparser.NewReservedVars("bv", sqlparser.BindVars{}), map[string]*querypb.BindVariable{}) + rv := sqlparser.NewReservedVars("", known) + out, err := sqlparser.PrepareAST(parse, rv, map[string]*querypb.BindVariable{}, true, "d", 0, "", map[string]string{}, nil, nil) require.NoError(t, err) - st, err := Analyze(parse, "d", fakeSchemaInfo()) + st, err := Analyze(out.AST, "d", fakeSchemaInfo()) require.NoError(t, err) - bv := parse.(*sqlparser.Select).SelectExprs[0].(*sqlparser.AliasedExpr).Expr.(*sqlparser.Argument) + bv := out.AST.(*sqlparser.Select).SelectExprs[0].(*sqlparser.AliasedExpr).Expr.(*sqlparser.Argument) typ, found := st.ExprTypes[bv] require.True(t, found, "bindvar was not typed") require.Equal(t, test.typ, typ.Type().String()) @@ -68,15 +69,15 @@ func TestColumnCollations(t *testing.T) { for _, test := range tests { t.Run(test.query, func(t *testing.T) { - parse, err := sqlparser.NewTestParser().Parse(test.query) + ast, err := sqlparser.NewTestParser().Parse(test.query) require.NoError(t, err) - err = sqlparser.Normalize(parse, sqlparser.NewReservedVars("bv", sqlparser.BindVars{}), map[string]*querypb.BindVariable{}) + out, err := sqlparser.PrepareAST(ast, sqlparser.NewReservedVars("bv", sqlparser.BindVars{}), map[string]*querypb.BindVariable{}, true, "d", 0, "", map[string]string{}, nil, nil) require.NoError(t, err) - st, err := Analyze(parse, "d", fakeSchemaInfo()) + st, err := Analyze(out.AST, "d", fakeSchemaInfo()) require.NoError(t, err) - col := extract(parse.(*sqlparser.Select), 0) + col := extract(out.AST.(*sqlparser.Select), 0) typ, found := st.TypeForExpr(col) require.True(t, found, "column was not typed") From fd1186c6a92ce41c663fe3b45b58e0299e99173d Mon Sep 17 00:00:00 2001 From: Chaitanya Rangavajhala Date: Tue, 28 Jan 2025 08:48:29 -0500 Subject: [PATCH 09/11] VTAdmin to use VTGate's vexplain (#17508) Signed-off-by: c-r-dev --- go/vt/proto/vtadmin/vtadmin.pb.go | 641 +++++++++++------- go/vt/proto/vtadmin/vtadmin_grpc.pb.go | 40 ++ go/vt/proto/vtadmin/vtadmin_vtproto.pb.go | 396 +++++++++++ go/vt/vtadmin/api.go | 54 ++ go/vt/vtadmin/api_test.go | 180 +++++ go/vt/vtadmin/http/vexplain.go | 34 + go/vt/vtadmin/rbac/rbac.go | 2 + go/vt/vtadmin/vtsql/fakevtsql/conn.go | 11 + go/vt/vtadmin/vtsql/vtsql.go | 74 ++ proto/vtadmin.proto | 13 + web/vtadmin/src/api/http.ts | 16 + web/vtadmin/src/components/App.tsx | 5 + web/vtadmin/src/components/NavRail.tsx | 3 + .../src/components/routes/VExplain.tsx | 153 +++++ .../components/routes/VTExplain.module.scss | 2 +- web/vtadmin/src/hooks/api.ts | 8 + web/vtadmin/src/proto/vtadmin.d.ts | 227 +++++++ web/vtadmin/src/proto/vtadmin.js | 486 +++++++++++++ 18 files changed, 2085 insertions(+), 260 deletions(-) create mode 100644 go/vt/vtadmin/http/vexplain.go create mode 100644 web/vtadmin/src/components/routes/VExplain.tsx diff --git a/go/vt/proto/vtadmin/vtadmin.pb.go b/go/vt/proto/vtadmin/vtadmin.pb.go index 8b6a6997c8d..c086eb01443 100644 --- a/go/vt/proto/vtadmin/vtadmin.pb.go +++ b/go/vt/proto/vtadmin/vtadmin.pb.go @@ -7200,6 +7200,112 @@ func (x *VTExplainResponse) GetResponse() string { return "" } +type VExplainRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + ClusterId string `protobuf:"bytes,1,opt,name=cluster_id,json=clusterId,proto3" json:"cluster_id,omitempty"` + Keyspace string `protobuf:"bytes,2,opt,name=keyspace,proto3" json:"keyspace,omitempty"` + Sql string `protobuf:"bytes,3,opt,name=sql,proto3" json:"sql,omitempty"` +} + +func (x *VExplainRequest) Reset() { + *x = VExplainRequest{} + mi := &file_vtadmin_proto_msgTypes[126] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *VExplainRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*VExplainRequest) ProtoMessage() {} + +func (x *VExplainRequest) ProtoReflect() protoreflect.Message { + mi := &file_vtadmin_proto_msgTypes[126] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use VExplainRequest.ProtoReflect.Descriptor instead. +func (*VExplainRequest) Descriptor() ([]byte, []int) { + return file_vtadmin_proto_rawDescGZIP(), []int{126} +} + +func (x *VExplainRequest) GetClusterId() string { + if x != nil { + return x.ClusterId + } + return "" +} + +func (x *VExplainRequest) GetKeyspace() string { + if x != nil { + return x.Keyspace + } + return "" +} + +func (x *VExplainRequest) GetSql() string { + if x != nil { + return x.Sql + } + return "" +} + +type VExplainResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Response string `protobuf:"bytes,1,opt,name=response,proto3" json:"response,omitempty"` +} + +func (x *VExplainResponse) Reset() { + *x = VExplainResponse{} + mi := &file_vtadmin_proto_msgTypes[127] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *VExplainResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*VExplainResponse) ProtoMessage() {} + +func (x *VExplainResponse) ProtoReflect() protoreflect.Message { + mi := &file_vtadmin_proto_msgTypes[127] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use VExplainResponse.ProtoReflect.Descriptor instead. +func (*VExplainResponse) Descriptor() ([]byte, []int) { + return file_vtadmin_proto_rawDescGZIP(), []int{127} +} + +func (x *VExplainResponse) GetResponse() string { + if x != nil { + return x.Response + } + return "" +} + type Schema_ShardTableSize struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -7211,7 +7317,7 @@ type Schema_ShardTableSize struct { func (x *Schema_ShardTableSize) Reset() { *x = Schema_ShardTableSize{} - mi := &file_vtadmin_proto_msgTypes[129] + mi := &file_vtadmin_proto_msgTypes[131] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7223,7 +7329,7 @@ func (x *Schema_ShardTableSize) String() string { func (*Schema_ShardTableSize) ProtoMessage() {} func (x *Schema_ShardTableSize) ProtoReflect() protoreflect.Message { - mi := &file_vtadmin_proto_msgTypes[129] + mi := &file_vtadmin_proto_msgTypes[131] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7267,7 +7373,7 @@ type Schema_TableSize struct { func (x *Schema_TableSize) Reset() { *x = Schema_TableSize{} - mi := &file_vtadmin_proto_msgTypes[130] + mi := &file_vtadmin_proto_msgTypes[132] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7279,7 +7385,7 @@ func (x *Schema_TableSize) String() string { func (*Schema_TableSize) ProtoMessage() {} func (x *Schema_TableSize) ProtoReflect() protoreflect.Message { - mi := &file_vtadmin_proto_msgTypes[130] + mi := &file_vtadmin_proto_msgTypes[132] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7327,7 +7433,7 @@ type GetSchemaMigrationsRequest_ClusterRequest struct { func (x *GetSchemaMigrationsRequest_ClusterRequest) Reset() { *x = GetSchemaMigrationsRequest_ClusterRequest{} - mi := &file_vtadmin_proto_msgTypes[132] + mi := &file_vtadmin_proto_msgTypes[134] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7339,7 +7445,7 @@ func (x *GetSchemaMigrationsRequest_ClusterRequest) String() string { func (*GetSchemaMigrationsRequest_ClusterRequest) ProtoMessage() {} func (x *GetSchemaMigrationsRequest_ClusterRequest) ProtoReflect() protoreflect.Message { - mi := &file_vtadmin_proto_msgTypes[132] + mi := &file_vtadmin_proto_msgTypes[134] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7386,7 +7492,7 @@ type ReloadSchemasResponse_KeyspaceResult struct { func (x *ReloadSchemasResponse_KeyspaceResult) Reset() { *x = ReloadSchemasResponse_KeyspaceResult{} - mi := &file_vtadmin_proto_msgTypes[135] + mi := &file_vtadmin_proto_msgTypes[137] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7398,7 +7504,7 @@ func (x *ReloadSchemasResponse_KeyspaceResult) String() string { func (*ReloadSchemasResponse_KeyspaceResult) ProtoMessage() {} func (x *ReloadSchemasResponse_KeyspaceResult) ProtoReflect() protoreflect.Message { - mi := &file_vtadmin_proto_msgTypes[135] + mi := &file_vtadmin_proto_msgTypes[137] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7445,7 +7551,7 @@ type ReloadSchemasResponse_ShardResult struct { func (x *ReloadSchemasResponse_ShardResult) Reset() { *x = ReloadSchemasResponse_ShardResult{} - mi := &file_vtadmin_proto_msgTypes[136] + mi := &file_vtadmin_proto_msgTypes[138] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7457,7 +7563,7 @@ func (x *ReloadSchemasResponse_ShardResult) String() string { func (*ReloadSchemasResponse_ShardResult) ProtoMessage() {} func (x *ReloadSchemasResponse_ShardResult) ProtoReflect() protoreflect.Message { - mi := &file_vtadmin_proto_msgTypes[136] + mi := &file_vtadmin_proto_msgTypes[138] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7505,7 +7611,7 @@ type ReloadSchemasResponse_TabletResult struct { func (x *ReloadSchemasResponse_TabletResult) Reset() { *x = ReloadSchemasResponse_TabletResult{} - mi := &file_vtadmin_proto_msgTypes[137] + mi := &file_vtadmin_proto_msgTypes[139] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7517,7 +7623,7 @@ func (x *ReloadSchemasResponse_TabletResult) String() string { func (*ReloadSchemasResponse_TabletResult) ProtoMessage() {} func (x *ReloadSchemasResponse_TabletResult) ProtoReflect() protoreflect.Message { - mi := &file_vtadmin_proto_msgTypes[137] + mi := &file_vtadmin_proto_msgTypes[139] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8545,7 +8651,16 @@ var file_vtadmin_proto_rawDesc = []byte{ 0x52, 0x03, 0x73, 0x71, 0x6c, 0x22, 0x2f, 0x0a, 0x11, 0x56, 0x54, 0x45, 0x78, 0x70, 0x6c, 0x61, 0x69, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x72, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0xbd, 0x31, 0x0a, 0x07, 0x56, 0x54, 0x41, 0x64, 0x6d, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x5e, 0x0a, 0x0f, 0x56, 0x45, 0x78, 0x70, 0x6c, 0x61, + 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6c, 0x75, + 0x73, 0x74, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, + 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x73, + 0x70, 0x61, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6b, 0x65, 0x79, 0x73, + 0x70, 0x61, 0x63, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x71, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x03, 0x73, 0x71, 0x6c, 0x22, 0x2e, 0x0a, 0x10, 0x56, 0x45, 0x78, 0x70, 0x6c, 0x61, + 0x69, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x72, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x72, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0x80, 0x32, 0x0a, 0x07, 0x56, 0x54, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x12, 0x4c, 0x0a, 0x0b, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x12, 0x1b, 0x2e, 0x76, 0x74, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x2e, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, @@ -8929,22 +9044,26 @@ var file_vtadmin_proto_rawDesc = []byte{ 0x64, 0x6d, 0x69, 0x6e, 0x2e, 0x56, 0x54, 0x45, 0x78, 0x70, 0x6c, 0x61, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x76, 0x74, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x2e, 0x56, 0x54, 0x45, 0x78, 0x70, 0x6c, 0x61, 0x69, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x22, 0x00, 0x12, 0x55, 0x0a, 0x0e, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x44, - 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x1e, 0x2e, 0x76, 0x74, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x2e, - 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x76, 0x74, 0x63, 0x74, 0x6c, 0x64, 0x61, 0x74, - 0x61, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x6a, 0x0a, 0x15, 0x57, 0x6f, - 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x53, 0x77, 0x69, 0x74, 0x63, 0x68, 0x54, 0x72, 0x61, 0x66, - 0x66, 0x69, 0x63, 0x12, 0x25, 0x2e, 0x76, 0x74, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x2e, 0x57, 0x6f, - 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x53, 0x77, 0x69, 0x74, 0x63, 0x68, 0x54, 0x72, 0x61, 0x66, - 0x66, 0x69, 0x63, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x76, 0x74, 0x63, - 0x74, 0x6c, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x53, - 0x77, 0x69, 0x74, 0x63, 0x68, 0x54, 0x72, 0x61, 0x66, 0x66, 0x69, 0x63, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x26, 0x5a, 0x24, 0x76, 0x69, 0x74, 0x65, 0x73, 0x73, - 0x2e, 0x69, 0x6f, 0x2f, 0x76, 0x69, 0x74, 0x65, 0x73, 0x73, 0x2f, 0x67, 0x6f, 0x2f, 0x76, 0x74, - 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x76, 0x74, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x62, 0x06, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x65, 0x22, 0x00, 0x12, 0x41, 0x0a, 0x08, 0x56, 0x45, 0x78, 0x70, 0x6c, 0x61, 0x69, 0x6e, 0x12, + 0x18, 0x2e, 0x76, 0x74, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x2e, 0x56, 0x45, 0x78, 0x70, 0x6c, 0x61, + 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x76, 0x74, 0x61, 0x64, + 0x6d, 0x69, 0x6e, 0x2e, 0x56, 0x45, 0x78, 0x70, 0x6c, 0x61, 0x69, 0x6e, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x55, 0x0a, 0x0e, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, + 0x6f, 0x77, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x1e, 0x2e, 0x76, 0x74, 0x61, 0x64, 0x6d, + 0x69, 0x6e, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x44, 0x65, 0x6c, 0x65, 0x74, + 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x76, 0x74, 0x63, 0x74, 0x6c, + 0x64, 0x61, 0x74, 0x61, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x44, 0x65, 0x6c, + 0x65, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x6a, 0x0a, + 0x15, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x53, 0x77, 0x69, 0x74, 0x63, 0x68, 0x54, + 0x72, 0x61, 0x66, 0x66, 0x69, 0x63, 0x12, 0x25, 0x2e, 0x76, 0x74, 0x61, 0x64, 0x6d, 0x69, 0x6e, + 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x53, 0x77, 0x69, 0x74, 0x63, 0x68, 0x54, + 0x72, 0x61, 0x66, 0x66, 0x69, 0x63, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, + 0x76, 0x74, 0x63, 0x74, 0x6c, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, + 0x6f, 0x77, 0x53, 0x77, 0x69, 0x74, 0x63, 0x68, 0x54, 0x72, 0x61, 0x66, 0x66, 0x69, 0x63, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x26, 0x5a, 0x24, 0x76, 0x69, 0x74, + 0x65, 0x73, 0x73, 0x2e, 0x69, 0x6f, 0x2f, 0x76, 0x69, 0x74, 0x65, 0x73, 0x73, 0x2f, 0x67, 0x6f, + 0x2f, 0x76, 0x74, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x76, 0x74, 0x61, 0x64, 0x6d, 0x69, + 0x6e, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -8960,7 +9079,7 @@ func file_vtadmin_proto_rawDescGZIP() []byte { } var file_vtadmin_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_vtadmin_proto_msgTypes = make([]protoimpl.MessageInfo, 139) +var file_vtadmin_proto_msgTypes = make([]protoimpl.MessageInfo, 141) var file_vtadmin_proto_goTypes = []any{ (Tablet_ServingState)(0), // 0: vtadmin.Tablet.ServingState (*Cluster)(nil), // 1: vtadmin.Cluster @@ -9089,207 +9208,209 @@ var file_vtadmin_proto_goTypes = []any{ (*VDiffShowResponse)(nil), // 124: vtadmin.VDiffShowResponse (*VTExplainRequest)(nil), // 125: vtadmin.VTExplainRequest (*VTExplainResponse)(nil), // 126: vtadmin.VTExplainResponse - nil, // 127: vtadmin.ClusterCellsAliases.AliasesEntry - nil, // 128: vtadmin.Keyspace.ShardsEntry - nil, // 129: vtadmin.Schema.TableSizesEntry - (*Schema_ShardTableSize)(nil), // 130: vtadmin.Schema.ShardTableSize - (*Schema_TableSize)(nil), // 131: vtadmin.Schema.TableSize - nil, // 132: vtadmin.Schema.TableSize.ByShardEntry - (*GetSchemaMigrationsRequest_ClusterRequest)(nil), // 133: vtadmin.GetSchemaMigrationsRequest.ClusterRequest - nil, // 134: vtadmin.GetSrvKeyspacesResponse.SrvKeyspacesEntry - nil, // 135: vtadmin.GetWorkflowsResponse.WorkflowsByClusterEntry - (*ReloadSchemasResponse_KeyspaceResult)(nil), // 136: vtadmin.ReloadSchemasResponse.KeyspaceResult - (*ReloadSchemasResponse_ShardResult)(nil), // 137: vtadmin.ReloadSchemasResponse.ShardResult - (*ReloadSchemasResponse_TabletResult)(nil), // 138: vtadmin.ReloadSchemasResponse.TabletResult - nil, // 139: vtadmin.VDiffShowResponse.ShardReportEntry - (*mysqlctl.BackupInfo)(nil), // 140: mysqlctl.BackupInfo - (*topodata.CellInfo)(nil), // 141: topodata.CellInfo - (*vtctldata.ShardReplicationPositionsResponse)(nil), // 142: vtctldata.ShardReplicationPositionsResponse - (*vtctldata.Keyspace)(nil), // 143: vtctldata.Keyspace - (*tabletmanagerdata.TableDefinition)(nil), // 144: tabletmanagerdata.TableDefinition - (*vtctldata.SchemaMigration)(nil), // 145: vtctldata.SchemaMigration - (*vtctldata.Shard)(nil), // 146: vtctldata.Shard - (*vschema.SrvVSchema)(nil), // 147: vschema.SrvVSchema - (*topodata.Tablet)(nil), // 148: topodata.Tablet - (*vschema.Keyspace)(nil), // 149: vschema.Keyspace - (*vtctldata.Workflow)(nil), // 150: vtctldata.Workflow - (*vtctldata.WorkflowDeleteRequest)(nil), // 151: vtctldata.WorkflowDeleteRequest - (*vtctldata.WorkflowSwitchTrafficRequest)(nil), // 152: vtctldata.WorkflowSwitchTrafficRequest - (*vtctldata.ApplySchemaRequest)(nil), // 153: vtctldata.ApplySchemaRequest - (*vtctldata.CancelSchemaMigrationRequest)(nil), // 154: vtctldata.CancelSchemaMigrationRequest - (*vtctldata.CleanupSchemaMigrationRequest)(nil), // 155: vtctldata.CleanupSchemaMigrationRequest - (*vtctldata.CompleteSchemaMigrationRequest)(nil), // 156: vtctldata.CompleteSchemaMigrationRequest - (*vtctldata.CreateKeyspaceRequest)(nil), // 157: vtctldata.CreateKeyspaceRequest - (*vtctldata.CreateShardRequest)(nil), // 158: vtctldata.CreateShardRequest - (*vtctldata.DeleteKeyspaceRequest)(nil), // 159: vtctldata.DeleteKeyspaceRequest - (*vtctldata.DeleteShardsRequest)(nil), // 160: vtctldata.DeleteShardsRequest - (*topodata.TabletAlias)(nil), // 161: topodata.TabletAlias - (*vtctldata.EmergencyReparentShardRequest)(nil), // 162: vtctldata.EmergencyReparentShardRequest - (*logutil.Event)(nil), // 163: logutil.Event - (*vtctldata.GetBackupsRequest)(nil), // 164: vtctldata.GetBackupsRequest - (*vtctldata.GetTransactionInfoRequest)(nil), // 165: vtctldata.GetTransactionInfoRequest - (*vtctldata.LaunchSchemaMigrationRequest)(nil), // 166: vtctldata.LaunchSchemaMigrationRequest - (*vtctldata.MaterializeCreateRequest)(nil), // 167: vtctldata.MaterializeCreateRequest - (*vtctldata.MoveTablesCompleteRequest)(nil), // 168: vtctldata.MoveTablesCompleteRequest - (*vtctldata.MoveTablesCreateRequest)(nil), // 169: vtctldata.MoveTablesCreateRequest - (*vtctldata.PlannedReparentShardRequest)(nil), // 170: vtctldata.PlannedReparentShardRequest - (*vtctldata.RetrySchemaMigrationRequest)(nil), // 171: vtctldata.RetrySchemaMigrationRequest - (*vtctldata.ReshardCreateRequest)(nil), // 172: vtctldata.ReshardCreateRequest - (*vtctldata.VDiffCreateRequest)(nil), // 173: vtctldata.VDiffCreateRequest - (*vtctldata.VDiffShowRequest)(nil), // 174: vtctldata.VDiffShowRequest - (*topodata.CellsAlias)(nil), // 175: topodata.CellsAlias - (*vtctldata.GetSchemaMigrationsRequest)(nil), // 176: vtctldata.GetSchemaMigrationsRequest - (*vtctldata.GetSrvKeyspacesResponse)(nil), // 177: vtctldata.GetSrvKeyspacesResponse - (*vtctldata.ApplySchemaResponse)(nil), // 178: vtctldata.ApplySchemaResponse - (*vtctldata.CancelSchemaMigrationResponse)(nil), // 179: vtctldata.CancelSchemaMigrationResponse - (*vtctldata.CleanupSchemaMigrationResponse)(nil), // 180: vtctldata.CleanupSchemaMigrationResponse - (*vtctldata.CompleteSchemaMigrationResponse)(nil), // 181: vtctldata.CompleteSchemaMigrationResponse - (*vtctldata.ConcludeTransactionResponse)(nil), // 182: vtctldata.ConcludeTransactionResponse - (*vtctldata.CreateShardResponse)(nil), // 183: vtctldata.CreateShardResponse - (*vtctldata.DeleteKeyspaceResponse)(nil), // 184: vtctldata.DeleteKeyspaceResponse - (*vtctldata.DeleteShardsResponse)(nil), // 185: vtctldata.DeleteShardsResponse - (*vtctldata.GetFullStatusResponse)(nil), // 186: vtctldata.GetFullStatusResponse - (*vtctldata.GetTopologyPathResponse)(nil), // 187: vtctldata.GetTopologyPathResponse - (*vtctldata.GetTransactionInfoResponse)(nil), // 188: vtctldata.GetTransactionInfoResponse - (*vtctldata.GetUnresolvedTransactionsResponse)(nil), // 189: vtctldata.GetUnresolvedTransactionsResponse - (*vtctldata.WorkflowStatusResponse)(nil), // 190: vtctldata.WorkflowStatusResponse - (*vtctldata.WorkflowUpdateResponse)(nil), // 191: vtctldata.WorkflowUpdateResponse - (*vtctldata.LaunchSchemaMigrationResponse)(nil), // 192: vtctldata.LaunchSchemaMigrationResponse - (*vtctldata.MoveTablesCompleteResponse)(nil), // 193: vtctldata.MoveTablesCompleteResponse - (*vtctldata.MaterializeCreateResponse)(nil), // 194: vtctldata.MaterializeCreateResponse - (*vtctldata.RetrySchemaMigrationResponse)(nil), // 195: vtctldata.RetrySchemaMigrationResponse - (*vtctldata.ValidateResponse)(nil), // 196: vtctldata.ValidateResponse - (*vtctldata.ValidateKeyspaceResponse)(nil), // 197: vtctldata.ValidateKeyspaceResponse - (*vtctldata.ValidateSchemaKeyspaceResponse)(nil), // 198: vtctldata.ValidateSchemaKeyspaceResponse - (*vtctldata.ValidateShardResponse)(nil), // 199: vtctldata.ValidateShardResponse - (*vtctldata.ValidateVersionKeyspaceResponse)(nil), // 200: vtctldata.ValidateVersionKeyspaceResponse - (*vtctldata.ValidateVersionShardResponse)(nil), // 201: vtctldata.ValidateVersionShardResponse - (*vtctldata.VDiffCreateResponse)(nil), // 202: vtctldata.VDiffCreateResponse - (*vtctldata.WorkflowDeleteResponse)(nil), // 203: vtctldata.WorkflowDeleteResponse - (*vtctldata.WorkflowSwitchTrafficResponse)(nil), // 204: vtctldata.WorkflowSwitchTrafficResponse + (*VExplainRequest)(nil), // 127: vtadmin.VExplainRequest + (*VExplainResponse)(nil), // 128: vtadmin.VExplainResponse + nil, // 129: vtadmin.ClusterCellsAliases.AliasesEntry + nil, // 130: vtadmin.Keyspace.ShardsEntry + nil, // 131: vtadmin.Schema.TableSizesEntry + (*Schema_ShardTableSize)(nil), // 132: vtadmin.Schema.ShardTableSize + (*Schema_TableSize)(nil), // 133: vtadmin.Schema.TableSize + nil, // 134: vtadmin.Schema.TableSize.ByShardEntry + (*GetSchemaMigrationsRequest_ClusterRequest)(nil), // 135: vtadmin.GetSchemaMigrationsRequest.ClusterRequest + nil, // 136: vtadmin.GetSrvKeyspacesResponse.SrvKeyspacesEntry + nil, // 137: vtadmin.GetWorkflowsResponse.WorkflowsByClusterEntry + (*ReloadSchemasResponse_KeyspaceResult)(nil), // 138: vtadmin.ReloadSchemasResponse.KeyspaceResult + (*ReloadSchemasResponse_ShardResult)(nil), // 139: vtadmin.ReloadSchemasResponse.ShardResult + (*ReloadSchemasResponse_TabletResult)(nil), // 140: vtadmin.ReloadSchemasResponse.TabletResult + nil, // 141: vtadmin.VDiffShowResponse.ShardReportEntry + (*mysqlctl.BackupInfo)(nil), // 142: mysqlctl.BackupInfo + (*topodata.CellInfo)(nil), // 143: topodata.CellInfo + (*vtctldata.ShardReplicationPositionsResponse)(nil), // 144: vtctldata.ShardReplicationPositionsResponse + (*vtctldata.Keyspace)(nil), // 145: vtctldata.Keyspace + (*tabletmanagerdata.TableDefinition)(nil), // 146: tabletmanagerdata.TableDefinition + (*vtctldata.SchemaMigration)(nil), // 147: vtctldata.SchemaMigration + (*vtctldata.Shard)(nil), // 148: vtctldata.Shard + (*vschema.SrvVSchema)(nil), // 149: vschema.SrvVSchema + (*topodata.Tablet)(nil), // 150: topodata.Tablet + (*vschema.Keyspace)(nil), // 151: vschema.Keyspace + (*vtctldata.Workflow)(nil), // 152: vtctldata.Workflow + (*vtctldata.WorkflowDeleteRequest)(nil), // 153: vtctldata.WorkflowDeleteRequest + (*vtctldata.WorkflowSwitchTrafficRequest)(nil), // 154: vtctldata.WorkflowSwitchTrafficRequest + (*vtctldata.ApplySchemaRequest)(nil), // 155: vtctldata.ApplySchemaRequest + (*vtctldata.CancelSchemaMigrationRequest)(nil), // 156: vtctldata.CancelSchemaMigrationRequest + (*vtctldata.CleanupSchemaMigrationRequest)(nil), // 157: vtctldata.CleanupSchemaMigrationRequest + (*vtctldata.CompleteSchemaMigrationRequest)(nil), // 158: vtctldata.CompleteSchemaMigrationRequest + (*vtctldata.CreateKeyspaceRequest)(nil), // 159: vtctldata.CreateKeyspaceRequest + (*vtctldata.CreateShardRequest)(nil), // 160: vtctldata.CreateShardRequest + (*vtctldata.DeleteKeyspaceRequest)(nil), // 161: vtctldata.DeleteKeyspaceRequest + (*vtctldata.DeleteShardsRequest)(nil), // 162: vtctldata.DeleteShardsRequest + (*topodata.TabletAlias)(nil), // 163: topodata.TabletAlias + (*vtctldata.EmergencyReparentShardRequest)(nil), // 164: vtctldata.EmergencyReparentShardRequest + (*logutil.Event)(nil), // 165: logutil.Event + (*vtctldata.GetBackupsRequest)(nil), // 166: vtctldata.GetBackupsRequest + (*vtctldata.GetTransactionInfoRequest)(nil), // 167: vtctldata.GetTransactionInfoRequest + (*vtctldata.LaunchSchemaMigrationRequest)(nil), // 168: vtctldata.LaunchSchemaMigrationRequest + (*vtctldata.MaterializeCreateRequest)(nil), // 169: vtctldata.MaterializeCreateRequest + (*vtctldata.MoveTablesCompleteRequest)(nil), // 170: vtctldata.MoveTablesCompleteRequest + (*vtctldata.MoveTablesCreateRequest)(nil), // 171: vtctldata.MoveTablesCreateRequest + (*vtctldata.PlannedReparentShardRequest)(nil), // 172: vtctldata.PlannedReparentShardRequest + (*vtctldata.RetrySchemaMigrationRequest)(nil), // 173: vtctldata.RetrySchemaMigrationRequest + (*vtctldata.ReshardCreateRequest)(nil), // 174: vtctldata.ReshardCreateRequest + (*vtctldata.VDiffCreateRequest)(nil), // 175: vtctldata.VDiffCreateRequest + (*vtctldata.VDiffShowRequest)(nil), // 176: vtctldata.VDiffShowRequest + (*topodata.CellsAlias)(nil), // 177: topodata.CellsAlias + (*vtctldata.GetSchemaMigrationsRequest)(nil), // 178: vtctldata.GetSchemaMigrationsRequest + (*vtctldata.GetSrvKeyspacesResponse)(nil), // 179: vtctldata.GetSrvKeyspacesResponse + (*vtctldata.ApplySchemaResponse)(nil), // 180: vtctldata.ApplySchemaResponse + (*vtctldata.CancelSchemaMigrationResponse)(nil), // 181: vtctldata.CancelSchemaMigrationResponse + (*vtctldata.CleanupSchemaMigrationResponse)(nil), // 182: vtctldata.CleanupSchemaMigrationResponse + (*vtctldata.CompleteSchemaMigrationResponse)(nil), // 183: vtctldata.CompleteSchemaMigrationResponse + (*vtctldata.ConcludeTransactionResponse)(nil), // 184: vtctldata.ConcludeTransactionResponse + (*vtctldata.CreateShardResponse)(nil), // 185: vtctldata.CreateShardResponse + (*vtctldata.DeleteKeyspaceResponse)(nil), // 186: vtctldata.DeleteKeyspaceResponse + (*vtctldata.DeleteShardsResponse)(nil), // 187: vtctldata.DeleteShardsResponse + (*vtctldata.GetFullStatusResponse)(nil), // 188: vtctldata.GetFullStatusResponse + (*vtctldata.GetTopologyPathResponse)(nil), // 189: vtctldata.GetTopologyPathResponse + (*vtctldata.GetTransactionInfoResponse)(nil), // 190: vtctldata.GetTransactionInfoResponse + (*vtctldata.GetUnresolvedTransactionsResponse)(nil), // 191: vtctldata.GetUnresolvedTransactionsResponse + (*vtctldata.WorkflowStatusResponse)(nil), // 192: vtctldata.WorkflowStatusResponse + (*vtctldata.WorkflowUpdateResponse)(nil), // 193: vtctldata.WorkflowUpdateResponse + (*vtctldata.LaunchSchemaMigrationResponse)(nil), // 194: vtctldata.LaunchSchemaMigrationResponse + (*vtctldata.MoveTablesCompleteResponse)(nil), // 195: vtctldata.MoveTablesCompleteResponse + (*vtctldata.MaterializeCreateResponse)(nil), // 196: vtctldata.MaterializeCreateResponse + (*vtctldata.RetrySchemaMigrationResponse)(nil), // 197: vtctldata.RetrySchemaMigrationResponse + (*vtctldata.ValidateResponse)(nil), // 198: vtctldata.ValidateResponse + (*vtctldata.ValidateKeyspaceResponse)(nil), // 199: vtctldata.ValidateKeyspaceResponse + (*vtctldata.ValidateSchemaKeyspaceResponse)(nil), // 200: vtctldata.ValidateSchemaKeyspaceResponse + (*vtctldata.ValidateShardResponse)(nil), // 201: vtctldata.ValidateShardResponse + (*vtctldata.ValidateVersionKeyspaceResponse)(nil), // 202: vtctldata.ValidateVersionKeyspaceResponse + (*vtctldata.ValidateVersionShardResponse)(nil), // 203: vtctldata.ValidateVersionShardResponse + (*vtctldata.VDiffCreateResponse)(nil), // 204: vtctldata.VDiffCreateResponse + (*vtctldata.WorkflowDeleteResponse)(nil), // 205: vtctldata.WorkflowDeleteResponse + (*vtctldata.WorkflowSwitchTrafficResponse)(nil), // 206: vtctldata.WorkflowSwitchTrafficResponse } var file_vtadmin_proto_depIdxs = []int32{ 1, // 0: vtadmin.ClusterBackup.cluster:type_name -> vtadmin.Cluster - 140, // 1: vtadmin.ClusterBackup.backup:type_name -> mysqlctl.BackupInfo + 142, // 1: vtadmin.ClusterBackup.backup:type_name -> mysqlctl.BackupInfo 1, // 2: vtadmin.ClusterCellsAliases.cluster:type_name -> vtadmin.Cluster - 127, // 3: vtadmin.ClusterCellsAliases.aliases:type_name -> vtadmin.ClusterCellsAliases.AliasesEntry + 129, // 3: vtadmin.ClusterCellsAliases.aliases:type_name -> vtadmin.ClusterCellsAliases.AliasesEntry 1, // 4: vtadmin.ClusterCellInfo.cluster:type_name -> vtadmin.Cluster - 141, // 5: vtadmin.ClusterCellInfo.cell_info:type_name -> topodata.CellInfo + 143, // 5: vtadmin.ClusterCellInfo.cell_info:type_name -> topodata.CellInfo 1, // 6: vtadmin.ClusterShardReplicationPosition.cluster:type_name -> vtadmin.Cluster - 142, // 7: vtadmin.ClusterShardReplicationPosition.position_info:type_name -> vtctldata.ShardReplicationPositionsResponse + 144, // 7: vtadmin.ClusterShardReplicationPosition.position_info:type_name -> vtctldata.ShardReplicationPositionsResponse 16, // 8: vtadmin.ClusterWorkflows.workflows:type_name -> vtadmin.Workflow 1, // 9: vtadmin.Keyspace.cluster:type_name -> vtadmin.Cluster - 143, // 10: vtadmin.Keyspace.keyspace:type_name -> vtctldata.Keyspace - 128, // 11: vtadmin.Keyspace.shards:type_name -> vtadmin.Keyspace.ShardsEntry + 145, // 10: vtadmin.Keyspace.keyspace:type_name -> vtctldata.Keyspace + 130, // 11: vtadmin.Keyspace.shards:type_name -> vtadmin.Keyspace.ShardsEntry 1, // 12: vtadmin.Schema.cluster:type_name -> vtadmin.Cluster - 144, // 13: vtadmin.Schema.table_definitions:type_name -> tabletmanagerdata.TableDefinition - 129, // 14: vtadmin.Schema.table_sizes:type_name -> vtadmin.Schema.TableSizesEntry + 146, // 13: vtadmin.Schema.table_definitions:type_name -> tabletmanagerdata.TableDefinition + 131, // 14: vtadmin.Schema.table_sizes:type_name -> vtadmin.Schema.TableSizesEntry 1, // 15: vtadmin.SchemaMigration.cluster:type_name -> vtadmin.Cluster - 145, // 16: vtadmin.SchemaMigration.schema_migration:type_name -> vtctldata.SchemaMigration + 147, // 16: vtadmin.SchemaMigration.schema_migration:type_name -> vtctldata.SchemaMigration 1, // 17: vtadmin.Shard.cluster:type_name -> vtadmin.Cluster - 146, // 18: vtadmin.Shard.shard:type_name -> vtctldata.Shard + 148, // 18: vtadmin.Shard.shard:type_name -> vtctldata.Shard 1, // 19: vtadmin.SrvVSchema.cluster:type_name -> vtadmin.Cluster - 147, // 20: vtadmin.SrvVSchema.srv_v_schema:type_name -> vschema.SrvVSchema + 149, // 20: vtadmin.SrvVSchema.srv_v_schema:type_name -> vschema.SrvVSchema 1, // 21: vtadmin.Tablet.cluster:type_name -> vtadmin.Cluster - 148, // 22: vtadmin.Tablet.tablet:type_name -> topodata.Tablet + 150, // 22: vtadmin.Tablet.tablet:type_name -> topodata.Tablet 0, // 23: vtadmin.Tablet.state:type_name -> vtadmin.Tablet.ServingState 1, // 24: vtadmin.VSchema.cluster:type_name -> vtadmin.Cluster - 149, // 25: vtadmin.VSchema.v_schema:type_name -> vschema.Keyspace + 151, // 25: vtadmin.VSchema.v_schema:type_name -> vschema.Keyspace 1, // 26: vtadmin.Vtctld.cluster:type_name -> vtadmin.Cluster 1, // 27: vtadmin.VTGate.cluster:type_name -> vtadmin.Cluster 1, // 28: vtadmin.Workflow.cluster:type_name -> vtadmin.Cluster - 150, // 29: vtadmin.Workflow.workflow:type_name -> vtctldata.Workflow - 151, // 30: vtadmin.WorkflowDeleteRequest.request:type_name -> vtctldata.WorkflowDeleteRequest - 152, // 31: vtadmin.WorkflowSwitchTrafficRequest.request:type_name -> vtctldata.WorkflowSwitchTrafficRequest - 153, // 32: vtadmin.ApplySchemaRequest.request:type_name -> vtctldata.ApplySchemaRequest - 154, // 33: vtadmin.CancelSchemaMigrationRequest.request:type_name -> vtctldata.CancelSchemaMigrationRequest - 155, // 34: vtadmin.CleanupSchemaMigrationRequest.request:type_name -> vtctldata.CleanupSchemaMigrationRequest - 156, // 35: vtadmin.CompleteSchemaMigrationRequest.request:type_name -> vtctldata.CompleteSchemaMigrationRequest - 157, // 36: vtadmin.CreateKeyspaceRequest.options:type_name -> vtctldata.CreateKeyspaceRequest + 152, // 29: vtadmin.Workflow.workflow:type_name -> vtctldata.Workflow + 153, // 30: vtadmin.WorkflowDeleteRequest.request:type_name -> vtctldata.WorkflowDeleteRequest + 154, // 31: vtadmin.WorkflowSwitchTrafficRequest.request:type_name -> vtctldata.WorkflowSwitchTrafficRequest + 155, // 32: vtadmin.ApplySchemaRequest.request:type_name -> vtctldata.ApplySchemaRequest + 156, // 33: vtadmin.CancelSchemaMigrationRequest.request:type_name -> vtctldata.CancelSchemaMigrationRequest + 157, // 34: vtadmin.CleanupSchemaMigrationRequest.request:type_name -> vtctldata.CleanupSchemaMigrationRequest + 158, // 35: vtadmin.CompleteSchemaMigrationRequest.request:type_name -> vtctldata.CompleteSchemaMigrationRequest + 159, // 36: vtadmin.CreateKeyspaceRequest.options:type_name -> vtctldata.CreateKeyspaceRequest 7, // 37: vtadmin.CreateKeyspaceResponse.keyspace:type_name -> vtadmin.Keyspace - 158, // 38: vtadmin.CreateShardRequest.options:type_name -> vtctldata.CreateShardRequest - 159, // 39: vtadmin.DeleteKeyspaceRequest.options:type_name -> vtctldata.DeleteKeyspaceRequest - 160, // 40: vtadmin.DeleteShardsRequest.options:type_name -> vtctldata.DeleteShardsRequest - 161, // 41: vtadmin.DeleteTabletRequest.alias:type_name -> topodata.TabletAlias + 160, // 38: vtadmin.CreateShardRequest.options:type_name -> vtctldata.CreateShardRequest + 161, // 39: vtadmin.DeleteKeyspaceRequest.options:type_name -> vtctldata.DeleteKeyspaceRequest + 162, // 40: vtadmin.DeleteShardsRequest.options:type_name -> vtctldata.DeleteShardsRequest + 163, // 41: vtadmin.DeleteTabletRequest.alias:type_name -> topodata.TabletAlias 1, // 42: vtadmin.DeleteTabletResponse.cluster:type_name -> vtadmin.Cluster - 162, // 43: vtadmin.EmergencyFailoverShardRequest.options:type_name -> vtctldata.EmergencyReparentShardRequest + 164, // 43: vtadmin.EmergencyFailoverShardRequest.options:type_name -> vtctldata.EmergencyReparentShardRequest 1, // 44: vtadmin.EmergencyFailoverShardResponse.cluster:type_name -> vtadmin.Cluster - 161, // 45: vtadmin.EmergencyFailoverShardResponse.promoted_primary:type_name -> topodata.TabletAlias - 163, // 46: vtadmin.EmergencyFailoverShardResponse.events:type_name -> logutil.Event + 163, // 45: vtadmin.EmergencyFailoverShardResponse.promoted_primary:type_name -> topodata.TabletAlias + 165, // 46: vtadmin.EmergencyFailoverShardResponse.events:type_name -> logutil.Event 61, // 47: vtadmin.FindSchemaRequest.table_size_options:type_name -> vtadmin.GetSchemaTableSizeOptions - 164, // 48: vtadmin.GetBackupsRequest.request_options:type_name -> vtctldata.GetBackupsRequest + 166, // 48: vtadmin.GetBackupsRequest.request_options:type_name -> vtctldata.GetBackupsRequest 2, // 49: vtadmin.GetBackupsResponse.backups:type_name -> vtadmin.ClusterBackup 4, // 50: vtadmin.GetCellInfosResponse.cell_infos:type_name -> vtadmin.ClusterCellInfo 3, // 51: vtadmin.GetCellsAliasesResponse.aliases:type_name -> vtadmin.ClusterCellsAliases 1, // 52: vtadmin.GetClustersResponse.clusters:type_name -> vtadmin.Cluster - 161, // 53: vtadmin.GetFullStatusRequest.alias:type_name -> topodata.TabletAlias + 163, // 53: vtadmin.GetFullStatusRequest.alias:type_name -> topodata.TabletAlias 15, // 54: vtadmin.GetGatesResponse.gates:type_name -> vtadmin.VTGate 7, // 55: vtadmin.GetKeyspacesResponse.keyspaces:type_name -> vtadmin.Keyspace 61, // 56: vtadmin.GetSchemaRequest.table_size_options:type_name -> vtadmin.GetSchemaTableSizeOptions 61, // 57: vtadmin.GetSchemasRequest.table_size_options:type_name -> vtadmin.GetSchemaTableSizeOptions 8, // 58: vtadmin.GetSchemasResponse.schemas:type_name -> vtadmin.Schema - 133, // 59: vtadmin.GetSchemaMigrationsRequest.cluster_requests:type_name -> vtadmin.GetSchemaMigrationsRequest.ClusterRequest + 135, // 59: vtadmin.GetSchemaMigrationsRequest.cluster_requests:type_name -> vtadmin.GetSchemaMigrationsRequest.ClusterRequest 9, // 60: vtadmin.GetSchemaMigrationsResponse.schema_migrations:type_name -> vtadmin.SchemaMigration 5, // 61: vtadmin.GetShardReplicationPositionsResponse.replication_positions:type_name -> vtadmin.ClusterShardReplicationPosition - 134, // 62: vtadmin.GetSrvKeyspacesResponse.srv_keyspaces:type_name -> vtadmin.GetSrvKeyspacesResponse.SrvKeyspacesEntry + 136, // 62: vtadmin.GetSrvKeyspacesResponse.srv_keyspaces:type_name -> vtadmin.GetSrvKeyspacesResponse.SrvKeyspacesEntry 11, // 63: vtadmin.GetSrvVSchemasResponse.srv_v_schemas:type_name -> vtadmin.SrvVSchema - 161, // 64: vtadmin.GetTabletRequest.alias:type_name -> topodata.TabletAlias + 163, // 64: vtadmin.GetTabletRequest.alias:type_name -> topodata.TabletAlias 12, // 65: vtadmin.GetTabletsResponse.tablets:type_name -> vtadmin.Tablet - 165, // 66: vtadmin.GetTransactionInfoRequest.request:type_name -> vtctldata.GetTransactionInfoRequest + 167, // 66: vtadmin.GetTransactionInfoRequest.request:type_name -> vtctldata.GetTransactionInfoRequest 13, // 67: vtadmin.GetVSchemasResponse.v_schemas:type_name -> vtadmin.VSchema 14, // 68: vtadmin.GetVtctldsResponse.vtctlds:type_name -> vtadmin.Vtctld - 135, // 69: vtadmin.GetWorkflowsResponse.workflows_by_cluster:type_name -> vtadmin.GetWorkflowsResponse.WorkflowsByClusterEntry - 166, // 70: vtadmin.LaunchSchemaMigrationRequest.request:type_name -> vtctldata.LaunchSchemaMigrationRequest - 167, // 71: vtadmin.MaterializeCreateRequest.request:type_name -> vtctldata.MaterializeCreateRequest - 168, // 72: vtadmin.MoveTablesCompleteRequest.request:type_name -> vtctldata.MoveTablesCompleteRequest - 169, // 73: vtadmin.MoveTablesCreateRequest.request:type_name -> vtctldata.MoveTablesCreateRequest - 161, // 74: vtadmin.PingTabletRequest.alias:type_name -> topodata.TabletAlias + 137, // 69: vtadmin.GetWorkflowsResponse.workflows_by_cluster:type_name -> vtadmin.GetWorkflowsResponse.WorkflowsByClusterEntry + 168, // 70: vtadmin.LaunchSchemaMigrationRequest.request:type_name -> vtctldata.LaunchSchemaMigrationRequest + 169, // 71: vtadmin.MaterializeCreateRequest.request:type_name -> vtctldata.MaterializeCreateRequest + 170, // 72: vtadmin.MoveTablesCompleteRequest.request:type_name -> vtctldata.MoveTablesCompleteRequest + 171, // 73: vtadmin.MoveTablesCreateRequest.request:type_name -> vtctldata.MoveTablesCreateRequest + 163, // 74: vtadmin.PingTabletRequest.alias:type_name -> topodata.TabletAlias 1, // 75: vtadmin.PingTabletResponse.cluster:type_name -> vtadmin.Cluster - 170, // 76: vtadmin.PlannedFailoverShardRequest.options:type_name -> vtctldata.PlannedReparentShardRequest + 172, // 76: vtadmin.PlannedFailoverShardRequest.options:type_name -> vtctldata.PlannedReparentShardRequest 1, // 77: vtadmin.PlannedFailoverShardResponse.cluster:type_name -> vtadmin.Cluster - 161, // 78: vtadmin.PlannedFailoverShardResponse.promoted_primary:type_name -> topodata.TabletAlias - 163, // 79: vtadmin.PlannedFailoverShardResponse.events:type_name -> logutil.Event - 161, // 80: vtadmin.RefreshStateRequest.alias:type_name -> topodata.TabletAlias + 163, // 78: vtadmin.PlannedFailoverShardResponse.promoted_primary:type_name -> topodata.TabletAlias + 165, // 79: vtadmin.PlannedFailoverShardResponse.events:type_name -> logutil.Event + 163, // 80: vtadmin.RefreshStateRequest.alias:type_name -> topodata.TabletAlias 1, // 81: vtadmin.RefreshStateResponse.cluster:type_name -> vtadmin.Cluster - 161, // 82: vtadmin.ReloadSchemasRequest.tablets:type_name -> topodata.TabletAlias - 136, // 83: vtadmin.ReloadSchemasResponse.keyspace_results:type_name -> vtadmin.ReloadSchemasResponse.KeyspaceResult - 137, // 84: vtadmin.ReloadSchemasResponse.shard_results:type_name -> vtadmin.ReloadSchemasResponse.ShardResult - 138, // 85: vtadmin.ReloadSchemasResponse.tablet_results:type_name -> vtadmin.ReloadSchemasResponse.TabletResult - 163, // 86: vtadmin.ReloadSchemaShardResponse.events:type_name -> logutil.Event - 161, // 87: vtadmin.RefreshTabletReplicationSourceRequest.alias:type_name -> topodata.TabletAlias - 161, // 88: vtadmin.RefreshTabletReplicationSourceResponse.primary:type_name -> topodata.TabletAlias + 163, // 82: vtadmin.ReloadSchemasRequest.tablets:type_name -> topodata.TabletAlias + 138, // 83: vtadmin.ReloadSchemasResponse.keyspace_results:type_name -> vtadmin.ReloadSchemasResponse.KeyspaceResult + 139, // 84: vtadmin.ReloadSchemasResponse.shard_results:type_name -> vtadmin.ReloadSchemasResponse.ShardResult + 140, // 85: vtadmin.ReloadSchemasResponse.tablet_results:type_name -> vtadmin.ReloadSchemasResponse.TabletResult + 165, // 86: vtadmin.ReloadSchemaShardResponse.events:type_name -> logutil.Event + 163, // 87: vtadmin.RefreshTabletReplicationSourceRequest.alias:type_name -> topodata.TabletAlias + 163, // 88: vtadmin.RefreshTabletReplicationSourceResponse.primary:type_name -> topodata.TabletAlias 1, // 89: vtadmin.RefreshTabletReplicationSourceResponse.cluster:type_name -> vtadmin.Cluster - 171, // 90: vtadmin.RetrySchemaMigrationRequest.request:type_name -> vtctldata.RetrySchemaMigrationRequest - 161, // 91: vtadmin.RunHealthCheckRequest.alias:type_name -> topodata.TabletAlias + 173, // 90: vtadmin.RetrySchemaMigrationRequest.request:type_name -> vtctldata.RetrySchemaMigrationRequest + 163, // 91: vtadmin.RunHealthCheckRequest.alias:type_name -> topodata.TabletAlias 1, // 92: vtadmin.RunHealthCheckResponse.cluster:type_name -> vtadmin.Cluster - 172, // 93: vtadmin.ReshardCreateRequest.request:type_name -> vtctldata.ReshardCreateRequest - 161, // 94: vtadmin.SetReadOnlyRequest.alias:type_name -> topodata.TabletAlias - 161, // 95: vtadmin.SetReadWriteRequest.alias:type_name -> topodata.TabletAlias - 161, // 96: vtadmin.StartReplicationRequest.alias:type_name -> topodata.TabletAlias + 174, // 93: vtadmin.ReshardCreateRequest.request:type_name -> vtctldata.ReshardCreateRequest + 163, // 94: vtadmin.SetReadOnlyRequest.alias:type_name -> topodata.TabletAlias + 163, // 95: vtadmin.SetReadWriteRequest.alias:type_name -> topodata.TabletAlias + 163, // 96: vtadmin.StartReplicationRequest.alias:type_name -> topodata.TabletAlias 1, // 97: vtadmin.StartReplicationResponse.cluster:type_name -> vtadmin.Cluster - 161, // 98: vtadmin.StopReplicationRequest.alias:type_name -> topodata.TabletAlias + 163, // 98: vtadmin.StopReplicationRequest.alias:type_name -> topodata.TabletAlias 1, // 99: vtadmin.StopReplicationResponse.cluster:type_name -> vtadmin.Cluster - 161, // 100: vtadmin.TabletExternallyPromotedRequest.alias:type_name -> topodata.TabletAlias + 163, // 100: vtadmin.TabletExternallyPromotedRequest.alias:type_name -> topodata.TabletAlias 1, // 101: vtadmin.TabletExternallyPromotedResponse.cluster:type_name -> vtadmin.Cluster - 161, // 102: vtadmin.TabletExternallyPromotedResponse.new_primary:type_name -> topodata.TabletAlias - 161, // 103: vtadmin.TabletExternallyPromotedResponse.old_primary:type_name -> topodata.TabletAlias - 161, // 104: vtadmin.TabletExternallyReparentedRequest.alias:type_name -> topodata.TabletAlias - 173, // 105: vtadmin.VDiffCreateRequest.request:type_name -> vtctldata.VDiffCreateRequest - 174, // 106: vtadmin.VDiffShowRequest.request:type_name -> vtctldata.VDiffShowRequest + 163, // 102: vtadmin.TabletExternallyPromotedResponse.new_primary:type_name -> topodata.TabletAlias + 163, // 103: vtadmin.TabletExternallyPromotedResponse.old_primary:type_name -> topodata.TabletAlias + 163, // 104: vtadmin.TabletExternallyReparentedRequest.alias:type_name -> topodata.TabletAlias + 175, // 105: vtadmin.VDiffCreateRequest.request:type_name -> vtctldata.VDiffCreateRequest + 176, // 106: vtadmin.VDiffShowRequest.request:type_name -> vtctldata.VDiffShowRequest 122, // 107: vtadmin.VDiffShardReport.progress:type_name -> vtadmin.VDiffProgress - 139, // 108: vtadmin.VDiffShowResponse.shard_report:type_name -> vtadmin.VDiffShowResponse.ShardReportEntry - 175, // 109: vtadmin.ClusterCellsAliases.AliasesEntry.value:type_name -> topodata.CellsAlias - 146, // 110: vtadmin.Keyspace.ShardsEntry.value:type_name -> vtctldata.Shard - 131, // 111: vtadmin.Schema.TableSizesEntry.value:type_name -> vtadmin.Schema.TableSize - 132, // 112: vtadmin.Schema.TableSize.by_shard:type_name -> vtadmin.Schema.TableSize.ByShardEntry - 130, // 113: vtadmin.Schema.TableSize.ByShardEntry.value:type_name -> vtadmin.Schema.ShardTableSize - 176, // 114: vtadmin.GetSchemaMigrationsRequest.ClusterRequest.request:type_name -> vtctldata.GetSchemaMigrationsRequest - 177, // 115: vtadmin.GetSrvKeyspacesResponse.SrvKeyspacesEntry.value:type_name -> vtctldata.GetSrvKeyspacesResponse + 141, // 108: vtadmin.VDiffShowResponse.shard_report:type_name -> vtadmin.VDiffShowResponse.ShardReportEntry + 177, // 109: vtadmin.ClusterCellsAliases.AliasesEntry.value:type_name -> topodata.CellsAlias + 148, // 110: vtadmin.Keyspace.ShardsEntry.value:type_name -> vtctldata.Shard + 133, // 111: vtadmin.Schema.TableSizesEntry.value:type_name -> vtadmin.Schema.TableSize + 134, // 112: vtadmin.Schema.TableSize.by_shard:type_name -> vtadmin.Schema.TableSize.ByShardEntry + 132, // 113: vtadmin.Schema.TableSize.ByShardEntry.value:type_name -> vtadmin.Schema.ShardTableSize + 178, // 114: vtadmin.GetSchemaMigrationsRequest.ClusterRequest.request:type_name -> vtctldata.GetSchemaMigrationsRequest + 179, // 115: vtadmin.GetSrvKeyspacesResponse.SrvKeyspacesEntry.value:type_name -> vtctldata.GetSrvKeyspacesResponse 6, // 116: vtadmin.GetWorkflowsResponse.WorkflowsByClusterEntry.value:type_name -> vtadmin.ClusterWorkflows 7, // 117: vtadmin.ReloadSchemasResponse.KeyspaceResult.keyspace:type_name -> vtadmin.Keyspace - 163, // 118: vtadmin.ReloadSchemasResponse.KeyspaceResult.events:type_name -> logutil.Event + 165, // 118: vtadmin.ReloadSchemasResponse.KeyspaceResult.events:type_name -> logutil.Event 10, // 119: vtadmin.ReloadSchemasResponse.ShardResult.shard:type_name -> vtadmin.Shard - 163, // 120: vtadmin.ReloadSchemasResponse.ShardResult.events:type_name -> logutil.Event + 165, // 120: vtadmin.ReloadSchemasResponse.ShardResult.events:type_name -> logutil.Event 12, // 121: vtadmin.ReloadSchemasResponse.TabletResult.tablet:type_name -> vtadmin.Tablet 123, // 122: vtadmin.VDiffShowResponse.ShardReportEntry.value:type_name -> vtadmin.VDiffShardReport 19, // 123: vtadmin.VTAdmin.ApplySchema:input_type -> vtadmin.ApplySchemaRequest @@ -9362,82 +9483,84 @@ var file_vtadmin_proto_depIdxs = []int32{ 120, // 190: vtadmin.VTAdmin.VDiffCreate:input_type -> vtadmin.VDiffCreateRequest 121, // 191: vtadmin.VTAdmin.VDiffShow:input_type -> vtadmin.VDiffShowRequest 125, // 192: vtadmin.VTAdmin.VTExplain:input_type -> vtadmin.VTExplainRequest - 17, // 193: vtadmin.VTAdmin.WorkflowDelete:input_type -> vtadmin.WorkflowDeleteRequest - 18, // 194: vtadmin.VTAdmin.WorkflowSwitchTraffic:input_type -> vtadmin.WorkflowSwitchTrafficRequest - 178, // 195: vtadmin.VTAdmin.ApplySchema:output_type -> vtctldata.ApplySchemaResponse - 179, // 196: vtadmin.VTAdmin.CancelSchemaMigration:output_type -> vtctldata.CancelSchemaMigrationResponse - 180, // 197: vtadmin.VTAdmin.CleanupSchemaMigration:output_type -> vtctldata.CleanupSchemaMigrationResponse - 181, // 198: vtadmin.VTAdmin.CompleteSchemaMigration:output_type -> vtctldata.CompleteSchemaMigrationResponse - 182, // 199: vtadmin.VTAdmin.ConcludeTransaction:output_type -> vtctldata.ConcludeTransactionResponse - 25, // 200: vtadmin.VTAdmin.CreateKeyspace:output_type -> vtadmin.CreateKeyspaceResponse - 183, // 201: vtadmin.VTAdmin.CreateShard:output_type -> vtctldata.CreateShardResponse - 184, // 202: vtadmin.VTAdmin.DeleteKeyspace:output_type -> vtctldata.DeleteKeyspaceResponse - 185, // 203: vtadmin.VTAdmin.DeleteShards:output_type -> vtctldata.DeleteShardsResponse - 30, // 204: vtadmin.VTAdmin.DeleteTablet:output_type -> vtadmin.DeleteTabletResponse - 32, // 205: vtadmin.VTAdmin.EmergencyFailoverShard:output_type -> vtadmin.EmergencyFailoverShardResponse - 8, // 206: vtadmin.VTAdmin.FindSchema:output_type -> vtadmin.Schema - 35, // 207: vtadmin.VTAdmin.GetBackups:output_type -> vtadmin.GetBackupsResponse - 37, // 208: vtadmin.VTAdmin.GetCellInfos:output_type -> vtadmin.GetCellInfosResponse - 39, // 209: vtadmin.VTAdmin.GetCellsAliases:output_type -> vtadmin.GetCellsAliasesResponse - 41, // 210: vtadmin.VTAdmin.GetClusters:output_type -> vtadmin.GetClustersResponse - 186, // 211: vtadmin.VTAdmin.GetFullStatus:output_type -> vtctldata.GetFullStatusResponse - 44, // 212: vtadmin.VTAdmin.GetGates:output_type -> vtadmin.GetGatesResponse - 7, // 213: vtadmin.VTAdmin.GetKeyspace:output_type -> vtadmin.Keyspace - 47, // 214: vtadmin.VTAdmin.GetKeyspaces:output_type -> vtadmin.GetKeyspacesResponse - 8, // 215: vtadmin.VTAdmin.GetSchema:output_type -> vtadmin.Schema - 50, // 216: vtadmin.VTAdmin.GetSchemas:output_type -> vtadmin.GetSchemasResponse - 52, // 217: vtadmin.VTAdmin.GetSchemaMigrations:output_type -> vtadmin.GetSchemaMigrationsResponse - 54, // 218: vtadmin.VTAdmin.GetShardReplicationPositions:output_type -> vtadmin.GetShardReplicationPositionsResponse - 177, // 219: vtadmin.VTAdmin.GetSrvKeyspace:output_type -> vtctldata.GetSrvKeyspacesResponse - 57, // 220: vtadmin.VTAdmin.GetSrvKeyspaces:output_type -> vtadmin.GetSrvKeyspacesResponse - 11, // 221: vtadmin.VTAdmin.GetSrvVSchema:output_type -> vtadmin.SrvVSchema - 60, // 222: vtadmin.VTAdmin.GetSrvVSchemas:output_type -> vtadmin.GetSrvVSchemasResponse - 12, // 223: vtadmin.VTAdmin.GetTablet:output_type -> vtadmin.Tablet - 64, // 224: vtadmin.VTAdmin.GetTablets:output_type -> vtadmin.GetTabletsResponse - 187, // 225: vtadmin.VTAdmin.GetTopologyPath:output_type -> vtctldata.GetTopologyPathResponse - 188, // 226: vtadmin.VTAdmin.GetTransactionInfo:output_type -> vtctldata.GetTransactionInfoResponse - 189, // 227: vtadmin.VTAdmin.GetUnresolvedTransactions:output_type -> vtctldata.GetUnresolvedTransactionsResponse - 13, // 228: vtadmin.VTAdmin.GetVSchema:output_type -> vtadmin.VSchema - 70, // 229: vtadmin.VTAdmin.GetVSchemas:output_type -> vtadmin.GetVSchemasResponse - 72, // 230: vtadmin.VTAdmin.GetVtctlds:output_type -> vtadmin.GetVtctldsResponse - 16, // 231: vtadmin.VTAdmin.GetWorkflow:output_type -> vtadmin.Workflow - 78, // 232: vtadmin.VTAdmin.GetWorkflows:output_type -> vtadmin.GetWorkflowsResponse - 190, // 233: vtadmin.VTAdmin.GetWorkflowStatus:output_type -> vtctldata.WorkflowStatusResponse - 191, // 234: vtadmin.VTAdmin.StartWorkflow:output_type -> vtctldata.WorkflowUpdateResponse - 191, // 235: vtadmin.VTAdmin.StopWorkflow:output_type -> vtctldata.WorkflowUpdateResponse - 192, // 236: vtadmin.VTAdmin.LaunchSchemaMigration:output_type -> vtctldata.LaunchSchemaMigrationResponse - 193, // 237: vtadmin.VTAdmin.MoveTablesComplete:output_type -> vtctldata.MoveTablesCompleteResponse - 190, // 238: vtadmin.VTAdmin.MoveTablesCreate:output_type -> vtctldata.WorkflowStatusResponse - 194, // 239: vtadmin.VTAdmin.MaterializeCreate:output_type -> vtctldata.MaterializeCreateResponse - 84, // 240: vtadmin.VTAdmin.PingTablet:output_type -> vtadmin.PingTabletResponse - 86, // 241: vtadmin.VTAdmin.PlannedFailoverShard:output_type -> vtadmin.PlannedFailoverShardResponse - 88, // 242: vtadmin.VTAdmin.RebuildKeyspaceGraph:output_type -> vtadmin.RebuildKeyspaceGraphResponse - 90, // 243: vtadmin.VTAdmin.RefreshState:output_type -> vtadmin.RefreshStateResponse - 96, // 244: vtadmin.VTAdmin.RefreshTabletReplicationSource:output_type -> vtadmin.RefreshTabletReplicationSourceResponse - 92, // 245: vtadmin.VTAdmin.ReloadSchemas:output_type -> vtadmin.ReloadSchemasResponse - 94, // 246: vtadmin.VTAdmin.ReloadSchemaShard:output_type -> vtadmin.ReloadSchemaShardResponse - 98, // 247: vtadmin.VTAdmin.RemoveKeyspaceCell:output_type -> vtadmin.RemoveKeyspaceCellResponse - 195, // 248: vtadmin.VTAdmin.RetrySchemaMigration:output_type -> vtctldata.RetrySchemaMigrationResponse - 101, // 249: vtadmin.VTAdmin.RunHealthCheck:output_type -> vtadmin.RunHealthCheckResponse - 190, // 250: vtadmin.VTAdmin.ReshardCreate:output_type -> vtctldata.WorkflowStatusResponse - 104, // 251: vtadmin.VTAdmin.SetReadOnly:output_type -> vtadmin.SetReadOnlyResponse - 106, // 252: vtadmin.VTAdmin.SetReadWrite:output_type -> vtadmin.SetReadWriteResponse - 108, // 253: vtadmin.VTAdmin.StartReplication:output_type -> vtadmin.StartReplicationResponse - 110, // 254: vtadmin.VTAdmin.StopReplication:output_type -> vtadmin.StopReplicationResponse - 112, // 255: vtadmin.VTAdmin.TabletExternallyPromoted:output_type -> vtadmin.TabletExternallyPromotedResponse - 196, // 256: vtadmin.VTAdmin.Validate:output_type -> vtctldata.ValidateResponse - 197, // 257: vtadmin.VTAdmin.ValidateKeyspace:output_type -> vtctldata.ValidateKeyspaceResponse - 198, // 258: vtadmin.VTAdmin.ValidateSchemaKeyspace:output_type -> vtctldata.ValidateSchemaKeyspaceResponse - 199, // 259: vtadmin.VTAdmin.ValidateShard:output_type -> vtctldata.ValidateShardResponse - 200, // 260: vtadmin.VTAdmin.ValidateVersionKeyspace:output_type -> vtctldata.ValidateVersionKeyspaceResponse - 201, // 261: vtadmin.VTAdmin.ValidateVersionShard:output_type -> vtctldata.ValidateVersionShardResponse - 202, // 262: vtadmin.VTAdmin.VDiffCreate:output_type -> vtctldata.VDiffCreateResponse - 124, // 263: vtadmin.VTAdmin.VDiffShow:output_type -> vtadmin.VDiffShowResponse - 126, // 264: vtadmin.VTAdmin.VTExplain:output_type -> vtadmin.VTExplainResponse - 203, // 265: vtadmin.VTAdmin.WorkflowDelete:output_type -> vtctldata.WorkflowDeleteResponse - 204, // 266: vtadmin.VTAdmin.WorkflowSwitchTraffic:output_type -> vtctldata.WorkflowSwitchTrafficResponse - 195, // [195:267] is the sub-list for method output_type - 123, // [123:195] is the sub-list for method input_type + 127, // 193: vtadmin.VTAdmin.VExplain:input_type -> vtadmin.VExplainRequest + 17, // 194: vtadmin.VTAdmin.WorkflowDelete:input_type -> vtadmin.WorkflowDeleteRequest + 18, // 195: vtadmin.VTAdmin.WorkflowSwitchTraffic:input_type -> vtadmin.WorkflowSwitchTrafficRequest + 180, // 196: vtadmin.VTAdmin.ApplySchema:output_type -> vtctldata.ApplySchemaResponse + 181, // 197: vtadmin.VTAdmin.CancelSchemaMigration:output_type -> vtctldata.CancelSchemaMigrationResponse + 182, // 198: vtadmin.VTAdmin.CleanupSchemaMigration:output_type -> vtctldata.CleanupSchemaMigrationResponse + 183, // 199: vtadmin.VTAdmin.CompleteSchemaMigration:output_type -> vtctldata.CompleteSchemaMigrationResponse + 184, // 200: vtadmin.VTAdmin.ConcludeTransaction:output_type -> vtctldata.ConcludeTransactionResponse + 25, // 201: vtadmin.VTAdmin.CreateKeyspace:output_type -> vtadmin.CreateKeyspaceResponse + 185, // 202: vtadmin.VTAdmin.CreateShard:output_type -> vtctldata.CreateShardResponse + 186, // 203: vtadmin.VTAdmin.DeleteKeyspace:output_type -> vtctldata.DeleteKeyspaceResponse + 187, // 204: vtadmin.VTAdmin.DeleteShards:output_type -> vtctldata.DeleteShardsResponse + 30, // 205: vtadmin.VTAdmin.DeleteTablet:output_type -> vtadmin.DeleteTabletResponse + 32, // 206: vtadmin.VTAdmin.EmergencyFailoverShard:output_type -> vtadmin.EmergencyFailoverShardResponse + 8, // 207: vtadmin.VTAdmin.FindSchema:output_type -> vtadmin.Schema + 35, // 208: vtadmin.VTAdmin.GetBackups:output_type -> vtadmin.GetBackupsResponse + 37, // 209: vtadmin.VTAdmin.GetCellInfos:output_type -> vtadmin.GetCellInfosResponse + 39, // 210: vtadmin.VTAdmin.GetCellsAliases:output_type -> vtadmin.GetCellsAliasesResponse + 41, // 211: vtadmin.VTAdmin.GetClusters:output_type -> vtadmin.GetClustersResponse + 188, // 212: vtadmin.VTAdmin.GetFullStatus:output_type -> vtctldata.GetFullStatusResponse + 44, // 213: vtadmin.VTAdmin.GetGates:output_type -> vtadmin.GetGatesResponse + 7, // 214: vtadmin.VTAdmin.GetKeyspace:output_type -> vtadmin.Keyspace + 47, // 215: vtadmin.VTAdmin.GetKeyspaces:output_type -> vtadmin.GetKeyspacesResponse + 8, // 216: vtadmin.VTAdmin.GetSchema:output_type -> vtadmin.Schema + 50, // 217: vtadmin.VTAdmin.GetSchemas:output_type -> vtadmin.GetSchemasResponse + 52, // 218: vtadmin.VTAdmin.GetSchemaMigrations:output_type -> vtadmin.GetSchemaMigrationsResponse + 54, // 219: vtadmin.VTAdmin.GetShardReplicationPositions:output_type -> vtadmin.GetShardReplicationPositionsResponse + 179, // 220: vtadmin.VTAdmin.GetSrvKeyspace:output_type -> vtctldata.GetSrvKeyspacesResponse + 57, // 221: vtadmin.VTAdmin.GetSrvKeyspaces:output_type -> vtadmin.GetSrvKeyspacesResponse + 11, // 222: vtadmin.VTAdmin.GetSrvVSchema:output_type -> vtadmin.SrvVSchema + 60, // 223: vtadmin.VTAdmin.GetSrvVSchemas:output_type -> vtadmin.GetSrvVSchemasResponse + 12, // 224: vtadmin.VTAdmin.GetTablet:output_type -> vtadmin.Tablet + 64, // 225: vtadmin.VTAdmin.GetTablets:output_type -> vtadmin.GetTabletsResponse + 189, // 226: vtadmin.VTAdmin.GetTopologyPath:output_type -> vtctldata.GetTopologyPathResponse + 190, // 227: vtadmin.VTAdmin.GetTransactionInfo:output_type -> vtctldata.GetTransactionInfoResponse + 191, // 228: vtadmin.VTAdmin.GetUnresolvedTransactions:output_type -> vtctldata.GetUnresolvedTransactionsResponse + 13, // 229: vtadmin.VTAdmin.GetVSchema:output_type -> vtadmin.VSchema + 70, // 230: vtadmin.VTAdmin.GetVSchemas:output_type -> vtadmin.GetVSchemasResponse + 72, // 231: vtadmin.VTAdmin.GetVtctlds:output_type -> vtadmin.GetVtctldsResponse + 16, // 232: vtadmin.VTAdmin.GetWorkflow:output_type -> vtadmin.Workflow + 78, // 233: vtadmin.VTAdmin.GetWorkflows:output_type -> vtadmin.GetWorkflowsResponse + 192, // 234: vtadmin.VTAdmin.GetWorkflowStatus:output_type -> vtctldata.WorkflowStatusResponse + 193, // 235: vtadmin.VTAdmin.StartWorkflow:output_type -> vtctldata.WorkflowUpdateResponse + 193, // 236: vtadmin.VTAdmin.StopWorkflow:output_type -> vtctldata.WorkflowUpdateResponse + 194, // 237: vtadmin.VTAdmin.LaunchSchemaMigration:output_type -> vtctldata.LaunchSchemaMigrationResponse + 195, // 238: vtadmin.VTAdmin.MoveTablesComplete:output_type -> vtctldata.MoveTablesCompleteResponse + 192, // 239: vtadmin.VTAdmin.MoveTablesCreate:output_type -> vtctldata.WorkflowStatusResponse + 196, // 240: vtadmin.VTAdmin.MaterializeCreate:output_type -> vtctldata.MaterializeCreateResponse + 84, // 241: vtadmin.VTAdmin.PingTablet:output_type -> vtadmin.PingTabletResponse + 86, // 242: vtadmin.VTAdmin.PlannedFailoverShard:output_type -> vtadmin.PlannedFailoverShardResponse + 88, // 243: vtadmin.VTAdmin.RebuildKeyspaceGraph:output_type -> vtadmin.RebuildKeyspaceGraphResponse + 90, // 244: vtadmin.VTAdmin.RefreshState:output_type -> vtadmin.RefreshStateResponse + 96, // 245: vtadmin.VTAdmin.RefreshTabletReplicationSource:output_type -> vtadmin.RefreshTabletReplicationSourceResponse + 92, // 246: vtadmin.VTAdmin.ReloadSchemas:output_type -> vtadmin.ReloadSchemasResponse + 94, // 247: vtadmin.VTAdmin.ReloadSchemaShard:output_type -> vtadmin.ReloadSchemaShardResponse + 98, // 248: vtadmin.VTAdmin.RemoveKeyspaceCell:output_type -> vtadmin.RemoveKeyspaceCellResponse + 197, // 249: vtadmin.VTAdmin.RetrySchemaMigration:output_type -> vtctldata.RetrySchemaMigrationResponse + 101, // 250: vtadmin.VTAdmin.RunHealthCheck:output_type -> vtadmin.RunHealthCheckResponse + 192, // 251: vtadmin.VTAdmin.ReshardCreate:output_type -> vtctldata.WorkflowStatusResponse + 104, // 252: vtadmin.VTAdmin.SetReadOnly:output_type -> vtadmin.SetReadOnlyResponse + 106, // 253: vtadmin.VTAdmin.SetReadWrite:output_type -> vtadmin.SetReadWriteResponse + 108, // 254: vtadmin.VTAdmin.StartReplication:output_type -> vtadmin.StartReplicationResponse + 110, // 255: vtadmin.VTAdmin.StopReplication:output_type -> vtadmin.StopReplicationResponse + 112, // 256: vtadmin.VTAdmin.TabletExternallyPromoted:output_type -> vtadmin.TabletExternallyPromotedResponse + 198, // 257: vtadmin.VTAdmin.Validate:output_type -> vtctldata.ValidateResponse + 199, // 258: vtadmin.VTAdmin.ValidateKeyspace:output_type -> vtctldata.ValidateKeyspaceResponse + 200, // 259: vtadmin.VTAdmin.ValidateSchemaKeyspace:output_type -> vtctldata.ValidateSchemaKeyspaceResponse + 201, // 260: vtadmin.VTAdmin.ValidateShard:output_type -> vtctldata.ValidateShardResponse + 202, // 261: vtadmin.VTAdmin.ValidateVersionKeyspace:output_type -> vtctldata.ValidateVersionKeyspaceResponse + 203, // 262: vtadmin.VTAdmin.ValidateVersionShard:output_type -> vtctldata.ValidateVersionShardResponse + 204, // 263: vtadmin.VTAdmin.VDiffCreate:output_type -> vtctldata.VDiffCreateResponse + 124, // 264: vtadmin.VTAdmin.VDiffShow:output_type -> vtadmin.VDiffShowResponse + 126, // 265: vtadmin.VTAdmin.VTExplain:output_type -> vtadmin.VTExplainResponse + 128, // 266: vtadmin.VTAdmin.VExplain:output_type -> vtadmin.VExplainResponse + 205, // 267: vtadmin.VTAdmin.WorkflowDelete:output_type -> vtctldata.WorkflowDeleteResponse + 206, // 268: vtadmin.VTAdmin.WorkflowSwitchTraffic:output_type -> vtctldata.WorkflowSwitchTrafficResponse + 196, // [196:269] is the sub-list for method output_type + 123, // [123:196] is the sub-list for method input_type 123, // [123:123] is the sub-list for extension type_name 123, // [123:123] is the sub-list for extension extendee 0, // [0:123] is the sub-list for field type_name @@ -9454,7 +9577,7 @@ func file_vtadmin_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_vtadmin_proto_rawDesc, NumEnums: 1, - NumMessages: 139, + NumMessages: 141, NumExtensions: 0, NumServices: 1, }, diff --git a/go/vt/proto/vtadmin/vtadmin_grpc.pb.go b/go/vt/proto/vtadmin/vtadmin_grpc.pb.go index 89fffccd424..b1b84e6c2db 100644 --- a/go/vt/proto/vtadmin/vtadmin_grpc.pb.go +++ b/go/vt/proto/vtadmin/vtadmin_grpc.pb.go @@ -215,6 +215,9 @@ type VTAdminClient interface { // VTExplain provides information on how Vitess plans to execute a // particular query. VTExplain(ctx context.Context, in *VTExplainRequest, opts ...grpc.CallOption) (*VTExplainResponse, error) + // VExplain provides information on how Vitess plans to execute a + // particular query. + VExplain(ctx context.Context, in *VExplainRequest, opts ...grpc.CallOption) (*VExplainResponse, error) // WorkflowDelete deletes a vreplication workflow. WorkflowDelete(ctx context.Context, in *WorkflowDeleteRequest, opts ...grpc.CallOption) (*vtctldata.WorkflowDeleteResponse, error) // WorkflowSwitchTraffic switches traffic for a VReplication workflow. @@ -859,6 +862,15 @@ func (c *vTAdminClient) VTExplain(ctx context.Context, in *VTExplainRequest, opt return out, nil } +func (c *vTAdminClient) VExplain(ctx context.Context, in *VExplainRequest, opts ...grpc.CallOption) (*VExplainResponse, error) { + out := new(VExplainResponse) + err := c.cc.Invoke(ctx, "/vtadmin.VTAdmin/VExplain", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *vTAdminClient) WorkflowDelete(ctx context.Context, in *WorkflowDeleteRequest, opts ...grpc.CallOption) (*vtctldata.WorkflowDeleteResponse, error) { out := new(vtctldata.WorkflowDeleteResponse) err := c.cc.Invoke(ctx, "/vtadmin.VTAdmin/WorkflowDelete", in, out, opts...) @@ -1073,6 +1085,9 @@ type VTAdminServer interface { // VTExplain provides information on how Vitess plans to execute a // particular query. VTExplain(context.Context, *VTExplainRequest) (*VTExplainResponse, error) + // VExplain provides information on how Vitess plans to execute a + // particular query. + VExplain(context.Context, *VExplainRequest) (*VExplainResponse, error) // WorkflowDelete deletes a vreplication workflow. WorkflowDelete(context.Context, *WorkflowDeleteRequest) (*vtctldata.WorkflowDeleteResponse, error) // WorkflowSwitchTraffic switches traffic for a VReplication workflow. @@ -1294,6 +1309,9 @@ func (UnimplementedVTAdminServer) VDiffShow(context.Context, *VDiffShowRequest) func (UnimplementedVTAdminServer) VTExplain(context.Context, *VTExplainRequest) (*VTExplainResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method VTExplain not implemented") } +func (UnimplementedVTAdminServer) VExplain(context.Context, *VExplainRequest) (*VExplainResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method VExplain not implemented") +} func (UnimplementedVTAdminServer) WorkflowDelete(context.Context, *WorkflowDeleteRequest) (*vtctldata.WorkflowDeleteResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method WorkflowDelete not implemented") } @@ -2573,6 +2591,24 @@ func _VTAdmin_VTExplain_Handler(srv interface{}, ctx context.Context, dec func(i return interceptor(ctx, in, info, handler) } +func _VTAdmin_VExplain_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(VExplainRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(VTAdminServer).VExplain(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/vtadmin.VTAdmin/VExplain", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(VTAdminServer).VExplain(ctx, req.(*VExplainRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _VTAdmin_WorkflowDelete_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(WorkflowDeleteRequest) if err := dec(in); err != nil { @@ -2896,6 +2932,10 @@ var VTAdmin_ServiceDesc = grpc.ServiceDesc{ MethodName: "VTExplain", Handler: _VTAdmin_VTExplain_Handler, }, + { + MethodName: "VExplain", + Handler: _VTAdmin_VExplain_Handler, + }, { MethodName: "WorkflowDelete", Handler: _VTAdmin_WorkflowDelete_Handler, diff --git a/go/vt/proto/vtadmin/vtadmin_vtproto.pb.go b/go/vt/proto/vtadmin/vtadmin_vtproto.pb.go index 82cca2cea06..31cfd018921 100644 --- a/go/vt/proto/vtadmin/vtadmin_vtproto.pb.go +++ b/go/vt/proto/vtadmin/vtadmin_vtproto.pb.go @@ -2801,6 +2801,42 @@ func (m *VTExplainResponse) CloneMessageVT() proto.Message { return m.CloneVT() } +func (m *VExplainRequest) CloneVT() *VExplainRequest { + if m == nil { + return (*VExplainRequest)(nil) + } + r := new(VExplainRequest) + r.ClusterId = m.ClusterId + r.Keyspace = m.Keyspace + r.Sql = m.Sql + if len(m.unknownFields) > 0 { + r.unknownFields = make([]byte, len(m.unknownFields)) + copy(r.unknownFields, m.unknownFields) + } + return r +} + +func (m *VExplainRequest) CloneMessageVT() proto.Message { + return m.CloneVT() +} + +func (m *VExplainResponse) CloneVT() *VExplainResponse { + if m == nil { + return (*VExplainResponse)(nil) + } + r := new(VExplainResponse) + r.Response = m.Response + if len(m.unknownFields) > 0 { + r.unknownFields = make([]byte, len(m.unknownFields)) + copy(r.unknownFields, m.unknownFields) + } + return r +} + +func (m *VExplainResponse) CloneMessageVT() proto.Message { + return m.CloneVT() +} + func (m *Cluster) MarshalVT() (dAtA []byte, err error) { if m == nil { return nil, nil @@ -9860,6 +9896,100 @@ func (m *VTExplainResponse) MarshalToSizedBufferVT(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *VExplainRequest) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *VExplainRequest) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *VExplainRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.Sql) > 0 { + i -= len(m.Sql) + copy(dAtA[i:], m.Sql) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Sql))) + i-- + dAtA[i] = 0x1a + } + if len(m.Keyspace) > 0 { + i -= len(m.Keyspace) + copy(dAtA[i:], m.Keyspace) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Keyspace))) + i-- + dAtA[i] = 0x12 + } + if len(m.ClusterId) > 0 { + i -= len(m.ClusterId) + copy(dAtA[i:], m.ClusterId) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.ClusterId))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *VExplainResponse) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *VExplainResponse) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *VExplainResponse) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.Response) > 0 { + i -= len(m.Response) + copy(dAtA[i:], m.Response) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Response))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + func (m *Cluster) SizeVT() (n int) { if m == nil { return 0 @@ -12560,6 +12690,42 @@ func (m *VTExplainResponse) SizeVT() (n int) { return n } +func (m *VExplainRequest) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.ClusterId) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + l = len(m.Keyspace) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + l = len(m.Sql) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + n += len(m.unknownFields) + return n +} + +func (m *VExplainResponse) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Response) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + n += len(m.unknownFields) + return n +} + func (m *Cluster) UnmarshalVT(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -29699,3 +29865,233 @@ func (m *VTExplainResponse) UnmarshalVT(dAtA []byte) error { } return nil } +func (m *VExplainRequest) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: VExplainRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: VExplainRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ClusterId", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ClusterId = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Keyspace", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Keyspace = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Sql", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Sql = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *VExplainResponse) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: VExplainResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: VExplainResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Response", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Response = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} diff --git a/go/vt/vtadmin/api.go b/go/vt/vtadmin/api.go index a54090bb044..1182d694e19 100644 --- a/go/vt/vtadmin/api.go +++ b/go/vt/vtadmin/api.go @@ -433,6 +433,7 @@ func (api *API) Handler() http.Handler { router.HandleFunc("/vdiff/{cluster_id}/show", httpAPI.Adapt(vtadminhttp.VDiffShow)).Name("API.VDiffShow") router.HandleFunc("/vtctlds", httpAPI.Adapt(vtadminhttp.GetVtctlds)).Name("API.GetVtctlds") router.HandleFunc("/vtexplain", httpAPI.Adapt(vtadminhttp.VTExplain)).Name("API.VTExplain") + router.HandleFunc("/vexplain", httpAPI.Adapt(vtadminhttp.VExplain)).Name("API.VExplain") router.HandleFunc("/workflow/{cluster_id}/{keyspace}/{name}", httpAPI.Adapt(vtadminhttp.GetWorkflow)).Name("API.GetWorkflow") router.HandleFunc("/workflows", httpAPI.Adapt(vtadminhttp.GetWorkflows)).Name("API.GetWorkflows") router.HandleFunc("/workflow/{cluster_id}/{keyspace}/{name}/status", httpAPI.Adapt(vtadminhttp.GetWorkflowStatus)).Name("API.GetWorkflowStatus") @@ -2616,6 +2617,59 @@ func (api *API) ValidateVersionShard(ctx context.Context, req *vtadminpb.Validat return res, nil } +// VExplain is part of the vtadminpb.VTAdminServer interface. +func (api *API) VExplain(ctx context.Context, req *vtadminpb.VExplainRequest) (*vtadminpb.VExplainResponse, error) { + span, ctx := trace.NewSpan(ctx, "API.VExplain") + defer span.Finish() + + if req.ClusterId == "" { + return nil, fmt.Errorf("%w: clusterID is required", errors.ErrInvalidRequest) + } + + if req.Keyspace == "" { + return nil, fmt.Errorf("%w: keyspace name is required", errors.ErrInvalidRequest) + } + + if req.Sql == "" { + return nil, fmt.Errorf("%w: SQL query is required", errors.ErrInvalidRequest) + } + + c, err := api.getClusterForRequest(req.ClusterId) + if err != nil { + return nil, err + } + + if !api.authz.IsAuthorized(ctx, c.ID, rbac.VExplainResource, rbac.GetAction) { + return nil, nil + } + + // Parser with default options. New() itself initializes with default MySQL version. + parser, err := sqlparser.New(sqlparser.Options{ + TruncateUILen: 512, + TruncateErrLen: 0, + }) + if err != nil { + return nil, err + } + + stmt, err := parser.Parse(req.GetSql()) + if err != nil { + return nil, err + } + + if _, ok := stmt.(*sqlparser.VExplainStmt); !ok { + return nil, vterrors.VT09017("Invalid VExplain statement") + } + + response, err := c.DB.VExplain(ctx, req.GetSql(), stmt.(*sqlparser.VExplainStmt)) + + if err != nil { + return nil, err + } + + return response, nil +} + // VTExplain is part of the vtadminpb.VTAdminServer interface. func (api *API) VTExplain(ctx context.Context, req *vtadminpb.VTExplainRequest) (*vtadminpb.VTExplainResponse, error) { // TODO (andrew): https://github.com/vitessio/vitess/issues/12161. diff --git a/go/vt/vtadmin/api_test.go b/go/vt/vtadmin/api_test.go index 82c744b95db..011acdf3e59 100644 --- a/go/vt/vtadmin/api_test.go +++ b/go/vt/vtadmin/api_test.go @@ -5136,6 +5136,186 @@ func TestVTExplain(t *testing.T) { } } +func TestVExplain(t *testing.T) { + tests := []struct { + name string + keyspaces []*vtctldatapb.Keyspace + shards []*vtctldatapb.Shard + srvVSchema *vschemapb.SrvVSchema + tabletSchemas map[string]*tabletmanagerdatapb.SchemaDefinition + tablets []*vtadminpb.Tablet + req *vtadminpb.VExplainRequest + expectedError error + }{ + { + name: "returns an error if cluster unspecified in request", + req: &vtadminpb.VExplainRequest{ + Keyspace: "commerce", + Sql: "vexplain all select * from customers", + }, + expectedError: vtadminerrors.ErrInvalidRequest, + }, + { + name: "returns an error if keyspace unspecified in request", + req: &vtadminpb.VExplainRequest{ + ClusterId: "c0", + Sql: "vexplain all select * from customers", + }, + expectedError: vtadminerrors.ErrInvalidRequest, + }, + { + name: "returns an error if SQL unspecified in request", + req: &vtadminpb.VExplainRequest{ + ClusterId: "c0", + Keyspace: "commerce", + }, + expectedError: vtadminerrors.ErrInvalidRequest, + }, + { + name: "runs VExplain given a valid request in a valid topology", + keyspaces: []*vtctldatapb.Keyspace{ + { + Name: "commerce", + Keyspace: &topodatapb.Keyspace{}, + }, + }, + shards: []*vtctldatapb.Shard{ + { + Name: "-", + Keyspace: "commerce", + }, + }, + srvVSchema: &vschemapb.SrvVSchema{ + Keyspaces: map[string]*vschemapb.Keyspace{ + "commerce": { + Sharded: false, + Tables: map[string]*vschemapb.Table{ + "customers": {}, + }, + }, + }, + RoutingRules: &vschemapb.RoutingRules{ + Rules: []*vschemapb.RoutingRule{}, + }, + }, + tabletSchemas: map[string]*tabletmanagerdatapb.SchemaDefinition{ + "c0_cell1-0000000100": { + DatabaseSchema: "CREATE DATABASE commerce", + TableDefinitions: []*tabletmanagerdatapb.TableDefinition{ + { + Name: "t1", + Schema: `CREATE TABLE customers (id int(11) not null,PRIMARY KEY (id));`, + Type: "BASE", + Columns: []string{"id"}, + DataLength: 100, + RowCount: 50, + Fields: []*querypb.Field{ + { + Name: "id", + Type: querypb.Type_INT32, + }, + }, + }, + }, + }, + }, + tablets: []*vtadminpb.Tablet{ + { + Cluster: &vtadminpb.Cluster{ + Id: "c0", + Name: "cluster0", + }, + State: vtadminpb.Tablet_SERVING, + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Uid: 100, + Cell: "c0_cell1", + }, + Hostname: "tablet-cell1-a", + Keyspace: "commerce", + Shard: "-", + Type: topodatapb.TabletType_REPLICA, + }, + }, + }, + req: &vtadminpb.VExplainRequest{ + ClusterId: "c0", + Keyspace: "commerce", + Sql: "vexplain all select * from customers", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + toposerver := memorytopo.NewServer(ctx, "c0_cell1") + + tmc := testutil.TabletManagerClient{ + GetSchemaResults: map[string]struct { + Schema *tabletmanagerdatapb.SchemaDefinition + Error error + }{}, + } + + vtctldserver := testutil.NewVtctldServerWithTabletManagerClient(t, toposerver, &tmc, func(ts *topo.Server) vtctlservicepb.VtctldServer { + return grpcvtctldserver.NewVtctldServer(vtenv.NewTestEnv(), ts) + }) + + testutil.WithTestServer(ctx, t, vtctldserver, func(t *testing.T, vtctldClient vtctldclient.VtctldClient) { + if tt.srvVSchema != nil { + err := toposerver.UpdateSrvVSchema(ctx, "c0_cell1", tt.srvVSchema) + require.NoError(t, err) + } + testutil.AddKeyspaces(ctx, t, toposerver, tt.keyspaces...) + testutil.AddShards(ctx, t, toposerver, tt.shards...) + + for _, tablet := range tt.tablets { + testutil.AddTablet(ctx, t, toposerver, tablet.Tablet, nil) + + // Adds each SchemaDefinition to the fake TabletManagerClient, or nil + // if there are no schemas for that tablet. (All tablet aliases must + // exist in the map. Otherwise, TabletManagerClient will return an error when + // looking up the schema with tablet alias that doesn't exist.) + alias := topoproto.TabletAliasString(tablet.Tablet.Alias) + tmc.GetSchemaResults[alias] = struct { + Schema *tabletmanagerdatapb.SchemaDefinition + Error error + }{ + Schema: tt.tabletSchemas[alias], + Error: nil, + } + } + + clusters := []*cluster.Cluster{ + vtadmintestutil.BuildCluster(t, vtadmintestutil.TestClusterConfig{ + Cluster: &vtadminpb.Cluster{ + Id: "c0", + Name: "cluster0", + }, + VtctldClient: vtctldClient, + Tablets: tt.tablets, + }), + } + + api := NewAPI(vtenv.NewTestEnv(), clusters, Options{}) + resp, err := api.VExplain(ctx, tt.req) + + if tt.expectedError != nil { + assert.True(t, errors.Is(err, tt.expectedError), "expected error type %w does not match actual error type %w", err, tt.expectedError) + } else { + require.NoError(t, err) + + // We don't particularly care to test the contents of the VExplain response, + // just that it exists. + assert.NotEmpty(t, resp.Response) + } + }) + }) + } +} + type ServeHTTPVtctldResponse struct { Result ServeHTTPVtctldResult `json:"result"` Ok bool `json:"ok"` diff --git a/go/vt/vtadmin/http/vexplain.go b/go/vt/vtadmin/http/vexplain.go new file mode 100644 index 00000000000..32b705e062a --- /dev/null +++ b/go/vt/vtadmin/http/vexplain.go @@ -0,0 +1,34 @@ +/* +Copyright 2025 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package http + +import ( + "context" + + vtadminpb "vitess.io/vitess/go/vt/proto/vtadmin" +) + +// VExplain implements the http wrapper for /vexplain?cluster_id=&keyspace=&sql= +func VExplain(ctx context.Context, r Request, api *API) *JSONResponse { + query := r.URL.Query() + res, err := api.server.VExplain(ctx, &vtadminpb.VExplainRequest{ + ClusterId: query.Get("cluster_id"), + Keyspace: query.Get("keyspace"), + Sql: query.Get("sql"), + }) + return NewJSONResponse(res, err) +} diff --git a/go/vt/vtadmin/rbac/rbac.go b/go/vt/vtadmin/rbac/rbac.go index 038db46fbd5..1184b57f7ab 100644 --- a/go/vt/vtadmin/rbac/rbac.go +++ b/go/vt/vtadmin/rbac/rbac.go @@ -136,5 +136,7 @@ const ( VTExplainResource Resource = "VTExplain" + VExplainResource Resource = "VExplain" + TabletFullStatusResource Resource = "TabletFullStatus" ) diff --git a/go/vt/vtadmin/vtsql/fakevtsql/conn.go b/go/vt/vtadmin/vtsql/fakevtsql/conn.go index af9ac44ad0d..62e3ba7277a 100644 --- a/go/vt/vtadmin/vtsql/fakevtsql/conn.go +++ b/go/vt/vtadmin/vtsql/fakevtsql/conn.go @@ -90,6 +90,17 @@ func (c *conn) QueryContext(ctx context.Context, query string, args []driver.Nam }) } + return &rows{ + cols: columns, + vals: vals, + pos: 0, + closed: false, + }, nil + case "vexplain all select * from customers": + columns := []string{"VExplain"} + vals := [][]any{} + vals = append(vals, []any{"{'Table' : 'customer, 'TestPlan' : 'TestPlan'}"}) + return &rows{ cols: columns, vals: vals, diff --git a/go/vt/vtadmin/vtsql/vtsql.go b/go/vt/vtadmin/vtsql/vtsql.go index 9f23eb70443..d923730192a 100644 --- a/go/vt/vtadmin/vtsql/vtsql.go +++ b/go/vt/vtadmin/vtsql/vtsql.go @@ -17,10 +17,13 @@ limitations under the License. package vtsql import ( + "bytes" "context" "database/sql" "fmt" + "strings" "sync" + "text/tabwriter" "time" "google.golang.org/grpc/credentials/insecure" @@ -31,6 +34,7 @@ import ( "vitess.io/vitess/go/trace" "vitess.io/vitess/go/vt/callerid" "vitess.io/vitess/go/vt/log" + "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vitessdriver" "vitess.io/vitess/go/vt/vtadmin/cluster/resolver" "vitess.io/vitess/go/vt/vtadmin/debug" @@ -45,6 +49,9 @@ type DB interface { // ShowTablets executes `SHOW vitess_tablets` and returns the result. ShowTablets(ctx context.Context) (*sql.Rows, error) + // VExplain executes query - `vexplain [ALL|PLAN|QUERIES|TRACE|KEYS] query` and returns the results + VExplain(ctx context.Context, query string, vexplainStmt *sqlparser.VExplainStmt) (*vtadminpb.VExplainResponse, error) + // Ping behaves like (*sql.DB).Ping. Ping() error // PingContext behaves like (*sql.DB).PingContext. @@ -174,6 +181,73 @@ func (vtgate *VTGateProxy) ShowTablets(ctx context.Context) (*sql.Rows, error) { return vtgate.conn.QueryContext(vtgate.getQueryContext(ctx), "SHOW vitess_tablets") } +// VExplain is part of the DB interface. +func (vtgate *VTGateProxy) VExplain(ctx context.Context, query string, vexplainStmt *sqlparser.VExplainStmt) (*vtadminpb.VExplainResponse, error) { + span, ctx := trace.NewSpan(ctx, "VTGateProxy.VExplain") + defer span.Finish() + + vtadminproto.AnnotateClusterSpan(vtgate.cluster, span) + + rows, err := vtgate.conn.QueryContext(vtgate.getQueryContext(ctx), query) + + if err != nil { + return nil, err + } + switch vexplainStmt.Type { + case sqlparser.QueriesVExplainType: + return convertVExplainQueriesResultToString(rows) + case sqlparser.AllVExplainType, sqlparser.TraceVExplainType, sqlparser.PlanVExplainType, sqlparser.KeysVExplainType: + return convertVExplainResultToString(rows) + default: + return nil, nil + } +} + +func convertVExplainResultToString(rows *sql.Rows) (*vtadminpb.VExplainResponse, error) { + var queryPlan string + for rows.Next() { + if err := rows.Scan(&queryPlan); err != nil { + return nil, err + } + } + return &vtadminpb.VExplainResponse{ + Response: queryPlan, + }, nil +} + +func convertVExplainQueriesResultToString(rows *sql.Rows) (*vtadminpb.VExplainResponse, error) { + var buf bytes.Buffer + w := tabwriter.NewWriter(&buf, 0, 0, 0, ' ', tabwriter.AlignRight) + + sep := []byte("|") + newLine := []byte("\n") + cols, _ := rows.Columns() + + if _, err := w.Write([]byte(strings.Join(cols, string(sep)) + string(newLine))); err != nil { + return nil, err + } + + row := make([][]byte, len(cols)) + rowPtr := make([]any, len(cols)) + for i := range row { + rowPtr[i] = &row[i] + } + + for rows.Next() { + if err := rows.Scan(rowPtr...); err != nil { + return nil, err + } + if _, err := w.Write(append(bytes.Join(row, sep), newLine...)); err != nil { + return nil, err + } + } + w.Flush() + + return &vtadminpb.VExplainResponse{ + Response: buf.String(), + }, nil +} + // Ping is part of the DB interface. func (vtgate *VTGateProxy) Ping() error { return vtgate.pingContext(context.Background()) diff --git a/proto/vtadmin.proto b/proto/vtadmin.proto index 963d1fa5779..1485cb485c2 100644 --- a/proto/vtadmin.proto +++ b/proto/vtadmin.proto @@ -225,6 +225,9 @@ service VTAdmin { // VTExplain provides information on how Vitess plans to execute a // particular query. rpc VTExplain(VTExplainRequest) returns (VTExplainResponse) {}; + // VExplain provides information on how Vitess plans to execute a + // particular query. + rpc VExplain(VExplainRequest) returns (VExplainResponse) {}; // WorkflowDelete deletes a vreplication workflow. rpc WorkflowDelete(WorkflowDeleteRequest) returns (vtctldata.WorkflowDeleteResponse) {}; // WorkflowSwitchTraffic switches traffic for a VReplication workflow. @@ -1097,3 +1100,13 @@ message VTExplainRequest { message VTExplainResponse { string response = 1; } + +message VExplainRequest { + string cluster_id = 1; + string keyspace = 2; + string sql = 3; +} + +message VExplainResponse { + string response = 1; +} diff --git a/web/vtadmin/src/api/http.ts b/web/vtadmin/src/api/http.ts index 674df961ef0..846f929ec52 100644 --- a/web/vtadmin/src/api/http.ts +++ b/web/vtadmin/src/api/http.ts @@ -636,6 +636,22 @@ export const fetchVTExplain = async ({ cluster, return pb.VTExplainResponse.create(result); }; +export const fetchVExplain = async ({ cluster_id, keyspace, sql }: R) => { + // As an easy enhancement for later, we can also validate the request parameters on the front-end + // instead of defaulting to '', to save a round trip. + const req = new URLSearchParams(); + req.append('cluster_id', cluster_id || ''); + req.append('keyspace', keyspace || ''); + req.append('sql', sql || ''); + + const { result } = await vtfetch(`/api/vexplain?${req}`); + + const err = pb.VExplainResponse.verify(result); + if (err) throw Error(err); + + return pb.VExplainResponse.create(result); +}; + export interface ValidateKeyspaceParams { clusterID: string; keyspace: string; diff --git a/web/vtadmin/src/components/App.tsx b/web/vtadmin/src/components/App.tsx index fd0f772ae19..e9f4bbb1844 100644 --- a/web/vtadmin/src/components/App.tsx +++ b/web/vtadmin/src/components/App.tsx @@ -30,6 +30,7 @@ import { Stream } from './routes/stream/Stream'; import { Workflows } from './routes/Workflows'; import { Workflow } from './routes/workflow/Workflow'; import { VTExplain } from './routes/VTExplain'; +import { VExplain } from './routes/VExplain'; import { Keyspace } from './routes/keyspace/Keyspace'; import { Tablet } from './routes/tablet/Tablet'; import { Backups } from './routes/Backups'; @@ -113,6 +114,10 @@ export const App = () => { + + + + diff --git a/web/vtadmin/src/components/NavRail.tsx b/web/vtadmin/src/components/NavRail.tsx index b30cd165684..c5e0c528bdd 100644 --- a/web/vtadmin/src/components/NavRail.tsx +++ b/web/vtadmin/src/components/NavRail.tsx @@ -77,6 +77,9 @@ export const NavRail = () => {
    • +
  • + +
    • diff --git a/web/vtadmin/src/components/routes/VExplain.tsx b/web/vtadmin/src/components/routes/VExplain.tsx new file mode 100644 index 00000000000..06969e63ca3 --- /dev/null +++ b/web/vtadmin/src/components/routes/VExplain.tsx @@ -0,0 +1,153 @@ +/** + * Copyright 2025 The Vitess Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import React, { useMemo } from 'react'; +import { orderBy } from 'lodash-es'; + +import { vtadmin as pb } from '../../proto/vtadmin'; +import { useKeyspaces, useVExplain } from '../../hooks/api'; +import { Select } from '../inputs/Select'; +import { ContentContainer } from '../layout/ContentContainer'; +import { WorkspaceHeader } from '../layout/WorkspaceHeader'; +import { WorkspaceTitle } from '../layout/WorkspaceTitle'; +import style from './VTExplain.module.scss'; +import { Code } from '../Code'; +import { useDocumentTitle } from '../../hooks/useDocumentTitle'; +import { Label } from '../inputs/Label'; + +export const VExplain = () => { + useDocumentTitle('VExplain'); + + const { data: keyspaces = [] } = useKeyspaces(); + + const [clusterID, updateCluster] = React.useState(null); + const [keyspaceName, updateKeyspace] = React.useState(null); + const [sql, updateSQL] = React.useState(null); + const [vexplainOption, updateVExplainOption] = React.useState('ALL'); + + const fetchVExplainRequestSql = function () { + return 'VEXPLAIN ' + vexplainOption + ' ' + sql; + }; + + const selectedKeyspace = + clusterID && keyspaceName + ? keyspaces?.find((k) => k.cluster?.id === clusterID && k.keyspace?.name === keyspaceName) + : null; + + const { data, error, refetch } = useVExplain( + { cluster_id: clusterID, keyspace: keyspaceName, sql: fetchVExplainRequestSql() }, + { + // Never cache, never refetch. + cacheTime: 0, + enabled: false, + refetchOnWindowFocus: false, + retry: false, + } + ); + + const onChangeKeyspace = (selectedKeyspace: pb.Keyspace | null | undefined) => { + updateCluster(selectedKeyspace?.cluster?.id); + updateKeyspace(selectedKeyspace?.keyspace?.name); + updateSQL(null); + }; + + const onChangeSQL: React.ChangeEventHandler = (e) => { + updateSQL(e.target.value); + }; + + const onSubmit: React.FormEventHandler = (e) => { + e.preventDefault(); + refetch(); + }; + + const VEXPLAIN_OPTIONS = ['ALL', 'PLAN', 'QUERIES', 'TRACE', 'KEYS']; + + const isReadyForSubmit = useMemo(() => { + return ( + typeof keyspaceName !== 'undefined' && + keyspaceName !== null && + keyspaceName !== '' && + typeof sql !== 'undefined' && + sql !== null && + sql !== '' + ); + }, [keyspaceName, sql]); + + return ( +
    + + VExplain + + +
    +
    +
    +