From af09e309bcd165ff4793bbd9c6a4de06912881ff Mon Sep 17 00:00:00 2001 From: Neil Macneale IV Date: Thu, 11 Jul 2024 09:27:22 -0700 Subject: [PATCH 01/18] Return empty dolt_procedures table if one hasn't been materialized --- go/libraries/doltcore/sqle/database.go | 11 ++ .../doltcore/sqle/procedures_table.go | 128 +++++++++++++----- .../doltcore/sqle/procedures_table_test.go | 20 ++- 3 files changed, 124 insertions(+), 35 deletions(-) diff --git a/go/libraries/doltcore/sqle/database.go b/go/libraries/doltcore/sqle/database.go index 9d2e80ee71c..a3c66ea9860 100644 --- a/go/libraries/doltcore/sqle/database.go +++ b/go/libraries/doltcore/sqle/database.go @@ -499,6 +499,17 @@ func (db Database) getTableInsensitive(ctx *sql.Context, head *doltdb.Commit, ds } case doltdb.StatisticsTableName: dt, found = dtables.NewStatisticsTable(ctx, db.Name(), db.ddb, asOf), true + case doltdb.ProceduresTableName: + backingTable, _, err := db.getTable(ctx, root, doltdb.ProceduresTableName) + if err != nil { + return nil, false, err + } + if backingTable == nil { + dt, found = NewEmptyProceduresTable(), true + } else { + writeTable := backingTable.(*WritableDoltTable) + dt, found = NewProceduresTable(writeTable), true + } } if found { diff --git a/go/libraries/doltcore/sqle/procedures_table.go b/go/libraries/doltcore/sqle/procedures_table.go index 39c2b25d64f..cf5815ac3ce 100644 --- a/go/libraries/doltcore/sqle/procedures_table.go +++ b/go/libraries/doltcore/sqle/procedures_table.go @@ -43,6 +43,53 @@ const ( ProceduresTableModifiedAtCol = "modified_at" ) +type ProceduresTable struct { + backingTable *WritableDoltTable +} + +func (pt *ProceduresTable) Name() string { + return ProceduresTableName +} + +func (pt *ProceduresTable) String() string { + return ProceduresTableName +} + +func (pt *ProceduresTable) Schema() sql.Schema { + return ProceduresTableSqlSchema().Schema +} + +func (pt *ProceduresTable) Collation() sql.CollationID { + return sql.Collation_Default +} + +func (pt *ProceduresTable) Partitions(ctx *sql.Context) (sql.PartitionIter, error) { + if pt.backingTable == nil { + // no backing table; return an empty iter. + return index.SinglePartitionIterFromNomsMap(nil), nil + } + return pt.backingTable.Partitions(ctx) +} + +func (pt *ProceduresTable) PartitionRows(ctx *sql.Context, partition sql.Partition) (sql.RowIter, error) { + if pt.backingTable == nil { + // no backing table; return an empty iter. + return sql.RowsToRowIter(), nil + } + + return pt.backingTable.PartitionRows(ctx, partition) +} + +func NewProceduresTable(backing *WritableDoltTable) sql.Table { + return &ProceduresTable{backingTable: backing} +} + +func NewEmptyProceduresTable() sql.Table { + return &ProceduresTable{} +} + +var _ sql.Table = (*ProceduresTable)(nil) + // The fixed SQL schema for the `dolt_procedures` table. func ProceduresTableSqlSchema() sql.PrimaryKeySchema { sqlSchema, err := sqlutil.FromDoltSchema("", doltdb.ProceduresTableName, ProceduresTableSchema()) @@ -65,37 +112,47 @@ func ProceduresTableSchema() schema.Schema { } // DoltProceduresGetOrCreateTable returns the `dolt_procedures` table from the given db, creating it in the db's -// current root if it doesn't exist +// current root if it doesn't exist. func DoltProceduresGetOrCreateTable(ctx *sql.Context, db Database) (*WritableDoltTable, error) { tbl, found, err := db.GetTableInsensitive(ctx, doltdb.ProceduresTableName) if err != nil { return nil, err } - if found { - // Make sure the schema is up to date - writableDoltTable := tbl.(*WritableDoltTable) - return migrateDoltProceduresSchema(ctx, db, writableDoltTable) - } - - root, err := db.GetRoot(ctx) - if err != nil { - return nil, err + if !found { + // Should never happen. + panic("runtime error. dolt_procedures table not found") } - err = db.createDoltTable(ctx, doltdb.ProceduresTableName, doltdb.DefaultSchemaName, root, ProceduresTableSchema()) - if err != nil { - return nil, err + wrapper, ok := tbl.(*ProceduresTable) + if !ok { + return nil, fmt.Errorf("expected a ProceduresTable, but got %T", tbl) } - tbl, found, err = db.GetTableInsensitive(ctx, doltdb.ProceduresTableName) - if err != nil { - return nil, err - } - // Verify it was created successfully - if !found { - return nil, sql.ErrTableNotFound.New(ProceduresTableName) + if wrapper.backingTable == nil { + // We haven't materialized the table yet. Go ahead and do so. + root, err := db.GetRoot(ctx) + if err != nil { + return nil, err + } + err = db.createDoltTable(ctx, doltdb.ProceduresTableName, doltdb.DefaultSchemaName, root, ProceduresTableSchema()) + if err != nil { + return nil, err + } + tbl, _, err = db.GetTableInsensitive(ctx, doltdb.ProceduresTableName) + if err != nil { + return nil, err + } + wrapper, ok = tbl.(*ProceduresTable) + if !ok { + return nil, fmt.Errorf("expected a ProceduresTable, but got %T", tbl) + } + if wrapper.backingTable == nil { + return nil, sql.ErrTableNotFound.New(ProceduresTableName) + } + return wrapper.backingTable, nil + } else { + return migrateDoltProceduresSchema(ctx, db, wrapper.backingTable) } - return tbl.(*WritableDoltTable), nil } // migrateDoltProceduresSchema migrates the dolt_procedures system table from a previous schema version to the current @@ -162,15 +219,20 @@ func migrateDoltProceduresSchema(ctx *sql.Context, db Database, oldTable *Writab return nil, err } - tbl, found, err := db.GetTableInsensitive(ctx, doltdb.ProceduresTableName) + tbl, _, err := db.GetTableInsensitive(ctx, doltdb.ProceduresTableName) if err != nil { return nil, err } - if !found { + + wrapper, ok := tbl.(*ProceduresTable) + if !ok { + return nil, fmt.Errorf("expected a ProceduresTable, but got %T", tbl) + } + if wrapper.backingTable == nil { return nil, sql.ErrTableNotFound.New(doltdb.ProceduresTableName) } - inserter := tbl.(*WritableDoltTable).Inserter(ctx) + inserter := wrapper.backingTable.Inserter(ctx) for _, row := range newRows { err = inserter.Insert(ctx, row) if err != nil { @@ -183,22 +245,24 @@ func migrateDoltProceduresSchema(ctx *sql.Context, db Database, oldTable *Writab return nil, err } - return tbl.(*WritableDoltTable), nil + return wrapper.backingTable, nil } // DoltProceduresGetTable returns the `dolt_procedures` table from the given db, or nil if the table doesn't exist func DoltProceduresGetTable(ctx *sql.Context, db Database) (*WritableDoltTable, error) { - tbl, found, err := db.GetTableInsensitive(ctx, doltdb.ProceduresTableName) + tbl, _, err := db.GetTableInsensitive(ctx, doltdb.ProceduresTableName) if err != nil { return nil, err } - if found { - // Make sure the schema is up to date - writableDoltTable := tbl.(*WritableDoltTable) - return migrateDoltProceduresSchema(ctx, db, writableDoltTable) - } else { - return nil, nil + + wrapper, ok := tbl.(*ProceduresTable) + if !ok { + return nil, fmt.Errorf("expected a ProceduresTable, but got %T", tbl) + } + if wrapper.backingTable != nil { + return migrateDoltProceduresSchema(ctx, db, wrapper.backingTable) } + return nil, nil } // DoltProceduresGetAll returns all stored procedures for the database if the procedureName is blank (and empty string), diff --git a/go/libraries/doltcore/sqle/procedures_table_test.go b/go/libraries/doltcore/sqle/procedures_table_test.go index 89dbcfe3e70..13cb86abd24 100644 --- a/go/libraries/doltcore/sqle/procedures_table_test.go +++ b/go/libraries/doltcore/sqle/procedures_table_test.go @@ -67,7 +67,12 @@ func TestProceduresMigration(t *testing.T) { tbl, found, err := db.GetTableInsensitive(ctx, doltdb.ProceduresTableName) require.NoError(t, err) require.True(t, found) - rows := readAllRows(ctx, t, tbl.(*WritableDoltTable)) + + wrapper, ok := tbl.(*ProceduresTable) + require.True(t, ok) + require.NotNil(t, wrapper.backingTable) + + rows := readAllRows(ctx, t, wrapper.backingTable) expectedRows := []sql.Row{ {"proc1", "create procedure proc1() SELECT 42 as pk from dual;", timestamp, timestamp, nil}, {"proc2", "create procedure proc2() SELECT 'HELLO' as greeting from dual;", timestamp, timestamp, nil}, @@ -91,7 +96,12 @@ func TestProceduresMigration(t *testing.T) { tbl, found, err := db.GetTableInsensitive(ctx, doltdb.ProceduresTableName) require.NoError(t, err) require.True(t, found) - rows := readAllRows(ctx, t, tbl.(*WritableDoltTable)) + + wrapper, ok := tbl.(*ProceduresTable) + require.True(t, ok) + require.NotNil(t, wrapper.backingTable) + + rows := readAllRows(ctx, t, wrapper.backingTable) expectedRows := []sql.Row{ {"proc1", "create procedure proc1() SELECT 42 as pk from dual;", timestamp, timestamp, nil}, {"proc2", "create procedure proc2() SELECT 'HELLO' as greeting from dual;", timestamp, timestamp, nil}, @@ -122,8 +132,12 @@ func newDatabaseWithProcedures(t *testing.T, dEnv *env.DoltEnv, opts editor.Opti require.NoError(t, err) require.True(t, found) + wrapper, ok := sqlTbl.(*ProceduresTable) + require.True(t, ok) + require.NotNil(t, wrapper.backingTable) + // Insert some test data for procedures - inserter := sqlTbl.(*WritableDoltTable).Inserter(ctx) + inserter := wrapper.backingTable.Inserter(ctx) require.NoError(t, inserter.Insert(ctx, sql.Row{"proc1", "create procedure proc1() SELECT 42 as pk from dual;", timestamp, timestamp})) require.NoError(t, inserter.Insert(ctx, sql.Row{"proc2", "create procedure proc2() SELECT 'HELLO' as greeting from dual;", timestamp, timestamp})) require.NoError(t, inserter.Close(ctx)) From 42bfc90979f224c7a96b4c87057f480923f0ae41 Mon Sep 17 00:00:00 2001 From: Neil Macneale IV Date: Thu, 11 Jul 2024 13:37:19 -0700 Subject: [PATCH 02/18] More complete imple of empty dolt_procedures. 'dolt diff' works again --- go/cmd/dolt/commands/diff.go | 6 +++-- .../doltcore/sqle/procedures_table.go | 24 +++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/go/cmd/dolt/commands/diff.go b/go/cmd/dolt/commands/diff.go index 8e594b44382..e558ebf04ba 100644 --- a/go/cmd/dolt/commands/diff.go +++ b/go/cmd/dolt/commands/diff.go @@ -385,9 +385,11 @@ func getTableNamesAtRef(queryist cli.Queryist, sqlCtx *sql.Context, ref string) if err != nil { return nil, fmt.Errorf("error interpolating query: %w", err) } - _, err = GetRowsForSql(queryist, sqlCtx, interpolatedQuery) + result, err := GetRowsForSql(queryist, sqlCtx, interpolatedQuery) if err == nil { - tableNames[sysTable] = true + if len(result) > 0 { + tableNames[sysTable] = true + } } else if isTableNotFoundError(err) { continue } else { diff --git a/go/libraries/doltcore/sqle/procedures_table.go b/go/libraries/doltcore/sqle/procedures_table.go index cf5815ac3ce..3c59e494236 100644 --- a/go/libraries/doltcore/sqle/procedures_table.go +++ b/go/libraries/doltcore/sqle/procedures_table.go @@ -20,6 +20,7 @@ import ( "strings" "time" + "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dtables" "github.com/dolthub/go-mysql-server/sql" "gopkg.in/src-d/go-errors.v1" @@ -80,6 +81,27 @@ func (pt *ProceduresTable) PartitionRows(ctx *sql.Context, partition sql.Partiti return pt.backingTable.PartitionRows(ctx, partition) } +func (pt *ProceduresTable) LockedToRoot(ctx *sql.Context, root doltdb.RootValue) (sql.IndexAddressableTable, error) { + if pt.backingTable == nil { + return pt, nil + + } + return pt.backingTable.LockedToRoot(ctx, root) +} + +func (pt *ProceduresTable) IndexedAccess(lookup sql.IndexLookup) sql.IndexedTable { + // Never reached. Interface required for LockedToRoot to be implemented. + panic("Unreachable") +} + +func (pt *ProceduresTable) GetIndexes(ctx *sql.Context) ([]sql.Index, error) { + return nil, nil +} + +func (pt *ProceduresTable) PreciseMatch() bool { + return true +} + func NewProceduresTable(backing *WritableDoltTable) sql.Table { return &ProceduresTable{backingTable: backing} } @@ -89,6 +111,8 @@ func NewEmptyProceduresTable() sql.Table { } var _ sql.Table = (*ProceduresTable)(nil) +var _ dtables.VersionableTable = (*ProceduresTable)(nil) +var _ sql.IndexAddressableTable = (*ProceduresTable)(nil) // The fixed SQL schema for the `dolt_procedures` table. func ProceduresTableSqlSchema() sql.PrimaryKeySchema { From 8f0e5823959bc4f59f9a5b0e50bf48071121bf13 Mon Sep 17 00:00:00 2001 From: macneale4 Date: Thu, 11 Jul 2024 20:45:49 +0000 Subject: [PATCH 03/18] [ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/update.sh --- go/libraries/doltcore/sqle/procedures_table.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/libraries/doltcore/sqle/procedures_table.go b/go/libraries/doltcore/sqle/procedures_table.go index 3c59e494236..5ac0614792c 100644 --- a/go/libraries/doltcore/sqle/procedures_table.go +++ b/go/libraries/doltcore/sqle/procedures_table.go @@ -20,12 +20,12 @@ import ( "strings" "time" - "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dtables" "github.com/dolthub/go-mysql-server/sql" "gopkg.in/src-d/go-errors.v1" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" "github.com/dolthub/dolt/go/libraries/doltcore/schema" + "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dtables" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/index" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/sqlutil" "github.com/dolthub/dolt/go/store/types" From ea77bb00c6dc91ee1952e8b97772840665e0f907 Mon Sep 17 00:00:00 2001 From: Neil Macneale IV Date: Thu, 11 Jul 2024 15:15:03 -0700 Subject: [PATCH 04/18] return empty dolt_schemas table when appropriate --- go/cmd/dolt/commands/diff.go | 12 +- go/libraries/doltcore/sqle/database.go | 69 +++++++-- go/libraries/doltcore/sqle/schema_table.go | 139 +++++++++++++++--- .../doltcore/sqle/schema_table_test.go | 6 +- go/libraries/doltcore/sqle/sqlddl_test.go | 2 +- go/libraries/doltcore/sqle/sqldelete_test.go | 4 +- go/libraries/doltcore/sqle/sqlinsert_test.go | 6 +- go/libraries/doltcore/sqle/sqlreplace_test.go | 4 +- go/libraries/doltcore/sqle/sqlselect_test.go | 4 +- go/libraries/doltcore/sqle/sqlupdate_test.go | 4 +- 10 files changed, 195 insertions(+), 55 deletions(-) diff --git a/go/cmd/dolt/commands/diff.go b/go/cmd/dolt/commands/diff.go index e558ebf04ba..2ffd637c2e6 100644 --- a/go/cmd/dolt/commands/diff.go +++ b/go/cmd/dolt/commands/diff.go @@ -386,15 +386,13 @@ func getTableNamesAtRef(queryist cli.Queryist, sqlCtx *sql.Context, ref string) return nil, fmt.Errorf("error interpolating query: %w", err) } result, err := GetRowsForSql(queryist, sqlCtx, interpolatedQuery) - if err == nil { - if len(result) > 0 { - tableNames[sysTable] = true - } - } else if isTableNotFoundError(err) { - continue - } else { + if err != nil { return nil, fmt.Errorf("error getting system table %s: %w", sysTable, err) } + + if len(result) > 0 { + tableNames[sysTable] = true + } } return tableNames, nil diff --git a/go/libraries/doltcore/sqle/database.go b/go/libraries/doltcore/sqle/database.go index a3c66ea9860..7c74770fea3 100644 --- a/go/libraries/doltcore/sqle/database.go +++ b/go/libraries/doltcore/sqle/database.go @@ -510,6 +510,17 @@ func (db Database) getTableInsensitive(ctx *sql.Context, head *doltdb.Commit, ds writeTable := backingTable.(*WritableDoltTable) dt, found = NewProceduresTable(writeTable), true } + case doltdb.SchemasTableName: + backingTable, _, err := db.getTable(ctx, root, doltdb.SchemasTableName) + if err != nil { + return nil, false, err + } + if backingTable == nil { + dt, found = NewEmptySchemaTable(), true + } else { + writeTable := backingTable.(*WritableDoltTable) + dt, found = NewSchemaTable(writeTable), true + } } if found { @@ -1418,16 +1429,22 @@ func (db Database) GetViewDefinition(ctx *sql.Context, viewName string) (sql.Vie } } - tbl, ok, err := db.GetTableInsensitive(ctx, doltdb.SchemasTableName) + tbl, _, err := db.GetTableInsensitive(ctx, doltdb.SchemasTableName) if err != nil { return sql.ViewDefinition{}, false, err } + + wrapper, ok := tbl.(*SchemaTable) if !ok { + return sql.ViewDefinition{}, false, fmt.Errorf("expected a SchemaTable, but found %T", tbl) + } + + if wrapper.backingTable == nil { dbState.SessionCache().CacheViews(key, nil, db.schemaName) return sql.ViewDefinition{}, false, nil } - views, viewDef, found, err := getViewDefinitionFromSchemaFragmentsOfView(ctx, tbl.(*WritableDoltTable), viewName) + views, viewDef, found, err := getViewDefinitionFromSchemaFragmentsOfView(ctx, wrapper.backingTable, viewName) if err != nil { return sql.ViewDefinition{}, false, err } @@ -1475,15 +1492,20 @@ func getViewDefinitionFromSchemaFragmentsOfView(ctx *sql.Context, tbl *WritableD // AllViews implements sql.ViewDatabase func (db Database) AllViews(ctx *sql.Context) ([]sql.ViewDefinition, error) { - tbl, ok, err := db.GetTableInsensitive(ctx, doltdb.SchemasTableName) + tbl, _, err := db.GetTableInsensitive(ctx, doltdb.SchemasTableName) if err != nil { return nil, err } + + wrapper, ok := tbl.(*SchemaTable) if !ok { + return nil, fmt.Errorf("expected a SchemaTable, but found %T", tbl) + } + if wrapper.backingTable == nil { return nil, nil } - views, _, _, err := getViewDefinitionFromSchemaFragmentsOfView(ctx, tbl.(*WritableDoltTable), "") + views, _, _, err := getViewDefinitionFromSchemaFragmentsOfView(ctx, wrapper.backingTable, "") if err != nil { return nil, err } @@ -1535,15 +1557,21 @@ func (db Database) GetTriggers(ctx *sql.Context) ([]sql.TriggerDefinition, error dbState.SessionCache().CacheTriggers(key, triggers, db.schemaName) }() - tbl, ok, err := db.GetTableInsensitive(ctx, doltdb.SchemasTableName) + tbl, _, err := db.GetTableInsensitive(ctx, doltdb.SchemasTableName) if err != nil { return nil, err } + + wrapper, ok := tbl.(*SchemaTable) if !ok { + return nil, fmt.Errorf("expected a SchemaTable, but found %T", tbl) + } + + if wrapper.backingTable == nil { return nil, nil } - frags, err := getSchemaFragmentsOfType(ctx, tbl.(*WritableDoltTable), triggerFragment) + frags, err := getSchemaFragmentsOfType(ctx, wrapper.backingTable, triggerFragment) if err != nil { return nil, err } @@ -1582,15 +1610,20 @@ func (db Database) DropTrigger(ctx *sql.Context, name string) error { // GetEvent implements sql.EventDatabase. func (db Database) GetEvent(ctx *sql.Context, name string) (sql.EventDefinition, bool, error) { - tbl, ok, err := db.GetTableInsensitive(ctx, doltdb.SchemasTableName) + tbl, _, err := db.GetTableInsensitive(ctx, doltdb.SchemasTableName) if err != nil { return sql.EventDefinition{}, false, err } + + wrapper, ok := tbl.(*SchemaTable) if !ok { + return sql.EventDefinition{}, false, fmt.Errorf("expected a SchemaTable, but found %T", tbl) + } + if wrapper.backingTable == nil { return sql.EventDefinition{}, false, nil } - frags, err := getSchemaFragmentsOfType(ctx, tbl.(*WritableDoltTable), eventFragment) + frags, err := getSchemaFragmentsOfType(ctx, wrapper.backingTable, eventFragment) if err != nil { return sql.EventDefinition{}, false, err } @@ -1609,16 +1642,21 @@ func (db Database) GetEvent(ctx *sql.Context, name string) (sql.EventDefinition, // GetEvents implements sql.EventDatabase. func (db Database) GetEvents(ctx *sql.Context) (events []sql.EventDefinition, token interface{}, err error) { - tbl, ok, err := db.GetTableInsensitive(ctx, doltdb.SchemasTableName) + tbl, _, err := db.GetTableInsensitive(ctx, doltdb.SchemasTableName) if err != nil { return nil, nil, err } + wrapper, ok := tbl.(*SchemaTable) if !ok { + return nil, nil, fmt.Errorf("expected a SchemaTable, but found %T", tbl) + } + + if wrapper.backingTable == nil { // If the dolt_schemas table doesn't exist, it's not an error, just no events return nil, nil, nil } - frags, err := getSchemaFragmentsOfType(ctx, tbl.(*WritableDoltTable), eventFragment) + frags, err := getSchemaFragmentsOfType(ctx, wrapper.backingTable, eventFragment) if err != nil { return nil, nil, err } @@ -1855,15 +1893,20 @@ func (db Database) dropFragFromSchemasTable(ctx *sql.Context, fragType, name str db.schemaName = schemaName } - stbl, found, err := db.GetTableInsensitive(ctx, doltdb.SchemasTableName) + stbl, _, err := db.GetTableInsensitive(ctx, doltdb.SchemasTableName) if err != nil { return err } - if !found { + + swrapper, ok := stbl.(*SchemaTable) + if !ok { + return fmt.Errorf("expected a SchemaTable, but found %T", stbl) + } + if swrapper.backingTable == nil { return missingErr } - tbl := stbl.(*WritableDoltTable) + tbl := swrapper.backingTable row, exists, err := fragFromSchemasTable(ctx, tbl, fragType, name) if err != nil { return err diff --git a/go/libraries/doltcore/sqle/schema_table.go b/go/libraries/doltcore/sqle/schema_table.go index 46d4544a751..f4e4e7b7f3d 100644 --- a/go/libraries/doltcore/sqle/schema_table.go +++ b/go/libraries/doltcore/sqle/schema_table.go @@ -20,6 +20,9 @@ import ( "strings" "time" + "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dtables" + "github.com/dolthub/dolt/go/libraries/doltcore/sqle/index" + "github.com/dolthub/dolt/go/libraries/doltcore/sqle/sqlutil" "github.com/dolthub/go-mysql-server/sql" gmstypes "github.com/dolthub/go-mysql-server/sql/types" "github.com/dolthub/vitess/go/vt/proto/query" @@ -40,6 +43,72 @@ type Extra struct { CreatedAt int64 } +type SchemaTable struct { + backingTable *WritableDoltTable +} + +func (st *SchemaTable) Name() string { + return doltdb.SchemasTableName +} + +func (st *SchemaTable) String() string { + return doltdb.SchemasTableName +} + +func (st *SchemaTable) Schema() sql.Schema { + return SchemaTableSqlSchema().Schema +} + +func (st *SchemaTable) Collation() sql.CollationID { + return sql.Collation_Default +} + +func (st *SchemaTable) Partitions(ctx *sql.Context) (sql.PartitionIter, error) { + if st.backingTable == nil { + return index.SinglePartitionIterFromNomsMap(nil), nil + } + return st.backingTable.Partitions(ctx) +} + +func (st *SchemaTable) PartitionRows(ctx *sql.Context, partition sql.Partition) (sql.RowIter, error) { + if st.backingTable == nil { + return sql.RowsToRowIter(), nil + } + return st.backingTable.PartitionRows(ctx, partition) +} + +func (st *SchemaTable) LockedToRoot(ctx *sql.Context, root doltdb.RootValue) (sql.IndexAddressableTable, error) { + if st.backingTable == nil { + return st, nil + } + return st.backingTable.LockedToRoot(ctx, root) +} + +func (st *SchemaTable) IndexedAccess(lookup sql.IndexLookup) sql.IndexedTable { + // Never reached. Interface required for LockedToRoot to be implemented. + panic("Unreachable") +} + +func (st *SchemaTable) GetIndexes(ctx *sql.Context) ([]sql.Index, error) { + return nil, nil +} + +func (st *SchemaTable) PreciseMatch() bool { + return true +} + +var _ sql.Table = (*SchemaTable)(nil) +var _ dtables.VersionableTable = (*SchemaTable)(nil) +var _ sql.IndexAddressableTable = (*SchemaTable)(nil) + +func SchemaTableSqlSchema() sql.PrimaryKeySchema { + sqlSchema, err := sqlutil.FromDoltSchema("", doltdb.SchemasTableName, SchemaTableSchema()) + if err != nil { + panic(err) // should never happen + } + return sqlSchema +} + func mustNewColWithTypeInfo(name string, tag uint64, typeInfo typeinfo.TypeInfo, partOfPK bool, defaultVal string, autoIncrement bool, comment string, constraints ...schema.ColConstraint) schema.Column { col, err := schema.NewColumnWithTypeInfo(name, tag, typeInfo, partOfPK, defaultVal, autoIncrement, comment, constraints...) if err != nil { @@ -57,15 +126,25 @@ func mustCreateStringType(baseType query.Type, length int64, collation sql.Colla } // dolt_schemas columns -var schemasTableCols = schema.NewColCollection( - mustNewColWithTypeInfo(doltdb.SchemasTablesTypeCol, schema.DoltSchemasTypeTag, typeinfo.CreateVarStringTypeFromSqlType(mustCreateStringType(query.Type_VARCHAR, 64, sql.Collation_utf8mb4_0900_ai_ci)), true, "", false, ""), - mustNewColWithTypeInfo(doltdb.SchemasTablesNameCol, schema.DoltSchemasNameTag, typeinfo.CreateVarStringTypeFromSqlType(mustCreateStringType(query.Type_VARCHAR, 64, sql.Collation_utf8mb4_0900_ai_ci)), true, "", false, ""), - mustNewColWithTypeInfo(doltdb.SchemasTablesFragmentCol, schema.DoltSchemasFragmentTag, typeinfo.CreateVarStringTypeFromSqlType(gmstypes.LongText), false, "", false, ""), - mustNewColWithTypeInfo(doltdb.SchemasTablesExtraCol, schema.DoltSchemasExtraTag, typeinfo.JSONType, false, "", false, ""), - mustNewColWithTypeInfo(doltdb.SchemasTablesSqlModeCol, schema.DoltSchemasSqlModeTag, typeinfo.CreateVarStringTypeFromSqlType(mustCreateStringType(query.Type_VARCHAR, 256, sql.Collation_utf8mb4_0900_ai_ci)), false, "", false, ""), -) +func SchemaTableSchema() schema.Schema { + var schemasTableCols = schema.NewColCollection( + mustNewColWithTypeInfo(doltdb.SchemasTablesTypeCol, schema.DoltSchemasTypeTag, typeinfo.CreateVarStringTypeFromSqlType(mustCreateStringType(query.Type_VARCHAR, 64, sql.Collation_utf8mb4_0900_ai_ci)), true, "", false, ""), + mustNewColWithTypeInfo(doltdb.SchemasTablesNameCol, schema.DoltSchemasNameTag, typeinfo.CreateVarStringTypeFromSqlType(mustCreateStringType(query.Type_VARCHAR, 64, sql.Collation_utf8mb4_0900_ai_ci)), true, "", false, ""), + mustNewColWithTypeInfo(doltdb.SchemasTablesFragmentCol, schema.DoltSchemasFragmentTag, typeinfo.CreateVarStringTypeFromSqlType(gmstypes.LongText), false, "", false, ""), + mustNewColWithTypeInfo(doltdb.SchemasTablesExtraCol, schema.DoltSchemasExtraTag, typeinfo.JSONType, false, "", false, ""), + mustNewColWithTypeInfo(doltdb.SchemasTablesSqlModeCol, schema.DoltSchemasSqlModeTag, typeinfo.CreateVarStringTypeFromSqlType(mustCreateStringType(query.Type_VARCHAR, 256, sql.Collation_utf8mb4_0900_ai_ci)), false, "", false, ""), + ) + + return schema.MustSchemaFromCols(schemasTableCols) +} + +func NewEmptySchemaTable() sql.Table { + return &SchemaTable{} +} -var schemaTableSchema = schema.MustSchemaFromCols(schemasTableCols) +func NewSchemaTable(backingTable *WritableDoltTable) sql.Table { + return &SchemaTable{backingTable: backingTable} +} // getOrCreateDoltSchemasTable returns the `dolt_schemas` table in `db`, creating it if it does not already exist. // Also migrates data to the correct format if necessary. @@ -88,15 +167,20 @@ func getOrCreateDoltSchemasTable(ctx *sql.Context, db Database) (retTbl *Writabl } } - tbl, found, err := db.GetTableInsensitive(ctx, doltdb.SchemasTableName) + tbl, _, err := db.GetTableInsensitive(ctx, doltdb.SchemasTableName) if err != nil { return nil, err } - if found { - schemasTable := tbl.(*WritableDoltTable) + wrapper, ok := tbl.(*SchemaTable) + if !ok { + return nil, fmt.Errorf("expected a SchemaTable, but found %T", tbl) + } + + if wrapper.backingTable != nil { + schemasTable := wrapper.backingTable // Old schemas table contains the `id` column or is missing an `extra` column. - if tbl.Schema().Contains(doltdb.SchemasTablesIdCol, doltdb.SchemasTableName) || !tbl.Schema().Contains(doltdb.SchemasTablesExtraCol, doltdb.SchemasTableName) { + if schemasTable.Schema().Contains(doltdb.SchemasTablesIdCol, doltdb.SchemasTableName) || !schemasTable.Schema().Contains(doltdb.SchemasTablesExtraCol, doltdb.SchemasTableName) { return migrateOldSchemasTableToNew(ctx, db, schemasTable) } else { return schemasTable, nil @@ -104,19 +188,24 @@ func getOrCreateDoltSchemasTable(ctx *sql.Context, db Database) (retTbl *Writabl } // Create new empty table - err = db.createDoltTable(ctx, doltdb.SchemasTableName, schemaName, root, schemaTableSchema) + err = db.createDoltTable(ctx, doltdb.SchemasTableName, schemaName, root, SchemaTableSchema()) if err != nil { return nil, err } - tbl, found, err = db.GetTableInsensitive(ctx, doltdb.SchemasTableName) + tbl, _, err = db.GetTableInsensitive(ctx, doltdb.SchemasTableName) if err != nil { return nil, err } - if !found { + + wrapper, ok = tbl.(*SchemaTable) + if !ok { + return nil, fmt.Errorf("expected a SchemaTable, but found %T", tbl) + } + if wrapper.backingTable == nil { return nil, sql.ErrTableNotFound.New(doltdb.SchemasTableName) } - return tbl.(*WritableDoltTable), nil + return wrapper.backingTable, nil } func migrateOldSchemasTableToNew(ctx *sql.Context, db Database, schemasTable *WritableDoltTable) (newTable *WritableDoltTable, rerr error) { @@ -151,7 +240,7 @@ func migrateOldSchemasTableToNew(ctx *sql.Context, db Database, schemasTable *Wr return nil, err } - newRow := make(sql.Row, schemasTableCols.Size()) + newRow := make(sql.Row, SchemaTableSchema().GetAllCols().Size()) newRow[0] = sqlRow[typeIdx] newRow[1] = sqlRow[nameIdx] newRow[2] = sqlRow[fragmentIdx] @@ -175,20 +264,26 @@ func migrateOldSchemasTableToNew(ctx *sql.Context, db Database, schemasTable *Wr return nil, err } - err = db.createDoltTable(ctx, doltdb.SchemasTableName, doltdb.DefaultSchemaName, root, schemaTableSchema) + err = db.createDoltTable(ctx, doltdb.SchemasTableName, doltdb.DefaultSchemaName, root, SchemaTableSchema()) if err != nil { return nil, err } - tbl, found, err := db.GetTableInsensitive(ctx, doltdb.SchemasTableName) + tbl, _, err := db.GetTableInsensitive(ctx, doltdb.SchemasTableName) if err != nil { return nil, err } - if !found { + + wrapper, ok := tbl.(*SchemaTable) + if !ok { + return nil, fmt.Errorf("expected a SchemaTable, but found %T", tbl) + } + + if wrapper.backingTable == nil { return nil, sql.ErrTableNotFound.New(doltdb.SchemasTableName) } - inserter := tbl.(*WritableDoltTable).Inserter(ctx) + inserter := wrapper.backingTable.Inserter(ctx) for _, row := range newRows { err = inserter.Insert(ctx, row) if err != nil { @@ -201,7 +296,7 @@ func migrateOldSchemasTableToNew(ctx *sql.Context, db Database, schemasTable *Wr return nil, err } - return tbl.(*WritableDoltTable), nil + return wrapper.backingTable, nil } // fragFromSchemasTable returns the row with the given schema fragment if it exists. diff --git a/go/libraries/doltcore/sqle/schema_table_test.go b/go/libraries/doltcore/sqle/schema_table_test.go index 8272429ff88..aaedba3ae80 100644 --- a/go/libraries/doltcore/sqle/schema_table_test.go +++ b/go/libraries/doltcore/sqle/schema_table_test.go @@ -52,7 +52,11 @@ func TestSchemaTableMigrationOriginal(t *testing.T) { require.NoError(t, err) require.True(t, found) - inserter := sqlTbl.(*WritableDoltTable).Inserter(ctx) + wrapper, ok := sqlTbl.(*SchemaTable) + require.True(t, ok) + require.NotNil(t, wrapper.backingTable) + + inserter := wrapper.backingTable.Inserter(ctx) err = inserter.Insert(ctx, sql.Row{"view", "view1", "SELECT v1 FROM test;"}) require.NoError(t, err) err = inserter.Insert(ctx, sql.Row{"view", "view2", "SELECT v2 FROM test;"}) diff --git a/go/libraries/doltcore/sqle/sqlddl_test.go b/go/libraries/doltcore/sqle/sqlddl_test.go index 5a9b5115c17..48732906fa0 100644 --- a/go/libraries/doltcore/sqle/sqlddl_test.go +++ b/go/libraries/doltcore/sqle/sqlddl_test.go @@ -819,7 +819,7 @@ func TestAlterSystemTables(t *testing.T) { "INSERT INTO dolt_docs VALUES ('LICENSE.md','A license')") CreateTestTable(t, dEnv, doltdb.DoltQueryCatalogTableName, dtables.DoltQueryCatalogSchema, "INSERT INTO dolt_query_catalog VALUES ('abc123', 1, 'example', 'select 2+2 from dual', 'description')") - CreateTestTable(t, dEnv, doltdb.SchemasTableName, schemaTableSchema, + CreateTestTable(t, dEnv, doltdb.SchemasTableName, SchemaTableSchema(), "INSERT INTO dolt_schemas (type, name, fragment) VALUES ('view', 'name', 'create view name as select 2+2 from dual')") ExecuteSetupSQL(context.Background(), ` CREATE PROCEDURE simple_proc2() SELECT 1+1; diff --git a/go/libraries/doltcore/sqle/sqldelete_test.go b/go/libraries/doltcore/sqle/sqldelete_test.go index 9ca25c97bed..9aab4d942eb 100644 --- a/go/libraries/doltcore/sqle/sqldelete_test.go +++ b/go/libraries/doltcore/sqle/sqldelete_test.go @@ -207,12 +207,12 @@ var systemTableDeleteTests = []DeleteTest{ }, { Name: "delete dolt_schemas", - AdditionalSetup: CreateTableFn(doltdb.SchemasTableName, schemaTableSchema, + AdditionalSetup: CreateTableFn(doltdb.SchemasTableName, SchemaTableSchema(), "INSERT INTO dolt_schemas (type, name, fragment) VALUES ('view', 'name', 'create view name as select 2+2 from dual')"), DeleteQuery: "delete from dolt_schemas", SelectQuery: "select * from dolt_schemas", ExpectedRows: ToSqlRows(dtables.DoltQueryCatalogSchema), - ExpectedSchema: schemaTableSchema, + ExpectedSchema: SchemaTableSchema(), }, } diff --git a/go/libraries/doltcore/sqle/sqlinsert_test.go b/go/libraries/doltcore/sqle/sqlinsert_test.go index 2dc980a5ae9..6dee3273d08 100644 --- a/go/libraries/doltcore/sqle/sqlinsert_test.go +++ b/go/libraries/doltcore/sqle/sqlinsert_test.go @@ -397,13 +397,13 @@ var systemTableInsertTests = []InsertTest{ }, { Name: "insert into dolt_schemas", - AdditionalSetup: CreateTableFn(doltdb.SchemasTableName, schemaTableSchema, ""), + AdditionalSetup: CreateTableFn(doltdb.SchemasTableName, SchemaTableSchema(), ""), InsertQuery: "insert into dolt_schemas (type, name, fragment) values ('view', 'name', 'create view name as select 2+2 from dual')", SelectQuery: "select * from dolt_schemas ORDER BY name", - ExpectedRows: ToSqlRows(CompressSchema(schemaTableSchema), + ExpectedRows: ToSqlRows(CompressSchema(SchemaTableSchema()), NewRow(types.String("view"), types.String("name"), types.String("create view name as select 2+2 from dual")), ), - ExpectedSchema: CompressSchema(schemaTableSchema), + ExpectedSchema: CompressSchema(SchemaTableSchema()), }, } diff --git a/go/libraries/doltcore/sqle/sqlreplace_test.go b/go/libraries/doltcore/sqle/sqlreplace_test.go index a5469af2f5c..d3501e11be2 100644 --- a/go/libraries/doltcore/sqle/sqlreplace_test.go +++ b/go/libraries/doltcore/sqle/sqlreplace_test.go @@ -272,12 +272,12 @@ var systemTableReplaceTests = []ReplaceTest{ }, { Name: "replace into dolt_schemas", - AdditionalSetup: CreateTableFn(doltdb.SchemasTableName, schemaTableSchema, + AdditionalSetup: CreateTableFn(doltdb.SchemasTableName, SchemaTableSchema(), "INSERT INTO dolt_schemas VALUES ('view', 'name', 'create view name as select 2+2 from dual', NULL, NULL)"), ReplaceQuery: "replace into dolt_schemas (type, name, fragment) values ('view', 'name', 'create view name as select 1+1 from dual')", SelectQuery: "select type, name, fragment, extra, sql_mode from dolt_schemas", ExpectedRows: []sql.Row{{"view", "name", "create view name as select 1+1 from dual", nil, nil}}, - ExpectedSchema: CompressSchema(schemaTableSchema), + ExpectedSchema: CompressSchema(SchemaTableSchema()), }, } diff --git a/go/libraries/doltcore/sqle/sqlselect_test.go b/go/libraries/doltcore/sqle/sqlselect_test.go index 3c7bd541a9f..c31eaad3cca 100644 --- a/go/libraries/doltcore/sqle/sqlselect_test.go +++ b/go/libraries/doltcore/sqle/sqlselect_test.go @@ -1308,11 +1308,11 @@ var systemTableSelectTests = []SelectTest{ }, { Name: "select from dolt_schemas", - AdditionalSetup: CreateTableFn(doltdb.SchemasTableName, schemaTableSchema, + AdditionalSetup: CreateTableFn(doltdb.SchemasTableName, SchemaTableSchema(), `INSERT INTO dolt_schemas VALUES ('view', 'name', 'create view name as select 2+2 from dual', NULL, NULL)`), Query: "select * from dolt_schemas", ExpectedRows: []sql.Row{{"view", "name", "create view name as select 2+2 from dual", nil, nil}}, - ExpectedSchema: CompressSchema(schemaTableSchema), + ExpectedSchema: CompressSchema(SchemaTableSchema()), }, } diff --git a/go/libraries/doltcore/sqle/sqlupdate_test.go b/go/libraries/doltcore/sqle/sqlupdate_test.go index 8d92735b6e0..c1cf6db5a17 100644 --- a/go/libraries/doltcore/sqle/sqlupdate_test.go +++ b/go/libraries/doltcore/sqle/sqlupdate_test.go @@ -378,12 +378,12 @@ var systemTableUpdateTests = []UpdateTest{ }, { Name: "update dolt_schemas", - AdditionalSetup: CreateTableFn(doltdb.SchemasTableName, schemaTableSchema, + AdditionalSetup: CreateTableFn(doltdb.SchemasTableName, SchemaTableSchema(), `INSERT INTO dolt_schemas VALUES ('view', 'name', 'create view name as select 2+2 from dual', NULL, NULL)`), UpdateQuery: "update dolt_schemas set type = 'not a view'", SelectQuery: "select * from dolt_schemas", ExpectedRows: []sql.Row{{"not a view", "name", "create view name as select 2+2 from dual", nil, nil}}, - ExpectedSchema: CompressSchema(schemaTableSchema), + ExpectedSchema: CompressSchema(SchemaTableSchema()), }, } From cdbbfba7bce2007d331cad62ebb7887d3c17162e Mon Sep 17 00:00:00 2001 From: macneale4 Date: Thu, 11 Jul 2024 22:23:32 +0000 Subject: [PATCH 05/18] [ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/update.sh --- go/libraries/doltcore/sqle/schema_table.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/go/libraries/doltcore/sqle/schema_table.go b/go/libraries/doltcore/sqle/schema_table.go index f4e4e7b7f3d..ad40d1cc8f2 100644 --- a/go/libraries/doltcore/sqle/schema_table.go +++ b/go/libraries/doltcore/sqle/schema_table.go @@ -20,9 +20,6 @@ import ( "strings" "time" - "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dtables" - "github.com/dolthub/dolt/go/libraries/doltcore/sqle/index" - "github.com/dolthub/dolt/go/libraries/doltcore/sqle/sqlutil" "github.com/dolthub/go-mysql-server/sql" gmstypes "github.com/dolthub/go-mysql-server/sql/types" "github.com/dolthub/vitess/go/vt/proto/query" @@ -30,7 +27,10 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" "github.com/dolthub/dolt/go/libraries/doltcore/schema" "github.com/dolthub/dolt/go/libraries/doltcore/schema/typeinfo" + "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dtables" + "github.com/dolthub/dolt/go/libraries/doltcore/sqle/index" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/resolve" + "github.com/dolthub/dolt/go/libraries/doltcore/sqle/sqlutil" ) const ( From d0a715180d7feb26708b7da1da4d1d58d4144359 Mon Sep 17 00:00:00 2001 From: Neil Macneale IV Date: Thu, 11 Jul 2024 15:53:08 -0700 Subject: [PATCH 06/18] Allow for the 'deletion' of fake tables --- go/libraries/doltcore/sqle/database.go | 7 +++++++ go/libraries/doltcore/sqle/procedures_table.go | 5 +++++ go/libraries/doltcore/sqle/schema_table.go | 5 +++++ go/libraries/doltcore/sqle/tables.go | 8 ++++++++ 4 files changed, 25 insertions(+) diff --git a/go/libraries/doltcore/sqle/database.go b/go/libraries/doltcore/sqle/database.go index 7c74770fea3..703f79cfb85 100644 --- a/go/libraries/doltcore/sqle/database.go +++ b/go/libraries/doltcore/sqle/database.go @@ -1940,6 +1940,13 @@ func (db Database) dropTableIfEmpty(ctx *sql.Context, tableName string) error { return nil } + if wrapped, ok := stbl.(WritableDoltTableWrapper); ok { + stbl = wrapped.UnWrap() + if stbl == nil { + return nil + } + } + table, err := stbl.(*WritableDoltTable).DoltTable.DoltTable(ctx) if err != nil { return err diff --git a/go/libraries/doltcore/sqle/procedures_table.go b/go/libraries/doltcore/sqle/procedures_table.go index 5ac0614792c..89e49ff83cf 100644 --- a/go/libraries/doltcore/sqle/procedures_table.go +++ b/go/libraries/doltcore/sqle/procedures_table.go @@ -102,6 +102,10 @@ func (pt *ProceduresTable) PreciseMatch() bool { return true } +func (pt *ProceduresTable) UnWrap() *WritableDoltTable { + return pt.backingTable +} + func NewProceduresTable(backing *WritableDoltTable) sql.Table { return &ProceduresTable{backingTable: backing} } @@ -113,6 +117,7 @@ func NewEmptyProceduresTable() sql.Table { var _ sql.Table = (*ProceduresTable)(nil) var _ dtables.VersionableTable = (*ProceduresTable)(nil) var _ sql.IndexAddressableTable = (*ProceduresTable)(nil) +var _ WritableDoltTableWrapper = (*ProceduresTable)(nil) // The fixed SQL schema for the `dolt_procedures` table. func ProceduresTableSqlSchema() sql.PrimaryKeySchema { diff --git a/go/libraries/doltcore/sqle/schema_table.go b/go/libraries/doltcore/sqle/schema_table.go index ad40d1cc8f2..af125d8a934 100644 --- a/go/libraries/doltcore/sqle/schema_table.go +++ b/go/libraries/doltcore/sqle/schema_table.go @@ -97,9 +97,14 @@ func (st *SchemaTable) PreciseMatch() bool { return true } +func (st *SchemaTable) UnWrap() *WritableDoltTable { + return st.backingTable +} + var _ sql.Table = (*SchemaTable)(nil) var _ dtables.VersionableTable = (*SchemaTable)(nil) var _ sql.IndexAddressableTable = (*SchemaTable)(nil) +var _ WritableDoltTableWrapper = (*SchemaTable)(nil) func SchemaTableSqlSchema() sql.PrimaryKeySchema { sqlSchema, err := sqlutil.FromDoltSchema("", doltdb.SchemasTableName, SchemaTableSchema()) diff --git a/go/libraries/doltcore/sqle/tables.go b/go/libraries/doltcore/sqle/tables.go index d56e0149fbd..8fbccd4f6a2 100644 --- a/go/libraries/doltcore/sqle/tables.go +++ b/go/libraries/doltcore/sqle/tables.go @@ -536,6 +536,14 @@ type WritableDoltTable struct { var _ doltTableInterface = (*WritableDoltTable)(nil) +// WritableDoltTableWrapper is an interface that allows a table to be returned as an sql.Table, but actually be a wrapped +// fake table. Specifically, databases.getTableInsensitive will returns an sql.Table, and there are cases where we +// want to return a table that hasn't been materialized yet. +type WritableDoltTableWrapper interface { + // Return the underlying WritableDoltTable, nil returns are expected when the wrapped table hasn't been materialized + UnWrap() *WritableDoltTable +} + // Internal interface for declaring the interfaces that writable dolt tables are expected to implement type doltTableInterface interface { sql.UpdatableTable From e99bcda5d2771ebf881ab189fe425d59ef5577ca Mon Sep 17 00:00:00 2001 From: Neil Macneale IV Date: Thu, 11 Jul 2024 16:06:01 -0700 Subject: [PATCH 07/18] fixed schema test --- go/libraries/doltcore/sqle/schema_table_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/go/libraries/doltcore/sqle/schema_table_test.go b/go/libraries/doltcore/sqle/schema_table_test.go index aaedba3ae80..3af0d95f778 100644 --- a/go/libraries/doltcore/sqle/schema_table_test.go +++ b/go/libraries/doltcore/sqle/schema_table_test.go @@ -115,7 +115,11 @@ func TestSchemaTableMigrationV1(t *testing.T) { require.NoError(t, err) require.True(t, found) - inserter := sqlTbl.(*WritableDoltTable).Inserter(ctx) + wrapper, ok := sqlTbl.(*SchemaTable) + require.True(t, ok) + require.NotNil(t, wrapper.backingTable) + + inserter := wrapper.UnWrap().Inserter(ctx) // JSON string has no spaces because our various JSON libraries don't agree on how to marshall it err = inserter.Insert(ctx, sql.Row{"view", "view1", "SELECT v1 FROM test;", 1, `{"extra":"data"}`}) require.NoError(t, err) From 626fc0184024549c5d2173a10b3c5974e984838f Mon Sep 17 00:00:00 2001 From: Neil Macneale IV Date: Fri, 12 Jul 2024 09:30:10 -0700 Subject: [PATCH 08/18] Remove the dolt_schema migration code and test --- go/libraries/doltcore/doltdb/system_table.go | 3 - go/libraries/doltcore/sqle/schema_table.go | 97 +--------------- .../doltcore/sqle/schema_table_test.go | 109 +----------------- .../bats/helper/old_dolt_schemas/config.json | 1 - .../noms/4bujnjuatdli0klda7ucot0tl836jujt | Bin 57 -> 0 bytes .../noms/63c0dojobm5j7arp99hiq78hj1rtp7m5 | Bin 775 -> 0 bytes .../bats/helper/old_dolt_schemas/noms/LOCK | 0 .../noms/aum9p7826iqaeq0pjtqlcn56hoj5nhht | Bin 529 -> 0 bytes .../noms/chghnlvugtiui6khfo29cpfnfr3166qc | Bin 211 -> 0 bytes .../noms/d0b8ibrrjvvv3f0mtd7hathtjq2vhig9 | Bin 211 -> 0 bytes .../noms/erturgf7to0f6r8prlbllflbshg5li4n | Bin 126 -> 0 bytes .../helper/old_dolt_schemas/noms/manifest | 1 - .../noms/oik6v50p7q6as4t7qg7f1i06iomt9vn9 | Bin 375 -> 0 bytes .../noms/oktjimrpqi8m9dbpt3c7ikoaolhgtr2n | Bin 1562 -> 0 bytes .../noms/qe0mct6rf8puhdnqecm185p63dacaqrt | Bin 353 -> 0 bytes .../noms/ro000svvgga0k7hatg7fta9j8ismtc3b | Bin 211 -> 0 bytes .../noms/u9c6ttq6ddrgqdfbod9pg2vv79ab6crt | Bin 587 -> 0 bytes .../helper/old_dolt_schemas/repo_state.json | 8 -- integration-tests/bats/triggers.bats | 48 +------- 19 files changed, 10 insertions(+), 257 deletions(-) delete mode 100644 integration-tests/bats/helper/old_dolt_schemas/config.json delete mode 100644 integration-tests/bats/helper/old_dolt_schemas/noms/4bujnjuatdli0klda7ucot0tl836jujt delete mode 100644 integration-tests/bats/helper/old_dolt_schemas/noms/63c0dojobm5j7arp99hiq78hj1rtp7m5 delete mode 100644 integration-tests/bats/helper/old_dolt_schemas/noms/LOCK delete mode 100644 integration-tests/bats/helper/old_dolt_schemas/noms/aum9p7826iqaeq0pjtqlcn56hoj5nhht delete mode 100644 integration-tests/bats/helper/old_dolt_schemas/noms/chghnlvugtiui6khfo29cpfnfr3166qc delete mode 100644 integration-tests/bats/helper/old_dolt_schemas/noms/d0b8ibrrjvvv3f0mtd7hathtjq2vhig9 delete mode 100644 integration-tests/bats/helper/old_dolt_schemas/noms/erturgf7to0f6r8prlbllflbshg5li4n delete mode 100644 integration-tests/bats/helper/old_dolt_schemas/noms/manifest delete mode 100644 integration-tests/bats/helper/old_dolt_schemas/noms/oik6v50p7q6as4t7qg7f1i06iomt9vn9 delete mode 100644 integration-tests/bats/helper/old_dolt_schemas/noms/oktjimrpqi8m9dbpt3c7ikoaolhgtr2n delete mode 100644 integration-tests/bats/helper/old_dolt_schemas/noms/qe0mct6rf8puhdnqecm185p63dacaqrt delete mode 100644 integration-tests/bats/helper/old_dolt_schemas/noms/ro000svvgga0k7hatg7fta9j8ismtc3b delete mode 100644 integration-tests/bats/helper/old_dolt_schemas/noms/u9c6ttq6ddrgqdfbod9pg2vv79ab6crt delete mode 100644 integration-tests/bats/helper/old_dolt_schemas/repo_state.json diff --git a/go/libraries/doltcore/doltdb/system_table.go b/go/libraries/doltcore/doltdb/system_table.go index cc60da73dc5..87d59e57b29 100644 --- a/go/libraries/doltcore/doltdb/system_table.go +++ b/go/libraries/doltcore/doltdb/system_table.go @@ -236,9 +236,6 @@ const ( const ( // SchemasTableName is the name of the dolt schema fragment table SchemasTableName = "dolt_schemas" - // SchemasTablesIdCol is an incrementing integer that represents the insertion index. - // Deprecated: This column is no longer used and will be removed in a future release. - SchemasTablesIdCol = "id" // SchemasTablesTypeCol is the name of the column that stores the type of a schema fragment in the dolt_schemas table SchemasTablesTypeCol = "type" // SchemasTablesNameCol The name of the column that stores the name of a schema fragment in the dolt_schemas table diff --git a/go/libraries/doltcore/sqle/schema_table.go b/go/libraries/doltcore/sqle/schema_table.go index af125d8a934..147f9629a78 100644 --- a/go/libraries/doltcore/sqle/schema_table.go +++ b/go/libraries/doltcore/sqle/schema_table.go @@ -184,9 +184,9 @@ func getOrCreateDoltSchemasTable(ctx *sql.Context, db Database) (retTbl *Writabl if wrapper.backingTable != nil { schemasTable := wrapper.backingTable - // Old schemas table contains the `id` column or is missing an `extra` column. - if schemasTable.Schema().Contains(doltdb.SchemasTablesIdCol, doltdb.SchemasTableName) || !schemasTable.Schema().Contains(doltdb.SchemasTablesExtraCol, doltdb.SchemasTableName) { - return migrateOldSchemasTableToNew(ctx, db, schemasTable) + // Old schemas are missing the `extra` column. Very ancient. Provide error message and bail. + if !schemasTable.Schema().Contains(doltdb.SchemasTablesExtraCol, doltdb.SchemasTableName) { + return nil, fmt.Errorf("cannot migrate dolt_schemas table from v0.19.1 or earlier") } else { return schemasTable, nil } @@ -213,97 +213,6 @@ func getOrCreateDoltSchemasTable(ctx *sql.Context, db Database) (retTbl *Writabl return wrapper.backingTable, nil } -func migrateOldSchemasTableToNew(ctx *sql.Context, db Database, schemasTable *WritableDoltTable) (newTable *WritableDoltTable, rerr error) { - // Copy all of the old data over and add an index column and an extra column - iter, err := SqlTableToRowIter(ctx, schemasTable.DoltTable, nil) - if err != nil { - return nil, err - } - - // The dolt_schemas table has undergone various changes over time and multiple possible schemas for it exist, so we - // need to get the column indexes from the current schema - nameIdx := schemasTable.sqlSchema().IndexOfColName(doltdb.SchemasTablesNameCol) - typeIdx := schemasTable.sqlSchema().IndexOfColName(doltdb.SchemasTablesTypeCol) - fragmentIdx := schemasTable.sqlSchema().IndexOfColName(doltdb.SchemasTablesFragmentCol) - extraIdx := schemasTable.sqlSchema().IndexOfColName(doltdb.SchemasTablesExtraCol) - sqlModeIdx := schemasTable.sqlSchema().IndexOfColName(doltdb.SchemasTablesSqlModeCol) - - defer func(iter sql.RowIter, ctx *sql.Context) { - err := iter.Close(ctx) - if err != nil && rerr == nil { - rerr = err - } - }(iter, ctx) - - var newRows []sql.Row - for { - sqlRow, err := iter.Next(ctx) - if err == io.EOF { - break - } - if err != nil { - return nil, err - } - - newRow := make(sql.Row, SchemaTableSchema().GetAllCols().Size()) - newRow[0] = sqlRow[typeIdx] - newRow[1] = sqlRow[nameIdx] - newRow[2] = sqlRow[fragmentIdx] - if extraIdx >= 0 { - newRow[3] = sqlRow[extraIdx] - } - if sqlModeIdx >= 0 { - newRow[4] = sqlRow[sqlModeIdx] - } - - newRows = append(newRows, newRow) - } - - err = db.dropTable(ctx, doltdb.SchemasTableName) - if err != nil { - return nil, err - } - - root, err := db.GetRoot(ctx) - if err != nil { - return nil, err - } - - err = db.createDoltTable(ctx, doltdb.SchemasTableName, doltdb.DefaultSchemaName, root, SchemaTableSchema()) - if err != nil { - return nil, err - } - - tbl, _, err := db.GetTableInsensitive(ctx, doltdb.SchemasTableName) - if err != nil { - return nil, err - } - - wrapper, ok := tbl.(*SchemaTable) - if !ok { - return nil, fmt.Errorf("expected a SchemaTable, but found %T", tbl) - } - - if wrapper.backingTable == nil { - return nil, sql.ErrTableNotFound.New(doltdb.SchemasTableName) - } - - inserter := wrapper.backingTable.Inserter(ctx) - for _, row := range newRows { - err = inserter.Insert(ctx, row) - if err != nil { - return nil, err - } - } - - err = inserter.Close(ctx) - if err != nil { - return nil, err - } - - return wrapper.backingTable, nil -} - // fragFromSchemasTable returns the row with the given schema fragment if it exists. func fragFromSchemasTable(ctx *sql.Context, tbl *WritableDoltTable, fragType string, name string) (r sql.Row, found bool, rerr error) { fragType, name = strings.ToLower(fragType), strings.ToLower(name) diff --git a/go/libraries/doltcore/sqle/schema_table_test.go b/go/libraries/doltcore/sqle/schema_table_test.go index 3af0d95f778..53e4cc3ceba 100644 --- a/go/libraries/doltcore/sqle/schema_table_test.go +++ b/go/libraries/doltcore/sqle/schema_table_test.go @@ -16,8 +16,6 @@ package sqle import ( "context" - "io" - "strings" "testing" "github.com/dolthub/go-mysql-server/sql" @@ -30,7 +28,7 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/table/editor" ) -func TestSchemaTableMigrationOriginal(t *testing.T) { +func TestAncientSchemaTableError(t *testing.T) { dEnv := dtestutils.CreateTestEnv() tmpDir, err := dEnv.TempTableFilesDir() require.NoError(t, err) @@ -64,104 +62,9 @@ func TestSchemaTableMigrationOriginal(t *testing.T) { err = inserter.Close(ctx) require.NoError(t, err) - tbl, err := getOrCreateDoltSchemasTable(ctx, db) // removes the old table and recreates it with the new schema - require.NoError(t, err) - - iter, err := SqlTableToRowIter(ctx, tbl.DoltTable, nil) - require.NoError(t, err) - - var rows []sql.Row - for { - row, err := iter.Next(ctx) - if err == io.EOF { - break - } - - require.NoError(t, err) - rows = append(rows, row) - } - - require.NoError(t, iter.Close(ctx)) - expectedRows := []sql.Row{ - {"view", "view1", "SELECT v1 FROM test;", nil, nil}, - {"view", "view2", "SELECT v2 FROM test;", nil, nil}, - } - - assert.Equal(t, expectedRows, rows) -} - -func TestSchemaTableMigrationV1(t *testing.T) { - dEnv := dtestutils.CreateTestEnv() - tmpDir, err := dEnv.TempTableFilesDir() - require.NoError(t, err) - opts := editor.Options{Deaf: dEnv.DbEaFactory(), Tempdir: tmpDir} - db, err := NewDatabase(context.Background(), "dolt", dEnv.DbData(), opts) - require.NoError(t, err) - - _, ctx, err := NewTestEngine(dEnv, context.Background(), db) - require.NoError(t, err) - - // original schema of dolt_schemas table with the ID column - err = db.createSqlTable(ctx, doltdb.SchemasTableName, "", sql.NewPrimaryKeySchema(sql.Schema{ - {Name: doltdb.SchemasTablesTypeCol, Type: gmstypes.Text, Source: doltdb.SchemasTableName, PrimaryKey: false}, - {Name: doltdb.SchemasTablesNameCol, Type: gmstypes.Text, Source: doltdb.SchemasTableName, PrimaryKey: false}, - {Name: doltdb.SchemasTablesFragmentCol, Type: gmstypes.Text, Source: doltdb.SchemasTableName, PrimaryKey: false}, - {Name: doltdb.SchemasTablesIdCol, Type: gmstypes.Int64, Source: doltdb.SchemasTableName, PrimaryKey: true}, - {Name: doltdb.SchemasTablesExtraCol, Type: gmstypes.JsonType{}, Source: doltdb.SchemasTableName, PrimaryKey: false, Nullable: true}, - }), sql.Collation_Default, "") - require.NoError(t, err) - - sqlTbl, found, err := db.GetTableInsensitive(ctx, doltdb.SchemasTableName) - require.NoError(t, err) - require.True(t, found) - - wrapper, ok := sqlTbl.(*SchemaTable) - require.True(t, ok) - require.NotNil(t, wrapper.backingTable) - - inserter := wrapper.UnWrap().Inserter(ctx) - // JSON string has no spaces because our various JSON libraries don't agree on how to marshall it - err = inserter.Insert(ctx, sql.Row{"view", "view1", "SELECT v1 FROM test;", 1, `{"extra":"data"}`}) - require.NoError(t, err) - err = inserter.Insert(ctx, sql.Row{"view", "view2", "SELECT v2 FROM test;", 2, nil}) - require.NoError(t, err) - err = inserter.Close(ctx) - require.NoError(t, err) - - tbl, err := getOrCreateDoltSchemasTable(ctx, db) // removes the old table and recreates it with the new schema - require.NoError(t, err) - - iter, err := SqlTableToRowIter(ctx, tbl.DoltTable, nil) - require.NoError(t, err) - - var rows []sql.Row - for { - row, err := iter.Next(ctx) - if err == io.EOF { - break - } - - require.NoError(t, err) - // convert the JSONDocument to a string for comparison - if row[3] != nil { - jsonDoc, ok := row[3].(sql.JSONWrapper) - if ok { - row[3], err = gmstypes.StringifyJSON(jsonDoc) - row[3] = strings.ReplaceAll(row[3].(string), " ", "") // remove spaces - } - - require.NoError(t, err) - } - - rows = append(rows, row) - } - - require.NoError(t, iter.Close(ctx)) - - expectedRows := []sql.Row{ - {"view", "view1", "SELECT v1 FROM test;", `{"extra":"data"}`, nil}, - {"view", "view2", "SELECT v2 FROM test;", nil, nil}, - } - - assert.Equal(t, expectedRows, rows) + // Ancient dolt_schemas table. Verify error. + tbl, err := getOrCreateDoltSchemasTable(ctx, db) // + require.Error(t, err) + assert.Contains(t, err.Error(), "cannot migrate dolt_schemas table") + require.Nil(t, tbl) } diff --git a/integration-tests/bats/helper/old_dolt_schemas/config.json b/integration-tests/bats/helper/old_dolt_schemas/config.json deleted file mode 100644 index 9e26dfeeb6e..00000000000 --- a/integration-tests/bats/helper/old_dolt_schemas/config.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/integration-tests/bats/helper/old_dolt_schemas/noms/4bujnjuatdli0klda7ucot0tl836jujt b/integration-tests/bats/helper/old_dolt_schemas/noms/4bujnjuatdli0klda7ucot0tl836jujt deleted file mode 100644 index c41b419f7c20cf86bd1a0030dd7abbbd06e7ef12..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 57 zcmZSPU}Ip=+7fK=!u{H&Tf*+M7$AT%eP#FypGmJ@6s*v61`07k#hL$ay>Un-`CR}2 D-mnpz diff --git a/integration-tests/bats/helper/old_dolt_schemas/noms/63c0dojobm5j7arp99hiq78hj1rtp7m5 b/integration-tests/bats/helper/old_dolt_schemas/noms/63c0dojobm5j7arp99hiq78hj1rtp7m5 deleted file mode 100644 index c9381672d3addaac5b49a7f3c72309c08377ffc3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 775 zcmZvaT}V_x6vxlpy&rq^uB(+|*n^8%Y3yp0_CctnOGP4yQ7Y+W_TJgNct6%Vvz9$X zp!8Bkk&y{KMqfTiUj#iw5~}E578H-VQG<2ja1M>hv6`DIOojy|2SWQ>s7hd za~z9_qC*%~gGfGLVuoeWB-9X4QX&qvY+=5Mdu-3Gv8}GOg(6za@G?=fzMJ^e@?0UX zB33TC*danzowR&!;ZDo$v|Oq(%OR9uCzaj*YU0j#EGtckujl3shVR{c(SN#4NnxM3 zj0OQIcVE7k9{4#k(y;Ww*qM&m@+*+~stIZ!X`pJRXEWVQ=)UK%KxWeOiM89+J4sK7rc-C$^9AJ2NkoBa9F zlqlJ{Pm)S}Q@$e4(>Pa2(iAnLsUiR<=kt$A2u+G=PUt<`R`7Ms4LJqIIu7F@Y~kB8*>u>Hsp71LK}DO|2F?PbK!Wyyo^KW X{=31sgt;ufJ>x&GJ{w(ZOlK;UGg(8+PolG{N$s{I|q9C#t zZ>1opU_HuSv;?6RJjo&++C%XmN$q+$AkBK@BQA_WB7HY03>wkjOGSe z>S^tkV>ZXJ<>;8;mgN}6xJihs>%_8*vxZsRKNAhEj_ND$iuEo(6#rb#hb0-IYC_#q z^C)&wEyO9ONpbV@r*HRnzCXWs^6l2p`>nq?y3~~+YV0a~CL|088c8VT^grVKL38Y> zvaMF8vzR8+#G@gez#O9ri2|EMEaeFnqf+q$>>L3ChbI*Pk<4$!v|5v=6jP+stvg9>7w z&-#knKtSTSUk&8R5?pHd%#GAad9FPp)?N?+Du`wr!dC#nyBmS{5Y;m7w!?YT*3ap> zjeVQ=s1Q)r=N?j;Eu(E9DZLSh)9Fnv!|VHyx`-2X@Mi>iBLJp^7q%!UxHEvDCxUzg zUaoIU%xCw8$1BTqQGsh_&NwEC=b UP}I?`3P|0Z_3hUu$d^lh0X=!G1ONa4 diff --git a/integration-tests/bats/helper/old_dolt_schemas/noms/chghnlvugtiui6khfo29cpfnfr3166qc b/integration-tests/bats/helper/old_dolt_schemas/noms/chghnlvugtiui6khfo29cpfnfr3166qc deleted file mode 100644 index db6bee870895b87841827e6868690771ea2025fe..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 211 zcmbQu_+b_s12dC`u6|lhX>o>rl2LMEvT?GhX_85rWuie!stqXl^sQUU!NSBOSd^Mpte=sZm{P2tn^;_uTEu?x-rcWL{(sxx zr+)tI(v05*TNHa-Ss0l_z$!EIfJ*WbbM%voQWHy3aVXA~QFxHZzsNj!b(uQ@1T6lW eo9U~$Mdn~ZXrT^Jh!HA2`~TJ(hg6c^1pol1*G{Sc diff --git a/integration-tests/bats/helper/old_dolt_schemas/noms/d0b8ibrrjvvv3f0mtd7hathtjq2vhig9 b/integration-tests/bats/helper/old_dolt_schemas/noms/d0b8ibrrjvvv3f0mtd7hathtjq2vhig9 deleted file mode 100644 index ad2f1859d31980c3a1eee0eda97e90f26149f919..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 211 zcmbQu_+b_s12dC`u6|lhX>o?WiLs%fp}C=Hs)dnZa;j-sYKoDOVWNq-rJy1My$?pOHKrB!F diff --git a/integration-tests/bats/helper/old_dolt_schemas/noms/erturgf7to0f6r8prlbllflbshg5li4n b/integration-tests/bats/helper/old_dolt_schemas/noms/erturgf7to0f6r8prlbllflbshg5li4n deleted file mode 100644 index 4c4f95b67e6d8e0b48ce1fc69345826f77e56e74..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 126 zcma$0;LOIr#3WdhnpUizk(!uNte=}$T#{PEe)8VkuT%bi+u)~u{_N6>-v(P0dt6x< znMA-UGxLB-@)C3OlZ#RlOH$dxbQxSbSXs_z_cG6{+w8{x0e;09Vm0^QRy?qrAiNJK R#0V9S`oHzYA(iBJ0RRRqE)xI% diff --git a/integration-tests/bats/helper/old_dolt_schemas/noms/manifest b/integration-tests/bats/helper/old_dolt_schemas/noms/manifest deleted file mode 100644 index 1641eab0ce4..00000000000 --- a/integration-tests/bats/helper/old_dolt_schemas/noms/manifest +++ /dev/null @@ -1 +0,0 @@ -4:__LD_1__:ma67ier4cau1ai1jhk2fom2m3bggmfa3:iht2tkfo9cbbnrfmla9ks4gahc0snq4o:63c0dojobm5j7arp99hiq78hj1rtp7m5:2:ro000svvgga0k7hatg7fta9j8ismtc3b:1:chghnlvugtiui6khfo29cpfnfr3166qc:1:d0b8ibrrjvvv3f0mtd7hathtjq2vhig9:1:oktjimrpqi8m9dbpt3c7ikoaolhgtr2n:6:oik6v50p7q6as4t7qg7f1i06iomt9vn9:2:4bujnjuatdli0klda7ucot0tl836jujt:1:qe0mct6rf8puhdnqecm185p63dacaqrt:4:erturgf7to0f6r8prlbllflbshg5li4n:1:u9c6ttq6ddrgqdfbod9pg2vv79ab6crt:2:aum9p7826iqaeq0pjtqlcn56hoj5nhht:2 \ No newline at end of file diff --git a/integration-tests/bats/helper/old_dolt_schemas/noms/oik6v50p7q6as4t7qg7f1i06iomt9vn9 b/integration-tests/bats/helper/old_dolt_schemas/noms/oik6v50p7q6as4t7qg7f1i06iomt9vn9 deleted file mode 100644 index acdfd6744edef4ce4d05244b28a01071edc13791..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 375 zcmeyvXv4|ooS&PUS;E4Sn_7~{$-&4Lk&;-F$i|YATAa)z<(ZdRl9`y3S(U25C|;pZ zlv4XD=g7))4>eUn&qu7mA# zo5Z7n3HAYvX#$2iiC0&NcA``jj3W5JQ({PLAO?F#OnhLh6KE<$4HHBy*Dmc+1_=1T zwXb!KeSEF^`~QzqB1?fV9U6LJ005DObianSg$@WoiUGk;JPbp_LbyB9liHMO>lGWX zYZTiz-*KC$7G?eMbF0&P*j>D@N6DeQe2Z+$fKv>;n8y}FdH6sID~5{HHUm`kvO)4h zQVeX_rmSI`Q~+M6UL#yV!{Ev_)AvaBjsk*SY}gVm>N;DHO*wC|6cx}ci6pjUWTZkL z1?WBvtB@t@XgEt_IgcRiyz8;$ENBxcGFG>HojV%N7<@3$FZPn;R+7N#dua~@?|rPx zHiF@zooUIZ6H+W0i%GI3DH>#TgtK-IGVygyEltp4J)o1-(RP~#I0E^8*|eCQb7kt6 zGF1y3$`sw{1uv3(;oMjzAMd&VL~q8Fvw4K=1`1GJa}*e(z?ru`IXDid&+v>br*-7v z$TrY+WN}j+Urn5<>KM$9+v@Ap%MSFUMe{gj_<)ikzQ+)Ki@D@vDZlh5{y_zumS zxNpdN^c^ zj@^as!geq((PemHN#sRkAyFDze0$7Q^miR0bF7ZAfe`S)6lT_6lq673(>935AcT%| z?f^9pmG^xPr+B|==(eP$CDSl$&t)0IL|PV0eQ1Yc469f`rnIno$0@7}mP7#hnPgzq zR_@`M``51cZdKjREf~%wkJKN3Y$8JVgNBwR{4Qf3-?+i$=W^#3=rT-*bJziW>vC^j zM8^o<{AY00?ZSy&)54`$@Xhttuxo9HU zoM>6sNRC8vQ#{#(lFjiLQj-})A+A~fq}gsu_!gKq?m(&frpoSLVQkUaS1B} z18Z4gPH8H8LvT)F!QD2sI~-CUJuc5X^f#81hk^0N@7wcTJ~*>6FfwWA>Zj$D7H8<2 zq$Qe}87HP1n5CH}C7N0$nDi-zPkgdPx{#TaCId7W&*kG^$F zIarwVxefkUKH%g@$4 DysCCF diff --git a/integration-tests/bats/helper/old_dolt_schemas/noms/ro000svvgga0k7hatg7fta9j8ismtc3b b/integration-tests/bats/helper/old_dolt_schemas/noms/ro000svvgga0k7hatg7fta9j8ismtc3b deleted file mode 100644 index 275122771159cd4fa2b9a51deb31d9aee404cf32..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 211 zcmbQu_+b_s12dC`u6|lhX>o?Wg>i~ml3|ibvbm|HSxTy*v5|p+kwKz?S&E@)Vw!;w zyJ$#$L+AnFS&TuZm)FS_{ODV^l!JwdNw6q2tyn)JH8G`FKR2 f<-6@Wep$B9kmGYY2NYt2iqHPP^~NEU7_=%=1tIU zRInRCB0@JtH&U{ylqiaBgwll|XkAFvjUq1mY>ZT@V!`@!?}fwtoy$4g8}MsG0zR}m z8nf&gE5`OE_70lqBuQ9Wpm@s20Wp?nBQ9cS)XtE z1%yH_u~@7~u*qT-%sk@CPu{*+omv0hI=1Wivu1HEey{gRR)8q{k6Cg!#ld#Wa0&Lv zf5^t%VCtr@pr|IbJ>4wmE~TDtgGF6ZnsoFsnUVkquv(S4Ix~udD@y<(Rg48f6LO(; zt=Si>cb)$A(trF%&-L4LnJr2%2Q?vQz*Or1_Am+>)Yc8^Fwe!@6O(ygt2j)8A`+EA z)-k9_%nXGQAPq;s!$67YRqT2?we)IPD3=^laBzkAJUr?5rmCdmSX5L2KnNeAh?B8q#4 zd@|P{=Li#AdX{wz$Rj%cpe(6C1+uhBMW|C0RaxoCX+8i$-+s42pzT*eS>-dxXs3ED z;6w{R7u(7jKw<-c#qH3~{oR>vT^)Tq+dW)g9tS+$#f|QueyhSu$T7>)#Qw`t@*uQl jj)a%CJ$hPSx$vp)Q2gV2%J+!lZ+ftP|Mg Date: Fri, 12 Jul 2024 15:04:01 -0700 Subject: [PATCH 09/18] More test fixes for the change in behavior of dolt_schema --- go/libraries/doltcore/sqle/sqlddl_test.go | 3 +-- go/libraries/doltcore/sqle/sqldelete_test.go | 17 ++++++------ go/libraries/doltcore/sqle/sqlinsert_test.go | 23 +++++++--------- go/libraries/doltcore/sqle/sqlreplace_test.go | 27 +++++++++---------- go/libraries/doltcore/sqle/sqlselect_test.go | 21 +++++++-------- go/libraries/doltcore/sqle/sqlupdate_test.go | 9 ------- integration-tests/bats/triggers.bats | 12 +++------ 7 files changed, 46 insertions(+), 66 deletions(-) diff --git a/go/libraries/doltcore/sqle/sqlddl_test.go b/go/libraries/doltcore/sqle/sqlddl_test.go index 48732906fa0..2fd3d6acd85 100644 --- a/go/libraries/doltcore/sqle/sqlddl_test.go +++ b/go/libraries/doltcore/sqle/sqlddl_test.go @@ -819,9 +819,8 @@ func TestAlterSystemTables(t *testing.T) { "INSERT INTO dolt_docs VALUES ('LICENSE.md','A license')") CreateTestTable(t, dEnv, doltdb.DoltQueryCatalogTableName, dtables.DoltQueryCatalogSchema, "INSERT INTO dolt_query_catalog VALUES ('abc123', 1, 'example', 'select 2+2 from dual', 'description')") - CreateTestTable(t, dEnv, doltdb.SchemasTableName, SchemaTableSchema(), - "INSERT INTO dolt_schemas (type, name, fragment) VALUES ('view', 'name', 'create view name as select 2+2 from dual')") ExecuteSetupSQL(context.Background(), ` + CREATE VIEW name as select 2+2 from dual; CREATE PROCEDURE simple_proc2() SELECT 1+1; INSERT INTO dolt_ignore VALUES ('test', 1);`)(t, dEnv) } diff --git a/go/libraries/doltcore/sqle/sqldelete_test.go b/go/libraries/doltcore/sqle/sqldelete_test.go index 9aab4d942eb..b5e62b04359 100644 --- a/go/libraries/doltcore/sqle/sqldelete_test.go +++ b/go/libraries/doltcore/sqle/sqldelete_test.go @@ -151,22 +151,22 @@ var BasicDeleteTests = []DeleteTest{ { Name: "delete invalid table", DeleteQuery: "delete from nobody", - ExpectedErr: "invalid table", + ExpectedErr: "table not found: nobody", }, { Name: "delete invalid column", DeleteQuery: "delete from people where z = 'dne'", - ExpectedErr: "invalid column", + ExpectedErr: "column \"z\" could not be found in any table in scope", }, { Name: "delete negative limit", DeleteQuery: "delete from people limit -1", - ExpectedErr: "invalid limit number", + ExpectedErr: "syntax error", // syntax error at position 27 near 'limit' }, { Name: "delete negative offset", DeleteQuery: "delete from people limit 1 offset -1", - ExpectedErr: "invalid limit number", + ExpectedErr: "syntax error", // syntax error at position 36 near 'offset' }, } @@ -208,11 +208,9 @@ var systemTableDeleteTests = []DeleteTest{ { Name: "delete dolt_schemas", AdditionalSetup: CreateTableFn(doltdb.SchemasTableName, SchemaTableSchema(), - "INSERT INTO dolt_schemas (type, name, fragment) VALUES ('view', 'name', 'create view name as select 2+2 from dual')"), - DeleteQuery: "delete from dolt_schemas", - SelectQuery: "select * from dolt_schemas", - ExpectedRows: ToSqlRows(dtables.DoltQueryCatalogSchema), - ExpectedSchema: SchemaTableSchema(), + "CREATE VIEW name as select 2+2 from dual"), + DeleteQuery: "delete from dolt_schemas", + ExpectedErr: "table doesn't support DELETE FROM", }, } @@ -239,6 +237,7 @@ func testDeleteQuery(t *testing.T, test DeleteTest) { root, err = executeModify(t, context.Background(), dEnv, root, test.DeleteQuery) if len(test.ExpectedErr) > 0 { require.Error(t, err) + assert.Contains(t, err.Error(), test.ExpectedErr) return } else { require.NoError(t, err) diff --git a/go/libraries/doltcore/sqle/sqlinsert_test.go b/go/libraries/doltcore/sqle/sqlinsert_test.go index 6dee3273d08..e734516bfc8 100644 --- a/go/libraries/doltcore/sqle/sqlinsert_test.go +++ b/go/libraries/doltcore/sqle/sqlinsert_test.go @@ -68,12 +68,12 @@ var BasicInsertTests = []InsertTest{ { Name: "insert no columns too few values", InsertQuery: "insert into people values (2, 'Bart', 'Simpson', false, 10, 9, '00000000-0000-0000-0000-000000000002')", - ExpectedErr: "too few values", + ExpectedErr: "number of values does not match number of columns provided", }, { Name: "insert no columns too many values", InsertQuery: "insert into people values (2, 'Bart', 'Simpson', false, 10, 9, '00000000-0000-0000-0000-000000000002', 222, 'abc')", - ExpectedErr: "too many values", + ExpectedErr: "number of values does not match number of columns provided", }, { Name: "insert full columns", @@ -127,27 +127,27 @@ var BasicInsertTests = []InsertTest{ { Name: "insert partial columns duplicate column", InsertQuery: "insert into people (id, first_name, last_name, first_name) values (2, 'Bart', 'Simpson', 'Bart')", - ExpectedErr: "duplicate column", + ExpectedErr: "column 'first_name' specified twice", }, { Name: "insert partial columns invalid column", InsertQuery: "insert into people (id, first_name, last_name, middle) values (2, 'Bart', 'Simpson', 'Nani')", - ExpectedErr: "duplicate column", + ExpectedErr: "Unknown column 'middle' in 'people'", }, { Name: "insert missing non-nullable column", InsertQuery: "insert into people (id, first_name) values (2, 'Bart')", - ExpectedErr: "column received nil but is non-nullable", + ExpectedErr: "Field 'last_name' doesn't have a default value", }, { Name: "insert partial columns mismatch too many values", InsertQuery: "insert into people (id, first_name, last_name) values (2, 'Bart', 'Simpson', false)", - ExpectedErr: "too many values", + ExpectedErr: "number of values does not match number of columns provided", }, { Name: "insert partial columns mismatch too few values", InsertQuery: "insert into people (id, first_name, last_name) values (2, 'Bart')", - ExpectedErr: "too few values", + ExpectedErr: "number of values does not match number of columns provided", }, { Name: "insert partial columns functions", @@ -228,7 +228,7 @@ var BasicInsertTests = []InsertTest{ { Name: "insert partial columns multiple rows null pk", InsertQuery: "insert into people (id, first_name, last_name) values (0, 'Bart', 'Simpson'), (1, 'Homer', null)", - ExpectedErr: "column received nil but is non-nullable", + ExpectedErr: "column name 'last_name' is non-nullable but attempted to set a value of null", }, { Name: "insert partial columns multiple rows duplicate", @@ -399,11 +399,7 @@ var systemTableInsertTests = []InsertTest{ Name: "insert into dolt_schemas", AdditionalSetup: CreateTableFn(doltdb.SchemasTableName, SchemaTableSchema(), ""), InsertQuery: "insert into dolt_schemas (type, name, fragment) values ('view', 'name', 'create view name as select 2+2 from dual')", - SelectQuery: "select * from dolt_schemas ORDER BY name", - ExpectedRows: ToSqlRows(CompressSchema(SchemaTableSchema()), - NewRow(types.String("view"), types.String("name"), types.String("create view name as select 2+2 from dual")), - ), - ExpectedSchema: CompressSchema(SchemaTableSchema()), + ExpectedErr: "table doesn't support INSERT INTO", }, } @@ -442,6 +438,7 @@ func testInsertQuery(t *testing.T, test InsertTest) { root, err = executeModify(t, context.Background(), dEnv, root, test.InsertQuery) if len(test.ExpectedErr) > 0 { require.Error(t, err) + assert.Contains(t, err.Error(), test.ExpectedErr) return } else { require.NoError(t, err) diff --git a/go/libraries/doltcore/sqle/sqlreplace_test.go b/go/libraries/doltcore/sqle/sqlreplace_test.go index d3501e11be2..790abd85ff3 100644 --- a/go/libraries/doltcore/sqle/sqlreplace_test.go +++ b/go/libraries/doltcore/sqle/sqlreplace_test.go @@ -70,12 +70,12 @@ var BasicReplaceTests = []ReplaceTest{ { Name: "replace no columns too few values", ReplaceQuery: "replace into people values (2, 'Bart', 'Simpson', false, 10, 9, '00000000-0000-0000-0000-000000000002')", - ExpectedErr: "too few values", + ExpectedErr: "number of values does not match number of columns provided", }, { Name: "replace no columns too many values", ReplaceQuery: "replace into people values (2, 'Bart', 'Simpson', false, 10, 9, '00000000-0000-0000-0000-000000000002', 222, 'abc')", - ExpectedErr: "too many values", + ExpectedErr: "number of values does not match number of columns provided", }, { Name: "replace full columns", @@ -129,27 +129,27 @@ var BasicReplaceTests = []ReplaceTest{ { Name: "replace partial columns duplicate column", ReplaceQuery: "replace into people (id, first_name, last_name, first_name) values (2, 'Bart', 'Simpson', 'Bart')", - ExpectedErr: "duplicate column", + ExpectedErr: "column 'first_name' specified twice", }, { Name: "replace partial columns invalid column", ReplaceQuery: "replace into people (id, first_name, last_name, middle) values (2, 'Bart', 'Simpson', 'Nani')", - ExpectedErr: "duplicate column", + ExpectedErr: "Unknown column 'middle' in 'people'", }, { Name: "replace missing non-nullable column", ReplaceQuery: "replace into people (id, first_name) values (2, 'Bart')", - ExpectedErr: "column received nil but is non-nullable", + ExpectedErr: "Field 'last_name' doesn't have a default value", }, { Name: "replace partial columns mismatch too many values", ReplaceQuery: "replace into people (id, first_name, last_name) values (2, 'Bart', 'Simpson', false)", - ExpectedErr: "too many values", + ExpectedErr: "number of values does not match number of columns provided", }, { Name: "replace partial columns mismatch too few values", ReplaceQuery: "replace into people (id, first_name, last_name) values (2, 'Bart')", - ExpectedErr: "too few values", + ExpectedErr: "number of values does not match number of columns provided", }, { Name: "replace partial columns functions", @@ -194,7 +194,7 @@ var BasicReplaceTests = []ReplaceTest{ { Name: "replace partial columns multiple rows null pk", ReplaceQuery: "replace into people (id, first_name, last_name) values (0, 'Bart', 'Simpson'), (1, 'Homer', null)", - ExpectedErr: "column received nil but is non-nullable", + ExpectedErr: "column name 'last_name' is non-nullable but attempted to set a value of null", }, { Name: "replace partial columns multiple rows duplicate", @@ -225,7 +225,7 @@ var BasicReplaceTests = []ReplaceTest{ (0, "Homer", "Simpson", true, 45, 100), (8, "Milhouse", "Van Houten", false, 8, 3.5), (7, "Maggie", null, false, 1, 5.1)`, - ExpectedErr: "Constraint failed for column 'last_name': Not null", + ExpectedErr: "column name 'last_name' is non-nullable but attempted to set a value of null", }, { Name: "replace partial columns existing pk", @@ -273,11 +273,9 @@ var systemTableReplaceTests = []ReplaceTest{ { Name: "replace into dolt_schemas", AdditionalSetup: CreateTableFn(doltdb.SchemasTableName, SchemaTableSchema(), - "INSERT INTO dolt_schemas VALUES ('view', 'name', 'create view name as select 2+2 from dual', NULL, NULL)"), - ReplaceQuery: "replace into dolt_schemas (type, name, fragment) values ('view', 'name', 'create view name as select 1+1 from dual')", - SelectQuery: "select type, name, fragment, extra, sql_mode from dolt_schemas", - ExpectedRows: []sql.Row{{"view", "name", "create view name as select 1+1 from dual", nil, nil}}, - ExpectedSchema: CompressSchema(SchemaTableSchema()), + "CREATE VIEW name as select 2+2 from dual"), + ReplaceQuery: "replace into dolt_schemas (type, name, fragment) values ('view', 'name', 'create view name as select 1+1 from dual')", + ExpectedErr: "table doesn't support INSERT INTO", }, } @@ -312,6 +310,7 @@ func testReplaceQuery(t *testing.T, test ReplaceTest) { root, err = executeModify(t, context.Background(), dEnv, root, test.ReplaceQuery) if len(test.ExpectedErr) > 0 { require.Error(t, err) + assert.Contains(t, err.Error(), test.ExpectedErr) return } else { require.NoError(t, err) diff --git a/go/libraries/doltcore/sqle/sqlselect_test.go b/go/libraries/doltcore/sqle/sqlselect_test.go index c31eaad3cca..3197b157c04 100644 --- a/go/libraries/doltcore/sqle/sqlselect_test.go +++ b/go/libraries/doltcore/sqle/sqlselect_test.go @@ -19,6 +19,7 @@ import ( "testing" "time" + "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dtables" "github.com/dolthub/go-mysql-server/sql" gmstypes "github.com/dolthub/go-mysql-server/sql/types" "github.com/stretchr/testify/assert" @@ -32,8 +33,6 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/row" "github.com/dolthub/dolt/go/libraries/doltcore/schema" "github.com/dolthub/dolt/go/libraries/doltcore/schema/typeinfo" - "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dtables" - "github.com/dolthub/dolt/go/libraries/doltcore/sqle/json" "github.com/dolthub/dolt/go/store/datas" "github.com/dolthub/dolt/go/store/types" ) @@ -1309,9 +1308,9 @@ var systemTableSelectTests = []SelectTest{ { Name: "select from dolt_schemas", AdditionalSetup: CreateTableFn(doltdb.SchemasTableName, SchemaTableSchema(), - `INSERT INTO dolt_schemas VALUES ('view', 'name', 'create view name as select 2+2 from dual', NULL, NULL)`), + `CREATE VIEW name as select 2+2 from dual`), Query: "select * from dolt_schemas", - ExpectedRows: []sql.Row{{"view", "name", "create view name as select 2+2 from dual", nil, nil}}, + ExpectedRows: []sql.Row{{"view", "name", "CREATE VIEW name as select 2+2 from dual", ignoreVal, "NO_ENGINE_SUBSTITUTION,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES"}}, ExpectedSchema: CompressSchema(SchemaTableSchema()), }, } @@ -1346,6 +1345,10 @@ func installTestCommitClock() func() { } } +type testIgnoredValue struct{} + +var ignoreVal = testIgnoredValue{} + // Tests the given query on a freshly created dataset, asserting that the result has the given schema and rows. If // expectedErr is set, asserts instead that the execution returns an error that matches. func testSelectQuery(t *testing.T, test SelectTest) { @@ -1378,14 +1381,10 @@ func testSelectQuery(t *testing.T, test SelectTest) { for i := 0; i < len(test.ExpectedRows); i++ { assert.Equal(t, len(test.ExpectedRows[i]), len(actualRows[i])) for j := 0; j < len(test.ExpectedRows[i]); j++ { - if _, ok := actualRows[i][j].(json.NomsJSON); ok { - cmp, err := gmstypes.CompareJSON(actualRows[i][j].(json.NomsJSON), test.ExpectedRows[i][j].(json.NomsJSON)) - assert.NoError(t, err) - assert.Equal(t, 0, cmp) - } else { - assert.Equal(t, test.ExpectedRows[i][j], actualRows[i][j]) + if test.ExpectedRows[i][j] == ignoreVal { + continue } - + assert.Equal(t, test.ExpectedRows[i][j], actualRows[i][j]) } } diff --git a/go/libraries/doltcore/sqle/sqlupdate_test.go b/go/libraries/doltcore/sqle/sqlupdate_test.go index c1cf6db5a17..2ca58d78858 100644 --- a/go/libraries/doltcore/sqle/sqlupdate_test.go +++ b/go/libraries/doltcore/sqle/sqlupdate_test.go @@ -376,15 +376,6 @@ var systemTableUpdateTests = []UpdateTest{ ExpectedRows: []sql.Row{{"abc123", uint64(2), "example", "select 2+2 from dual", "description"}}, ExpectedSchema: CompressSchema(dtables.DoltQueryCatalogSchema), }, - { - Name: "update dolt_schemas", - AdditionalSetup: CreateTableFn(doltdb.SchemasTableName, SchemaTableSchema(), - `INSERT INTO dolt_schemas VALUES ('view', 'name', 'create view name as select 2+2 from dual', NULL, NULL)`), - UpdateQuery: "update dolt_schemas set type = 'not a view'", - SelectQuery: "select * from dolt_schemas", - ExpectedRows: []sql.Row{{"not a view", "name", "create view name as select 2+2 from dual", nil, nil}}, - ExpectedSchema: CompressSchema(SchemaTableSchema()), - }, } // Tests the given query on a freshly created dataset, asserting that the result has the given schema and rows. If diff --git a/integration-tests/bats/triggers.bats b/integration-tests/bats/triggers.bats index 5876f890767..3888eeaf665 100644 --- a/integration-tests/bats/triggers.bats +++ b/integration-tests/bats/triggers.bats @@ -119,16 +119,12 @@ SQL [[ "$output" =~ "trigger1,INSERT,test,1,,SET new.v1 = new.v1 + 1,BEFORE,root@localhost,utf8mb4,utf8mb4_0900_bin,utf8mb4_0900_bin" ]] || false } -@test "triggers: Writing directly into dolt_schemas" { +@test "triggers: Writing directly into dolt_schemas is forbidden" { dolt sql -q "CREATE TABLE test(pk BIGINT PRIMARY KEY, v1 BIGINT);" dolt sql -q "CREATE VIEW view1 AS SELECT v1 FROM test;" - dolt sql -q "INSERT INTO dolt_schemas VALUES ('trigger', 'trigger1', 'CREATE TRIGGER trigger1 BEFORE INSERT ON test FOR EACH ROW SET new.v1 = -new.v1;', json_object('CreatedAt', 1), NULL);" - dolt sql -q "INSERT INTO test VALUES (1, 1);" - run dolt sql -q "SELECT * FROM test" -r=csv - [ "$status" -eq "0" ] - [[ "$output" =~ "pk,v1" ]] || false - [[ "$output" =~ "1,-1" ]] || false - [[ "${#lines[@]}" = "2" ]] || false + run dolt sql -q "INSERT INTO dolt_schemas VALUES ('trigger', 'trigger1', 'CREATE TRIGGER trigger1 BEFORE INSERT ON test FOR EACH ROW SET new.v1 = -new.v1;', json_object('CreatedAt', 1), NULL);" + [ "$status" -eq 1 ] + [[ "$output" =~ "doesn't support INSERT INTO" ]] || false } @test "triggers: Merge triggers on different branches, no conflict" { From a228d1e9afa1e8d9e61ba24bba4ecfd6f448ecc6 Mon Sep 17 00:00:00 2001 From: macneale4 Date: Fri, 12 Jul 2024 22:11:40 +0000 Subject: [PATCH 10/18] [ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/update.sh --- go/libraries/doltcore/sqle/sqlselect_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/libraries/doltcore/sqle/sqlselect_test.go b/go/libraries/doltcore/sqle/sqlselect_test.go index 3197b157c04..7f8fc08464d 100644 --- a/go/libraries/doltcore/sqle/sqlselect_test.go +++ b/go/libraries/doltcore/sqle/sqlselect_test.go @@ -19,7 +19,6 @@ import ( "testing" "time" - "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dtables" "github.com/dolthub/go-mysql-server/sql" gmstypes "github.com/dolthub/go-mysql-server/sql/types" "github.com/stretchr/testify/assert" @@ -33,6 +32,7 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/row" "github.com/dolthub/dolt/go/libraries/doltcore/schema" "github.com/dolthub/dolt/go/libraries/doltcore/schema/typeinfo" + "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dtables" "github.com/dolthub/dolt/go/store/datas" "github.com/dolthub/dolt/go/store/types" ) From 4bade522b43be69d3953acb6fbafc85d684034af Mon Sep 17 00:00:00 2001 From: Neil Macneale IV Date: Fri, 12 Jul 2024 16:34:23 -0700 Subject: [PATCH 11/18] Allow the conflicts table to be generated --- go/libraries/doltcore/sqle/schema_table.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/go/libraries/doltcore/sqle/schema_table.go b/go/libraries/doltcore/sqle/schema_table.go index 147f9629a78..526ad33153e 100644 --- a/go/libraries/doltcore/sqle/schema_table.go +++ b/go/libraries/doltcore/sqle/schema_table.go @@ -97,6 +97,11 @@ func (st *SchemaTable) PreciseMatch() bool { return true } +// Updater implements sql.UpdatableTable. The SchemaTable is only every modified by using it's wrapped backing table. +func (st *SchemaTable) Updater(ctx *sql.Context) sql.RowUpdater { + panic("Runtime error: SchemaTable.Updater() should never be called") +} + func (st *SchemaTable) UnWrap() *WritableDoltTable { return st.backingTable } @@ -104,6 +109,7 @@ func (st *SchemaTable) UnWrap() *WritableDoltTable { var _ sql.Table = (*SchemaTable)(nil) var _ dtables.VersionableTable = (*SchemaTable)(nil) var _ sql.IndexAddressableTable = (*SchemaTable)(nil) +var _ sql.UpdatableTable = (*SchemaTable)(nil) var _ WritableDoltTableWrapper = (*SchemaTable)(nil) func SchemaTableSqlSchema() sql.PrimaryKeySchema { From 89ea4e487d48eefce61a508a0c3ccf14fc313626 Mon Sep 17 00:00:00 2001 From: Neil Macneale IV Date: Mon, 15 Jul 2024 11:03:46 -0700 Subject: [PATCH 12/18] Fix mysql workbench tests --- .../mysql-client-tests/node/workbenchTests/views.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-tests/mysql-client-tests/node/workbenchTests/views.js b/integration-tests/mysql-client-tests/node/workbenchTests/views.js index 31fbad9c424..965a804ab03 100644 --- a/integration-tests/mysql-client-tests/node/workbenchTests/views.js +++ b/integration-tests/mysql-client-tests/node/workbenchTests/views.js @@ -11,7 +11,7 @@ export const viewsTests = [ { q: "SELECT * FROM ::tableName ::col0 LIMIT :limit OFFSET :offset", p: { tableName: "dolt_schemas", col0: "id", limit: 10, offset: 0 }, - expectedErr: "table not found: dolt_schemas", + res: [], }, { q: "CREATE VIEW ::name AS SELECT * FROM test", From 739a98f2dceed96f6a7817991f0bbd75264f1e2f Mon Sep 17 00:00:00 2001 From: Neil Macneale IV Date: Mon, 15 Jul 2024 15:21:33 -0700 Subject: [PATCH 13/18] WIP --- go/libraries/doltcore/sqle/database.go | 13 ++- go/libraries/doltcore/sqle/schema_table.go | 107 ++++++++++++++++++++- integration-tests/bats/triggers.bats | 48 ++++++++- 3 files changed, 161 insertions(+), 7 deletions(-) diff --git a/go/libraries/doltcore/sqle/database.go b/go/libraries/doltcore/sqle/database.go index 703f79cfb85..7f254221ea8 100644 --- a/go/libraries/doltcore/sqle/database.go +++ b/go/libraries/doltcore/sqle/database.go @@ -500,26 +500,31 @@ func (db Database) getTableInsensitive(ctx *sql.Context, head *doltdb.Commit, ds case doltdb.StatisticsTableName: dt, found = dtables.NewStatisticsTable(ctx, db.Name(), db.ddb, asOf), true case doltdb.ProceduresTableName: + found = true backingTable, _, err := db.getTable(ctx, root, doltdb.ProceduresTableName) if err != nil { return nil, false, err } if backingTable == nil { - dt, found = NewEmptyProceduresTable(), true + dt = NewEmptyProceduresTable() } else { writeTable := backingTable.(*WritableDoltTable) - dt, found = NewProceduresTable(writeTable), true + dt = NewProceduresTable(writeTable) } case doltdb.SchemasTableName: + found = true backingTable, _, err := db.getTable(ctx, root, doltdb.SchemasTableName) if err != nil { return nil, false, err } if backingTable == nil { - dt, found = NewEmptySchemaTable(), true + dt = NewEmptySchemaTable() } else { writeTable := backingTable.(*WritableDoltTable) - dt, found = NewSchemaTable(writeTable), true + dt, err = NewSchemaTable(ctx, db, writeTable) + if err != nil { + return nil, false, err + } } } diff --git a/go/libraries/doltcore/sqle/schema_table.go b/go/libraries/doltcore/sqle/schema_table.go index 526ad33153e..9d9033a6990 100644 --- a/go/libraries/doltcore/sqle/schema_table.go +++ b/go/libraries/doltcore/sqle/schema_table.go @@ -153,8 +153,18 @@ func NewEmptySchemaTable() sql.Table { return &SchemaTable{} } -func NewSchemaTable(backingTable *WritableDoltTable) sql.Table { - return &SchemaTable{backingTable: backingTable} +func NewSchemaTable(ctx *sql.Context, db Database, backingTable *WritableDoltTable) (sql.Table, error) { + if !backingTable.Schema().Contains(doltdb.SchemasTablesExtraCol, doltdb.SchemasTableName) { + return nil, fmt.Errorf("cannot migrate dolt_schemas table from v0.19.1 or earlier") + } else if !backingTable.Schema().Contains(doltdb.SchemasTablesSqlModeCol, doltdb.SchemasTableName) { + migrated, err := migrateOldSchemasTableToNew(ctx, db, backingTable) + if err != nil { + return nil, err + } + backingTable = migrated + } + + return &SchemaTable{backingTable: backingTable}, nil } // getOrCreateDoltSchemasTable returns the `dolt_schemas` table in `db`, creating it if it does not already exist. @@ -193,6 +203,8 @@ func getOrCreateDoltSchemasTable(ctx *sql.Context, db Database) (retTbl *Writabl // Old schemas are missing the `extra` column. Very ancient. Provide error message and bail. if !schemasTable.Schema().Contains(doltdb.SchemasTablesExtraCol, doltdb.SchemasTableName) { return nil, fmt.Errorf("cannot migrate dolt_schemas table from v0.19.1 or earlier") + } else if !schemasTable.Schema().Contains(doltdb.SchemasTablesSqlModeCol, doltdb.SchemasTableName) { + return migrateOldSchemasTableToNew(ctx, db, schemasTable) } else { return schemasTable, nil } @@ -219,6 +231,97 @@ func getOrCreateDoltSchemasTable(ctx *sql.Context, db Database) (retTbl *Writabl return wrapper.backingTable, nil } +func migrateOldSchemasTableToNew(ctx *sql.Context, db Database, schemasTable *WritableDoltTable) (newTable *WritableDoltTable, rerr error) { + // Copy all of the old data over and add an index column and an extra column + iter, err := SqlTableToRowIter(ctx, schemasTable.DoltTable, nil) + if err != nil { + return nil, err + } + + // The dolt_schemas table has undergone various changes over time and multiple possible schemas for it exist, so we + // need to get the column indexes from the current schema + nameIdx := schemasTable.sqlSchema().IndexOfColName(doltdb.SchemasTablesNameCol) + typeIdx := schemasTable.sqlSchema().IndexOfColName(doltdb.SchemasTablesTypeCol) + fragmentIdx := schemasTable.sqlSchema().IndexOfColName(doltdb.SchemasTablesFragmentCol) + extraIdx := schemasTable.sqlSchema().IndexOfColName(doltdb.SchemasTablesExtraCol) + sqlModeIdx := schemasTable.sqlSchema().IndexOfColName(doltdb.SchemasTablesSqlModeCol) + + defer func(iter sql.RowIter, ctx *sql.Context) { + err := iter.Close(ctx) + if err != nil && rerr == nil { + rerr = err + } + }(iter, ctx) + + var newRows []sql.Row + for { + sqlRow, err := iter.Next(ctx) + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + + newRow := make(sql.Row, SchemaTableSchema().GetAllCols().Size()) + newRow[0] = sqlRow[typeIdx] + newRow[1] = sqlRow[nameIdx] + newRow[2] = sqlRow[fragmentIdx] + if extraIdx >= 0 { + newRow[3] = sqlRow[extraIdx] + } + if sqlModeIdx >= 0 { + newRow[4] = sqlRow[sqlModeIdx] + } + + newRows = append(newRows, newRow) + } + + err = db.dropTable(ctx, doltdb.SchemasTableName) + if err != nil { + return nil, err + } + + root, err := db.GetRoot(ctx) + if err != nil { + return nil, err + } + + err = db.createDoltTable(ctx, doltdb.SchemasTableName, doltdb.DefaultSchemaName, root, SchemaTableSchema()) + if err != nil { + return nil, err + } + + tbl, _, err := db.GetTableInsensitive(ctx, doltdb.SchemasTableName) + if err != nil { + return nil, err + } + + wrapper, ok := tbl.(*SchemaTable) + if !ok { + return nil, fmt.Errorf("expected a SchemaTable, but found %T", tbl) + } + + if wrapper.backingTable == nil { + return nil, sql.ErrTableNotFound.New(doltdb.SchemasTableName) + } + + inserter := wrapper.backingTable.Inserter(ctx) + for _, row := range newRows { + err = inserter.Insert(ctx, row) + if err != nil { + return nil, err + } + } + + err = inserter.Close(ctx) + if err != nil { + return nil, err + } + + return wrapper.backingTable, nil +} + // fragFromSchemasTable returns the row with the given schema fragment if it exists. func fragFromSchemasTable(ctx *sql.Context, tbl *WritableDoltTable, fragType string, name string) (r sql.Row, found bool, rerr error) { fragType, name = strings.ToLower(fragType), strings.ToLower(name) diff --git a/integration-tests/bats/triggers.bats b/integration-tests/bats/triggers.bats index 3888eeaf665..813c3d1988e 100644 --- a/integration-tests/bats/triggers.bats +++ b/integration-tests/bats/triggers.bats @@ -164,4 +164,50 @@ SQL [[ "$output" =~ "trigger,trigger3,CREATE TRIGGER trigger3 BEFORE INSERT ON x FOR EACH ROW SET new.a = (new.a * 2) + 100" ]] || false [[ "$output" =~ "trigger,trigger4,CREATE TRIGGER trigger4 BEFORE INSERT ON x FOR EACH ROW SET new.a = (new.a * 2) + 1000" ]] || false [[ "${#lines[@]}" = "5" ]] || false -} \ No newline at end of file +} + +@test "triggers: Upgrade dolt_schemas" { + rm -rf .dolt + # old_dolt_schemas was created using v1.0.0, which is pre-sqlMode change + cp -a $BATS_TEST_DIRNAME/helper/old_dolt_schemas/. ./.dolt/ + + run dolt sql -q "SELECT * FROM dolt_schemas" -r=csv + [ "$status" -eq "0" ] + [[ "$output" =~ "type,name,fragment,extra" ]] || false + [[ "$output" =~ "view,view1,SELECT 2+2 FROM dual" ]] || false + [[ "${#lines[@]}" = "2" ]] || false + + run dolt sql -q "SELECT * FROM view1" -r=csv + [ "$status" -eq "0" ] + [[ "$output" =~ "2+2" ]] || false + [[ "$output" =~ "4" ]] || false + [[ "${#lines[@]}" = "2" ]] || false + + # creating a new view/trigger will recreate the dolt_schemas table + dolt sql -q "CREATE VIEW view2 AS SELECT 3+3 FROM dual;" + + skip "diff is broken on schema change" + run dolt diff + [ "$status" -eq "0" ] + [[ "$output" =~ "deleted table" ]] || false + [[ "$output" =~ "added table" ]] || false + + run dolt sql -q "SELECT * FROM dolt_schemas" -r=csv + [ "$status" -eq "0" ] + [[ "$output" =~ "type,name,fragment,id" ]] || false + [[ "$output" =~ "view,view1,CREATE VIEW view1 AS SELECT 2+2 FROM dual,1" ]] || false + [[ "$output" =~ "view,view2,CREATE VIEW view2 AS SELECT 3+3 FROM dual,2" ]] || false + [[ "${#lines[@]}" = "3" ]] || false + + run dolt sql -q "SELECT * FROM view1" -r=csv + [ "$status" -eq "0" ] + [[ "$output" =~ "2+2" ]] || false + [[ "$output" =~ "4" ]] || false + [[ "${#lines[@]}" = "2" ]] || false + + run dolt sql -q "SELECT * FROM view2" -r=csv + [ "$status" -eq "0" ] + [[ "$output" =~ "3+3" ]] || false + [[ "$output" =~ "6" ]] || false + [[ "${#lines[@]}" = "2" ]] || false +} From 779bc92581e2513a5bdd8fe786f38ede6f18f2e2 Mon Sep 17 00:00:00 2001 From: Neil Macneale IV Date: Mon, 15 Jul 2024 16:26:08 -0700 Subject: [PATCH 14/18] Bring back dolt_schemas table migration for the sql-mode column which is post v1.0 --- go/libraries/doltcore/sqle/schema_table.go | 5 +-- .../noms/5fms6l3ffldbk6t8qjumnjstmurvppk8 | Bin 0 -> 518 bytes .../bats/helper/old_dolt_schemas/noms/LOCK | 0 .../helper/old_dolt_schemas/noms/manifest | 1 + .../oldgen/6vvlbeph7glt2sskq43npe56la8p9akc | Bin 0 -> 1878 bytes .../helper/old_dolt_schemas/noms/oldgen/LOCK | 0 .../old_dolt_schemas/noms/oldgen/manifest | 1 + .../helper/old_dolt_schemas/repo_state.json | 6 +++ integration-tests/bats/triggers.bats | 41 +++--------------- 9 files changed, 17 insertions(+), 37 deletions(-) create mode 100644 integration-tests/bats/helper/old_dolt_schemas/noms/5fms6l3ffldbk6t8qjumnjstmurvppk8 create mode 100644 integration-tests/bats/helper/old_dolt_schemas/noms/LOCK create mode 100644 integration-tests/bats/helper/old_dolt_schemas/noms/manifest create mode 100644 integration-tests/bats/helper/old_dolt_schemas/noms/oldgen/6vvlbeph7glt2sskq43npe56la8p9akc create mode 100644 integration-tests/bats/helper/old_dolt_schemas/noms/oldgen/LOCK create mode 100644 integration-tests/bats/helper/old_dolt_schemas/noms/oldgen/manifest create mode 100755 integration-tests/bats/helper/old_dolt_schemas/repo_state.json diff --git a/go/libraries/doltcore/sqle/schema_table.go b/go/libraries/doltcore/sqle/schema_table.go index 9d9033a6990..5c37bd4c95e 100644 --- a/go/libraries/doltcore/sqle/schema_table.go +++ b/go/libraries/doltcore/sqle/schema_table.go @@ -267,9 +267,8 @@ func migrateOldSchemasTableToNew(ctx *sql.Context, db Database, schemasTable *Wr newRow[0] = sqlRow[typeIdx] newRow[1] = sqlRow[nameIdx] newRow[2] = sqlRow[fragmentIdx] - if extraIdx >= 0 { - newRow[3] = sqlRow[extraIdx] - } + newRow[3] = sqlRow[extraIdx] + if sqlModeIdx >= 0 { newRow[4] = sqlRow[sqlModeIdx] } diff --git a/integration-tests/bats/helper/old_dolt_schemas/noms/5fms6l3ffldbk6t8qjumnjstmurvppk8 b/integration-tests/bats/helper/old_dolt_schemas/noms/5fms6l3ffldbk6t8qjumnjstmurvppk8 new file mode 100644 index 0000000000000000000000000000000000000000..fd7fa87f96e2d904873559666e54eded86de61c2 GIT binary patch literal 518 zcmaFD_(Gb2;eh}H14D2~PzVD98v_Re3y@?0k_-$-fEXx$Kn6%Vx&-+GDInlukYEsD z5Mbb8RTJO?@|jsB7#LWEK6rWnIRQXy1H>+8U+i%6@&EZIZQf2^oeN13Oblm_7_ZX# zda627XC8zZ!B_z2@}?bech%_Ac2JN!yn_r#6QLn5S=j+!9|Q8!T1pqMHd~o7%(ca`n-dVl4#;hp1kLM z?>+B(&U2pk-2FP&B4Vm0BA++VP7)7U$wO{(k^`#>*~vym(vuZmoL*fuHq1n4Ofqv& z!AcMa1AV}LK*#0NUW~&)4d6{jR4tXB{*0uV>O|6CnG8^!D$vu+aCMk#hdj%eeL&cvY$Wt z!p4JdEjYk7|9dM=J8xg!v#IK0YJC5tgQ>>a&4;nP?H6msb5q++{d)XBZN-tn4}HGz ziI?vj`Q$4`_H^z4W&hXh@|oAqM?sRs}36BLT?eh=Nu-}LH zTLdBAfY)yXHD*L%)1zb}oH-^o>tWd|EdNT)jn;h&FYqw-7_;HCQ2Pcy1~}WWkBq9% zo!Ln2@8EJHt>gkpHdvlws*ht0eo;HdMqemksCLYq5sL_cqz%Y~4doJZ*sS1_YEwt> z+(*%4moI)`{qc6-IJt_m=jPuE>r6}8>CBhH!QS=emv}Gs9(ph)`vWntot*bWVzL;D znwadKyJ5Lmafqmhis5CHLUJ*JYUd8UnGEcNhG(BG@}=JCg6RfagDH`Wue}K*l-|z; z6bLFx@lFKf1O!VLfLp*xYzBftIt(Ezafmk?I%Fe`>oE&qc}!ckc_?e65*tU2NXw0@ zGaH*}n2M#7{Hoz7qtGmwSUwBp>S>f;H@2@226%yDyLSiF{QH}^iD{r$Ha zbf=?c{DSl8)zuc#nLO?iD#1qLs#0QirzDePSj6A^M}7_&k8P>Ca{iT%KRa{w#Mqjw z(evFE&!KuDJ&%w^lfSjeC}>7Nq=zGfC+>hy-aKGW)!jA_Zq%yQxSpkK&!L7-_1YrR z1x4t-Hg5|-X+m?B8L|s8W45eBQpiki#%oe|ei_T9WRY-`RA(-tLMoNaXhcSnn`Ij^ zCmlha8Mw?GEyzC!*ioR&Op)&OGbu!}|%=YPvhBrX8!?4=dhQzrV4)SsBoJJc%xMr|S`yt5c12cE^Zo>zHaXP|lhR=}*n$ zWt@Q+G9)L}#GhEVR%8wPkWe96c@R`BI9--;=+JBLO3JpJ_=vqHt@b7&NsVjSY>TIo zy;@|KWM0K2Tgg$jgRIgI6;ZwIOj(p!E+E(P(aNF9s3KvL$Sy6dn)VuZ^rw=%w1H{G z(Qk&rqDb5St4PyL@Do*933x-zBDD?cnl~z_ztDlw*jC)u>~HqAD|O52FhdLLABbzp z+E8mi2@j}o__?oNF3gDv(?Z|GmRUZyEIH)t_Q;|$Y~08QLZb%mU8VIHs{0( zTh6sV*dp94Fb;@6&0(MqpSv+OVC==%JJoaIoXL5$t3yW<`M-kvzeWDPAiu5Vcc=VI ZlCS^#3x)Fx03NyH?>DZ!Tity#_z$8ot(gD- literal 0 HcmV?d00001 diff --git a/integration-tests/bats/helper/old_dolt_schemas/noms/oldgen/LOCK b/integration-tests/bats/helper/old_dolt_schemas/noms/oldgen/LOCK new file mode 100644 index 00000000000..e69de29bb2d diff --git a/integration-tests/bats/helper/old_dolt_schemas/noms/oldgen/manifest b/integration-tests/bats/helper/old_dolt_schemas/noms/oldgen/manifest new file mode 100644 index 00000000000..d36d7f961a5 --- /dev/null +++ b/integration-tests/bats/helper/old_dolt_schemas/noms/oldgen/manifest @@ -0,0 +1 @@ +5:__DOLT__:9trs9rc9jmja9h13v3bqrf1k8i8pvjie:00000000000000000000000000000000:00000000000000000000000000000000:6vvlbeph7glt2sskq43npe56la8p9akc:9 \ No newline at end of file diff --git a/integration-tests/bats/helper/old_dolt_schemas/repo_state.json b/integration-tests/bats/helper/old_dolt_schemas/repo_state.json new file mode 100755 index 00000000000..32f2d6197d5 --- /dev/null +++ b/integration-tests/bats/helper/old_dolt_schemas/repo_state.json @@ -0,0 +1,6 @@ +{ + "head": "refs/heads/main", + "remotes": {}, + "backups": {}, + "branches": {} +} \ No newline at end of file diff --git a/integration-tests/bats/triggers.bats b/integration-tests/bats/triggers.bats index 813c3d1988e..705db4a61e6 100644 --- a/integration-tests/bats/triggers.bats +++ b/integration-tests/bats/triggers.bats @@ -171,43 +171,16 @@ SQL # old_dolt_schemas was created using v1.0.0, which is pre-sqlMode change cp -a $BATS_TEST_DIRNAME/helper/old_dolt_schemas/. ./.dolt/ - run dolt sql -q "SELECT * FROM dolt_schemas" -r=csv - [ "$status" -eq "0" ] - [[ "$output" =~ "type,name,fragment,extra" ]] || false - [[ "$output" =~ "view,view1,SELECT 2+2 FROM dual" ]] || false - [[ "${#lines[@]}" = "2" ]] || false - - run dolt sql -q "SELECT * FROM view1" -r=csv - [ "$status" -eq "0" ] - [[ "$output" =~ "2+2" ]] || false - [[ "$output" =~ "4" ]] || false - [[ "${#lines[@]}" = "2" ]] || false - - # creating a new view/trigger will recreate the dolt_schemas table - dolt sql -q "CREATE VIEW view2 AS SELECT 3+3 FROM dual;" - - skip "diff is broken on schema change" - run dolt diff - [ "$status" -eq "0" ] - [[ "$output" =~ "deleted table" ]] || false - [[ "$output" =~ "added table" ]] || false + # The column will automatically be added if it doesn't exist in the original schema, + # and just by selecting on the table, you will dirty the workspace. run dolt sql -q "SELECT * FROM dolt_schemas" -r=csv [ "$status" -eq "0" ] - [[ "$output" =~ "type,name,fragment,id" ]] || false - [[ "$output" =~ "view,view1,CREATE VIEW view1 AS SELECT 2+2 FROM dual,1" ]] || false - [[ "$output" =~ "view,view2,CREATE VIEW view2 AS SELECT 3+3 FROM dual,2" ]] || false - [[ "${#lines[@]}" = "3" ]] || false + [[ "$output" =~ "type,name,fragment,extra,sql_mode" ]] || false + [[ "$output" =~ "view,my_view" ]] || false + [[ "$output" =~ "SELECT 2+2" ]] || false - run dolt sql -q "SELECT * FROM view1" -r=csv + run dolt status [ "$status" -eq "0" ] - [[ "$output" =~ "2+2" ]] || false - [[ "$output" =~ "4" ]] || false - [[ "${#lines[@]}" = "2" ]] || false - - run dolt sql -q "SELECT * FROM view2" -r=csv - [ "$status" -eq "0" ] - [[ "$output" =~ "3+3" ]] || false - [[ "$output" =~ "6" ]] || false - [[ "${#lines[@]}" = "2" ]] || false + [[ "$output" =~ "modified: dolt_schemas" ]] || false } From 5524d82ee8b0773a243f3d8e41a326ded0da9975 Mon Sep 17 00:00:00 2001 From: Neil Macneale IV Date: Mon, 15 Jul 2024 16:51:28 -0700 Subject: [PATCH 15/18] Fix a golang test related to dolt_schema migration --- .../doltcore/sqle/schema_table_test.go | 44 ++++++++++++------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/go/libraries/doltcore/sqle/schema_table_test.go b/go/libraries/doltcore/sqle/schema_table_test.go index 53e4cc3ceba..533b38bb9f2 100644 --- a/go/libraries/doltcore/sqle/schema_table_test.go +++ b/go/libraries/doltcore/sqle/schema_table_test.go @@ -20,7 +20,6 @@ import ( "github.com/dolthub/go-mysql-server/sql" gmstypes "github.com/dolthub/go-mysql-server/sql/types" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" @@ -46,25 +45,38 @@ func TestAncientSchemaTableError(t *testing.T) { }), sql.Collation_Default, "") require.NoError(t, err) - sqlTbl, found, err := db.GetTableInsensitive(ctx, doltdb.SchemasTableName) - require.NoError(t, err) - require.True(t, found) + _, _, err = db.GetTableInsensitive(ctx, doltdb.SchemasTableName) + require.Error(t, err) + require.Contains(t, err.Error(), "cannot migrate dolt_schemas table from v0.19.1 or earlier") - wrapper, ok := sqlTbl.(*SchemaTable) - require.True(t, ok) - require.NotNil(t, wrapper.backingTable) +} - inserter := wrapper.backingTable.Inserter(ctx) - err = inserter.Insert(ctx, sql.Row{"view", "view1", "SELECT v1 FROM test;"}) +func TestV1SchemasTable(t *testing.T) { + dEnv := dtestutils.CreateTestEnv() + tmpDir, err := dEnv.TempTableFilesDir() require.NoError(t, err) - err = inserter.Insert(ctx, sql.Row{"view", "view2", "SELECT v2 FROM test;"}) + opts := editor.Options{Deaf: dEnv.DbEaFactory(), Tempdir: tmpDir} + db, err := NewDatabase(context.Background(), "dolt", dEnv.DbData(), opts) require.NoError(t, err) - err = inserter.Close(ctx) + + _, ctx, err := NewTestEngine(dEnv, context.Background(), db) require.NoError(t, err) - // Ancient dolt_schemas table. Verify error. - tbl, err := getOrCreateDoltSchemasTable(ctx, db) // - require.Error(t, err) - assert.Contains(t, err.Error(), "cannot migrate dolt_schemas table") - require.Nil(t, tbl) + err = db.createSqlTable(ctx, doltdb.SchemasTableName, "", sql.NewPrimaryKeySchema(sql.Schema{ // original schema of dolt_schemas table + {Name: doltdb.SchemasTablesTypeCol, Type: gmstypes.Text, Source: doltdb.SchemasTableName, PrimaryKey: true}, + {Name: doltdb.SchemasTablesNameCol, Type: gmstypes.Text, Source: doltdb.SchemasTableName, PrimaryKey: true}, + {Name: doltdb.SchemasTablesFragmentCol, Type: gmstypes.Text, Source: doltdb.SchemasTableName, PrimaryKey: false}, + {Name: doltdb.SchemasTablesExtraCol, Type: gmstypes.JSON, Source: doltdb.SchemasTableName, PrimaryKey: false}, + }), sql.Collation_Default, "") + require.NoError(t, err) + + tbl, _, err := db.GetTableInsensitive(ctx, doltdb.SchemasTableName) + require.NoError(t, err) + + wrapper, ok := tbl.(*SchemaTable) + require.True(t, ok) + require.NotNil(t, wrapper.backingTable) + + // Comparing the full schema is awkward because the are different instacne. So we'll just compare the new column name. + require.Equal(t, SchemaTableSqlSchema().Schema[4].Name, wrapper.backingTable.Schema()[4].Name) } From 88813997f545b5c89c000b1c7936aa93377cb7cd Mon Sep 17 00:00:00 2001 From: Neil Macneale IV Date: Tue, 16 Jul 2024 08:46:25 -0700 Subject: [PATCH 16/18] Late bind the schema on the wrapper object for dolt_schemas table --- go/libraries/doltcore/sqle/schema_table.go | 33 ++++++++++++++++++---- integration-tests/bats/triggers.bats | 12 ++++---- 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/go/libraries/doltcore/sqle/schema_table.go b/go/libraries/doltcore/sqle/schema_table.go index 5c37bd4c95e..e3b20ed7083 100644 --- a/go/libraries/doltcore/sqle/schema_table.go +++ b/go/libraries/doltcore/sqle/schema_table.go @@ -56,6 +56,16 @@ func (st *SchemaTable) String() string { } func (st *SchemaTable) Schema() sql.Schema { + if st.backingTable == nil { + // No backing table; return an current schema. + return SchemaTableSqlSchema().Schema + } + + if !st.backingTable.Schema().Contains(doltdb.SchemasTablesSqlModeCol, doltdb.SchemasTableName) { + // No SQL_MODE column; return an old schema. + return SchemaTableV1SqlSchema() + } + return SchemaTableSqlSchema().Schema } @@ -136,6 +146,23 @@ func mustCreateStringType(baseType query.Type, length int64, collation sql.Colla return ti } +// dolt_schemas columns, for dolt_schemas of v1.0.0. +func SchemaTableV1SqlSchema() sql.Schema { + var schemasTableCols = schema.NewColCollection( + mustNewColWithTypeInfo(doltdb.SchemasTablesTypeCol, schema.DoltSchemasTypeTag, typeinfo.CreateVarStringTypeFromSqlType(mustCreateStringType(query.Type_VARCHAR, 64, sql.Collation_utf8mb4_0900_ai_ci)), true, "", false, ""), + mustNewColWithTypeInfo(doltdb.SchemasTablesNameCol, schema.DoltSchemasNameTag, typeinfo.CreateVarStringTypeFromSqlType(mustCreateStringType(query.Type_VARCHAR, 64, sql.Collation_utf8mb4_0900_ai_ci)), true, "", false, ""), + mustNewColWithTypeInfo(doltdb.SchemasTablesFragmentCol, schema.DoltSchemasFragmentTag, typeinfo.CreateVarStringTypeFromSqlType(gmstypes.LongText), false, "", false, ""), + mustNewColWithTypeInfo(doltdb.SchemasTablesExtraCol, schema.DoltSchemasExtraTag, typeinfo.JSONType, false, "", false, ""), + ) + + legacy := schema.MustSchemaFromCols(schemasTableCols) + sqlSchema, err := sqlutil.FromDoltSchema("", doltdb.SchemasTableName, legacy) + if err != nil { + panic(err) // should never happen + } + return sqlSchema.Schema +} + // dolt_schemas columns func SchemaTableSchema() schema.Schema { var schemasTableCols = schema.NewColCollection( @@ -156,12 +183,6 @@ func NewEmptySchemaTable() sql.Table { func NewSchemaTable(ctx *sql.Context, db Database, backingTable *WritableDoltTable) (sql.Table, error) { if !backingTable.Schema().Contains(doltdb.SchemasTablesExtraCol, doltdb.SchemasTableName) { return nil, fmt.Errorf("cannot migrate dolt_schemas table from v0.19.1 or earlier") - } else if !backingTable.Schema().Contains(doltdb.SchemasTablesSqlModeCol, doltdb.SchemasTableName) { - migrated, err := migrateOldSchemasTableToNew(ctx, db, backingTable) - if err != nil { - return nil, err - } - backingTable = migrated } return &SchemaTable{backingTable: backingTable}, nil diff --git a/integration-tests/bats/triggers.bats b/integration-tests/bats/triggers.bats index 705db4a61e6..79b2d998f25 100644 --- a/integration-tests/bats/triggers.bats +++ b/integration-tests/bats/triggers.bats @@ -168,19 +168,19 @@ SQL @test "triggers: Upgrade dolt_schemas" { rm -rf .dolt + # old_dolt_schemas was created using v1.0.0, which is pre-sqlMode change cp -a $BATS_TEST_DIRNAME/helper/old_dolt_schemas/. ./.dolt/ - # The column will automatically be added if it doesn't exist in the original schema, - # and just by selecting on the table, you will dirty the workspace. - run dolt sql -q "SELECT * FROM dolt_schemas" -r=csv [ "$status" -eq "0" ] - [[ "$output" =~ "type,name,fragment,extra,sql_mode" ]] || false + [[ "$output" =~ "type,name,fragment,extra" ]] || false [[ "$output" =~ "view,my_view" ]] || false [[ "$output" =~ "SELECT 2+2" ]] || false - run dolt status + dolt sql -q "CREATE VIEW another_view AS SELECT 3+3" + + run dolt sql -q "SELECT * FROM dolt_schemas" -r=csv [ "$status" -eq "0" ] - [[ "$output" =~ "modified: dolt_schemas" ]] || false + [[ "$output" =~ "type,name,fragment,extra,sql_mode" ]] || false } From 7be3a1a0ae9bf30c403249513ea0dac5cf03bb56 Mon Sep 17 00:00:00 2001 From: Neil Macneale IV <46170177+macneale4@users.noreply.github.com> Date: Tue, 16 Jul 2024 09:35:05 -0700 Subject: [PATCH 17/18] Update go/libraries/doltcore/sqle/tables.go Co-authored-by: Jason Fulghum --- go/libraries/doltcore/sqle/tables.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/libraries/doltcore/sqle/tables.go b/go/libraries/doltcore/sqle/tables.go index 8fbccd4f6a2..6596a815910 100644 --- a/go/libraries/doltcore/sqle/tables.go +++ b/go/libraries/doltcore/sqle/tables.go @@ -540,7 +540,7 @@ var _ doltTableInterface = (*WritableDoltTable)(nil) // fake table. Specifically, databases.getTableInsensitive will returns an sql.Table, and there are cases where we // want to return a table that hasn't been materialized yet. type WritableDoltTableWrapper interface { - // Return the underlying WritableDoltTable, nil returns are expected when the wrapped table hasn't been materialized + // Unwrap returns the underlying WritableDoltTable, nil returns are expected when the wrapped table hasn't been materialized UnWrap() *WritableDoltTable } From 25d1c98d84a63c395f93f12e6ba4c990b1702306 Mon Sep 17 00:00:00 2001 From: Neil Macneale IV Date: Tue, 16 Jul 2024 09:44:34 -0700 Subject: [PATCH 18/18] One more rev to fix a test --- .../doltcore/sqle/schema_table_test.go | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/go/libraries/doltcore/sqle/schema_table_test.go b/go/libraries/doltcore/sqle/schema_table_test.go index 533b38bb9f2..cb115bede76 100644 --- a/go/libraries/doltcore/sqle/schema_table_test.go +++ b/go/libraries/doltcore/sqle/schema_table_test.go @@ -77,6 +77,23 @@ func TestV1SchemasTable(t *testing.T) { require.True(t, ok) require.NotNil(t, wrapper.backingTable) - // Comparing the full schema is awkward because the are different instacne. So we'll just compare the new column name. - require.Equal(t, SchemaTableSqlSchema().Schema[4].Name, wrapper.backingTable.Schema()[4].Name) + // unmodified dolt_schemas table. + require.Equal(t, 4, len(wrapper.backingTable.Schema())) + + tbl, err = getOrCreateDoltSchemasTable(ctx, db) + require.NoError(t, err) + require.NotNil(t, tbl) + + // modified dolt_schemas table. + require.Equal(t, 5, len(tbl.Schema())) + + tbl, _, err = db.GetTableInsensitive(ctx, doltdb.SchemasTableName) + require.NoError(t, err) + wrapper, ok = tbl.(*SchemaTable) + require.True(t, ok) + require.NotNil(t, wrapper.backingTable) + + // modified dolt_schemas table. + require.Equal(t, 5, len(wrapper.backingTable.Schema())) + }