Merge pull request #8474 from dolthub/nicktobey/secondary

[no-release-notes] Replace ConvertToSecondaryKeylessIndex with AddHashToSchema

go/cmd/dolt/commands/schcmds/import.go

@@ -330,7 +330,7 @@ func putEmptyTableWithSchema(ctx context.Context, tblName string, root doltdb.Ro
 		return nil, errhand.BuildDError("error: failed to get table.").AddCause(err).Build()
 	}
 
-	empty, err := durable.NewEmptyIndex(ctx, root.VRW(), root.NodeStore(), sch)
+	empty, err := durable.NewEmptyIndex(ctx, root.VRW(), root.NodeStore(), sch, false)
 	if err != nil {
 		return nil, errhand.BuildDError("error: failed to get table.").AddCause(err).Build()
 	}

go/libraries/doltcore/doltdb/commit_hooks_test.go

@@ -251,7 +251,7 @@ func TestAsyncPushOnWrite(t *testing.T) {
 			assert.NoError(t, err)
 
 			tSchema := createTestSchema(t)
-			rowData, err := durable.NewEmptyIndex(ctx, ddb.vrw, ddb.ns, tSchema)
+			rowData, err := durable.NewEmptyIndex(ctx, ddb.vrw, ddb.ns, tSchema, false)
 			require.NoError(t, err)
 			tbl, err := CreateTestTable(ddb.vrw, ddb.ns, tSchema, rowData)
 			require.NoError(t, err)

go/libraries/doltcore/doltdb/doltdb_test.go

@@ -83,7 +83,7 @@ func CreateTestTable(vrw types.ValueReadWriter, ns tree.NodeStore, tSchema schem
 
 func createTestRowData(t *testing.T, vrw types.ValueReadWriter, ns tree.NodeStore, sch schema.Schema) durable.Index {
 	if types.Format_Default == types.Format_DOLT {
-		idx, err := durable.NewEmptyIndex(context.Background(), vrw, ns, sch)
+		idx, err := durable.NewEmptyIndex(context.Background(), vrw, ns, sch, false)
 		require.NoError(t, err)
 		return idx
 	}
@@ -303,7 +303,7 @@ func TestLDNoms(t *testing.T) {
 
 		ctx := context.Background()
 		tSchema := createTestSchema(t)
-		rowData, err := durable.NewEmptyIndex(ctx, ddb.vrw, ddb.ns, tSchema)
+		rowData, err := durable.NewEmptyIndex(ctx, ddb.vrw, ddb.ns, tSchema, false)
 		if err != nil {
 			t.Fatal("Failed to create new empty index")
 		}

go/libraries/doltcore/doltdb/durable/index.go

@@ -96,11 +96,12 @@ func RefFromIndex(ctx context.Context, vrw types.ValueReadWriter, idx Index) (ty
 }
 
 // indexFromRef reads the types.Ref from storage and returns the Index it points to.
+// This is only used by noms format and can be removed.
 func indexFromRef(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, sch schema.Schema, r types.Ref) (Index, error) {
-	return indexFromAddr(ctx, vrw, ns, sch, r.TargetHash())
+	return indexFromAddr(ctx, vrw, ns, sch, r.TargetHash(), false)
 }
 
-func indexFromAddr(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, sch schema.Schema, addr hash.Hash) (Index, error) {
+func indexFromAddr(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, sch schema.Schema, addr hash.Hash, isKeylessTable bool) (Index, error) {
 	v, err := vrw.ReadValue(ctx, addr)
 	if err != nil {
 		return nil, err
@@ -111,7 +112,7 @@ func indexFromAddr(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeS
 		return IndexFromNomsMap(v.(types.Map), vrw, ns), nil
 
 	case types.Format_DOLT:
-		pm, err := shim.MapFromValue(v, sch, ns)
+		pm, err := shim.MapFromValue(v, sch, ns, isKeylessTable)
 		if err != nil {
 			return nil, err
 		}
@@ -123,7 +124,7 @@ func indexFromAddr(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeS
 }
 
 // NewEmptyIndex returns an index with no rows.
-func NewEmptyIndex(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, sch schema.Schema) (Index, error) {
+func NewEmptyIndex(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, sch schema.Schema, isKeylessSecondary bool) (Index, error) {
 	switch vrw.Format() {
 	case types.Format_LD_1:
 		m, err := types.NewMap(ctx, vrw)
@@ -134,6 +135,9 @@ func NewEmptyIndex(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeS
 
 	case types.Format_DOLT:
 		kd, vd := sch.GetMapDescriptors()
+		if isKeylessSecondary {
+			kd = prolly.AddHashToSchema(kd)
+		}
 		m, err := prolly.NewMapFromTuples(ctx, ns, kd, vd)
 		if err != nil {
 			return nil, err
@@ -370,7 +374,7 @@ func NewIndexSetWithEmptyIndexes(ctx context.Context, vrw types.ValueReadWriter,
 		return nil, err
 	}
 	for _, index := range sch.Indexes().AllIndexes() {
-		empty, err := NewEmptyIndex(ctx, vrw, ns, index.Schema())
+		empty, err := NewEmptyIndex(ctx, vrw, ns, index.Schema(), false)
 		if err != nil {
 			return nil, err
 		}
@@ -507,7 +511,7 @@ func (is doltDevIndexSet) GetIndex(ctx context.Context, tableSch schema.Schema,
 	if idxSch == nil {
 		idxSch = idx.Schema()
 	}
-	return indexFromAddr(ctx, is.vrw, is.ns, idxSch, foundAddr)
+	return indexFromAddr(ctx, is.vrw, is.ns, idxSch, foundAddr, schema.IsKeyless(tableSch))
 }
 
 func (is doltDevIndexSet) PutIndex(ctx context.Context, name string, idx Index) (IndexSet, error) {

go/libraries/doltcore/doltdb/durable/table.go

@@ -850,7 +850,7 @@ func (t doltDevTable) GetTableRows(ctx context.Context) (Index, error) {
 	if err != nil {
 		return nil, err
 	}
-	m, err := shim.MapFromValue(types.SerialMessage(rowbytes), sch, t.ns)
+	m, err := shim.MapFromValue(types.SerialMessage(rowbytes), sch, t.ns, false)
 	if err != nil {
 		return nil, err
 	}

go/libraries/doltcore/doltdb/root_val.go

@@ -908,7 +908,7 @@ func (root *rootValue) putTable(ctx context.Context, tName TableName, ref types.
 func CreateEmptyTable(ctx context.Context, root RootValue, tName TableName, sch schema.Schema) (RootValue, error) {
 	ns := root.NodeStore()
 	vrw := root.VRW()
-	empty, err := durable.NewEmptyIndex(ctx, vrw, ns, sch)
+	empty, err := durable.NewEmptyIndex(ctx, vrw, ns, sch, false)
 	if err != nil {
 		return nil, err
 	}

go/libraries/doltcore/doltdb/table.go

@@ -92,7 +92,7 @@ func NewTableFromDurable(table durable.Table) *Table {
 }
 
 func NewEmptyTable(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, sch schema.Schema) (*Table, error) {
-	rows, err := durable.NewEmptyIndex(ctx, vrw, ns, sch)
+	rows, err := durable.NewEmptyIndex(ctx, vrw, ns, sch, false)
 	if err != nil {
 		return nil, err
 	}

go/libraries/doltcore/dtestutils/testcommands/multienv.go

@@ -390,7 +390,7 @@ func createTestTable(dEnv *env.DoltEnv, tableName string, sch schema.Schema) err
 	vrw := dEnv.DoltDB.ValueReadWriter()
 	ns := dEnv.DoltDB.NodeStore()
 
-	idx, err := durable.NewEmptyIndex(ctx, vrw, ns, sch)
+	idx, err := durable.NewEmptyIndex(ctx, vrw, ns, sch, false)
 	if err != nil {
 		return err
 	}

go/libraries/doltcore/merge/fulltext_rebuild.go

@@ -330,7 +330,7 @@ func purgeFulltextTableData(ctx *sql.Context, root doltdb.RootValue, tableNames
 		if err != nil {
 			return nil, err
 		}
-		rows, err := durable.NewEmptyIndex(ctx, tbl.ValueReadWriter(), tbl.NodeStore(), sch)
+		rows, err := durable.NewEmptyIndex(ctx, tbl.ValueReadWriter(), tbl.NodeStore(), sch, false)
 		if err != nil {
 			return nil, err
 		}

go/libraries/doltcore/merge/merge_prolly_indexes.go

@@ -60,9 +60,6 @@ func mergeProllySecondaryIndexes(
 				return prolly.Map{}, false, err
 			}
 			m := durable.ProllyMapFromIndex(idx)
-			if schema.IsKeyless(sch) {
-				m = prolly.ConvertToSecondaryKeylessIndex(m)
-			}
 			return m, true, nil
 		}
 		return prolly.Map{}, false, nil

go/libraries/doltcore/merge/merge_prolly_rows.go

@@ -715,9 +715,6 @@ func newUniqIndex(ctx *sql.Context, sch schema.Schema, tableName string, def sch
 		return uniqIndex{}, err
 	}
 
-	if schema.IsKeyless(sch) { // todo(andy): sad panda
-		secondary = prolly.ConvertToSecondaryKeylessIndex(secondary)
-	}
 	p := clustered.Pool()
 
 	prefixDesc := secondary.KeyDesc().PrefixDesc(def.Count())

go/libraries/doltcore/merge/mutable_secondary_index.go

@@ -35,9 +35,6 @@ func GetMutableSecondaryIdxs(ctx *sql.Context, ourSch, sch schema.Schema, tableN
 			return nil, err
 		}
 		m := durable.ProllyMapFromIndex(idx)
-		if schema.IsKeyless(sch) {
-			m = prolly.ConvertToSecondaryKeylessIndex(m)
-		}
 		mods[i], err = NewMutableSecondaryIdx(ctx, m, ourSch, sch, tableName, index)
 		if err != nil {
 			return nil, err
@@ -76,17 +73,19 @@ func GetMutableSecondaryIdxsWithPending(ctx *sql.Context, ourSch, sch schema.Sch
 		// TODO: This isn't technically required, but correctly handling updating secondary indexes when only some
 		// of the table's rows have been updated is difficult to get right.
 		// Dropping the index is potentially slower but guaranteed to be correct.
-		if !m.KeyDesc().Equals(index.Schema().GetKeyDescriptorWithNoConversion()) {
+		idxKeyDesc := m.KeyDesc()
+		if schema.IsKeyless(sch) {
+			idxKeyDesc = idxKeyDesc.PrefixDesc(idxKeyDesc.Count() - 1)
+		}
+
+		if !idxKeyDesc.Equals(index.Schema().GetKeyDescriptorWithNoConversion()) {
 			continue
 		}
 
 		if !m.ValDesc().Equals(index.Schema().GetValueDescriptor()) {
 			continue
 		}
 
-		if schema.IsKeyless(sch) {
-			m = prolly.ConvertToSecondaryKeylessIndex(m)
-		}
 		newMutableSecondaryIdx, err := NewMutableSecondaryIdx(ctx, m, ourSch, sch, tableName, index)
 		if err != nil {
 			return nil, err

go/libraries/doltcore/merge/violations_fk.go

@@ -107,7 +107,7 @@ func GetForeignKeyViolations(ctx context.Context, newRoot, baseRoot doltdb.RootV
 				return err
 			}
 			// Parent does not exist in the ancestor so we use an empty map
-			emptyIdx, err := durable.NewEmptyIndex(ctx, postParent.Table.ValueReadWriter(), postParent.Table.NodeStore(), postParent.Schema)
+			emptyIdx, err := durable.NewEmptyIndex(ctx, postParent.Table.ValueReadWriter(), postParent.Table.NodeStore(), postParent.Schema, false)
 			if err != nil {
 				return err
 			}
@@ -129,7 +129,7 @@ func GetForeignKeyViolations(ctx context.Context, newRoot, baseRoot doltdb.RootV
 				return err
 			}
 			// Child does not exist in the ancestor so we use an empty map
-			emptyIdx, err := durable.NewEmptyIndex(ctx, postChild.Table.ValueReadWriter(), postChild.Table.NodeStore(), postChild.Schema)
+			emptyIdx, err := durable.NewEmptyIndex(ctx, postChild.Table.ValueReadWriter(), postChild.Table.NodeStore(), postChild.Schema, false)
 			if err != nil {
 				return err
 			}
@@ -370,7 +370,7 @@ func parentFkConstraintViolations(
 	}
 	var idx durable.Index
 	if empty {
-		idx, err = durable.NewEmptyIndex(ctx, postChild.Table.ValueReadWriter(), postParent.Table.NodeStore(), postParent.Schema)
+		idx, err = durable.NewEmptyIndex(ctx, postChild.Table.ValueReadWriter(), postParent.Table.NodeStore(), postParent.Schema, false)
 		if err != nil {
 			return err
 		}
@@ -405,7 +405,7 @@ func childFkConstraintViolations(
 	}
 	var idx durable.Index
 	if empty {
-		idx, err = durable.NewEmptyIndex(ctx, postChild.Table.ValueReadWriter(), postChild.Table.NodeStore(), postChild.Schema)
+		idx, err = durable.NewEmptyIndex(ctx, postChild.Table.ValueReadWriter(), postChild.Table.NodeStore(), postChild.Schema, false)
 		if err != nil {
 			return err
 		}

go/libraries/doltcore/sqle/alterschema_test.go

@@ -269,7 +269,7 @@ func makePeopleTable(ctx context.Context, dEnv *env.DoltEnv) (*env.DoltEnv, erro
 	if err != nil {
 		return nil, err
 	}
-	rows, err := durable.NewEmptyIndex(ctx, root.VRW(), root.NodeStore(), sch)
+	rows, err := durable.NewEmptyIndex(ctx, root.VRW(), root.NodeStore(), sch, false)
 	if err != nil {
 		return nil, err
 	}

go/libraries/doltcore/sqle/common_test.go

@@ -181,7 +181,7 @@ func CreateTestTable(t *testing.T, dEnv *env.DoltEnv, tableName string, sch sche
 	vrw := dEnv.DoltDB.ValueReadWriter()
 	ns := dEnv.DoltDB.NodeStore()
 
-	rows, err := durable.NewEmptyIndex(ctx, vrw, ns, sch)
+	rows, err := durable.NewEmptyIndex(ctx, vrw, ns, sch, false)
 	require.NoError(t, err)
 	tbl, err := doltdb.NewTable(ctx, vrw, ns, sch, rows, nil, nil)
 	require.NoError(t, err)

go/libraries/doltcore/sqle/dtables/conflicts_tables_prolly.go

@@ -416,7 +416,7 @@ func (itr *prollyConflictRowIter) loadTableMaps(ctx *sql.Context, baseHash, thei
 
 		var idx durable.Index
 		if !ok {
-			idx, err = durable.NewEmptyIndex(ctx, itr.vrw, itr.ns, itr.ourSch)
+			idx, err = durable.NewEmptyIndex(ctx, itr.vrw, itr.ns, itr.ourSch, false)
 		} else {
 			idx, err = baseTbl.GetRowData(ctx)
 		}

go/libraries/doltcore/sqle/dtables/diff_table.go

@@ -595,7 +595,7 @@ func tableData(ctx *sql.Context, tbl *doltdb.Table, ddb *doltdb.DoltDB) (durable
 	var err error
 
 	if tbl == nil {
-		data, err = durable.NewEmptyIndex(ctx, ddb.ValueReadWriter(), ddb.NodeStore(), schema.EmptySchema)
+		data, err = durable.NewEmptyIndex(ctx, ddb.ValueReadWriter(), ddb.NodeStore(), schema.EmptySchema, false)
 		if err != nil {
 			return nil, nil, err
 		}

go/libraries/doltcore/sqle/enginetest/validation.go

@@ -175,7 +175,6 @@ func validateKeylessIndex(ctx context.Context, sch schema.Schema, def schema.Ind
 		return nil
 	}
 
-	secondary = prolly.ConvertToSecondaryKeylessIndex(secondary)
 	idxDesc, _ := secondary.Descriptors()
 	builder := val.NewTupleBuilder(idxDesc)
 	mapping := ordinalMappingsForSecondaryIndex(sch, def)

go/libraries/doltcore/sqle/tables.go

@@ -932,7 +932,7 @@ func emptyFulltextTable(
 		return nil, nil, err
 	}
 
-	empty, err := durable.NewEmptyIndex(ctx, dt.ValueReadWriter(), dt.NodeStore(), doltSchema)
+	empty, err := durable.NewEmptyIndex(ctx, dt.ValueReadWriter(), dt.NodeStore(), doltSchema, false)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -1036,7 +1036,7 @@ func (t *WritableDoltTable) truncate(
 	}
 
 	for _, idx := range sch.Indexes().AllIndexes() {
-		empty, err := durable.NewEmptyIndex(ctx, table.ValueReadWriter(), table.NodeStore(), idx.Schema())
+		empty, err := durable.NewEmptyIndex(ctx, table.ValueReadWriter(), table.NodeStore(), idx.Schema(), false)
 		if err != nil {
 			return nil, err
 		}
@@ -1060,7 +1060,7 @@ func (t *WritableDoltTable) truncate(
 		}
 	}
 
-	empty, err := durable.NewEmptyIndex(ctx, table.ValueReadWriter(), table.NodeStore(), sch)
+	empty, err := durable.NewEmptyIndex(ctx, table.ValueReadWriter(), table.NodeStore(), sch, false)
 	if err != nil {
 		return nil, err
 	}

go/libraries/doltcore/sqle/temp_table.go

@@ -109,7 +109,7 @@ func NewTempTable(
 	vrw := ddb.ValueReadWriter()
 	ns := ddb.NodeStore()
 
-	idx, err := durable.NewEmptyIndex(ctx, vrw, ns, sch)
+	idx, err := durable.NewEmptyIndex(ctx, vrw, ns, sch, false)
 	if err != nil {
 		return nil, err
 	}

go/libraries/doltcore/sqle/testutil.go

@@ -425,7 +425,7 @@ func CreateEmptyTestTable(dEnv *env.DoltEnv, tableName string, sch schema.Schema
 	vrw := dEnv.DoltDB.ValueReadWriter()
 	ns := dEnv.DoltDB.NodeStore()
 
-	rows, err := durable.NewEmptyIndex(ctx, vrw, ns, sch)
+	rows, err := durable.NewEmptyIndex(ctx, vrw, ns, sch, false)
 	if err != nil {
 		return err
 	}

go/libraries/doltcore/sqle/writer/prolly_table_writer.go

@@ -26,7 +26,6 @@ import (
 	"github.com/dolthub/dolt/go/libraries/doltcore/sqle/globalstate"
 	"github.com/dolthub/dolt/go/libraries/doltcore/sqle/index"
 	"github.com/dolthub/dolt/go/store/pool"
-	"github.com/dolthub/dolt/go/store/prolly"
 	"github.com/dolthub/dolt/go/store/val"
 )
 
@@ -118,7 +117,6 @@ func getSecondaryKeylessProllyWriters(ctx context.Context, t *doltdb.Table, schS
 			return nil, err
 		}
 		m := durable.ProllyMapFromIndex(idxRows)
-		m = prolly.ConvertToSecondaryKeylessIndex(m)
 
 		keyDesc, _ := m.Descriptors()
 

go/libraries/doltcore/table/editor/creation/external_build_index.go

@@ -41,14 +41,11 @@ const (
 // single prolly tree materialization by presorting the index keys in an
 // intermediate file format.
 func BuildProllyIndexExternal(ctx *sql.Context, vrw types.ValueReadWriter, ns tree.NodeStore, sch schema.Schema, tableName string, idx schema.Index, primary prolly.Map, uniqCb DupEntryCb) (durable.Index, error) {
-	empty, err := durable.NewEmptyIndex(ctx, vrw, ns, idx.Schema())
+	empty, err := durable.NewEmptyIndex(ctx, vrw, ns, idx.Schema(), schema.IsKeyless(sch))
 	if err != nil {
 		return nil, err
 	}
 	secondary := durable.ProllyMapFromIndex(empty)
-	if schema.IsKeyless(sch) {
-		secondary = prolly.ConvertToSecondaryKeylessIndex(secondary)
-	}
 
 	iter, err := primary.IterAll(ctx)
 	if err != nil {

go/libraries/doltcore/table/editor/creation/index.go

@@ -214,14 +214,11 @@ func BuildUniqueProllyIndex(
 	primary prolly.Map,
 	cb DupEntryCb,
 ) (durable.Index, error) {
-	empty, err := durable.NewEmptyIndex(ctx, vrw, ns, idx.Schema())
+	empty, err := durable.NewEmptyIndex(ctx, vrw, ns, idx.Schema(), schema.IsKeyless(sch))
 	if err != nil {
 		return nil, err
 	}
 	secondary := durable.ProllyMapFromIndex(empty)
-	if schema.IsKeyless(sch) {
-		secondary = prolly.ConvertToSecondaryKeylessIndex(secondary)
-	}
 
 	iter, err := primary.IterAll(ctx)
 	if err != nil {

go/libraries/doltcore/table/untyped/sqlexport/sqlwriter_test.go

@@ -92,7 +92,7 @@ func TestEndToEnd(t *testing.T) {
 			root, err := dEnv.WorkingRoot(ctx)
 			require.NoError(t, err)
 
-			empty, err := durable.NewEmptyIndex(ctx, root.VRW(), root.NodeStore(), tt.sch)
+			empty, err := durable.NewEmptyIndex(ctx, root.VRW(), root.NodeStore(), tt.sch, false)
 			require.NoError(t, err)
 
 			indexes, err := durable.NewIndexSet(ctx, root.VRW(), root.NodeStore())

go/store/datas/statistics.go

@@ -122,7 +122,7 @@ func parse_Statistics(ctx context.Context, bs []byte, ns tree.NodeStore, vr type
 		return nil, err
 	}
 
-	m, err := shim.MapFromValue(value, schema.StatsTableDoltSchema, ns)
+	m, err := shim.MapFromValue(value, schema.StatsTableDoltSchema, ns, false)
 	if err != nil {
 		return nil, err
 	}