Merge remote-tracking branch 'origin/main' into aaron/gc-safepoint-co…

…ntroller

go/go.mod

@@ -15,7 +15,7 @@ require (
 	github.com/dolthub/fslock v0.0.3
 	github.com/dolthub/ishell v0.0.0-20240701202509-2b217167d718
 	github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81
-	github.com/dolthub/vitess v0.0.0-20250115003116-d6f17c220028
+	github.com/dolthub/vitess v0.0.0-20250123002143-3b45b8cacbfa
 	github.com/dustin/go-humanize v1.0.1
 	github.com/fatih/color v1.13.0
 	github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568
@@ -39,7 +39,7 @@ require (
 	github.com/tealeg/xlsx v1.0.5
 	go.uber.org/zap v1.24.0
 	golang.org/x/crypto v0.31.0
-	golang.org/x/net v0.25.0
+	golang.org/x/net v0.33.0
 	golang.org/x/sync v0.10.0
 	golang.org/x/sys v0.28.0
 	google.golang.org/api v0.126.0
@@ -56,7 +56,7 @@ require (
 	github.com/cespare/xxhash/v2 v2.2.0
 	github.com/creasty/defaults v1.6.0
 	github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2
-	github.com/dolthub/go-mysql-server v0.19.1-0.20250122180808-0e32f1d293ee
+	github.com/dolthub/go-mysql-server v0.19.1-0.20250123004221-f5a5bcea7eed
 	github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63
 	github.com/dolthub/swiss v0.1.0
 	github.com/esote/minmaxheap v1.0.0

go/go.sum

@@ -179,10 +179,8 @@ github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U=
 github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0=
 github.com/dolthub/go-icu-regex v0.0.0-20241215010122-db690dd53c90 h1:Sni8jrP0sy/w9ZYXoff4g/ixe+7bFCZlfCqXKJSU+zM=
 github.com/dolthub/go-icu-regex v0.0.0-20241215010122-db690dd53c90/go.mod h1:ylU4XjUpsMcvl/BKeRRMXSH7e7WBrPXdSLvnRJYrxEA=
-github.com/dolthub/go-mysql-server v0.19.1-0.20250122100341-c5d0e527c855 h1:TpoMmjRofO1JOTxp14t/pDyr0RLi29ybLAMhV3ZZ+R0=
-github.com/dolthub/go-mysql-server v0.19.1-0.20250122100341-c5d0e527c855/go.mod h1:5HtKnb+IAiv+27bo50KGANbUB4HAzGEF9rlFF2ZBLZg=
-github.com/dolthub/go-mysql-server v0.19.1-0.20250122180808-0e32f1d293ee h1:/rxD9Pn5WpHso6dH0HE3FKRDMMZ1b7ueqo9Q18rOr6s=
-github.com/dolthub/go-mysql-server v0.19.1-0.20250122180808-0e32f1d293ee/go.mod h1:5HtKnb+IAiv+27bo50KGANbUB4HAzGEF9rlFF2ZBLZg=
+github.com/dolthub/go-mysql-server v0.19.1-0.20250123004221-f5a5bcea7eed h1:2EQHWtMkjyN/SNfbg/nh/a0RANq8V8gxNynYum2Kq+s=
+github.com/dolthub/go-mysql-server v0.19.1-0.20250123004221-f5a5bcea7eed/go.mod h1:jYEJ8tNkA7K3k39X8iMqaX3MSMmViRgh222JSLHDgVc=
 github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63 h1:OAsXLAPL4du6tfbBgK0xXHZkOlos63RdKYS3Sgw/dfI=
 github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63/go.mod h1:lV7lUeuDhH5thVGDCKXbatwKy2KW80L4rMT46n+Y2/Q=
 github.com/dolthub/ishell v0.0.0-20240701202509-2b217167d718 h1:lT7hE5k+0nkBdj/1UOSFwjWpNxf+LCApbRHgnCA17XE=
@@ -195,8 +193,8 @@ github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81 h1:7/v8q9X
 github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81/go.mod h1:siLfyv2c92W1eN/R4QqG/+RjjX5W2+gCTRjZxBjI3TY=
 github.com/dolthub/swiss v0.1.0 h1:EaGQct3AqeP/MjASHLiH6i4TAmgbG/c4rA6a1bzCOPc=
 github.com/dolthub/swiss v0.1.0/go.mod h1:BeucyB08Vb1G9tumVN3Vp/pyY4AMUnr9p7Rz7wJ7kAQ=
-github.com/dolthub/vitess v0.0.0-20250115003116-d6f17c220028 h1:lgcIsCUaNDde+lS+aYGYGML95qrQlBMEpaXFN1Pmk+4=
-github.com/dolthub/vitess v0.0.0-20250115003116-d6f17c220028/go.mod h1:1gQZs/byeHLMSul3Lvl3MzioMtOW1je79QYGyi2fd70=
+github.com/dolthub/vitess v0.0.0-20250123002143-3b45b8cacbfa h1:kyoPzxViSXAyqfO0Mab7Qo1UogFIrxZKKyBU6kBOl+E=
+github.com/dolthub/vitess v0.0.0-20250123002143-3b45b8cacbfa/go.mod h1:1gQZs/byeHLMSul3Lvl3MzioMtOW1je79QYGyi2fd70=
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
@@ -867,8 +865,8 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx
 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
-golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
+golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
+golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=

go/libraries/doltcore/mvdata/engine_table_writer.go

@@ -345,7 +345,7 @@ func (s *SqlEngineTableWriter) createTable() error {
 		sqlCols = append(sqlCols, fmt.Sprintf("PRIMARY KEY (%s)", pks))
 	}
 
-	createTable := sql.GenerateCreateTableStatement(s.tableName, sqlCols, "", sql.CharacterSet_utf8mb4.String(), sql.Collation_Default.String(), "")
+	createTable := sql.GenerateCreateTableStatement(s.tableName, sqlCols, "", "", sql.CharacterSet_utf8mb4.String(), sql.Collation_Default.String(), "")
 	_, iter, _, err := s.se.Query(s.sqlCtx, createTable)
 	if err != nil {
 		return err

go/libraries/doltcore/sqle/enginetest/dolt_queries.go

@@ -7591,7 +7591,7 @@ var DoltTempTableScripts = []queries.ScriptTest{
 			{
 				Query: "show create table t;",
 				Expected: []sql.Row{
-					{"t", "CREATE TABLE `t` (\n" +
+					{"t", "CREATE TEMPORARY TABLE `t` (\n" +
 						"  `i` int NOT NULL AUTO_INCREMENT,\n" +
 						"  PRIMARY KEY (`i`)\n" +
 						") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin"},
@@ -7606,7 +7606,7 @@ var DoltTempTableScripts = []queries.ScriptTest{
 			{
 				Query: "show create table t;",
 				Expected: []sql.Row{
-					{"t", "CREATE TABLE `t` (\n" +
+					{"t", "CREATE TEMPORARY TABLE `t` (\n" +
 						"  `i` int NOT NULL AUTO_INCREMENT,\n" +
 						"  PRIMARY KEY (`i`)\n" +
 						") ENGINE=InnoDB AUTO_INCREMENT=4 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin"},
@@ -7629,7 +7629,7 @@ var DoltTempTableScripts = []queries.ScriptTest{
 			{
 				Query: "show create table t;",
 				Expected: []sql.Row{
-					{"t", "CREATE TABLE `t` (\n" +
+					{"t", "CREATE TEMPORARY TABLE `t` (\n" +
 						"  `i` int NOT NULL AUTO_INCREMENT,\n" +
 						"  PRIMARY KEY (`i`)\n" +
 						") ENGINE=InnoDB AUTO_INCREMENT=1001 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin"},
@@ -7670,4 +7670,62 @@ var DoltTempTableScripts = []queries.ScriptTest{
 			},
 		},
 	},
+	{
+		Name: "drop temporary table behavior",
+		Dialect: "mysql",
+		SetUpScript: []string{
+			"create table t (i int);",
+			"create temporary table tmp (i int);",
+		},
+		Assertions: []queries.ScriptTestAssertion{
+			{
+				Query: "show create table tmp;",
+				Expected: []sql.Row{
+					{"tmp", "CREATE TEMPORARY TABLE `tmp` (\n" +
+						"  `i` int\n" +
+						") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin"},
+				},
+			},
+			{
+				Query: "drop temporary table tmp;",
+				Expected: []sql.Row{
+					{types.NewOkResult(0)},
+				},
+			},
+
+			{
+				Query: "create temporary table t (i int, j int);",
+				Expected: []sql.Row{
+					{types.NewOkResult(0)},
+				},
+			},
+			{
+				Query: "show create table t;",
+				Expected: []sql.Row{
+					{"t", "CREATE TEMPORARY TABLE `t` (\n" +
+						"  `i` int,\n" +
+						"  `j` int\n" +
+						") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin"},
+				},
+			},
+			{
+				Query: "drop temporary table t;",
+				Expected: []sql.Row{
+					{types.NewOkResult(0)},
+				},
+			},
+			{
+				Query: "show create table t;",
+				Expected: []sql.Row{
+					{"t", "CREATE TABLE `t` (\n" +
+						"  `i` int\n" +
+						") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin"},
+				},
+			},
+			{
+				Query: "drop temporary table t;",
+				ExpectedErr: sql.ErrUnknownTable,
+			},
+		},
+	},
 }

go/libraries/doltcore/sqle/sqlfmt/schema_fmt.go

@@ -473,7 +473,7 @@ func GenerateCreateTableStatement(tblName string, sch schema.Schema, fks []doltd
 	}
 
 	coll := sql.CollationID(sch.GetCollation())
-	createTableStmt := sql.GenerateCreateTableStatement(tblName, colStmts, "", coll.CharacterSet().Name(), coll.Name(), sch.GetComment())
+	createTableStmt := sql.GenerateCreateTableStatement(tblName, colStmts, "", "", coll.CharacterSet().Name(), coll.Name(), sch.GetComment())
 	return fmt.Sprintf("%s;", createTableStmt), nil
 }
 

go/store/chunks/chunk_store.go

@@ -172,7 +172,9 @@ type MarkAndSweeper interface {
 	// chunks is accessed and copied.
 	SaveHashes(context.Context, []hash.Hash) error
 
-	Close(context.Context) (GCFinalizer, error)
+	Finalize(context.Context) (GCFinalizer, error)
+
+	Close(context.Context) error
 }
 
 // A GCFinalizer is returned from a MarkAndSweeper after it is closed.

go/store/chunks/memory_store.go

@@ -398,10 +398,14 @@ func (i *msvMarkAndSweeper) SaveHashes(ctx context.Context, hashes []hash.Hash)
 	return nil
 }
 
-func (i *msvMarkAndSweeper) Close(context.Context) (GCFinalizer, error) {
+func (i *msvMarkAndSweeper) Finalize(context.Context) (GCFinalizer, error) {
 	return msvGcFinalizer{i.ms, i.keepers}, nil
 }
 
+func (i *msvMarkAndSweeper) Close(context.Context) error {
+	return nil
+}
+
 func (ms *MemoryStoreView) MarkAndSweepChunks(ctx context.Context, getAddrs GetAddrsCurry, filter HasManyFunc, dest ChunkStore, mode GCMode) (MarkAndSweeper, error) {
 	if dest != ms {
 		panic("unsupported")

go/store/nbs/cmp_chunk_table_writer.go

@@ -173,6 +173,21 @@ func (tw *CmpChunkTableWriter) Remove() error {
 	return os.Remove(tw.path)
 }
 
+// Cancel the inprogress write and attempt to cleanup any
+// resources associated with it. It is an error to call
+// Flush{,ToFile} or Reader after canceling the writer.
+func (tw *CmpChunkTableWriter) Cancel() error {
+	closer, err := tw.sink.Reader()
+	if err != nil {
+		return err
+	}
+	err = closer.Close()
+	if err != nil {
+		return err
+	}
+	return tw.Remove()
+}
+
 func containsDuplicates(prefixes prefixIndexSlice) bool {
 	if len(prefixes) == 0 {
 		return false

go/store/nbs/gc_copier.go

@@ -45,21 +45,28 @@ func (ea gcErrAccum) Error() string {
 
 type gcCopier struct {
 	writer *CmpChunkTableWriter
+	tfp    tableFilePersister
 }
 
-func newGarbageCollectionCopier() (*gcCopier, error) {
+func newGarbageCollectionCopier(tfp tableFilePersister) (*gcCopier, error) {
 	writer, err := NewCmpChunkTableWriter("")
 	if err != nil {
 		return nil, err
 	}
-	return &gcCopier{writer}, nil
+	return &gcCopier{writer, tfp}, nil
 }
 
 func (gcc *gcCopier) addChunk(ctx context.Context, c CompressedChunk) error {
 	return gcc.writer.AddCmpChunk(c)
 }
 
-func (gcc *gcCopier) copyTablesToDir(ctx context.Context, tfp tableFilePersister) (ts []tableSpec, err error) {
+// If the writer should be closed and deleted, instead of being used with
+// copyTablesToDir, call this method.
+func (gcc *gcCopier) cancel(_ context.Context) error {
+	return gcc.writer.Cancel()
+}
+
+func (gcc *gcCopier) copyTablesToDir(ctx context.Context) (ts []tableSpec, err error) {
 	var filename string
 	filename, err = gcc.writer.Finish()
 	if err != nil {
@@ -79,7 +86,7 @@ func (gcc *gcCopier) copyTablesToDir(ctx context.Context, tfp tableFilePersister
 		return nil, fmt.Errorf("invalid filename: %s", filename)
 	}
 
-	exists, err := tfp.Exists(ctx, addr, uint32(gcc.writer.ChunkCount()), nil)
+	exists, err := gcc.tfp.Exists(ctx, addr, uint32(gcc.writer.ChunkCount()), nil)
 	if err != nil {
 		return nil, err
 	}
@@ -94,7 +101,7 @@ func (gcc *gcCopier) copyTablesToDir(ctx context.Context, tfp tableFilePersister
 	}
 
 	// Attempt to rename the file to the destination if we are working with a fsTablePersister...
-	if mover, ok := tfp.(movingTableFilePersister); ok {
+	if mover, ok := gcc.tfp.(movingTableFilePersister); ok {
 		err = mover.TryMoveCmpChunkTableWriter(ctx, filename, gcc.writer)
 		if err == nil {
 			return []tableSpec{
@@ -114,7 +121,7 @@ func (gcc *gcCopier) copyTablesToDir(ctx context.Context, tfp tableFilePersister
 	defer r.Close()
 	sz := gcc.writer.ContentLength()
 
-	err = tfp.CopyTableFile(ctx, r, filename, sz, uint32(gcc.writer.ChunkCount()))
+	err = gcc.tfp.CopyTableFile(ctx, r, filename, sz, uint32(gcc.writer.ChunkCount()))
 	if err != nil {
 		return nil, err
 	}

go/store/nbs/store.go

@@ -1652,7 +1652,7 @@ func markAndSweepChunks(ctx context.Context, nbs *NomsBlockStore, src NBSCompres
 		return nil, fmt.Errorf("NBS does not support copying garbage collection")
 	}
 
-	gcc, err := newGarbageCollectionCopier()
+	gcc, err := newGarbageCollectionCopier(tfp)
 	if err != nil {
 		return nil, err
 	}
@@ -1758,11 +1758,12 @@ func (i *markAndSweeper) SaveHashes(ctx context.Context, hashes []hash.Hash) err
 	return nil
 }
 
-func (i *markAndSweeper) Close(ctx context.Context) (chunks.GCFinalizer, error) {
-	specs, err := i.gcc.copyTablesToDir(ctx, i.tfp)
+func (i *markAndSweeper) Finalize(ctx context.Context) (chunks.GCFinalizer, error) {
+	specs, err := i.gcc.copyTablesToDir(ctx)
 	if err != nil {
 		return nil, err
 	}
+	i.gcc = nil
 
 	return gcFinalizer{
 		nbs:   i.dest,
@@ -1771,6 +1772,13 @@ func (i *markAndSweeper) Close(ctx context.Context) (chunks.GCFinalizer, error)
 	}, nil
 }
 
+func (i *markAndSweeper) Close(ctx context.Context) error {
+	if i.gcc != nil {
+		return i.gcc.cancel(ctx)
+	}
+	return nil
+}
+
 type gcFinalizer struct {
 	nbs   *NomsBlockStore
 	specs []tableSpec

go/store/nbs/store_test.go

@@ -345,8 +345,9 @@ func TestNBSCopyGC(t *testing.T) {
 		keepersSlice = append(keepersSlice, h)
 	}
 	require.NoError(t, sweeper.SaveHashes(ctx, keepersSlice))
-	finalizer, err := sweeper.Close(ctx)
+	finalizer, err := sweeper.Finalize(ctx)
 	require.NoError(t, err)
+	require.NoError(t, sweeper.Close(ctx))
 	require.NoError(t, finalizer.SwapChunksInStore(ctx))
 	st.EndGC()