refactor: reorg & modularize the code (apecloud#76)

.github/workflows/go.yml

@@ -43,7 +43,7 @@ jobs:
       run: go build -v
 
     - name: Test packages
-      run: go test -v -cover ./charset
+      run: go test -v -cover ./charset ./transpiler ./backend ./harness
 
     - name: Test Query Engine
       run: go test -v -cover --timeout 300s .

Makefile

@@ -46,7 +46,7 @@ test-full:
 	@$(SHOW_TOTAL_COVERAGE)
 
 test:
-	$(TEST_CMD) $(shell go list ./... | grep -v './binlogreplication' | grep -v './transpiler')
+	$(TEST_CMD) $(shell go list ./... | grep -v './binlogreplication')
 	@$(SHOW_TOTAL_COVERAGE)
 
 cover:

executor.go → backend/executor.go

@@ -11,15 +11,16 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-package main
+package backend
 
 import (
 	"context"
 	stdsql "database/sql"
 	"fmt"
 	"sync"
 
-	"github.com/apecloud/myduckserver/meta"
+	"github.com/apecloud/myduckserver/catalog"
+	"github.com/apecloud/myduckserver/transpiler"
 	"github.com/dolthub/go-mysql-server/sql"
 	"github.com/dolthub/go-mysql-server/sql/expression"
 	"github.com/dolthub/go-mysql-server/sql/plan"
@@ -66,8 +67,8 @@ func (b *DuckBuilder) GetConn(ctx context.Context, id uint32, schemaName string)
 			logrus.WithError(err).Error("Failed to get current schema")
 			return nil, err
 		} else if currentSchema != schemaName {
-			if _, err := conn.ExecContext(ctx, "USE "+meta.FullSchemaName(b.catalogName, schemaName)); err != nil {
-				if meta.IsDuckDBSetSchemaNotFoundError(err) {
+			if _, err := conn.ExecContext(ctx, "USE "+catalog.FullSchemaName(b.catalogName, schemaName)); err != nil {
+				if catalog.IsDuckDBSetSchemaNotFoundError(err) {
 					return nil, sql.ErrDatabaseNotFound.New(schemaName)
 				}
 				logrus.WithField("schema", schemaName).WithError(err).Error("Failed to switch schema")
@@ -108,7 +109,7 @@ func (b *DuckBuilder) Build(ctx *sql.Context, root sql.Node, r sql.Row) (sql.Row
 	}
 
 	// Fallback to the base builder if the plan contains system/user variables or is not a pure data query.
-	if containsVariable(n) || !isPureDataQuery(n) {
+	if containsVariable(n) || !IsPureDataQuery(n) {
 		return b.base.Build(ctx, root, r)
 	}
 
@@ -121,9 +122,9 @@ func (b *DuckBuilder) Build(ctx *sql.Context, root sql.Node, r sql.Row) (sql.Row
 
 	switch node := n.(type) {
 	case *plan.Use:
-		useStmt := "USE " + meta.FullSchemaName(b.catalogName, node.Database().Name())
+		useStmt := "USE " + catalog.FullSchemaName(b.catalogName, node.Database().Name())
 		if _, err := conn.ExecContext(ctx.Context, useStmt); err != nil {
-			if meta.IsDuckDBSetSchemaNotFoundError(err) {
+			if catalog.IsDuckDBSetSchemaNotFoundError(err) {
 				return nil, sql.ErrDatabaseNotFound.New(node.Database().Name())
 			}
 			return nil, err
@@ -182,7 +183,7 @@ func (b *DuckBuilder) executeQuery(ctx *sql.Context, n sql.Node, conn *stdsql.Co
 	case *plan.ShowTables:
 		duckSQL = ctx.Query()
 	default:
-		duckSQL, err = translate(n, ctx.Query())
+		duckSQL, err = transpiler.Translate(n, ctx.Query())
 	}
 	if err != nil {
 		return nil, err
@@ -204,7 +205,7 @@ func (b *DuckBuilder) executeQuery(ctx *sql.Context, n sql.Node, conn *stdsql.Co
 
 func (b *DuckBuilder) executeDML(ctx *sql.Context, n sql.Node, conn *stdsql.Conn) (sql.RowIter, error) {
 	// Translate the MySQL query to a DuckDB query
-	duckSQL, err := translate(n, ctx.Query())
+	duckSQL, err := transpiler.Translate(n, ctx.Query())
 	if err != nil {
 		return nil, err
 	}
@@ -250,13 +251,13 @@ func containsVariable(n sql.Node) bool {
 	return found
 }
 
-// isPureDataQuery inspects if the plan is a pure data query,
+// IsPureDataQuery inspects if the plan is a pure data query,
 // i.e., it operates on (>=1) data tables and does not touch any system tables.
 // The following examples are NOT pure data queries:
 // - `SELECT * FROM mysql.*`
 // - `TRUNCATE mysql.user`
 // - `SELECT DATABASE()`
-func isPureDataQuery(n sql.Node) bool {
+func IsPureDataQuery(n sql.Node) bool {
 	c := &tableNodeCollector{}
 	transform.Walk(c, n)
 	if len(c.tables) == 0 {

executor_test.go → backend/executor_test.go

@@ -1,13 +1,11 @@
-package main
+package backend
 
 import (
 	"context"
 	"io"
 	"testing"
 
 	"github.com/DATA-DOG/go-sqlmock"
-	"github.com/dolthub/go-mysql-server/enginetest"
-	"github.com/dolthub/go-mysql-server/enginetest/scriptgen/setup"
 	"github.com/dolthub/go-mysql-server/memory"
 	"github.com/dolthub/go-mysql-server/sql"
 	"github.com/dolthub/go-mysql-server/sql/plan"
@@ -100,61 +98,3 @@ func TestDuckBuilder_Select(t *testing.T) {
 
 	assert.NoError(t, mock.ExpectationsWereMet())
 }
-
-func TestIsPureDataQuery(t *testing.T) {
-	harness := NewDefaultDuckHarness()
-	harness.Setup(
-		setup.MydbData,
-		[]setup.SetupScript{
-			{
-				"CREATE TABLE users (id INT PRIMARY KEY, name VARCHAR(255))",
-				"CREATE table xy (x int primary key, y int, unique index y_idx(y));",
-			},
-		})
-	ctx := enginetest.NewContext(harness)
-	engine, err := harness.NewEngine(t)
-	require.NoError(t, err)
-	tests := []struct {
-		name     string
-		query    string
-		expected bool
-	}{
-		{
-			name:     "Simple SELECT query",
-			query:    "SELECT * FROM users",
-			expected: true,
-		},
-		{
-			name:     "Query from mysql system table",
-			query:    "SELECT * FROM mysql.user",
-			expected: false,
-		},
-		{
-			name:     "Query with system function",
-			query:    "SELECT DATABASE()",
-			expected: false,
-		},
-		// {
-		// 	name:     "Query with subquery from system table",
-		// 	query:    "SELECT u.name, (SELECT COUNT(*) FROM mysql.user) FROM users u",
-		// 	expected: false,
-		// },
-		{
-			name:     "Query from information_schema",
-			query:    "SELECT * FROM information_schema.tables",
-			expected: false,
-		},
-		// {
-		// 	name:     "Query with subquery",
-		// 	query:    "select * from xy where x in (select 1 having false);",
-		// 	expected: true,
-		// },
-	}
-	for _, tt := range tests {
-		analyzed, err := engine.AnalyzeQuery(ctx, tt.query)
-		require.NoError(t, err)
-		result := isPureDataQuery(analyzed)
-		assert.Equal(t, tt.expected, result, "isPureDataQuery() for query '%s'", tt.query)
-	}
-
-}

handler.go → backend/handler.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+package backend
 
 import (
 	"context"
@@ -49,7 +49,7 @@ func (h *MyHandler) ComInitDB(c *mysql.Conn, schemaName string) error {
 	return h.Handler.ComInitDB(c, schemaName)
 }
 
-func wrapHandler(b *DuckBuilder) server.HandlerWrapper {
+func WrapHandler(b *DuckBuilder) server.HandlerWrapper {
 	return func(h mysql.Handler) (mysql.Handler, error) {
 		handler, ok := h.(*server.Handler)
 		if !ok {

iter.go → backend/iter.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+package backend
 
 import (
 	stdsql "database/sql"

session.go → backend/session.go

@@ -11,27 +11,27 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-package main
+package backend
 
 import (
 	"context"
 	stdsql "database/sql"
 	"fmt"
 	"strconv"
 
-	"github.com/apecloud/myduckserver/meta"
+	"github.com/apecloud/myduckserver/catalog"
 	"github.com/dolthub/go-mysql-server/memory"
 	"github.com/dolthub/go-mysql-server/sql"
 	"github.com/dolthub/vitess/go/mysql"
 )
 
 type Session struct {
 	*memory.Session
-	db *meta.DbProvider
+	db *catalog.DatabaseProvider
 }
 
 // NewSessionBuilder returns a session builder for the given database provider.
-func NewSessionBuilder(provider *meta.DbProvider) func(ctx context.Context, conn *mysql.Conn, addr string) (sql.Session, error) {
+func NewSessionBuilder(provider *catalog.DatabaseProvider) func(ctx context.Context, conn *mysql.Conn, addr string) (sql.Session, error) {
 	_, err := provider.Storage().Exec("CREATE TABLE IF NOT EXISTS main.persistent_variables (name TEXT PRIMARY KEY, value TEXT, type TEXT)")
 	if err != nil {
 		panic(err)

meta/comment.go → catalog/comment.go

@@ -1,4 +1,4 @@
-package meta
+package catalog
 
 import (
 	"encoding/base64"

meta/database.go → catalog/database.go

@@ -1,4 +1,4 @@
-package meta
+package catalog
 
 import (
 	stdsql "database/sql"

meta/errors.go → catalog/errors.go

@@ -1,4 +1,4 @@
-package meta
+package catalog
 
 import (
 	"strings"

meta/identifier.go → catalog/identifier.go

@@ -1,4 +1,4 @@
-package meta
+package catalog
 
 import (
 	"strings"

meta/index.go → catalog/index.go

@@ -1,4 +1,4 @@
-package meta
+package catalog
 
 import "github.com/dolthub/go-mysql-server/sql"
 

meta/provider.go → catalog/provider.go

@@ -1,4 +1,4 @@
-package meta
+package catalog
 
 import (
 	"fmt"
@@ -15,26 +15,26 @@ import (
 	"github.com/apecloud/myduckserver/configuration"
 )
 
-type DbProvider struct {
+type DatabaseProvider struct {
 	mu          *sync.RWMutex
 	storage     *stdsql.DB
 	catalogName string
 	dataDir     string
 }
 
-var _ sql.DatabaseProvider = (*DbProvider)(nil)
-var _ sql.MutableDatabaseProvider = (*DbProvider)(nil)
-var _ configuration.DataDirProvider = (*DbProvider)(nil)
+var _ sql.DatabaseProvider = (*DatabaseProvider)(nil)
+var _ sql.MutableDatabaseProvider = (*DatabaseProvider)(nil)
+var _ configuration.DataDirProvider = (*DatabaseProvider)(nil)
 
-func NewInMemoryDBProvider() *DbProvider {
+func NewInMemoryDBProvider() *DatabaseProvider {
 	prov, err := NewDBProvider(".", "")
 	if err != nil {
 		panic(err)
 	}
 	return prov
 }
 
-func NewDBProvider(dataDir, dbFile string) (*DbProvider, error) {
+func NewDBProvider(dataDir, dbFile string) (*DatabaseProvider, error) {
 	dbFile = strings.TrimSpace(dbFile)
 	name := ""
 	dsn := ""
@@ -62,32 +62,32 @@ func NewDBProvider(dataDir, dbFile string) (*DbProvider, error) {
 		return nil, err
 	}
 
-	return &DbProvider{
+	return &DatabaseProvider{
 		mu:          &sync.RWMutex{},
 		storage:     storage,
 		catalogName: name,
 		dataDir:     dataDir,
 	}, nil
 }
 
-func (prov *DbProvider) Close() error {
+func (prov *DatabaseProvider) Close() error {
 	return prov.storage.Close()
 }
 
-func (prov *DbProvider) Storage() *stdsql.DB {
+func (prov *DatabaseProvider) Storage() *stdsql.DB {
 	return prov.storage
 }
 
-func (prov *DbProvider) CatalogName() string {
+func (prov *DatabaseProvider) CatalogName() string {
 	return prov.catalogName
 }
 
-func (prov *DbProvider) DataDir() string {
+func (prov *DatabaseProvider) DataDir() string {
 	return prov.dataDir
 }
 
 // AllDatabases implements sql.DatabaseProvider.
-func (prov *DbProvider) AllDatabases(ctx *sql.Context) []sql.Database {
+func (prov *DatabaseProvider) AllDatabases(ctx *sql.Context) []sql.Database {
 	prov.mu.RLock()
 	defer prov.mu.RUnlock()
 
@@ -120,7 +120,7 @@ func (prov *DbProvider) AllDatabases(ctx *sql.Context) []sql.Database {
 }
 
 // Database implements sql.DatabaseProvider.
-func (prov *DbProvider) Database(ctx *sql.Context, name string) (sql.Database, error) {
+func (prov *DatabaseProvider) Database(ctx *sql.Context, name string) (sql.Database, error) {
 	prov.mu.RLock()
 	defer prov.mu.RUnlock()
 
@@ -136,7 +136,7 @@ func (prov *DbProvider) Database(ctx *sql.Context, name string) (sql.Database, e
 }
 
 // HasDatabase implements sql.DatabaseProvider.
-func (prov *DbProvider) HasDatabase(ctx *sql.Context, name string) bool {
+func (prov *DatabaseProvider) HasDatabase(ctx *sql.Context, name string) bool {
 	prov.mu.RLock()
 	defer prov.mu.RUnlock()
 
@@ -158,7 +158,7 @@ func hasDatabase(engine *stdsql.DB, dstName string, name string) (bool, error) {
 }
 
 // CreateDatabase implements sql.MutableDatabaseProvider.
-func (prov *DbProvider) CreateDatabase(ctx *sql.Context, name string) error {
+func (prov *DatabaseProvider) CreateDatabase(ctx *sql.Context, name string) error {
 	prov.mu.Lock()
 	defer prov.mu.Unlock()
 
@@ -171,7 +171,7 @@ func (prov *DbProvider) CreateDatabase(ctx *sql.Context, name string) error {
 }
 
 // DropDatabase implements sql.MutableDatabaseProvider.
-func (prov *DbProvider) DropDatabase(ctx *sql.Context, name string) error {
+func (prov *DatabaseProvider) DropDatabase(ctx *sql.Context, name string) error {
 	prov.mu.Lock()
 	defer prov.mu.Unlock()
 

meta/table.go → catalog/table.go

@@ -1,4 +1,4 @@
-package meta
+package catalog
 
 import (
 	stdsql "database/sql"

meta/type_mapping.go → catalog/type_mapping.go

@@ -1,4 +1,4 @@
-package meta
+package catalog
 
 import (
 	"fmt"