setup for sqllite3 (#37)

* wip: setup for sqllite3

* sqlite support via bun sqlite dialect

- Keep same version of bun.DB and dialects
- Move (shared) SQL code into `datastore/sql`
- update instances of postgres.New* to cfg.GetDatabase().New*

* reset config

chore: run lint

* chore: integrate sqllite tests

chore: cleanup

* refactor bun initiation

* refactor integration tests

* remove integration test for sqlite

* switch to `logrusbun` to logQueries

* fix driver example in config.yml

Makefile

@@ -2,10 +2,10 @@ tidy_dependencies:
 	go mod tidy
 
 migrate:
-	migrate -path ./datastore/postgres/migrations/ -database "postgres://sdump:sdump@localhost:3432/sdump?sslmode=disable" up
+	migrate -path ./datastore/sql/migrations/ -database "postgres://sdump:sdump@localhost:3432/sdump?sslmode=disable" up
 
 migrate-down:
-	migrate -path ./datastore/postgres/migrations/ -database "postgres://sdump:sdump@localhost:3432/sdump?sslmode=disable" down
+	migrate -path ./datastore/sql/migrations/ -database "postgres://sdump:sdump@localhost:3432/sdump?sslmode=disable" down
 
 run-http:
 	go run cmd/*.go

cmd/delete.go

@@ -6,7 +6,7 @@ import (
 
 	"github.com/adelowo/sdump"
 	"github.com/adelowo/sdump/config"
-	"github.com/adelowo/sdump/datastore/postgres"
+	sdumpSql "github.com/adelowo/sdump/datastore/sql"
 	"github.com/spf13/cobra"
 )
 
@@ -16,12 +16,12 @@ func createDeleteCommand(rootCmd *cobra.Command, cfg *config.Config) {
 		Aliases: []string{"d"},
 		Short:   "Deletes all old HTTP requests to preserve DB space",
 		RunE: func(_ *cobra.Command, _ []string) error {
-			db, err := postgres.New(cfg.HTTP.Database.DSN, cfg.HTTP.Database.LogQueries)
+			db, err := sdumpSql.New(cfg.HTTP.Database)
 			if err != nil {
 				return err
 			}
 
-			ingestStore := postgres.NewIngestRepository(db)
+			ingestStore := sdumpSql.NewIngestRepository(db)
 
 			before := time.Now().Add(-1 * cfg.Cron.TTL)
 

cmd/http.go

@@ -8,7 +8,7 @@ import (
 	"time"
 
 	"github.com/adelowo/sdump/config"
-	"github.com/adelowo/sdump/datastore/postgres"
+	sdumpSql "github.com/adelowo/sdump/datastore/sql"
 	"github.com/adelowo/sdump/server/httpd"
 	"github.com/r3labs/sse/v2"
 	"github.com/sethvargo/go-limiter/memorystore"
@@ -53,11 +53,6 @@ func createHTTPCommand(cmd *cobra.Command, cfg *config.Config) {
 			logrus.SetOutput(os.Stdout)
 			logrus.SetLevel(lvl)
 
-			db, err := postgres.New(cfg.HTTP.Database.DSN, cfg.HTTP.Database.LogQueries)
-			if err != nil {
-				return err
-			}
-
 			var tokensPerMinute uint64 = 60
 			if cfg.HTTP.RateLimit.RequestsPerMinute > 0 {
 				tokensPerMinute = cfg.HTTP.RateLimit.RequestsPerMinute
@@ -71,9 +66,13 @@ func createHTTPCommand(cmd *cobra.Command, cfg *config.Config) {
 				return err
 			}
 
-			urlStore := postgres.NewURLRepositoryTable(db)
-			ingestStore := postgres.NewIngestRepository(db)
-			userStore := postgres.NewUserRepositoryTable(db)
+			db, err := sdumpSql.New(cfg.HTTP.Database)
+			if err != nil {
+				return err
+			}
+			urlStore := sdumpSql.NewURLRepositoryTable(db)
+			ingestStore := sdumpSql.NewIngestRepository(db)
+			userStore := sdumpSql.NewUserRepositoryTable(db)
 
 			hostName, err := os.Hostname()
 			if err != nil {

config.yml

@@ -39,8 +39,10 @@ http:
   database:
     ## database dsn
     dsn: postgres://sdump:sdump@localhost:3432/sdump?sslmode=disable
+    # dsn: "file::memory:?cache=shared"
     ## should we log sql queries? In prod, no but in local mode, you probably want to
     log_queries: true
+    # driver: "sqlite"
 
   #  limit the size of jSON request body that can be sent to endpoints
   max_request_body_size: 500

config/config.go

@@ -1,8 +1,10 @@
 package config
 
-import "time"
+import (
+	"time"
+)
 
-// ENUM(psql)
+// ENUM(psql, sqlite)
 type DatabaseType string
 
 type SSHConfig struct {
@@ -19,6 +21,12 @@ type SSHConfig struct {
 	AllowList []string `json:"allow_list,omitempty" mapstructure:"allow_list" yaml:"allow_list"`
 }
 
+type DatabaseConfig struct {
+	DSN        string       `mapstructure:"dsn" json:"dsn,omitempty" yaml:"dsn"`
+	LogQueries bool         `mapstructure:"log_queries" json:"log_queries,omitempty" yaml:"log_queries"`
+	Driver     DatabaseType `mapstructure:"driver" json:"driver,omitempty" yaml:"driver,omitempty"`
+}
+
 type HTTPConfig struct {
 	// Port to run http server on
 	// The server
@@ -31,10 +39,7 @@ type HTTPConfig struct {
 	// If empty, server would crash
 	AdminSecret string `mapstructure:"admin_secret" json:"admin_secret,omitempty" yaml:"admin_secret"`
 
-	Database struct {
-		DSN        string `mapstructure:"dsn" json:"dsn,omitempty" yaml:"dsn"`
-		LogQueries bool   `mapstructure:"log_queries" json:"log_queries,omitempty" yaml:"log_queries"`
-	} `mapstructure:"database" json:"database,omitempty" yaml:"database"`
+	Database DatabaseConfig `mapstructure:"database" json:"database,omitempty" yaml:"database"`
 
 	Domain             string `json:"domain,omitempty" yaml:"domain" mapstructure:"domain"`
 	MaxRequestBodySize int64  `json:"max_request_body_size,omitempty" yaml:"max_request_body_size" mapstructure:"max_request_body_size"`

datastore/sql/driver.go

@@ -0,0 +1,54 @@
+package sql
+
+import (
+	"database/sql"
+
+	"github.com/adelowo/sdump/config"
+	"github.com/oiime/logrusbun"
+	"github.com/sirupsen/logrus"
+	"github.com/uptrace/bun"
+	"github.com/uptrace/bun/dialect/pgdialect"
+	"github.com/uptrace/bun/dialect/sqlitedialect"
+	"github.com/uptrace/bun/driver/pgdriver"
+	"github.com/uptrace/bun/driver/sqliteshim"
+	"github.com/uptrace/bun/extra/bunotel"
+)
+
+func newPsql(cfg config.DatabaseConfig) (*bun.DB, error) {
+	pgdb := sql.OpenDB(pgdriver.NewConnector(pgdriver.WithDSN(cfg.DSN)))
+
+	db := bun.NewDB(pgdb, pgdialect.New())
+	log := logrus.New()
+
+	db.AddQueryHook(bunotel.NewQueryHook(bunotel.WithDBName("getclaimclaim")))
+
+	if cfg.LogQueries {
+		db.AddQueryHook(logrusbun.NewQueryHook(logrusbun.QueryHookOptions{Logger: log}))
+	}
+
+	return db, db.Ping()
+}
+
+func newSqlite(cfg config.DatabaseConfig) (*bun.DB, error) {
+	sqlite, err := sql.Open(sqliteshim.ShimName, cfg.DSN)
+	if err != nil {
+		panic(err)
+	}
+
+	db := bun.NewDB(sqlite, sqlitedialect.New())
+	log := logrus.New()
+
+	if cfg.LogQueries {
+		db.AddQueryHook(logrusbun.NewQueryHook(logrusbun.QueryHookOptions{Logger: log}))
+	}
+
+	return db, db.Ping()
+}
+
+func New(cfg config.DatabaseConfig) (*bun.DB, error) {
+	if cfg.Driver == config.DatabaseTypeSqlite {
+		return newSqlite(cfg)
+	}
+
+	return newPsql(cfg)
+}

datastore/postgres/ingest.go → datastore/sql/ingest.go

@@ -1,4 +1,4 @@
-package postgres
+package sql
 
 import (
 	"context"

datastore/postgres/ingest_test.go → datastore/sql/ingest_test.go

@@ -1,7 +1,7 @@
 //go:build integration
 // +build integration
 
-package postgres
+package sql
 
 import (
 	"context"
@@ -12,7 +12,7 @@ import (
 )
 
 func TestIngestRepository_Create(t *testing.T) {
-	client, teardownFunc := setupDatabase(t)
+	client, teardownFunc := setupPostgresDatabase(t)
 	defer teardownFunc()
 
 	ingestStore := NewIngestRepository(client)

datastore/postgres/postgres_test.go → datastore/sql/postgres_test.go

@@ -1,14 +1,15 @@
 //go:build integration
 // +build integration
 
-package postgres
+package sql
 
 import (
 	"context"
 	"database/sql"
 	"fmt"
 	"testing"
 
+	"github.com/adelowo/sdump/config"
 	testfixtures "github.com/go-testfixtures/testfixtures/v3"
 	"github.com/golang-migrate/migrate/v4"
 	"github.com/golang-migrate/migrate/v4/database/postgres"
@@ -19,7 +20,7 @@ import (
 	"github.com/uptrace/bun"
 )
 
-func prepareTestDatabase(t *testing.T, dsn string) {
+func preparePostgresTestDatabase(t *testing.T, dsn string) {
 	t.Helper()
 
 	var err error
@@ -50,9 +51,9 @@ func prepareTestDatabase(t *testing.T, dsn string) {
 	require.NoError(t, fixtures.Load())
 }
 
-// setupDatabase spins up a new Postgres container and returns a closure
+// setupPostgresDatabase spins up a new Postgres container and returns a closure
 // please always make sure to call the closure as it is the teardown function
-func setupDatabase(t *testing.T) (*bun.DB, func()) {
+func setupPostgresDatabase(t *testing.T) (*bun.DB, func()) {
 	t.Helper()
 
 	var dsn string
@@ -83,9 +84,11 @@ func setupDatabase(t *testing.T) (*bun.DB, func()) {
 	dsn = fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=disable", "sdump", "sdump",
 		fmt.Sprintf("localhost:%s", port.Port()), "sdumptest")
 
-	prepareTestDatabase(t, dsn)
+	preparePostgresTestDatabase(t, dsn)
 
-	client, err := New(dsn, false)
+	client, err := New(config.DatabaseConfig{
+		DSN: dsn,
+	})
 	require.NoError(t, err)
 
 	return client, func() {

datastore/sql/sqlite_test.go

@@ -0,0 +1,67 @@
+//go:build integration
+// +build integration
+
+package sql
+
+import (
+	"database/sql"
+	"fmt"
+	"testing"
+
+	"github.com/adelowo/sdump/config"
+	testfixtures "github.com/go-testfixtures/testfixtures/v3"
+	"github.com/stretchr/testify/require"
+	"github.com/uptrace/bun"
+	"github.com/uptrace/bun/driver/sqliteshim"
+
+	"github.com/golang-migrate/migrate/v4"
+	"github.com/golang-migrate/migrate/v4/database/sqlite3"
+)
+
+func prepareSqliteTestDatabase(t *testing.T, dsn string) {
+	t.Helper()
+
+	var err error
+
+	db, err := sql.Open(sqliteshim.ShimName, dsn)
+	require.NoError(t, err)
+
+	require.NoError(t, db.Ping())
+
+	driver, err := sqlite3.WithInstance(db, &sqlite3.Config{})
+	require.NoError(t, err)
+
+	migrator, err := migrate.NewWithDatabaseInstance(
+		fmt.Sprintf("file://%s", "migrations"), "sqllite3", driver)
+	require.NoError(t, err)
+
+	if err := migrator.Up(); err != nil && err != migrate.ErrNoChange {
+		require.NoError(t, err)
+	}
+
+	fixtures, err := testfixtures.New(
+		testfixtures.Database(db),
+		testfixtures.Dialect("sqlite3"),
+		testfixtures.Directory("testdata/fixtures"),
+	)
+	require.NoError(t, err)
+
+	require.NoError(t, fixtures.Load())
+}
+
+func setupSqliteDatabase(t *testing.T) (*bun.DB, func()) {
+	t.Helper()
+
+	dsn := "file::memory:?cache=shared"
+
+	prepareSqliteTestDatabase(t, dsn)
+
+	client, err := New(config.DatabaseConfig{
+		DSN:    dsn,
+		Driver: config.DatabaseTypeSqlite,
+	})
+	require.NoError(t, err)
+
+	return client, func() {
+	}
+}

datastore/postgres/url.go → datastore/sql/url.go

@@ -1,4 +1,4 @@
-package postgres
+package sql
 
 import (
 	"context"