SchemaMigrationsHistory to determine unapplied out of order migrations

This commit adds a `SchemaMigrationsHistory` table which:

- Provides users with more information about the time and order in
which migration scripts were applied to their database. This is exposed
via the `wrench migrate history` command

- Allows out of order migration scripts to be applied to a database by
determining what subset of migrations are yet to be applied in contrast
to comparing a single version number. This flexibility allows production
hotfixes to be applied on a diverging branch and having those changes
merged back to non production environments which may already be ahead of
production.

In addition to the new functionality around `SchemaMigrationsHistory`,
existing users of wrench will have their databases upgraded to use the
new tracking table by backfilling the missing history. New users of
wrench will have both `SchemaMigrations` and `SchemaMigrationsHistory`
from the start and no backfilling is necessary.

README.md

@@ -30,18 +30,34 @@ $ wrench migrate up --directory ./_examples
 # load ddl from database to file ./_examples/schema.sql
 $ wrench load --directory ./_examples
 
+# show time and date of migrations
+$ wrench migrate history
+Version	Dirty	Created					Modified
+1	false	2020-06-16 08:07:11.763755 +0000 UTC	2020-06-16 08:07:11.76998 +0000 UTC
+
 # finally, we have successfully migrated database!
 $ cat ./_examples/schema.sql
-CREATE TABLE SchemaMigrations (
-  Version INT64 NOT NULL,
-  Dirty BOOL NOT NULL,
-) PRIMARY KEY(Version);
-
 CREATE TABLE Singers (
   SingerID STRING(36) NOT NULL,
   FirstName STRING(1024),
   LastName STRING(1024),
 ) PRIMARY KEY(SingerID);
+
+CREATE TABLE SchemaMigrations (
+  Version INT64 NOT NULL,
+  Dirty BOOL NOT NULL,
+) PRIMARY KEY(Version);
+
+CREATE TABLE SchemaMigrationsHistory (
+  Version INT64 NOT NULL,
+  Dirty BOOL NOT NULL,
+  Created TIMESTAMP NOT NULL OPTIONS (
+    allow_commit_timestamp = true
+  ),
+  Modified TIMESTAMP NOT NULL OPTIONS (
+    allow_commit_timestamp = true
+  ),
+) PRIMARY KEY(Version);
 ```
 
 ## Installation
@@ -107,7 +123,14 @@ This creates a next migration file like `_examples/migrations/000001.sql`. You w
 $ wrench migrate up --directory ./_examples
 ```
 
-This executes migrations. This also creates `SchemaMigrations` table into your database to manage schema version if it does not exist.
+This executes migrations. This also creates `SchemaMigrations` & `SchemaMigrationsHistory` tables in your database to manage schema version if it does not exist.
+
+### Migrations history
+```sh
+$ wrench migrate history
+```
+This displays the history of migrations applied to your database, ordered by when they were first attempted.
+Migrations left in a dirty state and subsequently retried are reflected in the Modified timestamp.
 
 ### Apply single DDL/DML
 

_examples/schema.sql

@@ -1,4 +1,21 @@
 CREATE TABLE Singers (
   SingerID STRING(36) NOT NULL,
   FirstName STRING(1024),
+  LastName STRING(1024),
 ) PRIMARY KEY(SingerID);
+
+CREATE TABLE SchemaMigrations (
+  Version INT64 NOT NULL,
+  Dirty BOOL NOT NULL,
+) PRIMARY KEY(Version);
+
+CREATE TABLE SchemaMigrationsHistory (
+  Version INT64 NOT NULL,
+  Dirty BOOL NOT NULL,
+  Created TIMESTAMP NOT NULL OPTIONS (
+    allow_commit_timestamp = true
+  ),
+  Modified TIMESTAMP NOT NULL OPTIONS (
+    allow_commit_timestamp = true
+  ),
+) PRIMARY KEY(Version);

cmd/migrate.go

@@ -25,8 +25,10 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"sort"
 	"strconv"
 	"strings"
+	"text/tabwriter"
 
 	"github.com/cloudspannerecosystem/wrench/pkg/spanner"
 	"github.com/spf13/cobra"
@@ -64,12 +66,18 @@ func init() {
 		Short: "Set version V but don't run migration (ignores dirty state)",
 		RunE:  migrateSet,
 	}
+	migrateHistoryCmd := &cobra.Command{
+		Use: "history",
+		Short: "Print migration version history",
+		RunE: migrateHistory,
+	}
 
 	migrateCmd.AddCommand(
 		migrateCreateCmd,
 		migrateUpCmd,
 		migrateVersionCmd,
 		migrateSetCmd,
+		migrateHistoryCmd,
 	)
 
 	migrateCmd.PersistentFlags().String(flagNameDirectory, "", "Directory that migration files placed (required)")
@@ -127,23 +135,40 @@ func migrateUp(c *cobra.Command, args []string) error {
 	}
 	defer client.Close()
 
+	dir := filepath.Join(c.Flag(flagNameDirectory).Value.String(), migrationsDirName)
+	migrations, err := spanner.LoadMigrations(dir)
+	if err != nil {
+		return &Error{
+			cmd: c,
+			err: err,
+		}
+	}
+
 	if err = client.EnsureMigrationTable(ctx, migrationTableName); err != nil {
 		return &Error{
 			cmd: c,
 			err: err,
 		}
 	}
 
-	dir := filepath.Join(c.Flag(flagNameDirectory).Value.String(), migrationsDirName)
-	migrations, err := spanner.LoadMigrations(dir)
+	status, err := client.DetermineUpgradeStatus(ctx, migrationTableName)
 	if err != nil {
 		return &Error{
 			cmd: c,
 			err: err,
 		}
 	}
-
-	return client.ExecuteMigrations(ctx, migrations, limit, migrationTableName)
+	switch status {
+	case spanner.ExistingMigrationsUpgradeStarted:
+		return client.UpgradeExecuteMigrations(ctx, migrations, limit, migrationTableName)
+	case spanner.ExistingMigrationsUpgradeCompleted:
+		return client.ExecuteMigrations(ctx, migrations, limit, migrationTableName)
+	default:
+		return &Error{
+			cmd: c,
+			err: errors.New("migration in undetermined state"),
+		}
+	}
 }
 
 func migrateVersion(c *cobra.Command, args []string) error {
@@ -180,6 +205,34 @@ func migrateVersion(c *cobra.Command, args []string) error {
 	return nil
 }
 
+func migrateHistory(c *cobra.Command, args []string) error {
+	ctx := context.Background()
+
+	client, err := newSpannerClient(ctx, c)
+	if err != nil {
+		return err
+	}
+	defer client.Close()
+
+	history, err := client.GetMigrationHistory(ctx, migrationTableName)
+	if err != nil {
+		return err
+	}
+	sort.SliceStable(history, func(i, j int) bool {
+		return history[i].Created.Before(history[j].Created) // order by Created
+	})
+
+	writer := tabwriter.NewWriter(os.Stdout, 0, 8, 1, '\t', tabwriter.AlignRight)
+	fmt.Fprintln(writer, "Version\tDirty\tCreated\tModified")
+	for i := range history {
+		h := history[i]
+		fmt.Fprintf(writer, "%d\t%v\t%v\t%v\n", h.Version, h.Dirty, h.Created, h.Modified)
+	}
+	writer.Flush()
+
+	return nil
+}
+
 func migrateSet(c *cobra.Command, args []string) error {
 	ctx := context.Background()