WIP: Update DBs and clients for ingest and storage

Makefile

@@ -106,8 +106,8 @@ gen-enums: # @HELP Generate go-enum assets.
 gen-enums: ENUM_FLAGS = --names --template=$(CURDIR)/hack/make/enums.tmpl
 gen-enums: $(GO_ENUM)
 	go-enum $(ENUM_FLAGS) \
-		-f internal/enums/package_type.go \
-		-f internal/enums/pkg_status.go \
+		-f internal/enums/sip_type.go \
+		-f internal/enums/sip_status.go \
 		-f internal/enums/preprocessing_task_outcome.go \
 		-f internal/enums/pres_action_status.go \
 		-f internal/enums/pres_action_type.go \

cmd/migrate/main.go

@@ -1,22 +1,33 @@
-// Example:
+// Example (ingest):
+//
+//  1. Make changes to schema files (internal/persistence/ent/schema),
+//  2. Re-generate (make gen-ent),
+//  3. Use an empty MySQL database,
+//  4. Run:
+//     $ go run ./cmd/migrate/ \
+//     --db="ingest" \
+//     --dsn="mysql://root:root123@tcp(localhost:3306)/enduro_migrate" \
+//     --path="./internal/db/migrations" \
+//     --name="changes"
+//
+// Example (storage):
 //
 //  1. Make changes to schema files (internal/storage/persistence/ent/schema),
 //  2. Re-generate (make gen-ent),
-//  3. Drop any existing database tables or delete and re-create the database,
+//  3. Use an empty MySQL database,
 //  4. Run:
 //     $ go run ./cmd/migrate/ \
-//     --config="./enduro.toml" \
-//     --dsn="mysql://enduro:enduro123@tcp(localhost:3306)/enduro_storage" \
-//     --name="init" \
-//     --path="./internal/storage/persistence/migrations"
+//     --db="storage" \
+//     --dsn="mysql://root:root123@tcp(localhost:3306)/enduro_migrate" \
+//     --path="./internal/storage/persistence/migrations" \
+//     --name="changes"
 package main
 
 import (
 	"context"
 	"fmt"
 	"log"
 	"os"
-	"path/filepath"
 	"strings"
 
 	"ariga.io/atlas/sql/sqltool"
@@ -25,40 +36,38 @@ import (
 	"github.com/go-sql-driver/mysql"
 	"github.com/spf13/pflag"
 
-	"github.com/artefactual-sdps/enduro/internal/config"
-	"github.com/artefactual-sdps/enduro/internal/storage/persistence/ent/db/migrate"
+	ingest_migrate "github.com/artefactual-sdps/enduro/internal/persistence/ent/db/migrate"
+	storage_migrate "github.com/artefactual-sdps/enduro/internal/storage/persistence/ent/db/migrate"
 )
 
 func main() {
 	p := pflag.NewFlagSet("migrate", pflag.ExitOnError)
-	p.String("config", "", "Configuration file")
+	p.String("db", "", "Enduro database ('ingest' or 'storage')")
 	p.String("dsn", "", "MySQL DSN")
 	p.String("path", "", "Migration directory")
 	p.String("name", "changes", "Migration name")
 	_ = p.Parse(os.Args[1:])
 
-	path, _ := p.GetString("path")
-	if path == "" {
-		wd, err := os.Getwd()
-		if err != nil {
-			os.Exit(1)
-		}
-		// Guessing that running it from the root folder.
-		path = filepath.Join(wd, "internal/storage/persistence/migrations")
+	db, _ := p.GetString("db")
+	if db == "" {
+		fmt.Printf("--db flag is missing")
+		os.Exit(1)
+	}
+	if db != "ingest" && db != "storage" {
+		fmt.Printf("--db flag has an unexpected value (use 'ingest' or 'storage')")
+		os.Exit(1)
 	}
 
-	var cfg config.Configuration
-	configFile, _ := p.GetString("config")
-	_, _, err := config.Read(&cfg, configFile)
-	if err != nil {
-		fmt.Printf("Failed to read configuration: %v\n", err)
+	DSN, _ := p.GetString("dsn")
+	if DSN == "" {
+		fmt.Printf("--dsn flag is missing")
 		os.Exit(1)
 	}
 
-	DSN := cfg.Storage.Database.DSN
-	flagDSN, _ := p.GetString("dsn")
-	if flagDSN != "" {
-		DSN = flagDSN
+	path, _ := p.GetString("path")
+	if path == "" {
+		fmt.Printf("--path flag is missing")
+		os.Exit(1)
 	}
 
 	// MySQL's DSN format is not accepted by Ent, convert as needed (remove Net).
@@ -89,11 +98,17 @@ func main() {
 		schema.WithDir(dir),                         // provide migration directory
 		schema.WithMigrationMode(schema.ModeReplay), // provide migration mode
 		schema.WithDialect(dialect.MySQL),           // Ent dialect to use
+		schema.WithDropIndex(true),
+		schema.WithDropColumn(true),
 	}
 
 	// Generate migrations using Atlas support for TiDB (note the Ent dialect option passed above).
 	name, _ := p.GetString("name")
-	err = migrate.NamedDiff(ctx, entDSN, name, opts...)
+	if db == "ingest" {
+		err = ingest_migrate.NamedDiff(ctx, entDSN, name, opts...)
+	} else {
+		err = storage_migrate.NamedDiff(ctx, entDSN, name, opts...)
+	}
 	if err != nil {
 		log.Fatalf("failed generating migration file: %v", err)
 	}

internal/am/job_tracker_test.go

@@ -211,7 +211,7 @@ func TestConvertJobToPreservationTask(t *testing.T) {
 			want: datatypes.PreservationTask{
 				TaskID: "f60018ac-da79-4769-9509-c6c41d5efe7e",
 				Name:   "Move to processing directory",
-				Status: enums.PreservationTaskStatus(enums.PackageStatusDone),
+				Status: enums.PreservationTaskStatus(enums.SIPStatusDone),
 				StartedAt: sql.NullTime{
 					Time:  time.Date(2024, time.January, 18, 1, 27, 49, 0, time.UTC),
 					Valid: true,
@@ -251,7 +251,7 @@ func TestConvertJobToPreservationTask(t *testing.T) {
 			want: datatypes.PreservationTask{
 				TaskID: "c2128d39-2ace-47c5-8cac-39ded8d9c9ef",
 				Name:   "Verify SIP compliance",
-				Status: enums.PreservationTaskStatus(enums.PackageStatusInProgress),
+				Status: enums.PreservationTaskStatus(enums.SIPStatusInProgress),
 				StartedAt: sql.NullTime{
 					Time:  time.Date(2024, time.January, 18, 1, 27, 49, 0, time.UTC),
 					Valid: true,

internal/api/design/package_.go

@@ -266,7 +266,7 @@ var _ = Service("package", func() {
 })
 
 var EnumPackageStatus = func() {
-	Enum(enums.PackageStatusInterfaces()...)
+	Enum(enums.SIPStatusInterfaces()...)
 }
 
 var Package_ = Type("Package", func() {
@@ -275,7 +275,7 @@ var Package_ = Type("Package", func() {
 	TypedAttributeUUID("location_id", "Identifier of storage location")
 	Attribute("status", String, "Status of the package", func() {
 		EnumPackageStatus()
-		Default(enums.PackageStatusNew.String())
+		Default(enums.SIPStatusNew.String())
 	})
 	AttributeUUID("workflow_id", "Identifier of processing workflow")
 	AttributeUUID("run_id", "Identifier of latest processing workflow run")

internal/datatypes/preservation_action.go

@@ -14,5 +14,5 @@ type PreservationAction struct {
 	Status      enums.PreservationActionStatus `db:"status"`
 	StartedAt   sql.NullTime                   `db:"started_at"`
 	CompletedAt sql.NullTime                   `db:"completed_at"`
-	PackageID   int                            `db:"package_id"`
+	SIPID       int                            `db:"sip_id"`
 }

internal/datatypes/package_.go → internal/datatypes/sip.go

@@ -12,15 +12,15 @@ import (
 	"github.com/artefactual-sdps/enduro/internal/enums"
 )
 
-// Package represents a package in the package table.
-type Package struct {
-	ID         int                 `db:"id"`
-	Name       string              `db:"name"`
-	WorkflowID string              `db:"workflow_id"`
-	RunID      string              `db:"run_id"`
-	AIPID      uuid.NullUUID       `db:"aip_id"`      // Nullable.
-	LocationID uuid.NullUUID       `db:"location_id"` // Nullable.
-	Status     enums.PackageStatus `db:"status"`
+// SIP represents a SIP in the sip table.
+type SIP struct {
+	ID         int             `db:"id"`
+	Name       string          `db:"name"`
+	WorkflowID string          `db:"workflow_id"`
+	RunID      string          `db:"run_id"`
+	AIPID      uuid.NullUUID   `db:"aip_id"`      // Nullable.
+	LocationID uuid.NullUUID   `db:"location_id"` // Nullable.
+	Status     enums.SIPStatus `db:"status"`
 
 	// It defaults to CURRENT_TIMESTAMP(6) so populated as soon as possible.
 	CreatedAt time.Time `db:"created_at"`
@@ -33,7 +33,7 @@ type Package struct {
 }
 
 // Goa returns the API representation of the package.
-func (p *Package) Goa() *goapackage.EnduroStoredPackage {
+func (p *SIP) Goa() *goapackage.EnduroStoredPackage {
 	if p == nil {
 		return nil
 	}

internal/db/migrations/20250207193001_rename_package_table.down.sql

@@ -0,0 +1 @@
+RENAME TABLE sip TO package;

internal/db/migrations/20250207193001_rename_package_table.up.sql

@@ -0,0 +1 @@
+RENAME TABLE package TO sip;

internal/db/migrations/20250207204127_use_atlas.down.sql

@@ -0,0 +1,6 @@
+-- reverse: modify "preservation_task" table
+ALTER TABLE `preservation_task` DROP FOREIGN KEY `preservation_task_preservation_action_tasks`, ADD CONSTRAINT `preservation_task_ibfk_1` FOREIGN KEY (`preservation_action_id`) REFERENCES `preservation_action` (`id`) ON UPDATE NO ACTION ON DELETE CASCADE, DROP INDEX `preservation_task_preservation_action_tasks`, MODIFY COLUMN `preservation_action_id` int unsigned NOT NULL, MODIFY COLUMN `note` longtext NOT NULL COLLATE utf8mb4_0900_ai_ci, MODIFY COLUMN `completed_at` timestamp(6) NULL, MODIFY COLUMN `started_at` timestamp(6) NULL, MODIFY COLUMN `name` varchar(2048) NOT NULL COLLATE utf8mb4_0900_ai_ci, MODIFY COLUMN `task_id` varchar(36) NOT NULL COLLATE utf8mb4_0900_ai_ci, MODIFY COLUMN `id` int unsigned NOT NULL AUTO_INCREMENT, COLLATE utf8mb4_0900_ai_ci;
+-- reverse: modify "preservation_action" table
+ALTER TABLE `preservation_action` DROP FOREIGN KEY `preservation_action_sip_preservation_actions`, ADD CONSTRAINT `preservation_action_ibfk_1` FOREIGN KEY (`package_id`) REFERENCES `sip` (`id`) ON UPDATE NO ACTION ON DELETE CASCADE, DROP INDEX `preservation_action_sip_preservation_actions`, DROP COLUMN `sip_id`, ADD COLUMN `package_id` int unsigned NOT NULL, MODIFY COLUMN `completed_at` timestamp(6) NULL, MODIFY COLUMN `started_at` timestamp(6) NULL, MODIFY COLUMN `workflow_id` varchar(255) NOT NULL COLLATE utf8mb4_0900_ai_ci, MODIFY COLUMN `id` int unsigned NOT NULL AUTO_INCREMENT, COLLATE utf8mb4_0900_ai_ci;
+-- reverse: modify "sip" table
+ALTER TABLE `sip` DROP INDEX `sip_status_idx`, DROP INDEX `sip_started_at_idx`, DROP INDEX `sip_name_idx`, DROP INDEX `sip_location_id_idx`, DROP INDEX `sip_created_at_idx`, DROP INDEX `sip_aip_id_idx`, DROP INDEX `run_id`, ADD INDEX `package_status_idx` (`status`), ADD INDEX `package_started_at_idx` (`started_at`), ADD INDEX `package_name_idx` (`name` (50)), ADD INDEX `package_location_id_idx` (`location_id`), ADD INDEX `package_created_at_idx` (`created_at`), ADD INDEX `package_aip_id_idx` (`aip_id`), MODIFY COLUMN `completed_at` timestamp(6) NULL, MODIFY COLUMN `started_at` timestamp(6) NULL, MODIFY COLUMN `created_at` timestamp(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), MODIFY COLUMN `location_id` varchar(36) NULL COLLATE utf8mb4_0900_ai_ci, MODIFY COLUMN `aip_id` varchar(36) NULL COLLATE utf8mb4_0900_ai_ci, MODIFY COLUMN `run_id` varchar(36) NOT NULL COLLATE utf8mb4_0900_ai_ci, MODIFY COLUMN `workflow_id` varchar(255) NOT NULL COLLATE utf8mb4_0900_ai_ci, MODIFY COLUMN `name` varchar(2048) NOT NULL COLLATE utf8mb4_0900_ai_ci, MODIFY COLUMN `id` int unsigned NOT NULL AUTO_INCREMENT, COLLATE utf8mb4_0900_ai_ci;

internal/db/migrations/20250207204127_use_atlas.up.sql

@@ -0,0 +1,6 @@
+-- modify "sip" table
+ALTER TABLE `sip` COLLATE utf8mb4_bin, MODIFY COLUMN `id` bigint NOT NULL AUTO_INCREMENT, MODIFY COLUMN `name` varchar(2048) NOT NULL, MODIFY COLUMN `workflow_id` varchar(255) NOT NULL, MODIFY COLUMN `run_id` char(36) NOT NULL, MODIFY COLUMN `aip_id` char(36) NULL, MODIFY COLUMN `location_id` char(36) NULL, MODIFY COLUMN `created_at` timestamp NOT NULL, MODIFY COLUMN `started_at` timestamp NULL, MODIFY COLUMN `completed_at` timestamp NULL, DROP INDEX `package_aip_id_idx`, DROP INDEX `package_created_at_idx`, DROP INDEX `package_location_id_idx`, DROP INDEX `package_name_idx`, DROP INDEX `package_started_at_idx`, DROP INDEX `package_status_idx`, ADD UNIQUE INDEX `run_id` (`run_id`), ADD INDEX `sip_aip_id_idx` (`aip_id`), ADD INDEX `sip_created_at_idx` (`created_at`), ADD INDEX `sip_location_id_idx` (`location_id`), ADD INDEX `sip_name_idx` (`name` (50)), ADD INDEX `sip_started_at_idx` (`started_at`), ADD INDEX `sip_status_idx` (`status`);
+-- modify "preservation_action" table
+ALTER TABLE `preservation_action` COLLATE utf8mb4_bin, MODIFY COLUMN `id` bigint NOT NULL AUTO_INCREMENT, MODIFY COLUMN `workflow_id` varchar(255) NOT NULL, MODIFY COLUMN `started_at` timestamp NULL, MODIFY COLUMN `completed_at` timestamp NULL, DROP COLUMN `package_id`, ADD COLUMN `sip_id` bigint NOT NULL, ADD INDEX `preservation_action_sip_preservation_actions` (`sip_id`), DROP FOREIGN KEY `preservation_action_ibfk_1`, ADD CONSTRAINT `preservation_action_sip_preservation_actions` FOREIGN KEY (`sip_id`) REFERENCES `sip` (`id`) ON UPDATE NO ACTION ON DELETE CASCADE;
+-- modify "preservation_task" table
+ALTER TABLE `preservation_task` COLLATE utf8mb4_bin, MODIFY COLUMN `id` bigint NOT NULL AUTO_INCREMENT, MODIFY COLUMN `task_id` char(36) NOT NULL, MODIFY COLUMN `name` varchar(2048) NOT NULL, MODIFY COLUMN `started_at` timestamp NULL, MODIFY COLUMN `completed_at` timestamp NULL, MODIFY COLUMN `note` longtext NOT NULL, MODIFY COLUMN `preservation_action_id` bigint NOT NULL, ADD INDEX `preservation_task_preservation_action_tasks` (`preservation_action_id`), DROP FOREIGN KEY `preservation_task_ibfk_1`, ADD CONSTRAINT `preservation_task_preservation_action_tasks` FOREIGN KEY (`preservation_action_id`) REFERENCES `preservation_action` (`id`) ON UPDATE NO ACTION ON DELETE CASCADE;

internal/db/migrations/atlas.sum

@@ -0,0 +1,9 @@
+h1:w+PCoLZubbqC3OjanelOkLkm8latla/U8szWEuCeUCw=
+1570659451_init.down.sql h1:hZkExHm7J+Njznl8jVlCA3ePmlKfnCi3h0UFLvgU0kU=
+1570659451_init.up.sql h1:/Bi8TK4Lt/bWcuxnQMqAvRO3e/NhFDQusiR3fjk5Z3c=
+1710442322_nullable_aip_id.down.sql h1:naFYM+Aybp/Vz8EwJOG7fRf/CWmV87qwylR4YR/76XE=
+1710442322_nullable_aip_id.up.sql h1:POGPYt8Mt0rWGMfns5JtXF4WAadA+q2Lpk9i69BanKc=
+20250207193001_rename_package_table.down.sql h1:LtX8QzPCvUyR7lqfEJRuX394D3I6YuedEBbE/o3VBkc=
+20250207193001_rename_package_table.up.sql h1:d4wwV9M/xPJhmTLB2fxMG00PMkKtyfRdXA/8pxPB3J8=
+20250207204127_use_atlas.down.sql h1:gZ3Cz1x5FG9HHgNpYEFltj2x4eAnTU/U+evjoS+/tx8=
+20250207204127_use_atlas.up.sql h1:qZqCNH5J6wjoAtRHAkrtdsbStkLLmRZV/PHgq5fSxoc=