[Flyway]: Fixed integration tests + add new test cases (#111)

.github/workflows/ci-flyway-dialect.yaml

@@ -20,7 +20,7 @@ jobs:
 
     strategy:
       matrix:
-        java: [ '8', '11', '17' ]
+        java: [ '17', '21' ]
 
     steps:
       - uses: actions/checkout@v4

flyway-dialect/pom.xml

@@ -9,11 +9,12 @@
     <version>0.9.1-SNAPSHOT</version>
 
     <properties>
-        <maven.compiler.source>1.8</maven.compiler.source>
-        <maven.compiler.target>1.8</maven.compiler.target>
+        <maven.compiler.source>17</maven.compiler.source>
+        <maven.compiler.target>17</maven.compiler.target>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <flyway.version>10.7.1</flyway.version>
         <jdbc.ydb.driver.version>2.0.7</jdbc.ydb.driver.version>
+        <log4j2.version>2.17.2</log4j2.version>
     </properties>
 
     <dependencies>
@@ -30,6 +31,12 @@
             <version>${jdbc.ydb.driver.version}</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-slf4j-impl</artifactId>
+            <version>${log4j2.version}</version>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>tech.ydb.test</groupId>
             <artifactId>ydb-junit5-support</artifactId>

flyway-dialect/src/main/java/tech/ydb/flywaydb/database/YdbSchema.java

@@ -50,4 +50,9 @@ protected YdbTable[] doAllTables() {
     public YdbTable getTable(String tableName) {
         return new YdbTable(jdbcTemplate, database, this, tableName);
     }
+
+    @Override
+    public String toString() {
+        return "ydb_schema";
+    }
 }

flyway-dialect/src/main/java/tech/ydb/flywaydb/database/YdbTable.java

@@ -37,7 +37,12 @@ protected boolean doExists() throws SQLException {
     }
 
     @Override
-    protected void doLock() throws SQLException {
+    protected void doLock() {
+        if (lockDepth > 0) {
+            // Lock has already been taken - so the relevant row in the table already exists
+            return;
+        }
+
         Instant startLock = Instant.now();
 
         do {
@@ -49,17 +54,23 @@ protected void doLock() throws SQLException {
                 Thread.sleep(100 + random.nextInt(1000)); // pause 0.1s .. 1.1s
             } catch (InterruptedException ignored) {
             }
-        } while (startLock.minus(WAIT_LOCK_TIMEOUT).isAfter(Instant.now()));
+        } while (startLock.isAfter(Instant.now().minus(WAIT_LOCK_TIMEOUT)));
 
         throw new FlywayException("Unable to obtain table lock - another Flyway instance may be running");
     }
 
     @Override
     protected void doUnlock() {
+        if (lockDepth > 1) {
+            // Leave the locking row alone until we get to the final level of unlocking
+
+            return;
+        }
+
         for (int attempt = 0; attempt < RELEASE_MAX_ATTEMPT; attempt++) {
             SQLException sqlException = jdbcTemplate
-                    .executeStatement("DELETE FROM " + this + " WHERE installed_rank = -100")
-                    .getException();
+                    .executeStatement("DELETE FROM " + this +
+                            " WHERE installed_rank = -100 AND version = '" + tableLockId + "'").getException();
 
             if (sqlException == null) {
                 return;

flyway-dialect/src/test/java/tech/ydb/flyway/database/YdbFlywayTest.java

@@ -1,6 +1,21 @@
 package tech.ydb.flyway.database;
 
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
 import org.flywaydb.core.Flyway;
+import org.junit.jupiter.api.AfterEach;
+import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.RegisterExtension;
@@ -14,11 +29,140 @@ public class YdbFlywayTest {
     @RegisterExtension
     private static final YdbHelperExtension ydb = new YdbHelperExtension();
 
+    private static final String[] EVOLUTION_SCHEMA_MIGRATION_DIRS = new String[]{
+            "migration-step-1", "migration-step-2",
+            "migration-step-3", "migration-step-4",
+            "migration-step-5", "migration",
+    };
+
+    private static final Set<String> EXPECTED_SCRIPTS = Set.of(
+            "V1__create_series.sql", "V2__create_seasons.sql",
+            "V3__create_episodes.sql", "V4__load_data.sql",
+            "V5__create_series_title_index.sql", "V6__rename_index_title_index.sql"
+    );
+
+    @Test
+    void simpleTest() {
+        assertTrue(createFlyway("classpath:db/migration").migrate().success);
+
+        verifyTest();
+    }
+
     @Test
-    void migrationRollerTest() {
-        Flyway flyway = Flyway.configure().dataSource(jdbcUrl(), "", "").load();
+    void schemaHistoryStateTest() {
+        for (String migrationDir : EVOLUTION_SCHEMA_MIGRATION_DIRS) {
+            assertTrue(createFlyway("classpath:db/" + migrationDir).migrate().success);
+        }
+
+        verifyTest();
+    }
+
+    @Test
+    void schemaHistoryConcurrencyUpdateTest() throws ExecutionException, InterruptedException {
+        int threadPoolSize = 10;
+
+        ExecutorService threadPool = Executors.newFixedThreadPool(threadPoolSize);
+
+        for (int migrationStep = 0; migrationStep < EVOLUTION_SCHEMA_MIGRATION_DIRS.length; migrationStep++) {
+            List<Future<?>> taskFutures = new ArrayList<>();
+
+            for (int i = 0; i < threadPoolSize * 2; i++) {
+                int finalMigrationStep = migrationStep;
+
+                taskFutures.add(
+                        threadPool.submit(() -> assertTrue(
+                                createFlyway("classpath:db/" + EVOLUTION_SCHEMA_MIGRATION_DIRS[finalMigrationStep])
+                                        .migrate().success
+                        ))
+                );
+            }
+
+            for (Future<?> taskFuture : taskFutures) {
+                taskFuture.get();
+            }
+        }
+
+        verifyTest();
+    }
+
+    @Test
+        // https://documentation.red-gate.com/flyway/flyway-cli-and-api/commands/baseline
+    void baselineTest() throws SQLException {
+        try (Statement statement = DriverManager.getConnection(jdbcUrl()).createStatement()) {
+            statement.execute(
+                    "CREATE TABLE series " +
+                            "(" +
+                            "    series_id Uint64," +
+                            "    title Utf8," +
+                            "    series_info Utf8," +
+                            "    release_date Uint64," +
+                            "    PRIMARY KEY (series_id) " +
+                            ");" +
+                            "CREATE TABLE seasons" +
+                            "(" +
+                            "    series_id Uint64," +
+                            "    season_id Uint64," +
+                            "    title Utf8," +
+                            "    first_aired Uint64," +
+                            "    last_aired Uint64," +
+                            "    PRIMARY KEY (series_id, season_id)" +
+                            ");" +
+                            "CREATE TABLE episodes" +
+                            "(" +
+                            "    series_id Uint64," +
+                            "    season_id Uint64," +
+                            "    episode_id Uint64," +
+                            "    title Utf8," +
+                            "    air_date Uint64," +
+                            "    PRIMARY KEY (series_id, season_id, episode_id)" +
+                            ");"
+            );
+        }
+
+        assertTrue(createFlyway("empty").baseline().successfullyBaselined);
+    }
+
+    void verifyTest() {
+        try (Connection connection = DriverManager.getConnection(jdbcUrl())) {
+            assertCountTable(2, "SELECT COUNT(*) FROM series", connection);
+            assertCountTable(9, "SELECT COUNT(*) FROM seasons", connection);
+            assertCountTable(70, "SELECT COUNT(*) FROM episodes", connection);
+
+            ResultSet rs = connection.createStatement().executeQuery("SELECT script FROM flyway_schema_history;");
+            HashSet<String> scripts = new HashSet<>();
+
+            while (rs.next()) {
+                scripts.add(rs.getString(1));
+            }
+
+            assertEquals(EXPECTED_SCRIPTS, scripts);
+        } catch (SQLException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @AfterEach
+    void cleanYdb() throws SQLException {
+        try (Connection connection = DriverManager.getConnection(jdbcUrl())) {
+            connection.createStatement().execute("DROP TABLE series; DROP TABLE seasons; " +
+                    "DROP TABLE episodes; DROP TABLE flyway_schema_history;");
+        }
+    }
+
+    private static void assertCountTable(int expectedSize, String sql, Connection connection) throws SQLException {
+        Statement statement = connection.createStatement();
+
+        ResultSet rs = statement.executeQuery(sql);
+        rs.next();
+
+        assertEquals(expectedSize, rs.getLong(1));
+    }
 
-        assertTrue(flyway.migrate().success);
+    private static Flyway createFlyway(String migrationsDir) {
+        return Flyway.configure()
+                .locations(migrationsDir)
+                .dataSource(jdbcUrl(), "", "")
+                .load();
     }
 
     private static String jdbcUrl() {

flyway-dialect/src/test/resources/db/migration/V1_create_tables.sql → flyway-dialect/src/test/resources/db/migration-step-1/V1__create_series.sql

@@ -10,26 +10,4 @@ CREATE TABLE series -- series is the table name.
     -- non-repeating values). A table can have
     -- only one primary key. For every table
     -- in YDB, the primary key is required.
-);
-
-CREATE TABLE seasons
-(
-    series_id   Int64,
-    season_id   Int64,
-    title       Text,
-    first_aired Int64,
-    last_aired  Int64,
-    PRIMARY KEY (series_id, season_id)
-);
-
-CREATE TABLE episodes
-(
-    series_id  Int64,
-    season_id  Int64,
-    episode_id Int64,
-    title      Text,
-    air_date   Int64,
-    PRIMARY KEY (series_id, season_id, episode_id)
-);
-
-COMMIT;
+);

flyway-dialect/src/test/resources/db/migration-step-2/V1__create_series.sql

@@ -0,0 +1,13 @@
+CREATE TABLE series -- series is the table name.
+(                           -- Must be unique within the folder.
+    series_id    Int64,
+    title        Text,
+    series_info  Text,
+    release_date Int64,
+    PRIMARY KEY (series_id) -- The primary key is a column or
+    -- combination of columns that uniquely identifies
+    -- each table row (contains only
+    -- non-repeating values). A table can have
+    -- only one primary key. For every table
+    -- in YDB, the primary key is required.
+);

flyway-dialect/src/test/resources/db/migration-step-2/V2__create_seasons.sql

@@ -0,0 +1,10 @@
+
+CREATE TABLE seasons
+(
+    series_id Uint64,
+    season_id Uint64,
+    title Utf8,
+    first_aired Uint64,
+    last_aired Uint64,
+    PRIMARY KEY (series_id, season_id)
+);

flyway-dialect/src/test/resources/db/migration-step-3/V1__create_series.sql

@@ -0,0 +1,13 @@
+CREATE TABLE series -- series is the table name.
+(                           -- Must be unique within the folder.
+    series_id    Int64,
+    title        Text,
+    series_info  Text,
+    release_date Int64,
+    PRIMARY KEY (series_id) -- The primary key is a column or
+    -- combination of columns that uniquely identifies
+    -- each table row (contains only
+    -- non-repeating values). A table can have
+    -- only one primary key. For every table
+    -- in YDB, the primary key is required.
+);

flyway-dialect/src/test/resources/db/migration-step-3/V2__create_seasons.sql

@@ -0,0 +1,10 @@
+
+CREATE TABLE seasons
+(
+    series_id Uint64,
+    season_id Uint64,
+    title Utf8,
+    first_aired Uint64,
+    last_aired Uint64,
+    PRIMARY KEY (series_id, season_id)
+);

flyway-dialect/src/test/resources/db/migration-step-3/V3__create_episodes.sql

@@ -0,0 +1,9 @@
+CREATE TABLE episodes
+(
+    series_id Uint64,
+    season_id Uint64,
+    episode_id Uint64,
+    title Utf8,
+    air_date Uint64,
+    PRIMARY KEY (series_id, season_id, episode_id)
+);

flyway-dialect/src/test/resources/db/migration-step-4/V1__create_series.sql

@@ -0,0 +1,13 @@
+CREATE TABLE series -- series is the table name.
+(                           -- Must be unique within the folder.
+    series_id    Int64,
+    title        Text,
+    series_info  Text,
+    release_date Int64,
+    PRIMARY KEY (series_id) -- The primary key is a column or
+    -- combination of columns that uniquely identifies
+    -- each table row (contains only
+    -- non-repeating values). A table can have
+    -- only one primary key. For every table
+    -- in YDB, the primary key is required.
+);

flyway-dialect/src/test/resources/db/migration-step-4/V2__create_seasons.sql

@@ -0,0 +1,10 @@
+
+CREATE TABLE seasons
+(
+    series_id Uint64,
+    season_id Uint64,
+    title Utf8,
+    first_aired Uint64,
+    last_aired Uint64,
+    PRIMARY KEY (series_id, season_id)
+);

flyway-dialect/src/test/resources/db/migration-step-4/V3__create_episodes.sql

@@ -0,0 +1,9 @@
+CREATE TABLE episodes
+(
+    series_id Uint64,
+    season_id Uint64,
+    episode_id Uint64,
+    title Utf8,
+    air_date Uint64,
+    PRIMARY KEY (series_id, season_id, episode_id)
+);