Merge pull request #25 from keboola/COM-1343-ondra

Migrate Data of Tables directly
keboola · Feb 15, 2022 · 813d898 · 813d898
2 parents f9d5f75 + d2bf6ad
commit 813d898
Show file tree

Hide file tree

Showing 5 changed files with 109 additions and 31 deletions.
diff --git a/src/Component.php b/src/Component.php
@@ -73,6 +73,7 @@ protected function run(): void
             JobRunnerFactory::create($destProjectClient, $logger),
             $config->getSourceProjectUrl(),
             $config->getSourceProjectToken(),
+            $config->directDataMigration(),
             $logger
         );
         $migrate->run();

diff --git a/src/Config.php b/src/Config.php
@@ -12,6 +12,7 @@ class Config extends BaseConfig
     public const PROJECT_RESTORE_COMPONENT = 'keboola.project-restore';
     public const ORCHESTRATOR_MIGRATE_COMPONENT = 'keboola.app-orchestrator-migrate';
     public const SNOWFLAKE_WRITER_MIGRATE_COMPONENT = 'keboola.app-snowflake-writer-migrate';
+    public const DATA_OF_TABLES_MIGRATE_COMPONENT = 'keboola.app-project-migrate-large-tables';
 
     public function getSourceProjectUrl(): string
     {
@@ -22,4 +23,9 @@ public function getSourceProjectToken(): string
     {
         return $this->getValue(['parameters', '#sourceKbcToken']);
     }
+
+    public function directDataMigration(): bool
+    {
+        return $this->getValue(['parameters', 'directDataMigration']);
+    }
 }
diff --git a/src/ConfigDefinition.php b/src/ConfigDefinition.php
@@ -25,6 +25,7 @@ protected function getParametersDefinition(): ArrayNodeDefinition
                     ->isRequired()
                     ->cannotBeEmpty()
                 ->end()
+                ->booleanNode('directDataMigration')->defaultTrue()->end()
             ->end()
         ;
         // @formatter:on

diff --git a/src/Migrate.php b/src/Migrate.php
@@ -24,17 +24,21 @@ class Migrate
 
     private LoggerInterface $logger;
 
+    private bool $directDataMigration;
+
     public function __construct(
         JobRunner $sourceJobRunner,
         JobRunner $destJobRunner,
         string $sourceProjectUrl,
         string $sourceProjectToken,
+        bool $directDataMigration,
         LoggerInterface $logger
     ) {
         $this->sourceJobRunner = $sourceJobRunner;
         $this->destJobRunner = $destJobRunner;
         $this->sourceProjectUrl = $sourceProjectUrl;
         $this->sourceProjectToken = $sourceProjectToken;
+        $this->directDataMigration = $directDataMigration;
         $this->logger = $logger;
     }
 
@@ -44,6 +48,11 @@ public function run(): void
         try {
             $this->backupSourceProject($restoreCredentials['backupId']);
             $this->restoreDestinationProject($restoreCredentials);
+
+            if ($this->directDataMigration) {
+                $this->migrateDataOfTablesDirectly();
+            }
+
             $this->migrateSnowflakeWriters();
             if ($this->sourceJobRunner instanceof SyrupJobRunner) {
                 $this->migrateOrchestrations();
@@ -78,6 +87,7 @@ private function backupSourceProject(string $backupId): void
             [
                 'parameters' => [
                     'backupId' => $backupId,
+                    'exportStructureOnly' => $this->directDataMigration,
                 ],
             ]
         );
@@ -102,6 +112,23 @@ private function restoreDestinationProject(array $restoreCredentials): void
         $this->logger->info('Current project restored');
     }
 
+    private function migrateDataOfTablesDirectly(): void
+    {
+        $this->logger->info('Migrate data of tables directly.');
+
+        $this->destJobRunner->runJob(
+            Config::DATA_OF_TABLES_MIGRATE_COMPONENT,
+            [
+                'parameters' => [
+                    'sourceKbcUrl' => $this->sourceProjectUrl,
+                    '#sourceKbcToken' => $this->sourceProjectToken,
+                ],
+            ]
+        );
+
+        $this->logger->info('Data of tables has been migrated.');
+    }
+
     private function migrateOrchestrations(): void
     {
         $this->logger->info('Migrating orchestrations');

diff --git a/tests/phpunit/MigrateTest.php b/tests/phpunit/MigrateTest.php
@@ -26,6 +26,7 @@ class MigrateTest extends TestCase
     public function testMigrateSuccess(
         array $expectedCredentialsData,
         string $jobRunnerClass,
+        bool $migrateDataOfTablesDirectly,
         int $expectsRunJobs
     ): void {
         $sourceClientMock = $this->createMock($jobRunnerClass);
@@ -42,44 +43,62 @@ public function testMigrateSuccess(
             [
                 'id' => '222',
                 'status' => 'success',
-            ]
+            ],
+            $migrateDataOfTablesDirectly
         );
 
         $sourceProjectUrl = 'https://connection.keboola.com';
         $sourceProjectToken = 'xyz';
 
-        // run restore with credentials from step 1
-        $destClientMock->expects($this->exactly($expectsRunJobs))
-            ->method('runJob')
-            ->withConsecutive(
+        $destinationMockJobs = [
             // restore data
+            [
+                Config::PROJECT_RESTORE_COMPONENT,
                 [
-                    Config::PROJECT_RESTORE_COMPONENT,
-                    [
-                        'parameters' => array_merge($expectedCredentialsData, ['useDefaultBackend' => true]),
-                    ],
+                    'parameters' => array_merge($expectedCredentialsData, ['useDefaultBackend' => true]),
                 ],
-                // restore snowflake writers
+            ],
+        ];
+
+        // migrate data of tables
+        if ($migrateDataOfTablesDirectly) {
+            $destinationMockJobs[] = [
+                Config::DATA_OF_TABLES_MIGRATE_COMPONENT,
                 [
-                    Config::SNOWFLAKE_WRITER_MIGRATE_COMPONENT,
-                    [
-                        'parameters' => [
-                            'sourceKbcUrl' => $sourceProjectUrl,
-                            '#sourceKbcToken' => $sourceProjectToken,
-                        ],
+                    'parameters' => [
+                        'sourceKbcUrl' => $sourceProjectUrl,
+                        '#sourceKbcToken' => $sourceProjectToken,
                     ],
                 ],
-                // restore orchestrations
-                [
-                    Config::ORCHESTRATOR_MIGRATE_COMPONENT,
-                    [
-                        'parameters' => [
-                            'sourceKbcUrl' => $sourceProjectUrl,
-                            '#sourceKbcToken' => $sourceProjectToken,
-                        ],
-                    ],
-                ]
-            )->willReturn([
+            ];
+        }
+
+        // restore snowflake writers
+        $destinationMockJobs[] = [
+            Config::SNOWFLAKE_WRITER_MIGRATE_COMPONENT,
+            [
+                'parameters' => [
+                    'sourceKbcUrl' => $sourceProjectUrl,
+                    '#sourceKbcToken' => $sourceProjectToken,
+                ],
+            ],
+        ];
+
+        // restore orchestrations
+        $destinationMockJobs[] = [
+            Config::ORCHESTRATOR_MIGRATE_COMPONENT,
+            [
+                'parameters' => [
+                    'sourceKbcUrl' => $sourceProjectUrl,
+                    '#sourceKbcToken' => $sourceProjectToken,
+                ],
+            ],
+        ];
+
+        // run restore with credentials from step 1
+        $destClientMock->expects($this->exactly($expectsRunJobs))
+            ->method('runJob')
+            ->withConsecutive(...$destinationMockJobs)->willReturn([
                 'id' => '222',
                 'status' => 'success',
             ]);
@@ -91,6 +110,7 @@ public function testMigrateSuccess(
             $destClientMock,
             $sourceProjectUrl,
             $sourceProjectToken,
+            $migrateDataOfTablesDirectly,
             new NullLogger()
         );
         $migrate->run();
@@ -111,7 +131,8 @@ public function testShouldFailOnSnapshotError(): void
                 'result' => [
                     'message' => 'Cannot snapshot project',
                 ],
-            ]
+            ],
+            false
         );
 
         $destClientMock->expects($this->never())
@@ -124,6 +145,7 @@ public function testShouldFailOnSnapshotError(): void
             $destClientMock,
             'xxx',
             'yyy',
+            false,
             new NullLogger()
         );
         $migrate->run();
@@ -140,7 +162,8 @@ public function testShouldFailOnRestoreError(): void
             [
             'id' => '222',
                 'status' => 'success',
-            ]
+            ],
+            false
         );
 
         $destClientMock->expects($this->any())
@@ -160,6 +183,7 @@ public function testShouldFailOnRestoreError(): void
             $destClientMock,
             'xxx',
             'yyy',
+            false,
             new NullLogger()
         );
         $migrate->run();
@@ -176,7 +200,8 @@ public function testCatchSyrupClientException(): void
             [
                 'id' => '222',
                 'status' => 'success',
-            ]
+            ],
+            false
         );
 
         $destinationClientMock
@@ -191,6 +216,7 @@ public function testCatchSyrupClientException(): void
             $destinationClientMock,
             'xxx',
             'yyy',
+            false,
             new NullLogger()
         );
 
@@ -254,7 +280,7 @@ private function mockAddMethodGenerateAbsReadCredentials(MockObject $mockObject)
         ;
     }
 
-    private function mockAddMethodBackupProject(MockObject $mockObject, array $return): void
+    private function mockAddMethodBackupProject(MockObject $mockObject, array $return, bool $exportStructureOnly): void
     {
         $mockObject
             ->method('runJob')
@@ -263,6 +289,7 @@ private function mockAddMethodBackupProject(MockObject $mockObject, array $retur
                 [
                     'parameters' => [
                         'backupId' => '123',
+                        'exportStructureOnly' => $exportStructureOnly,
                     ],
                 ]
             )
@@ -282,6 +309,7 @@ public function successMigrateDataProvider(): Generator
                 ],
             ],
             SyrupJobRunner::class,
+            false,
             3,
         ];
 
@@ -293,6 +321,7 @@ public function successMigrateDataProvider(): Generator
                 ],
             ],
             SyrupJobRunner::class,
+            false,
             3,
         ];
 
@@ -306,6 +335,7 @@ public function successMigrateDataProvider(): Generator
                 ],
             ],
             QueueV2JobRunner::class,
+            false,
             2,
         ];
 
@@ -317,7 +347,20 @@ public function successMigrateDataProvider(): Generator
                 ],
             ],
             QueueV2JobRunner::class,
+            false,
             2,
         ];
+
+        yield 'migrateABS-queuev2-data-directly' => [
+            [
+                'abs' => [
+                    'container' => 'abcdefgh',
+                    '#connectionString' => 'https://testConnectionString',
+                ],
+            ],
+            QueueV2JobRunner::class,
+            true,
+            3,
+        ];
     }
 }