From e014ba7a633587ed5b6299b4d5d4d7956034acdf Mon Sep 17 00:00:00 2001 From: Ondra Jodas Date: Mon, 14 Feb 2022 16:32:47 +0100 Subject: [PATCH 1/2] Run App Migrate Data of Tables --- src/Component.php | 1 + src/Config.php | 6 ++ src/ConfigDefinition.php | 1 + src/Migrate.php | 27 +++++++++ tests/phpunit/MigrateTest.php | 105 ++++++++++++++++++++++++---------- 5 files changed, 109 insertions(+), 31 deletions(-) diff --git a/src/Component.php b/src/Component.php index f6698f6..2aee1a3 100644 --- a/src/Component.php +++ b/src/Component.php @@ -73,6 +73,7 @@ protected function run(): void JobRunnerFactory::create($destProjectClient, $logger), $config->getSourceProjectUrl(), $config->getSourceProjectToken(), + $config->migrateDataDirectly(), $logger ); $migrate->run(); diff --git a/src/Config.php b/src/Config.php index 36bff9d..3aae62c 100644 --- a/src/Config.php +++ b/src/Config.php @@ -12,6 +12,7 @@ class Config extends BaseConfig public const PROJECT_RESTORE_COMPONENT = 'keboola.project-restore'; public const ORCHESTRATOR_MIGRATE_COMPONENT = 'keboola.app-orchestrator-migrate'; public const SNOWFLAKE_WRITER_MIGRATE_COMPONENT = 'keboola.app-snowflake-writer-migrate'; + public const DATA_OF_TABLES_MIGRATE_COMPONENT = 'keboola.app-project-migrate-large-tables'; public function getSourceProjectUrl(): string { @@ -22,4 +23,9 @@ public function getSourceProjectToken(): string { return $this->getValue(['parameters', '#sourceKbcToken']); } + + public function migrateDataDirectly(): bool + { + return $this->getValue(['parameters', 'migrateDataDirectly']); + } } diff --git a/src/ConfigDefinition.php b/src/ConfigDefinition.php index a9395b7..e0732e2 100644 --- a/src/ConfigDefinition.php +++ b/src/ConfigDefinition.php @@ -25,6 +25,7 @@ protected function getParametersDefinition(): ArrayNodeDefinition ->isRequired() ->cannotBeEmpty() ->end() + ->booleanNode('migrateDataDirectly')->defaultTrue()->end() ->end() ; // @formatter:on diff --git a/src/Migrate.php b/src/Migrate.php index cca473f..9a75b92 100644 --- a/src/Migrate.php +++ b/src/Migrate.php @@ -24,17 +24,21 @@ class Migrate private LoggerInterface $logger; + private bool $migrateDataDirectly; + public function __construct( JobRunner $sourceJobRunner, JobRunner $destJobRunner, string $sourceProjectUrl, string $sourceProjectToken, + bool $migrateDataDirectly, LoggerInterface $logger ) { $this->sourceJobRunner = $sourceJobRunner; $this->destJobRunner = $destJobRunner; $this->sourceProjectUrl = $sourceProjectUrl; $this->sourceProjectToken = $sourceProjectToken; + $this->migrateDataDirectly = $migrateDataDirectly; $this->logger = $logger; } @@ -44,6 +48,11 @@ public function run(): void try { $this->backupSourceProject($restoreCredentials['backupId']); $this->restoreDestinationProject($restoreCredentials); + + if ($this->migrateDataDirectly) { + $this->migrateDataOfTablesDirectly(); + } + $this->migrateSnowflakeWriters(); if ($this->sourceJobRunner instanceof SyrupJobRunner) { $this->migrateOrchestrations(); @@ -78,6 +87,7 @@ private function backupSourceProject(string $backupId): void [ 'parameters' => [ 'backupId' => $backupId, + 'exportStructureOnly' => $this->migrateDataDirectly, ], ] ); @@ -102,6 +112,23 @@ private function restoreDestinationProject(array $restoreCredentials): void $this->logger->info('Current project restored'); } + private function migrateDataOfTablesDirectly(): void + { + $this->logger->info('Migrate data of tables directly.'); + + $this->destJobRunner->runJob( + Config::DATA_OF_TABLES_MIGRATE_COMPONENT, + [ + 'parameters' => [ + 'sourceKbcUrl' => $this->sourceProjectUrl, + '#sourceKbcToken' => $this->sourceProjectToken, + ], + ] + ); + + $this->logger->info('Data of tables has been migrated.'); + } + private function migrateOrchestrations(): void { $this->logger->info('Migrating orchestrations'); diff --git a/tests/phpunit/MigrateTest.php b/tests/phpunit/MigrateTest.php index 296ea07..995db71 100644 --- a/tests/phpunit/MigrateTest.php +++ b/tests/phpunit/MigrateTest.php @@ -26,6 +26,7 @@ class MigrateTest extends TestCase public function testMigrateSuccess( array $expectedCredentialsData, string $jobRunnerClass, + bool $migrateDataOfTablesDirectly, int $expectsRunJobs ): void { $sourceClientMock = $this->createMock($jobRunnerClass); @@ -42,44 +43,62 @@ public function testMigrateSuccess( [ 'id' => '222', 'status' => 'success', - ] + ], + $migrateDataOfTablesDirectly ); $sourceProjectUrl = 'https://connection.keboola.com'; $sourceProjectToken = 'xyz'; - // run restore with credentials from step 1 - $destClientMock->expects($this->exactly($expectsRunJobs)) - ->method('runJob') - ->withConsecutive( + $destinationMockJobs = [ // restore data + [ + Config::PROJECT_RESTORE_COMPONENT, [ - Config::PROJECT_RESTORE_COMPONENT, - [ - 'parameters' => array_merge($expectedCredentialsData, ['useDefaultBackend' => true]), - ], + 'parameters' => array_merge($expectedCredentialsData, ['useDefaultBackend' => true]), ], - // restore snowflake writers + ], + ]; + + // migrate data of tables + if ($migrateDataOfTablesDirectly) { + $destinationMockJobs[] = [ + Config::DATA_OF_TABLES_MIGRATE_COMPONENT, [ - Config::SNOWFLAKE_WRITER_MIGRATE_COMPONENT, - [ - 'parameters' => [ - 'sourceKbcUrl' => $sourceProjectUrl, - '#sourceKbcToken' => $sourceProjectToken, - ], + 'parameters' => [ + 'sourceKbcUrl' => $sourceProjectUrl, + '#sourceKbcToken' => $sourceProjectToken, ], ], - // restore orchestrations - [ - Config::ORCHESTRATOR_MIGRATE_COMPONENT, - [ - 'parameters' => [ - 'sourceKbcUrl' => $sourceProjectUrl, - '#sourceKbcToken' => $sourceProjectToken, - ], - ], - ] - )->willReturn([ + ]; + } + + // restore snowflake writers + $destinationMockJobs[] = [ + Config::SNOWFLAKE_WRITER_MIGRATE_COMPONENT, + [ + 'parameters' => [ + 'sourceKbcUrl' => $sourceProjectUrl, + '#sourceKbcToken' => $sourceProjectToken, + ], + ], + ]; + + // restore orchestrations + $destinationMockJobs[] = [ + Config::ORCHESTRATOR_MIGRATE_COMPONENT, + [ + 'parameters' => [ + 'sourceKbcUrl' => $sourceProjectUrl, + '#sourceKbcToken' => $sourceProjectToken, + ], + ], + ]; + + // run restore with credentials from step 1 + $destClientMock->expects($this->exactly($expectsRunJobs)) + ->method('runJob') + ->withConsecutive(...$destinationMockJobs)->willReturn([ 'id' => '222', 'status' => 'success', ]); @@ -91,6 +110,7 @@ public function testMigrateSuccess( $destClientMock, $sourceProjectUrl, $sourceProjectToken, + $migrateDataOfTablesDirectly, new NullLogger() ); $migrate->run(); @@ -111,7 +131,8 @@ public function testShouldFailOnSnapshotError(): void 'result' => [ 'message' => 'Cannot snapshot project', ], - ] + ], + false ); $destClientMock->expects($this->never()) @@ -124,6 +145,7 @@ public function testShouldFailOnSnapshotError(): void $destClientMock, 'xxx', 'yyy', + false, new NullLogger() ); $migrate->run(); @@ -140,7 +162,8 @@ public function testShouldFailOnRestoreError(): void [ 'id' => '222', 'status' => 'success', - ] + ], + false ); $destClientMock->expects($this->any()) @@ -160,6 +183,7 @@ public function testShouldFailOnRestoreError(): void $destClientMock, 'xxx', 'yyy', + false, new NullLogger() ); $migrate->run(); @@ -176,7 +200,8 @@ public function testCatchSyrupClientException(): void [ 'id' => '222', 'status' => 'success', - ] + ], + false ); $destinationClientMock @@ -191,6 +216,7 @@ public function testCatchSyrupClientException(): void $destinationClientMock, 'xxx', 'yyy', + false, new NullLogger() ); @@ -254,7 +280,7 @@ private function mockAddMethodGenerateAbsReadCredentials(MockObject $mockObject) ; } - private function mockAddMethodBackupProject(MockObject $mockObject, array $return): void + private function mockAddMethodBackupProject(MockObject $mockObject, array $return, bool $exportStructureOnly): void { $mockObject ->method('runJob') @@ -263,6 +289,7 @@ private function mockAddMethodBackupProject(MockObject $mockObject, array $retur [ 'parameters' => [ 'backupId' => '123', + 'exportStructureOnly' => $exportStructureOnly, ], ] ) @@ -282,6 +309,7 @@ public function successMigrateDataProvider(): Generator ], ], SyrupJobRunner::class, + false, 3, ]; @@ -293,6 +321,7 @@ public function successMigrateDataProvider(): Generator ], ], SyrupJobRunner::class, + false, 3, ]; @@ -306,6 +335,7 @@ public function successMigrateDataProvider(): Generator ], ], QueueV2JobRunner::class, + false, 2, ]; @@ -317,7 +347,20 @@ public function successMigrateDataProvider(): Generator ], ], QueueV2JobRunner::class, + false, 2, ]; + + yield 'migrateABS-queuev2-data-directly' => [ + [ + 'abs' => [ + 'container' => 'abcdefgh', + '#connectionString' => 'https://testConnectionString', + ], + ], + QueueV2JobRunner::class, + true, + 3, + ]; } } From d2bf6ad571874110bc9a9da7c7ab72ff07e43c96 Mon Sep 17 00:00:00 2001 From: Ondra Jodas Date: Tue, 15 Feb 2022 10:46:50 +0100 Subject: [PATCH 2/2] Rename params node --- src/Component.php | 2 +- src/Config.php | 4 ++-- src/ConfigDefinition.php | 2 +- src/Migrate.php | 10 +++++----- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/Component.php b/src/Component.php index 2aee1a3..30a82d6 100644 --- a/src/Component.php +++ b/src/Component.php @@ -73,7 +73,7 @@ protected function run(): void JobRunnerFactory::create($destProjectClient, $logger), $config->getSourceProjectUrl(), $config->getSourceProjectToken(), - $config->migrateDataDirectly(), + $config->directDataMigration(), $logger ); $migrate->run(); diff --git a/src/Config.php b/src/Config.php index 3aae62c..154e800 100644 --- a/src/Config.php +++ b/src/Config.php @@ -24,8 +24,8 @@ public function getSourceProjectToken(): string return $this->getValue(['parameters', '#sourceKbcToken']); } - public function migrateDataDirectly(): bool + public function directDataMigration(): bool { - return $this->getValue(['parameters', 'migrateDataDirectly']); + return $this->getValue(['parameters', 'directDataMigration']); } } diff --git a/src/ConfigDefinition.php b/src/ConfigDefinition.php index e0732e2..dc48c79 100644 --- a/src/ConfigDefinition.php +++ b/src/ConfigDefinition.php @@ -25,7 +25,7 @@ protected function getParametersDefinition(): ArrayNodeDefinition ->isRequired() ->cannotBeEmpty() ->end() - ->booleanNode('migrateDataDirectly')->defaultTrue()->end() + ->booleanNode('directDataMigration')->defaultTrue()->end() ->end() ; // @formatter:on diff --git a/src/Migrate.php b/src/Migrate.php index 9a75b92..237ed1b 100644 --- a/src/Migrate.php +++ b/src/Migrate.php @@ -24,21 +24,21 @@ class Migrate private LoggerInterface $logger; - private bool $migrateDataDirectly; + private bool $directDataMigration; public function __construct( JobRunner $sourceJobRunner, JobRunner $destJobRunner, string $sourceProjectUrl, string $sourceProjectToken, - bool $migrateDataDirectly, + bool $directDataMigration, LoggerInterface $logger ) { $this->sourceJobRunner = $sourceJobRunner; $this->destJobRunner = $destJobRunner; $this->sourceProjectUrl = $sourceProjectUrl; $this->sourceProjectToken = $sourceProjectToken; - $this->migrateDataDirectly = $migrateDataDirectly; + $this->directDataMigration = $directDataMigration; $this->logger = $logger; } @@ -49,7 +49,7 @@ public function run(): void $this->backupSourceProject($restoreCredentials['backupId']); $this->restoreDestinationProject($restoreCredentials); - if ($this->migrateDataDirectly) { + if ($this->directDataMigration) { $this->migrateDataOfTablesDirectly(); } @@ -87,7 +87,7 @@ private function backupSourceProject(string $backupId): void [ 'parameters' => [ 'backupId' => $backupId, - 'exportStructureOnly' => $this->migrateDataDirectly, + 'exportStructureOnly' => $this->directDataMigration, ], ] );