From 83d491db6b5c6a32783d2e8053d7526ed0bffb9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roman=20Pi=C5=A1t=C4=9Bk?= Date: Thu, 19 Sep 2024 16:46:44 +0200 Subject: [PATCH 1/3] fix: Use same dest. snflk workspace for snflk wr configs that use the same workspace at the source --- src/Component.php | 1 + src/Migrate.php | 70 +++++++++++++++++++++++++++++++++++ tests/phpunit/MigrateTest.php | 15 ++++++++ 3 files changed, 86 insertions(+) diff --git a/src/Component.php b/src/Component.php index 1f9dd1f..afc4ec9 100644 --- a/src/Component.php +++ b/src/Component.php @@ -75,6 +75,7 @@ protected function run(): void $sourceJobRunner, $destJobRunner, $sourceProjectClient, + $destProjectClient, $migrationsClient, $destProjectClient->getApiUrl(), $destProjectClient->getTokenString(), diff --git a/src/Migrate.php b/src/Migrate.php index 8e084e4..81ae4a3 100644 --- a/src/Migrate.php +++ b/src/Migrate.php @@ -12,6 +12,7 @@ use Keboola\StorageApi\Client as StorageClient; use Keboola\StorageApi\Components; use Keboola\StorageApi\DevBranches; +use Keboola\StorageApi\Options\Components\Configuration; use Keboola\Syrup\ClientException as SyrupClientException; use Psr\Log\LoggerInterface; @@ -25,6 +26,8 @@ class Migrate private StorageClient $sourceProjectStorageClient; + private StorageClient $destProjectStorageClient; + private MigrationsClient $migrationsClient; private string $sourceProjectUrl; @@ -52,15 +55,25 @@ class Migrate 'gooddata-writer', ]; + public const SNOWFLAKE_WRITER_COMPONENT_IDS = [ + 'keboola.wr-db-snowflake', // aws + 'keboola.wr-snowflake-blob-storage', // azure + 'keboola.wr-db-snowflake-gcs', // gcp + 'keboola.wr-db-snowflake-gcs-s3', // gcp with s3 + ]; + private string $migrateDataMode; private array $db; + private array $migratedSnowflakeWorkspaces = []; + public function __construct( Config $config, JobRunner $sourceJobRunner, JobRunner $destJobRunner, StorageClient $sourceProjectStorageClient, + StorageClient $destProjectStorageClient, MigrationsClient $migrationsClient, string $destinationProjectUrl, string $destinationProjectToken, @@ -69,6 +82,7 @@ public function __construct( $this->sourceJobRunner = $sourceJobRunner; $this->destJobRunner = $destJobRunner; $this->sourceProjectStorageClient = $sourceProjectStorageClient; + $this->destProjectStorageClient = $destProjectStorageClient; $this->migrationsClient = $migrationsClient; $this->sourceProjectUrl = $config->getSourceProjectUrl(); $this->sourceProjectToken = $config->getSourceProjectToken(); @@ -203,6 +217,14 @@ private function migrateSecrets(): void $this->dryRun ); + if (in_array($component['id'], self::SNOWFLAKE_WRITER_COMPONENT_IDS, true)) { + $this->preserveProperSnowflakeWorkspace( + $response['data']['componentId'], + $response['data']['configId'], + $config + ); + } + $message = $response['message']; if ($this->dryRun) { $message = '[dry-run] ' . $message; @@ -316,4 +338,52 @@ private function getRestoreConfigData(array $restoreCredentials): array throw new UserException('Unrecognized restore credentials.'); } } + + private function preserveProperSnowflakeWorkspace( + string $destinationComponentId, + string $destinationConfigurationId, + array $sourceConfigurationData + ): void { + if ($this->dryRun) { + return; + } + + $snowflakeUser = $sourceConfigurationData['configuration']['parameters']['db']['user']; + + $destinationComponentsApi = new Components($this->destProjectStorageClient); + $destinationConfigurationData = (array) $destinationComponentsApi + ->getConfiguration($destinationComponentId, $destinationConfigurationId); + + $migratedWorkspaceParameters = $this->migratedSnowflakeWorkspaces[$snowflakeUser] ?? null; + + if ($migratedWorkspaceParameters) { + // Use the existing Snowflake workspace from a previous configuration that has the same source workspace + $destinationConfigurationData['configuration']['parameters']['db'] = $migratedWorkspaceParameters; + + $destinationConfiguration = (new Configuration()) + ->setConfigurationId($destinationConfigurationId) + ->setComponentId($destinationComponentId) + ->setName($destinationConfigurationData['name']) + ->setDescription($destinationConfigurationData['description']) + ->setIsDisabled($destinationConfigurationData['isDisabled']) + ->setConfiguration($destinationConfigurationData['configuration']); + + $destinationComponentsApi->updateConfiguration($destinationConfiguration); + + $this->logger->info( + sprintf( + "Used existing Snowflake workspace '%s' for configuration with ID '%s' (%s).", + $migratedWorkspaceParameters['user'], + $destinationConfigurationId, + $destinationComponentId, + ), + ['secrets'] + ); + return; + } + + // Store Snowflake workspace for next configurations + $workspaceParameters = $destinationConfigurationData['configuration']['parameters']['db']; + $this->migratedSnowflakeWorkspaces[$snowflakeUser] = $workspaceParameters; + } } diff --git a/tests/phpunit/MigrateTest.php b/tests/phpunit/MigrateTest.php index bd7bd54..2f8abb7 100644 --- a/tests/phpunit/MigrateTest.php +++ b/tests/phpunit/MigrateTest.php @@ -191,6 +191,8 @@ public function testMigrateSuccess( ->willReturn('https://encryption.keboola.com') ; + $destClientMock = $this->createMock(StorageClient::class); + $migrationsClientMock = $this->createMock(Migrations::class); $migrationsClientMock->expects(self::never())->method('migrateConfiguration'); @@ -201,6 +203,7 @@ public function testMigrateSuccess( $sourceJobRunnerMock, $destJobRunnerMock, $sourceClientMock, + $destClientMock, $migrationsClientMock, 'https://dest-stack/', 'dest-token', @@ -296,6 +299,8 @@ public function testMigrateSecretsSuccess(): void ->willReturn('https://encryption.keboola.com') ; + $destClientMock = $this->createMock(StorageClient::class); + $migrationsClientMock = $this->createMock(Migrations::class); $migrationsClientMock ->expects(self::exactly(3)) @@ -316,6 +321,7 @@ public function testMigrateSecretsSuccess(): void $sourceJobRunnerMock, $destJobRunnerMock, $sourceClientMock, + $destClientMock, $migrationsClientMock, 'https://dest-stack/', 'dest-token', @@ -356,6 +362,7 @@ public function testShouldFailOnSnapshotError(): void $sourceJobRunnerMock = $this->createMock(SyrupJobRunner::class); $destJobRunnerMock = $this->createMock(SyrupJobRunner::class); $sourceClientMock = $this->createMock(StorageClient::class); + $destClientMock = $this->createMock(StorageClient::class); $migrationsClientMock = $this->createMock(Migrations::class); // generate credentials @@ -395,6 +402,7 @@ public function testShouldFailOnSnapshotError(): void $sourceJobRunnerMock, $destJobRunnerMock, $sourceClientMock, + $destClientMock, $migrationsClientMock, 'xxx-b', 'yyy-b', @@ -408,6 +416,7 @@ public function testShouldFailOnRestoreError(): void $sourceJobRunnerMock = $this->createMock(SyrupJobRunner::class); $destJobRunnerMock = $this->createMock(SyrupJobRunner::class); $sourceClientMock = $this->createMock(StorageClient::class); + $destClientMock = $this->createMock(StorageClient::class); $migrationsClientMock = $this->createMock(Migrations::class); $this->mockAddMethodGenerateS3ReadCredentials($sourceJobRunnerMock); @@ -450,6 +459,7 @@ public function testShouldFailOnRestoreError(): void $sourceJobRunnerMock, $destJobRunnerMock, $sourceClientMock, + $destClientMock, $migrationsClientMock, 'xxx-b', 'yyy-b', @@ -463,6 +473,7 @@ public function testCatchSyrupClientException(): void $sourceJobRunnerMock = $this->createMock(SyrupJobRunner::class); $destJobRunnerMock = $this->createMock(SyrupJobRunner::class); $sourceClientMock = $this->createMock(StorageClient::class); + $destClientMock = $this->createMock(StorageClient::class); $migrationsClientMock = $this->createMock(Migrations::class); $this->mockAddMethodGenerateS3ReadCredentials($sourceJobRunnerMock); @@ -499,6 +510,7 @@ public function testCatchSyrupClientException(): void $sourceJobRunnerMock, $destJobRunnerMock, $sourceClientMock, + $destClientMock, $migrationsClientMock, 'xxx-b', 'yyy-b', @@ -590,6 +602,8 @@ public function testDryRunMode( ], ]); + $destClientMock = $this->createMock(StorageClient::class); + $migrationsClientMock = $this->createMock(Migrations::class); $migrationsClientMock ->method('migrateConfiguration') @@ -619,6 +633,7 @@ public function testDryRunMode( $sourceJobRunnerMock, $destJobRunnerMock, $sourceClientMock, + $destClientMock, $migrationsClientMock, 'https://dest-stack/', 'dest-token', From 20082ff26a2a2481ec223f28923b5d17773dd9c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roman=20Pi=C5=A1t=C4=9Bk?= Date: Fri, 20 Sep 2024 09:18:19 +0200 Subject: [PATCH 2/3] test: Cover reuse of the same snflk workspace with unit test --- tests/phpunit/MigrateTest.php | 206 ++++++++++++++++++++++++++++++++++ 1 file changed, 206 insertions(+) diff --git a/tests/phpunit/MigrateTest.php b/tests/phpunit/MigrateTest.php index 2f8abb7..9d18573 100644 --- a/tests/phpunit/MigrateTest.php +++ b/tests/phpunit/MigrateTest.php @@ -357,6 +357,212 @@ public function testMigrateSecretsSuccess(): void ); } + public function testMigrateSnowflakeWritersWithSharedWorkspacesSuccess(): void + { + $sourceJobRunnerMock = $this->createMock(QueueV2JobRunner::class); + $destJobRunnerMock = $this->createMock(QueueV2JobRunner::class); + + // generate credentials + $this->mockAddMethodGenerateAbsReadCredentials($sourceJobRunnerMock); + $this->mockAddMethodBackupProject( + $sourceJobRunnerMock, + [ + 'id' => '222', + 'status' => 'success', + ], + true + ); + + $destJobRunnerMock->method('runJob') + ->willReturn([ + 'id' => '222', + 'status' => 'success', + ]); + + $config = new Config( + [ + 'parameters' => [ + 'sourceKbcUrl' => 'https://connection.keboola.com', + '#sourceKbcToken' => 'xyz', + 'migrateSecrets' => true, + '#sourceManageToken' => 'manage-token', + ], + ], + new ConfigDefinition() + ); + + $logsHandler = new TestHandler(); + $logger = new Logger('tests', [$logsHandler]); + + $testConfigurations = [ + [ + 'id' => '101', + 'name' => 'My Snowflake Data Destination #1', + 'description' => '', + 'isDisabled' => false, + 'configuration' => [ + 'parameters' => [ + 'db' => [ + 'port' => 'port', + 'schema' => 'schema', + 'warehouse' => 'warehouse', + 'driver' => 'snowflake', + 'host' => 'host', + 'user' => 'USER_01', + 'database' => 'database', + '#password' => 'encrypted-password', + ], + ], + ], + ], + [ + 'id' => '102', + 'name' => 'My Snowflake Data Destination #2', + 'description' => '', + 'isDisabled' => false, + 'configuration' => [ + 'parameters' => [ + 'db' => [ + 'port' => 'port', + 'schema' => 'schema', + 'warehouse' => 'warehouse', + 'driver' => 'snowflake', + 'host' => 'host', + 'user' => 'USER_02', + 'database' => 'database', + '#password' => 'encrypted-password', + ], + ], + ], + ], + [ + 'id' => '103', + 'name' => 'My Snowflake Data Destination #3', + 'description' => '', + 'isDisabled' => false, + 'configuration' => [ + 'parameters' => [ + 'db' => [ + 'port' => 'port', + 'schema' => 'schema', + 'warehouse' => 'warehouse', + 'driver' => 'snowflake', + 'host' => 'host', + 'user' => 'USER_01', + 'database' => 'database', + '#password' => 'encrypted-password', + ], + ], + ], + ], + ]; + + $sourceClientMock = $this->createMock(StorageClient::class); + $sourceClientMock + ->method('apiGet') + ->willReturnMap([ + [ + 'dev-branches/', null, [], + [ + [ + 'id' => '123', + 'name' => 'default', + 'isDefault' => true, + ], + ], + ], + [ + 'components?include=', null, [], + [ + [ + 'id' => 'keboola.wr-db-snowflake', + 'configurations' => $testConfigurations, + ], + ], + ], + ]) + ; + $sourceClientMock + ->method('getServiceUrl') + ->with('encryption') + ->willReturn('https://encryption.keboola.com') + ; + + $destClientMock = $this->createMock(StorageClient::class); + $destClientMock + ->method('apiGet') + ->willReturnCallback(function ($url) use ($testConfigurations): ?array { + preg_match('~components/([^/]+)/configs/([^/]+)~', $url, $matches); + [, , $configId] = $matches + [null, null, null]; + return current(array_filter($testConfigurations, fn ($c) => $c['id'] === $configId)) ?: null; + }) + ; + + $migrationsClientMock = $this->createMock(Migrations::class); + $migrationsClientMock + ->expects(self::exactly(3)) + ->method('migrateConfiguration') + ->willReturnCallback(function (...$args) { + [, $destinationStack, , $componentId, $configId, $branchId] = $args; + return [ + 'message' => "Configuration with ID '$configId' successfully " . + "migrated to stack '$destinationStack'.", + 'data' => [ + 'destinationStack' => $destinationStack, + 'componentId' => $componentId, + 'configId' => $configId, + 'branchId' => $branchId, + ], + ]; + }); + + /** @var JobRunner $sourceJobRunnerMock */ + /** @var JobRunner $destJobRunnerMock */ + $migrate = new Migrate( + $config, + $sourceJobRunnerMock, + $destJobRunnerMock, + $sourceClientMock, + $destClientMock, + $migrationsClientMock, + 'https://dest-stack/', + 'dest-token', + $logger, + ); + + $migrate->run(); + + $records = array_filter( + $logsHandler->getRecords(), + fn(array $record) => in_array('secrets', $record['context'] ?? [], true) + ); + self::assertCount(5, $records); + + $record = array_shift($records); + self::assertSame('Migrating configurations with secrets', $record['message']); + $record = array_shift($records); + self::assertSame( + 'Configuration with ID \'101\' successfully migrated to stack \'dest-stack\'.', + $record['message'] + ); + $record = array_shift($records); + self::assertSame( + 'Configuration with ID \'102\' successfully migrated to stack \'dest-stack\'.', + $record['message'] + ); + $record = array_shift($records); + self::assertSame( + 'Used existing Snowflake workspace \'USER_01\' for configuration with ID \'103\' ' + . '(keboola.wr-db-snowflake).', + $record['message'] + ); + $record = array_shift($records); + self::assertSame( + 'Configuration with ID \'103\' successfully migrated to stack \'dest-stack\'.', + $record['message'] + ); + } + public function testShouldFailOnSnapshotError(): void { $sourceJobRunnerMock = $this->createMock(SyrupJobRunner::class); From 3e576a687ddf9f214dcdbb8067b6911b93438432 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roman=20Pi=C5=A1t=C4=9Bk?= Date: Fri, 20 Sep 2024 14:31:02 +0200 Subject: [PATCH 3/3] fix: Fetch detailed configurations in Migrate::preserveProperSnowflakeWorkspace() --- src/Migrate.php | 16 ++++++++++------ tests/phpunit/MigrateTest.php | 28 ++++++++++++++++------------ 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/src/Migrate.php b/src/Migrate.php index 81ae4a3..a417123 100644 --- a/src/Migrate.php +++ b/src/Migrate.php @@ -219,9 +219,10 @@ private function migrateSecrets(): void if (in_array($component['id'], self::SNOWFLAKE_WRITER_COMPONENT_IDS, true)) { $this->preserveProperSnowflakeWorkspace( + $component['id'], + $config['id'], $response['data']['componentId'], - $response['data']['configId'], - $config + $response['data']['configId'] ); } @@ -340,20 +341,23 @@ private function getRestoreConfigData(array $restoreCredentials): array } private function preserveProperSnowflakeWorkspace( + string $sourceComponentId, + string $sourceConfigurationId, string $destinationComponentId, - string $destinationConfigurationId, - array $sourceConfigurationData + string $destinationConfigurationId ): void { if ($this->dryRun) { return; } - - $snowflakeUser = $sourceConfigurationData['configuration']['parameters']['db']['user']; + $sourceComponentsApi = new Components($this->sourceProjectStorageClient); + $sourceConfigurationData = (array) $sourceComponentsApi + ->getConfiguration($sourceComponentId, $sourceConfigurationId); $destinationComponentsApi = new Components($this->destProjectStorageClient); $destinationConfigurationData = (array) $destinationComponentsApi ->getConfiguration($destinationComponentId, $destinationConfigurationId); + $snowflakeUser = $sourceConfigurationData['configuration']['parameters']['db']['user']; $migratedWorkspaceParameters = $this->migratedSnowflakeWorkspaces[$snowflakeUser] ?? null; if ($migratedWorkspaceParameters) { diff --git a/tests/phpunit/MigrateTest.php b/tests/phpunit/MigrateTest.php index 9d18573..4f02022 100644 --- a/tests/phpunit/MigrateTest.php +++ b/tests/phpunit/MigrateTest.php @@ -5,6 +5,7 @@ namespace Keboola\AppProjectMigrate\Tests; use Generator; +use InvalidArgumentException; use Keboola\AppProjectMigrate\Config; use Keboola\AppProjectMigrate\ConfigDefinition; use Keboola\AppProjectMigrate\JobRunner\JobRunner; @@ -460,27 +461,30 @@ public function testMigrateSnowflakeWritersWithSharedWorkspacesSuccess(): void $sourceClientMock = $this->createMock(StorageClient::class); $sourceClientMock ->method('apiGet') - ->willReturnMap([ - [ - 'dev-branches/', null, [], - [ + ->willReturnCallback(function ($url) use ($testConfigurations) { + if ($url === 'dev-branches/') { + return [ [ 'id' => '123', 'name' => 'default', 'isDefault' => true, ], - ], - ], - [ - 'components?include=', null, [], - [ + ]; + } + if ($url === 'components?include=') { + return [ [ 'id' => 'keboola.wr-db-snowflake', 'configurations' => $testConfigurations, ], - ], - ], - ]) + ]; + } + if (preg_match('~components/([^/]+)/configs/([^/]+)~', $url, $matches)) { + [, , $configId] = $matches + [null, null, null]; + return current(array_filter($testConfigurations, fn ($c) => $c['id'] === $configId)) ?: null; + } + throw new InvalidArgumentException(sprintf('Unexpected URL "%s"', $url)); + }) ; $sourceClientMock ->method('getServiceUrl')