Merge pull request #50 from keboola/ondra-migrate-permanent-files-str…

…uncture-only
keboola · Jul 29, 2024 · 32596e7 · 32596e7
2 parents cf49c78 + 5aa205f
commit 32596e7
Show file tree

Hide file tree

Showing 4 changed files with 75 additions and 6 deletions.
diff --git a/src/Config.php b/src/Config.php
@@ -53,4 +53,14 @@ public function getDb(): array
     {
         return (array) $this->getValue(['parameters', 'db'], []);
     }
+
+    public function shouldMigratePermanentFiles(): bool
+    {
+        return $this->getValue(['parameters', 'migratePermanentFiles']);
+    }
+
+    public function shouldMigrateStructureOnly(): bool
+    {
+        return $this->getValue(['parameters', 'migrateStructureOnly']);
+    }
 }
diff --git a/src/ConfigDefinition.php b/src/ConfigDefinition.php
@@ -27,6 +27,8 @@ protected function getParametersDefinition(): ArrayNodeDefinition
                 ->end()
                 ->booleanNode('dryRun')->defaultFalse()->end()
                 ->booleanNode('directDataMigration')->defaultTrue()->end()
+                ->booleanNode('migratePermanentFiles')->defaultTrue()->end()
+                ->booleanNode('migrateStructureOnly')->defaultFalse()->end()
                 ->booleanNode('migrateSecrets')->defaultFalse()->end()
                 ->enumNode('dataMode')->values(['sapi', 'database'])->defaultValue('sapi')->end()
                 ->arrayNode('db')

diff --git a/src/Migrate.php b/src/Migrate.php
@@ -43,6 +43,10 @@ class Migrate
 
     private bool $migrateSecrets;
 
+    private bool $migratePermanentFiles;
+
+    private bool $migrateStructureOnly;
+
     public const OBSOLETE_COMPONENTS = [
         'orchestrator',
         'gooddata-writer',
@@ -73,6 +77,8 @@ public function __construct(
         $this->dryRun = $config->isDryRun();
         $this->directDataMigration = $config->directDataMigration();
         $this->migrateSecrets = $config->shouldMigrateSecrets();
+        $this->migratePermanentFiles = $config->shouldMigratePermanentFiles();
+        $this->migrateStructureOnly = $config->shouldMigrateStructureOnly();
         $this->logger = $logger;
         $this->migrateDataMode = $config->getMigrateDataMode();
         $this->db = $config->getDb();
@@ -89,7 +95,7 @@ public function run(): void
                 $this->migrateSecrets();
             }
 
-            if ($this->directDataMigration) {
+            if ($this->directDataMigration && !$this->migrateStructureOnly) {
                 $this->migrateDataOfTablesDirectly();
             }
 
@@ -133,7 +139,7 @@ private function backupSourceProject(string $backupId): void
             [
                 'parameters' => [
                     'backupId' => $backupId,
-                    'exportStructureOnly' => $this->directDataMigration,
+                    'exportStructureOnly' => $this->directDataMigration || $this->migrateStructureOnly,
                 ],
             ]
         );
@@ -291,6 +297,7 @@ private function getRestoreConfigData(array $restoreCredentials): array
                     ],
                     'useDefaultBackend' => true,
                     'restoreConfigs' => $this->migrateSecrets === false,
+                    'restorePermanentFiles' => $this->migratePermanentFiles,
                 ],
             ];
         } elseif (isset($restoreCredentials['credentials']['connectionString'])) {
@@ -302,6 +309,7 @@ private function getRestoreConfigData(array $restoreCredentials): array
                     ],
                     'useDefaultBackend' => true,
                     'restoreConfigs' => $this->migrateSecrets === false,
+                    'restorePermanentFiles' => $this->migratePermanentFiles,
                 ],
             ];
         } else {

diff --git a/tests/phpunit/MigrateTest.php b/tests/phpunit/MigrateTest.php
@@ -34,7 +34,9 @@ public function testMigrateSuccess(
         string $jobRunnerClass,
         bool $migrateDataOfTablesDirectly,
         int $expectsRunJobs,
-        bool $restoreConfigs
+        bool $restoreConfigs,
+        bool $migrateStructureOnly,
+        bool $restorePermanentFiles
     ): void {
         $sourceJobRunnerMock = $this->createMock($jobRunnerClass);
         $destJobRunnerMock = $this->createMock($jobRunnerClass);
@@ -51,7 +53,8 @@ public function testMigrateSuccess(
                 'id' => '222',
                 'status' => 'success',
             ],
-            $migrateDataOfTablesDirectly
+            $migrateDataOfTablesDirectly,
+            $migrateStructureOnly,
         );
 
         $sourceProjectUrl = 'https://connection.keboola.com';
@@ -68,6 +71,7 @@ public function testMigrateSuccess(
                             'useDefaultBackend' => true,
                             'restoreConfigs' => $restoreConfigs,
                             'dryRun' => false,
+                            'restorePermanentFiles' => $restorePermanentFiles,
                         ]
                     ),
                 ],
@@ -128,6 +132,8 @@ public function testMigrateSuccess(
                     'migrateSecrets' => false,
                     'directDataMigration' => $migrateDataOfTablesDirectly,
                     '#sourceManageToken' => 'manage-token',
+                    'migrateStructureOnly' => $migrateStructureOnly,
+                    'migratePermanentFiles' => $restorePermanentFiles,
                 ],
             ],
             new ConfigDefinition()
@@ -715,7 +721,8 @@ private function mockAddMethodGenerateAbsReadCredentials(MockObject $mockObject)
     private function mockAddMethodBackupProject(
         MockObject $mockObject,
         array $return,
-        bool $exportStructureOnly
+        bool $migrateDataOfTablesDirectly,
+        bool $exportStructureOnly = false
     ): void {
         $mockObject
             ->method('runJob')
@@ -724,7 +731,7 @@ private function mockAddMethodBackupProject(
                 [
                     'parameters' => [
                         'backupId' => '123',
-                        'exportStructureOnly' => $exportStructureOnly,
+                        'exportStructureOnly' => $exportStructureOnly || $migrateDataOfTablesDirectly,
                     ],
                 ]
             )
@@ -747,6 +754,8 @@ public function successMigrateDataProvider(): Generator
             'migrateDataOfTablesDirectly' => false,
             'expectsRunJobs' => 3,
             'restoreConfigs' => true,
+            'migrateStructureOnly' => false,
+            'restorePermanentFiles' => true,
         ];
 
         yield 'migrate-ABS-syrup' => [
@@ -760,6 +769,8 @@ public function successMigrateDataProvider(): Generator
             'migrateDataOfTablesDirectly' => false,
             'expectsRunJobs' => 3,
             'restoreConfigs' => true,
+            'migrateStructureOnly' => false,
+            'restorePermanentFiles' => true,
         ];
 
         yield 'migrate-S3-queuev2' => [
@@ -775,6 +786,8 @@ public function successMigrateDataProvider(): Generator
             'migrateDataOfTablesDirectly' => false,
             'expectsRunJobs' => 2,
             'restoreConfigs' => true,
+            'migrateStructureOnly' => false,
+            'restorePermanentFiles' => true,
         ];
 
         yield 'migrateABS-queuev2' => [
@@ -788,6 +801,8 @@ public function successMigrateDataProvider(): Generator
             'migrateDataOfTablesDirectly' => false,
             'expectsRunJobs' => 2,
             'restoreConfigs' => true,
+            'migrateStructureOnly' => false,
+            'restorePermanentFiles' => true,
         ];
 
         yield 'migrateABS-queuev2-data-directly' => [
@@ -801,6 +816,23 @@ public function successMigrateDataProvider(): Generator
             'migrateDataOfTablesDirectly' => true,
             'expectsRunJobs' => 3,
             'restoreConfigs' => true,
+            'migrateStructureOnly' => false,
+            'restorePermanentFiles' => true,
+        ];
+
+        yield 'migrateABS-queuev2-structure-only' => [
+            'expectedCredentialsData' => [
+                'abs' => [
+                    'container' => 'abcdefgh',
+                    '#connectionString' => 'https://testConnectionString',
+                ],
+            ],
+            'jobRunnerClass' => QueueV2JobRunner::class,
+            'migrateDataOfTablesDirectly' => false,
+            'expectsRunJobs' => 2,
+            'restoreConfigs' => true,
+            'migrateStructureOnly' => true,
+            'restorePermanentFiles' => true,
         ];
 
         yield 'migrate-secrets-false' => [
@@ -814,6 +846,23 @@ public function successMigrateDataProvider(): Generator
             'migrateDataOfTablesDirectly' => true,
             'expectsRunJobs' => 3,
             'restoreConfigs' => true,
+            'migrateStructureOnly' => false,
+            'restorePermanentFiles' => true,
+        ];
+
+        yield 'migrate-permanentFiles-false' => [
+            'expectedCredentialsData' => [
+                'abs' => [
+                    'container' => 'abcdefgh',
+                    '#connectionString' => 'https://testConnectionString',
+                ],
+            ],
+            'jobRunnerClass' => QueueV2JobRunner::class,
+            'migrateDataOfTablesDirectly' => true,
+            'expectsRunJobs' => 3,
+            'restoreConfigs' => true,
+            'migrateStructureOnly' => false,
+            'restorePermanentFiles' => false,
         ];
     }
 }