Merge pull request #55 from keboola/ct-843-fix-numeric-value-is-empty…

…-string add NULL_IF=(\'\'))
keboola · Mar 17, 2023 · 440b9eb · 440b9eb
2 parents 47a4a48 + 173db64
commit 440b9eb
Show file tree

Hide file tree

Showing 8 changed files with 37 additions and 16 deletions.
diff --git a/packages/php-db-import-export/src/Backend/Snowflake/ToStage/FromABSCopyIntoAdapter.php b/packages/php-db-import-export/src/Backend/Snowflake/ToStage/FromABSCopyIntoAdapter.php
@@ -92,7 +92,7 @@ static function ($entry) use ($source) {
             'COPY INTO %s.%s 
 FROM %s
 CREDENTIALS=(AZURE_SAS_TOKEN=\'%s\')
-FILE_FORMAT = (TYPE=CSV %s)
+FILE_FORMAT = (TYPE=CSV %s, NULL_IF=(\'\'))
 FILES = (%s)',
             SnowflakeQuote::quoteSingleIdentifier($destination->getSchemaName()),
             SnowflakeQuote::quoteSingleIdentifier($destination->getTableName()),

diff --git a/packages/php-db-import-export/src/Backend/Snowflake/ToStage/FromGCSCopyIntoAdapter.php b/packages/php-db-import-export/src/Backend/Snowflake/ToStage/FromGCSCopyIntoAdapter.php
@@ -78,7 +78,7 @@ private function getCopyCommand(
         return sprintf(
             'COPY INTO %s.%s FROM %s
                 STORAGE_INTEGRATION = %s, 
-                FILE_FORMAT = (TYPE=CSV %s)
+                FILE_FORMAT = (TYPE=CSV %s, NULL_IF=(\'\'))
                 FILES = (%s)',
             SnowflakeQuote::quoteSingleIdentifier($destination->getSchemaName()),
             SnowflakeQuote::quoteSingleIdentifier($destination->getTableName()),

diff --git a/packages/php-db-import-export/src/Backend/Snowflake/ToStage/FromS3CopyIntoAdapter.php b/packages/php-db-import-export/src/Backend/Snowflake/ToStage/FromS3CopyIntoAdapter.php
@@ -78,7 +78,7 @@ private function getCopyCommand(
             'COPY INTO %s.%s FROM %s 
                 CREDENTIALS = (AWS_KEY_ID = %s AWS_SECRET_KEY = %s)
                 REGION = %s
-                FILE_FORMAT = (TYPE=CSV %s)
+                FILE_FORMAT = (TYPE=CSV %s, NULL_IF=(\'\'))
                 FILES = (%s)',
             SnowflakeQuote::quoteSingleIdentifier($destination->getSchemaName()),
             SnowflakeQuote::quoteSingleIdentifier($destination->getTableName()),

diff --git a/packages/php-db-import-export/tests/data/nullify-with-quotes.csv b/packages/php-db-import-export/tests/data/nullify-with-quotes.csv
@@ -0,0 +1,4 @@
+"id","name","price"
+"1","test","50"
+"2","","500"
+"3","Bageta",""
diff --git a/packages/php-db-import-export/tests/functional/Snowflake/ToStage/StageImportFileTest.php b/packages/php-db-import-export/tests/functional/Snowflake/ToStage/StageImportFileTest.php
@@ -204,7 +204,10 @@ public function importSettingProvider(): Generator
         ];
     }
 
-    public function testWithNullifyValue(): void
+    /**
+     * @dataProvider nullifyFileProvider
+     */
+    public function testWithNullifyValue(string $fileName): void
     {
         $this->initTable(self::TABLE_NULLIFY);
 
@@ -224,7 +227,7 @@ public function testWithNullifyValue(): void
         );
         $importer->importToStagingTable(
             $this->getSourceInstanceFromCsv(
-                'nullify.csv',
+                $fileName,
                 new CsvOptions(),
                 [
                     'id',
@@ -238,7 +241,7 @@ public function testWithNullifyValue(): void
             $this->getSnowflakeImportOptions()
         );
 
-        self::assertEquals([
+        self::assertSame([
             ['id' => '1', 'col1' => 'test', 'col2' => '50'],
             ['id' => '2', 'col1' => null, 'col2' => '500'],
             ['id' => '3', 'col1' => 'Bageta', 'col2' => null],
@@ -251,6 +254,17 @@ public function testWithNullifyValue(): void
         ));
     }
 
+    public function nullifyFileProvider(): Generator
+    {
+        yield 'with empty value' => [
+            'nullify.csv',
+        ];
+
+        yield 'with empty value in quotes' => [
+            'nullify-with-quotes.csv',
+        ];
+    }
+
 // testCopyIntoInvalidTypes
     public function testInvalidManifestImport(): void
     {

diff --git a/.../php-db-import-export/tests/unit/Backend/Snowflake/ToStage/FromABSCopyIntoAdapterTest.php b/.../php-db-import-export/tests/unit/Backend/Snowflake/ToStage/FromABSCopyIntoAdapterTest.php
@@ -30,16 +30,17 @@ public function testGetCopyCommands(): void
             ->willReturn('azure://xxx.blob.core.windows.net/xx/');
 
         $conn = $this->mockConnection();
+        // @codingStandardsIgnoreStart
         $conn->expects(self::once())->method('executeStatement')->with(
             <<<EOT
 COPY INTO "schema"."stagingTable" 
 FROM 'azure://xxx.blob.core.windows.net/xx/'
 CREDENTIALS=(AZURE_SAS_TOKEN='')
-FILE_FORMAT = (TYPE=CSV FIELD_DELIMITER = ',' FIELD_OPTIONALLY_ENCLOSED_BY = '\"' ESCAPE_UNENCLOSED_FIELD = NONE)
+FILE_FORMAT = (TYPE=CSV FIELD_DELIMITER = ',' FIELD_OPTIONALLY_ENCLOSED_BY = '\"' ESCAPE_UNENCLOSED_FIELD = NONE, NULL_IF=(''))
 FILES = ('xxx.csv')
 EOT
         );
-
+        // @codingStandardsIgnoreEnd
         $conn->expects(self::once())->method('fetchAllAssociative')
             // phpcs:ignore
             ->with("SELECT TABLE_TYPE,BYTES,ROW_COUNT FROM information_schema.tables WHERE TABLE_SCHEMA = 'schema' AND TABLE_NAME = 'stagingTable';")
@@ -85,7 +86,7 @@ public function testGetCopyCommandsRowSkip(): void
 COPY INTO "schema"."stagingTable" 
 FROM 'azure://xxx.blob.core.windows.net/xx/'
 CREDENTIALS=(AZURE_SAS_TOKEN='')
-FILE_FORMAT = (TYPE=CSV FIELD_DELIMITER = ',' SKIP_HEADER = 3 FIELD_OPTIONALLY_ENCLOSED_BY = '\"' ESCAPE_UNENCLOSED_FIELD = NONE)
+FILE_FORMAT = (TYPE=CSV FIELD_DELIMITER = ',' SKIP_HEADER = 3 FIELD_OPTIONALLY_ENCLOSED_BY = '\"' ESCAPE_UNENCLOSED_FIELD = NONE, NULL_IF=(''))
 FILES = ('xxx.csv')
 EOT
         );
@@ -140,13 +141,15 @@ public function testGetCopyCommandWithMoreChunksOfFiles(): void
 
         $conn = $this->mockConnection();
 
+        // @codingStandardsIgnoreStart
         $qTemplate = <<<EOT
 COPY INTO "schema"."stagingTable" 
 FROM 'azure://xxx.blob.core.windows.net/xx/'
 CREDENTIALS=(AZURE_SAS_TOKEN='')
-FILE_FORMAT = (TYPE=CSV FIELD_DELIMITER = ',' FIELD_OPTIONALLY_ENCLOSED_BY = '\"' ESCAPE_UNENCLOSED_FIELD = NONE)
+FILE_FORMAT = (TYPE=CSV FIELD_DELIMITER = ',' FIELD_OPTIONALLY_ENCLOSED_BY = '\"' ESCAPE_UNENCLOSED_FIELD = NONE, NULL_IF=(''))
 FILES = (%s)
 EOT;
+        // @codingStandardsIgnoreEnd
         $q1 = sprintf($qTemplate, implode(', ', array_slice($entriesWithoutBucket, 0, 1000)));
         $q2 = sprintf($qTemplate, implode(', ', array_slice($entriesWithoutBucket, 1000, 5)));
         $conn->expects(self::exactly(2))->method('executeStatement')->withConsecutive([$q1], [$q2]);

diff --git a/.../php-db-import-export/tests/unit/Backend/Snowflake/ToStage/FromGCSCopyIntoAdapterTest.php b/.../php-db-import-export/tests/unit/Backend/Snowflake/ToStage/FromGCSCopyIntoAdapterTest.php
@@ -36,7 +36,7 @@ public function testGetCopyCommands(): void
         SKIP_HEADER = 0,
         FIELD_OPTIONALLY_ENCLOSED_BY = '\"',
         ESCAPE_UNENCLOSED_FIELD = NONE
-        )
+        , NULL_IF=(''))
                 FILES = ('https:url')
 EOT
         );
@@ -86,7 +86,7 @@ public function testGetCopyCommandsRowSkip(): void
         SKIP_HEADER = 3,
         FIELD_OPTIONALLY_ENCLOSED_BY = '\"',
         ESCAPE_UNENCLOSED_FIELD = NONE
-        )
+        , NULL_IF=(''))
                 FILES = ('https:url')
 EOT
         );
@@ -146,7 +146,7 @@ public function testGetCopyCommandWithMoreChunksOfFiles(): void
         SKIP_HEADER = 0,
         FIELD_OPTIONALLY_ENCLOSED_BY = '\"',
         ESCAPE_UNENCLOSED_FIELD = NONE
-        )
+        , NULL_IF=(''))
                 FILES = (%s)
 EOT;
         $q1 = sprintf($qTemplate, implode(', ', array_slice($entriesWithoutBucket, 0, 1000)));

diff --git a/...s/php-db-import-export/tests/unit/Backend/Snowflake/ToStage/FromS3CopyIntoAdapterTest.php b/...s/php-db-import-export/tests/unit/Backend/Snowflake/ToStage/FromS3CopyIntoAdapterTest.php
@@ -36,7 +36,7 @@ public function testGetCopyCommands(): void
         SKIP_HEADER = 0,
         FIELD_OPTIONALLY_ENCLOSED_BY = '\"',
         ESCAPE_UNENCLOSED_FIELD = NONE
-        )
+        , NULL_IF=(''))
                 FILES = ('https:url')
 EOT
         );
@@ -86,7 +86,7 @@ public function testGetCopyCommandsRowSkip(): void
         SKIP_HEADER = 3,
         FIELD_OPTIONALLY_ENCLOSED_BY = '\"',
         ESCAPE_UNENCLOSED_FIELD = NONE
-        )
+        , NULL_IF=(''))
                 FILES = ('https:url')
 EOT
         );
@@ -146,7 +146,7 @@ public function testGetCopyCommandWithMoreChunksOfFiles(): void
         SKIP_HEADER = 0,
         FIELD_OPTIONALLY_ENCLOSED_BY = '\"',
         ESCAPE_UNENCLOSED_FIELD = NONE
-        )
+        , NULL_IF=(''))
                 FILES = (%s)
 EOT;
         $q1 = sprintf($qTemplate, implode(', ', array_slice($entriesWithoutBucket, 0, 1000)));