HADOOP-19443: [ABFS][FnsOverBlob] Updating hadoop-azure Test Scripts …

…and Adding Tests For Metadata APIs (apache#7344)

Contributed by Anuj Modi
Reviewed by Anmol Asrani, Sneha Vijayarajan, Manish Bhatt

hadoop-tools/hadoop-azure/dev-support/testrun-scripts/runtests.sh

@@ -32,36 +32,87 @@ processCount=8
 
 ## SECTION: TEST COMBINATION METHODS
 
-runHNSOAuthTest()
+runHNSOAuthDFSTest()
 {
   accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.hnsTestAccountName"]/value' -n $azureTestXmlPath)
   PROPERTIES=("fs.azure.account.auth.type")
   VALUES=("OAuth")
-  triggerRun "HNS-OAuth" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
+  triggerRun "HNS-OAuth-DFS" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
 }
 
-runHNSSharedKeyTest()
+runHNSSharedKeyDFSTest()
 {
   accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.hnsTestAccountName"]/value' -n $azureTestXmlPath)
   PROPERTIES=("fs.azure.account.auth.type")
   VALUES=("SharedKey")
-  triggerRun "HNS-SharedKey" "$accountName"  "$runTest" $processCount "$cleanUpTestContainers"
+  triggerRun "HNS-SharedKey-DFS" "$accountName"  "$runTest" $processCount "$cleanUpTestContainers"
 }
 
-runNonHNSSharedKeyTest()
+runNonHNSSharedKeyDFSTest()
 {
   accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.nonHnsTestAccountName"]/value' -n $azureTestXmlPath)
   PROPERTIES=("fs.azure.account.auth.type")
   VALUES=("SharedKey")
-  triggerRun "NonHNS-SharedKey" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
+  triggerRun "NonHNS-SharedKey-DFS" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
 }
 
-runAppendBlobHNSOAuthTest()
+runAppendBlobHNSOAuthDFSTest()
 {
   accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.hnsTestAccountName"]/value' -n $azureTestXmlPath)
   PROPERTIES=("fs.azure.account.auth.type" "fs.azure.test.appendblob.enabled")
   VALUES=("OAuth" "true")
-  triggerRun "AppendBlob-HNS-OAuth" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
+  triggerRun "AppendBlob-HNS-OAuth-DFS" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
+}
+
+runNonHNSSharedKeyBlobTest()
+{
+  accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.nonHnsTestAccountName"]/value' -n $azureTestXmlPath)
+  fnsBlobConfigFileCheck "$accountName"
+  PROPERTIES=("fs.azure.account.auth.type")
+  VALUES=("SharedKey")
+  triggerRun "NonHNS-SharedKey-Blob" "${accountName}_blob" "$runTest" $processCount "$cleanUpTestContainers"
+}
+
+runNonHNSOAuthDFSTest()
+{
+  accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.nonHnsTestAccountName"]/value' -n $azureTestXmlPath)
+  PROPERTIES=("fs.azure.account.auth.type")
+  VALUES=("OAuth")
+  triggerRun "NonHNS-OAuth-DFS" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
+}
+
+runNonHNSOAuthBlobTest()
+{
+  accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.nonHnsTestAccountName"]/value' -n $azureTestXmlPath)
+  fnsBlobConfigFileCheck "$accountName"
+  PROPERTIES=("fs.azure.account.auth.type")
+  VALUES=("OAuth")
+  triggerRun "NonHNS-OAuth-Blob" "${accountName}_blob" "$runTest" $processCount "$cleanUpTestContainers"
+}
+
+runAppendBlobNonHNSOAuthBlobTest()
+{
+  accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.nonHnsTestAccountName"]/value' -n $azureTestXmlPath)
+  fnsBlobConfigFileCheck "$accountName"
+  PROPERTIES=("fs.azure.account.auth.type" "fs.azure.test.appendblob.enabled")
+  VALUES=("OAuth" "true")
+  triggerRun "AppendBlob-NonHNS-OAuth-Blob" "${accountName}_blob" "$runTest" $processCount "$cleanUpTestContainers"
+}
+
+runHNSOAuthDFSIngressBlobTest()
+{
+  accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.hnsTestAccountName"]/value' -n $azureTestXmlPath)
+  PROPERTIES=("fs.azure.account.auth.type" "fs.azure.ingress.service.type")
+  VALUES=("OAuth" "blob")
+  triggerRun "HNS-Oauth-DFS-IngressBlob" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
+}
+
+runNonHNSOAuthDFSIngressBlobTest()
+{
+  accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.nonHnsTestAccountName"]/value' -n $azureTestXmlPath)
+  PROPERTIES=("fs.azure.account.auth.type" "fs.azure.ingress.service.type")
+  VALUES=("OAuth" "blob")
+  triggerRun "NonHNS-OAuth-DFS-IngressBlob" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
 }
 
 runTest=false
@@ -130,23 +181,47 @@ done
 
 echo ' '
 echo 'Set the active test combination to run the action:'
-select combo in HNS-OAuth HNS-SharedKey nonHNS-SharedKey AppendBlob-HNS-OAuth AllCombinationsTestRun Quit
+select combo in HNS-OAuth-DFS HNS-SharedKey-DFS NonHNS-SharedKey-DFS AppendBlob-HNS-OAuth-DFS NonHNS-SharedKey-Blob NonHNS-OAuth-DFS NonHNS-OAuth-Blob AppendBlob-NonHNS-OAuth-Blob HNS-Oauth-DFS-IngressBlob NonHNS-Oauth-DFS-IngressBlob AllCombinationsTestRun Quit
 do
    case $combo in
-      HNS-OAuth)
-         runHNSOAuthTest
+      HNS-OAuth-DFS)
+         runHNSOAuthDFSTest
+         break
+         ;;
+      HNS-SharedKey-DFS)
+         runHNSSharedKeyDFSTest
+         break
+         ;;
+      NonHNS-SharedKey-DFS)
+         runNonHNSSharedKeyDFSTest
+         break
+         ;;
+       AppendBlob-HNS-OAuth-DFS)
+         runAppendBlobHNSOAuthDFSTest
+         break
+         ;;
+       NonHNS-SharedKey-Blob)
+         runNonHNSSharedKeyBlobTest
+         break
+         ;;
+        NonHNS-OAuth-DFS)
+         runNonHNSOAuthDFSTest
+         break
+         ;;
+        NonHNS-OAuth-Blob)
+         runNonHNSOAuthBlobTest
          break
          ;;
-      HNS-SharedKey)
-         runHNSSharedKeyTest
+        AppendBlob-NonHNS-OAuth-Blob)
+         runAppendBlobNonHNSOAuthBlobTest
          break
          ;;
-      nonHNS-SharedKey)
-         runNonHNSSharedKeyTest
+        HNS-Oauth-DFS-IngressBlob)
+         runHNSOAuthDFSIngressBlobTest
          break
          ;;
-       AppendBlob-HNS-OAuth)
-         runAppendBlobHNSOAuthTest
+        NonHNS-Oauth-DFS-IngressBlob)
+         runNonHNSOAuthDFSIngressBlobTest
          break
          ;;
       AllCombinationsTestRun)
@@ -155,10 +230,16 @@ do
           logOutput "ERROR: Invalid selection for SET_ACTIVE_TEST_CONFIG. This is applicable only for RUN_TEST."
           break
         fi
-         runHNSOAuthTest
-         runHNSSharedKeyTest
-         runNonHNSSharedKeyTest
-         runAppendBlobHNSOAuthTest ## Keep this as the last run scenario always
+        runHNSOAuthDFSTest
+        runHNSSharedKeyDFSTest
+        runNonHNSSharedKeyDFSTest
+        runAppendBlobHNSOAuthDFSTest
+        runNonHNSSharedKeyBlobTest
+        runNonHNSOAuthDFSTest
+        runNonHNSOAuthBlobTest
+        runAppendBlobNonHNSOAuthBlobTest
+        runHNSOAuthDFSIngressBlobTest
+        runNonHNSOAuthDFSIngressBlobTest
          break
          ;;
       Quit)

hadoop-tools/hadoop-azure/dev-support/testrun-scripts/testsupport.sh

@@ -45,6 +45,25 @@ ENDTIME=$(date +%s)
 outputFormatOn="\033[0;95m"
 outputFormatOff="\033[0m"
 
+# Function to check if the blob config file exists and create one if it doesn't.
+fnsBlobConfigFileCheck() {
+  baseFileName=$1
+  targetWord=".dfs."
+  replacementWord=".blob."
+  accountSettingsDir="src/test/resources/accountSettings/"
+  accountConfigFileSuffix="_settings.xml"
+  sourceFilePath="${accountSettingsDir}${baseFileName}${accountConfigFileSuffix}"
+  targetFilePath="${accountSettingsDir}${baseFileName}_blob${accountConfigFileSuffix}"
+
+  if [ ! -f "$targetFilePath" ]; then
+    cp "$sourceFilePath" "$targetFilePath"
+    sed -i "s/$targetWord/$replacementWord/g" "$targetFilePath"
+    echo "File created and word replaced."
+  else
+    echo "File already exists."
+  fi
+}
+
 triggerRun()
 {
   echo ' '

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java

@@ -550,11 +550,6 @@ public boolean isDfsToBlobFallbackEnabled() {
    */
   public void validateConfiguredServiceType(boolean isHNSEnabled)
       throws InvalidConfigurationValueException {
-    // TODO: [FnsOverBlob][HADOOP-19179] Remove this check when FNS over Blob is ready.
-    if (getFsConfiguredServiceType() == AbfsServiceType.BLOB) {
-      throw new InvalidConfigurationValueException(FS_DEFAULT_NAME_KEY,
-          "Blob Endpoint Support not yet available");
-    }
     if (isHNSEnabled && getConfiguredServiceTypeForFNSAccounts() == AbfsServiceType.BLOB) {
       throw new InvalidConfigurationValueException(
           FS_AZURE_FNS_ACCOUNT_SERVICE_TYPE, "Service Type Cannot be BLOB for HNS Account");

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java

@@ -25,6 +25,7 @@
 import java.util.UUID;
 import java.util.concurrent.Callable;
 
+import org.assertj.core.api.Assertions;
 import org.junit.After;
 import org.junit.Assume;
 import org.junit.Before;
@@ -64,6 +65,7 @@
 import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.FORWARD_SLASH;
 import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.*;
 import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.ABFS_BLOB_DOMAIN_NAME;
+import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.ABFS_DFS_DOMAIN_NAME;
 import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.HTTPS_SCHEME;
 import static org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode.FILE_SYSTEM_NOT_FOUND;
 import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.*;
@@ -632,8 +634,70 @@ private String getAzcopyAbsolutePath(Path path) throws IOException {
         + accountName + FORWARD_SLASH + fileSystemName + pathFromContainerRoot;
   }
 
+  /**
+   * Utility method to assume that the test is running against a Blob service.
+   * Otherwise, the test will be skipped.
+   */
   protected void assumeBlobServiceType() {
     Assume.assumeTrue("Blob service type is required for this test",
         getAbfsServiceType() == AbfsServiceType.BLOB);
   }
+
+  /**
+   * Utility method to assume that the test is running against a DFS service.
+   * Otherwise, the test will be skipped.
+   */
+  protected void assumeDfsServiceType() {
+    Assume.assumeTrue("DFS service type is required for this test",
+        getAbfsServiceType() == AbfsServiceType.DFS);
+  }
+
+  /**
+   * Utility method to assume that the test is running against a HNS Enabled account.
+   * Otherwise, the test will be skipped.
+   * @throws IOException if an error occurs while checking the account type.
+   */
+  protected void assumeHnsEnabled() throws IOException {
+    assumeHnsEnabled("HNS-Enabled account must be used for this test");
+  }
+
+  /**
+   * Utility method to assume that the test is running against a HNS Enabled account.
+   * @param errorMessage error message to be displayed if the test is skipped.
+   * @throws IOException if an error occurs while checking the account type.
+   */
+  protected void assumeHnsEnabled(String errorMessage) throws IOException {
+    Assume.assumeTrue(errorMessage, getIsNamespaceEnabled(getFileSystem()));
+  }
+
+  /**
+   * Utility method to assume that the test is running against a HNS Disabled account.
+   * Otherwise, the test will be skipped.
+   * @throws IOException if an error occurs while checking the account type.
+   */
+  protected void assumeHnsDisabled() throws IOException {
+    assumeHnsDisabled("HNS-Enabled account must not be used for this test");
+  }
+
+  /**
+   * Utility method to assume that the test is running against a HNS Disabled account.
+   * @param message error message to be displayed if the test is skipped.
+   * @throws IOException if an error occurs while checking the account type.
+   */
+  protected void assumeHnsDisabled(String message) throws IOException {
+    Assume.assumeFalse(message, getIsNamespaceEnabled(getFileSystem()));
+  }
+
+  /**
+   * Assert that the path contains the expected DNS suffix.
+   * If service type is blob, then path should have blob domain name.
+   * @param path to be asserted.
+   */
+  protected void assertPathDns(Path path) {
+    String expectedDns = getAbfsServiceType() == AbfsServiceType.BLOB
+        ? ABFS_BLOB_DOMAIN_NAME : ABFS_DFS_DOMAIN_NAME;
+    Assertions.assertThat(path.toString())
+        .describedAs("Path does not contain expected DNS")
+        .contains(expectedDns);
+  }
 }

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsListStatusRemoteIterator.java

@@ -225,6 +225,7 @@ public void testNonExistingPath() throws Exception {
 
   private void verifyIteratorResultContent(FileStatus fileStatus,
       List<String> fileNames) {
+    assertPathDns(fileStatus.getPath());
     String pathStr = fileStatus.getPath().toString();
     Assert.assertTrue(
         String.format("Could not remove path %s from filenames %s", pathStr,

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAppend.java

@@ -341,9 +341,7 @@ public void testCreateAppendBlobOverBlobEndpointAppendOverDfs()
   @Test
   public void testCreateAppendBlobOverDfsEndpointAppendOverBlob()
       throws IOException, NoSuchFieldException, IllegalAccessException {
-    Assume.assumeTrue(
-        "FNS does not support append blob creation for DFS endpoint",
-        getIsNamespaceEnabled(getFileSystem()));
+    assumeHnsEnabled("FNS does not support append blob creation for DFS endpoint");
     final AzureBlobFileSystem fs = Mockito.spy(getFileSystem());
     AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
     Mockito.doReturn(true).when(store).isAppendBlobKey(anyString());
@@ -777,7 +775,7 @@ public void testIntermittentAppendFailureToBeReported() throws Exception {
     Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled());
     try (AzureBlobFileSystem fs = Mockito.spy(
         (AzureBlobFileSystem) FileSystem.newInstance(getRawConfiguration()))) {
-      Assume.assumeTrue(!getIsNamespaceEnabled(fs));
+      assumeHnsDisabled();
       AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
       assumeBlobServiceType();
 
@@ -982,8 +980,7 @@ public void testFlushSuccessWithConnectionResetOnResponseValidMd5() throws Excep
     // Create a spy of AzureBlobFileSystem
     try (AzureBlobFileSystem fs = Mockito.spy(
         (AzureBlobFileSystem) FileSystem.newInstance(getRawConfiguration()))) {
-      Assume.assumeTrue(!getIsNamespaceEnabled(fs));
-
+      assumeHnsDisabled();
       // Create a spy of AzureBlobFileSystemStore
       AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
       assumeBlobServiceType();
@@ -1080,7 +1077,7 @@ public void testFlushSuccessWithConnectionResetOnResponseInvalidMd5() throws Exc
     // Create a spy of AzureBlobFileSystem
     try (AzureBlobFileSystem fs = Mockito.spy(
         (AzureBlobFileSystem) FileSystem.newInstance(getRawConfiguration()))) {
-      Assume.assumeTrue(!getIsNamespaceEnabled(fs));
+      assumeHnsDisabled();
 
       // Create a spy of AzureBlobFileSystemStore
       AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());