From fd8957fce72aa614678e203517a7f0f5cebc9431 Mon Sep 17 00:00:00 2001 From: bhattmanish98 Date: Thu, 30 Jan 2025 03:28:25 -0800 Subject: [PATCH] Reverted unrelated changes --- .../fs/azure/AzureNativeFileSystemStore.java | 6 +- .../hadoop/fs/azurebfs/AbfsConfiguration.java | 190 +--------------- .../hadoop/fs/azurebfs/AbfsStatistic.java | 2 +- .../fs/azurebfs/AzureBlobFileSystem.java | 213 +++++++----------- 4 files changed, 92 insertions(+), 319 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java index e3f995a6697ef..6e50cfc627e05 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java @@ -231,7 +231,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { /** * Configuration key to indicate the set of directories in WASB where - * we should do atomic folder rename synchronized with createNonRecursivePreCheck. + * we should do atomic folder rename synchronized with createNonRecursive. */ public static final String KEY_ATOMIC_RENAME_DIRECTORIES = "fs.azure.atomic.rename.dir"; @@ -254,7 +254,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { /** * The set of directories where we should apply atomic folder rename - * synchronized with createNonRecursivePreCheck. + * synchronized with createNonRecursive. */ private Set atomicRenameDirs; @@ -1237,7 +1237,7 @@ public boolean isBlockBlobWithCompactionKey(String key) { /** * Checks if the given key in Azure storage should have synchronized - * atomic folder rename createNonRecursivePreCheck implemented. + * atomic folder rename createNonRecursive implemented. */ @Override public boolean isAtomicRenameKey(String key) { diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index 622784fc9e135..19da9c5ade1aa 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -76,186 +76,8 @@ import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY; import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.EMPTY_STRING; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_BACKOFF_INTERVAL; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_BLOCK_LOCATION_HOST_PROPERTY_NAME; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_BLOCK_SIZE_PROPERTY_NAME; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_CONCURRENT_CONNECTION_VALUE_IN; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_CONCURRENT_CONNECTION_VALUE_OUT; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_CUSTOM_TOKEN_FETCH_RETRY_COUNT; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_ENABLE_SMALL_WRITE_OPTIMIZATION; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_FOOTER_READ_BUFFER_SIZE; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_HTTP_CONNECTION_TIMEOUT; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_HTTP_READ_TIMEOUT; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_KEY_ACCOUNT_KEYPROVIDER; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_LIST_MAX_RESULTS; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_MAX_BACKOFF_INTERVAL; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_MAX_IO_RETRIES; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_MIN_BACKOFF_INTERVAL; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_OAUTH_TOKEN_FETCH_RETRY_COUNT; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_OAUTH_TOKEN_FETCH_RETRY_DELTA_BACKOFF; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_OAUTH_TOKEN_FETCH_RETRY_MAX_BACKOFF; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_OAUTH_TOKEN_FETCH_RETRY_MIN_BACKOFF; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_READ_AHEAD_RANGE; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_READ_BUFFER_SIZE; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_READ_OPTIMIZE_FOOTER_READ; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_READ_SMALL_FILES_COMPLETELY; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_SKIP_USER_GROUP_METADATA_DURING_INITIALIZATION; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_STATIC_RETRY_FOR_CONNECTION_TIMEOUT_ENABLED; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_STATIC_RETRY_INTERVAL; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_TOLERATE_CONCURRENT_APPEND; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_WRITE_BUFFER_SIZE; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_WRITE_MAX_CONCURRENT_REQUESTS; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_WRITE_MAX_REQUESTS_TO_QUEUE; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ABFS_IO_RATE_LIMIT; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ABFS_LATENCY_TRACK; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ABFS_RENAME_RESILIENCE; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_IS_EXPECT_HEADER_ENABLED; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_IS_HNS_ENABLED; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_LEVEL_THROTTLING_ENABLED; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_OAUTH_CLIENT_ENDPOINT; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_OAUTH_CLIENT_SECRET; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_OAUTH_MSI_AUTHORITY; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_OAUTH_MSI_ENDPOINT; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_OAUTH_MSI_TENANT; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_OAUTH_REFRESH_TOKEN; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_OAUTH_REFRESH_TOKEN_ENDPOINT; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_OAUTH_TOKEN_FILE; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_OAUTH_USER_NAME; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_OAUTH_USER_PASSWORD; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_OPERATION_IDLE_TIMEOUT; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ALWAYS_READ_BUFFER_SIZE; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ALWAYS_USE_HTTPS; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ANALYSIS_PERIOD; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_APACHE_HTTP_CLIENT_IDLE_CONNECTION_TTL; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_APACHE_HTTP_CLIENT_MAX_IO_EXCEPTION_RETRIES; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_APPEND_BLOB_KEY; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ATOMIC_RENAME_KEY; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_BLOB_ATOMIC_RENAME_LEASE_REFRESH_DURATION; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_BLOB_COPY_MAX_WAIT_MILLIS; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_BLOB_COPY_PROGRESS_WAIT_MILLIS; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_BLOB_DIR_DELETE_MAX_THREAD; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_BLOB_DIR_RENAME_MAX_THREAD; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_CLIENT_CORRELATIONID; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_CLUSTER_NAME; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_CLUSTER_TYPE; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_CONSUMER_MAX_LAG; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_DISABLE_OUTPUTSTREAM_FLUSH; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ENABLE_ABFS_LIST_ITERATOR; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ENABLE_AUTOTHROTTLING; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ENABLE_CHECK_ACCESS; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ENABLE_CONDITIONAL_CREATE_OVERWRITE; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ENABLE_DELEGATION_TOKEN; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ENABLE_DFSTOBLOB_FALLBACK; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ENABLE_FLUSH; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ENABLE_MKDIR_OVERWRITE; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ENABLE_PAGINATED_DELETE; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ENABLE_READAHEAD; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ENCRYPTION_CONTEXT_PROVIDER_TYPE; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ENCRYPTION_ENCODED_CLIENT_PROVIDED_KEY; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ENCRYPTION_ENCODED_CLIENT_PROVIDED_KEY_SHA; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_FNS_ACCOUNT_SERVICE_TYPE; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_INFINITE_LEASE_KEY; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_INGRESS_SERVICE_TYPE; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_LEASE_THREADS; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_METRIC_ACCOUNT_KEY; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_METRIC_ACCOUNT_NAME; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_METRIC_ANALYSIS_TIMEOUT; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_METRIC_FORMAT; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_METRIC_IDLE_TIMEOUT; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_METRIC_URI; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_NETWORKING_LIBRARY; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_PRODUCER_QUEUE_MAX_SIZE; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_READ_AHEAD_BLOCK_SIZE; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_READ_AHEAD_QUEUE_DEPTH; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_SAS_FIXED_TOKEN; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_SAS_TOKEN_PROVIDER_TYPE; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_SAS_TOKEN_RENEW_PERIOD_FOR_STREAMS; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_SECURE_MODE; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_SSL_CHANNEL_MODE_KEY; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_TRACINGHEADER_FORMAT; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_USER_AGENT_PREFIX_KEY; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_USE_UPN; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.AZURE_BLOCK_LOCATION_HOST_DEFAULT; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_ABFS_LATENCY_TRACK; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_ACCOUNT_OPERATION_IDLE_TIMEOUT_MS; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_ALWAYS_READ_BUFFER_SIZE; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_ANALYSIS_PERIOD_MS; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_APACHE_HTTP_CLIENT_MAX_IO_EXCEPTION_RETRIES; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_AZURE_BLOB_ATOMIC_RENAME_LEASE_REFRESH_DURATION; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_AZURE_BLOB_COPY_MAX_WAIT_MILLIS; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_AZURE_BLOB_COPY_PROGRESS_WAIT_MILLIS; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_AZURE_ENABLE_SMALL_WRITE_OPTIMIZATION; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_AZURE_LIST_MAX_RESULTS; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_DELTA_BACKOFF; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_MAX_ATTEMPTS; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_MAX_BACKOFF_INTERVAL; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_MIN_BACKOFF_INTERVAL; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_AZURE_SKIP_USER_GROUP_METADATA_DURING_INITIALIZATION; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_BACKOFF_INTERVAL; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_CUSTOM_TOKEN_FETCH_RETRY_COUNT; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_DISABLE_OUTPUTSTREAM_FLUSH; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_ENABLE_ABFS_CHECKSUM_VALIDATION; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_ENABLE_ABFS_LIST_ITERATOR; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_ENABLE_ABFS_RENAME_RESILIENCE; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_ENABLE_AUTOTHROTTLING; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_ENABLE_CHECK_ACCESS; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_ENABLE_DELEGATION_TOKEN; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_ENABLE_FLUSH; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_ENABLE_HTTPS; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_ENABLE_PAGINATED_DELETE; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_ENABLE_READAHEAD; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_FOOTER_READ_BUFFER_SIZE; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_FS_AZURE_ACCOUNT_IS_EXPECT_HEADER_ENABLED; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_FS_AZURE_ACCOUNT_IS_HNS_ENABLED; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_FS_AZURE_ACCOUNT_LEVEL_THROTTLING_ENABLED; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_FS_AZURE_APPEND_BLOB_DIRECTORIES; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_FS_AZURE_ATOMIC_RENAME_DIRECTORIES; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_FS_AZURE_BLOB_DELETE_THREAD; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_FS_AZURE_BLOB_RENAME_THREAD; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_FS_AZURE_CONSUMER_MAX_LAG; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_FS_AZURE_ENABLE_CONDITIONAL_CREATE_OVERWRITE; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_FS_AZURE_ENABLE_DFSTOBLOB_FALLBACK; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_FS_AZURE_ENABLE_MKDIR_OVERWRITE; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_FS_AZURE_INFINITE_LEASE_DIRECTORIES; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_FS_AZURE_PRODUCER_QUEUE_MAX_SIZE; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_FS_AZURE_SSL_CHANNEL_MODE; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_FS_AZURE_USER_AGENT_PREFIX; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_HTTP_CLIENT_CONN_MAX_IDLE_TIME; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_HTTP_CONNECTION_TIMEOUT; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_HTTP_READ_TIMEOUT; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_LEASE_THREADS; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_MAX_BACKOFF_INTERVAL; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_MAX_RETRY_ATTEMPTS; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_METRIC_ANALYSIS_TIMEOUT_MS; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_METRIC_IDLE_TIMEOUT_MS; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_MIN_BACKOFF_INTERVAL; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_NETWORKING_LIBRARY; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_OPTIMIZE_FOOTER_READ; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_READ_AHEAD_BLOCK_SIZE; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_READ_AHEAD_QUEUE_DEPTH; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_READ_AHEAD_RANGE; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_READ_BUFFER_SIZE; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_READ_SMALL_FILES_COMPLETELY; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_READ_TOLERATE_CONCURRENT_APPEND; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_SAS_TOKEN_RENEW_PERIOD_FOR_STREAMS_IN_SECONDS; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_STATIC_RETRY_FOR_CONNECTION_TIMEOUT_ENABLED; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_STATIC_RETRY_INTERVAL; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_USE_UPN; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_VALUE_UNKNOWN; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_WRITE_BUFFER_SIZE; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.MAX_AZURE_BLOCK_SIZE; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.MAX_BUFFER_SIZE; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.MAX_CONCURRENT_READ_THREADS; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.MAX_CONCURRENT_WRITE_THREADS; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.MIN_BUFFER_SIZE; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.MIN_LEASE_THREADS; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.RATE_LIMIT_DEFAULT; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.*; +import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.*; /** * Configuration for Azure Blob FileSystem. @@ -729,10 +551,10 @@ public boolean isDfsToBlobFallbackEnabled() { public void validateConfiguredServiceType(boolean isHNSEnabled) throws InvalidConfigurationValueException { // TODO: [FnsOverBlob][HADOOP-19179] Remove this check when FNS over Blob is ready. -// if (getFsConfiguredServiceType() == AbfsServiceType.BLOB) { -// throw new InvalidConfigurationValueException(FS_DEFAULT_NAME_KEY, -// "Blob Endpoint Support not yet available"); -// } + if (getFsConfiguredServiceType() == AbfsServiceType.BLOB) { + throw new InvalidConfigurationValueException(FS_DEFAULT_NAME_KEY, + "Blob Endpoint Support not yet available"); + } if (isHNSEnabled && getConfiguredServiceTypeForFNSAccounts() == AbfsServiceType.BLOB) { throw new InvalidConfigurationValueException( FS_AZURE_FNS_ACCOUNT_SERVICE_TYPE, "Service Type Cannot be BLOB for HNS Account"); diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsStatistic.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsStatistic.java index cc80825fa0e34..3a77e82ffb4fb 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsStatistic.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsStatistic.java @@ -40,7 +40,7 @@ public enum AbfsStatistic { CALL_APPEND(CommonStatisticNames.OP_APPEND, "Calls of append()."), CALL_CREATE_NON_RECURSIVE(CommonStatisticNames.OP_CREATE_NON_RECURSIVE, - "Calls of createNonRecursivePreCheck()."), + "Calls of createNonRecursive()."), CALL_DELETE(CommonStatisticNames.OP_DELETE, "Calls of delete()."), CALL_EXIST(CommonStatisticNames.OP_EXISTS, diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java index 498067943bdf4..489bc1966e8c7 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java @@ -18,6 +18,7 @@ package org.apache.hadoop.fs.azurebfs; +import javax.annotation.Nullable; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; @@ -28,10 +29,10 @@ import java.net.URISyntaxException; import java.nio.file.AccessDeniedException; import java.time.Duration; -import java.util.Hashtable; -import java.util.List; import java.util.ArrayList; import java.util.EnumSet; +import java.util.Hashtable; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.UUID; @@ -41,27 +42,14 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; -import javax.annotation.Nullable; - -import org.apache.commons.lang3.StringUtils; -import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidConfigurationValueException; -import org.apache.hadoop.fs.azurebfs.services.AuthType; -import org.apache.hadoop.fs.impl.BackReference; -import org.apache.hadoop.security.ProviderUtils; -import org.apache.hadoop.util.Preconditions; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import org.apache.commons.lang3.ArrayUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.fs.azurebfs.commit.ResilientCommitByRename; -import org.apache.hadoop.fs.azurebfs.services.AbfsClient; -import org.apache.hadoop.fs.azurebfs.services.AbfsListStatusRemoteIterator; -import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.CommonPathCapabilities; @@ -73,27 +61,34 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.PathIOException; import org.apache.hadoop.fs.PathFilter; +import org.apache.hadoop.fs.PathIOException; +import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.XAttrSetFlag; +import org.apache.hadoop.fs.azurebfs.commit.ResilientCommitByRename; import org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants; +import org.apache.hadoop.fs.azurebfs.constants.FSOperationType; import org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations; import org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes; -import org.apache.hadoop.fs.azurebfs.constants.FSOperationType; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.FileSystemOperationUnhandledException; +import org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidConfigurationValueException; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidUriAuthorityException; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidUriException; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.SASTokenProviderException; import org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode; import org.apache.hadoop.fs.azurebfs.security.AbfsDelegationTokenManager; +import org.apache.hadoop.fs.azurebfs.services.AbfsClient; import org.apache.hadoop.fs.azurebfs.services.AbfsCounters; +import org.apache.hadoop.fs.azurebfs.services.AbfsListStatusRemoteIterator; import org.apache.hadoop.fs.azurebfs.services.AbfsLocatedFileStatus; +import org.apache.hadoop.fs.azurebfs.services.AuthType; import org.apache.hadoop.fs.azurebfs.utils.Listener; import org.apache.hadoop.fs.azurebfs.utils.TracingContext; import org.apache.hadoop.fs.azurebfs.utils.TracingHeaderFormat; import org.apache.hadoop.fs.impl.AbstractFSBuilderImpl; +import org.apache.hadoop.fs.impl.BackReference; import org.apache.hadoop.fs.impl.OpenFileParameters; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclStatus; @@ -104,14 +99,16 @@ import org.apache.hadoop.fs.store.DataBlocks; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.security.AccessControlException; -import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.ProviderUtils; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.util.RateLimiting; -import org.apache.hadoop.util.RateLimitingFactory; -import org.apache.hadoop.util.functional.RemoteIterators; +import org.apache.hadoop.security.token.Token; import org.apache.hadoop.util.DurationInfo; import org.apache.hadoop.util.LambdaUtils; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.util.Progressable; +import org.apache.hadoop.util.RateLimiting; +import org.apache.hadoop.util.RateLimitingFactory; +import org.apache.hadoop.util.functional.RemoteIterators; import static java.net.HttpURLConnection.HTTP_BAD_REQUEST; import static java.net.HttpURLConnection.HTTP_CONFLICT; @@ -119,7 +116,22 @@ import static org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_LOGGING_LEVEL; import static org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_LOGGING_LEVEL_DEFAULT; import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_STANDARD_OPTIONS; -import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.*; +import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.CALL_APPEND; +import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.CALL_CREATE; +import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.CALL_CREATE_NON_RECURSIVE; +import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.CALL_DELETE; +import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.CALL_EXIST; +import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.CALL_GET_DELEGATION_TOKEN; +import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.CALL_GET_FILE_STATUS; +import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.CALL_LIST_STATUS; +import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.CALL_MKDIRS; +import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.CALL_OPEN; +import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.CALL_RENAME; +import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.DIRECTORIES_CREATED; +import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.DIRECTORIES_DELETED; +import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.ERROR_IGNORED; +import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.FILES_CREATED; +import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.FILES_DELETED; import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.CPK_IN_NON_HNS_ACCOUNT_ERROR_MESSAGE; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.DATA_BLOCKS_BUFFER; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_IS_HNS_ENABLED; @@ -144,38 +156,24 @@ @InterfaceStability.Evolving public class AzureBlobFileSystem extends FileSystem implements IOStatisticsSource { - - public static final Logger LOG = LoggerFactory.getLogger( - AzureBlobFileSystem.class); - + public static final Logger LOG = LoggerFactory.getLogger(AzureBlobFileSystem.class); private URI uri; - private Path workingDir; - private AzureBlobFileSystemStore abfsStore; - private boolean isClosed; - private final String fileSystemId = UUID.randomUUID().toString(); private boolean delegationTokenEnabled = false; - private AbfsDelegationTokenManager delegationTokenManager; - private AbfsCounters abfsCounters; - private String clientCorrelationId; - private TracingHeaderFormat tracingHeaderFormat; - private Listener listener; /** Name of blockFactory to be used by AbfsOutputStream. */ private String blockOutputBuffer; - /** BlockFactory instance to be used. */ private DataBlocks.BlockFactory blockFactory; - /** Maximum Active blocks per OutputStream. */ private int blockOutputActiveBlocks; @@ -235,9 +233,8 @@ public void initialize(URI uri, Configuration configuration) tracingHeaderFormat = abfsConfiguration.getTracingHeaderFormat(); this.setWorkingDirectory(this.getHomeDirectory()); - TracingContext initFSTracingContext = new TracingContext( - clientCorrelationId, - fileSystemId, FSOperationType.INIT, tracingHeaderFormat, listener); + TracingContext initFSTracingContext = new TracingContext(clientCorrelationId, + fileSystemId, FSOperationType.INIT, tracingHeaderFormat, listener); /* * Validate the service type configured in the URI is valid for account type used. @@ -250,10 +247,8 @@ public void initialize(URI uri, Configuration configuration) LOG.debug("File system configured with Invalid Service Type", ex); throw ex; } catch (AzureBlobFileSystemException ex) { - LOG.debug("Failed to determine account type for service type validation", - ex); - throw new InvalidConfigurationValueException( - FS_AZURE_ACCOUNT_IS_HNS_ENABLED, ex); + LOG.debug("Failed to determine account type for service type validation", ex); + throw new InvalidConfigurationValueException(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, ex); } /* @@ -287,33 +282,27 @@ public void initialize(URI uri, Configuration configuration) try { if ((isEncryptionContextCPK(abfsConfiguration) || isGlobalKeyCPK( abfsConfiguration)) && !tryGetIsNamespaceEnabled(new TracingContext( - initFSTracingContext))) { + initFSTracingContext))) { throw new PathIOException(uri.getPath(), CPK_IN_NON_HNS_ACCOUNT_ERROR_MESSAGE); } } catch (InvalidConfigurationValueException ex) { - LOG.debug("Non-Hierarchical Namespace Accounts Cannot Have CPK Enabled", - ex); + LOG.debug("Non-Hierarchical Namespace Accounts Cannot Have CPK Enabled", ex); throw ex; } catch (AzureBlobFileSystemException ex) { - LOG.debug("Failed to determine account type for service type validation", - ex); - throw new InvalidConfigurationValueException( - FS_AZURE_ACCOUNT_IS_HNS_ENABLED, ex); + LOG.debug("Failed to determine account type for service type validation", ex); + throw new InvalidConfigurationValueException(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, ex); } // Create the file system if it does not exist. if (abfsConfiguration.getCreateRemoteFileSystemDuringInitialization()) { - TracingContext createFSTracingContext = new TracingContext( - initFSTracingContext); + TracingContext createFSTracingContext = new TracingContext(initFSTracingContext); createFSTracingContext.setOperation(CREATE_FILESYSTEM); - if (this.tryGetFileStatus(new Path(AbfsHttpConstants.ROOT_PATH), - createFSTracingContext) == null) { + if (this.tryGetFileStatus(new Path(AbfsHttpConstants.ROOT_PATH), createFSTracingContext) == null) { try { this.createFileSystem(createFSTracingContext); } catch (AzureBlobFileSystemException ex) { - checkException(null, ex, - AzureServiceErrorCode.FILE_SYSTEM_ALREADY_EXISTS); + checkException(null, ex, AzureServiceErrorCode.FILE_SYSTEM_ALREADY_EXISTS); } } } @@ -321,13 +310,11 @@ public void initialize(URI uri, Configuration configuration) LOG.trace("Initiate check for delegation token manager"); if (UserGroupInformation.isSecurityEnabled()) { - this.delegationTokenEnabled - = abfsConfiguration.isDelegationTokenManagerEnabled(); + this.delegationTokenEnabled = abfsConfiguration.isDelegationTokenManagerEnabled(); if (this.delegationTokenEnabled) { LOG.debug("Initializing DelegationTokenManager for {}", uri); - this.delegationTokenManager - = abfsConfiguration.getDelegationTokenManager(); + this.delegationTokenManager = abfsConfiguration.getDelegationTokenManager(); delegationTokenManager.bind(getUri(), configuration); LOG.debug("Created DelegationTokenManager {}", delegationTokenManager); } @@ -352,9 +339,7 @@ public String toString() { "AzureBlobFileSystem{"); sb.append("uri=").append(fullPathUri); sb.append(", user='").append(abfsStore.getUser()).append('\''); - sb.append(", primaryUserGroup='") - .append(abfsStore.getPrimaryGroup()) - .append('\''); + sb.append(", primaryUserGroup='").append(abfsStore.getPrimaryGroup()).append('\''); sb.append("[" + CAPABILITY_SAFE_READAHEAD + "]"); sb.append('}'); return sb.toString(); @@ -374,10 +359,8 @@ public void registerListener(Listener listener1) { } @Override - public FSDataInputStream open(final Path path, final int bufferSize) - throws IOException { - LOG.debug("AzureBlobFileSystem.open path: {} bufferSize: {}", path, - bufferSize); + public FSDataInputStream open(final Path path, final int bufferSize) throws IOException { + LOG.debug("AzureBlobFileSystem.open path: {} bufferSize: {}", path, bufferSize); // bufferSize is unused. return open(path, Optional.empty()); } @@ -391,8 +374,7 @@ private FSDataInputStream open(final Path path, TracingContext tracingContext = new TracingContext(clientCorrelationId, fileSystemId, FSOperationType.OPEN, tracingHeaderFormat, listener); InputStream inputStream = getAbfsStore() - .openFileForRead(qualifiedPath, parameters, statistics, - tracingContext); + .openFileForRead(qualifiedPath, parameters, statistics, tracingContext); return new FSDataInputStream(inputStream); } catch (AzureBlobFileSystemException ex) { checkException(path, ex); @@ -430,8 +412,7 @@ public FSDataOutputStream create(final Path f, final short replication, final long blockSize, final Progressable progress) throws IOException { - LOG.debug( - "AzureBlobFileSystem.create path: {} permission: {} overwrite: {} bufferSize: {}", + LOG.debug("AzureBlobFileSystem.create path: {} permission: {} overwrite: {} bufferSize: {}", f, permission, overwrite, @@ -450,10 +431,8 @@ public FSDataOutputStream create(final Path f, try { TracingContext tracingContext = new TracingContext(clientCorrelationId, - fileSystemId, FSOperationType.CREATE, overwrite, tracingHeaderFormat, - listener); - OutputStream outputStream = getAbfsStore().createFile(qualifiedPath, - statistics, overwrite, + fileSystemId, FSOperationType.CREATE, overwrite, tracingHeaderFormat, listener); + OutputStream outputStream = getAbfsStore().createFile(qualifiedPath, statistics, overwrite, permission == null ? FsPermission.getFileDefault() : permission, FsPermission.getUMask(getConf()), tracingContext); statIncrement(FILES_CREATED); @@ -526,9 +505,7 @@ public FSDataOutputStream createNonRecursive(final Path f, } @Override - public FSDataOutputStream append(final Path f, - final int bufferSize, - final Progressable progress) + public FSDataOutputStream append(final Path f, final int bufferSize, final Progressable progress) throws IOException { LOG.debug( "AzureBlobFileSystem.append path: {} bufferSize: {}", @@ -662,8 +639,7 @@ public Pair commitSingleFileByRename( final Path dest, @Nullable final String sourceEtag) throws IOException { - LOG.debug("renameFileWithEtag source: {} dest: {} etag {}", source, dest, - sourceEtag); + LOG.debug("renameFileWithEtag source: {} dest: {} etag {}", source, dest, sourceEtag); statIncrement(CALL_RENAME); trailingPeriodCheck(dest); @@ -676,8 +652,7 @@ public Pair commitSingleFileByRename( if (qualifiedSrcPath.equals(qualifiedDstPath)) { // rename to itself is forbidden - throw new PathIOException(qualifiedSrcPath.toString(), - "cannot rename object onto self"); + throw new PathIOException(qualifiedSrcPath.toString(), "cannot rename object onto self"); } // acquire one IO permit @@ -741,8 +716,7 @@ public FileStatus[] listStatus(final Path f) throws IOException { TracingContext tracingContext = new TracingContext(clientCorrelationId, fileSystemId, FSOperationType.LISTSTATUS, true, tracingHeaderFormat, listener); - FileStatus[] result = getAbfsStore().listStatus(qualifiedPath, - tracingContext); + FileStatus[] result = getAbfsStore().listStatus(qualifiedPath, tracingContext); return result; } catch (AzureBlobFileSystemException ex) { checkException(f, ex); @@ -795,8 +769,7 @@ private void trailingPeriodCheck(Path path) throws IllegalArgumentException { } @Override - public boolean mkdirs(final Path f, final FsPermission permission) - throws IOException { + public boolean mkdirs(final Path f, final FsPermission permission) throws IOException { LOG.debug( "AzureBlobFileSystem.mkdirs path: {} permissions: {}", f, permission); statIncrement(CALL_MKDIRS); @@ -893,8 +866,7 @@ public void breakLease(final Path f) throws IOException { Path qualifiedPath = makeQualified(f); - try (DurationInfo ignored = new DurationInfo(LOG, false, - "Break lease for %s", + try (DurationInfo ignored = new DurationInfo(LOG, false, "Break lease for %s", qualifiedPath)) { TracingContext tracingContext = new TracingContext(clientCorrelationId, fileSystemId, FSOperationType.BREAK_LEASE, tracingHeaderFormat, @@ -972,8 +944,7 @@ public BlockLocation[] getFileBlockLocations(FileStatus file, if (file.getLen() < start) { return new BlockLocation[0]; } - final String blobLocationHost = abfsStore.getAbfsConfiguration() - .getAzureBlockLocationHost(); + final String blobLocationHost = abfsStore.getAbfsConfiguration().getAzureBlockLocationHost(); final String[] name = {blobLocationHost}; final String[] host = {blobLocationHost}; @@ -989,8 +960,7 @@ public BlockLocation[] getFileBlockLocations(FileStatus file, for (int i = 0; i < locations.length; i++) { long currentOffset = start + (i * blockSize); long currentLength = Math.min(blockSize, start + len - currentOffset); - locations[i] = new BlockLocation(name, host, currentOffset, - currentLength); + locations[i] = new BlockLocation(name, host, currentOffset, currentLength); } return locations; @@ -1025,8 +995,7 @@ private boolean deleteRoot() throws IOException { final ExecutorService executorService = Executors.newFixedThreadPool(10); try { - final FileStatus[] ls = listStatus( - makeQualified(new Path(File.separator))); + final FileStatus[] ls = listStatus(makeQualified(new Path(File.separator))); final ArrayList deleteTasks = new ArrayList<>(); for (final FileStatus fs : ls) { final Future deleteTask = executorService.submit(new Callable() { @@ -1082,10 +1051,8 @@ public void setOwner(final Path path, final String owner, final String group) return; } - if ((owner == null || owner.isEmpty()) && (group == null - || group.isEmpty())) { - throw new IllegalArgumentException( - "A valid owner or group must be specified."); + if ((owner == null || owner.isEmpty()) && (group == null || group.isEmpty())) { + throw new IllegalArgumentException("A valid owner or group must be specified."); } Path qualifiedPath = makeQualified(path); @@ -1119,8 +1086,7 @@ public void setXAttr(final Path path, LOG.debug("AzureBlobFileSystem.setXAttr path: {}", path); if (name == null || name.isEmpty() || value == null) { - throw new IllegalArgumentException( - "A valid name and value must be specified."); + throw new IllegalArgumentException("A valid name and value must be specified."); } Path qualifiedPath = makeQualified(path); @@ -1198,8 +1164,7 @@ public void setPermission(final Path path, final FsPermission permission) throws IOException { LOG.debug("AzureBlobFileSystem.setPermission path: {}", path); TracingContext tracingContext = new TracingContext(clientCorrelationId, - fileSystemId, FSOperationType.SET_PERMISSION, true, tracingHeaderFormat, - listener); + fileSystemId, FSOperationType.SET_PERMISSION, true, tracingHeaderFormat, listener); if (!getIsNamespaceEnabled(tracingContext)) { super.setPermission(path, permission); @@ -1244,8 +1209,7 @@ public void modifyAclEntries(final Path path, final List aclSpec) } if (aclSpec == null || aclSpec.isEmpty()) { - throw new IllegalArgumentException( - "The value of the aclSpec parameter is invalid."); + throw new IllegalArgumentException("The value of the aclSpec parameter is invalid."); } Path qualifiedPath = makeQualified(path); @@ -1398,8 +1362,7 @@ public void setAcl(final Path path, final List aclSpec) public AclStatus getAclStatus(final Path path) throws IOException { LOG.debug("AzureBlobFileSystem.getAclStatus path: {}", path); TracingContext tracingContext = new TracingContext(clientCorrelationId, - fileSystemId, FSOperationType.GET_ACL_STATUS, true, tracingHeaderFormat, - listener); + fileSystemId, FSOperationType.GET_ACL_STATUS, true, tracingHeaderFormat, listener); if (!getIsNamespaceEnabled(tracingContext)) { throw new UnsupportedOperationException( @@ -1461,8 +1424,7 @@ public RemoteIterator listStatusIterator(Path path) LOG.debug("AzureBlobFileSystem.listStatusIterator path : {}", path); if (abfsStore.getAbfsConfiguration().enableAbfsListIterator()) { TracingContext tracingContext = new TracingContext(clientCorrelationId, - fileSystemId, FSOperationType.LISTSTATUS, true, tracingHeaderFormat, - listener); + fileSystemId, FSOperationType.LISTSTATUS, true, tracingHeaderFormat, listener); AbfsListStatusRemoteIterator abfsLsItr = new AbfsListStatusRemoteIterator(path, abfsStore, tracingContext); @@ -1502,8 +1464,7 @@ protected RemoteIterator listLocatedStatus( : null)); } - private FileStatus tryGetFileStatus(final Path f, - TracingContext tracingContext) { + private FileStatus tryGetFileStatus(final Path f, TracingContext tracingContext) { try { return getFileStatus(f, tracingContext); } catch (IOException ex) { @@ -1562,8 +1523,7 @@ private boolean fileSystemExists() throws IOException { return true; } - private void createFileSystem(TracingContext tracingContext) - throws IOException { + private void createFileSystem(TracingContext tracingContext) throws IOException { LOG.debug( "AzureBlobFileSystem.createFileSystem uri: {}", uri); try { @@ -1591,15 +1551,13 @@ private URI ensureAuthority(URI uri, final Configuration conf) { uri.getFragment()); } catch (URISyntaxException e) { // This should never happen. - throw new IllegalArgumentException( - new InvalidUriException(uri.toString())); + throw new IllegalArgumentException(new InvalidUriException(uri.toString())); } } } if (uri.getAuthority() == null) { - throw new IllegalArgumentException( - new InvalidUriAuthorityException(uri.toString())); + throw new IllegalArgumentException(new InvalidUriAuthorityException(uri.toString())); } return uri; @@ -1635,16 +1593,14 @@ FileSystemOperation execute( final T executionResult = callableFileOperation.call(); return new FileSystemOperation<>(executionResult, null); } catch (AbfsRestOperationException abfsRestOperationException) { - return new FileSystemOperation<>(defaultResultValue, - abfsRestOperationException); + return new FileSystemOperation<>(defaultResultValue, abfsRestOperationException); } catch (AzureBlobFileSystemException azureBlobFileSystemException) { throw new IOException(azureBlobFileSystemException); } catch (Exception exception) { if (exception instanceof ExecutionException) { exception = (Exception) getRootCause(exception); } - final FileSystemOperationUnhandledException - fileSystemOperationUnhandledException + final FileSystemOperationUnhandledException fileSystemOperationUnhandledException = new FileSystemOperationUnhandledException(exception); throw new IOException(fileSystemOperationUnhandledException); } @@ -1739,11 +1695,9 @@ private Throwable getRootCause(Throwable throwable) { * @throws IOException thrown when getting the current user. */ @Override - public synchronized Token getDelegationToken(final String renewer) - throws IOException { + public synchronized Token getDelegationToken(final String renewer) throws IOException { statIncrement(CALL_GET_DELEGATION_TOKEN); - return this.delegationTokenEnabled - ? this.delegationTokenManager.getDelegationToken(renewer) + return this.delegationTokenEnabled ? this.delegationTokenManager.getDelegationToken(renewer) : super.getDelegationToken(renewer); } @@ -1773,13 +1727,10 @@ void setListenerOperation(FSOperationType operation) { @VisibleForTesting static class FileSystemOperation { - private final T result; - private final AbfsRestOperationException exception; - FileSystemOperation(final T result, - final AbfsRestOperationException exception) { + FileSystemOperation(final T result, final AbfsRestOperationException exception) { this.result = result; this.exception = exception; }