Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

HADOOP-19448: [ABFS][FNSOverBlob] Optimizing the current create and mkdir flow #7353

Open
wants to merge 17 commits into
base: trunk
Choose a base branch
from
Open
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,6 @@
import org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations;
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException;
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException;
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.ConcurrentWriteOperationDetectedException;
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.FileSystemOperationUnhandledException;
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidAbfsRestOperationException;
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidFileSystemPropertyException;
Expand Down Expand Up @@ -762,53 +761,8 @@ private AbfsRestOperation conditionalCreateOverwriteFile(final String relativePa
final TracingContext tracingContext) throws IOException {
AbfsRestOperation op;
AbfsClient createClient = getClientHandler().getIngressClient();
try {
// Trigger a create with overwrite=false first so that eTag fetch can be
// avoided for cases when no pre-existing file is present (major portion
// of create file traffic falls into the case of no pre-existing file).
op = createClient.createPath(relativePath, true, false, permissions,
isAppendBlob, null, contextEncryptionAdapter, tracingContext);

} catch (AbfsRestOperationException e) {
if (e.getStatusCode() == HttpURLConnection.HTTP_CONFLICT) {
// File pre-exists, fetch eTag
try {
op = getClient().getPathStatus(relativePath, false, tracingContext, null);
} catch (AbfsRestOperationException ex) {
if (ex.getStatusCode() == HttpURLConnection.HTTP_NOT_FOUND) {
// Is a parallel access case, as file which was found to be
// present went missing by this request.
throw new ConcurrentWriteOperationDetectedException(
"Parallel access to the create path detected. Failing request "
+ "to honor single writer semantics");
} else {
throw ex;
}
}

String eTag = extractEtagHeader(op.getResult());

try {
// overwrite only if eTag matches with the file properties fetched befpre
op = createClient.createPath(relativePath, true, true, permissions,
isAppendBlob, eTag, contextEncryptionAdapter, tracingContext);
} catch (AbfsRestOperationException ex) {
if (ex.getStatusCode() == HttpURLConnection.HTTP_PRECON_FAILED) {
// Is a parallel access case, as file with eTag was just queried
// and precondition failure can happen only when another file with
// different etag got created.
throw new ConcurrentWriteOperationDetectedException(
"Parallel access to the create path detected. Failing request "
+ "to honor single writer semantics");
} else {
throw ex;
}
}
} else {
throw e;
}
}

op = createClient.conditionalCreateOverwriteFile(relativePath, statistics,
permissions, isAppendBlob, contextEncryptionAdapter, tracingContext);
return op;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,29 @@

/**
* Thrown when a concurrent write operation is detected.
* This exception is used to indicate that parallel access to the create path
* has been detected, which violates the single writer semantics.
*/
@org.apache.hadoop.classification.InterfaceAudience.Public
@org.apache.hadoop.classification.InterfaceStability.Evolving
public class ConcurrentWriteOperationDetectedException
extends AzureBlobFileSystemException {

private static final String ERROR_MESSAGE = "Parallel access to the create path detected. Failing request "
+ "to honor single writer semantics";

/**
* Constructs a new ConcurrentWriteOperationDetectedException with a default error message.
*/
public ConcurrentWriteOperationDetectedException() {
anmolanmol1234 marked this conversation as resolved.
Show resolved Hide resolved
super(ERROR_MESSAGE);
}

/**
* Constructs a new ConcurrentWriteOperationDetectedException with the specified error message.
*
* @param message the detail message.
*/
public ConcurrentWriteOperationDetectedException(String message) {
super(message);
}
Expand Down
Loading