-
Notifications
You must be signed in to change notification settings - Fork 8.9k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
HADOOP-19348. Add initial support for Analytics Accelerator Library f…
…or Amazon S3 (#7192)
- Loading branch information
1 parent
053afb7
commit e18d0a4
Showing
33 changed files
with
620 additions
and
32 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
183 changes: 183 additions & 0 deletions
183
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ASeekableStream.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,183 @@ | ||
/* | ||
* Licensed to the Apache Software Foundation (ASF) under one | ||
* or more contributor license agreements. See the NOTICE file | ||
* distributed with this work for additional information | ||
* regarding copyright ownership. The ASF licenses this file | ||
* to you under the Apache License, Version 2.0 (the | ||
* "License"); you may not use this file except in compliance | ||
* with the License. You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, | ||
* software distributed under the License is distributed on an | ||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
* KIND, either express or implied. See the License for the | ||
* specific language governing permissions and limitations | ||
* under the License. | ||
*/ | ||
|
||
package org.apache.hadoop.fs.s3a; | ||
|
||
import java.io.EOFException; | ||
import java.io.IOException; | ||
|
||
import org.apache.hadoop.fs.FSExceptionMessages; | ||
import org.apache.hadoop.fs.StreamCapabilities; | ||
import org.slf4j.Logger; | ||
import org.slf4j.LoggerFactory; | ||
|
||
import org.apache.hadoop.fs.FSInputStream; | ||
|
||
import software.amazon.s3.analyticsaccelerator.S3SeekableInputStream; | ||
import software.amazon.s3.analyticsaccelerator.S3SeekableInputStreamFactory; | ||
import software.amazon.s3.analyticsaccelerator.util.S3URI; | ||
|
||
public class S3ASeekableStream extends FSInputStream implements StreamCapabilities { | ||
|
||
private S3SeekableInputStream inputStream; | ||
private long lastReadCurrentPos = 0; | ||
private final String key; | ||
private volatile boolean closed; | ||
|
||
public static final Logger LOG = LoggerFactory.getLogger(S3ASeekableStream.class); | ||
|
||
public S3ASeekableStream(String bucket, String key, | ||
S3SeekableInputStreamFactory s3SeekableInputStreamFactory) { | ||
this.inputStream = s3SeekableInputStreamFactory.createStream(S3URI.of(bucket, key)); | ||
this.key = key; | ||
} | ||
|
||
/** | ||
* Indicates whether the given {@code capability} is supported by this stream. | ||
* | ||
* @param capability the capability to check. | ||
* @return true if the given {@code capability} is supported by this stream, false otherwise. | ||
*/ | ||
@Override | ||
public boolean hasCapability(String capability) { | ||
return false; | ||
} | ||
|
||
@Override | ||
public int read() throws IOException { | ||
throwIfClosed(); | ||
int bytesRead; | ||
try { | ||
bytesRead = inputStream.read(); | ||
} catch (IOException ioe) { | ||
onReadFailure(ioe); | ||
throw ioe; | ||
} | ||
return bytesRead; | ||
} | ||
|
||
@Override | ||
public void seek(long pos) throws IOException { | ||
throwIfClosed(); | ||
if (pos < 0) { | ||
throw new EOFException(FSExceptionMessages.NEGATIVE_SEEK | ||
+ " " + pos); | ||
} | ||
inputStream.seek(pos); | ||
} | ||
|
||
|
||
@Override | ||
public synchronized long getPos() { | ||
if (!closed) { | ||
lastReadCurrentPos = inputStream.getPos(); | ||
} | ||
return lastReadCurrentPos; | ||
} | ||
|
||
|
||
/** | ||
* Reads the last n bytes from the stream into a byte buffer. Blocks until end of stream is | ||
* reached. Leaves the position of the stream unaltered. | ||
* | ||
* @param buf buffer to read data into | ||
* @param off start position in buffer at which data is written | ||
* @param len the number of bytes to read; the n-th byte should be the last byte of the stream. | ||
* @return the total number of bytes read into the buffer | ||
* @throws IOException if an I/O error occurs | ||
*/ | ||
public int readTail(byte[] buf, int off, int len) throws IOException { | ||
throwIfClosed(); | ||
int bytesRead; | ||
try { | ||
bytesRead = inputStream.readTail(buf, off, len); | ||
} catch (IOException ioe) { | ||
onReadFailure(ioe); | ||
throw ioe; | ||
} | ||
return bytesRead; | ||
} | ||
|
||
@Override | ||
public int read(byte[] buf, int off, int len) throws IOException { | ||
throwIfClosed(); | ||
int bytesRead; | ||
try { | ||
bytesRead = inputStream.read(buf, off, len); | ||
} catch (IOException ioe) { | ||
onReadFailure(ioe); | ||
throw ioe; | ||
} | ||
return bytesRead; | ||
} | ||
|
||
|
||
@Override | ||
public boolean seekToNewSource(long l) throws IOException { | ||
return false; | ||
} | ||
|
||
@Override | ||
public int available() throws IOException { | ||
throwIfClosed(); | ||
return super.available(); | ||
} | ||
|
||
@Override | ||
public synchronized void close() throws IOException { | ||
if(!closed) { | ||
closed = true; | ||
try { | ||
inputStream.close(); | ||
inputStream = null; | ||
super.close(); | ||
} catch (IOException ioe) { | ||
LOG.debug("Failure closing stream {}: ", key); | ||
throw ioe; | ||
} | ||
} | ||
} | ||
|
||
/** | ||
* Close the stream on read failure. | ||
* No attempt to recover from failure | ||
* | ||
* @param ioe exception caught. | ||
*/ | ||
@Retries.OnceTranslated | ||
private void onReadFailure(IOException ioe) throws IOException { | ||
if (LOG.isDebugEnabled()) { | ||
LOG.debug("Got exception while trying to read from stream {}, " + | ||
"not trying to recover:", | ||
key, ioe); | ||
} else { | ||
LOG.info("Got exception while trying to read from stream {}, " + | ||
"not trying to recover:", | ||
key, ioe); | ||
} | ||
this.close(); | ||
} | ||
|
||
|
||
protected void throwIfClosed() throws IOException { | ||
if (closed) { | ||
throw new IOException(key + ": " + FSExceptionMessages.STREAM_IS_CLOSED); | ||
} | ||
} | ||
} |
Oops, something went wrong.