Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

HADOOP-19391. NNBench optimize: time consumption compute and usage information. #7302

Open
wants to merge 1 commit into
base: trunk
Choose a base branch
from
Open

HADOOP-19391. NNBench optimize: time consumption compute and usage information. #7302

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 25 additions & 24 deletions ...lient/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/hdfs/NNBench.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.SequenceFileInputFormat;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.apache.hadoop.util.concurrent.HadoopExecutors;
Expand Down Expand Up @@ -97,7 +98,7 @@ public class NNBench extends Configured implements Tool {
private long numberOfMaps = 1l; // default is 1
private long numberOfReduces = 1l; // default is 1
private long startTime =
System.currentTimeMillis() + (120 * 1000); // default is 'now' + 2min
Time.monotonicNow() + (120 * 1000); // default is 'now' + 2min
private long blockSize = 1l; // default is 1
private int bytesToWrite = 0; // default is 0
private long bytesPerChecksum = 1l; // default is 1
Expand Down Expand Up @@ -225,11 +226,11 @@ private static void displayUsage() {
"default is launch time + 2 mins. This is not mandatory>\n" +
"\t-blockSize <Block size in bytes. default is 1. " +
"This is not mandatory>\n" +
"\t-bytesToWrite <Bytes to write. default is 0. " +
"\t-bytesToWrite <Bytes to write per file. default is 0. " +
"This is not mandatory>\n" +
"\t-bytesPerChecksum <Bytes per checksum for the files. default is 1. " +
"This is not mandatory>\n" +
"\t-numberOfFiles <number of files to create. default is 1. " +
"\t-numberOfFiles <number of files to create per map. default is 1. " +
"This is not mandatory>\n" +
"\t-replicationFactorPerFile <Replication factor for the files." +
" default is 1. This is not mandatory>\n" +
Expand Down Expand Up @@ -463,9 +464,9 @@ private int analyzeResults() throws IOException {
" Maps to run: " + numberOfMaps,
" Reduces to run: " + numberOfReduces,
" Block Size (bytes): " + blockSize,
" Bytes to write: " + bytesToWrite,
" Bytes to write per file: " + bytesToWrite,
" Bytes per checksum: " + bytesPerChecksum,
" Number of files: " + numberOfFiles,
" Number of files per map: " + numberOfFiles,
" Replication factor: " + replicationFactorPerFile,
" Successful file operations: " + successfulFileOps,
"",
Expand Down Expand Up @@ -709,7 +710,7 @@ public void close() throws IOException {
*/
private boolean barrier() {
long startTime = getConf().getLong("test.nnbench.starttime", 0l);
long currentTime = System.currentTimeMillis();
long currentTime = Time.monotonicNow();
long sleepTime = startTime - currentTime;
boolean retVal = true;

Expand Down Expand Up @@ -759,23 +760,23 @@ public void map(Text key,
if (barrier()) {
String fileName = "file_" + value;
if (op.equals(OP_CREATE_WRITE)) {
startTimeTPmS = System.currentTimeMillis();
startTimeTPmS = Time.monotonicNow();
doCreateWriteOp(fileName, reporter);
} else if (op.equals(OP_OPEN_READ)) {
startTimeTPmS = System.currentTimeMillis();
startTimeTPmS = Time.monotonicNow();
doOpenReadOp(fileName, reporter);
} else if (op.equals(OP_RENAME)) {
startTimeTPmS = System.currentTimeMillis();
startTimeTPmS = Time.monotonicNow();
doRenameOp(fileName, reporter);
} else if (op.equals(OP_DELETE)) {
startTimeTPmS = System.currentTimeMillis();
startTimeTPmS = Time.monotonicNow();
doDeleteOp(fileName, reporter);
} else {
throw new IllegalArgumentException(
"unsupported operation [" + op + "]");
}

endTimeTPms = System.currentTimeMillis();
endTimeTPms = Time.monotonicNow();
totalTimeTPmS = endTimeTPms - startTimeTPmS;
} else {
output.collect(new Text("l:latemaps"), new Text("1"));
Expand Down Expand Up @@ -817,7 +818,7 @@ private void doCreateWriteOp(String name,
while (! successfulOp && numOfExceptions < MAX_OPERATION_EXCEPTIONS) {
try {
// Set up timer for measuring AL (transaction #1)
startTimeAL = System.currentTimeMillis();
startTimeAL = Time.monotonicNow();
// Create the file
// Use a buffer size of 512
out = filesystem.create(filePath,
Expand All @@ -826,14 +827,14 @@ private void doCreateWriteOp(String name,
replFactor,
blkSize);
out.write(buffer);
totalTimeAL1 += (System.currentTimeMillis() - startTimeAL);
totalTimeAL1 += (Time.monotonicNow() - startTimeAL);

// Close the file / file output stream
// Set up timers for measuring AL (transaction #2)
startTimeAL = System.currentTimeMillis();
startTimeAL = Time.monotonicNow();
out.close();

totalTimeAL2 += (System.currentTimeMillis() - startTimeAL);
totalTimeAL2 += (Time.monotonicNow() - startTimeAL);
successfulOp = true;
successfulFileOps ++;

Expand Down Expand Up @@ -866,16 +867,16 @@ private void doOpenReadOp(String name,
while (! successfulOp && numOfExceptions < MAX_OPERATION_EXCEPTIONS) {
try {
// Set up timer for measuring AL
startTimeAL = System.currentTimeMillis();
startTimeAL = Time.monotonicNow();
input = filesystem.open(filePath);
totalTimeAL1 += (System.currentTimeMillis() - startTimeAL);
totalTimeAL1 += (Time.monotonicNow() - startTimeAL);

// If the file needs to be read (specified at command line)
if (readFile) {
startTimeAL = System.currentTimeMillis();
startTimeAL = Time.monotonicNow();
input.readFully(buffer);

totalTimeAL2 += (System.currentTimeMillis() - startTimeAL);
totalTimeAL2 += (Time.monotonicNow() - startTimeAL);
}
input.close();
successfulOp = true;
Expand Down Expand Up @@ -909,12 +910,12 @@ private void doRenameOp(String name,
while (! successfulOp && numOfExceptions < MAX_OPERATION_EXCEPTIONS) {
try {
// Set up timer for measuring AL
startTimeAL = System.currentTimeMillis();
startTimeAL = Time.monotonicNow();
boolean result = filesystem.rename(filePath, filePathR);
if (!result) {
throw new IOException("rename failed for " + filePath);
}
totalTimeAL1 += (System.currentTimeMillis() - startTimeAL);
totalTimeAL1 += (Time.monotonicNow() - startTimeAL);

successfulOp = true;
successfulFileOps ++;
Expand Down Expand Up @@ -945,13 +946,13 @@ private void doDeleteOp(String name,
while (! successfulOp && numOfExceptions < MAX_OPERATION_EXCEPTIONS) {
try {
// Set up timer for measuring AL
startTimeAL = System.currentTimeMillis();
startTimeAL = Time.monotonicNow();
boolean result = filesystem.delete(filePath, true);
if (!result) {
throw new IOException("delete failed for " + filePath);
}
totalTimeAL1 += (System.currentTimeMillis() - startTimeAL);
totalTimeAL1 += (Time.monotonicNow() - startTimeAL);

successfulOp = true;
successfulFileOps ++;

Expand Down
Loading