From 4835c11cce4d7c0b59170cfdb5ab3c4afdeda19e Mon Sep 17 00:00:00 2001 From: Neil Ramaswamy Date: Wed, 17 Jan 2024 17:51:36 -0800 Subject: [PATCH] Add native logger support to RocksJava (#12213) Summary: ## Overview In this PR, we introduce support for setting the RocksDB native logger through Java. As mentioned in the discussion on the [Google Group discussion](https://groups.google.com/g/rocksdb/c/xYmbEs4sqRM/m/e73E4whJAQAJ), this work is primarily motivated by the JDK 17 [performance regression in JNI thread attach/detach calls](https://bugs.openjdk.org/browse/JDK-8314859): the only existing RocksJava logging configuration call, `setLogger`, invokes the provided logger over the JNI. ## Changes Specifically, these changes add support for the `devnull` and `stderr` native loggers. For the `stderr` logger, we add the ability to prefix every log with a `logPrefix`, so that it becomes possible know which database a particular log is coming from (if multiple databases are in use). The API looks like the following: ```java Options opts = new Options(); NativeLogger stderrNativeLogger = NativeLogger.newStderrLogger( InfoLogLevel.DEBUG_LEVEL, "[my prefix here]"); options.setLogger(stderrNativeLogger); try (final RocksDB db = RocksDB.open(options, ...)) {...} // Cleanup stderrNativeLogger.close() opts.close(); ``` Note that the API to set the logger is the same, via `Options::setLogger` (or `DBOptions::setLogger`). However, it will set the RocksDB logger to be native when the provided logger is an instance of `NativeLogger`. ## Testing Two tests have been added in `NativeLoggerTest.java`. The first test creates both the `devnull` and `stderr` loggers, and sets them on the associated `Options`. However, to avoid polluting the testing output with logs from `stderr`, only the `devnull` logger is actually used in the test. The second test does the same logic, but for `DBOptions`. It is possible to manually verify the `stderr` logger by modifying the tests slightly, and observing that the console indeed gets cluttered with logs from `stderr`. Pull Request resolved: https://github.com/facebook/rocksdb/pull/12213 Reviewed By: cbi42 Differential Revision: D52772306 Pulled By: ajkr fbshipit-source-id: 4026895f78f9cc250daf6bfa57427957e2d8b053 --- java/CMakeLists.txt | 6 ++ java/Makefile | 6 +- java/rocksjni/loggerjnicallback.cc | 32 ++----- java/rocksjni/options.cc | 63 +++++++++++--- java/rocksjni/stderr_logger.cc | 85 +++++++++++++++++++ java/src/main/java/org/rocksdb/DBOptions.java | 7 +- .../java/org/rocksdb/DBOptionsInterface.java | 4 +- java/src/main/java/org/rocksdb/Logger.java | 74 +++++++++------- .../java/org/rocksdb/LoggerInterface.java | 40 +++++++++ .../src/main/java/org/rocksdb/LoggerType.java | 48 +++++++++++ java/src/main/java/org/rocksdb/Options.java | 7 +- .../java/org/rocksdb/util/StdErrLogger.java | 56 ++++++++++++ .../src/test/java/org/rocksdb/LoggerTest.java | 35 ++++++++ .../org/rocksdb/util/StdErrLoggerTest.java | 45 ++++++++++ src.mk | 1 + util/stderr_logger.cc | 47 ++++++++-- util/stderr_logger.h | 12 ++- 17 files changed, 474 insertions(+), 94 deletions(-) create mode 100644 java/rocksjni/stderr_logger.cc create mode 100644 java/src/main/java/org/rocksdb/LoggerInterface.java create mode 100644 java/src/main/java/org/rocksdb/LoggerType.java create mode 100644 java/src/main/java/org/rocksdb/util/StdErrLogger.java create mode 100644 java/src/test/java/org/rocksdb/util/StdErrLoggerTest.java diff --git a/java/CMakeLists.txt b/java/CMakeLists.txt index e4121354ee0..06bea8706ca 100644 --- a/java/CMakeLists.txt +++ b/java/CMakeLists.txt @@ -74,6 +74,7 @@ set(JNI_NATIVE_SOURCES rocksjni/sst_partitioner.cc rocksjni/statistics.cc rocksjni/statisticsjni.cc + rocksjni/stderr_logger.cc rocksjni/table.cc rocksjni/table_filter.cc rocksjni/table_filter_jnicallback.cc @@ -188,6 +189,8 @@ set(JAVA_MAIN_CLASSES src/main/java/org/rocksdb/LiveFileMetaData.java src/main/java/org/rocksdb/LogFile.java src/main/java/org/rocksdb/Logger.java + src/main/java/org/rocksdb/LoggerInterface.java + src/main/java/org/rocksdb/LoggerType.java src/main/java/org/rocksdb/LRUCache.java src/main/java/org/rocksdb/MemoryUsageType.java src/main/java/org/rocksdb/MemoryUtil.java @@ -302,6 +305,7 @@ set(JAVA_MAIN_CLASSES src/test/java/org/rocksdb/WriteBatchTest.java src/test/java/org/rocksdb/RocksNativeLibraryResource.java src/test/java/org/rocksdb/util/CapturingWriteBatchHandler.java + src/main/java/org/rocksdb/util/StdErrLogger.java src/test/java/org/rocksdb/util/WriteBatchGetter.java ) @@ -413,6 +417,7 @@ set(JAVA_TEST_CLASSES src/test/java/org/rocksdb/MemoryUtilTest.java src/test/java/org/rocksdb/TableFilterTest.java src/test/java/org/rocksdb/TtlDBTest.java + src/test/java/org/rocksdb/util/StdErrLoggerTest.java ) set(JAVA_TEST_RUNNING_CLASSES @@ -511,6 +516,7 @@ set(JAVA_TEST_RUNNING_CLASSES org.rocksdb.MemoryUtilTest org.rocksdb.TableFilterTest org.rocksdb.TtlDBTest + org.rocksdb.util.StdErrLoggerTest ) include(FindJava) diff --git a/java/Makefile b/java/Makefile index a887a24b305..5e00921c62b 100644 --- a/java/Makefile +++ b/java/Makefile @@ -89,7 +89,8 @@ NATIVE_JAVA_CLASSES = \ org.rocksdb.WriteOptions\ org.rocksdb.WriteBatchWithIndex\ org.rocksdb.WriteBufferManager\ - org.rocksdb.WBWIRocksIterator + org.rocksdb.WBWIRocksIterator\ + org.rocksdb.util.StdErrLogger NATIVE_JAVA_TEST_CLASSES = \ org.rocksdb.RocksDBExceptionTest\ @@ -206,7 +207,8 @@ JAVA_TESTS = \ org.rocksdb.WriteBatchTest\ org.rocksdb.WriteBatchThreadedTest\ org.rocksdb.WriteOptionsTest\ - org.rocksdb.WriteBatchWithIndexTest + org.rocksdb.WriteBatchWithIndexTest\ + org.rocksdb.util.StdErrLoggerTest MAIN_SRC = src/main/java TEST_SRC = src/test/java diff --git a/java/rocksjni/loggerjnicallback.cc b/java/rocksjni/loggerjnicallback.cc index aa9f95cd4cf..28f64e638de 100644 --- a/java/rocksjni/loggerjnicallback.cc +++ b/java/rocksjni/loggerjnicallback.cc @@ -224,37 +224,15 @@ LoggerJniCallback::~LoggerJniCallback() { /* * Class: org_rocksdb_Logger - * Method: createNewLoggerOptions + * Method: newLogger * Signature: (J)J */ -jlong Java_org_rocksdb_Logger_createNewLoggerOptions(JNIEnv* env, jobject jobj, - jlong joptions) { +jlong Java_org_rocksdb_Logger_newLogger(JNIEnv* env, jobject jobj, + jlong jlog_level) { auto* sptr_logger = new std::shared_ptr( new ROCKSDB_NAMESPACE::LoggerJniCallback(env, jobj)); - - // set log level - auto* options = reinterpret_cast(joptions); - sptr_logger->get()->SetInfoLogLevel(options->info_log_level); - - return GET_CPLUSPLUS_POINTER(sptr_logger); -} - -/* - * Class: org_rocksdb_Logger - * Method: createNewLoggerDbOptions - * Signature: (J)J - */ -jlong Java_org_rocksdb_Logger_createNewLoggerDbOptions(JNIEnv* env, - jobject jobj, - jlong jdb_options) { - auto* sptr_logger = new std::shared_ptr( - new ROCKSDB_NAMESPACE::LoggerJniCallback(env, jobj)); - - // set log level - auto* db_options = - reinterpret_cast(jdb_options); - sptr_logger->get()->SetInfoLogLevel(db_options->info_log_level); - + auto log_level = static_cast(jlog_level); + sptr_logger->get()->SetInfoLogLevel(log_level); return GET_CPLUSPLUS_POINTER(sptr_logger); } diff --git a/java/rocksjni/options.cc b/java/rocksjni/options.cc index 596c8ad96b4..84c3f4481fa 100644 --- a/java/rocksjni/options.cc +++ b/java/rocksjni/options.cc @@ -38,6 +38,7 @@ #include "rocksjni/statisticsjni.h" #include "rocksjni/table_filter_jnicallback.h" #include "rocksjni/table_properties_collector_factory.h" +#include "util/stderr_logger.h" #include "utilities/merge_operators.h" /* @@ -1084,14 +1085,31 @@ void Java_org_rocksdb_Options_setSstFileManager( /* * Class: org_rocksdb_Options * Method: setLogger - * Signature: (JJ)V + * Signature: (JJB)V */ -void Java_org_rocksdb_Options_setLogger(JNIEnv*, jobject, jlong jhandle, - jlong jlogger_handle) { - std::shared_ptr* pLogger = - reinterpret_cast*>( - jlogger_handle); - reinterpret_cast(jhandle)->info_log = *pLogger; +void Java_org_rocksdb_Options_setLogger(JNIEnv* env, jobject, jlong jhandle, + jlong jlogger_handle, + jbyte jlogger_type) { + auto* options = reinterpret_cast(jhandle); + switch (jlogger_type) { + case 0x1: + // JAVA_IMPLEMENTATION + options->info_log = + *(reinterpret_cast< + std::shared_ptr*>( + jlogger_handle)); + break; + case 0x2: + // STDERR_IMPLEMENTATION + options->info_log = + *(reinterpret_cast*>( + jlogger_handle)); + break; + default: + ROCKSDB_NAMESPACE::IllegalArgumentExceptionJni::ThrowNew( + env, ROCKSDB_NAMESPACE::Status::InvalidArgument( + ROCKSDB_NAMESPACE::Slice("Unknown value for LoggerType"))); + } } /* @@ -6148,14 +6166,31 @@ void Java_org_rocksdb_DBOptions_setSstFileManager( /* * Class: org_rocksdb_DBOptions * Method: setLogger - * Signature: (JJ)V + * Signature: (JJB)V */ -void Java_org_rocksdb_DBOptions_setLogger(JNIEnv*, jobject, jlong jhandle, - jlong jlogger_handle) { - std::shared_ptr* pLogger = - reinterpret_cast*>( - jlogger_handle); - reinterpret_cast(jhandle)->info_log = *pLogger; +void Java_org_rocksdb_DBOptions_setLogger(JNIEnv* env, jobject, jlong jhandle, + jlong jlogger_handle, + jbyte jlogger_type) { + auto* options = reinterpret_cast(jhandle); + switch (jlogger_type) { + case 0x1: + // JAVA_IMPLEMENTATION + options->info_log = + *(reinterpret_cast< + std::shared_ptr*>( + jlogger_handle)); + break; + case 0x2: + // STDERR_IMPLEMENTATION + options->info_log = + *(reinterpret_cast*>( + jlogger_handle)); + break; + default: + ROCKSDB_NAMESPACE::IllegalArgumentExceptionJni::ThrowNew( + env, ROCKSDB_NAMESPACE::Status::InvalidArgument( + ROCKSDB_NAMESPACE::Slice("Unknown value for LoggerType"))); + } } /* diff --git a/java/rocksjni/stderr_logger.cc b/java/rocksjni/stderr_logger.cc new file mode 100644 index 00000000000..22da70c2e86 --- /dev/null +++ b/java/rocksjni/stderr_logger.cc @@ -0,0 +1,85 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "util/stderr_logger.h" + +#include + +#include + +#include "include/org_rocksdb_util_StdErrLogger.h" +#include "rocksjni/cplusplus_to_java_convert.h" +#include "rocksjni/portal.h" + +/* + * Class: org_rocksdb_util_StdErrLogger + * Method: newStdErrLogger + * Signature: (BLjava/lang/String;)J + */ +jlong Java_org_rocksdb_util_StdErrLogger_newStdErrLogger(JNIEnv* env, + jclass /*jcls*/, + jbyte jlog_level, + jstring jlog_prefix) { + auto log_level = static_cast(jlog_level); + std::shared_ptr* sptr_logger = nullptr; + if (jlog_prefix == nullptr) { + sptr_logger = new std::shared_ptr( + new ROCKSDB_NAMESPACE::StderrLogger(log_level)); + } else { + jboolean has_exception = JNI_FALSE; + auto log_prefix = ROCKSDB_NAMESPACE::JniUtil::copyStdString( + env, jlog_prefix, &has_exception); // also releases jlog_prefix + if (has_exception == JNI_TRUE) { + return 0; + } + sptr_logger = new std::shared_ptr( + new ROCKSDB_NAMESPACE::StderrLogger(log_level, log_prefix)); + } + return GET_CPLUSPLUS_POINTER(sptr_logger); +} + +/* + * Class: org_rocksdb_util_StdErrLogger + * Method: setInfoLogLevel + * Signature: (JB)V + */ +void Java_org_rocksdb_util_StdErrLogger_setInfoLogLevel(JNIEnv* /*env*/, + jclass /*jcls*/, + jlong jhandle, + jbyte jlog_level) { + auto* handle = + reinterpret_cast*>( + jhandle); + handle->get()->SetInfoLogLevel( + static_cast(jlog_level)); +} + +/* + * Class: org_rocksdb_util_StdErrLogger + * Method: infoLogLevel + * Signature: (J)B + */ +jbyte Java_org_rocksdb_util_StdErrLogger_infoLogLevel(JNIEnv* /*env*/, + jclass /*jcls*/, + jlong jhandle) { + auto* handle = + reinterpret_cast*>( + jhandle); + return static_cast(handle->get()->GetInfoLogLevel()); +} + +/* + * Class: org_rocksdb_util_StdErrLogger + * Method: disposeInternal + * Signature: (J)V + */ +void Java_org_rocksdb_util_StdErrLogger_disposeInternal(JNIEnv* /*env*/, + jobject /*jobj*/, + jlong jhandle) { + auto* handle = + reinterpret_cast*>( + jhandle); + delete handle; // delete std::shared_ptr +} diff --git a/java/src/main/java/org/rocksdb/DBOptions.java b/java/src/main/java/org/rocksdb/DBOptions.java index de10c058501..cd0fbeaeb51 100644 --- a/java/src/main/java/org/rocksdb/DBOptions.java +++ b/java/src/main/java/org/rocksdb/DBOptions.java @@ -213,9 +213,9 @@ public DBOptions setSstFileManager(final SstFileManager sstFileManager) { } @Override - public DBOptions setLogger(final Logger logger) { + public DBOptions setLogger(final LoggerInterface logger) { assert(isOwningHandle()); - setLogger(nativeHandle_, logger.nativeHandle_); + setLogger(nativeHandle_, logger.getNativeHandle(), logger.getLoggerType().getValue()); return this; } @@ -1275,8 +1275,7 @@ private native void setRateLimiter(long handle, long rateLimiterHandle); private native void setSstFileManager(final long handle, final long sstFileManagerHandle); - private native void setLogger(long handle, - long loggerHandle); + private native void setLogger(final long handle, final long loggerHandle, final byte loggerType); private native void setInfoLogLevel(long handle, byte logLevel); private native byte infoLogLevel(long handle); private native void setMaxOpenFiles(long handle, int maxOpenFiles); diff --git a/java/src/main/java/org/rocksdb/DBOptionsInterface.java b/java/src/main/java/org/rocksdb/DBOptionsInterface.java index 084a399cd03..5a351b4e45a 100644 --- a/java/src/main/java/org/rocksdb/DBOptionsInterface.java +++ b/java/src/main/java/org/rocksdb/DBOptionsInterface.java @@ -185,10 +185,10 @@ public interface DBOptionsInterface> { * *

Default: nullptr

* - * @param logger {@link Logger} instance. + * @param logger {@link LoggerInterface} instance. * @return the instance of the current object. */ - T setLogger(Logger logger); + T setLogger(LoggerInterface logger); /** *

Sets the RocksDB log level. Default level is INFO

diff --git a/java/src/main/java/org/rocksdb/Logger.java b/java/src/main/java/org/rocksdb/Logger.java index 614a7fa502f..b8d0e45efa0 100644 --- a/java/src/main/java/org/rocksdb/Logger.java +++ b/java/src/main/java/org/rocksdb/Logger.java @@ -35,10 +35,7 @@ * {@link org.rocksdb.InfoLogLevel#FATAL_LEVEL}. *

*/ -public abstract class Logger extends RocksCallbackObject { - private static final long WITH_OPTIONS = 0; - private static final long WITH_DBOPTIONS = 1; - +public abstract class Logger extends RocksCallbackObject implements LoggerInterface { /** *

AbstractLogger constructor.

* @@ -47,10 +44,13 @@ public abstract class Logger extends RocksCallbackObject { * maximum log level of RocksDB.

* * @param options {@link org.rocksdb.Options} instance. + * + * @deprecated Use {@link Logger#Logger(InfoLogLevel)} instead, e.g. {@code new + * Logger(options.infoLogLevel())}. */ + @Deprecated public Logger(final Options options) { - super(options.nativeHandle_, WITH_OPTIONS); - + this(options.infoLogLevel()); } /** @@ -61,56 +61,64 @@ public Logger(final Options options) { * as maximum log level of RocksDB.

* * @param dboptions {@link org.rocksdb.DBOptions} instance. + * + * @deprecated Use {@link Logger#Logger(InfoLogLevel)} instead, e.g. {@code new + * Logger(dbOptions.infoLogLevel())}. */ + @Deprecated public Logger(final DBOptions dboptions) { - super(dboptions.nativeHandle_, WITH_DBOPTIONS); + this(dboptions.infoLogLevel()); + } + + /** + *

AbstractLogger constructor.

+ * + * @param logLevel the log level. + */ + public Logger(final InfoLogLevel logLevel) { + super(logLevel.getValue()); } @Override protected long initializeNative(final long... nativeParameterHandles) { - if(nativeParameterHandles[1] == WITH_OPTIONS) { - return createNewLoggerOptions(nativeParameterHandles[0]); - } else if(nativeParameterHandles[1] == WITH_DBOPTIONS) { - return createNewLoggerDbOptions(nativeParameterHandles[0]); + if (nativeParameterHandles.length == 1) { + return newLogger(nativeParameterHandles[0]); } else { throw new IllegalArgumentException(); } } - /** - * Set {@link org.rocksdb.InfoLogLevel} to AbstractLogger. - * - * @param infoLogLevel {@link org.rocksdb.InfoLogLevel} instance. - */ - public void setInfoLogLevel(final InfoLogLevel infoLogLevel) { - setInfoLogLevel(nativeHandle_, infoLogLevel.getValue()); + @Override + public void setInfoLogLevel(final InfoLogLevel logLevel) { + setInfoLogLevel(nativeHandle_, logLevel.getValue()); } - /** - * Return the loggers log level. - * - * @return {@link org.rocksdb.InfoLogLevel} instance. - */ + @Override public InfoLogLevel infoLogLevel() { return InfoLogLevel.getInfoLogLevel( infoLogLevel(nativeHandle_)); } - protected abstract void log(InfoLogLevel infoLogLevel, - String logMsg); + @Override + public long getNativeHandle() { + return nativeHandle_; + } + + @Override + public final LoggerType getLoggerType() { + return LoggerType.JAVA_IMPLEMENTATION; + } + + protected abstract void log(final InfoLogLevel logLevel, final String logMsg); - protected native long createNewLoggerOptions( - long options); - protected native long createNewLoggerDbOptions( - long dbOptions); - protected native void setInfoLogLevel(long handle, - byte infoLogLevel); - protected native byte infoLogLevel(long handle); + protected native long newLogger(final long logLevel); + protected native void setInfoLogLevel(final long handle, final byte logLevel); + protected native byte infoLogLevel(final long handle); /** * We override {@link RocksCallbackObject#disposeInternal()} * as disposing of a rocksdb::LoggerJniCallback requires - * a slightly different approach as it is a std::shared_ptr + * a slightly different approach as it is a std::shared_ptr. */ @Override protected void disposeInternal() { diff --git a/java/src/main/java/org/rocksdb/LoggerInterface.java b/java/src/main/java/org/rocksdb/LoggerInterface.java new file mode 100644 index 00000000000..51239830b94 --- /dev/null +++ b/java/src/main/java/org/rocksdb/LoggerInterface.java @@ -0,0 +1,40 @@ +// Copyright (c) 2016, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb; + +/** + * LoggerInterface is a thin interface that specifies the most basic + * functionality for a Java wrapper around a RocksDB Logger. + */ +public interface LoggerInterface { + /** + * Set the log level. + * + * @param logLevel the level at which to log. + */ + void setInfoLogLevel(final InfoLogLevel logLevel); + + /** + * Get the log level + * + * @return the level at which to log. + */ + InfoLogLevel infoLogLevel(); + + /** + * Get the underlying Native Handle. + * + * @return the native handle. + */ + long getNativeHandle(); + + /** + * Get the type of this logger. + * + * @return the type of this logger. + */ + LoggerType getLoggerType(); +} diff --git a/java/src/main/java/org/rocksdb/LoggerType.java b/java/src/main/java/org/rocksdb/LoggerType.java new file mode 100644 index 00000000000..f5d0b0d954c --- /dev/null +++ b/java/src/main/java/org/rocksdb/LoggerType.java @@ -0,0 +1,48 @@ +// Copyright (c) 2016, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +package org.rocksdb; + +/** + * Simple enumeration used for differentiating + * the types of loggers when passing via the JNI + * boundary. + */ +public enum LoggerType { + JAVA_IMPLEMENTATION((byte) 0x1), + STDERR_IMPLEMENTATION((byte) 0x2); + + private final byte value; + + LoggerType(final byte value) { + this.value = value; + } + + /** + * Returns the byte value of the enumerations value + * + * @return byte representation + */ + byte getValue() { + return value; + } + + /** + * Get LoggerType by byte value. + * + * @param value byte representation of LoggerType. + * + * @return {@link org.rocksdb.LoggerType} instance. + * @throws java.lang.IllegalArgumentException if an invalid + * value is provided. + */ + static LoggerType getLoggerType(final byte value) { + for (final LoggerType loggerType : LoggerType.values()) { + if (loggerType.getValue() == value) { + return loggerType; + } + } + throw new IllegalArgumentException("Illegal value provided for LoggerType."); + } +} diff --git a/java/src/main/java/org/rocksdb/Options.java b/java/src/main/java/org/rocksdb/Options.java index e92e178ef50..32fa75f1d93 100644 --- a/java/src/main/java/org/rocksdb/Options.java +++ b/java/src/main/java/org/rocksdb/Options.java @@ -1246,9 +1246,9 @@ public Options setSstFileManager(final SstFileManager sstFileManager) { } @Override - public Options setLogger(final Logger logger) { + public Options setLogger(final LoggerInterface logger) { assert(isOwningHandle()); - setLogger(nativeHandle_, logger.nativeHandle_); + setLogger(nativeHandle_, logger.getNativeHandle(), logger.getLoggerType().getValue()); return this; } @@ -2180,8 +2180,7 @@ private native void setRateLimiter(long handle, long rateLimiterHandle); private native void setSstFileManager(final long handle, final long sstFileManagerHandle); - private native void setLogger(long handle, - long loggerHandle); + private native void setLogger(final long handle, final long loggerHandle, final byte loggerType); private native void setInfoLogLevel(long handle, byte logLevel); private native byte infoLogLevel(long handle); private native void setMaxOpenFiles(long handle, int maxOpenFiles); diff --git a/java/src/main/java/org/rocksdb/util/StdErrLogger.java b/java/src/main/java/org/rocksdb/util/StdErrLogger.java new file mode 100644 index 00000000000..00b08d38452 --- /dev/null +++ b/java/src/main/java/org/rocksdb/util/StdErrLogger.java @@ -0,0 +1,56 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +package org.rocksdb.util; + +import org.rocksdb.InfoLogLevel; +import org.rocksdb.LoggerInterface; +import org.rocksdb.LoggerType; +import org.rocksdb.RocksObject; + +/** + * Simply redirects all log messages to StdErr. + */ +public class StdErrLogger extends RocksObject implements LoggerInterface { + /** + * Constructs a new StdErrLogger. + * + * @param logLevel the level at which to log. + */ + public StdErrLogger(final InfoLogLevel logLevel) { + this(logLevel, null); + } + + /** + * Constructs a new StdErrLogger. + * + * @param logLevel the level at which to log. + * @param logPrefix the string with which to prefix all log messages. + */ + public StdErrLogger(final InfoLogLevel logLevel, /* @Nullable */ final String logPrefix) { + super(newStdErrLogger(logLevel.getValue(), logPrefix)); + } + + @Override + public void setInfoLogLevel(final InfoLogLevel logLevel) { + setInfoLogLevel(nativeHandle_, logLevel.getValue()); + } + + @Override + public InfoLogLevel infoLogLevel() { + return InfoLogLevel.getInfoLogLevel(infoLogLevel(nativeHandle_)); + } + + @Override + public LoggerType getLoggerType() { + return LoggerType.STDERR_IMPLEMENTATION; + } + + private static native long newStdErrLogger( + final byte logLevel, /* @Nullable */ final String logPrefix); + private static native void setInfoLogLevel(final long handle, final byte logLevel); + private static native byte infoLogLevel(final long handle); + + @Override protected native void disposeInternal(final long handle); +} diff --git a/java/src/test/java/org/rocksdb/LoggerTest.java b/java/src/test/java/org/rocksdb/LoggerTest.java index b6a7be55e7f..c174be52f02 100644 --- a/java/src/test/java/org/rocksdb/LoggerTest.java +++ b/java/src/test/java/org/rocksdb/LoggerTest.java @@ -232,4 +232,39 @@ protected void log(final InfoLogLevel infoLogLevel, final String logMsg) { } } } + + @Test + public void logLevelLogger() throws RocksDBException { + final AtomicInteger logMessageCounter = new AtomicInteger(); + try (final DBOptions options = new DBOptions().setCreateIfMissing(true); + final Logger logger = new Logger(InfoLogLevel.FATAL_LEVEL) { + // Create new logger with max log level passed by options + @Override + protected void log(final InfoLogLevel infoLogLevel, final String logMsg) { + assertThat(logMsg).isNotNull(); + assertThat(logMsg.length()).isGreaterThan(0); + logMessageCounter.incrementAndGet(); + } + }) { + // Set custom logger to options + options.setLogger(logger); + + final List cfDescriptors = + Collections.singletonList(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY)); + final List cfHandles = new ArrayList<>(); + + try (final RocksDB db = RocksDB.open( + options, dbFolder.getRoot().getAbsolutePath(), cfDescriptors, cfHandles)) { + try { + // there should be zero messages + // using fatal level as log level. + assertThat(logMessageCounter.get()).isEqualTo(0); + } finally { + for (final ColumnFamilyHandle columnFamilyHandle : cfHandles) { + columnFamilyHandle.close(); + } + } + } + } + } } diff --git a/java/src/test/java/org/rocksdb/util/StdErrLoggerTest.java b/java/src/test/java/org/rocksdb/util/StdErrLoggerTest.java new file mode 100644 index 00000000000..0788ec92e87 --- /dev/null +++ b/java/src/test/java/org/rocksdb/util/StdErrLoggerTest.java @@ -0,0 +1,45 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +package org.rocksdb.util; + +import org.junit.ClassRule; +import org.junit.Test; +import org.rocksdb.DBOptions; +import org.rocksdb.InfoLogLevel; +import org.rocksdb.Options; +import org.rocksdb.RocksNativeLibraryResource; + +public class StdErrLoggerTest { + @ClassRule + public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = + new RocksNativeLibraryResource(); + + // Logging with the stderr logger would pollute the console when tests were run (and + // from Java, we can't redirect or close stderr). + // So we just test creation of a StdErrLogger and setting it on Options + // without opening the DB. + @Test + public void nativeLoggersWithOptions() { + try (final Options options = new Options().setCreateIfMissing(true); + final StdErrLogger stdErrLogger = + new StdErrLogger(InfoLogLevel.DEBUG_LEVEL, "[Options prefix]")) { + options.setLogger(stdErrLogger); + } + } + + // Logging with the stderr logger would pollute the console when tests were run (and + // from Java, we can't redirect or close stderr). + // So we just test creation of a StdErrLogger and setting it on DBOptions + // without opening the DB. + @Test + public void nativeLoggersWithDBOptions() { + try (final DBOptions options = new DBOptions().setCreateIfMissing(true); + final StdErrLogger stdErrLogger = + new StdErrLogger(InfoLogLevel.DEBUG_LEVEL, "[DBOptions prefix]")) { + options.setLogger(stdErrLogger); + } + } +} diff --git a/src.mk b/src.mk index 96f637b4bd8..3acefe78031 100644 --- a/src.mk +++ b/src.mk @@ -702,6 +702,7 @@ JNI_NATIVE_SOURCES = \ java/rocksjni/sst_partitioner.cc \ java/rocksjni/statistics.cc \ java/rocksjni/statisticsjni.cc \ + java/rocksjni/stderr_logger.cc \ java/rocksjni/table.cc \ java/rocksjni/table_filter.cc \ java/rocksjni/table_filter_jnicallback.cc \ diff --git a/util/stderr_logger.cc b/util/stderr_logger.cc index 6044b8b9367..69e9989f04e 100644 --- a/util/stderr_logger.cc +++ b/util/stderr_logger.cc @@ -6,10 +6,15 @@ #include "util/stderr_logger.h" +#include "port/malloc.h" #include "port/sys_time.h" namespace ROCKSDB_NAMESPACE { -StderrLogger::~StderrLogger() {} +StderrLogger::~StderrLogger() { + if (log_prefix != nullptr) { + free((void*)log_prefix); + } +} void StderrLogger::Logv(const char* format, va_list ap) { const uint64_t thread_id = Env::Default()->GetThreadID(); @@ -19,12 +24,40 @@ void StderrLogger::Logv(const char* format, va_list ap) { const time_t seconds = now_tv.tv_sec; struct tm t; port::LocalTimeR(&seconds, &t); - fprintf(stderr, "%04d/%02d/%02d-%02d:%02d:%02d.%06d %llx ", t.tm_year + 1900, - t.tm_mon + 1, t.tm_mday, t.tm_hour, t.tm_min, t.tm_sec, - static_cast(now_tv.tv_usec), - static_cast(thread_id)); - vfprintf(stderr, format, ap); - fprintf(stderr, "\n"); + // The string we eventually log has three parts: the context (time, thread), + // optional user-supplied prefix, and the actual log message (the "suffix"). + // + // We compute their lengths so that we can allocate a buffer big enough to + // print it. The context string (with the date and thread id) is really only + // 44 bytes, but we allocate 50 to be safe. + // + // ctx_len = 44 = ( 4+ 1+ 2+1+2+ 1+2+ 1+2+ 1+ 2+1+6+ 1+16+1) + const char* ctx_prefix_fmt = "%04d/%02d/%02d-%02d:%02d:%02d.%06d %llx %s"; + size_t ctx_len = 50; + + va_list ap_copy; + va_copy(ap_copy, ap); + const size_t log_suffix_len = vsnprintf(nullptr, 0, format, ap_copy); + va_end(ap_copy); + + // Allocate space for the context, log_prefix, and log itself + // Extra byte for null termination + size_t buf_len = ctx_len + log_prefix_len + log_suffix_len + 1; + std::unique_ptr buf(new char[buf_len]); + + // If the logger was created without a prefix, the prefix is a nullptr + const char* prefix = log_prefix == nullptr ? "" : log_prefix; + + // Write out the context and prefix string + int written = + snprintf(buf.get(), ctx_len + log_prefix_len, ctx_prefix_fmt, + t.tm_year + 1900, t.tm_mon + 1, t.tm_mday, t.tm_hour, t.tm_min, + t.tm_sec, static_cast(now_tv.tv_usec), + static_cast(thread_id), prefix); + written += vsnprintf(buf.get() + written, log_suffix_len, format, ap); + buf[written] = '\0'; + + fprintf(stderr, "%s%c", buf.get(), '\n'); } } // namespace ROCKSDB_NAMESPACE diff --git a/util/stderr_logger.h b/util/stderr_logger.h index c3b01210cea..c4aa8e0c93d 100644 --- a/util/stderr_logger.h +++ b/util/stderr_logger.h @@ -17,7 +17,11 @@ namespace ROCKSDB_NAMESPACE { class StderrLogger : public Logger { public: explicit StderrLogger(const InfoLogLevel log_level = InfoLogLevel::INFO_LEVEL) - : Logger(log_level) {} + : Logger(log_level), log_prefix(nullptr) {} + explicit StderrLogger(const InfoLogLevel log_level, const std::string prefix) + : Logger(log_level), + log_prefix(strdup(prefix.c_str())), + log_prefix_len(strlen(log_prefix)) {} ~StderrLogger() override; @@ -26,6 +30,12 @@ class StderrLogger : public Logger { using Logger::Logv; virtual void Logv(const char* format, va_list ap) override; + + private: + // This prefix will be appended after the time/thread info of every log + const char* log_prefix; + // The length of the log_prefix + size_t log_prefix_len; }; } // namespace ROCKSDB_NAMESPACE