Query logger

CHANGES.md

@@ -1,5 +1,8 @@
 # Changes
 
+## Version 2.6.0
+* Support pure query logger deployment without authentication/authorization - #77
+
 ## Version 2.5.0
 * Add support for custom authenticator and the Subject audit record field - #146
 

bin/configure_ccm_querylog_chronicle.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+#
+# Copyright 2020 Telefonaktiebolaget LM Ericsson
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+shopt -s extglob
+
+SCRIPT_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+CCM_CONFIG=${CCM_CONFIG_DIR:=~/.ccm}
+JAR_FILE="${SCRIPT_PATH}/../ecaudit/target/ecaudit*.jar"
+
+if [ ! -f ${CCM_CONFIG}/CURRENT ]; then
+ echo "Unable to find an active ccm cluster"
+ exit 2
+fi
+
+if [ ! -f ${JAR_FILE} ]; then
+ echo "No jar file found. Build project and try again."
+ exit 3
+fi
+
+CCM_CLUSTER_NAME=`cat ${CCM_CONFIG}/CURRENT`
+echo "Installing ecAudit with Chronicle backend into ${CCM_CLUSTER_NAME}"
+
+CLUSTER_PATH=${CCM_CONFIG}/${CCM_CLUSTER_NAME}
+
+mkdir -p ${CLUSTER_PATH}/lib
+rm -f ${CLUSTER_PATH}/lib/ecaudit.jar
+ln -s ${JAR_FILE} ${CLUSTER_PATH}/lib/ecaudit.jar
+
+grep -sq ecaudit.jar ${CLUSTER_PATH}/cassandra.in.sh
+if [ $? -ne 0 ]; then
+ echo "CLASSPATH=\"\$CLASSPATH:${CLUSTER_PATH}/lib/ecaudit.jar\"" >> ${CLUSTER_PATH}/cassandra.in.sh
+ echo "JVM_EXTRA_OPTS=\"\$JVM_EXTRA_OPTS -Dcassandra.custom_query_handler_class=com.ericsson.bss.cassandra.ecaudit.handler.AuditQueryHandler\"" >> ${CLUSTER_PATH}/cassandra.in.sh
+fi
+
+grep -sq openhft ${CLUSTER_PATH}/cassandra.in.sh
+if [ $? -ne 0 ]; then
+ echo "JVM_EXTRA_OPTS=\"\$JVM_EXTRA_OPTS -da:net.openhft...\"" >> ${CLUSTER_PATH}/cassandra.in.sh
+fi
+
+grep -sq "filter_type=NONE" ${CLUSTER_PATH}/cassandra.in.sh
+if [ $? -ne 0 ]; then
+ sed -i '/filter_type/d' ${CLUSTER_PATH}/cassandra.in.sh
+ echo "JVM_EXTRA_OPTS=\"\$JVM_EXTRA_OPTS -Decaudit.filter_type=NONE\"" >> ${CLUSTER_PATH}/cassandra.in.sh
+fi
+
+update_audit_yaml() {
+ mkdir -p $2
+ rm -f $1
+ cat <<EOF > $1
+
+logger_backend:
+  - class_name: com.ericsson.bss.cassandra.ecaudit.logger.ChronicleAuditLogger
+    parameters:
+      - log_dir: $2
+        roll_cycle: MINUTELY
+        max_log_size: 1073741824 # 1GB
+        fields: TIMESTAMP, OPERATION, BATCH_ID, STATUS
+EOF
+}
+
+for NODE_PATH in ${CLUSTER_PATH}/node*;
+do
+ sed -i 's/^authenticator:.*/authenticator: AllowAllAuthenticator/' ${NODE_PATH}/conf/cassandra.yaml
+ sed -i 's/^authorizer:.*/authorizer: AllowAllAuthorizer/' ${NODE_PATH}/conf/cassandra.yaml
+ sed -i 's/^role_manager:.*/role_manager: com.ericsson.bss.cassandra.ecaudit.auth.AuditRoleManager/' ${NODE_PATH}/conf/cassandra.yaml
+ update_audit_yaml ${NODE_PATH}/conf/audit.yaml ${NODE_PATH}/logs/audit
+done

bin/import_cassandra_yaml_to_it.sh

@@ -55,3 +55,6 @@ for IT_TARGET in ${SCRIPT_PATH}/../integration-test-*/src/test/resources/cassand
 do
  format_for_it ${IMPORT_YAML} ${IT_TARGET}
 done
+
+sed -i 's/^authenticator:.*/authenticator: AllowAllAuthenticator/' ${SCRIPT_PATH}/../integration-test-query-logger/src/test/resources/cassandra.yaml
+sed -i 's/^authorizer:.*/authorizer: AllowAllAuthorizer/' ${SCRIPT_PATH}/../integration-test-query-logger/src/test/resources/cassandra.yaml

bin/run_ccm_performance_test.sh

@@ -147,6 +147,13 @@ ccm node1 cqlsh -u cassandra -p cassandra -x "ALTER ROLE cassandra WITH OPTIONS
 run_stress authentication-authorization-audit-role-whitelist-permission-derived
 stop_cassandra
 
+create_cluster vanilla-chron
+${SCRIPT_PATH}/configure_ccm_querylog_chronicle.sh
+start_cassandra
+create_dummy_whitelists
+run_stress vanilla-querylog-chronicle
+stop_cassandra
+
 create_cluster aaa-chron
 ${SCRIPT_PATH}/configure_ccm_audit_chronicle.sh
 start_cassandra

doc/example_query_logger.md

@@ -0,0 +1,78 @@
+# Query Logger Example
+
+This short guide will demonstrate how to setup ecAudit as a pure Query Logger in combination with the [Chronicle Logger](chronicle_logger.md) backend.
+In this example all CQL queries will be logged without any filtering, and users will be able to connect without any credentials.
+With a setup like this it is possible to log all CQL queries in a cluster for troubleshooting purposes and similar. 
+
+## cassandra.yaml
+
+Change the following setting in your ```cassandra.yaml```.
+
+```
+role_manager: com.ericsson.bss.cassandra.ecaudit.auth.AuditRoleManager
+```
+
+Leave the ```authenticator``` and ```authorizer``` settings at their default.
+
+```
+authenticator: AllowAllAuthenticator
+authorizer: AllowAllAuthorizer
+```
+
+
+## cassandra-env.sh
+
+Add the following JVM option to the ```cassandra-env.sh``` or the ```cassandra.in.sh```.
+
+**Note:** If you configure these settings in your ```cassandra-env.sh```,
+consider that the ```JVM_EXTRA_OPTS``` variable is consumed at the end of the file,
+so make sure to add the following lines *before* they are consumed.
+
+```
+JVM_EXTRA_OPTS="$JVM_EXTRA_OPTS -Dcassandra.custom_query_handler_class=com.ericsson.bss.cassandra.ecaudit.handler.AuditQueryHandler"
+JVM_EXTRA_OPTS="$JVM_EXTRA_OPTS -Decaudit.filter_type=NONE"
+JVM_EXTRA_OPTS="$JVM_EXTRA_OPTS -da:net.openhft..."
+```
+
+
+## audit.yaml
+
+Setup the ```audit.yaml``` as follows to roll a new Chronicle log file every minute,
+and store only fields which are relevant in this example.
+Also, use ```post_logging``` to make sure there is one audit record per CQL request.
+
+```
+log_timing_strategy: post_logging
+
+logger_backend:
+    - class_name: com.ericsson.bss.cassandra.ecaudit.logger.ChronicleAuditLogger
+      parameters:
+      - log_dir: /var/lib/cassandra/audit
+        roll_cycle: MINUTELY
+        fields: TIMESTAMP, OPERATION, BATCH_ID, STATUS
+```
+
+## eclog
+
+The query logs contain binary data since they are written with the Chronicle logger backend.
+
+The ```eclog``` tool will print the content in a human readable format.
+With a configuration file it is possible to get a custom format of each record.
+Create a ```eclog.yaml``` file with the following content in the same place as the chronicle log files under ```/var/lib/casssandra/audit/```.
+
+```
+log_format: "${TIMESTAMP}|{?${BATCH_ID}?}|${STATUS}|${OPERATION}"
+time_format: "yyyy-MM-dd HH:mm:ss z"
+```
+
+Here's an example on how that could look.
+
+```
+# java -jar eclog.jar -t 6 /var/lib/cassandra/audit/
+2020-04-01 12:42:02 CEST||ATTEMPT|INSERT INTO country.by_code (code, name, iso) VALUES ( 46, 'Sweden', 'SE');
+2020-04-01 12:42:10 CEST||ATTEMPT|INSERT INTO country.by_code (code, name, iso) VALUES ( 47, 'Norway', 'NO');
+2020-04-01 12:42:18 CEST||ATTEMPT|INSERT INTO country.by_code (code, name, iso) VALUES ( 48, 'Poland', 'PL');
+2020-04-01 12:42:24 CEST||ATTEMPT|INSERT INTO country.by_code (code, name, iso) VALUES ( 49, 'Germany', 'DE');
+2020-04-01 12:42:32 CEST||ATTEMPT|INSERT INTO country.by_code (code, name, iso) VALUES ( 51, 'Peru', 'PE');
+2020-04-01 12:43:10 CEST||ATTEMPT|SELECT iso FROM country.by_code WHERE code = 46;
+```

doc/install.md

@@ -1,6 +1,8 @@
 # Installation
 
 Follow these instructions to install ecAudit in each node of your Cassandra cluster.
+This will enable all the features provided by ecAudit.
+Visit the [sertup guide](setup.md) for a detailed list of options.
 
 
 ## Deploy Plug-In Jar File
@@ -36,7 +38,7 @@ Add the following JVM option to your ```cassandra-env.sh``` or your ```cassandra
 
 **Note:** If you configure these settings in your ```cassandra-env.sh```,
 consider that the ```JVM_EXTRA_OPTS``` variable is consumed at the end of the file,
-so make sure to add the following line *before* it is consumed.
+so make sure to add the following lines *before* they are consumed.
 
 ```
 JVM_EXTRA_OPTS="$JVM_EXTRA_OPTS -Dcassandra.custom_query_handler_class=com.ericsson.bss.cassandra.ecaudit.handler.AuditQueryHandler"

doc/setup.md

@@ -26,32 +26,9 @@ or with Java properties which typically are set in the ```cassandra-env.sh``` fi
 
 Read on below to learn how to tune the logger backend and manage audit whitelists.
 
-In the [audit.yaml reference](audit_yaml_reference.md) you'll find more details about different options.
+In the [audit.yaml reference](audit_yaml_reference.md) there are more details about different options.
 
-
-### Wrapped Authenticator Backend
-
-The ecAudit plug-in must be installed as the ```authenticator``` in the ```cassandra.yaml``` in order to capture authentication operations for auditing.
-The actual authentication must still be handled by an ```IDecoratedAuthenticator``` implementation.
-The ```IDecoratedAuthenticator``` is an extended version of the ```IAuthenticator``` interface provided by Cassandra. 
-ecAudit provides the ```DecoratedPasswordAuthenticator``` out of the box, which is used by default.
-The ```DecoratedPasswordAuthenticator``` is just and extended version of the ```PasswordAuthenticator``` provided by Cassandra.
-
-With the ```wrapped_authenticator``` setting in the ```audit.yaml``` file it is possible to configure another ```IDecoratedAuthenticator``` which may be your own custom implementation.
-This is useful if requiring a non-default (i.e. not ```PasswordAuthenticator```) implementation of an authenticator but still want authentication logs.
-
-For backwards compatibility reasons the ```AuditPasswordAuthenticator``` is also provided by ecAudit.
-It is an ```IAuthenticator``` which will emit audit records and it is hard wired to use the standard ```PasswordAuthenticator``` for authentication.
-It may be installed directly in the  ```cassandra.yaml```.
-
-
-### Wrapped Authorizer Backend
-
-The ecAudit plug-in must be installed as the ```authorizer``` in the ```cassandra.yaml``` in order to capture some operations.
-The actual authorization still has to be handled by another ```IAuthorizer``` implementation.
-Cassandra provides the ```AllowAllAuthorizer``` and the ```CassandraAuthorizer``` out of the box,
-where ecAudit will delegate authorization operations to the ```CassandraAuthorizer``` by default.
-With the ```wrapped_authorizer``` setting in the ```audit.yaml``` file it is possible to configure another ```IAuthorizer``` which may be your own custom implementation.
+A custom setup is described in the [Query Logger Example](example_query_logger.md).
 
 
 ### Logger Backend
@@ -92,3 +69,28 @@ To disable whitelists all together, add the following option near the end of you
 ```
 JVM_EXTRA_OPTS="$JVM_EXTRA_OPTS -Decaudit.filter_type=NONE"
 ```
+
+
+### Wrapped Authenticator Backend
+
+The ecAudit plug-in must be installed as the ```authenticator``` in the ```cassandra.yaml``` in order to capture authentication operations for auditing.
+The actual authentication must still be handled by an ```IDecoratedAuthenticator``` implementation.
+The ```IDecoratedAuthenticator``` is an extended version of the ```IAuthenticator``` interface provided by Cassandra.
+ecAudit provides the ```DecoratedPasswordAuthenticator``` out of the box, which is used by default.
+The ```DecoratedPasswordAuthenticator``` is just and extended version of the ```PasswordAuthenticator``` provided by Cassandra.
+
+With the ```wrapped_authenticator``` setting in the ```audit.yaml``` file it is possible to configure another ```IDecoratedAuthenticator``` which may be your own custom implementation.
+This is useful if requiring a non-default (i.e. not ```PasswordAuthenticator```) implementation of an authenticator but still want authentication logs.
+
+For backwards compatibility reasons the ```AuditPasswordAuthenticator``` is also provided by ecAudit.
+It is an ```IAuthenticator``` which will emit audit records and it is hard wired to use the standard ```PasswordAuthenticator``` for authentication.
+It may be installed directly in the  ```cassandra.yaml```.
+
+
+### Wrapped Authorizer Backend
+
+The ecAudit plug-in must be installed as the ```authorizer``` in the ```cassandra.yaml``` in order to capture some operations.
+The actual authorization still has to be handled by another ```IAuthorizer``` implementation.
+Cassandra provides the ```AllowAllAuthorizer``` and the ```CassandraAuthorizer``` out of the box,
+where ecAudit will delegate authorization operations to the ```CassandraAuthorizer``` by default.
+With the ```wrapped_authorizer``` setting in the ```audit.yaml``` file it is possible to configure another ```IAuthorizer``` which may be your own custom implementation.

ecaudit/src/main/java/com/ericsson/bss/cassandra/ecaudit/auth/AuditAuthenticator.java

@@ -117,6 +117,7 @@ public SaslNegotiator newSaslNegotiator()
     @Override
     public AuthenticatedUser legacyAuthenticate(Map<String, String> credentials) throws AuthenticationException
     {
+        LOG.debug("Setting up SASL negotiation with client peer");
         return wrappedAuthenticator.legacyAuthenticate(credentials);
     }
 

ecaudit/src/main/java/com/ericsson/bss/cassandra/ecaudit/auth/AuditRoleManager.java

@@ -25,6 +25,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.ericsson.bss.cassandra.ecaudit.AuditAdapter;
 import org.apache.cassandra.auth.AuthKeyspace;
 import org.apache.cassandra.auth.AuthenticatedUser;
 import org.apache.cassandra.auth.CassandraRoleManager;
@@ -57,6 +58,7 @@ public class AuditRoleManager implements IRoleManager
     private final IRoleManager wrappedRoleManager;
     private final AuditWhitelistManager whitelistManager;
     private final PermissionChecker permissionChecker;
+    private final AuditAdapter auditAdapter;
 
     private final Set<Option> supportedOptions;
     private final Set<Option> alterableOptions;
@@ -71,16 +73,18 @@ public AuditRoleManager()
     {
         this(new CassandraRoleManager(),
              new AuditWhitelistManager(),
+             AuditAdapter.getInstance(),
              DatabaseDescriptor.getAuthenticator());
     }
 
     @VisibleForTesting
-    AuditRoleManager(IRoleManager wrappedRoleManager, AuditWhitelistManager whitelistManager, IAuthenticator authenticator)
+    AuditRoleManager(IRoleManager wrappedRoleManager, AuditWhitelistManager whitelistManager, AuditAdapter auditAdapter, IAuthenticator authenticator)
     {
         LOG.info("Auditing enabled on role manager");
 
         this.wrappedRoleManager = wrappedRoleManager;
         this.whitelistManager = whitelistManager;
+        this.auditAdapter = auditAdapter;
         permissionChecker = new PermissionChecker();
         supportedOptions = resolveSupportedOptions(authenticator);
         alterableOptions = resolveAlterableOptions(authenticator);
@@ -134,6 +138,7 @@ public void setup()
     {
         wrappedRoleManager.setup();
         whitelistManager.setup();
+        auditAdapter.setup();
     }
 
     @Override

ecaudit/src/main/java/com/ericsson/bss/cassandra/ecaudit/auth/DecoratedPasswordAuthenticator.java

@@ -22,8 +22,6 @@
 import java.util.Set;
 
 import com.google.common.annotations.VisibleForTesting;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.auth.AuthenticatedUser;
 import org.apache.cassandra.auth.IResource;
@@ -37,8 +35,6 @@
  */
 public class DecoratedPasswordAuthenticator implements IDecoratedAuthenticator
 {
-    private static final Logger LOG = LoggerFactory.getLogger(DecoratedPasswordAuthenticator.class);
-
     private final PasswordAuthenticator wrappedAuthenticator;
 
     /**
@@ -54,7 +50,6 @@ public DecoratedPasswordAuthenticator()
     @VisibleForTesting
     DecoratedPasswordAuthenticator(PasswordAuthenticator authenticator)
     {
-        LOG.info("Auditing enabled on authenticator");
         this.wrappedAuthenticator = authenticator;
     }
 
@@ -103,7 +98,6 @@ public SaslNegotiator newSaslNegotiator()
     @Override
     public DecoratedSaslNegotiator newDecoratedSaslNegotiator()
     {
-        LOG.debug("Setting up SASL negotiation with client peer");
         return new DecoratedPlainTextSaslNegotiator(wrappedAuthenticator.newSaslNegotiator());
     }
 

ecaudit/src/main/java/com/ericsson/bss/cassandra/ecaudit/config/AuditYamlConfig.java

@@ -23,6 +23,7 @@
 import com.ericsson.bss.cassandra.ecaudit.auth.DecoratedPasswordAuthenticator;
 import com.ericsson.bss.cassandra.ecaudit.entry.suppressor.SuppressNothing;
 import com.ericsson.bss.cassandra.ecaudit.logger.Slf4jAuditLogger;
+import org.apache.cassandra.auth.CassandraAuthorizer;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.config.ParameterizedClass;
 
@@ -34,7 +35,7 @@ public final class AuditYamlConfig
 {
     private static final List<String> DEFAULT_WHITELIST = Collections.emptyList();
     private static final ParameterizedClass DEFAULT_LOGGER_BACKEND = new ParameterizedClass(Slf4jAuditLogger.class.getCanonicalName(), Collections.emptyMap());
-    private static final String DEFAULT_WRAPPED_AUTHORIZER = "org.apache.cassandra.auth.CassandraAuthorizer";
+    private static final String DEFAULT_WRAPPED_AUTHORIZER = CassandraAuthorizer.class.getName();
     private static final String DEFAULT_WRAPPED_AUTHENTICATOR = DecoratedPasswordAuthenticator.class.getName();
     private static final String DEFAULT_BOUND_VALUE_SUPPRESSOR = SuppressNothing.class.getName();
 

ecaudit/src/main/java/com/ericsson/bss/cassandra/ecaudit/handler/AuditQueryHandler.java

@@ -43,7 +43,7 @@
 
 /**
  * An implementation of {@link QueryHandler} that performs audit logging on queries.
- * <p>
+ *
  * It can be used as a stand-alone query handler, or wrapped inside another query handler if configuration is needed.
  */
 public class AuditQueryHandler implements QueryHandler
@@ -67,7 +67,7 @@ public AuditQueryHandler()
     /**
      * Creates an instance of {@link AuditQueryHandler} that uses the default audit logger configuration and wraps the
      * given query handler.
-     * <p>
+     *
      * The signature of this constructor must remain unchanged as it is used by other frameworks to create layers of
      * decorators on top of each other.
      *