Allow setting up per-cluster TLS connections

src/packaging/bin/spreaper

@@ -189,6 +189,9 @@ def _arguments_for_add_cluster(parser):
     parser.add_argument("jmx_port", help="the JMX port of the Cassandra cluster to be registered", default="7199")
     parser.add_argument("--jmx-username", help="JMX username in case authentication is activated", default=None)
     parser.add_argument("--jmx-password", help="JMX password in case authentication is activated", default=None)
+    parser.add_argument("--truststore-name",
+                        help="Name of the folder in Reaper's truststoreDir to load certs from",
+                        default=None)
 
 
 def _argument_owner(parser):
@@ -701,7 +704,8 @@ class ReaperCLI(object):
         payload = {'seedHost': args.seed_host,
                    'jmxPort': args.jmx_port,
                    'jmxUsername': args.jmx_username,
-                   'jmxPassword': args.jmx_password}
+                   'jmxPassword': args.jmx_password,
+                   'truststoreName': args.truststore_name}
         cluster_data = reaper.postFormData("cluster/auth", payload=payload)
         printq("# Registration succeeded:")
         print(json.dumps(json.loads(cluster_data), indent=2, sort_keys=True))

src/server/src/main/docker/Dockerfile

@@ -84,6 +84,7 @@ ENV REAPER_SEGMENT_COUNT_PER_NODE=64 \
     REAPER_HTTP_MANAGEMENT_ENABLE="false" \
     REAPER_HTTP_MANAGEMENT_KEYSTORE_PATH="" \
     REAPER_HTTP_MANAGEMENT_TRUSTSTORE_PATH="" \
+    REAPER_HTTP_MANAGEMENT_TRUSTSTORES_DIR="" \
     REAPER_TMP_DIRECTORY="/var/tmp/cassandra-reaper" \
     REAPER_MEMORY_STORAGE_DIRECTORY="/var/lib/cassandra-reaper/storage"
 

src/server/src/main/docker/cassandra-reaper.yml

@@ -86,4 +86,5 @@ httpManagement:
   mgmtApiMetricsPort: ${REAPER_MGMT_API_METRICS_PORT}
   keystore: ${REAPER_HTTP_MANAGEMENT_KEYSTORE_PATH}
   truststore: ${REAPER_HTTP_MANAGEMENT_TRUSTSTORE_PATH}
+  truststoreDir: ${REAPER_HTTP_MANAGEMENT_TRUSTSTORES_DIR}
 

src/server/src/main/java/io/cassandrareaper/ReaperApplication.java

@@ -342,7 +342,7 @@ private void maybeInitializeSidecarMode(ClusterResource addClusterResource) thro
   private boolean selfRegisterClusterForSidecar(ClusterResource addClusterResource, String seedHost)
       throws ReaperException {
     final Optional<Cluster> cluster = addClusterResource.findClusterWithSeedHost(seedHost, Optional.empty(),
-        Optional.empty());
+        Optional.empty(), Optional.empty());
     if (!cluster.isPresent()) {
       return false;
     }

src/server/src/main/java/io/cassandrareaper/ReaperApplicationConfiguration.java

@@ -753,6 +753,9 @@ public static final class HttpManagement {
     @JsonProperty
     private String truststore;
 
+    @JsonProperty
+    private String truststoreDir;
+
     @JsonProperty
     private Integer mgmtApiMetricsPort;
 
@@ -772,6 +775,10 @@ public String getTruststore() {
       return truststore;
     }
 
+    public String getTruststoreDir() {
+      return truststoreDir;
+    }
+
     @VisibleForTesting
     public void setEnabled(Boolean enabled) {
       this.enabled = enabled;
@@ -787,6 +794,11 @@ public void setTruststore(String truststore) {
       this.truststore = truststore;
     }
 
+    @VisibleForTesting
+    public void setTruststoreDir(String truststoreDir) {
+      this.truststoreDir = truststoreDir;
+    }
+
     public int getMgmtApiMetricsPort() {
       return mgmtApiMetricsPort == null ? DEFAULT_MGMT_API_METRICS_PORT : mgmtApiMetricsPort;
     }

src/server/src/main/java/io/cassandrareaper/core/Cluster.java

@@ -196,6 +196,12 @@ public Builder withJmxCredentials(JmxCredentials jmxCredentials) {
       return this;
     }
 
+    public Builder withTruststoreName(String truststoreName) {
+      Preconditions.checkNotNull(truststoreName);
+      this.properties.withTruststoreName(truststoreName);
+      return this;
+    }
+
     public Cluster build() {
       Preconditions.checkNotNull(name);
       Preconditions.checkNotNull(seedHosts);

src/server/src/main/java/io/cassandrareaper/core/ClusterProperties.java

@@ -25,9 +25,12 @@ public final class ClusterProperties {
   private final int jmxPort;
   private final JmxCredentials jmxCredentials;
 
+  private final String trustStoreName;
+
   private ClusterProperties(Builder builder) {
     this.jmxPort = builder.jmxPort;
     this.jmxCredentials = builder.jmxCredentials;
+    this.trustStoreName = builder.truststoreName;
   }
 
   public int getJmxPort() {
@@ -38,6 +41,10 @@ public JmxCredentials getJmxCredentials() {
     return jmxCredentials;
   }
 
+  public String getTruststoreName() {
+    return trustStoreName;
+  }
+
   public static Builder builder() {
     return new Builder();
   }
@@ -47,6 +54,8 @@ public static final class Builder {
     private int jmxPort;
     private JmxCredentials jmxCredentials;
 
+    private String truststoreName;
+
     private Builder() {}
 
     public Builder withJmxPort(int jmxPort) {
@@ -59,6 +68,11 @@ public Builder withJmxCredentials(JmxCredentials jmxCredentials) {
       return this;
     }
 
+    public Builder withTruststoreName(String truststoreName) {
+      this.truststoreName = truststoreName;
+      return this;
+    }
+
     public ClusterProperties build() {
       return new ClusterProperties(this);
     }

src/server/src/main/java/io/cassandrareaper/management/ClusterFacade.java

@@ -893,12 +893,10 @@ public ICassandraManagementProxy connect(Node node, Collection<String> endpoints
   private ICassandraManagementProxy connectImpl(Cluster cluster, Collection<String> endpoints)
           throws ReaperException {
     try {
-      ICassandraManagementProxy proxy = context.managementConnectionFactory
-              .connectAny(
-          endpoints
-              .stream()
-              .map(host -> Node.builder().withCluster(cluster).withHostname(host).build())
-              .collect(Collectors.toList()));
+      ICassandraManagementProxy proxy = context.managementConnectionFactory.connectAny(endpoints.stream()
+          .map(host -> Node.builder().withCluster(cluster).withHostname(host).build())
+          .collect(Collectors.toList())
+      );
 
       Async.markClusterActive(cluster, context);
       return proxy;

src/server/src/main/java/io/cassandrareaper/management/http/HttpManagementConnectionFactory.java

@@ -48,6 +48,7 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
+import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ExecutorService;
@@ -77,6 +78,11 @@
 
 public class HttpManagementConnectionFactory implements IManagementConnectionFactory {
   private static final char[] KEYSTORE_PASSWORD = "changeit".toCharArray();
+
+  private static final String KEYSTORE_COMPONENT_NAME = "keystore.jks";
+
+  private static final String TRUSTSTORE_COMPONENT_NAME = "truststore.jks";
+
   private static final Logger LOG = LoggerFactory.getLogger(HttpManagementConnectionFactory.class);
   private static final ConcurrentMap<String, HttpCassandraManagementProxy> HTTP_CONNECTIONS = Maps.newConcurrentMap();
   private final MetricRegistry metricRegistry;
@@ -95,13 +101,18 @@ public HttpManagementConnectionFactory(AppContext context, ScheduledExecutorServ
     this.config = context.config;
     registerConnectionsGauge();
     this.jobStatusPollerExecutor = jobStatusPollerExecutor;
-    if (context.config.getHttpManagement().getKeystore() != null && !context.config.getHttpManagement().getKeystore()
-        .isEmpty()) {
-      try {
-        createSslWatcher();
-      } catch (IOException e) {
-        throw new RuntimeException(e);
-      }
+
+    String ts = context.config.getHttpManagement().getTruststore();
+    boolean watchTruststore = ts != null && !ts.isEmpty();
+    String ks = context.config.getHttpManagement().getKeystore();
+    boolean watchKeystore = ks != null && !ks.isEmpty();
+    String tsd = context.config.getHttpManagement().getTruststoreDir();
+    boolean watchTruststoreDir = tsd != null && !tsd.isEmpty();
+
+    try {
+      createSslWatcher(watchTruststore, watchKeystore, watchTruststoreDir);
+    } catch (IOException e) {
+      throw new RuntimeException(e);
     }
   }
 
@@ -175,14 +186,18 @@ public HttpCassandraManagementProxy apply(@Nullable String hostName) {
         OkHttpClient.Builder clientBuilder = new OkHttpClient().newBuilder();
 
         String protocol = "http";
+
+        Path truststoreName = getTruststoreComponentPath(node, TRUSTSTORE_COMPONENT_NAME);
+        Path keystoreName = getTruststoreComponentPath(node, KEYSTORE_COMPONENT_NAME);
+
         if (useMtls) {
           LOG.debug("Using TLS connection to " + node.getHostname());
           // We have to split TrustManagers to its own function to please OkHttpClient
           TrustManager[] trustManagers;
           SSLContext sslContext;
           try {
-            trustManagers = getTrustManagers();
-            sslContext = createSslContext(trustManagers);
+            trustManagers = getTrustManagers(truststoreName);
+            sslContext = createSslContext(trustManagers, keystoreName);
           } catch (ReaperException e) {
             LOG.error("Failed to create SSLContext: " + e.getLocalizedMessage(), e);
             throw new RuntimeException(e);
@@ -218,8 +233,7 @@ public HttpCassandraManagementProxy apply(@Nullable String hostName) {
   }
 
   @VisibleForTesting
-  SSLContext createSslContext(TrustManager[] tms) throws ReaperException {
-    Path keyStorePath = Paths.get(config.getHttpManagement().getKeystore());
+  SSLContext createSslContext(TrustManager[] tms, Path keyStorePath) throws ReaperException {
 
     try (InputStream ksIs = Files.newInputStream(keyStorePath, StandardOpenOption.READ)) {
 
@@ -238,8 +252,11 @@ SSLContext createSslContext(TrustManager[] tms) throws ReaperException {
     }
   }
 
-  private TrustManager[] getTrustManagers() throws ReaperException {
-    Path trustStorePath = Paths.get(config.getHttpManagement().getTruststore());
+  @VisibleForTesting
+  TrustManager[] getTrustManagers(Path trustStorePath) throws ReaperException {
+
+    LOG.trace(String.format("Calling getSingleTrustManager with %s", trustStorePath));
+
     try (InputStream tsIs = Files.newInputStream(trustStorePath, StandardOpenOption.READ)) {
       KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType());
       trustStore.load(tsIs, KEYSTORE_PASSWORD);
@@ -249,30 +266,68 @@ private TrustManager[] getTrustManagers() throws ReaperException {
 
       return tmf.getTrustManagers();
     } catch (IOException | NoSuchAlgorithmException | KeyStoreException | CertificateException e) {
-      throw new ReaperException(e);
+      throw new ReaperException("Error loading trust managers");
     }
   }
 
   @VisibleForTesting
-  void createSslWatcher() throws IOException {
+  Path getTruststoreComponentPath(Node node, String truststoreComponentName) {
+    Path trustStorePath;
+
+    // using a .map() instead of .get() to nicely handle absence
+    Optional<String> truststoreName = node.getCluster().map(c -> c.getProperties().getTruststoreName());
+
+    if (truststoreName.isPresent()) {
+      // load a cluster-specific trust store which is a sub-dir of dir pointed to by truststoreDir
+      Path storesRootPath = Paths.get(config.getHttpManagement().getTruststoreDir());
+      trustStorePath = storesRootPath
+          .resolve(truststoreName.get())
+          .resolve(truststoreComponentName)
+          .toAbsolutePath();
+    } else {
+      // load the generic trust store from truststore/keystore options
+      trustStorePath = truststoreComponentName.equals(TRUSTSTORE_COMPONENT_NAME)
+          ? Paths.get(config.getHttpManagement().getTruststore()).toAbsolutePath()
+          : Paths.get(config.getHttpManagement().getKeystore()).toAbsolutePath();
+    }
+
+    return trustStorePath;
+  }
+
+  @VisibleForTesting
+  void createSslWatcher(boolean watchTruststore, boolean watchKeystore, boolean watchTruststoreDir) throws IOException {
+
     WatchService watchService = FileSystems.getDefault().newWatchService();
-    Path trustStorePath = Paths.get(config.getHttpManagement().getTruststore());
-    Path keyStorePath = Paths.get(config.getHttpManagement().getKeystore());
-    Path keystoreParent = trustStorePath.getParent();
-    Path trustStoreParent = keyStorePath.getParent();
-
-    keystoreParent.register(
-        watchService,
-        StandardWatchEventKinds.ENTRY_CREATE,
-        StandardWatchEventKinds.ENTRY_DELETE,
-        StandardWatchEventKinds.ENTRY_MODIFY);
-
-    if (!keystoreParent.equals(trustStoreParent)) {
-      trustStoreParent.register(
+
+    Path trustStorePath = watchTruststore ? Paths.get(config.getHttpManagement().getTruststore()) : null;
+    Path keyStorePath = watchKeystore ? Paths.get(config.getHttpManagement().getKeystore()) : null ;
+    Path truststoreDirPath = watchTruststoreDir ? Paths.get(config.getHttpManagement().getTruststoreDir()) : null ;
+
+    if (watchKeystore) {
+      keyStorePath.getParent().register(
           watchService,
           StandardWatchEventKinds.ENTRY_CREATE,
           StandardWatchEventKinds.ENTRY_DELETE,
-          StandardWatchEventKinds.ENTRY_MODIFY);
+          StandardWatchEventKinds.ENTRY_MODIFY
+      );
+    }
+    if (watchTruststore && watchKeystore) {
+      if (!trustStorePath.getParent().equals(keyStorePath.getParent())) {
+        trustStorePath.getParent().register(
+            watchService,
+            StandardWatchEventKinds.ENTRY_CREATE,
+            StandardWatchEventKinds.ENTRY_DELETE,
+            StandardWatchEventKinds.ENTRY_MODIFY
+        );
+      }
+    }
+    if (watchTruststoreDir) {
+      truststoreDirPath.register(
+          watchService,
+          StandardWatchEventKinds.ENTRY_CREATE,
+          StandardWatchEventKinds.ENTRY_DELETE,
+          StandardWatchEventKinds.ENTRY_MODIFY
+      );
     }
 
     ExecutorService executorService = Executors.newSingleThreadExecutor();
@@ -290,11 +345,23 @@ void createSslWatcher() throws IOException {
                 WatchEvent<java.nio.file.Path> ev = (WatchEvent<Path>) event;
                 Path eventFilename = ev.context();
 
-                if (keystoreParent.resolve(eventFilename).equals(keyStorePath)
-                    || trustStoreParent.resolve(eventFilename).equals(trustStorePath)) {
-                  // Something in the TLS has been modified.. recreate HTTP connections
-                  reloadNeeded = true;
+                if (watchKeystore) {
+                  if (keyStorePath.getParent().resolve(eventFilename).equals(keyStorePath)) {
+                    reloadNeeded = true;
+                  }
                 }
+                if (watchTruststore) {
+                  if (trustStorePath.getParent().resolve(eventFilename).equals(trustStorePath)) {
+                    // Something in the TLS has been modified.. recreate HTTP connections
+                    reloadNeeded = true;
+                  }
+                }
+                if (watchTruststoreDir) {
+                  if (Files.exists(truststoreDirPath.resolve(eventFilename))) {
+                    reloadNeeded = true;
+                  }
+                }
+
               }
               if (!key.reset()) {
                 // The watched directories have disappeared..