diff --git a/CHANGELOG.md b/CHANGELOG.md index cf0694ffe..19b234da3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ ## Change Log +### 3.6.1 (2024/06/11 07:14 +00:00) +- [#1507](https://github.com/thelastpickle/cassandra-reaper/pull/1507) Allow mounting a volume to host the config and enable read only root FS (#1507) (@adejanovski) + ### 3.6.0 (2024/04/18 08:53 +00:00) - [#1500](https://github.com/thelastpickle/cassandra-reaper/pull/1500) Upgrade snakeyaml (#1500) (@emerkle826) - [#1499](https://github.com/thelastpickle/cassandra-reaper/pull/1499) Update Dockerfile to specify restricted Java temp directory (#1499) (@emerkle826) diff --git a/pom.xml b/pom.xml index f5fac7cee..415a54f12 100644 --- a/pom.xml +++ b/pom.xml @@ -309,7 +309,7 @@ com.puppycrawl.tools checkstyle - 8.19 + 10.17.0 diff --git a/src/docs/content/docs/api.md b/src/docs/content/docs/api.md index 96c698d10..93f0b4d1c 100644 --- a/src/docs/content/docs/api.md +++ b/src/docs/content/docs/api.md @@ -196,7 +196,8 @@ Returns OK if all goes well NOT_MODIFIED if new state is the same as the old one * *segmentCountPerNode*: Defines the amount of segments per node to create for scheduled repair runs. (Optional) * *repairParallelism*: Defines the used repair parallelism for scheduled repair runs. (Optional) * *intensity*: Defines the repair intensity for scheduled repair runs. (Optional) - * *incrementalRepair*: Defines if incremental repair should be done. [true/false] (Optional) + * *incrementalRepair*: Defines if incremental repair should be done on all tokens of each node at once. [true/false] (Optional) + * *subrangeIncrementalRepair*: Defines if incremental repair should be done in subrange mode, against discrete token ranges. [true/false] (Optional) * *scheduleDaysBetween*: Defines the amount of days to wait between scheduling new repairs. For example, use value 7 for weekly schedule, and 0 for continuous. * *scheduleTriggerTime*: Defines the time for first scheduled trigger for the run. diff --git a/src/docs/content/docs/configuration/docker_vars.md b/src/docs/content/docs/configuration/docker_vars.md index e81d0fea2..cc2ef5161 100644 --- a/src/docs/content/docs/configuration/docker_vars.md +++ b/src/docs/content/docs/configuration/docker_vars.md @@ -27,6 +27,7 @@ The Docker environment variables listed in this section map directly to Reaper s REAPER_ENABLE_DYNAMIC_SEED_LIST | [enableDynamicSeedList]({{< relref "reaper_specific.md#enabledynamicseedlist" >}}) | true REAPER_HANGING_REPAIR_TIMEOUT_MINS | [hangingRepairTimeoutMins]({{< relref "reaper_specific.md#hangingrepairtimeoutmins" >}}) | 30 REAPER_INCREMENTAL_REPAIR | [incrementalRepair]({{< relref "reaper_specific.md#incrementalrepair" >}}) | false +REAPER_SUBRANGE_INCREMENTAL | [subrangeIncrementalRepair]({{< relref "reaper_specific.md#subrangeincremental" >}}) | false REAPER_JMX_AUTH_PASSWORD | [password]({{< relref "reaper_specific.md#password" >}}) | REAPER_JMX_AUTH_USERNAME | [username]({{< relref "reaper_specific.md#username" >}}) | REAPER_JMX_CREDENTIALS | [jmxCredentials]({{< relref "reaper_specific.md#jmxcredentials" >}}) | diff --git a/src/docs/content/docs/configuration/reaper_specific.md b/src/docs/content/docs/configuration/reaper_specific.md index 5a50893b8..f9e227ec2 100644 --- a/src/docs/content/docs/configuration/reaper_specific.md +++ b/src/docs/content/docs/configuration/reaper_specific.md @@ -154,6 +154,21 @@ Sets the default repair type unless specifically defined for each run. Note that
+### `subrangeIncrementalRepair` + +Type: *Boolean* + +Default: *false* + +Sets the default repair type unless specifically defined for each run. Note that this is only supported with the PARALLEL repairParallelism setting. For more details in incremental repair, please refer to the following article.http://www.datastax.com/dev/blog/more-efficient-repairs. +This mode will split the repair jobs into sets of token ranges using the incremental mode. +This will prevail over the `incrementalRepair` setting. + + +*Note*: Subrange incremental repair is only available since Cassandra 4.0. + +
+ ### `blacklistTwcsTables` Type: *Boolean* diff --git a/src/packaging/bin/spreaper b/src/packaging/bin/spreaper index 4a1a92475..8d1bebad9 100755 --- a/src/packaging/bin/spreaper +++ b/src/packaging/bin/spreaper @@ -233,6 +233,9 @@ def _arguments_for_repair_and_schedule(parser): parser.add_argument("--incremental", default="false", help=("Incremental repair (true or false), " "or use the configured default if not given (false)")) + parser.add_argument("--subrange-incremental", default="false", + help=("Subrange incremental repair (true or false), " + "or use the configured default if not given (false)")) parser.add_argument("--datacenters", default=None, help=("a comma separated list of datacenters to repair (do not use spaces after commas). " "Cannot be used in conjunction with --nodes.")) @@ -734,6 +737,7 @@ class ReaperCLI(object): repairParallelism=args.repair_parallelism, intensity=args.intensity, incrementalRepair=args.incremental, + subrangeIncrementalRepair=args.subrange_incremental, nodes=args.nodes, datacenters=args.datacenters, blacklistedTables=args.blacklisted_tables, @@ -755,6 +759,7 @@ class ReaperCLI(object): repairParallelism=args.repair_parallelism, intensity=args.intensity, incrementalRepair=args.incremental, + subrangeIncrementalRepair=args.subrange_incremental, nodes=args.nodes, datacenters=args.datacenters, blacklistedTables=args.blacklisted_tables, @@ -797,6 +802,7 @@ class ReaperCLI(object): scheduleDaysBetween=args.schedule_days_between, scheduleTriggerTime=args.schedule_trigger_time, incrementalRepair=args.incremental, + subrangeIncrementalRepair=args.subrange_incremental, nodes=args.nodes, datacenters=args.datacenters, blacklistedTables=args.blacklisted_tables, @@ -818,6 +824,7 @@ class ReaperCLI(object): scheduleDaysBetween=args.schedule_days_between, scheduleTriggerTime=args.schedule_trigger_time, incrementalRepair=args.incremental, + subrangeIncrementalRepair=args.subrange_incremental, nodes=args.nodes, datacenters=args.datacenters, blacklistedTables=args.blacklisted_tables, diff --git a/src/packaging/docker-services/reaper/reaper.env b/src/packaging/docker-services/reaper/reaper.env index 48e399172..a3bf203bc 100644 --- a/src/packaging/docker-services/reaper/reaper.env +++ b/src/packaging/docker-services/reaper/reaper.env @@ -35,6 +35,7 @@ REAPER_REPAIR_RUN_THREADS=32 REAPER_HANGING_REPAIR_TIMEOUT_MINS=90 REAPER_ENABLE_CROSS_ORIGIN=true REAPER_INCREMENTAL_REPAIR=true +REAPER_SUBRANGE_INCREMENTAL=false REAPER_BLACKLIST_TWCS=true REAPER_ENABLE_DYNAMIC_SEED_LIST=false REAPER_REPAIR_MANAGER_SCHEDULING_INTERVAL_SECONDS=10 diff --git a/src/server/checkstyle.xml b/src/server/checkstyle.xml index b709bb744..9c08941a0 100644 --- a/src/server/checkstyle.xml +++ b/src/server/checkstyle.xml @@ -14,30 +14,11 @@ limitations under the License. --> - - - - - + "-//Checkstyle//DTD Checkstyle Configuration 1.3//EN" + "https://checkstyle.org/dtds/configuration_1_3.dtd"> + - - - @@ -48,9 +29,12 @@ - - + + + + + @@ -70,10 +54,6 @@ - - - - @@ -234,6 +214,6 @@ - + diff --git a/src/server/src/main/docker/Dockerfile b/src/server/src/main/docker/Dockerfile index e9ea29a18..11df35ef3 100644 --- a/src/server/src/main/docker/Dockerfile +++ b/src/server/src/main/docker/Dockerfile @@ -27,6 +27,7 @@ ENV REAPER_SEGMENT_COUNT_PER_NODE=64 \ REAPER_STORAGE_TYPE=memory \ REAPER_ENABLE_CROSS_ORIGIN=true \ REAPER_INCREMENTAL_REPAIR=false \ + REAPER_SUBRANGE_INCREMENTAL=false \ REAPER_BLACKLIST_TWCS=false \ REAPER_ENABLE_DYNAMIC_SEED_LIST=true \ REAPER_REPAIR_MANAGER_SCHEDULING_INTERVAL_SECONDS=30 \ @@ -106,15 +107,10 @@ ADD ${SHADED_JAR} /usr/local/lib/cassandra-reaper.jar ADD src/packaging/bin/spreaper /usr/local/bin/spreaper -# get around `/usr/local/bin/configure-persistence.sh: line 65: can't create /etc/cassandra-reaper/cassandra-reaper.yml: Interrupted system call` unknown error -RUN touch /etc/cassandra-reaper/cassandra-reaper.yml -# get around `/usr/local/bin/configure-webui-authentication.sh: line 44: can't `create` /etc/cassandra-reaper/shiro.ini: Interrupted system call` unknown error -RUN touch /etc/cassandra-reaper/shiro.ini - - RUN addgroup -g 1001 -S reaper && adduser -G reaper -S -u 1001 reaper && \ mkdir -p /var/lib/cassandra-reaper && \ mkdir -p /etc/cassandra-reaper/shiro && \ + mkdir -p /etc/cassandra-reaper/config && \ mkdir -p /var/log/cassandra-reaper && \ mkdir -p ${REAPER_TMP_DIRECTORY} && \ chown reaper:reaper \ @@ -131,6 +127,7 @@ RUN addgroup -g 1001 -S reaper && adduser -G reaper -S -u 1001 reaper && \ chown -R reaper:reaper \ /var/lib/cassandra-reaper \ /etc/cassandra-reaper/shiro \ + /etc/cassandra-reaper/config \ /var/log/cassandra-reaper && \ chmod +x \ /usr/local/bin/entrypoint.sh \ @@ -142,6 +139,7 @@ RUN addgroup -g 1001 -S reaper && adduser -G reaper -S -u 1001 reaper && \ VOLUME /var/lib/cassandra-reaper VOLUME /etc/cassandra-reaper/shiro +VOLUME /etc/cassandra-reaper/config USER reaper diff --git a/src/server/src/main/docker/cassandra-reaper.yml b/src/server/src/main/docker/cassandra-reaper.yml index 72f12025e..5836e445c 100644 --- a/src/server/src/main/docker/cassandra-reaper.yml +++ b/src/server/src/main/docker/cassandra-reaper.yml @@ -23,6 +23,7 @@ hangingRepairTimeoutMins: ${REAPER_HANGING_REPAIR_TIMEOUT_MINS} storageType: ${REAPER_STORAGE_TYPE} enableCrossOrigin: ${REAPER_ENABLE_CROSS_ORIGIN} incrementalRepair: ${REAPER_INCREMENTAL_REPAIR} +subrangeIncrementalRepair: ${REAPER_SUBRANGE_INCREMENTAL} blacklistTwcsTables: ${REAPER_BLACKLIST_TWCS} enableDynamicSeedList: ${REAPER_ENABLE_DYNAMIC_SEED_LIST} repairManagerSchedulingIntervalSeconds: ${REAPER_REPAIR_MANAGER_SCHEDULING_INTERVAL_SECONDS} diff --git a/src/server/src/main/docker/configure-jmx-credentials.sh b/src/server/src/main/docker/configure-jmx-credentials.sh index e07d476bb..8da8aa30a 100644 --- a/src/server/src/main/docker/configure-jmx-credentials.sh +++ b/src/server/src/main/docker/configure-jmx-credentials.sh @@ -17,7 +17,7 @@ # we expect the jmx credentials to be a comma-separated list of 'user:password@cluster' entries if [ ! -z "${REAPER_JMX_CREDENTIALS}" ]; then -cat <> /etc/cassandra-reaper/cassandra-reaper.yml +cat <> /etc/cassandra-reaper/config/cassandra-reaper.yml jmxCredentials: EOT @@ -29,7 +29,7 @@ EOT PASSWORD=$(echo "${ENTRY}" | cut -d'@' -f1 | cut -d':' -f2 | sed 's/"/\\"/g') # finally, write out the YAML entries -cat <> /etc/cassandra-reaper/cassandra-reaper.yml +cat <> /etc/cassandra-reaper/config/cassandra-reaper.yml ${CLUSTER}: username: "${USERNAME}" password: "${PASSWORD}" @@ -41,7 +41,7 @@ fi if [ ! -z "${REAPER_JMX_AUTH_USERNAME}" ]; then -cat <> /etc/cassandra-reaper/cassandra-reaper.yml +cat <> /etc/cassandra-reaper/config/cassandra-reaper.yml jmxAuth: username: "$(echo "${REAPER_JMX_AUTH_USERNAME}" | sed 's/"/\\"/g')" password: "$(echo "${REAPER_JMX_AUTH_PASSWORD}" | sed 's/"/\\"/g')" @@ -50,7 +50,7 @@ EOT fi if [ ! -z "${CRYPTO_SYSTEM_PROPERTY_SECRET}" ]; then -cat <> /etc/cassandra-reaper/cassandra-reaper.yml +cat <> /etc/cassandra-reaper/config/cassandra-reaper.yml cryptograph: type: symmetric systemPropertySecret: ${CRYPTO_SYSTEM_PROPERTY_SECRET} diff --git a/src/server/src/main/docker/configure-metrics.sh b/src/server/src/main/docker/configure-metrics.sh index d2a8a56c0..d2b783676 100644 --- a/src/server/src/main/docker/configure-metrics.sh +++ b/src/server/src/main/docker/configure-metrics.sh @@ -15,7 +15,7 @@ # limitations under the License. if [ "true" = "${REAPER_METRICS_ENABLED}" ]; then -cat <> /etc/cassandra-reaper/cassandra-reaper.yml +cat <> /etc/cassandra-reaper/config/cassandra-reaper.yml metrics: frequency: ${REAPER_METRICS_FREQUENCY} reporters: ${REAPER_METRICS_REPORTERS} diff --git a/src/server/src/main/docker/configure-persistence.sh b/src/server/src/main/docker/configure-persistence.sh index b70ff4bec..c47450b79 100644 --- a/src/server/src/main/docker/configure-persistence.sh +++ b/src/server/src/main/docker/configure-persistence.sh @@ -16,20 +16,20 @@ # Add specific jmxAddressTranslator if [ ! -z "${JMX_ADDRESS_TRANSLATOR_TYPE}" ]; then -cat <> /etc/cassandra-reaper/cassandra-reaper.yml +cat <> /etc/cassandra-reaper/config/cassandra-reaper.yml jmxAddressTranslator: type: ${JMX_ADDRESS_TRANSLATOR_TYPE} EOT fi if [ "multiIpPerNode" = "${JMX_ADDRESS_TRANSLATOR_TYPE}" ] && [ -n "$JMX_ADDRESS_TRANSLATOR_MAPPING" ]; then -cat <> /etc/cassandra-reaper/cassandra-reaper.yml +cat <> /etc/cassandra-reaper/config/cassandra-reaper.yml ipTranslations: EOT IFS=',' read -ra mappings <<< "$JMX_ADDRESS_TRANSLATOR_MAPPING" for mapping in "${mappings[@]}"; do IFS=':' read -ra mapping <<< "$mapping" -cat <> /etc/cassandra-reaper/cassandra-reaper.yml +cat <> /etc/cassandra-reaper/config/cassandra-reaper.yml - from: "${mapping[0]}" to: "${mapping[1]}" EOT @@ -40,7 +40,7 @@ case ${REAPER_STORAGE_TYPE} in "cassandra") # BEGIN cassandra persistence options -cat <> /etc/cassandra-reaper/cassandra-reaper.yml +cat <> /etc/cassandra-reaper/config/cassandra-reaper.yml activateQueryLogger: ${REAPER_CASS_ACTIVATE_QUERY_LOGGER} cassandra: @@ -59,7 +59,7 @@ cassandra: EOT if [ "true" = "${REAPER_CASS_AUTH_ENABLED}" ]; then -cat <> /etc/cassandra-reaper/cassandra-reaper.yml +cat <> /etc/cassandra-reaper/config/cassandra-reaper.yml authProvider: type: plainText username: "$(echo "${REAPER_CASS_AUTH_USERNAME}" | sed 's/"/\\"/g')" @@ -68,27 +68,27 @@ EOT fi if [ "true" = "${REAPER_CASS_NATIVE_PROTOCOL_SSL_ENCRYPTION_ENABLED}" ]; then -cat <> /etc/cassandra-reaper/cassandra-reaper.yml +cat <> /etc/cassandra-reaper/config/cassandra-reaper.yml ssl: type: jdk EOT fi if [ "true" = "${REAPER_CASS_ADDRESS_TRANSLATOR_ENABLED}" ]; then -cat <> /etc/cassandra-reaper/cassandra-reaper.yml +cat <> /etc/cassandra-reaper/config/cassandra-reaper.yml addressTranslator: type: ${REAPER_CASS_ADDRESS_TRANSLATOR_TYPE} EOT fi if [ "multiIpPerNode" = "${REAPER_CASS_ADDRESS_TRANSLATOR_TYPE}" ] && [ -n "$REAPER_CASS_ADDRESS_TRANSLATOR_MAPPING" ]; then -cat <> /etc/cassandra-reaper/cassandra-reaper.yml +cat <> /etc/cassandra-reaper/config/cassandra-reaper.yml ipTranslations: EOT IFS=',' read -ra mappings <<< "$REAPER_CASS_ADDRESS_TRANSLATOR_MAPPING" for mapping in "${mappings[@]}"; do IFS=':' read -ra mapping <<< "$mapping" -cat <> /etc/cassandra-reaper/cassandra-reaper.yml +cat <> /etc/cassandra-reaper/config/cassandra-reaper.yml - from: "${mapping[0]}" to: "${mapping[1]}" EOT @@ -100,7 +100,7 @@ fi ;; "memory") # BEGIN cassandra persistence options -cat <> /etc/cassandra-reaper/cassandra-reaper.yml +cat <> /etc/cassandra-reaper/config/cassandra-reaper.yml persistenceStoragePath: ${REAPER_MEMORY_STORAGE_DIRECTORY} EOT diff --git a/src/server/src/main/docker/configure-webui-authentication.sh b/src/server/src/main/docker/configure-webui-authentication.sh index 48a38ea38..b3c5780d4 100644 --- a/src/server/src/main/docker/configure-webui-authentication.sh +++ b/src/server/src/main/docker/configure-webui-authentication.sh @@ -18,21 +18,21 @@ if [ "false" = "${REAPER_AUTH_ENABLED}" ]; then fi if [ ! -z "${REAPER_SHIRO_INI}" ]; then -cat <> /etc/cassandra-reaper/cassandra-reaper.yml +cat <> /etc/cassandra-reaper/config/cassandra-reaper.yml accessControl: sessionTimeout: PT10M shiro: iniConfigs: ["file:${REAPER_SHIRO_INI}"] EOT elif [ ! -z "${REAPER_AUTH_USER}" ]; then -cat <> /etc/cassandra-reaper/cassandra-reaper.yml +cat <> /etc/cassandra-reaper/config/cassandra-reaper.yml accessControl: sessionTimeout: PT10M shiro: - iniConfigs: ["file:/etc/cassandra-reaper/shiro.ini"] + iniConfigs: ["file:/etc/cassandra-reaper/config/shiro.ini"] EOT else -cat <> /etc/cassandra-reaper/cassandra-reaper.yml +cat <> /etc/cassandra-reaper/config/cassandra-reaper.yml accessControl: sessionTimeout: PT10M shiro: @@ -41,7 +41,7 @@ EOT fi if [ ! -z "${REAPER_AUTH_USER}" ]; then -cat <> /etc/cassandra-reaper/shiro.ini +cat <> /etc/cassandra-reaper/config/shiro.ini ${REAPER_AUTH_USER} = ${REAPER_AUTH_PASSWORD}, operator EOT2 fi diff --git a/src/server/src/main/docker/entrypoint.sh b/src/server/src/main/docker/entrypoint.sh index 593bd8d9e..70b8b71f6 100644 --- a/src/server/src/main/docker/entrypoint.sh +++ b/src/server/src/main/docker/entrypoint.sh @@ -36,8 +36,10 @@ if [ "$1" = 'cassandra-reaper' ]; then if [ -z "$REAPER_HEAP_SIZE" ]; then REAPER_HEAP_SIZE="1G" fi + # get around `/usr/local/bin/configure-persistence.sh: line 65: can't create /etc/cassandra-reaper/cassandra-reaper.yml: Interrupted system call` unknown error - touch /etc/cassandra-reaper/cassandra-reaper.yml + cp /etc/cassandra-reaper/cassandra-reaper.yml /etc/cassandra-reaper/config/cassandra-reaper.yml + cp /etc/cassandra-reaper/shiro.ini /etc/cassandra-reaper/config/shiro.ini /usr/local/bin/configure-persistence.sh /usr/local/bin/configure-webui-authentication.sh @@ -49,13 +51,14 @@ if [ "$1" = 'cassandra-reaper' ]; then -Xmx${REAPER_HEAP_SIZE} \ -Djava.io.tmpdir=${REAPER_TMP_DIRECTORY} \ -cp "/usr/local/lib/*" io.cassandrareaper.ReaperApplication server \ - /etc/cassandra-reaper/cassandra-reaper.yml + /etc/cassandra-reaper/config/cassandra-reaper.yml fi if [ "$1" = 'schema-migration' ]; then # get around `/usr/local/bin/configure-persistence.sh: line 65: can't create /etc/cassandra-reaper/cassandra-reaper.yml: Interrupted system call` unknown error - touch /etc/cassandra-reaper/cassandra-reaper.yml + cp /etc/cassandra-reaper/cassandra-reaper.yml /etc/cassandra-reaper/config/cassandra-reaper.yml + cp /etc/cassandra-reaper/shiro.ini /etc/cassandra-reaper/config/shiro.ini /usr/local/bin/configure-persistence.sh /usr/local/bin/configure-webui-authentication.sh @@ -65,7 +68,7 @@ if [ "$1" = 'schema-migration' ]; then ${JAVA_OPTS} \ -Djava.io.tmpdir=${REAPER_TMP_DIRECTORY} \ -cp "/usr/local/lib/*" io.cassandrareaper.ReaperApplication schema-migration \ - /etc/cassandra-reaper/cassandra-reaper.yml + /etc/cassandra-reaper/config/cassandra-reaper.yml fi if [ "$1" = 'register-clusters' ]; then diff --git a/src/server/src/main/java/io/cassandrareaper/AppContext.java b/src/server/src/main/java/io/cassandrareaper/AppContext.java index 2e71391ac..ec41abf20 100644 --- a/src/server/src/main/java/io/cassandrareaper/AppContext.java +++ b/src/server/src/main/java/io/cassandrareaper/AppContext.java @@ -56,7 +56,7 @@ public String getLocalNodeAddress() { return localNodeAddress; } - private static class Private { + private static final class Private { private static final Logger LOG = LoggerFactory.getLogger(AppContext.class); private static final String DEFAULT_INSTANCE_ADDRESS = "127.0.0.1"; diff --git a/src/server/src/main/java/io/cassandrareaper/ReaperApplication.java b/src/server/src/main/java/io/cassandrareaper/ReaperApplication.java index 538bbb934..110bc844f 100644 --- a/src/server/src/main/java/io/cassandrareaper/ReaperApplication.java +++ b/src/server/src/main/java/io/cassandrareaper/ReaperApplication.java @@ -421,6 +421,7 @@ private void schedulePurge(ScheduledExecutorService scheduler) { private void checkConfiguration(ReaperApplicationConfiguration config) { LOG.debug("repairIntensity: {}", config.getRepairIntensity()); LOG.debug("incrementalRepair: {}", config.getIncrementalRepair()); + LOG.debug("subrangeIncrementalRepair: {}", config.getSubrangeIncrementalRepair()); LOG.debug("repairRunThreadCount: {}", config.getRepairRunThreadCount()); LOG.debug("segmentCount: {}", config.getSegmentCount()); LOG.debug("repairParallelism: {}", config.getRepairParallelism()); diff --git a/src/server/src/main/java/io/cassandrareaper/ReaperApplicationConfiguration.java b/src/server/src/main/java/io/cassandrareaper/ReaperApplicationConfiguration.java index 4a598f557..45a004906 100644 --- a/src/server/src/main/java/io/cassandrareaper/ReaperApplicationConfiguration.java +++ b/src/server/src/main/java/io/cassandrareaper/ReaperApplicationConfiguration.java @@ -74,6 +74,10 @@ public final class ReaperApplicationConfiguration extends Configuration { @DefaultValue("false") private Boolean incrementalRepair; + @JsonProperty + @DefaultValue("false") + private Boolean subrangeIncrementalRepair; + @JsonProperty private Boolean blacklistTwcsTables; @@ -237,6 +241,14 @@ public void setIncrementalRepair(boolean incrementalRepair) { this.incrementalRepair = incrementalRepair; } + public boolean getSubrangeIncrementalRepair() { + return subrangeIncrementalRepair != null ? subrangeIncrementalRepair : false; + } + + public void setSubrangeIncrementalRepair(boolean subrangeIncrementalRepair) { + this.subrangeIncrementalRepair = subrangeIncrementalRepair; + } + public boolean getBlacklistTwcsTables() { return blacklistTwcsTables != null ? blacklistTwcsTables : false; } @@ -578,6 +590,9 @@ public static final class AutoSchedulingConfiguration { @JsonProperty private Boolean incremental; + @JsonProperty + private Boolean subrangeIncrementalRepair; + @JsonProperty private Integer percentUnrepairedThreshold; @@ -658,6 +673,14 @@ public void setIncremental(Boolean incremental) { this.incremental = incremental; } + public Boolean subrangeIncrementalRepair() { + return subrangeIncrementalRepair == null ? false : subrangeIncrementalRepair; + } + + public void setSubrangeIncrementalRepair(Boolean subrangeIncrementalRepair) { + this.subrangeIncrementalRepair = subrangeIncrementalRepair; + } + public Integer getPercentUnrepairedThreshold() { return percentUnrepairedThreshold == null ? -1 : percentUnrepairedThreshold; } diff --git a/src/server/src/main/java/io/cassandrareaper/core/RepairType.java b/src/server/src/main/java/io/cassandrareaper/core/RepairType.java new file mode 100644 index 000000000..acc15fe44 --- /dev/null +++ b/src/server/src/main/java/io/cassandrareaper/core/RepairType.java @@ -0,0 +1,44 @@ +/* + * Copyright 2024-2024 DataStax, Inc. + * + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cassandrareaper.core; + +public enum RepairType { + SUBRANGE_FULL, + INCREMENTAL, + SUBRANGE_INCREMENTAL; + + public String getRepairType() { + return name().toLowerCase(); + } + + public static RepairType fromName(String name) { + return valueOf(name.toUpperCase()); + } + + public boolean isIncremental() { + return this == INCREMENTAL || this == SUBRANGE_INCREMENTAL; + } + + public boolean isFull() { + return this == SUBRANGE_FULL; + } + + public boolean isSubrange() { + return this == SUBRANGE_FULL || this == SUBRANGE_INCREMENTAL; + } +} \ No newline at end of file diff --git a/src/server/src/main/java/io/cassandrareaper/core/RepairUnit.java b/src/server/src/main/java/io/cassandrareaper/core/RepairUnit.java index 43e14e2ac..e229cf21d 100644 --- a/src/server/src/main/java/io/cassandrareaper/core/RepairUnit.java +++ b/src/server/src/main/java/io/cassandrareaper/core/RepairUnit.java @@ -31,6 +31,7 @@ public final class RepairUnit { private final String keyspaceName; private final Set columnFamilies; private final boolean incrementalRepair; + private final boolean subrangeIncrementalRepair; private final Set nodes; private final Set datacenters; private final Set blacklistedTables; @@ -42,7 +43,9 @@ private RepairUnit(Builder builder, UUID id) { this.clusterName = builder.clusterName; this.keyspaceName = builder.keyspaceName; this.columnFamilies = builder.columnFamilies; - this.incrementalRepair = builder.incrementalRepair; + // If subrange incremental repair is true, we set incremental repair to true as well + this.incrementalRepair = builder.incrementalRepair || builder.subrangeIncrementalRepair; + this.subrangeIncrementalRepair = builder.subrangeIncrementalRepair; this.nodes = builder.nodes; this.datacenters = builder.datacenters; this.blacklistedTables = builder.blacklistedTables; @@ -74,6 +77,10 @@ public boolean getIncrementalRepair() { return incrementalRepair; } + public boolean getSubrangeIncrementalRepair() { + return subrangeIncrementalRepair; + } + public Set getNodes() { return nodes; } @@ -104,6 +111,7 @@ public static final class Builder { public String keyspaceName; public Set columnFamilies = Collections.emptySet(); public Boolean incrementalRepair; + public Boolean subrangeIncrementalRepair; public Set nodes = Collections.emptySet(); public Set datacenters = Collections.emptySet(); public Set blacklistedTables = Collections.emptySet(); @@ -117,6 +125,7 @@ private Builder(RepairUnit original) { keyspaceName = original.keyspaceName; columnFamilies = original.columnFamilies; incrementalRepair = original.incrementalRepair; + subrangeIncrementalRepair = original.subrangeIncrementalRepair; nodes = original.nodes; datacenters = original.datacenters; blacklistedTables = original.blacklistedTables; @@ -144,6 +153,11 @@ public Builder incrementalRepair(boolean incrementalRepair) { return this; } + public Builder subrangeIncrementalRepair(boolean subrangeIncrementalRepair) { + this.subrangeIncrementalRepair = subrangeIncrementalRepair; + return this; + } + public Builder nodes(Set nodes) { this.nodes = Collections.unmodifiableSet(nodes); return this; @@ -173,6 +187,8 @@ public RepairUnit build(UUID id) { Preconditions.checkState(null != clusterName, "clusterName(..) must be called before build(..)"); Preconditions.checkState(null != keyspaceName, "keyspaceName(..) must be called before build(..)"); Preconditions.checkState(null != incrementalRepair, "incrementalRepair(..) must be called before build(..)"); + Preconditions.checkState(null != subrangeIncrementalRepair, + "subrangeIncrementalRepair(..) must be called before build(..)"); Preconditions.checkState(null != repairThreadCount, "repairThreadCount(..) must be called before build(..)"); Preconditions.checkState(null != timeout, "timeout(..) must be called before build(..)"); return new RepairUnit(this, id); @@ -190,6 +206,8 @@ public int hashCode() { hash *= 59; hash += (this.incrementalRepair ? 2 : 1); hash *= 59; + hash += (this.subrangeIncrementalRepair ? 8 : 4); + hash *= 59; hash += Objects.hashCode(this.nodes); hash *= 59; hash += Objects.hashCode(this.datacenters); @@ -212,6 +230,7 @@ public boolean equals(Object obj) { } return Objects.equals(this.incrementalRepair, ((Builder) obj).incrementalRepair) + && Objects.equals(this.subrangeIncrementalRepair, ((Builder) obj).subrangeIncrementalRepair) && Objects.equals(this.clusterName, ((Builder) obj).clusterName) && Objects.equals(this.keyspaceName, ((Builder) obj).keyspaceName) && Objects.equals(this.columnFamilies, ((Builder) obj).columnFamilies) diff --git a/src/server/src/main/java/io/cassandrareaper/management/ClusterFacade.java b/src/server/src/main/java/io/cassandrareaper/management/ClusterFacade.java index ff860417b..2745008f7 100644 --- a/src/server/src/main/java/io/cassandrareaper/management/ClusterFacade.java +++ b/src/server/src/main/java/io/cassandrareaper/management/ClusterFacade.java @@ -924,7 +924,7 @@ private Collection enforceLocalNodeForSidecar(Collection endpoin : endpoints; } - private static class Async { + private static final class Async { private static final ExecutorService ASYNC = Executors.newSingleThreadExecutor(); private static boolean markClusterActive(Cluster cluster, AppContext context) { diff --git a/src/server/src/main/java/io/cassandrareaper/management/ICassandraManagementProxy.java b/src/server/src/main/java/io/cassandrareaper/management/ICassandraManagementProxy.java index 68ac8c36e..fc4b50610 100644 --- a/src/server/src/main/java/io/cassandrareaper/management/ICassandraManagementProxy.java +++ b/src/server/src/main/java/io/cassandrareaper/management/ICassandraManagementProxy.java @@ -18,6 +18,7 @@ package io.cassandrareaper.management; import io.cassandrareaper.ReaperException; +import io.cassandrareaper.core.RepairType; import io.cassandrareaper.core.Snapshot; import io.cassandrareaper.core.Table; import io.cassandrareaper.resources.view.NodesStatus; @@ -122,7 +123,7 @@ int triggerRepair( String keyspace, RepairParallelism repairParallelism, Collection columnFamilies, - boolean fullRepair, + RepairType repairType, Collection datacenters, RepairStatusHandler repairStatusHandler, List associatedTokens, diff --git a/src/server/src/main/java/io/cassandrareaper/management/MetricsProxy.java b/src/server/src/main/java/io/cassandrareaper/management/MetricsProxy.java index 93c8ad647..f393e4111 100644 --- a/src/server/src/main/java/io/cassandrareaper/management/MetricsProxy.java +++ b/src/server/src/main/java/io/cassandrareaper/management/MetricsProxy.java @@ -32,7 +32,6 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; - import javax.management.JMException; import com.google.common.collect.Lists; diff --git a/src/server/src/main/java/io/cassandrareaper/management/http/HttpCassandraManagementProxy.java b/src/server/src/main/java/io/cassandrareaper/management/http/HttpCassandraManagementProxy.java index b21233e0d..9a9d91025 100644 --- a/src/server/src/main/java/io/cassandrareaper/management/http/HttpCassandraManagementProxy.java +++ b/src/server/src/main/java/io/cassandrareaper/management/http/HttpCassandraManagementProxy.java @@ -20,6 +20,7 @@ import io.cassandrareaper.ReaperException; import io.cassandrareaper.core.GenericMetric; import io.cassandrareaper.core.Node; +import io.cassandrareaper.core.RepairType; import io.cassandrareaper.core.Snapshot; import io.cassandrareaper.core.Table; import io.cassandrareaper.management.ICassandraManagementProxy; @@ -68,7 +69,6 @@ import com.datastax.mgmtapi.client.model.TokenRangeToEndpoints; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Maps; - import org.apache.cassandra.repair.RepairParallelism; import org.apache.cassandra.utils.progress.ProgressEventType; import org.slf4j.Logger; @@ -354,7 +354,7 @@ public int triggerRepair( String keyspace, RepairParallelism repairParallelism, Collection columnFamilies, - boolean fullRepair, + RepairType repairType, Collection datacenters, RepairStatusHandler repairStatusHandler, List associatedTokens, @@ -365,7 +365,7 @@ public int triggerRepair( try { RepairRequestResponse resp = apiClient.putRepairV2( (new RepairRequest()) - .fullRepair(fullRepair) + .fullRepair(repairType.isFull()) .keyspace(keyspace) .tables(new ArrayList<>(columnFamilies)) .repairParallelism(RepairRequest.RepairParallelismEnum.fromValue(repairParallelism.getName())) diff --git a/src/server/src/main/java/io/cassandrareaper/management/http/HttpMetricsProxy.java b/src/server/src/main/java/io/cassandrareaper/management/http/HttpMetricsProxy.java index 00ae2c406..dca8448b9 100644 --- a/src/server/src/main/java/io/cassandrareaper/management/http/HttpMetricsProxy.java +++ b/src/server/src/main/java/io/cassandrareaper/management/http/HttpMetricsProxy.java @@ -31,13 +31,11 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; - import javax.management.JMException; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; import com.google.common.collect.Maps; - import okhttp3.HttpUrl; import okhttp3.OkHttpClient; import okhttp3.Request; diff --git a/src/server/src/main/java/io/cassandrareaper/management/jmx/JmxCassandraManagementProxy.java b/src/server/src/main/java/io/cassandrareaper/management/jmx/JmxCassandraManagementProxy.java index 717dd1020..7907b2254 100644 --- a/src/server/src/main/java/io/cassandrareaper/management/jmx/JmxCassandraManagementProxy.java +++ b/src/server/src/main/java/io/cassandrareaper/management/jmx/JmxCassandraManagementProxy.java @@ -21,6 +21,7 @@ import io.cassandrareaper.ReaperException; import io.cassandrareaper.core.Cluster; import io.cassandrareaper.core.JmxCredentials; +import io.cassandrareaper.core.RepairType; import io.cassandrareaper.core.Snapshot; import io.cassandrareaper.core.Table; import io.cassandrareaper.crypto.Cryptograph; @@ -556,7 +557,7 @@ public int triggerRepair( String keyspace, RepairParallelism repairParallelism, Collection columnFamilies, - boolean fullRepair, + RepairType repairType, Collection datacenters, RepairStatusHandler repairStatusHandler, List associatedTokens, @@ -589,7 +590,7 @@ public int triggerRepair( try { int repairNo; repairNo = triggerRepairPost2dot2( - fullRepair, + repairType, repairParallelism, keyspace, columnFamilies, @@ -606,7 +607,7 @@ public int triggerRepair( } private int triggerRepairPost2dot2( - boolean fullRepair, + RepairType repairType, RepairParallelism repairParallelism, String keyspace, Collection columnFamilies, @@ -617,14 +618,14 @@ private int triggerRepairPost2dot2( Map options = new HashMap<>(); options.put(RepairOption.PARALLELISM_KEY, repairParallelism.getName()); - options.put(RepairOption.INCREMENTAL_KEY, Boolean.toString(!fullRepair)); + options.put(RepairOption.INCREMENTAL_KEY, Boolean.toString(repairType.isIncremental())); options.put( RepairOption.JOB_THREADS_KEY, Integer.toString(repairThreadCount == 0 ? 1 : repairThreadCount)); options.put(RepairOption.TRACE_KEY, Boolean.toString(Boolean.FALSE)); options.put(RepairOption.COLUMNFAMILIES_KEY, StringUtils.join(columnFamilies, ",")); // options.put(RepairOption.PULL_REPAIR_KEY, Boolean.FALSE); - if (fullRepair) { + if (repairType.isSubrange()) { options.put( RepairOption.RANGES_KEY, StringUtils.join( diff --git a/src/server/src/main/java/io/cassandrareaper/resources/NodeStatsResource.java b/src/server/src/main/java/io/cassandrareaper/resources/NodeStatsResource.java index 0a94d541e..4f2de6645 100644 --- a/src/server/src/main/java/io/cassandrareaper/resources/NodeStatsResource.java +++ b/src/server/src/main/java/io/cassandrareaper/resources/NodeStatsResource.java @@ -28,7 +28,6 @@ import java.util.List; import java.util.Map; - import javax.ws.rs.GET; import javax.ws.rs.Path; import javax.ws.rs.PathParam; diff --git a/src/server/src/main/java/io/cassandrareaper/resources/RepairRunResource.java b/src/server/src/main/java/io/cassandrareaper/resources/RepairRunResource.java index 825bfa325..6a7c11368 100644 --- a/src/server/src/main/java/io/cassandrareaper/resources/RepairRunResource.java +++ b/src/server/src/main/java/io/cassandrareaper/resources/RepairRunResource.java @@ -25,6 +25,7 @@ import io.cassandrareaper.core.RepairSegment; import io.cassandrareaper.core.RepairUnit; import io.cassandrareaper.management.ClusterFacade; +import io.cassandrareaper.management.ICassandraManagementProxy; import io.cassandrareaper.resources.view.RepairRunStatus; import io.cassandrareaper.service.PurgeService; import io.cassandrareaper.service.RepairRunService; @@ -57,6 +58,7 @@ import javax.ws.rs.core.Response.Status; import javax.ws.rs.core.UriInfo; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import org.apache.cassandra.repair.RepairParallelism; @@ -100,6 +102,7 @@ static Response checkRequestForAddRepair( Optional repairParallelism, Optional intensityStr, Optional incrementalRepairStr, + Optional subrangeIncrementalStr, Optional nodesStr, Optional datacentersStr, Optional blacklistedTableNamesParam, @@ -148,6 +151,14 @@ static Response checkRequestForAddRepair( .entity("invalid query parameter \"incrementalRepair\", expecting [True,False]") .build(); } + if (subrangeIncrementalStr.isPresent() + && (!subrangeIncrementalStr.get().toUpperCase().contentEquals("TRUE") + && !subrangeIncrementalStr.get().toUpperCase().contentEquals("FALSE"))) { + + return Response.status(Response.Status.BAD_REQUEST) + .entity("invalid query parameter \"subrangeIncrementalStr\", expecting [True,False]") + .build(); + } try { Cluster cluster = context.storage.getClusterDao().getCluster(Cluster.toSymbolicName(clusterName.get())); @@ -157,10 +168,22 @@ static Response checkRequestForAddRepair( .build(); } - if (incrementalRepairStr.isPresent() && "true".equalsIgnoreCase(incrementalRepairStr.get())) { + if (subrangeIncrementalStr.isPresent() && "true".equalsIgnoreCase(subrangeIncrementalStr.get())) { + try { + String version = ClusterFacade.create(context).getCassandraVersion(cluster); + if (null != version && ICassandraManagementProxy.versionCompare(version, "4.0.0") < 0) { + String msg = "Subrange incremental repair does not work with Cassandra versions before 4.0.0"; + return Response.status(Response.Status.BAD_REQUEST).entity(msg).build(); + } + } catch (ReaperException e) { + String msg = String.format("find version of cluster %s failed", cluster.getName()); + LOG.error(msg, e); + return Response.serverError().entity(msg).build(); + } + } else if (incrementalRepairStr.isPresent() && "true".equalsIgnoreCase(incrementalRepairStr.get())) { try { String version = ClusterFacade.create(context).getCassandraVersion(cluster); - if (null != version && version.startsWith("2.0")) { + if (null != version && ICassandraManagementProxy.versionCompare(version, "2.1.0") < 0) { String msg = "Incremental repair does not work with Cassandra versions before 2.1"; return Response.status(Response.Status.BAD_REQUEST).entity(msg).build(); } @@ -309,6 +332,7 @@ public Response addRepairRun( @QueryParam("repairParallelism") Optional repairParallelism, @QueryParam("intensity") Optional intensityStr, @QueryParam("incrementalRepair") Optional incrementalRepairStr, + @QueryParam("subrangeIncrementalRepair") Optional subrangeIncrementalRepairStr, @QueryParam("nodes") Optional nodesToRepairParam, @QueryParam("datacenters") Optional datacentersToRepairParam, @QueryParam("blacklistedTables") Optional blacklistedTableNamesParam, @@ -328,6 +352,7 @@ public Response addRepairRun( repairParallelism, intensityStr, incrementalRepairStr, + subrangeIncrementalRepairStr, nodesToRepairParam, datacentersToRepairParam, blacklistedTableNamesParam, @@ -338,34 +363,9 @@ public Response addRepairRun( if (null != possibleFailedResponse) { return possibleFailedResponse; } - Double intensity; - if (intensityStr.isPresent()) { - intensity = Double.parseDouble(intensityStr.get()); - } else { - intensity = context.config.getRepairIntensity(); - LOG.debug("no intensity given, so using default value: {}", intensity); - } - boolean incrementalRepair; - if (incrementalRepairStr.isPresent()) { - incrementalRepair = Boolean.parseBoolean(incrementalRepairStr.get()); - } else { - incrementalRepair = context.config.getIncrementalRepair(); - LOG.debug("no incremental repair given, so using default value: {}", incrementalRepair); - } - int segments = context.config.getSegmentCountPerNode(); - if (!incrementalRepair) { - if (segmentCountPerNode.isPresent()) { - LOG.debug( - "using given segment count {} instead of configured value {}", - segmentCountPerNode.get(), - context.config.getSegmentCount()); - segments = segmentCountPerNode.get(); - } - } else { - // hijack the segment count in case of incremental repair - segments = -1; - } + final Cluster cluster = context.storage.getClusterDao().getCluster(Cluster.toSymbolicName(clusterName.get())); + Set tableNames; try { tableNames = repairRunService.getTableNamesBasedOnParam(cluster, keyspace.get(), tableNamesParam); @@ -373,6 +373,7 @@ public Response addRepairRun( LOG.error(ex.getMessage(), ex); return Response.status(Response.Status.NOT_FOUND).entity(ex.getMessage()).build(); } + Set blacklistedTableNames; try { blacklistedTableNames @@ -381,6 +382,7 @@ public Response addRepairRun( LOG.error(ex.getMessage(), ex); return Response.status(Response.Status.NOT_FOUND).entity(ex.getMessage()).build(); } + final Set nodesToRepair; try { nodesToRepair = repairRunService.getNodesToRepairBasedOnParam(cluster, nodesToRepairParam); @@ -388,6 +390,7 @@ public Response addRepairRun( LOG.error(ex.getMessage(), ex); return Response.status(Response.Status.NOT_FOUND).entity(ex.getMessage()).build(); } + final Set datacentersToRepair; try { datacentersToRepair = RepairRunService @@ -399,12 +402,15 @@ public Response addRepairRun( } int timeout = timeoutParam.orElse(context.config.getHangingRepairTimeoutMins()); boolean force = (forceParam.isPresent() ? Boolean.parseBoolean(forceParam.get()) : false); + boolean subrangeIncrementalRepair = getSubrangeIncrementalRepair(subrangeIncrementalRepairStr); + boolean incrementalRepair = getIncrementalRepair(incrementalRepairStr); RepairUnit.Builder builder = RepairUnit.builder() .clusterName(cluster.getName()) .keyspaceName(keyspace.get()) .columnFamilies(tableNames) .incrementalRepair(incrementalRepair) + .subrangeIncrementalRepair(subrangeIncrementalRepair) .nodes(nodesToRepair) .datacenters(datacentersToRepair) .blacklistedTables(blacklistedTableNames) @@ -414,7 +420,8 @@ public Response addRepairRun( final Optional maybeTheRepairUnit = repairUnitService.getOrCreateRepairUnit(cluster, builder, force); if (maybeTheRepairUnit.isPresent()) { RepairUnit theRepairUnit = maybeTheRepairUnit.get(); - if (theRepairUnit.getIncrementalRepair() != incrementalRepair) { + if (theRepairUnit.getIncrementalRepair() != incrementalRepair + && theRepairUnit.getSubrangeIncrementalRepair() != subrangeIncrementalRepair) { String msg = String.format( "A repair unit %s already exist for the same cluster/keyspace/tables" + " but with a different incremental repair value. Requested value %s | Existing value: %s", @@ -435,10 +442,12 @@ public Response addRepairRun( parallelism = RepairParallelism.valueOf(repairParallelism.get().toUpperCase()); } - if (incrementalRepair) { + if (incrementalRepair || subrangeIncrementalRepair) { parallelism = RepairParallelism.PARALLEL; } + Double intensity = getIntensity(intensityStr); + int segments = getSegments(segmentCountPerNode, subrangeIncrementalRepair, incrementalRepair); final RepairRun newRepairRun = repairRunService.registerRepairRun( cluster, theRepairUnit, @@ -464,6 +473,61 @@ public Response addRepairRun( } } + @VisibleForTesting + public int getSegments(Optional segmentCountPerNode, boolean subrangeIncrementalRepair, + boolean incrementalRepair) { + int segments = context.config.getSegmentCountPerNode(); + if (!incrementalRepair || subrangeIncrementalRepair) { + if (segmentCountPerNode.isPresent()) { + LOG.debug( + "using given segment count {} instead of configured value {}", + segmentCountPerNode.get(), + context.config.getSegmentCount()); + segments = segmentCountPerNode.get(); + } + } else { + // hijack the segment count in case of incremental repair + segments = -1; + } + return segments; + } + + @VisibleForTesting + public boolean getIncrementalRepair(Optional incrementalRepairStr) { + boolean incrementalRepair; + if (incrementalRepairStr.isPresent()) { + incrementalRepair = Boolean.parseBoolean(incrementalRepairStr.get()); + } else { + incrementalRepair = context.config.getIncrementalRepair(); + LOG.debug("no incremental repair given, so using default value: {}", incrementalRepair); + } + return incrementalRepair; + } + + @VisibleForTesting + public boolean getSubrangeIncrementalRepair(Optional subrangeIncrementalRepairStr) { + boolean subrangeIncrementalRepair; + if (subrangeIncrementalRepairStr.isPresent()) { + subrangeIncrementalRepair = Boolean.parseBoolean(subrangeIncrementalRepairStr.get()); + } else { + subrangeIncrementalRepair = context.config.getSubrangeIncrementalRepair(); + LOG.debug("no subrange incremental repair given, so using default value: {}", subrangeIncrementalRepair); + } + return subrangeIncrementalRepair; + } + + @VisibleForTesting + public Double getIntensity(Optional intensityStr) { + Double intensity; + if (intensityStr.isPresent()) { + intensity = Double.parseDouble(intensityStr.get()); + } else { + intensity = context.config.getRepairIntensity(); + LOG.debug("no intensity given, so using default value: {}", intensity); + } + return intensity; + } + /** * Modifies a state of the repair run. * diff --git a/src/server/src/main/java/io/cassandrareaper/resources/RepairScheduleResource.java b/src/server/src/main/java/io/cassandrareaper/resources/RepairScheduleResource.java index 609101320..961cc8f35 100644 --- a/src/server/src/main/java/io/cassandrareaper/resources/RepairScheduleResource.java +++ b/src/server/src/main/java/io/cassandrareaper/resources/RepairScheduleResource.java @@ -225,6 +225,7 @@ public Response addRepairSchedule( @QueryParam("repairParallelism") Optional repairParallelism, @QueryParam("intensity") Optional intensityStr, @QueryParam("incrementalRepair") Optional incrementalRepairStr, + @QueryParam("subrangeIncrementalRepair") Optional subrangeIncrementalRepairStr, @QueryParam("scheduleDaysBetween") Optional scheduleDaysBetween, @QueryParam("scheduleTriggerTime") Optional scheduleTriggerTime, @QueryParam("nodes") Optional nodesToRepairParam, @@ -247,6 +248,7 @@ public Response addRepairSchedule( repairParallelism, intensityStr, incrementalRepairStr, + subrangeIncrementalRepairStr, nodesToRepairParam, datacentersToRepairParam, blacklistedTableNamesParam, @@ -308,7 +310,8 @@ public Response addRepairSchedule( final Set datacentersToRepair = RepairRunService .getDatacentersToRepairBasedOnParam(datacentersToRepairParam); - boolean incremental = isIncrementalRepair(incrementalRepairStr); + boolean incremental + = isIncrementalRepair(incrementalRepairStr) || isIncrementalRepair(subrangeIncrementalRepairStr); RepairParallelism parallelism = context.config.getRepairParallelism(); if (repairParallelism.isPresent()) { LOG.debug("using given repair parallelism {} over configured value {}", repairParallelism.get(), parallelism); @@ -332,12 +335,13 @@ public Response addRepairSchedule( int timeout = timeoutParam.orElse(context.config.getHangingRepairTimeoutMins()); boolean adaptive = (adaptiveParam.isPresent() ? Boolean.parseBoolean(adaptiveParam.get()) : false); - + boolean subrangeIncremental = isIncrementalRepair(subrangeIncrementalRepairStr) ; RepairUnit.Builder unitBuilder = RepairUnit.builder() .clusterName(cluster.getName()) .keyspaceName(keyspace.get()) .columnFamilies(tableNames) .incrementalRepair(incremental) + .subrangeIncrementalRepair(subrangeIncremental) .nodes(nodesToRepair) .datacenters(datacentersToRepair) .blacklistedTables(blacklistedTableNames) @@ -419,7 +423,9 @@ private Response addRepairSchedule( if (maybeUnit.isPresent()) { RepairUnit unit = maybeUnit.get(); Preconditions - .checkState(unit.getIncrementalRepair() == incremental, "%s!=%s", unit.getIncrementalRepair(), incremental); + .checkState(unit.getIncrementalRepair() == incremental + || unit.getSubrangeIncrementalRepair() == incremental, + "%s!=%s", unit.getIncrementalRepair(), incremental); Preconditions .checkState((percentUnrepairedThreshold > 0 && incremental) || percentUnrepairedThreshold <= 0, "Setting a % repaired threshold can only be done on incremental schedules"); diff --git a/src/server/src/main/java/io/cassandrareaper/resources/RequestUtils.java b/src/server/src/main/java/io/cassandrareaper/resources/RequestUtils.java index 296927a96..95aba9760 100644 --- a/src/server/src/main/java/io/cassandrareaper/resources/RequestUtils.java +++ b/src/server/src/main/java/io/cassandrareaper/resources/RequestUtils.java @@ -18,7 +18,6 @@ package io.cassandrareaper.resources; import java.time.Duration; - import javax.servlet.ServletRequest; import javax.servlet.http.HttpServletRequest; import javax.ws.rs.HttpMethod; diff --git a/src/server/src/main/java/io/cassandrareaper/resources/SnapshotResource.java b/src/server/src/main/java/io/cassandrareaper/resources/SnapshotResource.java index 91b332e7f..b7c3b3866 100644 --- a/src/server/src/main/java/io/cassandrareaper/resources/SnapshotResource.java +++ b/src/server/src/main/java/io/cassandrareaper/resources/SnapshotResource.java @@ -26,7 +26,6 @@ import java.util.List; import java.util.Map; - import javax.ws.rs.DELETE; import javax.ws.rs.GET; import javax.ws.rs.POST; diff --git a/src/server/src/main/java/io/cassandrareaper/resources/auth/LoginResource.java b/src/server/src/main/java/io/cassandrareaper/resources/auth/LoginResource.java index 1b9774e6a..125a93056 100644 --- a/src/server/src/main/java/io/cassandrareaper/resources/auth/LoginResource.java +++ b/src/server/src/main/java/io/cassandrareaper/resources/auth/LoginResource.java @@ -18,7 +18,6 @@ package io.cassandrareaper.resources.auth; import java.io.IOException; - import javax.ws.rs.FormParam; import javax.ws.rs.POST; import javax.ws.rs.Path; diff --git a/src/server/src/main/java/io/cassandrareaper/resources/auth/RestPermissionsFilter.java b/src/server/src/main/java/io/cassandrareaper/resources/auth/RestPermissionsFilter.java index 6b719f0a2..d105169b1 100644 --- a/src/server/src/main/java/io/cassandrareaper/resources/auth/RestPermissionsFilter.java +++ b/src/server/src/main/java/io/cassandrareaper/resources/auth/RestPermissionsFilter.java @@ -21,7 +21,6 @@ import java.io.IOException; import java.util.Date; - import javax.servlet.ServletRequest; import javax.servlet.ServletResponse; import javax.servlet.http.HttpServletResponse; @@ -50,7 +49,7 @@ public boolean isAccessAllowed(ServletRequest request, ServletResponse response, if (!subject.getPrincipals().getRealmNames().contains("jwtRealm") && !RequestUtils.getSessionTimeout().isNegative() && subject.getSession().getStartTimestamp().before( - new Date(System.currentTimeMillis() - RequestUtils.getSessionTimeout().toMillis()))) { + new Date(System.currentTimeMillis() - RequestUtils.getSessionTimeout().toMillis()))) { // Session has lived longer than its timeout already. Force logout. subject.logout(); return false; diff --git a/src/server/src/main/java/io/cassandrareaper/resources/auth/ShiroJwtVerifyingFilter.java b/src/server/src/main/java/io/cassandrareaper/resources/auth/ShiroJwtVerifyingFilter.java index d6319aada..f1c689508 100644 --- a/src/server/src/main/java/io/cassandrareaper/resources/auth/ShiroJwtVerifyingFilter.java +++ b/src/server/src/main/java/io/cassandrareaper/resources/auth/ShiroJwtVerifyingFilter.java @@ -20,7 +20,6 @@ import io.cassandrareaper.resources.RequestUtils; import java.util.Optional; - import javax.servlet.ServletRequest; import javax.servlet.ServletResponse; import javax.servlet.http.HttpServletResponse; @@ -31,13 +30,11 @@ import io.jsonwebtoken.JwtException; import io.jsonwebtoken.Jwts; import io.jsonwebtoken.lang.Strings; - import org.apache.shiro.subject.SimplePrincipalCollection; import org.apache.shiro.subject.Subject; import org.apache.shiro.web.filter.AccessControlFilter; import org.apache.shiro.web.subject.WebSubject; import org.apache.shiro.web.util.WebUtils; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/server/src/main/java/io/cassandrareaper/resources/view/NodesStatus.java b/src/server/src/main/java/io/cassandrareaper/resources/view/NodesStatus.java index 753be59be..4f4f02b09 100644 --- a/src/server/src/main/java/io/cassandrareaper/resources/view/NodesStatus.java +++ b/src/server/src/main/java/io/cassandrareaper/resources/view/NodesStatus.java @@ -97,10 +97,6 @@ private GossipInfo parseEndpointStatesString( String allEndpointStates, Map simpleStates) { - List endpointStates = Lists.newArrayList(); - Set endpoints = Sets.newHashSet(); - Matcher matcher; - // Split into endpointState record strings String[] endpointLines = allEndpointStates.split("\n"); List strEndpoints = Lists.newArrayList(); @@ -132,7 +128,9 @@ private GossipInfo parseEndpointStatesString( simpleStates = simpleStatesCopy; Double totalLoad = 0.0; - + Set endpoints = Sets.newHashSet(); + Matcher matcher; + List endpointStates = Lists.newArrayList(); for (String endpointString : strEndpoints) { Optional status = Optional.empty(); Optional endpoint = parseEndpointState(ENDPOINT_NAME_PATTERNS, endpointString, 1, String.class); diff --git a/src/server/src/main/java/io/cassandrareaper/resources/view/RepairRunStatus.java b/src/server/src/main/java/io/cassandrareaper/resources/view/RepairRunStatus.java index 641b7fea2..8122eedd1 100644 --- a/src/server/src/main/java/io/cassandrareaper/resources/view/RepairRunStatus.java +++ b/src/server/src/main/java/io/cassandrareaper/resources/view/RepairRunStatus.java @@ -22,7 +22,6 @@ import java.util.Collection; import java.util.UUID; - import javax.annotation.Nullable; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -79,6 +78,9 @@ public final class RepairRunStatus { @JsonProperty("incremental_repair") private boolean incrementalRepair; + @JsonProperty("subrange_incremental_repair") + private boolean subrangeIncrementalRepair; + @JsonProperty("total_segments") private int totalSegments; @@ -142,6 +144,7 @@ public RepairRunStatus( DateTime pauseTime, double intensity, boolean incrementalRepair, + boolean subrangeIncrementalRepair, RepairParallelism repairParallelism, Collection nodes, Collection datacenters, @@ -165,6 +168,7 @@ public RepairRunStatus( this.currentTime = DateTime.now(); this.intensity = roundDoubleNicely(intensity); this.incrementalRepair = incrementalRepair; + this.subrangeIncrementalRepair = subrangeIncrementalRepair; this.totalSegments = totalSegments; this.repairParallelism = repairParallelism; this.segmentsRepaired = segmentsRepaired; @@ -231,6 +235,7 @@ public RepairRunStatus(RepairRun repairRun, RepairUnit repairUnit, int segmentsR repairRun.getPauseTime(), repairRun.getIntensity(), repairUnit.getIncrementalRepair(), + repairUnit.getSubrangeIncrementalRepair(), repairRun.getRepairParallelism(), repairUnit.getNodes(), repairUnit.getDatacenters(), @@ -413,6 +418,14 @@ public void setIncrementalRepair(boolean incrementalRepair) { this.incrementalRepair = incrementalRepair; } + public boolean getSubrangeIncrementalRepair() { + return subrangeIncrementalRepair; + } + + public void setSubrangeIncrementalRepair(boolean subrangeIncrementalRepair) { + this.subrangeIncrementalRepair = subrangeIncrementalRepair; + } + public int getTotalSegments() { return totalSegments; } diff --git a/src/server/src/main/java/io/cassandrareaper/resources/view/RepairScheduleStatus.java b/src/server/src/main/java/io/cassandrareaper/resources/view/RepairScheduleStatus.java index 175f155f0..89d3e2f00 100644 --- a/src/server/src/main/java/io/cassandrareaper/resources/view/RepairScheduleStatus.java +++ b/src/server/src/main/java/io/cassandrareaper/resources/view/RepairScheduleStatus.java @@ -64,6 +64,9 @@ public final class RepairScheduleStatus { @JsonProperty("incremental_repair") private boolean incrementalRepair; + @JsonProperty("subrange_incremental_repair") + private boolean subrangeIncrementalRepair; + @JsonProperty("repair_parallelism") private RepairParallelism repairParallelism; @@ -115,6 +118,7 @@ public RepairScheduleStatus( DateTime pauseTime, double intensity, boolean incrementalRepair, + boolean subrangeIncrementalRepair, RepairParallelism repairParallelism, int daysBetween, Collection nodes, @@ -138,6 +142,7 @@ public RepairScheduleStatus( this.pauseTime = pauseTime; this.intensity = RepairRunStatus.roundDoubleNicely(intensity); this.incrementalRepair = incrementalRepair; + this.subrangeIncrementalRepair = subrangeIncrementalRepair; this.repairParallelism = repairParallelism; this.daysBetween = daysBetween; this.nodes = nodes; @@ -164,6 +169,7 @@ public RepairScheduleStatus(RepairSchedule repairSchedule, RepairUnit repairUnit repairSchedule.getPauseTime(), repairSchedule.getIntensity(), repairUnit.getIncrementalRepair(), + repairUnit.getSubrangeIncrementalRepair(), repairSchedule.getRepairParallelism(), repairSchedule.getDaysBetween(), repairUnit.getNodes(), @@ -265,6 +271,14 @@ public void setIncrementalRepair(boolean incrementalRepair) { this.incrementalRepair = incrementalRepair; } + public boolean getSubrangeIncrementalRepair() { + return subrangeIncrementalRepair; + } + + public void setSubrangeIncrementalRepair(boolean subrangeIncrementalRepair) { + this.subrangeIncrementalRepair = subrangeIncrementalRepair; + } + public RepairParallelism getRepairParallelism() { return repairParallelism; } diff --git a/src/server/src/main/java/io/cassandrareaper/service/ClusterRepairScheduler.java b/src/server/src/main/java/io/cassandrareaper/service/ClusterRepairScheduler.java index 3818683cd..7827fb207 100644 --- a/src/server/src/main/java/io/cassandrareaper/service/ClusterRepairScheduler.java +++ b/src/server/src/main/java/io/cassandrareaper/service/ClusterRepairScheduler.java @@ -119,11 +119,13 @@ private boolean keyspaceCandidateForRepair(Cluster cluster, String keyspace) { private void createRepairSchedule(Cluster cluster, String keyspace, DateTime nextActivationTime) { boolean incrementalRepair = context.config.getAutoScheduling().incremental(); + boolean subrangeIncrementalRepair = context.config.getAutoScheduling().subrangeIncrementalRepair(); RepairUnit.Builder builder = RepairUnit.builder() .clusterName(cluster.getName()) .keyspaceName(keyspace) .incrementalRepair(incrementalRepair) + .subrangeIncrementalRepair(subrangeIncrementalRepair) .repairThreadCount(context.config.getRepairThreadCount()) .timeout(context.config.getHangingRepairTimeoutMins()); diff --git a/src/server/src/main/java/io/cassandrareaper/service/RepairManager.java b/src/server/src/main/java/io/cassandrareaper/service/RepairManager.java index ce344ca00..fdf5df234 100644 --- a/src/server/src/main/java/io/cassandrareaper/service/RepairManager.java +++ b/src/server/src/main/java/io/cassandrareaper/service/RepairManager.java @@ -256,13 +256,17 @@ private void resumeUnknownPausedRepairRuns(Collection pausedRepairRun private void abortSegmentsWithNoLeader(RepairRun repairRun, Collection runningSegments) { RepairUnit repairUnit = context.storage.getRepairUnitDao().getRepairUnit(repairRun.getRepairUnitId()); - if (repairUnit.getIncrementalRepair()) { + if (repairUnitIsNonSubrangeIncremental(repairUnit)) { abortSegmentsWithNoLeaderIncremental(repairRun, runningSegments); } else { abortSegmentsWithNoLeaderNonIncremental(repairRun, runningSegments); } } + private boolean repairUnitIsNonSubrangeIncremental(RepairUnit repairUnit) { + return repairUnit.getIncrementalRepair() && !repairUnit.getSubrangeIncrementalRepair(); + } + private void abortSegmentsWithNoLeaderIncremental(RepairRun repairRun, Collection runningSegments) { if (LOG.isDebugEnabled()) { @@ -326,7 +330,10 @@ public RepairSegment abortSegment(UUID runId, UUID segmentId) throws ReaperExcep try { if (null == segment.getCoordinatorHost() || RepairSegment.State.DONE == segment.getState()) { RepairUnit repairUnit = context.storage.getRepairUnitDao().getRepairUnit(segment.getRepairUnitId()); - UUID leaderElectionId = repairUnit.getIncrementalRepair() ? runId : segmentId; + // Incremental non subrange repairs will use the run id as leader election id + // to prevent multiple segments to run at once. Subrange incremental will allow multiple segments. + UUID leaderElectionId = repairUnit.getIncrementalRepair() && !repairUnit.getSubrangeIncrementalRepair() + ? runId : segmentId; boolean tookLead; if (tookLead = takeLead(context, leaderElectionId) || renewLead(context, leaderElectionId)) { try { diff --git a/src/server/src/main/java/io/cassandrareaper/service/RepairRunService.java b/src/server/src/main/java/io/cassandrareaper/service/RepairRunService.java index 37f0e6ba4..7b42f8414 100644 --- a/src/server/src/main/java/io/cassandrareaper/service/RepairRunService.java +++ b/src/server/src/main/java/io/cassandrareaper/service/RepairRunService.java @@ -251,15 +251,19 @@ public RepairRun registerRepairRun( // preparing a repair run involves several steps // the first step is to generate token segments - List tokenSegments = repairUnit.getIncrementalRepair() + // Non subrange incremental repair will generate a single segment per node + // Other types of repairs (full and subrange incremental) will generate segments + List tokenSegments = repairUnit.getIncrementalRepair() && !repairUnit.getSubrangeIncrementalRepair() ? Lists.newArrayList() : generateSegments(cluster, segmentsPerNode, repairUnit); checkNotNull(tokenSegments, "failed generating repair segments"); Map nodes = getClusterNodes(cluster, repairUnit); - // the next step is to prepare a repair run objec - int segments = repairUnit.getIncrementalRepair() ? nodes.keySet().size() : tokenSegments.size(); + // the next step is to prepare a repair run object + int segments = repairUnit.getIncrementalRepair() && !repairUnit.getSubrangeIncrementalRepair() + ? nodes.keySet().size() + : tokenSegments.size(); RepairRun.Builder runBuilder = RepairRun.builder(cluster.getName(), repairUnit.getId()) .intensity(intensity) @@ -271,7 +275,8 @@ public RepairRun registerRepairRun( .adaptiveSchedule(adaptiveSchedule); // the last preparation step is to generate actual repair segments - List segmentBuilders = repairUnit.getIncrementalRepair() + List segmentBuilders + = repairUnit.getIncrementalRepair() && !repairUnit.getSubrangeIncrementalRepair() ? createRepairSegmentsForIncrementalRepair(nodes, repairUnit, cluster, clusterFacade) : createRepairSegments(tokenSegments, repairUnit); @@ -322,7 +327,7 @@ List generateSegments( sg.generateSegments( globalSegmentCount, tokens, - repairUnit.getIncrementalRepair(), + repairUnit.getIncrementalRepair() && !repairUnit.getSubrangeIncrementalRepair(), replicasToRange, cassandraVersion), repairUnit, @@ -335,7 +340,7 @@ List generateSegments( throw new ReaperException("Couldn't get endpoints for tokens", e); } - if (segments.isEmpty() && !repairUnit.getIncrementalRepair()) { + if (segments.isEmpty() && (!repairUnit.getIncrementalRepair() || repairUnit.getSubrangeIncrementalRepair())) { String errMsg = String.format("failed to generate repair segments for cluster \"%s\"", targetCluster.getName()); LOG.error(errMsg); throw new ReaperException(errMsg); diff --git a/src/server/src/main/java/io/cassandrareaper/service/RepairRunner.java b/src/server/src/main/java/io/cassandrareaper/service/RepairRunner.java index 34eb67f67..c5f4f84b6 100644 --- a/src/server/src/main/java/io/cassandrareaper/service/RepairRunner.java +++ b/src/server/src/main/java/io/cassandrareaper/service/RepairRunner.java @@ -495,7 +495,7 @@ private void startNextSegment() throws ReaperException, InterruptedException { for (RepairSegment segment : nextRepairSegments) { Map potentialReplicaMap = this.repairRunService.getDCsByNodeForRepairSegment( cluster, segment.getTokenRange(), repairUnit.getKeyspaceName(), repairUnit); - if (repairUnit.getIncrementalRepair()) { + if (repairUnit.getIncrementalRepair() && !repairUnit.getSubrangeIncrementalRepair()) { Map endpointHostIdMap = clusterFacade.getEndpointToHostId(cluster); if (segment.getHostID() == null) { throw new ReaperException( @@ -672,8 +672,8 @@ private boolean repairSegment(final UUID segmentId, Segment segment, Collection< LOG.debug("preparing to repair segment {} on run with id {}", segmentId, repairRunId); List potentialCoordinators = Lists.newArrayList(); - if (!repairUnit.getIncrementalRepair()) { - // full repair + if (!repairUnit.getIncrementalRepair() || repairUnit.getSubrangeIncrementalRepair()) { + // full repair or subrange incremental try { potentialCoordinators = filterPotentialCoordinatorsByDatacenters( repairUnit.getDatacenters(), diff --git a/src/server/src/main/java/io/cassandrareaper/service/RepairUnitService.java b/src/server/src/main/java/io/cassandrareaper/service/RepairUnitService.java index 74ac98c19..3316cbe59 100644 --- a/src/server/src/main/java/io/cassandrareaper/service/RepairUnitService.java +++ b/src/server/src/main/java/io/cassandrareaper/service/RepairUnitService.java @@ -25,6 +25,7 @@ import io.cassandrareaper.core.RepairUnit; import io.cassandrareaper.core.Table; import io.cassandrareaper.management.ClusterFacade; +import io.cassandrareaper.management.ICassandraManagementProxy; import java.util.Collection; import java.util.Collections; @@ -94,13 +95,25 @@ public Optional getOrCreateRepairUnit(Cluster cluster, RepairUnit.Bu if (params.incrementalRepair) { try { String version = clusterFacade.getCassandraVersion(cluster); - if (null != version && version.startsWith("2.0")) { + if (null != version && ICassandraManagementProxy.versionCompare(version, "2.1.0") < 0) { throw new IllegalArgumentException("Incremental repair does not work with Cassandra versions before 2.1"); } } catch (ReaperException e) { LOG.warn("unknown version to cluster {}, maybe enabling incremental on 2.0...", cluster.getName(), e); } } + if (params.subrangeIncrementalRepair) { + try { + String version = clusterFacade.getCassandraVersion(cluster); + if (null != version && ICassandraManagementProxy.versionCompare(version, "4.0.0") < 0) { + throw new IllegalArgumentException( + "Subrange incremental repair does not work with Cassandra versions before 4.0.0"); + } + } catch (ReaperException e) { + LOG.warn("unknown version to cluster {}, maybe enabling subrange incremental before 4.0...", + cluster.getName(), e); + } + } Optional repairUnit = context.storage.getRepairUnitDao().getRepairUnit(params); if (repairUnit.isPresent()) { return repairUnit; @@ -211,7 +224,12 @@ boolean identicalUnits(Cluster cluster, RepairUnit unit, RepairUnit.Builder buil // if incremental repair is not the same, the units are not identical if (unit.getIncrementalRepair() != builder.incrementalRepair.booleanValue()) { - // incremental reapir is not the same + // incremental repair is not the same + return false; + } + + if (unit.getSubrangeIncrementalRepair() != builder.subrangeIncrementalRepair.booleanValue()) { + // subrange incremental repair is not the same return false; } diff --git a/src/server/src/main/java/io/cassandrareaper/service/SegmentRunner.java b/src/server/src/main/java/io/cassandrareaper/service/SegmentRunner.java index f22e765e4..68bf551e8 100644 --- a/src/server/src/main/java/io/cassandrareaper/service/SegmentRunner.java +++ b/src/server/src/main/java/io/cassandrareaper/service/SegmentRunner.java @@ -22,6 +22,7 @@ import io.cassandrareaper.ReaperException; import io.cassandrareaper.core.Cluster; import io.cassandrareaper.core.RepairSegment; +import io.cassandrareaper.core.RepairType; import io.cassandrareaper.core.RepairUnit; import io.cassandrareaper.management.ClusterFacade; import io.cassandrareaper.management.ICassandraManagementProxy; @@ -121,7 +122,9 @@ private SegmentRunner( this.repairUnit = repairUnit; this.repairRunner = repairRunner; this.segmentFailed = new AtomicBoolean(false); - this.leaderElectionId = repairUnit.getIncrementalRepair() ? repairRunner.getRepairRunId() : segmentId; + this.leaderElectionId = repairUnit.getIncrementalRepair() && !repairUnit.getSubrangeIncrementalRepair() + ? repairRunner.getRepairRunId() + : segmentId; this.tablesToRepair = tablesToRepair; } @@ -159,7 +162,9 @@ static void postponeSegment(AppContext context, RepairSegment segment) { = segment .reset() // set coordinator host to null only for full repairs - .withCoordinatorHost(unit.getIncrementalRepair() ? segment.getCoordinatorHost() : null) + .withCoordinatorHost(unit.getIncrementalRepair() && !unit.getSubrangeIncrementalRepair() + ? segment.getCoordinatorHost() + : null) .withFailCount(segment.getFailCount() + 1) .withId(segment.getId()) .build(); @@ -174,7 +179,9 @@ private static void postpone(AppContext context, RepairSegment segment, RepairUn segment .reset() // set coordinator host to null only for full repairs - .withCoordinatorHost(repairUnit.getIncrementalRepair() ? segment.getCoordinatorHost() : null) + .withCoordinatorHost(repairUnit.getIncrementalRepair() && !repairUnit.getSubrangeIncrementalRepair() + ? segment.getCoordinatorHost() + : null) .withFailCount(segment.getFailCount() + 1) .withId(segment.getId()) .build()); @@ -298,7 +305,8 @@ private boolean runRepair() { Cluster cluster = context.storage.getClusterDao().getCluster(clusterName); ICassandraManagementProxy coordinator = clusterFacade.connect(cluster, potentialCoordinators); String keyspace = repairUnit.getKeyspaceName(); - boolean fullRepair = !repairUnit.getIncrementalRepair(); + RepairType repairType = repairUnit.getSubrangeIncrementalRepair() ? RepairType.SUBRANGE_INCREMENTAL + : repairUnit.getIncrementalRepair() ? RepairType.INCREMENTAL : RepairType.SUBRANGE_FULL; try (Timer.Context cxt1 = context.metricRegistry.timer(metricNameForRepairing(segment)).time()) { try { @@ -321,7 +329,7 @@ private boolean runRepair() { keyspace, validationParallelism, tablesToRepair, - fullRepair, + repairType, repairUnit.getDatacenters(), this, segment.getTokenRange().getTokenRanges(), @@ -371,7 +379,7 @@ private void processTriggeredSegment(final RepairSegment segment, final ICassand // Timeout is extended for each attempt to prevent repairs from blocking if settings aren't accurate int attempt = segment.getFailCount() + 1; - long segmentTimeout = repairUnit.getIncrementalRepair() + long segmentTimeout = repairUnit.getIncrementalRepair() && !repairUnit.getSubrangeIncrementalRepair() ? timeoutMillis * MAX_TIMEOUT_EXTENSIONS * attempt : timeoutMillis * attempt; LOG.info("Repair for segment {} started, status wait will timeout in {} millis", segmentId, segmentTimeout); @@ -856,7 +864,7 @@ private boolean takeLead(RepairSegment segment) { = context.metricRegistry.timer(MetricRegistry.name(SegmentRunner.class, "takeLead")).time()) { boolean result = false; - if (repairUnit.getIncrementalRepair()) { + if (repairUnit.getIncrementalRepair() && !repairUnit.getSubrangeIncrementalRepair()) { result = context.storage instanceof IDistributedStorage ? ((IDistributedStorage) context.storage).takeLead(leaderElectionId) : true; @@ -877,7 +885,7 @@ private boolean renewLead(RepairSegment segment) { try (Timer.Context cx = context.metricRegistry.timer(MetricRegistry.name(SegmentRunner.class, "renewLead")).time()) { - if (repairUnit.getIncrementalRepair()) { + if (repairUnit.getIncrementalRepair() && !repairUnit.getSubrangeIncrementalRepair()) { boolean result = context.storage instanceof IDistributedStorage ? ((IDistributedStorage) context.storage).renewLead(leaderElectionId) : true; @@ -905,7 +913,7 @@ private void releaseLead(RepairSegment segment) { try (Timer.Context cx = context.metricRegistry.timer(MetricRegistry.name(SegmentRunner.class, "releaseLead")).time()) { if (context.storage instanceof IDistributedStorage) { - if (repairUnit.getIncrementalRepair()) { + if (repairUnit.getIncrementalRepair() && !repairUnit.getSubrangeIncrementalRepair()) { ((IDistributedStorage) context.storage).releaseLead(leaderElectionId); } else { ((IDistributedStorage) context.storage).releaseRunningRepairsForNodes(this.repairRunner.getRepairRunId(), diff --git a/src/server/src/main/java/io/cassandrareaper/storage/cassandra/CassandraStorageFacade.java b/src/server/src/main/java/io/cassandrareaper/storage/cassandra/CassandraStorageFacade.java index 7d2906865..13fb1c89e 100644 --- a/src/server/src/main/java/io/cassandrareaper/storage/cassandra/CassandraStorageFacade.java +++ b/src/server/src/main/java/io/cassandrareaper/storage/cassandra/CassandraStorageFacade.java @@ -454,7 +454,7 @@ public enum CassandraMode { *

* Writes keep retrying forever. */ - private static class RetryPolicyImpl implements RetryPolicy { + private static final class RetryPolicyImpl implements RetryPolicy { @Override public RetryDecision onReadTimeout( diff --git a/src/server/src/main/java/io/cassandrareaper/storage/metrics/CassandraMetricsDao.java b/src/server/src/main/java/io/cassandrareaper/storage/metrics/CassandraMetricsDao.java index 73da3fa50..6f81174a1 100644 --- a/src/server/src/main/java/io/cassandrareaper/storage/metrics/CassandraMetricsDao.java +++ b/src/server/src/main/java/io/cassandrareaper/storage/metrics/CassandraMetricsDao.java @@ -89,7 +89,8 @@ void prepareMetricStatements() { storePercentRepairedForSchedulePrepStmt = session .prepare( "INSERT INTO percent_repaired_by_schedule" - + " (cluster_name, repair_schedule_id, time_bucket, node, keyspace_name, table_name, percent_repaired, ts)" + + " (cluster_name, repair_schedule_id, time_bucket, node, keyspace_name," + + " table_name, percent_repaired, ts)" + " values(?, ?, ?, ?, ?, ?, ?, ?)" ); @@ -207,7 +208,6 @@ public void purgeMetrics() { @Override public List getPercentRepairedMetrics(String clusterName, UUID repairScheduleId, Long since) { - List metrics = Lists.newArrayList(); List futures = Lists.newArrayList(); List timeBuckets = Lists.newArrayList(); long now = DateTime.now().getMillis(); @@ -229,6 +229,7 @@ public List getPercentRepairedMetrics(String clusterName, timeBucket))); } + List metrics = Lists.newArrayList(); long maxTimeBucket = 0; for (ResultSetFuture future : futures) { for (Row row : future.getUninterruptibly()) { diff --git a/src/server/src/main/java/io/cassandrareaper/storage/repairrun/CassandraRepairRunDao.java b/src/server/src/main/java/io/cassandrareaper/storage/repairrun/CassandraRepairRunDao.java index f1671d78d..fdd025c25 100644 --- a/src/server/src/main/java/io/cassandrareaper/storage/repairrun/CassandraRepairRunDao.java +++ b/src/server/src/main/java/io/cassandrareaper/storage/repairrun/CassandraRepairRunDao.java @@ -215,7 +215,6 @@ public RepairRun addRepairRun( nbRanges = 0; } } - assert cassRepairUnitDao.getRepairUnit(newRepairRun.getRepairUnitId()).getIncrementalRepair() == isIncremental; futures.add(this.session.executeAsync(repairRunBatch)); futures.add( diff --git a/src/server/src/main/java/io/cassandrareaper/storage/repairrun/MemoryRepairRunDao.java b/src/server/src/main/java/io/cassandrareaper/storage/repairrun/MemoryRepairRunDao.java index 0f1dbffdb..ee3ccd366 100644 --- a/src/server/src/main/java/io/cassandrareaper/storage/repairrun/MemoryRepairRunDao.java +++ b/src/server/src/main/java/io/cassandrareaper/storage/repairrun/MemoryRepairRunDao.java @@ -84,6 +84,7 @@ public Collection getClusterRunStatuses(String clusterName, int run.getPauseTime(), run.getIntensity(), unit.getIncrementalRepair(), + unit.getSubrangeIncrementalRepair(), run.getRepairParallelism(), unit.getNodes(), unit.getDatacenters(), diff --git a/src/server/src/main/java/io/cassandrareaper/storage/repairunit/CassandraRepairUnitDao.java b/src/server/src/main/java/io/cassandrareaper/storage/repairunit/CassandraRepairUnitDao.java index f9a1eff2c..8ee41e81a 100644 --- a/src/server/src/main/java/io/cassandrareaper/storage/repairunit/CassandraRepairUnitDao.java +++ b/src/server/src/main/java/io/cassandrareaper/storage/repairunit/CassandraRepairUnitDao.java @@ -64,8 +64,9 @@ private void prepareStatements() { insertRepairUnitPrepStmt = session .prepare( "INSERT INTO repair_unit_v1(id, cluster_name, keyspace_name, column_families, " - + "incremental_repair, nodes, \"datacenters\", blacklisted_tables, repair_thread_count, timeout) " - + "VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") + + "incremental_repair, subrange_incremental, nodes, \"datacenters\", blacklisted_tables," + + "repair_thread_count, timeout) " + + "VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") .setConsistencyLevel(ConsistencyLevel.QUORUM); getRepairUnitPrepStmt = session .prepare("SELECT * FROM repair_unit_v1 WHERE id = ?") @@ -91,6 +92,7 @@ public void updateRepairUnit(RepairUnit updatedRepairUnit) { updatedRepairUnit.getKeyspaceName(), updatedRepairUnit.getColumnFamilies(), updatedRepairUnit.getIncrementalRepair(), + updatedRepairUnit.getSubrangeIncrementalRepair(), updatedRepairUnit.getNodes(), updatedRepairUnit.getDatacenters(), updatedRepairUnit.getBlacklistedTables(), @@ -106,6 +108,7 @@ private RepairUnit getRepairUnitImpl(UUID id) { .keyspaceName(repairUnitRow.getString("keyspace_name")) .columnFamilies(repairUnitRow.getSet("column_families", String.class)) .incrementalRepair(repairUnitRow.getBool("incremental_repair")) + .subrangeIncrementalRepair(repairUnitRow.getBool("subrange_incremental")) .nodes(repairUnitRow.getSet("nodes", String.class)) .datacenters(repairUnitRow.getSet("datacenters", String.class)) .blacklistedTables(repairUnitRow.getSet("blacklisted_tables", String.class)) @@ -134,6 +137,7 @@ public Optional getRepairUnit(RepairUnit.Builder params) { .keyspaceName(repairUnitRow.getString("keyspace_name")) .columnFamilies(repairUnitRow.getSet("column_families", String.class)) .incrementalRepair(repairUnitRow.getBool("incremental_repair")) + .subrangeIncrementalRepair(repairUnitRow.getBool("subrange_incremental")) .nodes(repairUnitRow.getSet("nodes", String.class)) .datacenters(repairUnitRow.getSet("datacenters", String.class)) .blacklistedTables(repairUnitRow.getSet("blacklisted_tables", String.class)) diff --git a/src/server/src/main/java/io/cassandrareaper/storage/repairunit/MemoryRepairUnitDao.java b/src/server/src/main/java/io/cassandrareaper/storage/repairunit/MemoryRepairUnitDao.java index 16a9964e1..d845b03ad 100644 --- a/src/server/src/main/java/io/cassandrareaper/storage/repairunit/MemoryRepairUnitDao.java +++ b/src/server/src/main/java/io/cassandrareaper/storage/repairunit/MemoryRepairUnitDao.java @@ -43,7 +43,8 @@ public MemoryRepairUnitDao(MemoryStorageFacade storage) { @Override public RepairUnit addRepairUnit(RepairUnit.Builder repairUnitBuilder) { Optional existing = getRepairUnit(repairUnitBuilder); - if (existing.isPresent() && repairUnitBuilder.incrementalRepair == existing.get().getIncrementalRepair()) { + if (existing.isPresent() && repairUnitBuilder.incrementalRepair == existing.get().getIncrementalRepair() + && repairUnitBuilder.subrangeIncrementalRepair == existing.get().getSubrangeIncrementalRepair()) { return existing.get(); } else { RepairUnit newRepairUnit = repairUnitBuilder.build(UUIDs.timeBased()); diff --git a/src/server/src/main/resources/db/astra/009_subrange_incremental.cql b/src/server/src/main/resources/db/astra/009_subrange_incremental.cql new file mode 100644 index 000000000..49d40eb13 --- /dev/null +++ b/src/server/src/main/resources/db/astra/009_subrange_incremental.cql @@ -0,0 +1,18 @@ +-- +-- Copyright 2024-2024 Datastax inc. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Add a new column to the repair_unit_v1 table to enable subrange incremental repairs + +ALTER TABLE repair_unit_v1 ADD subrange_incremental boolean; diff --git a/src/server/src/main/resources/db/cassandra/033_subrange_incremental.cql b/src/server/src/main/resources/db/cassandra/033_subrange_incremental.cql new file mode 100644 index 000000000..49d40eb13 --- /dev/null +++ b/src/server/src/main/resources/db/cassandra/033_subrange_incremental.cql @@ -0,0 +1,18 @@ +-- +-- Copyright 2024-2024 Datastax inc. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Add a new column to the repair_unit_v1 table to enable subrange incremental repairs + +ALTER TABLE repair_unit_v1 ADD subrange_incremental boolean; diff --git a/src/server/src/test/java/io/cassandrareaper/ReaperApplicationConfigurationTest.java b/src/server/src/test/java/io/cassandrareaper/ReaperApplicationConfigurationTest.java index 4645e4b0c..2eb079bdf 100644 --- a/src/server/src/test/java/io/cassandrareaper/ReaperApplicationConfigurationTest.java +++ b/src/server/src/test/java/io/cassandrareaper/ReaperApplicationConfigurationTest.java @@ -51,6 +51,7 @@ public void setUp() { config.setScheduleDaysBetween(7); config.setStorageType("foo"); config.setIncrementalRepair(false); + config.setSubrangeIncrementalRepair(false); config.setBlacklistTwcsTables(true); } diff --git a/src/server/src/test/java/io/cassandrareaper/SimpleReaperClient.java b/src/server/src/test/java/io/cassandrareaper/SimpleReaperClient.java index 3a2eea87f..211f77d75 100644 --- a/src/server/src/test/java/io/cassandrareaper/SimpleReaperClient.java +++ b/src/server/src/test/java/io/cassandrareaper/SimpleReaperClient.java @@ -34,7 +34,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; - import javax.ws.rs.client.Client; import javax.ws.rs.client.ClientBuilder; import javax.ws.rs.client.Entity; @@ -48,7 +47,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.datatype.joda.JodaModule; import com.google.common.collect.Maps; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/server/src/test/java/io/cassandrareaper/acceptance/BasicSteps.java b/src/server/src/test/java/io/cassandrareaper/acceptance/BasicSteps.java index cb40d02a5..d1cf856b8 100644 --- a/src/server/src/test/java/io/cassandrareaper/acceptance/BasicSteps.java +++ b/src/server/src/test/java/io/cassandrareaper/acceptance/BasicSteps.java @@ -1336,6 +1336,27 @@ public void a_new_incremental_repair_is_added_for_the_last_added_cluster_and_key } } + @When("^a new subrange incremental repair is added for the last added cluster and keyspace \"([^\"]*)\"$") + public void a_new_subrange_incremental_repair_is_added_for_the_last_added_cluster_and_keyspace(String keyspace) + throws Throwable { + synchronized (BasicSteps.class) { + ReaperTestJettyRunner runner = RUNNERS.get(RAND.nextInt(RUNNERS.size())); + Map params = Maps.newHashMap(); + params.put("clusterName", TestContext.TEST_CLUSTER); + params.put("keyspace", keyspace); + params.put("owner", TestContext.TEST_USER); + params.put("incrementalRepair", Boolean.FALSE.toString()); + params.put("subrangeIncrementalRepair", Boolean.TRUE.toString()); + Response response = runner.callReaper("POST", "/repair_run", Optional.of(params)); + String responseData = response.readEntity(String.class); + assertEquals(responseData, Response.Status.CREATED.getStatusCode(), response.getStatus()); + Assertions.assertThat(responseData).isNotBlank(); + RepairRunStatus run = SimpleReaperClient.parseRepairRunStatusJSON(responseData); + Assertions.assertThat(run.getTotalSegments()).isGreaterThan(3); + testContext.addCurrentRepairId(run.getId()); + } + } + @Then("^reaper has (\\d+) repairs for cluster called \"([^\"]*)\"$") public void reaper_has_repairs_for_cluster_called(int expected, String clusterName) throws Throwable { synchronized (BasicSteps.class) { diff --git a/src/server/src/test/java/io/cassandrareaper/acceptance/ReaperCassandraEachIT.java b/src/server/src/test/java/io/cassandrareaper/acceptance/ReaperCassandraEachIT.java index dc7519e94..f4598ef7d 100644 --- a/src/server/src/test/java/io/cassandrareaper/acceptance/ReaperCassandraEachIT.java +++ b/src/server/src/test/java/io/cassandrareaper/acceptance/ReaperCassandraEachIT.java @@ -45,10 +45,10 @@ public class ReaperCassandraEachIT { private static final Logger LOG = LoggerFactory.getLogger(ReaperCassandraSidecarIT.class); private static final List RUNNER_INSTANCES = new CopyOnWriteArrayList<>(); private static final String[] CASS_CONFIG_FILE - = { - "reaper-cassandra-each1-at.yaml", - "reaper-cassandra-each2-at.yaml" - }; + = { + "reaper-cassandra-each1-at.yaml", + "reaper-cassandra-each2-at.yaml" + }; protected ReaperCassandraEachIT() {} diff --git a/src/server/src/test/java/io/cassandrareaper/acceptance/ReaperCassandraIT.java b/src/server/src/test/java/io/cassandrareaper/acceptance/ReaperCassandraIT.java index 0baaea5bd..d6be6a992 100644 --- a/src/server/src/test/java/io/cassandrareaper/acceptance/ReaperCassandraIT.java +++ b/src/server/src/test/java/io/cassandrareaper/acceptance/ReaperCassandraIT.java @@ -25,7 +25,6 @@ import com.datastax.driver.core.Cluster; import com.datastax.driver.core.Session; import com.datastax.driver.core.SocketOptions; - import cucumber.api.CucumberOptions; import cucumber.api.junit.Cucumber; import org.junit.AfterClass; diff --git a/src/server/src/test/java/io/cassandrareaper/acceptance/ReaperCassandraSidecarIT.java b/src/server/src/test/java/io/cassandrareaper/acceptance/ReaperCassandraSidecarIT.java index 834ec36db..59c6865d8 100644 --- a/src/server/src/test/java/io/cassandrareaper/acceptance/ReaperCassandraSidecarIT.java +++ b/src/server/src/test/java/io/cassandrareaper/acceptance/ReaperCassandraSidecarIT.java @@ -46,12 +46,12 @@ public class ReaperCassandraSidecarIT { private static final Logger LOG = LoggerFactory.getLogger(ReaperCassandraSidecarIT.class); private static final List RUNNER_INSTANCES = new CopyOnWriteArrayList<>(); private static final String[] CASS_CONFIG_FILE - = { - "reaper-cassandra-sidecar1-at.yaml", - "reaper-cassandra-sidecar2-at.yaml", - "reaper-cassandra-sidecar3-at.yaml", - "reaper-cassandra-sidecar4-at.yaml", - }; + = { + "reaper-cassandra-sidecar1-at.yaml", + "reaper-cassandra-sidecar2-at.yaml", + "reaper-cassandra-sidecar3-at.yaml", + "reaper-cassandra-sidecar4-at.yaml", + }; protected ReaperCassandraSidecarIT() {} diff --git a/src/server/src/test/java/io/cassandrareaper/acceptance/ReaperHttpIT.java b/src/server/src/test/java/io/cassandrareaper/acceptance/ReaperHttpIT.java index cf6069766..c7a8e3ef7 100644 --- a/src/server/src/test/java/io/cassandrareaper/acceptance/ReaperHttpIT.java +++ b/src/server/src/test/java/io/cassandrareaper/acceptance/ReaperHttpIT.java @@ -25,7 +25,6 @@ import com.datastax.driver.core.Cluster; import com.datastax.driver.core.Session; import com.datastax.driver.core.SocketOptions; - import cucumber.api.CucumberOptions; import cucumber.api.junit.Cucumber; import org.junit.AfterClass; diff --git a/src/server/src/test/java/io/cassandrareaper/acceptance/ReaperTestJettyRunner.java b/src/server/src/test/java/io/cassandrareaper/acceptance/ReaperTestJettyRunner.java index 56ebcffd0..78e232f70 100644 --- a/src/server/src/test/java/io/cassandrareaper/acceptance/ReaperTestJettyRunner.java +++ b/src/server/src/test/java/io/cassandrareaper/acceptance/ReaperTestJettyRunner.java @@ -27,11 +27,9 @@ import java.util.Map; import java.util.Optional; import java.util.Set; - import javax.ws.rs.core.Response; import com.google.common.collect.Sets; - import io.dropwizard.testing.ConfigOverride; import io.dropwizard.testing.DropwizardTestSupport; import io.dropwizard.testing.ResourceHelpers; diff --git a/src/server/src/test/java/io/cassandrareaper/acceptance/TestContext.java b/src/server/src/test/java/io/cassandrareaper/acceptance/TestContext.java index 8f15db987..6979b1d08 100644 --- a/src/server/src/test/java/io/cassandrareaper/acceptance/TestContext.java +++ b/src/server/src/test/java/io/cassandrareaper/acceptance/TestContext.java @@ -27,7 +27,6 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.locks.ReentrantReadWriteLock; - import javax.ws.rs.sse.InboundSseEvent; import javax.ws.rs.sse.SseEventSource; diff --git a/src/server/src/test/java/io/cassandrareaper/management/http/HttpCassandraManagementProxyTest.java b/src/server/src/test/java/io/cassandrareaper/management/http/HttpCassandraManagementProxyTest.java index 8be9b697e..19f1ea7a7 100644 --- a/src/server/src/test/java/io/cassandrareaper/management/http/HttpCassandraManagementProxyTest.java +++ b/src/server/src/test/java/io/cassandrareaper/management/http/HttpCassandraManagementProxyTest.java @@ -22,6 +22,7 @@ import io.cassandrareaper.ReaperException; import io.cassandrareaper.core.GenericMetric; import io.cassandrareaper.core.Node; +import io.cassandrareaper.core.RepairType; import io.cassandrareaper.core.Snapshot; import io.cassandrareaper.core.Table; import io.cassandrareaper.management.RepairStatusHandler; @@ -155,7 +156,12 @@ public void testRepairProcessMapHandlers() throws Exception { int repairNo = httpCassandraManagementProxy.triggerRepair("ks", RepairParallelism.PARALLEL, - Collections.singleton("table"), true, Collections.emptyList(), repairStatusHandler, Collections.emptyList(), 1); + Collections.singleton("table"), + RepairType.SUBRANGE_FULL, + Collections.emptyList(), + repairStatusHandler, + Collections.emptyList(), + 1); assertEquals(123456789, repairNo); assertEquals(1, httpCassandraManagementProxy.jobTracker.size()); @@ -192,7 +198,7 @@ public void testNotificationsTracker() throws Exception { int repairNo = httpCassandraManagementProxy.triggerRepair("ks", RepairParallelism.PARALLEL, - Collections.singleton("table"), true, Collections.emptyList(), workAroundHandler, + Collections.singleton("table"), RepairType.SUBRANGE_FULL, Collections.emptyList(), workAroundHandler, Collections.emptyList(), 1); verify(mockClient).putRepairV2(eq( diff --git a/src/server/src/test/java/io/cassandrareaper/management/http/HttpMetricsProxyTest.java b/src/server/src/test/java/io/cassandrareaper/management/http/HttpMetricsProxyTest.java index ba224fd21..3a4f52573 100644 --- a/src/server/src/test/java/io/cassandrareaper/management/http/HttpMetricsProxyTest.java +++ b/src/server/src/test/java/io/cassandrareaper/management/http/HttpMetricsProxyTest.java @@ -28,7 +28,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; - import javax.management.JMException; import com.datastax.mgmtapi.client.api.DefaultApi; @@ -38,7 +37,6 @@ import okhttp3.OkHttpClient; import okhttp3.Response; import okhttp3.ResponseBody; - import org.junit.Test; import org.mockito.Mockito; import org.mockito.internal.util.collections.Sets; @@ -261,12 +259,29 @@ public void testParseThreadPoolMetric_withoutMatch() { @SuppressWarnings("checkstyle:LineLength") @Test public void testCollectTpStats() throws IOException, JMException { - String responseBodyStr = "org_apache_cassandra_metrics_thread_pools_pending_tasks{host=\"eae3481e-f803-49b5-8516-7ff271104db5\",instance=\"172.18.0.3\",cluster=\"test\",datacenter=\"dc1\",rack=\"default\",pod_name=\"test-dc1-default-sts-0\",node_name=\"mc-0-worker3\",pool_type=\"internal\",pool_name=\"TPC\",} 0.0\n" - + "org_apache_cassandra_metrics_thread_pools_pending_tasks{host=\"eae3481e-f803-49b5-8516-7ff271104db5\",instance=\"172.18.0.3\",cluster=\"test\",datacenter=\"dc1\",rack=\"default\",pod_name=\"test-dc1-default-sts-0\",node_name=\"mc-0-worker3\",pool_type=\"internal\",pool_name=\"ValidationExecutor\",} 0.0\n" - + "org_apache_cassandra_metrics_thread_pools_pending_tasks{host=\"eae3481e-f803-49b5-8516-7ff271104db5\",instance=\"172.18.0.3\",cluster=\"test\",datacenter=\"dc1\",rack=\"default\",pod_name=\"test-dc1-default-sts-0\",node_name=\"mc-0-worker3\",pool_type=\"internal\",pool_name=\"AntiCompactionExecutor\",} 0.0\n" - + "org_apache_cassandra_metrics_thread_pools_pending_tasks{host=\"eae3481e-f803-49b5-8516-7ff271104db5\",instance=\"172.18.0.3\",cluster=\"test\",datacenter=\"dc1\",rack=\"default\",pod_name=\"test-dc1-default-sts-0\",node_name=\"mc-0-worker3\",pool_type=\"internal\",pool_name=\"AntiEntropyStage\",} 0.0\n" - + "org_apache_cassandra_metrics_thread_pools_pending_tasks{host=\"eae3481e-f803-49b5-8516-7ff271104db5\",instance=\"172.18.0.3\",cluster=\"test\",datacenter=\"dc1\",rack=\"default\",pod_name=\"test-dc1-default-sts-0\",node_name=\"mc-0-worker3\",pool_type=\"internal\",pool_name=\"TPC\",} 0.0\n" - + "org_apache_cassandra_metrics_keyspace_view_read_time_solr_admin_bucket{host=\"eae3481e-f803-49b5-8516-7ff271104db5\",instance=\"172.18.0.3\",cluster=\"test\",datacenter=\"dc1\",rack=\"default\",pod_name=\"test-dc1-default-sts-0\",node_name=\"mc-0-worker3\",le=\"3379391\",} 0.0\n"; + String responseBodyStr = "org_apache_cassandra_metrics_thread_pools_pending_tasks{" + + "host=\"eae3481e-f803-49b5-8516-7ff271104db5\",instance=\"172.18.0.3\",cluster=\"test\"," + + "datacenter=\"dc1\",rack=\"default\",pod_name=\"test-dc1-default-sts-0\",node_name=\"mc-0-worker3\"," + + "pool_type=\"internal\",pool_name=\"TPC\",} 0.0\n" + + "org_apache_cassandra_metrics_thread_pools_pending_tasks{host=\"eae3481e-f803-49b5-8516-7ff271104db5\"," + + "instance=\"172.18.0.3\",cluster=\"test\",datacenter=\"dc1\",rack=\"default\"," + + "pod_name=\"test-dc1-default-sts-0\",node_name=\"mc-0-worker3\",pool_type=\"internal\"," + + "pool_name=\"ValidationExecutor\",} 0.0\n" + + "org_apache_cassandra_metrics_thread_pools_pending_tasks{host=\"eae3481e-f803-49b5-8516-7ff271104db5\"," + + "instance=\"172.18.0.3\",cluster=\"test\",datacenter=\"dc1\",rack=\"default\"," + + "pod_name=\"test-dc1-default-sts-0\",node_name=\"mc-0-worker3\",pool_type=\"internal\"," + + "pool_name=\"AntiCompactionExecutor\",} 0.0\n" + + "org_apache_cassandra_metrics_thread_pools_pending_tasks{host=\"eae3481e-f803-49b5-8516-7ff271104db5\"," + + "instance=\"172.18.0.3\",cluster=\"test\",datacenter=\"dc1\",rack=\"default\"," + + "pod_name=\"test-dc1-default-sts-0\",node_name=\"mc-0-worker3\",pool_type=\"internal\"," + + "pool_name=\"AntiEntropyStage\",} 0.0\n" + + "org_apache_cassandra_metrics_thread_pools_pending_tasks{host=\"eae3481e-f803-49b5-8516-7ff271104db5\"," + + "instance=\"172.18.0.3\",cluster=\"test\",datacenter=\"dc1\",rack=\"default\"," + + "pod_name=\"test-dc1-default-sts-0\",node_name=\"mc-0-worker3\",pool_type=\"internal\"," + + "pool_name=\"TPC\",} 0.0\n" + + "org_apache_cassandra_metrics_keyspace_view_read_time_solr_admin_bucket{" + + "host=\"eae3481e-f803-49b5-8516-7ff271104db5\",instance=\"172.18.0.3\",cluster=\"test\",datacenter=\"dc1\"," + + "rack=\"default\",pod_name=\"test-dc1-default-sts-0\",node_name=\"mc-0-worker3\",le=\"3379391\",} 0.0\n"; OkHttpClient httpClient = Mockito.mock(OkHttpClient.class); Call call = Mockito.mock(Call.class); @@ -304,13 +319,34 @@ public void testCollectTpStats() throws IOException, JMException { @SuppressWarnings("checkstyle:LineLength") @Test public void testCollectTpPendingTasks() throws JMException, IOException, ReaperException { - String responseBodyStr = "org_apache_cassandra_metrics_thread_pools_pending_tasks{host=\"eae3481e-f803-49b5-8516-7ff271104db5\",instance=\"172.18.0.3\",cluster=\"test\",datacenter=\"dc1\",rack=\"default\",pod_name=\"test-dc1-default-sts-0\",node_name=\"mc-0-worker3\",pool_type=\"internal\",pool_name=\"TPC\",} 0.0\n" - + "org_apache_cassandra_metrics_thread_pools_pending_tasks{host=\"eae3481e-f803-49b5-8516-7ff271104db5\",instance=\"172.18.0.3\",cluster=\"test\",datacenter=\"dc1\",rack=\"default\",pod_name=\"test-dc1-default-sts-0\",node_name=\"mc-0-worker3\",pool_type=\"internal\",pool_name=\"ValidationExecutor\",} 0.0\n" - + "org_apache_cassandra_metrics_thread_pools_pending_tasks{host=\"eae3481e-f803-49b5-8516-7ff271104db5\",instance=\"172.18.0.3\",cluster=\"test\",datacenter=\"dc1\",rack=\"default\",pod_name=\"test-dc1-default-sts-0\",node_name=\"mc-0-worker3\",pool_type=\"internal\",pool_name=\"AntiCompactionExecutor\",} 0.0\n" - + "org_apache_cassandra_metrics_thread_pools_pending_tasks{host=\"eae3481e-f803-49b5-8516-7ff271104db5\",instance=\"172.18.0.3\",cluster=\"test\",datacenter=\"dc1\",rack=\"default\",pod_name=\"test-dc1-default-sts-0\",node_name=\"mc-0-worker3\",pool_type=\"internal\",pool_name=\"AntiEntropyStage\",} 0.0\n" - + "org_apache_cassandra_metrics_thread_pools_pending_tasks{host=\"eae3481e-f803-49b5-8516-7ff271104db5\",instance=\"172.18.0.3\",cluster=\"test\",datacenter=\"dc1\",rack=\"default\",pod_name=\"test-dc1-default-sts-0\",node_name=\"mc-0-worker3\",pool_type=\"internal\",pool_name=\"TPC\",} 0.0\n" - + "org_apache_cassandra_metrics_thread_pools_pending_tasks{host=\"eae3481e-f803-49b5-8516-7ff271104db5\",instance=\"172.18.0.3\",cluster=\"test\",datacenter=\"dc1\",rack=\"default\",pod_name=\"test-dc1-default-sts-0\",node_name=\"mc-0-worker3\",pool_type=\"internal\",pool_name=\"CompactionExecutor\",} 0.0\n" - + "org_apache_cassandra_metrics_keyspace_view_read_time_solr_admin_bucket{host=\"eae3481e-f803-49b5-8516-7ff271104db5\",instance=\"172.18.0.3\",cluster=\"test\",datacenter=\"dc1\",rack=\"default\",pod_name=\"test-dc1-default-sts-0\",node_name=\"mc-0-worker3\",le=\"3379391\",} 0.0\n"; + String responseBodyStr = "org_apache_cassandra_metrics_thread_pools_pending_tasks{" + + "host=\"eae3481e-f803-49b5-8516-7ff271104db5\",instance=\"172.18.0.3\",cluster=\"test\",datacenter=\"dc1\"," + + "rack=\"default\",pod_name=\"test-dc1-default-sts-0\",node_name=\"mc-0-worker3\",pool_type=\"internal\"," + + "pool_name=\"TPC\",} 0.0\n" + + "org_apache_cassandra_metrics_thread_pools_pending_tasks{host=\"eae3481e-f803-49b5-8516-7ff271104db5\"," + + "instance=\"172.18.0.3\",cluster=\"test\",datacenter=\"dc1\",rack=\"default\"," + + "pod_name=\"test-dc1-default-sts-0\",node_name=\"mc-0-worker3\",pool_type=\"internal\"," + + "pool_name=\"ValidationExecutor\",} 0.0\n" + + "org_apache_cassandra_metrics_thread_pools_pending_tasks{host=\"eae3481e-f803-49b5-8516-7ff271104db5\"," + + "instance=\"172.18.0.3\",cluster=\"test\",datacenter=\"dc1\",rack=\"default\"," + + "pod_name=\"test-dc1-default-sts-0\",node_name=\"mc-0-worker3\",pool_type=\"internal\"," + + "pool_name=\"AntiCompactionExecutor\",} 0.0\n" + + "org_apache_cassandra_metrics_thread_pools_pending_tasks{host=\"eae3481e-f803-49b5-8516-7ff271104db5\"," + + "instance=\"172.18.0.3\",cluster=\"test\",datacenter=\"dc1\",rack=\"default\"," + + "pod_name=\"test-dc1-default-sts-0\",node_name=\"mc-0-worker3\",pool_type=\"internal\"," + + "pool_name=\"AntiEntropyStage\",} 0.0\n" + + "org_apache_cassandra_metrics_thread_pools_pending_tasks{host=\"eae3481e-f803-49b5-8516-7ff271104db5\"," + + "instance=\"172.18.0.3\",cluster=\"test\",datacenter=\"dc1\",rack=\"default\"," + + "pod_name=\"test-dc1-default-sts-0\",node_name=\"mc-0-worker3\",pool_type=\"internal\"," + + "pool_name=\"TPC\",} 0.0\n" + + "org_apache_cassandra_metrics_thread_pools_pending_tasks{host=\"eae3481e-f803-49b5-8516-7ff271104db5\"," + + "instance=\"172.18.0.3\",cluster=\"test\",datacenter=\"dc1\",rack=\"default\"," + + "pod_name=\"test-dc1-default-sts-0\",node_name=\"mc-0-worker3\",pool_type=\"internal\"," + + "pool_name=\"CompactionExecutor\",} 0.0\n" + + "org_apache_cassandra_metrics_keyspace_view_read_time_solr_admin_bucket{" + + "host=\"eae3481e-f803-49b5-8516-7ff271104db5\",instance=\"172.18.0.3\",cluster=\"test\"," + + "datacenter=\"dc1\",rack=\"default\",pod_name=\"test-dc1-default-sts-0\",node_name=\"mc-0-worker3\"," + + "le=\"3379391\",} 0.0\n"; OkHttpClient httpClient = Mockito.mock(OkHttpClient.class); Call call = Mockito.mock(Call.class); diff --git a/src/server/src/test/java/io/cassandrareaper/resources/RepairRunResourceTest.java b/src/server/src/test/java/io/cassandrareaper/resources/RepairRunResourceTest.java index e6d99bf3f..af9a5ddae 100644 --- a/src/server/src/test/java/io/cassandrareaper/resources/RepairRunResourceTest.java +++ b/src/server/src/test/java/io/cassandrareaper/resources/RepairRunResourceTest.java @@ -71,7 +71,6 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyCollection; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; @@ -159,7 +158,7 @@ public void setUp() throws Exception { anyString(), any(RepairParallelism.class), anyCollection(), - anyBoolean(), + any(), anyCollection(), any(), any(), @@ -186,6 +185,7 @@ public void setUp() throws Exception { .keyspaceName(keyspace) .columnFamilies(TABLES) .incrementalRepair(INCREMENTAL) + .subrangeIncrementalRepair(INCREMENTAL) .nodes(NODES) .datacenters(DATACENTERS) .blacklistedTables(BLACKLISTED_TABLES) @@ -238,6 +238,7 @@ private Response addRepairRun( Optional.of(REPAIR_PARALLELISM.name()), Optional.empty(), Optional.empty(), + Optional.empty(), nodes.isEmpty() ? Optional.empty() : Optional.of(StringUtils.join(nodes, ',')), Optional.empty(), blacklistedTables.isEmpty() ? Optional.empty() : Optional.of(StringUtils.join(blacklistedTables, ',')), @@ -562,6 +563,7 @@ private Response addRepairRunWithForceParam( Optional.of(REPAIR_PARALLELISM.name()), Optional.empty(), Optional.empty(), + Optional.empty(), nodes.isEmpty() ? Optional.empty() : Optional.of(StringUtils.join(nodes, ',')), Optional.empty(), blacklistedTables.isEmpty() ? Optional.empty() : Optional.of(StringUtils.join(blacklistedTables, ',')), @@ -593,4 +595,53 @@ public void testAddRunMalformedForceParam() { assertTrue(response.getEntity() instanceof String); assertEquals("invalid query parameter \"force\", expecting [True,False]", response.getEntity()); } + + @Test + public void testGetSegments() { + RepairRunResource repairRunResource = new RepairRunResource(context, context.storage.getRepairRunDao()); + // subrange incremental + int segments = repairRunResource.getSegments(Optional.of(10), true, true); + assertEquals(10, segments); + // non-subrange incremental + segments = repairRunResource.getSegments(Optional.of(10), false, true); + assertEquals(-1, segments); + // non-incremental + segments = repairRunResource.getSegments(Optional.of(10), false, false); + assertEquals(10, segments); + } + + @Test + public void testGetIncrementalRepair() { + AppContext ctx = mock(AppContext.class); + ctx.config = mock(ReaperApplicationConfiguration.class); + ctx.storage = new MemoryStorageFacade(); + RepairRunResource repairRunResource = new RepairRunResource(ctx, ctx.storage.getRepairRunDao()); + + boolean incrementalRepair = repairRunResource.getIncrementalRepair(Optional.of("false")); + assertFalse(incrementalRepair); + incrementalRepair = repairRunResource.getIncrementalRepair(Optional.of("true")); + assertTrue(incrementalRepair); + incrementalRepair = repairRunResource.getIncrementalRepair(Optional.empty()); + assertFalse(incrementalRepair); + } + + @Test + public void testGetSubrangeIncrementalRepair() { + AppContext ctx = mock(AppContext.class); + ctx.config = mock(ReaperApplicationConfiguration.class); + ctx.storage = new MemoryStorageFacade(); + RepairRunResource repairRunResource = new RepairRunResource(ctx, ctx.storage.getRepairRunDao()); + boolean subrangeIncremental = repairRunResource.getSubrangeIncrementalRepair(Optional.of("false")); + assertFalse(subrangeIncremental); + } + + @Test + public void testGetIntensity() { + AppContext ctx = mock(AppContext.class); + ctx.config = mock(ReaperApplicationConfiguration.class); + ctx.storage = new MemoryStorageFacade(); + RepairRunResource repairRunResource = new RepairRunResource(ctx, ctx.storage.getRepairRunDao()); + Double intensity = repairRunResource.getIntensity(Optional.of("0.8")); + assertEquals(0.8f, intensity.floatValue(), 0.0f); + } } \ No newline at end of file diff --git a/src/server/src/test/java/io/cassandrareaper/resources/RepairScheduleResourceTest.java b/src/server/src/test/java/io/cassandrareaper/resources/RepairScheduleResourceTest.java index 11e385030..a4e14ff1b 100644 --- a/src/server/src/test/java/io/cassandrareaper/resources/RepairScheduleResourceTest.java +++ b/src/server/src/test/java/io/cassandrareaper/resources/RepairScheduleResourceTest.java @@ -92,6 +92,7 @@ private static MockObjects initInMemoryMocks(URI uri) { RepairUnit.Builder mockRepairUnitBuilder = RepairUnit.builder() .incrementalRepair(false) + .subrangeIncrementalRepair(false) .repairThreadCount(1) .clusterName("cluster-test") .keyspaceName("keyspace-test") diff --git a/src/server/src/test/java/io/cassandrareaper/resources/view/RepairRunStatusTest.java b/src/server/src/test/java/io/cassandrareaper/resources/view/RepairRunStatusTest.java index e43a41a7a..8c358b521 100644 --- a/src/server/src/test/java/io/cassandrareaper/resources/view/RepairRunStatusTest.java +++ b/src/server/src/test/java/io/cassandrareaper/resources/view/RepairRunStatusTest.java @@ -73,6 +73,7 @@ public void testRunningRepairDuration() { null, // pauseTime 0.9, // intensity false, // incremental + false, // subrange incremental RepairParallelism.PARALLEL, // repairParellelism Collections.EMPTY_LIST, // nodes Collections.EMPTY_LIST, // datacenters @@ -104,6 +105,7 @@ public void testFinishedRepairDuration() { null, // pauseTime 0.9, // intensity false, // incremental + false, // subrange incremental RepairParallelism.PARALLEL, // repairParellelism Collections.EMPTY_LIST, // nodes Collections.EMPTY_LIST, // datacenters @@ -135,6 +137,7 @@ public void testPausedRepairDuration() { new DateTime().now().minusMinutes(1), // pauseTime 0.9, // intensity false, // incremental + false, // subrange incremental RepairParallelism.PARALLEL, // repairParellelism Collections.EMPTY_LIST, // nodes Collections.EMPTY_LIST, // datacenters @@ -166,6 +169,7 @@ public void testAbortedRepairDuration() { new DateTime().now().minusMinutes(1), // pauseTime 0.9, // intensity false, // incremental + false, // subrange incremental RepairParallelism.PARALLEL, // repairParellelism Collections.EMPTY_LIST, // nodes Collections.EMPTY_LIST, // datacenters diff --git a/src/server/src/test/java/io/cassandrareaper/resources/view/RepairScheduleStatusTest.java b/src/server/src/test/java/io/cassandrareaper/resources/view/RepairScheduleStatusTest.java index 43f3adf17..c98e94675 100644 --- a/src/server/src/test/java/io/cassandrareaper/resources/view/RepairScheduleStatusTest.java +++ b/src/server/src/test/java/io/cassandrareaper/resources/view/RepairScheduleStatusTest.java @@ -45,6 +45,7 @@ public void testJacksonJSONParsing() throws Exception { data.setId(UUIDs.timeBased()); data.setIntensity(0.75); data.setIncrementalRepair(false); + data.setSubrangeIncrementalRepair(false); data.setKeyspaceName("testKeyspace"); data.setOwner("testuser"); data.setRepairParallelism(RepairParallelism.PARALLEL); @@ -63,6 +64,7 @@ public void testJacksonJSONParsing() throws Exception { assertEquals(data.getId(), dataAfter.getId()); assertEquals(data.getIntensity(), dataAfter.getIntensity(), 0.0); assertEquals(data.getIncrementalRepair(), dataAfter.getIncrementalRepair()); + assertEquals(data.getSubrangeIncrementalRepair(), dataAfter.getSubrangeIncrementalRepair()); assertEquals(data.getKeyspaceName(), dataAfter.getKeyspaceName()); assertEquals(data.getRepairParallelism(), dataAfter.getRepairParallelism()); assertEquals(data.getState(), dataAfter.getState()); diff --git a/src/server/src/test/java/io/cassandrareaper/service/ClusterRepairSchedulerTest.java b/src/server/src/test/java/io/cassandrareaper/service/ClusterRepairSchedulerTest.java index 778a004c4..e4aede3cc 100644 --- a/src/server/src/test/java/io/cassandrareaper/service/ClusterRepairSchedulerTest.java +++ b/src/server/src/test/java/io/cassandrareaper/service/ClusterRepairSchedulerTest.java @@ -271,6 +271,7 @@ private RepairUnit.Builder oneRepair(Cluster cluster, String keyspace) { .clusterName(cluster.getName()) .keyspaceName(keyspace) .incrementalRepair(Boolean.FALSE) + .subrangeIncrementalRepair(Boolean.FALSE) .repairThreadCount(1) .timeout(30); } diff --git a/src/server/src/test/java/io/cassandrareaper/service/RepairManagerTest.java b/src/server/src/test/java/io/cassandrareaper/service/RepairManagerTest.java index 0c06d7ced..412191e9b 100644 --- a/src/server/src/test/java/io/cassandrareaper/service/RepairManagerTest.java +++ b/src/server/src/test/java/io/cassandrareaper/service/RepairManagerTest.java @@ -82,6 +82,7 @@ public void abortRunningSegmentWithNoLeader() throws ReaperException, Interrupte final String ksName = "reaper"; final Set cfNames = Sets.newHashSet("reaper"); final boolean incrementalRepair = false; + final boolean subrangeIncremental = false; final Set nodes = Sets.newHashSet("127.0.0.1"); final Set datacenters = Collections.emptySet(); final double intensity = 0.5f; @@ -96,6 +97,7 @@ public void abortRunningSegmentWithNoLeader() throws ReaperException, Interrupte .keyspaceName(ksName) .columnFamilies(cfNames) .incrementalRepair(incrementalRepair) + .subrangeIncrementalRepair(subrangeIncremental) .nodes(nodes) .datacenters(datacenters) .repairThreadCount(repairThreadCount) @@ -167,6 +169,7 @@ public void doNotAbortRunningSegmentWithLeader() throws ReaperException, Interru final String ksName = "reaper"; final Set cfNames = Sets.newHashSet("reaper"); final boolean incrementalRepair = false; + final boolean subrangeIncremental = false; final Set nodes = Sets.newHashSet("127.0.0.1"); final Set datacenters = Collections.emptySet(); final double intensity = 0.5f; @@ -187,7 +190,6 @@ public void doNotAbortRunningSegmentWithLeader() throws ReaperException, Interru AppContext context = new AppContext(); context.storage = storage; context.config = new ReaperApplicationConfiguration(); - RepairManager repairManager = RepairManager.create( context, Executors.newScheduledThreadPool(1), @@ -195,15 +197,14 @@ public void doNotAbortRunningSegmentWithLeader() throws ReaperException, Interru TimeUnit.MILLISECONDS, 1, context.storage.getRepairRunDao()); - repairManager = Mockito.spy(repairManager); context.repairManager = repairManager; - final RepairUnit cf = RepairUnit.builder() .clusterName(clusterName) .keyspaceName(ksName) .columnFamilies(cfNames) .incrementalRepair(incrementalRepair) + .subrangeIncrementalRepair(subrangeIncremental) .nodes(nodes) .datacenters(datacenters) .repairThreadCount(repairThreadCount) @@ -260,6 +261,7 @@ public void doNotAbortRunningSegmentWithRepairRunnerAndNoDistributedStorage() final String ksName = "reaper"; final Set cfNames = Sets.newHashSet("reaper"); final boolean incrementalRepair = false; + final boolean subrangeIncremental = false; final Set nodes = Sets.newHashSet("127.0.0.1"); final Set datacenters = Collections.emptySet(); final double intensity = 0.5f; @@ -279,6 +281,7 @@ public void doNotAbortRunningSegmentWithRepairRunnerAndNoDistributedStorage() .keyspaceName(ksName) .columnFamilies(cfNames) .incrementalRepair(incrementalRepair) + .subrangeIncrementalRepair(subrangeIncremental) .nodes(nodes) .datacenters(datacenters) .repairThreadCount(repairThreadCount) @@ -348,6 +351,7 @@ public void abortRunningSegmentWithNoRepairRunnerAndNoDistributedStorage() final String ksName = "reaper"; final Set cfNames = Sets.newHashSet("reaper"); final boolean incrementalRepair = false; + final boolean subrangeIncremental = false; final Set nodes = Sets.newHashSet("127.0.0.1"); final Set datacenters = Collections.emptySet(); final double intensity = 0.5f; @@ -382,6 +386,7 @@ public void abortRunningSegmentWithNoRepairRunnerAndNoDistributedStorage() .keyspaceName(ksName) .columnFamilies(cfNames) .incrementalRepair(incrementalRepair) + .subrangeIncrementalRepair(subrangeIncremental) .nodes(nodes) .datacenters(datacenters) .repairThreadCount(repairThreadCount) @@ -451,6 +456,7 @@ public void updateRepairRunIntensityTest() throws ReaperException, InterruptedEx final String ksName = "reaper"; final Set cfNames = Sets.newHashSet("reaper"); final boolean incrementalRepair = false; + final boolean subrangeIncremental = false; final Set nodes = Sets.newHashSet("127.0.0.1"); final Set datacenters = Collections.emptySet(); final int repairThreadCount = 1; @@ -461,6 +467,7 @@ public void updateRepairRunIntensityTest() throws ReaperException, InterruptedEx .keyspaceName(ksName) .columnFamilies(cfNames) .incrementalRepair(incrementalRepair) + .subrangeIncrementalRepair(subrangeIncremental) .nodes(nodes) .datacenters(datacenters) .repairThreadCount(repairThreadCount) @@ -568,6 +575,7 @@ private RepairUnit createRepairUnit(String clusterName) { final String ksName = "reaper"; final Set cfNames = Sets.newHashSet("reaper"); final boolean incrementalRepair = false; + final boolean subrangeIncremental = false; final Set nodes = Sets.newHashSet("127.0.0.1"); final Set datacenters = Collections.emptySet(); final int repairThreadCount = 1; @@ -578,6 +586,7 @@ private RepairUnit createRepairUnit(String clusterName) { .keyspaceName(ksName) .columnFamilies(cfNames) .incrementalRepair(incrementalRepair) + .subrangeIncrementalRepair(subrangeIncremental) .nodes(nodes) .datacenters(datacenters) .repairThreadCount(repairThreadCount) @@ -603,14 +612,14 @@ private RepairRun createRepairRun( return run; } - private static class NotEmptyList implements ArgumentMatcher> { + private static final class NotEmptyList implements ArgumentMatcher> { @Override public boolean matches(Collection segments) { return !segments.isEmpty(); } } - private static class EmptyList implements ArgumentMatcher> { + private static final class EmptyList implements ArgumentMatcher> { @Override public boolean matches(Collection segments) { return segments.isEmpty(); diff --git a/src/server/src/test/java/io/cassandrareaper/service/RepairRunServiceTest.java b/src/server/src/test/java/io/cassandrareaper/service/RepairRunServiceTest.java index 20af6113f..5b8d720e4 100644 --- a/src/server/src/test/java/io/cassandrareaper/service/RepairRunServiceTest.java +++ b/src/server/src/test/java/io/cassandrareaper/service/RepairRunServiceTest.java @@ -259,6 +259,7 @@ public void generateSegmentsFail1Test() throws ReaperException, UnknownHostExcep .keyspaceName(KS_NAME) .columnFamilies(CF_NAMES) .incrementalRepair(INCREMENTAL_REPAIR) + .subrangeIncrementalRepair(INCREMENTAL_REPAIR) .nodes(NODES) .datacenters(DATACENTERS) .blacklistedTables(BLACKLISTED_TABLES) @@ -305,6 +306,7 @@ protected JmxCassandraManagementProxy connectImpl(Node host) throws ReaperExcept .keyspaceName("test") .blacklistedTables(Sets.newHashSet("table1")) .incrementalRepair(false) + .subrangeIncrementalRepair(false) .repairThreadCount(4) .timeout(segmentTimeout) .build(UUIDs.timeBased()); @@ -345,6 +347,7 @@ public void generateSegmentsTest() throws ReaperException, UnknownHostException .keyspaceName(KS_NAME) .columnFamilies(CF_NAMES) .incrementalRepair(INCREMENTAL_REPAIR) + .subrangeIncrementalRepair(INCREMENTAL_REPAIR) .nodes(NODES) .datacenters(DATACENTERS) .blacklistedTables(BLACKLISTED_TABLES) @@ -392,6 +395,7 @@ protected JmxCassandraManagementProxy connectImpl(Node host) throws ReaperExcept .keyspaceName("test") .blacklistedTables(Sets.newHashSet("table1")) .incrementalRepair(false) + .subrangeIncrementalRepair(false) .repairThreadCount(4) .timeout(segmentTimeout) .build(UUIDs.timeBased()); @@ -406,6 +410,7 @@ public void failRepairRunCreationTest() throws ReaperException, UnknownHostExcep final String KS_NAME = "reaper"; final Set CF_NAMES = Sets.newHashSet("reaper"); final boolean INCREMENTAL_REPAIR = false; + final boolean SUBRANGE_INCREMENTAL_REPAIR = false; final Set NODES = Sets.newHashSet("127.0.0.1", "127.0.0.2", "127.0.0.3"); final Set DATACENTERS = Collections.emptySet(); final Set BLACKLISTED_TABLES = Collections.emptySet(); @@ -450,6 +455,7 @@ public void failRepairRunCreationTest() throws ReaperException, UnknownHostExcep .keyspaceName(KS_NAME) .columnFamilies(CF_NAMES) .incrementalRepair(INCREMENTAL_REPAIR) + .subrangeIncrementalRepair(SUBRANGE_INCREMENTAL_REPAIR) .nodes(NODES) .datacenters(DATACENTERS) .blacklistedTables(BLACKLISTED_TABLES) @@ -465,6 +471,7 @@ public void failIncrRepairRunCreationTest() throws ReaperException, UnknownHostE final String KS_NAME = "reaper"; final Set CF_NAMES = Sets.newHashSet("reaper"); final boolean INCREMENTAL_REPAIR = false; + final boolean SUBRANGE_INCREMENTAL_REPAIR = false; final Set NODES = Sets.newHashSet("127.0.0.1", "127.0.0.2", "127.0.0.3"); final Set DATACENTERS = Collections.emptySet(); final Set BLACKLISTED_TABLES = Collections.emptySet(); @@ -515,6 +522,7 @@ public void failIncrRepairRunCreationTest() throws ReaperException, UnknownHostE .keyspaceName(KS_NAME) .columnFamilies(CF_NAMES) .incrementalRepair(INCREMENTAL_REPAIR) + .subrangeIncrementalRepair(SUBRANGE_INCREMENTAL_REPAIR) .nodes(NODES) .datacenters(DATACENTERS) .blacklistedTables(BLACKLISTED_TABLES) @@ -531,6 +539,7 @@ public void getDCsByNodeForRepairSegmentPath1Test() throws ReaperException, Unkn final String KS_NAME = "reaper"; final Set CF_NAMES = Sets.newHashSet("reaper"); final boolean INCREMENTAL_REPAIR = false; + final boolean SUBRANGE_INCREMENTAL_REPAIR = false; final Set NODES = Sets.newHashSet("127.0.0.1", "127.0.0.2", "127.0.0.3"); final Set DATACENTERS = Sets.newHashSet("dc1"); final Set BLACKLISTED_TABLES = Collections.emptySet(); @@ -575,6 +584,7 @@ public JmxCassandraManagementProxy connectImpl(Node host) throws ReaperException .keyspaceName(KS_NAME) .columnFamilies(CF_NAMES) .incrementalRepair(INCREMENTAL_REPAIR) + .subrangeIncrementalRepair(SUBRANGE_INCREMENTAL_REPAIR) .nodes(NODES) .datacenters(DATACENTERS) .blacklistedTables(BLACKLISTED_TABLES) @@ -592,6 +602,7 @@ public void getDCsByNodeForRepairSegmentFailPathTest() throws ReaperException, U final String KS_NAME = "reaper"; final Set CF_NAMES = Sets.newHashSet("reaper"); final boolean INCREMENTAL_REPAIR = false; + final boolean SUBRANGE_INCREMENTAL_REPAIR = false; final Set NODES = Sets.newHashSet("127.0.0.1", "127.0.0.2", "127.0.0.3"); final Set DATACENTERS = Sets.newHashSet("dc1"); final Set BLACKLISTED_TABLES = Collections.emptySet(); @@ -639,6 +650,7 @@ public JmxCassandraManagementProxy connectImpl(Node host) throws ReaperException .keyspaceName(KS_NAME) .columnFamilies(CF_NAMES) .incrementalRepair(INCREMENTAL_REPAIR) + .subrangeIncrementalRepair(SUBRANGE_INCREMENTAL_REPAIR) .nodes(NODES) .datacenters(DATACENTERS) .blacklistedTables(BLACKLISTED_TABLES) @@ -655,6 +667,7 @@ public void createRepairSegmentsForIncrementalRepairTest() throws ReaperExceptio final String KS_NAME = "reaper"; final Set CF_NAMES = Sets.newHashSet("reaper"); final boolean INCREMENTAL_REPAIR = false; + final boolean SUBRANGE_INCREMENTAL_REPAIR = false; final Set NODES = Sets.newHashSet("127.0.0.1", "127.0.0.2", "127.0.0.3"); final Set DATACENTERS = Sets.newHashSet("dc1"); final Set BLACKLISTED_TABLES = Collections.emptySet(); @@ -682,6 +695,7 @@ public void createRepairSegmentsForIncrementalRepairTest() throws ReaperExceptio .keyspaceName(KS_NAME) .columnFamilies(CF_NAMES) .incrementalRepair(INCREMENTAL_REPAIR) + .subrangeIncrementalRepair(SUBRANGE_INCREMENTAL_REPAIR) .nodes(NODES) .datacenters(DATACENTERS) .blacklistedTables(BLACKLISTED_TABLES) @@ -821,6 +835,7 @@ public void generateSegmentsTestEmpty() throws ReaperException, UnknownHostExcep final String KS_NAME = "reaper"; final Set CF_NAMES = Sets.newHashSet("reaper"); final boolean INCREMENTAL_REPAIR = false; + final boolean SUBRANGE_INCREMENTAL_REPAIR = false; final Set NODES = Sets.newHashSet("127.0.0.1", "127.0.0.2", "127.0.0.3"); final Set DATACENTERS = Collections.emptySet(); final Set BLACKLISTED_TABLES = Collections.emptySet(); @@ -841,6 +856,7 @@ public void generateSegmentsTestEmpty() throws ReaperException, UnknownHostExcep .keyspaceName(KS_NAME) .columnFamilies(CF_NAMES) .incrementalRepair(INCREMENTAL_REPAIR) + .subrangeIncrementalRepair(SUBRANGE_INCREMENTAL_REPAIR) .nodes(NODES) .datacenters(DATACENTERS) .blacklistedTables(BLACKLISTED_TABLES) @@ -888,6 +904,7 @@ protected JmxCassandraManagementProxy connectImpl(Node host) throws ReaperExcept .keyspaceName("test") .blacklistedTables(Sets.newHashSet("table1")) .incrementalRepair(false) + .subrangeIncrementalRepair(false) .repairThreadCount(4) .timeout(segmentTimeout) .build(UUIDs.timeBased()); diff --git a/src/server/src/test/java/io/cassandrareaper/service/RepairRunnerHangingTest.java b/src/server/src/test/java/io/cassandrareaper/service/RepairRunnerHangingTest.java new file mode 100644 index 000000000..afbd073f8 --- /dev/null +++ b/src/server/src/test/java/io/cassandrareaper/service/RepairRunnerHangingTest.java @@ -0,0 +1,683 @@ +/* + * Copyright 2015-2017 Spotify AB + * Copyright 2016-2019 The Last Pickle Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cassandrareaper.service; + +import io.cassandrareaper.AppContext; +import io.cassandrareaper.ReaperApplicationConfiguration; +import io.cassandrareaper.ReaperException; +import io.cassandrareaper.core.Cluster; +import io.cassandrareaper.core.CompactionStats; +import io.cassandrareaper.core.Node; +import io.cassandrareaper.core.RepairRun; +import io.cassandrareaper.core.RepairSegment; +import io.cassandrareaper.core.RepairUnit; +import io.cassandrareaper.core.Segment; +import io.cassandrareaper.crypto.NoopCrypotograph; +import io.cassandrareaper.management.ClusterFacade; +import io.cassandrareaper.management.RepairStatusHandler; +import io.cassandrareaper.management.jmx.CassandraManagementProxyTest; +import io.cassandrareaper.management.jmx.JmxCassandraManagementProxy; +import io.cassandrareaper.management.jmx.JmxManagementConnectionFactory; +import io.cassandrareaper.storage.IStorageDao; +import io.cassandrareaper.storage.MemoryStorageFacade; + +import java.io.IOException; +import java.math.BigInteger; +import java.net.UnknownHostException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.Executors; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import javax.management.JMException; +import javax.management.MalformedObjectNameException; +import javax.management.ReflectionException; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import org.apache.cassandra.locator.EndpointSnitchInfoMBean; +import org.apache.cassandra.repair.RepairParallelism; +import org.apache.cassandra.service.ActiveRepairService; +import org.apache.cassandra.utils.progress.ProgressEventType; +import org.apache.commons.lang3.RandomStringUtils; +import org.joda.time.DateTime; +import org.joda.time.DateTimeUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.awaitility.Awaitility.await; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public final class RepairRunnerHangingTest { + + private static final Logger LOG = LoggerFactory.getLogger(RepairRunnerHangingTest.class); + private static final Set TABLES = ImmutableSet.of("table1"); + private static final List THREE_TOKENS = Lists.newArrayList( + BigInteger.valueOf(0L), + BigInteger.valueOf(100L), + BigInteger.valueOf(200L)); + + private final Cluster cluster = Cluster.builder() + .withName("test_" + RandomStringUtils.randomAlphabetic(12)) + .withSeedHosts(ImmutableSet.of("127.0.0.1")) + .withState(Cluster.State.ACTIVE) + .build(); + private Map replicas = ImmutableMap.of( + "127.0.0.1", "dc1" + ); + private Map currentRunners; + + @Before + public void setUp() throws Exception { + SegmentRunner.SEGMENT_RUNNERS.clear(); + currentRunners = new HashMap<>(); + // Create 3 runners for cluster1 and 1 for cluster2 + for (int i = 0; i < 3; i++) { + RepairRunner runner = mock(RepairRunner.class); + when(runner.getClusterName()).thenReturn("cluster1"); + when(runner.isRunning()).thenReturn(true); + currentRunners.put(UUID.randomUUID(), runner); + } + + RepairRunner runner = mock(RepairRunner.class); + when(runner.getClusterName()).thenReturn("cluster2"); + when(runner.isRunning()).thenReturn(true); + currentRunners.put(UUID.randomUUID(), runner); + } + + public static Map, List> threeNodeCluster() { + Map, List> map = new HashMap, List>(); + map = addRangeToMap(map, "0", "50", "a1", "a2", "a3"); + map = addRangeToMap(map, "50", "100", "a2", "a3", "a1"); + map = addRangeToMap(map, "100", "0", "a3", "a1", "a2"); + return map; + } + + public static Map, List> threeNodeClusterWithIps() { + Map, List> map = new HashMap, List>(); + map = addRangeToMap(map, "0", "100", "127.0.0.1", "127.0.0.2", "127.0.0.3"); + map = addRangeToMap(map, "100", "200", "127.0.0.2", "127.0.0.3", "127.0.0.1"); + map = addRangeToMap(map, "200", "0", "127.0.0.3", "127.0.0.1", "127.0.0.2"); + return map; + } + + public static Map, List> scyllaThreeNodeClusterWithIps() { + Map, List> map = new HashMap, List>(); + map = addRangeToMap(map, "", "0", "127.0.0.3", "127.0.0.1", "127.0.0.2"); + map = addRangeToMap(map, "0", "100", "127.0.0.1", "127.0.0.2", "127.0.0.3"); + map = addRangeToMap(map, "100", "200", "127.0.0.2", "127.0.0.3", "127.0.0.1"); + map = addRangeToMap(map, "200", "", "127.0.0.3", "127.0.0.1", "127.0.0.2"); + return map; + } + + public static Map, List> sixNodeCluster() { + Map, List> map = new HashMap, List>(); + map = addRangeToMap(map, "0", "50", "a1", "a2", "a3"); + map = addRangeToMap(map, "50", "100", "a2", "a3", "a4"); + map = addRangeToMap(map, "100", "150", "a3", "a4", "a5"); + map = addRangeToMap(map, "150", "200", "a4", "a5", "a6"); + map = addRangeToMap(map, "200", "250", "a5", "a6", "a1"); + map = addRangeToMap(map, "250", "0", "a6", "a1", "a2"); + return map; + } + + public static Map threeNodeClusterEndpoint() { + Map map = new HashMap(); + map.put("host1", "hostId1"); + map.put("host2", "hostId2"); + map.put("host3", "hostId3"); + return map; + } + + public static Map sixNodeClusterEndpoint() { + Map map = new HashMap(); + map.put("host1", "hostId1"); + map.put("host2", "hostId2"); + map.put("host3", "hostId3"); + map.put("host4", "hostId4"); + map.put("host5", "hostId5"); + map.put("host6", "hostId6"); + return map; + } + + private Map endpointToHostIDMap() { + Map endpointToHostIDMap = new HashMap(); + endpointToHostIDMap.put("127.0.0.1", UUID.randomUUID().toString()); + endpointToHostIDMap.put("127.0.0.2", UUID.randomUUID().toString()); + endpointToHostIDMap.put("127.0.0.3", UUID.randomUUID().toString()); + + return endpointToHostIDMap; + } + + private RepairRun addNewRepairRun( + final Map nodeMap, + final double intensity, + final IStorageDao storage, + UUID cf, + UUID hostID + ) { + return storage.getRepairRunDao().addRepairRun( + RepairRun.builder(cluster.getName(), cf) + .intensity(intensity) + .segmentCount(1) + .repairParallelism(RepairParallelism.PARALLEL) + .tables(TABLES), + Lists.newArrayList( + RepairSegment.builder( + Segment.builder() + .withTokenRange(new RingRange(BigInteger.ZERO, new BigInteger("100"))) + .withReplicas(nodeMap) + .build(), cf) + .withState(RepairSegment.State.RUNNING) + .withStartTime(DateTime.now()) + .withCoordinatorHost("reaper") + .withHostID(hostID), + RepairSegment.builder( + Segment.builder() + .withTokenRange(new RingRange(new BigInteger("100"), new BigInteger("200"))) + .withReplicas(nodeMap) + .build(), cf) + .withHostID(hostID) + ) + ); + } + + private static Map, List> addRangeToMap( + Map, List> map, + String start, + String end, + String... hosts) { + + List range = Lists.newArrayList(start, end); + List endPoints = Lists.newArrayList(hosts); + map.put(range, endPoints); + return map; + } + + @After + public void tearDown() { + DateTimeUtils.setCurrentMillisSystem(); + } + + @Test + public void testHangingRepair() throws InterruptedException, ReaperException, JMException, IOException { + final Set cfNames = Sets.newHashSet("reaper"); + final Set nodeSet = Sets.newHashSet("127.0.0.1"); + final Set datacenters = Collections.emptySet(); + final Set blacklistedTables = Collections.emptySet(); + final long timeRun = 41L; + final double intensity = 0.5f; + final int repairThreadCount = 1; + final int segmentTimeout = 1; + final IStorageDao storage = new MemoryStorageFacade(); + storage.getClusterDao().addCluster(cluster); + RepairUnit cf = storage.getRepairUnitDao().addRepairUnit( + RepairUnit.builder() + .clusterName(cluster.getName()) + .keyspaceName("reaper") + .columnFamilies(cfNames) + .incrementalRepair(false) + .subrangeIncrementalRepair(false) + .nodes(nodeSet) + .datacenters(datacenters) + .blacklistedTables(blacklistedTables) + .repairThreadCount(repairThreadCount) + .timeout(segmentTimeout)); + DateTimeUtils.setCurrentMillisFixed(timeRun); + RepairRun run = storage.getRepairRunDao().addRepairRun( + RepairRun.builder(cluster.getName(), cf.getId()) + .intensity(intensity) + .segmentCount(1) + .repairParallelism(RepairParallelism.PARALLEL) + .tables(TABLES), + Collections.singleton( + RepairSegment.builder( + Segment.builder() + .withTokenRange(new RingRange(BigInteger.ZERO, new BigInteger("100"))) + .withReplicas(replicas) + .build(), + cf.getId()))); + final UUID runId = run.getId(); + final UUID segmentId = storage.getRepairSegmentDao().getNextFreeSegments(run.getId()).get(0).getId(); + assertEquals(storage.getRepairSegmentDao().getRepairSegment(runId, segmentId).get().getState(), + RepairSegment.State.NOT_STARTED); + AppContext context = new AppContext(); + context.storage = storage; + context.config = new ReaperApplicationConfiguration(); + final Semaphore mutex = new Semaphore(0); + final JmxCassandraManagementProxy jmx = CassandraManagementProxyTest.mockJmxProxyImpl(); + Map endpointToHostIDMap = endpointToHostIDMap(); + when(jmx.getEndpointToHostId()).thenReturn(endpointToHostIDMap); + when(jmx.getClusterName()).thenReturn(cluster.getName()); + when(jmx.getRangeToEndpointMap(anyString())).thenReturn(RepairRunnerHangingTest.sixNodeCluster()); + EndpointSnitchInfoMBean endpointSnitchInfoMBean = mock(EndpointSnitchInfoMBean.class); + when(endpointSnitchInfoMBean.getDatacenter()).thenReturn("dc1"); + try { + when(endpointSnitchInfoMBean.getDatacenter(anyString())).thenReturn("dc1"); + } catch (UnknownHostException ex) { + throw new AssertionError(ex); + } + final AtomicInteger repairAttempts = new AtomicInteger(1); + when(jmx.triggerRepair(any(), any(), any(), any(), any(), any(), any(), anyInt())) + .then( + (invocation) -> { + assertEquals(RepairSegment.State.STARTED, + storage.getRepairSegmentDao().getRepairSegment(runId, segmentId).get().getState()); + final int repairNumber = repairAttempts.getAndIncrement(); + switch (repairNumber) { + case 1: + new Thread() { + @Override + public void run() { + ((RepairStatusHandler) invocation.getArgument(5)) + .handle(repairNumber, + Optional.of(ActiveRepairService.Status.STARTED), + Optional.empty(), null, jmx); + assertEquals( + RepairSegment.State.RUNNING, + storage.getRepairSegmentDao().getRepairSegment(runId, segmentId).get().getState()); + } + }.start(); + break; + case 2: + new Thread() { + @Override + public void run() { + ((RepairStatusHandler) invocation.getArgument(5)) + .handle( + repairNumber, + Optional.of(ActiveRepairService.Status.STARTED), + Optional.empty(), null, jmx); + assertEquals( + RepairSegment.State.RUNNING, + storage.getRepairSegmentDao().getRepairSegment(runId, segmentId).get().getState()); + ((RepairStatusHandler) invocation.getArgument(5)) + .handle( + repairNumber, + Optional.of(ActiveRepairService.Status.SESSION_SUCCESS), + Optional.empty(), null, jmx); + assertEquals( + RepairSegment.State.DONE, + storage.getRepairSegmentDao().getRepairSegment(runId, segmentId).get().getState()); + ((RepairStatusHandler) invocation.getArgument(5)) + .handle(repairNumber, + Optional.of(ActiveRepairService.Status.FINISHED), + Optional.empty(), null, jmx); + + mutex.release(); + LOG.info("MUTEX RELEASED"); + } + }.start(); + break; + default: + fail("triggerRepair should only have been called twice"); + } + LOG.info("repair number : " + repairNumber); + return repairNumber; + }); + ClusterFacade clusterFacade = mock(ClusterFacade.class); + when(clusterFacade.connect(any(Cluster.class), any())).thenReturn(jmx); + when(clusterFacade.nodeIsDirectlyAccessible(any(), any())).thenReturn(true); + when(clusterFacade.tokenRangeToEndpoint(any(), anyString(), any())).thenReturn(Lists.newArrayList(nodeSet)); + when(clusterFacade.getEndpointToHostId(any())).thenReturn(endpointToHostIDMap); + when(clusterFacade.listActiveCompactions(any())).thenReturn(CompactionStats.builder().withActiveCompactions( + Collections.emptyList()).withPendingCompactions(Optional.of(0)).build()); + when(clusterFacade.getRangeToEndpointMap(any(), anyString())) + .thenReturn((Map) ImmutableMap.of(Lists.newArrayList("0", "100"), Lists.newArrayList(nodeSet))); + context.repairManager = RepairManager.create( + context, + clusterFacade, + Executors.newScheduledThreadPool(1), + 1, + TimeUnit.MILLISECONDS, + 1, + context.storage.getRepairRunDao()); + context.managementConnectionFactory = new JmxManagementConnectionFactory(context, new NoopCrypotograph()) { + @Override + protected JmxCassandraManagementProxy connectImpl(Node host) throws ReaperException { + return jmx; + } + }; + context.repairManager.startRepairRun(run); + await().with().atMost(2, TimeUnit.MINUTES).until(() -> { + try { + mutex.acquire(); + Thread.sleep(1000); + return true; + } catch (InterruptedException ex) { + throw new IllegalStateException(ex); + } + }); + assertEquals(RepairRun.RunState.DONE, storage.getRepairRunDao().getRepairRun(runId).get().getRunState()); + } + + @Test + public void testHangingRepairNewApi() throws InterruptedException, ReaperException, MalformedObjectNameException, + ReflectionException, IOException { + final String ksName = "reaper"; + final Set cfNames = Sets.newHashSet("reaper"); + final boolean incrementalRepair = false; + final Set nodeSet = Sets.newHashSet("127.0.0.1"); + final Set datacenters = Collections.emptySet(); + final Set blacklistedTables = Collections.emptySet(); + final long timeRun = 41L; + final double intensity = 0.5f; + final int repairThreadCount = 1; + final int segmentTimeout = 1; + final IStorageDao storage = new MemoryStorageFacade(); + storage.getClusterDao().addCluster(cluster); + DateTimeUtils.setCurrentMillisFixed(timeRun); + RepairUnit cf = storage.getRepairUnitDao().addRepairUnit( + RepairUnit.builder() + .clusterName(cluster.getName()) + .keyspaceName(ksName) + .columnFamilies(cfNames) + .incrementalRepair(incrementalRepair) + .subrangeIncrementalRepair(incrementalRepair) + .nodes(nodeSet) + .datacenters(datacenters) + .blacklistedTables(blacklistedTables) + .repairThreadCount(repairThreadCount) + .timeout(segmentTimeout)); + RepairRun run = storage.getRepairRunDao().addRepairRun( + RepairRun.builder(cluster.getName(), cf.getId()) + .intensity(intensity).segmentCount(1) + .repairParallelism(RepairParallelism.PARALLEL) + .tables(TABLES), + Collections.singleton( + RepairSegment.builder( + Segment.builder() + .withTokenRange(new RingRange(BigInteger.ZERO, new BigInteger("100"))) + .withReplicas(replicas) + .build(), + cf.getId()))); + final UUID runId = run.getId(); + final UUID segmentId = storage.getRepairSegmentDao().getNextFreeSegments(run.getId()).get(0).getId(); + assertEquals(storage.getRepairSegmentDao().getRepairSegment(runId, segmentId).get().getState(), + RepairSegment.State.NOT_STARTED); + AppContext context = new AppContext(); + context.storage = storage; + context.config = new ReaperApplicationConfiguration(); + final Semaphore mutex = new Semaphore(0); + final JmxCassandraManagementProxy jmx = CassandraManagementProxyTest.mockJmxProxyImpl(); + when(jmx.getClusterName()).thenReturn(cluster.getName()); + when(jmx.getRangeToEndpointMap(anyString())).thenReturn(RepairRunnerHangingTest.sixNodeCluster()); + EndpointSnitchInfoMBean endpointSnitchInfoMBean = mock(EndpointSnitchInfoMBean.class); + when(endpointSnitchInfoMBean.getDatacenter()).thenReturn("dc1"); + try { + when(endpointSnitchInfoMBean.getDatacenter(anyString())).thenReturn("dc1"); + } catch (UnknownHostException ex) { + throw new AssertionError(ex); + } + final AtomicInteger repairAttempts = new AtomicInteger(1); + when(jmx.triggerRepair(any(), any(), any(), any(), any(), any(), any(), anyInt())) + .then( + (invocation) -> { + assertEquals( + RepairSegment.State.STARTED, + storage.getRepairSegmentDao().getRepairSegment(runId, segmentId).get().getState()); + final int repairNumber = repairAttempts.getAndIncrement(); + switch (repairNumber) { + case 1: + new Thread() { + @Override + public void run() { + ((RepairStatusHandler) invocation.getArgument(5)) + .handle( + repairNumber, Optional.empty(), + Optional.of(ProgressEventType.START), null, jmx); + assertEquals( + RepairSegment.State.RUNNING, + storage.getRepairSegmentDao().getRepairSegment(runId, segmentId).get().getState()); + } + }.start(); + break; + case 2: + new Thread() { + @Override + public void run() { + ((RepairStatusHandler) invocation.getArgument(5)) + .handle( + repairNumber, Optional.empty(), + Optional.of(ProgressEventType.START), null, jmx); + assertEquals( + RepairSegment.State.RUNNING, + storage.getRepairSegmentDao().getRepairSegment(runId, segmentId).get().getState()); + ((RepairStatusHandler) invocation.getArgument(5)) + .handle( + repairNumber, Optional.empty(), + Optional.of(ProgressEventType.SUCCESS), null, jmx); + assertEquals( + RepairSegment.State.DONE, + storage.getRepairSegmentDao().getRepairSegment(runId, segmentId).get().getState()); + ((RepairStatusHandler) invocation.getArgument(5)) + .handle( + repairNumber, Optional.empty(), + Optional.of(ProgressEventType.COMPLETE), null, jmx); + mutex.release(); + LOG.info("MUTEX RELEASED"); + } + }.start(); + break; + default: + fail("triggerRepair should only have been called twice"); + } + return repairNumber; + }); + ClusterFacade clusterFacade = mock(ClusterFacade.class); + when(clusterFacade.connect(any(Cluster.class), any())).thenReturn(jmx); + when(clusterFacade.nodeIsDirectlyAccessible(any(), any())).thenReturn(true); + when(clusterFacade.tokenRangeToEndpoint(any(), anyString(), any())) + .thenReturn(Lists.newArrayList(nodeSet)); + when(clusterFacade.getRangeToEndpointMap(any(), anyString())) + .thenReturn((Map) ImmutableMap.of(Lists.newArrayList("0", "100"), Lists.newArrayList(nodeSet))); + when(clusterFacade.listActiveCompactions(any())).thenReturn(CompactionStats.builder().withActiveCompactions( + Collections.emptyList()).withPendingCompactions(Optional.of(0)).build()); + context.repairManager + = RepairManager.create( + context, + clusterFacade, + Executors.newScheduledThreadPool(1), + 1, + TimeUnit.MILLISECONDS, + 1, + context.storage.getRepairRunDao()); + context.managementConnectionFactory = new JmxManagementConnectionFactory(context, new NoopCrypotograph()) { + @Override + protected JmxCassandraManagementProxy connectImpl(Node host) throws ReaperException { + return jmx; + } + }; + context.repairManager.startRepairRun(run); + await().with().atMost(2, TimeUnit.MINUTES).until(() -> { + try { + mutex.acquire(); + LOG.info("MUTEX ACQUIRED"); + // TODO: refactor so that we can properly wait for the repair runner to finish rather than using this sleep() + Thread.sleep(1000); + return true; + } catch (InterruptedException ex) { + throw new IllegalStateException(ex); + } + }); + assertEquals(RepairRun.RunState.DONE, storage.getRepairRunDao().getRepairRun(runId).get().getRunState()); + } + + @Test + public void testDontFailRepairAfterTopologyChangeIncrementalRepair() throws InterruptedException, ReaperException, + MalformedObjectNameException, ReflectionException, IOException { + final String ksName = "reaper"; + final Set cfNames = Sets.newHashSet("reaper"); + final boolean incrementalRepair = true; + final Set nodeSet = Sets.newHashSet("127.0.0.1", "127.0.0.2", "127.0.0.3"); + final List nodeSetAfterTopologyChange = Lists.newArrayList("127.0.0.3", "127.0.0.2", "127.0.0.4"); + final Map nodeMap = ImmutableMap.of("127.0.0.1", "dc1", "127.0.0.2", "dc1", "127.0.0.3", "dc1"); + final Map nodeMapAfterTopologyChange = ImmutableMap.of( + "127.0.0.3", "dc1", "127.0.0.2", "dc1", "127.0.0.4", "dc1"); + final Set datacenters = Collections.emptySet(); + final Set blacklistedTables = Collections.emptySet(); + final double intensity = 0.5f; + final int repairThreadCount = 1; + final int segmentTimeout = 30; + final List tokens = THREE_TOKENS; + final IStorageDao storage = new MemoryStorageFacade(); + AppContext context = new AppContext(); + context.storage = storage; + context.config = new ReaperApplicationConfiguration(); + storage.getClusterDao().addCluster(cluster); + UUID cf = storage.getRepairUnitDao().addRepairUnit( + RepairUnit.builder() + .clusterName(cluster.getName()) + .keyspaceName(ksName) + .columnFamilies(cfNames) + .incrementalRepair(incrementalRepair) + .subrangeIncrementalRepair(false) + .nodes(nodeSet) + .datacenters(datacenters) + .blacklistedTables(blacklistedTables) + .repairThreadCount(repairThreadCount) + .timeout(segmentTimeout)) + .getId(); + Map endpointToHostIDMap = endpointToHostIDMap(); + RepairRun run = addNewRepairRun(nodeMap, + intensity, + storage, + cf, + UUID.fromString(endpointToHostIDMap.get("127.0.0.1"))); + final UUID runId = run.getId(); + final UUID segmentId = storage.getRepairSegmentDao().getNextFreeSegments(run.getId()).get(0).getId(); + assertEquals(storage.getRepairSegmentDao().getRepairSegment(runId, segmentId).get().getState(), + RepairSegment.State.NOT_STARTED); + final JmxCassandraManagementProxy jmx = CassandraManagementProxyTest.mockJmxProxyImpl(); + when(jmx.getClusterName()).thenReturn(cluster.getName()); + when(jmx.getRangeToEndpointMap(anyString())).thenReturn(RepairRunnerTest.threeNodeClusterWithIps()); + when(jmx.getEndpointToHostId()).thenReturn(endpointToHostIDMap); + when(jmx.getTokens()).thenReturn(tokens); + EndpointSnitchInfoMBean endpointSnitchInfoMBean = mock(EndpointSnitchInfoMBean.class); + when(endpointSnitchInfoMBean.getDatacenter()).thenReturn("dc1"); + try { + when(endpointSnitchInfoMBean.getDatacenter(anyString())).thenReturn("dc1"); + } catch (UnknownHostException ex) { + throw new AssertionError(ex); + } + + ClusterFacade clusterFacade = mock(ClusterFacade.class); + when(clusterFacade.connect(any(Cluster.class), any())).thenReturn(jmx); + when(clusterFacade.nodeIsDirectlyAccessible(any(), any())).thenReturn(true); + when(clusterFacade.tokenRangeToEndpoint(any(), anyString(), any())) + .thenReturn(Lists.newArrayList(nodeSet)); + when(clusterFacade.getRangeToEndpointMap(any(), anyString())) + .thenReturn((Map) ImmutableMap.of( + Lists.newArrayList("0", "100"), Lists.newArrayList(nodeSet), + Lists.newArrayList("100", "200"), Lists.newArrayList(nodeSet))); + when(clusterFacade.getEndpointToHostId(any())).thenReturn(endpointToHostIDMap); + when(clusterFacade.listActiveCompactions(any())).thenReturn( + CompactionStats.builder() + .withActiveCompactions(Collections.emptyList()) + .withPendingCompactions(Optional.of(0)) + .build()); + context.repairManager = RepairManager.create( + context, + clusterFacade, + Executors.newScheduledThreadPool(10), + 1, + TimeUnit.MILLISECONDS, + 1, + context.storage.getRepairRunDao()); + AtomicInteger repairNumberCounter = new AtomicInteger(1); + when(jmx.triggerRepair(any(), any(), any(), any(), any(), any(), any(), anyInt())) + .then( + (invocation) -> { + final int repairNumber = repairNumberCounter.getAndIncrement(); + new Thread() { + @Override + public void run() { + ((RepairStatusHandler) invocation.getArgument(5)) + .handle( + repairNumber, + Optional.of(ActiveRepairService.Status.STARTED), + Optional.empty(), + null, + jmx); + ((RepairStatusHandler) invocation.getArgument(5)) + .handle( + repairNumber, + Optional.of(ActiveRepairService.Status.SESSION_SUCCESS), + Optional.empty(), + null, + jmx); + ((RepairStatusHandler) invocation.getArgument(5)) + .handle( + repairNumber, + Optional.of(ActiveRepairService.Status.FINISHED), + Optional.empty(), + null, + jmx); + } + }.start(); + return repairNumber; + }); + context.managementConnectionFactory = new JmxManagementConnectionFactory(context, new NoopCrypotograph()) { + @Override + protected JmxCassandraManagementProxy connectImpl(Node host) throws ReaperException { + return jmx; + } + }; + ClusterFacade clusterProxy = ClusterFacade.create(context); + ClusterFacade clusterProxySpy = Mockito.spy(clusterProxy); + Mockito.doReturn(nodeSetAfterTopologyChange).when(clusterProxySpy).tokenRangeToEndpoint(any(), any(), any()); + assertEquals(RepairRun.RunState.NOT_STARTED, storage.getRepairRunDao().getRepairRun(runId).get().getRunState()); + storage.getRepairRunDao().updateRepairRun( + run.with().runState(RepairRun.RunState.RUNNING).startTime(DateTime.now()).build(runId)); + // We'll now change the list of replicas for any segment, making the stored ones obsolete + when(clusterFacade.getRangeToEndpointMap(any(), anyString())) + .thenReturn((Map) ImmutableMap.of( + Lists.newArrayList("0", "100"), Lists.newArrayList(nodeSetAfterTopologyChange), + Lists.newArrayList("100", "200"), Lists.newArrayList(nodeSetAfterTopologyChange))); + String hostIdToChange = endpointToHostIDMap.get("127.0.0.1"); + endpointToHostIDMap.remove("127.0.0.1"); + endpointToHostIDMap.put("127.0.0.4", hostIdToChange); + when(clusterFacade.getEndpointToHostId(any())).thenReturn(endpointToHostIDMap); + when(clusterFacade.tokenRangeToEndpoint(any(), anyString(), any())) + .thenReturn(Lists.newArrayList(nodeSetAfterTopologyChange)); + context.repairManager.resumeRunningRepairRuns(); + + // The repair run should succeed despite the topology change. + await().with().atMost(60, TimeUnit.SECONDS).until(() -> { + return RepairRun.RunState.DONE == storage.getRepairRunDao().getRepairRun(runId).get().getRunState(); + }); + } +} \ No newline at end of file diff --git a/src/server/src/test/java/io/cassandrareaper/service/RepairRunnerTest.java b/src/server/src/test/java/io/cassandrareaper/service/RepairRunnerTest.java index 100e6fc0a..b7f16e250 100644 --- a/src/server/src/test/java/io/cassandrareaper/service/RepairRunnerTest.java +++ b/src/server/src/test/java/io/cassandrareaper/service/RepairRunnerTest.java @@ -27,6 +27,7 @@ import io.cassandrareaper.core.RepairRun; import io.cassandrareaper.core.RepairSchedule; import io.cassandrareaper.core.RepairSegment; +import io.cassandrareaper.core.RepairType; import io.cassandrareaper.core.RepairUnit; import io.cassandrareaper.core.Segment; import io.cassandrareaper.crypto.NoopCrypotograph; @@ -58,10 +59,8 @@ import java.util.UUID; import java.util.concurrent.Callable; import java.util.concurrent.Executors; -import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -import javax.management.JMException; import javax.management.MalformedObjectNameException; import javax.management.ReflectionException; @@ -73,7 +72,6 @@ import org.apache.cassandra.locator.EndpointSnitchInfoMBean; import org.apache.cassandra.repair.RepairParallelism; import org.apache.cassandra.service.ActiveRepairService; -import org.apache.cassandra.utils.progress.ProgressEventType; import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.tuple.Pair; import org.awaitility.Duration; @@ -84,28 +82,23 @@ import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import static java.util.concurrent.TimeUnit.SECONDS; import static org.awaitility.Awaitility.await; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyDouble; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public final class RepairRunnerTest { - - private static final Logger LOG = LoggerFactory.getLogger(RepairRunnerTest.class); private static final Set TABLES = ImmutableSet.of("table1"); private static final Duration POLL_INTERVAL = Duration.TWO_SECONDS; private static final List THREE_TOKENS = Lists.newArrayList( @@ -213,312 +206,6 @@ public void tearDown() { DateTimeUtils.setCurrentMillisSystem(); } - @Test - @SuppressWarnings("checkstyle:methodlength") - public void testHangingRepair() throws InterruptedException, ReaperException, JMException, IOException { - final String ksName = "reaper"; - final Set cfNames = Sets.newHashSet("reaper"); - final boolean incrementalRepair = false; - final Set nodeSet = Sets.newHashSet("127.0.0.1"); - final Set datacenters = Collections.emptySet(); - final Set blacklistedTables = Collections.emptySet(); - final long timeRun = 41L; - final double intensity = 0.5f; - final int repairThreadCount = 1; - final int segmentTimeout = 1; - final IStorageDao storage = new MemoryStorageFacade(); - storage.getClusterDao().addCluster(cluster); - RepairUnit cf = storage.getRepairUnitDao().addRepairUnit( - RepairUnit.builder() - .clusterName(cluster.getName()) - .keyspaceName(ksName) - .columnFamilies(cfNames) - .incrementalRepair(incrementalRepair) - .nodes(nodeSet) - .datacenters(datacenters) - .blacklistedTables(blacklistedTables) - .repairThreadCount(repairThreadCount) - .timeout(segmentTimeout)); - DateTimeUtils.setCurrentMillisFixed(timeRun); - RepairRun run = storage.getRepairRunDao().addRepairRun( - RepairRun.builder(cluster.getName(), cf.getId()) - .intensity(intensity) - .segmentCount(1) - .repairParallelism(RepairParallelism.PARALLEL) - .tables(TABLES), - Collections.singleton( - RepairSegment.builder( - Segment.builder() - .withTokenRange(new RingRange(BigInteger.ZERO, new BigInteger("100"))) - .withReplicas(replicas) - .build(), - cf.getId()))); - final UUID runId = run.getId(); - final UUID segmentId = storage.getRepairSegmentDao().getNextFreeSegments(run.getId()).get(0).getId(); - assertEquals(storage.getRepairSegmentDao().getRepairSegment(runId, segmentId).get().getState(), - RepairSegment.State.NOT_STARTED); - AppContext context = new AppContext(); - context.storage = storage; - context.config = new ReaperApplicationConfiguration(); - final Semaphore mutex = new Semaphore(0); - final JmxCassandraManagementProxy jmx = CassandraManagementProxyTest.mockJmxProxyImpl(); - Map endpointToHostIDMap = endpointToHostIDMap(); - when(jmx.getEndpointToHostId()).thenReturn(endpointToHostIDMap); - when(jmx.getClusterName()).thenReturn(cluster.getName()); - when(jmx.getRangeToEndpointMap(anyString())).thenReturn(RepairRunnerTest.sixNodeCluster()); - EndpointSnitchInfoMBean endpointSnitchInfoMBean = mock(EndpointSnitchInfoMBean.class); - when(endpointSnitchInfoMBean.getDatacenter()).thenReturn("dc1"); - try { - when(endpointSnitchInfoMBean.getDatacenter(anyString())).thenReturn("dc1"); - } catch (UnknownHostException ex) { - throw new AssertionError(ex); - } - final AtomicInteger repairAttempts = new AtomicInteger(1); - when(jmx.triggerRepair(any(), any(), any(), anyBoolean(), any(), any(), any(), anyInt())) - .then( - (invocation) -> { - assertEquals(RepairSegment.State.STARTED, - storage.getRepairSegmentDao().getRepairSegment(runId, segmentId).get().getState()); - final int repairNumber = repairAttempts.getAndIncrement(); - switch (repairNumber) { - case 1: - new Thread() { - @Override - public void run() { - ((RepairStatusHandler) invocation.getArgument(5)) - .handle(repairNumber, - Optional.of(ActiveRepairService.Status.STARTED), - Optional.empty(), null, jmx); - assertEquals( - RepairSegment.State.RUNNING, - storage.getRepairSegmentDao().getRepairSegment(runId, segmentId).get().getState()); - } - }.start(); - break; - case 2: - new Thread() { - @Override - public void run() { - ((RepairStatusHandler) invocation.getArgument(5)) - .handle( - repairNumber, - Optional.of(ActiveRepairService.Status.STARTED), - Optional.empty(), null, jmx); - assertEquals( - RepairSegment.State.RUNNING, - storage.getRepairSegmentDao().getRepairSegment(runId, segmentId).get().getState()); - ((RepairStatusHandler) invocation.getArgument(5)) - .handle( - repairNumber, - Optional.of(ActiveRepairService.Status.SESSION_SUCCESS), - Optional.empty(), null, jmx); - assertEquals( - RepairSegment.State.DONE, - storage.getRepairSegmentDao().getRepairSegment(runId, segmentId).get().getState()); - ((RepairStatusHandler) invocation.getArgument(5)) - .handle(repairNumber, - Optional.of(ActiveRepairService.Status.FINISHED), - Optional.empty(), null, jmx); - - mutex.release(); - LOG.info("MUTEX RELEASED"); - } - }.start(); - break; - default: - fail("triggerRepair should only have been called twice"); - } - LOG.info("repair number : " + repairNumber); - return repairNumber; - }); - ClusterFacade clusterFacade = mock(ClusterFacade.class); - when(clusterFacade.connect(any(Cluster.class), any())).thenReturn(jmx); - when(clusterFacade.nodeIsDirectlyAccessible(any(), any())).thenReturn(true); - when(clusterFacade.tokenRangeToEndpoint(any(), anyString(), any())).thenReturn(Lists.newArrayList(nodeSet)); - when(clusterFacade.getEndpointToHostId(any())).thenReturn(endpointToHostIDMap); - when(clusterFacade.listActiveCompactions(any())).thenReturn(CompactionStats.builder().withActiveCompactions( - Collections.emptyList()).withPendingCompactions(Optional.of(0)).build()); - when(clusterFacade.getRangeToEndpointMap(any(), anyString())) - .thenReturn((Map) ImmutableMap.of(Lists.newArrayList("0", "100"), Lists.newArrayList(nodeSet))); - context.repairManager = RepairManager.create( - context, - clusterFacade, - Executors.newScheduledThreadPool(1), - 1, - TimeUnit.MILLISECONDS, - 1, - context.storage.getRepairRunDao()); - context.managementConnectionFactory = new JmxManagementConnectionFactory(context, new NoopCrypotograph()) { - @Override - protected JmxCassandraManagementProxy connectImpl(Node host) throws ReaperException { - return jmx; - } - }; - context.repairManager.startRepairRun(run); - await().with().atMost(2, TimeUnit.MINUTES).until(() -> { - try { - mutex.acquire(); - LOG.info("MUTEX ACQUIRED"); - Thread.sleep(1000); - return true; - } catch (InterruptedException ex) { - throw new IllegalStateException(ex); - } - }); - assertEquals(RepairRun.RunState.DONE, storage.getRepairRunDao().getRepairRun(runId).get().getRunState()); - } - - @Test - public void testHangingRepairNewApi() throws InterruptedException, ReaperException, MalformedObjectNameException, - ReflectionException, IOException { - final String ksName = "reaper"; - final Set cfNames = Sets.newHashSet("reaper"); - final boolean incrementalRepair = false; - final Set nodeSet = Sets.newHashSet("127.0.0.1"); - final Set datacenters = Collections.emptySet(); - final Set blacklistedTables = Collections.emptySet(); - final long timeRun = 41L; - final double intensity = 0.5f; - final int repairThreadCount = 1; - final int segmentTimeout = 1; - final IStorageDao storage = new MemoryStorageFacade(); - storage.getClusterDao().addCluster(cluster); - DateTimeUtils.setCurrentMillisFixed(timeRun); - RepairUnit cf = storage.getRepairUnitDao().addRepairUnit( - RepairUnit.builder() - .clusterName(cluster.getName()) - .keyspaceName(ksName) - .columnFamilies(cfNames) - .incrementalRepair(incrementalRepair) - .nodes(nodeSet) - .datacenters(datacenters) - .blacklistedTables(blacklistedTables) - .repairThreadCount(repairThreadCount) - .timeout(segmentTimeout)); - RepairRun run = storage.getRepairRunDao().addRepairRun( - RepairRun.builder(cluster.getName(), cf.getId()) - .intensity(intensity).segmentCount(1) - .repairParallelism(RepairParallelism.PARALLEL) - .tables(TABLES), - Collections.singleton( - RepairSegment.builder( - Segment.builder() - .withTokenRange(new RingRange(BigInteger.ZERO, new BigInteger("100"))) - .withReplicas(replicas) - .build(), - cf.getId()))); - final UUID runId = run.getId(); - final UUID segmentId = storage.getRepairSegmentDao().getNextFreeSegments(run.getId()).get(0).getId(); - assertEquals(storage.getRepairSegmentDao().getRepairSegment(runId, segmentId).get().getState(), - RepairSegment.State.NOT_STARTED); - AppContext context = new AppContext(); - context.storage = storage; - context.config = new ReaperApplicationConfiguration(); - final Semaphore mutex = new Semaphore(0); - final JmxCassandraManagementProxy jmx = CassandraManagementProxyTest.mockJmxProxyImpl(); - when(jmx.getClusterName()).thenReturn(cluster.getName()); - when(jmx.getRangeToEndpointMap(anyString())).thenReturn(RepairRunnerTest.sixNodeCluster()); - EndpointSnitchInfoMBean endpointSnitchInfoMBean = mock(EndpointSnitchInfoMBean.class); - when(endpointSnitchInfoMBean.getDatacenter()).thenReturn("dc1"); - try { - when(endpointSnitchInfoMBean.getDatacenter(anyString())).thenReturn("dc1"); - } catch (UnknownHostException ex) { - throw new AssertionError(ex); - } - final AtomicInteger repairAttempts = new AtomicInteger(1); - when(jmx.triggerRepair(any(), any(), any(), anyBoolean(), any(), any(), any(), anyInt())) - .then( - (invocation) -> { - assertEquals( - RepairSegment.State.STARTED, - storage.getRepairSegmentDao().getRepairSegment(runId, segmentId).get().getState()); - final int repairNumber = repairAttempts.getAndIncrement(); - switch (repairNumber) { - case 1: - new Thread() { - @Override - public void run() { - ((RepairStatusHandler) invocation.getArgument(5)) - .handle( - repairNumber, Optional.empty(), - Optional.of(ProgressEventType.START), null, jmx); - assertEquals( - RepairSegment.State.RUNNING, - storage.getRepairSegmentDao().getRepairSegment(runId, segmentId).get().getState()); - } - }.start(); - break; - case 2: - new Thread() { - @Override - public void run() { - ((RepairStatusHandler) invocation.getArgument(5)) - .handle( - repairNumber, Optional.empty(), - Optional.of(ProgressEventType.START), null, jmx); - assertEquals( - RepairSegment.State.RUNNING, - storage.getRepairSegmentDao().getRepairSegment(runId, segmentId).get().getState()); - ((RepairStatusHandler) invocation.getArgument(5)) - .handle( - repairNumber, Optional.empty(), - Optional.of(ProgressEventType.SUCCESS), null, jmx); - assertEquals( - RepairSegment.State.DONE, - storage.getRepairSegmentDao().getRepairSegment(runId, segmentId).get().getState()); - ((RepairStatusHandler) invocation.getArgument(5)) - .handle( - repairNumber, Optional.empty(), - Optional.of(ProgressEventType.COMPLETE), null, jmx); - mutex.release(); - LOG.info("MUTEX RELEASED"); - } - }.start(); - break; - default: - fail("triggerRepair should only have been called twice"); - } - return repairNumber; - }); - ClusterFacade clusterFacade = mock(ClusterFacade.class); - when(clusterFacade.connect(any(Cluster.class), any())).thenReturn(jmx); - when(clusterFacade.nodeIsDirectlyAccessible(any(), any())).thenReturn(true); - when(clusterFacade.tokenRangeToEndpoint(any(), anyString(), any())) - .thenReturn(Lists.newArrayList(nodeSet)); - when(clusterFacade.getRangeToEndpointMap(any(), anyString())) - .thenReturn((Map) ImmutableMap.of(Lists.newArrayList("0", "100"), Lists.newArrayList(nodeSet))); - when(clusterFacade.listActiveCompactions(any())).thenReturn(CompactionStats.builder().withActiveCompactions( - Collections.emptyList()).withPendingCompactions(Optional.of(0)).build()); - context.repairManager - = RepairManager.create( - context, - clusterFacade, - Executors.newScheduledThreadPool(1), - 1, - TimeUnit.MILLISECONDS, - 1, - context.storage.getRepairRunDao()); - context.managementConnectionFactory = new JmxManagementConnectionFactory(context, new NoopCrypotograph()) { - @Override - protected JmxCassandraManagementProxy connectImpl(Node host) throws ReaperException { - return jmx; - } - }; - context.repairManager.startRepairRun(run); - await().with().atMost(2, TimeUnit.MINUTES).until(() -> { - try { - mutex.acquire(); - LOG.info("MUTEX ACQUIRED"); - // TODO: refactor so that we can properly wait for the repair runner to finish rather than using this sleep() - Thread.sleep(1000); - return true; - } catch (InterruptedException ex) { - throw new IllegalStateException(ex); - } - }); - assertEquals(RepairRun.RunState.DONE, storage.getRepairRunDao().getRepairRun(runId).get().getRunState()); - } - @Test public void testResumeRepair() throws InterruptedException, ReaperException, MalformedObjectNameException, ReflectionException, IOException { @@ -547,6 +234,7 @@ public void testResumeRepair() throws InterruptedException, ReaperException, Mal .keyspaceName(ksName) .columnFamilies(cfNames) .incrementalRepair(incrementalRepair) + .subrangeIncrementalRepair(incrementalRepair) .nodes(nodeSet) .datacenters(datacenters) .blacklistedTables(blacklistedTables) @@ -594,7 +282,7 @@ public void testResumeRepair() throws InterruptedException, ReaperException, Mal context.storage.getRepairRunDao()); AtomicInteger repairNumberCounter = new AtomicInteger(1); - when(jmx.triggerRepair(any(), any(), any(), anyBoolean(), any(), any(), any(), anyInt())) + when(jmx.triggerRepair(any(), any(), any(), any(), any(), any(), any(), anyInt())) .then( (invocation) -> { final int repairNumber = repairNumberCounter.getAndIncrement(); @@ -676,6 +364,7 @@ public void testTooManyPendingCompactions() .keyspaceName(ksName) .columnFamilies(cfNames) .incrementalRepair(incrementalRepair) + .subrangeIncrementalRepair(incrementalRepair) .nodes(nodeSet) .datacenters(datacenters) .blacklistedTables(blacklistedTables) @@ -721,7 +410,7 @@ public void testTooManyPendingCompactions() 1, context.storage.getRepairRunDao()); AtomicInteger repairNumberCounter = new AtomicInteger(1); - when(jmx.triggerRepair(any(), any(), any(), anyBoolean(), any(), any(), any(), anyInt())) + when(jmx.triggerRepair(any(), any(), any(), any(), any(), any(), any(), anyInt())) .then( (invocation) -> { final int repairNumber = repairNumberCounter.getAndIncrement(); @@ -868,6 +557,7 @@ public void testDontFailRepairAfterTopologyChange() throws InterruptedException, .keyspaceName(ksName) .columnFamilies(cfNames) .incrementalRepair(incrementalRepair) + .subrangeIncrementalRepair(incrementalRepair) .nodes(nodeSet) .datacenters(datacenters) .blacklistedTables(blacklistedTables) @@ -879,7 +569,8 @@ public void testDontFailRepairAfterTopologyChange() throws InterruptedException, intensity, storage, cf, - null); + null, + 1); final UUID runId = run.getId(); final UUID segmentId = storage.getRepairSegmentDao().getNextFreeSegments(run.getId()).get(0).getId(); assertEquals(storage.getRepairSegmentDao().getRepairSegment(runId, segmentId).get().getState(), @@ -921,7 +612,7 @@ public void testDontFailRepairAfterTopologyChange() throws InterruptedException, 1, context.storage.getRepairRunDao()); AtomicInteger repairNumberCounter = new AtomicInteger(1); - when(jmx.triggerRepair(any(), any(), any(), anyBoolean(), any(), any(), any(), anyInt())) + when(jmx.triggerRepair(any(), any(), any(), any(), any(), any(), any(), anyInt())) .then( (invocation) -> { final int repairNumber = repairNumberCounter.getAndIncrement(); @@ -981,74 +672,15 @@ protected JmxCassandraManagementProxy connectImpl(Node host) throws ReaperExcept }); } - private Map endpointToHostIDMap() { - Map endpointToHostIDMap = new HashMap(); - endpointToHostIDMap.put("127.0.0.1", UUID.randomUUID().toString()); - endpointToHostIDMap.put("127.0.0.2", UUID.randomUUID().toString()); - endpointToHostIDMap.put("127.0.0.3", UUID.randomUUID().toString()); - - return endpointToHostIDMap; - } - - private RepairRun addNewRepairRun( - final Map nodeMap, - final double intensity, - final IStorageDao storage, - UUID cf, - UUID hostID - ) { - return storage.getRepairRunDao().addRepairRun( - RepairRun.builder(cluster.getName(), cf) - .intensity(intensity) - .segmentCount(1) - .repairParallelism(RepairParallelism.PARALLEL) - .tables(TABLES), - Lists.newArrayList( - RepairSegment.builder( - Segment.builder() - .withTokenRange(new RingRange(BigInteger.ZERO, new BigInteger("100"))) - .withReplicas(nodeMap) - .build(), cf) - .withState(RepairSegment.State.RUNNING) - .withStartTime(DateTime.now()) - .withCoordinatorHost("reaper") - .withHostID(hostID), - RepairSegment.builder( - Segment.builder() - .withTokenRange(new RingRange(new BigInteger("100"), new BigInteger("200"))) - .withReplicas(nodeMap) - .build(), cf) - .withHostID(hostID) - ) - ); - } - @Test - public void isItOkToRepairTest() { - assertFalse(RepairRunner.okToRepairSegment(false, true, DatacenterAvailability.ALL)); - assertFalse(RepairRunner.okToRepairSegment(false, false, DatacenterAvailability.ALL)); - assertTrue(RepairRunner.okToRepairSegment(true, true, DatacenterAvailability.ALL)); - - assertTrue(RepairRunner.okToRepairSegment(false, true, DatacenterAvailability.LOCAL)); - assertFalse(RepairRunner.okToRepairSegment(false, false, DatacenterAvailability.LOCAL)); - assertTrue(RepairRunner.okToRepairSegment(true, true, DatacenterAvailability.LOCAL)); - - assertFalse(RepairRunner.okToRepairSegment(false, true, DatacenterAvailability.EACH)); - assertFalse(RepairRunner.okToRepairSegment(false, false, DatacenterAvailability.EACH)); - assertTrue(RepairRunner.okToRepairSegment(true, true, DatacenterAvailability.EACH)); - } - - @Test - public void testDontFailRepairAfterTopologyChangeIncrementalRepair() throws InterruptedException, ReaperException, + public void testSubrangeIncrementalRepair() throws InterruptedException, ReaperException, MalformedObjectNameException, ReflectionException, IOException { final String ksName = "reaper"; final Set cfNames = Sets.newHashSet("reaper"); final boolean incrementalRepair = true; + final boolean subrangeIncrementalRepair = true; final Set nodeSet = Sets.newHashSet("127.0.0.1", "127.0.0.2", "127.0.0.3"); - final List nodeSetAfterTopologyChange = Lists.newArrayList("127.0.0.3", "127.0.0.2", "127.0.0.4"); final Map nodeMap = ImmutableMap.of("127.0.0.1", "dc1", "127.0.0.2", "dc1", "127.0.0.3", "dc1"); - final Map nodeMapAfterTopologyChange = ImmutableMap.of( - "127.0.0.3", "dc1", "127.0.0.2", "dc1", "127.0.0.4", "dc1"); final Set datacenters = Collections.emptySet(); final Set blacklistedTables = Collections.emptySet(); final double intensity = 0.5f; @@ -1066,25 +698,29 @@ public void testDontFailRepairAfterTopologyChangeIncrementalRepair() throws Inte .keyspaceName(ksName) .columnFamilies(cfNames) .incrementalRepair(incrementalRepair) + .subrangeIncrementalRepair(subrangeIncrementalRepair) .nodes(nodeSet) .datacenters(datacenters) .blacklistedTables(blacklistedTables) .repairThreadCount(repairThreadCount) .timeout(segmentTimeout)) .getId(); - Map endpointToHostIDMap = endpointToHostIDMap(); + final Map endpointToHostIDMap = endpointToHostIDMap(); RepairRun run = addNewRepairRun(nodeMap, intensity, storage, cf, - UUID.fromString(endpointToHostIDMap.get("127.0.0.1"))); + null, + 30); final UUID runId = run.getId(); + assertTrue(storage.getRepairRunDao().getRepairRun(runId).get().getSegmentCount() > 3); final UUID segmentId = storage.getRepairSegmentDao().getNextFreeSegments(run.getId()).get(0).getId(); assertEquals(storage.getRepairSegmentDao().getRepairSegment(runId, segmentId).get().getState(), RepairSegment.State.NOT_STARTED); final JmxCassandraManagementProxy jmx = CassandraManagementProxyTest.mockJmxProxyImpl(); when(jmx.getClusterName()).thenReturn(cluster.getName()); when(jmx.getRangeToEndpointMap(anyString())).thenReturn(RepairRunnerTest.threeNodeClusterWithIps()); + when(jmx.getEndpointToHostId()).thenReturn(endpointToHostIDMap); when(jmx.getTokens()).thenReturn(tokens); EndpointSnitchInfoMBean endpointSnitchInfoMBean = mock(EndpointSnitchInfoMBean.class); @@ -1094,7 +730,6 @@ public void testDontFailRepairAfterTopologyChangeIncrementalRepair() throws Inte } catch (UnknownHostException ex) { throw new AssertionError(ex); } - ClusterFacade clusterFacade = mock(ClusterFacade.class); when(clusterFacade.connect(any(Cluster.class), any())).thenReturn(jmx); when(clusterFacade.nodeIsDirectlyAccessible(any(), any())).thenReturn(true); @@ -1104,7 +739,7 @@ public void testDontFailRepairAfterTopologyChangeIncrementalRepair() throws Inte .thenReturn((Map) ImmutableMap.of( Lists.newArrayList("0", "100"), Lists.newArrayList(nodeSet), Lists.newArrayList("100", "200"), Lists.newArrayList(nodeSet))); - when(clusterFacade.getEndpointToHostId(any())).thenReturn(endpointToHostIDMap); + when(clusterFacade.getEndpointToHostId(any())).thenReturn(nodeMap); when(clusterFacade.listActiveCompactions(any())).thenReturn( CompactionStats.builder() .withActiveCompactions(Collections.emptyList()) @@ -1119,7 +754,12 @@ public void testDontFailRepairAfterTopologyChangeIncrementalRepair() throws Inte 1, context.storage.getRepairRunDao()); AtomicInteger repairNumberCounter = new AtomicInteger(1); - when(jmx.triggerRepair(any(), any(), any(), anyBoolean(), any(), any(), any(), anyInt())) + // triggerRepair is only configured for incremental runs and will fail if full repair is requested + when(jmx.triggerRepair(any(), any(), any(), eq(RepairType.INCREMENTAL), any(), any(), any(), anyInt())) + .thenThrow(new RuntimeException()); + when(jmx.triggerRepair(any(), any(), any(), eq(RepairType.SUBRANGE_FULL), any(), any(), any(), anyInt())) + .thenThrow(new RuntimeException()); + when(jmx.triggerRepair(any(), any(), any(), eq(RepairType.SUBRANGE_INCREMENTAL), any(), any(), any(), anyInt())) .then( (invocation) -> { final int repairNumber = repairNumberCounter.getAndIncrement(); @@ -1157,36 +797,80 @@ protected JmxCassandraManagementProxy connectImpl(Node host) throws ReaperExcept return jmx; } }; - ClusterFacade clusterProxy = ClusterFacade.create(context); - ClusterFacade clusterProxySpy = Mockito.spy(clusterProxy); - Mockito.doReturn(nodeSetAfterTopologyChange).when(clusterProxySpy).tokenRangeToEndpoint(any(), any(), any()); assertEquals(RepairRun.RunState.NOT_STARTED, storage.getRepairRunDao().getRepairRun(runId).get().getRunState()); storage.getRepairRunDao().updateRepairRun( run.with().runState(RepairRun.RunState.RUNNING).startTime(DateTime.now()).build(runId)); - // We'll now change the list of replicas for any segment, making the stored ones obsolete - when(clusterFacade.getRangeToEndpointMap(any(), anyString())) - .thenReturn((Map) ImmutableMap.of( - Lists.newArrayList("0", "100"), Lists.newArrayList(nodeSetAfterTopologyChange), - Lists.newArrayList("100", "200"), Lists.newArrayList(nodeSetAfterTopologyChange))); - String hostIdToChange = endpointToHostIDMap.get("127.0.0.1"); - endpointToHostIDMap.remove("127.0.0.1"); - endpointToHostIDMap.put("127.0.0.4", hostIdToChange); - when(clusterFacade.getEndpointToHostId(any())).thenReturn(endpointToHostIDMap); - when(clusterFacade.tokenRangeToEndpoint(any(), anyString(), any())) - .thenReturn(Lists.newArrayList(nodeSetAfterTopologyChange)); context.repairManager.resumeRunningRepairRuns(); - // The repair run should succeed despite the topology change. - await().with().atMost(60, TimeUnit.SECONDS).until(() -> { + await().with().atMost(20, TimeUnit.SECONDS).until(() -> { return RepairRun.RunState.DONE == storage.getRepairRunDao().getRepairRun(runId).get().getRunState(); }); } + private Map endpointToHostIDMap() { + Map endpointToHostIDMap = new HashMap(); + endpointToHostIDMap.put("127.0.0.1", UUID.randomUUID().toString()); + endpointToHostIDMap.put("127.0.0.2", UUID.randomUUID().toString()); + endpointToHostIDMap.put("127.0.0.3", UUID.randomUUID().toString()); + + return endpointToHostIDMap; + } + + private RepairRun addNewRepairRun( + final Map nodeMap, + final double intensity, + final IStorageDao storage, + UUID cf, + UUID hostID, + int segmentCount + ) { + return storage.getRepairRunDao().addRepairRun( + RepairRun.builder(cluster.getName(), cf) + .intensity(intensity) + .segmentCount(segmentCount) + .repairParallelism(RepairParallelism.PARALLEL) + .tables(TABLES), + Lists.newArrayList( + RepairSegment.builder( + Segment.builder() + .withTokenRange(new RingRange(BigInteger.ZERO, new BigInteger("100"))) + .withReplicas(nodeMap) + .build(), cf) + .withState(RepairSegment.State.RUNNING) + .withStartTime(DateTime.now()) + .withCoordinatorHost("reaper") + .withHostID(hostID), + RepairSegment.builder( + Segment.builder() + .withTokenRange(new RingRange(new BigInteger("100"), new BigInteger("200"))) + .withReplicas(nodeMap) + .build(), cf) + .withHostID(hostID) + ) + ); + } + + @Test + public void isItOkToRepairTest() { + assertFalse(RepairRunner.okToRepairSegment(false, true, DatacenterAvailability.ALL)); + assertFalse(RepairRunner.okToRepairSegment(false, false, DatacenterAvailability.ALL)); + assertTrue(RepairRunner.okToRepairSegment(true, true, DatacenterAvailability.ALL)); + + assertTrue(RepairRunner.okToRepairSegment(false, true, DatacenterAvailability.LOCAL)); + assertFalse(RepairRunner.okToRepairSegment(false, false, DatacenterAvailability.LOCAL)); + assertTrue(RepairRunner.okToRepairSegment(true, true, DatacenterAvailability.LOCAL)); + + assertFalse(RepairRunner.okToRepairSegment(false, true, DatacenterAvailability.EACH)); + assertFalse(RepairRunner.okToRepairSegment(false, false, DatacenterAvailability.EACH)); + assertTrue(RepairRunner.okToRepairSegment(true, true, DatacenterAvailability.EACH)); + } + @Test public void getNodeMetricsInLocalDcAvailabilityForRemoteDcNodeTest() throws Exception { final String ksName = "reaper"; final Set cfNames = Sets.newHashSet("reaper"); final boolean incrementalRepair = false; + final boolean subrangeIncremental = false; final Set nodeSet = Sets.newHashSet("127.0.0.1", "127.0.0.2", "127.0.0.3"); final List nodeSetAfterTopologyChange = Lists.newArrayList("127.0.0.1", "127.0.0.2", "127.0.0.4"); final Map nodeMap = ImmutableMap.of( @@ -1205,6 +889,7 @@ public void getNodeMetricsInLocalDcAvailabilityForRemoteDcNodeTest() throws Exce .keyspaceName(ksName) .columnFamilies(cfNames) .incrementalRepair(incrementalRepair) + .subrangeIncrementalRepair(subrangeIncremental) .nodes(nodeSet) .datacenters(datacenters) .blacklistedTables(blacklistedTables) @@ -1288,6 +973,7 @@ public void getNodeMetricsInLocalDcAvailabilityForLocalDcNodeTest() throws Excep .keyspaceName(ksName) .columnFamilies(cfNames) .incrementalRepair(incrementalRepair) + .subrangeIncrementalRepair(incrementalRepair) .nodes(nodeSet) .datacenters(datacenters) .blacklistedTables(blacklistedTables) @@ -1300,7 +986,8 @@ public void getNodeMetricsInLocalDcAvailabilityForLocalDcNodeTest() throws Excep intensity, storage, cf, - null + null, + 1 ); final UUID runId = run.getId(); final UUID segmentId = storage.getRepairSegmentDao().getNextFreeSegments(run.getId()).get(0).getId(); @@ -1350,6 +1037,7 @@ public void adaptiveRepairReduceSegmentsTest() throws Exception { final String ksName = "reaper"; final Set cfNames = Sets.newHashSet("reaper"); final boolean incrementalRepair = false; + final boolean subrangeIncremental = false; final Set nodeSet = Sets.newHashSet("127.0.0.1", "127.0.0.2", "127.0.0.3"); final Map nodeMap = ImmutableMap.of( "127.0.0.1", "dc1", "127.0.0.2", "dc1", "127.0.0.3", "dc1" @@ -1367,6 +1055,7 @@ public void adaptiveRepairReduceSegmentsTest() throws Exception { .keyspaceName(ksName) .columnFamilies(cfNames) .incrementalRepair(incrementalRepair) + .subrangeIncrementalRepair(subrangeIncremental) .nodes(nodeSet) .datacenters(datacenters) .blacklistedTables(blacklistedTables) @@ -1448,6 +1137,7 @@ public void adaptiveRepairAddSegmentsTest() throws Exception { final String ksName = "reaper"; final Set cfNames = Sets.newHashSet("reaper"); final boolean incrementalRepair = false; + final boolean subrangeIncremental = false; final Set nodeSet = Sets.newHashSet("127.0.0.1", "127.0.0.2", "127.0.0.3"); final Map nodeMap = ImmutableMap.of( "127.0.0.1", "dc1", "127.0.0.2", "dc1", "127.0.0.3", "dc1" @@ -1465,6 +1155,7 @@ public void adaptiveRepairAddSegmentsTest() throws Exception { .keyspaceName(ksName) .columnFamilies(cfNames) .incrementalRepair(incrementalRepair) + .subrangeIncrementalRepair(subrangeIncremental) .nodes(nodeSet) .datacenters(datacenters) .blacklistedTables(blacklistedTables) @@ -1545,6 +1236,7 @@ public void adaptiveRepairRaiseTimeoutTest() throws Exception { final String ksName = "reaper"; final Set cfNames = Sets.newHashSet("reaper"); final boolean incrementalRepair = false; + final boolean subrangeIncremental = false; final Set nodeSet = Sets.newHashSet("127.0.0.1", "127.0.0.2", "127.0.0.3"); final Map nodeMap = ImmutableMap.of( "127.0.0.1", "dc1", "127.0.0.2", "dc1", "127.0.0.3", "dc1" @@ -1563,6 +1255,7 @@ public void adaptiveRepairRaiseTimeoutTest() throws Exception { .keyspaceName(ksName) .columnFamilies(cfNames) .incrementalRepair(incrementalRepair) + .subrangeIncrementalRepair(subrangeIncremental) .nodes(nodeSet) .datacenters(datacenters) .blacklistedTables(blacklistedTables) @@ -1647,6 +1340,7 @@ public void notAdaptiveScheduleTest() throws Exception { final String ksName = "reaper"; final Set cfNames = Sets.newHashSet("reaper"); final boolean incrementalRepair = false; + final boolean subrangeIncremental = false; final Set nodeSet = Sets.newHashSet("127.0.0.1", "127.0.0.2", "127.0.0.3"); final Map nodeMap = ImmutableMap.of( "127.0.0.1", "dc1", "127.0.0.2", "dc1", "127.0.0.3", "dc1" @@ -1665,6 +1359,7 @@ public void notAdaptiveScheduleTest() throws Exception { .keyspaceName(ksName) .columnFamilies(cfNames) .incrementalRepair(incrementalRepair) + .subrangeIncrementalRepair(subrangeIncremental) .nodes(nodeSet) .datacenters(datacenters) .blacklistedTables(blacklistedTables) @@ -1751,6 +1446,7 @@ public void isAllowedToRunTest() throws ReaperException { final String ksName = "reaper"; final Set cfNames = Sets.newHashSet("reaper"); final boolean incrementalRepair = false; + final boolean subrangeIncremental = false; final Set nodeSet = Sets.newHashSet("127.0.0.1", "127.0.0.2", "127.0.0.3"); final Map nodeMap = ImmutableMap.of( "127.0.0.1", "dc1", "127.0.0.2", "dc1", "127.0.0.3", "dc1" @@ -1774,6 +1470,7 @@ public void isAllowedToRunTest() throws ReaperException { .keyspaceName(ksName) .columnFamilies(cfNames) .incrementalRepair(incrementalRepair) + .subrangeIncrementalRepair(subrangeIncremental) .nodes(nodeSet) .datacenters(datacenters) .blacklistedTables(blacklistedTables) @@ -1826,6 +1523,7 @@ public void isNotAllowedToRunTest() throws ReaperException { final String ksName = "reaper"; final Set cfNames = Sets.newHashSet("reaper"); final boolean incrementalRepair = false; + final boolean subrangeIncremental = false; final Set nodeSet = Sets.newHashSet("127.0.0.1", "127.0.0.2", "127.0.0.3"); final Map nodeMap = ImmutableMap.of( "127.0.0.1", "dc1", "127.0.0.2", "dc1", "127.0.0.3", "dc1" @@ -1849,6 +1547,7 @@ public void isNotAllowedToRunTest() throws ReaperException { .keyspaceName(ksName) .columnFamilies(cfNames) .incrementalRepair(incrementalRepair) + .subrangeIncrementalRepair(subrangeIncremental) .nodes(nodeSet) .datacenters(datacenters) .blacklistedTables(blacklistedTables) diff --git a/src/server/src/test/java/io/cassandrareaper/service/RepairUnitServiceTest.java b/src/server/src/test/java/io/cassandrareaper/service/RepairUnitServiceTest.java index ae0b979a4..548a4bea4 100644 --- a/src/server/src/test/java/io/cassandrareaper/service/RepairUnitServiceTest.java +++ b/src/server/src/test/java/io/cassandrareaper/service/RepairUnitServiceTest.java @@ -106,6 +106,7 @@ public void getTablesToRepairRemoveOneTableTest() throws ReaperException, Unknow .keyspaceName("test") .blacklistedTables(Sets.newHashSet("table1")) .incrementalRepair(false) + .subrangeIncrementalRepair(false) .repairThreadCount(4) .timeout(30) .build(UUIDs.timeBased()); @@ -130,6 +131,7 @@ public void getTablesToRepairDefaultCompactionStrategyTable() throws ReaperExcep .keyspaceName("test") .blacklistedTables(Sets.newHashSet("table1")) .incrementalRepair(false) + .subrangeIncrementalRepair(false) .repairThreadCount(4) .timeout(30) .build(UUIDs.timeBased()); @@ -153,6 +155,7 @@ public void getTablesToRepairRemoveOneTableWithTwcsTest() throws ReaperException .clusterName(cluster.getName()) .keyspaceName("test") .incrementalRepair(false) + .subrangeIncrementalRepair(false) .repairThreadCount(4) .timeout(30) .build(UUIDs.timeBased()); @@ -177,6 +180,7 @@ public void getTablesToRepairRemoveTwoTablesTest() throws ReaperException, Unkno .keyspaceName("test") .blacklistedTables(Sets.newHashSet("table1", "table3")) .incrementalRepair(false) + .subrangeIncrementalRepair(false) .repairThreadCount(4) .timeout(30) .build(UUIDs.timeBased()); @@ -201,6 +205,7 @@ public void getTablesToRepairRemoveTwoTablesOneWithTwcsTest() throws ReaperExcep .keyspaceName("test") .blacklistedTables(Sets.newHashSet("table1")) .incrementalRepair(false) + .subrangeIncrementalRepair(false) .repairThreadCount(4) .timeout(30) .build(UUIDs.timeBased()); @@ -226,6 +231,7 @@ public void getTablesToRepairRemoveOneTableFromListTest() throws ReaperException .columnFamilies(Sets.newHashSet("table1", "table2")) .blacklistedTables(Sets.newHashSet("table1")) .incrementalRepair(false) + .subrangeIncrementalRepair(false) .repairThreadCount(4) .timeout(30) .build(UUIDs.timeBased()); @@ -251,6 +257,7 @@ public void getTablesToRepairRemoveOneTableFromListOneWithTwcsTest() throws Reap .columnFamilies(Sets.newHashSet("table1", "table2")) .blacklistedTables(Sets.newHashSet("table1")) .incrementalRepair(false) + .subrangeIncrementalRepair(false) .repairThreadCount(4) .timeout(30) .build(UUIDs.timeBased()); @@ -275,6 +282,7 @@ public void getTablesToRepairRemoveAllFailingTest() throws ReaperException, Unkn .keyspaceName("test") .blacklistedTables(Sets.newHashSet("table1", "table2", "table3")) .incrementalRepair(false) + .subrangeIncrementalRepair(false) .repairThreadCount(4) .timeout(30) .build(UUIDs.timeBased()); @@ -301,6 +309,7 @@ public void getTablesToRepairRemoveAllFromListFailingTest() throws ReaperExcepti .columnFamilies(Sets.newHashSet("table1", "table2", "table3")) .blacklistedTables(Sets.newHashSet("table1", "table2", "table3")) .incrementalRepair(false) + .subrangeIncrementalRepair(false) .repairThreadCount(4) .timeout(30) .build(UUIDs.timeBased()); @@ -327,6 +336,7 @@ public void conflictingRepairUnitsTest() throws ReaperException, UnknownHostExce .columnFamilies(Sets.newHashSet("table1", "table2", "table3")) .blacklistedTables(Sets.newHashSet("table1", "table2", "table3")) .incrementalRepair(false) + .subrangeIncrementalRepair(false) .repairThreadCount(4) .timeout(30) .build(UUIDs.timeBased()); @@ -337,6 +347,7 @@ public void conflictingRepairUnitsTest() throws ReaperException, UnknownHostExce .columnFamilies(Sets.newHashSet("table1", "table2", "table3")) .blacklistedTables(Sets.newHashSet("table1", "table2", "table3")) .incrementalRepair(false) + .subrangeIncrementalRepair(false) .repairThreadCount(4) .timeout(30); @@ -362,6 +373,7 @@ public void conflictingRepairUnitsDiffKSTest() throws ReaperException, UnknownHo .columnFamilies(Sets.newHashSet("table1", "table2", "table3")) .blacklistedTables(Sets.newHashSet("table1", "table2", "table3")) .incrementalRepair(false) + .subrangeIncrementalRepair(false) .repairThreadCount(4) .timeout(30) .build(UUIDs.timeBased()); @@ -395,6 +407,7 @@ public void conflictingRepairUnitsNoTablesTest() throws ReaperException, Unknown .clusterName(cluster.getName()) .keyspaceName("test") .incrementalRepair(false) + .subrangeIncrementalRepair(false) .repairThreadCount(4) .timeout(30) .build(UUIDs.timeBased()); @@ -426,6 +439,7 @@ public void notConflictingRepairUnitsTest() throws ReaperException, UnknownHostE .clusterName(cluster.getName()) .keyspaceName("test") .incrementalRepair(false) + .subrangeIncrementalRepair(false) .repairThreadCount(4) .timeout(30) .build(UUIDs.timeBased()); @@ -459,6 +473,7 @@ public void identicalRepairUnitsIncrFullTest() throws ReaperException, UnknownHo .columnFamilies(Sets.newHashSet("table1", "table2", "table3")) .blacklistedTables(Sets.newHashSet("table1", "table2", "table3")) .incrementalRepair(true) + .subrangeIncrementalRepair(false) .repairThreadCount(4) .timeout(30) .build(UUIDs.timeBased()); @@ -469,6 +484,7 @@ public void identicalRepairUnitsIncrFullTest() throws ReaperException, UnknownHo .columnFamilies(Sets.newHashSet("table1", "table2", "table3")) .blacklistedTables(Sets.newHashSet("table1", "table2", "table3")) .incrementalRepair(false) + .subrangeIncrementalRepair(false) .repairThreadCount(4) .timeout(30); @@ -494,6 +510,7 @@ public void identicalRepairUnitsDiffTablesTest() throws ReaperException, Unknown .columnFamilies(Sets.newHashSet("table1", "table2", "table3")) .blacklistedTables(Sets.newHashSet("table1", "table2", "table3")) .incrementalRepair(false) + .subrangeIncrementalRepair(false) .repairThreadCount(4) .timeout(30) .build(UUIDs.timeBased()); @@ -504,6 +521,7 @@ public void identicalRepairUnitsDiffTablesTest() throws ReaperException, Unknown .columnFamilies(Sets.newHashSet("table1", "table2", "table4")) .blacklistedTables(Sets.newHashSet("table1", "table2", "table3")) .incrementalRepair(false) + .subrangeIncrementalRepair(false) .repairThreadCount(4) .timeout(30); @@ -514,6 +532,7 @@ public void identicalRepairUnitsDiffTablesTest() throws ReaperException, Unknown .keyspaceName("test") .blacklistedTables(Sets.newHashSet("table1", "table2", "table3")) .incrementalRepair(false) + .subrangeIncrementalRepair(false) .repairThreadCount(4) .timeout(30); @@ -542,6 +561,7 @@ public void identicalRepairUnitsDiffNodesTest() throws ReaperException, UnknownH .blacklistedTables(Sets.newHashSet("table1", "table2", "table3")) .nodes(Sets.newHashSet("node1")) .incrementalRepair(false) + .subrangeIncrementalRepair(false) .repairThreadCount(4) .timeout(30) .build(UUIDs.timeBased()); @@ -553,6 +573,7 @@ public void identicalRepairUnitsDiffNodesTest() throws ReaperException, UnknownH .blacklistedTables(Sets.newHashSet("table1", "table2", "table3")) .nodes(Sets.newHashSet("node2")) .incrementalRepair(false) + .subrangeIncrementalRepair(false) .repairThreadCount(4) .timeout(30); @@ -578,6 +599,7 @@ public void conflictingRepairUnitsSameDcsTest() throws ReaperException, UnknownH .datacenters(Sets.newHashSet("dc1")) .nodes(Sets.newHashSet("node1")) .incrementalRepair(false) + .subrangeIncrementalRepair(false) .repairThreadCount(4) .timeout(30) .build(UUIDs.timeBased()); @@ -589,6 +611,7 @@ public void conflictingRepairUnitsSameDcsTest() throws ReaperException, UnknownH .blacklistedTables(Sets.newHashSet("table1", "table2", "table3")) .nodes(Sets.newHashSet("node1")) .incrementalRepair(false) + .subrangeIncrementalRepair(false) .repairThreadCount(4) .timeout(30); @@ -621,6 +644,7 @@ public void identicalRepairUnitsDifferentDcsTest() .datacenters(Sets.newHashSet("dc1")) .nodes(Sets.newHashSet("node1")) .incrementalRepair(false) + .subrangeIncrementalRepair(false) .repairThreadCount(4) .timeout(30) .build(UUIDs.timeBased()); @@ -633,6 +657,7 @@ public void identicalRepairUnitsDifferentDcsTest() .datacenters(Sets.newHashSet("dc2")) .nodes(Sets.newHashSet("node1")) .incrementalRepair(false) + .subrangeIncrementalRepair(false) .repairThreadCount(4) .timeout(30); @@ -663,6 +688,7 @@ public void identicalRepairUnitsNonExistentNodesTest() .columnFamilies(Sets.newHashSet("table1", "table2", "table3")) .blacklistedTables(Sets.newHashSet("table1", "table2", "table3")) .incrementalRepair(false) + .subrangeIncrementalRepair(false) .repairThreadCount(4) .timeout(30) .build(UUIDs.timeBased()); @@ -673,6 +699,7 @@ public void identicalRepairUnitsNonExistentNodesTest() .columnFamilies(Sets.newHashSet("table1", "table2", "table3")) .blacklistedTables(Sets.newHashSet("table1", "table2", "table3")) .incrementalRepair(false) + .subrangeIncrementalRepair(false) .repairThreadCount(3) .timeout(30); @@ -701,6 +728,7 @@ public void identicalRepairUnitsFailGetDcsTest() throws ReaperException, Unknown .blacklistedTables(Sets.newHashSet("table1", "table2", "table3")) .nodes(Sets.newHashSet("node1")) .incrementalRepair(false) + .subrangeIncrementalRepair(false) .repairThreadCount(4) .timeout(30) .build(UUIDs.timeBased()); @@ -712,6 +740,7 @@ public void identicalRepairUnitsFailGetDcsTest() throws ReaperException, Unknown .blacklistedTables(Sets.newHashSet("table1", "table2", "table3")) .nodes(Sets.newHashSet("node3")) .incrementalRepair(false) + .subrangeIncrementalRepair(false) .repairThreadCount(4) .timeout(30); @@ -733,6 +762,7 @@ public void unknownTablesTest() throws ReaperException, UnknownHostException { .clusterName(cluster.getName()) .keyspaceName("test") .incrementalRepair(false) + .subrangeIncrementalRepair(false) .repairThreadCount(4) .timeout(30) .build(UUIDs.timeBased()); @@ -741,6 +771,7 @@ public void unknownTablesTest() throws ReaperException, UnknownHostException { .clusterName(cluster.getName()) .keyspaceName("test") .incrementalRepair(false) + .subrangeIncrementalRepair(false) .repairThreadCount(3) .timeout(30); @@ -764,6 +795,7 @@ public void missingLiveNodesTest() .columnFamilies(Sets.newHashSet("table1", "table2", "table3")) .blacklistedTables(Sets.newHashSet("table1", "table2", "table3")) .incrementalRepair(false) + .subrangeIncrementalRepair(false) .repairThreadCount(4) .timeout(30) .build(UUIDs.timeBased()); @@ -774,6 +806,7 @@ public void missingLiveNodesTest() .columnFamilies(Sets.newHashSet("table1", "table2", "table3")) .blacklistedTables(Sets.newHashSet("table1", "table2", "table3")) .incrementalRepair(false) + .subrangeIncrementalRepair(false) .repairThreadCount(3) .timeout(30); @@ -816,6 +849,7 @@ public void createRepairUnitIncrUnknownVersionTest() .columnFamilies(Sets.newHashSet("table1", "table2", "table3")) .blacklistedTables(Sets.newHashSet("table1", "table2", "table3")) .incrementalRepair(true) + .subrangeIncrementalRepair(false) .repairThreadCount(4) .timeout(30); @@ -845,6 +879,7 @@ public void unitScheduleConflictsTest() .columnFamilies(Sets.newHashSet("table1", "table2", "table3")) .blacklistedTables(Sets.newHashSet("table1", "table2", "table3")) .incrementalRepair(true) + .subrangeIncrementalRepair(false) .repairThreadCount(4) .timeout(30); diff --git a/src/server/src/test/java/io/cassandrareaper/service/SchedulingManagerTest.java b/src/server/src/test/java/io/cassandrareaper/service/SchedulingManagerTest.java index e9cea223a..2149b41db 100644 --- a/src/server/src/test/java/io/cassandrareaper/service/SchedulingManagerTest.java +++ b/src/server/src/test/java/io/cassandrareaper/service/SchedulingManagerTest.java @@ -195,6 +195,7 @@ public void manageIncRepairAbovePercentThresholdSchedule() throws ReaperExceptio RepairUnit repairUnit = RepairUnit.builder() .clusterName("test") .incrementalRepair(true) + .subrangeIncrementalRepair(true) .keyspaceName("test") .repairThreadCount(1) .timeout(30) @@ -277,6 +278,7 @@ public void manageIncRepairBelowPercentThresholdSchedule() throws ReaperExceptio RepairUnit repairUnit = RepairUnit.builder() .clusterName("test") .incrementalRepair(true) + .subrangeIncrementalRepair(true) .keyspaceName("test") .repairThreadCount(1) .timeout(30) @@ -354,6 +356,7 @@ public void managePausedRepairSchedule() throws ReaperException { RepairUnit repairUnit = RepairUnit.builder() .clusterName("test") .incrementalRepair(true) + .subrangeIncrementalRepair(true) .keyspaceName("test") .repairThreadCount(1) .timeout(30) diff --git a/src/server/src/test/java/io/cassandrareaper/service/SegmentRunnerTest.java b/src/server/src/test/java/io/cassandrareaper/service/SegmentRunnerTest.java index 20f5c280d..c4cdbff11 100644 --- a/src/server/src/test/java/io/cassandrareaper/service/SegmentRunnerTest.java +++ b/src/server/src/test/java/io/cassandrareaper/service/SegmentRunnerTest.java @@ -73,7 +73,6 @@ import static org.apache.cassandra.repair.RepairParallelism.PARALLEL; import static org.junit.Assert.assertEquals; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doThrow; @@ -107,6 +106,7 @@ public void timeoutTest() throws InterruptedException, ReaperException, Executio .keyspaceName("reaper") .columnFamilies(Sets.newHashSet("reaper")) .incrementalRepair(false) + .subrangeIncrementalRepair(false) .nodes(Sets.newHashSet("127.0.0.1")) .repairThreadCount(1) .timeout(segmentTimeout)); @@ -154,7 +154,7 @@ public void timeoutTest() throws InterruptedException, ReaperException, Executio } - when(jmx.triggerRepair(any(), any(), any(), anyBoolean(), any(), any(), any(), anyInt())) + when(jmx.triggerRepair(any(), any(), any(), any(), any(), any(), any(), anyInt())) .then( (invocation) -> { assertEquals( @@ -230,6 +230,7 @@ public void successTest() throws InterruptedException, ReaperException, .keyspaceName("reaper") .columnFamilies(Sets.newHashSet("reaper")) .incrementalRepair(false) + .subrangeIncrementalRepair(false) .nodes(Sets.newHashSet("127.0.0.1")) .repairThreadCount(1) .timeout(segmentTimeout)); @@ -276,7 +277,7 @@ public void successTest() throws InterruptedException, ReaperException, throw new AssertionError(ex); } - when(jmx.triggerRepair(any(), any(), any(), anyBoolean(), any(), any(), any(), anyInt())) + when(jmx.triggerRepair(any(), any(), any(), any(), any(), any(), any(), anyInt())) .then( (invocation) -> { assertEquals( @@ -382,6 +383,7 @@ public void failureTest() throws InterruptedException, ReaperException, Executio .keyspaceName("reaper") .columnFamilies(Sets.newHashSet("reaper")) .incrementalRepair(false) + .subrangeIncrementalRepair(false) .nodes(Sets.newHashSet("127.0.0.1")) .repairThreadCount(1) .timeout(segmentTimeout)); @@ -434,7 +436,7 @@ public void failureTest() throws InterruptedException, ReaperException, Executio } - when(jmx.triggerRepair(any(), any(), any(), anyBoolean(), any(), any(), any(), anyInt())) + when(jmx.triggerRepair(any(), any(), any(), any(), any(), any(), any(), anyInt())) .then( (invocation) -> { assertEquals( @@ -529,6 +531,7 @@ public void outOfOrderSuccessCass21Test() .keyspaceName("reaper") .columnFamilies(Sets.newHashSet("reaper")) .incrementalRepair(false) + .subrangeIncrementalRepair(false) .nodes(Sets.newHashSet("127.0.0.1")) .repairThreadCount(1) .timeout(30)); @@ -581,7 +584,7 @@ public void outOfOrderSuccessCass21Test() } - when(jmx.triggerRepair(any(), any(), any(), anyBoolean(), any(), any(), any(), anyInt())) + when(jmx.triggerRepair(any(), any(), any(), any(), any(), any(), any(), anyInt())) .then( invocation -> { assertEquals( @@ -671,6 +674,7 @@ public void outOfOrderSuccessCass22Test() .keyspaceName("reaper") .columnFamilies(Sets.newHashSet("reaper")) .incrementalRepair(false) + .subrangeIncrementalRepair(false) .nodes(Sets.newHashSet("127.0.0.1")) .repairThreadCount(1) .timeout(30)); @@ -723,7 +727,7 @@ public void outOfOrderSuccessCass22Test() } - when(jmx.triggerRepair(any(), any(), any(), anyBoolean(), any(), any(), any(), anyInt())) + when(jmx.triggerRepair(any(), any(), any(), any(), any(), any(), any(), anyInt())) .then( invocation -> { assertEquals( @@ -815,6 +819,7 @@ public void outOfOrderFailureCass21Test() .keyspaceName("reaper") .columnFamilies(Sets.newHashSet("reaper")) .incrementalRepair(false) + .subrangeIncrementalRepair(false) .nodes(Sets.newHashSet("127.0.0.1")) .repairThreadCount(1) .timeout(30)); @@ -867,7 +872,7 @@ public void outOfOrderFailureCass21Test() } - when(jmx.triggerRepair(any(), any(), any(), anyBoolean(), any(), any(), any(), anyInt())) + when(jmx.triggerRepair(any(), any(), any(), any(), any(), any(), any(), anyInt())) .then( invocation -> { assertEquals( @@ -960,6 +965,7 @@ public void outOfOrderFailureTestCass22() .keyspaceName("reaper") .columnFamilies(Sets.newHashSet("reaper")) .incrementalRepair(false) + .subrangeIncrementalRepair(false) .nodes(Sets.newHashSet("127.0.0.1")) .repairThreadCount(1) .timeout(30)); @@ -1012,7 +1018,7 @@ public void outOfOrderFailureTestCass22() } - when(jmx.triggerRepair(any(), any(), any(), anyBoolean(), any(), any(), any(), anyInt())) + when(jmx.triggerRepair(any(), any(), any(), any(), any(), any(), any(), anyInt())) .then( invocation -> { assertEquals( @@ -1136,6 +1142,7 @@ public void triggerFailureTest() throws InterruptedException, ReaperException, E .keyspaceName("reaper") .columnFamilies(Sets.newHashSet("reaper")) .incrementalRepair(false) + .subrangeIncrementalRepair(false) .nodes(Sets.newHashSet("127.0.0.1")) .repairThreadCount(1) .timeout(segmentTimeout)); @@ -1185,7 +1192,7 @@ public void triggerFailureTest() throws InterruptedException, ReaperException, E } - when(jmx.triggerRepair(any(), any(), any(), anyBoolean(), any(), any(), any(), anyInt())) + when(jmx.triggerRepair(any(), any(), any(), any(), any(), any(), any(), anyInt())) .thenThrow(new ReaperException("failure")); context.managementConnectionFactory = new JmxManagementConnectionFactory(context, new NoopCrypotograph()) { @@ -1230,6 +1237,7 @@ public void nothingToRepairTest() throws InterruptedException, ReaperException, .keyspaceName("reaper") .columnFamilies(Sets.newHashSet("reaper")) .incrementalRepair(false) + .subrangeIncrementalRepair(false) .nodes(Sets.newHashSet("127.0.0.1")) .repairThreadCount(1) .timeout(segmentTimeout)); @@ -1279,7 +1287,7 @@ public void nothingToRepairTest() throws InterruptedException, ReaperException, } - when(jmx.triggerRepair(any(), any(), any(), anyBoolean(), any(), any(), any(), anyInt())) + when(jmx.triggerRepair(any(), any(), any(), any(), any(), any(), any(), anyInt())) .thenReturn(0); context.managementConnectionFactory = new JmxManagementConnectionFactory(context, new NoopCrypotograph()) { @@ -1351,6 +1359,7 @@ public void clearSnapshotTest() throws InterruptedException, ReaperException, Ex .keyspaceName("reaper") .columnFamilies(Sets.newHashSet("reaper")) .incrementalRepair(false) + .subrangeIncrementalRepair(false) .nodes(Sets.newHashSet("127.0.0.1")) .repairThreadCount(1) .timeout(segmentTimeout)); @@ -1430,6 +1439,7 @@ public void clearSnapshotFailTest() throws InterruptedException, ReaperException .keyspaceName("reaper") .columnFamilies(Sets.newHashSet("reaper")) .incrementalRepair(false) + .subrangeIncrementalRepair(false) .nodes(Sets.newHashSet("127.0.0.1")) .repairThreadCount(1) .timeout(segmentTimeout)); diff --git a/src/server/src/test/resources/io.cassandrareaper.acceptance/integration_reaper_functionality.feature b/src/server/src/test/resources/io.cassandrareaper.acceptance/integration_reaper_functionality.feature index 85c370d83..f2fa15d74 100644 --- a/src/server/src/test/resources/io.cassandrareaper.acceptance/integration_reaper_functionality.feature +++ b/src/server/src/test/resources/io.cassandrareaper.acceptance/integration_reaper_functionality.feature @@ -309,6 +309,33 @@ Feature: Using Reaper Then reaper has no longer the last added cluster in storage ${cucumber.upgrade-versions} + @sidecar + @all_nodes_reachable + @cassandra_4_0_onwards + @focus + Scenario Outline: Create a cluster and a subrange incremental repair run and delete them + Given that reaper is running + And reaper has no cluster in storage + When an add-cluster request is made to reaper + Then reaper has the last added cluster in storage + And reaper has 0 repairs for the last added cluster + When a new subrange incremental repair is added for the last added cluster and keyspace "booya" + And deleting cluster called "test" fails + And the last added repair is activated + And we wait for at least 1 segments to be repaired + Then reaper has 1 started or done repairs for the last added cluster + Then reaper has 1 started or done repairs for the last added cluster + When the last added repair is stopped + When a new repair is added for the last added cluster and keyspace "booya" with force option + And the last added repair is activated + And we wait for at least 1 segments to be repaired + Then reaper has 2 repairs for cluster called "test" + When the last added repair is stopped + And all added repair runs are deleted for the last added cluster + And the last added cluster is deleted + Then reaper has no longer the last added cluster in storage + ${cucumber.upgrade-versions} + @sidecar @all_nodes_reachable @cassandra_2_1_onwards diff --git a/src/ui/app/jsx/repair-form.jsx b/src/ui/app/jsx/repair-form.jsx index 10d6b86cb..19841a9c0 100644 --- a/src/ui/app/jsx/repair-form.jsx +++ b/src/ui/app/jsx/repair-form.jsx @@ -61,7 +61,7 @@ const repairForm = CreateReactClass({ cause: "", startTime: this.props.formType === "schedule" ? moment().toDate() : null, intervalDays: "", - incrementalRepair: "false", + repairType: "full", repairThreadCount: 1, timeout: null, formCollapsed: true, @@ -203,14 +203,17 @@ const repairForm = CreateReactClass({ if (this.state.parallelism) repair.repairParallelism = this.state.parallelism; if (this.state.intensity) repair.intensity = this.state.intensity; if (this.state.cause) repair.cause = this.state.cause; - if (this.state.incrementalRepair) { - repair.incrementalRepair = this.state.incrementalRepair; - if (repair.incrementalRepair == "true") { + repair.incrementalRepair = "false"; + if (this.state.repairType) { + if (this.state.repairType === "incremental_subrange") { + repair.incrementalRepair = "true"; + repair.subrangeIncrementalRepair = "true"; + repair.repairParallelism = "PARALLEL"; + } + if (this.state.repairType === "incremental") { + repair.incrementalRepair = "true"; repair.repairParallelism = "PARALLEL"; } - } - else { - repair.incrementalRepair = "false"; } if (this.state.nodes) repair.nodes = this.state.nodes; if (this.state.datacenters) repair.datacenters = this.state.datacenters; @@ -712,14 +715,14 @@ const repairForm = CreateReactClass({

- +