Skip to content

Commit

Permalink
tests/node_ops: avoid interference betwen failure injections and ops
Browse files Browse the repository at this point in the history
They can already be separated with a lock, but it's not sufficient.
We need certain delay between them.
  • Loading branch information
bashtanov committed Jun 26, 2024
1 parent 4cdf78d commit 64db89e
Show file tree
Hide file tree
Showing 2 changed files with 7 additions and 2 deletions.
4 changes: 3 additions & 1 deletion tests/rptest/tests/random_node_operations_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import random
import re
import threading
import time
from rptest.clients.rpk import RpkTool
from rptest.services.admin import Admin
from rptest.tests.prealloc_nodes import PreallocNodesTest
Expand Down Expand Up @@ -463,7 +464,8 @@ def enable_write_caching_testing():
self.redpanda,
self.logger,
lock,
progress_timeout=120 if enable_failures else 60)
progress_timeout=120 if enable_failures else 60,
wait_after_decommission=60 enable_failures else 0)
for i, op in enumerate(
generate_random_workload(
available_nodes=self.active_node_idxs)):
Expand Down
5 changes: 4 additions & 1 deletion tests/rptest/utils/node_operations.py
Original file line number Diff line number Diff line change
Expand Up @@ -215,12 +215,14 @@ def __init__(self,
redpanda: RedpandaService,
logger,
lock: threading.Lock,
progress_timeout=60):
progress_timeout=60,
wait_after_decommission=0):
self.redpanda = redpanda
self.logger = logger
self.timeout = 360
self.lock = lock
self.progress_timeout = progress_timeout
self.wait_after_decommission = wait_after_decommission

def node_id(self, idx):
return self.redpanda.node_id(self.redpanda.get_node(idx),
Expand Down Expand Up @@ -257,6 +259,7 @@ def decommissioned():
node_removed(node_id),
timeout_sec=self.timeout,
backoff_sec=1)
time.sleep(wait_after_decommission)

def get_statuses(self, node_to_query=None):
admin = Admin(self.redpanda)
Expand Down

0 comments on commit 64db89e

Please sign in to comment.