diff --git a/scripts/addr2line.py b/scripts/addr2line.py index 3f568f4d21..d48738f6cd 100755 --- a/scripts/addr2line.py +++ b/scripts/addr2line.py @@ -30,6 +30,7 @@ # special binary path/module indicating that the address is from the kernel KERNEL_MODULE = '' + class Addr2Line: # Matcher for a line that appears at the end a single decoded @@ -42,10 +43,10 @@ class Addr2Line: # 0x0: ?? at /v/llvm/llvm/src/compiler-rt/lib/asan/asan_fake_stack.h:133 # so that's why we liberally accept .* as the part after "at" below dummy_pattern = re.compile( - r"(.*0x0000000000000000: \?\? \?\?:0\n)" # addr2line pattern + r"(.*0x0000000000000000: \?\? \?\?:0\n)" # addr2line pattern r"|" r"(.*0x0: \?\? at .*\n)" # llvm-addr2line pattern - ) + ) def __init__(self, binary, concise=False, cmd_path="addr2line"): self._binary = binary @@ -59,9 +60,19 @@ def __init__(self, binary, concise=False, cmd_path="addr2line"): print('{}'.format(s)) options = f"-{'C' if not concise else ''}fpia" - self._input = subprocess.Popen([cmd_path, options, "-e", self._binary], stdin=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True) + self._input = subprocess.Popen( + [cmd_path, options, "-e", self._binary], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + universal_newlines=True, + ) if concise: - self._output = subprocess.Popen(["c++filt", "-p"], stdin=self._input.stdout, stdout=subprocess.PIPE, universal_newlines=True) + self._output = subprocess.Popen( + ["c++filt", "-p"], + stdin=self._input.stdout, + stdout=subprocess.PIPE, + universal_newlines=True, + ) else: self._output = self._input @@ -93,13 +104,14 @@ def __call__(self, address): self._input.stdin.flush() return self._read_resolved_address() + class KernelResolver: """A resolver for kernel addresses which tries to read from /proc/kallsyms.""" LAST_SYMBOL_MAX_SIZE = 1024 def __init__(self, kallsyms='/proc/kallsyms'): - syms : list[tuple[int, str]] = [] + syms: list[tuple[int, str]] = [] ksym_re = re.compile(r'(?P[0-9a-f]+) (?P.+) (?P\S+)') warnings_left = 10 @@ -116,8 +128,11 @@ def __init__(self, kallsyms='/proc/kallsyms'): for line in f: m = ksym_re.match(line) if not m: - if warnings_left > 0: # don't spam too much - print(f'WARNING: {kallsyms} regex match failure: {line.strip()}', file=sys.stdout) + if warnings_left > 0: # don't spam too much + print( + f'WARNING: {kallsyms} regex match failure: {line.strip()}', + file=sys.stdout, + ) warnings_left -= 1 else: syms.append((int(m.group('addr'), 16), m.group('name'))) @@ -140,10 +155,9 @@ def __init__(self, kallsyms='/proc/kallsyms'): return # split because bisect can't take a key func before 3.10 - self.sym_addrs : tuple[int] - self.sym_names : tuple[str] - self.sym_addrs, self.sym_names = zip(*syms) # type: ignore - + self.sym_addrs: tuple[int] + self.sym_names: tuple[str] + self.sym_addrs, self.sym_names = zip(*syms) # type: ignore def __call__(self, addrstr): if self.error: @@ -187,22 +201,30 @@ def __init__(self): token = fr"(?:{path}\+)?{addr}" full_addr_match = fr"(?:(?P{path})\s*\+\s*)?(?P{addr})" ignore_addr_match = fr"(?:(?P{path})\s*\+\s*)?(?:{addr})" - self.oneline_re = re.compile(fr"^((?:.*(?:(?:at|backtrace):?|:))?(?:\s+))?({token}(?:\s+{token})*)(?:\).*|\s*)$", flags=re.IGNORECASE) + self.oneline_re = re.compile( + fr"^((?:.*(?:(?:at|backtrace):?|:))?(?:\s+))?({token}(?:\s+{token})*)(?:\).*|\s*)$", + flags=re.IGNORECASE, + ) self.address_re = re.compile(full_addr_match, flags=re.IGNORECASE) - self.syslog_re = re.compile(fr"^(?:#\d+\s+)(?P{addr})(?:.*\s+)\({ignore_addr_match}\)\s*$", flags=re.IGNORECASE) + self.syslog_re = re.compile( + fr"^(?:#\d+\s+)(?P{addr})(?:.*\s+)\({ignore_addr_match}\)\s*$", + flags=re.IGNORECASE, + ) self.kernel_re = re.compile(fr'^.*kernel callstack: (?P(?:{addr}\s*)+)$') - self.asan_re = re.compile(fr"^(?:.*\s+)\({full_addr_match}\)(\s+\(BuildId: [0-9a-fA-F]+\))?$", flags=re.IGNORECASE) + self.asan_re = re.compile( + fr"^(?:.*\s+)\({full_addr_match}\)(\s+\(BuildId: [0-9a-fA-F]+\))?$", + flags=re.IGNORECASE, + ) self.asan_ignore_re = re.compile(f"^=.*$", flags=re.IGNORECASE) self.generic_re = re.compile(fr"^(?:.*\s+){full_addr_match}\s*$", flags=re.IGNORECASE) self.separator_re = re.compile(r'^\W*-+\W*$') - def split_addresses(self, addrstring: str, default_path=None): - addresses : list[dict[str, Any]] = [] + addresses: list[dict[str, Any]] = [] for obj in addrstring.split(): m = re.match(self.address_re, obj) assert m, f'addr did not match address regex: {obj}' - #print(f" >>> '{obj}': address {m.groups()}") + # print(f" >>> '{obj}': address {m.groups()}") addresses.append({'path': m.group(1) or default_path, 'addr': m.group(2)}) return addresses @@ -220,21 +242,21 @@ def get_prefix(s): return { 'type': self.Type.ADDRESS, 'prefix': 'kernel callstack: ', - 'addresses' : self.split_addresses(m.group('addrs'), KERNEL_MODULE) + 'addresses': self.split_addresses(m.group('addrs'), KERNEL_MODULE), } m = re.match(self.oneline_re, line) if m: - #print(f">>> '{line}': oneline {m.groups()}") + # print(f">>> '{line}': oneline {m.groups()}") return { 'type': self.Type.ADDRESS, 'prefix': get_prefix(m.group(1)), - 'addresses': self.split_addresses(m.group(2)) + 'addresses': self.split_addresses(m.group(2)), } m = re.match(self.syslog_re, line) if m: - #print(f">>> '{line}': syslog {m.groups()}") + # print(f">>> '{line}': syslog {m.groups()}") ret = {'type': self.Type.ADDRESS} ret['prefix'] = None ret['addresses'] = [{'path': m.group('path'), 'addr': m.group('addr')}] @@ -242,12 +264,12 @@ def get_prefix(s): m = re.match(self.asan_ignore_re, line) if m: - #print(f">>> '{line}': asan ignore") + # print(f">>> '{line}': asan ignore") return None m = re.match(self.asan_re, line) if m: - #print(f">>> '{line}': asan {m.groups()}") + # print(f">>> '{line}': asan {m.groups()}") ret = {'type': self.Type.ADDRESS} ret['prefix'] = None ret['addresses'] = [{'path': m.group('path'), 'addr': m.group('addr')}] @@ -255,7 +277,7 @@ def get_prefix(s): m = re.match(self.generic_re, line) if m: - #print(f">>> '{line}': generic {m.groups()}") + # print(f">>> '{line}': generic {m.groups()}") ret = {'type': self.Type.ADDRESS} ret['prefix'] = None ret['addresses'] = [{'path': m.group('path'), 'addr': m.group('addr')}] @@ -265,10 +287,19 @@ def get_prefix(s): if match: return {'type': self.Type.SEPARATOR} - #print(f">>> '{line}': None") + # print(f">>> '{line}': None") return None - def __init__(self, executable, kallsyms='/proc/kallsyms', before_lines=1, context_re='', verbose=False, concise=False, cmd_path='addr2line'): + def __init__( + self, + executable, + kallsyms='/proc/kallsyms', + before_lines=1, + context_re='', + verbose=False, + concise=False, + cmd_path='addr2line', + ): self._executable = executable self._kallsyms = kallsyms self._current_backtrace = [] @@ -285,7 +316,9 @@ def __init__(self, executable, kallsyms='/proc/kallsyms', before_lines=1, contex self._concise = concise self._cmd_path = cmd_path self._known_modules = {} - self._get_resolver_for_module(self._executable) # fail fast if there is something wrong with the exe resolver + self._get_resolver_for_module( + self._executable + ) # fail fast if there is something wrong with the exe resolver self.parser = self.BacktraceParser() def _get_resolver_for_module(self, module): @@ -346,8 +379,12 @@ def _print_current_backtrace(self): backtrace = "".join(map(str, self._current_backtrace)) if backtrace in self._known_backtraces: - print("[Backtrace #{}] Already seen, not resolving again.".format(self._known_backtraces[backtrace])) - print("") # To separate traces with an empty line + print( + "[Backtrace #{}] Already seen, not resolving again.".format( + self._known_backtraces[backtrace] + ) + ) + print("") # To separate traces with an empty line self._current_backtrace = [] return @@ -358,7 +395,7 @@ def _print_current_backtrace(self): for module, addr in self._current_backtrace: self._print_resolved_address(module, addr) - print("") # To separate traces with an empty line + print("") # To separate traces with an empty line self._current_backtrace = [] self._i += 1 @@ -371,9 +408,9 @@ def __call__(self, line): if self._before_lines > 0: self._before_lines_queue.append(line) elif self._before_lines < 0: - sys.stdout.write(line) # line already has a trailing newline + sys.stdout.write(line) # line already has a trailing newline else: - pass # when == 0 no non-backtrace lines are printed + pass # when == 0 no non-backtrace lines are printed elif res['type'] == self.BacktraceParser.Type.SEPARATOR: pass elif res['type'] == self.BacktraceParser.Type.ADDRESS: @@ -392,4 +429,3 @@ def __call__(self, line): else: print(f"Unknown '{line}': {res}") raise RuntimeError("Unknown result type {res}") - diff --git a/scripts/seastar-addr2line b/scripts/seastar-addr2line index ffbf448bb3..859fd25f0f 100755 --- a/scripts/seastar-addr2line +++ b/scripts/seastar-addr2line @@ -63,173 +63,327 @@ class TestStringMethods(unittest.TestCase): self.assertEqual(res, expected, f"failed to parse {line}") def test_separator(self): - data = [ - ('---', - {'type': BacktraceResolver.BacktraceParser.Type.SEPARATOR}) - ] + data = [('---', {'type': BacktraceResolver.BacktraceParser.Type.SEPARATOR})] self._test(data) def test_addresses_only(self): data = [ - ('0x12f34', - {'type': BacktraceResolver.BacktraceParser.Type.ADDRESS, - 'prefix': None, - 'addresses': [{'path': None, 'addr': '0x12f34'}]}), - ('0xa1234 0xb4567', - {'type': BacktraceResolver.BacktraceParser.Type.ADDRESS, - 'prefix': None, - 'addresses': [{'path': None, 'addr': '0xa1234'}, {'path': None, 'addr': '0xb4567'}]}), - (' 0xa1234 /my/path+0xb4567', - {'type': BacktraceResolver.BacktraceParser.Type.ADDRESS, - 'prefix': None, - 'addresses': [{'path': None, 'addr': '0xa1234'}, {'path': '/my/path', 'addr': '0xb4567'}]}), - ('/my/path+0x12f34', - {'type': BacktraceResolver.BacktraceParser.Type.ADDRESS, - 'prefix': None, - 'addresses': [{'path': '/my/path', 'addr': '0x12f34'}]}), - (' /my/path+0x12f34', - {'type': BacktraceResolver.BacktraceParser.Type.ADDRESS, - 'prefix': None, - 'addresses': [{'path': '/my/path', 'addr': '0x12f34'}]}) + ( + '0x12f34', + { + 'type': BacktraceResolver.BacktraceParser.Type.ADDRESS, + 'prefix': None, + 'addresses': [{'path': None, 'addr': '0x12f34'}], + }, + ), + ( + '0xa1234 0xb4567', + { + 'type': BacktraceResolver.BacktraceParser.Type.ADDRESS, + 'prefix': None, + 'addresses': [ + {'path': None, 'addr': '0xa1234'}, + {'path': None, 'addr': '0xb4567'}, + ], + }, + ), + ( + ' 0xa1234 /my/path+0xb4567', + { + 'type': BacktraceResolver.BacktraceParser.Type.ADDRESS, + 'prefix': None, + 'addresses': [ + {'path': None, 'addr': '0xa1234'}, + {'path': '/my/path', 'addr': '0xb4567'}, + ], + }, + ), + ( + '/my/path+0x12f34', + { + 'type': BacktraceResolver.BacktraceParser.Type.ADDRESS, + 'prefix': None, + 'addresses': [{'path': '/my/path', 'addr': '0x12f34'}], + }, + ), + ( + ' /my/path+0x12f34', + { + 'type': BacktraceResolver.BacktraceParser.Type.ADDRESS, + 'prefix': None, + 'addresses': [{'path': '/my/path', 'addr': '0x12f34'}], + }, + ), ] self._test(data) def test_without_prefix(self): data = [ - ('Some prefix 0x12f34', - {'type': BacktraceResolver.BacktraceParser.Type.ADDRESS, - 'prefix': None, - 'addresses': [{'path': None, 'addr': '0x12f34'}]}), - ('Some prefix /my/path+0x12f34', - {'type': BacktraceResolver.BacktraceParser.Type.ADDRESS, - 'prefix': None, - 'addresses': [{'path': '/my/path', 'addr': '0x12f34'}]}) + ( + 'Some prefix 0x12f34', + { + 'type': BacktraceResolver.BacktraceParser.Type.ADDRESS, + 'prefix': None, + 'addresses': [{'path': None, 'addr': '0x12f34'}], + }, + ), + ( + 'Some prefix /my/path+0x12f34', + { + 'type': BacktraceResolver.BacktraceParser.Type.ADDRESS, + 'prefix': None, + 'addresses': [{'path': '/my/path', 'addr': '0x12f34'}], + }, + ), ] self._test(data) def test_reactor_stall_reports(self): data = [ - ('Reactor stalled on shard 1. Backtrace: 0x12f34', - {'type': BacktraceResolver.BacktraceParser.Type.ADDRESS, - 'prefix': 'Reactor stalled on shard 1. Backtrace:', - 'addresses': [{'path': None, 'addr': '0x12f34'}]}), - ('Reactor stalled on shard 1. Backtrace: 0xa1234 0xb5678', - {'type': BacktraceResolver.BacktraceParser.Type.ADDRESS, - 'prefix': 'Reactor stalled on shard 1. Backtrace:', - 'addresses': [{'path': None, 'addr': '0xa1234'}, {'path': None, 'addr': '0xb5678'}]}), - ('Reactor stalled on shard 1. Backtrace: /my/path+0xabcd', - {'type': BacktraceResolver.BacktraceParser.Type.ADDRESS, - 'prefix': 'Reactor stalled on shard 1. Backtrace:', - 'addresses': [{'path': '/my/path', 'addr': '0xabcd'}]}), - ('Apr 28 11:42:58 ip-172-31-2-154.ec2.internal scylla[10612]: Reactor stalled for 260 ms on shard 20.', None), + ( + 'Reactor stalled on shard 1. Backtrace: 0x12f34', + { + 'type': BacktraceResolver.BacktraceParser.Type.ADDRESS, + 'prefix': 'Reactor stalled on shard 1. Backtrace:', + 'addresses': [{'path': None, 'addr': '0x12f34'}], + }, + ), + ( + 'Reactor stalled on shard 1. Backtrace: 0xa1234 0xb5678', + { + 'type': BacktraceResolver.BacktraceParser.Type.ADDRESS, + 'prefix': 'Reactor stalled on shard 1. Backtrace:', + 'addresses': [ + {'path': None, 'addr': '0xa1234'}, + {'path': None, 'addr': '0xb5678'}, + ], + }, + ), + ( + 'Reactor stalled on shard 1. Backtrace: /my/path+0xabcd', + { + 'type': BacktraceResolver.BacktraceParser.Type.ADDRESS, + 'prefix': 'Reactor stalled on shard 1. Backtrace:', + 'addresses': [{'path': '/my/path', 'addr': '0xabcd'}], + }, + ), + ( + 'Apr 28 11:42:58 ip-172-31-2-154.ec2.internal scylla[10612]: Reactor stalled for 260 ms on shard 20.', + None, + ), ('Apr 28 11:42:58 ip-172-31-2-154.ec2.internal scylla[10612]: Backtrace:', None), - ('Apr 28 11:42:58 ip-172-31-2-154.ec2.internal scylla[10612]: 0x0000000003163dc2', - {'type': BacktraceResolver.BacktraceParser.Type.ADDRESS, - 'prefix': 'Apr 28 11:42:58 ip-172-31-2-154.ec2.internal scylla[10612]:', - 'addresses': [{'path': None, 'addr': '0x0000000003163dc2'}]}), + ( + 'Apr 28 11:42:58 ip-172-31-2-154.ec2.internal scylla[10612]: 0x0000000003163dc2', + { + 'type': BacktraceResolver.BacktraceParser.Type.ADDRESS, + 'prefix': 'Apr 28 11:42:58 ip-172-31-2-154.ec2.internal scylla[10612]:', + 'addresses': [{'path': None, 'addr': '0x0000000003163dc2'}], + }, + ), ] self._test(data) def test_boost_failure(self): data = [ - ('Expected partition_end(), but got end of stream, at 0xa1234 0xb5678 /my/path+0xabcd', - {'type': BacktraceResolver.BacktraceParser.Type.ADDRESS, - 'prefix': 'Expected partition_end(), but got end of stream, at', - 'addresses': [{'path': None, 'addr': '0xa1234'}, {'path': None, 'addr': '0xb5678'}, {'path': '/my/path', 'addr': '0xabcd'}]})] + ( + 'Expected partition_end(), but got end of stream, at 0xa1234 0xb5678 /my/path+0xabcd', + { + 'type': BacktraceResolver.BacktraceParser.Type.ADDRESS, + 'prefix': 'Expected partition_end(), but got end of stream, at', + 'addresses': [ + {'path': None, 'addr': '0xa1234'}, + {'path': None, 'addr': '0xb5678'}, + {'path': '/my/path', 'addr': '0xabcd'}, + ], + }, + ) + ] self._test(data) def test_asan_failure(self): data = [ - ('==16118==ERROR: AddressSanitizer: heap-use-after-free on address 0x60700019c710 at pc 0x000014d24643 bp 0x7ffc51f72220 sp 0x7ffc51f72218', None), + ( + '==16118==ERROR: AddressSanitizer: heap-use-after-free on address 0x60700019c710 at pc 0x000014d24643 bp 0x7ffc51f72220 sp 0x7ffc51f72218', + None, + ), ('READ of size 8 at 0x60700019c710 thread T0', None), - ('#0 0x14d24642 (/jenkins/workspace/scylla-enterprise/dtest-debug/scylla/.ccm/scylla-repository/1a5173bd45d01697d98ba2a7645f5d86afb2d0be/scylla/libexec/scylla+0x14d24642)', - {'type': BacktraceResolver.BacktraceParser.Type.ADDRESS, - 'prefix': None, - 'addresses': [{'path': '/jenkins/workspace/scylla-enterprise/dtest-debug/scylla/.ccm/scylla-repository/1a5173bd45d01697d98ba2a7645f5d86afb2d0be/scylla/libexec/scylla', 'addr': '0x14d24642'}]}), - (' #1 0xd8d910f (/home/myhome/.dtest/dtest-84j9064d/test/node1/bin/scylla+0xd8d910f) (BuildId: 05a1d3d58d2b07e526decdad717e71a4590be2e0)', - {'type': BacktraceResolver.BacktraceParser.Type.ADDRESS, - 'prefix': None, - 'addresses': [{'path': '/home/myhome/.dtest/dtest-84j9064d/test/node1/bin/scylla', 'addr': '0xd8d910f'}]}), + ( + '#0 0x14d24642 (/jenkins/workspace/scylla-enterprise/dtest-debug/scylla/.ccm/scylla-repository/1a5173bd45d01697d98ba2a7645f5d86afb2d0be/scylla/libexec/scylla+0x14d24642)', + { + 'type': BacktraceResolver.BacktraceParser.Type.ADDRESS, + 'prefix': None, + 'addresses': [ + { + 'path': '/jenkins/workspace/scylla-enterprise/dtest-debug/scylla/.ccm/scylla-repository/1a5173bd45d01697d98ba2a7645f5d86afb2d0be/scylla/libexec/scylla', + 'addr': '0x14d24642', + } + ], + }, + ), + ( + ' #1 0xd8d910f (/home/myhome/.dtest/dtest-84j9064d/test/node1/bin/scylla+0xd8d910f) (BuildId: 05a1d3d58d2b07e526decdad717e71a4590be2e0)', + { + 'type': BacktraceResolver.BacktraceParser.Type.ADDRESS, + 'prefix': None, + 'addresses': [ + { + 'path': '/home/myhome/.dtest/dtest-84j9064d/test/node1/bin/scylla', + 'addr': '0xd8d910f', + } + ], + }, + ), ] self._test(data) - def test_thrown_exception(self): data = [ - ('seastar::internal::backtraced (throw_with_backtrace_exception_logging Backtrace: 0x42bc95 /lib64/libc.so.6+0x281e1 0x412cfd)', - {'type': BacktraceResolver.BacktraceParser.Type.ADDRESS, - 'prefix': 'seastar::internal::backtraced (throw_with_backtrace_exception_logging Backtrace:', - 'addresses': [{'path': None, 'addr': '0x42bc95'}, {'path': '/lib64/libc.so.6', 'addr': '0x281e1'}, {'path': None, 'addr': '0x412cfd'}]}), - ('seastar::nested_exception: seastar::internal::backtraced (inner Backtrace: 0x42bc95 /lib64/libc.so.6+0x281e1 0x412cfd) (while cleaning up after unknown_obj)', - {'type': BacktraceResolver.BacktraceParser.Type.ADDRESS, - 'prefix': 'seastar::nested_exception: seastar::internal::backtraced (inner Backtrace:', - 'addresses': [{'path': None, 'addr': '0x42bc95'}, {'path': '/lib64/libc.so.6', 'addr': '0x281e1'}, {'path': None, 'addr': '0x412cfd'}]}), - ('seastar::nested_exception: seastar::internal::backtraced (inner Backtrace: 0x42bc95 /lib64/libc.so.6+0x281e1 0x412cfd) ' - '(while cleaning up after seastar::internal::backtraced (outer Backtrace: 0x1234 /lib64/libc.so.6+0x5678 0xabcd))', - {'type': BacktraceResolver.BacktraceParser.Type.ADDRESS, - 'prefix': 'seastar::nested_exception: seastar::internal::backtraced (inner Backtrace: 0x42bc95 /lib64/libc.so.6+0x281e1 0x412cfd)' - ' (while cleaning up after seastar::internal::backtraced (outer Backtrace:', - 'addresses': [{'path': None, 'addr': '0x1234'}, {'path': '/lib64/libc.so.6', 'addr': '0x5678'}, {'path': None, 'addr': '0xabcd'}]}) - ] + ( + 'seastar::internal::backtraced (throw_with_backtrace_exception_logging Backtrace: 0x42bc95 /lib64/libc.so.6+0x281e1 0x412cfd)', + { + 'type': BacktraceResolver.BacktraceParser.Type.ADDRESS, + 'prefix': 'seastar::internal::backtraced (throw_with_backtrace_exception_logging Backtrace:', + 'addresses': [ + {'path': None, 'addr': '0x42bc95'}, + {'path': '/lib64/libc.so.6', 'addr': '0x281e1'}, + {'path': None, 'addr': '0x412cfd'}, + ], + }, + ), + ( + 'seastar::nested_exception: seastar::internal::backtraced (inner Backtrace: 0x42bc95 /lib64/libc.so.6+0x281e1 0x412cfd) (while cleaning up after unknown_obj)', + { + 'type': BacktraceResolver.BacktraceParser.Type.ADDRESS, + 'prefix': 'seastar::nested_exception: seastar::internal::backtraced (inner Backtrace:', + 'addresses': [ + {'path': None, 'addr': '0x42bc95'}, + {'path': '/lib64/libc.so.6', 'addr': '0x281e1'}, + {'path': None, 'addr': '0x412cfd'}, + ], + }, + ), + ( + 'seastar::nested_exception: seastar::internal::backtraced (inner Backtrace: 0x42bc95 /lib64/libc.so.6+0x281e1 0x412cfd) ' + '(while cleaning up after seastar::internal::backtraced (outer Backtrace: 0x1234 /lib64/libc.so.6+0x5678 0xabcd))', + { + 'type': BacktraceResolver.BacktraceParser.Type.ADDRESS, + 'prefix': 'seastar::nested_exception: seastar::internal::backtraced (inner Backtrace: 0x42bc95 /lib64/libc.so.6+0x281e1 0x412cfd)' + ' (while cleaning up after seastar::internal::backtraced (outer Backtrace:', + 'addresses': [ + {'path': None, 'addr': '0x1234'}, + {'path': '/lib64/libc.so.6', 'addr': '0x5678'}, + {'path': None, 'addr': '0xabcd'}, + ], + }, + ), + ] self._test(data) def test_assertion_failure(self): data = [ - ("2022-04-15T06:19:24+00:00 gemini-1tb-10h-master-db-node-c0c7fc43-4 ! INFO | " - "scylla: sstables/consumer.hh:610: future<> data_consumer::continuous_data_consumer" - ">::fast_forward_to" - "(size_t, size_t) [StateProcessor = sstables::index_consume_entry_context" - "]: Assertion `begin >= _stream_position.position' failed.", None), - ('2022-04-15T06:19:24+00:00 gemini-1tb-10h-master-db-node-c0c7fc43-4 ! INFO | Backtrace:', None), - ('2022-04-15T06:19:24+00:00 gemini-1tb-10h-master-db-node-c0c7fc43-4 ! INFO | 0x199d312', - {'type': BacktraceResolver.BacktraceParser.Type.ADDRESS, - 'prefix': None, - 'addresses': [{'path': None, 'addr': '0x199d312'}]}), - ('2022-04-15T06:19:24+00:00 gemini-1tb-10h-master-db-node-c0c7fc43-4 ! INFO | (throw_with_backtrace_exception_logging Backtrace: 0x42bc95 /lib64/libc.so.6+0x281e1 0x412cfd)', - {'type': BacktraceResolver.BacktraceParser.Type.ADDRESS, - 'prefix': '2022-04-15T06:19:24+00:00 gemini-1tb-10h-master-db-node-c0c7fc43-4 ! INFO | (throw_with_backtrace_exception_logging Backtrace:', - 'addresses': [{'path': None, 'addr': '0x42bc95'}, {'path': '/lib64/libc.so.6', 'addr': '0x281e1'}, {'path': None, 'addr': '0x412cfd'}]}), - ] + ( + "2022-04-15T06:19:24+00:00 gemini-1tb-10h-master-db-node-c0c7fc43-4 ! INFO | " + "scylla: sstables/consumer.hh:610: future<> data_consumer::continuous_data_consumer" + ">::fast_forward_to" + "(size_t, size_t) [StateProcessor = sstables::index_consume_entry_context" + "]: Assertion `begin >= _stream_position.position' failed.", + None, + ), + ( + '2022-04-15T06:19:24+00:00 gemini-1tb-10h-master-db-node-c0c7fc43-4 ! INFO | Backtrace:', + None, + ), + ( + '2022-04-15T06:19:24+00:00 gemini-1tb-10h-master-db-node-c0c7fc43-4 ! INFO | 0x199d312', + { + 'type': BacktraceResolver.BacktraceParser.Type.ADDRESS, + 'prefix': None, + 'addresses': [{'path': None, 'addr': '0x199d312'}], + }, + ), + ( + '2022-04-15T06:19:24+00:00 gemini-1tb-10h-master-db-node-c0c7fc43-4 ! INFO | (throw_with_backtrace_exception_logging Backtrace: 0x42bc95 /lib64/libc.so.6+0x281e1 0x412cfd)', + { + 'type': BacktraceResolver.BacktraceParser.Type.ADDRESS, + 'prefix': '2022-04-15T06:19:24+00:00 gemini-1tb-10h-master-db-node-c0c7fc43-4 ! INFO | (throw_with_backtrace_exception_logging Backtrace:', + 'addresses': [ + {'path': None, 'addr': '0x42bc95'}, + {'path': '/lib64/libc.so.6', 'addr': '0x281e1'}, + {'path': None, 'addr': '0x412cfd'}, + ], + }, + ), + ] self._test(data) def test_segfault_failure(self): data = [ ('[2022-04-19T23:09:28.311Z] Segmentation fault on shard 1.', None), ('[2022-04-19T23:09:28.311Z] Backtrace:', None), - ('[2022-04-19T23:09:28.311Z] 0x461bbb8', - {'type': BacktraceResolver.BacktraceParser.Type.ADDRESS, - 'prefix': None, - 'addresses': [{'path': None, 'addr': '0x461bbb8'}]}), - ('[2022-04-19T23:09:28.311Z] /lib64/libpthread.so.0+0x92a4', - {'type': BacktraceResolver.BacktraceParser.Type.ADDRESS, - 'prefix': None, - 'addresses': [{'path': '/lib64/libpthread.so.0', 'addr': '0x92a4'}]}), - - ('#0 0x19c01681 (/path/to/scylla+0xdeadbeef)', - {'type': BacktraceResolver.BacktraceParser.Type.ADDRESS, - 'prefix': None, - 'addresses': [{'path': '/path/to/scylla', 'addr': '0x19c01681'}]}), - ('#1 0x00007fd2dab4f950 abort (libc.so.6 + 0x26950)', - {'type': BacktraceResolver.BacktraceParser.Type.ADDRESS, - 'prefix': None, - 'addresses': [{'path': 'libc.so.6', 'addr': '0x00007fd2dab4f950'}]}), - ('#2 0x00000000015c4cd3 n/a (/path/to/scylla + 0x15c4cd3)', - {'type': BacktraceResolver.BacktraceParser.Type.ADDRESS, - 'prefix': None, - 'addresses': [{'path': '/path/to/scylla', 'addr': '0x00000000015c4cd3'}]}), - + ( + '[2022-04-19T23:09:28.311Z] 0x461bbb8', + { + 'type': BacktraceResolver.BacktraceParser.Type.ADDRESS, + 'prefix': None, + 'addresses': [{'path': None, 'addr': '0x461bbb8'}], + }, + ), + ( + '[2022-04-19T23:09:28.311Z] /lib64/libpthread.so.0+0x92a4', + { + 'type': BacktraceResolver.BacktraceParser.Type.ADDRESS, + 'prefix': None, + 'addresses': [{'path': '/lib64/libpthread.so.0', 'addr': '0x92a4'}], + }, + ), + ( + '#0 0x19c01681 (/path/to/scylla+0xdeadbeef)', + { + 'type': BacktraceResolver.BacktraceParser.Type.ADDRESS, + 'prefix': None, + 'addresses': [{'path': '/path/to/scylla', 'addr': '0x19c01681'}], + }, + ), + ( + '#1 0x00007fd2dab4f950 abort (libc.so.6 + 0x26950)', + { + 'type': BacktraceResolver.BacktraceParser.Type.ADDRESS, + 'prefix': None, + 'addresses': [{'path': 'libc.so.6', 'addr': '0x00007fd2dab4f950'}], + }, + ), + ( + '#2 0x00000000015c4cd3 n/a (/path/to/scylla + 0x15c4cd3)', + { + 'type': BacktraceResolver.BacktraceParser.Type.ADDRESS, + 'prefix': None, + 'addresses': [{'path': '/path/to/scylla', 'addr': '0x00000000015c4cd3'}], + }, + ), ('kernel callstack: ', None), - ('kernel callstack: 0xffffffffffffff80', - {'type': BacktraceResolver.BacktraceParser.Type.ADDRESS, - 'prefix': 'kernel callstack: ', - 'addresses': [{'path': '', 'addr': '0xffffffffffffff80'}]}), - ('kernel callstack: 0xffffffffffffff80 0xffffffffaf15ccca', - {'type': BacktraceResolver.BacktraceParser.Type.ADDRESS, - 'prefix': 'kernel callstack: ', - 'addresses': [{'path': '', 'addr': '0xffffffffffffff80'}, {'path': '', 'addr': '0xffffffffaf15ccca'}]}) + ( + 'kernel callstack: 0xffffffffffffff80', + { + 'type': BacktraceResolver.BacktraceParser.Type.ADDRESS, + 'prefix': 'kernel callstack: ', + 'addresses': [{'path': '', 'addr': '0xffffffffffffff80'}], + }, + ), + ( + 'kernel callstack: 0xffffffffffffff80 0xffffffffaf15ccca', + { + 'type': BacktraceResolver.BacktraceParser.Type.ADDRESS, + 'prefix': 'kernel callstack: ', + 'addresses': [ + {'path': '', 'addr': '0xffffffffffffff80'}, + {'path': '', 'addr': '0xffffffffaf15ccca'}, + ], + }, + ), ] self._test(data) + def main(): description = 'Massage and pass addresses to the real addr2line for symbol lookup.' epilog = ''' @@ -252,7 +406,8 @@ There are three operational modes: required=True, metavar='EXECUTABLE', dest='executable', - help='The executable where the addresses originate from') + help='The executable where the addresses originate from', + ) cmdline_parser.add_argument( '-f', @@ -261,7 +416,8 @@ There are three operational modes: required=False, metavar='FILE', dest='file', - help='The file containing the addresses') + help='The file containing the addresses', + ) cmdline_parser.add_argument( '-b', @@ -271,7 +427,8 @@ There are three operational modes: default=1, help='Non-backtrace lines to print before resolved backtraces for context.' ' Set to 0 to print only resolved backtraces.' - ' Set to -1 to print all non-backtrace lines. Default is 1.') + ' Set to -1 to print all non-backtrace lines. Default is 1.', + ) cmdline_parser.add_argument( '-m', @@ -280,7 +437,8 @@ There are three operational modes: metavar='MATCH', help='Only resolve backtraces whose non-backtrace lines match the regular-expression.' ' The amount of non-backtrace lines considered can be set with --before.' - ' By default no matching is performed.') + ' By default no matching is performed.', + ) cmdline_parser.add_argument( '-a', @@ -289,7 +447,8 @@ There are three operational modes: metavar='CMD_PATH', default='addr2line', help='The path or name of the addr2line command, which should behave as and ' - 'accept the same options as binutils addr2line or llvm-addr2line.') + 'accept the same options as binutils addr2line or llvm-addr2line.', + ) cmdline_parser.add_argument( '-v', @@ -297,28 +456,24 @@ There are three operational modes: action='store_true', default=False, help='Make resolved backtraces verbose, prepend to each line the module' - ' it originates from, as well as the address being resolved') + ' it originates from, as well as the address being resolved', + ) cmdline_parser.add_argument( - '-t', - '--test', - action='store_true', - default=False, - help='Self-test') + '-t', '--test', action='store_true', default=False, help='Self-test' + ) cmdline_parser.add_argument( - 'addresses', - type=str, - metavar='ADDRESS', - nargs='*', - help='Addresses to parse') + 'addresses', type=str, metavar='ADDRESS', nargs='*', help='Addresses to parse' + ) cmdline_parser.add_argument( '--kallsyms', type=str, metavar='KALLSYMS', default='/proc/kallsyms', - help='Alternative path for kallsyms file to resolve kernel addresses') + help='Alternative path for kallsyms file to resolve kernel addresses', + ) args = cmdline_parser.parse_args() @@ -336,12 +491,14 @@ There are three operational modes: else: lines = sys.stdin - with BacktraceResolver(executable=args.executable, - kallsyms=args.kallsyms, - before_lines=args.before, - context_re=args.match, - verbose=args.verbose, - cmd_path=args.addr2line) as resolve: + with BacktraceResolver( + executable=args.executable, + kallsyms=args.kallsyms, + before_lines=args.before, + context_re=args.match, + verbose=args.verbose, + cmd_path=args.addr2line, + ) as resolve: p = re.compile(r'\W+') for line in list(lines): resolve(line.strip() + '\n') @@ -350,11 +507,8 @@ There are three operational modes: if __name__ == '__main__': cmdline_parser = argparse.ArgumentParser(add_help=False) cmdline_parser.add_argument( - '-t', - '--test', - action='store_true', - default=False, - help='Self-test') + '-t', '--test', action='store_true', default=False, help='Self-test' + ) args, unrecognized = cmdline_parser.parse_known_args() if args.test: