Skip to content

Commit

Permalink
Merge pull request #700 from irq0/pr/s3tr-selective-lc-debug
Browse files Browse the repository at this point in the history
  • Loading branch information
Marcel Lauhoff authored Oct 12, 2023
2 parents eaae805 + d4c938a commit 69f0927
Show file tree
Hide file tree
Showing 3 changed files with 108 additions and 81 deletions.
136 changes: 69 additions & 67 deletions tools/s3tests/analyze.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,13 @@
Simple analysis tasks for s3tr JSON results
"""

import csv
import json
import logging
import pathlib
import sys

import click
import requests
import rich
from rich.console import Console
from rich.table import Table
Expand All @@ -23,100 +23,102 @@ def analyze():
"""Analyze s3tr JSON results"""


def get_upstream_list():
resp = requests.get(
"https://raw.githubusercontent.com/aquarist-labs/ceph"
"/s3gw/qa/rgw/store/sfs/tests/fixtures/s3-tests.txt"
)
resp.raise_for_status()
return resp.text


def get_known_good(file=None):
if file:
with open(file) as fp:
data = fp.read().split("\n")
else:
data = get_upstream_list().split("\n")

return frozenset(test for test in data if test and not test.startswith("#"))
def get_excuses(file):
result = {}
with open(file) as fp:
csvreader = csv.reader(fp, delimiter=";")
for row in csvreader:
result[row[0]] = {
"url": row[1],
"excuse": row[2],
}
return result


@analyze.command()
@click.option(
"--known-good-file",
@click.argument(
"file",
type=click.Path(
file_okay=True, dir_okay=False, allow_dash=False, path_type=pathlib.Path
),
required=True,
nargs=1,
)
@click.argument(
"file",
"excuses-file",
type=click.Path(
file_okay=True, dir_okay=False, allow_dash=False, path_type=pathlib.Path
),
required=True,
required=False,
nargs=1,
)
def new_failures(known_good_file, file):
def summary(file, excuses_file):
"""
Compare results to known good from latest main branch
"""
console = Console()
if excuses_file:
excuses = get_excuses(excuses_file)
else:
excuses = None

known_good = get_known_good(known_good_file)
with open(file) as fp:
results = json.load(fp)

results = {result["test"].split("::")[1]: result for result in results}

success = frozenset(
failures = frozenset(
(name for name, result in results.items() if result["test_return"] != "success")
)
successes = frozenset(
(name for name, result in results.items() if result["test_return"] == "success")
)

known_good_that_fail_now = known_good - success
table = Table(box=rich.box.SIMPLE, title="S3 Test Stats")
table.add_column("")
table.add_column("")
table.add_row("Failed tests", str(len(failures)))
table.add_row("Successful tests", str(len(successes)))
table.add_row("Total tests", str(len(results)))
if excuses:
table.add_row("Tests OK to fail", str(len(excuses)))

if not excuses:
sys.exit(0)

failures_that_must_not_be = failures - excuses.keys()
new_successes = excuses.keys() & successes
table.add_row("Failures, not excused", str(len(failures_that_must_not_be)))
table.add_row("Successes, excused", str(len(new_successes)))
console.print(table)

table = Table(box=rich.box.SIMPLE, caption="Known good tests that fail now")
table.add_column("Test Name")
table.add_column("Test Result")
table.add_column("Container Exit")
for test in known_good_that_fail_now:
table.add_row(
test, results[test]["test_return"], results[test]["container_return"]
if failures_that_must_not_be:
table = Table(box=rich.box.SIMPLE, title="Failures not in excuse file")
table.add_column("Test Name")
table.add_column("Test Result")
table.add_column("Container Exit")
for test in sorted(failures_that_must_not_be):
table.add_row(
test, results[test]["test_return"], results[test]["container_return"]
)
console.print(table)

if new_successes:
table = Table(
box=rich.box.SIMPLE, title="Tests in excuse file no longer failing"
)
console.print(table)
if len(known_good_that_fail_now) > 0:
table.add_column("Test Name")
table.add_column("URL")
table.add_column("Excuse")
for test in sorted(new_successes):
table.add_row(test, excuses[test]["url"], excuses[test]["excuse"])
console.print(table)
console.print("Please remove no longer failing tests from excuse file")

if len(failures_that_must_not_be) > 0 or len(new_successes) > 0:
console.print("💥")
sys.exit(23)


@analyze.command()
@click.option(
"--known-good-file",
type=click.Path(
file_okay=True, dir_okay=False, allow_dash=False, path_type=pathlib.Path
),
)
@click.argument(
"file",
type=click.Path(
file_okay=True, dir_okay=False, allow_dash=False, path_type=pathlib.Path
),
required=True,
nargs=1,
)
def new_successes(known_good_file, file):
"""
What to add to s3-tests.txt?
"""
known_good = get_known_good(known_good_file)
with open(file) as fp:
results = json.load(fp)
results = {result["test"].split("::")[1]: result for result in results}
success = frozenset(
(name for name, result in results.items() if result["test_return"] == "success")
)

succeeding_not_in_known_good = success - known_good
print("\n".join(succeeding_not_in_known_good))
else:
console.print("🥳")


def get_result(file, test_name):
Expand Down
31 changes: 24 additions & 7 deletions tools/s3tests/runner.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@
}


def make_radosgw_command(id, port):
def make_radosgw_command(id: str, port: int, lifecycle_debug: bool):
return [
"stdbuf",
"-oL",
Expand Down Expand Up @@ -88,7 +88,7 @@ def make_radosgw_command(id, port):
"--rgw_crypt_default_encryption_key",
"4YSmvJtBv0aZ7geVgAsdpRnLBEwWSWlMIGnRS8a9TSA=",
"--rgw-lc-debug-interval",
"10",
"10" if lifecycle_debug else "-1",
"--log-flush-on-exit",
"1",
"--log-to-stderr",
Expand All @@ -100,7 +100,7 @@ def make_radosgw_command(id, port):
"--debug-rgw",
"10",
"--rgw-frontends",
f'"beast port={port}"',
f'"beast port={port}, status bind=0.0.0.0 port={port + 10000}"',
"2>&1 1> /log",
]

Expand All @@ -115,16 +115,17 @@ class S3GW:
Admin OPs: create_user()
"""

def __init__(self, cri, image, container_run_args, name, port):
def __init__(self, cri, image, container_run_args, name, port, hints):
self.cri = cri
self.image = image
self.container_run_args = container_run_args
self.name = name
self.port = port
self.container = None
self.hints = hints

def start(self):
command = make_radosgw_command(self.name, self.port)
command = make_radosgw_command(self.name, self.port, "lifecycle" in self.hints)
kwargs = self.container_run_args | {
"image": self.image,
"name": f"s3gw_{self.name}",
Expand Down Expand Up @@ -165,6 +166,15 @@ def http_up(self):
except requests.exceptions.ConnectionError:
return False

def metrics(self):
try:
resp = requests.get(
f"http://{self.network_address()}:{self.port + 10000}/prometheus"
)
return resp.text
except requests.exceptions.ConnectionError:
return ""

def create_user(self, **kwargs):
rgwadmin = radosgw.connection.RadosGWAdminConnection(
host=self.network_address(),
Expand Down Expand Up @@ -298,7 +308,12 @@ def run_test(docker_api, image, container_run_args, s3_tests, name, port):
start_time_ns = time.perf_counter_ns()
cri = docker.DockerClient(base_url=docker_api)
container_name = name.split("::")[1]
container = S3GW(cri, image, container_run_args, container_name, port)
container_hints = set()
if "_lifecycle" in name:
container_hints.add("lifecycle")
container = S3GW(
cri, image, container_run_args, container_name, port, container_hints
)
container.start()

for retry in range(10):
Expand Down Expand Up @@ -366,6 +381,7 @@ def run_test(docker_api, image, container_run_args, s3_tests, name, port):
LOG.exception("unhandled exception during test %s. rethrowing.", name)
raise e

metrics = container.metrics()
container_ret = container.stop()
logs = container.logfile()
container.remove()
Expand All @@ -374,6 +390,7 @@ def run_test(docker_api, image, container_run_args, s3_tests, name, port):
"test_return": ret,
"container_return": container_ret,
"container_logs": logs,
"metrics": metrics,
"test_output": out,
"test_data": data_out,
"runtime_ns": time.perf_counter_ns() - start_time_ns,
Expand Down Expand Up @@ -509,7 +526,7 @@ def run(
)
LOG.info(
'Running radosgw with command "%s"',
" ".join(make_radosgw_command("PLACEHOLDER", "PLACEHOLDER")),
" ".join(make_radosgw_command("PLACEHOLDER", -1, True)),
)
try:
results = run_tests(
Expand Down
22 changes: 15 additions & 7 deletions tools/s3tests/to_sqlite.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ def make_full_results_database(results, pytest_markers, db_path):
"result": get_test_result(result),
"out": result["test_output"],
"log_container": result["container_logs"],
"metrics": result["metrics"],
}
db["results"].insert(row, pk="test")
for keyword in keywords:
Expand All @@ -73,7 +74,7 @@ def make_full_results_database(results, pytest_markers, db_path):
group by results.test
""",
)
db["results"].enable_fts(["out", "log_container"])
db["results"].enable_fts(["out", "log_container", "metrics"])
db["results"].create_index(["result"])
db["results_keywords"].create_index(["keyword"])
db["results_keywords"].create_index(["test"])
Expand All @@ -83,18 +84,18 @@ def make_comparison_database(results_by_versions, db_path):
db = sqlite_utils.Database(db_path)
db["versions"].insert_all(
[
{"name": version, "id": result["index"]}
for version, result in results_by_versions.items()
{"name": version, "id": index}
for (version, index), result in results_by_versions.items()
],
pk="id",
)
for results in results_by_versions.values():
for (_, index), results in results_by_versions.items():
for result in results:
db["results"].insert(
{
"test": result["test"].split("::")[1],
"result": get_test_result(result),
"version_id": result["index"],
"version_id": index,
},
pk="id",
foreign_keys=("version_id", "versions"),
Expand Down Expand Up @@ -187,6 +188,14 @@ def serve(pytest_ini, datasette_metadata, input):


@to_sqlite.command()
@click.option(
"--pytest-ini",
envvar="PYTEST_INI",
type=click.Path(
file_okay=True, dir_okay=False, allow_dash=False, path_type=pathlib.Path
),
required=True,
)
@click.argument(
"db_path",
type=click.Path(
Expand All @@ -210,8 +219,7 @@ def comparison(pytest_ini, db_path, input_files):
for i, file in enumerate(input_files):
with open(file) as fp:
results = json.load(fp)
results["index"] = i
results_by_versions[file.name] = results
results_by_versions[(file.name, i)] = results

make_comparison_database(results_by_versions, db_path)

Expand Down

0 comments on commit 69f0927

Please sign in to comment.