Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

test OOM by slow read attack #618

Draft
wants to merge 7 commits into
base: master
Choose a base branch
from
Draft
Changes from 6 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
148 changes: 148 additions & 0 deletions t_stress/test_slow_read.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,148 @@
__author__ = "Tempesta Technologies, Inc."
__copyright__ = "Copyright (C) 2022 Tempesta Technologies, Inc."
__license__ = "GPL2"

import os
from pathlib import Path

from framework import tester
from helpers import tf_cfg


class TestH2SlowRead(tester.TempestaTest):
parallel = 10
clients = [
{
"id": f"deproxy-{i}",
"type": "deproxy_h2",
"addr": "${tempesta_ip}",
"port": "443",
"ssl": True,
"ssl_hostname": "tempesta-tech.com",
}
for i in range(parallel)
]

backends = [
{
"id": "nginx",
"type": "nginx",
"status_uri": "http://${server_ip}:8000/nginx_status",
"config": """
pid ${pid};
worker_processes auto;

events {
worker_connections 1024;
use epoll;
}

http {
keepalive_timeout ${server_keepalive_timeout};
keepalive_requests 10;
sendfile on;
tcp_nopush on;
tcp_nodelay on;

open_file_cache max=1000;
open_file_cache_valid 30s;
open_file_cache_min_uses 2;
open_file_cache_errors off;

# [ debug | info | notice | warn | error | crit | alert | emerg ]
# Fully disable log errors.
error_log /dev/null emerg;

# Disable access log altogether.
access_log off;

server {
listen ${server_ip}:8000;

location / {
root ${server_resources};
}
location /nginx_status {
stub_status on;
}
}
}
""",
}
]

tempesta = {
"config": """
cache 0;
keepalive_timeout 10;
listen 443 proto=h2;

tls_match_any_server_name;

srv_group default {
server ${server_ip}:8000;
}

vhost tempesta-tech.com {
tls_certificate ${tempesta_workdir}/tempesta.crt;
tls_certificate_key ${tempesta_workdir}/tempesta.key;
proxy_pass default;
}

frang_limits {
http_body_len 1000000;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should not use http_body_len here. This directive does not affect the slowread attack

Copy link
Contributor Author

@kingluo kingluo May 13, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No, as said, it's one of the workarounds of slow-read attacks. Please see my description in my comment below.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

you set http_body_len 10**6, but attacker creates 100 streams with a body of 10**5KB. What will Tempesta's behavior be in this case? This test will not answer on my question, so I think it check nothing.

Copy link
Contributor Author

@kingluo kingluo May 13, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

http_body_len = 1MB < BODY_SIZE = 1024 * 1024 * 100 = 100MB, then the slow read attack will be broken at 10MB per stream.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We plan to remove http_body_len directive for responses in 498 so I think you should not use it for these tests.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think we should rely on a future PR, let alone no PR schedule for tempesta-tech/tempesta#498. This test should serve as a placeholder, at least to describe the workaround, otherwise, there is no point in keeping it as it will definitely fail under the current implementation, thus blocking CI.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@kingluo in such cases please write comment like # TODO tempesta#498: remove the body length limit and add a comment to tempesta-tech/tempesta#498 to update the place.

Also if you see comment questions, especially with long discussions, please address them with comments in code. In this case please describe in a comment for the Tempesta configuration how does the workaround for the slow-read attack prevention actually works and reference the CVE

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

okay. I'll add the comment.

}
"""
}

def setUp(self):
super().setUp()

BODY_SIZE = 1024 * 1024 * 100 # 100MB
body = "x" * BODY_SIZE

fp = str(Path(tf_cfg.cfg.get("Server", "resources")) / "large.file")
with open(fp, "w") as f:
f.write(body)

self.addCleanup(lambda: os.remove(fp))

def test_window_update(self):
self.start_all_services()

for client in self.get_clients():
client.update_initial_settings(initial_window_size=1)
client.send_bytes(client.h2_connection.data_to_send())
self.assertTrue(client.wait_for_ack_settings())
for _ in range(100):
# send HEADERS frame with END_STREAM flag
client.make_request(
client.create_request(
method="GET", headers=[], authority="tempesta-tech.com", uri="/large.file"
),
end_stream=True,
)

for client in self.get_clients():
self.assertTrue(client.wait_for_connection_close())

def test_tcp(self):
self.start_all_services()

for client in self.get_clients():
client.update_initial_settings(initial_window_size=(1 << 31) - 10)
client.send_bytes(client.h2_connection.data_to_send())
self.assertTrue(client.wait_for_ack_settings())
client.readable = lambda: False
for _ in range(100):
client.make_request(
client.create_request(
method="GET", headers=[], authority="tempesta-tech.com", uri="/large.file"
),
end_stream=True,
)

for client in self.get_clients():
self.assertFalse(client.wait_for_response(10))
client.readable = lambda: True
self.assertTrue(client.wait_for_connection_close())