-
Notifications
You must be signed in to change notification settings - Fork 5
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
test OOM by slow read attack #618
Draft
kingluo
wants to merge
7
commits into
master
Choose a base branch
from
jinhua/fix-1346-slow-read
base: master
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
+149
−0
Draft
Changes from 6 commits
Commits
Show all changes
7 commits
Select commit
Hold shift + click to select a range
ca3c3e9
test OOM by slow read attack
kingluo 597abd7
use nginx to send large file
kingluo ae554bf
catch ssl.SSLEOFError
kingluo 4d99b16
split into two test functions, because not much shared code
kingluo 7e98c74
catch ssl.SSLEOFError
kingluo fede44d
use DeproxyClientH2
kingluo d3f7ce4
Update test_slow_read.py
kingluo File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,148 @@ | ||
__author__ = "Tempesta Technologies, Inc." | ||
__copyright__ = "Copyright (C) 2022 Tempesta Technologies, Inc." | ||
__license__ = "GPL2" | ||
|
||
import os | ||
from pathlib import Path | ||
|
||
from framework import tester | ||
from helpers import tf_cfg | ||
|
||
|
||
class TestH2SlowRead(tester.TempestaTest): | ||
parallel = 10 | ||
clients = [ | ||
{ | ||
"id": f"deproxy-{i}", | ||
"type": "deproxy_h2", | ||
"addr": "${tempesta_ip}", | ||
"port": "443", | ||
"ssl": True, | ||
"ssl_hostname": "tempesta-tech.com", | ||
} | ||
for i in range(parallel) | ||
] | ||
|
||
backends = [ | ||
{ | ||
"id": "nginx", | ||
"type": "nginx", | ||
"status_uri": "http://${server_ip}:8000/nginx_status", | ||
"config": """ | ||
pid ${pid}; | ||
worker_processes auto; | ||
|
||
events { | ||
worker_connections 1024; | ||
use epoll; | ||
} | ||
|
||
http { | ||
keepalive_timeout ${server_keepalive_timeout}; | ||
keepalive_requests 10; | ||
sendfile on; | ||
tcp_nopush on; | ||
tcp_nodelay on; | ||
|
||
open_file_cache max=1000; | ||
open_file_cache_valid 30s; | ||
open_file_cache_min_uses 2; | ||
open_file_cache_errors off; | ||
|
||
# [ debug | info | notice | warn | error | crit | alert | emerg ] | ||
# Fully disable log errors. | ||
error_log /dev/null emerg; | ||
|
||
# Disable access log altogether. | ||
access_log off; | ||
|
||
server { | ||
listen ${server_ip}:8000; | ||
|
||
location / { | ||
root ${server_resources}; | ||
} | ||
location /nginx_status { | ||
stub_status on; | ||
} | ||
} | ||
} | ||
""", | ||
} | ||
] | ||
|
||
tempesta = { | ||
"config": """ | ||
cache 0; | ||
keepalive_timeout 10; | ||
listen 443 proto=h2; | ||
|
||
tls_match_any_server_name; | ||
|
||
srv_group default { | ||
server ${server_ip}:8000; | ||
} | ||
|
||
vhost tempesta-tech.com { | ||
tls_certificate ${tempesta_workdir}/tempesta.crt; | ||
tls_certificate_key ${tempesta_workdir}/tempesta.key; | ||
proxy_pass default; | ||
} | ||
|
||
frang_limits { | ||
http_body_len 1000000; | ||
} | ||
""" | ||
} | ||
|
||
def setUp(self): | ||
super().setUp() | ||
|
||
BODY_SIZE = 1024 * 1024 * 100 # 100MB | ||
body = "x" * BODY_SIZE | ||
|
||
fp = str(Path(tf_cfg.cfg.get("Server", "resources")) / "large.file") | ||
with open(fp, "w") as f: | ||
f.write(body) | ||
|
||
self.addCleanup(lambda: os.remove(fp)) | ||
|
||
def test_window_update(self): | ||
self.start_all_services() | ||
|
||
for client in self.get_clients(): | ||
client.update_initial_settings(initial_window_size=1) | ||
client.send_bytes(client.h2_connection.data_to_send()) | ||
self.assertTrue(client.wait_for_ack_settings()) | ||
for _ in range(100): | ||
# send HEADERS frame with END_STREAM flag | ||
client.make_request( | ||
client.create_request( | ||
method="GET", headers=[], authority="tempesta-tech.com", uri="/large.file" | ||
), | ||
end_stream=True, | ||
) | ||
|
||
for client in self.get_clients(): | ||
self.assertTrue(client.wait_for_connection_close()) | ||
|
||
def test_tcp(self): | ||
self.start_all_services() | ||
|
||
for client in self.get_clients(): | ||
client.update_initial_settings(initial_window_size=(1 << 31) - 10) | ||
client.send_bytes(client.h2_connection.data_to_send()) | ||
self.assertTrue(client.wait_for_ack_settings()) | ||
client.readable = lambda: False | ||
for _ in range(100): | ||
client.make_request( | ||
client.create_request( | ||
method="GET", headers=[], authority="tempesta-tech.com", uri="/large.file" | ||
), | ||
end_stream=True, | ||
) | ||
|
||
for client in self.get_clients(): | ||
self.assertFalse(client.wait_for_response(10)) | ||
client.readable = lambda: True | ||
self.assertTrue(client.wait_for_connection_close()) |
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
We should not use
http_body_len
here. This directive does not affect the slowread attackThere was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
No, as said, it's one of the workarounds of slow-read attacks. Please see my description in my comment below.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
you set
http_body_len 10**6
, but attacker creates 100 streams with a body of 10**5KB. What will Tempesta's behavior be in this case? This test will not answer on my question, so I think it check nothing.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
http_body_len = 1MB < BODY_SIZE = 1024 * 1024 * 100 = 100MB
, then the slow read attack will be broken at 10MB per stream.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
We plan to remove
http_body_len
directive for responses in 498 so I think you should not use it for these tests.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I don't think we should rely on a future PR, let alone no PR schedule for tempesta-tech/tempesta#498. This test should serve as a placeholder, at least to describe the workaround, otherwise, there is no point in keeping it as it will definitely fail under the current implementation, thus blocking CI.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@kingluo in such cases please write comment like
# TODO tempesta#498: remove the body length limit
and add a comment to tempesta-tech/tempesta#498 to update the place.Also if you see comment questions, especially with long discussions, please address them with comments in code. In this case please describe in a comment for the Tempesta configuration how does the workaround for the slow-read attack prevention actually works and reference the CVE
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
okay. I'll add the comment.