Skip to content
This repository has been archived by the owner on Mar 24, 2021. It is now read-only.

Fix tests for librdkafka 0.11.3+ #928

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
+5 −2
Open

Fix tests for librdkafka 0.11.3+ #928

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
72a12b3
Fix tests for librdkafka 0.11.3+
carsonip Mar 17, 2019
04d3c2e
Move constant to top of file
carsonip Mar 21, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions pykafka/rdkafka/simple_consumer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@

log = logging.getLogger(__name__)

LIBRDKAFKA_MAX_ALLOWED_QUEUED_MAX_MESSAGE_BYTES = 2097151 # see librdkafka commit cc43f4


class RdKafkaSimpleConsumer(SimpleConsumer):
"""A pykafka.SimpleConsumer with librdkafka-based fetchers
Expand Down Expand Up @@ -248,9 +250,10 @@ def _mk_rdkafka_config_lists(self):
# queued.max.messages.kbytes so for now we infer the implied
# maximum (which, with default settings, is ~2GB per partition):
"queued.min.messages": self._queued_max_messages,
"queued.max.messages.kbytes": str(
"queued.max.messages.kbytes": str(min(
self._queued_max_messages
* self._fetch_message_max_bytes // 1024),
* self._fetch_message_max_bytes // 1024,
LIBRDKAFKA_MAX_ALLOWED_QUEUED_MAX_MESSAGE_BYTES)),

"fetch.wait.max.ms": self._fetch_wait_max_ms,
"fetch.message.max.bytes": self._fetch_message_max_bytes,
Expand Down