diff --git a/itests/containers/data-poller/app/poller-kafka.py b/itests/containers/data-poller/app/poller-kafka.py index 9cc3c877..5f21c278 100644 --- a/itests/containers/data-poller/app/poller-kafka.py +++ b/itests/containers/data-poller/app/poller-kafka.py @@ -8,7 +8,7 @@ from confluent_kafka.admin import AdminClient, NewTopic KAFKA_BOOTSTRAP_HOST = os.getenv('KAFKA_BOOTSTRAP_HOST', 'kafka1') -KAFKA_BOOTSTRAP_PORT = os.getenv('KAFKA_BOOTSTRAP_PORT', '19092') +KAFKA_BOOTSTRAP_PORT = os.getenv('KAFKA_BOOTSTRAP_PORT', '9092') TOPIC_NAME = os.getenv('BIFROST_KAFKA_TOPIC', 'itests') EXPECTED_COUNT = int(os.getenv('EXPECTED_COUNT', '1')) WAIT_TIME = int(os.getenv('KAFKA_POLLER_WAIT_TIME', '90')) diff --git a/itests/contexts/kafka.env b/itests/contexts/kafka.env index 6df63ab7..9fe58a3c 100644 --- a/itests/contexts/kafka.env +++ b/itests/contexts/kafka.env @@ -2,7 +2,7 @@ TRANSPORT_SINK=kafka BIFROST_KAFKA_TOPIC=itests KAFKA_BOOTSTRAP_HOST=kafka1 -KAFKA_BOOTSTRAP_PORT=19092 +KAFKA_BOOTSTRAP_PORT=9092 # We don't use it but needs to be specified for docker compose file LOCALSTACK_PORT=1337 diff --git a/itests/docker-compose.yml b/itests/docker-compose.yml index f31736bb..a80caa1a 100644 --- a/itests/docker-compose.yml +++ b/itests/docker-compose.yml @@ -27,46 +27,26 @@ services: networks: - net - zoo1: - profiles: ["all", "kafka" ] - image: confluentinc/cp-zookeeper:7.3.2 - hostname: zoo1 - container_name: zoo1 - ports: - - "2181:2181" - environment: - ZOOKEEPER_CLIENT_PORT: 2181 - ZOOKEEPER_SERVER_ID: 1 - ZOOKEEPER_SERVERS: zoo1:2888:3888 - networks: - - net - kafka1: profiles: ["all", "kafka"] - image: confluentinc/cp-kafka:7.3.2 + image: apache/kafka:3.8.0 hostname: kafka1 container_name: kafka1 ports: - "9092:9092" - - "29092:29092" - - "19092:19092" - - "9999:9999" environment: - KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka1:19092,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092,DOCKER://host.docker.internal:29092 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT - KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL - KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181" - KAFKA_BROKER_ID: 1 - KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" + KAFKA_NODE_ID: 1 + KAFKA_PROCESS_ROLES: broker,controller + KAFKA_LISTENERS: PLAINTEXT://localhost:9092,CONTROLLER://localhost:9093 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092 + KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT + KAFKA_CONTROLLER_QUORUM_VOTERS: 1@localhost:9093 KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 - KAFKA_JMX_PORT: 9999 - KAFKA_JMX_HOSTNAME: ${DOCKER_HOST_IP:-127.0.0.1} - KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer - KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true" - depends_on: - - zoo1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_NUM_PARTITIONS: 3 networks: - net