Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Convert wurstmeister images to bitnami images #168

Merged
merged 17 commits into from
Dec 11, 2023
Merged

Convert wurstmeister images to bitnami images #168

Show file tree
Hide file tree
Changes from 13 commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
83e945e
Convert wurstmeister images to bitnami images
ssingudasu Nov 28, 2023
5b0273f
Modified ci workflows
ssingudasu Nov 28, 2023
ee76ecd
Minor changes to README.md
ssingudasu Nov 28, 2023
1e9a318
Minor changes to environment variable
ssingudasu Nov 28, 2023
b4f2859
ci test271 enable scram
ssingudasu Nov 28, 2023
aa6d67e
ci changing to SASL_PLAINTEXT
ssingudasu Nov 28, 2023
4187b8e
modifying advertized listeners
ssingudasu Nov 28, 2023
c3c760c
reverting test271
ssingudasu Nov 28, 2023
78f50d5
modifying test271 authroizer class name
ssingudasu Nov 28, 2023
c7feb77
removing listeners
ssingudasu Nov 28, 2023
b32d6fc
Bumping the Kafka version to 2.7.0 for SCRAM
ssingudasu Nov 28, 2023
126227c
Minor README.md fixtures
ssingudasu Nov 28, 2023
46bf022
Minor README.md fixtures
ssingudasu Nov 28, 2023
1460c00
Fixtures for kafka versions, KAFKA_IMAGE_TAG
ssingudasu Nov 28, 2023
b2f065d
ci fixture for kafka version 0.10.2.1
ssingudasu Nov 28, 2023
b3308d2
Minor README changes
ssingudasu Nov 28, 2023
40168d4
Minor README changes
ssingudasu Nov 29, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
180 changes: 98 additions & 82 deletions .github/workflows/ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,78 +50,86 @@ jobs:

services:
zookeeper:
image: wurstmeister/zookeeper
image: bitnami/zookeeper:latest
ports:
- "2181:2181"
env:
ALLOW_ANONYMOUS_LOGIN: yes

kafka1:
image: wurstmeister/kafka:2.11-0.10.2.2
ssingudasu marked this conversation as resolved.
Show resolved Hide resolved
image: bitnami/kafka:0.10.2.1
ports:
- "9092:9092"
env:
KAFKA_BROKER_ID: 1
KAFKA_BROKER_RACK: zone1
KAFKA_ADVERTISED_HOST_NAME: kafka1
KAFKA_ADVERTISED_PORT: 9092
KAFKA_CFG_BROKER_ID: 1
KAFKA_CFG_BROKER_RACK: zone1
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
ALLOW_PLAINTEXT_LISTENER: yes
KAFKA_CFG_LISTENERS: PLAINTEXT://:9092
KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://kafka1:9092

kafka2:
image: wurstmeister/kafka:2.11-0.10.2.2
image: bitnami/kafka:0.10.2.1
ports:
- "9093:9092"
env:
KAFKA_BROKER_ID: 2
KAFKA_BROKER_RACK: zone1
KAFKA_ADVERTISED_HOST_NAME: kafka2
KAFKA_ADVERTISED_PORT: 9092
KAFKA_CFG_BROKER_ID: 2
KAFKA_CFG_BROKER_RACK: zone1
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
ALLOW_PLAINTEXT_LISTENER: yes
KAFKA_CFG_LISTENERS: PLAINTEXT://:9092
KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://kafka2:9092

kafka3:
image: wurstmeister/kafka:2.11-0.10.2.2
image: bitnami/kafka:0.10.2.1
ports:
- "9094:9092"
env:
KAFKA_BROKER_ID: 3
KAFKA_BROKER_RACK: zone2
KAFKA_ADVERTISED_HOST_NAME: kafka3
KAFKA_ADVERTISED_PORT: 9092
KAFKA_CFG_BROKER_ID: 3
KAFKA_CFG_BROKER_RACK: zone2
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
ALLOW_PLAINTEXT_LISTENER: yes
KAFKA_CFG_LISTENERS: PLAINTEXT://:9092
KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://kafka3:9092

kafka4:
image: wurstmeister/kafka:2.11-0.10.2.2
image: bitnami/kafka:0.10.2.1
ports:
- "9095:9092"
env:
KAFKA_BROKER_ID: 4
KAFKA_BROKER_RACK: zone2
KAFKA_ADVERTISED_HOST_NAME: kafka4
KAFKA_ADVERTISED_PORT: 9092
env:
KAFKA_CFG_BROKER_ID: 4
KAFKA_CFG_BROKER_RACK: zone2
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
ALLOW_PLAINTEXT_LISTENER: yes
KAFKA_CFG_LISTENERS: PLAINTEXT://:9092
KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://kafka4:9092

kafka5:
image: wurstmeister/kafka:2.11-0.10.2.2
image: bitnami/kafka:0.10.2.1
ports:
- "9096:9092"
env:
KAFKA_BROKER_ID: 5
KAFKA_BROKER_RACK: zone3
KAFKA_ADVERTISED_HOST_NAME: kafka5
KAFKA_ADVERTISED_PORT: 9092
env:
KAFKA_CFG_BROKER_ID: 5
KAFKA_CFG_BROKER_RACK: zone3
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
ALLOW_PLAINTEXT_LISTENER: yes
KAFKA_CFG_LISTENERS: PLAINTEXT://:9092
KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://kafka5:9092

kafka6:
image: wurstmeister/kafka:2.11-0.10.2.2
image: bitnami/kafka:0.10.2.1
ports:
- "9097:9092"
env:
KAFKA_BROKER_ID: 6
KAFKA_BROKER_RACK: zone3
KAFKA_ADVERTISED_HOST_NAME: kafka6
KAFKA_ADVERTISED_PORT: 9092
env:
KAFKA_CFG_BROKER_ID: 6
KAFKA_CFG_BROKER_RACK: zone3
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
ALLOW_PLAINTEXT_LISTENER: yes
KAFKA_CFG_LISTENERS: PLAINTEXT://:9092
KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://kafka6:9092


test271:
test270:
runs-on: ubuntu-latest
container:
image: cimg/go:1.19
Expand All @@ -147,91 +155,99 @@ jobs:

services:
zookeeper:
image: wurstmeister/zookeeper
image: bitnami/zookeeper:latest
ports:
- "2181:2181"
env:
ALLOW_ANONYMOUS_LOGIN: yes

kafka1:
image: wurstmeister/kafka:2.13-2.7.1
ssingudasu marked this conversation as resolved.
Show resolved Hide resolved
image: bitnami/kafka:2.7.0
ports:
- "9092:9092"
env:
KAFKA_BROKER_ID: 1
KAFKA_BROKER_RACK: zone1
KAFKA_ADVERTISED_HOST_NAME: kafka1
KAFKA_ADVERTISED_PORT: 9092
KAFKA_CFG_BROKER_ID: 1
KAFKA_CFG_BROKER_RACK: zone1
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer
KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true
ALLOW_PLAINTEXT_LISTENER: yes
KAFKA_CFG_ADVERTISED_HOST_NAME: kafka1
KAFKA_CFG_ADVERTISED_PORT: 9092
KAFKA_CFG_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
KAFKA_CFG_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true

kafka2:
image: wurstmeister/kafka:2.13-2.7.1
image: bitnami/kafka:2.7.0
ports:
- "9093:9092"
env:
KAFKA_BROKER_ID: 2
KAFKA_BROKER_RACK: zone1
KAFKA_ADVERTISED_HOST_NAME: kafka2
KAFKA_ADVERTISED_PORT: 9092
KAFKA_CFG_BROKER_ID: 2
KAFKA_CFG_BROKER_RACK: zone1
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer
KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true
ALLOW_PLAINTEXT_LISTENER: yes
KAFKA_CFG_ADVERTISED_HOST_NAME: kafka2
KAFKA_CFG_ADVERTISED_PORT: 9092
KAFKA_CFG_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
KAFKA_CFG_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true

kafka3:
image: wurstmeister/kafka:2.13-2.7.1
image: bitnami/kafka:2.7.0
ports:
- "9094:9092"
env:
KAFKA_BROKER_ID: 3
KAFKA_BROKER_RACK: zone2
KAFKA_ADVERTISED_HOST_NAME: kafka3
KAFKA_ADVERTISED_PORT: 9092
KAFKA_CFG_BROKER_ID: 3
KAFKA_CFG_BROKER_RACK: zone2
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer
KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true
ALLOW_PLAINTEXT_LISTENER: yes
KAFKA_CFG_ADVERTISED_HOST_NAME: kafka3
KAFKA_CFG_ADVERTISED_PORT: 9092
KAFKA_CFG_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
KAFKA_CFG_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true

kafka4:
image: wurstmeister/kafka:2.13-2.7.1
image: bitnami/kafka:2.7.0
ports:
- "9095:9092"
env:
KAFKA_BROKER_ID: 4
KAFKA_BROKER_RACK: zone2
KAFKA_ADVERTISED_HOST_NAME: kafka4
KAFKA_ADVERTISED_PORT: 9092
env:
KAFKA_CFG_BROKER_ID: 4
KAFKA_CFG_BROKER_RACK: zone2
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer
KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true
ALLOW_PLAINTEXT_LISTENER: yes
KAFKA_CFG_ADVERTISED_HOST_NAME: kafka4
KAFKA_CFG_ADVERTISED_PORT: 9092
KAFKA_CFG_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
KAFKA_CFG_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true

kafka5:
image: wurstmeister/kafka:2.13-2.7.1
image: bitnami/kafka:2.7.0
ports:
- "9096:9092"
env:
KAFKA_BROKER_ID: 5
KAFKA_BROKER_RACK: zone3
KAFKA_ADVERTISED_HOST_NAME: kafka5
KAFKA_ADVERTISED_PORT: 9092
env:
KAFKA_CFG_BROKER_ID: 5
KAFKA_CFG_BROKER_RACK: zone3
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer
KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true
ALLOW_PLAINTEXT_LISTENER: yes
KAFKA_CFG_ADVERTISED_HOST_NAME: kafka5
KAFKA_CFG_ADVERTISED_PORT: 9092
KAFKA_CFG_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
KAFKA_CFG_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true

kafka6:
image: wurstmeister/kafka:2.13-2.7.1
image: bitnami/kafka:2.7.0
ports:
- "9097:9092"
env:
KAFKA_BROKER_ID: 6
KAFKA_BROKER_RACK: zone3
KAFKA_ADVERTISED_HOST_NAME: kafka6
KAFKA_ADVERTISED_PORT: 9092
env:
KAFKA_CFG_BROKER_ID: 6
KAFKA_CFG_BROKER_RACK: zone3
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer
KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true
ALLOW_PLAINTEXT_LISTENER: yes
KAFKA_CFG_ADVERTISED_HOST_NAME: kafka6
KAFKA_CFG_ADVERTISED_PORT: 9092
KAFKA_CFG_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
KAFKA_CFG_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true

snyk:
runs-on: ubuntu-latest
needs: [test010, test271]
needs: [test010, test270]
steps:
- uses: actions/checkout@v3
- name: Run Snyk to check for vulnerabilities
Expand Down
7 changes: 4 additions & 3 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -552,11 +552,12 @@ Then, run:
make test
```

You can change the Kafka version of the local cluster by setting the
`KAFKA_IMAGE_TAG` environment variable when running `docker-compose up -d`. See the
[`wurstmeister/kafka` dockerhub page](https://hub.docker.com/r/wurstmeister/kafka/tags) for more
You can change the Kafka version in docker-compose.yml and docker-compose-auth.yml files before running `docker-compose up -d`. See the
[`bitnami/kafka` dockerhub page](https://hub.docker.com/r/bitnami/kafka/tags) for more
details on the available versions.

For SCRAM Config API calls, All `bitnami/kafka` images in topicctl should start using kafka:2.7.0+ version. SCRAM Config API calls have been implemented since `Kafka:2.7.0`. See the [`KIP-554` page](Add Broker-side SCRAM Config API) for more details.

#### Run against local cluster

To run the `get`, `repl`, and `tail` subcommands against the local cluster,
Expand Down
65 changes: 38 additions & 27 deletions docker-compose-auth.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,46 +7,57 @@
# 4. 9095: SASL over SSL
#
# See examples/auth for the associated cluster configs and certs.
version: '2'
version: '3'

services:
zookeeper:
image: "wurstmeister/zookeeper:latest"
container_name: zookeeper
hostname: zookeeper
image: bitnami/zookeeper:latest
ports:
- "2181:2181"
environment:
ALLOW_ANONYMOUS_LOGIN: yes

kafka:
image: wurstmeister/kafka:2.13-2.7.1
container_name: kafka
hostname: kafka
image: bitnami/kafka:2.7.0
ssingudasu marked this conversation as resolved.
Show resolved Hide resolved
depends_on:
- zookeeper
restart: on-failure:3
links:
- zookeeper
ports:
- 9092:9092
- 9093:9093
- 9094:9094
- 9095:9095
environment:
KAFKA_BROKER_ID: 1
KAFKA_ADVERTISED_HOST_NAME: localhost
KAFKA_ADVERTISED_PORT: 9092
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_MESSAGE_MAX_BYTES: 200000000
KAFKA_LISTENERS: "PLAINTEXT://:9092,SSL://:9093,SASL_PLAINTEXT://:9094,SASL_SSL://:9095"
KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://localhost:9092,SSL://localhost:9093,SASL_PLAINTEXT://localhost:9094,SASL_SSL://localhost:9095"
KAFKA_SASL_ENABLED_MECHANISMS: "PLAIN,SCRAM-SHA-256,SCRAM-SHA-512"
KAFKA_AUTHORIZER_CLASS_NAME: 'kafka.security.auth.SimpleAclAuthorizer'
KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: 'true'
KAFKA_SSL_KEYSTORE_LOCATION: /certs/kafka.keystore.jks
KAFKA_SSL_KEYSTORE_PASSWORD: test123
KAFKA_SSL_KEY_PASSWORD: test123
KAFKA_SSL_TRUSTSTORE_LOCATION: /certs/kafka.truststore.jks
KAFKA_SSL_TRUSTSTORE_PASSWORD: test123
KAFKA_SSL_CLIENT_AUTH: none
KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: ""
KAFKA_OPTS: "-Djava.security.auth.login.config=/opt/kafka/config/kafka_server_jaas.conf"
CUSTOM_INIT_SCRIPT: |-
echo -e 'KafkaServer {\norg.apache.kafka.common.security.scram.ScramLoginModule required\n username="adminscram"\n password="admin-secret";\n org.apache.kafka.common.security.plain.PlainLoginModule required\n username="adminplain"\n password="admin-secret"\n user_adminplain="admin-secret";\n };' > /opt/kafka/config/kafka_server_jaas.conf;
/opt/kafka/bin/kafka-configs.sh --zookeeper zookeeper:2181 --alter --add-config 'SCRAM-SHA-256=[password=admin-secret-256],SCRAM-SHA-512=[password=admin-secret-512]' --entity-type users --entity-name adminscram
KAFKA_CFG_BROKER_ID: 1
KAFKA_CFG_BROKER_RACK: zone1
KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_CFG_MESSAGE_MAX_BYTES: 200000000
KAFKA_CFG_LISTENERS: "PLAINTEXT://:9092,SSL://:9093,SASL_PLAINTEXT://:9094,SASL_SSL://:9095"
KAFKA_CFG_ADVERTISED_LISTENERS: "PLAINTEXT://localhost:9092,SSL://localhost:9093,SASL_PLAINTEXT://localhost:9094,SASL_SSL://localhost:9095"
KAFKA_CFG_SASL_ENABLED_MECHANISMS: "PLAIN,SCRAM-SHA-256,SCRAM-SHA-512"
KAFKA_CFG_AUTHORIZER_CLASS_NAME: "kafka.security.auth.SimpleAclAuthorizer"

KAFKA_CFG_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"

KAFKA_CFG_SSL_KEYSTORE_LOCATION: /opt/bitnami/kafka/config/certs/kafka.truststore.jks
KAFKA_CFG_SSL_KEYSTORE_PASSWORD: test123

KAFKA_CFG_SSL_TRUSTSTORE_LOCATION: /opt/bitnami/kafka/config/certs/kafka.truststore.jks
KAFKA_CFG_SSL_TRUSTSTORE_PASSWORD: test123


KAFKA_CFG_SSL_CLIENT_AUTH: none
KAFKA_CFG_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: ""
KAFKA_OPTS: "-Djava.security.auth.login.config=/opt/bitnami/kafka/config/kafka_jaas.conf"
ALLOW_PLAINTEXT_LISTENER: "yes"
entrypoint:
- "/bin/bash"
- "-c"
- echo -e 'KafkaServer {\norg.apache.kafka.common.security.scram.ScramLoginModule required\n username="adminscram"\n password="admin-secret";\n org.apache.kafka.common.security.plain.PlainLoginModule required\n username="adminplain"\n password="admin-secret"\n user_adminplain="admin-secret";\n };' > /opt/bitnami/kafka/config/kafka_jaas.conf; /opt/bitnami/kafka/bin/kafka-configs.sh --zookeeper zookeeper:2181 --alter --add-config "SCRAM-SHA-256=[password=admin-secret-256],SCRAM-SHA-512=[password=admin-secret-512]" --entity-type users --entity-name adminscram; exec /entrypoint.sh /run.sh
volumes:
- /var/run/docker.sock:/var/run/docker.sock
- ./examples/auth/certs:/certs
- ./examples/auth/certs:/opt/bitnami/kafka/config/certs
Loading
Loading