Kafka Cluster
A three node kafka cluster.
- kafka
- kafka settings
- project location: kafka-cluster
- kafka version: cp 7.5.2
- kafka ports:
19092
,29092
,39092
- kafka jmx ports:
19999
,29999
,39999
Run kafka cluster:
cd kafka-cluster
docker compose up -d
cd ..
Create a topic:
kafka-cli kafka-topics --create --bootstrap-server kafka1:9092 \
--replication-factor 3 \
--partitions 3 \
--topic kafka-cluster.test
kafka-cli kafka-topics --bootstrap-server kafka1:9092 --list
Produce a message:
kafka-cli kafka-console-producer --broker-list kafka1:9092 --topic kafka-cluster.test
Consume messages:
kafka-cli kafka-console-consumer --from-beginning --group kafka-cluster.test \
--topic kafka-cluster.test \
--bootstrap-server kafka1:9092
⚠️ The
JMX
ports were opened to monitor kafka usingjconsole
.
Run jconsole
:
jconsole localhost:19999
Docker Compose
services:
kafka1:
image: confluentinc/cp-kafka:${VERSION}
environment:
KAFKA_BROKER_ID: "1"
KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka1:9092,EXTERNAL://localhost:19092
KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
KAFKA_LISTENERS: CONTROLLER://0.0.0.0:9093,INTERNAL://0.0.0.0:9092,EXTERNAL://0.0.0.0:19092
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
KAFKA_JMX_PORT: "19999"
CLUSTER_ID: "arN_n1N_QHqihZJPxv7URA"
KAFKA_PROCESS_ROLES: "broker,controller"
KAFKA_CONTROLLER_QUORUM_VOTERS: "1@kafka1:9093,2@kafka2:9093,3@kafka3:9093"
ports:
- "19092:19092"
- "19999:19999"
restart: on-failure
volumes:
- kafka1_data:/var/lib/kafka/data
healthcheck:
test: kafka-topics --bootstrap-server localhost:9092 --list > /dev/null 2>&1
interval: 10s
timeout: 10s
retries: 5
start_period: 10s
kafka2:
image: confluentinc/cp-kafka:${VERSION}
environment:
KAFKA_BROKER_ID: "2"
KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka2:9092,EXTERNAL://localhost:29092
KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
KAFKA_LISTENERS: CONTROLLER://0.0.0.0:9093,INTERNAL://0.0.0.0:9092,EXTERNAL://0.0.0.0:29092
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
KAFKA_JMX_PORT: "29999"
CLUSTER_ID: "arN_n1N_QHqihZJPxv7URA"
KAFKA_PROCESS_ROLES: "broker,controller"
KAFKA_CONTROLLER_QUORUM_VOTERS: "1@kafka1:9093,2@kafka2:9093,3@kafka3:9093"
ports:
- "29092:29092"
- "29999:29999"
restart: on-failure
volumes:
- kafka2_data:/var/lib/kafka/data
healthcheck:
test: kafka-topics --bootstrap-server localhost:9092 --list > /dev/null 2>&1
interval: 10s
timeout: 10s
retries: 5
start_period: 10s
kafka3:
image: confluentinc/cp-kafka:${VERSION}
environment:
KAFKA_BROKER_ID: "3"
KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka3:9092,EXTERNAL://localhost:39092
KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
KAFKA_LISTENERS: CONTROLLER://0.0.0.0:9093,INTERNAL://0.0.0.0:9092,EXTERNAL://0.0.0.0:39092
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
KAFKA_JMX_PORT: "39999"
CLUSTER_ID: "arN_n1N_QHqihZJPxv7URA"
KAFKA_PROCESS_ROLES: "broker,controller"
KAFKA_CONTROLLER_QUORUM_VOTERS: "1@kafka1:9093,2@kafka2:9093,3@kafka3:9093"
ports:
- "39092:39092"
- "39999:39999"
restart: on-failure
volumes:
- kafka3_data:/var/lib/kafka/data
healthcheck:
test: kafka-topics --bootstrap-server localhost:9092 --list > /dev/null 2>&1
interval: 10s
timeout: 10s
retries: 5
start_period: 10s
volumes:
kafka1_data:
kafka2_data:
kafka3_data:
networks:
default:
external: true
name: kafka-sandbox_network