Kafka Cluster

A three node kafka cluster.

Run kafka cluster:

cd kafka-cluster
docker compose up -d
cd ..

Create a topic:

kafka-cli kafka-topics --create --bootstrap-server kafka1:9092 \
                       --replication-factor 3 \
                       --partitions 3 \
                       --topic kafka-cluster.test
kafka-cli kafka-topics --bootstrap-server kafka1:9092 --list

Produce a message:

kafka-cli kafka-console-producer --broker-list kafka1:9092 --topic kafka-cluster.test

Consume messages:

kafka-cli kafka-console-consumer --from-beginning --group kafka-cluster.test \
                                 --topic kafka-cluster.test  \
                                 --bootstrap-server kafka1:9092

⚠️ The JMX ports were opened to monitor kafka using jconsole.

Run jconsole:

jconsole localhost:19999

Docker Compose

services:
  kafka1:
    image: confluentinc/cp-kafka:${VERSION}
    environment:
      KAFKA_BROKER_ID: "1"
      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka1:9092,EXTERNAL://localhost:19092
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
      KAFKA_LISTENERS: CONTROLLER://0.0.0.0:9093,INTERNAL://0.0.0.0:9092,EXTERNAL://0.0.0.0:19092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
      KAFKA_JMX_PORT: "19999"
      CLUSTER_ID: "arN_n1N_QHqihZJPxv7URA"
      KAFKA_PROCESS_ROLES: "broker,controller"
      KAFKA_CONTROLLER_QUORUM_VOTERS: "1@kafka1:9093,2@kafka2:9093,3@kafka3:9093"
    ports:
      - "19092:19092"
      - "19999:19999"
    restart: on-failure
    volumes:
      - kafka1_data:/var/lib/kafka/data
    healthcheck:
      test: kafka-topics --bootstrap-server localhost:9092 --list > /dev/null 2>&1
      interval: 10s
      timeout: 10s
      retries: 5
      start_period: 10s

  kafka2:
    image: confluentinc/cp-kafka:${VERSION}
    environment:
      KAFKA_BROKER_ID: "2"
      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka2:9092,EXTERNAL://localhost:29092
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
      KAFKA_LISTENERS: CONTROLLER://0.0.0.0:9093,INTERNAL://0.0.0.0:9092,EXTERNAL://0.0.0.0:29092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
      KAFKA_JMX_PORT: "29999"
      CLUSTER_ID: "arN_n1N_QHqihZJPxv7URA"
      KAFKA_PROCESS_ROLES: "broker,controller"
      KAFKA_CONTROLLER_QUORUM_VOTERS: "1@kafka1:9093,2@kafka2:9093,3@kafka3:9093"
    ports:
      - "29092:29092"
      - "29999:29999"
    restart: on-failure
    volumes:
      - kafka2_data:/var/lib/kafka/data
    healthcheck:
      test: kafka-topics --bootstrap-server localhost:9092 --list > /dev/null 2>&1
      interval: 10s
      timeout: 10s
      retries: 5
      start_period: 10s

  kafka3:
    image: confluentinc/cp-kafka:${VERSION}
    environment:
      KAFKA_BROKER_ID: "3"
      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka3:9092,EXTERNAL://localhost:39092
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
      KAFKA_LISTENERS: CONTROLLER://0.0.0.0:9093,INTERNAL://0.0.0.0:9092,EXTERNAL://0.0.0.0:39092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
      KAFKA_JMX_PORT: "39999"
      CLUSTER_ID: "arN_n1N_QHqihZJPxv7URA"
      KAFKA_PROCESS_ROLES: "broker,controller"
      KAFKA_CONTROLLER_QUORUM_VOTERS: "1@kafka1:9093,2@kafka2:9093,3@kafka3:9093"
    ports:
      - "39092:39092"
      - "39999:39999"
    restart: on-failure
    volumes:
      - kafka3_data:/var/lib/kafka/data
    healthcheck:
      test: kafka-topics --bootstrap-server localhost:9092 --list > /dev/null 2>&1
      interval: 10s
      timeout: 10s
      retries: 5
      start_period: 10s

volumes:
  kafka1_data:
  kafka2_data:
  kafka3_data:

networks:
  default:
    external: true
    name: kafka-sandbox_network