#
# https://hub.docker.com/r/wurstmeister/kafka/
# https://better-coding.com/building-apache-kafka-cluster-using-docker-compose-and-virtualbox/
# https://gist.github.com/everpeace/7a317860cab6c7fb39d5b0c13ec2543e
# https://github.com/zoidbergwill/docker-compose-kafka/blob/master/docker-compose.yml
# https://www.kaaproject.org/kafka-docker/
#
version: '3.3'
services:
emq:
image: emqx/emqx:latest
ports:
- "1883:1883"
- "8083:8083"
- "18083:18083"
kafka1:
image: wurstmeister/kafka
healthcheck:
test: ps augwwx | egrep [S]upportedKafka
depends_on:
- zk1
- zk2
- zk3
volumes:
- "./kafka1/mqtt.properties:/opt/kafka/config/mqtt.properties"
- "./kafka-connect/kafka-connect-mqtt-1.0-SNAPSHOT.jar:/opt/kafka/libs/kafka-connect-mqtt-1.0-SNAPSHOT.jar"
- "./kafka-connect/kafka-connect-mqtt-1.2.1.jar:/opt/kafka/libs/kafka-connect-mqtt-1.2.1.jar"
- "./kafka-connect/org.eclipse.paho.client.mqttv3-1.0.2.jar:/opt/kafka/libs/org.eclipse.paho.client.mqttv3-1.0.2.jar"
ports:
- "9091:9092"
environment:
KAFKA_BROKER_ID: 1
KAFKA_CREATE_TOPICS: hello-mqtt:1:2,hello-mqtt-kafka:1:2,test-topic:1:2
KAFKA_ZOOKEEPER_CONNECT: zk1:2181,zk2:2181,zk3:2181
KAFKA_ADVERTISED_HOST_NAME: 192.168.35.106
KAFKA_ADVERTISED_PORT: 9091
KAFKA_LOG_RETENTION_HOURS: "168"
KAFKA_LOG_RETENTION_BYTES: "100000000"
kafka2:
image: wurstmeister/kafka
healthcheck:
test: ps augwwx | egrep [S]upportedKafka
depends_on:
- zk1
- zk2
- zk3
volumes:
- "./kafka2/mqtt.properties:/opt/kafka/config/mqtt.properties"
- "./kafka-connect/kafka-connect-mqtt-1.0-SNAPSHOT.jar:/opt/kafka/libs/kafka-connect-mqtt-1.0-SNAPSHOT.jar"
- "./kafka-connect/kafka-connect-mqtt-1.2.1.jar:/opt/kafka/libs/kafka-connect-mqtt-1.2.1.jar"
- "./kafka-connect/org.eclipse.paho.client.mqttv3-1.0.2.jar:/opt/kafka/libs/org.eclipse.paho.client.mqttv3-1.0.2.jar"
ports:
- "9092:9092"
environment:
KAFKA_BROKER_ID: 2
KAFKA_ZOOKEEPER_CONNECT: zk1:2181,zk2:2181,zk3:2181
KAFKA_ADVERTISED_HOST_NAME: 192.168.35.106
KAFKA_ADVERTISED_PORT: 9092
KAFKA_LOG_RETENTION_HOURS: "168"
KAFKA_LOG_RETENTION_BYTES: "100000000"
kafka3:
image: wurstmeister/kafka
healthcheck:
test: ps augwwx | egrep [S]upportedKafka
depends_on:
- zk1
- zk2
- zk3
volumes:
- "./kafka3/mqtt.properties:/opt/kafka/config/mqtt.properties"
- "./kafka-connect/kafka-connect-mqtt-1.0-SNAPSHOT.jar:/opt/kafka/libs/kafka-connect-mqtt-1.0-SNAPSHOT.jar"
- "./kafka-connect/kafka-connect-mqtt-1.2.1.jar:/opt/kafka/libs/kafka-connect-mqtt-1.2.1.jar"
- "./kafka-connect/org.eclipse.paho.client.mqttv3-1.0.2.jar:/opt/kafka/libs/org.eclipse.paho.client.mqttv3-1.0.2.jar"
ports:
- "9093:9092"
environment:
KAFKA_BROKER_ID: 3
KAFKA_ZOOKEEPER_CONNECT: zk1:2181,zk2:2181,zk3:2181
KAFKA_ADVERTISED_HOST_NAME: 192.168.35.106
KAFKA_ADVERTISED_PORT: 9093
KAFKA_LOG_RETENTION_HOURS: "168"
KAFKA_LOG_RETENTION_BYTES: "100000000"
zk1:
image: elevy/zookeeper:latest
environment:
MYID: 1
SERVERS: zk1,zk2,zk3
ports:
- "2181:2181"
- "2888"
- "3888"
healthcheck:
test: echo stat | nc localhost 2181
interval: 10s
timeout: 10s
retries: 3
zk2:
image: elevy/zookeeper:latest
environment:
MYID: 2
SERVERS: zk1,zk2,zk3
ports:
- "2182:2181"
- "2888"
- "3888"
healthcheck:
test: echo stat | nc localhost 2181
interval: 10s
timeout: 10s
retries: 3
zk3:
image: elevy/zookeeper:latest
environment:
MYID: 3
SERVERS: zk1,zk2,zk3
ports:
- "2183:2181"
- "2888"
- "3888"
healthcheck:
test: echo stat | nc localhost 2181
interval: 10s
timeout: 10s
retries: 3
kafka-manager:
image: sheepkiller/kafka-manager:latest
depends_on:
- kafka1
- kafka2
- kafka3
- zk1
- zk2
- zk3
links:
- zk1
- zk2
- zk3
ports:
- "9000:9000"
environment:
ZK_HOSTS: zk1:2181,zk2:2181,zk3:2181
APPLICATION_SECRET: letmein
KM_ARGS: -Djava.net.preferIPv4Stack=true
kafka-bridge:
build: ./mqtt-to-kafka
# NOTE: adding -u unbuffers STDIN/STDOUT and makes print statements
# available in the logs
command: ["python", "-u", "message_intercept.py"]
volumes:
- ./mqtt-to-kafka:/app
depends_on:
- kafka1
- kafka2
- kafka3
- emq
environment:
- KAFKA_HOSTS="kafka1,kafka2,kafka3"
restart: unless-stopped
elasticsearch:
build:
context: elk/elasticsearch/
args:
ELK_VERSION: $ELK_VERSION
volumes:
- ./elk/elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro
- ./elk/elasticsearch/data:/usr/share/elasticsearch/data
ports:
- "9200:9200"
- "9300:9300"
environment:
ES_JAVA_OPTS: "-Xmx256m -Xms256m"
healthcheck:
test: echo stat | nc localhost 9200
interval: 10s
timeout: 10s
retries: 3
logstash:
build:
context: elk/logstash/
args:
ELK_VERSION: $ELK_VERSION
volumes:
- ./elk/logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml:ro
- ./elk/logstash/pipeline:/usr/share/logstash/pipeline:ro
ports:
- "5000:5000"
- "9600:9600"
environment:
LS_JAVA_OPTS: "-Xmx256m -Xms256m"
depends_on:
- elasticsearch
kibana:
build:
context: elk/kibana/
args:
ELK_VERSION: $ELK_VERSION
volumes:
- ./elk/kibana/config/:/usr/share/kibana/config:ro
ports:
- "5601:5601"
depends_on:
- elasticsearch
healthcheck:
test: echo stat | nc localhost 5601
interval: 10s
timeout: 10s
retries: 3
grafana:
image: grafana/grafana
ports:
- "3000:3000"
depends_on:
- elasticsearch
healthcheck:
test: echo stat | nc localhost 3000
interval: 10s
timeout: 10s
retries: 3
emq-kafka-elk.zip