kafka

1
# Licensed to the Apache Software Foundation (ASF) under one or more
2
# contributor license agreements.  See the NOTICE file distributed with
3
# this work for additional information regarding copyright ownership.
4
# The ASF licenses this file to You under the Apache License, Version 2.0
5
# (the "License"); you may not use this file except in compliance with
6
# the License.  You may obtain a copy of the License at
7
#
8
#    http://www.apache.org/licenses/LICENSE-2.0
9
#
10
# Unless required by applicable law or agreed to in writing, software
11
# distributed under the License is distributed on an "AS IS" BASIS,
12
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
# See the License for the specific language governing permissions and
14
# limitations under the License.
15

16
---
17
version: '2'
18
services:
19
  controller1:
20
    image: {$IMAGE}
21
    hostname: controller1
22
    container_name: controller1
23
    environment:
24
      KAFKA_NODE_ID: 1
25
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT'
26
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
27
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
28
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
29
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
30
      KAFKA_PROCESS_ROLES: 'controller'
31
      KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller1:19092,2@controller2:19092,3@controller3:19092'
32
      KAFKA_LISTENERS: 'CONTROLLER://:19092'
33
      KAFKA_INTER_BROKER_LISTENER_NAME: 'CONTROLLER'
34
      KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
35
      KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'
36
      CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw'
37

38
  controller2:
39
    image: {$IMAGE}
40
    hostname: controller2
41
    container_name: controller2
42
    environment:
43
      KAFKA_NODE_ID: 2
44
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT'
45
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
46
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
47
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
48
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
49
      KAFKA_PROCESS_ROLES: 'controller'
50
      KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller1:19092,2@controller2:19092,3@controller3:19092'
51
      KAFKA_LISTENERS: 'CONTROLLER://:19092'
52
      KAFKA_INTER_BROKER_LISTENER_NAME: 'CONTROLLER'
53
      KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
54
      KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'
55
      CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw'
56

57
  controller3:
58
    image: {$IMAGE}
59
    hostname: controller3
60
    container_name: controller3
61
    environment:
62
      KAFKA_NODE_ID: 3
63
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT'
64
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
65
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
66
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
67
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
68
      KAFKA_PROCESS_ROLES: 'controller'
69
      KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller1:19092,2@controller2:19092,3@controller3:19092'
70
      KAFKA_LISTENERS: 'CONTROLLER://:19092'
71
      KAFKA_INTER_BROKER_LISTENER_NAME: 'CONTROLLER'
72
      KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
73
      KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'
74
      CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw'
75

76
  broker1:
77
    image: {$IMAGE}
78
    hostname: broker1
79
    container_name: broker1
80
    ports:
81
      - "9092:9092"
82
      - "19093:9093"
83
      - "9101:9101"
84
    volumes:
85
      - ../../secrets:/etc/kafka/secrets
86
    environment:
87
      KAFKA_NODE_ID: 4
88
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,SSL:SSL,PLAINTEXT_HOST:PLAINTEXT'
89
      KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT_HOST://localhost:9092,SSL://localhost:19093,PLAINTEXT://broker1:29092'
90
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
91
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
92
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
93
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
94
      KAFKA_PROCESS_ROLES: 'broker'
95
      KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller1:19092,2@controller2:19092,3@controller3:19092'
96
      KAFKA_LISTENERS: 'PLAINTEXT_HOST://:9092,SSL://:9093,PLAINTEXT://:29092'
97
      KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT'
98
      KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
99
      KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'
100
      CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw'
101
      KAFKA_JMX_PORT: 9101
102
      KAFKA_JMX_HOSTNAME: localhost
103
      KAFKA_SSL_KEYSTORE_FILENAME: "kafka01.keystore.jks"
104
      KAFKA_SSL_KEYSTORE_CREDENTIALS: "kafka_keystore_creds"
105
      KAFKA_SSL_KEY_CREDENTIALS: "kafka_ssl_key_creds"
106
      KAFKA_SSL_TRUSTSTORE_FILENAME: "kafka.truststore.jks"
107
      KAFKA_SSL_TRUSTSTORE_CREDENTIALS: "kafka_truststore_creds"
108
      KAFKA_SSL_CLIENT_AUTH: "required"
109
    depends_on:
110
      - controller1
111
      - controller2
112
      - controller3
113

114
  broker2:
115
    image: {$IMAGE}
116
    hostname: broker2
117
    container_name: broker2
118
    ports:
119
      - "9093:9093"
120
      - "19092:9092"
121
    volumes:
122
      - ../../secrets:/etc/kafka/secrets
123
    environment:
124
      KAFKA_NODE_ID: 5
125
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,SSL:SSL,PLAINTEXT_HOST:PLAINTEXT'
126
      KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT_HOST://localhost:19092,SSL://localhost:9093,PLAINTEXT://broker2:29092"
127
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
128
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
129
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
130
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
131
      KAFKA_PROCESS_ROLES: 'broker'
132
      KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller1:19092,2@controller2:19092,3@controller3:19092'
133
      KAFKA_LISTENERS: 'PLAINTEXT_HOST://:9092,SSL://:9093,PLAINTEXT://:29092'
134
      KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT'
135
      KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
136
      KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'
137
      CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw'
138
      KAFKA_SSL_KEYSTORE_FILENAME: "kafka01.keystore.jks"
139
      KAFKA_SSL_KEYSTORE_CREDENTIALS: "kafka_keystore_creds"
140
      KAFKA_SSL_KEY_CREDENTIALS: "kafka_ssl_key_creds"
141
      KAFKA_SSL_TRUSTSTORE_FILENAME: "kafka.truststore.jks"
142
      KAFKA_SSL_TRUSTSTORE_CREDENTIALS: "kafka_truststore_creds"
143
      KAFKA_SSL_CLIENT_AUTH: "required"
144
    depends_on:
145
      - controller1
146
      - controller2
147
      - controller3
148

149
  broker3:
150
    image: {$IMAGE}
151
    hostname: broker3
152
    container_name: broker3
153
    ports:
154
      - "39092:9092"
155
      - "9094:9093"
156
    volumes:
157
      - ../../secrets:/etc/kafka/secrets
158
      - ../../file-input:/mnt/shared/config
159
    environment:
160
      CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw'
161
      # Set a property absent from the file
162
      KAFKA_NODE_ID: 6
163
      # Override existing properties
164
      KAFKA_PROCESS_ROLES: 'broker'
165
      KAFKA_LISTENERS: 'PLAINTEXT_HOST://:9092,SSL://:9093,PLAINTEXT://:29092'
166
      KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller1:19092,2@controller2:19092,3@controller3:19092'
167
    depends_on:
168
      - controller1
169
      - controller2
170
      - controller3
171