From 094e5acf30461108c06d1798b116f52417fed64c Mon Sep 17 00:00:00 2001 From: Alex Burdusel Date: Sat, 28 Mar 2026 12:54:26 +0200 Subject: [PATCH 1/3] Update Kafka to 4.0.2 and configured KRaft mode. Update Kafka-UI to the latest version. --- applications/events/deploy/charts/Chart.yaml | 4 +- applications/events/deploy/charts/README.md | 35 +- .../deploy/charts/templates/_helpers.tpl | 6 +- .../deploy/charts/templates/configmap.yaml | 3 +- .../templates/configmap_fromValues.yaml | 3 +- .../deploy/charts/templates/deployment.yaml | 3 +- .../events/deploy/charts/templates/hpa.yaml | 1 + .../deploy/charts/templates/ingress.yaml | 12 +- .../templates/networkpolicy-egress.yaml | 1 + .../templates/networkpolicy-ingress.yaml | 1 + .../deploy/charts/templates/secret.yaml | 3 + .../deploy/charts/templates/service.yaml | 4 + .../charts/templates/serviceaccount.yaml | 1 + applications/events/deploy/charts/values.yaml | 5 + .../events/deploy/resources/broker/init.sh | 47 --- .../deploy/resources/broker/log4j.properties | 76 ---- .../deploy/resources/broker/server.properties | 134 ------- .../events/deploy/resources/zookeeper/init.sh | 15 - .../resources/zookeeper/log4j.properties | 8 - .../resources/zookeeper/zookeeper.properties | 12 - .../deploy/templates-compose/deployments.yaml | 114 ++---- .../events/deploy/templates/broker-config.yml | 9 - .../events/deploy/templates/deployments.yml | 346 +++--------------- .../events/deploy/templates/roles.yml | 66 ---- .../events/deploy/templates/services.yml | 52 +-- .../events/deploy/templates/zoo-config.yml | 7 - applications/events/deploy/values.yaml | 32 +- docs/events.md | 38 +- 28 files changed, 169 insertions(+), 869 deletions(-) delete mode 100644 applications/events/deploy/resources/broker/init.sh delete mode 100644 applications/events/deploy/resources/broker/log4j.properties delete mode 100644 applications/events/deploy/resources/broker/server.properties delete mode 100644 applications/events/deploy/resources/zookeeper/init.sh delete mode 100644 applications/events/deploy/resources/zookeeper/log4j.properties delete mode 100644 applications/events/deploy/resources/zookeeper/zookeeper.properties delete mode 100644 applications/events/deploy/templates/broker-config.yml delete mode 100644 applications/events/deploy/templates/roles.yml delete mode 100644 applications/events/deploy/templates/zoo-config.yml diff --git a/applications/events/deploy/charts/Chart.yaml b/applications/events/deploy/charts/Chart.yaml index 2ae296b4e..460bb6503 100644 --- a/applications/events/deploy/charts/Chart.yaml +++ b/applications/events/deploy/charts/Chart.yaml @@ -2,6 +2,6 @@ apiVersion: v2 name: kafka-ui description: A Helm chart for kafka-UI type: application -version: 0.4.6 -appVersion: v0.4.0 +version: 0.7.6 +appVersion: v0.7.2 icon: https://github.com/provectus/kafka-ui/raw/master/documentation/images/kafka-ui-logo.png diff --git a/applications/events/deploy/charts/README.md b/applications/events/deploy/charts/README.md index 5d08629dc..3b71ce11c 100644 --- a/applications/events/deploy/charts/README.md +++ b/applications/events/deploy/charts/README.md @@ -1,34 +1 @@ -# Kafka-UI Helm Chart - -## Configuration - -Most of the Helm charts parameters are common, follow table describe unique parameters related to application configuration. - -### Kafka-UI parameters - -| Parameter | Description | Default | -| ---------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | ------- | -| `existingConfigMap` | Name of the existing ConfigMap with Kafka-UI environment variables | `nil` | -| `existingSecret` | Name of the existing Secret with Kafka-UI environment variables | `nil` | -| `envs.secret` | Set of the sensitive environment variables to pass to Kafka-UI | `{}` | -| `envs.config` | Set of the environment variables to pass to Kafka-UI | `{}` | -| `yamlApplicationConfigConfigMap` | Map with name and keyName keys, name refers to the existing ConfigMap, keyName refers to the ConfigMap key with Kafka-UI config in Yaml format | `{}` | -| `yamlApplicationConfig` | Kafka-UI config in Yaml format | `{}` | -| `networkPolicy.enabled` | Enable network policies | `false` | -| `networkPolicy.egressRules.customRules` | Custom network egress policy rules | `[]` | -| `networkPolicy.ingressRules.customRules` | Custom network ingress policy rules | `[]` | -| `podLabels` | Extra labels for Kafka-UI pod | `{}` | - - -## Example - -To install Kafka-UI need to execute follow: -``` bash -helm repo add kafka-ui https://provectus.github.io/kafka-ui -helm install kafka-ui kafka-ui/kafka-ui --set envs.config.KAFKA_CLUSTERS_0_NAME=local --set envs.config.KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS=kafka:9092 -``` -To connect to Kafka-UI web application need to execute: -``` bash -kubectl port-forward svc/kafka-ui 8080:80 -``` -Open the `http://127.0.0.1:8080` on the browser to access Kafka-UI. +Please refer to our [documentation](https://docs.kafka-ui.provectus.io/configuration/helm-charts) to get some info on our helm charts. diff --git a/applications/events/deploy/charts/templates/_helpers.tpl b/applications/events/deploy/charts/templates/_helpers.tpl index 510452d4c..91246bcb5 100644 --- a/applications/events/deploy/charts/templates/_helpers.tpl +++ b/applications/events/deploy/charts/templates/_helpers.tpl @@ -68,6 +68,11 @@ This allows us to check if the registry of the image is specified or not. */}} {{- define "kafka-ui.imageName" -}} {{- $registryName := .Values.image.registry -}} +{{- if .Values.global }} + {{- if .Values.global.imageRegistry }} + {{- $registryName = .Values.global.imageRegistry -}} + {{- end -}} +{{- end -}} {{- $repository := .Values.image.repository -}} {{- $tag := .Values.image.tag | default .Chart.AppVersion -}} {{- if $registryName }} @@ -76,4 +81,3 @@ This allows us to check if the registry of the image is specified or not. {{- printf "%s:%s" $repository $tag -}} {{- end }} {{- end -}} - diff --git a/applications/events/deploy/charts/templates/configmap.yaml b/applications/events/deploy/charts/templates/configmap.yaml index 22d2a6994..56b2bf4dc 100644 --- a/applications/events/deploy/charts/templates/configmap.yaml +++ b/applications/events/deploy/charts/templates/configmap.yaml @@ -3,8 +3,9 @@ apiVersion: v1 kind: ConfigMap metadata: name: {{ include "kafka-ui.fullname" . }} + namespace: {{ .Release.Namespace }} labels: {{- include "kafka-ui.labels" . | nindent 4 }} data: {{- toYaml .Values.envs.config | nindent 2 }} -{{- end -}} \ No newline at end of file +{{- end -}} diff --git a/applications/events/deploy/charts/templates/configmap_fromValues.yaml b/applications/events/deploy/charts/templates/configmap_fromValues.yaml index ae8b7695c..7e5823835 100644 --- a/applications/events/deploy/charts/templates/configmap_fromValues.yaml +++ b/applications/events/deploy/charts/templates/configmap_fromValues.yaml @@ -3,9 +3,10 @@ apiVersion: v1 kind: ConfigMap metadata: name: {{ include "kafka-ui.fullname" . }}-fromvalues + namespace: {{ .Release.Namespace }} labels: {{- include "kafka-ui.labels" . | nindent 4 }} data: config.yml: |- {{- toYaml .Values.yamlApplicationConfig | nindent 4}} -{{ end }} \ No newline at end of file +{{ end }} diff --git a/applications/events/deploy/charts/templates/deployment.yaml b/applications/events/deploy/charts/templates/deployment.yaml index 219e9b440..7a92712b8 100644 --- a/applications/events/deploy/charts/templates/deployment.yaml +++ b/applications/events/deploy/charts/templates/deployment.yaml @@ -2,6 +2,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: {{ include "kafka-ui.fullname" . }} + namespace: {{ .Release.Namespace }} labels: {{- include "kafka-ui.labels" . | nindent 4 }} {{- with .Values.annotations }} @@ -53,7 +54,7 @@ spec: {{- toYaml . | nindent 12 }} {{- end }} {{- if or .Values.yamlApplicationConfig .Values.yamlApplicationConfigConfigMap}} - - name: SPRING_CONFIG_LOCATION + - name: SPRING_CONFIG_ADDITIONAL-LOCATION {{- if .Values.yamlApplicationConfig }} value: /kafka-ui/config.yml {{- else if .Values.yamlApplicationConfigConfigMap }} diff --git a/applications/events/deploy/charts/templates/hpa.yaml b/applications/events/deploy/charts/templates/hpa.yaml index 1cc131c9e..77188bdc7 100644 --- a/applications/events/deploy/charts/templates/hpa.yaml +++ b/applications/events/deploy/charts/templates/hpa.yaml @@ -9,6 +9,7 @@ apiVersion: autoscaling/v2beta1 kind: HorizontalPodAutoscaler metadata: name: {{ include "kafka-ui.fullname" . }} + namespace: {{ .Release.Namespace }} labels: {{- include "kafka-ui.labels" . | nindent 4 }} spec: diff --git a/applications/events/deploy/charts/templates/ingress.yaml b/applications/events/deploy/charts/templates/ingress.yaml index e4b33439c..75b107bd4 100644 --- a/applications/events/deploy/charts/templates/ingress.yaml +++ b/applications/events/deploy/charts/templates/ingress.yaml @@ -13,6 +13,7 @@ apiVersion: extensions/v1beta1 kind: Ingress metadata: name: {{ $fullName }} + namespace: {{ .Release.Namespace }} labels: {{- include "kafka-ui.labels" . | nindent 4 }} {{- with .Values.ingress.annotations }} @@ -35,25 +36,30 @@ spec: {{- if and ($.Capabilities.APIVersions.Has "networking.k8s.io/v1") $isHigher1p19 -}} {{- range .Values.ingress.precedingPaths }} - path: {{ .path }} - pathType: Prefix + pathType: {{ .pathType }} backend: service: name: {{ .serviceName }} port: + {{- if .servicePort }} number: {{ .servicePort }} + {{- end }} + {{- if .servicePortName }} + name: {{ .servicePortName }} + {{- end }} {{- end }} - backend: service: name: {{ $fullName }} port: number: {{ $svcPort }} - pathType: Prefix + pathType: {{ .Values.ingress.pathType }} {{- if .Values.ingress.path }} path: {{ .Values.ingress.path }} {{- end }} {{- range .Values.ingress.succeedingPaths }} - path: {{ .path }} - pathType: Prefix + pathType: {{ .pathType }} backend: service: name: {{ .serviceName }} diff --git a/applications/events/deploy/charts/templates/networkpolicy-egress.yaml b/applications/events/deploy/charts/templates/networkpolicy-egress.yaml index 4f5828027..1ba60ec1b 100644 --- a/applications/events/deploy/charts/templates/networkpolicy-egress.yaml +++ b/applications/events/deploy/charts/templates/networkpolicy-egress.yaml @@ -3,6 +3,7 @@ apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: {{ printf "%s-egress" (include "kafka-ui.fullname" .) }} + namespace: {{ .Release.Namespace }} labels: {{- include "kafka-ui.labels" . | nindent 4 }} spec: diff --git a/applications/events/deploy/charts/templates/networkpolicy-ingress.yaml b/applications/events/deploy/charts/templates/networkpolicy-ingress.yaml index 74988676b..5afbc55e8 100644 --- a/applications/events/deploy/charts/templates/networkpolicy-ingress.yaml +++ b/applications/events/deploy/charts/templates/networkpolicy-ingress.yaml @@ -3,6 +3,7 @@ apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: {{ printf "%s-ingress" (include "kafka-ui.fullname" .) }} + namespace: {{ .Release.Namespace }} labels: {{- include "kafka-ui.labels" . | nindent 4 }} spec: diff --git a/applications/events/deploy/charts/templates/secret.yaml b/applications/events/deploy/charts/templates/secret.yaml index a2d1f25fa..387666f84 100644 --- a/applications/events/deploy/charts/templates/secret.yaml +++ b/applications/events/deploy/charts/templates/secret.yaml @@ -1,7 +1,9 @@ +{{- if .Values.envs.secret -}} apiVersion: v1 kind: Secret metadata: name: {{ include "kafka-ui.fullname" . }} + namespace: {{ .Release.Namespace }} labels: {{- include "kafka-ui.labels" . | nindent 4 }} type: Opaque @@ -9,3 +11,4 @@ data: {{- range $key, $val := .Values.envs.secret }} {{ $key }}: {{ $val | b64enc | quote }} {{- end -}} +{{- end}} diff --git a/applications/events/deploy/charts/templates/service.yaml b/applications/events/deploy/charts/templates/service.yaml index 5801135c4..337e1457d 100644 --- a/applications/events/deploy/charts/templates/service.yaml +++ b/applications/events/deploy/charts/templates/service.yaml @@ -2,6 +2,7 @@ apiVersion: v1 kind: Service metadata: name: {{ include "kafka-ui.fullname" . }} + namespace: {{ .Release.Namespace }} labels: {{- include "kafka-ui.labels" . | nindent 4 }} {{- if .Values.service.annotations }} @@ -10,6 +11,9 @@ metadata: {{- end }} spec: type: {{ .Values.service.type }} +{{- if and (eq .Values.service.type "LoadBalancer") (.Values.service.loadBalancerIP) }} + loadBalancerIP: {{ .Values.service.loadBalancerIP }} +{{- end }} ports: - port: {{ .Values.service.port }} targetPort: http diff --git a/applications/events/deploy/charts/templates/serviceaccount.yaml b/applications/events/deploy/charts/templates/serviceaccount.yaml index b89551c83..ddefde371 100644 --- a/applications/events/deploy/charts/templates/serviceaccount.yaml +++ b/applications/events/deploy/charts/templates/serviceaccount.yaml @@ -3,6 +3,7 @@ apiVersion: v1 kind: ServiceAccount metadata: name: {{ include "kafka-ui.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} labels: {{- include "kafka-ui.labels" . | nindent 4 }} {{- with .Values.serviceAccount.annotations }} diff --git a/applications/events/deploy/charts/values.yaml b/applications/events/deploy/charts/values.yaml index 570f1cb43..190172eeb 100644 --- a/applications/events/deploy/charts/values.yaml +++ b/applications/events/deploy/charts/values.yaml @@ -93,6 +93,8 @@ securityContext: service: type: ClusterIP port: 80 + # In case of service type LoadBalancer, you can specify reserved static IP + # loadBalancerIP: 10.11.12.13 # if you want to force a specific nodePort. Must be use with service.type=NodePort # nodePort: @@ -110,6 +112,9 @@ ingress: # The path for the Ingress path: "/" + # The path type for the Ingress + pathType: "Prefix" + # The hostname for the Ingress host: "" diff --git a/applications/events/deploy/resources/broker/init.sh b/applications/events/deploy/resources/broker/init.sh deleted file mode 100644 index 18e88f3f2..000000000 --- a/applications/events/deploy/resources/broker/init.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -set -e -set -x -cp /etc/kafka-configmap/log4j.properties /etc/kafka/ - -KAFKA_BROKER_ID=${HOSTNAME##*-} -SEDS=("s/#init#broker.id=#init#/broker.id=$KAFKA_BROKER_ID/") -LABELS="kafka-broker-id=$KAFKA_BROKER_ID" -ANNOTATIONS="" - -hash kubectl 2>/dev/null || { - SEDS+=("s/#init#broker.rack=#init#/#init#broker.rack=# kubectl not found in path/") -} && { - ZONE=$(kubectl get node "$NODE_NAME" -o=go-template='{{index .metadata.labels "failure-domain.beta.kubernetes.io/zone"}}') - if [ "x$ZONE" == "x" ]; then - SEDS+=("s/#init#broker.rack=#init#/#init#broker.rack=# zone label not found for node $NODE_NAME/") - else - SEDS+=("s/#init#broker.rack=#init#/broker.rack=$ZONE/") - LABELS="$LABELS kafka-broker-rack=$ZONE" - fi - - OUTSIDE_HOST=$(kubectl get node "$NODE_NAME" -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}') - OUTSIDE_PORT=$((32400 + ${KAFKA_BROKER_ID})) - SEDS+=("s|#init#advertised.listeners=PLAINTEXT://#init#|advertised.listeners=PLAINTEXT://:9092,OUTSIDE://${OUTSIDE_HOST}:${OUTSIDE_PORT}|") - ANNOTATIONS="$ANNOTATIONS kafka-listener-outside-host=$OUTSIDE_HOST kafka-listener-outside-port=$OUTSIDE_PORT" - - if [ ! -z "$LABELS" ]; then - kubectl -n $POD_NAMESPACE label pod $POD_NAME $LABELS || echo "Failed to label $POD_NAMESPACE.$POD_NAME - RBAC issue?" - fi - if [ ! -z "$ANNOTATIONS" ]; then - kubectl -n $POD_NAMESPACE annotate pod $POD_NAME $ANNOTATIONS || echo "Failed to annotate $POD_NAMESPACE.$POD_NAME - RBAC issue?" - fi -} -printf '%s\n' "${SEDS[@]}" | sed -f - /etc/kafka-configmap/server.properties > /etc/kafka/server.properties.tmp -[ $? -eq 0 ] && mv /etc/kafka/server.properties.tmp /etc/kafka/server.properties - -# wait for events to be up & running -# install curl first -# apt update && apt install -y curl -# for i in {1..100} -# do -# sleep 5 -# if curl events; then -# curl events:80/clusters --data 'name=cluster-'${POD_NAMESPACE}'&zkHosts=zookeeper.'${POD_NAMESPACE}':2181&kafkaVersion=2.2.0&jmxEnabled=true&jmxUser=&jmxPass=&logkafkaEnabled=true&pollConsumers=true&activeOffsetCacheEnabled=true&tuning.brokerViewUpdatePeriodSeconds=30&tuning.clusterManagerThreadPoolSize=2&tuning.clusterManagerThreadPoolQueueSize=100&tuning.kafkaCommandThreadPoolSize=2&tuning.kafkaCommandThreadPoolQueueSize=100&tuning.logkafkaCommandThreadPoolSize=2&tuning.logkafkaCommandThreadPoolQueueSize=100&tuning.logkafkaUpdatePeriodSeconds=30&tuning.partitionOffsetCacheTimeoutSecs=5&tuning.brokerViewThreadPoolSize=2&tuning.brokerViewThreadPoolQueueSize=1000&tuning.offsetCacheThreadPoolSize=2&tuning.offsetCacheThreadPoolQueueSize=1000&tuning.kafkaAdminClientThreadPoolSize=2&tuning.kafkaAdminClientThreadPoolQueueSize=1000&tuning.kafkaManagedOffsetMetadataCheckMillis=30000&tuning.kafkaManagedOffsetGroupCacheSize=1000000&tuning.kafkaManagedOffsetGroupExpireDays=7&securityProtocol=PLAINTEXT&saslMechanism=DEFAULT&jaasConfig=' -X POST -# exit 0 -# fi -# done diff --git a/applications/events/deploy/resources/broker/log4j.properties b/applications/events/deploy/resources/broker/log4j.properties deleted file mode 100644 index 085a7e41b..000000000 --- a/applications/events/deploy/resources/broker/log4j.properties +++ /dev/null @@ -1,76 +0,0 @@ -# Unspecified loggers and loggers with additivity=true output to server.log and stdout -# Note that INFO only applies to unspecified loggers, the log level of the child logger is used otherwise -log4j.rootLogger=INFO, stdout - -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n - -log4j.appender.kafkaAppender=org.apache.log4j.DailyRollingFileAppender -log4j.appender.kafkaAppender.DatePattern='.'yyyy-MM-dd-HH -log4j.appender.kafkaAppender.File=${kafka.logs.dir}/server.log -log4j.appender.kafkaAppender.layout=org.apache.log4j.PatternLayout -log4j.appender.kafkaAppender.layout.ConversionPattern=[%d] %p %m (%c)%n - -log4j.appender.stateChangeAppender=org.apache.log4j.DailyRollingFileAppender -log4j.appender.stateChangeAppender.DatePattern='.'yyyy-MM-dd-HH -log4j.appender.stateChangeAppender.File=${kafka.logs.dir}/state-change.log -log4j.appender.stateChangeAppender.layout=org.apache.log4j.PatternLayout -log4j.appender.stateChangeAppender.layout.ConversionPattern=[%d] %p %m (%c)%n - -log4j.appender.requestAppender=org.apache.log4j.DailyRollingFileAppender -log4j.appender.requestAppender.DatePattern='.'yyyy-MM-dd-HH -log4j.appender.requestAppender.File=${kafka.logs.dir}/kafka-request.log -log4j.appender.requestAppender.layout=org.apache.log4j.PatternLayout -log4j.appender.requestAppender.layout.ConversionPattern=[%d] %p %m (%c)%n - -log4j.appender.cleanerAppender=org.apache.log4j.DailyRollingFileAppender -log4j.appender.cleanerAppender.DatePattern='.'yyyy-MM-dd-HH -log4j.appender.cleanerAppender.File=${kafka.logs.dir}/log-cleaner.log -log4j.appender.cleanerAppender.layout=org.apache.log4j.PatternLayout -log4j.appender.cleanerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n - -log4j.appender.controllerAppender=org.apache.log4j.DailyRollingFileAppender -log4j.appender.controllerAppender.DatePattern='.'yyyy-MM-dd-HH -log4j.appender.controllerAppender.File=${kafka.logs.dir}/controller.log -log4j.appender.controllerAppender.layout=org.apache.log4j.PatternLayout -log4j.appender.controllerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n - -log4j.appender.authorizerAppender=org.apache.log4j.DailyRollingFileAppender -log4j.appender.authorizerAppender.DatePattern='.'yyyy-MM-dd-HH -log4j.appender.authorizerAppender.File=${kafka.logs.dir}/kafka-authorizer.log -log4j.appender.authorizerAppender.layout=org.apache.log4j.PatternLayout -log4j.appender.authorizerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n - -# Change the two lines below to adjust ZK client logging -log4j.logger.org.I0Itec.zkclient.ZkClient=INFO -log4j.logger.org.apache.zookeeper=INFO - -# Change the two lines below to adjust the general broker logging level (output to server.log and stdout) -log4j.logger.kafka=INFO -log4j.logger.org.apache.kafka=INFO - -# Change to DEBUG or TRACE to enable request logging -log4j.logger.kafka.request.logger=WARN, requestAppender -log4j.additivity.kafka.request.logger=false - -# Uncomment the lines below and change log4j.logger.kafka.network.RequestChannel$ to TRACE for additional output -# related to the handling of requests -#log4j.logger.kafka.network.Processor=TRACE, requestAppender -#log4j.logger.kafka.server.KafkaApis=TRACE, requestAppender -#log4j.additivity.kafka.server.KafkaApis=false -log4j.logger.kafka.network.RequestChannel$=WARN, requestAppender -log4j.additivity.kafka.network.RequestChannel$=false - -log4j.logger.kafka.controller=TRACE, controllerAppender -log4j.additivity.kafka.controller=false - -log4j.logger.kafka.log.LogCleaner=INFO, cleanerAppender -log4j.additivity.kafka.log.LogCleaner=false - -log4j.logger.state.change.logger=TRACE, stateChangeAppender -log4j.additivity.state.change.logger=false - -# Change to DEBUG to enable audit log for the authorizer -log4j.logger.kafka.authorizer.logger=WARN, authorizerAppender -log4j.additivity.kafka.authorizer.logger=false \ No newline at end of file diff --git a/applications/events/deploy/resources/broker/server.properties b/applications/events/deploy/resources/broker/server.properties deleted file mode 100644 index 584136d92..000000000 --- a/applications/events/deploy/resources/broker/server.properties +++ /dev/null @@ -1,134 +0,0 @@ -############################# Log Basics ############################## - -# A comma seperated list of directories under which to store log files -# Overrideslog.dir -log.dirs=/var/lib/kafka/data/topics - -# The default number of log partitions per topic. More partitions allow greater -# parallelism for consumption, but this will also result in more files across -# the brokers. -num.partitions=12 - -default.replication.factor=3 - -min.insync.replicas=2 - -auto.create.topics.enable=true - -# The number of threads per data directory to be used for log recovery at startup and flushing at shutdown. -# This value is recommended to be increased for installations with data dirs located in RAID array. -# num.recovery.threads.per.data.dir=1 - -############################# Server Basics ############################# - -# The id of the broker. This must be set to a unique integer for each broker. -#init#broker.id=#init# - -#init#broker.rack=#init# - -############################# Socket Server Settings ############################# - -# The address the socket server listens on. It will get the value returned from -# java.net.InetAddress.getCanonicalHostName() if not configured. -# FORMAT: -# listeners = listener_name://host_name:port -# EXAMPLE: -# listeners = PLAINTEXT://your.host.name:9092 -#listeners=PLAINTEXT://:9092 -listeners=PLAINTEXT://:9092,OUTSIDE://:9094 - -# Hostname and port the broker will advertise to producers and consumers. If not set, -# it uses the value for "listeners" if configured. Otherwise, it will use the value -# returned from java.net.InetAddress.getCanonicalHostName(). -#advertised.listeners=PLAINTEXT://your.host.name:9092 -#init#advertised.listeners=PLAINTEXT://#init# - -# Maps listener names to security protocols, the default is for them to be the same. See the config documentation for more details -#listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL -listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL,OUTSIDE:PLAINTEXT -inter.broker.listener.name=PLAINTEXT - -# The number of threads that the server uses for receiving requests from the network and sending responses to the network -#num.network.threads=3 - -# The number of threads that the server uses for processing requests, which may include disk I/O -#num.io.threads=8 - -#The send buffer (SO_SNDBUF) used by the socket server -#socket.send.buffer.bytes=102400 - -#The receive buffer (SO_RCVBUF) used by the socket server -#socket.receive.buffer.bytes=102400 - -#The maximum size of a request that the socket server will accept (protection againstOOM) -#socket.request.max.bytes=104857600 - -############################# Internal Topic Settings ############################# -# The replication factor for the group metadata internal topics \"__consumer_offsets\" and \"__transaction_state\" -#For anything other than development testing, a value greater than 1 is recommended for to ensure availability such as 3. -#offsets.topic.replication.factor=1 -#transaction.state.log.replication.factor=1 -#transaction.state.log.min.isr=1 - -############################# Log Flush Policy ############################# - -# Messages are immediately written to the filesystem but by default we only fsync() to sync -# the OS cache lazily. The following configurations control the flush of data to disk. -# There are a few important trade-offs here: -#1. Durability: Unflushed data may be lost if you are not using replication. -#2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush. -#3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to excessive seeks. -# The settings below allow one to configure the flush policy to flush data after a period of time or -# every N messages (or both). This can be done globally and overridden on a per-topic basis. - -# The number of messages to accept before forcing a flush of data to disk -#log.flush.interval.messages=10000 - -# The maximum amount of time a message can sit in a log before we force a flush -#log.flush.interval.ms=1000 - -############################# Log Retention Policy ############################# - -# The following configurations control the disposal of log segments. The policy can -# be set to delete segments after a period of time, or after a given size has accumulated. -# A segment will be deleted whenever *either* of these criteria are met. Deletion always happens -# from the end of the log. - -# https://cwiki.apache.org/confluence/display/KAFKA/KIP-186%3A+Increase+offsets+retention+default+to+7+days -offsets.retention.minutes=10080 - -# The minimum age of a log file to be eligible for deletion due to age -log.retention.hours=-1 - -# A size-based retention policy for logs. Segments are pruned from the log unless the remaining -# segments drop below log.retention.bytes. Functions independently of log.retention.hours. -#log.retention.bytes=1073741824 - -# The maximum size of a log segment file. When this size is reached a new log segment will be created. -#log.segment.bytes=1073741824 - -# The interval at which log segments are checked to see if they can be deleted according -# to the retention policies -#log.retention.check.interval.ms=300000 - -############################# Zookeeper ############################# - -# Zookeeper connection string (see zookeeper docs for details). -# This is a comma separated host:port pairs, each corresponding to a zk -# server. e.g. \"127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002\". -# You can also append an optional chroot string to the urls to specify the -# root directory for all kafka znodes. -zookeeper.connect=zookeeper:2181 - -# Timeout in ms for connecting to zookeeper -zookeeper.connection.timeout.ms=600000 -atlas.kafka.zookeeper.session.timeout.ms=600000 - -############################# Group Coordinator Settings ############################# - -# The following configuration specifies the time, in milliseconds, that the GroupCoordinator will delay the initial consumer rebalance. -# The rebalance will be further delayed by the value of group.initial.rebalance.delay.ms as new members join the group, up to a maximum of max.poll.interval.ms. -# The default value for this is 3 seconds. -# We override this to 0 here as it makes for a better out-of-the-box experience for development and testing. -# However, in production environments the default value of 3 seconds is more suitable as this will help to avoid unnecessary, and potentially expensive, rebalances during application startup. -#group.initial.rebalance.delay.ms=0" diff --git a/applications/events/deploy/resources/zookeeper/init.sh b/applications/events/deploy/resources/zookeeper/init.sh deleted file mode 100644 index eb9f567c8..000000000 --- a/applications/events/deploy/resources/zookeeper/init.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -set -e -set -x - -[ -d /var/lib/zookeeper/data ] || mkdir /var/lib/zookeeper/data -[ -z "$ID_OFFSET" ] && ID_OFFSET=1 -export ZOOKEEPER_SERVER_ID=$((${HOSTNAME##*-} + $ID_OFFSET)) -echo "${ZOOKEEPER_SERVER_ID:-1}" | tee /var/lib/zookeeper/data/myid -cp -Lur /etc/kafka-configmap/* /etc/kafka/ -[ ! -z "$PZOO_REPLICAS" ] && [ ! -z "$ZOO_REPLICAS" ] && { - sed -i "s/^server\\./#server./" /etc/kafka/zookeeper.properties - for N in $(seq $PZOO_REPLICAS); do echo "server.$N=pzoo-$(( $N - 1 )).pzoo:2888:3888:participant" >> /etc/kafka/zookeeper.properties; done - for N in $(seq $ZOO_REPLICAS); do echo "server.$(( $PZOO_REPLICAS + $N ))=zoo-$(( $N - 1 )).zoo:2888:3888:participant" >> /etc/kafka/zookeeper.properties; done -} -sed -i "s/server\.$ZOOKEEPER_SERVER_ID\=[a-z0-9.-]*/server.$ZOOKEEPER_SERVER_ID=0.0.0.0/" /etc/kafka/zookeeper.properties diff --git a/applications/events/deploy/resources/zookeeper/log4j.properties b/applications/events/deploy/resources/zookeeper/log4j.properties deleted file mode 100644 index 9f71fa756..000000000 --- a/applications/events/deploy/resources/zookeeper/log4j.properties +++ /dev/null @@ -1,8 +0,0 @@ -log4j.rootLogger=INFO, stdout -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n - -# Suppress connection log messages, three lines per livenessProbe execution -log4j.logger.org.apache.zookeeper.server.NIOServerCnxnFactory=WARN -log4j.logger.org.apache.zookeeper.server.NIOServerCnxn=WARN \ No newline at end of file diff --git a/applications/events/deploy/resources/zookeeper/zookeeper.properties b/applications/events/deploy/resources/zookeeper/zookeeper.properties deleted file mode 100644 index ee890a08a..000000000 --- a/applications/events/deploy/resources/zookeeper/zookeeper.properties +++ /dev/null @@ -1,12 +0,0 @@ -tickTime=2000 -dataDir=/var/lib/zookeeper/data -dataLogDir=/var/lib/zookeeper/log -clientPort=2181 -maxClientCnxns=100 -initLimit=5 -syncLimit=2 -server.1=pzoo-0.pzoo:2888:3888:participant -server.2=pzoo-1.pzoo:2888:3888:participant -server.3=pzoo-2.pzoo:2888:3888:participant -server.4=zoo-0.zoo:2888:3888:participant -server.5=zoo-1.zoo:2888:3888:participant \ No newline at end of file diff --git a/applications/events/deploy/templates-compose/deployments.yaml b/applications/events/deploy/templates-compose/deployments.yaml index 0001fde48..a984a108a 100644 --- a/applications/events/deploy/templates-compose/deployments.yaml +++ b/applications/events/deploy/templates-compose/deployments.yaml @@ -1,97 +1,29 @@ {{- define "events.deployment" }} events: networks: - - ch - image: solsson/kafka:2.3.0@sha256:b59603a8c0645f792fb54e9571500e975206352a021d6a116b110945ca6c3a1d + ch: + aliases: + - bootstrap + - broker + image: {{ .Values.apps.events.kafka.image }} ports: - - "9094:9092" + - "9094:{{ .Values.apps.events.kafka.port }}" expose: - - 5555 - - 9094 - - 9092 + - {{ .Values.apps.events.kafka.port }} + - {{ .Values.apps.events.kafka.controllerPort }} environment: - - CLASSPATH=/opt/kafka/libs/extensions/* - - KAFKA_LOG4J_OPTS=-Dlog4j.configuration=file:/etc/kafka/log4j.properties - - JMX_PORT=5555 - command: - - "./bin/kafka-server-start.sh" - - "/etc/kafka/server.properties" - - "--override" - - "default.replication.factor=1" - - "--override" - - "min.insync.replicas=1" - - "--override" - - "offsets.topic.replication.factor=1" - - "--override" - - "offsets.topic.num.partitions=1" - depends_on: - events-kafka-init: - condition: service_completed_successfully - -events-kafka-init: - networks: - - ch - image: solsson/kafka-initutils@sha256:f6d9850c6c3ad5ecc35e717308fddb47daffbde18eb93e98e031128fe8b899ef - command: - - "/bin/bash" - - "/etc/kafka-configmap/init.sh" - environment: - -pzoo: - networks: - - ch - expose: - - 2181 - - 2888 - - 3888 - image: solsson/kafka:2.3.0@sha256:b59603a8c0645f792fb54e9571500e975206352a021d6a116b110945ca6c3a1d - command: - - "./bin/zookeeper-server-start.sh" - - "/etc/kafka/zookeeper.properties" - environment: - - KAFKA_LOG4J_OPTS=-Dlog4j.configuration=file:/etc/kafka/log4j.properties - depends_on: - events-pzoo-init: - condition: service_completed_successfully - -events-pzoo-init: - networks: - - ch - expose: - image: solsson/kafka-initutils@sha256:f6d9850c6c3ad5ecc35e717308fddb47daffbde18eb93e98e031128fe8b899ef - command: - - "/bin/bash" - - "/etc/kafka-configmap/init.sh" - environment: - - PZOO_REPLICAS=1 - - ZOO_REPLICAS=0 - -zoo: - networks: - - ch - expose: - - 2181 - - 2888 - - 3888 - image: solsson/kafka:2.3.0@sha256:b59603a8c0645f792fb54e9571500e975206352a021d6a116b110945ca6c3a1d - command: - - "./bin/zookeeper-server-start.sh" - - "/etc/kafka/zookeeper.properties" - environment: - - KAFKA_LOG4J_OPTS=-Dlog4j.configuration=file:/etc/kafka/log4j.properties - depends_on: - events-zoo-init: - condition: service_completed_successfully - -events-zoo-init: - networks: - - ch - image: solsson/kafka-initutils@sha256:f6d9850c6c3ad5ecc35e717308fddb47daffbde18eb93e98e031128fe8b899ef - command: - - "/bin/bash" - - "/etc/kafka-configmap/init.sh" - environment: - - PZOO_REPLICAS=1 - - ZOO_REPLICAS=0 - - ID_OFFSET=2 -{{- end }} \ No newline at end of file + - ALLOW_PLAINTEXT_LISTENER=yes + - KAFKA_CLUSTER_ID={{ .Values.apps.events.kafka.clusterId }} + - KAFKA_CFG_NODE_ID=0 + - KAFKA_CFG_PROCESS_ROLES=controller,broker + - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER + - KAFKA_CFG_LISTENERS=PLAINTEXT://:{{ .Values.apps.events.kafka.port }},CONTROLLER://:{{ .Values.apps.events.kafka.controllerPort }} + - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://events:{{ .Values.apps.events.kafka.port }} + - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT + - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT + - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@broker:{{ .Values.apps.events.kafka.controllerPort }} + - KAFKA_CFG_LOG_DIRS=/bitnami/kafka/data + {{- range $key, $value := .Values.apps.events.kafka.config }} + - KAFKA_CFG_{{ $key | upper | replace "." "_" | replace "-" "_" }}={{ $value }} + {{- end }} +{{- end }} diff --git a/applications/events/deploy/templates/broker-config.yml b/applications/events/deploy/templates/broker-config.yml deleted file mode 100644 index e49bbda2e..000000000 --- a/applications/events/deploy/templates/broker-config.yml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: broker-config -data: -{{ (.Files.Glob "resources/events/broker/*").AsConfig | indent 2 }} - - - \ No newline at end of file diff --git a/applications/events/deploy/templates/deployments.yml b/applications/events/deploy/templates/deployments.yml index ed9da9ee3..b2f14f9b9 100644 --- a/applications/events/deploy/templates/deployments.yml +++ b/applications/events/deploy/templates/deployments.yml @@ -3,7 +3,6 @@ kind: StatefulSet metadata: name: kafka spec: - podManagementPolicy: Parallel replicas: 1 selector: matchLabels: @@ -11,311 +10,76 @@ spec: serviceName: broker template: metadata: - annotations: null labels: app: kafka spec: - serviceAccountName: kafka containers: - - command: - - ./bin/kafka-server-start.sh - - /etc/kafka/server.properties - - --override - - default.replication.factor=1 - - --override - - min.insync.replicas=1 - - --override - - offsets.topic.replication.factor=1 - - --override - - offsets.topic.num.partitions=1 + - name: broker + image: {{ .Values.apps.events.kafka.image | quote }} + imagePullPolicy: {{ .Values.apps.events.kafka.imagePullPolicy }} env: - - name: CLASSPATH - value: /opt/kafka/libs/extensions/* - - name: KAFKA_LOG4J_OPTS - value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties - - name: JMX_PORT - value: "5555" - image: solsson/kafka:2.3.0@sha256:b59603a8c0645f792fb54e9571500e975206352a021d6a116b110945ca6c3a1d - lifecycle: - preStop: - exec: - command: - - sh - - -ce - - kill -s TERM 1; while $(kill -0 1 2>/dev/null); do sleep 1; done - name: broker - ports: - - containerPort: 9092 - name: inside - - containerPort: 9094 - name: outside - - containerPort: 5555 - name: jmx - readinessProbe: - tcpSocket: - port: 9092 - timeoutSeconds: 2 - initialDelaySeconds: 30 - periodSeconds: 30 - resources: - {{- .Values.apps.events.kafka.resources | toYaml | nindent 10 }} - volumeMounts: - - mountPath: /etc/kafka - name: config - - mountPath: /var/lib/kafka/data - name: data - - mountPath: /opt/kafka/libs/extensions - name: extensions - initContainers: - - command: - - /bin/bash - - /etc/kafka-configmap/init.sh - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: solsson/kafka-initutils@sha256:f6d9850c6c3ad5ecc35e717308fddb47daffbde18eb93e98e031128fe8b899ef - name: init-config - volumeMounts: - - mountPath: /etc/kafka-configmap - name: configmap - - mountPath: /etc/kafka - name: config - - mountPath: /opt/kafka/libs/extensions - name: extensions - terminationGracePeriodSeconds: 30 - volumes: - - configMap: - name: broker-config - name: configmap - - emptyDir: {} - name: config - - emptyDir: {} - name: extensions - updateStrategy: - type: RollingUpdate - volumeClaimTemplates: - - metadata: - name: data - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: {{ .Values.apps.events.kafka.storage }} - storageClassName: standard ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: pzoo -spec: - podManagementPolicy: Parallel - replicas: 1 - selector: - matchLabels: - app: zookeeper - storage: persistent - serviceName: pzoo - template: - metadata: - annotations: null - labels: - app: zookeeper - storage: persistent - spec: - serviceAccountName: kafka - containers: - - command: - - ./bin/zookeeper-server-start.sh - - /etc/kafka/zookeeper.properties - env: - - name: KAFKA_LOG4J_OPTS - value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties - image: solsson/kafka:2.3.0@sha256:b59603a8c0645f792fb54e9571500e975206352a021d6a116b110945ca6c3a1d - lifecycle: - preStop: - exec: - command: - - sh - - -ce - - kill -s TERM 1; while $(kill -0 1 2>/dev/null); do sleep 1; done - name: zookeeper + - name: CLUSTER_ID + value: {{ .Values.apps.events.kafka.clusterId | quote }} + - name: KAFKA_NODE_ID + value: "1" + - name: KAFKA_PROCESS_ROLES + value: controller,broker + - name: KAFKA_CONTROLLER_LISTENER_NAMES + value: CONTROLLER + - name: KAFKA_LISTENERS + value: PLAINTEXT://:{{ .Values.apps.events.kafka.port }},CONTROLLER://:{{ .Values.apps.events.kafka.controllerPort }} + - name: KAFKA_ADVERTISED_LISTENERS + value: PLAINTEXT://kafka-0.broker.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.apps.events.kafka.port }} + - name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP + value: PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT + - name: KAFKA_INTER_BROKER_LISTENER_NAME + value: PLAINTEXT + - name: KAFKA_CONTROLLER_QUORUM_VOTERS + value: 1@kafka-0.broker.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.apps.events.kafka.controllerPort }} + - name: KAFKA_LOG_DIRS + value: /var/lib/kafka/data + - name: KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS + value: "0" + {{- range $key, $value := .Values.apps.events.kafka.config }} + - name: KAFKA_{{ $key | upper | replace "." "_" | replace "-" "_" }} + value: {{ $value | quote }} + {{- end }} ports: - - containerPort: 2181 + - containerPort: {{ .Values.apps.events.kafka.port }} name: client - - containerPort: 2888 - name: peer - - containerPort: 3888 - name: leader-election + - containerPort: {{ .Values.apps.events.kafka.controllerPort }} + name: controller + startupProbe: + tcpSocket: + port: client + failureThreshold: 30 + periodSeconds: 10 + timeoutSeconds: 5 readinessProbe: - exec: - command: - - /bin/sh - - -c - - '[ "imok" = "$(echo ruok | nc -w 1 -q 1 127.0.0.1 2181)" ]' - timeoutSeconds: 3 + tcpSocket: + port: client + initialDelaySeconds: 15 + periodSeconds: 15 + timeoutSeconds: 5 + livenessProbe: + tcpSocket: + port: client initialDelaySeconds: 30 periodSeconds: 30 - livenessProbe: - exec: - command: - - /bin/sh - - -c - - '[ "imok" = "$(echo ruok | nc -w 1 -q 1 127.0.0.1 2181)" ]' - timeoutSeconds: 3 - initialDelaySeconds: 12 - periodSeconds: 60 - resources: - {{- .Values.apps.events.pzoo.resources | toYaml | nindent 10 }} - volumeMounts: - - mountPath: /etc/kafka - name: config - - mountPath: /var/lib/zookeeper - name: data - initContainers: - - command: - - /bin/bash - - /etc/kafka-configmap/init.sh - env: - - name: PZOO_REPLICAS - value: "1" - - name: ZOO_REPLICAS - value: "0" - image: solsson/kafka-initutils@sha256:f6d9850c6c3ad5ecc35e717308fddb47daffbde18eb93e98e031128fe8b899ef - name: init-config - volumeMounts: - - mountPath: /etc/kafka-configmap - name: configmap - - mountPath: /etc/kafka - name: config - - mountPath: /var/lib/zookeeper - name: data - terminationGracePeriodSeconds: 10 - volumes: - - configMap: - name: zookeeper-config - name: configmap - - emptyDir: {} - name: config - updateStrategy: - type: RollingUpdate - volumeClaimTemplates: - - metadata: - name: data - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: {{ .Values.apps.events.pzoo.storage }} - storageClassName: standard ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: zoo -spec: - podManagementPolicy: Parallel - replicas: 0 - selector: - matchLabels: - app: zookeeper - storage: persistent-regional - serviceName: zoo - template: - metadata: - annotations: null - labels: - app: zookeeper - storage: persistent-regional - spec: - serviceAccountName: kafka - containers: - - command: - - ./bin/zookeeper-server-start.sh - - /etc/kafka/zookeeper.properties - env: - - name: KAFKA_LOG4J_OPTS - value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties - image: solsson/kafka:2.3.0@sha256:b59603a8c0645f792fb54e9571500e975206352a021d6a116b110945ca6c3a1d + timeoutSeconds: 5 lifecycle: preStop: exec: command: - - sh - - -ce - - kill -s TERM 1; while $(kill -0 1 2>/dev/null); do sleep 1; done - name: zookeeper - ports: - - containerPort: 2181 - name: client - - containerPort: 2888 - name: peer - - containerPort: 3888 - name: leader-election - readinessProbe: - exec: - command: - - /bin/sh - - -c - - '[ "imok" = "$(echo ruok | nc -w 1 -q 1 127.0.0.1 2181)" ]' - timeoutSeconds: 3 - initialDelaySeconds: 30 - periodSeconds: 30 - livenessProbe: - exec: - command: - - /bin/sh - - -c - - '[ "imok" = "$(echo ruok | nc -w 1 -q 1 127.0.0.1 2181)" ]' - timeoutSeconds: 3 - initialDelaySeconds: 12 - periodSeconds: 60 - + - /bin/sh + - -ec + - /opt/kafka/bin/kafka-server-stop.sh resources: - {{- .Values.apps.events.zoo.resources | toYaml | nindent 10 }} - volumeMounts: - - mountPath: /etc/kafka - name: config - - mountPath: /var/lib/zookeeper - name: data - initContainers: - - command: - - /bin/bash - - /etc/kafka-configmap/init.sh - env: - - name: PZOO_REPLICAS - value: "1" - - name: ZOO_REPLICAS - value: "0" - - name: ID_OFFSET - value: "2" - image: solsson/kafka-initutils@sha256:f6d9850c6c3ad5ecc35e717308fddb47daffbde18eb93e98e031128fe8b899ef - name: init-config + {{- .Values.apps.events.kafka.resources | toYaml | nindent 10 }} volumeMounts: - - mountPath: /etc/kafka-configmap - name: configmap - - mountPath: /etc/kafka - name: config - - mountPath: /var/lib/zookeeper + - mountPath: /var/lib/kafka/data name: data - terminationGracePeriodSeconds: 10 - volumes: - - configMap: - name: zookeeper-config - name: configmap - - emptyDir: {} - name: config + terminationGracePeriodSeconds: 30 updateStrategy: type: RollingUpdate volumeClaimTemplates: @@ -326,5 +90,5 @@ spec: - ReadWriteOnce resources: requests: - storage: {{ .Values.apps.events.zoo.storage }} - storageClassName: standard + storage: {{ .Values.apps.events.kafka.storage }} + storageClassName: {{ .Values.apps.events.kafka.storageClassName }} diff --git a/applications/events/deploy/templates/roles.yml b/applications/events/deploy/templates/roles.yml deleted file mode 100644 index 3d19bdd0f..000000000 --- a/applications/events/deploy/templates/roles.yml +++ /dev/null @@ -1,66 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: kafka - namespace: {{ .Release.Namespace }} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - origin: github.com_Yolean_kubernetes-kafka - name: pod-labler - -rules: -- apiGroups: - - "" - resources: - - pods - verbs: - - get - - update - - patch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ .Release.Name }}-node-reader - labels: - origin: github.com_Yolean_kubernetes-kafka -rules: -- apiGroups: - - "" - resources: - - nodes - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - origin: github.com_Yolean_kubernetes-kafka - name: kafka-pod-labler -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: pod-labler -subjects: -- kind: ServiceAccount - name: kafka - namespace: {{ .Release.Namespace }} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - origin: github.com_Yolean_kubernetes-kafka - name: {{ .Release.Name }}-kafka-node-reader -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{ .Release.Name }}-node-reader -subjects: -- kind: ServiceAccount - name: kafka - namespace: {{ .Release.Namespace }} \ No newline at end of file diff --git a/applications/events/deploy/templates/services.yml b/applications/events/deploy/templates/services.yml index b192a735f..a95d07b3f 100644 --- a/applications/events/deploy/templates/services.yml +++ b/applications/events/deploy/templates/services.yml @@ -4,7 +4,9 @@ metadata: name: {{ .Values.apps.events.kafka.name }} spec: ports: - - port: {{ .Values.apps.events.kafka.port }} + - name: client + port: {{ .Values.apps.events.kafka.port }} + targetPort: client selector: app: kafka --- @@ -14,48 +16,12 @@ metadata: name: broker spec: clusterIP: None - ports: - - port: {{ .Values.apps.events.kafka.port }} - selector: - app: kafka ---- -apiVersion: v1 -kind: Service -metadata: - name: pzoo -spec: - clusterIP: None - ports: - - name: peer - port: 2888 - - name: leader-election - port: 3888 - selector: - app: zookeeper - storage: persistent ---- -apiVersion: v1 -kind: Service -metadata: - name: zookeeper -spec: ports: - name: client - port: 2181 + port: {{ .Values.apps.events.kafka.port }} + targetPort: client + - name: controller + port: {{ .Values.apps.events.kafka.controllerPort }} + targetPort: controller selector: - app: zookeeper ---- -apiVersion: v1 -kind: Service -metadata: - name: zoo -spec: - clusterIP: None - ports: - - name: peer - port: 2888 - - name: leader-election - port: 3888 - selector: - app: zookeeper - storage: persistent-regional \ No newline at end of file + app: kafka diff --git a/applications/events/deploy/templates/zoo-config.yml b/applications/events/deploy/templates/zoo-config.yml deleted file mode 100644 index db948e5df..000000000 --- a/applications/events/deploy/templates/zoo-config.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: zookeeper-config -data: -{{ (.Files.Glob "resources/events/zookeeper/*").AsConfig | indent 2 }} \ No newline at end of file diff --git a/applications/events/deploy/values.yaml b/applications/events/deploy/values.yaml index 071e423c0..d738ab75f 100644 --- a/applications/events/deploy/values.yaml +++ b/applications/events/deploy/values.yaml @@ -9,13 +9,23 @@ harness: test: e2e: enabled: true - env: - - name: ZK_HOSTS - value: zookeeper:2181 kafka: name: bootstrap port: 9092 + controllerPort: 9093 + image: docker.io/apache/kafka:4.0.2 + imagePullPolicy: IfNotPresent + clusterId: 5L6g3nShT-eMCtK--X86sw storage: 10Gi + storageClassName: standard + config: + auto.create.topics.enable: "true" + num.partitions: "3" + default.replication.factor: "1" + min.insync.replicas: "1" + offsets.topic.replication.factor: "1" + transaction.state.log.replication.factor: "1" + transaction.state.log.min.isr: "1" resources: limits: cpu: 500m @@ -23,19 +33,3 @@ kafka: requests: cpu: 50m memory: 100Mi -pzoo: - storage: 1Gi - resources: - limits: - memory: 500Mi - requests: - cpu: 10m - memory: 100Mi -zoo: - storage: 1Gi - resources: - limits: - memory: 500Mi - requests: - cpu: 10m - memory: 100Mi diff --git a/docs/events.md b/docs/events.md index 28f16fbf2..924d48b65 100644 --- a/docs/events.md +++ b/docs/events.md @@ -1,27 +1,49 @@ # Events -The events application is created upon a Kafka StatefulSet. +The events application runs a Kafka `StatefulSet` in KRaft mode. -## Access the Kafka manager +## Access Kafka UI -The [Kafka Manager](https://github.com/yahoo/CMAK) is preconfigured and accessible at the address https://events.MYDOMAIN. +The [Kafka UI](https://github.com/provectus/kafka-ui) application is preconfigured and accessible at the address `https://events.MYDOMAIN`. ## Configure the Kafka parameters -Override the [broker server configuration file](../applications/events/deploy/resources/broker/server.properties) to change most of the relevant configurations. +Override the `kafka` values in [applications/events/deploy/values.yaml](../applications/events/deploy/values.yaml) to change the relevant broker parameters. + +The `kafka.config` map is converted into Kafka `server.properties` settings automatically. For example: + +```yaml +kafka: + config: + num.partitions: "6" + auto.create.topics.enable: "false" +``` + +The default override set is intentionally small and only keeps the single-node KRaft deployment behavior stable. + +## Reset local KRaft state + +Kafka metadata and log state are stored on the `kafka` PVC. If a local Minikube deployment gets stuck because of stale KRaft state, delete the broker pod and its PVC before redeploying: + +```bash +kubectl delete pod -n test kafka-0 +kubectl delete pvc -n test data-kafka-0 +``` + +Then run `harness-deployment` again to recreate the broker with a fresh volume. ## Locally test Kafka queue calls The following allows to call/test to Kafka locally. It is useful to test and debug an application which listens/writes to the queue -Kafka broker to local 9092 +Kafka broker to local `9092` ``` -kubectl port-forward --namespace mnp $(kubectl get po -n mnp | grep kafka-0 | \awk '{print $1;}') 9092:9092 +kubectl port-forward --namespace mnp svc/bootstrap 9092:9092 ``` Also add to your hosts file ``` -127.0.0.1 kafka-0.broker.mnp.svc.cluster.local bootstrap.mnp.svc.cluster.local +127.0.0.1 bootstrap.mnp.svc.cluster.local kafka-0.broker.mnp.svc.cluster.local ``` ## Backend library @@ -78,4 +100,4 @@ from cloudharness.workflows.utils import notify_queue my_message = {"a": "b"} notify_queue("my-topic", my_message) -``` \ No newline at end of file +``` From 8acc7f003ff0758cff768f6b633d6cb0fe83b985 Mon Sep 17 00:00:00 2001 From: Alex Burdusel Date: Tue, 31 Mar 2026 15:01:27 +0300 Subject: [PATCH 2/3] Increase Kafka memory limit to avoid OOM restarts. --- applications/events/deploy/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/applications/events/deploy/values.yaml b/applications/events/deploy/values.yaml index d738ab75f..740a71fa8 100644 --- a/applications/events/deploy/values.yaml +++ b/applications/events/deploy/values.yaml @@ -29,7 +29,7 @@ kafka: resources: limits: cpu: 500m - memory: 600Mi + memory: 1Gi requests: cpu: 50m memory: 100Mi From 04d660ca88cc9d369e4ed3d0fbf03a17e1bdee7f Mon Sep 17 00:00:00 2001 From: Alex Burdusel Date: Thu, 2 Apr 2026 16:39:21 +0300 Subject: [PATCH 3/3] Remove redundant template file from events config --- .../deploy/templates-compose/deployments.yaml | 29 ------------------- 1 file changed, 29 deletions(-) delete mode 100644 applications/events/deploy/templates-compose/deployments.yaml diff --git a/applications/events/deploy/templates-compose/deployments.yaml b/applications/events/deploy/templates-compose/deployments.yaml deleted file mode 100644 index a984a108a..000000000 --- a/applications/events/deploy/templates-compose/deployments.yaml +++ /dev/null @@ -1,29 +0,0 @@ -{{- define "events.deployment" }} -events: - networks: - ch: - aliases: - - bootstrap - - broker - image: {{ .Values.apps.events.kafka.image }} - ports: - - "9094:{{ .Values.apps.events.kafka.port }}" - expose: - - {{ .Values.apps.events.kafka.port }} - - {{ .Values.apps.events.kafka.controllerPort }} - environment: - - ALLOW_PLAINTEXT_LISTENER=yes - - KAFKA_CLUSTER_ID={{ .Values.apps.events.kafka.clusterId }} - - KAFKA_CFG_NODE_ID=0 - - KAFKA_CFG_PROCESS_ROLES=controller,broker - - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER - - KAFKA_CFG_LISTENERS=PLAINTEXT://:{{ .Values.apps.events.kafka.port }},CONTROLLER://:{{ .Values.apps.events.kafka.controllerPort }} - - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://events:{{ .Values.apps.events.kafka.port }} - - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT - - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT - - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@broker:{{ .Values.apps.events.kafka.controllerPort }} - - KAFKA_CFG_LOG_DIRS=/bitnami/kafka/data - {{- range $key, $value := .Values.apps.events.kafka.config }} - - KAFKA_CFG_{{ $key | upper | replace "." "_" | replace "-" "_" }}={{ $value }} - {{- end }} -{{- end }}