confluentinc/cp-kafka, zookeeper, kafka-ui

예상과 달랐던 사내 교육의 방향성에 조금 놀란가슴을 진정시키는 중이다

 

교육에서 제공받은 주키퍼와 카프카 클러스터링 하고, 카프카 UI 환경 구성하는 docker-compose를 k8s 용으로 바꿔봤다.

apiVersion: apps/v1
kind: Deployment
metadata:
  name: zookeeper1
  namespace: default
spec:
  progressDeadlineSeconds: 600
  replicas: 1
  revisionHistoryLimit: 1
  selector:
    matchLabels:
      app: zookeeper1
  strategy:
    rollingUpdate:
      maxSurge: 25%
      maxUnavailable: 25%
    type: RollingUpdate
  template:
    metadata:
      creationTimestamp: null
      labels:
        app: zookeeper1
    spec:
      containers:
      - env:
        - name: ZOOKEEPER_SERVER_ID
          value: "1"
        - name: ZOOKEEPER_SERVERS
          value: 0.0.0.0:2888:3888;zookeeper2:2888:3888;zookeeper3:2888:3888
        - name: ZOOKEEPER_CLIENT_PORT
          value: "2181"
        envFrom:
        - configMapRef:
            name: kafka-cm
        image: confluentinc/cp-zookeeper
        imagePullPolicy: IfNotPresent
        name: zookeeper1
        ports:
        - containerPort: 2181
          protocol: TCP
        - containerPort: 2888
          protocol: TCP
        - containerPort: 3888
          protocol: TCP
        resources: {}
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: File
      dnsPolicy: ClusterFirst
      restartPolicy: Always
      schedulerName: default-scheduler
      securityContext: {}
      terminationGracePeriodSeconds: 30
      
---
 
apiVersion: apps/v1
kind: Deployment
metadata:
  name: zookeeper2
  namespace: default
spec:
  progressDeadlineSeconds: 600
  replicas: 1
  revisionHistoryLimit: 1
  selector:
    matchLabels:
      app: zookeeper2
  strategy:
    rollingUpdate:
      maxSurge: 25%
      maxUnavailable: 25%
    type: RollingUpdate
  template:
    metadata:
      creationTimestamp: null
      labels:
        app: zookeeper2
    spec:
      containers:
      - env:
        - name: ZOOKEEPER_SERVER_ID
          value: "2"
        - name: ZOOKEEPER_SERVERS
          value: zookeeper1:2888:3888;0.0.0.0:2888:3888;zookeeper3:2888:3888
        - name: ZOOKEEPER_CLIENT_PORT
          value: "2182"
        envFrom:
        - configMapRef:
            name: kafka-cm
        image: confluentinc/cp-zookeeper
        imagePullPolicy: IfNotPresent
        name: zookeeper2
        ports:
        - containerPort: 2182
          protocol: TCP
        - containerPort: 2888
          protocol: TCP
        - containerPort: 3888
          protocol: TCP
        resources: {}
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: File
      dnsPolicy: ClusterFirst
      restartPolicy: Always
      schedulerName: default-scheduler
      securityContext: {}
      terminationGracePeriodSeconds: 30
      
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: zookeeper3
  namespace: default
spec:
  progressDeadlineSeconds: 600
  replicas: 1
  revisionHistoryLimit: 1
  selector:
    matchLabels:
      app: zookeeper3
  strategy:
    rollingUpdate:
      maxSurge: 25%
      maxUnavailable: 25%
    type: RollingUpdate
  template:
    metadata:
      creationTimestamp: null
      labels:
        app: zookeeper3
    spec:
      containers:
      - env:
        - name: ZOOKEEPER_SERVER_ID
          value: "3"
        - name: ZOOKEEPER_SERVERS
          value: zookeeper1:2888:3888;zookeeper2:2888:3888;0.0.0.0:2888:3888
        - name: ZOOKEEPER_CLIENT_PORT
          value: "2183"
        envFrom:
        - configMapRef:
            name: kafka-cm
        image: confluentinc/cp-zookeeper
        imagePullPolicy: IfNotPresent
        name: zookeeper3
        ports:
        - containerPort: 2183
          protocol: TCP
        - containerPort: 2888
          protocol: TCP
        - containerPort: 3888
          protocol: TCP
        resources: {}
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: File
      dnsPolicy: ClusterFirst
      restartPolicy: Always
      schedulerName: default-scheduler
      securityContext: {}
      terminationGracePeriodSeconds: 30
      
---
apiVersion: v1
data:
  KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
  KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka1:9092,kafka2:9093,kafka3:9094
  KAFKA_CLUSTERS_0_NAME: test-cluster
  KAFKA_CLUSTERS_0_ZOOKEEPER: zookeeper1:2181,zookeeper2:2182,zookeeper3:2183
  KAFKA_DEFAULT_REPLICATION_FACTOR: "3"
  KAFKA_DELETE_TOPIC_ENABLE: "true"
  KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
  KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT
  KAFKA_MIN_INSYNC_REPLICAS: "2"
  KAFKA_NUM_PARTITIONS: "1"
  KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: "3"
  KAFKA_UI_AUTH_ENABLED: "false"
  KAFKA_ZOOKEEPER_CONNECT: zookeeper1:2181,zookeeper2:2182,zookeeper3:2183
  ZOOKEEPER_INIT_LIMIT: "5"
  ZOOKEEPER_SYNC_LIMIT: "2"
  ZOOKEEPER_TICK_TIME: "2000"
kind: ConfigMap
metadata:
  name: kafka-cm
  namespace: default
  
---
apiVersion: v1
kind: Service
metadata:
  labels:
    app: zookeeper1
  name: zookeeper1
  namespace: default
spec:
  internalTrafficPolicy: Cluster
  ipFamilies:
  - IPv4
  ipFamilyPolicy: SingleStack
  ports:
  - name: tcp-clients
    port: 2181
    protocol: TCP
    targetPort: 2181
  - name: replication
    port: 2888
    protocol: TCP
    targetPort: 2888
  - name: election
    port: 3888
    protocol: TCP
    targetPort: 3888
  selector:
    app: zookeeper1
  sessionAffinity: None
  type: ClusterIP
  
---
apiVersion: v1
kind: Service
metadata:
  labels:
    app: zookeeper2
  name: zookeeper2
  namespace: default
spec:
  internalTrafficPolicy: Cluster
  ipFamilies:
  - IPv4
  ipFamilyPolicy: SingleStack
  ports:
  - name: tcp-clients
    port: 2182
    protocol: TCP
    targetPort: 2181
  - name: replication
    port: 2888
    protocol: TCP
    targetPort: 2888
  - name: election
    port: 3888
    protocol: TCP
    targetPort: 3888
  selector:
    app: zookeeper2
  sessionAffinity: None
  type: ClusterIP

---
apiVersion: v1
kind: Service
metadata:
  labels:
    app: zookeeper3
  name: zookeeper3
  namespace: default
spec:
  internalTrafficPolicy: Cluster
  ipFamilies:
  - IPv4
  ipFamilyPolicy: SingleStack
  ports:
  - name: tcp-clients
    port: 2183
    protocol: TCP
    targetPort: 2183
  - name: replication
    port: 2888
    protocol: TCP
    targetPort: 2888
  - name: election
    port: 3888
    protocol: TCP
    targetPort: 3888
  selector:
    app: zookeeper3
  sessionAffinity: None
  type: ClusterIP
  
---
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: kafka1
  name: kafka1
  namespace: default
spec:
  progressDeadlineSeconds: 600
  replicas: 1
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      app: kafka1
  strategy:
    rollingUpdate:
      maxSurge: 25%
      maxUnavailable: 25%
    type: RollingUpdate
  template:
    metadata:
      creationTimestamp: null
      labels:
        app: kafka1
    spec:
      containers:
      - env:
        - name: KAFKA_ADVERTISED_LISTENERS
          value: INTERNAL://kafka1:9092
        - name: KAFKA_BROKER_ID
          value: "1"
        envFrom:
        - configMapRef:
            name: kafka-cm
        image: confluentinc/cp-kafka:latest
        imagePullPolicy: Always
        name: kafka1
        ports:
        - containerPort: 9092
          protocol: TCP
        resources: {}
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: File
      dnsPolicy: ClusterFirst
      restartPolicy: Always
      schedulerName: default-scheduler
      securityContext: {}
      terminationGracePeriodSeconds: 30

---

apiVersion: v1
kind: Service
metadata:
  labels:
    app: kafka1
  name: kafka1
  namespace: default
spec:
  internalTrafficPolicy: Cluster
  ipFamilies:
  - IPv4
  ipFamilyPolicy: SingleStack
  ports:
  - port: 9092
    protocol: TCP
    targetPort: 9092
  selector:
    app: kafka1
  sessionAffinity: None
  type: ClusterIP
  
---
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: kafka2
  name: kafka2
  namespace: default
spec:
  progressDeadlineSeconds: 600
  replicas: 1
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      app: kafka2
  strategy:
    rollingUpdate:
      maxSurge: 25%
      maxUnavailable: 25%
    type: RollingUpdate
  template:
    metadata:
      creationTimestamp: null
      labels:
        app: kafka2
    spec:
      containers:
      - env:
        - name: KAFKA_ADVERTISED_LISTENERS
          value: INTERNAL://kafka2:9093
        - name: KAFKA_BROKER_ID
          value: "2"
        envFrom:
        - configMapRef:
            name: kafka-cm
        image: confluentinc/cp-kafka:latest
        imagePullPolicy: Always
        name: kafka2
        ports:
        - containerPort: 9093
          protocol: TCP
        resources: {}
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: File
      dnsPolicy: ClusterFirst
      restartPolicy: Always
      schedulerName: default-scheduler
      securityContext: {}
      terminationGracePeriodSeconds: 30

---

apiVersion: v1
kind: Service
metadata:
  labels:
    app: kafka2
  name: kafka2
  namespace: default
spec:
  internalTrafficPolicy: Cluster
  ipFamilies:
  - IPv4
  ipFamilyPolicy: SingleStack
  ports:
  - port: 9093
    protocol: TCP
    targetPort: 9093
  selector:
    app: kafka2
  sessionAffinity: None
  type: ClusterIP
  
---
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: kafka3
  name: kafka3
  namespace: default
spec:
  progressDeadlineSeconds: 600
  replicas: 1
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      app: kafka3
  strategy:
    rollingUpdate:
      maxSurge: 25%
      maxUnavailable: 25%
    type: RollingUpdate
  template:
    metadata:
      creationTimestamp: null
      labels:
        app: kafka3
    spec:
      containers:
      - env:
        - name: KAFKA_ADVERTISED_LISTENERS
          value: INTERNAL://kafka3:9094
        - name: KAFKA_BROKER_ID
          value: "3"
        envFrom:
        - configMapRef:
            name: kafka-cm
        image: confluentinc/cp-kafka:latest
        imagePullPolicy: Always
        name: kafka3
        ports:
        - containerPort: 9094
          protocol: TCP
        resources: {}
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: File
      dnsPolicy: ClusterFirst
      restartPolicy: Always
      schedulerName: default-scheduler
      securityContext: {}
      terminationGracePeriodSeconds: 30

---

apiVersion: v1
kind: Service
metadata:
  labels:
    app: kafka3
  name: kafka3
  namespace: default
spec:
  internalTrafficPolicy: Cluster
  ipFamilies:
  - IPv4
  ipFamilyPolicy: SingleStack
  ports:
  - port: 9094
    protocol: TCP
    targetPort: 9094
  selector:
    app: kafka3
  sessionAffinity: None
  type: ClusterIP
  
---
apiVersion: apps/v1
kind: Deployment
metadata:
  creationTimestamp: null
  labels:
    app: kafka-ui
  name: kafka-ui
spec:
  replicas: 1
  selector:
    matchLabels:
      app: kafka-ui
  strategy: {}
  template:
    metadata:
      creationTimestamp: null
      labels:
        app: kafka-ui
    spec:
      containers:
      - image: provectuslabs/kafka-ui:latest
        name: kafka-ui
        ports:
        - containerPort: 8080
        - containerPort: 8081
        envFrom:
        - configMapRef:
            name: kafka-cm
        resources: {}

---
apiVersion: v1
kind: Service
metadata:
  labels:
    app: kafka-ui
  name: kafka-ui
  namespace: default
spec:
  externalTrafficPolicy: Cluster
  internalTrafficPolicy: Cluster
  ipFamilies:
  - IPv4
  ipFamilyPolicy: SingleStack
  ports:
  - name: web
    nodePort: 32051
    port: 8080
    protocol: TCP
    targetPort: 8080
  - name: web2
    nodePort: 31119
    port: 8081
    protocol: TCP
    targetPort: 8081
  selector:
    app: kafka-ui
  sessionAffinity: None
  type: NodePort

 

노출시킨 노드포트로 접근해보자

 

KRaft 는 여기를 참고한다

https://fullmooney.tistory.com/77

confluentinc/cp-kafka KRaft yaml (zookeeper out)