[Kafka] Source connect API 민감 정보 숨기기!

 

https://docs.confluent.io/platform/7.1/connect/security.html

Kafka Connect Security Basics | Confluent Documentation

 

민감 정보를 파일로 생성하여, REST API를 통한 connector 생성시 참조하도록 구성한다.

Helm, Kustomize를 통해 이미지 빌드 없이 외부 주입으로 설정 파일을 덮어쓴다.

 

Dockerfile

FROM openjdk:17.0.1-jdk-slim AS builder

# 기본 작업 디렉토리 설정
WORKDIR /confluent

# 필수 도구 설치
RUN apt-get update && \
    apt-get install -y curl git vim unzip zip

# Kafka Connect 설치
RUN curl -O http://packages.confluent.io/archive/7.0/confluent-community-7.0.1.tar.gz && \
    tar -zxvf confluent-community-7.0.1.tar.gz

# Confluent Hub 설치
RUN mkdir -p /confluent/confluent-hub/plugins
RUN curl -fL -o /confluent/confluent-hub/confluent-hub-client-latest.tar.gz http://client.hub.confluent.io/confluent-hub-client-latest.tar.gz
RUN tar -zxvf /confluent/confluent-hub/confluent-hub-client-latest.tar.gz -C /confluent/confluent-hub

# JMX Exporter 설치
RUN mkdir -p /opt/jmx && \
    curl -fL -o /opt/jmx/jmx_prometheus_javaagent.jar \
    https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.20.0/jmx_prometheus_javaagent-0.20.0.jar

# SMT 커넥터
COPY *.jar /confluent/confluent-hub/plugins/
COPY connect-distributed.properties /confluent/confluent-7.0.1/etc/kafka
COPY jmx-exporter.yaml /opt/jmx/

# debezium mysql connector
RUN yes | confluent-hub/bin/confluent-hub install debezium/debezium-connector-mysql:1.7.0 --component-dir /confluent/confluent-hub/plugins --worker-configs /confluent/confluent-7.0.1/etc/kafka/connect-distributed.properties

# File provider
COPY security.properties .


# 이미지 사이즈 축소 : 불필요한 파일 정리
RUN rm -rf kafka-env-var-config-provider-1.1.0.tar.gz kafka-env-var-config-provider-1.1.0 confluent-community-7.0.1.tar.gz
RUN rm -rf confluent-hub/confluent-hub-client-latest.tar.gz confluent-hub/share/doc
RUN rm -rf confluent-7.0.1/README confluent-7.0.1/share/doc confluent-7.0.1/share/java/ksqldb
RUN rm -rf confluent-7.0.1/bin/ksqldb confluent-7.0.1/bin/rest-utils confluent-7.0.1/bin/ksql*

# 환경 변수 설정
ENV CONFLUENT_HOME=/confluent/confluent-hub
ENV PATH=$PATH:$CONFLUENT_HOME/bin

# 기본 명령어
CMD ["/bin/sh" , "-c" ,"/confluent/confluent-7.0.1/bin/connect-distributed /confluent/confluent-7.0.1/etc/kafka/connect-distributed.properties"]

적용 전

 

적용 후

 

security.properties

KAFKA_BOOTSTRAP_SERVERS=10.105.90.26:9092,10.105.90.27:9092,10.105.90.33:9092
DEV_DATABASE_HOSTNAME=cdb-krs.gov-ntruss.com
DEV_DATABASE_PORT=3306
DEV_DATABASE_USER=유저명
DEV_DATABASE_PASSWORD=비밀번호
STG_DATABASE_HOSTNAME=cdb-krs.gov-ntruss.com
STG_DATABASE_PORT=3306
STG_DATABASE_USER=유저명
STG_DATABASE_PASSWORD=비밀번호
STG1_DATABASE_HOSTNAME=cdb-krs.gov-ntruss.com
STG1_DATABASE_PORT=3306
STG1_DATABASE_USER=유저명
STG1_DATABASE_PASSWORD=비밀번호
RELEASE_MATH_DATABASE_HOSTNAME=cdb-krs.gov-ntruss.com
RELEASE_MATH_DATABASE_PORT=3306
RELEASE_MATH_DATABASE_USER=유저명
RELEASE_MATH_DATABASE_PASSWORD=비밀번호
RELEASE_ENGL_DATABASE_HOSTNAME=cdb-krs.gov-ntruss.com
RELEASE_ENGL_DATABASE_PORT=3306
RELEASE_ENGL_DATABASE_USER=유저명
RELEASE_ENGL_DATABASE_PASSWORD=비밀번호

 

deployment-patch.yaml

- op: replace
  path: /spec/template/spec/containers
  value:
      volumeMounts:
        - mountPath: /confluent/security.properties
          name: security-config
          subPath: security.properties     
          
- op: add
  path: /spec/template/spec/volumes
  value:
    - name: security-config
      configMap:
        name: security-config

 

kustomization.yaml

configMapGenerator:
  - name: security-config
    files:
      - security.properties=./security.properties

 

Connector REST API

${file:/confluent/security.properties:참조할 필드}

curl -X POST https://connect.restapi.com/connectors \
  -H "Content-Type: application/json" \
  -d '{
  "name": "connector",
  "config": {
    "connector.class": "io.debezium.connector.mysql.MySqlConnector",
    "producer.override.max.request.size": 5242880,
    "database.hostname": "${file:/confluent/security.properties:DEV_DATABASE_HOSTNAME}",
    "database.port":     "3306",
    "database.user":     "${file:/confluent/security.properties:DEV_DATABASE_USER}",
    "database.password": "${file:/confluent/security.properties:DEV_DATABASE_PASSWORD}",
    "database.server.id": "3318378",
    "database.server.name": "cdc",
    "database.history.store.only.captured.tables.ddl": "true",
    "table.include.list": "a,b,c",    
    "column.exclude.list": "d,e,f",
    "ddl.filter": "CREATE VIEW",
    "database.history.kafka.bootstrap.servers": "${file:/confluent/security.properties:KAFKA_BOOTSTRAP_SERVERS}",
    "database.history.kafka.topic": "HISTORY",
    "snapshot.mode": "initial",    
    "snapshot.locking.mode": "none",
    "value.converter": "org.apache.kafka.connect.json.JsonConverter",
    "key.converter": "org.apache.kafka.connect.json.JsonConverter",
    "value.converter.schemas.enable": false,
    "key.converter.schemas.enable": false,
    "topic.creation.enable": true,
    "topic.creation.default.partitions": 4,
    "topic.creation.default.replication.factor": 1,
    "database.history.skip.unparseable.ddl": "true",
    "internal.key.converter.schemas.enable": "false",
    "internal.value.converter.schemas.enable": "false"
  }
}'