I'm currently trying to implement a debezium connector. I've used JDBC connector as introduction but I need log-based CDC. I'm using a docker-compose.yml file to establish all configuration and the database is based on Oracle.
I'm stuck because I'm getting this error "ERROR io.confluent.admin.utils.ClusterStatus - Expected 1 brokers but found only 0. Brokers found []."
Here is my docker-compose file:
Is there a simpler way to build this docker-compose?
version: '2'
services:
zookeeper:
image: debezium/zookeeper:1.4
hostname: zookeeper
container_name: zookeeper
volumes:
- /dados/persistence/zookeeper/data:/var/lib/zookeeper/data
- /dados/persistence/zookeeper/log:/var/lib/zookeeper/log
ports:
- "2181:2181"
- "2888:2888"
- "3888:3888"
broker:
image: debezium/kafka:1.4
hostname: broker
container_name: broker
volumes:
- /dados/persistence/broker/data:/var/lib/kafka/data
depends_on:
- zookeeper
ports:
- "9092:9092"
- "9101:9101"
environment:
BROKER_ID: 1
ZOOKEEPER_CONNECT: 'zookeeper:2181'
listener.security.protocol.map: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
advertised.listeners: PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092
metric.reports: io.confluent.metrics.reporter.ConfluentMetricsReporter
offset.topic.replication.factor: 1
group.initial.rebalance.delay.ms: 0
confluent.license.topic.replication.factor: 1
confluent.balancer.topic.replication.factor: 1
transaction.state.log.min.isr: 1
transaction.state.log.replication.factor: 1
jmx.port: 9101
jmx.hostname: localhost
confluent.schema.registry.url: http://schema-registry:8081
CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: broker:29092
CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1
CONFLUENT_METRICS_ENABLE: 'true'
CONFLUENT_SUPPORT_CUSTOMER_ID: 'anonymous'
schema-registry:
image: confluentinc/cp-schema-registry:6.0.1
hostname: schema-registry
container_name: schema-registry
depends_on:
- broker
ports:
- "8081:8081"
environment:
SCHEMA_REGISTRY_HOST_NAME: schema-registry
SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'broker:29092'
SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081
connect:
image: debezium/connect:1.4
hostname: connect
container_name: connect
volumes:
- /dados/packages/confluent-hub/dados/packages/confluent-hub/share/confluent-hub-components:/usr/share/confluent-hub-components/custom
- /dados/persistence/kafka-connect/jars:/etc/kafka-connect/jars
depends_on:
- broker
- schema-registry
ports:
- "8083:8083"
environment:
bootstrap.servers: 'broker:29092'
advertised.host.name: connect
rest.port: 8083
group.id: compose-connect-group
CONFIG_STORAGE_TOPIC: docker-connect-configs
config.storage.replication.factor: 1
offset.fulsh.interval.ms: 10000
OFFSET_STORAGE_TOPIC: docker-connect-offsets
offset.storage.replication.factor: 1
status.storage.topic: docker-connect-status
status.storage.replication.factor: 1
key.converter: org.apache.kafka.connect.storage.StringConverter
value.converter: io.confluent.connect.avro.AvroConverter
value.converter.schema.registry.url: http://schema-registry:8081
internal.key.converter: "org.apache.kafka.connect.json.JsonConverter"
internal.value.converter: "org.apache.kafka.connect.json.JsonConverter"
# CLASSPATH required due to CC-2422
CLASSPATH: /usr/share/java/monitoring-interceptors/monitoring-interceptors-6.0.1.jar
producer.interceptor.classes: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor"
consumer.interceptor.classes: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor"
plugin_path: "/usr/share/java,/usr/share/confluent-hub-components,/usr/share/confluent-hub-components/custom"
log4j.loggers: org.apache.zookeeper=ERROR,org.I0Itec.zkclient=ERROR,org.reflections=ERROR
control-center:
image: confluentinc/cp-enterprise-control-center:6.0.1
hostname: control-center
container_name: control-center
depends_on:
- broker
- schema-registry
- connect
- ksqldb-server
ports:
- "9021:9021"
environment:
CONTROL_CENTER_BOOTSTRAP_SERVERS: 'broker:29092'
CONTROL_CENTER_CONNECT_CLUSTER: 'connect:8083'
CONTROL_CENTER_KSQL_KSQLDB1_URL: "http://10.58.0.207:8088"
CONTROL_CENTER_KSQL_KSQLDB1_ADVERTISED_URL: "http://10.58.0.207:8088"
CONTROL_CENTER_SCHEMA_REGISTRY_URL: "http://10.58.0.207:8081"
CONTROL_CENTER_REPLICATION_FACTOR: 1
CONTROL_CENTER_INTERNAL_TOPICS_PARTITIONS: 1
CONTROL_CENTER_MONITORING_INTERCEPTOR_TOPIC_PARTITIONS: 1
CONFLUENT_METRICS_TOPIC_REPLICATION: 1
PORT: 9021
ksqldb-server:
image: confluentinc/cp-ksqldb-server:6.0.1
hostname: ksqldb-server
container_name: ksqldb-server
depends_on:
- broker
- connect
ports:
- "8088:8088"
environment:
KSQL_CONFIG_DIR: "/etc/ksql"
KSQL_BOOTSTRAP_SERVERS: "broker:29092"
KSQL_HOST_NAME: ksqldb-server
KSQL_LISTENERS: "http://0.0.0.0:8088"
KSQL_CACHE_MAX_BYTES_BUFFERING: 0
KSQL_KSQL_SCHEMA_REGISTRY_URL: "http://schema-registry:8081"
KSQL_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor"
KSQL_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor"
KSQL_KSQL_CONNECT_URL: "http://connect:8083"
KSQL_KSQL_LOGGING_PROCESSING_TOPIC_REPLICATION_FACTOR: 1
KSQL_KSQL_LOGGING_PROCESSING_TOPIC_AUTO_CREATE: 'true'
KSQL_KSQL_LOGGING_PROCESSING_STREAM_AUTO_CREATE: 'true'
ksqldb-cli:
image: confluentinc/cp-ksqldb-cli:6.0.1
container_name: ksqldb-cli
depends_on:
- broker
- connect
- ksqldb-server
entrypoint: /bin/sh
tty: true
ksql-datagen:
image: confluentinc/ksqldb-examples:6.0.1
hostname: ksql-datagen
container_name: ksql-datagen
depends_on:
- ksqldb-server
- broker
- schema-registry
- connect
command: "bash -c 'echo Waiting for Kafka to be ready... && \
cub kafka-ready -b broker:29092 1 40 && \
echo Waiting for Confluent Schema Registry to be ready... && \
cub sr-ready schema-registry 8081 40 && \
echo Waiting a few seconds for topic creation to finish... && \
sleep 11 && \
tail -f /dev/null'"
environment:
KSQL_CONFIG_DIR: "/etc/ksql"
STREAMS_BOOTSTRAP_SERVERS: broker:29092
STREAMS_SCHEMA_REGISTRY_HOST: schema-registry
STREAMS_SCHEMA_REGISTRY_PORT: 8081
rest-proxy:
image: confluentinc/cp-kafka-rest:6.0.1
depends_on:
- broker
- schema-registry
ports:
- 8082:8082
hostname: rest-proxy
container_name: rest-proxy
environment:
KAFKA_REST_HOST_NAME: rest-proxy
KAFKA_REST_BOOTSTRAP_SERVERS: 'broker:29092'
KAFKA_REST_LISTENERS: "http://0.0.0.0:8082"
KAFKA_REST_SCHEMA_REGISTRY_URL: 'http://schema-registry:8081'
Here's a working Docker Compose which uses the Debezium Oracle connector.
---
version: '2'
services:
zookeeper:
image: confluentinc/cp-zookeeper:5.5.0
container_name: zookeeper
environment:
ZOOKEEPER_CLIENT_PORT: 2181
ZOOKEEPER_TICK_TIME: 2000
kafka:
image: confluentinc/cp-kafka:5.5.0
container_name: kafka
depends_on:
- zookeeper
ports:
# "`-._,-'"`-._,-'"`-._,-'"`-._,-'"`-._,-'"`-._,-'"`-._,-'"`-._,-'"`-._,-
# An important note about accessing Kafka from clients on other machines:
# -----------------------------------------------------------------------
#
# The config used here exposes port 9092 for _external_ connections to the broker
# i.e. those from _outside_ the docker network. This could be from the host machine
# running docker, or maybe further afield if you've got a more complicated setup.
# If the latter is true, you will need to change the value 'localhost' in
# KAFKA_ADVERTISED_LISTENERS to one that is resolvable to the docker host from those
# remote clients
#
# For connections _internal_ to the docker network, such as from other services
# and components, use kafka:29092.
#
# See https://rmoff.net/2018/08/02/kafka-listeners-explained/ for details
# "`-._,-'"`-._,-'"`-._,-'"`-._,-'"`-._,-'"`-._,-'"`-._,-'"`-._,-'"`-._,-
#
- 9092:9092
environment:
KAFKA_BROKER_ID: 1
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 100
schema-registry:
image: confluentinc/cp-schema-registry:5.5.0
container_name: schema-registry
ports:
- 8081:8081
depends_on:
- zookeeper
- kafka
environment:
SCHEMA_REGISTRY_HOST_NAME: schema-registry
SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: zookeeper:2181
kafka-connect:
image: confluentinc/cp-kafka-connect-base:5.5.0
container_name: kafka-connect
depends_on:
- zookeeper
- kafka
- schema-registry
ports:
- 8083:8083
environment:
CONNECT_BOOTSTRAP_SERVERS: "kafka:29092"
CONNECT_REST_PORT: 8083
CONNECT_GROUP_ID: compose-connect-group
CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs
CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets
CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status
CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter
CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema-registry:8081'
CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
CONNECT_REST_ADVERTISED_HOST_NAME: "kafka-connect"
CONNECT_LOG4J_ROOT_LOGLEVEL: "INFO"
CONNECT_LOG4J_APPENDER_STDOUT_LAYOUT_CONVERSIONPATTERN: "[%d] %p %X{connector.context}%m (%c:%L)%n"
CONNECT_LOG4J_LOGGERS: "org.apache.kafka.connect.runtime.rest=WARN,org.reflections=ERROR"
CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: "1"
CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: "1"
CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: "1"
CONNECT_PLUGIN_PATH: '/usr/share/java,/usr/share/confluent-hub-components/'
#
# Make sure you set this for the correct path that the library files for InstantClient have been installed into
LD_LIBRARY_PATH: '/usr/share/java/debezium-connector-oracle/instantclient_19_6/'
command:
- bash
- -c
- |
echo "Installing connector plugins"
confluent-hub install --no-prompt confluentinc/kafka-connect-jdbc:5.5.0
confluent-hub install --no-prompt confluentinc/kafka-connect-datagen:0.3.2
#
echo "Downloading JDBC drivers"
cd /usr/share/confluent-hub-components/confluentinc-kafka-connect-jdbc/lib
curl https://maven.xwiki.org/externals/com/oracle/jdbc/ojdbc8/12.2.0.1/ojdbc8-12.2.0.1.jar -o ojdbc8-12.2.0.1.jar
# -----------
# Install Debezium Oracle connector and required libraries
# -- Download
curl "https://oss.sonatype.org/service/local/repositories/snapshots/content/io/debezium/debezium-connector-oracle/1.2.0-SNAPSHOT/debezium-connector-oracle-1.2.0-20200612.041347-34-plugin.tar.gz" | \
tar xvfz - --directory /usr/share/java/
# -- Install the required library files
apt-get update
apt-get install -y --force-yes unzip libaio1
wget "https://download.oracle.com/otn_software/linux/instantclient/19600/instantclient-basiclite-linux.x64-19.6.0.0.0dbru.zip" -O /tmp/ic.zip
unzip /tmp/ic.zip -d /usr/share/java/debezium-connector-oracle/
# -----------
# Launch the Kafka Connect worker
/etc/confluent/docker/run &
#
# Don't exit
sleep infinity
ksqldb:
image: confluentinc/ksqldb-server:0.9.0
container_name: ksqldb
depends_on:
- kafka
- kafka-connect
ports:
- "8088:8088"
environment:
KSQL_LISTENERS: http://0.0.0.0:8088
KSQL_BOOTSTRAP_SERVERS: kafka:29092
KSQL_KSQL_LOGGING_PROCESSING_STREAM_AUTO_CREATE: "true"
KSQL_KSQL_LOGGING_PROCESSING_TOPIC_AUTO_CREATE: "true"
KSQL_KSQL_CONNECT_URL: http://kafka-connect:8083
KSQL_KSQL_SCHEMA_REGISTRY_URL: http://schema-registry:8081
KSQL_KSQL_SERVICE_ID: confluent_rmoff_01
KSQL_KSQL_HIDDEN_TOPICS: '^_.*'
# Other systems
oracle:
# You need to build this image first, following the instructions at
# https://github.com/oracle/docker-images/blob/master/OracleDatabase/SingleInstance/README.md
#
# NB: There's no sudo on the image but you can run stuff as root using
# docker exec -it -u root oracle <cmd>
# e.g.: docker exec -it -u root oracle bash -c "rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm && yum install -y rlwrap"
image: oracle/database:12.2.0.1-ee
container_name: oracle
environment:
- ORACLE_PWD=Admin123
ports:
- 1521:1521
volumes:
- $PWD/oradata:/opt/oracle/oradata
- $PWD/oradata/recovery_area:/opt/oracle/oradata/recovery_area
- $PWD/ora-setup-scripts:/opt/oracle/scripts/setup
- $PWD/ora-startup-scripts:/opt/oracle/scripts/startup
postgres:
# *-----------------------------*
# To connect to the DB:
# docker exec -it postgres bash -c 'psql -U $POSTGRES_USER $POSTGRES_DB'
# *-----------------------------*
image: postgres:12
container_name: postgres
environment:
- POSTGRES_USER=postgres
- POSTGRES_PASSWORD=postgres
kafkacat:
image: edenhill/kafkacat:1.5.0
container_name: kafkacat
links:
- kafka
- schema-registry
entrypoint:
- /bin/sh
- -c
- |
apk add jq;
while [ 1 -eq 1 ];do sleep 60;done