I can only find old and incomplete examples of using opentracing/jaeger with Kafka. I want to run an example locally as a proof of concept - opentracing spans to kafka.
I managed to get some of this working, but on jeager-query service I keep getting:
"msg":"Failed to init storage factory","error":"kafka: client has run out of available brokers to talk to (Is your cluster reachable?)"
I'm not sure if I need to use some sort of storage i.e. cassandra too?
version: '3.8'
services:
zookeeper:
image: confluentinc/cp-zookeeper
networks:
- kafka-net
container_name: zookeeper
environment:
- ZOOKEEPER_CLIENT_PORT=2181
ports:
- 2181:2181
kafka:
image: confluentinc/cp-kafka
networks:
- kafka-net
container_name: kafka
environment:
- KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
- KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
- ALLOW_PLAINTEXT_LISTENER=yes
- KAFKA_LISTENERS-INTERNAL://kafka:29092,EXTERNAL://localhost:9092
- KAFKA_ADVERTISED=INTERNAL://kafka:29092,EXTERNAL://localhost:9092
- KAFKA_ADVERTISED_LISTENERS=INTERNAL://kafka:29092,EXTERNAL://localhost:9092
- KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
- KAFKA_INTER_BROKER_LISTENER_NAME=INTERNAL
ports:
- 9092:9092
depends_on:
- zookeeper
restart: on-failure
jaeger-collector:
image: jaegertracing/jaeger-collector
container_name: jaeger-collector
networks:
- kafka-net
ports:
- "14250:14250"
# - "14267:14267"
- "14268:14268" # HTTP collector port to receive spans
- "14269:14269" # HTTP health check port
restart: on-failure
environment:
LOG_LEVEL: "debug"
SPAN_STORAGE_TYPE: "kafka"
KAFKA_TOPIC: "somekafkatag"
KAFKA_BROKERS: "kafka:9092"
KAFKA_PRODUCER_BROKERS: "kafka:29092"
jaeger-agent:
image: jaegertracing/jaeger-agent
container_name: jaeger-agent
networks:
- kafka-net
command: ["--reporter.grpc.host-port=jaeger-collector:14250"]
ports:
- "5775:5775/udp"
- "6831:6831/udp"
- "6832:6832/udp"
- "5778:5778"
environment:
LOG_LEVEL: "debug"
SPAN_STORAGE_TYPE: "kafka"
restart: on-failure
depends_on:
- jaeger-collector
jaeger-ingester:
image: jaegertracing/jaeger-ingester
container_name: jaeger-ingester
networks:
- kafka-net
ports:
- "14270:14270" # HTTP health check port: http://localhost:14270/
- "14271:14271" # Metrics port: http://localhost:14271/metrics
restart: on-failure
command: ["--kafka.producer.brokers=kafka:9092"]
environment:
LOG_LEVEL: "debug"
INGESTER_PARALLELISM: "1"
INGESTER_DEADLOCKINTERVAL: "0ms"
SPAN_STORAGE_TYPE: "kafka"
KAFKA_CONSUMER_BROKERS: "kafka:9092"
METRICS_BACKEND: "expvar"
KAFKA_BROKERS: "kafka:29092"
jaeger-query:
image: jaegertracing/jaeger-query
container_name: jaeger-query
networks:
- kafka-net
ports:
- "16686:16686" # Jaeger UI port
- "16687:16687" # HTTP health check port: http://localhost:16687/
restart: on-failure
environment:
LOG_LEVEL: "debug"
SPAN_STORAGE_TYPE: "kafka"
KAFKA_CONSUMER_BROKERS: "kafka:9092"
KAFKA_BROKERS: "kafka:29092"
networks:
kafka-net:
driver: bridge
I was missing a lot of information. I managed to get it working:
version: '3.8'
services:
zookeeper:
image: confluentinc/cp-zookeeper
networks:
- kafka-net
container_name: zookeeper
environment:
ZOOKEEPER_CLIENT_PORT: 2181
ports:
- 2181:2181
cassandra:
hostname: cassandra
image: cassandra
networks:
- kafka-net
environment:
MAX_HEAP_SIZE: 1G
HEAP_NEWSIZE: 256M
ports:
- "9042:9042"
cassandra-schema:
image: jaegertracing/jaeger-cassandra-schema
networks:
- kafka-net
depends_on:
- cassandra
kafka:
image: confluentinc/cp-kafka
networks:
- kafka-net
container_name: kafka
environment:
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
ALLOW_PLAINTEXT_LISTENER: "yes"
KAFKA_LISTENERS-INTERNAL: //kafka:29092,EXTERNAL://localhost:9092
KAFKA_ADVERTISED: INTERNAL://kafka:29092,EXTERNAL://localhost:9092
KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka:29092,EXTERNAL://localhost:9092
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
ports:
- 9092:9092
- 29092:29092
depends_on:
- zookeeper
restart: on-failure
jaeger-collector:
image: jaegertracing/jaeger-collector
container_name: jaeger-collector
networks:
- kafka-net
ports:
- "14250:14250"
- "14267:14267"
- "14268:14268" # HTTP collector port to receive spans
- "14269:14269" # HTTP health check port
restart: on-failure
environment:
LOG_LEVEL: "debug"
SPAN_STORAGE_TYPE: "kafka"
KAFKA_BROKERS: "kafka:9092"
KAFKA_PRODUCER_BROKERS: "kafka:29092"
jaeger-agent:
image: jaegertracing/jaeger-agent
container_name: jaeger-agent
networks:
- kafka-net
command: ["--reporter.grpc.host-port=jaeger-collector:14250"]
ports:
- "5775:5775/udp"
- "6831:6831/udp"
- "6832:6832/udp"
- "5778:5778"
environment:
LOG_LEVEL: "debug"
SPAN_STORAGE_TYPE: "kafka"
restart: on-failure
depends_on:
- jaeger-collector
jaeger-ingester:
image: jaegertracing/jaeger-ingester
container_name: jaeger-ingester
networks:
- kafka-net
ports:
- "14270:14270" # HTTP health check port: http://localhost:14270/
- "14271:14271" # Metrics port: http://localhost:14270/metrics
restart: on-failure
environment:
LOG_LEVEL: debug
INGESTER_PARALLELISM: 1
INGESTER_DEADLOCKINTERVAL: ms
SPAN_STORAGE_TYPE: cassandra
CASSANDRA_SERVERS: cassandra
CASSANDRA_KEYSPACE: jaeger_v1_dc1
METRICS_BACKEND: expvar
KAFKA_CONSUMER_BROKERS: kafka:29092
KAFKA_CONSUMER_TOPIC: jaeger-spans
depends_on:
- cassandra-schema
jaeger-query:
image: jaegertracing/jaeger-query
container_name: jaeger-query
networks:
- kafka-net
ports:
- "16686:16686" # Jaeger UI port
- "16687:16687" # HTTP health check port: http://localhost:16687/
restart: on-failure
depends_on:
- cassandra-schema
environment:
LOG_LEVEL: debug
SPAN_STORAGE_TYPE: cassandra
CASSANDRA_SERVERS: cassandra
CASSANDRA_KEYSPACE: jaeger_v1_dc1
JAEGER_ENDPOINT: http://jaeger-collector:14268/api/traces
networks:
kafka-net:
driver: bridge