Cassandra datastax driver unable to connect to one of the nodes in the data center but still I am able to read and write to database. Datacenter contains two nodes in one rack. I specified one of them as seed node.
Connected to cluster: Test Cluster
Datacenter: datacenter1; Host: /PVT IP1; Rack: rack1
Datacenter: datacenter1; Host: /PUBLIC IP2; Rack: rack1
2017-11-04 02:19:50 WARN com.datastax.driver.core.HostConnectionPool:184 - Error creating connection to /PVT IP1:9042
com.datastax.driver.core.exceptions.TransportException: [/PVT IP1:9042] Cannot connect
at com.datastax.driver.core.Connection$1.operationComplete(Connection.java:165)
at com.datastax.driver.core.Connection$1.operationComplete(Connection.java:148)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:507)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:500)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:479)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:420)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:122)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe$1.run(AbstractNioChannel.java:220)
at io.netty.util.concurrent.PromiseTask$RunnableAdapter.call(PromiseTask.java:38)
at io.netty.util.concurrent.ScheduledFutureTask.run(ScheduledFutureTask.java:120)
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:399)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:463)
at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:131)
at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:138)
at java.lang.Thread.run(Thread.java:745)
Caused by: io.netty.channel.ConnectTimeoutException: connection timed out: /PVT IP1:9042
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe$1.run(AbstractNioChannel.java:218)
... 7 more
2017-11-04 02:19:50 WARN com.datastax.driver.core.Session:378 - Error creating pool to /PVT IP1:9042
com.datastax.driver.core.exceptions.ConnectionException: [/PVT IP1:9042] Pool was closed during initialization
at com.datastax.driver.core.HostConnectionPool$2.onSuccess(HostConnectionPool.java:148)
at com.datastax.driver.core.HostConnectionPool$2.onSuccess(HostConnectionPool.java:134)
at com.google.common.util.concurrent.Futures$6.run(Futures.java:1773)
at com.google.common.util.concurrent.MoreExecutors$DirectExecutor.execute(MoreExecutors.java:456)
at com.google.common.util.concurrent.AbstractFuture.executeListener(AbstractFuture.java:817)
at com.google.common.util.concurrent.AbstractFuture.complete(AbstractFuture.java:753)
at com.google.common.util.concurrent.AbstractFuture.set(AbstractFuture.java:613)
at com.google.common.util.concurrent.CollectionFuture$CollectionFutureRunningState.handleAllCompleted(CollectionFuture.java:76)
at com.google.common.util.concurrent.AggregateFuture$RunningState.processCompleted(AggregateFuture.java:255)
at com.google.common.util.concurrent.AggregateFuture$RunningState.decrementCountAndMaybeComplete(AggregateFuture.java:242)
at com.google.common.util.concurrent.AggregateFuture$RunningState.access$300(AggregateFuture.java:91)
at com.google.common.util.concurrent.AggregateFuture$RunningState$1.run(AggregateFuture.java:146)
at com.google.common.util.concurrent.MoreExecutors$DirectExecutor.execute(MoreExecutors.java:456)
at com.google.common.util.concurrent.AbstractFuture.executeListener(AbstractFuture.java:817)
at com.google.common.util.concurrent.AbstractFuture.complete(AbstractFuture.java:753)
at com.google.common.util.concurrent.AbstractFuture.completeWithFuture(AbstractFuture.java:730)
at com.google.common.util.concurrent.AbstractFuture.setFuture(AbstractFuture.java:666)
at com.google.common.util.concurrent.Futures$AsyncCatchingFuture.doFallback(Futures.java:826)
at com.google.common.util.concurrent.Futures$AsyncCatchingFuture.doFallback(Futures.java:813)
at com.google.common.util.concurrent.Futures$AbstractCatchingFuture.run(Futures.java:789)
at com.google.common.util.concurrent.MoreExecutors$DirectExecutor.execute(MoreExecutors.java:456)
at com.google.common.util.concurrent.AbstractFuture.executeListener(AbstractFuture.java:817)
at com.google.common.util.concurrent.AbstractFuture.complete(AbstractFuture.java:753)
at com.google.common.util.concurrent.AbstractFuture.completeWithFuture(AbstractFuture.java:730)
at com.google.common.util.concurrent.AbstractFuture.setFuture(AbstractFuture.java:666)
at com.google.common.util.concurrent.Futures$AsyncCatchingFuture.doFallback(Futures.java:826)
at com.google.common.util.concurrent.Futures$AsyncCatchingFuture.doFallback(Futures.java:813)
at com.google.common.util.concurrent.Futures$AbstractCatchingFuture.run(Futures.java:789)
at com.google.common.util.concurrent.MoreExecutors$DirectExecutor.execute(MoreExecutors.java:456)
at com.google.common.util.concurrent.AbstractFuture.executeListener(AbstractFuture.java:817)
at com.google.common.util.concurrent.AbstractFuture.complete(AbstractFuture.java:753)
at com.google.common.util.concurrent.AbstractFuture.setException(AbstractFuture.java:634)
at com.google.common.util.concurrent.Futures$AbstractChainingFuture.run(Futures.java:1405)
at com.google.common.util.concurrent.MoreExecutors$DirectExecutor.execute(MoreExecutors.java:456)
at com.google.common.util.concurrent.AbstractFuture.executeListener(AbstractFuture.java:817)
at com.google.common.util.concurrent.AbstractFuture.complete(AbstractFuture.java:753)
at com.google.common.util.concurrent.AbstractFuture.setException(AbstractFuture.java:634)
at com.google.common.util.concurrent.SettableFuture.setException(SettableFuture.java:53)
at com.datastax.driver.core.Connection$1.operationComplete(Connection.java:165)
at com.datastax.driver.core.Connection$1.operationComplete(Connection.java:148)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:507)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:500)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:479)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:420)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:122)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe$1.run(AbstractNioChannel.java:220)
at io.netty.util.concurrent.PromiseTask$RunnableAdapter.call(PromiseTask.java:38)
at io.netty.util.concurrent.ScheduledFutureTask.run(ScheduledFutureTask.java:120)
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:399)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:463)
at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:131)
at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:138)
at java.lang.Thread.run(Thread.java:745)
docker-compose.yaml file for each node looks like this:
Yaml file Node 1:
version: "3"
services:
cassandra:
image: cassandra:3.11
container_name: cassandra_node
ports:
- 7000:7000
- 9042:9042
volumes:
- /home/******/docker/cassandra/cassandra-data:/var/lib/cassandra
environment:
CASSANDRA_BROADCAST_ADDRESS: PVT IP1
CASSANDRA_SEEDS: PVT IP1
For node2:
Yaml File Node 2
version: "3"
services:
cassandra:
image: cassandra:3.11
container_name: cassandra_node
ports:
- 7000:7000
- 9042:9042
volumes:
- /home/******/docker/cassandra/cassandra-data:/var/lib/cassandra
environment:
CASSANDRA_BROADCAST_ADDRESS: PVT IP2
CASSANDRA_SEEDS: PVT IP1
Could someone help me in figuring out the issue here. Thanks!!
I think the issue here is with values passed for CASSANDRA_BROADCAST_ADDRESS and CASSANDRA_SEEDS. There is space character in values, hence cassandra driver might be unable to parse.
I have been using below docker-compose.yml for creating cassandra cluster. Please give it a try.
version: '3'
networks:
cassandra-cluster:
driver: bridge
volumes:
data-volume-dc1-n1:
data-volume-dc1-n2:
data-volume-dc2-n1:
data-volume-dc2-n2:
services:
###############################################################################################
# DC1 node 1
###############################################################################################
DC1_N1:
container_name: DC1_N1
image: cassandra:latest
command: bash -c 'if [ -z "$$(ls -A /var/lib/cassandra/)" ] ; then sleep 0; fi && /docker-entrypoint.sh cassandra -f'
volumes:
- data-volume-dc1-n1:/var/lib/cassandra
environment:
- CASSANDRA_CLUSTER_NAME=cassandra_cluster
- CASSANDRA_SEEDS=DC1_N1,DC1_N2,DC2_N1,DC2_N2
- CASSANDRA_ENDPOINT_SNITCH=GossipingPropertyFileSnitch
- CASSANDRA_DC=DC1
- CASSANDRA_RACK=rack1
# Expose ports for cassandra cluster
expose:
- 7000
- 7001
- 7199
- 9042
- 9160
ports:
- "9042:9042"
# Cassandra ulimt recommended settings
ulimits:
memlock: -1
nproc: 32768
nofile: 100000
networks:
- cassandra-cluster
restart: unless-stopped
###############################################################################################
# DC1 node 2
###############################################################################################
DC1_N2:
container_name: DC1_N2
image: cassandra:latest
command: bash -c 'if [ -z "$$(ls -A /var/lib/cassandra/)" ] ; then sleep 0; fi && /docker-entrypoint.sh cassandra -f'
volumes:
- data-volume-dc2-n2:/var/lib/cassandra
environment:
- CASSANDRA_CLUSTER_NAME=cassandra_cluster
- CASSANDRA_SEEDS=DC1_N1,DC1_N2,DC2_N1,DC2_N2
- CASSANDRA_ENDPOINT_SNITCH=GossipingPropertyFileSnitch
- CASSANDRA_DC=DC1
- CASSANDRA_RACK=rack2
# Expose ports for cassandra cluster
expose:
- 7000
- 7001
- 7199
- 9042
- 9160
# Cassandra ulimt recommended settings
ulimits:
memlock: -1
nproc: 32768
nofile: 100000
networks:
- cassandra-cluster
restart: unless-stopped
###############################################################################################
# DC2 node 1
###############################################################################################
DC2_N1:
container_name: DC2_N1
image: cassandra:latest
command: bash -c 'if [ -z "$$(ls -A /var/lib/cassandra/)" ] ; then sleep 0; fi && /docker-entrypoint.sh cassandra -f'
volumes:
- data-volume-dc2-n1:/var/lib/cassandra
environment:
- CASSANDRA_CLUSTER_NAME=cassandra_cluster
- CASSANDRA_SEEDS=DC1_N1,DC1_N2,DC2_N1,DC2_N2
- CASSANDRA_ENDPOINT_SNITCH=GossipingPropertyFileSnitch
- CASSANDRA_DC=DC2
- CASSANDRA_RACK=rack1
# Expose ports for cassandra cluster
expose:
- 7000
- 7001
- 7199
- 9042
- 9160
# Cassandra ulimt recommended settings
ulimits:
memlock: -1
nproc: 32768
nofile: 100000
networks:
- cassandra-cluster
restart: unless-stopped