Service Discovery versus DCOS Overlay Network

Question

I've setup a DCOS 1.8 cluster and am currently familiarizing. So far I have marathon-lb working like a charm with Jenkins via Host networking. Now I am trying to set things up using Overlay. I have a couple of test containers, some in the dcos overlay network, some not. So far they can reach each other via IP, which is nice. However when I try to resolv containers on the overlay network using mesos-dns, all it resolves is the host address (not exactly what I am expecting).

So I played around some with marathon to figure it out. What I did was add a discovery block to ipAddress:

{
  "volumes": null,
  "id": "/mariadb10",
  "cmd": null,
  "args": null,
  "user": null,
  "env": {
    "MYSQL_ROOT_PASSWORD": "foo"
  },
  "instances": 1,
  "cpus": 1,
  "mem": 1024,
  "disk": 0,
  "gpus": 0,
  "executor": null,
  "constraints": null,
  "fetch": null,
  "storeUrls": null,
  "backoffSeconds": 1,
  "backoffFactor": 1.15,
  "maxLaunchDelaySeconds": 3600,
  "container": {
    "docker": {
      "image": "mariadb:10.0",
      "forcePullImage": false,
      "privileged": false,
      "network": "USER"
    },
    "type": "DOCKER",
    "volumes": [
      {
        "containerPath": "/var/lib/mysql",
        "hostPath": "/mnt/foo",
        "mode": "RW"
      }
    ]
  },
  "healthChecks": [
    {
      "protocol": "TCP",
      "gracePeriodSeconds": 30,
      "intervalSeconds": 10,
      "timeoutSeconds": 10,
      "maxConsecutiveFailures": 3,
      "port": 3306
    }
  ],
  "readinessChecks": null,
  "dependencies": null,
  "upgradeStrategy": {
    "minimumHealthCapacity": 1,
    "maximumOverCapacity": 1
  },
  "labels": null,
  "acceptedResourceRoles": null,
  "ipAddress": {
    "networkName": "dcos",
    "discovery": {
        "ports": [
            { "number": 3306, "name": "mysql", "protocol": "tcp" }
            ]
    }
  },
  "residency": null,
  "secrets": null,
  "taskKillGracePeriodSeconds": null
}

Marathon tells me this is not allowed with "Bridge" or "User" networks. However it did not complain about the following and launched the container:

{
"volumes": null,
"id": "/mariadb10",
"cmd": null,
"args": null,
"user": null,
"env": {
  "MYSQL_ROOT_PASSWORD": "foo"
},
"instances": 1,
"cpus": 1,
"mem": 1024,
"disk": 0,
"gpus": 0,
"executor": null,
"constraints": null,
"fetch": null,
"storeUrls": null,
"backoffSeconds": 1,
"backoffFactor": 1.15,
"maxLaunchDelaySeconds": 3600,
"container": {
  "docker": {
    "image": "mariadb:10.0",
    "forcePullImage": false,
    "privileged": false,
    "network": "USER"
  },
  "type": "DOCKER",
  "volumes": [
    {
      "containerPath": "/var/lib/mysql",
      "hostPath": "/mnt/foo",
      "mode": "RW"
    }
  ]
},
"healthChecks": [
  {
    "protocol": "TCP",
    "gracePeriodSeconds": 30,
    "intervalSeconds": 10,
    "timeoutSeconds": 10,
    "maxConsecutiveFailures": 3,
    "port": 3306
  }
],
"readinessChecks": null,
"dependencies": null,
"upgradeStrategy": {
  "minimumHealthCapacity": 1,
  "maximumOverCapacity": 1
},
"labels": null,
"acceptedResourceRoles": null,
"ipAddress": {
  "networkName": "dcos"
},
"residency": null,
"secrets": null,
"taskKillGracePeriodSeconds": null
}

Funny thing is, it does not use the overlay address anymore, but now listens to the hosts address and also announces the hosts address into the overlay network.

Am I just doing it wrong or does that not work as expected, yet?

Answer 1

So,

I found the solution myself. The easy workaround is to edit /opt/mesosphere/etc/mesos-dns.json. Then change the order of IPSources to list netinfo first.

For more information, you can also check here