Actor stats not being sent to StatsD from Kamon

Question

I'm trying to use Kamon with StatsD backend for Akka monitoring/telemetry. I currently only have one actor and I'm sending an ask which replies to the caller. I can see my custom counter data in Graphana and Graphite as well as JVM and OS stats, but no actor stats from Akka. My project is setup just like this intro but I have added the Kamon stuff and my own HttpService. What am I missing?

Here's my config:

kamon {
  trace.level = simple-trace
  metric {
    tick-interval = 1 second
    filters {
      akka-actor.includes = [ "**" ]
      akka-router.includes = [ "**" ]
      akka-dispatcher.includes = [ "**" ]
      trace.includes =  [ "**" ]
      trace-segment.includes =  [ "**" ]
      histogram.includes =  [ "**" ]
      min-max-counter.includes =  [ "**" ]
      gauge.includes =  [ "**" ]
      counters.includes =  [ "**" ]
      # For web projects
      http-server.includes = [ "**" ]

      akka-actor.excludes = [ "**" ]
      akka-router.excludes = [ "**" ]
      akka-dispatcher.excludes = [ "**" ]
      counters.excludes =  [ "" ]
      trace-segment.excludes =  [ "" ]
      histogram.excludes =  [ "" ]
      min-max-counter.excludes =  [ "" ]
      gauge.excludes =  [ "" ]
      http-server.excludes =  [ "" ]
    }
  }

  akka {
    ask-pattern-timeout-warning = "lightweight"
  }

  statsd {
    hostname = "localhost" //Graphite Host
    port = "8125"
    flush-interval = 1 second
    max-packet-size = 1024 bytes
    subscriptions {
      histogram = ["**"]
      min-max-counter = ["**"]
      gauge = ["**"]
      counter = ["**"]
      trace = ["**"]
      trace-segment = ["**"]
      system-metric = ["**"]
      akka-actor = ["**"]
      akka-dispatcher = ["**"]
      http-server = ["**"]
    }
    report-system-metrics = true

    simple-metric-key-generator {
      application = "data-id-generator"
    }
  }
  system-metrics {
    #sigar is enabled by default
    sigar-enabled = false

    #jmx related metrics are enabled by default
    jmx-enabled = true
  }
}

Here's my service:

import akka.event.Logging
import akka.http.scaladsl.server.Directives._
import akka.http.scaladsl.server.Route
import akka.pattern.ask
import akka.util.{ByteString, Timeout}
import com.incontact.data.GeneratorActor.GenerateId
import com.incontact.data.IdGenerator.{GeneratorError, IdResult}
import com.incontact.http.HttpService
import akka.http.scaladsl.model.StatusCodes._
import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport
import spray.json._
import kamon.Kamon

import scala.concurrent.duration._

trait IdService extends HttpService with SprayJsonSupport with DefaultJsonProtocol {
  private lazy val log = Logging(system, classOf[IdService])

  implicit val timeout: Timeout = 1.second
  //implicit val errorFormat = jsonFormat2(GeneratorError)

  abstract override def route: Route = {
    log.info("creating generator actor")
    val generator = system.actorOf(GeneratorActor.props, "generator")
    val counter = Kamon.metrics.counter("generate-long-id")

    path("") {
      get {
        counter.increment()
        log.info("IdService sending single request to actor")
        onSuccess((generator ? GenerateId).mapTo[IdResult]) {
          case Right(id) => complete(ByteString(s"${id}\n"))
          case Left(e:GeneratorError) => failWith(e.Error)
        }
      }
    }
  } ~ super.route
}

Main.scala (startup)

import akka.event.Logging
import com.incontact.data.IdService
import com.incontact.http.HttpServer
import kamon.Kamon

import scala.io.StdIn
import scala.util.Success

object Main extends App
  with HttpServer
  with IdService {

  Kamon.start()

  private lazy val log = Logging(system, getClass)
  bindingFuture.andThen {
    case Success(binding) =>
      // this really doesn't support any production behavior since [ENTER] won't be used to shut down the docker container
      log.info("Press ENTER to exit")
      StdIn.readLine()
      log.info("Shutting down...")
      Kamon.shutdown()
      system.terminate()
  }
}

build.sbt

scalaVersion := "2.11.8"
val akkaVersion = "2.4.14"
val kamonVersion = "0.6.3"

libraryDependencies ++= Seq(
 // remove multiple-dependency warnings
 "org.scala-lang" % "scala-reflect" % scalaVersion.value,
 "org.scala-lang.modules" %% "scala-xml" % "1.0.5",

 // application dependencies
 "com.typesafe.akka" %% "akka-actor" % akkaVersion,
 "com.typesafe.akka" %% "akka-http" % "10.0.0",
 "com.typesafe.akka" %% "akka-http-spray-json" % "10.0.0",
 "com.amazonaws" % "aws-java-sdk-dynamodb" % "1.11.63",

 // kamon dependencies
 "io.kamon" %% "kamon-core" % kamonVersion,
 "io.kamon" %% "kamon-statsd" % kamonVersion,
 "io.kamon" %% "kamon-system-metrics" % kamonVersion,
 "org.aspectj" % "aspectjweaver" % "1.8.6",

 // test dependencies
 "com.typesafe.akka" %% "akka-testkit" % akkaVersion % "test",
 "org.scalatest" %% "scalatest" % "3.0.1" % "test",
 "com.typesafe.akka" %% "akka-http-testkit" % "10.0.0" % "test",
 "org.pegdown" % "pegdown" % "1.1.0" % "test"
)

project/plugins.sbt

addSbtPlugin("io.kamon" % "aspectj-runner" % "0.1.3")

launch from command-line:

$ sbt aspectj-runner:run

Answer 1

It looks like you're excluding all the actors/dispatcher/router metrics in your config. This should rule all metrics out (even if you're also including them all).

  akka-actor.excludes = [ "**" ]
  akka-router.excludes = [ "**" ]
  akka-dispatcher.excludes = [ "**" ]

Try and change it to

  akka-actor.excludes = [  ]
  akka-router.excludes = [  ]
  akka-dispatcher.excludes = [  ]