Telemetry

object Implementation extends StrictLogging {
  def foo = BusinessLogic.formatSQL("SELECT * FROM foo;")
}
object TelemetryV1 extends StrictLogging {
  def metered[A](f: () => A): A = {
    val start  = System.currentTimeMillis()
    val result = f()
    val end    = System.currentTimeMillis()
    logger.info(s"elapsed: ${end - start}")
    result
  }
}
object Implementation extends StrictLogging {
  def foo = TelemetryV1.metered(
  	() => BusinessLogic.formatSQL("SELECT * FROM foo;")
  )
}
//elapsed: 10s
//elapsed: 1s
//elapsed: 20s
object TelemetryV2 extends StrictLogging {
  def metered[A](name: String)(f: () => A): A = {
    val start  = System.currentTimeMillis()
    val result = f()
    val end    = System.currentTimeMillis()
    logger.info(s"name: ${name} elapsed: ${end - start}")
    result
  }
}
object ImplementationV2 {
  def foo =
    TelemetryV2.metered("foo")(
      () => BusinessLogic.formatSQL("SELECT * FROM foo;")
    )

  def bar =
    TelemetryV2.metered("bar")(
      () => BusinessLogic.formatSQL("SELECT * FROM foo;")
    )
}
//name: "foo" elapsed: 10s
//name: "bar" elapsed: 1s
//name: "foo" elapsed: 20s
object TelemetryV3 extends StrictLogging {
  def metered[A](name: String)(f: () => A): A = {
    val start  = System.currentTimeMillis()
    val result = Try(f())
    val end    = System.currentTimeMillis()
    logger.info(s"name: ${name} error: ${result.isFailure} elapsed: ${end - start}")
    result.get
  }
}
object ImplementationV3 {
  def foo =
    TelemetryV3.metered("foo")(
      () => BusinessLogic.formatSQL("SELECT * FROM foo;")
    )

  def bar =
    TelemetryV3.metered("bar")(
      () => BusinessLogic.formatSQL("SELECT * FROM foo;")
    )
}
//name: "foo" error: true elapsed: 10s
//name: "bar" error: false elapsed: 1s
//name: "foo" error: true elapsed: 20s
object BusinessLogicV1 {
  def formatSQL(mode: String)(sql: String): String = ???
}
object TelemetryV4 extends StrictLogging {
  def metered[A](
                  name: String,
                  tags: Map[String, Any]
                )(f: () => A): A = {
    val start  = System.currentTimeMillis()
    val result = Try(f())
    val end    = System.currentTimeMillis()
    val tags   = (Map("error" -> result.isFailure) ++ tags).map{
      case (k,v) => s"$k: $v"
    }.toList.mkString(" ")
    logger.info(s"name: ${name} ${tags} elapsed: ${end - start}")
    result.get
  }
}
object ImplementationV4 {
  def foo(mode: String) =
    TelemetryV4.metered(
      "foo",
      Map("mode" -> mode)
    )(() => BusinessLogicV1.formatSQL(mode)("SELECT * FROM foo;"))

  def bar(mode: String) =
    TelemetryV4.metered(
      "bar",
      Map("mode" -> mode)
    )(() => BusinessLogicV1.formatSQL(mode)("SELECT * FROM foo;"))
}
//name: "foo" error: true  mode: "http"  elapsed: 10s
//name: "bar" error: false mode: "local" elapsed: 1s
//name: "foo" error: true  mode: "http"  elapsed: 20s
//name: "foo" error: true  mode: "http"  elapsed: 10s
//name: "bar" error: false mode: "local" elapsed: 1s
//name: "foo" error: true  mode: "http"  elapsed: 20s
//name: "foo" error: true  mode: "http"  elapsed: 20s
//name: "foo" error: true  mode: "http"  elapsed: 20s
//name: "foo" error: true  mode: "http"  elapsed: 20s
//name: "foo" error: true  mode: "http"  elapsed: 20s
//name: "foo" error: true  mode: "http"  elapsed: 30s
//name: "foo" error: true  mode: "http"  elapsed: 20s
//name: "foo" error: true  mode: "http"  elapsed: 20s

8*50*5*60*60 ~= 0.9MB/h

object TelemetryReporter {
  type Name = String
  type Tags = Map[String, Any]
  private case class Metric(count: Int, sum: Long, max:Long)

  private var buffer = Map.empty[(Name, Tags), Metric]
  
  def addOperation(name: String, tags: Map[String, Any], v: Long) = {
    val newV = buffer
      .get(name -> tags)
      .map(m => m.copy(count = m.count + 1, sum = m.sum + v, max = Math.max(v,m.max)))
      .getOrElse(Metric(1, 0, Long.MinValue))
    buffer = buffer + ((name -> tags) -> newV)
  }

  def reportingTask() = {
    while(true) {
      //report to some external system
      buffer = Map.empty
    }
  }
}

And that's it for telemetry!

Concepts

- name

- tags/labels

- reporter

- aggregation

Telemetry

By Carlos Teixeira

Telemetry

  • 532