Kc/trace

README.md

@@ -78,3 +78,7 @@ file.
 TODO: Instructions on how to generate an Okta access token locally.     
 We use the Okta Code org locally because this makes it easier to develop and test with users that have already been
 registered in [Code gateway](https://profile.code.dev-theguardian.com/).
+
+### Docs
+
+See the [docs](docs/) for more information on how to work with the app.

app/controllers/TelemetryFilter.scala

@@ -0,0 +1,27 @@
+package controllers
+
+import io.opentelemetry.api.trace.SpanKind.SERVER
+import io.opentelemetry.api.trace.{SpanKind, StatusCode, Tracer}
+import play.api.mvc.*
+
+import scala.concurrent.ExecutionContext
+
+class TelemetryFilter(tracer: Tracer)(implicit ec: ExecutionContext) extends EssentialFilter {
+
+  override def apply(next: EssentialAction): EssentialAction = request => {
+    val span = tracer.spanBuilder(request.path).setSpanKind(SERVER).startSpan()
+    span.makeCurrent()
+    val accumulator = next(request)
+    accumulator
+      .map { result =>
+        span.end()
+        result
+      }
+      .recover { case e: Exception =>
+        span.setStatus(StatusCode.ERROR, e.getMessage)
+        span.recordException(e)
+        span.end()
+        Results.InternalServerError(s"Telemetry failure: ${e.getMessage}")
+      }
+  }
+}

app/load/AppComponents.scala

@@ -5,7 +5,17 @@ import com.gu.identity.auth.{OktaAudience, OktaAuthService, OktaIssuerUrl, OktaT
 import com.okta.sdk.client.AuthorizationMode.PRIVATE_KEY
 import com.okta.sdk.client.Clients
 import com.okta.sdk.resource.api.UserApi
-import controllers.{HealthCheckController, UserController}
+import controllers.{HealthCheckController, TelemetryFilter, UserController}
+import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator
+import io.opentelemetry.context.propagation.{ContextPropagators, TextMapPropagator}
+import io.opentelemetry.contrib.aws.resource.Ec2Resource
+import io.opentelemetry.contrib.awsxray.{AwsXrayIdGenerator, AwsXrayRemoteSampler}
+import io.opentelemetry.contrib.awsxray.propagator.AwsXrayPropagator
+import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter
+import io.opentelemetry.sdk.OpenTelemetrySdk
+import io.opentelemetry.sdk.resources.Resource
+import io.opentelemetry.sdk.trace.SdkTracerProvider
+import io.opentelemetry.sdk.trace.`export`.BatchSpanProcessor
 import logging.RequestLoggingFilter
 import play.api.ApplicationLoader.Context
 import play.api.BuiltInComponentsFromContext
@@ -25,7 +35,33 @@ class AppComponents(context: Context)
     with SlickComponents
     with AhcWSComponents {
 
-  override def httpFilters: Seq[EssentialFilter] = super.httpFilters :+ new RequestLoggingFilter(materializer)
+  private val openTelemetry = {
+    val propagators = ContextPropagators.create(
+      TextMapPropagator.composite(
+        W3CTraceContextPropagator.getInstance,
+        AwsXrayPropagator.getInstance
+      )
+    )
+    val spanProcessor = BatchSpanProcessor.builder(OtlpGrpcSpanExporter.getDefault).build()
+    val idGenerator = AwsXrayIdGenerator.getInstance
+    val resource = Resource.getDefault.merge(Ec2Resource.get)
+    val sampler = AwsXrayRemoteSampler.newBuilder(resource).build()
+    val tracerProvider = SdkTracerProvider.builder
+      .addSpanProcessor(spanProcessor)
+      .setIdGenerator(idGenerator)
+      .setResource(resource)
+      .setSampler(sampler)
+      .build()
+    OpenTelemetrySdk.builder
+      .setPropagators(propagators)
+      .setTracerProvider(tracerProvider)
+      .buildAndRegisterGlobal()
+  }
+
+  override def httpFilters: Seq[EssentialFilter] = super.httpFilters :++ Seq(
+    new RequestLoggingFilter(materializer),
+    new TelemetryFilter(openTelemetry.getTracer("Gatehouse-manual"))
+  )
 
   private lazy val oktaOrgUrl = s"https://${configuration.get[String]("oktaApi.domain")}"
 

app/load/AppLoader.scala

@@ -36,11 +36,20 @@ class AppLoader extends ApplicationLoader {
 
     // To validate the SSL certificate of the RDS instance - remove this and the associated params when we no longer use RDS
     def configureTrustStore(config: Config) = {
-      System.setProperty("javax.net.ssl.trustStore", config.getString("trustStore.path"))
-      System.setProperty("javax.net.ssl.trustStorePassword", config.getString("trustStore.password"))
+      sys.Prop.StringProp("javax.net.ssl.trustStore").set(config.getString("trustStore.path"))
+      sys.Prop.StringProp("javax.net.ssl.trustStorePassword").set(config.getString("trustStore.password"))
+    }
+
+    def configureTelemetry() = {
+      sys.Prop.StringProp("otel.service.name").set("Gatehouse")
+      sys.Prop.StringProp("otel.traces.exporter").set("logging,otlp")
+      sys.Prop.StringProp("otel.metrics.exporter").set("none")
+      sys.Prop.StringProp("otel.logs.exporter").set("none")
+      sys.Prop.IntProp("otel.metric.export.interval").set("15000")
     }
 
     if (isDev)
+      configureTelemetry()
       Try(configFor(AwsIdentity(app = appName, stack = awsProfileName, stage = "DEV", region = "eu-west-1")))
     else
       for {

app/utils/FutureHelper.scala

@@ -1,13 +1,20 @@
 package utils
 
-import scala.concurrent.{ExecutionContext, Future, blocking}
+import io.opentelemetry.context.Context as TelemetryContext
+
+import scala.concurrent.{ExecutionContext, Future}
 import scala.util.{Failure, Success, Try}
 
 object FutureHelper {
 
-  def tryAsync[A](a: => A)(implicit ctx: ExecutionContext): Future[A] =
-    Future(Try(a)) flatMap {
+  def tryAsync[A](a: => A)(implicit ctx: ExecutionContext): Future[A] = {
+    val telemetryContext = TelemetryContext.current()
+    Future(Try {
+      telemetryContext.makeCurrent()
+      a
+    }) flatMap {
       case Success(a)         => Future.successful(a)
       case Failure(exception) => Future.failed(exception)
     }
+  }
 }

build.sbt

@@ -8,17 +8,22 @@ lazy val root = (project in file("."))
     scalacOptions ++= Seq(
       "-explain",
       "-feature",
-      "-Werror",
+//      "-Werror",
     ),
     scalafmtOnCompile := true,
     Universal / javaOptions ++= Seq(
       "-javaagent:/opt/aws-opentelemetry-agent/aws-opentelemetry-agent.jar",
       "-Dotel.service.name=Gatehouse",
-      "-Dotel.exporter=otlp",
+//      "-Dotel.resource.providers.aws.enabled=true",
+//      "-Dotel.instrumentation.common.experimental.controller-telemetry.enabled=true",
       "-Dotel.traces.sampler=xray",
-//      "-Dotel.javaagent.debug=true",
+      "-Dotel.traces.exporter=logging,otlp",
+      "-Dotel.metrics.exporter=none",
+      "-Dotel.logs.exporter=none",
+      "-Dotel.javaagent.debug=true",
+//      s"-Dotel.javaagent.configuration-file=${baseDirectory.value}/conf/telemetry.conf",
       "-Dpidfile.path=/dev/null",
-      s"-J-Dlogs.home=/var/log/${packageName.value}",
+      s"-Dlogs.home=/var/log/${packageName.value}",
     ),
     Test / javaOptions += "-Dlogback.configurationFile=logback-test.xml",
     libraryDependencies ++= Seq(
@@ -38,6 +43,15 @@ lazy val root = (project in file("."))
       "com.okta.sdk" % "okta-sdk-api" % "15.0.0",
       "com.okta.sdk" % "okta-sdk-impl" % "15.0.0" % Runtime,
       "com.googlecode.libphonenumber" % "libphonenumber" % "8.13.34",
+      "io.opentelemetry" % "opentelemetry-api" % "1.37.0",
+      "io.opentelemetry" % "opentelemetry-sdk" % "1.37.0",
+      "io.opentelemetry" % "opentelemetry-exporter-otlp" % "1.37.0",
+      "io.opentelemetry.semconv" % "opentelemetry-semconv" % "1.25.0-alpha",
+      "io.opentelemetry" % "opentelemetry-extension-aws" % "1.20.1" % Runtime,
+      "io.opentelemetry" % "opentelemetry-sdk-extension-aws" % "1.19.0" % Runtime,
+      "io.opentelemetry.contrib" % "opentelemetry-aws-xray" % "1.35.0",
+      "io.opentelemetry.contrib" % "opentelemetry-aws-xray-propagator" % "1.35.0-alpha",
+      "io.opentelemetry.contrib" % "opentelemetry-aws-resources" % "1.35.0-alpha",
       "org.scalatestplus.play" %% "scalatestplus-play" % "7.0.1" % Test,
     ),
     dependencyOverrides ++= {

conf/logback.xml

@@ -22,6 +22,8 @@
         <appender-ref ref="STDOUT"/>
     </appender>
 
+    <logger name="io.opentelemetry" level="DEBUG"/>
+
     <root level="INFO">
         <appender-ref ref="ASYNCFILE"/>
         <appender-ref ref="ASYNCSTDOUT"/>

conf/telemetry.conf

@@ -0,0 +1,15 @@
+otel.service.name = Gatehouse
+
+otel.resource.providers.aws.enabled = true
+
+otel.instrumentation.common.experimental.controller-telemetry.enabled = true
+
+otel.traces.sampler = xray
+
+//otel.exporter=otlp
+otel.traces.exporter = logging,otlp
+//otel.traces.exporter=none
+otel.metrics.exporter = logging,otlp
+otel.logs.exporter = logging,otlp
+
+otel.javaagent.debug = true

docs/Telemetry.md

@@ -0,0 +1,19 @@
+# Telemetry
+
+We're using AWS X-Ray in Code and Prod to (eventually) get distributed traces of client apps calling Gatehouse and Gatehouse then calling Okta.
+We're using the OpenTelemetry Java agent to instrument the JVM.
+This sends traces to a local collector, which forwards them to AWS X-Ray.
+
+## Troubleshooting traces locally
+
+It doesn't seem to be straightforward to send JVM arguments to Play apps in Dev mode.  
+This is how I have been working locally.
+
+1. In Intellij, I set `JAVA_TOOL_OPTIONS` in my SBT config:  
+sbt Tool Window > Build Tool Settings > sbt Settings > Environment variables:  
+`JAVA_TOOL_OPTIONS`=`-javaagent:/<path>/aws-opentelemetry-agent.jar -Dotel.javaagent.configuration-file=conf/telemetry.conf`
+
+2. Download the agent to the path specified above.  
+See cdk/lib/gatehouse.ts for steps to download the agent.
+
+3. Start a new SBT session.

test/controllers/HealthCheckControllerSpec.scala

@@ -15,7 +15,12 @@ import scala.concurrent.Future
 
 class HealthCheckControllerSpec extends PlaySpec with OneAppPerTestWithComponents with MockitoSugar {
 
-  override def components: BuiltInComponents = new AppComponents(context)
+  override def components: BuiltInComponents = {
+    sys.Prop.StringProp("otel.traces.exporter").set("none")
+    sys.Prop.StringProp("otel.metrics.exporter").set("none")
+    sys.Prop.StringProp("otel.logs.exporter").set("none")
+    new AppComponents(context)
+  }
 
   "GET healthcheck" should {