Skip to content

Commit

Permalink
Adding Tag configuration InfluxDb publisher (#275)
Browse files Browse the repository at this point in the history
* adding BuildMetrics tagging filtering

* adding enum build propereties

* removing in class function

* fixing problem toString BuildMetrics

* adding tests TagFieldProvider

* Update TagFieldProvider.kt

removing main function

* adding publisher tests

* adding doc tags configuration

* updating README

* change type tags to list
  • Loading branch information
cdsap authored Mar 5, 2021
1 parent fcc38fc commit 3e8a707
Show file tree
Hide file tree
Showing 12 changed files with 292 additions and 59 deletions.
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -314,6 +314,7 @@ Talaiot will send to the InfluxDb server defined in the configuration the values
| retentionPolicyConfiguration | retention policy which is used for writing points |
| publishBuildMetrics | Publish build metrics of the publisher, true by default |
| publishTaskMetrics | Publish tasks metrics of the publisher, true by default |
| tags | Collection of BuildMetrics used as tags |

Included in: `com.cdsap.talaiot` and `com.cdsap.talaiot.plugin.influxdb` plugins.

Expand Down
9 changes: 9 additions & 0 deletions build.gradle.kts
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
subprojects {
// Address https://github.com/gradle/gradle/issues/4823: Force parent project evaluation before sub-project evaluation for Kotlin build scripts
if (gradle.startParameter.isConfigureOnDemand
&& buildscript.sourceFile?.extension?.toLowerCase() == "kts"
&& parent != rootProject) {
generateSequence(parent) { project -> project.parent.takeIf { it != rootProject } }
.forEach { evaluationDependsOn(it.path) }
}
}
2 changes: 1 addition & 1 deletion library/core/talaiot-test-utils/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ talaiotLib {
dependencies {
implementation(project(":library:core:talaiot-logger"))
implementation("org.testcontainers:testcontainers:1.11.3")
api("org.testcontainers:influxdb:1.11.3")
api("org.testcontainers:influxdb:1.15.2")
api("org.testcontainers:elasticsearch:1.12.0")

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
package com.cdsap.talaiot.metrics

enum class BuildMetrics {
Duration,
Configuration,
Success,
BuildId,
BuildInvocationId,
RequestedTasks,
CacheRatio,
Start,
RootProject,
OsVersion,
MaxWorkers,
JavaRuntime,
JavaVmName,
JavaXmsBytes,
JavaXmxBytes,
JavaMaxPermSize,
TotalRamAvailableBytes,
CpuCount,
Locale,
Username,
DefaultCharset,
IdeVersion,
GradleVersion,
GitBranch,
GitUser,
Hostname,
OsManufacturer,
PublicIp,
CacheUrl,
LocalCacheHit,
LocalCacheMiss,
RemoteCacheHit,
RemoteCacheMiss,
CacheStore,
SwitchCache,
SwitchScan,
SwitchConfigurationOnDemand,
SwitchContinueOnFailure,
SwitchDaemon,
SwitchDryRun,
SwitchOffline,
SwitchParallel,
SwitchRefreshDependencies,
SwitchRerunTasks,
Custom;

override fun toString(): String {
return if (super.toString().startsWith("Switch")) {
val temp = super.toString().split("Switch")
"switch.${temp[1].decapitalize()}"
} else {
super.toString().decapitalize()
}
}
}
Original file line number Diff line number Diff line change
@@ -1,62 +1,61 @@
package com.cdsap.talaiot.metrics

import com.cdsap.talaiot.entities.ExecutionReport

import com.cdsap.talaiot.metrics.BuildMetrics.*
class DefaultBuildMetricsProvider(
private val report: ExecutionReport
) : ValuesProvider {

override fun get(): Map<String, Any> {
val map = mutableMapOf<String, Any>()

map["duration"] = report.durationMs?.toLong() ?: 0L
map["configuration"] = report.configurationDurationMs?.toLong() ?: 0L
map[Duration.toString()] = report.durationMs?.toLong() ?: 0L
map[Configuration.toString()] = report.configurationDurationMs?.toLong() ?: 0L

with(report) {
success.let { map["success"] = it }
buildId?.let { map["buildId"] = it }
buildInvocationId?.let { map["buildInvocationId"] = it }
requestedTasks?.let { map["requestedTasks"] = it }
cacheRatio?.let { map["cacheRatio"] = it.toDouble() }
beginMs?.let { map["start"] = it.toDouble() }
rootProject?.let { map["rootProject"] = it }
success.let { map[Success.toString()] = it }
buildId?.let { map[BuildId.toString()] = it }
buildInvocationId?.let { map[BuildInvocationId.toString()] = it }
requestedTasks?.let { map[RequestedTasks.toString()] = it }
cacheRatio?.let { map[CacheRatio.toString()] = it.toDouble() }
beginMs?.let { map[Start.toString()] = it.toDouble() }
rootProject?.let { map[RootProject.toString()] = it }
with(environment) {
osVersion?.let { map["osVersion"] = it }
maxWorkers?.let { map["maxWorkers"] = it.toInt() }
javaRuntime?.let { map["javaRuntime"] = it }
javaVmName?.let { map["javaVmName"] = it }
javaXmsBytes?.let { map["javaXmsBytes"] = it.toLong() }
javaXmxBytes?.let { map["javaXmxBytes"] = it.toLong() }
javaMaxPermSize?.let { map["javaMaxPermSize"] = it.toLong() }
totalRamAvailableBytes?.let { map["totalRamAvailableBytes"] = it.toLong() }
cpuCount?.let { map["cpuCount"] = it.toInt() }
locale?.let { map["locale"] = it }
username?.let { map["username"] = it }
publicIp?.let { map["publicIp"] = it }
defaultChartset?.let { map["defaultCharset"] = it }
ideVersion?.let { map["ideVersion"] = it }
gradleVersion?.let { map["gradleVersion"] = it }
gitBranch?.let { map["gitBranch"] = it }
gitUser?.let { map["gitUser"] = it }
hostname?.let { map["hostname"] = it }
osManufacturer?.let { map["osManufacturer"] = it }
publicIp?.let { map["publicIp"] = it }
cacheUrl?.let { map["cacheUrl"] = it }
localCacheHit?.let { map["localCacheHit"] = it.toString() }
localCacheMiss?.let { map["localCacheMiss"] = it.toString() }
remoteCacheHit?.let { map["remoteCacheHit"] = it.toString() }
remoteCacheMiss?.let { map["remoteCacheMiss"] = it.toString() }
cacheStore?.let { map["cacheStore"] = it }
switches.buildCache?.let { map["switch.cache"] = it }
switches.buildScan?.let { map["switch.scan"] = it }
switches.configurationOnDemand?.let { map["switch.configurationOnDemand"] = it }
switches.continueOnFailure?.let { map["switch.continueOnFailure"] = it }
switches.daemon?.let { map["switch.daemon"] = it }
switches.dryRun?.let { map["switch.dryRun"] = it }
switches.offline?.let { map["switch.offline"] = it }
switches.parallel?.let { map["switch.parallel"] = it }
switches.refreshDependencies?.let { map["switch.refreshDependencies"] = it }
switches.rerunTasks?.let { map["switch.rerunTasks"] = it }
osVersion?.let { map[OsVersion.toString()] = it }
maxWorkers?.let { map[MaxWorkers.toString()] = it.toInt() }
javaRuntime?.let { map[JavaRuntime.toString()] = it }
javaVmName?.let { map[JavaVmName.toString()] = it }
javaXmsBytes?.let { map[JavaXmsBytes.toString()] = it.toLong() }
javaXmxBytes?.let { map[JavaXmxBytes.toString()] = it.toLong() }
javaMaxPermSize?.let { map[JavaMaxPermSize.toString()] = it.toLong() }
totalRamAvailableBytes?.let { map[TotalRamAvailableBytes.toString()] = it.toLong() }
cpuCount?.let { map[CpuCount.toString()] = it.toInt() }
locale?.let { map[Locale.toString()] = it }
username?.let { map[Username.toString()] = it }
defaultChartset?.let { map[DefaultCharset.toString()] = it }
ideVersion?.let { map[IdeVersion.toString()] = it }
gradleVersion?.let { map[GradleVersion.toString()] = it }
gitBranch?.let { map[GitBranch.toString()] = it }
gitUser?.let { map[GitUser.toString()] = it }
hostname?.let { map[Hostname.toString()] = it }
osManufacturer?.let { map[OsManufacturer.toString()] = it }
publicIp?.let { map[PublicIp.toString()] = it }
cacheUrl?.let { map[CacheUrl.toString()] = it }
localCacheHit?.let { map[LocalCacheHit.toString()] = it.toString() }
localCacheMiss?.let { map[LocalCacheMiss.toString()] = it.toString() }
remoteCacheHit?.let { map[RemoteCacheHit.toString()] = it.toString() }
remoteCacheMiss?.let { map[RemoteCacheMiss.toString()] = it.toString() }
cacheStore?.let { map[CacheStore.toString()] = it }
switches.buildCache?.let { map[SwitchCache.toString()] = it }
switches.buildScan?.let { map[SwitchScan.toString()] = it }
switches.configurationOnDemand?.let { map[SwitchConfigurationOnDemand.toString()] = it }
switches.continueOnFailure?.let { map[SwitchContinueOnFailure.toString()] = it }
switches.daemon?.let { map[SwitchDaemon.toString()] = it }
switches.dryRun?.let { map[SwitchDryRun.toString()] = it }
switches.offline?.let { map[SwitchOffline.toString()] = it }
switches.parallel?.let { map[SwitchParallel.toString()] = it }
switches.refreshDependencies?.let { map[SwitchRefreshDependencies.toString()] = it }
switches.rerunTasks?.let { map[SwitchRerunTasks.toString()] = it }
}
}
map.putAll(report.customProperties.buildProperties)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,6 @@ talaiotLib {

dependencies {
implementation(project(":library:core:talaiot"))
implementation("org.influxdb:influxdb-java:2.19")
implementation("org.influxdb:influxdb-java:2.21")
testImplementation(project(":library:core:talaiot-test-utils"))
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ package com.cdsap.talaiot.publisher.influxdb

import com.cdsap.talaiot.entities.ExecutionReport
import com.cdsap.talaiot.logger.LogTracker
import com.cdsap.talaiot.metrics.DefaultBuildMetricsProvider
import com.cdsap.talaiot.metrics.DefaultTaskDataProvider
import com.cdsap.talaiot.publisher.Publisher
import okhttp3.OkHttpClient
Expand Down Expand Up @@ -125,18 +124,19 @@ class InfluxDbPublisher(
}

private fun createBuildPoint(report: ExecutionReport): Point {
val metricsProvider = DefaultBuildMetricsProvider(report)
val tagFieldProvider = TagFieldProvider(report,influxDbPublisherConfiguration.tags)
return Point.measurement(influxDbPublisherConfiguration.buildMetricName)
.time(report.endMs?.toLong() ?: System.currentTimeMillis(), TimeUnit.MILLISECONDS)
.fields(metricsProvider.get())
.build()
.time(report.endMs?.toLong() ?: System.currentTimeMillis(), TimeUnit.MILLISECONDS)
.tag(tagFieldProvider.tags())
.fields(tagFieldProvider.fields())
.build()
}

private fun createDb(): InfluxDB {
val okHttpBuilder = OkHttpClient.Builder()
.connectTimeout(TIMEOUT_SEC, TimeUnit.SECONDS)
.readTimeout(TIMEOUT_SEC, TimeUnit.SECONDS)
.writeTimeout(TIMEOUT_SEC, TimeUnit.SECONDS)
.connectTimeout(TIMEOUT_SEC, TimeUnit.SECONDS)
.readTimeout(TIMEOUT_SEC, TimeUnit.SECONDS)
.writeTimeout(TIMEOUT_SEC, TimeUnit.SECONDS)
val user = influxDbPublisherConfiguration.username
val password = influxDbPublisherConfiguration.password
val url = influxDbPublisherConfiguration.url
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package com.cdsap.talaiot.publisher.influxdb

import com.cdsap.talaiot.configuration.PublisherConfiguration
import com.cdsap.talaiot.metrics.BuildMetrics
import groovy.lang.Closure

/**
Expand Down Expand Up @@ -52,6 +53,11 @@ class InfluxDbPublisherConfiguration : PublisherConfiguration {
*/
var retentionPolicyConfiguration: RetentionPolicyConfiguration = RetentionPolicyConfiguration()

/**
* by default all build metrics are considered fields, specify required [BuildMetrics] to be consider as Tags in InfluxDb
*/
var tags : List<BuildMetrics> = emptyList()

/**
* Configuration accessor within the [InfluxDbPublisherConfiguration] for the [com.cdsap.talaiot.configuration.RetentionPolicyConfiguration]
*
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
package com.cdsap.talaiot.publisher.influxdb

import com.cdsap.talaiot.entities.CustomProperties
import com.cdsap.talaiot.entities.Environment
import com.cdsap.talaiot.entities.ExecutionReport
import com.cdsap.talaiot.metrics.BuildMetrics
import com.cdsap.talaiot.metrics.DefaultBuildMetricsProvider
import java.lang.IllegalArgumentException

class TagFieldProvider(
private val executionReport: ExecutionReport,
private val tagsConfiguration: List<BuildMetrics>
) {
private val buildMetrics = DefaultBuildMetricsProvider(executionReport).get()

fun tags(): Map<String, String> = buildMetrics
.filter {
customMetrics(it.key) || defaultMetrics(it.key)
}.mapValues { it.value.toString() }


fun fields(): Map<String, Any> {
val tags = tags()
return buildMetrics.filter { !tags.containsKey(it.key) }
}

private fun customMetrics(key: String) =
tagsConfiguration.contains(BuildMetrics.Custom) &&
executionReport.customProperties.buildProperties.contains(key)

private fun defaultMetrics(key: String) =
try {
tagsConfiguration.contains(BuildMetrics.valueOf(key.capitalize()))
} catch (e: IllegalArgumentException){
false
}
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
package com.cdsap.talaiot.publisher.influxdb

import com.cdsap.talaiot.publisher.influxdb.InfluxDbPublisherConfiguration
import io.kotlintest.specs.BehaviorSpec

class InfluxDbConfigurationTest : BehaviorSpec({
Expand All @@ -10,13 +9,15 @@ class InfluxDbConfigurationTest : BehaviorSpec({
val influxDbPublisherConfiguration = InfluxDbPublisherConfiguration()
then("default values are given") {
assert(
influxDbPublisherConfiguration.retentionPolicyConfiguration.name == "rpTalaiot"
&& influxDbPublisherConfiguration.retentionPolicyConfiguration.duration == "30d"
influxDbPublisherConfiguration.retentionPolicyConfiguration.name == "rpTalaiot" &&
influxDbPublisherConfiguration.retentionPolicyConfiguration.duration == "30d" &&
influxDbPublisherConfiguration.tags.isEmpty()
)
}
}
`when`("There is custom retention policy defined") {
val influxDbPublisherConfiguration = InfluxDbPublisherConfiguration()

influxDbPublisherConfiguration.retentionPolicyConfiguration {
name = "customRp"
duration = "99w"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package com.cdsap.talaiot.publisher.influxdb
import com.cdsap.talaiot.entities.*
import com.cdsap.talaiot.utils.TestExecutor
import com.cdsap.talaiot.logger.TestLogTrackerRecorder
import com.cdsap.talaiot.metrics.BuildMetrics
import io.kotlintest.Spec
import io.kotlintest.specs.BehaviorSpec
import org.influxdb.dto.Query
Expand Down Expand Up @@ -183,7 +184,33 @@ class InfluxDbPublisherTest : BehaviorSpec() {

}
}
`when`("custom metrics are included as tags") {
val databaseTags = "databaseTags"
val influxDbConfiguration = InfluxDbPublisherConfiguration().apply {
dbName = databaseTags
url = container.url
taskMetricName = "task"
buildMetricName = "build"
publishTaskMetrics = false
tags = listOf(BuildMetrics.Custom, BuildMetrics.MaxWorkers)
}
val influxDbPublisher = InfluxDbPublisher(
influxDbConfiguration, logger, TestExecutor()
)

then("build metrics are sent and task metrics doesn't") {
influxDbPublisher.publish(executionReport())

val buildResult =
influxDB.query(Query("select * from $databaseTags.rpTalaiot.build group by *"))
assert(buildResult.results[0].series[0].tags.containsKey(BuildMetrics.MaxWorkers.toString()))
assert(buildResult.results[0].series[0].tags.containsKey("metric3"))
assert(buildResult.results[0].series[0].tags.containsKey("metric4"))
assert(!buildResult.results[0].series[0].columns.contains(BuildMetrics.MaxWorkers.toString()))
assert(!buildResult.results[0].series[0].columns.contains("metric3"))
assert(!buildResult.results[0].series[0].columns.contains("metric4"))
}
}
}
}

Expand Down
Loading

0 comments on commit 3e8a707

Please sign in to comment.