Update JFR image and make logs plugin params configurable (#406)

SAP · Sep 1, 2023 · 4b167fc · 4b167fc
1 parent cc37e30
commit 4b167fc
Show file tree

Hide file tree

Showing 5 changed files with 60 additions and 8 deletions.
diff --git a/changelog.yaml b/changelog.yaml
@@ -52,6 +52,16 @@
   date: TBD
   changes:
 
+    - type: enhancement
+      impact: minor
+      title: Update stewardci-jenkinsfile-runner image to version '230825_1c09345'
+      description: |-
+        The new image contains new version "0.12.0" of elasticsearch-logs-plugin
+        which enables configuring more parameters.
+
+        See release note for [version 230825_1c09345](https://github.com/SAP/stewardci-jenkinsfilerunner-image/releases/tag/230825_1c09345)
+      pullRequestNumber: 406
+
 - version: "0.36.0"
   date: 2023-08-22
   changes:

diff --git a/charts/steward/Chart.yaml b/charts/steward/Chart.yaml
@@ -5,8 +5,8 @@ type: application
 
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 0.36.1-dev
+version: 0.37.0-dev
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
-appVersion: 0.36.1-dev
+appVersion: 0.37.0-dev
diff --git a/charts/steward/templates/_helpers.tpl b/charts/steward/templates/_helpers.tpl
@@ -108,3 +108,10 @@ to the empty string.
 true
 {{- end -}}
 {{- end -}}
+
+{{- define "steward.provideAsKeyValueEnv" -}}
+{{- if .dictValue -}}
+- name: {{ .dictKey }}
+  value: {{ .dictValue | int | squote }}
+{{- end -}}
+{{- end -}}
diff --git a/charts/steward/templates/task-jenkinsfile-runner.yaml b/charts/steward/templates/task-jenkinsfile-runner.yaml
@@ -123,20 +123,39 @@ spec:
       value: '$(params.RUN_CAUSE)'
     - name: TERMINATION_LOG_PATH
       value: /tekton/results/jfr-termination-log
-    {{ with .Values.pipelineRuns.logging.forwarding }}{{ if .enabled }}
+    {{- include "steward.provideAsKeyValueEnv" (dict "dictKey" "PIPELINE_LOGS_PLUGIN_SPLIT_MESSAGES_LONGER_THAN" "dictValue" .Values.pipelineRuns.logging.elasticsearch.splitMessagesLongerThan) | nindent 4 }}
+    {{- with .Values.pipelineRuns.logging.forwarding }}{{- if .enabled }}
     - name: PIPELINE_LOG_FLUENTD_HOST
-      {{ if .useSidecar }}
+      {{- if .useSidecar }}
       valueFrom:
         fieldRef:
           fieldPath: status.podIP
-      {{ else }}
+      {{- else }}
       value: {{ required "the host name of the log-forwarder" .host  }}
-      {{ end }}
+      {{- end }}
     - name: PIPELINE_LOG_FLUENTD_PORT
       value: {{ .port | quote }}
     - name: PIPELINE_LOG_FLUENTD_TAG
       value: {{ .tag }}
-    {{ end }}{{ end}}
+    - name: PIPELINE_LOGS_PLUGIN_SENDER_BASE_RETRY_INTERVAL_MILLIS
+      value: {{ required "the senderBaseRetryIntervalMillis parameter in the 'elasticsearch-logs-plugin'" .senderBaseRetryIntervalMillis | int | squote }}
+    - name: PIPELINE_LOGS_PLUGIN_SENDER_MAX_RETRY_INTERVAL_MILLIS
+      value: {{ required "the senderMaxRetryIntervalMillis parameter in the 'elasticsearch-logs-plugin'" .senderMaxRetryIntervalMillis | int | squote }}
+    - name: PIPELINE_LOGS_PLUGIN_SENDER_MAX_RETRY_COUNT
+      value: {{ required "the senderMaxRetryCount parameter in the 'elasticsearch-logs-plugin'" .senderMaxRetryCount | int | squote }}
+    - name: PIPELINE_LOGS_PLUGIN_CONNECTION_TIMEOUT_MILLIS
+      value: {{ required "the connectionTimeoutMillis parameter in the 'elasticsearch-logs-plugin'" .connectionTimeoutMillis | int | squote }}
+    - name: PIPELINE_LOGS_PLUGIN_READ_TIMEOUT_MILLIS
+      value: {{ required "the readTimeoutMillis parameter in the 'elasticsearch-logs-plugin'" .readTimeoutMillis | int | squote }}
+    {{- include "steward.provideAsKeyValueEnv" (dict "dictKey" "PIPELINE_LOGS_PLUGIN_BUFFER_CHUNK_INITIAL_SIZE" "dictValue" .bufferChunkInitialSize) | nindent 4 }}
+    {{- include "steward.provideAsKeyValueEnv" (dict "dictKey" "PIPELINE_LOGS_PLUGIN_BUFFER_CHUNK_RETENTION_SIZE" "dictValue" .bufferChunkRetentionSize) | nindent 4 }}
+    {{- include "steward.provideAsKeyValueEnv" (dict "dictKey" "PIPELINE_LOGS_PLUGIN_MAX_BUFFER_SIZE" "dictValue" .maxBufferSize) | nindent 4 }}
+    {{- include "steward.provideAsKeyValueEnv" (dict "dictKey" "PIPELINE_LOGS_PLUGIN_BUFFER_CHUNK_RETENTION_TIME_MILLIS" "dictValue" .bufferChunkRetentionTimeMillis) | nindent 4 }}
+    {{- include "steward.provideAsKeyValueEnv" (dict "dictKey" "PIPELINE_LOGS_PLUGIN_MAX_WAIT_SECONDS_UNTIL_BUFFER_FLUSHED" "dictValue" .maxWaitSecondsUntilBufferFlushed) | nindent 4 }}
+    {{- include "steward.provideAsKeyValueEnv" (dict "dictKey" "PIPELINE_LOGS_PLUGIN_MAX_WAIT_SECONDS_UNTIL_FLUSHER_TERMINATED" "dictValue" .maxWaitSecondsUntilFlusherTerminated) | nindent 4 }}
+    {{- include "steward.provideAsKeyValueEnv" (dict "dictKey" "PIPELINE_LOGS_PLUGIN_FLUSH_ATTEMPT_INTERVAL_MILLIS" "dictValue" .flushAttemptIntervalMillis) | nindent 4 }}
+    {{- include "steward.provideAsKeyValueEnv" (dict "dictKey" "PIPELINE_LOGS_PLUGIN_EMIT_MAX_RETRIES_IF_BUFFER_FULL" "dictValue" .emitMaxRetriesIfBufferFull) | nindent 4 }}
+    {{- end }}{{- end}}
     resources:
       {{- toYaml .Values.pipelineRuns.jenkinsfileRunner.resources | nindent 6 }}
     terminationMessagePath: /tekton/results/jfr-termination-log

diff --git a/charts/steward/values.yaml b/charts/steward/values.yaml
@@ -48,14 +48,30 @@ pipelineRuns:
   logging:
     elasticsearch:
       indexURL: ""
+      splitMessagesLongerThan: 1048576
     forwarding:
       enabled: false
       host: ""
       useSidecar: true
       port: '24224'
       tag: BUILD_LOGS
+      ## mandatory parameters of the elasticsearch-logs-plugin
+      senderBaseRetryIntervalMillis: 1000
+      senderMaxRetryIntervalMillis: 1000
+      senderMaxRetryCount: 30
+      connectionTimeoutMillis: 3000
+      readTimeoutMillis: 3000
+      ## optional parameters of the elasticsearch-logs-plugin
+      #  flushAttemptIntervalMillis: 100
+      #  bufferChunkInitialSize: 1048576 # 1 MB
+      #  bufferChunkRetentionSize: 3145728 # 3 MB
+      #  maxBufferSize: 10485760 # 10 MB
+      #  bufferChunkRetentionTimeMillis: 1000
+      #  maxWaitSecondsUntilBufferFlushed: 30
+      #  maxWaitSecondsUntilFlusherTerminated: 30
+      #  emitMaxRetriesIfBufferFull: -1 # forever
   jenkinsfileRunner:
-    image: "stewardci/stewardci-jenkinsfile-runner:230822_3c378b2"
+    image: "stewardci/stewardci-jenkinsfile-runner:230825_1c09345"
     imagePullPolicy: IfNotPresent
     javaOpts: >-
       -Dhudson.slaves.NodeProvisioner.initialDelay=0