feat(sourcegraph): add support to scale worker horizontally

sourcegraph · Nov 20, 2024 · 090699e · 090699e
1 parent 5059ec3
commit 090699e
Show file tree

Hide file tree

Showing 5 changed files with 276 additions and 122 deletions.
diff --git a/charts/sourcegraph/README.md b/charts/sourcegraph/README.md
@@ -362,6 +362,7 @@ In addition to the documented values, all services also support the following va
 | worker.name | string | `"worker"` | Name used by resources. Does not affect service names or PVCs. |
 | worker.podSecurityContext | object | `{}` | Security context for the `worker` pod, learn more from the [Kubernetes documentation](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod) |
 | worker.replicaCount | int | `1` | Number of `worker` pod |
+| worker.replicas | list | `[]` | Scale worker horizontally by configuring additional replicas dedicated to specific jobs. for each replica, configure the dedicated jobs to run on this replica. learn more from https://sourcegraph.com/docs/admin/workers#3-split-jobs-and-scale-independently |
 | worker.resources | object | `{"limits":{"cpu":"2","memory":"4G"},"requests":{"cpu":"500m","memory":"2G"}}` | Resource requests & limits for the `worker` container, learn more from the [Kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) |
 | worker.serviceAccount.create | bool | `false` | Enable creation of ServiceAccount for `worker` |
 | worker.serviceAccount.name | string | `""` | Name of the ServiceAccount to be created or an existing ServiceAccount |
diff --git a/charts/sourcegraph/templates/_worker.tpl b/charts/sourcegraph/templates/_worker.tpl
@@ -0,0 +1,148 @@
+{{- define "sourcegraph.worker" -}}
+{{- $top := index . 0 }}
+{{- $suffix := index . 1 -}}
+{{- $allowlist := index . 2 -}} 
+{{- $blocklist := index . 3 -}}
+{{- $resources := index . 4 -}}
+
+{{- $name := $top.Values.worker.name -}}
+{{- if $suffix -}}
+{{- $name = printf "%s-%s" $name $suffix -}}
+{{- end -}}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  annotations:
+    description: Manages background processes.
+  labels:
+    {{- include "sourcegraph.labels" $top | nindent 4 }}
+    {{- if $top.Values.worker.labels }}
+      {{- toYaml $top.Values.worker.labels | nindent 4 }}
+    {{- end }}
+    deploy: sourcegraph
+    app.kubernetes.io/component: worker
+  name: {{ $name }}
+spec:
+  minReadySeconds: 10
+  replicas: {{ $top.Values.worker.replicaCount }}
+  revisionHistoryLimit: {{ $top.Values.sourcegraph.revisionHistoryLimit }}
+  selector:
+    matchLabels:
+      {{- include "sourcegraph.selectorLabels" $top | nindent 6 }}
+      app: worker
+      {{- if $suffix }}
+      worker-replica: {{ $name | quote }}
+      {{- end }}
+  strategy:
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 1
+    type: RollingUpdate
+  template:
+    metadata:
+      annotations:
+        kubectl.kubernetes.io/default-container: worker
+      {{- include "sourcegraph.redisChecksum" $top | nindent 8 }}
+      {{- if $top.Values.sourcegraph.podAnnotations }}
+      {{- toYaml $top.Values.sourcegraph.podAnnotations | nindent 8 }}
+      {{- end }}
+      {{- if $top.Values.worker.podAnnotations }}
+      {{- toYaml $top.Values.worker.podAnnotations | nindent 8 }}
+      {{- end }}
+      labels:
+      {{- include "sourcegraph.selectorLabels" $top | nindent 8 }}
+      {{- if $top.Values.sourcegraph.podLabels }}
+      {{- toYaml $top.Values.sourcegraph.podLabels | nindent 8 }}
+      {{- end }}
+      {{- if $top.Values.worker.podLabels }}
+      {{- toYaml $top.Values.worker.podLabels | nindent 8 }}
+      {{- end }}
+        deploy: sourcegraph
+        app: worker
+        {{- if $suffix }}
+        worker-replica: {{ $name | quote }}
+        {{- end }}
+    spec:
+      containers:
+      - name: worker
+        env:
+        {{- include "sourcegraph.redisConnection" $top | nindent 8 }}
+        {{- if $allowlist }}
+        - name: WORKER_JOB_ALLOWLIST
+          value: {{ $allowlist }}
+        {{- end }}
+        {{- if $blocklist }}
+        - name: WORKER_JOB_BLOCKLIST
+          value: {{ $blocklist }}
+        {{- end }}
+        {{- range $name, $item := $top.Values.worker.env}}
+        - name: {{ $name }}
+          {{- $item | toYaml | nindent 10 }}
+        {{- end }}
+        {{- if $top.Values.blobstore.enabled }}
+        - name: PRECISE_CODE_INTEL_UPLOAD_BACKEND
+          value: blobstore
+        - name: PRECISE_CODE_INTEL_UPLOAD_AWS_ENDPOINT
+          value: http://blobstore:9000
+        {{- end }}
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        {{- include "sourcegraph.openTelemetryEnv" $top | nindent 8 }}
+        image: {{ include "sourcegraph.image" (list $top "worker" ) }}
+        imagePullPolicy: {{ $top.Values.sourcegraph.image.pullPolicy }}
+        {{- with $top.Values.worker.args }}
+        args:
+          {{- toYaml . | nindent 8 }}
+        {{- end }}
+        terminationMessagePolicy: FallbackToLogsOnError
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: debug
+            scheme: HTTP
+          initialDelaySeconds: 60
+          timeoutSeconds: 5
+        readinessProbe:
+          httpGet:
+            path: /ready
+            port: debug
+            scheme: HTTP
+          periodSeconds: 5
+          timeoutSeconds: 5
+        ports:
+        - containerPort: 3189
+          name: http
+        - containerPort: 6060
+          name: debug
+        - containerPort: 6996
+          name: prom
+        {{- if not $top.Values.sourcegraph.localDevMode }}
+        resources:
+          {{- toYaml $resources | nindent 10 }}
+        {{- end }}
+        securityContext:
+          {{- toYaml $top.Values.worker.containerSecurityContext | nindent 10 }}
+        volumeMounts:
+        {{- if $top.Values.worker.extraVolumeMounts }}
+        {{- toYaml $top.Values.worker.extraVolumeMounts | nindent 8 }}
+        {{- end }}
+      {{- if $top.Values.worker.extraContainers }}
+        {{- toYaml $top.Values.worker.extraContainers | nindent 6 }}
+      {{- end }}
+      securityContext:
+        {{- toYaml $top.Values.worker.podSecurityContext | nindent 8 }}
+      {{- include "sourcegraph.nodeSelector" (list $top "worker" ) | trim | nindent 6 }}
+      {{- include "sourcegraph.affinity" (list $top "worker" ) | trim | nindent 6 }}
+      {{- include "sourcegraph.tolerations" (list $top "worker" ) | trim | nindent 6 }}
+      {{- with $top.Values.sourcegraph.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- include "sourcegraph.renderServiceAccountName" (list $top "worker") | trim | nindent 6 }}
+      volumes:
+      {{- if $top.Values.worker.extraVolumes }}
+      {{- toYaml $top.Values.worker.extraVolumes | nindent 6 }}
+      {{- end }}
+{{- end -}}
diff --git a/charts/sourcegraph/templates/worker/worker.Deployment.yaml b/charts/sourcegraph/templates/worker/worker.Deployment.yaml
@@ -1,122 +1,22 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  annotations:
-    description: Manages background processes.
-  labels:
-    {{- include "sourcegraph.labels" . | nindent 4 }}
-    {{- if .Values.worker.labels }}
-      {{- toYaml .Values.worker.labels | nindent 4 }}
-    {{- end }}
-    deploy: sourcegraph
-    app.kubernetes.io/component: worker
-  name: {{ .Values.worker.name }}
-spec:
-  minReadySeconds: 10
-  replicas: {{ .Values.worker.replicaCount }}
-  revisionHistoryLimit: {{ .Values.sourcegraph.revisionHistoryLimit }}
-  selector:
-    matchLabels:
-      {{- include "sourcegraph.selectorLabels" . | nindent 6 }}
-      app: worker
-  strategy:
-    rollingUpdate:
-      maxSurge: 1
-      maxUnavailable: 1
-    type: RollingUpdate
-  template:
-    metadata:
-      annotations:
-        kubectl.kubernetes.io/default-container: worker
-      {{- include "sourcegraph.redisChecksum" . | nindent 8 }}
-      {{- if .Values.sourcegraph.podAnnotations }}
-      {{- toYaml .Values.sourcegraph.podAnnotations | nindent 8 }}
-      {{- end }}
-      {{- if .Values.worker.podAnnotations }}
-      {{- toYaml .Values.worker.podAnnotations | nindent 8 }}
-      {{- end }}
-      labels:
-      {{- include "sourcegraph.selectorLabels" . | nindent 8 }}
-      {{- if .Values.sourcegraph.podLabels }}
-      {{- toYaml .Values.sourcegraph.podLabels | nindent 8 }}
-      {{- end }}
-      {{- if .Values.worker.podLabels }}
-      {{- toYaml .Values.worker.podLabels | nindent 8 }}
-      {{- end }}
-        deploy: sourcegraph
-        app: worker
-    spec:
-      containers:
-      - name: worker
-        env:
-        {{- include "sourcegraph.redisConnection" .| nindent 8 }}
-        {{- range $name, $item := .Values.worker.env}}
-        - name: {{ $name }}
-          {{- $item | toYaml | nindent 10 }}
-        {{- end }}
-        {{- if .Values.blobstore.enabled }}
-        - name: PRECISE_CODE_INTEL_UPLOAD_BACKEND
-          value: blobstore
-        - name: PRECISE_CODE_INTEL_UPLOAD_AWS_ENDPOINT
-          value: http://blobstore:9000
-        {{- end }}
-        - name: POD_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.name
-        {{- include "sourcegraph.openTelemetryEnv" . | nindent 8 }}
-        image: {{ include "sourcegraph.image" (list . "worker" ) }}
-        imagePullPolicy: {{ .Values.sourcegraph.image.pullPolicy }}
-        {{- with .Values.worker.args }}
-        args:
-          {{- toYaml . | nindent 8 }}
-        {{- end }}
-        terminationMessagePolicy: FallbackToLogsOnError
-        livenessProbe:
-          httpGet:
-            path: /healthz
-            port: debug
-            scheme: HTTP
-          initialDelaySeconds: 60
-          timeoutSeconds: 5
-        readinessProbe:
-          httpGet:
-            path: /ready
-            port: debug
-            scheme: HTTP
-          periodSeconds: 5
-          timeoutSeconds: 5
-        ports:
-        - containerPort: 3189
-          name: http
-        - containerPort: 6060
-          name: debug
-        - containerPort: 6996
-          name: prom
-        {{- if not .Values.sourcegraph.localDevMode }}
-        resources:
-          {{- toYaml .Values.worker.resources | nindent 10 }}
-        {{- end }}
-        securityContext:
-          {{- toYaml .Values.worker.containerSecurityContext | nindent 10 }}
-        volumeMounts:
-        {{- if .Values.worker.extraVolumeMounts }}
-        {{- toYaml .Values.worker.extraVolumeMounts | nindent 8 }}
-        {{- end }}
-      {{- if .Values.worker.extraContainers }}
-        {{- toYaml .Values.worker.extraContainers | nindent 6 }}
-      {{- end }}
-      securityContext:
-        {{- toYaml .Values.worker.podSecurityContext | nindent 8 }}
-      {{- include "sourcegraph.nodeSelector" (list . "worker" ) | trim | nindent 6 }}
-      {{- include "sourcegraph.affinity" (list . "worker" ) | trim | nindent 6 }}
-      {{- include "sourcegraph.tolerations" (list . "worker" ) | trim | nindent 6 }}
-      {{- with .Values.sourcegraph.imagePullSecrets }}
-      imagePullSecrets:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      {{- include "sourcegraph.renderServiceAccountName" (list . "worker") | trim | nindent 6 }}
-      volumes:
-      {{- if .Values.worker.extraVolumes }}
-      {{- toYaml .Values.worker.extraVolumes | nindent 6 }}
-      {{- end }}
+{{- if not .Values.worker.replicas }}
+  {{- include "sourcegraph.worker" (list . "" "" "" .Values.worker.resources ) | nindent 0 }}
+{{- else }}
+  {{- $dedicatedJobs := list }}
+  {{- range .Values.worker.replicas }}
+    {{- $dedicatedJobs = $dedicatedJobs | concat .jobs }}
+  {{- end }}
+  {{- $primaryBlocklist := join "," ($dedicatedJobs | uniq | sortAlpha) }}
+---
+  {{-  include "sourcegraph.worker" (list . "" "all" $primaryBlocklist $.Values.worker.resources) | nindent 0 }}
+
+  {{- range $idx, $item := .Values.worker.replicas }}
+---
+    {{- $replicaName := printf "%d" $idx }}
+    {{- $allowlist := join "," ($item.jobs | uniq | sortAlpha) }}
+    {{- $resources := $.Values.worker.resources -}}
+    {{- if $item.resources -}}
+    {{- $resources = $item.resources -}}
+    {{- end -}}
+    {{- include "sourcegraph.worker" (list $ $replicaName $allowlist "" $resources) | nindent 0 }}
+  {{- end }}
+{{- end }}
diff --git a/charts/sourcegraph/tests/worker_test.yaml b/charts/sourcegraph/tests/worker_test.yaml
@@ -0,0 +1,92 @@
+suite: worker
+release:
+  name: sourcegraph
+  namespace: sourcegraph
+tests:
+- it: should render only the worker deployment when replicas are not set
+  template: worker/worker.Deployment.yaml
+  set:
+    worker:
+      replicas: []
+  asserts:
+  - hasDocuments:
+      count: 1
+- it: should render multiple worker deployment with replicas set
+  template: worker/worker.Deployment.yaml
+  set:
+    worker:
+      replicas:
+      - jobs: ["job1", "job2"]
+      - jobs: ["job3", "job4"]
+  asserts:
+  - hasDocuments:
+      count: 3
+- it: should have the correct env var in primary worker deployment
+  template: worker/worker.Deployment.yaml
+  set:
+    worker:
+      replicas:
+      - jobs: ["job1", "job2"]
+      - jobs: ["job3", "job4"]
+  documentIndex: 0
+  asserts:
+  - contains:
+      path: spec.template.spec.containers[0].env
+      content:
+        name: WORKER_JOB_ALLOWLIST
+        value: all
+  - contains:
+      path: spec.template.spec.containers[0].env
+      content:
+        name: WORKER_JOB_BLOCKLIST
+        value: job1,job2,job3,job4
+  - isEmpty:
+      path: spec.selector.matchLabels.worker-replica
+- it: should have the correct env var in worker-0 deployment
+  template: worker/worker.Deployment.yaml
+  set:
+    worker:
+      replicas:
+      - jobs: ["job1", "job2"]
+      - jobs: ["job3", "job4"]
+  documentIndex: 1
+  asserts:
+  - containsDocument:
+      kind: Deployment
+      apiVersion: apps/v1
+      name: worker-0
+  - contains:
+      path: spec.template.spec.containers[0].env
+      content:
+        name: WORKER_JOB_ALLOWLIST
+        value: job1,job2
+  - equal:
+      path: spec.selector.matchLabels.worker-replica
+      value: worker-0
+  - equal:
+      path: spec.template.metadata.labels.worker-replica
+      value: worker-0
+- it: should have the correct env var in worker-1 deployment
+  template: worker/worker.Deployment.yaml
+  set:
+    worker:
+      replicas:
+      - jobs: ["job1", "job2"]
+      - jobs: ["job3", "job4"]
+  documentIndex: 2
+  asserts:
+  - containsDocument:
+      kind: Deployment
+      apiVersion: apps/v1
+      name: worker-1
+  - contains:
+      path: spec.template.spec.containers[0].env
+      content:
+        name: WORKER_JOB_ALLOWLIST
+        value: job3,job4
+  - equal:
+      path: spec.selector.matchLabels.worker-replica
+      value: worker-1
+  - equal:
+      path: spec.template.metadata.labels.worker-replica
+      value: worker-1