Add tooling to run "oneoff" pods (#44)

as with replicate/web admin tooling. My eventual goal here is to run
monobase builds in each cluster as part of CI on the `main` branch
instead of managing the separate daemonset in replicate/cluster.

Connected to PLAT-582

kubernetes/oneoff/kustomization.yaml

@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - pod.yaml

kubernetes/oneoff/pod.yaml

@@ -0,0 +1,63 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: .
+spec:
+  containers:
+    - name: oneoff
+      image: monobase:latest
+      command: ["sh", "-c", "This default command should be overridden!"]
+      envFrom:
+        - configMapRef:
+            name: fuse-rpc-env
+        - secretRef:
+            name: fuse-rpc-secrets
+      env:
+        - name: R8_CUDA_PREFIX
+          value: https://monobase.replicate.delivery/cuda
+        - name: R8_CUDNN_PREFIX
+          value: https://monobase.replicate.delivery/cudnn
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: NAMESPACE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        - name: OTEL_RESOURCE_ATTRIBUTES
+          value: >-
+            k8s.namespace.name=$(NAMESPACE_NAME),
+            k8s.node.name=$(NODE_NAME),
+            k8s.pod.name=$(POD_NAME),
+            $(EXTRA_OTEL_RESOURCE_ATTRIBUTES)
+        - name: OTEL_SERVICE_NAME
+          value: monobase
+      volumeMounts:
+        - name: monobase-prefix
+          mountPath: /srv/r8/monobase
+        - name: nfd
+          mountPath: /etc/kubernetes/node-feature-discovery
+      tty: true
+      stdin: true
+  nodeSelector:
+    kubernetes.io/arch: amd64
+    node.coreweave.cloud/class: gpu
+  tolerations:
+    - key: arch
+      operator: Equal
+      value: amd64
+      effect: NoSchedule
+  volumes:
+    - name: monobase-prefix
+      hostPath:
+        path: /srv/r8/monobase # kustomize this per provider
+        type: DirectoryOrCreate
+    - name: nfd
+      hostPath:
+        path: /etc/kubernetes/node-feature-discovery
+        type: Directory

script/build-prod

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -o errexit
+set -o pipefail
+set -o nounset
+
+main() {
+  cd "$(git rev-parse --show-toplevel)"
+
+  printf '# ---> using kubectl context "%s"\n' "$(kubectl config current-context)"
+  exec ./script/run-oneoff-pod build /opt/r8/monobase/build.sh "${@}"
+}
+
+main "${@}"

script/run-oneoff-pod

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+set -o errexit
+set -o pipefail
+set -o nounset
+
+main() {
+  local name="${1}"
+  shift
+
+  cd "$(git rev-parse --show-toplevel)"
+
+
+  local podname image fuse_rpc_env overrides
+  podname="monobase-${name}-$(id -u -n | tr A-Z a-z)-$(date +%s)"
+  image="$(_get-image)"
+  fuse_rpc_env="$(_get-fuse-rpc-env)"
+  overrides="$(_generate-overrides shell "${image}" "${fuse_rpc_env}" "${@}")"
+
+  echo "running pod ${podname}"
+  exec kubectl run \
+    --rm -i --tty \
+    --namespace services \
+    --restart=Never \
+    --image="${image}" \
+    --overrides="${overrides}" \
+    "${podname}"
+}
+
+_get-image() {
+  kubectl \
+    --namespace=services \
+    get daemonset/monobase \
+    -o jsonpath='{.spec.template.spec.initContainers[0].image}'
+}
+
+_get-fuse-rpc-env() {
+  kubectl \
+    --namespace=services \
+    get configmaps -o name |
+    awk -F/ '/fuse-rpc-env/ { print $2 }'
+}
+
+_generate-overrides() {
+  local name="${1}"
+  local image="${2}"
+  local fuse_rpc_env="${3}"
+  shift 3
+
+  local host_path='/srv/r8/monobase'
+  case "$(kubectl config current-context)" in
+  tailscale-operator-coreweave-*)
+    host_path='/mnt/local/srv/r8/monobase'
+    ;;
+  esac
+
+  kustomize build kubernetes/oneoff |
+    yq -o json |
+    jq 'del(.metadata) |
+       .spec.containers[0].name = "'${name}'" |
+       .spec.containers[0].image = "'${image}'" |
+       .spec.containers[0].envFrom[0].configMapRef.name = "'${fuse_rpc_env}'" |
+       .spec.volumes[0].hostPath.path = "'${host_path}'" |
+       .spec.containers[0].command = $ARGS.positional' --args -- "${@}"
+}
+
+main "${@}"