diff --git a/01-prerequisites.md b/01-prerequisites.md index 0f3c3115..b293ff3c 100644 --- a/01-prerequisites.md +++ b/01-prerequisites.md @@ -24,6 +24,9 @@ This is the starting point for the instructions on deploying the [AKS Secure Bas [![Launch Azure Cloud Shell](https://docs.microsoft.com/azure/includes/media/cloud-shell-try-it/launchcloudshell.png)](https://shell.azure.com) +1. While the following feature(s) are still in _preview_, please enable them in your subscription. + 1. [Register the Azure Key Vault Secrets Provider for AKS preview feature - `AKS-AzureKeyVaultSecretsProvider`](https://docs.microsoft.com/azure/aks/csi-secrets-store-driver#register-the-aks-azurekeyvaultsecretsprovider-preview-feature). + 1. Clone/download this repo locally, or even better fork this repository. > :twisted_rightwards_arrows: If you have forked this reference implementation repo, you'll be able to customize some of the files and commands for a more personalized and production-like experience; ensure references to this git repository mentioned throughout the walkthrough are updated to use your own fork. diff --git a/README.md b/README.md index 05903f01..19bff7a7 100644 --- a/README.md +++ b/README.md @@ -39,8 +39,8 @@ Finally, this implementation uses the [ASP.NET Core Docker sample web app](https - [Flux GitOps Operator](https://fluxcd.io) - [Traefik Ingress Controller](https://doc.traefik.io/traefik/v2.4/routing/providers/kubernetes-ingress/) -- [Azure AD Pod Identity](https://github.com/Azure/aad-pod-identity) -- [Azure KeyVault Secret Store CSI Provider](https://github.com/Azure/secrets-store-csi-driver-provider-azure) +- [Azure AD Pod Identity](https://docs.microsoft.com/azure/aks/use-azure-ad-pod-identity) +- [Secrets Store CSI Driver for Kubernetes](https://docs.microsoft.com/azure/aks/csi-secrets-store-driver) - [Kured](https://docs.microsoft.com/azure/aks/node-updates-kured) ![Network diagram depicting a hub-spoke network with two peered VNets, each with three subnets and main Azure resources.](https://docs.microsoft.com/azure/architecture/reference-architectures/containers/aks/images/secure-baseline-architecture.svg) diff --git a/cluster-manifests/README.md b/cluster-manifests/README.md index 7235a592..963f95f0 100644 --- a/cluster-manifests/README.md +++ b/cluster-manifests/README.md @@ -12,7 +12,6 @@ This is the root of the GitOps configuration directory. These Kubernetes object * Ingress Network Policy * Flux (self-managing) * Azure Monitor Prometheus Scraping -* Azure KeyVault Secret Store CSI Provider * Azure AD Pod Identity ### Kured diff --git a/cluster-manifests/cluster-baseline-settings/akv-secrets-store-csi.yaml b/cluster-manifests/cluster-baseline-settings/akv-secrets-store-csi.yaml deleted file mode 100644 index e0e0e776..00000000 --- a/cluster-manifests/cluster-baseline-settings/akv-secrets-store-csi.yaml +++ /dev/null @@ -1,510 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: secrets-store-csi-driver - namespace: cluster-baseline-settings - labels: - app.kubernetes.io/name: secrets-store-csi-driver - app.kubernetes.io/component: csi-driver ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: csi-secrets-store-provider-azure - namespace: cluster-baseline-settings - labels: - app.kubernetes.io/name: csi-secrets-store-provider-azure - app.kubernetes.io/component: csi-provider ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - creationTimestamp: null - name: secretproviderclasses-role - labels: - app.kubernetes.io/name: secrets-store-csi-driver - app.kubernetes.io/component: csi-driver -rules: -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch -- apiGroups: - - "" - resources: - - pods - verbs: - - get - - list - - watch -- apiGroups: - - secrets-store.csi.x-k8s.io - resources: - - secretproviderclasses - verbs: - - get - - list - - watch -- apiGroups: - - secrets-store.csi.x-k8s.io - resources: - - secretproviderclasspodstatuses - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - secrets-store.csi.x-k8s.io - resources: - - secretproviderclasspodstatuses/status - verbs: - - get - - patch - - update ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: secretproviderclasses-rolebinding - labels: - app.kubernetes.io/name: secrets-store-csi-driver - app.kubernetes.io/component: csi-driver -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: secretproviderclasses-role -subjects: -- kind: ServiceAccount - name: secrets-store-csi-driver - namespace: cluster-baseline-settings ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - creationTimestamp: null - name: secretprovidersyncing-role - labels: - app.kubernetes.io/name: secrets-store-csi-driver - app.kubernetes.io/component: csi-driver -rules: -- apiGroups: - - "" - resources: - - secrets - verbs: - - create - - delete - - get - - list - - patch - - update - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: secretprovidersyncing-rolebinding - labels: - app.kubernetes.io/name: secrets-store-csi-driver - app.kubernetes.io/component: csi-driver -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: secretprovidersyncing-role -subjects: -- kind: ServiceAccount - name: secrets-store-csi-driver - namespace: cluster-baseline-settings ---- -apiVersion: storage.k8s.io/v1 -kind: CSIDriver -metadata: - name: secrets-store.csi.k8s.io - labels: - app.kubernetes.io/name: secrets-store-csi-driver - app.kubernetes.io/component: csi-driver -spec: - podInfoOnMount: true - attachRequired: false - volumeLifecycleModes: - - Ephemeral ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.4.0 - creationTimestamp: null - name: secretproviderclasses.secrets-store.csi.x-k8s.io - labels: - app.kubernetes.io/name: secrets-store-csi-driver-secretproviderclasses-crd - app.kubernetes.io/component: csi-driver -spec: - group: secrets-store.csi.x-k8s.io - names: - kind: SecretProviderClass - listKind: SecretProviderClassList - plural: secretproviderclasses - singular: secretproviderclass - scope: Namespaced - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: SecretProviderClass is the Schema for the secretproviderclasses - API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: SecretProviderClassSpec defines the desired state of SecretProviderClass - properties: - parameters: - additionalProperties: - type: string - description: Configuration for specific provider - type: object - provider: - description: Configuration for provider name - type: string - secretObjects: - items: - description: SecretObject defines the desired state of synced K8s - secret objects - properties: - data: - items: - description: SecretObjectData defines the desired state of - synced K8s secret object data - properties: - key: - description: data field to populate - type: string - objectName: - description: name of the object to sync - type: string - type: object - type: array - labels: - additionalProperties: - type: string - description: labels of K8s secret object - type: object - secretName: - description: name of the K8s secret object - type: string - type: - description: type of K8s secret object - type: string - type: object - type: array - type: object - status: - description: SecretProviderClassStatus defines the observed state of SecretProviderClass - properties: - byPod: - items: - description: ByPodStatus defines the state of SecretProviderClass - as seen by an individual controller - properties: - id: - description: id of the pod that wrote the status - type: string - namespace: - description: namespace of the pod that wrote the status - type: string - type: object - type: array - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.4.0 - creationTimestamp: null - name: secretproviderclasspodstatuses.secrets-store.csi.x-k8s.io - labels: - app.kubernetes.io/name: secrets-store-csi-driver-secretproviderclasspodstatuses-crd - app.kubernetes.io/component: csi-driver -spec: - group: secrets-store.csi.x-k8s.io - names: - kind: SecretProviderClassPodStatus - listKind: SecretProviderClassPodStatusList - plural: secretproviderclasspodstatuses - singular: secretproviderclasspodstatus - scope: Namespaced - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: SecretProviderClassPodStatus is the Schema for the secretproviderclassespodstatus - API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - status: - description: SecretProviderClassPodStatusStatus defines the observed state - of SecretProviderClassPodStatus - properties: - mounted: - type: boolean - objects: - items: - description: SecretProviderClassObject defines the object fetched - from external secrets store - properties: - id: - type: string - version: - type: string - type: object - type: array - podName: - type: string - secretProviderClassName: - type: string - targetPath: - type: string - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: csi-secrets-store - namespace: cluster-baseline-settings - labels: - app: csi-secrets-store - app.kubernetes.io/name: csi-secrets-store - app.kubernetes.io/component: csi-driver -spec: - selector: - matchLabels: - app: csi-secrets-store - template: - metadata: - labels: - app: csi-secrets-store - app.kubernetes.io/name: csi-secrets-store - app.kubernetes.io/component: csi-driver - spec: - nodeSelector: - kubernetes.io/os: linux - agentpool: npuser01 - serviceAccountName: secrets-store-csi-driver - hostNetwork: true - containers: - - name: node-driver-registrar - image: mcr.microsoft.com/oss/kubernetes-csi/csi-node-driver-registrar:v2.0.1 - args: - - --v=5 - - --csi-address=/csi/csi.sock - - --kubelet-registration-path=/var/lib/kubelet/plugins/csi-secrets-store/csi.sock - env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - imagePullPolicy: Always - volumeMounts: - - name: plugin-dir - mountPath: /csi - - name: registration-dir - mountPath: /registration - resources: - limits: - cpu: 100m - memory: 100Mi - requests: - cpu: 10m - memory: 20Mi - - name: secrets-store - image: mcr.microsoft.com/oss/kubernetes-csi/secrets-store/driver:v0.0.20 - args: - - "--debug=false" - - "--endpoint=$(CSI_ENDPOINT)" - - "--nodeid=$(KUBE_NODE_NAME)" - - "--provider-volume=/etc/kubernetes/secrets-store-csi-providers" - - "--metrics-addr=:8095" - - "--enable-secret-rotation=false" # To use this alpha feature, follow guidance at https://github.com/kubernetes-sigs/secrets-store-csi-driver/blob/master/docs/README.rotation.md - - "--rotation-poll-interval=5m" - - "--grpc-supported-providers=azure" - env: - - name: CSI_ENDPOINT - value: unix:///csi/csi.sock - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - imagePullPolicy: Always - securityContext: - privileged: true - ports: - - containerPort: 9808 - name: healthz - protocol: TCP - livenessProbe: - failureThreshold: 5 - httpGet: - path: /healthz - port: healthz - initialDelaySeconds: 30 - timeoutSeconds: 10 - periodSeconds: 15 - volumeMounts: - - name: plugin-dir - mountPath: /csi - - name: mountpoint-dir - mountPath: /var/lib/kubelet/pods - mountPropagation: Bidirectional - - name: providers-dir - mountPath: /etc/kubernetes/secrets-store-csi-providers - resources: - limits: - cpu: 200m - memory: 200Mi - requests: - cpu: 50m - memory: 100Mi - - name: liveness-probe - image: mcr.microsoft.com/oss/kubernetes-csi/livenessprobe:v2.2.0 - imagePullPolicy: Always - args: - - --csi-address=/csi/csi.sock - - --probe-timeout=3s - - --health-port=9808 - - -v=2 - volumeMounts: - - name: plugin-dir - mountPath: /csi - resources: - limits: - cpu: 100m - memory: 100Mi - requests: - cpu: 10m - memory: 20Mi - volumes: - - name: mountpoint-dir - hostPath: - path: /var/lib/kubelet/pods - type: DirectoryOrCreate - - name: registration-dir - hostPath: - path: /var/lib/kubelet/plugins_registry/ - type: Directory - - name: plugin-dir - hostPath: - path: /var/lib/kubelet/plugins/csi-secrets-store/ - type: DirectoryOrCreate - - name: providers-dir - hostPath: - path: /etc/kubernetes/secrets-store-csi-providers - type: DirectoryOrCreate ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: csi-secrets-store-provider-azure - namespace: cluster-baseline-settings - labels: - app: csi-secrets-store-provider-azure - app.kubernetes.io/name: csi-secrets-store-provider-azure - app.kubernetes.io/component: csi-provider -spec: - updateStrategy: - type: RollingUpdate - selector: - matchLabels: - app: csi-secrets-store-provider-azure - template: - metadata: - labels: - app: csi-secrets-store-provider-azure - app.kubernetes.io/name: csi-secrets-store-provider-azure - app.kubernetes.io/component: csi-provider - spec: - serviceAccountName: csi-secrets-store-provider-azure - hostNetwork: true - containers: - - name: provider-azure-installer - image: mcr.microsoft.com/oss/azure/secrets-store/provider-azure:0.0.13 - imagePullPolicy: IfNotPresent - args: - - --endpoint=unix:///provider/azure.sock - lifecycle: - preStop: - exec: - command: - - "rm /provider/azure.sock" - resources: - requests: - cpu: 50m - memory: 100Mi - limits: - cpu: 50m - memory: 100Mi - volumeMounts: - - mountPath: "/provider" - name: providervol - - name: mountpoint-dir - mountPath: /var/lib/kubelet/pods - mountPropagation: HostToContainer - volumes: - - name: providervol - hostPath: - path: "/etc/kubernetes/secrets-store-csi-providers" - - name: mountpoint-dir - hostPath: - path: /var/lib/kubelet/pods - nodeSelector: - kubernetes.io/os: linux - agentpool: npuser01 diff --git a/cluster-stamp.json b/cluster-stamp.json index 4f3b1e41..1450a406 100644 --- a/cluster-stamp.json +++ b/cluster-stamp.json @@ -1093,6 +1093,12 @@ "config": { "version": "v2" } + }, + "azureKeyvaultSecretsProvider": { + "enabled": true, + "config": { + "enableSecretRotation": "false" + } } }, "nodeResourceGroup": "[variables('nodeResourceGroupName')]",