[Bug]: system-upgrade-controller FailedScheduling

[ryadom]

Description

system-upgrade-controller pod is failed to schedule on control plane nodes.
Error message:
0/14 nodes are available: 11 node(s) didn't match Pod's node affinity/selector, 3 node(s) were unschedulable. preemption: 0/14 nodes are available: 14 Preemption is not helpful for scheduling.

pod's tolerations

  tolerations:
    - key: CriticalAddonsOnly
      operator: Exists
    - effect: NoSchedule
      key: node-role.kubernetes.io/master
      operator: Exists
    - effect: NoSchedule
      key: node-role.kubernetes.io/controlplane
      operator: Exists
    - effect: NoSchedule
      key: node-role.kubernetes.io/control-plane
      operator: Exists
    - effect: NoExecute
      key: node-role.kubernetes.io/etcd
      operator: Exists
    - effect: NoExecute
      key: node.kubernetes.io/not-ready
      operator: Exists
      tolerationSeconds: 300
    - effect: NoExecute
      key: node.kubernetes.io/unreachable
      operator: Exists
      tolerationSeconds: 300

Node's taints

spec:
  taints:
    - effect: NoSchedule
      key: node-role.kubernetes.io/control-plane
    - effect: NoSchedule
      key: node.kubernetes.io/unschedulable
      timeAdded: '2024-12-06T18:38:01Z'
  unschedulable: true

Kube.tf file

module "kube-hetzner" {
  providers = {
    hcloud = hcloud
  }
  hcloud_token = var.hcloud_token
  source       = "kube-hetzner/kube-hetzner/hcloud"
  version      = "2.14.5"

  ssh_public_key             = ...
  ssh_private_key            = ...
  ssh_additional_public_keys = ...

  ssh_hcloud_key_label = "kube-${var.env}-admin"

  network_region = "eu-central"

  control_plane_nodepools = [
    {
      name        = "control-plane-fsn1",
      server_type = "cx22",
      location    = "fsn1",
      labels      = [],
      taints      = [],
      count       = 1
    },
    {
      name        = "control-plane-nbg1",
      server_type = "cx22",
      location    = "nbg1",
      labels      = [],
      taints      = [],
      count       = 1
    },
    {
      name        = "control-plane-hel1",
      server_type = "cx22",
      location    = "hel1",
      labels      = [],
      taints      = [],
      count       = 1
    }
  ]

  agent_nodepools = [
    {
      name        = "agent-main",
      server_type = var.agent_main_pool_size,
      location    = var.agent_main_pool_location,
      labels      = [],
      taints      = [],
      count       = var.agent_main_pool_count,
      # placement_group = "1"
    },
    {
      name            = "agent-secondary",
      server_type     = var.agent_secondary_pool_size,
      location        = var.agent_secondary_pool_location,
      labels          = [],
      taints          = [],
      count           = var.agent_secondary_pool_count,
      placement_group = "2"
    }
  ]

  load_balancer_type     = "lb11"
  load_balancer_location = "fsn1"
  base_domain = ...

  enable_longhorn        = true
  longhorn_namespace     = "longhorn-system"
  longhorn_replica_count = 3
  system_upgrade_use_drain = true
  cluster_name = "kube-${var.env}"
  use_cluster_name_in_node_name = true
  restrict_outbound_traffic = false
  dns_servers = [
    "1.1.1.1",
    "8.8.8.8",
    "2606:4700:4700::1111",
  ]
  create_kubeconfig    = false
  create_kustomization = false
}

Screenshots

No response

Platform

Linux

[BrammyS]

I am having the same issue. Tried to upgrade to 2.16.0 but that did not resolve the issue.