update to latest 2.26.0 kubespray version and add dns zones settings to coredns

kubespray/project/roles/kubernetes-apps/ansible/defaults/main.yml

@@ -16,8 +16,6 @@ coredns_ordinal_suffix: ""
 coredns_deployment_nodeselector: "kubernetes.io/os: linux"
 coredns_default_zone_cache_block: |
   cache 30
-coredns_host_network: false
-coredns_port: 53
 
 coredns_pod_disruption_budget: false
 # value for coredns pdb
@@ -96,6 +94,11 @@ netchecker_server_user: 1000
 netchecker_agent_group: 1000
 netchecker_server_group: 1000
 
+# Log levels
+netchecker_agent_log_level: 5
+netchecker_server_log_level: 5
+netchecker_etcd_log_level: info
+
 # Dashboard
 dashboard_replicas: 1
 

kubespray/project/roles/kubernetes-apps/ansible/tasks/coredns.yml

@@ -3,7 +3,7 @@
   template:
     src: "{{ item.file }}.j2"
     dest: "{{ kube_config_dir }}/{{ item.file }}"
-    mode: 0644
+    mode: "0644"
   loop:
     - { name: coredns, file: coredns-clusterrole.yml, type: clusterrole }
     - { name: coredns, file: coredns-clusterrolebinding.yml, type: clusterrolebinding }
@@ -31,7 +31,7 @@
   template:
     src: "{{ item.src }}.j2"
     dest: "{{ kube_config_dir }}/{{ item.file }}"
-    mode: 0644
+    mode: "0644"
   with_items:
     - { name: coredns, src: coredns-deployment.yml, file: coredns-deployment-secondary.yml, type: deployment }
     - { name: coredns, src: coredns-svc.yml, file: coredns-svc-secondary.yml, type: svc }

kubespray/project/roles/kubernetes-apps/ansible/tasks/dashboard.yml

@@ -3,7 +3,7 @@
   template:
     src: "{{ item.file }}.j2"
     dest: "{{ kube_config_dir }}/{{ item.file }}"
-    mode: 0644
+    mode: "0644"
   with_items:
     - { file: dashboard.yml, type: deploy, name: kubernetes-dashboard }
   register: manifests

kubespray/project/roles/kubernetes-apps/ansible/tasks/etcd_metrics.yml

@@ -3,7 +3,7 @@
   template:
     src: "{{ item.file }}.j2"
     dest: "{{ kube_config_dir }}/{{ item.file }}"
-    mode: 0644
+    mode: "0644"
   with_items:
     - { file: etcd_metrics-endpoints.yml, type: endpoints, name: etcd-metrics }
     - { file: etcd_metrics-service.yml, type: service, name: etcd-metrics }

kubespray/project/roles/kubernetes-apps/ansible/tasks/main.yml

@@ -2,7 +2,7 @@
 - name: Kubernetes Apps | Wait for kube-apiserver
   uri:
     url: "{{ kube_apiserver_endpoint }}/healthz"
-    validate_certs: no
+    validate_certs: false
     client_cert: "{{ kube_apiserver_client_cert }}"
     client_key: "{{ kube_apiserver_client_key }}"
   register: result

kubespray/project/roles/kubernetes-apps/ansible/tasks/netchecker.yml

@@ -29,7 +29,7 @@
   template:
     src: "{{ item.file }}.j2"
     dest: "{{ kube_config_dir }}/{{ item.file }}"
-    mode: 0644
+    mode: "0644"
   with_items: "{{ netchecker_templates }}"
   register: manifests
   when:

kubespray/project/roles/kubernetes-apps/ansible/tasks/nodelocaldns.yml

@@ -20,7 +20,7 @@
   template:
     src: "{{ item.file }}.j2"
     dest: "{{ kube_config_dir }}/{{ item.file }}"
-    mode: 0644
+    mode: "0644"
   with_items:
     - { name: nodelocaldns, file: nodelocaldns-config.yml, type: configmap }
     - { name: nodelocaldns, file: nodelocaldns-sa.yml, type: sa }
@@ -51,7 +51,7 @@
   template:
     src: "{{ item.file }}.j2"
     dest: "{{ kube_config_dir }}/{{ item.file }}"
-    mode: 0644
+    mode: "0644"
   with_items:
     - { name: nodelocaldns, file: nodelocaldns-second-daemonset.yml, type: daemonset }
   register: nodelocaldns_second_manifests

kubespray/project/roles/kubernetes-apps/ansible/templates/coredns-config.yml.j2

@@ -34,7 +34,7 @@ data:
     }
 {%   endfor %}
 {% endif %}
-    .:{{ coredns_port }} {
+    .:53 {
 {% if coredns_additional_configs is defined %}
         {{ coredns_additional_configs | indent(width=8, first=False) }}
 {% endif %}

kubespray/project/roles/kubernetes-apps/ansible/templates/coredns-deployment.yml.j2

@@ -24,7 +24,6 @@ spec:
       annotations:
         createdby: 'kubespray'
     spec:
-      hostNetwork: {{ coredns_host_network | default(false) }}
       securityContext:
         seccompProfile:
           type: RuntimeDefault
@@ -76,10 +75,10 @@ spec:
         - name: config-volume
           mountPath: /etc/coredns
         ports:
-        - containerPort: {{ coredns_port }}
+        - containerPort: 53
           name: dns
           protocol: UDP
-        - containerPort: {{ coredns_port }}
+        - containerPort: 53
           name: dns-tcp
           protocol: TCP
         - containerPort: 9153

kubespray/project/roles/kubernetes-apps/ansible/templates/coredns-svc.yml.j2

@@ -20,11 +20,9 @@ spec:
     - name: dns
       port: 53
       protocol: UDP
-      targetPort: "dns"
     - name: dns-tcp
       port: 53
       protocol: TCP
-      targetPort: "dns-tcp"
     - name: metrics
       port: 9153
       protocol: TCP

kubespray/project/roles/kubernetes-apps/ansible/templates/dashboard.yml.j2

@@ -247,7 +247,7 @@ spec:
     k8s-app: kubernetes-dashboard
 
 ---
-# ------------------- Metrics Scrapper Service Account ------------------- #
+# ------------------- Metrics Scraper Service Account ------------------- #
 
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
@@ -263,7 +263,7 @@ rules:
 
 ---
 
-# ------------------- Metrics Scrapper Service  ------------------- #
+# ------------------- Metrics Scraper Service  ------------------- #
 kind: Service
 apiVersion: v1
 metadata:
@@ -280,7 +280,7 @@ spec:
 
 ---
 
-# ------------------- Metrics Scrapper Deployment  ------------------- #
+# ------------------- Metrics Scraper Deployment  ------------------- #
 kind: Deployment
 apiVersion: apps/v1
 metadata:

kubespray/project/roles/kubernetes-apps/ansible/templates/netchecker-agent-ds.yml.j2

@@ -35,7 +35,7 @@ spec:
                 fieldRef:
                   fieldPath: spec.nodeName
           args:
-            - "-v=5"
+            - "-v={{ netchecker_agent_log_level }}"
             - "-alsologtostderr=true"
             - "-serverendpoint=netchecker-service:8081"
             - "-reportinterval={{ agent_report_interval }}"

kubespray/project/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.yml.j2

@@ -37,7 +37,7 @@ spec:
                 fieldRef:
                   fieldPath: spec.nodeName
           args:
-            - "-v=5"
+            - "-v={{ netchecker_agent_log_level }}"
             - "-alsologtostderr=true"
             - "-serverendpoint=netchecker-service:8081"
             - "-reportinterval={{ agent_report_interval }}"

kubespray/project/roles/kubernetes-apps/ansible/templates/netchecker-server-deployment.yml.j2

@@ -43,7 +43,7 @@ spec:
           ports:
             - containerPort: 8081
           args:
-            - -v=5
+            - -v={{ netchecker_server_log_level }}
             - -logtostderr
             - -kubeproxyinit=false
             - -endpoint=0.0.0.0:8081
@@ -51,6 +51,9 @@ spec:
         - name: etcd
           image: "{{ etcd_image_repo }}:{{ netcheck_etcd_image_tag }}"
           imagePullPolicy: {{ k8s_image_pull_policy }}
+          env:
+            - name: ETCD_LOG_LEVEL
+              value: "{{ netchecker_etcd_log_level }}"
           command:
             - etcd
             - --listen-client-urls=http://127.0.0.1:2379

kubespray/project/roles/kubernetes-apps/ansible/templates/nodelocaldns-daemonset.yml.j2

@@ -20,7 +20,7 @@ spec:
     spec:
       nodeSelector:
         {{ nodelocaldns_ds_nodeselector }}
-      priorityClassName: system-cluster-critical
+      priorityClassName: system-node-critical
       serviceAccountName: nodelocaldns
       hostNetwork: true
       dnsPolicy: Default  # Don't use cluster DNS.

kubespray/project/roles/kubernetes-apps/argocd/defaults/main.yml

@@ -1,6 +1,6 @@
 ---
 argocd_enabled: false
-argocd_version: v2.8.4
+argocd_version: v2.11.0
 argocd_namespace: argocd
 # argocd_admin_password:
 argocd_install_url: "https://raw.githubusercontent.com/argoproj/argo-cd/{{ argocd_version }}/manifests/install.yaml"

kubespray/project/roles/kubernetes-apps/argocd/tasks/main.yml

@@ -8,10 +8,10 @@
   ansible.posix.synchronize:
     src: "{{ downloads.yq.dest }}"
     dest: "{{ bin_dir }}/yq"
-    compress: no
-    perms: yes
-    owner: no
-    group: no
+    compress: false
+    perms: true
+    owner: false
+    group: false
   delegate_to: "{{ inventory_hostname }}"
 
 - name: Kubernetes Apps | Set ArgoCD template list
@@ -36,7 +36,7 @@
       url: "{{ item.url }}"
       unarchive: false
       owner: "root"
-      mode: 0644
+      mode: "0644"
       sha256: ""
     download: "{{ download_defaults | combine(download_argocd) }}"
   with_items: "{{ argocd_templates | selectattr('url', 'defined') | list }}"
@@ -49,17 +49,17 @@
   ansible.posix.synchronize:
     src: "{{ local_release_dir }}/{{ item.file }}"
     dest: "{{ kube_config_dir }}/{{ item.file }}"
-    compress: no
-    perms: yes
-    owner: no
-    group: no
+    compress: false
+    perms: true
+    owner: false
+    group: false
   delegate_to: "{{ inventory_hostname }}"
   with_items: "{{ argocd_templates | selectattr('url', 'defined') | list }}"
   when:
     - "inventory_hostname == groups['kube_control_plane'][0]"
 
 - name: Kubernetes Apps | Set ArgoCD namespace for remote manifests
-  become: yes
+  become: true
   command: |
     {{ bin_dir }}/yq eval-all -i '.metadata.namespace="{{ argocd_namespace }}"' {{ kube_config_dir }}/{{ item.file }}
   with_items: "{{ argocd_templates | selectattr('url', 'defined') | list }}"
@@ -69,11 +69,11 @@
     - "inventory_hostname == groups['kube_control_plane'][0]"
 
 - name: Kubernetes Apps | Create ArgoCD manifests from templates
-  become: yes
+  become: true
   template:
     src: "{{ item.file }}.j2"
     dest: "{{ kube_config_dir }}/{{ item.file }}"
-    mode: 0644
+    mode: "0644"
   with_items: "{{ argocd_templates | selectattr('url', 'undefined') | list }}"
   loop_control:
     label: "{{ item.file }}"
@@ -81,7 +81,7 @@
     - "inventory_hostname == groups['kube_control_plane'][0]"
 
 - name: Kubernetes Apps | Install ArgoCD
-  become: yes
+  become: true
   kube:
     name: ArgoCD
     kubectl: "{{ bin_dir }}/kubectl"
@@ -93,7 +93,7 @@
 
 # https://github.com/argoproj/argo-cd/blob/master/docs/faq.md#i-forgot-the-admin-password-how-do-i-reset-it
 - name: Kubernetes Apps | Set ArgoCD custom admin password
-  become: yes
+  become: true
   shell: |
     {{ bin_dir }}/kubectl --kubeconfig /etc/kubernetes/admin.conf -n {{ argocd_namespace }} patch secret argocd-secret -p \
       '{

kubespray/project/roles/kubernetes-apps/cloud_controller/oci/tasks/main.yml

@@ -7,7 +7,7 @@
   template:
     src: controller-manager-config.yml.j2
     dest: "{{ kube_config_dir }}/controller-manager-config.yml"
-    mode: 0644
+    mode: "0644"
   when: inventory_hostname == groups['kube_control_plane'][0]
 
 - name: "OCI Cloud Controller | Slurp Configuration"
@@ -24,7 +24,7 @@
   template:
     src: oci-cloud-provider.yml.j2
     dest: "{{ kube_config_dir }}/oci-cloud-provider.yml"
-    mode: 0644
+    mode: "0644"
   when: inventory_hostname == groups['kube_control_plane'][0]
 
 - name: "OCI Cloud Controller | Apply Manifests"

kubespray/project/roles/kubernetes-apps/cluster_roles/tasks/main.yml

@@ -2,7 +2,7 @@
 - name: Kubernetes Apps | Wait for kube-apiserver
   uri:
     url: "{{ kube_apiserver_endpoint }}/healthz"
-    validate_certs: no
+    validate_certs: false
     client_cert: "{{ kube_apiserver_client_cert }}"
     client_key: "{{ kube_apiserver_client_key }}"
   register: result
@@ -15,7 +15,7 @@
   template:
     src: "node-crb.yml.j2"
     dest: "{{ kube_config_dir }}/node-crb.yml"
-    mode: 0640
+    mode: "0640"
   register: node_crb_manifest
   when:
     - rbac_enabled
@@ -70,7 +70,7 @@
   copy:
     src: k8s-cluster-critical-pc.yml
     dest: "{{ kube_config_dir }}/k8s-cluster-critical-pc.yml"
-    mode: 0640
+    mode: "0640"
   when: inventory_hostname == groups['kube_control_plane'] | last
 
 - name: PriorityClass | Create k8s-cluster-critical

kubespray/project/roles/kubernetes-apps/cluster_roles/tasks/oci.yml

@@ -3,7 +3,7 @@
   copy:
     src: "oci-rbac.yml"
     dest: "{{ kube_config_dir }}/oci-rbac.yml"
-    mode: 0640
+    mode: "0640"
   when:
   - cloud_provider is defined
   - cloud_provider == 'oci'

kubespray/project/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/tasks/main.yml

@@ -26,14 +26,14 @@
     path: "{{ kube_config_dir }}/addons/container_engine_accelerator"
     owner: root
     group: root
-    mode: 0755
+    mode: "0755"
     recurse: true
 
 - name: Container Engine Acceleration Nvidia GPU | Create manifests for nvidia accelerators
   template:
     src: "{{ item.file }}.j2"
     dest: "{{ kube_config_dir }}/addons/container_engine_accelerator/{{ item.file }}"
-    mode: 0644
+    mode: "0644"
   with_items:
     - { name: nvidia-driver-install-daemonset, file: nvidia-driver-install-daemonset.yml, type: daemonset }
     - { name: k8s-device-plugin-nvidia-daemonset, file: k8s-device-plugin-nvidia-daemonset.yml, type: daemonset }

kubespray/project/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/vars/centos-7.yml

@@ -1,3 +0,0 @@
----
-nvidia_driver_install_container: "{{ nvidia_driver_install_centos_container }}"
-nvidia_driver_install_supported: true

kubespray/project/roles/kubernetes-apps/container_runtimes/gvisor/tasks/main.yaml

@@ -4,7 +4,7 @@
     path: "{{ kube_config_dir }}/addons/gvisor"
     owner: root
     group: root
-    mode: 0755
+    mode: "0755"
     recurse: true
 
 - name: GVisor | Templates List
@@ -16,7 +16,7 @@
   template:
     src: "{{ item.file }}.j2"
     dest: "{{ kube_config_dir }}/addons/gvisor/{{ item.file }}"
-    mode: 0644
+    mode: "0644"
   with_items: "{{ gvisor_templates }}"
   register: gvisor_manifests
   when:

kubespray/project/roles/kubernetes-apps/container_runtimes/kata_containers/tasks/main.yaml

@@ -5,7 +5,7 @@
     path: "{{ kube_config_dir }}/addons/kata_containers"
     owner: root
     group: root
-    mode: 0755
+    mode: "0755"
     recurse: true
 
 - name: Kata Containers | Templates list
@@ -17,7 +17,7 @@
   template:
     src: "{{ item.file }}.j2"
     dest: "{{ kube_config_dir }}/addons/kata_containers/{{ item.file }}"
-    mode: 0644
+    mode: "0644"
   with_items: "{{ kata_containers_templates }}"
   register: kata_containers_manifests
   when:

kubespray/project/roles/kubernetes-apps/csi_driver/aws_ebs/tasks/main.yml

@@ -3,7 +3,7 @@
   template:
     src: "{{ item.file }}.j2"
     dest: "{{ kube_config_dir }}/{{ item.file }}"
-    mode: 0644
+    mode: "0644"
   with_items:
     - {name: aws-ebs-csi-driver, file: aws-ebs-csi-driver.yml}
     - {name: aws-ebs-csi-controllerservice, file: aws-ebs-csi-controllerservice-rbac.yml}

kubespray/project/roles/kubernetes-apps/csi_driver/azuredisk/tasks/main.yml

@@ -7,7 +7,7 @@
     src: "azure-csi-cloud-config.j2"
     dest: "{{ kube_config_dir }}/azure_csi_cloud_config"
     group: "{{ kube_cert_group }}"
-    mode: 0640
+    mode: "0640"
   when: inventory_hostname == groups['kube_control_plane'][0]
 
 - name: Azure CSI Driver | Get base64 cloud-config
@@ -20,7 +20,7 @@
   template:
     src: "{{ item.file }}.j2"
     dest: "{{ kube_config_dir }}/{{ item.file }}"
-    mode: 0644
+    mode: "0644"
   with_items:
     - {name: azure-csi-azuredisk-driver, file: azure-csi-azuredisk-driver.yml}
     - {name: azure-csi-cloud-config-secret, file: azure-csi-cloud-config-secret.yml}

kubespray/project/roles/kubernetes-apps/csi_driver/cinder/defaults/main.yml

@@ -1,5 +1,4 @@
 ---
-
 cinder_csi_attacher_image_tag: "v4.4.2"
 cinder_csi_provisioner_image_tag: "v3.6.2"
 cinder_csi_snapshotter_image_tag: "v6.3.2"
@@ -35,3 +34,15 @@ cinder_csi_controller_replicas: 1
 # cinder_csi_rescan_on_resize: true
 
 cinder_tolerations: []
+
+## Dictionaries of extra arguments to add to the cinder CSI plugin containers
+## Format:
+##  cinder_csi_attacher_extra_args:
+##    arg1: "value1"
+##    arg2: "value2"
+cinder_csi_attacher_extra_args: {}
+cinder_csi_provisioner_extra_args: {}
+cinder_csi_snapshotter_extra_args: {}
+cinder_csi_resizer_extra_args: {}
+cinder_csi_plugin_extra_args: {}
+cinder_liveness_probe_extra_args: {}

kubespray/project/roles/kubernetes-apps/csi_driver/cinder/tasks/cinder-write-cacert.yml

@@ -7,5 +7,5 @@
     src: "{{ cinder_cacert }}"
     dest: "{{ kube_config_dir }}/cinder-cacert.pem"
     group: "{{ kube_cert_group }}"
-    mode: 0640
+    mode: "0640"
   delegate_to: "{{ delegate_host_to_write_cacert }}"

kubespray/project/roles/kubernetes-apps/csi_driver/cinder/tasks/main.yml

@@ -18,7 +18,7 @@
     src: "cinder-csi-cloud-config.j2"
     dest: "{{ kube_config_dir }}/cinder_cloud_config"
     group: "{{ kube_cert_group }}"
-    mode: 0640
+    mode: "0640"
   when: inventory_hostname == groups['kube_control_plane'][0]
 
 - name: Cinder CSI Driver | Get base64 cloud-config
@@ -31,7 +31,7 @@
   template:
     src: "{{ item.file }}.j2"
     dest: "{{ kube_config_dir }}/{{ item.file }}"
-    mode: 0644
+    mode: "0644"
   with_items:
     - {name: cinder-csi-driver, file: cinder-csi-driver.yml}
     - {name: cinder-csi-cloud-config-secret, file: cinder-csi-cloud-config-secret.yml}

kubespray/project/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-controllerplugin.yml.j2

@@ -30,6 +30,9 @@ spec:
             - --leader-election=true
 {% endif %}
             - "--default-fstype=ext4"
+{% for key, value in cinder_csi_attacher_extra_args.items() %}
+            - "{{ '--' + key + '=' + value }}"
+{% endfor %}
           env:
             - name: ADDRESS
               value: /var/lib/csi/sockets/pluginproxy/csi.sock
@@ -50,6 +53,9 @@ spec:
 {% if cinder_csi_controller_replicas is defined and cinder_csi_controller_replicas > 1 %}
             - "--leader-election=true"
 {% endif %}
+{% for key, value in cinder_csi_provisioner_extra_args.items() %}
+            - "{{ '--' + key + '=' + value }}"
+{% endfor %}
           env:
             - name: ADDRESS
               value: /var/lib/csi/sockets/pluginproxy/csi.sock
@@ -66,6 +72,9 @@ spec:
 {% if cinder_csi_controller_replicas is defined and cinder_csi_controller_replicas > 1 %}
             - --leader-election=true
 {% endif %}
+{% for key, value in cinder_csi_snapshotter_extra_args.items() %}
+            - "{{ '--' + key + '=' + value }}"
+{% endfor %}
           env:
             - name: ADDRESS
               value: /var/lib/csi/sockets/pluginproxy/csi.sock
@@ -82,6 +91,9 @@ spec:
 {% if cinder_csi_controller_replicas is defined and cinder_csi_controller_replicas > 1 %}
             - --leader-election=true
 {% endif %}
+{% for key, value in cinder_csi_resizer_extra_args.items() %}
+            - "{{ '--' + key + '=' + value }}"
+{% endfor %}
           env:
             - name: ADDRESS
               value: /var/lib/csi/sockets/pluginproxy/csi.sock
@@ -93,6 +105,9 @@ spec:
           imagePullPolicy: {{ k8s_image_pull_policy }}
           args:
             - "--csi-address=$(ADDRESS)"
+{% for key, value in cinder_liveness_probe_extra_args.items() %}
+            - "{{ '--' + key + '=' + value }}"
+{% endfor %}
           env:
             - name: ADDRESS
               value: /var/lib/csi/sockets/pluginproxy/csi.sock
@@ -107,6 +122,9 @@ spec:
             - "--endpoint=$(CSI_ENDPOINT)"
             - "--cloud-config=$(CLOUD_CONFIG)"
             - "--cluster=$(CLUSTER_NAME)"
+{% for key, value in cinder_csi_plugin_extra_args.items() %}
+            - "{{ '--' + key + '=' + value }}"
+{% endfor %}
           env:
             - name: CSI_ENDPOINT
               value: unix://csi/csi.sock

kubespray/project/roles/kubernetes-apps/csi_driver/csi_crd/tasks/main.yml

@@ -3,7 +3,7 @@
   template:
     src: "{{ item.file }}.j2"
     dest: "{{ kube_config_dir }}/{{ item.file }}"
-    mode: 0644
+    mode: "0644"
   with_items:
     - {name: volumesnapshotclasses, file: volumesnapshotclasses.yml}
     - {name: volumesnapshotcontents, file: volumesnapshotcontents.yml}

kubespray/project/roles/kubernetes-apps/csi_driver/gcp_pd/tasks/main.yml

@@ -9,7 +9,7 @@
     src: "{{ gcp_pd_csi_sa_cred_file }}"
     dest: "{{ kube_config_dir }}/cloud-sa.json"
     group: "{{ kube_cert_group }}"
-    mode: 0640
+    mode: "0640"
   when: inventory_hostname == groups['kube_control_plane'][0]
 
 - name: GCP PD CSI Driver | Get base64 cloud-sa.json
@@ -22,7 +22,7 @@
   template:
     src: "{{ item.file }}.j2"
     dest: "{{ kube_config_dir }}/{{ item.file }}"
-    mode: 0644
+    mode: "0644"
   with_items:
     - {name: gcp-pd-csi-cred-secret, file: gcp-pd-csi-cred-secret.yml}
     - {name: gcp-pd-csi-setup, file: gcp-pd-csi-setup.yml}

kubespray/project/roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-controller.yml.j2

@@ -162,4 +162,4 @@ metadata:
   name: pd.csi.storage.gke.io
 spec:
   attachRequired: true
-  podInfoOnMount: false
+  podInfoOnMount: false

kubespray/project/roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-node.yml.j2

@@ -109,4 +109,4 @@ spec:
       # See "special case". This will tolerate everything. Node component should
       # be scheduled on all nodes.
       tolerations:
-      - operator: Exists
+      - operator: Exists

kubespray/project/roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-sc-regional.yml.j2

@@ -6,4 +6,4 @@ provisioner: pd.csi.storage.gke.io
 parameters:
   type: pd-balanced
   replication-type: regional-pd
-volumeBindingMode: WaitForFirstConsumer
+volumeBindingMode: WaitForFirstConsumer

kubespray/project/roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-sc-zonal.yml.j2

@@ -5,4 +5,4 @@ metadata:
 provisioner: pd.csi.storage.gke.io
 parameters:
   type: pd-balanced
-volumeBindingMode: WaitForFirstConsumer
+volumeBindingMode: WaitForFirstConsumer

kubespray/project/roles/kubernetes-apps/csi_driver/upcloud/defaults/main.yml

@@ -3,7 +3,7 @@ upcloud_csi_controller_replicas: 1
 upcloud_csi_provisioner_image_tag: "v3.1.0"
 upcloud_csi_attacher_image_tag: "v3.4.0"
 upcloud_csi_resizer_image_tag: "v1.4.0"
-upcloud_csi_plugin_image_tag: "v0.3.3"
+upcloud_csi_plugin_image_tag: "v1.1.0"
 upcloud_csi_node_image_tag: "v2.5.0"
 upcloud_username: "{{ lookup('env', 'UPCLOUD_USERNAME') }}"
 upcloud_password: "{{ lookup('env', 'UPCLOUD_PASSWORD') }}"

kubespray/project/roles/kubernetes-apps/csi_driver/upcloud/tasks/main.yml

@@ -16,7 +16,7 @@
   template:
     src: "{{ item.file }}.j2"
     dest: "{{ kube_config_dir }}/{{ item.file }}"
-    mode: 0644
+    mode: "0644"
   with_items:
     - {name: upcloud-csi-cred-secret, file: upcloud-csi-cred-secret.yml}
     - {name: upcloud-csi-setup, file: upcloud-csi-setup.yml}

kubespray/project/roles/kubernetes-apps/csi_driver/upcloud/templates/upcloud-csi-controller.yml.j2

@@ -58,13 +58,26 @@ spec:
           volumeMounts:
             - name: socket-dir
               mountPath: /var/lib/csi/sockets/pluginproxy/
+        - name: csi-snapshotter
+          image: k8s.gcr.io/sig-storage/csi-snapshotter:{{ upcloud_csi_snapshotter_image_tag }}
+          args:
+            - "--csi-address=$(ADDRESS)"
+            - "--v=5"
+            - "--timeout=600s"
+            - "--leader-election=false"
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+          imagePullPolicy: "Always"
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
         - name: csi-upcloud-plugin
           image: ghcr.io/upcloudltd/upcloud-csi:{{ upcloud_csi_plugin_image_tag }}
           args:
             - "--endpoint=$(CSI_ENDPOINT)"
             - "--nodehost=$(NODE_ID)"
-            - "--username=$(UPCLOUD_USERNAME)"
-            - "--password=$(UPCLOUD_PASSWORD)"
+            - "--mode=monolith"
           env:
             - name: CSI_ENDPOINT
               value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock

kubespray/project/roles/kubernetes-apps/csi_driver/upcloud/templates/upcloud-csi-node.yml.j2

@@ -42,8 +42,7 @@ spec:
           args:
             - "--endpoint=$(CSI_ENDPOINT)"
             - "--nodehost=$(NODE_ID)"
-            - "--username=$(UPCLOUD_USERNAME)"
-            - "--password=$(UPCLOUD_PASSWORD)"
+            - "--mode=monolith"
           env:
             - name: CSI_ENDPOINT
               value: unix:///csi/csi.sock

kubespray/project/roles/kubernetes-apps/csi_driver/upcloud/templates/upcloud-csi-setup.yml.j2

@@ -62,6 +62,18 @@ rules:
   - apiGroups: [ "" ]
     resources: [ "events" ]
     verbs: [ "list", "watch", "create", "update", "patch" ]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents"]
+    verbs: ["create", "get", "list", "watch", "update", "delete", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents/status"]
+    verbs: ["update"]
+  - apiGroups: [ "snapshot.storage.k8s.io" ]
+    resources: [ "volumesnapshots" ]
+    verbs: [ "get", "list" ]
   - apiGroups: [ "" ]
     resources: [ "nodes" ]
     verbs: [ "get", "list", "watch" ]
@@ -183,3 +195,54 @@ roleRef:
   kind: ClusterRole
   name: csi-upcloud-resizer-role
   apiGroup: rbac.authorization.k8s.io
+
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-upcloud-snapshotter-role
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "list"]
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "create", "delete"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["list", "watch", "create", "update", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots"]
+    verbs: ["get", "list"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents"]
+    verbs: ["get", "list"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["csinodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattachments"]
+    verbs: ["get", "list", "watch"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-upcloud-snapshotter-binding
+subjects:
+  - kind: ServiceAccount
+    name: csi-upcloud-controller-sa
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: csi-upcloud-snapshotter-role
+  apiGroup: rbac.authorization.k8s.io

kubespray/project/roles/kubernetes-apps/csi_driver/vsphere/defaults/main.yml

@@ -21,7 +21,7 @@ vsphere_csi_controller_replicas: 1
 
 csi_endpoint: '{% if external_vsphere_version >= "7.0u1" %}/csi{% else %}/var/lib/csi/sockets/pluginproxy{% endif %}'
 
-vsphere_csi_aggressive_node_drain: False
+vsphere_csi_aggressive_node_drain: false
 vsphere_csi_aggressive_node_unreachable_timeout: 300
 vsphere_csi_aggressive_node_not_ready_timeout: 300
 

kubespray/project/roles/kubernetes-apps/csi_driver/vsphere/tasks/main.yml

@@ -6,7 +6,7 @@
   template:
     src: "{{ item }}.j2"
     dest: "{{ kube_config_dir }}/{{ item }}"
-    mode: 0640
+    mode: "0640"
   with_items:
     - vsphere-csi-cloud-config
   when: inventory_hostname == groups['kube_control_plane'][0]
@@ -15,7 +15,7 @@
   template:
     src: "{{ item }}.j2"
     dest: "{{ kube_config_dir }}/{{ item }}"
-    mode: 0644
+    mode: "0644"
   with_items:
     - vsphere-csi-namespace.yml
     - vsphere-csi-driver.yml

kubespray/project/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-config.yml.j2

@@ -18,7 +18,7 @@ data:
   "max-pvscsi-targets-per-vm": "true"
   "multi-vcenter-csi-topology": "true"
   "csi-internal-generated-cluster-id": "true"
-  "listview-tasks": "true" 
+  "listview-tasks": "true"
 {% if vsphere_csi_controller is version('v2.7.0', '>=') %}
   "improved-csi-idempotency": "true"
   "improved-volume-topology": "true"

kubespray/project/roles/kubernetes-apps/external_cloud_controller/hcloud/tasks/main.yml

@@ -4,7 +4,7 @@
     src: "{{ item.file }}.j2"
     dest: "{{ kube_config_dir }}/{{ item.file }}"
     group: "{{ kube_cert_group }}"
-    mode: 0640
+    mode: "0640"
   with_items:
     - {name: external-hcloud-cloud-secret, file: external-hcloud-cloud-secret.yml}
     - {name: external-hcloud-cloud-service-account, file: external-hcloud-cloud-service-account.yml}

kubespray/project/roles/kubernetes-apps/external_cloud_controller/huaweicloud/defaults/main.yml

@@ -16,4 +16,4 @@ external_huaweicloud_cloud: "{{ lookup('env','OS_CLOUD') }}"
 ##    arg2: "value2"
 external_huawei_cloud_controller_extra_args: {}
 external_huawei_cloud_controller_image_repo: "swr.ap-southeast-1.myhuaweicloud.com"
-external_huawei_cloud_controller_image_tag: "v0.26.6"
+external_huawei_cloud_controller_image_tag: "v0.26.8"

kubespray/project/roles/kubernetes-apps/external_cloud_controller/huaweicloud/tasks/main.yml

@@ -24,7 +24,7 @@
     src: "{{ item.file }}.j2"
     dest: "{{ kube_config_dir }}/{{ item.file }}"
     group: "{{ kube_cert_group }}"
-    mode: 0640
+    mode: "0640"
   with_items:
     - {name: external-huawei-cloud-config-secret, file: external-huawei-cloud-config-secret.yml}
     - {name: external-huawei-cloud-controller-manager-roles, file: external-huawei-cloud-controller-manager-roles.yml}

kubespray/project/roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-config.j2

@@ -21,3 +21,6 @@ subnet-id={{ external_huaweicloud_lbaas_subnet_id }}
 {% if external_huaweicloud_lbaas_network_id is defined %}
 id={{ external_huaweicloud_lbaas_network_id }}
 {% endif %}
+{% if external_huaweicloud_security_group_id is defined %}
+security-group-id={{ external_huaweicloud_security_group_id }}
+{% endif %}

kubespray/project/roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-controller-manager-ds.yml.j2

@@ -47,6 +47,11 @@ spec:
             - --cloud-config=$(CLOUD_CONFIG)
             - --cloud-provider=huaweicloud
             - --use-service-account-credentials=true
+            - --node-status-update-frequency=5s
+            - --node-monitor-period=5s
+            - --leader-elect-lease-duration=30s
+            - --leader-elect-renew-deadline=20s
+            - --leader-elect-retry-period=2s
 {% for key, value in external_huawei_cloud_controller_extra_args.items() %}
             - "{{ '--' + key + '=' + value }}"
 {% endfor %}

kubespray/project/roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-controller-manager-role-bindings.yml.j2

@@ -1,16 +1,12 @@
-apiVersion: v1
-items:
-- apiVersion: rbac.authorization.k8s.io/v1
-  kind: ClusterRoleBinding
-  metadata:
-    name: system:cloud-controller-manager
-  roleRef:
-    apiGroup: rbac.authorization.k8s.io
-    kind: ClusterRole
-    name: system:cloud-controller-manager
-  subjects:
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: system:cloud-controller-manager
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:cloud-controller-manager
+subjects:
   - kind: ServiceAccount
     name: cloud-controller-manager
     namespace: kube-system
-kind: List
-metadata: {}

kubespray/project/roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-controller-manager-roles.yml.j2

@@ -1,117 +1,113 @@
-apiVersion: v1
-items:
-- apiVersion: rbac.authorization.k8s.io/v1
-  kind: ClusterRole
-  metadata:
-    name: system:cloud-controller-manager
-  rules:
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: system:cloud-controller-manager
+rules:
   - resources:
-    - tokenreviews
+      - tokenreviews
     verbs:
-    - get
-    - list
-    - watch
-    - create
-    - update
-    - patch
+      - get
+      - list
+      - watch
+      - create
+      - update
+      - patch
     apiGroups:
-    - authentication.k8s.io
+      - authentication.k8s.io
   - resources:
-    - configmaps
-    - endpoints
-    - pods
-    - services
-    - secrets
-    - serviceaccounts
-    - serviceaccounts/token
+      - configmaps
+      - endpoints
+      - pods
+      - services
+      - secrets
+      - serviceaccounts
+      - serviceaccounts/token
     verbs:
-    - get
-    - list
-    - watch
-    - create
-    - update
-    - patch
+      - get
+      - list
+      - watch
+      - create
+      - update
+      - patch
     apiGroups:
-    - ''
+      - ''
   - resources:
-    - nodes
+      - nodes
     verbs:
-    - get
-    - list
-    - watch
-    - delete
-    - patch
-    - update
+      - get
+      - list
+      - watch
+      - delete
+      - patch
+      - update
     apiGroups:
-    - ''
+      - ''
   - resources:
-    - services/status
-    - pods/status
+      - services/status
+      - pods/status
     verbs:
-    - update
-    - patch
+      - update
+      - patch
     apiGroups:
-    - ''
+      - ''
   - resources:
-    - nodes/status
+      - nodes/status
     verbs:
-    - patch
-    - update
+      - patch
+      - update
     apiGroups:
-    - ''
+      - ''
   - resources:
-    - events
-    - endpoints
+      - events
+      - endpoints
     verbs:
-    - create
-    - patch
-    - update
+      - create
+      - patch
+      - update
     apiGroups:
-    - ''
+      - ''
   - resources:
-    - leases
+      - leases
     verbs:
-    - get
-    - update
-    - create
-    - delete
+      - get
+      - update
+      - create
+      - delete
     apiGroups:
-    - coordination.k8s.io
+      - coordination.k8s.io
   - resources:
-    - customresourcedefinitions
+      - customresourcedefinitions
     verbs:
-    - get
-    - update
-    - create
-    - delete
+      - get
+      - update
+      - create
+      - delete
     apiGroups:
       - apiextensions.k8s.io
   - resources:
-    - ingresses
+      - ingresses
     verbs:
-    - get
-    - list
-    - watch
-    - update
-    - create
-    - patch
-    - delete
+      - get
+      - list
+      - watch
+      - update
+      - create
+      - patch
+      - delete
     apiGroups:
-    - networking.k8s.io
+      - networking.k8s.io
   - resources:
-    - ingresses/status
+      - ingresses/status
     verbs:
-    - update
-    - patch
+      - update
+      - patch
     apiGroups:
-    - networking.k8s.io
+      - networking.k8s.io
   - resources:
-    - endpointslices
+      - endpointslices
     verbs:
-    - get
-    - list
-    - watch
+      - get
+      - list
+      - watch
     apiGroups:
-    - discovery.k8s.io
-kind: List
-metadata: {}
+      - discovery.k8s.io

kubespray/project/roles/kubernetes-apps/external_cloud_controller/openstack/defaults/main.yml

@@ -21,5 +21,6 @@ external_openstack_cacert: "{{ lookup('env', 'OS_CACERT') }}"
 ##    arg1: "value1"
 ##    arg2: "value2"
 external_openstack_cloud_controller_extra_args: {}
-external_openstack_cloud_controller_image_tag: "v1.25.3"
+external_openstack_cloud_controller_image_tag: "v1.30.0"
 external_openstack_cloud_controller_bind_address: 127.0.0.1
+external_openstack_cloud_controller_dns_policy: ClusterFirst

kubespray/project/roles/kubernetes-apps/external_cloud_controller/openstack/tasks/main.yml

@@ -24,7 +24,7 @@
     src: "{{ item.file }}.j2"
     dest: "{{ kube_config_dir }}/{{ item.file }}"
     group: "{{ kube_cert_group }}"
-    mode: 0640
+    mode: "0640"
   with_items:
     - {name: external-openstack-cloud-config-secret, file: external-openstack-cloud-config-secret.yml}
     - {name: external-openstack-cloud-controller-manager-roles, file: external-openstack-cloud-controller-manager-roles.yml}

kubespray/project/roles/kubernetes-apps/external_cloud_controller/openstack/templates/external-openstack-cloud-controller-manager-ds.yml.j2

@@ -36,7 +36,7 @@ spec:
       serviceAccountName: cloud-controller-manager
       containers:
         - name: openstack-cloud-controller-manager
-          image: {{ docker_image_repo }}/k8scloudprovider/openstack-cloud-controller-manager:{{ external_openstack_cloud_controller_image_tag }}
+          image: {{ external_openstack_cloud_controller_image_repo }}:{{ external_openstack_cloud_controller_image_tag }}
           args:
             - /bin/openstack-cloud-controller-manager
             - --v=1
@@ -81,7 +81,9 @@ spec:
             - name: CLOUD_CONFIG
               value: /etc/config/cloud.conf
       hostNetwork: true
-      dnsPolicy: ClusterFirstWithHostNet
+{% if external_openstack_cloud_controller_dns_policy is defined %}
+      dnsPolicy: {{ external_openstack_cloud_controller_dns_policy }}
+{% endif %}
       volumes:
 {% if kubelet_flexvolumes_plugins_dir is defined %}
       - name: flexvolume-dir

kubespray/project/roles/kubernetes-apps/external_cloud_controller/vsphere/tasks/main.yml

@@ -6,7 +6,7 @@
   template:
     src: "{{ item }}.j2"
     dest: "{{ kube_config_dir }}/{{ item }}"
-    mode: 0640
+    mode: "0640"
   with_items:
     - external-vsphere-cpi-cloud-config
   when: inventory_hostname == groups['kube_control_plane'][0]
@@ -15,7 +15,7 @@
   template:
     src: "{{ item }}.j2"
     dest: "{{ kube_config_dir }}/{{ item }}"
-    mode: 0644
+    mode: "0644"
   with_items:
     - external-vsphere-cpi-cloud-config-secret.yml
     - external-vsphere-cloud-controller-manager-roles.yml

kubespray/project/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/tasks/main.yml

@@ -33,7 +33,7 @@
     state: directory
     owner: root
     group: root
-    mode: 0755
+    mode: "0755"
   when:
     - inventory_hostname == groups['kube_control_plane'][0]
 
@@ -54,7 +54,7 @@
   template:
     src: "{{ item.file }}.j2"
     dest: "{{ kube_config_dir }}/addons/cephfs_provisioner/{{ item.file }}"
-    mode: 0644
+    mode: "0644"
   with_items: "{{ cephfs_provisioner_templates }}"
   register: cephfs_provisioner_manifests
   when: inventory_hostname == groups['kube_control_plane'][0]

kubespray/project/roles/kubernetes-apps/external_provisioner/local_path_provisioner/tasks/main.yml

@@ -5,7 +5,7 @@
     state: directory
     owner: root
     group: root
-    mode: 0755
+    mode: "0755"
   when:
     - inventory_hostname == groups['kube_control_plane'][0]
 
@@ -13,7 +13,7 @@
   file:
     path: "{{ local_path_provisioner_claim_root }}"
     state: directory
-    mode: 0755
+    mode: "0755"
 
 - name: Local Path Provisioner | Render Template
   set_fact:
@@ -30,7 +30,7 @@
   template:
     src: "{{ item.file }}.j2"
     dest: "{{ kube_config_dir }}/addons/local_path_provisioner/{{ item.file }}"
-    mode: 0644
+    mode: "0644"
   with_items: "{{ local_path_provisioner_templates }}"
   register: local_path_provisioner_manifests
   when: inventory_hostname == groups['kube_control_plane'][0]

kubespray/project/roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-cm.yml.j2

@@ -32,4 +32,3 @@ data:
       - name: helper-pod
         image: "{{ local_path_provisioner_helper_image_repo }}:{{ local_path_provisioner_helper_image_tag }}"
         imagePullPolicy: IfNotPresent
-

kubespray/project/roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-cr.yml.j2

@@ -15,4 +15,4 @@ rules:
     verbs: [ "create", "patch" ]
   - apiGroups: [ "storage.k8s.io" ]
     resources: [ "storageclasses" ]
-    verbs: [ "get", "list", "watch" ]
+    verbs: [ "get", "list", "watch" ]

kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/defaults/main.yml

@@ -18,3 +18,4 @@ local_volume_provisioner_storage_classes: |
       "fs_type": "ext4"
     }
   }
+local_volume_provisioner_log_level: 2

kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/tasks/main.yml

@@ -12,7 +12,7 @@
     state: directory
     owner: root
     group: root
-    mode: 0755
+    mode: "0755"
 
 - name: Local Volume Provisioner | Templates list
   set_fact:
@@ -29,7 +29,7 @@
   template:
     src: "{{ item.file }}.j2"
     dest: "{{ kube_config_dir }}/addons/local_volume_provisioner/{{ item.file }}"
-    mode: 0644
+    mode: "0644"
   with_items: "{{ local_volume_provisioner_templates }}"
   register: local_volume_provisioner_manifests
   when: inventory_hostname == groups['kube_control_plane'][0]

kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-ds.yml.j2

@@ -30,6 +30,8 @@ spec:
         - name: provisioner
           image: {{ local_volume_provisioner_image_repo }}:{{ local_volume_provisioner_image_tag }}
           imagePullPolicy: {{ k8s_image_pull_policy }}
+          args:
+          - "-v={{ local_volume_provisioner_log_level }}"
           securityContext:
             privileged: true
           env:

kubespray/project/roles/kubernetes-apps/external_provisioner/rbd_provisioner/tasks/main.yml

@@ -33,7 +33,7 @@
     state: directory
     owner: root
     group: root
-    mode: 0755
+    mode: "0755"
   when:
     - inventory_hostname == groups['kube_control_plane'][0]
 
@@ -54,7 +54,7 @@
   template:
     src: "{{ item.file }}.j2"
     dest: "{{ kube_config_dir }}/addons/rbd_provisioner/{{ item.file }}"
-    mode: 0644
+    mode: "0644"
   with_items: "{{ rbd_provisioner_templates }}"
   register: rbd_provisioner_manifests
   when: inventory_hostname == groups['kube_control_plane'][0]

kubespray/project/roles/kubernetes-apps/gateway_api/defaults/main.yml

@@ -0,0 +1,4 @@
+---
+gateway_api_enabled: false
+gateway_api_version: v1.1.0
+gateway_api_experimental_channel: false

kubespray/project/roles/kubernetes-apps/gateway_api/tasks/main.yml

@@ -0,0 +1,33 @@
+---
+- name: Gateway API | Create addon dir
+  file:
+    path: "{{ kube_config_dir }}/addons/gateway_api"
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+
+- name: Gateway API | Set channel
+  set_fact:
+    gateway_api_channel: "{{ 'experimental' if gateway_api_experimental_channel else 'standard' }}"
+  when:
+    - "inventory_hostname == groups['kube_control_plane'][0]"
+
+- name: Gateway API | Copy Gateway API manifests to remote
+  template:
+    src: "{{ gateway_api_channel }}-install.yaml.j2"
+    dest: "{{ kube_config_dir }}/addons/gateway_api/{{ gateway_api_channel }}-install.yaml"
+    mode: "0644"
+  when:
+    - "inventory_hostname == groups['kube_control_plane'][0]"
+
+- name: Gateway API | Install Gateway API
+  kube:
+    name: Gateway API
+    kubectl: "{{ bin_dir }}/kubectl"
+    filename: "{{ kube_config_dir }}/addons/gateway_api/{{ gateway_api_channel }}-install.yaml"
+    state: latest
+  when:
+    - "inventory_hostname == groups['kube_control_plane'][0]"

kubespray/project/roles/kubernetes-apps/helm/tasks/main.yml

@@ -32,18 +32,18 @@
   copy:
     src: "{{ local_release_dir }}/helm-{{ helm_version }}/linux-{{ image_arch }}/helm"
     dest: "{{ bin_dir }}/helm"
-    mode: 0755
+    mode: "0755"
     remote_src: true
 
 - name: Helm | Get helm completion
   command: "{{ bin_dir }}/helm completion bash"
-  changed_when: False
+  changed_when: false
   register: helm_completion
-  check_mode: False
+  check_mode: false
 
 - name: Helm | Install helm completion
   copy:
     dest: /etc/bash_completion.d/helm.sh
     content: "{{ helm_completion.stdout }}"
-    mode: 0755
-  become: True
+    mode: "0755"
+  become: true

kubespray/project/roles/kubernetes-apps/helm/tasks/pyyaml-flatcar.yml

@@ -2,13 +2,13 @@
 - name: Get installed pip version
   command: "{{ ansible_python_interpreter if ansible_python_interpreter is defined else 'python' }} -m pip --version"
   register: pip_version_output
-  ignore_errors: yes
+  ignore_errors: true
   changed_when: false
 
 - name: Get installed PyYAML version
   command: "{{ ansible_python_interpreter if ansible_python_interpreter is defined else 'python' }} -m pip show PyYAML"
   register: pyyaml_version_output
-  ignore_errors: yes
+  ignore_errors: true
   changed_when: false
 
 - name: Install pip

kubespray/project/roles/kubernetes-apps/ingress_controller/alb_ingress_controller/tasks/main.yml

@@ -6,13 +6,13 @@
     state: directory
     owner: root
     group: root
-    mode: 0755
+    mode: "0755"
 
 - name: ALB Ingress Controller | Create manifests
   template:
     src: "{{ item.file }}.j2"
     dest: "{{ kube_config_dir }}/addons/alb_ingress/{{ item.file }}"
-    mode: 0644
+    mode: "0644"
   with_items:
     - { name: alb-ingress-clusterrole, file: alb-ingress-clusterrole.yml, type: clusterrole }
     - { name: alb-ingress-clusterrolebinding, file: alb-ingress-clusterrolebinding.yml, type: clusterrolebinding }

kubespray/project/roles/kubernetes-apps/ingress_controller/cert_manager/tasks/main.yml

@@ -24,7 +24,7 @@
     state: directory
     owner: root
     group: root
-    mode: 0755
+    mode: "0755"
   when:
     - inventory_hostname == groups['kube_control_plane'][0]
 
@@ -38,7 +38,7 @@
   template:
     src: "{{ item.file }}.j2"
     dest: "{{ kube_config_dir }}/addons/cert_manager/{{ item.file }}"
-    mode: 0644
+    mode: "0644"
   with_items: "{{ cert_manager_templates }}"
   register: cert_manager_manifests
   when:

kubespray/project/roles/kubernetes-apps/ingress_controller/cert_manager/templates/cert-manager.crds.yml.j2

@@ -226,10 +226,10 @@ spec:
                                   - subscriptionID
                                 properties:
                                   clientID:
-                                    description: if both this and ClientSecret are left unset MSI will be used
+                                    description: 'Auth: Azure Service Principal: The ClientID of the Azure Service Principal used to authenticate with Azure DNS. If set, ClientSecret and TenantID must also be set.'
                                     type: string
                                   clientSecretSecretRef:
-                                    description: if both this and ClientID are left unset MSI will be used
+                                    description: 'Auth: Azure Service Principal: A reference to a Secret containing the password associated with the Service Principal. If set, ClientID and TenantID must also be set.'
                                     type: object
                                     required:
                                       - name
@@ -252,14 +252,14 @@ spec:
                                     description: name of the DNS zone that should be used
                                     type: string
                                   managedIdentity:
-                                    description: managed identity configuration, can not be used at the same time as clientID, clientSecretSecretRef or tenantID
+                                    description: 'Auth: Azure Workload Identity or Azure Managed Service Identity: Settings to enable Azure Workload Identity or Azure Managed Service Identity If set, ClientID, ClientSecret and TenantID must not be set.'
                                     type: object
                                     properties:
                                       clientID:
                                         description: client ID of the managed identity, can not be used at the same time as resourceID
                                         type: string
                                       resourceID:
-                                        description: resource ID of the managed identity, can not be used at the same time as clientID
+                                        description: resource ID of the managed identity, can not be used at the same time as clientID Cannot be used for Azure Managed Service Identity
                                         type: string
                                   resourceGroupName:
                                     description: resource group the DNS zone is located in
@@ -268,7 +268,7 @@ spec:
                                     description: ID of the Azure subscription
                                     type: string
                                   tenantID:
-                                    description: when specifying ClientID and ClientSecret then this field is also needed
+                                    description: 'Auth: Azure Service Principal: The TenantID of the Azure Service Principal used to authenticate with Azure DNS. If set, ClientID and ClientSecret must also be set.'
                                     type: string
                               cloudDNS:
                                 description: Use the Google Cloud DNS API to manage DNS01 challenge records.
@@ -474,13 +474,13 @@ spec:
                                           maxLength: 253
                                           minLength: 1
                                         namespace:
-                                          description: "Namespace is the namespace of the referent. When unspecified, this refers to the local namespace of the Route. \n Note that there are specific rules for ParentRefs which cross namespace boundaries. Cross-namespace references are only valid if they are explicitly allowed by something in the namespace they are referring to. For example: Gateway has the AllowedRoutes field, and ReferenceGrant provides a generic way to enable any other kind of cross-namespace reference. \n ParentRefs from a Route to a Service in the same namespace are \"producer\" routes, which apply default routing rules to inbound connections from any namespace to the Service. \n ParentRefs from a Route to a Service in a different namespace are \"consumer\" routes, and these routing rules are only applied to outbound connections originating from the same namespace as the Route, for which the intended destination of the connections are a Service targeted as a ParentRef of the Route. \n Support: Core"
+                                          description: "Namespace is the namespace of the referent. When unspecified, this refers to the local namespace of the Route. \n Note that there are specific rules for ParentRefs which cross namespace boundaries. Cross-namespace references are only valid if they are explicitly allowed by something in the namespace they are referring to. For example: Gateway has the AllowedRoutes field, and ReferenceGrant provides a generic way to enable any other kind of cross-namespace reference. \n <gateway:experimental:description> ParentRefs from a Route to a Service in the same namespace are \"producer\" routes, which apply default routing rules to inbound connections from any namespace to the Service. \n ParentRefs from a Route to a Service in a different namespace are \"consumer\" routes, and these routing rules are only applied to outbound connections originating from the same namespace as the Route, for which the intended destination of the connections are a Service targeted as a ParentRef of the Route. </gateway:experimental:description> \n Support: Core"
                                           type: string
                                           maxLength: 63
                                           minLength: 1
                                           pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$
                                         port:
-                                          description: "Port is the network port this Route targets. It can be interpreted differently based on the type of parent resource. \n When the parent resource is a Gateway, this targets all listeners listening on the specified port that also support this kind of Route(and select this Route). It's not recommended to set `Port` unless the networking behaviors specified in a Route must apply to a specific port as opposed to a listener(s) whose port(s) may be changed. When both Port and SectionName are specified, the name and port of the selected listener must match both specified values. \n When the parent resource is a Service, this targets a specific port in the Service spec. When both Port (experimental) and SectionName are specified, the name and port of the selected port must match both specified values. \n Implementations MAY choose to support other parent resources. Implementations supporting other types of parent resources MUST clearly document how/if Port is interpreted. \n For the purpose of status, an attachment is considered successful as long as the parent resource accepts it partially. For example, Gateway listeners can restrict which Routes can attach to them by Route kind, namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from the referencing Route, the Route MUST be considered successfully attached. If no Gateway listeners accept attachment from this Route, the Route MUST be considered detached from the Gateway. \n Support: Extended \n <gateway:experimental>"
+                                          description: "Port is the network port this Route targets. It can be interpreted differently based on the type of parent resource. \n When the parent resource is a Gateway, this targets all listeners listening on the specified port that also support this kind of Route(and select this Route). It's not recommended to set `Port` unless the networking behaviors specified in a Route must apply to a specific port as opposed to a listener(s) whose port(s) may be changed. When both Port and SectionName are specified, the name and port of the selected listener must match both specified values. \n <gateway:experimental:description> When the parent resource is a Service, this targets a specific port in the Service spec. When both Port (experimental) and SectionName are specified, the name and port of the selected port must match both specified values. </gateway:experimental:description> \n Implementations MAY choose to support other parent resources. Implementations supporting other types of parent resources MUST clearly document how/if Port is interpreted. \n For the purpose of status, an attachment is considered successful as long as the parent resource accepts it partially. For example, Gateway listeners can restrict which Routes can attach to them by Route kind, namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from the referencing Route, the Route MUST be considered successfully attached. If no Gateway listeners accept attachment from this Route, the Route MUST be considered detached from the Gateway. \n Support: Extended \n <gateway:experimental>"
                                           type: integer
                                           format: int32
                                           maximum: 65535
@@ -694,7 +694,7 @@ spec:
                                                             - topologyKey
                                                           properties:
                                                             labelSelector:
-                                                              description: A label query over a set of resources, in this case pods.
+                                                              description: A label query over a set of resources, in this case pods. If it's null, this PodAffinityTerm matches with no Pods.
                                                               type: object
                                                               properties:
                                                                 matchExpressions:
@@ -724,6 +724,18 @@ spec:
                                                                   additionalProperties:
                                                                     type: string
                                                               x-kubernetes-map-type: atomic
+                                                            matchLabelKeys:
+                                                              description: MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. Also, MatchLabelKeys cannot be set when LabelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                                              type: array
+                                                              items:
+                                                                type: string
+                                                              x-kubernetes-list-type: atomic
+                                                            mismatchLabelKeys:
+                                                              description: MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                                              type: array
+                                                              items:
+                                                                type: string
+                                                              x-kubernetes-list-type: atomic
                                                             namespaceSelector:
                                                               description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces.
                                                               type: object
@@ -777,7 +789,7 @@ spec:
                                                         - topologyKey
                                                       properties:
                                                         labelSelector:
-                                                          description: A label query over a set of resources, in this case pods.
+                                                          description: A label query over a set of resources, in this case pods. If it's null, this PodAffinityTerm matches with no Pods.
                                                           type: object
                                                           properties:
                                                             matchExpressions:
@@ -807,6 +819,18 @@ spec:
                                                               additionalProperties:
                                                                 type: string
                                                           x-kubernetes-map-type: atomic
+                                                        matchLabelKeys:
+                                                          description: MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. Also, MatchLabelKeys cannot be set when LabelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                                          type: array
+                                                          items:
+                                                            type: string
+                                                          x-kubernetes-list-type: atomic
+                                                        mismatchLabelKeys:
+                                                          description: MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                                          type: array
+                                                          items:
+                                                            type: string
+                                                          x-kubernetes-list-type: atomic
                                                         namespaceSelector:
                                                           description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces.
                                                           type: object
@@ -867,7 +891,7 @@ spec:
                                                             - topologyKey
                                                           properties:
                                                             labelSelector:
-                                                              description: A label query over a set of resources, in this case pods.
+                                                              description: A label query over a set of resources, in this case pods. If it's null, this PodAffinityTerm matches with no Pods.
                                                               type: object
                                                               properties:
                                                                 matchExpressions:
@@ -897,6 +921,18 @@ spec:
                                                                   additionalProperties:
                                                                     type: string
                                                               x-kubernetes-map-type: atomic
+                                                            matchLabelKeys:
+                                                              description: MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. Also, MatchLabelKeys cannot be set when LabelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                                              type: array
+                                                              items:
+                                                                type: string
+                                                              x-kubernetes-list-type: atomic
+                                                            mismatchLabelKeys:
+                                                              description: MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                                              type: array
+                                                              items:
+                                                                type: string
+                                                              x-kubernetes-list-type: atomic
                                                             namespaceSelector:
                                                               description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces.
                                                               type: object
@@ -950,7 +986,7 @@ spec:
                                                         - topologyKey
                                                       properties:
                                                         labelSelector:
-                                                          description: A label query over a set of resources, in this case pods.
+                                                          description: A label query over a set of resources, in this case pods. If it's null, this PodAffinityTerm matches with no Pods.
                                                           type: object
                                                           properties:
                                                             matchExpressions:
@@ -980,6 +1016,18 @@ spec:
                                                               additionalProperties:
                                                                 type: string
                                                           x-kubernetes-map-type: atomic
+                                                        matchLabelKeys:
+                                                          description: MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. Also, MatchLabelKeys cannot be set when LabelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                                          type: array
+                                                          items:
+                                                            type: string
+                                                          x-kubernetes-list-type: atomic
+                                                        mismatchLabelKeys:
+                                                          description: MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                                          type: array
+                                                          items:
+                                                            type: string
+                                                          x-kubernetes-list-type: atomic
                                                         namespaceSelector:
                                                           description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces.
                                                           type: object
@@ -1097,6 +1145,11 @@ spec:
                       type: array
                       items:
                         type: string
+                    issuingCertificateURLs:
+                      description: IssuingCertificateURLs is a list of URLs which this issuer should embed into certificates it creates. See https://www.rfc-editor.org/rfc/rfc5280#section-4.2.2.1 for more details. As an example, such a URL might be "http://ca.domain.com/ca.crt".
+                      type: array
+                      items:
+                        type: string
                     ocspServers:
                       description: The OCSP server list is an X.509 v3 extension that defines a list of URLs of OCSP responders. The OCSP responders can be queried for the revocation status of an issued certificate. If not set, the certificate will be issued with no OCSP servers set. For example, an OCSP server URL could be "http://ocsp.int-x3.letsencrypt.org".
                       type: array
@@ -1507,10 +1560,10 @@ spec:
                             - subscriptionID
                           properties:
                             clientID:
-                              description: if both this and ClientSecret are left unset MSI will be used
+                              description: 'Auth: Azure Service Principal: The ClientID of the Azure Service Principal used to authenticate with Azure DNS. If set, ClientSecret and TenantID must also be set.'
                               type: string
                             clientSecretSecretRef:
-                              description: if both this and ClientID are left unset MSI will be used
+                              description: 'Auth: Azure Service Principal: A reference to a Secret containing the password associated with the Service Principal. If set, ClientID and TenantID must also be set.'
                               type: object
                               required:
                                 - name
@@ -1533,14 +1586,14 @@ spec:
                               description: name of the DNS zone that should be used
                               type: string
                             managedIdentity:
-                              description: managed identity configuration, can not be used at the same time as clientID, clientSecretSecretRef or tenantID
+                              description: 'Auth: Azure Workload Identity or Azure Managed Service Identity: Settings to enable Azure Workload Identity or Azure Managed Service Identity If set, ClientID, ClientSecret and TenantID must not be set.'
                               type: object
                               properties:
                                 clientID:
                                   description: client ID of the managed identity, can not be used at the same time as resourceID
                                   type: string
                                 resourceID:
-                                  description: resource ID of the managed identity, can not be used at the same time as clientID
+                                  description: resource ID of the managed identity, can not be used at the same time as clientID Cannot be used for Azure Managed Service Identity
                                   type: string
                             resourceGroupName:
                               description: resource group the DNS zone is located in
@@ -1549,7 +1602,7 @@ spec:
                               description: ID of the Azure subscription
                               type: string
                             tenantID:
-                              description: when specifying ClientID and ClientSecret then this field is also needed
+                              description: 'Auth: Azure Service Principal: The TenantID of the Azure Service Principal used to authenticate with Azure DNS. If set, ClientID and ClientSecret must also be set.'
                               type: string
                         cloudDNS:
                           description: Use the Google Cloud DNS API to manage DNS01 challenge records.
@@ -1755,13 +1808,13 @@ spec:
                                     maxLength: 253
                                     minLength: 1
                                   namespace:
-                                    description: "Namespace is the namespace of the referent. When unspecified, this refers to the local namespace of the Route. \n Note that there are specific rules for ParentRefs which cross namespace boundaries. Cross-namespace references are only valid if they are explicitly allowed by something in the namespace they are referring to. For example: Gateway has the AllowedRoutes field, and ReferenceGrant provides a generic way to enable any other kind of cross-namespace reference. \n ParentRefs from a Route to a Service in the same namespace are \"producer\" routes, which apply default routing rules to inbound connections from any namespace to the Service. \n ParentRefs from a Route to a Service in a different namespace are \"consumer\" routes, and these routing rules are only applied to outbound connections originating from the same namespace as the Route, for which the intended destination of the connections are a Service targeted as a ParentRef of the Route. \n Support: Core"
+                                    description: "Namespace is the namespace of the referent. When unspecified, this refers to the local namespace of the Route. \n Note that there are specific rules for ParentRefs which cross namespace boundaries. Cross-namespace references are only valid if they are explicitly allowed by something in the namespace they are referring to. For example: Gateway has the AllowedRoutes field, and ReferenceGrant provides a generic way to enable any other kind of cross-namespace reference. \n <gateway:experimental:description> ParentRefs from a Route to a Service in the same namespace are \"producer\" routes, which apply default routing rules to inbound connections from any namespace to the Service. \n ParentRefs from a Route to a Service in a different namespace are \"consumer\" routes, and these routing rules are only applied to outbound connections originating from the same namespace as the Route, for which the intended destination of the connections are a Service targeted as a ParentRef of the Route. </gateway:experimental:description> \n Support: Core"
                                     type: string
                                     maxLength: 63
                                     minLength: 1
                                     pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$
                                   port:
-                                    description: "Port is the network port this Route targets. It can be interpreted differently based on the type of parent resource. \n When the parent resource is a Gateway, this targets all listeners listening on the specified port that also support this kind of Route(and select this Route). It's not recommended to set `Port` unless the networking behaviors specified in a Route must apply to a specific port as opposed to a listener(s) whose port(s) may be changed. When both Port and SectionName are specified, the name and port of the selected listener must match both specified values. \n When the parent resource is a Service, this targets a specific port in the Service spec. When both Port (experimental) and SectionName are specified, the name and port of the selected port must match both specified values. \n Implementations MAY choose to support other parent resources. Implementations supporting other types of parent resources MUST clearly document how/if Port is interpreted. \n For the purpose of status, an attachment is considered successful as long as the parent resource accepts it partially. For example, Gateway listeners can restrict which Routes can attach to them by Route kind, namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from the referencing Route, the Route MUST be considered successfully attached. If no Gateway listeners accept attachment from this Route, the Route MUST be considered detached from the Gateway. \n Support: Extended \n <gateway:experimental>"
+                                    description: "Port is the network port this Route targets. It can be interpreted differently based on the type of parent resource. \n When the parent resource is a Gateway, this targets all listeners listening on the specified port that also support this kind of Route(and select this Route). It's not recommended to set `Port` unless the networking behaviors specified in a Route must apply to a specific port as opposed to a listener(s) whose port(s) may be changed. When both Port and SectionName are specified, the name and port of the selected listener must match both specified values. \n <gateway:experimental:description> When the parent resource is a Service, this targets a specific port in the Service spec. When both Port (experimental) and SectionName are specified, the name and port of the selected port must match both specified values. </gateway:experimental:description> \n Implementations MAY choose to support other parent resources. Implementations supporting other types of parent resources MUST clearly document how/if Port is interpreted. \n For the purpose of status, an attachment is considered successful as long as the parent resource accepts it partially. For example, Gateway listeners can restrict which Routes can attach to them by Route kind, namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from the referencing Route, the Route MUST be considered successfully attached. If no Gateway listeners accept attachment from this Route, the Route MUST be considered detached from the Gateway. \n Support: Extended \n <gateway:experimental>"
                                     type: integer
                                     format: int32
                                     maximum: 65535
@@ -1975,7 +2028,7 @@ spec:
                                                       - topologyKey
                                                     properties:
                                                       labelSelector:
-                                                        description: A label query over a set of resources, in this case pods.
+                                                        description: A label query over a set of resources, in this case pods. If it's null, this PodAffinityTerm matches with no Pods.
                                                         type: object
                                                         properties:
                                                           matchExpressions:
@@ -2005,6 +2058,18 @@ spec:
                                                             additionalProperties:
                                                               type: string
                                                         x-kubernetes-map-type: atomic
+                                                      matchLabelKeys:
+                                                        description: MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. Also, MatchLabelKeys cannot be set when LabelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                                        type: array
+                                                        items:
+                                                          type: string
+                                                        x-kubernetes-list-type: atomic
+                                                      mismatchLabelKeys:
+                                                        description: MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                                        type: array
+                                                        items:
+                                                          type: string
+                                                        x-kubernetes-list-type: atomic
                                                       namespaceSelector:
                                                         description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces.
                                                         type: object
@@ -2058,7 +2123,7 @@ spec:
                                                   - topologyKey
                                                 properties:
                                                   labelSelector:
-                                                    description: A label query over a set of resources, in this case pods.
+                                                    description: A label query over a set of resources, in this case pods. If it's null, this PodAffinityTerm matches with no Pods.
                                                     type: object
                                                     properties:
                                                       matchExpressions:
@@ -2088,6 +2153,18 @@ spec:
                                                         additionalProperties:
                                                           type: string
                                                     x-kubernetes-map-type: atomic
+                                                  matchLabelKeys:
+                                                    description: MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. Also, MatchLabelKeys cannot be set when LabelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                                    type: array
+                                                    items:
+                                                      type: string
+                                                    x-kubernetes-list-type: atomic
+                                                  mismatchLabelKeys:
+                                                    description: MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                                    type: array
+                                                    items:
+                                                      type: string
+                                                    x-kubernetes-list-type: atomic
                                                   namespaceSelector:
                                                     description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces.
                                                     type: object
@@ -2148,7 +2225,7 @@ spec:
                                                       - topologyKey
                                                     properties:
                                                       labelSelector:
-                                                        description: A label query over a set of resources, in this case pods.
+                                                        description: A label query over a set of resources, in this case pods. If it's null, this PodAffinityTerm matches with no Pods.
                                                         type: object
                                                         properties:
                                                           matchExpressions:
@@ -2178,6 +2255,18 @@ spec:
                                                             additionalProperties:
                                                               type: string
                                                         x-kubernetes-map-type: atomic
+                                                      matchLabelKeys:
+                                                        description: MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. Also, MatchLabelKeys cannot be set when LabelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                                        type: array
+                                                        items:
+                                                          type: string
+                                                        x-kubernetes-list-type: atomic
+                                                      mismatchLabelKeys:
+                                                        description: MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                                        type: array
+                                                        items:
+                                                          type: string
+                                                        x-kubernetes-list-type: atomic
                                                       namespaceSelector:
                                                         description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces.
                                                         type: object
@@ -2231,7 +2320,7 @@ spec:
                                                   - topologyKey
                                                 properties:
                                                   labelSelector:
-                                                    description: A label query over a set of resources, in this case pods.
+                                                    description: A label query over a set of resources, in this case pods. If it's null, this PodAffinityTerm matches with no Pods.
                                                     type: object
                                                     properties:
                                                       matchExpressions:
@@ -2261,6 +2350,18 @@ spec:
                                                         additionalProperties:
                                                           type: string
                                                     x-kubernetes-map-type: atomic
+                                                  matchLabelKeys:
+                                                    description: MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. Also, MatchLabelKeys cannot be set when LabelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                                    type: array
+                                                    items:
+                                                      type: string
+                                                    x-kubernetes-list-type: atomic
+                                                  mismatchLabelKeys:
+                                                    description: MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                                    type: array
+                                                    items:
+                                                      type: string
+                                                    x-kubernetes-list-type: atomic
                                                   namespaceSelector:
                                                     description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces.
                                                     type: object
@@ -2822,10 +2923,10 @@ spec:
                                   - subscriptionID
                                 properties:
                                   clientID:
-                                    description: if both this and ClientSecret are left unset MSI will be used
+                                    description: 'Auth: Azure Service Principal: The ClientID of the Azure Service Principal used to authenticate with Azure DNS. If set, ClientSecret and TenantID must also be set.'
                                     type: string
                                   clientSecretSecretRef:
-                                    description: if both this and ClientID are left unset MSI will be used
+                                    description: 'Auth: Azure Service Principal: A reference to a Secret containing the password associated with the Service Principal. If set, ClientID and TenantID must also be set.'
                                     type: object
                                     required:
                                       - name
@@ -2848,14 +2949,14 @@ spec:
                                     description: name of the DNS zone that should be used
                                     type: string
                                   managedIdentity:
-                                    description: managed identity configuration, can not be used at the same time as clientID, clientSecretSecretRef or tenantID
+                                    description: 'Auth: Azure Workload Identity or Azure Managed Service Identity: Settings to enable Azure Workload Identity or Azure Managed Service Identity If set, ClientID, ClientSecret and TenantID must not be set.'
                                     type: object
                                     properties:
                                       clientID:
                                         description: client ID of the managed identity, can not be used at the same time as resourceID
                                         type: string
                                       resourceID:
-                                        description: resource ID of the managed identity, can not be used at the same time as clientID
+                                        description: resource ID of the managed identity, can not be used at the same time as clientID Cannot be used for Azure Managed Service Identity
                                         type: string
                                   resourceGroupName:
                                     description: resource group the DNS zone is located in
@@ -2864,7 +2965,7 @@ spec:
                                     description: ID of the Azure subscription
                                     type: string
                                   tenantID:
-                                    description: when specifying ClientID and ClientSecret then this field is also needed
+                                    description: 'Auth: Azure Service Principal: The TenantID of the Azure Service Principal used to authenticate with Azure DNS. If set, ClientID and ClientSecret must also be set.'
                                     type: string
                               cloudDNS:
                                 description: Use the Google Cloud DNS API to manage DNS01 challenge records.
@@ -3070,13 +3171,13 @@ spec:
                                           maxLength: 253
                                           minLength: 1
                                         namespace:
-                                          description: "Namespace is the namespace of the referent. When unspecified, this refers to the local namespace of the Route. \n Note that there are specific rules for ParentRefs which cross namespace boundaries. Cross-namespace references are only valid if they are explicitly allowed by something in the namespace they are referring to. For example: Gateway has the AllowedRoutes field, and ReferenceGrant provides a generic way to enable any other kind of cross-namespace reference. \n ParentRefs from a Route to a Service in the same namespace are \"producer\" routes, which apply default routing rules to inbound connections from any namespace to the Service. \n ParentRefs from a Route to a Service in a different namespace are \"consumer\" routes, and these routing rules are only applied to outbound connections originating from the same namespace as the Route, for which the intended destination of the connections are a Service targeted as a ParentRef of the Route. \n Support: Core"
+                                          description: "Namespace is the namespace of the referent. When unspecified, this refers to the local namespace of the Route. \n Note that there are specific rules for ParentRefs which cross namespace boundaries. Cross-namespace references are only valid if they are explicitly allowed by something in the namespace they are referring to. For example: Gateway has the AllowedRoutes field, and ReferenceGrant provides a generic way to enable any other kind of cross-namespace reference. \n <gateway:experimental:description> ParentRefs from a Route to a Service in the same namespace are \"producer\" routes, which apply default routing rules to inbound connections from any namespace to the Service. \n ParentRefs from a Route to a Service in a different namespace are \"consumer\" routes, and these routing rules are only applied to outbound connections originating from the same namespace as the Route, for which the intended destination of the connections are a Service targeted as a ParentRef of the Route. </gateway:experimental:description> \n Support: Core"
                                           type: string
                                           maxLength: 63
                                           minLength: 1
                                           pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$
                                         port:
-                                          description: "Port is the network port this Route targets. It can be interpreted differently based on the type of parent resource. \n When the parent resource is a Gateway, this targets all listeners listening on the specified port that also support this kind of Route(and select this Route). It's not recommended to set `Port` unless the networking behaviors specified in a Route must apply to a specific port as opposed to a listener(s) whose port(s) may be changed. When both Port and SectionName are specified, the name and port of the selected listener must match both specified values. \n When the parent resource is a Service, this targets a specific port in the Service spec. When both Port (experimental) and SectionName are specified, the name and port of the selected port must match both specified values. \n Implementations MAY choose to support other parent resources. Implementations supporting other types of parent resources MUST clearly document how/if Port is interpreted. \n For the purpose of status, an attachment is considered successful as long as the parent resource accepts it partially. For example, Gateway listeners can restrict which Routes can attach to them by Route kind, namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from the referencing Route, the Route MUST be considered successfully attached. If no Gateway listeners accept attachment from this Route, the Route MUST be considered detached from the Gateway. \n Support: Extended \n <gateway:experimental>"
+                                          description: "Port is the network port this Route targets. It can be interpreted differently based on the type of parent resource. \n When the parent resource is a Gateway, this targets all listeners listening on the specified port that also support this kind of Route(and select this Route). It's not recommended to set `Port` unless the networking behaviors specified in a Route must apply to a specific port as opposed to a listener(s) whose port(s) may be changed. When both Port and SectionName are specified, the name and port of the selected listener must match both specified values. \n <gateway:experimental:description> When the parent resource is a Service, this targets a specific port in the Service spec. When both Port (experimental) and SectionName are specified, the name and port of the selected port must match both specified values. </gateway:experimental:description> \n Implementations MAY choose to support other parent resources. Implementations supporting other types of parent resources MUST clearly document how/if Port is interpreted. \n For the purpose of status, an attachment is considered successful as long as the parent resource accepts it partially. For example, Gateway listeners can restrict which Routes can attach to them by Route kind, namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from the referencing Route, the Route MUST be considered successfully attached. If no Gateway listeners accept attachment from this Route, the Route MUST be considered detached from the Gateway. \n Support: Extended \n <gateway:experimental>"
                                           type: integer
                                           format: int32
                                           maximum: 65535
@@ -3290,7 +3391,7 @@ spec:
                                                             - topologyKey
                                                           properties:
                                                             labelSelector:
-                                                              description: A label query over a set of resources, in this case pods.
+                                                              description: A label query over a set of resources, in this case pods. If it's null, this PodAffinityTerm matches with no Pods.
                                                               type: object
                                                               properties:
                                                                 matchExpressions:
@@ -3320,6 +3421,18 @@ spec:
                                                                   additionalProperties:
                                                                     type: string
                                                               x-kubernetes-map-type: atomic
+                                                            matchLabelKeys:
+                                                              description: MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. Also, MatchLabelKeys cannot be set when LabelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                                              type: array
+                                                              items:
+                                                                type: string
+                                                              x-kubernetes-list-type: atomic
+                                                            mismatchLabelKeys:
+                                                              description: MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                                              type: array
+                                                              items:
+                                                                type: string
+                                                              x-kubernetes-list-type: atomic
                                                             namespaceSelector:
                                                               description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces.
                                                               type: object
@@ -3373,7 +3486,7 @@ spec:
                                                         - topologyKey
                                                       properties:
                                                         labelSelector:
-                                                          description: A label query over a set of resources, in this case pods.
+                                                          description: A label query over a set of resources, in this case pods. If it's null, this PodAffinityTerm matches with no Pods.
                                                           type: object
                                                           properties:
                                                             matchExpressions:
@@ -3403,6 +3516,18 @@ spec:
                                                               additionalProperties:
                                                                 type: string
                                                           x-kubernetes-map-type: atomic
+                                                        matchLabelKeys:
+                                                          description: MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. Also, MatchLabelKeys cannot be set when LabelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                                          type: array
+                                                          items:
+                                                            type: string
+                                                          x-kubernetes-list-type: atomic
+                                                        mismatchLabelKeys:
+                                                          description: MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                                          type: array
+                                                          items:
+                                                            type: string
+                                                          x-kubernetes-list-type: atomic
                                                         namespaceSelector:
                                                           description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces.
                                                           type: object
@@ -3463,7 +3588,7 @@ spec:
                                                             - topologyKey
                                                           properties:
                                                             labelSelector:
-                                                              description: A label query over a set of resources, in this case pods.
+                                                              description: A label query over a set of resources, in this case pods. If it's null, this PodAffinityTerm matches with no Pods.
                                                               type: object
                                                               properties:
                                                                 matchExpressions:
@@ -3493,6 +3618,18 @@ spec:
                                                                   additionalProperties:
                                                                     type: string
                                                               x-kubernetes-map-type: atomic
+                                                            matchLabelKeys:
+                                                              description: MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. Also, MatchLabelKeys cannot be set when LabelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                                              type: array
+                                                              items:
+                                                                type: string
+                                                              x-kubernetes-list-type: atomic
+                                                            mismatchLabelKeys:
+                                                              description: MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                                              type: array
+                                                              items:
+                                                                type: string
+                                                              x-kubernetes-list-type: atomic
                                                             namespaceSelector:
                                                               description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces.
                                                               type: object
@@ -3546,7 +3683,7 @@ spec:
                                                         - topologyKey
                                                       properties:
                                                         labelSelector:
-                                                          description: A label query over a set of resources, in this case pods.
+                                                          description: A label query over a set of resources, in this case pods. If it's null, this PodAffinityTerm matches with no Pods.
                                                           type: object
                                                           properties:
                                                             matchExpressions:
@@ -3576,6 +3713,18 @@ spec:
                                                               additionalProperties:
                                                                 type: string
                                                           x-kubernetes-map-type: atomic
+                                                        matchLabelKeys:
+                                                          description: MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. Also, MatchLabelKeys cannot be set when LabelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                                          type: array
+                                                          items:
+                                                            type: string
+                                                          x-kubernetes-list-type: atomic
+                                                        mismatchLabelKeys:
+                                                          description: MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+                                                          type: array
+                                                          items:
+                                                            type: string
+                                                          x-kubernetes-list-type: atomic
                                                         namespaceSelector:
                                                           description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces.
                                                           type: object
@@ -3693,6 +3842,11 @@ spec:
                       type: array
                       items:
                         type: string
+                    issuingCertificateURLs:
+                      description: IssuingCertificateURLs is a list of URLs which this issuer should embed into certificates it creates. See https://www.rfc-editor.org/rfc/rfc5280#section-4.2.2.1 for more details. As an example, such a URL might be "http://ca.domain.com/ca.crt".
+                      type: array
+                      items:
+                        type: string
                     ocspServers:
                       description: The OCSP server list is an X.509 v3 extension that defines a list of URLs of OCSP responders. The OCSP responders can be queried for the revocation status of an issued certificate. If not set, the certificate will be issued with no OCSP servers set. For example, an OCSP server URL could be "http://ocsp.int-x3.letsencrypt.org".
                       type: array
@@ -4100,9 +4254,83 @@ spec:
                             name:
                               description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
                               type: string
+                        profile:
+                          description: "Profile specifies the key and certificate encryption algorithms and the HMAC algorithm used to create the PKCS12 keystore. Default value is `LegacyRC2` for backward compatibility. \n If provided, allowed values are: `LegacyRC2`: Deprecated. Not supported by default in OpenSSL 3 or Java 20. `LegacyDES`: Less secure algorithm. Use this option for maximal compatibility. `Modern2023`: Secure algorithm. Use this option in case you have to always use secure algorithms (eg. because of company policy). Please note that the security of the algorithm is not that important in reality, because the unencrypted certificate and private key are also stored in the Secret."
+                          type: string
+                          enum:
+                            - LegacyRC2
+                            - LegacyDES
+                            - Modern2023
                 literalSubject:
                   description: "Requested X.509 certificate subject, represented using the LDAP \"String Representation of a Distinguished Name\" [1]. Important: the LDAP string format also specifies the order of the attributes in the subject, this is important when issuing certs for LDAP authentication. Example: `CN=foo,DC=corp,DC=example,DC=com` More info [1]: https://datatracker.ietf.org/doc/html/rfc4514 More info: https://github.com/cert-manager/cert-manager/issues/3203 More info: https://github.com/cert-manager/cert-manager/issues/4424 \n Cannot be set if the `subject` or `commonName` field is set. This is an Alpha Feature and is only enabled with the `--feature-gates=LiteralCertificateSubject=true` option set on both the controller and webhook components."
                   type: string
+                nameConstraints:
+                  description: "x.509 certificate NameConstraint extension which MUST NOT be used in a non-CA certificate. More Info: https://datatracker.ietf.org/doc/html/rfc5280#section-4.2.1.10 \n This is an Alpha Feature and is only enabled with the `--feature-gates=NameConstraints=true` option set on both the controller and webhook components."
+                  type: object
+                  properties:
+                    critical:
+                      description: if true then the name constraints are marked critical.
+                      type: boolean
+                    excluded:
+                      description: Excluded contains the constraints which must be disallowed. Any name matching a restriction in the excluded field is invalid regardless of information appearing in the permitted
+                      type: object
+                      properties:
+                        dnsDomains:
+                          description: DNSDomains is a list of DNS domains that are permitted or excluded.
+                          type: array
+                          items:
+                            type: string
+                        emailAddresses:
+                          description: EmailAddresses is a list of Email Addresses that are permitted or excluded.
+                          type: array
+                          items:
+                            type: string
+                        ipRanges:
+                          description: IPRanges is a list of IP Ranges that are permitted or excluded. This should be a valid CIDR notation.
+                          type: array
+                          items:
+                            type: string
+                        uriDomains:
+                          description: URIDomains is a list of URI domains that are permitted or excluded.
+                          type: array
+                          items:
+                            type: string
+                    permitted:
+                      description: Permitted contains the constraints in which the names must be located.
+                      type: object
+                      properties:
+                        dnsDomains:
+                          description: DNSDomains is a list of DNS domains that are permitted or excluded.
+                          type: array
+                          items:
+                            type: string
+                        emailAddresses:
+                          description: EmailAddresses is a list of Email Addresses that are permitted or excluded.
+                          type: array
+                          items:
+                            type: string
+                        ipRanges:
+                          description: IPRanges is a list of IP Ranges that are permitted or excluded. This should be a valid CIDR notation.
+                          type: array
+                          items:
+                            type: string
+                        uriDomains:
+                          description: URIDomains is a list of URI domains that are permitted or excluded.
+                          type: array
+                          items:
+                            type: string
+                otherNames:
+                  description: '`otherNames` is an escape hatch for SAN that allows any type. We currently restrict the support to string like otherNames, cf RFC 5280 p 37 Any UTF8 String valued otherName can be passed with by setting the keys oid: x.x.x.x and UTF8Value: somevalue for `otherName`. Most commonly this would be UPN set with oid: 1.3.6.1.4.1.311.20.2.3 You should ensure that any OID passed is valid for the UTF8String type as we do not explicitly validate this.'
+                  type: array
+                  items:
+                    type: object
+                    properties:
+                      oid:
+                        description: OID is the object identifier for the otherName SAN. The object identifier must be expressed as a dotted string, for example, "1.2.840.113556.1.4.221".
+                        type: string
+                      utf8Value:
+                        description: utf8Value is the string value of the otherName SAN. The utf8Value accepts any valid UTF8 string to set as value for the otherName SAN.
+                        type: string
                 privateKey:
                   description: Private key options. These include the key algorithm and size, the used encoding and the rotation policy.
                   type: object

kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/defaults/main.yml

@@ -2,6 +2,8 @@
 ingress_nginx_namespace: "ingress-nginx"
 ingress_nginx_host_network: false
 ingress_nginx_service_type: LoadBalancer
+ingress_nginx_service_nodeport_http: ""
+ingress_nginx_service_nodeport_https: ""
 ingress_publish_status_address: ""
 ingress_nginx_nodeselector:
   kubernetes.io/os: "linux"
@@ -19,5 +21,6 @@ ingress_nginx_without_class: true
 ingress_nginx_default: false
 ingress_nginx_webhook_enabled: false
 ingress_nginx_webhook_job_ttl: 1800
+ingress_nginx_opentelemetry_enabled: false
 
 ingress_nginx_probe_initial_delay_seconds: 10

kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/tasks/main.yml

@@ -6,7 +6,7 @@
     state: directory
     owner: root
     group: root
-    mode: 0755
+    mode: "0755"
   when:
     - inventory_hostname == groups['kube_control_plane'][0]
 
@@ -34,6 +34,7 @@
       - { name: role-admission-webhook, file: role-admission-webhook.yml, type: role }
       - { name: rolebinding-admission-webhook, file: rolebinding-admission-webhook.yml, type: rolebinding }
       - { name: admission-webhook-job, file: admission-webhook-job.yml, type: job }
+      - { name: svc-ingress-nginx-controller-admission, file: svc-ingress-nginx-controller-admission.yml, type: svc }
 
 - name: NGINX Ingress Controller | Append extra templates to NGINX Ingress Template list for service
   set_fact:
@@ -49,7 +50,7 @@
   template:
     src: "{{ item.file }}.j2"
     dest: "{{ kube_config_dir }}/addons/ingress_nginx/{{ item.file }}"
-    mode: 0644
+    mode: "0644"
   with_items: "{{ ingress_nginx_templates }}"
   register: ingress_nginx_manifests
   when:

kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ds-ingress-nginx-controller.yml.j2

@@ -23,6 +23,26 @@ spec:
     spec:
       serviceAccountName: ingress-nginx
       terminationGracePeriodSeconds: {{ ingress_nginx_termination_grace_period_seconds }}
+{% if ingress_nginx_opentelemetry_enabled %}
+      initContainers:
+      - name: opentelemetry
+        command:
+        - /init_module
+        image: {{ ingress_nginx_opentelemetry_image_repo }}:{{ ingress_nginx_opentelemetry_image_tag }}
+        securityContext:
+          runAsNonRoot: true
+          runAsUser: 65532
+          allowPrivilegeEscalation: false
+          seccompProfile:
+            type: RuntimeDefault
+          capabilities:
+            drop:
+              - ALL
+          readOnlyRootFilesystem: true
+        volumeMounts:
+        - mountPath: /modules_mount
+          name: modules
+{% endif %}
 {% if ingress_nginx_host_network %}
       hostNetwork: true
       dnsPolicy: ClusterFirstWithHostNet
@@ -127,15 +147,27 @@ spec:
             timeoutSeconds: 5
             successThreshold: 1
             failureThreshold: 3
-{% if ingress_nginx_webhook_enabled %}
+{% if ingress_nginx_webhook_enabled or ingress_nginx_opentelemetry_enabled %}
           volumeMounts:
+{% if ingress_nginx_webhook_enabled %}
             - mountPath: /usr/local/certificates/
               name: webhook-cert
               readOnly: true
 {% endif %}
-{% if ingress_nginx_webhook_enabled %}
+{% if ingress_nginx_opentelemetry_enabled %}
+            - name: modules
+              mountPath: /modules_mount
+{% endif %}
+{% endif %}
+{% if ingress_nginx_webhook_enabled or ingress_nginx_opentelemetry_enabled %}
       volumes:
+{% if ingress_nginx_webhook_enabled %}
         - name: webhook-cert
           secret:
             secretName: ingress-nginx-admission
 {% endif %}
+{% if ingress_nginx_opentelemetry_enabled %}
+        - name: modules
+          emptyDir: {}
+{% endif %}
+{% endif %}

kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/svc-ingress-nginx-controller-admission.yml.j2

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+  name: ingress-nginx-controller-admission
+  namespace: {{ ingress_nginx_namespace }}
+spec:
+  type: ClusterIP
+  ports:
+  - appProtocol: https
+    name: https-webhook
+    port: 443
+    targetPort: webhook
+  selector:
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx

kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/svc-ingress-nginx.yml.j2

@@ -14,11 +14,17 @@ spec:
       port: 80
       targetPort: 80
       protocol: TCP
+{% if (ingress_nginx_service_type == 'NodePort' or ingress_nginx_service_type == 'LoadBalancer') and ingress_nginx_service_nodeport_http %}
+      nodePort: {{ingress_nginx_service_nodeport_http | int}}
+{% endif %}
     - name: https
       port: 443
       targetPort: 443
       protocol: TCP
+{% if (ingress_nginx_service_type == 'NodePort' or ingress_nginx_service_type == 'LoadBalancer') and ingress_nginx_service_nodeport_https %}
+      nodePort: {{ingress_nginx_service_nodeport_https | int}}
+{% endif %}
   selector:
     app.kubernetes.io/name: ingress-nginx
-    app.kubernetes.io/port-of: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
 {% endif %}

kubespray/project/roles/kubernetes-apps/krew/tasks/krew.yml

@@ -8,13 +8,13 @@
   template:
     src: krew.j2
     dest: /etc/bash_completion.d/krew
-    mode: 0644
+    mode: "0644"
 
 - name: Krew | Copy krew manifest
   template:
     src: krew.yml.j2
     dest: "{{ local_release_dir }}/krew.yml"
-    mode: 0644
+    mode: "0644"
 
 - name: Krew | Install krew  # noqa command-instead-of-shell
   shell: "{{ local_release_dir }}/krew-{{ host_os }}_{{ image_arch }} install --archive={{ local_release_dir }}/krew-{{ host_os }}_{{ image_arch }}.tar.gz --manifest={{ local_release_dir }}/krew.yml"
@@ -24,15 +24,15 @@
 
 - name: Krew | Get krew completion
   command: "{{ local_release_dir }}/krew-{{ host_os }}_{{ image_arch }} completion bash"
-  changed_when: False
+  changed_when: false
   register: krew_completion
-  check_mode: False
-  ignore_errors: yes  # noqa ignore-errors
+  check_mode: false
+  ignore_errors: true  # noqa ignore-errors
 
 - name: Krew | Install krew completion
   copy:
     dest: /etc/bash_completion.d/krew.sh
     content: "{{ krew_completion.stdout }}"
-    mode: 0755
-  become: True
+    mode: "0755"
+  become: true
   when: krew_completion.rc == 0

kubespray/project/roles/kubernetes-apps/kubelet-csr-approver/defaults/main.yml

@@ -5,7 +5,7 @@ kubelet_csr_approver_namespace: kube-system
 kubelet_csr_approver_repository_name: kubelet-csr-approver
 kubelet_csr_approver_repository_url: https://postfinance.github.io/kubelet-csr-approver
 kubelet_csr_approver_chart_ref: "{{ kubelet_csr_approver_repository_name }}/kubelet-csr-approver"
-kubelet_csr_approver_chart_version: 0.2.8
+kubelet_csr_approver_chart_version: 1.1.0
 
 # Fill values override here
 # See upstream https://github.com/postfinance/kubelet-csr-approver

kubespray/project/roles/kubernetes-apps/meta/main.yml

@@ -111,6 +111,13 @@ dependencies:
     tags:
       - oci
 
+  - role: kubernetes-apps/gateway_api
+    when:
+      - gateway_api_enabled
+      - inventory_hostname == groups['kube_control_plane'][0]
+    tags:
+      - gateway_api
+
   - role: kubernetes-apps/metallb
     when:
       - metallb_enabled
@@ -128,7 +135,14 @@ dependencies:
   - role: kubernetes-apps/scheduler_plugins
     when:
       - scheduler_plugins_enabled
-      - kube_major_version is version('v1.28', '<')
+      - kube_major_version is version('v1.29', '<')
       - inventory_hostname == groups['kube_control_plane'][0]
     tags:
       - scheduler_plugins
+
+  - role: kubernetes-apps/node_feature_discovery
+    when:
+      - node_feature_discovery_enabled
+      - inventory_hostname == groups['kube_control_plane'][0]
+    tags:
+      - node_feature_discovery

kubespray/project/roles/kubernetes-apps/metallb/defaults/main.yml

@@ -13,4 +13,4 @@ metallb_speaker_tolerations:
     key: node-role.kubernetes.io/control-plane
     operator: Exists
 metallb_controller_tolerations: []
-metallb_loadbalancer_class: ""
+metallb_loadbalancer_class: ""

kubespray/project/roles/kubernetes-apps/metallb/tasks/main.yml

@@ -16,7 +16,7 @@
   template:
     src: "metallb.yaml.j2"
     dest: "{{ kube_config_dir }}/metallb.yaml"
-    mode: 0644
+    mode: "0644"
   register: metallb_rendering
   when:
     - inventory_hostname == groups['kube_control_plane'][0]
@@ -33,7 +33,7 @@
     - inventory_hostname == groups['kube_control_plane'][0]
 
 - name: Kubernetes Apps | Wait for MetalLB controller to be running
-  command: "{{ bin_dir }}/kubectl rollout status -n metallb-system deployment -l app=metallb,component=controller --timeout=2m"
+  command: "{{ bin_dir }}/kubectl rollout status -n {{ metallb_namespace }} deployment -l app=metallb,component=controller --timeout=2m"
   become: true
   when:
     - inventory_hostname == groups['kube_control_plane'][0]
@@ -47,7 +47,7 @@
       ansible.builtin.template:
         src: pools.yaml.j2
         dest: "{{ kube_config_dir }}/pools.yaml"
-        mode: 0644
+        mode: "0644"
       register: pools_rendering
 
     - name: MetalLB | Create address pools configuration
@@ -67,7 +67,7 @@
       ansible.builtin.template:
         src: layer2.yaml.j2
         dest: "{{ kube_config_dir }}/layer2.yaml"
-        mode: 0644
+        mode: "0644"
       register: layer2_rendering
 
     - name: MetalLB | Create layer2 configuration
@@ -87,7 +87,7 @@
       ansible.builtin.template:
         src: layer3.yaml.j2
         dest: "{{ kube_config_dir }}/layer3.yaml"
-        mode: 0644
+        mode: "0644"
       register: layer3_rendering
 
     - name: MetalLB | Create layer3 configuration
@@ -104,5 +104,5 @@
     name: config
     kubectl: "{{ bin_dir }}/kubectl"
     resource: ConfigMap
-    namespace: metallb-system
+    namespace: "{{ metallb_namespace }}"
     state: absent

kubespray/project/roles/kubernetes-apps/metallb/templates/layer2.yaml.j2

@@ -11,7 +11,7 @@ apiVersion: metallb.io/v1beta1
 kind: L2Advertisement
 metadata:
   name: "{{ entry }}"
-  namespace: metallb-system
+  namespace: "{{ metallb_namespace }}"
 spec:
   ipAddressPools:
   - "{{ entry }}"

kubespray/project/roles/kubernetes-apps/metallb/templates/layer3.yaml.j2

@@ -9,7 +9,7 @@ apiVersion: metallb.io/v1beta1
 kind: Community
 metadata:
   name: "{{ community_name }}"
-  namespace: metallb-system
+  namespace: "{{ metallb_namespace }}"
 spec:
   communities:
   - name: "{{ community_name }}"
@@ -21,7 +21,7 @@ apiVersion: metallb.io/v1beta1
 kind: Community
 metadata:
   name: well-known
-  namespace: metallb-system
+  namespace: "{{ metallb_namespace }}"
 spec:
   communities:
   - name: no-export
@@ -51,7 +51,7 @@ apiVersion: metallb.io/v1beta1
 kind: BGPAdvertisement
 metadata:
   name: "{{ peer_name }}-local"
-  namespace: metallb-system
+  namespace: "{{ metallb_namespace }}"
 spec:
   aggregationLength: 32
   aggregationLengthV6: 128
@@ -70,7 +70,7 @@ apiVersion: metallb.io/v1beta1
 kind: BGPAdvertisement
 metadata:
   name: "{{ peer_name }}-external"
-  namespace: metallb-system
+  namespace: "{{ metallb_namespace }}"
 spec:
   {% if peer.aggregation_length is defined and peer.aggregation_length <= 30 %}
   aggregationLength: {{ peer.aggregation_length }}
@@ -93,7 +93,7 @@ apiVersion: metallb.io/v1beta2
 kind: BGPPeer
 metadata:
   name: "{{ peer_name }}"
-  namespace: metallb-system
+  namespace: "{{ metallb_namespace }}"
 spec:
   myASN: {{ peer.my_asn }}
   peerASN: {{ peer.peer_asn }}

kubespray/project/roles/kubernetes-apps/metallb/templates/metallb.yaml.j2

@@ -6,7 +6,7 @@ metadata:
     pod-security.kubernetes.io/audit: privileged
     pod-security.kubernetes.io/enforce: privileged
     pod-security.kubernetes.io/warn: privileged
-  name: metallb-system
+  name: {{ metallb_namespace }}
 
 ---
 apiVersion: apiextensions.k8s.io/v1
@@ -23,7 +23,7 @@ spec:
         caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tDQpNSUlGWlRDQ0EwMmdBd0lCQWdJVU5GRW1XcTM3MVpKdGkrMmlSQzk1WmpBV1MxZ3dEUVlKS29aSWh2Y05BUUVMDQpCUUF3UWpFTE1Ba0dBMVVFQmhNQ1dGZ3hGVEFUQmdOVkJBY01ERVJsWm1GMWJIUWdRMmwwZVRFY01Cb0dBMVVFDQpDZ3dUUkdWbVlYVnNkQ0JEYjIxd1lXNTVJRXgwWkRBZUZ3MHlNakEzTVRrd09UTXlNek5hRncweU1qQTRNVGd3DQpPVE15TXpOYU1FSXhDekFKQmdOVkJBWVRBbGhZTVJVd0V3WURWUVFIREF4RVpXWmhkV3gwSUVOcGRIa3hIREFhDQpCZ05WQkFvTUUwUmxabUYxYkhRZ1EyOXRjR0Z1ZVNCTWRHUXdnZ0lpTUEwR0NTcUdTSWIzRFFFQkFRVUFBNElDDQpEd0F3Z2dJS0FvSUNBUUNxVFpxMWZRcC9vYkdlenhES0o3OVB3Ny94azJwellualNzMlkzb1ZYSm5sRmM4YjVlDQpma2ZZQnY2bndscW1keW5PL2phWFBaQmRQSS82aFdOUDBkdVhadEtWU0NCUUpyZzEyOGNXb3F0MGNTN3pLb1VpDQpvcU1tQ0QvRXVBeFFNZjhRZDF2c1gvVllkZ0poVTZBRXJLZEpIaXpFOUJtUkNkTDBGMW1OVW55Rk82UnRtWFZUDQpidkxsTDVYeTc2R0FaQVBLOFB4aVlDa0NtbDdxN0VnTWNiOXlLWldCYmlxQ3VkTXE5TGJLNmdKNzF6YkZnSXV4DQo1L1pXK2JraTB2RlplWk9ZODUxb1psckFUNzJvMDI4NHNTWW9uN0pHZVZkY3NoUnh5R1VpSFpSTzdkaXZVTDVTDQpmM2JmSDFYbWY1ZDQzT0NWTWRuUUV2NWVaOG8zeWVLa3ZrbkZQUGVJMU9BbjdGbDlFRVNNR2dhOGFaSG1URSttDQpsLzlMSmdDYjBnQmtPT0M0WnV4bWh2aERKV1EzWnJCS3pMQlNUZXN0NWlLNVlwcXRWVVk2THRyRW9FelVTK1lsDQpwWndXY2VQWHlHeHM5ZURsR3lNVmQraW15Y3NTU1UvVno2Mmx6MnZCS21NTXBkYldDQWhud0RsRTVqU2dyMjRRDQp0eGNXLys2N3d5KzhuQlI3UXdqVTFITndVRjBzeERWdEwrZ1NHVERnSEVZSlhZelYvT05zMy94TkpoVFNPSkxNDQpoeXNVdyttaGdackdhbUdXcHVIVU1DUitvTWJzMTc1UkcrQjJnUFFHVytPTjJnUTRyOXN2b0ZBNHBBQm8xd1dLDQpRYjRhY3pmeVVscElBOVFoSmFsZEY3S3dPSHVlV3gwRUNrNXg0T2tvVDBvWVp0dzFiR0JjRGtaSmF3SURBUUFCDQpvMU13VVRBZEJnTlZIUTRFRmdRVW90UlNIUm9IWTEyRFZ4R0NCdEhpb1g2ZmVFQXdId1lEVlIwakJCZ3dGb0FVDQpvdFJTSFJvSFkxMkRWeEdDQnRIaW9YNmZlRUF3RHdZRFZSMFRBUUgvQkFVd0F3RUIvekFOQmdrcWhraUc5dzBCDQpBUXNGQUFPQ0FnRUFSbkpsWWRjMTFHd0VxWnh6RDF2R3BDR2pDN2VWTlQ3aVY1d3IybXlybHdPYi9aUWFEa0xYDQpvVStaOVVXT1VlSXJTdzUydDdmQUpvVVAwSm5iYkMveVIrU1lqUGhvUXNiVHduOTc2ZldBWTduM3FMOXhCd1Y0DQphek41OXNjeUp0dlhMeUtOL2N5ak1ReDRLajBIMFg0bWJ6bzVZNUtzWWtYVU0vOEFPdWZMcEd0S1NGVGgrSEFDDQpab1Q5YnZHS25adnNHd0tYZFF0Wnh0akhaUjVqK3U3ZGtQOTJBT051RFNabS8rWVV4b2tBK09JbzdSR3BwSHNXDQo1ZTdNY0FTVXRtb1FORXd6dVFoVkJaRWQ1OGtKYjUrV0VWbGNzanlXNnRTbzErZ25tTWNqR1BsMWgxR2hVbjV4DQpFY0lWRnBIWXM5YWo1NmpBSjk1MVQvZjhMaWxmTlVnanBLQ0c1bnl0SUt3emxhOHNtdGlPdm1UNEpYbXBwSkI2DQo4bmdHRVluVjUrUTYwWFJ2OEhSSGp1VG9CRHVhaERrVDA2R1JGODU1d09FR2V4bkZpMXZYWUxLVllWb1V2MXRKDQo4dVdUR1pwNllDSVJldlBqbzg5ZytWTlJSaVFYUThJd0dybXE5c0RoVTlqTjA0SjdVL1RvRDFpNHE3VnlsRUc5DQorV1VGNkNLaEdBeTJIaEhwVncyTGFoOS9lUzdZMUZ1YURrWmhPZG1laG1BOCtqdHNZamJadnR5Mm1SWlF0UUZzDQpUU1VUUjREbUR2bVVPRVRmeStpRHdzK2RkWXVNTnJGeVVYV2dkMnpBQU4ydVl1UHFGY2pRcFNPODFzVTJTU3R3DQoxVzAyeUtYOGJEYmZFdjBzbUh3UzliQnFlSGo5NEM1Mjg0YXpsdTBmaUdpTm1OUEM4ckJLRmhBPQ0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ==
         service:
           name: webhook-service
-          namespace: metallb-system
+          namespace: "{{ metallb_namespace }}"
           path: /convert
       conversionReviewVersions:
       - v1alpha1
@@ -544,7 +544,7 @@ spec:
         caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tDQpNSUlGWlRDQ0EwMmdBd0lCQWdJVU5GRW1XcTM3MVpKdGkrMmlSQzk1WmpBV1MxZ3dEUVlKS29aSWh2Y05BUUVMDQpCUUF3UWpFTE1Ba0dBMVVFQmhNQ1dGZ3hGVEFUQmdOVkJBY01ERVJsWm1GMWJIUWdRMmwwZVRFY01Cb0dBMVVFDQpDZ3dUUkdWbVlYVnNkQ0JEYjIxd1lXNTVJRXgwWkRBZUZ3MHlNakEzTVRrd09UTXlNek5hRncweU1qQTRNVGd3DQpPVE15TXpOYU1FSXhDekFKQmdOVkJBWVRBbGhZTVJVd0V3WURWUVFIREF4RVpXWmhkV3gwSUVOcGRIa3hIREFhDQpCZ05WQkFvTUUwUmxabUYxYkhRZ1EyOXRjR0Z1ZVNCTWRHUXdnZ0lpTUEwR0NTcUdTSWIzRFFFQkFRVUFBNElDDQpEd0F3Z2dJS0FvSUNBUUNxVFpxMWZRcC9vYkdlenhES0o3OVB3Ny94azJwellualNzMlkzb1ZYSm5sRmM4YjVlDQpma2ZZQnY2bndscW1keW5PL2phWFBaQmRQSS82aFdOUDBkdVhadEtWU0NCUUpyZzEyOGNXb3F0MGNTN3pLb1VpDQpvcU1tQ0QvRXVBeFFNZjhRZDF2c1gvVllkZ0poVTZBRXJLZEpIaXpFOUJtUkNkTDBGMW1OVW55Rk82UnRtWFZUDQpidkxsTDVYeTc2R0FaQVBLOFB4aVlDa0NtbDdxN0VnTWNiOXlLWldCYmlxQ3VkTXE5TGJLNmdKNzF6YkZnSXV4DQo1L1pXK2JraTB2RlplWk9ZODUxb1psckFUNzJvMDI4NHNTWW9uN0pHZVZkY3NoUnh5R1VpSFpSTzdkaXZVTDVTDQpmM2JmSDFYbWY1ZDQzT0NWTWRuUUV2NWVaOG8zeWVLa3ZrbkZQUGVJMU9BbjdGbDlFRVNNR2dhOGFaSG1URSttDQpsLzlMSmdDYjBnQmtPT0M0WnV4bWh2aERKV1EzWnJCS3pMQlNUZXN0NWlLNVlwcXRWVVk2THRyRW9FelVTK1lsDQpwWndXY2VQWHlHeHM5ZURsR3lNVmQraW15Y3NTU1UvVno2Mmx6MnZCS21NTXBkYldDQWhud0RsRTVqU2dyMjRRDQp0eGNXLys2N3d5KzhuQlI3UXdqVTFITndVRjBzeERWdEwrZ1NHVERnSEVZSlhZelYvT05zMy94TkpoVFNPSkxNDQpoeXNVdyttaGdackdhbUdXcHVIVU1DUitvTWJzMTc1UkcrQjJnUFFHVytPTjJnUTRyOXN2b0ZBNHBBQm8xd1dLDQpRYjRhY3pmeVVscElBOVFoSmFsZEY3S3dPSHVlV3gwRUNrNXg0T2tvVDBvWVp0dzFiR0JjRGtaSmF3SURBUUFCDQpvMU13VVRBZEJnTlZIUTRFRmdRVW90UlNIUm9IWTEyRFZ4R0NCdEhpb1g2ZmVFQXdId1lEVlIwakJCZ3dGb0FVDQpvdFJTSFJvSFkxMkRWeEdDQnRIaW9YNmZlRUF3RHdZRFZSMFRBUUgvQkFVd0F3RUIvekFOQmdrcWhraUc5dzBCDQpBUXNGQUFPQ0FnRUFSbkpsWWRjMTFHd0VxWnh6RDF2R3BDR2pDN2VWTlQ3aVY1d3IybXlybHdPYi9aUWFEa0xYDQpvVStaOVVXT1VlSXJTdzUydDdmQUpvVVAwSm5iYkMveVIrU1lqUGhvUXNiVHduOTc2ZldBWTduM3FMOXhCd1Y0DQphek41OXNjeUp0dlhMeUtOL2N5ak1ReDRLajBIMFg0bWJ6bzVZNUtzWWtYVU0vOEFPdWZMcEd0S1NGVGgrSEFDDQpab1Q5YnZHS25adnNHd0tYZFF0Wnh0akhaUjVqK3U3ZGtQOTJBT051RFNabS8rWVV4b2tBK09JbzdSR3BwSHNXDQo1ZTdNY0FTVXRtb1FORXd6dVFoVkJaRWQ1OGtKYjUrV0VWbGNzanlXNnRTbzErZ25tTWNqR1BsMWgxR2hVbjV4DQpFY0lWRnBIWXM5YWo1NmpBSjk1MVQvZjhMaWxmTlVnanBLQ0c1bnl0SUt3emxhOHNtdGlPdm1UNEpYbXBwSkI2DQo4bmdHRVluVjUrUTYwWFJ2OEhSSGp1VG9CRHVhaERrVDA2R1JGODU1d09FR2V4bkZpMXZYWUxLVllWb1V2MXRKDQo4dVdUR1pwNllDSVJldlBqbzg5ZytWTlJSaVFYUThJd0dybXE5c0RoVTlqTjA0SjdVL1RvRDFpNHE3VnlsRUc5DQorV1VGNkNLaEdBeTJIaEhwVncyTGFoOS9lUzdZMUZ1YURrWmhPZG1laG1BOCtqdHNZamJadnR5Mm1SWlF0UUZzDQpUU1VUUjREbUR2bVVPRVRmeStpRHdzK2RkWXVNTnJGeVVYV2dkMnpBQU4ydVl1UHFGY2pRcFNPODFzVTJTU3R3DQoxVzAyeUtYOGJEYmZFdjBzbUh3UzliQnFlSGo5NEM1Mjg0YXpsdTBmaUdpTm1OUEM4ckJLRmhBPQ0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ==
         service:
           name: webhook-service
-          namespace: metallb-system
+          namespace: "{{ metallb_namespace }}"
           path: /convert
       conversionReviewVersions:
       - v1beta1
@@ -1291,7 +1291,7 @@ metadata:
     pod-security.kubernetes.io/enforce: privileged
     pod-security.kubernetes.io/warn: privileged
   name: controller
-  namespace: metallb-system
+  namespace: "{{ metallb_namespace }}"
 
 {% if metallb_speaker_enabled %}
 ---
@@ -1301,7 +1301,7 @@ metadata:
   labels:
     app: metallb
   name: speaker
-  namespace: metallb-system
+  namespace: "{{ metallb_namespace }}"
 {% endif %}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
@@ -1310,7 +1310,7 @@ metadata:
   labels:
     app: metallb
   name: controller
-  namespace: metallb-system
+  namespace: "{{ metallb_namespace }}"
 rules:
 - apiGroups:
   - ""
@@ -1402,7 +1402,7 @@ metadata:
   labels:
     app: metallb
   name: pod-lister
-  namespace: metallb-system
+  namespace: "{{ metallb_namespace }}"
 rules:
 - apiGroups:
   - ""
@@ -1480,7 +1480,7 @@ kind: ClusterRole
 metadata:
   labels:
     app: metallb
-  name: metallb-system:controller
+  name: {{ metallb_namespace }}:controller
 rules:
 - apiGroups:
   - ""
@@ -1561,7 +1561,7 @@ kind: ClusterRole
 metadata:
   labels:
     app: metallb
-  name: metallb-system:speaker
+  name: {{ metallb_namespace }}:speaker
 rules:
 - apiGroups:
   - ""
@@ -1598,7 +1598,7 @@ metadata:
   labels:
     app: metallb
   name: controller
-  namespace: metallb-system
+  namespace: "{{ metallb_namespace }}"
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: Role
@@ -1606,7 +1606,7 @@ roleRef:
 subjects:
 - kind: ServiceAccount
   name: controller
-  namespace: metallb-system
+  namespace: "{{ metallb_namespace }}"
 
 ---
 apiVersion: rbac.authorization.k8s.io/v1
@@ -1615,7 +1615,7 @@ metadata:
   labels:
     app: metallb
   name: pod-lister
-  namespace: metallb-system
+  namespace: "{{ metallb_namespace }}"
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: Role
@@ -1623,7 +1623,7 @@ roleRef:
 subjects:
 - kind: ServiceAccount
   name: speaker
-  namespace: metallb-system
+  namespace: "{{ metallb_namespace }}"
 
 ---
 apiVersion: rbac.authorization.k8s.io/v1
@@ -1631,15 +1631,15 @@ kind: ClusterRoleBinding
 metadata:
   labels:
     app: metallb
-  name: metallb-system:controller
+  name: {{ metallb_namespace }}:controller
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: metallb-system:controller
+  name: {{ metallb_namespace }}:controller
 subjects:
 - kind: ServiceAccount
   name: controller
-  namespace: metallb-system
+  namespace: "{{ metallb_namespace }}"
 
 {% if metallb_speaker_enabled %}
 ---
@@ -1648,15 +1648,15 @@ kind: ClusterRoleBinding
 metadata:
   labels:
     app: metallb
-  name: metallb-system:speaker
+  name: {{ metallb_namespace }}:speaker
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: metallb-system:speaker
+  name: {{ metallb_namespace }}:speaker
 subjects:
 - kind: ServiceAccount
   name: speaker
-  namespace: metallb-system
+  namespace: "{{ metallb_namespace }}"
 {% endif %}
 
 ---
@@ -1664,14 +1664,14 @@ apiVersion: v1
 kind: Secret
 metadata:
   name: webhook-server-cert
-  namespace: metallb-system
+  namespace: "{{ metallb_namespace }}"
 
 ---
 apiVersion: v1
 kind: Service
 metadata:
   name: webhook-service
-  namespace: metallb-system
+  namespace: "{{ metallb_namespace }}"
 spec:
   ports:
   - port: 443
@@ -1687,7 +1687,7 @@ metadata:
     app: metallb
     component: controller
   name: controller
-  namespace: metallb-system
+  namespace: "{{ metallb_namespace }}"
 spec:
   revisionHistoryLimit: 3
   selector:
@@ -1782,7 +1782,7 @@ metadata:
     app: metallb
     component: speaker
   name: speaker
-  namespace: metallb-system
+  namespace: "{{ metallb_namespace }}"
 spec:
   selector:
     matchLabels:
@@ -1888,7 +1888,7 @@ webhooks:
   clientConfig:
     service:
       name: webhook-service
-      namespace: metallb-system
+      namespace: "{{ metallb_namespace }}"
       path: /validate-metallb-io-v1beta2-bgppeer
   failurePolicy: Fail
   name: bgppeersvalidationwebhook.metallb.io
@@ -1908,7 +1908,7 @@ webhooks:
   clientConfig:
     service:
       name: webhook-service
-      namespace: metallb-system
+      namespace: "{{ metallb_namespace }}"
       path: /validate-metallb-io-v1beta1-addresspool
   failurePolicy: Fail
   name: addresspoolvalidationwebhook.metallb.io
@@ -1928,7 +1928,7 @@ webhooks:
   clientConfig:
     service:
       name: webhook-service
-      namespace: metallb-system
+      namespace: "{{ metallb_namespace }}"
       path: /validate-metallb-io-v1beta1-bfdprofile
   failurePolicy: Fail
   name: bfdprofilevalidationwebhook.metallb.io
@@ -1948,7 +1948,7 @@ webhooks:
   clientConfig:
     service:
       name: webhook-service
-      namespace: metallb-system
+      namespace: "{{ metallb_namespace }}"
       path: /validate-metallb-io-v1beta1-bgpadvertisement
   failurePolicy: Fail
   name: bgpadvertisementvalidationwebhook.metallb.io
@@ -1968,7 +1968,7 @@ webhooks:
   clientConfig:
     service:
       name: webhook-service
-      namespace: metallb-system
+      namespace: "{{ metallb_namespace }}"
       path: /validate-metallb-io-v1beta1-community
   failurePolicy: Fail
   name: communityvalidationwebhook.metallb.io
@@ -1988,7 +1988,7 @@ webhooks:
   clientConfig:
     service:
       name: webhook-service
-      namespace: metallb-system
+      namespace: "{{ metallb_namespace }}"
       path: /validate-metallb-io-v1beta1-ipaddresspool
   failurePolicy: Fail
   name: ipaddresspoolvalidationwebhook.metallb.io
@@ -2008,7 +2008,7 @@ webhooks:
   clientConfig:
     service:
       name: webhook-service
-      namespace: metallb-system
+      namespace: "{{ metallb_namespace }}"
       path: /validate-metallb-io-v1beta1-l2advertisement
   failurePolicy: Fail
   name: l2advertisementvalidationwebhook.metallb.io

kubespray/project/roles/kubernetes-apps/metallb/templates/pools.yaml.j2

@@ -9,7 +9,7 @@
 apiVersion: metallb.io/v1beta1
 kind: IPAddressPool
 metadata:
-  namespace: metallb-system
+  namespace: "{{ metallb_namespace }}"
   name: "{{ pool_name }}"
 spec:
   addresses:

kubespray/project/roles/kubernetes-apps/metrics_server/tasks/main.yml

@@ -19,7 +19,7 @@
     state: directory
     owner: root
     group: root
-    mode: 0755
+    mode: "0755"
   when:
     - inventory_hostname == groups['kube_control_plane'][0]
 
@@ -39,7 +39,7 @@
   template:
     src: "{{ item.file }}.j2"
     dest: "{{ kube_config_dir }}/addons/metrics_server/{{ item.file }}"
-    mode: 0644
+    mode: "0644"
   with_items: "{{ metrics_server_templates }}"
   register: metrics_server_manifests
   when:

kubespray/project/roles/kubernetes-apps/network_plugin/multus/tasks/main.yml

@@ -9,7 +9,7 @@
     state: "latest"
   delegate_to: "{{ groups['kube_control_plane'][0] }}"
   run_once: true
-  with_items: "{{ multus_manifest_1.results + (multus_nodes_list | map('extract', hostvars, 'multus_manifest_2') | list | json_query('[].results')) }}"
+  with_items: "{{ (multus_manifest_1.results | default([])) + (multus_nodes_list | map('extract', hostvars, 'multus_manifest_2.results') | default([]) | list) }}"
   loop_control:
     label: "{{ item.item.name if item != None else 'skipped' }}"
   vars:

kubespray/project/roles/kubernetes-apps/network_plugin/weave/tasks/main.yml

@@ -13,7 +13,7 @@
 - name: Weave | Wait for Weave to become available
   uri:
     url: http://127.0.0.1:6784/status
-    return_content: yes
+    return_content: true
   register: weave_status
   retries: 180
   delay: 5

kubespray/project/roles/kubernetes-apps/node_feature_discovery/defaults/main.yml

@@ -0,0 +1,16 @@
+---
+node_feature_discovery_enabled: false
+node_feature_discovery_namespace: node-feature-discovery
+node_feature_discovery_enable_nodefeature_api: true
+node_feature_discovery_gc_replicas: 1
+node_feature_discovery_gc_interval: 1h
+node_feature_discovery_gc_sa_name: node-feature-discovery-gc
+node_feature_discovery_gc_sa_create: true
+node_feature_discovery_master_replicas: 1
+node_feature_discovery_master_crd_controller: null
+node_feature_discovery_master_instance: null
+node_feature_discovery_master_config: null
+node_feature_discovery_worker_sa_name: node-feature-discovery-worker
+node_feature_discovery_worker_sa_create: true
+node_feature_discovery_worker_config: null
+node_feature_discovery_worker_tolerations: null

kubespray/project/roles/kubernetes-apps/node_feature_discovery/tasks/main.yml

@@ -0,0 +1,49 @@
+---
+- name: Node Feature Discovery | Create addon dir
+  file:
+    path: "{{ kube_config_dir }}/addons/node_feature_discovery"
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+
+- name: Node Feature Discovery | Templates list
+  set_fact:
+    node_feature_discovery_templates:
+      - { name: nfd-ns, file: nfd-ns.yaml, type: ns }
+      - { name: nfd-api-crd, file: nfd-api-crds.yaml, type: crd }
+      - { name: nfd-serviceaccount, file: nfd-serviceaccount.yaml, type: sa }
+      - { name: nfd-role, file: nfd-role.yaml, type: role }
+      - { name: nfd-clusterrole, file: nfd-clusterrole.yaml, type: clusterrole }
+      - { name: nfd-rolebinding, file: nfd-rolebinding.yaml, type: rolebinding }
+      - { name: nfd-clusterrolebinding, file: nfd-clusterrolebinding.yaml, type: clusterrolebinding }
+      - { name: nfd-master-conf, file: nfd-master-conf.yaml, type: cm }
+      - { name: nfd-worker-conf, file: nfd-worker-conf.yaml, type: cm }
+      - { name: nfd-topologyupdater-conf, file: nfd-topologyupdater-conf.yaml, type: cm }
+      - { name: nfd-gc, file: nfd-gc.yaml, type: deploy }
+      - { name: nfd-master, file: nfd-master.yaml, type: deploy }
+      - { name: nfd-worker, file: nfd-worker.yaml, type: ds }
+      - { name: nfd-service, file: nfd-service.yaml, type: srv }
+
+- name: Node Feature Discovery | Create manifests
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/addons/node_feature_discovery/{{ item.file }}"
+    mode: "0644"
+  with_items: "{{ node_feature_discovery_templates }}"
+  register: node_feature_discovery_manifests
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+
+- name: Node Feature Discovery | Apply manifests
+  kube:
+    name: "{{ item.item.name }}"
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: "{{ item.item.type }}"
+    filename: "{{ kube_config_dir }}/addons/node_feature_discovery/{{ item.item.file }}"
+    state: "latest"
+  with_items: "{{ node_feature_discovery_manifests.results }}"
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]

kubespray/project/roles/kubernetes-apps/node_feature_discovery/templates/nfd-api-crds.yaml.j2

@@ -0,0 +1,710 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.14.0
+  name: nodefeatures.nfd.k8s-sigs.io
+spec:
+  group: nfd.k8s-sigs.io
+  names:
+    kind: NodeFeature
+    listKind: NodeFeatureList
+    plural: nodefeatures
+    singular: nodefeature
+  scope: Namespaced
+  versions:
+  - name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: |-
+          NodeFeature resource holds the features discovered for one node in the
+          cluster.
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: Specification of the NodeFeature, containing features discovered
+              for a node.
+            properties:
+              features:
+                description: Features is the full "raw" features data that has been
+                  discovered.
+                properties:
+                  attributes:
+                    additionalProperties:
+                      description: AttributeFeatureSet is a set of features having
+                        string value.
+                      properties:
+                        elements:
+                          additionalProperties:
+                            type: string
+                          description: Individual features of the feature set.
+                          type: object
+                      required:
+                      - elements
+                      type: object
+                    description: Attributes contains all the attribute-type features
+                      of the node.
+                    type: object
+                  flags:
+                    additionalProperties:
+                      description: FlagFeatureSet is a set of simple features only
+                        containing names without values.
+                      properties:
+                        elements:
+                          additionalProperties:
+                            description: Nil is a dummy empty struct for protobuf
+                              compatibility
+                            type: object
+                          description: Individual features of the feature set.
+                          type: object
+                      required:
+                      - elements
+                      type: object
+                    description: Flags contains all the flag-type features of the
+                      node.
+                    type: object
+                  instances:
+                    additionalProperties:
+                      description: InstanceFeatureSet is a set of features each of
+                        which is an instance having multiple attributes.
+                      properties:
+                        elements:
+                          description: Individual features of the feature set.
+                          items:
+                            description: InstanceFeature represents one instance of
+                              a complex features, e.g. a device.
+                            properties:
+                              attributes:
+                                additionalProperties:
+                                  type: string
+                                description: Attributes of the instance feature.
+                                type: object
+                            required:
+                            - attributes
+                            type: object
+                          type: array
+                      required:
+                      - elements
+                      type: object
+                    description: Instances contains all the instance-type features
+                      of the node.
+                    type: object
+                type: object
+              labels:
+                additionalProperties:
+                  type: string
+                description: Labels is the set of node labels that are requested to
+                  be created.
+                type: object
+            type: object
+        required:
+        - spec
+        type: object
+    served: true
+    storage: true
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.14.0
+  name: nodefeaturegroups.nfd.k8s-sigs.io
+spec:
+  group: nfd.k8s-sigs.io
+  names:
+    kind: NodeFeatureGroup
+    listKind: NodeFeatureGroupList
+    plural: nodefeaturegroups
+    shortNames:
+    - nfg
+    singular: nodefeaturegroup
+  scope: Namespaced
+  versions:
+  - name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: NodeFeatureGroup resource holds Node pools by featureGroup
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: Spec defines the rules to be evaluated.
+            properties:
+              featureGroupRules:
+                description: List of rules to evaluate to determine nodes that belong
+                  in this group.
+                items:
+                  description: GroupRule defines a rule for nodegroup filtering.
+                  properties:
+                    matchAny:
+                      description: MatchAny specifies a list of matchers one of which
+                        must match.
+                      items:
+                        description: MatchAnyElem specifies one sub-matcher of MatchAny.
+                        properties:
+                          matchFeatures:
+                            description: MatchFeatures specifies a set of matcher
+                              terms all of which must match.
+                            items:
+                              description: |-
+                                FeatureMatcherTerm defines requirements against one feature set. All
+                                requirements (specified as MatchExpressions) are evaluated against each
+                                element in the feature set.
+                              properties:
+                                feature:
+                                  description: Feature is the name of the feature
+                                    set to match against.
+                                  type: string
+                                matchExpressions:
+                                  additionalProperties:
+                                    description: |-
+                                      MatchExpression specifies an expression to evaluate against a set of input
+                                      values. It contains an operator that is applied when matching the input and
+                                      an array of values that the operator evaluates the input against.
+                                    properties:
+                                      op:
+                                        description: Op is the operator to be applied.
+                                        enum:
+                                        - In
+                                        - NotIn
+                                        - InRegexp
+                                        - Exists
+                                        - DoesNotExist
+                                        - Gt
+                                        - Lt
+                                        - GtLt
+                                        - IsTrue
+                                        - IsFalse
+                                        type: string
+                                      value:
+                                        description: |-
+                                          Value is the list of values that the operand evaluates the input
+                                          against. Value should be empty if the operator is Exists, DoesNotExist,
+                                          IsTrue or IsFalse. Value should contain exactly one element if the
+                                          operator is Gt or Lt and exactly two elements if the operator is GtLt.
+                                          In other cases Value should contain at least one element.
+                                        items:
+                                          type: string
+                                        type: array
+                                    required:
+                                    - op
+                                    type: object
+                                  description: |-
+                                    MatchExpressions is the set of per-element expressions evaluated. These
+                                    match against the value of the specified elements.
+                                  type: object
+                                matchName:
+                                  description: |-
+                                    MatchName in an expression that is matched against the name of each
+                                    element in the feature set.
+                                  properties:
+                                    op:
+                                      description: Op is the operator to be applied.
+                                      enum:
+                                      - In
+                                      - NotIn
+                                      - InRegexp
+                                      - Exists
+                                      - DoesNotExist
+                                      - Gt
+                                      - Lt
+                                      - GtLt
+                                      - IsTrue
+                                      - IsFalse
+                                      type: string
+                                    value:
+                                      description: |-
+                                        Value is the list of values that the operand evaluates the input
+                                        against. Value should be empty if the operator is Exists, DoesNotExist,
+                                        IsTrue or IsFalse. Value should contain exactly one element if the
+                                        operator is Gt or Lt and exactly two elements if the operator is GtLt.
+                                        In other cases Value should contain at least one element.
+                                      items:
+                                        type: string
+                                      type: array
+                                  required:
+                                  - op
+                                  type: object
+                              required:
+                              - feature
+                              type: object
+                            type: array
+                        required:
+                        - matchFeatures
+                        type: object
+                      type: array
+                    matchFeatures:
+                      description: MatchFeatures specifies a set of matcher terms
+                        all of which must match.
+                      items:
+                        description: |-
+                          FeatureMatcherTerm defines requirements against one feature set. All
+                          requirements (specified as MatchExpressions) are evaluated against each
+                          element in the feature set.
+                        properties:
+                          feature:
+                            description: Feature is the name of the feature set to
+                              match against.
+                            type: string
+                          matchExpressions:
+                            additionalProperties:
+                              description: |-
+                                MatchExpression specifies an expression to evaluate against a set of input
+                                values. It contains an operator that is applied when matching the input and
+                                an array of values that the operator evaluates the input against.
+                              properties:
+                                op:
+                                  description: Op is the operator to be applied.
+                                  enum:
+                                  - In
+                                  - NotIn
+                                  - InRegexp
+                                  - Exists
+                                  - DoesNotExist
+                                  - Gt
+                                  - Lt
+                                  - GtLt
+                                  - IsTrue
+                                  - IsFalse
+                                  type: string
+                                value:
+                                  description: |-
+                                    Value is the list of values that the operand evaluates the input
+                                    against. Value should be empty if the operator is Exists, DoesNotExist,
+                                    IsTrue or IsFalse. Value should contain exactly one element if the
+                                    operator is Gt or Lt and exactly two elements if the operator is GtLt.
+                                    In other cases Value should contain at least one element.
+                                  items:
+                                    type: string
+                                  type: array
+                              required:
+                              - op
+                              type: object
+                            description: |-
+                              MatchExpressions is the set of per-element expressions evaluated. These
+                              match against the value of the specified elements.
+                            type: object
+                          matchName:
+                            description: |-
+                              MatchName in an expression that is matched against the name of each
+                              element in the feature set.
+                            properties:
+                              op:
+                                description: Op is the operator to be applied.
+                                enum:
+                                - In
+                                - NotIn
+                                - InRegexp
+                                - Exists
+                                - DoesNotExist
+                                - Gt
+                                - Lt
+                                - GtLt
+                                - IsTrue
+                                - IsFalse
+                                type: string
+                              value:
+                                description: |-
+                                  Value is the list of values that the operand evaluates the input
+                                  against. Value should be empty if the operator is Exists, DoesNotExist,
+                                  IsTrue or IsFalse. Value should contain exactly one element if the
+                                  operator is Gt or Lt and exactly two elements if the operator is GtLt.
+                                  In other cases Value should contain at least one element.
+                                items:
+                                  type: string
+                                type: array
+                            required:
+                            - op
+                            type: object
+                        required:
+                        - feature
+                        type: object
+                      type: array
+                    name:
+                      description: Name of the rule.
+                      type: string
+                  required:
+                  - name
+                  type: object
+                type: array
+            required:
+            - featureGroupRules
+            type: object
+          status:
+            description: |-
+              Status of the NodeFeatureGroup after the most recent evaluation of the
+              specification.
+            properties:
+              nodes:
+                description: Nodes is a list of FeatureGroupNode in the cluster that
+                  match the featureGroupRules
+                items:
+                  properties:
+                    name:
+                      description: Name of the node.
+                      type: string
+                  required:
+                  - name
+                  type: object
+                type: array
+                x-kubernetes-list-map-keys:
+                - name
+                x-kubernetes-list-type: map
+            type: object
+        required:
+        - spec
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.14.0
+  name: nodefeaturerules.nfd.k8s-sigs.io
+spec:
+  group: nfd.k8s-sigs.io
+  names:
+    kind: NodeFeatureRule
+    listKind: NodeFeatureRuleList
+    plural: nodefeaturerules
+    shortNames:
+    - nfr
+    singular: nodefeaturerule
+  scope: Cluster
+  versions:
+  - name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: |-
+          NodeFeatureRule resource specifies a configuration for feature-based
+          customization of node objects, such as node labeling.
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: Spec defines the rules to be evaluated.
+            properties:
+              rules:
+                description: Rules is a list of node customization rules.
+                items:
+                  description: Rule defines a rule for node customization such as
+                    labeling.
+                  properties:
+                    annotations:
+                      additionalProperties:
+                        type: string
+                      description: Annotations to create if the rule matches.
+                      type: object
+                    extendedResources:
+                      additionalProperties:
+                        type: string
+                      description: ExtendedResources to create if the rule matches.
+                      type: object
+                    labels:
+                      additionalProperties:
+                        type: string
+                      description: Labels to create if the rule matches.
+                      type: object
+                    labelsTemplate:
+                      description: |-
+                        LabelsTemplate specifies a template to expand for dynamically generating
+                        multiple labels. Data (after template expansion) must be keys with an
+                        optional value (<key>[=<value>]) separated by newlines.
+                      type: string
+                    matchAny:
+                      description: MatchAny specifies a list of matchers one of which
+                        must match.
+                      items:
+                        description: MatchAnyElem specifies one sub-matcher of MatchAny.
+                        properties:
+                          matchFeatures:
+                            description: MatchFeatures specifies a set of matcher
+                              terms all of which must match.
+                            items:
+                              description: |-
+                                FeatureMatcherTerm defines requirements against one feature set. All
+                                requirements (specified as MatchExpressions) are evaluated against each
+                                element in the feature set.
+                              properties:
+                                feature:
+                                  description: Feature is the name of the feature
+                                    set to match against.
+                                  type: string
+                                matchExpressions:
+                                  additionalProperties:
+                                    description: |-
+                                      MatchExpression specifies an expression to evaluate against a set of input
+                                      values. It contains an operator that is applied when matching the input and
+                                      an array of values that the operator evaluates the input against.
+                                    properties:
+                                      op:
+                                        description: Op is the operator to be applied.
+                                        enum:
+                                        - In
+                                        - NotIn
+                                        - InRegexp
+                                        - Exists
+                                        - DoesNotExist
+                                        - Gt
+                                        - Lt
+                                        - GtLt
+                                        - IsTrue
+                                        - IsFalse
+                                        type: string
+                                      value:
+                                        description: |-
+                                          Value is the list of values that the operand evaluates the input
+                                          against. Value should be empty if the operator is Exists, DoesNotExist,
+                                          IsTrue or IsFalse. Value should contain exactly one element if the
+                                          operator is Gt or Lt and exactly two elements if the operator is GtLt.
+                                          In other cases Value should contain at least one element.
+                                        items:
+                                          type: string
+                                        type: array
+                                    required:
+                                    - op
+                                    type: object
+                                  description: |-
+                                    MatchExpressions is the set of per-element expressions evaluated. These
+                                    match against the value of the specified elements.
+                                  type: object
+                                matchName:
+                                  description: |-
+                                    MatchName in an expression that is matched against the name of each
+                                    element in the feature set.
+                                  properties:
+                                    op:
+                                      description: Op is the operator to be applied.
+                                      enum:
+                                      - In
+                                      - NotIn
+                                      - InRegexp
+                                      - Exists
+                                      - DoesNotExist
+                                      - Gt
+                                      - Lt
+                                      - GtLt
+                                      - IsTrue
+                                      - IsFalse
+                                      type: string
+                                    value:
+                                      description: |-
+                                        Value is the list of values that the operand evaluates the input
+                                        against. Value should be empty if the operator is Exists, DoesNotExist,
+                                        IsTrue or IsFalse. Value should contain exactly one element if the
+                                        operator is Gt or Lt and exactly two elements if the operator is GtLt.
+                                        In other cases Value should contain at least one element.
+                                      items:
+                                        type: string
+                                      type: array
+                                  required:
+                                  - op
+                                  type: object
+                              required:
+                              - feature
+                              type: object
+                            type: array
+                        required:
+                        - matchFeatures
+                        type: object
+                      type: array
+                    matchFeatures:
+                      description: MatchFeatures specifies a set of matcher terms
+                        all of which must match.
+                      items:
+                        description: |-
+                          FeatureMatcherTerm defines requirements against one feature set. All
+                          requirements (specified as MatchExpressions) are evaluated against each
+                          element in the feature set.
+                        properties:
+                          feature:
+                            description: Feature is the name of the feature set to
+                              match against.
+                            type: string
+                          matchExpressions:
+                            additionalProperties:
+                              description: |-
+                                MatchExpression specifies an expression to evaluate against a set of input
+                                values. It contains an operator that is applied when matching the input and
+                                an array of values that the operator evaluates the input against.
+                              properties:
+                                op:
+                                  description: Op is the operator to be applied.
+                                  enum:
+                                  - In
+                                  - NotIn
+                                  - InRegexp
+                                  - Exists
+                                  - DoesNotExist
+                                  - Gt
+                                  - Lt
+                                  - GtLt
+                                  - IsTrue
+                                  - IsFalse
+                                  type: string
+                                value:
+                                  description: |-
+                                    Value is the list of values that the operand evaluates the input
+                                    against. Value should be empty if the operator is Exists, DoesNotExist,
+                                    IsTrue or IsFalse. Value should contain exactly one element if the
+                                    operator is Gt or Lt and exactly two elements if the operator is GtLt.
+                                    In other cases Value should contain at least one element.
+                                  items:
+                                    type: string
+                                  type: array
+                              required:
+                              - op
+                              type: object
+                            description: |-
+                              MatchExpressions is the set of per-element expressions evaluated. These
+                              match against the value of the specified elements.
+                            type: object
+                          matchName:
+                            description: |-
+                              MatchName in an expression that is matched against the name of each
+                              element in the feature set.
+                            properties:
+                              op:
+                                description: Op is the operator to be applied.
+                                enum:
+                                - In
+                                - NotIn
+                                - InRegexp
+                                - Exists
+                                - DoesNotExist
+                                - Gt
+                                - Lt
+                                - GtLt
+                                - IsTrue
+                                - IsFalse
+                                type: string
+                              value:
+                                description: |-
+                                  Value is the list of values that the operand evaluates the input
+                                  against. Value should be empty if the operator is Exists, DoesNotExist,
+                                  IsTrue or IsFalse. Value should contain exactly one element if the
+                                  operator is Gt or Lt and exactly two elements if the operator is GtLt.
+                                  In other cases Value should contain at least one element.
+                                items:
+                                  type: string
+                                type: array
+                            required:
+                            - op
+                            type: object
+                        required:
+                        - feature
+                        type: object
+                      type: array
+                    name:
+                      description: Name of the rule.
+                      type: string
+                    taints:
+                      description: Taints to create if the rule matches.
+                      items:
+                        description: |-
+                          The node this Taint is attached to has the "effect" on
+                          any pod that does not tolerate the Taint.
+                        properties:
+                          effect:
+                            description: |-
+                              Required. The effect of the taint on pods
+                              that do not tolerate the taint.
+                              Valid effects are NoSchedule, PreferNoSchedule and NoExecute.
+                            type: string
+                          key:
+                            description: Required. The taint key to be applied to
+                              a node.
+                            type: string
+                          timeAdded:
+                            description: |-
+                              TimeAdded represents the time at which the taint was added.
+                              It is only written for NoExecute taints.
+                            format: date-time
+                            type: string
+                          value:
+                            description: The taint value corresponding to the taint
+                              key.
+                            type: string
+                        required:
+                        - effect
+                        - key
+                        type: object
+                      type: array
+                    vars:
+                      additionalProperties:
+                        type: string
+                      description: |-
+                        Vars is the variables to store if the rule matches. Variables do not
+                        directly inflict any changes in the node object. However, they can be
+                        referenced from other rules enabling more complex rule hierarchies,
+                        without exposing intermediary output values as labels.
+                      type: object
+                    varsTemplate:
+                      description: |-
+                        VarsTemplate specifies a template to expand for dynamically generating
+                        multiple variables. Data (after template expansion) must be keys with an
+                        optional value (<key>[=<value>]) separated by newlines.
+                      type: string
+                  required:
+                  - name
+                  type: object
+                type: array
+            required:
+            - rules
+            type: object
+        required:
+        - spec
+        type: object
+    served: true
+    storage: true

kubespray/project/roles/kubernetes-apps/node_feature_discovery/templates/nfd-clusterrole.yaml.j2

@@ -0,0 +1,80 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: node-feature-discovery
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  - nodes/status
+  verbs:
+  - get
+  - patch
+  - update
+  - list
+- apiGroups:
+  - nfd.k8s-sigs.io
+  resources:
+  - nodefeatures
+  - nodefeaturerules
+  - nodefeaturegroups
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - nfd.k8s-sigs.io
+  resources:
+  - nodefeaturegroup/status
+  verbs:
+  - patch
+  - update
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - create
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  resourceNames:
+  - "nfd-master.nfd.kubernetes.io"
+  verbs:
+  - get
+  - update
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: node-feature-discovery-gc
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - nodes/proxy
+  verbs:
+  - get
+- apiGroups:
+  - topology.node.k8s.io
+  resources:
+  - noderesourcetopologies
+  verbs:
+  - delete
+  - list
+- apiGroups:
+  - nfd.k8s-sigs.io
+  resources:
+  - nodefeatures
+  verbs:
+  - delete
+  - list

kubespray/project/roles/kubernetes-apps/node_feature_discovery/templates/nfd-clusterrolebinding.yaml.j2

@@ -0,0 +1,25 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: node-feature-discovery
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: node-feature-discovery
+subjects:
+- kind: ServiceAccount
+  name: node-feature-discovery
+  namespace: {{ node_feature_discovery_namespace }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: node-feature-discovery-gc
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: node-feature-discovery-gc
+subjects:
+- kind: ServiceAccount
+  name: {{ node_feature_discovery_gc_sa_name }}
+  namespace: {{ node_feature_discovery_namespace }}

kubespray/project/roles/kubernetes-apps/node_feature_discovery/templates/nfd-gc.yaml.j2

@@ -0,0 +1,42 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: node-feature-discovery-gc
+  namespace: {{ node_feature_discovery_namespace }}
+  labels:
+    app.kubernetes.io/name: node-feature-discovery
+    role: gc
+spec:
+  replicas: {{ node_feature_discovery_gc_replicas }}
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: node-feature-discovery
+      role: gc
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: node-feature-discovery
+        role: gc
+    spec:
+      serviceAccountName: {{ node_feature_discovery_gc_sa_name }}
+      dnsPolicy: ClusterFirstWithHostNet
+      containers:
+      - name: gc
+        image: {{ node_feature_discovery_image_repo }}:{{ node_feature_discovery_image_tag }}
+        imagePullPolicy: IfNotPresent
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        command:
+          - "nfd-gc"
+        args:
+          - "-gc-interval={{ node_feature_discovery_gc_interval }}"
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - ALL
+          readOnlyRootFilesystem: true
+          runAsNonRoot: true

kubespray/project/roles/kubernetes-apps/node_feature_discovery/templates/nfd-master-conf.yaml.j2

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: node-feature-discovery-master-conf
+  namespace: {{ node_feature_discovery_namespace }}
+data:
+{% if node_feature_discovery_master_config %}
+  nfd-master.conf: |-
+    {{ node_feature_discovery_master_config | to_yaml(indent=2, width=1337) | indent(width=4) }}
+{% else %}
+  nfd-master.conf: "null"
+{% endif %}

kubespray/project/roles/kubernetes-apps/node_feature_discovery/templates/nfd-master.yaml.j2

@@ -0,0 +1,111 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name:  node-feature-discovery-master
+  namespace: {{ node_feature_discovery_namespace }}
+  labels:
+    app.kubernetes.io/name: node-feature-discovery
+    role: master
+spec:
+  replicas: {{ node_feature_discovery_master_replicas }}
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: node-feature-discovery
+      role: master
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: node-feature-discovery
+        role: master
+    spec:
+      serviceAccountName: node-feature-discovery
+      enableServiceLinks: false
+      containers:
+      - name: master
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - ALL
+          readOnlyRootFilesystem: true
+          runAsNonRoot: true
+        image: {{ node_feature_discovery_image_repo }}:{{ node_feature_discovery_image_tag }}
+        imagePullPolicy: IfNotPresent
+        livenessProbe:
+          grpc:
+            port: 8082
+          initialDelaySeconds: 10
+          periodSeconds: 10
+        readinessProbe:
+          grpc:
+            port: 8082
+          initialDelaySeconds: 5
+          periodSeconds: 10
+          failureThreshold: 10
+        ports:
+        - containerPort: 8082
+          name: grpc
+        - containerPort: 8081
+          name: metrics
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        command:
+          - "nfd-master"
+        args:
+          - "-port=8080"
+{% if not node_feature_discovery_enable_nodefeature_api %}
+          - "-enable-nodefeature-api=false"
+{% elif node_feature_discovery_master_replicas > 1 %}
+          - "-enable-leader-election"
+{% endif %}
+{% if node_feature_discovery_master_crd_controller != none %}
+          - "-crd-controller={{ node_feature_discovery_master_crd_controller }}"
+{% else %}
+{% if node_feature_discovery_master_instance  %}
+          ## By default, disable crd controller for other than the default instances
+          - "-crd-controller=false"
+{% else %}
+          ## By default, disable crd controller for other than the default instances
+          - "-crd-controller=true"
+{% endif %}
+{% endif %}
+          - "-metrics=8081"
+        volumeMounts:
+          - name: nfd-master-conf
+            mountPath: "/etc/kubernetes/node-feature-discovery"
+            readOnly: true
+      volumes:
+        - name: nfd-master-conf
+          configMap:
+            name: node-feature-discovery-master-conf
+            items:
+              - key: nfd-master.conf
+                path: nfd-master.conf
+      affinity:
+        nodeAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - preference:
+              matchExpressions:
+              - key: node-role.kubernetes.io/master
+                operator: In
+                values:
+                - ""
+            weight: 1
+          - preference:
+              matchExpressions:
+              - key: node-role.kubernetes.io/control-plane
+                operator: In
+                values:
+                - ""
+            weight: 1
+      tolerations:
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/master
+        operator: Equal
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/control-plane
+        operator: Equal

kubespray/project/roles/kubernetes-apps/node_feature_discovery/templates/nfd-ns.yaml.j2

@@ -0,0 +1,7 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: {{ node_feature_discovery_namespace }}
+  labels:
+    name: {{ node_feature_discovery_namespace }}

kubespray/project/roles/kubernetes-apps/node_feature_discovery/templates/nfd-role.yaml.j2

@@ -0,0 +1,20 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: node-feature-discovery-worker
+  namespace: {{ node_feature_discovery_namespace }}
+rules:
+- apiGroups:
+  - nfd.k8s-sigs.io
+  resources:
+  - nodefeatures
+  verbs:
+  - create
+  - get
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get

kubespray/project/roles/kubernetes-apps/node_feature_discovery/templates/nfd-rolebinding.yaml.j2

@@ -0,0 +1,13 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: node-feature-discovery-worker
+  namespace: {{ node_feature_discovery_namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: node-feature-discovery-worker
+subjects:
+- kind: ServiceAccount
+  name: {{ node_feature_discovery_worker_sa_name }}
+  namespace: {{ node_feature_discovery_namespace }}