add ability to log to file, update to v2.28.1 and fix 'first upgrade' issue

internal/resources/kubespray/project/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml

@@ -25,7 +25,7 @@
 
 - name: Kubeadm | aggregate all SANs
   set_fact:
-    apiserver_sans: "{{ (sans_base + groups['kube_control_plane'] + sans_lb + sans_lb_ip + sans_supp + sans_access_ip + sans_ip + sans_address + sans_override + sans_hostname + sans_fqdn + sans_kube_vip_address) | unique }}"
+    apiserver_sans: "{{ (sans_base + groups['kube_control_plane'] + sans_lb + sans_lb_ip + sans_supp + sans_access_ip + sans_ip + sans_ipv4_address + sans_ipv6_address + sans_override + sans_hostname + sans_fqdn + sans_kube_vip_address) | unique }}"
   vars:
     sans_base:
       - "kubernetes"
@@ -41,7 +41,8 @@
     sans_supp: "{{ supplementary_addresses_in_ssl_keys if supplementary_addresses_in_ssl_keys is defined else [] }}"
     sans_access_ip: "{{ groups['kube_control_plane'] | map('extract', hostvars, 'main_access_ip') | list | select('defined') | list }}"
     sans_ip: "{{ groups['kube_control_plane'] | map('extract', hostvars, 'main_ip') | list | select('defined') | list }}"
-    sans_address: "{{ groups['kube_control_plane'] | map('extract', hostvars, ['ansible_default_ipv6', 'ansible_default_ipv4', 'address']) | list | select('defined') | list }}"
+    sans_ipv4_address: "{{ groups['kube_control_plane'] | map('extract', hostvars, ['ansible_default_ipv4', 'address']) | list | select('defined') | list }}"
+    sans_ipv6_address: "{{ groups['kube_control_plane'] | map('extract', hostvars, ['ansible_default_ipv6', 'address']) | list | select('defined') | list }}"
     sans_override: "{{ [kube_override_hostname] if kube_override_hostname else [] }}"
     sans_hostname: "{{ groups['kube_control_plane'] | map('extract', hostvars, ['ansible_hostname']) | list | select('defined') | list }}"
     sans_fqdn: "{{ groups['kube_control_plane'] | map('extract', hostvars, ['ansible_fqdn']) | list | select('defined') | list }}"
@@ -170,6 +171,7 @@
     - apiserver_sans_ip_check.changed or apiserver_sans_host_check.changed
     - not kube_external_ca_mode
 
+  # TODO: Remove --skip-phases from command when v1beta4 UpgradeConfiguration supports skipPhases
 - name: Kubeadm | Initialize first control plane node
   when: inventory_hostname == first_kube_control_plane and not kubeadm_already_run.stat.exists
   vars:

internal/resources/kubespray/project/roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml

@@ -3,7 +3,8 @@
   import_tasks: check-api.yml
 
   # kubeadm-config.v1beta4 with UpgradeConfiguration requires some values that were previously allowed as args to be specified in the config file
-- name: Kubeadm | Upgrade first control plane node
+  # TODO: Remove --skip-phases from command when v1beta4 UpgradeConfiguration supports skipPhases
+- name: Kubeadm | Upgrade first control plane node to {{ kube_version }}
   command: >-
     timeout -k 600s 600s
     {{ bin_dir }}/kubeadm upgrade apply -y v{{ kube_version }}
@@ -16,14 +17,18 @@
     --force
     {%- else %}
     --config={{ kube_config_dir }}/kubeadm-config.yaml
-    {%- endif -%}
+    {%- endif %}
+    {%- if kube_version is version('1.32.0', '>=') %}
+    --skip-phases={{ kubeadm_init_phases_skip | join(',') }}
+    {%- endif %}
   register: kubeadm_upgrade
   when: inventory_hostname == first_kube_control_plane
   failed_when: kubeadm_upgrade.rc != 0 and "field is immutable" not in kubeadm_upgrade.stderr
   environment:
     PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}"
 
-- name: Kubeadm | Upgrade other control plane nodes
+# TODO: When we retire kubeadm-config.v1beta3, remove --certificate-renewal, --ignore-preflight-errors, --etcd-upgrade, --patches, and --skip-phases from command, since v1beta4+ supports these in UpgradeConfiguration.node
+- name: Kubeadm | Upgrade other control plane nodes to {{ kube_version }}
   command: >-
     {{ bin_dir }}/kubeadm upgrade node
     {%- if kubeadm_config_api_version == 'v1beta3' %}
@@ -33,7 +38,8 @@
     {% if kubeadm_patches | length > 0 %}--patches={{ kubeadm_patches_dir }}{% endif %}
     {%- else %}
     --config={{ kube_config_dir }}/kubeadm-config.yaml
-    {%- endif -%}
+    {%- endif %}
+    --skip-phases={{ kubeadm_upgrade_node_phases_skip | join(',') }}
   register: kubeadm_upgrade
   when: inventory_hostname != first_kube_control_plane
   failed_when: kubeadm_upgrade.rc != 0 and "field is immutable" not in kubeadm_upgrade.stderr

internal/resources/kubespray/project/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta4.yaml.j2

@@ -502,6 +502,12 @@ apply:
 {% endif %}
   imagePullPolicy: {{ k8s_image_pull_policy }}
   imagePullSerial: {{ kubeadm_image_pull_serial | lower }}
+{% for skip_phase in kubeadm_upgrade_node_phases_skip %}
+{% if loop.first %}
+  skipPhases:
+{% endif %}
+  - "{{ skip_phase }}"
+{% endfor %}
 node:
   certificateRenewal: {{ kubeadm_upgrade_auto_cert_renewal | lower }}
   etcdUpgrade: {{ (etcd_deployment_type == "kubeadm") | lower }}
@@ -517,6 +523,12 @@ node:
 {% endif %}
   imagePullPolicy: {{ k8s_image_pull_policy }}
   imagePullSerial: {{ kubeadm_image_pull_serial | lower }}
+{% for skip_phase in kubeadm_init_phases_skip %}
+{% if loop.first %}
+  skipPhases:
+{% endif %}
+  - "{{ skip_phase }}"
+{% endfor %}
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

internal/resources/kubespray/project/roles/kubernetes/control-plane/templates/podsecurity.yaml.j2

@@ -1,6 +1,6 @@
-{% if kube_pod_security_use_default %}
 apiVersion: pod-security.admission.config.k8s.io/v1
 kind: PodSecurityConfiguration
+{% if kube_pod_security_use_default %}
 defaults:
   enforce: "{{ kube_pod_security_default_enforce }}"
   enforce-version: "{{ kube_pod_security_default_enforce_version }}"

internal/resources/kubespray/project/roles/kubernetes/kubeadm_common/defaults/main.yml

@@ -20,4 +20,4 @@ kubeadm_patches: []
 # Patches are applied in the order they are specified.
 
 # List of errors to ignore during kubeadm preflight checks
-kubeadm_ignore_preflight_errors: []
+kubeadm_ignore_preflight_errors: ["CreateJob"]

internal/resources/kubespray/project/roles/kubernetes/node/defaults/main.yml

@@ -10,7 +10,7 @@ kubelet_bind_address: "{{ main_ip | default('::') }}"
 kube_resolv_conf: "/etc/resolv.conf"
 
 # Set to empty to avoid cgroup creation
-kubelet_enforce_node_allocatable: "\"\""
+kubelet_enforce_node_allocatable: '""'
 
 # Set runtime and kubelet cgroups when using systemd as cgroup driver (default)
 kube_service_cgroups: "{% if kube_reserved %}{{ kube_reserved_cgroups_for_service_slice }}{% else %}system.slice{% endif %}"
@@ -34,7 +34,6 @@ kube_node_addresses: >-
     {{ hostvars[host]['main_ips'] | join(' ') }}{{ ' ' if not loop.last else '' }}
   {%- endfor -%}
 kubelet_secure_addresses: "localhost link-local {{ kube_pods_subnets | regex_replace(',', ' ') }} {{ kube_node_addresses }}"
-
 # Reserve this space for kube resources
 # Whether to run kubelet and container-engine daemons in a dedicated cgroup. (Not required for resource reservations).
 kube_reserved: false
@@ -188,7 +187,6 @@ conntrack_modules:
   - nf_conntrack
   - nf_conntrack_ipv4
 
-
 ## Enable distributed tracing for kubelet
 kubelet_tracing: false
 kubelet_tracing_endpoint: "[::]:4317"