kube-forge v2: embede python with ansible (kubespray) and cobra as cli building tool

internal/resources/kubespray/project/roles/kubernetes/control-plane/tasks/check-api.yml

@@ -0,0 +1,10 @@
+---
+- name: Kubeadm | Check api is up
+  uri:
+    url: "https://{{ ip | default(fallback_ip) }}:{{ kube_apiserver_port }}/healthz"
+    validate_certs: false
+  when: ('kube_control_plane' in group_names)
+  register: _result
+  retries: 60
+  delay: 5
+  until: _result.status == 200

internal/resources/kubespray/project/roles/kubernetes/control-plane/tasks/define-first-kube-control.yml

@@ -0,0 +1,19 @@
+---
+
+- name: Check which kube-control nodes are already members of the cluster
+  command: "{{ bin_dir }}/kubectl get nodes --selector=node-role.kubernetes.io/control-plane -o json"
+  register: kube_control_planes_raw
+  ignore_errors: true
+  changed_when: false
+
+- name: Set fact joined_control_planes
+  set_fact:
+    joined_control_planes: "{{ ((kube_control_planes_raw.stdout | from_json)['items']) | default([]) | map(attribute='metadata') | map(attribute='name') | list }}"
+  delegate_to: "{{ item }}"
+  loop: "{{ groups['kube_control_plane'] }}"
+  when: kube_control_planes_raw is succeeded
+  run_once: true
+
+- name: Set fact first_kube_control_plane
+  set_fact:
+    first_kube_control_plane: "{{ joined_control_planes | default([]) | first | default(groups['kube_control_plane'] | first) }}"

internal/resources/kubespray/project/roles/kubernetes/control-plane/tasks/encrypt-at-rest.yml

@@ -0,0 +1,40 @@
+---
+- name: Check if secret for encrypting data at rest already exist
+  stat:
+    path: "{{ kube_cert_dir }}/secrets_encryption.yaml"
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: secrets_encryption_file
+
+- name: Slurp secrets_encryption file if it exists
+  slurp:
+    src: "{{ kube_cert_dir }}/secrets_encryption.yaml"
+  register: secret_file_encoded
+  when: secrets_encryption_file.stat.exists
+
+- name: Base 64 Decode slurped secrets_encryption.yaml file
+  set_fact:
+    secret_file_decoded: "{{ secret_file_encoded['content'] | b64decode | from_yaml }}"
+  when: secrets_encryption_file.stat.exists
+
+- name: Extract secret value from secrets_encryption.yaml
+  set_fact:
+    kube_encrypt_token_extracted: "{{ secret_file_decoded | json_query(secrets_encryption_query) | first | b64decode }}"
+  when: secrets_encryption_file.stat.exists
+
+- name: Set kube_encrypt_token across control plane nodes
+  set_fact:
+    kube_encrypt_token: "{{ kube_encrypt_token_extracted }}"
+  delegate_to: "{{ item }}"
+  delegate_facts: true
+  with_inventory_hostnames: kube_control_plane
+  when: kube_encrypt_token_extracted is defined
+
+- name: Write secrets for encrypting secret data at rest
+  template:
+    src: secrets_encryption.yaml.j2
+    dest: "{{ kube_cert_dir }}/secrets_encryption.yaml"
+    owner: root
+    group: "{{ kube_cert_group }}"
+    mode: "0640"

internal/resources/kubespray/project/roles/kubernetes/control-plane/tasks/kubeadm-backup.yml

@@ -0,0 +1,28 @@
+---
+- name: Backup old certs and keys
+  copy:
+    src: "{{ kube_cert_dir }}/{{ item }}"
+    dest: "{{ kube_cert_dir }}/{{ item }}.old"
+    mode: preserve
+    remote_src: true
+  with_items:
+    - apiserver.crt
+    - apiserver.key
+    - apiserver-kubelet-client.crt
+    - apiserver-kubelet-client.key
+    - front-proxy-client.crt
+    - front-proxy-client.key
+  ignore_errors: true  # noqa ignore-errors
+
+- name: Backup old confs
+  copy:
+    src: "{{ kube_config_dir }}/{{ item }}"
+    dest: "{{ kube_config_dir }}/{{ item }}.old"
+    mode: preserve
+    remote_src: true
+  with_items:
+    - admin.conf
+    - controller-manager.conf
+    - kubelet.conf
+    - scheduler.conf
+  ignore_errors: true  # noqa ignore-errors

internal/resources/kubespray/project/roles/kubernetes/control-plane/tasks/kubeadm-etcd.yml

@@ -0,0 +1,29 @@
+---
+- name: Calculate etcd cert serial
+  command: "openssl x509 -in {{ kube_cert_dir }}/apiserver-etcd-client.crt -noout -serial"
+  register: "etcd_client_cert_serial_result"
+  changed_when: false
+  tags:
+    - network
+
+- name: Set etcd_client_cert_serial
+  set_fact:
+    etcd_client_cert_serial: "{{ etcd_client_cert_serial_result.stdout.split('=')[1] }}"
+  tags:
+    - network
+
+- name: Ensure etcdctl and etcdutl script is installed
+  import_role:
+    name: etcdctl_etcdutl
+  when: etcd_deployment_type == "kubeadm"
+  tags:
+    - etcdctl
+    - etcdutl
+
+- name: Set ownership for etcd data directory
+  file:
+    path: "{{ etcd_data_dir }}"
+    owner: "{{ etcd_owner }}"
+    group: "{{ etcd_owner }}"
+    mode: "0700"
+  when: etcd_deployment_type == "kubeadm"

internal/resources/kubespray/project/roles/kubernetes/control-plane/tasks/kubeadm-fix-apiserver.yml

@@ -0,0 +1,17 @@
+---
+
+- name: Update server field in component kubeconfigs
+  lineinfile:
+    dest: "{{ kube_config_dir }}/{{ item }}"
+    regexp: '^    server: https'
+    line: '    server: {{ kube_apiserver_endpoint }}'
+    backup: true
+  with_items:
+    - admin.conf
+    - controller-manager.conf
+    - kubelet.conf
+    - scheduler.conf
+  notify:
+    - "Control plane | Restart kube-controller-manager"
+    - "Control plane | Restart kube-scheduler"
+    - "Control plane | reload kubelet"

internal/resources/kubespray/project/roles/kubernetes/control-plane/tasks/kubeadm-secondary.yml

@@ -0,0 +1,101 @@
+---
+- name: Set kubeadm_discovery_address
+  set_fact:
+    # noqa: jinja[spacing]
+    kubeadm_discovery_address: >-
+      {%- if "127.0.0.1" in kube_apiserver_endpoint or "localhost" in kube_apiserver_endpoint -%}
+      {{ first_kube_control_plane_address | ansible.utils.ipwrap }}:{{ kube_apiserver_port }}
+      {%- else -%}
+      {{ kube_apiserver_endpoint | regex_replace('https://', '') }}
+      {%- endif %}
+  tags:
+    - facts
+
+- name: Upload certificates so they are fresh and not expired
+  command: >-
+    {{ bin_dir }}/kubeadm init phase
+    --config {{ kube_config_dir }}/kubeadm-config.yaml
+    upload-certs
+    --upload-certs
+  register: kubeadm_upload_cert
+  when:
+    - inventory_hostname == first_kube_control_plane
+    - not kube_external_ca_mode
+
+- name: Parse certificate key if not set
+  set_fact:
+    kubeadm_certificate_key: "{{ hostvars[first_kube_control_plane]['kubeadm_upload_cert'].stdout_lines[-1] | trim }}"
+  run_once: true
+  when:
+    - hostvars[first_kube_control_plane]['kubeadm_upload_cert'] is defined
+    - hostvars[first_kube_control_plane]['kubeadm_upload_cert'] is not skipped
+
+- name: Create kubeadm ControlPlane config
+  template:
+    src: "kubeadm-controlplane.yaml.j2"
+    dest: "{{ kube_config_dir }}/kubeadm-controlplane.yaml"
+    mode: "0640"
+    backup: true
+    validate: "{{ kubeadm_config_validate_enabled | ternary(bin_dir + '/kubeadm config validate --config %s', omit) }}"
+  when:
+    - inventory_hostname != first_kube_control_plane
+    - not kubeadm_already_run.stat.exists
+
+- name: Wait for k8s apiserver
+  wait_for:
+    host: "{{ kubeadm_discovery_address | regex_replace('\\]?:\\d+$', '') | regex_replace('^\\[', '') }}"
+    port: "{{ kubeadm_discovery_address.split(':')[-1] }}"
+    timeout: 180
+
+
+- name: Check already run
+  debug:
+    msg: "{{ kubeadm_already_run.stat.exists }}"
+
+- name: Reset cert directory
+  shell: >-
+    if [ -f /etc/kubernetes/manifests/kube-apiserver.yaml ]; then
+    {{ bin_dir }}/kubeadm reset -f --cert-dir {{ kube_cert_dir }};
+    fi
+  environment:
+    PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}"
+  when:
+    - inventory_hostname != first_kube_control_plane
+    - kubeadm_already_run is not defined or not kubeadm_already_run.stat.exists
+    - not kube_external_ca_mode
+
+- name: Get kubeconfig for join discovery process
+  command: "{{ kubectl }} -n kube-public get cm cluster-info -o jsonpath='{.data.kubeconfig}'"
+  register: kubeconfig_file_discovery
+  run_once: true
+  delegate_to: "{{ groups['kube_control_plane'] | first }}"
+  when:
+    - kubeadm_use_file_discovery
+    - kubeadm_already_run is not defined or not kubeadm_already_run.stat.exists
+
+- name: Copy discovery kubeconfig
+  copy:
+    dest: "{{ kube_config_dir }}/cluster-info-discovery-kubeconfig.yaml"
+    content: "{{ kubeconfig_file_discovery.stdout }}"
+    owner: "root"
+    mode: "0644"
+  when:
+    - inventory_hostname != first_kube_control_plane
+    - kubeadm_use_file_discovery
+    - kubeadm_already_run is not defined or not kubeadm_already_run.stat.exists
+
+- name: Joining control plane node to the cluster.
+  command: >-
+    {{ bin_dir }}/kubeadm join
+    --config {{ kube_config_dir }}/kubeadm-controlplane.yaml
+    --ignore-preflight-errors={{ kubeadm_ignore_preflight_errors | join(',') }}
+    --skip-phases={{ kubeadm_join_phases_skip | join(',') }}
+  environment:
+    PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}"
+  register: kubeadm_join_control_plane
+  retries: 3
+  throttle: 1
+  until: kubeadm_join_control_plane is succeeded
+  when:
+    - inventory_hostname != first_kube_control_plane
+    - kubeadm_already_run is not defined or not kubeadm_already_run.stat.exists

internal/resources/kubespray/project/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml

@@ -0,0 +1,260 @@
+---
+- name: Install OIDC certificate
+  copy:
+    content: "{{ kube_oidc_ca_cert | b64decode }}"
+    dest: "{{ kube_oidc_ca_file }}"
+    owner: root
+    group: root
+    mode: "0644"
+  when:
+    - kube_oidc_auth
+    - kube_oidc_ca_cert is defined
+
+- name: Kubeadm | Check if kubeadm has already run
+  stat:
+    path: "/var/lib/kubelet/config.yaml"
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: kubeadm_already_run
+
+- name: Kubeadm | Backup kubeadm certs / kubeconfig
+  import_tasks: kubeadm-backup.yml
+  when:
+    - kubeadm_already_run.stat.exists
+
+- name: Kubeadm | aggregate all SANs
+  set_fact:
+    apiserver_sans: "{{ (sans_base + groups['kube_control_plane'] + sans_lb + sans_lb_ip + sans_supp + sans_access_ip + sans_ip + sans_address + sans_override + sans_hostname + sans_fqdn + sans_kube_vip_address) | unique }}"
+  vars:
+    sans_base:
+      - "kubernetes"
+      - "kubernetes.default"
+      - "kubernetes.default.svc"
+      - "kubernetes.default.svc.{{ dns_domain }}"
+      - "{{ kube_apiserver_ip }}"
+      - "localhost"
+      - "127.0.0.1"
+      - "::1"
+    sans_lb: "{{ [apiserver_loadbalancer_domain_name] if apiserver_loadbalancer_domain_name is defined else [] }}"
+    sans_lb_ip: "{{ [loadbalancer_apiserver.address] if loadbalancer_apiserver is defined and loadbalancer_apiserver.address is defined else [] }}"
+    sans_supp: "{{ supplementary_addresses_in_ssl_keys if supplementary_addresses_in_ssl_keys is defined else [] }}"
+    sans_access_ip: "{{ groups['kube_control_plane'] | map('extract', hostvars, 'main_access_ip') | list | select('defined') | list }}"
+    sans_ip: "{{ groups['kube_control_plane'] | map('extract', hostvars, 'main_ip') | list | select('defined') | list }}"
+    sans_address: "{{ groups['kube_control_plane'] | map('extract', hostvars, ['ansible_default_ipv6', 'ansible_default_ipv4', 'address']) | list | select('defined') | list }}"
+    sans_override: "{{ [kube_override_hostname] if kube_override_hostname else [] }}"
+    sans_hostname: "{{ groups['kube_control_plane'] | map('extract', hostvars, ['ansible_hostname']) | list | select('defined') | list }}"
+    sans_fqdn: "{{ groups['kube_control_plane'] | map('extract', hostvars, ['ansible_fqdn']) | list | select('defined') | list }}"
+    sans_kube_vip_address: "{{ [kube_vip_address] if kube_vip_address is defined and kube_vip_address else [] }}"
+  tags: facts
+
+- name: Create audit-policy directory
+  file:
+    path: "{{ audit_policy_file | dirname }}"
+    state: directory
+    mode: "0640"
+  when: kubernetes_audit or kubernetes_audit_webhook
+
+- name: Write api audit policy yaml
+  template:
+    src: apiserver-audit-policy.yaml.j2
+    dest: "{{ audit_policy_file }}"
+    mode: "0640"
+  when: kubernetes_audit or kubernetes_audit_webhook
+  notify: Control plane | Restart apiserver
+
+- name: Write api audit webhook config yaml
+  template:
+    src: apiserver-audit-webhook-config.yaml.j2
+    dest: "{{ audit_webhook_config_file }}"
+    mode: "0640"
+  when: kubernetes_audit_webhook
+  notify: Control plane | Restart apiserver
+
+- name: Create apiserver tracing config directory
+  file:
+    path: "{{ kube_config_dir }}/tracing"
+    state: directory
+    mode: "0640"
+  when: kube_apiserver_tracing
+
+- name: Write apiserver tracing config yaml
+  template:
+    src: apiserver-tracing.yaml.j2
+    dest: "{{ kube_config_dir }}/tracing/apiserver-tracing.yaml"
+    mode: "0640"
+  when: kube_apiserver_tracing
+  notify: Control plane | Restart apiserver
+
+# Nginx LB(default), If kubeadm_config_api_fqdn is defined, use other LB by kubeadm controlPlaneEndpoint.
+- name: Set kubeadm_config_api_fqdn define
+  set_fact:
+    kubeadm_config_api_fqdn: "{{ apiserver_loadbalancer_domain_name | default('lb-apiserver.kubernetes.local') }}"
+  when: loadbalancer_apiserver is defined
+
+- name: Kubeadm | Create kubeadm config
+  template:
+    src: "kubeadm-config.{{ kubeadm_config_api_version }}.yaml.j2"
+    dest: "{{ kube_config_dir }}/kubeadm-config.yaml"
+    mode: "0640"
+    validate: "{{ kubeadm_config_validate_enabled | ternary(bin_dir + '/kubeadm config validate --config %s', omit) }}"
+
+- name: Kubeadm | Create directory to store admission control configurations
+  file:
+    path: "{{ kube_config_dir }}/admission-controls"
+    state: directory
+    mode: "0640"
+  when: kube_apiserver_admission_control_config_file
+
+- name: Kubeadm | Push admission control config file
+  template:
+    src: "admission-controls.yaml.j2"
+    dest: "{{ kube_config_dir }}/admission-controls/admission-controls.yaml"
+    mode: "0640"
+  when: kube_apiserver_admission_control_config_file
+  notify: Control plane | Restart apiserver
+
+- name: Kubeadm | Push admission control config files
+  template:
+    src: "{{ item | lower }}.yaml.j2"
+    dest: "{{ kube_config_dir }}/admission-controls/{{ item | lower }}.yaml"
+    mode: "0640"
+  when:
+    - kube_apiserver_admission_control_config_file
+    - item in kube_apiserver_admission_plugins_needs_configuration
+  loop: "{{ kube_apiserver_enable_admission_plugins }}"
+  notify: Control plane | Restart apiserver
+
+- name: Kubeadm | Check apiserver.crt SANs
+  vars:
+    apiserver_ips: "{{ apiserver_sans | map('ansible.utils.ipaddr') | reject('equalto', False) | list }}"
+    apiserver_hosts: "{{ apiserver_sans | difference(apiserver_ips) }}"
+  when:
+    - kubeadm_already_run.stat.exists
+    - not kube_external_ca_mode
+  block:
+    - name: Kubeadm | Check apiserver.crt SAN IPs
+      command:
+        cmd: "openssl x509 -noout -in {{ kube_cert_dir }}/apiserver.crt -checkip {{ item }}"
+      loop: "{{ apiserver_ips }}"
+      register: apiserver_sans_ip_check
+      changed_when: apiserver_sans_ip_check.stdout is not search('does match certificate')
+      failed_when: apiserver_sans_ip_check.rc != 0 and apiserver_sans_ip_check.stdout is not search('does NOT match certificate')
+    - name: Kubeadm | Check apiserver.crt SAN hosts
+      command:
+        cmd: "openssl x509 -noout -in {{ kube_cert_dir }}/apiserver.crt -checkhost {{ item }}"
+      loop: "{{ apiserver_hosts }}"
+      register: apiserver_sans_host_check
+      changed_when: apiserver_sans_host_check.stdout is not search('does match certificate')
+      failed_when: apiserver_sans_host_check.rc != 0 and apiserver_sans_host_check.stdout is not search('does NOT match certificate')
+
+- name: Kubeadm | regenerate apiserver cert 1/2
+  file:
+    state: absent
+    path: "{{ kube_cert_dir }}/{{ item }}"
+  with_items:
+    - apiserver.crt
+    - apiserver.key
+  when:
+    - kubeadm_already_run.stat.exists
+    - apiserver_sans_ip_check.changed or apiserver_sans_host_check.changed
+    - not kube_external_ca_mode
+
+- name: Kubeadm | regenerate apiserver cert 2/2
+  command: >-
+    {{ bin_dir }}/kubeadm
+    init phase certs apiserver
+    --config={{ kube_config_dir }}/kubeadm-config.yaml
+  when:
+    - kubeadm_already_run.stat.exists
+    - apiserver_sans_ip_check.changed or apiserver_sans_host_check.changed
+    - not kube_external_ca_mode
+
+- name: Kubeadm | Initialize first control plane node
+  when: inventory_hostname == first_kube_control_plane and not kubeadm_already_run.stat.exists
+  vars:
+    kubeadm_init_first_control_plane_cmd: >-
+      timeout -k {{ kubeadm_init_timeout }} {{ kubeadm_init_timeout }}
+      {{ bin_dir }}/kubeadm init
+      --config={{ kube_config_dir }}/kubeadm-config.yaml
+      --ignore-preflight-errors={{ kubeadm_ignore_preflight_errors | join(',') }}
+      --skip-phases={{ kubeadm_init_phases_skip | join(',') }}
+      {{ kube_external_ca_mode | ternary('', '--upload-certs') }}
+  environment:
+    PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}"
+  notify: Control plane | restart kubelet
+  block:
+    - name: Kubeadm | Initialize first control plane node (1st try)
+      command: "{{ kubeadm_init_first_control_plane_cmd }}"
+      register: kubeadm_init
+      failed_when: kubeadm_init.rc != 0 and "field is immutable" not in kubeadm_init.stderr
+  rescue:
+    # Retry is because upload config sometimes fails
+    # This retry task is separated from 1st task to show log of failure of 1st task.
+    - name: Kubeadm | Initialize first control plane node (retry)
+      command: "{{ kubeadm_init_first_control_plane_cmd }}"
+      register: kubeadm_init
+      retries: 2
+      until: kubeadm_init is succeeded or "field is immutable" in kubeadm_init.stderr
+      failed_when: kubeadm_init.rc != 0 and "field is immutable" not in kubeadm_init.stderr
+
+- name: Set kubeadm certificate key
+  set_fact:
+    kubeadm_certificate_key: "{{ item | regex_search('--certificate-key ([^ ]+)', '\\1') | first }}"
+  with_items: "{{ hostvars[groups['kube_control_plane'][0]]['kubeadm_init'].stdout_lines | default([]) }}"
+  when:
+    - kubeadm_certificate_key is not defined
+    - (item | trim) is match('.*--certificate-key.*')
+
+- name: Create hardcoded kubeadm token for joining nodes with 24h expiration (if defined)
+  shell: >-
+    {{ bin_dir }}/kubeadm --kubeconfig {{ kube_config_dir }}/admin.conf token delete {{ kubeadm_token }} || :;
+    {{ bin_dir }}/kubeadm --kubeconfig {{ kube_config_dir }}/admin.conf token create {{ kubeadm_token }}
+  changed_when: false
+  when:
+    - inventory_hostname == first_kube_control_plane
+    - kubeadm_token is defined
+    - kubeadm_refresh_token
+  tags:
+    - kubeadm_token
+
+- name: Remove binding to anonymous user
+  command: "{{ kubectl }} -n kube-public delete rolebinding kubeadm:bootstrap-signer-clusterinfo --ignore-not-found"
+  when: inventory_hostname == first_kube_control_plane and remove_anonymous_access
+
+- name: Create kubeadm token for joining nodes with 24h expiration (default)
+  command: "{{ bin_dir }}/kubeadm --kubeconfig {{ kube_config_dir }}/admin.conf token create"
+  changed_when: false
+  register: temp_token
+  retries: 5
+  delay: 5
+  until: temp_token is succeeded
+  delegate_to: "{{ first_kube_control_plane }}"
+  when: kubeadm_token is not defined
+  tags:
+    - kubeadm_token
+
+- name: Set kubeadm_token
+  set_fact:
+    kubeadm_token: "{{ temp_token.stdout }}"
+  when: temp_token.stdout is defined
+  tags:
+    - kubeadm_token
+
+- name: Kubeadm | Join other control plane nodes
+  include_tasks: kubeadm-secondary.yml
+
+- name: Kubeadm | upgrade kubernetes cluster to {{ kube_version }}
+  include_tasks: kubeadm-upgrade.yml
+  when:
+    - upgrade_cluster_setup
+    - kubeadm_already_run.stat.exists
+
+# FIXME(mattymo): from docs: If you don't want to taint your control-plane node, set this field to an empty slice, i.e. `taints: {}` in the YAML file.
+- name: Kubeadm | Remove taint for control plane node with node role
+  command: "{{ kubectl }} taint node {{ inventory_hostname }} {{ item }}"
+  delegate_to: "{{ first_kube_control_plane }}"
+  with_items:
+    - "node-role.kubernetes.io/control-plane:NoSchedule-"
+  when: ('kube_node' in group_names)
+  failed_when: false

internal/resources/kubespray/project/roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml

@@ -0,0 +1,105 @@
+---
+- name: Ensure kube-apiserver is up before upgrade
+  import_tasks: check-api.yml
+
+  # kubeadm-config.v1beta4 with UpgradeConfiguration requires some values that were previously allowed as args to be specified in the config file
+- name: Kubeadm | Upgrade first control plane node
+  command: >-
+    timeout -k 600s 600s
+    {{ bin_dir }}/kubeadm upgrade apply -y v{{ kube_version }}
+    {%- if kubeadm_config_api_version == 'v1beta3' %}
+    --certificate-renewal={{ kubeadm_upgrade_auto_cert_renewal }}
+    --ignore-preflight-errors={{ kubeadm_ignore_preflight_errors | join(',') }}
+    --allow-experimental-upgrades
+    --etcd-upgrade={{ (etcd_deployment_type == "kubeadm") | lower }}
+    {% if kubeadm_patches | length > 0 %}--patches={{ kubeadm_patches_dir }}{% endif %}
+    --force
+    {%- else %}
+    --config={{ kube_config_dir }}/kubeadm-config.yaml
+    {%- endif -%}
+  register: kubeadm_upgrade
+  when: inventory_hostname == first_kube_control_plane
+  failed_when: kubeadm_upgrade.rc != 0 and "field is immutable" not in kubeadm_upgrade.stderr
+  environment:
+    PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}"
+
+- name: Kubeadm | Upgrade other control plane nodes
+  command: >-
+    {{ bin_dir }}/kubeadm upgrade node
+    {%- if kubeadm_config_api_version == 'v1beta3' %}
+    --certificate-renewal={{ kubeadm_upgrade_auto_cert_renewal }}
+    --ignore-preflight-errors={{ kubeadm_ignore_preflight_errors | join(',') }}
+    --etcd-upgrade={{ (etcd_deployment_type == "kubeadm") | lower }}
+    {% if kubeadm_patches | length > 0 %}--patches={{ kubeadm_patches_dir }}{% endif %}
+    {%- else %}
+    --config={{ kube_config_dir }}/kubeadm-config.yaml
+    {%- endif -%}
+  register: kubeadm_upgrade
+  when: inventory_hostname != first_kube_control_plane
+  failed_when: kubeadm_upgrade.rc != 0 and "field is immutable" not in kubeadm_upgrade.stderr
+  environment:
+    PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}"
+
+# kubeadm upgrade no longer reconciles ClusterConfiguration and KubeProxyConfiguration changes, this must be done separately after upgrade to ensure the latest config is applied
+- name: Update kubeadm and kubelet configmaps after upgrade
+  command: "{{ bin_dir }}/kubeadm init phase upload-config all --config {{ kube_config_dir }}/kubeadm-config.yaml"
+  register: kubeadm_upload_config
+  # Retry is because upload config sometimes fails
+  retries: 3
+  until: kubeadm_upload_config.rc == 0
+  when:
+  - inventory_hostname == first_kube_control_plane
+
+- name: Update kube-proxy configmap after upgrade
+  command: "{{ bin_dir }}/kubeadm init phase addon kube-proxy --config {{ kube_config_dir }}/kubeadm-config.yaml"
+  register: kube_proxy_upload_config
+  # Retry is because upload config sometimes fails
+  retries: 3
+  until: kube_proxy_upload_config.rc == 0
+  when:
+  - inventory_hostname == first_kube_control_plane
+  - ('addon/kube-proxy' not in kubeadm_init_phases_skip)
+
+- name: Rewrite kubeadm managed etcd static pod manifests with updated configmap
+  command: "{{ bin_dir }}/kubeadm init phase etcd local --config {{ kube_config_dir }}/kubeadm-config.yaml"
+  when:
+  - etcd_deployment_type == "kubeadm"
+  notify: Control plane | restart kubelet
+
+- name: Rewrite kubernetes control plane static pod manifests with updated configmap
+  command: "{{ bin_dir }}/kubeadm init phase control-plane all --config {{ kube_config_dir }}/kubeadm-config.yaml"
+  notify: Control plane | restart kubelet
+
+- name: Flush kubelet handlers
+  meta: flush_handlers
+
+- name: Ensure kube-apiserver is up after upgrade and control plane configuration updates
+  import_tasks: check-api.yml
+
+- name: Kubeadm | Remove binding to anonymous user
+  command: "{{ kubectl }} -n kube-public delete rolebinding kubeadm:bootstrap-signer-clusterinfo --ignore-not-found"
+  when: remove_anonymous_access
+
+- name: Kubeadm | clean kubectl cache to refresh api types
+  file:
+    path: "{{ item }}"
+    state: absent
+  with_items:
+  - /root/.kube/cache
+  - /root/.kube/http-cache
+
+# FIXME: https://github.com/kubernetes/kubeadm/issues/1318
+- name: Kubeadm | scale down coredns replicas to 0 if not using coredns dns_mode
+  command: >-
+    {{ kubectl }}
+    -n kube-system
+    scale deployment/coredns --replicas 0
+  register: scale_down_coredns
+  retries: 6
+  delay: 5
+  until: scale_down_coredns is succeeded
+  run_once: true
+  when:
+  - kubeadm_scale_down_coredns_enabled
+  - dns_mode not in ['coredns', 'coredns_dual']
+  changed_when: false

internal/resources/kubespray/project/roles/kubernetes/control-plane/tasks/kubelet-fix-client-cert-rotation.yml

@@ -0,0 +1,18 @@
+---
+- name: Fixup kubelet client cert rotation 1/2
+  lineinfile:
+    path: "{{ kube_config_dir }}/kubelet.conf"
+    regexp: '^    client-certificate-data: '
+    line: '    client-certificate: /var/lib/kubelet/pki/kubelet-client-current.pem'
+    backup: true
+  notify:
+    - "Control plane | reload kubelet"
+
+- name: Fixup kubelet client cert rotation 2/2
+  lineinfile:
+    path: "{{ kube_config_dir }}/kubelet.conf"
+    regexp: '^    client-key-data: '
+    line: '    client-key: /var/lib/kubelet/pki/kubelet-client-current.pem'
+    backup: true
+  notify:
+    - "Control plane | reload kubelet"

internal/resources/kubespray/project/roles/kubernetes/control-plane/tasks/main.yml

@@ -0,0 +1,145 @@
+---
+- name: Pre-upgrade control plane
+  import_tasks: pre-upgrade.yml
+  tags:
+    - k8s-pre-upgrade
+
+- name: Create webhook token auth config
+  template:
+    src: webhook-token-auth-config.yaml.j2
+    dest: "{{ kube_config_dir }}/webhook-token-auth-config.yaml"
+    mode: "0640"
+  when: kube_webhook_token_auth | default(false)
+
+- name: Create webhook authorization config
+  template:
+    src: webhook-authorization-config.yaml.j2
+    dest: "{{ kube_config_dir }}/webhook-authorization-config.yaml"
+    mode: "0640"
+  when: kube_webhook_authorization | default(false)
+
+- name: Create structured AuthorizationConfiguration file
+  copy:
+    content: "{{ authz_config | to_nice_yaml(indent=2, sort_keys=false) }}"
+    dest: "{{ kube_config_dir }}/apiserver-authorization-config-{{ kube_apiserver_authorization_config_api_version }}.yaml"
+    mode: "0640"
+  vars:
+    authz_config:
+      apiVersion: apiserver.config.k8s.io/{{ kube_apiserver_authorization_config_api_version }}
+      kind: AuthorizationConfiguration
+      authorizers: "{{ kube_apiserver_authorization_config_authorizers }}"
+  when: kube_apiserver_use_authorization_config_file
+
+- name: Create kube-scheduler config
+  template:
+    src: kubescheduler-config.yaml.j2
+    dest: "{{ kube_config_dir }}/kubescheduler-config.yaml"
+    mode: "0644"
+
+- name: Apply Kubernetes encrypt at rest config
+  import_tasks: encrypt-at-rest.yml
+  when:
+    - kube_encrypt_secret_data
+  tags:
+    - kube-apiserver
+
+- name: Install | Copy kubectl binary from download dir
+  copy:
+    src: "{{ downloads.kubectl.dest }}"
+    dest: "{{ bin_dir }}/kubectl"
+    mode: "0755"
+    remote_src: true
+  tags:
+    - kubectl
+    - upgrade
+
+- name: Install kubectl bash completion
+  shell: "{{ bin_dir }}/kubectl completion bash >/etc/bash_completion.d/kubectl.sh"
+  when: ansible_os_family in ["Debian","RedHat", "Suse"]
+  tags:
+    - kubectl
+  ignore_errors: true  # noqa ignore-errors
+
+- name: Set kubectl bash completion file permissions
+  file:
+    path: /etc/bash_completion.d/kubectl.sh
+    owner: root
+    group: root
+    mode: "0755"
+  when: ansible_os_family in ["Debian","RedHat", "Suse"]
+  tags:
+    - kubectl
+    - upgrade
+  ignore_errors: true  # noqa ignore-errors
+
+- name: Set bash alias for kubectl
+  blockinfile:
+    path: /etc/bash_completion.d/kubectl.sh
+    block: |-
+      alias {{ kubectl_alias }}=kubectl
+      if [[ $(type -t compopt) = "builtin" ]]; then
+        complete -o default -F __start_kubectl {{ kubectl_alias }}
+      else
+        complete -o default -o nospace -F __start_kubectl {{ kubectl_alias }}
+      fi
+    state: present
+    marker: "# Ansible entries {mark}"
+  when:
+    - ansible_os_family in ["Debian","RedHat", "Suse"]
+    - kubectl_alias is defined and kubectl_alias != ""
+  tags:
+    - kubectl
+    - upgrade
+  ignore_errors: true  # noqa ignore-errors
+
+- name: Define nodes already joined to existing cluster and first_kube_control_plane
+  import_tasks: define-first-kube-control.yml
+
+- name: Include kubeadm setup
+  import_tasks: kubeadm-setup.yml
+
+- name: Include kubeadm etcd extra tasks
+  include_tasks: kubeadm-etcd.yml
+  when: etcd_deployment_type == "kubeadm"
+
+- name: Include kubeadm secondary server apiserver fixes
+  include_tasks: kubeadm-fix-apiserver.yml
+
+- name: Cleanup unused AuthorizationConfiguration file versions
+  file:
+    path: "{{ kube_config_dir }}/apiserver-authorization-config-{{ item }}.yaml"
+    state: absent
+  loop: "{{ ['v1alpha1', 'v1beta1', 'v1'] | reject('equalto', kube_apiserver_authorization_config_api_version) | list }}"
+  when: kube_apiserver_use_authorization_config_file
+
+- name: Include kubelet client cert rotation fixes
+  include_tasks: kubelet-fix-client-cert-rotation.yml
+  when: kubelet_rotate_certificates
+
+- name: Install script to renew K8S control plane certificates
+  template:
+    src: k8s-certs-renew.sh.j2
+    dest: "{{ bin_dir }}/k8s-certs-renew.sh"
+    mode: "0755"
+
+- name: Renew K8S control plane certificates monthly 1/2
+  template:
+    src: "{{ item }}.j2"
+    dest: "/etc/systemd/system/{{ item }}"
+    mode: "0644"
+    validate: "sh -c '[ -f /usr/bin/systemd/system/factory-reset.target ] || exit 0 && systemd-analyze verify %s:{{item}}'"
+    # FIXME: check that systemd version >= 250 (factory-reset.target was introduced in that release)
+    # Remove once we drop support for systemd < 250
+  with_items:
+    - k8s-certs-renew.service
+    - k8s-certs-renew.timer
+  register: k8s_certs_units
+  when: auto_renew_certificates
+
+- name: Renew K8S control plane certificates monthly 2/2
+  systemd_service:
+    name: k8s-certs-renew.timer
+    enabled: true
+    state: started
+    daemon_reload: "{{ k8s_certs_units is changed }}"
+  when: auto_renew_certificates

internal/resources/kubespray/project/roles/kubernetes/control-plane/tasks/pre-upgrade.yml

@@ -0,0 +1,21 @@
+---
+- name: "Pre-upgrade | Delete control plane manifests if etcd secrets changed"
+  file:
+    path: "/etc/kubernetes/manifests/{{ item }}.manifest"
+    state: absent
+  with_items:
+    - ["kube-apiserver", "kube-controller-manager", "kube-scheduler"]
+  register: kube_apiserver_manifest_replaced
+  when: etcd_secret_changed | default(false)
+
+- name: "Pre-upgrade | Delete control plane containers forcefully"  # noqa no-handler
+  shell: "set -o pipefail && docker ps -af name=k8s_{{ item }}* -q | xargs --no-run-if-empty docker rm -f"
+  args:
+    executable: /bin/bash
+  with_items:
+    - ["kube-apiserver", "kube-controller-manager", "kube-scheduler"]
+  when: kube_apiserver_manifest_replaced.changed
+  register: remove_control_plane_container
+  retries: 10
+  until: remove_control_plane_container.rc == 0
+  delay: 1