gstykalin/kube-forge
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

actualized for kubespray version 2.28.0

Browse Source
This commit is contained in:
George Stykalin
2025-06-15 20:25:45 +03:00
parent c819d4f51f
commit c2df2abc78
366 changed files with 9939 additions and 40388 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
kubespray/project/roles/kubernetes/control-plane/tasks/check-api.yml Normal file
View File

@@ -0,0 +1,10 @@
---
- name: Kubeadm | Check api is up
uri:
url: "https://{{ ip | default(fallback_ip) }}:{{ kube_apiserver_port }}/healthz"
validate_certs: false
when: ('kube_control_plane' in group_names)
register: _result
retries: 60
delay: 5
until: _result.status == 200

2
kubespray/project/roles/kubernetes/control-plane/tasks/encrypt-at-rest.yml
View File

@@ -23,7 +23,7 @@
kube_encrypt_token_extracted: "{{ secret_file_decoded | json_query(secrets_encryption_query) | first | b64decode }}"
when: secrets_encryption_file.stat.exists
- name: Set kube_encrypt_token across master nodes
- name: Set kube_encrypt_token across control plane nodes
set_fact:
kube_encrypt_token: "{{ kube_encrypt_token_extracted }}"
delegate_to: "{{ item }}"

6
kubespray/project/roles/kubernetes/control-plane/tasks/kubeadm-fix-apiserver.yml
View File

@@ -12,6 +12,6 @@
- kubelet.conf
- scheduler.conf
notify:
- "Master | Restart kube-controller-manager"
- "Master | Restart kube-scheduler"
- "Master | reload kubelet"
- "Control plane | Restart kube-controller-manager"
- "Control plane | Restart kube-scheduler"
- "Control plane | reload kubelet"

17
kubespray/project/roles/kubernetes/control-plane/tasks/kubeadm-secondary.yml
View File

@@ -4,7 +4,7 @@
# noqa: jinja[spacing]
kubeadm_discovery_address: >-
{%- if "127.0.0.1" in kube_apiserver_endpoint or "localhost" in kube_apiserver_endpoint -%}
{{ first_kube_control_plane_address }}:{{ kube_apiserver_port }}
{{ first_kube_control_plane_address | ansible.utils.ipwrap }}:{{ kube_apiserver_port }}
{%- else -%}
{{ kube_apiserver_endpoint | regex_replace('https://', '') }}
{%- endif %}
@@ -24,26 +24,27 @@
- name: Parse certificate key if not set
set_fact:
kubeadm_certificate_key: "{{ hostvars[groups['kube_control_plane'][0]]['kubeadm_upload_cert'].stdout_lines[-1] | trim }}"
kubeadm_certificate_key: "{{ hostvars[first_kube_control_plane]['kubeadm_upload_cert'].stdout_lines[-1] | trim }}"
run_once: true
when:
- hostvars[groups['kube_control_plane'][0]]['kubeadm_upload_cert'] is defined
- hostvars[groups['kube_control_plane'][0]]['kubeadm_upload_cert'] is not skipped
- hostvars[first_kube_control_plane]['kubeadm_upload_cert'] is defined
- hostvars[first_kube_control_plane]['kubeadm_upload_cert'] is not skipped
- name: Create kubeadm ControlPlane config
template:
src: "kubeadm-controlplane.{{ kubeadmConfig_api_version }}.yaml.j2"
src: "kubeadm-controlplane.yaml.j2"
dest: "{{ kube_config_dir }}/kubeadm-controlplane.yaml"
mode: "0640"
backup: true
validate: "{{ kubeadm_config_validate_enabled | ternary(bin_dir + '/kubeadm config validate --config %s', omit) }}"
when:
- inventory_hostname != first_kube_control_plane
- not kubeadm_already_run.stat.exists
- name: Wait for k8s apiserver
wait_for:
host: "{{ kubeadm_discovery_address.split(':')[0] }}"
port: "{{ kubeadm_discovery_address.split(':')[1] }}"
host: "{{ kubeadm_discovery_address | regex_replace('\\]?:\\d+$', '') | regex_replace('^\\[', '') }}"
port: "{{ kubeadm_discovery_address.split(':')[-1] }}"
timeout: 180
@@ -87,7 +88,7 @@
command: >-
{{ bin_dir }}/kubeadm join
--config {{ kube_config_dir }}/kubeadm-controlplane.yaml
--ignore-preflight-errors=all
--ignore-preflight-errors={{ kubeadm_ignore_preflight_errors | join(',') }}
--skip-phases={{ kubeadm_join_phases_skip | join(',') }}
environment:
PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}"

97
kubespray/project/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml
View File

@@ -35,12 +35,13 @@
- "{{ kube_apiserver_ip }}"
- "localhost"
- "127.0.0.1"
- "::1"
sans_lb: "{{ [apiserver_loadbalancer_domain_name] if apiserver_loadbalancer_domain_name is defined else [] }}"
sans_lb_ip: "{{ [loadbalancer_apiserver.address] if loadbalancer_apiserver is defined and loadbalancer_apiserver.address is defined else [] }}"
sans_supp: "{{ supplementary_addresses_in_ssl_keys if supplementary_addresses_in_ssl_keys is defined else [] }}"
sans_access_ip: "{{ groups['kube_control_plane'] | map('extract', hostvars, 'access_ip') | list | select('defined') | list }}"
sans_ip: "{{ groups['kube_control_plane'] | map('extract', hostvars, 'ip') | list | select('defined') | list }}"
sans_address: "{{ groups['kube_control_plane'] | map('extract', hostvars, ['ansible_default_ipv4', 'address']) | list | select('defined') | list }}"
sans_access_ip: "{{ groups['kube_control_plane'] | map('extract', hostvars, 'main_access_ip') | list | select('defined') | list }}"
sans_ip: "{{ groups['kube_control_plane'] | map('extract', hostvars, 'main_ip') | list | select('defined') | list }}"
sans_address: "{{ groups['kube_control_plane'] | map('extract', hostvars, ['ansible_default_ipv6', 'ansible_default_ipv4', 'address']) | list | select('defined') | list }}"
sans_override: "{{ [kube_override_hostname] if kube_override_hostname else [] }}"
sans_hostname: "{{ groups['kube_control_plane'] | map('extract', hostvars, ['ansible_hostname']) | list | select('defined') | list }}"
sans_fqdn: "{{ groups['kube_control_plane'] | map('extract', hostvars, ['ansible_fqdn']) | list | select('defined') | list }}"
@@ -52,21 +53,23 @@
path: "{{ audit_policy_file | dirname }}"
state: directory
mode: "0640"
when: kubernetes_audit | default(false) or kubernetes_audit_webhook | default(false)
when: kubernetes_audit or kubernetes_audit_webhook
- name: Write api audit policy yaml
template:
src: apiserver-audit-policy.yaml.j2
dest: "{{ audit_policy_file }}"
mode: "0640"
when: kubernetes_audit | default(false) or kubernetes_audit_webhook | default(false)
when: kubernetes_audit or kubernetes_audit_webhook
notify: Control plane | Restart apiserver
- name: Write api audit webhook config yaml
template:
src: apiserver-audit-webhook-config.yaml.j2
dest: "{{ audit_webhook_config_file }}"
mode: "0640"
when: kubernetes_audit_webhook | default(false)
when: kubernetes_audit_webhook
notify: Control plane | Restart apiserver
- name: Create apiserver tracing config directory
file:
@@ -81,6 +84,7 @@
dest: "{{ kube_config_dir }}/tracing/apiserver-tracing.yaml"
mode: "0640"
when: kube_apiserver_tracing
notify: Control plane | Restart apiserver
# Nginx LB(default), If kubeadm_config_api_fqdn is defined, use other LB by kubeadm controlPlaneEndpoint.
- name: Set kubeadm_config_api_fqdn define
@@ -88,15 +92,12 @@
kubeadm_config_api_fqdn: "{{ apiserver_loadbalancer_domain_name | default('lb-apiserver.kubernetes.local') }}"
when: loadbalancer_apiserver is defined
- name: Set kubeadm api version to v1beta3
set_fact:
kubeadmConfig_api_version: v1beta3
- name: Kubeadm | Create kubeadm config
template:
src: "kubeadm-config.{{ kubeadmConfig_api_version }}.yaml.j2"
src: "kubeadm-config.{{ kubeadm_config_api_version }}.yaml.j2"
dest: "{{ kube_config_dir }}/kubeadm-config.yaml"
mode: "0640"
validate: "{{ kubeadm_config_validate_enabled | ternary(bin_dir + '/kubeadm config validate --config %s', omit) }}"
- name: Kubeadm | Create directory to store admission control configurations
file:
@@ -111,6 +112,7 @@
dest: "{{ kube_config_dir }}/admission-controls/admission-controls.yaml"
mode: "0640"
when: kube_apiserver_admission_control_config_file
notify: Control plane | Restart apiserver
- name: Kubeadm | Push admission control config files
template:
@@ -121,15 +123,7 @@
- kube_apiserver_admission_control_config_file
- item in kube_apiserver_admission_plugins_needs_configuration
loop: "{{ kube_apiserver_enable_admission_plugins }}"
- name: Kubeadm | Configure default cluster podnodeslector
template:
src: "podnodeselector.yaml.j2"
dest: "{{ kube_config_dir }}/admission-controls/podnodeselector.yaml"
mode: "0640"
when:
- kube_apiserver_admission_plugins_podnodeselector_default_node_selector is defined
- kube_apiserver_admission_plugins_podnodeselector_default_node_selector | length > 0
notify: Control plane | Restart apiserver
- name: Kubeadm | Check apiserver.crt SANs
vars:
@@ -145,12 +139,14 @@
loop: "{{ apiserver_ips }}"
register: apiserver_sans_ip_check
changed_when: apiserver_sans_ip_check.stdout is not search('does match certificate')
failed_when: apiserver_sans_ip_check.rc != 0 and apiserver_sans_ip_check.stdout is not search('does NOT match certificate')
- name: Kubeadm | Check apiserver.crt SAN hosts
command:
cmd: "openssl x509 -noout -in {{ kube_cert_dir }}/apiserver.crt -checkhost {{ item }}"
loop: "{{ apiserver_hosts }}"
register: apiserver_sans_host_check
changed_when: apiserver_sans_host_check.stdout is not search('does match certificate')
failed_when: apiserver_sans_host_check.rc != 0 and apiserver_sans_host_check.stdout is not search('does NOT match certificate')
- name: Kubeadm | regenerate apiserver cert 1/2
file:
@@ -174,38 +170,33 @@
- apiserver_sans_ip_check.changed or apiserver_sans_host_check.changed
- not kube_external_ca_mode
- name: Kubeadm | Create directory to store kubeadm patches
file:
path: "{{ kubeadm_patches.dest_dir }}"
state: directory
mode: "0640"
when: kubeadm_patches is defined and kubeadm_patches.enabled
- name: Kubeadm | Copy kubeadm patches from inventory files
copy:
src: "{{ kubeadm_patches.source_dir }}/"
dest: "{{ kubeadm_patches.dest_dir }}"
owner: "root"
mode: "0644"
when: kubeadm_patches is defined and kubeadm_patches.enabled
- name: Kubeadm | Initialize first master
command: >-
timeout -k {{ kubeadm_init_timeout }} {{ kubeadm_init_timeout }}
{{ bin_dir }}/kubeadm init
--config={{ kube_config_dir }}/kubeadm-config.yaml
--ignore-preflight-errors=all
--skip-phases={{ kubeadm_init_phases_skip | join(',') }}
{{ kube_external_ca_mode | ternary('', '--upload-certs') }}
register: kubeadm_init
# Retry is because upload config sometimes fails
retries: 3
until: kubeadm_init is succeeded or "field is immutable" in kubeadm_init.stderr
- name: Kubeadm | Initialize first control plane node
when: inventory_hostname == first_kube_control_plane and not kubeadm_already_run.stat.exists
failed_when: kubeadm_init.rc != 0 and "field is immutable" not in kubeadm_init.stderr
vars:
kubeadm_init_first_control_plane_cmd: >-
timeout -k {{ kubeadm_init_timeout }} {{ kubeadm_init_timeout }}
{{ bin_dir }}/kubeadm init
--config={{ kube_config_dir }}/kubeadm-config.yaml
--ignore-preflight-errors={{ kubeadm_ignore_preflight_errors | join(',') }}
--skip-phases={{ kubeadm_init_phases_skip | join(',') }}
{{ kube_external_ca_mode | ternary('', '--upload-certs') }}
environment:
PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}"
notify: Master | restart kubelet
notify: Control plane | restart kubelet
block:
- name: Kubeadm | Initialize first control plane node (1st try)
command: "{{ kubeadm_init_first_control_plane_cmd }}"
register: kubeadm_init
failed_when: kubeadm_init.rc != 0 and "field is immutable" not in kubeadm_init.stderr
rescue:
# Retry is because upload config sometimes fails
# This retry task is separated from 1st task to show log of failure of 1st task.
- name: Kubeadm | Initialize first control plane node (retry)
command: "{{ kubeadm_init_first_control_plane_cmd }}"
register: kubeadm_init
retries: 2
until: kubeadm_init is succeeded or "field is immutable" in kubeadm_init.stderr
failed_when: kubeadm_init.rc != 0 and "field is immutable" not in kubeadm_init.stderr
- name: Set kubeadm certificate key
set_fact:
@@ -250,20 +241,20 @@
tags:
- kubeadm_token
- name: Kubeadm | Join other masters
- name: Kubeadm | Join other control plane nodes
include_tasks: kubeadm-secondary.yml
- name: Kubeadm | upgrade kubernetes cluster
- name: Kubeadm | upgrade kubernetes cluster to {{ kube_version }}
include_tasks: kubeadm-upgrade.yml
when:
- upgrade_cluster_setup
- kubeadm_already_run.stat.exists
# FIXME(mattymo): from docs: If you don't want to taint your control-plane node, set this field to an empty slice, i.e. `taints: {}` in the YAML file.
- name: Kubeadm | Remove taint for master with node role
- name: Kubeadm | Remove taint for control plane node with node role
command: "{{ kubectl }} taint node {{ inventory_hostname }} {{ item }}"
delegate_to: "{{ first_kube_control_plane }}"
with_items:
- "node-role.kubernetes.io/control-plane:NoSchedule-"
when: inventory_hostname in groups['kube_node']
when: ('kube_node' in group_names)
failed_when: false

97
kubespray/project/roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml
View File

@@ -1,55 +1,80 @@
---
- name: Kubeadm | Check api is up
uri:
url: "https://{{ ip | default(fallback_ips[inventory_hostname]) }}:{{ kube_apiserver_port }}/healthz"
validate_certs: false
when: inventory_hostname in groups['kube_control_plane']
register: _result
retries: 60
delay: 5
until: _result.status == 200
- name: Ensure kube-apiserver is up before upgrade
import_tasks: check-api.yml
- name: Kubeadm | Upgrade first master
# kubeadm-config.v1beta4 with UpgradeConfiguration requires some values that were previously allowed as args to be specified in the config file
- name: Kubeadm | Upgrade first control plane node
command: >-
timeout -k 600s 600s
{{ bin_dir }}/kubeadm
upgrade apply -y {{ kube_version }}
{{ bin_dir }}/kubeadm upgrade apply -y v{{ kube_version }}
{%- if kubeadm_config_api_version == 'v1beta3' %}
--certificate-renewal={{ kubeadm_upgrade_auto_cert_renewal }}
--ignore-preflight-errors=all
--ignore-preflight-errors={{ kubeadm_ignore_preflight_errors | join(',') }}
--allow-experimental-upgrades
--etcd-upgrade={{ (etcd_deployment_type == "kubeadm") | bool | lower }}
{% if kubeadm_patches is defined and kubeadm_patches.enabled %}--patches={{ kubeadm_patches.dest_dir }}{% endif %}
--etcd-upgrade={{ (etcd_deployment_type == "kubeadm") | lower }}
{% if kubeadm_patches | length > 0 %}--patches={{ kubeadm_patches_dir }}{% endif %}
--force
{%- else %}
--config={{ kube_config_dir }}/kubeadm-config.yaml
{%- endif -%}
register: kubeadm_upgrade
# Retry is because upload config sometimes fails
retries: 3
until: kubeadm_upgrade.rc == 0
when: inventory_hostname == first_kube_control_plane
failed_when: kubeadm_upgrade.rc != 0 and "field is immutable" not in kubeadm_upgrade.stderr
environment:
PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}"
notify: Master | restart kubelet
- name: Kubeadm | Upgrade other masters
- name: Kubeadm | Upgrade other control plane nodes
command: >-
timeout -k 600s 600s
{{ bin_dir }}/kubeadm
upgrade apply -y {{ kube_version }}
{{ bin_dir }}/kubeadm upgrade node
{%- if kubeadm_config_api_version == 'v1beta3' %}
--certificate-renewal={{ kubeadm_upgrade_auto_cert_renewal }}
--ignore-preflight-errors=all
--allow-experimental-upgrades
--etcd-upgrade={{ (etcd_deployment_type == "kubeadm") | bool | lower }}
{% if kubeadm_patches is defined and kubeadm_patches.enabled %}--patches={{ kubeadm_patches.dest_dir }}{% endif %}
--force
--ignore-preflight-errors={{ kubeadm_ignore_preflight_errors | join(',') }}
--etcd-upgrade={{ (etcd_deployment_type == "kubeadm") | lower }}
{% if kubeadm_patches | length > 0 %}--patches={{ kubeadm_patches_dir }}{% endif %}
{%- else %}
--config={{ kube_config_dir }}/kubeadm-config.yaml
{%- endif -%}
register: kubeadm_upgrade
# Retry is because upload config sometimes fails
retries: 3
until: kubeadm_upgrade.rc == 0
when: inventory_hostname != first_kube_control_plane
failed_when: kubeadm_upgrade.rc != 0 and "field is immutable" not in kubeadm_upgrade.stderr
environment:
PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}"
notify: Master | restart kubelet
# kubeadm upgrade no longer reconciles ClusterConfiguration and KubeProxyConfiguration changes, this must be done separately after upgrade to ensure the latest config is applied
- name: Update kubeadm and kubelet configmaps after upgrade
command: "{{ bin_dir }}/kubeadm init phase upload-config all --config {{ kube_config_dir }}/kubeadm-config.yaml"
register: kubeadm_upload_config
# Retry is because upload config sometimes fails
retries: 3
until: kubeadm_upload_config.rc == 0
when:
- inventory_hostname == first_kube_control_plane
- name: Update kube-proxy configmap after upgrade
command: "{{ bin_dir }}/kubeadm init phase addon kube-proxy --config {{ kube_config_dir }}/kubeadm-config.yaml"
register: kube_proxy_upload_config
# Retry is because upload config sometimes fails
retries: 3
until: kube_proxy_upload_config.rc == 0
when:
- inventory_hostname == first_kube_control_plane
- ('addon/kube-proxy' not in kubeadm_init_phases_skip)
- name: Rewrite kubeadm managed etcd static pod manifests with updated configmap
command: "{{ bin_dir }}/kubeadm init phase etcd local --config {{ kube_config_dir }}/kubeadm-config.yaml"
when:
- etcd_deployment_type == "kubeadm"
notify: Control plane | restart kubelet
- name: Rewrite kubernetes control plane static pod manifests with updated configmap
command: "{{ bin_dir }}/kubeadm init phase control-plane all --config {{ kube_config_dir }}/kubeadm-config.yaml"
notify: Control plane | restart kubelet
- name: Flush kubelet handlers
meta: flush_handlers
- name: Ensure kube-apiserver is up after upgrade and control plane configuration updates
import_tasks: check-api.yml
- name: Kubeadm | Remove binding to anonymous user
command: "{{ kubectl }} -n kube-public delete rolebinding kubeadm:bootstrap-signer-clusterinfo --ignore-not-found"
@@ -60,8 +85,8 @@
path: "{{ item }}"
state: absent
with_items:
- /root/.kube/cache
- /root/.kube/http-cache
- /root/.kube/cache
- /root/.kube/http-cache
# FIXME: https://github.com/kubernetes/kubeadm/issues/1318
- name: Kubeadm | scale down coredns replicas to 0 if not using coredns dns_mode
@@ -75,6 +100,6 @@
until: scale_down_coredns is succeeded
run_once: true
when:
- kubeadm_scale_down_coredns_enabled
- dns_mode not in ['coredns', 'coredns_dual']
- kubeadm_scale_down_coredns_enabled
- dns_mode not in ['coredns', 'coredns_dual']
changed_when: false

4
kubespray/project/roles/kubernetes/control-plane/tasks/kubelet-fix-client-cert-rotation.yml
View File

@@ -6,7 +6,7 @@
line: ' client-certificate: /var/lib/kubelet/pki/kubelet-client-current.pem'
backup: true
notify:
- "Master | reload kubelet"
- "Control plane | reload kubelet"
- name: Fixup kubelet client cert rotation 2/2
lineinfile:
@@ -15,4 +15,4 @@
line: ' client-key: /var/lib/kubelet/pki/kubelet-client-current.pem'
backup: true
notify:
- "Master | reload kubelet"
- "Control plane | reload kubelet"

25
kubespray/project/roles/kubernetes/control-plane/tasks/main.yml
View File

@@ -18,6 +18,18 @@
mode: "0640"
when: kube_webhook_authorization | default(false)
- name: Create structured AuthorizationConfiguration file
copy:
content: "{{ authz_config | to_nice_yaml(indent=2, sort_keys=false) }}"
dest: "{{ kube_config_dir }}/apiserver-authorization-config-{{ kube_apiserver_authorization_config_api_version }}.yaml"
mode: "0640"
vars:
authz_config:
apiVersion: apiserver.config.k8s.io/{{ kube_apiserver_authorization_config_api_version }}
kind: AuthorizationConfiguration
authorizers: "{{ kube_apiserver_authorization_config_authorizers }}"
when: kube_apiserver_use_authorization_config_file
- name: Create kube-scheduler config
template:
src: kubescheduler-config.yaml.j2
@@ -43,7 +55,7 @@
- name: Install kubectl bash completion
shell: "{{ bin_dir }}/kubectl completion bash >/etc/bash_completion.d/kubectl.sh"
when: ansible_os_family in ["Debian","RedHat"]
when: ansible_os_family in ["Debian","RedHat", "Suse"]
tags:
- kubectl
ignore_errors: true # noqa ignore-errors
@@ -54,7 +66,7 @@
owner: root
group: root
mode: "0755"
when: ansible_os_family in ["Debian","RedHat"]
when: ansible_os_family in ["Debian","RedHat", "Suse"]
tags:
- kubectl
- upgrade
@@ -73,7 +85,7 @@
state: present
marker: "# Ansible entries {mark}"
when:
- ansible_os_family in ["Debian","RedHat"]
- ansible_os_family in ["Debian","RedHat", "Suse"]
- kubectl_alias is defined and kubectl_alias != ""
tags:
- kubectl
@@ -93,6 +105,13 @@
- name: Include kubeadm secondary server apiserver fixes
include_tasks: kubeadm-fix-apiserver.yml
- name: Cleanup unused AuthorizationConfiguration file versions
file:
path: "{{ kube_config_dir }}/apiserver-authorization-config-{{ item }}.yaml"
state: absent
loop: "{{ ['v1alpha1', 'v1beta1', 'v1'] | reject('equalto', kube_apiserver_authorization_config_api_version) | list }}"
when: kube_apiserver_use_authorization_config_file
- name: Include kubelet client cert rotation fixes
include_tasks: kubelet-fix-client-cert-rotation.yml
when: kubelet_rotate_certificates

8
kubespray/project/roles/kubernetes/control-plane/tasks/pre-upgrade.yml
View File

@@ -1,5 +1,5 @@
---
- name: "Pre-upgrade | Delete master manifests if etcd secrets changed"
- name: "Pre-upgrade | Delete control plane manifests if etcd secrets changed"
file:
path: "/etc/kubernetes/manifests/{{ item }}.manifest"
state: absent
@@ -8,14 +8,14 @@
register: kube_apiserver_manifest_replaced
when: etcd_secret_changed | default(false)
- name: "Pre-upgrade | Delete master containers forcefully" # noqa no-handler
- name: "Pre-upgrade | Delete control plane containers forcefully" # noqa no-handler
shell: "set -o pipefail && docker ps -af name=k8s_{{ item }}* -q | xargs --no-run-if-empty docker rm -f"
args:
executable: /bin/bash
with_items:
- ["kube-apiserver", "kube-controller-manager", "kube-scheduler"]
when: kube_apiserver_manifest_replaced.changed
register: remove_master_container
register: remove_control_plane_container
retries: 10
until: remove_master_container.rc == 0
until: remove_control_plane_container.rc == 0
delay: 1