kube-forge v2: embede python with ansible (kubespray) and cobra as cli building tool

internal/resources/kubespray/project/roles/kubernetes/client/defaults/main.yml

@@ -0,0 +1,8 @@
+---
+kubeconfig_localhost: false
+kubeconfig_localhost_ansible_host: false
+kubectl_localhost: false
+artifacts_dir: "{{ inventory_dir }}/artifacts"
+
+kube_config_dir: "/etc/kubernetes"
+kube_apiserver_port: "6443"

internal/resources/kubespray/project/roles/kubernetes/client/tasks/main.yml

@@ -0,0 +1,114 @@
+---
+- name: Set external kube-apiserver endpoint
+  set_fact:
+    # noqa: jinja[spacing]
+    external_apiserver_address: >-
+      {%- if loadbalancer_apiserver is defined and loadbalancer_apiserver.address is defined -%}
+      {{ loadbalancer_apiserver.address }}
+      {%- elif kubeconfig_localhost_ansible_host is defined and kubeconfig_localhost_ansible_host -%}
+      {{ hostvars[groups['kube_control_plane'][0]].ansible_host }}
+      {%- else -%}
+      {{ kube_apiserver_access_address }}
+      {%- endif -%}
+    # noqa: jinja[spacing]
+    external_apiserver_port: >-
+      {%- if loadbalancer_apiserver is defined and loadbalancer_apiserver.address is defined and loadbalancer_apiserver.port is defined -%}
+      {{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}
+      {%- else -%}
+      {{ kube_apiserver_port }}
+      {%- endif -%}
+  tags:
+    - facts
+
+- name: Create kube config dir for current/ansible become user
+  file:
+    path: "{{ ansible_env.HOME | default('/root') }}/.kube"
+    mode: "0700"
+    state: directory
+
+- name: Copy admin kubeconfig to current/ansible become user home
+  copy:
+    src: "{{ kube_config_dir }}/admin.conf"
+    dest: "{{ ansible_env.HOME | default('/root') }}/.kube/config"
+    remote_src: true
+    mode: "0600"
+    backup: true
+
+- name: Create kube artifacts dir
+  file:
+    path: "{{ artifacts_dir }}"
+    mode: "0750"
+    state: directory
+  delegate_to: localhost
+  connection: local
+  become: false
+  run_once: true
+  when: kubeconfig_localhost
+
+- name: Wait for k8s apiserver
+  wait_for:
+    host: "{{ kube_apiserver_access_address }}"
+    port: "{{ kube_apiserver_port }}"
+    timeout: 180
+
+- name: Get admin kubeconfig from remote host
+  slurp:
+    src: "{{ kube_config_dir }}/admin.conf"
+  run_once: true
+  register: raw_admin_kubeconfig
+  when: kubeconfig_localhost
+
+- name: Convert kubeconfig to YAML
+  set_fact:
+    admin_kubeconfig: "{{ raw_admin_kubeconfig.content | b64decode | from_yaml }}"
+  when: kubeconfig_localhost
+
+- name: Override username in kubeconfig
+  set_fact:
+    final_admin_kubeconfig: "{{ admin_kubeconfig | combine(override_cluster_name, recursive=true) | combine(override_context, recursive=true) | combine(override_user, recursive=true) }}"
+  vars:
+    cluster_infos: "{{ admin_kubeconfig['clusters'][0]['cluster'] }}"
+    user_certs: "{{ admin_kubeconfig['users'][0]['user'] }}"
+    username: "kubernetes-admin-{{ cluster_name }}"
+    context: "kubernetes-admin-{{ cluster_name }}@{{ cluster_name }}"
+    override_cluster_name: "{{ {'clusters': [{'cluster': (cluster_infos | combine({'server': 'https://' + (external_apiserver_address | ansible.utils.ipwrap) + ':' + (external_apiserver_port | string)})), 'name': cluster_name}]} }}"
+    override_context: "{{ {'contexts': [{'context': {'user': username, 'cluster': cluster_name}, 'name': context}], 'current-context': context} }}"
+    override_user: "{{ {'users': [{'name': username, 'user': user_certs}]} }}"
+  when: kubeconfig_localhost
+
+- name: Write admin kubeconfig on ansible host
+  copy:
+    content: "{{ final_admin_kubeconfig | to_nice_yaml(indent=2) }}"
+    dest: "{{ artifacts_dir }}/admin.conf"
+    mode: "0600"
+  delegate_to: localhost
+  connection: local
+  become: false
+  run_once: true
+  when: kubeconfig_localhost
+
+- name: Copy kubectl binary to ansible host
+  fetch:
+    src: "{{ bin_dir }}/kubectl"
+    dest: "{{ artifacts_dir }}/kubectl"
+    flat: true
+    validate_checksum: false
+  register: copy_binary_result
+  until: copy_binary_result is not failed
+  retries: 20
+  become: false
+  run_once: true
+  when: kubectl_localhost
+
+- name: Create helper script kubectl.sh on ansible host
+  copy:
+    content: |
+      #!/bin/bash
+      ${BASH_SOURCE%/*}/kubectl --kubeconfig=${BASH_SOURCE%/*}/admin.conf "$@"
+    dest: "{{ artifacts_dir }}/kubectl.sh"
+    mode: "0755"
+  become: false
+  run_once: true
+  delegate_to: localhost
+  connection: local
+  when: kubectl_localhost and kubeconfig_localhost

internal/resources/kubespray/project/roles/kubernetes/control-plane/defaults/main/etcd.yml

@@ -0,0 +1,29 @@
+---
+# Set etcd user/group
+etcd_owner: etcd
+
+# Note: This does not set up DNS entries. It simply adds the following DNS
+# entries to the certificate
+etcd_cert_alt_names:
+  - "etcd.kube-system.svc.{{ dns_domain }}"
+  - "etcd.kube-system.svc"
+  - "etcd.kube-system"
+  - "etcd"
+etcd_cert_alt_ips: []
+
+etcd_heartbeat_interval: "250"
+etcd_election_timeout: "5000"
+
+etcd_metrics: "basic"
+
+## A dictionary of extra environment variables to add to etcd.env, formatted like:
+##  etcd_extra_vars:
+##    var1: "value1"
+##    var2: "value2"
+## Note this is different from the etcd role with ETCD_ prfexi, caps, and underscores
+etcd_extra_vars: {}
+
+# etcd_quota_backend_bytes: "2147483648"
+# etcd_max_request_bytes: "1572864"
+
+etcd_compaction_retention: "8"

internal/resources/kubespray/project/roles/kubernetes/control-plane/defaults/main/kube-proxy.yml

@@ -0,0 +1,114 @@
+---
+# bind address for kube-proxy
+kube_proxy_bind_address: '0.0.0.0'
+
+# acceptContentTypes defines the Accept header sent by clients when connecting to a server, overriding the
+# default value of 'application/json'. This field will control all connections to the server used by a particular
+# client.
+kube_proxy_client_accept_content_types: ''
+
+# burst allows extra queries to accumulate when a client is exceeding its rate.
+kube_proxy_client_burst: 10
+
+# contentType is the content type used when sending data to the server from this client.
+kube_proxy_client_content_type: application/vnd.kubernetes.protobuf
+
+# kubeconfig is the path to a KubeConfig file.
+# Leave as empty string to generate from other fields
+kube_proxy_client_kubeconfig: ''
+
+# qps controls the number of queries per second allowed for this connection.
+kube_proxy_client_qps: 5
+
+# How often configuration from the apiserver is refreshed. Must be greater than 0.
+kube_proxy_config_sync_period: 15m0s
+
+### Conntrack
+# maxPerCore is the maximum number of NAT connections to track
+# per CPU core (0 to leave the limit as-is and ignore min).
+kube_proxy_conntrack_max_per_core: 32768
+
+# min is the minimum value of connect-tracking records to allocate,
+# regardless of conntrackMaxPerCore (set maxPerCore=0 to leave the limit as-is).
+kube_proxy_conntrack_min: 131072
+
+# tcpCloseWaitTimeout is how long an idle conntrack entry
+# in CLOSE_WAIT state will remain in the conntrack
+# table. (e.g. '60s'). Must be greater than 0 to set.
+kube_proxy_conntrack_tcp_close_wait_timeout: 1h0m0s
+
+# tcpEstablishedTimeout is how long an idle TCP connection will be kept open
+# (e.g. '2s').  Must be greater than 0 to set.
+kube_proxy_conntrack_tcp_established_timeout: 24h0m0s
+
+# Enables profiling via web interface on /debug/pprof handler.
+# Profiling handlers will be handled by metrics server.
+kube_proxy_enable_profiling: false
+
+# bind address for kube-proxy health check
+kube_proxy_healthz_bind_address: 0.0.0.0:10256
+
+# If using the pure iptables proxy, SNAT everything. Note that it breaks any
+# policy engine.
+kube_proxy_masquerade_all: false
+
+# If using the pure iptables proxy, the bit of the fwmark space to mark packets requiring SNAT with.
+# Must be within the range [0, 31].
+kube_proxy_masquerade_bit: 14
+
+# The minimum interval of how often the iptables or ipvs rules can be refreshed as
+# endpoints and services change (e.g. '5s', '1m', '2h22m').
+kube_proxy_min_sync_period: 0s
+
+# The maximum interval of how often iptables or ipvs rules are refreshed (e.g. '5s', '1m', '2h22m').
+# Must be greater than 0.
+kube_proxy_sync_period: 30s
+
+# A comma-separated list of CIDR's which the ipvs proxier should not touch when cleaning up IPVS rules.
+kube_proxy_exclude_cidrs: []
+
+# The ipvs scheduler type when proxy mode is ipvs
+# rr: round-robin
+# lc: least connection
+# dh: destination hashing
+# sh: source hashing
+# sed: shortest expected delay
+# nq: never queue
+kube_proxy_scheduler: rr
+
+# configure arp_ignore and arp_announce to avoid answering ARP queries from kube-ipvs0 interface
+# must be set to true for MetalLB, kube-vip(ARP enabled) to work
+kube_proxy_strict_arp: false
+
+# kube_proxy_tcp_timeout is the timeout value used for idle IPVS TCP sessions.
+# The default value is 0, which preserves the current timeout value on the system.
+kube_proxy_tcp_timeout: 0s
+
+# kube_proxy_tcp_fin_timeout is the timeout value used for IPVS TCP sessions after receiving a FIN.
+# The default value is 0, which preserves the current timeout value on the system.
+kube_proxy_tcp_fin_timeout: 0s
+
+# kube_proxy_udp_timeout is the timeout value used for IPVS UDP packets.
+# The default value is 0, which preserves the current timeout value on the system.
+kube_proxy_udp_timeout: 0s
+
+# The IP address and port for the metrics server to serve on
+# (set to 0.0.0.0 for all IPv4 interfaces and `::` for all IPv6 interfaces)
+kube_proxy_metrics_bind_address: 127.0.0.1:10249
+
+# A string slice of values which specify the addresses to use for NodePorts.
+# Values may be valid IP blocks (e.g. 1.2.3.0/24, 1.2.3.4/32).
+# The default empty string slice ([]) means to use all local addresses.
+kube_proxy_nodeport_addresses: >-
+  {%- if kube_proxy_nodeport_addresses_cidr is defined -%}
+  [{{ kube_proxy_nodeport_addresses_cidr }}]
+  {%- else -%}
+  []
+  {%- endif -%}
+
+# oom-score-adj value for kube-proxy process. Values must be within the range [-1000, 1000]
+kube_proxy_oom_score_adj: -999
+
+# portRange is the range of host ports (beginPort-endPort, inclusive) that may be consumed
+# in order to proxy service traffic. If unspecified, 0, or (0-0) then ports will be randomly chosen.
+kube_proxy_port_range: ''

internal/resources/kubespray/project/roles/kubernetes/control-plane/defaults/main/kube-scheduler.yml

@@ -0,0 +1,33 @@
+---
+# Extra args passed by kubeadm
+kube_kubeadm_scheduler_extra_args: {}
+
+# Associated interface must be reachable by the rest of the cluster, and by
+# CLI/web clients.
+kube_scheduler_bind_address: "::"
+
+# ClientConnection options (e.g. Burst, QPS) except from kubeconfig.
+kube_scheduler_client_conn_extra_opts: {}
+
+# Additional KubeSchedulerConfiguration settings (e.g. metricsBindAddress).
+kube_scheduler_config_extra_opts: {}
+
+# List of scheduler extenders (dicts), each holding the values of how to
+# communicate with the extender.
+kube_scheduler_extenders: []
+
+# Leader Election options (e.g. ResourceName, RetryPerion) except from
+# LeaseDuration and Renew deadline which are defined in following vars.
+kube_scheduler_leader_elect_extra_opts: {}
+
+# Leader election lease duration
+kube_scheduler_leader_elect_lease_duration: 15s
+
+# Leader election lease timeout
+kube_scheduler_leader_elect_renew_deadline: 10s
+
+# Lisf of scheduling profiles (ditcs) supported by kube-scheduler
+kube_scheduler_profiles: []
+
+# Extra volume mounts
+scheduler_extra_volumes: {}

internal/resources/kubespray/project/roles/kubernetes/control-plane/defaults/main/main.yml

@@ -0,0 +1,257 @@
+---
+# disable upgrade cluster
+upgrade_cluster_setup: false
+
+# By default the external API listens on all interfaces, this can be changed to
+# listen on a specific address/interface.
+# NOTE: If you specific address/interface and use loadbalancer_apiserver_localhost
+# loadbalancer_apiserver_localhost (nginx/haproxy) will deploy on control plane nodes on 127.0.0.1:{{ loadbalancer_apiserver_port | default(kube_apiserver_port) }} too.
+kube_apiserver_bind_address: "::"
+
+# A port range to reserve for services with NodePort visibility.
+# Inclusive at both ends of the range.
+kube_apiserver_node_port_range: "30000-32767"
+
+# ETCD backend for k8s data
+kube_apiserver_storage_backend: etcd3
+
+# The interval of compaction requests. If 0, the compaction request from apiserver is disabled.
+kube_apiserver_etcd_compaction_interval: "5m0s"
+
+# CIS 1.2.26
+# Validate that the service account token
+# in the request is actually present in etcd.
+kube_apiserver_service_account_lookup: true
+
+kube_etcd_cacert_file: ca.pem
+kube_etcd_cert_file: node-{{ inventory_hostname }}.pem
+kube_etcd_key_file: node-{{ inventory_hostname }}-key.pem
+
+# Associated interfaces must be reachable by the rest of the cluster, and by
+# CLI/web clients.
+kube_controller_manager_bind_address: "::"
+
+# Leader election lease durations and timeouts for controller-manager
+kube_controller_manager_leader_elect_lease_duration: 15s
+kube_controller_manager_leader_elect_renew_deadline: 10s
+
+# discovery_timeout modifies the discovery timeout
+discovery_timeout: 5m0s
+
+# Instruct first control plane node to refresh kubeadm token
+kubeadm_refresh_token: true
+
+# Scale down coredns replicas to 0 if not using coredns dns_mode
+kubeadm_scale_down_coredns_enabled: true
+
+# audit support
+kubernetes_audit: false
+# path to audit log file
+audit_log_path: /var/log/audit/kube-apiserver-audit.log
+# num days
+audit_log_maxage: 30
+# the num of audit logs to retain
+audit_log_maxbackups: 10
+# the max size in MB to retain
+audit_log_maxsize: 100
+# policy file
+audit_policy_file: "{{ kube_config_dir }}/audit-policy/apiserver-audit-policy.yaml"
+# custom audit policy rules (to replace the default ones)
+# audit_policy_custom_rules: |
+#   - level: None
+#     users: []
+#     verbs: []
+#     resources: []
+
+# audit log hostpath
+audit_log_name: audit-logs
+audit_log_hostpath: /var/log/kubernetes/audit
+audit_log_mountpath: "{{ audit_log_path | dirname }}"
+
+# audit policy hostpath
+audit_policy_name: audit-policy
+audit_policy_hostpath: "{{ audit_policy_file | dirname }}"
+audit_policy_mountpath: "{{ audit_policy_hostpath }}"
+
+# audit webhook support
+kubernetes_audit_webhook: false
+
+# path to audit webhook config file
+audit_webhook_config_file: "{{ kube_config_dir }}/audit-policy/apiserver-audit-webhook-config.yaml"
+audit_webhook_server_url: "https://audit.app"
+audit_webhook_server_extra_args: {}
+audit_webhook_mode: batch
+audit_webhook_batch_max_size: 100
+audit_webhook_batch_max_wait: 1s
+
+kube_controller_node_monitor_grace_period: 40s
+kube_controller_node_monitor_period: 5s
+kube_controller_terminated_pod_gc_threshold: 12500
+kube_apiserver_request_timeout: "1m0s"
+kube_apiserver_pod_eviction_not_ready_timeout_seconds: "300"
+kube_apiserver_pod_eviction_unreachable_timeout_seconds: "300"
+
+# 1.10+ admission plugins
+kube_apiserver_enable_admission_plugins: []
+
+# enable admission plugins configuration
+kube_apiserver_admission_control_config_file: false
+
+# data structure to configure EventRateLimit admission plugin
+# this should have the following structure:
+# kube_apiserver_admission_event_rate_limits:
+# <limit_name>:
+#   type: <limit_type>
+#   qps: <qps_value>
+#   burst: <burst_value>
+#   cache_size: <cache_size_value>
+kube_apiserver_admission_event_rate_limits: {}
+
+## PodSecurityAdmission plugin configuration
+kube_pod_security_use_default: false
+kube_pod_security_default_enforce: baseline
+kube_pod_security_default_enforce_version: "v{{ kube_major_version }}"
+kube_pod_security_default_audit: restricted
+kube_pod_security_default_audit_version: "v{{ kube_major_version }}"
+kube_pod_security_default_warn: restricted
+kube_pod_security_default_warn_version: "v{{ kube_major_version }}"
+kube_pod_security_exemptions_usernames: []
+kube_pod_security_exemptions_runtime_class_names: []
+kube_pod_security_exemptions_namespaces:
+  - kube-system
+
+## ResourceQuota plugin configuration
+## Resources that ResourceQuota should limit by default if no quota exists
+## Example below enforces quota on all storage classes
+# kube_resource_quota_limited_resources:
+# - apiGroup: ""
+#   resource: persistentvolumeclaims
+#   matchContains:
+#   - .storageclass.storage.k8s.io/requests.storage
+kube_resource_quota_limited_resources: []
+
+# 1.10+ list of disabled admission plugins
+kube_apiserver_disable_admission_plugins: []
+
+# extra runtime config
+kube_api_runtime_config: []
+
+## Enable/Disable Kube API Server Authentication Methods
+kube_token_auth: false
+kube_oidc_auth: false
+
+## Variables for webhook token auth https://kubernetes.io/docs/reference/access-authn-authz/authentication/#webhook-token-authentication
+kube_webhook_token_auth: false
+kube_webhook_token_auth_url_skip_tls_verify: false
+# kube_webhook_token_auth_url: https://...
+## base64-encoded string of the webhook's CA certificate
+# kube_webhook_token_auth_ca_data: "LS0t..."
+
+## Variables for webhook token authz https://kubernetes.io/docs/reference/access-authn-authz/webhook/
+# kube_webhook_authorization_url: https://...
+kube_webhook_authorization: false
+kube_webhook_authorization_url_skip_tls_verify: false
+
+# Default podnodeselector
+kube_apiserver_admission_plugins_podnodeselector_default_node_selector: ""
+
+## Variables for OpenID Connect Configuration https://kubernetes.io/docs/admin/authentication/
+## To use OpenID you have to deploy additional an OpenID Provider (e.g Dex, Keycloak, ...)
+
+# kube_oidc_url: https:// ...
+# kube_oidc_client_id: kubernetes
+## Optional settings for OIDC
+# kube_oidc_username_claim: sub
+# kube_oidc_username_prefix: 'oidc:'
+# kube_oidc_groups_claim: groups
+# kube_oidc_groups_prefix: 'oidc:'
+# Copy oidc CA file to the following path if needed
+# kube_oidc_ca_file: {{ kube_cert_dir }}/ca.pem
+# Optionally include a base64-encoded oidc CA cert
+# kube_oidc_ca_cert: c3RhY2thYnVzZS5jb20...
+
+# List of the preferred NodeAddressTypes to use for kubelet connections.
+kubelet_preferred_address_types: 'InternalDNS,InternalIP,Hostname,ExternalDNS,ExternalIP'
+
+## Extra args for k8s components passing by kubeadm
+kube_kubeadm_apiserver_extra_args: {}
+kube_kubeadm_controller_extra_args: {}
+
+## Extra control plane host volume mounts
+## Example:
+# apiserver_extra_volumes:
+#  - name: name
+#    hostPath: /host/path
+#    mountPath: /mount/path
+#    readOnly: true
+apiserver_extra_volumes: {}
+controller_manager_extra_volumes: {}
+
+## Encrypting Secret Data at Rest
+kube_encrypt_secret_data: false
+kube_encrypt_token: "{{ lookup('password', credentials_dir + '/kube_encrypt_token.creds length=32 chars=ascii_letters,digits') }}"
+# Must be either: aescbc, secretbox or aesgcm
+kube_encryption_algorithm: "secretbox"
+# Which kubernetes resources to encrypt
+kube_encryption_resources: [secrets]
+
+secrets_encryption_query: "resources[*].providers[0].{{ kube_encryption_algorithm }}.keys[0].secret"
+
+## Support tls min version, Possible values: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13.
+# tls_min_version: ""
+
+## Support tls cipher suites.
+# tls_cipher_suites:
+#   - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA
+#   - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256
+#   - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256
+#   - TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA
+#   - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
+#   - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256
+#   - TLS_ECDHE_ECDSA_WITH_RC4_128_SHA
+#   - TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA
+#   - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA
+#   - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256
+#   - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256
+#   - TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA
+#   - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384
+#   - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256
+#   - TLS_ECDHE_RSA_WITH_RC4_128_SHA
+#   - TLS_RSA_WITH_3DES_EDE_CBC_SHA
+#   - TLS_RSA_WITH_AES_128_CBC_SHA
+#   - TLS_RSA_WITH_AES_128_CBC_SHA256
+#   - TLS_RSA_WITH_AES_128_GCM_SHA256
+#   - TLS_RSA_WITH_AES_256_CBC_SHA
+#   - TLS_RSA_WITH_AES_256_GCM_SHA384
+#   - TLS_RSA_WITH_RC4_128_SHA
+
+## Amount of time to retain events. (default 1h0m0s)
+event_ttl_duration: "1h0m0s"
+
+## Automatically renew K8S control plane certificates on first Monday of each month
+auto_renew_certificates: false
+# First Monday of each month
+auto_renew_certificates_systemd_calendar: "Mon *-*-1,2,3,4,5,6,7 03:00:00"
+# kubeadm renews all the certificates during control plane upgrade.
+# If we have requirement like without renewing certs upgrade the cluster,
+# we can opt out from the default behavior by setting kubeadm_upgrade_auto_cert_renewal to false
+kubeadm_upgrade_auto_cert_renewal: true
+
+# Bash alias of kubectl to interact with Kubernetes cluster much easier
+# kubectl_alias: k
+
+## Enable distributed tracing for kube-apiserver
+kube_apiserver_tracing: false
+kube_apiserver_tracing_endpoint: "[::]:4317"
+kube_apiserver_tracing_sampling_rate_per_million: 100
+
+# Enable kubeadm file discovery if anonymous access has been removed
+kubeadm_use_file_discovery: "{{ remove_anonymous_access }}"
+
+# imagePullSerial specifies if image pulling performed by kubeadm must be done serially or in parallel. Default: true
+kubeadm_image_pull_serial: true
+
+# Supported asymmetric encryption algorithm types for the cluster's keys and certificates.
+# can be one of RSA-2048(default), RSA-3072, RSA-4096, ECDSA-P256
+# ref: https://kubernetes.io/docs/reference/config-api/kubeadm-config.v1beta4/#kubeadm-k8s-io-v1beta4-ClusterConfiguration
+kube_asymmetric_encryption_algorithm: "RSA-2048"

internal/resources/kubespray/project/roles/kubernetes/control-plane/handlers/main.yml

@@ -0,0 +1,117 @@
+---
+- name: Control plane | reload systemd
+  systemd_service:
+    daemon_reload: true
+  listen: Control plane | restart kubelet
+
+- name: Control plane | reload kubelet
+  service:
+    name: kubelet
+    state: restarted
+  listen: Control plane | restart kubelet
+
+- name: Control plane | Remove apiserver container docker
+  shell: "set -o pipefail && docker ps -af name=k8s_kube-apiserver* -q | xargs --no-run-if-empty docker rm -f"
+  args:
+    executable: /bin/bash
+  register: remove_apiserver_container
+  retries: 10
+  until: remove_apiserver_container.rc == 0
+  delay: 1
+  when: container_manager == "docker"
+  listen: Control plane | Restart apiserver
+
+- name: Control plane | Remove apiserver container containerd/crio
+  shell: "set -o pipefail && {{ bin_dir }}/crictl pods --name 'kube-apiserver*' -q | xargs -I% --no-run-if-empty bash -c '{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %'"
+  args:
+    executable: /bin/bash
+  register: remove_apiserver_container
+  retries: 10
+  until: remove_apiserver_container.rc == 0
+  delay: 1
+  when: container_manager in ['containerd', 'crio']
+  listen: Control plane | Restart apiserver
+
+- name: Control plane | Remove scheduler container docker
+  shell: "set -o pipefail && {{ docker_bin_dir }}/docker ps -af name=k8s_kube-scheduler* -q | xargs --no-run-if-empty {{ docker_bin_dir }}/docker rm -f"
+  args:
+    executable: /bin/bash
+  register: remove_scheduler_container
+  retries: 10
+  until: remove_scheduler_container.rc == 0
+  delay: 1
+  when: container_manager == "docker"
+  listen: Control plane | Restart kube-scheduler
+
+- name: Control plane | Remove scheduler container containerd/crio
+  shell: "set -o pipefail && {{ bin_dir }}/crictl pods --name 'kube-scheduler*' -q | xargs -I% --no-run-if-empty bash -c '{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %'"
+  args:
+    executable: /bin/bash
+  register: remove_scheduler_container
+  retries: 10
+  until: remove_scheduler_container.rc == 0
+  delay: 1
+  when: container_manager in ['containerd', 'crio']
+  listen: Control plane | Restart kube-scheduler
+
+- name: Control plane | Remove controller manager container docker
+  shell: "set -o pipefail && {{ docker_bin_dir }}/docker ps -af name=k8s_kube-controller-manager* -q | xargs --no-run-if-empty {{ docker_bin_dir }}/docker rm -f"
+  args:
+    executable: /bin/bash
+  register: remove_cm_container
+  retries: 10
+  until: remove_cm_container.rc == 0
+  delay: 1
+  when: container_manager == "docker"
+  listen: Control plane | Restart kube-controller-manager
+
+- name: Control plane | Remove controller manager container containerd/crio
+  shell: "set -o pipefail && {{ bin_dir }}/crictl pods --name 'kube-controller-manager*' -q | xargs -I% --no-run-if-empty bash -c '{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %'"
+  args:
+    executable: /bin/bash
+  register: remove_cm_container
+  retries: 10
+  until: remove_cm_container.rc == 0
+  delay: 1
+  when: container_manager in ['containerd', 'crio']
+  listen: Control plane | Restart kube-controller-manager
+
+- name: Control plane | wait for kube-scheduler
+  vars:
+    endpoint: "{{ kube_scheduler_bind_address if kube_scheduler_bind_address != '::' else 'localhost' }}"
+  uri:
+    url: https://{{ endpoint }}:10259/healthz
+    validate_certs: false
+  register: scheduler_result
+  until: scheduler_result.status == 200
+  retries: 60
+  delay: 1
+  listen:
+    - Control plane | restart kubelet
+    - Control plane | Restart kube-scheduler
+
+- name: Control plane | wait for kube-controller-manager
+  vars:
+    endpoint: "{{ kube_controller_manager_bind_address if kube_controller_manager_bind_address != '::' else 'localhost' }}"
+  uri:
+    url: https://{{ endpoint }}:10257/healthz
+    validate_certs: false
+  register: controller_manager_result
+  until: controller_manager_result.status == 200
+  retries: 60
+  delay: 1
+  listen:
+    - Control plane | restart kubelet
+    - Control plane | Restart kube-controller-manager
+
+- name: Control plane | wait for the apiserver to be running
+  uri:
+    url: "{{ kube_apiserver_endpoint }}/healthz"
+    validate_certs: false
+  register: result
+  until: result.status == 200
+  retries: 60
+  delay: 1
+  listen:
+    - Control plane | restart kubelet
+    - Control plane | Restart apiserver

internal/resources/kubespray/project/roles/kubernetes/control-plane/meta/main.yml

@@ -0,0 +1,9 @@
+---
+dependencies:
+  - role: kubernetes/kubeadm_common
+  - role: adduser
+    user: "{{ addusers.etcd }}"
+    when:
+      - etcd_deployment_type == "kubeadm"
+      - not (ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk", "ClearLinux"] or is_fedora_coreos)
+  - role: network_plugin/calico_defaults

internal/resources/kubespray/project/roles/kubernetes/control-plane/tasks/check-api.yml

@@ -0,0 +1,10 @@
+---
+- name: Kubeadm | Check api is up
+  uri:
+    url: "https://{{ ip | default(fallback_ip) }}:{{ kube_apiserver_port }}/healthz"
+    validate_certs: false
+  when: ('kube_control_plane' in group_names)
+  register: _result
+  retries: 60
+  delay: 5
+  until: _result.status == 200

internal/resources/kubespray/project/roles/kubernetes/control-plane/tasks/define-first-kube-control.yml

@@ -0,0 +1,19 @@
+---
+
+- name: Check which kube-control nodes are already members of the cluster
+  command: "{{ bin_dir }}/kubectl get nodes --selector=node-role.kubernetes.io/control-plane -o json"
+  register: kube_control_planes_raw
+  ignore_errors: true
+  changed_when: false
+
+- name: Set fact joined_control_planes
+  set_fact:
+    joined_control_planes: "{{ ((kube_control_planes_raw.stdout | from_json)['items']) | default([]) | map(attribute='metadata') | map(attribute='name') | list }}"
+  delegate_to: "{{ item }}"
+  loop: "{{ groups['kube_control_plane'] }}"
+  when: kube_control_planes_raw is succeeded
+  run_once: true
+
+- name: Set fact first_kube_control_plane
+  set_fact:
+    first_kube_control_plane: "{{ joined_control_planes | default([]) | first | default(groups['kube_control_plane'] | first) }}"

internal/resources/kubespray/project/roles/kubernetes/control-plane/tasks/encrypt-at-rest.yml

@@ -0,0 +1,40 @@
+---
+- name: Check if secret for encrypting data at rest already exist
+  stat:
+    path: "{{ kube_cert_dir }}/secrets_encryption.yaml"
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: secrets_encryption_file
+
+- name: Slurp secrets_encryption file if it exists
+  slurp:
+    src: "{{ kube_cert_dir }}/secrets_encryption.yaml"
+  register: secret_file_encoded
+  when: secrets_encryption_file.stat.exists
+
+- name: Base 64 Decode slurped secrets_encryption.yaml file
+  set_fact:
+    secret_file_decoded: "{{ secret_file_encoded['content'] | b64decode | from_yaml }}"
+  when: secrets_encryption_file.stat.exists
+
+- name: Extract secret value from secrets_encryption.yaml
+  set_fact:
+    kube_encrypt_token_extracted: "{{ secret_file_decoded | json_query(secrets_encryption_query) | first | b64decode }}"
+  when: secrets_encryption_file.stat.exists
+
+- name: Set kube_encrypt_token across control plane nodes
+  set_fact:
+    kube_encrypt_token: "{{ kube_encrypt_token_extracted }}"
+  delegate_to: "{{ item }}"
+  delegate_facts: true
+  with_inventory_hostnames: kube_control_plane
+  when: kube_encrypt_token_extracted is defined
+
+- name: Write secrets for encrypting secret data at rest
+  template:
+    src: secrets_encryption.yaml.j2
+    dest: "{{ kube_cert_dir }}/secrets_encryption.yaml"
+    owner: root
+    group: "{{ kube_cert_group }}"
+    mode: "0640"

internal/resources/kubespray/project/roles/kubernetes/control-plane/tasks/kubeadm-backup.yml

@@ -0,0 +1,28 @@
+---
+- name: Backup old certs and keys
+  copy:
+    src: "{{ kube_cert_dir }}/{{ item }}"
+    dest: "{{ kube_cert_dir }}/{{ item }}.old"
+    mode: preserve
+    remote_src: true
+  with_items:
+    - apiserver.crt
+    - apiserver.key
+    - apiserver-kubelet-client.crt
+    - apiserver-kubelet-client.key
+    - front-proxy-client.crt
+    - front-proxy-client.key
+  ignore_errors: true  # noqa ignore-errors
+
+- name: Backup old confs
+  copy:
+    src: "{{ kube_config_dir }}/{{ item }}"
+    dest: "{{ kube_config_dir }}/{{ item }}.old"
+    mode: preserve
+    remote_src: true
+  with_items:
+    - admin.conf
+    - controller-manager.conf
+    - kubelet.conf
+    - scheduler.conf
+  ignore_errors: true  # noqa ignore-errors

internal/resources/kubespray/project/roles/kubernetes/control-plane/tasks/kubeadm-etcd.yml

@@ -0,0 +1,29 @@
+---
+- name: Calculate etcd cert serial
+  command: "openssl x509 -in {{ kube_cert_dir }}/apiserver-etcd-client.crt -noout -serial"
+  register: "etcd_client_cert_serial_result"
+  changed_when: false
+  tags:
+    - network
+
+- name: Set etcd_client_cert_serial
+  set_fact:
+    etcd_client_cert_serial: "{{ etcd_client_cert_serial_result.stdout.split('=')[1] }}"
+  tags:
+    - network
+
+- name: Ensure etcdctl and etcdutl script is installed
+  import_role:
+    name: etcdctl_etcdutl
+  when: etcd_deployment_type == "kubeadm"
+  tags:
+    - etcdctl
+    - etcdutl
+
+- name: Set ownership for etcd data directory
+  file:
+    path: "{{ etcd_data_dir }}"
+    owner: "{{ etcd_owner }}"
+    group: "{{ etcd_owner }}"
+    mode: "0700"
+  when: etcd_deployment_type == "kubeadm"

internal/resources/kubespray/project/roles/kubernetes/control-plane/tasks/kubeadm-fix-apiserver.yml

@@ -0,0 +1,17 @@
+---
+
+- name: Update server field in component kubeconfigs
+  lineinfile:
+    dest: "{{ kube_config_dir }}/{{ item }}"
+    regexp: '^    server: https'
+    line: '    server: {{ kube_apiserver_endpoint }}'
+    backup: true
+  with_items:
+    - admin.conf
+    - controller-manager.conf
+    - kubelet.conf
+    - scheduler.conf
+  notify:
+    - "Control plane | Restart kube-controller-manager"
+    - "Control plane | Restart kube-scheduler"
+    - "Control plane | reload kubelet"

internal/resources/kubespray/project/roles/kubernetes/control-plane/tasks/kubeadm-secondary.yml

@@ -0,0 +1,101 @@
+---
+- name: Set kubeadm_discovery_address
+  set_fact:
+    # noqa: jinja[spacing]
+    kubeadm_discovery_address: >-
+      {%- if "127.0.0.1" in kube_apiserver_endpoint or "localhost" in kube_apiserver_endpoint -%}
+      {{ first_kube_control_plane_address | ansible.utils.ipwrap }}:{{ kube_apiserver_port }}
+      {%- else -%}
+      {{ kube_apiserver_endpoint | regex_replace('https://', '') }}
+      {%- endif %}
+  tags:
+    - facts
+
+- name: Upload certificates so they are fresh and not expired
+  command: >-
+    {{ bin_dir }}/kubeadm init phase
+    --config {{ kube_config_dir }}/kubeadm-config.yaml
+    upload-certs
+    --upload-certs
+  register: kubeadm_upload_cert
+  when:
+    - inventory_hostname == first_kube_control_plane
+    - not kube_external_ca_mode
+
+- name: Parse certificate key if not set
+  set_fact:
+    kubeadm_certificate_key: "{{ hostvars[first_kube_control_plane]['kubeadm_upload_cert'].stdout_lines[-1] | trim }}"
+  run_once: true
+  when:
+    - hostvars[first_kube_control_plane]['kubeadm_upload_cert'] is defined
+    - hostvars[first_kube_control_plane]['kubeadm_upload_cert'] is not skipped
+
+- name: Create kubeadm ControlPlane config
+  template:
+    src: "kubeadm-controlplane.yaml.j2"
+    dest: "{{ kube_config_dir }}/kubeadm-controlplane.yaml"
+    mode: "0640"
+    backup: true
+    validate: "{{ kubeadm_config_validate_enabled | ternary(bin_dir + '/kubeadm config validate --config %s', omit) }}"
+  when:
+    - inventory_hostname != first_kube_control_plane
+    - not kubeadm_already_run.stat.exists
+
+- name: Wait for k8s apiserver
+  wait_for:
+    host: "{{ kubeadm_discovery_address | regex_replace('\\]?:\\d+$', '') | regex_replace('^\\[', '') }}"
+    port: "{{ kubeadm_discovery_address.split(':')[-1] }}"
+    timeout: 180
+
+
+- name: Check already run
+  debug:
+    msg: "{{ kubeadm_already_run.stat.exists }}"
+
+- name: Reset cert directory
+  shell: >-
+    if [ -f /etc/kubernetes/manifests/kube-apiserver.yaml ]; then
+    {{ bin_dir }}/kubeadm reset -f --cert-dir {{ kube_cert_dir }};
+    fi
+  environment:
+    PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}"
+  when:
+    - inventory_hostname != first_kube_control_plane
+    - kubeadm_already_run is not defined or not kubeadm_already_run.stat.exists
+    - not kube_external_ca_mode
+
+- name: Get kubeconfig for join discovery process
+  command: "{{ kubectl }} -n kube-public get cm cluster-info -o jsonpath='{.data.kubeconfig}'"
+  register: kubeconfig_file_discovery
+  run_once: true
+  delegate_to: "{{ groups['kube_control_plane'] | first }}"
+  when:
+    - kubeadm_use_file_discovery
+    - kubeadm_already_run is not defined or not kubeadm_already_run.stat.exists
+
+- name: Copy discovery kubeconfig
+  copy:
+    dest: "{{ kube_config_dir }}/cluster-info-discovery-kubeconfig.yaml"
+    content: "{{ kubeconfig_file_discovery.stdout }}"
+    owner: "root"
+    mode: "0644"
+  when:
+    - inventory_hostname != first_kube_control_plane
+    - kubeadm_use_file_discovery
+    - kubeadm_already_run is not defined or not kubeadm_already_run.stat.exists
+
+- name: Joining control plane node to the cluster.
+  command: >-
+    {{ bin_dir }}/kubeadm join
+    --config {{ kube_config_dir }}/kubeadm-controlplane.yaml
+    --ignore-preflight-errors={{ kubeadm_ignore_preflight_errors | join(',') }}
+    --skip-phases={{ kubeadm_join_phases_skip | join(',') }}
+  environment:
+    PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}"
+  register: kubeadm_join_control_plane
+  retries: 3
+  throttle: 1
+  until: kubeadm_join_control_plane is succeeded
+  when:
+    - inventory_hostname != first_kube_control_plane
+    - kubeadm_already_run is not defined or not kubeadm_already_run.stat.exists

internal/resources/kubespray/project/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml

@@ -0,0 +1,260 @@
+---
+- name: Install OIDC certificate
+  copy:
+    content: "{{ kube_oidc_ca_cert | b64decode }}"
+    dest: "{{ kube_oidc_ca_file }}"
+    owner: root
+    group: root
+    mode: "0644"
+  when:
+    - kube_oidc_auth
+    - kube_oidc_ca_cert is defined
+
+- name: Kubeadm | Check if kubeadm has already run
+  stat:
+    path: "/var/lib/kubelet/config.yaml"
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: kubeadm_already_run
+
+- name: Kubeadm | Backup kubeadm certs / kubeconfig
+  import_tasks: kubeadm-backup.yml
+  when:
+    - kubeadm_already_run.stat.exists
+
+- name: Kubeadm | aggregate all SANs
+  set_fact:
+    apiserver_sans: "{{ (sans_base + groups['kube_control_plane'] + sans_lb + sans_lb_ip + sans_supp + sans_access_ip + sans_ip + sans_address + sans_override + sans_hostname + sans_fqdn + sans_kube_vip_address) | unique }}"
+  vars:
+    sans_base:
+      - "kubernetes"
+      - "kubernetes.default"
+      - "kubernetes.default.svc"
+      - "kubernetes.default.svc.{{ dns_domain }}"
+      - "{{ kube_apiserver_ip }}"
+      - "localhost"
+      - "127.0.0.1"
+      - "::1"
+    sans_lb: "{{ [apiserver_loadbalancer_domain_name] if apiserver_loadbalancer_domain_name is defined else [] }}"
+    sans_lb_ip: "{{ [loadbalancer_apiserver.address] if loadbalancer_apiserver is defined and loadbalancer_apiserver.address is defined else [] }}"
+    sans_supp: "{{ supplementary_addresses_in_ssl_keys if supplementary_addresses_in_ssl_keys is defined else [] }}"
+    sans_access_ip: "{{ groups['kube_control_plane'] | map('extract', hostvars, 'main_access_ip') | list | select('defined') | list }}"
+    sans_ip: "{{ groups['kube_control_plane'] | map('extract', hostvars, 'main_ip') | list | select('defined') | list }}"
+    sans_address: "{{ groups['kube_control_plane'] | map('extract', hostvars, ['ansible_default_ipv6', 'ansible_default_ipv4', 'address']) | list | select('defined') | list }}"
+    sans_override: "{{ [kube_override_hostname] if kube_override_hostname else [] }}"
+    sans_hostname: "{{ groups['kube_control_plane'] | map('extract', hostvars, ['ansible_hostname']) | list | select('defined') | list }}"
+    sans_fqdn: "{{ groups['kube_control_plane'] | map('extract', hostvars, ['ansible_fqdn']) | list | select('defined') | list }}"
+    sans_kube_vip_address: "{{ [kube_vip_address] if kube_vip_address is defined and kube_vip_address else [] }}"
+  tags: facts
+
+- name: Create audit-policy directory
+  file:
+    path: "{{ audit_policy_file | dirname }}"
+    state: directory
+    mode: "0640"
+  when: kubernetes_audit or kubernetes_audit_webhook
+
+- name: Write api audit policy yaml
+  template:
+    src: apiserver-audit-policy.yaml.j2
+    dest: "{{ audit_policy_file }}"
+    mode: "0640"
+  when: kubernetes_audit or kubernetes_audit_webhook
+  notify: Control plane | Restart apiserver
+
+- name: Write api audit webhook config yaml
+  template:
+    src: apiserver-audit-webhook-config.yaml.j2
+    dest: "{{ audit_webhook_config_file }}"
+    mode: "0640"
+  when: kubernetes_audit_webhook
+  notify: Control plane | Restart apiserver
+
+- name: Create apiserver tracing config directory
+  file:
+    path: "{{ kube_config_dir }}/tracing"
+    state: directory
+    mode: "0640"
+  when: kube_apiserver_tracing
+
+- name: Write apiserver tracing config yaml
+  template:
+    src: apiserver-tracing.yaml.j2
+    dest: "{{ kube_config_dir }}/tracing/apiserver-tracing.yaml"
+    mode: "0640"
+  when: kube_apiserver_tracing
+  notify: Control plane | Restart apiserver
+
+# Nginx LB(default), If kubeadm_config_api_fqdn is defined, use other LB by kubeadm controlPlaneEndpoint.
+- name: Set kubeadm_config_api_fqdn define
+  set_fact:
+    kubeadm_config_api_fqdn: "{{ apiserver_loadbalancer_domain_name | default('lb-apiserver.kubernetes.local') }}"
+  when: loadbalancer_apiserver is defined
+
+- name: Kubeadm | Create kubeadm config
+  template:
+    src: "kubeadm-config.{{ kubeadm_config_api_version }}.yaml.j2"
+    dest: "{{ kube_config_dir }}/kubeadm-config.yaml"
+    mode: "0640"
+    validate: "{{ kubeadm_config_validate_enabled | ternary(bin_dir + '/kubeadm config validate --config %s', omit) }}"
+
+- name: Kubeadm | Create directory to store admission control configurations
+  file:
+    path: "{{ kube_config_dir }}/admission-controls"
+    state: directory
+    mode: "0640"
+  when: kube_apiserver_admission_control_config_file
+
+- name: Kubeadm | Push admission control config file
+  template:
+    src: "admission-controls.yaml.j2"
+    dest: "{{ kube_config_dir }}/admission-controls/admission-controls.yaml"
+    mode: "0640"
+  when: kube_apiserver_admission_control_config_file
+  notify: Control plane | Restart apiserver
+
+- name: Kubeadm | Push admission control config files
+  template:
+    src: "{{ item | lower }}.yaml.j2"
+    dest: "{{ kube_config_dir }}/admission-controls/{{ item | lower }}.yaml"
+    mode: "0640"
+  when:
+    - kube_apiserver_admission_control_config_file
+    - item in kube_apiserver_admission_plugins_needs_configuration
+  loop: "{{ kube_apiserver_enable_admission_plugins }}"
+  notify: Control plane | Restart apiserver
+
+- name: Kubeadm | Check apiserver.crt SANs
+  vars:
+    apiserver_ips: "{{ apiserver_sans | map('ansible.utils.ipaddr') | reject('equalto', False) | list }}"
+    apiserver_hosts: "{{ apiserver_sans | difference(apiserver_ips) }}"
+  when:
+    - kubeadm_already_run.stat.exists
+    - not kube_external_ca_mode
+  block:
+    - name: Kubeadm | Check apiserver.crt SAN IPs
+      command:
+        cmd: "openssl x509 -noout -in {{ kube_cert_dir }}/apiserver.crt -checkip {{ item }}"
+      loop: "{{ apiserver_ips }}"
+      register: apiserver_sans_ip_check
+      changed_when: apiserver_sans_ip_check.stdout is not search('does match certificate')
+      failed_when: apiserver_sans_ip_check.rc != 0 and apiserver_sans_ip_check.stdout is not search('does NOT match certificate')
+    - name: Kubeadm | Check apiserver.crt SAN hosts
+      command:
+        cmd: "openssl x509 -noout -in {{ kube_cert_dir }}/apiserver.crt -checkhost {{ item }}"
+      loop: "{{ apiserver_hosts }}"
+      register: apiserver_sans_host_check
+      changed_when: apiserver_sans_host_check.stdout is not search('does match certificate')
+      failed_when: apiserver_sans_host_check.rc != 0 and apiserver_sans_host_check.stdout is not search('does NOT match certificate')
+
+- name: Kubeadm | regenerate apiserver cert 1/2
+  file:
+    state: absent
+    path: "{{ kube_cert_dir }}/{{ item }}"
+  with_items:
+    - apiserver.crt
+    - apiserver.key
+  when:
+    - kubeadm_already_run.stat.exists
+    - apiserver_sans_ip_check.changed or apiserver_sans_host_check.changed
+    - not kube_external_ca_mode
+
+- name: Kubeadm | regenerate apiserver cert 2/2
+  command: >-
+    {{ bin_dir }}/kubeadm
+    init phase certs apiserver
+    --config={{ kube_config_dir }}/kubeadm-config.yaml
+  when:
+    - kubeadm_already_run.stat.exists
+    - apiserver_sans_ip_check.changed or apiserver_sans_host_check.changed
+    - not kube_external_ca_mode
+
+- name: Kubeadm | Initialize first control plane node
+  when: inventory_hostname == first_kube_control_plane and not kubeadm_already_run.stat.exists
+  vars:
+    kubeadm_init_first_control_plane_cmd: >-
+      timeout -k {{ kubeadm_init_timeout }} {{ kubeadm_init_timeout }}
+      {{ bin_dir }}/kubeadm init
+      --config={{ kube_config_dir }}/kubeadm-config.yaml
+      --ignore-preflight-errors={{ kubeadm_ignore_preflight_errors | join(',') }}
+      --skip-phases={{ kubeadm_init_phases_skip | join(',') }}
+      {{ kube_external_ca_mode | ternary('', '--upload-certs') }}
+  environment:
+    PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}"
+  notify: Control plane | restart kubelet
+  block:
+    - name: Kubeadm | Initialize first control plane node (1st try)
+      command: "{{ kubeadm_init_first_control_plane_cmd }}"
+      register: kubeadm_init
+      failed_when: kubeadm_init.rc != 0 and "field is immutable" not in kubeadm_init.stderr
+  rescue:
+    # Retry is because upload config sometimes fails
+    # This retry task is separated from 1st task to show log of failure of 1st task.
+    - name: Kubeadm | Initialize first control plane node (retry)
+      command: "{{ kubeadm_init_first_control_plane_cmd }}"
+      register: kubeadm_init
+      retries: 2
+      until: kubeadm_init is succeeded or "field is immutable" in kubeadm_init.stderr
+      failed_when: kubeadm_init.rc != 0 and "field is immutable" not in kubeadm_init.stderr
+
+- name: Set kubeadm certificate key
+  set_fact:
+    kubeadm_certificate_key: "{{ item | regex_search('--certificate-key ([^ ]+)', '\\1') | first }}"
+  with_items: "{{ hostvars[groups['kube_control_plane'][0]]['kubeadm_init'].stdout_lines | default([]) }}"
+  when:
+    - kubeadm_certificate_key is not defined
+    - (item | trim) is match('.*--certificate-key.*')
+
+- name: Create hardcoded kubeadm token for joining nodes with 24h expiration (if defined)
+  shell: >-
+    {{ bin_dir }}/kubeadm --kubeconfig {{ kube_config_dir }}/admin.conf token delete {{ kubeadm_token }} || :;
+    {{ bin_dir }}/kubeadm --kubeconfig {{ kube_config_dir }}/admin.conf token create {{ kubeadm_token }}
+  changed_when: false
+  when:
+    - inventory_hostname == first_kube_control_plane
+    - kubeadm_token is defined
+    - kubeadm_refresh_token
+  tags:
+    - kubeadm_token
+
+- name: Remove binding to anonymous user
+  command: "{{ kubectl }} -n kube-public delete rolebinding kubeadm:bootstrap-signer-clusterinfo --ignore-not-found"
+  when: inventory_hostname == first_kube_control_plane and remove_anonymous_access
+
+- name: Create kubeadm token for joining nodes with 24h expiration (default)
+  command: "{{ bin_dir }}/kubeadm --kubeconfig {{ kube_config_dir }}/admin.conf token create"
+  changed_when: false
+  register: temp_token
+  retries: 5
+  delay: 5
+  until: temp_token is succeeded
+  delegate_to: "{{ first_kube_control_plane }}"
+  when: kubeadm_token is not defined
+  tags:
+    - kubeadm_token
+
+- name: Set kubeadm_token
+  set_fact:
+    kubeadm_token: "{{ temp_token.stdout }}"
+  when: temp_token.stdout is defined
+  tags:
+    - kubeadm_token
+
+- name: Kubeadm | Join other control plane nodes
+  include_tasks: kubeadm-secondary.yml
+
+- name: Kubeadm | upgrade kubernetes cluster to {{ kube_version }}
+  include_tasks: kubeadm-upgrade.yml
+  when:
+    - upgrade_cluster_setup
+    - kubeadm_already_run.stat.exists
+
+# FIXME(mattymo): from docs: If you don't want to taint your control-plane node, set this field to an empty slice, i.e. `taints: {}` in the YAML file.
+- name: Kubeadm | Remove taint for control plane node with node role
+  command: "{{ kubectl }} taint node {{ inventory_hostname }} {{ item }}"
+  delegate_to: "{{ first_kube_control_plane }}"
+  with_items:
+    - "node-role.kubernetes.io/control-plane:NoSchedule-"
+  when: ('kube_node' in group_names)
+  failed_when: false

internal/resources/kubespray/project/roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml

@@ -0,0 +1,105 @@
+---
+- name: Ensure kube-apiserver is up before upgrade
+  import_tasks: check-api.yml
+
+  # kubeadm-config.v1beta4 with UpgradeConfiguration requires some values that were previously allowed as args to be specified in the config file
+- name: Kubeadm | Upgrade first control plane node
+  command: >-
+    timeout -k 600s 600s
+    {{ bin_dir }}/kubeadm upgrade apply -y v{{ kube_version }}
+    {%- if kubeadm_config_api_version == 'v1beta3' %}
+    --certificate-renewal={{ kubeadm_upgrade_auto_cert_renewal }}
+    --ignore-preflight-errors={{ kubeadm_ignore_preflight_errors | join(',') }}
+    --allow-experimental-upgrades
+    --etcd-upgrade={{ (etcd_deployment_type == "kubeadm") | lower }}
+    {% if kubeadm_patches | length > 0 %}--patches={{ kubeadm_patches_dir }}{% endif %}
+    --force
+    {%- else %}
+    --config={{ kube_config_dir }}/kubeadm-config.yaml
+    {%- endif -%}
+  register: kubeadm_upgrade
+  when: inventory_hostname == first_kube_control_plane
+  failed_when: kubeadm_upgrade.rc != 0 and "field is immutable" not in kubeadm_upgrade.stderr
+  environment:
+    PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}"
+
+- name: Kubeadm | Upgrade other control plane nodes
+  command: >-
+    {{ bin_dir }}/kubeadm upgrade node
+    {%- if kubeadm_config_api_version == 'v1beta3' %}
+    --certificate-renewal={{ kubeadm_upgrade_auto_cert_renewal }}
+    --ignore-preflight-errors={{ kubeadm_ignore_preflight_errors | join(',') }}
+    --etcd-upgrade={{ (etcd_deployment_type == "kubeadm") | lower }}
+    {% if kubeadm_patches | length > 0 %}--patches={{ kubeadm_patches_dir }}{% endif %}
+    {%- else %}
+    --config={{ kube_config_dir }}/kubeadm-config.yaml
+    {%- endif -%}
+  register: kubeadm_upgrade
+  when: inventory_hostname != first_kube_control_plane
+  failed_when: kubeadm_upgrade.rc != 0 and "field is immutable" not in kubeadm_upgrade.stderr
+  environment:
+    PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}"
+
+# kubeadm upgrade no longer reconciles ClusterConfiguration and KubeProxyConfiguration changes, this must be done separately after upgrade to ensure the latest config is applied
+- name: Update kubeadm and kubelet configmaps after upgrade
+  command: "{{ bin_dir }}/kubeadm init phase upload-config all --config {{ kube_config_dir }}/kubeadm-config.yaml"
+  register: kubeadm_upload_config
+  # Retry is because upload config sometimes fails
+  retries: 3
+  until: kubeadm_upload_config.rc == 0
+  when:
+  - inventory_hostname == first_kube_control_plane
+
+- name: Update kube-proxy configmap after upgrade
+  command: "{{ bin_dir }}/kubeadm init phase addon kube-proxy --config {{ kube_config_dir }}/kubeadm-config.yaml"
+  register: kube_proxy_upload_config
+  # Retry is because upload config sometimes fails
+  retries: 3
+  until: kube_proxy_upload_config.rc == 0
+  when:
+  - inventory_hostname == first_kube_control_plane
+  - ('addon/kube-proxy' not in kubeadm_init_phases_skip)
+
+- name: Rewrite kubeadm managed etcd static pod manifests with updated configmap
+  command: "{{ bin_dir }}/kubeadm init phase etcd local --config {{ kube_config_dir }}/kubeadm-config.yaml"
+  when:
+  - etcd_deployment_type == "kubeadm"
+  notify: Control plane | restart kubelet
+
+- name: Rewrite kubernetes control plane static pod manifests with updated configmap
+  command: "{{ bin_dir }}/kubeadm init phase control-plane all --config {{ kube_config_dir }}/kubeadm-config.yaml"
+  notify: Control plane | restart kubelet
+
+- name: Flush kubelet handlers
+  meta: flush_handlers
+
+- name: Ensure kube-apiserver is up after upgrade and control plane configuration updates
+  import_tasks: check-api.yml
+
+- name: Kubeadm | Remove binding to anonymous user
+  command: "{{ kubectl }} -n kube-public delete rolebinding kubeadm:bootstrap-signer-clusterinfo --ignore-not-found"
+  when: remove_anonymous_access
+
+- name: Kubeadm | clean kubectl cache to refresh api types
+  file:
+    path: "{{ item }}"
+    state: absent
+  with_items:
+  - /root/.kube/cache
+  - /root/.kube/http-cache
+
+# FIXME: https://github.com/kubernetes/kubeadm/issues/1318
+- name: Kubeadm | scale down coredns replicas to 0 if not using coredns dns_mode
+  command: >-
+    {{ kubectl }}
+    -n kube-system
+    scale deployment/coredns --replicas 0
+  register: scale_down_coredns
+  retries: 6
+  delay: 5
+  until: scale_down_coredns is succeeded
+  run_once: true
+  when:
+  - kubeadm_scale_down_coredns_enabled
+  - dns_mode not in ['coredns', 'coredns_dual']
+  changed_when: false

internal/resources/kubespray/project/roles/kubernetes/control-plane/tasks/kubelet-fix-client-cert-rotation.yml

@@ -0,0 +1,18 @@
+---
+- name: Fixup kubelet client cert rotation 1/2
+  lineinfile:
+    path: "{{ kube_config_dir }}/kubelet.conf"
+    regexp: '^    client-certificate-data: '
+    line: '    client-certificate: /var/lib/kubelet/pki/kubelet-client-current.pem'
+    backup: true
+  notify:
+    - "Control plane | reload kubelet"
+
+- name: Fixup kubelet client cert rotation 2/2
+  lineinfile:
+    path: "{{ kube_config_dir }}/kubelet.conf"
+    regexp: '^    client-key-data: '
+    line: '    client-key: /var/lib/kubelet/pki/kubelet-client-current.pem'
+    backup: true
+  notify:
+    - "Control plane | reload kubelet"

internal/resources/kubespray/project/roles/kubernetes/control-plane/tasks/main.yml

@@ -0,0 +1,145 @@
+---
+- name: Pre-upgrade control plane
+  import_tasks: pre-upgrade.yml
+  tags:
+    - k8s-pre-upgrade
+
+- name: Create webhook token auth config
+  template:
+    src: webhook-token-auth-config.yaml.j2
+    dest: "{{ kube_config_dir }}/webhook-token-auth-config.yaml"
+    mode: "0640"
+  when: kube_webhook_token_auth | default(false)
+
+- name: Create webhook authorization config
+  template:
+    src: webhook-authorization-config.yaml.j2
+    dest: "{{ kube_config_dir }}/webhook-authorization-config.yaml"
+    mode: "0640"
+  when: kube_webhook_authorization | default(false)
+
+- name: Create structured AuthorizationConfiguration file
+  copy:
+    content: "{{ authz_config | to_nice_yaml(indent=2, sort_keys=false) }}"
+    dest: "{{ kube_config_dir }}/apiserver-authorization-config-{{ kube_apiserver_authorization_config_api_version }}.yaml"
+    mode: "0640"
+  vars:
+    authz_config:
+      apiVersion: apiserver.config.k8s.io/{{ kube_apiserver_authorization_config_api_version }}
+      kind: AuthorizationConfiguration
+      authorizers: "{{ kube_apiserver_authorization_config_authorizers }}"
+  when: kube_apiserver_use_authorization_config_file
+
+- name: Create kube-scheduler config
+  template:
+    src: kubescheduler-config.yaml.j2
+    dest: "{{ kube_config_dir }}/kubescheduler-config.yaml"
+    mode: "0644"
+
+- name: Apply Kubernetes encrypt at rest config
+  import_tasks: encrypt-at-rest.yml
+  when:
+    - kube_encrypt_secret_data
+  tags:
+    - kube-apiserver
+
+- name: Install | Copy kubectl binary from download dir
+  copy:
+    src: "{{ downloads.kubectl.dest }}"
+    dest: "{{ bin_dir }}/kubectl"
+    mode: "0755"
+    remote_src: true
+  tags:
+    - kubectl
+    - upgrade
+
+- name: Install kubectl bash completion
+  shell: "{{ bin_dir }}/kubectl completion bash >/etc/bash_completion.d/kubectl.sh"
+  when: ansible_os_family in ["Debian","RedHat", "Suse"]
+  tags:
+    - kubectl
+  ignore_errors: true  # noqa ignore-errors
+
+- name: Set kubectl bash completion file permissions
+  file:
+    path: /etc/bash_completion.d/kubectl.sh
+    owner: root
+    group: root
+    mode: "0755"
+  when: ansible_os_family in ["Debian","RedHat", "Suse"]
+  tags:
+    - kubectl
+    - upgrade
+  ignore_errors: true  # noqa ignore-errors
+
+- name: Set bash alias for kubectl
+  blockinfile:
+    path: /etc/bash_completion.d/kubectl.sh
+    block: |-
+      alias {{ kubectl_alias }}=kubectl
+      if [[ $(type -t compopt) = "builtin" ]]; then
+        complete -o default -F __start_kubectl {{ kubectl_alias }}
+      else
+        complete -o default -o nospace -F __start_kubectl {{ kubectl_alias }}
+      fi
+    state: present
+    marker: "# Ansible entries {mark}"
+  when:
+    - ansible_os_family in ["Debian","RedHat", "Suse"]
+    - kubectl_alias is defined and kubectl_alias != ""
+  tags:
+    - kubectl
+    - upgrade
+  ignore_errors: true  # noqa ignore-errors
+
+- name: Define nodes already joined to existing cluster and first_kube_control_plane
+  import_tasks: define-first-kube-control.yml
+
+- name: Include kubeadm setup
+  import_tasks: kubeadm-setup.yml
+
+- name: Include kubeadm etcd extra tasks
+  include_tasks: kubeadm-etcd.yml
+  when: etcd_deployment_type == "kubeadm"
+
+- name: Include kubeadm secondary server apiserver fixes
+  include_tasks: kubeadm-fix-apiserver.yml
+
+- name: Cleanup unused AuthorizationConfiguration file versions
+  file:
+    path: "{{ kube_config_dir }}/apiserver-authorization-config-{{ item }}.yaml"
+    state: absent
+  loop: "{{ ['v1alpha1', 'v1beta1', 'v1'] | reject('equalto', kube_apiserver_authorization_config_api_version) | list }}"
+  when: kube_apiserver_use_authorization_config_file
+
+- name: Include kubelet client cert rotation fixes
+  include_tasks: kubelet-fix-client-cert-rotation.yml
+  when: kubelet_rotate_certificates
+
+- name: Install script to renew K8S control plane certificates
+  template:
+    src: k8s-certs-renew.sh.j2
+    dest: "{{ bin_dir }}/k8s-certs-renew.sh"
+    mode: "0755"
+
+- name: Renew K8S control plane certificates monthly 1/2
+  template:
+    src: "{{ item }}.j2"
+    dest: "/etc/systemd/system/{{ item }}"
+    mode: "0644"
+    validate: "sh -c '[ -f /usr/bin/systemd/system/factory-reset.target ] || exit 0 && systemd-analyze verify %s:{{item}}'"
+    # FIXME: check that systemd version >= 250 (factory-reset.target was introduced in that release)
+    # Remove once we drop support for systemd < 250
+  with_items:
+    - k8s-certs-renew.service
+    - k8s-certs-renew.timer
+  register: k8s_certs_units
+  when: auto_renew_certificates
+
+- name: Renew K8S control plane certificates monthly 2/2
+  systemd_service:
+    name: k8s-certs-renew.timer
+    enabled: true
+    state: started
+    daemon_reload: "{{ k8s_certs_units is changed }}"
+  when: auto_renew_certificates

internal/resources/kubespray/project/roles/kubernetes/control-plane/tasks/pre-upgrade.yml

@@ -0,0 +1,21 @@
+---
+- name: "Pre-upgrade | Delete control plane manifests if etcd secrets changed"
+  file:
+    path: "/etc/kubernetes/manifests/{{ item }}.manifest"
+    state: absent
+  with_items:
+    - ["kube-apiserver", "kube-controller-manager", "kube-scheduler"]
+  register: kube_apiserver_manifest_replaced
+  when: etcd_secret_changed | default(false)
+
+- name: "Pre-upgrade | Delete control plane containers forcefully"  # noqa no-handler
+  shell: "set -o pipefail && docker ps -af name=k8s_{{ item }}* -q | xargs --no-run-if-empty docker rm -f"
+  args:
+    executable: /bin/bash
+  with_items:
+    - ["kube-apiserver", "kube-controller-manager", "kube-scheduler"]
+  when: kube_apiserver_manifest_replaced.changed
+  register: remove_control_plane_container
+  retries: 10
+  until: remove_control_plane_container.rc == 0
+  delay: 1

internal/resources/kubespray/project/roles/kubernetes/control-plane/templates/admission-controls.yaml.j2

@@ -0,0 +1,9 @@
+apiVersion: apiserver.config.k8s.io/v1
+kind: AdmissionConfiguration
+plugins:
+{% for plugin in kube_apiserver_enable_admission_plugins %}
+{% if plugin in kube_apiserver_admission_plugins_needs_configuration %}
+- name: {{ plugin }}
+  path: {{ kube_config_dir }}/{{ plugin | lower }}.yaml
+{% endif %}
+{% endfor %}

internal/resources/kubespray/project/roles/kubernetes/control-plane/templates/apiserver-audit-policy.yaml.j2

@@ -0,0 +1,129 @@
+apiVersion: audit.k8s.io/v1
+kind: Policy
+rules:
+{% if audit_policy_custom_rules is defined and audit_policy_custom_rules != "" %}
+{{ audit_policy_custom_rules | indent(2, true) }}
+{% else %}
+  # The following requests were manually identified as high-volume and low-risk,
+  # so drop them.
+  - level: None
+    users: ["system:kube-proxy"]
+    verbs: ["watch"]
+    resources:
+      - group: "" # core
+        resources: ["endpoints", "services", "services/status"]
+  - level: None
+    # Ingress controller reads `configmaps/ingress-uid` through the unsecured port.
+    # TODO(#46983): Change this to the ingress controller service account.
+    users: ["system:unsecured"]
+    namespaces: ["kube-system"]
+    verbs: ["get"]
+    resources:
+      - group: "" # core
+        resources: ["configmaps"]
+  - level: None
+    users: ["kubelet"] # legacy kubelet identity
+    verbs: ["get"]
+    resources:
+      - group: "" # core
+        resources: ["nodes", "nodes/status"]
+  - level: None
+    userGroups: ["system:nodes"]
+    verbs: ["get"]
+    resources:
+      - group: "" # core
+        resources: ["nodes", "nodes/status"]
+  - level: None
+    users:
+      - system:kube-controller-manager
+      - system:kube-scheduler
+      - system:serviceaccount:kube-system:endpoint-controller
+    verbs: ["get", "update"]
+    namespaces: ["kube-system"]
+    resources:
+      - group: "" # core
+        resources: ["endpoints"]
+  - level: None
+    users: ["system:apiserver"]
+    verbs: ["get"]
+    resources:
+      - group: "" # core
+        resources: ["namespaces", "namespaces/status", "namespaces/finalize"]
+  # Don't log HPA fetching metrics.
+  - level: None
+    users:
+      - system:kube-controller-manager
+    verbs: ["get", "list"]
+    resources:
+      - group: "metrics.k8s.io"
+  # Don't log these read-only URLs.
+  - level: None
+    nonResourceURLs:
+      - /healthz*
+      - /version
+      - /swagger*
+  # Don't log events requests.
+  - level: None
+    resources:
+      - group: "" # core
+        resources: ["events"]
+  # Secrets, ConfigMaps, TokenRequest and TokenReviews can contain sensitive & binary data,
+  # so only log at the Metadata level.
+  - level: Metadata
+    resources:
+      - group: "" # core
+        resources: ["secrets", "configmaps", "serviceaccounts/token"]
+      - group: authentication.k8s.io
+        resources: ["tokenreviews"]
+    omitStages:
+      - "RequestReceived"
+  # Get responses can be large; skip them.
+  - level: Request
+    verbs: ["get", "list", "watch"]
+    resources:
+      - group: "" # core
+      - group: "admissionregistration.k8s.io"
+      - group: "apiextensions.k8s.io"
+      - group: "apiregistration.k8s.io"
+      - group: "apps"
+      - group: "authentication.k8s.io"
+      - group: "authorization.k8s.io"
+      - group: "autoscaling"
+      - group: "batch"
+      - group: "certificates.k8s.io"
+      - group: "extensions"
+      - group: "metrics.k8s.io"
+      - group: "networking.k8s.io"
+      - group: "policy"
+      - group: "rbac.authorization.k8s.io"
+      - group: "settings.k8s.io"
+      - group: "storage.k8s.io"
+    omitStages:
+      - "RequestReceived"
+  # Default level for known APIs
+  - level: RequestResponse
+    resources:
+      - group: "" # core
+      - group: "admissionregistration.k8s.io"
+      - group: "apiextensions.k8s.io"
+      - group: "apiregistration.k8s.io"
+      - group: "apps"
+      - group: "authentication.k8s.io"
+      - group: "authorization.k8s.io"
+      - group: "autoscaling"
+      - group: "batch"
+      - group: "certificates.k8s.io"
+      - group: "extensions"
+      - group: "metrics.k8s.io"
+      - group: "networking.k8s.io"
+      - group: "policy"
+      - group: "rbac.authorization.k8s.io"
+      - group: "settings.k8s.io"
+      - group: "storage.k8s.io"
+    omitStages:
+      - "RequestReceived"
+  # Default level for all other requests.
+  - level: Metadata
+    omitStages:
+      - "RequestReceived"
+{% endif %}

internal/resources/kubespray/project/roles/kubernetes/control-plane/templates/apiserver-audit-webhook-config.yaml.j2

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Config
+clusters:
+- cluster:
+    server: {{ audit_webhook_server_url }}
+{% for key in audit_webhook_server_extra_args %}
+    {{ key }}: "{{ audit_webhook_server_extra_args[key] }}"
+{% endfor %}
+  name: auditsink
+contexts:
+- context:
+    cluster: auditsink
+    user: ""
+  name: default-context
+current-context: default-context
+preferences: {}
+users: []

internal/resources/kubespray/project/roles/kubernetes/control-plane/templates/apiserver-tracing.yaml.j2

@@ -0,0 +1,4 @@
+apiVersion: apiserver.config.k8s.io/v1beta1
+kind: TracingConfiguration
+endpoint: {{ kube_apiserver_tracing_endpoint }}
+samplingRatePerMillion: {{ kube_apiserver_tracing_sampling_rate_per_million }}

internal/resources/kubespray/project/roles/kubernetes/control-plane/templates/eventratelimit.yaml.j2

@@ -0,0 +1,11 @@
+apiVersion: eventratelimit.admission.k8s.io/v1alpha1
+kind: Configuration
+limits:
+{% for limit in kube_apiserver_admission_event_rate_limits.values() %}
+- type: {{ limit.type }}
+  qps: {{ limit.qps }}
+  burst: {{ limit.burst }}
+{% if limit.cache_size is defined %}
+  cacheSize: {{ limit.cache_size }}
+{% endif %}
+{% endfor %}

internal/resources/kubespray/project/roles/kubernetes/control-plane/templates/k8s-certs-renew.service.j2

@@ -0,0 +1,6 @@
+[Unit]
+Description=Renew K8S control plane certificates
+
+[Service]
+Type=oneshot
+ExecStart={{ bin_dir }}/k8s-certs-renew.sh

internal/resources/kubespray/project/roles/kubernetes/control-plane/templates/k8s-certs-renew.sh.j2

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+echo "## Expiration before renewal ##"
+{{ bin_dir }}/kubeadm certs check-expiration
+
+echo "## Renewing certificates managed by kubeadm ##"
+{{ bin_dir }}/kubeadm certs renew all
+
+echo "## Restarting control plane pods managed by kubeadm ##"
+{% if container_manager == "docker" %}
+{{ docker_bin_dir }}/docker ps -af 'name=k8s_POD_(kube-apiserver|kube-controller-manager|kube-scheduler|etcd)-*' -q | /usr/bin/xargs {{ docker_bin_dir }}/docker rm -f
+{% else %}
+{{ bin_dir }}/crictl pods --namespace kube-system --name 'kube-scheduler-*|kube-controller-manager-*|kube-apiserver-*|etcd-*' -q | /usr/bin/xargs {{ bin_dir }}/crictl rmp -f
+{% endif %}
+
+echo "## Updating /root/.kube/config ##"
+cp {{ kube_config_dir }}/admin.conf /root/.kube/config
+
+echo "## Waiting for apiserver to be up again ##"
+until printf "" 2>>/dev/null >>/dev/tcp/127.0.0.1/{{ kube_apiserver_port | default(6443) }}; do sleep 1; done
+
+echo "## Expiration after renewal ##"
+{{ bin_dir }}/kubeadm certs check-expiration

internal/resources/kubespray/project/roles/kubernetes/control-plane/templates/k8s-certs-renew.timer.j2

@@ -0,0 +1,11 @@
+[Unit]
+Description=Timer to renew K8S control plane certificates
+
+[Timer]
+OnCalendar={{ auto_renew_certificates_systemd_calendar }}
+RandomizedDelaySec={{ 10 * (groups['kube_control_plane'] | length) }}min
+FixedRandomDelay=yes
+Persistent=yes
+
+[Install]
+WantedBy=multi-user.target

internal/resources/kubespray/project/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2

@@ -0,0 +1,457 @@
+apiVersion: kubeadm.k8s.io/v1beta3
+kind: InitConfiguration
+{% if kubeadm_token is defined %}
+bootstrapTokens:
+- token: "{{ kubeadm_token }}"
+  description: "kubespray kubeadm bootstrap token"
+  ttl: "24h"
+{% endif %}
+localAPIEndpoint:
+  advertiseAddress: "{{ kube_apiserver_address }}"
+  bindPort: {{ kube_apiserver_port }}
+{% if kubeadm_certificate_key is defined %}
+certificateKey: {{ kubeadm_certificate_key }}
+{% endif %}
+nodeRegistration:
+{% if kube_override_hostname | default('') %}
+  name: "{{ kube_override_hostname }}"
+{% endif %}
+{% if 'kube_control_plane' in group_names and 'kube_node' not in group_names %}
+  taints:
+  - effect: NoSchedule
+    key: node-role.kubernetes.io/control-plane
+{% else %}
+  taints: []
+{% endif %}
+  criSocket: {{ cri_socket }}
+{% if cloud_provider == "external" %}
+  kubeletExtraArgs:
+    cloud-provider: external
+{% endif %}
+{% if kubeadm_patches | length > 0 %}
+patches:
+  directory: {{ kubeadm_patches_dir }}
+{% endif %}
+---
+apiVersion: kubeadm.k8s.io/v1beta3
+kind: ClusterConfiguration
+clusterName: {{ cluster_name }}
+etcd:
+{% if etcd_deployment_type != "kubeadm" %}
+  external:
+      endpoints:
+{% for endpoint in etcd_access_addresses.split(',') %}
+      - "{{ endpoint }}"
+{% endfor %}
+      caFile: {{ etcd_cert_dir }}/{{ kube_etcd_cacert_file }}
+      certFile: {{ etcd_cert_dir }}/{{ kube_etcd_cert_file }}
+      keyFile: {{ etcd_cert_dir }}/{{ kube_etcd_key_file }}
+{% elif etcd_deployment_type == "kubeadm" %}
+  local:
+    imageRepository: "{{ etcd_image_repo | regex_replace("/etcd$","") }}"
+    imageTag: "{{ etcd_image_tag }}"
+    dataDir: "{{ etcd_data_dir }}"
+    extraArgs:
+      metrics: {{ etcd_metrics }}
+      election-timeout: "{{ etcd_election_timeout }}"
+      heartbeat-interval: "{{ etcd_heartbeat_interval }}"
+      auto-compaction-retention: "{{ etcd_compaction_retention }}"
+{% if etcd_listen_metrics_urls is defined %}
+      listen-metrics-urls: "{{ etcd_listen_metrics_urls }}"
+{% endif %}
+{% if etcd_snapshot_count is defined %}
+      snapshot-count: "{{ etcd_snapshot_count }}"
+{% endif %}
+{% if etcd_quota_backend_bytes is defined %}
+      quota-backend-bytes: "{{ etcd_quota_backend_bytes }}"
+{% endif %}
+{% if etcd_max_request_bytes is defined %}
+      max-request-bytes: "{{ etcd_max_request_bytes }}"
+{% endif %}
+{% if etcd_log_level is defined %}
+      log-level: "{{ etcd_log_level }}"
+{% endif %}
+{% for key, value in etcd_extra_vars.items() %}
+      {{ key }}: "{{ value }}"
+{% endfor %}
+    serverCertSANs:
+{% for san in etcd_cert_alt_names %}
+      - "{{ san }}"
+{% endfor %}
+{% for san in etcd_cert_alt_ips %}
+      - "{{ san }}"
+{% endfor %}
+    peerCertSANs:
+{% for san in etcd_cert_alt_names %}
+      - "{{ san }}"
+{% endfor %}
+{% for san in etcd_cert_alt_ips %}
+      - "{{ san }}"
+{% endfor %}
+{% endif %}
+dns:
+  imageRepository: {{ coredns_image_repo | regex_replace('/coredns(?!/coredns).*$', '') }}
+  imageTag: {{ coredns_image_tag }}
+networking:
+  dnsDomain: {{ dns_domain }}
+  serviceSubnet: "{{ kube_service_subnets }}"
+{% if kube_network_plugin is defined and kube_network_plugin not in ["kube-ovn"] %}
+  podSubnet: "{{ kube_pods_subnets }}"
+{% endif %}
+{% if kubeadm_feature_gates %}
+featureGates:
+{%   for feature in kubeadm_feature_gates %}
+  {{ feature | replace("=", ": ") }}
+{%   endfor %}
+{% endif %}
+kubernetesVersion: v{{ kube_version }}
+{% if kubeadm_config_api_fqdn is defined %}
+controlPlaneEndpoint: "{{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}"
+{% else %}
+controlPlaneEndpoint: "{{ main_ip | ansible.utils.ipwrap }}:{{ kube_apiserver_port }}"
+{% endif %}
+certificatesDir: {{ kube_cert_dir }}
+imageRepository: {{ kube_image_repo }}
+apiServer:
+  extraArgs:
+{% if kube_apiserver_etcd_compaction_interval is defined %}
+    etcd-compaction-interval: "{{ kube_apiserver_etcd_compaction_interval }}"
+{% endif %}
+{% if kube_apiserver_pod_eviction_not_ready_timeout_seconds is defined %}
+    default-not-ready-toleration-seconds: "{{ kube_apiserver_pod_eviction_not_ready_timeout_seconds }}"
+{% endif %}
+{% if kube_apiserver_pod_eviction_unreachable_timeout_seconds is defined %}
+    default-unreachable-toleration-seconds: "{{ kube_apiserver_pod_eviction_unreachable_timeout_seconds }}"
+{% endif %}
+{% if kube_api_anonymous_auth is defined %}
+    anonymous-auth: "{{ kube_api_anonymous_auth }}"
+{% endif %}
+{% if kube_apiserver_use_authorization_config_file %}
+    authorization-config: "{{ kube_config_dir }}/apiserver-authorization-config-{{ kube_apiserver_authorization_config_api_version }}.yaml"
+{% else %}
+    authorization-mode: {{ authorization_modes | join(',') }}
+{% endif %}
+    bind-address: "{{ kube_apiserver_bind_address }}"
+{% if kube_apiserver_enable_admission_plugins | length > 0 %}
+    enable-admission-plugins: {{ kube_apiserver_enable_admission_plugins | join(',') }}
+{% endif %}
+{% if kube_apiserver_admission_control_config_file %}
+    admission-control-config-file: {{ kube_config_dir }}/admission-controls.yaml
+{% endif %}
+{% if kube_apiserver_disable_admission_plugins | length > 0 %}
+    disable-admission-plugins: {{ kube_apiserver_disable_admission_plugins | join(',') }}
+{% endif %}
+    apiserver-count: "{{ kube_apiserver_count }}"
+    endpoint-reconciler-type: lease
+{% if etcd_events_cluster_enabled %}
+    etcd-servers-overrides: "/events#{{ etcd_events_access_addresses_semicolon }}"
+{% endif %}
+    service-node-port-range: {{ kube_apiserver_node_port_range }}
+    service-cluster-ip-range: "{{ kube_service_subnets }}"
+    kubelet-preferred-address-types: "{{ kubelet_preferred_address_types }}"
+    profiling: "{{ kube_profiling }}"
+    request-timeout: "{{ kube_apiserver_request_timeout }}"
+    enable-aggregator-routing: "{{ kube_api_aggregator_routing }}"
+{% if kube_token_auth %}
+    token-auth-file: {{ kube_token_dir }}/known_tokens.csv
+{% endif %}
+{% if kube_apiserver_service_account_lookup %}
+    service-account-lookup: "{{ kube_apiserver_service_account_lookup }}"
+{% endif %}
+{% if kube_oidc_auth | default(false) and kube_oidc_url is defined and kube_oidc_client_id is defined %}
+    oidc-issuer-url: "{{ kube_oidc_url }}"
+    oidc-client-id: "{{ kube_oidc_client_id }}"
+{%   if kube_oidc_ca_file is defined %}
+    oidc-ca-file: "{{ kube_oidc_ca_file }}"
+{%   endif %}
+{%   if kube_oidc_username_claim is defined %}
+    oidc-username-claim: "{{ kube_oidc_username_claim }}"
+{%   endif %}
+{%   if kube_oidc_groups_claim is defined %}
+    oidc-groups-claim: "{{ kube_oidc_groups_claim }}"
+{%   endif %}
+{%   if kube_oidc_username_prefix is defined %}
+    oidc-username-prefix: "{{ kube_oidc_username_prefix }}"
+{%   endif %}
+{%   if kube_oidc_groups_prefix is defined %}
+    oidc-groups-prefix: "{{ kube_oidc_groups_prefix }}"
+{%   endif %}
+{% endif %}
+{% if kube_webhook_token_auth | default(false) %}
+    authentication-token-webhook-config-file: {{ kube_config_dir }}/webhook-token-auth-config.yaml
+{% endif %}
+{% if kube_webhook_authorization and not kube_apiserver_use_authorization_config_file %}
+    authorization-webhook-config-file: {{ kube_config_dir }}/webhook-authorization-config.yaml
+{% endif %}
+{% if kube_encrypt_secret_data %}
+    encryption-provider-config: {{ kube_cert_dir }}/secrets_encryption.yaml
+{% endif %}
+    storage-backend: {{ kube_apiserver_storage_backend }}
+{% if kube_api_runtime_config | length > 0 %}
+    runtime-config: {{ kube_api_runtime_config | join(',') }}
+{% endif %}
+    allow-privileged: "true"
+{% if kubernetes_audit or kubernetes_audit_webhook %}
+    audit-policy-file: {{ audit_policy_file }}
+{% endif %}
+{% if kubernetes_audit %}
+    audit-log-path: "{{ audit_log_path }}"
+    audit-log-maxage: "{{ audit_log_maxage }}"
+    audit-log-maxbackup: "{{ audit_log_maxbackups }}"
+    audit-log-maxsize: "{{ audit_log_maxsize }}"
+{% endif %}
+{% if kubernetes_audit_webhook %}
+    audit-webhook-config-file: {{ audit_webhook_config_file }}
+    audit-webhook-mode: {{ audit_webhook_mode }}
+{% if audit_webhook_mode == "batch" %}
+    audit-webhook-batch-max-size: "{{ audit_webhook_batch_max_size }}"
+    audit-webhook-batch-max-wait: "{{ audit_webhook_batch_max_wait }}"
+{% endif %}
+{% endif %}
+{% for key in kube_kubeadm_apiserver_extra_args %}
+    {{ key }}: "{{ kube_kubeadm_apiserver_extra_args[key] }}"
+{% endfor %}
+{% if kube_apiserver_feature_gates or kube_feature_gates %}
+    feature-gates: "{{ kube_apiserver_feature_gates | default(kube_feature_gates, true) | join(',') }}"
+{% endif %}
+{% if tls_min_version is defined %}
+    tls-min-version: {{ tls_min_version }}
+{% endif %}
+{% if tls_cipher_suites is defined %}
+    tls-cipher-suites: {% for tls in tls_cipher_suites %}{{ tls }}{{ "," if not loop.last else "" }}{% endfor %}
+
+{% endif %}
+{% if event_ttl_duration is defined %}
+    event-ttl: {{ event_ttl_duration }}
+{% endif %}
+{% if kubelet_rotate_server_certificates %}
+    kubelet-certificate-authority: {{ kube_cert_dir }}/ca.crt
+{% endif %}
+{% if kube_apiserver_tracing %}
+    tracing-config-file: {{ kube_config_dir }}/tracing/apiserver-tracing.yaml
+{% endif %}
+{% if kubernetes_audit or kube_token_auth or kube_webhook_token_auth or apiserver_extra_volumes or ssl_ca_dirs | length %}
+  extraVolumes:
+{% if kube_token_auth %}
+  - name: token-auth-config
+    hostPath: {{ kube_token_dir }}
+    mountPath: {{ kube_token_dir }}
+{% endif %}
+{% if kube_webhook_token_auth | default(false) %}
+  - name: webhook-token-auth-config
+    hostPath: {{ kube_config_dir }}/webhook-token-auth-config.yaml
+    mountPath: {{ kube_config_dir }}/webhook-token-auth-config.yaml
+{% endif %}
+{% if kube_webhook_authorization | default(false) %}
+  - name: webhook-authorization-config
+    hostPath: {{ kube_config_dir }}/webhook-authorization-config.yaml
+    mountPath: {{ kube_config_dir }}/webhook-authorization-config.yaml
+{% endif %}
+{% if kube_apiserver_use_authorization_config_file %}
+  - name: authorization-config
+    hostPath: {{ kube_config_dir }}/apiserver-authorization-config-{{ kube_apiserver_authorization_config_api_version }}.yaml
+    mountPath: {{ kube_config_dir }}/apiserver-authorization-config-{{ kube_apiserver_authorization_config_api_version }}.yaml
+{% endif %}
+{% if kubernetes_audit or kubernetes_audit_webhook %}
+  - name: {{ audit_policy_name }}
+    hostPath: {{ audit_policy_hostpath }}
+    mountPath: {{ audit_policy_mountpath }}
+{% if audit_log_path != "-" %}
+  - name: {{ audit_log_name }}
+    hostPath: {{ audit_log_hostpath }}
+    mountPath: {{ audit_log_mountpath }}
+    readOnly: false
+{% endif %}
+{% endif %}
+{% if kube_apiserver_admission_control_config_file %}
+  - name: admission-control-configs
+    hostPath: {{ kube_config_dir }}/admission-controls
+    mountPath: {{ kube_config_dir }}
+    readOnly: false
+    pathType: DirectoryOrCreate
+{% endif %}
+{% if kube_apiserver_tracing %}
+  - name: tracing
+    hostPath: {{ kube_config_dir }}/tracing
+    mountPath: {{ kube_config_dir }}/tracing
+    readOnly: true
+    pathType: DirectoryOrCreate
+{% endif %}
+{% for volume in apiserver_extra_volumes %}
+  - name: {{ volume.name }}
+    hostPath: {{ volume.hostPath }}
+    mountPath: {{ volume.mountPath }}
+    readOnly: {{ volume.readOnly | d(not (volume.writable | d(false))) }}
+{% endfor %}
+{% if ssl_ca_dirs | length %}
+{% for dir in ssl_ca_dirs %}
+  - name: {{ dir | regex_replace('^/(.*)$', '\\1' ) | regex_replace('/', '-') }}
+    hostPath: {{ dir }}
+    mountPath: {{ dir }}
+    readOnly: true
+{% endfor %}
+{% endif %}
+{% endif %}
+  certSANs:
+{% for san in apiserver_sans %}
+  - {{ san }}
+{% endfor %}
+  timeoutForControlPlane: 5m0s
+controllerManager:
+  extraArgs:
+    node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }}
+    node-monitor-period: {{ kube_controller_node_monitor_period }}
+{% if kube_network_plugin is defined and kube_network_plugin not in ["kube-ovn"] %}
+    cluster-cidr: "{{ kube_pods_subnets }}"
+{% endif %}
+    service-cluster-ip-range: "{{ kube_service_subnets }}"
+{% if kube_network_plugin is defined and kube_network_plugin == "calico" and not calico_ipam_host_local %}
+    allocate-node-cidrs: "false"
+{% else %}
+{% if ipv4_stack %}
+    node-cidr-mask-size-ipv4: "{{ kube_network_node_prefix }}"
+{% endif %}
+{% if ipv6_stack %}
+    node-cidr-mask-size-ipv6: "{{ kube_network_node_prefix_ipv6 }}"
+{% endif %}
+{% endif %}
+    profiling: "{{ kube_profiling }}"
+    terminated-pod-gc-threshold: "{{ kube_controller_terminated_pod_gc_threshold }}"
+    bind-address: "{{ kube_controller_manager_bind_address }}"
+    leader-elect-lease-duration: {{ kube_controller_manager_leader_elect_lease_duration }}
+    leader-elect-renew-deadline: {{ kube_controller_manager_leader_elect_renew_deadline }}
+{% if kube_controller_feature_gates or kube_feature_gates %}
+    feature-gates: "{{ kube_controller_feature_gates | default(kube_feature_gates, true) | join(',') }}"
+{% endif %}
+{% for key in kube_kubeadm_controller_extra_args %}
+    {{ key }}: "{{ kube_kubeadm_controller_extra_args[key] }}"
+{% endfor %}
+{% if kube_network_plugin is defined and kube_network_plugin not in ["cloud"] %}
+    configure-cloud-routes: "false"
+{% endif %}
+{% if kubelet_flexvolumes_plugins_dir is defined %}
+    flex-volume-plugin-dir: {{ kubelet_flexvolumes_plugins_dir }}
+{% endif %}
+{% if tls_min_version is defined %}
+    tls-min-version: {{ tls_min_version }}
+{% endif %}
+{% if tls_cipher_suites is defined %}
+    tls-cipher-suites: {% for tls in tls_cipher_suites %}{{ tls }}{{ "," if not loop.last else "" }}{% endfor %}
+
+{% endif %}
+{% if controller_manager_extra_volumes %}
+  extraVolumes:
+{% for volume in controller_manager_extra_volumes %}
+  - name: {{ volume.name }}
+    hostPath: {{ volume.hostPath }}
+    mountPath: {{ volume.mountPath }}
+    readOnly: {{ volume.readOnly | d(not (volume.writable | d(false))) }}
+{% endfor %}
+{% endif %}
+scheduler:
+  extraArgs:
+    bind-address: "{{ kube_scheduler_bind_address }}"
+    config: {{ kube_config_dir }}/kubescheduler-config.yaml
+{% if kube_scheduler_feature_gates or kube_feature_gates %}
+    feature-gates: "{{ kube_scheduler_feature_gates | default(kube_feature_gates, true) | join(',') }}"
+{% endif %}
+    profiling: "{{ kube_profiling }}"
+{% if kube_kubeadm_scheduler_extra_args | length > 0 %}
+{% for key in kube_kubeadm_scheduler_extra_args %}
+    {{ key }}: "{{ kube_kubeadm_scheduler_extra_args[key] }}"
+{% endfor %}
+{% endif %}
+{% if tls_min_version is defined %}
+    tls-min-version: {{ tls_min_version }}
+{% endif %}
+{% if tls_cipher_suites is defined %}
+    tls-cipher-suites: {% for tls in tls_cipher_suites %}{{ tls }}{{ "," if not loop.last else "" }}{% endfor %}
+
+{% endif %}
+  extraVolumes:
+  - name: kubescheduler-config
+    hostPath: {{ kube_config_dir }}/kubescheduler-config.yaml
+    mountPath: {{ kube_config_dir }}/kubescheduler-config.yaml
+    readOnly: true
+{% if scheduler_extra_volumes %}
+{% for volume in scheduler_extra_volumes %}
+  - name: {{ volume.name }}
+    hostPath: {{ volume.hostPath }}
+    mountPath: {{ volume.mountPath }}
+    readOnly: {{ volume.readOnly | d(not (volume.writable | d(false))) }}
+{% endfor %}
+{% endif %}
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+bindAddress: "{{ kube_proxy_bind_address }}"
+clientConnection:
+  acceptContentTypes: {{ kube_proxy_client_accept_content_types }}
+  burst: {{ kube_proxy_client_burst }}
+  contentType: {{ kube_proxy_client_content_type }}
+  kubeconfig: {{ kube_proxy_client_kubeconfig }}
+  qps: {{ kube_proxy_client_qps }}
+{% if kube_network_plugin is defined and kube_network_plugin not in ["kube-ovn"] %}
+clusterCIDR: "{{ kube_pods_subnets }}"
+{% endif %}
+configSyncPeriod: {{ kube_proxy_config_sync_period }}
+conntrack:
+  maxPerCore: {{ kube_proxy_conntrack_max_per_core }}
+  min: {{ kube_proxy_conntrack_min }}
+  tcpCloseWaitTimeout: {{ kube_proxy_conntrack_tcp_close_wait_timeout }}
+  tcpEstablishedTimeout: {{ kube_proxy_conntrack_tcp_established_timeout }}
+enableProfiling: {{ kube_proxy_enable_profiling }}
+healthzBindAddress: "{{ kube_proxy_healthz_bind_address }}"
+hostnameOverride: "{{ kube_override_hostname }}"
+iptables:
+  masqueradeAll: {{ kube_proxy_masquerade_all }}
+  masqueradeBit: {{ kube_proxy_masquerade_bit }}
+  minSyncPeriod: {{ kube_proxy_min_sync_period }}
+  syncPeriod: {{ kube_proxy_sync_period }}
+ipvs:
+  excludeCIDRs: {{ kube_proxy_exclude_cidrs }}
+  minSyncPeriod: {{ kube_proxy_min_sync_period }}
+  scheduler: {{ kube_proxy_scheduler }}
+  syncPeriod: {{ kube_proxy_sync_period }}
+  strictARP: {{ kube_proxy_strict_arp }}
+  tcpTimeout: {{ kube_proxy_tcp_timeout }}
+  tcpFinTimeout: {{ kube_proxy_tcp_fin_timeout }}
+  udpTimeout: {{ kube_proxy_udp_timeout }}
+metricsBindAddress: "{{ kube_proxy_metrics_bind_address }}"
+mode: {{ kube_proxy_mode }}
+nodePortAddresses: {{ kube_proxy_nodeport_addresses }}
+oomScoreAdj: {{ kube_proxy_oom_score_adj }}
+portRange: {{ kube_proxy_port_range }}
+{% if kube_proxy_feature_gates or kube_feature_gates %}
+{% set feature_gates = ( kube_proxy_feature_gates | default(kube_feature_gates, true) ) %}
+featureGates:
+{%   for feature in feature_gates %}
+  {{ feature | replace("=", ": ") }}
+{%   endfor %}
+{% endif %}
+{# DNS settings for kubelet #}
+{% if enable_nodelocaldns %}
+{% set kubelet_cluster_dns = [nodelocaldns_ip] %}
+{% elif dns_mode in ['coredns'] %}
+{% set kubelet_cluster_dns = [skydns_server] %}
+{% elif dns_mode == 'coredns_dual' %}
+{% set kubelet_cluster_dns = [skydns_server,skydns_server_secondary] %}
+{% elif dns_mode == 'manual' %}
+{% set kubelet_cluster_dns = [manual_dns_server] %}
+{% else %}
+{% set kubelet_cluster_dns = [] %}
+{% endif %}
+---
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+clusterDNS:
+{% for dns_address in kubelet_cluster_dns %}
+- {{ dns_address }}
+{% endfor %}
+{% if kubelet_feature_gates or kube_feature_gates %}
+{% set feature_gates = ( kubelet_feature_gates | default(kube_feature_gates, true) ) %}
+featureGates:
+{%   for feature in feature_gates %}
+  {{ feature | replace("=", ": ") }}
+{%   endfor %}
+{% endif %}

internal/resources/kubespray/project/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta4.yaml.j2

@@ -0,0 +1,593 @@
+apiVersion: kubeadm.k8s.io/v1beta4
+kind: InitConfiguration
+{% if kubeadm_token is defined %}
+bootstrapTokens:
+- token: "{{ kubeadm_token }}"
+  description: "kubespray kubeadm bootstrap token"
+  ttl: "24h"
+{% endif %}
+localAPIEndpoint:
+  advertiseAddress: "{{ kube_apiserver_address }}"
+  bindPort: {{ kube_apiserver_port }}
+{% if kubeadm_certificate_key is defined %}
+certificateKey: {{ kubeadm_certificate_key }}
+{% endif %}
+nodeRegistration:
+{% if kube_override_hostname | default('') %}
+  name: "{{ kube_override_hostname }}"
+{% endif %}
+{% if 'kube_control_plane' in group_names and 'kube_node' not in group_names %}
+  taints:
+  - effect: NoSchedule
+    key: node-role.kubernetes.io/control-plane
+{% else %}
+  taints: []
+{% endif %}
+  criSocket: {{ cri_socket }}
+{% if cloud_provider == "external" %}
+  kubeletExtraArgs:
+  - name: cloud-provider
+    value: external
+{% endif %}
+  imagePullPolicy: {{ k8s_image_pull_policy }}
+  imagePullSerial: {{ kubeadm_image_pull_serial | lower }}
+{% if kubeadm_patches | length > 0 %}
+patches:
+  directory: {{ kubeadm_patches_dir }}
+{% endif %}
+---
+apiVersion: kubeadm.k8s.io/v1beta4
+kind: ClusterConfiguration
+clusterName: {{ cluster_name }}
+encryptionAlgorithm: {{ kube_asymmetric_encryption_algorithm }}
+etcd:
+{% if etcd_deployment_type != "kubeadm" %}
+  external:
+      endpoints:
+{% for endpoint in etcd_access_addresses.split(',') %}
+      - "{{ endpoint }}"
+{% endfor %}
+      caFile: {{ etcd_cert_dir }}/{{ kube_etcd_cacert_file }}
+      certFile: {{ etcd_cert_dir }}/{{ kube_etcd_cert_file }}
+      keyFile: {{ etcd_cert_dir }}/{{ kube_etcd_key_file }}
+{% elif etcd_deployment_type == "kubeadm" %}
+  local:
+    imageRepository: "{{ etcd_image_repo | regex_replace("/etcd$","") }}"
+    imageTag: "{{ etcd_image_tag }}"
+    dataDir: "{{ etcd_data_dir }}"
+    extraArgs:
+    - name: metrics
+      value: {{ etcd_metrics }}
+    - name: election-timeout
+      value: "{{ etcd_election_timeout }}"
+    - name: heartbeat-interval
+      value: "{{ etcd_heartbeat_interval }}"
+    - name: auto-compaction-retention
+      value: "{{ etcd_compaction_retention }}"
+{% if etcd_listen_metrics_urls is defined %}
+    - name: listen-metrics-urls
+      value: "{{ etcd_listen_metrics_urls }}"
+{% endif %}
+{% if etcd_snapshot_count is defined %}
+    - name: snapshot-count
+      value: "{{ etcd_snapshot_count }}"
+{% endif %}
+{% if etcd_quota_backend_bytes is defined %}
+    - name: quota-backend-bytes
+      value: "{{ etcd_quota_backend_bytes }}"
+{% endif %}
+{% if etcd_max_request_bytes is defined %}
+    - name: max-request-bytes
+      value: "{{ etcd_max_request_bytes }}"
+{% endif %}
+{% if etcd_log_level is defined %}
+    - name: log-level
+      value: "{{ etcd_log_level }}"
+{% endif %}
+{% for key, value in etcd_extra_vars.items() %}
+    - name: {{ key }}
+      value: "{{ value }}"
+{% endfor %}
+    serverCertSANs:
+{% for san in etcd_cert_alt_names %}
+      - "{{ san }}"
+{% endfor %}
+{% for san in etcd_cert_alt_ips %}
+      - "{{ san }}"
+{% endfor %}
+    peerCertSANs:
+{% for san in etcd_cert_alt_names %}
+      - "{{ san }}"
+{% endfor %}
+{% for san in etcd_cert_alt_ips %}
+      - "{{ san }}"
+{% endfor %}
+{% endif %}
+dns:
+{% if 'addon/coredns' in kubeadm_init_phases_skip  %}
+  disabled: true
+{% endif %}
+  imageRepository: {{ coredns_image_repo | regex_replace('/coredns(?!/coredns).*$', '') }}
+  imageTag: {{ coredns_image_tag }}
+networking:
+  dnsDomain: {{ dns_domain }}
+  serviceSubnet: "{{ kube_service_subnets }}"
+{% if kube_network_plugin is defined and kube_network_plugin not in ["kube-ovn"] %}
+  podSubnet: "{{ kube_pods_subnets }}"
+{% endif %}
+{% if kubeadm_feature_gates %}
+featureGates:
+{%   for feature in kubeadm_feature_gates %}
+  {{ feature | replace("=", ": ") }}
+{%   endfor %}
+{% endif %}
+kubernetesVersion: v{{ kube_version }}
+{% if kubeadm_config_api_fqdn is defined %}
+controlPlaneEndpoint: "{{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}"
+{% else %}
+controlPlaneEndpoint: "{{ main_ip | ansible.utils.ipwrap }}:{{ kube_apiserver_port }}"
+{% endif %}
+certificatesDir: {{ kube_cert_dir }}
+imageRepository: {{ kube_image_repo }}
+apiServer:
+  extraArgs:
+{% if kube_apiserver_etcd_compaction_interval is defined %}
+  - name: etcd-compaction-interval
+    value: "{{ kube_apiserver_etcd_compaction_interval }}"
+{% endif %}
+{% if kube_apiserver_pod_eviction_not_ready_timeout_seconds is defined %}
+  - name: default-not-ready-toleration-seconds
+    value: "{{ kube_apiserver_pod_eviction_not_ready_timeout_seconds }}"
+{% endif %}
+{% if kube_apiserver_pod_eviction_unreachable_timeout_seconds is defined %}
+  - name: default-unreachable-toleration-seconds
+    value: "{{ kube_apiserver_pod_eviction_unreachable_timeout_seconds }}"
+{% endif %}
+{% if kube_api_anonymous_auth is defined %}
+  - name: anonymous-auth
+    value: "{{ kube_api_anonymous_auth }}"
+{% endif %}
+{% if kube_apiserver_use_authorization_config_file %}
+  - name: authorization-config
+    value: "{{ kube_config_dir }}/apiserver-authorization-config-{{ kube_apiserver_authorization_config_api_version }}.yaml"
+{% else %}
+  - name: authorization-mode
+    value: "{{ authorization_modes | join(',') }}"
+{% endif %}
+  - name: bind-address
+    value: "{{ kube_apiserver_bind_address }}"
+{% if kube_apiserver_enable_admission_plugins | length > 0 %}
+  - name: enable-admission-plugins
+    value: "{{ kube_apiserver_enable_admission_plugins | join(',') }}"
+{% endif %}
+{% if kube_apiserver_admission_control_config_file %}
+  - name: admission-control-config-file
+    value: "{{ kube_config_dir }}/admission-controls.yaml"
+{% endif %}
+{% if kube_apiserver_disable_admission_plugins | length > 0 %}
+  - name: disable-admission-plugins
+    value: "{{ kube_apiserver_disable_admission_plugins | join(',') }}"
+{% endif %}
+  - name: apiserver-count
+    value: "{{ kube_apiserver_count }}"
+  - name: endpoint-reconciler-type
+    value: lease
+{% if etcd_events_cluster_enabled %}
+  - name: etcd-servers-overrides
+    value: "/events#{{ etcd_events_access_addresses_semicolon }}"
+{% endif %}
+  - name: service-node-port-range
+    value: "{{ kube_apiserver_node_port_range }}"
+  - name: service-cluster-ip-range
+    value: "{{ kube_service_subnets }}"
+  - name: kubelet-preferred-address-types
+    value: "{{ kubelet_preferred_address_types }}"
+  - name: profiling
+    value: "{{ kube_profiling }}"
+  - name: request-timeout
+    value: "{{ kube_apiserver_request_timeout }}"
+  - name: enable-aggregator-routing
+    value: "{{ kube_api_aggregator_routing }}"
+{% if kube_apiserver_service_account_lookup %}
+  - name: service-account-lookup
+    value: "{{ kube_apiserver_service_account_lookup }}"
+{% endif %}
+{% if kube_oidc_auth | default(false) and kube_oidc_url is defined and kube_oidc_client_id is defined %}
+  - name: oidc-issuer-url
+    value: "{{ kube_oidc_url }}"
+  - name: oidc-client-id
+    value: "{{ kube_oidc_client_id }}"
+{%   if kube_oidc_ca_file is defined %}
+  - name: oidc-ca-file
+    value: "{{ kube_oidc_ca_file }}"
+{%   endif %}
+{%   if kube_oidc_username_claim is defined %}
+  - name: oidc-username-claim
+    value: "{{ kube_oidc_username_claim }}"
+{%   endif %}
+{%   if kube_oidc_groups_claim is defined %}
+  - name: oidc-groups-claim
+    value: "{{ kube_oidc_groups_claim }}"
+{%   endif %}
+{%   if kube_oidc_username_prefix is defined %}
+  - name: oidc-username-prefix
+    value: "{{ kube_oidc_username_prefix }}"
+{%   endif %}
+{%   if kube_oidc_groups_prefix is defined %}
+  - name: oidc-groups-prefix
+    value: "{{ kube_oidc_groups_prefix }}"
+{%   endif %}
+{% endif %}
+{% if kube_webhook_token_auth | default(false) %}
+  - name: authentication-token-webhook-config-file
+    value: "{{ kube_config_dir }}/webhook-token-auth-config.yaml"
+{% endif %}
+{% if kube_webhook_authorization and not kube_apiserver_use_authorization_config_file %}
+  - name: authorization-webhook-config-file
+    value: "{{ kube_config_dir }}/webhook-authorization-config.yaml"
+{% endif %}
+{% if kube_encrypt_secret_data %}
+  - name: encryption-provider-config
+    value: "{{ kube_cert_dir }}/secrets_encryption.yaml"
+{% endif %}
+  - name: storage-backend
+    value: "{{ kube_apiserver_storage_backend }}"
+{% if kube_api_runtime_config | length > 0 %}
+  - name: runtime-config
+    value: "{{ kube_api_runtime_config | join(',') }}"
+{% endif %}
+  - name: allow-privileged
+    value: "true"
+{% if kubernetes_audit or kubernetes_audit_webhook %}
+  - name: audit-policy-file
+    value: "{{ audit_policy_file }}"
+{% endif %}
+{% if kubernetes_audit %}
+  - name: audit-log-path
+    value: "{{ audit_log_path }}"
+  - name: audit-log-maxage
+    value: "{{ audit_log_maxage }}"
+  - name: audit-log-maxbackup
+    value: "{{ audit_log_maxbackups }}"
+  - name: audit-log-maxsize
+    value: "{{ audit_log_maxsize }}"
+{% endif %}
+{% if kubernetes_audit_webhook %}
+  - name: audit-webhook-config-file
+    value: "{{ audit_webhook_config_file }}"
+  - name: audit-webhook-mode
+    value: "{{ audit_webhook_mode }}"
+{% if audit_webhook_mode == "batch" %}
+  - name: audit-webhook-batch-max-size
+    value: "{{ audit_webhook_batch_max_size }}"
+  - name: audit-webhook-batch-max-wait
+    value: "{{ audit_webhook_batch_max_wait }}"
+{% endif %}
+{% endif %}
+{% for key in kube_kubeadm_apiserver_extra_args %}
+  - name: "{{ key }}"
+    value: "{{ kube_kubeadm_apiserver_extra_args[key] }}"
+{% endfor %}
+{% if kube_apiserver_feature_gates or kube_feature_gates %}
+  - name: feature-gates
+    value: "{{ kube_apiserver_feature_gates | default(kube_feature_gates, true) | join(',') }}"
+{% endif %}
+{% if tls_min_version is defined %}
+  - name: tls-min-version
+    value: "{{ tls_min_version }}"
+{% endif %}
+{% if tls_cipher_suites is defined %}
+  - name: tls-cipher-suites
+    value: "{% for tls in tls_cipher_suites %}{{ tls }}{{ ',' if not loop.last else '' }}{% endfor %}"
+{% endif %}
+{% if event_ttl_duration is defined %}
+  - name: event-ttl
+    value: "{{ event_ttl_duration }}"
+{% endif %}
+{% if kubelet_rotate_server_certificates %}
+  - name: kubelet-certificate-authority
+    value: "{{ kube_cert_dir }}/ca.crt"
+{% endif %}
+{% if kube_apiserver_tracing %}
+  - name: tracing-config-file
+    value: "{{ kube_config_dir }}/tracing/apiserver-tracing.yaml"
+{% endif %}
+{% if kubernetes_audit or kube_token_auth or kube_webhook_token_auth or apiserver_extra_volumes or ssl_ca_dirs | length %}
+  extraVolumes:
+{% if kube_token_auth %}
+  - name: token-auth-config
+    hostPath: {{ kube_token_dir }}
+    mountPath: {{ kube_token_dir }}
+{% endif %}
+{% if kube_webhook_token_auth | default(false) %}
+  - name: webhook-token-auth-config
+    hostPath: {{ kube_config_dir }}/webhook-token-auth-config.yaml
+    mountPath: {{ kube_config_dir }}/webhook-token-auth-config.yaml
+{% endif %}
+{% if kube_webhook_authorization | default(false) %}
+  - name: webhook-authorization-config
+    hostPath: {{ kube_config_dir }}/webhook-authorization-config.yaml
+    mountPath: {{ kube_config_dir }}/webhook-authorization-config.yaml
+{% endif %}
+{% if kube_apiserver_use_authorization_config_file %}
+  - name: authorization-config
+    hostPath: {{ kube_config_dir }}/apiserver-authorization-config-{{ kube_apiserver_authorization_config_api_version }}.yaml
+    mountPath: {{ kube_config_dir }}/apiserver-authorization-config-{{ kube_apiserver_authorization_config_api_version }}.yaml
+{% endif %}
+{% if kubernetes_audit or kubernetes_audit_webhook %}
+  - name: {{ audit_policy_name }}
+    hostPath: {{ audit_policy_hostpath }}
+    mountPath: {{ audit_policy_mountpath }}
+{% if audit_log_path != "-" %}
+  - name: {{ audit_log_name }}
+    hostPath: {{ audit_log_hostpath }}
+    mountPath: {{ audit_log_mountpath }}
+    readOnly: false
+{% endif %}
+{% endif %}
+{% if kube_apiserver_admission_control_config_file %}
+  - name: admission-control-configs
+    hostPath: {{ kube_config_dir }}/admission-controls
+    mountPath: {{ kube_config_dir }}
+    readOnly: false
+    pathType: DirectoryOrCreate
+{% endif %}
+{% if kube_apiserver_tracing %}
+  - name: tracing
+    hostPath: {{ kube_config_dir }}/tracing
+    mountPath: {{ kube_config_dir }}/tracing
+    readOnly: true
+    pathType: DirectoryOrCreate
+{% endif %}
+{% for volume in apiserver_extra_volumes %}
+  - name: {{ volume.name }}
+    hostPath: {{ volume.hostPath }}
+    mountPath: {{ volume.mountPath }}
+    readOnly: {{ volume.readOnly | d(not (volume.writable | d(false))) }}
+{% endfor %}
+{% if ssl_ca_dirs | length %}
+{% for dir in ssl_ca_dirs %}
+  - name: {{ dir | regex_replace('^/(.*)$', '\\1' ) | regex_replace('/', '-') }}
+    hostPath: {{ dir }}
+    mountPath: {{ dir }}
+    readOnly: true
+{% endfor %}
+{% endif %}
+{% endif %}
+  certSANs:
+{% for san in apiserver_sans %}
+  - {{ san }}
+{% endfor %}
+controllerManager:
+  extraArgs:
+  - name: node-monitor-grace-period
+    value: "{{ kube_controller_node_monitor_grace_period }}"
+  - name: node-monitor-period
+    value: "{{ kube_controller_node_monitor_period }}"
+{% if kube_network_plugin is defined and kube_network_plugin not in ["kube-ovn"] %}
+  - name: cluster-cidr
+    value: "{{ kube_pods_subnets }}"
+{% endif %}
+  - name: service-cluster-ip-range
+    value: "{{ kube_service_subnets }}"
+{% if kube_network_plugin is defined and kube_network_plugin == "calico" and not calico_ipam_host_local %}
+  - name: allocate-node-cidrs
+    value: "false"
+{% else %}
+{% if ipv4_stack %}
+  - name: node-cidr-mask-size-ipv4
+    value: "{{ kube_network_node_prefix }}"
+{% endif %}
+{% if ipv6_stack %}
+  - name: node-cidr-mask-size-ipv6
+    value: "{{ kube_network_node_prefix_ipv6 }}"
+{% endif %}
+{% endif %}
+  - name: profiling
+    value: "{{ kube_profiling }}"
+  - name: terminated-pod-gc-threshold
+    value: "{{ kube_controller_terminated_pod_gc_threshold }}"
+  - name: bind-address
+    value: "{{ kube_controller_manager_bind_address }}"
+  - name: leader-elect-lease-duration
+    value: "{{ kube_controller_manager_leader_elect_lease_duration }}"
+  - name: leader-elect-renew-deadline
+    value: "{{ kube_controller_manager_leader_elect_renew_deadline }}"
+{% if kube_controller_feature_gates or kube_feature_gates %}
+  - name: feature-gates
+    value: "{{ kube_controller_feature_gates | default(kube_feature_gates, true) | join(',') }}"
+{% endif %}
+{% for key in kube_kubeadm_controller_extra_args %}
+  - name: "{{ key }}"
+    value: "{{ kube_kubeadm_controller_extra_args[key] }}"
+{% endfor %}
+{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "gce"] %}
+  - name: cloud-provider
+    value: "{{ cloud_provider }}"
+  - name: cloud-config
+    value: "{{ kube_config_dir }}/cloud_config"
+{% endif %}
+{% if kube_network_plugin is defined and kube_network_plugin not in ["cloud"] %}
+  - name: configure-cloud-routes
+    value: "false"
+{% endif %}
+{% if kubelet_flexvolumes_plugins_dir is defined %}
+  - name: flex-volume-plugin-dir
+    value: "{{ kubelet_flexvolumes_plugins_dir }}"
+{% endif %}
+{% if tls_min_version is defined %}
+  - name: tls-min-version
+    value: "{{ tls_min_version }}"
+{% endif %}
+{% if tls_cipher_suites is defined %}
+  - name: tls-cipher-suites
+    value: "{% for tls in tls_cipher_suites %}{{ tls }}{{ ',' if not loop.last else '' }}{% endfor %}"
+{% endif %}
+{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "gce"] or controller_manager_extra_volumes %}
+  extraVolumes:
+{% if cloud_provider is defined and cloud_provider in ["openstack"] and openstack_cacert is defined %}
+  - name: openstackcacert
+    hostPath: "{{ kube_config_dir }}/openstack-cacert.pem"
+    mountPath: "{{ kube_config_dir }}/openstack-cacert.pem"
+{% endif %}
+{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "gce"] %}
+  - name: cloud-config
+    hostPath: {{ kube_config_dir }}/cloud_config
+    mountPath: {{ kube_config_dir }}/cloud_config
+{% endif %}
+{% for volume in controller_manager_extra_volumes %}
+  - name: {{ volume.name }}
+    hostPath: {{ volume.hostPath }}
+    mountPath: {{ volume.mountPath }}
+    readOnly: {{ volume.readOnly | d(not (volume.writable | d(false))) }}
+{% endfor %}
+{% endif %}
+scheduler:
+  extraArgs:
+  - name: bind-address
+    value: "{{ kube_scheduler_bind_address }}"
+  - name: config
+    value: "{{ kube_config_dir }}/kubescheduler-config.yaml"
+{% if kube_scheduler_feature_gates or kube_feature_gates %}
+  - name: feature-gates
+    value: "{{ kube_scheduler_feature_gates | default(kube_feature_gates, true) | join(',') }}"
+{% endif %}
+  - name: profiling
+    value: "{{ kube_profiling }}"
+{% if kube_kubeadm_scheduler_extra_args | length > 0 %}
+{% for key in kube_kubeadm_scheduler_extra_args %}
+  - name: "{{ key }}"
+    value: "{{ kube_kubeadm_scheduler_extra_args[key] }}"
+{% endfor %}
+{% endif %}
+{% if tls_min_version is defined %}
+  - name: tls-min-version
+    value: "{{ tls_min_version }}"
+{% endif %}
+{% if tls_cipher_suites is defined %}
+  - name: tls-cipher-suites
+    value: "{% for tls in tls_cipher_suites %}{{ tls }}{{ ',' if not loop.last else '' }}{% endfor %}"
+{% endif %}
+  extraVolumes:
+  - name: kubescheduler-config
+    hostPath: {{ kube_config_dir }}/kubescheduler-config.yaml
+    mountPath: {{ kube_config_dir }}/kubescheduler-config.yaml
+    readOnly: true
+{% if scheduler_extra_volumes %}
+{% for volume in scheduler_extra_volumes %}
+  - name: {{ volume.name }}
+    hostPath: {{ volume.hostPath }}
+    mountPath: {{ volume.mountPath }}
+    readOnly: {{ volume.readOnly | d(not (volume.writable | d(false))) }}
+{% endfor %}
+{% endif %}
+---
+apiVersion: kubeadm.k8s.io/v1beta4
+kind: UpgradeConfiguration
+apply:
+  kubernetesVersion: v{{ kube_version }}
+  allowExperimentalUpgrades: true
+  certificateRenewal: {{ kubeadm_upgrade_auto_cert_renewal | lower }}
+  etcdUpgrade: {{ (etcd_deployment_type == "kubeadm") | lower }}
+  forceUpgrade: true
+{% if kubeadm_ignore_preflight_errors | length > 0 %}
+  ignorePreflightErrors:
+{% for ignore_error in kubeadm_ignore_preflight_errors %}
+  - "{{ ignore_error }}"
+{% endfor %}
+{% endif %}
+{% if kubeadm_patches | length > 0 %}
+  patches:
+    directory: {{ kubeadm_patches_dir }}
+{% endif %}
+  imagePullPolicy: {{ k8s_image_pull_policy }}
+  imagePullSerial: {{ kubeadm_image_pull_serial | lower }}
+node:
+  certificateRenewal: {{ kubeadm_upgrade_auto_cert_renewal | lower }}
+  etcdUpgrade: {{ (etcd_deployment_type == "kubeadm") | lower }}
+{% if kubeadm_ignore_preflight_errors | length > 0 %}
+  ignorePreflightErrors:
+{% for ignore_error in kubeadm_ignore_preflight_errors %}
+  - "{{ ignore_error }}"
+{% endfor %}
+{% endif %}
+{% if kubeadm_patches | length > 0 %}
+  patches:
+    directory: {{ kubeadm_patches_dir }}
+{% endif %}
+  imagePullPolicy: {{ k8s_image_pull_policy }}
+  imagePullSerial: {{ kubeadm_image_pull_serial | lower }}
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+bindAddress: "{{ kube_proxy_bind_address }}"
+clientConnection:
+  acceptContentTypes: {{ kube_proxy_client_accept_content_types }}
+  burst: {{ kube_proxy_client_burst }}
+  contentType: {{ kube_proxy_client_content_type }}
+  kubeconfig: {{ kube_proxy_client_kubeconfig }}
+  qps: {{ kube_proxy_client_qps }}
+{% if kube_network_plugin is defined and kube_network_plugin not in ["kube-ovn"] %}
+clusterCIDR: "{{ kube_pods_subnets }}"
+{% endif %}
+configSyncPeriod: {{ kube_proxy_config_sync_period }}
+conntrack:
+  maxPerCore: {{ kube_proxy_conntrack_max_per_core }}
+  min: {{ kube_proxy_conntrack_min }}
+  tcpCloseWaitTimeout: {{ kube_proxy_conntrack_tcp_close_wait_timeout }}
+  tcpEstablishedTimeout: {{ kube_proxy_conntrack_tcp_established_timeout }}
+enableProfiling: {{ kube_proxy_enable_profiling }}
+healthzBindAddress: "{{ kube_proxy_healthz_bind_address }}"
+hostnameOverride: "{{ kube_override_hostname }}"
+iptables:
+  masqueradeAll: {{ kube_proxy_masquerade_all }}
+  masqueradeBit: {{ kube_proxy_masquerade_bit }}
+  minSyncPeriod: {{ kube_proxy_min_sync_period }}
+  syncPeriod: {{ kube_proxy_sync_period }}
+ipvs:
+  excludeCIDRs: {{ kube_proxy_exclude_cidrs }}
+  minSyncPeriod: {{ kube_proxy_min_sync_period }}
+  scheduler: {{ kube_proxy_scheduler }}
+  syncPeriod: {{ kube_proxy_sync_period }}
+  strictARP: {{ kube_proxy_strict_arp }}
+  tcpTimeout: {{ kube_proxy_tcp_timeout }}
+  tcpFinTimeout: {{ kube_proxy_tcp_fin_timeout }}
+  udpTimeout: {{ kube_proxy_udp_timeout }}
+metricsBindAddress: "{{ kube_proxy_metrics_bind_address }}"
+mode: {{ kube_proxy_mode }}
+nodePortAddresses: {{ kube_proxy_nodeport_addresses }}
+oomScoreAdj: {{ kube_proxy_oom_score_adj }}
+portRange: {{ kube_proxy_port_range }}
+{% if kube_proxy_feature_gates or kube_feature_gates %}
+{% set feature_gates = ( kube_proxy_feature_gates | default(kube_feature_gates, true) ) %}
+featureGates:
+{%   for feature in feature_gates %}
+  {{ feature | replace("=", ": ") }}
+{%   endfor %}
+{% endif %}
+{# DNS settings for kubelet #}
+{% if enable_nodelocaldns %}
+{% set kubelet_cluster_dns = [nodelocaldns_ip] %}
+{% elif dns_mode in ['coredns'] %}
+{% set kubelet_cluster_dns = [skydns_server] %}
+{% elif dns_mode == 'coredns_dual' %}
+{% set kubelet_cluster_dns = [skydns_server,skydns_server_secondary] %}
+{% elif dns_mode == 'manual' %}
+{% set kubelet_cluster_dns = [manual_dns_server] %}
+{% else %}
+{% set kubelet_cluster_dns = [] %}
+{% endif %}
+---
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+clusterDNS:
+{% for dns_address in kubelet_cluster_dns %}
+- {{ dns_address }}
+{% endfor %}
+{% if kubelet_feature_gates or kube_feature_gates %}
+{% set feature_gates = ( kubelet_feature_gates | default(kube_feature_gates, true) ) %}
+featureGates:
+{%   for feature in feature_gates %}
+  {{ feature | replace("=", ": ") }}
+{%   endfor %}
+{% endif %}

internal/resources/kubespray/project/roles/kubernetes/control-plane/templates/kubeadm-controlplane.yaml.j2

@@ -0,0 +1,43 @@
+apiVersion: kubeadm.k8s.io/{{ kubeadm_config_api_version }}
+kind: JoinConfiguration
+discovery:
+{% if kubeadm_use_file_discovery %}
+  file:
+    kubeConfigPath: {{ kube_config_dir }}/cluster-info-discovery-kubeconfig.yaml
+{% else %}
+  bootstrapToken:
+{% if kubeadm_config_api_fqdn is defined %}
+    apiServerEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}
+{% else %}
+    apiServerEndpoint: "{{ kubeadm_discovery_address }}"
+{% endif %}
+    token: {{ kubeadm_token }}
+    unsafeSkipCAVerification: true
+{% endif %}
+  tlsBootstrapToken: {{ kubeadm_token }}
+{# TODO: drop the if when we drop support for k8s<1.31 #}
+{% if kubeadm_config_api_version == 'v1beta3' %}
+  timeout: {{ discovery_timeout }}
+{% else %}
+timeouts:
+  discovery: {{ discovery_timeout }}
+{% endif %}
+controlPlane:
+  localAPIEndpoint:
+    advertiseAddress: "{{ kube_apiserver_address }}"
+    bindPort: {{ kube_apiserver_port }}
+  certificateKey: {{ kubeadm_certificate_key }}
+nodeRegistration:
+  name: {{ kube_override_hostname | default(inventory_hostname) }}
+  criSocket: {{ cri_socket }}
+{% if 'kube_control_plane' in group_names and 'kube_node' not in group_names %}
+  taints:
+  - effect: NoSchedule
+    key: node-role.kubernetes.io/control-plane
+{% else %}
+  taints: []
+{% endif %}
+{% if kubeadm_patches | length > 0 %}
+patches:
+  directory: {{ kubeadm_patches_dir }}
+{% endif %}

internal/resources/kubespray/project/roles/kubernetes/control-plane/templates/kubescheduler-config.yaml.j2

@@ -0,0 +1,24 @@
+apiVersion: kubescheduler.config.k8s.io/v1
+kind: KubeSchedulerConfiguration
+clientConnection:
+  kubeconfig: "{{ kube_config_dir }}/scheduler.conf"
+{% for key in kube_scheduler_client_conn_extra_opts %}
+  {{ key }}: {{ kube_scheduler_client_conn_extra_opts[key] }}
+{% endfor %}
+{% if kube_scheduler_extenders %}
+extenders:
+{{ kube_scheduler_extenders | to_nice_yaml(indent=2, width=256) }}
+{% endif %}
+leaderElection:
+  leaseDuration: {{ kube_scheduler_leader_elect_lease_duration }}
+  renewDeadline: {{ kube_scheduler_leader_elect_renew_deadline }}
+{% for key in kube_scheduler_leader_elect_extra_opts %}
+  {{ key }}: {{ kube_scheduler_leader_elect_extra_opts[key] }}
+{% endfor %}
+{% if kube_scheduler_profiles %}
+profiles:
+{{ kube_scheduler_profiles | to_nice_yaml(indent=2, width=256) }}
+{% endif %}
+{% for key in kube_scheduler_config_extra_opts %}
+{{ key }}: {{ kube_scheduler_config_extra_opts[key] }}
+{% endfor %}

internal/resources/kubespray/project/roles/kubernetes/control-plane/templates/podnodeselector.yaml.j2

@@ -0,0 +1,2 @@
+podNodeSelectorPluginConfig:
+  clusterDefaultNodeSelector: {{ kube_apiserver_admission_plugins_podnodeselector_default_node_selector }}

internal/resources/kubespray/project/roles/kubernetes/control-plane/templates/podsecurity.yaml.j2

@@ -0,0 +1,17 @@
+{% if kube_pod_security_use_default %}
+apiVersion: pod-security.admission.config.k8s.io/v1
+kind: PodSecurityConfiguration
+defaults:
+  enforce: "{{ kube_pod_security_default_enforce }}"
+  enforce-version: "{{ kube_pod_security_default_enforce_version }}"
+  audit: "{{ kube_pod_security_default_audit }}"
+  audit-version: "{{ kube_pod_security_default_audit_version }}"
+  warn: "{{ kube_pod_security_default_warn }}"
+  warn-version: "{{ kube_pod_security_default_warn_version }}"
+exemptions:
+  usernames: {{ kube_pod_security_exemptions_usernames | to_json }}
+  runtimeClasses: {{ kube_pod_security_exemptions_runtime_class_names | to_json }}
+  namespaces: {{ kube_pod_security_exemptions_namespaces | to_json }}
+{% else %}
+# This file is intentinally left empty as kube_pod_security_use_default={{ kube_pod_security_use_default }}
+{% endif %}

internal/resources/kubespray/project/roles/kubernetes/control-plane/templates/resourcequota.yaml.j2

@@ -0,0 +1,8 @@
+apiVersion: apiserver.config.k8s.io/v1
+kind: ResourceQuotaConfiguration
+{% if kube_resource_quota_limited_resources | d(false) -%}
+limitedResources:
+{{ kube_resource_quota_limited_resources | to_nice_yaml(indent=2, sort_keys=false) }}
+{% else %}
+# No limitedResources configured. If limitedResources are required, please set kube_resource_quota_limited_resources.
+{%- endif %}

internal/resources/kubespray/project/roles/kubernetes/control-plane/templates/secrets_encryption.yaml.j2

@@ -0,0 +1,11 @@
+apiVersion: apiserver.config.k8s.io/v1
+kind: EncryptionConfiguration
+resources:
+  - resources:
+{{ kube_encryption_resources | to_nice_yaml | indent(4, True) }}
+    providers:
+    - {{ kube_encryption_algorithm }}:
+        keys:
+        - name: key
+          secret: {{ kube_encrypt_token | b64encode }}
+    - identity: {}

internal/resources/kubespray/project/roles/kubernetes/control-plane/templates/webhook-authorization-config.yaml.j2

@@ -0,0 +1,18 @@
+# clusters refers to the remote service.
+clusters:
+- name: webhook-token-authz-cluster
+  cluster:
+    server: {{ kube_webhook_authorization_url }}
+    insecure-skip-tls-verify: {{ kube_webhook_authorization_url_skip_tls_verify }}
+
+# users refers to the API server's webhook configuration.
+users:
+- name: webhook-token-authz-user
+
+# kubeconfig files require a context. Provide one for the API server.
+current-context: webhook-token-authz
+contexts:
+- context:
+    cluster: webhook-token-authz-cluster
+    user: webhook-token-authz-user
+  name: webhook-token-authz

internal/resources/kubespray/project/roles/kubernetes/control-plane/templates/webhook-token-auth-config.yaml.j2

@@ -0,0 +1,21 @@
+# clusters refers to the remote service.
+clusters:
+- name: webhook-token-auth-cluster
+  cluster:
+    server: {{ kube_webhook_token_auth_url }}
+    insecure-skip-tls-verify: {{ kube_webhook_token_auth_url_skip_tls_verify }}
+{% if kube_webhook_token_auth_ca_data is defined %}
+    certificate-authority-data: {{ kube_webhook_token_auth_ca_data }}
+{% endif %}
+
+# users refers to the API server's webhook configuration.
+users:
+- name: webhook-token-auth-user
+
+# kubeconfig files require a context. Provide one for the API server.
+current-context: webhook-token-auth
+contexts:
+- context:
+    cluster: webhook-token-auth-cluster
+    user: webhook-token-auth-user
+  name: webhook-token-auth

internal/resources/kubespray/project/roles/kubernetes/control-plane/vars/main.yaml

@@ -0,0 +1,9 @@
+---
+# list of admission plugins that needs to be configured
+# https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/
+kube_apiserver_admission_plugins_needs_configuration:
+- EventRateLimit
+- ImagePolicyWebhook
+- PodSecurity
+- PodNodeSelector
+- ResourceQuota

internal/resources/kubespray/project/roles/kubernetes/kubeadm/defaults/main.yml

@@ -0,0 +1,8 @@
+---
+# discovery_timeout modifies the discovery timeout
+# This value must be smaller than kubeadm_join_timeout
+discovery_timeout: 60s
+kubeadm_join_timeout: 120s
+
+# Enable kubeadm file discovery if anonymous access has been removed
+kubeadm_use_file_discovery: "{{ remove_anonymous_access }}"

internal/resources/kubespray/project/roles/kubernetes/kubeadm/handlers/main.yml

@@ -0,0 +1,11 @@
+---
+- name: Kubeadm | reload systemd
+  systemd_service:
+    daemon_reload: true
+  listen: Kubeadm | restart kubelet
+
+- name: Kubeadm | reload kubelet
+  service:
+    name: kubelet
+    state: restarted
+  listen: Kubeadm | restart kubelet

internal/resources/kubespray/project/roles/kubernetes/kubeadm/meta/main.yml

@@ -0,0 +1,3 @@
+---
+dependencies:
+  - role: kubernetes/kubeadm_common

internal/resources/kubespray/project/roles/kubernetes/kubeadm/tasks/kubeadm_etcd_node.yml

@@ -0,0 +1,63 @@
+---
+- name: Parse certificate key if not set
+  set_fact:
+    kubeadm_certificate_key: "{{ hostvars[groups['kube_control_plane'][0]]['kubeadm_certificate_key'] }}"
+  when: kubeadm_certificate_key is undefined
+
+- name: Create kubeadm cert controlplane config
+  template:
+    src: "kubeadm-client.conf.j2"
+    dest: "{{ kube_config_dir }}/kubeadm-cert-controlplane.conf"
+    mode: "0640"
+    validate: "{{ kubeadm_config_validate_enabled | ternary(bin_dir + '/kubeadm config validate --config %s', omit) }}"
+  vars:
+    kubeadm_cert_controlplane: true
+
+- name: Pull control plane certs down
+  shell: >-
+    {{ bin_dir }}/kubeadm join phase
+    control-plane-prepare download-certs
+    --config {{ kube_config_dir }}/kubeadm-cert-controlplane.conf
+    &&
+    {{ bin_dir }}/kubeadm join phase
+    control-plane-prepare certs
+    --config {{ kube_config_dir }}/kubeadm-cert-controlplane.conf
+  args:
+    creates: "{{ kube_cert_dir }}/apiserver-etcd-client.key"
+
+- name: Delete unneeded certificates
+  file:
+    path: "{{ item }}"
+    state: absent
+  with_items:
+    - "{{ kube_cert_dir }}/apiserver.crt"
+    - "{{ kube_cert_dir }}/apiserver.key"
+    - "{{ kube_cert_dir }}/ca.key"
+    - "{{ kube_cert_dir }}/etcd/ca.key"
+    - "{{ kube_cert_dir }}/etcd/healthcheck-client.crt"
+    - "{{ kube_cert_dir }}/etcd/healthcheck-client.key"
+    - "{{ kube_cert_dir }}/etcd/peer.crt"
+    - "{{ kube_cert_dir }}/etcd/peer.key"
+    - "{{ kube_cert_dir }}/etcd/server.crt"
+    - "{{ kube_cert_dir }}/etcd/server.key"
+    - "{{ kube_cert_dir }}/front-proxy-ca.crt"
+    - "{{ kube_cert_dir }}/front-proxy-ca.key"
+    - "{{ kube_cert_dir }}/front-proxy-client.crt"
+    - "{{ kube_cert_dir }}/front-proxy-client.key"
+    - "{{ kube_cert_dir }}/sa.key"
+    - "{{ kube_cert_dir }}/sa.pub"
+
+- name: Calculate etcd cert serial
+  command: "openssl x509 -in {{ kube_cert_dir }}/apiserver-etcd-client.crt -noout -serial"
+  register: "etcd_client_cert_serial_result"
+  changed_when: false
+  when:
+    - group_names | intersect(['k8s_cluster', 'calico_rr']) | length > 0
+  tags:
+    - network
+
+- name: Set etcd_client_cert_serial
+  set_fact:
+    etcd_client_cert_serial: "{{ etcd_client_cert_serial_result.stdout.split('=')[1] }}"
+  tags:
+    - network

internal/resources/kubespray/project/roles/kubernetes/kubeadm/tasks/main.yml

@@ -0,0 +1,205 @@
+---
+- name: Set kubeadm_discovery_address
+  set_fact:
+    # noqa: jinja[spacing]
+    kubeadm_discovery_address: >-
+      {%- if "127.0.0.1" in kube_apiserver_endpoint or "localhost" in kube_apiserver_endpoint -%}
+      {{ first_kube_control_plane_address | ansible.utils.ipwrap }}:{{ kube_apiserver_port }}
+      {%- else -%}
+      {{ kube_apiserver_endpoint | replace("https://", "") }}
+      {%- endif %}
+  tags:
+    - facts
+
+- name: Check if kubelet.conf exists
+  stat:
+    path: "{{ kube_config_dir }}/kubelet.conf"
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: kubelet_conf
+
+- name: Check if kubeadm CA cert is accessible
+  stat:
+    path: "{{ kube_cert_dir }}/ca.crt"
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: kubeadm_ca_stat
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+  run_once: true
+
+- name: Fetch CA certificate from control plane node
+  slurp:
+    src: "{{ kube_cert_dir }}/ca.crt"
+  register: ca_cert_content
+  when:
+    - kubeadm_ca_stat.stat is defined
+    - kubeadm_ca_stat.stat.exists
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+  run_once: true
+
+- name: Create kubeadm token for joining nodes with 24h expiration (default)
+  command: "{{ bin_dir }}/kubeadm token create"
+  register: temp_token
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+  when: kubeadm_token is not defined
+  changed_when: false
+
+- name: Set kubeadm_token to generated token
+  set_fact:
+    kubeadm_token: "{{ temp_token.stdout }}"
+  when: kubeadm_token is not defined
+
+- name: Get kubeconfig for join discovery process
+  command: "{{ kubectl }} -n kube-public get cm cluster-info -o jsonpath='{.data.kubeconfig}'"
+  register: kubeconfig_file_discovery
+  run_once: true
+  delegate_to: "{{ groups['kube_control_plane'] | first }}"
+  when: kubeadm_use_file_discovery
+
+- name: Copy discovery kubeconfig
+  copy:
+    dest: "{{ kube_config_dir }}/cluster-info-discovery-kubeconfig.yaml"
+    content: "{{ kubeconfig_file_discovery.stdout }}"
+    owner: "root"
+    mode: "0644"
+  when:
+    - ('kube_control_plane' not in group_names)
+    - not kubelet_conf.stat.exists
+    - kubeadm_use_file_discovery
+
+- name: Create kubeadm client config
+  template:
+    src: "kubeadm-client.conf.j2"
+    dest: "{{ kube_config_dir }}/kubeadm-client.conf"
+    backup: true
+    mode: "0640"
+    validate: "{{ kubeadm_config_validate_enabled | ternary(bin_dir + '/kubeadm config validate --config %s', omit) }}"
+  when: ('kube_control_plane' not in group_names)
+
+- name: Join to cluster if needed
+  environment:
+    PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}:/sbin"
+  when:
+    - ('kube_control_plane' not in group_names)
+    - not kubelet_conf.stat.exists
+  vars:
+    ignored:
+      - DirAvailable--etc-kubernetes-manifests
+      - "{{ kubeadm_ignore_preflight_errors }}"
+  command: >-
+    timeout -k {{ kubeadm_join_timeout }} {{ kubeadm_join_timeout }}
+    {{ bin_dir }}/kubeadm join
+    --config {{ kube_config_dir }}/kubeadm-client.conf
+    --ignore-preflight-errors={{ ignored | flatten | join(',') }}
+    --skip-phases={{ kubeadm_join_phases_skip | join(',') }}
+
+- name: Update server field in kubelet kubeconfig
+  lineinfile:
+    dest: "{{ kube_config_dir }}/kubelet.conf"
+    regexp: 'server:'
+    line: '    server: {{ kube_apiserver_endpoint }}'
+    backup: true
+  when:
+    - kubeadm_config_api_fqdn is not defined
+    - ('kube_control_plane' not in group_names)
+    - kubeadm_discovery_address != kube_apiserver_endpoint | replace("https://", "")
+  notify: Kubeadm | restart kubelet
+
+- name: Update server field in kubelet kubeconfig - external lb
+  lineinfile:
+    dest: "{{ kube_config_dir }}/kubelet.conf"
+    regexp: '^    server: https'
+    line: '    server: {{ kube_apiserver_endpoint }}'
+    backup: true
+  when:
+    - ('kube_control_plane' not in group_names)
+    - loadbalancer_apiserver is defined
+  notify: Kubeadm | restart kubelet
+
+- name: Get current resourceVersion of kube-proxy configmap
+  command: "{{ kubectl }} get configmap kube-proxy -n kube-system -o jsonpath='{.metadata.resourceVersion}'"
+  register: original_configmap_resource_version
+  run_once: true
+  delegate_to: "{{ groups['kube_control_plane'] | first }}"
+  delegate_facts: false
+  when:
+    - kube_proxy_deployed
+  tags:
+    - kube-proxy
+
+# FIXME(mattymo): Need to point to localhost, otherwise control plane nodes will all point
+#                 incorrectly to first control plane node, creating SPoF.
+- name: Update server field in kube-proxy kubeconfig
+  shell: >-
+    set -o pipefail && {{ kubectl }} get configmap kube-proxy -n kube-system -o yaml
+    | sed 's#server:.*#server: https://127.0.0.1:{{ kube_apiserver_port }}#g'
+    | {{ kubectl }} replace -f -
+  args:
+    executable: /bin/bash
+  run_once: true
+  delegate_to: "{{ groups['kube_control_plane'] | first }}"
+  delegate_facts: false
+  when:
+    - kubeadm_config_api_fqdn is not defined
+    - kubeadm_discovery_address != kube_apiserver_endpoint | replace("https://", "")
+    - kube_proxy_deployed
+    - loadbalancer_apiserver_localhost
+  tags:
+    - kube-proxy
+
+- name: Update server field in kube-proxy kubeconfig - external lb
+  shell: >-
+    set -o pipefail && {{ kubectl }} get configmap kube-proxy -n kube-system -o yaml
+    | sed 's#server:.*#server: {{kube_apiserver_endpoint}}#g'
+    | {{ kubectl }} replace -f -
+  args:
+    executable: /bin/bash
+  run_once: true
+  delegate_to: "{{ groups['kube_control_plane'] | first }}"
+  delegate_facts: false
+  when:
+    - kube_proxy_deployed
+    - loadbalancer_apiserver is defined
+  tags:
+    - kube-proxy
+
+- name: Get new resourceVersion of kube-proxy configmap
+  command: "{{ kubectl }} get configmap kube-proxy -n kube-system -o jsonpath='{.metadata.resourceVersion}'"
+  register: new_configmap_resource_version
+  run_once: true
+  delegate_to: "{{ groups['kube_control_plane'] | first }}"
+  delegate_facts: false
+  when:
+    - kube_proxy_deployed
+  tags:
+    - kube-proxy
+
+- name: Set ca.crt file permission
+  file:
+    path: "{{ kube_cert_dir }}/ca.crt"
+    owner: root
+    group: root
+    mode: "0644"
+
+- name: Restart all kube-proxy pods to ensure that they load the new configmap
+  command: "{{ kubectl }} delete pod -n kube-system -l k8s-app=kube-proxy --force --grace-period=0"
+  run_once: true
+  delegate_to: "{{ groups['kube_control_plane'] | first }}"
+  delegate_facts: false
+  when:
+    - kubeadm_config_api_fqdn is not defined or loadbalancer_apiserver is defined
+    - kubeadm_discovery_address != kube_apiserver_endpoint | replace("https://", "") or loadbalancer_apiserver is defined
+    - kube_proxy_deployed
+    - original_configmap_resource_version.stdout != new_configmap_resource_version.stdout
+  tags:
+    - kube-proxy
+
+- name: Extract etcd certs from control plane if using etcd kubeadm mode
+  include_tasks: kubeadm_etcd_node.yml
+  when:
+    - etcd_deployment_type == "kubeadm"
+    - inventory_hostname not in groups['kube_control_plane']
+    - kube_network_plugin in ["calico", "flannel", "cilium"] or cilium_deploy_additionally
+    - kube_network_plugin != "calico" or calico_datastore == "etcd"

internal/resources/kubespray/project/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.j2

@@ -0,0 +1,50 @@
+---
+apiVersion: kubeadm.k8s.io/{{ kubeadm_config_api_version }}
+kind: JoinConfiguration
+discovery:
+{% if kubeadm_use_file_discovery %}
+  file:
+    kubeConfigPath: {{ kube_config_dir }}/cluster-info-discovery-kubeconfig.yaml
+{% else %}
+  bootstrapToken:
+{% if kubeadm_config_api_fqdn is defined %}
+    apiServerEndpoint: "{{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}"
+{% else %}
+    apiServerEndpoint: "{{ kubeadm_discovery_address }}"
+{% endif %}
+    token: {{ kubeadm_token }}
+{% if ca_cert_content is defined %}
+    caCertHashes:
+    - sha256:{{ (ca_cert_content.content | b64decode | community.crypto.x509_certificate_info).public_key_fingerprints.sha256.replace(':', '') }}
+{% else %}
+    unsafeSkipCAVerification: true
+{% endif %}
+{% endif %}
+  tlsBootstrapToken: {{ kubeadm_token }}
+{# TODO: drop the if when we drop support for k8s<1.31 #}
+{% if kubeadm_config_api_version == 'v1beta3' %}
+  timeout: {{ discovery_timeout }}
+{% else %}
+timeouts:
+  discovery: {{ discovery_timeout }}
+{% endif %}
+caCertPath: {{ kube_cert_dir }}/ca.crt
+{% if kubeadm_cert_controlplane is defined and kubeadm_cert_controlplane %}
+controlPlane:
+  localAPIEndpoint:
+    advertiseAddress: "{{ kube_apiserver_address }}"
+    bindPort: {{ kube_apiserver_port }}
+  certificateKey: {{ kubeadm_certificate_key }}
+{% endif %}
+nodeRegistration:
+  name: '{{ kube_override_hostname }}'
+  criSocket: {{ cri_socket }}
+{% if 'calico_rr' in group_names and 'kube_node' not in group_names %}
+  taints:
+  - effect: NoSchedule
+    key: node-role.kubernetes.io/calico-rr
+{% endif %}
+{% if kubeadm_patches | length > 0 %}
+patches:
+  directory: {{ kubeadm_patches_dir }}
+{% endif %}

internal/resources/kubespray/project/roles/kubernetes/kubeadm_common/defaults/main.yml

@@ -0,0 +1,23 @@
+---
+kubeadm_patches_dir: "{{ kube_config_dir }}/patches"
+kubeadm_patches: []
+# See https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/control-plane-flags/#patches
+# Correspondance with this link
+# patchtype = type
+# target = target
+# suffix -> managed automatically
+# extension -> always "yaml"
+# kubeadm_patches:
+# - target: kube-apiserver|kube-controller-manager|kube-scheduler|etcd|kubeletconfiguration
+#   type: strategic(default)|json|merge
+#   patch:
+#    metadata:
+#      annotations:
+#        example.com/test: "true"
+#      labels:
+#        example.com/prod_level: "{{ prod_level }}"
+# - ...
+# Patches are applied in the order they are specified.
+
+# List of errors to ignore during kubeadm preflight checks
+kubeadm_ignore_preflight_errors: []

internal/resources/kubespray/project/roles/kubernetes/kubeadm_common/tasks/main.yml

@@ -0,0 +1,17 @@
+---
+- name: Kubeadm | Create directory to store kubeadm patches
+  file:
+    path: "{{ kubeadm_patches_dir }}"
+    state: directory
+    mode: "0640"
+  when: kubeadm_patches | length > 0
+
+- name: Kubeadm | Copy kubeadm patches from inventory files
+  copy:
+    content: "{{ item.patch | to_yaml }}"
+    dest: "{{ kubeadm_patches_dir }}/{{ item.target }}{{ suffix }}+{{ item.type | d('strategic') }}.yaml"
+    owner: "root"
+    mode: "0644"
+  loop: "{{ kubeadm_patches }}"
+  loop_control:
+    index_var: suffix

internal/resources/kubespray/project/roles/kubernetes/node-label/tasks/main.yml

@@ -0,0 +1,49 @@
+---
+- name: Kubernetes Apps | Wait for kube-apiserver
+  uri:
+    url: "{{ kube_apiserver_endpoint }}/healthz"
+    validate_certs: false
+    client_cert: "{{ kube_apiserver_client_cert }}"
+    client_key: "{{ kube_apiserver_client_key }}"
+  register: result
+  until: result.status == 200
+  retries: 10
+  delay: 6
+  when: inventory_hostname == groups['kube_control_plane'][0]
+
+- name: Set role node label to empty list
+  set_fact:
+    role_node_labels: []
+
+- name: Node label for nvidia GPU nodes
+  set_fact:
+    role_node_labels: "{{ role_node_labels + ['nvidia.com/gpu=true'] }}"
+  when:
+    - nvidia_gpu_nodes is defined
+    - nvidia_accelerator_enabled | bool
+    - inventory_hostname in nvidia_gpu_nodes
+
+- name: Set inventory node label to empty list
+  set_fact:
+    inventory_node_labels: []
+
+- name: Populate inventory node label
+  set_fact:
+    inventory_node_labels: "{{ inventory_node_labels + ['%s=%s' | format(item.key, item.value)] }}"
+  loop: "{{ node_labels | d({}) | dict2items }}"
+  when:
+    - node_labels is defined
+    - node_labels is mapping
+
+- debug:  # noqa name[missing]
+    var: role_node_labels
+- debug:  # noqa name[missing]
+    var: inventory_node_labels
+
+- name: Set label to node
+  command: >-
+      {{ kubectl }} label node {% if kube_override_hostname %}{{ kube_override_hostname }}{% else %}{{ inventory_hostname }}{% endif %} {{ item }} --overwrite=true
+  loop: "{{ role_node_labels + inventory_node_labels }}"
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+  changed_when: false
+...

internal/resources/kubespray/project/roles/kubernetes/node-taint/tasks/main.yml

@@ -0,0 +1,35 @@
+---
+- name: Set role and inventory node taint to empty list
+  set_fact:
+    role_node_taints: []
+    inventory_node_taints: []
+
+- name: Node taint for nvidia GPU nodes
+  set_fact:
+    role_node_taints: "{{ role_node_taints + ['nvidia.com/gpu=:NoSchedule'] }}"
+  when:
+    - nvidia_gpu_nodes is defined
+    - nvidia_accelerator_enabled | bool
+    - inventory_hostname in nvidia_gpu_nodes
+
+- name: Populate inventory node taint
+  set_fact:
+    inventory_node_taints: "{{ inventory_node_taints + ['%s' | format(item)] }}"
+  loop: "{{ node_taints | d([]) }}"
+  when:
+    - node_taints is defined
+    - node_taints is not string
+    - node_taints is not mapping
+    - node_taints is iterable
+- debug:  # noqa name[missing]
+    var: role_node_taints
+- debug:  # noqa name[missing]
+    var: inventory_node_taints
+
+- name: Set taint to node
+  command: >-
+      {{ kubectl }} taint node {{ kube_override_hostname | default(inventory_hostname) }} {{ (role_node_taints + inventory_node_taints) | join(' ') }} --overwrite=true
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+  changed_when: false
+  when:
+    - (role_node_taints + inventory_node_taints) | length > 0

internal/resources/kubespray/project/roles/kubernetes/node/defaults/main.yml

@@ -0,0 +1,198 @@
+---
+# advertised host IP for kubelet. This affects network plugin config. Take caution
+# add ipv6 manual for dualstack mode because ipv4 priority in main_ip for dualstack
+kubelet_address: "{{ main_ips | join(',') }}"
+
+# bind address for kubelet. Set to :: to listen on all interfaces
+kubelet_bind_address: "{{ main_ip | default('::') }}"
+
+# resolv.conf to base dns config
+kube_resolv_conf: "/etc/resolv.conf"
+
+# Set to empty to avoid cgroup creation
+kubelet_enforce_node_allocatable: "\"\""
+
+# Set runtime and kubelet cgroups when using systemd as cgroup driver (default)
+kube_service_cgroups: "{% if kube_reserved %}{{ kube_reserved_cgroups_for_service_slice }}{% else %}system.slice{% endif %}"
+kubelet_runtime_cgroups: "/{{ kube_service_cgroups }}/{{ container_manager }}.service"
+kubelet_kubelet_cgroups: "/{{ kube_service_cgroups }}/kubelet.service"
+
+# Set runtime and kubelet cgroups when using cgroupfs as cgroup driver
+kubelet_runtime_cgroups_cgroupfs: "/system.slice/{{ container_manager }}.service"
+kubelet_kubelet_cgroups_cgroupfs: "/system.slice/kubelet.service"
+
+# Set systemd service hardening features
+kubelet_systemd_hardening: false
+
+# Kubelet service dependencies other than container runtime
+kubelet_systemd_wants_dependencies: []
+
+# List of secure IPs for kubelet
+# don't forget ipv6 addresses for dualstack(because "main_ip" prioritizes ipv4)
+kube_node_addresses: >-
+  {%- for host in (groups['k8s_cluster'] | union(groups['etcd'])) -%}
+    {{ hostvars[host]['main_ips'] | join(' ') }}{{ ' ' if not loop.last else '' }}
+  {%- endfor -%}
+kubelet_secure_addresses: "localhost link-local {{ kube_pods_subnets | regex_replace(',', ' ') }} {{ kube_node_addresses }}"
+
+# Reserve this space for kube resources
+# Whether to run kubelet and container-engine daemons in a dedicated cgroup. (Not required for resource reservations).
+kube_reserved: false
+kube_reserved_cgroups: "/{{ kube_reserved_cgroups_for_service_slice }}"
+kube_memory_reserved: "256Mi"
+kube_cpu_reserved: "100m"
+kube_ephemeral_storage_reserved: "500Mi"
+kube_pid_reserved: "1000"
+
+# Set to true to reserve resources for system daemons
+system_reserved: false
+system_reserved_cgroups_for_service_slice: system.slice
+system_reserved_cgroups: "/{{ system_reserved_cgroups_for_service_slice }}"
+system_memory_reserved: "512Mi"
+system_cpu_reserved: "500m"
+system_ephemeral_storage_reserved: "500Mi"
+system_pid_reserved: 1000
+
+## Eviction Thresholds to avoid system OOMs
+# https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/#eviction-thresholds
+eviction_hard: {}
+eviction_hard_control_plane: {}
+
+kubelet_status_update_frequency: 10s
+
+# kube-vip
+kube_vip_version: 0.8.0
+
+kube_vip_arp_enabled: false
+kube_vip_interface:
+kube_vip_services_interface:
+kube_vip_cidr: 32
+kube_vip_dns_mode: first
+kube_vip_controlplane_enabled: false
+kube_vip_ddns_enabled: false
+kube_vip_cp_detect: false
+kube_vip_services_enabled: false
+kube_vip_leader_election_enabled: "{{ kube_vip_arp_enabled }}"
+kube_vip_bgp_enabled: false
+kube_vip_bgp_routerid:
+kube_vip_local_as: 65000
+kube_vip_bgp_peeraddress:
+kube_vip_bgp_peerpass:
+kube_vip_bgp_peeras: 65000
+kube_vip_bgppeers:
+kube_vip_address:
+kube_vip_enableServicesElection: false
+kube_vip_lb_enable: false
+kube_vip_leasename: plndr-cp-lock
+kube_vip_svc_leasename: plndr-svcs-lock
+kube_vip_leaseduration: 5
+kube_vip_renewdeadline: 3
+kube_vip_retryperiod: 1
+kube_vip_enable_node_labeling: false
+
+# Requests for load balancer app
+loadbalancer_apiserver_memory_requests: 32M
+loadbalancer_apiserver_cpu_requests: 25m
+
+loadbalancer_apiserver_keepalive_timeout: 5m
+loadbalancer_apiserver_pod_name: "{% if loadbalancer_apiserver_type == 'nginx' %}nginx-proxy{% else %}haproxy{% endif %}"
+
+# Uncomment if you need to enable deprecated runtimes
+# kube_api_runtime_config:
+#   - apps/v1beta1=true
+#   - apps/v1beta2=true
+#   - extensions/v1beta1/daemonsets=true
+#   - extensions/v1beta1/deployments=true
+#   - extensions/v1beta1/replicasets=true
+#   - extensions/v1beta1/networkpolicies=true
+
+# A port range to reserve for services with NodePort visibility.
+# Inclusive at both ends of the range.
+kube_apiserver_node_port_range: "30000-32767"
+
+# Configure the amount of pods able to run on single node
+# default is equal to application default
+kubelet_max_pods: 110
+
+# Sets the maximum number of processes running per Pod
+# Default value -1 = unlimited
+kubelet_pod_pids_limit: -1
+
+## Support parameters to be passed to kubelet via kubelet-config.yaml
+kubelet_config_extra_args: {}
+
+## Parameters to be passed to kubelet via kubelet-config.yaml when cgroupfs is used as cgroup driver
+kubelet_config_extra_args_cgroupfs:
+  systemCgroups: /system.slice
+  cgroupRoot: /
+
+# Maximum number of container log files that can be present for a container.
+kubelet_logfiles_max_nr: 5
+
+# Maximum size of the container log file before it is rotated
+kubelet_logfiles_max_size: 10Mi
+
+## Support custom flags to be passed to kubelet
+kubelet_custom_flags: []
+
+# The read-only port for the Kubelet to serve on with no authentication/authorization.
+kube_read_only_port: 0
+
+# Port for healthz for Kubelet
+kubelet_healthz_port: 10248
+
+# Bind address for healthz for Kubelet
+kubelet_healthz_bind_address: 127.0.0.1
+
+# sysctl_file_path to add sysctl conf to
+sysctl_file_path: "/etc/sysctl.d/99-sysctl.conf"
+
+## Support tls min version, Possible values: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13.
+# tls_min_version: ""
+
+## Support tls cipher suites.
+# tls_cipher_suites:
+#   - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA
+#   - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256
+#   - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256
+#   - TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA
+#   - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
+#   - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256
+#   - TLS_ECDHE_ECDSA_WITH_RC4_128_SHA
+#   - TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA
+#   - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA
+#   - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256
+#   - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256
+#   - TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA
+#   - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384
+#   - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256
+#   - TLS_ECDHE_RSA_WITH_RC4_128_SHA
+#   - TLS_RSA_WITH_3DES_EDE_CBC_SHA
+#   - TLS_RSA_WITH_AES_128_CBC_SHA
+#   - TLS_RSA_WITH_AES_128_CBC_SHA256
+#   - TLS_RSA_WITH_AES_128_GCM_SHA256
+#   - TLS_RSA_WITH_AES_256_CBC_SHA
+#   - TLS_RSA_WITH_AES_256_GCM_SHA384
+#   - TLS_RSA_WITH_RC4_128_SHA
+
+kube_proxy_ipvs_modules:
+  - ip_vs
+  - ip_vs_rr
+  - ip_vs_wrr
+  - ip_vs_sh
+  - ip_vs_wlc
+  - ip_vs_lc
+
+# Kubespray will use the first module of this list which it can successfully modprobe
+conntrack_modules:
+  - nf_conntrack
+  - nf_conntrack_ipv4
+
+
+## Enable distributed tracing for kubelet
+kubelet_tracing: false
+kubelet_tracing_endpoint: "[::]:4317"
+kubelet_tracing_sampling_rate_per_million: 100
+
+# The maximum number of image pulls in parallel. Set it to a integer great than 1 to enable image pulling in parallel.
+kubelet_max_parallel_image_pulls: 1

internal/resources/kubespray/project/roles/kubernetes/node/handlers/main.yml

@@ -0,0 +1,11 @@
+---
+- name: Kubelet | reload systemd
+  systemd_service:
+    daemon_reload: true
+  listen: Node | restart kubelet
+
+- name: Kubelet | restart kubelet
+  service:
+    name: kubelet
+    state: restarted
+  listen: Node | restart kubelet

internal/resources/kubespray/project/roles/kubernetes/node/tasks/facts.yml

@@ -0,0 +1,62 @@
+---
+- name: Gather cgroups facts for docker
+  when: container_manager == 'docker'
+  block:
+  - name: Look up docker cgroup driver
+    shell: "set -o pipefail && docker info | grep 'Cgroup Driver' | awk -F': ' '{ print $2; }'"
+    args:
+      executable: /bin/bash
+    register: docker_cgroup_driver_result
+    changed_when: false
+    check_mode: false
+
+  - name: Set kubelet_cgroup_driver_detected fact for docker
+    set_fact:
+      kubelet_cgroup_driver_detected: "{{ docker_cgroup_driver_result.stdout }}"
+
+- name: Gather cgroups facts for crio
+  when: container_manager == 'crio'
+  block:
+  - name: Look up crio cgroup driver
+    shell: "set -o pipefail && {{ bin_dir }}/{{ crio_status_command }} info | grep 'cgroup driver' | awk -F': ' '{ print $2; }'"
+    args:
+      executable: /bin/bash
+    register: crio_cgroup_driver_result
+    changed_when: false
+
+  - name: Set kubelet_cgroup_driver_detected fact for crio
+    set_fact:
+      kubelet_cgroup_driver_detected: "{{ crio_cgroup_driver_result.stdout }}"
+
+- name: Set kubelet_cgroup_driver_detected fact for containerd
+  when: container_manager == 'containerd'
+  set_fact:
+    kubelet_cgroup_driver_detected: >-
+      {%- if containerd_use_systemd_cgroup -%}systemd{%- else -%}cgroupfs{%- endif -%}
+
+- name: Set kubelet_cgroup_driver
+  set_fact:
+    kubelet_cgroup_driver: "{{ kubelet_cgroup_driver_detected }}"
+  when: kubelet_cgroup_driver is undefined
+
+- name: Set kubelet_cgroups options when cgroupfs is used
+  set_fact:
+    kubelet_runtime_cgroups: "{{ kubelet_runtime_cgroups_cgroupfs }}"
+    kubelet_kubelet_cgroups: "{{ kubelet_kubelet_cgroups_cgroupfs }}"
+  when: kubelet_cgroup_driver == 'cgroupfs'
+
+- name: Set kubelet_config_extra_args options when cgroupfs is used
+  set_fact:
+    kubelet_config_extra_args: "{{ kubelet_config_extra_args | combine(kubelet_config_extra_args_cgroupfs) }}"
+  when: kubelet_cgroup_driver == 'cgroupfs'
+
+- name: Os specific vars
+  include_vars: "{{ item }}"
+  with_first_found:
+  - files:
+    - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower | replace('/', '_') }}.yml"
+    - "{{ ansible_distribution | lower }}-{{ ansible_distribution_release }}.yml"
+    - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower | replace('/', '_') }}.yml"
+    - "{{ ansible_distribution | lower }}.yml"
+    - "{{ ansible_os_family | lower }}.yml"
+    skip: true

internal/resources/kubespray/project/roles/kubernetes/node/tasks/install.yml

@@ -0,0 +1,22 @@
+---
+- name: Install | Copy kubeadm binary from download dir
+  copy:
+    src: "{{ downloads.kubeadm.dest }}"
+    dest: "{{ bin_dir }}/kubeadm"
+    mode: "0755"
+    remote_src: true
+  tags:
+    - kubeadm
+  when:
+    - not ('kube_control_plane' in group_names)
+
+- name: Install | Copy kubelet binary from download dir
+  copy:
+    src: "{{ downloads.kubelet.dest }}"
+    dest: "{{ bin_dir }}/kubelet"
+    mode: "0755"
+    remote_src: true
+  tags:
+    - kubelet
+    - upgrade
+  notify: Node | restart kubelet

internal/resources/kubespray/project/roles/kubernetes/node/tasks/kubelet.yml

@@ -0,0 +1,55 @@
+---
+- name: Set kubelet api version to v1beta1
+  set_fact:
+    kubeletConfig_api_version: v1beta1
+  tags:
+    - kubelet
+    - kubeadm
+
+- name: Write kubelet environment config file (kubeadm)
+  template:
+    src: "kubelet.env.{{ kubeletConfig_api_version }}.j2"
+    dest: "{{ kube_config_dir }}/kubelet.env"
+    setype: "{{ (preinstall_selinux_state != 'disabled') | ternary('etc_t', omit) }}"
+    backup: true
+    mode: "0600"
+  notify: Node | restart kubelet
+  tags:
+    - kubelet
+    - kubeadm
+
+- name: Write kubelet config file
+  template:
+    src: "kubelet-config.{{ kubeletConfig_api_version }}.yaml.j2"
+    dest: "{{ kube_config_dir }}/kubelet-config.yaml"
+    mode: "0600"
+  notify: Kubelet | restart kubelet
+  tags:
+    - kubelet
+    - kubeadm
+
+- name: Write kubelet systemd init file
+  template:
+    src: "kubelet.service.j2"
+    dest: "/etc/systemd/system/kubelet.service"
+    backup: true
+    mode: "0600"
+    validate: "sh -c '[ -f /usr/bin/systemd/system/factory-reset.target ] || exit 0 && systemd-analyze verify %s:kubelet.service'"
+    # FIXME: check that systemd version >= 250 (factory-reset.target was introduced in that release)
+    # Remove once we drop support for systemd < 250
+  notify: Node | restart kubelet
+  tags:
+    - kubelet
+    - kubeadm
+
+- name: Flush_handlers and reload-systemd
+  meta: flush_handlers
+
+- name: Enable kubelet
+  service:
+    name: kubelet
+    enabled: true
+    state: started
+  tags:
+    - kubelet
+  notify: Kubelet | restart kubelet

internal/resources/kubespray/project/roles/kubernetes/node/tasks/loadbalancer/haproxy.yml

@@ -0,0 +1,34 @@
+---
+- name: Haproxy | Cleanup potentially deployed nginx-proxy
+  file:
+    path: "{{ kube_manifest_dir }}/nginx-proxy.yml"
+    state: absent
+
+- name: Haproxy | Make haproxy directory
+  file:
+    path: "{{ haproxy_config_dir }}"
+    state: directory
+    mode: "0755"
+    owner: root
+
+- name: Haproxy | Write haproxy configuration
+  template:
+    src: "loadbalancer/haproxy.cfg.j2"
+    dest: "{{ haproxy_config_dir }}/haproxy.cfg"
+    owner: root
+    mode: "0755"
+    backup: true
+
+- name: Haproxy | Get checksum from config
+  stat:
+    path: "{{ haproxy_config_dir }}/haproxy.cfg"
+    get_attributes: false
+    get_checksum: true
+    get_mime: false
+  register: haproxy_stat
+
+- name: Haproxy | Write static pod
+  template:
+    src: manifests/haproxy.manifest.j2
+    dest: "{{ kube_manifest_dir }}/haproxy.yml"
+    mode: "0640"

internal/resources/kubespray/project/roles/kubernetes/node/tasks/loadbalancer/kube-vip.yml

@@ -0,0 +1,39 @@
+---
+- name: Kube-vip  | Check cluster settings for kube-vip
+  fail:
+    msg: "kube-vip require kube_proxy_strict_arp = true, see https://github.com/kube-vip/kube-vip/blob/main/docs/kubernetes/arp/index.md"
+  when:
+    - kube_proxy_mode == 'ipvs' and not kube_proxy_strict_arp
+    - kube_vip_arp_enabled
+
+- name: Kube-vip | Check if super-admin.conf exists
+  stat:
+    path: "{{ kube_config_dir }}/super-admin.conf"
+  failed_when: false
+  changed_when: false
+  register: stat_kube_vip_super_admin
+
+- name: Kube-vip | Check if kubeadm has already run
+  stat:
+    path: "/var/lib/kubelet/config.yaml"
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: kubeadm_already_run
+
+- name: Kube-vip | Set admin.conf
+  set_fact:
+    kube_vip_admin_conf: admin.conf
+
+- name: Kube-vip | Set admin.conf for first Control Plane
+  set_fact:
+    kube_vip_admin_conf: super-admin.conf
+  when:
+    - inventory_hostname == groups['kube_control_plane'] | first
+    - (stat_kube_vip_super_admin.stat.exists and stat_kube_vip_super_admin.stat.isreg) or (not kubeadm_already_run.stat.exists )
+
+- name: Kube-vip | Write static pod
+  template:
+    src: manifests/kube-vip.manifest.j2
+    dest: "{{ kube_manifest_dir }}/kube-vip.yml"
+    mode: "0640"

internal/resources/kubespray/project/roles/kubernetes/node/tasks/loadbalancer/nginx-proxy.yml

@@ -0,0 +1,34 @@
+---
+- name: Haproxy | Cleanup potentially deployed haproxy
+  file:
+    path: "{{ kube_manifest_dir }}/haproxy.yml"
+    state: absent
+
+- name: Nginx-proxy | Make nginx directory
+  file:
+    path: "{{ nginx_config_dir }}"
+    state: directory
+    mode: "0700"
+    owner: root
+
+- name: Nginx-proxy | Write nginx-proxy configuration
+  template:
+    src: "loadbalancer/nginx.conf.j2"
+    dest: "{{ nginx_config_dir }}/nginx.conf"
+    owner: root
+    mode: "0755"
+    backup: true
+
+- name: Nginx-proxy | Get checksum from config
+  stat:
+    path: "{{ nginx_config_dir }}/nginx.conf"
+    get_attributes: false
+    get_checksum: true
+    get_mime: false
+  register: nginx_stat
+
+- name: Nginx-proxy | Write static pod
+  template:
+    src: manifests/nginx-proxy.manifest.j2
+    dest: "{{ kube_manifest_dir }}/nginx-proxy.yml"
+    mode: "0640"

internal/resources/kubespray/project/roles/kubernetes/node/tasks/main.yml

@@ -0,0 +1,148 @@
+---
+- name: Fetch facts
+  import_tasks: facts.yml
+  tags:
+    - facts
+    - kubelet
+
+- name: Ensure /var/lib/cni exists
+  file:
+    path: /var/lib/cni
+    state: directory
+    mode: "0755"
+
+- name: Install kubelet binary
+  import_tasks: install.yml
+  tags:
+    - kubelet
+
+- name: Install kube-vip
+  import_tasks: loadbalancer/kube-vip.yml
+  when:
+    - ('kube_control_plane' in group_names)
+    - kube_vip_enabled
+  tags:
+    - kube-vip
+
+- name: Install nginx-proxy
+  import_tasks: loadbalancer/nginx-proxy.yml
+  when:
+    - ('kube_control_plane' not in group_names) or (kube_apiserver_bind_address != '::')
+    - loadbalancer_apiserver_localhost
+    - loadbalancer_apiserver_type == 'nginx'
+  tags:
+    - nginx
+
+- name: Install haproxy
+  import_tasks: loadbalancer/haproxy.yml
+  when:
+    - ('kube_control_plane' not in group_names) or (kube_apiserver_bind_address != '::')
+    - loadbalancer_apiserver_localhost
+    - loadbalancer_apiserver_type == 'haproxy'
+  tags:
+    - haproxy
+
+- name: Ensure nodePort range is reserved
+  ansible.posix.sysctl:
+    name: net.ipv4.ip_local_reserved_ports
+    value: "{{ kube_apiserver_node_port_range }}"
+    sysctl_set: true
+    sysctl_file: "{{ sysctl_file_path }}"
+    state: present
+    reload: true
+  when: kube_apiserver_node_port_range is defined
+  tags:
+    - kube-proxy
+
+- name: Verify if br_netfilter module exists
+  command: "modinfo br_netfilter"
+  environment:
+    PATH: "{{ ansible_env.PATH }}:/sbin"  # Make sure we can workaround RH's conservative path management
+  register: modinfo_br_netfilter
+  failed_when: modinfo_br_netfilter.rc not in [0, 1]
+  changed_when: false
+  check_mode: false
+
+# TODO: Remove once upstream issue is fixed
+# https://github.com/ansible-collections/community.general/issues/7717
+- name: Verify br_netfilter module path exists
+  file:
+    path: "{{ item }}"
+    state: directory
+    mode: "0755"
+  loop:
+    - /etc/modules-load.d
+    - /etc/modprobe.d
+
+- name: Enable br_netfilter module
+  community.general.modprobe:
+    name: br_netfilter
+    state: present
+  when: modinfo_br_netfilter.rc == 0
+
+- name: Persist br_netfilter module
+  copy:
+    dest: /etc/modules-load.d/kubespray-br_netfilter.conf
+    content: br_netfilter
+    mode: "0644"
+  when: modinfo_br_netfilter.rc == 0
+
+# kube-proxy needs net.bridge.bridge-nf-call-iptables enabled when found if br_netfilter is not a module
+- name: Check if bridge-nf-call-iptables key exists
+  command: "sysctl net.bridge.bridge-nf-call-iptables"
+  failed_when: false
+  changed_when: false
+  check_mode: false
+  register: sysctl_bridge_nf_call_iptables
+
+- name: Enable bridge-nf-call tables
+  ansible.posix.sysctl:
+    name: "{{ item }}"
+    state: present
+    sysctl_file: "{{ sysctl_file_path }}"
+    value: "1"
+    reload: true
+  when: sysctl_bridge_nf_call_iptables.rc == 0
+  with_items:
+    - net.bridge.bridge-nf-call-iptables
+    - net.bridge.bridge-nf-call-arptables
+    - net.bridge.bridge-nf-call-ip6tables
+
+- name: Modprobe Kernel Module for IPVS
+  community.general.modprobe:
+    name: "{{ item }}"
+    state: present
+    persistent: present
+  loop: "{{ kube_proxy_ipvs_modules }}"
+  when: kube_proxy_mode == 'ipvs'
+  tags:
+    - kube-proxy
+
+- name: Modprobe conntrack module
+  community.general.modprobe:
+    name: "{{ item }}"
+    state: present
+    persistent: present
+  register: modprobe_conntrack_module
+  ignore_errors: true  # noqa ignore-errors
+  loop: "{{ conntrack_modules }}"
+  when:
+    - kube_proxy_mode == 'ipvs'
+    - "(modprobe_conntrack_module|default({'rc': 1})).rc != 0"  # loop until first success
+  tags:
+    - kube-proxy
+
+- name: Modprobe Kernel Module for nftables
+  community.general.modprobe:
+    name: "nf_tables"
+    state: present
+    persistent: present
+  when: kube_proxy_mode == 'nftables'
+  tags:
+    - kube-proxy
+
+- name: Install kubelet
+  import_tasks: kubelet.yml
+  tags:
+    - kubelet
+    - kubeadm

internal/resources/kubespray/project/roles/kubernetes/node/templates/http-proxy.conf.j2

@@ -0,0 +1,2 @@
+[Service]
+Environment={% if http_proxy %}"HTTP_PROXY={{ http_proxy }}"{% endif %} {% if https_proxy %}"HTTPS_PROXY={{ https_proxy }}"{% endif %} {% if no_proxy %}"NO_PROXY={{ no_proxy }}"{% endif %}

internal/resources/kubespray/project/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2

@@ -0,0 +1,136 @@
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+nodeStatusUpdateFrequency: "{{ kubelet_status_update_frequency }}"
+failSwapOn: {{ kubelet_fail_swap_on }}
+authentication:
+  anonymous:
+    enabled: false
+  webhook:
+    enabled: {{ kubelet_authentication_token_webhook }}
+  x509:
+    clientCAFile: {{ kube_cert_dir }}/ca.crt
+authorization:
+{% if kubelet_authorization_mode_webhook %}
+  mode: Webhook
+{% else %}
+  mode: AlwaysAllow
+{% endif %}
+{% if kubelet_enforce_node_allocatable is defined and kubelet_enforce_node_allocatable != "\"\"" %}
+{% set kubelet_enforce_node_allocatable_list = kubelet_enforce_node_allocatable.split(",") %}
+enforceNodeAllocatable:
+{% for item in kubelet_enforce_node_allocatable_list %}
+- {{ item }}
+{% endfor %}
+{% endif %}
+staticPodPath: {{ kube_manifest_dir }}
+cgroupDriver: {{ kubelet_cgroup_driver | default('systemd') }}
+containerLogMaxFiles: {{ kubelet_logfiles_max_nr }}
+containerLogMaxSize: {{ kubelet_logfiles_max_size }}
+containerRuntimeEndpoint : {{ cri_socket }}
+maxPods: {{ kubelet_max_pods }}
+podPidsLimit: {{ kubelet_pod_pids_limit }}
+address: "{{ kubelet_bind_address }}"
+readOnlyPort: {{ kube_read_only_port }}
+healthzPort: {{ kubelet_healthz_port }}
+healthzBindAddress: "{{ kubelet_healthz_bind_address }}"
+kubeletCgroups: {{ kubelet_kubelet_cgroups }}
+clusterDomain: {{ dns_domain }}
+{% if kubelet_protect_kernel_defaults | bool %}
+protectKernelDefaults: true
+{% endif %}
+{% if kubelet_rotate_certificates | bool %}
+rotateCertificates: true
+{% endif %}
+{% if kubelet_rotate_server_certificates | bool %}
+serverTLSBootstrap: true
+{% endif %}
+{# DNS settings for kubelet #}
+{% if enable_nodelocaldns %}
+{% set kubelet_cluster_dns = [nodelocaldns_ip] %}
+{% elif dns_mode in ['coredns'] %}
+{% set kubelet_cluster_dns = [skydns_server] %}
+{% elif dns_mode == 'coredns_dual' %}
+{% set kubelet_cluster_dns = [skydns_server,skydns_server_secondary] %}
+{% elif dns_mode == 'manual' %}
+{% set kubelet_cluster_dns = [manual_dns_server] %}
+{% else %}
+{% set kubelet_cluster_dns = [] %}
+{% endif %}
+clusterDNS:
+{% for dns_address in kubelet_cluster_dns %}
+- {{ dns_address }}
+{% endfor %}
+{# Node reserved CPU/memory #}
+{% for scope in "kube", "system" %}
+{% if lookup('ansible.builtin.vars', scope + "_reserved") | bool %}
+{{ scope }}ReservedCgroup: {{ lookup('ansible.builtin.vars', scope + '_reserved_cgroups') }}
+{% endif %}
+{{ scope }}Reserved:
+{% for resource in "cpu", "memory", "ephemeral-storage", "pid" %}
+  {{ resource }}: "{{ lookup('ansible.builtin.vars', scope + '_' ~ (resource | replace('-', '_')) + '_reserved') }}"
+{% endfor %}
+{% endfor %}
+{% if eviction_hard is defined and eviction_hard %}
+evictionHard:
+  {{ eviction_hard | to_nice_yaml(indent=2) | indent(2) }}
+{% endif %}
+resolvConf: "{{ kube_resolv_conf }}"
+{% if kubelet_config_extra_args %}
+{{ kubelet_config_extra_args | to_nice_yaml(indent=2) }}
+{% endif %}
+{% if kubelet_feature_gates or kube_feature_gates %}
+featureGates:
+{% for feature in (kubelet_feature_gates | default(kube_feature_gates, true)) %}
+  {{ feature | replace("=", ": ") }}
+{% endfor %}
+{% endif %}
+{% if tls_min_version is defined %}
+tlsMinVersion: {{ tls_min_version }}
+{% endif %}
+{% if tls_cipher_suites is defined %}
+tlsCipherSuites:
+{% for tls in tls_cipher_suites %}
+- {{ tls }}
+{% endfor %}
+{% endif %}
+eventRecordQPS: {{ kubelet_event_record_qps }}
+shutdownGracePeriod: {{ kubelet_shutdown_grace_period }}
+shutdownGracePeriodCriticalPods: {{ kubelet_shutdown_grace_period_critical_pods }}
+{% if not kubelet_fail_swap_on %}
+memorySwap:
+  swapBehavior: {{ kubelet_swap_behavior }}
+{% endif %}
+{% if kubelet_streaming_connection_idle_timeout is defined %}
+streamingConnectionIdleTimeout: {{ kubelet_streaming_connection_idle_timeout }}
+{% endif %}
+{% if kubelet_image_gc_high_threshold is defined %}
+imageGCHighThresholdPercent: {{ kubelet_image_gc_high_threshold }}
+{% endif %}
+{% if kubelet_image_gc_low_threshold is defined %}
+imageGCLowThresholdPercent: {{ kubelet_image_gc_low_threshold }}
+{% endif %}
+{% if kubelet_make_iptables_util_chains is defined %}
+makeIPTablesUtilChains: {{ kubelet_make_iptables_util_chains | bool }}
+{% endif %}
+{% if kubelet_seccomp_default is defined %}
+seccompDefault: {{ kubelet_seccomp_default | bool }}
+{% endif %}
+{% if kubelet_cpu_manager_policy is defined %}
+cpuManagerPolicy: {{ kubelet_cpu_manager_policy }}
+{% endif %}
+{% if kubelet_cpu_manager_policy_options is defined %}
+cpuManagerPolicyOptions:
+  {{ kubelet_cpu_manager_policy_options | to_nice_yaml(indent=2) | indent(width=2) }}
+{% endif %}
+{% if kubelet_topology_manager_policy is defined %}
+topologyManagerPolicy: {{ kubelet_topology_manager_policy }}
+{% endif %}
+{% if kubelet_topology_manager_scope is defined %}
+topologyManagerScope: {{ kubelet_topology_manager_scope }}
+{% endif %}
+{% if kubelet_tracing %}
+tracing:
+  endpoint: "{{ kubelet_tracing_endpoint }}"
+  samplingRatePerMillion: {{ kubelet_tracing_sampling_rate_per_million }}
+{% endif %}
+maxParallelImagePulls: {{ kubelet_max_parallel_image_pulls }}

internal/resources/kubespray/project/roles/kubernetes/node/templates/kubelet.env.v1beta1.j2

@@ -0,0 +1,30 @@
+KUBE_LOG_LEVEL="--v={{ kube_log_level }}"
+KUBELET_ADDRESS="--node-ip={{ kubelet_address }}"
+{% if kube_override_hostname|default('') %}
+KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
+{% endif %}
+
+{# Base kubelet args #}
+{% set kubelet_args_base -%}
+{# start kubeadm specific settings #}
+--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \
+--config={{ kube_config_dir }}/kubelet-config.yaml \
+--kubeconfig={{ kube_config_dir }}/kubelet.conf \
+{# end kubeadm specific settings #}
+--runtime-cgroups={{ kubelet_runtime_cgroups }} \
+{% endset %}
+
+KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_custom_flags | join(' ') }}"
+{% if kubelet_flexvolumes_plugins_dir is defined %}
+KUBELET_VOLUME_PLUGIN="--volume-plugin-dir={{ kubelet_flexvolumes_plugins_dir }}"
+{% endif %}
+{% if kube_network_plugin is defined and kube_network_plugin == "cloud" %}
+KUBELET_NETWORK_PLUGIN="--hairpin-mode=promiscuous-bridge --network-plugin=kubenet"
+{% endif %}
+{% if cloud_provider == "external" %}
+KUBELET_CLOUDPROVIDER="--cloud-provider={{ cloud_provider }}"
+{% else %}
+KUBELET_CLOUDPROVIDER=""
+{% endif %}
+
+PATH={{ bin_dir }}:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin

internal/resources/kubespray/project/roles/kubernetes/node/templates/kubelet.service.j2

@@ -0,0 +1,57 @@
+[Unit]
+Description=Kubernetes Kubelet Server
+Documentation=https://github.com/GoogleCloudPlatform/kubernetes
+After={{ container_manager }}.service
+{% if container_manager == 'docker' %}
+Wants=docker.socket
+{% else %}
+Wants={{ container_manager }}.service
+{% endif %}
+{% for kubelet_dependency in kubelet_systemd_wants_dependencies|default([]) %}
+{% if kubelet_dependency|length > 0 %}
+Wants={{ kubelet_dependency }}
+{% endif %}
+{% endfor %}
+
+[Service]
+EnvironmentFile=-{{ kube_config_dir }}/kubelet.env
+{% if system_reserved|bool %}
+ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/cpu/{{ system_reserved_cgroups_for_service_slice }}
+ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/cpuacct/{{ system_reserved_cgroups_for_service_slice }}
+ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/cpuset/{{ system_reserved_cgroups_for_service_slice }}
+ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/hugetlb/{{ system_reserved_cgroups_for_service_slice }}
+ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/memory/{{ system_reserved_cgroups_for_service_slice }}
+ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/pids/{{ system_reserved_cgroups_for_service_slice }}
+ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/systemd/{{ system_reserved_cgroups_for_service_slice }}
+{% endif %}
+{% if kube_reserved|bool %}
+ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/cpu/{{ kube_reserved_cgroups_for_service_slice }}
+ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/cpuacct/{{ kube_reserved_cgroups_for_service_slice }}
+ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/cpuset/{{ kube_reserved_cgroups_for_service_slice }}
+ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/hugetlb/{{ kube_reserved_cgroups_for_service_slice }}
+ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/memory/{{ kube_reserved_cgroups_for_service_slice }}
+ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/pids/{{ kube_reserved_cgroups_for_service_slice }}
+ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/systemd/{{ kube_reserved_cgroups_for_service_slice }}
+{% endif %}
+ExecStart={{ bin_dir }}/kubelet \
+		$KUBE_LOGTOSTDERR \
+		$KUBE_LOG_LEVEL \
+		$KUBELET_API_SERVER \
+		$KUBELET_ADDRESS \
+		$KUBELET_PORT \
+		$KUBELET_HOSTNAME \
+		$KUBELET_ARGS \
+		$DOCKER_SOCKET \
+		$KUBELET_NETWORK_PLUGIN \
+		$KUBELET_VOLUME_PLUGIN \
+		$KUBELET_CLOUDPROVIDER
+Restart=always
+RestartSec=10s
+{% if kubelet_systemd_hardening %}
+# Hardening setup
+IPAddressDeny=any
+IPAddressAllow={{ kubelet_secure_addresses }}
+{% endif %}
+
+[Install]
+WantedBy=multi-user.target

internal/resources/kubespray/project/roles/kubernetes/node/templates/loadbalancer/haproxy.cfg.j2

@@ -0,0 +1,49 @@
+global
+    maxconn                 4000
+    log                     127.0.0.1 local0
+
+defaults
+    mode                    http
+    log                     global
+    option                  httplog
+    option                  dontlognull
+    option                  http-server-close
+    option                  redispatch
+    retries                 5
+    timeout http-request    5m
+    timeout queue           5m
+    timeout connect         30s
+    timeout client          {{ loadbalancer_apiserver_keepalive_timeout }}
+    timeout server          15m
+    timeout http-keep-alive 30s
+    timeout check           30s
+    maxconn                 4000
+
+{% if loadbalancer_apiserver_healthcheck_port is defined -%}
+frontend healthz
+  bind 0.0.0.0:{{ loadbalancer_apiserver_healthcheck_port }}
+  {% if ipv6_stack -%}
+  bind :::{{ loadbalancer_apiserver_healthcheck_port }}
+  {% endif -%}
+  mode http
+  monitor-uri /healthz
+{% endif %}
+
+frontend kube_api_frontend
+  bind 127.0.0.1:{{ loadbalancer_apiserver_port|default(kube_apiserver_port) }}
+  {% if ipv6_stack -%}
+  bind [::1]:{{ loadbalancer_apiserver_port|default(kube_apiserver_port) }};
+  {% endif -%}
+  mode tcp
+  option tcplog
+  default_backend kube_api_backend
+
+backend kube_api_backend
+  mode tcp
+  balance leastconn
+  default-server inter 15s downinter 15s rise 2 fall 2 slowstart 60s maxconn 1000 maxqueue 256 weight 100
+  option httpchk GET /healthz
+  http-check expect status 200
+  {% for host in groups['kube_control_plane'] -%}
+  server {{ host }} {{ hostvars[host]['main_access_ip'] | ansible.utils.ipwrap }}:{{ kube_apiserver_port }} check check-ssl verify none
+  {% endfor -%}

internal/resources/kubespray/project/roles/kubernetes/node/templates/loadbalancer/nginx.conf.j2

@@ -0,0 +1,60 @@
+error_log stderr notice;
+
+worker_processes 2;
+worker_rlimit_nofile 130048;
+worker_shutdown_timeout 10s;
+
+events {
+  multi_accept on;
+  use epoll;
+  worker_connections 16384;
+}
+
+stream {
+  upstream kube_apiserver {
+    least_conn;
+    {% for host in groups['kube_control_plane'] -%}
+    server {{ hostvars[host]['main_access_ip'] | ansible.utils.ipwrap }}:{{ kube_apiserver_port }};
+    {% endfor -%}
+  }
+
+  server {
+    listen        127.0.0.1:{{ loadbalancer_apiserver_port|default(kube_apiserver_port) }};
+    {% if ipv6_stack -%}
+    listen        [::1]:{{ loadbalancer_apiserver_port|default(kube_apiserver_port) }};
+    {% endif -%}
+    proxy_pass    kube_apiserver;
+    proxy_timeout 10m;
+    proxy_connect_timeout 1s;
+  }
+}
+
+http {
+  aio threads;
+  aio_write on;
+  tcp_nopush on;
+  tcp_nodelay on;
+
+  keepalive_timeout {{ loadbalancer_apiserver_keepalive_timeout }};
+  keepalive_requests 100;
+  reset_timedout_connection on;
+  server_tokens off;
+  autoindex off;
+
+  {% if loadbalancer_apiserver_healthcheck_port is defined -%}
+  server {
+    listen {{ loadbalancer_apiserver_healthcheck_port }};
+    {% if ipv6_stack -%}
+    listen [::]:{{ loadbalancer_apiserver_healthcheck_port }};
+    {% endif -%}
+    location /healthz {
+      access_log off;
+      return 200;
+    }
+    location /stub_status {
+      stub_status on;
+      access_log off;
+    }
+  }
+  {% endif %}
+}

internal/resources/kubespray/project/roles/kubernetes/node/templates/manifests/haproxy.manifest.j2

@@ -0,0 +1,42 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: {{ loadbalancer_apiserver_pod_name }}
+  namespace: kube-system
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
+    k8s-app: kube-haproxy
+  annotations:
+    haproxy-cfg-checksum: "{{ haproxy_stat.stat.checksum }}"
+spec:
+  hostNetwork: true
+  dnsPolicy: ClusterFirstWithHostNet
+  nodeSelector:
+    kubernetes.io/os: linux
+  priorityClassName: system-node-critical
+  containers:
+  - name: haproxy
+    image: {{ haproxy_image_repo }}:{{ haproxy_image_tag }}
+    imagePullPolicy: {{ k8s_image_pull_policy }}
+    resources:
+      requests:
+        cpu: {{ loadbalancer_apiserver_cpu_requests }}
+        memory: {{ loadbalancer_apiserver_memory_requests }}
+    {% if loadbalancer_apiserver_healthcheck_port is defined -%}
+    livenessProbe:
+      httpGet:
+        path: /healthz
+        port: {{ loadbalancer_apiserver_healthcheck_port }}
+    readinessProbe:
+      httpGet:
+        path: /healthz
+        port: {{ loadbalancer_apiserver_healthcheck_port }}
+    {% endif -%}
+    volumeMounts:
+    - mountPath: /usr/local/etc/haproxy/
+      name: etc-haproxy
+      readOnly: true
+  volumes:
+  - name: etc-haproxy
+    hostPath:
+      path: {{ haproxy_config_dir }}

internal/resources/kubespray/project/roles/kubernetes/node/templates/manifests/kube-vip.manifest.j2

@@ -0,0 +1,129 @@
+# Inspired by https://github.com/kube-vip/kube-vip/blob/v0.8.0/pkg/kubevip/config_generator.go#L103
+apiVersion: v1
+kind: Pod
+metadata:
+  name: kube-vip
+  namespace: kube-system
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
+    k8s-app: kube-vip
+spec:
+  containers:
+  - args:
+    - manager
+    env:
+    - name: vip_arp
+      value: {{ kube_vip_arp_enabled | string | to_json }}
+    - name: port
+      value: {{ kube_apiserver_port | string | to_json }}
+    - name: vip_nodename
+      value: {{ inventory_hostname }}
+{% if kube_vip_interface %}
+    - name: vip_interface
+      value: {{ kube_vip_interface | string | to_json }}
+{% endif %}
+{% if kube_vip_services_interface %}
+    - name: vip_servicesinterface
+      value: {{ kube_vip_services_interface | string | to_json }}
+{% endif %}
+{% if kube_vip_cidr %}
+    - name: vip_cidr
+      value: {{ kube_vip_cidr | string | to_json }}
+{% endif %}
+{% if kube_vip_dns_mode %}
+    - name: dns_mode
+      value: {{ kube_vip_dns_mode | string | to_json }}
+{% endif %}
+{% if kube_vip_controlplane_enabled %}
+    - name: cp_enable
+      value: "true"
+    - name: cp_namespace
+      value: kube-system
+    - name: vip_ddns
+      value: {{ kube_vip_ddns_enabled | string | to_json }}
+    - name: cp_detect
+      value: {{ kube_vip_cp_detect | string | to_json }}
+{% endif %}
+{% if kube_vip_services_enabled %}
+    - name: svc_enable
+      value: "true"
+{% endif %}
+{% if kube_vip_svc_leasename %}
+    - name: svc_leasename
+      value: {{ kube_vip_svc_leasename | string | to_json }}
+{% endif %}
+{% if kube_vip_enableServicesElection %}
+    - name: svc_election
+      value: "true"
+{% endif %}
+{% if kube_vip_leader_election_enabled %}
+    - name: vip_leaderelection
+      value: "true"
+    - name: vip_leasename
+      value: {{ kube_vip_leasename | string | to_json }}
+    - name: vip_leaseduration
+      value: {{ kube_vip_leaseduration | string | to_json }}
+    - name: vip_renewdeadline
+      value: {{ kube_vip_renewdeadline | string | to_json }}
+    - name: vip_retryperiod
+      value: {{ kube_vip_retryperiod | string | to_json }}
+{% endif %}
+{% if kube_vip_enable_node_labeling %}
+    - name: enable_node_labeling
+      value: {{ kube_vip_enable_node_labeling | string | to_json }}
+{% endif %}
+{% if kube_vip_bgp_enabled %}
+    - name: bgp_enable
+      value: "true"
+    - name: bgp_routerid
+      value: {{ kube_vip_bgp_routerid | string | to_json }}
+    - name: bgp_as
+      value: {{ kube_vip_local_as | string | to_json }}
+    - name: bgp_peeraddress
+      value: {{ kube_vip_bgp_peeraddress | to_json }}
+    - name: bgp_peerpass
+      value: {{ kube_vip_bgp_peerpass | to_json }}
+    - name: bgp_peeras
+      value: {{ kube_vip_bgp_peeras | string | to_json }}
+{% if kube_vip_bgppeers %}
+    - name: bgp_peers
+      value: {{ kube_vip_bgppeers | join(',')  | to_json }}
+{% endif %}
+{% endif %}
+    - name: address
+      value: {{ kube_vip_address | to_json }}
+{% if kube_vip_lb_enable %}
+    - name: lb_enable
+      value: "true"
+{% endif %}
+{% if kube_vip_lb_fwdmethod %}
+    - name: lb_fwdmethod
+      value: {{ kube_vip_lb_fwdmethod | string | to_json }}
+{% endif %}
+    image: {{ kube_vip_image_repo }}:{{ kube_vip_image_tag }}
+    imagePullPolicy: {{ k8s_image_pull_policy }}
+    name: kube-vip
+    resources: {}
+{% if kube_vip_lb_fwdmethod == "masquerade" %}
+    securityContext:
+      privileged: true
+{% else %}
+    securityContext:
+      capabilities:
+        add:
+        - NET_ADMIN
+        - NET_RAW
+{% endif %}
+    volumeMounts:
+    - mountPath: /etc/kubernetes/admin.conf
+      name: kubeconfig
+  hostAliases:
+  - hostnames:
+    - kubernetes
+    ip: 127.0.0.1
+  hostNetwork: true
+  volumes:
+  - hostPath:
+      path: /etc/kubernetes/{{kube_vip_admin_conf}}
+    name: kubeconfig
+status: {}

internal/resources/kubespray/project/roles/kubernetes/node/templates/manifests/nginx-proxy.manifest.j2

@@ -0,0 +1,42 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: {{ loadbalancer_apiserver_pod_name }}
+  namespace: kube-system
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
+    k8s-app: kube-nginx
+  annotations:
+    nginx-cfg-checksum: "{{ nginx_stat.stat.checksum }}"
+spec:
+  hostNetwork: true
+  dnsPolicy: ClusterFirstWithHostNet
+  nodeSelector:
+    kubernetes.io/os: linux
+  priorityClassName: system-node-critical
+  containers:
+  - name: nginx-proxy
+    image: {{ nginx_image_repo }}:{{ nginx_image_tag }}
+    imagePullPolicy: {{ k8s_image_pull_policy }}
+    resources:
+      requests:
+        cpu: {{ loadbalancer_apiserver_cpu_requests }}
+        memory: {{ loadbalancer_apiserver_memory_requests }}
+    {% if loadbalancer_apiserver_healthcheck_port is defined -%}
+    livenessProbe:
+      httpGet:
+        path: /healthz
+        port: {{ loadbalancer_apiserver_healthcheck_port }}
+    readinessProbe:
+      httpGet:
+        path: /healthz
+        port: {{ loadbalancer_apiserver_healthcheck_port }}
+    {% endif -%}
+    volumeMounts:
+    - mountPath: /etc/nginx
+      name: etc-nginx
+      readOnly: true
+  volumes:
+  - name: etc-nginx
+    hostPath:
+      path: {{ nginx_config_dir }}

internal/resources/kubespray/project/roles/kubernetes/node/templates/node-kubeconfig.yaml.j2

@@ -0,0 +1,19 @@
+---
+apiVersion: v1
+kind: Config
+clusters:
+- name: local
+  cluster:
+    certificate-authority: {{ kube_cert_dir }}/ca.pem
+    server: "{{ kube_apiserver_endpoint }}"
+users:
+- name: kubelet
+  user:
+    client-certificate: {{ kube_cert_dir }}/node-{{ inventory_hostname }}.pem
+    client-key: {{ kube_cert_dir }}/node-{{ inventory_hostname }}-key.pem
+contexts:
+- context:
+    cluster: local
+    user: kubelet
+  name: kubelet-{{ cluster_name }}
+current-context: kubelet-{{ cluster_name }}

internal/resources/kubespray/project/roles/kubernetes/node/vars/fedora.yml

@@ -0,0 +1,2 @@
+---
+kube_resolv_conf: "/run/systemd/resolve/resolv.conf"

internal/resources/kubespray/project/roles/kubernetes/node/vars/ubuntu-18.yml

@@ -0,0 +1,2 @@
+---
+kube_resolv_conf: "/run/systemd/resolve/resolv.conf"

internal/resources/kubespray/project/roles/kubernetes/node/vars/ubuntu-20.yml

@@ -0,0 +1,2 @@
+---
+kube_resolv_conf: "/run/systemd/resolve/resolv.conf"

internal/resources/kubespray/project/roles/kubernetes/node/vars/ubuntu-22.yml

@@ -0,0 +1,2 @@
+---
+kube_resolv_conf: "/run/systemd/resolve/resolv.conf"

internal/resources/kubespray/project/roles/kubernetes/node/vars/ubuntu-24.yml

@@ -0,0 +1,2 @@
+---
+kube_resolv_conf: "/run/systemd/resolve/resolv.conf"

internal/resources/kubespray/project/roles/kubernetes/preinstall/defaults/main.yml

@@ -0,0 +1,130 @@
+---
+# Set to true to allow pre-checks to fail and continue deployment
+ignore_assert_errors: false
+# Set to false to disable the backup parameter, set to true to accumulate backups of config files.
+leave_etc_backup_files: true
+nameservers: []
+cloud_resolver: []
+disable_host_nameservers: false
+# Kubespray sets this to true after clusterDNS is running to apply changes to the host resolv.conf
+dns_late: false
+
+# Set to true if your network does not support IPv6
+# This may be necessary for pulling Docker images from
+# GCE docker repository
+disable_ipv6_dns: false
+
+# Remove default cluster search domains (``default.svc.{{ dns_domain }}, svc.{{ dns_domain }}``).
+remove_default_searchdomains: false
+
+kube_owner: kube
+kube_cert_group: kube-cert
+kube_config_dir: /etc/kubernetes
+kube_cert_dir: "{{ kube_config_dir }}/ssl"
+kube_cert_compat_dir: /etc/kubernetes/pki
+kubelet_flexvolumes_plugins_dir: /usr/libexec/kubernetes/kubelet-plugins/volume/exec
+
+# Flatcar Container Linux by Kinvolk cloud init config file to define /etc/resolv.conf content
+# for hostnet pods and infra needs
+resolveconf_cloud_init_conf: /etc/resolveconf_cloud_init.conf
+
+# All inventory hostnames will be written into each /etc/hosts file.
+populate_inventory_to_hosts_file: true
+# K8S Api FQDN will be written into /etc/hosts file.
+populate_loadbalancer_apiserver_to_hosts_file: true
+# etc_hosts_localhost_entries will be written into /etc/hosts file.
+populate_localhost_entries_to_hosts_file: true
+
+sysctl_file_path: "/etc/sysctl.d/99-sysctl.conf"
+
+etc_hosts_localhost_entries:
+  127.0.0.1:
+    expected:
+      - localhost
+      - localhost.localdomain
+  ::1:
+    expected:
+      - localhost6
+      - localhost6.localdomain
+    unexpected:
+      - localhost
+      - localhost.localdomain
+
+# Minimal memory requirement in MB for safety checks
+minimal_node_memory_mb: 1024
+minimal_master_memory_mb: 1500
+
+## NTP Settings
+
+# Manage the NTP configuration file.
+ntp_manage_config: false
+# Specify the NTP servers
+# Only takes effect when ntp_manage_config is true.
+ntp_servers:
+  - "0.pool.ntp.org iburst"
+  - "1.pool.ntp.org iburst"
+  - "2.pool.ntp.org iburst"
+  - "3.pool.ntp.org iburst"
+# Restrict NTP access to these hosts.
+# Only takes effect when ntp_manage_config is true.
+ntp_restrict:
+  - "127.0.0.1"
+  - "::1"
+# Specify whether to filter interfaces
+ntp_filter_interface: false
+# Specify the interfaces
+# Only takes effect when ntp_filter_interface is true
+# ntp_interfaces:
+#   - ignore wildcard
+#   - listen xxx
+# The NTP driftfile path
+# Only takes effect when ntp_manage_config is true.
+# Default value is `/var/lib/ntp/ntp.drift`, for ntpsec use '/var/lib/ntpsec/ntp.drift'
+ntp_driftfile: >-
+      {% if ntp_package == "ntpsec" -%}
+      /var/lib/ntpsec/ntp.drift
+      {%- else -%}
+      /var/lib/ntp/ntp.drift
+      {%- endif -%}
+# Only takes effect when ntp_manage_config is true.
+ntp_tinker_panic: false
+
+# Force sync time immediately after the ntp installed, which is useful in a newly installed system.
+ntp_force_sync_immediately: false
+
+# Set the timezone for your server.  eg: "Etc/UTC","Etc/GMT-8". If not set, the timezone will not change.
+ntp_timezone: ""
+
+# Currently known os distributions
+supported_os_distributions:
+  - 'RedHat'
+  - 'CentOS'
+  - 'Fedora'
+  - 'Ubuntu'
+  - 'Debian'
+  - 'Flatcar'
+  - 'Flatcar Container Linux by Kinvolk'
+  - 'Suse'
+  - 'openSUSE Leap'
+  - 'openSUSE Tumbleweed'
+  - 'ClearLinux'
+  - 'OracleLinux'
+  - 'AlmaLinux'
+  - 'Rocky'
+  - 'Amazon'
+  - 'Kylin Linux Advanced Server'
+  - 'UnionTech'
+  - 'UniontechOS'
+  - 'openEuler'
+
+# Extending some distributions into the redhat os family
+redhat_os_family_extensions:
+  - "UnionTech"
+  - "UniontechOS"
+
+# Sets DNSStubListener=no, useful if you get "0.0.0.0:53: bind: address already in use"
+systemd_resolved_disable_stub_listener: "{{ ansible_os_family in ['Flatcar', 'Flatcar Container Linux by Kinvolk'] }}"
+
+# Used to disable File Access Policy Daemon service.
+# If service is enabled, the CNI plugin installation will fail
+disable_fapolicyd: true

internal/resources/kubespray/project/roles/kubernetes/preinstall/files/dhclient_nodnsupdate

@@ -0,0 +1,4 @@
+#!/bin/sh
+make_resolv_conf() {
+    :
+}

internal/resources/kubespray/project/roles/kubernetes/preinstall/handlers/main.yml

@@ -0,0 +1,134 @@
+---
+- name: Preinstall | apply resolvconf cloud-init
+  command: /usr/bin/coreos-cloudinit --from-file {{ resolveconf_cloud_init_conf }}
+  when: ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"]
+  listen: Preinstall | update resolvconf for Flatcar Container Linux by Kinvolk
+
+- name: Preinstall | reload NetworkManager
+  service:
+    name: NetworkManager.service
+    state: restarted
+  listen: Preinstall | update resolvconf for networkmanager
+
+- name: Preinstall | reload kubelet
+  service:
+    name: kubelet
+    state: restarted
+  notify:
+    - Preinstall | kube-controller configured
+    - Preinstall | kube-apiserver configured
+    - Preinstall | restart kube-controller-manager docker
+    - Preinstall | restart kube-controller-manager crio/containerd
+    - Preinstall | restart kube-apiserver docker
+    - Preinstall | restart kube-apiserver crio/containerd
+  when: not dns_early | bool
+  listen:
+    - Preinstall | propagate resolvconf to k8s components
+    - Preinstall | update resolvconf for Flatcar Container Linux by Kinvolk
+    - Preinstall | update resolvconf for networkmanager
+
+# FIXME(mattymo): Also restart for kubeadm mode
+- name: Preinstall | kube-apiserver configured
+  stat:
+    path: "{{ kube_manifest_dir }}/kube-apiserver.yaml"
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: kube_apiserver_set
+  when: ('kube_control_plane' in group_names) and dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'
+  listen: Preinstall | propagate resolvconf to k8s components
+
+# FIXME(mattymo): Also restart for kubeadm mode
+- name: Preinstall | kube-controller configured
+  stat:
+    path: "{{ kube_manifest_dir }}/kube-controller-manager.yaml"
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: kube_controller_set
+  when: ('kube_control_plane' in group_names) and dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'
+  listen: Preinstall | propagate resolvconf to k8s components
+
+- name: Preinstall | restart kube-controller-manager docker
+  shell: "set -o pipefail && {{ docker_bin_dir }}/docker ps -f name=k8s_POD_kube-controller-manager* -q | xargs --no-run-if-empty {{ docker_bin_dir }}/docker rm -f"
+  args:
+    executable: /bin/bash
+  when:
+    - container_manager == "docker"
+    - ('kube_control_plane' in group_names)
+    - dns_mode != 'none'
+    - resolvconf_mode == 'host_resolvconf'
+    - kube_controller_set.stat.exists
+  listen: Preinstall | propagate resolvconf to k8s components
+
+- name: Preinstall | restart kube-controller-manager crio/containerd
+  shell: "set -o pipefail && {{ bin_dir }}/crictl pods --name kube-controller-manager* -q | xargs -I% --no-run-if-empty bash -c '{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %'"
+  args:
+    executable: /bin/bash
+  register: preinstall_restart_controller_manager
+  retries: 10
+  delay: 1
+  until: preinstall_restart_controller_manager.rc == 0
+  when:
+    - container_manager in ['crio', 'containerd']
+    - ('kube_control_plane' in group_names)
+    - dns_mode != 'none'
+    - resolvconf_mode == 'host_resolvconf'
+    - kube_controller_set.stat.exists
+  listen: Preinstall | propagate resolvconf to k8s components
+
+- name: Preinstall | restart kube-apiserver docker
+  shell: "set -o pipefail && {{ docker_bin_dir }}/docker ps -f name=k8s_POD_kube-apiserver* -q | xargs --no-run-if-empty {{ docker_bin_dir }}/docker rm -f"
+  args:
+    executable: /bin/bash
+  when:
+    - container_manager == "docker"
+    - ('kube_control_plane' in group_names)
+    - dns_mode != 'none'
+    - resolvconf_mode == 'host_resolvconf'
+    - kube_apiserver_set.stat.exists
+  listen: Preinstall | propagate resolvconf to k8s components
+
+- name: Preinstall | restart kube-apiserver crio/containerd
+  shell: "set -o pipefail && {{ bin_dir }}/crictl pods --name kube-apiserver* -q | xargs -I% --no-run-if-empty bash -c '{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %'"
+  args:
+    executable: /bin/bash
+  register: preinstall_restart_apiserver
+  retries: 10
+  until: preinstall_restart_apiserver.rc == 0
+  delay: 1
+  when:
+    - container_manager in ['crio', 'containerd']
+    - ('kube_control_plane' in group_names)
+    - dns_mode != 'none'
+    - resolvconf_mode == 'host_resolvconf'
+    - kube_apiserver_set.stat.exists
+  listen: Preinstall | propagate resolvconf to k8s components
+
+# When running this as the last phase ensure we wait for kube-apiserver to come up
+- name: Preinstall | wait for the apiserver to be running
+  uri:
+    url: "{{ kube_apiserver_endpoint }}/healthz"
+    validate_certs: false
+  register: result
+  until: result.status == 200
+  retries: 60
+  delay: 1
+  when:
+    - dns_late
+    - ('kube_control_plane' in group_names)
+    - dns_mode != 'none'
+    - resolvconf_mode == 'host_resolvconf'
+    - not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] and not is_fedora_coreos
+  listen: Preinstall | propagate resolvconf to k8s components
+
+- name: Preinstall | Restart systemd-resolved
+  service:
+    name: systemd-resolved
+    state: restarted
+
+- name: Preinstall | restart ntp
+  service:
+    name: "{{ ntp_service_name }}"
+    state: restarted
+  when: ntp_enabled

internal/resources/kubespray/project/roles/kubernetes/preinstall/meta/main.yml

@@ -0,0 +1,8 @@
+---
+dependencies:
+  - role: adduser
+    user: "{{ addusers.kube }}"
+    when:
+      - not is_fedora_coreos
+    tags:
+      - kubelet

internal/resources/kubespray/project/roles/kubernetes/preinstall/tasks/0010-swapoff.yml

@@ -0,0 +1,26 @@
+---
+- name: Check if /etc/fstab exists
+  stat:
+    path: "/etc/fstab"
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: fstab_file
+
+- name: Remove swapfile from /etc/fstab
+  ansible.posix.mount:
+    name: "{{ item }}"
+    fstype: swap
+    state: absent
+  loop:
+    - swap
+    - none
+  when: fstab_file.stat.exists
+
+- name: Mask swap.target (persist swapoff)
+  ansible.builtin.systemd_service:
+    name: swap.target
+    masked: true
+
+- name: Disable swap
+  command: /sbin/swapoff -a

internal/resources/kubespray/project/roles/kubernetes/preinstall/tasks/0020-set_facts.yml

@@ -0,0 +1,152 @@
+---
+- name: Set os_family fact for other redhat-based operating systems
+  set_fact:
+    ansible_os_family: "RedHat"
+    ansible_distribution_major_version: "8"
+  when: ansible_distribution in redhat_os_family_extensions
+  tags:
+    - facts
+
+- name: Check resolvconf
+  command: which resolvconf
+  register: resolvconf
+  failed_when: false
+  changed_when: false
+  check_mode: false
+
+- name: Check existence of /etc/resolvconf/resolv.conf.d
+  stat:
+    path: /etc/resolvconf/resolv.conf.d
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  failed_when: false
+  register: resolvconfd_path
+
+- name: Check status of /etc/resolv.conf
+  stat:
+    path: /etc/resolv.conf
+    follow: false
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  failed_when: false
+  register: resolvconf_stat
+
+  # Used in vars/
+- name: Fetch resolv.conf
+  when: resolvconf_stat.stat.exists
+  slurp:
+    src: /etc/resolv.conf
+  register: resolvconf_slurp
+
+- name: NetworkManager | Check if host has NetworkManager
+  # noqa command-instead-of-module - Should we use service_facts for this?
+  command: systemctl is-active --quiet NetworkManager.service
+  register: networkmanager_enabled
+  failed_when: false
+  changed_when: false
+  check_mode: false
+
+- name: Check systemd-resolved
+  # noqa command-instead-of-module - Should we use service_facts for this?
+  command: systemctl is-active systemd-resolved
+  register: systemd_resolved_enabled
+  failed_when: false
+  changed_when: false
+  check_mode: false
+
+- name: Set default dns if remove_default_searchdomains is false
+  set_fact:
+    default_searchdomains: ["default.svc.{{ dns_domain }}", "svc.{{ dns_domain }}"]
+  when: not remove_default_searchdomains | default() | bool or (remove_default_searchdomains | default() | bool and searchdomains | length == 0)
+
+- name: Set dns facts
+  set_fact:
+    resolvconf: >-
+      {%- if resolvconf.rc == 0 and resolvconfd_path.stat.isdir is defined and resolvconfd_path.stat.isdir -%}true{%- else -%}false{%- endif -%}
+
+- name: Check if kubelet is configured
+  stat:
+    path: "{{ kube_config_dir }}/kubelet.env"
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: kubelet_configured
+  changed_when: false
+
+- name: Check if early DNS configuration stage
+  set_fact:
+    dns_early: "{{ not kubelet_configured.stat.exists }}"
+
+- name: Target resolv.conf files
+  set_fact:
+    resolvconffile: /etc/resolv.conf
+    base: >-
+      {%- if resolvconf | bool -%}/etc/resolvconf/resolv.conf.d/base{%- endif -%}
+    head: >-
+      {%- if resolvconf | bool -%}/etc/resolvconf/resolv.conf.d/head{%- endif -%}
+  when: not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] and not is_fedora_coreos
+
+- name: Target temporary resolvconf cloud init file (Flatcar Container Linux by Kinvolk / Fedora CoreOS)
+  set_fact:
+    resolvconffile: /tmp/resolveconf_cloud_init_conf
+  when: ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] or is_fedora_coreos
+
+- name: Check if /etc/dhclient.conf exists
+  stat:
+    path: /etc/dhclient.conf
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: dhclient_stat
+
+- name: Target dhclient conf file for /etc/dhclient.conf
+  set_fact:
+    dhclientconffile: /etc/dhclient.conf
+  when: dhclient_stat.stat.exists
+
+- name: Check if /etc/dhcp/dhclient.conf exists
+  stat:
+    path: /etc/dhcp/dhclient.conf
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: dhcp_dhclient_stat
+
+- name: Target dhclient conf file for /etc/dhcp/dhclient.conf
+  set_fact:
+    dhclientconffile: /etc/dhcp/dhclient.conf
+  when: dhcp_dhclient_stat.stat.exists
+
+- name: Target dhclient hook file for Red Hat family
+  set_fact:
+    dhclienthookfile: /etc/dhcp/dhclient.d/zdnsupdate.sh
+  when: ansible_os_family == "RedHat"
+
+- name: Target dhclient hook file for Debian family
+  set_fact:
+    dhclienthookfile: /etc/dhcp/dhclient-exit-hooks.d/zdnsupdate
+  when: ansible_os_family == "Debian"
+
+- name: Set etcd vars if using kubeadm mode
+  set_fact:
+    etcd_cert_dir: "{{ kube_cert_dir }}"
+    kube_etcd_cacert_file: "etcd/ca.crt"
+    kube_etcd_cert_file: "apiserver-etcd-client.crt"
+    kube_etcd_key_file: "apiserver-etcd-client.key"
+  when:
+    - etcd_deployment_type == "kubeadm"
+
+- name: Check /usr readonly
+  stat:
+    path: "/usr"
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: usr
+
+- name: Set alternate flexvolume path
+  set_fact:
+    kubelet_flexvolumes_plugins_dir: /var/lib/kubelet/volumeplugins
+  when: not usr.stat.writeable

internal/resources/kubespray/project/roles/kubernetes/preinstall/tasks/0040-verify-settings.yml

@@ -0,0 +1,97 @@
+---
+- name: Stop if any host not in '--limit' does not have a fact cache
+  vars:
+    uncached_hosts: "{{ hostvars | dict2items | selectattr('value.ansible_default_ipv6', 'undefined') | selectattr('value.ansible_default_ipv4', 'undefined') | map(attribute='key') }}"
+    excluded_hosts: "{{ groups['k8s_cluster'] | difference(query('inventory_hostnames', ansible_limit)) }}"
+  assert:
+    that: uncached_hosts | intersect(excluded_hosts) == []
+    fail_msg: |
+      Kubespray does not support '--limit' without a populated facts cache for the excluded hosts.
+      Please run the facts.yml playbook first without '--limit'.
+      The following excluded hosts are not cached: {{ uncached_hosts | intersect(excluded_hosts) }}
+  run_once: true
+  when:
+    - ansible_limit is defined
+    - not ignore_assert_errors
+- name: Stop if non systemd OS type
+  assert:
+    that: ansible_service_mgr == "systemd"
+  when: not ignore_assert_errors
+
+- name: Stop if the os does not support
+  assert:
+    that: (allow_unsupported_distribution_setup | default(false)) or ansible_distribution in supported_os_distributions
+    msg: "{{ ansible_distribution }} is not a known OS"
+  when: not ignore_assert_errors
+
+- name: Stop if memory is too small for control plane nodes
+  assert:
+    that: ansible_memtotal_mb >= minimal_master_memory_mb
+  when:
+    - not ignore_assert_errors
+    - ('kube_control_plane' in group_names)
+
+- name: Stop if memory is too small for nodes
+  assert:
+    that: ansible_memtotal_mb >= minimal_node_memory_mb
+  when:
+    - not ignore_assert_errors
+    - ('kube_node' in group_names)
+
+# This command will fail if cgroups are not enabled on the node.
+# For reference: https://kubernetes.io/docs/concepts/architecture/cgroups/#check-cgroup-version
+- name: Stop if cgroups are not enabled on nodes
+  command: stat -fc %T /sys/fs/cgroup/
+  changed_when: false
+  when: not ignore_assert_errors
+
+- name: Stop if ip var does not match local ips
+  assert:
+    that: (ip in ansible_all_ipv4_addresses) or (ip in ansible_all_ipv6_addresses)
+    msg: "IPv4: '{{ ansible_all_ipv4_addresses }}' and IPv6: '{{ ansible_all_ipv6_addresses }}' do not contain '{{ ip }}'"
+  when:
+    - not ignore_assert_errors
+    - ip is defined
+
+- name: Stop if access_ip is not pingable
+  command: ping -c1 {{ main_access_ip }}
+  when:
+    - main_access_ip is defined
+    - not ignore_assert_errors
+    - ping_access_ip
+  changed_when: false
+
+- name: Stop if kernel version is too low for cilium
+  assert:
+    that: ansible_kernel.split('-')[0] is version('4.9.17', '>=')
+  when:
+    - kube_network_plugin == 'cilium' or cilium_deploy_additionally
+    - not ignore_assert_errors
+
+- name: Stop if kernel version is too low for nftables
+  assert:
+    that: ansible_kernel.split('-')[0] is version('5.13', '>=')
+  when:
+    - kube_proxy_mode == 'nftables'
+    - not ignore_assert_errors
+
+- name: Stop if bad hostname
+  assert:
+    that: inventory_hostname is match("[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$")
+    msg: "Hostname must consist of lower case alphanumeric characters, '.' or '-', and must start and end with an alphanumeric character"
+  when: not ignore_assert_errors
+
+- name: Stop if /etc/resolv.conf has no configured nameservers
+  assert:
+    that: configured_nameservers | length>0
+    fail_msg: "nameserver should not be empty in /etc/resolv.conf"
+  when:
+    - upstream_dns_servers | length == 0
+    - not disable_host_nameservers
+    - dns_mode in ['coredns', 'coredns_dual']
+
+- name: Stop if download_localhost is enabled for Flatcar Container Linux
+  assert:
+    that: ansible_os_family not in ["Flatcar", "Flatcar Container Linux by Kinvolk"]
+    msg: "download_run_once not supported for Flatcar Container Linux"
+  when: download_run_once or download_force_cache

internal/resources/kubespray/project/roles/kubernetes/preinstall/tasks/0050-create_directories.yml

@@ -0,0 +1,119 @@
+---
+- name: Create kubernetes directories
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: "{{ kube_owner }}"
+    mode: "0755"
+  when: ('k8s_cluster' in group_names)
+  become: true
+  tags:
+    - kubelet
+    - kube-controller-manager
+    - kube-apiserver
+    - bootstrap_os
+    - apps
+    - network
+    - master    # master tag is deprecated and replaced by control-plane
+    - control-plane
+    - node
+  with_items:
+    - "{{ kube_config_dir }}"
+    - "{{ kube_manifest_dir }}"
+    - "{{ kube_script_dir }}"
+    - "{{ kubelet_flexvolumes_plugins_dir }}"
+
+- name: Create other directories of root owner
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: root
+    mode: "0755"
+  when: ('k8s_cluster' in group_names)
+  become: true
+  tags:
+    - kubelet
+    - kube-controller-manager
+    - kube-apiserver
+    - bootstrap_os
+    - apps
+    - network
+    - master    # master tag is deprecated and replaced by control-plane
+    - control-plane
+    - node
+  with_items:
+    - "{{ kube_cert_dir }}"
+    - "{{ bin_dir }}"
+
+- name: Check if kubernetes kubeadm compat cert dir exists
+  stat:
+    path: "{{ kube_cert_compat_dir }}"
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: kube_cert_compat_dir_check
+  when:
+    - ('k8s_cluster' in group_names)
+    - kube_cert_dir != kube_cert_compat_dir
+
+- name: Create kubernetes kubeadm compat cert dir (kubernetes/kubeadm issue 1498)
+  file:
+    src: "{{ kube_cert_dir }}"
+    dest: "{{ kube_cert_compat_dir }}"
+    state: link
+    mode: "0755"
+  when:
+    - ('k8s_cluster' in group_names)
+    - kube_cert_dir != kube_cert_compat_dir
+    - not kube_cert_compat_dir_check.stat.exists
+
+- name: Create cni directories
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: "{{ kube_owner }}"
+    mode: "0755"
+  with_items:
+    - "/etc/cni/net.d"
+    - "/opt/cni/bin"
+  when:
+    - kube_network_plugin in ["calico", "weave", "flannel", "cilium", "kube-ovn", "kube-router", "macvlan"]
+    - ('k8s_cluster' in group_names)
+  tags:
+    - network
+    - cilium
+    - calico
+    - weave
+    - kube-ovn
+    - kube-router
+    - bootstrap_os
+
+- name: Create calico cni directories
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: "{{ kube_owner }}"
+    mode: "0755"
+  with_items:
+    - "/var/lib/calico"
+  when:
+    - kube_network_plugin == "calico"
+    - ('k8s_cluster' in group_names)
+  tags:
+    - network
+    - calico
+    - bootstrap_os
+
+- name: Create local volume provisioner directories
+  file:
+    path: "{{ local_volume_provisioner_storage_classes[item].host_dir }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "{{ local_volume_provisioner_directory_mode }}"
+  with_items: "{{ local_volume_provisioner_storage_classes.keys() | list }}"
+  when:
+    - ('k8s_cluster' in group_names)
+    - local_volume_provisioner_enabled
+  tags:
+    - persistent_volumes

internal/resources/kubespray/project/roles/kubernetes/preinstall/tasks/0060-resolvconf.yml

@@ -0,0 +1,57 @@
+---
+- name: Create temporary resolveconf cloud init file
+  command: cp -f /etc/resolv.conf "{{ resolvconffile }}"
+  when: ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"]
+
+- name: Add domain/search/nameservers/options to resolv.conf
+  blockinfile:
+    path: "{{ resolvconffile }}"
+    block: |-
+      domain {{ dns_domain }}
+      search {{ (default_searchdomains + searchdomains) | join(' ') }}
+      {% for item in nameserverentries %}
+      nameserver {{ item }}
+      {% endfor %}
+      options ndots:{{ ndots }} timeout:{{ dns_timeout | default('2') }} attempts:{{ dns_attempts | default('2') }}
+    state: present
+    insertbefore: BOF
+    create: true
+    backup: "{{ not resolvconf_stat.stat.islnk }}"
+    marker: "# Ansible entries {mark}"
+    mode: "0644"
+  notify: Preinstall | propagate resolvconf to k8s components
+
+- name: Remove search/domain/nameserver options before block
+  replace:
+    path: "{{ item[0] }}"
+    regexp: '^{{ item[1] }}[^#]*(?=# Ansible entries BEGIN)'
+    backup: "{{ not resolvconf_stat.stat.islnk }}"
+  with_nested:
+    - "{{ [resolvconffile, base | default(''), head | default('')] | difference(['']) }}"
+    - [ 'search\s', 'nameserver\s', 'domain\s', 'options\s' ]
+  notify: Preinstall | propagate resolvconf to k8s components
+
+- name: Remove search/domain/nameserver options after block
+  replace:
+    path: "{{ item[0] }}"
+    regexp: '(# Ansible entries END\n(?:(?!^{{ item[1] }}).*\n)*)(?:^{{ item[1] }}.*\n?)+'
+    replace: '\1'
+    backup: "{{ not resolvconf_stat.stat.islnk }}"
+  with_nested:
+    - "{{ [resolvconffile, base | default(''), head | default('')] | difference(['']) }}"
+    - [ 'search\s', 'nameserver\s', 'domain\s', 'options\s' ]
+  notify: Preinstall | propagate resolvconf to k8s components
+
+- name: Get temporary resolveconf cloud init file content
+  command: cat {{ resolvconffile }}
+  register: cloud_config
+  when: ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"]
+
+- name: Persist resolvconf cloud init file
+  template:
+    dest: "{{ resolveconf_cloud_init_conf }}"
+    src: resolvconf.j2
+    owner: root
+    mode: "0644"
+  notify: Preinstall | update resolvconf for Flatcar Container Linux by Kinvolk
+  when: ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"]

internal/resources/kubespray/project/roles/kubernetes/preinstall/tasks/0061-systemd-resolved.yml

@@ -0,0 +1,15 @@
+---
+- name: Create systemd-resolved drop-in directory
+  file:
+    state: directory
+    name: /etc/systemd/resolved.conf.d/
+    mode: "0755"
+
+- name: Write Kubespray DNS settings to systemd-resolved
+  template:
+    src: resolved.conf.j2
+    dest: /etc/systemd/resolved.conf.d/kubespray.conf
+    owner: root
+    group: root
+    mode: "0644"
+  notify: Preinstall | Restart systemd-resolved

internal/resources/kubespray/project/roles/kubernetes/preinstall/tasks/0062-networkmanager-unmanaged-devices.yml

@@ -0,0 +1,28 @@
+---
+- name: NetworkManager | Ensure NetworkManager conf.d dir
+  file:
+    path: "/etc/NetworkManager/conf.d"
+    state: directory
+    recurse: true
+
+- name: NetworkManager | Prevent NetworkManager from managing Calico interfaces (cali*/tunl*/vxlan.calico)
+  copy:
+    content: |
+      [keyfile]
+      unmanaged-devices+=interface-name:cali*;interface-name:tunl*;interface-name:vxlan.calico;interface-name:vxlan-v6.calico
+    dest: /etc/NetworkManager/conf.d/calico.conf
+    mode: "0644"
+  when:
+    - kube_network_plugin == "calico"
+  notify: Preinstall | reload NetworkManager
+
+# TODO: add other network_plugin interfaces
+
+- name: NetworkManager | Prevent NetworkManager from managing K8S interfaces (kube-ipvs0/nodelocaldns)
+  copy:
+    content: |
+      [keyfile]
+      unmanaged-devices+=interface-name:kube-ipvs0;interface-name:nodelocaldns
+    dest: /etc/NetworkManager/conf.d/k8s.conf
+    mode: "0644"
+  notify: Preinstall | reload NetworkManager

internal/resources/kubespray/project/roles/kubernetes/preinstall/tasks/0063-networkmanager-dns.yml

@@ -0,0 +1,38 @@
+---
+- name: NetworkManager | Add nameservers to NM configuration
+  community.general.ini_file:
+    path: /etc/NetworkManager/conf.d/dns.conf
+    section: global-dns-domain-*
+    option: servers
+    value: "{{ nameserverentries | join(',') }}"
+    mode: '0600'
+    backup: "{{ leave_etc_backup_files }}"
+  when:
+    - ('127.0.0.53' not in nameserverentries
+       or systemd_resolved_enabled.rc != 0)
+  notify: Preinstall | update resolvconf for networkmanager
+
+- name: Set default dns if remove_default_searchdomains is false
+  set_fact:
+    default_searchdomains: ["default.svc.{{ dns_domain }}", "svc.{{ dns_domain }}"]
+  when: not remove_default_searchdomains | default() | bool or (remove_default_searchdomains | default() | bool and searchdomains | default([]) | length==0)
+
+- name: NetworkManager | Add DNS search to NM configuration
+  community.general.ini_file:
+    path: /etc/NetworkManager/conf.d/dns.conf
+    section: global-dns
+    option: searches
+    value: "{{ (default_searchdomains | default([]) + searchdomains) | join(',') }}"
+    mode: '0600'
+    backup: "{{ leave_etc_backup_files }}"
+  notify: Preinstall | update resolvconf for networkmanager
+
+- name: NetworkManager | Add DNS options to NM configuration
+  community.general.ini_file:
+    path: /etc/NetworkManager/conf.d/dns.conf
+    section: global-dns
+    option: options
+    value: "ndots:{{ ndots }},timeout:{{ dns_timeout | default('2') }},attempts:{{ dns_attempts | default('2') }}"
+    mode: '0600'
+    backup: "{{ leave_etc_backup_files }}"
+  notify: Preinstall | update resolvconf for networkmanager

internal/resources/kubespray/project/roles/kubernetes/preinstall/tasks/0080-system-configurations.yml

@@ -0,0 +1,146 @@
+---
+# Todo : selinux configuration
+- name: Confirm selinux deployed
+  stat:
+    path: /etc/selinux/config
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  when:
+    - ansible_os_family == "RedHat"
+    - "'Amazon' not in ansible_distribution"
+  register: slc
+
+- name: Set selinux policy
+  ansible.posix.selinux:
+    policy: targeted
+    state: "{{ preinstall_selinux_state }}"
+  when:
+    - ansible_os_family == "RedHat"
+    - "'Amazon' not in ansible_distribution"
+    - slc.stat.exists
+  tags:
+    - bootstrap_os
+
+- name: Disable IPv6 DNS lookup
+  lineinfile:
+    dest: /etc/gai.conf
+    line: "precedence ::ffff:0:0/96  100"
+    state: present
+    create: true
+    backup: "{{ leave_etc_backup_files }}"
+    mode: "0644"
+  when:
+    - disable_ipv6_dns
+    - not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"]
+  tags:
+    - bootstrap_os
+
+- name: Clean previously used sysctl file locations
+  file:
+    path: "/etc/sysctl.d/{{ item }}"
+    state: absent
+  with_items:
+    - ipv4-ip_forward.conf
+    - bridge-nf-call.conf
+
+- name: Stat sysctl file configuration
+  stat:
+    path: "{{ sysctl_file_path }}"
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: sysctl_file_stat
+  tags:
+    - bootstrap_os
+
+- name: Change sysctl file path to link source if linked
+  set_fact:
+    sysctl_file_path: "{{ sysctl_file_stat.stat.lnk_source }}"
+  when:
+    - sysctl_file_stat.stat.islnk is defined
+    - sysctl_file_stat.stat.islnk
+  tags:
+    - bootstrap_os
+
+- name: Make sure sysctl file path folder exists
+  file:
+    name: "{{ sysctl_file_path | dirname }}"
+    state: directory
+    mode: "0755"
+
+- name: Enable ip forwarding
+  ansible.posix.sysctl:
+    sysctl_file: "{{ sysctl_file_path }}"
+    name: net.ipv4.ip_forward
+    value: "1"
+    state: present
+    reload: true
+  when: ipv4_stack | bool
+
+- name: Enable ipv6 forwarding
+  ansible.posix.sysctl:
+    sysctl_file: "{{ sysctl_file_path }}"
+    name: net.ipv6.conf.all.forwarding
+    value: "1"
+    state: present
+    reload: true
+  when: ipv6_stack | bool
+
+- name: Check if we need to set fs.may_detach_mounts
+  stat:
+    path: /proc/sys/fs/may_detach_mounts
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: fs_may_detach_mounts
+  ignore_errors: true  # noqa ignore-errors
+
+- name: Set fs.may_detach_mounts if needed
+  ansible.posix.sysctl:
+    sysctl_file: "{{ sysctl_file_path }}"
+    name: fs.may_detach_mounts
+    value: 1
+    state: present
+    reload: true
+  when: fs_may_detach_mounts.stat.exists | d(false)
+
+- name: Ensure kubelet expected parameters are set
+  ansible.posix.sysctl:
+    sysctl_file: "{{ sysctl_file_path }}"
+    name: "{{ item.name }}"
+    value: "{{ item.value }}"
+    state: present
+    reload: true
+  with_items:
+    - { name: kernel.keys.root_maxbytes, value: 25000000 }
+    - { name: kernel.keys.root_maxkeys, value: 1000000 }
+    - { name: kernel.panic, value: 10 }
+    - { name: kernel.panic_on_oops, value: 1 }
+    - { name: vm.overcommit_memory, value: 1 }
+    - { name: vm.panic_on_oom, value: 0 }
+  when: kubelet_protect_kernel_defaults | bool
+
+- name: Check dummy module
+  community.general.modprobe:
+    name: dummy
+    state: present
+    params: 'numdummies=0'
+  when: enable_nodelocaldns
+
+- name: Set additional sysctl variables
+  ansible.posix.sysctl:
+    sysctl_file: "{{ sysctl_file_path }}"
+    name: "{{ item.name }}"
+    value: "{{ item.value }}"
+    state: present
+    reload: true
+  with_items: "{{ additional_sysctl }}"
+
+- name: Disable fapolicyd service
+  failed_when: false
+  systemd_service:
+    name: fapolicyd
+    state: stopped
+    enabled: false
+  when: disable_fapolicyd

internal/resources/kubespray/project/roles/kubernetes/preinstall/tasks/0081-ntp-configurations.yml

@@ -0,0 +1,81 @@
+---
+
+- name: Disable systemd-timesyncd
+  service:
+    name: systemd-timesyncd.service
+    enabled: false
+    state: stopped
+  failed_when: false
+
+- name: Set fact NTP settings
+  set_fact:
+    # noqa: jinja[spacing]
+    ntp_config_file: >-
+      {% if ntp_package == "ntp" -%}
+      /etc/ntp.conf
+      {%- elif ntp_package == "ntpsec" -%}
+      /etc/ntpsec/ntp.conf
+      {%- elif ansible_os_family in ['RedHat', 'Suse'] -%}
+      /etc/chrony.conf
+      {%- else -%}
+      /etc/chrony/chrony.conf
+      {%- endif -%}
+    # noqa: jinja[spacing]
+    ntp_service_name: >-
+      {% if ntp_package == "chrony" -%}
+      chronyd
+      {%- elif ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk", "RedHat", "Suse"] -%}
+      ntpd
+      {%- else -%}
+      ntp
+      {%- endif %}
+
+- name: Generate NTP configuration file.
+  template:
+    src: "{{ ntp_config_file | basename }}.j2"
+    dest: "{{ ntp_config_file }}"
+    mode: "0644"
+  notify: Preinstall | restart ntp
+  when:
+    - ntp_manage_config
+
+- name: Stop the NTP Deamon For Sync Immediately   # `ntpd -gq`,`chronyd -q` requires the ntp daemon stop
+  service:
+    name: "{{ ntp_service_name }}"
+    state: stopped
+  when:
+    - ntp_force_sync_immediately
+
+- name: Force Sync NTP Immediately
+  # noqa: jinja[spacing]
+  command: >-
+      timeout -k 60s 60s
+      {% if ntp_package == "chrony" -%}
+      chronyd -q
+      {%- else -%}
+      ntpd -gq
+      {%- endif -%}
+  when:
+    - ntp_force_sync_immediately
+
+- name: Ensure NTP service is started and enabled
+  service:
+    name: "{{ ntp_service_name }}"
+    state: started
+    enabled: true
+
+- name: Ensure tzdata package
+  package:
+    name:
+      - tzdata
+    state: present
+  when:
+    - ntp_timezone
+    - not is_fedora_coreos
+    - not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"]
+
+- name: Set timezone
+  community.general.timezone:
+    name: "{{ ntp_timezone }}"
+  when:
+    - ntp_timezone

internal/resources/kubespray/project/roles/kubernetes/preinstall/tasks/0090-etchosts.yml

@@ -0,0 +1,80 @@
+---
+- name: Hosts | create hosts list from inventory
+  set_fact:
+    etc_hosts_inventory_block: |-
+      {% for item in (groups['k8s_cluster'] + groups['etcd'] | default([]) + groups['calico_rr'] | default([])) | unique %}
+      {{ hostvars[item]['main_access_ip'] }} {{ hostvars[item]['ansible_hostname'] | default(item) }}.{{ dns_domain }} {{ hostvars[item]['ansible_hostname'] | default(item) }}
+      {% if ipv4_stack and ipv6_stack %}
+      {{ hostvars[item]['access_ip6'] | default(hostvars[item]['ip6'] | default(hostvars[item]['ansible_default_ipv6']['address'])) }} {{ hostvars[item]['ansible_hostname'] | default(item) }}.{{ dns_domain }} {{ hostvars[item]['ansible_hostname'] | default(item) }}
+      {% endif %}
+      {% endfor %}
+  delegate_to: localhost
+  connection: local
+  delegate_facts: true
+  run_once: true
+
+- name: Hosts | populate inventory into hosts file
+  blockinfile:
+    path: /etc/hosts
+    block: "{{ hostvars.localhost.etc_hosts_inventory_block }}"
+    state: "{{ 'present' if populate_inventory_to_hosts_file else 'absent' }}"
+    create: true
+    backup: "{{ leave_etc_backup_files }}"
+    unsafe_writes: true
+    marker: "# Ansible inventory hosts {mark}"
+    mode: "0644"
+
+- name: Hosts | populate kubernetes loadbalancer address into hosts file
+  lineinfile:
+    dest: /etc/hosts
+    regexp: ".*{{ apiserver_loadbalancer_domain_name }}$"
+    line: "{{ loadbalancer_apiserver.address }} {{ apiserver_loadbalancer_domain_name }}"
+    state: present
+    backup: "{{ leave_etc_backup_files }}"
+    unsafe_writes: true
+  when:
+    - populate_loadbalancer_apiserver_to_hosts_file
+    - loadbalancer_apiserver is defined
+    - loadbalancer_apiserver.address is defined
+
+- name: Hosts | Update localhost entries in hosts file
+  when: populate_localhost_entries_to_hosts_file
+  block:
+    - name: Hosts | Retrieve hosts file content
+      slurp:
+        src: /etc/hosts
+      register: etc_hosts_content
+
+    - name: Hosts | Extract existing entries for localhost from hosts file
+      set_fact:
+        etc_hosts_localhosts_dict: >-
+          {%- set splitted = (item | regex_replace('[ \t]+', ' ') | regex_replace('#.*$') | trim).split(' ') -%}
+          {{ etc_hosts_localhosts_dict | default({}) | combine({splitted[0]: splitted[1::]}) }}
+      with_items: "{{ (etc_hosts_content['content'] | b64decode).splitlines() }}"
+      when:
+        - etc_hosts_content.content is defined
+        - (item is match('^::1 .*') or item is match('^127.0.0.1 .*'))
+
+    - name: Hosts | Update target hosts file entries dict with required entries
+      set_fact:
+        etc_hosts_localhosts_dict_target: >-
+          {%- set target_entries = (etc_hosts_localhosts_dict | default({})).get(item.key, []) | difference(item.value.get('unexpected', [])) -%}
+          {{ etc_hosts_localhosts_dict_target | default({}) | combine({item.key: (target_entries + item.value.expected) | unique}) }}
+      loop: "{{ etc_hosts_localhost_entries | dict2items }}"
+
+    - name: Hosts | Update (if necessary) hosts file
+      lineinfile:
+        dest: /etc/hosts
+        line: "{{ item.key }} {{ item.value | join(' ') }}"
+        regexp: "^{{ item.key }}.*$"
+        state: present
+        backup: "{{ leave_etc_backup_files }}"
+        unsafe_writes: true
+      loop: "{{ etc_hosts_localhosts_dict_target | default({}) | dict2items }}"
+
+# gather facts to update ansible_fqdn
+- name: Update facts
+  setup:
+    gather_subset: min
+  when:
+    - not dns_late

internal/resources/kubespray/project/roles/kubernetes/preinstall/tasks/0100-dhclient-hooks.yml

@@ -0,0 +1,38 @@
+---
+- name: Configure dhclient to supersede search/domain/nameservers
+  blockinfile:
+    # 1 is the 2nd item of a tuple in items()
+    block: |-
+      {% for key, val in dhclient_supersede.items() | rejectattr(1, '==', []) -%}
+      {% if key == "domain-name-servers" -%}
+      supersede {{ key }} {{ val | join(',') }};
+      {% else -%}
+      supersede {{ key }} "{{ val | join('","') }}";
+      {% endif -%}
+      {% endfor %}
+    path: "{{ dhclientconffile }}"
+    create: true
+    state: present
+    insertbefore: BOF
+    backup: "{{ leave_etc_backup_files }}"
+    marker: "# Ansible entries {mark}"
+    mode: "0644"
+  notify: Preinstall | propagate resolvconf to k8s components
+
+- name: Configure dhclient hooks for resolv.conf (non-RH)
+  template:
+    src: dhclient_dnsupdate.sh.j2
+    dest: "{{ dhclienthookfile }}"
+    owner: root
+    mode: "0755"
+  notify: Preinstall | propagate resolvconf to k8s components
+  when: ansible_os_family not in [ "RedHat", "Suse" ]
+
+- name: Configure dhclient hooks for resolv.conf (RH-only)
+  template:
+    src: dhclient_dnsupdate_rh.sh.j2
+    dest: "{{ dhclienthookfile }}"
+    owner: root
+    mode: "0755"
+  notify: Preinstall | propagate resolvconf to k8s components
+  when: ansible_os_family == "RedHat"

internal/resources/kubespray/project/roles/kubernetes/preinstall/tasks/0110-dhclient-hooks-undo.yml

@@ -0,0 +1,18 @@
+---
+
+# These tasks will undo changes done by kubespray in the past if needed (e.g. when upgrading from kubespray 2.0.x
+# or when changing resolvconf_mode)
+
+- name: Remove kubespray specific config from dhclient config
+  blockinfile:
+    path: "{{ dhclientconffile }}"
+    state: absent
+    backup: "{{ leave_etc_backup_files }}"
+    marker: "# Ansible entries {mark}"
+  notify: Preinstall | propagate resolvconf to k8s components
+
+- name: Remove kubespray specific dhclient hook
+  file:
+    path: "{{ dhclienthookfile }}"
+    state: absent
+  notify: Preinstall | propagate resolvconf to k8s components

internal/resources/kubespray/project/roles/kubernetes/preinstall/tasks/main.yml

@@ -0,0 +1,131 @@
+---
+# Disable swap
+- name: Disable swap
+  import_tasks: 0010-swapoff.yml
+  when:
+    - not dns_late
+    - kubelet_fail_swap_on
+
+- name: Set facts
+  import_tasks: 0020-set_facts.yml
+  tags:
+    - resolvconf
+    - facts
+
+- name: Check settings
+  import_tasks: 0040-verify-settings.yml
+  when:
+    - not dns_late
+  tags:
+    - asserts
+
+- name: Create directories
+  import_tasks: 0050-create_directories.yml
+  when:
+    - not dns_late
+
+- name: Apply resolvconf settings
+  import_tasks: 0060-resolvconf.yml
+  when:
+    - dns_mode != 'none'
+    - resolvconf_mode == 'host_resolvconf'
+    - systemd_resolved_enabled.rc != 0
+    - networkmanager_enabled.rc != 0
+  tags:
+    - bootstrap_os
+    - resolvconf
+
+- name: Apply systemd-resolved settings
+  import_tasks: 0061-systemd-resolved.yml
+  when:
+    - dns_mode != 'none'
+    - resolvconf_mode == 'host_resolvconf'
+    - systemd_resolved_enabled.rc == 0
+  tags:
+    - bootstrap_os
+    - resolvconf
+
+- name: Apply networkmanager unmanaged devices settings
+  import_tasks: 0062-networkmanager-unmanaged-devices.yml
+  when:
+    - networkmanager_enabled.rc == 0
+  tags:
+    - bootstrap_os
+
+- name: Apply networkmanager DNS settings
+  import_tasks: 0063-networkmanager-dns.yml
+  when:
+    - dns_mode != 'none'
+    - resolvconf_mode == 'host_resolvconf'
+    - networkmanager_enabled.rc == 0
+  tags:
+    - bootstrap_os
+    - resolvconf
+
+- name: Apply system configurations
+  import_tasks: 0080-system-configurations.yml
+  when:
+    - not dns_late
+  tags:
+    - bootstrap_os
+
+- name: Configure NTP
+  import_tasks: 0081-ntp-configurations.yml
+  when:
+    - not dns_late
+    - ntp_enabled
+  tags:
+    - bootstrap_os
+
+- name: Configure /etc/hosts
+  import_tasks: 0090-etchosts.yml
+  tags:
+    - bootstrap_os
+    - etchosts
+
+- name: Configure dhclient
+  import_tasks: 0100-dhclient-hooks.yml
+  when:
+    - dns_mode != 'none'
+    - resolvconf_mode == 'host_resolvconf'
+    - dhclientconffile is defined
+    - not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"]
+  tags:
+    - bootstrap_os
+    - resolvconf
+
+- name: Configure dhclient dhclient hooks
+  import_tasks: 0110-dhclient-hooks-undo.yml
+  when:
+    - dns_mode != 'none'
+    - resolvconf_mode != 'host_resolvconf'
+    - dhclientconffile is defined
+    - not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"]
+  tags:
+    - bootstrap_os
+    - resolvconf
+
+# We need to make sure the network is restarted early enough so that docker can later pick up the correct system
+# nameservers and search domains
+- name: Flush handlers
+  meta: flush_handlers
+
+- name: Check if we are running inside a Azure VM
+  stat:
+    path: /var/lib/waagent/
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: azure_check
+  when:
+    - not dns_late
+  tags:
+    - bootstrap_os
+
+- name: Run calico checks
+  include_role:
+    name: network_plugin/calico
+    tasks_from: check
+  when:
+    - kube_network_plugin == 'calico'
+    - not ignore_assert_errors

internal/resources/kubespray/project/roles/kubernetes/preinstall/templates/ansible_git.j2

@@ -0,0 +1,3 @@
+; This file contains the information which identifies the deployment state relative to the git repo
+[default]
+{{ gitinfo.stdout }}

internal/resources/kubespray/project/roles/kubernetes/preinstall/templates/chrony.conf.j2

@@ -0,0 +1,27 @@
+# {{ ansible_managed }}
+
+# Specify one or more NTP servers.
+# Use public servers from the pool.ntp.org project.
+# Please consider joining the pool (http://www.pool.ntp.org/join.html).
+{% for server in ntp_servers %}
+server {{ server }}
+{% endfor %}
+
+# Record the rate at which the system clock gains/losses time.
+driftfile /var/lib/chrony/drift
+
+{% if ntp_tinker_panic is sameas true %}
+# Force time sync if the drift exceeds the threshold specified
+# Useful for VMs that can be paused and much later resumed.
+makestep 1.0 -1
+{% else %}
+# Allow the system clock to be stepped in the first three updates
+# if its offset is larger than 1 second.
+makestep 1.0 3
+{% endif %}
+
+# Enable kernel synchronization of the real-time clock (RTC).
+rtcsync
+
+# Specify directory for log files.
+logdir /var/log/chrony

internal/resources/kubespray/project/roles/kubernetes/preinstall/templates/dhclient_dnsupdate.sh.j2

@@ -0,0 +1,13 @@
+#!/bin/sh
+#
+# Prepend resolver options to /etc/resolv.conf after dhclient`
+# regenerates the file. See man (5) resolver for more details.
+#
+if [ $reason = "BOUND" ]; then
+  if [ -n "$new_domain_search" -o -n "$new_domain_name_servers" ]; then
+    RESOLV_CONF=$(cat /etc/resolv.conf | sed -r '/^options (timeout|attempts|ndots).*$/d')
+    OPTIONS="options timeout:{{ dns_timeout|default('2') }} attempts:{{ dns_attempts|default('2') }} ndots:{{ ndots }}"
+
+    printf "%b\n" "$RESOLV_CONF\n$OPTIONS" > /etc/resolv.conf
+  fi
+fi

internal/resources/kubespray/project/roles/kubernetes/preinstall/templates/dhclient_dnsupdate_rh.sh.j2

@@ -0,0 +1,17 @@
+#!/bin/sh
+#
+# Prepend resolver options to /etc/resolv.conf after dhclient`
+# regenerates the file. See man (5) resolver for more details.
+#
+zdnsupdate_config() {
+  if [ -n "$new_domain_search" -o -n "$new_domain_name_servers" ]; then
+    RESOLV_CONF=$(cat /etc/resolv.conf | sed -r '/^options (timeout|attempts|ndots).*$/d')
+    OPTIONS="options timeout:{{ dns_timeout|default('2') }} attempts:{{ dns_attempts|default('2') }} ndots:{{ ndots }}"
+
+    echo -e "$RESOLV_CONF\n$OPTIONS" > /etc/resolv.conf
+  fi
+}
+
+zdnsupdate_restore() {
+  :
+}

internal/resources/kubespray/project/roles/kubernetes/preinstall/templates/ntp.conf.j2

@@ -0,0 +1,52 @@
+# {{ ansible_managed }}
+
+# /etc/ntp.conf, configuration for ntpd; see ntp.conf(5) for help
+
+driftfile {{ ntp_driftfile }}
+
+{% if ntp_tinker_panic is sameas true %}
+# Always reset the clock, even if the new time is more than 1000s away
+# from the current system time. Useful for VMs that can be paused
+# and much later resumed.
+tinker panic 0
+{% endif %}
+
+# Specify one or more NTP servers.
+# Use public servers from the pool.ntp.org project.
+# Please consider joining the pool (http://www.pool.ntp.org/join.html).
+{% for item in ntp_servers %}
+pool {{ item }}
+{% endfor %}
+
+# Access control configuration; see /usr/share/doc/ntp-doc/html/accopt.html for
+# details.  The web page <http://support.ntp.org/bin/view/Support/AccessRestrictions>
+# might also be helpful.
+#
+# Note that "restrict" applies to both servers and clients, so a configuration
+# that might be intended to block requests from certain clients could also end
+# up blocking replies from your own upstream servers.
+
+# By default, exchange time with everybody, but don't allow configuration.
+restrict -4 default kod notrap nomodify nopeer noquery limited
+restrict -6 default kod notrap nomodify nopeer noquery limited
+
+# Local users may interrogate the ntp server more closely.
+{% for item in ntp_restrict %}
+restrict {{ item }}
+{% endfor %}
+
+# Needed for filtering interfaces
+{% if ntp_filter_interface %}
+{% for item in ntp_interfaces %}
+interface {{ item }}
+{% endfor %}
+{% endif %}
+
+# Needed for adding pool entries
+restrict source notrap nomodify noquery
+
+# Disable the monitoring facility to prevent amplification attacks using ntpdc
+# monlist command when default restrict does not include the noquery flag. See
+# CVE-2013-5211 for more details.
+# Note: Monitoring will not be disabled with the limited restriction flag.
+disable monitor

internal/resources/kubespray/project/roles/kubernetes/preinstall/templates/resolvconf.j2

@@ -0,0 +1,10 @@
+#cloud-config
+write_files:
+  - path: "/etc/resolv.conf"
+    permissions: "0644"
+    owner: "root"
+    content: |
+    {% for l in cloud_config.stdout_lines %}
+      {{ l }}
+    {% endfor %}
+    #

internal/resources/kubespray/project/roles/kubernetes/preinstall/templates/resolved.conf.j2

@@ -0,0 +1,15 @@
+[Resolve]
+{% if not dns_early and dns_late %}
+DNS={{ ([nodelocaldns_ip] if enable_nodelocaldns else coredns_server) | list | join(' ') }}
+{% endif %}
+FallbackDNS={{ ( upstream_dns_servers + nameservers + cloud_resolver) | unique | join(' ') }}
+{% if remove_default_searchdomains and searchdomains | length != 0 %}
+Domains={{ searchdomains | join(' ') }}
+{% else %}
+Domains={{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains) | join(' ') }}
+{% endif %}
+DNSSEC=no
+Cache=no-negative
+{% if systemd_resolved_disable_stub_listener | bool %}
+DNSStubListener=no
+{% endif %}

internal/resources/kubespray/project/roles/kubernetes/preinstall/vars/main.yml

@@ -0,0 +1,22 @@
+---
+coredns_server_by_mode:
+  coredns: "{{ [skydns_server] }}"
+  coredns_dual: "{{ [skydns_server, skydns_server_secondary] }}"
+  manual: "{{ manual_dns_server.split(',') }}"
+  none: []
+coredns_server: "{{ upstream_dns_servers if dns_early else coredns_server_by_mode[dns_mode] }}"
+
+_nameserverentries:
+  late:
+    - "{{ nodelocaldns_ip if enable_nodelocaldns else coredns_server }}"
+  early:
+    - "{{ nameservers }}"
+    - "{{ cloud_resolver }}"
+    - "{{ configured_nameservers if not disable_host_nameservers else [] }}"
+nameserverentries: "{{ ((_nameserverentries['late'] if not dns_early else []) + _nameserverentries['early']) | flatten | unique }}"
+dhclient_supersede:
+  domain-name-servers: "{{ ([nameservers, cloud_resolver] | flatten | unique) if dns_early else nameserverentries }}"
+  domain-name: "{{ [dns_domain] }}"
+  domain-search: "{{ default_searchdomains + searchdomains }}"
+configured_nameservers: "{{ (resolvconf_slurp.content | b64decode | regex_findall('^nameserver\\s*(\\S*)', multiline=True) | ansible.utils.ipaddr)
+                            if resolvconf_stat.stat.exists else [] }}"