kube-forge v2: embede python with ansible (kubespray) and cobra as cli building tool

internal/resources/kubespray/project/roles/kubernetes/preinstall/tasks/0010-swapoff.yml

@@ -0,0 +1,26 @@
+---
+- name: Check if /etc/fstab exists
+  stat:
+    path: "/etc/fstab"
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: fstab_file
+
+- name: Remove swapfile from /etc/fstab
+  ansible.posix.mount:
+    name: "{{ item }}"
+    fstype: swap
+    state: absent
+  loop:
+    - swap
+    - none
+  when: fstab_file.stat.exists
+
+- name: Mask swap.target (persist swapoff)
+  ansible.builtin.systemd_service:
+    name: swap.target
+    masked: true
+
+- name: Disable swap
+  command: /sbin/swapoff -a

internal/resources/kubespray/project/roles/kubernetes/preinstall/tasks/0020-set_facts.yml

@@ -0,0 +1,152 @@
+---
+- name: Set os_family fact for other redhat-based operating systems
+  set_fact:
+    ansible_os_family: "RedHat"
+    ansible_distribution_major_version: "8"
+  when: ansible_distribution in redhat_os_family_extensions
+  tags:
+    - facts
+
+- name: Check resolvconf
+  command: which resolvconf
+  register: resolvconf
+  failed_when: false
+  changed_when: false
+  check_mode: false
+
+- name: Check existence of /etc/resolvconf/resolv.conf.d
+  stat:
+    path: /etc/resolvconf/resolv.conf.d
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  failed_when: false
+  register: resolvconfd_path
+
+- name: Check status of /etc/resolv.conf
+  stat:
+    path: /etc/resolv.conf
+    follow: false
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  failed_when: false
+  register: resolvconf_stat
+
+  # Used in vars/
+- name: Fetch resolv.conf
+  when: resolvconf_stat.stat.exists
+  slurp:
+    src: /etc/resolv.conf
+  register: resolvconf_slurp
+
+- name: NetworkManager | Check if host has NetworkManager
+  # noqa command-instead-of-module - Should we use service_facts for this?
+  command: systemctl is-active --quiet NetworkManager.service
+  register: networkmanager_enabled
+  failed_when: false
+  changed_when: false
+  check_mode: false
+
+- name: Check systemd-resolved
+  # noqa command-instead-of-module - Should we use service_facts for this?
+  command: systemctl is-active systemd-resolved
+  register: systemd_resolved_enabled
+  failed_when: false
+  changed_when: false
+  check_mode: false
+
+- name: Set default dns if remove_default_searchdomains is false
+  set_fact:
+    default_searchdomains: ["default.svc.{{ dns_domain }}", "svc.{{ dns_domain }}"]
+  when: not remove_default_searchdomains | default() | bool or (remove_default_searchdomains | default() | bool and searchdomains | length == 0)
+
+- name: Set dns facts
+  set_fact:
+    resolvconf: >-
+      {%- if resolvconf.rc == 0 and resolvconfd_path.stat.isdir is defined and resolvconfd_path.stat.isdir -%}true{%- else -%}false{%- endif -%}
+
+- name: Check if kubelet is configured
+  stat:
+    path: "{{ kube_config_dir }}/kubelet.env"
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: kubelet_configured
+  changed_when: false
+
+- name: Check if early DNS configuration stage
+  set_fact:
+    dns_early: "{{ not kubelet_configured.stat.exists }}"
+
+- name: Target resolv.conf files
+  set_fact:
+    resolvconffile: /etc/resolv.conf
+    base: >-
+      {%- if resolvconf | bool -%}/etc/resolvconf/resolv.conf.d/base{%- endif -%}
+    head: >-
+      {%- if resolvconf | bool -%}/etc/resolvconf/resolv.conf.d/head{%- endif -%}
+  when: not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] and not is_fedora_coreos
+
+- name: Target temporary resolvconf cloud init file (Flatcar Container Linux by Kinvolk / Fedora CoreOS)
+  set_fact:
+    resolvconffile: /tmp/resolveconf_cloud_init_conf
+  when: ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] or is_fedora_coreos
+
+- name: Check if /etc/dhclient.conf exists
+  stat:
+    path: /etc/dhclient.conf
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: dhclient_stat
+
+- name: Target dhclient conf file for /etc/dhclient.conf
+  set_fact:
+    dhclientconffile: /etc/dhclient.conf
+  when: dhclient_stat.stat.exists
+
+- name: Check if /etc/dhcp/dhclient.conf exists
+  stat:
+    path: /etc/dhcp/dhclient.conf
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: dhcp_dhclient_stat
+
+- name: Target dhclient conf file for /etc/dhcp/dhclient.conf
+  set_fact:
+    dhclientconffile: /etc/dhcp/dhclient.conf
+  when: dhcp_dhclient_stat.stat.exists
+
+- name: Target dhclient hook file for Red Hat family
+  set_fact:
+    dhclienthookfile: /etc/dhcp/dhclient.d/zdnsupdate.sh
+  when: ansible_os_family == "RedHat"
+
+- name: Target dhclient hook file for Debian family
+  set_fact:
+    dhclienthookfile: /etc/dhcp/dhclient-exit-hooks.d/zdnsupdate
+  when: ansible_os_family == "Debian"
+
+- name: Set etcd vars if using kubeadm mode
+  set_fact:
+    etcd_cert_dir: "{{ kube_cert_dir }}"
+    kube_etcd_cacert_file: "etcd/ca.crt"
+    kube_etcd_cert_file: "apiserver-etcd-client.crt"
+    kube_etcd_key_file: "apiserver-etcd-client.key"
+  when:
+    - etcd_deployment_type == "kubeadm"
+
+- name: Check /usr readonly
+  stat:
+    path: "/usr"
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: usr
+
+- name: Set alternate flexvolume path
+  set_fact:
+    kubelet_flexvolumes_plugins_dir: /var/lib/kubelet/volumeplugins
+  when: not usr.stat.writeable

internal/resources/kubespray/project/roles/kubernetes/preinstall/tasks/0040-verify-settings.yml

@@ -0,0 +1,97 @@
+---
+- name: Stop if any host not in '--limit' does not have a fact cache
+  vars:
+    uncached_hosts: "{{ hostvars | dict2items | selectattr('value.ansible_default_ipv6', 'undefined') | selectattr('value.ansible_default_ipv4', 'undefined') | map(attribute='key') }}"
+    excluded_hosts: "{{ groups['k8s_cluster'] | difference(query('inventory_hostnames', ansible_limit)) }}"
+  assert:
+    that: uncached_hosts | intersect(excluded_hosts) == []
+    fail_msg: |
+      Kubespray does not support '--limit' without a populated facts cache for the excluded hosts.
+      Please run the facts.yml playbook first without '--limit'.
+      The following excluded hosts are not cached: {{ uncached_hosts | intersect(excluded_hosts) }}
+  run_once: true
+  when:
+    - ansible_limit is defined
+    - not ignore_assert_errors
+- name: Stop if non systemd OS type
+  assert:
+    that: ansible_service_mgr == "systemd"
+  when: not ignore_assert_errors
+
+- name: Stop if the os does not support
+  assert:
+    that: (allow_unsupported_distribution_setup | default(false)) or ansible_distribution in supported_os_distributions
+    msg: "{{ ansible_distribution }} is not a known OS"
+  when: not ignore_assert_errors
+
+- name: Stop if memory is too small for control plane nodes
+  assert:
+    that: ansible_memtotal_mb >= minimal_master_memory_mb
+  when:
+    - not ignore_assert_errors
+    - ('kube_control_plane' in group_names)
+
+- name: Stop if memory is too small for nodes
+  assert:
+    that: ansible_memtotal_mb >= minimal_node_memory_mb
+  when:
+    - not ignore_assert_errors
+    - ('kube_node' in group_names)
+
+# This command will fail if cgroups are not enabled on the node.
+# For reference: https://kubernetes.io/docs/concepts/architecture/cgroups/#check-cgroup-version
+- name: Stop if cgroups are not enabled on nodes
+  command: stat -fc %T /sys/fs/cgroup/
+  changed_when: false
+  when: not ignore_assert_errors
+
+- name: Stop if ip var does not match local ips
+  assert:
+    that: (ip in ansible_all_ipv4_addresses) or (ip in ansible_all_ipv6_addresses)
+    msg: "IPv4: '{{ ansible_all_ipv4_addresses }}' and IPv6: '{{ ansible_all_ipv6_addresses }}' do not contain '{{ ip }}'"
+  when:
+    - not ignore_assert_errors
+    - ip is defined
+
+- name: Stop if access_ip is not pingable
+  command: ping -c1 {{ main_access_ip }}
+  when:
+    - main_access_ip is defined
+    - not ignore_assert_errors
+    - ping_access_ip
+  changed_when: false
+
+- name: Stop if kernel version is too low for cilium
+  assert:
+    that: ansible_kernel.split('-')[0] is version('4.9.17', '>=')
+  when:
+    - kube_network_plugin == 'cilium' or cilium_deploy_additionally
+    - not ignore_assert_errors
+
+- name: Stop if kernel version is too low for nftables
+  assert:
+    that: ansible_kernel.split('-')[0] is version('5.13', '>=')
+  when:
+    - kube_proxy_mode == 'nftables'
+    - not ignore_assert_errors
+
+- name: Stop if bad hostname
+  assert:
+    that: inventory_hostname is match("[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$")
+    msg: "Hostname must consist of lower case alphanumeric characters, '.' or '-', and must start and end with an alphanumeric character"
+  when: not ignore_assert_errors
+
+- name: Stop if /etc/resolv.conf has no configured nameservers
+  assert:
+    that: configured_nameservers | length>0
+    fail_msg: "nameserver should not be empty in /etc/resolv.conf"
+  when:
+    - upstream_dns_servers | length == 0
+    - not disable_host_nameservers
+    - dns_mode in ['coredns', 'coredns_dual']
+
+- name: Stop if download_localhost is enabled for Flatcar Container Linux
+  assert:
+    that: ansible_os_family not in ["Flatcar", "Flatcar Container Linux by Kinvolk"]
+    msg: "download_run_once not supported for Flatcar Container Linux"
+  when: download_run_once or download_force_cache

internal/resources/kubespray/project/roles/kubernetes/preinstall/tasks/0050-create_directories.yml

@@ -0,0 +1,119 @@
+---
+- name: Create kubernetes directories
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: "{{ kube_owner }}"
+    mode: "0755"
+  when: ('k8s_cluster' in group_names)
+  become: true
+  tags:
+    - kubelet
+    - kube-controller-manager
+    - kube-apiserver
+    - bootstrap_os
+    - apps
+    - network
+    - master    # master tag is deprecated and replaced by control-plane
+    - control-plane
+    - node
+  with_items:
+    - "{{ kube_config_dir }}"
+    - "{{ kube_manifest_dir }}"
+    - "{{ kube_script_dir }}"
+    - "{{ kubelet_flexvolumes_plugins_dir }}"
+
+- name: Create other directories of root owner
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: root
+    mode: "0755"
+  when: ('k8s_cluster' in group_names)
+  become: true
+  tags:
+    - kubelet
+    - kube-controller-manager
+    - kube-apiserver
+    - bootstrap_os
+    - apps
+    - network
+    - master    # master tag is deprecated and replaced by control-plane
+    - control-plane
+    - node
+  with_items:
+    - "{{ kube_cert_dir }}"
+    - "{{ bin_dir }}"
+
+- name: Check if kubernetes kubeadm compat cert dir exists
+  stat:
+    path: "{{ kube_cert_compat_dir }}"
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: kube_cert_compat_dir_check
+  when:
+    - ('k8s_cluster' in group_names)
+    - kube_cert_dir != kube_cert_compat_dir
+
+- name: Create kubernetes kubeadm compat cert dir (kubernetes/kubeadm issue 1498)
+  file:
+    src: "{{ kube_cert_dir }}"
+    dest: "{{ kube_cert_compat_dir }}"
+    state: link
+    mode: "0755"
+  when:
+    - ('k8s_cluster' in group_names)
+    - kube_cert_dir != kube_cert_compat_dir
+    - not kube_cert_compat_dir_check.stat.exists
+
+- name: Create cni directories
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: "{{ kube_owner }}"
+    mode: "0755"
+  with_items:
+    - "/etc/cni/net.d"
+    - "/opt/cni/bin"
+  when:
+    - kube_network_plugin in ["calico", "weave", "flannel", "cilium", "kube-ovn", "kube-router", "macvlan"]
+    - ('k8s_cluster' in group_names)
+  tags:
+    - network
+    - cilium
+    - calico
+    - weave
+    - kube-ovn
+    - kube-router
+    - bootstrap_os
+
+- name: Create calico cni directories
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: "{{ kube_owner }}"
+    mode: "0755"
+  with_items:
+    - "/var/lib/calico"
+  when:
+    - kube_network_plugin == "calico"
+    - ('k8s_cluster' in group_names)
+  tags:
+    - network
+    - calico
+    - bootstrap_os
+
+- name: Create local volume provisioner directories
+  file:
+    path: "{{ local_volume_provisioner_storage_classes[item].host_dir }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "{{ local_volume_provisioner_directory_mode }}"
+  with_items: "{{ local_volume_provisioner_storage_classes.keys() | list }}"
+  when:
+    - ('k8s_cluster' in group_names)
+    - local_volume_provisioner_enabled
+  tags:
+    - persistent_volumes

internal/resources/kubespray/project/roles/kubernetes/preinstall/tasks/0060-resolvconf.yml

@@ -0,0 +1,57 @@
+---
+- name: Create temporary resolveconf cloud init file
+  command: cp -f /etc/resolv.conf "{{ resolvconffile }}"
+  when: ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"]
+
+- name: Add domain/search/nameservers/options to resolv.conf
+  blockinfile:
+    path: "{{ resolvconffile }}"
+    block: |-
+      domain {{ dns_domain }}
+      search {{ (default_searchdomains + searchdomains) | join(' ') }}
+      {% for item in nameserverentries %}
+      nameserver {{ item }}
+      {% endfor %}
+      options ndots:{{ ndots }} timeout:{{ dns_timeout | default('2') }} attempts:{{ dns_attempts | default('2') }}
+    state: present
+    insertbefore: BOF
+    create: true
+    backup: "{{ not resolvconf_stat.stat.islnk }}"
+    marker: "# Ansible entries {mark}"
+    mode: "0644"
+  notify: Preinstall | propagate resolvconf to k8s components
+
+- name: Remove search/domain/nameserver options before block
+  replace:
+    path: "{{ item[0] }}"
+    regexp: '^{{ item[1] }}[^#]*(?=# Ansible entries BEGIN)'
+    backup: "{{ not resolvconf_stat.stat.islnk }}"
+  with_nested:
+    - "{{ [resolvconffile, base | default(''), head | default('')] | difference(['']) }}"
+    - [ 'search\s', 'nameserver\s', 'domain\s', 'options\s' ]
+  notify: Preinstall | propagate resolvconf to k8s components
+
+- name: Remove search/domain/nameserver options after block
+  replace:
+    path: "{{ item[0] }}"
+    regexp: '(# Ansible entries END\n(?:(?!^{{ item[1] }}).*\n)*)(?:^{{ item[1] }}.*\n?)+'
+    replace: '\1'
+    backup: "{{ not resolvconf_stat.stat.islnk }}"
+  with_nested:
+    - "{{ [resolvconffile, base | default(''), head | default('')] | difference(['']) }}"
+    - [ 'search\s', 'nameserver\s', 'domain\s', 'options\s' ]
+  notify: Preinstall | propagate resolvconf to k8s components
+
+- name: Get temporary resolveconf cloud init file content
+  command: cat {{ resolvconffile }}
+  register: cloud_config
+  when: ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"]
+
+- name: Persist resolvconf cloud init file
+  template:
+    dest: "{{ resolveconf_cloud_init_conf }}"
+    src: resolvconf.j2
+    owner: root
+    mode: "0644"
+  notify: Preinstall | update resolvconf for Flatcar Container Linux by Kinvolk
+  when: ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"]

internal/resources/kubespray/project/roles/kubernetes/preinstall/tasks/0061-systemd-resolved.yml

@@ -0,0 +1,15 @@
+---
+- name: Create systemd-resolved drop-in directory
+  file:
+    state: directory
+    name: /etc/systemd/resolved.conf.d/
+    mode: "0755"
+
+- name: Write Kubespray DNS settings to systemd-resolved
+  template:
+    src: resolved.conf.j2
+    dest: /etc/systemd/resolved.conf.d/kubespray.conf
+    owner: root
+    group: root
+    mode: "0644"
+  notify: Preinstall | Restart systemd-resolved

internal/resources/kubespray/project/roles/kubernetes/preinstall/tasks/0062-networkmanager-unmanaged-devices.yml

@@ -0,0 +1,28 @@
+---
+- name: NetworkManager | Ensure NetworkManager conf.d dir
+  file:
+    path: "/etc/NetworkManager/conf.d"
+    state: directory
+    recurse: true
+
+- name: NetworkManager | Prevent NetworkManager from managing Calico interfaces (cali*/tunl*/vxlan.calico)
+  copy:
+    content: |
+      [keyfile]
+      unmanaged-devices+=interface-name:cali*;interface-name:tunl*;interface-name:vxlan.calico;interface-name:vxlan-v6.calico
+    dest: /etc/NetworkManager/conf.d/calico.conf
+    mode: "0644"
+  when:
+    - kube_network_plugin == "calico"
+  notify: Preinstall | reload NetworkManager
+
+# TODO: add other network_plugin interfaces
+
+- name: NetworkManager | Prevent NetworkManager from managing K8S interfaces (kube-ipvs0/nodelocaldns)
+  copy:
+    content: |
+      [keyfile]
+      unmanaged-devices+=interface-name:kube-ipvs0;interface-name:nodelocaldns
+    dest: /etc/NetworkManager/conf.d/k8s.conf
+    mode: "0644"
+  notify: Preinstall | reload NetworkManager

internal/resources/kubespray/project/roles/kubernetes/preinstall/tasks/0063-networkmanager-dns.yml

@@ -0,0 +1,38 @@
+---
+- name: NetworkManager | Add nameservers to NM configuration
+  community.general.ini_file:
+    path: /etc/NetworkManager/conf.d/dns.conf
+    section: global-dns-domain-*
+    option: servers
+    value: "{{ nameserverentries | join(',') }}"
+    mode: '0600'
+    backup: "{{ leave_etc_backup_files }}"
+  when:
+    - ('127.0.0.53' not in nameserverentries
+       or systemd_resolved_enabled.rc != 0)
+  notify: Preinstall | update resolvconf for networkmanager
+
+- name: Set default dns if remove_default_searchdomains is false
+  set_fact:
+    default_searchdomains: ["default.svc.{{ dns_domain }}", "svc.{{ dns_domain }}"]
+  when: not remove_default_searchdomains | default() | bool or (remove_default_searchdomains | default() | bool and searchdomains | default([]) | length==0)
+
+- name: NetworkManager | Add DNS search to NM configuration
+  community.general.ini_file:
+    path: /etc/NetworkManager/conf.d/dns.conf
+    section: global-dns
+    option: searches
+    value: "{{ (default_searchdomains | default([]) + searchdomains) | join(',') }}"
+    mode: '0600'
+    backup: "{{ leave_etc_backup_files }}"
+  notify: Preinstall | update resolvconf for networkmanager
+
+- name: NetworkManager | Add DNS options to NM configuration
+  community.general.ini_file:
+    path: /etc/NetworkManager/conf.d/dns.conf
+    section: global-dns
+    option: options
+    value: "ndots:{{ ndots }},timeout:{{ dns_timeout | default('2') }},attempts:{{ dns_attempts | default('2') }}"
+    mode: '0600'
+    backup: "{{ leave_etc_backup_files }}"
+  notify: Preinstall | update resolvconf for networkmanager

internal/resources/kubespray/project/roles/kubernetes/preinstall/tasks/0080-system-configurations.yml

@@ -0,0 +1,146 @@
+---
+# Todo : selinux configuration
+- name: Confirm selinux deployed
+  stat:
+    path: /etc/selinux/config
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  when:
+    - ansible_os_family == "RedHat"
+    - "'Amazon' not in ansible_distribution"
+  register: slc
+
+- name: Set selinux policy
+  ansible.posix.selinux:
+    policy: targeted
+    state: "{{ preinstall_selinux_state }}"
+  when:
+    - ansible_os_family == "RedHat"
+    - "'Amazon' not in ansible_distribution"
+    - slc.stat.exists
+  tags:
+    - bootstrap_os
+
+- name: Disable IPv6 DNS lookup
+  lineinfile:
+    dest: /etc/gai.conf
+    line: "precedence ::ffff:0:0/96  100"
+    state: present
+    create: true
+    backup: "{{ leave_etc_backup_files }}"
+    mode: "0644"
+  when:
+    - disable_ipv6_dns
+    - not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"]
+  tags:
+    - bootstrap_os
+
+- name: Clean previously used sysctl file locations
+  file:
+    path: "/etc/sysctl.d/{{ item }}"
+    state: absent
+  with_items:
+    - ipv4-ip_forward.conf
+    - bridge-nf-call.conf
+
+- name: Stat sysctl file configuration
+  stat:
+    path: "{{ sysctl_file_path }}"
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: sysctl_file_stat
+  tags:
+    - bootstrap_os
+
+- name: Change sysctl file path to link source if linked
+  set_fact:
+    sysctl_file_path: "{{ sysctl_file_stat.stat.lnk_source }}"
+  when:
+    - sysctl_file_stat.stat.islnk is defined
+    - sysctl_file_stat.stat.islnk
+  tags:
+    - bootstrap_os
+
+- name: Make sure sysctl file path folder exists
+  file:
+    name: "{{ sysctl_file_path | dirname }}"
+    state: directory
+    mode: "0755"
+
+- name: Enable ip forwarding
+  ansible.posix.sysctl:
+    sysctl_file: "{{ sysctl_file_path }}"
+    name: net.ipv4.ip_forward
+    value: "1"
+    state: present
+    reload: true
+  when: ipv4_stack | bool
+
+- name: Enable ipv6 forwarding
+  ansible.posix.sysctl:
+    sysctl_file: "{{ sysctl_file_path }}"
+    name: net.ipv6.conf.all.forwarding
+    value: "1"
+    state: present
+    reload: true
+  when: ipv6_stack | bool
+
+- name: Check if we need to set fs.may_detach_mounts
+  stat:
+    path: /proc/sys/fs/may_detach_mounts
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: fs_may_detach_mounts
+  ignore_errors: true  # noqa ignore-errors
+
+- name: Set fs.may_detach_mounts if needed
+  ansible.posix.sysctl:
+    sysctl_file: "{{ sysctl_file_path }}"
+    name: fs.may_detach_mounts
+    value: 1
+    state: present
+    reload: true
+  when: fs_may_detach_mounts.stat.exists | d(false)
+
+- name: Ensure kubelet expected parameters are set
+  ansible.posix.sysctl:
+    sysctl_file: "{{ sysctl_file_path }}"
+    name: "{{ item.name }}"
+    value: "{{ item.value }}"
+    state: present
+    reload: true
+  with_items:
+    - { name: kernel.keys.root_maxbytes, value: 25000000 }
+    - { name: kernel.keys.root_maxkeys, value: 1000000 }
+    - { name: kernel.panic, value: 10 }
+    - { name: kernel.panic_on_oops, value: 1 }
+    - { name: vm.overcommit_memory, value: 1 }
+    - { name: vm.panic_on_oom, value: 0 }
+  when: kubelet_protect_kernel_defaults | bool
+
+- name: Check dummy module
+  community.general.modprobe:
+    name: dummy
+    state: present
+    params: 'numdummies=0'
+  when: enable_nodelocaldns
+
+- name: Set additional sysctl variables
+  ansible.posix.sysctl:
+    sysctl_file: "{{ sysctl_file_path }}"
+    name: "{{ item.name }}"
+    value: "{{ item.value }}"
+    state: present
+    reload: true
+  with_items: "{{ additional_sysctl }}"
+
+- name: Disable fapolicyd service
+  failed_when: false
+  systemd_service:
+    name: fapolicyd
+    state: stopped
+    enabled: false
+  when: disable_fapolicyd

internal/resources/kubespray/project/roles/kubernetes/preinstall/tasks/0081-ntp-configurations.yml

@@ -0,0 +1,81 @@
+---
+
+- name: Disable systemd-timesyncd
+  service:
+    name: systemd-timesyncd.service
+    enabled: false
+    state: stopped
+  failed_when: false
+
+- name: Set fact NTP settings
+  set_fact:
+    # noqa: jinja[spacing]
+    ntp_config_file: >-
+      {% if ntp_package == "ntp" -%}
+      /etc/ntp.conf
+      {%- elif ntp_package == "ntpsec" -%}
+      /etc/ntpsec/ntp.conf
+      {%- elif ansible_os_family in ['RedHat', 'Suse'] -%}
+      /etc/chrony.conf
+      {%- else -%}
+      /etc/chrony/chrony.conf
+      {%- endif -%}
+    # noqa: jinja[spacing]
+    ntp_service_name: >-
+      {% if ntp_package == "chrony" -%}
+      chronyd
+      {%- elif ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk", "RedHat", "Suse"] -%}
+      ntpd
+      {%- else -%}
+      ntp
+      {%- endif %}
+
+- name: Generate NTP configuration file.
+  template:
+    src: "{{ ntp_config_file | basename }}.j2"
+    dest: "{{ ntp_config_file }}"
+    mode: "0644"
+  notify: Preinstall | restart ntp
+  when:
+    - ntp_manage_config
+
+- name: Stop the NTP Deamon For Sync Immediately   # `ntpd -gq`,`chronyd -q` requires the ntp daemon stop
+  service:
+    name: "{{ ntp_service_name }}"
+    state: stopped
+  when:
+    - ntp_force_sync_immediately
+
+- name: Force Sync NTP Immediately
+  # noqa: jinja[spacing]
+  command: >-
+      timeout -k 60s 60s
+      {% if ntp_package == "chrony" -%}
+      chronyd -q
+      {%- else -%}
+      ntpd -gq
+      {%- endif -%}
+  when:
+    - ntp_force_sync_immediately
+
+- name: Ensure NTP service is started and enabled
+  service:
+    name: "{{ ntp_service_name }}"
+    state: started
+    enabled: true
+
+- name: Ensure tzdata package
+  package:
+    name:
+      - tzdata
+    state: present
+  when:
+    - ntp_timezone
+    - not is_fedora_coreos
+    - not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"]
+
+- name: Set timezone
+  community.general.timezone:
+    name: "{{ ntp_timezone }}"
+  when:
+    - ntp_timezone

internal/resources/kubespray/project/roles/kubernetes/preinstall/tasks/0090-etchosts.yml

@@ -0,0 +1,80 @@
+---
+- name: Hosts | create hosts list from inventory
+  set_fact:
+    etc_hosts_inventory_block: |-
+      {% for item in (groups['k8s_cluster'] + groups['etcd'] | default([]) + groups['calico_rr'] | default([])) | unique %}
+      {{ hostvars[item]['main_access_ip'] }} {{ hostvars[item]['ansible_hostname'] | default(item) }}.{{ dns_domain }} {{ hostvars[item]['ansible_hostname'] | default(item) }}
+      {% if ipv4_stack and ipv6_stack %}
+      {{ hostvars[item]['access_ip6'] | default(hostvars[item]['ip6'] | default(hostvars[item]['ansible_default_ipv6']['address'])) }} {{ hostvars[item]['ansible_hostname'] | default(item) }}.{{ dns_domain }} {{ hostvars[item]['ansible_hostname'] | default(item) }}
+      {% endif %}
+      {% endfor %}
+  delegate_to: localhost
+  connection: local
+  delegate_facts: true
+  run_once: true
+
+- name: Hosts | populate inventory into hosts file
+  blockinfile:
+    path: /etc/hosts
+    block: "{{ hostvars.localhost.etc_hosts_inventory_block }}"
+    state: "{{ 'present' if populate_inventory_to_hosts_file else 'absent' }}"
+    create: true
+    backup: "{{ leave_etc_backup_files }}"
+    unsafe_writes: true
+    marker: "# Ansible inventory hosts {mark}"
+    mode: "0644"
+
+- name: Hosts | populate kubernetes loadbalancer address into hosts file
+  lineinfile:
+    dest: /etc/hosts
+    regexp: ".*{{ apiserver_loadbalancer_domain_name }}$"
+    line: "{{ loadbalancer_apiserver.address }} {{ apiserver_loadbalancer_domain_name }}"
+    state: present
+    backup: "{{ leave_etc_backup_files }}"
+    unsafe_writes: true
+  when:
+    - populate_loadbalancer_apiserver_to_hosts_file
+    - loadbalancer_apiserver is defined
+    - loadbalancer_apiserver.address is defined
+
+- name: Hosts | Update localhost entries in hosts file
+  when: populate_localhost_entries_to_hosts_file
+  block:
+    - name: Hosts | Retrieve hosts file content
+      slurp:
+        src: /etc/hosts
+      register: etc_hosts_content
+
+    - name: Hosts | Extract existing entries for localhost from hosts file
+      set_fact:
+        etc_hosts_localhosts_dict: >-
+          {%- set splitted = (item | regex_replace('[ \t]+', ' ') | regex_replace('#.*$') | trim).split(' ') -%}
+          {{ etc_hosts_localhosts_dict | default({}) | combine({splitted[0]: splitted[1::]}) }}
+      with_items: "{{ (etc_hosts_content['content'] | b64decode).splitlines() }}"
+      when:
+        - etc_hosts_content.content is defined
+        - (item is match('^::1 .*') or item is match('^127.0.0.1 .*'))
+
+    - name: Hosts | Update target hosts file entries dict with required entries
+      set_fact:
+        etc_hosts_localhosts_dict_target: >-
+          {%- set target_entries = (etc_hosts_localhosts_dict | default({})).get(item.key, []) | difference(item.value.get('unexpected', [])) -%}
+          {{ etc_hosts_localhosts_dict_target | default({}) | combine({item.key: (target_entries + item.value.expected) | unique}) }}
+      loop: "{{ etc_hosts_localhost_entries | dict2items }}"
+
+    - name: Hosts | Update (if necessary) hosts file
+      lineinfile:
+        dest: /etc/hosts
+        line: "{{ item.key }} {{ item.value | join(' ') }}"
+        regexp: "^{{ item.key }}.*$"
+        state: present
+        backup: "{{ leave_etc_backup_files }}"
+        unsafe_writes: true
+      loop: "{{ etc_hosts_localhosts_dict_target | default({}) | dict2items }}"
+
+# gather facts to update ansible_fqdn
+- name: Update facts
+  setup:
+    gather_subset: min
+  when:
+    - not dns_late

internal/resources/kubespray/project/roles/kubernetes/preinstall/tasks/0100-dhclient-hooks.yml

@@ -0,0 +1,38 @@
+---
+- name: Configure dhclient to supersede search/domain/nameservers
+  blockinfile:
+    # 1 is the 2nd item of a tuple in items()
+    block: |-
+      {% for key, val in dhclient_supersede.items() | rejectattr(1, '==', []) -%}
+      {% if key == "domain-name-servers" -%}
+      supersede {{ key }} {{ val | join(',') }};
+      {% else -%}
+      supersede {{ key }} "{{ val | join('","') }}";
+      {% endif -%}
+      {% endfor %}
+    path: "{{ dhclientconffile }}"
+    create: true
+    state: present
+    insertbefore: BOF
+    backup: "{{ leave_etc_backup_files }}"
+    marker: "# Ansible entries {mark}"
+    mode: "0644"
+  notify: Preinstall | propagate resolvconf to k8s components
+
+- name: Configure dhclient hooks for resolv.conf (non-RH)
+  template:
+    src: dhclient_dnsupdate.sh.j2
+    dest: "{{ dhclienthookfile }}"
+    owner: root
+    mode: "0755"
+  notify: Preinstall | propagate resolvconf to k8s components
+  when: ansible_os_family not in [ "RedHat", "Suse" ]
+
+- name: Configure dhclient hooks for resolv.conf (RH-only)
+  template:
+    src: dhclient_dnsupdate_rh.sh.j2
+    dest: "{{ dhclienthookfile }}"
+    owner: root
+    mode: "0755"
+  notify: Preinstall | propagate resolvconf to k8s components
+  when: ansible_os_family == "RedHat"

internal/resources/kubespray/project/roles/kubernetes/preinstall/tasks/0110-dhclient-hooks-undo.yml

@@ -0,0 +1,18 @@
+---
+
+# These tasks will undo changes done by kubespray in the past if needed (e.g. when upgrading from kubespray 2.0.x
+# or when changing resolvconf_mode)
+
+- name: Remove kubespray specific config from dhclient config
+  blockinfile:
+    path: "{{ dhclientconffile }}"
+    state: absent
+    backup: "{{ leave_etc_backup_files }}"
+    marker: "# Ansible entries {mark}"
+  notify: Preinstall | propagate resolvconf to k8s components
+
+- name: Remove kubespray specific dhclient hook
+  file:
+    path: "{{ dhclienthookfile }}"
+    state: absent
+  notify: Preinstall | propagate resolvconf to k8s components

internal/resources/kubespray/project/roles/kubernetes/preinstall/tasks/main.yml

@@ -0,0 +1,131 @@
+---
+# Disable swap
+- name: Disable swap
+  import_tasks: 0010-swapoff.yml
+  when:
+    - not dns_late
+    - kubelet_fail_swap_on
+
+- name: Set facts
+  import_tasks: 0020-set_facts.yml
+  tags:
+    - resolvconf
+    - facts
+
+- name: Check settings
+  import_tasks: 0040-verify-settings.yml
+  when:
+    - not dns_late
+  tags:
+    - asserts
+
+- name: Create directories
+  import_tasks: 0050-create_directories.yml
+  when:
+    - not dns_late
+
+- name: Apply resolvconf settings
+  import_tasks: 0060-resolvconf.yml
+  when:
+    - dns_mode != 'none'
+    - resolvconf_mode == 'host_resolvconf'
+    - systemd_resolved_enabled.rc != 0
+    - networkmanager_enabled.rc != 0
+  tags:
+    - bootstrap_os
+    - resolvconf
+
+- name: Apply systemd-resolved settings
+  import_tasks: 0061-systemd-resolved.yml
+  when:
+    - dns_mode != 'none'
+    - resolvconf_mode == 'host_resolvconf'
+    - systemd_resolved_enabled.rc == 0
+  tags:
+    - bootstrap_os
+    - resolvconf
+
+- name: Apply networkmanager unmanaged devices settings
+  import_tasks: 0062-networkmanager-unmanaged-devices.yml
+  when:
+    - networkmanager_enabled.rc == 0
+  tags:
+    - bootstrap_os
+
+- name: Apply networkmanager DNS settings
+  import_tasks: 0063-networkmanager-dns.yml
+  when:
+    - dns_mode != 'none'
+    - resolvconf_mode == 'host_resolvconf'
+    - networkmanager_enabled.rc == 0
+  tags:
+    - bootstrap_os
+    - resolvconf
+
+- name: Apply system configurations
+  import_tasks: 0080-system-configurations.yml
+  when:
+    - not dns_late
+  tags:
+    - bootstrap_os
+
+- name: Configure NTP
+  import_tasks: 0081-ntp-configurations.yml
+  when:
+    - not dns_late
+    - ntp_enabled
+  tags:
+    - bootstrap_os
+
+- name: Configure /etc/hosts
+  import_tasks: 0090-etchosts.yml
+  tags:
+    - bootstrap_os
+    - etchosts
+
+- name: Configure dhclient
+  import_tasks: 0100-dhclient-hooks.yml
+  when:
+    - dns_mode != 'none'
+    - resolvconf_mode == 'host_resolvconf'
+    - dhclientconffile is defined
+    - not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"]
+  tags:
+    - bootstrap_os
+    - resolvconf
+
+- name: Configure dhclient dhclient hooks
+  import_tasks: 0110-dhclient-hooks-undo.yml
+  when:
+    - dns_mode != 'none'
+    - resolvconf_mode != 'host_resolvconf'
+    - dhclientconffile is defined
+    - not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"]
+  tags:
+    - bootstrap_os
+    - resolvconf
+
+# We need to make sure the network is restarted early enough so that docker can later pick up the correct system
+# nameservers and search domains
+- name: Flush handlers
+  meta: flush_handlers
+
+- name: Check if we are running inside a Azure VM
+  stat:
+    path: /var/lib/waagent/
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: azure_check
+  when:
+    - not dns_late
+  tags:
+    - bootstrap_os
+
+- name: Run calico checks
+  include_role:
+    name: network_plugin/calico
+    tasks_from: check
+  when:
+    - kube_network_plugin == 'calico'
+    - not ignore_assert_errors