kube-forge v2: embede python with ansible (kubespray) and cobra as cli building tool

internal/resources/kubespray/project/roles/kubernetes/node/tasks/facts.yml

@@ -0,0 +1,62 @@
+---
+- name: Gather cgroups facts for docker
+  when: container_manager == 'docker'
+  block:
+  - name: Look up docker cgroup driver
+    shell: "set -o pipefail && docker info | grep 'Cgroup Driver' | awk -F': ' '{ print $2; }'"
+    args:
+      executable: /bin/bash
+    register: docker_cgroup_driver_result
+    changed_when: false
+    check_mode: false
+
+  - name: Set kubelet_cgroup_driver_detected fact for docker
+    set_fact:
+      kubelet_cgroup_driver_detected: "{{ docker_cgroup_driver_result.stdout }}"
+
+- name: Gather cgroups facts for crio
+  when: container_manager == 'crio'
+  block:
+  - name: Look up crio cgroup driver
+    shell: "set -o pipefail && {{ bin_dir }}/{{ crio_status_command }} info | grep 'cgroup driver' | awk -F': ' '{ print $2; }'"
+    args:
+      executable: /bin/bash
+    register: crio_cgroup_driver_result
+    changed_when: false
+
+  - name: Set kubelet_cgroup_driver_detected fact for crio
+    set_fact:
+      kubelet_cgroup_driver_detected: "{{ crio_cgroup_driver_result.stdout }}"
+
+- name: Set kubelet_cgroup_driver_detected fact for containerd
+  when: container_manager == 'containerd'
+  set_fact:
+    kubelet_cgroup_driver_detected: >-
+      {%- if containerd_use_systemd_cgroup -%}systemd{%- else -%}cgroupfs{%- endif -%}
+
+- name: Set kubelet_cgroup_driver
+  set_fact:
+    kubelet_cgroup_driver: "{{ kubelet_cgroup_driver_detected }}"
+  when: kubelet_cgroup_driver is undefined
+
+- name: Set kubelet_cgroups options when cgroupfs is used
+  set_fact:
+    kubelet_runtime_cgroups: "{{ kubelet_runtime_cgroups_cgroupfs }}"
+    kubelet_kubelet_cgroups: "{{ kubelet_kubelet_cgroups_cgroupfs }}"
+  when: kubelet_cgroup_driver == 'cgroupfs'
+
+- name: Set kubelet_config_extra_args options when cgroupfs is used
+  set_fact:
+    kubelet_config_extra_args: "{{ kubelet_config_extra_args | combine(kubelet_config_extra_args_cgroupfs) }}"
+  when: kubelet_cgroup_driver == 'cgroupfs'
+
+- name: Os specific vars
+  include_vars: "{{ item }}"
+  with_first_found:
+  - files:
+    - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower | replace('/', '_') }}.yml"
+    - "{{ ansible_distribution | lower }}-{{ ansible_distribution_release }}.yml"
+    - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower | replace('/', '_') }}.yml"
+    - "{{ ansible_distribution | lower }}.yml"
+    - "{{ ansible_os_family | lower }}.yml"
+    skip: true

internal/resources/kubespray/project/roles/kubernetes/node/tasks/install.yml

@@ -0,0 +1,22 @@
+---
+- name: Install | Copy kubeadm binary from download dir
+  copy:
+    src: "{{ downloads.kubeadm.dest }}"
+    dest: "{{ bin_dir }}/kubeadm"
+    mode: "0755"
+    remote_src: true
+  tags:
+    - kubeadm
+  when:
+    - not ('kube_control_plane' in group_names)
+
+- name: Install | Copy kubelet binary from download dir
+  copy:
+    src: "{{ downloads.kubelet.dest }}"
+    dest: "{{ bin_dir }}/kubelet"
+    mode: "0755"
+    remote_src: true
+  tags:
+    - kubelet
+    - upgrade
+  notify: Node | restart kubelet

internal/resources/kubespray/project/roles/kubernetes/node/tasks/kubelet.yml

@@ -0,0 +1,55 @@
+---
+- name: Set kubelet api version to v1beta1
+  set_fact:
+    kubeletConfig_api_version: v1beta1
+  tags:
+    - kubelet
+    - kubeadm
+
+- name: Write kubelet environment config file (kubeadm)
+  template:
+    src: "kubelet.env.{{ kubeletConfig_api_version }}.j2"
+    dest: "{{ kube_config_dir }}/kubelet.env"
+    setype: "{{ (preinstall_selinux_state != 'disabled') | ternary('etc_t', omit) }}"
+    backup: true
+    mode: "0600"
+  notify: Node | restart kubelet
+  tags:
+    - kubelet
+    - kubeadm
+
+- name: Write kubelet config file
+  template:
+    src: "kubelet-config.{{ kubeletConfig_api_version }}.yaml.j2"
+    dest: "{{ kube_config_dir }}/kubelet-config.yaml"
+    mode: "0600"
+  notify: Kubelet | restart kubelet
+  tags:
+    - kubelet
+    - kubeadm
+
+- name: Write kubelet systemd init file
+  template:
+    src: "kubelet.service.j2"
+    dest: "/etc/systemd/system/kubelet.service"
+    backup: true
+    mode: "0600"
+    validate: "sh -c '[ -f /usr/bin/systemd/system/factory-reset.target ] || exit 0 && systemd-analyze verify %s:kubelet.service'"
+    # FIXME: check that systemd version >= 250 (factory-reset.target was introduced in that release)
+    # Remove once we drop support for systemd < 250
+  notify: Node | restart kubelet
+  tags:
+    - kubelet
+    - kubeadm
+
+- name: Flush_handlers and reload-systemd
+  meta: flush_handlers
+
+- name: Enable kubelet
+  service:
+    name: kubelet
+    enabled: true
+    state: started
+  tags:
+    - kubelet
+  notify: Kubelet | restart kubelet

internal/resources/kubespray/project/roles/kubernetes/node/tasks/loadbalancer/haproxy.yml

@@ -0,0 +1,34 @@
+---
+- name: Haproxy | Cleanup potentially deployed nginx-proxy
+  file:
+    path: "{{ kube_manifest_dir }}/nginx-proxy.yml"
+    state: absent
+
+- name: Haproxy | Make haproxy directory
+  file:
+    path: "{{ haproxy_config_dir }}"
+    state: directory
+    mode: "0755"
+    owner: root
+
+- name: Haproxy | Write haproxy configuration
+  template:
+    src: "loadbalancer/haproxy.cfg.j2"
+    dest: "{{ haproxy_config_dir }}/haproxy.cfg"
+    owner: root
+    mode: "0755"
+    backup: true
+
+- name: Haproxy | Get checksum from config
+  stat:
+    path: "{{ haproxy_config_dir }}/haproxy.cfg"
+    get_attributes: false
+    get_checksum: true
+    get_mime: false
+  register: haproxy_stat
+
+- name: Haproxy | Write static pod
+  template:
+    src: manifests/haproxy.manifest.j2
+    dest: "{{ kube_manifest_dir }}/haproxy.yml"
+    mode: "0640"

internal/resources/kubespray/project/roles/kubernetes/node/tasks/loadbalancer/kube-vip.yml

@@ -0,0 +1,39 @@
+---
+- name: Kube-vip  | Check cluster settings for kube-vip
+  fail:
+    msg: "kube-vip require kube_proxy_strict_arp = true, see https://github.com/kube-vip/kube-vip/blob/main/docs/kubernetes/arp/index.md"
+  when:
+    - kube_proxy_mode == 'ipvs' and not kube_proxy_strict_arp
+    - kube_vip_arp_enabled
+
+- name: Kube-vip | Check if super-admin.conf exists
+  stat:
+    path: "{{ kube_config_dir }}/super-admin.conf"
+  failed_when: false
+  changed_when: false
+  register: stat_kube_vip_super_admin
+
+- name: Kube-vip | Check if kubeadm has already run
+  stat:
+    path: "/var/lib/kubelet/config.yaml"
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: kubeadm_already_run
+
+- name: Kube-vip | Set admin.conf
+  set_fact:
+    kube_vip_admin_conf: admin.conf
+
+- name: Kube-vip | Set admin.conf for first Control Plane
+  set_fact:
+    kube_vip_admin_conf: super-admin.conf
+  when:
+    - inventory_hostname == groups['kube_control_plane'] | first
+    - (stat_kube_vip_super_admin.stat.exists and stat_kube_vip_super_admin.stat.isreg) or (not kubeadm_already_run.stat.exists )
+
+- name: Kube-vip | Write static pod
+  template:
+    src: manifests/kube-vip.manifest.j2
+    dest: "{{ kube_manifest_dir }}/kube-vip.yml"
+    mode: "0640"

internal/resources/kubespray/project/roles/kubernetes/node/tasks/loadbalancer/nginx-proxy.yml

@@ -0,0 +1,34 @@
+---
+- name: Haproxy | Cleanup potentially deployed haproxy
+  file:
+    path: "{{ kube_manifest_dir }}/haproxy.yml"
+    state: absent
+
+- name: Nginx-proxy | Make nginx directory
+  file:
+    path: "{{ nginx_config_dir }}"
+    state: directory
+    mode: "0700"
+    owner: root
+
+- name: Nginx-proxy | Write nginx-proxy configuration
+  template:
+    src: "loadbalancer/nginx.conf.j2"
+    dest: "{{ nginx_config_dir }}/nginx.conf"
+    owner: root
+    mode: "0755"
+    backup: true
+
+- name: Nginx-proxy | Get checksum from config
+  stat:
+    path: "{{ nginx_config_dir }}/nginx.conf"
+    get_attributes: false
+    get_checksum: true
+    get_mime: false
+  register: nginx_stat
+
+- name: Nginx-proxy | Write static pod
+  template:
+    src: manifests/nginx-proxy.manifest.j2
+    dest: "{{ kube_manifest_dir }}/nginx-proxy.yml"
+    mode: "0640"

internal/resources/kubespray/project/roles/kubernetes/node/tasks/main.yml

@@ -0,0 +1,148 @@
+---
+- name: Fetch facts
+  import_tasks: facts.yml
+  tags:
+    - facts
+    - kubelet
+
+- name: Ensure /var/lib/cni exists
+  file:
+    path: /var/lib/cni
+    state: directory
+    mode: "0755"
+
+- name: Install kubelet binary
+  import_tasks: install.yml
+  tags:
+    - kubelet
+
+- name: Install kube-vip
+  import_tasks: loadbalancer/kube-vip.yml
+  when:
+    - ('kube_control_plane' in group_names)
+    - kube_vip_enabled
+  tags:
+    - kube-vip
+
+- name: Install nginx-proxy
+  import_tasks: loadbalancer/nginx-proxy.yml
+  when:
+    - ('kube_control_plane' not in group_names) or (kube_apiserver_bind_address != '::')
+    - loadbalancer_apiserver_localhost
+    - loadbalancer_apiserver_type == 'nginx'
+  tags:
+    - nginx
+
+- name: Install haproxy
+  import_tasks: loadbalancer/haproxy.yml
+  when:
+    - ('kube_control_plane' not in group_names) or (kube_apiserver_bind_address != '::')
+    - loadbalancer_apiserver_localhost
+    - loadbalancer_apiserver_type == 'haproxy'
+  tags:
+    - haproxy
+
+- name: Ensure nodePort range is reserved
+  ansible.posix.sysctl:
+    name: net.ipv4.ip_local_reserved_ports
+    value: "{{ kube_apiserver_node_port_range }}"
+    sysctl_set: true
+    sysctl_file: "{{ sysctl_file_path }}"
+    state: present
+    reload: true
+  when: kube_apiserver_node_port_range is defined
+  tags:
+    - kube-proxy
+
+- name: Verify if br_netfilter module exists
+  command: "modinfo br_netfilter"
+  environment:
+    PATH: "{{ ansible_env.PATH }}:/sbin"  # Make sure we can workaround RH's conservative path management
+  register: modinfo_br_netfilter
+  failed_when: modinfo_br_netfilter.rc not in [0, 1]
+  changed_when: false
+  check_mode: false
+
+# TODO: Remove once upstream issue is fixed
+# https://github.com/ansible-collections/community.general/issues/7717
+- name: Verify br_netfilter module path exists
+  file:
+    path: "{{ item }}"
+    state: directory
+    mode: "0755"
+  loop:
+    - /etc/modules-load.d
+    - /etc/modprobe.d
+
+- name: Enable br_netfilter module
+  community.general.modprobe:
+    name: br_netfilter
+    state: present
+  when: modinfo_br_netfilter.rc == 0
+
+- name: Persist br_netfilter module
+  copy:
+    dest: /etc/modules-load.d/kubespray-br_netfilter.conf
+    content: br_netfilter
+    mode: "0644"
+  when: modinfo_br_netfilter.rc == 0
+
+# kube-proxy needs net.bridge.bridge-nf-call-iptables enabled when found if br_netfilter is not a module
+- name: Check if bridge-nf-call-iptables key exists
+  command: "sysctl net.bridge.bridge-nf-call-iptables"
+  failed_when: false
+  changed_when: false
+  check_mode: false
+  register: sysctl_bridge_nf_call_iptables
+
+- name: Enable bridge-nf-call tables
+  ansible.posix.sysctl:
+    name: "{{ item }}"
+    state: present
+    sysctl_file: "{{ sysctl_file_path }}"
+    value: "1"
+    reload: true
+  when: sysctl_bridge_nf_call_iptables.rc == 0
+  with_items:
+    - net.bridge.bridge-nf-call-iptables
+    - net.bridge.bridge-nf-call-arptables
+    - net.bridge.bridge-nf-call-ip6tables
+
+- name: Modprobe Kernel Module for IPVS
+  community.general.modprobe:
+    name: "{{ item }}"
+    state: present
+    persistent: present
+  loop: "{{ kube_proxy_ipvs_modules }}"
+  when: kube_proxy_mode == 'ipvs'
+  tags:
+    - kube-proxy
+
+- name: Modprobe conntrack module
+  community.general.modprobe:
+    name: "{{ item }}"
+    state: present
+    persistent: present
+  register: modprobe_conntrack_module
+  ignore_errors: true  # noqa ignore-errors
+  loop: "{{ conntrack_modules }}"
+  when:
+    - kube_proxy_mode == 'ipvs'
+    - "(modprobe_conntrack_module|default({'rc': 1})).rc != 0"  # loop until first success
+  tags:
+    - kube-proxy
+
+- name: Modprobe Kernel Module for nftables
+  community.general.modprobe:
+    name: "nf_tables"
+    state: present
+    persistent: present
+  when: kube_proxy_mode == 'nftables'
+  tags:
+    - kube-proxy
+
+- name: Install kubelet
+  import_tasks: kubelet.yml
+  tags:
+    - kubelet
+    - kubeadm