kube-forge v2: embede python with ansible (kubespray) and cobra as cli building tool

internal/resources/kubespray/project/roles/kubernetes/kubeadm/defaults/main.yml

@@ -0,0 +1,8 @@
+---
+# discovery_timeout modifies the discovery timeout
+# This value must be smaller than kubeadm_join_timeout
+discovery_timeout: 60s
+kubeadm_join_timeout: 120s
+
+# Enable kubeadm file discovery if anonymous access has been removed
+kubeadm_use_file_discovery: "{{ remove_anonymous_access }}"

internal/resources/kubespray/project/roles/kubernetes/kubeadm/handlers/main.yml

@@ -0,0 +1,11 @@
+---
+- name: Kubeadm | reload systemd
+  systemd_service:
+    daemon_reload: true
+  listen: Kubeadm | restart kubelet
+
+- name: Kubeadm | reload kubelet
+  service:
+    name: kubelet
+    state: restarted
+  listen: Kubeadm | restart kubelet

internal/resources/kubespray/project/roles/kubernetes/kubeadm/meta/main.yml

@@ -0,0 +1,3 @@
+---
+dependencies:
+  - role: kubernetes/kubeadm_common

internal/resources/kubespray/project/roles/kubernetes/kubeadm/tasks/kubeadm_etcd_node.yml

@@ -0,0 +1,63 @@
+---
+- name: Parse certificate key if not set
+  set_fact:
+    kubeadm_certificate_key: "{{ hostvars[groups['kube_control_plane'][0]]['kubeadm_certificate_key'] }}"
+  when: kubeadm_certificate_key is undefined
+
+- name: Create kubeadm cert controlplane config
+  template:
+    src: "kubeadm-client.conf.j2"
+    dest: "{{ kube_config_dir }}/kubeadm-cert-controlplane.conf"
+    mode: "0640"
+    validate: "{{ kubeadm_config_validate_enabled | ternary(bin_dir + '/kubeadm config validate --config %s', omit) }}"
+  vars:
+    kubeadm_cert_controlplane: true
+
+- name: Pull control plane certs down
+  shell: >-
+    {{ bin_dir }}/kubeadm join phase
+    control-plane-prepare download-certs
+    --config {{ kube_config_dir }}/kubeadm-cert-controlplane.conf
+    &&
+    {{ bin_dir }}/kubeadm join phase
+    control-plane-prepare certs
+    --config {{ kube_config_dir }}/kubeadm-cert-controlplane.conf
+  args:
+    creates: "{{ kube_cert_dir }}/apiserver-etcd-client.key"
+
+- name: Delete unneeded certificates
+  file:
+    path: "{{ item }}"
+    state: absent
+  with_items:
+    - "{{ kube_cert_dir }}/apiserver.crt"
+    - "{{ kube_cert_dir }}/apiserver.key"
+    - "{{ kube_cert_dir }}/ca.key"
+    - "{{ kube_cert_dir }}/etcd/ca.key"
+    - "{{ kube_cert_dir }}/etcd/healthcheck-client.crt"
+    - "{{ kube_cert_dir }}/etcd/healthcheck-client.key"
+    - "{{ kube_cert_dir }}/etcd/peer.crt"
+    - "{{ kube_cert_dir }}/etcd/peer.key"
+    - "{{ kube_cert_dir }}/etcd/server.crt"
+    - "{{ kube_cert_dir }}/etcd/server.key"
+    - "{{ kube_cert_dir }}/front-proxy-ca.crt"
+    - "{{ kube_cert_dir }}/front-proxy-ca.key"
+    - "{{ kube_cert_dir }}/front-proxy-client.crt"
+    - "{{ kube_cert_dir }}/front-proxy-client.key"
+    - "{{ kube_cert_dir }}/sa.key"
+    - "{{ kube_cert_dir }}/sa.pub"
+
+- name: Calculate etcd cert serial
+  command: "openssl x509 -in {{ kube_cert_dir }}/apiserver-etcd-client.crt -noout -serial"
+  register: "etcd_client_cert_serial_result"
+  changed_when: false
+  when:
+    - group_names | intersect(['k8s_cluster', 'calico_rr']) | length > 0
+  tags:
+    - network
+
+- name: Set etcd_client_cert_serial
+  set_fact:
+    etcd_client_cert_serial: "{{ etcd_client_cert_serial_result.stdout.split('=')[1] }}"
+  tags:
+    - network

internal/resources/kubespray/project/roles/kubernetes/kubeadm/tasks/main.yml

@@ -0,0 +1,205 @@
+---
+- name: Set kubeadm_discovery_address
+  set_fact:
+    # noqa: jinja[spacing]
+    kubeadm_discovery_address: >-
+      {%- if "127.0.0.1" in kube_apiserver_endpoint or "localhost" in kube_apiserver_endpoint -%}
+      {{ first_kube_control_plane_address | ansible.utils.ipwrap }}:{{ kube_apiserver_port }}
+      {%- else -%}
+      {{ kube_apiserver_endpoint | replace("https://", "") }}
+      {%- endif %}
+  tags:
+    - facts
+
+- name: Check if kubelet.conf exists
+  stat:
+    path: "{{ kube_config_dir }}/kubelet.conf"
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: kubelet_conf
+
+- name: Check if kubeadm CA cert is accessible
+  stat:
+    path: "{{ kube_cert_dir }}/ca.crt"
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: kubeadm_ca_stat
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+  run_once: true
+
+- name: Fetch CA certificate from control plane node
+  slurp:
+    src: "{{ kube_cert_dir }}/ca.crt"
+  register: ca_cert_content
+  when:
+    - kubeadm_ca_stat.stat is defined
+    - kubeadm_ca_stat.stat.exists
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+  run_once: true
+
+- name: Create kubeadm token for joining nodes with 24h expiration (default)
+  command: "{{ bin_dir }}/kubeadm token create"
+  register: temp_token
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+  when: kubeadm_token is not defined
+  changed_when: false
+
+- name: Set kubeadm_token to generated token
+  set_fact:
+    kubeadm_token: "{{ temp_token.stdout }}"
+  when: kubeadm_token is not defined
+
+- name: Get kubeconfig for join discovery process
+  command: "{{ kubectl }} -n kube-public get cm cluster-info -o jsonpath='{.data.kubeconfig}'"
+  register: kubeconfig_file_discovery
+  run_once: true
+  delegate_to: "{{ groups['kube_control_plane'] | first }}"
+  when: kubeadm_use_file_discovery
+
+- name: Copy discovery kubeconfig
+  copy:
+    dest: "{{ kube_config_dir }}/cluster-info-discovery-kubeconfig.yaml"
+    content: "{{ kubeconfig_file_discovery.stdout }}"
+    owner: "root"
+    mode: "0644"
+  when:
+    - ('kube_control_plane' not in group_names)
+    - not kubelet_conf.stat.exists
+    - kubeadm_use_file_discovery
+
+- name: Create kubeadm client config
+  template:
+    src: "kubeadm-client.conf.j2"
+    dest: "{{ kube_config_dir }}/kubeadm-client.conf"
+    backup: true
+    mode: "0640"
+    validate: "{{ kubeadm_config_validate_enabled | ternary(bin_dir + '/kubeadm config validate --config %s', omit) }}"
+  when: ('kube_control_plane' not in group_names)
+
+- name: Join to cluster if needed
+  environment:
+    PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}:/sbin"
+  when:
+    - ('kube_control_plane' not in group_names)
+    - not kubelet_conf.stat.exists
+  vars:
+    ignored:
+      - DirAvailable--etc-kubernetes-manifests
+      - "{{ kubeadm_ignore_preflight_errors }}"
+  command: >-
+    timeout -k {{ kubeadm_join_timeout }} {{ kubeadm_join_timeout }}
+    {{ bin_dir }}/kubeadm join
+    --config {{ kube_config_dir }}/kubeadm-client.conf
+    --ignore-preflight-errors={{ ignored | flatten | join(',') }}
+    --skip-phases={{ kubeadm_join_phases_skip | join(',') }}
+
+- name: Update server field in kubelet kubeconfig
+  lineinfile:
+    dest: "{{ kube_config_dir }}/kubelet.conf"
+    regexp: 'server:'
+    line: '    server: {{ kube_apiserver_endpoint }}'
+    backup: true
+  when:
+    - kubeadm_config_api_fqdn is not defined
+    - ('kube_control_plane' not in group_names)
+    - kubeadm_discovery_address != kube_apiserver_endpoint | replace("https://", "")
+  notify: Kubeadm | restart kubelet
+
+- name: Update server field in kubelet kubeconfig - external lb
+  lineinfile:
+    dest: "{{ kube_config_dir }}/kubelet.conf"
+    regexp: '^    server: https'
+    line: '    server: {{ kube_apiserver_endpoint }}'
+    backup: true
+  when:
+    - ('kube_control_plane' not in group_names)
+    - loadbalancer_apiserver is defined
+  notify: Kubeadm | restart kubelet
+
+- name: Get current resourceVersion of kube-proxy configmap
+  command: "{{ kubectl }} get configmap kube-proxy -n kube-system -o jsonpath='{.metadata.resourceVersion}'"
+  register: original_configmap_resource_version
+  run_once: true
+  delegate_to: "{{ groups['kube_control_plane'] | first }}"
+  delegate_facts: false
+  when:
+    - kube_proxy_deployed
+  tags:
+    - kube-proxy
+
+# FIXME(mattymo): Need to point to localhost, otherwise control plane nodes will all point
+#                 incorrectly to first control plane node, creating SPoF.
+- name: Update server field in kube-proxy kubeconfig
+  shell: >-
+    set -o pipefail && {{ kubectl }} get configmap kube-proxy -n kube-system -o yaml
+    | sed 's#server:.*#server: https://127.0.0.1:{{ kube_apiserver_port }}#g'
+    | {{ kubectl }} replace -f -
+  args:
+    executable: /bin/bash
+  run_once: true
+  delegate_to: "{{ groups['kube_control_plane'] | first }}"
+  delegate_facts: false
+  when:
+    - kubeadm_config_api_fqdn is not defined
+    - kubeadm_discovery_address != kube_apiserver_endpoint | replace("https://", "")
+    - kube_proxy_deployed
+    - loadbalancer_apiserver_localhost
+  tags:
+    - kube-proxy
+
+- name: Update server field in kube-proxy kubeconfig - external lb
+  shell: >-
+    set -o pipefail && {{ kubectl }} get configmap kube-proxy -n kube-system -o yaml
+    | sed 's#server:.*#server: {{kube_apiserver_endpoint}}#g'
+    | {{ kubectl }} replace -f -
+  args:
+    executable: /bin/bash
+  run_once: true
+  delegate_to: "{{ groups['kube_control_plane'] | first }}"
+  delegate_facts: false
+  when:
+    - kube_proxy_deployed
+    - loadbalancer_apiserver is defined
+  tags:
+    - kube-proxy
+
+- name: Get new resourceVersion of kube-proxy configmap
+  command: "{{ kubectl }} get configmap kube-proxy -n kube-system -o jsonpath='{.metadata.resourceVersion}'"
+  register: new_configmap_resource_version
+  run_once: true
+  delegate_to: "{{ groups['kube_control_plane'] | first }}"
+  delegate_facts: false
+  when:
+    - kube_proxy_deployed
+  tags:
+    - kube-proxy
+
+- name: Set ca.crt file permission
+  file:
+    path: "{{ kube_cert_dir }}/ca.crt"
+    owner: root
+    group: root
+    mode: "0644"
+
+- name: Restart all kube-proxy pods to ensure that they load the new configmap
+  command: "{{ kubectl }} delete pod -n kube-system -l k8s-app=kube-proxy --force --grace-period=0"
+  run_once: true
+  delegate_to: "{{ groups['kube_control_plane'] | first }}"
+  delegate_facts: false
+  when:
+    - kubeadm_config_api_fqdn is not defined or loadbalancer_apiserver is defined
+    - kubeadm_discovery_address != kube_apiserver_endpoint | replace("https://", "") or loadbalancer_apiserver is defined
+    - kube_proxy_deployed
+    - original_configmap_resource_version.stdout != new_configmap_resource_version.stdout
+  tags:
+    - kube-proxy
+
+- name: Extract etcd certs from control plane if using etcd kubeadm mode
+  include_tasks: kubeadm_etcd_node.yml
+  when:
+    - etcd_deployment_type == "kubeadm"
+    - inventory_hostname not in groups['kube_control_plane']
+    - kube_network_plugin in ["calico", "flannel", "cilium"] or cilium_deploy_additionally
+    - kube_network_plugin != "calico" or calico_datastore == "etcd"

internal/resources/kubespray/project/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.j2

@@ -0,0 +1,50 @@
+---
+apiVersion: kubeadm.k8s.io/{{ kubeadm_config_api_version }}
+kind: JoinConfiguration
+discovery:
+{% if kubeadm_use_file_discovery %}
+  file:
+    kubeConfigPath: {{ kube_config_dir }}/cluster-info-discovery-kubeconfig.yaml
+{% else %}
+  bootstrapToken:
+{% if kubeadm_config_api_fqdn is defined %}
+    apiServerEndpoint: "{{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}"
+{% else %}
+    apiServerEndpoint: "{{ kubeadm_discovery_address }}"
+{% endif %}
+    token: {{ kubeadm_token }}
+{% if ca_cert_content is defined %}
+    caCertHashes:
+    - sha256:{{ (ca_cert_content.content | b64decode | community.crypto.x509_certificate_info).public_key_fingerprints.sha256.replace(':', '') }}
+{% else %}
+    unsafeSkipCAVerification: true
+{% endif %}
+{% endif %}
+  tlsBootstrapToken: {{ kubeadm_token }}
+{# TODO: drop the if when we drop support for k8s<1.31 #}
+{% if kubeadm_config_api_version == 'v1beta3' %}
+  timeout: {{ discovery_timeout }}
+{% else %}
+timeouts:
+  discovery: {{ discovery_timeout }}
+{% endif %}
+caCertPath: {{ kube_cert_dir }}/ca.crt
+{% if kubeadm_cert_controlplane is defined and kubeadm_cert_controlplane %}
+controlPlane:
+  localAPIEndpoint:
+    advertiseAddress: "{{ kube_apiserver_address }}"
+    bindPort: {{ kube_apiserver_port }}
+  certificateKey: {{ kubeadm_certificate_key }}
+{% endif %}
+nodeRegistration:
+  name: '{{ kube_override_hostname }}'
+  criSocket: {{ cri_socket }}
+{% if 'calico_rr' in group_names and 'kube_node' not in group_names %}
+  taints:
+  - effect: NoSchedule
+    key: node-role.kubernetes.io/calico-rr
+{% endif %}
+{% if kubeadm_patches | length > 0 %}
+patches:
+  directory: {{ kubeadm_patches_dir }}
+{% endif %}