fix download issues when using scale

kubespray/inventory/group_vars/all.yml

@@ -130,4 +130,4 @@ unsafe_show_logs: false
 allow_unsupported_distribution_setup: false
 
 ## Kubelet additional settings
-kubelet_custom_flags: "--root-dir=/var/lib/kubelet"
+kubelet_custom_flags: "--root-dir=/app/k8s/kubelet"

kubespray/inventory/group_vars/all/containerd.yml

@@ -1,8 +1,8 @@
 ---
 # Please see roles/container-engine/containerd/defaults/main.yml for more configuration options
 
-containerd_storage_dir: /var/lib/containerd
-containerd_state_dir:  /run/containerd
+containerd_storage_dir: /app/containerd
+containerd_state_dir:  /app/containerd-state
 
 # containerd_oom_score: 0
 

kubespray/inventory/group_vars/k8s_cluster/k8s-net-calico.yml

@@ -100,7 +100,7 @@ calico_pool_blocksize: 26
 # calico_vxlan_port: 4789
 
 # Enable eBPF mode
-# calico_bpf_enabled: false
+calico_bpf_enabled: false
 
 # If you want to use non default IP_AUTODETECTION_METHOD, IP6_AUTODETECTION_METHOD for calico node set this option to one of:
 # * can-reach=DESTINATION

kubespray/inventory/group_vars/k8s_cluster/k8s-net-cilium.yml

@@ -142,19 +142,19 @@
 ### Enable Hubble without install
 # cilium_enable_hubble: false
 ### Enable Hubble Metrics
-# cilium_enable_hubble_metrics: false
+cilium_enable_hubble_metrics: false
 ### if cilium_enable_hubble_metrics: true
-# cilium_hubble_metrics: {}
-# - dns
-# - drop
-# - tcp
-# - flow
-# - icmp
-# - http
+# cilium_hubble_metrics:
+#   - dns
+#   - drop
+#   - tcp
+#   - flow
+#   - icmp
+#   - http
 ### Enable Hubble install
-# cilium_hubble_install: false
+cilium_hubble_install:  false
 ### Enable auto generate certs if cilium_hubble_install: true
-# cilium_hubble_tls_generate: false
+cilium_hubble_tls_generate: true
 
 # IP address management mode for v1.9+.
 # https://docs.cilium.io/en/v1.9/concepts/networking/ipam/
@@ -224,7 +224,7 @@
 # cilium_enable_ipv6_masquerade: true
 
 # -- Enable native IP masquerade support in eBPF
-# cilium_enable_bpf_masquerade: false
+cilium_enable_bpf_masquerade: false
 
 # -- Configure whether direct routing mode should route traffic via
 # host stack (true) or directly and more efficiently out of BPF (false) if

kubespray/inventory/hosts

@@ -1,17 +1,31 @@
 
-t1-disk-lt-k8s-test-master-01 ansible_host=10.100.20.49 ip=10.100.20.49  
-t1-disk-lt-k8s-test-slave-01 ansible_host=10.100.20.48 ip=10.100.20.48  
-t1-disk-lt-k8s-test-slave-02 ansible_host=10.100.20.47 ip=10.100.20.47  
+t1-disk-lt-k8s-master-01 ansible_host=10.100.20.43 ip=10.100.20.43  
+t1-disk-lt-k8s-master-02 ansible_host=10.100.20.38 ip=10.100.20.38  
+t1-disk-lt-k8s-master-03 ansible_host=10.100.20.42 ip=10.100.20.42  
+t1-disk-lt-k8s-slave-01 ansible_host=10.100.20.40 ip=10.100.20.40  
+t1-disk-lt-k8s-slave-02 ansible_host=10.100.20.41 ip=10.100.20.41  
+t1-disk-lt-k8s-slave-03 ansible_host=10.100.20.39 ip=10.100.20.39  
+t1-disk-lt-k8s-slave-04 ansible_host=10.100.20.45 ip=10.100.20.45  
+t1-disk-lt-k8s-slave-05 ansible_host=10.100.20.44 ip=10.100.20.44  
+t1-disk-lt-k8s-slave-06 ansible_host=10.100.20.50 ip=10.100.20.50  
 
 [kube_control_plane]
-t1-disk-lt-k8s-test-master-01
+t1-disk-lt-k8s-master-01
+t1-disk-lt-k8s-master-02
+t1-disk-lt-k8s-master-03
 
 [etcd]
-t1-disk-lt-k8s-test-master-01
+t1-disk-lt-k8s-master-01
+t1-disk-lt-k8s-master-02
+t1-disk-lt-k8s-master-03
 
 [kube_node]
-t1-disk-lt-k8s-test-slave-01
-t1-disk-lt-k8s-test-slave-02
+t1-disk-lt-k8s-slave-01
+t1-disk-lt-k8s-slave-02
+t1-disk-lt-k8s-slave-03
+t1-disk-lt-k8s-slave-04
+t1-disk-lt-k8s-slave-05
+t1-disk-lt-k8s-slave-06
 
 [k8s_cluster:children]
 kube_control_plane

kubespray/project/roles/download/tasks/main.yml

@@ -2,7 +2,7 @@
 - name: Download | Prepare working directories and variables
   import_tasks: prep_download.yml
   when:
-    - not skip_downloads
+    - not skip_downloads | default(false)
   tags:
     - download
     - upload
@@ -10,7 +10,7 @@
 - name: Download | Get kubeadm binary and list of required images
   include_tasks: prep_kubeadm_images.yml
   when:
-    - not skip_downloads
+    - not skip_downloads | default(false)
     - inventory_hostname in groups['kube_control_plane']
   tags:
     - download
@@ -22,44 +22,8 @@
   vars:
     download: "{{ download_defaults | combine(item.value) }}"
     include_file: "download_{% if download.container %}container{% else %}file{% endif %}.yml"
-    kubeadm_images: "{{ skip_kubeadm_images | ternary({}, _kubeadm_images) }}"
-    # The trick (converting list of tuples to list of dicts) below come from
-    # https://docs.ansible.com/ansible/latest/collections/community/general/dict_filter.html#examples
-    _kubeadm_images: "{{ dict(names | map('regex_replace', '^(.*)', 'kubeadm_\\1') |
-      zip( repos | zip(_tags, _groups) |
-      map('zip', keys) | map('map', 'reverse') | map('community.general.dict') |
-      map('combine', defaults))) |
-      dict2items | rejectattr('key', 'in', excluded) | items2dict }}"
-    keys:
-      - repo
-      - tag
-      - groups
-    images: "{{ kubeadm_images_raw.stdout_lines | map('split', ':') }}"
-    _tags: "{{ images | map(attribute=1) }}"
-    repos: "{{ images | map(attribute=0) }}"
-    names: "{{ repos | map('split', '/') | map(attribute=-1) }}"
-    _groups: "{{ names | map('extract', images_groups) }}"
-    defaults:
-      enabled: true
-      container: true
-    excluded:
-      - kubeadm_coredns
-      - kubeadm_pause
-    images_groups:
-      coredns: []
-      pause: []
-      kube-proxy:
-        - k8s_cluster
-      etcd:
-        - etcd
-      kube-scheduler:
-        - kube_control_plane
-      kube-controller-manager:
-        - kube_control_plane
-      kube-apiserver:
-        - kube_control_plane
   when:
-    - not skip_downloads
+    - not skip_downloads | default(false)
     - download.enabled
     - item.value.enabled
     - (not (item.value.container | default(false))) or (item.value.container and download_container)

kubespray/project/roles/download/tasks/prep_kubeadm_images.yml

@@ -20,7 +20,7 @@
     dest: "{{ kube_config_dir }}/kubeadm-images.yaml"
     mode: 0644
   when:
-    - not skip_kubeadm_images
+    - not skip_kubeadm_images | default(false)
 
 - name: Prep_kubeadm_images | Copy kubeadm binary from download dir to system path
   copy:
@@ -36,9 +36,36 @@
     state: file
 
 - name: Prep_kubeadm_images | Generate list of required images
-  command: "{{ bin_dir }}/kubeadm config images list --config={{ kube_config_dir }}/kubeadm-images.yaml"
+  shell: "set -o pipefail && {{ bin_dir }}/kubeadm config images list --config={{ kube_config_dir }}/kubeadm-images.yaml | grep -Ev 'coredns|pause'"
+  args:
+    executable: /bin/bash
   register: kubeadm_images_raw
   run_once: true
   changed_when: false
   when:
-    - not skip_kubeadm_images
+    - not skip_kubeadm_images | default(false)
+
+- name: Prep_kubeadm_images | Parse list of images
+  vars:
+    kubeadm_images_list: "{{ kubeadm_images_raw.stdout_lines }}"
+  set_fact:
+    kubeadm_image:
+      key: "kubeadm_{{ (item | regex_replace('^(?:.*\\/)*', '')).split(':')[0] }}"
+      value:
+        enabled: true
+        container: true
+        repo: "{{ item | regex_replace('^(.*):.*$', '\\1') }}"
+        tag: "{{ item | regex_replace('^.*:(.*)$', '\\1') }}"
+        groups: k8s_cluster
+  loop: "{{ kubeadm_images_list | flatten(levels=1) }}"
+  register: kubeadm_images_cooked
+  run_once: true
+  when:
+    - not skip_kubeadm_images | default(false)
+
+- name: Prep_kubeadm_images | Convert list of images to dict for later use
+  set_fact:
+    kubeadm_images: "{{ kubeadm_images_cooked.results | map(attribute='ansible_facts.kubeadm_image') | list | items2dict }}"
+  run_once: true
+  when:
+    - not skip_kubeadm_images | default(false)