add ability to log to file, update to v2.28.1 and fix 'first upgrade' issue

internal/resources/kubespray/project/roles/network_plugin/calico/tasks/install.yml

@@ -353,7 +353,7 @@
               {% if not calico_no_global_as_num | default(false) %}"asNumber": {{ global_as_num }},{% endif %}
               "nodeToNodeMeshEnabled": {{ nodeToNodeMeshEnabled | default('true') }} ,
               {% if calico_advertise_cluster_ips | default(false) %}
-              "serviceClusterIPs": >-
+              "serviceClusterIPs":
                 {%- if ipv4_stack and ipv6_stack-%}
                 [{"cidr": "{{ kube_service_addresses }}", "cidr": "{{ kube_service_addresses_ipv6 }}"}],
                 {%- elif ipv6_stack-%}

internal/resources/kubespray/project/roles/network_plugin/cilium/defaults/main.yml

@@ -1,5 +1,8 @@
 ---
 cilium_min_version_required: "1.15"
+
+# remove migrate after 2.29 released
+cilium_remove_old_resources: false
 # Log-level
 cilium_debug: false
 
@@ -173,6 +176,10 @@ cilium_hubble_export_dynamic_config_content:
     excludeFilters: []
     filePath: "/var/run/cilium/hubble/events.log"
 
+# Override the DNS suffix that Hubble-Relay uses to resolve its peer service.
+# It defaults to the inventory's `dns_domain`.
+cilium_hubble_peer_service_cluster_domain: "{{ dns_domain }}"
+
 ### Capacity of Hubble events buffer. The provided value must be one less than an integer power of two and no larger than 65535
 ### (ie: 1, 3, ..., 2047, 4095, ..., 65535) (default 4095)
 # cilium_hubble_event_buffer_capacity: 4095

internal/resources/kubespray/project/roles/network_plugin/cilium/tasks/apply.yml

@@ -1,6 +1,17 @@
 ---
+- name: Check if Cilium Helm release exists (via cilium version)
+  command: "{{ bin_dir }}/cilium version"
+  register: cilium_release_info
+  when: inventory_hostname == groups['kube_control_plane'][0]
+  failed_when: false
+  changed_when: false
+
+- name: Set action to install or upgrade
+  set_fact:
+    cilium_action: "{{ 'install' if ('release: not found' in cilium_release_info.stderr | default('') or 'release: not found' in cilium_release_info.stdout | default('')) else 'upgrade' }}"
+
 - name: Cilium | Install
-  command: "{{ bin_dir }}/cilium install --version {{ cilium_version }} -f {{ kube_config_dir }}/cilium-values.yaml"
+  command: "{{ bin_dir }}/cilium {{ cilium_action }} --version {{ cilium_version }} -f {{ kube_config_dir }}/cilium-values.yaml"
   when: inventory_hostname == groups['kube_control_plane'][0]
 
 - name: Cilium | Wait for pods to run

internal/resources/kubespray/project/roles/network_plugin/cilium/tasks/main.yml

@@ -5,5 +5,10 @@
 - name: Cilium install
   include_tasks: install.yml
 
+# Remove after 2.29 released
+- name: Cilium remove old resources
+  when: cilium_remove_old_resources
+  include_tasks: remove_old_resources.yml
+
 - name: Cilium apply
   include_tasks: apply.yml

internal/resources/kubespray/project/roles/network_plugin/cilium/tasks/remove_old_resources.yml

@@ -0,0 +1,45 @@
+---
+# Remove after 2.29 released
+- name: Cilium | Delete Old Resource
+  command: |
+    {{ kubectl }} delete {{ item.kind | lower }} {{ item.name }} \
+    {{ '-n kube-system' if item.kind not in ['ClusterRole', 'ClusterRoleBinding'] else '' }} \
+  loop:
+    - { kind: ServiceAccount, name: cilium }
+    - { kind: ServiceAccount, name: cilium-operator }
+    - { kind: ServiceAccount, name: hubble-generate-certs }
+    - { kind: ServiceAccount, name: hubble-relay }
+    - { kind: ServiceAccount, name: hubble-ui }
+    - { kind: Service, name: hubble-metrics }
+    - { kind: Service, name: hubble-relay-metrics }
+    - { kind: Service, name: hubble-relay }
+    - { kind: Service, name: hubble-ui }
+    - { kind: Service, name: hubble-peer }
+    - { kind: Deployment, name: cilium-operator }
+    - { kind: Deployment, name: hubble-relay }
+    - { kind: Deployment, name: hubble-ui }
+    - { kind: DaemonSet, name: cilium }
+    - { kind: CronJob, name: hubble-generate-certs }
+    - { kind: Job, name: hubble-generate-certs }
+    - { kind: ConfigMap, name: cilium-config }
+    - { kind: ConfigMap, name: ip-masq-agent }
+    - { kind: ConfigMap, name: hubble-relay-config }
+    - { kind: ConfigMap, name: hubble-ui-nginx }
+    - { kind: ClusterRole, name: cilium }
+    - { kind: ClusterRole, name: cilium-operator }
+    - { kind: ClusterRole, name: hubble-generate-certs }
+    - { kind: ClusterRole, name: hubble-relay }
+    - { kind: ClusterRole, name: hubble-ui }
+    - { kind: ClusterRoleBinding, name: cilium }
+    - { kind: ClusterRoleBinding, name: cilium-operator }
+    - { kind: ClusterRoleBinding, name: hubble-generate-certs }
+    - { kind: ClusterRoleBinding, name: hubble-relay }
+    - { kind: ClusterRoleBinding, name: hubble-ui }
+    - { kind: Secret, name: hubble-ca-secret }
+    - { kind: Secret, name: hubble-relay-client-certs }
+    - { kind: Secret, name: hubble-server-certs }
+  register: patch_result
+  when: inventory_hostname == groups['kube_control_plane'][0]
+  failed_when:
+    - patch_result.rc != 0
+    - "'not found' not in patch_result.stderr"

internal/resources/kubespray/project/roles/network_plugin/cilium/templates/values.yaml.j2

@@ -1,6 +1,7 @@
+#jinja2: trim_blocks: True, lstrip_blocks: True
 MTU: {{ cilium_mtu }}
 debug:
-  enabled: {{ cilium_debug }}
+  enabled: {{ cilium_debug | to_json }}
 
 image:
   repository: {{ cilium_image_repo }}
@@ -10,12 +11,15 @@ k8sServiceHost: "auto"
 k8sServicePort: "auto"
 
 ipv4:
-  enabled: {{ cilium_enable_ipv4 }}
+  enabled: {{ cilium_enable_ipv4 | to_json }}
 ipv6:
-  enabled: {{ cilium_enable_ipv6 }}
+  enabled: {{ cilium_enable_ipv6 | to_json }}
 
 l2announcements:
-  enabled: {{ cilium_l2announcements }}
+  enabled: {{ cilium_l2announcements | to_json }}
+
+bgpControlPlane:
+  enabled: {{ cilium_enable_bgp_control_plane | to_json }}
 
 healthPort: {{ cilium_agent_health_port }}
 
@@ -26,11 +30,11 @@ tunnelProtocol: {{ cilium_tunnel_mode }}
 loadbalancer:
   mode: {{ cilium_loadbalancer_mode }}
 
-kubeProxyReplacement: {{ cilium_kube_proxy_replacement }}
+kubeProxyReplacement: {{ cilium_kube_proxy_replacement | to_json }}
 
 {% if cilium_dns_proxy_enable_transparent_mode is defined %}
 dnsProxy:
-  enableTransparentMode: {{ cilium_dns_proxy_enable_transparent_mode }}
+  enableTransparentMode: {{ cilium_dns_proxy_enable_transparent_mode | to_json }}
 {% endif %}
 
 extraVolumes:
@@ -43,54 +47,55 @@ extraArgs:
   {{ cilium_agent_extra_args | to_nice_yaml(indent=2) | indent(2) }}
 
 bpf:
-  masquerade: {{ cilium_enable_bpf_masquerade }}
-  hostLegacyRouting: {{ cilium_enable_host_legacy_routing }}
+  masquerade: {{ cilium_enable_bpf_masquerade | to_json }}
+  hostLegacyRouting: {{ cilium_enable_host_legacy_routing | to_json }}
   monitorAggregation: {{ cilium_monitor_aggregation }}
-  preallocateMaps: {{ cilium_preallocate_bpf_maps }}
+  preallocateMaps: {{ cilium_preallocate_bpf_maps | to_json }}
   mapDynamicSizeRatio: {{ cilium_bpf_map_dynamic_size_ratio }}
 
 cni:
-  exclusive: {{ cilium_cni_exclusive }}
+  exclusive: {{ cilium_cni_exclusive | to_json }}
   logFile: {{ cilium_cni_log_file }}
 
-autoDirectNodeRoutes: {{ cilium_auto_direct_node_routes }}
+autoDirectNodeRoutes: {{ cilium_auto_direct_node_routes | to_json }}
 
 ipv4NativeRoutingCIDR: {{ cilium_native_routing_cidr }}
 ipv6NativeRoutingCIDR: {{ cilium_native_routing_cidr_ipv6 }}
 
 encryption:
-  enabled: {{ cilium_encryption_enabled }}
-  {% if cilium_encryption_enabled %}
+  enabled: {{ cilium_encryption_enabled | to_json }}
+{% if cilium_encryption_enabled %}
   type: {{ cilium_encryption_type }}
-  {% if cilium_encryption_type == 'wireguard' %}
-  nodeEncryption: {{ cilium_encryption_node_encryption }}
-  {% endif %}
-
-  {% endif %}
+{% if cilium_encryption_type == 'wireguard' %}
+  nodeEncryption: {{ cilium_encryption_node_encryption | to_json }}
+{% endif %}
+{% endif %}
 
 bandwidthManager:
-  enabled: {{ cilium_enable_bandwidth_manager }}
-  bbr: {{ cilium_enable_bandwidth_manager_bbr }}
+  enabled: {{ cilium_enable_bandwidth_manager | to_json }}
+  bbr: {{ cilium_enable_bandwidth_manager_bbr | to_json }}
 
 ipMasqAgent:
-  enabled: {{ cilium_ip_masq_agent_enable }}
-  {% if cilium_ip_masq_agent_enable %}
+  enabled: {{ cilium_ip_masq_agent_enable | to_json }}
+{% if cilium_ip_masq_agent_enable %}
   config:
     nonMasqueradeCIDRs: {{ cilium_non_masquerade_cidrs }}
-    masqLinkLocal: {{ cilium_masq_link_local }}
-    masqLinkLocalIPv6: {{ cilium_masq_link_local_ipv6 }}
+    masqLinkLocal: {{ cilium_masq_link_local | to_json }}
+    masqLinkLocalIPv6: {{ cilium_masq_link_local_ipv6 | to_json }}
   # cilium_ip_masq_resync_interval
-  {% endif %}
+{% endif %}
 
 hubble:
-  enabled: {{ cilium_enable_hubble }}
+  peerService:
+    clusterDomain: {{ cilium_hubble_peer_service_cluster_domain }}
+  enabled: {{ cilium_enable_hubble | to_json }}
   relay:
-    enabled: {{ cilium_enable_hubble }}
+    enabled: {{ cilium_enable_hubble | to_json }}
     image:
       repository: {{ cilium_hubble_relay_image_repo }}
       tag: {{ cilium_hubble_relay_image_tag }}
   ui:
-    enabled: {{ cilium_enable_hubble_ui }}
+    enabled: {{ cilium_enable_hubble_ui | to_json }}
     backend:
       image:
         repository: {{ cilium_hubble_ui_backend_image_repo }}
@@ -100,18 +105,18 @@ hubble:
         repository: {{ cilium_hubble_ui_image_repo }}
         tag: {{ cilium_hubble_ui_image_tag }}
   metrics:
-    enabled: {{ cilium_hubble_metrics }}
+    enabled: {{ cilium_hubble_metrics | to_json }}
   export:
     fileMaxBackups: {{ cilium_hubble_export_file_max_backups }}
     fileMaxSizeMb: {{ cilium_hubble_export_file_max_size_mb }}
     dynamic:
-      enabled: {{ cilium_hubble_export_dynamic_enabled }}
+      enabled: {{ cilium_hubble_export_dynamic_enabled | to_json }}
       config:
         content:
           {{ cilium_hubble_export_dynamic_config_content | to_nice_yaml(indent=10) | indent(10) }}
 
 gatewayAPI:
-  enabled: {{ cilium_gateway_api_enabled }}
+  enabled: {{ cilium_gateway_api_enabled | to_json }}
 
 ipam:
   mode: {{ cilium_ipam_mode }}
@@ -126,7 +131,7 @@ ipam:
 
 cgroup:
   autoMount:
-    enabled: {{ cilium_cgroup_auto_mount }}
+    enabled: {{ cilium_cgroup_auto_mount | to_json }}
   hostRoot: {{ cilium_cgroup_host_root }}
 
 operator:
@@ -147,18 +152,21 @@ cluster:
   id: {{ cilium_cluster_id }}
   name: {{ cilium_cluster_name }}
 
-enableIPv4Masquerade: {{ cilium_enable_ipv4_masquerade }}
-enableIPv6Masquerade: {{ cilium_enable_ipv6_masquerade }}
+enableIPv4Masquerade: {{ cilium_enable_ipv4_masquerade | to_json }}
+enableIPv6Masquerade: {{ cilium_enable_ipv6_masquerade | to_json }}
 
 hostFirewall:
-  enabled: {{ cilium_enable_host_firewall }}
+  enabled: {{ cilium_enable_host_firewall | to_json }}
 
 certgen:
   image:
-    repositry: {{ cilium_hubble_certgen_image_repo }}
+    repository: {{ cilium_hubble_certgen_image_repo }}
     tag: {{ cilium_hubble_certgen_image_tag }}
 
 envoy:
   image:
-    repositry: {{ cilium_hubble_envoy_image_repo }}
+    repository: {{ cilium_hubble_envoy_image_repo }}
     tag: {{ cilium_hubble_envoy_image_tag }}
+
+extraConfig:
+  {{ cilium_config_extra_vars | to_yaml | indent(2) }}