update to latest 2.26.0 kubespray version and add dns zones settings to coredns

2024-12-14 21:56:19 +03:00
parent be9c8a28fd
commit 4294f52703
368 changed files with 37761 additions and 2942 deletions
--- a/kubespray/project/roles/network_plugin/calico/rr/tasks/pre.yml
+++ b/kubespray/project/roles/network_plugin/calico/rr/tasks/pre.yml
@@ -3,7 +3,7 @@
  service:
    name: calico-rr
    state: stopped
-    enabled: no
+    enabled: false
  failed_when: false

 - name: Calico-rr | Delete obsolete files
--- a/kubespray/project/roles/network_plugin/calico/tasks/calico_apiserver_certs.yml
+++ b/kubespray/project/roles/network_plugin/calico/tasks/calico_apiserver_certs.yml
@@ -9,7 +9,7 @@
  template:
    src: "calico-apiserver-ns.yml.j2"
    dest: "{{ kube_config_dir }}/calico-apiserver-ns.yml"
-    mode: 0644
+    mode: "0644"

 - name: Calico | Apply ns manifests
  kube:
@@ -21,21 +21,21 @@
  file:
    path: /etc/calico/certs
    state: directory
-    mode: 0755
+    mode: "0755"
  when: calico_apiserver_secret.rc != 0

 - name: Calico | Copy ssl script for apiserver certs
  template:
    src: make-ssl-calico.sh.j2
    dest: "{{ bin_dir }}/make-ssl-apiserver.sh"
-    mode: 0755
+    mode: "0755"
  when: calico_apiserver_secret.rc != 0

 - name: Calico | Copy ssl config for apiserver certs
  copy:
    src: openssl.conf
    dest: /etc/calico/certs/openssl.conf
-    mode: 0644
+    mode: "0644"
  when: calico_apiserver_secret.rc != 0

 - name: Calico | Generate apiserver certs
--- a/kubespray/project/roles/network_plugin/calico/tasks/check.yml
+++ b/kubespray/project/roles/network_plugin/calico/tasks/check.yml
@@ -4,7 +4,7 @@
    that:
      - ipip is not defined
    msg: "'ipip' configuration variable is deprecated, please configure your inventory with 'calico_ipip_mode' set to 'Always' or 'CrossSubnet' according to your specific needs"
-  run_once: True
+  run_once: true
  delegate_to: "{{ groups['kube_control_plane'][0] }}"

 - name: Stop if legacy encapsulation variables are detected (ipip_mode)
@@ -12,7 +12,7 @@
    that:
      - ipip_mode is not defined
    msg: "'ipip_mode' configuration variable is deprecated, please configure your inventory with 'calico_ipip_mode' set to 'Always' or 'CrossSubnet' according to your specific needs"
-  run_once: True
+  run_once: true
  delegate_to: "{{ groups['kube_control_plane'][0] }}"

 - name: Stop if legacy encapsulation variables are detected (calcio_ipam_autoallocateblocks)
@@ -20,7 +20,7 @@
    that:
      - calcio_ipam_autoallocateblocks is not defined
    msg: "'calcio_ipam_autoallocateblocks' configuration variable is deprecated, it's a typo, please configure your inventory with 'calico_ipam_autoallocateblocks' set to 'true' or 'false' according to your specific needs"
-  run_once: True
+  run_once: true
  delegate_to: "{{ groups['kube_control_plane'][0] }}"


@@ -32,7 +32,7 @@
    msg: "When using cloud_provider azure and network_plugin calico calico_ipip_mode must be 'Never' and calico_vxlan_mode 'Always' or 'CrossSubnet'"
  when:
    - cloud_provider is defined and cloud_provider == 'azure'
-  run_once: True
+  run_once: true
  delegate_to: "{{ groups['kube_control_plane'][0] }}"

 - name: Stop if supported Calico versions
@@ -40,21 +40,21 @@
    that:
      - "calico_version in calico_crds_archive_checksums.keys()"
    msg: "Calico version not supported {{ calico_version }} not in {{ calico_crds_archive_checksums.keys() }}"
-  run_once: True
+  run_once: true
  delegate_to: "{{ groups['kube_control_plane'][0] }}"

 - name: Check if calicoctl.sh exists
  stat:
    path: "{{ bin_dir }}/calicoctl.sh"
  register: calicoctl_sh_exists
-  run_once: True
+  run_once: true
  delegate_to: "{{ groups['kube_control_plane'][0] }}"

 - name: Check if calico ready
  command: "{{ bin_dir }}/calicoctl.sh get ClusterInformation default"
  register: calico_ready
-  run_once: True
-  ignore_errors: True
+  run_once: true
+  ignore_errors: true
  retries: 5
  delay: 10
  until: calico_ready.rc == 0
@@ -62,7 +62,7 @@
  when: calicoctl_sh_exists.stat.exists

 - name: Check that current calico version is enough for upgrade
-  run_once: True
+  run_once: true
  delegate_to: "{{ groups['kube_control_plane'][0] }}"
  when: calicoctl_sh_exists.stat.exists and calico_ready.rc == 0
  block:
@@ -82,15 +82,16 @@
          Minimum version is {{ calico_min_version_required }} supported by the previous kubespray release.
          But current version is {{ calico_version_on_server.stdout }}.

- name: "Check that cluster_id is set if calico_rr enabled"
+- name: "Check that cluster_id is set and a valid IPv4 address if calico_rr enabled"
  assert:
    that:
      - cluster_id is defined
-    msg: "A unique cluster_id is required if using calico_rr"
+      - cluster_id is ansible.utils.ipv4
+    msg: "A unique cluster_id is required if using calico_rr, and it must be a valid IPv4 address"
  when:
    - peer_with_calico_rr
    - inventory_hostname == groups['kube_control_plane'][0]
-  run_once: True
+  run_once: true
  delegate_to: "{{ groups['kube_control_plane'][0] }}"

 - name: "Check that calico_rr nodes are in k8s_cluster group"
@@ -100,7 +101,7 @@
    msg: "calico_rr must be a child group of k8s_cluster group"
  when:
    - '"calico_rr" in group_names'
-  run_once: True
+  run_once: true
  delegate_to: "{{ groups['kube_control_plane'][0] }}"

 - name: "Check vars defined correctly"
@@ -109,7 +110,7 @@
      - "calico_pool_name is defined"
      - "calico_pool_name is match('^[a-zA-Z0-9-_\\\\.]{2,63}$')"
    msg: "calico_pool_name contains invalid characters"
-  run_once: True
+  run_once: true
  delegate_to: "{{ groups['kube_control_plane'][0] }}"

 - name: "Check calico network backend defined correctly"
@@ -117,11 +118,11 @@
    that:
      - "calico_network_backend in ['bird', 'vxlan', 'none']"
    msg: "calico network backend is not 'bird', 'vxlan' or 'none'"
-  run_once: True
+  run_once: true
  delegate_to: "{{ groups['kube_control_plane'][0] }}"

 - name: "Check ipip and vxlan mode defined correctly"
-  run_once: True
+  run_once: true
  delegate_to: "{{ groups['kube_control_plane'][0] }}"
  assert:
    that:
@@ -136,7 +137,7 @@
    msg: "IP in IP and VXLAN mode is mutualy exclusive modes"
  when:
    - "calico_ipip_mode in ['Always', 'CrossSubnet']"
-  run_once: True
+  run_once: true
  delegate_to: "{{ groups['kube_control_plane'][0] }}"

 - name: "Check ipip and vxlan mode if simultaneously enabled"
@@ -146,23 +147,23 @@
    msg: "IP in IP and VXLAN mode is mutualy exclusive modes"
  when:
    - "calico_vxlan_mode in ['Always', 'CrossSubnet']"
-  run_once: True
+  run_once: true
  delegate_to: "{{ groups['kube_control_plane'][0] }}"

 - name: "Get Calico {{ calico_pool_name }} configuration"
  command: "{{ bin_dir }}/calicoctl.sh get ipPool {{ calico_pool_name }} -o json"
-  failed_when: False
-  changed_when: False
-  check_mode: no
+  failed_when: false
+  changed_when: false
+  check_mode: false
  register: calico
-  run_once: True
+  run_once: true
  delegate_to: "{{ groups['kube_control_plane'][0] }}"

 - name: "Set calico_pool_conf"
  set_fact:
    calico_pool_conf: '{{ calico.stdout | from_json }}'
  when: calico.rc == 0 and calico.stdout
-  run_once: True
+  run_once: true
  delegate_to: "{{ groups['kube_control_plane'][0] }}"

 - name: "Check if inventory match current cluster configuration"
@@ -175,7 +176,7 @@
    msg: "Your inventory doesn't match the current cluster configuration"
  when:
    - calico_pool_conf is defined
-  run_once: True
+  run_once: true
  delegate_to: "{{ groups['kube_control_plane'][0] }}"

 - name: "Check kdd calico_datastore if calico_apiserver_enabled"
@@ -184,7 +185,7 @@
    msg: "When using calico apiserver you need to use the kubernetes datastore"
  when:
    - calico_apiserver_enabled
-  run_once: True
+  run_once: true
  delegate_to: "{{ groups['kube_control_plane'][0] }}"

 - name: "Check kdd calico_datastore if typha_enabled"
@@ -193,7 +194,7 @@
    msg: "When using typha you need to use the kubernetes datastore"
  when:
    - typha_enabled
-  run_once: True
+  run_once: true
  delegate_to: "{{ groups['kube_control_plane'][0] }}"

 - name: "Check ipip mode is Never for calico ipv6"
@@ -203,5 +204,5 @@
    msg: "Calico doesn't support ipip tunneling for the IPv6"
  when:
    - enable_dual_stack_networks
-  run_once: True
+  run_once: true
  delegate_to: "{{ groups['kube_control_plane'][0] }}"
--- a/kubespray/project/roles/network_plugin/calico/tasks/install.yml
+++ b/kubespray/project/roles/network_plugin/calico/tasks/install.yml
@@ -13,14 +13,14 @@
  copy:
    src: "{{ downloads.calicoctl.dest }}"
    dest: "{{ bin_dir }}/calicoctl"
-    mode: 0755
-    remote_src: yes
+    mode: "0755"
+    remote_src: true

 - name: Calico | Create calico certs directory
  file:
    dest: "{{ calico_cert_dir }}"
    state: directory
-    mode: 0750
+    mode: "0750"
    owner: root
    group: root
  when: calico_datastore == "etcd"
@@ -30,8 +30,8 @@
    src: "{{ etcd_cert_dir }}/{{ item.s }}"
    dest: "{{ calico_cert_dir }}/{{ item.d }}"
    state: hard
-    mode: 0640
-    force: yes
+    mode: "0640"
+    force: true
  with_items:
    - {s: "{{ kube_etcd_cacert_file }}", d: "ca_cert.crt"}
    - {s: "{{ kube_etcd_cert_file }}", d: "cert.crt"}
@@ -54,14 +54,14 @@
  template:
    src: "calicoctl.{{ calico_datastore }}.sh.j2"
    dest: "{{ bin_dir }}/calicoctl.sh"
-    mode: 0755
+    mode: "0755"
    owner: root
    group: root

 - name: Calico | wait for etcd
  uri:
    url: "{{ etcd_access_addresses.split(',') | first }}/health"
-    validate_certs: no
+    validate_certs: false
    client_cert: "{{ calico_cert_dir }}/cert.crt"
    client_key: "{{ calico_cert_dir }}/key.pem"
  register: result
@@ -142,7 +142,7 @@
      assemble:
        src: "{{ calico_kdd_path }}"
        dest: "{{ kube_config_dir }}/kdd-crds.yml"
-        mode: 0644
+        mode: "0644"
        delimiter: "---\n"
        regexp: ".*\\.yaml"
        remote_src: true
@@ -165,8 +165,8 @@
    - name: Calico | Get existing FelixConfiguration
      command: "{{ bin_dir }}/calicoctl.sh get felixconfig default -o json"
      register: _felix_cmd
-      ignore_errors: True
-      changed_when: False
+      ignore_errors: true
+      changed_when: false

    - name: Calico | Set kubespray FelixConfiguration
      set_fact:
@@ -201,7 +201,7 @@
      command:
        cmd: "{{ bin_dir }}/calicoctl.sh apply -f -"
        stdin: "{{ _felix_config is string | ternary(_felix_config, _felix_config | to_json) }}"
-      changed_when: False
+      changed_when: false

 - name: Calico | Configure Calico IP Pool
  when:
@@ -210,8 +210,8 @@
    - name: Calico | Get existing calico network pool
      command: "{{ bin_dir }}/calicoctl.sh get ippool {{ calico_pool_name }} -o json"
      register: _calico_pool_cmd
-      ignore_errors: True
-      changed_when: False
+      ignore_errors: true
+      changed_when: false

    - name: Calico | Set kubespray calico network pool
      set_fact:
@@ -251,7 +251,7 @@
      command:
        cmd: "{{ bin_dir }}/calicoctl.sh apply -f -"
        stdin: "{{ _calico_pool is string | ternary(_calico_pool, _calico_pool | to_json) }}"
-      changed_when: False
+      changed_when: false

 - name: Calico | Configure Calico IPv6 Pool
  when:
@@ -261,8 +261,8 @@
    - name: Calico | Get existing calico ipv6 network pool
      command: "{{ bin_dir }}/calicoctl.sh get ippool {{ calico_pool_name }}-ipv6 -o json"
      register: _calico_pool_ipv6_cmd
-      ignore_errors: True
-      changed_when: False
+      ignore_errors: true
+      changed_when: false

    - name: Calico | Set kubespray calico network pool
      set_fact:
@@ -302,19 +302,19 @@
      command:
        cmd: "{{ bin_dir }}/calicoctl.sh apply -f -"
        stdin: "{{ _calico_pool_ipv6 is string | ternary(_calico_pool_ipv6, _calico_pool_ipv6 | to_json) }}"
-      changed_when: False
+      changed_when: false

 - name: Populate Service External IPs
  set_fact:
    _service_external_ips: "{{ _service_external_ips | default([]) + [{'cidr': item}] }}"
  with_items: "{{ calico_advertise_service_external_ips }}"
-  run_once: yes
+  run_once: true

 - name: Populate Service LoadBalancer IPs
  set_fact:
    _service_loadbalancer_ips: "{{ _service_loadbalancer_ips | default([]) + [{'cidr': item}] }}"
  with_items: "{{ calico_advertise_service_loadbalancer_ips }}"
-  run_once: yes
+  run_once: true

 - name: "Determine nodeToNodeMesh needed state"
  set_fact:
@@ -322,7 +322,7 @@
  when:
    - peer_with_router | default(false) or peer_with_calico_rr | default(false)
    - inventory_hostname in groups['k8s_cluster']
-  run_once: yes
+  run_once: true

 - name: Calico | Configure Calico BGP
  when:
@@ -331,8 +331,8 @@
    - name: Calico | Get existing BGP Configuration
      command: "{{ bin_dir }}/calicoctl.sh get bgpconfig default -o json"
      register: _bgp_config_cmd
-      ignore_errors: True
-      changed_when: False
+      ignore_errors: true
+      changed_when: false

    - name: Calico | Set kubespray BGP Configuration
      set_fact:
@@ -366,13 +366,13 @@
      command:
        cmd: "{{ bin_dir }}/calicoctl.sh apply -f -"
        stdin: "{{ _bgp_config is string | ternary(_bgp_config, _bgp_config | to_json) }}"
-      changed_when: False
+      changed_when: false

 - name: Calico | Create calico manifests
  template:
    src: "{{ item.file }}.j2"
    dest: "{{ kube_config_dir }}/{{ item.file }}"
-    mode: 0644
+    mode: "0644"
  with_items:
    - {name: calico-config, file: calico-config.yml, type: cm}
    - {name: calico-node, file: calico-node.yml, type: ds}
@@ -389,7 +389,7 @@
  template:
    src: "{{ item.file }}.j2"
    dest: "{{ kube_config_dir }}/{{ item.file }}"
-    mode: 0644
+    mode: "0644"
  with_items:
    - {name: calico, file: calico-typha.yml, type: typha}
  register: calico_node_typha_manifest
@@ -416,7 +416,7 @@
  template:
    src: "{{ item.file }}.j2"
    dest: "{{ kube_config_dir }}/{{ item.file }}"
-    mode: 0644
+    mode: "0644"
  with_items:
    - {name: calico, file: calico-apiserver.yml, type: calico-apiserver}
  register: calico_apiserver_manifest
@@ -469,7 +469,7 @@
  template:
    src: "{{ item.file }}.j2"
    dest: "{{ kube_config_dir }}/{{ item.file }}"
-    mode: 0644
+    mode: "0644"
  with_items:
    - {name: calico, file: calico-ipamconfig.yml, type: ipam}
  when:
--- a/kubespray/project/roles/network_plugin/calico/tasks/peer_with_router.yml
+++ b/kubespray/project/roles/network_plugin/calico/tasks/peer_with_router.yml
@@ -19,10 +19,42 @@
  until: output.rc == 0
  delay: "{{ retry_stagger | random + 3 }}"
  with_items:
-    - "{{ peers | selectattr('scope', 'defined') | selectattr('scope', 'equalto', 'global') | list | default([]) }}"
+    - "{{ peers | default([]) | selectattr('scope', 'defined') | selectattr('scope', 'equalto', 'global') | list }}"
  when:
    - inventory_hostname == groups['kube_control_plane'][0]

+- name: Calico | Get node for per node peering
+  command:
+    cmd: "{{ bin_dir }}/calicoctl.sh get node {{ inventory_hostname }}"
+  register: output_get_node
+  when:
+    - inventory_hostname in groups['k8s_cluster']
+    - local_as is defined
+    - groups['calico_rr'] | default([]) | length == 0
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+
+- name: Calico | Patch node asNumber for per node peering
+  command:
+    cmd: |-
+      {{ bin_dir }}/calicoctl.sh patch node "{{ inventory_hostname }}" --patch '{{ patch is string | ternary(patch, patch | to_json) }}'
+  vars:
+    patch: >
+      {"spec": {
+        "bgp": {
+          "asNumber": "{{ local_as }}"
+        },
+        "orchRefs": [{"nodeName": "{{ inventory_hostname }}", "orchestrator": "k8s"}]
+      }}
+  register: output
+  retries: 0
+  until: output.rc == 0
+  delay: "{{ retry_stagger | random + 3 }}"
+  when:
+    - inventory_hostname in groups['k8s_cluster']
+    - local_as is defined
+    - groups['calico_rr'] | default([]) | length == 0
+    - output_get_node.rc == 0
+
 - name: Calico | Configure node asNumber for per node peering
  command:
    cmd: "{{ bin_dir }}/calicoctl.sh apply -f -"
@@ -48,6 +80,7 @@
    - inventory_hostname in groups['k8s_cluster']
    - local_as is defined
    - groups['calico_rr'] | default([]) | length == 0
+    - output_get_node.rc != 0

 - name: Calico | Configure peering with router(s) at node scope
  command:
@@ -64,6 +97,9 @@
        "asNumber": "{{ item.as }}",
        "node": "{{ inventory_hostname }}",
        "peerIP": "{{ item.router_id }}",
+        {% if calico_version is version('v3.26.0', '>=') and (item.filters | default([]) | length > 0) %}
+        "filters": {{ item.filters }},
+        {% endif %}
        "sourceAddress": "{{ item.sourceaddress | default('UseNodeIP') }}"
      }}
  register: output
@@ -71,7 +107,7 @@
  until: output.rc == 0
  delay: "{{ retry_stagger | random + 3 }}"
  with_items:
-    - "{{ peers | selectattr('scope', 'undefined') | list | default([]) | union(peers | selectattr('scope', 'defined') | selectattr('scope', 'equalto', 'node') | list | default([])) }}"
+    - "{{ peers | default([]) | selectattr('scope', 'undefined') | list | union(peers | default([]) | selectattr('scope', 'defined') | selectattr('scope', 'equalto', 'node') | list ) }}"
  delegate_to: "{{ groups['kube_control_plane'][0] }}"
  when:
    - inventory_hostname in groups['k8s_cluster']
--- a/kubespray/project/roles/network_plugin/calico/tasks/repos.yml
+++ b/kubespray/project/roles/network_plugin/calico/tasks/repos.yml
@@ -10,11 +10,11 @@
        file: _copr:copr.fedorainfracloud.org:jdoss:wireguard
        description: Copr repo for wireguard owned by jdoss
        baseurl: "{{ calico_wireguard_repo }}"
-        gpgcheck: yes
+        gpgcheck: true
        gpgkey: https://download.copr.fedorainfracloud.org/results/jdoss/wireguard/pubkey.gpg
-        skip_if_unavailable: yes
-        enabled: yes
-        repo_gpgcheck: no
+        skip_if_unavailable: true
+        enabled: true
+        repo_gpgcheck: false
      when:
        - ansible_os_family in ['RedHat']
        - ansible_distribution not in ['Fedora']
--- a/kubespray/project/roles/network_plugin/calico/tasks/reset.yml
+++ b/kubespray/project/roles/network_plugin/calico/tasks/reset.yml
@@ -2,9 +2,9 @@
 - name: Reset | check vxlan.calico network device
  stat:
    path: /sys/class/net/vxlan.calico
-    get_attributes: no
-    get_checksum: no
-    get_mime: no
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
  register: vxlan

 - name: Reset | remove the network vxlan.calico device created by calico
@@ -14,9 +14,9 @@
 - name: Reset | check dummy0 network device
  stat:
    path: /sys/class/net/dummy0
-    get_attributes: no
-    get_checksum: no
-    get_mime: no
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
  register: dummy0

 - name: Reset | remove the network device created by calico
--- a/kubespray/project/roles/network_plugin/calico/tasks/typha_certs.yml
+++ b/kubespray/project/roles/network_plugin/calico/tasks/typha_certs.yml
@@ -9,21 +9,22 @@
  file:
    path: /etc/calico/certs
    state: directory
-    mode: 0755
+    mode: "0755"
  when: typha_server_secret.rc != 0

 - name: Calico | Copy ssl script for typha certs
  template:
    src: make-ssl-calico.sh.j2
    dest: "{{ bin_dir }}/make-ssl-typha.sh"
-    mode: 0755
+    mode: "0755"
+
  when: typha_server_secret.rc != 0

 - name: Calico | Copy ssl config for typha certs
  copy:
    src: openssl.conf
    dest: /etc/calico/certs/openssl.conf
-    mode: 0644
+    mode: "0644"
  when: typha_server_secret.rc != 0

 - name: Calico | Generate typha certs
--- a/kubespray/project/roles/network_plugin/calico/templates/calico-apiserver.yml.j2
+++ b/kubespray/project/roles/network_plugin/calico/templates/calico-apiserver.yml.j2
@@ -72,6 +72,15 @@ spec:
          initialDelaySeconds: 90
          periodSeconds: 10
        name: calico-apiserver
+{% if calico_version is version('v3.28.0', '>=') %}
+        readinessProbe:
+          httpGet:
+            path: /readyz
+            port: 5443
+            scheme: HTTPS
+          timeoutSeconds: 5
+          periodSeconds: 60
+{% else %}
        readinessProbe:
          exec:
            command:
@@ -79,6 +88,7 @@ spec:
          failureThreshold: 5
          initialDelaySeconds: 5
          periodSeconds: 10
+{% endif %}
        securityContext:
          privileged: false
          runAsUser: 0
@@ -157,6 +167,7 @@ rules:
  - networksets
  - bgpconfigurations
  - bgppeers
+  - bgpfilters
  - felixconfigurations
  - kubecontrollersconfigurations
  - ippools
@@ -172,7 +183,16 @@ rules:
  - create
  - update
  - delete
-
+{% if calico_version is version('v3.28.0', '>=') %}
+- apiGroups:
+  - policy
+  resourceNames:
+  - calico-apiserver
+  resources:
+  - podsecuritypolicies
+  verbs:
+  - use
+{% endif %}
 ---

 apiVersion: rbac.authorization.k8s.io/v1
--- a/kubespray/project/roles/network_plugin/calico/templates/calico-config.yml.j2
+++ b/kubespray/project/roles/network_plugin/calico/templates/calico-config.yml.j2
@@ -102,4 +102,3 @@ data:
        }
      ]
    }
-
--- a/kubespray/project/roles/network_plugin/calico/templates/calico-cr.yml.j2
+++ b/kubespray/project/roles/network_plugin/calico/templates/calico-cr.yml.j2
@@ -16,6 +16,11 @@ rules:
      - pods/status
    verbs:
      - patch
+  - apiGroups: [""]
+    resources:
+      - nodes/status
+    verbs:
+      - update
  - apiGroups: ["crd.projectcalico.org"]
    resources:
      - blockaffinities
--- a/kubespray/project/roles/network_plugin/calico/templates/calico-node.yml.j2
+++ b/kubespray/project/roles/network_plugin/calico/templates/calico-node.yml.j2
@@ -38,7 +38,7 @@ spec:
      # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods.
      terminationGracePeriodSeconds: 0
      initContainers:
-{% if calico_datastore == "kdd" %}
+{% if calico_datastore == "kdd" and not calico_ipam_host_local %}
        # This container performs upgrade from host-local IPAM to calico-ipam.
        # It can be deleted if this is a fresh installation, or if you have already
        # upgraded to use calico-ipam.
@@ -310,6 +310,10 @@ spec:
              value: "{{ calico_node_ignorelooserpf }}"
            - name: CALICO_MANAGE_CNI
              value: "true"
+{% if calico_ipam_host_local %}
+            - name: USE_POD_CIDR
+              value: "true"
+{% endif %}
 {% if calico_node_extra_envs is defined %}
 {% for key in calico_node_extra_envs %}
            - name: {{ key }}
@@ -407,9 +411,11 @@ spec:
        - name: var-run-calico
          hostPath:
            path: /var/run/calico
+            type: DirectoryOrCreate
        - name: var-lib-calico
          hostPath:
            path: /var/lib/calico
+            type: DirectoryOrCreate
        # Used to install CNI.
        - name: cni-net-dir
          hostPath:
@@ -417,6 +423,7 @@ spec:
        - name: cni-bin-dir
          hostPath:
            path: /opt/cni/bin
+            type: DirectoryOrCreate
 {% if calico_datastore == "etcd" %}
        # Mount in the etcd TLS secrets.
        - name: etcd-certs
@@ -428,7 +435,7 @@ spec:
          hostPath:
            path: /run/xtables.lock
            type: FileOrCreate
-{% if calico_datastore == "kdd" %}
+{% if calico_datastore == "kdd" and not calico_ipam_host_local %}
        # Mount in the directory for host-local IPAM allocations. This is
        # used when upgrading from host-local to calico-ipam, and can be removed
        # if not using the upgrade-ipam init container.
--- a/kubespray/project/roles/network_plugin/calico/templates/calico-typha.yml.j2
+++ b/kubespray/project/roles/network_plugin/calico/templates/calico-typha.yml.j2
@@ -136,11 +136,10 @@ spec:
            name: cacert
            readOnly: true
 {% endif %}
-          # Needed for version >=3.7 when the 'host-local' ipam is used
-          # Should never happen given templates/cni-calico.conflist.j2
-          # Configure route aggregation based on pod CIDR.
-          # - name: USE_POD_CIDR
-          #   value: "true"
+{% if calico_ipam_host_local %}
+          - name: USE_POD_CIDR
+            value: "true"
+{% endif %}
        livenessProbe:
          httpGet:
            path: /liveness
--- a/kubespray/project/roles/network_plugin/cilium/defaults/main.yml
+++ b/kubespray/project/roles/network_plugin/cilium/defaults/main.yml
@@ -7,6 +7,9 @@ cilium_mtu: ""
 cilium_enable_ipv4: true
 cilium_enable_ipv6: false

+# Enable l2 announcement from cilium to replace Metallb Ref: https://docs.cilium.io/en/v1.14/network/l2-announcements/
+cilium_l2announcements: false
+
 # Cilium agent health port
 cilium_agent_health_port: "{%- if cilium_version | regex_replace('v') is version('1.11.6', '>=') -%}9879{%- else -%}9876{%- endif -%}"

@@ -39,6 +42,10 @@ cilium_cpu_requests: 100m

 # Overlay Network Mode
 cilium_tunnel_mode: vxlan
+
+# LoadBalancer Mode (snat/dsr/hybrid) Ref: https://docs.cilium.io/en/stable/network/kubernetes/kubeproxy-free/#dsr-mode
+cilium_loadbalancer_mode: snat
+
 # Optional features
 cilium_enable_prometheus: false
 # Enable if you want to make use of hostPort mappings
@@ -152,6 +159,12 @@ cilium_hubble_install: false
 ### Enable auto generate certs if cilium_hubble_install: true
 cilium_hubble_tls_generate: false

+### Capacity of Hubble events buffer. The provided value must be one less than an integer power of two and no larger than 65535
+### (ie: 1, 3, ..., 2047, 4095, ..., 65535) (default 4095)
+# cilium_hubble_event_buffer_capacity: 4095
+### Buffer size of the channel to receive monitor events.
+# cilium_hubble_event_queue_size: 50
+
 # The default IP address management mode is "Cluster Scope".
 # https://docs.cilium.io/en/stable/concepts/networking/ipam/
 cilium_ipam_mode: cluster-pool
--- a/kubespray/project/roles/network_plugin/cilium/tasks/check.yml
+++ b/kubespray/project/roles/network_plugin/cilium/tasks/check.yml
@@ -61,3 +61,9 @@
  when:
    - cilium_ipsec_enabled is defined
    - cilium_ipsec_enabled
+
+- name: Stop if cilium_hubble_event_buffer_capacity is not a power of 2 minus 1 and is not between 1 and 65535
+  assert:
+    that: "cilium_hubble_event_buffer_capacity in [1, 3, 7, 15, 31, 63, 127, 255, 511, 1023, 2047, 4095, 8191, 16383, 32767, 65535]"
+    msg: "Error: cilium_hubble_event_buffer_capacity:{{ cilium_hubble_event_buffer_capacity }} is not a power of 2 minus 1 and it should be between 1 and 65535."
+  when: cilium_hubble_event_buffer_capacity is defined
--- a/kubespray/project/roles/network_plugin/cilium/tasks/install.yml
+++ b/kubespray/project/roles/network_plugin/cilium/tasks/install.yml
@@ -10,7 +10,7 @@
  file:
    dest: "{{ cilium_cert_dir }}"
    state: directory
-    mode: 0750
+    mode: "0750"
    owner: root
    group: root
  when:
@@ -20,9 +20,9 @@
  file:
    src: "{{ etcd_cert_dir }}/{{ item.s }}"
    dest: "{{ cilium_cert_dir }}/{{ item.d }}"
-    mode: 0644
+    mode: "0644"
    state: hard
-    force: yes
+    force: true
  loop:
    - {s: "{{ kube_etcd_cacert_file }}", d: "ca_cert.crt"}
    - {s: "{{ kube_etcd_cert_file }}", d: "cert.crt"}
@@ -36,7 +36,7 @@
    state: directory
    owner: root
    group: root
-    mode: 0755
+    mode: "0755"
  when:
    - inventory_hostname == groups['kube_control_plane'][0]
    - cilium_hubble_install
@@ -45,7 +45,7 @@
  template:
    src: "{{ item.name }}/{{ item.file }}.j2"
    dest: "{{ kube_config_dir }}/{{ item.name }}-{{ item.file }}"
-    mode: 0644
+    mode: "0644"
  loop:
    - {name: cilium, file: config.yml, type: cm}
    - {name: cilium-operator, file: crb.yml, type: clusterrolebinding}
@@ -66,7 +66,7 @@
  template:
    src: "{{ item.name }}/{{ item.file }}.j2"
    dest: "{{ kube_config_dir }}/addons/hubble/{{ item.name }}-{{ item.file }}"
-    mode: 0644
+    mode: "0644"
  loop:
    - {name: hubble, file: config.yml, type: cm}
    - {name: hubble, file: crb.yml, type: clusterrolebinding}
@@ -86,12 +86,12 @@
  template:
    src: 000-cilium-portmap.conflist.j2
    dest: /etc/cni/net.d/000-cilium-portmap.conflist
-    mode: 0644
+    mode: "0644"
  when: cilium_enable_portmap

 - name: Cilium | Copy Ciliumcli binary from download dir
  copy:
    src: "{{ local_release_dir }}/cilium"
    dest: "{{ bin_dir }}/cilium"
-    mode: 0755
-    remote_src: yes
+    mode: "0755"
+    remote_src: true
--- a/kubespray/project/roles/network_plugin/cilium/tasks/reset_iface.yml
+++ b/kubespray/project/roles/network_plugin/cilium/tasks/reset_iface.yml
@@ -2,9 +2,9 @@
 - name: "Reset | check if network device {{ iface }} is present"
  stat:
    path: "/sys/class/net/{{ iface }}"
-    get_attributes: no
-    get_checksum: no
-    get_mime: no
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
  register: device_remains

 - name: "Reset | remove network device {{ iface }}"
--- a/kubespray/project/roles/network_plugin/cilium/templates/cilium-operator/cr.yml.j2
+++ b/kubespray/project/roles/network_plugin/cilium/templates/cilium-operator/cr.yml.j2
@@ -97,6 +97,11 @@ rules:
  - ciliumloadbalancerippools/status
  - ciliumbgppeeringpolicies
  - ciliumenvoyconfigs
+{% endif %}
+{% if cilium_version | regex_replace('v') is version('1.15', '>=') %}
+  - ciliumbgppeerconfigs
+  - ciliumbgpadvertisements
+  - ciliumbgpnodeconfigs
 {% endif %}
  verbs:
  - '*'
@@ -146,6 +151,20 @@ rules:
  - ciliumlocalredirectpolicies.cilium.io
  - ciliumnetworkpolicies.cilium.io
  - ciliumnodes.cilium.io
+{% if cilium_version | regex_replace('v') is version('1.14', '>=') %}
+  - ciliumnodeconfigs.cilium.io
+  - ciliumcidrgroups.cilium.io
+  - ciliuml2announcementpolicies.cilium.io
+  - ciliumpodippools.cilium.io
+  - ciliumloadbalancerippools.cilium.io
+{% endif %}
+{% if cilium_version | regex_replace('v') is version('1.15', '>=') %}
+  - ciliumbgpclusterconfigs.cilium.io
+  - ciliumbgppeerconfigs.cilium.io
+  - ciliumbgpadvertisements.cilium.io
+  - ciliumbgpnodeconfigs.cilium.io
+  - ciliumbgpnodeconfigoverrides.cilium.io
+{% endif %}
 {% endif %}
 {% for rules in cilium_clusterrole_rules_operator_extra_vars %}
 - apiGroups:
--- a/kubespray/project/roles/network_plugin/cilium/templates/cilium/config.yml.j2
+++ b/kubespray/project/roles/network_plugin/cilium/templates/cilium/config.yml.j2
@@ -69,6 +69,13 @@ data:
  # custom-cni-conf to "true", otherwise Cilium may overwrite the configuration.
  custom-cni-conf: "false"

+{% if cilium_version | regex_replace('v') is version('1.14.0', '>=') %}
+  # Tell the agent to generate and write a CNI configuration file
+  write-cni-conf-when-ready: /host/etc/cni/net.d/05-cilium.conflist
+  cni-exclusive: "{{ cilium_cni_exclusive }}"
+  cni-log-file: "{{ cilium_cni_log_file }}"
+{% endif %}
+
  # If you want cilium monitor to aggregate tracing for packets, set this level
  # to "low", "medium", or "maximum". The higher the level, the less packets
  # that will be seen in monitor output.
@@ -124,6 +131,12 @@ data:
  tunnel-protocol: "{{ cilium_tunnel_mode }}"
 {% endif %}

+  ## DSR setting
+  bpf-lb-mode: "{{ cilium_loadbalancer_mode }}"
+
+  # l2
+  enable-l2-announcements: "{{ cilium_l2announcements }}"
+
  # Enable Bandwidth Manager
  # Cilium’s bandwidth manager supports the kubernetes.io/egress-bandwidth Pod annotation.
  # Bandwidth enforcement currently does not work in combination with L7 Cilium Network Policies.
@@ -179,6 +192,12 @@ data:
 {% for hubble_metrics_cycle in cilium_hubble_metrics %}
    {{ hubble_metrics_cycle }}
 {% endfor %}
+{% endif %}
+{% if cilium_hubble_event_buffer_capacity is defined %}
+  hubble-event-buffer-capacity: "{{ cilium_hubble_event_buffer_capacity }}"
+{% endif %}
+{% if cilium_hubble_event_queue_size is defined %}
+  hubble-event-queue-size: "{{ cilium_hubble_event_queue_size }}"
 {% endif %}
  hubble-listen-address: ":4244"
 {% if cilium_enable_hubble and cilium_hubble_install %}
--- a/kubespray/project/roles/network_plugin/cilium/templates/cilium/cr.yml.j2
+++ b/kubespray/project/roles/network_plugin/cilium/templates/cilium/cr.yml.j2
@@ -106,6 +106,15 @@ rules:
  - ciliumnodes/finalizers
  - ciliumidentities/finalizers
  - ciliumlocalredirectpolicies/finalizers
+{% endif %}
+{% if cilium_version | regex_replace('v') is version('1.14', '>=') %}
+  - ciliuml2announcementpolicies/status
+{% endif %}
+{% if cilium_version | regex_replace('v') is version('1.15', '>=') %}
+  - ciliumbgpnodeconfigs
+  - ciliumbgpnodeconfigs/status
+  - ciliumbgpadvertisements
+  - ciliumbgppeerconfigs
 {% endif %}
  verbs:
  - '*'
@@ -125,7 +134,22 @@ rules:
  - cilium.io
  resources:
  - ciliumcidrgroups
+  - ciliuml2announcementpolicies
+  - ciliumpodippools
+  - ciliuml2announcementpolicies/status
  verbs:
  - list
  - watch
+{% if cilium_version %}
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - create
+  - get
+  - update
+  - list
+  - delete
+{% endif %}
 {% endif %}
--- a/kubespray/project/roles/network_plugin/cilium/templates/hubble/config.yml.j2
+++ b/kubespray/project/roles/network_plugin/cilium/templates/hubble/config.yml.j2
@@ -12,10 +12,10 @@ data:
    peer-service: "hubble-peer.kube-system.svc.{{ dns_domain }}:443"
    listen-address: :4245
    metrics-listen-address: ":9966"
-    dial-timeout: 
-    retry-timeout: 
-    sort-buffer-len-max: 
-    sort-buffer-drain-timeout: 
+    dial-timeout:
+    retry-timeout:
+    sort-buffer-len-max:
+    sort-buffer-drain-timeout:
    tls-client-cert-file: /var/lib/hubble-relay/tls/client.crt
    tls-client-key-file: /var/lib/hubble-relay/tls/client.key
    tls-server-cert-file: /var/lib/hubble-relay/tls/server.crt
--- a/kubespray/project/roles/network_plugin/cilium/templates/hubble/service.yml.j2
+++ b/kubespray/project/roles/network_plugin/cilium/templates/hubble/service.yml.j2
@@ -102,4 +102,3 @@ spec:
    protocol: TCP
    targetPort: 4244
  internalTrafficPolicy: Local
-
--- a/kubespray/project/roles/network_plugin/cni/tasks/main.yml
+++ b/kubespray/project/roles/network_plugin/cni/tasks/main.yml
@@ -3,7 +3,7 @@
  file:
    path: /opt/cni/bin
    state: directory
-    mode: 0755
+    mode: "0755"
    owner: "{{ cni_bin_owner }}"
    recurse: true

@@ -11,6 +11,6 @@
  unarchive:
    src: "{{ downloads.cni.dest }}"
    dest: "/opt/cni/bin"
-    mode: 0755
+    mode: "0755"
    owner: "{{ cni_bin_owner }}"
-    remote_src: yes
+    remote_src: true
--- a/kubespray/project/roles/network_plugin/custom_cni/tasks/main.yml
+++ b/kubespray/project/roles/network_plugin/custom_cni/tasks/main.yml
@@ -12,7 +12,7 @@
    template:
      src: "{{ item }}"
      dest: "{{ kube_config_dir }}/{{ item | basename | replace('.j2', '') }}"
-      mode: 0644
+      mode: "0644"
    loop: "{{ custom_cni_manifests }}"
    delegate_to: "{{ groups['kube_control_plane'] | first }}"
    run_once: true
--- a/kubespray/project/roles/network_plugin/flannel/tasks/main.yml
+++ b/kubespray/project/roles/network_plugin/flannel/tasks/main.yml
@@ -12,7 +12,7 @@
  template:
    src: "{{ item.file }}.j2"
    dest: "{{ kube_config_dir }}/{{ item.file }}"
-    mode: 0644
+    mode: "0644"
  with_items:
    - {name: flannel, file: cni-flannel-rbac.yml, type: sa}
    - {name: kube-flannel, file: cni-flannel.yml, type: ds}
--- a/kubespray/project/roles/network_plugin/flannel/tasks/reset.yml
+++ b/kubespray/project/roles/network_plugin/flannel/tasks/reset.yml
@@ -2,9 +2,9 @@
 - name: Reset | check cni network device
  stat:
    path: /sys/class/net/cni0
-    get_attributes: no
-    get_checksum: no
-    get_mime: no
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
  register: cni

 - name: Reset | remove the network device created by the flannel
@@ -14,9 +14,9 @@
 - name: Reset | check flannel network device
  stat:
    path: /sys/class/net/flannel.1
-    get_attributes: no
-    get_checksum: no
-    get_mime: no
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
  register: flannel

 - name: Reset | remove the network device created by the flannel
--- a/kubespray/project/roles/network_plugin/kube-ovn/defaults/main.yml
+++ b/kubespray/project/roles/network_plugin/kube-ovn/defaults/main.yml
@@ -115,4 +115,4 @@ kube_ovn_ls_dnat_mod_dl_dst: true
 kube_ovn_keep_vm_ip: true

 ## cni config priority, default: 01
-kube_ovn_cni_config_priority: 01
+kube_ovn_cni_config_priority: '01'
--- a/kubespray/project/roles/network_plugin/kube-ovn/tasks/main.yml
+++ b/kubespray/project/roles/network_plugin/kube-ovn/tasks/main.yml
@@ -9,7 +9,7 @@
  template:
    src: "{{ item.file }}.j2"
    dest: "{{ kube_config_dir }}/{{ item.file }}"
-    mode: 0644
+    mode: "0644"
  with_items:
    - {name: kube-ovn-crd, file: cni-kube-ovn-crd.yml}
    - {name: ovn, file: cni-ovn.yml}
--- a/kubespray/project/roles/network_plugin/kube-ovn/templates/cni-kube-ovn-crd.yml.j2
+++ b/kubespray/project/roles/network_plugin/kube-ovn/templates/cni-kube-ovn-crd.yml.j2
--- a/kubespray/project/roles/network_plugin/kube-ovn/templates/cni-kube-ovn.yml.j2
+++ b/kubespray/project/roles/network_plugin/kube-ovn/templates/cni-kube-ovn.yml.j2
@@ -1,4 +1,190 @@
 ---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: ovn-vpc-nat-config
+  namespace: kube-system
+  annotations:
+    kubernetes.io/description: |
+      kube-ovn vpc-nat common config
+data:
+  image: {{ kube_ovn_vpc_container_image_repo }}:{{ kube_ovn_vpc_container_image_tag }}
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: ovn-vpc-nat-gw-config
+  namespace: kube-system
+data:
+  enable-vpc-nat-gw: "true"
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kube-ovn-cni
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  annotations:
+    rbac.authorization.k8s.io/system-only: "true"
+  name: system:kube-ovn-cni
+rules:
+  - apiGroups:
+      - "kubeovn.io"
+    resources:
+      - subnets
+      - vlans
+      - provider-networks
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - ""
+      - "kubeovn.io"
+    resources:
+      - ovn-eips
+      - ovn-eips/status
+      - nodes
+      - pods
+      - vlans
+    verbs:
+      - get
+      - list
+      - patch
+      - watch
+  - apiGroups:
+      - "kubeovn.io"
+    resources:
+      - ips
+    verbs:
+      - get
+      - update
+  - apiGroups:
+      - ""
+    resources:
+      - events
+    verbs:
+      - create
+      - patch
+      - update
+  - apiGroups:
+      - ""
+    resources:
+      - configmaps
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - authentication.k8s.io
+    resources:
+      - tokenreviews
+    verbs:
+      - create
+  - apiGroups:
+      - authorization.k8s.io
+    resources:
+      - subjectaccessreviews
+    verbs:
+      - create
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kube-ovn-cni
+roleRef:
+  name: system:kube-ovn-cni
+  kind: ClusterRole
+  apiGroup: rbac.authorization.k8s.io
+subjects:
+  - kind: ServiceAccount
+    name: kube-ovn-cni
+    namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: kube-ovn-cni
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader
+subjects:
+  - kind: ServiceAccount
+    name: kube-ovn-cni
+    namespace: kube-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kube-ovn-app
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  annotations:
+    rbac.authorization.k8s.io/system-only: "true"
+  name: system:kube-ovn-app
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - nodes
+    verbs:
+      - get
+      - list
+  - apiGroups:
+      - apps
+    resources:
+      - daemonsets
+    verbs:
+      - get
+  - apiGroups:
+      - authentication.k8s.io
+    resources:
+      - tokenreviews
+    verbs:
+      - create
+  - apiGroups:
+      - authorization.k8s.io
+    resources:
+      - subjectaccessreviews
+    verbs:
+      - create
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kube-ovn-app
+roleRef:
+  name: system:kube-ovn-app
+  kind: ClusterRole
+  apiGroup: rbac.authorization.k8s.io
+subjects:
+  - kind: ServiceAccount
+    name: kube-ovn-app
+    namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: kube-ovn-app
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader
+subjects:
+  - kind: ServiceAccount
+    name: kube-ovn-app
+    namespace: kube-system
+---
 kind: Deployment
 apiVersion: apps/v1
 metadata:
@@ -25,8 +211,20 @@ spec:
        type: infra
    spec:
      tolerations:
-      - operator: Exists
+        - effect: NoSchedule
+          operator: Exists
+        - key: CriticalAddonsOnly
+          operator: Exists
      affinity:
+        nodeAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - preference:
+              matchExpressions:
+              - key: "ovn.kubernetes.io/ic-gw"
+                operator: NotIn
+                values:
+                - "true"
+            weight: 100
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            - labelSelector:
@@ -67,7 +265,12 @@ spec:
          - --log_file_max_size=0
          - --enable-lb-svc=false
          - --keep-vm-ip={{ kube_ovn_keep_vm_ip }}
-          - --pod-default-fip-type=""
+          securityContext:
+            runAsUser: 0
+            privileged: false
+            capabilities:
+              add:
+                - NET_BIND_SERVICE
          env:
            - name: ENABLE_SSL
              value: "{{ kube_ovn_enable_ssl | lower }}"
@@ -85,6 +288,10 @@ spec:
                  fieldPath: spec.nodeName
            - name: OVN_DB_IPS
              value: "{{ kube_ovn_central_ips }}"
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
            - name: POD_IPS
              valueFrom:
                fieldRef:
@@ -96,18 +303,24 @@ spec:
              name: localtime
            - mountPath: /var/log/kube-ovn
              name: kube-ovn-log
+            - mountPath: /var/log/ovn
+              name: ovn-log
            - mountPath: /var/run/tls
              name: kube-ovn-tls
          readinessProbe:
            exec:
              command:
-                - /kube-ovn/kube-ovn-controller-healthcheck
+                - /kube-ovn/kube-ovn-healthcheck
+                - --port=10660
+                - --tls=false
            periodSeconds: 3
            timeoutSeconds: 45
          livenessProbe:
            exec:
              command:
-                - /kube-ovn/kube-ovn-controller-healthcheck
+                - /kube-ovn/kube-ovn-healthcheck
+                - --port=10660
+                - --tls=false
            initialDelaySeconds: 300
            periodSeconds: 7
            failureThreshold: 5
@@ -128,6 +341,9 @@ spec:
        - name: kube-ovn-log
          hostPath:
            path: /var/log/kube-ovn
+        - name: ovn-log
+          hostPath:
+            path: /var/log/ovn
        - name: kube-ovn-tls
          secret:
            optional: true
@@ -161,7 +377,7 @@ spec:
        - key: CriticalAddonsOnly
          operator: Exists
      priorityClassName: system-node-critical
-      serviceAccountName: ovn
+      serviceAccountName: kube-ovn-cni
      hostNetwork: true
      hostPID: true
      initContainers:
@@ -202,7 +418,13 @@ spec:
          - --log_file_max_size=0
        securityContext:
          runAsUser: 0
-          privileged: true
+          privileged: false
+          capabilities:
+            add:
+              - NET_ADMIN
+              - NET_BIND_SERVICE
+              - NET_RAW
+              - SYS_ADMIN
        env:
          - name: ENABLE_SSL
            value: "{{ kube_ovn_enable_ssl | lower }}"
@@ -231,14 +453,15 @@ spec:
            mountPath: /lib/modules
            readOnly: true
          - name: shared-dir
-            mountPath: /var/lib/kubelet/pods
+            mountPath: $KUBELET_DIR/pods
          - mountPath: /etc/openvswitch
            name: systemid
+            readOnly: true
          - mountPath: /etc/cni/net.d
            name: cni-conf
          - mountPath: /run/openvswitch
            name: host-run-ovs
-            mountPropagation: Bidirectional
+            mountPropagation: HostToContainer
          - mountPath: /run/ovn
            name: host-run-ovn
          - mountPath: /host/var/run/dbus
@@ -255,6 +478,7 @@ spec:
            name: host-log-ovn
          - mountPath: /etc/localtime
            name: localtime
+            readOnly: true
          - mountPath: /tmp
            name: tmp
        livenessProbe:
@@ -262,17 +486,22 @@ spec:
          initialDelaySeconds: 30
          periodSeconds: 7
          successThreshold: 1
-          tcpSocket:
-            port: 10665
-          timeoutSeconds: 3
+          exec:
+            command:
+              - /kube-ovn/kube-ovn-healthcheck
+              - --port=10665
+              - --tls=false
+          timeoutSeconds: 5
        readinessProbe:
          failureThreshold: 3
-          initialDelaySeconds: 30
          periodSeconds: 7
          successThreshold: 1
-          tcpSocket:
-            port: 10665
-          timeoutSeconds: 3
+          exec:
+            command:
+              - /kube-ovn/kube-ovn-healthcheck
+              - --port=10665
+              - --tls=false
+          timeoutSeconds: 5
        resources:
          requests:
            cpu: {{ kube_ovn_cni_server_cpu_request }}
@@ -389,28 +618,23 @@ spec:
                fieldRef:
                  fieldPath: spec.nodeName
          volumeMounts:
-            - mountPath: /lib/modules
-              name: host-modules
-              readOnly: true
-            - mountPath: /run/openvswitch
-              name: host-run-ovs
            - mountPath: /var/run/openvswitch
              name: host-run-ovs
            - mountPath: /var/run/ovn
              name: host-run-ovn
-            - mountPath: /sys
-              name: host-sys
-              readOnly: true
            - mountPath: /etc/openvswitch
              name: host-config-openvswitch
            - mountPath: /var/log/openvswitch
              name: host-log-ovs
+              readOnly: true
            - mountPath: /var/log/ovn
              name: host-log-ovn
+              readOnly: true
            - mountPath: /var/log/kube-ovn
              name: kube-ovn-log
            - mountPath: /etc/localtime
              name: localtime
+              readOnly: true
            - mountPath: /var/run/tls
              name: kube-ovn-tls
          resources:
@@ -423,18 +647,12 @@ spec:
      nodeSelector:
        kubernetes.io/os: "linux"
      volumes:
-        - name: host-modules
-          hostPath:
-            path: /lib/modules
        - name: host-run-ovs
          hostPath:
            path: /run/openvswitch
        - name: host-run-ovn
          hostPath:
            path: /run/ovn
-        - name: host-sys
-          hostPath:
-            path: /sys
        - name: host-config-openvswitch
          hostPath:
            path: /etc/origin/openvswitch
@@ -500,6 +718,12 @@ spec:
          image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }}
          imagePullPolicy: {{ k8s_image_pull_policy }}
          command: ["/kube-ovn/start-ovn-monitor.sh"]
+          args:
+          - --secure-serving=false
+          - --log_file=/var/log/kube-ovn/kube-ovn-monitor.log
+          - --logtostderr=false
+          - --alsologtostderr=true
+          - --log_file_max_size=200
          securityContext:
            runAsUser: 0
            privileged: false
@@ -510,6 +734,10 @@ spec:
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
            - name: POD_IPS
              valueFrom:
                fieldRef:
@@ -532,30 +760,38 @@ spec:
              name: host-config-openvswitch
            - mountPath: /etc/ovn
              name: host-config-ovn
-            - mountPath: /var/log/openvswitch
-              name: host-log-ovs
            - mountPath: /var/log/ovn
              name: host-log-ovn
+              readOnly: true
            - mountPath: /etc/localtime
              name: localtime
+              readOnly: true
            - mountPath: /var/run/tls
              name: kube-ovn-tls
-          readinessProbe:
-            exec:
-              command:
-              - cat
-              - /var/run/ovn/ovn-controller.pid
-            periodSeconds: 10
-            timeoutSeconds: 45
+            - mountPath: /var/log/kube-ovn
+              name: kube-ovn-log
          livenessProbe:
+            failureThreshold: 3
+            initialDelaySeconds: 30
+            periodSeconds: 7
+            successThreshold: 1
            exec:
              command:
-              - cat
-              - /var/run/ovn/ovn-controller.pid
+                - /kube-ovn/kube-ovn-healthcheck
+                - --port=10661
+                - --tls=false
+            timeoutSeconds: 5
+          readinessProbe:
+            failureThreshold: 3
            initialDelaySeconds: 30
-            periodSeconds: 10
-            failureThreshold: 5
-            timeoutSeconds: 45
+            periodSeconds: 7
+            successThreshold: 1
+            exec:
+              command:
+                - /kube-ovn/kube-ovn-healthcheck
+                - --port=10661
+                - --tls=false
+            timeoutSeconds: 5
      nodeSelector:
        kubernetes.io/os: "linux"
        kube-ovn/role: "master"
@@ -585,6 +821,9 @@ spec:
          secret:
            optional: true
            secretName: kube-ovn-tls
+        - name: kube-ovn-log
+          hostPath:
+            path: /var/log/kube-ovn
 ---
 kind: Service
 apiVersion: v1
@@ -655,7 +894,7 @@ spec:
  ports:
    - port: 10665
      name: metrics
-      {% if kube_ovn_ic_enable %}
+{% if kube_ovn_ic_enable %}
 ---
 kind: ConfigMap
 apiVersion: v1
--- a/kubespray/project/roles/network_plugin/kube-ovn/templates/cni-ovn.yml.j2
+++ b/kubespray/project/roles/network_plugin/kube-ovn/templates/cni-ovn.yml.j2
@@ -1,6 +1,54 @@
 ---
 apiVersion: v1
 kind: ServiceAccount
+metadata:
+  name: ovn-ovs
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  annotations:
+    rbac.authorization.k8s.io/system-only: "true"
+  name: system:ovn-ovs
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - get
+      - patch
+  - apiGroups:
+      - ""
+    resources:
+      - services
+      - endpoints
+    verbs:
+      - get
+  - apiGroups:
+      - apps
+    resources:
+      - controllerrevisions
+    verbs:
+      - get
+      - list
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: ovn-ovs
+roleRef:
+  name: system:ovn-ovs
+  kind: ClusterRole
+  apiGroup: rbac.authorization.k8s.io
+subjects:
+  - kind: ServiceAccount
+    name: ovn-ovs
+    namespace: kube-system
+---
+apiVersion: v1
+kind: ServiceAccount
 metadata:
  name: ovn
  namespace: kube-system
@@ -18,8 +66,11 @@ rules:
      - vpcs
      - vpcs/status
      - vpc-nat-gateways
+      - vpc-nat-gateways/status
      - subnets
      - subnets/status
+      - ippools
+      - ippools/status
      - ips
      - vips
      - vips/status
@@ -43,59 +94,98 @@ rules:
      - ovn-eips/status
      - ovn-fips/status
      - ovn-snat-rules/status
+      - ovn-dnat-rules
+      - ovn-dnat-rules/status
      - switch-lb-rules
      - switch-lb-rules/status
      - vpc-dnses
      - vpc-dnses/status
+      - qos-policies
+      - qos-policies/status
    verbs:
      - "*"
  - apiGroups:
      - ""
    resources:
      - pods
-      - pods/exec
      - namespaces
-      - nodes
-      - configmaps
    verbs:
-      - create
      - get
      - list
+      - patch
      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - nodes
+    verbs:
+      - get
+      - list
      - patch
      - update
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - pods/exec
+    verbs:
+      - create
  - apiGroups:
      - "k8s.cni.cncf.io"
    resources:
      - network-attachment-definitions
    verbs:
-      - create
-      - delete
      - get
-      - list
-      - update
  - apiGroups:
      - ""
      - networking.k8s.io
-      - apps
-      - extensions
    resources:
      - networkpolicies
-      - services
-      - services/status
-      - endpoints
-      - statefulsets
-      - daemonsets
-      - deployments
-      - deployments/scale
+      - configmaps
    verbs:
-      - create
-      - delete
-      - update
-      - patch
      - get
      - list
      - watch
+  - apiGroups:
+      - apps
+    resources:
+      - daemonsets
+    verbs:
+      - get
+  - apiGroups:
+      - ""
+    resources:
+      - services
+      - services/status
+    verbs:
+      - get
+      - list
+      - update
+      - create
+      - delete
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - endpoints
+    verbs:
+      - create
+      - update
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - apps
+    resources:
+      - statefulsets
+      - deployments
+      - deployments/scale
+    verbs:
+      - get
+      - list
+      - create
+      - delete
+      - update
  - apiGroups:
      - ""
    resources:
@@ -118,6 +208,18 @@ rules:
    verbs:
      - get
      - list
+  - apiGroups:
+      - authentication.k8s.io
+    resources:
+      - tokenreviews
+    verbs:
+      - create
+  - apiGroups:
+      - authorization.k8s.io
+    resources:
+      - subjectaccessreviews
+    verbs:
+      - create
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -132,6 +234,20 @@ subjects:
    name: ovn
    namespace: kube-system
 ---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: ovn
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader
+subjects:
+  - kind: ServiceAccount
+    name: ovn
+    namespace: kube-system
+---
 kind: Service
 apiVersion: v1
 metadata:
@@ -218,7 +334,12 @@ spec:
        type: infra
    spec:
      tolerations:
-      - operator: Exists
+        - effect: NoSchedule
+          operator: Exists
+        - effect: NoExecute
+          operator: Exists
+        - key: CriticalAddonsOnly
+          operator: Exists
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
@@ -227,7 +348,7 @@ spec:
                  app: ovn-central
              topologyKey: kubernetes.io/hostname
      priorityClassName: system-cluster-critical
-      serviceAccountName: ovn
+      serviceAccountName: ovn-ovs
      hostNetwork: true
      containers:
        - name: ovn-central
@@ -236,7 +357,9 @@ spec:
          command: ["/kube-ovn/start-db.sh"]
          securityContext:
            capabilities:
-              add: ["SYS_NICE"]
+              add:
+                - NET_BIND_SERVICE
+                - SYS_NICE
          env:
            - name: ENABLE_SSL
              value: "{{ kube_ovn_enable_ssl | lower }}"
@@ -260,6 +383,12 @@ spec:
                  fieldPath: status.podIPs
            - name: ENABLE_BIND_LOCAL_IP
              value: "{{ kube_ovn_bind_local_ip_enabled }}"
+            - name: PROBE_INTERVAL
+              value: "180000"
+            - name: OVN_NORTHD_PROBE_INTERVAL
+              value: "5000"
+            - name: OVN_LEADER_PROBE_INTERVAL
+              value: "5"
          resources:
            requests:
              cpu: {{ kube_ovn_db_cpu_request }}
@@ -349,7 +478,10 @@ spec:
    matchLabels:
      app: ovs
  updateStrategy:
-    type: OnDelete
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 0
  template:
    metadata:
      labels:
@@ -358,9 +490,14 @@ spec:
        type: infra
    spec:
      tolerations:
-      - operator: Exists
+        - effect: NoSchedule
+          operator: Exists
+        - effect: NoExecute
+          operator: Exists
+        - key: CriticalAddonsOnly
+          operator: Exists
      priorityClassName: system-node-critical
-      serviceAccountName: ovn
+      serviceAccountName: ovn-ovs
      hostNetwork: true
      hostPID: true
      containers:
@@ -371,7 +508,13 @@ spec:
          command: [{% if kube_ovn_dpdk_enabled %}"/kube-ovn/start-ovs-dpdk.sh"{% else %}"/kube-ovn/start-ovs.sh"{% endif %}]
          securityContext:
            runAsUser: 0
-            privileged: true
+            privileged: false
+            capabilities:
+              add:
+                - NET_ADMIN
+                - NET_BIND_SERVICE
+                - SYS_MODULE
+                - SYS_NICE
          env:
            - name: ENABLE_SSL
              value: "{{ kube_ovn_enable_ssl | lower }}"
@@ -379,6 +522,14 @@ spec:
              valueFrom:
                fieldRef:
                  fieldPath: status.podIP
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
 {% if not kube_ovn_dpdk_enabled %}
            - name: HW_OFFLOAD
              value: "{{ kube_ovn_hw_offload | string | lower }}"
@@ -425,6 +576,9 @@ spec:
              name: localtime
            - mountPath: /var/run/tls
              name: kube-ovn-tls
+            - mountPath: /var/run/containerd
+              name: cruntime
+              readOnly: true
          readinessProbe:
            exec:
              command:
@@ -511,6 +665,9 @@ spec:
        - name: localtime
          hostPath:
            path: /etc/localtime
+        - name: cruntime
+          hostPath:
+            path: /var/run/containerd
        - name: kube-ovn-tls
          secret:
            optional: true
--- a/kubespray/project/roles/network_plugin/kube-router/tasks/main.yml
+++ b/kubespray/project/roles/network_plugin/kube-router/tasks/main.yml
@@ -9,13 +9,13 @@
    state: directory
    owner: "{{ kube_owner }}"
    recurse: true
-    mode: 0755
+    mode: "0755"

 - name: Kube-router | Create kubeconfig
  template:
    src: kubeconfig.yml.j2
    dest: /var/lib/kube-router/kubeconfig
-    mode: 0644
+    mode: "0644"
    owner: "{{ kube_owner }}"
  notify:
    - Reset_kube_router
@@ -43,7 +43,7 @@
  template:
    src: cni-conf.json.j2
    dest: /etc/cni/net.d/10-kuberouter.conflist
-    mode: 0644
+    mode: "0644"
    owner: "{{ kube_owner }}"
  notify:
    - Reset_kube_router
@@ -57,6 +57,6 @@
  template:
    src: kube-router.yml.j2
    dest: "{{ kube_config_dir }}/kube-router.yml"
-    mode: 0644
+    mode: "0644"
  delegate_to: "{{ groups['kube_control_plane'] | first }}"
  run_once: true
--- a/kubespray/project/roles/network_plugin/kube-router/tasks/reset.yml
+++ b/kubespray/project/roles/network_plugin/kube-router/tasks/reset.yml
@@ -2,9 +2,9 @@
 - name: Reset | check kube-dummy-if network device
  stat:
    path: /sys/class/net/kube-dummy-if
-    get_attributes: no
-    get_checksum: no
-    get_mime: no
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
  register: kube_dummy_if

 - name: Reset | remove the network device created by kube-router
@@ -14,9 +14,9 @@
 - name: Check kube-bridge exists
  stat:
    path: /sys/class/net/kube-bridge
-    get_attributes: no
-    get_checksum: no
-    get_mime: no
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
  register: kube_bridge_if

 - name: Reset | donw the network bridge create by kube-router
--- a/kubespray/project/roles/network_plugin/macvlan/tasks/main.yml
+++ b/kubespray/project/roles/network_plugin/macvlan/tasks/main.yml
@@ -23,7 +23,7 @@
  template:
    src: debian-network-macvlan.cfg.j2
    dest: /etc/network/interfaces.d/60-mac0.cfg
-    mode: 0644
+    mode: "0644"
  notify: Macvlan | restart network
  when: ansible_os_family in ["Debian"]

@@ -53,7 +53,7 @@
    template:
      src: "{{ item.src }}.j2"
      dest: "/etc/sysconfig/network-scripts/{{ item.dst }}"
-      mode: 0644
+      mode: "0644"
    with_items:
    - {src: centos-network-macvlan.cfg, dst: ifcfg-mac0 }
    - {src: centos-routes-macvlan.cfg, dst: route-mac0 }
@@ -67,7 +67,7 @@
    template:
      src: coreos-service-nat_ouside.j2
      dest: /etc/systemd/system/enable_nat_ouside.service
-      mode: 0644
+      mode: "0644"
    when: enable_nat_default_gateway

  - name: Macvlan | Enable service nat via gateway on Flatcar Container Linux
@@ -81,7 +81,7 @@
    template:
      src: "{{ item.src }}.j2"
      dest: "/etc/systemd/network/{{ item.dst }}"
-      mode: 0644
+      mode: "0644"
    with_items:
    - {src: coreos-device-macvlan.cfg, dst: macvlan.netdev }
    - {src: coreos-interface-macvlan.cfg, dst: output.network }
@@ -92,19 +92,19 @@
  template:
    src: 10-macvlan.conf.j2
    dest: /etc/cni/net.d/10-macvlan.conf
-    mode: 0644
+    mode: "0644"

 - name: Macvlan | Install loopback definition for Macvlan
  template:
    src: 99-loopback.conf.j2
    dest: /etc/cni/net.d/99-loopback.conf
-    mode: 0644
+    mode: "0644"

 - name: Enable net.ipv4.conf.all.arp_notify in sysctl
  ansible.posix.sysctl:
    name: net.ipv4.conf.all.arp_notify
    value: 1
-    sysctl_set: yes
+    sysctl_set: true
    sysctl_file: "{{ sysctl_file_path }}"
    state: present
-    reload: yes
+    reload: true
--- a/kubespray/project/roles/network_plugin/multus/tasks/main.yml
+++ b/kubespray/project/roles/network_plugin/multus/tasks/main.yml
@@ -3,7 +3,7 @@
  copy:
    src: "{{ item.file }}"
    dest: "{{ kube_config_dir }}"
-    mode: 0644
+    mode: "0644"
  with_items:
    - {name: multus-crd, file: multus-crd.yml, type: customresourcedefinition}
    - {name: multus-serviceaccount, file: multus-serviceaccount.yml, type: serviceaccount}
@@ -20,7 +20,7 @@
  template:
    src: multus-daemonset.yml.j2
    dest: "{{ kube_config_dir }}/{{ item.file }}"
-    mode: 0644
+    mode: "0644"
  with_items:
    - {name: multus-daemonset-containerd, file: multus-daemonset-containerd.yml, type: daemonset, engine: containerd }
    - {name: multus-daemonset-docker, file: multus-daemonset-docker.yml, type: daemonset, engine: docker }
--- a/kubespray/project/roles/network_plugin/ovn4nfv/tasks/main.yml
+++ b/kubespray/project/roles/network_plugin/ovn4nfv/tasks/main.yml
@@ -9,7 +9,7 @@
  template:
    src: "{{ item.file }}.j2"
    dest: "{{ kube_config_dir }}/{{ item.file }}"
-    mode: 0644
+    mode: "0644"
  with_items:
    - {name: ovn-daemonset, file: ovn-daemonset.yml}
    - {name: ovn4nfv-k8s-plugin, file: ovn4nfv-k8s-plugin.yml}
--- a/kubespray/project/roles/network_plugin/weave/tasks/main.yml
+++ b/kubespray/project/roles/network_plugin/weave/tasks/main.yml
@@ -3,10 +3,10 @@
  template:
    src: weave-net.yml.j2
    dest: "{{ kube_config_dir }}/weave-net.yml"
-    mode: 0644
+    mode: "0644"

 - name: Weave | Fix nodePort for Weave
  template:
    src: 10-weave.conflist.j2
    dest: /etc/cni/net.d/10-weave.conflist
-    mode: 0644
+    mode: "0644"