kube-forge v2: embede python with ansible (kubespray) and cobra as cli building tool

internal/resources/kubespray/project/roles/network_plugin/calico/files/openssl.conf

@@ -0,0 +1,27 @@
+req_extensions = v3_req
+distinguished_name = req_distinguished_name
+
+[req_distinguished_name]
+
+[ v3_req ]
+basicConstraints = CA:FALSE
+keyUsage = digitalSignature, keyEncipherment
+
+[ ssl_client ]
+extendedKeyUsage = clientAuth, serverAuth
+basicConstraints = CA:FALSE
+subjectKeyIdentifier=hash
+authorityKeyIdentifier=keyid,issuer
+
+[ v3_ca ]
+basicConstraints = CA:TRUE
+keyUsage = cRLSign, digitalSignature, keyCertSign
+subjectKeyIdentifier=hash
+authorityKeyIdentifier=keyid:always,issuer
+
+[ ssl_client_apiserver ]
+extendedKeyUsage = clientAuth, serverAuth
+basicConstraints = CA:FALSE
+subjectKeyIdentifier=hash
+authorityKeyIdentifier=keyid,issuer
+subjectAltName = DNS:calico-api.calico-apiserver.svc

internal/resources/kubespray/project/roles/network_plugin/calico/handlers/main.yml

@@ -0,0 +1,31 @@
+---
+- name: Delete 10-calico.conflist
+  file:
+    path: /etc/cni/net.d/10-calico.conflist
+    state: absent
+  listen: Reset_calico_cni
+  when: calico_cni_config is defined
+
+- name: Calico | delete calico-node docker containers
+  shell: "set -o pipefail && {{ docker_bin_dir }}/docker ps -af name=k8s_POD_calico-node* -q | xargs --no-run-if-empty {{ docker_bin_dir }}/docker rm -f"
+  args:
+    executable: /bin/bash
+  register: docker_calico_node_remove
+  until: docker_calico_node_remove is succeeded
+  retries: 5
+  when:
+    - container_manager in ["docker"]
+    - calico_cni_config is defined
+  listen: Reset_calico_cni
+
+- name: Calico | delete calico-node crio/containerd containers
+  shell: 'set -o pipefail && {{ bin_dir }}/crictl pods --name calico-node-* -q | xargs -I% --no-run-if-empty bash -c "{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %"'
+  args:
+    executable: /bin/bash
+  register: crictl_calico_node_remove
+  until: crictl_calico_node_remove is succeeded
+  retries: 5
+  when:
+    - container_manager in ["crio", "containerd"]
+    - calico_cni_config is defined
+  listen: Reset_calico_cni

internal/resources/kubespray/project/roles/network_plugin/calico/meta/main.yml

@@ -0,0 +1,3 @@
+---
+dependencies:
+  - role: network_plugin/calico_defaults

internal/resources/kubespray/project/roles/network_plugin/calico/rr/defaults/main.yml

@@ -0,0 +1,5 @@
+---
+# Global as_num (/calico/bgp/v1/global/as_num)
+# should be the same as in calico role
+global_as_num: "64512"
+calico_baremetal_nodename: "{{ kube_override_hostname | default(inventory_hostname) }}"

internal/resources/kubespray/project/roles/network_plugin/calico/rr/tasks/main.yml

@@ -0,0 +1,16 @@
+---
+- name: Calico-rr | Pre-upgrade tasks
+  include_tasks: pre.yml
+
+- name: Calico-rr | Configuring node tasks
+  include_tasks: update-node.yml
+
+- name: Calico-rr | Set label for route reflector
+  command: >-
+    {{ bin_dir }}/calicoctl.sh label node {{ inventory_hostname }}
+    'i-am-a-route-reflector=true' --overwrite
+  changed_when: false
+  register: calico_rr_label
+  until: calico_rr_label is succeeded
+  delay: "{{ retry_stagger | random + 3 }}"
+  retries: 10

internal/resources/kubespray/project/roles/network_plugin/calico/rr/tasks/pre.yml

@@ -0,0 +1,15 @@
+---
+- name: Calico-rr | Disable calico-rr service if it exists
+  service:
+    name: calico-rr
+    state: stopped
+    enabled: false
+  failed_when: false
+
+- name: Calico-rr | Delete obsolete files
+  file:
+    path: "{{ item }}"
+    state: absent
+  with_items:
+    - /etc/calico/calico-rr.env
+    - /etc/systemd/system/calico-rr.service

internal/resources/kubespray/project/roles/network_plugin/calico/rr/tasks/update-node.yml

@@ -0,0 +1,50 @@
+---
+# Workaround to retry a block of tasks, ansible doesn't have a direct way to do it,
+# you can follow the block loop request in: https://github.com/ansible/ansible/issues/46203
+- name: Calico-rr | Configure route reflector
+  block:
+  - name: Set the retry count
+    set_fact:
+      retry_count: "{{ 0 if retry_count is undefined else retry_count | int + 1 }}"
+
+  - name: Calico | Set label for route reflector  # noqa command-instead-of-shell
+    shell: "{{ bin_dir }}/calicoctl.sh label node  {{ inventory_hostname }} calico-rr-id={{ calico_rr_id }} --overwrite"
+    changed_when: false
+    register: calico_rr_id_label
+    until: calico_rr_id_label is succeeded
+    delay: "{{ retry_stagger | random + 3 }}"
+    retries: 10
+    when: calico_rr_id is defined
+
+  - name: Calico-rr | Fetch current node object
+    command: "{{ bin_dir }}/calicoctl.sh get node {{ inventory_hostname }} -ojson"
+    changed_when: false
+    register: calico_rr_node
+    until: calico_rr_node is succeeded
+    delay: "{{ retry_stagger | random + 3 }}"
+    retries: 10
+
+  - name: Calico-rr | Set route reflector cluster ID
+    # noqa: jinja[spacing]
+    set_fact:
+      calico_rr_node_patched: >-
+        {{ calico_rr_node.stdout | from_json | combine({ 'spec': { 'bgp':
+        { 'routeReflectorClusterID': cluster_id }}}, recursive=True) }}
+
+  - name: Calico-rr | Configure route reflector  # noqa command-instead-of-shell
+    shell: "{{ bin_dir }}/calicoctl.sh replace -f-"
+    args:
+      stdin: "{{ calico_rr_node_patched | to_json }}"
+
+  rescue:
+  - name: Fail if retry limit is reached
+    fail:
+      msg: Ended after 10 retries
+    when: retry_count | int == 10
+
+  - name: Retrying node configuration
+    debug:
+      msg: "Failed to configure route reflector - Retrying..."
+
+  - name: Retry node configuration
+    include_tasks: update-node.yml

internal/resources/kubespray/project/roles/network_plugin/calico/tasks/calico_apiserver_certs.yml

@@ -0,0 +1,60 @@
+---
+- name: Calico | Check if calico apiserver exists
+  command: "{{ kubectl }} -n calico-apiserver get secret calico-apiserver-certs"
+  register: calico_apiserver_secret
+  changed_when: false
+  failed_when: false
+
+- name: Calico | Create ns manifests
+  template:
+    src: "calico-apiserver-ns.yml.j2"
+    dest: "{{ kube_config_dir }}/calico-apiserver-ns.yml"
+    mode: "0644"
+
+- name: Calico | Apply ns manifests
+  kube:
+    kubectl: "{{ bin_dir }}/kubectl"
+    filename: "{{ kube_config_dir }}/calico-apiserver-ns.yml"
+    state: "latest"
+
+- name: Calico | Ensure calico certs dir
+  file:
+    path: /etc/calico/certs
+    state: directory
+    mode: "0755"
+  when: calico_apiserver_secret.rc != 0
+
+- name: Calico | Copy ssl script for apiserver certs
+  template:
+    src: make-ssl-calico.sh.j2
+    dest: "{{ bin_dir }}/make-ssl-apiserver.sh"
+    mode: "0755"
+  when: calico_apiserver_secret.rc != 0
+
+- name: Calico | Copy ssl config for apiserver certs
+  copy:
+    src: openssl.conf
+    dest: /etc/calico/certs/openssl.conf
+    mode: "0644"
+  when: calico_apiserver_secret.rc != 0
+
+- name: Calico | Generate apiserver certs
+  command: >-
+    {{ bin_dir }}/make-ssl-apiserver.sh
+    -f /etc/calico/certs/openssl.conf
+    -c {{ kube_cert_dir }}
+    -d /etc/calico/certs
+    -s apiserver
+  when: calico_apiserver_secret.rc != 0
+
+- name: Calico | Create calico apiserver generic secrets
+  command: >-
+    {{ kubectl }} -n calico-apiserver
+    create secret generic {{ item.name }}
+    --from-file={{ item.cert }}
+    --from-file={{ item.key }}
+  with_items:
+    - name: calico-apiserver-certs
+      cert: /etc/calico/certs/apiserver.crt
+      key: /etc/calico/certs/apiserver.key
+  when: calico_apiserver_secret.rc != 0

internal/resources/kubespray/project/roles/network_plugin/calico/tasks/check.yml

@@ -0,0 +1,235 @@
+---
+- name: Stop if legacy encapsulation variables are detected (ipip)
+  assert:
+    that:
+      - ipip is not defined
+    msg: "'ipip' configuration variable is deprecated, please configure your inventory with 'calico_ipip_mode' set to 'Always' or 'CrossSubnet' according to your specific needs"
+  run_once: true
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+
+- name: Stop if legacy encapsulation variables are detected (ipip_mode)
+  assert:
+    that:
+      - ipip_mode is not defined
+    msg: "'ipip_mode' configuration variable is deprecated, please configure your inventory with 'calico_ipip_mode' set to 'Always' or 'CrossSubnet' according to your specific needs"
+  run_once: true
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+
+- name: Stop if legacy encapsulation variables are detected (calcio_ipam_autoallocateblocks)
+  assert:
+    that:
+      - calcio_ipam_autoallocateblocks is not defined
+    msg: "'calcio_ipam_autoallocateblocks' configuration variable is deprecated, it's a typo, please configure your inventory with 'calico_ipam_autoallocateblocks' set to 'true' or 'false' according to your specific needs"
+  run_once: true
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+
+
+- name: Stop if supported Calico versions
+  assert:
+    that:
+      - "calico_version in calico_crds_archive_checksums.no_arch.keys()"
+    msg: "Calico version not supported {{ calico_version }} not in {{ calico_crds_archive_checksums.no_arch.keys() }}"
+  run_once: true
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+
+- name: Check if calicoctl.sh exists
+  stat:
+    path: "{{ bin_dir }}/calicoctl.sh"
+  register: calicoctl_sh_exists
+  run_once: true
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+
+- name: Check if calico ready
+  command: "{{ bin_dir }}/calicoctl.sh get ClusterInformation default"
+  register: calico_ready
+  run_once: true
+  ignore_errors: true
+  retries: 5
+  delay: 10
+  until: calico_ready.rc == 0
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+  when: calicoctl_sh_exists.stat.exists
+
+- name: Check that current calico version is enough for upgrade
+  run_once: true
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+  when: calicoctl_sh_exists.stat.exists and calico_ready.rc == 0
+  block:
+    - name: Get current calico version
+      shell: "set -o pipefail && {{ bin_dir }}/calicoctl.sh version | grep 'Client Version:' | awk '{ print $3}'"
+      args:
+        executable: /bin/bash
+      register: calico_version_on_server
+      changed_when: false
+
+    - name: Assert that current calico version is enough for upgrade
+      assert:
+        that:
+          - calico_version_on_server.stdout.removeprefix('v') is version(calico_min_version_required, '>=')
+        msg: >
+          Your version of calico is not fresh enough for upgrade.
+          Minimum version is {{ calico_min_version_required }} supported by the previous kubespray release.
+          But current version is {{ calico_version_on_server.stdout }}.
+
+- name: "Check that cluster_id is set and a valid IPv4 address if calico_rr enabled"
+  assert:
+    that:
+      - cluster_id is defined
+      - cluster_id is ansible.utils.ipv4
+    msg: "A unique cluster_id is required if using calico_rr, and it must be a valid IPv4 address"
+  when:
+    - peer_with_calico_rr
+    - inventory_hostname == groups['kube_control_plane'][0]
+  run_once: true
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+
+- name: "Check that calico_rr nodes are in k8s_cluster group"
+  assert:
+    that:
+      - '"k8s_cluster" in group_names'
+    msg: "calico_rr must be a child group of k8s_cluster group"
+  when:
+    - '"calico_rr" in group_names'
+  run_once: true
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+
+- name: "Check vars defined correctly"
+  assert:
+    that:
+      - "calico_pool_name is defined"
+      - "calico_pool_name is match('^[a-zA-Z0-9-_\\\\.]{2,63}$')"
+    msg: "calico_pool_name contains invalid characters"
+  run_once: true
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+
+- name: "Check calico network backend defined correctly"
+  assert:
+    that:
+      - "calico_network_backend in ['bird', 'vxlan', 'none']"
+    msg: "calico network backend is not 'bird', 'vxlan' or 'none'"
+  run_once: true
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+
+- name: "Check ipip and vxlan mode defined correctly"
+  run_once: true
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+  assert:
+    that:
+      - "calico_ipip_mode in ['Always', 'CrossSubnet', 'Never']"
+      - "calico_vxlan_mode in ['Always', 'CrossSubnet', 'Never']"
+    msg: "calico inter host encapsulation mode is not 'Always', 'CrossSubnet' or 'Never'"
+
+- name: "Check ipip and vxlan mode if simultaneously enabled"
+  assert:
+    that:
+      - "calico_vxlan_mode in ['Never']"
+    msg: "IP in IP and VXLAN mode is mutualy exclusive modes"
+  when:
+    - "calico_ipip_mode in ['Always', 'CrossSubnet']"
+  run_once: true
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+
+- name: "Check ipip and vxlan mode if simultaneously enabled"
+  assert:
+    that:
+      - "calico_ipip_mode in ['Never']"
+    msg: "IP in IP and VXLAN mode is mutualy exclusive modes"
+  when:
+    - "calico_vxlan_mode in ['Always', 'CrossSubnet']"
+  run_once: true
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+
+- name: "Get Calico {{ calico_pool_name }} configuration"
+  command: "{{ bin_dir }}/calicoctl.sh get ipPool {{ calico_pool_name }} -o json"
+  failed_when: false
+  changed_when: false
+  check_mode: false
+  register: calico
+  run_once: true
+  when: ipv4_stack | bool
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+
+- name: "Set calico_pool_conf"
+  set_fact:
+    calico_pool_conf: '{{ calico.stdout | from_json }}'
+  when:
+    - ipv4_stack | bool
+    - calico is defined
+    - calico.rc == 0 and calico.stdout
+  run_once: true
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+
+- name: "Check if inventory match current cluster configuration"
+  assert:
+    that:
+      - calico_pool_conf.spec.blockSize | int == calico_pool_blocksize | int
+      - calico_pool_conf.spec.cidr == (calico_pool_cidr | default(kube_pods_subnet))
+      - not calico_pool_conf.spec.ipipMode is defined or calico_pool_conf.spec.ipipMode == calico_ipip_mode
+      - not calico_pool_conf.spec.vxlanMode is defined or calico_pool_conf.spec.vxlanMode == calico_vxlan_mode
+    msg: "Your inventory doesn't match the current cluster configuration"
+  when:
+    - ipv4_stack | bool
+    - calico_pool_conf is defined
+  run_once: true
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+
+- name: "Get Calico {{ calico_pool_name }}-ipv6 configuration"
+  command: "{{ bin_dir }}/calicoctl.sh get ipPool {{ calico_pool_name }}-ipv6 -o json"
+  failed_when: false
+  changed_when: false
+  check_mode: false
+  register: calico_ipv6
+  run_once: true
+  when: ipv6_stack | bool
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+
+- name: "Set calico_pool_ipv6_conf"
+  set_fact:
+    calico_pool_conf: '{{ calico_ipv6.stdout | from_json }}'
+  when:
+    - ipv6_stack | bool
+    - alico_ipv6 is defined
+    - calico_ipv6.rc == 0 and calico_ipv6.stdout
+  run_once: true
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+
+- name: "Check if ipv6 inventory match current cluster configuration"
+  assert:
+    that:
+      - calico_pool_conf.spec.blockSize | int == calico_pool_blocksize_ipv6 | int
+      - calico_pool_conf.spec.cidr == (calico_pool_cidr_ipv6 | default(kube_pods_subnet_ipv6))
+      - not calico_pool_conf.spec.ipipMode is defined or calico_pool_conf.spec.ipipMode == calico_ipip_mode_ipv6
+      - not calico_pool_conf.spec.vxlanMode is defined or calico_pool_conf.spec.vxlanMode == calico_vxlan_mode_ipv6
+    msg: "Your ipv6 inventory doesn't match the current cluster configuration"
+  when:
+    - ipv6_stack | bool
+    - calico_pool_ipv6_conf is defined
+  run_once: true
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+
+- name: "Check kdd calico_datastore if calico_apiserver_enabled"
+  assert:
+    that: calico_datastore == "kdd"
+    msg: "When using calico apiserver you need to use the kubernetes datastore"
+  when:
+    - calico_apiserver_enabled
+  run_once: true
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+
+- name: "Check kdd calico_datastore if typha_enabled"
+  assert:
+    that: calico_datastore == "kdd"
+    msg: "When using typha you need to use the kubernetes datastore"
+  when:
+    - typha_enabled
+  run_once: true
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+
+- name: "Check ipip mode is Never for calico ipv6"
+  assert:
+    that:
+      - "calico_ipip_mode_ipv6 in ['Never']"
+    msg: "Calico doesn't support ipip tunneling for the IPv6"
+  when: ipv6_stack | bool
+  run_once: true
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"

internal/resources/kubespray/project/roles/network_plugin/calico/tasks/install.yml

@@ -0,0 +1,510 @@
+---
+- name: Calico | Install Wireguard packages
+  package:
+    name: "{{ item }}"
+    state: present
+  with_items: "{{ calico_wireguard_packages }}"
+  register: calico_package_install
+  until: calico_package_install is succeeded
+  retries: 4
+  when: calico_wireguard_enabled
+
+- name: Calico | Copy calicoctl binary from download dir
+  copy:
+    src: "{{ downloads.calicoctl.dest }}"
+    dest: "{{ bin_dir }}/calicoctl"
+    mode: "0755"
+    remote_src: true
+
+- name: Calico | Create calico certs directory
+  file:
+    dest: "{{ calico_cert_dir }}"
+    state: directory
+    mode: "0750"
+    owner: root
+    group: root
+  when: calico_datastore == "etcd"
+
+- name: Calico | Link etcd certificates for calico-node
+  file:
+    src: "{{ etcd_cert_dir }}/{{ item.s }}"
+    dest: "{{ calico_cert_dir }}/{{ item.d }}"
+    state: hard
+    mode: "0640"
+    force: true
+  with_items:
+    - {s: "{{ kube_etcd_cacert_file }}", d: "ca_cert.crt"}
+    - {s: "{{ kube_etcd_cert_file }}", d: "cert.crt"}
+    - {s: "{{ kube_etcd_key_file }}", d: "key.pem"}
+  when: calico_datastore == "etcd"
+
+- name: Calico | Generate typha certs
+  include_tasks: typha_certs.yml
+  when:
+    - typha_secure
+    - inventory_hostname == groups['kube_control_plane'][0]
+
+- name: Calico | Generate apiserver certs
+  include_tasks: calico_apiserver_certs.yml
+  when:
+    - calico_apiserver_enabled
+    - inventory_hostname == groups['kube_control_plane'][0]
+
+- name: Calico | Install calicoctl wrapper script
+  template:
+    src: "calicoctl.{{ calico_datastore }}.sh.j2"
+    dest: "{{ bin_dir }}/calicoctl.sh"
+    mode: "0755"
+    owner: root
+    group: root
+
+- name: Calico | wait for etcd
+  uri:
+    url: "{{ etcd_access_addresses.split(',') | first }}/health"
+    validate_certs: false
+    client_cert: "{{ calico_cert_dir }}/cert.crt"
+    client_key: "{{ calico_cert_dir }}/key.pem"
+  register: result
+  until: result.status == 200 or result.status == 401
+  retries: 10
+  delay: 5
+  run_once: true
+  when: calico_datastore == "etcd"
+
+- name: Calico | Check if calico network pool has already been configured
+  # noqa risky-shell-pipe - grep will exit 1 if no match found
+  shell: >
+    {{ bin_dir }}/calicoctl.sh get ippool | grep -w "{{ calico_pool_cidr | default(kube_pods_subnet) }}" | wc -l
+  args:
+    executable: /bin/bash
+  register: calico_conf
+  retries: 4
+  until: calico_conf.rc == 0
+  delay: "{{ retry_stagger | random + 3 }}"
+  changed_when: false
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - ipv4_stack | bool
+
+- name: Calico | Ensure that calico_pool_cidr is within kube_pods_subnet when defined
+  assert:
+    that: "[calico_pool_cidr] | ansible.utils.ipaddr(kube_pods_subnet) | length == 1"
+    msg: "{{ calico_pool_cidr }} is not within or equal to {{ kube_pods_subnet }}"
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - ipv4_stack | bool
+    - calico_pool_cidr is defined
+    - 'calico_conf.stdout == "0"'
+
+- name: Calico | Check if calico IPv6 network pool has already been configured
+  # noqa risky-shell-pipe - grep will exit 1 if no match found
+  shell: >
+    {{ bin_dir }}/calicoctl.sh get ippool | grep -w "{{ calico_pool_cidr_ipv6 | default(kube_pods_subnet_ipv6) }}" | wc -l
+  args:
+    executable: /bin/bash
+  register: calico_conf_ipv6
+  retries: 4
+  until: calico_conf_ipv6.rc == 0
+  delay: "{{ retry_stagger | random + 3 }}"
+  changed_when: false
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - ipv6_stack
+
+- name: Calico | Ensure that calico_pool_cidr_ipv6 is within kube_pods_subnet_ipv6 when defined
+  assert:
+    that: "[calico_pool_cidr_ipv6] | ansible.utils.ipaddr(kube_pods_subnet_ipv6) | length == 1"
+    msg: "{{ calico_pool_cidr_ipv6 }} is not within or equal to {{ kube_pods_subnet_ipv6 }}"
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - ipv6_stack | bool
+    - calico_conf_ipv6.stdout is defined and calico_conf_ipv6.stdout == "0"
+    - calico_pool_cidr_ipv6 is defined
+
+- name: Calico | kdd specific configuration
+  when:
+    - ('kube_control_plane' in group_names)
+    - calico_datastore == "kdd"
+  block:
+    - name: Calico | Check if extra directory is needed
+      stat:
+        path: "{{ local_release_dir }}/calico-{{ calico_version }}-kdd-crds/{{ 'kdd' if (calico_version is version('3.22.3', '<')) else 'crd' }}"
+      register: kdd_path
+    - name: Calico | Set kdd path when calico < v3.22.3
+      set_fact:
+        calico_kdd_path: "{{ local_release_dir }}/calico-{{ calico_version }}-kdd-crds{{ '/kdd' if kdd_path.stat.exists is defined and kdd_path.stat.exists }}"
+      when:
+        - calico_version is version('3.22.3', '<')
+    - name: Calico | Set kdd path when calico > 3.22.2
+      set_fact:
+        calico_kdd_path: "{{ local_release_dir }}/calico-{{ calico_version }}-kdd-crds{{ '/crd' if kdd_path.stat.exists is defined and kdd_path.stat.exists }}"
+      when:
+        - calico_version is version('3.22.2', '>')
+    - name: Calico | Create calico manifests for kdd
+      assemble:
+        src: "{{ calico_kdd_path }}"
+        dest: "{{ kube_config_dir }}/kdd-crds.yml"
+        mode: "0644"
+        delimiter: "---\n"
+        regexp: ".*\\.yaml"
+        remote_src: true
+
+    - name: Calico | Create Calico Kubernetes datastore resources
+      kube:
+        kubectl: "{{ bin_dir }}/kubectl"
+        filename: "{{ kube_config_dir }}/kdd-crds.yml"
+        state: "latest"
+      register: kubectl_result
+      until: kubectl_result is succeeded
+      retries: 5
+      when:
+        - inventory_hostname == groups['kube_control_plane'][0]
+
+- name: Calico | Configure Felix
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+  block:
+    - name: Calico | Get existing FelixConfiguration
+      command: "{{ bin_dir }}/calicoctl.sh get felixconfig default -o json"
+      register: _felix_cmd
+      ignore_errors: true
+      changed_when: false
+
+    - name: Calico | Set kubespray FelixConfiguration
+      set_fact:
+        _felix_config: >
+          {
+            "kind": "FelixConfiguration",
+            "apiVersion": "projectcalico.org/v3",
+            "metadata": {
+              "name": "default",
+            },
+            "spec": {
+              "ipipEnabled": {{ calico_ipip_mode != 'Never' }},
+              "reportingInterval": "{{ calico_felix_reporting_interval }}",
+              "bpfLogLevel": "{{ calico_bpf_log_level }}",
+              "bpfEnabled": {{ calico_bpf_enabled | bool }},
+              "bpfExternalServiceMode": "{{ calico_bpf_service_mode }}",
+              "wireguardEnabled": {{ calico_wireguard_enabled | bool }},
+              "logSeverityScreen": "{{ calico_felix_log_severity_screen }}",
+              "vxlanEnabled": {{ calico_vxlan_mode != 'Never' }},
+              "featureDetectOverride": "{{ calico_feature_detect_override }}",
+              "floatingIPs": "{{ calico_felix_floating_ips }}"
+            }
+          }
+
+    - name: Calico | Process FelixConfiguration
+      set_fact:
+        _felix_config: "{{ _felix_cmd.stdout | from_json | combine(_felix_config, recursive=True) }}"
+      when:
+        - _felix_cmd is success
+
+    - name: Calico | Configure calico FelixConfiguration
+      command:
+        cmd: "{{ bin_dir }}/calicoctl.sh apply -f -"
+        stdin: "{{ _felix_config is string | ternary(_felix_config, _felix_config | to_json) }}"
+      changed_when: false
+
+- name: Calico | Configure Calico IP Pool
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - ipv4_stack | bool
+  block:
+    - name: Calico | Get existing calico network pool
+      command: "{{ bin_dir }}/calicoctl.sh get ippool {{ calico_pool_name }} -o json"
+      register: _calico_pool_cmd
+      ignore_errors: true
+      changed_when: false
+
+    - name: Calico | Set kubespray calico network pool
+      set_fact:
+        _calico_pool: >
+          {
+            "kind": "IPPool",
+            "apiVersion": "projectcalico.org/v3",
+            "metadata": {
+              "name": "{{ calico_pool_name }}",
+            },
+            "spec": {
+              "blockSize": {{ calico_pool_blocksize }},
+              "cidr": "{{ calico_pool_cidr | default(kube_pods_subnet) }}",
+              "ipipMode": "{{ calico_ipip_mode }}",
+              "vxlanMode": "{{ calico_vxlan_mode }}",
+              "natOutgoing": {{ nat_outgoing | default(false) }}
+            }
+          }
+
+    - name: Calico | Process calico network pool
+      when:
+        - _calico_pool_cmd is success
+      block:
+        - name: Calico | Get current calico network pool blocksize
+          set_fact:
+            _calico_blocksize: >
+              {
+                "spec": {
+                  "blockSize": {{ (_calico_pool_cmd.stdout | from_json).spec.blockSize }}
+                }
+              }
+        - name: Calico | Merge calico network pool
+          set_fact:
+            _calico_pool: "{{ _calico_pool_cmd.stdout | from_json | combine(_calico_pool, _calico_blocksize, recursive=True) }}"
+
+    - name: Calico | Configure calico network pool
+      command:
+        cmd: "{{ bin_dir }}/calicoctl.sh apply -f -"
+        stdin: "{{ _calico_pool is string | ternary(_calico_pool, _calico_pool | to_json) }}"
+      changed_when: false
+
+- name: Calico | Configure Calico IPv6 Pool
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - ipv6_stack | bool
+  block:
+    - name: Calico | Get existing calico ipv6 network pool
+      command: "{{ bin_dir }}/calicoctl.sh get ippool {{ calico_pool_name }}-ipv6 -o json"
+      register: _calico_pool_ipv6_cmd
+      ignore_errors: true
+      changed_when: false
+
+    - name: Calico | Set kubespray calico network pool
+      set_fact:
+        _calico_pool_ipv6: >
+          {
+            "kind": "IPPool",
+            "apiVersion": "projectcalico.org/v3",
+            "metadata": {
+              "name": "{{ calico_pool_name }}-ipv6",
+            },
+            "spec": {
+              "blockSize": {{ calico_pool_blocksize_ipv6 }},
+              "cidr": "{{ calico_pool_cidr_ipv6 | default(kube_pods_subnet_ipv6) }}",
+              "ipipMode": "{{ calico_ipip_mode_ipv6 }}",
+              "vxlanMode": "{{ calico_vxlan_mode_ipv6 }}",
+              "natOutgoing": {{ nat_outgoing_ipv6 | default(false) }}
+            }
+          }
+
+    - name: Calico | Process calico ipv6 network pool
+      when:
+        - _calico_pool_ipv6_cmd is success
+      block:
+        - name: Calico | Get current calico ipv6 network pool blocksize
+          set_fact:
+            _calico_blocksize_ipv6: >
+              {
+                "spec": {
+                  "blockSize": {{ (_calico_pool_ipv6_cmd.stdout | from_json).spec.blockSize }}
+                }
+              }
+        - name: Calico | Merge calico ipv6 network pool
+          set_fact:
+            _calico_pool_ipv6: "{{ _calico_pool_ipv6_cmd.stdout | from_json | combine(_calico_pool_ipv6, _calico_blocksize_ipv6, recursive=True) }}"
+
+    - name: Calico | Configure calico ipv6 network pool
+      command:
+        cmd: "{{ bin_dir }}/calicoctl.sh apply -f -"
+        stdin: "{{ _calico_pool_ipv6 is string | ternary(_calico_pool_ipv6, _calico_pool_ipv6 | to_json) }}"
+      changed_when: false
+
+- name: Populate Service External IPs
+  set_fact:
+    _service_external_ips: "{{ _service_external_ips | default([]) + [{'cidr': item}] }}"
+  with_items: "{{ calico_advertise_service_external_ips }}"
+  run_once: true
+
+- name: Populate Service LoadBalancer IPs
+  set_fact:
+    _service_loadbalancer_ips: "{{ _service_loadbalancer_ips | default([]) + [{'cidr': item}] }}"
+  with_items: "{{ calico_advertise_service_loadbalancer_ips }}"
+  run_once: true
+
+- name: "Determine nodeToNodeMesh needed state"
+  set_fact:
+    nodeToNodeMeshEnabled: "false"
+  when:
+    - peer_with_router | default(false) or peer_with_calico_rr | default(false)
+    - ('k8s_cluster' in group_names)
+  run_once: true
+
+- name: Calico | Configure Calico BGP
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+  block:
+    - name: Calico | Get existing BGP Configuration
+      command: "{{ bin_dir }}/calicoctl.sh get bgpconfig default -o json"
+      register: _bgp_config_cmd
+      ignore_errors: true
+      changed_when: false
+
+    - name: Calico | Set kubespray BGP Configuration
+      set_fact:
+        # noqa: jinja[spacing]
+        _bgp_config: >
+          {
+            "kind": "BGPConfiguration",
+            "apiVersion": "projectcalico.org/v3",
+            "metadata": {
+              "name": "default",
+            },
+            "spec": {
+              "listenPort": {{ calico_bgp_listen_port }},
+              "logSeverityScreen": "Info",
+              {% if not calico_no_global_as_num | default(false) %}"asNumber": {{ global_as_num }},{% endif %}
+              "nodeToNodeMeshEnabled": {{ nodeToNodeMeshEnabled | default('true') }} ,
+              {% if calico_advertise_cluster_ips | default(false) %}
+              "serviceClusterIPs": >-
+                {%- if ipv4_stack and ipv6_stack-%}
+                [{"cidr": "{{ kube_service_addresses }}", "cidr": "{{ kube_service_addresses_ipv6 }}"}],
+                {%- elif ipv6_stack-%}
+                [{"cidr": "{{ kube_service_addresses_ipv6 }}"}],
+                {%- else -%}
+                [{"cidr": "{{ kube_service_addresses }}"}],
+                {%- endif -%}
+              {% endif %}
+              {% if calico_advertise_service_loadbalancer_ips | length > 0  %}"serviceLoadBalancerIPs": {{ _service_loadbalancer_ips }},{% endif %}
+              "serviceExternalIPs": {{ _service_external_ips | default([]) }}
+            }
+          }
+
+    - name: Calico | Process BGP Configuration
+      set_fact:
+        _bgp_config: "{{ _bgp_config_cmd.stdout | from_json | combine(_bgp_config, recursive=True) }}"
+      when:
+        - _bgp_config_cmd is success
+
+    - name: Calico | Set up BGP Configuration
+      command:
+        cmd: "{{ bin_dir }}/calicoctl.sh apply -f -"
+        stdin: "{{ _bgp_config is string | ternary(_bgp_config, _bgp_config | to_json) }}"
+      changed_when: false
+
+- name: Calico | Create calico manifests
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.file }}"
+    mode: "0644"
+  with_items:
+    - {name: calico-config, file: calico-config.yml, type: cm}
+    - {name: calico-node, file: calico-node.yml, type: ds}
+    - {name: calico, file: calico-node-sa.yml, type: sa}
+    - {name: calico, file: calico-cr.yml, type: clusterrole}
+    - {name: calico, file: calico-crb.yml, type: clusterrolebinding}
+    - {name: kubernetes-services-endpoint, file: kubernetes-services-endpoint.yml, type: cm }
+  register: calico_node_manifests
+  when:
+    - ('kube_control_plane' in group_names)
+    - rbac_enabled or item.type not in rbac_resources
+
+- name: Calico | Create calico manifests for typha
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.file }}"
+    mode: "0644"
+  with_items:
+    - {name: calico, file: calico-typha.yml, type: typha}
+  register: calico_node_typha_manifest
+  when:
+    - ('kube_control_plane' in group_names)
+    - typha_enabled
+
+- name: Calico | get calico apiserver caBundle
+  command: "{{ bin_dir }}/kubectl get secret -n calico-apiserver calico-apiserver-certs -o jsonpath='{.data.apiserver\\.crt}'"
+  changed_when: false
+  register: calico_apiserver_cabundle
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - calico_apiserver_enabled
+
+- name: Calico | set calico apiserver caBundle fact
+  set_fact:
+    calico_apiserver_cabundle: "{{ calico_apiserver_cabundle.stdout }}"
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - calico_apiserver_enabled
+
+- name: Calico | Create calico manifests for apiserver
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.file }}"
+    mode: "0644"
+  with_items:
+    - {name: calico, file: calico-apiserver.yml, type: calico-apiserver}
+  register: calico_apiserver_manifest
+  when:
+    - ('kube_control_plane' in group_names)
+    - calico_apiserver_enabled
+
+- name: Start Calico resources
+  kube:
+    name: "{{ item.item.name }}"
+    namespace: "kube-system"
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: "{{ item.item.type }}"
+    filename: "{{ kube_config_dir }}/{{ item.item.file }}"
+    state: "latest"
+  with_items:
+    - "{{ calico_node_manifests.results }}"
+    - "{{ calico_node_typha_manifest.results }}"
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - not item is skipped
+  loop_control:
+    label: "{{ item.item.file }}"
+
+- name: Start Calico apiserver resources
+  kube:
+    name: "{{ item.item.name }}"
+    namespace: "calico-apiserver"
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: "{{ item.item.type }}"
+    filename: "{{ kube_config_dir }}/{{ item.item.file }}"
+    state: "latest"
+  with_items:
+    - "{{ calico_apiserver_manifest.results }}"
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - not item is skipped
+  loop_control:
+    label: "{{ item.item.file }}"
+
+- name: Wait for calico kubeconfig to be created
+  wait_for:
+    path: /etc/cni/net.d/calico-kubeconfig
+    timeout: "{{ calico_kubeconfig_wait_timeout }}"
+  when:
+    - inventory_hostname not in groups['kube_control_plane']
+    - calico_datastore == "kdd"
+
+- name: Calico | Create Calico ipam manifests
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.file }}"
+    mode: "0644"
+  with_items:
+    - {name: calico, file: calico-ipamconfig.yml, type: ipam}
+  when:
+    - ('kube_control_plane' in group_names)
+    - calico_datastore == "kdd"
+
+- name: Calico | Create ipamconfig resources
+  kube:
+    kubectl: "{{ bin_dir }}/kubectl"
+    filename: "{{ kube_config_dir }}/calico-ipamconfig.yml"
+    state: "latest"
+  register: resource_result
+  until: resource_result is succeeded
+  retries: 4
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - calico_datastore == "kdd"
+
+- name: Calico | Peer with Calico Route Reflector
+  include_tasks: peer_with_calico_rr.yml
+  when:
+    - peer_with_calico_rr | default(false)
+
+- name: Calico | Peer with the router
+  include_tasks: peer_with_router.yml
+  when:
+    - peer_with_router | default(false)

internal/resources/kubespray/project/roles/network_plugin/calico/tasks/main.yml

@@ -0,0 +1,9 @@
+---
+- name: Calico Pre tasks
+  import_tasks: pre.yml
+
+- name: Calico repos
+  import_tasks: repos.yml
+
+- name: Calico install
+  include_tasks: install.yml

internal/resources/kubespray/project/roles/network_plugin/calico/tasks/peer_with_calico_rr.yml

@@ -0,0 +1,86 @@
+---
+- name: Calico | Set label for groups nodes
+  command: "{{ bin_dir }}/calicoctl.sh label node  {{ inventory_hostname }} calico-group-id={{ calico_group_id }} --overwrite"
+  changed_when: false
+  register: calico_group_id_label
+  until: calico_group_id_label is succeeded
+  delay: "{{ retry_stagger | random + 3 }}"
+  retries: 10
+  when:
+    - calico_group_id is defined
+
+- name: Calico | Configure peering with route reflectors at global scope
+  command:
+    cmd: "{{ bin_dir }}/calicoctl.sh apply -f -"
+    # revert when it's already a string
+    stdin: "{{ stdin is string | ternary(stdin, stdin | to_json) }}"
+  vars:
+    stdin: >
+      {"apiVersion": "projectcalico.org/v3",
+      "kind": "BGPPeer",
+      "metadata": {
+        "name": "{{ calico_rr_id }}-to-node"
+      },
+      "spec": {
+        "peerSelector": "calico-rr-id == '{{ calico_rr_id }}'",
+        "nodeSelector": "calico-group-id == '{{ calico_group_id }}'"
+      }}
+  register: output
+  retries: 4
+  until: output.rc == 0
+  delay: "{{ retry_stagger | random + 3 }}"
+  when:
+    - calico_rr_id is defined
+    - calico_group_id is defined
+    - ('calico_rr' in group_names)
+
+- name: Calico | Configure peering with route reflectors at global scope
+  command:
+    cmd: "{{ bin_dir }}/calicoctl.sh apply -f -"
+    # revert when it's already a string
+    stdin: "{{ stdin is string | ternary(stdin, stdin | to_json) }}"
+  vars:
+    stdin: >
+      {"apiVersion": "projectcalico.org/v3",
+      "kind": "BGPPeer",
+      "metadata": {
+        "name": "peer-to-rrs"
+      },
+      "spec": {
+        "nodeSelector": "!has(i-am-a-route-reflector)",
+        "peerSelector": "has(i-am-a-route-reflector)"
+      }}
+  register: output
+  retries: 4
+  until: output.rc == 0
+  delay: "{{ retry_stagger | random + 3 }}"
+  with_items:
+    - "{{ groups['calico_rr'] | default([]) }}"
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - calico_rr_id is not defined or calico_group_id is not defined
+
+- name: Calico | Configure route reflectors to peer with each other
+  command:
+    cmd: "{{ bin_dir }}/calicoctl.sh apply -f -"
+    # revert when it's already a string
+    stdin: "{{ stdin is string | ternary(stdin, stdin | to_json) }}"
+  vars:
+    stdin: >
+      {"apiVersion": "projectcalico.org/v3",
+      "kind": "BGPPeer",
+      "metadata": {
+        "name": "rr-mesh"
+      },
+      "spec": {
+        "nodeSelector": "has(i-am-a-route-reflector)",
+        "peerSelector": "has(i-am-a-route-reflector)"
+      }}
+  register: output
+  retries: 4
+  until: output.rc == 0
+  delay: "{{ retry_stagger | random + 3 }}"
+  with_items:
+    - "{{ groups['calico_rr'] | default([]) }}"
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]

internal/resources/kubespray/project/roles/network_plugin/calico/tasks/peer_with_router.yml

@@ -0,0 +1,116 @@
+---
+- name: Calico | Configure peering with router(s) at global scope
+  command:
+    cmd: "{{ bin_dir }}/calicoctl.sh apply -f -"
+    stdin: "{{ stdin is string | ternary(stdin, stdin | to_json) }}"
+  vars:
+    stdin: >
+      {"apiVersion": "projectcalico.org/v3",
+      "kind": "BGPPeer",
+      "metadata": {
+        "name": "global-{{ item.name | default(item.router_id | replace(':', '-')) }}"
+      },
+      "spec": {
+        "asNumber": "{{ item.as }}",
+        "peerIP": "{{ item.router_id }}"
+      }}
+  register: output
+  retries: 4
+  until: output.rc == 0
+  delay: "{{ retry_stagger | random + 3 }}"
+  with_items:
+    - "{{ peers | default([]) | selectattr('scope', 'defined') | selectattr('scope', 'equalto', 'global') | list }}"
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+
+- name: Calico | Get node for per node peering
+  command:
+    cmd: "{{ bin_dir }}/calicoctl.sh get node {{ inventory_hostname }}"
+  register: output_get_node
+  when:
+    - ('k8s_cluster' in group_names)
+    - local_as is defined
+    - groups['calico_rr'] | default([]) | length == 0
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+
+- name: Calico | Patch node asNumber for per node peering
+  command:
+    cmd: |-
+      {{ bin_dir }}/calicoctl.sh patch node "{{ inventory_hostname }}" --patch '{{ patch is string | ternary(patch, patch | to_json) }}'
+  vars:
+    patch: >
+      {"spec": {
+        "bgp": {
+          "asNumber": "{{ local_as }}"
+        },
+        "orchRefs": [{"nodeName": "{{ inventory_hostname }}", "orchestrator": "k8s"}]
+      }}
+  register: output
+  retries: 0
+  until: output.rc == 0
+  delay: "{{ retry_stagger | random + 3 }}"
+  when:
+    - ('k8s_cluster' in group_names)
+    - local_as is defined
+    - groups['calico_rr'] | default([]) | length == 0
+    - output_get_node.rc == 0
+
+- name: Calico | Configure node asNumber for per node peering
+  command:
+    cmd: "{{ bin_dir }}/calicoctl.sh apply -f -"
+    stdin: "{{ stdin is string | ternary(stdin, stdin | to_json) }}"
+  vars:
+    stdin: >
+      {"apiVersion": "projectcalico.org/v3",
+      "kind": "Node",
+      "metadata": {
+        "name": "{{ inventory_hostname }}"
+      },
+      "spec": {
+        "bgp": {
+          "asNumber": "{{ local_as }}"
+        },
+        "orchRefs":[{"nodeName":"{{ inventory_hostname }}","orchestrator":"k8s"}]
+      }}
+  register: output
+  retries: 4
+  until: output.rc == 0
+  delay: "{{ retry_stagger | random + 3 }}"
+  when:
+    - ('k8s_cluster' in group_names)
+    - local_as is defined
+    - groups['calico_rr'] | default([]) | length == 0
+    - output_get_node.rc != 0
+
+- name: Calico | Configure peering with router(s) at node scope
+  command:
+    cmd: "{{ bin_dir }}/calicoctl.sh apply -f -"
+    stdin: "{{ stdin is string | ternary(stdin, stdin | to_json) }}"
+  vars:
+    stdin: >
+      {"apiVersion": "projectcalico.org/v3",
+      "kind": "BGPPeer",
+      "metadata": {
+        "name": "{{ inventory_hostname }}-{{ item.name | default(item.router_id | replace(':', '-')) }}"
+      },
+      "spec": {
+        "asNumber": "{{ item.as }}",
+        "node": "{{ inventory_hostname }}",
+        "peerIP": "{{ item.router_id }}",
+        {% if calico_version is version('3.26.0', '>=') and (item.filters | default([]) | length > 0) %}
+        "filters": {{ item.filters }},
+        {% endif %}
+        {% if calico_version is version('3.23.0', '>=') and (item.numallowedlocalasnumbers | default(0) > 0) %}
+        "numAllowedLocalASNumbers": {{ item.numallowedlocalasnumbers }},
+        {% endif %}
+        "sourceAddress": "{{ item.sourceaddress | default('UseNodeIP') }}"
+      }}
+  register: output
+  retries: 4
+  until: output.rc == 0
+  delay: "{{ retry_stagger | random + 3 }}"
+  with_items:
+    - "{{ peers | default([]) | selectattr('scope', 'undefined') | list | union(peers | default([]) | selectattr('scope', 'defined') | selectattr('scope', 'equalto', 'node') | list ) }}"
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+  when:
+    - ('k8s_cluster' in group_names)

internal/resources/kubespray/project/roles/network_plugin/calico/tasks/pre.yml

@@ -0,0 +1,36 @@
+---
+- name: Slurp CNI config
+  slurp:
+    src: /etc/cni/net.d/10-calico.conflist
+  register: calico_cni_config_slurp
+  failed_when: false
+
+- name: Gather calico facts
+  tags:
+  - facts
+  when: calico_cni_config_slurp.content is defined
+  block:
+  - name: Set fact calico_cni_config from slurped CNI config
+    set_fact:
+      calico_cni_config: "{{ calico_cni_config_slurp['content'] | b64decode | from_json }}"
+  - name: Set fact calico_datastore to etcd if needed
+    set_fact:
+      calico_datastore: etcd
+    when:
+    - "'plugins' in calico_cni_config"
+    - "'etcd_endpoints' in calico_cni_config.plugins.0"
+
+- name: Calico | Gather os specific variables
+  include_vars: "{{ item }}"
+  with_first_found:
+  - files:
+    - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower | replace('/', '_') }}.yml"
+    - "{{ ansible_distribution | lower }}-{{ ansible_distribution_release }}.yml"
+    - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower | replace('/', '_') }}.yml"
+    - "{{ ansible_distribution | lower }}.yml"
+    - "{{ ansible_os_family | lower }}-{{ ansible_architecture }}.yml"
+    - "{{ ansible_os_family | lower }}.yml"
+    - defaults.yml
+    paths:
+    - ../vars
+    skip: true

internal/resources/kubespray/project/roles/network_plugin/calico/tasks/repos.yml

@@ -0,0 +1,21 @@
+---
+- name: Calico | Add wireguard yum repo
+  when:
+    - calico_wireguard_enabled
+  block:
+
+    - name: Calico | Add wireguard yum repo
+      yum_repository:
+        name: copr:copr.fedorainfracloud.org:jdoss:wireguard
+        file: _copr:copr.fedorainfracloud.org:jdoss:wireguard
+        description: Copr repo for wireguard owned by jdoss
+        baseurl: "{{ calico_wireguard_repo }}"
+        gpgcheck: true
+        gpgkey: https://download.copr.fedorainfracloud.org/results/jdoss/wireguard/pubkey.gpg
+        skip_if_unavailable: true
+        enabled: true
+        repo_gpgcheck: false
+      when:
+        - ansible_os_family in ['RedHat']
+        - ansible_distribution not in ['Fedora']
+        - ansible_facts['distribution_major_version'] | int < 9

internal/resources/kubespray/project/roles/network_plugin/calico/tasks/reset.yml

@@ -0,0 +1,30 @@
+---
+- name: Reset | check vxlan.calico network device
+  stat:
+    path: /sys/class/net/vxlan.calico
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: vxlan
+
+- name: Reset | remove the network vxlan.calico device created by calico
+  command: ip link del vxlan.calico
+  when: vxlan.stat.exists
+
+- name: Reset | check dummy0 network device
+  stat:
+    path: /sys/class/net/dummy0
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: dummy0
+
+- name: Reset | remove the network device created by calico
+  command: ip link del dummy0
+  when: dummy0.stat.exists
+
+- name: Reset | get and remove remaining routes set by bird
+  shell: set -o pipefail && ip route show proto bird | xargs -i bash -c "ip route del {} proto bird "
+  args:
+    executable: /bin/bash
+  changed_when: false

internal/resources/kubespray/project/roles/network_plugin/calico/tasks/typha_certs.yml

@@ -0,0 +1,52 @@
+---
+- name: Calico | Check if typha-server exists
+  command: "{{ kubectl }} -n kube-system get secret typha-server"
+  register: typha_server_secret
+  changed_when: false
+  failed_when: false
+
+- name: Calico | Ensure calico certs dir
+  file:
+    path: /etc/calico/certs
+    state: directory
+    mode: "0755"
+  when: typha_server_secret.rc != 0
+
+- name: Calico | Copy ssl script for typha certs
+  template:
+    src: make-ssl-calico.sh.j2
+    dest: "{{ bin_dir }}/make-ssl-typha.sh"
+    mode: "0755"
+
+  when: typha_server_secret.rc != 0
+
+- name: Calico | Copy ssl config for typha certs
+  copy:
+    src: openssl.conf
+    dest: /etc/calico/certs/openssl.conf
+    mode: "0644"
+  when: typha_server_secret.rc != 0
+
+- name: Calico | Generate typha certs
+  command: >-
+    {{ bin_dir }}/make-ssl-typha.sh
+    -f /etc/calico/certs/openssl.conf
+    -c {{ kube_cert_dir }}
+    -d /etc/calico/certs
+    -s typha
+  when: typha_server_secret.rc != 0
+
+- name: Calico | Create typha tls secrets
+  command: >-
+    {{ kubectl }} -n kube-system
+    create secret tls {{ item.name }}
+    --cert {{ item.cert }}
+    --key {{ item.key }}
+  with_items:
+    - name: typha-server
+      cert: /etc/calico/certs/typha-server.crt
+      key: /etc/calico/certs/typha-server.key
+    - name: typha-client
+      cert: /etc/calico/certs/typha-client.crt
+      key: /etc/calico/certs/typha-client.key
+  when: typha_server_secret.rc != 0

internal/resources/kubespray/project/roles/network_plugin/calico/templates/calico-apiserver-ns.yml.j2

@@ -0,0 +1,10 @@
+# This is a tech-preview manifest which installs the Calico API server. Note that this manifest is liable to change
+# or be removed in future releases without further warning.
+#
+# Namespace and namespace-scoped resources.
+apiVersion: v1
+kind: Namespace
+metadata:
+  labels:
+    name: calico-apiserver
+  name: calico-apiserver

internal/resources/kubespray/project/roles/network_plugin/calico/templates/calico-apiserver.yml.j2

@@ -0,0 +1,301 @@
+# Policy to ensure the API server isn't cut off. Can be modified, but ensure
+# that the main API server is always able to reach the Calico API server.
+kind: NetworkPolicy
+apiVersion: networking.k8s.io/v1
+metadata:
+  name: allow-apiserver
+  namespace: calico-apiserver
+spec:
+  podSelector:
+    matchLabels:
+      apiserver: "true"
+  ingress:
+  - ports:
+    - protocol: TCP
+      port: 5443
+
+---
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: calico-api
+  namespace: calico-apiserver
+spec:
+  ports:
+  - name: apiserver
+    port: 443
+    protocol: TCP
+    targetPort: 5443
+  selector:
+    apiserver: "true"
+  type: ClusterIP
+
+---
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    apiserver: "true"
+    k8s-app: calico-apiserver
+  name: calico-apiserver
+  namespace: calico-apiserver
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      apiserver: "true"
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        apiserver: "true"
+        k8s-app: calico-apiserver
+      name: calico-apiserver
+      namespace: calico-apiserver
+    spec:
+      containers:
+      - args:
+        - --secure-port=5443
+        env:
+        - name: DATASTORE_TYPE
+          value: kubernetes
+        image: {{ calico_apiserver_image_repo }}:{{ calico_apiserver_image_tag }}
+        imagePullPolicy: {{ k8s_image_pull_policy }}
+        livenessProbe:
+          httpGet:
+            path: /version
+            port: 5443
+            scheme: HTTPS
+          initialDelaySeconds: 90
+          periodSeconds: 10
+        name: calico-apiserver
+{% if calico_version is version('3.28.0', '>=') %}
+        readinessProbe:
+          httpGet:
+            path: /readyz
+            port: 5443
+            scheme: HTTPS
+          timeoutSeconds: 5
+          periodSeconds: 60
+{% else %}
+        readinessProbe:
+          exec:
+            command:
+            - /code/filecheck
+          failureThreshold: 5
+          initialDelaySeconds: 5
+          periodSeconds: 10
+{% endif %}
+        securityContext:
+          privileged: false
+          runAsUser: 0
+        volumeMounts:
+        - mountPath: /code/apiserver.local.config/certificates
+          name: calico-apiserver-certs
+      dnsPolicy: ClusterFirst
+      nodeSelector:
+        kubernetes.io/os: linux
+      restartPolicy: Always
+      serviceAccount: calico-apiserver
+      serviceAccountName: calico-apiserver
+      tolerations:
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/control-plane
+      volumes:
+      - name: calico-apiserver-certs
+        secret:
+          secretName: calico-apiserver-certs
+
+---
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: calico-apiserver
+  namespace: calico-apiserver
+
+---
+
+# Cluster-scoped resources below here.
+apiVersion: apiregistration.k8s.io/v1
+kind: APIService
+metadata:
+  name: v3.projectcalico.org
+spec:
+  group: projectcalico.org
+  groupPriorityMinimum: 1500
+  caBundle: {{ calico_apiserver_cabundle }}
+  service:
+    name: calico-api
+    namespace: calico-apiserver
+    port: 443
+  version: v3
+  versionPriority: 200
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: calico-crds
+rules:
+- apiGroups:
+  - extensions
+  - networking.k8s.io
+  - ""
+  resources:
+  - networkpolicies
+  - nodes
+  - namespaces
+  - pods
+  - serviceaccounts
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - crd.projectcalico.org
+  resources:
+  - globalnetworkpolicies
+  - networkpolicies
+  - clusterinformations
+  - hostendpoints
+  - globalnetworksets
+  - networksets
+  - bgpconfigurations
+  - bgppeers
+  - bgpfilters
+  - felixconfigurations
+  - kubecontrollersconfigurations
+  - ippools
+  - ipamconfigs
+  - ipreservations
+  - ipamblocks
+  - blockaffinities
+  - caliconodestatuses
+  - tiers
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - update
+  - delete
+{% if calico_version is version('3.28.0', '>=') %}
+- apiGroups:
+  - policy
+  resourceNames:
+  - calico-apiserver
+  resources:
+  - podsecuritypolicies
+  verbs:
+  - use
+{% endif %}
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: calico-extension-apiserver-auth-access
+rules:
+- apiGroups:
+  - ""
+  resourceNames:
+  - extension-apiserver-authentication
+  resources:
+  - configmaps
+  verbs:
+  - list
+  - watch
+  - get
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - clusterroles
+  - clusterrolebindings
+  - roles
+  - rolebindings
+  verbs:
+  - get
+  - list
+  - watch
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: calico-webhook-reader
+rules:
+- apiGroups:
+  - admissionregistration.k8s.io
+  resources:
+  - mutatingwebhookconfigurations
+  - validatingwebhookconfigurations
+  verbs:
+  - get
+  - list
+  - watch
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: calico-apiserver-access-crds
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: calico-crds
+subjects:
+- kind: ServiceAccount
+  name: calico-apiserver
+  namespace: calico-apiserver
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: calico-apiserver-delegate-auth
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:auth-delegator
+subjects:
+- kind: ServiceAccount
+  name: calico-apiserver
+  namespace: calico-apiserver
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: calico-apiserver-webhook-reader
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: calico-webhook-reader
+subjects:
+- kind: ServiceAccount
+  name: calico-apiserver
+  namespace: calico-apiserver
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: calico-extension-apiserver-auth-access
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: calico-extension-apiserver-auth-access
+subjects:
+- kind: ServiceAccount
+  name: calico-apiserver
+  namespace: calico-apiserver

internal/resources/kubespray/project/roles/network_plugin/calico/templates/calico-config.yml.j2

@@ -0,0 +1,106 @@
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: calico-config
+  namespace: kube-system
+data:
+{% if calico_datastore == "etcd" %}
+  etcd_endpoints: "{{ etcd_access_addresses }}"
+  etcd_ca: "/calico-secrets/ca_cert.crt"
+  etcd_cert: "/calico-secrets/cert.crt"
+  etcd_key: "/calico-secrets/key.pem"
+{% elif calico_datastore == "kdd" and typha_enabled %}
+  # To enable Typha, set this to "calico-typha" *and* set a non-zero value for Typha replicas
+  # below.  We recommend using Typha if you have more than 50 nodes. Above 100 nodes it is
+  # essential.
+  typha_service_name: "calico-typha"
+{% endif %}
+{% if calico_network_backend == 'bird' %}
+  cluster_type: "kubespray,bgp"
+  calico_backend: "bird"
+{% else %}
+  cluster_type: "kubespray"
+  calico_backend: "{{ calico_network_backend }}"
+{% endif %}
+{% if inventory_hostname in groups['k8s_cluster'] and peer_with_router | default(false) %}
+  as: "{{ local_as | default(global_as_num) }}"
+{% endif -%}
+  # The CNI network configuration to install on each node. The special
+  # values in this config will be automatically populated.
+  cni_network_config: |-
+    {
+      "name": "{{ calico_cni_name }}",
+      "cniVersion":"0.3.1",
+      "plugins":[
+        {
+          {% if calico_datastore == "kdd" %}
+            "datastore_type": "kubernetes",
+            "nodename": "__KUBERNETES_NODE_NAME__",
+          {% endif %}
+            "type": "calico",
+            "log_level": "info",
+          {% if calico_cni_log_file_path %}
+            "log_file_path": "{{ calico_cni_log_file_path }}",
+          {% endif %}
+          {% if calico_datastore == "etcd" %}
+            "etcd_endpoints": "{{ etcd_access_addresses }}",
+            "etcd_cert_file": "{{ calico_cert_dir }}/cert.crt",
+            "etcd_key_file": "{{ calico_cert_dir }}/key.pem",
+            "etcd_ca_cert_file": "{{ calico_cert_dir }}/ca_cert.crt",
+          {% endif %}
+          {% if calico_ipam_host_local %}
+            "ipam": {
+              "type": "host-local",
+              "subnet": "usePodCidr"
+            },
+          {% else %}
+            "ipam": {
+              "type": "calico-ipam",
+            {% if ipv4_stack %}
+              "assign_ipv4": "true"{{ ',' if (ipv6_stack and ipv4_stack) }}
+            {% endif %}
+            {% if ipv6_stack %}
+              "assign_ipv6": "true"
+            {% endif %}
+            },
+          {% endif %}
+          {% if calico_allow_ip_forwarding %}
+            "container_settings": {
+              "allow_ip_forwarding": true
+            },
+          {% endif %}
+          {% if (calico_feature_control is defined) and (calico_feature_control | length > 0) %}
+            "feature_control": {
+              {% for fc in calico_feature_control -%}
+              {% set fcval = calico_feature_control[fc] -%}
+                "{{ fc }}": {{ (fcval | string | lower) if (fcval == true or fcval == false) else "\"" + fcval + "\"" }}{{ "," if not loop.last else "" }}
+              {% endfor -%}
+              {{- "" }}
+            },
+          {% endif %}
+          {% if enable_network_policy %}
+            "policy": {
+              "type": "k8s"
+            },
+          {% endif %}
+          {% if calico_mtu is defined and calico_mtu is number %}
+            "mtu": {{ calico_mtu }},
+          {% endif %}
+            "kubernetes": {
+              "kubeconfig": "__KUBECONFIG_FILEPATH__"
+            }
+        },
+        {
+          "type":"portmap",
+          "capabilities": {
+            "portMappings": true
+          }
+        },
+        {
+          "type":"bandwidth",
+          "capabilities": {
+            "bandwidth": true
+          }
+        }
+      ]
+    }

internal/resources/kubespray/project/roles/network_plugin/calico/templates/calico-cr.yml.j2

@@ -0,0 +1,213 @@
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: calico-cni-plugin
+rules:
+  - apiGroups: [""]
+    resources:
+      - pods
+      - nodes
+      - namespaces
+    verbs:
+      - get
+  - apiGroups: [""]
+    resources:
+      - pods/status
+    verbs:
+      - patch
+  - apiGroups: [""]
+    resources:
+      - nodes/status
+    verbs:
+      - update
+  - apiGroups: ["crd.projectcalico.org"]
+    resources:
+      - blockaffinities
+      - ipamblocks
+      - ipamhandles
+      - clusterinformations
+      - ippools
+      - ipreservations
+      - ipamconfigs
+    verbs:
+      - get
+      - list
+      - create
+      - update
+      - delete
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: calico-node
+  namespace: kube-system
+rules:
+  - apiGroups: [""]
+    resources:
+      - pods
+      - nodes
+      - namespaces
+      - configmaps
+    verbs:
+      - get
+  # EndpointSlices are used for Service-based network policy rule
+  # enforcement.
+  - apiGroups: ["discovery.k8s.io"]
+    resources:
+      - endpointslices
+    verbs:
+      - watch
+      - list
+  - apiGroups: [""]
+    resources:
+      - endpoints
+      - services
+    verbs:
+      - watch
+      - list
+{% if calico_datastore == "kdd" %}
+      # Used to discover Typhas.
+      - get
+{% endif %}
+  - apiGroups: [""]
+    resources:
+      - nodes/status
+    verbs:
+      # Needed for clearing NodeNetworkUnavailable flag.
+      - patch
+{% if calico_datastore == "kdd" %}
+      # Calico stores some configuration information in node annotations.
+      - update
+  # Watch for changes to Kubernetes NetworkPolicies.
+  - apiGroups: ["networking.k8s.io"]
+    resources:
+      - networkpolicies
+    verbs:
+      - watch
+      - list
+  # Watch for changes to Kubernetes AdminNetworkPolicies.
+  - apiGroups: ["policy.networking.k8s.io"]
+    resources:
+      - adminnetworkpolicies
+    verbs:
+      - watch
+      - list
+  # Used by Calico for policy information.
+  - apiGroups: [""]
+    resources:
+      - pods
+      - namespaces
+      - serviceaccounts
+    verbs:
+      - list
+      - watch
+  # The CNI plugin patches pods/status.
+  - apiGroups: [""]
+    resources:
+      - pods/status
+    verbs:
+      - patch
+  # Calico monitors various CRDs for config.
+  - apiGroups: ["crd.projectcalico.org"]
+    resources:
+      - globalfelixconfigs
+      - felixconfigurations
+      - bgppeers
+      - bgpfilters
+      - globalbgpconfigs
+      - bgpconfigurations
+      - ippools
+      - ipreservations
+      - ipamblocks
+      - globalnetworkpolicies
+      - globalnetworksets
+      - networkpolicies
+      - networksets
+      - clusterinformations
+      - hostendpoints
+      - blockaffinities
+      - caliconodestatuses
+      - tiers
+    verbs:
+      - get
+      - list
+      - watch
+   # Calico creates some tiers on startup.
+  - apiGroups: ["crd.projectcalico.org"]
+    resources:
+      - tiers
+    verbs:
+      - create
+  # Calico must create and update some CRDs on startup.
+  - apiGroups: ["crd.projectcalico.org"]
+    resources:
+      - ippools
+      - felixconfigurations
+      - clusterinformations
+    verbs:
+      - create
+      - update
+  # Calico must update some CRDs.
+  - apiGroups: [ "crd.projectcalico.org" ]
+    resources:
+      - caliconodestatuses
+    verbs:
+      - update
+  # Calico stores some configuration information on the node.
+  - apiGroups: [""]
+    resources:
+      - nodes
+    verbs:
+      - get
+      - list
+      - watch
+  # These permissions are only required for upgrade from v2.6, and can
+  # be removed after upgrade or on fresh installations.
+  - apiGroups: ["crd.projectcalico.org"]
+    resources:
+      - bgpconfigurations
+      - bgppeers
+    verbs:
+      - create
+      - update
+  # These permissions are required for Calico CNI to perform IPAM allocations.
+  - apiGroups: ["crd.projectcalico.org"]
+    resources:
+      - blockaffinities
+      - ipamblocks
+      - ipamhandles
+    verbs:
+      - get
+      - list
+      - create
+      - update
+      - delete
+  - apiGroups: ["crd.projectcalico.org"]
+    resources:
+      - ipamconfigs
+    verbs:
+      - get
+      - create
+  # Block affinities must also be watchable by confd for route aggregation.
+  - apiGroups: ["crd.projectcalico.org"]
+    resources:
+      - blockaffinities
+    verbs:
+      - watch
+  # The Calico IPAM migration needs to get daemonsets. These permissions can be
+  # removed if not upgrading from an installation using host-local IPAM.
+  - apiGroups: ["apps"]
+    resources:
+      - daemonsets
+    verbs:
+      - get
+{% endif %}
+  # Used for creating service account tokens to be used by the CNI plugin
+  - apiGroups: [""]
+    resources:
+      - serviceaccounts/token
+    resourceNames:
+      - calico-cni-plugin
+    verbs:
+      - create

internal/resources/kubespray/project/roles/network_plugin/calico/templates/calico-crb.yml.j2

@@ -0,0 +1,28 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: calico-node
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: calico-node
+subjects:
+- kind: ServiceAccount
+  name: calico-node
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: calico-cni-plugin
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: calico-cni-plugin
+subjects:
+- kind: ServiceAccount
+  name: calico-cni-plugin
+  namespace: kube-system

internal/resources/kubespray/project/roles/network_plugin/calico/templates/calico-ipamconfig.yml.j2

@@ -0,0 +1,8 @@
+apiVersion: crd.projectcalico.org/v1
+kind: IPAMConfig
+metadata:
+  name: default
+spec:
+  autoAllocateBlocks: {{ calico_ipam_autoallocateblocks }}
+  strictAffinity: {{ calico_ipam_strictaffinity }}
+  maxBlocksPerHost: {{ calico_ipam_maxblocksperhost }}

internal/resources/kubespray/project/roles/network_plugin/calico/templates/calico-node-sa.yml.j2

@@ -0,0 +1,13 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: calico-node
+  namespace: kube-system
+
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: calico-cni-plugin
+  namespace: kube-system

internal/resources/kubespray/project/roles/network_plugin/calico/templates/calico-node.yml.j2

@@ -0,0 +1,509 @@
+---
+# This manifest installs the calico/node container, as well
+# as the Calico CNI plugins and network config on
+# each control plane and worker node in a Kubernetes cluster.
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: calico-node
+  namespace: kube-system
+  labels:
+    k8s-app: calico-node
+spec:
+  selector:
+    matchLabels:
+      k8s-app: calico-node
+  updateStrategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 1
+  template:
+    metadata:
+      labels:
+        k8s-app: calico-node
+      annotations:
+{% if calico_datastore == "etcd" %}
+        kubespray.etcd-cert/serial: "{{ etcd_client_cert_serial }}"
+{% endif %}
+{% if calico_felix_prometheusmetricsenabled %}
+        prometheus.io/scrape: 'true'
+        prometheus.io/port: "{{ calico_felix_prometheusmetricsport }}"
+{% endif %}
+    spec:
+      nodeSelector:
+        {{ calico_ds_nodeselector }}
+      priorityClassName: system-node-critical
+      hostNetwork: true
+      serviceAccountName: calico-node
+      tolerations:
+        # Make sure calico-node gets scheduled on all nodes.
+        - effect: NoSchedule
+          operator: Exists
+        # Mark the pod as a critical add-on for rescheduling.
+        - key: CriticalAddonsOnly
+          operator: Exists
+        - effect: NoExecute
+          operator: Exists
+      # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force
+      # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods.
+      terminationGracePeriodSeconds: 0
+      initContainers:
+{% if calico_datastore == "kdd" and not calico_ipam_host_local %}
+        # This container performs upgrade from host-local IPAM to calico-ipam.
+        # It can be deleted if this is a fresh installation, or if you have already
+        # upgraded to use calico-ipam.
+        - name: upgrade-ipam
+          image: {{ calico_cni_image_repo }}:{{ calico_cni_image_tag }}
+          imagePullPolicy: {{ k8s_image_pull_policy }}
+          command: ["/opt/cni/bin/calico-ipam", "-upgrade"]
+          envFrom:
+            - configMapRef:
+                # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode.
+                name: kubernetes-services-endpoint
+                optional: true
+          env:
+            - name: KUBERNETES_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: CALICO_NETWORKING_BACKEND
+              valueFrom:
+                configMapKeyRef:
+                  name: calico-config
+                  key: calico_backend
+          volumeMounts:
+            - mountPath: /var/lib/cni/networks
+              name: host-local-net-dir
+            - mountPath: /host/opt/cni/bin
+              name: cni-bin-dir
+          securityContext:
+            privileged: true
+{% endif %}
+        # This container installs the Calico CNI binaries
+        # and CNI network config file on each node.
+        - name: install-cni
+          image: {{ calico_cni_image_repo }}:{{ calico_cni_image_tag }}
+          imagePullPolicy: {{ k8s_image_pull_policy }}
+          command: ["/opt/cni/bin/install"]
+          envFrom:
+          - configMapRef:
+              # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode.
+              name: kubernetes-services-endpoint
+              optional: true
+          env:
+            # The CNI network config to install on each node.
+            - name: CNI_NETWORK_CONFIG
+              valueFrom:
+                configMapKeyRef:
+                  name: calico-config
+                  key: cni_network_config
+            # Name of the CNI config file to create.
+            - name: CNI_CONF_NAME
+              value: "10-calico.conflist"
+{% if calico_mtu is defined %}
+            # CNI MTU Config variable
+            - name: CNI_MTU
+              value: "{{ calico_veth_mtu | default(calico_mtu) }}"
+{% endif %}
+            # Prevents the container from sleeping forever.
+            - name: SLEEP
+              value: "false"
+{% if calico_datastore == "etcd" %}
+            - name: ETCD_ENDPOINTS
+              valueFrom:
+                configMapKeyRef:
+                  name: calico-config
+                  key: etcd_endpoints
+{% endif %}
+{% if calico_datastore == "kdd" %}
+            # Set the hostname based on the k8s node name.
+            - name: KUBERNETES_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+{% endif %}
+          volumeMounts:
+            - mountPath: /host/etc/cni/net.d
+              name: cni-net-dir
+            - mountPath: /host/opt/cni/bin
+              name: cni-bin-dir
+          securityContext:
+            privileged: true
+        # This init container mounts the necessary filesystems needed by the BPF data plane
+        # i.e. bpf at /sys/fs/bpf and cgroup2 at /run/calico/cgroup. Calico-node initialisation is executed
+        # in best effort fashion, i.e. no failure for errors, to not disrupt pod creation in iptable mode.
+        - name: "mount-bpffs"
+          image: {{ calico_node_image_repo }}:{{ calico_node_image_tag }}
+          imagePullPolicy: {{ k8s_image_pull_policy }}
+          command: ["calico-node", "-init", "-best-effort"]
+          volumeMounts:
+            - mountPath: /sys/fs
+              name: sys-fs
+              # Bidirectional is required to ensure that the new mount we make at /sys/fs/bpf propagates to the host
+              # so that it outlives the init container.
+              mountPropagation: Bidirectional
+            - mountPath: /var/run/calico
+              name: var-run-calico
+              # Bidirectional is required to ensure that the new mount we make at /run/calico/cgroup propagates to the host
+              # so that it outlives the init container.
+              mountPropagation: Bidirectional
+            # Mount /proc/ from host which usually is an init program at /nodeproc. It's needed by mountns binary,
+            # executed by calico-node, to mount root cgroup2 fs at /run/calico/cgroup to attach CTLB programs correctly.
+            - mountPath: /nodeproc
+              name: nodeproc
+              readOnly: true
+          securityContext:
+            privileged: true
+      containers:
+        # Runs calico/node container on each Kubernetes node.  This
+        # container programs network policy and routes on each
+        # host.
+        - name: calico-node
+          image: {{ calico_node_image_repo }}:{{ calico_node_image_tag }}
+          imagePullPolicy: {{ k8s_image_pull_policy }}
+          envFrom:
+            - configMapRef:
+                # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode.
+                name: kubernetes-services-endpoint
+                optional: true
+          env:
+            # The location of the Calico etcd cluster.
+{% if calico_datastore == "etcd" %}
+            - name: ETCD_ENDPOINTS
+              valueFrom:
+                configMapKeyRef:
+                  name: calico-config
+                  key: etcd_endpoints
+            # Location of the CA certificate for etcd.
+            - name: ETCD_CA_CERT_FILE
+              valueFrom:
+                configMapKeyRef:
+                  name: calico-config
+                  key: etcd_ca
+            # Location of the client key for etcd.
+            - name: ETCD_KEY_FILE
+              valueFrom:
+                configMapKeyRef:
+                  name: calico-config
+                  key: etcd_key
+            # Location of the client certificate for etcd.
+            - name: ETCD_CERT_FILE
+              valueFrom:
+                configMapKeyRef:
+                  name: calico-config
+                  key: etcd_cert
+{% elif calico_datastore == "kdd" %}
+            # Use Kubernetes API as the backing datastore.
+            - name: DATASTORE_TYPE
+              value: "kubernetes"
+{% if typha_enabled %}
+            # Typha support: controlled by the ConfigMap.
+            - name: FELIX_TYPHAK8SSERVICENAME
+              valueFrom:
+                configMapKeyRef:
+                  name: calico-config
+                  key: typha_service_name
+{% if typha_secure %}
+            - name: FELIX_TYPHACN
+              value: typha-server
+            - name: FELIX_TYPHACAFILE
+              value: /etc/typha-ca/ca.crt
+            - name: FELIX_TYPHACERTFILE
+              value: /etc/typha-client/typha-client.crt
+            - name: FELIX_TYPHAKEYFILE
+              value: /etc/typha-client/typha-client.key
+{% endif %}
+{% endif %}
+            # Wait for the datastore.
+            - name: WAIT_FOR_DATASTORE
+              value: "true"
+{% endif %}
+{% if calico_network_backend == 'vxlan' %}
+            - name: FELIX_VXLANVNI
+              value: "{{ calico_vxlan_vni }}"
+            - name: FELIX_VXLANPORT
+              value: "{{ calico_vxlan_port }}"
+{% endif %}
+            # Choose the backend to use.
+            - name: CALICO_NETWORKING_BACKEND
+              valueFrom:
+                configMapKeyRef:
+                  name: calico-config
+                  key: calico_backend
+            # Cluster type to identify the deployment type
+            - name: CLUSTER_TYPE
+              value: "k8s,bgp"
+            # Set noderef for node controller.
+            - name: CALICO_K8S_NODE_REF
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            # Disable file logging so `kubectl logs` works.
+            - name: CALICO_DISABLE_FILE_LOGGING
+              value: "true"
+            # Set Felix endpoint to host default action to ACCEPT.
+            - name: FELIX_DEFAULTENDPOINTTOHOSTACTION
+              value: "{{ calico_endpoint_to_host_action | default('RETURN') }}"
+            - name: FELIX_HEALTHHOST
+              value: "{{ calico_healthhost }}"
+{% if kube_proxy_mode == 'ipvs' and kube_apiserver_node_port_range is defined %}
+            - name: FELIX_KUBENODEPORTRANGES
+              value: "{{ kube_apiserver_node_port_range.split('-')[0] }}:{{ kube_apiserver_node_port_range.split('-')[1] }}"
+{% endif %}
+            - name: FELIX_IPTABLESBACKEND
+              value: "{{ calico_iptables_backend }}"
+            - name: FELIX_IPTABLESLOCKTIMEOUTSECS
+              value: "{{ calico_iptables_lock_timeout_secs }}"
+            # The default IPv4 pool to create on startup if none exists. Pod IPs will be
+            # chosen from this range. Changing this value after installation will have
+            # no effect. This should fall within `--cluster-cidr`.
+            # - name: CALICO_IPV4POOL_CIDR
+            #   value: "192.168.0.0/16"
+            - name: CALICO_IPV4POOL_IPIP
+              value: "{{ calico_ipv4pool_ipip }}"
+            # Enable or Disable VXLAN on the default IP pool.
+            - name: CALICO_IPV4POOL_VXLAN
+              value: "Never"
+            - name: FELIX_IPV6SUPPORT
+              value: "{{ ipv6_stack | default(false) }}"
+            # Set Felix logging to "info"
+            - name: FELIX_LOGSEVERITYSCREEN
+              value: "{{ calico_loglevel }}"
+            # Set Calico startup logging to "error"
+            - name: CALICO_STARTUP_LOGLEVEL
+              value: "{{ calico_node_startup_loglevel }}"
+            # Enable or disable usage report
+            - name: FELIX_USAGEREPORTINGENABLED
+              value: "{{ calico_usage_reporting }}"
+            # Set MTU for tunnel device used if ipip is enabled
+{% if calico_mtu is defined %}
+            # Set MTU for tunnel device used if ipip is enabled
+            - name: FELIX_IPINIPMTU
+              value: "{{ calico_veth_mtu | default(calico_mtu) }}"
+            # Set MTU for the VXLAN tunnel device.
+            - name: FELIX_VXLANMTU
+              value: "{{ calico_veth_mtu | default(calico_mtu) }}"
+            # Set MTU for the Wireguard tunnel device.
+            - name: FELIX_WIREGUARDMTU
+              value: "{{ calico_veth_mtu | default(calico_mtu) }}"
+{% endif %}
+            - name: FELIX_CHAININSERTMODE
+              value: "{{ calico_felix_chaininsertmode }}"
+            - name: FELIX_PROMETHEUSMETRICSENABLED
+              value: "{{ calico_felix_prometheusmetricsenabled }}"
+            - name: FELIX_PROMETHEUSMETRICSPORT
+              value: "{{ calico_felix_prometheusmetricsport }}"
+            - name: FELIX_PROMETHEUSGOMETRICSENABLED
+              value: "{{ calico_felix_prometheusgometricsenabled }}"
+            - name: FELIX_PROMETHEUSPROCESSMETRICSENABLED
+              value: "{{ calico_felix_prometheusprocessmetricsenabled }}"
+{% if calico_ip_auto_method is defined %}
+            - name: IP_AUTODETECTION_METHOD
+              value: "{{ calico_ip_auto_method }}"
+{% else %}
+            - name: NODEIP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.hostIP
+            - name: IP_AUTODETECTION_METHOD
+              value: "can-reach=$(NODEIP)"
+{% endif %}
+{% if ipv4_stack %}
+            - name: IP
+              value: "autodetect"
+{% else %}
+            - name: IP
+              value: none
+{% endif %}
+{% if ipv6_stack %}
+            - name: IP6
+              value: autodetect
+{% endif %}
+{% if calico_ip6_auto_method is defined and ipv6_stack %}
+            - name: IP6_AUTODETECTION_METHOD
+              value: "{{ calico_ip6_auto_method }}"
+{% endif %}
+{% if calico_felix_mtu_iface_pattern is defined %}
+            - name: FELIX_MTUIFACEPATTERN
+              value: "{{ calico_felix_mtu_iface_pattern }}"
+{% endif %}
+{% if calico_use_default_route_src_ipaddr | default(false) %}
+            - name: FELIX_DEVICEROUTESOURCEADDRESS
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.hostIP
+{% endif %}
+            - name: NODENAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: FELIX_HEALTHENABLED
+              value: "true"
+            - name: FELIX_IGNORELOOSERPF
+              value: "{{ calico_node_ignorelooserpf }}"
+            - name: CALICO_MANAGE_CNI
+              value: "true"
+{% if calico_ipam_host_local %}
+            - name: USE_POD_CIDR
+              value: "true"
+{% endif %}
+{% if calico_node_extra_envs is defined %}
+{% for key in calico_node_extra_envs %}
+            - name: {{ key }}
+              value: "{{ calico_node_extra_envs[key] }}"
+{% endfor %}
+{% endif %}
+          securityContext:
+            privileged: true
+          resources:
+            limits:
+{% if calico_node_cpu_limit != "0" %}
+              cpu: {{ calico_node_cpu_limit }}
+{% endif %}
+              memory: {{ calico_node_memory_limit }}
+            requests:
+              cpu: {{ calico_node_cpu_requests }}
+              memory: {{ calico_node_memory_requests }}
+          lifecycle:
+            preStop:
+              exec:
+                command:
+                - /bin/calico-node
+                - -shutdown
+          livenessProbe:
+            exec:
+              command:
+              - /bin/calico-node
+              - -felix-live
+{% if calico_network_backend == "bird" %}
+              - -bird-live
+{% endif %}
+            periodSeconds: 10
+            initialDelaySeconds: 10
+            timeoutSeconds: {{ calico_node_livenessprobe_timeout | default(10) }}
+            failureThreshold: 6
+          readinessProbe:
+            exec:
+              command:
+              - /bin/calico-node
+{% if calico_network_backend == "bird" %}
+              - -bird-ready
+{% endif %}
+              - -felix-ready
+            periodSeconds: 10
+            timeoutSeconds: {{ calico_node_readinessprobe_timeout | default(10) }}
+            failureThreshold: 6
+          volumeMounts:
+            - mountPath: /lib/modules
+              name: lib-modules
+              readOnly: true
+            - mountPath: /var/run/calico
+              name: var-run-calico
+              readOnly: false
+            - mountPath: /var/lib/calico
+              name: var-lib-calico
+              readOnly: false
+{% if calico_datastore == "etcd" %}
+            - mountPath: /calico-secrets
+              name: etcd-certs
+              readOnly: true
+{% endif %}
+            - name: xtables-lock
+              mountPath: /run/xtables.lock
+              readOnly: false
+            # For maintaining CNI plugin API credentials.
+            - mountPath: /host/etc/cni/net.d
+              name: cni-net-dir
+              readOnly: false
+{% if typha_secure %}
+            - name: typha-client
+              mountPath: /etc/typha-client
+              readOnly: true
+            - name: typha-cacert
+              subPath: ca.crt
+              mountPath: /etc/typha-ca/ca.crt
+              readOnly: true
+{% endif %}
+            - name: policysync
+              mountPath: /var/run/nodeagent
+            # For eBPF mode, we need to be able to mount the BPF filesystem at /sys/fs/bpf so we mount in the
+            # parent directory.
+            - name: bpffs
+              mountPath: /sys/fs/bpf
+            - name: cni-log-dir
+              mountPath: /var/log/calico/cni
+              readOnly: true
+      volumes:
+        # Used by calico/node.
+        - name: lib-modules
+          hostPath:
+            path: /lib/modules
+        - name: var-run-calico
+          hostPath:
+            path: /var/run/calico
+            type: DirectoryOrCreate
+        - name: var-lib-calico
+          hostPath:
+            path: /var/lib/calico
+            type: DirectoryOrCreate
+        # Used to install CNI.
+        - name: cni-net-dir
+          hostPath:
+            path: /etc/cni/net.d
+        - name: cni-bin-dir
+          hostPath:
+            path: /opt/cni/bin
+            type: DirectoryOrCreate
+{% if calico_datastore == "etcd" %}
+        # Mount in the etcd TLS secrets.
+        - name: etcd-certs
+          hostPath:
+            path: "{{ calico_cert_dir }}"
+{% endif %}
+        # Mount the global iptables lock file, used by calico/node
+        - name: xtables-lock
+          hostPath:
+            path: /run/xtables.lock
+            type: FileOrCreate
+{% if calico_datastore == "kdd" and not calico_ipam_host_local %}
+        # Mount in the directory for host-local IPAM allocations. This is
+        # used when upgrading from host-local to calico-ipam, and can be removed
+        # if not using the upgrade-ipam init container.
+        - name: host-local-net-dir
+          hostPath:
+            path: /var/lib/cni/networks
+{% endif %}
+{% if typha_enabled and typha_secure %}
+        - name: typha-client
+          secret:
+            secretName: typha-client
+            items:
+                - key: tls.crt
+                  path: typha-client.crt
+                - key: tls.key
+                  path: typha-client.key
+        - name: typha-cacert
+          hostPath:
+            path: "/etc/kubernetes/ssl/"
+{% endif %}
+        - name: sys-fs
+          hostPath:
+            path: /sys/fs/
+            type: DirectoryOrCreate
+        - name: bpffs
+          hostPath:
+            path: /sys/fs/bpf
+            type: Directory
+        # mount /proc at /nodeproc to be used by mount-bpffs initContainer to mount root cgroup2 fs.
+        - name: nodeproc
+          hostPath:
+            path: /proc
+        # Used to access CNI logs.
+        - name: cni-log-dir
+          hostPath:
+            path: /var/log/calico/cni
+        # Used to create per-pod Unix Domain Sockets
+        - name: policysync
+          hostPath:
+            type: DirectoryOrCreate
+            path: /var/run/nodeagent

internal/resources/kubespray/project/roles/network_plugin/calico/templates/calico-typha.yml.j2

@@ -0,0 +1,186 @@
+# This manifest creates a Service, which will be backed by Calico's Typha daemon.
+# Typha sits in between Felix and the API server, reducing Calico's load on the API server.
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: calico-typha
+  namespace: kube-system
+  labels:
+    k8s-app: calico-typha
+spec:
+  ports:
+    - port: 5473
+      protocol: TCP
+      targetPort: calico-typha
+      name: calico-typha
+{% if typha_prometheusmetricsenabled %}
+    - port: {{ typha_prometheusmetricsport }}
+      protocol: TCP
+      targetPort: http-metrics
+      name: metrics
+{% endif %}
+  selector:
+    k8s-app: calico-typha
+
+---
+
+# This manifest creates a Deployment of Typha to back the above service.
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: calico-typha
+  namespace: kube-system
+  labels:
+    k8s-app: calico-typha
+spec:
+  # Number of Typha replicas.  To enable Typha, set this to a non-zero value *and* set the
+  # typha_service_name variable in the calico-config ConfigMap above.
+  #
+  # We recommend using Typha if you have more than 50 nodes.  Above 100 nodes it is essential
+  # (when using the Kubernetes datastore).  Use one replica for every 100-200 nodes.  In
+  # production, we recommend running at least 3 replicas to reduce the impact of rolling upgrade.
+  replicas: {{ typha_replicas }}
+  revisionHistoryLimit: 2
+  selector:
+    matchLabels:
+      k8s-app: calico-typha
+  template:
+    metadata:
+      labels:
+        k8s-app: calico-typha
+      annotations:
+        cluster-autoscaler.kubernetes.io/safe-to-evict: 'true'
+{% if typha_prometheusmetricsenabled %}
+        prometheus.io/scrape: 'true'
+        prometheus.io/port: "{{ typha_prometheusmetricsport }}"
+{% endif %}
+    spec:
+      nodeSelector:
+        kubernetes.io/os: linux
+      hostNetwork: true
+      tolerations:
+        - key: node-role.kubernetes.io/control-plane
+          operator: Exists
+          effect: NoSchedule
+      # Since Calico can't network a pod until Typha is up, we need to run Typha itself
+      # as a host-networked pod.
+      serviceAccountName: calico-node
+      priorityClassName: system-cluster-critical
+      # fsGroup allows using projected serviceaccount tokens as described here kubernetes/kubernetes#82573
+      securityContext:
+        fsGroup: 65534
+      containers:
+      - image: {{ calico_typha_image_repo }}:{{ calico_typha_image_tag }}
+        imagePullPolicy: {{ k8s_image_pull_policy }}
+        name: calico-typha
+        ports:
+        - containerPort: 5473
+          name: calico-typha
+          protocol: TCP
+{% if typha_prometheusmetricsenabled %}
+        - containerPort: {{ typha_prometheusmetricsport }}
+          name: http-metrics
+          protocol: TCP
+{% endif %}
+        envFrom:
+        - configMapRef:
+            # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode.
+            name: kubernetes-services-endpoint
+            optional: true
+        env:
+          # Enable "info" logging by default.  Can be set to "debug" to increase verbosity.
+          - name: TYPHA_LOGSEVERITYSCREEN
+            value: "info"
+          # Disable logging to file and syslog since those don't make sense in Kubernetes.
+          - name: TYPHA_LOGFILEPATH
+            value: "none"
+          - name: TYPHA_LOGSEVERITYSYS
+            value: "none"
+          # Monitor the Kubernetes API to find the number of running instances and rebalance
+          # connections.
+          - name: TYPHA_CONNECTIONREBALANCINGMODE
+            value: "kubernetes"
+          - name: TYPHA_DATASTORETYPE
+            value: "kubernetes"
+          - name: TYPHA_HEALTHENABLED
+            value: "true"
+          - name: TYPHA_MAXCONNECTIONSLOWERLIMIT
+            value: "{{ typha_max_connections_lower_limit }}"
+{% if typha_secure %}
+          - name: TYPHA_CAFILE
+            value: /etc/ca/ca.crt
+          - name: TYPHA_CLIENTCN
+            value: typha-client
+          - name: TYPHA_SERVERCERTFILE
+            value: /etc/typha/server_certificate.pem
+          - name: TYPHA_SERVERKEYFILE
+            value: /etc/typha/server_key.pem
+{% endif %}
+{% if typha_prometheusmetricsenabled %}
+          # Since Typha is host-networked,
+          # this opens a port on the host, which may need to be secured.
+          - name: TYPHA_PROMETHEUSMETRICSENABLED
+            value: "true"
+          - name: TYPHA_PROMETHEUSMETRICSPORT
+            value: "{{ typha_prometheusmetricsport }}"
+{% endif %}
+{% if calico_ipam_host_local %}
+          - name: USE_POD_CIDR
+            value: "true"
+{% endif %}
+{% if typha_secure %}
+        volumeMounts:
+          - mountPath: /etc/typha
+            name: typha-server
+            readOnly: true
+          - mountPath: /etc/ca/ca.crt
+            subPath: ca.crt
+            name: cacert
+            readOnly: true
+{% endif %}
+        livenessProbe:
+          httpGet:
+            path: /liveness
+            port: 9098
+            host: localhost
+          periodSeconds: 30
+          initialDelaySeconds: 30
+        readinessProbe:
+          httpGet:
+            path: /readiness
+            port: 9098
+            host: localhost
+          periodSeconds: 10
+{% if typha_secure %}
+      volumes:
+        - name: typha-server
+          secret:
+            secretName: typha-server
+            items:
+              - key: tls.crt
+                path: server_certificate.pem
+              - key: tls.key
+                path: server_key.pem
+        - name: cacert
+          hostPath:
+            path: "{{ kube_cert_dir }}"
+{% endif %}
+
+---
+
+# This manifest creates a Pod Disruption Budget for Typha to allow K8s Cluster Autoscaler to evict
+
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: calico-typha
+  namespace: kube-system
+  labels:
+    k8s-app: calico-typha
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      k8s-app: calico-typha

internal/resources/kubespray/project/roles/network_plugin/calico/templates/calicoctl.etcd.sh.j2

@@ -0,0 +1,6 @@
+#!/bin/bash
+ETCD_ENDPOINTS={{ etcd_access_addresses }} \
+ETCD_CA_CERT_FILE={{ calico_cert_dir }}/ca_cert.crt \
+ETCD_CERT_FILE={{ calico_cert_dir }}/cert.crt \
+ETCD_KEY_FILE={{ calico_cert_dir }}/key.pem \
+{{ bin_dir }}/calicoctl --allow-version-mismatch "$@"

internal/resources/kubespray/project/roles/network_plugin/calico/templates/calicoctl.kdd.sh.j2

@@ -0,0 +1,8 @@
+#!/bin/bash
+DATASTORE_TYPE=kubernetes \
+{% if inventory_hostname in groups['kube_control_plane'] %}
+KUBECONFIG=/etc/kubernetes/admin.conf \
+{% else %}
+KUBECONFIG=/etc/cni/net.d/calico-kubeconfig \
+{% endif %}
+{{ bin_dir }}/calicoctl --allow-version-mismatch "$@"

internal/resources/kubespray/project/roles/network_plugin/calico/templates/kubernetes-services-endpoint.yml.j2

@@ -0,0 +1,11 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  namespace: kube-system
+  name: kubernetes-services-endpoint
+data:
+{% if calico_bpf_enabled %}
+  KUBERNETES_SERVICE_HOST: "{{ kube_apiserver_global_endpoint | urlsplit('hostname') }}"
+  KUBERNETES_SERVICE_PORT: "{{ kube_apiserver_global_endpoint | urlsplit('port') }}"
+{% endif %}

internal/resources/kubespray/project/roles/network_plugin/calico/templates/make-ssl-calico.sh.j2

@@ -0,0 +1,102 @@
+#!/bin/bash
+
+# Author: Smana smainklh@gmail.com
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o pipefail
+usage()
+{
+    cat << EOF
+Create self signed certificates
+
+Usage : $(basename $0) -f <config> [-d <ssldir>]
+      -h | --help         : Show this message
+      -f | --config       : Openssl configuration file
+      -d | --ssldir       : Directory where the certificates will be installed
+      -c | --cadir        : Directory where the existing CA is located
+      -s | --service      : Service for the ca
+
+               ex :
+               $(basename $0) -f openssl.conf -d /srv/ssl
+EOF
+}
+
+# Options parsing
+while (($#)); do
+    case "$1" in
+        -h | --help)   usage;   exit 0;;
+        -f | --config) CONFIG=${2}; shift 2;;
+        -d | --ssldir) SSLDIR="${2}"; shift 2;;
+        -c | --cadir) CADIR="${2}"; shift 2;;
+        -s | --service) SERVICE="${2}"; shift 2;;
+        *)
+            usage
+            echo "ERROR : Unknown option"
+            exit 3
+        ;;
+    esac
+done
+
+if [ -z ${CONFIG} ]; then
+    echo "ERROR: the openssl configuration file is missing. option -f"
+    exit 1
+fi
+if [ -z ${SSLDIR} ]; then
+    SSLDIR="/etc/calico/certs"
+fi
+
+tmpdir=$(mktemp -d /tmp/calico_${SERVICE}_certs.XXXXXX)
+trap 'rm -rf "${tmpdir}"' EXIT
+cd "${tmpdir}"
+
+mkdir -p ${SSLDIR} ${CADIR}
+
+# Root CA
+if [ -e "$CADIR/ca.key" ]; then
+  # Reuse existing CA
+  cp $CADIR/{ca.crt,ca.key} .
+else
+  openssl genrsa -out ca.key {{certificates_key_size}} > /dev/null 2>&1
+  openssl req -x509 -new -nodes -key ca.key -days {{certificates_duration}}  -out ca.crt -subj "/CN=calico-${SERVICE}-ca" > /dev/null 2>&1
+fi
+
+if [ $SERVICE == "typha" ]; then
+    # Typha server
+    openssl genrsa -out typha-server.key {{certificates_key_size}} > /dev/null 2>&1
+    openssl req -new -key typha-server.key -out typha-server.csr -subj "/CN=typha-server" -config ${CONFIG} > /dev/null 2>&1
+    openssl x509 -req -in typha-server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out typha-server.crt -days {{certificates_duration}}  -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1
+
+    # Typha client
+    openssl genrsa -out typha-client.key {{certificates_key_size}} > /dev/null 2>&1
+    openssl req -new -key typha-client.key -out typha-client.csr -subj "/CN=typha-client" -config ${CONFIG} > /dev/null 2>&1
+    openssl x509 -req -in typha-client.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out typha-client.crt -days {{certificates_duration}}  -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1
+
+elif [ $SERVICE == "apiserver" ]; then
+    # calico-apiserver
+    openssl genrsa -out apiserver.key {{certificates_key_size}} > /dev/null 2>&1
+    openssl req -new -key apiserver.key -out apiserver.csr -subj "/CN=calico-apiserver" -config ${CONFIG} > /dev/null 2>&1
+    openssl x509 -req -in apiserver.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out apiserver.crt -days {{certificates_duration}}  -extensions ssl_client_apiserver -extfile ${CONFIG} > /dev/null 2>&1
+else
+    echo "ERROR: the openssl configuration file is missing. option -s"
+    exit 1
+fi
+
+# Install certs
+if [ -e "$CADIR/ca.key" ]; then
+    # No pass existing CA
+    rm -f ca.crt ca.key
+fi
+
+mv {*.crt,*.key} ${SSLDIR}/

internal/resources/kubespray/project/roles/network_plugin/calico/vars/amazon.yml

@@ -0,0 +1,5 @@
+---
+calico_wireguard_repo: https://download.copr.fedorainfracloud.org/results/jdoss/wireguard/epel-7-$basearch/
+calico_wireguard_packages:
+  - wireguard-dkms
+  - wireguard-tools

internal/resources/kubespray/project/roles/network_plugin/calico/vars/centos-9.yml

@@ -0,0 +1,3 @@
+---
+calico_wireguard_packages:
+  - wireguard-tools

internal/resources/kubespray/project/roles/network_plugin/calico/vars/debian.yml

@@ -0,0 +1,3 @@
+---
+calico_wireguard_packages:
+  - wireguard

internal/resources/kubespray/project/roles/network_plugin/calico/vars/fedora.yml

@@ -0,0 +1,3 @@
+---
+calico_wireguard_packages:
+  - wireguard-tools

internal/resources/kubespray/project/roles/network_plugin/calico/vars/opensuse.yml

@@ -0,0 +1,3 @@
+---
+calico_wireguard_packages:
+  - wireguard-tools

internal/resources/kubespray/project/roles/network_plugin/calico/vars/redhat-9.yml

@@ -0,0 +1,3 @@
+---
+calico_wireguard_packages:
+  - wireguard-tools

internal/resources/kubespray/project/roles/network_plugin/calico/vars/redhat.yml

@@ -0,0 +1,4 @@
+---
+calico_wireguard_packages:
+  - wireguard-dkms
+  - wireguard-tools

internal/resources/kubespray/project/roles/network_plugin/calico/vars/rocky-9.yml

@@ -0,0 +1,3 @@
+---
+calico_wireguard_packages:
+  - wireguard-tools

internal/resources/kubespray/project/roles/network_plugin/calico_defaults/defaults/main.yml

@@ -0,0 +1,176 @@
+---
+# the default value of name
+calico_cni_name: k8s-pod-network
+
+# Enables Internet connectivity from containers
+nat_outgoing: true
+nat_outgoing_ipv6: false
+
+# add default ippool name
+calico_pool_name: "default-pool"
+calico_ipv4pool_ipip: "Off"
+
+# Change encapsulation mode, by default we enable vxlan which is the most mature and well tested mode
+calico_ipip_mode: Never  # valid values are 'Always', 'Never' and 'CrossSubnet'
+calico_vxlan_mode: Always  # valid values are 'Always', 'Never' and 'CrossSubnet'
+
+calico_cni_pool: true
+calico_cni_pool_ipv6: true
+
+# add default ippool blockSize
+calico_pool_blocksize: 26
+
+# Calico doesn't support ipip tunneling for the IPv6.
+calico_ipip_mode_ipv6: Never
+calico_vxlan_mode_ipv6: Always
+
+# add default ipv6 ippool blockSize
+calico_pool_blocksize_ipv6: 122
+
+# Calico network backend can be 'bird', 'vxlan' and 'none'
+calico_network_backend: vxlan
+
+calico_cert_dir: /etc/calico/certs
+
+# Global as_num (/calico/bgp/v1/global/as_num)
+global_as_num: "64512"
+
+# You can set MTU value here. If left undefined or empty, it will
+# not be specified in calico CNI config, so Calico will use built-in
+# defaults. The value should be a number, not a string.
+# calico_mtu: 1500
+
+# Advertise Service External IPs
+calico_advertise_service_external_ips: []
+
+# Advertise Service LoadBalancer IPs
+calico_advertise_service_loadbalancer_ips: []
+
+# Calico eBPF support
+calico_bpf_enabled: false
+calico_bpf_log_level: ""
+# Valid option for service mode: Tunnel (default), DSR=Direct Server Return
+calico_bpf_service_mode: Tunnel
+
+# Calico floatingIPs support
+# Valid option for floatingIPs: Disabled (default), Enabled
+calico_felix_floating_ips: Disabled
+
+# Limits for apps
+calico_node_memory_limit: 500M
+calico_node_cpu_limit: "0"
+calico_node_memory_requests: 64M
+calico_node_cpu_requests: 150m
+calico_felix_chaininsertmode: Insert
+
+# Calico daemonset nodeselector
+calico_ds_nodeselector: "kubernetes.io/os: linux"
+
+# Virtual network ID to use for VXLAN traffic. A value of 0 means “use the kernel default”.
+calico_vxlan_vni: 4096
+
+# Port to use for VXLAN traffic. A value of 0 means “use the kernel default”.
+calico_vxlan_port: 4789
+
+# Enable Prometheus Metrics endpoint for felix
+calico_felix_prometheusmetricsenabled: false
+calico_felix_prometheusmetricsport: 9091
+calico_felix_prometheusgometricsenabled: true
+calico_felix_prometheusprocessmetricsenabled: true
+
+# Set the agent log level. Can be debug, warning, info or fatal
+calico_loglevel: info
+calico_node_startup_loglevel: error
+
+# Set log path for calico CNI plugin. Set to false to disable logging to disk.
+calico_cni_log_file_path: /var/log/calico/cni/cni.log
+
+# Enable or disable usage report to 'usage.projectcalico.org'
+calico_usage_reporting: false
+
+# Should calico ignore kernel's RPF check setting,
+# see https://github.com/projectcalico/felix/blob/ab8799eaea66627e5db7717e62fca61fd9c08646/python/calico/felix/config.py#L198
+calico_node_ignorelooserpf: false
+
+# Define address on which Felix will respond to health requests
+calico_healthhost: "localhost"
+
+# Configure time in seconds that calico will wait for the iptables lock
+calico_iptables_lock_timeout_secs: 10
+
+# Choose Calico iptables backend: "Legacy", "Auto" or "NFT" (FELIX_IPTABLESBACKEND)
+calico_iptables_backend: "Auto"
+
+# Calico Wireguard support
+calico_wireguard_enabled: false
+calico_wireguard_packages: []
+calico_wireguard_repo: https://download.copr.fedorainfracloud.org/results/jdoss/wireguard/epel-{{ ansible_distribution_major_version }}-$basearch/
+
+# If you want to use non default IP_AUTODETECTION_METHOD, IP6_AUTODETECTION_METHOD for calico node set this option to one of:
+# * can-reach=DESTINATION
+# * interface=INTERFACE-REGEX
+# see https://projectcalico.docs.tigera.io/reference/node/configuration#ip-autodetection-methods
+# calico_ip_auto_method: "interface=eth.*"
+# calico_ip6_auto_method: "interface=eth.*"
+
+# Set FELIX_MTUIFACEPATTERN, Pattern used to discover the host’s interface for MTU auto-detection.
+# see https://projectcalico.docs.tigera.io/reference/felix/configuration
+# calico_felix_mtu_iface_pattern: "^((en|wl|ww|sl|ib)[opsx].*|(eth|wlan|wwan).*)"
+
+calico_baremetal_nodename: "{{ kube_override_hostname | default(inventory_hostname) }}"
+
+kube_etcd_cacert_file: ca.pem
+kube_etcd_cert_file: node-{{ inventory_hostname }}.pem
+kube_etcd_key_file: node-{{ inventory_hostname }}-key.pem
+
+# Choose data store type for calico: "etcd" or "kdd" (kubernetes datastore)
+# The default value for calico_datastore is set in role kubespray-default
+
+# Use typha (only with kdd)
+typha_enabled: false
+typha_prometheusmetricsenabled: false
+typha_prometheusmetricsport: 9093
+
+# Scaling typha: 1 replica per 100 nodes is adequate
+# Number of typha replicas
+typha_replicas: 1
+
+# Set max typha connections
+typha_max_connections_lower_limit: 300
+
+# Generate certifcates for typha<->calico-node communication
+typha_secure: false
+
+calico_feature_control: {}
+
+# Calico default BGP port
+calico_bgp_listen_port: 179
+
+# Calico FelixConfiguration options
+calico_felix_reporting_interval: 0s
+calico_felix_log_severity_screen: Info
+
+# Calico container settings
+calico_allow_ip_forwarding: false
+
+# Calico IPAM strictAffinity
+calico_ipam_strictaffinity: false
+
+# Calico IPAM autoAllocateBlocks
+calico_ipam_autoallocateblocks: true
+
+# Calico IPAM maxBlocksPerHost, default 0
+calico_ipam_maxblocksperhost: 0
+
+# Calico host local IPAM (use node .spec.podCIDR)
+
+calico_ipam_host_local: false
+
+# Calico apiserver (only with kdd)
+calico_apiserver_enabled: false
+
+# Calico feature detect override
+calico_feature_detect_override: ""
+
+# Calico kubeconfig wait timeout in seconds
+calico_kubeconfig_wait_timeout: 300

internal/resources/kubespray/project/roles/network_plugin/cilium/defaults/main.yml

@@ -0,0 +1,354 @@
+---
+cilium_min_version_required: "1.15"
+# Log-level
+cilium_debug: false
+
+cilium_mtu: "0"
+cilium_enable_ipv4: "{{ ipv4_stack }}"
+cilium_enable_ipv6: "{{ ipv6_stack }}"
+
+# Enable l2 announcement from cilium to replace Metallb Ref: https://docs.cilium.io/en/v1.14/network/l2-announcements/
+cilium_l2announcements: false
+
+# Cilium agent health port
+cilium_agent_health_port: "9879"
+
+# Identity allocation mode selects how identities are shared between cilium
+# nodes by setting how they are stored. The options are "crd" or "kvstore".
+# - "crd" stores identities in kubernetes as CRDs (custom resource definition).
+#   These can be queried with:
+#     `kubectl get ciliumid`
+# - "kvstore" stores identities in an etcd kvstore.
+# - In order to support External Workloads, "crd" is required
+#   - Ref: https://docs.cilium.io/en/stable/gettingstarted/external-workloads/#setting-up-support-for-external-workloads-beta
+# - KVStore operations are only required when cilium-operator is running with any of the below options:
+#   - --synchronize-k8s-services
+#   - --synchronize-k8s-nodes
+#   - --identity-allocation-mode=kvstore
+#   - Ref: https://docs.cilium.io/en/stable/internals/cilium_operator/#kvstore-operations
+cilium_identity_allocation_mode: crd
+
+# Etcd SSL dirs
+cilium_cert_dir: /etc/cilium/certs
+kube_etcd_cacert_file: ca.pem
+kube_etcd_cert_file: node-{{ inventory_hostname }}.pem
+kube_etcd_key_file: node-{{ inventory_hostname }}-key.pem
+
+# Limits for apps
+cilium_memory_limit: 500M
+cilium_cpu_limit: 500m
+cilium_memory_requests: 64M
+cilium_cpu_requests: 100m
+
+# Overlay Network Mode
+cilium_tunnel_mode: vxlan
+
+# LoadBalancer Mode (snat/dsr/hybrid) Ref: https://docs.cilium.io/en/stable/network/kubernetes/kubeproxy-free/#dsr-mode
+cilium_loadbalancer_mode: snat
+
+# -- Configure Loadbalancer IP Pools
+cilium_loadbalancer_ip_pools: []
+
+# Optional features
+cilium_enable_prometheus: false
+# Enable if you want to make use of hostPort mappings
+cilium_enable_portmap: false
+# Monitor aggregation level (none/low/medium/maximum)
+cilium_monitor_aggregation: medium
+# Kube Proxy Replacement mode (true/false)
+cilium_kube_proxy_replacement: false
+
+# If not defined `cilium_dns_proxy_enable_transparent_mode`, it will following the Cilium behavior.
+# When Cilium is configured to replace kube-proxy, it automatically enables dnsProxy, which will conflict with nodelocaldns.
+# You can set `false` avoid conflict with nodelocaldns.
+# https://github.com/cilium/cilium/issues/33144
+# cilium_dns_proxy_enable_transparent_mode:
+
+# If upgrading from Cilium < 1.5, you may want to override some of these options
+# to prevent service disruptions. See also:
+# http://docs.cilium.io/en/stable/install/upgrade/#changes-that-may-require-action
+cilium_preallocate_bpf_maps: false
+
+# Auto direct nodes routes can be used to advertise pods routes in your cluster
+# without any tunelling (with `cilium_tunnel_mode` sets to `disabled`).
+# This works only if you have a L2 connectivity between all your nodes.
+# You wil also have to specify the variable `cilium_native_routing_cidr` to
+# make this work. Please refer to the cilium documentation for more
+# information about this kind of setups.
+cilium_auto_direct_node_routes: false
+
+# Allows to explicitly specify the IPv4 CIDR for native routing.
+# When specified, Cilium assumes networking for this CIDR is preconfigured and
+# hands traffic destined for that range to the Linux network stack without
+# applying any SNAT.
+# Generally speaking, specifying a native routing CIDR implies that Cilium can
+# depend on the underlying networking stack to route packets to their
+# destination. To offer a concrete example, if Cilium is configured to use
+# direct routing and the Kubernetes CIDR is included in the native routing CIDR,
+# the user must configure the routes to reach pods, either manually or by
+# setting the auto-direct-node-routes flag.
+cilium_native_routing_cidr: ""
+
+# Allows to explicitly specify the IPv6 CIDR for native routing.
+cilium_native_routing_cidr_ipv6: ""
+
+# Enable transparent network encryption.
+cilium_encryption_enabled: false
+
+# Encryption method. Can be either ipsec or wireguard.
+# Only effective when `cilium_encryption_enabled` is set to true.
+cilium_encryption_type: "ipsec"
+
+# Enable encryption for pure node to node traffic.
+# This option is only effective when `cilium_encryption_type` is set to `wireguard`.
+cilium_encryption_node_encryption: false
+
+# If your kernel or distribution does not support WireGuard, Cilium agent can be configured to fall back on the user-space implementation.
+# When this flag is enabled and Cilium detects that the kernel has no native support for WireGuard,
+# it will fallback on the wireguard-go user-space implementation of WireGuard.
+# This option is only effective when `cilium_encryption_type` is set to `wireguard`.
+cilium_wireguard_userspace_fallback: false
+
+# Enable Bandwidth Manager
+# Cilium’s bandwidth manager supports the kubernetes.io/egress-bandwidth Pod annotation.
+# Bandwidth enforcement currently does not work in combination with L7 Cilium Network Policies.
+# In case they select the Pod at egress, then the bandwidth enforcement will be disabled for those Pods.
+# Bandwidth Manager requires a v5.1.x or more recent Linux kernel.
+cilium_enable_bandwidth_manager: false
+cilium_enable_bandwidth_manager_bbr: false
+
+# IP Masquerade Agent
+# https://docs.cilium.io/en/stable/concepts/networking/masquerading/
+# By default, all packets from a pod destined to an IP address outside of the cilium_native_routing_cidr range are masqueraded
+cilium_ip_masq_agent_enable: false
+
+### A packet sent from a pod to a destination which belongs to any CIDR from the nonMasqueradeCIDRs is not going to be masqueraded
+cilium_non_masquerade_cidrs:
+  - 10.0.0.0/8
+  - 172.16.0.0/12
+  - 192.168.0.0/16
+  - 100.64.0.0/10
+  - 192.0.0.0/24
+  - 192.0.2.0/24
+  - 192.88.99.0/24
+  - 198.18.0.0/15
+  - 198.51.100.0/24
+  - 203.0.113.0/24
+  - 240.0.0.0/4
+### Indicates whether to masquerade traffic to the link local prefix.
+### If the masqLinkLocal is not set or set to false, then 169.254.0.0/16 is appended to the non-masquerade CIDRs list.
+cilium_masq_link_local: false
+cilium_masq_link_local_ipv6: false
+### A time interval at which the agent attempts to reload config from disk
+cilium_ip_masq_resync_interval: 60s
+
+# Hubble
+### Enable Hubble without install
+cilium_enable_hubble: false
+### Enable Hubble-ui
+cilium_enable_hubble_ui: "{{ cilium_enable_hubble }}"
+### Enable Hubble Metrics (deprecated)
+cilium_enable_hubble_metrics: false
+### if cilium_enable_hubble_metrics: true
+cilium_hubble_metrics: []
+# - dns
+# - drop
+# - tcp
+# - flow
+# - icmp
+# - http
+### Enable Hubble install
+cilium_hubble_install: false
+### Enable auto generate certs if cilium_hubble_install: true
+cilium_hubble_tls_generate: false
+
+cilium_hubble_export_file_max_backups: "5"
+cilium_hubble_export_file_max_size_mb: "10"
+
+cilium_hubble_export_dynamic_enabled: false
+cilium_hubble_export_dynamic_config_content:
+  - name: all
+    fieldMask: []
+    includeFilters: []
+    excludeFilters: []
+    filePath: "/var/run/cilium/hubble/events.log"
+
+### Capacity of Hubble events buffer. The provided value must be one less than an integer power of two and no larger than 65535
+### (ie: 1, 3, ..., 2047, 4095, ..., 65535) (default 4095)
+# cilium_hubble_event_buffer_capacity: 4095
+### Buffer size of the channel to receive monitor events.
+# cilium_hubble_event_queue_size: 50
+
+cilium_gateway_api_enabled: false
+
+# The default IP address management mode is "Cluster Scope".
+# https://docs.cilium.io/en/stable/concepts/networking/ipam/
+cilium_ipam_mode: cluster-pool
+
+# Cluster Pod CIDRs use the kube_pods_subnet value by default.
+# If your node network is in the same range you will lose connectivity to other nodes.
+# Defaults to kube_pods_subnet if not set.
+# cilium_pool_cidr: 10.233.64.0/18
+
+# When cilium_enable_ipv6 is used, you need to set the IPV6 value.  Defaults to kube_pods_subnet_ipv6 if not set.
+# cilium_pool_cidr_ipv6: fd85:ee78:d8a6:8607::1:0000/112
+
+# When cilium IPAM uses the "Cluster Scope" mode, it will pre-allocate a segment of IP to each node,
+# schedule the Pod to this node, and then allocate IP from here. cilium_pool_mask_size Specifies
+# the size allocated from cluster Pod CIDR to node.ipam.podCIDRs
+# Defaults to kube_network_node_prefix if not set.
+# cilium_pool_mask_size: "24"
+
+# cilium_pool_mask_size Specifies the size allocated to node.ipam.podCIDRs from cluster Pod IPV6 CIDR
+# Defaults to kube_network_node_prefix_ipv6 if not set.
+# cilium_pool_mask_size_ipv6: "120"
+
+
+# Extra arguments for the Cilium agent
+cilium_agent_custom_args: [] # deprecated
+cilium_agent_extra_args: []
+
+# For adding and mounting extra volumes to the cilium agent
+cilium_agent_extra_volumes: []
+cilium_agent_extra_volume_mounts: []
+
+cilium_agent_extra_env_vars: []
+
+cilium_operator_replicas: 2
+
+# The address at which the cillium operator bind health check api
+cilium_operator_api_serve_addr: "127.0.0.1:9234"
+
+## A dictionary of extra config variables to add to cilium-config, formatted like:
+##  cilium_config_extra_vars:
+##    var1: "value1"
+##    var2: "value2"
+cilium_config_extra_vars: {}
+
+# For adding and mounting extra volumes to the cilium operator
+cilium_operator_extra_volumes: []
+cilium_operator_extra_volume_mounts: []
+
+# Extra arguments for the Cilium Operator
+cilium_operator_custom_args: [] # deprecated
+cilium_operator_extra_args: []
+
+# Tolerations of the cilium operator
+cilium_operator_tolerations:
+  - operator: "Exists"
+
+# Unique ID of the cluster. Must be unique across all connected
+# clusters and in the range of 1 to 255. Only required for Cluster Mesh,
+# may be 0 if Cluster Mesh is not used.
+cilium_cluster_id: 0
+# Name of the cluster. Only relevant when building a mesh of clusters.
+# The "default" name cannot be used if the Cluster ID is different from 0.
+cilium_cluster_name: default
+
+# Make Cilium take ownership over the `/etc/cni/net.d` directory on the node, renaming all non-Cilium CNI configurations to `*.cilium_bak`.
+# This ensures no Pods can be scheduled using other CNI plugins during Cilium agent downtime.
+# Available for Cilium v1.10 and up.
+cilium_cni_exclusive: true
+
+# Configure the log file for CNI logging with retention policy of 7 days.
+# Disable CNI file logging by setting this field to empty explicitly.
+# Available for Cilium v1.12 and up.
+cilium_cni_log_file: "/var/run/cilium/cilium-cni.log"
+
+# -- Configure cgroup related configuration
+# -- Enable auto mount of cgroup2 filesystem.
+# When `cilium_cgroup_auto_mount` is enabled, cgroup2 filesystem is mounted at
+# `cilium_cgroup_host_root` path on the underlying host and inside the cilium agent pod.
+# If users disable `cilium_cgroup_auto_mount`, it's expected that users have mounted
+# cgroup2 filesystem at the specified `cilium_cgroup_auto_mount` volume, and then the
+# volume will be mounted inside the cilium agent pod at the same path.
+# Available for Cilium v1.11 and up
+cilium_cgroup_auto_mount: true
+# -- Configure cgroup root where cgroup2 filesystem is mounted on the host
+cilium_cgroup_host_root: "/run/cilium/cgroupv2"
+
+# Specifies the ratio (0.0-1.0) of total system memory to use for dynamic
+# sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps.
+cilium_bpf_map_dynamic_size_ratio: "0.0025"
+
+# -- Enables masquerading of IPv4 traffic leaving the node from endpoints.
+# Available for Cilium v1.10 and up
+cilium_enable_ipv4_masquerade: true
+# -- Enables masquerading of IPv6 traffic leaving the node from endpoints.
+# Available for Cilium v1.10 and up
+cilium_enable_ipv6_masquerade: true
+
+# -- Enable native IP masquerade support in eBPF
+cilium_enable_bpf_masquerade: false
+
+# -- Configure whether direct routing mode should route traffic via
+# host stack (true) or directly and more efficiently out of BPF (false) if
+# the kernel supports it. The latter has the implication that it will also
+# bypass netfilter in the host namespace.
+cilium_enable_host_legacy_routing: false
+
+# -- Enable use of the remote node identity.
+# ref: https://docs.cilium.io/en/v1.7/install/upgrade/#configmap-remote-node-identity
+cilium_enable_remote_node_identity: true
+
+# -- Enable the use of well-known identities.
+cilium_enable_well_known_identities: false
+
+# The monitor aggregation flags determine which TCP flags which, upon the
+# first observation, cause monitor notifications to be generated.
+#
+# Only effective when monitor aggregation is set to "medium" or higher.
+cilium_monitor_aggregation_flags: "all"
+
+cilium_enable_bpf_clock_probe: true
+
+# -- Enable BGP Control Plane
+cilium_enable_bgp_control_plane: false
+
+
+# -- Configure BGP Instances (New bgpv2 API v1.16+)
+cilium_bgp_cluster_configs: []
+
+# -- Configure BGP Peers (New bgpv2 API v1.16+)
+cilium_bgp_peer_configs: []
+
+# -- Configure BGP Advertisements (New bgpv2 API v1.16+)
+cilium_bgp_advertisements: []
+
+# -- Configure BGP Node Config Overrides (New bgpv2 API v1.16+)
+cilium_bgp_node_config_overrides: []
+
+# -- Configure BGP Peers (Legacy < v1.16)
+cilium_bgp_peering_policies: []
+
+# -- Whether to enable CNP status updates.
+cilium_disable_cnp_status_updates: true
+
+# Configure how long to wait for the Cilium DaemonSet to be ready again
+cilium_rolling_restart_wait_retries_count: 30
+cilium_rolling_restart_wait_retries_delay_seconds: 10
+
+# Cilium changed the default metrics exporter ports in 1.12
+cilium_agent_scrape_port: "9962"
+cilium_operator_scrape_port: "9963"
+cilium_hubble_scrape_port: "9965"
+
+# Cilium certgen args for generate certificate for hubble mTLS
+cilium_certgen_args:
+  cilium-namespace: kube-system
+  ca-reuse-secret: true
+  ca-secret-name: hubble-ca-secret
+  ca-generate: true
+  ca-validity-duration: 94608000s
+  hubble-server-cert-generate: true
+  hubble-server-cert-common-name: '*.{{ cilium_cluster_name }}.hubble-grpc.cilium.io'
+  hubble-server-cert-validity-duration: 94608000s
+  hubble-server-cert-secret-name: hubble-server-certs
+  hubble-relay-client-cert-generate: true
+  hubble-relay-client-cert-common-name: '*.{{ cilium_cluster_name }}.hubble-grpc.cilium.io'
+  hubble-relay-client-cert-validity-duration: 94608000s
+  hubble-relay-client-cert-secret-name: hubble-relay-client-certs
+  hubble-relay-server-cert-generate: false
+
+cilium_enable_host_firewall: false
+cilium_policy_audit_mode: false

internal/resources/kubespray/project/roles/network_plugin/cilium/tasks/apply.yml

@@ -0,0 +1,229 @@
+---
+- name: Cilium | Install
+  command: "{{ bin_dir }}/cilium install --version {{ cilium_version }} -f {{ kube_config_dir }}/cilium-values.yaml"
+  when: inventory_hostname == groups['kube_control_plane'][0]
+
+- name: Cilium | Wait for pods to run
+  command: "{{ kubectl }} -n kube-system get pods -l k8s-app=cilium -o jsonpath='{.items[?(@.status.containerStatuses[0].ready==false)].metadata.name}'"  # noqa literal-compare
+  register: pods_not_ready
+  until: pods_not_ready.stdout.find("cilium")==-1
+  retries: "{{ cilium_rolling_restart_wait_retries_count | int }}"
+  delay: "{{ cilium_rolling_restart_wait_retries_delay_seconds | int }}"
+  failed_when: false
+  when: inventory_hostname == groups['kube_control_plane'][0]
+
+- name: Cilium | Wait for CiliumLoadBalancerIPPool CRD to be present
+  command: "{{ kubectl }} wait --for condition=established --timeout=60s crd/ciliumloadbalancerippools.cilium.io"
+  register: cillium_lbippool_crd_ready
+  retries: "{{ cilium_rolling_restart_wait_retries_count | int }}"
+  delay: "{{ cilium_rolling_restart_wait_retries_delay_seconds | int }}"
+  failed_when: false
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - cilium_loadbalancer_ip_pools is defined and (cilium_loadbalancer_ip_pools|length>0)
+
+- name: Cilium | Create CiliumLoadBalancerIPPool manifests
+  template:
+    src: "{{ item.name }}/{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.name }}-{{ item.file }}"
+    mode: "0644"
+  with_items:
+    - {name: cilium, file: cilium-loadbalancer-ip-pool.yml, type: CiliumLoadBalancerIPPool}
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - cillium_lbippool_crd_ready is defined and cillium_lbippool_crd_ready.rc is defined and cillium_lbippool_crd_ready.rc == 0
+    - cilium_loadbalancer_ip_pools is defined and (cilium_loadbalancer_ip_pools|length>0)
+
+- name: Cilium | Apply CiliumLoadBalancerIPPool from cilium_loadbalancer_ip_pools
+  kube:
+    name: "{{ item.name }}"
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: "{{ item.type }}"
+    filename: "{{ kube_config_dir }}/{{ item.name }}-{{ item.file }}"
+    state: "latest"
+  loop:
+    - {name: cilium, file: cilium-loadbalancer-ip-pool.yml, type: CiliumLoadBalancerIPPool}
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - cillium_lbippool_crd_ready is defined and cillium_lbippool_crd_ready.rc is defined and cillium_lbippool_crd_ready.rc == 0
+    - cilium_loadbalancer_ip_pools is defined and (cilium_loadbalancer_ip_pools|length>0)
+
+- name: Cilium | Wait for CiliumBGPPeeringPolicy CRD to be present
+  command: "{{ kubectl }} wait --for condition=established --timeout=60s crd/ciliumbgppeeringpolicies.cilium.io"
+  register: cillium_bgpppolicy_crd_ready
+  retries: "{{ cilium_rolling_restart_wait_retries_count | int }}"
+  delay: "{{ cilium_rolling_restart_wait_retries_delay_seconds | int }}"
+  failed_when: false
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - cilium_bgp_peering_policies is defined and (cilium_bgp_peering_policies|length>0)
+
+- name: Cilium | Create CiliumBGPPeeringPolicy manifests
+  template:
+    src: "{{ item.name }}/{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.name }}-{{ item.file }}"
+    mode: "0644"
+  with_items:
+    - {name: cilium, file: cilium-bgp-peering-policy.yml, type: CiliumBGPPeeringPolicy}
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - cillium_bgpppolicy_crd_ready is defined and cillium_bgpppolicy_crd_ready.rc is defined and cillium_bgpppolicy_crd_ready.rc == 0
+    - cilium_bgp_peering_policies is defined and (cilium_bgp_peering_policies|length>0)
+
+- name: Cilium | Apply CiliumBGPPeeringPolicy from cilium_bgp_peering_policies
+  kube:
+    name: "{{ item.name }}"
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: "{{ item.type }}"
+    filename: "{{ kube_config_dir }}/{{ item.name }}-{{ item.file }}"
+    state: "latest"
+  loop:
+    - {name: cilium, file: cilium-bgp-peering-policy.yml, type: CiliumBGPPeeringPolicy}
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - cillium_bgpppolicy_crd_ready is defined and cillium_bgpppolicy_crd_ready.rc is defined and cillium_bgpppolicy_crd_ready.rc == 0
+    - cilium_bgp_peering_policies is defined and (cilium_bgp_peering_policies|length>0)
+
+- name: Cilium | Wait for CiliumBGPClusterConfig CRD to be present
+  command: "{{ kubectl }} wait --for condition=established --timeout=60s crd/ciliumbgpclusterconfigs.cilium.io"
+  register: cillium_bgpcconfig_crd_ready
+  retries: "{{ cilium_rolling_restart_wait_retries_count | int }}"
+  delay: "{{ cilium_rolling_restart_wait_retries_delay_seconds | int }}"
+  failed_when: false
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - cilium_bgp_cluster_configs is defined and (cilium_bgp_cluster_configs|length>0)
+
+- name: Cilium | Create CiliumBGPClusterConfig manifests
+  template:
+    src: "{{ item.name }}/{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.name }}-{{ item.file }}"
+    mode: "0644"
+  with_items:
+    - {name: cilium, file: cilium-bgp-cluster-config.yml, type: CiliumBGPClusterConfig}
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - cillium_bgpcconfig_crd_ready is defined and cillium_bgpcconfig_crd_ready.rc is defined and cillium_bgpcconfig_crd_ready.rc == 0
+    - cilium_bgp_cluster_configs is defined and (cilium_bgp_cluster_configs|length>0)
+
+- name: Cilium | Apply CiliumBGPClusterConfig from cilium_bgp_cluster_configs
+  kube:
+    name: "{{ item.name }}"
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: "{{ item.type }}"
+    filename: "{{ kube_config_dir }}/{{ item.name }}-{{ item.file }}"
+    state: "latest"
+  loop:
+    - {name: cilium, file: cilium-bgp-cluster-config.yml, type: CiliumBGPClusterConfig}
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - cillium_bgpcconfig_crd_ready is defined and cillium_bgpcconfig_crd_ready.rc is defined and cillium_bgpcconfig_crd_ready.rc == 0
+    - cilium_bgp_cluster_configs is defined and (cilium_bgp_cluster_configs|length>0)
+
+- name: Cilium | Wait for CiliumBGPPeerConfig CRD to be present
+  command: "{{ kubectl }} wait --for condition=established --timeout=60s crd/ciliumbgppeerconfigs.cilium.io"
+  register: cillium_bgppconfig_crd_ready
+  retries: "{{ cilium_rolling_restart_wait_retries_count | int }}"
+  delay: "{{ cilium_rolling_restart_wait_retries_delay_seconds | int }}"
+  failed_when: false
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - cilium_bgp_peer_configs is defined and (cilium_bgp_peer_configs|length>0)
+
+- name: Cilium | Create CiliumBGPPeerConfig manifests
+  template:
+    src: "{{ item.name }}/{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.name }}-{{ item.file }}"
+    mode: "0644"
+  with_items:
+    - {name: cilium, file: cilium-bgp-peer-config.yml, type: CiliumBGPPeerConfig}
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - cillium_bgppconfig_crd_ready is defined and cillium_bgppconfig_crd_ready.rc is defined and cillium_bgppconfig_crd_ready.rc == 0
+    - cilium_bgp_peer_configs is defined and (cilium_bgp_peer_configs|length>0)
+
+- name: Cilium | Apply CiliumBGPPeerConfig from cilium_bgp_peer_configs
+  kube:
+    name: "{{ item.name }}"
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: "{{ item.type }}"
+    filename: "{{ kube_config_dir }}/{{ item.name }}-{{ item.file }}"
+    state: "latest"
+  loop:
+    - {name: cilium, file: cilium-bgp-peer-config.yml, type: CiliumBGPPeerConfig}
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - cillium_bgppconfig_crd_ready is defined and cillium_bgppconfig_crd_ready.rc is defined and cillium_bgppconfig_crd_ready.rc == 0
+    - cilium_bgp_peer_configs is defined and (cilium_bgp_peer_configs|length>0)
+
+- name: Cilium | Wait for CiliumBGPAdvertisement CRD to be present
+  command: "{{ kubectl }} wait --for condition=established --timeout=60s crd/ciliumbgpadvertisements.cilium.io"
+  register: cillium_bgpadvert_crd_ready
+  retries: "{{ cilium_rolling_restart_wait_retries_count | int }}"
+  delay: "{{ cilium_rolling_restart_wait_retries_delay_seconds | int }}"
+  failed_when: false
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - cilium_bgp_advertisements is defined and (cilium_bgp_advertisements|length>0)
+
+- name: Cilium | Create CiliumBGPAdvertisement manifests
+  template:
+    src: "{{ item.name }}/{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.name }}-{{ item.file }}"
+    mode: "0644"
+  with_items:
+    - {name: cilium, file: cilium-bgp-advertisement.yml, type: CiliumBGPAdvertisement}
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - cillium_bgpadvert_crd_ready is defined and cillium_bgpadvert_crd_ready.rc is defined and cillium_bgpadvert_crd_ready.rc == 0
+    - cilium_bgp_advertisements is defined and (cilium_bgp_advertisements|length>0)
+
+- name: Cilium | Apply CiliumBGPAdvertisement from cilium_bgp_advertisements
+  kube:
+    name: "{{ item.name }}"
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: "{{ item.type }}"
+    filename: "{{ kube_config_dir }}/{{ item.name }}-{{ item.file }}"
+    state: "latest"
+  loop:
+    - {name: cilium, file: cilium-bgp-advertisement.yml, type: CiliumBGPAdvertisement}
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - cillium_bgpadvert_crd_ready is defined and cillium_bgpadvert_crd_ready.rc is defined and cillium_bgpadvert_crd_ready.rc == 0
+    - cilium_bgp_advertisements is defined and (cilium_bgp_advertisements|length>0)
+
+- name: Cilium | Wait for CiliumBGPNodeConfigOverride CRD to be present
+  command: "{{ kubectl }} wait --for condition=established --timeout=60s crd/ciliumbgpnodeconfigoverrides.cilium.io"
+  register: cilium_bgp_node_config_crd_ready
+  retries: "{{ cilium_rolling_restart_wait_retries_count | int }}"
+  delay: "{{ cilium_rolling_restart_wait_retries_delay_seconds | int }}"
+  failed_when: false
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - cilium_bgp_advertisements is defined and (cilium_bgp_advertisements|length>0)
+
+- name: Cilium | Create CiliumBGPNodeConfigOverride manifests
+  template:
+    src: "{{ item.name }}/{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.name }}-{{ item.file }}"
+    mode: "0644"
+  with_items:
+    - {name: cilium, file: cilium-bgp-node-config-override.yml, type: CiliumBGPNodeConfigOverride}
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - cilium_bgp_node_config_crd_ready is defined and cilium_bgp_node_config_crd_ready.rc is defined and cilium_bgp_node_config_crd_ready.rc == 0
+    - cilium_bgp_node_config_overrides is defined and (cilium_bgp_node_config_overrides|length>0)
+
+- name: Cilium | Apply CiliumBGPNodeConfigOverride from cilium_bgp_node_config_overrides
+  kube:
+    name: "{{ item.name }}"
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: "{{ item.type }}"
+    filename: "{{ kube_config_dir }}/{{ item.name }}-{{ item.file }}"
+    state: "latest"
+  loop:
+    - {name: cilium, file: cilium-bgp-node-config-override.yml, type: CiliumBGPNodeConfigOverride}
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - cilium_bgp_node_config_crd_ready is defined and cilium_bgp_node_config_crd_ready.rc is defined and cilium_bgp_node_config_crd_ready.rc == 0
+    - cilium_bgp_node_config_overrides is defined and (cilium_bgp_node_config_overrides|length>0)

internal/resources/kubespray/project/roles/network_plugin/cilium/tasks/check.yml

@@ -0,0 +1,69 @@
+---
+- name: Cilium | Check Cilium encryption `cilium_ipsec_key` for ipsec
+  assert:
+    that:
+      - "cilium_ipsec_key is defined"
+    msg: "cilium_ipsec_key should be defined to enable encryption using ipsec"
+  when:
+    - cilium_encryption_enabled
+    - cilium_encryption_type == "ipsec"
+    - cilium_tunnel_mode in ['vxlan']
+
+# TODO: Clean this task up when we drop backward compatibility support for `cilium_ipsec_enabled`
+- name: Stop if `cilium_ipsec_enabled` is defined and `cilium_encryption_type` is not `ipsec`
+  assert:
+    that: cilium_encryption_type == 'ipsec'
+    msg: >
+      It is not possible to use `cilium_ipsec_enabled` when `cilium_encryption_type` is set to {{ cilium_encryption_type }}.
+  when:
+    - cilium_ipsec_enabled is defined
+    - cilium_ipsec_enabled
+    - kube_network_plugin == 'cilium' or cilium_deploy_additionally
+
+- name: Stop if kernel version is too low for Cilium Wireguard encryption
+  assert:
+    that: ansible_kernel.split('-')[0] is version('5.6.0', '>=')
+  when:
+    - kube_network_plugin == 'cilium' or cilium_deploy_additionally
+    - cilium_encryption_enabled
+    - cilium_encryption_type == "wireguard"
+    - not ignore_assert_errors
+
+- name: Stop if bad Cilium identity allocation mode
+  assert:
+    that: cilium_identity_allocation_mode in ['crd', 'kvstore']
+    msg: "cilium_identity_allocation_mode must be either 'crd' or 'kvstore'"
+
+- name: Stop if bad Cilium Cluster ID
+  assert:
+    that:
+      - cilium_cluster_id <= 255
+      - cilium_cluster_id >= 0
+    msg: "'cilium_cluster_id' must be between 1 and 255"
+  when: cilium_cluster_id is defined
+
+- name: Stop if bad encryption type
+  assert:
+    that: cilium_encryption_type in ['ipsec', 'wireguard']
+    msg: "cilium_encryption_type must be either 'ipsec' or 'wireguard'"
+  when: cilium_encryption_enabled
+
+- name: Stop if cilium_version is < {{ cilium_min_version_required }}
+  assert:
+    that: cilium_version is version(cilium_min_version_required, '>=')
+    msg: "cilium_version is too low. Minimum version {{ cilium_min_version_required }}"
+
+# TODO: Clean this task up when we drop backward compatibility support for `cilium_ipsec_enabled`
+- name: Set `cilium_encryption_type` to "ipsec" and  if `cilium_ipsec_enabled` is true
+  set_fact:
+    cilium_encryption_type: ipsec
+    cilium_encryption_enabled: true
+  when:
+    - cilium_ipsec_enabled is defined
+    - cilium_ipsec_enabled
+
+- name: Stop if cilium_hubble_event_buffer_capacity is not a power of 2 minus 1 and is not between 1 and 65535
+  assert:
+    that: "cilium_hubble_event_buffer_capacity in [1, 3, 7, 15, 31, 63, 127, 255, 511, 1023, 2047, 4095, 8191, 16383, 32767, 65535]"
+    msg: "Error: cilium_hubble_event_buffer_capacity:{{ cilium_hubble_event_buffer_capacity }} is not a power of 2 minus 1 and it should be between 1 and 65535."
+  when: cilium_hubble_event_buffer_capacity is defined

internal/resources/kubespray/project/roles/network_plugin/cilium/tasks/install.yml

@@ -0,0 +1,53 @@
+---
+- name: Cilium | Ensure BPFFS mounted
+  ansible.posix.mount:
+    fstype: bpf
+    path: /sys/fs/bpf
+    src: bpffs
+    state: mounted
+
+- name: Cilium | Create Cilium certs directory
+  file:
+    dest: "{{ cilium_cert_dir }}"
+    state: directory
+    mode: "0750"
+    owner: root
+    group: root
+  when:
+    - cilium_identity_allocation_mode == "kvstore"
+
+- name: Cilium | Link etcd certificates for cilium
+  file:
+    src: "{{ etcd_cert_dir }}/{{ item.s }}"
+    dest: "{{ cilium_cert_dir }}/{{ item.d }}"
+    mode: "0644"
+    state: hard
+    force: true
+  loop:
+    - {s: "{{ kube_etcd_cacert_file }}", d: "ca_cert.crt"}
+    - {s: "{{ kube_etcd_cert_file }}", d: "cert.crt"}
+    - {s: "{{ kube_etcd_key_file }}", d: "key.pem"}
+  when:
+    - cilium_identity_allocation_mode == "kvstore"
+
+- name: Cilium | Enable portmap addon
+  template:
+    src: 000-cilium-portmap.conflist.j2
+    dest: /etc/cni/net.d/000-cilium-portmap.conflist
+    mode: "0644"
+  when: cilium_enable_portmap
+
+- name: Cilium | Render values
+  template:
+    src: values.yaml.j2
+    dest: "{{ kube_config_dir }}/cilium-values.yaml"
+    mode: "0644"
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+
+- name: Cilium | Copy Ciliumcli binary from download dir
+  copy:
+    src: "{{ local_release_dir }}/cilium"
+    dest: "{{ bin_dir }}/cilium"
+    mode: "0755"
+    remote_src: true

internal/resources/kubespray/project/roles/network_plugin/cilium/tasks/main.yml

@@ -0,0 +1,9 @@
+---
+- name: Cilium check
+  import_tasks: check.yml
+
+- name: Cilium install
+  include_tasks: install.yml
+
+- name: Cilium apply
+  include_tasks: apply.yml

internal/resources/kubespray/project/roles/network_plugin/cilium/tasks/reset.yml

@@ -0,0 +1,9 @@
+---
+- name: Reset | check and remove devices if still present
+  include_tasks: reset_iface.yml
+  vars:
+    iface: "{{ item }}"
+  loop:
+    - cilium_host
+    - cilium_net
+    - cilium_vxlan

internal/resources/kubespray/project/roles/network_plugin/cilium/tasks/reset_iface.yml

@@ -0,0 +1,12 @@
+---
+- name: "Reset | check if network device {{ iface }} is present"
+  stat:
+    path: "/sys/class/net/{{ iface }}"
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: device_remains
+
+- name: "Reset | remove network device {{ iface }}"
+  command: "ip link del {{ iface }}"
+  when: device_remains.stat.exists

internal/resources/kubespray/project/roles/network_plugin/cilium/templates/000-cilium-portmap.conflist.j2

@@ -0,0 +1,13 @@
+{
+    "cniVersion": "0.3.1",
+    "name": "cilium-portmap",
+    "plugins": [
+        {
+            "type": "cilium-cni"
+        },
+        {
+            "type": "portmap",
+            "capabilities": { "portMappings": true }
+        }
+    ]
+}

internal/resources/kubespray/project/roles/network_plugin/cilium/templates/cilium/cilium-bgp-advertisement.yml.j2

@@ -0,0 +1,12 @@
+{% for cilium_bgp_advertisement in cilium_bgp_advertisements %}
+---
+apiVersion: "cilium.io/v2alpha1"
+kind: CiliumBGPAdvertisement
+metadata:
+  name: "{{ cilium_bgp_advertisement.name }}"
+{% if cilium_bgp_advertisement.labels %}
+  labels: {{ cilium_bgp_advertisement.labels | to_yaml }}
+{% endif %}
+spec:
+  {{ cilium_bgp_advertisement.spec | to_yaml | indent(4) }}
+{% endfor %}

internal/resources/kubespray/project/roles/network_plugin/cilium/templates/cilium/cilium-bgp-cluster-config.yml.j2

@@ -0,0 +1,9 @@
+{% for cilium_bgp_cluster_config in cilium_bgp_cluster_configs %}
+---
+apiVersion: "cilium.io/v2alpha1"
+kind: CiliumBGPClusterConfig
+metadata:
+  name: "{{ cilium_bgp_cluster_config.name }}"
+spec:
+  {{ cilium_bgp_cluster_config.spec | to_yaml | indent(2) }}
+{% endfor %}

internal/resources/kubespray/project/roles/network_plugin/cilium/templates/cilium/cilium-bgp-node-config-override.yml.j2

@@ -0,0 +1,9 @@
+{% for cilium_bgp_node_config_override in cilium_bgp_node_config_overrides %}
+---
+apiVersion: "cilium.io/v2alpha1"
+kind: CiliumBGPNodeConfigOverride
+metadata:
+  name: "{{ cilium_bgp_node_config_override.name }}"
+spec:
+  {{ cilium_bgp_node_config_override.spec | to_yaml | indent(2) }}
+{% endfor %}

internal/resources/kubespray/project/roles/network_plugin/cilium/templates/cilium/cilium-bgp-peer-config.yml.j2

@@ -0,0 +1,9 @@
+{% for cilium_bgp_peer_config in cilium_bgp_peer_configs %}
+---
+apiVersion: "cilium.io/v2alpha1"
+kind: CiliumBGPPeerConfig
+metadata:
+  name: "{{ cilium_bgp_peer_config.name }}"
+spec:
+  {{ cilium_bgp_peer_config.spec | to_yaml | indent(2) }}
+{% endfor %}

internal/resources/kubespray/project/roles/network_plugin/cilium/templates/cilium/cilium-bgp-peering-policy.yml.j2

@@ -0,0 +1,9 @@
+{% for cilium_bgp_peering_policy in cilium_bgp_peering_policies %}
+---
+apiVersion: "cilium.io/v2alpha1"
+kind: CiliumBGPPeeringPolicy
+metadata:
+  name: "{{ cilium_bgp_peering_policy.name }}"
+spec:
+  {{ cilium_bgp_peering_policy.spec | to_yaml | indent(2) }}
+{% endfor %}

internal/resources/kubespray/project/roles/network_plugin/cilium/templates/cilium/cilium-loadbalancer-ip-pool.yml.j2

@@ -0,0 +1,16 @@
+{% for cilium_loadbalancer_ip_pool in cilium_loadbalancer_ip_pools %}
+---
+apiVersion: "cilium.io/v2alpha1"
+kind: CiliumLoadBalancerIPPool
+metadata:
+  name: "{{ cilium_loadbalancer_ip_pool.name }}"
+spec:
+  blocks:
+{% for cblock in cilium_loadbalancer_ip_pool.cidrs | default([]) %}
+  - cidr: "{{ cblock }}"
+{% endfor %}
+{% for rblock in cilium_loadbalancer_ip_pool.ranges | default([]) %}
+  - start: "{{ rblock.start }}"
+    stop: "{{ rblock.stop | default(rblock.start) }}"
+{% endfor %}
+{% endfor %}

internal/resources/kubespray/project/roles/network_plugin/cilium/templates/values.yaml.j2

@@ -0,0 +1,164 @@
+MTU: {{ cilium_mtu }}
+debug:
+  enabled: {{ cilium_debug }}
+
+image:
+  repository: {{ cilium_image_repo }}
+  tag: {{ cilium_image_tag }}
+
+k8sServiceHost: "auto"
+k8sServicePort: "auto"
+
+ipv4:
+  enabled: {{ cilium_enable_ipv4 }}
+ipv6:
+  enabled: {{ cilium_enable_ipv6 }}
+
+l2announcements:
+  enabled: {{ cilium_l2announcements }}
+
+healthPort: {{ cilium_agent_health_port }}
+
+identityAllocationMode: {{ cilium_identity_allocation_mode }}
+
+tunnelProtocol: {{ cilium_tunnel_mode }}
+
+loadbalancer:
+  mode: {{ cilium_loadbalancer_mode }}
+
+kubeProxyReplacement: {{ cilium_kube_proxy_replacement }}
+
+{% if cilium_dns_proxy_enable_transparent_mode is defined %}
+dnsProxy:
+  enableTransparentMode: {{ cilium_dns_proxy_enable_transparent_mode }}
+{% endif %}
+
+extraVolumes:
+  {{ cilium_agent_extra_volumes | to_nice_yaml(indent=2) | indent(2) }}
+
+extraVolumeMounts:
+  {{ cilium_agent_extra_volume_mounts | to_nice_yaml(indent=2) | indent(2) }}
+
+extraArgs:
+  {{ cilium_agent_extra_args | to_nice_yaml(indent=2) | indent(2) }}
+
+bpf:
+  masquerade: {{ cilium_enable_bpf_masquerade }}
+  hostLegacyRouting: {{ cilium_enable_host_legacy_routing }}
+  monitorAggregation: {{ cilium_monitor_aggregation }}
+  preallocateMaps: {{ cilium_preallocate_bpf_maps }}
+  mapDynamicSizeRatio: {{ cilium_bpf_map_dynamic_size_ratio }}
+
+cni:
+  exclusive: {{ cilium_cni_exclusive }}
+  logFile: {{ cilium_cni_log_file }}
+
+autoDirectNodeRoutes: {{ cilium_auto_direct_node_routes }}
+
+ipv4NativeRoutingCIDR: {{ cilium_native_routing_cidr }}
+ipv6NativeRoutingCIDR: {{ cilium_native_routing_cidr_ipv6 }}
+
+encryption:
+  enabled: {{ cilium_encryption_enabled }}
+  {% if cilium_encryption_enabled %}
+  type: {{ cilium_encryption_type }}
+  {% if cilium_encryption_type == 'wireguard' %}
+  nodeEncryption: {{ cilium_encryption_node_encryption }}
+  {% endif %}
+
+  {% endif %}
+
+bandwidthManager:
+  enabled: {{ cilium_enable_bandwidth_manager }}
+  bbr: {{ cilium_enable_bandwidth_manager_bbr }}
+
+ipMasqAgent:
+  enabled: {{ cilium_ip_masq_agent_enable }}
+  {% if cilium_ip_masq_agent_enable %}
+  config:
+    nonMasqueradeCIDRs: {{ cilium_non_masquerade_cidrs }}
+    masqLinkLocal: {{ cilium_masq_link_local }}
+    masqLinkLocalIPv6: {{ cilium_masq_link_local_ipv6 }}
+  # cilium_ip_masq_resync_interval
+  {% endif %}
+
+hubble:
+  enabled: {{ cilium_enable_hubble }}
+  relay:
+    enabled: {{ cilium_enable_hubble }}
+    image:
+      repository: {{ cilium_hubble_relay_image_repo }}
+      tag: {{ cilium_hubble_relay_image_tag }}
+  ui:
+    enabled: {{ cilium_enable_hubble_ui }}
+    backend:
+      image:
+        repository: {{ cilium_hubble_ui_backend_image_repo }}
+        tag: {{ cilium_hubble_ui_backend_image_tag }}
+    frontend:
+      image:
+        repository: {{ cilium_hubble_ui_image_repo }}
+        tag: {{ cilium_hubble_ui_image_tag }}
+  metrics:
+    enabled: {{ cilium_hubble_metrics }}
+  export:
+    fileMaxBackups: {{ cilium_hubble_export_file_max_backups }}
+    fileMaxSizeMb: {{ cilium_hubble_export_file_max_size_mb }}
+    dynamic:
+      enabled: {{ cilium_hubble_export_dynamic_enabled }}
+      config:
+        content:
+          {{ cilium_hubble_export_dynamic_config_content | to_nice_yaml(indent=10) | indent(10) }}
+
+gatewayAPI:
+  enabled: {{ cilium_gateway_api_enabled }}
+
+ipam:
+  mode: {{ cilium_ipam_mode }}
+  operator:
+    clusterPoolIPv4PodCIDRList:
+      - {{ cilium_pool_cidr | default(kube_pods_subnet) }}
+    clusterPoolIPv4MaskSize: {{ cilium_pool_mask_size | default(kube_network_node_prefix) }}
+
+    clusterPoolIPv6PodCIDRList:
+      - {{ cilium_pool_cidr_ipv6 | default(kube_pods_subnet_ipv6) }}
+    clusterPoolIPv6MaskSize: {{ cilium_pool_mask_size_ipv6 | default(kube_network_node_prefix_ipv6) }}
+
+cgroup:
+  autoMount:
+    enabled: {{ cilium_cgroup_auto_mount }}
+  hostRoot: {{ cilium_cgroup_host_root }}
+
+operator:
+  image:
+    repository: {{ cilium_operator_image_repo }}
+    tag: {{ cilium_operator_image_tag }}
+  replicas: {{ cilium_operator_replicas }}
+  extraArgs:
+    {{ cilium_operator_extra_args | to_nice_yaml(indent=2) | indent(4) }}
+  extraVolumes:
+    {{ cilium_operator_extra_volumes | to_nice_yaml(indent=2) | indent(4) }}
+  extraVolumeMounts:
+    {{ cilium_operator_extra_volume_mounts | to_nice_yaml(indent=2) | indent(4) }}
+  tolerations:
+    {{ cilium_operator_tolerations | to_nice_yaml(indent=2) | indent(4) }}
+
+cluster:
+  id: {{ cilium_cluster_id }}
+  name: {{ cilium_cluster_name }}
+
+enableIPv4Masquerade: {{ cilium_enable_ipv4_masquerade }}
+enableIPv6Masquerade: {{ cilium_enable_ipv6_masquerade }}
+
+hostFirewall:
+  enabled: {{ cilium_enable_host_firewall }}
+
+certgen:
+  image:
+    repositry: {{ cilium_hubble_certgen_image_repo }}
+    tag: {{ cilium_hubble_certgen_image_tag }}
+
+envoy:
+  image:
+    repositry: {{ cilium_hubble_envoy_image_repo }}
+    tag: {{ cilium_hubble_envoy_image_tag }}

internal/resources/kubespray/project/roles/network_plugin/cni/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+cni_bin_owner: "{{ kube_owner }}"

internal/resources/kubespray/project/roles/network_plugin/cni/tasks/main.yml

@@ -0,0 +1,16 @@
+---
+- name: CNI | make sure /opt/cni/bin exists
+  file:
+    path: /opt/cni/bin
+    state: directory
+    mode: "0755"
+    owner: "{{ cni_bin_owner }}"
+    recurse: true
+
+- name: CNI | Copy cni plugins
+  unarchive:
+    src: "{{ downloads.cni.dest }}"
+    dest: "/opt/cni/bin"
+    mode: "0755"
+    owner: "{{ cni_bin_owner }}"
+    remote_src: true

internal/resources/kubespray/project/roles/network_plugin/custom_cni/defaults/main.yml

@@ -0,0 +1,11 @@
+---
+
+custom_cni_manifests: []
+
+custom_cni_chart_namespace: kube-system
+custom_cni_chart_release_name: ""
+custom_cni_chart_repository_name: ""
+custom_cni_chart_repository_url: ""
+custom_cni_chart_ref: ""
+custom_cni_chart_version: ""
+custom_cni_chart_values: {}

internal/resources/kubespray/project/roles/network_plugin/custom_cni/meta/main.yml

@@ -0,0 +1,20 @@
+---
+dependencies:
+  - role: helm-apps
+    when:
+      - inventory_hostname == groups['kube_control_plane'][0]
+      - custom_cni_chart_release_name | length > 0
+    environment:
+      http_proxy: "{{ http_proxy | default('') }}"
+      https_proxy: "{{ https_proxy | default('') }}"
+    release_common_opts: {}
+    releases:
+      - name: "{{ custom_cni_chart_release_name }}"
+        namespace: "{{ custom_cni_chart_namespace }}"
+        chart_ref: "{{ custom_cni_chart_ref }}"
+        chart_version: "{{ custom_cni_chart_version }}"
+        wait: true
+        values: "{{ custom_cni_chart_values }}"
+    repositories:
+      - name: "{{ custom_cni_chart_repository_name }}"
+        url: "{{ custom_cni_chart_repository_url }}"

internal/resources/kubespray/project/roles/network_plugin/custom_cni/tasks/main.yml

@@ -0,0 +1,29 @@
+---
+- name: Custom CNI | Manifest deployment
+  when: not custom_cni_chart_release_name | length > 0
+  block:
+  - name: Custom CNI | Check Custom CNI Manifests
+    assert:
+      that:
+      - "custom_cni_manifests | length > 0"
+      msg: "custom_cni_manifests should not be empty"
+
+  - name: Custom CNI | Copy Custom manifests
+    template:
+      src: "{{ item }}"
+      dest: "{{ kube_config_dir }}/{{ item | basename | replace('.j2', '') }}"
+      mode: "0644"
+    loop: "{{ custom_cni_manifests }}"
+    delegate_to: "{{ groups['kube_control_plane'] | first }}"
+    run_once: true
+
+  - name: Custom CNI | Start Resources
+    kube:
+      namespace: "kube-system"
+      kubectl: "{{ bin_dir }}/kubectl"
+      filename: "{{ kube_config_dir }}/{{ item | basename | replace('.j2', '') }}"
+      state: "latest"
+      wait: true
+    loop: "{{ custom_cni_manifests }}"
+    delegate_to: "{{ groups['kube_control_plane'] | first }}"
+    run_once: true

internal/resources/kubespray/project/roles/network_plugin/flannel/defaults/main.yml

@@ -0,0 +1,28 @@
+---
+# Flannel public IP
+# The address that flannel should advertise as how to access the system
+# Disabled until https://github.com/coreos/flannel/issues/712 is fixed
+# flannel_public_ip: "{{ main_access_ip }}"
+
+## interface that should be used for flannel operations
+## This is actually an inventory cluster-level item
+# flannel_interface:
+
+## Select interface that should be used for flannel operations by regexp on Name or IP
+## This is actually an inventory cluster-level item
+## example: select interface with ip from net 10.0.0.0/23
+## single quote and escape backslashes
+# flannel_interface_regexp: '10\\.0\\.[0-2]\\.\\d{1,3}'
+
+# You can choose what type of flannel backend to use
+# please refer to flannel's docs : https://github.com/coreos/flannel/blob/master/README.md
+flannel_backend_type: "vxlan"
+flannel_vxlan_vni: 1
+flannel_vxlan_port: 8472
+flannel_vxlan_direct_routing: false
+
+# Limits for apps
+flannel_memory_limit: 500M
+flannel_cpu_limit: 300m
+flannel_memory_requests: 64M
+flannel_cpu_requests: 150m

internal/resources/kubespray/project/roles/network_plugin/flannel/meta/main.yml

@@ -0,0 +1,3 @@
+---
+dependencies:
+  - role: network_plugin/cni

internal/resources/kubespray/project/roles/network_plugin/flannel/tasks/main.yml

@@ -0,0 +1,21 @@
+---
+
+- name: Flannel | Stop if kernel version is too low for Flannel Wireguard encryption
+  assert:
+    that: ansible_kernel.split('-')[0] is version('5.6.0', '>=')
+  when:
+    - kube_network_plugin == 'flannel'
+    - flannel_backend_type == 'wireguard'
+    - not ignore_assert_errors
+
+- name: Flannel | Create Flannel manifests
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.file }}"
+    mode: "0644"
+  with_items:
+    - {name: flannel, file: cni-flannel-rbac.yml, type: sa}
+    - {name: kube-flannel, file: cni-flannel.yml, type: ds}
+  register: flannel_node_manifests
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]

internal/resources/kubespray/project/roles/network_plugin/flannel/tasks/reset.yml

@@ -0,0 +1,24 @@
+---
+- name: Reset | check cni network device
+  stat:
+    path: /sys/class/net/cni0
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: cni
+
+- name: Reset | remove the network device created by the flannel
+  command: ip link del cni0
+  when: cni.stat.exists
+
+- name: Reset | check flannel network device
+  stat:
+    path: /sys/class/net/flannel.1
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: flannel
+
+- name: Reset | remove the network device created by the flannel
+  command: ip link del flannel.1
+  when: flannel.stat.exists

internal/resources/kubespray/project/roles/network_plugin/flannel/templates/cni-flannel-rbac.yml.j2

@@ -0,0 +1,52 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: flannel
+  namespace: kube-system
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: flannel
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - nodes/status
+  verbs:
+  - patch
+- apiGroups:
+  - "networking.k8s.io"
+  resources:
+  - clustercidrs
+  verbs:
+  - list
+  - watch
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: flannel
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: flannel
+subjects:
+- kind: ServiceAccount
+  name: flannel
+  namespace: kube-system

internal/resources/kubespray/project/roles/network_plugin/flannel/templates/cni-flannel.yml.j2

@@ -0,0 +1,172 @@
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: kube-flannel-cfg
+  namespace: kube-system
+  labels:
+    tier: node
+    app: flannel
+data:
+  cni-conf.json: |
+    {
+      "name": "cbr0",
+      "cniVersion": "0.3.1",
+      "plugins": [
+        {
+          "type": "flannel",
+          "delegate": {
+            "hairpinMode": true,
+            "isDefaultGateway": true
+          }
+        },
+        {
+          "type": "portmap",
+          "capabilities": {
+            "portMappings": true
+          }
+        }
+      ]
+    }
+  net-conf.json: |
+    {
+{% if ipv4_stack %}
+      "Network": "{{ kube_pods_subnet }}",
+      "EnableIPv4": true,
+{% endif %}
+{% if ipv6_stack %}
+      "EnableIPv6": true,
+      "IPv6Network": "{{ kube_pods_subnet_ipv6 }}",
+{% endif %}
+      "Backend": {
+        "Type": "{{ flannel_backend_type }}"{% if flannel_backend_type == "vxlan" %},
+        "VNI": {{ flannel_vxlan_vni }},
+        "Port": {{ flannel_vxlan_port }},
+        "DirectRouting": {{ flannel_vxlan_direct_routing | to_json }}
+{% endif %}
+      }
+    }
+{% for arch in ['amd64', 'arm64', 'arm', 'ppc64le', 's390x'] %}
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+{% if arch == 'amd64' %}
+  name: kube-flannel
+{% else %}
+  name: kube-flannel-ds-{{ arch }}
+{% endif %}
+  namespace: kube-system
+  labels:
+    tier: node
+    app: flannel
+spec:
+  selector:
+    matchLabels:
+      app: flannel
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: flannel
+    spec:
+      priorityClassName: system-node-critical
+      serviceAccountName: flannel
+      containers:
+      - name: kube-flannel
+        image: {{ flannel_image_repo }}:{{ flannel_image_tag }}
+        imagePullPolicy: {{ k8s_image_pull_policy }}
+        resources:
+          limits:
+            cpu: {{ flannel_cpu_limit }}
+            memory: {{ flannel_memory_limit }}
+          requests:
+            cpu: {{ flannel_cpu_requests }}
+            memory: {{ flannel_memory_requests }}
+        command: [ "/opt/bin/flanneld", "--ip-masq", "--kube-subnet-mgr"{% if flannel_interface is defined %}, "--iface={{ flannel_interface }}"{% endif %}{% if flannel_interface_regexp is defined %}, "--iface-regex={{ flannel_interface_regexp }}"{% endif %} ]
+        securityContext:
+          privileged: false
+          capabilities:
+            add: ["NET_ADMIN", "NET_RAW"]
+        env:
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: EVENT_QUEUE_DEPTH
+          value: "5000"
+        volumeMounts:
+        - name: run
+          mountPath: /run/flannel
+        - name: flannel-cfg
+          mountPath: /etc/kube-flannel/
+        - name: xtables-lock
+          mountPath: /run/xtables.lock
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: kubernetes.io/os
+                    operator: In
+                    values:
+                      - linux
+                  - key: kubernetes.io/arch
+                    operator: In
+                    values:
+                      - {{ arch }}
+      initContainers:
+      - name: install-cni-plugin
+        image: {{ flannel_init_image_repo }}:{{ flannel_init_image_tag }}
+        command:
+        - cp
+        args:
+        - -f
+        - /flannel
+        - /opt/cni/bin/flannel
+        volumeMounts:
+        - name: cni-plugin
+          mountPath: /opt/cni/bin
+      - name: install-cni
+        image: {{ flannel_image_repo }}:{{ flannel_image_tag }}
+        command:
+        - cp
+        args:
+        - -f
+        - /etc/kube-flannel/cni-conf.json
+        - /etc/cni/net.d/10-flannel.conflist
+        volumeMounts:
+        - name: cni
+          mountPath: /etc/cni/net.d
+        - name: flannel-cfg
+          mountPath: /etc/kube-flannel/
+      hostNetwork: true
+      dnsPolicy: ClusterFirstWithHostNet
+      tolerations:
+      - operator: Exists
+      volumes:
+        - name: run
+          hostPath:
+            path: /run/flannel
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
+        - name: flannel-cfg
+          configMap:
+            name: kube-flannel-cfg
+        - name: xtables-lock
+          hostPath:
+            path: /run/xtables.lock
+            type: FileOrCreate
+        - name: cni-plugin
+          hostPath:
+            path: /opt/cni/bin
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: {{ serial | default('20%') }}
+    type: RollingUpdate
+{% endfor %}

internal/resources/kubespray/project/roles/network_plugin/kube-ovn/defaults/main.yml

@@ -0,0 +1,135 @@
+---
+kube_ovn_db_cpu_request: 500m
+kube_ovn_db_memory_request: 200Mi
+kube_ovn_db_cpu_limit: 3000m
+kube_ovn_db_memory_limit: 3000Mi
+kube_ovn_node_cpu_request: 200m
+kube_ovn_node_memory_request: 200Mi
+kube_ovn_node_cpu_limit: 1000m
+kube_ovn_node_memory_limit: 800Mi
+kube_ovn_cni_server_cpu_request: 200m
+kube_ovn_cni_server_memory_request: 200Mi
+kube_ovn_cni_server_cpu_limit: 1000m
+kube_ovn_cni_server_memory_limit: 1Gi
+kube_ovn_controller_cpu_request: 200m
+kube_ovn_controller_memory_request: 200Mi
+kube_ovn_controller_cpu_limit: 1000m
+kube_ovn_controller_memory_limit: 1Gi
+kube_ovn_pinger_cpu_request: 100m
+kube_ovn_pinger_memory_request: 200Mi
+kube_ovn_pinger_cpu_limit: 200m
+kube_ovn_pinger_memory_limit: 400Mi
+kube_ovn_monitor_memory_request: 200Mi
+kube_ovn_monitor_cpu_request: 200m
+kube_ovn_monitor_memory_limit: 200Mi
+kube_ovn_monitor_cpu_limit: 200m
+kube_ovn_dpdk_node_cpu_request: 1000m
+kube_ovn_dpdk_node_memory_request: 2Gi
+kube_ovn_dpdk_node_cpu_limit: 1000m
+kube_ovn_dpdk_node_memory_limit: 2Gi
+
+kube_ovn_central_hosts: "{{ groups['kube_control_plane'] }}"
+kube_ovn_central_replics: "{{ kube_ovn_central_hosts | length }}"
+kube_ovn_controller_replics: "{{ kube_ovn_central_hosts | length }}"
+kube_ovn_central_ips: |-
+  {% for item in kube_ovn_central_hosts -%}
+    {{ hostvars[item]['main_ip'] }}{% if not loop.last %},{% endif %}
+  {%- endfor %}
+
+kube_ovn_ic_enable: false
+kube_ovn_ic_autoroute: true
+kube_ovn_ic_dbhost: "127.0.0.1"
+kube_ovn_ic_zone: "kubernetes"
+
+# geneve or vlan
+kube_ovn_network_type: geneve
+
+# geneve, vxlan or stt. ATTENTION: some networkpolicy cannot take effect when using vxlan and stt need custom compile ovs kernel module
+kube_ovn_tunnel_type: geneve
+
+## The nic to support container network can be a nic name or a group of regex separated by comma e.g: 'enp6s0f0,eth.*', if empty will use the nic that the default route use.
+# kube_ovn_iface: eth1
+## The MTU used by pod iface in overlay networks (default iface MTU - 100)
+# kube_ovn_mtu: 1333
+
+## Enable hw-offload, disable traffic mirror and set the iface to the physical port. Make sure that there is an IP address bind to the physical port.
+kube_ovn_hw_offload: false
+# traffic mirror
+kube_ovn_traffic_mirror: false
+
+# kube_ovn_pool_cidr_ipv6: fd85:ee78:d8a6:8607::1:0000/112
+# kube_ovn_default_interface_name: eth0
+
+kube_ovn_external_address: 8.8.8.8
+kube_ovn_external_address_ipv6: 2400:3200::1
+kube_ovn_external_address_merged: >-
+  {%- if ipv4_stack and ipv6_stack -%}
+  {{ kube_ovn_external_address }},{{ kube_ovn_external_address_ipv6 }}
+  {%- elif ipv4_stack -%}
+  {{ kube_ovn_external_address }}
+  {%- else -%}
+  {{ kube_ovn_external_address_ipv6 }}
+  {%- endif -%}
+
+kube_ovn_external_dns: alauda.cn
+
+# kube_ovn_default_gateway: 10.233.64.1,fd85:ee78:d8a6:8607::1:0
+kube_ovn_default_gateway_check: true
+kube_ovn_default_logical_gateway: false
+
+# u2o_interconnection
+kube_ovn_u2o_interconnection: false
+
+# kube_ovn_default_exclude_ips: 10.16.0.1
+kube_ovn_node_switch_cidr: 100.64.0.0/16
+kube_ovn_node_switch_cidr_ipv6: fd00:100:64::/64
+kube_ovn_node_switch_cidr_merged: >-
+  {%- if ipv4_stack and ipv6_stack -%}
+  {{ kube_ovn_node_switch_cidr }},{{ kube_ovn_node_switch_cidr_ipv6 }}
+  {%- elif ipv4_stack -%}
+  {{ kube_ovn_node_switch_cidr }}
+  {%- else -%}
+  {{ kube_ovn_node_switch_cidr_ipv6 }}
+  {%- endif -%}
+
+## vlan config, set default interface name and vlan id
+# kube_ovn_default_interface_name: eth0
+kube_ovn_default_vlan_id: 100
+kube_ovn_vlan_name: product
+
+## pod nic type, support: veth-pair or internal-port
+kube_ovn_pod_nic_type: veth_pair
+
+## Enable load balancer
+kube_ovn_enable_lb: true
+
+## Enable network policy support
+kube_ovn_enable_np: true
+
+## Enable external vpc support
+kube_ovn_enable_external_vpc: true
+
+## Enable checksum
+kube_ovn_encap_checksum: true
+
+## enable ssl
+kube_ovn_enable_ssl: false
+
+## dpdk
+kube_ovn_dpdk_enabled: false
+kube_ovn_dpdk_tunnel_iface: br-phy
+
+## bind local ip
+kube_ovn_bind_local_ip_enabled: true
+
+## eip snat
+kube_ovn_eip_snat_enabled: true
+
+# ls dnat mod dl dst
+kube_ovn_ls_dnat_mod_dl_dst: true
+
+## keep vm ip
+kube_ovn_keep_vm_ip: true
+
+## cni config priority, default: 01
+kube_ovn_cni_config_priority: '01'

internal/resources/kubespray/project/roles/network_plugin/kube-ovn/tasks/main.yml

@@ -0,0 +1,17 @@
+---
+- name: Kube-OVN | Label ovn-db node
+  command: "{{ kubectl }} label --overwrite node {{ item }} kube-ovn/role=master"
+  loop: "{{ kube_ovn_central_hosts }}"
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+
+- name: Kube-OVN | Create Kube-OVN manifests
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.file }}"
+    mode: "0644"
+  with_items:
+    - {name: kube-ovn-crd, file: cni-kube-ovn-crd.yml}
+    - {name: ovn, file: cni-ovn.yml}
+    - {name: kube-ovn, file: cni-kube-ovn.yml}
+  register: kube_ovn_node_manifests

internal/resources/kubespray/project/roles/network_plugin/kube-ovn/templates/cni-kube-ovn.yml.j2

@@ -0,0 +1,912 @@
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: ovn-vpc-nat-config
+  namespace: kube-system
+  annotations:
+    kubernetes.io/description: |
+      kube-ovn vpc-nat common config
+data:
+  image: {{ kube_ovn_vpc_container_image_repo }}:{{ kube_ovn_vpc_container_image_tag }}
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: ovn-vpc-nat-gw-config
+  namespace: kube-system
+data:
+  enable-vpc-nat-gw: "true"
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kube-ovn-cni
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  annotations:
+    rbac.authorization.k8s.io/system-only: "true"
+  name: system:kube-ovn-cni
+rules:
+  - apiGroups:
+      - "kubeovn.io"
+    resources:
+      - subnets
+      - vlans
+      - provider-networks
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - ""
+      - "kubeovn.io"
+    resources:
+      - ovn-eips
+      - ovn-eips/status
+      - nodes
+      - pods
+      - vlans
+    verbs:
+      - get
+      - list
+      - patch
+      - watch
+  - apiGroups:
+      - "kubeovn.io"
+    resources:
+      - ips
+    verbs:
+      - get
+      - update
+  - apiGroups:
+      - ""
+    resources:
+      - events
+    verbs:
+      - create
+      - patch
+      - update
+  - apiGroups:
+      - ""
+    resources:
+      - configmaps
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - authentication.k8s.io
+    resources:
+      - tokenreviews
+    verbs:
+      - create
+  - apiGroups:
+      - authorization.k8s.io
+    resources:
+      - subjectaccessreviews
+    verbs:
+      - create
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kube-ovn-cni
+roleRef:
+  name: system:kube-ovn-cni
+  kind: ClusterRole
+  apiGroup: rbac.authorization.k8s.io
+subjects:
+  - kind: ServiceAccount
+    name: kube-ovn-cni
+    namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: kube-ovn-cni
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader
+subjects:
+  - kind: ServiceAccount
+    name: kube-ovn-cni
+    namespace: kube-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kube-ovn-app
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  annotations:
+    rbac.authorization.k8s.io/system-only: "true"
+  name: system:kube-ovn-app
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - nodes
+    verbs:
+      - get
+      - list
+  - apiGroups:
+      - apps
+    resources:
+      - daemonsets
+    verbs:
+      - get
+  - apiGroups:
+      - authentication.k8s.io
+    resources:
+      - tokenreviews
+    verbs:
+      - create
+  - apiGroups:
+      - authorization.k8s.io
+    resources:
+      - subjectaccessreviews
+    verbs:
+      - create
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kube-ovn-app
+roleRef:
+  name: system:kube-ovn-app
+  kind: ClusterRole
+  apiGroup: rbac.authorization.k8s.io
+subjects:
+  - kind: ServiceAccount
+    name: kube-ovn-app
+    namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: kube-ovn-app
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader
+subjects:
+  - kind: ServiceAccount
+    name: kube-ovn-app
+    namespace: kube-system
+---
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: kube-ovn-controller
+  namespace: kube-system
+  annotations:
+    kubernetes.io/description: |
+      kube-ovn controller
+spec:
+  replicas: {{ kube_ovn_controller_replics }}
+  selector:
+    matchLabels:
+      app: kube-ovn-controller
+  strategy:
+    rollingUpdate:
+      maxSurge: 0%
+      maxUnavailable: 100%
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: kube-ovn-controller
+        component: network
+        type: infra
+    spec:
+      tolerations:
+        - effect: NoSchedule
+          operator: Exists
+        - key: CriticalAddonsOnly
+          operator: Exists
+      affinity:
+        nodeAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - preference:
+              matchExpressions:
+              - key: "ovn.kubernetes.io/ic-gw"
+                operator: NotIn
+                values:
+                - "true"
+            weight: 100
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            - labelSelector:
+                matchLabels:
+                  app: kube-ovn-controller
+              topologyKey: kubernetes.io/hostname
+      priorityClassName: system-cluster-critical
+      serviceAccountName: ovn
+      hostNetwork: true
+      containers:
+        - name: kube-ovn-controller
+          image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }}
+          imagePullPolicy: {{ k8s_image_pull_policy }}
+          args:
+          - /kube-ovn/start-controller.sh
+          - --default-cidr={{ kube_pods_subnets }}
+          - --default-gateway={% if kube_ovn_default_gateway is defined %}{{ kube_ovn_default_gateway }}{% endif %}{{ '' }}
+          - --default-gateway-check={{ kube_ovn_default_gateway_check | string }}
+          - --default-logical-gateway={{ kube_ovn_default_logical_gateway | string }}
+          - --default-u2o-interconnection={{ kube_ovn_u2o_interconnection }}
+          - --default-exclude-ips={% if kube_ovn_default_exclude_ips is defined %}{{ kube_ovn_default_exclude_ips }}{% endif %}{{ '' }}
+          - --node-switch-cidr={{ kube_ovn_node_switch_cidr_merged }}
+          - --service-cluster-ip-range={{ kube_service_subnets }}
+          - --network-type={{ kube_ovn_network_type }}
+          - --default-interface-name={{ kube_ovn_default_interface_name | default('') }}
+          - --default-vlan-id={{ kube_ovn_default_vlan_id }}
+          - --ls-dnat-mod-dl-dst={{ kube_ovn_ls_dnat_mod_dl_dst }}
+          - --pod-nic-type={{ kube_ovn_pod_nic_type }}
+          - --enable-lb={{ kube_ovn_enable_lb | string }}
+          - --enable-np={{ kube_ovn_enable_np | string }}
+          - --enable-eip-snat={{ kube_ovn_eip_snat_enabled }}
+          - --enable-external-vpc={{ kube_ovn_enable_external_vpc | string }}
+          - --logtostderr=false
+          - --alsologtostderr=true
+          - --gc-interval=360
+          - --inspect-interval=20
+          - --log_file=/var/log/kube-ovn/kube-ovn-controller.log
+          - --log_file_max_size=0
+          - --enable-lb-svc=false
+          - --keep-vm-ip={{ kube_ovn_keep_vm_ip }}
+          securityContext:
+            runAsUser: 0
+            privileged: false
+            capabilities:
+              add:
+                - NET_BIND_SERVICE
+          env:
+            - name: ENABLE_SSL
+              value: "{{ kube_ovn_enable_ssl | lower }}"
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: KUBE_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: OVN_DB_IPS
+              value: "{{ kube_ovn_central_ips }}"
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: POD_IPS
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIPs
+            - name: ENABLE_BIND_LOCAL_IP
+              value: "{{ kube_ovn_bind_local_ip_enabled }}"
+          volumeMounts:
+            - mountPath: /etc/localtime
+              name: localtime
+            - mountPath: /var/log/kube-ovn
+              name: kube-ovn-log
+            - mountPath: /var/log/ovn
+              name: ovn-log
+            - mountPath: /var/run/tls
+              name: kube-ovn-tls
+          readinessProbe:
+            exec:
+              command:
+                - /kube-ovn/kube-ovn-healthcheck
+                - --port=10660
+                - --tls=false
+            periodSeconds: 3
+            timeoutSeconds: 45
+          livenessProbe:
+            exec:
+              command:
+                - /kube-ovn/kube-ovn-healthcheck
+                - --port=10660
+                - --tls=false
+            initialDelaySeconds: 300
+            periodSeconds: 7
+            failureThreshold: 5
+            timeoutSeconds: 45
+          resources:
+            requests:
+              cpu: {{ kube_ovn_controller_cpu_request }}
+              memory: {{ kube_ovn_controller_memory_request }}
+            limits:
+              cpu: {{ kube_ovn_controller_cpu_limit }}
+              memory: {{ kube_ovn_controller_memory_limit }}
+      nodeSelector:
+        kubernetes.io/os: "linux"
+      volumes:
+        - name: localtime
+          hostPath:
+            path: /etc/localtime
+        - name: kube-ovn-log
+          hostPath:
+            path: /var/log/kube-ovn
+        - name: ovn-log
+          hostPath:
+            path: /var/log/ovn
+        - name: kube-ovn-tls
+          secret:
+            optional: true
+            secretName: kube-ovn-tls
+
+---
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: kube-ovn-cni
+  namespace: kube-system
+  annotations:
+    kubernetes.io/description: |
+      This daemon set launches the kube-ovn cni daemon.
+spec:
+  selector:
+    matchLabels:
+      app: kube-ovn-cni
+  template:
+    metadata:
+      labels:
+        app: kube-ovn-cni
+        component: network
+        type: infra
+    spec:
+      tolerations:
+        - effect: NoSchedule
+          operator: Exists
+        - effect: NoExecute
+          operator: Exists
+        - key: CriticalAddonsOnly
+          operator: Exists
+      priorityClassName: system-node-critical
+      serviceAccountName: kube-ovn-cni
+      hostNetwork: true
+      hostPID: true
+      initContainers:
+      - name: install-cni
+        image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }}
+        imagePullPolicy: {{ k8s_image_pull_policy }}
+        command: ["/kube-ovn/install-cni.sh"]
+        securityContext:
+          runAsUser: 0
+          privileged: true
+        volumeMounts:
+          - mountPath: /opt/cni/bin
+            name: cni-bin
+          - mountPath: /usr/local/bin
+            name: local-bin
+      containers:
+      - name: cni-server
+        image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }}
+        imagePullPolicy: {{ k8s_image_pull_policy }}
+        command:
+          - bash
+          - /kube-ovn/start-cniserver.sh
+        args:
+          - --enable-mirror={{ kube_ovn_traffic_mirror | lower }}
+          - --encap-checksum={{ kube_ovn_encap_checksum | lower }}
+          - --service-cluster-ip-range={{ kube_service_subnets }}
+          - --iface={{ kube_ovn_iface | default('') }}
+          - --dpdk-tunnel-iface={{ kube_ovn_dpdk_tunnel_iface }}
+          - --network-type={{ kube_ovn_network_type }}
+          - --default-interface-name={{ kube_ovn_default_interface_name | default('') }}
+          {% if kube_ovn_mtu is defined %}
+          - --mtu={{ kube_ovn_mtu }}
+{% endif %}
+          - --cni-conf-name={{ kube_ovn_cni_config_priority }}-kube-ovn.conflist
+          - --logtostderr=false
+          - --alsologtostderr=true
+          - --log_file=/var/log/kube-ovn/kube-ovn-cni.log
+          - --log_file_max_size=0
+        securityContext:
+          runAsUser: 0
+          privileged: false
+          capabilities:
+            add:
+              - NET_ADMIN
+              - NET_BIND_SERVICE
+              - NET_RAW
+              - SYS_ADMIN
+        env:
+          - name: ENABLE_SSL
+            value: "{{ kube_ovn_enable_ssl | lower }}"
+          - name: POD_IP
+            valueFrom:
+              fieldRef:
+                fieldPath: status.podIP
+          - name: KUBE_NODE_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: spec.nodeName
+          - name: MODULES
+            value: kube_ovn_fastpath.ko
+          - name: RPMS
+            value: openvswitch-kmod
+          - name: POD_IPS
+            valueFrom:
+              fieldRef:
+                fieldPath: status.podIPs
+          - name: ENABLE_BIND_LOCAL_IP
+            value: "{{ kube_ovn_bind_local_ip_enabled }}"
+          - name: DBUS_SYSTEM_BUS_ADDRESS
+            value: "unix:path=/host/var/run/dbus/system_bus_socket"
+        volumeMounts:
+          - name: host-modules
+            mountPath: /lib/modules
+            readOnly: true
+          - name: shared-dir
+            mountPath: $KUBELET_DIR/pods
+          - mountPath: /etc/openvswitch
+            name: systemid
+            readOnly: true
+          - mountPath: /etc/cni/net.d
+            name: cni-conf
+          - mountPath: /run/openvswitch
+            name: host-run-ovs
+            mountPropagation: HostToContainer
+          - mountPath: /run/ovn
+            name: host-run-ovn
+          - mountPath: /host/var/run/dbus
+            name: host-dbus
+            mountPropagation: HostToContainer
+          - mountPath: /var/run/netns
+            name: host-ns
+            mountPropagation: HostToContainer
+          - mountPath: /var/log/kube-ovn
+            name: kube-ovn-log
+          - mountPath: /var/log/openvswitch
+            name: host-log-ovs
+          - mountPath: /var/log/ovn
+            name: host-log-ovn
+          - mountPath: /etc/localtime
+            name: localtime
+            readOnly: true
+          - mountPath: /tmp
+            name: tmp
+        livenessProbe:
+          failureThreshold: 3
+          initialDelaySeconds: 30
+          periodSeconds: 7
+          successThreshold: 1
+          exec:
+            command:
+              - /kube-ovn/kube-ovn-healthcheck
+              - --port=10665
+              - --tls=false
+          timeoutSeconds: 5
+        readinessProbe:
+          failureThreshold: 3
+          periodSeconds: 7
+          successThreshold: 1
+          exec:
+            command:
+              - /kube-ovn/kube-ovn-healthcheck
+              - --port=10665
+              - --tls=false
+          timeoutSeconds: 5
+        resources:
+          requests:
+            cpu: {{ kube_ovn_cni_server_cpu_request }}
+            memory: {{ kube_ovn_cni_server_memory_request }}
+          limits:
+            cpu: {{ kube_ovn_cni_server_cpu_limit }}
+            memory: {{ kube_ovn_cni_server_memory_limit }}
+      nodeSelector:
+        kubernetes.io/os: "linux"
+      volumes:
+        - name: host-modules
+          hostPath:
+            path: /lib/modules
+        - name: shared-dir
+          hostPath:
+            path: /var/lib/kubelet/pods
+        - name: systemid
+          hostPath:
+            path: /etc/origin/openvswitch
+        - name: host-run-ovs
+          hostPath:
+            path: /run/openvswitch
+        - name: host-run-ovn
+          hostPath:
+            path: /run/ovn
+        - name: cni-conf
+          hostPath:
+            path: /etc/cni/net.d
+        - name: cni-bin
+          hostPath:
+            path: /opt/cni/bin
+        - name: host-ns
+          hostPath:
+            path: /var/run/netns
+        - name: host-dbus
+          hostPath:
+            path: /var/run/dbus
+        - name: host-log-ovs
+          hostPath:
+            path: /var/log/openvswitch
+        - name: kube-ovn-log
+          hostPath:
+            path: /var/log/kube-ovn
+        - name: host-log-ovn
+          hostPath:
+            path: /var/log/ovn
+        - name: localtime
+          hostPath:
+            path: /etc/localtime
+        - name: tmp
+          hostPath:
+            path: /tmp
+        - name: local-bin
+          hostPath:
+            path: /usr/local/bin
+---
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: kube-ovn-pinger
+  namespace: kube-system
+  annotations:
+    kubernetes.io/description: |
+      This daemon set launches the openvswitch daemon.
+spec:
+  selector:
+    matchLabels:
+      app: kube-ovn-pinger
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: kube-ovn-pinger
+        component: network
+        type: infra
+    spec:
+      priorityClassName: system-node-critical
+      serviceAccountName: ovn
+      hostPID: true
+      containers:
+        - name: pinger
+          image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }}
+          command:
+          - /kube-ovn/kube-ovn-pinger
+          args:
+          - --external-address={{ kube_ovn_external_address_merged }}
+          - --external-dns={{ kube_ovn_external_dns }}
+          - --logtostderr=false
+          - --alsologtostderr=true
+          - --log_file=/var/log/kube-ovn/kube-ovn-pinger.log
+          - --log_file_max_size=0
+          imagePullPolicy: {{ k8s_image_pull_policy }}
+          securityContext:
+            runAsUser: 0
+            privileged: false
+          env:
+            - name: ENABLE_SSL
+              value: "{{ kube_ovn_enable_ssl | lower }}"
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: HOST_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.hostIP
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          volumeMounts:
+            - mountPath: /var/run/openvswitch
+              name: host-run-ovs
+            - mountPath: /var/run/ovn
+              name: host-run-ovn
+            - mountPath: /etc/openvswitch
+              name: host-config-openvswitch
+            - mountPath: /var/log/openvswitch
+              name: host-log-ovs
+              readOnly: true
+            - mountPath: /var/log/ovn
+              name: host-log-ovn
+              readOnly: true
+            - mountPath: /var/log/kube-ovn
+              name: kube-ovn-log
+            - mountPath: /etc/localtime
+              name: localtime
+              readOnly: true
+            - mountPath: /var/run/tls
+              name: kube-ovn-tls
+          resources:
+            requests:
+              cpu: {{ kube_ovn_pinger_cpu_request }}
+              memory: {{ kube_ovn_pinger_memory_request }}
+            limits:
+              cpu: {{ kube_ovn_pinger_cpu_limit }}
+              memory: {{ kube_ovn_pinger_memory_limit }}
+      nodeSelector:
+        kubernetes.io/os: "linux"
+      volumes:
+        - name: host-run-ovs
+          hostPath:
+            path: /run/openvswitch
+        - name: host-run-ovn
+          hostPath:
+            path: /run/ovn
+        - name: host-config-openvswitch
+          hostPath:
+            path: /etc/origin/openvswitch
+        - name: host-log-ovs
+          hostPath:
+            path: /var/log/openvswitch
+        - name: kube-ovn-log
+          hostPath:
+            path: /var/log/kube-ovn
+        - name: host-log-ovn
+          hostPath:
+            path: /var/log/ovn
+        - name: localtime
+          hostPath:
+            path: /etc/localtime
+        - name: kube-ovn-tls
+          secret:
+            optional: true
+            secretName: kube-ovn-tls
+---
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: kube-ovn-monitor
+  namespace: kube-system
+  annotations:
+    kubernetes.io/description: |
+      Metrics for OVN components: northd, nb and sb.
+spec:
+  replicas: 1
+  strategy:
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 1
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      app: kube-ovn-monitor
+  template:
+    metadata:
+      labels:
+        app: kube-ovn-monitor
+        component: network
+        type: infra
+    spec:
+      tolerations:
+        - effect: NoSchedule
+          operator: Exists
+        - key: CriticalAddonsOnly
+          operator: Exists
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            - labelSelector:
+                matchLabels:
+                  app: kube-ovn-monitor
+              topologyKey: kubernetes.io/hostname
+      priorityClassName: system-cluster-critical
+      serviceAccountName: ovn
+      hostNetwork: true
+      containers:
+        - name: kube-ovn-monitor
+          image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }}
+          imagePullPolicy: {{ k8s_image_pull_policy }}
+          command: ["/kube-ovn/start-ovn-monitor.sh"]
+          args:
+          - --secure-serving=false
+          - --log_file=/var/log/kube-ovn/kube-ovn-monitor.log
+          - --logtostderr=false
+          - --alsologtostderr=true
+          - --log_file_max_size=200
+          securityContext:
+            runAsUser: 0
+            privileged: false
+          env:
+            - name: ENABLE_SSL
+              value: "{{ kube_ovn_enable_ssl | lower }}"
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: POD_IPS
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIPs
+            - name: ENABLE_BIND_LOCAL_IP
+              value: "{{ kube_ovn_bind_local_ip_enabled }}"
+          resources:
+            requests:
+              cpu: {{ kube_ovn_monitor_cpu_request }}
+              memory: {{ kube_ovn_monitor_memory_request }}
+            limits:
+              cpu: {{ kube_ovn_monitor_cpu_limit }}
+              memory: {{ kube_ovn_monitor_memory_limit }}
+          volumeMounts:
+            - mountPath: /var/run/openvswitch
+              name: host-run-ovs
+            - mountPath: /var/run/ovn
+              name: host-run-ovn
+            - mountPath: /etc/openvswitch
+              name: host-config-openvswitch
+            - mountPath: /etc/ovn
+              name: host-config-ovn
+            - mountPath: /var/log/ovn
+              name: host-log-ovn
+              readOnly: true
+            - mountPath: /etc/localtime
+              name: localtime
+              readOnly: true
+            - mountPath: /var/run/tls
+              name: kube-ovn-tls
+            - mountPath: /var/log/kube-ovn
+              name: kube-ovn-log
+          livenessProbe:
+            failureThreshold: 3
+            initialDelaySeconds: 30
+            periodSeconds: 7
+            successThreshold: 1
+            exec:
+              command:
+                - /kube-ovn/kube-ovn-healthcheck
+                - --port=10661
+                - --tls=false
+            timeoutSeconds: 5
+          readinessProbe:
+            failureThreshold: 3
+            initialDelaySeconds: 30
+            periodSeconds: 7
+            successThreshold: 1
+            exec:
+              command:
+                - /kube-ovn/kube-ovn-healthcheck
+                - --port=10661
+                - --tls=false
+            timeoutSeconds: 5
+      nodeSelector:
+        kubernetes.io/os: "linux"
+        kube-ovn/role: "master"
+      volumes:
+        - name: host-run-ovs
+          hostPath:
+            path: /run/openvswitch
+        - name: host-run-ovn
+          hostPath:
+            path: /run/ovn
+        - name: host-config-openvswitch
+          hostPath:
+            path: /etc/origin/openvswitch
+        - name: host-config-ovn
+          hostPath:
+            path: /etc/origin/ovn
+        - name: host-log-ovs
+          hostPath:
+            path: /var/log/openvswitch
+        - name: host-log-ovn
+          hostPath:
+            path: /var/log/ovn
+        - name: localtime
+          hostPath:
+            path: /etc/localtime
+        - name: kube-ovn-tls
+          secret:
+            optional: true
+            secretName: kube-ovn-tls
+        - name: kube-ovn-log
+          hostPath:
+            path: /var/log/kube-ovn
+---
+kind: Service
+apiVersion: v1
+metadata:
+  name: kube-ovn-monitor
+  namespace: kube-system
+  labels:
+    app: kube-ovn-monitor
+spec:
+  ports:
+    - name: metrics
+      port: 10661
+  type: ClusterIP
+{% if ipv6_stack %}
+  ipFamilyPolicy: PreferDualStack
+{% endif %}
+  selector:
+    app: kube-ovn-monitor
+  sessionAffinity: None
+---
+kind: Service
+apiVersion: v1
+metadata:
+  name: kube-ovn-pinger
+  namespace: kube-system
+  labels:
+    app: kube-ovn-pinger
+spec:
+{% if ipv6_stack %}
+  ipFamilyPolicy: PreferDualStack
+{% endif %}
+  selector:
+    app: kube-ovn-pinger
+  ports:
+    - port: 8080
+      name: metrics
+---
+kind: Service
+apiVersion: v1
+metadata:
+  name: kube-ovn-controller
+  namespace: kube-system
+  labels:
+    app: kube-ovn-controller
+spec:
+{% if ipv6_stack %}
+  ipFamilyPolicy: PreferDualStack
+{% endif %}
+  selector:
+    app: kube-ovn-controller
+  ports:
+    - port: 10660
+      name: metrics
+---
+kind: Service
+apiVersion: v1
+metadata:
+  name: kube-ovn-cni
+  namespace: kube-system
+  labels:
+    app: kube-ovn-cni
+spec:
+{% if ipv6_stack %}
+  ipFamilyPolicy: PreferDualStack
+{% endif %}
+  selector:
+    app: kube-ovn-cni
+  ports:
+    - port: 10665
+      name: metrics
+{% if kube_ovn_ic_enable %}
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: ovn-ic-config
+  namespace: kube-system
+data:
+  enable-ic: "{{ kube_ovn_ic_enable | lower }}"
+  az-name: "{{ kube_ovn_ic_zone }}"
+  ic-db-host: "{{ kube_ovn_ic_dbhost }}"
+  ic-nb-port: "6645"
+  ic-sb-port: "6646"
+  gw-nodes: "{{ kube_ovn_central_hosts | join(',') }}"
+  auto-route: "{{ kube_ovn_ic_autoroute | lower }}"
+{% endif %}

internal/resources/kubespray/project/roles/network_plugin/kube-ovn/templates/cni-ovn.yml.j2

@@ -0,0 +1,674 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: ovn-ovs
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  annotations:
+    rbac.authorization.k8s.io/system-only: "true"
+  name: system:ovn-ovs
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - get
+      - patch
+  - apiGroups:
+      - ""
+    resources:
+      - services
+      - endpoints
+    verbs:
+      - get
+  - apiGroups:
+      - apps
+    resources:
+      - controllerrevisions
+    verbs:
+      - get
+      - list
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: ovn-ovs
+roleRef:
+  name: system:ovn-ovs
+  kind: ClusterRole
+  apiGroup: rbac.authorization.k8s.io
+subjects:
+  - kind: ServiceAccount
+    name: ovn-ovs
+    namespace: kube-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: ovn
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  annotations:
+    rbac.authorization.k8s.io/system-only: "true"
+  name: system:ovn
+rules:
+  - apiGroups:
+      - "kubeovn.io"
+    resources:
+      - vpcs
+      - vpcs/status
+      - vpc-nat-gateways
+      - vpc-nat-gateways/status
+      - subnets
+      - subnets/status
+      - ippools
+      - ippools/status
+      - ips
+      - vips
+      - vips/status
+      - vlans
+      - vlans/status
+      - provider-networks
+      - provider-networks/status
+      - security-groups
+      - security-groups/status
+      - iptables-eips
+      - iptables-fip-rules
+      - iptables-dnat-rules
+      - iptables-snat-rules
+      - iptables-eips/status
+      - iptables-fip-rules/status
+      - iptables-dnat-rules/status
+      - iptables-snat-rules/status
+      - ovn-eips
+      - ovn-fips
+      - ovn-snat-rules
+      - ovn-eips/status
+      - ovn-fips/status
+      - ovn-snat-rules/status
+      - ovn-dnat-rules
+      - ovn-dnat-rules/status
+      - switch-lb-rules
+      - switch-lb-rules/status
+      - vpc-dnses
+      - vpc-dnses/status
+      - qos-policies
+      - qos-policies/status
+    verbs:
+      - "*"
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - namespaces
+    verbs:
+      - get
+      - list
+      - patch
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - nodes
+    verbs:
+      - get
+      - list
+      - patch
+      - update
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - pods/exec
+    verbs:
+      - create
+  - apiGroups:
+      - "k8s.cni.cncf.io"
+    resources:
+      - network-attachment-definitions
+    verbs:
+      - get
+  - apiGroups:
+      - ""
+      - networking.k8s.io
+    resources:
+      - networkpolicies
+      - configmaps
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - apps
+    resources:
+      - daemonsets
+    verbs:
+      - get
+  - apiGroups:
+      - ""
+    resources:
+      - services
+      - services/status
+    verbs:
+      - get
+      - list
+      - update
+      - create
+      - delete
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - endpoints
+    verbs:
+      - create
+      - update
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - apps
+    resources:
+      - statefulsets
+      - deployments
+      - deployments/scale
+    verbs:
+      - get
+      - list
+      - create
+      - delete
+      - update
+  - apiGroups:
+      - ""
+    resources:
+      - events
+    verbs:
+      - create
+      - patch
+      - update
+  - apiGroups:
+      - coordination.k8s.io
+    resources:
+      - leases
+    verbs:
+      - "*"
+  - apiGroups:
+      - "kubevirt.io"
+    resources:
+      - virtualmachines
+      - virtualmachineinstances
+    verbs:
+      - get
+      - list
+  - apiGroups:
+      - authentication.k8s.io
+    resources:
+      - tokenreviews
+    verbs:
+      - create
+  - apiGroups:
+      - authorization.k8s.io
+    resources:
+      - subjectaccessreviews
+    verbs:
+      - create
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: ovn
+roleRef:
+  name: system:ovn
+  kind: ClusterRole
+  apiGroup: rbac.authorization.k8s.io
+subjects:
+  - kind: ServiceAccount
+    name: ovn
+    namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: ovn
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader
+subjects:
+  - kind: ServiceAccount
+    name: ovn
+    namespace: kube-system
+---
+kind: Service
+apiVersion: v1
+metadata:
+  name: ovn-nb
+  namespace: kube-system
+spec:
+  ports:
+    - name: ovn-nb
+      protocol: TCP
+      port: 6641
+      targetPort: 6641
+  type: ClusterIP
+{% if ipv6_stack %}
+  ipFamilyPolicy: PreferDualStack
+{% endif %}
+  selector:
+    app: ovn-central
+    ovn-nb-leader: "true"
+  sessionAffinity: None
+---
+kind: Service
+apiVersion: v1
+metadata:
+  name: ovn-sb
+  namespace: kube-system
+spec:
+  ports:
+    - name: ovn-sb
+      protocol: TCP
+      port: 6642
+      targetPort: 6642
+  type: ClusterIP
+{% if ipv6_stack %}
+  ipFamilyPolicy: PreferDualStack
+{% endif %}
+  selector:
+    app: ovn-central
+    ovn-sb-leader: "true"
+  sessionAffinity: None
+---
+kind: Service
+apiVersion: v1
+metadata:
+  name: ovn-northd
+  namespace: kube-system
+spec:
+  ports:
+    - name: ovn-northd
+      protocol: TCP
+      port: 6643
+      targetPort: 6643
+  type: ClusterIP
+{% if ipv6_stack %}
+  ipFamilyPolicy: PreferDualStack
+{% endif %}
+  selector:
+    app: ovn-central
+    ovn-northd-leader: "true"
+  sessionAffinity: None
+---
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: ovn-central
+  namespace: kube-system
+  annotations:
+    kubernetes.io/description: |
+      OVN components: northd, nb and sb.
+spec:
+  replicas: {{ kube_ovn_central_replics }}
+  strategy:
+    rollingUpdate:
+      maxSurge: 0
+      maxUnavailable: 1
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      app: ovn-central
+  template:
+    metadata:
+      labels:
+        app: ovn-central
+        component: network
+        type: infra
+    spec:
+      tolerations:
+        - effect: NoSchedule
+          operator: Exists
+        - effect: NoExecute
+          operator: Exists
+        - key: CriticalAddonsOnly
+          operator: Exists
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            - labelSelector:
+                matchLabels:
+                  app: ovn-central
+              topologyKey: kubernetes.io/hostname
+      priorityClassName: system-cluster-critical
+      serviceAccountName: ovn-ovs
+      hostNetwork: true
+      containers:
+        - name: ovn-central
+          image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }}
+          imagePullPolicy: {{ k8s_image_pull_policy }}
+          command: ["/kube-ovn/start-db.sh"]
+          securityContext:
+            capabilities:
+              add:
+                - NET_BIND_SERVICE
+                - SYS_NICE
+          env:
+            - name: ENABLE_SSL
+              value: "{{ kube_ovn_enable_ssl | lower }}"
+            - name: NODE_IPS
+              value: "{{ kube_ovn_central_ips }}"
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: POD_IPS
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIPs
+            - name: ENABLE_BIND_LOCAL_IP
+              value: "{{ kube_ovn_bind_local_ip_enabled }}"
+            - name: PROBE_INTERVAL
+              value: "180000"
+            - name: OVN_NORTHD_PROBE_INTERVAL
+              value: "5000"
+            - name: OVN_LEADER_PROBE_INTERVAL
+              value: "5"
+          resources:
+            requests:
+              cpu: {{ kube_ovn_db_cpu_request }}
+              memory: {{ kube_ovn_db_memory_request }}
+            limits:
+              cpu: {{ kube_ovn_db_cpu_limit }}
+              memory: {{ kube_ovn_db_memory_limit }}
+          volumeMounts:
+            - mountPath: /var/run/openvswitch
+              name: host-run-ovs
+            - mountPath: /var/run/ovn
+              name: host-run-ovn
+            - mountPath: /sys
+              name: host-sys
+              readOnly: true
+            - mountPath: /etc/openvswitch
+              name: host-config-openvswitch
+            - mountPath: /etc/ovn
+              name: host-config-ovn
+            - mountPath: /var/log/openvswitch
+              name: host-log-ovs
+            - mountPath: /var/log/ovn
+              name: host-log-ovn
+            - mountPath: /etc/localtime
+              name: localtime
+            - mountPath: /var/run/tls
+              name: kube-ovn-tls
+          readinessProbe:
+            exec:
+              command:
+                - bash
+                - /kube-ovn/ovn-healthcheck.sh
+            periodSeconds: 15
+            timeoutSeconds: 45
+          livenessProbe:
+            exec:
+              command:
+                - bash
+                - /kube-ovn/ovn-healthcheck.sh
+            initialDelaySeconds: 30
+            periodSeconds: 15
+            failureThreshold: 5
+            timeoutSeconds: 45
+      nodeSelector:
+        kubernetes.io/os: "linux"
+        kube-ovn/role: "master"
+      volumes:
+        - name: host-run-ovs
+          hostPath:
+            path: /run/openvswitch
+        - name: host-run-ovn
+          hostPath:
+            path: /run/ovn
+        - name: host-sys
+          hostPath:
+            path: /sys
+        - name: host-config-openvswitch
+          hostPath:
+            path: /etc/origin/openvswitch
+        - name: host-config-ovn
+          hostPath:
+            path: /etc/origin/ovn
+        - name: host-log-ovs
+          hostPath:
+            path: /var/log/openvswitch
+        - name: host-log-ovn
+          hostPath:
+            path: /var/log/ovn
+        - name: localtime
+          hostPath:
+            path: /etc/localtime
+        - name: kube-ovn-tls
+          secret:
+            optional: true
+            secretName: kube-ovn-tls
+---
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: ovs-ovn
+  namespace: kube-system
+  annotations:
+    kubernetes.io/description: |
+      This daemon set launches the openvswitch daemon.
+spec:
+  selector:
+    matchLabels:
+      app: ovs
+  updateStrategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 0
+  template:
+    metadata:
+      labels:
+        app: ovs
+        component: network
+        type: infra
+    spec:
+      tolerations:
+        - effect: NoSchedule
+          operator: Exists
+        - effect: NoExecute
+          operator: Exists
+        - key: CriticalAddonsOnly
+          operator: Exists
+      priorityClassName: system-node-critical
+      serviceAccountName: ovn-ovs
+      hostNetwork: true
+      hostPID: true
+      containers:
+        - name: openvswitch
+          image: {% if kube_ovn_dpdk_enabled %}{{ kube_ovn_dpdk_container_image_repo }}:{{ kube_ovn_dpdk_container_image_tag }}{% else %}{{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }}{% endif %}
+
+          imagePullPolicy: {{ k8s_image_pull_policy }}
+          command: [{% if kube_ovn_dpdk_enabled %}"/kube-ovn/start-ovs-dpdk.sh"{% else %}"/kube-ovn/start-ovs.sh"{% endif %}]
+          securityContext:
+            runAsUser: 0
+            privileged: false
+            capabilities:
+              add:
+                - NET_ADMIN
+                - NET_BIND_SERVICE
+                - SYS_MODULE
+                - SYS_NICE
+          env:
+            - name: ENABLE_SSL
+              value: "{{ kube_ovn_enable_ssl | lower }}"
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+{% if not kube_ovn_dpdk_enabled %}
+            - name: HW_OFFLOAD
+              value: "{{ kube_ovn_hw_offload | string | lower }}"
+            - name: TUNNEL_TYPE
+              value: "{{ kube_ovn_tunnel_type }}"
+{% endif %}
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: OVN_DB_IPS
+              value: "{{ kube_ovn_central_ips }}"
+          volumeMounts:
+            - mountPath: /var/run/netns
+              name: host-ns
+              mountPropagation: HostToContainer
+            - mountPath: /lib/modules
+              name: host-modules
+              readOnly: true
+            - mountPath: /var/run/openvswitch
+              name: host-run-ovs
+            - mountPath: /var/run/ovn
+              name: host-run-ovn
+            - mountPath: /sys
+              name: host-sys
+              readOnly: true
+            - mountPath: /etc/cni/net.d
+              name: cni-conf
+            - mountPath: /etc/openvswitch
+              name: host-config-openvswitch
+            - mountPath: /etc/ovn
+              name: host-config-ovn
+            - mountPath: /var/log/openvswitch
+              name: host-log-ovs
+            - mountPath: /var/log/ovn
+              name: host-log-ovn
+{% if kube_ovn_dpdk_enabled %}
+            - mountPath: /opt/ovs-config
+              name: host-config-ovs
+            - mountPath: /dev/hugepages
+              name: hugepage
+{% endif %}
+            - mountPath: /etc/localtime
+              name: localtime
+            - mountPath: /var/run/tls
+              name: kube-ovn-tls
+            - mountPath: /var/run/containerd
+              name: cruntime
+              readOnly: true
+          readinessProbe:
+            exec:
+              command:
+                - bash
+{% if kube_ovn_dpdk_enabled %}
+                - /kube-ovn/ovs-dpdk-healthcheck.sh
+{% else %}
+                - /kube-ovn/ovs-healthcheck.sh
+{% endif %}
+            periodSeconds: 5
+            timeoutSeconds: 45
+          livenessProbe:
+            exec:
+              command:
+                - bash
+{% if kube_ovn_dpdk_enabled %}
+                - /kube-ovn/ovs-dpdk-healthcheck.sh
+{% else %}
+                - /kube-ovn/ovs-healthcheck.sh
+{% endif %}
+            initialDelaySeconds: 60
+            periodSeconds: 5
+            failureThreshold: 5
+            timeoutSeconds: 45
+          resources:
+{% if kube_ovn_dpdk_enabled %}
+            requests:
+              cpu: {{ kube_ovn_dpdk_node_cpu_request }}
+              memory: {{ kube_ovn_dpdk_node_memory_request }}
+            limits:
+              cpu: {{ kube_ovn_dpdk_node_cpu_limit }}
+              memory: {{ kube_ovn_dpdk_node_memory_limit }}
+              hugepages-1Gi: 1Gi
+{% else %}
+            requests:
+              cpu: {{ kube_ovn_node_cpu_request }}
+              memory: {{ kube_ovn_node_memory_request }}
+            limits:
+              cpu: {{ kube_ovn_node_cpu_limit }}
+              memory: {{ kube_ovn_node_memory_limit }}
+{% endif %}
+      nodeSelector:
+        kubernetes.io/os: "linux"
+      volumes:
+        - name: host-modules
+          hostPath:
+            path: /lib/modules
+        - name: host-run-ovs
+          hostPath:
+            path: /run/openvswitch
+        - name: host-run-ovn
+          hostPath:
+            path: /run/ovn
+        - name: host-sys
+          hostPath:
+            path: /sys
+        - name: host-ns
+          hostPath:
+            path: /var/run/netns
+        - name: cni-conf
+          hostPath:
+            path: /etc/cni/net.d
+        - name: host-config-openvswitch
+          hostPath:
+            path: /etc/origin/openvswitch
+        - name: host-config-ovn
+          hostPath:
+            path: /etc/origin/ovn
+        - name: host-log-ovs
+          hostPath:
+            path: /var/log/openvswitch
+        - name: host-log-ovn
+          hostPath:
+            path: /var/log/ovn
+{% if kube_ovn_dpdk_enabled %}
+        - name: host-config-ovs
+          hostPath:
+            path: /opt/ovs-config
+            type: DirectoryOrCreate
+        - name: hugepage
+          emptyDir:
+            medium: HugePages
+{% endif %}
+        - name: localtime
+          hostPath:
+            path: /etc/localtime
+        - name: cruntime
+          hostPath:
+            path: /var/run/containerd
+        - name: kube-ovn-tls
+          secret:
+            optional: true
+            secretName: kube-ovn-tls

internal/resources/kubespray/project/roles/network_plugin/kube-router/defaults/main.yml

@@ -0,0 +1,69 @@
+---
+# Enables Pod Networking -- Advertises and learns the routes to Pods via iBGP
+kube_router_run_router: true
+
+# Enables Network Policy -- sets up iptables to provide ingress firewall for pods
+kube_router_run_firewall: true
+
+# Enables Service Proxy -- sets up IPVS for Kubernetes Services
+# see docs/kube-router.md "Caveats" section
+kube_router_run_service_proxy: false
+
+# Add Cluster IP of the service to the RIB so that it gets advertises to the BGP peers.
+kube_router_advertise_cluster_ip: false
+
+# Add External IP of service to the RIB so that it gets advertised to the BGP peers.
+kube_router_advertise_external_ip: false
+
+# Add LoadBalancer IP of service status as set by the LB provider to the RIB so that it gets advertised to the BGP peers.
+kube_router_advertise_loadbalancer_ip: false
+
+# Enables BGP graceful restarts
+kube_router_bgp_graceful_restart: true
+
+# Adjust manifest of kube-router daemonset template with DSR needed changes
+kube_router_enable_dsr: false
+
+# Array of arbitrary extra arguments to kube-router, see
+# https://github.com/cloudnativelabs/kube-router/blob/master/docs/user-guide.md
+kube_router_extra_args: []
+
+# ASN number of the cluster, used when communicating with external BGP routers
+kube_router_cluster_asn: ~
+
+# ASN numbers of the BGP peer to which cluster nodes will advertise cluster ip and node's pod cidr.
+kube_router_peer_router_asns: ~
+
+# The ip address of the external router to which all nodes will peer and advertise the cluster ip and pod cidr's.
+kube_router_peer_router_ips: ~
+
+# The remote port of the external BGP to which all nodes will peer. If not set, default BGP port (179) will be used.
+kube_router_peer_router_ports: ~
+
+# Setups node CNI to allow hairpin mode, requires node reboots, see
+# https://github.com/cloudnativelabs/kube-router/blob/master/docs/user-guide.md#hairpin-mode
+kube_router_support_hairpin_mode: false
+
+# Select DNS Policy ClusterFirstWithHostNet, ClusterFirst, etc.
+kube_router_dns_policy: ClusterFirstWithHostNet
+
+# Adds annotations to kubernetes nodes for advanced configuration of BGP Peers.
+# https://github.com/cloudnativelabs/kube-router/blob/master/docs/bgp.md
+
+# Array of annotations for master
+kube_router_annotations_master: []
+
+# Array of annotations for every node
+kube_router_annotations_node: []
+
+# Array of common annotations for every node
+kube_router_annotations_all: []
+
+# Enables scraping kube-router metrics with Prometheus
+kube_router_enable_metrics: false
+
+# Path to serve Prometheus metrics on
+kube_router_metrics_path: /metrics
+
+# Prometheus metrics port to use
+kube_router_metrics_port: 9255

internal/resources/kubespray/project/roles/network_plugin/kube-router/handlers/main.yml

@@ -0,0 +1,20 @@
+---
+- name: Kube-router | delete kube-router docker containers
+  shell: "set -o pipefail && {{ docker_bin_dir }}/docker ps -af name=k8s_POD_kube-router* -q | xargs --no-run-if-empty docker rm -f"
+  args:
+    executable: /bin/bash
+  register: docker_kube_router_remove
+  until: docker_kube_router_remove is succeeded
+  retries: 5
+  when: container_manager in ["docker"]
+  listen: Reset_kube_router
+
+- name: Kube-router | delete kube-router crio/containerd containers
+  shell: 'set -o pipefail && {{ bin_dir }}/crictl pods --name kube-router* -q | xargs -I% --no-run-if-empty bash -c "{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %"'
+  args:
+    executable: /bin/bash
+  register: crictl_kube_router_remove
+  until: crictl_kube_router_remove is succeeded
+  retries: 5
+  when: container_manager in ["crio", "containerd"]
+  listen: Reset_kube_router

internal/resources/kubespray/project/roles/network_plugin/kube-router/meta/main.yml

@@ -0,0 +1,3 @@
+---
+dependencies:
+  - role: network_plugin/cni

internal/resources/kubespray/project/roles/network_plugin/kube-router/tasks/annotate.yml

@@ -0,0 +1,21 @@
+---
+- name: Kube-router | Add annotations on kube_control_plane
+  command: "{{ kubectl }} annotate --overwrite node {{ ansible_hostname }} {{ item }}"
+  with_items:
+  - "{{ kube_router_annotations_master }}"
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+  when: kube_router_annotations_master is defined and 'kube_control_plane' in group_names
+
+- name: Kube-router | Add annotations on kube_node
+  command: "{{ kubectl }} annotate --overwrite node {{ ansible_hostname }} {{ item }}"
+  with_items:
+  - "{{ kube_router_annotations_node }}"
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+  when: kube_router_annotations_node is defined and 'kube_node' in group_names
+
+- name: Kube-router | Add common annotations on all servers
+  command: "{{ kubectl }} annotate --overwrite node {{ ansible_hostname }} {{ item }}"
+  with_items:
+  - "{{ kube_router_annotations_all }}"
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+  when: kube_router_annotations_all is defined and 'k8s_cluster' in group_names

internal/resources/kubespray/project/roles/network_plugin/kube-router/tasks/main.yml

@@ -0,0 +1,62 @@
+---
+- name: Kube-router | Create annotations
+  import_tasks: annotate.yml
+  tags: annotate
+
+- name: Kube-router | Create config directory
+  file:
+    path: /var/lib/kube-router
+    state: directory
+    owner: "{{ kube_owner }}"
+    recurse: true
+    mode: "0755"
+
+- name: Kube-router | Create kubeconfig
+  template:
+    src: kubeconfig.yml.j2
+    dest: /var/lib/kube-router/kubeconfig
+    mode: "0644"
+    owner: "{{ kube_owner }}"
+  notify:
+    - Reset_kube_router
+
+- name: Kube-router | Slurp cni config
+  slurp:
+    src: /etc/cni/net.d/10-kuberouter.conflist
+  register: cni_config_slurp
+  ignore_errors: true  # noqa ignore-errors
+
+- name: Kube-router | Set cni_config variable
+  set_fact:
+    cni_config: "{{ cni_config_slurp.content | b64decode | from_json }}"
+  when:
+    - not cni_config_slurp.failed
+
+- name: Kube-router | Set host_subnet variable
+  when:
+    - cni_config is defined
+    - cni_config | json_query('plugins[?bridge==`kube-bridge`].ipam.subnet') | length > 0
+  set_fact:
+    host_subnet: "{{ cni_config | json_query('plugins[?bridge==`kube-bridge`].ipam.subnet') | first }}"
+
+- name: Kube-router | Create cni config
+  template:
+    src: cni-conf.json.j2
+    dest: /etc/cni/net.d/10-kuberouter.conflist
+    mode: "0644"
+    owner: "{{ kube_owner }}"
+  notify:
+    - Reset_kube_router
+
+- name: Kube-router | Delete old configuration
+  file:
+    path: /etc/cni/net.d/10-kuberouter.conf
+    state: absent
+
+- name: Kube-router | Create manifest
+  template:
+    src: kube-router.yml.j2
+    dest: "{{ kube_config_dir }}/kube-router.yml"
+    mode: "0644"
+  delegate_to: "{{ groups['kube_control_plane'] | first }}"
+  run_once: true

internal/resources/kubespray/project/roles/network_plugin/kube-router/tasks/reset.yml

@@ -0,0 +1,28 @@
+---
+- name: Reset | check kube-dummy-if network device
+  stat:
+    path: /sys/class/net/kube-dummy-if
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: kube_dummy_if
+
+- name: Reset | remove the network device created by kube-router
+  command: ip link del kube-dummy-if
+  when: kube_dummy_if.stat.exists
+
+- name: Check kube-bridge exists
+  stat:
+    path: /sys/class/net/kube-bridge
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: kube_bridge_if
+
+- name: Reset | donw the network bridge create by kube-router
+  command: ip link set kube-bridge down
+  when: kube_bridge_if.stat.exists
+
+- name: Reset | remove the network bridge create by kube-router
+  command: ip link del kube-bridge
+  when: kube_bridge_if.stat.exists

internal/resources/kubespray/project/roles/network_plugin/kube-router/templates/cni-conf.json.j2

@@ -0,0 +1,27 @@
+{
+   "cniVersion":"0.3.0",
+   "name":"kubernetes",
+   "plugins":[
+      {
+         "name":"kubernetes",
+         "type":"bridge",
+         "bridge":"kube-bridge",
+         "isDefaultGateway":true,
+{% if kube_router_support_hairpin_mode %}
+          "hairpinMode":true,
+{% endif %}
+         "ipam":{
+{% if host_subnet is defined %}
+            "subnet": "{{ host_subnet }}",
+{% endif %}
+            "type":"host-local"
+         }
+      },
+      {
+         "type":"portmap",
+         "capabilities":{
+            "portMappings":true
+         }
+      }
+   ]
+}

internal/resources/kubespray/project/roles/network_plugin/kube-router/templates/kube-router.yml.j2

@@ -0,0 +1,228 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  labels:
+    k8s-app: kube-router
+    tier: node
+  name: kube-router
+  namespace: kube-system
+spec:
+  minReadySeconds: 3
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 1
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      k8s-app: kube-router
+      tier: node
+  template:
+    metadata:
+      labels:
+        k8s-app: kube-router
+        tier: node
+      annotations:
+{% if kube_router_enable_metrics %}
+        prometheus.io/path: {{ kube_router_metrics_path }}
+        prometheus.io/port: "{{ kube_router_metrics_port }}"
+        prometheus.io/scrape: "true"
+{% endif %}
+    spec:
+      priorityClassName: system-node-critical
+      serviceAccountName: kube-router
+      containers:
+      - name: kube-router
+        image: {{ kube_router_image_repo }}:{{ kube_router_image_tag }}
+        imagePullPolicy: {{ k8s_image_pull_policy }}
+        args:
+        - --run-router={{ kube_router_run_router | bool }}
+        - --run-firewall={{ kube_router_run_firewall | bool }}
+        - --run-service-proxy={{ kube_router_run_service_proxy | bool }}
+        - --kubeconfig=/var/lib/kube-router/kubeconfig
+        - --bgp-graceful-restart={{ kube_router_bgp_graceful_restart }}
+{% if kube_router_advertise_cluster_ip %}
+        - --advertise-cluster-ip
+{% endif %}
+{% if kube_router_advertise_external_ip %}
+        - --advertise-external-ip
+{% endif %}
+{% if kube_router_advertise_loadbalancer_ip %}
+        - --advertise-loadbalancer-ip
+{% endif %}
+{% if kube_router_cluster_asn %}
+        - --cluster-asn={{ kube_router_cluster_asn }}
+{% endif %}
+{% if kube_router_peer_router_asns %}
+        - --peer-router-asns={{ kube_router_peer_router_asns }}
+{% endif %}
+{% if kube_router_peer_router_ips %}
+        - --peer-router-ips={{ kube_router_peer_router_ips }}
+{% endif %}
+{% if kube_router_peer_router_ports %}
+        - --peer-router-ports={{ kube_router_peer_router_ports }}
+{% endif %}
+{% if kube_router_enable_metrics %}
+        - --metrics-path={{ kube_router_metrics_path }}
+        - --metrics-port={{ kube_router_metrics_port }}
+{% endif %}
+{% if kube_router_enable_dsr %}
+{% if container_manager == "docker" %}
+        - --runtime-endpoint=unix:///var/run/docker.sock
+{% endif %}
+{% if container_manager == "containerd" %}
+{% endif %}
+        - --runtime-endpoint=unix:///run/containerd/containerd.sock
+{% endif %}
+{% for arg in kube_router_extra_args %}
+        - "{{ arg }}"
+{% endfor %}
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        - name: KUBE_ROUTER_CNI_CONF_FILE
+          value: /etc/cni/net.d/10-kuberouter.conflist
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 20244
+          initialDelaySeconds: 10
+          periodSeconds: 3
+        resources:
+          requests:
+            cpu: 250m
+            memory: 250Mi
+        securityContext:
+          privileged: true
+        volumeMounts:
+{% if kube_router_enable_dsr %}
+{% if container_manager == "docker" %}
+        - name: docker-socket
+          mountPath: /var/run/docker.sock
+          readOnly: true
+{% endif %}
+{% if container_manager == "containerd" %}
+        - name: containerd-socket
+          mountPath: /run/containerd/containerd.sock
+          readOnly: true
+{% endif %}
+{% endif %}
+        - name: lib-modules
+          mountPath: /lib/modules
+          readOnly: true
+        - name: cni-conf-dir
+          mountPath: /etc/cni/net.d
+        - name: kubeconfig
+          mountPath: /var/lib/kube-router
+          readOnly: true
+        - name: xtables-lock
+          mountPath: /run/xtables.lock
+          readOnly: false
+{% if kube_router_enable_metrics %}
+        ports:
+        - containerPort: {{ kube_router_metrics_port }}
+          hostPort: {{ kube_router_metrics_port }}
+          name: metrics
+          protocol: TCP
+{% endif %}
+      hostNetwork: true
+      dnsPolicy: {{ kube_router_dns_policy }}
+{% if kube_router_enable_dsr %}
+      hostIPC: true
+      hostPID: true
+{% endif %}
+      tolerations:
+      - operator: Exists
+      volumes:
+{% if kube_router_enable_dsr %}
+{% if container_manager == "docker" %}
+      - name: docker-socket
+        hostPath:
+          path: /var/run/docker.sock
+          type: Socket
+{% endif %}
+{% if container_manager == "containerd" %}
+      - name: containerd-socket
+        hostPath:
+          path: /run/containerd/containerd.sock
+          type: Socket
+{% endif %}
+{% endif %}
+      - name: lib-modules
+        hostPath:
+          path: /lib/modules
+      - name: cni-conf-dir
+        hostPath:
+          path: /etc/cni/net.d
+      - name: kubeconfig
+        hostPath:
+          path: /var/lib/kube-router
+      - name: xtables-lock
+        hostPath:
+          path: /run/xtables.lock
+          type: FileOrCreate
+
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kube-router
+  namespace: kube-system
+
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: kube-router
+  namespace: kube-system
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - namespaces
+      - pods
+      - services
+      - nodes
+      - endpoints
+    verbs:
+      - list
+      - get
+      - watch
+  - apiGroups:
+      - "networking.k8s.io"
+    resources:
+      - networkpolicies
+    verbs:
+      - list
+      - get
+      - watch
+  - apiGroups:
+      - extensions
+    resources:
+      - networkpolicies
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - discovery.k8s.io
+    resources:
+      - endpointslices
+    verbs:
+      - get
+      - list
+      - watch
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: kube-router
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kube-router
+subjects:
+- kind: ServiceAccount
+  name: kube-router
+  namespace: kube-system

internal/resources/kubespray/project/roles/network_plugin/kube-router/templates/kubeconfig.yml.j2

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Config
+clusterCIDR: {{ kube_pods_subnets }}
+clusters:
+- name: cluster
+  cluster:
+    certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+    server: {{ kube_apiserver_endpoint }}
+users:
+- name: kube-router
+  user:
+    tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+contexts:
+- context:
+    cluster: cluster
+    user: kube-router
+  name: kube-router-context
+current-context: kube-router-context

internal/resources/kubespray/project/roles/network_plugin/macvlan/defaults/main.yml

@@ -0,0 +1,6 @@
+---
+macvlan_interface: eth0
+enable_nat_default_gateway: true
+
+# sysctl_file_path to add sysctl conf to
+sysctl_file_path: "/etc/sysctl.d/99-sysctl.conf"

internal/resources/kubespray/project/roles/network_plugin/macvlan/files/ifdown-local

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+POSTDOWNNAME="/etc/sysconfig/network-scripts/post-down-$1"
+if [ -x $POSTDOWNNAME ]; then
+  exec $POSTDOWNNAME
+fi

internal/resources/kubespray/project/roles/network_plugin/macvlan/files/ifdown-macvlan

@@ -0,0 +1,40 @@
+#!/bin/bash
+#
+# initscripts-macvlan
+# Copyright (C) 2014 Lars Kellogg-Stedman
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+. /etc/init.d/functions
+
+cd /etc/sysconfig/network-scripts
+. ./network-functions
+
+[ -f ../network ] && . ../network
+
+CONFIG=${1}
+
+need_config ${CONFIG}
+
+source_config
+
+OTHERSCRIPT="/etc/sysconfig/network-scripts/ifdown-${REAL_DEVICETYPE}"
+
+if [ ! -x ${OTHERSCRIPT} ]; then
+	OTHERSCRIPT="/etc/sysconfig/network-scripts/ifdown-eth"
+fi
+
+${OTHERSCRIPT} ${CONFIG}
+
+ip link del ${DEVICE} type ${TYPE:-macvlan}

internal/resources/kubespray/project/roles/network_plugin/macvlan/files/ifup-local

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+POSTUPNAME="/etc/sysconfig/network-scripts/post-up-$1"
+if [ -x $POSTUPNAME ]; then
+  exec $POSTUPNAME
+fi

internal/resources/kubespray/project/roles/network_plugin/macvlan/files/ifup-macvlan

@@ -0,0 +1,43 @@
+#!/bin/bash
+#
+# initscripts-macvlan
+# Copyright (C) 2014 Lars Kellogg-Stedman
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+. /etc/init.d/functions
+
+cd /etc/sysconfig/network-scripts
+. ./network-functions
+
+[ -f ../network ] && . ../network
+
+CONFIG=${1}
+
+need_config ${CONFIG}
+
+source_config
+
+OTHERSCRIPT="/etc/sysconfig/network-scripts/ifup-${REAL_DEVICETYPE}"
+
+if [ ! -x ${OTHERSCRIPT} ]; then
+	OTHERSCRIPT="/etc/sysconfig/network-scripts/ifup-eth"
+fi
+
+ip link add \
+	link ${MACVLAN_PARENT} \
+	name ${DEVICE} \
+	type ${TYPE:-macvlan} mode ${MACVLAN_MODE:-private}
+
+${OTHERSCRIPT} ${CONFIG}

internal/resources/kubespray/project/roles/network_plugin/macvlan/handlers/main.yml

@@ -0,0 +1,15 @@
+---
+- name: Macvlan | reload network
+  service:
+    # noqa: jinja[spacing]
+    name: >-
+      {% if ansible_os_family == "RedHat" -%}
+      network
+      {%- elif ansible_distribution == "Ubuntu" and ansible_distribution_release == "bionic" -%}
+      systemd-networkd
+      {%- elif ansible_os_family == "Debian" -%}
+      networking
+      {%- endif %}
+    state: restarted
+  when: not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] and kube_network_plugin not in ['calico']
+  listen: Macvlan | restart network

internal/resources/kubespray/project/roles/network_plugin/macvlan/meta/main.yml

@@ -0,0 +1,3 @@
+---
+dependencies:
+  - role: network_plugin/cni

internal/resources/kubespray/project/roles/network_plugin/macvlan/tasks/main.yml

@@ -0,0 +1,110 @@
+---
+- name: Macvlan | Retrieve Pod Cidr
+  command: "{{ kubectl }} get nodes {{ kube_override_hostname | default(inventory_hostname) }} -o jsonpath='{.spec.podCIDR}'"
+  changed_when: false
+  register: node_pod_cidr_cmd
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+
+- name: Macvlan | set node_pod_cidr
+  set_fact:
+    node_pod_cidr: "{{ node_pod_cidr_cmd.stdout }}"
+
+- name: Macvlan | Retrieve default gateway network interface
+  become: false
+  raw: ip -4 route list 0/0 | sed 's/.*dev \([[:alnum:]]*\).*/\1/'
+  changed_when: false
+  register: node_default_gateway_interface_cmd
+
+- name: Macvlan | set node_default_gateway_interface
+  set_fact:
+    node_default_gateway_interface: "{{ node_default_gateway_interface_cmd.stdout | trim }}"
+
+- name: Macvlan | Install network gateway interface on debian
+  template:
+    src: debian-network-macvlan.cfg.j2
+    dest: /etc/network/interfaces.d/60-mac0.cfg
+    mode: "0644"
+  notify: Macvlan | restart network
+  when: ansible_os_family in ["Debian"]
+
+- name: Install macvlan config on RH distros
+  when: ansible_os_family == "RedHat"
+  block:
+  - name: Macvlan | Install macvlan script on centos
+    copy:
+      src: "{{ item }}"
+      dest: /etc/sysconfig/network-scripts/
+      owner: root
+      group: root
+      mode: "0755"
+    with_fileglob:
+    - files/*
+
+  - name: Macvlan | Install post-up script on centos
+    copy:
+      src: "files/ifup-local"
+      dest: /sbin/
+      owner: root
+      group: root
+      mode: "0755"
+    when: enable_nat_default_gateway
+
+  - name: Macvlan | Install network gateway interface on centos
+    template:
+      src: "{{ item.src }}.j2"
+      dest: "/etc/sysconfig/network-scripts/{{ item.dst }}"
+      mode: "0644"
+    with_items:
+    - {src: centos-network-macvlan.cfg, dst: ifcfg-mac0 }
+    - {src: centos-routes-macvlan.cfg, dst: route-mac0 }
+    - {src: centos-postup-macvlan.cfg, dst: post-up-mac0 }
+    notify: Macvlan | restart network
+
+- name: Install macvlan config on Flatcar
+  when: ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"]
+  block:
+  - name: Macvlan | Install service nat via gateway on Flatcar Container Linux
+    template:
+      src: coreos-service-nat_ouside.j2
+      dest: /etc/systemd/system/enable_nat_ouside.service
+      mode: "0644"
+    when: enable_nat_default_gateway
+
+  - name: Macvlan | Enable service nat via gateway on Flatcar Container Linux
+    command: "{{ item }}"
+    with_items:
+    - systemctl daemon-reload
+    - systemctl enable enable_nat_ouside.service
+    when: enable_nat_default_gateway
+
+  - name: Macvlan | Install network gateway interface on Flatcar Container Linux
+    template:
+      src: "{{ item.src }}.j2"
+      dest: "/etc/systemd/network/{{ item.dst }}"
+      mode: "0644"
+    with_items:
+    - {src: coreos-device-macvlan.cfg, dst: macvlan.netdev }
+    - {src: coreos-interface-macvlan.cfg, dst: output.network }
+    - {src: coreos-network-macvlan.cfg, dst: macvlan.network }
+    notify: Macvlan | restart network
+
+- name: Macvlan | Install cni definition for Macvlan
+  template:
+    src: 10-macvlan.conf.j2
+    dest: /etc/cni/net.d/10-macvlan.conf
+    mode: "0644"
+
+- name: Macvlan | Install loopback definition for Macvlan
+  template:
+    src: 99-loopback.conf.j2
+    dest: /etc/cni/net.d/99-loopback.conf
+    mode: "0644"
+
+- name: Enable net.ipv4.conf.all.arp_notify in sysctl
+  ansible.posix.sysctl:
+    name: net.ipv4.conf.all.arp_notify
+    value: 1
+    sysctl_set: true
+    sysctl_file: "{{ sysctl_file_path }}"
+    state: present
+    reload: true

internal/resources/kubespray/project/roles/network_plugin/macvlan/templates/10-macvlan.conf.j2

@@ -0,0 +1,15 @@
+{
+  "cniVersion": "0.4.0",
+  "name": "mynet",
+  "type": "macvlan",
+  "master": "{{ macvlan_interface }}",
+  "hairpinMode": true,
+  "ipam": {
+    "type": "host-local",
+    "subnet": "{{ node_pod_cidr }}",
+    "routes": [
+      { "dst": "0.0.0.0/0" }
+    ],
+    "gateway": "{{ node_pod_cidr|ansible.utils.ipaddr('net')|ansible.utils.ipaddr(1)|ansible.utils.ipaddr('address') }}"
+  }
+}

internal/resources/kubespray/project/roles/network_plugin/macvlan/templates/99-loopback.conf.j2

@@ -0,0 +1,5 @@
+{
+  "cniVersion": "0.2.0",
+  "name": "lo",
+  "type": "loopback"
+}

internal/resources/kubespray/project/roles/network_plugin/macvlan/templates/centos-network-macvlan.cfg.j2

@@ -0,0 +1,13 @@
+DEVICE=mac0
+DEVICETYPE=macvlan
+TYPE=macvlan
+BOOTPROTO=none
+ONBOOT=yes
+NM_CONTROLLED=no
+
+MACVLAN_PARENT={{ macvlan_interface }}
+MACVLAN_MODE=bridge
+
+IPADDR={{ node_pod_cidr|ansible.utils.ipaddr('net')|ansible.utils.ipaddr(1)|ansible.utils.ipaddr('address') }}
+NETMASK={{ node_pod_cidr|ansible.utils.ipaddr('netmask') }}
+NETWORK={{ node_pod_cidr|ansible.utils.ipaddr('network') }}

internal/resources/kubespray/project/roles/network_plugin/macvlan/templates/centos-postdown-macvlan.cfg.j2

@@ -0,0 +1,3 @@
+{% if enable_nat_default_gateway %}
+iptables -t nat -D POSTROUTING -s {{ node_pod_cidr|ansible.utils.ipaddr('net') }} -o {{ node_default_gateway_interface }} -j MASQUERADE
+{% endif %}

internal/resources/kubespray/project/roles/network_plugin/macvlan/templates/centos-postup-macvlan.cfg.j2

@@ -0,0 +1,3 @@
+{% if enable_nat_default_gateway %}
+iptables -t nat -I POSTROUTING -s {{ node_pod_cidr|ansible.utils.ipaddr('net') }} -o {{ node_default_gateway_interface }} -j MASQUERADE
+{% endif %}

internal/resources/kubespray/project/roles/network_plugin/macvlan/templates/centos-routes-macvlan.cfg.j2

@@ -0,0 +1,7 @@
+{% for host in groups['kube_node'] %}
+{% if hostvars[host]['access_ip'] is defined  %}
+{% if hostvars[host]['node_pod_cidr'] != node_pod_cidr  %}
+{{ hostvars[host]['node_pod_cidr'] }} via {{ hostvars[host]['access_ip'] }}
+{% endif %}
+{% endif %}
+{% endfor %}

internal/resources/kubespray/project/roles/network_plugin/macvlan/templates/coreos-device-macvlan.cfg.j2

@@ -0,0 +1,6 @@
+[NetDev]
+Name=mac0
+Kind=macvlan
+
+[MACVLAN]
+Mode=bridge

internal/resources/kubespray/project/roles/network_plugin/macvlan/templates/coreos-interface-macvlan.cfg.j2

@@ -0,0 +1,6 @@
+[Match]
+Name={{ macvlan_interface }}
+
+[Network]
+MACVLAN=mac0
+DHCP=yes

internal/resources/kubespray/project/roles/network_plugin/macvlan/templates/coreos-network-macvlan.cfg.j2

@@ -0,0 +1,17 @@
+[Match]
+Name=mac0
+
+[Network]
+Address={{ node_pod_cidr|ansible.utils.ipaddr('net')|ansible.utils.ipaddr(1)|ansible.utils.ipaddr('address') }}/{{ node_pod_cidr|ansible.utils.ipaddr('prefix') }}
+
+{% for host in groups['kube_node'] %}
+{% if hostvars[host]['access_ip'] is defined  %}
+{% if hostvars[host]['node_pod_cidr'] != node_pod_cidr  %}
+[Route]
+Gateway={{ hostvars[host]['access_ip'] }}
+Destination={{ hostvars[host]['node_pod_cidr'] }}
+GatewayOnlink=yes
+
+{% endif %}
+{% endif %}
+{% endfor %}

internal/resources/kubespray/project/roles/network_plugin/macvlan/templates/coreos-service-nat_ouside.j2

@@ -0,0 +1,6 @@
+[Service]
+Type=oneshot
+ExecStart=/bin/bash -c "iptables -t nat -I POSTROUTING -s {{ node_pod_cidr|ansible.utils.ipaddr('net') }} -o {{ node_default_gateway_interface }} -j MASQUERADE"
+
+[Install]
+WantedBy=sys-subsystem-net-devices-mac0.device

internal/resources/kubespray/project/roles/network_plugin/macvlan/templates/debian-network-macvlan.cfg.j2

@@ -0,0 +1,26 @@
+auto mac0
+iface mac0 inet static
+    address {{ node_pod_cidr|ansible.utils.ipaddr('net')|ansible.utils.ipaddr(1)|ansible.utils.ipaddr('address') }}
+    network {{ node_pod_cidr|ansible.utils.ipaddr('network') }}
+    netmask {{ node_pod_cidr|ansible.utils.ipaddr('netmask') }}
+    broadcast {{ node_pod_cidr|ansible.utils.ipaddr('broadcast') }}
+    pre-up ip link add link {{ macvlan_interface }} mac0 type macvlan mode bridge
+{% for host in groups['kube_node'] %}
+{% if hostvars[host]['access_ip'] is defined  %}
+{% if hostvars[host]['node_pod_cidr'] != node_pod_cidr  %}
+    post-up ip route add {{ hostvars[host]['node_pod_cidr'] }} via {{ hostvars[host]['access_ip'] }}
+{% endif %}
+{% endif %}
+{% endfor %}
+{% if enable_nat_default_gateway %}
+    post-up iptables -t nat -I POSTROUTING -s {{ node_pod_cidr|ansible.utils.ipaddr('net') }} -o {{ node_default_gateway_interface }} -j MASQUERADE
+{% endif %}
+{% for host in groups['kube_node'] %}
+{% if hostvars[host]['access_ip'] is defined  %}
+{% if hostvars[host]['node_pod_cidr'] != node_pod_cidr  %}
+    post-down ip route del {{ hostvars[host]['node_pod_cidr'] }} via {{ hostvars[host]['access_ip'] }}
+{% endif %}
+{% endif %}
+{% endfor %}
+    post-down iptables -t nat -D POSTROUTING -s {{ node_pod_cidr|ansible.utils.ipaddr('net') }} -o {{ node_default_gateway_interface }} -j MASQUERADE
+    post-down ip link delete mac0

internal/resources/kubespray/project/roles/network_plugin/meta/main.yml

@@ -0,0 +1,49 @@
+---
+dependencies:
+  - role: network_plugin/cni
+    when: kube_network_plugin != 'none'
+
+  - role: network_plugin/cilium
+    when: kube_network_plugin == 'cilium' or cilium_deploy_additionally
+    tags:
+      - cilium
+
+  - role: network_plugin/calico
+    when: kube_network_plugin == 'calico'
+    tags:
+      - calico
+
+  - role: network_plugin/flannel
+    when: kube_network_plugin == 'flannel'
+    tags:
+      - flannel
+
+  - role: network_plugin/weave
+    when: kube_network_plugin == 'weave'
+    tags:
+      - weave
+
+  - role: network_plugin/macvlan
+    when: kube_network_plugin == 'macvlan'
+    tags:
+      - macvlan
+
+  - role: network_plugin/kube-ovn
+    when: kube_network_plugin == 'kube-ovn'
+    tags:
+      - kube-ovn
+
+  - role: network_plugin/kube-router
+    when: kube_network_plugin == 'kube-router'
+    tags:
+      - kube-router
+
+  - role: network_plugin/custom_cni
+    when: kube_network_plugin == 'custom_cni'
+    tags:
+      - custom_cni
+
+  - role: network_plugin/multus
+    when: kube_network_plugin_multus
+    tags:
+      - multus

internal/resources/kubespray/project/roles/network_plugin/multus/defaults/main.yml

@@ -0,0 +1,10 @@
+---
+multus_conf_file: "auto"
+multus_cni_conf_dir_host: "/etc/cni/net.d"
+multus_cni_bin_dir_host: "/opt/cni/bin"
+multus_cni_run_dir_host: "/run"
+multus_cni_conf_dir: "{{ ('/host', multus_cni_conf_dir_host) | join }}"
+multus_cni_bin_dir: "{{ ('/host', multus_cni_bin_dir_host) | join }}"
+multus_cni_run_dir: "{{ ('/host', multus_cni_run_dir_host) | join }}"
+multus_kubeconfig_file_host: "{{ (multus_cni_conf_dir_host, '/multus.d/multus.kubeconfig') | join }}"
+multus_namespace_isolation: false