kube-forge v2: embede python with ansible (kubespray) and cobra as cli building tool

internal/resources/kubespray/project/roles/kubernetes-apps/ansible/defaults/main.yml

@@ -0,0 +1,133 @@
+---
+# Limits for coredns
+
+# uncomment the line below to customize the DNS cpu limit value
+# dns_cpu_limit: 300m
+dns_memory_limit: 300Mi
+dns_cpu_requests: 100m
+dns_memory_requests: 70Mi
+dns_min_replicas: "{{ [2, groups['k8s_cluster'] | length] | min }}"
+dns_nodes_per_replica: 16
+dns_cores_per_replica: 256
+dns_prevent_single_point_failure: "{{ 'true' if dns_min_replicas | int > 1 else 'false' }}"
+enable_coredns_reverse_dns_lookups: true
+coredns_ordinal_suffix: ""
+# dns_extra_tolerations: [{effect: NoSchedule, operator: "Exists"}]
+coredns_deployment_nodeselector: "kubernetes.io/os: linux"
+coredns_default_zone_cache_block: |
+  cache 30
+
+coredns_pod_disruption_budget: false
+# value for coredns pdb
+coredns_pod_disruption_budget_max_unavailable: "30%"
+deploy_coredns: true
+# coredns_additional_configs adds any extra configuration to coredns
+# coredns_additional_configs: |
+#   whoami
+#   local
+
+# coredns_rewrite_block: |
+#   rewrite stop {
+#     name regex (.*)\.my\.domain {1}.svc.cluster.local
+#     answer name (.*)\.svc\.cluster\.local {1}.my.domain
+#   }
+
+# coredns_additional_error_config: |
+#   consolidate 5m ".* i/o timeout$" warning
+
+# Configure coredns and nodelocaldns to correctly answer DNS queries when you changed
+# your 'dns_domain' and some workloads used it directly.
+old_dns_domains: []
+
+# dns_upstream_forward_extra_opts apply to coredns forward section as well as nodelocaldns upstream target forward section
+# dns_upstream_forward_extra_opts:
+#   policy: sequential
+
+# Apply extra options to coredns kubernetes plugin
+# coredns_kubernetes_extra_opts:
+#   - 'fallthrough example.local'
+
+# nodelocaldns
+nodelocaldns_cpu_requests: 100m
+nodelocaldns_memory_limit: 200Mi
+nodelocaldns_memory_requests: 70Mi
+nodelocaldns_ds_nodeselector: "kubernetes.io/os: linux"
+nodelocaldns_prometheus_port: 9253
+nodelocaldns_secondary_prometheus_port: 9255
+
+# nodelocaldns_additional_configs adds any extra configuration to coredns
+# nodelocaldns_additional_configs: |
+#   whoami
+#   local
+
+# Limits for dns-autoscaler
+dns_autoscaler_cpu_requests: 20m
+dns_autoscaler_memory_requests: 10Mi
+dns_autoscaler_deployment_nodeselector: "kubernetes.io/os: linux"
+# dns_autoscaler_extra_tolerations: [{effect: NoSchedule, operator: "Exists"}]
+dns_autoscaler_affinity: {}
+
+# etcd metrics
+# etcd_metrics_service_labels:
+#   k8s-app: etcd
+#   app.kubernetes.io/managed-by: Kubespray
+#   app: kube-prometheus-stack-kube-etcd
+#   release: prometheus-stack
+
+# Netchecker
+deploy_netchecker: false
+netchecker_port: 31081
+agent_report_interval: 15
+netcheck_namespace: default
+
+# Limits for netchecker apps
+netchecker_agent_cpu_limit: 30m
+netchecker_agent_memory_limit: 100M
+netchecker_agent_cpu_requests: 15m
+netchecker_agent_memory_requests: 64M
+netchecker_server_cpu_limit: 100m
+netchecker_server_memory_limit: 256M
+netchecker_server_cpu_requests: 50m
+netchecker_server_memory_requests: 64M
+netchecker_etcd_cpu_limit: 200m
+netchecker_etcd_memory_limit: 256M
+netchecker_etcd_cpu_requests: 100m
+netchecker_etcd_memory_requests: 128M
+
+# SecurityContext (user/group)
+netchecker_agent_user: 1000
+netchecker_server_user: 1000
+netchecker_agent_group: 1000
+netchecker_server_group: 1000
+
+# Log levels
+netchecker_agent_log_level: 5
+netchecker_server_log_level: 5
+netchecker_etcd_log_level: info
+
+# Dashboard
+dashboard_replicas: 1
+
+# Namespace for dashboard
+dashboard_namespace: kube-system
+
+# Limits for dashboard
+dashboard_cpu_limit: 100m
+dashboard_memory_limit: 256M
+dashboard_cpu_requests: 50m
+dashboard_memory_requests: 64M
+
+# Set dashboard_use_custom_certs to true if overriding dashboard_certs_secret_name with a secret that
+# contains dashboard_tls_key_file and dashboard_tls_cert_file instead of using the initContainer provisioned certs
+dashboard_use_custom_certs: false
+dashboard_certs_secret_name: kubernetes-dashboard-certs
+dashboard_tls_key_file: dashboard.key
+dashboard_tls_cert_file: dashboard.crt
+dashboard_master_toleration: true
+
+# Override dashboard default settings
+dashboard_token_ttl: 900
+dashboard_skip_login: false
+
+# Policy Controllers
+# policy_controller_extra_tolerations: [{effect: NoSchedule, operator: "Exists"}]

internal/resources/kubespray/project/roles/kubernetes-apps/ansible/tasks/main.yml

@@ -0,0 +1,123 @@
+---
+- name: Kubernetes Apps | Wait for kube-apiserver
+  uri:
+    url: "{{ kube_apiserver_endpoint }}/healthz"
+    validate_certs: false
+    client_cert: "{{ kube_apiserver_client_cert }}"
+    client_key: "{{ kube_apiserver_client_key }}"
+  register: result
+  until: result.status == 200
+  retries: 20
+  delay: 1
+  when: inventory_hostname == groups['kube_control_plane'][0]
+
+- name: Kubernetes Apps | CoreDNS
+  command:
+    cmd: "{{ kubectl_apply_stdin }}"
+    stdin: "{{ lookup('template', item) }}"
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+  run_once: true
+  loop: "{{ coredns_manifests | flatten }}"
+  tags:
+    - coredns
+  vars:
+    clusterIP: "{{ skydns_server }}"
+  when:
+    - dns_mode in ['coredns', 'coredns_dual']
+    - deploy_coredns
+
+- name: Kubernetes Apps | CoreDNS Secondary
+  command:
+    cmd: "{{ kubectl_apply_stdin }}"
+    stdin: "{{ lookup('template', item) }}"
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+  run_once: true
+  loop: "{{ coredns_manifests | flatten }}"
+  tags:
+    - coredns
+  vars:
+    clusterIP: "{{ skydns_server_secondary }}"
+    coredns_ordinal_suffix: "-secondary"
+  when:
+    - dns_mode == 'coredns_dual'
+    - deploy_coredns
+
+- name: Kubernetes Apps | nodelocalDNS
+  command:
+    cmd: "{{ kubectl_apply_stdin }}"
+    stdin: "{{ lookup('template', item) }}"
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+  run_once: true
+  loop: "{{ nodelocaldns_manifests | flatten }}"
+  when:
+    - enable_nodelocaldns
+  tags:
+    - nodelocaldns
+    - coredns
+  vars:
+    primaryClusterIP: >-
+      {%- if dns_mode in ['coredns', 'coredns_dual'] -%}
+      {{ skydns_server }}
+      {%- elif dns_mode == 'manual' -%}
+      {{ manual_dns_server }}
+      {%- endif -%}
+    secondaryclusterIP: "{{ skydns_server_secondary }}"
+    forwardTarget: >-
+      {%- if secondaryclusterIP is defined and dns_mode == 'coredns_dual' -%}
+      {{ primaryClusterIP }} {{ secondaryclusterIP }}
+      {%- else -%}
+      {{ primaryClusterIP }}
+      {%- endif -%}
+    upstreamForwardTarget: >-
+      {%- if upstream_dns_servers | length > 0 -%}
+      {{ upstream_dns_servers | join(' ') }}
+      {%- else -%}
+      /etc/resolv.conf
+      {%- endif -%}
+
+- name: Kubernetes Apps | Etcd metrics endpoints
+  command:
+    cmd: "{{ kubectl_apply_stdin }}"
+    stdin: "{{ lookup('template', item) }}"
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+  run_once: true
+  loop:
+    - etcd_metrics-endpoints.yml.j2
+    - etcd_metrics-service.yml.j2
+  when: etcd_metrics_port is defined and etcd_metrics_service_labels is defined
+  tags:
+    - etcd_metrics
+
+- name: Kubernetes Apps | Netchecker
+  command:
+    cmd: "{{ kubectl_apply_stdin }}"
+    stdin: "{{ lookup('template', item) }}"
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+  run_once: true
+  vars:
+    namespace: "{{ netcheck_namespace }}"
+  when: deploy_netchecker
+  tags:
+    - netchecker
+  loop:
+    - netchecker-ns.yml.j2
+    - netchecker-agent-sa.yml.j2
+    - netchecker-agent-ds.yml.j2
+    - netchecker-agent-hostnet-ds.yml.j2
+    - netchecker-server-sa.yml.j2
+    - netchecker-server-clusterrole.yml.j2
+    - netchecker-server-clusterrolebinding.yml.j2
+    - netchecker-server-deployment.yml.j2
+    - netchecker-server-svc.yml.j2
+
+- name: Kubernetes Apps | Dashboard
+  command:
+    cmd: "{{ kubectl_apply_stdin }}"
+    stdin: "{{ lookup('template', 'dashboard.yml.j2') }}"
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+  run_once: true
+  vars:
+    namespace: "{{ dashboard_namespace }}"
+  when: dashboard_enabled
+  tags:
+    - dashboard

internal/resources/kubespray/project/roles/kubernetes-apps/ansible/templates/coredns-clusterrole.yml.j2

@@ -0,0 +1,32 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    kubernetes.io/bootstrapping: rbac-defaults
+    addonmanager.kubernetes.io/mode: Reconcile
+  name: system:coredns
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  - services
+  - pods
+  - namespaces
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+- apiGroups:
+  - discovery.k8s.io
+  resources:
+  - endpointslices
+  verbs:
+  - list
+  - watch

internal/resources/kubespray/project/roles/kubernetes-apps/ansible/templates/coredns-clusterrolebinding.yml.j2

@@ -0,0 +1,18 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  annotations:
+    rbac.authorization.kubernetes.io/autoupdate: "true"
+  labels:
+    kubernetes.io/bootstrapping: rbac-defaults
+    addonmanager.kubernetes.io/mode: EnsureExists
+  name: system:coredns
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:coredns
+subjects:
+  - kind: ServiceAccount
+    name: coredns
+    namespace: kube-system

internal/resources/kubespray/project/roles/kubernetes-apps/ansible/templates/coredns-config.yml.j2

@@ -0,0 +1,98 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: coredns
+  namespace: kube-system
+  labels:
+      addonmanager.kubernetes.io/mode: EnsureExists
+data:
+  Corefile: |
+{% if coredns_external_zones is defined and coredns_external_zones | length > 0 %}
+{%   for block in coredns_external_zones %}
+    {{ block['zones'] | join(' ') }} {
+        log
+        errors {
+{% if coredns_additional_error_config is defined %}
+          {{ coredns_additional_error_config | indent(width=10, first=False) }}
+{% endif %}
+        }
+{% if block['rewrite'] is defined and block['rewrite'] | length > 0 %}
+{% for rewrite_match in block['rewrite'] %}
+        rewrite {{ rewrite_match }}
+{% endfor %}
+{% endif %}
+        forward . {{ block['nameservers'] | join(' ') }}
+        loadbalance
+        cache {{ block['cache'] | default(5) }}
+        reload
+{% if dns_etchosts | default(None) %}
+        hosts /etc/coredns/hosts {
+          fallthrough
+        }
+{% endif %}
+    }
+{%   endfor %}
+{% endif %}
+    .:53 {
+{% if coredns_additional_configs is defined %}
+        {{ coredns_additional_configs | indent(width=8, first=False) }}
+{% endif %}
+        errors {
+{% if coredns_additional_error_config is defined %}
+          {{ coredns_additional_error_config | indent(width=10, first=False) }}
+{% endif %}
+        }
+        health {
+            lameduck 5s
+        }
+{% if coredns_rewrite_block is defined %}
+        {{ coredns_rewrite_block | indent(width=8, first=False) }}
+{% endif %}
+{% for old_dns_domain in old_dns_domains %}
+        rewrite name suffix {{ old_dns_domain }} {{ dns_domain }} answer auto
+{% endfor %}
+        ready
+        kubernetes {{ dns_domain }} {% if coredns_kubernetes_extra_domains is defined %}{{ coredns_kubernetes_extra_domains }} {% endif %}{% if enable_coredns_reverse_dns_lookups %}in-addr.arpa ip6.arpa {% endif %}{
+          pods insecure
+{% if enable_coredns_k8s_endpoint_pod_names %}
+          endpoint_pod_names
+{% endif %}
+{% if enable_coredns_reverse_dns_lookups %}
+          fallthrough in-addr.arpa ip6.arpa
+{% endif %}
+{% if coredns_kubernetes_extra_opts is defined %}
+{% for opt in coredns_kubernetes_extra_opts %}
+          {{ opt }}
+{% endfor %}
+{% endif %}
+        }
+        prometheus :9153
+        forward . {{ upstream_dns_servers | join(' ') if upstream_dns_servers | length > 0 else '/etc/resolv.conf' }} {
+          prefer_udp
+          max_concurrent 1000
+{% if dns_upstream_forward_extra_opts is defined %}
+{% for optname, optvalue in dns_upstream_forward_extra_opts.items() %}
+          {{ (optname ~ ' ' ~ optvalue) | trim }}
+          {# do not add a trailing space when optvalue == ''
+             workaround for: https://github.com/kubernetes/kubernetes/issues/36222 #}
+{% endfor %}
+{% endif %}
+        }
+{% if enable_coredns_k8s_external %}
+        k8s_external {{ coredns_k8s_external_zone }}
+{% endif %}
+        {{ coredns_default_zone_cache_block | indent(width=8, first=False) }}
+        loop
+        reload
+        loadbalance
+{% if dns_etchosts | default(None) %}
+        hosts /etc/coredns/hosts {
+          fallthrough
+        }
+{% endif %}
+    }
+{% if dns_etchosts | default(None) %}
+  hosts: |
+    {{ dns_etchosts | indent(width=4, first=False) }}
+{% endif %}

internal/resources/kubespray/project/roles/kubernetes-apps/ansible/templates/coredns-deployment.yml.j2

@@ -0,0 +1,122 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: "coredns{{ coredns_ordinal_suffix }}"
+  namespace: kube-system
+  labels:
+    k8s-app: "kube-dns{{ coredns_ordinal_suffix }}"
+    addonmanager.kubernetes.io/mode: Reconcile
+    kubernetes.io/name: "coredns{{ coredns_ordinal_suffix }}"
+spec:
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 1
+      maxSurge: 10%
+  selector:
+    matchLabels:
+      k8s-app: kube-dns{{ coredns_ordinal_suffix }}
+  template:
+    metadata:
+      labels:
+        k8s-app: kube-dns{{ coredns_ordinal_suffix }}
+      annotations:
+        createdby: 'kubespray'
+    spec:
+      securityContext:
+        seccompProfile:
+          type: RuntimeDefault
+      nodeSelector:
+        {{ coredns_deployment_nodeselector }}
+      priorityClassName: system-cluster-critical
+      serviceAccountName: coredns
+      tolerations:
+        - key: node-role.kubernetes.io/control-plane
+          effect: NoSchedule
+{% if dns_extra_tolerations is defined %}
+        {{ dns_extra_tolerations | list | to_nice_yaml(indent=2) | indent(8) }}
+{% endif %}
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+          - topologyKey: "kubernetes.io/hostname"
+            labelSelector:
+              matchLabels:
+                k8s-app: kube-dns{{ coredns_ordinal_suffix }}
+        nodeAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            preference:
+              matchExpressions:
+              - key: node-role.kubernetes.io/control-plane
+                operator: In
+                values:
+                - ""
+      containers:
+      - name: coredns
+        image: "{{ coredns_image_repo }}:{{ coredns_image_tag }}"
+        imagePullPolicy: {{ k8s_image_pull_policy }}
+        resources:
+          # TODO: Set memory limits when we've profiled the container for large
+          # clusters, then set request = limit to keep this container in
+          # guaranteed class. Currently, this container falls into the
+          # "burstable" category so the kubelet doesn't backoff from restarting it.
+          limits:
+{% if dns_cpu_limit is defined %}
+            cpu: {{ dns_cpu_limit }}
+{% endif %}
+            memory: {{ dns_memory_limit }}
+          requests:
+            cpu: {{ dns_cpu_requests }}
+            memory: {{ dns_memory_requests }}
+        args: [ "-conf", "/etc/coredns/Corefile" ]
+        volumeMounts:
+        - name: config-volume
+          mountPath: /etc/coredns
+        ports:
+        - containerPort: 53
+          name: dns
+          protocol: UDP
+        - containerPort: 53
+          name: dns-tcp
+          protocol: TCP
+        - containerPort: 9153
+          name: metrics
+          protocol: TCP
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            add:
+            - NET_BIND_SERVICE
+            drop:
+            - all
+          readOnlyRootFilesystem: true
+        livenessProbe:
+          httpGet:
+            path: /health
+            port: 8080
+            scheme: HTTP
+          timeoutSeconds: 5
+          successThreshold: 1
+          failureThreshold: 10
+        readinessProbe:
+          httpGet:
+            path: /ready
+            port: 8181
+            scheme: HTTP
+          timeoutSeconds: 5
+          successThreshold: 1
+          failureThreshold: 10
+      dnsPolicy: Default
+      volumes:
+        - name: config-volume
+          configMap:
+            name: coredns
+            items:
+            - key: Corefile
+              path: Corefile
+{% if dns_etchosts | default(None) %}
+            - key: hosts
+              path: hosts
+{% endif %}

internal/resources/kubespray/project/roles/kubernetes-apps/ansible/templates/coredns-poddisruptionbudget.yml.j2

@@ -0,0 +1,9 @@
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: coredns{{ coredns_ordinal_suffix }}
+spec:
+  maxUnavailable: {{ coredns_pod_disruption_budget_max_unavailable }}
+  selector:
+    matchLabels:
+      k8s-app: kube-dns{{ coredns_ordinal_suffix }}

internal/resources/kubespray/project/roles/kubernetes-apps/ansible/templates/coredns-sa.yml.j2

@@ -0,0 +1,9 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: coredns
+  namespace: kube-system
+  labels:
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile

internal/resources/kubespray/project/roles/kubernetes-apps/ansible/templates/coredns-svc.yml.j2

@@ -0,0 +1,28 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: coredns{{ coredns_ordinal_suffix }}
+  namespace: kube-system
+  labels:
+    k8s-app: kube-dns{{ coredns_ordinal_suffix }}
+    kubernetes.io/name: "coredns{{ coredns_ordinal_suffix }}"
+    addonmanager.kubernetes.io/mode: Reconcile
+  annotations:
+    prometheus.io/port: "9153"
+    prometheus.io/scrape: "true"
+    createdby: 'kubespray'
+spec:
+  selector:
+    k8s-app: kube-dns{{ coredns_ordinal_suffix }}
+  clusterIP: {{ clusterIP }}
+  ports:
+    - name: dns
+      port: 53
+      protocol: UDP
+    - name: dns-tcp
+      port: 53
+      protocol: TCP
+    - name: metrics
+      port: 9153
+      protocol: TCP

internal/resources/kubespray/project/roles/kubernetes-apps/ansible/templates/dashboard.yml.j2

@@ -0,0 +1,323 @@
+# Copyright 2017 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Configuration to deploy release version of the Dashboard UI compatible with
+# Kubernetes 1.8.
+#
+# Example usage: kubectl create -f <this_file>
+
+{% if namespace != 'kube-system' %}
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: {{ namespace }}
+  labels:
+    name: {{ namespace }}
+{% endif %}
+---
+# ------------------- Dashboard Secrets ------------------- #
+apiVersion: v1
+kind: Secret
+metadata:
+  labels:
+    k8s-app: kubernetes-dashboard
+  name: kubernetes-dashboard-certs
+type: Opaque
+
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  labels:
+    k8s-app: kubernetes-dashboard
+  name: kubernetes-dashboard-csrf
+type: Opaque
+data:
+  csrf: ""
+
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  labels:
+    k8s-app: kubernetes-dashboard
+  name: kubernetes-dashboard-key-holder
+type: Opaque
+
+---
+# ------------------- Dashboard ConfigMap ------------------- #
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  labels:
+    k8s-app: kubernetes-dashboard
+  name: kubernetes-dashboard-settings
+
+---
+# ------------------- Dashboard Service Account ------------------- #
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    k8s-app: kubernetes-dashboard
+  name: kubernetes-dashboard
+
+---
+# ------------------- Dashboard Role & Role Binding ------------------- #
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  labels:
+    k8s-app: kubernetes-dashboard
+  name: kubernetes-dashboard
+rules:
+  # Allow Dashboard to get, update and delete Dashboard exclusive secrets.
+  - apiGroups: [""]
+    resources: ["secrets"]
+    resourceNames: ["kubernetes-dashboard-key-holder", "kubernetes-dashboard-certs", "kubernetes-dashboard-csrf"]
+    verbs: ["get", "update", "delete"]
+    # Allow Dashboard to get and update 'kubernetes-dashboard-settings' config map.
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    resourceNames: ["kubernetes-dashboard-settings"]
+    verbs: ["get", "update"]
+    # Allow Dashboard to get metrics.
+  - apiGroups: [""]
+    resources: ["services"]
+    resourceNames: ["heapster", "dashboard-metrics-scraper"]
+    verbs: ["proxy"]
+  - apiGroups: [""]
+    resources: ["services/proxy"]
+    resourceNames: ["heapster", "http:heapster:", "https:heapster:", "dashboard-metrics-scraper", "http:dashboard-metrics-scraper"]
+    verbs: ["get"]
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    k8s-app: kubernetes-dashboard
+  name: kubernetes-dashboard
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: kubernetes-dashboard
+subjects:
+  - kind: ServiceAccount
+    name: kubernetes-dashboard
+    namespace: {{ namespace }}
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kubernetes-dashboard
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kubernetes-dashboard
+subjects:
+  - kind: ServiceAccount
+    name: kubernetes-dashboard
+    namespace: {{ namespace }}
+
+---
+# ------------------- Dashboard Deployment ------------------- #
+
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  labels:
+    k8s-app: kubernetes-dashboard
+  name: kubernetes-dashboard
+spec:
+  replicas: {{ dashboard_replicas }}
+  revisionHistoryLimit: 10
+  selector:
+    matchLabels:
+      k8s-app: kubernetes-dashboard
+  template:
+    metadata:
+      labels:
+        k8s-app: kubernetes-dashboard
+    spec:
+      securityContext:
+        seccompProfile:
+          type: RuntimeDefault
+      priorityClassName: system-cluster-critical
+      containers:
+        - name: kubernetes-dashboard
+          image: {{ dashboard_image_repo }}:{{ dashboard_image_tag }}
+          imagePullPolicy: {{ k8s_image_pull_policy }}
+          resources:
+            limits:
+              cpu: {{ dashboard_cpu_limit }}
+              memory: {{ dashboard_memory_limit }}
+            requests:
+              cpu: {{ dashboard_cpu_requests }}
+              memory: {{ dashboard_memory_requests }}
+          ports:
+            - containerPort: 8443
+              protocol: TCP
+          args:
+            - --namespace={{ namespace }}
+{% if dashboard_use_custom_certs %}
+            - --tls-key-file={{ dashboard_tls_key_file }}
+            - --tls-cert-file={{ dashboard_tls_cert_file }}
+{% else %}
+            - --auto-generate-certificates
+{% endif %}
+{% if dashboard_skip_login %}
+            - --enable-skip-login
+{% endif %}
+            - --authentication-mode=token
+            # Uncomment the following line to manually specify Kubernetes API server Host
+            # If not specified, Dashboard will attempt to auto discover the API server and connect
+            # to it. Uncomment only if the default does not work.
+            # - --apiserver-host=http://my-address:port
+            - --token-ttl={{ dashboard_token_ttl }}
+          volumeMounts:
+            - name: kubernetes-dashboard-certs
+              mountPath: /certs
+              # Create on-disk volume to store exec logs
+            - mountPath: /tmp
+              name: tmp-volume
+          livenessProbe:
+            httpGet:
+              scheme: HTTPS
+              path: /
+              port: 8443
+            initialDelaySeconds: 30
+            timeoutSeconds: 30
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            runAsUser: 1001
+            runAsGroup: 2001
+      volumes:
+        - name: kubernetes-dashboard-certs
+          secret:
+            secretName: {{ dashboard_certs_secret_name }}
+        - name: tmp-volume
+          emptyDir: {}
+      serviceAccountName: kubernetes-dashboard
+{% if dashboard_master_toleration %}
+      tolerations:
+        - key: node-role.kubernetes.io/control-plane
+          effect: NoSchedule
+{% endif %}
+
+---
+# ------------------- Dashboard Service ------------------- #
+
+kind: Service
+apiVersion: v1
+metadata:
+  labels:
+    k8s-app: kubernetes-dashboard
+  name: kubernetes-dashboard
+spec:
+  ports:
+    - port: 443
+      targetPort: 8443
+  selector:
+    k8s-app: kubernetes-dashboard
+
+---
+# ------------------- Metrics Scraper Service Account ------------------- #
+
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  labels:
+    k8s-app: kubernetes-dashboard
+  name: kubernetes-dashboard
+rules:
+  # Allow Metrics Scraper to get metrics from the Metrics server
+  - apiGroups: ["metrics.k8s.io"]
+    resources: ["pods", "nodes"]
+    verbs: ["get", "list", "watch"]
+
+---
+
+# ------------------- Metrics Scraper Service  ------------------- #
+kind: Service
+apiVersion: v1
+metadata:
+  labels:
+    k8s-app: kubernetes-metrics-scraper
+  name: dashboard-metrics-scraper
+spec:
+  ports:
+    - port: 8000
+      targetPort: 8000
+  selector:
+    k8s-app: kubernetes-metrics-scraper
+
+---
+
+# ------------------- Metrics Scraper Deployment  ------------------- #
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  labels:
+    k8s-app: kubernetes-metrics-scraper
+  name: kubernetes-metrics-scraper
+spec:
+  replicas: 1
+  revisionHistoryLimit: 10
+  selector:
+    matchLabels:
+      k8s-app: kubernetes-metrics-scraper
+  template:
+    metadata:
+      labels:
+        k8s-app: kubernetes-metrics-scraper
+    spec:
+      securityContext:
+        seccompProfile:
+          type: RuntimeDefault
+      priorityClassName: system-cluster-critical
+      containers:
+        - name: kubernetes-metrics-scraper
+          image: {{ dashboard_metrics_scraper_repo }}:{{ dashboard_metrics_scraper_tag }}
+          ports:
+            - containerPort: 8000
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              scheme: HTTP
+              path: /
+              port: 8000
+            initialDelaySeconds: 30
+            timeoutSeconds: 30
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            runAsUser: 1001
+            runAsGroup: 2001
+          volumeMounts:
+          - mountPath: /tmp
+            name: tmp-volume
+      serviceAccountName: kubernetes-dashboard
+      volumes:
+      - name: tmp-volume
+        emptyDir: {}
+{% if dashboard_master_toleration %}
+      tolerations:
+        - key: node-role.kubernetes.io/control-plane
+          effect: NoSchedule
+{% endif %}

internal/resources/kubespray/project/roles/kubernetes-apps/ansible/templates/dns-autoscaler-clusterrole.yml.j2

@@ -0,0 +1,34 @@
+---
+# Copyright 2016 The Kubernetes Authors. All rights reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: system:dns-autoscaler
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
+rules:
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["list", "watch"]
+  - apiGroups: [""]
+    resources: ["replicationcontrollers/scale"]
+    verbs: ["get", "update"]
+  - apiGroups: ["extensions", "apps"]
+    resources: ["deployments/scale", "replicasets/scale"]
+    verbs: ["get", "update"]
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    verbs: ["get", "create"]

internal/resources/kubespray/project/roles/kubernetes-apps/ansible/templates/dns-autoscaler-clusterrolebinding.yml.j2

@@ -0,0 +1,29 @@
+---
+# Copyright 2016 The Kubernetes Authors. All rights reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: system:dns-autoscaler
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
+subjects:
+  - kind: ServiceAccount
+    name: dns-autoscaler
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: system:dns-autoscaler
+  apiGroup: rbac.authorization.k8s.io

internal/resources/kubespray/project/roles/kubernetes-apps/ansible/templates/dns-autoscaler-sa.yml.j2

@@ -0,0 +1,22 @@
+---
+# Copyright 2016 The Kubernetes Authors. All rights reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+kind: ServiceAccount
+apiVersion: v1
+metadata:
+  name: dns-autoscaler
+  namespace: kube-system
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile

internal/resources/kubespray/project/roles/kubernetes-apps/ansible/templates/dns-autoscaler.yml.j2

@@ -0,0 +1,72 @@
+---
+# Copyright 2016 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: dns-autoscaler{{ coredns_ordinal_suffix }}
+  namespace: kube-system
+  labels:
+    k8s-app: dns-autoscaler{{ coredns_ordinal_suffix }}
+    addonmanager.kubernetes.io/mode: Reconcile
+spec:
+  selector:
+    matchLabels:
+      k8s-app: dns-autoscaler{{ coredns_ordinal_suffix }}
+  template:
+    metadata:
+      labels:
+        k8s-app: dns-autoscaler{{ coredns_ordinal_suffix }}
+      annotations:
+    spec:
+      nodeSelector:
+        {{ dns_autoscaler_deployment_nodeselector }}
+      priorityClassName: system-cluster-critical
+      securityContext:
+        seccompProfile:
+          type: RuntimeDefault
+        supplementalGroups: [ 65534 ]
+        fsGroup: 65534
+      nodeSelector:
+        kubernetes.io/os: linux
+      tolerations:
+        - effect: NoSchedule
+          key: node-role.kubernetes.io/control-plane
+{% if dns_autoscaler_extra_tolerations is defined %}
+        {{ dns_autoscaler_extra_tolerations | list | to_nice_yaml(indent=2) | indent(8) }}
+{% endif %}
+      affinity:
+        {{ dns_autoscaler_affinity | to_nice_yaml(indent=2) | indent(8) }}
+      containers:
+      - name: autoscaler
+        image: "{{ dnsautoscaler_image_repo }}:{{ dnsautoscaler_image_tag }}"
+        resources:
+          requests:
+            cpu: {{ dns_autoscaler_cpu_requests }}
+            memory: {{ dns_autoscaler_memory_requests }}
+        readinessProbe:
+          httpGet:
+            path: /healthz
+            port: 8080
+            scheme: HTTP
+        command:
+        - /cluster-proportional-autoscaler
+        - --namespace=kube-system
+        - --default-params={"linear":{"preventSinglePointFailure":{{ dns_prevent_single_point_failure }},"coresPerReplica":{{ dns_cores_per_replica }},"nodesPerReplica":{{ dns_nodes_per_replica }},"min":{{ dns_min_replicas }}}}
+        - --logtostderr=true
+        - --v=2
+        - --configmap=dns-autoscaler{{ coredns_ordinal_suffix }}
+        - --target=Deployment/coredns{{ coredns_ordinal_suffix }}
+      serviceAccountName: dns-autoscaler

internal/resources/kubespray/project/roles/kubernetes-apps/ansible/templates/etcd_metrics-endpoints.yml.j2

@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: Endpoints
+metadata:
+  name: etcd-metrics
+  namespace: kube-system
+  labels:
+    k8s-app: etcd
+    app.kubernetes.io/managed-by: Kubespray
+subsets:
+{% for etcd_metrics_address, etcd_host in etcd_metrics_addresses.split(',') | zip(etcd_hosts) %}
+  - addresses:
+      - ip: {{ etcd_metrics_address | urlsplit('hostname') }}
+        targetRef:
+          kind: Node
+          name: {{ etcd_host }}
+    ports:
+      - name: http-metrics
+        port: {{ etcd_metrics_address | urlsplit('port') }}
+        protocol: TCP
+{% endfor %}

internal/resources/kubespray/project/roles/kubernetes-apps/ansible/templates/etcd_metrics-service.yml.j2

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: etcd-metrics
+  namespace: kube-system
+  labels:
+    {{ etcd_metrics_service_labels | to_yaml(indent=2, width=1337) | indent(width=4) }}
+spec:
+  ports:
+    - name: http-metrics
+      protocol: TCP
+      port: {{ etcd_metrics_port }}
+      # targetPort:

internal/resources/kubespray/project/roles/kubernetes-apps/ansible/templates/netchecker-agent-ds.yml.j2

@@ -0,0 +1,56 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  labels:
+    app: netchecker-agent
+  name: netchecker-agent
+  namespace: {{ netcheck_namespace }}
+spec:
+  selector:
+    matchLabels:
+      app: netchecker-agent
+  template:
+    metadata:
+      name: netchecker-agent
+      labels:
+        app: netchecker-agent
+    spec:
+      priorityClassName: {% if netcheck_namespace == 'kube-system' %}system-node-critical{% else %}k8s-cluster-critical{% endif %}{{ '' }}
+      tolerations:
+        - effect: NoSchedule
+          operator: Exists
+      nodeSelector:
+        kubernetes.io/os: linux
+      containers:
+        - name: netchecker-agent
+          image: "{{ netcheck_agent_image_repo }}:{{ netcheck_agent_image_tag }}"
+          imagePullPolicy: {{ k8s_image_pull_policy }}
+          env:
+            - name: MY_POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: MY_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          args:
+            - "-v={{ netchecker_agent_log_level }}"
+            - "-alsologtostderr=true"
+            - "-serverendpoint=netchecker-service:8081"
+            - "-reportinterval={{ agent_report_interval }}"
+          resources:
+            limits:
+              cpu: {{ netchecker_agent_cpu_limit }}
+              memory: {{ netchecker_agent_memory_limit }}
+            requests:
+              cpu: {{ netchecker_agent_cpu_requests }}
+              memory: {{ netchecker_agent_memory_requests }}
+          securityContext:
+            runAsUser: {{ netchecker_agent_user | default('0') }}
+            runAsGroup: {{ netchecker_agent_group | default('0') }}
+      serviceAccountName: netchecker-agent
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 100%
+    type: RollingUpdate

internal/resources/kubespray/project/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.yml.j2

@@ -0,0 +1,58 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  labels:
+    app: netchecker-agent-hostnet
+  name: netchecker-agent-hostnet
+  namespace: {{ netcheck_namespace }}
+spec:
+  selector:
+    matchLabels:
+      app: netchecker-agent-hostnet
+  template:
+    metadata:
+      name: netchecker-agent-hostnet
+      labels:
+        app: netchecker-agent-hostnet
+    spec:
+      hostNetwork: true
+      dnsPolicy: ClusterFirstWithHostNet
+      nodeSelector:
+        kubernetes.io/os: linux
+      priorityClassName: {% if netcheck_namespace == 'kube-system' %}system-node-critical{% else %}k8s-cluster-critical{% endif %}{{ '' }}
+      tolerations:
+        - effect: NoSchedule
+          operator: Exists
+      containers:
+        - name: netchecker-agent
+          image: "{{ netcheck_agent_image_repo }}:{{ netcheck_agent_image_tag }}"
+          imagePullPolicy: {{ k8s_image_pull_policy }}
+          env:
+            - name: MY_POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: MY_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          args:
+            - "-v={{ netchecker_agent_log_level }}"
+            - "-alsologtostderr=true"
+            - "-serverendpoint=netchecker-service:8081"
+            - "-reportinterval={{ agent_report_interval }}"
+          resources:
+            limits:
+              cpu: {{ netchecker_agent_cpu_limit }}
+              memory: {{ netchecker_agent_memory_limit }}
+            requests:
+              cpu: {{ netchecker_agent_cpu_requests }}
+              memory: {{ netchecker_agent_memory_requests }}
+          securityContext:
+            runAsUser: {{ netchecker_agent_user | default('0') }}
+            runAsGroup: {{ netchecker_agent_group | default('0') }}
+      serviceAccountName: netchecker-agent
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 100%
+    type: RollingUpdate

internal/resources/kubespray/project/roles/kubernetes-apps/ansible/templates/netchecker-agent-sa.yml.j2

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: netchecker-agent
+  namespace: {{ netcheck_namespace }}

internal/resources/kubespray/project/roles/kubernetes-apps/ansible/templates/netchecker-ns.yml.j2

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: "{{ netcheck_namespace }}"
+  labels:
+    name: "{{ netcheck_namespace }}"

internal/resources/kubespray/project/roles/kubernetes-apps/ansible/templates/netchecker-server-clusterrole.yml.j2

@@ -0,0 +1,9 @@
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: netchecker-server
+  namespace: {{ netcheck_namespace }}
+rules:
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["list", "get"]

internal/resources/kubespray/project/roles/kubernetes-apps/ansible/templates/netchecker-server-clusterrolebinding.yml.j2

@@ -0,0 +1,13 @@
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: netchecker-server
+  namespace: {{ netcheck_namespace }}
+subjects:
+  - kind: ServiceAccount
+    name: netchecker-server
+    namespace: {{ netcheck_namespace }}
+roleRef:
+  kind: ClusterRole
+  name: netchecker-server
+  apiGroup: rbac.authorization.k8s.io

internal/resources/kubespray/project/roles/kubernetes-apps/ansible/templates/netchecker-server-deployment.yml.j2

@@ -0,0 +1,86 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: netchecker-server
+  namespace: {{ netcheck_namespace }}
+  labels:
+    app: netchecker-server
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: netchecker-server
+  template:
+    metadata:
+      name: netchecker-server
+      labels:
+        app: netchecker-server
+    spec:
+      priorityClassName: {% if netcheck_namespace == 'kube-system' %}system-cluster-critical{% else %}k8s-cluster-critical{% endif %}{{ '' }}
+      volumes:
+        - name: etcd-data
+          emptyDir: {}
+      containers:
+        - name: netchecker-server
+          image: "{{ netcheck_server_image_repo }}:{{ netcheck_server_image_tag }}"
+          imagePullPolicy: {{ k8s_image_pull_policy }}
+          resources:
+            limits:
+              cpu: {{ netchecker_server_cpu_limit }}
+              memory: {{ netchecker_server_memory_limit }}
+            requests:
+              cpu: {{ netchecker_server_cpu_requests }}
+              memory: {{ netchecker_server_memory_requests }}
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ['ALL']
+            runAsUser: {{ netchecker_server_user | default('0') }}
+            runAsGroup: {{ netchecker_server_group | default('0') }}
+            runAsNonRoot: true
+            seccompProfile:
+              type: RuntimeDefault
+          ports:
+            - containerPort: 8081
+          args:
+            - -v={{ netchecker_server_log_level }}
+            - -logtostderr
+            - -kubeproxyinit=false
+            - -endpoint=0.0.0.0:8081
+            - -etcd-endpoints=http://127.0.0.1:2379
+        - name: etcd
+          image: "{{ etcd_image_repo }}:{{ netcheck_etcd_image_tag }}"
+          imagePullPolicy: {{ k8s_image_pull_policy }}
+          env:
+            - name: ETCD_LOG_LEVEL
+              value: "{{ netchecker_etcd_log_level }}"
+          command:
+            - etcd
+            - --listen-client-urls=http://127.0.0.1:2379
+            - --advertise-client-urls=http://127.0.0.1:2379
+            - --data-dir=/var/lib/etcd
+            - --enable-v2
+            - --force-new-cluster
+          volumeMounts:
+            - mountPath: /var/lib/etcd
+              name: etcd-data
+          resources:
+            limits:
+              cpu: {{ netchecker_etcd_cpu_limit }}
+              memory: {{ netchecker_etcd_memory_limit }}
+            requests:
+              cpu: {{ netchecker_etcd_cpu_requests }}
+              memory: {{ netchecker_etcd_memory_requests }}
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ['ALL']
+            runAsUser: {{ netchecker_server_user | default('0') }}
+            runAsGroup: {{ netchecker_server_group | default('0') }}
+            runAsNonRoot: true
+            seccompProfile:
+              type: RuntimeDefault
+      tolerations:
+        - effect: NoSchedule
+          operator: Exists
+      serviceAccountName: netchecker-server

internal/resources/kubespray/project/roles/kubernetes-apps/ansible/templates/netchecker-server-sa.yml.j2

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: netchecker-server
+  namespace: {{ netcheck_namespace }}

internal/resources/kubespray/project/roles/kubernetes-apps/ansible/templates/netchecker-server-svc.yml.j2

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: netchecker-service
+  namespace: {{ netcheck_namespace }}
+spec:
+  selector:
+    app: netchecker-server
+  ports:
+    -
+      protocol: TCP
+      port: 8081
+      targetPort: 8081
+      nodePort: {{ netchecker_port }}
+  type: NodePort

internal/resources/kubespray/project/roles/kubernetes-apps/ansible/templates/nodelocaldns-config.yml.j2

@@ -0,0 +1,192 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: nodelocaldns
+  namespace: kube-system
+  labels:
+    addonmanager.kubernetes.io/mode: EnsureExists
+
+data:
+  Corefile: |
+{% if nodelocaldns_external_zones is defined and nodelocaldns_external_zones | length > 0 %}
+{% for block in nodelocaldns_external_zones %}
+    {{ block['zones'] | join(' ') }} {
+        errors
+        cache {{ block['cache'] | default(30) }}
+        reload
+{% if block['rewrite'] is defined and block['rewrite'] | length > 0 %}
+{% for rewrite_match in block['rewrite'] %}
+        rewrite {{ rewrite_match }}
+{% endfor %}
+{% endif %}
+        loop
+        bind {{ nodelocaldns_ip }}
+        forward . {{ block['nameservers'] | join(' ') }}
+        prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:{{ nodelocaldns_prometheus_port }}
+        log
+{% if dns_etchosts | default(None) %}
+        hosts /etc/coredns/hosts {
+          fallthrough
+        }
+{% endif %}
+    }
+{% endfor %}
+{% endif %}
+    {{ ([dns_domain] + old_dns_domains) | join(' ') }}:53 {
+        errors
+        cache {
+            success 9984 30
+            denial 9984 5
+        }
+        reload
+        loop
+        bind {{ nodelocaldns_ip }}
+        forward . {{ forwardTarget }} {
+            force_tcp
+        }
+        prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:{{ nodelocaldns_prometheus_port }}
+        health {{ nodelocaldns_ip }}:{{ nodelocaldns_health_port }}
+{% if dns_etchosts | default(None) %}
+        hosts /etc/coredns/hosts {
+          fallthrough
+        }
+{% endif %}
+    }
+    in-addr.arpa:53 {
+        errors
+        cache 30
+        reload
+        loop
+        bind {{ nodelocaldns_ip }}
+        forward . {{ forwardTarget }} {
+            force_tcp
+        }
+        prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:{{ nodelocaldns_prometheus_port }}
+    }
+    ip6.arpa:53 {
+        errors
+        cache 30
+        reload
+        loop
+        bind {{ nodelocaldns_ip }}
+        forward . {{ forwardTarget }} {
+            force_tcp
+        }
+        prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:{{ nodelocaldns_prometheus_port }}
+    }
+    .:53 {
+{% if nodelocaldns_additional_configs is defined %}
+        {{ nodelocaldns_additional_configs | indent(width=8, first=False) }}
+{% endif %}
+        errors
+        cache 30
+        reload
+        loop
+        bind {{ nodelocaldns_ip }}
+        forward . {{ upstreamForwardTarget }}{% if dns_upstream_forward_extra_opts is defined %} {
+{% for optname, optvalue in dns_upstream_forward_extra_opts.items() %}
+          {{ (optname ~ ' ' ~ optvalue) | trim }}
+          {# do not add a trailing space when optvalue == ''
+             workaround for: https://github.com/kubernetes/kubernetes/issues/36222 #}
+{% endfor %}
+        }{% endif %}
+
+        prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:{{ nodelocaldns_prometheus_port }}
+{% if dns_etchosts | default(None) %}
+        hosts /etc/coredns/hosts {
+          fallthrough
+        }
+{% endif %}
+    }
+{% if enable_nodelocaldns_secondary %}
+  Corefile-second: |
+{% if nodelocaldns_external_zones is defined and nodelocaldns_external_zones | length > 0 %}
+{% for block in nodelocaldns_external_zones %}
+    {{ block['zones'] | join(' ') }} {
+        errors
+        cache {{ block['cache'] | default(30) }}
+        reload
+        loop
+        bind {{ nodelocaldns_ip }}
+        forward . {{ block['nameservers'] | join(' ') }}
+        prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:{{ nodelocaldns_secondary_prometheus_port }}
+        log
+{% if dns_etchosts | default(None) %}
+        hosts /etc/coredns/hosts {
+          fallthrough
+        }
+{% endif %}
+    }
+{% endfor %}
+{% endif %}
+    {{ dns_domain }}:53 {
+        errors
+        cache {
+            success 9984 30
+            denial 9984 5
+        }
+        reload
+        loop
+        bind {{ nodelocaldns_ip }}
+        forward . {{ forwardTarget }} {
+            force_tcp
+        }
+        prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:{{ nodelocaldns_secondary_prometheus_port }}
+        health {{ nodelocaldns_ip }}:{{ nodelocaldns_second_health_port }}
+{% if dns_etchosts | default(None) %}
+        hosts /etc/coredns/hosts {
+          fallthrough
+        }
+{% endif %}
+    }
+    in-addr.arpa:53 {
+        errors
+        cache 30
+        reload
+        loop
+        bind {{ nodelocaldns_ip }}
+        forward . {{ forwardTarget }} {
+            force_tcp
+        }
+        prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:{{ nodelocaldns_secondary_prometheus_port }}
+    }
+    ip6.arpa:53 {
+        errors
+        cache 30
+        reload
+        loop
+        bind {{ nodelocaldns_ip }}
+        forward . {{ forwardTarget }} {
+            force_tcp
+        }
+        prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:{{ nodelocaldns_secondary_prometheus_port }}
+    }
+    .:53 {
+{% if nodelocaldns_additional_configs is defined %}
+        {{ nodelocaldns_additional_configs | indent(width=8, first=False) }}
+{% endif %}
+        errors
+        cache 30
+        reload
+        loop
+        bind {{ nodelocaldns_ip }}
+        forward . {{ upstreamForwardTarget }}{% if dns_upstream_forward_extra_opts is defined %} {
+{% for optname, optvalue in dns_upstream_forward_extra_opts.items() %}
+          {{ (optname ~ ' ' ~ optvalue) | trim }}
+          {# do not add a trailing space when optvalue == ''
+             workaround for: https://github.com/kubernetes/kubernetes/issues/36222 #}
+{% endfor %}
+        }{% endif %}
+
+        prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:{{ nodelocaldns_secondary_prometheus_port }}
+{% if dns_etchosts | default(None) %}
+        hosts /etc/coredns/hosts {
+          fallthrough
+        }
+{% endif %}
+    }
+{% endif %}
+{% if dns_etchosts | default(None) %}
+  hosts: |
+    {{ dns_etchosts | indent(width=4, first=False) }}
+{% endif %}

internal/resources/kubespray/project/roles/kubernetes-apps/ansible/templates/nodelocaldns-daemonset.yml.j2

@@ -0,0 +1,114 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: nodelocaldns
+  namespace: kube-system
+  labels:
+    k8s-app: kube-dns
+    addonmanager.kubernetes.io/mode: Reconcile
+spec:
+  selector:
+    matchLabels:
+      k8s-app: node-local-dns
+  template:
+    metadata:
+      labels:
+        k8s-app: node-local-dns
+      annotations:
+        prometheus.io/scrape: 'true'
+        prometheus.io/port: '{{ nodelocaldns_prometheus_port }}'
+    spec:
+      nodeSelector:
+        {{ nodelocaldns_ds_nodeselector }}
+      priorityClassName: system-node-critical
+      serviceAccountName: nodelocaldns
+      hostNetwork: true
+      dnsPolicy: Default  # Don't use cluster DNS.
+      tolerations:
+      - effect: NoSchedule
+        operator: "Exists"
+      - effect: NoExecute
+        operator: "Exists"
+      containers:
+      - name: node-cache
+        image: "{{ nodelocaldns_image_repo }}:{{ nodelocaldns_image_tag }}"
+        resources:
+          limits:
+            memory: {{ nodelocaldns_memory_limit }}
+          requests:
+            cpu: {{ nodelocaldns_cpu_requests }}
+            memory: {{ nodelocaldns_memory_requests }}
+        args:
+        - -localip
+        - {{ nodelocaldns_ip }}
+        - -conf
+        - /etc/coredns/Corefile
+        - -upstreamsvc
+        - coredns
+{% if enable_nodelocaldns_secondary %}
+        - -skipteardown
+{% endif %}
+        ports:
+        - containerPort: 53
+          name: dns
+          protocol: UDP
+        - containerPort: 53
+          name: dns-tcp
+          protocol: TCP
+        - containerPort: {{ nodelocaldns_prometheus_port }}
+          name: metrics
+          protocol: TCP
+        securityContext:
+          privileged: true
+{% if nodelocaldns_bind_metrics_host_ip %}
+        env:
+          - name: MY_HOST_IP
+            valueFrom:
+              fieldRef:
+                fieldPath: status.hostIP
+{% endif %}
+        livenessProbe:
+          httpGet:
+            host: {{ nodelocaldns_ip }}
+            path: /health
+            port: {{ nodelocaldns_health_port }}
+            scheme: HTTP
+          timeoutSeconds: 5
+          successThreshold: 1
+          failureThreshold: 10
+        readinessProbe:
+          httpGet:
+            host: {{ nodelocaldns_ip }}
+            path: /health
+            port: {{ nodelocaldns_health_port }}
+            scheme: HTTP
+          timeoutSeconds: 5
+          successThreshold: 1
+          failureThreshold: 10
+        volumeMounts:
+        - name: config-volume
+          mountPath: /etc/coredns
+        - name: xtables-lock
+          mountPath: /run/xtables.lock
+      volumes:
+        - name: config-volume
+          configMap:
+            name: nodelocaldns
+            items:
+            - key: Corefile
+              path: Corefile
+{% if dns_etchosts | default(None) %}
+            - key: hosts
+              path: hosts
+{% endif %}
+        - name: xtables-lock
+          hostPath:
+            path: /run/xtables.lock
+            type: FileOrCreate
+      # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force
+      # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods.
+      terminationGracePeriodSeconds: 0
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: {{ serial | default('20%') }}
+    type: RollingUpdate

internal/resources/kubespray/project/roles/kubernetes-apps/ansible/templates/nodelocaldns-sa.yml.j2

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: nodelocaldns
+  namespace: kube-system
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile

internal/resources/kubespray/project/roles/kubernetes-apps/ansible/templates/nodelocaldns-second-daemonset.yml.j2

@@ -0,0 +1,107 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: nodelocaldns-second
+  namespace: kube-system
+  labels:
+    k8s-app: kube-dns
+    addonmanager.kubernetes.io/mode: Reconcile
+spec:
+  selector:
+    matchLabels:
+      k8s-app: node-local-dns-second
+  template:
+    metadata:
+      labels:
+        k8s-app: node-local-dns-second
+      annotations:
+        prometheus.io/scrape: 'true'
+        prometheus.io/port: '{{ nodelocaldns_secondary_prometheus_port }}'
+    spec:
+      nodeSelector:
+        {{ nodelocaldns_ds_nodeselector }}
+      priorityClassName: system-cluster-critical
+      serviceAccountName: nodelocaldns
+      hostNetwork: true
+      dnsPolicy: Default  # Don't use cluster DNS.
+      tolerations:
+      - effect: NoSchedule
+        operator: "Exists"
+      - effect: NoExecute
+        operator: "Exists"
+      containers:
+      - name: node-cache
+        image: "{{ nodelocaldns_image_repo }}:{{ nodelocaldns_image_tag }}"
+        resources:
+          limits:
+            memory: {{ nodelocaldns_memory_limit }}
+          requests:
+            cpu: {{ nodelocaldns_cpu_requests }}
+            memory: {{ nodelocaldns_memory_requests }}
+        args: [ "-localip", "{{ nodelocaldns_ip }}", "-conf", "/etc/coredns/Corefile", "-upstreamsvc", "coredns", "-skipteardown" ]
+        ports:
+        - containerPort: {{ nodelocaldns_secondary_prometheus_port }}
+          name: metrics
+          protocol: TCP
+        securityContext:
+          privileged: true
+{% if nodelocaldns_bind_metrics_host_ip %}
+        env:
+          - name: MY_HOST_IP
+            valueFrom:
+              fieldRef:
+                fieldPath: status.hostIP
+{% endif %}
+        livenessProbe:
+          httpGet:
+            host: {{ nodelocaldns_ip }}
+            path: /health
+            port: {{ nodelocaldns_health_port }}
+            scheme: HTTP
+          timeoutSeconds: 5
+          successThreshold: 1
+          failureThreshold: 10
+        readinessProbe:
+          httpGet:
+            host: {{ nodelocaldns_ip }}
+            path: /health
+            port: {{ nodelocaldns_health_port }}
+            scheme: HTTP
+          timeoutSeconds: 5
+          successThreshold: 1
+          failureThreshold: 10
+        volumeMounts:
+        - name: config-volume
+          mountPath: /etc/coredns
+        - name: xtables-lock
+          mountPath: /run/xtables.lock
+        lifecycle:
+          preStop:
+            exec:
+              command:
+                - sh
+                - -c
+                - sleep {{ nodelocaldns_secondary_skew_seconds }} && kill -9 1
+      volumes:
+        - name: config-volume
+          configMap:
+            name: nodelocaldns
+            items:
+            - key: Corefile-second
+              path: Corefile
+{% if dns_etchosts | default(None) %}
+            - key: hosts
+              path: hosts
+{% endif %}
+        - name: xtables-lock
+          hostPath:
+            path: /run/xtables.lock
+            type: FileOrCreate
+      # Implement a time skew between the main nodelocaldns and this secondary.
+      # Since the two nodelocaldns instances share the :53 port, we want to keep
+      # at least one running at any time enven if the manifests are replaced simultaneously
+      terminationGracePeriodSeconds: {{ nodelocaldns_secondary_skew_seconds }}
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: {{ serial | default('20%') }}
+    type: RollingUpdate

internal/resources/kubespray/project/roles/kubernetes-apps/ansible/vars/main.yml

@@ -0,0 +1,22 @@
+---
+dns_autoscaler_manifests:
+- dns-autoscaler-sa.yml.j2
+- dns-autoscaler.yml.j2
+- dns-autoscaler-clusterrole.yml.j2
+- dns-autoscaler-clusterrolebinding.yml.j2
+
+coredns_manifests:
+- coredns-clusterrole.yml.j2
+- coredns-clusterrolebinding.yml.j2
+- coredns-config.yml.j2
+- coredns-deployment.yml.j2
+- coredns-sa.yml.j2
+- coredns-svc.yml.j2
+- "{{ dns_autoscaler_manifests if enable_dns_autoscaler else [] }}"
+- "{{ 'coredns-poddisruptionbudget.yml.j2' if coredns_pod_disruption_budget else [] }}"
+
+nodelocaldns_manifests:
+- nodelocaldns-config.yml.j2
+- nodelocaldns-daemonset.yml.j2
+- nodelocaldns-sa.yml.j2
+- "{{ 'nodelocaldns-second-daemonset.yml.j2' if enable_nodelocaldns_secondary else [] }}"

internal/resources/kubespray/project/roles/kubernetes-apps/argocd/defaults/main.yml

@@ -0,0 +1,6 @@
+---
+argocd_enabled: false
+argocd_version: 2.14.5
+argocd_namespace: argocd
+# argocd_admin_password:
+argocd_install_url: "https://raw.githubusercontent.com/argoproj/argo-cd/v{{ argocd_version }}/manifests/install.yaml"

internal/resources/kubespray/project/roles/kubernetes-apps/argocd/tasks/main.yml

@@ -0,0 +1,107 @@
+---
+- name: Kubernetes Apps | Download yq
+  include_tasks: "../../../download/tasks/download_file.yml"
+  vars:
+    download: "{{ download_defaults | combine(downloads.yq) }}"
+
+- name: Kubernetes Apps | Copy yq binary from download dir
+  ansible.posix.synchronize:
+    src: "{{ downloads.yq.dest }}"
+    dest: "{{ bin_dir }}/yq"
+    compress: false
+    perms: true
+    owner: false
+    group: false
+  delegate_to: "{{ inventory_hostname }}"
+
+- name: Kubernetes Apps | Set ArgoCD template list
+  set_fact:
+    argocd_templates:
+      - name: namespace
+        file: argocd-namespace.yml
+      - name: install
+        file: argocd-install.yml
+        namespace: "{{ argocd_namespace }}"
+        url: "{{ argocd_install_url }}"
+  when:
+    - "inventory_hostname == groups['kube_control_plane'][0]"
+
+- name: Kubernetes Apps | Download ArgoCD remote manifests
+  include_tasks: "../../../download/tasks/download_file.yml"
+  vars:
+    download_argocd:
+      enabled: "{{ argocd_enabled }}"
+      file: true
+      dest: "{{ local_release_dir }}/{{ item.file }}"
+      url: "{{ item.url }}"
+      unarchive: false
+      owner: "root"
+      mode: "0644"
+      sha256: ""
+    download: "{{ download_defaults | combine(download_argocd) }}"
+  with_items: "{{ argocd_templates | selectattr('url', 'defined') | list }}"
+  loop_control:
+    label: "{{ item.file }}"
+  when:
+    - "inventory_hostname == groups['kube_control_plane'][0]"
+
+- name: Kubernetes Apps | Copy ArgoCD remote manifests from download dir
+  ansible.posix.synchronize:
+    src: "{{ local_release_dir }}/{{ item.file }}"
+    dest: "{{ kube_config_dir }}/{{ item.file }}"
+    compress: false
+    perms: true
+    owner: false
+    group: false
+  delegate_to: "{{ inventory_hostname }}"
+  with_items: "{{ argocd_templates | selectattr('url', 'defined') | list }}"
+  when:
+    - "inventory_hostname == groups['kube_control_plane'][0]"
+
+- name: Kubernetes Apps | Set ArgoCD namespace for remote manifests
+  become: true
+  command: |
+    {{ bin_dir }}/yq eval-all -i '.metadata.namespace="{{ argocd_namespace }}"' {{ kube_config_dir }}/{{ item.file }}
+  with_items: "{{ argocd_templates | selectattr('url', 'defined') | list }}"
+  loop_control:
+    label: "{{ item.file }}"
+  when:
+    - "inventory_hostname == groups['kube_control_plane'][0]"
+
+- name: Kubernetes Apps | Create ArgoCD manifests from templates
+  become: true
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.file }}"
+    mode: "0644"
+  with_items: "{{ argocd_templates | selectattr('url', 'undefined') | list }}"
+  loop_control:
+    label: "{{ item.file }}"
+  when:
+    - "inventory_hostname == groups['kube_control_plane'][0]"
+
+- name: Kubernetes Apps | Install ArgoCD
+  become: true
+  kube:
+    name: ArgoCD
+    kubectl: "{{ bin_dir }}/kubectl"
+    filename: "{{ kube_config_dir }}/{{ item.file }}"
+    state: latest
+  with_items: "{{ argocd_templates }}"
+  when:
+    - "inventory_hostname == groups['kube_control_plane'][0]"
+
+# https://github.com/argoproj/argo-cd/blob/master/docs/faq.md#i-forgot-the-admin-password-how-do-i-reset-it
+- name: Kubernetes Apps | Set ArgoCD custom admin password
+  become: true
+  shell: |
+    {{ bin_dir }}/kubectl --kubeconfig /etc/kubernetes/admin.conf -n {{ argocd_namespace }} patch secret argocd-secret -p \
+      '{
+        "stringData": {
+          "admin.password": "{{ argocd_admin_password | password_hash('bcrypt') }}",
+          "admin.passwordMtime": "'$(date +%FT%T%Z)'"
+        }
+      }'
+  when:
+    - argocd_admin_password is defined
+    - "inventory_hostname == groups['kube_control_plane'][0]"

internal/resources/kubespray/project/roles/kubernetes-apps/argocd/templates/argocd-namespace.yml.j2

@@ -0,0 +1,7 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: {{ argocd_namespace }}
+  labels:
+    app: argocd

internal/resources/kubespray/project/roles/kubernetes-apps/cluster_roles/files/k8s-cluster-critical-pc.yml

@@ -0,0 +1,8 @@
+---
+apiVersion: scheduling.k8s.io/v1
+kind: PriorityClass
+metadata:
+  name: k8s-cluster-critical
+value: 1000000000
+globalDefault: false
+description: "This priority class should only be used by the pods installed using kubespray."

internal/resources/kubespray/project/roles/kubernetes-apps/cluster_roles/tasks/main.yml

@@ -0,0 +1,80 @@
+---
+- name: Kubernetes Apps | Wait for kube-apiserver
+  uri:
+    url: "{{ kube_apiserver_endpoint }}/healthz"
+    validate_certs: false
+    client_cert: "{{ kube_apiserver_client_cert }}"
+    client_key: "{{ kube_apiserver_client_key }}"
+  register: result
+  until: result.status == 200
+  retries: 10
+  delay: 6
+  when: inventory_hostname == groups['kube_control_plane'][0]
+
+- name: Kubernetes Apps | Add ClusterRoleBinding to admit nodes
+  template:
+    src: "node-crb.yml.j2"
+    dest: "{{ kube_config_dir }}/node-crb.yml"
+    mode: "0640"
+  register: node_crb_manifest
+  when:
+    - rbac_enabled
+    - inventory_hostname == groups['kube_control_plane'][0]
+
+- name: Apply workaround to allow all nodes with cert O=system:nodes to register
+  kube:
+    name: "kubespray:system:node"
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: "clusterrolebinding"
+    filename: "{{ kube_config_dir }}/node-crb.yml"
+    state: latest
+  register: result
+  until: result is succeeded
+  retries: 10
+  delay: 6
+  when:
+    - rbac_enabled
+    - node_crb_manifest.changed
+    - inventory_hostname == groups['kube_control_plane'][0]
+
+- name: Kubernetes Apps | Remove old webhook ClusterRole
+  kube:
+    name: "system:node-webhook"
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: "clusterrole"
+    state: absent
+  when:
+    - rbac_enabled
+    - inventory_hostname == groups['kube_control_plane'][0]
+  tags: node-webhook
+
+- name: Kubernetes Apps | Remove old webhook ClusterRoleBinding
+  kube:
+    name: "system:node-webhook"
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: "clusterrolebinding"
+    state: absent
+  when:
+    - rbac_enabled
+    - inventory_hostname == groups['kube_control_plane'][0]
+  tags: node-webhook
+
+- name: PriorityClass | Copy k8s-cluster-critical-pc.yml file
+  copy:
+    src: k8s-cluster-critical-pc.yml
+    dest: "{{ kube_config_dir }}/k8s-cluster-critical-pc.yml"
+    mode: "0640"
+  when: inventory_hostname == groups['kube_control_plane'] | last
+
+- name: PriorityClass | Create k8s-cluster-critical
+  kube:
+    name: k8s-cluster-critical
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: "PriorityClass"
+    filename: "{{ kube_config_dir }}/k8s-cluster-critical-pc.yml"
+    state: latest
+  register: result
+  until: result is succeeded
+  retries: 10
+  delay: 6
+  when: inventory_hostname == groups['kube_control_plane'] | last

internal/resources/kubespray/project/roles/kubernetes-apps/cluster_roles/templates/namespace.j2

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: "kube-system"

internal/resources/kubespray/project/roles/kubernetes-apps/cluster_roles/templates/node-crb.yml.j2

@@ -0,0 +1,17 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  annotations:
+    rbac.authorization.kubernetes.io/autoupdate: "true"
+  labels:
+    kubernetes.io/bootstrapping: rbac-defaults
+  name: kubespray:system:node
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:node
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: Group
+  name: system:nodes

internal/resources/kubespray/project/roles/kubernetes-apps/cluster_roles/templates/vsphere-rbac.yml.j2

@@ -0,0 +1,35 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: system:vsphere-cloud-provider
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+  - update
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: system:vsphere-cloud-provider
+roleRef:
+  kind: ClusterRole
+  name: system:vsphere-cloud-provider
+  apiGroup: rbac.authorization.k8s.io
+subjects:
+- kind: ServiceAccount
+  name: vsphere-cloud-provider
+  namespace: kube-system

internal/resources/kubespray/project/roles/kubernetes-apps/container_engine_accelerator/meta/main.yml

@@ -0,0 +1,8 @@
+---
+dependencies:
+  - role: kubernetes-apps/container_engine_accelerator/nvidia_gpu
+    when: nvidia_accelerator_enabled
+    tags:
+      - apps
+      - nvidia_gpu
+      - container_engine_accelerator

internal/resources/kubespray/project/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/defaults/main.yml

@@ -0,0 +1,14 @@
+---
+nvidia_accelerator_enabled: false
+nvidia_driver_version: "390.87"
+nvidia_gpu_tesla_base_url: https://us.download.nvidia.com/tesla/
+nvidia_gpu_gtx_base_url: http://us.download.nvidia.com/XFree86/Linux-x86_64/
+nvidia_gpu_flavor: tesla
+nvidia_url_end: "{{ nvidia_driver_version }}/NVIDIA-Linux-x86_64-{{ nvidia_driver_version }}.run"
+nvidia_driver_install_container: false
+nvidia_driver_install_centos_container: atzedevries/nvidia-centos-driver-installer:2
+nvidia_driver_install_ubuntu_container: registry.k8s.io/ubuntu-nvidia-driver-installer@sha256:7df76a0f0a17294e86f691c81de6bbb7c04a1b4b3d4ea4e7e2cccdc42e1f6d63
+nvidia_driver_install_supported: false
+nvidia_gpu_device_plugin_container: "registry.k8s.io/nvidia-gpu-device-plugin@sha256:0842734032018be107fa2490c98156992911e3e1f2a21e059ff0105b07dd8e9e"
+nvidia_gpu_nodes: []
+nvidia_gpu_device_plugin_memory: 30Mi

internal/resources/kubespray/project/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/tasks/main.yml

@@ -0,0 +1,55 @@
+---
+
+- name: Container Engine Acceleration Nvidia GPU | gather os specific variables
+  include_vars: "{{ item }}"
+  with_first_found:
+    - files:
+        - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower | replace('/', '_') }}.yml"
+        - "{{ ansible_distribution | lower }}-{{ ansible_distribution_release }}.yml"
+        - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower | replace('/', '_') }}.yml"
+        - "{{ ansible_distribution | lower }}.yml"
+        - "{{ ansible_os_family | lower }}.yml"
+      skip: true
+
+- name: Container Engine Acceleration Nvidia GPU | Set fact of download url Tesla
+  set_fact:
+    nvidia_driver_download_url_default: "{{ nvidia_gpu_tesla_base_url }}{{ nvidia_url_end }}"
+  when: nvidia_gpu_flavor | lower == "tesla"
+
+- name: Container Engine Acceleration Nvidia GPU | Set fact of download url GTX
+  set_fact:
+    nvidia_driver_download_url_default: "{{ nvidia_gpu_gtx_base_url }}{{ nvidia_url_end }}"
+  when: nvidia_gpu_flavor | lower == "gtx"
+
+- name: Container Engine Acceleration Nvidia GPU | Create addon dir
+  file:
+    path: "{{ kube_config_dir }}/addons/container_engine_accelerator"
+    owner: root
+    group: root
+    mode: "0755"
+    recurse: true
+
+- name: Container Engine Acceleration Nvidia GPU | Create manifests for nvidia accelerators
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/addons/container_engine_accelerator/{{ item.file }}"
+    mode: "0644"
+  with_items:
+    - { name: nvidia-driver-install-daemonset, file: nvidia-driver-install-daemonset.yml, type: daemonset }
+    - { name: k8s-device-plugin-nvidia-daemonset, file: k8s-device-plugin-nvidia-daemonset.yml, type: daemonset }
+  register: container_engine_accelerator_manifests
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0] and nvidia_driver_install_container
+
+- name: Container Engine Acceleration Nvidia GPU | Apply manifests for nvidia accelerators
+  kube:
+    name: "{{ item.item.name }}"
+    namespace: "kube-system"
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: "{{ item.item.type }}"
+    filename: "{{ kube_config_dir }}/addons/container_engine_accelerator/{{ item.item.file }}"
+    state: "latest"
+  with_items:
+    - "{{ container_engine_accelerator_manifests.results }}"
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0] and nvidia_driver_install_container and nvidia_driver_install_supported

internal/resources/kubespray/project/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/templates/k8s-device-plugin-nvidia-daemonset.yml.j2

@@ -0,0 +1,60 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: nvidia-gpu-device-plugin
+  namespace: kube-system
+  labels:
+    k8s-app: nvidia-gpu-device-plugin
+    addonmanager.kubernetes.io/mode: Reconcile
+spec:
+  selector:
+    matchLabels:
+      k8s-app: nvidia-gpu-device-plugin
+  template:
+    metadata:
+      labels:
+        k8s-app: nvidia-gpu-device-plugin
+    spec:
+      priorityClassName: system-node-critical
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: "nvidia.com/gpu"
+                operator: Exists
+      tolerations:
+      - operator: "Exists"
+        effect: "NoExecute"
+      - operator: "Exists"
+        effect: "NoSchedule"
+      hostNetwork: true
+      dnsPolicy: ClusterFirstWithHostNet
+      hostPID: true
+      volumes:
+      - name: device-plugin
+        hostPath:
+          path: /var/lib/kubelet/device-plugins
+      - name: dev
+        hostPath:
+          path: /dev
+      containers:
+      - image: "{{ nvidia_gpu_device_plugin_container }}"
+        command: ["/usr/bin/nvidia-gpu-device-plugin", "-logtostderr"]
+        name: nvidia-gpu-device-plugin
+        resources:
+          requests:
+            cpu: 50m
+            memory: {{ nvidia_gpu_device_plugin_memory }}
+          limits:
+            cpu: 50m
+            memory: {{ nvidia_gpu_device_plugin_memory }}
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: device-plugin
+          mountPath: /device-plugin
+        - name: dev
+          mountPath: /dev
+  updateStrategy:
+    type: RollingUpdate

internal/resources/kubespray/project/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/templates/nvidia-driver-install-daemonset.yml.j2

@@ -0,0 +1,82 @@
+# Copyright 2017 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: nvidia-driver-installer
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      name: nvidia-driver-installer
+  template:
+    metadata:
+      labels:
+        name: nvidia-driver-installer
+    spec:
+      priorityClassName: system-node-critical
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: "nvidia.com/gpu"
+                operator: Exists
+      tolerations:
+      - key: "nvidia.com/gpu"
+        effect: "NoSchedule"
+        operator: "Exists"
+      hostNetwork: true
+      dnsPolicy: ClusterFirstWithHostNet
+      hostPID: true
+      volumes:
+      - name: dev
+        hostPath:
+          path: /dev
+      - name: nvidia-install-dir-host
+        hostPath:
+          path: /home/kubernetes/bin/nvidia
+      - name: root-mount
+        hostPath:
+          path: /
+      initContainers:
+      - image: "{{ nvidia_driver_install_container }}"
+        name: nvidia-driver-installer
+        resources:
+          requests:
+            cpu: 0.15
+        securityContext:
+          privileged: true
+        env:
+          - name: NVIDIA_INSTALL_DIR_HOST
+            value: /home/kubernetes/bin/nvidia
+          - name: NVIDIA_INSTALL_DIR_CONTAINER
+            value: /usr/local/nvidia
+          - name: ROOT_MOUNT_DIR
+            value: /root
+          - name: NVIDIA_DRIVER_VERSION
+            value: "{{ nvidia_driver_version }}"
+          - name: NVIDIA_DRIVER_DOWNLOAD_URL
+            value: "{{ nvidia_driver_download_url_default }}"
+        volumeMounts:
+        - name: nvidia-install-dir-host
+          mountPath: /usr/local/nvidia
+        - name: dev
+          mountPath: /dev
+        - name: root-mount
+          mountPath: /root
+      containers:
+      - image: "{{ pod_infra_image_repo }}:{{ pod_infra_image_tag }}"
+        name: pause

internal/resources/kubespray/project/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/vars/ubuntu-16.yml

@@ -0,0 +1,3 @@
+---
+nvidia_driver_install_container: "{{ nvidia_driver_install_ubuntu_container }}"
+nvidia_driver_install_supported: true

internal/resources/kubespray/project/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/vars/ubuntu-18.yml

@@ -0,0 +1,3 @@
+---
+nvidia_driver_install_container: "{{ nvidia_driver_install_ubuntu_container }}"
+nvidia_driver_install_supported: true

internal/resources/kubespray/project/roles/kubernetes-apps/container_runtimes/crun/tasks/main.yaml

@@ -0,0 +1,19 @@
+---
+
+- name: Crun | Copy runtime class manifest
+  template:
+    src: runtimeclass-crun.yml
+    dest: "{{ kube_config_dir }}/runtimeclass-crun.yml"
+    mode: "0664"
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+
+- name: Crun | Apply manifests
+  kube:
+    name: "runtimeclass-crun"
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: "runtimeclass"
+    filename: "{{ kube_config_dir }}/runtimeclass-crun.yml"
+    state: "latest"
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]

internal/resources/kubespray/project/roles/kubernetes-apps/container_runtimes/crun/templates/runtimeclass-crun.yml

@@ -0,0 +1,6 @@
+---
+kind: RuntimeClass
+apiVersion: node.k8s.io/v1
+metadata:
+  name: crun
+handler: crun

internal/resources/kubespray/project/roles/kubernetes-apps/container_runtimes/gvisor/tasks/main.yaml

@@ -0,0 +1,34 @@
+---
+- name: GVisor | Create addon dir
+  file:
+    path: "{{ kube_config_dir }}/addons/gvisor"
+    owner: root
+    group: root
+    mode: "0755"
+    recurse: true
+
+- name: GVisor | Templates List
+  set_fact:
+    gvisor_templates:
+      - { name: runtimeclass-gvisor, file: runtimeclass-gvisor.yml, type: runtimeclass }
+
+- name: GVisort | Create manifests
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/addons/gvisor/{{ item.file }}"
+    mode: "0644"
+  with_items: "{{ gvisor_templates }}"
+  register: gvisor_manifests
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+
+- name: GVisor | Apply manifests
+  kube:
+    name: "{{ item.item.name }}"
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: "{{ item.item.type }}"
+    filename: "{{ kube_config_dir }}/addons/gvisor/{{ item.item.file }}"
+    state: "latest"
+  with_items: "{{ gvisor_manifests.results }}"
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]

internal/resources/kubespray/project/roles/kubernetes-apps/container_runtimes/gvisor/templates/runtimeclass-gvisor.yml.j2

@@ -0,0 +1,6 @@
+---
+kind: RuntimeClass
+apiVersion: node.k8s.io/v1
+metadata:
+  name: gvisor
+handler: runsc

internal/resources/kubespray/project/roles/kubernetes-apps/container_runtimes/kata_containers/defaults/main.yaml

@@ -0,0 +1,5 @@
+---
+
+kata_containers_qemu_overhead: true
+kata_containers_qemu_overhead_fixed_cpu: 250m
+kata_containers_qemu_overhead_fixed_memory: 160Mi

internal/resources/kubespray/project/roles/kubernetes-apps/container_runtimes/kata_containers/tasks/main.yaml

@@ -0,0 +1,35 @@
+---
+
+- name: Kata Containers | Create addon dir
+  file:
+    path: "{{ kube_config_dir }}/addons/kata_containers"
+    owner: root
+    group: root
+    mode: "0755"
+    recurse: true
+
+- name: Kata Containers | Templates list
+  set_fact:
+    kata_containers_templates:
+      - { name: runtimeclass-kata-qemu, file: runtimeclass-kata-qemu.yml, type: runtimeclass }
+
+- name: Kata Containers | Create manifests
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/addons/kata_containers/{{ item.file }}"
+    mode: "0644"
+  with_items: "{{ kata_containers_templates }}"
+  register: kata_containers_manifests
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+
+- name: Kata Containers | Apply manifests
+  kube:
+    name: "{{ item.item.name }}"
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: "{{ item.item.type }}"
+    filename: "{{ kube_config_dir }}/addons/kata_containers/{{ item.item.file }}"
+    state: "latest"
+  with_items: "{{ kata_containers_manifests.results }}"
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]

internal/resources/kubespray/project/roles/kubernetes-apps/container_runtimes/kata_containers/templates/runtimeclass-kata-qemu.yml.j2

@@ -0,0 +1,12 @@
+---
+kind: RuntimeClass
+apiVersion: node.k8s.io/v1
+metadata:
+  name: kata-qemu
+handler: kata-qemu
+{% if kata_containers_qemu_overhead %}
+overhead:
+  podFixed:
+    cpu: {{ kata_containers_qemu_overhead_fixed_cpu }}
+    memory: {{ kata_containers_qemu_overhead_fixed_memory }}
+{% endif %}

internal/resources/kubespray/project/roles/kubernetes-apps/container_runtimes/meta/main.yml

@@ -0,0 +1,31 @@
+---
+dependencies:
+  - role: kubernetes-apps/container_runtimes/kata_containers
+    when: kata_containers_enabled
+    tags:
+      - apps
+      - kata-containers
+      - container-runtimes
+
+  - role: kubernetes-apps/container_runtimes/gvisor
+    when: gvisor_enabled
+    tags:
+      - apps
+      - gvisor
+      - container-runtimes
+
+  - role: kubernetes-apps/container_runtimes/crun
+    when: crun_enabled
+    tags:
+      - apps
+      - crun
+      - container-runtimes
+
+  - role: kubernetes-apps/container_runtimes/youki
+    when:
+      - youki_enabled
+      - container_manager == 'crio'
+    tags:
+      - apps
+      - youki
+      - container-runtimes

internal/resources/kubespray/project/roles/kubernetes-apps/container_runtimes/youki/tasks/main.yaml

@@ -0,0 +1,19 @@
+---
+
+- name: Youki | Copy runtime class manifest
+  template:
+    src: runtimeclass-youki.yml
+    dest: "{{ kube_config_dir }}/runtimeclass-youki.yml"
+    mode: "0664"
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+
+- name: Youki | Apply manifests
+  kube:
+    name: "runtimeclass-youki"
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: "runtimeclass"
+    filename: "{{ kube_config_dir }}/runtimeclass-youki.yml"
+    state: "latest"
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]

internal/resources/kubespray/project/roles/kubernetes-apps/container_runtimes/youki/templates/runtimeclass-youki.yml

@@ -0,0 +1,6 @@
+---
+kind: RuntimeClass
+apiVersion: node.k8s.io/v1
+metadata:
+  name: youki
+handler: youki

internal/resources/kubespray/project/roles/kubernetes-apps/csi_driver/aws_ebs/defaults/main.yml

@@ -0,0 +1,11 @@
+---
+aws_ebs_csi_enable_volume_scheduling: true
+aws_ebs_csi_enable_volume_snapshot: false
+aws_ebs_csi_enable_volume_resizing: false
+aws_ebs_csi_controller_replicas: 1
+aws_ebs_csi_plugin_image_tag: latest
+
+# Add annotions to ebs_csi_controller. Useful if using kube2iam for role assumption
+# aws_ebs_csi_annotations:
+#   - key: iam.amazonaws.com/role
+#     value: your-ebs-role-arn

internal/resources/kubespray/project/roles/kubernetes-apps/csi_driver/aws_ebs/tasks/main.yml

@@ -0,0 +1,26 @@
+---
+- name: AWS CSI Driver | Generate Manifests
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.file }}"
+    mode: "0644"
+  with_items:
+    - {name: aws-ebs-csi-driver, file: aws-ebs-csi-driver.yml}
+    - {name: aws-ebs-csi-controllerservice, file: aws-ebs-csi-controllerservice-rbac.yml}
+    - {name: aws-ebs-csi-controllerservice, file: aws-ebs-csi-controllerservice.yml}
+    - {name: aws-ebs-csi-nodeservice, file: aws-ebs-csi-nodeservice.yml}
+  register: aws_csi_manifests
+  when: inventory_hostname == groups['kube_control_plane'][0]
+
+- name: AWS CSI Driver | Apply Manifests
+  kube:
+    kubectl: "{{ bin_dir }}/kubectl"
+    filename: "{{ kube_config_dir }}/{{ item.item.file }}"
+    state: "latest"
+  with_items:
+    - "{{ aws_csi_manifests.results }}"
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - not item is skipped
+  loop_control:
+    label: "{{ item.item.file }}"

internal/resources/kubespray/project/roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-controllerservice-rbac.yml.j2

@@ -0,0 +1,180 @@
+# Controller Service
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: ebs-csi-controller-sa
+  namespace: kube-system
+
+---
+
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-external-provisioner-role
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["list", "watch", "create", "delete"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["get", "list", "watch", "create", "update", "patch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["csinodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "watch", "list", "delete", "update", "create"]
+
+---
+
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-csi-provisioner-binding
+subjects:
+  - kind: ServiceAccount
+    name: ebs-csi-controller-sa
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: ebs-external-provisioner-role
+  apiGroup: rbac.authorization.k8s.io
+
+---
+
+# The permissions in this ClusterRole are tightly coupled with the version of csi-attacher used. More information about this can be found in kubernetes-csi/external-attacher.
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-external-attacher-role
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "patch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["csinodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattachments"]
+    verbs: ["get", "list", "watch", "patch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattachments/status"]
+    verbs: ["patch"]
+
+---
+
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-csi-attacher-binding
+subjects:
+  - kind: ServiceAccount
+    name: ebs-csi-controller-sa
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: ebs-external-attacher-role
+  apiGroup: rbac.authorization.k8s.io
+
+{% if aws_ebs_csi_enable_volume_snapshot %}
+---
+
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-external-snapshotter-role
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["list", "watch", "create", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "list"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents"]
+    verbs: ["create", "get", "list", "watch", "update", "delete"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: ["apiextensions.k8s.io"]
+    resources: ["customresourcedefinitions"]
+    verbs: ["create", "list", "watch", "delete"]
+
+---
+
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-csi-snapshotter-binding
+subjects:
+  - kind: ServiceAccount
+    name: ebs-csi-controller-sa
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: ebs-external-snapshotter-role
+  apiGroup: rbac.authorization.k8s.io
+
+{% endif %}
+
+{% if aws_ebs_csi_enable_volume_resizing %}
+---
+
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-external-resizer-role
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims/status"]
+    verbs: ["update", "patch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["list", "watch", "create", "update", "patch"]
+
+---
+
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-csi-resizer-binding
+subjects:
+  - kind: ServiceAccount
+    name: ebs-csi-controller-sa
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: ebs-external-resizer-role
+  apiGroup: rbac.authorization.k8s.io
+
+{% endif %}

internal/resources/kubespray/project/roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-controllerservice.yml.j2

@@ -0,0 +1,131 @@
+---
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: ebs-csi-controller
+  namespace: kube-system
+spec:
+  replicas: {{ aws_ebs_csi_controller_replicas }}
+  selector:
+    matchLabels:
+      app: ebs-csi-controller
+      app.kubernetes.io/name: aws-ebs-csi-driver
+  template:
+    metadata:
+      labels:
+        app: ebs-csi-controller
+        app.kubernetes.io/name: aws-ebs-csi-driver
+{% if aws_ebs_csi_annotations is defined %}
+      annotations:
+{% for annotation in aws_ebs_csi_annotations %}
+        {{ annotation.key }}: {{ annotation.value }}
+{% endfor %}
+{% endif %}
+    spec:
+      nodeSelector:
+        kubernetes.io/os: linux
+      serviceAccountName: ebs-csi-controller-sa
+      priorityClassName: system-cluster-critical
+      containers:
+        - name: ebs-plugin
+          image: {{ aws_ebs_csi_plugin_image_repo }}:{{ aws_ebs_csi_plugin_image_tag }}
+          args:
+            - --endpoint=$(CSI_ENDPOINT)
+{% if aws_ebs_csi_extra_volume_tags is defined %}
+            - --extra-volume-tags={{ aws_ebs_csi_extra_volume_tags }}
+{% endif %}
+            - --logtostderr
+            - --v=5
+          env:
+            - name: CSI_ENDPOINT
+              value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock
+            - name: AWS_ACCESS_KEY_ID
+              valueFrom:
+                secretKeyRef:
+                  name: aws-secret
+                  key: key_id
+                  optional: true
+            - name: AWS_SECRET_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: aws-secret
+                  key: access_key
+                  optional: true
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+          ports:
+            - name: healthz
+              containerPort: 9808
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: healthz
+            initialDelaySeconds: 10
+            timeoutSeconds: 3
+            periodSeconds: 10
+            failureThreshold: 5
+        - name: csi-provisioner
+          image: {{ csi_provisioner_image_repo }}:{{ csi_provisioner_image_tag }}
+          args:
+            - --csi-address=$(ADDRESS)
+            - --v=5
+{% if aws_ebs_csi_enable_volume_scheduling %}
+            - --feature-gates=Topology=true
+{% endif %}
+            - --leader-election=true
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+        - name: csi-attacher
+          image: {{ csi_attacher_image_repo }}:{{ csi_attacher_image_tag }}
+          args:
+            - --csi-address=$(ADDRESS)
+            - --v=5
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+{% if aws_ebs_csi_enable_volume_snapshot %}
+        - name: csi-snapshotter
+          image: {{ csi_snapshotter_image_repo }}:{{ csi_snapshotter_image_tag }}
+          args:
+            - --csi-address=$(ADDRESS)
+            - --timeout=15s
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+{% endif %}
+{% if aws_ebs_csi_enable_volume_resizing %}
+        - name: csi-resizer
+          image: {{ csi_resizer_image_repo }}:{{ csi_resizer_image_tag }}
+          imagePullPolicy: {{ k8s_image_pull_policy }}
+          args:
+            - --csi-address=$(ADDRESS)
+            - --v=5
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+{% endif %}
+        - name: liveness-probe
+          image: {{ csi_livenessprobe_image_repo }}:{{ csi_livenessprobe_image_tag }}
+          args:
+            - --csi-address=/csi/csi.sock
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+      volumes:
+        - name: socket-dir
+          emptyDir: {}

internal/resources/kubespray/project/roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-driver.yml.j2

@@ -0,0 +1,8 @@
+---
+apiVersion: storage.k8s.io/v1
+kind: CSIDriver
+metadata:
+  name: ebs.csi.aws.com
+spec:
+  attachRequired: true
+  podInfoOnMount: false

internal/resources/kubespray/project/roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-nodeservice.yml.j2

@@ -0,0 +1,101 @@
+---
+# Node Service
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: ebs-csi-node
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      app: ebs-csi-node
+      app.kubernetes.io/name: aws-ebs-csi-driver
+  template:
+    metadata:
+      labels:
+        app: ebs-csi-node
+        app.kubernetes.io/name: aws-ebs-csi-driver
+    spec:
+      nodeSelector:
+        kubernetes.io/os: linux
+      hostNetwork: true
+      priorityClassName: system-node-critical
+      containers:
+        - name: ebs-plugin
+          securityContext:
+            privileged: true
+          image: {{ aws_ebs_csi_plugin_image_repo }}:{{ aws_ebs_csi_plugin_image_tag }}
+          args:
+            - --endpoint=$(CSI_ENDPOINT)
+{% if aws_ebs_csi_extra_volume_tags is defined %}
+            - --extra-volume-tags={{ aws_ebs_csi_extra_volume_tags }}
+{% endif %}
+            - --logtostderr
+            - --v=5
+          env:
+            - name: CSI_ENDPOINT
+              value: unix:/csi/csi.sock
+          volumeMounts:
+            - name: kubelet-dir
+              mountPath: /var/lib/kubelet
+              mountPropagation: "Bidirectional"
+            - name: plugin-dir
+              mountPath: /csi
+            - name: device-dir
+              mountPath: /dev
+          ports:
+            - name: healthz
+              containerPort: 9808
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: healthz
+            initialDelaySeconds: 10
+            timeoutSeconds: 3
+            periodSeconds: 10
+            failureThreshold: 5
+        - name: node-driver-registrar
+          image: {{ csi_node_driver_registrar_image_repo }}:{{ csi_node_driver_registrar_image_tag }}
+          args:
+            - --csi-address=$(ADDRESS)
+            - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
+            - --v=5
+          lifecycle:
+            preStop:
+              exec:
+                command: ["/bin/sh", "-c", "rm -rf /registration/ebs.csi.aws.com-reg.sock /csi/csi.sock"]
+          env:
+            - name: ADDRESS
+              value: /csi/csi.sock
+            - name: DRIVER_REG_SOCK_PATH
+              value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock
+          volumeMounts:
+            - name: plugin-dir
+              mountPath: /csi
+            - name: registration-dir
+              mountPath: /registration
+        - name: liveness-probe
+          image: {{ csi_livenessprobe_image_repo }}:{{ csi_livenessprobe_image_tag }}
+          args:
+            - --csi-address=/csi/csi.sock
+          volumeMounts:
+            - name: plugin-dir
+              mountPath: /csi
+      volumes:
+        - name: kubelet-dir
+          hostPath:
+            path: /var/lib/kubelet
+            type: Directory
+        - name: plugin-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins/ebs.csi.aws.com/
+            type: DirectoryOrCreate
+        - name: registration-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins_registry/
+            type: Directory
+        - name: device-dir
+          hostPath:
+            path: /dev
+            type: Directory

internal/resources/kubespray/project/roles/kubernetes-apps/csi_driver/azuredisk/defaults/main.yml

@@ -0,0 +1,6 @@
+---
+azure_csi_use_instance_metadata: true
+azure_csi_controller_replicas: 2
+azure_csi_plugin_image_tag: latest
+azure_csi_controller_affinity: {}
+azure_csi_node_affinity: {}

internal/resources/kubespray/project/roles/kubernetes-apps/csi_driver/azuredisk/tasks/azure-credential-check.yml

@@ -0,0 +1,54 @@
+---
+- name: Azure CSI Driver | check azure_csi_tenant_id value
+  fail:
+    msg: "azure_csi_tenant_id is missing"
+  when: azure_csi_tenant_id is not defined or not azure_csi_tenant_id
+
+- name: Azure CSI Driver | check azure_csi_subscription_id value
+  fail:
+    msg: "azure_csi_subscription_id is missing"
+  when: azure_csi_subscription_id is not defined or not azure_csi_subscription_id
+
+- name: Azure CSI Driver | check azure_csi_aad_client_id value
+  fail:
+    msg: "azure_csi_aad_client_id is missing"
+  when: azure_csi_aad_client_id is not defined or not azure_csi_aad_client_id
+
+- name: Azure CSI Driver | check azure_csi_aad_client_secret value
+  fail:
+    msg: "azure_csi_aad_client_secret is missing"
+  when: azure_csi_aad_client_secret is not defined or not azure_csi_aad_client_secret
+
+- name: Azure CSI Driver | check azure_csi_resource_group value
+  fail:
+    msg: "azure_csi_resource_group is missing"
+  when: azure_csi_resource_group is not defined or not azure_csi_resource_group
+
+- name: Azure CSI Driver | check azure_csi_location value
+  fail:
+    msg: "azure_csi_location is missing"
+  when: azure_csi_location is not defined or not azure_csi_location
+
+- name: Azure CSI Driver | check azure_csi_subnet_name value
+  fail:
+    msg: "azure_csi_subnet_name is missing"
+  when: azure_csi_subnet_name is not defined or not azure_csi_subnet_name
+
+- name: Azure CSI Driver | check azure_csi_security_group_name value
+  fail:
+    msg: "azure_csi_security_group_name is missing"
+  when: azure_csi_security_group_name is not defined or not azure_csi_security_group_name
+
+- name: Azure CSI Driver | check azure_csi_vnet_name value
+  fail:
+    msg: "azure_csi_vnet_name is missing"
+  when: azure_csi_vnet_name is not defined or not azure_csi_vnet_name
+
+- name: Azure CSI Driver | check azure_csi_vnet_resource_group value
+  fail:
+    msg: "azure_csi_vnet_resource_group is missing"
+  when: azure_csi_vnet_resource_group is not defined or not azure_csi_vnet_resource_group
+
+- name: "Azure CSI Driver | check azure_csi_use_instance_metadata is a bool"
+  assert:
+    that: azure_csi_use_instance_metadata | type_debug == 'bool'

internal/resources/kubespray/project/roles/kubernetes-apps/csi_driver/azuredisk/tasks/main.yml

@@ -0,0 +1,45 @@
+---
+- name: Azure CSI Driver | Check Azure credentials
+  include_tasks: azure-credential-check.yml
+
+- name: Azure CSI Driver | Write Azure CSI cloud-config
+  template:
+    src: "azure-csi-cloud-config.j2"
+    dest: "{{ kube_config_dir }}/azure_csi_cloud_config"
+    group: "{{ kube_cert_group }}"
+    mode: "0640"
+  when: inventory_hostname == groups['kube_control_plane'][0]
+
+- name: Azure CSI Driver | Get base64 cloud-config
+  slurp:
+    src: "{{ kube_config_dir }}/azure_csi_cloud_config"
+  register: cloud_config_secret
+  when: inventory_hostname == groups['kube_control_plane'][0]
+
+- name: Azure CSI Driver | Generate Manifests
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.file }}"
+    mode: "0644"
+  with_items:
+    - {name: azure-csi-azuredisk-driver, file: azure-csi-azuredisk-driver.yml}
+    - {name: azure-csi-cloud-config-secret, file: azure-csi-cloud-config-secret.yml}
+    - {name: azure-csi-azuredisk-controller, file: azure-csi-azuredisk-controller-rbac.yml}
+    - {name: azure-csi-azuredisk-controller, file: azure-csi-azuredisk-controller.yml}
+    - {name: azure-csi-azuredisk-node-rbac, file: azure-csi-azuredisk-node-rbac.yml}
+    - {name: azure-csi-azuredisk-node, file: azure-csi-azuredisk-node.yml}
+  register: azure_csi_manifests
+  when: inventory_hostname == groups['kube_control_plane'][0]
+
+- name: Azure CSI Driver | Apply Manifests
+  kube:
+    kubectl: "{{ bin_dir }}/kubectl"
+    filename: "{{ kube_config_dir }}/{{ item.item.file }}"
+    state: "latest"
+  with_items:
+    - "{{ azure_csi_manifests.results }}"
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - not item is skipped
+  loop_control:
+    label: "{{ item.item.file }}"

internal/resources/kubespray/project/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-controller-rbac.yml.j2

@@ -0,0 +1,230 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: csi-azuredisk-controller-sa
+  namespace: kube-system
+---
+
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: azuredisk-external-provisioner-role
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "create", "delete"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["get", "list", "watch", "create", "update", "patch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["csinodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "list", "watch", "create", "update", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots"]
+    verbs: ["get", "list"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents"]
+    verbs: ["get", "list"]
+---
+
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: azuredisk-csi-provisioner-binding
+subjects:
+  - kind: ServiceAccount
+    name: csi-azuredisk-controller-sa
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: azuredisk-external-provisioner-role
+  apiGroup: rbac.authorization.k8s.io
+
+---
+
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: azuredisk-external-attacher-role
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["csi.storage.k8s.io"]
+    resources: ["csinodeinfos"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattachments"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattachments/status"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "list", "watch", "create", "update", "patch"]
+---
+
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: azuredisk-csi-attacher-binding
+subjects:
+  - kind: ServiceAccount
+    name: csi-azuredisk-controller-sa
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: azuredisk-external-attacher-role
+  apiGroup: rbac.authorization.k8s.io
+
+---
+
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: azuredisk-cluster-driver-registrar-role
+rules:
+  - apiGroups: ["apiextensions.k8s.io"]
+    resources: ["customresourcedefinitions"]
+    verbs: ["create", "list", "watch", "delete"]
+  - apiGroups: ["csi.storage.k8s.io"]
+    resources: ["csidrivers"]
+    verbs: ["create", "delete"]
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "list", "watch", "create", "update", "patch"]
+---
+
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: azuredisk-csi-driver-registrar-binding
+subjects:
+  - kind: ServiceAccount
+    name: csi-azuredisk-controller-sa
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: azuredisk-cluster-driver-registrar-role
+  apiGroup: rbac.authorization.k8s.io
+
+---
+
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: azuredisk-external-snapshotter-role
+rules:
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["list", "watch", "create", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "list"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents"]
+    verbs: ["create", "get", "list", "watch", "update", "delete"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents/status"]
+    verbs: ["update"]
+  - apiGroups: ["apiextensions.k8s.io"]
+    resources: ["customresourcedefinitions"]
+    verbs: ["create", "list", "watch", "delete"]
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "watch", "list", "delete", "update", "create"]
+---
+
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: azuredisk-csi-snapshotter-binding
+subjects:
+  - kind: ServiceAccount
+    name: csi-azuredisk-controller-sa
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: azuredisk-external-snapshotter-role
+  apiGroup: rbac.authorization.k8s.io
+---
+
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: azuredisk-external-resizer-role
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims/status"]
+    verbs: ["update", "patch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["list", "watch", "create", "update", "patch"]
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "list", "watch", "create", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list", "watch"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: azuredisk-csi-resizer-role
+subjects:
+  - kind: ServiceAccount
+    name: csi-azuredisk-controller-sa
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: azuredisk-external-resizer-role
+  apiGroup: rbac.authorization.k8s.io
+
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-azuredisk-controller-secret-role
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "list"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-azuredisk-controller-secret-binding
+subjects:
+  - kind: ServiceAccount
+    name: csi-azuredisk-controller-sa
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: csi-azuredisk-controller-secret-role
+  apiGroup: rbac.authorization.k8s.io

internal/resources/kubespray/project/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-controller.yml.j2

@@ -0,0 +1,177 @@
+---
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: csi-azuredisk-controller
+  namespace: kube-system
+spec:
+  replicas: {{ azure_csi_controller_replicas }}
+  selector:
+    matchLabels:
+      app: csi-azuredisk-controller
+  template:
+    metadata:
+      labels:
+        app: csi-azuredisk-controller
+    spec:
+      hostNetwork: true
+      serviceAccountName: csi-azuredisk-controller-sa
+      nodeSelector:
+        kubernetes.io/os: linux
+      priorityClassName: system-cluster-critical
+      tolerations:
+        - key: "node-role.kubernetes.io/control-plane"
+          effect: "NoSchedule"
+{% if azure_csi_controller_affinity %}
+      affinity:
+        {{ azure_csi_controller_affinity | to_nice_yaml | indent(width=8) }}
+{% endif %}
+      containers:
+        - name: csi-provisioner
+          image: {{ azure_csi_image_repo }}/csi-provisioner:{{ azure_csi_provisioner_image_tag }}
+          imagePullPolicy: {{ k8s_image_pull_policy }}
+          args:
+            - "--feature-gates=Topology=true"
+            - "--csi-address=$(ADDRESS)"
+            - "--v=2"
+            - "--timeout=15s"
+            - "--leader-election"
+            - "--worker-threads=40"
+            - "--extra-create-metadata=true"
+            - "--strict-topology=true"
+          env:
+            - name: ADDRESS
+              value: /csi/csi.sock
+          volumeMounts:
+            - mountPath: /csi
+              name: socket-dir
+          resources:
+            limits:
+              memory: 500Mi
+            requests:
+              cpu: 10m
+              memory: 20Mi
+        - name: csi-attacher
+          image: {{ azure_csi_image_repo }}/csi-attacher:{{ azure_csi_attacher_image_tag }}
+          imagePullPolicy: {{ k8s_image_pull_policy }}
+          args:
+            - "-v=2"
+            - "-csi-address=$(ADDRESS)"
+            - "-timeout=600s"
+            - "-leader-election"
+            - "-worker-threads=500"
+          env:
+            - name: ADDRESS
+              value: /csi/csi.sock
+          volumeMounts:
+          - mountPath: /csi
+            name: socket-dir
+          resources:
+            limits:
+              memory: 500Mi
+            requests:
+              cpu: 10m
+              memory: 20Mi
+        - name: csi-snapshotter
+          image: {{ azure_csi_image_repo }}/csi-snapshotter:{{ azure_csi_snapshotter_image_tag }}
+          args:
+            - "-csi-address=$(ADDRESS)"
+            - "-leader-election"
+            - "-v=2"
+          env:
+            - name: ADDRESS
+              value: /csi/csi.sock
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+          resources:
+            limits:
+              memory: 100Mi
+            requests:
+              cpu: 10m
+              memory: 20Mi
+        - name: csi-resizer
+          image: {{ azure_csi_image_repo }}/csi-resizer:{{ azure_csi_resizer_image_tag }}
+          args:
+            - "-csi-address=$(ADDRESS)"
+            - "-v=2"
+            - "-leader-election"
+            - '-handle-volume-inuse-error=false'
+            - "-timeout=60s"
+          env:
+            - name: ADDRESS
+              value: /csi/csi.sock
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+          resources:
+            limits:
+              memory: 500Mi
+            requests:
+              cpu: 10m
+              memory: 20Mi
+        - name: liveness-probe
+          image: {{ azure_csi_image_repo }}/livenessprobe:{{ azure_csi_livenessprobe_image_tag }}
+          args:
+            - --csi-address=/csi/csi.sock
+            - --probe-timeout=3s
+            - --health-port=29602
+            - --v=2
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+          resources:
+            limits:
+              memory: 100Mi
+            requests:
+              cpu: 10m
+              memory: 20Mi
+        - name: azuredisk
+          image: {{ azure_csi_plugin_image_repo }}/azuredisk-csi:{{ azure_csi_plugin_image_tag }}
+          imagePullPolicy: {{ k8s_image_pull_policy }}
+          args:
+            - "--v=5"
+            - "--endpoint=$(CSI_ENDPOINT)"
+            - "--metrics-address=0.0.0.0:29604"
+            - "--disable-avset-nodes=true"
+            - "--drivername=disk.csi.azure.com"
+            - "--cloud-config-secret-name=cloud-config"
+            - "--cloud-config-secret-namespace=kube-system"
+          ports:
+            - containerPort: 29602
+              name: healthz
+              protocol: TCP
+            - containerPort: 29604
+              name: metrics
+              protocol: TCP
+          livenessProbe:
+            failureThreshold: 5
+            httpGet:
+              path: /healthz
+              port: healthz
+            initialDelaySeconds: 30
+            timeoutSeconds: 10
+            periodSeconds: 30
+          env:
+            - name: AZURE_CREDENTIAL_FILE
+              value: "/etc/kubernetes/azure.json"
+            - name: CSI_ENDPOINT
+              value: unix:///csi/csi.sock
+          volumeMounts:
+            - mountPath: /csi
+              name: socket-dir
+            - mountPath: /etc/kubernetes/
+              name: azure-cred
+              readOnly: true
+          resources:
+            limits:
+              memory: 500Mi
+            requests:
+              cpu: 10m
+              memory: 20Mi
+      volumes:
+        - name: socket-dir
+          emptyDir: {}
+        - name: azure-cred
+          secret:
+            secretName: cloud-config

internal/resources/kubespray/project/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-driver.yml.j2

@@ -0,0 +1,10 @@
+---
+apiVersion: storage.k8s.io/v1
+kind: CSIDriver
+metadata:
+  name: disk.csi.azure.com
+spec:
+  attachRequired: true
+  podInfoOnMount: true
+  volumeLifecycleModes:  # added in Kubernetes 1.16
+    - Persistent

internal/resources/kubespray/project/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-node-rbac.yml.j2

@@ -0,0 +1,30 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: csi-azuredisk-node-sa
+  namespace: kube-system
+
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-azuredisk-node-secret-role
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "list"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-azuredisk-node-secret-binding
+subjects:
+  - kind: ServiceAccount
+    name: csi-azuredisk-node-sa
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: csi-azuredisk-node-secret-role
+  apiGroup: rbac.authorization.k8s.io

internal/resources/kubespray/project/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-node.yml.j2

@@ -0,0 +1,168 @@
+---
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: csi-azuredisk-node
+  namespace: kube-system
+spec:
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 1
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      app: csi-azuredisk-node
+  template:
+    metadata:
+      labels:
+        app: csi-azuredisk-node
+    spec:
+      hostNetwork: true
+      dnsPolicy: Default
+      serviceAccountName: csi-azuredisk-node-sa
+      nodeSelector:
+        kubernetes.io/os: linux
+{% if azure_csi_node_affinity %}
+      affinity:
+        {{ azure_csi_node_affinity | to_nice_yaml | indent(width=8) }}
+{% endif %}
+      priorityClassName: system-node-critical
+      tolerations:
+        - operator: Exists
+      containers:
+        - name: liveness-probe
+          volumeMounts:
+            - mountPath: /csi
+              name: socket-dir
+          image: {{ azure_csi_image_repo }}/livenessprobe:{{ azure_csi_livenessprobe_image_tag }}
+          imagePullPolicy: {{ k8s_image_pull_policy }}
+          args:
+            - --csi-address=/csi/csi.sock
+            - --probe-timeout=3s
+            - --health-port=29603
+            - --v=2
+          resources:
+            limits:
+              memory: 100Mi
+            requests:
+              cpu: 10m
+              memory: 20Mi
+        - name: node-driver-registrar
+          image: {{ azure_csi_image_repo }}/csi-node-driver-registrar:{{ azure_csi_node_registrar_image_tag }}
+          args:
+            - --csi-address=$(ADDRESS)
+            - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
+            - --v=2
+          livenessProbe:
+            exec:
+              command:
+                - /csi-node-driver-registrar
+                - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
+                - --mode=kubelet-registration-probe
+            initialDelaySeconds: 30
+            timeoutSeconds: 15
+          env:
+            - name: ADDRESS
+              value: /csi/csi.sock
+            - name: DRIVER_REG_SOCK_PATH
+              value: /var/lib/kubelet/plugins/disk.csi.azure.com/csi.sock
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+            - name: registration-dir
+              mountPath: /registration
+          resources:
+            limits:
+              memory: 100Mi
+            requests:
+              cpu: 10m
+              memory: 20Mi
+        - name: azuredisk
+          image: {{ azure_csi_plugin_image_repo }}/azuredisk-csi:{{ azure_csi_plugin_image_tag }}
+          imagePullPolicy: {{ k8s_image_pull_policy }}
+          args:
+            - "--v=5"
+            - "--endpoint=$(CSI_ENDPOINT)"
+            - "--nodeid=$(KUBE_NODE_NAME)"
+            - "--metrics-address=0.0.0.0:29605"
+            - "--enable-perf-optimization=true"
+            - "--drivername=disk.csi.azure.com"
+            - "--volume-attach-limit=-1"
+            - "--cloud-config-secret-name=cloud-config"
+            - "--cloud-config-secret-namespace=kube-system"
+          ports:
+            - containerPort: 29603
+              name: healthz
+              protocol: TCP
+            - containerPort: 29605
+              name: metrics
+              protocol: TCP
+          livenessProbe:
+            failureThreshold: 5
+            httpGet:
+              path: /healthz
+              port: healthz
+            initialDelaySeconds: 30
+            timeoutSeconds: 10
+            periodSeconds: 30
+          env:
+            - name: AZURE_CREDENTIAL_FILE
+              value: "/etc/kubernetes/azure.json"
+            - name: CSI_ENDPOINT
+              value: unix:///csi/csi.sock
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: spec.nodeName
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - mountPath: /csi
+              name: socket-dir
+            - mountPath: /var/lib/kubelet/
+              mountPropagation: Bidirectional
+              name: mountpoint-dir
+            - mountPath: /etc/kubernetes/
+              name: azure-cred
+            - mountPath: /dev
+              name: device-dir
+            - mountPath: /sys/bus/scsi/devices
+              name: sys-devices-dir
+            - mountPath: /sys/class/scsi_host/
+              name: scsi-host-dir
+          resources:
+            limits:
+              memory: 200Mi
+            requests:
+              cpu: 10m
+              memory: 20Mi
+      volumes:
+        - hostPath:
+            path: /var/lib/kubelet/plugins/disk.csi.azure.com
+            type: DirectoryOrCreate
+          name: socket-dir
+        - hostPath:
+            path: /var/lib/kubelet/
+            type: DirectoryOrCreate
+          name: mountpoint-dir
+        - hostPath:
+            path: /var/lib/kubelet/plugins_registry/
+            type: DirectoryOrCreate
+          name: registration-dir
+        - secret:
+            defaultMode: 0644
+            secretName: cloud-config
+          name: azure-cred
+        - hostPath:
+            path: /dev
+            type: Directory
+          name: device-dir
+        - hostPath:
+            path: /sys/bus/scsi/devices
+            type: Directory
+          name: sys-devices-dir
+        - hostPath:
+            path: /sys/class/scsi_host/
+            type: Directory
+          name: scsi-host-dir

internal/resources/kubespray/project/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-cloud-config-secret.yml.j2

@@ -0,0 +1,7 @@
+kind: Secret
+apiVersion: v1
+metadata:
+  name: cloud-config
+  namespace: kube-system
+data:
+  azure.json: {{ cloud_config_secret.content }}

internal/resources/kubespray/project/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-cloud-config.j2

@@ -0,0 +1,14 @@
+{
+    "cloud":"AzurePublicCloud",
+    "tenantId": "{{ azure_csi_tenant_id }}",
+    "subscriptionId": "{{ azure_csi_subscription_id }}",
+    "aadClientId": "{{ azure_csi_aad_client_id }}",
+    "aadClientSecret": "{{ azure_csi_aad_client_secret }}",
+    "location": "{{ azure_csi_location }}",
+    "resourceGroup": "{{ azure_csi_resource_group }}",
+    "vnetName": "{{ azure_csi_vnet_name }}",
+    "vnetResourceGroup": "{{ azure_csi_vnet_resource_group }}",
+    "subnetName": "{{ azure_csi_subnet_name }}",
+    "securityGroupName": "{{ azure_csi_security_group_name }}",
+    "useInstanceMetadata": {{ azure_csi_use_instance_metadata }},
+}

internal/resources/kubespray/project/roles/kubernetes-apps/csi_driver/cinder/defaults/main.yml

@@ -0,0 +1,48 @@
+---
+cinder_csi_attacher_image_tag: "v4.4.2"
+cinder_csi_provisioner_image_tag: "v3.6.2"
+cinder_csi_snapshotter_image_tag: "v6.3.2"
+cinder_csi_resizer_image_tag: "v1.9.2"
+cinder_csi_livenessprobe_image_tag: "v2.11.0"
+
+# To access Cinder, the CSI controller will need credentials to access
+# openstack apis. Per default this values will be
+# read from the environment.
+cinder_auth_url: "{{ lookup('env', 'OS_AUTH_URL') }}"
+cinder_username: "{{ lookup('env', 'OS_USERNAME') }}"
+cinder_password: "{{ lookup('env', 'OS_PASSWORD') }}"
+cinder_application_credential_id: "{{ lookup('env', 'OS_APPLICATION_CREDENTIAL_ID') }}"
+cinder_application_credential_name: "{{ lookup('env', 'OS_APPLICATION_CREDENTIAL_NAME') }}"
+cinder_application_credential_secret: "{{ lookup('env', 'OS_APPLICATION_CREDENTIAL_SECRET') }}"
+cinder_region: "{{ lookup('env', 'OS_REGION_NAME') }}"
+cinder_tenant_id: "{{ lookup('env', 'OS_TENANT_ID') | default(lookup('env', 'OS_PROJECT_ID'), true) }}"
+cinder_tenant_name: "{{ lookup('env', 'OS_TENANT_NAME') | default(lookup('env', 'OS_PROJECT_NAME'), true) }}"
+cinder_domain_name: "{{ lookup('env', 'OS_USER_DOMAIN_NAME') }}"
+cinder_domain_id: "{{ lookup('env', 'OS_USER_DOMAIN_ID') }}"
+cinder_cacert: "{{ lookup('env', 'OS_CACERT') }}"
+
+# For now, only Cinder v3 is supported in Cinder CSI driver
+cinder_blockstorage_version: "v3"
+cinder_csi_controller_replicas: 1
+
+# Optional. Set to true, to rescan block device and verify its size before expanding
+# the filesystem.
+# Not all hypervizors have a /sys/class/block/XXX/device/rescan location, therefore if
+# you enable this option and your hypervizor doesn't support this, you'll get a warning
+# log on resize event. It is recommended to disable this option in this case.
+# Defaults to false
+# cinder_csi_rescan_on_resize: true
+
+cinder_tolerations: []
+
+## Dictionaries of extra arguments to add to the cinder CSI plugin containers
+## Format:
+##  cinder_csi_attacher_extra_args:
+##    arg1: "value1"
+##    arg2: "value2"
+cinder_csi_attacher_extra_args: {}
+cinder_csi_provisioner_extra_args: {}
+cinder_csi_snapshotter_extra_args: {}
+cinder_csi_resizer_extra_args: {}
+cinder_csi_plugin_extra_args: {}
+cinder_liveness_probe_extra_args: {}

internal/resources/kubespray/project/roles/kubernetes-apps/csi_driver/cinder/tasks/cinder-credential-check.yml

@@ -0,0 +1,59 @@
+---
+- name: Cinder CSI Driver | check cinder_auth_url value
+  fail:
+    msg: "cinder_auth_url is missing"
+  when: cinder_auth_url is not defined or not cinder_auth_url
+
+- name: Cinder CSI Driver | check cinder_username value cinder_application_credential_name value
+  fail:
+    msg: "you must either set cinder_username or cinder_application_credential_name"
+  when:
+    - cinder_username is not defined or not cinder_username
+    - cinder_application_credential_name is not defined or not cinder_application_credential_name
+
+- name: Cinder CSI Driver | check cinder_application_credential_id value
+  fail:
+    msg: "cinder_application_credential_id is missing"
+  when:
+    - cinder_application_credential_name is defined
+    - cinder_application_credential_name | length > 0
+    - cinder_application_credential_id is not defined or not cinder_application_credential_id
+
+- name: Cinder CSI Driver | check cinder_application_credential_secret value
+  fail:
+    msg: "cinder_application_credential_secret is missing"
+  when:
+    - cinder_application_credential_name is defined
+    - cinder_application_credential_name | length > 0
+    - cinder_application_credential_secret is not defined or not cinder_application_credential_secret
+
+- name: Cinder CSI Driver | check cinder_password value
+  fail:
+    msg: "cinder_password is missing"
+  when:
+    - cinder_username is defined
+    - cinder_username | length > 0
+    - cinder_application_credential_name is not defined or not cinder_application_credential_name
+    - cinder_application_credential_secret is not defined or not cinder_application_credential_secret
+    - cinder_password is not defined or not cinder_password
+
+- name: Cinder CSI Driver | check cinder_region value
+  fail:
+    msg: "cinder_region is missing"
+  when: cinder_region is not defined or not cinder_region
+
+- name: Cinder CSI Driver | check cinder_tenant_id value
+  fail:
+    msg: "one of cinder_tenant_id or cinder_tenant_name must be specified"
+  when:
+    - cinder_tenant_id is not defined or not cinder_tenant_id
+    - cinder_tenant_name is not defined or not cinder_tenant_name
+    - cinder_application_credential_name is not defined or not cinder_application_credential_name
+
+- name: Cinder CSI Driver | check cinder_domain_id value
+  fail:
+    msg: "one of cinder_domain_id or cinder_domain_name must be specified"
+  when:
+    - cinder_domain_id is not defined or not cinder_domain_id
+    - cinder_domain_name is not defined or not cinder_domain_name
+    - cinder_application_credential_name is not defined or not cinder_application_credential_name

internal/resources/kubespray/project/roles/kubernetes-apps/csi_driver/cinder/tasks/cinder-write-cacert.yml

@@ -0,0 +1,11 @@
+---
+# include to workaround mitogen issue
+# https://github.com/dw/mitogen/issues/663
+
+- name: Cinder CSI Driver | Write cacert file
+  copy:
+    src: "{{ cinder_cacert }}"
+    dest: "{{ kube_config_dir }}/cinder-cacert.pem"
+    group: "{{ kube_cert_group }}"
+    mode: "0640"
+  delegate_to: "{{ delegate_host_to_write_cacert }}"

internal/resources/kubespray/project/roles/kubernetes-apps/csi_driver/cinder/tasks/main.yml

@@ -0,0 +1,57 @@
+---
+- name: Cinder CSI Driver | Check Cinder credentials
+  include_tasks: cinder-credential-check.yml
+
+- name: Cinder CSI Driver | Write cacert file
+  include_tasks: cinder-write-cacert.yml
+  run_once: true
+  loop: "{{ groups['k8s_cluster'] }}"
+  loop_control:
+    loop_var: delegate_host_to_write_cacert
+  when:
+    - ('k8s_cluster' in group_names)
+    - cinder_cacert is defined
+    - cinder_cacert | length > 0
+
+- name: Cinder CSI Driver | Write Cinder cloud-config
+  template:
+    src: "cinder-csi-cloud-config.j2"
+    dest: "{{ kube_config_dir }}/cinder_cloud_config"
+    group: "{{ kube_cert_group }}"
+    mode: "0640"
+  when: inventory_hostname == groups['kube_control_plane'][0]
+
+- name: Cinder CSI Driver | Get base64 cloud-config
+  slurp:
+    src: "{{ kube_config_dir }}/cinder_cloud_config"
+  register: cloud_config_secret
+  when: inventory_hostname == groups['kube_control_plane'][0]
+
+- name: Cinder CSI Driver | Generate Manifests
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.file }}"
+    mode: "0644"
+  with_items:
+    - {name: cinder-csi-driver, file: cinder-csi-driver.yml}
+    - {name: cinder-csi-cloud-config-secret, file: cinder-csi-cloud-config-secret.yml}
+    - {name: cinder-csi-controllerplugin, file: cinder-csi-controllerplugin-rbac.yml}
+    - {name: cinder-csi-controllerplugin, file: cinder-csi-controllerplugin.yml}
+    - {name: cinder-csi-nodeplugin, file: cinder-csi-nodeplugin-rbac.yml}
+    - {name: cinder-csi-nodeplugin, file: cinder-csi-nodeplugin.yml}
+    - {name: cinder-csi-poddisruptionbudget, file: cinder-csi-poddisruptionbudget.yml}
+  register: cinder_csi_manifests
+  when: inventory_hostname == groups['kube_control_plane'][0]
+
+- name: Cinder CSI Driver | Apply Manifests
+  kube:
+    kubectl: "{{ bin_dir }}/kubectl"
+    filename: "{{ kube_config_dir }}/{{ item.item.file }}"
+    state: "latest"
+  with_items:
+    - "{{ cinder_csi_manifests.results }}"
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - not item is skipped
+  loop_control:
+    label: "{{ item.item.file }}"

internal/resources/kubespray/project/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-cloud-config-secret.yml.j2

@@ -0,0 +1,10 @@
+# This YAML file contains secret objects,
+# which are necessary to run csi cinder plugin.
+
+kind: Secret
+apiVersion: v1
+metadata:
+  name: cloud-config
+  namespace: kube-system
+data:
+  cloud.conf: {{ cloud_config_secret.content }}

internal/resources/kubespray/project/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-cloud-config.j2

@@ -0,0 +1,44 @@
+[Global]
+auth-url="{{ cinder_auth_url }}"
+{% if cinder_application_credential_id|length == 0 and cinder_application_credential_name|length == 0 %}
+username="{{ cinder_username }}"
+password="{{ cinder_password }}"
+{% endif %}
+{% if cinder_application_credential_id|length > 0 %}
+application-credential-id={{ cinder_application_credential_id }}
+{% endif %}
+{% if cinder_application_credential_name|length > 0 %}
+application-credential-name={{ cinder_application_credential_name }}
+{% endif %}
+{% if cinder_application_credential_secret|length > 0 %}
+application-credential-secret={{ cinder_application_credential_secret }}
+{% endif %}
+region="{{ cinder_region }}"
+{% if cinder_tenant_id|length > 0 %}
+tenant-id="{{ cinder_tenant_id }}"
+{% endif %}
+{% if cinder_tenant_name|length > 0 %}
+tenant-name="{{ cinder_tenant_name }}"
+{% endif %}
+{% if cinder_domain_name|length > 0 %}
+domain-name="{{ cinder_domain_name }}"
+{% elif cinder_domain_id|length > 0 %}
+domain-id ="{{ cinder_domain_id }}"
+{% endif %}
+{% if cinder_cacert|length > 0 %}
+ca-file="{{ kube_config_dir }}/cinder-cacert.pem"
+{% endif %}
+
+[BlockStorage]
+{% if cinder_blockstorage_version is defined %}
+bs-version={{ cinder_blockstorage_version }}
+{% endif %}
+{% if cinder_csi_ignore_volume_az is defined %}
+ignore-volume-az={{ cinder_csi_ignore_volume_az | bool }}
+{% endif %}
+{% if node_volume_attach_limit is defined and node_volume_attach_limit != "" %}
+node-volume-attach-limit="{{ node_volume_attach_limit }}"
+{% endif %}
+{% if cinder_csi_rescan_on_resize is defined %}
+rescan-on-resize={{ cinder_csi_rescan_on_resize | bool }}
+{% endif %}

internal/resources/kubespray/project/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-controllerplugin-rbac.yml.j2

@@ -0,0 +1,179 @@
+# This YAML file contains RBAC API objects,
+# which are necessary to run csi controller plugin
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: csi-cinder-controller-sa
+  namespace: kube-system
+
+---
+# external attacher
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-attacher-role
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "patch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["csinodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattachments"]
+    verbs: ["get", "list", "watch", "patch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattachments/status"]
+    verbs: ["patch"]
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "watch", "list", "delete", "update", "create"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-attacher-binding
+subjects:
+  - kind: ServiceAccount
+    name: csi-cinder-controller-sa
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: csi-attacher-role
+  apiGroup: rbac.authorization.k8s.io
+
+---
+# external Provisioner
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-provisioner-role
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "create", "delete"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["csinodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["list", "watch", "create", "update", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots"]
+    verbs: ["get", "list"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents"]
+    verbs: ["get", "list"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattachments"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "watch", "list", "delete", "update", "create"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-provisioner-binding
+subjects:
+  - kind: ServiceAccount
+    name: csi-cinder-controller-sa
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: csi-provisioner-role
+  apiGroup: rbac.authorization.k8s.io
+
+---
+# external snapshotter
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-snapshotter-role
+rules:
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["list", "watch", "create", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "list"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents"]
+    verbs: ["create", "get", "list", "watch", "update", "patch", "delete"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents/status"]
+    verbs: ["update"]
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "watch", "list", "delete", "update", "create"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-snapshotter-binding
+subjects:
+  - kind: ServiceAccount
+    name: csi-cinder-controller-sa
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: csi-snapshotter-role
+  apiGroup: rbac.authorization.k8s.io
+---
+
+# External Resizer
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-resizer-role
+rules:
+  # The following rule should be uncommented for plugins that require secrets
+  # for provisioning.
+  # - apiGroups: [""]
+  #   resources: ["secrets"]
+  #   verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "patch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims/status"]
+    verbs: ["patch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["list", "watch", "create", "update", "patch"]
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "watch", "list", "delete", "update", "create"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-resizer-binding
+subjects:
+  - kind: ServiceAccount
+    name: csi-cinder-controller-sa
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: csi-resizer-role
+  apiGroup: rbac.authorization.k8s.io

internal/resources/kubespray/project/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-controllerplugin.yml.j2

@@ -0,0 +1,191 @@
+# This YAML file contains CSI Controller Plugin Sidecars
+# external-attacher, external-provisioner, external-snapshotter
+# external-resize, liveness-probe
+
+---
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: csi-cinder-controllerplugin
+  namespace: kube-system
+spec:
+  replicas: {{ cinder_csi_controller_replicas }}
+  selector:
+    matchLabels:
+      app: csi-cinder-controllerplugin
+  template:
+    metadata:
+      labels:
+        app: csi-cinder-controllerplugin
+    spec:
+      serviceAccount: csi-cinder-controller-sa
+      containers:
+        - name: csi-attacher
+          image: {{ csi_attacher_image_repo }}:{{ cinder_csi_attacher_image_tag }}
+          imagePullPolicy: {{ k8s_image_pull_policy }}
+          args:
+            - "--csi-address=$(ADDRESS)"
+            - "--timeout=3m"
+{% if cinder_csi_controller_replicas is defined and cinder_csi_controller_replicas > 1 %}
+            - --leader-election=true
+{% endif %}
+            - "--default-fstype=ext4"
+{% for key, value in cinder_csi_attacher_extra_args.items() %}
+            - "{{ '--' + key + '=' + value }}"
+{% endfor %}
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+        - name: csi-provisioner
+          image: {{ csi_provisioner_image_repo }}:{{ cinder_csi_provisioner_image_tag }}
+          imagePullPolicy: {{ k8s_image_pull_policy }}
+          args:
+            - "--csi-address=$(ADDRESS)"
+            - "--timeout=3m"
+            - "--default-fstype=ext4"
+            - "--extra-create-metadata"
+{% if cinder_topology is defined and cinder_topology %}
+            - --feature-gates=Topology=true
+{% endif %}
+{% if cinder_csi_controller_replicas is defined and cinder_csi_controller_replicas > 1 %}
+            - "--leader-election=true"
+{% endif %}
+{% for key, value in cinder_csi_provisioner_extra_args.items() %}
+            - "{{ '--' + key + '=' + value }}"
+{% endfor %}
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+        - name: csi-snapshotter
+          image: {{ csi_snapshotter_image_repo }}:{{ cinder_csi_snapshotter_image_tag }}
+          imagePullPolicy: {{ k8s_image_pull_policy }}
+          args:
+            - "--csi-address=$(ADDRESS)"
+            - "--timeout=3m"
+            - "--extra-create-metadata"
+{% if cinder_csi_controller_replicas is defined and cinder_csi_controller_replicas > 1 %}
+            - --leader-election=true
+{% endif %}
+{% for key, value in cinder_csi_snapshotter_extra_args.items() %}
+            - "{{ '--' + key + '=' + value }}"
+{% endfor %}
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+          volumeMounts:
+            - mountPath: /var/lib/csi/sockets/pluginproxy/
+              name: socket-dir
+        - name: csi-resizer
+          image: {{ csi_resizer_image_repo }}:{{ cinder_csi_resizer_image_tag }}
+          imagePullPolicy: {{ k8s_image_pull_policy }}
+          args:
+            - "--csi-address=$(ADDRESS)"
+            - "--timeout=3m"
+            - "--handle-volume-inuse-error=false"
+{% if cinder_csi_controller_replicas is defined and cinder_csi_controller_replicas > 1 %}
+            - --leader-election=true
+{% endif %}
+{% for key, value in cinder_csi_resizer_extra_args.items() %}
+            - "{{ '--' + key + '=' + value }}"
+{% endfor %}
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+        - name: liveness-probe
+          image: {{ csi_livenessprobe_image_repo }}:{{ cinder_csi_livenessprobe_image_tag }}
+          imagePullPolicy: {{ k8s_image_pull_policy }}
+          args:
+            - "--csi-address=$(ADDRESS)"
+{% for key, value in cinder_liveness_probe_extra_args.items() %}
+            - "{{ '--' + key + '=' + value }}"
+{% endfor %}
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+          volumeMounts:
+            - mountPath: /var/lib/csi/sockets/pluginproxy/
+              name: socket-dir
+        - name: cinder-csi-plugin
+          image: {{ cinder_csi_plugin_image_repo }}:{{ cinder_csi_plugin_image_tag }}
+          imagePullPolicy: {{ k8s_image_pull_policy }}
+          args:
+            - /bin/cinder-csi-plugin
+            - "--endpoint=$(CSI_ENDPOINT)"
+            - "--cloud-config=$(CLOUD_CONFIG)"
+            - "--cluster=$(CLUSTER_NAME)"
+{% for key, value in cinder_csi_plugin_extra_args.items() %}
+            - "{{ '--' + key + '=' + value }}"
+{% endfor %}
+          env:
+            - name: CSI_ENDPOINT
+              value: unix://csi/csi.sock
+            - name: CLOUD_CONFIG
+              value: /etc/config/cloud.conf
+            - name: CLUSTER_NAME
+              value: {{ cluster_name }}
+          ports:
+            - containerPort: 9808
+              name: healthz
+              protocol: TCP
+          livenessProbe:
+            failureThreshold: 5
+            httpGet:
+              path: /healthz
+              port: healthz
+            initialDelaySeconds: 10
+            timeoutSeconds: 10
+            periodSeconds: 60
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+            - name: secret-cinderplugin
+              mountPath: /etc/config
+              readOnly: true
+            - name: ca-certs
+              mountPath: /etc/ssl/certs
+              readOnly: true
+{% if ssl_ca_dirs | length %}
+{% for dir in ssl_ca_dirs %}
+            - name: {{ dir | regex_replace('^/(.*)$', '\\1' ) | regex_replace('/', '-') }}
+              mountPath: {{ dir }}
+              readOnly: true
+{% endfor %}
+{% endif %}
+{% if cinder_cacert is defined and cinder_cacert != "" %}
+            - name: cinder-cacert
+              mountPath: {{ kube_config_dir }}/cinder-cacert.pem
+              readOnly: true
+{% endif %}
+      volumes:
+        - name: socket-dir
+          emptyDir:
+        - name: secret-cinderplugin
+          secret:
+            secretName: cloud-config
+        - name: ca-certs
+          hostPath:
+            path: /etc/ssl/certs
+            type: DirectoryOrCreate
+{% if ssl_ca_dirs | length %}
+{% for dir in ssl_ca_dirs %}
+        - name: {{ dir | regex_replace('^/(.*)$', '\\1' ) | regex_replace('/', '-') }}
+          hostPath:
+            path: {{ dir }}
+            type: DirectoryOrCreate
+{% endfor %}
+{% endif %}
+{% if cinder_cacert is defined and cinder_cacert != "" %}
+        - name: cinder-cacert
+          hostPath:
+            path: {{ kube_config_dir }}/cinder-cacert.pem
+            type: FileOrCreate
+{% endif %}

internal/resources/kubespray/project/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-driver.yml.j2

@@ -0,0 +1,10 @@
+apiVersion: storage.k8s.io/v1
+kind: CSIDriver
+metadata:
+  name: cinder.csi.openstack.org
+spec:
+  attachRequired: true
+  podInfoOnMount: true
+  volumeLifecycleModes:
+  - Persistent
+  - Ephemeral

internal/resources/kubespray/project/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-nodeplugin-rbac.yml.j2

@@ -0,0 +1,30 @@
+# This YAML defines all API objects to create RBAC roles for csi node plugin.
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: csi-cinder-node-sa
+  namespace: kube-system
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-nodeplugin-role
+rules:
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["get", "list", "watch", "create", "update", "patch"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-nodeplugin-binding
+subjects:
+  - kind: ServiceAccount
+    name: csi-cinder-node-sa
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: csi-nodeplugin-role
+  apiGroup: rbac.authorization.k8s.io

internal/resources/kubespray/project/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-nodeplugin.yml.j2

@@ -0,0 +1,148 @@
+# This YAML file contains driver-registrar & csi driver nodeplugin API objects,
+# which are necessary to run csi nodeplugin for cinder.
+
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: csi-cinder-nodeplugin
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      app: csi-cinder-nodeplugin
+  template:
+    metadata:
+      labels:
+        app: csi-cinder-nodeplugin
+    spec:
+      tolerations:
+        - operator: Exists
+      serviceAccountName: csi-cinder-node-sa
+      hostNetwork: true
+      dnsPolicy: ClusterFirstWithHostNet
+      containers:
+        - name: node-driver-registrar
+          image: {{ csi_node_driver_registrar_image_repo }}:{{ csi_node_driver_registrar_image_tag }}
+          imagePullPolicy: {{ k8s_image_pull_policy }}
+          args:
+            - "--csi-address=$(ADDRESS)"
+            - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)"
+          env:
+            - name: ADDRESS
+              value: /csi/csi.sock
+            - name: DRIVER_REG_SOCK_PATH
+              value: /var/lib/kubelet/plugins/cinder.csi.openstack.org/csi.sock
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+            - name: registration-dir
+              mountPath: /registration
+        - name: liveness-probe
+          image: {{ csi_livenessprobe_image_repo }}:{{ csi_livenessprobe_image_tag }}
+          args:
+            - "--csi-address=/csi/csi.sock"
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+        - name: cinder-csi-plugin
+          securityContext:
+            privileged: true
+            capabilities:
+              add: ["SYS_ADMIN"]
+            allowPrivilegeEscalation: true
+          image: {{ cinder_csi_plugin_image_repo }}:{{ cinder_csi_plugin_image_tag }}
+          imagePullPolicy: {{ k8s_image_pull_policy }}
+          args:
+            - /bin/cinder-csi-plugin
+            - "--endpoint=$(CSI_ENDPOINT)"
+            - "--cloud-config=$(CLOUD_CONFIG)"
+          env:
+            - name: CSI_ENDPOINT
+              value: unix://csi/csi.sock
+            - name: CLOUD_CONFIG
+              value: /etc/config/cloud.conf
+          ports:
+            - containerPort: 9808
+              name: healthz
+              protocol: TCP
+          livenessProbe:
+            failureThreshold: 5
+            httpGet:
+              path: /healthz
+              port: healthz
+            initialDelaySeconds: 10
+            timeoutSeconds: 3
+            periodSeconds: 10
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+            - name: kubelet-dir
+              mountPath: /var/lib/kubelet
+              mountPropagation: "Bidirectional"
+            - name: pods-probe-dir
+              mountPath: /dev
+              mountPropagation: "HostToContainer"
+            - name: secret-cinderplugin
+              mountPath: /etc/config
+              readOnly: true
+            - name: ca-certs
+              mountPath: /etc/ssl/certs
+              readOnly: true
+{% if ssl_ca_dirs | length %}
+{% for dir in ssl_ca_dirs %}
+            - name: {{ dir | regex_replace('^/(.*)$', '\\1' ) | regex_replace('/', '-') }}
+              mountPath: {{ dir }}
+              readOnly: true
+{% endfor %}
+{% endif %}
+{% if cinder_cacert is defined and cinder_cacert != "" %}
+            - name: cinder-cacert
+              mountPath: {{ kube_config_dir }}/cinder-cacert.pem
+              readOnly: true
+{% endif %}
+      volumes:
+        - name: socket-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins/cinder.csi.openstack.org
+            type: DirectoryOrCreate
+        - name: registration-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins_registry/
+            type: Directory
+        - name: kubelet-dir
+          hostPath:
+            path: /var/lib/kubelet
+            type: Directory
+        - name: pods-probe-dir
+          hostPath:
+            path: /dev
+            type: Directory
+        - name: secret-cinderplugin
+          secret:
+            secretName: cloud-config
+        - name: ca-certs
+          hostPath:
+            path: /etc/ssl/certs
+            type: DirectoryOrCreate
+{% if ssl_ca_dirs | length %}
+{% for dir in ssl_ca_dirs %}
+        - name: {{ dir | regex_replace('^/(.*)$', '\\1' ) | regex_replace('/', '-') }}
+          hostPath:
+            path: {{ dir }}
+            type: DirectoryOrCreate
+{% endfor %}
+{% endif %}
+{% if cinder_cacert is defined and cinder_cacert != "" %}
+        - name: cinder-cacert
+          hostPath:
+            path: {{ kube_config_dir }}/cinder-cacert.pem
+            type: FileOrCreate
+{% endif %}
+{% if cinder_tolerations %}
+      tolerations:
+        {{ cinder_tolerations | to_nice_yaml(indent=2) | indent(width=8) }}
+{% endif %}

internal/resources/kubespray/project/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-poddisruptionbudget.yml.j2

@@ -0,0 +1,14 @@
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: cinder-csi-pdb
+  namespace: kube-system
+spec:
+{% if cinder_csi_controller_replicas is defined and cinder_csi_controller_replicas > 1 %}
+  minAvailable: 1
+{% else %}
+  minAvailable: 0
+{% endif %}
+  selector:
+    matchLabels:
+      app: csi-cinder-controllerplugin

internal/resources/kubespray/project/roles/kubernetes-apps/csi_driver/csi_crd/tasks/main.yml

@@ -0,0 +1,26 @@
+---
+- name: CSI CRD | Generate Manifests
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.file }}"
+    mode: "0644"
+  with_items:
+    - {name: volumesnapshotclasses, file: volumesnapshotclasses.yml}
+    - {name: volumesnapshotcontents, file: volumesnapshotcontents.yml}
+    - {name: volumesnapshots, file: volumesnapshots.yml}
+  register: csi_crd_manifests
+  when: inventory_hostname == groups['kube_control_plane'][0]
+
+- name: CSI CRD | Apply Manifests
+  kube:
+    kubectl: "{{ bin_dir }}/kubectl"
+    filename: "{{ kube_config_dir }}/{{ item.item.file }}"
+    state: "latest"
+    wait: true
+  with_items:
+    - "{{ csi_crd_manifests.results }}"
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - not item is skipped
+  loop_control:
+    label: "{{ item.item.file }}"

internal/resources/kubespray/project/roles/kubernetes-apps/csi_driver/csi_crd/templates/volumesnapshotclasses.yml.j2

@@ -0,0 +1,116 @@
+
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.4.0
+    api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419"
+  creationTimestamp: null
+  name: volumesnapshotclasses.snapshot.storage.k8s.io
+spec:
+  group: snapshot.storage.k8s.io
+  names:
+    kind: VolumeSnapshotClass
+    listKind: VolumeSnapshotClassList
+    plural: volumesnapshotclasses
+    singular: volumesnapshotclass
+  scope: Cluster
+  versions:
+  - additionalPrinterColumns:
+    - jsonPath: .driver
+      name: Driver
+      type: string
+    - description: Determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted.
+      jsonPath: .deletionPolicy
+      name: DeletionPolicy
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1
+    schema:
+      openAPIV3Schema:
+        description: VolumeSnapshotClass specifies parameters that a underlying storage system uses when creating a volume snapshot. A specific VolumeSnapshotClass is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses are non-namespaced
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          deletionPolicy:
+            description: deletionPolicy determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. Required.
+            enum:
+            - Delete
+            - Retain
+            type: string
+          driver:
+            description: driver is the name of the storage driver that handles this VolumeSnapshotClass. Required.
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          parameters:
+            additionalProperties:
+              type: string
+            description: parameters is a key-value map with storage driver specific parameters for creating snapshots. These values are opaque to Kubernetes.
+            type: object
+        required:
+        - deletionPolicy
+        - driver
+        type: object
+    served: true
+    storage: true
+    subresources: {}
+  - additionalPrinterColumns:
+    - jsonPath: .driver
+      name: Driver
+      type: string
+    - description: Determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted.
+      jsonPath: .deletionPolicy
+      name: DeletionPolicy
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta1
+    # This indicates the v1beta1 version of the custom resource is deprecated.
+    # API requests to this version receive a warning in the server response.
+    deprecated: true
+    # This overrides the default warning returned to clients making v1beta1 API requests.
+    deprecationWarning: "snapshot.storage.k8s.io/v1beta1 VolumeSnapshotClass is deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshotClass"
+    schema:
+      openAPIV3Schema:
+        description: VolumeSnapshotClass specifies parameters that a underlying storage system uses when creating a volume snapshot. A specific VolumeSnapshotClass is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses are non-namespaced
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          deletionPolicy:
+            description: deletionPolicy determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. Required.
+            enum:
+            - Delete
+            - Retain
+            type: string
+          driver:
+            description: driver is the name of the storage driver that handles this VolumeSnapshotClass. Required.
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          parameters:
+            additionalProperties:
+              type: string
+            description: parameters is a key-value map with storage driver specific parameters for creating snapshots. These values are opaque to Kubernetes.
+            type: object
+        required:
+        - deletionPolicy
+        - driver
+        type: object
+    served: true
+    storage: false
+    subresources: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []

internal/resources/kubespray/project/roles/kubernetes-apps/csi_driver/csi_crd/templates/volumesnapshotcontents.yml.j2

@@ -0,0 +1,305 @@
+
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.4.0
+    api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419"
+  creationTimestamp: null
+  name: volumesnapshotcontents.snapshot.storage.k8s.io
+spec:
+  group: snapshot.storage.k8s.io
+  names:
+    kind: VolumeSnapshotContent
+    listKind: VolumeSnapshotContentList
+    plural: volumesnapshotcontents
+    singular: volumesnapshotcontent
+  scope: Cluster
+  versions:
+  - additionalPrinterColumns:
+    - description: Indicates if the snapshot is ready to be used to restore a volume.
+      jsonPath: .status.readyToUse
+      name: ReadyToUse
+      type: boolean
+    - description: Represents the complete size of the snapshot in bytes
+      jsonPath: .status.restoreSize
+      name: RestoreSize
+      type: integer
+    - description: Determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted.
+      jsonPath: .spec.deletionPolicy
+      name: DeletionPolicy
+      type: string
+    - description: Name of the CSI driver used to create the physical snapshot on the underlying storage system.
+      jsonPath: .spec.driver
+      name: Driver
+      type: string
+    - description: Name of the VolumeSnapshotClass to which this snapshot belongs.
+      jsonPath: .spec.volumeSnapshotClassName
+      name: VolumeSnapshotClass
+      type: string
+    - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound.
+      jsonPath: .spec.volumeSnapshotRef.name
+      name: VolumeSnapshot
+      type: string
+    - description: Namespace of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound.
+      jsonPath: .spec.volumeSnapshotRef.namespace
+      name: VolumeSnapshotNamespace
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1
+    schema:
+      openAPIV3Schema:
+        description: VolumeSnapshotContent represents the actual "on-disk" snapshot object in the underlying storage system
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          spec:
+            description: spec defines properties of a VolumeSnapshotContent created by the underlying storage system. Required.
+            properties:
+              deletionPolicy:
+                description: deletionPolicy determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. For dynamically provisioned snapshots, this field will automatically be filled in by the CSI snapshotter sidecar with the "DeletionPolicy" field defined in the corresponding VolumeSnapshotClass. For pre-existing snapshots, users MUST specify this field when creating the  VolumeSnapshotContent object. Required.
+                enum:
+                - Delete
+                - Retain
+                type: string
+              driver:
+                description: driver is the name of the CSI driver used to create the physical snapshot on the underlying storage system. This MUST be the same as the name returned by the CSI GetPluginName() call for that driver. Required.
+                type: string
+              source:
+                description: source specifies whether the snapshot is (or should be) dynamically provisioned or already exists, and just requires a Kubernetes object representation. This field is immutable after creation. Required.
+                properties:
+                  snapshotHandle:
+                    description: snapshotHandle specifies the CSI "snapshot_id" of a pre-existing snapshot on the underlying storage system for which a Kubernetes object representation was (or should be) created. This field is immutable.
+                    type: string
+                  volumeHandle:
+                    description: volumeHandle specifies the CSI "volume_id" of the volume from which a snapshot should be dynamically taken from. This field is immutable.
+                    type: string
+                type: object
+                oneOf:
+                - required: ["snapshotHandle"]
+                - required: ["volumeHandle"]
+              volumeSnapshotClassName:
+                description: name of the VolumeSnapshotClass from which this snapshot was (or will be) created. Note that after provisioning, the VolumeSnapshotClass may be deleted or recreated with different set of values, and as such, should not be referenced post-snapshot creation.
+                type: string
+              volumeSnapshotRef:
+                description: volumeSnapshotRef specifies the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName field must reference to this VolumeSnapshotContent's name for the bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent object, name and namespace of the VolumeSnapshot object MUST be provided for binding to happen. This field is immutable after creation. Required.
+                properties:
+                  apiVersion:
+                    description: API version of the referent.
+                    type: string
+                  fieldPath:
+                    description: 'If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. For example, if the object reference is to a container within a pod, this would take on a value like: "spec.containers{name}" (where "name" refers to the name of the container that triggered the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. TODO: this design is not final and this field is subject to change in the future.'
+                    type: string
+                  kind:
+                    description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+                    type: string
+                  name:
+                    description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+                    type: string
+                  namespace:
+                    description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/'
+                    type: string
+                  resourceVersion:
+                    description: 'Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency'
+                    type: string
+                  uid:
+                    description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids'
+                    type: string
+                type: object
+            required:
+            - deletionPolicy
+            - driver
+            - source
+            - volumeSnapshotRef
+            type: object
+          status:
+            description: status represents the current information of a snapshot.
+            properties:
+              creationTime:
+                description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it indicates the creation time is unknown. The format of this field is a Unix nanoseconds time encoded as an int64. On Unix, the command `date +%s%N` returns the current time in nanoseconds since 1970-01-01 00:00:00 UTC.
+                format: int64
+                type: integer
+              error:
+                description: error is the last observed error during snapshot creation, if any. Upon success after retry, this error field will be cleared.
+                properties:
+                  message:
+                    description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.'
+                    type: string
+                  time:
+                    description: time is the timestamp when the error was encountered.
+                    format: date-time
+                    type: string
+                type: object
+              readyToUse:
+                description: readyToUse indicates if a snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown.
+                type: boolean
+              restoreSize:
+                description: restoreSize represents the complete size of the snapshot in bytes. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown.
+                format: int64
+                minimum: 0
+                type: integer
+              snapshotHandle:
+                description: snapshotHandle is the CSI "snapshot_id" of a snapshot on the underlying storage system. If not specified, it indicates that dynamic snapshot creation has either failed or it is still in progress.
+                type: string
+            type: object
+        required:
+        - spec
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+  - additionalPrinterColumns:
+    - description: Indicates if the snapshot is ready to be used to restore a volume.
+      jsonPath: .status.readyToUse
+      name: ReadyToUse
+      type: boolean
+    - description: Represents the complete size of the snapshot in bytes
+      jsonPath: .status.restoreSize
+      name: RestoreSize
+      type: integer
+    - description: Determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted.
+      jsonPath: .spec.deletionPolicy
+      name: DeletionPolicy
+      type: string
+    - description: Name of the CSI driver used to create the physical snapshot on the underlying storage system.
+      jsonPath: .spec.driver
+      name: Driver
+      type: string
+    - description: Name of the VolumeSnapshotClass to which this snapshot belongs.
+      jsonPath: .spec.volumeSnapshotClassName
+      name: VolumeSnapshotClass
+      type: string
+    - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound.
+      jsonPath: .spec.volumeSnapshotRef.name
+      name: VolumeSnapshot
+      type: string
+    - description: Namespace of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound.
+      jsonPath: .spec.volumeSnapshotRef.namespace
+      name: VolumeSnapshotNamespace
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta1
+    # This indicates the v1beta1 version of the custom resource is deprecated.
+    # API requests to this version receive a warning in the server response.
+    deprecated: true
+    # This overrides the default warning returned to clients making v1beta1 API requests.
+    deprecationWarning: "snapshot.storage.k8s.io/v1beta1 VolumeSnapshotContent is deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshotContent"
+    schema:
+      openAPIV3Schema:
+        description: VolumeSnapshotContent represents the actual "on-disk" snapshot object in the underlying storage system
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          spec:
+            description: spec defines properties of a VolumeSnapshotContent created by the underlying storage system. Required.
+            properties:
+              deletionPolicy:
+                description: deletionPolicy determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. For dynamically provisioned snapshots, this field will automatically be filled in by the CSI snapshotter sidecar with the "DeletionPolicy" field defined in the corresponding VolumeSnapshotClass. For pre-existing snapshots, users MUST specify this field when creating the  VolumeSnapshotContent object. Required.
+                enum:
+                - Delete
+                - Retain
+                type: string
+              driver:
+                description: driver is the name of the CSI driver used to create the physical snapshot on the underlying storage system. This MUST be the same as the name returned by the CSI GetPluginName() call for that driver. Required.
+                type: string
+              source:
+                description: source specifies whether the snapshot is (or should be) dynamically provisioned or already exists, and just requires a Kubernetes object representation. This field is immutable after creation. Required.
+                properties:
+                  snapshotHandle:
+                    description: snapshotHandle specifies the CSI "snapshot_id" of a pre-existing snapshot on the underlying storage system for which a Kubernetes object representation was (or should be) created. This field is immutable.
+                    type: string
+                  volumeHandle:
+                    description: volumeHandle specifies the CSI "volume_id" of the volume from which a snapshot should be dynamically taken from. This field is immutable.
+                    type: string
+                type: object
+              volumeSnapshotClassName:
+                description: name of the VolumeSnapshotClass from which this snapshot was (or will be) created. Note that after provisioning, the VolumeSnapshotClass may be deleted or recreated with different set of values, and as such, should not be referenced post-snapshot creation.
+                type: string
+              volumeSnapshotRef:
+                description: volumeSnapshotRef specifies the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName field must reference to this VolumeSnapshotContent's name for the bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent object, name and namespace of the VolumeSnapshot object MUST be provided for binding to happen. This field is immutable after creation. Required.
+                properties:
+                  apiVersion:
+                    description: API version of the referent.
+                    type: string
+                  fieldPath:
+                    description: 'If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. For example, if the object reference is to a container within a pod, this would take on a value like: "spec.containers{name}" (where "name" refers to the name of the container that triggered the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. TODO: this design is not final and this field is subject to change in the future.'
+                    type: string
+                  kind:
+                    description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+                    type: string
+                  name:
+                    description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+                    type: string
+                  namespace:
+                    description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/'
+                    type: string
+                  resourceVersion:
+                    description: 'Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency'
+                    type: string
+                  uid:
+                    description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids'
+                    type: string
+                type: object
+            required:
+            - deletionPolicy
+            - driver
+            - source
+            - volumeSnapshotRef
+            type: object
+          status:
+            description: status represents the current information of a snapshot.
+            properties:
+              creationTime:
+                description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it indicates the creation time is unknown. The format of this field is a Unix nanoseconds time encoded as an int64. On Unix, the command `date +%s%N` returns the current time in nanoseconds since 1970-01-01 00:00:00 UTC.
+                format: int64
+                type: integer
+              error:
+                description: error is the last observed error during snapshot creation, if any. Upon success after retry, this error field will be cleared.
+                properties:
+                  message:
+                    description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.'
+                    type: string
+                  time:
+                    description: time is the timestamp when the error was encountered.
+                    format: date-time
+                    type: string
+                type: object
+              readyToUse:
+                description: readyToUse indicates if a snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown.
+                type: boolean
+              restoreSize:
+                description: restoreSize represents the complete size of the snapshot in bytes. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown.
+                format: int64
+                minimum: 0
+                type: integer
+              snapshotHandle:
+                description: snapshotHandle is the CSI "snapshot_id" of a snapshot on the underlying storage system. If not specified, it indicates that dynamic snapshot creation has either failed or it is still in progress.
+                type: string
+            type: object
+        required:
+        - spec
+        type: object
+    served: true
+    storage: false
+    subresources:
+      status: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []

internal/resources/kubespray/project/roles/kubernetes-apps/csi_driver/csi_crd/templates/volumesnapshots.yml.j2

@@ -0,0 +1,231 @@
+
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.4.0
+    api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419"
+  creationTimestamp: null
+  name: volumesnapshots.snapshot.storage.k8s.io
+spec:
+  group: snapshot.storage.k8s.io
+  names:
+    kind: VolumeSnapshot
+    listKind: VolumeSnapshotList
+    plural: volumesnapshots
+    singular: volumesnapshot
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - description: Indicates if the snapshot is ready to be used to restore a volume.
+      jsonPath: .status.readyToUse
+      name: ReadyToUse
+      type: boolean
+    - description: If a new snapshot needs to be created, this contains the name of the source PVC from which this snapshot was (or will be) created.
+      jsonPath: .spec.source.persistentVolumeClaimName
+      name: SourcePVC
+      type: string
+    - description: If a snapshot already exists, this contains the name of the existing VolumeSnapshotContent object representing the existing snapshot.
+      jsonPath: .spec.source.volumeSnapshotContentName
+      name: SourceSnapshotContent
+      type: string
+    - description: Represents the minimum size of volume required to rehydrate from this snapshot.
+      jsonPath: .status.restoreSize
+      name: RestoreSize
+      type: string
+    - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot.
+      jsonPath: .spec.volumeSnapshotClassName
+      name: SnapshotClass
+      type: string
+    - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot object intends to bind to. Please note that verification of binding actually requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure both are pointing at each other. Binding MUST be verified prior to usage of this object.
+      jsonPath: .status.boundVolumeSnapshotContentName
+      name: SnapshotContent
+      type: string
+    - description: Timestamp when the point-in-time snapshot was taken by the underlying storage system.
+      jsonPath: .status.creationTime
+      name: CreationTime
+      type: date
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1
+    schema:
+      openAPIV3Schema:
+        description: VolumeSnapshot is a user's request for either creating a point-in-time snapshot of a persistent volume, or binding to a pre-existing snapshot.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          spec:
+            description: 'spec defines the desired characteristics of a snapshot requested by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots Required.'
+            properties:
+              source:
+                description: source specifies where a snapshot will be created from. This field is immutable after creation. Required.
+                properties:
+                  persistentVolumeClaimName:
+                    description: persistentVolumeClaimName specifies the name of the PersistentVolumeClaim object representing the volume from which a snapshot should be created. This PVC is assumed to be in the same namespace as the VolumeSnapshot object. This field should be set if the snapshot does not exists, and needs to be created. This field is immutable.
+                    type: string
+                  volumeSnapshotContentName:
+                    description: volumeSnapshotContentName specifies the name of a pre-existing VolumeSnapshotContent object representing an existing volume snapshot. This field should be set if the snapshot already exists and only needs a representation in Kubernetes. This field is immutable.
+                    type: string
+                type: object
+                oneOf:
+                - required: ["persistentVolumeClaimName"]
+                - required: ["volumeSnapshotContentName"]
+              volumeSnapshotClassName:
+                description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass requested by the VolumeSnapshot. VolumeSnapshotClassName may be left nil to indicate that the default SnapshotClass should be used. A given cluster may have multiple default Volume SnapshotClasses: one default per CSI Driver. If a VolumeSnapshot does not specify a SnapshotClass, VolumeSnapshotSource will be checked to figure out what the associated CSI Driver is, and the default VolumeSnapshotClass associated with that CSI Driver will be used. If more than one VolumeSnapshotClass exist for a given CSI Driver and more than one have been marked as default, CreateSnapshot will fail and generate an event. Empty string is not allowed for this field.'
+                type: string
+            required:
+            - source
+            type: object
+          status:
+            description: status represents the current information of a snapshot. Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object.
+            properties:
+              boundVolumeSnapshotContentName:
+                description: 'boundVolumeSnapshotContentName is the name of the VolumeSnapshotContent object to which this VolumeSnapshot object intends to bind to. If not specified, it indicates that the VolumeSnapshot object has not been successfully bound to a VolumeSnapshotContent object yet. NOTE: To avoid possible security issues, consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object.'
+                type: string
+              creationTime:
+                description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it may indicate that the creation time of the snapshot is unknown.
+                format: date-time
+                type: string
+              error:
+                description: error is the last observed error during snapshot creation, if any. This field could be helpful to upper level controllers(i.e., application controller) to decide whether they should continue on waiting for the snapshot to be created based on the type of error reported. The snapshot controller will keep retrying when an error occurrs during the snapshot creation. Upon success, this error field will be cleared.
+                properties:
+                  message:
+                    description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.'
+                    type: string
+                  time:
+                    description: time is the timestamp when the error was encountered.
+                    format: date-time
+                    type: string
+                type: object
+              readyToUse:
+                description: readyToUse indicates if the snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown.
+                type: boolean
+              restoreSize:
+                type: string
+                description: restoreSize represents the minimum size of volume required to create a volume from this snapshot. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown.
+                pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                x-kubernetes-int-or-string: true
+            type: object
+        required:
+        - spec
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+  - additionalPrinterColumns:
+    - description: Indicates if the snapshot is ready to be used to restore a volume.
+      jsonPath: .status.readyToUse
+      name: ReadyToUse
+      type: boolean
+    - description: If a new snapshot needs to be created, this contains the name of the source PVC from which this snapshot was (or will be) created.
+      jsonPath: .spec.source.persistentVolumeClaimName
+      name: SourcePVC
+      type: string
+    - description: If a snapshot already exists, this contains the name of the existing VolumeSnapshotContent object representing the existing snapshot.
+      jsonPath: .spec.source.volumeSnapshotContentName
+      name: SourceSnapshotContent
+      type: string
+    - description: Represents the minimum size of volume required to rehydrate from this snapshot.
+      jsonPath: .status.restoreSize
+      name: RestoreSize
+      type: string
+    - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot.
+      jsonPath: .spec.volumeSnapshotClassName
+      name: SnapshotClass
+      type: string
+    - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot object intends to bind to. Please note that verification of binding actually requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure both are pointing at each other. Binding MUST be verified prior to usage of this object.
+      jsonPath: .status.boundVolumeSnapshotContentName
+      name: SnapshotContent
+      type: string
+    - description: Timestamp when the point-in-time snapshot was taken by the underlying storage system.
+      jsonPath: .status.creationTime
+      name: CreationTime
+      type: date
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta1
+    # This indicates the v1beta1 version of the custom resource is deprecated.
+    # API requests to this version receive a warning in the server response.
+    deprecated: true
+    # This overrides the default warning returned to clients making v1beta1 API requests.
+    deprecationWarning: "snapshot.storage.k8s.io/v1beta1 VolumeSnapshot is deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshot"
+    schema:
+      openAPIV3Schema:
+        description: VolumeSnapshot is a user's request for either creating a point-in-time snapshot of a persistent volume, or binding to a pre-existing snapshot.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          spec:
+            description: 'spec defines the desired characteristics of a snapshot requested by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots Required.'
+            properties:
+              source:
+                description: source specifies where a snapshot will be created from. This field is immutable after creation. Required.
+                properties:
+                  persistentVolumeClaimName:
+                    description: persistentVolumeClaimName specifies the name of the PersistentVolumeClaim object representing the volume from which a snapshot should be created. This PVC is assumed to be in the same namespace as the VolumeSnapshot object. This field should be set if the snapshot does not exists, and needs to be created. This field is immutable.
+                    type: string
+                  volumeSnapshotContentName:
+                    description: volumeSnapshotContentName specifies the name of a pre-existing VolumeSnapshotContent object representing an existing volume snapshot. This field should be set if the snapshot already exists and only needs a representation in Kubernetes. This field is immutable.
+                    type: string
+                type: object
+              volumeSnapshotClassName:
+                description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass requested by the VolumeSnapshot. VolumeSnapshotClassName may be left nil to indicate that the default SnapshotClass should be used. A given cluster may have multiple default Volume SnapshotClasses: one default per CSI Driver. If a VolumeSnapshot does not specify a SnapshotClass, VolumeSnapshotSource will be checked to figure out what the associated CSI Driver is, and the default VolumeSnapshotClass associated with that CSI Driver will be used. If more than one VolumeSnapshotClass exist for a given CSI Driver and more than one have been marked as default, CreateSnapshot will fail and generate an event. Empty string is not allowed for this field.'
+                type: string
+            required:
+            - source
+            type: object
+          status:
+            description: status represents the current information of a snapshot. Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object.
+            properties:
+              boundVolumeSnapshotContentName:
+                description: 'boundVolumeSnapshotContentName is the name of the VolumeSnapshotContent object to which this VolumeSnapshot object intends to bind to. If not specified, it indicates that the VolumeSnapshot object has not been successfully bound to a VolumeSnapshotContent object yet. NOTE: To avoid possible security issues, consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object.'
+                type: string
+              creationTime:
+                description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it may indicate that the creation time of the snapshot is unknown.
+                format: date-time
+                type: string
+              error:
+                description: error is the last observed error during snapshot creation, if any. This field could be helpful to upper level controllers(i.e., application controller) to decide whether they should continue on waiting for the snapshot to be created based on the type of error reported. The snapshot controller will keep retrying when an error occurrs during the snapshot creation. Upon success, this error field will be cleared.
+                properties:
+                  message:
+                    description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.'
+                    type: string
+                  time:
+                    description: time is the timestamp when the error was encountered.
+                    format: date-time
+                    type: string
+                type: object
+              readyToUse:
+                description: readyToUse indicates if the snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown.
+                type: boolean
+              restoreSize:
+                type: string
+                description: restoreSize represents the minimum size of volume required to create a volume from this snapshot. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown.
+                pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                x-kubernetes-int-or-string: true
+            type: object
+        required:
+        - spec
+        type: object
+    served: true
+    storage: false
+    subresources:
+      status: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []

internal/resources/kubespray/project/roles/kubernetes-apps/csi_driver/gcp_pd/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+gcp_pd_csi_controller_replicas: 1

internal/resources/kubespray/project/roles/kubernetes-apps/csi_driver/gcp_pd/tasks/main.yml

@@ -0,0 +1,47 @@
+---
+- name: GCP PD CSI Driver | Check if cloud-sa.json exists
+  fail:
+    msg: "Credentials file cloud-sa.json is mandatory"
+  when: gcp_pd_csi_sa_cred_file is not defined or not gcp_pd_csi_sa_cred_file
+
+- name: GCP PD CSI Driver | Copy GCP credentials file
+  copy:
+    src: "{{ gcp_pd_csi_sa_cred_file }}"
+    dest: "{{ kube_config_dir }}/cloud-sa.json"
+    group: "{{ kube_cert_group }}"
+    mode: "0640"
+  when: inventory_hostname == groups['kube_control_plane'][0]
+
+- name: GCP PD CSI Driver | Get base64 cloud-sa.json
+  slurp:
+    src: "{{ kube_config_dir }}/cloud-sa.json"
+  register: gcp_cred_secret
+  when: inventory_hostname == groups['kube_control_plane'][0]
+
+- name: GCP PD CSI Driver | Generate Manifests
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.file }}"
+    mode: "0644"
+  with_items:
+    - {name: gcp-pd-csi-cred-secret, file: gcp-pd-csi-cred-secret.yml}
+    - {name: gcp-pd-csi-setup, file: gcp-pd-csi-setup.yml}
+    - {name: gcp-pd-csi-controller, file: gcp-pd-csi-controller.yml}
+    - {name: gcp-pd-csi-node, file: gcp-pd-csi-node.yml}
+    - {name: gcp-pd-csi-sc-regional, file: gcp-pd-csi-sc-regional.yml}
+    - {name: gcp-pd-csi-sc-zonal, file: gcp-pd-csi-sc-zonal.yml}
+  register: gcp_pd_csi_manifests
+  when: inventory_hostname == groups['kube_control_plane'][0]
+
+- name: GCP PD CSI Driver | Apply Manifests
+  kube:
+    kubectl: "{{ bin_dir }}/kubectl"
+    filename: "{{ kube_config_dir }}/{{ item.item.file }}"
+    state: "latest"
+  with_items:
+    - "{{ gcp_pd_csi_manifests.results }}"
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - not item is skipped
+  loop_control:
+    label: "{{ item.item.file }}"

internal/resources/kubespray/project/roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-controller.yml.j2

@@ -0,0 +1,165 @@
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: csi-gce-pd-controller
+  namespace: kube-system
+spec:
+  replicas: {{ gcp_pd_csi_controller_replicas }}
+  selector:
+    matchLabels:
+      app: gcp-compute-persistent-disk-csi-driver
+  template:
+    metadata:
+      labels:
+        app: gcp-compute-persistent-disk-csi-driver
+    spec:
+      # Host network must be used for interaction with Workload Identity in GKE
+      # since it replaces GCE Metadata Server with GKE Metadata Server. Remove
+      # this requirement when issue is resolved and before any exposure of
+      # metrics ports
+      hostNetwork: true
+      nodeSelector:
+        kubernetes.io/os: linux
+      serviceAccountName: csi-gce-pd-controller-sa
+      priorityClassName: csi-gce-pd-controller
+      containers:
+        - name: csi-provisioner
+          image: {{ csi_provisioner_image_repo }}:{{ csi_provisioner_image_tag }}
+          args:
+            - "--v=5"
+            - "--csi-address=/csi/csi.sock"
+            - "--feature-gates=Topology=true"
+            - "--http-endpoint=:22011"
+            - "--leader-election-namespace=$(PDCSI_NAMESPACE)"
+            - "--timeout=250s"
+            - "--extra-create-metadata"
+          # - "--run-controller-service=false"  # disable the controller service of the CSI driver
+          # - "--run-node-service=false"        # disable the node service of the CSI driver
+            - "--leader-election"
+            - "--default-fstype=ext4"
+            - "--controller-publish-readonly=true"
+          env:
+            - name: PDCSI_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+          ports:
+            - containerPort: 22011
+              name: http-endpoint
+              protocol: TCP
+          livenessProbe:
+            failureThreshold: 1
+            httpGet:
+              path: /healthz/leader-election
+              port: http-endpoint
+            initialDelaySeconds: 10
+            timeoutSeconds: 10
+            periodSeconds: 20
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+        - name: csi-attacher
+          image: {{ csi_attacher_image_repo }}:{{ csi_attacher_image_tag }}
+          args:
+            - "--v=5"
+            - "--csi-address=/csi/csi.sock"
+            - "--http-endpoint=:22012"
+            - "--leader-election"
+            - "--leader-election-namespace=$(PDCSI_NAMESPACE)"
+            - "--timeout=250s"
+          env:
+            - name: PDCSI_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+          ports:
+            - containerPort: 22012
+              name: http-endpoint
+              protocol: TCP
+          livenessProbe:
+            failureThreshold: 1
+            httpGet:
+              path: /healthz/leader-election
+              port: http-endpoint
+            initialDelaySeconds: 10
+            timeoutSeconds: 10
+            periodSeconds: 20
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+        - name: csi-resizer
+          image: {{ csi_resizer_image_repo }}:{{ csi_resizer_image_tag }}
+          args:
+            - "--v=5"
+            - "--csi-address=/csi/csi.sock"
+            - "--http-endpoint=:22013"
+            - "--leader-election"
+            - "--leader-election-namespace=$(PDCSI_NAMESPACE)"
+            - "--handle-volume-inuse-error=false"
+          env:
+            - name: PDCSI_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+          ports:
+            - containerPort: 22013
+              name: http-endpoint
+              protocol: TCP
+          livenessProbe:
+            failureThreshold: 1
+            httpGet:
+              path: /healthz/leader-election
+              port: http-endpoint
+            initialDelaySeconds: 10
+            timeoutSeconds: 10
+            periodSeconds: 20
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+        - name: csi-snapshotter
+          image: {{ csi_snapshotter_image_repo }}:{{ csi_snapshotter_image_tag }}
+          args:
+            - "--v=5"
+            - "--csi-address=/csi/csi.sock"
+            - "--metrics-address=:22014"
+            - "--leader-election"
+            - "--leader-election-namespace=$(PDCSI_NAMESPACE)"
+            - "--timeout=300s"
+          env:
+            - name: PDCSI_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+        - name: gce-pd-driver
+          # Don't change base image without changing pdImagePlaceholder in
+          # test/k8s-integration/main.go
+          image: {{ gcp_pd_csi_plugin_image_repo }}:{{ gcp_pd_csi_plugin_image_tag }}
+          args:
+            - "--v=5"
+            - "--endpoint=unix:/csi/csi.sock"
+          env:
+            - name: GOOGLE_APPLICATION_CREDENTIALS
+              value: "/etc/cloud-sa/cloud-sa.json"
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+            - name: cloud-sa-volume
+              readOnly: true
+              mountPath: "/etc/cloud-sa"
+      volumes:
+        - name: socket-dir
+          emptyDir: {}
+        - name: cloud-sa-volume
+          secret:
+            secretName: cloud-sa
+---
+apiVersion: storage.k8s.io/v1
+kind: CSIDriver
+metadata:
+  name: pd.csi.storage.gke.io
+spec:
+  attachRequired: true
+  podInfoOnMount: false

internal/resources/kubespray/project/roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-cred-secret.yml.j2

@@ -0,0 +1,8 @@
+---
+kind: Secret
+apiVersion: v1
+metadata:
+  name: cloud-sa
+  namespace: kube-system
+data:
+  cloud-sa.json: {{ gcp_cred_secret.content }}

internal/resources/kubespray/project/roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-node.yml.j2

@@ -0,0 +1,112 @@
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: csi-gce-pd-node
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      app: gcp-compute-persistent-disk-csi-driver
+  template:
+    metadata:
+      labels:
+        app: gcp-compute-persistent-disk-csi-driver
+    spec:
+      # Host network must be used for interaction with Workload Identity in GKE
+      # since it replaces GCE Metadata Server with GKE Metadata Server. Remove
+      # this requirement when issue is resolved and before any exposure of
+      # metrics ports.
+      hostNetwork: true
+      priorityClassName: csi-gce-pd-node
+      serviceAccountName: csi-gce-pd-node-sa
+      containers:
+        - name: csi-driver-registrar
+          image: {{ csi_node_driver_registrar_image_repo }}:{{ csi_node_driver_registrar_image_tag }}
+          args:
+            - "--v=5"
+            - "--csi-address=/csi/csi.sock"
+            - "--kubelet-registration-path=/var/lib/kubelet/plugins/pd.csi.storage.gke.io/csi.sock"
+          lifecycle:
+            preStop:
+              exec:
+                command: ["/bin/sh", "-c", "rm -rf /registration/pd.csi.storage.gke.io /registration/pd.csi.storage.gke.io-reg.sock"]
+          env:
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          volumeMounts:
+            - name: plugin-dir
+              mountPath: /csi
+            - name: registration-dir
+              mountPath: /registration
+        - name: gce-pd-driver
+          securityContext:
+            privileged: true
+          # Don't change base image without changing pdImagePlaceholder in
+          # test/k8s-integration/main.go
+          image: {{ gcp_pd_csi_plugin_image_repo }}:{{ gcp_pd_csi_plugin_image_tag }}
+          args:
+            - "--v=5"
+            - "--endpoint=unix:/csi/csi.sock"
+            - "--run-controller-service=false"
+          volumeMounts:
+            - name: kubelet-dir
+              mountPath: /var/lib/kubelet
+              mountPropagation: "Bidirectional"
+            - name: plugin-dir
+              mountPath: /csi
+            - name: device-dir
+              mountPath: /dev
+            # The following mounts are required to trigger host udevadm from
+            # container
+            - name: udev-rules-etc
+              mountPath: /etc/udev
+            - name: udev-rules-lib
+              mountPath: /lib/udev
+            - name: udev-socket
+              mountPath: /run/udev
+            - name: sys
+              mountPath: /sys
+      nodeSelector:
+        kubernetes.io/os: linux
+      volumes:
+        - name: registration-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins_registry/
+            type: Directory
+        - name: kubelet-dir
+          hostPath:
+            path: /var/lib/kubelet
+            type: Directory
+        - name: plugin-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins/pd.csi.storage.gke.io/
+            type: DirectoryOrCreate
+        - name: device-dir
+          hostPath:
+            path: /dev
+            type: Directory
+        # The following mounts are required to trigger host udevadm from
+        # container
+        - name: udev-rules-etc
+          hostPath:
+            path: /etc/udev
+            type: Directory
+        - name: udev-rules-lib
+          hostPath:
+            path: /lib/udev
+            type: Directory
+        - name: udev-socket
+          hostPath:
+            path: /run/udev
+            type: Directory
+        - name: sys
+          hostPath:
+            path: /sys
+            type: Directory
+      # https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/
+      # See "special case". This will tolerate everything. Node component should
+      # be scheduled on all nodes.
+      tolerations:
+      - operator: Exists

internal/resources/kubespray/project/roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-sc-regional.yml.j2

@@ -0,0 +1,9 @@
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: csi-gce-pd-regional
+provisioner: pd.csi.storage.gke.io
+parameters:
+  type: pd-balanced
+  replication-type: regional-pd
+volumeBindingMode: WaitForFirstConsumer

internal/resources/kubespray/project/roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-sc-zonal.yml.j2

@@ -0,0 +1,8 @@
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: csi-gce-pd-zonal
+provisioner: pd.csi.storage.gke.io
+parameters:
+  type: pd-balanced
+volumeBindingMode: WaitForFirstConsumer

internal/resources/kubespray/project/roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-setup.yml.j2

@@ -0,0 +1,241 @@
+##### Node Service Account, Roles, RoleBindings
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: csi-gce-pd-node-sa
+  namespace: kube-system
+
+---
+##### Controller Service Account, Roles, Rolebindings
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: csi-gce-pd-controller-sa
+  namespace: kube-system
+
+---
+# xref: https://github.com/kubernetes-csi/external-provisioner/blob/master/deploy/kubernetes/rbac.yaml
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-gce-pd-provisioner-role
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "create", "delete"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["list", "watch", "create", "update", "patch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["csinodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots"]
+    verbs: ["get", "list"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents"]
+    verbs: ["get", "list"]
+  # Access to volumeattachments is only needed when the CSI driver
+  # has the PUBLISH_UNPUBLISH_VOLUME controller capability.
+  # In that case, external-provisioner will watch volumeattachments
+  # to determine when it is safe to delete a volume.
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattachments"]
+    verbs: ["get", "list", "watch"]
+---
+
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-gce-pd-controller-provisioner-binding
+subjects:
+  - kind: ServiceAccount
+    name: csi-gce-pd-controller-sa
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: csi-gce-pd-provisioner-role
+  apiGroup: rbac.authorization.k8s.io
+
+---
+# xref: https://github.com/kubernetes-csi/external-attacher/blob/master/deploy/kubernetes/rbac.yaml
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-gce-pd-attacher-role
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["csinodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattachments"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattachments/status"]
+    verbs: ["patch"]
+---
+
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-gce-pd-controller-attacher-binding
+subjects:
+  - kind: ServiceAccount
+    name: csi-gce-pd-controller-sa
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: csi-gce-pd-attacher-role
+  apiGroup: rbac.authorization.k8s.io
+
+---
+
+apiVersion: scheduling.k8s.io/v1
+kind: PriorityClass
+metadata:
+  name: csi-gce-pd-controller
+value: 900000000
+globalDefault: false
+description: "This priority class should be used for the GCE PD CSI driver controller deployment only."
+
+---
+
+apiVersion: scheduling.k8s.io/v1
+kind: PriorityClass
+metadata:
+  name: csi-gce-pd-node
+value: 900001000
+globalDefault: false
+description: "This priority class should be used for the GCE PD CSI driver node deployment only."
+
+---
+
+# Resizer must be able to work with PVCs, PVs, SCs.
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-gce-pd-resizer-role
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims/status"]
+    verbs: ["update", "patch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["list", "watch", "create", "update", "patch"]
+  # If handle-volume-inuse-error=true, the pod specific rbac is needed
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list", "watch"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-gce-pd-resizer-binding
+subjects:
+  - kind: ServiceAccount
+    name: csi-gce-pd-controller-sa
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: csi-gce-pd-resizer-role
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: csi-gce-pd-controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: csi-gce-pd-node-deploy
+subjects:
+- kind: ServiceAccount
+  name: csi-gce-pd-controller-sa
+  namespace: kube-system
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: csi-gce-pd-snapshotter-role
+rules:
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["list", "watch", "create", "update", "patch"]
+  # Secrets resource omitted since GCE PD snapshots does not require them
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents"]
+    verbs: ["create", "get", "list", "watch", "update", "delete", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents/status"]
+    verbs: ["update", "patch"]
+---
+
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-gce-pd-controller-snapshotter-binding
+subjects:
+  - kind: ServiceAccount
+    name: csi-gce-pd-controller-sa
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: csi-gce-pd-snapshotter-role
+  apiGroup: rbac.authorization.k8s.io
+---
+
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-gce-pd-leaderelection-role
+  namespace: kube-system
+  labels:
+    k8s-app: gcp-compute-persistent-disk-csi-driver
+rules:
+- apiGroups: ["coordination.k8s.io"]
+  resources: ["leases"]
+  verbs: ["get", "watch", "list", "delete", "update", "create"]
+
+---
+
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-gce-pd-controller-leaderelection-binding
+  namespace: kube-system
+  labels:
+    k8s-app: gcp-compute-persistent-disk-csi-driver
+subjects:
+- kind: ServiceAccount
+  name: csi-gce-pd-controller-sa
+  namespace: kube-system
+roleRef:
+  kind: Role
+  name: csi-gce-pd-leaderelection-role
+  apiGroup: rbac.authorization.k8s.io

internal/resources/kubespray/project/roles/kubernetes-apps/csi_driver/upcloud/defaults/main.yml

@@ -0,0 +1,16 @@
+---
+upcloud_csi_controller_replicas: 1
+upcloud_csi_provisioner_image_tag: "v3.1.0"
+upcloud_csi_attacher_image_tag: "v3.4.0"
+upcloud_csi_resizer_image_tag: "v1.4.0"
+upcloud_csi_plugin_image_tag: "v1.1.0"
+upcloud_csi_node_image_tag: "v2.5.0"
+upcloud_username: "{{ lookup('env', 'UPCLOUD_USERNAME') }}"
+upcloud_password: "{{ lookup('env', 'UPCLOUD_PASSWORD') }}"
+upcloud_tolerations: []
+upcloud_csi_enable_volume_snapshot: false
+upcloud_csi_snapshot_controller_replicas: 2
+upcloud_csi_snapshotter_image_tag: "v4.2.1"
+upcloud_csi_snapshot_controller_image_tag: "v4.2.1"
+upcloud_csi_snapshot_validation_webhook_image_tag: "v4.2.1"
+upcloud_cacert: "{{ lookup('env', 'OS_CACERT') }}"

internal/resources/kubespray/project/roles/kubernetes-apps/csi_driver/upcloud/tasks/main.yml

@@ -0,0 +1,40 @@
+---
+- name: UpCloud CSI Driver | Check if UPCLOUD_USERNAME exists
+  fail:
+    msg: "UpCloud username is missing. Env UPCLOUD_USERNAME is mandatory"
+  when: upcloud_username is not defined or not upcloud_username
+
+- name: UpCloud CSI Driver | Check if UPCLOUD_PASSWORD exists
+  fail:
+    msg: "UpCloud password is missing. Env UPCLOUD_PASSWORD is mandatory"
+  when:
+    - upcloud_username is defined
+    - upcloud_username | length > 0
+    - upcloud_password is not defined or not upcloud_password
+
+- name: UpCloud CSI Driver | Generate Manifests
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.file }}"
+    mode: "0644"
+  with_items:
+    - {name: upcloud-csi-cred-secret, file: upcloud-csi-cred-secret.yml}
+    - {name: upcloud-csi-setup, file: upcloud-csi-setup.yml}
+    - {name: upcloud-csi-controller, file: upcloud-csi-controller.yml}
+    - {name: upcloud-csi-node, file: upcloud-csi-node.yml}
+    - {name: upcloud-csi-driver, file: upcloud-csi-driver.yml}
+  register: upcloud_csi_manifests
+  when: inventory_hostname == groups['kube_control_plane'][0]
+
+- name: UpCloud CSI Driver | Apply Manifests
+  kube:
+    kubectl: "{{ bin_dir }}/kubectl"
+    filename: "{{ kube_config_dir }}/{{ item.item.file }}"
+    state: "latest"
+  with_items:
+    - "{{ upcloud_csi_manifests.results }}"
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - not item is skipped
+  loop_control:
+    label: "{{ item.item.file }}"

internal/resources/kubespray/project/roles/kubernetes-apps/csi_driver/upcloud/templates/upcloud-csi-controller.yml.j2

@@ -0,0 +1,106 @@
+kind: StatefulSet
+apiVersion: apps/v1
+metadata:
+  name: csi-upcloud-controller
+  namespace: kube-system
+spec:
+  serviceName: "csi-upcloud"
+  replicas: {{ upcloud_csi_controller_replicas }}
+  selector:
+    matchLabels:
+      app: csi-upcloud-controller
+  template:
+    metadata:
+      labels:
+        app: csi-upcloud-controller
+        role: csi-upcloud
+    spec:
+      priorityClassName: system-cluster-critical
+      serviceAccount: csi-upcloud-controller-sa
+      containers:
+        - name: csi-provisioner
+          image: registry.k8s.io/sig-storage/csi-provisioner:{{ upcloud_csi_provisioner_image_tag }}
+          args:
+            - "--csi-address=$(ADDRESS)"
+            - "--v=5"
+            - "--timeout=600s"
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+          imagePullPolicy: "Always"
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+        - name: csi-attacher
+          image: registry.k8s.io/sig-storage/csi-attacher:{{ upcloud_csi_attacher_image_tag }}
+          args:
+            - "--v=5"
+            - "--csi-address=$(ADDRESS)"
+            - "--timeout=120s"
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+          imagePullPolicy: "Always"
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+        - name: csi-resizer
+          image: registry.k8s.io/sig-storage/csi-resizer:{{ upcloud_csi_resizer_image_tag }}
+          args:
+            - "--v=5"
+            - "--timeout=120s"
+            - "--csi-address=$(ADDRESS)"
+            - "--handle-volume-inuse-error=true"
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+          imagePullPolicy: "Always"
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+        - name: csi-snapshotter
+          image: k8s.gcr.io/sig-storage/csi-snapshotter:{{ upcloud_csi_snapshotter_image_tag }}
+          args:
+            - "--csi-address=$(ADDRESS)"
+            - "--v=5"
+            - "--timeout=600s"
+            - "--leader-election=false"
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+          imagePullPolicy: "Always"
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+        - name: csi-upcloud-plugin
+          image: ghcr.io/upcloudltd/upcloud-csi:{{ upcloud_csi_plugin_image_tag }}
+          args:
+            - "--endpoint=$(CSI_ENDPOINT)"
+            - "--nodehost=$(NODE_ID)"
+            - "--mode=monolith"
+          env:
+            - name: CSI_ENDPOINT
+              value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock
+            - name: UPCLOUD_USERNAME
+              valueFrom:
+                secretKeyRef:
+                  name: upcloud
+                  key: username
+            - name: UPCLOUD_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: upcloud
+                  key: password
+            - name: NODE_ID
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          imagePullPolicy: "Always"
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+      imagePullSecrets:
+        - name: upcloud
+      volumes:
+        - name: socket-dir
+          emptyDir: {}

internal/resources/kubespray/project/roles/kubernetes-apps/csi_driver/upcloud/templates/upcloud-csi-cred-secret.yml.j2

@@ -0,0 +1,9 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+    name: upcloud
+    namespace: kube-system
+stringData:
+    username: {{ upcloud_username }}
+    password: {{ upcloud_password }}

internal/resources/kubespray/project/roles/kubernetes-apps/csi_driver/upcloud/templates/upcloud-csi-driver.yml.j2

@@ -0,0 +1,8 @@
+apiVersion: storage.k8s.io/v1
+kind: CSIDriver
+metadata:
+  name: storage.csi.upcloud.com
+spec:
+  attachRequired: true
+  podInfoOnMount: true
+  fsGroupPolicy: File