kube-forge v2: embede python with ansible (kubespray) and cobra as cli building tool

internal/resources/kubespray/project/roles/kubernetes-apps/external_provisioner/local_path_provisioner/defaults/main.yml

@@ -0,0 +1,10 @@
+---
+local_path_provisioner_enabled: false
+local_path_provisioner_namespace: "local-path-storage"
+local_path_provisioner_storage_class: "local-path"
+local_path_provisioner_reclaim_policy: Delete
+local_path_provisioner_claim_root: /opt/local-path-provisioner/
+local_path_provisioner_is_default_storageclass: "true"
+local_path_provisioner_debug: false
+local_path_provisioner_helper_image_repo: "busybox"
+local_path_provisioner_helper_image_tag: "latest"

internal/resources/kubespray/project/roles/kubernetes-apps/external_provisioner/local_path_provisioner/tasks/main.yml

@@ -0,0 +1,47 @@
+---
+- name: Local Path Provisioner | Create addon dir
+  file:
+    path: "{{ kube_config_dir }}/addons/local_path_provisioner"
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+
+- name: Local Path Provisioner | Create claim root dir
+  file:
+    path: "{{ local_path_provisioner_claim_root }}"
+    state: directory
+    mode: "0755"
+
+- name: Local Path Provisioner | Render Template
+  set_fact:
+    local_path_provisioner_templates:
+      - { name: local-path-storage-ns, file: local-path-storage-ns.yml, type: ns }
+      - { name: local-path-storage-sa, file: local-path-storage-sa.yml, type: sa }
+      - { name: local-path-storage-cr, file: local-path-storage-cr.yml, type: cr }
+      - { name: local-path-storage-clusterrolebinding, file: local-path-storage-clusterrolebinding.yml, type: clusterrolebinding }
+      - { name: local-path-storage-cm, file: local-path-storage-cm.yml, type: cm }
+      - { name: local-path-storage-deployment, file: local-path-storage-deployment.yml, type: deployment }
+      - { name: local-path-storage-sc, file: local-path-storage-sc.yml, type: sc }
+
+- name: Local Path Provisioner | Create manifests
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/addons/local_path_provisioner/{{ item.file }}"
+    mode: "0644"
+  with_items: "{{ local_path_provisioner_templates }}"
+  register: local_path_provisioner_manifests
+  when: inventory_hostname == groups['kube_control_plane'][0]
+
+- name: Local Path Provisioner | Apply manifests
+  kube:
+    name: "{{ item.item.name }}"
+    namespace: "{{ local_path_provisioner_namespace }}"
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: "{{ item.item.type }}"
+    filename: "{{ kube_config_dir }}/addons/local_path_provisioner/{{ item.item.file }}"
+    state: "latest"
+  with_items: "{{ local_path_provisioner_manifests.results }}"
+  when: inventory_hostname == groups['kube_control_plane'][0]

internal/resources/kubespray/project/roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-clusterrolebinding.yml.j2

@@ -0,0 +1,13 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: local-path-provisioner-bind
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: local-path-provisioner-role
+subjects:
+  - kind: ServiceAccount
+    name: local-path-provisioner-service-account
+    namespace: {{ local_path_provisioner_namespace }}

internal/resources/kubespray/project/roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-cm.yml.j2

@@ -0,0 +1,34 @@
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: local-path-config
+  namespace: {{ local_path_provisioner_namespace }}
+data:
+  config.json: |-
+    {
+            "nodePathMap":[
+            {
+                    "node":"DEFAULT_PATH_FOR_NON_LISTED_NODES",
+                    "paths":["{{ local_path_provisioner_claim_root }}"]
+            }
+            ]
+    }
+  setup: |-
+    #!/bin/sh
+    set -eu
+    mkdir -m 0777 -p "$VOL_DIR"
+  teardown: |-
+    #!/bin/sh
+    set -eu
+    rm -rf "$VOL_DIR"
+  helperPod.yaml: |-
+    apiVersion: v1
+    kind: Pod
+    metadata:
+      name: helper-pod
+    spec:
+      containers:
+      - name: helper-pod
+        image: "{{ local_path_provisioner_helper_image_repo }}:{{ local_path_provisioner_helper_image_tag }}"
+        imagePullPolicy: IfNotPresent

internal/resources/kubespray/project/roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-cr.yml.j2

@@ -0,0 +1,18 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: local-path-provisioner-role
+rules:
+  - apiGroups: [ "" ]
+    resources: [ "nodes", "persistentvolumeclaims", "configmaps" ]
+    verbs: [ "get", "list", "watch" ]
+  - apiGroups: [ "" ]
+    resources: [ "endpoints", "persistentvolumes", "pods" ]
+    verbs: [ "*" ]
+  - apiGroups: [ "" ]
+    resources: [ "events" ]
+    verbs: [ "create", "patch" ]
+  - apiGroups: [ "storage.k8s.io" ]
+    resources: [ "storageclasses" ]
+    verbs: [ "get", "list", "watch" ]

internal/resources/kubespray/project/roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-deployment.yml.j2

@@ -0,0 +1,41 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: local-path-provisioner
+  namespace: {{ local_path_provisioner_namespace }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: local-path-provisioner
+  template:
+    metadata:
+      labels:
+        app: local-path-provisioner
+    spec:
+      serviceAccountName: local-path-provisioner-service-account
+      containers:
+      - name: local-path-provisioner
+        image: {{ local_path_provisioner_image_repo }}:{{ local_path_provisioner_image_tag }}
+        imagePullPolicy: {{ k8s_image_pull_policy }}
+        command:
+        - local-path-provisioner
+        - start
+        - --config
+        - /etc/config/config.json
+{% if local_path_provisioner_debug | default(false) %}
+        - --debug
+{% endif %}
+        volumeMounts:
+        - name: config-volume
+          mountPath: /etc/config/
+        env:
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+      volumes:
+        - name: config-volume
+          configMap:
+            name: local-path-config

internal/resources/kubespray/project/roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-ns.yml.j2

@@ -0,0 +1,5 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: {{ local_path_provisioner_namespace }}

internal/resources/kubespray/project/roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-sa.yml.j2

@@ -0,0 +1,6 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: local-path-provisioner-service-account
+  namespace: {{ local_path_provisioner_namespace }}

internal/resources/kubespray/project/roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-sc.yml.j2

@@ -0,0 +1,10 @@
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: {{ local_path_provisioner_storage_class }}
+  annotations:
+    storageclass.kubernetes.io/is-default-class: "{{ local_path_provisioner_is_default_storageclass }}"
+provisioner: rancher.io/local-path
+volumeBindingMode: WaitForFirstConsumer
+reclaimPolicy: {{ local_path_provisioner_reclaim_policy }}

internal/resources/kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/defaults/main.yml

@@ -0,0 +1,21 @@
+---
+local_volume_provisioner_namespace: "kube-system"
+# List of node labels to be copied to the PVs created by the provisioner
+local_volume_provisioner_nodelabels: []
+#   - kubernetes.io/hostname
+#   - topology.kubernetes.io/region
+#   - topology.kubernetes.io/zone
+local_volume_provisioner_tolerations: []
+local_volume_provisioner_use_node_name_only: false
+# Leverages Ansible's string to Python datatype casting. Otherwise the dict_key isn't substituted.
+# see https://github.com/ansible/ansible/issues/17324
+local_volume_provisioner_storage_classes: |
+  {
+    "{{ local_volume_provisioner_storage_class | default('local-storage') }}": {
+      "host_dir": "{{ local_volume_provisioner_base_dir | default('/mnt/disks') }}",
+      "mount_dir": "{{ local_volume_provisioner_mount_dir | default('/mnt/disks') }}",
+      "volume_mode": "Filesystem",
+      "fs_type": "ext4"
+    }
+  }
+local_volume_provisioner_log_level: 2

internal/resources/kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/tasks/basedirs.yml

@@ -0,0 +1,12 @@
+---
+# include to workaround mitogen issue
+# https://github.com/dw/mitogen/issues/663
+
+- name: "Local Volume Provisioner | Ensure base dir {{ delegate_host_base_dir.1 }} is created on {{ delegate_host_base_dir.0 }}"
+  file:
+    path: "{{ local_volume_provisioner_storage_classes[delegate_host_base_dir.1].host_dir }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "{{ local_volume_provisioner_directory_mode }}"
+  delegate_to: "{{ delegate_host_base_dir.0 }}"

internal/resources/kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/tasks/main.yml

@@ -0,0 +1,48 @@
+---
+
+- name: Local Volume Provisioner | Ensure base dir is created on all hosts
+  include_tasks: basedirs.yml
+  loop_control:
+    loop_var: delegate_host_base_dir
+  loop: "{{ groups['k8s_cluster'] | product(local_volume_provisioner_storage_classes.keys()) | list }}"
+
+- name: Local Volume Provisioner | Create addon dir
+  file:
+    path: "{{ kube_config_dir }}/addons/local_volume_provisioner"
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+
+- name: Local Volume Provisioner | Templates list
+  set_fact:
+    local_volume_provisioner_templates:
+      - { name: local-volume-provisioner-ns, file: local-volume-provisioner-ns.yml, type: ns }
+      - { name: local-volume-provisioner-sa, file: local-volume-provisioner-sa.yml, type: sa }
+      - { name: local-volume-provisioner-clusterrole, file: local-volume-provisioner-clusterrole.yml, type: clusterrole }
+      - { name: local-volume-provisioner-clusterrolebinding, file: local-volume-provisioner-clusterrolebinding.yml, type: clusterrolebinding }
+      - { name: local-volume-provisioner-cm, file: local-volume-provisioner-cm.yml, type: cm }
+      - { name: local-volume-provisioner-ds, file: local-volume-provisioner-ds.yml, type: ds }
+      - { name: local-volume-provisioner-sc, file: local-volume-provisioner-sc.yml, type: sc }
+
+- name: Local Volume Provisioner | Create manifests
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/addons/local_volume_provisioner/{{ item.file }}"
+    mode: "0644"
+  with_items: "{{ local_volume_provisioner_templates }}"
+  register: local_volume_provisioner_manifests
+  when: inventory_hostname == groups['kube_control_plane'][0]
+
+- name: Local Volume Provisioner | Apply manifests
+  kube:
+    name: "{{ item.item.name }}"
+    namespace: "{{ local_volume_provisioner_namespace }}"
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: "{{ item.item.type }}"
+    filename: "{{ kube_config_dir }}/addons/local_volume_provisioner/{{ item.item.file }}"
+    state: "latest"
+  with_items: "{{ local_volume_provisioner_manifests.results }}"
+  when: inventory_hostname == groups['kube_control_plane'][0]
+  loop_control:
+    label: "{{ item.item.file }}"

internal/resources/kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-clusterrole.yml.j2

@@ -0,0 +1,22 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: local-volume-provisioner-node-clusterrole
+  namespace: {{ local_volume_provisioner_namespace }}
+rules:
+- apiGroups: [""]
+  resources: ["persistentvolumes"]
+  verbs: ["get", "list", "watch", "create", "delete"]
+- apiGroups: ["storage.k8s.io"]
+  resources: ["storageclasses"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: [""]
+  resources: ["events"]
+  verbs: ["watch"]
+- apiGroups: ["", "events.k8s.io"]
+  resources: ["events"]
+  verbs: ["create", "update", "patch"]
+- apiGroups: [""]
+  resources: ["nodes"]
+  verbs: ["get"]

internal/resources/kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-clusterrolebinding.yml.j2

@@ -0,0 +1,14 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: local-volume-provisioner-system-node
+  namespace: {{ local_volume_provisioner_namespace }}
+subjects:
+- kind: ServiceAccount
+  name: local-volume-provisioner
+  namespace: {{ local_volume_provisioner_namespace }}
+roleRef:
+  kind: ClusterRole
+  name: local-volume-provisioner-node-clusterrole
+  apiGroup: rbac.authorization.k8s.io

internal/resources/kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-cm.yml.j2

@@ -0,0 +1,33 @@
+# Macro to convert camelCase dictionary keys to snake_case keys
+{% macro convert_keys(mydict) -%}
+  {% for key in mydict.keys() | list -%}
+    {% set key_split = key.split('_') -%}
+    {% set new_key = key_split[0] + key_split[1:] | map('capitalize') | join -%}
+    {% set value = mydict.pop(key) -%}
+    {{ mydict.__setitem__(new_key, value) -}}
+    {{ convert_keys(value) if value is mapping else None -}}
+  {% endfor -%}
+{% endmacro -%}
+
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: local-volume-provisioner
+  namespace: {{ local_volume_provisioner_namespace }}
+data:
+{% if local_volume_provisioner_nodelabels | length > 0 %}
+  nodeLabelsForPV: |
+{% for nodelabel in local_volume_provisioner_nodelabels %}
+    - {{ nodelabel }}
+{% endfor %}
+{% endif %}
+{% if local_volume_provisioner_use_node_name_only %}
+  useNodeNameOnly: "true"
+{% endif %}
+  storageClassMap: |
+{% for class_name, storage_class in local_volume_provisioner_storage_classes.items() %}
+    {{ class_name }}:
+      {{- convert_keys(storage_class) }}
+      {{ storage_class | to_nice_yaml(indent=2) | indent(6) }}
+{%- endfor %}

internal/resources/kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-ds.yml.j2

@@ -0,0 +1,68 @@
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: local-volume-provisioner
+  namespace: {{ local_volume_provisioner_namespace }}
+  labels:
+    k8s-app: local-volume-provisioner
+    version: {{ local_volume_provisioner_image_tag }}
+spec:
+  selector:
+    matchLabels:
+      k8s-app: local-volume-provisioner
+      version: {{ local_volume_provisioner_image_tag }}
+  template:
+    metadata:
+      labels:
+        k8s-app: local-volume-provisioner
+        version: {{ local_volume_provisioner_image_tag }}
+    spec:
+      priorityClassName: {% if local_volume_provisioner_namespace == 'kube-system' %}system-node-critical{% else %}k8s-cluster-critical{% endif %}{{ '' }}
+      serviceAccountName: local-volume-provisioner
+      nodeSelector:
+        kubernetes.io/os: linux
+{% if local_volume_provisioner_tolerations %}
+      tolerations:
+        {{ local_volume_provisioner_tolerations | to_nice_yaml(indent=2) | indent(width=8) }}
+{% endif %}
+      containers:
+        - name: provisioner
+          image: {{ local_volume_provisioner_image_repo }}:{{ local_volume_provisioner_image_tag }}
+          imagePullPolicy: {{ k8s_image_pull_policy }}
+          args:
+          - "-v={{ local_volume_provisioner_log_level }}"
+          securityContext:
+            privileged: true
+          env:
+          - name: MY_NODE_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: spec.nodeName
+          - name: MY_NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
+          volumeMounts:
+            - name: local-volume-provisioner
+              mountPath: /etc/provisioner/config
+              readOnly: true
+            - mountPath: /dev
+              name: provisioner-dev
+{% for class_name, class_config in local_volume_provisioner_storage_classes.items() %}
+            - name: local-volume-provisioner-hostpath-{{ class_name }}
+              mountPath: {{ class_config.mount_dir }}
+              mountPropagation: "HostToContainer"
+{% endfor %}
+      volumes:
+        - name: local-volume-provisioner
+          configMap:
+            name: local-volume-provisioner
+        - name: provisioner-dev
+          hostPath:
+            path: /dev
+{% for class_name, class_config in local_volume_provisioner_storage_classes.items() %}
+        - name: local-volume-provisioner-hostpath-{{ class_name }}
+          hostPath:
+            path: {{ class_config.host_dir }}
+{% endfor %}

internal/resources/kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-ns.yml.j2

@@ -0,0 +1,7 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: {{ local_volume_provisioner_namespace }}
+  labels:
+    name: {{ local_volume_provisioner_namespace }}

internal/resources/kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-sa.yml.j2

@@ -0,0 +1,6 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: local-volume-provisioner
+  namespace: {{ local_volume_provisioner_namespace }}

internal/resources/kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-sc.yml.j2

@@ -0,0 +1,12 @@
+{% for class_name, class_config in local_volume_provisioner_storage_classes.items() %}
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: {{ class_name }}
+provisioner: kubernetes.io/no-provisioner
+volumeBindingMode: WaitForFirstConsumer
+{% if class_config.reclaim_policy is defined %}
+reclaimPolicy: {{ class_config.reclaim_policy }}
+{% endif %}
+{% endfor %}

internal/resources/kubespray/project/roles/kubernetes-apps/external_provisioner/meta/main.yml

@@ -0,0 +1,17 @@
+---
+dependencies:
+  - role: kubernetes-apps/external_provisioner/local_volume_provisioner
+    when:
+      - local_volume_provisioner_enabled
+      - inventory_hostname == groups['kube_control_plane'][0]
+    tags:
+      - apps
+      - local-volume-provisioner
+      - external-provisioner
+
+  - role: kubernetes-apps/external_provisioner/local_path_provisioner
+    when: local_path_provisioner_enabled
+    tags:
+      - apps
+      - local-path-provisioner
+      - external-provisioner