move cert-manager and ingress-nginx from kubespray to separate modules, remove reset action and update observability template

cmd/main/main.go

@@ -8,21 +8,17 @@ import (
 	"os"
 )
 
-func parseArgs() (string, string, string, string) {
+func parseArgs() (string, string, string) {
-	var password string
+	var password, configPath, dataDir string
-	var configPath string
-	var dataDir string
-	var tags string
 	flag.StringVar(&password, "p", "", "Password to access hosts")
 	flag.StringVar(&configPath, "c", "/etc/kube-forge/config.yaml", "Path to config file")
 	flag.StringVar(&dataDir, "d", "/var/lib/kube-forge", "Path to data dir")
-	flag.StringVar(&tags, "t", "", "List of tags to apply (ex: ingress,dns)")
 	flag.Parse()
-	return password, configPath, dataDir, tags
+	return password, configPath, dataDir
 }
 
 func main() {
-	password, configPath, dataDir, tags := parseArgs()
+	password, configPath, dataDir := parseArgs()
 	config := config.CreateConfig(configPath, dataDir, password)
 
 	repositories, releases := templates.GetHelmAppsConfigData()
@@ -34,12 +30,14 @@ func main() {
 	for _, cmd := range os.Args {
 		switch cmd {
 		case "apply":
-			kubespray.InstallCluster(tags)
+			kubespray.InstallCluster("")
 			kubespray.CopyK8SAdminConfig("k8s-admin.conf")
-		case "reset":
+		case "apply-modules":
-			kubespray.ResetCluster()
+			kubespray.InstallCluster("helm-apps")
 		case "upgrade":
-			kubespray.UpgradeCluster(tags)
+			kubespray.UpgradeCluster("")
+		case "scale":
+			kubespray.ScaleCluster()
 		}
 	}
 }

config.yaml

@@ -96,32 +96,34 @@ modules:
           retention: 7d
 
       alert_manager:
-        enabled: false
+        enabled: true
+        # additionalMessageTemplates: {}
 
-        # route:
+        route:
-        #   receiver: kvazaric_notifications
+          receiver: kvazaric_notifications
-        #   group_wait: 1s
+          group_wait: 1s
-        #   repeat_interval: 180m
+          repeat_interval: 180m
+
+          routes:
+            - receiver: "kvazaric_notifications"
+              group_by: ["alertname"]
+              group_wait: 1s
+              repeat_interval: 8737h
+              matchers:
+                - fireOnce=true
+        receivers:
+          - name: kvazaric_notifications
+            telegram_configs:
+              - send_resolved: true
+                api_url: https://api.telegram.org
+                bot_token: 6364937365:AAHcJKc-McN11gBWuQqoM87zr18eK0VKX9I
+                chat_id: -968770779
+                parse_mode: HTML
+                message: '{% raw %}{{ "{{" }} template "default-message" . {{ "}}" }}{% endraw %}'
+                http_config:
+                  follow_redirects: true
+                  enable_http2: false
 
-        #   routes:
-        #     - receiver: "kvazaric_notifications"
-        #       group_by: ["alertname"]
-        #       group_wait: 1s
-        #       repeat_interval: 8737h
-        #       matchers:
-        #         - fireOnce=true
-        # receivers:
-        #   - name: kvazaric_notifications
-        #     telegram_configs:
-        #       - send_resolved: true
-        #         api_url: https://api.telegram.org
-        #         bot_token: 6364937365:AAHcJKc-McN11gBWuQqoM87zr18eK0VKX9I
-        #         chat_id: -968770779
-        #         parse_mode: HTML
-        #         message: '{{ template "telegram-message" . }}'
-        #         http_config:
-        #           follow_redirects: true
-        #           enable_http2: false
       blackbox:
         enabled: true
         additional_modules: |
@@ -245,7 +247,8 @@ modules:
 
   additional:
     cert_manager:
-      install: true
+      enabled: true
+      ha_enabled: false
       dns_servers:
         - "1.1.1.1"
         - "8.8.8.8"
@@ -253,9 +256,7 @@ modules:
 
     ingress:
       type: nginx # nginx
-      install: true
+      enabled: true
-      insecure_port: 80
-      secure_port: 443
       ingress_class_name: nginx
       host_network: true
 

kubespray/inventory/group_vars/all.yml

@@ -127,3 +127,8 @@ unsafe_show_logs: false
 
 ## If enabled it will allow kubespray to attempt setup even if the distribution is not supported. For unsupported distributions this can lead to unexpected failures in some cases.
 allow_unsupported_distribution_setup: false
+
+## Containerd settings
+# containerd_metadata_root_dir: /app/lib/containerd
+# The state directory for containerd
+# containerd_state_dir: /app/run/containerd

kubespray/project/cluster.yml

@@ -90,11 +90,7 @@
     - { role: kubernetes-apps/ingress_controller, tags: ingress-controller }
     - { role: kubernetes-apps/external_provisioner, tags: external-provisioner }
     - { role: kubernetes-apps, tags: apps }
-    - {
+    - { role: helm-apps, tags: helm-apps }
-        role: helm-apps,
-        when: "releases is defined and repositories is defined",
-        tags: helm-apps,
-      }
 
 - name: Apply resolv.conf changes now that cluster DNS is up
   hosts: k8s_cluster

pkg/config/modules_additional.go

@@ -2,18 +2,32 @@ package config
 
 type Additional struct {
 	CertManager struct {
-		Install      bool     `yaml:"install"`
+		Enabled      bool     `yaml:"enabled"`
+		HaEnabled    bool     `yaml:"ha_enabled"`
 		DnsServers   []string `yaml:"dns_servers" env-default:"[8.8.8.8,1.1.1.1]"`
 		AccountEmail string   `yaml:"account_email"`
+		Controller   struct {
+			Image string `yaml:"image" env-default:"quay.io/jetstack/cert-manager-controller"`
+			Tag   string `yaml:"tag" env-default:"v1.14.5"`
+		} `yaml:"controller"`
 	} `yaml:"cert_manager"`
 
 	Ingress struct {
 		Type                    string `yaml:"type" env-default:"nginx"`
-		Install          bool   `yaml:"install" env-default:"false"`
+		Enabled                 bool   `yaml:"enabled"`
-		InsecurePort     int    `yaml:"insecure_port" env-default:"80"`
-		SecurePort       int    `yaml:"secure_port" env-default:"443"`
 		IngressClassName        string `yaml:"ingress_class_name" env-default:"nginx"`
-		HostNetwork      string `yaml:"host_network" env-default:"false"`
+		HostNetwork             bool   `yaml:"host_network"`
+		EnableAdmissionWebhooks bool   `yaml:"enable_admission_webhooks"`
+		Nginx                   struct {
+			Controller struct {
+				Image string `yaml:"image" env-default:"registry.k8s.io/ingress-nginx/controller"`
+				Tag   string `yaml:"tag" env-default:"v1.9.4"`
+			} `yaml:"controller"`
+			DefaultBackend struct {
+				Image string `yaml:"image" env-default:"registry.k8s.io/defaultbackend-amd64"`
+				Tag   string `yaml:"tag" env-default:"1.5"`
+			} `yaml:"default_backend"`
+		} `yaml:"nginx"`
 	} `yaml:"ingress"`
 
 	LoadBalancer struct {

pkg/config/modules_observability.go

@@ -107,6 +107,7 @@ type Monitoring struct {
 		Enabled                    bool        `yaml:"enabled"`
 		Image                      string      `yaml:"image" env-default:"prom/alertmanager"`
 		Tag                        string      `yaml:"tag" env-default:"v0.26.0"`
+		AdditionalMessageTemplates interface{} `yaml:"additionalMessageTemplates"`
 		Route                      interface{} `yaml:"route"`
 		Receivers                  interface{} `yaml:"receivers"`
 	} `yaml:"alert_manager"`
@@ -114,7 +115,7 @@ type Monitoring struct {
 		Enabled           bool   `yaml:"enabled"`
 		Image             string `yaml:"image" env-default:"prom/blackbox-exporter"`
 		Tag               string `yaml:"tag" env-default:"v0.24.0"`
-		additionalModules string `yaml:"routes" env-default:""`
+		AdditionalModules string `yaml:"routes" env-default:""`
 	} `yaml:"blackbox"`
 	KubeState struct {
 		Enabled bool   `yaml:"enabled"`

pkg/config/orchestrator.go

@@ -21,7 +21,7 @@ type Orchestrator struct {
 	PingAccessIp             bool   `yaml:"ping_access_ip" env-default:"true"`
 	AutoRenewCertificates    bool   `yaml:"auto_renew_certificates" env-default:"false"`
 	EventTtl                 string `yaml:"event_ttl" env-default:"1h0m0s"`
-	PodSecurotyPolicyEnabled bool   `yaml:"pod_security_policy_enabled" env-default:"false"`
+	PodSecurityPolicyEnabled bool   `yaml:"pod_security_policy_enabled" env-default:"false"`
 
 	Network struct {
 		Plugin           string `yaml:"plugin" env-default:"calico"`

pkg/kubespray/cluster.go

@@ -8,6 +8,6 @@ func UpgradeCluster(tags string) {
 	runPlaybook("kubespray/project/upgrade_cluster.yml", tags)
 }
 
-func ResetCluster() {
+func ScaleCluster() {
-	runPlaybook("kubespray/project/reset.yml", "reset")
+	runPlaybook("kubespray/project/scale.yml", "")
 }

pkg/kubespray/utility.go

@@ -16,7 +16,7 @@ func getPlaybookParameters(tags string) playbook.AnsiblePlaybookOptions {
 	cfg := config.GetConfig()
 
 	ansiblePlaybookOptions := playbook.AnsiblePlaybookOptions{
-		Inventory: "kubespray/inventory/hosts",
+		Inventory: filepath.Join(cfg.DataDir, "kubespray/inventory/hosts"),
 		Tags:      tags,
 		User:      cfg.Credentials.User,
 		Become:    true,

pkg/resources/templates/helm-apps/releases/additional-modules/ingress-nginx.yml.tmpl

@@ -0,0 +1,908 @@
+- name: ingress-nginx
+  namespace: ingress-nginx
+  create_namespace: true
+  chart_ref: kube-forge/ingress-nginx
+  chart_version: 4.10.1
+  {{- if and .Modules.Additional.Ingress.Enabled (eq .Modules.Additional.Ingress.Type "nginx") }}
+  release_state: "present"
+  {{- else }}
+  release_state: "absent"
+  {{- end }}
+  values:
+    commonLabels: {}
+    controller:
+      name: controller
+      enableAnnotationValidations: false
+      image:
+        ## Keep false as default for now!
+        chroot: false
+        ## for backwards compatibility consider setting the full image url via the repository value below
+        ## use *either* current default registry/image or repository format or installing chart by providing the values.yaml will fail
+        repository: {{ .Modules.Additional.Ingress.Nginx.Controller.Image }}
+        tag: "{{ .Modules.Additional.Ingress.Nginx.Controller.Tag }}"
+        digest: sha256:5b161f051d017e55d358435f295f5e9a297e66158f136321d9b04520ec6c48a3
+        digestChroot: sha256:5976b1067cfbca8a21d0ba53d71f83543a73316a61ea7f7e436d6cf84ddf9b26
+        pullPolicy: IfNotPresent
+        # www-data -> uid 101
+        runAsUser: 101
+        allowPrivilegeEscalation: true
+      # -- Use an existing PSP instead of creating one
+      existingPsp: ""
+      # -- Configures the controller container name
+      containerName: controller
+      # -- Configures the ports that the nginx-controller listens on
+      containerPort:
+        http: 80
+        https: 443
+      # -- Will add custom configuration options to Nginx https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/
+      config: {}
+      # -- Annotations to be added to the controller config configuration configmap.
+      configAnnotations: {}
+      # -- Will add custom headers before sending traffic to backends according to https://github.com/kubernetes/ingress-nginx/tree/main/docs/examples/customization/custom-headers
+      proxySetHeaders: {}
+      # -- Will add custom headers before sending response traffic to the client according to: https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#add-headers
+      addHeaders: {}
+      # -- Optionally customize the pod dnsConfig.
+      dnsConfig: {}
+      # -- Optionally customize the pod hostAliases.
+      hostAliases: []
+      # - ip: 127.0.0.1
+      #   hostnames:
+      #   - foo.local
+      #   - bar.local
+      # - ip: 10.1.2.3
+      #   hostnames:
+      #   - foo.remote
+      #   - bar.remote
+      # -- Optionally customize the pod hostname.
+      hostname: {}
+      # -- Optionally change this to ClusterFirstWithHostNet in case you have 'hostNetwork: true'.
+      # By default, while using host network, name resolution uses the host's DNS. If you wish nginx-controller
+      # to keep resolving names inside the k8s network, use ClusterFirstWithHostNet.
+      dnsPolicy: ClusterFirst
+      # -- Bare-metal considerations via the host network https://kubernetes.github.io/ingress-nginx/deploy/baremetal/#via-the-host-network
+      # Ingress status was blank because there is no Service exposing the Ingress-Nginx Controller in a configuration using the host network, the default --publish-service flag used in standard cloud setups does not apply
+      reportNodeInternalIp: false
+      # -- Process Ingress objects without ingressClass annotation/ingressClassName field
+      # Overrides value for --watch-ingress-without-class flag of the controller binary
+      # Defaults to false
+      watchIngressWithoutClass: false
+      # -- Process IngressClass per name (additionally as per spec.controller).
+      ingressClassByName: false
+      # -- This configuration enables Topology Aware Routing feature, used together with service annotation service.kubernetes.io/topology-mode="auto"
+      # Defaults to false
+      enableTopologyAwareRouting: false
+      # -- This configuration defines if Ingress Controller should allow users to set
+      # their own *-snippet annotations, otherwise this is forbidden / dropped
+      # when users add those annotations.
+      # Global snippets in ConfigMap are still respected
+      allowSnippetAnnotations: true
+      # -- Required for use with CNI based kubernetes installations (such as ones set up by kubeadm),
+      # since CNI and hostport don't mix yet. Can be deprecated once https://github.com/kubernetes/kubernetes/issues/23920
+      # is merged
+      hostNetwork: {{ .Modules.Additional.Ingress.HostNetwork }}
+      ## Use host ports 80 and 443
+      ## Disabled by default
+      hostPort:
+        # -- Enable 'hostPort' or not
+        enabled: false
+        ports:
+          # -- 'hostPort' http port
+          http: 80
+          # -- 'hostPort' https port
+          https: 443
+      # NetworkPolicy for controller component.
+      networkPolicy:
+        # -- Enable 'networkPolicy' or not
+        enabled: false
+      # -- Election ID to use for status update, by default it uses the controller name combined with a suffix of 'leader'
+      electionID: ""
+      ## This section refers to the creation of the IngressClass resource
+      ## IngressClass resources are supported since k8s >= 1.18 and required since k8s >= 1.19
+      ingressClassResource:
+        # -- Name of the ingressClass
+        name: nginx
+        # -- Is this ingressClass enabled or not
+        enabled: true
+        # -- Is this the default ingressClass for the cluster
+        default: false
+        # -- Controller-value of the controller that is processing this ingressClass
+        controllerValue: "k8s.io/ingress-nginx"
+        # -- Parameters is a link to a custom resource containing additional
+        # configuration for the controller. This is optional if the controller
+        # does not require extra parameters.
+        parameters: {}
+      # -- For backwards compatibility with ingress.class annotation, use ingressClass.
+      # Algorithm is as follows, first ingressClassName is considered, if not present, controller looks for ingress.class annotation
+      ingressClass: {{ .Modules.Additional.Ingress.IngressClassName }}
+      # -- Labels to add to the pod container metadata
+      podLabels: {}
+
+      # -- Security Context policies for controller pods
+      podSecurityContext: {}
+      # -- See https://kubernetes.io/docs/tasks/administer-cluster/sysctl-cluster/ for notes on enabling and using sysctls
+      sysctls: {}
+      # sysctls:
+      #   "net.core.somaxconn": "8192"
+
+      # -- Allows customization of the source of the IP address or FQDN to report
+      # in the ingress status field. By default, it reads the information provided
+      # by the service. If disable, the status field reports the IP address of the
+      # node or nodes where an ingress controller pod is running.
+      publishService:
+        # -- Enable 'publishService' or not
+        enabled: true
+        # -- Allows overriding of the publish service to bind to
+        # Must be <namespace>/<service_name>
+        pathOverride: ""
+      # Limit the scope of the controller to a specific namespace
+      scope:
+        # -- Enable 'scope' or not
+        enabled: false
+        # -- Namespace to limit the controller to; defaults to $(POD_NAMESPACE)
+        namespace: ""
+        # -- When scope.enabled == false, instead of watching all namespaces, we watching namespaces whose labels
+        # only match with namespaceSelector. Format like foo=bar. Defaults to empty, means watching all namespaces.
+        namespaceSelector: ""
+      # -- Allows customization of the configmap / nginx-configmap namespace; defaults to $(POD_NAMESPACE)
+      configMapNamespace: ""
+      tcp:
+        # -- Allows customization of the tcp-services-configmap; defaults to $(POD_NAMESPACE)
+        configMapNamespace: ""
+        # -- Annotations to be added to the tcp config configmap
+        annotations: {}
+      udp:
+        # -- Allows customization of the udp-services-configmap; defaults to $(POD_NAMESPACE)
+        configMapNamespace: ""
+        # -- Annotations to be added to the udp config configmap
+        annotations: {}
+      # -- Maxmind license key to download GeoLite2 Databases.
+      ## https://blog.maxmind.com/2019/12/18/significant-changes-to-accessing-and-using-geolite2-databases
+      maxmindLicenseKey: ""
+      # -- Additional command line arguments to pass to Ingress-Nginx Controller
+      # E.g. to specify the default SSL certificate you can use
+      extraArgs: {}
+      ## extraArgs:
+      ##   default-ssl-certificate: "<namespace>/<secret_name>"
+
+      # -- Additional environment variables to set
+      extraEnvs: []
+      # extraEnvs:
+      #   - name: FOO
+      #     valueFrom:
+      #       secretKeyRef:
+      #         key: FOO
+      #         name: secret-resource
+
+      # -- Use a `DaemonSet` or `Deployment`
+      kind: DaemonSet
+      # -- Annotations to be added to the controller Deployment or DaemonSet
+      ##
+      annotations: {}
+      #  keel.sh/pollSchedule: "@every 60m"
+
+      # -- Labels to be added to the controller Deployment or DaemonSet and other resources that do not have option to specify labels
+      ##
+      labels: {}
+      #  keel.sh/policy: patch
+      #  keel.sh/trigger: poll
+
+      # -- The update strategy to apply to the Deployment or DaemonSet
+      ##
+      updateStrategy: {}
+      #  rollingUpdate:
+      #    maxUnavailable: 1
+      #  type: RollingUpdate
+
+      # -- `minReadySeconds` to avoid killing pods before we are ready
+      ##
+      minReadySeconds: 0
+      # -- Node tolerations for server scheduling to nodes with taints
+      ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
+      ##
+      tolerations: []
+      #  - key: "key"
+      #    operator: "Equal|Exists"
+      #    value: "value"
+      #    effect: "NoSchedule|PreferNoSchedule|NoExecute(1.6 only)"
+
+      # -- Affinity and anti-affinity rules for server scheduling to nodes
+      ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity
+      ##
+      affinity: {}
+      # # An example of preferred pod anti-affinity, weight is in the range 1-100
+      # podAntiAffinity:
+      #   preferredDuringSchedulingIgnoredDuringExecution:
+      #   - weight: 100
+      #     podAffinityTerm:
+      #       labelSelector:
+      #         matchExpressions:
+      #         - key: app.kubernetes.io/name
+      #           operator: In
+      #           values:
+      #           - ingress-nginx
+      #         - key: app.kubernetes.io/instance
+      #           operator: In
+      #           values:
+      #           - ingress-nginx
+      #         - key: app.kubernetes.io/component
+      #           operator: In
+      #           values:
+      #           - controller
+      #       topologyKey: kubernetes.io/hostname
+
+      # # An example of required pod anti-affinity
+      # podAntiAffinity:
+      #   requiredDuringSchedulingIgnoredDuringExecution:
+      #   - labelSelector:
+      #       matchExpressions:
+      #       - key: app.kubernetes.io/name
+      #         operator: In
+      #         values:
+      #         - ingress-nginx
+      #       - key: app.kubernetes.io/instance
+      #         operator: In
+      #         values:
+      #         - ingress-nginx
+      #       - key: app.kubernetes.io/component
+      #         operator: In
+      #         values:
+      #         - controller
+      #     topologyKey: "kubernetes.io/hostname"
+
+      # -- Topology spread constraints rely on node labels to identify the topology domain(s) that each Node is in.
+      ## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/
+      ##
+      topologySpreadConstraints: []
+
+      # -- `terminationGracePeriodSeconds` to avoid killing pods before we are ready
+      ## wait up to five minutes for the drain of connections
+      ##
+      terminationGracePeriodSeconds: 300
+      # -- Node labels for controller pod assignment
+      ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/
+      ##
+      nodeSelector:
+        kubernetes.io/os: linux
+      ## Liveness and readiness probe values
+      ## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#container-probes
+      ##
+      ## startupProbe:
+      ##   httpGet:
+      ##     # should match container.healthCheckPath
+      ##     path: "/healthz"
+      ##     port: 10254
+      ##     scheme: HTTP
+      ##   initialDelaySeconds: 5
+      ##   periodSeconds: 5
+      ##   timeoutSeconds: 2
+      ##   successThreshold: 1
+      ##   failureThreshold: 5
+      livenessProbe:
+        httpGet:
+          # should match container.healthCheckPath
+          path: "/healthz"
+          port: 10254
+          scheme: HTTP
+        initialDelaySeconds: 10
+        periodSeconds: 10
+        timeoutSeconds: 1
+        successThreshold: 1
+        failureThreshold: 5
+      readinessProbe:
+        httpGet:
+          # should match container.healthCheckPath
+          path: "/healthz"
+          port: 10254
+          scheme: HTTP
+        initialDelaySeconds: 10
+        periodSeconds: 10
+        timeoutSeconds: 1
+        successThreshold: 1
+        failureThreshold: 3
+      # -- Path of the health check endpoint. All requests received on the port defined by
+      # the healthz-port parameter are forwarded internally to this path.
+      healthCheckPath: "/healthz"
+      # -- Address to bind the health check endpoint.
+      # It is better to set this option to the internal node address
+      # if the Ingress-Nginx Controller is running in the `hostNetwork: true` mode.
+      healthCheckHost: ""
+      # -- Annotations to be added to controller pods
+      ##
+      podAnnotations: {}
+      replicaCount: 1
+      # -- Minimum available pods set in PodDisruptionBudget.
+      # Define either 'minAvailable' or 'maxUnavailable', never both.
+      minAvailable: 1
+      # -- Maximum unavalaile pods set in PodDisruptionBudget. If set, 'minAvailable' is ignored.
+      # maxUnavailable: 1
+
+      ## Define requests resources to avoid probe issues due to CPU utilization in busy nodes
+      ## ref: https://github.com/kubernetes/ingress-nginx/issues/4735#issuecomment-551204903
+      ## Ideally, there should be no limits.
+      ## https://engineering.indeedblog.com/blog/2019/12/cpu-throttling-regression-fix/
+      resources:
+        ##  limits:
+        ##    cpu: 100m
+        ##    memory: 90Mi
+        requests:
+          cpu: 100m
+          memory: 90Mi
+      # Mutually exclusive with keda autoscaling
+      autoscaling:
+        enabled: false
+        annotations: {}
+        minReplicas: 1
+        maxReplicas: 11
+        targetCPUUtilizationPercentage: 50
+        targetMemoryUtilizationPercentage: 50
+        behavior: {}
+        # scaleDown:
+        #   stabilizationWindowSeconds: 300
+        #   policies:
+        #   - type: Pods
+        #     value: 1
+        #     periodSeconds: 180
+        # scaleUp:
+        #   stabilizationWindowSeconds: 300
+        #   policies:
+        #   - type: Pods
+        #     value: 2
+        #     periodSeconds: 60
+      autoscalingTemplate: []
+      # Custom or additional autoscaling metrics
+      # ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-custom-metrics
+      # - type: Pods
+      #   pods:
+      #     metric:
+      #       name: nginx_ingress_controller_nginx_process_requests_total
+      #     target:
+      #       type: AverageValue
+      #       averageValue: 10000m
+
+      # Mutually exclusive with hpa autoscaling
+      keda:
+        apiVersion: "keda.sh/v1alpha1"
+        ## apiVersion changes with keda 1.x vs 2.x
+        ## 2.x = keda.sh/v1alpha1
+        ## 1.x = keda.k8s.io/v1alpha1
+        enabled: false
+        minReplicas: 1
+        maxReplicas: 11
+        pollingInterval: 30
+        cooldownPeriod: 300
+        # fallback:
+        #   failureThreshold: 3
+        #   replicas: 11
+        restoreToOriginalReplicaCount: false
+        scaledObject:
+          annotations: {}
+          # Custom annotations for ScaledObject resource
+          #  annotations:
+          # key: value
+        triggers: []
+        # - type: prometheus
+        #   metadata:
+        #     serverAddress: http://<prometheus-host>:9090
+        #     metricName: http_requests_total
+        #     threshold: '100'
+        #     query: sum(rate(http_requests_total{deployment="my-deployment"}[2m]))
+
+        behavior: {}
+        # scaleDown:
+        #   stabilizationWindowSeconds: 300
+        #   policies:
+        #   - type: Pods
+        #     value: 1
+        #     periodSeconds: 180
+        # scaleUp:
+        #   stabilizationWindowSeconds: 300
+        #   policies:
+        #   - type: Pods
+        #     value: 2
+        #     periodSeconds: 60
+      # -- Enable mimalloc as a drop-in replacement for malloc.
+      ## ref: https://github.com/microsoft/mimalloc
+      ##
+      enableMimalloc: true
+      ## Override NGINX template
+      customTemplate:
+        configMapName: ""
+        configMapKey: ""
+      service:
+        enabled: true
+        # -- If enabled is adding an appProtocol option for Kubernetes service. An appProtocol field replacing annotations that were
+        # using for setting a backend protocol. Here is an example for AWS: service.beta.kubernetes.io/aws-load-balancer-backend-protocol: http
+        # It allows choosing the protocol for each backend specified in the Kubernetes service.
+        # See the following GitHub issue for more details about the purpose: https://github.com/kubernetes/kubernetes/issues/40244
+        # Will be ignored for Kubernetes versions older than 1.20
+        ##
+        appProtocol: true
+        # -- Annotations are mandatory for the load balancer to come up. Varies with the cloud service. Values passed through helm tpl engine.
+        annotations: {}
+        labels: {}
+        # clusterIP: ""
+
+        # -- List of IP addresses at which the controller services are available
+        ## Ref: https://kubernetes.io/docs/concepts/services-networking/service/#external-ips
+        ##
+        externalIPs: []
+        # -- Used by cloud providers to connect the resulting `LoadBalancer` to a pre-existing static IP according to https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer
+        loadBalancerIP: ""
+        loadBalancerSourceRanges: []
+        # -- Used by cloud providers to select a load balancer implementation other than the cloud provider default. https://kubernetes.io/docs/concepts/services-networking/service/#load-balancer-class
+        loadBalancerClass: ""
+        enableHttp: true
+        enableHttps: true
+        ## Set external traffic policy to: "Local" to preserve source IP on providers supporting it.
+        ## Ref: https://kubernetes.io/docs/tutorials/services/source-ip/#source-ip-for-services-with-typeloadbalancer
+        # externalTrafficPolicy: ""
+
+        ## Must be either "None" or "ClientIP" if set. Kubernetes will default to "None".
+        ## Ref: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies
+        # sessionAffinity: ""
+
+        ## Specifies the health check node port (numeric port number) for the service. If healthCheckNodePort isn’t specified,
+        ## the service controller allocates a port from your cluster’s NodePort range.
+        ## Ref: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip
+        # healthCheckNodePort: 0
+
+        # -- Represents the dual-stack-ness requested or required by this Service. Possible values are
+        # SingleStack, PreferDualStack or RequireDualStack.
+        # The ipFamilies and clusterIPs fields depend on the value of this field.
+        ## Ref: https://kubernetes.io/docs/concepts/services-networking/dual-stack/
+        ipFamilyPolicy: "SingleStack"
+        # -- List of IP families (e.g. IPv4, IPv6) assigned to the service. This field is usually assigned automatically
+        # based on cluster configuration and the ipFamilyPolicy field.
+        ## Ref: https://kubernetes.io/docs/concepts/services-networking/dual-stack/
+        ipFamilies:
+          - IPv4
+        ports:
+          http: 80
+          https: 443
+        targetPorts:
+          http: http
+          https: https
+        type: ""
+        # type: NodePort
+        # nodePorts:
+        #   http: 32080
+        #   https: 32443
+        #   tcp:
+        #     8080: 32808
+        nodePorts:
+          http: ""
+          https: ""
+          tcp: {}
+          udp: {}
+        external:
+          enabled: true
+        internal:
+          # -- Enables an additional internal load balancer (besides the external one).
+          enabled: false
+          # -- Annotations are mandatory for the load balancer to come up. Varies with the cloud service. Values passed through helm tpl engine.
+          annotations: {}
+          # -- Used by cloud providers to connect the resulting internal LoadBalancer to a pre-existing static IP. Make sure to add to the service the needed annotation to specify the subnet which the static IP belongs to. For instance, `networking.gke.io/internal-load-balancer-subnet` for GCP and `service.beta.kubernetes.io/aws-load-balancer-subnets` for AWS.
+          loadBalancerIP: ""
+          # -- Restrict access For LoadBalancer service. Defaults to 0.0.0.0/0.
+          loadBalancerSourceRanges: []
+          ## Set external traffic policy to: "Local" to preserve source IP on
+          ## providers supporting it
+          ## Ref: https://kubernetes.io/docs/tutorials/services/source-ip/#source-ip-for-services-with-typeloadbalancer
+          # externalTrafficPolicy: ""
+
+          # -- Custom port mapping for internal service
+          ports: {}
+          #  http: 80
+          #  https: 443
+
+          # -- Custom target port mapping for internal service
+          targetPorts: {}
+          #  http: http
+          #  https: https
+      # shareProcessNamespace enables process namespace sharing within the pod.
+      # This can be used for example to signal log rotation using `kill -USR1` from a sidecar.
+      shareProcessNamespace: false
+      # -- Additional containers to be added to the controller pod.
+      # See https://github.com/lemonldap-ng-controller/lemonldap-ng-controller as example.
+      extraContainers: []
+      #  - name: my-sidecar
+      #    image: nginx:latest
+      #  - name: lemonldap-ng-controller
+      #    image: lemonldapng/lemonldap-ng-controller:0.2.0
+      #    args:
+      #      - /lemonldap-ng-controller
+      #      - --alsologtostderr
+      #      - --configmap=$(POD_NAMESPACE)/lemonldap-ng-configuration
+      #    env:
+      #      - name: POD_NAME
+      #        valueFrom:
+      #          fieldRef:
+      #            fieldPath: metadata.name
+      #      - name: POD_NAMESPACE
+      #        valueFrom:
+      #          fieldRef:
+      #            fieldPath: metadata.namespace
+      #    volumeMounts:
+      #    - name: copy-portal-skins
+      #      mountPath: /srv/var/lib/lemonldap-ng/portal/skins
+
+      # -- Additional volumeMounts to the controller main container.
+      extraVolumeMounts: []
+      #  - name: copy-portal-skins
+      #   mountPath: /var/lib/lemonldap-ng/portal/skins
+
+      # -- Additional volumes to the controller pod.
+      extraVolumes: []
+      #  - name: copy-portal-skins
+      #    emptyDir: {}
+
+      # -- Containers, which are run before the app containers are started.
+      extraInitContainers: []
+      # - name: init-myservice
+      #   image: busybox
+      #   command: ['sh', '-c', 'until nslookup myservice; do echo waiting for myservice; sleep 2; done;']
+
+      # -- Modules, which are mounted into the core nginx image. See values.yaml for a sample to add opentelemetry module
+      extraModules: []
+      # - name: mytestmodule
+      #   image: registry.k8s.io/ingress-nginx/mytestmodule
+      #   containerSecurityContext:
+      #     allowPrivilegeEscalation: false
+      #
+      # The image must contain a `/usr/local/bin/init_module.sh` executable, which
+      # will be executed as initContainers, to move its config files within the
+      # mounted volume.
+
+      opentelemetry:
+        enabled: false
+        image: registry.k8s.io/ingress-nginx/opentelemetry:v20230721-3e2062ee5@sha256:13bee3f5223883d3ca62fee7309ad02d22ec00ff0d7033e3e9aca7a9f60fd472
+        containerSecurityContext:
+          allowPrivilegeEscalation: false
+        resources: {}
+      admissionWebhooks:
+        annotations: {}
+        # ignore-check.kube-linter.io/no-read-only-rootfs: "This deployment needs write access to root filesystem".
+
+        ## Additional annotations to the admission webhooks.
+        ## These annotations will be added to the ValidatingWebhookConfiguration and
+        ## the Jobs Spec of the admission webhooks.
+        enabled: {{ .Modules.Additional.Ingress.EnableAdmissionWebhooks }}
+        # -- Additional environment variables to set
+        extraEnvs: []
+        # extraEnvs:
+        #   - name: FOO
+        #     valueFrom:
+        #       secretKeyRef:
+        #         key: FOO
+        #         name: secret-resource
+        # -- Admission Webhook failure policy to use
+        failurePolicy: Fail
+        # timeoutSeconds: 10
+        port: 8443
+        certificate: "/usr/local/certificates/cert"
+        key: "/usr/local/certificates/key"
+        namespaceSelector: {}
+        objectSelector: {}
+        # -- Labels to be added to admission webhooks
+        labels: {}
+        # -- Use an existing PSP instead of creating one
+        existingPsp: ""
+        service:
+          annotations: {}
+          # clusterIP: ""
+          externalIPs: []
+          # loadBalancerIP: ""
+          loadBalancerSourceRanges: []
+          servicePort: 443
+          type: ClusterIP
+        createSecretJob:
+          securityContext:
+            allowPrivilegeEscalation: false
+          resources: {}
+          # limits:
+          #   cpu: 10m
+          #   memory: 20Mi
+          # requests:
+          #   cpu: 10m
+          #   memory: 20Mi
+        patchWebhookJob:
+          securityContext:
+            allowPrivilegeEscalation: false
+          resources: {}
+        patch:
+          enabled: true
+          image:
+            registry: registry.k8s.io
+            image: ingress-nginx/kube-webhook-certgen
+            ## for backwards compatibility consider setting the full image url via the repository value below
+            ## use *either* current default registry/image or repository format or installing chart by providing the values.yaml will fail
+            ## repository:
+            tag: v20231011-8b53cabe0
+            digest: sha256:a7943503b45d552785aa3b5e457f169a5661fb94d82b8a3373bcd9ebaf9aac80
+            pullPolicy: IfNotPresent
+          # -- Provide a priority class name to the webhook patching job
+          ##
+          priorityClassName: ""
+          podAnnotations: {}
+          nodeSelector:
+            kubernetes.io/os: linux
+          tolerations: []
+          # -- Labels to be added to patch job resources
+          labels: {}
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 2000
+            fsGroup: 2000
+        # Use certmanager to generate webhook certs
+        certManager:
+          enabled: false
+          # self-signed root certificate
+          rootCert:
+            # default to be 5y
+            duration: ""
+          admissionCert:
+            # default to be 1y
+            duration: ""
+            # issuerRef:
+            #   name: "issuer"
+            #   kind: "ClusterIssuer"
+      metrics:
+        port: 10254
+        portName: metrics
+        # if this port is changed, change healthz-port: in extraArgs: accordingly
+        enabled: false
+        service:
+          annotations: {}
+          # prometheus.io/scrape: "true"
+          # prometheus.io/port: "10254"
+          # -- Labels to be added to the metrics service resource
+          labels: {}
+          # clusterIP: ""
+
+          # -- List of IP addresses at which the stats-exporter service is available
+          ## Ref: https://kubernetes.io/docs/concepts/services-networking/service/#external-ips
+          ##
+          externalIPs: []
+          # loadBalancerIP: ""
+          loadBalancerSourceRanges: []
+          servicePort: 10254
+          type: ClusterIP
+          # externalTrafficPolicy: ""
+          # nodePort: ""
+        serviceMonitor:
+          enabled: false
+          additionalLabels: {}
+          ## The label to use to retrieve the job name from.
+          ## jobLabel: "app.kubernetes.io/name"
+          namespace: ""
+          namespaceSelector: {}
+          ## Default: scrape .Release.Namespace or namespaceOverride only
+          ## To scrape all, use the following:
+          ## namespaceSelector:
+          ##   any: true
+          scrapeInterval: 30s
+          # honorLabels: true
+          targetLabels: []
+          relabelings: []
+          metricRelabelings: []
+        prometheusRule:
+          enabled: false
+          additionalLabels: {}
+          # namespace: ""
+          rules: []
+          # # These are just examples rules, please adapt them to your needs
+          # - alert: NGINXConfigFailed
+          #   expr: count(nginx_ingress_controller_config_last_reload_successful == 0) > 0
+          #   for: 1s
+          #   labels:
+          #     severity: critical
+          #   annotations:
+          #     description: bad ingress config - nginx config test failed
+          #     summary: uninstall the latest ingress changes to allow config reloads to resume
+          # - alert: NGINXCertificateExpiry
+          #   expr: (avg(nginx_ingress_controller_ssl_expire_time_seconds) by (host) - time()) < 604800
+          #   for: 1s
+          #   labels:
+          #     severity: critical
+          #   annotations:
+          #     description: ssl certificate(s) will expire in less then a week
+          #     summary: renew expiring certificates to avoid downtime
+          # - alert: NGINXTooMany500s
+          #   expr: 100 * ( sum( nginx_ingress_controller_requests{status=~"5.+"} ) / sum(nginx_ingress_controller_requests) ) > 5
+          #   for: 1m
+          #   labels:
+          #     severity: warning
+          #   annotations:
+          #     description: Too many 5XXs
+          #     summary: More than 5% of all requests returned 5XX, this requires your attention
+          # - alert: NGINXTooMany400s
+          #   expr: 100 * ( sum( nginx_ingress_controller_requests{status=~"4.+"} ) / sum(nginx_ingress_controller_requests) ) > 5
+          #   for: 1m
+          #   labels:
+          #     severity: warning
+          #   annotations:
+          #     description: Too many 4XXs
+          #     summary: More than 5% of all requests returned 4XX, this requires your attention
+      # -- Improve connection draining when ingress controller pod is deleted using a lifecycle hook:
+      # With this new hook, we increased the default terminationGracePeriodSeconds from 30 seconds
+      # to 300, allowing the draining of connections up to five minutes.
+      # If the active connections end before that, the pod will terminate gracefully at that time.
+      # To effectively take advantage of this feature, the Configmap feature
+      # worker-shutdown-timeout new value is 240s instead of 10s.
+      ##
+      lifecycle:
+        preStop:
+          exec:
+            command:
+              - /wait-shutdown
+      priorityClassName: ""
+    # -- Rollback limit
+    ##
+    revisionHistoryLimit: 10
+    ## Default 404 backend
+    ##
+    defaultBackend:
+      ##
+      enabled: false
+      name: defaultbackend
+      image:
+        ## for backwards compatibility consider setting the full image url via the repository value below
+        ## use *either* current default registry/image or repository format or installing chart by providing the values.yaml will fail
+        repository: {{ .Modules.Additional.Ingress.Nginx.DefaultBackend.Image }}
+        tag: "{{ .Modules.Additional.Ingress.Nginx.DefaultBackend.Tag }}"
+        pullPolicy: IfNotPresent
+        # nobody user -> uid 65534
+        runAsUser: 65534
+        runAsNonRoot: true
+        readOnlyRootFilesystem: true
+        allowPrivilegeEscalation: false
+      # -- Use an existing PSP instead of creating one
+      existingPsp: ""
+      extraArgs: {}
+      serviceAccount:
+        create: true
+        name: ""
+        automountServiceAccountToken: true
+      # -- Additional environment variables to set for defaultBackend pods
+      extraEnvs: []
+      port: 8080
+      ## Readiness and liveness probes for default backend
+      ## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/
+      ##
+      livenessProbe:
+        failureThreshold: 3
+        initialDelaySeconds: 30
+        periodSeconds: 10
+        successThreshold: 1
+        timeoutSeconds: 5
+      readinessProbe:
+        failureThreshold: 6
+        initialDelaySeconds: 0
+        periodSeconds: 5
+        successThreshold: 1
+        timeoutSeconds: 5
+      # -- The update strategy to apply to the Deployment or DaemonSet
+      ##
+      updateStrategy: {}
+      #  rollingUpdate:
+      #    maxUnavailable: 1
+      #  type: RollingUpdate
+
+      # -- `minReadySeconds` to avoid killing pods before we are ready
+      ##
+      minReadySeconds: 0
+      # -- Node tolerations for server scheduling to nodes with taints
+      ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
+      ##
+      tolerations: []
+      #  - key: "key"
+      #    operator: "Equal|Exists"
+      #    value: "value"
+      #    effect: "NoSchedule|PreferNoSchedule|NoExecute(1.6 only)"
+
+      affinity: {}
+      # -- Security Context policies for controller pods
+      # See https://kubernetes.io/docs/tasks/administer-cluster/sysctl-cluster/ for
+      # notes on enabling and using sysctls
+      ##
+      podSecurityContext: {}
+      # -- Security Context policies for controller main container.
+      # See https://kubernetes.io/docs/tasks/administer-cluster/sysctl-cluster/ for
+      # notes on enabling and using sysctls
+      ##
+      containerSecurityContext: {}
+      # -- Labels to add to the pod container metadata
+      podLabels: {}
+      # -- Node labels for default backend pod assignment
+      ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/
+      ##
+      nodeSelector:
+        kubernetes.io/os: linux
+      # -- Annotations to be added to default backend pods
+      ##
+      podAnnotations: {}
+      replicaCount: 1
+      minAvailable: 1
+      resources: {}
+      # limits:
+      #   cpu: 10m
+      #   memory: 20Mi
+      # requests:
+      #   cpu: 10m
+      #   memory: 20Mi
+
+      extraVolumeMounts: []
+      ## Additional volumeMounts to the default backend container.
+      #  - name: copy-portal-skins
+      #   mountPath: /var/lib/lemonldap-ng/portal/skins
+
+      extraVolumes: []
+      ## Additional volumes to the default backend pod.
+      #  - name: copy-portal-skins
+      #    emptyDir: {}
+
+      autoscaling:
+        annotations: {}
+        enabled: false
+        minReplicas: 1
+        maxReplicas: 2
+        targetCPUUtilizationPercentage: 50
+        targetMemoryUtilizationPercentage: 50
+      # NetworkPolicy for default backend component.
+      networkPolicy:
+        # -- Enable 'networkPolicy' or not
+        enabled: false
+      service:
+        annotations: {}
+        # clusterIP: ""
+
+        # -- List of IP addresses at which the default backend service is available
+        ## Ref: https://kubernetes.io/docs/concepts/services-networking/service/#external-ips
+        ##
+        externalIPs: []
+        # loadBalancerIP: ""
+        loadBalancerSourceRanges: []
+        servicePort: 80
+        type: ClusterIP
+      priorityClassName: ""
+      # -- Labels to be added to the default backend resources
+      labels: {}
+    ## Enable RBAC as per https://github.com/kubernetes/ingress-nginx/blob/main/docs/deploy/rbac.md and https://github.com/kubernetes/ingress-nginx/issues/266
+    rbac:
+      create: true
+      scope: false
+    ## If true, create & use Pod Security Policy resources
+    ## https://kubernetes.io/docs/concepts/policy/pod-security-policy/
+    podSecurityPolicy:
+      enabled: false
+    serviceAccount:
+      create: true
+      name: ""
+      automountServiceAccountToken: true
+      # -- Annotations for the controller service account
+      annotations: {}
+    # -- Optional array of imagePullSecrets containing private registry credentials
+    ## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
+    imagePullSecrets: []
+    # - name: secretName
+
+    # -- TCP service key-value pairs
+    ## Ref: https://github.com/kubernetes/ingress-nginx/blob/main/docs/user-guide/exposing-tcp-udp-services.md
+    ##
+    tcp: {}
+    #  8080: "default/example-tcp-svc:9000"
+
+    # -- UDP service key-value pairs
+    ## Ref: https://github.com/kubernetes/ingress-nginx/blob/main/docs/user-guide/exposing-tcp-udp-services.md
+    ##
+    udp: {}
+    #  53: "kube-system/kube-dns:53"
+
+    # -- Prefix for TCP and UDP ports names in ingress controller service
+    ## Some cloud providers, like Yandex Cloud may have a requirements for a port name regex to support cloud load balancer integration
+    portNamePrefix: ""
+    # -- (string) A base64-encoded Diffie-Hellman parameter.
+    # This can be generated with: `openssl dhparam 4096 2> /dev/null | base64`
+    ## Ref: https://github.com/kubernetes/ingress-nginx/tree/main/docs/examples/customization/ssl-dh-param
+    dhParam: ""

pkg/resources/templates/helm-apps/releases/additional-modules/longhorn.yml.tmpl

@@ -1,9 +1,13 @@
-{{- if .Modules.Additional.Storage.Longhorn.Enabled }}
 - name: longhorn
   namespace: longhorn-system
   create_namespace: true
   chart_ref: kube-forge/longhorn
   chart_version: 1.6.1
+  {{- if .Modules.Additional.Storage.Longhorn.Enabled }}
+  release_state: "present"
+  {{- else }}
+  release_state: "absent"
+  {{- end }}
   values:
     global:
       cattle:
@@ -484,4 +488,3 @@
 
     # -- Setting that allows Longhorn to generate code coverage profiles.
     enableGoCoverDir: false
-{{- end }}

pkg/resources/templates/helm-apps/releases/observability/observability.yml.tmpl

@@ -33,6 +33,74 @@
       scrapeInterval: {{ .Modules.Observability.Monitoring.Prometheus.ScrapeInterval }}
       retention: {{ .Modules.Observability.Monitoring.Prometheus.Persistence.Retention }}
       # serviceNodePort: 30008
+      additionalConfigs: |
+        - job_name: "kubelet"
+          scheme: https
+          metrics_path: /metrics/cadvisor
+          tls_config:
+            insecure_skip_verify: true
+          authorization:
+            credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+          kubernetes_sd_configs:
+            - role: node
+          relabel_configs:
+            - action: labelmap
+              regex: __meta_kubernetes_node_label_(.+)
+
+        - job_name: "kubernetes-apiservers"
+          kubernetes_sd_configs:
+            - role: endpoints
+          scheme: https
+          tls_config:
+            ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+          authorization:
+            credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+          relabel_configs:
+            - source_labels:
+                [
+                  __meta_kubernetes_namespace,
+                  __meta_kubernetes_service_name,
+                  __meta_kubernetes_endpoint_port_name,
+                ]
+              action: keep
+              regex: default;kubernetes;https
+
+        - job_name: "coredns"
+          kubernetes_sd_configs:
+            - role: endpoints
+          scheme: http
+          relabel_configs:
+            - source_labels:
+                [
+                  __meta_kubernetes_namespace,
+                  __meta_kubernetes_service_name,
+                  __meta_kubernetes_endpoint_port_name,
+                ]
+              action: keep
+              regex: kube-system;.*dns.*;metrics
+
+        - job_name: 'ingress-endpoints'
+          metrics_path: /probe
+          params:
+            module: [https_ok]
+          kubernetes_sd_configs:
+            - role: ingress
+          relabel_configs:
+            - source_labels: [__address__]
+              target_label: __param_target
+            - source_labels: [__param_target]
+              target_label: instance
+            - source_labels:
+                [
+                  __meta_kubernetes_ingress_scheme,
+                  __address__,
+                  __meta_kubernetes_ingress_path,
+                ]
+              regex: (.+);(.+);(.+)
+              replacement: https://${2}${3}
+              target_label: __param_target
+            - target_label: __address__
+              replacement: observability-blackbox-exporter:9115
 
     alertManager:
       enabled: {{ .Modules.Observability.Monitoring.AlertManager.Enabled }}
@@ -46,13 +114,18 @@
       # serviceNodePort: 30009
 
       configPath: /etc/alertmanager
-
+      {{- if .Modules.Observability.Monitoring.AlertManager.AdditionalMessageTemplates }}
+      additionalMessageTemplates:
+        {{- .Modules.Observability.Monitoring.AlertManager.AdditionalMessageTemplates | toYaml | nindent 8 }}
+      {{- end }}
+      {{- if .Modules.Observability.Monitoring.AlertManager.Route }}
       route:
         {{- .Modules.Observability.Monitoring.AlertManager.Route | toYaml | nindent 8 }}
-
+      {{- end }}
+      {{- if .Modules.Observability.Monitoring.AlertManager.Receivers }}
       receivers:
         {{- .Modules.Observability.Monitoring.AlertManager.Receivers | toYaml | nindent 8 }}
-
+      {{- end }}
 
     blackboxExporter:
       enabled: {{ .Modules.Observability.Monitoring.Blackbox.Enabled }}

pkg/resources/templates/kubespray/inventory/group_vars/all.yml.tmpl

@@ -134,3 +134,8 @@ unsafe_show_logs: false
 
 ## If enabled it will allow kubespray to attempt setup even if the distribution is not supported. For unsupported distributions this can lead to unexpected failures in some cases.
 allow_unsupported_distribution_setup: false
+
+## Containerd settings
+# containerd_metadata_root_dir: /app/lib/containerd
+# The state directory for containerd
+# containerd_state_dir: /app/run/containerd

pkg/resources/templates/kubespray/inventory/group_vars/k8s_cluster/addons.yml.tmpl

@@ -83,9 +83,8 @@ rbd_provisioner_enabled: false
 # rbd_provisioner_reclaim_policy: Delete
 
 # Nginx ingress controller deployment
-{{- if eq .Modules.Additional.Ingress.Type "nginx" }}
+ingress_nginx_enabled: false
-ingress_nginx_enabled: {{ .Modules.Additional.Ingress.Install }}
+ingress_nginx_host_network: false
-ingress_nginx_host_network: {{ .Modules.Additional.Ingress.HostNetwork }}
 ingress_publish_status_address: ""
 # ingress_nginx_nodeselector:
 #   kubernetes.io/os: "linux"
@@ -99,8 +98,8 @@ ingress_publish_status_address: ""
 #     value: ""
 #     effect: "NoSchedule"
 # ingress_nginx_namespace: "ingress-nginx"
-ingress_nginx_insecure_port: {{ .Modules.Additional.Ingress.InsecurePort }}
+ingress_nginx_insecure_port: 80
-ingress_nginx_secure_port: {{ .Modules.Additional.Ingress.SecurePort }}
+ingress_nginx_secure_port: 443
 # ingress_nginx_configmap:
 #   map-hash-bucket-size: "128"
 #   ssl-protocols: "TLSv1.2 TLSv1.3"
@@ -111,8 +110,7 @@ ingress_nginx_secure_port: {{ .Modules.Additional.Ingress.SecurePort }}
 # ingress_nginx_extra_args:
 #   - --default-ssl-certificate=default/foo-tls
 # ingress_nginx_termination_grace_period_seconds: 300
-ingress_nginx_class: {{ .Modules.Additional.Ingress.IngressClassName }}
+ingress_nginx_class: "nginx"
-{{- end }}
 
 # ALB ingress controller deployment
 ingress_alb_enabled: false
@@ -123,7 +121,7 @@ ingress_alb_enabled: false
 # alb_ingress_aws_debug: "false"
 
 # Cert manager deployment
-cert_manager_enabled: {{ .Modules.Additional.CertManager.Install }}
+cert_manager_enabled: false
 # cert_manager_namespace: "cert-manager"
 # cert_manager_tolerations:
 #   - key: node-role.kubernetes.io/master
@@ -152,9 +150,8 @@ cert_manager_enabled: {{ .Modules.Additional.CertManager.Install }}
 # cert_manager_dns_policy: "ClusterFirst"
 cert_manager_dns_config:
   nameservers:
-    {{- range .Modules.Additional.CertManager.DnsServers }}
+    - "8.8.8.8"
-    - "{{ . }}"
+    - "8.8.4.4"
-    {{- end }}
 
 # MetalLB deployment
 {{- if and (eq .Modules.Additional.LoadBalancer.Type "metallb") .Modules.Additional.LoadBalancer.Install }}

pkg/resources/templates/kubespray/inventory/group_vars/k8s_cluster/k8s-cluster.yml.tmpl

@@ -247,7 +247,7 @@ kubernetes_audit: false
 default_kubelet_config_dir: {{ `"{{ kube_config_dir }}/dynamic_kubelet_dir"` }}
 
 # pod security policy (RBAC must be enabled either by having 'RBAC' in authorization_modes or kubeadm enabled)
-podsecuritypolicy_enabled: {{ .Orchestrator.PodSecurotyPolicyEnabled }}
+podsecuritypolicy_enabled: {{ .Orchestrator.PodSecurityPolicyEnabled }}
 
 # Custom PodSecurityPolicySpec for restricted policy
 # podsecuritypolicy_restricted_spec: {}

pkg/templates/templates.go

@@ -20,6 +20,8 @@ var K8S_TEMPLATES = [...][2]string{
 
 var HELM_APPS_TEMPLATES = [...]string{
 	"templates/helm-apps/releases/additional-modules/longhorn.yml.tmpl",
+	"templates/helm-apps/releases/additional-modules/cert-manager.yml.tmpl",
+	"templates/helm-apps/releases/additional-modules/ingress-nginx.yml.tmpl",
 	"templates/helm-apps/releases/additional-modules/secrets-store-csi-driver.yml.tmpl",
 	"templates/helm-apps/releases/observability/fluent-operator.yml.tmpl",
 	"templates/helm-apps/releases/observability/opentelemetry-operator.yml.tmpl",