migrate from kubespray to library helm client

This commit is contained in:
2024-06-20 00:20:50 +03:00
parent 3109816dee
commit 78cf06e314
46 changed files with 6283 additions and 12035 deletions

View File

@@ -2,13 +2,28 @@ package main
import (
"flag"
"fmt"
"kube-forge/internal/additional"
"kube-forge/internal/cicd"
"kube-forge/internal/config"
"kube-forge/internal/csi"
"kube-forge/internal/kubespray"
"kube-forge/internal/logging"
"kube-forge/internal/observability"
"kube-forge/internal/registry"
"kube-forge/internal/secrets_storage"
"kube-forge/internal/templates"
"os"
)
func installAndConfigureModules() {
csi.ApplyCharts()
additional.ApplyCharts()
registry.ApplyCharts()
secrets_storage.ApplyCharts()
cicd.ApplyCharts()
observability.ApplyCharts()
}
func main() {
var password, configPath, workDir string
var verbose bool
@@ -20,9 +35,6 @@ func main() {
flag.Parse()
config := config.CreateConfig(configPath, workDir, password)
config.Verbose = verbose
repositories, releases := templates.GetHelmAppsConfigData()
config.Repositories = repositories
config.Releases = releases
templates.ApplyK8sTemplates()
@@ -30,9 +42,10 @@ func main() {
switch cmd {
case "apply":
kubespray.InstallCluster("")
installAndConfigureModules()
return
case "apply-modules":
kubespray.InstallCluster("helm-apps")
installAndConfigureModules()
return
case "upgrade":
kubespray.UpgradeCluster("")
@@ -42,5 +55,5 @@ func main() {
return
}
}
fmt.Println("No such command\nAvailable commands: apply, apply-modules, upgrade, scale")
logging.Log.Error("No such command\nAvailable commands: apply, apply-modules, upgrade, scale")
}

4
go.mod
View File

@@ -11,7 +11,9 @@ require (
github.com/apenella/go-ansible/v2 v2.0.0
github.com/ilyakaznacheev/cleanenv v1.5.0
github.com/mittwald/go-helm-client v0.12.9
github.com/sirupsen/logrus v1.9.3
golang.org/x/crypto v0.22.0
helm.sh/helm/v3 v3.14.2
k8s.io/api v0.30.0
k8s.io/apimachinery v0.30.0
k8s.io/client-go v0.30.0
@@ -113,7 +115,6 @@ require (
github.com/rubenv/sql-migrate v1.6.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/shopspring/decimal v1.3.1 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/spf13/cast v1.6.0 // indirect
github.com/spf13/cobra v1.8.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
@@ -144,7 +145,6 @@ require (
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
helm.sh/helm/v3 v3.14.2 // indirect
k8s.io/apiextensions-apiserver v0.29.0 // indirect
k8s.io/apiserver v0.29.0 // indirect
k8s.io/cli-runtime v0.29.0 // indirect

View File

@@ -0,0 +1,79 @@
package additional
import (
"kube-forge/internal/config"
"kube-forge/internal/helm_client"
"kube-forge/internal/templates"
"time"
go_helm_client "github.com/mittwald/go-helm-client"
)
var HELM_REPOS = []config.RepoSettings{
{
Name: "kube-forge",
URL: "https://git.kvazaric.ru/api/v4/projects/41/packages/helm/stable",
},
}
func getCertManagerSpec() go_helm_client.ChartSpec {
appConfig := config.GetConfig()
return go_helm_client.ChartSpec{
ReleaseName: "cert-manager",
ChartName: appConfig.Modules.Additional.CertManager.ChartRef,
Version: appConfig.Modules.Additional.CertManager.ChartVersion,
Namespace: appConfig.Modules.Additional.CertManager.Namespace,
CreateNamespace: true,
Atomic: true,
Timeout: time.Second * 600,
ValuesYaml: templates.GetHelmValuesByTemplate("templates/helm-apps/releases/additional-modules/cert-manager.yml.tmpl"),
}
}
func getIngressNginxSpec() go_helm_client.ChartSpec {
appConfig := config.GetConfig()
return go_helm_client.ChartSpec{
ReleaseName: "ingress-nginx",
ChartName: appConfig.Modules.Additional.Ingress.ChartRef,
Version: appConfig.Modules.Additional.Ingress.ChartVersion,
Namespace: appConfig.Modules.Additional.Ingress.Namespace,
CreateNamespace: true,
Atomic: true,
Timeout: time.Second * 600,
ValuesYaml: templates.GetHelmValuesByTemplate("templates/helm-apps/releases/additional-modules/ingress-nginx.yml.tmpl"),
}
}
func getDockerSecretsGeneratorSpec() go_helm_client.ChartSpec {
appConfig := config.GetConfig()
return go_helm_client.ChartSpec{
ReleaseName: "docker-secrets-generator",
ChartName: appConfig.Modules.Additional.DockerSecrets.ChartRef,
Version: appConfig.Modules.Additional.DockerSecrets.ChartVersion,
Namespace: appConfig.Modules.Additional.DockerSecrets.Namespace,
CreateNamespace: true,
Atomic: true,
Timeout: time.Second * 600,
ValuesYaml: templates.GetHelmValuesByTemplate("templates/helm-apps/releases/additional-modules/docker-secrets-generator.yml.tmpl"),
}
}
func ApplyCharts() {
appConfig := config.GetConfig()
helm_client.AddHelmRepos("kube-system", HELM_REPOS)
if appConfig.Modules.Additional.CertManager.Enabled {
helm_client.InstallChart(getCertManagerSpec())
} else {
helm_client.DeleteChart(getCertManagerSpec())
}
if appConfig.Modules.Additional.Ingress.Enabled && appConfig.Modules.Additional.Ingress.Type == "nginx" {
helm_client.InstallChart(getIngressNginxSpec())
} else {
helm_client.DeleteChart(getIngressNginxSpec())
}
if appConfig.Modules.Additional.DockerSecrets.Repositories != nil {
helm_client.InstallChart(getDockerSecretsGeneratorSpec())
} else {
helm_client.DeleteChart(getDockerSecretsGeneratorSpec())
}
}

108
internal/cicd/helm.go Normal file
View File

@@ -0,0 +1,108 @@
package cicd
import (
"kube-forge/internal/config"
"kube-forge/internal/helm_client"
"kube-forge/internal/templates"
"time"
go_helm_client "github.com/mittwald/go-helm-client"
)
var HELM_REPOS = [...]config.RepoSettings{
{
Name: "kube-forge",
URL: "https://git.kvazaric.ru/api/v4/projects/41/packages/helm/stable",
},
}
func getArgoCdSpec() go_helm_client.ChartSpec {
appConfig := config.GetConfig()
return go_helm_client.ChartSpec{
ReleaseName: "argo-cd",
ChartName: appConfig.Modules.Cicd.ArgoCd.ChartRef,
Version: appConfig.Modules.Cicd.ArgoCd.ChartVersion,
Namespace: appConfig.Modules.Cicd.Namespace,
CreateNamespace: true,
Atomic: true,
Timeout: time.Second * 600,
ValuesYaml: templates.GetHelmValuesByTemplate("templates/helm-apps/releases/cicd/argo-cd.yml.tmpl"),
}
}
func getArgoRolloutsSpec() go_helm_client.ChartSpec {
appConfig := config.GetConfig()
return go_helm_client.ChartSpec{
ReleaseName: "argo-rollouts",
ChartName: appConfig.Modules.Cicd.Rollouts.ChartRef,
Version: appConfig.Modules.Cicd.Rollouts.ChartVersion,
Namespace: appConfig.Modules.Cicd.Namespace,
CreateNamespace: true,
Atomic: true,
Timeout: time.Second * 600,
ValuesYaml: templates.GetHelmValuesByTemplate("templates/helm-apps/releases/cicd/argo-rollouts.yml.tmpl"),
}
}
func getKeelSpec() go_helm_client.ChartSpec {
appConfig := config.GetConfig()
return go_helm_client.ChartSpec{
ReleaseName: "keel",
ChartName: appConfig.Modules.Cicd.UpdatesOperator.ChartRef,
Version: appConfig.Modules.Cicd.UpdatesOperator.ChartVersion,
Namespace: appConfig.Modules.Cicd.UpdatesOperator.Namespace,
CreateNamespace: true,
Atomic: true,
Timeout: time.Second * 600,
ValuesYaml: templates.GetHelmValuesByTemplate("templates/helm-apps/releases/cicd/keel.yml.tmpl"),
}
}
func getArgoCdIngressSpec() go_helm_client.ChartSpec {
appConfig := config.GetConfig()
return go_helm_client.ChartSpec{
ReleaseName: "argo-cd-ingress",
ChartName: appConfig.Modules.Cicd.ArgoCd.ServiceIngress.ChartRef,
Version: appConfig.Modules.Cicd.ArgoCd.ServiceIngress.ChartVersion,
Namespace: appConfig.Modules.Cicd.Namespace,
CreateNamespace: true,
Atomic: true,
Timeout: time.Second * 600,
ValuesYaml: templates.GetHelmValuesByTemplate("templates/helm-apps/releases/cicd/argo-cd-ingress.yml.tmpl"),
}
}
func addCicdHelmRepos() {
appConfig := config.GetConfig()
for _, repoSettings := range HELM_REPOS {
helm_client.AddHelmRepo(appConfig.Modules.SecretsStorage.Namespace, repoSettings)
}
}
func ApplyCharts() {
addCicdHelmRepos()
appConfig := config.GetConfig()
if appConfig.Modules.Cicd.Enabled {
helm_client.InstallChart(getArgoCdSpec())
if appConfig.Modules.Cicd.ArgoCd.Expose.Type == "ingress" {
helm_client.InstallChart(getArgoCdIngressSpec())
} else {
helm_client.DeleteChart(getArgoCdIngressSpec())
}
if appConfig.Modules.Cicd.Rollouts.Enabled {
helm_client.InstallChart(getArgoRolloutsSpec())
} else {
helm_client.DeleteChart(getArgoRolloutsSpec())
}
if appConfig.Modules.Cicd.UpdatesOperator.Enabled {
helm_client.InstallChart(getKeelSpec())
} else {
helm_client.DeleteChart(getKeelSpec())
}
} else {
helm_client.DeleteChart(getArgoCdIngressSpec())
helm_client.DeleteChart(getArgoCdSpec())
helm_client.DeleteChart(getArgoRolloutsSpec())
helm_client.DeleteChart(getKeelSpec())
}
}

View File

@@ -38,9 +38,6 @@ type Config struct {
Cicd Cicd `yaml:"cicd"`
SecretsStorage SecretsStorage `yaml:"secrets_storage"`
} `yaml:"modules"`
Repositories string
Releases string
}
var instance *Config

View File

@@ -5,13 +5,30 @@ import (
"log"
"os"
"path/filepath"
"time"
helm_client "github.com/mittwald/go-helm-client"
)
var client *helm_client.Client
type RepoSettings struct {
Name string
URL string
Username string
Password string
}
func CreateHelmClient() helm_client.Client {
type ChartSettings struct {
ReleaseName string
ChartRef string
ChartVersion string
Namespace string
CreateNamespace bool
Atomic bool
Timeout time.Duration
ValuesYaml string
}
func GetHelmClient(namespace string) helm_client.Client {
config := GetConfig()
file, err := os.Open(filepath.Join(config.WorkDir, config.KubeconfigFile))
if err != nil {
@@ -24,7 +41,7 @@ func CreateHelmClient() helm_client.Client {
}
opts := &helm_client.KubeConfClientOptions{
Options: &helm_client.Options{
Namespace: "default", // Change this to the namespace you wish to install the chart in.
Namespace: namespace,
RepositoryCache: "/tmp/.helmcache",
RepositoryConfig: "/tmp/.helmrepo",
Debug: true,
@@ -40,9 +57,6 @@ func CreateHelmClient() helm_client.Client {
if err != nil {
log.Fatalf("error while creating helm client: %s", err)
}
return client
}
func GetHelmClient() *helm_client.Client {
return client
}

View File

@@ -5,6 +5,7 @@ type Additional struct {
ChartRef string `yaml:"chart_ref" env-default:"kube-forge/cert-manager"`
ChartVersion string `yaml:"chart_version" env-default:"v1.14.5"`
Enabled bool `yaml:"enabled"`
Namespace string `yaml:"namespace" env-default:"cert-manager"`
HaEnabled bool `yaml:"ha_enabled"`
DnsServers []string `yaml:"dns_servers" env-default:"8.8.8.8,1.1.1.1"`
AccountEmail string `yaml:"account_email"`
@@ -19,6 +20,7 @@ type Additional struct {
ChartVersion string `yaml:"chart_version" env-default:"4.10.1"`
Type string `yaml:"type" env-default:"nginx"`
Enabled bool `yaml:"enabled"`
Namespace string `yaml:"namespace" env-default:"ingress-nginx"`
IngressClassName string `yaml:"ingress_class_name" env-default:"nginx"`
HostNetwork bool `yaml:"host_network"`
EnableAdmissionWebhooks bool `yaml:"enable_admission_webhooks"`
@@ -42,6 +44,7 @@ type Additional struct {
DockerSecrets struct {
ChartRef string `yaml:"chart_ref" env-default:"kube-forge/docker-secrets-generator"`
ChartVersion string `yaml:"chart_version" env-default:"0.1.0"`
Namespace string `yaml:"namespace" env-default:"kube-system"`
Repositories interface{} `yaml:"repositories"`
} `yaml:"docker_secrets"`
@@ -54,11 +57,13 @@ type Additional struct {
ChartRef string `yaml:"chart_ref" env-default:"kube-forge/longhorn"`
ChartVersion string `yaml:"chart_version" env-default:"1.6.1"`
Enabled bool `yaml:"enabled"`
Namespace string `yaml:"namespace" env-default:"longhorn-system"`
} `yaml:"longhorn"`
SecretsStoreCsiDriver struct {
ChartRef string `yaml:"chart_ref" env-default:"kube-forge/secrets-store-csi-driver"`
ChartVersion string `yaml:"chart_version" env-default:"1.4.3"`
Enabled bool `yaml:"enabled"`
Namespace string `yaml:"namespace" env-default:"kube-system"`
} `yaml:"secrets_store_csi_driver"`
} `yaml:"storage"`
}

View File

@@ -2,6 +2,7 @@ package config
type Cicd struct {
Enabled bool `yaml:"enabled"`
Namespace string `yaml:"namespace" env-default:"cicd"`
ArgoCd struct {
ChartRef string `yaml:"chart_ref" env-default:"kube-forge/argo-cd"`
ChartVersion string `yaml:"chart_version" env-default:"6.7.10"`
@@ -69,6 +70,7 @@ type Cicd struct {
ChartRef string `yaml:"chart_ref" env-default:"kube-forge/keel"`
ChartVersion string `yaml:"chart_version" env-default:"1.0.3"`
Enabled bool `yaml:"enabled"`
Namespace string `yaml:"namespace" env-default:"kube-system"`
Image string `yaml:"image" env-default:"harbor.kvazaric.ru/kube-forge/keelhq/keel"`
Tag string `yaml:"tag" env-default:"latest"`
} `yaml:"updates_operator"`

View File

@@ -8,6 +8,7 @@ type Observability struct {
Tracing Tracing `yaml:"tracing"`
Monitoring Monitoring `yaml:"monitoring"`
Visualization Visualization `yaml:"visualization"`
Namespace string `yaml:"namespace" env-default:"observability"`
}
type Logging struct {
@@ -17,6 +18,7 @@ type Logging struct {
ChartVersion string `yaml:"chart_version" env-default:"2.7.0"`
Image string `yaml:"image" env-default:"harbor.kvazaric.ru/kube-forge/kubesphere/fluent-operator"`
Tag string `yaml:"tag" env-default:"v2.7.0"`
Namespace string `yaml:"namespace" env-default:"observability"`
InitContainer struct {
Image string `yaml:"image" env-default:"harbor.kvazaric.ru/kube-forge/docker"`
Tag string `yaml:"tag" env-default:"20.10"`
@@ -36,6 +38,7 @@ type Logging struct {
Registry string `yaml:"registry" env-default:"harbor.kvazaric.ru"`
Image string `yaml:"image" env-default:"kube-forge/grafana/loki"`
Tag string `yaml:"tag" env-default:"latest"`
Namespace string `yaml:"namespace" env-default:"observability"`
Persistence struct {
StorageClass string `yaml:"storage_class" env-default:"local-path"`
StorageSize string `yaml:"storage_size" env-default:"10Gi"`
@@ -65,6 +68,7 @@ type Tracing struct {
ChartVersion string `yaml:"chart_version" env-default:"0.55.0"`
Image string `yaml:"image" env-default:"ghcr.io/open-telemetry/opentelemetry-operator/opentelemetry-operator"`
Tag string `yaml:"tag" env-default:""`
Namespace string `yaml:"namespace" env-default:"observability"`
} `yaml:"operator"`
Collector struct {
Image string `yaml:"image" env-default:"harbor.kvazaric.ru/kube-forge/otel/opentelemetry-collector-contrib"`
@@ -75,6 +79,7 @@ type Tracing struct {
ChartVersion string `yaml:"chart_version" env-default:"1.7.2"`
Image string `yaml:"image" env-default:"harbor.kvazaric.ru/kube-forge/grafana/tempo"`
Tag string `yaml:"tag" env-default:"2.3.1"`
Namespace string `yaml:"namespace" env-default:"observability"`
Retention string `yaml:"retention" env-default:"24h"`
ListenPort int `yaml:"listen_port" env-default:"3100"`
Persistence struct {
@@ -142,6 +147,7 @@ type Monitoring struct {
ChartVersion string `yaml:"chart_version" env-default:"3.12.1"`
Image string `yaml:"image" env-default:"registry.k8s.io/metrics-server/metrics-server"`
Tag string `yaml:"tag" env-default:""`
Namespace string `yaml:"namespace" env-default:"kube-system"`
}
}

View File

@@ -3,6 +3,7 @@ package config
type SecretsStorage struct {
ChartRef string `yaml:"chart_ref" env-default:"kube-forge/vault"`
ChartVersion string `yaml:"chart_version" env-default:"0.1.0"`
Namespace string `yaml:"namespace" env-default:"secrets-storage"`
Enabled bool `yaml:"enabled"`
KeyShares int `yaml:"key_shares" env-default:"5"`
KeyThreshold int `yaml:"key_threshold" env-default:"3"`

60
internal/csi/helm.go Normal file
View File

@@ -0,0 +1,60 @@
package csi
import (
"kube-forge/internal/config"
"kube-forge/internal/helm_client"
"kube-forge/internal/templates"
"time"
go_helm_client "github.com/mittwald/go-helm-client"
)
var HELM_REPOS = []config.RepoSettings{
{
Name: "kube-forge",
URL: "https://git.kvazaric.ru/api/v4/projects/41/packages/helm/stable",
},
}
func getLonghornSpec() go_helm_client.ChartSpec {
appConfig := config.GetConfig()
return go_helm_client.ChartSpec{
ReleaseName: "longhorn",
ChartName: appConfig.Modules.Additional.Storage.Longhorn.ChartRef,
Version: appConfig.Modules.Additional.Storage.Longhorn.ChartVersion,
Namespace: appConfig.Modules.Additional.Storage.Longhorn.Namespace,
CreateNamespace: true,
Atomic: true,
Timeout: time.Second * 600,
ValuesYaml: templates.GetHelmValuesByTemplate("templates/helm-apps/releases/additional-modules/longhorn.yml.tmpl"),
}
}
func getSecretsStoreSpec() go_helm_client.ChartSpec {
appConfig := config.GetConfig()
return go_helm_client.ChartSpec{
ReleaseName: "csi-secrets-store",
ChartName: appConfig.Modules.Additional.Storage.SecretsStoreCsiDriver.ChartRef,
Version: appConfig.Modules.Additional.Storage.SecretsStoreCsiDriver.ChartVersion,
Namespace: appConfig.Modules.Additional.Storage.SecretsStoreCsiDriver.Namespace,
CreateNamespace: true,
Atomic: true,
Timeout: time.Second * 60,
ValuesYaml: templates.GetHelmValuesByTemplate("templates/helm-apps/releases/additional-modules/secrets-store-csi-driver.yml.tmpl"),
}
}
func ApplyCharts() {
appConfig := config.GetConfig()
helm_client.AddHelmRepos("kube-system", HELM_REPOS)
if appConfig.Modules.Additional.Storage.Longhorn.Enabled {
helm_client.InstallChart(getLonghornSpec())
} else {
helm_client.DeleteChart(getLonghornSpec())
}
if appConfig.Modules.Additional.Storage.SecretsStoreCsiDriver.Enabled {
helm_client.InstallChart(getSecretsStoreSpec())
} else {
helm_client.DeleteChart(getSecretsStoreSpec())
}
}

View File

@@ -0,0 +1,32 @@
package helm_client
import (
"context"
"kube-forge/internal/config"
"kube-forge/internal/logging"
go_helm_client "github.com/mittwald/go-helm-client"
)
func InstallChart(chartSpec go_helm_client.ChartSpec) {
helmClient := config.GetHelmClient(chartSpec.Namespace)
_, error := helmClient.GetRelease(chartSpec.ReleaseName)
if error != nil {
logging.Log.Infof("Installing %s", chartSpec.ChartName)
} else {
logging.Log.Infof("Upgrading %s", chartSpec.ChartName)
}
if _, err := helmClient.InstallOrUpgradeChart(context.Background(), &chartSpec, nil); err != nil {
panic(err)
}
}
func DeleteChart(chartSpec go_helm_client.ChartSpec) {
helmClient := config.GetHelmClient(chartSpec.Namespace)
_, error := helmClient.GetRelease(chartSpec.ReleaseName)
if error != nil {
return
}
logging.Log.Warnf("Uninstalling %s", chartSpec.ChartName)
helmClient.UninstallRelease(&chartSpec)
}

View File

@@ -0,0 +1,27 @@
package helm_client
import (
"kube-forge/internal/config"
"helm.sh/helm/v3/pkg/repo"
)
func AddHelmRepo(namespace string, repoSettings config.RepoSettings) {
helmClient := config.GetHelmClient(namespace)
chartRepo := repo.Entry{
Name: repoSettings.Name,
URL: repoSettings.URL,
Username: repoSettings.Username,
Password: repoSettings.Password,
}
if err := helmClient.AddOrUpdateChartRepo(chartRepo); err != nil {
panic(err)
}
}
func AddHelmRepos(namespace string, helmRepos []config.RepoSettings) {
for _, repoSettings := range helmRepos {
AddHelmRepo(namespace, repoSettings)
}
}

View File

@@ -1,22 +1,13 @@
package kubespray
import (
"fmt"
"kube-forge/internal/config"
"kube-forge/internal/secrets_storage"
)
func InstallCluster(tags string) {
appConfig := config.GetConfig()
runPlaybook("kubespray/project/cluster.yml", tags)
CopyK8SAdminConfig(appConfig.KubeconfigFile)
config.CreateHelmClient()
if appConfig.Modules.SecretsStorage.Enabled {
fmt.Println("## Additional Vault Configuration")
secrets_storage.InitVault()
secrets_storage.UnsealVault()
secrets_storage.AddKubernetesLocalIntegration()
}
}
func UpgradeCluster(tags string) {
@@ -26,7 +17,7 @@ func UpgradeCluster(tags string) {
}
func ScaleCluster() {
runPlaybook("kubespray/project/scale.yml", "")
appConfig := config.GetConfig()
runPlaybook("kubespray/project/scale.yml", "")
CopyK8SAdminConfig(appConfig.KubeconfigFile)
}

12
internal/logging/log.go Normal file
View File

@@ -0,0 +1,12 @@
package logging
import (
"github.com/sirupsen/logrus"
)
var Log = logrus.New()
func init() {
Log.SetLevel(logrus.InfoLevel)
Log.SetFormatter(&logrus.TextFormatter{})
}

View File

@@ -0,0 +1,122 @@
package observability
import (
"kube-forge/internal/config"
"kube-forge/internal/helm_client"
"kube-forge/internal/templates"
"time"
go_helm_client "github.com/mittwald/go-helm-client"
)
var HELM_REPOS = []config.RepoSettings{
{
Name: "kube-forge",
URL: "https://git.kvazaric.ru/api/v4/projects/41/packages/helm/stable",
},
}
func getFluentOperatorSpec() go_helm_client.ChartSpec {
appConfig := config.GetConfig()
return go_helm_client.ChartSpec{
ReleaseName: "fluent-operator",
ChartName: appConfig.Modules.Observability.Logging.Operator.ChartRef,
Version: appConfig.Modules.Observability.Logging.Operator.ChartVersion,
Namespace: appConfig.Modules.Observability.Logging.Operator.Namespace,
CreateNamespace: true,
Atomic: true,
UpgradeCRDs: true,
Timeout: time.Second * 60,
ValuesYaml: templates.GetHelmValuesByTemplate("templates/helm-apps/releases/observability/fluent-operator.yml.tmpl"),
}
}
func getOpenTelemetryOperatorSpec() go_helm_client.ChartSpec {
appConfig := config.GetConfig()
return go_helm_client.ChartSpec{
ReleaseName: "opentelemetry-operator",
ChartName: appConfig.Modules.Observability.Tracing.Operator.ChartRef,
Version: appConfig.Modules.Observability.Tracing.Operator.ChartVersion,
Namespace: appConfig.Modules.Observability.Tracing.Operator.Namespace,
CreateNamespace: true,
Atomic: true,
UpgradeCRDs: true,
Timeout: time.Second * 60,
ValuesYaml: templates.GetHelmValuesByTemplate("templates/helm-apps/releases/observability/opentelemetry-operator.yml.tmpl"),
}
}
func getMetricsServerSpec() go_helm_client.ChartSpec {
appConfig := config.GetConfig()
return go_helm_client.ChartSpec{
ReleaseName: "metrics-server",
ChartName: appConfig.Modules.Observability.Monitoring.MetricsServer.ChartRef,
Version: appConfig.Modules.Observability.Monitoring.MetricsServer.ChartVersion,
Namespace: appConfig.Modules.Observability.Monitoring.MetricsServer.Namespace,
CreateNamespace: true,
Atomic: true,
Timeout: time.Second * 60,
ValuesYaml: templates.GetHelmValuesByTemplate("templates/helm-apps/releases/observability/metrics-server.yml.tmpl"),
}
}
func getTempoSpec() go_helm_client.ChartSpec {
appConfig := config.GetConfig()
return go_helm_client.ChartSpec{
ReleaseName: "tempo",
ChartName: appConfig.Modules.Observability.Tracing.Tempo.ChartRef,
Version: appConfig.Modules.Observability.Tracing.Tempo.ChartVersion,
Namespace: appConfig.Modules.Observability.Tracing.Tempo.Namespace,
CreateNamespace: true,
Atomic: true,
Timeout: time.Second * 60,
ValuesYaml: templates.GetHelmValuesByTemplate("templates/helm-apps/releases/observability/tempo.yml.tmpl"),
}
}
func getLokiSpec() go_helm_client.ChartSpec {
appConfig := config.GetConfig()
return go_helm_client.ChartSpec{
ReleaseName: "loki",
ChartName: appConfig.Modules.Observability.Logging.Loki.ChartRef,
Version: appConfig.Modules.Observability.Logging.Loki.ChartVersion,
Namespace: appConfig.Modules.Observability.Logging.Loki.Namespace,
CreateNamespace: true,
Atomic: true,
Timeout: time.Second * 60,
ValuesYaml: templates.GetHelmValuesByTemplate("templates/helm-apps/releases/observability/loki.yml.tmpl"),
}
}
func getObservabilitySpec() go_helm_client.ChartSpec {
appConfig := config.GetConfig()
return go_helm_client.ChartSpec{
ReleaseName: "observability",
ChartName: appConfig.Modules.Observability.ChartRef,
Version: appConfig.Modules.Observability.ChartVersion,
Namespace: appConfig.Modules.Observability.Namespace,
CreateNamespace: true,
Atomic: true,
Timeout: time.Second * 600,
ValuesYaml: templates.GetHelmValuesByTemplate("templates/helm-apps/releases/observability/observability.yml.tmpl"),
}
}
func ApplyCharts() {
appConfig := config.GetConfig()
helm_client.AddHelmRepos(appConfig.Modules.Registry.Namespace, HELM_REPOS)
if appConfig.Modules.Observability.Enabled {
if appConfig.Modules.Observability.Logging.Enabled {
helm_client.InstallChart(getFluentOperatorSpec())
helm_client.InstallChart(getLokiSpec())
}
if appConfig.Modules.Observability.Tracing.Enabled {
helm_client.InstallChart(getOpenTelemetryOperatorSpec())
helm_client.InstallChart(getTempoSpec())
}
if appConfig.Modules.Observability.Monitoring.Enabled {
helm_client.InstallChart(getMetricsServerSpec())
}
helm_client.InstallChart(getObservabilitySpec())
}
}

View File

@@ -1,5 +0,0 @@
package registry
func InstallRegistryCharts() {
}

View File

@@ -1,9 +0,0 @@
package registry
func CreateDockerRepositories() {
}
func CreateHelmRepositories() {
}

61
internal/registry/helm.go Normal file
View File

@@ -0,0 +1,61 @@
package registry
import (
"kube-forge/internal/config"
"kube-forge/internal/helm_client"
"kube-forge/internal/templates"
"time"
go_helm_client "github.com/mittwald/go-helm-client"
)
var HELM_REPOS = []config.RepoSettings{
{
Name: "kube-forge",
URL: "https://git.kvazaric.ru/api/v4/projects/41/packages/helm/stable",
},
}
func getHarborCertificateGeneratorSpec() go_helm_client.ChartSpec {
appConfig := config.GetConfig()
return go_helm_client.ChartSpec{
ReleaseName: "harbor-certificate-generator",
ChartName: appConfig.Modules.Registry.Tls.CertificateGenerator.ChartRef,
Version: appConfig.Modules.Registry.Tls.CertificateGenerator.ChartVersion,
Namespace: appConfig.Modules.Registry.Namespace,
CreateNamespace: true,
Atomic: true,
Timeout: time.Second * 60,
ValuesYaml: templates.GetHelmValuesByTemplate("templates/helm-apps/releases/registry/harbor-certificate-generator.yml.tmpl"),
}
}
func getHarborSpec() go_helm_client.ChartSpec {
appConfig := config.GetConfig()
return go_helm_client.ChartSpec{
ReleaseName: "harbor",
ChartName: appConfig.Modules.Registry.ChartRef,
Version: appConfig.Modules.Registry.ChartVersion,
Namespace: appConfig.Modules.Registry.Namespace,
CreateNamespace: true,
Atomic: true,
Timeout: time.Second * 600,
ValuesYaml: templates.GetHelmValuesByTemplate("templates/helm-apps/releases/registry/harbor.yml.tmpl"),
}
}
func ApplyCharts() {
appConfig := config.GetConfig()
helm_client.AddHelmRepos(appConfig.Modules.Registry.Namespace, HELM_REPOS)
if appConfig.Modules.Registry.Enabled {
if appConfig.Modules.Registry.Expose.Type == "ingress" {
helm_client.InstallChart(getHarborCertificateGeneratorSpec())
} else {
helm_client.DeleteChart(getHarborCertificateGeneratorSpec())
}
helm_client.InstallChart(getHarborSpec())
} else {
helm_client.DeleteChart(getHarborSpec())
helm_client.DeleteChart(getHarborCertificateGeneratorSpec())
}
}

View File

@@ -1,14 +1,3 @@
- name: cert-manager
namespace: cert-manager
create_namespace: true
chart_ref: {{ .Modules.Additional.CertManager.ChartRef }}
chart_version: {{ .Modules.Additional.CertManager.ChartVersion }}
{{- if .Modules.Additional.CertManager.Enabled }}
release_state: "present"
{{- else }}
release_state: "absent"
{{- end }}
values:
global:
imagePullSecrets: []

View File

@@ -1,14 +1,2 @@
- name: docker-secrets-generator
namespace: kube-system
chart_ref: {{ .Modules.Additional.DockerSecrets.ChartRef }}
chart_version: {{ .Modules.Additional.DockerSecrets.ChartVersion }}
{{- if .Modules.Additional.DockerSecrets.Repositories }}
release_state: "present"
{{- else }}
release_state: "absent"
{{- end }}
{{- if .Modules.Additional.DockerSecrets.Repositories }}
values:
repositories:
{{- .Modules.Additional.DockerSecrets.Repositories | toYaml | nindent 6 }}
{{- end }}

View File

@@ -1,14 +1,3 @@
- name: ingress-nginx
namespace: ingress-nginx
create_namespace: true
chart_ref: {{ .Modules.Additional.Ingress.ChartRef }}
chart_version: {{ .Modules.Additional.Ingress.ChartVersion }}
{{- if and .Modules.Additional.Ingress.Enabled (eq .Modules.Additional.Ingress.Type "nginx") }}
release_state: "present"
{{- else }}
release_state: "absent"
{{- end }}
values:
commonLabels: {}
controller:
name: controller
@@ -20,8 +9,6 @@
## use *either* current default registry/image or repository format or installing chart by providing the values.yaml will fail
repository: {{ .Modules.Additional.Ingress.Nginx.Controller.Image }}
tag: "{{ .Modules.Additional.Ingress.Nginx.Controller.Tag }}"
digest: sha256:5b161f051d017e55d358435f295f5e9a297e66158f136321d9b04520ec6c48a3
digestChroot: sha256:5976b1067cfbca8a21d0ba53d71f83543a73316a61ea7f7e436d6cf84ddf9b26
pullPolicy: IfNotPresent
# www-data -> uid 101
runAsUser: 101
@@ -556,7 +543,7 @@
opentelemetry:
enabled: false
image: registry.k8s.io/ingress-nginx/opentelemetry:v20230721-3e2062ee5@sha256:13bee3f5223883d3ca62fee7309ad02d22ec00ff0d7033e3e9aca7a9f60fd472
# image: registry.k8s.io/ingress-nginx/opentelemetry:v20230721-3e2062ee5@sha256:13bee3f5223883d3ca62fee7309ad02d22ec00ff0d7033e3e9aca7a9f60fd472
containerSecurityContext:
allowPrivilegeEscalation: false
resources: {}

View File

@@ -1,14 +1,3 @@
- name: longhorn
namespace: longhorn-system
create_namespace: true
chart_ref: {{ .Modules.Additional.Storage.Longhorn.ChartRef }}
chart_version: {{ .Modules.Additional.Storage.Longhorn.ChartVersion }}
{{- if .Modules.Additional.Storage.Longhorn.Enabled }}
release_state: "present"
{{- else }}
release_state: "absent"
{{- end }}
values:
global:
cattle:
# -- Default system registry.

View File

@@ -1,13 +1,3 @@
- name: csi-secrets-store
namespace: kube-system
chart_ref: {{ .Modules.Additional.Storage.SecretsStoreCsiDriver.ChartRef }}
chart_version: {{ .Modules.Additional.Storage.SecretsStoreCsiDriver.ChartVersion }}
{{- if .Modules.Additional.Storage.SecretsStoreCsiDriver.Enabled }}
release_state: "present"
{{- else }}
release_state: "absent"
{{- end }}
values:
linux:
enabled: true
image:

View File

@@ -1,14 +1,3 @@
- name: argo-cd-ingress
namespace: cicd
create_namespace: true
chart_ref: {{ .Modules.Cicd.ArgoCd.ServiceIngress.ChartRef }}
chart_version: {{ .Modules.Cicd.ArgoCd.ServiceIngress.ChartVersion }}
{{- if and .Modules.Cicd.Enabled (eq .Modules.Cicd.ArgoCd.Expose.Type "ingress") }}
release_state: "present"
{{- else }}
release_state: "absent"
{{- end }}
values:
services:
- domain: {{ .Modules.Cicd.ArgoCd.Expose.Domain }}
path: {{ .Modules.Cicd.ArgoCd.Expose.Path }}

View File

@@ -1,14 +1,3 @@
- name: argo-cd
namespace: cicd
create_namespace: true
chart_ref: {{ .Modules.Cicd.ArgoCd.ChartRef }}
chart_version: {{ .Modules.Cicd.ArgoCd.ChartVersion }}
{{- if .Modules.Cicd.Enabled }}
release_state: "present"
{{- else }}
release_state: "absent"
{{- end }}
values:
crds:
install: true

View File

@@ -1,14 +1,3 @@
- name: argo-rollouts
namespace: cicd
create_namespace: true
chart_ref: {{ .Modules.Cicd.Rollouts.ChartRef }}
chart_version: {{ .Modules.Cicd.Rollouts.ChartVersion }}
{{- if and .Modules.Cicd.Enabled .Modules.Cicd.Rollouts.Enabled }}
release_state: "present"
{{- else }}
release_state: "absent"
{{- end }}
values:
installCRDs: true
keepCRDs: false
clusterInstall: true

View File

@@ -1,13 +1,3 @@
- name: keel
namespace: kube-system
chart_ref: {{ .Modules.Cicd.UpdatesOperator.ChartRef }}
chart_version: {{ .Modules.Cicd.UpdatesOperator.ChartVersion }}
{{- if and .Modules.Cicd.Enabled .Modules.Cicd.UpdatesOperator.Enabled }}
release_state: "present"
{{- else }}
release_state: "absent"
{{- end }}
values:
image:
repository: {{ .Modules.Cicd.UpdatesOperator.Image }}
tag: {{ .Modules.Cicd.UpdatesOperator.Tag }}

View File

@@ -1,14 +1,3 @@
- name: fluent-operator
namespace: observability
create_namespace: true
chart_ref: {{ .Modules.Observability.Logging.Operator.ChartRef }}
chart_version: {{ .Modules.Observability.Logging.Operator.ChartVersion }}
{{- if and .Modules.Observability.Enabled .Modules.Observability.Logging.Enabled }}
release_state: "present"
{{- else }}
release_state: "absent"
{{- end }}
values:
# Set this to containerd or crio if you want to collect CRI format logs
containerRuntime: {{ .Orchestrator.ContainerEngine.Type }}
Kubernetes: false

View File

@@ -1,14 +1,3 @@
- name: loki
namespace: observability
create_namespace: true
chart_ref: {{ .Modules.Observability.Logging.Loki.ChartRef }}
chart_version: {{ .Modules.Observability.Logging.Loki.ChartVersion }}
{{- if and .Modules.Observability.Enabled .Modules.Observability.Logging.Enabled }}
release_state: "present"
{{- else }}
release_state: "absent"
{{- end }}
values:
loki:
image:
registry: {{ .Modules.Observability.Logging.Loki.Registry }}
@@ -88,7 +77,6 @@
rules:
enabled: true
alerting: true
additionalGroups: {}
extraObjects:
- apiVersion: v1

View File

@@ -1,14 +1,3 @@
- name: metrics-server
namespace: kube-system
create_namespace: true
chart_ref: {{ .Modules.Observability.Monitoring.MetricsServer.ChartRef }}
chart_version: {{ .Modules.Observability.Monitoring.MetricsServer.ChartVersion }}
{{- if and .Modules.Observability.Enabled .Modules.Observability.Monitoring.Enabled }}
release_state: "present"
{{- else }}
release_state: "absent"
{{- end }}
values:
image:
repository: {{.Modules.Observability.Monitoring.MetricsServer.Image }}
tag: "{{ .Modules.Observability.Monitoring.MetricsServer.Tag }}"

View File

@@ -1,14 +1,3 @@
- name: observability
namespace: observability
create_namespace: true
chart_ref: {{ .Modules.Observability.ChartRef }}
chart_version: {{ .Modules.Observability.ChartVersion }}
{{- if .Modules.Observability.Enabled }}
release_state: "present"
{{- else }}
release_state: "absent"
{{- end }}
values:
prometheus:
enabled: {{ .Modules.Observability.Monitoring.Enabled }}
serviceMonitor: true

View File

@@ -1,14 +1,3 @@
- name: opentelemetry-operator
namespace: observability
create_namespace: true
chart_ref: {{ .Modules.Observability.Tracing.Operator.ChartRef }}
chart_version: {{ .Modules.Observability.Tracing.Operator.ChartVersion }}
{{- if and .Modules.Observability.Enabled .Modules.Observability.Tracing.Enabled }}
release_state: "present"
{{- else }}
release_state: "absent"
{{- end }}
values:
replicaCount: 1
nameOverride: ""
imagePullSecrets: []

View File

@@ -1,14 +1,3 @@
- name: tempo
namespace: observability
create_namespace: true
chart_ref: {{ .Modules.Observability.Tracing.Tempo.ChartRef }}
chart_version: {{ .Modules.Observability.Tracing.Tempo.ChartVersion }}
{{- if and .Modules.Observability.Enabled .Modules.Observability.Tracing.Enabled }}
release_state: "present"
{{- else }}
release_state: "absent"
{{- end }}
values:
replicas: 1
tempo:

View File

@@ -1,14 +1,3 @@
- name: harbor-certificate-generator
namespace: {{ .Modules.Registry.Namespace }}
create_namespace: true
chart_ref: {{ .Modules.Registry.Tls.CertificateGenerator.ChartRef }}
chart_version: {{ .Modules.Registry.Tls.CertificateGenerator.ChartVersion }}
{{- if and .Modules.Registry.Enabled (eq .Modules.Registry.Expose.Type "ingress") }}
release_state: "present"
{{- else }}
release_state: "absent"
{{- end }}
values:
issuer_email: {{ .Modules.Additional.CertManager.AccountEmail }}
solver_ingress_class: {{ .Modules.Additional.Ingress.Type }}

View File

@@ -1,14 +1,3 @@
- name: harbor
namespace: {{ .Modules.Registry.Namespace }}
create_namespace: true
chart_ref: {{ .Modules.Registry.ChartRef }}
chart_version: {{ .Modules.Registry.ChartVersion }}
{{- if .Modules.Registry.Enabled }}
release_state: "present"
{{- else }}
release_state: "absent"
{{- end }}
values:
expose:
type: {{ .Modules.Registry.Expose.Type }}
tls:

View File

@@ -1,14 +1,3 @@
- name: vault
namespace: secrets-storage
create_namespace: true
chart_ref: {{ .Modules.SecretsStorage.ChartRef }}
chart_version: {{ .Modules.SecretsStorage.ChartVersion }}
{{- if .Modules.SecretsStorage.Enabled }}
release_state: "present"
{{- else }}
release_state: "absent"
{{- end }}
values:
global:
enabled: true

View File

@@ -1,5 +0,0 @@
- name: kube-forge
url: "https://git.kvazaric.ru/api/v4/projects/41/packages/helm/stable"
{{ if .Modules.AdditionalRepositories }}
{{ .Modules.AdditionalRepositories | toYaml }}
{{- end }}

View File

@@ -232,13 +232,3 @@ argocd_enabled: false
# The plugin manager for kubectl
krew_enabled: false
krew_root_dir: "/usr/local/krew"
########################################
# Helm apps configuration
########################################
repositories:
{{- .Repositories | nindent 2 }}
releases:
{{- .Releases | nindent 2 }}

View File

@@ -4,44 +4,45 @@ import (
"fmt"
"kube-forge/internal/config"
"kube-forge/internal/kubernetes_client"
"kube-forge/internal/logging"
"kube-forge/internal/templates"
"regexp"
"strings"
)
func InitVault() {
func initVault() {
_, err := kubernetes_client.GetPodByName("vault-0", "secrets-storage")
if err != nil {
fmt.Println(err.Error())
logging.Log.Error(err.Error())
return
}
err = commandToInitVault()
if err != nil {
fmt.Println(err.Error())
logging.Log.Warn(err.Error())
return
}
fmt.Println("Vault initialized")
logging.Log.Info("Vault initialized")
templates.ApplyVaultInitKeysTemplate()
}
func AddKubernetesLocalIntegration() {
func addKubernetesLocalIntegration() {
_, err := kubernetes_client.GetPodByName("vault-0", "secrets-storage")
if err != nil {
fmt.Println(err.Error())
logging.Log.Error(err.Error())
return
}
err = commandToAddKubernetesLocalIntegration()
if err != nil {
fmt.Println(err.Error())
logging.Log.Error(err.Error())
return
}
fmt.Println("Vault local Kubernetes integration added")
logging.Log.Info("Vault local Kubernetes integration added")
}
func UnsealVault() {
func unsealVault() {
_, err := kubernetes_client.GetPodByName("vault-0", "secrets-storage")
if err != nil {
fmt.Println(err.Error())
logging.Log.Error(err.Error())
return
}
commandToUnsealVault()
@@ -92,7 +93,7 @@ func commandToUnsealVault() {
commandArray, "secrets-storage", "vault-0", "vault",
)
}
fmt.Println("Vault unsealed")
logging.Log.Info("Vault unsealed")
}
func commandToAddKubernetesLocalIntegration() error {

View File

@@ -0,0 +1,44 @@
package secrets_storage
import (
"kube-forge/internal/config"
"kube-forge/internal/helm_client"
"kube-forge/internal/templates"
"time"
go_helm_client "github.com/mittwald/go-helm-client"
)
var HELM_REPOS = []config.RepoSettings{
{
Name: "kube-forge",
URL: "https://git.kvazaric.ru/api/v4/projects/41/packages/helm/stable",
},
}
func getVaultSpec() go_helm_client.ChartSpec {
appConfig := config.GetConfig()
return go_helm_client.ChartSpec{
ReleaseName: "vault",
ChartName: appConfig.Modules.SecretsStorage.ChartRef,
Version: appConfig.Modules.SecretsStorage.ChartVersion,
Namespace: appConfig.Modules.SecretsStorage.Namespace,
CreateNamespace: true,
Atomic: true,
Timeout: time.Second * 600,
ValuesYaml: templates.GetHelmValuesByTemplate("templates/helm-apps/releases/secrets-storage/vault.yml.tmpl"),
}
}
func ApplyCharts() {
appConfig := config.GetConfig()
helm_client.AddHelmRepos(appConfig.Modules.SecretsStorage.Namespace, HELM_REPOS)
if appConfig.Modules.SecretsStorage.Enabled {
helm_client.InstallChart(getVaultSpec())
initVault()
unsealVault()
addKubernetesLocalIntegration()
} else {
helm_client.DeleteChart(getVaultSpec())
}
}

View File

@@ -0,0 +1,12 @@
package templates
import (
"kube-forge/internal/config"
"kube-forge/internal/resources"
)
func GetHelmValuesByTemplate(templateFile string) string {
cfg := config.GetConfig()
template := getTemplateFromEmbedFSFolder(resources.Templates, templateFile)
return executeTemplateToString(template, cfg)
}

View File

@@ -1,46 +0,0 @@
package templates
import (
"kube-forge/internal/config"
"kube-forge/internal/resources"
"strings"
)
var HELM_APPS_TEMPLATES = [...]string{
"templates/helm-apps/releases/additional-modules/docker-secrets-generator.yml.tmpl",
"templates/helm-apps/releases/additional-modules/longhorn.yml.tmpl",
"templates/helm-apps/releases/additional-modules/cert-manager.yml.tmpl",
"templates/helm-apps/releases/additional-modules/ingress-nginx.yml.tmpl",
"templates/helm-apps/releases/additional-modules/secrets-store-csi-driver.yml.tmpl",
"templates/helm-apps/releases/observability/fluent-operator.yml.tmpl",
"templates/helm-apps/releases/observability/opentelemetry-operator.yml.tmpl",
"templates/helm-apps/releases/observability/metrics-server.yml.tmpl",
"templates/helm-apps/releases/observability/tempo.yml.tmpl",
"templates/helm-apps/releases/observability/loki.yml.tmpl",
"templates/helm-apps/releases/observability/observability.yml.tmpl",
"templates/helm-apps/releases/registry/harbor-certificate-generator.yml.tmpl",
"templates/helm-apps/releases/registry/harbor.yml.tmpl",
"templates/helm-apps/releases/cicd/argo-cd.yml.tmpl",
"templates/helm-apps/releases/cicd/argo-rollouts.yml.tmpl",
"templates/helm-apps/releases/cicd/keel.yml.tmpl",
"templates/helm-apps/releases/cicd/argo-cd-ingress.yml.tmpl",
"templates/helm-apps/releases/secrets-storage/vault.yml.tmpl",
}
var HELM_REPOSITORIES_TEMPLATES = [...]string{
"templates/helm-apps/repositories/repositories.yml.tmpl",
}
func GetHelmAppsConfigData() (string, string) {
cfg := config.GetConfig()
helmAppsTemplateResults := []string{}
for _, templateFile := range HELM_APPS_TEMPLATES {
template := getTemplateFromEmbedFSFolder(resources.Templates, templateFile)
helmAppsTemplateResults = append(helmAppsTemplateResults, executeTemplateToString(template, cfg))
}
repositoriesTemplateResults := []string{}
for _, templateFile := range HELM_REPOSITORIES_TEMPLATES {
template := getTemplateFromEmbedFSFolder(resources.Templates, templateFile)
repositoriesTemplateResults = append(repositoriesTemplateResults, executeTemplateToString(template, cfg))
}
return strings.Join(repositoriesTemplateResults, "\n"), strings.Join(helmAppsTemplateResults, "\n")
}

File diff suppressed because it is too large Load Diff

View File

@@ -90,7 +90,7 @@
- { role: kubernetes-apps/ingress_controller, tags: ingress-controller }
- { role: kubernetes-apps/external_provisioner, tags: external-provisioner }
- { role: kubernetes-apps, tags: apps }
- { role: helm-apps, tags: helm-apps }
# - { role: helm-apps, tags: helm-apps }
- name: Apply resolv.conf changes now that cluster DNS is up
hosts: k8s_cluster