add auto init/unseal for vault

pkg/config/config.go

@@ -15,7 +15,7 @@ type Host struct {
 }
 
 type Config struct {
-	DataDir     string
+	WorkDir     string
 	Credentials struct {
 		User            string `yaml:"user"`
 		Password        string `yaml:"password"`
@@ -43,9 +43,9 @@ type Config struct {
 
 var instance *Config
 
-func CreateConfig(configPath string, dataDir string, password string) *Config {
+func CreateConfig(configPath string, workDir string, password string) *Config {
 	instance = &Config{}
-	instance.DataDir = dataDir
+	instance.WorkDir = workDir
 
 	if err := cleanenv.ReadConfig(configPath, instance); err != nil {
 		helper, _ := cleanenv.GetDescription(instance, nil)

pkg/config/kubernetes.go

@@ -0,0 +1,21 @@
+package config
+
+import (
+	"path/filepath"
+
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+)
+
+var kubernetesConfig *rest.Config
+
+func GetKubernetesConfig() *rest.Config {
+	appConfig := GetConfig()
+	kubeconfigPath := filepath.Join(appConfig.WorkDir, "k8s-admin.conf")
+
+	kubernetesConfig, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath)
+	if err != nil {
+		panic(err.Error())
+	}
+	return kubernetesConfig
+}

pkg/config/modules_secrets_storage.go

@@ -1,11 +1,13 @@
 package config
 
 type SecretsStorage struct {
-	ChartRef     string `yaml:"chart_ref" env-default:"kube-forge/vault"`
-	ChartVersion string `yaml:"chart_version" env-default:"0.1.0"`
-	Enabled      bool   `yaml:"enabled"`
-	Image        string `yaml:"image" env-default:"kubesphere/fluent-operator"`
-	Tag          string `yaml:"tag" env-default:"v2.7.0"`
+	ChartRef     string   `yaml:"chart_ref" env-default:"kube-forge/vault"`
+	ChartVersion string   `yaml:"chart_version" env-default:"0.1.0"`
+	Enabled      bool     `yaml:"enabled"`
+	KeyShares    int      `yaml:"key_shares" env-default:"5"`
+	KeyThreshold int      `yaml:"key_threshold" env-default:"3"`
+	UnsealKeys   []string `yaml:"unseal_keys" env-default:"[]"`
+	RootToken    string
 	Expose       struct {
 		Type     string `yaml:"type"`
 		Domain   string `yaml:"domain"`

pkg/kubernetes_client/pod.go

@@ -0,0 +1,80 @@
+package kubernetes_client
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"kube-forge/pkg/config"
+	"net/http"
+	"strings"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/remotecommand"
+)
+
+func ExecuteCommandInPodContainer(command string, namespace string, podName string, container string) (string, error) {
+	clientset, err := kubernetes.NewForConfig(config.GetKubernetesConfig())
+	if err != nil {
+		panic(err.Error())
+	}
+	commandArray := strings.Split(command, " ")
+
+	execRequest := clientset.CoreV1().RESTClient().Post().
+		Resource("pods").
+		Name(podName).
+		Namespace(namespace).
+		SubResource("exec").
+		Param("container", container).
+		Param("stderr", "true").
+		Param("stdout", "true")
+	for _, com := range commandArray {
+		execRequest = execRequest.Param("command", com)
+	}
+
+	stderr := bytes.NewBufferString("")
+	stdout := bytes.NewBufferString("")
+
+	streamOptions := remotecommand.StreamOptions{
+		Stdout: stdout,
+		Stderr: stderr,
+		Tty:    false,
+	}
+
+	exec, err := remotecommand.NewSPDYExecutor(config.GetKubernetesConfig(), http.MethodPost, execRequest.URL())
+	if err != nil {
+		fmt.Println(err.Error())
+	}
+
+	err = exec.StreamWithContext(context.Background(), streamOptions)
+	if err != nil {
+		if stderr.Len() == 0 {
+			panic(err)
+		}
+		outputErr := stderr.String()
+		return outputErr, err
+	}
+	output := stdout.String()
+	return output, nil
+}
+
+func GetPodByName(podName string, podNamespace string) (*corev1.Pod, error) {
+	clientset, err := kubernetes.NewForConfig(config.GetKubernetesConfig())
+	if err != nil {
+		panic(err.Error())
+	}
+	for i := 15; i > 0; i-- {
+		time.Sleep(time.Second * 1)
+
+		pod, err := clientset.CoreV1().
+			Pods(podNamespace).
+			Get(context.Background(), podName, metav1.GetOptions{})
+		if err != nil || pod.Status.Phase != "Running" {
+			continue
+		}
+		return pod, nil
+	}
+	return nil, err
+}

pkg/kubespray/cluster.go

@@ -1,7 +1,18 @@
 package kubespray
 
+import (
+	"kube-forge/pkg/config"
+	"kube-forge/pkg/secrets_storage"
+)
+
 func InstallCluster(tags string) {
 	runPlaybook("kubespray/project/cluster.yml", tags)
+
+	config := config.GetConfig()
+	if config.Modules.SecretsStorage.Enabled {
+		secrets_storage.InitVault()
+		secrets_storage.UnsealVault()
+	}
 }
 
 func UpgradeCluster(tags string) {

pkg/kubespray/utility.go

@@ -16,7 +16,7 @@ func getPlaybookParameters(tags string) playbook.AnsiblePlaybookOptions {
 	cfg := config.GetConfig()
 
 	ansiblePlaybookOptions := playbook.AnsiblePlaybookOptions{
-		Inventory: filepath.Join(cfg.DataDir, "kubespray/inventory/hosts"),
+		Inventory: filepath.Join(cfg.WorkDir, "kubespray/inventory/hosts"),
 		Tags:      tags,
 		User:      cfg.Credentials.User,
 		Become:    true,
@@ -35,7 +35,7 @@ func getPlaybookParameters(tags string) playbook.AnsiblePlaybookOptions {
 
 func CopyK8SAdminConfig(pathInDataDir string) {
 	config := config.GetConfig()
-	dataDir := config.DataDir
+	dataDir := config.WorkDir
 	var adminDefaultConfigPath = filepath.Join(dataDir, "kubespray/inventory/artifacts/admin.conf")
 	var adminOutConfigPath = filepath.Join(dataDir, pathInDataDir)
 	source, err := os.Open(adminDefaultConfigPath)

pkg/resources/templates/helm-apps/releases/additional-modules/docker-secrets-generator.yml.tmpl

@@ -7,8 +7,8 @@
   {{- else }}
   release_state: "absent"
   {{- end }}
-  values:
     {{- if .Modules.Additional.DockerSecrets.Repositories }}
+  values:
     repositories:
       {{- .Modules.Additional.DockerSecrets.Repositories | toYaml | nindent 6 }}
     {{- end }}

pkg/resources/templates/helm-apps/releases/secrets-storage/vault.yml.tmpl

@@ -344,7 +344,7 @@
         #   name: http-monitoring
 
       readinessProbe:
-        enabled: true
+        enabled: false
         # If you need to use a http path instead of the default exec
         # path: /v1/sys/health?standbyok=true
 
@@ -388,11 +388,6 @@
       # Used to set the sleep time during the preStop step
       preStopSleepSeconds: 5
 
-      postStart:
-      # - /bin/sh
-      # - -c
-      # - /vault/userconfig/myscript/run.sh
-
       extraEnvironmentVars: {}
 
       extraSecretEnvironmentVars: []
@@ -816,6 +811,7 @@
         extraLabels: {}
 
       readinessProbe:
+        enabled: true
         failureThreshold: 2
         initialDelaySeconds: 5
         periodSeconds: 5

pkg/secrets_storage/commands.go

@@ -0,0 +1,98 @@
+package secrets_storage
+
+import (
+	"fmt"
+	"kube-forge/pkg/config"
+	"kube-forge/pkg/kubernetes_client"
+	"kube-forge/pkg/templates"
+	"regexp"
+	"strings"
+)
+
+func InitVault() {
+	_, err := kubernetes_client.GetPodByName("vault-0", "secrets-storage")
+	if err != nil {
+		fmt.Println(err.Error())
+		return
+	}
+	err = commandToInitVault()
+	if err != nil {
+		fmt.Println(err.Error())
+		return
+	}
+	fmt.Println("Vault initialized")
+	templates.ApplyVaultInitKeysTemplate()
+}
+
+func UnsealVault() {
+	_, err := kubernetes_client.GetPodByName("vault-0", "secrets-storage")
+	if err != nil {
+		fmt.Println(err.Error())
+		return
+	}
+	commandToUnsealVault()
+}
+
+// func getRunningVault(ctx context.Context, podName string, podNamespace string, responseChan chan<- error) {
+// 	time.Sleep(1 * time.Minute)
+// 	_, err := kubernetes_client.GetPodByName(ctx, podName, podNamespace)
+// 	if err != nil {
+// 		responseChan <- err
+// 	}
+
+// 	commandToUnsealVault()
+// 	responseChan <- nil
+// 	close(responseChan)
+// }
+
+func commandToInitVault() error {
+	config := config.GetConfig()
+
+	command := fmt.Sprintf(
+		"vault operator init -key-shares=%d -key-threshold=%d",
+		config.Modules.SecretsStorage.KeyShares,
+		config.Modules.SecretsStorage.KeyThreshold,
+	)
+	output, err := kubernetes_client.ExecuteCommandInPodContainer(
+		command, "secrets-storage", "vault-0", "vault",
+	)
+	if err != nil && strings.Contains(output, "Vault is already initialized") {
+		return VaultAlreadyInitialised
+	}
+
+	unsealKeys, rootToken := parseVaultInitKeys(output)
+	config.Modules.SecretsStorage.UnsealKeys = unsealKeys
+	config.Modules.SecretsStorage.RootToken = rootToken
+	return nil
+}
+
+func parseVaultInitKeys(input string) ([]string, string) {
+	unsealKeyPattern := regexp.MustCompile(`Unseal Key \d+: (\S+)`)
+	rootTokenPattern := regexp.MustCompile(`Initial Root Token: (\S+)`)
+
+	unsealKeysMatches := unsealKeyPattern.FindAllStringSubmatch(input, -1)
+	var unsealKeys []string
+	for _, match := range unsealKeysMatches {
+		unsealKeys = append(unsealKeys, match[1])
+	}
+
+	rootTokenMatches := rootTokenPattern.FindStringSubmatch(input)
+	rootToken := rootTokenMatches[1]
+
+	return unsealKeys, rootToken
+}
+
+func commandToUnsealVault() {
+	config := config.GetConfig()
+
+	for _, unsealKey := range config.Modules.SecretsStorage.UnsealKeys {
+		command := fmt.Sprintf(
+			"vault operator unseal %s",
+			unsealKey,
+		)
+		kubernetes_client.ExecuteCommandInPodContainer(
+			command, "secrets-storage", "vault-0", "vault",
+		)
+	}
+	fmt.Println("Vault unsealed")
+}

pkg/secrets_storage/errors.go

@@ -0,0 +1,5 @@
+package secrets_storage
+
+import "errors"
+
+var VaultAlreadyInitialised = errors.New("Vault already initialised")

pkg/templates/helm_apps.go

@@ -1,23 +1,11 @@
 package templates
 
 import (
-	"bytes"
-	"embed"
 	"kube-forge/pkg/config"
 	"kube-forge/pkg/resources"
-	"os"
-	"path/filepath"
 	"strings"
-	"text/template"
 )
 
-var K8S_TEMPLATES = [...][2]string{
-	{"templates/kubespray/inventory/hosts.tmpl", "kubespray/inventory/hosts"},
-	{"templates/kubespray/inventory/group_vars/all.yml.tmpl", "kubespray/inventory/group_vars/all.yml"},
-	{"templates/kubespray/inventory/group_vars/k8s_cluster/addons.yml.tmpl", "kubespray/inventory/group_vars/k8s_cluster/addons.yml"},
-	{"templates/kubespray/inventory/group_vars/k8s_cluster/k8s-cluster.yml.tmpl", "kubespray/inventory/group_vars/k8s_cluster/k8s-cluster.yml"},
-}
-
 var HELM_APPS_TEMPLATES = [...]string{
 	"templates/helm-apps/releases/additional-modules/docker-secrets-generator.yml.tmpl",
 	"templates/helm-apps/releases/additional-modules/longhorn.yml.tmpl",
@@ -41,49 +29,6 @@ var HELM_REPOSITORIES_TEMPLATES = [...]string{
 	"templates/helm-apps/repositories/repositories.yml.tmpl",
 }
 
-func executeTemplateToString(template *template.Template, config *config.Config) string {
-	templateResult := &bytes.Buffer{}
-	err := template.Execute(templateResult, config)
-	if err != nil {
-		panic(err)
-	}
-	templateResultString := templateResult.String()
-	return templateResultString
-}
-
-func getTemplateFromEmbedFSFolder(embedFS embed.FS, templateFile string) *template.Template {
-	templateData, err := embedFS.ReadFile(templateFile)
-	if err != nil {
-		panic(err)
-	}
-	templateDataString := string(templateData)
-	template, err := template.New("tmpl").Funcs(funcMap()).Parse(templateDataString)
-	if err != nil {
-		panic(err)
-	}
-	return template
-}
-
-func applyTemplates() {
-	config := config.GetConfig()
-	for _, templateData := range K8S_TEMPLATES {
-		var templateFile = templateData[0]
-		var outFile = filepath.Join(config.DataDir, templateData[1])
-
-		tmpl := getTemplateFromEmbedFSFolder(resources.Templates, templateFile)
-
-		file, err := os.Create(outFile)
-		if err != nil {
-			panic(err)
-		}
-		defer file.Close()
-		err = tmpl.Execute(file, config)
-		if err != nil {
-			panic(err)
-		}
-	}
-}
-
 func GetHelmAppsConfigData() (string, string) {
 	cfg := config.GetConfig()
 	helmAppsTemplateResults := []string{}
@@ -98,7 +43,3 @@ func GetHelmAppsConfigData() (string, string) {
 	}
 	return strings.Join(repositoriesTemplateResults, "\n"), strings.Join(helmAppsTemplateResults, "\n")
 }
-
-func ApplyTemplates() {
-	applyTemplates()
-}

pkg/templates/kubespray.go

@@ -0,0 +1,12 @@
+package templates
+
+var K8S_TEMPLATES = [...][2]string{
+	{"templates/kubespray/inventory/hosts.tmpl", "kubespray/inventory/hosts"},
+	{"templates/kubespray/inventory/group_vars/all.yml.tmpl", "kubespray/inventory/group_vars/all.yml"},
+	{"templates/kubespray/inventory/group_vars/k8s_cluster/addons.yml.tmpl", "kubespray/inventory/group_vars/k8s_cluster/addons.yml"},
+	{"templates/kubespray/inventory/group_vars/k8s_cluster/k8s-cluster.yml.tmpl", "kubespray/inventory/group_vars/k8s_cluster/k8s-cluster.yml"},
+}
+
+func ApplyK8sTemplates() {
+	applyTemplates(K8S_TEMPLATES[:])
+}

pkg/templates/secrets_storage.go

@@ -0,0 +1,14 @@
+package templates
+
+import (
+	"fmt"
+	"kube-forge/pkg/config"
+)
+
+func ApplyVaultInitKeysTemplate() {
+	config := config.GetConfig()
+	var VAULT_INIT_KEYS_TEMPLATE = [...][2]string{
+		{"templates/secrets-storage/vault-keys.json.tmpl", fmt.Sprintf("%s/vault-keys.json", config.WorkDir)},
+	}
+	applyTemplates(VAULT_INIT_KEYS_TEMPLATE[:])
+}

pkg/templates/utility.go

@@ -0,0 +1,54 @@
+package templates
+
+import (
+	"bytes"
+	"embed"
+	"kube-forge/pkg/config"
+	"kube-forge/pkg/resources"
+	"os"
+	"path/filepath"
+	"text/template"
+)
+
+func executeTemplateToString(template *template.Template, config *config.Config) string {
+	templateResult := &bytes.Buffer{}
+	err := template.Execute(templateResult, config)
+	if err != nil {
+		panic(err)
+	}
+	templateResultString := templateResult.String()
+	return templateResultString
+}
+
+func getTemplateFromEmbedFSFolder(embedFS embed.FS, templateFile string) *template.Template {
+	templateData, err := embedFS.ReadFile(templateFile)
+	if err != nil {
+		panic(err)
+	}
+	templateDataString := string(templateData)
+	template, err := template.New("tmpl").Funcs(funcMap()).Parse(templateDataString)
+	if err != nil {
+		panic(err)
+	}
+	return template
+}
+
+func applyTemplates(templates [][2]string) {
+	config := config.GetConfig()
+	for _, templateData := range templates {
+		var templateFile = templateData[0]
+		var outFile = filepath.Join(config.WorkDir, templateData[1])
+
+		tmpl := getTemplateFromEmbedFSFolder(resources.Templates, templateFile)
+
+		file, err := os.Create(outFile)
+		if err != nil {
+			panic(err)
+		}
+		defer file.Close()
+		err = tmpl.Execute(file, config)
+		if err != nil {
+			panic(err)
+		}
+	}
+}