kube-forge v2: embede python with ansible (kubespray) and cobra as cli building tool

internal/config/ansible_playbook_config.go

@@ -2,7 +2,7 @@ package config
 
 import (
 	"github.com/apenella/go-ansible/pkg/options"
-	"github.com/apenella/go-ansible/pkg/playbook"
+	"github.com/apenella/go-ansible/v2/pkg/playbook"
 )
 
 type AnsiblePlaybookConfig struct {

internal/config/config.go

@@ -2,10 +2,19 @@ package config
 
 import (
 	"fmt"
+	"kube-forge/internal/files"
+	"path/filepath"
 
 	"github.com/ilyakaznacheev/cleanenv"
 )
 
+var (
+	ConfigFile string
+	Output     string
+	Password   string
+	Verbose    bool
+)
+
 type Host struct {
 	Hostname string   `yaml:"hostname"`
 	Ip       string   `yaml:"ip"`
@@ -15,10 +24,12 @@ type Host struct {
 }
 
 type Config struct {
-	WorkDir        string
-	KubeconfigFile string `yaml:"kubeconfig_file" env-default:"k8s-admin.conf"`
-	Verbose        bool
-	Credentials    struct {
+	BaseDir          string
+	InventoryDir     string
+	Verbose          bool
+	KubeconfigFile   string
+	InstallationName string `yaml:"installation_name" env-default:"cluster.local"`
+	Credentials      struct {
 		User            string `yaml:"user"`
 		Password        string `yaml:"password"`
 		AskSudoPassword bool   `yaml:"ask_sudo_password"`
@@ -36,21 +47,37 @@ type Config struct {
 
 var instance *Config
 
-func CreateConfig(configPath string, workDir string, password string) *Config {
+func createConfig(configFile string, out string, password string, verbose bool) *Config {
 	instance = &Config{}
-	instance.WorkDir = workDir
 
-	if err := cleanenv.ReadConfig(configPath, instance); err != nil {
+	if err := cleanenv.ReadConfig(configFile, instance); err != nil {
 		helper, _ := cleanenv.GetDescription(instance, nil)
 		panic(fmt.Sprintf("%s\n%s", helper, err))
 	}
 
+	instance.BaseDir = files.CreateTempDir()
+	instance.InventoryDir = filepath.Join(instance.BaseDir, "inventory")
+
+	files.CreateDirWithAllParents(instance.InventoryDir)
+
+	if out != "" {
+		files.CreateDirWithAllParents(out)
+		instance.KubeconfigFile = filepath.Join(out, "k8s-admin.conf")
+	} else {
+		instance.KubeconfigFile = filepath.Join(files.GetWorkingDir(), "k8s-admin.conf")
+	}
+
 	if password != "" {
 		instance.Credentials.Password = password
 	}
+
+	instance.Verbose = verbose
 	return instance
 }
 
 func GetConfig() *Config {
+	if instance == nil {
+		createConfig(ConfigFile, Output, Password, Verbose)
+	}
 	return instance
 }

internal/config/helm.go

@@ -4,7 +4,6 @@ import (
 	"io"
 	"log"
 	"os"
-	"path/filepath"
 	"time"
 
 	helm_client "github.com/mittwald/go-helm-client"
@@ -30,7 +29,7 @@ type ChartSettings struct {
 
 func GetHelmClient(namespace string) helm_client.Client {
 	config := GetConfig()
-	file, err := os.Open(filepath.Join(config.WorkDir, config.KubeconfigFile))
+	file, err := os.Open(config.KubeconfigFile)
 	if err != nil {
 		log.Fatalf("failed to open file: %s", err)
 	}
@@ -46,7 +45,7 @@ func GetHelmClient(namespace string) helm_client.Client {
 			RepositoryConfig: "/tmp/.helmrepo",
 			Debug:            true,
 			Linting:          true, // Change this to false if you don't want linting.
-			DebugLog: func(format string, v ...interface{}) {
+			DebugLog: func(format string, v ...any) {
 				// Change this to your own logger. Default is 'log.Printf(format, v...)'.
 			},
 		},

internal/config/kubernetes.go

@@ -1,21 +0,0 @@
-package config
-
-import (
-	"path/filepath"
-
-	"k8s.io/client-go/rest"
-	"k8s.io/client-go/tools/clientcmd"
-)
-
-var kubernetesConfig *rest.Config
-
-func GetKubernetesConfig() *rest.Config {
-	appConfig := GetConfig()
-	kubeconfigPath := filepath.Join(appConfig.WorkDir, appConfig.KubeconfigFile)
-
-	kubernetesConfig, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath)
-	if err != nil {
-		panic(err.Error())
-	}
-	return kubernetesConfig
-}

internal/config/orchestrator.go

@@ -1,9 +1,9 @@
 package config
 
 type Dns struct {
-	Servers                []string      `yaml:"servers" env-default:"8.8.8.8,8.8.4.4"`
-	DisableHostNameservers bool          `yaml:"disable_host_nameservers"`
-	CoreDNSExternalZones   []interface{} `yaml:"coredns_external_zones"`
+	Servers                []string `yaml:"servers" env-default:"8.8.8.8,8.8.4.4"`
+	DisableHostNameservers bool     `yaml:"disable_host_nameservers"`
+	CoreDNSExternalZones   []any    `yaml:"coredns_external_zones"`
 }
 
 type RegistryMirror struct {

internal/files/main.go

@@ -0,0 +1,30 @@
+package files
+
+import (
+	"kube-forge/internal/logging"
+	"os"
+)
+
+func GetWorkingDir() string {
+	cwd, err := os.Getwd()
+	if err != nil {
+		logging.Log.Fatal(err)
+	}
+	return cwd
+
+}
+
+func CreateTempDir() string {
+	tempDir, err := os.MkdirTemp("", "kube-forge-")
+	if err != nil {
+		logging.Log.Fatal(err)
+	}
+	return tempDir
+}
+
+func CreateDirWithAllParents(directory string) {
+	err := os.MkdirAll(directory, 0755)
+	if err != nil {
+		logging.Log.Fatal(err)
+	}
+}

internal/kubernetes_client/errors.go

@@ -1,5 +0,0 @@
-package kubernetes_client
-
-import "errors"
-
-var NoSuchVarInPod = errors.New("No such variable in pod!")

internal/kubernetes_client/pod.go

@@ -1,101 +0,0 @@
-package kubernetes_client
-
-import (
-	"bytes"
-	"context"
-	"fmt"
-	"kube-forge/internal/config"
-	"net/http"
-	"strings"
-	"time"
-
-	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/tools/remotecommand"
-)
-
-func ExecuteCommandInPodContainer(commandArray []string, namespace string, podName string, container string) (string, error) {
-	clientset, err := kubernetes.NewForConfig(config.GetKubernetesConfig())
-	if err != nil {
-		panic(err.Error())
-	}
-
-	execRequest := clientset.CoreV1().RESTClient().Post().
-		Resource("pods").
-		Name(podName).
-		Namespace(namespace).
-		SubResource("exec").
-		Param("container", container).
-		Param("stderr", "true").
-		Param("stdout", "true")
-	for _, com := range commandArray {
-		execRequest = execRequest.Param("command", com)
-	}
-
-	stderr := bytes.NewBufferString("")
-	stdout := bytes.NewBufferString("")
-
-	streamOptions := remotecommand.StreamOptions{
-		Stdout: stdout,
-		Stderr: stderr,
-		Tty:    false,
-	}
-
-	exec, err := remotecommand.NewSPDYExecutor(config.GetKubernetesConfig(), http.MethodPost, execRequest.URL())
-	if err != nil {
-		fmt.Println(err.Error())
-	}
-
-	err = exec.StreamWithContext(context.Background(), streamOptions)
-	if err != nil {
-		if stderr.Len() == 0 {
-			panic(err)
-		}
-		outputErr := stderr.String()
-		return outputErr, err
-	}
-	output := stdout.String()
-	return output, nil
-}
-
-func GetPodByName(podName string, podNamespace string) (*corev1.Pod, error) {
-	clientset, err := kubernetes.NewForConfig(config.GetKubernetesConfig())
-	if err != nil {
-		panic(err.Error())
-	}
-	for i := 15; i > 0; i-- {
-		time.Sleep(time.Second * 1)
-
-		pod, err := clientset.CoreV1().
-			Pods(podNamespace).
-			Get(context.Background(), podName, metav1.GetOptions{})
-		if err != nil || pod.Status.Phase != "Running" {
-			continue
-		}
-		return pod, nil
-	}
-	return nil, err
-}
-
-func GetEnvVariableFromPodContainer(envVarName string, podNamespace string, podName string, containerName string) (string, error) {
-	commandArray := []string{"env"}
-	output, err := ExecuteCommandInPodContainer(
-		commandArray, podNamespace, podName, containerName,
-	)
-	if err != nil {
-		return output, err
-	}
-	lines := strings.Split(output, "\n")
-
-	for _, line := range lines {
-		if strings.HasPrefix(line, envVarName+"=") {
-			parts := strings.Split(line, "=")
-			if len(parts) == 2 {
-				value := parts[1]
-				return value, nil
-			}
-		}
-	}
-	return "", NoSuchVarInPod
-}

internal/kubespray/cluster.go

@@ -23,7 +23,5 @@ func ScaleCluster() {
 }
 
 func ResetCluster() {
-	appConfig := config.GetConfig()
 	runPlaybook("kubespray/project/reset.yml", "")
-	CopyK8SAdminConfig(appConfig.KubeconfigFile)
 }

internal/kubespray/utility.go

@@ -2,8 +2,11 @@ package kubespray
 
 import (
 	"context"
+	"fmt"
 	"io"
 	"kube-forge/internal/config"
+	"kube-forge/internal/logging"
+	"kube-forge/internal/python"
 	"os"
 	"path/filepath"
 
@@ -16,7 +19,7 @@ func getPlaybookParameters(tags string) playbook.AnsiblePlaybookOptions {
 	cfg := config.GetConfig()
 
 	ansiblePlaybookOptions := playbook.AnsiblePlaybookOptions{
-		Inventory: filepath.Join(cfg.WorkDir, "kubespray/inventory/hosts"),
+		Inventory: filepath.Join(cfg.InventoryDir, "hosts"),
 		Tags:      tags,
 		User:      cfg.Credentials.User,
 		Become:    true,
@@ -33,18 +36,16 @@ func getPlaybookParameters(tags string) playbook.AnsiblePlaybookOptions {
 	return ansiblePlaybookOptions
 }
 
-func CopyK8SAdminConfig(pathInDataDir string) {
+func CopyK8SAdminConfig(outFile string) {
 	config := config.GetConfig()
-	dataDir := config.WorkDir
-	var adminDefaultConfigPath = filepath.Join(dataDir, "kubespray/inventory/artifacts/admin.conf")
-	var adminOutConfigPath = filepath.Join(dataDir, pathInDataDir)
+	var adminDefaultConfigPath = filepath.Join(config.InventoryDir, "artifacts/admin.conf")
 	source, err := os.Open(adminDefaultConfigPath)
 	if err != nil {
 		panic(err)
 	}
 	defer source.Close()
 
-	destination, err := os.Create(adminOutConfigPath)
+	destination, err := os.Create(outFile)
 	if err != nil {
 		panic(err)
 	}
@@ -53,26 +54,58 @@ func CopyK8SAdminConfig(pathInDataDir string) {
 	if err != nil {
 		panic(err)
 	}
+	logging.Log.Info(fmt.Sprintf("K8s admin config: %s", outFile))
 }
 
 func runPlaybook(playbookPath string, tags string) {
 	config := config.GetConfig()
 	var callbackExecute execute.Executor
+	epExec := python.NewPythonExec()
+
+	envVars := map[string]string{
+		"ANSIBLE_DISPLAY_SKIPPED_HOSTS": "false",
+		"ANSIBLE_DISPLAY_OK_HOSTS":      "false",
+		"ANSIBLE_FORCE_COLOR":           "true",
+		"ANSIBLE_HOST_KEY_CHECKING":     "false",
+	}
 
 	playbookOptions := getPlaybookParameters(tags)
+
 	playbookCmd := playbook.NewAnsiblePlaybookCmd(
-		playbook.WithPlaybooks(playbookPath),
+		playbook.WithPlaybooks(
+			filepath.Join(epExec.ResourcesFsPath, playbookPath),
+		),
 		playbook.WithPlaybookOptions(&playbookOptions),
+		playbook.WithBinary(
+			filepath.Join(epExec.PythonLibFsPath, "bin", "ansible-playbook"),
+		),
 	)
+
 	execute := execute.NewDefaultExecute(
 		execute.WithCmd(playbookCmd),
 		execute.WithErrorEnrich(playbook.NewAnsiblePlaybookErrorEnrich()),
+		execute.WithExecutable(epExec),
 	)
 
 	if config.Verbose {
-		callbackExecute = stdoutcallback.NewDebugStdoutCallbackExecute(execute)
+		envVars["ANSIBLE_STDOUT_CALLBACK"] = "unixy"
+		envVars["ANSIBLE_VERBOSITY"] = "3"
+		envVars["ANSIBLE_DISPLAY_SKIPPED_HOSTS"] = "true"
+		envVars["ANSIBLE_DISPLAY_OK_HOSTS"] = "true"
 	} else {
-		callbackExecute = stdoutcallback.NewDenseStdoutCallbackExecute(execute)
+		envVars["ANSIBLE_STDOUT_CALLBACK"] = "dense"
+		envVars["ANSIBLE_ACTION_WARNINGS"] = "false"
+		envVars["ANSIBLE_DEVEL_WARNING"] = "false"
+		envVars["ANSIBLE_DEPRECATION_WARNINGS"] = "false"
+		envVars["ANSIBLE_DUPLICATE_YAML_DICT_KEY"] = "ignore"
+		envVars["ANSIBLE_HOST_PATTERN_MISMATCH"] = "ignore"
+		envVars["ANSIBLE_SYSTEM_WARNINGS"] = "false"
+	}
+
+	callbackExecute = stdoutcallback.NewDefaultStdoutCallbackExecute(execute)
+
+	for key, value := range envVars {
+		os.Setenv(key, value)
 	}
 
 	err := callbackExecute.Execute(context.Background())

internal/python/generate/main.go

@@ -0,0 +1,27 @@
+package main
+
+import (
+	"runtime"
+	"strings"
+
+	"github.com/kluctl/go-embed-python/pip"
+)
+
+var platforms = map[string][]string{
+	"darwin-amd64":  {"macosx_11_0_x86_64", "macosx_12_0_x86_64"},
+	"darwin-arm64":  {"macosx_11_0_arm64", "macosx_12_0_arm64"},
+	"linux-amd64":   {"manylinux_2_17_x86_64", "manylinux_2_28_x86_64", "manylinux2014_x86_64"},
+	"linux-arm64":   {"manylinux_2_17_aarch64", "manylinux_2_28_aarch64", "manylinux2014_aarch64"},
+	"windows-amd64": {"win_amd64"},
+}
+
+func main() {
+	os := runtime.GOOS
+
+	arch := runtime.GOARCH
+	pipPlatform := platforms[strings.Join([]string{os, arch}, "-")]
+	err := pip.CreateEmbeddedPipPackages("requirements.txt", os, arch, pipPlatform, "./data/")
+	if err != nil {
+		panic(err)
+	}
+}

internal/python/main.go

@@ -0,0 +1,3 @@
+package python
+
+//go:generate go run ./generate

internal/python/requirements.txt

@@ -0,0 +1,4 @@
+ansible
+cryptography
+jmespath
+netaddr

internal/python/utils.go

@@ -0,0 +1,161 @@
+package python
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"kube-forge/internal/python/data"
+	"kube-forge/internal/resources"
+	"os"
+	"os/exec"
+	"path/filepath"
+
+	executable "github.com/apenella/go-ansible/v2/pkg/execute/exec"
+	"github.com/kluctl/go-embed-python/embed_util"
+	"github.com/kluctl/go-embed-python/python"
+)
+
+type PythonExec struct {
+	ep              *python.EmbeddedPython
+	PythonLibFsPath string
+	ResourcesFsPath string
+}
+
+type CmdWrapper struct {
+	*exec.Cmd
+}
+
+func (c *CmdWrapper) CombinedOutput() ([]byte, error) {
+	return c.Cmd.CombinedOutput()
+}
+
+func (c *CmdWrapper) Environ() []string {
+	return c.Cmd.Environ()
+}
+
+func (c *CmdWrapper) Output() ([]byte, error) {
+	return c.Cmd.Output()
+}
+
+func (c *CmdWrapper) Run() error {
+	return c.Cmd.Run()
+}
+
+func (c *CmdWrapper) Start() error {
+	return c.Cmd.Start()
+}
+
+func (c *CmdWrapper) StderrPipe() (io.ReadCloser, error) {
+	return c.Cmd.StderrPipe()
+}
+
+func (c *CmdWrapper) StdinPipe() (io.WriteCloser, error) {
+	return c.Cmd.StdinPipe()
+}
+
+func (c *CmdWrapper) StdoutPipe() (io.ReadCloser, error) {
+	return c.Cmd.StdoutPipe()
+}
+
+func (c *CmdWrapper) String() string {
+	return c.Cmd.String()
+}
+
+func (c *CmdWrapper) Wait() error {
+	return c.Cmd.Wait()
+}
+
+type ErrorCmd struct {
+	err error
+}
+
+func (e *ErrorCmd) CombinedOutput() ([]byte, error) {
+	return nil, e.err
+}
+
+func (e *ErrorCmd) Environ() []string {
+	return nil
+}
+
+func (e *ErrorCmd) Output() ([]byte, error) {
+	return nil, e.err
+}
+
+func (e *ErrorCmd) Run() error {
+	return e.err
+}
+
+func (e *ErrorCmd) Start() error {
+	return e.err
+}
+
+func (e *ErrorCmd) StderrPipe() (io.ReadCloser, error) {
+	return nil, e.err
+}
+
+func (e *ErrorCmd) StdinPipe() (io.WriteCloser, error) {
+	return nil, e.err
+}
+
+func (e *ErrorCmd) StdoutPipe() (io.ReadCloser, error) {
+	return nil, e.err
+}
+
+func (e *ErrorCmd) String() string {
+	return fmt.Sprintf("ErrorCmd: %v", e.err)
+}
+
+func (e *ErrorCmd) Wait() error {
+	return e.err
+}
+
+func NewPythonExec() *PythonExec {
+	tmpDir := filepath.Join(os.TempDir(), "go-ansible")
+	pythonDir := tmpDir + "-python"
+	pythonLibDir := tmpDir + "-python-lib"
+	resourcesDir := tmpDir + "-resources"
+
+	pythonLibFs, err := embed_util.NewEmbeddedFilesWithTmpDir(data.Data, pythonLibDir, true)
+	if err != nil {
+		panic(err)
+	}
+	pythonLibFsPath := pythonLibFs.GetExtractedPath()
+	resourcesFs, _ := embed_util.NewEmbeddedFilesWithTmpDir(resources.Kubespray, resourcesDir, true)
+	resourcesFsPath := resourcesFs.GetExtractedPath()
+
+	ep, _ := python.NewEmbeddedPythonWithTmpDir(pythonDir, true)
+	ep.AddPythonPath(pythonLibFs.GetExtractedPath())
+	ep.AddPythonPath(resourcesFs.GetExtractedPath())
+
+	return &PythonExec{
+		ep:              ep,
+		PythonLibFsPath: pythonLibFsPath,
+		ResourcesFsPath: resourcesFsPath,
+	}
+}
+
+func (e *PythonExec) Command(name string, arg ...string) executable.Cmder {
+	cmd := append([]string{name}, arg...)
+	execCmd, err := e.ep.PythonCmd2(cmd)
+	if err != nil {
+		return &ErrorCmd{err: fmt.Errorf("failed to create python command: %w", err)}
+	}
+	return &CmdWrapper{Cmd: execCmd}
+}
+
+func (e *PythonExec) CommandContext(ctx context.Context, name string, arg ...string) executable.Cmder {
+	cmd := append([]string{name}, arg...)
+	execCmd, err := e.ep.PythonCmd2(cmd)
+	if err != nil {
+		return &ErrorCmd{err: fmt.Errorf("failed to create python command: %w", err)}
+	}
+
+	ctxCmd := exec.CommandContext(ctx, execCmd.Path, execCmd.Args[1:]...)
+	ctxCmd.Env = execCmd.Env
+	ctxCmd.Dir = execCmd.Dir
+	ctxCmd.Stdin = execCmd.Stdin
+	ctxCmd.Stdout = execCmd.Stdout
+	ctxCmd.Stderr = execCmd.Stderr
+
+	return &CmdWrapper{Cmd: ctxCmd}
+}

internal/resources/kubespray/project/ansible_version.yml

@@ -0,0 +1,33 @@
+---
+- name: Check Ansible version
+  hosts: all
+  gather_facts: false
+  become: false
+  run_once: true
+  vars:
+    minimal_ansible_version: 2.16.4
+    maximal_ansible_version: 2.17.0
+  tags: always
+  tasks:
+    # - name: "Check {{ minimal_ansible_version }} <= Ansible version < {{ maximal_ansible_version }}"
+    #   assert:
+    #     msg: "Ansible must be between {{ minimal_ansible_version }} and {{ maximal_ansible_version }} exclusive - you have {{ ansible_version.string }}"
+    #     that:
+    #       - ansible_version.string is version(minimal_ansible_version, ">=")
+    #       - ansible_version.string is version(maximal_ansible_version, "<")
+    #   tags:
+    #     - check
+
+    - name: "Check that python netaddr is installed"
+      assert:
+        msg: "Python netaddr is not present"
+        that: "'127.0.0.1' | ansible.utils.ipaddr"
+      tags:
+        - check
+
+    - name: "Check that jinja is not too old (install via pip)"
+      assert:
+        msg: "Your Jinja version is too old, install via pip"
+        that: "{% set test %}It works{% endset %}{{ test == 'It works' }}"
+      tags:
+        - check

internal/resources/kubespray/project/boilerplate.yml

@@ -0,0 +1,46 @@
+---
+- name: Check ansible version
+  import_playbook: ansible_version.yml
+
+# These are inventory compatibility tasks with two purposes:
+# - to ensure we keep compatibility with old style group names
+# - to reduce inventory boilerplate (defining parent groups / empty groups)
+
+- name: Define groups for legacy less structured inventories
+  hosts: all
+  gather_facts: false
+  tags: always
+  tasks:
+    - name: Match needed groups by their old names or definition
+      vars:
+        group_mappings:
+          kube_control_plane:
+            - kube-master
+          kube_node:
+            - kube-node
+          calico_rr:
+            - calico-rr
+          no_floating:
+            - no-floating
+          k8s_cluster:
+            - kube_node
+            - kube_control_plane
+            - calico_rr
+      group_by:
+        key: "{{ (group_names | intersect(item.value) | length > 0) | ternary(item.key, '_all') }}"
+      loop: "{{ group_mappings | dict2items }}"
+
+- name: Check inventory settings
+  hosts: all
+  gather_facts: false
+  tags: always
+  roles:
+    - validate_inventory
+
+- name: Install bastion ssh config
+  hosts: bastion[0]
+  gather_facts: false
+  environment: "{{ proxy_disable_env }}"
+  roles:
+    - { role: kubespray_defaults }
+    - { role: bastion-ssh-config, tags: ["localhost", "bastion"] }

internal/resources/kubespray/project/cluster.yml

@@ -0,0 +1,101 @@
+---
+- name: Common tasks for every playbooks
+  import_playbook: boilerplate.yml
+
+- name: Gather facts
+  import_playbook: facts.yml
+
+- name: Prepare for etcd install
+  hosts: k8s_cluster:etcd
+  gather_facts: false
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  environment: "{{ proxy_disable_env }}"
+  roles:
+    - { role: kubespray_defaults }
+    - { role: kubernetes/preinstall, tags: preinstall }
+    - { role: "container-engine", tags: "container-engine", when: deploy_container_engine }
+    - { role: download, tags: download, when: "not skip_downloads" }
+
+- name: Install etcd
+  vars:
+    etcd_cluster_setup: true
+    etcd_events_cluster_setup: "{{ etcd_events_cluster_enabled }}"
+  import_playbook: install_etcd.yml
+
+- name: Install Kubernetes nodes
+  hosts: k8s_cluster
+  gather_facts: false
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  environment: "{{ proxy_disable_env }}"
+  roles:
+    - { role: kubespray_defaults }
+    - { role: kubernetes/node, tags: node }
+
+- name: Install the control plane
+  hosts: kube_control_plane
+  gather_facts: false
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  environment: "{{ proxy_disable_env }}"
+  roles:
+    - { role: kubespray_defaults }
+    - { role: kubernetes/control-plane, tags: master }
+    - { role: kubernetes/client, tags: client }
+    - { role: kubernetes-apps/cluster_roles, tags: cluster-roles }
+
+- name: Invoke kubeadm and install a CNI
+  hosts: k8s_cluster
+  gather_facts: false
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  environment: "{{ proxy_disable_env }}"
+  roles:
+    - { role: kubespray_defaults }
+    - { role: kubernetes/kubeadm, tags: kubeadm}
+    - { role: kubernetes/node-label, tags: node-label }
+    - { role: kubernetes/node-taint, tags: node-taint }
+    - role: kubernetes-apps/gateway_api
+      when: gateway_api_enabled
+      tags: gateway_api
+      delegate_to: "{{ groups['kube_control_plane'][0] }}"
+      run_once: true
+    - { role: network_plugin, tags: network }
+
+- name: Install Calico Route Reflector
+  hosts: calico_rr
+  gather_facts: false
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  environment: "{{ proxy_disable_env }}"
+  roles:
+    - { role: kubespray_defaults }
+    - { role: network_plugin/calico/rr, tags: ['network', 'calico_rr'] }
+
+- name: Patch Kubernetes for Windows
+  hosts: kube_control_plane[0]
+  gather_facts: false
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  environment: "{{ proxy_disable_env }}"
+  roles:
+    - { role: kubespray_defaults }
+    - { role: win_nodes/kubernetes_patch, tags: ["master", "win_nodes"] }
+
+- name: Install Kubernetes apps
+  hosts: kube_control_plane
+  gather_facts: false
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  environment: "{{ proxy_disable_env }}"
+  roles:
+    - { role: kubespray_defaults }
+    - { role: kubernetes-apps/external_cloud_controller, tags: external-cloud-controller }
+    - { role: kubernetes-apps/network_plugin, tags: network }
+    - { role: kubernetes-apps/policy_controller, tags: policy-controller }
+    - { role: kubernetes-apps/ingress_controller, tags: ingress-controller }
+    - { role: kubernetes-apps/external_provisioner, tags: external-provisioner }
+    - { role: kubernetes-apps, tags: apps }
+
+- name: Apply resolv.conf changes now that cluster DNS is up
+  hosts: k8s_cluster
+  gather_facts: false
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  environment: "{{ proxy_disable_env }}"
+  roles:
+    - { role: kubespray_defaults }
+    - { role: kubernetes/preinstall, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf, dns_late: true }

internal/resources/kubespray/project/facts.yml

@@ -0,0 +1,39 @@
+---
+- name: Bootstrap hosts for Ansible
+  hosts: k8s_cluster:etcd:calico_rr
+  strategy: linear
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  gather_facts: false
+  environment: "{{ proxy_disable_env }}"
+  roles:
+    - { role: bootstrap_os, tags: bootstrap_os}
+
+- name: Gather facts
+  hosts: k8s_cluster:etcd:calico_rr
+  gather_facts: false
+  tags: always
+  tasks:
+    - name: Gather and compute network facts
+      import_role:
+        name: network_facts
+    - name: Gather minimal facts
+      setup:
+        gather_subset: '!all'
+
+    # filter match the following variables:
+    # ansible_default_ipv4
+    # ansible_default_ipv6
+    # ansible_all_ipv4_addresses
+    # ansible_all_ipv6_addresses
+    - name: Gather necessary facts (network)
+      setup:
+        gather_subset: '!all,!min,network'
+        filter: "ansible_*_ipv[46]*"
+
+    # filter match the following variables:
+    # ansible_memtotal_mb
+    # ansible_swaptotal_mb
+    - name: Gather necessary facts (hardware)
+      setup:
+        gather_subset: '!all,!min,hardware'
+        filter: "ansible_*total_mb"

internal/resources/kubespray/project/install_etcd.yml

@@ -0,0 +1,26 @@
+---
+- name: Add worker nodes to the etcd play if needed
+  hosts: kube_node
+  roles:
+    - { role: kubespray_defaults }
+  tasks:
+    - name: Check if nodes needs etcd client certs (depends on network_plugin)
+      group_by:
+        key: "_kubespray_needs_etcd"
+      when:
+        - kube_network_plugin in ["flannel", "canal", "cilium"] or
+          (cilium_deploy_additionally | default(false)) or
+          (kube_network_plugin == "calico" and calico_datastore == "etcd")
+        - etcd_deployment_type != "kubeadm"
+      tags: etcd
+
+- name: Install etcd
+  hosts: etcd:kube_control_plane:_kubespray_needs_etcd
+  gather_facts: false
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  environment: "{{ proxy_disable_env }}"
+  roles:
+    - { role: kubespray_defaults }
+    - role: etcd
+      tags: etcd
+      when: etcd_deployment_type != "kubeadm"

internal/resources/kubespray/project/library/kube.py

@@ -0,0 +1,366 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+DOCUMENTATION = """
+---
+module: kube
+short_description: Manage Kubernetes Cluster
+description:
+  - Create, replace, remove, and stop resources within a Kubernetes Cluster
+version_added: "2.0"
+options:
+  name:
+    required: false
+    default: null
+    description:
+      - The name associated with resource
+  filename:
+    required: false
+    default: null
+    description:
+      - The path and filename of the resource(s) definition file(s).
+      - To operate on several files this can accept a comma separated list of files or a list of files.
+    aliases: [ 'files', 'file', 'filenames' ]
+  kubectl:
+    required: false
+    default: null
+    description:
+      - The path to the kubectl bin
+  namespace:
+    required: false
+    default: null
+    description:
+      - The namespace associated with the resource(s)
+  resource:
+    required: false
+    default: null
+    description:
+      - The resource to perform an action on. pods (po), replicationControllers (rc), services (svc)
+  label:
+    required: false
+    default: null
+    description:
+      - The labels used to filter specific resources.
+  server:
+    required: false
+    default: null
+    description:
+      - The url for the API server that commands are executed against.
+  kubeconfig:
+    required: false
+    default: null
+    description:
+      - The path to the kubeconfig.
+  force:
+    required: false
+    default: false
+    description:
+      - A flag to indicate to force delete, replace, or stop.
+  wait:
+    required: false
+    default: false
+    description:
+      - A flag to indicate to wait for resources to be created before continuing to the next step
+  all:
+    required: false
+    default: false
+    description:
+      - A flag to indicate delete all, stop all, or all namespaces when checking exists.
+  log_level:
+    required: false
+    default: 0
+    description:
+      - Indicates the level of verbosity of logging by kubectl.
+  state:
+    required: false
+    choices: ['present', 'absent', 'latest', 'reloaded', 'stopped']
+    default: present
+    description:
+      - present handles checking existence or creating if definition file provided,
+        absent handles deleting resource(s) based on other options,
+        latest handles creating or updating based on existence,
+        reloaded handles updating resource(s) definition using definition file,
+        stopped handles stopping resource(s) based on other options.
+  recursive:
+    required: false
+    default: false
+    description:
+      - Process the directory used in -f, --filename recursively.
+        Useful when you want to manage related manifests organized
+        within the same directory.
+requirements:
+  - kubectl
+author: "Kenny Jones (@kenjones-cisco)"
+"""
+
+EXAMPLES = """
+- name: test nginx is present
+  kube: name=nginx resource=rc state=present
+
+- name: test nginx is stopped
+  kube: name=nginx resource=rc state=stopped
+
+- name: test nginx is absent
+  kube: name=nginx resource=rc state=absent
+
+- name: test nginx is present
+  kube: filename=/tmp/nginx.yml
+
+- name: test nginx and postgresql are present
+  kube: files=/tmp/nginx.yml,/tmp/postgresql.yml
+
+- name: test nginx and postgresql are present
+  kube:
+    files:
+      - /tmp/nginx.yml
+      - /tmp/postgresql.yml
+"""
+
+
+class KubeManager(object):
+
+    def __init__(self, module):
+
+        self.module = module
+
+        self.kubectl = module.params.get('kubectl')
+        if self.kubectl is None:
+            self.kubectl =  module.get_bin_path('kubectl', True)
+        self.base_cmd = [self.kubectl]
+
+        if module.params.get('server'):
+            self.base_cmd.append('--server=' + module.params.get('server'))
+
+        if module.params.get('kubeconfig'):
+            self.base_cmd.append('--kubeconfig=' + module.params.get('kubeconfig'))
+
+        if module.params.get('log_level'):
+            self.base_cmd.append('--v=' + str(module.params.get('log_level')))
+
+        if module.params.get('namespace'):
+            self.base_cmd.append('--namespace=' + module.params.get('namespace'))
+
+
+        self.all = module.params.get('all')
+        self.force = module.params.get('force')
+        self.wait = module.params.get('wait')
+        self.name = module.params.get('name')
+        self.filename = [f.strip() for f in module.params.get('filename') or []]
+        self.resource = module.params.get('resource')
+        self.label = module.params.get('label')
+        self.recursive = module.params.get('recursive')
+
+    def _execute(self, cmd):
+        args = self.base_cmd + cmd
+        try:
+            rc, out, err = self.module.run_command(args)
+            if rc != 0:
+                self.module.fail_json(
+                    msg='error running kubectl (%s) command (rc=%d), out=\'%s\', err=\'%s\'' % (' '.join(args), rc, out, err))
+        except Exception as exc:
+            self.module.fail_json(
+                msg='error running kubectl (%s) command: %s' % (' '.join(args), str(exc)))
+        return out.splitlines()
+
+    def _execute_nofail(self, cmd):
+        args = self.base_cmd + cmd
+        rc, out, err = self.module.run_command(args)
+        if rc != 0:
+            return None
+        return out.splitlines()
+
+    def create(self, check=True, force=True):
+        if check and self.exists():
+            return []
+
+        cmd = ['apply']
+
+        if force:
+            cmd.append('--force')
+
+        if self.wait:
+            cmd.append('--wait')
+
+        if self.recursive:
+            cmd.append('--recursive={}'.format(self.recursive))
+
+        if not self.filename:
+            self.module.fail_json(msg='filename required to create')
+
+        cmd.append('--filename=' + ','.join(self.filename))
+
+        return self._execute(cmd)
+
+    def replace(self, force=True):
+
+        cmd = ['apply']
+
+        if force:
+            cmd.append('--force')
+
+        if self.wait:
+            cmd.append('--wait')
+
+        if self.recursive:
+            cmd.append('--recursive={}'.format(self.recursive))
+
+        if not self.filename:
+            self.module.fail_json(msg='filename required to reload')
+
+        cmd.append('--filename=' + ','.join(self.filename))
+
+        return self._execute(cmd)
+
+    def delete(self):
+
+        if not self.force and not self.exists():
+            return []
+
+        cmd = ['delete']
+
+        if self.filename:
+            cmd.append('--filename=' + ','.join(self.filename))
+            if self.recursive:
+                cmd.append('--recursive={}'.format(self.recursive))
+        else:
+            if not self.resource:
+                self.module.fail_json(msg='resource required to delete without filename')
+
+            cmd.append(self.resource)
+
+            if self.name:
+                cmd.append(self.name)
+
+            if self.label:
+                cmd.append('--selector=' + self.label)
+
+            if self.all:
+                cmd.append('--all')
+
+            if self.force:
+                cmd.append('--ignore-not-found')
+
+            if self.recursive:
+                cmd.append('--recursive={}'.format(self.recursive))
+
+        return self._execute(cmd)
+
+    def exists(self):
+        cmd = ['get']
+
+        if self.filename:
+            cmd.append('--filename=' + ','.join(self.filename))
+            if self.recursive:
+                cmd.append('--recursive={}'.format(self.recursive))
+        else:
+            if not self.resource:
+                self.module.fail_json(msg='resource required without filename')
+
+            cmd.append(self.resource)
+
+            if self.name:
+                cmd.append(self.name)
+
+            if self.label:
+                cmd.append('--selector=' + self.label)
+
+            if self.all:
+                cmd.append('--all-namespaces')
+
+        cmd.append('--no-headers')
+
+        result = self._execute_nofail(cmd)
+        if not result:
+            return False
+        return True
+
+    # TODO: This is currently unused, perhaps convert to 'scale' with a replicas param?
+    def stop(self):
+
+        if not self.force and not self.exists():
+            return []
+
+        cmd = ['stop']
+
+        if self.filename:
+            cmd.append('--filename=' + ','.join(self.filename))
+            if self.recursive:
+                cmd.append('--recursive={}'.format(self.recursive))
+        else:
+            if not self.resource:
+                self.module.fail_json(msg='resource required to stop without filename')
+
+            cmd.append(self.resource)
+
+            if self.name:
+                cmd.append(self.name)
+
+            if self.label:
+                cmd.append('--selector=' + self.label)
+
+            if self.all:
+                cmd.append('--all')
+
+            if self.force:
+                cmd.append('--ignore-not-found')
+
+        return self._execute(cmd)
+
+
+def main():
+
+    module = AnsibleModule(
+        argument_spec=dict(
+            name=dict(),
+            filename=dict(type='list', aliases=['files', 'file', 'filenames']),
+            namespace=dict(),
+            resource=dict(),
+            label=dict(),
+            server=dict(),
+            kubeconfig=dict(),
+            kubectl=dict(),
+            force=dict(default=False, type='bool'),
+            wait=dict(default=False, type='bool'),
+            all=dict(default=False, type='bool'),
+            log_level=dict(default=0, type='int'),
+            state=dict(default='present', choices=['present', 'absent', 'latest', 'reloaded', 'stopped', 'exists']),
+            recursive=dict(default=False, type='bool'),
+            ),
+            mutually_exclusive=[['filename', 'list']]
+        )
+
+    changed = False
+
+    manager = KubeManager(module)
+    state = module.params.get('state')
+    if state == 'present':
+        result = manager.create(check=False)
+
+    elif state == 'absent':
+        result = manager.delete()
+
+    elif state == 'reloaded':
+        result = manager.replace()
+
+    elif state == 'stopped':
+        result = manager.stop()
+
+    elif state == 'latest':
+        result = manager.replace()
+
+    elif state == 'exists':
+        result = manager.exists()
+        module.exit_json(changed=changed,
+                     msg='%s' % result)
+
+    else:
+        module.fail_json(msg='Unrecognized state %s.' % state)
+
+    module.exit_json(changed=changed,
+                     msg='success: %s' % (' '.join(result))
+                     )
+
+
+from ansible.module_utils.basic import *  # noqa
+if __name__ == '__main__':
+    main()

internal/resources/kubespray/project/modules/kube.py

@@ -0,0 +1,366 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+DOCUMENTATION = """
+---
+module: kube
+short_description: Manage Kubernetes Cluster
+description:
+  - Create, replace, remove, and stop resources within a Kubernetes Cluster
+version_added: "2.0"
+options:
+  name:
+    required: false
+    default: null
+    description:
+      - The name associated with resource
+  filename:
+    required: false
+    default: null
+    description:
+      - The path and filename of the resource(s) definition file(s).
+      - To operate on several files this can accept a comma separated list of files or a list of files.
+    aliases: [ 'files', 'file', 'filenames' ]
+  kubectl:
+    required: false
+    default: null
+    description:
+      - The path to the kubectl bin
+  namespace:
+    required: false
+    default: null
+    description:
+      - The namespace associated with the resource(s)
+  resource:
+    required: false
+    default: null
+    description:
+      - The resource to perform an action on. pods (po), replicationControllers (rc), services (svc)
+  label:
+    required: false
+    default: null
+    description:
+      - The labels used to filter specific resources.
+  server:
+    required: false
+    default: null
+    description:
+      - The url for the API server that commands are executed against.
+  kubeconfig:
+    required: false
+    default: null
+    description:
+      - The path to the kubeconfig.
+  force:
+    required: false
+    default: false
+    description:
+      - A flag to indicate to force delete, replace, or stop.
+  wait:
+    required: false
+    default: false
+    description:
+      - A flag to indicate to wait for resources to be created before continuing to the next step
+  all:
+    required: false
+    default: false
+    description:
+      - A flag to indicate delete all, stop all, or all namespaces when checking exists.
+  log_level:
+    required: false
+    default: 0
+    description:
+      - Indicates the level of verbosity of logging by kubectl.
+  state:
+    required: false
+    choices: ['present', 'absent', 'latest', 'reloaded', 'stopped']
+    default: present
+    description:
+      - present handles checking existence or creating if definition file provided,
+        absent handles deleting resource(s) based on other options,
+        latest handles creating or updating based on existence,
+        reloaded handles updating resource(s) definition using definition file,
+        stopped handles stopping resource(s) based on other options.
+  recursive:
+    required: false
+    default: false
+    description:
+      - Process the directory used in -f, --filename recursively.
+        Useful when you want to manage related manifests organized
+        within the same directory.
+requirements:
+  - kubectl
+author: "Kenny Jones (@kenjones-cisco)"
+"""
+
+EXAMPLES = """
+- name: test nginx is present
+  kube: name=nginx resource=rc state=present
+
+- name: test nginx is stopped
+  kube: name=nginx resource=rc state=stopped
+
+- name: test nginx is absent
+  kube: name=nginx resource=rc state=absent
+
+- name: test nginx is present
+  kube: filename=/tmp/nginx.yml
+
+- name: test nginx and postgresql are present
+  kube: files=/tmp/nginx.yml,/tmp/postgresql.yml
+
+- name: test nginx and postgresql are present
+  kube:
+    files:
+      - /tmp/nginx.yml
+      - /tmp/postgresql.yml
+"""
+
+
+class KubeManager(object):
+
+    def __init__(self, module):
+
+        self.module = module
+
+        self.kubectl = module.params.get('kubectl')
+        if self.kubectl is None:
+            self.kubectl =  module.get_bin_path('kubectl', True)
+        self.base_cmd = [self.kubectl]
+
+        if module.params.get('server'):
+            self.base_cmd.append('--server=' + module.params.get('server'))
+
+        if module.params.get('kubeconfig'):
+            self.base_cmd.append('--kubeconfig=' + module.params.get('kubeconfig'))
+
+        if module.params.get('log_level'):
+            self.base_cmd.append('--v=' + str(module.params.get('log_level')))
+
+        if module.params.get('namespace'):
+            self.base_cmd.append('--namespace=' + module.params.get('namespace'))
+
+
+        self.all = module.params.get('all')
+        self.force = module.params.get('force')
+        self.wait = module.params.get('wait')
+        self.name = module.params.get('name')
+        self.filename = [f.strip() for f in module.params.get('filename') or []]
+        self.resource = module.params.get('resource')
+        self.label = module.params.get('label')
+        self.recursive = module.params.get('recursive')
+
+    def _execute(self, cmd):
+        args = self.base_cmd + cmd
+        try:
+            rc, out, err = self.module.run_command(args)
+            if rc != 0:
+                self.module.fail_json(
+                    msg='error running kubectl (%s) command (rc=%d), out=\'%s\', err=\'%s\'' % (' '.join(args), rc, out, err))
+        except Exception as exc:
+            self.module.fail_json(
+                msg='error running kubectl (%s) command: %s' % (' '.join(args), str(exc)))
+        return out.splitlines()
+
+    def _execute_nofail(self, cmd):
+        args = self.base_cmd + cmd
+        rc, out, err = self.module.run_command(args)
+        if rc != 0:
+            return None
+        return out.splitlines()
+
+    def create(self, check=True, force=True):
+        if check and self.exists():
+            return []
+
+        cmd = ['apply']
+
+        if force:
+            cmd.append('--force')
+
+        if self.wait:
+            cmd.append('--wait')
+
+        if self.recursive:
+            cmd.append('--recursive={}'.format(self.recursive))
+
+        if not self.filename:
+            self.module.fail_json(msg='filename required to create')
+
+        cmd.append('--filename=' + ','.join(self.filename))
+
+        return self._execute(cmd)
+
+    def replace(self, force=True):
+
+        cmd = ['apply']
+
+        if force:
+            cmd.append('--force')
+
+        if self.wait:
+            cmd.append('--wait')
+
+        if self.recursive:
+            cmd.append('--recursive={}'.format(self.recursive))
+
+        if not self.filename:
+            self.module.fail_json(msg='filename required to reload')
+
+        cmd.append('--filename=' + ','.join(self.filename))
+
+        return self._execute(cmd)
+
+    def delete(self):
+
+        if not self.force and not self.exists():
+            return []
+
+        cmd = ['delete']
+
+        if self.filename:
+            cmd.append('--filename=' + ','.join(self.filename))
+            if self.recursive:
+                cmd.append('--recursive={}'.format(self.recursive))
+        else:
+            if not self.resource:
+                self.module.fail_json(msg='resource required to delete without filename')
+
+            cmd.append(self.resource)
+
+            if self.name:
+                cmd.append(self.name)
+
+            if self.label:
+                cmd.append('--selector=' + self.label)
+
+            if self.all:
+                cmd.append('--all')
+
+            if self.force:
+                cmd.append('--ignore-not-found')
+
+            if self.recursive:
+                cmd.append('--recursive={}'.format(self.recursive))
+
+        return self._execute(cmd)
+
+    def exists(self):
+        cmd = ['get']
+
+        if self.filename:
+            cmd.append('--filename=' + ','.join(self.filename))
+            if self.recursive:
+                cmd.append('--recursive={}'.format(self.recursive))
+        else:
+            if not self.resource:
+                self.module.fail_json(msg='resource required without filename')
+
+            cmd.append(self.resource)
+
+            if self.name:
+                cmd.append(self.name)
+
+            if self.label:
+                cmd.append('--selector=' + self.label)
+
+            if self.all:
+                cmd.append('--all-namespaces')
+
+        cmd.append('--no-headers')
+
+        result = self._execute_nofail(cmd)
+        if not result:
+            return False
+        return True
+
+    # TODO: This is currently unused, perhaps convert to 'scale' with a replicas param?
+    def stop(self):
+
+        if not self.force and not self.exists():
+            return []
+
+        cmd = ['stop']
+
+        if self.filename:
+            cmd.append('--filename=' + ','.join(self.filename))
+            if self.recursive:
+                cmd.append('--recursive={}'.format(self.recursive))
+        else:
+            if not self.resource:
+                self.module.fail_json(msg='resource required to stop without filename')
+
+            cmd.append(self.resource)
+
+            if self.name:
+                cmd.append(self.name)
+
+            if self.label:
+                cmd.append('--selector=' + self.label)
+
+            if self.all:
+                cmd.append('--all')
+
+            if self.force:
+                cmd.append('--ignore-not-found')
+
+        return self._execute(cmd)
+
+
+def main():
+
+    module = AnsibleModule(
+        argument_spec=dict(
+            name=dict(),
+            filename=dict(type='list', aliases=['files', 'file', 'filenames']),
+            namespace=dict(),
+            resource=dict(),
+            label=dict(),
+            server=dict(),
+            kubeconfig=dict(),
+            kubectl=dict(),
+            force=dict(default=False, type='bool'),
+            wait=dict(default=False, type='bool'),
+            all=dict(default=False, type='bool'),
+            log_level=dict(default=0, type='int'),
+            state=dict(default='present', choices=['present', 'absent', 'latest', 'reloaded', 'stopped', 'exists']),
+            recursive=dict(default=False, type='bool'),
+            ),
+            mutually_exclusive=[['filename', 'list']]
+        )
+
+    changed = False
+
+    manager = KubeManager(module)
+    state = module.params.get('state')
+    if state == 'present':
+        result = manager.create(check=False)
+
+    elif state == 'absent':
+        result = manager.delete()
+
+    elif state == 'reloaded':
+        result = manager.replace()
+
+    elif state == 'stopped':
+        result = manager.stop()
+
+    elif state == 'latest':
+        result = manager.replace()
+
+    elif state == 'exists':
+        result = manager.exists()
+        module.exit_json(changed=changed,
+                     msg='%s' % result)
+
+    else:
+        module.fail_json(msg='Unrecognized state %s.' % state)
+
+    module.exit_json(changed=changed,
+                     msg='success: %s' % (' '.join(result))
+                     )
+
+
+from ansible.module_utils.basic import *  # noqa
+if __name__ == '__main__':
+    main()

internal/resources/kubespray/project/recover_control_plane.yml

@@ -0,0 +1,28 @@
+---
+- name: Common tasks for every playbooks
+  import_playbook: boilerplate.yml
+
+- name: Recover etcd
+  hosts: etcd[0]
+  environment: "{{ proxy_disable_env }}"
+  roles:
+    - { role: kubespray_defaults}
+    - role: recover_control_plane/etcd
+      when: etcd_deployment_type != "kubeadm"
+
+- name: Recover control plane
+  hosts: kube_control_plane[0]
+  environment: "{{ proxy_disable_env }}"
+  roles:
+    - { role: kubespray_defaults}
+    - { role: recover_control_plane/control-plane }
+
+- name: Apply whole cluster install
+  import_playbook: cluster.yml
+
+- name: Perform post recover tasks
+  hosts: kube_control_plane
+  environment: "{{ proxy_disable_env }}"
+  roles:
+    - { role: kubespray_defaults}
+    - { role: recover_control_plane/post-recover }

internal/resources/kubespray/project/remove_node.yml

@@ -0,0 +1,58 @@
+---
+- name: Validate nodes for removal
+  hosts: localhost
+  tasks:
+    - name: Assert that nodes are specified for removal
+      assert:
+        that:
+          - node is defined
+          - node | length > 0
+        msg: "No nodes specified for removal. The `node` variable must be set explicitly."
+
+- name: Common tasks for every playbooks
+  import_playbook: boilerplate.yml
+
+- name: Confirm node removal
+  hosts: "{{ node | default('this_is_unreachable') }}"
+  gather_facts: false
+  tasks:
+    - name: Confirm Execution
+      pause:
+        prompt: "Are you sure you want to delete nodes state? Type 'yes' to delete nodes."
+      register: pause_result
+      run_once: true
+      when:
+        - not (skip_confirmation | default(false) | bool)
+
+    - name: Fail if user does not confirm deletion
+      fail:
+        msg: "Delete nodes confirmation failed"
+      when: pause_result.user_input | default('yes') != 'yes'
+
+- name: Gather facts
+  import_playbook: facts.yml
+  when: reset_nodes | default(True) | bool
+
+- name: Reset node
+  hosts: "{{ node | default('this_is_unreachable') }}"
+  gather_facts: false
+  environment: "{{ proxy_disable_env }}"
+  pre_tasks:
+    - name: Gather information about installed services
+      service_facts:
+      when: reset_nodes | default(True) | bool
+  roles:
+    - { role: kubespray_defaults, when: reset_nodes | default(True) | bool }
+    - { role: remove_node/pre_remove, tags: pre-remove }
+    - role: remove-node/remove-etcd-node
+      when: "'etcd' in group_names"
+    - { role: reset, tags: reset, when: reset_nodes | default(True) | bool }
+
+# Currently cannot remove first control plane node or first etcd node
+- name: Post node removal
+  hosts: "{{ node | default('this_is_unreachable') }}"
+  gather_facts: false
+  environment: "{{ proxy_disable_env }}"
+  roles:
+    - { role: kubespray_defaults, when: reset_nodes | default(True) | bool }
+    - { role: remove-node/post-remove, tags: post-remove }

internal/resources/kubespray/project/reset.yml

@@ -0,0 +1,24 @@
+---
+- name: Common tasks for every playbooks
+  import_playbook: boilerplate.yml
+
+- name: Gather facts
+  import_playbook: facts.yml
+
+- name: Reset cluster
+  hosts: etcd:k8s_cluster:calico_rr
+  gather_facts: false
+  pre_tasks:
+    - name: Gather information about installed services
+      service_facts:
+
+  environment: "{{ proxy_disable_env }}"
+  roles:
+    - { role: kubespray_defaults }
+    - {
+        role: kubernetes/preinstall,
+        when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'",
+        tags: resolvconf,
+        dns_early: true,
+      }
+    - { role: reset, tags: reset }

internal/resources/kubespray/project/roles/adduser/defaults/main.yml

@@ -0,0 +1,27 @@
+---
+kube_owner: kube
+kube_cert_group: kube-cert
+etcd_data_dir: "/var/lib/etcd"
+
+addusers:
+  etcd:
+    name: etcd
+    comment: "Etcd user"
+    create_home: false
+    system: true
+    shell: /sbin/nologin
+  kube:
+    name: kube
+    comment: "Kubernetes user"
+    create_home: false
+    system: true
+    shell: /sbin/nologin
+    group: "{{ kube_cert_group }}"
+
+adduser:
+  name: "{{ user.name }}"
+  group: "{{ user.name | default(None) }}"
+  comment: "{{ user.comment | default(None) }}"
+  shell: "{{ user.shell | default(None) }}"
+  system: "{{ user.system | default(None) }}"
+  create_home: "{{ user.create_home | default(None) }}"

internal/resources/kubespray/project/roles/adduser/molecule/default/converge.yml

@@ -0,0 +1,10 @@
+---
+- name: Converge
+  hosts: all
+  become: true
+  gather_facts: false
+  roles:
+    - role: adduser
+  vars:
+    user:
+      name: foo

internal/resources/kubespray/project/roles/adduser/molecule/default/molecule.yml

@@ -0,0 +1,19 @@
+---
+role_name_check: 1
+dependency:
+  name: galaxy
+platforms:
+  - name: ubuntu20
+    cloud_image: ubuntu-2004
+    vm_cpu_cores: 1
+    vm_memory: 512
+provisioner:
+  name: ansible
+  config_options:
+    defaults:
+      callbacks_enabled: profile_tasks
+      timeout: 120
+  playbooks:
+    create: ../../../../tests/cloud_playbooks/create-kubevirt.yml
+verifier:
+  name: testinfra

internal/resources/kubespray/project/roles/adduser/tasks/main.yml

@@ -0,0 +1,16 @@
+---
+- name: User | Create User Group
+  group:
+    name: "{{ user.group | default(user.name) }}"
+    system: "{{ user.system | default(omit) }}"
+
+- name: User | Create User
+  user:
+    comment: "{{ user.comment | default(omit) }}"
+    create_home: "{{ user.create_home | default(omit) }}"
+    group: "{{ user.group | default(user.name) }}"
+    home: "{{ user.home | default(omit) }}"
+    shell: "{{ user.shell | default(omit) }}"
+    name: "{{ user.name }}"
+    system: "{{ user.system | default(omit) }}"
+  when: user.name != "root"

internal/resources/kubespray/project/roles/adduser/vars/coreos.yml

@@ -0,0 +1,8 @@
+---
+addusers:
+  - name: kube
+    comment: "Kubernetes user"
+    shell: /sbin/nologin
+    system: true
+    group: "{{ kube_cert_group }}"
+    create_home: false

internal/resources/kubespray/project/roles/adduser/vars/debian.yml

@@ -0,0 +1,15 @@
+---
+addusers:
+  - name: etcd
+    comment: "Etcd user"
+    create_home: true
+    home: "{{ etcd_data_dir }}"
+    system: true
+    shell: /sbin/nologin
+
+  - name: kube
+    comment: "Kubernetes user"
+    create_home: false
+    system: true
+    shell: /sbin/nologin
+    group: "{{ kube_cert_group }}"

internal/resources/kubespray/project/roles/adduser/vars/redhat.yml

@@ -0,0 +1,15 @@
+---
+addusers:
+  - name: etcd
+    comment: "Etcd user"
+    create_home: true
+    home: "{{ etcd_data_dir }}"
+    system: true
+    shell: /sbin/nologin
+
+  - name: kube
+    comment: "Kubernetes user"
+    create_home: false
+    system: true
+    shell: /sbin/nologin
+    group: "{{ kube_cert_group }}"

internal/resources/kubespray/project/roles/bastion-ssh-config/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+ssh_bastion_confing__name: ssh-bastion.conf

internal/resources/kubespray/project/roles/bastion-ssh-config/molecule/default/converge.yml

@@ -0,0 +1,15 @@
+---
+- name: Converge
+  hosts: all
+  become: true
+  gather_facts: false
+  roles:
+    - role: bastion-ssh-config
+  tasks:
+    - name: Copy config to remote host
+      copy:
+        src: "{{ playbook_dir }}/{{ ssh_bastion_confing__name }}"
+        dest: "{{ ssh_bastion_confing__name }}"
+        owner: "{{ ansible_user }}"
+        group: "{{ ansible_user }}"
+        mode: "0644"

internal/resources/kubespray/project/roles/bastion-ssh-config/molecule/default/molecule.yml

@@ -0,0 +1,27 @@
+---
+role_name_check: 1
+dependency:
+  name: galaxy
+platforms:
+  - name: bastion-01
+    cloud_image: ubuntu-2004
+    vm_cpu_cores: 1
+    vm_memory: 512
+provisioner:
+  name: ansible
+  config_options:
+    defaults:
+      callbacks_enabled: profile_tasks
+      timeout: 120
+  inventory:
+    hosts:
+      all:
+        hosts:
+        children:
+          bastion:
+            hosts:
+              bastion-01:
+  playbooks:
+    create: ../../../../tests/cloud_playbooks/create-kubevirt.yml
+verifier:
+  name: testinfra

internal/resources/kubespray/project/roles/bastion-ssh-config/tasks/main.yml

@@ -0,0 +1,22 @@
+---
+- name: Set bastion host IP and port
+  set_fact:
+    bastion_ip: "{{ hostvars[groups['bastion'][0]]['ansible_host'] | d(hostvars[groups['bastion'][0]]['ansible_ssh_host']) }}"
+    bastion_port: "{{ hostvars[groups['bastion'][0]]['ansible_port'] | d(hostvars[groups['bastion'][0]]['ansible_ssh_port']) | d(22) }}"
+  delegate_to: localhost
+  connection: local
+
+# As we are actually running on localhost, the ansible_ssh_user is your local user when you try to use it directly
+# To figure out the real ssh user, we delegate this task to the bastion and store the ansible_user in real_user
+- name: Store the current ansible_user in the real_user fact
+  set_fact:
+    real_user: "{{ ansible_user }}"
+
+- name: Create ssh bastion conf
+  become: false
+  delegate_to: localhost
+  connection: local
+  template:
+    src: "{{ ssh_bastion_confing__name }}.j2"
+    dest: "{{ playbook_dir }}/{{ ssh_bastion_confing__name }}"
+    mode: "0640"

internal/resources/kubespray/project/roles/bastion-ssh-config/templates/ssh-bastion.conf.j2

@@ -0,0 +1,18 @@
+{% set vars={'hosts': ''} %}
+{% set user='' %}
+
+{% for h in groups['all'] %}
+{% if h not in groups['bastion'] %}
+{% if vars.update({'hosts': vars['hosts'] + ' ' + (hostvars[h].get('ansible_ssh_host') or hostvars[h]['ansible_host'])}) %}{% endif %}
+{% endif %}
+{% endfor %}
+
+Host {{ bastion_ip }}
+  Hostname {{ bastion_ip }}
+  StrictHostKeyChecking no
+  ControlMaster auto
+  ControlPath ~/.ssh/ansible-%r@%h:%p
+  ControlPersist 5m
+
+Host {{ vars['hosts'] }}
+  ProxyCommand ssh -F /dev/null -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -W %h:%p -p {{ bastion_port }} {{ real_user }}@{{ bastion_ip }} {% if ansible_ssh_private_key_file is defined %}-i {{ ansible_ssh_private_key_file }}{% endif %}

internal/resources/kubespray/project/roles/bootstrap-os/tasks/main.yml

@@ -0,0 +1,10 @@
+---
+- name: Warn for usage of deprecated role
+  fail:
+    msg: bootstrap-os is deprecated, switch to bootstrap_os
+  ignore_errors: true # noqa ignore-errors
+  run_once: true
+
+- name: Compat for direct role import
+  import_role:
+    name: bootstrap_os

internal/resources/kubespray/project/roles/bootstrap_os/defaults/main.yml

@@ -0,0 +1,44 @@
+---
+## CentOS/RHEL/AlmaLinux specific variables
+# Use the fastestmirror yum plugin
+centos_fastestmirror_enabled: false
+# Timeout (in seconds) for checking RHEL subscription status
+rh_subscription_check_timeout: 180
+
+## Flatcar Container Linux specific variables
+# Disable locksmithd or leave it in its current state
+coreos_locksmithd_disable: false
+
+# Install epel repo on Centos/RHEL
+epel_enabled: false
+
+## Oracle Linux specific variables
+# Install public repo on Oracle Linux
+use_oracle_public_repo: true
+
+## Ubuntu specific variables
+# Disable unattended-upgrades for Linux kernel and all packages start with linux- on Ubuntu
+ubuntu_kernel_unattended_upgrades_disabled: false
+# Stop unattended-upgrades if it is currently running on Ubuntu
+ubuntu_stop_unattended_upgrades: false
+
+fedora_coreos_packages:
+  - python
+  - python3-libselinux
+  - ethtool                 # required in kubeadm preflight phase for verifying the environment
+  - ipset                   # required in kubeadm preflight phase for verifying the environment
+  - conntrack-tools         # required by kube-proxy
+  - containernetworking-plugins  # required by crio
+
+## General
+# Set the hostname to inventory_hostname
+override_system_hostname: true
+
+is_fedora_coreos: false
+
+skip_http_proxy_on_os_packages: false
+
+# If this is true, debug information will be displayed but
+# may contain some private data, so it is recommended to set it to false
+# in the production environment.
+unsafe_show_logs: false

internal/resources/kubespray/project/roles/bootstrap_os/files/bootstrap.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+set -e
+
+BINDIR="/opt/bin"
+if [[ -e $BINDIR/.bootstrapped ]]; then
+  exit 0
+fi
+
+ARCH=$(uname -m)
+case $ARCH in
+  "x86_64")
+    PYPY_ARCH=linux64
+    PYPI_HASH=46818cb3d74b96b34787548343d266e2562b531ddbaf330383ba930ff1930ed5
+    ;;
+  "aarch64")
+    PYPY_ARCH=aarch64
+    PYPI_HASH=2e1ae193d98bc51439642a7618d521ea019f45b8fb226940f7e334c548d2b4b9
+    ;;
+  *)
+    echo "Unsupported Architecture: ${ARCH}"
+    exit 1
+esac
+
+PYTHON_VERSION=3.9
+PYPY_VERSION=7.3.9
+PYPY_FILENAME="pypy${PYTHON_VERSION}-v${PYPY_VERSION}-${PYPY_ARCH}"
+PYPI_URL="https://downloads.python.org/pypy/${PYPY_FILENAME}.tar.bz2"
+
+mkdir -p $BINDIR
+
+cd $BINDIR
+
+TAR_FILE=pyp.tar.bz2
+wget -O "${TAR_FILE}" "${PYPI_URL}"
+echo "${PYPI_HASH} ${TAR_FILE}" | sha256sum -c -
+tar -xjf "${TAR_FILE}" && rm "${TAR_FILE}"
+mv -n "${PYPY_FILENAME}" pypy3
+
+ln -s ./pypy3/bin/pypy3 python
+$BINDIR/python --version
+
+# install PyYAML
+./python -m ensurepip
+./python -m pip install pyyaml
+
+touch $BINDIR/.bootstrapped

internal/resources/kubespray/project/roles/bootstrap_os/handlers/main.yml

@@ -0,0 +1,4 @@
+---
+- name: RHEL auto-attach subscription
+  command: /sbin/subscription-manager attach --auto
+  become: true

internal/resources/kubespray/project/roles/bootstrap_os/meta/main.yml

@@ -0,0 +1,3 @@
+---
+dependencies:
+  - role: kubespray_defaults

internal/resources/kubespray/project/roles/bootstrap_os/molecule/default/converge.yml

@@ -0,0 +1,7 @@
+---
+- name: Converge
+  hosts: all
+  gather_facts: false
+  become: true
+  roles:
+    - role: bootstrap_os

internal/resources/kubespray/project/roles/bootstrap_os/molecule/default/molecule.yml

@@ -0,0 +1,37 @@
+---
+role_name_check: 1
+dependency:
+  name: galaxy
+platforms:
+  - name: ubuntu20
+    cloud_image: ubuntu-2004
+    vm_cpu_cores: 1
+    vm_memory: 512
+  - name: ubuntu22
+    cloud_image: ubuntu-2204
+    vm_cpu_cores: 1
+    vm_memory: 512
+  - name: almalinux9
+    cloud_image: almalinux-9
+    vm_cpu_cores: 1
+    vm_memory: 512
+  - name: debian12
+    cloud_image: debian-12
+    vm_cpu_cores: 1
+    vm_memory: 512
+provisioner:
+  name: ansible
+  config_options:
+    defaults:
+      callbacks_enabled: profile_tasks
+      timeout: 120
+  inventory:
+    group_vars:
+      all:
+        user:
+          name: foo
+          comment: My test comment
+  playbooks:
+    create: ../../../../tests/cloud_playbooks/create-kubevirt.yml
+verifier:
+  name: testinfra

internal/resources/kubespray/project/roles/bootstrap_os/molecule/default/tests/test_default.py

@@ -0,0 +1,11 @@
+import os
+
+import testinfra.utils.ansible_runner
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+    os.environ['MOLECULE_INVENTORY_FILE']
+).get_hosts('all')
+
+
+def test_python(host):
+    assert host.exists('python3') or host.exists('python')

internal/resources/kubespray/project/roles/bootstrap_os/tasks/amzn.yml

@@ -0,0 +1,16 @@
+---
+- name: Enable selinux-ng repo for Amazon Linux for container-selinux
+  command: amazon-linux-extras enable selinux-ng
+
+- name: Enable EPEL repo for Amazon Linux
+  yum_repository:
+    name: epel
+    file: epel
+    description: Extra Packages for Enterprise Linux 7 - $basearch
+    baseurl: http://download.fedoraproject.org/pub/epel/7/$basearch
+    gpgcheck: true
+    gpgkey: http://download.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-7
+    skip_if_unavailable: true
+    enabled: true
+    repo_gpgcheck: false
+  when: epel_enabled

internal/resources/kubespray/project/roles/bootstrap_os/tasks/centos.yml

@@ -0,0 +1,110 @@
+---
+- name: Gather host facts to get ansible_distribution_version ansible_distribution_major_version
+  setup:
+    gather_subset: '!all'
+    filter: ansible_distribution_*version
+
+- name: Add proxy to yum.conf or dnf.conf if http_proxy is defined
+  community.general.ini_file:
+    path: "{{ ((ansible_distribution_major_version | int) < 8) | ternary('/etc/yum.conf', '/etc/dnf/dnf.conf') }}"
+    section: main
+    option: proxy
+    value: "{{ http_proxy | default(omit) }}"
+    state: "{{ http_proxy | default(False) | ternary('present', 'absent') }}"
+    no_extra_spaces: true
+    mode: "0644"
+  become: true
+  when: not skip_http_proxy_on_os_packages
+
+# For Oracle Linux install public repo
+- name: Download Oracle Linux public yum repo
+  get_url:
+    url: https://yum.oracle.com/public-yum-ol7.repo
+    dest: /etc/yum.repos.d/public-yum-ol7.repo
+    mode: "0644"
+  when:
+    - use_oracle_public_repo | default(true)
+    - '''ID="ol"'' in os_release.stdout_lines'
+    - (ansible_distribution_version | float) < 7.6
+  environment: "{{ proxy_env }}"
+
+- name: Enable Oracle Linux repo
+  community.general.ini_file:
+    dest: /etc/yum.repos.d/public-yum-ol7.repo
+    section: "{{ item }}"
+    option: enabled
+    value: "1"
+    mode: "0644"
+  with_items:
+    - ol7_latest
+    - ol7_addons
+    - ol7_developer_EPEL
+  when:
+    - use_oracle_public_repo | default(true)
+    - '''ID="ol"'' in os_release.stdout_lines'
+    - (ansible_distribution_version | float) < 7.6
+
+- name: Install EPEL for Oracle Linux repo package
+  package:
+    name: "oracle-epel-release-el{{ ansible_distribution_major_version }}"
+    state: present
+  when:
+    - use_oracle_public_repo | default(true)
+    - '''ID="ol"'' in os_release.stdout_lines'
+    - (ansible_distribution_version | float) >= 7.6
+
+- name: Enable Oracle Linux repo
+  community.general.ini_file:
+    dest: "/etc/yum.repos.d/oracle-linux-ol{{ ansible_distribution_major_version }}.repo"
+    section: "ol{{ ansible_distribution_major_version }}_addons"
+    option: "{{ item.option }}"
+    value: "{{ item.value }}"
+    mode: "0644"
+  with_items:
+    - { option: "name", value: "ol{{ ansible_distribution_major_version }}_addons" }
+    - { option: "enabled", value: "1" }
+    - { option: "baseurl", value: "http://yum.oracle.com/repo/OracleLinux/OL{{ ansible_distribution_major_version }}/addons/$basearch/" }
+  when:
+    - use_oracle_public_repo | default(true)
+    - '''ID="ol"'' in os_release.stdout_lines'
+    - (ansible_distribution_version | float) >= 7.6
+
+- name: Enable Centos extra repo for Oracle Linux
+  community.general.ini_file:
+    dest: "/etc/yum.repos.d/centos-extras.repo"
+    section: "extras"
+    option: "{{ item.option }}"
+    value: "{{ item.value }}"
+    mode: "0644"
+  with_items:
+    - { option: "name", value: "CentOS-{{ ansible_distribution_major_version }} - Extras" }
+    - { option: "enabled", value: "1" }
+    - { option: "gpgcheck", value: "0" }
+    - { option: "baseurl", value: "http://mirror.centos.org/centos/{{ ansible_distribution_major_version }}/extras/$basearch/os/" }
+  when:
+    - use_oracle_public_repo | default(true)
+    - '''ID="ol"'' in os_release.stdout_lines'
+    - (ansible_distribution_version | float) >= 7.6
+    - (ansible_distribution_version | float) < 9
+
+# CentOS ships with python installed
+
+- name: Check presence of fastestmirror.conf
+  stat:
+    path: /etc/yum/pluginconf.d/fastestmirror.conf
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: fastestmirror
+
+# the fastestmirror plugin can actually slow down Ansible deployments
+- name: Disable fastestmirror plugin if requested
+  lineinfile:
+    dest: /etc/yum/pluginconf.d/fastestmirror.conf
+    regexp: "^enabled=.*"
+    line: "enabled=0"
+    state: present
+  become: true
+  when:
+    - fastestmirror.stat.exists
+    - not centos_fastestmirror_enabled

internal/resources/kubespray/project/roles/bootstrap_os/tasks/clear-linux-os.yml

@@ -0,0 +1,16 @@
+---
+# ClearLinux ships with Python installed
+
+- name: Install basic package to run containers
+  package:
+    name: containers-basic
+    state: present
+
+- name: Make sure docker service is enabled
+  systemd_service:
+    name: docker
+    masked: false
+    enabled: true
+    daemon_reload: true
+    state: started
+  become: true

internal/resources/kubespray/project/roles/bootstrap_os/tasks/debian.yml

@@ -0,0 +1,64 @@
+---
+# Some Debian based distros ship without Python installed
+
+- name: Check if bootstrap is needed
+  raw: which python3
+  register: need_bootstrap
+  failed_when: false
+  changed_when: false
+  # This command should always run, even in check mode
+  check_mode: false
+  tags:
+    - facts
+
+- name: Check http::proxy in apt configuration files
+  raw: apt-config dump | grep -qsi 'Acquire::http::proxy'
+  register: need_http_proxy
+  failed_when: false
+  changed_when: false
+  # This command should always run, even in check mode
+  check_mode: false
+
+- name: Add http_proxy to /etc/apt/apt.conf if http_proxy is defined
+  raw: echo 'Acquire::http::proxy "{{ http_proxy }}";' >> /etc/apt/apt.conf
+  become: true
+  when:
+    - http_proxy is defined
+    - need_http_proxy.rc != 0
+    - not skip_http_proxy_on_os_packages
+
+- name: Check https::proxy in apt configuration files
+  raw: apt-config dump | grep -qsi 'Acquire::https::proxy'
+  register: need_https_proxy
+  failed_when: false
+  changed_when: false
+  # This command should always run, even in check mode
+  check_mode: false
+
+- name: Add https_proxy to /etc/apt/apt.conf if https_proxy is defined
+  raw: echo 'Acquire::https::proxy "{{ https_proxy }}";' >> /etc/apt/apt.conf
+  become: true
+  when:
+    - https_proxy is defined
+    - need_https_proxy.rc != 0
+    - not skip_http_proxy_on_os_packages
+
+- name: Install python3
+  raw:
+    apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y python3-minimal
+  become: true
+  when:
+    - need_bootstrap.rc != 0
+
+- name: Update Apt cache
+  raw: apt-get update --allow-releaseinfo-change
+  become: true
+  when:
+    - os_release_dict['ID'] == 'debian'
+    - os_release_dict['VERSION_ID'] in ["10", "11"]
+  register: bootstrap_update_apt_result
+  changed_when:
+    - '"changed its" in bootstrap_update_apt_result.stdout'
+    - '"value from" in bootstrap_update_apt_result.stdout'
+  ignore_errors: true

internal/resources/kubespray/project/roles/bootstrap_os/tasks/fedora-coreos.yml

@@ -0,0 +1,40 @@
+---
+
+- name: Check if bootstrap is needed
+  raw: which python
+  register: need_bootstrap
+  failed_when: false
+  changed_when: false
+  tags:
+    - facts
+
+- name: Remove podman network cni
+  raw: "podman network rm podman"
+  become: true
+  ignore_errors: true  # noqa ignore-errors
+  when: need_bootstrap.rc != 0
+
+- name: Clean up possible pending packages on fedora coreos
+  raw: "export http_proxy={{ http_proxy | default('') }};rpm-ostree cleanup -p }}"
+  become: true
+  when: need_bootstrap.rc != 0
+
+- name: Install required packages on fedora coreos
+  raw: "export http_proxy={{ http_proxy | default('') }};rpm-ostree install --allow-inactive {{ fedora_coreos_packages | join(' ') }}"
+  become: true
+  when: need_bootstrap.rc != 0
+
+- name: Reboot immediately for updated ostree
+  raw: "nohup bash -c 'sleep 5s && shutdown -r now'"
+  become: true
+  ignore_errors: true  # noqa ignore-errors
+  ignore_unreachable: true
+  when: need_bootstrap.rc != 0
+
+- name: Wait for the reboot to complete
+  wait_for_connection:
+    timeout: 240
+    connect_timeout: 20
+    delay: 5
+    sleep: 5
+  when: need_bootstrap.rc != 0

internal/resources/kubespray/project/roles/bootstrap_os/tasks/fedora.yml

@@ -0,0 +1,30 @@
+---
+# Some Fedora based distros ship without Python installed
+
+- name: Check if bootstrap is needed
+  raw: which python
+  register: need_bootstrap
+  failed_when: false
+  changed_when: false
+  tags:
+    - facts
+
+- name: Add proxy to dnf.conf if http_proxy is defined
+  community.general.ini_file:
+    path: "/etc/dnf/dnf.conf"
+    section: main
+    option: proxy
+    value: "{{ http_proxy | default(omit) }}"
+    state: "{{ http_proxy | default(False) | ternary('present', 'absent') }}"
+    no_extra_spaces: true
+    mode: "0644"
+  become: true
+  when: not skip_http_proxy_on_os_packages
+
+# libselinux-python3 is required on SELinux enabled hosts
+# See https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html#managed-node-requirements
+- name: Install ansible requirements
+  raw: "dnf install --assumeyes python3 python3-dnf libselinux-python3"
+  become: true
+  when:
+    - need_bootstrap.rc != 0

internal/resources/kubespray/project/roles/bootstrap_os/tasks/flatcar.yml

@@ -0,0 +1,34 @@
+---
+# Flatcar Container Linux ships without Python installed
+
+- name: Check if bootstrap is needed
+  raw: stat /opt/bin/.bootstrapped
+  register: need_bootstrap
+  failed_when: false
+  changed_when: false
+  tags:
+    - facts
+
+- name: Run bootstrap.sh
+  script: bootstrap.sh
+  become: true
+  environment: "{{ proxy_env }}"
+  when:
+    - need_bootstrap.rc != 0
+
+# Workaround ansible https://github.com/ansible/ansible/pull/82821
+# We set the interpreter rather than ansible_python_interpreter to allow
+# - using virtual env with task level ansible_python_interpreter later
+# - let users specify an ansible_python_interpreter in group_vars
+
+- name: Make interpreter discovery works on Flatcar
+  set_fact:
+    ansible_interpreter_python_fallback: "{{ (ansible_interpreter_python_fallback | default([])) + ['/opt/bin/python'] }}"
+
+- name: Disable auto-upgrade
+  systemd_service:
+    name: locksmithd.service
+    masked: true
+    state: stopped
+  when:
+    - coreos_locksmithd_disable

internal/resources/kubespray/project/roles/bootstrap_os/tasks/main.yml

@@ -0,0 +1,62 @@
+---
+- name: Fetch /etc/os-release
+  raw: cat /etc/os-release
+  register: os_release
+  changed_when: false
+  # This command should always run, even in check mode
+  check_mode: false
+
+- name: Include distro specifics vars and tasks
+  vars:
+    os_release_dict: "{{ os_release.stdout_lines | select('regex', '^.+=.*$') | map('regex_replace', '\"', '') |
+                         map('split', '=') | community.general.dict }}"
+  block:
+  - name: Include vars
+    include_vars: "{{ item }}"
+    tags:
+    - facts
+    with_first_found:
+    - &search
+      files:
+      - "{{ os_release_dict['ID'] }}-{{ os_release_dict['VARIANT_ID'] }}.yml"
+      - "{{ os_release_dict['ID'] }}.yml"
+      paths:
+      - vars/
+      skip: true
+  - name: Include tasks
+    include_tasks: "{{ included_tasks_file }}"
+    with_first_found:
+    - <<: *search
+      paths: []
+    loop_control:
+      loop_var: included_tasks_file
+
+- name: Install system packages
+  import_role:
+    name: system_packages
+  tags:
+  - system-packages
+
+- name: Create remote_tmp for it is used by another module
+  file:
+    path: "{{ ansible_remote_tmp | default('~/.ansible/tmp') }}"
+    state: directory
+    mode: "0700"
+
+- name: Gather facts
+  setup:
+    gather_subset: '!all'
+    filter: ansible_*
+
+- name: Assign inventory name to unconfigured hostnames (non-CoreOS, non-Flatcar, Suse and ClearLinux, non-Fedora)
+  hostname:
+    name: "{{ inventory_hostname }}"
+  when: override_system_hostname
+
+- name: Ensure bash_completion.d folder exists
+  file:
+    name: /etc/bash_completion.d/
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"

internal/resources/kubespray/project/roles/bootstrap_os/tasks/openEuler.yml

@@ -0,0 +1,3 @@
+---
+- name: Import Centos boostrap for openEuler
+  import_tasks: centos.yml

internal/resources/kubespray/project/roles/bootstrap_os/tasks/opensuse-leap.yml

@@ -0,0 +1,3 @@
+---
+- name: Import Opensuse bootstrap
+  import_tasks: opensuse.yml

internal/resources/kubespray/project/roles/bootstrap_os/tasks/opensuse-tumbleweed.yml

@@ -0,0 +1,3 @@
+---
+- name: Import Opensuse bootstrap
+  import_tasks: opensuse.yml

internal/resources/kubespray/project/roles/bootstrap_os/tasks/opensuse.yml

@@ -0,0 +1,85 @@
+---
+# OpenSUSE ships with Python installed
+- name: Gather host facts to get ansible_distribution_version ansible_distribution_major_version
+  setup:
+    gather_subset: '!all'
+    filter: ansible_distribution_*version
+
+- name: Check that /etc/sysconfig/proxy file exists
+  stat:
+    path: /etc/sysconfig/proxy
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: stat_result
+
+- name: Create the /etc/sysconfig/proxy empty file
+  file:  # noqa risky-file-permissions
+    path: /etc/sysconfig/proxy
+    state: touch
+  when:
+    - http_proxy is defined or https_proxy is defined
+    - not stat_result.stat.exists
+
+- name: Set the http_proxy in /etc/sysconfig/proxy
+  lineinfile:
+    path: /etc/sysconfig/proxy
+    regexp: '^HTTP_PROXY='
+    line: 'HTTP_PROXY="{{ http_proxy }}"'
+  become: true
+  when:
+    - http_proxy is defined
+
+- name: Set the https_proxy in /etc/sysconfig/proxy
+  lineinfile:
+    path: /etc/sysconfig/proxy
+    regexp: '^HTTPS_PROXY='
+    line: 'HTTPS_PROXY="{{ https_proxy }}"'
+  become: true
+  when:
+    - https_proxy is defined
+
+- name: Enable proxies
+  lineinfile:
+    path: /etc/sysconfig/proxy
+    regexp: '^PROXY_ENABLED='
+    line: 'PROXY_ENABLED="yes"'
+  become: true
+  when:
+    - http_proxy is defined or https_proxy is defined
+
+# Required for zypper module
+- name: Install python-xml
+  shell: zypper refresh && zypper --non-interactive install python-xml
+  changed_when: false
+  become: true
+  tags:
+    - facts
+
+# Without this package, the get_url module fails when trying to handle https
+- name: Install python-cryptography
+  community.general.zypper:
+    name: python-cryptography
+    state: present
+    update_cache: true
+  become: true
+  when:
+    - ansible_distribution_version is version('15.4', '<')
+
+- name: Install python3-cryptography
+  community.general.zypper:
+    name: python3-cryptography
+    state: present
+    update_cache: true
+  become: true
+  when:
+    - ansible_distribution_version is version('15.4', '>=')
+
+# Nerdctl needs some basic packages to get an environment up
+- name: Install basic dependencies
+  community.general.zypper:
+    name:
+      - iptables
+      - apparmor-parser
+    state: present
+  become: true

internal/resources/kubespray/project/roles/bootstrap_os/tasks/rhel.yml

@@ -0,0 +1,95 @@
+---
+- name: Gather host facts to get ansible_distribution_version ansible_distribution_major_version
+  setup:
+    gather_subset: '!all'
+    filter: ansible_distribution_*version
+
+- name: Add proxy to yum.conf or dnf.conf if http_proxy is defined
+  community.general.ini_file:
+    path: "{{ ((ansible_distribution_major_version | int) < 8) | ternary('/etc/yum.conf', '/etc/dnf/dnf.conf') }}"
+    section: main
+    option: proxy
+    value: "{{ http_proxy | default(omit) }}"
+    state: "{{ http_proxy | default(False) | ternary('present', 'absent') }}"
+    no_extra_spaces: true
+    mode: "0644"
+  become: true
+  when: not skip_http_proxy_on_os_packages
+
+- name: Add proxy to RHEL subscription-manager if http_proxy is defined
+  command: /sbin/subscription-manager config --server.proxy_hostname={{ http_proxy | regex_replace(':\d+$') | regex_replace('^.*://') }} --server.proxy_port={{ http_proxy | regex_replace('^.*:') }}
+  become: true
+  when:
+    - not skip_http_proxy_on_os_packages
+    - http_proxy is defined
+
+- name: Check RHEL subscription-manager status
+  command: /sbin/subscription-manager status
+  register: rh_subscription_status
+  changed_when: "rh_subscription_status.rc != 0"
+  ignore_errors: true  # noqa ignore-errors
+  timeout: "{{ rh_subscription_check_timeout }}"
+  become: true
+
+- name: RHEL subscription Organization ID/Activation Key registration
+  community.general.redhat_subscription:
+    state: present
+    org_id: "{{ rh_subscription_org_id }}"
+    activationkey: "{{ rh_subscription_activation_key }}"
+    force_register: true
+  notify: RHEL auto-attach subscription
+  become: true
+  when:
+    - rh_subscription_org_id is defined
+    - rh_subscription_status.changed
+
+# this task has no_log set to prevent logging security sensitive information such as subscription passwords
+- name: RHEL subscription Username/Password registration
+  community.general.redhat_subscription:
+    state: present
+    username: "{{ rh_subscription_username }}"
+    password: "{{ rh_subscription_password }}"
+    auto_attach: true
+    force_register: true
+    syspurpose:
+      usage: "{{ rh_subscription_usage }}"
+      role: "{{ rh_subscription_role }}"
+      service_level_agreement: "{{ rh_subscription_sla }}"
+      sync: true
+  notify: RHEL auto-attach subscription
+  become: true
+  no_log: "{{ not (unsafe_show_logs | bool) }}"
+  when:
+    - rh_subscription_username is defined
+    - rh_subscription_status.changed
+
+# container-selinux is in appstream repo
+- name: Enable RHEL 8 repos
+  community.general.rhsm_repository:
+    name:
+      - "rhel-8-for-*-baseos-rpms"
+      - "rhel-8-for-*-appstream-rpms"
+    state: "{{ 'enabled' if (rhel_enable_repos | default(True) | bool) else 'disabled' }}"
+  when:
+    - ansible_distribution_major_version == "8"
+    - (not rh_subscription_status.changed) or (rh_subscription_username is defined) or (rh_subscription_org_id is defined)
+
+- name: Check presence of fastestmirror.conf
+  stat:
+    path: /etc/yum/pluginconf.d/fastestmirror.conf
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: fastestmirror
+
+# the fastestmirror plugin can actually slow down Ansible deployments
+- name: Disable fastestmirror plugin if requested
+  lineinfile:
+    dest: /etc/yum/pluginconf.d/fastestmirror.conf
+    regexp: "^enabled=.*"
+    line: "enabled=0"
+    state: present
+  become: true
+  when:
+    - fastestmirror.stat.exists
+    - not centos_fastestmirror_enabled

internal/resources/kubespray/project/roles/bootstrap_os/tasks/ubuntu.yml

@@ -0,0 +1,29 @@
+---
+- name: Import Debian bootstrap
+  import_tasks: debian.yml
+
+- name: Check unattended-upgrades file exist
+  stat:
+    path: /etc/apt/apt.conf.d/50unattended-upgrades
+  register: unattended_upgrades_file_stat
+  when:
+    - ubuntu_kernel_unattended_upgrades_disabled
+
+- name: Disable kernel unattended-upgrades
+  lineinfile:
+    path: "{{ unattended_upgrades_file_stat.stat.path }}"
+    insertafter: "Unattended-Upgrade::Package-Blacklist"
+    line: '"linux-";'
+    state: present
+  become: true
+  when:
+    - ubuntu_kernel_unattended_upgrades_disabled
+    - unattended_upgrades_file_stat.stat.exists
+
+- name: Stop unattended-upgrades service
+  service:
+    name: unattended-upgrades
+    state: stopped
+    enabled: false
+  become: true
+  when: ubuntu_stop_unattended_upgrades

internal/resources/kubespray/project/roles/bootstrap_os/vars/fedora-coreos.yml

@@ -0,0 +1,2 @@
+---
+is_fedora_coreos: true

internal/resources/kubespray/project/roles/bootstrap_os/vars/flatcar.yml

@@ -0,0 +1,2 @@
+---
+bin_dir: "/opt/bin"

internal/resources/kubespray/project/roles/container-engine/containerd-common/defaults/main.yml

@@ -0,0 +1,5 @@
+---
+# We keep these variables around to allow migration from package
+# manager controlled installs to direct download ones.
+containerd_package: 'containerd.io'
+yum_repo_dir: /etc/yum.repos.d

internal/resources/kubespray/project/roles/container-engine/containerd-common/meta/main.yml

@@ -0,0 +1,2 @@
+---
+allow_duplicates: true

internal/resources/kubespray/project/roles/container-engine/containerd-common/tasks/main.yml

@@ -0,0 +1,31 @@
+---
+- name: Containerd-common | check if fedora coreos
+  stat:
+    path: /run/ostree-booted
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: ostree
+
+- name: Containerd-common | set is_ostree
+  set_fact:
+    is_ostree: "{{ ostree.stat.exists }}"
+
+- name: Containerd-common | gather os specific variables
+  include_vars: "{{ item }}"
+  with_first_found:
+    - files:
+        - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower | replace('/', '_') }}.yml"
+        - "{{ ansible_distribution | lower }}-{{ ansible_distribution_release | lower }}-{{ host_architecture }}.yml"
+        - "{{ ansible_distribution | lower }}-{{ ansible_distribution_release | lower }}.yml"
+        - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower | replace('/', '_') }}.yml"
+        - "{{ ansible_distribution | lower }}-{{ host_architecture }}.yml"
+        - "{{ ansible_distribution | lower }}.yml"
+        - "{{ ansible_os_family | lower }}-{{ host_architecture }}.yml"
+        - "{{ ansible_os_family | lower }}.yml"
+        - defaults.yml
+      paths:
+        - ../vars
+      skip: true
+  tags:
+    - facts

internal/resources/kubespray/project/roles/container-engine/containerd-common/vars/amazon.yml

@@ -0,0 +1,2 @@
+---
+containerd_package: containerd

internal/resources/kubespray/project/roles/container-engine/containerd-common/vars/suse.yml

@@ -0,0 +1,2 @@
+---
+containerd_package: containerd

internal/resources/kubespray/project/roles/container-engine/containerd/defaults/main.yml

@@ -0,0 +1,128 @@
+---
+containerd_storage_dir: "/var/lib/containerd"
+containerd_state_dir: "/run/containerd"
+containerd_systemd_dir: "/etc/systemd/system/containerd.service.d"
+# The default value is not -999 here because containerd's oom_score_adj has been
+# set to the -999 even if containerd_oom_score is 0.
+# Ref: https://github.com/kubernetes-sigs/kubespray/pull/9275#issuecomment-1246499242
+containerd_oom_score: 0
+
+containerd_default_runtime: "runc"
+containerd_snapshotter: "overlayfs"
+
+containerd_runc_runtime:
+  name: runc
+  type: "io.containerd.runc.v2"
+  engine: ""
+  root: ""
+  base_runtime_spec: cri-base.json
+  options:
+    SystemdCgroup: "{{ containerd_use_systemd_cgroup | ternary('true', 'false') }}"
+    BinaryName: "{{ bin_dir }}/runc"
+
+containerd_additional_runtimes: []
+# Example for Kata Containers as additional runtime:
+#  - name: kata
+#    type: "io.containerd.kata.v2"
+#    engine: ""
+#    root: ""
+
+containerd_base_runtime_spec_rlimit_nofile: 65535
+
+containerd_default_base_runtime_spec_patch:
+  process:
+    rlimits:
+      - type: RLIMIT_NOFILE
+        hard: "{{ containerd_base_runtime_spec_rlimit_nofile }}"
+        soft: "{{ containerd_base_runtime_spec_rlimit_nofile }}"
+
+# Can help reduce disk usage
+# https://github.com/containerd/containerd/discussions/6295
+containerd_discard_unpacked_layers: true
+
+containerd_base_runtime_specs:
+  cri-base.json: "{{ containerd_default_base_runtime_spec | combine(containerd_default_base_runtime_spec_patch, recursive=1) }}"
+
+containerd_grpc_max_recv_message_size: 16777216
+containerd_grpc_max_send_message_size: 16777216
+
+containerd_debug_address: ""
+containerd_debug_level: "info"
+containerd_debug_format: ""
+containerd_debug_uid: 0
+containerd_debug_gid: 0
+
+containerd_metrics_address: ""
+
+containerd_metrics_grpc_histogram: false
+
+containerd_registries_mirrors:
+  - prefix: docker.io
+    mirrors:
+      - host: https://registry-1.docker.io
+        capabilities: ["pull", "resolve"]
+        skip_verify: false
+#        ca: ["/etc/certs/mirror.pem"]
+#        client: [["/etc/certs/client.pem", ""],["/etc/certs/client.cert", "/etc/certs/client.key"]]
+
+containerd_max_container_log_line_size: 16384
+
+# If enabled it will allow non root users to use port numbers <1024
+containerd_enable_unprivileged_ports: false
+# If enabled it will allow non root users to use icmp sockets
+containerd_enable_unprivileged_icmp: false
+
+containerd_enable_selinux: false
+containerd_disable_apparmor: false
+containerd_tolerate_missing_hugetlb_controller: true
+containerd_disable_hugetlb_controller: true
+containerd_image_pull_progress_timeout: 5m
+
+containerd_cfg_dir: /etc/containerd
+
+# Extra config to be put in {{ containerd_cfg_dir }}/config.toml literally
+containerd_extra_args: ''
+
+# Configure registry auth (if applicable to secure/insecure registries)
+containerd_registry_auth: []
+#  - registry: 10.0.0.2:5000
+#    username: user
+#    password: pass
+
+# Configure containerd service
+containerd_limit_proc_num: "infinity"
+containerd_limit_core: "infinity"
+containerd_limit_open_file_num: 1048576
+containerd_limit_mem_lock: "infinity"
+
+# OS distributions that already support containerd
+containerd_supported_distributions:
+  - "CentOS"
+  - "OracleLinux"
+  - "RedHat"
+  - "Ubuntu"
+  - "Debian"
+  - "Fedora"
+  - "AlmaLinux"
+  - "Rocky"
+  - "Amazon"
+  - "Flatcar"
+  - "Flatcar Container Linux by Kinvolk"
+  - "Suse"
+  - "openSUSE Leap"
+  - "openSUSE Tumbleweed"
+  - "Kylin Linux Advanced Server"
+  - "UnionTech"
+  - "UniontechOS"
+  - "openEuler"
+
+# Enable container device interface
+enable_cdi: false
+
+# For containerd tracing configuration please check out the official documentation:
+# https://github.com/containerd/containerd/blob/main/docs/tracing.md
+containerd_tracing_enabled: false
+containerd_tracing_endpoint: "[::]:4317"
+containerd_tracing_protocol: "grpc"
+containerd_tracing_sampling_ratio: 1.0
+containerd_tracing_service_name: "containerd"

internal/resources/kubespray/project/roles/container-engine/containerd/handlers/main.yml

@@ -0,0 +1,17 @@
+---
+- name: Containerd | restart containerd
+  systemd_service:
+    name: containerd
+    state: restarted
+    enabled: true
+    daemon-reload: true
+    masked: false
+  listen: Restart containerd
+
+- name: Containerd | wait for containerd
+  command: "{{ containerd_bin_dir }}/ctr images ls -q"
+  register: containerd_ready
+  retries: 8
+  delay: 4
+  until: containerd_ready.rc == 0
+  listen: Restart containerd

internal/resources/kubespray/project/roles/container-engine/containerd/handlers/reset.yml

@@ -0,0 +1 @@
+---

internal/resources/kubespray/project/roles/container-engine/containerd/meta/main.yml

@@ -0,0 +1,6 @@
+---
+dependencies:
+  - role: container-engine/containerd-common
+  - role: container-engine/runc
+  - role: container-engine/crictl
+  - role: container-engine/nerdctl

internal/resources/kubespray/project/roles/container-engine/containerd/molecule/default/converge.yml

@@ -0,0 +1,9 @@
+---
+- name: Converge
+  hosts: all
+  become: true
+  vars:
+    container_manager: containerd
+  roles:
+    - role: kubespray_defaults
+    - role: container-engine/containerd

internal/resources/kubespray/project/roles/container-engine/containerd/molecule/default/molecule.yml

@@ -0,0 +1,39 @@
+---
+role_name_check: 1
+platforms:
+  - cloud_image: ubuntu-2004
+    name: ubuntu20
+    vm_cpu_cores: 1
+    vm_memory: 1024
+    node_groups:
+      - kube_control_plane
+      - kube_node
+      - k8s_cluster
+  - cloud_image: debian-11
+    name: debian11
+    vm_cpu_cores: 1
+    vm_memory: 1024
+    node_groups:
+      - kube_control_plane
+      - kube_node
+      - k8s_cluster
+  - cloud_image: almalinux-9
+    name: almalinux9
+    vm_cpu_cores: 1
+    vm_memory: 1024
+    node_groups:
+      - kube_control_plane
+      - kube_node
+      - k8s_cluster
+provisioner:
+  name: ansible
+  env:
+    ANSIBLE_ROLES_PATH: ../../../../
+  config_options:
+    defaults:
+      callbacks_enabled: profile_tasks
+      timeout: 120
+  playbooks:
+    create: ../../../../../tests/cloud_playbooks/create-kubevirt.yml
+verifier:
+  name: testinfra

internal/resources/kubespray/project/roles/container-engine/containerd/molecule/default/prepare.yml

@@ -0,0 +1,30 @@
+---
+- name: Prepare
+  hosts: all
+  gather_facts: false
+  become: true
+  vars:
+    ignore_assert_errors: true
+  roles:
+    - role: kubespray_defaults
+    - role: bootstrap_os
+    - role: network_facts
+    - role: kubernetes/preinstall
+    - role: adduser
+      user: "{{ addusers.kube }}"
+  tasks:
+    - name: Download CNI
+      include_tasks: "../../../../download/tasks/download_file.yml"
+      vars:
+        download: "{{ download_defaults | combine(downloads.cni) }}"
+
+- name: Prepare CNI
+  hosts: all
+  gather_facts: false
+  become: true
+  vars:
+    ignore_assert_errors: true
+    kube_network_plugin: cni
+  roles:
+    - role: kubespray_defaults
+    - role: network_plugin/cni

internal/resources/kubespray/project/roles/container-engine/containerd/molecule/default/tests/test_default.py

@@ -0,0 +1,55 @@
+import os
+import pytest
+
+import testinfra.utils.ansible_runner
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
+
+
+def test_service(host):
+    svc = host.service("containerd")
+    assert svc.is_running
+    assert svc.is_enabled
+
+
+def test_version(host):
+    crictl = "/usr/local/bin/crictl"
+    path = "unix:///var/run/containerd/containerd.sock"
+    with host.sudo():
+        cmd = host.command(crictl + " --runtime-endpoint " + path + " version")
+    assert cmd.rc == 0
+    assert "RuntimeName:  containerd" in cmd.stdout
+
+
+@pytest.mark.parametrize('image, dest', [
+    ('quay.io/kubespray/hello-world:latest', '/tmp/hello-world.tar')
+])
+def test_image_pull_save_load(host, image, dest):
+    nerdctl = "/usr/local/bin/nerdctl"
+    dest_file = host.file(dest)
+
+    with host.sudo():
+        pull_cmd = host.command(nerdctl + " pull " + image)
+    assert pull_cmd.rc ==0
+
+    with host.sudo():
+        save_cmd = host.command(nerdctl + " save -o " + dest + " " + image)
+    assert save_cmd.rc == 0
+    assert dest_file.exists
+
+    with host.sudo():
+        load_cmd = host.command(nerdctl + " load < " + dest)
+    assert load_cmd.rc == 0
+
+
+@pytest.mark.parametrize('image', [
+    ('quay.io/kubespray/hello-world:latest')
+])
+def test_run(host, image):
+    nerdctl = "/usr/local/bin/nerdctl"
+
+    with host.sudo():
+        cmd = host.command(nerdctl + " -n k8s.io run " + image)
+    assert cmd.rc == 0
+    assert "Hello from Docker" in cmd.stdout

internal/resources/kubespray/project/roles/container-engine/containerd/tasks/main.yml

@@ -0,0 +1,100 @@
+---
+- name: Containerd | Download containerd
+  include_tasks: "../../../download/tasks/download_file.yml"
+  vars:
+    download: "{{ download_defaults | combine(downloads.containerd) }}"
+
+- name: Containerd | Unpack containerd archive
+  unarchive:
+    src: "{{ downloads.containerd.dest }}"
+    dest: "{{ containerd_bin_dir }}"
+    mode: "0755"
+    remote_src: true
+    extra_opts:
+      - --strip-components=1
+  notify: Restart containerd
+
+- name: Containerd | Generate systemd service for containerd
+  template:
+    src: containerd.service.j2
+    dest: /etc/systemd/system/containerd.service
+    mode: "0644"
+    validate: "sh -c '[ -f /usr/bin/systemd/system/factory-reset.target ] || exit 0 && systemd-analyze verify %s:containerd.service'"
+    # FIXME: check that systemd version >= 250 (factory-reset.target was introduced in that release)
+    # Remove once we drop support for systemd < 250
+  notify: Restart containerd
+
+- name: Containerd | Ensure containerd directories exist
+  file:
+    dest: "{{ item }}"
+    state: directory
+    mode: "0755"
+    owner: root
+    group: root
+  with_items:
+    - "{{ containerd_systemd_dir }}"
+    - "{{ containerd_cfg_dir }}"
+    - "{{ containerd_storage_dir }}"
+    - "{{ containerd_state_dir }}"
+
+- name: Containerd | Write containerd proxy drop-in
+  template:
+    src: http-proxy.conf.j2
+    dest: "{{ containerd_systemd_dir }}/http-proxy.conf"
+    mode: "0644"
+  notify: Restart containerd
+  when: http_proxy is defined or https_proxy is defined
+
+- name: Containerd | Generate default base_runtime_spec
+  register: ctr_oci_spec
+  command: "{{ containerd_bin_dir }}/ctr oci spec"
+  check_mode: false
+  changed_when: false
+
+- name: Containerd | Store generated default base_runtime_spec
+  set_fact:
+    containerd_default_base_runtime_spec: "{{ ctr_oci_spec.stdout | from_json }}"
+
+- name: Containerd | Write base_runtime_specs
+  copy:
+    content: "{{ item.value }}"
+    dest: "{{ containerd_cfg_dir }}/{{ item.key }}"
+    owner: "root"
+    mode: "0644"
+  with_dict: "{{ containerd_base_runtime_specs | default({}) }}"
+  notify: Restart containerd
+
+- name: Containerd | Copy containerd config file
+  template:
+    src: "{{ 'config.toml.j2' if containerd_version is version('2.0.0', '>=') else 'config-v1.toml.j2' }}"
+    dest: "{{ containerd_cfg_dir }}/config.toml"
+    owner: "root"
+    mode: "0640"
+  notify: Restart containerd
+
+- name: Containerd | Configure containerd registries
+  block:
+    - name: Containerd | Create registry directories
+      file:
+        path: "{{ containerd_cfg_dir }}/certs.d/{{ item.prefix }}"
+        state: directory
+        mode: "0755"
+      loop: "{{ containerd_registries_mirrors }}"
+    - name: Containerd | Write hosts.toml file
+      template:
+        src: hosts.toml.j2
+        dest: "{{ containerd_cfg_dir }}/certs.d/{{ item.prefix }}/hosts.toml"
+        mode: "0640"
+      loop: "{{ containerd_registries_mirrors }}"
+
+# you can sometimes end up in a state where everything is installed
+# but containerd was not started / enabled
+- name: Containerd | Flush handlers
+  meta: flush_handlers
+
+- name: Containerd | Ensure containerd is started and enabled
+  systemd_service:
+    name: containerd
+    daemon_reload: true
+    enabled: true
+    state: started

internal/resources/kubespray/project/roles/container-engine/containerd/tasks/reset.yml

@@ -0,0 +1,22 @@
+---
+- name: Containerd | Stop containerd service
+  service:
+    name: containerd
+    daemon_reload: true
+    enabled: false
+    state: stopped
+  tags:
+    - reset_containerd
+
+- name: Containerd | Remove configuration files
+  file:
+    path: "{{ item }}"
+    state: absent
+  loop:
+    - /etc/systemd/system/containerd.service
+    - "{{ containerd_systemd_dir }}"
+    - "{{ containerd_cfg_dir }}"
+    - "{{ containerd_storage_dir }}"
+    - "{{ containerd_state_dir }}"
+  tags:
+    - reset_containerd

internal/resources/kubespray/project/roles/container-engine/containerd/templates/config-v1.toml.j2

@@ -0,0 +1,102 @@
+# This is for containerd v1 for compatibility
+version = 2
+
+root = "{{ containerd_storage_dir }}"
+state = "{{ containerd_state_dir }}"
+oom_score = {{ containerd_oom_score }}
+
+{% if containerd_extra_args is defined %}
+{{ containerd_extra_args }}
+{% endif %}
+
+[grpc]
+  max_recv_message_size = {{ containerd_grpc_max_recv_message_size }}
+  max_send_message_size = {{ containerd_grpc_max_send_message_size }}
+
+[debug]
+  address = "{{ containerd_debug_address }}"
+  level = "{{ containerd_debug_level }}"
+  format = "{{ containerd_debug_format }}"
+  uid = {{ containerd_debug_uid }}
+  gid = {{ containerd_debug_gid }}
+
+[metrics]
+  address = "{{ containerd_metrics_address }}"
+  grpc_histogram = {{ containerd_metrics_grpc_histogram | lower }}
+
+[plugins]
+  [plugins."io.containerd.grpc.v1.cri"]
+    sandbox_image = "{{ pod_infra_image_repo }}:{{ pod_infra_image_tag }}"
+    max_container_log_line_size = {{ containerd_max_container_log_line_size }}
+    enable_unprivileged_ports = {{ containerd_enable_unprivileged_ports | lower }}
+    enable_unprivileged_icmp = {{ containerd_enable_unprivileged_icmp | lower }}
+    enable_selinux = {{ containerd_enable_selinux | lower }}
+    disable_apparmor = {{ containerd_disable_apparmor | lower }}
+    tolerate_missing_hugetlb_controller = {{ containerd_tolerate_missing_hugetlb_controller | lower }}
+    disable_hugetlb_controller = {{ containerd_disable_hugetlb_controller | lower }}
+    image_pull_progress_timeout = "{{ containerd_image_pull_progress_timeout }}"
+{% if enable_cdi %}
+    enable_cdi = true
+    cdi_spec_dirs = ["/etc/cdi", "/var/run/cdi"]
+{% endif %}
+    [plugins."io.containerd.grpc.v1.cri".containerd]
+      default_runtime_name = "{{ containerd_default_runtime }}"
+      snapshotter = "{{ containerd_snapshotter }}"
+      discard_unpacked_layers = {{ containerd_discard_unpacked_layers | lower }}
+      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+{% for runtime in [containerd_runc_runtime] + containerd_additional_runtimes %}
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.{{ runtime.name }}]
+          runtime_type = "{{ runtime.type }}"
+          runtime_engine = "{{ runtime.engine }}"
+          runtime_root = "{{ runtime.root }}"
+{% if runtime.base_runtime_spec is defined %}
+          base_runtime_spec = "{{ containerd_cfg_dir }}/{{ runtime.base_runtime_spec }}"
+{% endif %}
+
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.{{ runtime.name }}.options]
+{% for key, value in runtime.options.items() %}
+{% if value | string != "true" and value | string != "false" %}
+            {{ key }} = "{{ value }}"
+{% else %}
+            {{ key }} = {{ value }}
+{% endif %}
+{% endfor %}
+{% endfor %}
+{% if kata_containers_enabled %}
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-qemu]
+          runtime_type = "io.containerd.kata-qemu.v2"
+{% endif %}
+{% if gvisor_enabled %}
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runsc]
+          runtime_type = "io.containerd.runsc.v1"
+{% endif %}
+    [plugins."io.containerd.grpc.v1.cri".registry]
+      config_path = "{{ containerd_cfg_dir }}/certs.d"
+{% for registry in containerd_registry_auth if registry['registry'] is defined %}
+{% if (registry['username'] is defined and registry['password'] is defined) or registry['auth'] is defined %}
+      [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ registry['registry'] }}".auth]
+{% if registry['username'] is defined and registry['password'] is defined %}
+        password = "{{ registry['password'] }}"
+        username = "{{ registry['username'] }}"
+{% else %}
+        auth = "{{ registry['auth'] }}"
+{% endif %}
+{% endif %}
+{% endfor %}
+
+{% if nri_enabled and containerd_version is version('1.7.0', '>=') %}
+  [plugins."io.containerd.nri.v1.nri"]
+    disable = false
+{% endif %}
+
+{% if containerd_tracing_enabled %}
+  [plugins."io.containerd.tracing.processor.v1.otlp"]
+    endpoint = "{{ containerd_tracing_endpoint }}"
+    protocol = "{{ containerd_tracing_protocol }}"
+{% if containerd_tracing_protocol == "grpc" %}
+    insecure = false
+{% endif %}
+  [plugins."io.containerd.internal.v1.tracing"]
+    sampling_ratio = {{ containerd_tracing_sampling_ratio }}
+    service_name = "{{ containerd_tracing_service_name }}"
+{% endif %}

internal/resources/kubespray/project/roles/container-engine/containerd/templates/config.toml.j2

@@ -0,0 +1,92 @@
+version = 3
+
+root = "{{ containerd_storage_dir }}"
+state = "{{ containerd_state_dir }}"
+oom_score = {{ containerd_oom_score }}
+
+{% if containerd_extra_args is defined %}
+{{ containerd_extra_args }}
+{% endif %}
+
+[grpc]
+  max_recv_message_size = {{ containerd_grpc_max_recv_message_size }}
+  max_send_message_size = {{ containerd_grpc_max_send_message_size }}
+
+[debug]
+  address = "{{ containerd_debug_address }}"
+  level = "{{ containerd_debug_level }}"
+  format = "{{ containerd_debug_format }}"
+  uid = {{ containerd_debug_uid }}
+  gid = {{ containerd_debug_gid }}
+
+[metrics]
+  address = "{{ containerd_metrics_address }}"
+  grpc_histogram = {{ containerd_metrics_grpc_histogram | lower }}
+
+[plugins]
+  [plugins."io.containerd.cri.v1.runtime"]
+    max_container_log_line_size = {{ containerd_max_container_log_line_size }}
+    enable_unprivileged_ports = {{ containerd_enable_unprivileged_ports | lower }}
+    enable_unprivileged_icmp = {{ containerd_enable_unprivileged_icmp | lower }}
+    enable_selinux = {{ containerd_enable_selinux | lower }}
+    disable_apparmor = {{ containerd_disable_apparmor | lower }}
+    tolerate_missing_hugetlb_controller = {{ containerd_tolerate_missing_hugetlb_controller | lower }}
+    disable_hugetlb_controller = {{ containerd_disable_hugetlb_controller | lower }}
+{% if enable_cdi %}
+    enable_cdi = true
+    cdi_spec_dirs = ["/etc/cdi", "/var/run/cdi"]
+{% endif %}
+
+   [plugins."io.containerd.cri.v1.runtime".containerd]
+     default_runtime_name = "{{ containerd_default_runtime }}"
+     [plugins."io.containerd.cri.v1.runtime".containerd.runtimes]
+{% for runtime in [containerd_runc_runtime] + containerd_additional_runtimes %}
+       [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.{{ runtime.name }}]
+         runtime_type = "{{ runtime.type }}"
+         runtime_engine = "{{ runtime.engine }}"
+         runtime_root = "{{ runtime.root }}"
+{% if runtime.base_runtime_spec is defined %}
+         base_runtime_spec = "{{ containerd_cfg_dir }}/{{ runtime.base_runtime_spec }}"
+{% endif %}
+
+         [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.{{ runtime.name }}.options]
+{% for key, value in runtime.options.items() %}
+{% if value | string != "true" and value | string != "false" %}
+           {{ key }} = "{{ value }}"
+{% else %}
+           {{ key }} = {{ value }}
+{% endif %}
+{% endfor %}
+{% endfor %}
+{% if kata_containers_enabled %}
+       [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.kata-qemu]
+         runtime_type = "io.containerd.kata-qemu.v2"
+{% endif %}
+{% if gvisor_enabled %}
+       [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runsc]
+         runtime_type = "io.containerd.runsc.v1"
+{% endif %}
+
+  [plugins."io.containerd.cri.v1.images"]
+    snapshotter = "{{ containerd_snapshotter }}"
+    discard_unpacked_layers = {{ containerd_discard_unpacked_layers | lower }}
+    image_pull_progress_timeout = "{{ containerd_image_pull_progress_timeout }}"
+  [plugins."io.containerd.cri.v1.images".pinned_images]
+    sandbox = "{{ pod_infra_image_repo }}:{{ pod_infra_image_tag }}"
+  [plugins."io.containerd.cri.v1.images".registry]
+    config_path = "{{ containerd_cfg_dir }}/certs.d"
+
+  [plugins."io.containerd.nri.v1.nri"]
+    disable = {{ 'false' if nri_enabled else 'true' }}
+
+{% if containerd_tracing_enabled %}
+  [plugins."io.containerd.tracing.processor.v1.otlp"]
+    endpoint = "{{ containerd_tracing_endpoint }}"
+    protocol = "{{ containerd_tracing_protocol }}"
+{% if containerd_tracing_protocol == "grpc" %}
+    insecure = false
+{% endif %}
+  [plugins."io.containerd.internal.v1.tracing"]
+    sampling_ratio = {{ containerd_tracing_sampling_ratio }}
+    service_name = "{{ containerd_tracing_service_name }}"
+{% endif %}

internal/resources/kubespray/project/roles/container-engine/containerd/templates/containerd.service.j2

@@ -0,0 +1,45 @@
+# Copyright The containerd Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[Unit]
+Description=containerd container runtime
+Documentation=https://containerd.io
+After=network.target local-fs.target dbus.service
+
+[Service]
+ExecStartPre=-/sbin/modprobe overlay
+ExecStart={{ containerd_bin_dir }}/containerd
+
+Type=notify
+Delegate=yes
+KillMode=process
+Restart=always
+RestartSec=5
+# Having non-zero Limit*s causes performance problems due to accounting overhead
+# in the kernel. We recommend using cgroups to do container-local accounting.
+LimitNPROC={{ containerd_limit_proc_num }}
+LimitCORE={{ containerd_limit_core }}
+LimitNOFILE={{ containerd_limit_open_file_num }}
+LimitMEMLOCK={{ containerd_limit_mem_lock }}
+# Comment TasksMax if your systemd version does not supports it.
+# Only systemd 226 and above support this version.
+TasksMax=infinity
+OOMScoreAdjust=-999
+# Set the cgroup slice of the service so that kube reserved takes effect
+{% if kube_reserved is defined and kube_reserved|bool %}
+Slice={{ kube_reserved_cgroups_for_service_slice }}
+{% endif %}
+
+[Install]
+WantedBy=multi-user.target

internal/resources/kubespray/project/roles/container-engine/containerd/templates/hosts.toml.j2

@@ -0,0 +1,13 @@
+server = "{{ item.server | default("https://" + item.prefix) }}"
+{% for mirror in item.mirrors %}
+[host."{{ mirror.host }}"]
+  capabilities = ["{{ ([ mirror.capabilities ] | flatten ) | join('","') }}"]
+  skip_verify = {{ mirror.skip_verify | default('false') | string | lower }}
+  override_path = {{ mirror.override_path | default('false') | string | lower }}
+{% if mirror.ca is defined %}
+  ca = ["{{ ([ mirror.ca ] | flatten ) | join('","') }}"]
+{% endif %}
+{% if mirror.client is defined %}
+  client = [{% for pair in mirror.client %}["{{ pair[0] }}", "{{ pair[1] }}"]{% if not loop.last %},{% endif %}{% endfor %}]
+{% endif %}
+{% endfor %}

internal/resources/kubespray/project/roles/container-engine/containerd/templates/http-proxy.conf.j2

@@ -0,0 +1,2 @@
+[Service]
+Environment={% if http_proxy is defined %}"HTTP_PROXY={{ http_proxy }}"{% endif %} {% if https_proxy is defined %}"HTTPS_PROXY={{ https_proxy }}"{% endif %} {% if no_proxy is defined %}"NO_PROXY={{ no_proxy }}"{% endif %}

internal/resources/kubespray/project/roles/container-engine/cri-dockerd/defaults/main.yml

@@ -0,0 +1,3 @@
+---
+# Default is "info" (like if not provided). Possible values are any log level string parseable by logrus
+cri_dockerd_log_level: "info"

internal/resources/kubespray/project/roles/container-engine/cri-dockerd/handlers/main.yml

@@ -0,0 +1,31 @@
+---
+- name: Cri-dockerd | reload systemd
+  systemd_service:
+    name: cri-dockerd
+    daemon_reload: true
+    masked: false
+  listen: Restart and enable cri-dockerd
+
+- name: Cri-dockerd | restart docker.service
+  service:
+    name: docker.service
+    state: restarted
+  listen: Restart and enable cri-dockerd
+
+- name: Cri-dockerd | reload cri-dockerd.socket
+  service:
+    name: cri-dockerd.socket
+    state: restarted
+  listen: Restart and enable cri-dockerd
+
+- name: Cri-dockerd | reload cri-dockerd.service
+  service:
+    name: cri-dockerd.service
+    state: restarted
+  listen: Restart and enable cri-dockerd
+
+- name: Cri-dockerd | enable cri-dockerd service
+  service:
+    name: cri-dockerd.service
+    enabled: true
+  listen: Restart and enable cri-dockerd

internal/resources/kubespray/project/roles/container-engine/cri-dockerd/meta/main.yml

@@ -0,0 +1,4 @@
+---
+dependencies:
+  - role: container-engine/docker
+  - role: container-engine/crictl

internal/resources/kubespray/project/roles/container-engine/cri-dockerd/molecule/default/converge.yml

@@ -0,0 +1,9 @@
+---
+- name: Converge
+  hosts: all
+  become: true
+  vars:
+    container_manager: docker
+  roles:
+    - role: kubespray_defaults
+    - role: container-engine/cri-dockerd

internal/resources/kubespray/project/roles/container-engine/cri-dockerd/molecule/default/files/10-mynet.conf

@@ -0,0 +1,17 @@
+{
+  "cniVersion": "0.2.0",
+  "name": "mynet",
+  "type": "bridge",
+  "bridge": "cni0",
+  "isGateway": true,
+  "ipMasq": true,
+  "ipam": {
+    "type": "host-local",
+    "subnet": "172.19.0.0/24",
+    "routes": [
+      {
+        "dst": "0.0.0.0/0"
+      }
+    ]
+  }
+}

internal/resources/kubespray/project/roles/container-engine/cri-dockerd/molecule/default/files/container.json

@@ -0,0 +1,10 @@
+{
+  "metadata": {
+    "name": "cri-dockerd1"
+  },
+  "image": {
+    "image": "quay.io/kubespray/hello-world:latest"
+  },
+  "log_path": "cri-dockerd1.0.log",
+  "linux": {}
+}

internal/resources/kubespray/project/roles/container-engine/cri-dockerd/molecule/default/files/sandbox.json

@@ -0,0 +1,10 @@
+{
+  "metadata": {
+    "name": "cri-dockerd1",
+    "namespace": "default",
+    "attempt": 1,
+    "uid": "hdishd83djaidwnduwk28bcsb"
+  },
+  "linux": {},
+  "log_directory": "/tmp"
+}

internal/resources/kubespray/project/roles/container-engine/cri-dockerd/molecule/default/molecule.yml

@@ -0,0 +1,31 @@
+---
+role_name_check: 1
+platforms:
+  - name: almalinux9
+    cloud_image: almalinux-9
+    vm_cpu_cores: 1
+    vm_memory: 1024
+    node_groups:
+      - kube_control_plane
+  - name: ubuntu20
+    cloud_image: ubuntu-2004
+    vm_cpu_cores: 1
+    vm_memory: 1024
+    node_groups:
+      - kube_control_plane
+provisioner:
+  name: ansible
+  env:
+    ANSIBLE_ROLES_PATH: ../../../../
+  config_options:
+    defaults:
+      callbacks_enabled: profile_tasks
+      timeout: 120
+  inventory:
+    group_vars:
+      all:
+        become: true
+  playbooks:
+    create: ../../../../../tests/cloud_playbooks/create-kubevirt.yml
+verifier:
+  name: testinfra

internal/resources/kubespray/project/roles/container-engine/cri-dockerd/molecule/default/prepare.yml

@@ -0,0 +1,48 @@
+---
+- name: Prepare
+  hosts: all
+  become: true
+  roles:
+    - role: kubespray_defaults
+    - role: bootstrap_os
+    - role: adduser
+      user: "{{ addusers.kube }}"
+  tasks:
+    - name: Download CNI
+      include_tasks: "../../../../download/tasks/download_file.yml"
+      vars:
+        download: "{{ download_defaults | combine(downloads.cni) }}"
+
+- name: Prepare container runtime
+  hosts: all
+  become: true
+  vars:
+    container_manager: containerd
+    kube_network_plugin: cni
+  roles:
+    - role: kubespray_defaults
+    - role: network_plugin/cni
+  tasks:
+    - name: Copy test container files
+      copy:
+        src: "{{ item }}"
+        dest: "/tmp/{{ item }}"
+        owner: root
+        mode: "0644"
+      with_items:
+        - container.json
+        - sandbox.json
+    - name: Create /etc/cni/net.d directory
+      file:
+        path: /etc/cni/net.d
+        state: directory
+        owner: "{{ kube_owner }}"
+        mode: "0755"
+    - name: Setup CNI
+      copy:
+        src: "{{ item }}"
+        dest: "/etc/cni/net.d/{{ item }}"
+        owner: root
+        mode: "0644"
+      with_items:
+        - 10-mynet.conf

internal/resources/kubespray/project/roles/container-engine/cri-dockerd/molecule/default/tests/test_default.py

@@ -0,0 +1,19 @@
+import os
+
+import testinfra.utils.ansible_runner
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
+
+
+def test_run_pod(host):
+    run_command = "/usr/local/bin/crictl run --with-pull /tmp/container.json /tmp/sandbox.json"
+    with host.sudo():
+        cmd = host.command(run_command)
+    assert cmd.rc == 0
+
+    with host.sudo():
+      log_f = host.file("/tmp/cri-dockerd1.0.log")
+
+      assert log_f.exists
+      assert b"Hello from Docker" in log_f.content

internal/resources/kubespray/project/roles/container-engine/cri-dockerd/tasks/main.yml

@@ -0,0 +1,31 @@
+---
+- name: Runc | Download cri-dockerd binary
+  include_tasks: "../../../download/tasks/download_file.yml"
+  vars:
+    download: "{{ download_defaults | combine(downloads.cri_dockerd) }}"
+
+- name: Copy cri-dockerd binary from download dir
+  copy:
+    src: "{{ local_release_dir }}/cri-dockerd"
+    dest: "{{ bin_dir }}/cri-dockerd"
+    mode: "0755"
+    remote_src: true
+  notify:
+    - Restart and enable cri-dockerd
+
+- name: Generate cri-dockerd systemd unit files
+  template:
+    src: "{{ item }}.j2"
+    dest: "/etc/systemd/system/{{ item }}"
+    mode: "0644"
+    validate: "sh -c '[ -f /usr/bin/systemd/system/factory-reset.target ] || exit 0 && systemd-analyze verify %s:{{ item }}'"
+    # FIXME: check that systemd version >= 250 (factory-reset.target was introduced in that release)
+    # Remove once we drop support for systemd < 250
+  with_items:
+    - cri-dockerd.service
+    - cri-dockerd.socket
+  notify:
+    - Restart and enable cri-dockerd
+
+- name: Flush handlers
+  meta: flush_handlers

internal/resources/kubespray/project/roles/container-engine/cri-dockerd/templates/cri-dockerd.service.j2

@@ -0,0 +1,44 @@
+[Unit]
+Description=CRI Interface for Docker Application Container Engine
+Documentation=https://docs.mirantis.com
+After=network-online.target firewalld.service docker.service
+Wants=network-online.target docker.service
+Requires=cri-dockerd.socket
+
+[Service]
+Type=notify
+ExecStart={{ bin_dir }}/cri-dockerd --container-runtime-endpoint {{ cri_socket }} --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin --network-plugin=cni --pod-cidr={{ kube_pods_subnets }} --pod-infra-container-image={{ pod_infra_image_repo }}:{{ pod_infra_version }} --log-level {{ cri_dockerd_log_level }} {% if ipv6_stack %}--ipv6-dual-stack=True{% endif %}
+
+ExecReload=/bin/kill -s HUP $MAINPID
+TimeoutSec=0
+RestartSec=2
+Restart=always
+
+# Note that StartLimit* options were moved from "Service" to "Unit" in systemd 229.
+# Both the old, and new location are accepted by systemd 229 and up, so using the old location
+# to make them work for either version of systemd.
+StartLimitBurst=3
+
+# Note that StartLimitInterval was renamed to StartLimitIntervalSec in systemd 230.
+# Both the old, and new name are accepted by systemd 230 and up, so using the old name to make
+# this option work for either version of systemd.
+StartLimitInterval=60s
+
+# Having non-zero Limit*s causes performance problems due to accounting overhead
+# in the kernel. We recommend using cgroups to do container-local accounting.
+LimitNOFILE=infinity
+LimitNPROC=infinity
+LimitCORE=infinity
+
+# Comment TasksMax if your systemd version does not support it.
+# Only systemd 226 and above support this option.
+TasksMax=infinity
+Delegate=yes
+KillMode=process
+# Set the cgroup slice of the service so that kube reserved takes effect
+{% if kube_reserved is defined and kube_reserved|bool %}
+Slice={{ kube_reserved_cgroups_for_service_slice }}
+{% endif %}
+
+[Install]
+WantedBy=multi-user.target

internal/resources/kubespray/project/roles/container-engine/cri-dockerd/templates/cri-dockerd.socket.j2

@@ -0,0 +1,12 @@
+[Unit]
+Description=CRI Docker Socket for the API
+PartOf=cri-dockerd.service
+
+[Socket]
+ListenStream=%t/cri-dockerd.sock
+SocketMode=0660
+SocketUser=root
+SocketGroup=docker
+
+[Install]
+WantedBy=sockets.target

internal/resources/kubespray/project/roles/container-engine/cri-o/defaults/main.yml

@@ -0,0 +1,113 @@
+---
+
+crio_cgroup_manager: "{{ kubelet_cgroup_driver | default('systemd') }}"
+crio_conmon: "{{ bin_dir }}/conmon"
+crio_default_runtime: "crun"
+crio_libexec_dir: "/usr/libexec/crio"
+crio_enable_metrics: false
+crio_log_level: "info"
+crio_metrics_port: "9090"
+crio_pause_image: "{{ pod_infra_image_repo }}:{{ pod_infra_version }}"
+
+# Registries defined within cri-o.
+# By default unqualified images are not allowed for security reasons
+crio_registries: []
+#  - prefix: docker.io
+#    insecure: false
+#    blocked: false
+#    location: registry-1.docker.io ## REQUIRED
+#    unqualified: false
+#    mirrors:
+#      - location: 172.20.100.52:5000
+#        insecure: true
+#      - location: mirror.gcr.io
+#        insecure: false
+
+crio_registry_auth: []
+#  - registry: 10.0.0.2:5000
+#    username: user
+#    password: pass
+
+crio_seccomp_profile: ""
+crio_selinux: "{{ (preinstall_selinux_state == 'enforcing') | lower }}"
+crio_signature_policy: "{% if ansible_os_family == 'ClearLinux' %}/usr/share/defaults/crio/policy.json{% endif %}"
+
+# Override system default for storage driver
+# crio_storage_driver: "overlay"
+
+crio_stream_port: "10010"
+
+crio_required_version: "{{ kube_version | regex_replace('^(?P<major>\\d+).(?P<minor>\\d+).(?P<patch>\\d+)$', '\\g<major>.\\g<minor>') }}"
+
+crio_root: "/var/lib/containers/storage"
+
+# The crio_runtimes variable defines a list of OCI compatible runtimes.
+crio_runtimes:
+  - name: crun
+    path: "{{ crio_runtime_bin_dir }}/crun"
+    type: oci
+    root: /run/crun
+
+# Kata Containers is an OCI runtime, where containers are run inside lightweight
+# VMs. Kata provides additional isolation towards the host, minimizing the host attack
+# surface and mitigating the consequences of containers breakout.
+kata_runtimes:
+  # Kata Containers with the default configured VMM
+  - name: kata-qemu
+    path: /usr/local/bin/containerd-shim-kata-qemu-v2
+    type: vm
+    root: /run/kata-containers
+    privileged_without_host_devices: true
+
+runc_runtime:
+  name: runc
+  path: "{{ crio_runtime_bin_dir }}/runc"
+  type: oci
+  root: /run/runc
+
+# crun is a fast and low-memory footprint OCI Container Runtime fully written in C.
+crun_runtime:
+  name: crun
+  path: "{{ crio_runtime_bin_dir }}/crun"
+  type: oci
+  root: /run/crun
+
+# youki is an implementation of the OCI runtime-spec in Rust, similar to runc.
+youki_runtime:
+  name: youki
+  path: "{{ youki_bin_dir }}/youki"
+  type: oci
+  root: /run/youki
+
+# Reserve 16M uids and gids for user namespaces (256 pods * 65536 uids/gids)
+# at the end of the uid/gid space
+crio_remap_enable: false
+crio_remap_user: containers
+crio_subuid_start: 2130706432
+crio_subuid_length: 16777216
+crio_subgid_start: 2130706432
+crio_subgid_length: 16777216
+
+# cri-o manual files
+crio_man_files:
+  5:
+    - crio.conf
+    - crio.conf.d
+  8:
+    - crio
+    - crio-status
+
+# If set to true, it will enable the CRIU support in cri-o
+crio_criu_support_enabled: false
+
+# Configure default_capabilities in crio.conf
+crio_default_capabilities:
+  - CHOWN
+  - DAC_OVERRIDE
+  - FSETID
+  - FOWNER
+  - SETGID
+  - SETUID
+  - SETPCAP
+  - NET_BIND_SERVICE
+  - KILL

internal/resources/kubespray/project/roles/container-engine/cri-o/files/mounts.conf

@@ -0,0 +1 @@
+/usr/share/rhel/secrets:/run/secrets

internal/resources/kubespray/project/roles/container-engine/cri-o/handlers/main.yml

@@ -0,0 +1,12 @@
+---
+- name: CRI-O | reload systemd
+  systemd_service:
+    daemon_reload: true
+  listen: Restart crio
+
+- name: CRI-O | reload crio
+  service:
+    name: crio
+    state: restarted
+    enabled: true
+  listen: Restart crio

internal/resources/kubespray/project/roles/container-engine/cri-o/meta/main.yml

@@ -0,0 +1,5 @@
+---
+dependencies:
+  - role: container-engine/crun
+  - role: container-engine/crictl
+  - role: container-engine/skopeo

internal/resources/kubespray/project/roles/container-engine/cri-o/molecule/default/converge.yml

@@ -0,0 +1,9 @@
+---
+- name: Converge
+  hosts: all
+  become: true
+  vars:
+    container_manager: crio
+  roles:
+    - role: kubespray_defaults
+    - role: container-engine/cri-o

internal/resources/kubespray/project/roles/container-engine/cri-o/molecule/default/files/10-mynet.conf

@@ -0,0 +1,17 @@
+{
+  "cniVersion": "0.2.0",
+  "name": "mynet",
+  "type": "bridge",
+  "bridge": "cni0",
+  "isGateway": true,
+  "ipMasq": true,
+  "ipam": {
+    "type": "host-local",
+    "subnet": "172.19.0.0/24",
+    "routes": [
+      {
+        "dst": "0.0.0.0/0"
+      }
+    ]
+  }
+}

internal/resources/kubespray/project/roles/container-engine/cri-o/molecule/default/files/container.json

@@ -0,0 +1,10 @@
+{
+  "metadata": {
+    "name": "runc1"
+  },
+  "image": {
+    "image": "quay.io/kubespray/hello-world:latest"
+  },
+  "log_path": "runc1.0.log",
+  "linux": {}
+}