From 4c795c172ae2a50a31f04202efaf88f00376b675 Mon Sep 17 00:00:00 2001 From: George Stykalin Date: Thu, 25 Apr 2024 22:15:40 +0300 Subject: [PATCH] first version --- .gitignore | 5 + cmd/main/main.go | 45 + config.yaml | 277 + go.mod | 43 + go.sum | 119 + kubespray/env/cmdline | 1 + kubespray/env/extravars | 2 + .../credentials/kubeadm_certificate_key.creds | 1 + kubespray/inventory/group_vars/all.yml | 129 + .../group_vars/k8s_cluster/addons.yml | 3318 ++++++++++++ .../group_vars/k8s_cluster/k8s-cluster.yml | 380 ++ .../group_vars/k8s_cluster/k8s-net-calico.yml | 131 + .../group_vars/k8s_cluster/k8s-net-cilium.yml | 245 + .../k8s_cluster/k8s-net-flannel.yml | 18 + .../k8s_cluster/k8s-net-kube-ovn.yml | 63 + .../k8s_cluster/k8s-net-kube-router.yml | 64 + .../k8s_cluster/k8s-net-macvlan.yml | 6 + .../group_vars/k8s_cluster/k8s-net-weave.yml | 64 + kubespray/inventory/hosts | 23 + kubespray/project/ansible_version.yml | 34 + kubespray/project/boilerplate.yml | 58 + kubespray/project/cluster.yml | 111 + kubespray/project/facts.yml | 41 + kubespray/project/install_etcd.yml | 29 + .../library/__pycache__/kube.cpython-312.pyc | Bin 0 -> 14789 bytes kubespray/project/library/kube.py | 366 ++ kubespray/project/modules/kube.py | 366 ++ kubespray/project/recover_control_plane.yml | 28 + kubespray/project/remove_node.yml | 43 + kubespray/project/reset.yml | 35 + .../project/roles/adduser/defaults/main.yml | 27 + .../adduser/molecule/default/converge.yml | 10 + .../adduser/molecule/default/molecule.yml | 23 + .../project/roles/adduser/tasks/main.yml | 16 + .../project/roles/adduser/vars/coreos.yml | 8 + .../project/roles/adduser/vars/debian.yml | 15 + .../project/roles/adduser/vars/redhat.yml | 15 + .../bastion-ssh-config/defaults/main.yml | 2 + .../molecule/default/converge.yml | 15 + .../molecule/default/molecule.yml | 31 + .../roles/bastion-ssh-config/tasks/main.yml | 22 + .../templates/ssh-bastion.conf.j2 | 18 + .../roles/bootstrap-os/defaults/main.yml | 32 + .../roles/bootstrap-os/files/bootstrap.sh | 46 + .../roles/bootstrap-os/handlers/main.yml | 4 + .../molecule/default/converge.yml | 6 + .../molecule/default/molecule.yml | 53 + .../molecule/default/tests/test_default.py | 11 + .../bootstrap-os/tasks/bootstrap-amazon.yml | 13 + .../bootstrap-os/tasks/bootstrap-centos.yml | 118 + .../tasks/bootstrap-clearlinux.yml | 16 + .../bootstrap-os/tasks/bootstrap-coreos.yml | 37 + .../bootstrap-os/tasks/bootstrap-debian.yml | 76 + .../tasks/bootstrap-fedora-coreos.yml | 46 + .../bootstrap-os/tasks/bootstrap-fedora.yml | 36 + .../bootstrap-os/tasks/bootstrap-flatcar.yml | 37 + .../bootstrap-os/tasks/bootstrap-opensuse.yml | 85 + .../bootstrap-os/tasks/bootstrap-redhat.yml | 113 + .../project/roles/bootstrap-os/tasks/main.yml | 109 + .../containerd-common/defaults/main.yml | 17 + .../containerd-common/meta/main.yml | 2 + .../containerd-common/tasks/main.yml | 31 + .../containerd-common/vars/amazon.yml | 2 + .../containerd-common/vars/suse.yml | 2 + .../containerd/defaults/main.yml | 111 + .../containerd/handlers/main.yml | 17 + .../containerd/handlers/reset.yml | 1 + .../container-engine/containerd/meta/main.yml | 6 + .../containerd/molecule/default/converge.yml | 9 + .../containerd/molecule/default/molecule.yml | 47 + .../containerd/molecule/default/prepare.yml | 29 + .../molecule/default/tests/test_default.py | 55 + .../containerd/tasks/main.yml | 143 + .../containerd/tasks/reset.yml | 40 + .../containerd/templates/config.toml.j2 | 97 + .../templates/containerd.service.j2 | 45 + .../containerd/templates/hosts.toml.j2 | 7 + .../containerd/templates/http-proxy.conf.j2 | 2 + .../containerd/vars/debian.yml | 7 + .../containerd/vars/ubuntu.yml | 7 + .../cri-dockerd/handlers/main.yml | 31 + .../cri-dockerd/meta/main.yml | 4 + .../cri-dockerd/molecule/default/converge.yml | 9 + .../molecule/default/files/10-mynet.conf | 17 + .../molecule/default/files/container.json | 10 + .../molecule/default/files/sandbox.json | 10 + .../cri-dockerd/molecule/default/molecule.yml | 39 + .../cri-dockerd/molecule/default/prepare.yml | 48 + .../molecule/default/tests/test_default.py | 19 + .../cri-dockerd/tasks/main.yml | 31 + .../templates/cri-dockerd.service.j2 | 44 + .../templates/cri-dockerd.socket.j2 | 12 + .../container-engine/cri-o/defaults/main.yml | 91 + .../container-engine/cri-o/files/mounts.conf | 1 + .../container-engine/cri-o/handlers/main.yml | 12 + .../container-engine/cri-o/meta/main.yml | 5 + .../cri-o/molecule/default/converge.yml | 9 + .../molecule/default/files/10-mynet.conf | 17 + .../molecule/default/files/container.json | 10 + .../cri-o/molecule/default/files/sandbox.json | 10 + .../cri-o/molecule/default/molecule.yml | 57 + .../cri-o/molecule/default/prepare.yml | 53 + .../molecule/default/tests/test_default.py | 35 + .../cri-o/tasks/load_vars.yml | 8 + .../container-engine/cri-o/tasks/main.yaml | 214 + .../container-engine/cri-o/tasks/reset.yml | 90 + .../cri-o/tasks/setup-amazon.yaml | 38 + .../cri-o/templates/config.json.j2 | 17 + .../cri-o/templates/crio.conf.j2 | 389 ++ .../cri-o/templates/http-proxy.conf.j2 | 2 + .../cri-o/templates/registry.conf.j2 | 13 + .../cri-o/templates/unqualified.conf.j2 | 10 + .../container-engine/cri-o/vars/v1.28.yml | 9 + .../container-engine/cri-o/vars/v1.29.yml | 9 + .../container-engine/crictl/handlers/main.yml | 12 + .../container-engine/crictl/tasks/crictl.yml | 22 + .../container-engine/crictl/tasks/main.yml | 3 + .../crictl/templates/crictl.yaml.j2 | 4 + .../container-engine/crun/tasks/main.yml | 12 + .../docker-storage/defaults/main.yml | 19 + .../files/install_container_storage_setup.sh | 23 + .../docker-storage/tasks/main.yml | 48 + .../templates/docker-storage-setup.j2 | 35 + .../container-engine/docker/defaults/main.yml | 67 + .../docker/files/cleanup-docker-orphans.sh | 38 + .../container-engine/docker/handlers/main.yml | 30 + .../container-engine/docker/meta/main.yml | 5 + .../docker/tasks/docker_plugin.yml | 8 + .../container-engine/docker/tasks/main.yml | 181 + .../docker/tasks/pre-upgrade.yml | 36 + .../container-engine/docker/tasks/reset.yml | 106 + .../docker/tasks/set_facts_dns.yml | 66 + .../container-engine/docker/tasks/systemd.yml | 68 + .../docker/templates/docker-dns.conf.j2 | 6 + .../docker/templates/docker-options.conf.j2 | 11 + .../templates/docker-orphan-cleanup.conf.j2 | 2 + .../docker/templates/docker.service.j2 | 51 + .../docker/templates/fedora_docker.repo.j2 | 7 + .../docker/templates/http-proxy.conf.j2 | 2 + .../docker/templates/rh_docker.repo.j2 | 10 + .../container-engine/docker/vars/amazon.yml | 15 + .../docker/vars/clearlinux.yml | 4 + .../container-engine/docker/vars/debian.yml | 66 + .../container-engine/docker/vars/fedora.yml | 54 + .../container-engine/docker/vars/kylin.yml | 58 + .../container-engine/docker/vars/redhat-7.yml | 57 + .../container-engine/docker/vars/redhat.yml | 57 + .../container-engine/docker/vars/suse.yml | 6 + .../container-engine/docker/vars/ubuntu.yml | 66 + .../docker/vars/uniontech.yml | 59 + .../gvisor/molecule/default/converge.yml | 11 + .../molecule/default/files/10-mynet.conf | 17 + .../molecule/default/files/container.json | 10 + .../molecule/default/files/sandbox.json | 10 + .../gvisor/molecule/default/molecule.yml | 39 + .../gvisor/molecule/default/prepare.yml | 49 + .../molecule/default/tests/test_default.py | 29 + .../container-engine/gvisor/tasks/main.yml | 20 + .../container-engine/kata-containers/OWNERS | 6 + .../kata-containers/defaults/main.yml | 10 + .../molecule/default/converge.yml | 11 + .../molecule/default/files/10-mynet.conf | 17 + .../molecule/default/files/container.json | 10 + .../molecule/default/files/sandbox.json | 10 + .../molecule/default/molecule.yml | 39 + .../molecule/default/prepare.yml | 49 + .../molecule/default/tests/test_default.py | 37 + .../kata-containers/tasks/main.yml | 54 + .../templates/configuration-qemu.toml.j2 | 706 +++ .../templates/containerd-shim-kata-v2.j2 | 2 + .../roles/container-engine/meta/main.yml | 58 + .../nerdctl/handlers/main.yml | 12 + .../container-engine/nerdctl/tasks/main.yml | 36 + .../nerdctl/templates/nerdctl.toml.j2 | 9 + .../container-engine/runc/defaults/main.yml | 5 + .../container-engine/runc/tasks/main.yml | 38 + .../container-engine/skopeo/tasks/main.yml | 32 + .../validate-container-engine/tasks/main.yml | 153 + .../container-engine/youki/defaults/main.yml | 3 + .../youki/molecule/default/converge.yml | 11 + .../molecule/default/files/10-mynet.conf | 17 + .../molecule/default/files/container.json | 10 + .../youki/molecule/default/files/sandbox.json | 10 + .../youki/molecule/default/molecule.yml | 39 + .../youki/molecule/default/prepare.yml | 49 + .../molecule/default/tests/test_default.py | 29 + .../container-engine/youki/tasks/main.yml | 12 + .../project/roles/download/meta/main.yml | 2 + .../download/tasks/check_pull_required.yml | 25 + .../download/tasks/download_container.yml | 125 + .../roles/download/tasks/download_file.yml | 115 + .../roles/download/tasks/extract_file.yml | 11 + .../project/roles/download/tasks/main.yml | 66 + .../roles/download/tasks/prep_download.yml | 90 + .../download/tasks/prep_kubeadm_images.yml | 44 + .../download/tasks/set_container_facts.yml | 55 + .../download/templates/kubeadm-images.yaml.j2 | 24 + .../project/roles/etcd/defaults/main.yml | 127 + .../project/roles/etcd/handlers/backup.yml | 64 + .../roles/etcd/handlers/backup_cleanup.yml | 18 + .../project/roles/etcd/handlers/main.yml | 55 + kubespray/project/roles/etcd/meta/main.yml | 8 + .../project/roles/etcd/tasks/check_certs.yml | 134 + .../project/roles/etcd/tasks/configure.yml | 179 + .../roles/etcd/tasks/gen_certs_script.yml | 156 + .../etcd/tasks/gen_nodes_certs_script.yml | 33 + .../roles/etcd/tasks/install_docker.yml | 42 + .../project/roles/etcd/tasks/install_host.yml | 31 + .../etcd/tasks/join_etcd-events_member.yml | 49 + .../roles/etcd/tasks/join_etcd_member.yml | 53 + kubespray/project/roles/etcd/tasks/main.yml | 96 + .../roles/etcd/tasks/refresh_config.yml | 16 + .../project/roles/etcd/tasks/upd_ca_trust.yml | 37 + .../etcd/templates/etcd-docker.service.j2 | 18 + .../templates/etcd-events-docker.service.j2 | 18 + .../templates/etcd-events-host.service.j2 | 16 + .../roles/etcd/templates/etcd-events.env.j2 | 43 + .../roles/etcd/templates/etcd-events.j2 | 21 + .../roles/etcd/templates/etcd-host.service.j2 | 16 + .../project/roles/etcd/templates/etcd.env.j2 | 78 + .../project/roles/etcd/templates/etcd.j2 | 21 + .../roles/etcd/templates/make-ssl-etcd.sh.j2 | 105 + .../roles/etcd/templates/openssl.conf.j2 | 45 + kubespray/project/roles/etcd/vars/main.yml | 10 + .../roles/etcdctl_etcdutl/tasks/main.yml | 45 + .../etcdctl_etcdutl/templates/etcdctl.sh.j2 | 14 + kubespray/project/roles/helm-apps/README.md | 39 + .../roles/helm-apps/meta/argument_specs.yml | 95 + .../project/roles/helm-apps/meta/main.yml | 3 + .../project/roles/helm-apps/tasks/main.yml | 19 + .../project/roles/helm-apps/vars/main.yml | 12 + .../kubernetes-apps/ansible/defaults/main.yml | 124 + .../kubernetes-apps/ansible/tasks/coredns.yml | 50 + .../ansible/tasks/dashboard.yml | 21 + .../ansible/tasks/etcd_metrics.yml | 22 + .../kubernetes-apps/ansible/tasks/main.yml | 73 + .../ansible/tasks/netchecker.yml | 47 + .../ansible/tasks/nodelocaldns.yml | 79 + .../templates/coredns-clusterrole.yml.j2 | 32 + .../coredns-clusterrolebinding.yml.j2 | 18 + .../ansible/templates/coredns-config.yml.j2 | 96 + .../templates/coredns-deployment.yml.j2 | 123 + .../coredns-poddisruptionbudget.yml.j2 | 9 + .../ansible/templates/coredns-sa.yml.j2 | 9 + .../ansible/templates/coredns-svc.yml.j2 | 30 + .../ansible/templates/dashboard.yml.j2 | 335 ++ .../dns-autoscaler-clusterrole.yml.j2 | 34 + .../dns-autoscaler-clusterrolebinding.yml.j2 | 29 + .../templates/dns-autoscaler-sa.yml.j2 | 22 + .../ansible/templates/dns-autoscaler.yml.j2 | 86 + .../templates/etcd_metrics-endpoints.yml.j2 | 20 + .../templates/etcd_metrics-service.yml.j2 | 13 + .../templates/netchecker-agent-ds.yml.j2 | 56 + .../netchecker-agent-hostnet-ds.yml.j2 | 58 + .../templates/netchecker-agent-sa.yml.j2 | 5 + .../ansible/templates/netchecker-ns.yml.j2 | 6 + .../netchecker-server-clusterrole.yml.j2 | 9 + ...etchecker-server-clusterrolebinding.yml.j2 | 13 + .../netchecker-server-deployment.yml.j2 | 83 + .../templates/netchecker-server-sa.yml.j2 | 5 + .../templates/netchecker-server-svc.yml.j2 | 15 + .../templates/nodelocaldns-config.yml.j2 | 182 + .../templates/nodelocaldns-daemonset.yml.j2 | 115 + .../ansible/templates/nodelocaldns-sa.yml.j2 | 7 + .../nodelocaldns-second-daemonset.yml.j2 | 103 + .../kubernetes-apps/argocd/defaults/main.yml | 6 + .../kubernetes-apps/argocd/tasks/main.yml | 107 + .../argocd/templates/argocd-namespace.yml.j2 | 7 + .../cloud_controller/oci/defaults/main.yml | 6 + .../oci/tasks/credentials-check.yml | 67 + .../cloud_controller/oci/tasks/main.yml | 35 + .../controller-manager-config.yml.j2 | 89 + .../oci/templates/oci-cloud-provider.yml.j2 | 69 + .../files/k8s-cluster-critical-pc.yml | 8 + .../cluster_roles/files/oci-rbac.yml | 124 + .../cluster_roles/tasks/main.yml | 87 + .../cluster_roles/tasks/oci.yml | 19 + .../cluster_roles/templates/namespace.j2 | 4 + .../cluster_roles/templates/node-crb.yml.j2 | 17 + .../templates/vsphere-rbac.yml.j2 | 35 + .../meta/main.yml | 8 + .../nvidia_gpu/defaults/main.yml | 14 + .../nvidia_gpu/tasks/main.yml | 55 + .../k8s-device-plugin-nvidia-daemonset.yml.j2 | 60 + .../nvidia-driver-install-daemonset.yml.j2 | 82 + .../nvidia_gpu/vars/centos-7.yml | 3 + .../nvidia_gpu/vars/ubuntu-16.yml | 3 + .../nvidia_gpu/vars/ubuntu-18.yml | 3 + .../container_runtimes/crun/tasks/main.yaml | 19 + .../crun/templates/runtimeclass-crun.yml | 6 + .../container_runtimes/gvisor/tasks/main.yaml | 34 + .../templates/runtimeclass-gvisor.yml.j2 | 6 + .../kata_containers/defaults/main.yaml | 5 + .../kata_containers/tasks/main.yaml | 35 + .../templates/runtimeclass-kata-qemu.yml.j2 | 12 + .../container_runtimes/meta/main.yml | 31 + .../container_runtimes/youki/tasks/main.yaml | 19 + .../youki/templates/runtimeclass-youki.yml | 6 + .../roles/kubernetes-apps/csi_driver/OWNERS | 6 + .../csi_driver/aws_ebs/defaults/main.yml | 11 + .../csi_driver/aws_ebs/tasks/main.yml | 26 + .../aws-ebs-csi-controllerservice-rbac.yml.j2 | 180 + .../aws-ebs-csi-controllerservice.yml.j2 | 131 + .../templates/aws-ebs-csi-driver.yml.j2 | 8 + .../templates/aws-ebs-csi-nodeservice.yml.j2 | 101 + .../csi_driver/azuredisk/defaults/main.yml | 6 + .../tasks/azure-credential-check.yml | 54 + .../csi_driver/azuredisk/tasks/main.yml | 45 + ...azure-csi-azuredisk-controller-rbac.yml.j2 | 230 + .../azure-csi-azuredisk-controller.yml.j2 | 177 + .../azure-csi-azuredisk-driver.yml.j2 | 10 + .../azure-csi-azuredisk-node-rbac.yml.j2 | 30 + .../templates/azure-csi-azuredisk-node.yml.j2 | 168 + .../azure-csi-cloud-config-secret.yml.j2 | 7 + .../templates/azure-csi-cloud-config.j2 | 14 + .../csi_driver/cinder/defaults/main.yml | 37 + .../cinder/tasks/cinder-credential-check.yml | 59 + .../cinder/tasks/cinder-write-cacert.yml | 11 + .../csi_driver/cinder/tasks/main.yml | 57 + .../cinder-csi-cloud-config-secret.yml.j2 | 10 + .../templates/cinder-csi-cloud-config.j2 | 44 + .../cinder-csi-controllerplugin-rbac.yml.j2 | 179 + .../cinder-csi-controllerplugin.yml.j2 | 173 + .../cinder/templates/cinder-csi-driver.yml.j2 | 10 + .../cinder-csi-nodeplugin-rbac.yml.j2 | 30 + .../templates/cinder-csi-nodeplugin.yml.j2 | 148 + .../cinder-csi-poddisruptionbudget.yml.j2 | 14 + .../csi_driver/csi_crd/tasks/main.yml | 26 + .../templates/volumesnapshotclasses.yml.j2 | 116 + .../templates/volumesnapshotcontents.yml.j2 | 305 ++ .../csi_crd/templates/volumesnapshots.yml.j2 | 231 + .../csi_driver/gcp_pd/defaults/main.yml | 2 + .../csi_driver/gcp_pd/tasks/main.yml | 47 + .../templates/gcp-pd-csi-controller.yml.j2 | 165 + .../templates/gcp-pd-csi-cred-secret.yml.j2 | 8 + .../gcp_pd/templates/gcp-pd-csi-node.yml.j2 | 112 + .../templates/gcp-pd-csi-sc-regional.yml.j2 | 9 + .../templates/gcp-pd-csi-sc-zonal.yml.j2 | 8 + .../gcp_pd/templates/gcp-pd-csi-setup.yml.j2 | 241 + .../csi_driver/upcloud/defaults/main.yml | 16 + .../csi_driver/upcloud/tasks/main.yml | 40 + .../templates/upcloud-csi-controller.yml.j2 | 93 + .../templates/upcloud-csi-cred-secret.yml.j2 | 9 + .../templates/upcloud-csi-driver.yml.j2 | 8 + .../upcloud/templates/upcloud-csi-node.yml.j2 | 101 + .../templates/upcloud-csi-setup.yml.j2 | 185 + .../csi_driver/vsphere/defaults/main.yml | 54 + .../csi_driver/vsphere/tasks/main.yml | 55 + .../tasks/vsphere-credentials-check.yml | 38 + .../templates/vsphere-csi-cloud-config.j2 | 9 + .../vsphere-csi-controller-config.yml.j2 | 31 + .../vsphere-csi-controller-deployment.yml.j2 | 260 + .../vsphere-csi-controller-rbac.yml.j2 | 89 + .../vsphere-csi-controller-service.yml.j2 | 19 + .../templates/vsphere-csi-driver.yml.j2 | 7 + .../templates/vsphere-csi-namespace.yml.j2 | 4 + .../templates/vsphere-csi-node-rbac.yml.j2 | 55 + .../vsphere/templates/vsphere-csi-node.yml.j2 | 170 + .../hcloud/defaults/main.yml | 14 + .../hcloud/tasks/main.yml | 30 + ...controller-manager-ds-with-networks.yml.j2 | 93 + ...-hcloud-cloud-controller-manager-ds.yml.j2 | 92 + ...external-hcloud-cloud-role-bindings.yml.j2 | 13 + .../external-hcloud-cloud-secret.yml.j2 | 15 + ...ternal-hcloud-cloud-service-account.yml.j2 | 6 + .../huaweicloud/defaults/main.yml | 19 + .../tasks/huaweicloud-credential-check.yml | 33 + .../huaweicloud/tasks/main.yml | 49 + ...external-huawei-cloud-config-secret.yml.j2 | 10 + .../templates/external-huawei-cloud-config.j2 | 23 + ...-huawei-cloud-controller-manager-ds.yml.j2 | 91 + ...ud-controller-manager-role-bindings.yml.j2 | 16 + ...awei-cloud-controller-manager-roles.yml.j2 | 117 + .../external_cloud_controller/meta/main.yml | 42 + .../openstack/OWNERS | 6 + .../openstack/defaults/main.yml | 25 + .../openstack/tasks/main.yml | 49 + .../tasks/openstack-credential-check.yml | 66 + ...ernal-openstack-cloud-config-secret.yml.j2 | 13 + .../external-openstack-cloud-config.j2 | 92 + ...enstack-cloud-controller-manager-ds.yml.j2 | 110 + ...ud-controller-manager-role-bindings.yml.j2 | 16 + ...tack-cloud-controller-manager-roles.yml.j2 | 109 + .../vsphere/defaults/main.yml | 14 + .../vsphere/tasks/main.yml | 49 + .../tasks/vsphere-credentials-check.yml | 32 + ...vsphere-cloud-controller-manager-ds.yml.j2 | 74 + ...ud-controller-manager-role-bindings.yml.j2 | 35 + ...here-cloud-controller-manager-roles.yml.j2 | 91 + ...nal-vsphere-cpi-cloud-config-secret.yml.j2 | 11 + .../external-vsphere-cpi-cloud-config.j2 | 8 + .../cephfs_provisioner/defaults/main.yml | 10 + .../cephfs_provisioner/tasks/main.yml | 71 + .../templates/00-namespace.yml.j2 | 7 + .../clusterrole-cephfs-provisioner.yml.j2 | 22 + ...usterrolebinding-cephfs-provisioner.yml.j2 | 13 + .../deploy-cephfs-provisioner.yml.j2 | 34 + .../templates/role-cephfs-provisioner.yml.j2 | 13 + .../rolebinding-cephfs-provisioner.yml.j2 | 14 + .../templates/sa-cephfs-provisioner.yml.j2 | 6 + .../templates/sc-cephfs-provisioner.yml.j2 | 15 + .../secret-cephfs-provisioner.yml.j2 | 9 + .../local_path_provisioner/defaults/main.yml | 10 + .../local_path_provisioner/tasks/main.yml | 47 + ...cal-path-storage-clusterrolebinding.yml.j2 | 13 + .../templates/local-path-storage-cm.yml.j2 | 35 + .../templates/local-path-storage-cr.yml.j2 | 18 + .../local-path-storage-deployment.yml.j2 | 41 + .../templates/local-path-storage-ns.yml.j2 | 5 + .../templates/local-path-storage-sa.yml.j2 | 6 + .../templates/local-path-storage-sc.yml.j2 | 10 + .../defaults/main.yml | 20 + .../tasks/basedirs.yml | 12 + .../local_volume_provisioner/tasks/main.yml | 48 + ...ocal-volume-provisioner-clusterrole.yml.j2 | 22 + ...lume-provisioner-clusterrolebinding.yml.j2 | 14 + .../local-volume-provisioner-cm.yml.j2 | 33 + .../local-volume-provisioner-ds.yml.j2 | 66 + .../local-volume-provisioner-ns.yml.j2 | 7 + .../local-volume-provisioner-sa.yml.j2 | 6 + .../local-volume-provisioner-sc.yml.j2 | 12 + .../external_provisioner/meta/main.yml | 30 + .../rbd_provisioner/defaults/main.yml | 17 + .../rbd_provisioner/tasks/main.yml | 71 + .../templates/00-namespace.yml.j2 | 7 + .../clusterrole-rbd-provisioner.yml.j2 | 26 + .../clusterrolebinding-rbd-provisioner.yml.j2 | 13 + .../templates/deploy-rbd-provisioner.yml.j2 | 40 + .../templates/role-rbd-provisioner.yml.j2 | 13 + .../rolebinding-rbd-provisioner.yml.j2 | 14 + .../templates/sa-rbd-provisioner.yml.j2 | 6 + .../templates/sc-rbd-provisioner.yml.j2 | 19 + .../templates/secret-rbd-provisioner.yml.j2 | 18 + .../roles/kubernetes-apps/helm/.gitkeep | 0 .../kubernetes-apps/helm/defaults/main.yml | 2 + .../roles/kubernetes-apps/helm/tasks/main.yml | 49 + .../helm/tasks/pyyaml-flatcar.yml | 22 + .../kubernetes-apps/helm/vars/amazon.yml | 2 + .../kubernetes-apps/helm/vars/centos-7.yml | 2 + .../kubernetes-apps/helm/vars/centos.yml | 2 + .../kubernetes-apps/helm/vars/debian.yml | 2 + .../kubernetes-apps/helm/vars/fedora.yml | 2 + .../kubernetes-apps/helm/vars/redhat-7.yml | 2 + .../kubernetes-apps/helm/vars/redhat.yml | 2 + .../roles/kubernetes-apps/helm/vars/suse.yml | 2 + .../kubernetes-apps/helm/vars/ubuntu.yml | 2 + .../alb_ingress_controller/OWNERS | 6 + .../alb_ingress_controller/defaults/main.yml | 7 + .../alb_ingress_controller/tasks/main.yml | 36 + .../templates/alb-ingress-clusterrole.yml.j2 | 13 + .../alb-ingress-clusterrolebinding.yml.j2 | 14 + .../templates/alb-ingress-deploy.yml.j2 | 74 + .../templates/alb-ingress-ns.yml.j2 | 7 + .../templates/alb-ingress-sa.yml.j2 | 6 + .../cert_manager/defaults/main.yml | 19 + .../cert_manager/tasks/main.yml | 56 + .../templates/cert-manager.crds.yml.j2 | 4482 +++++++++++++++++ .../templates/cert-manager.yml.j2 | 1259 +++++ .../ingress_nginx/defaults/main.yml | 23 + .../ingress_nginx/tasks/main.yml | 68 + .../templates/00-namespace.yml.j2 | 7 + .../admission-webhook-configuration.yml.j2 | 29 + .../templates/admission-webhook-job.yml.j2 | 86 + .../clusterrole-admission-webhook.yml.j2 | 15 + .../clusterrole-ingress-nginx.yml.j2 | 36 + ...lusterrolebinding-admission-webhook.yml.j2 | 16 + .../clusterrolebinding-ingress-nginx.yml.j2 | 16 + .../templates/cm-ingress-nginx.yml.j2 | 13 + .../templates/cm-tcp-services.yml.j2 | 13 + .../templates/cm-udp-services.yml.j2 | 13 + .../ds-ingress-nginx-controller.yml.j2 | 141 + .../templates/ingressclass-nginx.yml.j2 | 13 + .../templates/role-admission-webhook.yml.j2 | 17 + .../templates/role-ingress-nginx.yml.j2 | 47 + .../rolebinding-admission-webhook.yml.j2 | 17 + .../rolebinding-ingress-nginx.yml.j2 | 17 + .../templates/sa-admission-webhook.yml.j2 | 8 + .../templates/sa-ingress-nginx.yml.j2 | 9 + .../templates/svc-ingress-nginx.yml.j2 | 24 + .../ingress_controller/meta/main.yml | 22 + .../kubernetes-apps/krew/defaults/main.yml | 5 + .../roles/kubernetes-apps/krew/tasks/krew.yml | 38 + .../roles/kubernetes-apps/krew/tasks/main.yml | 10 + .../kubernetes-apps/krew/templates/krew.j2 | 7 + .../krew/templates/krew.yml.j2 | 100 + .../kubelet-csr-approver/defaults/main.yml | 12 + .../kubelet-csr-approver/meta/main.yml | 20 + .../roles/kubernetes-apps/meta/main.yml | 134 + .../roles/kubernetes-apps/metallb/OWNERS | 5 + .../kubernetes-apps/metallb/defaults/main.yml | 16 + .../kubernetes-apps/metallb/tasks/main.yml | 108 + .../metallb/templates/layer2.yaml.j2 | 19 + .../metallb/templates/layer3.yaml.j2 | 125 + .../metallb/templates/metallb.yaml.j2 | 2025 ++++++++ .../metallb/templates/pools.yaml.j2 | 22 + .../metrics_server/defaults/main.yml | 14 + .../metrics_server/tasks/main.yml | 57 + .../templates/auth-delegator.yaml.j2 | 14 + .../templates/auth-reader.yaml.j2 | 15 + .../templates/metrics-apiservice.yaml.j2 | 15 + .../metrics-server-deployment.yaml.j2 | 118 + .../templates/metrics-server-sa.yaml.j2 | 8 + .../templates/metrics-server-service.yaml.j2 | 17 + ...resource-reader-clusterrolebinding.yaml.j2 | 15 + .../templates/resource-reader.yaml.j2 | 22 + .../network_plugin/calico/tasks/main.yml | 2 + .../network_plugin/flannel/tasks/main.yml | 17 + .../network_plugin/kube-ovn/tasks/main.yml | 9 + .../network_plugin/kube-router/OWNERS | 6 + .../network_plugin/kube-router/tasks/main.yml | 23 + .../network_plugin/meta/main.yml | 31 + .../network_plugin/multus/tasks/main.yml | 18 + .../network_plugin/weave/tasks/main.yml | 21 + .../persistent_volumes/aws-ebs-csi/OWNERS | 5 + .../aws-ebs-csi/defaults/main.yml | 8 + .../aws-ebs-csi/tasks/main.yml | 20 + .../aws-ebs-csi-storage-class.yml.j2 | 18 + .../azuredisk-csi/defaults/main.yml | 3 + .../azuredisk-csi/tasks/main.yml | 20 + .../templates/azure-csi-storage-class.yml.j2 | 14 + .../cinder-csi/defaults/main.yml | 7 + .../cinder-csi/tasks/main.yml | 20 + .../templates/cinder-csi-storage-class.yml.j2 | 25 + .../gcp-pd-csi/defaults/main.yml | 8 + .../gcp-pd-csi/tasks/main.yml | 20 + .../templates/gcp-pd-csi-storage-class.yml.j2 | 20 + .../persistent_volumes/meta/main.yml | 43 + .../openstack/defaults/main.yml | 7 + .../openstack/tasks/main.yml | 20 + .../templates/openstack-storage-class.yml.j2 | 27 + .../upcloud-csi/defaults/main.yml | 12 + .../upcloud-csi/tasks/main.yml | 20 + .../upcloud-csi-storage-class.yml.j2 | 16 + .../calico/defaults/main.yml | 10 + .../policy_controller/calico/tasks/main.yml | 34 + .../templates/calico-kube-controllers.yml.j2 | 85 + .../calico/templates/calico-kube-cr.yml.j2 | 110 + .../calico/templates/calico-kube-crb.yml.j2 | 13 + .../calico/templates/calico-kube-sa.yml.j2 | 6 + .../policy_controller/meta/main.yml | 8 + .../registry/defaults/main.yml | 48 + .../kubernetes-apps/registry/tasks/main.yml | 98 + .../registry/templates/registry-cm.yml.j2 | 10 + .../registry/templates/registry-ing.yml.j2 | 27 + .../registry/templates/registry-ns.yml.j2 | 7 + .../registry/templates/registry-pvc.yml.j2 | 15 + .../registry/templates/registry-rs.yml.j2 | 115 + .../registry/templates/registry-sa.yml.j2 | 5 + .../templates/registry-secrets.yml.j2 | 10 + .../registry/templates/registry-svc.yml.j2 | 32 + .../scheduler_plugins/defaults/main.yml | 29 + .../scheduler_plugins/tasks/main.yml | 68 + ...appgroup.diktyo.x-k8s.io_appgroups.yaml.j2 | 197 + .../templates/cm-scheduler-plugins.yaml.j2 | 28 + .../deploy-scheduler-plugins.yaml.j2 | 74 + .../templates/namespace.yaml.j2 | 7 + ....diktyo.x-k8s.io_networktopologies.yaml.j2 | 148 + .../templates/rbac-scheduler-plugins.yaml.j2 | 140 + .../templates/sa-scheduler-plugins.yaml.j2 | 11 + .../scheduling.x-k8s.io_elasticquotas.yaml.j2 | 82 + .../scheduling.x-k8s.io_podgroups.yaml.j2 | 97 + ...node.k8s.io_noderesourcetopologies.yaml.j2 | 153 + .../snapshots/cinder-csi/defaults/main.yml | 6 + .../snapshots/cinder-csi/tasks/main.yml | 18 + .../cinder-csi-snapshot-class.yml.j2 | 13 + .../kubernetes-apps/snapshots/meta/main.yml | 14 + .../snapshot-controller/defaults/main.yml | 3 + .../snapshot-controller/tasks/main.yml | 39 + .../templates/rbac-snapshot-controller.yml.j2 | 85 + .../templates/snapshot-controller.yml.j2 | 40 + .../templates/snapshot-ns.yml.j2 | 7 + .../roles/kubernetes/client/defaults/main.yml | 8 + .../roles/kubernetes/client/tasks/main.yml | 114 + .../control-plane/defaults/main/etcd.yml | 31 + .../defaults/main/kube-proxy.yml | 114 + .../defaults/main/kube-scheduler.yml | 33 + .../control-plane/defaults/main/main.yml | 242 + .../control-plane/handlers/main.yml | 117 + .../kubernetes/control-plane/meta/main.yml | 12 + .../tasks/define-first-kube-control.yml | 19 + .../control-plane/tasks/encrypt-at-rest.yml | 40 + .../control-plane/tasks/kubeadm-backup.yml | 28 + .../control-plane/tasks/kubeadm-etcd.yml | 29 + .../tasks/kubeadm-fix-apiserver.yml | 17 + .../control-plane/tasks/kubeadm-secondary.yml | 80 + .../control-plane/tasks/kubeadm-setup.yml | 265 + .../control-plane/tasks/kubeadm-upgrade.yml | 78 + .../kubelet-fix-client-cert-rotation.yml | 18 + .../kubernetes/control-plane/tasks/main.yml | 126 + .../control-plane/tasks/pre-upgrade.yml | 21 + .../templates/admission-controls.yaml.j2 | 9 + .../templates/apiserver-audit-policy.yaml.j2 | 129 + .../apiserver-audit-webhook-config.yaml.j2 | 17 + .../templates/apiserver-tracing.yaml.j2 | 4 + .../templates/eventratelimit.yaml.j2 | 11 + .../templates/k8s-certs-renew.service.j2 | 6 + .../templates/k8s-certs-renew.sh.j2 | 23 + .../templates/k8s-certs-renew.timer.j2 | 11 + .../templates/kubeadm-config.v1beta3.yaml.j2 | 471 ++ .../kubeadm-controlplane.v1beta3.yaml.j2 | 32 + .../templates/kubescheduler-config.yaml.j2 | 24 + .../templates/podnodeselector.yaml.j2 | 2 + .../templates/podsecurity.yaml.j2 | 17 + .../templates/secrets_encryption.yaml.j2 | 11 + .../webhook-authorization-config.yaml.j2 | 18 + .../webhook-token-auth-config.yaml.j2 | 21 + .../kubernetes/control-plane/vars/main.yaml | 3 + .../kubernetes/kubeadm/defaults/main.yml | 12 + .../kubernetes/kubeadm/handlers/main.yml | 11 + .../kubeadm/tasks/kubeadm_etcd_node.yml | 62 + .../roles/kubernetes/kubeadm/tasks/main.yml | 204 + .../templates/kubeadm-client.conf.v1beta3.j2 | 39 + .../kubernetes/node-label/tasks/main.yml | 49 + .../kubernetes/node-taint/tasks/main.yml | 35 + .../roles/kubernetes/node/defaults/main.yml | 261 + .../roles/kubernetes/node/handlers/main.yml | 11 + .../azure-credential-check.yml | 82 + .../openstack-credential-check.yml | 34 + .../vsphere-credential-check.yml | 22 + .../roles/kubernetes/node/tasks/facts.yml | 62 + .../roles/kubernetes/node/tasks/install.yml | 22 + .../roles/kubernetes/node/tasks/kubelet.yml | 55 + .../node/tasks/loadbalancer/haproxy.yml | 34 + .../node/tasks/loadbalancer/kube-vip.yml | 13 + .../node/tasks/loadbalancer/nginx-proxy.yml | 34 + .../roles/kubernetes/node/tasks/main.yml | 191 + .../kubernetes/node/tasks/pre_upgrade.yml | 48 + .../cloud-configs/aws-cloud-config.j2 | 11 + .../cloud-configs/azure-cloud-config.j2 | 26 + .../cloud-configs/gce-cloud-config.j2 | 2 + .../cloud-configs/openstack-cloud-config.j2 | 54 + .../cloud-configs/vsphere-cloud-config.j2 | 36 + .../node/templates/http-proxy.conf.j2 | 2 + .../templates/kubelet-config.v1beta1.yaml.j2 | 173 + .../node/templates/kubelet.env.v1beta1.j2 | 33 + .../node/templates/kubelet.service.j2 | 52 + .../templates/loadbalancer/haproxy.cfg.j2 | 49 + .../node/templates/loadbalancer/nginx.conf.j2 | 60 + .../templates/manifests/haproxy.manifest.j2 | 42 + .../templates/manifests/kube-vip.manifest.j2 | 106 + .../manifests/nginx-proxy.manifest.j2 | 42 + .../node/templates/node-kubeconfig.yaml.j2 | 19 + .../roles/kubernetes/node/vars/fedora.yml | 2 + .../roles/kubernetes/node/vars/ubuntu-18.yml | 2 + .../roles/kubernetes/node/vars/ubuntu-20.yml | 2 + .../roles/kubernetes/node/vars/ubuntu-22.yml | 2 + .../kubernetes/preinstall/defaults/main.yml | 155 + .../preinstall/files/dhclient_nodnsupdate | 4 + .../kubernetes/preinstall/gen-gitinfos.sh | 73 + .../kubernetes/preinstall/handlers/main.yml | 134 + .../roles/kubernetes/preinstall/meta/main.yml | 8 + .../preinstall/tasks/0010-swapoff.yml | 26 + .../preinstall/tasks/0020-set_facts.yml | 271 + .../preinstall/tasks/0040-verify-settings.yml | 318 ++ .../tasks/0050-create_directories.yml | 119 + .../preinstall/tasks/0060-resolvconf.yml | 58 + .../tasks/0061-systemd-resolved.yml | 15 + .../0062-networkmanager-unmanaged-devices.yml | 28 + .../tasks/0063-networkmanager-dns.yml | 37 + .../preinstall/tasks/0070-system-packages.yml | 99 + .../tasks/0080-system-configurations.yml | 145 + .../tasks/0081-ntp-configurations.yml | 87 + .../preinstall/tasks/0090-etchosts.yml | 81 + .../preinstall/tasks/0100-dhclient-hooks.yml | 33 + .../tasks/0110-dhclient-hooks-undo.yml | 18 + .../tasks/0120-growpart-azure-centos-7.yml | 44 + .../kubernetes/preinstall/tasks/main.yml | 148 + .../preinstall/templates/ansible_git.j2 | 3 + .../preinstall/templates/chrony.conf.j2 | 27 + .../templates/dhclient_dnsupdate.sh.j2 | 13 + .../templates/dhclient_dnsupdate_rh.sh.j2 | 17 + .../preinstall/templates/ntp.conf.j2 | 45 + .../preinstall/templates/resolvconf.j2 | 10 + .../preinstall/templates/resolved.conf.j2 | 15 + .../kubernetes/preinstall/vars/amazon.yml | 7 + .../kubernetes/preinstall/vars/centos.yml | 8 + .../kubernetes/preinstall/vars/debian-11.yml | 10 + .../kubernetes/preinstall/vars/debian-12.yml | 11 + .../kubernetes/preinstall/vars/debian.yml | 9 + .../kubernetes/preinstall/vars/fedora.yml | 8 + .../kubernetes/preinstall/vars/redhat.yml | 8 + .../roles/kubernetes/preinstall/vars/suse.yml | 5 + .../kubernetes/preinstall/vars/ubuntu.yml | 8 + .../kubernetes/tokens/files/kube-gen-token.sh | 34 + .../kubernetes/tokens/tasks/check-tokens.yml | 41 + .../kubernetes/tokens/tasks/gen_tokens.yml | 63 + .../roles/kubernetes/tokens/tasks/main.yml | 21 + .../defaults/main/checksums.yml | 1486 ++++++ .../defaults/main/download.yml | 1191 +++++ .../kubespray-defaults/defaults/main/main.yml | 686 +++ .../kubespray-defaults/tasks/fallback_ips.yml | 34 + .../roles/kubespray-defaults/tasks/main.yaml | 28 + .../kubespray-defaults/tasks/no_proxy.yml | 40 + .../roles/kubespray-defaults/vars/main.yml | 9 + .../network_plugin/calico/files/openssl.conf | 27 + .../network_plugin/calico/handlers/main.yml | 31 + .../roles/network_plugin/calico/meta/main.yml | 3 + .../calico/rr/defaults/main.yml | 5 + .../network_plugin/calico/rr/tasks/main.yml | 16 + .../network_plugin/calico/rr/tasks/pre.yml | 15 + .../calico/rr/tasks/update-node.yml | 50 + .../calico/tasks/calico_apiserver_certs.yml | 60 + .../network_plugin/calico/tasks/check.yml | 207 + .../network_plugin/calico/tasks/install.yml | 499 ++ .../network_plugin/calico/tasks/main.yml | 9 + .../calico/tasks/peer_with_calico_rr.yml | 86 + .../calico/tasks/peer_with_router.yml | 77 + .../roles/network_plugin/calico/tasks/pre.yml | 47 + .../network_plugin/calico/tasks/repos.yml | 21 + .../network_plugin/calico/tasks/reset.yml | 30 + .../calico/tasks/typha_certs.yml | 51 + .../templates/calico-apiserver-ns.yml.j2 | 10 + .../calico/templates/calico-apiserver.yml.j2 | 280 + .../calico/templates/calico-config.yml.j2 | 105 + .../calico/templates/calico-cr.yml.j2 | 194 + .../calico/templates/calico-crb.yml.j2 | 28 + .../calico/templates/calico-ipamconfig.yml.j2 | 8 + .../calico/templates/calico-node-sa.yml.j2 | 13 + .../calico/templates/calico-node.yml.j2 | 475 ++ .../calico/templates/calico-typha.yml.j2 | 187 + .../calico/templates/calicoctl.etcd.sh.j2 | 6 + .../calico/templates/calicoctl.kdd.sh.j2 | 8 + .../kubernetes-services-endpoint.yml.j2 | 11 + .../calico/templates/make-ssl-calico.sh.j2 | 102 + .../network_plugin/calico/vars/amazon.yml | 5 + .../network_plugin/calico/vars/centos-9.yml | 3 + .../network_plugin/calico/vars/debian.yml | 3 + .../network_plugin/calico/vars/fedora.yml | 3 + .../network_plugin/calico/vars/opensuse.yml | 3 + .../network_plugin/calico/vars/redhat-9.yml | 3 + .../network_plugin/calico/vars/redhat.yml | 4 + .../network_plugin/calico/vars/rocky-9.yml | 3 + .../calico_defaults/defaults/main.yml | 176 + .../network_plugin/cilium/defaults/main.yml | 311 ++ .../network_plugin/cilium/tasks/apply.yml | 33 + .../network_plugin/cilium/tasks/check.yml | 63 + .../network_plugin/cilium/tasks/install.yml | 97 + .../network_plugin/cilium/tasks/main.yml | 9 + .../network_plugin/cilium/tasks/reset.yml | 9 + .../cilium/tasks/reset_iface.yml | 12 + .../templates/000-cilium-portmap.conflist.j2 | 13 + .../templates/cilium-operator/cr.yml.j2 | 169 + .../templates/cilium-operator/crb.yml.j2 | 13 + .../templates/cilium-operator/deploy.yml.j2 | 170 + .../templates/cilium-operator/sa.yml.j2 | 6 + .../cilium/templates/cilium/config.yml.j2 | 263 + .../cilium/templates/cilium/cr.yml.j2 | 131 + .../cilium/templates/cilium/crb.yml.j2 | 13 + .../cilium/templates/cilium/ds.yml.j2 | 446 ++ .../cilium/templates/cilium/sa.yml.j2 | 6 + .../cilium/templates/cilium/secret.yml.j2 | 9 + .../cilium/templates/hubble/config.yml.j2 | 71 + .../cilium/templates/hubble/cr.yml.j2 | 106 + .../cilium/templates/hubble/crb.yml.j2 | 44 + .../cilium/templates/hubble/cronjob.yml.j2 | 38 + .../cilium/templates/hubble/deploy.yml.j2 | 197 + .../cilium/templates/hubble/job.yml.j2 | 34 + .../cilium/templates/hubble/sa.yml.j2 | 23 + .../cilium/templates/hubble/service.yml.j2 | 105 + .../network_plugin/cni/defaults/main.yml | 2 + .../roles/network_plugin/cni/tasks/main.yml | 16 + .../custom_cni/defaults/main.yml | 11 + .../network_plugin/custom_cni/meta/main.yml | 20 + .../network_plugin/custom_cni/tasks/main.yml | 29 + .../network_plugin/flannel/defaults/main.yml | 28 + .../network_plugin/flannel/meta/main.yml | 3 + .../network_plugin/flannel/tasks/main.yml | 21 + .../network_plugin/flannel/tasks/reset.yml | 24 + .../flannel/templates/cni-flannel-rbac.yml.j2 | 52 + .../flannel/templates/cni-flannel.yml.j2 | 170 + .../roles/network_plugin/kube-ovn/OWNERS | 4 + .../network_plugin/kube-ovn/defaults/main.yml | 118 + .../network_plugin/kube-ovn/tasks/main.yml | 17 + .../templates/cni-kube-ovn-crd.yml.j2 | 1533 ++++++ .../kube-ovn/templates/cni-kube-ovn.yml.j2 | 673 +++ .../kube-ovn/templates/cni-ovn.yml.j2 | 517 ++ .../roles/network_plugin/kube-router/OWNERS | 6 + .../kube-router/defaults/main.yml | 69 + .../kube-router/handlers/main.yml | 20 + .../network_plugin/kube-router/meta/main.yml | 3 + .../kube-router/tasks/annotate.yml | 21 + .../network_plugin/kube-router/tasks/main.yml | 62 + .../kube-router/tasks/reset.yml | 28 + .../kube-router/templates/cni-conf.json.j2 | 27 + .../kube-router/templates/kube-router.yml.j2 | 228 + .../kube-router/templates/kubeconfig.yml.j2 | 18 + .../roles/network_plugin/macvlan/OWNERS | 6 + .../network_plugin/macvlan/defaults/main.yml | 6 + .../network_plugin/macvlan/files/ifdown-local | 6 + .../macvlan/files/ifdown-macvlan | 40 + .../network_plugin/macvlan/files/ifup-local | 6 + .../network_plugin/macvlan/files/ifup-macvlan | 43 + .../network_plugin/macvlan/handlers/main.yml | 15 + .../network_plugin/macvlan/meta/main.yml | 3 + .../network_plugin/macvlan/tasks/main.yml | 110 + .../macvlan/templates/10-macvlan.conf.j2 | 15 + .../macvlan/templates/99-loopback.conf.j2 | 5 + .../templates/centos-network-macvlan.cfg.j2 | 13 + .../templates/centos-postdown-macvlan.cfg.j2 | 3 + .../templates/centos-postup-macvlan.cfg.j2 | 3 + .../templates/centos-routes-macvlan.cfg.j2 | 7 + .../templates/coreos-device-macvlan.cfg.j2 | 6 + .../templates/coreos-interface-macvlan.cfg.j2 | 6 + .../templates/coreos-network-macvlan.cfg.j2 | 17 + .../templates/coreos-service-nat_ouside.j2 | 6 + .../templates/debian-network-macvlan.cfg.j2 | 26 + .../roles/network_plugin/meta/main.yml | 48 + .../network_plugin/multus/defaults/main.yml | 10 + .../multus/files/multus-clusterrole.yml | 28 + .../files/multus-clusterrolebinding.yml | 13 + .../multus/files/multus-crd.yml | 45 + .../multus/files/multus-serviceaccount.yml | 6 + .../roles/network_plugin/multus/meta/main.yml | 3 + .../network_plugin/multus/tasks/main.yml | 36 + .../multus/templates/multus-daemonset.yml.j2 | 79 + .../network_plugin/ovn4nfv/tasks/main.yml | 16 + .../network_plugin/weave/defaults/main.yml | 64 + .../roles/network_plugin/weave/meta/main.yml | 3 + .../roles/network_plugin/weave/tasks/main.yml | 12 + .../weave/templates/10-weave.conflist.j2 | 16 + .../weave/templates/weave-net.yml.j2 | 297 ++ .../roles/recover_control_plane/OWNERS | 8 + .../control-plane/defaults/main.yml | 2 + .../control-plane/tasks/main.yml | 29 + .../recover_control_plane/etcd/tasks/main.yml | 94 + .../etcd/tasks/recover_lost_quorum.yml | 59 + .../post-recover/tasks/main.yml | 20 + .../remove-node/post-remove/defaults/main.yml | 3 + .../remove-node/post-remove/tasks/main.yml | 13 + .../remove-node/pre-remove/defaults/main.yml | 6 + .../remove-node/pre-remove/tasks/main.yml | 43 + .../remove-etcd-node/tasks/main.yml | 58 + .../project/roles/reset/defaults/main.yml | 21 + kubespray/project/roles/reset/tasks/main.yml | 440 ++ .../upgrade/post-upgrade/defaults/main.yml | 5 + .../roles/upgrade/post-upgrade/tasks/main.yml | 32 + .../upgrade/pre-upgrade/defaults/main.yml | 20 + .../roles/upgrade/pre-upgrade/tasks/main.yml | 98 + .../upgrade/system-upgrade/tasks/apt.yml | 13 + .../upgrade/system-upgrade/tasks/main.yml | 17 + .../upgrade/system-upgrade/tasks/yum.yml | 12 + .../kubernetes_patch/defaults/main.yml | 4 + .../win_nodes/kubernetes_patch/tasks/main.yml | 41 + kubespray/project/scale.yml | 104 + kubespray/project/upgrade_cluster.yml | 130 + pkg/config/ansible_playbook_config.go | 12 + pkg/config/config.go | 66 + pkg/config/modules_additional.go | 37 + pkg/config/modules_cicd.go | 29 + pkg/config/modules_observability.go | 154 + pkg/config/modules_registry.go | 26 + pkg/config/modules_secrets_storage.go | 42 + pkg/config/orchestrator.go | 31 + pkg/config/utility.go | 18 + pkg/kubespray/cluster.go | 16 + pkg/kubespray/utility.go | 91 + pkg/resources/resources.go | 6 + .../additional-modules/cert-manager.yml.tmpl | 0 .../additional-modules/ingress-nginx.yml.tmpl | 0 .../additional-modules/longhorn.yml.tmpl | 487 ++ .../secrets-store-csi-driver.yml.tmpl | 6 + .../releases/cicd/argo-cd-ingress.yml.tmpl | 34 + .../helm-apps/releases/cicd/argo-cd.yml.tmpl | 144 + .../releases/cicd/argo-rollouts.yml.tmpl | 421 ++ .../helm-apps/releases/cicd/keel.yml.tmpl | 9 + .../observability/fluent-operator.yml.tmpl | 66 + .../releases/observability/loki.yml.tmpl | 200 + .../observability/observability.yml.tmpl | 315 ++ .../opentelemetry-operator.yml.tmpl | 145 + .../releases/observability/tempo.yml.tmpl | 79 + .../harbor-certificate-generator.yml.tmpl | 17 + .../releases/registry/harbor.yml.tmpl | 382 ++ .../releases/secrets-storage/vault.yml.tmpl | 883 ++++ .../repositories/repositories.yml.tmpl | 5 + .../inventory/group_vars/all.yml.tmpl | 136 + .../group_vars/k8s_cluster/addons.yml.tmpl | 247 + .../k8s_cluster/k8s-cluster.yml.tmpl | 384 ++ .../templates/kubespray/inventory/hosts.tmpl | 61 + pkg/templates/funcs.go | 105 + pkg/templates/templates.go | 101 + 880 files changed, 62224 insertions(+) create mode 100644 .gitignore create mode 100644 cmd/main/main.go create mode 100644 config.yaml create mode 100644 go.mod create mode 100644 go.sum create mode 100644 kubespray/env/cmdline create mode 100644 kubespray/env/extravars create mode 100644 kubespray/inventory/credentials/kubeadm_certificate_key.creds create mode 100644 kubespray/inventory/group_vars/all.yml create mode 100644 kubespray/inventory/group_vars/k8s_cluster/addons.yml create mode 100644 kubespray/inventory/group_vars/k8s_cluster/k8s-cluster.yml create mode 100644 kubespray/inventory/group_vars/k8s_cluster/k8s-net-calico.yml create mode 100644 kubespray/inventory/group_vars/k8s_cluster/k8s-net-cilium.yml create mode 100644 kubespray/inventory/group_vars/k8s_cluster/k8s-net-flannel.yml create mode 100644 kubespray/inventory/group_vars/k8s_cluster/k8s-net-kube-ovn.yml create mode 100644 kubespray/inventory/group_vars/k8s_cluster/k8s-net-kube-router.yml create mode 100644 kubespray/inventory/group_vars/k8s_cluster/k8s-net-macvlan.yml create mode 100644 kubespray/inventory/group_vars/k8s_cluster/k8s-net-weave.yml create mode 100644 kubespray/inventory/hosts create mode 100644 kubespray/project/ansible_version.yml create mode 100644 kubespray/project/boilerplate.yml create mode 100644 kubespray/project/cluster.yml create mode 100644 kubespray/project/facts.yml create mode 100644 kubespray/project/install_etcd.yml create mode 100644 kubespray/project/library/__pycache__/kube.cpython-312.pyc create mode 100644 kubespray/project/library/kube.py create mode 100644 kubespray/project/modules/kube.py create mode 100644 kubespray/project/recover_control_plane.yml create mode 100644 kubespray/project/remove_node.yml create mode 100644 kubespray/project/reset.yml create mode 100644 kubespray/project/roles/adduser/defaults/main.yml create mode 100644 kubespray/project/roles/adduser/molecule/default/converge.yml create mode 100644 kubespray/project/roles/adduser/molecule/default/molecule.yml create mode 100644 kubespray/project/roles/adduser/tasks/main.yml create mode 100644 kubespray/project/roles/adduser/vars/coreos.yml create mode 100644 kubespray/project/roles/adduser/vars/debian.yml create mode 100644 kubespray/project/roles/adduser/vars/redhat.yml create mode 100644 kubespray/project/roles/bastion-ssh-config/defaults/main.yml create mode 100644 kubespray/project/roles/bastion-ssh-config/molecule/default/converge.yml create mode 100644 kubespray/project/roles/bastion-ssh-config/molecule/default/molecule.yml create mode 100644 kubespray/project/roles/bastion-ssh-config/tasks/main.yml create mode 100644 kubespray/project/roles/bastion-ssh-config/templates/ssh-bastion.conf.j2 create mode 100644 kubespray/project/roles/bootstrap-os/defaults/main.yml create mode 100755 kubespray/project/roles/bootstrap-os/files/bootstrap.sh create mode 100644 kubespray/project/roles/bootstrap-os/handlers/main.yml create mode 100644 kubespray/project/roles/bootstrap-os/molecule/default/converge.yml create mode 100644 kubespray/project/roles/bootstrap-os/molecule/default/molecule.yml create mode 100644 kubespray/project/roles/bootstrap-os/molecule/default/tests/test_default.py create mode 100644 kubespray/project/roles/bootstrap-os/tasks/bootstrap-amazon.yml create mode 100644 kubespray/project/roles/bootstrap-os/tasks/bootstrap-centos.yml create mode 100644 kubespray/project/roles/bootstrap-os/tasks/bootstrap-clearlinux.yml create mode 100644 kubespray/project/roles/bootstrap-os/tasks/bootstrap-coreos.yml create mode 100644 kubespray/project/roles/bootstrap-os/tasks/bootstrap-debian.yml create mode 100644 kubespray/project/roles/bootstrap-os/tasks/bootstrap-fedora-coreos.yml create mode 100644 kubespray/project/roles/bootstrap-os/tasks/bootstrap-fedora.yml create mode 100644 kubespray/project/roles/bootstrap-os/tasks/bootstrap-flatcar.yml create mode 100644 kubespray/project/roles/bootstrap-os/tasks/bootstrap-opensuse.yml create mode 100644 kubespray/project/roles/bootstrap-os/tasks/bootstrap-redhat.yml create mode 100644 kubespray/project/roles/bootstrap-os/tasks/main.yml create mode 100644 kubespray/project/roles/container-engine/containerd-common/defaults/main.yml create mode 100644 kubespray/project/roles/container-engine/containerd-common/meta/main.yml create mode 100644 kubespray/project/roles/container-engine/containerd-common/tasks/main.yml create mode 100644 kubespray/project/roles/container-engine/containerd-common/vars/amazon.yml create mode 100644 kubespray/project/roles/container-engine/containerd-common/vars/suse.yml create mode 100644 kubespray/project/roles/container-engine/containerd/defaults/main.yml create mode 100644 kubespray/project/roles/container-engine/containerd/handlers/main.yml create mode 100644 kubespray/project/roles/container-engine/containerd/handlers/reset.yml create mode 100644 kubespray/project/roles/container-engine/containerd/meta/main.yml create mode 100644 kubespray/project/roles/container-engine/containerd/molecule/default/converge.yml create mode 100644 kubespray/project/roles/container-engine/containerd/molecule/default/molecule.yml create mode 100644 kubespray/project/roles/container-engine/containerd/molecule/default/prepare.yml create mode 100644 kubespray/project/roles/container-engine/containerd/molecule/default/tests/test_default.py create mode 100644 kubespray/project/roles/container-engine/containerd/tasks/main.yml create mode 100644 kubespray/project/roles/container-engine/containerd/tasks/reset.yml create mode 100644 kubespray/project/roles/container-engine/containerd/templates/config.toml.j2 create mode 100644 kubespray/project/roles/container-engine/containerd/templates/containerd.service.j2 create mode 100644 kubespray/project/roles/container-engine/containerd/templates/hosts.toml.j2 create mode 100644 kubespray/project/roles/container-engine/containerd/templates/http-proxy.conf.j2 create mode 100644 kubespray/project/roles/container-engine/containerd/vars/debian.yml create mode 100644 kubespray/project/roles/container-engine/containerd/vars/ubuntu.yml create mode 100644 kubespray/project/roles/container-engine/cri-dockerd/handlers/main.yml create mode 100644 kubespray/project/roles/container-engine/cri-dockerd/meta/main.yml create mode 100644 kubespray/project/roles/container-engine/cri-dockerd/molecule/default/converge.yml create mode 100644 kubespray/project/roles/container-engine/cri-dockerd/molecule/default/files/10-mynet.conf create mode 100644 kubespray/project/roles/container-engine/cri-dockerd/molecule/default/files/container.json create mode 100644 kubespray/project/roles/container-engine/cri-dockerd/molecule/default/files/sandbox.json create mode 100644 kubespray/project/roles/container-engine/cri-dockerd/molecule/default/molecule.yml create mode 100644 kubespray/project/roles/container-engine/cri-dockerd/molecule/default/prepare.yml create mode 100644 kubespray/project/roles/container-engine/cri-dockerd/molecule/default/tests/test_default.py create mode 100644 kubespray/project/roles/container-engine/cri-dockerd/tasks/main.yml create mode 100644 kubespray/project/roles/container-engine/cri-dockerd/templates/cri-dockerd.service.j2 create mode 100644 kubespray/project/roles/container-engine/cri-dockerd/templates/cri-dockerd.socket.j2 create mode 100644 kubespray/project/roles/container-engine/cri-o/defaults/main.yml create mode 100644 kubespray/project/roles/container-engine/cri-o/files/mounts.conf create mode 100644 kubespray/project/roles/container-engine/cri-o/handlers/main.yml create mode 100644 kubespray/project/roles/container-engine/cri-o/meta/main.yml create mode 100644 kubespray/project/roles/container-engine/cri-o/molecule/default/converge.yml create mode 100644 kubespray/project/roles/container-engine/cri-o/molecule/default/files/10-mynet.conf create mode 100644 kubespray/project/roles/container-engine/cri-o/molecule/default/files/container.json create mode 100644 kubespray/project/roles/container-engine/cri-o/molecule/default/files/sandbox.json create mode 100644 kubespray/project/roles/container-engine/cri-o/molecule/default/molecule.yml create mode 100644 kubespray/project/roles/container-engine/cri-o/molecule/default/prepare.yml create mode 100644 kubespray/project/roles/container-engine/cri-o/molecule/default/tests/test_default.py create mode 100644 kubespray/project/roles/container-engine/cri-o/tasks/load_vars.yml create mode 100644 kubespray/project/roles/container-engine/cri-o/tasks/main.yaml create mode 100644 kubespray/project/roles/container-engine/cri-o/tasks/reset.yml create mode 100644 kubespray/project/roles/container-engine/cri-o/tasks/setup-amazon.yaml create mode 100644 kubespray/project/roles/container-engine/cri-o/templates/config.json.j2 create mode 100644 kubespray/project/roles/container-engine/cri-o/templates/crio.conf.j2 create mode 100644 kubespray/project/roles/container-engine/cri-o/templates/http-proxy.conf.j2 create mode 100644 kubespray/project/roles/container-engine/cri-o/templates/registry.conf.j2 create mode 100644 kubespray/project/roles/container-engine/cri-o/templates/unqualified.conf.j2 create mode 100644 kubespray/project/roles/container-engine/cri-o/vars/v1.28.yml create mode 100644 kubespray/project/roles/container-engine/cri-o/vars/v1.29.yml create mode 100644 kubespray/project/roles/container-engine/crictl/handlers/main.yml create mode 100644 kubespray/project/roles/container-engine/crictl/tasks/crictl.yml create mode 100644 kubespray/project/roles/container-engine/crictl/tasks/main.yml create mode 100644 kubespray/project/roles/container-engine/crictl/templates/crictl.yaml.j2 create mode 100644 kubespray/project/roles/container-engine/crun/tasks/main.yml create mode 100644 kubespray/project/roles/container-engine/docker-storage/defaults/main.yml create mode 100644 kubespray/project/roles/container-engine/docker-storage/files/install_container_storage_setup.sh create mode 100644 kubespray/project/roles/container-engine/docker-storage/tasks/main.yml create mode 100644 kubespray/project/roles/container-engine/docker-storage/templates/docker-storage-setup.j2 create mode 100644 kubespray/project/roles/container-engine/docker/defaults/main.yml create mode 100644 kubespray/project/roles/container-engine/docker/files/cleanup-docker-orphans.sh create mode 100644 kubespray/project/roles/container-engine/docker/handlers/main.yml create mode 100644 kubespray/project/roles/container-engine/docker/meta/main.yml create mode 100644 kubespray/project/roles/container-engine/docker/tasks/docker_plugin.yml create mode 100644 kubespray/project/roles/container-engine/docker/tasks/main.yml create mode 100644 kubespray/project/roles/container-engine/docker/tasks/pre-upgrade.yml create mode 100644 kubespray/project/roles/container-engine/docker/tasks/reset.yml create mode 100644 kubespray/project/roles/container-engine/docker/tasks/set_facts_dns.yml create mode 100644 kubespray/project/roles/container-engine/docker/tasks/systemd.yml create mode 100644 kubespray/project/roles/container-engine/docker/templates/docker-dns.conf.j2 create mode 100644 kubespray/project/roles/container-engine/docker/templates/docker-options.conf.j2 create mode 100644 kubespray/project/roles/container-engine/docker/templates/docker-orphan-cleanup.conf.j2 create mode 100644 kubespray/project/roles/container-engine/docker/templates/docker.service.j2 create mode 100644 kubespray/project/roles/container-engine/docker/templates/fedora_docker.repo.j2 create mode 100644 kubespray/project/roles/container-engine/docker/templates/http-proxy.conf.j2 create mode 100644 kubespray/project/roles/container-engine/docker/templates/rh_docker.repo.j2 create mode 100644 kubespray/project/roles/container-engine/docker/vars/amazon.yml create mode 100644 kubespray/project/roles/container-engine/docker/vars/clearlinux.yml create mode 100644 kubespray/project/roles/container-engine/docker/vars/debian.yml create mode 100644 kubespray/project/roles/container-engine/docker/vars/fedora.yml create mode 100644 kubespray/project/roles/container-engine/docker/vars/kylin.yml create mode 100644 kubespray/project/roles/container-engine/docker/vars/redhat-7.yml create mode 100644 kubespray/project/roles/container-engine/docker/vars/redhat.yml create mode 100644 kubespray/project/roles/container-engine/docker/vars/suse.yml create mode 100644 kubespray/project/roles/container-engine/docker/vars/ubuntu.yml create mode 100644 kubespray/project/roles/container-engine/docker/vars/uniontech.yml create mode 100644 kubespray/project/roles/container-engine/gvisor/molecule/default/converge.yml create mode 100644 kubespray/project/roles/container-engine/gvisor/molecule/default/files/10-mynet.conf create mode 100644 kubespray/project/roles/container-engine/gvisor/molecule/default/files/container.json create mode 100644 kubespray/project/roles/container-engine/gvisor/molecule/default/files/sandbox.json create mode 100644 kubespray/project/roles/container-engine/gvisor/molecule/default/molecule.yml create mode 100644 kubespray/project/roles/container-engine/gvisor/molecule/default/prepare.yml create mode 100644 kubespray/project/roles/container-engine/gvisor/molecule/default/tests/test_default.py create mode 100644 kubespray/project/roles/container-engine/gvisor/tasks/main.yml create mode 100644 kubespray/project/roles/container-engine/kata-containers/OWNERS create mode 100644 kubespray/project/roles/container-engine/kata-containers/defaults/main.yml create mode 100644 kubespray/project/roles/container-engine/kata-containers/molecule/default/converge.yml create mode 100644 kubespray/project/roles/container-engine/kata-containers/molecule/default/files/10-mynet.conf create mode 100644 kubespray/project/roles/container-engine/kata-containers/molecule/default/files/container.json create mode 100644 kubespray/project/roles/container-engine/kata-containers/molecule/default/files/sandbox.json create mode 100644 kubespray/project/roles/container-engine/kata-containers/molecule/default/molecule.yml create mode 100644 kubespray/project/roles/container-engine/kata-containers/molecule/default/prepare.yml create mode 100644 kubespray/project/roles/container-engine/kata-containers/molecule/default/tests/test_default.py create mode 100644 kubespray/project/roles/container-engine/kata-containers/tasks/main.yml create mode 100644 kubespray/project/roles/container-engine/kata-containers/templates/configuration-qemu.toml.j2 create mode 100644 kubespray/project/roles/container-engine/kata-containers/templates/containerd-shim-kata-v2.j2 create mode 100644 kubespray/project/roles/container-engine/meta/main.yml create mode 100644 kubespray/project/roles/container-engine/nerdctl/handlers/main.yml create mode 100644 kubespray/project/roles/container-engine/nerdctl/tasks/main.yml create mode 100644 kubespray/project/roles/container-engine/nerdctl/templates/nerdctl.toml.j2 create mode 100644 kubespray/project/roles/container-engine/runc/defaults/main.yml create mode 100644 kubespray/project/roles/container-engine/runc/tasks/main.yml create mode 100644 kubespray/project/roles/container-engine/skopeo/tasks/main.yml create mode 100644 kubespray/project/roles/container-engine/validate-container-engine/tasks/main.yml create mode 100644 kubespray/project/roles/container-engine/youki/defaults/main.yml create mode 100644 kubespray/project/roles/container-engine/youki/molecule/default/converge.yml create mode 100644 kubespray/project/roles/container-engine/youki/molecule/default/files/10-mynet.conf create mode 100644 kubespray/project/roles/container-engine/youki/molecule/default/files/container.json create mode 100644 kubespray/project/roles/container-engine/youki/molecule/default/files/sandbox.json create mode 100644 kubespray/project/roles/container-engine/youki/molecule/default/molecule.yml create mode 100644 kubespray/project/roles/container-engine/youki/molecule/default/prepare.yml create mode 100644 kubespray/project/roles/container-engine/youki/molecule/default/tests/test_default.py create mode 100644 kubespray/project/roles/container-engine/youki/tasks/main.yml create mode 100644 kubespray/project/roles/download/meta/main.yml create mode 100644 kubespray/project/roles/download/tasks/check_pull_required.yml create mode 100644 kubespray/project/roles/download/tasks/download_container.yml create mode 100644 kubespray/project/roles/download/tasks/download_file.yml create mode 100644 kubespray/project/roles/download/tasks/extract_file.yml create mode 100644 kubespray/project/roles/download/tasks/main.yml create mode 100644 kubespray/project/roles/download/tasks/prep_download.yml create mode 100644 kubespray/project/roles/download/tasks/prep_kubeadm_images.yml create mode 100644 kubespray/project/roles/download/tasks/set_container_facts.yml create mode 100644 kubespray/project/roles/download/templates/kubeadm-images.yaml.j2 create mode 100644 kubespray/project/roles/etcd/defaults/main.yml create mode 100644 kubespray/project/roles/etcd/handlers/backup.yml create mode 100644 kubespray/project/roles/etcd/handlers/backup_cleanup.yml create mode 100644 kubespray/project/roles/etcd/handlers/main.yml create mode 100644 kubespray/project/roles/etcd/meta/main.yml create mode 100644 kubespray/project/roles/etcd/tasks/check_certs.yml create mode 100644 kubespray/project/roles/etcd/tasks/configure.yml create mode 100644 kubespray/project/roles/etcd/tasks/gen_certs_script.yml create mode 100644 kubespray/project/roles/etcd/tasks/gen_nodes_certs_script.yml create mode 100644 kubespray/project/roles/etcd/tasks/install_docker.yml create mode 100644 kubespray/project/roles/etcd/tasks/install_host.yml create mode 100644 kubespray/project/roles/etcd/tasks/join_etcd-events_member.yml create mode 100644 kubespray/project/roles/etcd/tasks/join_etcd_member.yml create mode 100644 kubespray/project/roles/etcd/tasks/main.yml create mode 100644 kubespray/project/roles/etcd/tasks/refresh_config.yml create mode 100644 kubespray/project/roles/etcd/tasks/upd_ca_trust.yml create mode 100644 kubespray/project/roles/etcd/templates/etcd-docker.service.j2 create mode 100644 kubespray/project/roles/etcd/templates/etcd-events-docker.service.j2 create mode 100644 kubespray/project/roles/etcd/templates/etcd-events-host.service.j2 create mode 100644 kubespray/project/roles/etcd/templates/etcd-events.env.j2 create mode 100644 kubespray/project/roles/etcd/templates/etcd-events.j2 create mode 100644 kubespray/project/roles/etcd/templates/etcd-host.service.j2 create mode 100644 kubespray/project/roles/etcd/templates/etcd.env.j2 create mode 100644 kubespray/project/roles/etcd/templates/etcd.j2 create mode 100644 kubespray/project/roles/etcd/templates/make-ssl-etcd.sh.j2 create mode 100644 kubespray/project/roles/etcd/templates/openssl.conf.j2 create mode 100644 kubespray/project/roles/etcd/vars/main.yml create mode 100644 kubespray/project/roles/etcdctl_etcdutl/tasks/main.yml create mode 100644 kubespray/project/roles/etcdctl_etcdutl/templates/etcdctl.sh.j2 create mode 100644 kubespray/project/roles/helm-apps/README.md create mode 100644 kubespray/project/roles/helm-apps/meta/argument_specs.yml create mode 100644 kubespray/project/roles/helm-apps/meta/main.yml create mode 100644 kubespray/project/roles/helm-apps/tasks/main.yml create mode 100644 kubespray/project/roles/helm-apps/vars/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/ansible/defaults/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/ansible/tasks/coredns.yml create mode 100644 kubespray/project/roles/kubernetes-apps/ansible/tasks/dashboard.yml create mode 100644 kubespray/project/roles/kubernetes-apps/ansible/tasks/etcd_metrics.yml create mode 100644 kubespray/project/roles/kubernetes-apps/ansible/tasks/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/ansible/tasks/netchecker.yml create mode 100644 kubespray/project/roles/kubernetes-apps/ansible/tasks/nodelocaldns.yml create mode 100644 kubespray/project/roles/kubernetes-apps/ansible/templates/coredns-clusterrole.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/ansible/templates/coredns-clusterrolebinding.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/ansible/templates/coredns-config.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/ansible/templates/coredns-deployment.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/ansible/templates/coredns-poddisruptionbudget.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/ansible/templates/coredns-sa.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/ansible/templates/coredns-svc.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/ansible/templates/dashboard.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/ansible/templates/dns-autoscaler-clusterrole.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/ansible/templates/dns-autoscaler-clusterrolebinding.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/ansible/templates/dns-autoscaler-sa.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/ansible/templates/dns-autoscaler.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/ansible/templates/etcd_metrics-endpoints.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/ansible/templates/etcd_metrics-service.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/ansible/templates/netchecker-agent-ds.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/ansible/templates/netchecker-agent-sa.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/ansible/templates/netchecker-ns.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/ansible/templates/netchecker-server-clusterrole.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/ansible/templates/netchecker-server-clusterrolebinding.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/ansible/templates/netchecker-server-deployment.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/ansible/templates/netchecker-server-sa.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/ansible/templates/netchecker-server-svc.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/ansible/templates/nodelocaldns-config.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/ansible/templates/nodelocaldns-daemonset.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/ansible/templates/nodelocaldns-sa.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/ansible/templates/nodelocaldns-second-daemonset.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/argocd/defaults/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/argocd/tasks/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/argocd/templates/argocd-namespace.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/cloud_controller/oci/defaults/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/cloud_controller/oci/tasks/credentials-check.yml create mode 100644 kubespray/project/roles/kubernetes-apps/cloud_controller/oci/tasks/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/cloud_controller/oci/templates/controller-manager-config.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/cloud_controller/oci/templates/oci-cloud-provider.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/cluster_roles/files/k8s-cluster-critical-pc.yml create mode 100644 kubespray/project/roles/kubernetes-apps/cluster_roles/files/oci-rbac.yml create mode 100644 kubespray/project/roles/kubernetes-apps/cluster_roles/tasks/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/cluster_roles/tasks/oci.yml create mode 100644 kubespray/project/roles/kubernetes-apps/cluster_roles/templates/namespace.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/cluster_roles/templates/node-crb.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/cluster_roles/templates/vsphere-rbac.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/container_engine_accelerator/meta/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/defaults/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/tasks/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/templates/k8s-device-plugin-nvidia-daemonset.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/templates/nvidia-driver-install-daemonset.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/vars/centos-7.yml create mode 100644 kubespray/project/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/vars/ubuntu-16.yml create mode 100644 kubespray/project/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/vars/ubuntu-18.yml create mode 100644 kubespray/project/roles/kubernetes-apps/container_runtimes/crun/tasks/main.yaml create mode 100644 kubespray/project/roles/kubernetes-apps/container_runtimes/crun/templates/runtimeclass-crun.yml create mode 100644 kubespray/project/roles/kubernetes-apps/container_runtimes/gvisor/tasks/main.yaml create mode 100644 kubespray/project/roles/kubernetes-apps/container_runtimes/gvisor/templates/runtimeclass-gvisor.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/container_runtimes/kata_containers/defaults/main.yaml create mode 100644 kubespray/project/roles/kubernetes-apps/container_runtimes/kata_containers/tasks/main.yaml create mode 100644 kubespray/project/roles/kubernetes-apps/container_runtimes/kata_containers/templates/runtimeclass-kata-qemu.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/container_runtimes/meta/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/container_runtimes/youki/tasks/main.yaml create mode 100644 kubespray/project/roles/kubernetes-apps/container_runtimes/youki/templates/runtimeclass-youki.yml create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/OWNERS create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/aws_ebs/defaults/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/aws_ebs/tasks/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-controllerservice-rbac.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-controllerservice.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-driver.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-nodeservice.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/azuredisk/defaults/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/azuredisk/tasks/azure-credential-check.yml create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/azuredisk/tasks/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-controller-rbac.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-controller.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-driver.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-node-rbac.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-node.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-cloud-config-secret.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-cloud-config.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/cinder/defaults/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/cinder/tasks/cinder-credential-check.yml create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/cinder/tasks/cinder-write-cacert.yml create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/cinder/tasks/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-cloud-config-secret.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-cloud-config.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-controllerplugin-rbac.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-controllerplugin.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-driver.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-nodeplugin-rbac.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-nodeplugin.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-poddisruptionbudget.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/csi_crd/tasks/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/csi_crd/templates/volumesnapshotclasses.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/csi_crd/templates/volumesnapshotcontents.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/csi_crd/templates/volumesnapshots.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/gcp_pd/defaults/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/gcp_pd/tasks/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-controller.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-cred-secret.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-node.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-sc-regional.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-sc-zonal.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-setup.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/upcloud/defaults/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/upcloud/tasks/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/upcloud/templates/upcloud-csi-controller.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/upcloud/templates/upcloud-csi-cred-secret.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/upcloud/templates/upcloud-csi-driver.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/upcloud/templates/upcloud-csi-node.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/upcloud/templates/upcloud-csi-setup.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/vsphere/defaults/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/vsphere/tasks/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/vsphere/tasks/vsphere-credentials-check.yml create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-cloud-config.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-config.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-deployment.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-rbac.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-service.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-driver.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-namespace.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-node-rbac.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-node.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/external_cloud_controller/hcloud/defaults/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/external_cloud_controller/hcloud/tasks/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-controller-manager-ds-with-networks.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-controller-manager-ds.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-role-bindings.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-secret.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-service-account.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/external_cloud_controller/huaweicloud/defaults/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/external_cloud_controller/huaweicloud/tasks/huaweicloud-credential-check.yml create mode 100644 kubespray/project/roles/kubernetes-apps/external_cloud_controller/huaweicloud/tasks/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-config-secret.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-config.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-controller-manager-ds.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-controller-manager-role-bindings.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-controller-manager-roles.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/external_cloud_controller/meta/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/external_cloud_controller/openstack/OWNERS create mode 100644 kubespray/project/roles/kubernetes-apps/external_cloud_controller/openstack/defaults/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/external_cloud_controller/openstack/tasks/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/external_cloud_controller/openstack/tasks/openstack-credential-check.yml create mode 100644 kubespray/project/roles/kubernetes-apps/external_cloud_controller/openstack/templates/external-openstack-cloud-config-secret.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/external_cloud_controller/openstack/templates/external-openstack-cloud-config.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/external_cloud_controller/openstack/templates/external-openstack-cloud-controller-manager-ds.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/external_cloud_controller/openstack/templates/external-openstack-cloud-controller-manager-role-bindings.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/external_cloud_controller/openstack/templates/external-openstack-cloud-controller-manager-roles.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/external_cloud_controller/vsphere/defaults/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/external_cloud_controller/vsphere/tasks/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/external_cloud_controller/vsphere/tasks/vsphere-credentials-check.yml create mode 100644 kubespray/project/roles/kubernetes-apps/external_cloud_controller/vsphere/templates/external-vsphere-cloud-controller-manager-ds.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/external_cloud_controller/vsphere/templates/external-vsphere-cloud-controller-manager-role-bindings.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/external_cloud_controller/vsphere/templates/external-vsphere-cloud-controller-manager-roles.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/external_cloud_controller/vsphere/templates/external-vsphere-cpi-cloud-config-secret.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/external_cloud_controller/vsphere/templates/external-vsphere-cpi-cloud-config.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/defaults/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/tasks/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/00-namespace.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/clusterrole-cephfs-provisioner.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/clusterrolebinding-cephfs-provisioner.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/deploy-cephfs-provisioner.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/role-cephfs-provisioner.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/rolebinding-cephfs-provisioner.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/sa-cephfs-provisioner.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/sc-cephfs-provisioner.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/secret-cephfs-provisioner.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/external_provisioner/local_path_provisioner/defaults/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/external_provisioner/local_path_provisioner/tasks/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-clusterrolebinding.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-cm.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-cr.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-deployment.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-ns.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-sa.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-sc.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/defaults/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/tasks/basedirs.yml create mode 100644 kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/tasks/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-clusterrole.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-clusterrolebinding.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-cm.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-ds.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-ns.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-sa.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-sc.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/external_provisioner/meta/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/external_provisioner/rbd_provisioner/defaults/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/external_provisioner/rbd_provisioner/tasks/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/00-namespace.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/clusterrole-rbd-provisioner.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/clusterrolebinding-rbd-provisioner.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/deploy-rbd-provisioner.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/role-rbd-provisioner.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/rolebinding-rbd-provisioner.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/sa-rbd-provisioner.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/sc-rbd-provisioner.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/secret-rbd-provisioner.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/helm/.gitkeep create mode 100644 kubespray/project/roles/kubernetes-apps/helm/defaults/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/helm/tasks/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/helm/tasks/pyyaml-flatcar.yml create mode 100644 kubespray/project/roles/kubernetes-apps/helm/vars/amazon.yml create mode 100644 kubespray/project/roles/kubernetes-apps/helm/vars/centos-7.yml create mode 100644 kubespray/project/roles/kubernetes-apps/helm/vars/centos.yml create mode 100644 kubespray/project/roles/kubernetes-apps/helm/vars/debian.yml create mode 100644 kubespray/project/roles/kubernetes-apps/helm/vars/fedora.yml create mode 100644 kubespray/project/roles/kubernetes-apps/helm/vars/redhat-7.yml create mode 100644 kubespray/project/roles/kubernetes-apps/helm/vars/redhat.yml create mode 100644 kubespray/project/roles/kubernetes-apps/helm/vars/suse.yml create mode 100644 kubespray/project/roles/kubernetes-apps/helm/vars/ubuntu.yml create mode 100644 kubespray/project/roles/kubernetes-apps/ingress_controller/alb_ingress_controller/OWNERS create mode 100644 kubespray/project/roles/kubernetes-apps/ingress_controller/alb_ingress_controller/defaults/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/ingress_controller/alb_ingress_controller/tasks/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/ingress_controller/alb_ingress_controller/templates/alb-ingress-clusterrole.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/ingress_controller/alb_ingress_controller/templates/alb-ingress-clusterrolebinding.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/ingress_controller/alb_ingress_controller/templates/alb-ingress-deploy.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/ingress_controller/alb_ingress_controller/templates/alb-ingress-ns.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/ingress_controller/alb_ingress_controller/templates/alb-ingress-sa.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/ingress_controller/cert_manager/defaults/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/ingress_controller/cert_manager/tasks/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/ingress_controller/cert_manager/templates/cert-manager.crds.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/ingress_controller/cert_manager/templates/cert-manager.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/defaults/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/tasks/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/00-namespace.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/admission-webhook-configuration.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/admission-webhook-job.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/clusterrole-admission-webhook.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/clusterrole-ingress-nginx.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/clusterrolebinding-admission-webhook.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/clusterrolebinding-ingress-nginx.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/cm-ingress-nginx.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/cm-tcp-services.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/cm-udp-services.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ds-ingress-nginx-controller.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingressclass-nginx.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/role-admission-webhook.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/role-ingress-nginx.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/rolebinding-admission-webhook.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/rolebinding-ingress-nginx.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/sa-admission-webhook.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/sa-ingress-nginx.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/svc-ingress-nginx.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/ingress_controller/meta/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/krew/defaults/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/krew/tasks/krew.yml create mode 100644 kubespray/project/roles/kubernetes-apps/krew/tasks/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/krew/templates/krew.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/krew/templates/krew.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/kubelet-csr-approver/defaults/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/kubelet-csr-approver/meta/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/meta/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/metallb/OWNERS create mode 100644 kubespray/project/roles/kubernetes-apps/metallb/defaults/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/metallb/tasks/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/metallb/templates/layer2.yaml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/metallb/templates/layer3.yaml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/metallb/templates/metallb.yaml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/metallb/templates/pools.yaml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/metrics_server/defaults/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/metrics_server/tasks/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/metrics_server/templates/auth-delegator.yaml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/metrics_server/templates/auth-reader.yaml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/metrics_server/templates/metrics-apiservice.yaml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/metrics_server/templates/metrics-server-deployment.yaml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/metrics_server/templates/metrics-server-sa.yaml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/metrics_server/templates/metrics-server-service.yaml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/metrics_server/templates/resource-reader-clusterrolebinding.yaml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/metrics_server/templates/resource-reader.yaml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/network_plugin/calico/tasks/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/network_plugin/flannel/tasks/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/network_plugin/kube-ovn/tasks/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/network_plugin/kube-router/OWNERS create mode 100644 kubespray/project/roles/kubernetes-apps/network_plugin/kube-router/tasks/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/network_plugin/meta/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/network_plugin/multus/tasks/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/network_plugin/weave/tasks/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/persistent_volumes/aws-ebs-csi/OWNERS create mode 100644 kubespray/project/roles/kubernetes-apps/persistent_volumes/aws-ebs-csi/defaults/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/persistent_volumes/aws-ebs-csi/tasks/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/persistent_volumes/aws-ebs-csi/templates/aws-ebs-csi-storage-class.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/persistent_volumes/azuredisk-csi/defaults/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/persistent_volumes/azuredisk-csi/tasks/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/persistent_volumes/azuredisk-csi/templates/azure-csi-storage-class.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/persistent_volumes/cinder-csi/defaults/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/persistent_volumes/cinder-csi/tasks/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/persistent_volumes/cinder-csi/templates/cinder-csi-storage-class.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/persistent_volumes/gcp-pd-csi/defaults/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/persistent_volumes/gcp-pd-csi/tasks/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/persistent_volumes/gcp-pd-csi/templates/gcp-pd-csi-storage-class.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/persistent_volumes/meta/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/persistent_volumes/openstack/defaults/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/persistent_volumes/openstack/tasks/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/persistent_volumes/openstack/templates/openstack-storage-class.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/persistent_volumes/upcloud-csi/defaults/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/persistent_volumes/upcloud-csi/tasks/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/persistent_volumes/upcloud-csi/templates/upcloud-csi-storage-class.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/policy_controller/calico/defaults/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/policy_controller/calico/tasks/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-controllers.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-cr.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-crb.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-sa.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/policy_controller/meta/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/registry/defaults/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/registry/tasks/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/registry/templates/registry-cm.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/registry/templates/registry-ing.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/registry/templates/registry-ns.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/registry/templates/registry-pvc.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/registry/templates/registry-rs.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/registry/templates/registry-sa.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/registry/templates/registry-secrets.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/registry/templates/registry-svc.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/scheduler_plugins/defaults/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/scheduler_plugins/tasks/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/scheduler_plugins/templates/appgroup.diktyo.x-k8s.io_appgroups.yaml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/scheduler_plugins/templates/cm-scheduler-plugins.yaml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/scheduler_plugins/templates/deploy-scheduler-plugins.yaml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/scheduler_plugins/templates/namespace.yaml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/scheduler_plugins/templates/networktopology.diktyo.x-k8s.io_networktopologies.yaml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/scheduler_plugins/templates/rbac-scheduler-plugins.yaml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/scheduler_plugins/templates/sa-scheduler-plugins.yaml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/scheduler_plugins/templates/scheduling.x-k8s.io_elasticquotas.yaml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/scheduler_plugins/templates/scheduling.x-k8s.io_podgroups.yaml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/scheduler_plugins/templates/topology.node.k8s.io_noderesourcetopologies.yaml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/snapshots/cinder-csi/defaults/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/snapshots/cinder-csi/tasks/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/snapshots/cinder-csi/templates/cinder-csi-snapshot-class.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/snapshots/meta/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/snapshots/snapshot-controller/defaults/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/snapshots/snapshot-controller/tasks/main.yml create mode 100644 kubespray/project/roles/kubernetes-apps/snapshots/snapshot-controller/templates/rbac-snapshot-controller.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/snapshots/snapshot-controller/templates/snapshot-controller.yml.j2 create mode 100644 kubespray/project/roles/kubernetes-apps/snapshots/snapshot-controller/templates/snapshot-ns.yml.j2 create mode 100644 kubespray/project/roles/kubernetes/client/defaults/main.yml create mode 100644 kubespray/project/roles/kubernetes/client/tasks/main.yml create mode 100644 kubespray/project/roles/kubernetes/control-plane/defaults/main/etcd.yml create mode 100644 kubespray/project/roles/kubernetes/control-plane/defaults/main/kube-proxy.yml create mode 100644 kubespray/project/roles/kubernetes/control-plane/defaults/main/kube-scheduler.yml create mode 100644 kubespray/project/roles/kubernetes/control-plane/defaults/main/main.yml create mode 100644 kubespray/project/roles/kubernetes/control-plane/handlers/main.yml create mode 100644 kubespray/project/roles/kubernetes/control-plane/meta/main.yml create mode 100644 kubespray/project/roles/kubernetes/control-plane/tasks/define-first-kube-control.yml create mode 100644 kubespray/project/roles/kubernetes/control-plane/tasks/encrypt-at-rest.yml create mode 100644 kubespray/project/roles/kubernetes/control-plane/tasks/kubeadm-backup.yml create mode 100644 kubespray/project/roles/kubernetes/control-plane/tasks/kubeadm-etcd.yml create mode 100644 kubespray/project/roles/kubernetes/control-plane/tasks/kubeadm-fix-apiserver.yml create mode 100644 kubespray/project/roles/kubernetes/control-plane/tasks/kubeadm-secondary.yml create mode 100644 kubespray/project/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml create mode 100644 kubespray/project/roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml create mode 100644 kubespray/project/roles/kubernetes/control-plane/tasks/kubelet-fix-client-cert-rotation.yml create mode 100644 kubespray/project/roles/kubernetes/control-plane/tasks/main.yml create mode 100644 kubespray/project/roles/kubernetes/control-plane/tasks/pre-upgrade.yml create mode 100644 kubespray/project/roles/kubernetes/control-plane/templates/admission-controls.yaml.j2 create mode 100644 kubespray/project/roles/kubernetes/control-plane/templates/apiserver-audit-policy.yaml.j2 create mode 100644 kubespray/project/roles/kubernetes/control-plane/templates/apiserver-audit-webhook-config.yaml.j2 create mode 100644 kubespray/project/roles/kubernetes/control-plane/templates/apiserver-tracing.yaml.j2 create mode 100644 kubespray/project/roles/kubernetes/control-plane/templates/eventratelimit.yaml.j2 create mode 100644 kubespray/project/roles/kubernetes/control-plane/templates/k8s-certs-renew.service.j2 create mode 100644 kubespray/project/roles/kubernetes/control-plane/templates/k8s-certs-renew.sh.j2 create mode 100644 kubespray/project/roles/kubernetes/control-plane/templates/k8s-certs-renew.timer.j2 create mode 100644 kubespray/project/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2 create mode 100644 kubespray/project/roles/kubernetes/control-plane/templates/kubeadm-controlplane.v1beta3.yaml.j2 create mode 100644 kubespray/project/roles/kubernetes/control-plane/templates/kubescheduler-config.yaml.j2 create mode 100644 kubespray/project/roles/kubernetes/control-plane/templates/podnodeselector.yaml.j2 create mode 100644 kubespray/project/roles/kubernetes/control-plane/templates/podsecurity.yaml.j2 create mode 100644 kubespray/project/roles/kubernetes/control-plane/templates/secrets_encryption.yaml.j2 create mode 100644 kubespray/project/roles/kubernetes/control-plane/templates/webhook-authorization-config.yaml.j2 create mode 100644 kubespray/project/roles/kubernetes/control-plane/templates/webhook-token-auth-config.yaml.j2 create mode 100644 kubespray/project/roles/kubernetes/control-plane/vars/main.yaml create mode 100644 kubespray/project/roles/kubernetes/kubeadm/defaults/main.yml create mode 100644 kubespray/project/roles/kubernetes/kubeadm/handlers/main.yml create mode 100644 kubespray/project/roles/kubernetes/kubeadm/tasks/kubeadm_etcd_node.yml create mode 100644 kubespray/project/roles/kubernetes/kubeadm/tasks/main.yml create mode 100644 kubespray/project/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta3.j2 create mode 100644 kubespray/project/roles/kubernetes/node-label/tasks/main.yml create mode 100644 kubespray/project/roles/kubernetes/node-taint/tasks/main.yml create mode 100644 kubespray/project/roles/kubernetes/node/defaults/main.yml create mode 100644 kubespray/project/roles/kubernetes/node/handlers/main.yml create mode 100644 kubespray/project/roles/kubernetes/node/tasks/cloud-credentials/azure-credential-check.yml create mode 100644 kubespray/project/roles/kubernetes/node/tasks/cloud-credentials/openstack-credential-check.yml create mode 100644 kubespray/project/roles/kubernetes/node/tasks/cloud-credentials/vsphere-credential-check.yml create mode 100644 kubespray/project/roles/kubernetes/node/tasks/facts.yml create mode 100644 kubespray/project/roles/kubernetes/node/tasks/install.yml create mode 100644 kubespray/project/roles/kubernetes/node/tasks/kubelet.yml create mode 100644 kubespray/project/roles/kubernetes/node/tasks/loadbalancer/haproxy.yml create mode 100644 kubespray/project/roles/kubernetes/node/tasks/loadbalancer/kube-vip.yml create mode 100644 kubespray/project/roles/kubernetes/node/tasks/loadbalancer/nginx-proxy.yml create mode 100644 kubespray/project/roles/kubernetes/node/tasks/main.yml create mode 100644 kubespray/project/roles/kubernetes/node/tasks/pre_upgrade.yml create mode 100644 kubespray/project/roles/kubernetes/node/templates/cloud-configs/aws-cloud-config.j2 create mode 100644 kubespray/project/roles/kubernetes/node/templates/cloud-configs/azure-cloud-config.j2 create mode 100644 kubespray/project/roles/kubernetes/node/templates/cloud-configs/gce-cloud-config.j2 create mode 100644 kubespray/project/roles/kubernetes/node/templates/cloud-configs/openstack-cloud-config.j2 create mode 100644 kubespray/project/roles/kubernetes/node/templates/cloud-configs/vsphere-cloud-config.j2 create mode 100644 kubespray/project/roles/kubernetes/node/templates/http-proxy.conf.j2 create mode 100644 kubespray/project/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2 create mode 100644 kubespray/project/roles/kubernetes/node/templates/kubelet.env.v1beta1.j2 create mode 100644 kubespray/project/roles/kubernetes/node/templates/kubelet.service.j2 create mode 100644 kubespray/project/roles/kubernetes/node/templates/loadbalancer/haproxy.cfg.j2 create mode 100644 kubespray/project/roles/kubernetes/node/templates/loadbalancer/nginx.conf.j2 create mode 100644 kubespray/project/roles/kubernetes/node/templates/manifests/haproxy.manifest.j2 create mode 100644 kubespray/project/roles/kubernetes/node/templates/manifests/kube-vip.manifest.j2 create mode 100644 kubespray/project/roles/kubernetes/node/templates/manifests/nginx-proxy.manifest.j2 create mode 100644 kubespray/project/roles/kubernetes/node/templates/node-kubeconfig.yaml.j2 create mode 100644 kubespray/project/roles/kubernetes/node/vars/fedora.yml create mode 100644 kubespray/project/roles/kubernetes/node/vars/ubuntu-18.yml create mode 100644 kubespray/project/roles/kubernetes/node/vars/ubuntu-20.yml create mode 100644 kubespray/project/roles/kubernetes/node/vars/ubuntu-22.yml create mode 100644 kubespray/project/roles/kubernetes/preinstall/defaults/main.yml create mode 100644 kubespray/project/roles/kubernetes/preinstall/files/dhclient_nodnsupdate create mode 100755 kubespray/project/roles/kubernetes/preinstall/gen-gitinfos.sh create mode 100644 kubespray/project/roles/kubernetes/preinstall/handlers/main.yml create mode 100644 kubespray/project/roles/kubernetes/preinstall/meta/main.yml create mode 100644 kubespray/project/roles/kubernetes/preinstall/tasks/0010-swapoff.yml create mode 100644 kubespray/project/roles/kubernetes/preinstall/tasks/0020-set_facts.yml create mode 100644 kubespray/project/roles/kubernetes/preinstall/tasks/0040-verify-settings.yml create mode 100644 kubespray/project/roles/kubernetes/preinstall/tasks/0050-create_directories.yml create mode 100644 kubespray/project/roles/kubernetes/preinstall/tasks/0060-resolvconf.yml create mode 100644 kubespray/project/roles/kubernetes/preinstall/tasks/0061-systemd-resolved.yml create mode 100644 kubespray/project/roles/kubernetes/preinstall/tasks/0062-networkmanager-unmanaged-devices.yml create mode 100644 kubespray/project/roles/kubernetes/preinstall/tasks/0063-networkmanager-dns.yml create mode 100644 kubespray/project/roles/kubernetes/preinstall/tasks/0070-system-packages.yml create mode 100644 kubespray/project/roles/kubernetes/preinstall/tasks/0080-system-configurations.yml create mode 100644 kubespray/project/roles/kubernetes/preinstall/tasks/0081-ntp-configurations.yml create mode 100644 kubespray/project/roles/kubernetes/preinstall/tasks/0090-etchosts.yml create mode 100644 kubespray/project/roles/kubernetes/preinstall/tasks/0100-dhclient-hooks.yml create mode 100644 kubespray/project/roles/kubernetes/preinstall/tasks/0110-dhclient-hooks-undo.yml create mode 100644 kubespray/project/roles/kubernetes/preinstall/tasks/0120-growpart-azure-centos-7.yml create mode 100644 kubespray/project/roles/kubernetes/preinstall/tasks/main.yml create mode 100644 kubespray/project/roles/kubernetes/preinstall/templates/ansible_git.j2 create mode 100644 kubespray/project/roles/kubernetes/preinstall/templates/chrony.conf.j2 create mode 100644 kubespray/project/roles/kubernetes/preinstall/templates/dhclient_dnsupdate.sh.j2 create mode 100644 kubespray/project/roles/kubernetes/preinstall/templates/dhclient_dnsupdate_rh.sh.j2 create mode 100644 kubespray/project/roles/kubernetes/preinstall/templates/ntp.conf.j2 create mode 100644 kubespray/project/roles/kubernetes/preinstall/templates/resolvconf.j2 create mode 100644 kubespray/project/roles/kubernetes/preinstall/templates/resolved.conf.j2 create mode 100644 kubespray/project/roles/kubernetes/preinstall/vars/amazon.yml create mode 100644 kubespray/project/roles/kubernetes/preinstall/vars/centos.yml create mode 100644 kubespray/project/roles/kubernetes/preinstall/vars/debian-11.yml create mode 100644 kubespray/project/roles/kubernetes/preinstall/vars/debian-12.yml create mode 100644 kubespray/project/roles/kubernetes/preinstall/vars/debian.yml create mode 100644 kubespray/project/roles/kubernetes/preinstall/vars/fedora.yml create mode 100644 kubespray/project/roles/kubernetes/preinstall/vars/redhat.yml create mode 100644 kubespray/project/roles/kubernetes/preinstall/vars/suse.yml create mode 100644 kubespray/project/roles/kubernetes/preinstall/vars/ubuntu.yml create mode 100644 kubespray/project/roles/kubernetes/tokens/files/kube-gen-token.sh create mode 100644 kubespray/project/roles/kubernetes/tokens/tasks/check-tokens.yml create mode 100644 kubespray/project/roles/kubernetes/tokens/tasks/gen_tokens.yml create mode 100644 kubespray/project/roles/kubernetes/tokens/tasks/main.yml create mode 100644 kubespray/project/roles/kubespray-defaults/defaults/main/checksums.yml create mode 100644 kubespray/project/roles/kubespray-defaults/defaults/main/download.yml create mode 100644 kubespray/project/roles/kubespray-defaults/defaults/main/main.yml create mode 100644 kubespray/project/roles/kubespray-defaults/tasks/fallback_ips.yml create mode 100644 kubespray/project/roles/kubespray-defaults/tasks/main.yaml create mode 100644 kubespray/project/roles/kubespray-defaults/tasks/no_proxy.yml create mode 100644 kubespray/project/roles/kubespray-defaults/vars/main.yml create mode 100644 kubespray/project/roles/network_plugin/calico/files/openssl.conf create mode 100644 kubespray/project/roles/network_plugin/calico/handlers/main.yml create mode 100644 kubespray/project/roles/network_plugin/calico/meta/main.yml create mode 100644 kubespray/project/roles/network_plugin/calico/rr/defaults/main.yml create mode 100644 kubespray/project/roles/network_plugin/calico/rr/tasks/main.yml create mode 100644 kubespray/project/roles/network_plugin/calico/rr/tasks/pre.yml create mode 100644 kubespray/project/roles/network_plugin/calico/rr/tasks/update-node.yml create mode 100644 kubespray/project/roles/network_plugin/calico/tasks/calico_apiserver_certs.yml create mode 100644 kubespray/project/roles/network_plugin/calico/tasks/check.yml create mode 100644 kubespray/project/roles/network_plugin/calico/tasks/install.yml create mode 100644 kubespray/project/roles/network_plugin/calico/tasks/main.yml create mode 100644 kubespray/project/roles/network_plugin/calico/tasks/peer_with_calico_rr.yml create mode 100644 kubespray/project/roles/network_plugin/calico/tasks/peer_with_router.yml create mode 100644 kubespray/project/roles/network_plugin/calico/tasks/pre.yml create mode 100644 kubespray/project/roles/network_plugin/calico/tasks/repos.yml create mode 100644 kubespray/project/roles/network_plugin/calico/tasks/reset.yml create mode 100644 kubespray/project/roles/network_plugin/calico/tasks/typha_certs.yml create mode 100644 kubespray/project/roles/network_plugin/calico/templates/calico-apiserver-ns.yml.j2 create mode 100644 kubespray/project/roles/network_plugin/calico/templates/calico-apiserver.yml.j2 create mode 100644 kubespray/project/roles/network_plugin/calico/templates/calico-config.yml.j2 create mode 100644 kubespray/project/roles/network_plugin/calico/templates/calico-cr.yml.j2 create mode 100644 kubespray/project/roles/network_plugin/calico/templates/calico-crb.yml.j2 create mode 100644 kubespray/project/roles/network_plugin/calico/templates/calico-ipamconfig.yml.j2 create mode 100644 kubespray/project/roles/network_plugin/calico/templates/calico-node-sa.yml.j2 create mode 100644 kubespray/project/roles/network_plugin/calico/templates/calico-node.yml.j2 create mode 100644 kubespray/project/roles/network_plugin/calico/templates/calico-typha.yml.j2 create mode 100644 kubespray/project/roles/network_plugin/calico/templates/calicoctl.etcd.sh.j2 create mode 100644 kubespray/project/roles/network_plugin/calico/templates/calicoctl.kdd.sh.j2 create mode 100644 kubespray/project/roles/network_plugin/calico/templates/kubernetes-services-endpoint.yml.j2 create mode 100644 kubespray/project/roles/network_plugin/calico/templates/make-ssl-calico.sh.j2 create mode 100644 kubespray/project/roles/network_plugin/calico/vars/amazon.yml create mode 100644 kubespray/project/roles/network_plugin/calico/vars/centos-9.yml create mode 100644 kubespray/project/roles/network_plugin/calico/vars/debian.yml create mode 100644 kubespray/project/roles/network_plugin/calico/vars/fedora.yml create mode 100644 kubespray/project/roles/network_plugin/calico/vars/opensuse.yml create mode 100644 kubespray/project/roles/network_plugin/calico/vars/redhat-9.yml create mode 100644 kubespray/project/roles/network_plugin/calico/vars/redhat.yml create mode 100644 kubespray/project/roles/network_plugin/calico/vars/rocky-9.yml create mode 100644 kubespray/project/roles/network_plugin/calico_defaults/defaults/main.yml create mode 100644 kubespray/project/roles/network_plugin/cilium/defaults/main.yml create mode 100644 kubespray/project/roles/network_plugin/cilium/tasks/apply.yml create mode 100644 kubespray/project/roles/network_plugin/cilium/tasks/check.yml create mode 100644 kubespray/project/roles/network_plugin/cilium/tasks/install.yml create mode 100644 kubespray/project/roles/network_plugin/cilium/tasks/main.yml create mode 100644 kubespray/project/roles/network_plugin/cilium/tasks/reset.yml create mode 100644 kubespray/project/roles/network_plugin/cilium/tasks/reset_iface.yml create mode 100644 kubespray/project/roles/network_plugin/cilium/templates/000-cilium-portmap.conflist.j2 create mode 100644 kubespray/project/roles/network_plugin/cilium/templates/cilium-operator/cr.yml.j2 create mode 100644 kubespray/project/roles/network_plugin/cilium/templates/cilium-operator/crb.yml.j2 create mode 100644 kubespray/project/roles/network_plugin/cilium/templates/cilium-operator/deploy.yml.j2 create mode 100644 kubespray/project/roles/network_plugin/cilium/templates/cilium-operator/sa.yml.j2 create mode 100644 kubespray/project/roles/network_plugin/cilium/templates/cilium/config.yml.j2 create mode 100644 kubespray/project/roles/network_plugin/cilium/templates/cilium/cr.yml.j2 create mode 100644 kubespray/project/roles/network_plugin/cilium/templates/cilium/crb.yml.j2 create mode 100644 kubespray/project/roles/network_plugin/cilium/templates/cilium/ds.yml.j2 create mode 100644 kubespray/project/roles/network_plugin/cilium/templates/cilium/sa.yml.j2 create mode 100644 kubespray/project/roles/network_plugin/cilium/templates/cilium/secret.yml.j2 create mode 100644 kubespray/project/roles/network_plugin/cilium/templates/hubble/config.yml.j2 create mode 100644 kubespray/project/roles/network_plugin/cilium/templates/hubble/cr.yml.j2 create mode 100644 kubespray/project/roles/network_plugin/cilium/templates/hubble/crb.yml.j2 create mode 100644 kubespray/project/roles/network_plugin/cilium/templates/hubble/cronjob.yml.j2 create mode 100644 kubespray/project/roles/network_plugin/cilium/templates/hubble/deploy.yml.j2 create mode 100644 kubespray/project/roles/network_plugin/cilium/templates/hubble/job.yml.j2 create mode 100644 kubespray/project/roles/network_plugin/cilium/templates/hubble/sa.yml.j2 create mode 100644 kubespray/project/roles/network_plugin/cilium/templates/hubble/service.yml.j2 create mode 100644 kubespray/project/roles/network_plugin/cni/defaults/main.yml create mode 100644 kubespray/project/roles/network_plugin/cni/tasks/main.yml create mode 100644 kubespray/project/roles/network_plugin/custom_cni/defaults/main.yml create mode 100644 kubespray/project/roles/network_plugin/custom_cni/meta/main.yml create mode 100644 kubespray/project/roles/network_plugin/custom_cni/tasks/main.yml create mode 100644 kubespray/project/roles/network_plugin/flannel/defaults/main.yml create mode 100644 kubespray/project/roles/network_plugin/flannel/meta/main.yml create mode 100644 kubespray/project/roles/network_plugin/flannel/tasks/main.yml create mode 100644 kubespray/project/roles/network_plugin/flannel/tasks/reset.yml create mode 100644 kubespray/project/roles/network_plugin/flannel/templates/cni-flannel-rbac.yml.j2 create mode 100644 kubespray/project/roles/network_plugin/flannel/templates/cni-flannel.yml.j2 create mode 100644 kubespray/project/roles/network_plugin/kube-ovn/OWNERS create mode 100644 kubespray/project/roles/network_plugin/kube-ovn/defaults/main.yml create mode 100644 kubespray/project/roles/network_plugin/kube-ovn/tasks/main.yml create mode 100644 kubespray/project/roles/network_plugin/kube-ovn/templates/cni-kube-ovn-crd.yml.j2 create mode 100644 kubespray/project/roles/network_plugin/kube-ovn/templates/cni-kube-ovn.yml.j2 create mode 100644 kubespray/project/roles/network_plugin/kube-ovn/templates/cni-ovn.yml.j2 create mode 100644 kubespray/project/roles/network_plugin/kube-router/OWNERS create mode 100644 kubespray/project/roles/network_plugin/kube-router/defaults/main.yml create mode 100644 kubespray/project/roles/network_plugin/kube-router/handlers/main.yml create mode 100644 kubespray/project/roles/network_plugin/kube-router/meta/main.yml create mode 100644 kubespray/project/roles/network_plugin/kube-router/tasks/annotate.yml create mode 100644 kubespray/project/roles/network_plugin/kube-router/tasks/main.yml create mode 100644 kubespray/project/roles/network_plugin/kube-router/tasks/reset.yml create mode 100644 kubespray/project/roles/network_plugin/kube-router/templates/cni-conf.json.j2 create mode 100644 kubespray/project/roles/network_plugin/kube-router/templates/kube-router.yml.j2 create mode 100644 kubespray/project/roles/network_plugin/kube-router/templates/kubeconfig.yml.j2 create mode 100644 kubespray/project/roles/network_plugin/macvlan/OWNERS create mode 100644 kubespray/project/roles/network_plugin/macvlan/defaults/main.yml create mode 100644 kubespray/project/roles/network_plugin/macvlan/files/ifdown-local create mode 100755 kubespray/project/roles/network_plugin/macvlan/files/ifdown-macvlan create mode 100755 kubespray/project/roles/network_plugin/macvlan/files/ifup-local create mode 100755 kubespray/project/roles/network_plugin/macvlan/files/ifup-macvlan create mode 100644 kubespray/project/roles/network_plugin/macvlan/handlers/main.yml create mode 100644 kubespray/project/roles/network_plugin/macvlan/meta/main.yml create mode 100644 kubespray/project/roles/network_plugin/macvlan/tasks/main.yml create mode 100644 kubespray/project/roles/network_plugin/macvlan/templates/10-macvlan.conf.j2 create mode 100644 kubespray/project/roles/network_plugin/macvlan/templates/99-loopback.conf.j2 create mode 100644 kubespray/project/roles/network_plugin/macvlan/templates/centos-network-macvlan.cfg.j2 create mode 100644 kubespray/project/roles/network_plugin/macvlan/templates/centos-postdown-macvlan.cfg.j2 create mode 100644 kubespray/project/roles/network_plugin/macvlan/templates/centos-postup-macvlan.cfg.j2 create mode 100644 kubespray/project/roles/network_plugin/macvlan/templates/centos-routes-macvlan.cfg.j2 create mode 100644 kubespray/project/roles/network_plugin/macvlan/templates/coreos-device-macvlan.cfg.j2 create mode 100644 kubespray/project/roles/network_plugin/macvlan/templates/coreos-interface-macvlan.cfg.j2 create mode 100644 kubespray/project/roles/network_plugin/macvlan/templates/coreos-network-macvlan.cfg.j2 create mode 100644 kubespray/project/roles/network_plugin/macvlan/templates/coreos-service-nat_ouside.j2 create mode 100644 kubespray/project/roles/network_plugin/macvlan/templates/debian-network-macvlan.cfg.j2 create mode 100644 kubespray/project/roles/network_plugin/meta/main.yml create mode 100644 kubespray/project/roles/network_plugin/multus/defaults/main.yml create mode 100644 kubespray/project/roles/network_plugin/multus/files/multus-clusterrole.yml create mode 100644 kubespray/project/roles/network_plugin/multus/files/multus-clusterrolebinding.yml create mode 100644 kubespray/project/roles/network_plugin/multus/files/multus-crd.yml create mode 100644 kubespray/project/roles/network_plugin/multus/files/multus-serviceaccount.yml create mode 100644 kubespray/project/roles/network_plugin/multus/meta/main.yml create mode 100644 kubespray/project/roles/network_plugin/multus/tasks/main.yml create mode 100644 kubespray/project/roles/network_plugin/multus/templates/multus-daemonset.yml.j2 create mode 100644 kubespray/project/roles/network_plugin/ovn4nfv/tasks/main.yml create mode 100644 kubespray/project/roles/network_plugin/weave/defaults/main.yml create mode 100644 kubespray/project/roles/network_plugin/weave/meta/main.yml create mode 100644 kubespray/project/roles/network_plugin/weave/tasks/main.yml create mode 100644 kubespray/project/roles/network_plugin/weave/templates/10-weave.conflist.j2 create mode 100644 kubespray/project/roles/network_plugin/weave/templates/weave-net.yml.j2 create mode 100644 kubespray/project/roles/recover_control_plane/OWNERS create mode 100644 kubespray/project/roles/recover_control_plane/control-plane/defaults/main.yml create mode 100644 kubespray/project/roles/recover_control_plane/control-plane/tasks/main.yml create mode 100644 kubespray/project/roles/recover_control_plane/etcd/tasks/main.yml create mode 100644 kubespray/project/roles/recover_control_plane/etcd/tasks/recover_lost_quorum.yml create mode 100644 kubespray/project/roles/recover_control_plane/post-recover/tasks/main.yml create mode 100644 kubespray/project/roles/remove-node/post-remove/defaults/main.yml create mode 100644 kubespray/project/roles/remove-node/post-remove/tasks/main.yml create mode 100644 kubespray/project/roles/remove-node/pre-remove/defaults/main.yml create mode 100644 kubespray/project/roles/remove-node/pre-remove/tasks/main.yml create mode 100644 kubespray/project/roles/remove-node/remove-etcd-node/tasks/main.yml create mode 100644 kubespray/project/roles/reset/defaults/main.yml create mode 100644 kubespray/project/roles/reset/tasks/main.yml create mode 100644 kubespray/project/roles/upgrade/post-upgrade/defaults/main.yml create mode 100644 kubespray/project/roles/upgrade/post-upgrade/tasks/main.yml create mode 100644 kubespray/project/roles/upgrade/pre-upgrade/defaults/main.yml create mode 100644 kubespray/project/roles/upgrade/pre-upgrade/tasks/main.yml create mode 100644 kubespray/project/roles/upgrade/system-upgrade/tasks/apt.yml create mode 100644 kubespray/project/roles/upgrade/system-upgrade/tasks/main.yml create mode 100644 kubespray/project/roles/upgrade/system-upgrade/tasks/yum.yml create mode 100644 kubespray/project/roles/win_nodes/kubernetes_patch/defaults/main.yml create mode 100644 kubespray/project/roles/win_nodes/kubernetes_patch/tasks/main.yml create mode 100644 kubespray/project/scale.yml create mode 100644 kubespray/project/upgrade_cluster.yml create mode 100644 pkg/config/ansible_playbook_config.go create mode 100644 pkg/config/config.go create mode 100644 pkg/config/modules_additional.go create mode 100644 pkg/config/modules_cicd.go create mode 100644 pkg/config/modules_observability.go create mode 100644 pkg/config/modules_registry.go create mode 100644 pkg/config/modules_secrets_storage.go create mode 100644 pkg/config/orchestrator.go create mode 100644 pkg/config/utility.go create mode 100644 pkg/kubespray/cluster.go create mode 100644 pkg/kubespray/utility.go create mode 100644 pkg/resources/resources.go create mode 100644 pkg/resources/templates/helm-apps/releases/additional-modules/cert-manager.yml.tmpl create mode 100644 pkg/resources/templates/helm-apps/releases/additional-modules/ingress-nginx.yml.tmpl create mode 100644 pkg/resources/templates/helm-apps/releases/additional-modules/longhorn.yml.tmpl create mode 100644 pkg/resources/templates/helm-apps/releases/additional-modules/secrets-store-csi-driver.yml.tmpl create mode 100644 pkg/resources/templates/helm-apps/releases/cicd/argo-cd-ingress.yml.tmpl create mode 100644 pkg/resources/templates/helm-apps/releases/cicd/argo-cd.yml.tmpl create mode 100644 pkg/resources/templates/helm-apps/releases/cicd/argo-rollouts.yml.tmpl create mode 100644 pkg/resources/templates/helm-apps/releases/cicd/keel.yml.tmpl create mode 100644 pkg/resources/templates/helm-apps/releases/observability/fluent-operator.yml.tmpl create mode 100644 pkg/resources/templates/helm-apps/releases/observability/loki.yml.tmpl create mode 100644 pkg/resources/templates/helm-apps/releases/observability/observability.yml.tmpl create mode 100644 pkg/resources/templates/helm-apps/releases/observability/opentelemetry-operator.yml.tmpl create mode 100644 pkg/resources/templates/helm-apps/releases/observability/tempo.yml.tmpl create mode 100644 pkg/resources/templates/helm-apps/releases/registry/harbor-certificate-generator.yml.tmpl create mode 100644 pkg/resources/templates/helm-apps/releases/registry/harbor.yml.tmpl create mode 100644 pkg/resources/templates/helm-apps/releases/secrets-storage/vault.yml.tmpl create mode 100644 pkg/resources/templates/helm-apps/repositories/repositories.yml.tmpl create mode 100644 pkg/resources/templates/kubespray/inventory/group_vars/all.yml.tmpl create mode 100644 pkg/resources/templates/kubespray/inventory/group_vars/k8s_cluster/addons.yml.tmpl create mode 100644 pkg/resources/templates/kubespray/inventory/group_vars/k8s_cluster/k8s-cluster.yml.tmpl create mode 100644 pkg/resources/templates/kubespray/inventory/hosts.tmpl create mode 100644 pkg/templates/funcs.go create mode 100644 pkg/templates/templates.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7f4949c --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +*.DS_Store +*.vscode +*.env +*artifacts* +*k8s-admin* \ No newline at end of file diff --git a/cmd/main/main.go b/cmd/main/main.go new file mode 100644 index 0000000..fad25a0 --- /dev/null +++ b/cmd/main/main.go @@ -0,0 +1,45 @@ +package main + +import ( + "flag" + "kube-forge/pkg/config" + "kube-forge/pkg/kubespray" + "kube-forge/pkg/templates" + "os" +) + +func parseArgs() (string, string, string, string) { + var password string + var configPath string + var dataDir string + var tags string + flag.StringVar(&password, "p", "", "Password to access hosts") + flag.StringVar(&configPath, "c", "/etc/kube-forge/config.yaml", "Path to config file") + flag.StringVar(&dataDir, "d", "/var/lib/kube-forge", "Path to data dir") + flag.StringVar(&tags, "t", "", "List of tags to apply (ex: ingress,dns)") + flag.Parse() + return password, configPath, dataDir, tags +} + +func main() { + password, configPath, dataDir, tags := parseArgs() + config := config.CreateConfig(configPath, dataDir, password) + + repositories, releases := templates.GetHelmAppsConfigData() + config.Repositories = repositories + config.Releases = releases + + templates.ApplyTemplates() + + for _, cmd := range os.Args { + switch cmd { + case "apply": + kubespray.InstallCluster(tags) + kubespray.CopyK8SAdminConfig("k8s-admin.conf") + case "reset": + kubespray.ResetCluster() + case "upgrade": + kubespray.UpgradeCluster(tags) + } + } +} diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..2946df7 --- /dev/null +++ b/config.yaml @@ -0,0 +1,277 @@ +credentials: + user: sre-admin + ask_sudo_password: false + private_key_file: /home/sre-admin/.ssh/id_rsa + +hosts: + - hostname: vbox-prod-k8s-master-01 + ip: "10.250.50.22" + roles: [control_plane, etcd] + + # # Optional creds for each host + # user: user + # password: nF5S8nuKi87Dh42Jnjik + + - hostname: vbox-prod-k8s-slave-01 + ip: "10.250.50.23" + roles: [node] + + - hostname: vbox-prod-k8s-slave-02 + ip: "10.250.50.21" + roles: [node] + +default_registry: "" + +orchestrator: + version: v1.29.0 + cluster_name: k8s-cluster.local + # bin_dir: /usr/local/bin + # sysctl_file_path: /etc/sysctl.d/99-sysctl.conf + loadbalancer_apiserver_port: 6443 + dns: + servers: + - 8.8.8.8 + - 8.8.4.4 + disable_host_nameservers: false + # cloud_provider: "" # 'gce', 'aws', 'azure', 'openstack', 'vsphere', 'oci', or 'external' + # external_cloud_provider: "" # 'openstack', 'vsphere' and 'hcloud' + + container_engine: + type: containerd # docker, crio and containerd + install: true + + ping_access_ip: true + auto_renew_certificates: true + event_ttl: "1h0m0s" + pod_security_policy_enabled: true + + network: + plugin: calico # cilium, calico, kube-ovn, weave or flannel + # service_addresses: 10.233.0.0/18 + # pods_subnet: 10.233.64.0/18 + +modules: + admin_password: changeit + + # additional_repositories: + # - name: argo-helm + # url: "https://argoproj.github.io/argo-helm" + + observability: + enabled: true + + logging: + enabled: true + + loki: + persistence: + storage_size: 10Gi + retention: 168h + + events: + enabled: true + cron: + schedule: "*/2 * * * *" + + tracing: + enabled: true + + tempo: + retention: 24h + listen_port: 3100 + + persistence: + storage_size: 10Gi + + tempo_query: + listen_port: 16686 + + monitoring: + enabled: true + + prometheus: + scrape_interval: 15s + persistence: + storage_size: 3Gi + retention: 7d + + alert_manager: + enabled: false + + # route: + # receiver: kvazaric_notifications + # group_wait: 1s + # repeat_interval: 180m + + # routes: + # - receiver: "kvazaric_notifications" + # group_by: ["alertname"] + # group_wait: 1s + # repeat_interval: 8737h + # matchers: + # - fireOnce=true + # receivers: + # - name: kvazaric_notifications + # telegram_configs: + # - send_resolved: true + # api_url: https://api.telegram.org + # bot_token: 6364937365:AAHcJKc-McN11gBWuQqoM87zr18eK0VKX9I + # chat_id: -968770779 + # parse_mode: HTML + # message: '{{ template "telegram-message" . }}' + # http_config: + # follow_redirects: true + # enable_http2: false + blackbox: + enabled: true + additional_modules: | + + kube_state: + enabled: true + + node: + enabled: true + + visualization: + enabled: true + + grafana: + enabled: true + expose: + type: ingress + + domain: grafana.disk.lt.t1.cloud + + tls: + enabled: true + + persistence: + storage_size: 2Gi + + cicd: + enabled: true + + argo_cd: + expose: + type: ingress + domain: argocd.disk.lt.t1.cloud + + tls: + enabled: true + + repositories: + {} + # harbor-helm: + # enableOCI: "true" + # type: helm + # name: helm-mountbit + # url: harbor.disk.t1.cloud/mountbit-helm + # username: username + # password: password + rbac: + additional_policies: | + p, developer, applications, *, */*, allow + + p, developer, applications, *, */*logging*, deny + p, developer, repositories, get, *, allow + p, developer, projects, get, *, allow + + p, developer, logs, get, *, allow + + p, guest, applications, get, */*, allow + p, guest, projects, get, *, allow + + ha: + enabled: false + autoscaling: false + + rollouts: + enabled: true + + updates_operator: + enabled: true + + secrets_storage: + enabled: true + expose: + type: ingress # NodePort + domain: vault.disk.lt.t1.cloud + node_port: 30004 + + tls: + enabled: true + + # requires secrets_store_csi_driver in additional modules + csi_integration: + enabled: true + image: hashicorp/vault-csi-provider + version: 1.4.1 + agent: + image: hashicorp/vault + version: 1.15.6 + injector: + image: hashicorp/vault-k8s + version: 1.3.1 + server: + image: "hashicorp/vault" + version: 1.15.6 + persistence: + size: 10Gi + + registry: + enabled: true + expose: + type: ingress # ingress or NodePort + # if expose_type is "ingress" + domain: harbor.disk.lt.t1.cloud + + # if expose_type is "NodePort" + node_port_http: 30002 + node_port_https: 30003 + tls: + enabled: true + + persistence: + registry_size: 10Gi + jobservice_size: 1Gi + database_size: 2Gi + redis_size: 1Gi + trivy_size: 5Gi + + defaultProjects: + - name: harbor-helm + public: false + enabled_scanner: true + + additional: + cert_manager: + install: true + dns_servers: + - "1.1.1.1" + - "8.8.8.8" + account_email: reversstorm@gmail.com + + ingress: + type: nginx # nginx + install: true + insecure_port: 80 + secure_port: 443 + ingress_class_name: nginx + host_network: true + + # load_balancer: + # type: metallb + # install: true + + storage: + local_path_provisioner: + storage_class_name: local-path + reclaim_policy: Delete + + longhorn: + enabled: true + + secrets_store_csi_driver: + enabled: true + + ## TODO: ceph diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..9ee3150 --- /dev/null +++ b/go.mod @@ -0,0 +1,43 @@ +module kube-forge + +go 1.22 + +toolchain go1.22.2 + +require ( + github.com/BurntSushi/toml v1.2.1 + github.com/Masterminds/sprig/v3 v3.2.3 + github.com/apenella/go-ansible v1.3.0 + github.com/ilyakaznacheev/cleanenv v1.5.0 + golang.org/x/crypto v0.22.0 + sigs.k8s.io/yaml v1.4.0 +) + +require ( + github.com/Masterminds/goutils v1.1.1 // indirect + github.com/Masterminds/semver/v3 v3.2.0 // indirect + github.com/apenella/go-ansible/v2 v2.0.0 // indirect + github.com/apenella/go-common-utils/data v0.0.0-20220913191136-86daaa87e7df // indirect + github.com/apenella/go-common-utils/error v0.0.0-20220913191136-86daaa87e7df // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/fatih/color v1.16.0 // indirect + github.com/google/uuid v1.1.1 // indirect + github.com/huandu/xstrings v1.3.3 // indirect + github.com/imdario/mergo v0.3.11 // indirect + github.com/joho/godotenv v1.5.1 // indirect + github.com/kr/pretty v0.2.1 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mitchellh/copystructure v1.0.0 // indirect + github.com/mitchellh/reflectwalk v1.0.0 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/shopspring/decimal v1.2.0 // indirect + github.com/spf13/cast v1.3.1 // indirect + github.com/stretchr/objx v0.5.2 // indirect + github.com/stretchr/testify v1.9.0 // indirect + golang.org/x/sys v0.19.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..ab179bf --- /dev/null +++ b/go.sum @@ -0,0 +1,119 @@ +github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= +github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= +github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= +github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= +github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= +github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA= +github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM= +github.com/apenella/go-ansible v1.3.0 h1:INNaVzrgQMNIO9Yvnly6lA5KfN/pY3m1Z37gpmlUzPM= +github.com/apenella/go-ansible v1.3.0/go.mod h1:qiLWolQqLkCoqFlPvdsKhxk7O0qjIBCPNhUKpH/vjaU= +github.com/apenella/go-ansible/v2 v2.0.0 h1:9aZkU8dQ2fVITYX5V1CiQ3VX4PnNPBTgv31wJFd2Ito= +github.com/apenella/go-ansible/v2 v2.0.0/go.mod h1:ifhiX4d0bpynb8yhdzLTmGl/38HqTYr/26PfjB1enXQ= +github.com/apenella/go-common-utils/data v0.0.0-20220913191136-86daaa87e7df h1:sEikY2P+NZK/7VZUwIsnXIGElhsuFDSxh1bZYwHxdcI= +github.com/apenella/go-common-utils/data v0.0.0-20220913191136-86daaa87e7df/go.mod h1:cLVL6GjUiKG/WyBzX+KD6h/XRV/HnNZIZbMNNiBgQ9o= +github.com/apenella/go-common-utils/error v0.0.0-20220913191136-86daaa87e7df h1:SvlYbjlsSQDS7hbVT1h012/zdgvcwWJ+Yd9XRiiY/8s= +github.com/apenella/go-common-utils/error v0.0.0-20220913191136-86daaa87e7df/go.mod h1:+3dyIlHX350xJIUIffwMLswZXU+N2FwDE05VuKqxYdw= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= +github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8bk= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/huandu/xstrings v1.3.3 h1:/Gcsuc1x8JVbJ9/rlye4xZnVAbEkGauT8lbebqcQws4= +github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= +github.com/ilyakaznacheev/cleanenv v1.5.0 h1:0VNZXggJE2OYdXE87bfSSwGxeiGt9moSR2lOrsHHvr4= +github.com/ilyakaznacheev/cleanenv v1.5.0/go.mod h1:a5aDzaJrLCQZsazHol1w8InnDcOX0OColm64SlIi6gk= +github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA= +github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ= +github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= +github.com/mitchellh/reflectwalk v1.0.0 h1:9D+8oIskB4VJBN5SFlmc27fSlIBZaov1Wpk/IfikLNY= +github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/sosedoff/ansible-vault-go v0.2.0 h1:XqkBdqbXgTuFQ++NdrZvSdUTNozeb6S3V5x7FVs17vg= +github.com/sosedoff/ansible-vault-go v0.2.0/go.mod h1:wMU54HNJfY0n0KIgbpA9m15NBfaUDlJrAsaZp0FwzkI= +github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= +github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= +golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3 h1:slmdOY3vp8a7KQbHkL+FLbvbkgMqmXojpFUO/jENuqQ= +olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3/go.mod h1:oVgVk4OWVDi43qWBEyGhXgYxt7+ED4iYNpTngSLX2Iw= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/kubespray/env/cmdline b/kubespray/env/cmdline new file mode 100644 index 0000000..3a4cee0 --- /dev/null +++ b/kubespray/env/cmdline @@ -0,0 +1 @@ +-b -v --become \ No newline at end of file diff --git a/kubespray/env/extravars b/kubespray/env/extravars new file mode 100644 index 0000000..ea5601b --- /dev/null +++ b/kubespray/env/extravars @@ -0,0 +1,2 @@ +--- +ansible_sudo_pass: nF5S8nuKi87Dh42Jnjik \ No newline at end of file diff --git a/kubespray/inventory/credentials/kubeadm_certificate_key.creds b/kubespray/inventory/credentials/kubeadm_certificate_key.creds new file mode 100644 index 0000000..876b675 --- /dev/null +++ b/kubespray/inventory/credentials/kubeadm_certificate_key.creds @@ -0,0 +1 @@ +4EE863bb8CdA1331D43B4CBcB274dd2267fDCdaA97Dcfa5E7E6BBc31b10E9695 diff --git a/kubespray/inventory/group_vars/all.yml b/kubespray/inventory/group_vars/all.yml new file mode 100644 index 0000000..bc35886 --- /dev/null +++ b/kubespray/inventory/group_vars/all.yml @@ -0,0 +1,129 @@ +--- +bin_dir: /usr/local/bin + +## The access_ip variable is used to define how other nodes should access +## the node. This is used in flannel to allow other flannel nodes to see +## this node for example. The access_ip is really useful AWS and Google +## environments where the nodes are accessed remotely by the "public" ip, +## but don't know about that address themselves. +# access_ip: 1.1.1.1 + + +## External LB example config +## apiserver_loadbalancer_domain_name: "elb.some.domain" +# loadbalancer_apiserver: +# address: 1.2.3.4 +# port: 1234 + +## Internal loadbalancers for apiservers +# loadbalancer_apiserver_localhost: true +# valid options are "nginx" or "haproxy" +# loadbalancer_apiserver_type: nginx # valid values "nginx" or "haproxy" + +## Local loadbalancer should use this port +## And must be set port 6443 +loadbalancer_apiserver_port: 6443 + +## If loadbalancer_apiserver_healthcheck_port variable defined, enables proxy liveness check for nginx. +loadbalancer_apiserver_healthcheck_port: 8081 + +disable_host_nameservers: false + +upstream_dns_servers: + - 8.8.8.8 + - 8.8.4.4 + +## There are some changes specific to the cloud providers +## for instance we need to encapsulate packets with some network plugins +## If set the possible values are either 'gce', 'aws', 'azure', 'openstack', 'vsphere', 'oci', or 'external' +## When openstack is used make sure to source in the openstack credentials +## like you would do when using openstack-client before starting the playbook. + +## When cloud_provider is set to 'external', you can set the cloud controller to deploy +## Supported cloud controllers are: 'openstack', 'vsphere' and 'hcloud' +## When openstack or vsphere are used make sure to source in the required fields + +## Set these proxy values in order to update package manager and docker daemon to use proxies +# http_proxy: "" +# https_proxy: "" + +## Refer to roles/kubespray-defaults/defaults/main.yml before modifying no_proxy +# no_proxy: "" + +## Some problems may occur when downloading files over https proxy due to ansible bug +## https://github.com/ansible/ansible/issues/32750. Set this variable to False to disable +## SSL validation of get_url module. Note that kubespray will still be performing checksum validation. +# download_validate_certs: False + +## If you need exclude all cluster nodes from proxy and other resources, add other resources here. +# additional_no_proxy: "" + +## If you need to disable proxying of os package repositories but are still behind an http_proxy set +## skip_http_proxy_on_os_packages to true +## This will cause kubespray not to set proxy environment in /etc/yum.conf for centos and in /etc/apt/apt.conf for debian/ubuntu +## Special information for debian/ubuntu - you have to set the no_proxy variable, then apt package will install from your source of wish +# skip_http_proxy_on_os_packages: false + +## Since workers are included in the no_proxy variable by default, docker engine will be restarted on all nodes (all +## pods will restart) when adding or removing workers. To override this behaviour by only including master nodes in the +## no_proxy variable, set below to true: +no_proxy_exclude_workers: false + +## Certificate Management +## This setting determines whether certs are generated via scripts. +## Chose 'none' if you provide your own certificates. +## Option is "script", "none" +# cert_management: script + +## Set to true to allow pre-checks to fail and continue deployment +# ignore_assert_errors: false + +## The read-only port for the Kubelet to serve on with no authentication/authorization. Uncomment to enable. +# kube_read_only_port: 10255 + +## Set true to download and cache container +# download_container: true + +## Deploy container engine +# Set false if you want to deploy container engine manually. +deploy_container_engine: true + +## Red Hat Enterprise Linux subscription registration +## Add either RHEL subscription Username/Password or Organization ID/Activation Key combination +## Update RHEL subscription purpose usage, role and SLA if necessary +# rh_subscription_username: "" +# rh_subscription_password: "" +# rh_subscription_org_id: "" +# rh_subscription_activation_key: "" +# rh_subscription_usage: "Development" +# rh_subscription_role: "Red Hat Enterprise Server" +# rh_subscription_sla: "Self-Support" + +## Check if access_ip responds to ping. Set false if your firewall blocks ICMP. +ping_access_ip: true + +# sysctl_file_path to add sysctl conf to +sysctl_file_path: /etc/sysctl.d/99-sysctl.conf + +## Variables for webhook token auth https://kubernetes.io/docs/reference/access-authn-authz/authentication/#webhook-token-authentication +kube_webhook_token_auth: false +kube_webhook_token_auth_url_skip_tls_verify: false +# kube_webhook_token_auth_url: https://... +## base64-encoded string of the webhook's CA certificate +# kube_webhook_token_auth_ca_data: "LS0t..." + +## NTP Settings +# Start the ntpd or chrony service and enable it at system boot. +ntp_enabled: false +ntp_manage_config: false +ntp_servers: + - "0.pool.ntp.org iburst" + - "1.pool.ntp.org iburst" + - "2.pool.ntp.org iburst" + - "3.pool.ntp.org iburst" + +## Used to control no_log attribute +unsafe_show_logs: false + +## If enabled it will allow kubespray to attempt setup even if the distribution is not supported. For unsupported distributions this can lead to unexpected failures in some cases. +allow_unsupported_distribution_setup: false diff --git a/kubespray/inventory/group_vars/k8s_cluster/addons.yml b/kubespray/inventory/group_vars/k8s_cluster/addons.yml new file mode 100644 index 0000000..1af9fa5 --- /dev/null +++ b/kubespray/inventory/group_vars/k8s_cluster/addons.yml @@ -0,0 +1,3318 @@ +--- +dashboard_enabled: false + +helm_enabled: true + +registry_enabled: false + +metrics_server_enabled: false + +# Rancher Local Path Provisioner +local_path_provisioner_enabled: true +# local_path_provisioner_namespace: "local-path-storage" +local_path_provisioner_storage_class: local-path +local_path_provisioner_reclaim_policy: Delete +# local_path_provisioner_claim_root: /opt/local-path-provisioner/ +# local_path_provisioner_debug: false +# local_path_provisioner_image_repo: "rancher/local-path-provisioner" +# local_path_provisioner_image_tag: "v0.0.23" +# local_path_provisioner_helper_image_repo: "busybox" +# local_path_provisioner_helper_image_tag: "latest" + +# Local volume provisioner deployment +local_volume_provisioner_enabled: false +# local_volume_provisioner_namespace: kube-system +# local_volume_provisioner_nodelabels: +# - kubernetes.io/hostname +# - topology.kubernetes.io/region +# - topology.kubernetes.io/zone +# local_volume_provisioner_storage_classes: +# local-storage: +# host_dir: /mnt/disks +# mount_dir: /mnt/disks +# volume_mode: Filesystem +# fs_type: ext4 +# fast-disks: +# host_dir: /mnt/fast-disks +# mount_dir: /mnt/fast-disks +# block_cleaner_command: +# - "/scripts/shred.sh" +# - "2" +# volume_mode: Filesystem +# fs_type: ext4 +# local_volume_provisioner_tolerations: +# - effect: NoSchedule +# operator: Exists + +# CSI Volume Snapshot Controller deployment, set this to true if your CSI is able to manage snapshots +# currently, setting cinder_csi_enabled=true would automatically enable the snapshot controller +# Longhorn is an extenal CSI that would also require setting this to true but it is not included in kubespray +# csi_snapshot_controller_enabled: false +# csi snapshot namespace +# snapshot_controller_namespace: kube-system + +# CephFS provisioner deployment +cephfs_provisioner_enabled: false +# cephfs_provisioner_namespace: "cephfs-provisioner" +# cephfs_provisioner_cluster: ceph +# cephfs_provisioner_monitors: "172.24.0.1:6789,172.24.0.2:6789,172.24.0.3:6789" +# cephfs_provisioner_admin_id: admin +# cephfs_provisioner_secret: secret +# cephfs_provisioner_storage_class: cephfs +# cephfs_provisioner_reclaim_policy: Delete +# cephfs_provisioner_claim_root: /volumes +# cephfs_provisioner_deterministic_names: true + +# RBD provisioner deployment +rbd_provisioner_enabled: false +# rbd_provisioner_namespace: rbd-provisioner +# rbd_provisioner_replicas: 2 +# rbd_provisioner_monitors: "172.24.0.1:6789,172.24.0.2:6789,172.24.0.3:6789" +# rbd_provisioner_pool: kube +# rbd_provisioner_admin_id: admin +# rbd_provisioner_secret_name: ceph-secret-admin +# rbd_provisioner_secret: ceph-key-admin +# rbd_provisioner_user_id: kube +# rbd_provisioner_user_secret_name: ceph-secret-user +# rbd_provisioner_user_secret: ceph-key-user +# rbd_provisioner_user_secret_namespace: rbd-provisioner +# rbd_provisioner_fs_type: ext4 +# rbd_provisioner_image_format: "2" +# rbd_provisioner_image_features: layering +# rbd_provisioner_storage_class: rbd +# rbd_provisioner_reclaim_policy: Delete + +# Nginx ingress controller deployment +ingress_nginx_enabled: true +ingress_nginx_host_network: true +ingress_publish_status_address: "" +# ingress_nginx_nodeselector: +# kubernetes.io/os: "linux" +# ingress_nginx_tolerations: +# - key: "node-role.kubernetes.io/master" +# operator: "Equal" +# value: "" +# effect: "NoSchedule" +# - key: "node-role.kubernetes.io/control-plane" +# operator: "Equal" +# value: "" +# effect: "NoSchedule" +# ingress_nginx_namespace: "ingress-nginx" +ingress_nginx_insecure_port: 80 +ingress_nginx_secure_port: 443 +# ingress_nginx_configmap: +# map-hash-bucket-size: "128" +# ssl-protocols: "TLSv1.2 TLSv1.3" +# ingress_nginx_configmap_tcp_services: +# 9000: "default/example-go:8080" +# ingress_nginx_configmap_udp_services: +# 53: "kube-system/coredns:53" +# ingress_nginx_extra_args: +# - --default-ssl-certificate=default/foo-tls +# ingress_nginx_termination_grace_period_seconds: 300 +ingress_nginx_class: nginx + +# ALB ingress controller deployment +ingress_alb_enabled: false +# alb_ingress_aws_region: "us-east-1" +# alb_ingress_restrict_scheme: "false" +# Enables logging on all outbound requests sent to the AWS API. +# If logging is desired, set to true. +# alb_ingress_aws_debug: "false" + +# Cert manager deployment +cert_manager_enabled: true +# cert_manager_namespace: "cert-manager" +# cert_manager_tolerations: +# - key: node-role.kubernetes.io/master +# effect: NoSchedule +# - key: node-role.kubernetes.io/control-plane +# effect: NoSchedule +# cert_manager_affinity: +# nodeAffinity: +# preferredDuringSchedulingIgnoredDuringExecution: +# - weight: 100 +# preference: +# matchExpressions: +# - key: node-role.kubernetes.io/control-plane +# operator: In +# values: +# - "" +# cert_manager_nodeselector: +# kubernetes.io/os: "linux" + +# cert_manager_trusted_internal_ca: | +# -----BEGIN CERTIFICATE----- +# [REPLACE with your CA certificate] +# -----END CERTIFICATE----- +# cert_manager_leader_election_namespace: kube-system + +# cert_manager_dns_policy: "ClusterFirst" +cert_manager_dns_config: + nameservers: + - "1.1.1.1" + - "8.8.8.8" + +# MetalLB deployment +metallb_enabled: false +metallb_speaker_enabled: "{{ metallb_enabled }}" +# metallb_speaker_nodeselector: +# kubernetes.io/os: "linux" +# metallb_controller_nodeselector: +# kubernetes.io/os: "linux" +# metallb_speaker_tolerations: +# - key: "node-role.kubernetes.io/master" +# operator: "Equal" +# value: "" +# effect: "NoSchedule" +# - key: "node-role.kubernetes.io/control-plane" +# operator: "Equal" +# value: "" +# effect: "NoSchedule" +# metallb_controller_tolerations: +# - key: "node-role.kubernetes.io/master" +# operator: "Equal" +# value: "" +# effect: "NoSchedule" +# - key: "node-role.kubernetes.io/control-plane" +# operator: "Equal" +# value: "" +# effect: "NoSchedule" +# metallb_version: v0.13.9 +# metallb_protocol: "layer2" +# metallb_port: "7472" +# metallb_memberlist_port: "7946" +# metallb_config: +# address_pools: +# primary: +# ip_range: +# - 10.5.0.0/16 +# auto_assign: true +# pool1: +# ip_range: +# - 10.6.0.0/16 +# auto_assign: true +# pool2: +# ip_range: +# - 10.10.0.0/16 +# auto_assign: true +# layer2: +# - primary +# layer3: +# defaults: +# peer_port: 179 +# hold_time: 120s +# communities: +# vpn-only: "1234:1" +# NO_ADVERTISE: "65535:65282" +# metallb_peers: +# peer1: +# peer_address: 10.6.0.1 +# peer_asn: 64512 +# my_asn: 4200000000 +# communities: +# - vpn-only +# address_pool: +# - pool1 +# peer2: +# peer_address: 10.10.0.1 +# peer_asn: 64513 +# my_asn: 4200000000 +# communities: +# - NO_ADVERTISE +# address_pool: +# - pool2 + +argocd_enabled: false + +# The plugin manager for kubectl +krew_enabled: false +krew_root_dir: "/usr/local/krew" + + +######################################## +# Helm apps configuration +######################################## +repositories: + - name: kube-forge + url: "https://git.kvazaric.ru/api/v4/projects/41/packages/helm/stable" + + +releases: + + - name: longhorn + namespace: longhorn-system + create_namespace: true + chart_ref: kube-forge/longhorn + chart_version: 1.6.1 + values: + global: + cattle: + # -- Default system registry. + systemDefaultRegistry: "" + windowsCluster: + # -- Setting that allows Longhorn to run on a Rancher Windows cluster. + enabled: false + # -- Toleration for Linux nodes that can run user-deployed Longhorn components. + tolerations: + - key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" + # -- Node selector for Linux nodes that can run user-deployed Longhorn components. + nodeSelector: + kubernetes.io/os: "linux" + defaultSetting: + # -- Toleration for system-managed Longhorn components. + taintToleration: cattle.io/os=linux:NoSchedule + # -- Node selector for system-managed Longhorn components. + systemManagedComponentsNodeSelector: kubernetes.io/os:linux + + networkPolicies: + # -- Setting that allows you to enable network policies that control access to Longhorn pods. + enabled: false + # -- Distribution that determines the policy for allowing access for an ingress. (Options: "k3s", "rke2", "rke1") + type: "k3s" + + image: + longhorn: + engine: + # -- Repository for the Longhorn Engine image. + repository: longhornio/longhorn-engine + # -- Specify Longhorn engine image tag + tag: v1.6.1 + manager: + # -- Repository for the Longhorn Manager image. + repository: longhornio/longhorn-manager + # -- Specify Longhorn manager image tag + tag: v1.6.1 + ui: + # -- Repository for the Longhorn UI image. + repository: longhornio/longhorn-ui + # -- Specify Longhorn ui image tag + tag: v1.6.1 + instanceManager: + # -- Repository for the Longhorn Instance Manager image. + repository: longhornio/longhorn-instance-manager + # -- Specify Longhorn instance manager image tag + tag: v1.6.1 + shareManager: + # -- Repository for the Longhorn Share Manager image. + repository: longhornio/longhorn-share-manager + # -- Specify Longhorn share manager image tag + tag: v1.6.1 + backingImageManager: + # -- Repository for the Backing Image Manager image. When unspecified, Longhorn uses the default value. + repository: longhornio/backing-image-manager + # -- Specify Longhorn backing image manager image tag + tag: v1.6.1 + supportBundleKit: + # -- Repository for the Longhorn Support Bundle Manager image. + repository: longhornio/support-bundle-kit + # -- Tag for the Longhorn Support Bundle Manager image. + tag: v0.0.36 + csi: + attacher: + # -- Repository for the CSI attacher image. When unspecified, Longhorn uses the default value. + repository: longhornio/csi-attacher + # -- Tag for the CSI attacher image. When unspecified, Longhorn uses the default value. + tag: v4.4.2 + provisioner: + # -- Repository for the CSI Provisioner image. When unspecified, Longhorn uses the default value. + repository: longhornio/csi-provisioner + # -- Tag for the CSI Provisioner image. When unspecified, Longhorn uses the default value. + tag: v3.6.2 + nodeDriverRegistrar: + # -- Repository for the CSI Node Driver Registrar image. When unspecified, Longhorn uses the default value. + repository: longhornio/csi-node-driver-registrar + # -- Tag for the CSI Node Driver Registrar image. When unspecified, Longhorn uses the default value. + tag: v2.9.2 + resizer: + # -- Repository for the CSI Resizer image. When unspecified, Longhorn uses the default value. + repository: longhornio/csi-resizer + # -- Tag for the CSI Resizer image. When unspecified, Longhorn uses the default value. + tag: v1.9.2 + snapshotter: + # -- Repository for the CSI Snapshotter image. When unspecified, Longhorn uses the default value. + repository: longhornio/csi-snapshotter + # -- Tag for the CSI Snapshotter image. When unspecified, Longhorn uses the default value. + tag: v6.3.2 + livenessProbe: + # -- Repository for the CSI liveness probe image. When unspecified, Longhorn uses the default value. + repository: longhornio/livenessprobe + # -- Tag for the CSI liveness probe image. When unspecified, Longhorn uses the default value. + tag: v2.12.0 + openshift: + oauthProxy: + # -- Repository for the OAuth Proxy image. This setting applies only to OpenShift users. + repository: quay.io/openshift/origin-oauth-proxy + # -- Tag for the OAuth Proxy image. This setting applies only to OpenShift users. Specify OCP/OKD version 4.1 or later. The latest stable version is 4.14. + tag: 4.14 + # -- Image pull policy that applies to all user-deployed Longhorn components, such as Longhorn Manager, Longhorn driver, and Longhorn UI. + pullPolicy: IfNotPresent + + service: + ui: + # -- Service type for Longhorn UI. (Options: "ClusterIP", "NodePort", "LoadBalancer", "Rancher-Proxy") + type: ClusterIP + # -- NodePort port number for Longhorn UI. When unspecified, Longhorn selects a free port between 30000 and 32767. + nodePort: null + manager: + # -- Service type for Longhorn Manager. + type: ClusterIP + # -- NodePort port number for Longhorn Manager. When unspecified, Longhorn selects a free port between 30000 and 32767. + nodePort: "" + + persistence: + # -- Setting that allows you to specify the default Longhorn StorageClass. + defaultClass: true + # -- Filesystem type of the default Longhorn StorageClass. + defaultFsType: ext4 + # -- mkfs parameters of the default Longhorn StorageClass. + defaultMkfsParams: "" + # -- Replica count of the default Longhorn StorageClass. + defaultClassReplicaCount: 3 + # -- Data locality of the default Longhorn StorageClass. (Options: "disabled", "best-effort") + defaultDataLocality: disabled + # -- Reclaim policy that provides instructions for handling of a volume after its claim is released. (Options: "Retain", "Delete") + reclaimPolicy: Delete + # -- Setting that allows you to enable live migration of a Longhorn volume from one node to another. + migratable: false + # -- Set NFS mount options for Longhorn StorageClass for RWX volumes + nfsOptions: "" + recurringJobSelector: + # -- Setting that allows you to enable the recurring job selector for a Longhorn StorageClass. + enable: false + # -- Recurring job selector for a Longhorn StorageClass. Ensure that quotes are used correctly when specifying job parameters. (Example: `[{"name":"backup", "isGroup":true}]`) + jobList: [] + backingImage: + # -- Setting that allows you to use a backing image in a Longhorn StorageClass. + enable: false + # -- Backing image to be used for creating and restoring volumes in a Longhorn StorageClass. When no backing images are available, specify the data source type and parameters that Longhorn can use to create a backing image. + name: ~ + # -- Data source type of a backing image used in a Longhorn StorageClass. + # If the backing image exists in the cluster, Longhorn uses this setting to verify the image. + # If the backing image does not exist, Longhorn creates one using the specified data source type. + dataSourceType: ~ + # -- Data source parameters of a backing image used in a Longhorn StorageClass. + # You can specify a JSON string of a map. (Example: `'{\"url\":\"https://backing-image-example.s3-region.amazonaws.com/test-backing-image\"}'`) + dataSourceParameters: ~ + # -- Expected SHA-512 checksum of a backing image used in a Longhorn StorageClass. + expectedChecksum: ~ + defaultNodeSelector: + # -- Setting that allows you to enable the node selector for the default Longhorn StorageClass. + enable: false + # -- Node selector for the default Longhorn StorageClass. Longhorn uses only nodes with the specified tags for storing volume data. (Examples: "storage,fast") + selector: "" + # -- Setting that allows you to enable automatic snapshot removal during filesystem trim for a Longhorn StorageClass. (Options: "ignored", "enabled", "disabled") + removeSnapshotsDuringFilesystemTrim: ignored + + preUpgradeChecker: + # -- Setting that allows Longhorn to perform pre-upgrade checks. Disable this setting when installing Longhorn using Argo CD or other GitOps solutions. + jobEnabled: true + # -- Setting that allows Longhorn to perform upgrade version checks after starting the Longhorn Manager DaemonSet Pods. Disabling this setting also disables `preUpgradeChecker.jobEnabled`. Longhorn recommends keeping this setting enabled. + upgradeVersionCheck: true + + csi: + # -- kubelet root directory. When unspecified, Longhorn uses the default value. + kubeletRootDir: ~ + # -- Replica count of the CSI Attacher. When unspecified, Longhorn uses the default value ("3"). + attacherReplicaCount: ~ + # -- Replica count of the CSI Provisioner. When unspecified, Longhorn uses the default value ("3"). + provisionerReplicaCount: ~ + # -- Replica count of the CSI Resizer. When unspecified, Longhorn uses the default value ("3"). + resizerReplicaCount: ~ + # -- Replica count of the CSI Snapshotter. When unspecified, Longhorn uses the default value ("3"). + snapshotterReplicaCount: ~ + + defaultSettings: + # -- Endpoint used to access the backupstore. (Options: "NFS", "CIFS", "AWS", "GCP", "AZURE") + backupTarget: ~ + # -- Name of the Kubernetes secret associated with the backup target. + backupTargetCredentialSecret: ~ + # -- Setting that allows Longhorn to automatically attach a volume and create snapshots or backups when recurring jobs are run. + allowRecurringJobWhileVolumeDetached: ~ + # -- Setting that allows Longhorn to automatically create a default disk only on nodes with the label "node.longhorn.io/create-default-disk=true" (if no other disks exist). When this setting is disabled, Longhorn creates a default disk on each node that is added to the cluster. + createDefaultDiskLabeledNodes: ~ + # -- Default path for storing data on a host. The default value is "/var/lib/longhorn/". + defaultDataPath: ~ + # -- Default data locality. A Longhorn volume has data locality if a local replica of the volume exists on the same node as the pod that is using the volume. + defaultDataLocality: ~ + # -- Setting that allows scheduling on nodes with healthy replicas of the same volume. This setting is disabled by default. + replicaSoftAntiAffinity: ~ + # -- Setting that automatically rebalances replicas when an available node is discovered. + replicaAutoBalance: ~ + # -- Percentage of storage that can be allocated relative to hard drive capacity. The default value is "100". + storageOverProvisioningPercentage: ~ + # -- Percentage of minimum available disk capacity. When the minimum available capacity exceeds the total available capacity, the disk becomes unschedulable until more space is made available for use. The default value is "25". + storageMinimalAvailablePercentage: ~ + # -- Percentage of disk space that is not allocated to the default disk on each new Longhorn node. + storageReservedPercentageForDefaultDisk: ~ + # -- Upgrade Checker that periodically checks for new Longhorn versions. When a new version is available, a notification appears on the Longhorn UI. This setting is enabled by default + upgradeChecker: ~ + # -- Default number of replicas for volumes created using the Longhorn UI. For Kubernetes configuration, modify the `numberOfReplicas` field in the StorageClass. The default value is "3". + defaultReplicaCount: ~ + # -- Default Longhorn StorageClass. "storageClassName" is assigned to PVs and PVCs that are created for an existing Longhorn volume. "storageClassName" can also be used as a label, so it is possible to use a Longhorn StorageClass to bind a workload to an existing PV without creating a Kubernetes StorageClass object. The default value is "longhorn-static". + defaultLonghornStaticStorageClass: ~ + # -- Number of seconds that Longhorn waits before checking the backupstore for new backups. The default value is "300". When the value is "0", polling is disabled. + backupstorePollInterval: ~ + # -- Number of minutes that Longhorn keeps a failed backup resource. When the value is "0", automatic deletion is disabled. + failedBackupTTL: ~ + # -- Setting that restores recurring jobs from a backup volume on a backup target and creates recurring jobs if none exist during backup restoration. + restoreVolumeRecurringJobs: ~ + # -- Maximum number of successful recurring backup and snapshot jobs to be retained. When the value is "0", a history of successful recurring jobs is not retained. + recurringSuccessfulJobsHistoryLimit: ~ + # -- Maximum number of failed recurring backup and snapshot jobs to be retained. When the value is "0", a history of failed recurring jobs is not retained. + recurringFailedJobsHistoryLimit: ~ + # -- Maximum number of snapshots or backups to be retained. + recurringJobMaxRetention: ~ + # -- Maximum number of failed support bundles that can exist in the cluster. When the value is "0", Longhorn automatically purges all failed support bundles. + supportBundleFailedHistoryLimit: ~ + # -- Taint or toleration for system-managed Longhorn components. + taintToleration: ~ + # -- Node selector for system-managed Longhorn components. + systemManagedComponentsNodeSelector: ~ + # -- PriorityClass for system-managed Longhorn components. + # This setting can help prevent Longhorn components from being evicted under Node Pressure. + # Notice that this will be applied to Longhorn user-deployed components by default if there are no priority class values set yet, such as `longhornManager.priorityClass`. + priorityClass: &defaultPriorityClassNameRef "longhorn-critical" + # -- Setting that allows Longhorn to automatically salvage volumes when all replicas become faulty (for example, when the network connection is interrupted). Longhorn determines which replicas are usable and then uses these replicas for the volume. This setting is enabled by default. + autoSalvage: ~ + # -- Setting that allows Longhorn to automatically delete a workload pod that is managed by a controller (for example, daemonset) whenever a Longhorn volume is detached unexpectedly (for example, during Kubernetes upgrades). After deletion, the controller restarts the pod and then Kubernetes handles volume reattachment and remounting. + autoDeletePodWhenVolumeDetachedUnexpectedly: ~ + # -- Setting that prevents Longhorn Manager from scheduling replicas on a cordoned Kubernetes node. This setting is enabled by default. + disableSchedulingOnCordonedNode: ~ + # -- Setting that allows Longhorn to schedule new replicas of a volume to nodes in the same zone as existing healthy replicas. Nodes that do not belong to any zone are treated as existing in the zone that contains healthy replicas. When identifying zones, Longhorn relies on the label "topology.kubernetes.io/zone=" in the Kubernetes node object. + replicaZoneSoftAntiAffinity: ~ + # -- Setting that allows scheduling on disks with existing healthy replicas of the same volume. This setting is enabled by default. + replicaDiskSoftAntiAffinity: ~ + # -- Policy that defines the action Longhorn takes when a volume is stuck with a StatefulSet or Deployment pod on a node that failed. + nodeDownPodDeletionPolicy: ~ + # -- Policy that defines the action Longhorn takes when a node with the last healthy replica of a volume is drained. + nodeDrainPolicy: ~ + # -- Setting that allows automatic detaching of manually-attached volumes when a node is cordoned. + detachManuallyAttachedVolumesWhenCordoned: ~ + # -- Number of seconds that Longhorn waits before reusing existing data on a failed replica instead of creating a new replica of a degraded volume. + replicaReplenishmentWaitInterval: ~ + # -- Maximum number of replicas that can be concurrently rebuilt on each node. + concurrentReplicaRebuildPerNodeLimit: ~ + # -- Maximum number of volumes that can be concurrently restored on each node using a backup. When the value is "0", restoration of volumes using a backup is disabled. + concurrentVolumeBackupRestorePerNodeLimit: ~ + # -- Setting that disables the revision counter and thereby prevents Longhorn from tracking all write operations to a volume. When salvaging a volume, Longhorn uses properties of the "volume-head-xxx.img" file (the last file size and the last time the file was modified) to select the replica to be used for volume recovery. This setting applies only to volumes created using the Longhorn UI. + disableRevisionCounter: ~ + # -- Image pull policy for system-managed pods, such as Instance Manager, engine images, and CSI Driver. Changes to the image pull policy are applied only after the system-managed pods restart. + systemManagedPodsImagePullPolicy: ~ + # -- Setting that allows you to create and attach a volume without having all replicas scheduled at the time of creation. + allowVolumeCreationWithDegradedAvailability: ~ + # -- Setting that allows Longhorn to automatically clean up the system-generated snapshot after replica rebuilding is completed. + autoCleanupSystemGeneratedSnapshot: ~ + # -- Setting that allows Longhorn to automatically clean up the snapshot generated by a recurring backup job. + autoCleanupRecurringJobBackupSnapshot: ~ + # -- Maximum number of engines that are allowed to concurrently upgrade on each node after Longhorn Manager is upgraded. When the value is "0", Longhorn does not automatically upgrade volume engines to the new default engine image version. + concurrentAutomaticEngineUpgradePerNodeLimit: ~ + # -- Number of minutes that Longhorn waits before cleaning up the backing image file when no replicas in the disk are using it. + backingImageCleanupWaitInterval: ~ + # -- Number of seconds that Longhorn waits before downloading a backing image file again when the status of all image disk files changes to "failed" or "unknown". + backingImageRecoveryWaitInterval: ~ + # -- Percentage of the total allocatable CPU resources on each node to be reserved for each instance manager pod when the V1 Data Engine is enabled. The default value is "12". + guaranteedInstanceManagerCPU: ~ + # -- Setting that notifies Longhorn that the cluster is using the Kubernetes Cluster Autoscaler. + kubernetesClusterAutoscalerEnabled: ~ + # -- Setting that allows Longhorn to automatically delete an orphaned resource and the corresponding data (for example, stale replicas). Orphaned resources on failed or unknown nodes are not automatically cleaned up. + orphanAutoDeletion: ~ + # -- Storage network for in-cluster traffic. When unspecified, Longhorn uses the Kubernetes cluster network. + storageNetwork: ~ + # -- Flag that prevents accidental uninstallation of Longhorn. + deletingConfirmationFlag: ~ + # -- Timeout between the Longhorn Engine and replicas. Specify a value between "8" and "30" seconds. The default value is "8". + engineReplicaTimeout: ~ + # -- Setting that allows you to enable and disable snapshot hashing and data integrity checks. + snapshotDataIntegrity: ~ + # -- Setting that allows disabling of snapshot hashing after snapshot creation to minimize impact on system performance. + snapshotDataIntegrityImmediateCheckAfterSnapshotCreation: ~ + # -- Setting that defines when Longhorn checks the integrity of data in snapshot disk files. You must use the Unix cron expression format. + snapshotDataIntegrityCronjob: ~ + # -- Setting that allows Longhorn to automatically mark the latest snapshot and its parent files as removed during a filesystem trim. Longhorn does not remove snapshots containing multiple child files. + removeSnapshotsDuringFilesystemTrim: ~ + # -- Setting that allows fast rebuilding of replicas using the checksum of snapshot disk files. Before enabling this setting, you must set the snapshot-data-integrity value to "enable" or "fast-check". + fastReplicaRebuildEnabled: ~ + # -- Number of seconds that an HTTP client waits for a response from a File Sync server before considering the connection to have failed. + replicaFileSyncHttpClientTimeout: ~ + # -- Log levels that indicate the type and severity of logs in Longhorn Manager. The default value is "Info". (Options: "Panic", "Fatal", "Error", "Warn", "Info", "Debug", "Trace") + logLevel: ~ + # -- Setting that allows you to specify a backup compression method. + backupCompressionMethod: ~ + # -- Maximum number of worker threads that can concurrently run for each backup. + backupConcurrentLimit: ~ + # -- Maximum number of worker threads that can concurrently run for each restore operation. + restoreConcurrentLimit: ~ + # -- Setting that allows you to enable the V1 Data Engine. + v1DataEngine: ~ + # -- Setting that allows you to enable the V2 Data Engine, which is based on the Storage Performance Development Kit (SPDK). The V2 Data Engine is a preview feature and should not be used in production environments. + v2DataEngine: ~ + # -- Setting that allows you to configure maximum huge page size (in MiB) for the V2 Data Engine. + v2DataEngineHugepageLimit: ~ + # -- Setting that allows rebuilding of offline replicas for volumes using the V2 Data Engine. + offlineReplicaRebuilding: ~ + # -- Number of millicpus on each node to be reserved for each Instance Manager pod when the V2 Data Engine is enabled. The default value is "1250". + v2DataEngineGuaranteedInstanceManagerCPU: ~ + # -- Setting that allows scheduling of empty node selector volumes to any node. + allowEmptyNodeSelectorVolume: ~ + # -- Setting that allows scheduling of empty disk selector volumes to any disk. + allowEmptyDiskSelectorVolume: ~ + # -- Setting that allows Longhorn to periodically collect anonymous usage data for product improvement purposes. Longhorn sends collected data to the [Upgrade Responder](https://github.com/longhorn/upgrade-responder) server, which is the data source of the Longhorn Public Metrics Dashboard (https://metrics.longhorn.io). The Upgrade Responder server does not store data that can be used to identify clients, including IP addresses. + allowCollectingLonghornUsageMetrics: ~ + # -- Setting that temporarily prevents all attempts to purge volume snapshots. + disableSnapshotPurge: ~ + # -- Maximum snapshot count for a volume. The value should be between 2 to 250 + snapshotMaxCount: ~ + + privateRegistry: + # -- Setting that allows you to create a private registry secret. + createSecret: ~ + # -- URL of a private registry. When unspecified, Longhorn uses the default system registry. + registryUrl: ~ + # -- User account used for authenticating with a private registry. + registryUser: ~ + # -- Password for authenticating with a private registry. + registryPasswd: ~ + # -- Kubernetes secret that allows you to pull images from a private registry. This setting applies only when creation of private registry secrets is enabled. You must include the private registry name in the secret name. + registrySecret: ~ + + longhornManager: + log: + # -- Format of Longhorn Manager logs. (Options: "plain", "json") + format: plain + # -- PriorityClass for Longhorn Manager. + priorityClass: *defaultPriorityClassNameRef + # -- Toleration for Longhorn Manager on nodes allowed to run Longhorn Manager. + tolerations: [] + ## If you want to set tolerations for Longhorn Manager DaemonSet, delete the `[]` in the line above + ## and uncomment this example block + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + # -- Node selector for Longhorn Manager. Specify the nodes allowed to run Longhorn Manager. + nodeSelector: {} + ## If you want to set node selector for Longhorn Manager DaemonSet, delete the `{}` in the line above + ## and uncomment this example block + # label-key1: "label-value1" + # label-key2: "label-value2" + # -- Annotation for the Longhorn Manager service. + serviceAnnotations: {} + ## If you want to set annotations for the Longhorn Manager service, delete the `{}` in the line above + ## and uncomment this example block + # annotation-key1: "annotation-value1" + # annotation-key2: "annotation-value2" + + longhornDriver: + # -- PriorityClass for Longhorn Driver. + priorityClass: *defaultPriorityClassNameRef + # -- Toleration for Longhorn Driver on nodes allowed to run Longhorn components. + tolerations: [] + ## If you want to set tolerations for Longhorn Driver Deployer Deployment, delete the `[]` in the line above + ## and uncomment this example block + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + # -- Node selector for Longhorn Driver. Specify the nodes allowed to run Longhorn Driver. + nodeSelector: {} + ## If you want to set node selector for Longhorn Driver Deployer Deployment, delete the `{}` in the line above + ## and uncomment this example block + # label-key1: "label-value1" + # label-key2: "label-value2" + + longhornUI: + # -- Replica count for Longhorn UI. + replicas: 2 + # -- PriorityClass for Longhorn UI. + priorityClass: *defaultPriorityClassNameRef + # -- Toleration for Longhorn UI on nodes allowed to run Longhorn components. + tolerations: [] + ## If you want to set tolerations for Longhorn UI Deployment, delete the `[]` in the line above + ## and uncomment this example block + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + # -- Node selector for Longhorn UI. Specify the nodes allowed to run Longhorn UI. + nodeSelector: {} + ## If you want to set node selector for Longhorn UI Deployment, delete the `{}` in the line above + ## and uncomment this example block + # label-key1: "label-value1" + # label-key2: "label-value2" + + ingress: + # -- Setting that allows Longhorn to generate ingress records for the Longhorn UI service. + enabled: false + + # -- IngressClass resource that contains ingress configuration, including the name of the Ingress controller. + # ingressClassName can replace the kubernetes.io/ingress.class annotation used in earlier Kubernetes releases. + ingressClassName: ~ + + # -- Hostname of the Layer 7 load balancer. + host: sslip.io + + # -- Setting that allows you to enable TLS on ingress records. + tls: false + + # -- Setting that allows you to enable secure connections to the Longhorn UI service via port 443. + secureBackends: false + + # -- TLS secret that contains the private key and certificate to be used for TLS. This setting applies only when TLS is enabled on ingress records. + tlsSecret: longhorn.local-tls + + path: / + + ## If you're using kube-lego, you will want to add: + ## kubernetes.io/tls-acme: true + ## + ## For a full list of possible ingress annotations, please see + ## ref: https://github.com/kubernetes/ingress-nginx/blob/master/docs/annotations.md + ## + ## If tls is set to true, annotation ingress.kubernetes.io/secure-backends: "true" will automatically be set + # -- Ingress annotations in the form of key-value pairs. + annotations: + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: true + + # -- Secret that contains a TLS private key and certificate. Use secrets if you want to use your own certificates to secure ingresses. + secrets: + ## If you're providing your own certificates, please use this to add the certificates as secrets + ## key and certificate should start with -----BEGIN CERTIFICATE----- or + ## -----BEGIN RSA PRIVATE KEY----- + ## + ## name should line up with a tlsSecret set further up + ## If you're using kube-lego, this is unneeded, as it will create the secret for you if it is not set + ## + ## It is also possible to create and manage the certificates outside of this helm chart + ## Please see README.md for more information + # - name: longhorn.local-tls + # key: + # certificate: + + # -- Setting that allows you to enable pod security policies (PSPs) that allow privileged Longhorn pods to start. This setting applies only to clusters running Kubernetes 1.25 and earlier, and with the built-in Pod Security admission controller enabled. + enablePSP: false + + # -- Specify override namespace, specifically this is useful for using longhorn as sub-chart and its release namespace is not the `longhorn-system`. + namespaceOverride: "" + + # -- Annotation for the Longhorn Manager DaemonSet pods. This setting is optional. + annotations: {} + + serviceAccount: + # -- Annotations to add to the service account + annotations: {} + + metrics: + serviceMonitor: + # -- Setting that allows the creation of a Prometheus ServiceMonitor resource for Longhorn Manager components. + enabled: false + + ## openshift settings + openshift: + # -- Setting that allows Longhorn to integrate with OpenShift. + enabled: false + ui: + # -- Route for connections between Longhorn and the OpenShift web console. + route: "longhorn-ui" + # -- Port for accessing the OpenShift web console. + port: 443 + # -- Port for proxy that provides access to the OpenShift web console. + proxy: 8443 + + # -- Setting that allows Longhorn to generate code coverage profiles. + enableGoCoverDir: false + + + - name: csi-secrets-store + namespace: kube-system + chart_ref: kube-forge/secrets-store-csi-driver + chart_version: 1.4.3 + - name: fluent-operator + namespace: observability + create_namespace: true + chart_ref: kube-forge/fluent-operator + chart_version: 2.7.0 + release_state: "present" + values: + # Set this to containerd or crio if you want to collect CRI format logs + containerRuntime: containerd + Kubernetes: false + + operator: + initcontainer: + repository: "docker" + tag: "20.10" + + resources: + limits: + cpu: 100m + memory: 100Mi + + requests: + cpu: 50m + memory: 64Mi + container: + repository: "kubesphere/fluent-operator" + tag: "v2.7.0" + resources: + limits: + cpu: 100m + memory: 100Mi + requests: + cpu: 100m + memory: 60Mi + imagePullSecrets: [] + labels: {} + logPath: + # The operator currently assumes a Docker container runtime path for the logs as the default, for other container runtimes you can set the location explicitly below. + # crio: /var/log + containerd: /var/log + disableComponentControllers: "" + + fluentbit: + crdsEnable: true + enable: false + + fluentd: + crdsEnable: true + enable: false + name: fluentd + # Valid modes include "collector" and "agent". + # The "collector" mode will deploy Fluentd as a StatefulSet as before. + # The new "agent" mode will deploy Fluentd as a DaemonSet. + mode: "agent" + port: 24224 + image: + repository: "kubesphere/fluentd" + tag: "v1.15.3" + + nameOverride: "" + fullnameOverride: "" + namespaceOverride: "" + + - name: opentelemetry-operator + namespace: observability + create_namespace: true + chart_ref: kube-forge/opentelemetry-operator + chart_version: 0.55.0 + release_state: "present" + values: + replicaCount: 1 + nameOverride: "" + imagePullSecrets: [] + pdb: + create: false + minAvailable: 1 + maxUnavailable: "" + + manager: + image: + repository: ghcr.io/open-telemetry/opentelemetry-operator/opentelemetry-operator + tag: "" + collectorImage: + repository: otel/opentelemetry-collector-contrib + tag: 0.95.0 + + featureGates: "" + ports: + metricsPort: 8080 + webhookPort: 9443 + healthzPort: 8081 + resources: + limits: + cpu: 100m + memory: 128Mi + requests: + cpu: 100m + memory: 64Mi + env: + ENABLE_WEBHOOKS: "true" + + serviceAccount: + create: true + annotations: {} + + serviceMonitor: + enabled: true + metricsEndpoints: + - port: metrics + + prometheusRule: + enabled: true + groups: [] + defaultRules: + enabled: true + + extraArgs: [] + + leaderElection: + enabled: true + + verticalPodAutoscaler: + enabled: false + controlledResources: [] + maxAllowed: {} + minAllowed: {} + + updatePolicy: + updateMode: Auto + minReplicas: 2 + rolling: false + + securityContext: {} + + kubeRBACProxy: + enabled: true + image: + repository: quay.io/brancz/kube-rbac-proxy + tag: v0.15.0 + ports: + proxyPort: 8443 + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 5m + memory: 64Mi + + extraArgs: [] + + securityContext: {} + + admissionWebhooks: + create: true + servicePort: 443 + failurePolicy: Fail + secretName: "" + pods: + failurePolicy: Ignore + + namePrefix: "" + + timeoutSeconds: 10 + + namespaceSelector: {} + objectSelector: {} + certManager: + enabled: true + issuerRef: {} + certificateAnnotations: {} + issuerAnnotations: {} + + autoGenerateCert: + enabled: true + recreate: true + + secretAnnotations: {} + secretLabels: {} + + role: + create: true + + clusterRole: + create: true + + affinity: {} + tolerations: [] + nodeSelector: {} + topologySpreadConstraints: [] + hostNetwork: false + + priorityClassName: "" + + securityContext: + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 + fsGroup: 65532 + + testFramework: + image: + repository: busybox + tag: latest + + - name: tempo + namespace: observability + create_namespace: true + chart_ref: kube-forge/tempo + chart_version: 1.7.2 + release_state: "present" + values: + replicas: 1 + + tempo: + repository: grafana/tempo + tag: "" + pullPolicy: IfNotPresent + + updateStrategy: RollingUpdate + + memBallastSizeMbs: 1024 + multitenancyEnabled: false + reportingEnabled: false + + metricsGenerator: + enabled: false + remoteWriteUrl: "http://prometheus.monitoring:9090/api/v1/write" + retention: 24h + global_overrides: + per_tenant_override_config: /conf/overrides.yaml + + server: + http_listen_port: 3100 + storage: + trace: + backend: local + local: + path: /var/tempo/traces + wal: + path: /var/tempo/wal + receivers: + otlp: + protocols: + grpc: + endpoint: "0.0.0.0:4317" + http: + endpoint: "0.0.0.0:4318" + + tempoQuery: + repository: grafana/tempo-query + tag: null + pullPolicy: IfNotPresent + + enabled: true + + service: + port: 16686 + + ingress: + enabled: false + + serviceAccount: + create: true + automountServiceAccountToken: true + + service: + type: ClusterIP + + serviceMonitor: + enabled: true + + persistence: + enabled: true + storageClassName: local-path + accessModes: + - ReadWriteOnce + size: 10Gi + + priorityClassName: null + + - name: loki + namespace: observability + create_namespace: true + chart_ref: kube-forge/loki + chart_version: 5.47.2 + release_state: "present" + values: + loki: + image: + registry: docker.io + repository: grafana/loki + tag: null + podAnnotations: + app.kubernetes.io/component: "loki" + auth_enabled: false + commonConfig: + replication_factor: 1 + storage: + type: 'filesystem' + + frontend: + max_outstanding_per_tenant: 10000 + + limits_config: + reject_old_samples: false + split_queries_by_interval: 15m + max_query_parallelism: 32 + max_query_series: 10000 + retention_period: 168h + + compactor: + compaction_interval: 10m + retention_enabled: true + retention_delete_delay: 2h + + querier: + max_concurrent: 2048 + + query_scheduler: + max_outstanding_requests_per_tenant: 10000 + + rulerConfig: + storage: + type: local + local: + directory: /var/loki/rules + rule_path: /tmp/rules + + alertmanager_url: http://observability-alert-manager:9093 + + + singleBinary: + replicas: 1 + + extraVolumes: + - name: loki-default-rules + configMap: + name: loki-default-alerting-rules + + extraVolumeMounts: + - name: loki-default-rules + mountPath: /var/loki/rules + + + write: + persistence: + volumeClaimsEnabled: true + storageClass: "local-path" + size: 10Gi + + test: + enabled: false + + gateway: + enabled: false + + monitoring: + selfMonitoring: + enabled: false + grafanaAgent: + installOperator: false + lokiCanary: + enabled: false + rules: + enabled: true + alerting: true + additionalGroups: {} + + extraObjects: + - apiVersion: v1 + kind: ConfigMap + metadata: + name: loki-default-alerting-rules + labels: + loki_rule: "" + + data: + loki-default-alerting-rules.yaml: |- + groups: + - name: kube-events-alerts + rules: + - alert: FailedEventsOccured + expr: | + count(rate({logs_type="kube-events"} | json reason="reason", event_type="event_type" | event_type = `Warning` | reason = `Failed` [1h])) > 0 + for: 1m + labels: + severity: critical + annotations: + alertname: FailedEventsOccured + instance: kube-cluster + jobName: kube_events + summary: Failed events occured in cluster + addDefaultUrl: "true" + + - alert: OOMKilledEventsOccured + expr: | + count(rate({logs_type="kube-events"} | json reason="reason", event_type="event_type" | event_type = `Warning` | reason = `OOMKilled` [1h])) > 0 + for: 1m + labels: + severity: critical + annotations: + alertname: OOMKilledEventsOccured + instance: kube-cluster + jobName: kube_events + summary: OOMKilled events occured in cluster + addDefaultUrl: "true" + + - alert: EvictedEventsOccured + expr: | + count(rate({logs_type="kube-events"} | json reason="reason", event_type="event_type" | event_type = `Warning` | reason = `Evicted` [1h])) > 0 + for: 1m + labels: + severity: critical + annotations: + alertname: EvictedEventsOccured + instance: kube-cluster + jobName: kube_events + summary: Evicted events occured in cluster + addDefaultUrl: "true" + + - alert: ImagePullBackOffEventsOccured + expr: | + count(rate({logs_type="kube-events"} | json reason="reason", event_type="event_type" | event_type = `Warning` | reason = `ImagePullBackOff` [1h])) > 0 + for: 1m + labels: + severity: critical + annotations: + alertname: ImagePullBackOffEventsOccured + instance: kube-cluster + jobName: kube_events + summary: ImagePullBackOff events occured in cluster + addDefaultUrl: "true" + + - alert: BackOffEventsOccured + expr: | + count(rate({logs_type="kube-events"} | json reason="reason", event_type="event_type" | event_type = `Warning` | reason = `BackOff` [1h])) > 0 + for: 1m + labels: + severity: critical + annotations: + alertname: BackOffEventsOccured + instance: kube-cluster + jobName: kube_events + summary: BackOff events occured in cluster + addDefaultUrl: "true" + sidecar: + rules: + enabled: true + # -- Label that the configmaps/secrets with rules will be marked with. + label: loki_rule + # -- Label value that the configmaps/secrets with rules will be set to. + labelValue: "" + # -- Folder into which the rules will be placed. + folder: /var/loki/rules + # -- Comma separated list of namespaces. If specified, the sidecar will search for config-maps/secrets inside these namespaces. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify 'ALL' to search in all namespaces. + searchNamespace: 'ALL' + # -- Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH request, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # -- Search in configmap, secret, or both. + resource: both + # -- Absolute path to the shell script to execute after a configmap or secret has been reloaded. + script: null + # -- WatchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S. + watchServerTimeout: 60 + # + # -- WatchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # Defaults to 66sec. + watchClientTimeout: 60 + # -- Log level of the sidecar container. + logLevel: INFO + + - name: observability + namespace: observability + create_namespace: true + chart_ref: kube-forge/observability + chart_version: 0.1.0 + release_state: "present" + values: + prometheus: + enabled: true + serviceMonitor: true + image: + repository: prom/prometheus + tag: v2.45.0 + pullPolicy: IfNotPresent + + clustering: + enabled: false + replicas: 3 + shards: 1 + + persistence: + enabled: true + + storageClassName: "local-path" + storageResources: + requests: + storage: 3Gi + + scrapeInterval: 15s + retention: 7d + # serviceNodePort: 30008 + + alertManager: + enabled: false + serviceMonitor: true + enableDefaultRules: true + image: + repository: prom/alertmanager + tag: "v0.26.0" + pullPolicy: IfNotPresent + + # serviceNodePort: 30009 + + configPath: /etc/alertmanager + + route: + null + + receivers: + null + + + blackboxExporter: + enabled: true + serviceMonitor: true + image: + repository: prom/blackbox-exporter + tag: "v0.24.0" + pullPolicy: IfNotPresent + + # serviceNodePort: 30012 + + configPath: /etc/blackbox_exporter + additionalModules: + + + kubeStateMetrics: + enabled: true + image: + repository: bitnami/kube-state-metrics + tag: "2.9.2" + pullPolicy: IfNotPresent + resources: + requests: + cpu: 30m + memory: 120Mi + limits: + memory: 240Mi + cpu: 60m + + prometheusOperator: + enabled: true + image: + repository: ghcr.io/prometheus-operator/prometheus-operator + tag: v0.65.2 + pullPolicy: IfNotPresent + + prometheusConfigReloader: + image: + repository: ghcr.io/prometheus-operator/prometheus-config-reloader + tag: v0.65.2 + pullPolicy: IfNotPresent + + kubeRbacProxy: + image: + repository: bitnami/kube-rbac-proxy + tag: 0.14.1 + pullPolicy: IfNotPresent + + nodeExporter: + enabled: true + image: + repository: prom/node-exporter + tag: v1.5.0 + pullPolicy: IfNotPresent + + kubeEventsExporter: + enabled: true + image: + repository: ghcr.io/resmoio/kubernetes-event-exporter + tag: v1.4 + pullPolicy: IfNotPresent + lokiAddress: http://loki.observability.svc.cluster.local:3100 + logLevel: warn + logFormat: json + kubeQPS: 100 + kubeBurst: 500 + maxEventAgeSeconds: 120 + metricsNamePrefix: event_exporter_ + + cron: + restartSchedule: "*/2 * * * *" + image: + repository: bitnami/kubectl + tag: 1.27.5-debian-11-r8 + pullPolicy: IfNotPresent + additionalRoutes: + additionalReceivers: + + grafana: + enabled: true + serviceMonitor: true + domain: &grafanaDomain grafana.disk.lt.t1.cloud + image: + repository: grafana/grafana + tag: 10.4.1 + pullPolicy: IfNotPresent + + storageClassName: "local-path" + storageResources: + requests: + storage: 2Gi + + config: + server: | + root_url = https://grafana.disk.lt.t1.cloud + + security: | + admin_user = admin + admin_password = changeit + + auth: | + + + authGenericAuth: | + + + additionalDatasources: + - name: Kube-loki + type: loki + access: proxy + url: http://loki:3100 + editable: false + basicAuth: false + isDefault: false + jsonData: + maxLines: 1000 + - name: Kube-jaeger-query + type: jaeger + access: proxy + url: http://tempo:16686 + editable: false + basicAuth: false + isDefault: false + + opentelemetryCollector: + enabled: true + serviceMonitor: true + config: | + receivers: + otlp: + protocols: + grpc: + http: + exporters: + otlphttp: + endpoint: http://tempo:4318 + service: + telemetry: + logs: + level: "debug" + pipelines: + traces: + receivers: [otlp] + exporters: [otlphttp] + + ingress: + enabled: true + accountEmail: reversstorm@gmail.com + class: nginx + annotations: + nginx.ingress.kubernetes.io/proxy-buffer-size: "128k" + nginx.ingress.kubernetes.io/proxy-buffers: "4 256k" + nginx.ingress.kubernetes.io/proxy-busy-buffers-size: "256k" + tls: + enabled: true + hosts: + - host: grafana.disk.lt.t1.cloud + secretName: grafana-tls + + containerRuntime: containerd + + fluentbit: + enable: true + serviceMonitor: true + image: + repository: "kubesphere/fluent-bit" + tag: "v2.2.2" + + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/edge + operator: DoesNotExist + tolerations: + - operator: Exists + + input: + tail: + enable: true + refreshIntervalSeconds: 10 + memBufLimit: 100MB + bufferMaxSize: "" + path: "/var/log/containers/*.log" + skipLongLines: true + readFromHead: false + storageType: memory + pauseOnChunksOverlimit: "off" + systemd: + enable: true + systemdFilter: + enable: true + filters: [] + path: "/var/log/journal" + includeKubelet: true + stripUnderscores: "off" + storageType: memory + pauseOnChunksOverlimit: "off" + + nodeExporterMetrics: {} + fluentBitMetrics: {} + + output: + es: + enable: false + host: "" + port: 9200 + logstashPrefix: ks-logstash-log + bufferSize: 20MB + traceError: true + kafka: + enable: false + brokers: "" + topics: ks-log + opentelemetry: {} + opensearch: + enable: false + stdout: + enable: false + loki: + enable: true + host: loki + port: 3100 + + stackdriver: {} + + service: + storage: {} + + filter: + kubernetes: + enable: true + labels: true + annotations: true + containerd: + enable: true + systemd: + enable: true + + kubeedge: + enable: false + prometheusRemoteWrite: + # Change the host to the address of a cloud-side Prometheus-compatible server that can receive Prometheus remote write data + host: "" + # Change the port to the port of a cloud-side Prometheus-compatible server that can receive Prometheus remote write data + port: "" + + - name: harbor-certificate-generator + namespace: registry + create_namespace: true + chart_ref: kube-forge/certificate-generator + chart_version: 0.1.0 + release_state: "present" + values: + issuer_email: reversstorm@gmail.com + solver_ingress_class: nginx + + certificates: + - name: harbor-tls + domain: harbor.disk.lt.t1.cloud + + - name: harbor + namespace: registry + create_namespace: true + chart_ref: kube-forge/harbor + chart_version: 1.14.2 + release_state: "present" + values: + expose: + type: ingress + tls: + enabled: true + certSource: secret + secret: + secretName: harbor-tls + ingress: + hosts: + core: harbor.disk.lt.t1.cloud + controller: default + kubeVersionOverride: "" + className: "nginx" + annotations: + ingress.kubernetes.io/ssl-redirect: "true" + ingress.kubernetes.io/proxy-body-size: "0" + nginx.ingress.kubernetes.io/ssl-redirect: "true" + nginx.ingress.kubernetes.io/proxy-body-size: "0" + labels: {} + + nodePort: + name: harbor + ports: + http: + port: 80 + nodePort: 30002 + https: + port: 443 + nodePort: 30003 + + externalURL: https://harbor.disk.lt.t1.cloud + persistence: + resourcePolicy: "keep" + persistentVolumeClaim: + registry: + existingClaim: "" + storageClass: "local-path" + subPath: "" + accessMode: ReadWriteOnce + size: 10Gi + annotations: {} + jobservice: + jobLog: + existingClaim: "" + storageClass: "local-path" + subPath: "" + accessMode: ReadWriteOnce + size: 1Gi + annotations: {} + database: + existingClaim: "" + storageClass: "local-path" + subPath: "" + accessMode: ReadWriteOnce + size: 2Gi + annotations: {} + redis: + existingClaim: "" + storageClass: "local-path" + subPath: "" + accessMode: ReadWriteOnce + size: 1Gi + annotations: {} + trivy: + existingClaim: "" + storageClass: "local-path" + subPath: "" + accessMode: ReadWriteOnce + size: 5Gi + annotations: {} + + imageChartStorage: + disableredirect: false + + type: filesystem + filesystem: + rootdirectory: /storage + #maxthreads: 100 + + imagePullPolicy: IfNotPresent + + updateStrategy: + type: RollingUpdate + + harborAdminPassword: "changeit" + + logLevel: info + + metrics: + enabled: true + core: + path: /metrics + port: 8001 + registry: + path: /metrics + port: 8001 + jobservice: + path: /metrics + port: 8001 + exporter: + path: /metrics + port: 8001 + + serviceMonitor: + enabled: true + + trace: + enabled: true + provider: otel + sample_rate: 1 + attributes: + application: harbor + jaeger: + endpoint: http://hostname:14268/api/traces + otel: + endpoint: observability-opentelemetry-collector-collector.observability.svc.k8s-cluster.local:4318 + url_path: /v1/traces + compression: false + insecure: true + timeout: 10 + + portal: + image: + repository: goharbor/harbor-portal + tag: v2.10.1 + serviceAccountName: "" + automountServiceAccountToken: false + replicas: 1 + revisionHistoryLimit: 10 + + topologySpreadConstraints: [] + + podLabels: + "app.kubernetes.io/component": "harbor-portal" + priorityClassName: + + core: + image: + repository: goharbor/harbor-core + tag: v2.10.1 + serviceAccountName: "" + automountServiceAccountToken: false + replicas: 1 + revisionHistoryLimit: 10 + startupProbe: + enabled: true + initialDelaySeconds: 10 + extraEnvVars: [] + nodeSelector: {} + tolerations: [] + affinity: {} + topologySpreadConstraints: [] + podLabels: + "app.kubernetes.io/component": "harbor-core" + serviceAnnotations: {} + priorityClassName: + configureUserSettings: + quotaUpdateProvider: db # Or redis + secret: "" + existingSecret: "" + secretName: "" + tokenKey: "" + + tokenCert: "" + + xsrfKey: "" + existingXsrfSecret: "" + existingXsrfSecretKey: CSRF_KEY + artifactPullAsyncFlushDuration: + gdpr: + deleteUser: false + auditLogsCompliant: false + + + jobservice: + image: + repository: goharbor/harbor-jobservice + tag: v2.10.1 + serviceAccountName: "" + automountServiceAccountToken: false + replicas: 1 + revisionHistoryLimit: 10 + topologySpreadConstraints: + podLabels: + "app.kubernetes.io/component": "harbor-jobservice" + priorityClassName: + maxJobWorkers: 10 + jobLoggers: + - file + # - database + # - stdout + loggerSweeperDuration: 14 #days + notification: + webhook_job_max_retry: 3 + webhook_job_http_client_timeout: 3 # in seconds + reaper: + max_update_hours: 24 + max_dangling_hours: 168 + secret: "" + existingSecret: "" + existingSecretKey: JOBSERVICE_SECRET + + registry: + registry: + image: + repository: goharbor/registry-photon + tag: v2.10.1 + extraEnvVars: [] + controller: + image: + repository: goharbor/harbor-registryctl + tag: v2.10.1 + extraEnvVars: [] + serviceAccountName: "" + automountServiceAccountToken: false + replicas: 1 + revisionHistoryLimit: 10 + topologySpreadConstraints: [] + podLabels: + "app.kubernetes.io/component": "harbor-registry" + priorityClassName: + secret: "" + existingSecret: "" + existingSecretKey: REGISTRY_HTTP_SECRET + relativeurls: false + credentials: + # If using existingSecret, the key must be REGISTRY_PASSWD and REGISTRY_HTPASSWD + existingSecret: "" + # Login and password in htpasswd string format. Excludes `registry.credentials.username` and `registry.credentials.password`. May come in handy when integrating with tools like argocd or flux. This allows the same line to be generated each time the template is rendered, instead of the `htpasswd` function from helm, which generates different lines each time because of the salt. + # htpasswdString: $apr1$XLefHzeG$Xl4.s00sMSCCcMyJljSZb0 # example string + # htpasswdString: "" + middleware: + enabled: false + type: cloudFront + cloudFront: + baseurl: example.cloudfront.net + keypairid: KEYPAIRID + duration: 3000s + ipfilteredby: none + # The secret key that should be present is CLOUDFRONT_KEY_DATA, which should be the encoded private key + # that allows access to CloudFront + privateKeySecret: "my-secret" + # enable purge _upload directories + upload_purging: + enabled: true + # remove files in _upload directories which exist for a period of time, default is one week. + age: 168h + # the interval of the purge operations + interval: 24h + dryrun: false + + trivy: + enabled: true + image: + repository: goharbor/trivy-adapter-photon + tag: v2.10.1 + serviceAccountName: "" + automountServiceAccountToken: false + resources: + requests: + cpu: 200m + memory: 512Mi + limits: + cpu: 1 + memory: 1Gi + + + database: + # if external database is used, set "type" to "external" + # and fill the connection information in "external" section + type: internal + internal: + image: + repository: goharbor/harbor-db + tag: v2.10.1 + serviceAccountName: "" + automountServiceAccountToken: false + livenessProbe: + timeoutSeconds: 1 + readinessProbe: + timeoutSeconds: 1 + priorityClassName: + # The initial superuser password for internal database + # password: "changeit" + # The size limit for Shared memory, pgSQL use it for shared_buffer + # More details see: + # https://github.com/goharbor/harbor/issues/15034 + shmSizeLimit: 512Mi + initContainer: + migrator: {} + # resources: + # requests: + # memory: 128Mi + # cpu: 100m + permissions: {} + # resources: + # requests: + # memory: 128Mi + # cpu: 100m + external: + host: "192.168.0.1" + port: "5432" + username: "user" + password: "password" + coreDatabase: "registry" + # if using existing secret, the key must be "password" + existingSecret: "" + # "disable" - No SSL + # "require" - Always SSL (skip verification) + # "verify-ca" - Always SSL (verify that the certificate presented by the + # server was signed by a trusted CA) + # "verify-full" - Always SSL (verify that the certification presented by the + # server was signed by a trusted CA and the server host name matches the one + # in the certificate) + sslmode: "disable" + # The maximum number of connections in the idle connection pool per pod (core+exporter). + # If it <=0, no idle connections are retained. + maxIdleConns: 100 + # The maximum number of open connections to the database per pod (core+exporter). + # If it <= 0, then there is no limit on the number of open connections. + # Note: the default number of connections is 1024 for postgre of harbor. + maxOpenConns: 900 + ## Additional deployment annotations + podAnnotations: {} + ## Additional deployment labels + podLabels: {} + + + redis: + type: internal + internal: + image: + repository: goharbor/redis-photon + tag: v2.10.1 + serviceAccountName: "" + automountServiceAccountToken: false + extraEnvVars: [] + nodeSelector: {} + tolerations: [] + affinity: {} + priorityClassName: + jobserviceDatabaseIndex: "1" + registryDatabaseIndex: "2" + trivyAdapterIndex: "5" + # harborDatabaseIndex: "6" + # cacheLayerDatabaseIndex: "7" + external: + # support redis, redis+sentinel + # addr for redis: : + # addr for redis+sentinel: :,:,: + addr: "192.168.0.2:6379" + # The name of the set of Redis instances to monitor, it must be set to support redis+sentinel + sentinelMasterSet: "" + # The "coreDatabaseIndex" must be "0" as the library Harbor + # used doesn't support configuring it + # harborDatabaseIndex defaults to "0", but it can be configured to "6", this config is optional + # cacheLayerDatabaseIndex defaults to "0", but it can be configured to "7", this config is optional + coreDatabaseIndex: "0" + jobserviceDatabaseIndex: "1" + registryDatabaseIndex: "2" + trivyAdapterIndex: "5" + # harborDatabaseIndex: "6" + # cacheLayerDatabaseIndex: "7" + # username field can be an empty string, and it will be authenticated against the default user + username: "" + password: "" + existingSecret: "" + podAnnotations: {} + podLabels: {} + + - name: argo-cd + namespace: cicd + create_namespace: true + chart_ref: kube-forge/argo-cd + chart_version: 6.7.12 + release_state: "present" + values: + crds: + install: true + + server: + certificateSecret: + enabled: false + + metrics: + enabled: true + serviceMonitor: + enabled: true + + redis: + metrics: + enabled: true + serviceMonitor: + enabled: true + + controller: + replicas: 1 + metrics: + enabled: true + serviceMonitor: + enabled: true + + applicationSet: + metrics: + enabled: true + serviceMonitor: + enabled: true + + dex: + metrics: + enabled: true + ## check later + serviceMonitor: + enabled: false + + repoServer: + + metrics: + enabled: true + serviceMonitor: + enabled: true + + notifications: + metrics: + enabled: true + serviceMonitor: + enabled: true + + configs: + params: + server.insecure: true + + secret: + argocdServerAdminPassword: $2a$10$OUae.wI5UBlC0ISkvD82XOdbGc9iP8apcXcqNTy4gfjSyiA9Jmw76 + + repositories: + # add default helm-repository from harbor + {} + + cm: + create: true + url: "https://argocd.disk.lt.t1.cloud" + + accounts.developer: login + accounts.guest: login + + # oidc.config: "" + + rbac: + create: true + policy.csv: | + p, role:admin, applications, create, */*, allow + p, role:admin, applications, update, */*, allow + p, role:admin, applications, delete, */*, allow + p, role:admin, applications, sync, */*, allow + p, role:admin, applications, override, */*, allow + p, role:admin, applications, action/*, */*, allow + p, role:admin, applicationsets, get, */*, allow + p, role:admin, applicationsets, create, */*, allow + p, role:admin, applicationsets, update, */*, allow + p, role:admin, applicationsets, delete, */*, allow + p, role:admin, certificates, create, *, allow + p, role:admin, certificates, update, *, allow + p, role:admin, certificates, delete, *, allow + p, role:admin, clusters, create, *, allow + p, role:admin, clusters, update, *, allow + p, role:admin, clusters, delete, *, allow + p, role:admin, repositories, create, *, allow + p, role:admin, repositories, update, *, allow + p, role:admin, repositories, delete, *, allow + p, role:admin, projects, create, *, allow + p, role:admin, projects, update, *, allow + p, role:admin, projects, delete, *, allow + p, role:admin, accounts, update, *, allow + p, role:admin, gpgkeys, create, *, allow + p, role:admin, gpgkeys, delete, *, allow + p, role:admin, exec, create, */*, allow + + + policy.default: role:'' + # scopes: "[roles,email,groups]" + + - name: argo-rollouts + namespace: cicd + create_namespace: true + chart_ref: kube-forge/argo-rollouts + release_state: "present" + chart_version: 2.35.1 + values: + installCRDs: true + keepCRDs: false + clusterInstall: true + createClusterAggregateRoles: true + + apiVersionOverrides: + # -- String to override apiVersion of ingresses rendered by this helm chart + ingress: "" # networking.k8s.io/v1beta1 + + # -- Override the Kubernetes version, which is used to evaluate certain manifests + kubeVersionOverride: "" + + # -- Additional manifests to deploy within the chart. A list of objects. + ## Can be used to add secrets for Analysis with 3rd-party monitoring solutions. + extraObjects: [] + # - apiVersion: v1 + # kind: Secret + # metadata: + # name: datadog + # type: Opaque + # data: + # address: https://api.datadoghq.com + # api-key: + # app-key: + + global: + # -- Annotations for all deployed Deployments + deploymentAnnotations: {} + + controller: + # -- Value of label `app.kubernetes.io/component` + component: rollouts-controller + # -- Annotations to be added to the controller deployment + deploymentAnnotations: {} + # -- Annotations to be added to application controller pods + podAnnotations: {} + # -- [Node selector] + nodeSelector: {} + # -- [Tolerations] for use with node taints + tolerations: [] + # -- Assign custom [affinity] rules to the deployment + affinity: {} + logging: + # -- Set the logging level (one of: `debug`, `info`, `warn`, `error`) + level: info + # -- Set the klog logging level + kloglevel: "0" + # -- Set the logging format (one of: `text`, `json`) + format: "text" + + # -- Assign custom [TopologySpreadConstraints] rules to the controller + ## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ + ## If labelSelector is left out, it will default to the labelSelector configuration of the deployment + topologySpreadConstraints: [] + # - maxSkew: 1 + # topologyKey: topology.kubernetes.io/zone + # whenUnsatisfiable: DoNotSchedule + + # -- [priorityClassName] for the controller + priorityClassName: "" + # -- The number of controller pods to run + replicas: 2 + image: + # -- Registry to use + registry: quay.io + # -- Repository to use + repository: argoproj/argo-rollouts + # -- Overrides the image tag (default is the chart appVersion) + tag: "" + # -- Image pull policy + pullPolicy: IfNotPresent + + # -- flag to enable creation of cluster controller role (requires cluster RBAC) + createClusterRole: true + + # Controller container ports + containerPorts: + # -- Metrics container port + metrics: 8090 + # -- Healthz container port + healthz: 8080 + metrics: + # -- Deploy metrics service + enabled: true + service: + # -- Metrics service port name + portName: metrics + # -- Metrics service port + port: 8090 + # -- Service annotations + annotations: {} + serviceMonitor: + # -- Enable a prometheus ServiceMonitor + enabled: true + # -- Namespace to be used for the ServiceMonitor + namespace: "" + # -- Labels to be added to the ServiceMonitor + additionalLabels: {} + # -- Annotations to be added to the ServiceMonitor + additionalAnnotations: {} + # -- RelabelConfigs to apply to samples before scraping + relabelings: [] + # -- MetricRelabelConfigs to apply to samples before ingestion + metricRelabelings: [] + + # -- Configure liveness [probe] for the controller + # @default -- See [values.yaml] + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 30 + periodSeconds: 20 + failureThreshold: 3 + successThreshold: 1 + timeoutSeconds: 10 + + # -- Configure readiness [probe] for the controller + # @default -- See [values.yaml] + readinessProbe: + httpGet: + path: /metrics + port: metrics + initialDelaySeconds: 15 + periodSeconds: 5 + failureThreshold: 3 + successThreshold: 1 + timeoutSeconds: 4 + + ## Configure Pod Disruption Budget for the controller + pdb: + # -- Labels to be added to controller [Pod Disruption Budget] + labels: {} + # -- Annotations to be added to controller [Pod Disruption Budget] + annotations: {} + # -- Deploy a [Pod Disruption Budget] for the controller + enabled: false + # -- Minimum number / percentage of pods that should remain scheduled + minAvailable: # 1 + # -- Maximum number / percentage of pods that may be made unavailable + maxUnavailable: # 0 + + # -- Additional volumes to add to the controller pod + volumes: [] + # - configMap: + # name: my-certs-cm + # name: my-certs + + # -- Additional volumeMounts to add to the controller container + volumeMounts: [] + # - mountPath: /etc/ssl/certs + # name: my-certs + + # -- Configures 3rd party metric providers for controller + ## Ref: https://argo-rollouts.readthedocs.io/en/stable/analysis/plugins/ + metricProviderPlugins: {} + # metricProviderPlugins: |- + # - name: "argoproj-labs/sample-prometheus" # name of the plugin, it must match the name required by the plugin so that it can find its configuration + # location: "file://./my-custom-plugin" # supports http(s):// urls and file:// + + # -- Configures 3rd party traffic router plugins for controller + ## Ref: https://argo-rollouts.readthedocs.io/en/stable/features/traffic-management/plugins/ + trafficRouterPlugins: {} + # trafficRouterPlugins: |- + # - name: "argoproj-labs/sample-nginx" # name of the plugin, it must match the name required by the plugin so it can find it's configuration + # location: "file://./my-custom-plugin" # supports http(s):// urls and file:// + + serviceAccount: + # -- Specifies whether a service account should be created + create: true + # -- Annotations to add to the service account + annotations: {} + # -- The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + + # -- Annotations to be added to all CRDs + crdAnnotations: {} + + # -- Annotations for the all deployed pods + podAnnotations: {} + + # -- Security Context to set on pod level + podSecurityContext: + runAsNonRoot: true + + # -- Security Context to set on container level + containerSecurityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + + # -- Annotations to be added to the Rollout service + serviceAnnotations: {} + + # -- Labels to be added to the Rollout pods + podLabels: {} + + # -- Secrets with credentials to pull images from a private registry. Registry secret names as an array. + imagePullSecrets: [] + # - name: argo-pull-secret + + providerRBAC: + # -- Toggles addition of provider-specific RBAC rules to the controller Role and ClusterRole + enabled: true + # providerRBAC.enabled must be true in order to toggle the individual providers + providers: + # -- Adds RBAC rules for the Istio provider + istio: true + # -- Adds RBAC rules for the SMI provider + smi: true + # -- Adds RBAC rules for the Ambassador provider + ambassador: true + # -- Adds RBAC rules for the AWS Load Balancer Controller provider + awsLoadBalancerController: true + # -- Adds RBAC rules for the AWS App Mesh provider + awsAppMesh: true + # -- Adds RBAC rules for the Traefik provider + traefik: true + # -- Adds RBAC rules for the Apisix provider + apisix: true + # -- Adds RBAC rules for the Contour provider, see `https://github.com/argoproj-labs/rollouts-plugin-trafficrouter-contour/blob/main/README.md` + contour: true + # -- Adds RBAC rules for the Gloo Platform provider, see `https://github.com/argoproj-labs/rollouts-plugin-trafficrouter-glooplatform/blob/main/README.md` + glooPlatform: true + # -- Additional RBAC rules for others providers + additionalRules: [] + + dashboard: + # -- Deploy dashboard server + enabled: true + # -- Set cluster role to readonly + readonly: false + # -- Value of label `app.kubernetes.io/component` + component: rollouts-dashboard + # -- Annotations to be added to the dashboard deployment + deploymentAnnotations: {} + # -- Annotations to be added to application dashboard pods + podAnnotations: {} + # -- [Node selector] + nodeSelector: {} + # -- [Tolerations] for use with node taints + tolerations: [] + # -- Assign custom [affinity] rules to the deployment + affinity: {} + logging: + # -- Set the logging level (one of: `debug`, `info`, `warn`, `error`) + level: info + # -- Set the klog logging level + kloglevel: "0" + + # -- Assign custom [TopologySpreadConstraints] rules to the dashboard server + ## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ + ## If labelSelector is left out, it will default to the labelSelector configuration of the deployment + topologySpreadConstraints: [] + # - maxSkew: 1 + # topologyKey: topology.kubernetes.io/zone + # whenUnsatisfiable: DoNotSchedule + + # -- [priorityClassName] for the dashboard server + priorityClassName: "" + + # -- flag to enable creation of dashbord cluster role (requires cluster RBAC) + createClusterRole: true + + # -- The number of dashboard pods to run + replicas: 1 + image: + # -- Registry to use + registry: quay.io + # -- Repository to use + repository: argoproj/kubectl-argo-rollouts + # -- Overrides the image tag (default is the chart appVersion) + tag: "" + # -- Image pull policy + pullPolicy: IfNotPresent + # -- Additional command line arguments to pass to rollouts-dashboard. A list of flags. + extraArgs: [] + # -- Additional environment variables for rollouts-dashboard. A list of name/value maps. + extraEnv: [] + # - name: FOO + # value: bar + # -- Resource limits and requests for the dashboard pods. + resources: {} + # -- Security Context to set on pod level + podSecurityContext: + runAsNonRoot: true + # -- Security Context to set on container level + containerSecurityContext: {} + service: + # -- Sets the type of the Service + type: ClusterIP + # -- LoadBalancer will get created with the IP specified in this field + loadBalancerIP: "" + # -- Source IP ranges to allow access to service from + loadBalancerSourceRanges: [] + # -- Dashboard service external IPs + externalIPs: [] + # -- Service annotations + annotations: {} + # -- Service labels + labels: {} + # -- Service port name + portName: dashboard + # -- Service port + port: 3100 + # -- Service target port + targetPort: 3100 + # -- (int) Service nodePort + nodePort: + serviceAccount: + # -- Specifies whether a dashboard service account should be created + create: true + # -- Annotations to add to the dashboard service account + annotations: {} + # -- The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + + ## Configure Pod Disruption Budget for the dashboard + pdb: + # -- Labels to be added to dashboard [Pod Disruption Budget] + labels: {} + # -- Annotations to be added to dashboard [Pod Disruption Budget] + annotations: {} + # -- Deploy a [Pod Disruption Budget] for the dashboard + enabled: false + # -- Minimum number / percentage of pods that should remain scheduled + minAvailable: # 1 + # -- Maximum number / percentage of pods that may be made unavailable + maxUnavailable: # 0 + + ## Ingress configuration. + ## ref: https://kubernetes.io/docs/user-guide/ingress/ + ## + ingress: + # -- Enable dashboard ingress support + enabled: false + # -- Dashboard ingress annotations + annotations: {} + # -- Dashboard ingress labels + labels: {} + # -- Dashboard ingress class name + ingressClassName: "" + + # -- Dashboard ingress hosts + ## Argo Rollouts Dashboard Ingress. + ## Hostnames must be provided if Ingress is enabled. + ## Secrets must be manually created in the namespace + hosts: [] + # - argorollouts.example.com + + # -- Dashboard ingress paths + paths: + - / + # -- Dashboard ingress path type + pathType: Prefix + # -- Dashboard ingress extra paths + extraPaths: [] + # - path: /* + # backend: + # serviceName: ssl-redirect + # servicePort: use-annotation + ## for Kubernetes >=1.19 (when "networking.k8s.io/v1" is used) + # - path: /* + # pathType: Prefix + # backend: + # service + # name: ssl-redirect + # port: + # name: use-annotation + + # -- Dashboard ingress tls + tls: [] + # - secretName: argorollouts-example-tls + # hosts: + # - argorollouts.example.com + + # -- Additional volumes to add to the dashboard pod + volumes: [] + + # -- Additional volumeMounts to add to the dashboard container + volumeMounts: [] + + notifications: + secret: + # -- Whether to create notifications secret + create: false + # -- Generic key:value pairs to be inserted into the notifications secret + items: {} + # slack-token: + + # -- Configures notification services + notifiers: {} + # service.slack: | + # token: $slack-token + + # -- Notification templates + templates: {} + + # -- The trigger defines the condition when the notification should be sent + triggers: {} + # trigger.on-purple: | + # - send: [my-purple-template] + # when: rollout.spec.template.spec.containers[0].image == 'argoproj/rollouts-demo:purple' + + - name: keel + namespace: kube-system + chart_ref: kube-forge/keel + chart_version: 1.0.3 + release_state: "present" + + - name: argo-cd-ingress + namespace: cicd + create_namespace: true + chart_ref: kube-forge/service-ingress + chart_version: 0.1.0 + release_state: "present" + values: + services: + - domain: argocd.disk.lt.t1.cloud + address: argo-cd-argocd-server + port: 80 + secretName: argo-cd-server-tls + + ingress: + accountEmail: reversstorm@gmail.com + class: nginx + annotations: + nginx.ingress.kubernetes.io/proxy-buffer-size: "128k" + nginx.ingress.kubernetes.io/proxy-buffers: "4 256k" + nginx.ingress.kubernetes.io/proxy-busy-buffers-size: "256k" + nginx.ingress.kubernetes.io/proxy-body-size: "0" + tls: + enabled: true + useCertManager: true + + # used if "useCertManager" is false + crt: "" + key: "" + + - name: vault + namespace: secrets-storage + create_namespace: true + chart_ref: kube-forge/vault + chart_version: 0.1.0 + release_state: "present" + values: + global: + enabled: true + + imagePullSecrets: [] + tlsDisable: true + + externalVaultAddr: "" + + openshift: false + + # Create PodSecurityPolicy for pods + psp: + enable: false + # Annotation for PodSecurityPolicy. + # This is a multi-line templated string map, and can also be set as YAML. + annotations: | + seccomp.security.alpha.kubernetes.io/allowedProfileNames: docker/default,runtime/default + apparmor.security.beta.kubernetes.io/allowedProfileNames: runtime/default + seccomp.security.alpha.kubernetes.io/defaultProfileName: runtime/default + apparmor.security.beta.kubernetes.io/defaultProfileName: runtime/default + + serverTelemetry: + # Enable integration with the Prometheus Operator + # See the top level serverTelemetry section below before enabling this feature. + prometheusOperator: false + + injector: + enabled: true + + replicas: 1 + + # Configures the port the injector should listen on + port: 8080 + + # If multiple replicas are specified, by default a leader will be determined + # so that only one injector attempts to create TLS certificates. + leaderElector: + enabled: true + + # If true, will enable a node exporter metrics endpoint at /metrics. + metrics: + enabled: false + + # Deprecated: Please use global.externalVaultAddr instead. + externalVaultAddr: "" + + # image sets the repo and tag of the vault-k8s image to use for the injector. + image: + repository: "hashicorp/vault-k8s" + tag: "1.3.1" + pullPolicy: IfNotPresent + + # agentImage sets the repo and tag of the Vault image to use for the Vault Agent + # containers. This should be set to the official Vault image. Vault 1.3.1+ is + # required. + agentImage: + repository: "hashicorp/vault" + tag: "1.15.6" + agentDefaults: + cpuLimit: "500m" + cpuRequest: "250m" + memLimit: "128Mi" + memRequest: "64Mi" + # ephemeralLimit: "128Mi" + # ephemeralRequest: "64Mi" + + # Default template type for secrets when no custom template is specified. + # Possible values include: "json" and "map". + template: "map" + + # Default values within Agent's template_config stanza. + templateConfig: + exitOnRetryFailure: true + staticSecretRenderInterval: "" + + # Used to define custom livenessProbe settings + livenessProbe: + # When a probe fails, Kubernetes will try failureThreshold times before giving up + failureThreshold: 2 + # Number of seconds after the container has started before probe initiates + initialDelaySeconds: 5 + # How often (in seconds) to perform the probe + periodSeconds: 2 + # Minimum consecutive successes for the probe to be considered successful after having failed + successThreshold: 1 + # Number of seconds after which the probe times out. + timeoutSeconds: 5 + # Used to define custom readinessProbe settings + readinessProbe: + # When a probe fails, Kubernetes will try failureThreshold times before giving up + failureThreshold: 2 + # Number of seconds after the container has started before probe initiates + initialDelaySeconds: 5 + # How often (in seconds) to perform the probe + periodSeconds: 2 + # Minimum consecutive successes for the probe to be considered successful after having failed + successThreshold: 1 + # Number of seconds after which the probe times out. + timeoutSeconds: 5 + # Used to define custom startupProbe settings + startupProbe: + # When a probe fails, Kubernetes will try failureThreshold times before giving up + failureThreshold: 12 + # Number of seconds after the container has started before probe initiates + initialDelaySeconds: 5 + # How often (in seconds) to perform the probe + periodSeconds: 5 + # Minimum consecutive successes for the probe to be considered successful after having failed + successThreshold: 1 + # Number of seconds after which the probe times out. + timeoutSeconds: 5 + + # Mount Path of the Vault Kubernetes Auth Method. + authPath: "auth/kubernetes" + + # Configures the log verbosity of the injector. + # Supported log levels include: trace, debug, info, warn, error + logLevel: "info" + + # Configures the log format of the injector. Supported log formats: "standard", "json". + logFormat: "standard" + + # Configures all Vault Agent sidecars to revoke their token when shutting down + revokeOnShutdown: false + + webhook: + # Configures failurePolicy of the webhook. The "unspecified" default behaviour depends on the + # API Tag of the WebHook. + # To block pod creation while the webhook is unavailable, set the policy to `Fail` below. + # See https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy + # + failurePolicy: Ignore + + # matchPolicy specifies the approach to accepting changes based on the rules of + # the MutatingWebhookConfiguration. + # See https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy + # for more details. + # + matchPolicy: Exact + + # timeoutSeconds is the amount of seconds before the webhook request will be ignored + # or fails. + # If it is ignored or fails depends on the failurePolicy + # See https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#timeouts + # for more details. + # + timeoutSeconds: 30 + + # namespaceSelector is the selector for restricting the webhook to only + # specific namespaces. + # See https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-namespaceselector + # for more details. + # Example: + # namespaceSelector: + # matchLabels: + # sidecar-injector: enabled + namespaceSelector: {} + + # objectSelector is the selector for restricting the webhook to only + # specific labels. + # See https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector + # for more details. + # Example: + # objectSelector: + # matchLabels: + # vault-sidecar-injector: enabled + + # Extra annotations to attach to the webhook + annotations: {} + + # Deprecated: please use 'webhook.failurePolicy' instead + # Configures failurePolicy of the webhook. The "unspecified" default behaviour depends on the + # API Tag of the WebHook. + # To block pod creation while webhook is unavailable, set the policy to `Fail` below. + # See https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy + # + failurePolicy: Ignore + + # Deprecated: please use 'webhook.namespaceSelector' instead + # namespaceSelector is the selector for restricting the webhook to only + # specific namespaces. + # See https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-namespaceselector + # for more details. + # Example: + # namespaceSelector: + # matchLabels: + # sidecar-injector: enabled + namespaceSelector: {} + + # Deprecated: please use 'webhook.objectSelector' instead + # objectSelector is the selector for restricting the webhook to only + # specific labels. + # See https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector + # for more details. + # Example: + # objectSelector: + # matchLabels: + # vault-sidecar-injector: enabled + objectSelector: {} + + # Deprecated: please use 'webhook.annotations' instead + # Extra annotations to attach to the webhook + webhookAnnotations: {} + + certs: + # secretName is the name of the secret that has the TLS certificate and + # private key to serve the injector webhook. If this is null, then the + # injector will default to its automatic management mode that will assign + # a service account to the injector to generate its own certificates. + secretName: null + + # caBundle is a base64-encoded PEM-encoded certificate bundle for the CA + # that signed the TLS certificate that the webhook serves. This must be set + # if secretName is non-null unless an external service like cert-manager is + # keeping the caBundle updated. + caBundle: "" + + # certName and keyName are the names of the files within the secret for + # the TLS cert and private key, respectively. These have reasonable + # defaults but can be customized if necessary. + certName: tls.crt + keyName: tls.key + + securityContext: + pod: {} + container: {} + + resources: {} + + # extraEnvironmentVars is a list of extra environment variables to set in the + # injector deployment. + extraEnvironmentVars: {} + # KUBERNETES_SERVICE_HOST: kubernetes.default.svc + + topologySpreadConstraints: [] + + tolerations: [] + + nodeSelector: {} + + priorityClassName: "" + + annotations: {} + + extraLabels: {} + + hostNetwork: false + + + service: + # Extra annotations to attach to the injector service + annotations: {} + + # Injector serviceAccount specific config + serviceAccount: + # Extra annotations to attach to the injector serviceAccount + annotations: {} + + # A disruption budget limits the number of pods of a replicated application + # that are down simultaneously from voluntary disruptions + podDisruptionBudget: {} + # podDisruptionBudget: + # maxUnavailable: 1 + + # strategy for updating the deployment. This can be a multi-line string or a + # YAML map. + strategy: {} + # strategy: | + # rollingUpdate: + # maxSurge: 25% + # maxUnavailable: 25% + # type: RollingUpdate + + server: + enabled: true + enterpriseLicense: + # The name of the Kubernetes secret that holds the enterprise license. The + # secret must be in the same namespace that Vault is installed into. + secretName: "" + # The key within the Kubernetes secret that holds the enterprise license. + secretKey: "license" + + image: + repository: "hashicorp/vault" + tag: "1.15.6" + # Overrides the default Image Pull Policy + pullPolicy: IfNotPresent + + updateStrategyType: "RollingUpdate" + + # Supported log levels include: trace, debug, info, warn, error + logLevel: "" + + # Supported log formats include: standard, json + logFormat: "" + + resources: {} + + hostAliases: [] + # - ip: 127.0.0.1 + # hostnames: + # - chart-example.local + + route: + enabled: false + + # When HA mode is enabled and K8s service registration is being used, + # configure the route to point to the Vault active service. + activeService: true + + labels: {} + annotations: {} + host: chart-example.local + # tls will be passed directly to the route's TLS config, which + # can be used to configure other termination methods that terminate + # TLS at the router + tls: + termination: passthrough + + # authDelegator enables a cluster role binding to be attached to the service + # account. This cluster role binding can be used to setup Kubernetes auth + # method. See https://developer.hashicorp.com/vault/docs/auth/kubernetes + authDelegator: + enabled: true + + extraInitContainers: null + extraContainers: null + shareProcessNamespace: false + extraArgs: "" + + extraPorts: null + # - containerPort: 8300 + # name: http-monitoring + + readinessProbe: + enabled: true + # If you need to use a http path instead of the default exec + # path: /v1/sys/health?standbyok=true + + # Port number on which readinessProbe will be checked. + port: 8200 + # When a probe fails, Kubernetes will try failureThreshold times before giving up + failureThreshold: 2 + # Number of seconds after the container has started before probe initiates + initialDelaySeconds: 5 + # How often (in seconds) to perform the probe + periodSeconds: 5 + # Minimum consecutive successes for the probe to be considered successful after having failed + successThreshold: 1 + # Number of seconds after which the probe times out. + timeoutSeconds: 3 + # Used to enable a livenessProbe for the pods + livenessProbe: + enabled: false + # Used to define a liveness exec command. If provided, exec is preferred to httpGet (path) as the livenessProbe handler. + execCommand: [] + # - /bin/sh + # - -c + # - /vault/userconfig/mylivenessscript/run.sh + # Path for the livenessProbe to use httpGet as the livenessProbe handler + path: "/v1/sys/health?standbyok=true" + # Port number on which livenessProbe will be checked if httpGet is used as the livenessProbe handler + port: 8200 + # When a probe fails, Kubernetes will try failureThreshold times before giving up + failureThreshold: 2 + # Number of seconds after the container has started before probe initiates + initialDelaySeconds: 60 + # How often (in seconds) to perform the probe + periodSeconds: 5 + # Minimum consecutive successes for the probe to be considered successful after having failed + successThreshold: 1 + # Number of seconds after which the probe times out. + timeoutSeconds: 3 + + terminationGracePeriodSeconds: 10 + + # Used to set the sleep time during the preStop step + preStopSleepSeconds: 5 + + postStart: + # - /bin/sh + # - -c + # - /vault/userconfig/myscript/run.sh + + extraEnvironmentVars: {} + + extraSecretEnvironmentVars: [] + + extraVolumes: [] + + volumes: null + + volumeMounts: null + + topologySpreadConstraints: [] + + tolerations: [] + nodeSelector: {} + + # Enables network policy for server pods + networkPolicy: + enabled: false + egress: [] + # egress: + # - to: + # - ipBlock: + # cidr: 10.0.0.0/24 + # ports: + # - protocol: TCP + # port: 443 + ingress: + - from: + - namespaceSelector: {} + ports: + - port: 8200 + protocol: TCP + - port: 8201 + protocol: TCP + + priorityClassName: "" + extraLabels: {} + + annotations: {} + + service: + enabled: true + # Enable or disable the vault-active service, which selects Vault pods that + # have labeled themselves as the cluster leader with `vault-active: "true"`. + active: + enabled: true + # Extra annotations for the service definition. This can either be YAML or a + # YAML-formatted multi-line templated string map of the annotations to apply + # to the active service. + annotations: {} + # Enable or disable the vault-standby service, which selects Vault pods that + # have labeled themselves as a cluster follower with `vault-active: "false"`. + standby: + enabled: true + # Extra annotations for the service definition. This can either be YAML or a + # YAML-formatted multi-line templated string map of the annotations to apply + # to the standby service. + annotations: {} + # When disabled, services may select Vault pods not deployed from the chart. + # Does not affect the headless vault-internal service with `ClusterIP: None` + instanceSelector: + enabled: true + # clusterIP controls whether a Cluster IP address is attached to the + # Vault service within Kubernetes. By default, the Vault service will + # be given a Cluster IP address, set to None to disable. When disabled + # Kubernetes will create a "headless" service. Headless services can be + # used to communicate with pods directly through DNS instead of a round-robin + # load balancer. + # clusterIP: None + + # Configures the service type for the main Vault service. Can be ClusterIP + # or NodePort. + #type: ClusterIP + + # The IP family and IP families options are to set the behaviour in a dual-stack environment. + # Omitting these values will let the service fall back to whatever the CNI dictates the defaults + # should be. + # These are only supported for kubernetes versions >=1.23.0 + # + # Configures the service's supported IP family policy, can be either: + # SingleStack: Single-stack service. The control plane allocates a cluster IP for the Service, using the first configured service cluster IP range. + # PreferDualStack: Allocates IPv4 and IPv6 cluster IPs for the Service. + # RequireDualStack: Allocates Service .spec.ClusterIPs from both IPv4 and IPv6 address ranges. + ipFamilyPolicy: "" + + # Sets the families that should be supported and the order in which they should be applied to ClusterIP as well. + # Can be IPv4 and/or IPv6. + ipFamilies: [] + + # Do not wait for pods to be ready before including them in the services' + # targets. Does not apply to the headless service, which is used for + # cluster-internal communication. + publishNotReadyAddresses: true + + # The externalTrafficPolicy can be set to either Cluster or Local + # and is only valid for LoadBalancer and NodePort service types. + # The default value is Cluster. + # ref: https://kubernetes.io/docs/concepts/services-networking/service/#external-traffic-policy + externalTrafficPolicy: Cluster + + # If type is set to "NodePort", a specific nodePort value can be configured, + # will be random if left blank. + #nodePort: 30000 + + # When HA mode is enabled + # If type is set to "NodePort", a specific nodePort value can be configured, + # will be random if left blank. + #activeNodePort: 30001 + + # When HA mode is enabled + # If type is set to "NodePort", a specific nodePort value can be configured, + # will be random if left blank. + #standbyNodePort: 30002 + + # Port on which Vault server is listening + port: 8200 + # Target port to which the service should be mapped to + targetPort: 8200 + # Extra annotations for the service definition. This can either be YAML or a + # YAML-formatted multi-line templated string map of the annotations to apply + # to the service. + annotations: {} + + dataStorage: + enabled: true + size: 10Gi + mountPath: "/vault/data" + storageClass: local-path + accessMode: ReadWriteOnce + annotations: {} + labels: {} + + persistentVolumeClaimRetentionPolicy: {} + + # required for ha installation + auditStorage: + enabled: false + # Size of the PVC created + size: 10Gi + # Location where the PVC will be mounted. + mountPath: "/vault/audit" + # Name of the storage class to use. If null it will use the + # configured default Storage Class. + storageClass: local-path + # Access Mode of the storage device being used for the PVC + accessMode: ReadWriteOnce + # Annotations to apply to the PVC + annotations: {} + # Labels to apply to the PVC + labels: {} + + dev: + enabled: false + + # Set VAULT_DEV_ROOT_TOKEN_ID value + devRootToken: "root" + + # Run Vault in "standalone" mode. This is the default mode that will deploy if + # no arguments are given to helm. This requires a PVC for data storage to use + # the "file" backend. This mode is not highly available and should not be scaled + # past a single replica. + standalone: + enabled: "-" + + # config is a raw string of default configuration when using a Stateful + # deployment. Default is to use a PersistentVolumeClaim mounted at /vault/data + # and store data there. This is only used when using a Replica count of 1, and + # using a stateful set. This should be HCL. + + # Note: Configuration files are stored in ConfigMaps so sensitive data + # such as passwords should be either mounted through extraSecretEnvironmentVars + # or through a Kube secret. For more information see: + # https://developer.hashicorp.com/vault/docs/platform/k8s/helm/run#protecting-sensitive-vault-configurations + config: | + ui = true + + listener "tcp" { + tls_disable = 1 + address = "[::]:8200" + cluster_address = "[::]:8201" + telemetry { + unauthenticated_metrics_access = "true" + } + } + storage "file" { + path = "/vault/data" + } + telemetry { + prometheus_retention_time = "30s" + disable_hostname = true + } + + # Run Vault in "HA" mode. There are no storage requirements unless the audit log + # persistence is required. In HA mode Vault will configure itself to use Consul + # for its storage backend. The default configuration provided will work the Consul + # Helm project by default. It is possible to manually configure Vault to use a + # different HA backend. + ha: + enabled: false + replicas: 3 + + # Set the api_addr configuration for Vault HA + # See https://developer.hashicorp.com/vault/docs/configuration#api_addr + # If set to null, this will be set to the Pod IP Address + apiAddr: null + + # Set the cluster_addr confuguration for Vault HA + # See https://developer.hashicorp.com/vault/docs/configuration#cluster_addr + clusterAddr: null + + # Enables Vault's integrated Raft storage. Unlike the typical HA modes where + # Vault's persistence is external (such as Consul), enabling Raft mode will create + # persistent volumes for Vault to store data according to the configuration under server.dataStorage. + # The Vault cluster will coordinate leader elections and failovers internally. + raft: + # Enables Raft integrated storage + enabled: false + # Set the Node Raft ID to the name of the pod + setNodeId: false + + config: | + ui = true + + listener "tcp" { + tls_disable = 1 + address = "[::]:8200" + cluster_address = "[::]:8201" + telemetry { + unauthenticated_metrics_access = "true" + } + } + + storage "raft" { + path = "/vault/data" + } + + service_registration "kubernetes" {} + + # config is a raw string of default configuration when using a Stateful + # deployment. Default is to use a Consul for its HA storage backend. + # This should be HCL. + + # Note: Configuration files are stored in ConfigMaps so sensitive data + # such as passwords should be either mounted through extraSecretEnvironmentVars + # or through a Kube secret. For more information see: + # https://developer.hashicorp.com/vault/docs/platform/k8s/helm/run#protecting-sensitive-vault-configurations + config: | + ui = true + + listener "tcp" { + tls_disable = 1 + address = "[::]:8200" + cluster_address = "[::]:8201" + telemetry { + unauthenticated_metrics_access = "true" + } + } + storage "consul" { + path = "vault" + address = "HOST_IP:8500" + } + + service_registration "kubernetes" {} + + # Example configuration for using auto-unseal, using Google Cloud KMS. The + # GKMS keys must already exist, and the cluster must have a service account + # that is authorized to access GCP KMS. + #seal "gcpckms" { + # project = "vault-helm-dev-246514" + # region = "global" + # key_ring = "vault-helm-unseal-kr" + # crypto_key = "vault-helm-unseal-key" + #} + telemetry { + prometheus_retention_time = "30s" + disable_hostname = true + } + + # A disruption budget limits the number of pods of a replicated application + # that are down simultaneously from voluntary disruptions + disruptionBudget: + enabled: true + + # maxUnavailable will default to (n/2)-1 where n is the number of + # replicas. If you'd like a custom value, you can specify an override here. + maxUnavailable: null + + serviceAccount: + create: true + name: "" + createSecret: false + annotations: {} + extraLabels: {} + serviceDiscovery: + enabled: true + + statefulSet: + annotations: {} + securityContext: + pod: {} + container: {} + + hostNetwork: false + + # Vault UI + ui: + enabled: true + domain: vault.disk.lt.t1.cloud + publishNotReadyAddresses: true + # The service should only contain selectors for active Vault pod + activeVaultPodOnly: false + serviceType: "ClusterIP" + serviceNodePort: null + externalPort: 8200 + targetPort: 8200 + + serviceIPFamilyPolicy: "" + + serviceIPFamilies: [] + + externalTrafficPolicy: Cluster + + #loadBalancerSourceRanges: + # - 10.0.0.0/16 + # - 1.78.23.3/32 + + # loadBalancerIP: + + annotations: {} + + csi: + # True if you want to install a secrets-store-csi-driver-provider-vault daemonset. + # + # Requires installing the secrets-store-csi-driver separately, see: + # https://github.com/kubernetes-sigs/secrets-store-csi-driver#install-the-secrets-store-csi-driver + # + # With the driver and provider installed, you can mount Vault secrets into volumes + # similar to the Vault Agent injector, and you can also sync those secrets into + # Kubernetes secrets. + enabled: true + + image: + repository: "hashicorp/vault-csi-provider" + tag: "1.4.1" + pullPolicy: IfNotPresent + + volumes: null + + volumeMounts: null + + resources: {} + + # Override the default secret name for the CSI Provider's HMAC key used for + # generating secret versions. + hmacSecretName: "" + + daemonSet: + updateStrategy: + type: RollingUpdate + maxUnavailable: "" + # Extra annotations for the daemonSet. This can either be YAML or a + # YAML-formatted multi-line templated string map of the annotations to apply + # to the daemonSet. + annotations: {} + # Provider host path (must match the CSI provider's path) + providersDir: "/etc/kubernetes/secrets-store-csi-providers" + # Kubelet host path + kubeletRootDir: "/var/lib/kubelet" + # Extra labels to attach to the vault-csi-provider daemonSet + # This should be a YAML map of the labels to apply to the csi provider daemonSet + extraLabels: {} + # security context for the pod template and container in the csi provider daemonSet + securityContext: + pod: {} + container: {} + + pod: + annotations: {} + tolerations: [] + nodeSelector: {} + affinity: {} + extraLabels: {} + + agent: + enabled: true + extraArgs: [] + + image: + repository: "hashicorp/vault" + tag: "1.15.6" + pullPolicy: IfNotPresent + + logFormat: standard + logLevel: info + + resources: {} + + priorityClassName: "" + + serviceAccount: + annotations: {} + extraLabels: {} + + readinessProbe: + failureThreshold: 2 + initialDelaySeconds: 5 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 3 + + livenessProbe: + failureThreshold: 2 + initialDelaySeconds: 5 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 3 + + debug: false + extraArgs: [] + + serverTelemetry: + # Enable support for the Prometheus Operator. Currently, this chart does not support + # authenticating to Vault's metrics endpoint, so the following `telemetry{}` must be included + # in the `listener "tcp"{}` stanza + # telemetry { + # unauthenticated_metrics_access = "true" + # } + # + # See the `standalone.config` for a more complete example of this. + # + # In addition, a top level `telemetry{}` stanza must also be included in the Vault configuration: + # + # example: + # telemetry { + # prometheus_retention_time = "30s" + # disable_hostname = true + # } + # + # Configuration for monitoring the Vault server. + serviceMonitor: + enabled: true + selectors: {} + interval: 30s + scrapeTimeout: 10s + + prometheusRules: + enabled: true + selectors: {} + rules: [] + + ingress: + enabled: true + accountEmail: reversstorm@gmail.com + class: nginx + annotations: + nginx.ingress.kubernetes.io/proxy-buffer-size: "128k" + nginx.ingress.kubernetes.io/proxy-buffers: "4 256k" + nginx.ingress.kubernetes.io/proxy-busy-buffers-size: "256k" + tls: + enabled: true + hosts: + - host: vault.disk.lt.t1.cloud + secretName: vault-tls + diff --git a/kubespray/inventory/group_vars/k8s_cluster/k8s-cluster.yml b/kubespray/inventory/group_vars/k8s_cluster/k8s-cluster.yml new file mode 100644 index 0000000..76586f8 --- /dev/null +++ b/kubespray/inventory/group_vars/k8s_cluster/k8s-cluster.yml @@ -0,0 +1,380 @@ +--- +# Kubernetes configuration dirs and system namespace. +# Those are where all the additional config stuff goes +# the kubernetes normally puts in /srv/kubernetes. +# This puts them in a sane location and namespace. +# Editing those values will almost surely break something. +kube_config_dir: /etc/kubernetes +kube_script_dir: "{{ bin_dir }}/kubernetes-scripts" +kube_manifest_dir: "{{ kube_config_dir }}/manifests" + +# This is where all the cert scripts and certs will be located +kube_cert_dir: "{{ kube_config_dir }}/ssl" + +# This is where all of the bearer tokens will be stored +kube_token_dir: "{{ kube_config_dir }}/tokens" + +kube_api_anonymous_auth: true + +## Change this to use another Kubernetes version, e.g. a current beta release +kube_version: v1.29.0 + +# Where the binaries will be downloaded. +# Note: ensure that you've enough disk space (about 1G) +local_release_dir: "/tmp/releases" +# Random shifts for retrying failed ops like pushing/downloading +retry_stagger: 5 + +# This is the user that owns tha cluster installation. +kube_owner: kube + +# This is the group that the cert creation scripts chgrp the +# cert files to. Not really changeable... +kube_cert_group: kube-cert + +# Cluster Loglevel configuration +kube_log_level: 2 + +# Directory where credentials will be stored +credentials_dir: "{{ inventory_dir }}/credentials" + +## It is possible to activate / deactivate selected authentication methods (oidc, static token auth) +# kube_oidc_auth: false +# kube_token_auth: false + + +## Variables for OpenID Connect Configuration https://kubernetes.io/docs/admin/authentication/ +## To use OpenID you have to deploy additional an OpenID Provider (e.g Dex, Keycloak, ...) + +# kube_oidc_url: https:// ... +# kube_oidc_client_id: kubernetes +## Optional settings for OIDC +# kube_oidc_ca_file: "{{ kube_cert_dir }}/ca.pem" +# kube_oidc_username_claim: sub +# kube_oidc_username_prefix: 'oidc:' +# kube_oidc_groups_claim: groups +# kube_oidc_groups_prefix: 'oidc:' + +## Variables to control webhook authn/authz +# kube_webhook_token_auth: false +# kube_webhook_token_auth_url: https://... +# kube_webhook_token_auth_url_skip_tls_verify: false + +## For webhook authorization, authorization_modes must include Webhook +# kube_webhook_authorization: false +# kube_webhook_authorization_url: https://... +# kube_webhook_authorization_url_skip_tls_verify: false + +# Choose network plugin (cilium, calico, kube-ovn, weave or flannel. Use cni for generic cni plugin) +# Can also be set to 'cloud', which lets the cloud provider setup appropriate routing +kube_network_plugin: calico + +# Setting multi_networking to true will install Multus: https://github.com/k8snetworkplumbingwg/multus-cni +kube_network_plugin_multus: false + +# Kubernetes internal network for services, unused block of space. +kube_service_addresses: 10.233.0.0/18 + +# internal network. When used, it will assign IP +# addresses from this range to individual pods. +# This network must be unused in your network infrastructure! +kube_pods_subnet: 10.233.64.0/18 + +# internal network node size allocation (optional). This is the size allocated +# to each node for pod IP address allocation. Note that the number of pods per node is +# also limited by the kubelet_max_pods variable which defaults to 110. +# +# Example: +# Up to 64 nodes and up to 254 or kubelet_max_pods (the lowest of the two) pods per node: +# - kube_pods_subnet: 10.233.64.0/18 +# - kube_network_node_prefix: 24 +# - kubelet_max_pods: 110 +# +# Example: +# Up to 128 nodes and up to 126 or kubelet_max_pods (the lowest of the two) pods per node: +# - kube_pods_subnet: 10.233.64.0/18 +# - kube_network_node_prefix: 25 +# - kubelet_max_pods: 110 +kube_network_node_prefix: 24 + +# Configure Dual Stack networking (i.e. both IPv4 and IPv6) +enable_dual_stack_networks: false + +# Kubernetes internal network for IPv6 services, unused block of space. +# This is only used if enable_dual_stack_networks is set to true +# This provides 4096 IPv6 IPs +kube_service_addresses_ipv6: fd85:ee78:d8a6:8607::1000/116 + +# Internal network. When used, it will assign IPv6 addresses from this range to individual pods. +# This network must not already be in your network infrastructure! +# This is only used if enable_dual_stack_networks is set to true. +# This provides room for 256 nodes with 254 pods per node. +kube_pods_subnet_ipv6: fd85:ee78:d8a6:8607::1:0000/112 + +# IPv6 subnet size allocated to each for pods. +# This is only used if enable_dual_stack_networks is set to true +# This provides room for 254 pods per node. +kube_network_node_prefix_ipv6: 120 + +# The port the API Server will be listening on. +kube_apiserver_ip: "{{ kube_service_addresses|ipaddr('net')|ipaddr(1)|ipaddr('address') }}" +kube_apiserver_port: 6443 # (https) + +# Kube-proxy proxyMode configuration. +# Can be ipvs, iptables +kube_proxy_mode: ipvs + +# configure arp_ignore and arp_announce to avoid answering ARP queries from kube-ipvs0 interface +# must be set to true for MetalLB, kube-vip(ARP enabled) to work +kube_proxy_strict_arp: false + +# A string slice of values which specify the addresses to use for NodePorts. +# Values may be valid IP blocks (e.g. 1.2.3.0/24, 1.2.3.4/32). +# The default empty string slice ([]) means to use all local addresses. +# kube_proxy_nodeport_addresses_cidr is retained for legacy config +kube_proxy_nodeport_addresses: >- + {%- if kube_proxy_nodeport_addresses_cidr is defined -%} + [{{ kube_proxy_nodeport_addresses_cidr }}] + {%- else -%} + [] + {%- endif -%} + +# If non-empty, will use this string as identification instead of the actual hostname +# kube_override_hostname: >- +# {%- if cloud_provider is defined and cloud_provider in [ 'aws' ] -%} +# {%- else -%} +# {{ inventory_hostname }} +# {%- endif -%} + +## Encrypting Secret Data at Rest +kube_encrypt_secret_data: false + +# Graceful Node Shutdown (Kubernetes >= 1.21.0), see https://kubernetes.io/blog/2021/04/21/graceful-node-shutdown-beta/ +# kubelet_shutdown_grace_period had to be greater than kubelet_shutdown_grace_period_critical_pods to allow +# non-critical podsa to also terminate gracefully +# kubelet_shutdown_grace_period: 60s +# kubelet_shutdown_grace_period_critical_pods: 20s + +# DNS configuration. +# Kubernetes cluster name, also will be used as DNS domain +cluster_name: k8s-cluster.local +# Subdomains of DNS domain to be resolved via /etc/resolv.conf for hostnet pods +ndots: 2 +# dns_timeout: 2 +# dns_attempts: 2 +# Custom search domains to be added in addition to the default cluster search domains +# searchdomains: +# - "svc.{{ cluster_name }}" +# - "default.svc.{{ cluster_name }}" +# remove_default_searchdomains: false +# Can be coredns, coredns_dual, manual or none +dns_mode: coredns +# Set manual server if using a custom cluster DNS server +# manual_dns_server: 10.x.x.x +# Enable nodelocal dns cache +enable_nodelocaldns: true +enable_nodelocaldns_secondary: false +nodelocaldns_ip: 169.254.25.10 +nodelocaldns_health_port: 9254 +nodelocaldns_second_health_port: 9256 +nodelocaldns_bind_metrics_host_ip: false +nodelocaldns_secondary_skew_seconds: 5 +# nodelocaldns_external_zones: +# - zones: +# - example.com +# - example.io:1053 +# nameservers: +# - 1.1.1.1 +# - 2.2.2.2 +# cache: 5 +# - zones: +# - https://mycompany.local:4453 +# nameservers: +# - 192.168.0.53 +# cache: 0 +# - zones: +# - mydomain.tld +# nameservers: +# - 10.233.0.3 +# cache: 5 +# rewrite: +# - name website.tld website.namespace.svc.cluster.local +# Enable k8s_external plugin for CoreDNS +enable_coredns_k8s_external: false +coredns_k8s_external_zone: k8s_external.local +# Enable endpoint_pod_names option for kubernetes plugin +enable_coredns_k8s_endpoint_pod_names: false +# Set forward options for upstream DNS servers in coredns (and nodelocaldns) config +# dns_upstream_forward_extra_opts: +# policy: sequential +# Apply extra options to coredns kubernetes plugin +# coredns_kubernetes_extra_opts: +# - 'fallthrough example.local' +# Forward extra domains to the coredns kubernetes plugin +# coredns_kubernetes_extra_domains: '' + +# Can be docker_dns, host_resolvconf or none +resolvconf_mode: host_resolvconf +# Deploy netchecker app to verify DNS resolve as an HTTP service +deploy_netchecker: false +# Ip address of the kubernetes skydns service +skydns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(3)|ipaddr('address') }}" +skydns_server_secondary: "{{ kube_service_addresses|ipaddr('net')|ipaddr(4)|ipaddr('address') }}" +dns_domain: "{{ cluster_name }}" + +## Container runtime +## docker for docker, crio for cri-o and containerd for containerd. +## Default: containerd +container_manager: containerd + +# Additional container runtimes +kata_containers_enabled: false + +kubeadm_certificate_key: "{{ lookup('password', credentials_dir + '/kubeadm_certificate_key.creds length=64 chars=hexdigits') | lower }}" + +# K8s image pull policy (imagePullPolicy) +k8s_image_pull_policy: IfNotPresent + +# audit log for kubernetes +kubernetes_audit: false + +# define kubelet config dir for dynamic kubelet +# kubelet_config_dir: +default_kubelet_config_dir: "{{ kube_config_dir }}/dynamic_kubelet_dir" + +# pod security policy (RBAC must be enabled either by having 'RBAC' in authorization_modes or kubeadm enabled) +podsecuritypolicy_enabled: true + +# Custom PodSecurityPolicySpec for restricted policy +# podsecuritypolicy_restricted_spec: {} + +# Custom PodSecurityPolicySpec for privileged policy +# podsecuritypolicy_privileged_spec: {} + +# Make a copy of kubeconfig on the host that runs Ansible in {{ inventory_dir }}/artifacts +kubeconfig_localhost: true +# Use ansible_host as external api ip when copying over kubeconfig. +kubeconfig_localhost_ansible_host: true +# kubectl_localhost: false + +# A comma separated list of levels of node allocatable enforcement to be enforced by kubelet. +# Acceptable options are 'pods', 'system-reserved', 'kube-reserved' and ''. Default is "". +# kubelet_enforce_node_allocatable: pods + +## Set runtime and kubelet cgroups when using systemd as cgroup driver (default) +# kubelet_runtime_cgroups: "/{{ kube_service_cgroups }}/{{ container_manager }}.service" +# kubelet_kubelet_cgroups: "/{{ kube_service_cgroups }}/kubelet.service" + +## Set runtime and kubelet cgroups when using cgroupfs as cgroup driver +# kubelet_runtime_cgroups_cgroupfs: "/system.slice/{{ container_manager }}.service" +# kubelet_kubelet_cgroups_cgroupfs: "/system.slice/kubelet.service" + +# Optionally reserve this space for kube daemons. +# kube_reserved: false +## Uncomment to override default values +## The following two items need to be set when kube_reserved is true +# kube_reserved_cgroups_for_service_slice: kube.slice +# kube_reserved_cgroups: "/{{ kube_reserved_cgroups_for_service_slice }}" +# kube_memory_reserved: 256Mi +# kube_cpu_reserved: 100m +# kube_ephemeral_storage_reserved: 2Gi +# kube_pid_reserved: "1000" +# Reservation for master hosts +# kube_master_memory_reserved: 512Mi +# kube_master_cpu_reserved: 200m +# kube_master_ephemeral_storage_reserved: 2Gi +# kube_master_pid_reserved: "1000" + +## Optionally reserve resources for OS system daemons. +# system_reserved: true +## Uncomment to override default values +## The following two items need to be set when system_reserved is true +# system_reserved_cgroups_for_service_slice: system.slice +# system_reserved_cgroups: "/{{ system_reserved_cgroups_for_service_slice }}" +# system_memory_reserved: 512Mi +# system_cpu_reserved: 500m +# system_ephemeral_storage_reserved: 2Gi +## Reservation for master hosts +# system_master_memory_reserved: 256Mi +# system_master_cpu_reserved: 250m +# system_master_ephemeral_storage_reserved: 2Gi + +## Eviction Thresholds to avoid system OOMs +# https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/#eviction-thresholds +# eviction_hard: {} +# eviction_hard_control_plane: {} + +# An alternative flexvolume plugin directory +# kubelet_flexvolumes_plugins_dir: /usr/libexec/kubernetes/kubelet-plugins/volume/exec + +## Supplementary addresses that can be added in kubernetes ssl keys. +## That can be useful for example to setup a keepalived virtual IP +# supplementary_addresses_in_ssl_keys: [10.0.0.1, 10.0.0.2, 10.0.0.3] + +## Running on top of openstack vms with cinder enabled may lead to unschedulable pods due to NoVolumeZoneConflict restriction in kube-scheduler. +## See https://github.com/kubernetes-sigs/kubespray/issues/2141 +## Set this variable to true to get rid of this issue +volume_cross_zone_attachment: false +## Add Persistent Volumes Storage Class for corresponding cloud provider (supported: in-tree OpenStack, Cinder CSI, +## AWS EBS CSI, Azure Disk CSI, GCP Persistent Disk CSI) +persistent_volumes_enabled: false + +## Container Engine Acceleration +## Enable container acceleration feature, for example use gpu acceleration in containers +# nvidia_accelerator_enabled: true +## Nvidia GPU driver install. Install will by done by a (init) pod running as a daemonset. +## Important: if you use Ubuntu then you should set in all.yml 'docker_storage_options: -s overlay2' +## Array with nvida_gpu_nodes, leave empty or comment if you don't want to install drivers. +## Labels and taints won't be set to nodes if they are not in the array. +# nvidia_gpu_nodes: +# - kube-gpu-001 +# nvidia_driver_version: "384.111" +## flavor can be tesla or gtx +# nvidia_gpu_flavor: gtx +## NVIDIA driver installer images. Change them if you have trouble accessing gcr.io. +# nvidia_driver_install_centos_container: atzedevries/nvidia-centos-driver-installer:2 +# nvidia_driver_install_ubuntu_container: gcr.io/google-containers/ubuntu-nvidia-driver-installer@sha256:7df76a0f0a17294e86f691c81de6bbb7c04a1b4b3d4ea4e7e2cccdc42e1f6d63 +## NVIDIA GPU device plugin image. +# nvidia_gpu_device_plugin_container: "registry.k8s.io/nvidia-gpu-device-plugin@sha256:0842734032018be107fa2490c98156992911e3e1f2a21e059ff0105b07dd8e9e" + +## Support tls min version, Possible values: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13. +# tls_min_version: "" + +## Support tls cipher suites. +# tls_cipher_suites: {} +# - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA +# - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256 +# - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 +# - TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA +# - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 +# - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 +# - TLS_ECDHE_ECDSA_WITH_RC4_128_SHA +# - TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA +# - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA +# - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 +# - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 +# - TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA +# - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 +# - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 +# - TLS_ECDHE_RSA_WITH_RC4_128_SHA +# - TLS_RSA_WITH_3DES_EDE_CBC_SHA +# - TLS_RSA_WITH_AES_128_CBC_SHA +# - TLS_RSA_WITH_AES_128_CBC_SHA256 +# - TLS_RSA_WITH_AES_128_GCM_SHA256 +# - TLS_RSA_WITH_AES_256_CBC_SHA +# - TLS_RSA_WITH_AES_256_GCM_SHA384 +# - TLS_RSA_WITH_RC4_128_SHA + +## Amount of time to retain events. (default 1h0m0s) +event_ttl_duration: "1h0m0s" + +## Automatically renew K8S control plane certificates on first Monday of each month +auto_renew_certificates: true +# First Monday of each month +# auto_renew_certificates_systemd_calendar: "Mon *-*-1,2,3,4,5,6,7 03:{{ groups['kube_control_plane'].index(inventory_hostname) }}0:00" + +# kubeadm patches path +kubeadm_patches: + enabled: false + source_dir: "{{ inventory_dir }}/patches" + dest_dir: "{{ kube_config_dir }}/patches" diff --git a/kubespray/inventory/group_vars/k8s_cluster/k8s-net-calico.yml b/kubespray/inventory/group_vars/k8s_cluster/k8s-net-calico.yml new file mode 100644 index 0000000..cc0499d --- /dev/null +++ b/kubespray/inventory/group_vars/k8s_cluster/k8s-net-calico.yml @@ -0,0 +1,131 @@ +--- +# see roles/network_plugin/calico/defaults/main.yml + +# the default value of name +calico_cni_name: k8s-pod-network + +## With calico it is possible to distributed routes with border routers of the datacenter. +## Warning : enabling router peering will disable calico's default behavior ('node mesh'). +## The subnets of each nodes will be distributed by the datacenter router +# peer_with_router: false + +# Enables Internet connectivity from containers +# nat_outgoing: true + +# Enables Calico CNI "host-local" IPAM plugin +# calico_ipam_host_local: true + +# add default ippool name +# calico_pool_name: "default-pool" + +# add default ippool blockSize (defaults kube_network_node_prefix) +calico_pool_blocksize: 26 + +# add default ippool CIDR (must be inside kube_pods_subnet, defaults to kube_pods_subnet otherwise) +# calico_pool_cidr: 1.2.3.4/5 + +# add default ippool CIDR to CNI config +# calico_cni_pool: true + +# Add default IPV6 IPPool CIDR. Must be inside kube_pods_subnet_ipv6. Defaults to kube_pods_subnet_ipv6 if not set. +# calico_pool_cidr_ipv6: fd85:ee78:d8a6:8607::1:0000/112 + +# Add default IPV6 IPPool CIDR to CNI config +# calico_cni_pool_ipv6: true + +# Global as_num (/calico/bgp/v1/global/as_num) +# global_as_num: "64512" + +# If doing peering with node-assigned asn where the globas does not match your nodes, you want this +# to be true. All other cases, false. +# calico_no_global_as_num: false + +# You can set MTU value here. If left undefined or empty, it will +# not be specified in calico CNI config, so Calico will use built-in +# defaults. The value should be a number, not a string. +# calico_mtu: 1500 + +# Configure the MTU to use for workload interfaces and tunnels. +# - If Wireguard is enabled, subtract 60 from your network MTU (i.e 1500-60=1440) +# - Otherwise, if VXLAN or BPF mode is enabled, subtract 50 from your network MTU (i.e. 1500-50=1450) +# - Otherwise, if IPIP is enabled, subtract 20 from your network MTU (i.e. 1500-20=1480) +# - Otherwise, if not using any encapsulation, set to your network MTU (i.e. 1500) +# calico_veth_mtu: 1440 + +# Advertise Cluster IPs +# calico_advertise_cluster_ips: true + +# Advertise Service External IPs +# calico_advertise_service_external_ips: +# - x.x.x.x/24 +# - y.y.y.y/32 + +# Advertise Service LoadBalancer IPs +# calico_advertise_service_loadbalancer_ips: +# - x.x.x.x/24 +# - y.y.y.y/16 + +# Choose data store type for calico: "etcd" or "kdd" (kubernetes datastore) +# calico_datastore: "kdd" + +# Choose Calico iptables backend: "Legacy", "Auto" or "NFT" +# calico_iptables_backend: "Auto" + +# Use typha (only with kdd) +# typha_enabled: false + +# Generate TLS certs for secure typha<->calico-node communication +# typha_secure: false + +# Scaling typha: 1 replica per 100 nodes is adequate +# Number of typha replicas +# typha_replicas: 1 + +# Set max typha connections +# typha_max_connections_lower_limit: 300 + +# Set calico network backend: "bird", "vxlan" or "none" +# bird enable BGP routing, required for ipip and no encapsulation modes +# calico_network_backend: vxlan + +# IP in IP and VXLAN is mutualy exclusive modes. +# set IP in IP encapsulation mode: "Always", "CrossSubnet", "Never" +# calico_ipip_mode: 'Never' + +# set VXLAN encapsulation mode: "Always", "CrossSubnet", "Never" +# calico_vxlan_mode: 'Always' + +# set VXLAN port and VNI +# calico_vxlan_vni: 4096 +# calico_vxlan_port: 4789 + +# Enable eBPF mode +# calico_bpf_enabled: false + +# If you want to use non default IP_AUTODETECTION_METHOD, IP6_AUTODETECTION_METHOD for calico node set this option to one of: +# * can-reach=DESTINATION +# * interface=INTERFACE-REGEX +# see https://docs.projectcalico.org/reference/node/configuration +# calico_ip_auto_method: "interface=eth.*" +# calico_ip6_auto_method: "interface=eth.*" + +# Set FELIX_MTUIFACEPATTERN, Pattern used to discover the host’s interface for MTU auto-detection. +# see https://projectcalico.docs.tigera.io/reference/felix/configuration +# calico_felix_mtu_iface_pattern: "^((en|wl|ww|sl|ib)[opsx].*|(eth|wlan|wwan).*)" + +# Choose the iptables insert mode for Calico: "Insert" or "Append". +# calico_felix_chaininsertmode: Insert + +# If you want use the default route interface when you use multiple interface with dynamique route (iproute2) +# see https://docs.projectcalico.org/reference/node/configuration : FELIX_DEVICEROUTESOURCEADDRESS +# calico_use_default_route_src_ipaddr: false + +# Enable calico traffic encryption with wireguard +# calico_wireguard_enabled: false + +# Under certain situations liveness and readiness probes may need tunning +# calico_node_livenessprobe_timeout: 10 +# calico_node_readinessprobe_timeout: 10 + +# Calico apiserver (only with kdd) +# calico_apiserver_enabled: false diff --git a/kubespray/inventory/group_vars/k8s_cluster/k8s-net-cilium.yml b/kubespray/inventory/group_vars/k8s_cluster/k8s-net-cilium.yml new file mode 100644 index 0000000..9023e09 --- /dev/null +++ b/kubespray/inventory/group_vars/k8s_cluster/k8s-net-cilium.yml @@ -0,0 +1,245 @@ +--- +# cilium_version: "v1.12.1" + +# Log-level +# cilium_debug: false + +# cilium_mtu: "" +# cilium_enable_ipv4: true +# cilium_enable_ipv6: false + +# Cilium agent health port +# cilium_agent_health_port: "9879" + +# Identity allocation mode selects how identities are shared between cilium +# nodes by setting how they are stored. The options are "crd" or "kvstore". +# - "crd" stores identities in kubernetes as CRDs (custom resource definition). +# These can be queried with: +# `kubectl get ciliumid` +# - "kvstore" stores identities in an etcd kvstore. +# - In order to support External Workloads, "crd" is required +# - Ref: https://docs.cilium.io/en/stable/gettingstarted/external-workloads/#setting-up-support-for-external-workloads-beta +# - KVStore operations are only required when cilium-operator is running with any of the below options: +# - --synchronize-k8s-services +# - --synchronize-k8s-nodes +# - --identity-allocation-mode=kvstore +# - Ref: https://docs.cilium.io/en/stable/internals/cilium_operator/#kvstore-operations +# cilium_identity_allocation_mode: kvstore + +# Etcd SSL dirs +# cilium_cert_dir: /etc/cilium/certs +# kube_etcd_cacert_file: ca.pem +# kube_etcd_cert_file: cert.pem +# kube_etcd_key_file: cert-key.pem + +# Limits for apps +# cilium_memory_limit: 500M +# cilium_cpu_limit: 500m +# cilium_memory_requests: 64M +# cilium_cpu_requests: 100m + +# Overlay Network Mode +# cilium_tunnel_mode: vxlan +# Optional features +# cilium_enable_prometheus: false +# Enable if you want to make use of hostPort mappings +# cilium_enable_portmap: false +# Monitor aggregation level (none/low/medium/maximum) +# cilium_monitor_aggregation: medium +# The monitor aggregation flags determine which TCP flags which, upon the +# first observation, cause monitor notifications to be generated. +# +# Only effective when monitor aggregation is set to "medium" or higher. +# cilium_monitor_aggregation_flags: "all" +# Kube Proxy Replacement mode (strict/partial) +# cilium_kube_proxy_replacement: partial + +# If upgrading from Cilium < 1.5, you may want to override some of these options +# to prevent service disruptions. See also: +# http://docs.cilium.io/en/stable/install/upgrade/#changes-that-may-require-action +# cilium_preallocate_bpf_maps: false + +# `cilium_tofqdns_enable_poller` is deprecated in 1.8, removed in 1.9 +# cilium_tofqdns_enable_poller: false + +# `cilium_enable_legacy_services` is deprecated in 1.6, removed in 1.9 +# cilium_enable_legacy_services: false + +# Unique ID of the cluster. Must be unique across all conneted clusters and +# in the range of 1 and 255. Only relevant when building a mesh of clusters. +# This value is not defined by default +# cilium_cluster_id: + +# Deploy cilium even if kube_network_plugin is not cilium. +# This enables to deploy cilium alongside another CNI to replace kube-proxy. +# cilium_deploy_additionally: false + +# Auto direct nodes routes can be used to advertise pods routes in your cluster +# without any tunelling (with `cilium_tunnel_mode` sets to `disabled`). +# This works only if you have a L2 connectivity between all your nodes. +# You wil also have to specify the variable `cilium_native_routing_cidr` to +# make this work. Please refer to the cilium documentation for more +# information about this kind of setups. +# cilium_auto_direct_node_routes: false + +# Allows to explicitly specify the IPv4 CIDR for native routing. +# When specified, Cilium assumes networking for this CIDR is preconfigured and +# hands traffic destined for that range to the Linux network stack without +# applying any SNAT. +# Generally speaking, specifying a native routing CIDR implies that Cilium can +# depend on the underlying networking stack to route packets to their +# destination. To offer a concrete example, if Cilium is configured to use +# direct routing and the Kubernetes CIDR is included in the native routing CIDR, +# the user must configure the routes to reach pods, either manually or by +# setting the auto-direct-node-routes flag. +# cilium_native_routing_cidr: "" + +# Allows to explicitly specify the IPv6 CIDR for native routing. +# cilium_native_routing_cidr_ipv6: "" + +# Enable transparent network encryption. +# cilium_encryption_enabled: false + +# Encryption method. Can be either ipsec or wireguard. +# Only effective when `cilium_encryption_enabled` is set to true. +# cilium_encryption_type: "ipsec" + +# Enable encryption for pure node to node traffic. +# This option is only effective when `cilium_encryption_type` is set to `ipsec`. +# cilium_ipsec_node_encryption: false + +# If your kernel or distribution does not support WireGuard, Cilium agent can be configured to fall back on the user-space implementation. +# When this flag is enabled and Cilium detects that the kernel has no native support for WireGuard, +# it will fallback on the wireguard-go user-space implementation of WireGuard. +# This option is only effective when `cilium_encryption_type` is set to `wireguard`. +# cilium_wireguard_userspace_fallback: false + +# IP Masquerade Agent +# https://docs.cilium.io/en/stable/concepts/networking/masquerading/ +# By default, all packets from a pod destined to an IP address outside of the cilium_native_routing_cidr range are masqueraded +# cilium_ip_masq_agent_enable: false + +### A packet sent from a pod to a destination which belongs to any CIDR from the nonMasqueradeCIDRs is not going to be masqueraded +# cilium_non_masquerade_cidrs: +# - 10.0.0.0/8 +# - 172.16.0.0/12 +# - 192.168.0.0/16 +# - 100.64.0.0/10 +# - 192.0.0.0/24 +# - 192.0.2.0/24 +# - 192.88.99.0/24 +# - 198.18.0.0/15 +# - 198.51.100.0/24 +# - 203.0.113.0/24 +# - 240.0.0.0/4 +### Indicates whether to masquerade traffic to the link local prefix. +### If the masqLinkLocal is not set or set to false, then 169.254.0.0/16 is appended to the non-masquerade CIDRs list. +# cilium_masq_link_local: false +### A time interval at which the agent attempts to reload config from disk +# cilium_ip_masq_resync_interval: 60s + +# Hubble +### Enable Hubble without install +# cilium_enable_hubble: false +### Enable Hubble Metrics +# cilium_enable_hubble_metrics: false +### if cilium_enable_hubble_metrics: true +# cilium_hubble_metrics: {} +# - dns +# - drop +# - tcp +# - flow +# - icmp +# - http +### Enable Hubble install +# cilium_hubble_install: false +### Enable auto generate certs if cilium_hubble_install: true +# cilium_hubble_tls_generate: false + +# IP address management mode for v1.9+. +# https://docs.cilium.io/en/v1.9/concepts/networking/ipam/ +# cilium_ipam_mode: kubernetes + +# Extra arguments for the Cilium agent +# cilium_agent_custom_args: [] + +# For adding and mounting extra volumes to the cilium agent +# cilium_agent_extra_volumes: [] +# cilium_agent_extra_volume_mounts: [] + +# cilium_agent_extra_env_vars: [] + +# cilium_operator_replicas: 2 + +# The address at which the cillium operator bind health check api +# cilium_operator_api_serve_addr: "127.0.0.1:9234" + +## A dictionary of extra config variables to add to cilium-config, formatted like: +## cilium_config_extra_vars: +## var1: "value1" +## var2: "value2" +# cilium_config_extra_vars: {} + +# For adding and mounting extra volumes to the cilium operator +# cilium_operator_extra_volumes: [] +# cilium_operator_extra_volume_mounts: [] + +# Extra arguments for the Cilium Operator +# cilium_operator_custom_args: [] + +# Name of the cluster. Only relevant when building a mesh of clusters. +# cilium_cluster_name: default + +# Make Cilium take ownership over the `/etc/cni/net.d` directory on the node, renaming all non-Cilium CNI configurations to `*.cilium_bak`. +# This ensures no Pods can be scheduled using other CNI plugins during Cilium agent downtime. +# Available for Cilium v1.10 and up. +# cilium_cni_exclusive: true + +# Configure the log file for CNI logging with retention policy of 7 days. +# Disable CNI file logging by setting this field to empty explicitly. +# Available for Cilium v1.12 and up. +# cilium_cni_log_file: "/var/run/cilium/cilium-cni.log" + +# -- Configure cgroup related configuration +# -- Enable auto mount of cgroup2 filesystem. +# When `cilium_cgroup_auto_mount` is enabled, cgroup2 filesystem is mounted at +# `cilium_cgroup_host_root` path on the underlying host and inside the cilium agent pod. +# If users disable `cilium_cgroup_auto_mount`, it's expected that users have mounted +# cgroup2 filesystem at the specified `cilium_cgroup_auto_mount` volume, and then the +# volume will be mounted inside the cilium agent pod at the same path. +# Available for Cilium v1.11 and up +# cilium_cgroup_auto_mount: true +# -- Configure cgroup root where cgroup2 filesystem is mounted on the host +# cilium_cgroup_host_root: "/run/cilium/cgroupv2" + +# Specifies the ratio (0.0-1.0) of total system memory to use for dynamic +# sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps. +# cilium_bpf_map_dynamic_size_ratio: "0.0" + +# -- Enables masquerading of IPv4 traffic leaving the node from endpoints. +# Available for Cilium v1.10 and up +# cilium_enable_ipv4_masquerade: true +# -- Enables masquerading of IPv6 traffic leaving the node from endpoints. +# Available for Cilium v1.10 and up +# cilium_enable_ipv6_masquerade: true + +# -- Enable native IP masquerade support in eBPF +# cilium_enable_bpf_masquerade: false + +# -- Configure whether direct routing mode should route traffic via +# host stack (true) or directly and more efficiently out of BPF (false) if +# the kernel supports it. The latter has the implication that it will also +# bypass netfilter in the host namespace. +# cilium_enable_host_legacy_routing: true + +# -- Enable use of the remote node identity. +# ref: https://docs.cilium.io/en/v1.7/install/upgrade/#configmap-remote-node-identity +# cilium_enable_remote_node_identity: true + +# -- Enable the use of well-known identities. +# cilium_enable_well_known_identities: false + +# cilium_enable_bpf_clock_probe: true + +# -- Whether to enable CNP status updates. +# cilium_disable_cnp_status_updates: true diff --git a/kubespray/inventory/group_vars/k8s_cluster/k8s-net-flannel.yml b/kubespray/inventory/group_vars/k8s_cluster/k8s-net-flannel.yml new file mode 100644 index 0000000..64d20a8 --- /dev/null +++ b/kubespray/inventory/group_vars/k8s_cluster/k8s-net-flannel.yml @@ -0,0 +1,18 @@ +# see roles/network_plugin/flannel/defaults/main.yml + +## interface that should be used for flannel operations +## This is actually an inventory cluster-level item +# flannel_interface: + +## Select interface that should be used for flannel operations by regexp on Name or IP +## This is actually an inventory cluster-level item +## example: select interface with ip from net 10.0.0.0/23 +## single quote and escape backslashes +# flannel_interface_regexp: '10\\.0\\.[0-2]\\.\\d{1,3}' + +# You can choose what type of flannel backend to use: 'vxlan', 'host-gw' or 'wireguard' +# please refer to flannel's docs : https://github.com/coreos/flannel/blob/master/README.md +# flannel_backend_type: "vxlan" +# flannel_vxlan_vni: 1 +# flannel_vxlan_port: 8472 +# flannel_vxlan_direct_routing: false diff --git a/kubespray/inventory/group_vars/k8s_cluster/k8s-net-kube-ovn.yml b/kubespray/inventory/group_vars/k8s_cluster/k8s-net-kube-ovn.yml new file mode 100644 index 0000000..c241a76 --- /dev/null +++ b/kubespray/inventory/group_vars/k8s_cluster/k8s-net-kube-ovn.yml @@ -0,0 +1,63 @@ +--- + +# geneve or vlan +kube_ovn_network_type: geneve + +# geneve, vxlan or stt. ATTENTION: some networkpolicy cannot take effect when using vxlan and stt need custom compile ovs kernel module +kube_ovn_tunnel_type: geneve + +## The nic to support container network can be a nic name or a group of regex separated by comma e.g: 'enp6s0f0,eth.*', if empty will use the nic that the default route use. +# kube_ovn_iface: eth1 +## The MTU used by pod iface in overlay networks (default iface MTU - 100) +# kube_ovn_mtu: 1333 + +## Enable hw-offload, disable traffic mirror and set the iface to the physical port. Make sure that there is an IP address bind to the physical port. +kube_ovn_hw_offload: false +# traffic mirror +kube_ovn_traffic_mirror: false + +# kube_ovn_pool_cidr_ipv6: fd85:ee78:d8a6:8607::1:0000/112 +# kube_ovn_default_interface_name: eth0 + +kube_ovn_external_address: 8.8.8.8 +kube_ovn_external_address_ipv6: 2400:3200::1 +kube_ovn_external_dns: alauda.cn + +# kube_ovn_default_gateway: 10.233.64.1,fd85:ee78:d8a6:8607::1:0 +kube_ovn_default_gateway_check: true +kube_ovn_default_logical_gateway: false +# kube_ovn_default_exclude_ips: 10.16.0.1 +kube_ovn_node_switch_cidr: 100.64.0.0/16 +kube_ovn_node_switch_cidr_ipv6: fd00:100:64::/64 + +## vlan config, set default interface name and vlan id +# kube_ovn_default_interface_name: eth0 +kube_ovn_default_vlan_id: 100 +kube_ovn_vlan_name: product + +## pod nic type, support: veth-pair or internal-port +kube_ovn_pod_nic_type: veth_pair + +## Enable load balancer +kube_ovn_enable_lb: true + +## Enable network policy support +kube_ovn_enable_np: true + +## Enable external vpc support +kube_ovn_enable_external_vpc: true + +## Enable checksum +kube_ovn_encap_checksum: true + +## enable ssl +kube_ovn_enable_ssl: false + +## dpdk +kube_ovn_dpdk_enabled: false + +## enable interconnection to an existing IC database server. +kube_ovn_ic_enable: false +kube_ovn_ic_autoroute: true +kube_ovn_ic_dbhost: "127.0.0.1" +kube_ovn_ic_zone: "kubernetes" diff --git a/kubespray/inventory/group_vars/k8s_cluster/k8s-net-kube-router.yml b/kubespray/inventory/group_vars/k8s_cluster/k8s-net-kube-router.yml new file mode 100644 index 0000000..e4dfcc9 --- /dev/null +++ b/kubespray/inventory/group_vars/k8s_cluster/k8s-net-kube-router.yml @@ -0,0 +1,64 @@ +# See roles/network_plugin/kube-router//defaults/main.yml + +# Enables Pod Networking -- Advertises and learns the routes to Pods via iBGP +# kube_router_run_router: true + +# Enables Network Policy -- sets up iptables to provide ingress firewall for pods +# kube_router_run_firewall: true + +# Enables Service Proxy -- sets up IPVS for Kubernetes Services +# see docs/kube-router.md "Caveats" section +# kube_router_run_service_proxy: false + +# Add Cluster IP of the service to the RIB so that it gets advertises to the BGP peers. +# kube_router_advertise_cluster_ip: false + +# Add External IP of service to the RIB so that it gets advertised to the BGP peers. +# kube_router_advertise_external_ip: false + +# Add LoadBalancer IP of service status as set by the LB provider to the RIB so that it gets advertised to the BGP peers. +# kube_router_advertise_loadbalancer_ip: false + +# Adjust manifest of kube-router daemonset template with DSR needed changes +# kube_router_enable_dsr: false + +# Array of arbitrary extra arguments to kube-router, see +# https://github.com/cloudnativelabs/kube-router/blob/master/docs/user-guide.md +# kube_router_extra_args: [] + +# ASN number of the cluster, used when communicating with external BGP routers +# kube_router_cluster_asn: ~ + +# ASN numbers of the BGP peer to which cluster nodes will advertise cluster ip and node's pod cidr. +# kube_router_peer_router_asns: ~ + +# The ip address of the external router to which all nodes will peer and advertise the cluster ip and pod cidr's. +# kube_router_peer_router_ips: ~ + +# The remote port of the external BGP to which all nodes will peer. If not set, default BGP port (179) will be used. +# kube_router_peer_router_ports: ~ + +# Setups node CNI to allow hairpin mode, requires node reboots, see +# https://github.com/cloudnativelabs/kube-router/blob/master/docs/user-guide.md#hairpin-mode +# kube_router_support_hairpin_mode: false + +# Select DNS Policy ClusterFirstWithHostNet, ClusterFirst, etc. +# kube_router_dns_policy: ClusterFirstWithHostNet + +# Array of annotations for master +# kube_router_annotations_master: [] + +# Array of annotations for every node +# kube_router_annotations_node: [] + +# Array of common annotations for every node +# kube_router_annotations_all: [] + +# Enables scraping kube-router metrics with Prometheus +# kube_router_enable_metrics: false + +# Path to serve Prometheus metrics on +# kube_router_metrics_path: /metrics + +# Prometheus metrics port to use +# kube_router_metrics_port: 9255 diff --git a/kubespray/inventory/group_vars/k8s_cluster/k8s-net-macvlan.yml b/kubespray/inventory/group_vars/k8s_cluster/k8s-net-macvlan.yml new file mode 100644 index 0000000..d2534e7 --- /dev/null +++ b/kubespray/inventory/group_vars/k8s_cluster/k8s-net-macvlan.yml @@ -0,0 +1,6 @@ +--- +# private interface, on a l2-network +macvlan_interface: "eth1" + +# Enable nat in default gateway network interface +enable_nat_default_gateway: true diff --git a/kubespray/inventory/group_vars/k8s_cluster/k8s-net-weave.yml b/kubespray/inventory/group_vars/k8s_cluster/k8s-net-weave.yml new file mode 100644 index 0000000..269a77c --- /dev/null +++ b/kubespray/inventory/group_vars/k8s_cluster/k8s-net-weave.yml @@ -0,0 +1,64 @@ +# see roles/network_plugin/weave/defaults/main.yml + +# Weave's network password for encryption, if null then no network encryption. +# weave_password: ~ + +# If set to 1, disable checking for new Weave Net versions (default is blank, +# i.e. check is enabled) +# weave_checkpoint_disable: false + +# Soft limit on the number of connections between peers. Defaults to 100. +# weave_conn_limit: 100 + +# Weave Net defaults to enabling hairpin on the bridge side of the veth pair +# for containers attached. If you need to disable hairpin, e.g. your kernel is +# one of those that can panic if hairpin is enabled, then you can disable it by +# setting `HAIRPIN_MODE=false`. +# weave_hairpin_mode: true + +# The range of IP addresses used by Weave Net and the subnet they are placed in +# (CIDR format; default 10.32.0.0/12) +# weave_ipalloc_range: "{{ kube_pods_subnet }}" + +# Set to 0 to disable Network Policy Controller (default is on) +# weave_expect_npc: "{{ enable_network_policy }}" + +# List of addresses of peers in the Kubernetes cluster (default is to fetch the +# list from the api-server) +# weave_kube_peers: ~ + +# Set the initialization mode of the IP Address Manager (defaults to consensus +# amongst the KUBE_PEERS) +# weave_ipalloc_init: ~ + +# Set the IP address used as a gateway from the Weave network to the host +# network - this is useful if you are configuring the addon as a static pod. +# weave_expose_ip: ~ + +# Address and port that the Weave Net daemon will serve Prometheus-style +# metrics on (defaults to 0.0.0.0:6782) +# weave_metrics_addr: ~ + +# Address and port that the Weave Net daemon will serve status requests on +# (defaults to disabled) +# weave_status_addr: ~ + +# Weave Net defaults to 1376 bytes, but you can set a smaller size if your +# underlying network has a tighter limit, or set a larger size for better +# performance if your network supports jumbo frames (e.g. 8916) +# weave_mtu: 1376 + +# Set to 1 to preserve the client source IP address when accessing Service +# annotated with `service.spec.externalTrafficPolicy=Local`. The feature works +# only with Weave IPAM (default). +# weave_no_masq_local: true + +# set to nft to use nftables backend for iptables (default is iptables) +# weave_iptables_backend: iptables + +# Extra variables that passing to launch.sh, useful for enabling seed mode, see +# https://www.weave.works/docs/net/latest/tasks/ipam/ipam/ +# weave_extra_args: ~ + +# Extra variables for weave_npc that passing to launch.sh, useful for change log level, ex --log-level=error +# weave_npc_extra_args: ~ diff --git a/kubespray/inventory/hosts b/kubespray/inventory/hosts new file mode 100644 index 0000000..65af680 --- /dev/null +++ b/kubespray/inventory/hosts @@ -0,0 +1,23 @@ + +vbox-prod-k8s-master-01 ansible_host=10.250.50.22 ip=10.250.50.22 +vbox-prod-k8s-slave-01 ansible_host=10.250.50.23 ip=10.250.50.23 +vbox-prod-k8s-slave-02 ansible_host=10.250.50.21 ip=10.250.50.21 + +[kube_control_plane] +vbox-prod-k8s-master-01 + +[etcd] +vbox-prod-k8s-master-01 + +[kube_node] +vbox-prod-k8s-slave-01 +vbox-prod-k8s-slave-02 + +[k8s_cluster:children] +kube_control_plane +kube_node + +[all:vars] +ansible_connection=ssh +ansible_user=sre-admin +ansible_ssh_private_key_file=/home/sre-admin/.ssh/id_rsa diff --git a/kubespray/project/ansible_version.yml b/kubespray/project/ansible_version.yml new file mode 100644 index 0000000..d5a83bd --- /dev/null +++ b/kubespray/project/ansible_version.yml @@ -0,0 +1,34 @@ +--- +- name: Check Ansible version + hosts: all + gather_facts: false + become: no + run_once: true + vars: + minimal_ansible_version: 2.15.5 # 2.15 versions before 2.15.5 are known to be buggy for kubespray + maximal_ansible_version: 2.17.0 + tags: always + tasks: + - name: "Check {{ minimal_ansible_version }} <= Ansible version < {{ maximal_ansible_version }}" + assert: + msg: "Ansible must be between {{ minimal_ansible_version }} and {{ maximal_ansible_version }} exclusive - you have {{ ansible_version.string }}" + that: + - ansible_version.string is version(minimal_ansible_version, ">=") + - ansible_version.string is version(maximal_ansible_version, "<") + tags: + - check + + - name: "Check that python netaddr is installed" + assert: + msg: "Python netaddr is not present" + that: "'127.0.0.1' | ansible.utils.ipaddr" + tags: + - check + + # CentOS 7 provides too old jinja version + - name: "Check that jinja is not too old (install via pip)" + assert: + msg: "Your Jinja version is too old, install via pip" + that: "{% set test %}It works{% endset %}{{ test == 'It works' }}" + tags: + - check diff --git a/kubespray/project/boilerplate.yml b/kubespray/project/boilerplate.yml new file mode 100644 index 0000000..137a4c2 --- /dev/null +++ b/kubespray/project/boilerplate.yml @@ -0,0 +1,58 @@ +--- +- name: Check ansible version + import_playbook: ansible_version.yml + +# These are inventory compatibility tasks to ensure we keep compatibility with old style group names + +- name: Add kube-master nodes to kube_control_plane + hosts: kube-master + gather_facts: false + tags: always + tasks: + - name: Add nodes to kube_control_plane group + group_by: + key: 'kube_control_plane' + +- name: Add kube-node nodes to kube_node + hosts: kube-node + gather_facts: false + tags: always + tasks: + - name: Add nodes to kube_node group + group_by: + key: 'kube_node' + +- name: Add k8s-cluster nodes to k8s_cluster + hosts: k8s-cluster + gather_facts: false + tags: always + tasks: + - name: Add nodes to k8s_cluster group + group_by: + key: 'k8s_cluster' + +- name: Add calico-rr nodes to calico_rr + hosts: calico-rr + gather_facts: false + tags: always + tasks: + - name: Add nodes to calico_rr group + group_by: + key: 'calico_rr' + +- name: Add no-floating nodes to no_floating + hosts: no-floating + gather_facts: false + tags: always + tasks: + - name: Add nodes to no-floating group + group_by: + key: 'no_floating' + +- name: Install bastion ssh config + hosts: bastion[0] + gather_facts: False + environment: "{{ proxy_disable_env }}" + roles: + - { role: kubespray-defaults } + - { role: bastion-ssh-config, tags: ["localhost", "bastion"] } diff --git a/kubespray/project/cluster.yml b/kubespray/project/cluster.yml new file mode 100644 index 0000000..bc3e4ed --- /dev/null +++ b/kubespray/project/cluster.yml @@ -0,0 +1,111 @@ +--- +- name: Common tasks for every playbooks + import_playbook: boilerplate.yml + +- name: Gather facts + import_playbook: facts.yml + +- name: Prepare for etcd install + hosts: k8s_cluster:etcd + gather_facts: False + any_errors_fatal: "{{ any_errors_fatal | default(true) }}" + environment: "{{ proxy_disable_env }}" + roles: + - { role: kubespray-defaults } + - { role: kubernetes/preinstall, tags: preinstall } + - { + role: "container-engine", + tags: "container-engine", + when: deploy_container_engine, + } + - { role: download, tags: download, when: "not skip_downloads" } + +- name: Install etcd + import_playbook: install_etcd.yml + +- name: Install Kubernetes nodes + hosts: k8s_cluster + gather_facts: False + any_errors_fatal: "{{ any_errors_fatal | default(true) }}" + environment: "{{ proxy_disable_env }}" + roles: + - { role: kubespray-defaults } + - { role: kubernetes/node, tags: node } + +- name: Install the control plane + hosts: kube_control_plane + gather_facts: False + any_errors_fatal: "{{ any_errors_fatal | default(true) }}" + environment: "{{ proxy_disable_env }}" + roles: + - { role: kubespray-defaults } + - { role: kubernetes/control-plane, tags: master } + - { role: kubernetes/client, tags: client } + - { role: kubernetes-apps/cluster_roles, tags: cluster-roles } + +- name: Invoke kubeadm and install a CNI + hosts: k8s_cluster + gather_facts: False + any_errors_fatal: "{{ any_errors_fatal | default(true) }}" + environment: "{{ proxy_disable_env }}" + roles: + - { role: kubespray-defaults } + - { role: kubernetes/kubeadm, tags: kubeadm } + - { role: kubernetes/node-label, tags: node-label } + - { role: kubernetes/node-taint, tags: node-taint } + - { role: network_plugin, tags: network } + - { role: kubernetes-apps/kubelet-csr-approver, tags: kubelet-csr-approver } + +- name: Install Calico Route Reflector + hosts: calico_rr + gather_facts: False + any_errors_fatal: "{{ any_errors_fatal | default(true) }}" + environment: "{{ proxy_disable_env }}" + roles: + - { role: kubespray-defaults } + - { role: network_plugin/calico/rr, tags: ["network", "calico_rr"] } + +- name: Patch Kubernetes for Windows + hosts: kube_control_plane[0] + gather_facts: False + any_errors_fatal: "{{ any_errors_fatal | default(true) }}" + environment: "{{ proxy_disable_env }}" + roles: + - { role: kubespray-defaults } + - { role: win_nodes/kubernetes_patch, tags: ["master", "win_nodes"] } + +- name: Install Kubernetes and Helm apps + hosts: kube_control_plane + gather_facts: False + any_errors_fatal: "{{ any_errors_fatal | default(true) }}" + environment: "{{ proxy_disable_env }}" + roles: + - { role: kubespray-defaults } + - { + role: kubernetes-apps/external_cloud_controller, + tags: external-cloud-controller, + } + - { role: kubernetes-apps/network_plugin, tags: network } + - { role: kubernetes-apps/policy_controller, tags: policy-controller } + - { role: kubernetes-apps/ingress_controller, tags: ingress-controller } + - { role: kubernetes-apps/external_provisioner, tags: external-provisioner } + - { role: kubernetes-apps, tags: apps } + - { + role: helm-apps, + when: "releases is defined and repositories is defined", + tags: helm-apps, + } + +- name: Apply resolv.conf changes now that cluster DNS is up + hosts: k8s_cluster + gather_facts: False + any_errors_fatal: "{{ any_errors_fatal | default(true) }}" + environment: "{{ proxy_disable_env }}" + roles: + - { role: kubespray-defaults } + - { + role: kubernetes/preinstall, + when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", + tags: resolvconf, + dns_late: true, + } diff --git a/kubespray/project/facts.yml b/kubespray/project/facts.yml new file mode 100644 index 0000000..77823ac --- /dev/null +++ b/kubespray/project/facts.yml @@ -0,0 +1,41 @@ +--- +- name: Bootstrap hosts for Ansible + hosts: k8s_cluster:etcd:calico_rr + strategy: linear + any_errors_fatal: "{{ any_errors_fatal | default(true) }}" + gather_facts: false + environment: "{{ proxy_disable_env }}" + vars: + # Need to disable pipelining for bootstrap-os as some systems have requiretty in sudoers set, which makes pipelining + # fail. bootstrap-os fixes this on these systems, so in later plays it can be enabled. + ansible_ssh_pipelining: false + roles: + - { role: kubespray-defaults } + - { role: bootstrap-os, tags: bootstrap-os} + +- name: Gather facts + hosts: k8s_cluster:etcd:calico_rr + gather_facts: False + tags: always + tasks: + - name: Gather minimal facts + setup: + gather_subset: '!all' + + # filter match the following variables: + # ansible_default_ipv4 + # ansible_default_ipv6 + # ansible_all_ipv4_addresses + # ansible_all_ipv6_addresses + - name: Gather necessary facts (network) + setup: + gather_subset: '!all,!min,network' + filter: "ansible_*_ipv[46]*" + + # filter match the following variables: + # ansible_memtotal_mb + # ansible_swaptotal_mb + - name: Gather necessary facts (hardware) + setup: + gather_subset: '!all,!min,hardware' + filter: "ansible_*total_mb" diff --git a/kubespray/project/install_etcd.yml b/kubespray/project/install_etcd.yml new file mode 100644 index 0000000..b8e4d1d --- /dev/null +++ b/kubespray/project/install_etcd.yml @@ -0,0 +1,29 @@ +--- +- name: Add worker nodes to the etcd play if needed + hosts: kube_node + roles: + - { role: kubespray-defaults } + tasks: + - name: Check if nodes needs etcd client certs (depends on network_plugin) + group_by: + key: "_kubespray_needs_etcd" + when: + - kube_network_plugin in ["flannel", "canal", "cilium"] or + (cilium_deploy_additionally | default(false)) or + (kube_network_plugin == "calico" and calico_datastore == "etcd") + - etcd_deployment_type != "kubeadm" + tags: etcd + +- name: Install etcd + hosts: etcd:kube_control_plane:_kubespray_needs_etcd + gather_facts: False + any_errors_fatal: "{{ any_errors_fatal | default(true) }}" + environment: "{{ proxy_disable_env }}" + roles: + - { role: kubespray-defaults } + - role: etcd + tags: etcd + vars: + etcd_cluster_setup: true + etcd_events_cluster_setup: "{{ etcd_events_cluster_enabled }}" + when: etcd_deployment_type != "kubeadm" diff --git a/kubespray/project/library/__pycache__/kube.cpython-312.pyc b/kubespray/project/library/__pycache__/kube.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c691b2f244dbd689cf438d9f501b7eaf914e3638 GIT binary patch literal 14789 zcmeHOYit|GcHZUtrT2r9CD|)EvPDId96u8|bnM7ZU^`Nr*m>DW8H&52s8A#`yR=0` zatz!QmE07T>Y|cyabcxs5gBn|=U4JG_ZCG{6evWKH)0n4p|jK|KxJrLzNQHsUkix^9gAP_khmPW%- zE}%#FtUyHI?GNz0!1r?;cX0*Q#! z&qWiFNG>Pe9-4u|o*ES>Aub?_v0xYk=c!Pdj8JkU91)%vvv@!nr9GiiY|sO8MgqgalNkj{ zN#qhD=nKdR3IKM_iE$wq9tj7v4iQmTPz-@Qk-kX~P-Dk&;G6AmJk^VIdOf;Lag#BoYW=_lBc9 z4zfVI8i|ATM*v|902Dd}aTD2}3xuUdXyIdVN7{4*R4oNSiE+aM7o>>J!y~jFUO^ed z(L^{(<;UTQ3KJ5DD8xa+K;%)haN`0hy@KRH`EeGZPl#ZVNFE+wN*NWRTyRtfet-fB z699NrQzNmEKLTj^DAMhKc5pBbIs3V<`E$h3&5O+-?1m+@Q6r^66Yqp9EI}D z$Q9s5VN%AKpt!??D5xQHEgg7lNqD4mPU7Mub~X%B^iU?qU6EBINe2uotLlU^+c3;4 zJWgCO=sp578KRq*r=he`qoD}^!X)B6U?Fy^N`nqA%9^jQ_KikCt5qmP!Tg#Kkz_#z z&Ublwx&GD2(f3Dd6plFoqyjJI><^5)Hf9eoi4!CS>Y=C|58h5tijkxO(%?aPM|wD~ z7iGt}EzdWSL@|`>4N(|LL=*!_#u9)fQ3)-4T=DIo>!^zWui+6;HFOU`@H{E-8i}00 zhN>?jA5D;6N6rZw2gRlCnOD_)M<5}ALkS+-aUmK_a>rs(oVE^rAVkOT(Hjhl!I-DR z5%^CQy-~%n<51hVDA>^i2To01p%Ll9s{`&n<*PcdJ`x0AfF}6V!ewaX#uOU$2(;9~ zWoRin|7fIgWy?@fg;u3f$I{m)jmP_FQ+t!+5rEmVq#il3gjNqF zov-*K=u-_rk?Gi@zInM4!4)OUWtT7w<^mIXGvr?G9)yPlMqo~}5cC*$7D5{%hCC49 z7$G(D4hVPH(-z*zo8fNdEs$&Dt&rMz8>Eh;-Q$v-$3bP30EPrr;>g1WKO_9^E<=p9 zz)Z1Ij34vqHN}n0FP#@{hYl<1OsL%QP1vk*O=ik*Qp5+Z+ z59FgUrz(MRAFC*S$2Z>eI2Q@xh*X&~h^(zbNJ0C;-Dz*LuE5iCsh&jw$G6W4FCTMuCmSLt1Q--l-m2Up7 zuI$VkKCL%FSt!T&HCx_*{M9Ods9&kl-$4DEg8Gl6?X?B@mSvmjcgSWMFw1t$+4I!M7G*(Aw&23cxF{P#f@Xc0|NU2&UWnulIq^N5sC60InSPQ+jBb zaS}-Ofqy>+PG(;uJWK*4Nx$^Qld{e4$F(WHf3p5~eqQa>a)Gt@-!LCuVAdL2XH1_u z*W7Ij)AQ4-?yi)(Yq>e?-UYd?th4IU^Rv&dIJab5J62o!QmuVA?O*Rsw;r51{OK#% z=C-S?S6UYL_|zj-&k>-ALo>lxR{;(BK? zPET>Y#_L;`M{bN?8(-~xA=UfBH_mkLE9vgRRQq7YHT1B`n{RIW%KN2vb>~28=fF3| z(>qV3+D~L$udQ2{`b`VIdEaW?Q>nVAmflF$Jq?YwWgA)+-kX1KwP9PTVcXKVbVDEH zI3{Om5H48-v#dS9^S^9^cnv>7JvvdS3nZ_3xTaEPc3G zbJcyt{j=snbrad%QUxel@SKF?%tu}Y3n!A_xr}3r$1$!GEJFBOq0;W>n4>ecsQmp z4e#=FRa`#FY-J>EK6#2EC#NkNkjnfpsea1x7AcSS7FDjFvOhBwO8E}#A52@$TRwh> zv`VRS>^Ww__zrW9^;n1g9a}YdKp+HyagvBeF$|{YDn+=iP6+PlVl#vyutMqU6ke$HbfjlfnxHYafP=a|U$kw7@&9}{Cy z*)$dlM`inw30(As^#sxcRb+=4kA$U27(z^sm2AQ+BLb}{KtiHyAVJv(^2tUJ1CoiL zi0YsODI96V;x>P7sj&p=a=BNajQC4PE-=}ux=Yiu(~DxNs$<5qZm` zqbuR@^!w43I9ZX-tt6*b{L`tX>2)Jhv+W+k)_lyaJDBRaOXp|Lr>nNC*tY!Uo|UO? zzQY)hDInwV+2gCO_FJy@?7@M5u>IWjZT;U@W=@=3IrUcN#M>+N?*OHWj{joViZkpw zP1mYw^VEK~Zbd@(MeM}+Lyd>_F+bmDAE-0_g6SWqH2z(s8J>%b8V#Bn^+kAim{FHe zdM%g%61gjHayll>Ceh$@3^;Fa9>WmX0<=7)yb6)6kU_R%u7#=ufxR(?crfY4 znla1_%++O`6>~Kg_hswc*}~zt3q+V8cCUINw9@YWR>4q~)Iz7ibLpja1p)8zBt)qp zcG@^)AT?9Qu|j&eY7l?sXaJTmW2RDS&@bg`YEdPvsT1{*?lL@0x>&0lIjhpCcgjmc z)V#D}D@c!CQ^}==&b;}$hLk{0zhA7pZ4AAvk_))#a95=(iGP@}x{~VfI;Hj7NycLz zlFi@&Mw0ifu#SiO0QW6kFM6K$E4^OLwe)>7J=voAps>uRY<%J>p0Y?!+JW$WXMQQr zC$l{ship+6!igI`D?TMQNwxr|@qnZ_mFPW^r!eWnq!$vf9SWRgx<0IUq-ZfBss~z! zVjC8P`DAUONho>OvGg(|7ntvz)pMs7&di_ri}$j14KqjAsv2j)Gsb^$R%cyRmyXRI zn@eU~?Q7r|K6mjsHESZ{;xOyL#RIvl758@4d~Te-c0TPska4}Rh5&qVIuCy-SNqDg zffZMK#&vkj=Dg&Xb;1wLewYpRw(yRROzCrmNalZ0+P35Vi>4 zG=p2B21gCaFvWs781&4p#E^6xn6foBg(z!+-DP;{Vxhhnt17DADK9P5hQ{YEYDhXN zuDe83Y)Jb3f`+8yB7u}T+*RowFeE)w)S!zR64?bUAiFWygUMb<${3BZT|kY7fTzfi zXCd*}OU#7y<0l$;>Dh=;nMs^jh#Ci?t7aS4+&PHJx)M;V?bJ)H-+ zb*QKu)jP|~u~8G0)fQ7=v~w4=Fl~`T&1G~ai;}5Iw{P^hrlNTqKp8EXh&z}BkZ_Yz-&y39&asKyR z^rG^DR-GXD@vC{Ii8hCwm;L8K)mKVRP z9$2e$FZll4R}^=xbich)-I=L=XD#Q0NVr&)ado0R$&1Mxy~W*EpS$v0rkZAhm}<6f zdH0R}YyGK~J(=pgSY_hkg!*k?rh03xRhm?Nnd%*wcm85|Ni!fR6v241$S2W7;lG4# zhC_kq+#`}gf0kZ^2Tj(&Vv3}CahEO?)zTiYS3P(Cb{UR8%H5BC3GLHC+#lia>-LL2 z7q$HqFLW@QqNq+U12vt}MbyxwN-m0`I@BnN>Xd(ARP=BjQB;p+A&TmkC`L=fhLS6f zV7NgcvPnnM>2VDa6r&WzIv#{EUGO4@pz>rj1+Dzjyhs=7W8P5#j=&J{rBfc)<094$ zpzp~!Z|Jwl7LW|ijH=7$G`1m#iN(;Rgi^nOeDOaZ!4SWGCbSWl`T^trWjUA{qA>O9 z6Njml7H_7yR}GO4WNaO4H4UpZovE76rC_?Id&cs8RYSI}3Gh`3jdWq?Unvaf1JDz2 z1`)#kPn|*X+cF$~)btAH+0RlEISnH2DLL8O1j8 zBA^Euz~nF{M<6NknvO!oWOZTKh!h*a0L9C6Zx5FK z{pIlRFB4cjMK2$&ybK%8hh-sMI(BHM1KR0)7?10&FLId5S3vI=LhSWQwazr0UYcel z&CQ+C&LrUp3B40qdMnbrb}9o_(2cyE{-dT2mg1aB>kCufbX_|`F>Q)of$}x9JT4b% zJ9POP>IgyEpFr7Kt*p*cSwY!xC|jqM)maoPDEk(at=G!_Fx^8?wt<%24)W@(TIcCP z|Di6ITvIIHIHl~4y46u&nP>x))^Qy9rYR%cNo_82iS_Aj;#>62zI?3h!cs~FT$7Ba zbx1bhKV0C9gqfaCludYp?ZezzSB$bo zf3+hE8e&)D6NXg;T^@RBq~I(F4YqN?6c2>JX;*pzgpvS(D`0Okfu}qC_;(Gtc03^g z07jCqcNK*HYr#Qe@<&kB!;<|-`W;NpVDc^|r!k@at^u}lq4_~L9v%YWJK)1);~P=f zhKPmeiD2cVE!Qdb(uIaerq(*KspCitI&M8TAx zS}CysMd5?^N}(75m`Whq!ONED_HKSl!egfUz$5^rWUF#)na~E2mmy0;D~EMgbyvbP z4u20JBapclr_4WrRGqpRR-lDL)EmQN%?dtl)H+MH_ht=O6}wgYP>$EvA5WvX9%E8DzndFzd? zYh5?@rn~!7&HJIcx#8E0!R%IMM!Gr6%4|1I-#Au&|B%NjR~&wI;Ell}L#Ga%I{NC6 zY&-Jyp}`Zc962dlV#9bMNl_sjR(VrE(Xig!l%J5o5wRDJRE2}&7!)9bm^_0CZs^~) zJ+BlzK*r$}CtkWqyKZDzcHMTGtubAwSf4O7xJ(yp_jY__U_0hu%N_4`rp&uo_q_= 7.6 + +- name: Enable Oracle Linux repo + community.general.ini_file: + dest: "/etc/yum.repos.d/oracle-linux-ol{{ ansible_distribution_major_version }}.repo" + section: "ol{{ ansible_distribution_major_version }}_addons" + option: "{{ item.option }}" + value: "{{ item.value }}" + mode: 0644 + with_items: + - { option: "name", value: "ol{{ ansible_distribution_major_version }}_addons" } + - { option: "enabled", value: "1" } + - { option: "baseurl", value: "http://yum.oracle.com/repo/OracleLinux/OL{{ ansible_distribution_major_version }}/addons/$basearch/" } + when: + - use_oracle_public_repo | default(true) + - '''ID="ol"'' in os_release.stdout_lines' + - (ansible_distribution_version | float) >= 7.6 + +- name: Enable Centos extra repo for Oracle Linux + community.general.ini_file: + dest: "/etc/yum.repos.d/centos-extras.repo" + section: "extras" + option: "{{ item.option }}" + value: "{{ item.value }}" + mode: 0644 + with_items: + - { option: "name", value: "CentOS-{{ ansible_distribution_major_version }} - Extras" } + - { option: "enabled", value: "1" } + - { option: "gpgcheck", value: "0" } + - { option: "baseurl", value: "http://mirror.centos.org/{{ 'altarch' if (ansible_distribution_major_version | int) <= 7 and ansible_architecture == 'aarch64' else 'centos' }}/{{ ansible_distribution_major_version }}/extras/$basearch/{% if ansible_distribution_major_version | int > 7 %}os/{% endif %}" } + when: + - use_oracle_public_repo | default(true) + - '''ID="ol"'' in os_release.stdout_lines' + - (ansible_distribution_version | float) >= 7.6 + - (ansible_distribution_version | float) < 9 + +# CentOS ships with python installed + +- name: Check presence of fastestmirror.conf + stat: + path: /etc/yum/pluginconf.d/fastestmirror.conf + get_attributes: no + get_checksum: no + get_mime: no + register: fastestmirror + +# the fastestmirror plugin can actually slow down Ansible deployments +- name: Disable fastestmirror plugin if requested + lineinfile: + dest: /etc/yum/pluginconf.d/fastestmirror.conf + regexp: "^enabled=.*" + line: "enabled=0" + state: present + become: true + when: + - fastestmirror.stat.exists + - not centos_fastestmirror_enabled + +# libselinux-python is required on SELinux enabled hosts +# See https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html#managed-node-requirements +- name: Install libselinux python package + package: + name: "{{ ((ansible_distribution_major_version | int) < 8) | ternary('libselinux-python', 'python3-libselinux') }}" + state: present + become: true diff --git a/kubespray/project/roles/bootstrap-os/tasks/bootstrap-clearlinux.yml b/kubespray/project/roles/bootstrap-os/tasks/bootstrap-clearlinux.yml new file mode 100644 index 0000000..de42e3c --- /dev/null +++ b/kubespray/project/roles/bootstrap-os/tasks/bootstrap-clearlinux.yml @@ -0,0 +1,16 @@ +--- +# ClearLinux ships with Python installed + +- name: Install basic package to run containers + package: + name: containers-basic + state: present + +- name: Make sure docker service is enabled + systemd: + name: docker + masked: false + enabled: true + daemon_reload: true + state: started + become: true diff --git a/kubespray/project/roles/bootstrap-os/tasks/bootstrap-coreos.yml b/kubespray/project/roles/bootstrap-os/tasks/bootstrap-coreos.yml new file mode 100644 index 0000000..737a7ec --- /dev/null +++ b/kubespray/project/roles/bootstrap-os/tasks/bootstrap-coreos.yml @@ -0,0 +1,37 @@ +--- +# CoreOS ships without Python installed + +- name: Check if bootstrap is needed + raw: stat /opt/bin/.bootstrapped + register: need_bootstrap + failed_when: false + changed_when: false + tags: + - facts + +- name: Force binaries directory for Container Linux by CoreOS and Flatcar + set_fact: + bin_dir: "/opt/bin" + tags: + - facts + +- name: Run bootstrap.sh + script: bootstrap.sh + become: true + environment: "{{ proxy_env }}" + when: + - need_bootstrap.rc != 0 + +- name: Set the ansible_python_interpreter fact + set_fact: + ansible_python_interpreter: "{{ bin_dir }}/python" + tags: + - facts + +- name: Disable auto-upgrade + systemd: + name: locksmithd.service + masked: true + state: stopped + when: + - coreos_locksmithd_disable diff --git a/kubespray/project/roles/bootstrap-os/tasks/bootstrap-debian.yml b/kubespray/project/roles/bootstrap-os/tasks/bootstrap-debian.yml new file mode 100644 index 0000000..47bad20 --- /dev/null +++ b/kubespray/project/roles/bootstrap-os/tasks/bootstrap-debian.yml @@ -0,0 +1,76 @@ +--- +# Some Debian based distros ship without Python installed + +- name: Check if bootstrap is needed + raw: which python3 + register: need_bootstrap + failed_when: false + changed_when: false + # This command should always run, even in check mode + check_mode: false + tags: + - facts + +- name: Check http::proxy in apt configuration files + raw: apt-config dump | grep -qsi 'Acquire::http::proxy' + register: need_http_proxy + failed_when: false + changed_when: false + # This command should always run, even in check mode + check_mode: false + +- name: Add http_proxy to /etc/apt/apt.conf if http_proxy is defined + raw: echo 'Acquire::http::proxy "{{ http_proxy }}";' >> /etc/apt/apt.conf + become: true + when: + - http_proxy is defined + - need_http_proxy.rc != 0 + - not skip_http_proxy_on_os_packages + +- name: Check https::proxy in apt configuration files + raw: apt-config dump | grep -qsi 'Acquire::https::proxy' + register: need_https_proxy + failed_when: false + changed_when: false + # This command should always run, even in check mode + check_mode: false + +- name: Add https_proxy to /etc/apt/apt.conf if https_proxy is defined + raw: echo 'Acquire::https::proxy "{{ https_proxy }}";' >> /etc/apt/apt.conf + become: true + when: + - https_proxy is defined + - need_https_proxy.rc != 0 + - not skip_http_proxy_on_os_packages + +- name: Install python3 + raw: + apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y python3-minimal + become: true + when: + - need_bootstrap.rc != 0 + +- name: Update Apt cache + raw: apt-get update --allow-releaseinfo-change + become: true + when: + - '''ID=debian'' in os_release.stdout_lines' + - '''VERSION_ID="10"'' in os_release.stdout_lines or ''VERSION_ID="11"'' in os_release.stdout_lines' + register: bootstrap_update_apt_result + changed_when: + - '"changed its" in bootstrap_update_apt_result.stdout' + - '"value from" in bootstrap_update_apt_result.stdout' + ignore_errors: true + +- name: Set the ansible_python_interpreter fact + set_fact: + ansible_python_interpreter: "/usr/bin/python3" + +# Workaround for https://github.com/ansible/ansible/issues/25543 +- name: Install dbus for the hostname module + package: + name: dbus + state: present + use: apt + become: true diff --git a/kubespray/project/roles/bootstrap-os/tasks/bootstrap-fedora-coreos.yml b/kubespray/project/roles/bootstrap-os/tasks/bootstrap-fedora-coreos.yml new file mode 100644 index 0000000..91dc020 --- /dev/null +++ b/kubespray/project/roles/bootstrap-os/tasks/bootstrap-fedora-coreos.yml @@ -0,0 +1,46 @@ +--- + +- name: Check if bootstrap is needed + raw: which python + register: need_bootstrap + failed_when: false + changed_when: false + tags: + - facts + +- name: Remove podman network cni + raw: "podman network rm podman" + become: true + ignore_errors: true # noqa ignore-errors + when: need_bootstrap.rc != 0 + +- name: Clean up possible pending packages on fedora coreos + raw: "export http_proxy={{ http_proxy | default('') }};rpm-ostree cleanup -p }}" + become: true + when: need_bootstrap.rc != 0 + +- name: Install required packages on fedora coreos + raw: "export http_proxy={{ http_proxy | default('') }};rpm-ostree install --allow-inactive {{ fedora_coreos_packages | join(' ') }}" + become: true + when: need_bootstrap.rc != 0 + +- name: Reboot immediately for updated ostree + raw: "nohup bash -c 'sleep 5s && shutdown -r now'" + become: true + ignore_errors: true # noqa ignore-errors + ignore_unreachable: yes + when: need_bootstrap.rc != 0 + +- name: Wait for the reboot to complete + wait_for_connection: + timeout: 240 + connect_timeout: 20 + delay: 5 + sleep: 5 + when: need_bootstrap.rc != 0 + +- name: Store the fact if this is an fedora core os host + set_fact: + is_fedora_coreos: True + tags: + - facts diff --git a/kubespray/project/roles/bootstrap-os/tasks/bootstrap-fedora.yml b/kubespray/project/roles/bootstrap-os/tasks/bootstrap-fedora.yml new file mode 100644 index 0000000..4ce77b4 --- /dev/null +++ b/kubespray/project/roles/bootstrap-os/tasks/bootstrap-fedora.yml @@ -0,0 +1,36 @@ +--- +# Some Fedora based distros ship without Python installed + +- name: Check if bootstrap is needed + raw: which python + register: need_bootstrap + failed_when: false + changed_when: false + tags: + - facts + +- name: Add proxy to dnf.conf if http_proxy is defined + community.general.ini_file: + path: "/etc/dnf/dnf.conf" + section: main + option: proxy + value: "{{ http_proxy | default(omit) }}" + state: "{{ http_proxy | default(False) | ternary('present', 'absent') }}" + no_extra_spaces: true + mode: 0644 + become: true + when: not skip_http_proxy_on_os_packages + +- name: Install python3 on fedora + raw: "dnf install --assumeyes --quiet python3" + become: true + when: + - need_bootstrap.rc != 0 + +# libselinux-python3 is required on SELinux enabled hosts +# See https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html#managed-node-requirements +- name: Install libselinux-python3 + package: + name: libselinux-python3 + state: present + become: true diff --git a/kubespray/project/roles/bootstrap-os/tasks/bootstrap-flatcar.yml b/kubespray/project/roles/bootstrap-os/tasks/bootstrap-flatcar.yml new file mode 100644 index 0000000..b0f3a9e --- /dev/null +++ b/kubespray/project/roles/bootstrap-os/tasks/bootstrap-flatcar.yml @@ -0,0 +1,37 @@ +--- +# Flatcar Container Linux ships without Python installed + +- name: Check if bootstrap is needed + raw: stat /opt/bin/.bootstrapped + register: need_bootstrap + failed_when: false + changed_when: false + tags: + - facts + +- name: Force binaries directory for Flatcar Container Linux by Kinvolk + set_fact: + bin_dir: "/opt/bin" + tags: + - facts + +- name: Run bootstrap.sh + script: bootstrap.sh + become: true + environment: "{{ proxy_env }}" + when: + - need_bootstrap.rc != 0 + +- name: Set the ansible_python_interpreter fact + set_fact: + ansible_python_interpreter: "{{ bin_dir }}/python" + tags: + - facts + +- name: Disable auto-upgrade + systemd: + name: locksmithd.service + masked: true + state: stopped + when: + - coreos_locksmithd_disable diff --git a/kubespray/project/roles/bootstrap-os/tasks/bootstrap-opensuse.yml b/kubespray/project/roles/bootstrap-os/tasks/bootstrap-opensuse.yml new file mode 100644 index 0000000..9b69dcd --- /dev/null +++ b/kubespray/project/roles/bootstrap-os/tasks/bootstrap-opensuse.yml @@ -0,0 +1,85 @@ +--- +# OpenSUSE ships with Python installed +- name: Gather host facts to get ansible_distribution_version ansible_distribution_major_version + setup: + gather_subset: '!all' + filter: ansible_distribution_*version + +- name: Check that /etc/sysconfig/proxy file exists + stat: + path: /etc/sysconfig/proxy + get_attributes: no + get_checksum: no + get_mime: no + register: stat_result + +- name: Create the /etc/sysconfig/proxy empty file + file: # noqa risky-file-permissions + path: /etc/sysconfig/proxy + state: touch + when: + - http_proxy is defined or https_proxy is defined + - not stat_result.stat.exists + +- name: Set the http_proxy in /etc/sysconfig/proxy + lineinfile: + path: /etc/sysconfig/proxy + regexp: '^HTTP_PROXY=' + line: 'HTTP_PROXY="{{ http_proxy }}"' + become: true + when: + - http_proxy is defined + +- name: Set the https_proxy in /etc/sysconfig/proxy + lineinfile: + path: /etc/sysconfig/proxy + regexp: '^HTTPS_PROXY=' + line: 'HTTPS_PROXY="{{ https_proxy }}"' + become: true + when: + - https_proxy is defined + +- name: Enable proxies + lineinfile: + path: /etc/sysconfig/proxy + regexp: '^PROXY_ENABLED=' + line: 'PROXY_ENABLED="yes"' + become: true + when: + - http_proxy is defined or https_proxy is defined + +# Required for zypper module +- name: Install python-xml + shell: zypper refresh && zypper --non-interactive install python-xml + changed_when: false + become: true + tags: + - facts + +# Without this package, the get_url module fails when trying to handle https +- name: Install python-cryptography + community.general.zypper: + name: python-cryptography + state: present + update_cache: true + become: true + when: + - ansible_distribution_version is version('15.4', '<') + +- name: Install python3-cryptography + community.general.zypper: + name: python3-cryptography + state: present + update_cache: true + become: true + when: + - ansible_distribution_version is version('15.4', '>=') + +# Nerdctl needs some basic packages to get an environment up +- name: Install basic dependencies + community.general.zypper: + name: + - iptables + - apparmor-parser + state: present + become: true diff --git a/kubespray/project/roles/bootstrap-os/tasks/bootstrap-redhat.yml b/kubespray/project/roles/bootstrap-os/tasks/bootstrap-redhat.yml new file mode 100644 index 0000000..c362146 --- /dev/null +++ b/kubespray/project/roles/bootstrap-os/tasks/bootstrap-redhat.yml @@ -0,0 +1,113 @@ +--- +- name: Gather host facts to get ansible_distribution_version ansible_distribution_major_version + setup: + gather_subset: '!all' + filter: ansible_distribution_*version + +- name: Add proxy to yum.conf or dnf.conf if http_proxy is defined + community.general.ini_file: + path: "{{ ((ansible_distribution_major_version | int) < 8) | ternary('/etc/yum.conf', '/etc/dnf/dnf.conf') }}" + section: main + option: proxy + value: "{{ http_proxy | default(omit) }}" + state: "{{ http_proxy | default(False) | ternary('present', 'absent') }}" + no_extra_spaces: true + mode: 0644 + become: true + when: not skip_http_proxy_on_os_packages + +- name: Add proxy to RHEL subscription-manager if http_proxy is defined + command: /sbin/subscription-manager config --server.proxy_hostname={{ http_proxy | regex_replace(':\d+$') | regex_replace('^.*://') }} --server.proxy_port={{ http_proxy | regex_replace('^.*:') }} + become: true + when: + - not skip_http_proxy_on_os_packages + - http_proxy is defined + +- name: Check RHEL subscription-manager status + command: /sbin/subscription-manager status + register: rh_subscription_status + changed_when: "rh_subscription_status.rc != 0" + ignore_errors: true # noqa ignore-errors + become: true + +- name: RHEL subscription Organization ID/Activation Key registration + community.general.redhat_subscription: + state: present + org_id: "{{ rh_subscription_org_id }}" + activationkey: "{{ rh_subscription_activation_key }}" + force_register: true + notify: RHEL auto-attach subscription + become: true + when: + - rh_subscription_org_id is defined + - rh_subscription_status.changed + +# this task has no_log set to prevent logging security sensitive information such as subscription passwords +- name: RHEL subscription Username/Password registration + community.general.redhat_subscription: + state: present + username: "{{ rh_subscription_username }}" + password: "{{ rh_subscription_password }}" + auto_attach: true + force_register: true + syspurpose: + usage: "{{ rh_subscription_usage }}" + role: "{{ rh_subscription_role }}" + service_level_agreement: "{{ rh_subscription_sla }}" + sync: true + notify: RHEL auto-attach subscription + become: true + no_log: "{{ not (unsafe_show_logs | bool) }}" + when: + - rh_subscription_username is defined + - rh_subscription_status.changed + +# container-selinux is in extras repo +- name: Enable RHEL 7 repos + community.general.rhsm_repository: + name: + - "rhel-7-server-rpms" + - "rhel-7-server-extras-rpms" + state: "{{ 'enabled' if (rhel_enable_repos | default(True) | bool) else 'disabled' }}" + when: + - ansible_distribution_major_version == "7" + - (not rh_subscription_status.changed) or (rh_subscription_username is defined) or (rh_subscription_org_id is defined) + +# container-selinux is in appstream repo +- name: Enable RHEL 8 repos + community.general.rhsm_repository: + name: + - "rhel-8-for-*-baseos-rpms" + - "rhel-8-for-*-appstream-rpms" + state: "{{ 'enabled' if (rhel_enable_repos | default(True) | bool) else 'disabled' }}" + when: + - ansible_distribution_major_version == "8" + - (not rh_subscription_status.changed) or (rh_subscription_username is defined) or (rh_subscription_org_id is defined) + +- name: Check presence of fastestmirror.conf + stat: + path: /etc/yum/pluginconf.d/fastestmirror.conf + get_attributes: no + get_checksum: no + get_mime: no + register: fastestmirror + +# the fastestmirror plugin can actually slow down Ansible deployments +- name: Disable fastestmirror plugin if requested + lineinfile: + dest: /etc/yum/pluginconf.d/fastestmirror.conf + regexp: "^enabled=.*" + line: "enabled=0" + state: present + become: true + when: + - fastestmirror.stat.exists + - not centos_fastestmirror_enabled + +# libselinux-python is required on SELinux enabled hosts +# See https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html#managed-node-requirements +- name: Install libselinux python package + package: + name: "{{ ((ansible_distribution_major_version | int) < 8) | ternary('libselinux-python', 'python3-libselinux') }}" + state: present + become: true diff --git a/kubespray/project/roles/bootstrap-os/tasks/main.yml b/kubespray/project/roles/bootstrap-os/tasks/main.yml new file mode 100644 index 0000000..73c9e06 --- /dev/null +++ b/kubespray/project/roles/bootstrap-os/tasks/main.yml @@ -0,0 +1,109 @@ +--- +- name: Fetch /etc/os-release + raw: cat /etc/os-release + register: os_release + changed_when: false + # This command should always run, even in check mode + check_mode: false + +- name: Bootstrap CentOS + include_tasks: bootstrap-centos.yml + when: '''ID="centos"'' in os_release.stdout_lines or ''ID="ol"'' in os_release.stdout_lines or ''ID="almalinux"'' in os_release.stdout_lines or ''ID="rocky"'' in os_release.stdout_lines or ''ID="kylin"'' in os_release.stdout_lines or ''ID="uos"'' in os_release.stdout_lines or ''ID="openEuler"'' in os_release.stdout_lines' + +- name: Bootstrap Amazon + include_tasks: bootstrap-amazon.yml + when: '''ID="amzn"'' in os_release.stdout_lines' + +- name: Bootstrap RedHat + include_tasks: bootstrap-redhat.yml + when: '''ID="rhel"'' in os_release.stdout_lines' + +- name: Bootstrap Clear Linux + include_tasks: bootstrap-clearlinux.yml + when: '''ID=clear-linux-os'' in os_release.stdout_lines' + +# Fedora CoreOS +- name: Bootstrap Fedora CoreOS + include_tasks: bootstrap-fedora-coreos.yml + when: + - '''ID=fedora'' in os_release.stdout_lines' + - '''VARIANT_ID=coreos'' in os_release.stdout_lines' + +- name: Bootstrap Flatcar + include_tasks: bootstrap-flatcar.yml + when: '''ID=flatcar'' in os_release.stdout_lines' + +- name: Bootstrap Debian + include_tasks: bootstrap-debian.yml + when: '''ID=debian'' in os_release.stdout_lines or ''ID=ubuntu'' in os_release.stdout_lines' + +# Fedora "classic" +- name: Boostrap Fedora + include_tasks: bootstrap-fedora.yml + when: + - '''ID=fedora'' in os_release.stdout_lines' + - '''VARIANT_ID=coreos'' not in os_release.stdout_lines' + +- name: Bootstrap OpenSUSE + include_tasks: bootstrap-opensuse.yml + when: '''ID="opensuse-leap"'' in os_release.stdout_lines or ''ID="opensuse-tumbleweed"'' in os_release.stdout_lines' + +- name: Create remote_tmp for it is used by another module + file: + path: "{{ ansible_remote_tmp | default('~/.ansible/tmp') }}" + state: directory + mode: 0700 + +# Workaround for https://github.com/ansible/ansible/issues/42726 +# (1/3) +- name: Gather host facts to get ansible_os_family + setup: + gather_subset: '!all' + filter: ansible_* + +- name: Assign inventory name to unconfigured hostnames (non-CoreOS, non-Flatcar, Suse and ClearLinux, non-Fedora) + hostname: + name: "{{ inventory_hostname }}" + when: + - override_system_hostname + - ansible_os_family not in ['Suse', 'Flatcar', 'Flatcar Container Linux by Kinvolk', 'ClearLinux'] + - not ansible_distribution == "Fedora" + - not is_fedora_coreos + +# (2/3) +- name: Assign inventory name to unconfigured hostnames (CoreOS, Flatcar, Suse, ClearLinux and Fedora only) + command: "hostnamectl set-hostname {{ inventory_hostname }}" + register: hostname_changed + become: true + changed_when: false + when: > + override_system_hostname + and (ansible_os_family in ['Suse', 'Flatcar', 'Flatcar Container Linux by Kinvolk', 'ClearLinux'] + or is_fedora_coreos + or ansible_distribution == "Fedora") + +# (3/3) +- name: Update hostname fact (CoreOS, Flatcar, Suse, ClearLinux and Fedora only) + setup: + gather_subset: '!all' + filter: ansible_hostname + when: > + override_system_hostname + and (ansible_os_family in ['Suse', 'Flatcar', 'Flatcar Container Linux by Kinvolk', 'ClearLinux'] + or is_fedora_coreos + or ansible_distribution == "Fedora") + +- name: Install ceph-commmon package + package: + name: + - ceph-common + state: present + when: rbd_provisioner_enabled | default(false) + +- name: Ensure bash_completion.d folder exists + file: + name: /etc/bash_completion.d/ + state: directory + owner: root + group: root + mode: 0755 diff --git a/kubespray/project/roles/container-engine/containerd-common/defaults/main.yml b/kubespray/project/roles/container-engine/containerd-common/defaults/main.yml new file mode 100644 index 0000000..ae1c6e0 --- /dev/null +++ b/kubespray/project/roles/container-engine/containerd-common/defaults/main.yml @@ -0,0 +1,17 @@ +--- +# We keep these variables around to allow migration from package +# manager controlled installs to direct download ones. +containerd_package: 'containerd.io' +yum_repo_dir: /etc/yum.repos.d + +# Keep minimal repo information around for cleanup +containerd_repo_info: + repos: + +# Ubuntu docker-ce repo +containerd_ubuntu_repo_base_url: "https://download.docker.com/linux/ubuntu" +containerd_ubuntu_repo_component: "stable" + +# Debian docker-ce repo +containerd_debian_repo_base_url: "https://download.docker.com/linux/debian" +containerd_debian_repo_component: "stable" diff --git a/kubespray/project/roles/container-engine/containerd-common/meta/main.yml b/kubespray/project/roles/container-engine/containerd-common/meta/main.yml new file mode 100644 index 0000000..61d3ffe --- /dev/null +++ b/kubespray/project/roles/container-engine/containerd-common/meta/main.yml @@ -0,0 +1,2 @@ +--- +allow_duplicates: true diff --git a/kubespray/project/roles/container-engine/containerd-common/tasks/main.yml b/kubespray/project/roles/container-engine/containerd-common/tasks/main.yml new file mode 100644 index 0000000..d0cf1f1 --- /dev/null +++ b/kubespray/project/roles/container-engine/containerd-common/tasks/main.yml @@ -0,0 +1,31 @@ +--- +- name: Containerd-common | check if fedora coreos + stat: + path: /run/ostree-booted + get_attributes: no + get_checksum: no + get_mime: no + register: ostree + +- name: Containerd-common | set is_ostree + set_fact: + is_ostree: "{{ ostree.stat.exists }}" + +- name: Containerd-common | gather os specific variables + include_vars: "{{ item }}" + with_first_found: + - files: + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower | replace('/', '_') }}.yml" + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_release | lower }}-{{ host_architecture }}.yml" + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_release | lower }}.yml" + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower | replace('/', '_') }}.yml" + - "{{ ansible_distribution | lower }}-{{ host_architecture }}.yml" + - "{{ ansible_distribution | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ host_architecture }}.yml" + - "{{ ansible_os_family | lower }}.yml" + - defaults.yml + paths: + - ../vars + skip: true + tags: + - facts diff --git a/kubespray/project/roles/container-engine/containerd-common/vars/amazon.yml b/kubespray/project/roles/container-engine/containerd-common/vars/amazon.yml new file mode 100644 index 0000000..0568169 --- /dev/null +++ b/kubespray/project/roles/container-engine/containerd-common/vars/amazon.yml @@ -0,0 +1,2 @@ +--- +containerd_package: containerd diff --git a/kubespray/project/roles/container-engine/containerd-common/vars/suse.yml b/kubespray/project/roles/container-engine/containerd-common/vars/suse.yml new file mode 100644 index 0000000..0568169 --- /dev/null +++ b/kubespray/project/roles/container-engine/containerd-common/vars/suse.yml @@ -0,0 +1,2 @@ +--- +containerd_package: containerd diff --git a/kubespray/project/roles/container-engine/containerd/defaults/main.yml b/kubespray/project/roles/container-engine/containerd/defaults/main.yml new file mode 100644 index 0000000..9ecace9 --- /dev/null +++ b/kubespray/project/roles/container-engine/containerd/defaults/main.yml @@ -0,0 +1,111 @@ +--- +containerd_storage_dir: "/var/lib/containerd" +containerd_state_dir: "/run/containerd" +containerd_systemd_dir: "/etc/systemd/system/containerd.service.d" +# The default value is not -999 here because containerd's oom_score_adj has been +# set to the -999 even if containerd_oom_score is 0. +# Ref: https://github.com/kubernetes-sigs/kubespray/pull/9275#issuecomment-1246499242 +containerd_oom_score: 0 + +containerd_default_runtime: "runc" +containerd_snapshotter: "overlayfs" + +containerd_runc_runtime: + name: runc + type: "io.containerd.runc.v2" + engine: "" + root: "" + base_runtime_spec: cri-base.json + options: + systemdCgroup: "{{ containerd_use_systemd_cgroup | ternary('true', 'false') }}" + binaryName: "{{ bin_dir }}/runc" + +containerd_additional_runtimes: [] +# Example for Kata Containers as additional runtime: +# - name: kata +# type: "io.containerd.kata.v2" +# engine: "" +# root: "" + +containerd_base_runtime_spec_rlimit_nofile: 65535 + +containerd_default_base_runtime_spec_patch: + process: + rlimits: + - type: RLIMIT_NOFILE + hard: "{{ containerd_base_runtime_spec_rlimit_nofile }}" + soft: "{{ containerd_base_runtime_spec_rlimit_nofile }}" + +# Can help reduce disk usage +# https://github.com/containerd/containerd/discussions/6295 +containerd_discard_unpacked_layers: true + +containerd_base_runtime_specs: + cri-base.json: "{{ containerd_default_base_runtime_spec | combine(containerd_default_base_runtime_spec_patch, recursive=1) }}" + +containerd_grpc_max_recv_message_size: 16777216 +containerd_grpc_max_send_message_size: 16777216 + +containerd_debug_level: "info" + +containerd_metrics_address: "" + +containerd_metrics_grpc_histogram: false + +containerd_registries_mirrors: + - prefix: docker.io + mirrors: + - host: https://registry-1.docker.io + capabilities: ["pull", "resolve"] + skip_verify: false + +containerd_max_container_log_line_size: -1 + +# If enabled it will allow non root users to use port numbers <1024 +containerd_enable_unprivileged_ports: false +# If enabled it will allow non root users to use icmp sockets +containerd_enable_unprivileged_icmp: false + +containerd_cfg_dir: /etc/containerd + +# Extra config to be put in {{ containerd_cfg_dir }}/config.toml literally +containerd_extra_args: '' + +# Configure registry auth (if applicable to secure/insecure registries) +containerd_registry_auth: [] +# - registry: 10.0.0.2:5000 +# username: user +# password: pass + +# Configure containerd service +containerd_limit_proc_num: "infinity" +containerd_limit_core: "infinity" +containerd_limit_open_file_num: "infinity" +containerd_limit_mem_lock: "infinity" + +# If enabled it will use config_path and config to be put in {{ containerd_cfg_dir }}/certs.d/ +containerd_use_config_path: false + +# OS distributions that already support containerd +containerd_supported_distributions: + - "CentOS" + - "OracleLinux" + - "RedHat" + - "Ubuntu" + - "Debian" + - "Fedora" + - "AlmaLinux" + - "Rocky" + - "Amazon" + - "Flatcar" + - "Flatcar Container Linux by Kinvolk" + - "Suse" + - "openSUSE Leap" + - "openSUSE Tumbleweed" + - "Kylin Linux Advanced Server" + - "UnionTech" + - "UniontechOS" + - "openEuler" + +# Enable container device interface +enable_cdi: false diff --git a/kubespray/project/roles/container-engine/containerd/handlers/main.yml b/kubespray/project/roles/container-engine/containerd/handlers/main.yml new file mode 100644 index 0000000..1959dc9 --- /dev/null +++ b/kubespray/project/roles/container-engine/containerd/handlers/main.yml @@ -0,0 +1,17 @@ +--- +- name: Containerd | restart containerd + systemd: + name: containerd + state: restarted + enabled: yes + daemon-reload: yes + masked: no + listen: Restart containerd + +- name: Containerd | wait for containerd + command: "{{ containerd_bin_dir }}/ctr images ls -q" + register: containerd_ready + retries: 8 + delay: 4 + until: containerd_ready.rc == 0 + listen: Restart containerd diff --git a/kubespray/project/roles/container-engine/containerd/handlers/reset.yml b/kubespray/project/roles/container-engine/containerd/handlers/reset.yml new file mode 100644 index 0000000..ed97d53 --- /dev/null +++ b/kubespray/project/roles/container-engine/containerd/handlers/reset.yml @@ -0,0 +1 @@ +--- diff --git a/kubespray/project/roles/container-engine/containerd/meta/main.yml b/kubespray/project/roles/container-engine/containerd/meta/main.yml new file mode 100644 index 0000000..41c5b6a --- /dev/null +++ b/kubespray/project/roles/container-engine/containerd/meta/main.yml @@ -0,0 +1,6 @@ +--- +dependencies: + - role: container-engine/containerd-common + - role: container-engine/runc + - role: container-engine/crictl + - role: container-engine/nerdctl diff --git a/kubespray/project/roles/container-engine/containerd/molecule/default/converge.yml b/kubespray/project/roles/container-engine/containerd/molecule/default/converge.yml new file mode 100644 index 0000000..7847871 --- /dev/null +++ b/kubespray/project/roles/container-engine/containerd/molecule/default/converge.yml @@ -0,0 +1,9 @@ +--- +- name: Converge + hosts: all + become: true + vars: + container_manager: containerd + roles: + - role: kubespray-defaults + - role: container-engine/containerd diff --git a/kubespray/project/roles/container-engine/containerd/molecule/default/molecule.yml b/kubespray/project/roles/container-engine/containerd/molecule/default/molecule.yml new file mode 100644 index 0000000..4c5c48f --- /dev/null +++ b/kubespray/project/roles/container-engine/containerd/molecule/default/molecule.yml @@ -0,0 +1,47 @@ +--- +role_name_check: 1 +driver: + name: vagrant + provider: + name: libvirt +platforms: + - name: ubuntu20 + box: generic/ubuntu2004 + cpus: 1 + memory: 1024 + groups: + - kube_control_plane + - kube_node + - k8s_cluster + provider_options: + driver: kvm + - name: debian11 + box: generic/debian11 + cpus: 1 + memory: 1024 + groups: + - kube_control_plane + - kube_node + - k8s_cluster + provider_options: + driver: kvm + - name: almalinux8 + box: almalinux/8 + cpus: 1 + memory: 1024 + groups: + - kube_control_plane + - kube_node + - k8s_cluster + provider_options: + driver: kvm +provisioner: + name: ansible + env: + ANSIBLE_ROLES_PATH: ../../../../ + config_options: + defaults: + callbacks_enabled: profile_tasks + timeout: 120 +verifier: + name: testinfra diff --git a/kubespray/project/roles/container-engine/containerd/molecule/default/prepare.yml b/kubespray/project/roles/container-engine/containerd/molecule/default/prepare.yml new file mode 100644 index 0000000..ddc9c04 --- /dev/null +++ b/kubespray/project/roles/container-engine/containerd/molecule/default/prepare.yml @@ -0,0 +1,29 @@ +--- +- name: Prepare + hosts: all + gather_facts: False + become: true + vars: + ignore_assert_errors: true + roles: + - role: kubespray-defaults + - role: bootstrap-os + - role: kubernetes/preinstall + - role: adduser + user: "{{ addusers.kube }}" + tasks: + - name: Download CNI + include_tasks: "../../../../download/tasks/download_file.yml" + vars: + download: "{{ download_defaults | combine(downloads.cni) }}" + +- name: Prepare CNI + hosts: all + gather_facts: False + become: true + vars: + ignore_assert_errors: true + kube_network_plugin: cni + roles: + - role: kubespray-defaults + - role: network_plugin/cni diff --git a/kubespray/project/roles/container-engine/containerd/molecule/default/tests/test_default.py b/kubespray/project/roles/container-engine/containerd/molecule/default/tests/test_default.py new file mode 100644 index 0000000..e1d9151 --- /dev/null +++ b/kubespray/project/roles/container-engine/containerd/molecule/default/tests/test_default.py @@ -0,0 +1,55 @@ +import os +import pytest + +import testinfra.utils.ansible_runner + +testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( + os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') + + +def test_service(host): + svc = host.service("containerd") + assert svc.is_running + assert svc.is_enabled + + +def test_version(host): + crictl = "/usr/local/bin/crictl" + path = "unix:///var/run/containerd/containerd.sock" + with host.sudo(): + cmd = host.command(crictl + " --runtime-endpoint " + path + " version") + assert cmd.rc == 0 + assert "RuntimeName: containerd" in cmd.stdout + + +@pytest.mark.parametrize('image, dest', [ + ('quay.io/kubespray/hello-world:latest', '/tmp/hello-world.tar') +]) +def test_image_pull_save_load(host, image, dest): + nerdctl = "/usr/local/bin/nerdctl" + dest_file = host.file(dest) + + with host.sudo(): + pull_cmd = host.command(nerdctl + " pull " + image) + assert pull_cmd.rc ==0 + + with host.sudo(): + save_cmd = host.command(nerdctl + " save -o " + dest + " " + image) + assert save_cmd.rc == 0 + assert dest_file.exists + + with host.sudo(): + load_cmd = host.command(nerdctl + " load < " + dest) + assert load_cmd.rc == 0 + + +@pytest.mark.parametrize('image', [ + ('quay.io/kubespray/hello-world:latest') +]) +def test_run(host, image): + nerdctl = "/usr/local/bin/nerdctl" + + with host.sudo(): + cmd = host.command(nerdctl + " -n k8s.io run " + image) + assert cmd.rc == 0 + assert "Hello from Docker" in cmd.stdout diff --git a/kubespray/project/roles/container-engine/containerd/tasks/main.yml b/kubespray/project/roles/container-engine/containerd/tasks/main.yml new file mode 100644 index 0000000..f1b9777 --- /dev/null +++ b/kubespray/project/roles/container-engine/containerd/tasks/main.yml @@ -0,0 +1,143 @@ +--- +- name: Fail containerd setup if distribution is not supported + fail: + msg: "{{ ansible_distribution }} is not supported by containerd." + when: + - not (allow_unsupported_distribution_setup | default(false)) and (ansible_distribution not in containerd_supported_distributions) + +- name: Containerd | Remove any package manager controlled containerd package + package: + name: "{{ containerd_package }}" + state: absent + when: + - not (is_ostree or (ansible_distribution == "Flatcar Container Linux by Kinvolk") or (ansible_distribution == "Flatcar")) + +- name: Containerd | Remove containerd repository + file: + path: "{{ yum_repo_dir }}/containerd.repo" + state: absent + when: + - ansible_os_family in ['RedHat'] + +- name: Containerd | Remove containerd repository + apt_repository: + repo: "{{ item }}" + state: absent + with_items: "{{ containerd_repo_info.repos }}" + when: ansible_pkg_mgr == 'apt' + +- name: Containerd | Download containerd + include_tasks: "../../../download/tasks/download_file.yml" + vars: + download: "{{ download_defaults | combine(downloads.containerd) }}" + +- name: Containerd | Unpack containerd archive + unarchive: + src: "{{ downloads.containerd.dest }}" + dest: "{{ containerd_bin_dir }}" + mode: 0755 + remote_src: yes + extra_opts: + - --strip-components=1 + notify: Restart containerd + +- name: Containerd | Remove orphaned binary + file: + path: "/usr/bin/{{ item }}" + state: absent + when: + - containerd_bin_dir != "/usr/bin" + - not (is_ostree or (ansible_distribution == "Flatcar Container Linux by Kinvolk") or (ansible_distribution == "Flatcar")) + ignore_errors: true # noqa ignore-errors + with_items: + - containerd + - containerd-shim + - containerd-shim-runc-v1 + - containerd-shim-runc-v2 + - ctr + +- name: Containerd | Generate systemd service for containerd + template: + src: containerd.service.j2 + dest: /etc/systemd/system/containerd.service + mode: 0644 + validate: "sh -c '[ -f /usr/bin/systemd/system/factory-reset.target ] || exit 0 && systemd-analyze verify %s:containerd.service'" + # FIXME: check that systemd version >= 250 (factory-reset.target was introduced in that release) + # Remove once we drop support for systemd < 250 + notify: Restart containerd + +- name: Containerd | Ensure containerd directories exist + file: + dest: "{{ item }}" + state: directory + mode: 0755 + owner: root + group: root + with_items: + - "{{ containerd_systemd_dir }}" + - "{{ containerd_cfg_dir }}" + - "{{ containerd_storage_dir }}" + - "{{ containerd_state_dir }}" + +- name: Containerd | Write containerd proxy drop-in + template: + src: http-proxy.conf.j2 + dest: "{{ containerd_systemd_dir }}/http-proxy.conf" + mode: 0644 + notify: Restart containerd + when: http_proxy is defined or https_proxy is defined + +- name: Containerd | Generate default base_runtime_spec + register: ctr_oci_spec + command: "{{ containerd_bin_dir }}/ctr oci spec" + check_mode: false + changed_when: false + +- name: Containerd | Store generated default base_runtime_spec + set_fact: + containerd_default_base_runtime_spec: "{{ ctr_oci_spec.stdout | from_json }}" + +- name: Containerd | Write base_runtime_specs + copy: + content: "{{ item.value }}" + dest: "{{ containerd_cfg_dir }}/{{ item.key }}" + owner: "root" + mode: 0644 + with_dict: "{{ containerd_base_runtime_specs | default({}) }}" + notify: Restart containerd + +- name: Containerd | Copy containerd config file + template: + src: config.toml.j2 + dest: "{{ containerd_cfg_dir }}/config.toml" + owner: "root" + mode: 0640 + notify: Restart containerd + +- name: Containerd | Configure containerd registries + when: containerd_registries_mirrors is defined + block: + - name: Containerd | Create registry directories + file: + path: "{{ containerd_cfg_dir }}/certs.d/{{ item.prefix }}" + state: directory + mode: 0755 + loop: "{{ containerd_registries_mirrors }}" + - name: Containerd | Write hosts.toml file + template: + src: hosts.toml.j2 + dest: "{{ containerd_cfg_dir }}/certs.d/{{ item.prefix }}/hosts.toml" + mode: 0640 + loop: "{{ containerd_registries_mirrors }}" + +# you can sometimes end up in a state where everything is installed +# but containerd was not started / enabled +- name: Containerd | Flush handlers + meta: flush_handlers + +- name: Containerd | Ensure containerd is started and enabled + systemd: + name: containerd + daemon_reload: yes + enabled: yes + state: started diff --git a/kubespray/project/roles/container-engine/containerd/tasks/reset.yml b/kubespray/project/roles/container-engine/containerd/tasks/reset.yml new file mode 100644 index 0000000..517e56d --- /dev/null +++ b/kubespray/project/roles/container-engine/containerd/tasks/reset.yml @@ -0,0 +1,40 @@ +--- +- name: Containerd | Remove containerd repository for RedHat os family + file: + path: "{{ yum_repo_dir }}/containerd.repo" + state: absent + when: + - ansible_os_family in ['RedHat'] + tags: + - reset_containerd + +- name: Containerd | Remove containerd repository for Debian os family + apt_repository: + repo: "{{ item }}" + state: absent + with_items: "{{ containerd_repo_info.repos }}" + when: ansible_pkg_mgr == 'apt' + tags: + - reset_containerd + +- name: Containerd | Stop containerd service + service: + name: containerd + daemon_reload: true + enabled: false + state: stopped + tags: + - reset_containerd + +- name: Containerd | Remove configuration files + file: + path: "{{ item }}" + state: absent + loop: + - /etc/systemd/system/containerd.service + - "{{ containerd_systemd_dir }}" + - "{{ containerd_cfg_dir }}" + - "{{ containerd_storage_dir }}" + - "{{ containerd_state_dir }}" + tags: + - reset_containerd diff --git a/kubespray/project/roles/container-engine/containerd/templates/config.toml.j2 b/kubespray/project/roles/container-engine/containerd/templates/config.toml.j2 new file mode 100644 index 0000000..30773aa --- /dev/null +++ b/kubespray/project/roles/container-engine/containerd/templates/config.toml.j2 @@ -0,0 +1,97 @@ +version = 2 +root = "{{ containerd_storage_dir }}" +state = "{{ containerd_state_dir }}" +oom_score = {{ containerd_oom_score }} + +[grpc] + max_recv_message_size = {{ containerd_grpc_max_recv_message_size }} + max_send_message_size = {{ containerd_grpc_max_send_message_size }} + +[debug] + level = "{{ containerd_debug_level }}" + +[metrics] + address = "{{ containerd_metrics_address }}" + grpc_histogram = {{ containerd_metrics_grpc_histogram | lower }} + +[plugins] + [plugins."io.containerd.grpc.v1.cri"] + sandbox_image = "{{ pod_infra_image_repo }}:{{ pod_infra_image_tag }}" + max_container_log_line_size = {{ containerd_max_container_log_line_size }} + enable_unprivileged_ports = {{ containerd_enable_unprivileged_ports | lower }} + enable_unprivileged_icmp = {{ containerd_enable_unprivileged_icmp | lower }} +{% if enable_cdi %} + enable_cdi = true + cdi_spec_dirs = ["/etc/cdi", "/var/run/cdi"] +{% endif %} + [plugins."io.containerd.grpc.v1.cri".containerd] + default_runtime_name = "{{ containerd_default_runtime }}" + snapshotter = "{{ containerd_snapshotter }}" + discard_unpacked_layers = {{ containerd_discard_unpacked_layers | lower }} + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] +{% for runtime in [containerd_runc_runtime] + containerd_additional_runtimes %} + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.{{ runtime.name }}] + runtime_type = "{{ runtime.type }}" + runtime_engine = "{{ runtime.engine }}" + runtime_root = "{{ runtime.root }}" +{% if runtime.base_runtime_spec is defined %} + base_runtime_spec = "{{ containerd_cfg_dir }}/{{ runtime.base_runtime_spec }}" +{% endif %} + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.{{ runtime.name }}.options] +{% for key, value in runtime.options.items() %} +{% if value | string != "true" and value | string != "false" %} + {{ key }} = "{{ value }}" +{% else %} + {{ key }} = {{ value }} +{% endif %} +{% endfor %} +{% endfor %} +{% if kata_containers_enabled %} + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-qemu] + runtime_type = "io.containerd.kata-qemu.v2" +{% endif %} +{% if gvisor_enabled %} + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runsc] + runtime_type = "io.containerd.runsc.v1" +{% endif %} + [plugins."io.containerd.grpc.v1.cri".registry] +{% if containerd_use_config_path is defined and containerd_use_config_path|bool %} + config_path = "{{ containerd_cfg_dir }}/certs.d" +{% else %} + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] +{% set insecure_registries_addr = [] %} +{% for registry in containerd_registries_mirrors %} + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{ registry.prefix }}"] +{% set endpoint = [] %} +{% for mirror in registry.mirrors %} +{% if endpoint.append(mirror.host) %}{% endif %} +{% if mirror.skip_verify is defined and mirror.skip_verify|bool %}{% if insecure_registries_addr.append(mirror.host | urlsplit('netloc')) %}{% endif %}{% endif %} +{% endfor %} + endpoint = ["{{ ( endpoint | unique ) | join('","') }}"] +{% endfor %} +{% for addr in insecure_registries_addr | unique %} + [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ addr }}".tls] + insecure_skip_verify = true +{% endfor %} +{% endif %} +{% for registry in containerd_registry_auth if registry['registry'] is defined %} +{% if (registry['username'] is defined and registry['password'] is defined) or registry['auth'] is defined %} + [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ registry['registry'] }}".auth] +{% if registry['username'] is defined and registry['password'] is defined %} + password = "{{ registry['password'] }}" + username = "{{ registry['username'] }}" +{% else %} + auth = "{{ registry['auth'] }}" +{% endif %} +{% endif %} +{% endfor %} + +{% if nri_enabled and containerd_version is version('1.7.0', '>=') %} + [plugins."io.containerd.nri.v1.nri"] + disable = false +{% endif %} + +{% if containerd_extra_args is defined %} +{{ containerd_extra_args }} +{% endif %} diff --git a/kubespray/project/roles/container-engine/containerd/templates/containerd.service.j2 b/kubespray/project/roles/container-engine/containerd/templates/containerd.service.j2 new file mode 100644 index 0000000..06b2290 --- /dev/null +++ b/kubespray/project/roles/container-engine/containerd/templates/containerd.service.j2 @@ -0,0 +1,45 @@ +# Copyright The containerd Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[Unit] +Description=containerd container runtime +Documentation=https://containerd.io +After=network.target local-fs.target + +[Service] +ExecStartPre=-/sbin/modprobe overlay +ExecStart={{ containerd_bin_dir }}/containerd + +Type=notify +Delegate=yes +KillMode=process +Restart=always +RestartSec=5 +# Having non-zero Limit*s causes performance problems due to accounting overhead +# in the kernel. We recommend using cgroups to do container-local accounting. +LimitNPROC={{ containerd_limit_proc_num }} +LimitCORE={{ containerd_limit_core }} +LimitNOFILE={{ containerd_limit_open_file_num }} +LimitMEMLOCK={{ containerd_limit_mem_lock }} +# Comment TasksMax if your systemd version does not supports it. +# Only systemd 226 and above support this version. +TasksMax=infinity +OOMScoreAdjust=-999 +# Set the cgroup slice of the service so that kube reserved takes effect +{% if kube_reserved is defined and kube_reserved|bool %} +Slice={{ kube_reserved_cgroups_for_service_slice }} +{% endif %} + +[Install] +WantedBy=multi-user.target diff --git a/kubespray/project/roles/container-engine/containerd/templates/hosts.toml.j2 b/kubespray/project/roles/container-engine/containerd/templates/hosts.toml.j2 new file mode 100644 index 0000000..ea003ed --- /dev/null +++ b/kubespray/project/roles/container-engine/containerd/templates/hosts.toml.j2 @@ -0,0 +1,7 @@ +server = "https://{{ item.prefix }}" +{% for mirror in item.mirrors %} +[host."{{ mirror.host }}"] + capabilities = ["{{ ([ mirror.capabilities ] | flatten ) | join('","') }}"] + skip_verify = {{ mirror.skip_verify | default('false') | string | lower }} + override_path = {{ mirror.override_path | default('false') | string | lower }} +{% endfor %} diff --git a/kubespray/project/roles/container-engine/containerd/templates/http-proxy.conf.j2 b/kubespray/project/roles/container-engine/containerd/templates/http-proxy.conf.j2 new file mode 100644 index 0000000..212f30f --- /dev/null +++ b/kubespray/project/roles/container-engine/containerd/templates/http-proxy.conf.j2 @@ -0,0 +1,2 @@ +[Service] +Environment={% if http_proxy is defined %}"HTTP_PROXY={{ http_proxy }}"{% endif %} {% if https_proxy is defined %}"HTTPS_PROXY={{ https_proxy }}"{% endif %} {% if no_proxy is defined %}"NO_PROXY={{ no_proxy }}"{% endif %} diff --git a/kubespray/project/roles/container-engine/containerd/vars/debian.yml b/kubespray/project/roles/container-engine/containerd/vars/debian.yml new file mode 100644 index 0000000..8b18d9a --- /dev/null +++ b/kubespray/project/roles/container-engine/containerd/vars/debian.yml @@ -0,0 +1,7 @@ +--- +containerd_repo_info: + repos: + - > + deb {{ containerd_debian_repo_base_url }} + {{ ansible_distribution_release | lower }} + {{ containerd_debian_repo_component }} diff --git a/kubespray/project/roles/container-engine/containerd/vars/ubuntu.yml b/kubespray/project/roles/container-engine/containerd/vars/ubuntu.yml new file mode 100644 index 0000000..dd77532 --- /dev/null +++ b/kubespray/project/roles/container-engine/containerd/vars/ubuntu.yml @@ -0,0 +1,7 @@ +--- +containerd_repo_info: + repos: + - > + deb {{ containerd_ubuntu_repo_base_url }} + {{ ansible_distribution_release | lower }} + {{ containerd_ubuntu_repo_component }} diff --git a/kubespray/project/roles/container-engine/cri-dockerd/handlers/main.yml b/kubespray/project/roles/container-engine/cri-dockerd/handlers/main.yml new file mode 100644 index 0000000..3a24979 --- /dev/null +++ b/kubespray/project/roles/container-engine/cri-dockerd/handlers/main.yml @@ -0,0 +1,31 @@ +--- +- name: Cri-dockerd | reload systemd + systemd: + name: cri-dockerd + daemon_reload: true + masked: no + listen: Restart and enable cri-dockerd + +- name: Cri-dockerd | restart docker.service + service: + name: docker.service + state: restarted + listen: Restart and enable cri-dockerd + +- name: Cri-dockerd | reload cri-dockerd.socket + service: + name: cri-dockerd.socket + state: restarted + listen: Restart and enable cri-dockerd + +- name: Cri-dockerd | reload cri-dockerd.service + service: + name: cri-dockerd.service + state: restarted + listen: Restart and enable cri-dockerd + +- name: Cri-dockerd | enable cri-dockerd service + service: + name: cri-dockerd.service + enabled: yes + listen: Restart and enable cri-dockerd diff --git a/kubespray/project/roles/container-engine/cri-dockerd/meta/main.yml b/kubespray/project/roles/container-engine/cri-dockerd/meta/main.yml new file mode 100644 index 0000000..4923f3b --- /dev/null +++ b/kubespray/project/roles/container-engine/cri-dockerd/meta/main.yml @@ -0,0 +1,4 @@ +--- +dependencies: + - role: container-engine/docker + - role: container-engine/crictl diff --git a/kubespray/project/roles/container-engine/cri-dockerd/molecule/default/converge.yml b/kubespray/project/roles/container-engine/cri-dockerd/molecule/default/converge.yml new file mode 100644 index 0000000..be6fa38 --- /dev/null +++ b/kubespray/project/roles/container-engine/cri-dockerd/molecule/default/converge.yml @@ -0,0 +1,9 @@ +--- +- name: Converge + hosts: all + become: true + vars: + container_manager: docker + roles: + - role: kubespray-defaults + - role: container-engine/cri-dockerd diff --git a/kubespray/project/roles/container-engine/cri-dockerd/molecule/default/files/10-mynet.conf b/kubespray/project/roles/container-engine/cri-dockerd/molecule/default/files/10-mynet.conf new file mode 100644 index 0000000..f10935b --- /dev/null +++ b/kubespray/project/roles/container-engine/cri-dockerd/molecule/default/files/10-mynet.conf @@ -0,0 +1,17 @@ +{ + "cniVersion": "0.2.0", + "name": "mynet", + "type": "bridge", + "bridge": "cni0", + "isGateway": true, + "ipMasq": true, + "ipam": { + "type": "host-local", + "subnet": "172.19.0.0/24", + "routes": [ + { + "dst": "0.0.0.0/0" + } + ] + } +} diff --git a/kubespray/project/roles/container-engine/cri-dockerd/molecule/default/files/container.json b/kubespray/project/roles/container-engine/cri-dockerd/molecule/default/files/container.json new file mode 100644 index 0000000..1d839e6 --- /dev/null +++ b/kubespray/project/roles/container-engine/cri-dockerd/molecule/default/files/container.json @@ -0,0 +1,10 @@ +{ + "metadata": { + "name": "cri-dockerd1" + }, + "image": { + "image": "quay.io/kubespray/hello-world:latest" + }, + "log_path": "cri-dockerd1.0.log", + "linux": {} +} diff --git a/kubespray/project/roles/container-engine/cri-dockerd/molecule/default/files/sandbox.json b/kubespray/project/roles/container-engine/cri-dockerd/molecule/default/files/sandbox.json new file mode 100644 index 0000000..f451e9e --- /dev/null +++ b/kubespray/project/roles/container-engine/cri-dockerd/molecule/default/files/sandbox.json @@ -0,0 +1,10 @@ +{ + "metadata": { + "name": "cri-dockerd1", + "namespace": "default", + "attempt": 1, + "uid": "hdishd83djaidwnduwk28bcsb" + }, + "linux": {}, + "log_directory": "/tmp" +} diff --git a/kubespray/project/roles/container-engine/cri-dockerd/molecule/default/molecule.yml b/kubespray/project/roles/container-engine/cri-dockerd/molecule/default/molecule.yml new file mode 100644 index 0000000..82cb778 --- /dev/null +++ b/kubespray/project/roles/container-engine/cri-dockerd/molecule/default/molecule.yml @@ -0,0 +1,39 @@ +--- +role_name_check: 1 +driver: + name: vagrant + provider: + name: libvirt +platforms: + - name: almalinux8 + box: almalinux/8 + cpus: 1 + memory: 1024 + nested: true + groups: + - kube_control_plane + provider_options: + driver: kvm + - name: ubuntu20 + box: generic/ubuntu2004 + cpus: 1 + memory: 1024 + nested: true + groups: + - kube_control_plane + provider_options: + driver: kvm +provisioner: + name: ansible + env: + ANSIBLE_ROLES_PATH: ../../../../ + config_options: + defaults: + callbacks_enabled: profile_tasks + timeout: 120 + inventory: + group_vars: + all: + become: true +verifier: + name: testinfra diff --git a/kubespray/project/roles/container-engine/cri-dockerd/molecule/default/prepare.yml b/kubespray/project/roles/container-engine/cri-dockerd/molecule/default/prepare.yml new file mode 100644 index 0000000..83449f8 --- /dev/null +++ b/kubespray/project/roles/container-engine/cri-dockerd/molecule/default/prepare.yml @@ -0,0 +1,48 @@ +--- +- name: Prepare + hosts: all + become: true + roles: + - role: kubespray-defaults + - role: bootstrap-os + - role: adduser + user: "{{ addusers.kube }}" + tasks: + - name: Download CNI + include_tasks: "../../../../download/tasks/download_file.yml" + vars: + download: "{{ download_defaults | combine(downloads.cni) }}" + +- name: Prepare container runtime + hosts: all + become: true + vars: + container_manager: containerd + kube_network_plugin: cni + roles: + - role: kubespray-defaults + - role: network_plugin/cni + tasks: + - name: Copy test container files + copy: + src: "{{ item }}" + dest: "/tmp/{{ item }}" + owner: root + mode: 0644 + with_items: + - container.json + - sandbox.json + - name: Create /etc/cni/net.d directory + file: + path: /etc/cni/net.d + state: directory + owner: "{{ kube_owner }}" + mode: 0755 + - name: Setup CNI + copy: + src: "{{ item }}" + dest: "/etc/cni/net.d/{{ item }}" + owner: root + mode: 0644 + with_items: + - 10-mynet.conf diff --git a/kubespray/project/roles/container-engine/cri-dockerd/molecule/default/tests/test_default.py b/kubespray/project/roles/container-engine/cri-dockerd/molecule/default/tests/test_default.py new file mode 100644 index 0000000..dc99b34 --- /dev/null +++ b/kubespray/project/roles/container-engine/cri-dockerd/molecule/default/tests/test_default.py @@ -0,0 +1,19 @@ +import os + +import testinfra.utils.ansible_runner + +testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( + os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') + + +def test_run_pod(host): + run_command = "/usr/local/bin/crictl run --with-pull /tmp/container.json /tmp/sandbox.json" + with host.sudo(): + cmd = host.command(run_command) + assert cmd.rc == 0 + + with host.sudo(): + log_f = host.file("/tmp/cri-dockerd1.0.log") + + assert log_f.exists + assert b"Hello from Docker" in log_f.content diff --git a/kubespray/project/roles/container-engine/cri-dockerd/tasks/main.yml b/kubespray/project/roles/container-engine/cri-dockerd/tasks/main.yml new file mode 100644 index 0000000..730e379 --- /dev/null +++ b/kubespray/project/roles/container-engine/cri-dockerd/tasks/main.yml @@ -0,0 +1,31 @@ +--- +- name: Runc | Download cri-dockerd binary + include_tasks: "../../../download/tasks/download_file.yml" + vars: + download: "{{ download_defaults | combine(downloads.cri_dockerd) }}" + +- name: Copy cri-dockerd binary from download dir + copy: + src: "{{ local_release_dir }}/cri-dockerd" + dest: "{{ bin_dir }}/cri-dockerd" + mode: 0755 + remote_src: true + notify: + - Restart and enable cri-dockerd + +- name: Generate cri-dockerd systemd unit files + template: + src: "{{ item }}.j2" + dest: "/etc/systemd/system/{{ item }}" + mode: 0644 + validate: "sh -c '[ -f /usr/bin/systemd/system/factory-reset.target ] || exit 0 && systemd-analyze verify %s:{{ item }}'" + # FIXME: check that systemd version >= 250 (factory-reset.target was introduced in that release) + # Remove once we drop support for systemd < 250 + with_items: + - cri-dockerd.service + - cri-dockerd.socket + notify: + - Restart and enable cri-dockerd + +- name: Flush handlers + meta: flush_handlers diff --git a/kubespray/project/roles/container-engine/cri-dockerd/templates/cri-dockerd.service.j2 b/kubespray/project/roles/container-engine/cri-dockerd/templates/cri-dockerd.service.j2 new file mode 100644 index 0000000..ec12815 --- /dev/null +++ b/kubespray/project/roles/container-engine/cri-dockerd/templates/cri-dockerd.service.j2 @@ -0,0 +1,44 @@ +[Unit] +Description=CRI Interface for Docker Application Container Engine +Documentation=https://docs.mirantis.com +After=network-online.target firewalld.service docker.service +Wants=network-online.target docker.service +Requires=cri-dockerd.socket + +[Service] +Type=notify +ExecStart={{ bin_dir }}/cri-dockerd --container-runtime-endpoint {{ cri_socket }} --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin --network-plugin=cni --pod-cidr={{ kube_pods_subnet }} --pod-infra-container-image={{ pod_infra_image_repo }}:{{ pod_infra_version }} {% if enable_dual_stack_networks %}--ipv6-dual-stack=True{% endif %} + +ExecReload=/bin/kill -s HUP $MAINPID +TimeoutSec=0 +RestartSec=2 +Restart=always + +# Note that StartLimit* options were moved from "Service" to "Unit" in systemd 229. +# Both the old, and new location are accepted by systemd 229 and up, so using the old location +# to make them work for either version of systemd. +StartLimitBurst=3 + +# Note that StartLimitInterval was renamed to StartLimitIntervalSec in systemd 230. +# Both the old, and new name are accepted by systemd 230 and up, so using the old name to make +# this option work for either version of systemd. +StartLimitInterval=60s + +# Having non-zero Limit*s causes performance problems due to accounting overhead +# in the kernel. We recommend using cgroups to do container-local accounting. +LimitNOFILE=infinity +LimitNPROC=infinity +LimitCORE=infinity + +# Comment TasksMax if your systemd version does not support it. +# Only systemd 226 and above support this option. +TasksMax=infinity +Delegate=yes +KillMode=process +# Set the cgroup slice of the service so that kube reserved takes effect +{% if kube_reserved is defined and kube_reserved|bool %} +Slice={{ kube_reserved_cgroups_for_service_slice }} +{% endif %} + +[Install] +WantedBy=multi-user.target diff --git a/kubespray/project/roles/container-engine/cri-dockerd/templates/cri-dockerd.socket.j2 b/kubespray/project/roles/container-engine/cri-dockerd/templates/cri-dockerd.socket.j2 new file mode 100644 index 0000000..8dfa27d --- /dev/null +++ b/kubespray/project/roles/container-engine/cri-dockerd/templates/cri-dockerd.socket.j2 @@ -0,0 +1,12 @@ +[Unit] +Description=CRI Docker Socket for the API +PartOf=cri-dockerd.service + +[Socket] +ListenStream=%t/cri-dockerd.sock +SocketMode=0660 +SocketUser=root +SocketGroup=docker + +[Install] +WantedBy=sockets.target diff --git a/kubespray/project/roles/container-engine/cri-o/defaults/main.yml b/kubespray/project/roles/container-engine/cri-o/defaults/main.yml new file mode 100644 index 0000000..6bc0d72 --- /dev/null +++ b/kubespray/project/roles/container-engine/cri-o/defaults/main.yml @@ -0,0 +1,91 @@ +--- + +crio_cgroup_manager: "{{ kubelet_cgroup_driver | default('systemd') }}" +crio_conmon: "{{ bin_dir }}/conmon" +crio_enable_metrics: false +crio_log_level: "info" +crio_metrics_port: "9090" +crio_pause_image: "{{ pod_infra_image_repo }}:{{ pod_infra_version }}" + +# Registries defined within cri-o. +# By default unqualified images are not allowed for security reasons +crio_registries: [] +# - prefix: docker.io +# insecure: false +# blocked: false +# location: registry-1.docker.io ## REQUIRED +# unqualified: false +# mirrors: +# - location: 172.20.100.52:5000 +# insecure: true +# - location: mirror.gcr.io +# insecure: false + +crio_registry_auth: [] +# - registry: 10.0.0.2:5000 +# username: user +# password: pass + +crio_seccomp_profile: "" +crio_selinux: "{{ (preinstall_selinux_state == 'enforcing') | lower }}" +crio_signature_policy: "{% if ansible_os_family == 'ClearLinux' %}/usr/share/defaults/crio/policy.json{% endif %}" + +# Override system default for storage driver +# crio_storage_driver: "overlay" + +crio_stream_port: "10010" + +crio_required_version: "{{ kube_version | regex_replace('^v(?P\\d+).(?P\\d+).(?P\\d+)$', '\\g.\\g') }}" + +# The crio_runtimes variable defines a list of OCI compatible runtimes. +crio_runtimes: + - name: runc + path: "{{ bin_dir }}/runc" + type: oci + root: /run/runc + +# Kata Containers is an OCI runtime, where containers are run inside lightweight +# VMs. Kata provides additional isolation towards the host, minimizing the host attack +# surface and mitigating the consequences of containers breakout. +kata_runtimes: + # Kata Containers with the default configured VMM + - name: kata-qemu + path: /usr/local/bin/containerd-shim-kata-qemu-v2 + type: vm + root: /run/kata-containers + privileged_without_host_devices: true + +# crun is a fast and low-memory footprint OCI Container Runtime fully written in C. +crun_runtime: + name: crun + path: "{{ bin_dir }}/crun" + type: oci + root: /run/crun + +# youki is an implementation of the OCI runtime-spec in Rust, similar to runc. +youki_runtime: + name: youki + path: "{{ youki_bin_dir }}/youki" + type: oci + root: /run/youki + +# Reserve 16M uids and gids for user namespaces (256 pods * 65536 uids/gids) +# at the end of the uid/gid space +crio_remap_enable: false +crio_remap_user: containers +crio_subuid_start: 2130706432 +crio_subuid_length: 16777216 +crio_subgid_start: 2130706432 +crio_subgid_length: 16777216 + +# cri-o manual files +crio_man_files: + 5: + - crio.conf + - crio.conf.d + 8: + - crio + - crio-status + +# If set to true, it will enable the CRIU support in cri-o +crio_criu_support_enabled: false diff --git a/kubespray/project/roles/container-engine/cri-o/files/mounts.conf b/kubespray/project/roles/container-engine/cri-o/files/mounts.conf new file mode 100644 index 0000000..b7cde9d --- /dev/null +++ b/kubespray/project/roles/container-engine/cri-o/files/mounts.conf @@ -0,0 +1 @@ +/usr/share/rhel/secrets:/run/secrets diff --git a/kubespray/project/roles/container-engine/cri-o/handlers/main.yml b/kubespray/project/roles/container-engine/cri-o/handlers/main.yml new file mode 100644 index 0000000..b0c5951 --- /dev/null +++ b/kubespray/project/roles/container-engine/cri-o/handlers/main.yml @@ -0,0 +1,12 @@ +--- +- name: CRI-O | reload systemd + systemd: + daemon_reload: true + listen: Restart crio + +- name: CRI-O | reload crio + service: + name: crio + state: restarted + enabled: yes + listen: Restart crio diff --git a/kubespray/project/roles/container-engine/cri-o/meta/main.yml b/kubespray/project/roles/container-engine/cri-o/meta/main.yml new file mode 100644 index 0000000..7259b46 --- /dev/null +++ b/kubespray/project/roles/container-engine/cri-o/meta/main.yml @@ -0,0 +1,5 @@ +--- +dependencies: + - role: container-engine/runc + - role: container-engine/crictl + - role: container-engine/skopeo diff --git a/kubespray/project/roles/container-engine/cri-o/molecule/default/converge.yml b/kubespray/project/roles/container-engine/cri-o/molecule/default/converge.yml new file mode 100644 index 0000000..376f07c --- /dev/null +++ b/kubespray/project/roles/container-engine/cri-o/molecule/default/converge.yml @@ -0,0 +1,9 @@ +--- +- name: Converge + hosts: all + become: true + vars: + container_manager: crio + roles: + - role: kubespray-defaults + - role: container-engine/cri-o diff --git a/kubespray/project/roles/container-engine/cri-o/molecule/default/files/10-mynet.conf b/kubespray/project/roles/container-engine/cri-o/molecule/default/files/10-mynet.conf new file mode 100644 index 0000000..f10935b --- /dev/null +++ b/kubespray/project/roles/container-engine/cri-o/molecule/default/files/10-mynet.conf @@ -0,0 +1,17 @@ +{ + "cniVersion": "0.2.0", + "name": "mynet", + "type": "bridge", + "bridge": "cni0", + "isGateway": true, + "ipMasq": true, + "ipam": { + "type": "host-local", + "subnet": "172.19.0.0/24", + "routes": [ + { + "dst": "0.0.0.0/0" + } + ] + } +} diff --git a/kubespray/project/roles/container-engine/cri-o/molecule/default/files/container.json b/kubespray/project/roles/container-engine/cri-o/molecule/default/files/container.json new file mode 100644 index 0000000..bcd71e7 --- /dev/null +++ b/kubespray/project/roles/container-engine/cri-o/molecule/default/files/container.json @@ -0,0 +1,10 @@ +{ + "metadata": { + "name": "runc1" + }, + "image": { + "image": "quay.io/kubespray/hello-world:latest" + }, + "log_path": "runc1.0.log", + "linux": {} +} diff --git a/kubespray/project/roles/container-engine/cri-o/molecule/default/files/sandbox.json b/kubespray/project/roles/container-engine/cri-o/molecule/default/files/sandbox.json new file mode 100644 index 0000000..eb9dcb9 --- /dev/null +++ b/kubespray/project/roles/container-engine/cri-o/molecule/default/files/sandbox.json @@ -0,0 +1,10 @@ +{ + "metadata": { + "name": "runc1", + "namespace": "default", + "attempt": 1, + "uid": "hdishd83djaidwnduwk28bcsb" + }, + "linux": {}, + "log_directory": "/tmp" +} diff --git a/kubespray/project/roles/container-engine/cri-o/molecule/default/molecule.yml b/kubespray/project/roles/container-engine/cri-o/molecule/default/molecule.yml new file mode 100644 index 0000000..99d44a3 --- /dev/null +++ b/kubespray/project/roles/container-engine/cri-o/molecule/default/molecule.yml @@ -0,0 +1,57 @@ +--- +role_name_check: 1 +driver: + name: vagrant + provider: + name: libvirt +platforms: + - name: ubuntu20 + box: generic/ubuntu2004 + cpus: 2 + memory: 1024 + groups: + - kube_control_plane + - kube_node + - k8s_cluster + provider_options: + driver: kvm + - name: almalinux8 + box: almalinux/8 + cpus: 2 + memory: 1024 + groups: + - kube_control_plane + - kube_node + - k8s_cluster + provider_options: + driver: kvm + - name: fedora + box: fedora/38-cloud-base + cpus: 2 + memory: 2048 + groups: + - kube_control_plane + - kube_node + - k8s_cluster + provider_options: + driver: kvm + - name: debian10 + box: generic/debian10 + cpus: 2 + memory: 1024 + groups: + - kube_control_plane + - kube_node + - k8s_cluster + provider_options: + driver: kvm +provisioner: + name: ansible + env: + ANSIBLE_ROLES_PATH: ../../../../ + config_options: + defaults: + callbacks_enabled: profile_tasks + timeout: 120 +verifier: + name: testinfra diff --git a/kubespray/project/roles/container-engine/cri-o/molecule/default/prepare.yml b/kubespray/project/roles/container-engine/cri-o/molecule/default/prepare.yml new file mode 100644 index 0000000..103b0d3 --- /dev/null +++ b/kubespray/project/roles/container-engine/cri-o/molecule/default/prepare.yml @@ -0,0 +1,53 @@ +--- +- name: Prepare + hosts: all + gather_facts: False + become: true + vars: + ignore_assert_errors: true + roles: + - role: kubespray-defaults + - role: bootstrap-os + - role: kubernetes/preinstall + - role: adduser + user: "{{ addusers.kube }}" + tasks: + - name: Download CNI + include_tasks: "../../../../download/tasks/download_file.yml" + vars: + download: "{{ download_defaults | combine(downloads.cni) }}" + +- name: Prepare CNI + hosts: all + gather_facts: False + become: true + vars: + ignore_assert_errors: true + kube_network_plugin: cni + roles: + - role: kubespray-defaults + - role: network_plugin/cni + tasks: + - name: Copy test container files + copy: + src: "{{ item }}" + dest: "/tmp/{{ item }}" + owner: root + mode: 0644 + with_items: + - container.json + - sandbox.json + - name: Create /etc/cni/net.d directory + file: + path: /etc/cni/net.d + state: directory + owner: "{{ kube_owner }}" + mode: 0755 + - name: Setup CNI + copy: + src: "{{ item }}" + dest: "/etc/cni/net.d/{{ item }}" + owner: root + mode: 0644 + with_items: + - 10-mynet.conf diff --git a/kubespray/project/roles/container-engine/cri-o/molecule/default/tests/test_default.py b/kubespray/project/roles/container-engine/cri-o/molecule/default/tests/test_default.py new file mode 100644 index 0000000..358a1b7 --- /dev/null +++ b/kubespray/project/roles/container-engine/cri-o/molecule/default/tests/test_default.py @@ -0,0 +1,35 @@ +import os + +import testinfra.utils.ansible_runner + +testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( + os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') + + +def test_service(host): + svc = host.service("crio") + assert svc.is_running + assert svc.is_enabled + + +def test_run(host): + crictl = "/usr/local/bin/crictl" + path = "unix:///var/run/crio/crio.sock" + with host.sudo(): + cmd = host.command(crictl + " --runtime-endpoint " + path + " version") + assert cmd.rc == 0 + assert "RuntimeName: cri-o" in cmd.stdout + +def test_run_pod(host): + runtime = "runc" + + run_command = "/usr/local/bin/crictl run --with-pull --runtime {} /tmp/container.json /tmp/sandbox.json".format(runtime) + with host.sudo(): + cmd = host.command(run_command) + assert cmd.rc == 0 + + with host.sudo(): + log_f = host.file("/tmp/runc1.0.log") + + assert log_f.exists + assert b"Hello from Docker" in log_f.content diff --git a/kubespray/project/roles/container-engine/cri-o/tasks/load_vars.yml b/kubespray/project/roles/container-engine/cri-o/tasks/load_vars.yml new file mode 100644 index 0000000..8a5daa3 --- /dev/null +++ b/kubespray/project/roles/container-engine/cri-o/tasks/load_vars.yml @@ -0,0 +1,8 @@ +--- +- name: Cri-o | include vars/v1.28.yml + include_vars: v1.28.yml + when: crio_version is version("v1.29.0", operator="<") + +- name: Cri-o | include vars/v1.29.yml + include_vars: v1.29.yml + when: crio_version is version("v1.29.0", operator=">=") diff --git a/kubespray/project/roles/container-engine/cri-o/tasks/main.yaml b/kubespray/project/roles/container-engine/cri-o/tasks/main.yaml new file mode 100644 index 0000000..99a3ed2 --- /dev/null +++ b/kubespray/project/roles/container-engine/cri-o/tasks/main.yaml @@ -0,0 +1,214 @@ +--- +- name: Cri-o | load vars + import_tasks: load_vars.yml + +- name: Cri-o | check if fedora coreos + stat: + path: /run/ostree-booted + get_attributes: no + get_checksum: no + get_mime: no + register: ostree + +- name: Cri-o | set is_ostree + set_fact: + is_ostree: "{{ ostree.stat.exists }}" + +- name: Cri-o | get ostree version + shell: "set -o pipefail && rpm-ostree --version | awk -F\\' '/Version/{print $2}'" + args: + executable: /bin/bash + register: ostree_version + when: is_ostree + +- name: Cri-o | Download cri-o + include_tasks: "../../../download/tasks/download_file.yml" + vars: + download: "{{ download_defaults | combine(downloads.crio) }}" + +- name: Cri-o | special handling for amazon linux + import_tasks: "setup-amazon.yaml" + when: ansible_distribution in ["Amazon"] + +- name: Cri-o | build a list of crio runtimes with Katacontainers runtimes + set_fact: + crio_runtimes: "{{ crio_runtimes + kata_runtimes }}" + when: + - kata_containers_enabled + +- name: Cri-o | build a list of crio runtimes with crun runtime + set_fact: + crio_runtimes: "{{ crio_runtimes + [crun_runtime] }}" + when: + - crun_enabled + +- name: Cri-o | build a list of crio runtimes with youki runtime + set_fact: + crio_runtimes: "{{ crio_runtimes + [youki_runtime] }}" + when: + - youki_enabled + +- name: Cri-o | make sure needed folders exist in the system + with_items: + - /etc/crio + - /etc/containers + - /etc/systemd/system/crio.service.d + file: + path: "{{ item }}" + state: directory + mode: 0755 + +- name: Cri-o | install cri-o config + template: + src: crio.conf.j2 + dest: /etc/crio/crio.conf + mode: 0644 + register: config_install + +- name: Cri-o | install config.json + template: + src: config.json.j2 + dest: /etc/crio/config.json + mode: 0644 + register: reg_auth_install + +- name: Cri-o | copy binaries + copy: + src: "{{ local_release_dir }}/cri-o/bin/{{ item }}" + dest: "{{ bin_dir }}/{{ item }}" + mode: 0755 + remote_src: true + with_items: + - "{{ crio_bin_files }}" + notify: Restart crio + +- name: Cri-o | copy service file + copy: + src: "{{ local_release_dir }}/cri-o/contrib/crio.service" + dest: /etc/systemd/system/crio.service + mode: 0755 + remote_src: true + notify: Restart crio + +- name: Cri-o | update the bin dir for crio.service file + replace: + dest: /etc/systemd/system/crio.service + regexp: "/usr/local/bin/crio" + replace: "{{ bin_dir }}/crio" + notify: Restart crio + +- name: Cri-o | copy default policy + copy: + src: "{{ local_release_dir }}/cri-o/contrib/policy.json" + dest: /etc/containers/policy.json + mode: 0755 + remote_src: true + notify: Restart crio + +- name: Cri-o | copy mounts.conf + copy: + src: mounts.conf + dest: /etc/containers/mounts.conf + mode: 0644 + when: + - ansible_os_family == 'RedHat' + notify: Restart crio + +- name: Cri-o | create directory for oci hooks + file: + path: /etc/containers/oci/hooks.d + state: directory + owner: root + mode: 0755 + +- name: Cri-o | set overlay driver + community.general.ini_file: + dest: /etc/containers/storage.conf + section: storage + option: "{{ item.option }}" + value: "{{ item.value }}" + mode: 0644 + with_items: + - option: driver + value: '"overlay"' + - option: graphroot + value: '"/var/lib/containers/storage"' + - option: runroot + value: '"/var/run/containers/storage"' + +# metacopy=on is available since 4.19 and was backported to RHEL 4.18 kernel +- name: Cri-o | set metacopy mount options correctly + community.general.ini_file: + dest: /etc/containers/storage.conf + section: storage.options.overlay + option: mountopt + value: '{{ ''"nodev"'' if ansible_kernel is version_compare(("4.18" if ansible_os_family == "RedHat" else "4.19"), "<") else ''"nodev,metacopy=on"'' }}' + mode: 0644 + +- name: Cri-o | create directory registries configs + file: + path: /etc/containers/registries.conf.d + state: directory + owner: root + mode: 0755 + +- name: Cri-o | write registries configs + template: + src: registry.conf.j2 + dest: "/etc/containers/registries.conf.d/10-{{ item.prefix | default(item.location) | regex_replace(':', '_') }}.conf" + mode: 0644 + loop: "{{ crio_registries }}" + notify: Restart crio + +- name: Cri-o | configure unqualified registry settings + template: + src: unqualified.conf.j2 + dest: "/etc/containers/registries.conf.d/01-unqualified.conf" + mode: 0644 + notify: Restart crio + +- name: Cri-o | write cri-o proxy drop-in + template: + src: http-proxy.conf.j2 + dest: /etc/systemd/system/crio.service.d/http-proxy.conf + mode: 0644 + notify: Restart crio + when: http_proxy is defined or https_proxy is defined + +- name: Cri-o | configure the uid/gid space for user namespaces + lineinfile: + path: '{{ item.path }}' + line: '{{ item.entry }}' + regex: '^\s*{{ crio_remap_user }}:' + state: '{{ "present" if crio_remap_enable | bool else "absent" }}' + loop: + - path: /etc/subuid + entry: '{{ crio_remap_user }}:{{ crio_subuid_start }}:{{ crio_subuid_length }}' + - path: /etc/subgid + entry: '{{ crio_remap_user }}:{{ crio_subgid_start }}:{{ crio_subgid_length }}' + loop_control: + label: '{{ item.path }}' + +- name: Cri-o | ensure crio service is started and enabled + service: + name: crio + daemon_reload: true + enabled: true + state: started + register: service_start + +- name: Cri-o | trigger service restart only when needed + service: + name: crio + state: restarted + when: + - config_install.changed or reg_auth_install.changed + - not service_start.changed + +- name: Cri-o | verify that crio is running + command: "{{ bin_dir }}/{{ crio_status_command }} info" + register: get_crio_info + until: get_crio_info is succeeded + changed_when: false + retries: 5 + delay: "{{ retry_stagger | random + 3 }}" diff --git a/kubespray/project/roles/container-engine/cri-o/tasks/reset.yml b/kubespray/project/roles/container-engine/cri-o/tasks/reset.yml new file mode 100644 index 0000000..53d4714 --- /dev/null +++ b/kubespray/project/roles/container-engine/cri-o/tasks/reset.yml @@ -0,0 +1,90 @@ +--- +- name: Cri-o | load vars + import_tasks: load_vars.yml + +- name: CRI-O | Kubic repo name for debian os family + set_fact: + crio_kubic_debian_repo_name: "{{ ((ansible_distribution == 'Ubuntu') | ternary('x', '')) ~ ansible_distribution ~ '_' ~ ansible_distribution_version }}" + when: ansible_os_family == "Debian" + tags: + - reset_crio + +- name: CRI-O | Remove kubic apt repo + apt_repository: + repo: "deb http://{{ crio_download_base }}/{{ crio_kubic_debian_repo_name }}/ /" + state: absent + when: crio_kubic_debian_repo_name is defined + tags: + - reset_crio + +- name: CRI-O | Remove cri-o apt repo + apt_repository: + repo: "deb {{ crio_download_crio }}{{ crio_version }}/{{ crio_kubic_debian_repo_name }}/ /" + state: absent + filename: devel-kubic-libcontainers-stable-cri-o + when: crio_kubic_debian_repo_name is defined + tags: + - reset_crio + +- name: CRI-O | Remove CRI-O kubic yum repo + yum_repository: + name: devel_kubic_libcontainers_stable + state: absent + when: ansible_distribution in ["Amazon"] + tags: + - reset_crio + +- name: CRI-O | Remove CRI-O kubic yum repo + yum_repository: + name: "devel_kubic_libcontainers_stable_cri-o_{{ crio_version }}" + state: absent + when: + - ansible_os_family == "RedHat" + - ansible_distribution not in ["Amazon", "Fedora"] + tags: + - reset_crio + +- name: CRI-O | Run yum-clean-metadata + command: yum clean metadata + when: + - ansible_os_family == "RedHat" + tags: + - reset_crio + +- name: CRI-O | Remove crictl + file: + name: "{{ item }}" + state: absent + loop: + - /etc/crictl.yaml + - "{{ bin_dir }}/crictl" + tags: + - reset_crio + +- name: CRI-O | Stop crio service + service: + name: crio + daemon_reload: true + enabled: false + state: stopped + tags: + - reset_crio + +- name: CRI-O | Remove CRI-O configuration files + file: + name: "{{ item }}" + state: absent + loop: + - /etc/crio + - /etc/containers + - /etc/systemd/system/crio.service.d + tags: + - reset_crio + +- name: CRI-O | Remove CRI-O binaries + file: + name: "{{ item }}" + state: absent + with_items: "{{ crio_bin_files }}" + tags: + - reset_crio diff --git a/kubespray/project/roles/container-engine/cri-o/tasks/setup-amazon.yaml b/kubespray/project/roles/container-engine/cri-o/tasks/setup-amazon.yaml new file mode 100644 index 0000000..843bc20 --- /dev/null +++ b/kubespray/project/roles/container-engine/cri-o/tasks/setup-amazon.yaml @@ -0,0 +1,38 @@ +--- +- name: Check that amzn2-extras.repo exists + stat: + path: /etc/yum.repos.d/amzn2-extras.repo + register: amzn2_extras_file_stat + +- name: Find docker repo in amzn2-extras.repo file + lineinfile: + dest: /etc/yum.repos.d/amzn2-extras.repo + line: "[amzn2extra-docker]" + check_mode: yes + register: amzn2_extras_docker_repo + when: + - amzn2_extras_file_stat.stat.exists + +- name: Remove docker repository + community.general.ini_file: + dest: /etc/yum.repos.d/amzn2-extras.repo + section: amzn2extra-docker + option: enabled + value: "0" + backup: yes + mode: 0644 + when: + - amzn2_extras_file_stat.stat.exists + - not amzn2_extras_docker_repo.changed + +- name: Add container-selinux yum repo + yum_repository: + name: copr:copr.fedorainfracloud.org:lsm5:container-selinux + file: _copr_lsm5-container-selinux.repo + description: Copr repo for container-selinux owned by lsm5 + baseurl: https://download.copr.fedorainfracloud.org/results/lsm5/container-selinux/epel-7-$basearch/ + gpgcheck: yes + gpgkey: https://download.copr.fedorainfracloud.org/results/lsm5/container-selinux/pubkey.gpg + skip_if_unavailable: yes + enabled: yes + repo_gpgcheck: no diff --git a/kubespray/project/roles/container-engine/cri-o/templates/config.json.j2 b/kubespray/project/roles/container-engine/cri-o/templates/config.json.j2 new file mode 100644 index 0000000..4afd49f --- /dev/null +++ b/kubespray/project/roles/container-engine/cri-o/templates/config.json.j2 @@ -0,0 +1,17 @@ +{% if crio_registry_auth is defined and crio_registry_auth|length %} +{ +{% for reg in crio_registry_auth %} + "auths": { + "{{ reg.registry }}": { + "auth": "{{ (reg.username + ':' + reg.password) | string | b64encode }}" + } +{% if not loop.last %} + }, +{% else %} + } +{% endif %} +{% endfor %} +} +{% else %} +{} +{% endif %} diff --git a/kubespray/project/roles/container-engine/cri-o/templates/crio.conf.j2 b/kubespray/project/roles/container-engine/cri-o/templates/crio.conf.j2 new file mode 100644 index 0000000..6f9b84f --- /dev/null +++ b/kubespray/project/roles/container-engine/cri-o/templates/crio.conf.j2 @@ -0,0 +1,389 @@ +# The CRI-O configuration file specifies all of the available configuration +# options and command-line flags for the crio(8) OCI Kubernetes Container Runtime +# daemon, but in a TOML format that can be more easily modified and versioned. +# +# Please refer to crio.conf(5) for details of all configuration options. + +# CRI-O supports partial configuration reload during runtime, which can be +# done by sending SIGHUP to the running process. Currently supported options +# are explicitly mentioned with: 'This option supports live configuration +# reload'. + +# CRI-O reads its storage defaults from the containers-storage.conf(5) file +# located at /etc/containers/storage.conf. Modify this storage configuration if +# you want to change the system's defaults. If you want to modify storage just +# for CRI-O, you can change the storage configuration options here. +[crio] + +# Path to the "root directory". CRI-O stores all of its data, including +# containers images, in this directory. +root = "/var/lib/containers/storage" + +# Path to the "run directory". CRI-O stores all of its state in this directory. +# Read from /etc/containers/storage.conf first so unnecessary here +# runroot = "/var/run/containers/storage" + +# Storage driver used to manage the storage of images and containers. Please +# refer to containers-storage.conf(5) to see all available storage drivers. +{% if crio_storage_driver is defined %} +storage_driver = "{{ crio_storage_driver }}" +{% endif %} + +# List to pass options to the storage driver. Please refer to +# containers-storage.conf(5) to see all available storage options. +#storage_option = [ +#] + +# The default log directory where all logs will go unless directly specified by +# the kubelet. The log directory specified must be an absolute directory. +log_dir = "/var/log/crio/pods" + +# Location for CRI-O to lay down the temporary version file. +# It is used to check if crio wipe should wipe containers, which should +# always happen on a node reboot +version_file = "/var/run/crio/version" + +# Location for CRI-O to lay down the persistent version file. +# It is used to check if crio wipe should wipe images, which should +# only happen when CRI-O has been upgraded +version_file_persist = "/var/lib/crio/version" + +# The crio.api table contains settings for the kubelet/gRPC interface. +[crio.api] + +# Path to AF_LOCAL socket on which CRI-O will listen. +listen = "/var/run/crio/crio.sock" + +# IP address on which the stream server will listen. +stream_address = "127.0.0.1" + +# The port on which the stream server will listen. If the port is set to "0", then +# CRI-O will allocate a random free port number. +stream_port = "{{ crio_stream_port }}" + +# Enable encrypted TLS transport of the stream server. +stream_enable_tls = false + +# Path to the x509 certificate file used to serve the encrypted stream. This +# file can change, and CRI-O will automatically pick up the changes within 5 +# minutes. +stream_tls_cert = "" + +# Path to the key file used to serve the encrypted stream. This file can +# change and CRI-O will automatically pick up the changes within 5 minutes. +stream_tls_key = "" + +# Path to the x509 CA(s) file used to verify and authenticate client +# communication with the encrypted stream. This file can change and CRI-O will +# automatically pick up the changes within 5 minutes. +stream_tls_ca = "" + +# Maximum grpc send message size in bytes. If not set or <=0, then CRI-O will default to 16 * 1024 * 1024. +grpc_max_send_msg_size = 16777216 + +# Maximum grpc receive message size. If not set or <= 0, then CRI-O will default to 16 * 1024 * 1024. +grpc_max_recv_msg_size = 16777216 + +# The crio.runtime table contains settings pertaining to the OCI runtime used +# and options for how to set up and manage the OCI runtime. +[crio.runtime] + +# A list of ulimits to be set in containers by default, specified as +# "=:", for example: +# "nofile=1024:2048" +# If nothing is set here, settings will be inherited from the CRI-O daemon +#default_ulimits = [ +#] + +# default_runtime is the _name_ of the OCI runtime to be used as the default. +# The name is matched against the runtimes map below. +default_runtime = "runc" + +# If true, the runtime will not use pivot_root, but instead use MS_MOVE. +no_pivot = false + +# decryption_keys_path is the path where the keys required for +# image decryption are stored. This option supports live configuration reload. +decryption_keys_path = "/etc/crio/keys/" + +# Path to the conmon binary, used for monitoring the OCI runtime. +# Will be searched for using $PATH if empty. +conmon = "{{ crio_conmon }}" + +# Cgroup setting for conmon +{% if crio_cgroup_manager == "cgroupfs" %} +conmon_cgroup = "pod" +{% else %} +{% if kube_reserved is defined and kube_reserved|bool %} +conmon_cgroup = "{{ kube_reserved_cgroups_for_service_slice }}" +{% else %} +conmon_cgroup = "system.slice" +{% endif %} +{% endif %} + +# Environment variable list for the conmon process, used for passing necessary +# environment variables to conmon or the runtime. +conmon_env = [ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", +] + +# Additional environment variables to set for all the +# containers. These are overridden if set in the +# container image spec or in the container runtime configuration. +default_env = [ +] + +# If true, SELinux will be used for pod separation on the host. +selinux = {{ crio_selinux }} + +# Path to the seccomp.json profile which is used as the default seccomp profile +# for the runtime. If not specified, then the internal default seccomp profile +# will be used. This option supports live configuration reload. +seccomp_profile = "{{ crio_seccomp_profile }}" + +# Used to change the name of the default AppArmor profile of CRI-O. The default +# profile name is "crio-default". This profile only takes effect if the user +# does not specify a profile via the Kubernetes Pod's metadata annotation. If +# the profile is set to "unconfined", then this equals to disabling AppArmor. +# This option supports live configuration reload. +# apparmor_profile = "crio-default" + +# Cgroup management implementation used for the runtime. +cgroup_manager = "{{ crio_cgroup_manager }}" + +# List of default capabilities for containers. If it is empty or commented out, +# only the capabilities defined in the containers json file by the user/kube +# will be added. +default_capabilities = [ + "CHOWN", + "DAC_OVERRIDE", + "FSETID", + "FOWNER", + "NET_RAW", + "SETGID", + "SETUID", + "SETPCAP", + "NET_BIND_SERVICE", + "SYS_CHROOT", + "KILL", +] + +# List of default sysctls. If it is empty or commented out, only the sysctls +# defined in the container json file by the user/kube will be added. +default_sysctls = [ +] + +# List of additional devices. specified as +# "::", for example: "--device=/dev/sdc:/dev/xvdc:rwm". +#If it is empty or commented out, only the devices +# defined in the container json file by the user/kube will be added. +additional_devices = [ +] + +# Path to OCI hooks directories for automatically executed hooks. If one of the +# directories does not exist, then CRI-O will automatically skip them. +hooks_dir = [ + "/usr/share/containers/oci/hooks.d", +] + +# List of default mounts for each container. **Deprecated:** this option will +# be removed in future versions in favor of default_mounts_file. +default_mounts = [ +] + +# Path to the file specifying the defaults mounts for each container. The +# format of the config is /SRC:/DST, one mount per line. Notice that CRI-O reads +# its default mounts from the following two files: +# +# 1) /etc/containers/mounts.conf (i.e., default_mounts_file): This is the +# override file, where users can either add in their own default mounts, or +# override the default mounts shipped with the package. +# +# 2) /usr/share/containers/mounts.conf: This is the default file read for +# mounts. If you want CRI-O to read from a different, specific mounts file, +# you can change the default_mounts_file. Note, if this is done, CRI-O will +# only add mounts it finds in this file. +# +#default_mounts_file = "" + +# Maximum sized allowed for the container log file. Negative numbers indicate +# that no size limit is imposed. If it is positive, it must be >= 8192 to +# match/exceed conmon's read buffer. The file is truncated and re-opened so the +# limit is never exceeded. +log_size_max = -1 + +# Whether container output should be logged to journald in addition to the kuberentes log file +log_to_journald = false + +# Path to directory in which container exit files are written to by conmon. +container_exits_dir = "/var/run/crio/exits" + +# Path to directory for container attach sockets. +container_attach_socket_dir = "/var/run/crio" + +# The prefix to use for the source of the bind mounts. +bind_mount_prefix = "" + +# If set to true, all containers will run in read-only mode. +read_only = false + +# Changes the verbosity of the logs based on the level it is set to. Options +# are fatal, panic, error, warn, info, debug and trace. This option supports +# live configuration reload. +log_level = "{{ crio_log_level }}" + +# Filter the log messages by the provided regular expression. +# This option supports live configuration reload. +log_filter = "" + +# The UID mappings for the user namespace of each container. A range is +# specified in the form containerUID:HostUID:Size. Multiple ranges must be +# separated by comma. +uid_mappings = "" + +# The GID mappings for the user namespace of each container. A range is +# specified in the form containerGID:HostGID:Size. Multiple ranges must be +# separated by comma. +gid_mappings = "" + +# The minimal amount of time in seconds to wait before issuing a timeout +# regarding the proper termination of the container. The lowest possible +# value is 30s, whereas lower values are not considered by CRI-O. +ctr_stop_timeout = 30 + +# **DEPRECATED** this option is being replaced by manage_ns_lifecycle, which is described below. +# manage_network_ns_lifecycle = false + +# manage_ns_lifecycle determines whether we pin and remove namespaces +# and manage their lifecycle +{% if kata_containers_enabled %} +manage_ns_lifecycle = true +{% else %} +manage_ns_lifecycle = false +{% endif %} + +# The directory where the state of the managed namespaces gets tracked. +# Only used when manage_ns_lifecycle is true. +namespaces_dir = "/var/run" + +# pinns_path is the path to find the pinns binary, which is needed to manage namespace lifecycle +{% if bin_dir == "/usr/local/bin" %} +pinns_path = "" +{% else %} +pinns_path = "{{ bin_dir }}/pinns" +{% endif %} + +{% if crio_criu_support_enabled %} +# Enable CRIU integration, requires that the criu binary is available in $PATH. +enable_criu_support = true +{% endif %} + +# The "crio.runtime.runtimes" table defines a list of OCI compatible runtimes. +# The runtime to use is picked based on the runtime_handler provided by the CRI. +# If no runtime_handler is provided, the runtime will be picked based on the level +# of trust of the workload. Each entry in the table should follow the format: +# +#[crio.runtime.runtimes.runtime-handler] +# runtime_path = "/path/to/the/executable" +# runtime_type = "oci" +# runtime_root = "/path/to/the/root" +# +# Where: +# - runtime-handler: name used to identify the runtime +# - runtime_path (optional, string): absolute path to the runtime executable in +# the host filesystem. If omitted, the runtime-handler identifier should match +# the runtime executable name, and the runtime executable should be placed +# in $PATH. +# - runtime_type (optional, string): type of runtime, one of: "oci", "vm". If +# omitted, an "oci" runtime is assumed. +# - runtime_root (optional, string): root directory for storage of containers +# state. + +{% for runtime in crio_runtimes %} +[crio.runtime.runtimes.{{ runtime.name }}] +runtime_path = "{{ runtime.path }}" +runtime_type = "{{ runtime.type }}" +runtime_root = "{{ runtime.root }}" +privileged_without_host_devices = {{ runtime.privileged_without_host_devices|default(false)|lower }} +allowed_annotations = {{ runtime.allowed_annotations|default([])|to_json }} +{% endfor %} + +# Kata Containers with the Firecracker VMM +#[crio.runtime.runtimes.kata-fc] + +# The crio.image table contains settings pertaining to the management of OCI images. +# +# CRI-O reads its configured registries defaults from the system wide +# containers-registries.conf(5) located in /etc/containers/registries.conf. If +# you want to modify just CRI-O, you can change the registries configuration in +# this file. Otherwise, leave insecure_registries and registries commented out to +# use the system's defaults from /etc/containers/registries.conf. +[crio.image] +{% if crio_insecure_registries is defined and crio_insecure_registries|length>0 %} +insecure_registries = {{ crio_insecure_registries }} +{% endif %} + +# Default transport for pulling images from a remote container storage. +default_transport = "docker://" + +# The path to a file containing credentials necessary for pulling images from +# secure registries. The file is similar to that of /var/lib/kubelet/config.json +global_auth_file = "/etc/crio/config.json" + +# The image used to instantiate infra containers. +# This option supports live configuration reload. +pause_image = "{{ crio_pause_image }}" + +# The path to a file containing credentials specific for pulling the pause_image from +# above. The file is similar to that of /var/lib/kubelet/config.json +# This option supports live configuration reload. +pause_image_auth_file = "" + +# The command to run to have a container stay in the paused state. +# When explicitly set to "", it will fallback to the entrypoint and command +# specified in the pause image. When commented out, it will fallback to the +# default: "/pause". This option supports live configuration reload. +pause_command = "/pause" + +# Path to the file which decides what sort of policy we use when deciding +# whether or not to trust an image that we've pulled. It is not recommended that +# this option be used, as the default behavior of using the system-wide default +# policy (i.e., /etc/containers/policy.json) is most often preferred. Please +# refer to containers-policy.json(5) for more details. +signature_policy = "{{ crio_signature_policy }}" + +# Controls how image volumes are handled. The valid values are mkdir, bind and +# ignore; the latter will ignore volumes entirely. +image_volumes = "mkdir" + +# The crio.network table containers settings pertaining to the management of +# CNI plugins. +[crio.network] + +# The default CNI network name to be selected. If not set or "", then +# CRI-O will pick-up the first one found in network_dir. +# cni_default_network = "" + +# Path to the directory where CNI configuration files are located. +network_dir = "/etc/cni/net.d/" + +# Paths to directories where CNI plugin binaries are located. +plugin_dirs = [ + "/opt/cni/bin", + "/usr/libexec/cni", +] + +# A necessary configuration for Prometheus based metrics retrieval +[crio.metrics] + +# Globally enable or disable metrics support. +enable_metrics = {{ crio_enable_metrics | bool | lower }} + +# The port on which the metrics server will listen. +metrics_port = {{ crio_metrics_port }} + +{% if nri_enabled and crio_version is version('v1.26.0', operator='>=') %} +[crio.nri] + +enable_nri=true +{% endif %} diff --git a/kubespray/project/roles/container-engine/cri-o/templates/http-proxy.conf.j2 b/kubespray/project/roles/container-engine/cri-o/templates/http-proxy.conf.j2 new file mode 100644 index 0000000..212f30f --- /dev/null +++ b/kubespray/project/roles/container-engine/cri-o/templates/http-proxy.conf.j2 @@ -0,0 +1,2 @@ +[Service] +Environment={% if http_proxy is defined %}"HTTP_PROXY={{ http_proxy }}"{% endif %} {% if https_proxy is defined %}"HTTPS_PROXY={{ https_proxy }}"{% endif %} {% if no_proxy is defined %}"NO_PROXY={{ no_proxy }}"{% endif %} diff --git a/kubespray/project/roles/container-engine/cri-o/templates/registry.conf.j2 b/kubespray/project/roles/container-engine/cri-o/templates/registry.conf.j2 new file mode 100644 index 0000000..38368f9 --- /dev/null +++ b/kubespray/project/roles/container-engine/cri-o/templates/registry.conf.j2 @@ -0,0 +1,13 @@ +[[registry]] +prefix = "{{ item.prefix | default(item.location) }}" +insecure = {{ item.insecure | default('false') | string | lower }} +blocked = {{ item.blocked | default('false') | string | lower }} +location = "{{ item.location }}" +{% if item.mirrors is defined %} +{% for mirror in item.mirrors %} + +[[registry.mirror]] +location = "{{ mirror.location }}" +insecure = {{ mirror.insecure | default('false') | string | lower }} +{% endfor %} +{% endif %} diff --git a/kubespray/project/roles/container-engine/cri-o/templates/unqualified.conf.j2 b/kubespray/project/roles/container-engine/cri-o/templates/unqualified.conf.j2 new file mode 100644 index 0000000..fc91f8b --- /dev/null +++ b/kubespray/project/roles/container-engine/cri-o/templates/unqualified.conf.j2 @@ -0,0 +1,10 @@ +{%- set _unqualified_registries = [] -%} +{% for _registry in crio_registries if _registry.unqualified -%} +{% if _registry.prefix is defined -%} +{{ _unqualified_registries.append(_registry.prefix) }} +{% else %} +{{ _unqualified_registries.append(_registry.location) }} +{%- endif %} +{%- endfor %} + +unqualified-search-registries = {{ _unqualified_registries | string }} diff --git a/kubespray/project/roles/container-engine/cri-o/vars/v1.28.yml b/kubespray/project/roles/container-engine/cri-o/vars/v1.28.yml new file mode 100644 index 0000000..a6a8b67 --- /dev/null +++ b/kubespray/project/roles/container-engine/cri-o/vars/v1.28.yml @@ -0,0 +1,9 @@ +--- +# cri-o binary files +crio_bin_files: + - conmon + - crio + - crio-status + - pinns + +crio_status_command: crio-status diff --git a/kubespray/project/roles/container-engine/cri-o/vars/v1.29.yml b/kubespray/project/roles/container-engine/cri-o/vars/v1.29.yml new file mode 100644 index 0000000..bc25e2e --- /dev/null +++ b/kubespray/project/roles/container-engine/cri-o/vars/v1.29.yml @@ -0,0 +1,9 @@ +--- +# cri-o binary files +crio_bin_files: + - conmon + - conmonrs + - crio + - pinns + +crio_status_command: crio status diff --git a/kubespray/project/roles/container-engine/crictl/handlers/main.yml b/kubespray/project/roles/container-engine/crictl/handlers/main.yml new file mode 100644 index 0000000..5319586 --- /dev/null +++ b/kubespray/project/roles/container-engine/crictl/handlers/main.yml @@ -0,0 +1,12 @@ +--- +- name: Get crictl completion + command: "{{ bin_dir }}/crictl completion" + changed_when: False + register: cri_completion + check_mode: false + +- name: Install crictl completion + copy: + dest: /etc/bash_completion.d/crictl + content: "{{ cri_completion.stdout }}" + mode: 0644 diff --git a/kubespray/project/roles/container-engine/crictl/tasks/crictl.yml b/kubespray/project/roles/container-engine/crictl/tasks/crictl.yml new file mode 100644 index 0000000..cffa050 --- /dev/null +++ b/kubespray/project/roles/container-engine/crictl/tasks/crictl.yml @@ -0,0 +1,22 @@ +--- +- name: Crictl | Download crictl + include_tasks: "../../../download/tasks/download_file.yml" + vars: + download: "{{ download_defaults | combine(downloads.crictl) }}" + +- name: Install crictl config + template: + src: crictl.yaml.j2 + dest: /etc/crictl.yaml + owner: root + mode: 0644 + +- name: Copy crictl binary from download dir + copy: + src: "{{ local_release_dir }}/crictl" + dest: "{{ bin_dir }}/crictl" + mode: 0755 + remote_src: true + notify: + - Get crictl completion + - Install crictl completion diff --git a/kubespray/project/roles/container-engine/crictl/tasks/main.yml b/kubespray/project/roles/container-engine/crictl/tasks/main.yml new file mode 100644 index 0000000..9337016 --- /dev/null +++ b/kubespray/project/roles/container-engine/crictl/tasks/main.yml @@ -0,0 +1,3 @@ +--- +- name: Install crictl + include_tasks: crictl.yml diff --git a/kubespray/project/roles/container-engine/crictl/templates/crictl.yaml.j2 b/kubespray/project/roles/container-engine/crictl/templates/crictl.yaml.j2 new file mode 100644 index 0000000..b97dbef --- /dev/null +++ b/kubespray/project/roles/container-engine/crictl/templates/crictl.yaml.j2 @@ -0,0 +1,4 @@ +runtime-endpoint: {{ cri_socket }} +image-endpoint: {{ cri_socket }} +timeout: 30 +debug: false diff --git a/kubespray/project/roles/container-engine/crun/tasks/main.yml b/kubespray/project/roles/container-engine/crun/tasks/main.yml new file mode 100644 index 0000000..c21bb3f --- /dev/null +++ b/kubespray/project/roles/container-engine/crun/tasks/main.yml @@ -0,0 +1,12 @@ +--- +- name: Crun | Download crun binary + include_tasks: "../../../download/tasks/download_file.yml" + vars: + download: "{{ download_defaults | combine(downloads.crun) }}" + +- name: Copy crun binary from download dir + copy: + src: "{{ downloads.crun.dest }}" + dest: "{{ bin_dir }}/crun" + mode: 0755 + remote_src: true diff --git a/kubespray/project/roles/container-engine/docker-storage/defaults/main.yml b/kubespray/project/roles/container-engine/docker-storage/defaults/main.yml new file mode 100644 index 0000000..6a69556 --- /dev/null +++ b/kubespray/project/roles/container-engine/docker-storage/defaults/main.yml @@ -0,0 +1,19 @@ +--- +docker_container_storage_setup_repository: https://github.com/projectatomic/container-storage-setup.git +docker_container_storage_setup_version: v0.6.0 +docker_container_storage_setup_profile_name: kubespray +docker_container_storage_setup_storage_driver: devicemapper +docker_container_storage_setup_container_thinpool: docker-pool +# It must be define a disk path for docker_container_storage_setup_devs. +# Otherwise docker-storage-setup will be executed incorrectly. +# docker_container_storage_setup_devs: /dev/vdb +docker_container_storage_setup_data_size: 40%FREE +docker_container_storage_setup_min_data_size: 2G +docker_container_storage_setup_chunk_size: 512K +docker_container_storage_setup_growpart: "false" +docker_container_storage_setup_auto_extend_pool: "yes" +docker_container_storage_setup_pool_autoextend_threshold: 60 +docker_container_storage_setup_pool_autoextend_percent: 20 +docker_container_storage_setup_device_wait_timeout: 60 +docker_container_storage_setup_wipe_signatures: "false" +docker_container_storage_setup_container_root_lv_size: 40%FREE diff --git a/kubespray/project/roles/container-engine/docker-storage/files/install_container_storage_setup.sh b/kubespray/project/roles/container-engine/docker-storage/files/install_container_storage_setup.sh new file mode 100644 index 0000000..604c843 --- /dev/null +++ b/kubespray/project/roles/container-engine/docker-storage/files/install_container_storage_setup.sh @@ -0,0 +1,23 @@ +#!/bin/sh + +set -e + +repository=${1:-https://github.com/projectatomic/container-storage-setup.git} +version=${2:-master} +profile_name=${3:-kubespray} +dir=`mktemp -d` +export GIT_DIR=$dir/.git +export GIT_WORK_TREE=$dir + +git init +git fetch --depth 1 $repository $version +git merge FETCH_HEAD +make -C $dir install +rm -rf /var/lib/container-storage-setup/$profile_name $dir + +set +e + +/usr/bin/container-storage-setup create $profile_name /etc/sysconfig/docker-storage-setup && /usr/bin/container-storage-setup activate $profile_name +# FIXME: exit status can be 1 for both fatal and non fatal errors in current release, +# could be improved by matching error strings +exit 0 diff --git a/kubespray/project/roles/container-engine/docker-storage/tasks/main.yml b/kubespray/project/roles/container-engine/docker-storage/tasks/main.yml new file mode 100644 index 0000000..ec12975 --- /dev/null +++ b/kubespray/project/roles/container-engine/docker-storage/tasks/main.yml @@ -0,0 +1,48 @@ +--- + +- name: Docker-storage-setup | install git and make + with_items: [git, make] + package: + pkg: "{{ item }}" + state: present + +- name: Docker-storage-setup | docker-storage-setup sysconfig template + template: + src: docker-storage-setup.j2 + dest: /etc/sysconfig/docker-storage-setup + mode: 0644 + +- name: Docker-storage-override-directory | docker service storage-setup override dir + file: + dest: /etc/systemd/system/docker.service.d + mode: 0755 + owner: root + group: root + state: directory + +- name: Docker-storage-override | docker service storage-setup override file + copy: + dest: /etc/systemd/system/docker.service.d/override.conf + content: |- + ### This file is managed by Ansible + [Service] + EnvironmentFile=-/etc/sysconfig/docker-storage + + owner: root + group: root + mode: 0644 + +# https://docs.docker.com/engine/installation/linux/docker-ce/centos/#install-using-the-repository +- name: Docker-storage-setup | install lvm2 + package: + name: lvm2 + state: present + +- name: Docker-storage-setup | install and run container-storage-setup + become: yes + script: | + install_container_storage_setup.sh \ + {{ docker_container_storage_setup_repository }} \ + {{ docker_container_storage_setup_version }} \ + {{ docker_container_storage_setup_profile_name }} + notify: Docker | reload systemd diff --git a/kubespray/project/roles/container-engine/docker-storage/templates/docker-storage-setup.j2 b/kubespray/project/roles/container-engine/docker-storage/templates/docker-storage-setup.j2 new file mode 100644 index 0000000..1a502b2 --- /dev/null +++ b/kubespray/project/roles/container-engine/docker-storage/templates/docker-storage-setup.j2 @@ -0,0 +1,35 @@ +{%if docker_container_storage_setup_storage_driver is defined%}STORAGE_DRIVER={{docker_container_storage_setup_storage_driver}}{%endif%} + +{%if docker_container_storage_setup_extra_storage_options is defined%}EXTRA_STORAGE_OPTIONS={{docker_container_storage_setup_extra_storage_options}}{%endif%} + +{%if docker_container_storage_setup_devs is defined%}DEVS={{docker_container_storage_setup_devs}}{%endif%} + +{%if docker_container_storage_setup_container_thinpool is defined%}CONTAINER_THINPOOL={{docker_container_storage_setup_container_thinpool}}{%endif%} + +{%if docker_container_storage_setup_vg is defined%}VG={{docker_container_storage_setup_vg}}{%endif%} + +{%if docker_container_storage_setup_root_size is defined%}ROOT_SIZE={{docker_container_storage_setup_root_size}}{%endif%} + +{%if docker_container_storage_setup_data_size is defined%}DATA_SIZE={{docker_container_storage_setup_data_size}}{%endif%} + +{%if docker_container_storage_setup_min_data_size is defined%}MIN_DATA_SIZE={{docker_container_storage_setup_min_data_size}}{%endif%} + +{%if docker_container_storage_setup_chunk_size is defined%}CHUNK_SIZE={{docker_container_storage_setup_chunk_size}}{%endif%} + +{%if docker_container_storage_setup_growpart is defined%}GROWPART={{docker_container_storage_setup_growpart}}{%endif%} + +{%if docker_container_storage_setup_auto_extend_pool is defined%}AUTO_EXTEND_POOL={{docker_container_storage_setup_auto_extend_pool}}{%endif%} + +{%if docker_container_storage_setup_pool_autoextend_threshold is defined%}POOL_AUTOEXTEND_THRESHOLD={{docker_container_storage_setup_pool_autoextend_threshold}}{%endif%} + +{%if docker_container_storage_setup_pool_autoextend_percent is defined%}POOL_AUTOEXTEND_PERCENT={{docker_container_storage_setup_pool_autoextend_percent}}{%endif%} + +{%if docker_container_storage_setup_device_wait_timeout is defined%}DEVICE_WAIT_TIMEOUT={{docker_container_storage_setup_device_wait_timeout}}{%endif%} + +{%if docker_container_storage_setup_wipe_signatures is defined%}WIPE_SIGNATURES={{docker_container_storage_setup_wipe_signatures}}{%endif%} + +{%if docker_container_storage_setup_container_root_lv_name is defined%}CONTAINER_ROOT_LV_NAME={{docker_container_storage_setup_container_root_lv_name}}{%endif%} + +{%if docker_container_storage_setup_container_root_lv_size is defined%}CONTAINER_ROOT_LV_SIZE={{docker_container_storage_setup_container_root_lv_size}}{%endif%} + +{%if docker_container_storage_setup_container_root_lv_mount_path is defined%}CONTAINER_ROOT_LV_MOUNT_PATH={{docker_container_storage_setup_container_root_lv_mount_path}}{%endif%} diff --git a/kubespray/project/roles/container-engine/docker/defaults/main.yml b/kubespray/project/roles/container-engine/docker/defaults/main.yml new file mode 100644 index 0000000..f756208 --- /dev/null +++ b/kubespray/project/roles/container-engine/docker/defaults/main.yml @@ -0,0 +1,67 @@ +--- +docker_version: '24.0' +docker_cli_version: "{{ docker_version }}" + +docker_package_info: + pkgs: + +# Path where to store repo key +# docker_repo_key_keyring: /etc/apt/trusted.gpg.d/docker.gpg + +docker_repo_key_info: + repo_keys: + +docker_repo_info: + repos: + +docker_cgroup_driver: systemd + +docker_bin_dir: "/usr/bin" + +# flag to enable/disable docker cleanup +docker_orphan_clean_up: false + +# old docker package names to be removed +docker_remove_packages_yum: + - docker + - docker-common + - docker-engine + - docker-selinux.noarch + - docker-client + - docker-client-latest + - docker-latest + - docker-latest-logrotate + - docker-logrotate + - docker-engine-selinux.noarch + +# remove podman to avoid containerd.io confliction +podman_remove_packages_yum: + - podman + +docker_remove_packages_apt: + - docker + - docker-engine + - docker.io + +# Docker specific repos should be part of the docker role not containerd-common anymore +# Optional values for containerd apt repo +containerd_package_info: + pkgs: + +# Fedora docker-ce repo +docker_fedora_repo_base_url: 'https://download.docker.com/linux/fedora/{{ ansible_distribution_major_version }}/$basearch/stable' +docker_fedora_repo_gpgkey: 'https://download.docker.com/linux/fedora/gpg' + +# CentOS/RedHat docker-ce repo +docker_rh_repo_base_url: 'https://download.docker.com/linux/centos/{{ ansible_distribution_major_version }}/$basearch/stable' +docker_rh_repo_gpgkey: 'https://download.docker.com/linux/centos/gpg' + +# Ubuntu docker-ce repo +docker_ubuntu_repo_base_url: "https://download.docker.com/linux/ubuntu" +docker_ubuntu_repo_gpgkey: 'https://download.docker.com/linux/ubuntu/gpg' +docker_ubuntu_repo_repokey: '9DC858229FC7DD38854AE2D88D81803C0EBFCD88' + +# Debian docker-ce repo +docker_debian_repo_base_url: "https://download.docker.com/linux/debian" +docker_debian_repo_gpgkey: 'https://download.docker.com/linux/debian/gpg' +docker_debian_repo_repokey: '9DC858229FC7DD38854AE2D88D81803C0EBFCD88' diff --git a/kubespray/project/roles/container-engine/docker/files/cleanup-docker-orphans.sh b/kubespray/project/roles/container-engine/docker/files/cleanup-docker-orphans.sh new file mode 100644 index 0000000..d7a9a8f --- /dev/null +++ b/kubespray/project/roles/container-engine/docker/files/cleanup-docker-orphans.sh @@ -0,0 +1,38 @@ +#!/bin/bash +list_descendants () +{ + local children=$(ps -o pid= --ppid "$1") + for pid in $children + do + list_descendants "$pid" + done + [[ -n "$children" ]] && echo "$children" +} + +shim_search="^docker-containerd-shim|^containerd-shim" +count_shim_processes=$(pgrep -f $shim_search | wc -l) + +if [ ${count_shim_processes} -gt 0 ]; then + # Find all container pids from shims + orphans=$(pgrep -P $(pgrep -d ',' -f $shim_search) |\ + # Filter out valid docker pids, leaving the orphans + egrep -v $(docker ps -q | xargs docker inspect --format '{{.State.Pid}}' | awk '{printf "%s%s",sep,$1; sep="|"}')) + + if [[ -n "$orphans" && -n "$(ps -o ppid= $orphans)" ]] + then + # Get shim pids of orphans + orphan_shim_pids=$(ps -o pid= $(ps -o ppid= $orphans)) + + # Find all orphaned container PIDs + orphan_container_pids=$(for pid in $orphan_shim_pids; do list_descendants $pid; done) + + # Recursively kill all child PIDs of orphan shims + echo -e "Killing orphan container PIDs and descendants: \n$(ps -O ppid= $orphan_container_pids)" + kill -9 $orphan_container_pids || true + + else + echo "No orphaned containers found" + fi +else + echo "The node doesn't have any shim processes." +fi diff --git a/kubespray/project/roles/container-engine/docker/handlers/main.yml b/kubespray/project/roles/container-engine/docker/handlers/main.yml new file mode 100644 index 0000000..4a8a639 --- /dev/null +++ b/kubespray/project/roles/container-engine/docker/handlers/main.yml @@ -0,0 +1,30 @@ +--- +- name: Docker | reload systemd + systemd: + name: docker + daemon_reload: true + masked: no + listen: Restart docker + +- name: Docker | reload docker.socket + service: + name: docker.socket + state: restarted + when: ansible_os_family in ['Flatcar', 'Flatcar Container Linux by Kinvolk'] or is_fedora_coreos + listen: Restart docker + + +- name: Docker | reload docker + service: + name: docker + state: restarted + listen: Restart docker + + +- name: Docker | wait for docker + command: "{{ docker_bin_dir }}/docker images" + register: docker_ready + retries: 20 + delay: 1 + until: docker_ready.rc == 0 + listen: Restart docker diff --git a/kubespray/project/roles/container-engine/docker/meta/main.yml b/kubespray/project/roles/container-engine/docker/meta/main.yml new file mode 100644 index 0000000..d7e4751 --- /dev/null +++ b/kubespray/project/roles/container-engine/docker/meta/main.yml @@ -0,0 +1,5 @@ +--- +dependencies: + - role: container-engine/containerd-common + - role: container-engine/docker-storage + when: docker_container_storage_setup and ansible_os_family == "RedHat" diff --git a/kubespray/project/roles/container-engine/docker/tasks/docker_plugin.yml b/kubespray/project/roles/container-engine/docker/tasks/docker_plugin.yml new file mode 100644 index 0000000..8ee530e --- /dev/null +++ b/kubespray/project/roles/container-engine/docker/tasks/docker_plugin.yml @@ -0,0 +1,8 @@ +--- +- name: Install Docker plugin + command: docker plugin install --grant-all-permissions {{ docker_plugin | quote }} + when: docker_plugin is defined + register: docker_plugin_status + failed_when: + - docker_plugin_status.failed + - '"already exists" not in docker_plugin_status.stderr' diff --git a/kubespray/project/roles/container-engine/docker/tasks/main.yml b/kubespray/project/roles/container-engine/docker/tasks/main.yml new file mode 100644 index 0000000..1fc490b --- /dev/null +++ b/kubespray/project/roles/container-engine/docker/tasks/main.yml @@ -0,0 +1,181 @@ +--- +- name: Check if fedora coreos + stat: + path: /run/ostree-booted + get_attributes: no + get_checksum: no + get_mime: no + register: ostree + +- name: Set is_ostree + set_fact: + is_ostree: "{{ ostree.stat.exists }}" + +- name: Set docker_version for openEuler + set_fact: + docker_version: '19.03' + when: ansible_distribution == "openEuler" + tags: + - facts + +- name: Gather os specific variables + include_vars: "{{ item }}" + with_first_found: + - files: + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower | replace('/', '_') }}.yml" + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_release | lower }}-{{ host_architecture }}.yml" + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_release | lower }}.yml" + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower | replace('/', '_') }}.yml" + - "{{ ansible_distribution | lower }}-{{ host_architecture }}.yml" + - "{{ ansible_distribution | lower }}.yml" + - "{{ ansible_distribution.split(' ')[0] | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower | replace('/', '_') }}.yml" + - "{{ ansible_os_family | lower }}-{{ host_architecture }}.yml" + - "{{ ansible_os_family | lower }}.yml" + - defaults.yml + paths: + - ../vars + skip: true + tags: + - facts + +- name: Warn about Docker version on SUSE + debug: + msg: "SUSE distributions always install Docker from the distro repos" + when: ansible_pkg_mgr == 'zypper' + +- name: Gather DNS facts + include_tasks: set_facts_dns.yml + when: dns_mode != 'none' and resolvconf_mode == 'docker_dns' + tags: + - facts + +- name: Pre-upgrade docker + import_tasks: pre-upgrade.yml + +- name: Ensure docker-ce repository public key is installed + apt_key: + id: "{{ item }}" + url: "{{ docker_repo_key_info.url }}" + keyring: "{{ docker_repo_key_keyring|default(omit) }}" + state: present + register: keyserver_task_result + until: keyserver_task_result is succeeded + retries: 4 + delay: "{{ retry_stagger | d(3) }}" + with_items: "{{ docker_repo_key_info.repo_keys }}" + environment: "{{ proxy_env }}" + when: ansible_pkg_mgr == 'apt' + +- name: Ensure docker-ce repository is enabled + apt_repository: + repo: "{{ item }}" + state: present + with_items: "{{ docker_repo_info.repos }}" + when: ansible_pkg_mgr == 'apt' + +- name: Configure docker repository on Fedora + template: + src: "fedora_docker.repo.j2" + dest: "{{ yum_repo_dir }}/docker.repo" + mode: 0644 + when: ansible_distribution == "Fedora" and not is_ostree + +- name: Configure docker repository on RedHat/CentOS/OracleLinux/AlmaLinux/KylinLinux + template: + src: "rh_docker.repo.j2" + dest: "{{ yum_repo_dir }}/docker-ce.repo" + mode: 0644 + when: + - ansible_os_family == "RedHat" + - ansible_distribution != "Fedora" + - not is_ostree + +- name: Remove dpkg hold + dpkg_selections: + name: "{{ item }}" + selection: install + when: ansible_pkg_mgr == 'apt' + changed_when: false + with_items: + - "{{ containerd_package }}" + - docker-ce + - docker-ce-cli + +- name: Ensure docker packages are installed + package: + name: "{{ docker_package_info.pkgs }}" + state: "{{ docker_package_info.state | default('present') }}" + module_defaults: + apt: + update_cache: true + dnf: + enablerepo: "{{ docker_package_info.enablerepo | default(omit) }}" + disablerepo: "{{ docker_package_info.disablerepo | default(omit) }}" + yum: + enablerepo: "{{ docker_package_info.enablerepo | default(omit) }}" + zypper: + update_cache: true + register: docker_task_result + until: docker_task_result is succeeded + retries: 4 + delay: "{{ retry_stagger | d(3) }}" + notify: Restart docker + when: + - not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] + - not is_ostree + - docker_package_info.pkgs | length > 0 + +# This is required to ensure any apt upgrade will not break kubernetes +- name: Tell Debian hosts not to change the docker version with apt upgrade + dpkg_selections: + name: "{{ item }}" + selection: hold + when: ansible_pkg_mgr == 'apt' + changed_when: false + with_items: + - "{{ containerd_package }}" + - docker-ce + - docker-ce-cli + +- name: Ensure docker started, remove our config if docker start failed and try again + block: + - name: Ensure service is started if docker packages are already present + service: + name: docker + state: started + when: docker_task_result is not changed + rescue: + - debug: # noqa name[missing] + msg: "Docker start failed. Try to remove our config" + - name: Remove kubespray generated config + file: + path: "{{ item }}" + state: absent + with_items: + - /etc/systemd/system/docker.service.d/http-proxy.conf + - /etc/systemd/system/docker.service.d/docker-options.conf + - /etc/systemd/system/docker.service.d/docker-dns.conf + - /etc/systemd/system/docker.service.d/docker-orphan-cleanup.conf + notify: Restart docker + +- name: Flush handlers so we can wait for docker to come up + meta: flush_handlers + +# Install each plugin using a looped include to make error handling in the included task simpler. +- name: Install docker plugin + include_tasks: docker_plugin.yml + loop: "{{ docker_plugins }}" + loop_control: + loop_var: docker_plugin + +- name: Set docker systemd config + import_tasks: systemd.yml + +- name: Ensure docker service is started and enabled + service: + name: "{{ item }}" + enabled: yes + state: started + with_items: + - docker diff --git a/kubespray/project/roles/container-engine/docker/tasks/pre-upgrade.yml b/kubespray/project/roles/container-engine/docker/tasks/pre-upgrade.yml new file mode 100644 index 0000000..f346b46 --- /dev/null +++ b/kubespray/project/roles/container-engine/docker/tasks/pre-upgrade.yml @@ -0,0 +1,36 @@ +--- +- name: Remove legacy docker repo file + file: + path: "{{ yum_repo_dir }}/docker.repo" + state: absent + when: + - ansible_os_family == 'RedHat' + - not is_ostree + +- name: Ensure old versions of Docker are not installed. | Debian + apt: + name: '{{ docker_remove_packages_apt }}' + state: absent + when: + - ansible_os_family == 'Debian' + - (docker_versioned_pkg[docker_version | string] is search('docker-ce')) + + +- name: Ensure podman not installed. | RedHat + package: + name: '{{ podman_remove_packages_yum }}' + state: absent + when: + - ansible_os_family == 'RedHat' + - (docker_versioned_pkg[docker_version | string] is search('docker-ce')) + - not is_ostree + + +- name: Ensure old versions of Docker are not installed. | RedHat + package: + name: '{{ docker_remove_packages_yum }}' + state: absent + when: + - ansible_os_family == 'RedHat' + - (docker_versioned_pkg[docker_version | string] is search('docker-ce')) + - not is_ostree diff --git a/kubespray/project/roles/container-engine/docker/tasks/reset.yml b/kubespray/project/roles/container-engine/docker/tasks/reset.yml new file mode 100644 index 0000000..4bca908 --- /dev/null +++ b/kubespray/project/roles/container-engine/docker/tasks/reset.yml @@ -0,0 +1,106 @@ +--- + +- name: Docker | Get package facts + package_facts: + manager: auto + +- name: Docker | Find docker packages + set_fact: + docker_packages_list: "{{ ansible_facts.packages.keys() | select('search', '^docker*') }}" + containerd_package: "{{ ansible_facts.packages.keys() | select('search', '^containerd*') }}" + +- name: Docker | Stop all running container + shell: "set -o pipefail && {{ docker_bin_dir }}/docker ps -q | xargs -r {{ docker_bin_dir }}/docker kill" + args: + executable: /bin/bash + register: stop_all_containers + retries: 5 + until: stop_all_containers.rc == 0 + changed_when: true + delay: 5 + ignore_errors: true # noqa ignore-errors + when: docker_packages_list | length>0 + +- name: Reset | remove all containers + shell: "set -o pipefail && {{ docker_bin_dir }}/docker ps -aq | xargs -r docker rm -fv" + args: + executable: /bin/bash + register: remove_all_containers + retries: 4 + until: remove_all_containers.rc == 0 + delay: 5 + when: docker_packages_list | length>0 + +- name: Docker | Stop docker service + service: + name: "{{ item }}" + enabled: false + state: stopped + loop: + - docker + - docker.socket + - containerd + when: docker_packages_list | length>0 + +- name: Docker | Remove dpkg hold + dpkg_selections: + name: "{{ item }}" + selection: install + when: ansible_pkg_mgr == 'apt' + changed_when: false + with_items: + - "{{ docker_packages_list }}" + - "{{ containerd_package }}" + +- name: Docker | Remove docker package + package: + name: "{{ item }}" + state: absent + changed_when: false + with_items: + - "{{ docker_packages_list }}" + - "{{ containerd_package }}" + when: + - not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] + - not is_ostree + - docker_packages_list | length > 0 + +- name: Docker | ensure docker-ce repository is removed + apt_repository: + repo: "{{ item }}" + state: absent + with_items: "{{ docker_repo_info.repos }}" + when: ansible_pkg_mgr == 'apt' + +- name: Docker | Remove docker repository on Fedora + file: + name: "{{ yum_repo_dir }}/docker.repo" + state: absent + when: ansible_distribution == "Fedora" and not is_ostree + +- name: Docker | Remove docker repository on RedHat/CentOS/Oracle/AlmaLinux Linux + file: + name: "{{ yum_repo_dir }}/docker-ce.repo" + state: absent + when: + - ansible_os_family == "RedHat" + - ansible_distribution != "Fedora" + - not is_ostree + +- name: Docker | Remove docker configuration files + file: + name: "{{ item }}" + state: absent + loop: + - /etc/systemd/system/docker.service.d/ + - /etc/systemd/system/docker.socket + - /etc/systemd/system/docker.service + - /etc/systemd/system/containerd.service + - /etc/systemd/system/containerd.service.d + - /var/lib/docker + - /etc/docker + ignore_errors: true # noqa ignore-errors + +- name: Docker | systemctl daemon-reload # noqa no-handler + systemd: + daemon_reload: true diff --git a/kubespray/project/roles/container-engine/docker/tasks/set_facts_dns.yml b/kubespray/project/roles/container-engine/docker/tasks/set_facts_dns.yml new file mode 100644 index 0000000..d7c1039 --- /dev/null +++ b/kubespray/project/roles/container-engine/docker/tasks/set_facts_dns.yml @@ -0,0 +1,66 @@ +--- + +- name: Set dns server for docker + set_fact: + docker_dns_servers: "{{ dns_servers }}" + +- name: Show docker_dns_servers + debug: + msg: "{{ docker_dns_servers }}" + +- name: Add upstream dns servers + set_fact: + docker_dns_servers: "{{ docker_dns_servers + upstream_dns_servers | default([]) }}" + when: dns_mode in ['coredns', 'coredns_dual'] + +- name: Add global searchdomains + set_fact: + docker_dns_search_domains: "{{ docker_dns_search_domains + searchdomains | default([]) }}" + +- name: Check system nameservers + shell: set -o pipefail && grep "^nameserver" /etc/resolv.conf | sed -r 's/^nameserver\s*([^#\s]+)\s*(#.*)?/\1/' + args: + executable: /bin/bash + changed_when: False + register: system_nameservers + check_mode: no + +- name: Check system search domains + # noqa risky-shell-pipe - if resolf.conf has no search domain, grep will exit 1 which would force us to add failed_when: false + # Therefore -o pipefail is not applicable in this specific instance + shell: grep "^search" /etc/resolv.conf | sed -r 's/^search\s*([^#]+)\s*(#.*)?/\1/' + args: + executable: /bin/bash + changed_when: False + register: system_search_domains + check_mode: no + +- name: Add system nameservers to docker options + set_fact: + docker_dns_servers: "{{ docker_dns_servers | union(system_nameservers.stdout_lines) | unique }}" + when: system_nameservers.stdout + +- name: Add system search domains to docker options + set_fact: + docker_dns_search_domains: "{{ docker_dns_search_domains | union(system_search_domains.stdout.split() | default([])) | unique }}" + when: system_search_domains.stdout + +- name: Check number of nameservers + fail: + msg: "Too many nameservers. You can relax this check by set docker_dns_servers_strict=false in docker.yml and we will only use the first 3." + when: docker_dns_servers | length > 3 and docker_dns_servers_strict | bool + +- name: Rtrim number of nameservers to 3 + set_fact: + docker_dns_servers: "{{ docker_dns_servers[0:3] }}" + when: docker_dns_servers | length > 3 and not docker_dns_servers_strict | bool + +- name: Check number of search domains + fail: + msg: "Too many search domains" + when: docker_dns_search_domains | length > 6 + +- name: Check length of search domains + fail: + msg: "Search domains exceeded limit of 256 characters" + when: docker_dns_search_domains | join(' ') | length > 256 diff --git a/kubespray/project/roles/container-engine/docker/tasks/systemd.yml b/kubespray/project/roles/container-engine/docker/tasks/systemd.yml new file mode 100644 index 0000000..57d9b9c --- /dev/null +++ b/kubespray/project/roles/container-engine/docker/tasks/systemd.yml @@ -0,0 +1,68 @@ +--- +- name: Create docker service systemd directory if it doesn't exist + file: + path: /etc/systemd/system/docker.service.d + state: directory + mode: 0755 + +- name: Write docker proxy drop-in + template: + src: http-proxy.conf.j2 + dest: /etc/systemd/system/docker.service.d/http-proxy.conf + mode: 0644 + notify: Restart docker + when: http_proxy is defined or https_proxy is defined + +- name: Get systemd version + # noqa command-instead-of-module - systemctl is called intentionally here + shell: set -o pipefail && systemctl --version | head -n 1 | cut -d " " -f 2 + args: + executable: /bin/bash + register: systemd_version + when: not is_ostree + changed_when: false + check_mode: false + +- name: Write docker.service systemd file + template: + src: docker.service.j2 + dest: /etc/systemd/system/docker.service + mode: 0644 + register: docker_service_file + notify: Restart docker + when: + - not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] + - not is_fedora_coreos + +- name: Write docker options systemd drop-in + template: + src: docker-options.conf.j2 + dest: "/etc/systemd/system/docker.service.d/docker-options.conf" + mode: 0644 + notify: Restart docker + +- name: Write docker dns systemd drop-in + template: + src: docker-dns.conf.j2 + dest: "/etc/systemd/system/docker.service.d/docker-dns.conf" + mode: 0644 + notify: Restart docker + when: dns_mode != 'none' and resolvconf_mode == 'docker_dns' + +- name: Copy docker orphan clean up script to the node + copy: + src: cleanup-docker-orphans.sh + dest: "{{ bin_dir }}/cleanup-docker-orphans.sh" + mode: 0755 + when: docker_orphan_clean_up | bool + +- name: Write docker orphan clean up systemd drop-in + template: + src: docker-orphan-cleanup.conf.j2 + dest: "/etc/systemd/system/docker.service.d/docker-orphan-cleanup.conf" + mode: 0644 + notify: Restart docker + when: docker_orphan_clean_up | bool + +- name: Flush handlers + meta: flush_handlers diff --git a/kubespray/project/roles/container-engine/docker/templates/docker-dns.conf.j2 b/kubespray/project/roles/container-engine/docker/templates/docker-dns.conf.j2 new file mode 100644 index 0000000..01dbd3b --- /dev/null +++ b/kubespray/project/roles/container-engine/docker/templates/docker-dns.conf.j2 @@ -0,0 +1,6 @@ +[Service] +Environment="DOCKER_DNS_OPTIONS=\ + {% for d in docker_dns_servers %}--dns {{ d }} {% endfor %} \ + {% for d in docker_dns_search_domains %}--dns-search {{ d }} {% endfor %} \ + {% for o in docker_dns_options %}--dns-opt {{ o }} {% endfor %} \ +" diff --git a/kubespray/project/roles/container-engine/docker/templates/docker-options.conf.j2 b/kubespray/project/roles/container-engine/docker/templates/docker-options.conf.j2 new file mode 100644 index 0000000..ae661ad --- /dev/null +++ b/kubespray/project/roles/container-engine/docker/templates/docker-options.conf.j2 @@ -0,0 +1,11 @@ +[Service] +Environment="DOCKER_OPTS={{ docker_options|default('') }} --iptables={{ docker_iptables_enabled | default('false') }} \ +--exec-opt native.cgroupdriver={{ docker_cgroup_driver }} \ +{% for i in docker_insecure_registries %}--insecure-registry={{ i }} {% endfor %} \ +{% for i in docker_registry_mirrors %}--registry-mirror={{ i }} {% endfor %} \ +--data-root={{ docker_daemon_graph }} \ +{% if ansible_os_family not in ["openSUSE Leap", "openSUSE Tumbleweed", "Suse"] %}{{ docker_log_opts }}{% endif %}" + +{% if docker_mount_flags is defined and docker_mount_flags != "" %} +MountFlags={{ docker_mount_flags }} +{% endif %} diff --git a/kubespray/project/roles/container-engine/docker/templates/docker-orphan-cleanup.conf.j2 b/kubespray/project/roles/container-engine/docker/templates/docker-orphan-cleanup.conf.j2 new file mode 100644 index 0000000..370f1e7 --- /dev/null +++ b/kubespray/project/roles/container-engine/docker/templates/docker-orphan-cleanup.conf.j2 @@ -0,0 +1,2 @@ +[Service] +ExecStartPost=-{{ bin_dir }}/cleanup-docker-orphans.sh diff --git a/kubespray/project/roles/container-engine/docker/templates/docker.service.j2 b/kubespray/project/roles/container-engine/docker/templates/docker.service.j2 new file mode 100644 index 0000000..539c3a5 --- /dev/null +++ b/kubespray/project/roles/container-engine/docker/templates/docker.service.j2 @@ -0,0 +1,51 @@ +[Unit] +Description=Docker Application Container Engine +Documentation=http://docs.docker.com +{% if ansible_os_family == "RedHat" %} +After=network.target {{ ' docker-storage-setup.service' if docker_container_storage_setup else '' }} containerd.service +BindsTo=containerd.service +{{ 'Wants=docker-storage-setup.service' if docker_container_storage_setup else '' }} +{% elif ansible_os_family == "Debian" %} +After=network.target docker.socket containerd.service +BindsTo=containerd.service +Wants=docker.socket +{% elif ansible_os_family == "Suse" %} +After=network.target lvm2-monitor.service SuSEfirewall2.service +# After=network.target containerd.service +# BindsTo=containerd.service +{% endif %} + +[Service] +Type=notify +{% if docker_storage_options is defined %} +Environment="DOCKER_STORAGE_OPTIONS={{ docker_storage_options }}" +{% endif %} +Environment=GOTRACEBACK=crash +ExecReload=/bin/kill -s HUP $MAINPID +Delegate=yes +KillMode=process +ExecStart={{ docker_bin_dir }}/dockerd \ +{% if ansible_os_family == "Suse" %} + --add-runtime oci=/usr/sbin/docker-runc \ +{% endif %} + $DOCKER_OPTS \ + $DOCKER_STORAGE_OPTIONS \ + $DOCKER_DNS_OPTIONS +{% if not is_ostree and systemd_version.stdout|int >= 226 %} +TasksMax=infinity +{% endif %} +LimitNOFILE=1048576 +LimitNPROC=1048576 +LimitCORE=infinity +TimeoutStartSec=1min +# restart the docker process if it exits prematurely +Restart=on-failure +StartLimitBurst=3 +StartLimitInterval=60s +# Set the cgroup slice of the service so that kube reserved takes effect +{% if kube_reserved is defined and kube_reserved|bool %} +Slice={{ kube_reserved_cgroups_for_service_slice }} +{% endif %} + +[Install] +WantedBy=multi-user.target diff --git a/kubespray/project/roles/container-engine/docker/templates/fedora_docker.repo.j2 b/kubespray/project/roles/container-engine/docker/templates/fedora_docker.repo.j2 new file mode 100644 index 0000000..3958ff0 --- /dev/null +++ b/kubespray/project/roles/container-engine/docker/templates/fedora_docker.repo.j2 @@ -0,0 +1,7 @@ +[docker-ce] +name=Docker-CE Repository +baseurl={{ docker_fedora_repo_base_url }} +enabled=1 +gpgcheck={{ '1' if docker_fedora_repo_gpgkey else '0' }} +gpgkey={{ docker_fedora_repo_gpgkey }} +{% if http_proxy is defined %}proxy={{ http_proxy }}{% endif %} diff --git a/kubespray/project/roles/container-engine/docker/templates/http-proxy.conf.j2 b/kubespray/project/roles/container-engine/docker/templates/http-proxy.conf.j2 new file mode 100644 index 0000000..212f30f --- /dev/null +++ b/kubespray/project/roles/container-engine/docker/templates/http-proxy.conf.j2 @@ -0,0 +1,2 @@ +[Service] +Environment={% if http_proxy is defined %}"HTTP_PROXY={{ http_proxy }}"{% endif %} {% if https_proxy is defined %}"HTTPS_PROXY={{ https_proxy }}"{% endif %} {% if no_proxy is defined %}"NO_PROXY={{ no_proxy }}"{% endif %} diff --git a/kubespray/project/roles/container-engine/docker/templates/rh_docker.repo.j2 b/kubespray/project/roles/container-engine/docker/templates/rh_docker.repo.j2 new file mode 100644 index 0000000..178bbc2 --- /dev/null +++ b/kubespray/project/roles/container-engine/docker/templates/rh_docker.repo.j2 @@ -0,0 +1,10 @@ +[docker-ce] +name=Docker-CE Repository +baseurl={{ docker_rh_repo_base_url }} +enabled=0 +gpgcheck={{ '1' if docker_rh_repo_gpgkey else '0' }} +keepcache={{ docker_rpm_keepcache | default('1') }} +gpgkey={{ docker_rh_repo_gpgkey }} +{% if http_proxy is defined %} +proxy={{ http_proxy }} +{% endif %} diff --git a/kubespray/project/roles/container-engine/docker/vars/amazon.yml b/kubespray/project/roles/container-engine/docker/vars/amazon.yml new file mode 100644 index 0000000..4871f4a --- /dev/null +++ b/kubespray/project/roles/container-engine/docker/vars/amazon.yml @@ -0,0 +1,15 @@ +--- +# https://docs.aws.amazon.com/en_us/AmazonECS/latest/developerguide/docker-basics.html + +docker_versioned_pkg: + 'latest': docker + '18.09': docker-18.09.9ce-2.amzn2 + '19.03': docker-19.03.13ce-1.amzn2 + '20.10': docker-20.10.7-5.amzn2 + +docker_version: "latest" + +docker_package_info: + pkgs: + - "{{ docker_versioned_pkg[docker_version | string] }}" + enablerepo: amzn2extra-docker diff --git a/kubespray/project/roles/container-engine/docker/vars/clearlinux.yml b/kubespray/project/roles/container-engine/docker/vars/clearlinux.yml new file mode 100644 index 0000000..fbb7a22 --- /dev/null +++ b/kubespray/project/roles/container-engine/docker/vars/clearlinux.yml @@ -0,0 +1,4 @@ +--- +docker_package_info: + pkgs: + - "containers-basic" diff --git a/kubespray/project/roles/container-engine/docker/vars/debian.yml b/kubespray/project/roles/container-engine/docker/vars/debian.yml new file mode 100644 index 0000000..02289fa --- /dev/null +++ b/kubespray/project/roles/container-engine/docker/vars/debian.yml @@ -0,0 +1,66 @@ +--- +# containerd package info is only relevant for docker +containerd_versioned_pkg: + 'latest': "{{ containerd_package }}" + '1.3.7': "{{ containerd_package }}=1.3.7-1" + '1.3.9': "{{ containerd_package }}=1.3.9-1" + '1.4.3': "{{ containerd_package }}=1.4.3-2" + '1.4.4': "{{ containerd_package }}=1.4.4-1" + '1.4.6': "{{ containerd_package }}=1.4.6-1" + '1.4.9': "{{ containerd_package }}=1.4.9-1" + '1.4.12': "{{ containerd_package }}=1.4.12-1" + '1.6.4': "{{ containerd_package }}=1.6.4-1" + '1.6.6': "{{ containerd_package }}=1.6.6-1" + '1.6.7': "{{ containerd_package }}=1.6.7-1" + '1.6.8': "{{ containerd_package }}=1.6.8-1" + '1.6.9': "{{ containerd_package }}=1.6.9-1" + '1.6.10': "{{ containerd_package }}=1.6.10-1" + '1.6.11': "{{ containerd_package }}=1.6.11-1" + '1.6.12': "{{ containerd_package }}=1.6.12-1" + '1.6.13': "{{ containerd_package }}=1.6.13-1" + '1.6.14': "{{ containerd_package }}=1.6.14-1" + '1.6.15': "{{ containerd_package }}=1.6.15-1" + '1.6.16': "{{ containerd_package }}=1.6.16-1" + '1.6.18': "{{ containerd_package }}=1.6.18-1" + '1.6.28': "{{ containerd_package }}=1.6.28-1" + 'stable': "{{ containerd_package }}=1.6.28-1" + 'edge': "{{ containerd_package }}=1.6.28-1" + +# https://download.docker.com/linux/debian/ +docker_versioned_pkg: + 'latest': docker-ce + '18.09': docker-ce=5:18.09.9~3-0~debian-{{ ansible_distribution_release | lower }} + '19.03': docker-ce=5:19.03.15~3-0~debian-{{ ansible_distribution_release | lower }} + '20.10': docker-ce=5:20.10.20~3-0~debian-{{ ansible_distribution_release | lower }} + '23.0': docker-ce=5:23.0.6-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} + '24.0': docker-ce=5:24.0.9-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} + 'stable': docker-ce=5:24.0.9-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} + 'edge': docker-ce=5:24.0.9-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} + +docker_cli_versioned_pkg: + 'latest': docker-ce-cli + '18.09': docker-ce-cli=5:18.09.9~3-0~debian-{{ ansible_distribution_release | lower }} + '19.03': docker-ce-cli=5:19.03.15~3-0~debian-{{ ansible_distribution_release | lower }} + '20.10': docker-ce-cli=5:20.10.20~3-0~debian-{{ ansible_distribution_release | lower }} + '23.0': docker-ce-cli=5:23.0.6-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} + '24.0': docker-ce-cli=5:24.0.9-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} + 'stable': docker-ce-cli=5:24.0.9-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} + 'edge': docker-ce-cli=5:24.0.9-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} + +docker_package_info: + pkgs: + - "{{ containerd_versioned_pkg[docker_containerd_version | string] }}" + - "{{ docker_cli_versioned_pkg[docker_cli_version | string] }}" + - "{{ docker_versioned_pkg[docker_version | string] }}" + +docker_repo_key_info: + url: '{{ docker_debian_repo_gpgkey }}' + repo_keys: + - '{{ docker_debian_repo_repokey }}' + +docker_repo_info: + repos: + - > + deb {{ docker_debian_repo_base_url }} + {{ ansible_distribution_release | lower }} + stable diff --git a/kubespray/project/roles/container-engine/docker/vars/fedora.yml b/kubespray/project/roles/container-engine/docker/vars/fedora.yml new file mode 100644 index 0000000..39dfbaf --- /dev/null +++ b/kubespray/project/roles/container-engine/docker/vars/fedora.yml @@ -0,0 +1,54 @@ +--- +# containerd versions are only relevant for docker +containerd_versioned_pkg: + 'latest': "{{ containerd_package }}" + '1.3.7': "{{ containerd_package }}-1.3.7-3.1.fc{{ ansible_distribution_major_version }}" + '1.3.9': "{{ containerd_package }}-1.3.9-3.1.fc{{ ansible_distribution_major_version }}" + '1.4.3': "{{ containerd_package }}-1.4.3-3.2.fc{{ ansible_distribution_major_version }}" + '1.4.4': "{{ containerd_package }}-1.4.4-3.1.fc{{ ansible_distribution_major_version }}" + '1.4.6': "{{ containerd_package }}-1.4.6-3.1.fc{{ ansible_distribution_major_version }}" + '1.4.9': "{{ containerd_package }}-1.4.9-3.1.fc{{ ansible_distribution_major_version }}" + '1.4.12': "{{ containerd_package }}-1.4.12-3.1.fc{{ ansible_distribution_major_version }}" + '1.6.4': "{{ containerd_package }}-1.6.4-3.1.fc{{ ansible_distribution_major_version }}" + '1.6.6': "{{ containerd_package }}-1.6.6-3.1.fc{{ ansible_distribution_major_version }}" + '1.6.7': "{{ containerd_package }}-1.6.7-3.1.fc{{ ansible_distribution_major_version }}" + '1.6.8': "{{ containerd_package }}-1.6.8-3.1.fc{{ ansible_distribution_major_version }}" + '1.6.9': "{{ containerd_package }}-1.6.9-3.1.fc{{ ansible_distribution_major_version }}" + '1.6.10': "{{ containerd_package }}-1.6.10-3.1.fc{{ ansible_distribution_major_version }}" + '1.6.11': "{{ containerd_package }}-1.6.11-3.1.fc{{ ansible_distribution_major_version }}" + '1.6.12': "{{ containerd_package }}-1.6.12-3.1.fc{{ ansible_distribution_major_version }}" + '1.6.13': "{{ containerd_package }}-1.6.13-3.1.fc{{ ansible_distribution_major_version }}" + '1.6.14': "{{ containerd_package }}-1.6.14-3.1.fc{{ ansible_distribution_major_version }}" + '1.6.15': "{{ containerd_package }}-1.6.15-3.1.fc{{ ansible_distribution_major_version }}" + '1.6.16': "{{ containerd_package }}-1.6.16-3.1.fc{{ ansible_distribution_major_version }}" + '1.6.18': "{{ containerd_package }}-1.6.18-3.1.fc{{ ansible_distribution_major_version }}" + '1.6.28': "{{ containerd_package }}-1.6.28-3.1.fc{{ ansible_distribution_major_version }}" + 'stable': "{{ containerd_package }}-1.6.28-3.1.fc{{ ansible_distribution_major_version }}" + 'edge': "{{ containerd_package }}-1.6.28-3.1.fc{{ ansible_distribution_major_version }}" + +# https://docs.docker.com/install/linux/docker-ce/fedora/ +# https://download.docker.com/linux/fedora//x86_64/stable/Packages/ +docker_versioned_pkg: + 'latest': docker-ce + '19.03': docker-ce-19.03.15-3.fc{{ ansible_distribution_major_version }} + '20.10': docker-ce-20.10.20-3.fc{{ ansible_distribution_major_version }} + '23.0': docker-ce-3:23.0.6-1.fc{{ ansible_distribution_major_version }} + '24.0': docker-ce-3:24.0.9-1.fc{{ ansible_distribution_major_version }} + 'stable': docker-ce-3:24.0.9-1.fc{{ ansible_distribution_major_version }} + 'edge': docker-ce-3:24.0.9-1.fc{{ ansible_distribution_major_version }} + +docker_cli_versioned_pkg: + 'latest': docker-ce-cli + '19.03': docker-ce-cli-19.03.15-3.fc{{ ansible_distribution_major_version }} + '20.10': docker-ce-cli-20.10.20-3.fc{{ ansible_distribution_major_version }} + '23.0': docker-ce-cli-1:23.0.6-1.fc{{ ansible_distribution_major_version }} + '24.0': docker-ce-cli-1:24.0.9-1.fc{{ ansible_distribution_major_version }} + 'stable': docker-ce-cli-1:24.0.9-1.fc{{ ansible_distribution_major_version }} + 'edge': docker-ce-cli-1:24.0.9-1.fc{{ ansible_distribution_major_version }} + +docker_package_info: + enablerepo: "docker-ce" + pkgs: + - "{{ containerd_versioned_pkg[docker_containerd_version | string] }}" + - "{{ docker_cli_versioned_pkg[docker_cli_version | string] }}" + - "{{ docker_versioned_pkg[docker_version | string] }}" diff --git a/kubespray/project/roles/container-engine/docker/vars/kylin.yml b/kubespray/project/roles/container-engine/docker/vars/kylin.yml new file mode 100644 index 0000000..2b8cdf7 --- /dev/null +++ b/kubespray/project/roles/container-engine/docker/vars/kylin.yml @@ -0,0 +1,58 @@ +--- +# containerd versions are only relevant for docker +containerd_versioned_pkg: + 'latest': "{{ containerd_package }}" + '1.3.7': "{{ containerd_package }}-1.3.7-3.1.el{{ ansible_distribution_major_version }}" + '1.3.9': "{{ containerd_package }}-1.3.9-3.1.el{{ ansible_distribution_major_version }}" + '1.4.3': "{{ containerd_package }}-1.4.3-3.2.el{{ ansible_distribution_major_version }}" + '1.4.4': "{{ containerd_package }}-1.4.4-3.1.el{{ ansible_distribution_major_version }}" + '1.4.6': "{{ containerd_package }}-1.4.6-3.1.el{{ ansible_distribution_major_version }}" + '1.4.9': "{{ containerd_package }}-1.4.9-3.1.el{{ ansible_distribution_major_version }}" + '1.4.12': "{{ containerd_package }}-1.4.12-3.1.el{{ ansible_distribution_major_version }}" + '1.6.4': "{{ containerd_package }}-1.6.4-3.1.el{{ ansible_distribution_major_version }}" + '1.6.6': "{{ containerd_package }}-1.6.6-3.1.el{{ ansible_distribution_major_version }}" + '1.6.7': "{{ containerd_package }}-1.6.7-3.1.el{{ ansible_distribution_major_version }}" + '1.6.8': "{{ containerd_package }}-1.6.8-3.1.el{{ ansible_distribution_major_version }}" + '1.6.9': "{{ containerd_package }}-1.6.9-3.1.el{{ ansible_distribution_major_version }}" + '1.6.10': "{{ containerd_package }}-1.6.10-3.1.el{{ ansible_distribution_major_version }}" + '1.6.11': "{{ containerd_package }}-1.6.11-3.1.el{{ ansible_distribution_major_version }}" + '1.6.12': "{{ containerd_package }}-1.6.12-3.1.el{{ ansible_distribution_major_version }}" + '1.6.13': "{{ containerd_package }}-1.6.13-3.1.el{{ ansible_distribution_major_version }}" + '1.6.14': "{{ containerd_package }}-1.6.14-3.1.el{{ ansible_distribution_major_version }}" + '1.6.15': "{{ containerd_package }}-1.6.15-3.1.el{{ ansible_distribution_major_version }}" + '1.6.16': "{{ containerd_package }}-1.6.16-3.1.el{{ ansible_distribution_major_version }}" + '1.6.18': "{{ containerd_package }}-1.6.18-3.1.el{{ ansible_distribution_major_version }}" + '1.6.28': "{{ containerd_package }}-1.6.28-3.1.el{{ ansible_distribution_major_version }}" + 'stable': "{{ containerd_package }}-1.6.28-3.1.el{{ ansible_distribution_major_version }}" + 'edge': "{{ containerd_package }}-1.6.28-3.1.el{{ ansible_distribution_major_version }}" + +docker_version: 19.03 +docker_cli_version: 19.03 + +# https://docs.docker.com/engine/installation/linux/centos/#install-from-a-package +# https://download.docker.com/linux/centos/>/x86_64/stable/Packages/ +# or do 'yum --showduplicates list docker-engine' +docker_versioned_pkg: + 'latest': docker-ce + '18.09': docker-ce-3:18.09.9-3.el{{ ansible_distribution_major_version }} + '19.03': docker-ce-3:19.03.15-3.el{{ ansible_distribution_major_version }} + '23.0': docker-ce-3:23.0.6-1.el{{ ansible_distribution_major_version }} + '24.0': docker-ce-3:24.0.9-1.el{{ ansible_distribution_major_version }} + 'stable': docker-ce-3:24.0.9-1.el{{ ansible_distribution_major_version }} + 'edge': docker-ce-3:24.0.9-1.el{{ ansible_distribution_major_version }} + +docker_cli_versioned_pkg: + 'latest': docker-ce-cli + '18.09': docker-ce-cli-1:18.09.9-3.el{{ ansible_distribution_major_version }} + '19.03': docker-ce-cli-1:19.03.15-3.el{{ ansible_distribution_major_version }} + '23.0': docker-ce-cli-1:23.0.6-1.el{{ ansible_distribution_major_version }} + '24.0': docker-ce-cli-1:24.0.9-1.el{{ ansible_distribution_major_version }} + 'stable': docker-ce-cli-1:24.0.9-1.el{{ ansible_distribution_major_version }} + 'edge': docker-ce-cli-1:24.0.9-1.el{{ ansible_distribution_major_version }} + +docker_package_info: + enablerepo: "docker-ce" + pkgs: + - "{{ containerd_versioned_pkg[docker_containerd_version | string] }}" + - "{{ docker_cli_versioned_pkg[docker_cli_version | string] }}" + - "{{ docker_versioned_pkg[docker_version | string] }}" diff --git a/kubespray/project/roles/container-engine/docker/vars/redhat-7.yml b/kubespray/project/roles/container-engine/docker/vars/redhat-7.yml new file mode 100644 index 0000000..c10e881 --- /dev/null +++ b/kubespray/project/roles/container-engine/docker/vars/redhat-7.yml @@ -0,0 +1,57 @@ +--- +# containerd versions are only relevant for docker +containerd_versioned_pkg: + 'latest': "{{ containerd_package }}" + '1.3.7': "{{ containerd_package }}-1.3.7-3.1.el7" + '1.3.9': "{{ containerd_package }}-1.3.9-3.1.el7" + '1.4.3': "{{ containerd_package }}-1.4.3-3.2.el7" + '1.4.4': "{{ containerd_package }}-1.4.4-3.1.el7" + '1.4.6': "{{ containerd_package }}-1.4.6-3.1.el7" + '1.4.9': "{{ containerd_package }}-1.4.9-3.1.el7" + '1.4.12': "{{ containerd_package }}-1.4.12-3.1.el7" + '1.6.4': "{{ containerd_package }}-1.6.4-3.1.el7" + '1.6.6': "{{ containerd_package }}-1.6.6-3.1.el7" + '1.6.7': "{{ containerd_package }}-1.6.7-3.1.el7" + '1.6.8': "{{ containerd_package }}-1.6.8-3.1.el7" + '1.6.9': "{{ containerd_package }}-1.6.9-3.1.el7" + '1.6.10': "{{ containerd_package }}-1.6.10-3.1.el7" + '1.6.11': "{{ containerd_package }}-1.6.11-3.1.el7" + '1.6.12': "{{ containerd_package }}-1.6.12-3.1.el7" + '1.6.13': "{{ containerd_package }}-1.6.13-3.1.el7" + '1.6.14': "{{ containerd_package }}-1.6.14-3.1.el7" + '1.6.15': "{{ containerd_package }}-1.6.15-3.1.el7" + '1.6.16': "{{ containerd_package }}-1.6.16-3.1.el7" + '1.6.18': "{{ containerd_package }}-1.6.18-3.1.el7" + '1.6.28': "{{ containerd_package }}-1.6.28-3.1.el7" + 'stable': "{{ containerd_package }}-1.6.28-3.1.el7" + 'edge': "{{ containerd_package }}-1.6.18-3.1.el7" + +# https://docs.docker.com/engine/installation/linux/centos/#install-from-a-package +# https://download.docker.com/linux/centos/>/x86_64/stable/Packages/ +# or do 'yum --showduplicates list docker-engine' +docker_versioned_pkg: + 'latest': docker-ce + '18.09': docker-ce-18.09.9-3.el7 + '19.03': docker-ce-19.03.15-3.el7 + '20.10': docker-ce-20.10.20-3.el7 + '23.0': docker-ce-23.0.6-1.el7 + '24.0': docker-ce-24.0.9-1.el7 + 'stable': docker-ce-24.0.9-1.el7 + 'edge': docker-ce-24.0.9-1.el7 + +docker_cli_versioned_pkg: + 'latest': docker-ce-cli + '18.09': docker-ce-cli-18.09.9-3.el7 + '19.03': docker-ce-cli-19.03.15-3.el7 + '20.10': docker-ce-cli-20.10.20-3.el7 + '23.0': docker-ce-cli-23.0.6-1.el7 + '24.0': docker-ce-cli-24.0.9-1.el7 + 'stable': docker-ce-cli-24.0.9-1.el7 + 'edge': docker-ce-cli-24.0.9-1.el7 + +docker_package_info: + enablerepo: "docker-ce" + pkgs: + - "{{ containerd_versioned_pkg[docker_containerd_version | string] }}" + - "{{ docker_cli_versioned_pkg[docker_cli_version | string] }}" + - "{{ docker_versioned_pkg[docker_version | string] }}" diff --git a/kubespray/project/roles/container-engine/docker/vars/redhat.yml b/kubespray/project/roles/container-engine/docker/vars/redhat.yml new file mode 100644 index 0000000..453f7f9 --- /dev/null +++ b/kubespray/project/roles/container-engine/docker/vars/redhat.yml @@ -0,0 +1,57 @@ +--- +# containerd versions are only relevant for docker +containerd_versioned_pkg: + 'latest': "{{ containerd_package }}" + '1.3.7': "{{ containerd_package }}-1.3.7-3.1.el{{ ansible_distribution_major_version }}" + '1.3.9': "{{ containerd_package }}-1.3.9-3.1.el{{ ansible_distribution_major_version }}" + '1.4.3': "{{ containerd_package }}-1.4.3-3.2.el{{ ansible_distribution_major_version }}" + '1.4.4': "{{ containerd_package }}-1.4.4-3.1.el{{ ansible_distribution_major_version }}" + '1.4.6': "{{ containerd_package }}-1.4.6-3.1.el{{ ansible_distribution_major_version }}" + '1.4.9': "{{ containerd_package }}-1.4.9-3.1.el{{ ansible_distribution_major_version }}" + '1.4.12': "{{ containerd_package }}-1.4.12-3.1.el{{ ansible_distribution_major_version }}" + '1.6.4': "{{ containerd_package }}-1.6.4-3.1.el{{ ansible_distribution_major_version }}" + '1.6.6': "{{ containerd_package }}-1.6.6-3.1.el{{ ansible_distribution_major_version }}" + '1.6.7': "{{ containerd_package }}-1.6.7-3.1.el{{ ansible_distribution_major_version }}" + '1.6.8': "{{ containerd_package }}-1.6.8-3.1.el{{ ansible_distribution_major_version }}" + '1.6.9': "{{ containerd_package }}-1.6.9-3.1.el{{ ansible_distribution_major_version }}" + '1.6.10': "{{ containerd_package }}-1.6.10-3.1.el{{ ansible_distribution_major_version }}" + '1.6.11': "{{ containerd_package }}-1.6.11-3.1.el{{ ansible_distribution_major_version }}" + '1.6.12': "{{ containerd_package }}-1.6.12-3.1.el{{ ansible_distribution_major_version }}" + '1.6.13': "{{ containerd_package }}-1.6.13-3.1.el{{ ansible_distribution_major_version }}" + '1.6.14': "{{ containerd_package }}-1.6.14-3.1.el{{ ansible_distribution_major_version }}" + '1.6.15': "{{ containerd_package }}-1.6.15-3.1.el{{ ansible_distribution_major_version }}" + '1.6.16': "{{ containerd_package }}-1.6.16-3.1.el{{ ansible_distribution_major_version }}" + '1.6.18': "{{ containerd_package }}-1.6.18-3.1.el{{ ansible_distribution_major_version }}" + '1.6.28': "{{ containerd_package }}-1.6.28-3.1.el{{ ansible_distribution_major_version }}" + 'stable': "{{ containerd_package }}-1.6.28-3.1.el{{ ansible_distribution_major_version }}" + 'edge': "{{ containerd_package }}-1.6.28-3.1.el{{ ansible_distribution_major_version }}" + +# https://docs.docker.com/engine/installation/linux/centos/#install-from-a-package +# https://download.docker.com/linux/centos/>/x86_64/stable/Packages/ +# or do 'yum --showduplicates list docker-engine' +docker_versioned_pkg: + 'latest': docker-ce + '18.09': docker-ce-3:18.09.9-3.el7 + '19.03': docker-ce-3:19.03.15-3.el{{ ansible_distribution_major_version }} + '20.10': docker-ce-3:20.10.20-3.el{{ ansible_distribution_major_version }} + '23.0': docker-ce-3:23.0.6-1.el{{ ansible_distribution_major_version }} + '24.0': docker-ce-3:24.0.9-1.el{{ ansible_distribution_major_version }} + 'stable': docker-ce-3:24.0.9-1.el{{ ansible_distribution_major_version }} + 'edge': docker-ce-3:24.0.9-1.el{{ ansible_distribution_major_version }} + +docker_cli_versioned_pkg: + 'latest': docker-ce-cli + '18.09': docker-ce-cli-1:18.09.9-3.el7 + '19.03': docker-ce-cli-1:19.03.15-3.el{{ ansible_distribution_major_version }} + '20.10': docker-ce-cli-1:20.10.20-3.el{{ ansible_distribution_major_version }} + '23.0': docker-ce-cli-1:23.0.6-1.el{{ ansible_distribution_major_version }} + '24.0': docker-ce-cli-1:24.0.9-1.el{{ ansible_distribution_major_version }} + 'stable': docker-ce-cli-1:24.0.9-1.el{{ ansible_distribution_major_version }} + 'edge': docker-ce-cli-1:24.0.9-1.el{{ ansible_distribution_major_version }} + +docker_package_info: + enablerepo: "docker-ce" + pkgs: + - "{{ containerd_versioned_pkg[docker_containerd_version | string] }}" + - "{{ docker_cli_versioned_pkg[docker_cli_version | string] }}" + - "{{ docker_versioned_pkg[docker_version | string] }}" diff --git a/kubespray/project/roles/container-engine/docker/vars/suse.yml b/kubespray/project/roles/container-engine/docker/vars/suse.yml new file mode 100644 index 0000000..2d9fbf0 --- /dev/null +++ b/kubespray/project/roles/container-engine/docker/vars/suse.yml @@ -0,0 +1,6 @@ +--- +docker_package_info: + state: latest + pkgs: + - docker + - containerd diff --git a/kubespray/project/roles/container-engine/docker/vars/ubuntu.yml b/kubespray/project/roles/container-engine/docker/vars/ubuntu.yml new file mode 100644 index 0000000..b4c1d28 --- /dev/null +++ b/kubespray/project/roles/container-engine/docker/vars/ubuntu.yml @@ -0,0 +1,66 @@ +--- +# containerd versions are only relevant for docker +containerd_versioned_pkg: + 'latest': "{{ containerd_package }}" + '1.3.7': "{{ containerd_package }}=1.3.7-1" + '1.3.9': "{{ containerd_package }}=1.3.9-1" + '1.4.3': "{{ containerd_package }}=1.4.3-2" + '1.4.4': "{{ containerd_package }}=1.4.4-1" + '1.4.6': "{{ containerd_package }}=1.4.6-1" + '1.4.9': "{{ containerd_package }}=1.4.9-1" + '1.4.12': "{{ containerd_package }}=1.4.12-1" + '1.6.4': "{{ containerd_package }}=1.6.4-1" + '1.6.6': "{{ containerd_package }}=1.6.6-1" + '1.6.7': "{{ containerd_package }}=1.6.7-1" + '1.6.8': "{{ containerd_package }}=1.6.8-1" + '1.6.9': "{{ containerd_package }}=1.6.9-1" + '1.6.10': "{{ containerd_package }}=1.6.10-1" + '1.6.11': "{{ containerd_package }}=1.6.11-1" + '1.6.12': "{{ containerd_package }}=1.6.12-1" + '1.6.13': "{{ containerd_package }}=1.6.13-1" + '1.6.14': "{{ containerd_package }}=1.6.14-1" + '1.6.15': "{{ containerd_package }}=1.6.15-1" + '1.6.16': "{{ containerd_package }}=1.6.16-1" + '1.6.18': "{{ containerd_package }}=1.6.18-1" + '1.6.28': "{{ containerd_package }}=1.6.28-1" + 'stable': "{{ containerd_package }}=1.6.28-1" + 'edge': "{{ containerd_package }}=1.6.28-1" + +# https://download.docker.com/linux/ubuntu/ +docker_versioned_pkg: + 'latest': docker-ce + '18.09': docker-ce=5:18.09.9~3-0~ubuntu-{{ ansible_distribution_release | lower }} + '19.03': docker-ce=5:19.03.15~3-0~ubuntu-{{ ansible_distribution_release | lower }} + '20.10': docker-ce=5:20.10.20~3-0~ubuntu-{{ ansible_distribution_release | lower }} + '23.0': docker-ce=5:23.0.6-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }} + '24.0': docker-ce=5:24.0.9-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }} + 'stable': docker-ce=5:24.0.9-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }} + 'edge': docker-ce=5:24.0.9-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }} + +docker_cli_versioned_pkg: + 'latest': docker-ce-cli + '18.09': docker-ce-cli=5:18.09.9~3-0~ubuntu-{{ ansible_distribution_release | lower }} + '19.03': docker-ce-cli=5:19.03.15~3-0~ubuntu-{{ ansible_distribution_release | lower }} + '20.10': docker-ce-cli=5:20.10.20~3-0~ubuntu-{{ ansible_distribution_release | lower }} + '23.0': docker-ce-cli=5:23.0.6-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }} + '24.0': docker-ce-cli=5:24.0.9-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }} + 'stable': docker-ce-cli=5:24.0.9-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }} + 'edge': docker-ce-cli=5:24.0.9-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }} + +docker_package_info: + pkgs: + - "{{ containerd_versioned_pkg[docker_containerd_version | string] }}" + - "{{ docker_cli_versioned_pkg[docker_cli_version | string] }}" + - "{{ docker_versioned_pkg[docker_version | string] }}" + +docker_repo_key_info: + url: '{{ docker_ubuntu_repo_gpgkey }}' + repo_keys: + - '{{ docker_ubuntu_repo_repokey }}' + +docker_repo_info: + repos: + - > + deb [arch={{ host_architecture }}] {{ docker_ubuntu_repo_base_url }} + {{ ansible_distribution_release | lower }} + stable diff --git a/kubespray/project/roles/container-engine/docker/vars/uniontech.yml b/kubespray/project/roles/container-engine/docker/vars/uniontech.yml new file mode 100644 index 0000000..0749726 --- /dev/null +++ b/kubespray/project/roles/container-engine/docker/vars/uniontech.yml @@ -0,0 +1,59 @@ +--- +# containerd versions are only relevant for docker +containerd_versioned_pkg: + 'latest': "{{ containerd_package }}" + '1.3.7': "{{ containerd_package }}-1.3.7-3.1.el{{ ansible_distribution_major_version }}" + '1.3.9': "{{ containerd_package }}-1.3.9-3.1.el{{ ansible_distribution_major_version }}" + '1.4.3': "{{ containerd_package }}-1.4.3-3.2.el{{ ansible_distribution_major_version }}" + '1.4.4': "{{ containerd_package }}-1.4.4-3.1.el{{ ansible_distribution_major_version }}" + '1.4.6': "{{ containerd_package }}-1.4.6-3.1.el{{ ansible_distribution_major_version }}" + '1.4.9': "{{ containerd_package }}-1.4.9-3.1.el{{ ansible_distribution_major_version }}" + '1.4.12': "{{ containerd_package }}-1.4.12-3.1.el{{ ansible_distribution_major_version }}" + '1.6.4': "{{ containerd_package }}-1.6.4-3.1.el{{ ansible_distribution_major_version }}" + '1.6.8': "{{ containerd_package }}-1.6.8-3.1.el{{ ansible_distribution_major_version }}" + '1.6.9': "{{ containerd_package }}-1.6.9-3.1.el{{ ansible_distribution_major_version }}" + '1.6.10': "{{ containerd_package }}-1.6.10-3.1.el{{ ansible_distribution_major_version }}" + '1.6.11': "{{ containerd_package }}-1.6.11-3.1.el{{ ansible_distribution_major_version }}" + '1.6.12': "{{ containerd_package }}-1.6.12-3.1.el{{ ansible_distribution_major_version }}" + '1.6.13': "{{ containerd_package }}-1.6.13-3.1.el{{ ansible_distribution_major_version }}" + '1.6.14': "{{ containerd_package }}-1.6.14-3.1.el{{ ansible_distribution_major_version }}" + '1.6.15': "{{ containerd_package }}-1.6.15-3.1.el{{ ansible_distribution_major_version }}" + '1.6.16': "{{ containerd_package }}-1.6.16-3.1.el{{ ansible_distribution_major_version }}" + '1.6.18': "{{ containerd_package }}-1.6.18-3.1.el{{ ansible_distribution_major_version }}" + '1.6.28': "{{ containerd_package }}-1.6.28-3.1.el{{ ansible_distribution_major_version }}" + 'stable': "{{ containerd_package }}-1.6.28-3.1.el{{ ansible_distribution_major_version }}" + 'edge': "{{ containerd_package }}-1.6.28-3.1.el{{ ansible_distribution_major_version }}" + +docker_version: 19.03 +docker_cli_version: 19.03 + +# https://docs.docker.com/engine/installation/linux/centos/#install-from-a-package +# https://download.docker.com/linux/centos/>/x86_64/stable/Packages/ +# or do 'yum --showduplicates list docker-engine' +docker_versioned_pkg: + 'latest': docker-ce + '18.09': docker-ce-3:18.09.9-3.el7 + '19.03': docker-ce-3:19.03.15-3.el{{ ansible_distribution_major_version }} + '20.10': docker-ce-3:20.10.17-3.el{{ ansible_distribution_major_version }} + '23.0': docker-ce-3:23.0.6-1.el{{ ansible_distribution_major_version }} + '24.0': docker-ce-3:24.0.9-1.el{{ ansible_distribution_major_version }} + 'stable': docker-ce-3:24.0.9-1.el{{ ansible_distribution_major_version }} + 'edge': docker-ce-3:24.0.9-1.el{{ ansible_distribution_major_version }} + +docker_cli_versioned_pkg: + 'latest': docker-ce-cli + '18.09': docker-ce-cli-1:18.09.9-3.el7 + '19.03': docker-ce-cli-1:19.03.15-3.el{{ ansible_distribution_major_version }} + '20.10': docker-ce-cli-1:20.10.17-3.el{{ ansible_distribution_major_version }} + '23.0': docker-ce-cli-1:23.0.6-1.el{{ ansible_distribution_major_version }} + '24.0': docker-ce-cli-1:24.0.9-1.el{{ ansible_distribution_major_version }} + 'stable': docker-ce-cli-1:24.0.9-1.el{{ ansible_distribution_major_version }} + 'edge': docker-ce-cli-1:24.0.9-1.el{{ ansible_distribution_major_version }} + +docker_package_info: + enablerepo: "docker-ce" + disablerepo: "UniontechOS-20-AppStream" + pkgs: + - "{{ containerd_versioned_pkg[docker_containerd_version | string] }}" + - "{{ docker_cli_versioned_pkg[docker_cli_version | string] }}" + - "{{ docker_versioned_pkg[docker_version | string] }}" diff --git a/kubespray/project/roles/container-engine/gvisor/molecule/default/converge.yml b/kubespray/project/roles/container-engine/gvisor/molecule/default/converge.yml new file mode 100644 index 0000000..b14d078 --- /dev/null +++ b/kubespray/project/roles/container-engine/gvisor/molecule/default/converge.yml @@ -0,0 +1,11 @@ +--- +- name: Converge + hosts: all + become: true + vars: + gvisor_enabled: true + container_manager: containerd + roles: + - role: kubespray-defaults + - role: container-engine/containerd + - role: container-engine/gvisor diff --git a/kubespray/project/roles/container-engine/gvisor/molecule/default/files/10-mynet.conf b/kubespray/project/roles/container-engine/gvisor/molecule/default/files/10-mynet.conf new file mode 100644 index 0000000..f10935b --- /dev/null +++ b/kubespray/project/roles/container-engine/gvisor/molecule/default/files/10-mynet.conf @@ -0,0 +1,17 @@ +{ + "cniVersion": "0.2.0", + "name": "mynet", + "type": "bridge", + "bridge": "cni0", + "isGateway": true, + "ipMasq": true, + "ipam": { + "type": "host-local", + "subnet": "172.19.0.0/24", + "routes": [ + { + "dst": "0.0.0.0/0" + } + ] + } +} diff --git a/kubespray/project/roles/container-engine/gvisor/molecule/default/files/container.json b/kubespray/project/roles/container-engine/gvisor/molecule/default/files/container.json new file mode 100644 index 0000000..acec0ce --- /dev/null +++ b/kubespray/project/roles/container-engine/gvisor/molecule/default/files/container.json @@ -0,0 +1,10 @@ +{ + "metadata": { + "name": "gvisor1" + }, + "image": { + "image": "quay.io/kubespray/hello-world:latest" + }, + "log_path": "gvisor1.0.log", + "linux": {} +} diff --git a/kubespray/project/roles/container-engine/gvisor/molecule/default/files/sandbox.json b/kubespray/project/roles/container-engine/gvisor/molecule/default/files/sandbox.json new file mode 100644 index 0000000..a8da54d --- /dev/null +++ b/kubespray/project/roles/container-engine/gvisor/molecule/default/files/sandbox.json @@ -0,0 +1,10 @@ +{ + "metadata": { + "name": "gvisor1", + "namespace": "default", + "attempt": 1, + "uid": "hdishd83djaidwnduwk28bcsb" + }, + "linux": {}, + "log_directory": "/tmp" +} diff --git a/kubespray/project/roles/container-engine/gvisor/molecule/default/molecule.yml b/kubespray/project/roles/container-engine/gvisor/molecule/default/molecule.yml new file mode 100644 index 0000000..9ba1927 --- /dev/null +++ b/kubespray/project/roles/container-engine/gvisor/molecule/default/molecule.yml @@ -0,0 +1,39 @@ +--- +role_name_check: 1 +driver: + name: vagrant + provider: + name: libvirt +platforms: + - name: ubuntu20 + box: generic/ubuntu2004 + cpus: 1 + memory: 1024 + nested: true + groups: + - kube_control_plane + provider_options: + driver: kvm + - name: almalinux8 + box: almalinux/8 + cpus: 1 + memory: 1024 + nested: true + groups: + - kube_control_plane + provider_options: + driver: kvm +provisioner: + name: ansible + env: + ANSIBLE_ROLES_PATH: ../../../../ + config_options: + defaults: + callbacks_enabled: profile_tasks + timeout: 120 + inventory: + group_vars: + all: + become: true +verifier: + name: testinfra diff --git a/kubespray/project/roles/container-engine/gvisor/molecule/default/prepare.yml b/kubespray/project/roles/container-engine/gvisor/molecule/default/prepare.yml new file mode 100644 index 0000000..3ec3602 --- /dev/null +++ b/kubespray/project/roles/container-engine/gvisor/molecule/default/prepare.yml @@ -0,0 +1,49 @@ +--- +- name: Prepare generic + hosts: all + become: true + roles: + - role: kubespray-defaults + - role: bootstrap-os + - role: adduser + user: "{{ addusers.kube }}" + tasks: + - name: Download CNI + include_tasks: "../../../../download/tasks/download_file.yml" + vars: + download: "{{ download_defaults | combine(downloads.cni) }}" + +- name: Prepare container runtime + hosts: all + become: true + vars: + container_manager: containerd + kube_network_plugin: cni + roles: + - role: kubespray-defaults + - role: network_plugin/cni + - role: container-engine/crictl + tasks: + - name: Copy test container files + copy: + src: "{{ item }}" + dest: "/tmp/{{ item }}" + owner: root + mode: 0644 + with_items: + - container.json + - sandbox.json + - name: Create /etc/cni/net.d directory + file: + path: /etc/cni/net.d + state: directory + owner: root + mode: 0755 + - name: Setup CNI + copy: + src: "{{ item }}" + dest: "/etc/cni/net.d/{{ item }}" + owner: root + mode: 0644 + with_items: + - 10-mynet.conf diff --git a/kubespray/project/roles/container-engine/gvisor/molecule/default/tests/test_default.py b/kubespray/project/roles/container-engine/gvisor/molecule/default/tests/test_default.py new file mode 100644 index 0000000..1cb7fb0 --- /dev/null +++ b/kubespray/project/roles/container-engine/gvisor/molecule/default/tests/test_default.py @@ -0,0 +1,29 @@ +import os + +import testinfra.utils.ansible_runner + +testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( + os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') + + +def test_run(host): + gvisorruntime = "/usr/local/bin/runsc" + with host.sudo(): + cmd = host.command(gvisorruntime + " --version") + assert cmd.rc == 0 + assert "runsc version" in cmd.stdout + + +def test_run_pod(host): + runtime = "runsc" + + run_command = "/usr/local/bin/crictl run --with-pull --runtime {} /tmp/container.json /tmp/sandbox.json".format(runtime) + with host.sudo(): + cmd = host.command(run_command) + assert cmd.rc == 0 + + with host.sudo(): + log_f = host.file("/tmp/gvisor1.0.log") + + assert log_f.exists + assert b"Hello from Docker" in log_f.content diff --git a/kubespray/project/roles/container-engine/gvisor/tasks/main.yml b/kubespray/project/roles/container-engine/gvisor/tasks/main.yml new file mode 100644 index 0000000..1a8277b --- /dev/null +++ b/kubespray/project/roles/container-engine/gvisor/tasks/main.yml @@ -0,0 +1,20 @@ +--- +- name: GVisor | Download runsc binary + include_tasks: "../../../download/tasks/download_file.yml" + vars: + download: "{{ download_defaults | combine(downloads.gvisor_runsc) }}" + +- name: GVisor | Download containerd-shim-runsc-v1 binary + include_tasks: "../../../download/tasks/download_file.yml" + vars: + download: "{{ download_defaults | combine(downloads.gvisor_containerd_shim) }}" + +- name: GVisor | Copy binaries + copy: + src: "{{ item.src }}" + dest: "{{ bin_dir }}/{{ item.dest }}" + mode: 0755 + remote_src: yes + with_items: + - { src: "{{ downloads.gvisor_runsc.dest }}", dest: "runsc" } + - { src: "{{ downloads.gvisor_containerd_shim.dest }}", dest: "containerd-shim-runsc-v1" } diff --git a/kubespray/project/roles/container-engine/kata-containers/OWNERS b/kubespray/project/roles/container-engine/kata-containers/OWNERS new file mode 100644 index 0000000..fa95926 --- /dev/null +++ b/kubespray/project/roles/container-engine/kata-containers/OWNERS @@ -0,0 +1,6 @@ +# See the OWNERS docs at https://go.k8s.io/owners + +approvers: + - pasqualet +reviewers: + - pasqualet diff --git a/kubespray/project/roles/container-engine/kata-containers/defaults/main.yml b/kubespray/project/roles/container-engine/kata-containers/defaults/main.yml new file mode 100644 index 0000000..40bbc33 --- /dev/null +++ b/kubespray/project/roles/container-engine/kata-containers/defaults/main.yml @@ -0,0 +1,10 @@ +--- +kata_containers_dir: /opt/kata +kata_containers_config_dir: /etc/kata-containers +kata_containers_containerd_bin_dir: /usr/local/bin + +kata_containers_qemu_default_memory: "{{ ansible_memtotal_mb }}" +kata_containers_qemu_debug: 'false' +kata_containers_qemu_sandbox_cgroup_only: 'true' +kata_containers_qemu_enable_mem_prealloc: 'false' +kata_containers_virtio_fs_cache: 'always' diff --git a/kubespray/project/roles/container-engine/kata-containers/molecule/default/converge.yml b/kubespray/project/roles/container-engine/kata-containers/molecule/default/converge.yml new file mode 100644 index 0000000..a6fdf81 --- /dev/null +++ b/kubespray/project/roles/container-engine/kata-containers/molecule/default/converge.yml @@ -0,0 +1,11 @@ +--- +- name: Converge + hosts: all + become: true + vars: + kata_containers_enabled: true + container_manager: containerd + roles: + - role: kubespray-defaults + - role: container-engine/containerd + - role: container-engine/kata-containers diff --git a/kubespray/project/roles/container-engine/kata-containers/molecule/default/files/10-mynet.conf b/kubespray/project/roles/container-engine/kata-containers/molecule/default/files/10-mynet.conf new file mode 100644 index 0000000..f10935b --- /dev/null +++ b/kubespray/project/roles/container-engine/kata-containers/molecule/default/files/10-mynet.conf @@ -0,0 +1,17 @@ +{ + "cniVersion": "0.2.0", + "name": "mynet", + "type": "bridge", + "bridge": "cni0", + "isGateway": true, + "ipMasq": true, + "ipam": { + "type": "host-local", + "subnet": "172.19.0.0/24", + "routes": [ + { + "dst": "0.0.0.0/0" + } + ] + } +} diff --git a/kubespray/project/roles/container-engine/kata-containers/molecule/default/files/container.json b/kubespray/project/roles/container-engine/kata-containers/molecule/default/files/container.json new file mode 100644 index 0000000..e2e9a56 --- /dev/null +++ b/kubespray/project/roles/container-engine/kata-containers/molecule/default/files/container.json @@ -0,0 +1,10 @@ +{ + "metadata": { + "name": "kata1" + }, + "image": { + "image": "quay.io/kubespray/hello-world:latest" + }, + "log_path": "kata1.0.log", + "linux": {} +} diff --git a/kubespray/project/roles/container-engine/kata-containers/molecule/default/files/sandbox.json b/kubespray/project/roles/container-engine/kata-containers/molecule/default/files/sandbox.json new file mode 100644 index 0000000..326a578 --- /dev/null +++ b/kubespray/project/roles/container-engine/kata-containers/molecule/default/files/sandbox.json @@ -0,0 +1,10 @@ +{ + "metadata": { + "name": "kata1", + "namespace": "default", + "attempt": 1, + "uid": "hdishd83djaidwnduwk28bcsb" + }, + "linux": {}, + "log_directory": "/tmp" +} diff --git a/kubespray/project/roles/container-engine/kata-containers/molecule/default/molecule.yml b/kubespray/project/roles/container-engine/kata-containers/molecule/default/molecule.yml new file mode 100644 index 0000000..8eaa5d7 --- /dev/null +++ b/kubespray/project/roles/container-engine/kata-containers/molecule/default/molecule.yml @@ -0,0 +1,39 @@ +--- +role_name_check: 1 +driver: + name: vagrant + provider: + name: libvirt +platforms: + - name: ubuntu20 + box: generic/ubuntu2004 + cpus: 1 + memory: 1024 + nested: true + groups: + - kube_control_plane + provider_options: + driver: kvm + - name: ubuntu22 + box: generic/ubuntu2204 + cpus: 1 + memory: 1024 + nested: true + groups: + - kube_control_plane + provider_options: + driver: kvm +provisioner: + name: ansible + env: + ANSIBLE_ROLES_PATH: ../../../../ + config_options: + defaults: + callbacks_enabled: profile_tasks + timeout: 120 + inventory: + group_vars: + all: + become: true +verifier: + name: testinfra diff --git a/kubespray/project/roles/container-engine/kata-containers/molecule/default/prepare.yml b/kubespray/project/roles/container-engine/kata-containers/molecule/default/prepare.yml new file mode 100644 index 0000000..9d7019a --- /dev/null +++ b/kubespray/project/roles/container-engine/kata-containers/molecule/default/prepare.yml @@ -0,0 +1,49 @@ +--- +- name: Prepare + hosts: all + become: true + roles: + - role: kubespray-defaults + - role: bootstrap-os + - role: adduser + user: "{{ addusers.kube }}" + tasks: + - name: Download CNI + include_tasks: "../../../../download/tasks/download_file.yml" + vars: + download: "{{ download_defaults | combine(downloads.cni) }}" + +- name: Prepare container runtime + hosts: all + become: true + vars: + container_manager: containerd + kube_network_plugin: cni + roles: + - role: kubespray-defaults + - role: network_plugin/cni + - role: container-engine/crictl + tasks: + - name: Copy test container files + copy: + src: "{{ item }}" + dest: "/tmp/{{ item }}" + owner: root + mode: 0644 + with_items: + - container.json + - sandbox.json + - name: Create /etc/cni/net.d directory + file: + path: /etc/cni/net.d + state: directory + owner: "{{ kube_owner }}" + mode: 0755 + - name: Setup CNI + copy: + src: "{{ item }}" + dest: "/etc/cni/net.d/{{ item }}" + owner: root + mode: 0644 + with_items: + - 10-mynet.conf diff --git a/kubespray/project/roles/container-engine/kata-containers/molecule/default/tests/test_default.py b/kubespray/project/roles/container-engine/kata-containers/molecule/default/tests/test_default.py new file mode 100644 index 0000000..e10fff4 --- /dev/null +++ b/kubespray/project/roles/container-engine/kata-containers/molecule/default/tests/test_default.py @@ -0,0 +1,37 @@ +import os + +import testinfra.utils.ansible_runner + +testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( + os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') + + +def test_run(host): + kataruntime = "/opt/kata/bin/kata-runtime" + with host.sudo(): + cmd = host.command(kataruntime + " version") + assert cmd.rc == 0 + assert "kata-runtime" in cmd.stdout + + +def test_run_check(host): + kataruntime = "/opt/kata/bin/kata-runtime" + with host.sudo(): + cmd = host.command(kataruntime + " check") + assert cmd.rc == 0 + assert "System is capable of running" in cmd.stdout + + +def test_run_pod(host): + runtime = "kata-qemu" + + run_command = "/usr/local/bin/crictl run --with-pull --runtime {} /tmp/container.json /tmp/sandbox.json".format(runtime) + with host.sudo(): + cmd = host.command(run_command) + assert cmd.rc == 0 + + with host.sudo(): + log_f = host.file("/tmp/kata1.0.log") + + assert log_f.exists + assert b"Hello from Docker" in log_f.content diff --git a/kubespray/project/roles/container-engine/kata-containers/tasks/main.yml b/kubespray/project/roles/container-engine/kata-containers/tasks/main.yml new file mode 100644 index 0000000..e795b1f --- /dev/null +++ b/kubespray/project/roles/container-engine/kata-containers/tasks/main.yml @@ -0,0 +1,54 @@ +--- +- name: Kata-containers | Download kata binary + include_tasks: "../../../download/tasks/download_file.yml" + vars: + download: "{{ download_defaults | combine(downloads.kata_containers) }}" + +- name: Kata-containers | Copy kata-containers binary + unarchive: + src: "{{ downloads.kata_containers.dest }}" + dest: "/" + mode: 0755 + owner: root + group: root + remote_src: yes + +- name: Kata-containers | Create config directory + file: + path: "{{ kata_containers_config_dir }}" + state: directory + mode: 0755 + +- name: Kata-containers | Set configuration + template: + src: "{{ item }}.j2" + dest: "{{ kata_containers_config_dir }}/{{ item }}" + mode: 0644 + with_items: + - configuration-qemu.toml + +- name: Kata-containers | Set containerd bin + vars: + shim: "{{ item }}" + template: + dest: "{{ kata_containers_containerd_bin_dir }}/containerd-shim-kata-{{ item }}-v2" + src: containerd-shim-kata-v2.j2 + mode: 0755 + with_items: + - qemu + +- name: Kata-containers | Load vhost kernel modules + community.general.modprobe: + state: present + name: "{{ item }}" + with_items: + - vhost_vsock + - vhost_net + +- name: Kata-containers | Persist vhost kernel modules + copy: + dest: /etc/modules-load.d/kubespray-kata-containers.conf + mode: 0644 + content: | + vhost_vsock + vhost_net diff --git a/kubespray/project/roles/container-engine/kata-containers/templates/configuration-qemu.toml.j2 b/kubespray/project/roles/container-engine/kata-containers/templates/configuration-qemu.toml.j2 new file mode 100644 index 0000000..1551144 --- /dev/null +++ b/kubespray/project/roles/container-engine/kata-containers/templates/configuration-qemu.toml.j2 @@ -0,0 +1,706 @@ +# Copyright (c) 2017-2019 Intel Corporation +# Copyright (c) 2021 Adobe Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# XXX: WARNING: this file is auto-generated. +# XXX: +# XXX: Source file: "config/configuration-qemu.toml.in" +# XXX: Project: +# XXX: Name: Kata Containers +# XXX: Type: kata + +[hypervisor.qemu] +path = "/opt/kata/bin/qemu-system-x86_64" +{% if kata_containers_version is version('2.2.0', '>=') %} +kernel = "/opt/kata/share/kata-containers/vmlinux.container" +{% else %} +kernel = "/opt/kata/share/kata-containers/vmlinuz.container" +{% endif %} +image = "/opt/kata/share/kata-containers/kata-containers.img" +# initrd = "/opt/kata/share/kata-containers/kata-containers-initrd.img" +machine_type = "q35" + +# rootfs filesystem type: +# - ext4 (default) +# - xfs +# - erofs +rootfs_type="ext4" + +# Enable confidential guest support. +# Toggling that setting may trigger different hardware features, ranging +# from memory encryption to both memory and CPU-state encryption and integrity. +# The Kata Containers runtime dynamically detects the available feature set and +# aims at enabling the largest possible one, returning an error if none is +# available, or none is supported by the hypervisor. +# +# Known limitations: +# * Does not work by design: +# - CPU Hotplug +# - Memory Hotplug +# - NVDIMM devices +# +# Default false +# confidential_guest = true + +# Choose AMD SEV-SNP confidential guests +# In case of using confidential guests on AMD hardware that supports both SEV +# and SEV-SNP, the following enables SEV-SNP guests. SEV guests are default. +# Default false +# sev_snp_guest = true + +# Enable running QEMU VMM as a non-root user. +# By default QEMU VMM run as root. When this is set to true, QEMU VMM process runs as +# a non-root random user. See documentation for the limitations of this mode. +# rootless = true + +# List of valid annotation names for the hypervisor +# Each member of the list is a regular expression, which is the base name +# of the annotation, e.g. "path" for io.katacontainers.config.hypervisor.path" +enable_annotations = ["enable_iommu"] + +# List of valid annotations values for the hypervisor +# Each member of the list is a path pattern as described by glob(3). +# The default if not set is empty (all annotations rejected.) +# Your distribution recommends: ["/opt/kata/bin/qemu-system-x86_64"] +valid_hypervisor_paths = ["/opt/kata/bin/qemu-system-x86_64"] + +# Optional space-separated list of options to pass to the guest kernel. +# For example, use `kernel_params = "vsyscall=emulate"` if you are having +# trouble running pre-2.15 glibc. +# +# WARNING: - any parameter specified here will take priority over the default +# parameter value of the same name used to start the virtual machine. +# Do not set values here unless you understand the impact of doing so as you +# may stop the virtual machine from booting. +# To see the list of default parameters, enable hypervisor debug, create a +# container and look for 'default-kernel-parameters' log entries. +kernel_params = "" + +# Path to the firmware. +# If you want that qemu uses the default firmware leave this option empty +firmware = "" + +# Path to the firmware volume. +# firmware TDVF or OVMF can be split into FIRMWARE_VARS.fd (UEFI variables +# as configuration) and FIRMWARE_CODE.fd (UEFI program image). UEFI variables +# can be customized per each user while UEFI code is kept same. +firmware_volume = "" + +# Machine accelerators +# comma-separated list of machine accelerators to pass to the hypervisor. +# For example, `machine_accelerators = "nosmm,nosmbus,nosata,nopit,static-prt,nofw"` +machine_accelerators="" + +# Qemu seccomp sandbox feature +# comma-separated list of seccomp sandbox features to control the syscall access. +# For example, `seccompsandbox= "on,obsolete=deny,spawn=deny,resourcecontrol=deny"` +# Note: "elevateprivileges=deny" doesn't work with daemonize option, so it's removed from the seccomp sandbox +# Another note: enabling this feature may reduce performance, you may enable +# /proc/sys/net/core/bpf_jit_enable to reduce the impact. see https://man7.org/linux/man-pages/man8/bpfc.8.html +#seccompsandbox="on,obsolete=deny,spawn=deny,resourcecontrol=deny" + +# CPU features +# comma-separated list of cpu features to pass to the cpu +# For example, `cpu_features = "pmu=off,vmx=off" +cpu_features="pmu=off" + +# Default number of vCPUs per SB/VM: +# unspecified or 0 --> will be set to 1 +# < 0 --> will be set to the actual number of physical cores +# > 0 <= number of physical cores --> will be set to the specified number +# > number of physical cores --> will be set to the actual number of physical cores +default_vcpus = 1 + +# Default maximum number of vCPUs per SB/VM: +# unspecified or == 0 --> will be set to the actual number of physical cores or to the maximum number +# of vCPUs supported by KVM if that number is exceeded +# > 0 <= number of physical cores --> will be set to the specified number +# > number of physical cores --> will be set to the actual number of physical cores or to the maximum number +# of vCPUs supported by KVM if that number is exceeded +# WARNING: Depending of the architecture, the maximum number of vCPUs supported by KVM is used when +# the actual number of physical cores is greater than it. +# WARNING: Be aware that this value impacts the virtual machine's memory footprint and CPU +# the hotplug functionality. For example, `default_maxvcpus = 240` specifies that until 240 vCPUs +# can be added to a SB/VM, but the memory footprint will be big. Another example, with +# `default_maxvcpus = 8` the memory footprint will be small, but 8 will be the maximum number of +# vCPUs supported by the SB/VM. In general, we recommend that you do not edit this variable, +# unless you know what are you doing. +# NOTICE: on arm platform with gicv2 interrupt controller, set it to 8. +default_maxvcpus = 0 + +# Bridges can be used to hot plug devices. +# Limitations: +# * Currently only pci bridges are supported +# * Until 30 devices per bridge can be hot plugged. +# * Until 5 PCI bridges can be cold plugged per VM. +# This limitation could be a bug in qemu or in the kernel +# Default number of bridges per SB/VM: +# unspecified or 0 --> will be set to 1 +# > 1 <= 5 --> will be set to the specified number +# > 5 --> will be set to 5 +default_bridges = 1 + +# Default memory size in MiB for SB/VM. +# If unspecified then it will be set 2048 MiB. +default_memory = {{ kata_containers_qemu_default_memory }} +# +# Default memory slots per SB/VM. +# If unspecified then it will be set 10. +# This is will determine the times that memory will be hotadded to sandbox/VM. +#memory_slots = 10 + +# Default maximum memory in MiB per SB / VM +# unspecified or == 0 --> will be set to the actual amount of physical RAM +# > 0 <= amount of physical RAM --> will be set to the specified number +# > amount of physical RAM --> will be set to the actual amount of physical RAM +default_maxmemory = 0 + +# The size in MiB will be plused to max memory of hypervisor. +# It is the memory address space for the NVDIMM devie. +# If set block storage driver (block_device_driver) to "nvdimm", +# should set memory_offset to the size of block device. +# Default 0 +#memory_offset = 0 + +# Specifies virtio-mem will be enabled or not. +# Please note that this option should be used with the command +# "echo 1 > /proc/sys/vm/overcommit_memory". +# Default false +#enable_virtio_mem = true + +# Disable block device from being used for a container's rootfs. +# In case of a storage driver like devicemapper where a container's +# root file system is backed by a block device, the block device is passed +# directly to the hypervisor for performance reasons. +# This flag prevents the block device from being passed to the hypervisor, +# virtio-fs is used instead to pass the rootfs. +disable_block_device_use = false + +# Shared file system type: +# - virtio-fs (default) +# - virtio-9p +# - virtio-fs-nydus +{% if kata_containers_version is version('2.2.0', '>=') %} +shared_fs = "virtio-fs" +{% else %} +shared_fs = "virtio-9p" +{% endif %} + +# Path to vhost-user-fs daemon. +{% if kata_containers_version is version('2.5.0', '>=') %} +virtio_fs_daemon = "/opt/kata/libexec/virtiofsd" +{% else %} +virtio_fs_daemon = "/opt/kata/libexec/kata-qemu/virtiofsd" +{% endif %} + +# List of valid annotations values for the virtiofs daemon +# The default if not set is empty (all annotations rejected.) +# Your distribution recommends: ["/opt/kata/libexec/virtiofsd"] +valid_virtio_fs_daemon_paths = [ + "/opt/kata/libexec/virtiofsd", + "/opt/kata/libexec/kata-qemu/virtiofsd", +] + +# Default size of DAX cache in MiB +virtio_fs_cache_size = 0 + +# Default size of virtqueues +virtio_fs_queue_size = 1024 + +# Extra args for virtiofsd daemon +# +# Format example: +# ["--arg1=xxx", "--arg2=yyy"] +# Examples: +# Set virtiofsd log level to debug : ["--log-level=debug"] +# +# see `virtiofsd -h` for possible options. +virtio_fs_extra_args = ["--thread-pool-size=1", "--announce-submounts"] + +# Cache mode: +# +# - never +# Metadata, data, and pathname lookup are not cached in guest. They are +# always fetched from host and any changes are immediately pushed to host. +# +# - auto +# Metadata and pathname lookup cache expires after a configured amount of +# time (default is 1 second). Data is cached while the file is open (close +# to open consistency). +# +# - always +# Metadata, data, and pathname lookup are cached in guest and never expire. +virtio_fs_cache = "{{ kata_containers_virtio_fs_cache }}" + +# Block storage driver to be used for the hypervisor in case the container +# rootfs is backed by a block device. This is virtio-scsi, virtio-blk +# or nvdimm. +block_device_driver = "virtio-scsi" + +# aio is the I/O mechanism used by qemu +# Options: +# +# - threads +# Pthread based disk I/O. +# +# - native +# Native Linux I/O. +# +# - io_uring +# Linux io_uring API. This provides the fastest I/O operations on Linux, requires kernel>5.1 and +# qemu >=5.0. +block_device_aio = "io_uring" + +# Specifies cache-related options will be set to block devices or not. +# Default false +#block_device_cache_set = true + +# Specifies cache-related options for block devices. +# Denotes whether use of O_DIRECT (bypass the host page cache) is enabled. +# Default false +#block_device_cache_direct = true + +# Specifies cache-related options for block devices. +# Denotes whether flush requests for the device are ignored. +# Default false +#block_device_cache_noflush = true + +# Enable iothreads (data-plane) to be used. This causes IO to be +# handled in a separate IO thread. This is currently only implemented +# for SCSI. +# +enable_iothreads = false + +# Enable pre allocation of VM RAM, default false +# Enabling this will result in lower container density +# as all of the memory will be allocated and locked +# This is useful when you want to reserve all the memory +# upfront or in the cases where you want memory latencies +# to be very predictable +# Default false +enable_mem_prealloc = {{ kata_containers_qemu_enable_mem_prealloc }} + +# Enable huge pages for VM RAM, default false +# Enabling this will result in the VM memory +# being allocated using huge pages. +# This is useful when you want to use vhost-user network +# stacks within the container. This will automatically +# result in memory pre allocation +#enable_hugepages = true + +# Enable vhost-user storage device, default false +# Enabling this will result in some Linux reserved block type +# major range 240-254 being chosen to represent vhost-user devices. +enable_vhost_user_store = false + +# The base directory specifically used for vhost-user devices. +# Its sub-path "block" is used for block devices; "block/sockets" is +# where we expect vhost-user sockets to live; "block/devices" is where +# simulated block device nodes for vhost-user devices to live. +vhost_user_store_path = "/var/run/kata-containers/vhost-user" + +# Enable vIOMMU, default false +# Enabling this will result in the VM having a vIOMMU device +# This will also add the following options to the kernel's +# command line: intel_iommu=on,iommu=pt +#enable_iommu = true + +# Enable IOMMU_PLATFORM, default false +# Enabling this will result in the VM device having iommu_platform=on set +#enable_iommu_platform = true + +# List of valid annotations values for the vhost user store path +# The default if not set is empty (all annotations rejected.) +# Your distribution recommends: ["/var/run/kata-containers/vhost-user"] +valid_vhost_user_store_paths = ["/var/run/kata-containers/vhost-user"] + +# The timeout for reconnecting on non-server spdk sockets when the remote end goes away. +# qemu will delay this many seconds and then attempt to reconnect. +# Zero disables reconnecting, and the default is zero. +vhost_user_reconnect_timeout_sec = 0 + +# Enable file based guest memory support. The default is an empty string which +# will disable this feature. In the case of virtio-fs, this is enabled +# automatically and '/dev/shm' is used as the backing folder. +# This option will be ignored if VM templating is enabled. +#file_mem_backend = "" + +# List of valid annotations values for the file_mem_backend annotation +# The default if not set is empty (all annotations rejected.) +# Your distribution recommends: [""] +valid_file_mem_backends = [""] + +# -pflash can add image file to VM. The arguments of it should be in format +# of ["/path/to/flash0.img", "/path/to/flash1.img"] +pflashes = [] + +# This option changes the default hypervisor and kernel parameters +# to enable debug output where available. And Debug also enables the hmp socket. +# +# Default false +enable_debug = {{ kata_containers_qemu_debug }} + +# Disable the customizations done in the runtime when it detects +# that it is running on top a VMM. This will result in the runtime +# behaving as it would when running on bare metal. +# +#disable_nesting_checks = true + +# This is the msize used for 9p shares. It is the number of bytes +# used for 9p packet payload. +#msize_9p = 8192 + +# If false and nvdimm is supported, use nvdimm device to plug guest image. +# Otherwise virtio-block device is used. +# +# nvdimm is not supported when `confidential_guest = true`. +# +# Default is false +#disable_image_nvdimm = true + +# VFIO devices are hotplugged on a bridge by default. +# Enable hotplugging on root bus. This may be required for devices with +# a large PCI bar, as this is a current limitation with hotplugging on +# a bridge. +# Default false +#hotplug_vfio_on_root_bus = true + +# Before hot plugging a PCIe device, you need to add a pcie_root_port device. +# Use this parameter when using some large PCI bar devices, such as Nvidia GPU +# The value means the number of pcie_root_port +# This value is valid when hotplug_vfio_on_root_bus is true and machine_type is "q35" +# Default 0 +#pcie_root_port = 2 + +# If vhost-net backend for virtio-net is not desired, set to true. Default is false, which trades off +# security (vhost-net runs ring0) for network I/O performance. +#disable_vhost_net = true + +# +# Default entropy source. +# The path to a host source of entropy (including a real hardware RNG) +# /dev/urandom and /dev/random are two main options. +# Be aware that /dev/random is a blocking source of entropy. If the host +# runs out of entropy, the VMs boot time will increase leading to get startup +# timeouts. +# The source of entropy /dev/urandom is non-blocking and provides a +# generally acceptable source of entropy. It should work well for pretty much +# all practical purposes. +#entropy_source= "/dev/urandom" + +# List of valid annotations values for entropy_source +# The default if not set is empty (all annotations rejected.) +# Your distribution recommends: ["/dev/urandom","/dev/random",""] +valid_entropy_sources = ["/dev/urandom","/dev/random",""] + +# Path to OCI hook binaries in the *guest rootfs*. +# This does not affect host-side hooks which must instead be added to +# the OCI spec passed to the runtime. +# +# You can create a rootfs with hooks by customizing the osbuilder scripts: +# https://github.com/kata-containers/kata-containers/tree/main/tools/osbuilder +# +# Hooks must be stored in a subdirectory of guest_hook_path according to their +# hook type, i.e. "guest_hook_path/{prestart,poststart,poststop}". +# The agent will scan these directories for executable files and add them, in +# lexicographical order, to the lifecycle of the guest container. +# Hooks are executed in the runtime namespace of the guest. See the official documentation: +# https://github.com/opencontainers/runtime-spec/blob/v1.0.1/config.md#posix-platform-hooks +# Warnings will be logged if any error is encountered while scanning for hooks, +# but it will not abort container execution. +#guest_hook_path = "/usr/share/oci/hooks" +# +# Use rx Rate Limiter to control network I/O inbound bandwidth(size in bits/sec for SB/VM). +# In Qemu, we use classful qdiscs HTB(Hierarchy Token Bucket) to discipline traffic. +# Default 0-sized value means unlimited rate. +#rx_rate_limiter_max_rate = 0 +# Use tx Rate Limiter to control network I/O outbound bandwidth(size in bits/sec for SB/VM). +# In Qemu, we use classful qdiscs HTB(Hierarchy Token Bucket) and ifb(Intermediate Functional Block) +# to discipline traffic. +# Default 0-sized value means unlimited rate. +#tx_rate_limiter_max_rate = 0 + +# Set where to save the guest memory dump file. +# If set, when GUEST_PANICKED event occurred, +# guest memeory will be dumped to host filesystem under guest_memory_dump_path, +# This directory will be created automatically if it does not exist. +# +# The dumped file(also called vmcore) can be processed with crash or gdb. +# +# WARNING: +# Dump guest’s memory can take very long depending on the amount of guest memory +# and use much disk space. +#guest_memory_dump_path="/var/crash/kata" + +# If enable paging. +# Basically, if you want to use "gdb" rather than "crash", +# or need the guest-virtual addresses in the ELF vmcore, +# then you should enable paging. +# +# See: https://www.qemu.org/docs/master/qemu-qmp-ref.html#Dump-guest-memory for details +#guest_memory_dump_paging=false + +# Enable swap in the guest. Default false. +# When enable_guest_swap is enabled, insert a raw file to the guest as the swap device +# if the swappiness of a container (set by annotation "io.katacontainers.container.resource.swappiness") +# is bigger than 0. +# The size of the swap device should be +# swap_in_bytes (set by annotation "io.katacontainers.container.resource.swap_in_bytes") - memory_limit_in_bytes. +# If swap_in_bytes is not set, the size should be memory_limit_in_bytes. +# If swap_in_bytes and memory_limit_in_bytes is not set, the size should +# be default_memory. +#enable_guest_swap = true + +# use legacy serial for guest console if available and implemented for architecture. Default false +#use_legacy_serial = true + +# disable applying SELinux on the VMM process (default false) +disable_selinux=false + +# disable applying SELinux on the container process +# If set to false, the type `container_t` is applied to the container process by default. +# Note: To enable guest SELinux, the guest rootfs must be CentOS that is created and built +# with `SELINUX=yes`. +# (default: true) +disable_guest_selinux=true + +[factory] +# VM templating support. Once enabled, new VMs are created from template +# using vm cloning. They will share the same initial kernel, initramfs and +# agent memory by mapping it readonly. It helps speeding up new container +# creation and saves a lot of memory if there are many kata containers running +# on the same host. +# +# When disabled, new VMs are created from scratch. +# +# Note: Requires "initrd=" to be set ("image=" is not supported). +# +# Default false +#enable_template = true + +# Specifies the path of template. +# +# Default "/run/vc/vm/template" +#template_path = "/run/vc/vm/template" + +# The number of caches of VMCache: +# unspecified or == 0 --> VMCache is disabled +# > 0 --> will be set to the specified number +# +# VMCache is a function that creates VMs as caches before using it. +# It helps speed up new container creation. +# The function consists of a server and some clients communicating +# through Unix socket. The protocol is gRPC in protocols/cache/cache.proto. +# The VMCache server will create some VMs and cache them by factory cache. +# It will convert the VM to gRPC format and transport it when gets +# requestion from clients. +# Factory grpccache is the VMCache client. It will request gRPC format +# VM and convert it back to a VM. If VMCache function is enabled, +# kata-runtime will request VM from factory grpccache when it creates +# a new sandbox. +# +# Default 0 +#vm_cache_number = 0 + +# Specify the address of the Unix socket that is used by VMCache. +# +# Default /var/run/kata-containers/cache.sock +#vm_cache_endpoint = "/var/run/kata-containers/cache.sock" + +[agent.kata] +# If enabled, make the agent display debug-level messages. +# (default: disabled) +enable_debug = {{ kata_containers_qemu_debug }} + +# Enable agent tracing. +# +# If enabled, the agent will generate OpenTelemetry trace spans. +# +# Notes: +# +# - If the runtime also has tracing enabled, the agent spans will be +# associated with the appropriate runtime parent span. +# - If enabled, the runtime will wait for the container to shutdown, +# increasing the container shutdown time slightly. +# +# (default: disabled) +#enable_tracing = true + +# Comma separated list of kernel modules and their parameters. +# These modules will be loaded in the guest kernel using modprobe(8). +# The following example can be used to load two kernel modules with parameters +# - kernel_modules=["e1000e InterruptThrottleRate=3000,3000,3000 EEE=1", "i915 enable_ppgtt=0"] +# The first word is considered as the module name and the rest as its parameters. +# Container will not be started when: +# * A kernel module is specified and the modprobe command is not installed in the guest +# or it fails loading the module. +# * The module is not available in the guest or it doesn't met the guest kernel +# requirements, like architecture and version. +# +kernel_modules=[] + +# Enable debug console. + +# If enabled, user can connect guest OS running inside hypervisor +# through "kata-runtime exec " command + +#debug_console_enabled = true + +# Agent connection dialing timeout value in seconds +# (default: 30) +#dial_timeout = 30 + +[runtime] +# If enabled, the runtime will log additional debug messages to the +# system log +# (default: disabled) +enable_debug = {{ kata_containers_qemu_debug }} +# +# Internetworking model +# Determines how the VM should be connected to the +# the container network interface +# Options: +# +# - macvtap +# Used when the Container network interface can be bridged using +# macvtap. +# +# - none +# Used when customize network. Only creates a tap device. No veth pair. +# +# - tcfilter +# Uses tc filter rules to redirect traffic from the network interface +# provided by plugin to a tap interface connected to the VM. +# +internetworking_model="tcfilter" + +# disable guest seccomp +# Determines whether container seccomp profiles are passed to the virtual +# machine and applied by the kata agent. If set to true, seccomp is not applied +# within the guest +# (default: true) +disable_guest_seccomp=true + +# vCPUs pinning settings +# if enabled, each vCPU thread will be scheduled to a fixed CPU +# qualified condition: num(vCPU threads) == num(CPUs in sandbox's CPUSet) +# enable_vcpus_pinning = false + +# Apply a custom SELinux security policy to the container process inside the VM. +# This is used when you want to apply a type other than the default `container_t`, +# so general users should not uncomment and apply it. +# (format: "user:role:type") +# Note: You cannot specify MCS policy with the label because the sensitivity levels and +# categories are determined automatically by high-level container runtimes such as containerd. +#guest_selinux_label="system_u:system_r:container_t" + +# If enabled, the runtime will create opentracing.io traces and spans. +# (See https://www.jaegertracing.io/docs/getting-started). +# (default: disabled) +#enable_tracing = true + +# Set the full url to the Jaeger HTTP Thrift collector. +# The default if not set will be "http://localhost:14268/api/traces" +#jaeger_endpoint = "" + +# Sets the username to be used if basic auth is required for Jaeger. +#jaeger_user = "" + +# Sets the password to be used if basic auth is required for Jaeger. +#jaeger_password = "" + +# If enabled, the runtime will not create a network namespace for shim and hypervisor processes. +# This option may have some potential impacts to your host. It should only be used when you know what you're doing. +# `disable_new_netns` conflicts with `internetworking_model=tcfilter` and `internetworking_model=macvtap`. It works only +# with `internetworking_model=none`. The tap device will be in the host network namespace and can connect to a bridge +# (like OVS) directly. +# (default: false) +#disable_new_netns = true + +# if enabled, the runtime will add all the kata processes inside one dedicated cgroup. +# The container cgroups in the host are not created, just one single cgroup per sandbox. +# The runtime caller is free to restrict or collect cgroup stats of the overall Kata sandbox. +# The sandbox cgroup path is the parent cgroup of a container with the PodSandbox annotation. +# The sandbox cgroup is constrained if there is no container type annotation. +# See: https://pkg.go.dev/github.com/kata-containers/kata-containers/src/runtime/virtcontainers#ContainerType +sandbox_cgroup_only={{ kata_containers_qemu_sandbox_cgroup_only }} + +# If enabled, the runtime will attempt to determine appropriate sandbox size (memory, CPU) before booting the virtual machine. In +# this case, the runtime will not dynamically update the amount of memory and CPU in the virtual machine. This is generally helpful +# when a hardware architecture or hypervisor solutions is utilized which does not support CPU and/or memory hotplug. +# Compatibility for determining appropriate sandbox (VM) size: +# - When running with pods, sandbox sizing information will only be available if using Kubernetes >= 1.23 and containerd >= 1.6. CRI-O +# does not yet support sandbox sizing annotations. +# - When running single containers using a tool like ctr, container sizing information will be available. +static_sandbox_resource_mgmt=false + +# If specified, sandbox_bind_mounts identifieds host paths to be mounted (ro) into the sandboxes shared path. +# This is only valid if filesystem sharing is utilized. The provided path(s) will be bindmounted into the shared fs directory. +# If defaults are utilized, these mounts should be available in the guest at `/run/kata-containers/shared/containers/sandbox-mounts` +# These will not be exposed to the container workloads, and are only provided for potential guest services. +sandbox_bind_mounts=[] + +# VFIO Mode +# Determines how VFIO devices should be be presented to the container. +# Options: +# +# - vfio +# Matches behaviour of OCI runtimes (e.g. runc) as much as +# possible. VFIO devices will appear in the container as VFIO +# character devices under /dev/vfio. The exact names may differ +# from the host (they need to match the VM's IOMMU group numbers +# rather than the host's) +# +# - guest-kernel +# This is a Kata-specific behaviour that's useful in certain cases. +# The VFIO device is managed by whatever driver in the VM kernel +# claims it. This means it will appear as one or more device nodes +# or network interfaces depending on the nature of the device. +# Using this mode requires specially built workloads that know how +# to locate the relevant device interfaces within the VM. +# +vfio_mode="guest-kernel" + +# If enabled, the runtime will not create Kubernetes emptyDir mounts on the guest filesystem. Instead, emptyDir mounts will +# be created on the host and shared via virtio-fs. This is potentially slower, but allows sharing of files from host to guest. +disable_guest_empty_dir=false + +# Enabled experimental feature list, format: ["a", "b"]. +# Experimental features are features not stable enough for production, +# they may break compatibility, and are prepared for a big version bump. +# Supported experimental features: +# (default: []) +experimental=[] + +# If enabled, user can run pprof tools with shim v2 process through kata-monitor. +# (default: false) +# enable_pprof = true + +# WARNING: All the options in the following section have not been implemented yet. +# This section was added as a placeholder. DO NOT USE IT! +[image] +# Container image service. +# +# Offload the CRI image management service to the Kata agent. +# (default: false) +#service_offload = true + +# Container image decryption keys provisioning. +# Applies only if service_offload is true. +# Keys can be provisioned locally (e.g. through a special command or +# a local file) or remotely (usually after the guest is remotely attested). +# The provision setting is a complete URL that lets the Kata agent decide +# which method to use in order to fetch the keys. +# +# Keys can be stored in a local file, in a measured and attested initrd: +#provision=data:///local/key/file +# +# Keys could be fetched through a special command or binary from the +# initrd (guest) image, e.g. a firmware call: +#provision=file:///path/to/bin/fetcher/in/guest +# +# Keys can be remotely provisioned. The Kata agent fetches them from e.g. +# a HTTPS URL: +#provision=https://my-key-broker.foo/tenant/ diff --git a/kubespray/project/roles/container-engine/kata-containers/templates/containerd-shim-kata-v2.j2 b/kubespray/project/roles/container-engine/kata-containers/templates/containerd-shim-kata-v2.j2 new file mode 100644 index 0000000..a3cb830 --- /dev/null +++ b/kubespray/project/roles/container-engine/kata-containers/templates/containerd-shim-kata-v2.j2 @@ -0,0 +1,2 @@ +#!/bin/bash +KATA_CONF_FILE={{ kata_containers_config_dir }}/configuration-{{ shim }}.toml {{ kata_containers_dir }}/bin/containerd-shim-kata-v2 $@ diff --git a/kubespray/project/roles/container-engine/meta/main.yml b/kubespray/project/roles/container-engine/meta/main.yml new file mode 100644 index 0000000..3e068d6 --- /dev/null +++ b/kubespray/project/roles/container-engine/meta/main.yml @@ -0,0 +1,58 @@ +# noqa role-name - this is a meta role that doesn't need a name +--- +dependencies: + - role: container-engine/validate-container-engine + tags: + - container-engine + - validate-container-engine + + - role: container-engine/kata-containers + when: + - kata_containers_enabled + tags: + - container-engine + - kata-containers + + - role: container-engine/gvisor + when: + - gvisor_enabled + - container_manager in ['docker', 'containerd'] + tags: + - container-engine + - gvisor + + - role: container-engine/crun + when: + - crun_enabled + tags: + - container-engine + - crun + + - role: container-engine/youki + when: + - youki_enabled + - container_manager == 'crio' + tags: + - container-engine + - youki + + - role: container-engine/cri-o + when: + - container_manager == 'crio' + tags: + - container-engine + - crio + + - role: container-engine/containerd + when: + - container_manager == 'containerd' + tags: + - container-engine + - containerd + + - role: container-engine/cri-dockerd + when: + - container_manager == 'docker' + tags: + - container-engine + - docker diff --git a/kubespray/project/roles/container-engine/nerdctl/handlers/main.yml b/kubespray/project/roles/container-engine/nerdctl/handlers/main.yml new file mode 100644 index 0000000..27895ff --- /dev/null +++ b/kubespray/project/roles/container-engine/nerdctl/handlers/main.yml @@ -0,0 +1,12 @@ +--- +- name: Get nerdctl completion + command: "{{ bin_dir }}/nerdctl completion bash" + changed_when: False + register: nerdctl_completion + check_mode: false + +- name: Install nerdctl completion + copy: + dest: /etc/bash_completion.d/nerdctl + content: "{{ nerdctl_completion.stdout }}" + mode: 0644 diff --git a/kubespray/project/roles/container-engine/nerdctl/tasks/main.yml b/kubespray/project/roles/container-engine/nerdctl/tasks/main.yml new file mode 100644 index 0000000..e4e4ebd --- /dev/null +++ b/kubespray/project/roles/container-engine/nerdctl/tasks/main.yml @@ -0,0 +1,36 @@ +--- +- name: Nerdctl | Download nerdctl + include_tasks: "../../../download/tasks/download_file.yml" + vars: + download: "{{ download_defaults | combine(downloads.nerdctl) }}" + +- name: Nerdctl | Copy nerdctl binary from download dir + copy: + src: "{{ local_release_dir }}/nerdctl" + dest: "{{ bin_dir }}/nerdctl" + mode: 0755 + remote_src: true + owner: root + group: root + become: true + notify: + - Get nerdctl completion + - Install nerdctl completion + +- name: Nerdctl | Create configuration dir + file: + path: /etc/nerdctl + state: directory + mode: 0755 + owner: root + group: root + become: true + +- name: Nerdctl | Install nerdctl configuration + template: + src: nerdctl.toml.j2 + dest: /etc/nerdctl/nerdctl.toml + mode: 0644 + owner: root + group: root + become: true diff --git a/kubespray/project/roles/container-engine/nerdctl/templates/nerdctl.toml.j2 b/kubespray/project/roles/container-engine/nerdctl/templates/nerdctl.toml.j2 new file mode 100644 index 0000000..8b590f6 --- /dev/null +++ b/kubespray/project/roles/container-engine/nerdctl/templates/nerdctl.toml.j2 @@ -0,0 +1,9 @@ +debug = false +debug_full = false +address = "{{ cri_socket }}" +namespace = "k8s.io" +snapshotter = "{{ nerdctl_snapshotter | default('overlayfs') }}" +cni_path = "/opt/cni/bin" +cni_netconfpath = "/etc/cni/net.d" +cgroup_manager = "{{ kubelet_cgroup_driver | default('systemd') }}" +hosts_dir = ["{{ containerd_cfg_dir }}/certs.d"] diff --git a/kubespray/project/roles/container-engine/runc/defaults/main.yml b/kubespray/project/roles/container-engine/runc/defaults/main.yml new file mode 100644 index 0000000..af8aa08 --- /dev/null +++ b/kubespray/project/roles/container-engine/runc/defaults/main.yml @@ -0,0 +1,5 @@ +--- + +runc_bin_dir: "{{ bin_dir }}" + +runc_package_name: runc diff --git a/kubespray/project/roles/container-engine/runc/tasks/main.yml b/kubespray/project/roles/container-engine/runc/tasks/main.yml new file mode 100644 index 0000000..542a447 --- /dev/null +++ b/kubespray/project/roles/container-engine/runc/tasks/main.yml @@ -0,0 +1,38 @@ +--- +- name: Runc | check if fedora coreos + stat: + path: /run/ostree-booted + get_attributes: no + get_checksum: no + get_mime: no + register: ostree + +- name: Runc | set is_ostree + set_fact: + is_ostree: "{{ ostree.stat.exists }}" + +- name: Runc | Uninstall runc package managed by package manager + package: + name: "{{ runc_package_name }}" + state: absent + when: + - not (is_ostree or (ansible_distribution == "Flatcar Container Linux by Kinvolk") or (ansible_distribution == "Flatcar")) + +- name: Runc | Download runc binary + include_tasks: "../../../download/tasks/download_file.yml" + vars: + download: "{{ download_defaults | combine(downloads.runc) }}" + +- name: Copy runc binary from download dir + copy: + src: "{{ downloads.runc.dest }}" + dest: "{{ runc_bin_dir }}/runc" + mode: 0755 + remote_src: true + +- name: Runc | Remove orphaned binary + file: + path: /usr/bin/runc + state: absent + when: runc_bin_dir != "/usr/bin" + ignore_errors: true # noqa ignore-errors diff --git a/kubespray/project/roles/container-engine/skopeo/tasks/main.yml b/kubespray/project/roles/container-engine/skopeo/tasks/main.yml new file mode 100644 index 0000000..cef0424 --- /dev/null +++ b/kubespray/project/roles/container-engine/skopeo/tasks/main.yml @@ -0,0 +1,32 @@ +--- +- name: Skopeo | check if fedora coreos + stat: + path: /run/ostree-booted + get_attributes: no + get_checksum: no + get_mime: no + register: ostree + +- name: Skopeo | set is_ostree + set_fact: + is_ostree: "{{ ostree.stat.exists }}" + +- name: Skopeo | Uninstall skopeo package managed by package manager + package: + name: skopeo + state: absent + when: + - not (is_ostree or (ansible_distribution == "Flatcar Container Linux by Kinvolk") or (ansible_distribution == "Flatcar")) + ignore_errors: true # noqa ignore-errors + +- name: Skopeo | Download skopeo binary + include_tasks: "../../../download/tasks/download_file.yml" + vars: + download: "{{ download_defaults | combine(downloads.skopeo) }}" + +- name: Copy skopeo binary from download dir + copy: + src: "{{ downloads.skopeo.dest }}" + dest: "{{ bin_dir }}/skopeo" + mode: 0755 + remote_src: true diff --git a/kubespray/project/roles/container-engine/validate-container-engine/tasks/main.yml b/kubespray/project/roles/container-engine/validate-container-engine/tasks/main.yml new file mode 100644 index 0000000..08ea1e5 --- /dev/null +++ b/kubespray/project/roles/container-engine/validate-container-engine/tasks/main.yml @@ -0,0 +1,153 @@ +--- +- name: Validate-container-engine | check if fedora coreos + stat: + path: /run/ostree-booted + get_attributes: no + get_checksum: no + get_mime: no + register: ostree + tags: + - facts + +- name: Validate-container-engine | set is_ostree + set_fact: + is_ostree: "{{ ostree.stat.exists }}" + tags: + - facts + +- name: Ensure kubelet systemd unit exists + stat: + path: "/etc/systemd/system/kubelet.service" + register: kubelet_systemd_unit_exists + tags: + - facts + +- name: Populate service facts + service_facts: + tags: + - facts + +- name: Check if containerd is installed + find: + file_type: file + recurse: yes + use_regex: yes + patterns: + - containerd.service$ + paths: + - /lib/systemd + - /etc/systemd + - /run/systemd + register: containerd_installed + tags: + - facts + +- name: Check if docker is installed + find: + file_type: file + recurse: yes + use_regex: yes + patterns: + - docker.service$ + paths: + - /lib/systemd + - /etc/systemd + - /run/systemd + register: docker_installed + tags: + - facts + +- name: Check if crio is installed + find: + file_type: file + recurse: yes + use_regex: yes + patterns: + - crio.service$ + paths: + - /lib/systemd + - /etc/systemd + - /run/systemd + register: crio_installed + tags: + - facts + +- name: Uninstall containerd + vars: + service_name: containerd.service + when: + - not (is_ostree or (ansible_distribution == "Flatcar Container Linux by Kinvolk") or (ansible_distribution == "Flatcar")) + - container_manager != "containerd" + - docker_installed.matched == 0 + - containerd_installed.matched > 0 + - ansible_facts.services[service_name]['state'] == 'running' + block: + - name: Drain node + include_role: + name: remove-node/pre-remove + apply: + tags: + - pre-remove + when: kubelet_systemd_unit_exists.stat.exists + - name: Stop kubelet + service: + name: kubelet + state: stopped + when: kubelet_systemd_unit_exists.stat.exists + - name: Remove Containerd + import_role: + name: container-engine/containerd + tasks_from: reset + handlers_from: reset + +- name: Uninstall docker + vars: + service_name: docker.service + when: + - not (is_ostree or (ansible_distribution == "Flatcar Container Linux by Kinvolk") or (ansible_distribution == "Flatcar")) + - container_manager != "docker" + - docker_installed.matched > 0 + - ansible_facts.services[service_name]['state'] == 'running' + block: + - name: Drain node + include_role: + name: remove-node/pre-remove + apply: + tags: + - pre-remove + when: kubelet_systemd_unit_exists.stat.exists + - name: Stop kubelet + service: + name: kubelet + state: stopped + when: kubelet_systemd_unit_exists.stat.exists + - name: Remove Docker + import_role: + name: container-engine/docker + tasks_from: reset + +- name: Uninstall crio + vars: + service_name: crio.service + when: + - not (is_ostree or (ansible_distribution == "Flatcar Container Linux by Kinvolk") or (ansible_distribution == "Flatcar")) + - container_manager != "crio" + - crio_installed.matched > 0 + - ansible_facts.services[service_name]['state'] == 'running' + block: + - name: Drain node + include_role: + name: remove-node/pre-remove + apply: + tags: + - pre-remove + when: kubelet_systemd_unit_exists.stat.exists + - name: Stop kubelet + service: + name: kubelet + state: stopped + when: kubelet_systemd_unit_exists.stat.exists + - name: Remove CRI-O + import_role: + name: container-engine/cri-o + tasks_from: reset diff --git a/kubespray/project/roles/container-engine/youki/defaults/main.yml b/kubespray/project/roles/container-engine/youki/defaults/main.yml new file mode 100644 index 0000000..2250f22 --- /dev/null +++ b/kubespray/project/roles/container-engine/youki/defaults/main.yml @@ -0,0 +1,3 @@ +--- + +youki_bin_dir: "{{ bin_dir }}" diff --git a/kubespray/project/roles/container-engine/youki/molecule/default/converge.yml b/kubespray/project/roles/container-engine/youki/molecule/default/converge.yml new file mode 100644 index 0000000..11ef8f6 --- /dev/null +++ b/kubespray/project/roles/container-engine/youki/molecule/default/converge.yml @@ -0,0 +1,11 @@ +--- +- name: Converge + hosts: all + become: true + vars: + youki_enabled: true + container_manager: crio + roles: + - role: kubespray-defaults + - role: container-engine/cri-o + - role: container-engine/youki diff --git a/kubespray/project/roles/container-engine/youki/molecule/default/files/10-mynet.conf b/kubespray/project/roles/container-engine/youki/molecule/default/files/10-mynet.conf new file mode 100644 index 0000000..b9fa3ba --- /dev/null +++ b/kubespray/project/roles/container-engine/youki/molecule/default/files/10-mynet.conf @@ -0,0 +1,17 @@ +{ + "cniVersion": "0.4.0", + "name": "mynet", + "type": "bridge", + "bridge": "cni0", + "isGateway": true, + "ipMasq": true, + "ipam": { + "type": "host-local", + "subnet": "172.19.0.0/24", + "routes": [ + { + "dst": "0.0.0.0/0" + } + ] + } +} diff --git a/kubespray/project/roles/container-engine/youki/molecule/default/files/container.json b/kubespray/project/roles/container-engine/youki/molecule/default/files/container.json new file mode 100644 index 0000000..a5d5094 --- /dev/null +++ b/kubespray/project/roles/container-engine/youki/molecule/default/files/container.json @@ -0,0 +1,10 @@ +{ + "metadata": { + "name": "youki1" + }, + "image": { + "image": "quay.io/kubespray/hello-world:latest" + }, + "log_path": "youki1.0.log", + "linux": {} +} diff --git a/kubespray/project/roles/container-engine/youki/molecule/default/files/sandbox.json b/kubespray/project/roles/container-engine/youki/molecule/default/files/sandbox.json new file mode 100644 index 0000000..b2a4ffe --- /dev/null +++ b/kubespray/project/roles/container-engine/youki/molecule/default/files/sandbox.json @@ -0,0 +1,10 @@ +{ + "metadata": { + "name": "youki1", + "namespace": "default", + "attempt": 1, + "uid": "hdishd83djaidwnduwk28bcsb" + }, + "linux": {}, + "log_directory": "/tmp" +} diff --git a/kubespray/project/roles/container-engine/youki/molecule/default/molecule.yml b/kubespray/project/roles/container-engine/youki/molecule/default/molecule.yml new file mode 100644 index 0000000..9ba1927 --- /dev/null +++ b/kubespray/project/roles/container-engine/youki/molecule/default/molecule.yml @@ -0,0 +1,39 @@ +--- +role_name_check: 1 +driver: + name: vagrant + provider: + name: libvirt +platforms: + - name: ubuntu20 + box: generic/ubuntu2004 + cpus: 1 + memory: 1024 + nested: true + groups: + - kube_control_plane + provider_options: + driver: kvm + - name: almalinux8 + box: almalinux/8 + cpus: 1 + memory: 1024 + nested: true + groups: + - kube_control_plane + provider_options: + driver: kvm +provisioner: + name: ansible + env: + ANSIBLE_ROLES_PATH: ../../../../ + config_options: + defaults: + callbacks_enabled: profile_tasks + timeout: 120 + inventory: + group_vars: + all: + become: true +verifier: + name: testinfra diff --git a/kubespray/project/roles/container-engine/youki/molecule/default/prepare.yml b/kubespray/project/roles/container-engine/youki/molecule/default/prepare.yml new file mode 100644 index 0000000..119f58a --- /dev/null +++ b/kubespray/project/roles/container-engine/youki/molecule/default/prepare.yml @@ -0,0 +1,49 @@ +--- +- name: Prepare generic + hosts: all + become: true + roles: + - role: kubespray-defaults + - role: bootstrap-os + - role: adduser + user: "{{ addusers.kube }}" + tasks: + - name: Download CNI + include_tasks: "../../../../download/tasks/download_file.yml" + vars: + download: "{{ download_defaults | combine(downloads.cni) }}" + +- name: Prepare container runtime + hosts: all + become: true + vars: + container_manager: crio + kube_network_plugin: cni + roles: + - role: kubespray-defaults + - role: network_plugin/cni + - role: container-engine/crictl + tasks: + - name: Copy test container files + copy: + src: "{{ item }}" + dest: "/tmp/{{ item }}" + owner: root + mode: 0644 + with_items: + - container.json + - sandbox.json + - name: Create /etc/cni/net.d directory + file: + path: /etc/cni/net.d + state: directory + owner: root + mode: 0755 + - name: Setup CNI + copy: + src: "{{ item }}" + dest: "/etc/cni/net.d/{{ item }}" + owner: root + mode: 0644 + with_items: + - 10-mynet.conf diff --git a/kubespray/project/roles/container-engine/youki/molecule/default/tests/test_default.py b/kubespray/project/roles/container-engine/youki/molecule/default/tests/test_default.py new file mode 100644 index 0000000..54ed5c5 --- /dev/null +++ b/kubespray/project/roles/container-engine/youki/molecule/default/tests/test_default.py @@ -0,0 +1,29 @@ +import os + +import testinfra.utils.ansible_runner + +testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( + os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') + + +def test_run(host): + youkiruntime = "/usr/local/bin/youki" + with host.sudo(): + cmd = host.command(youkiruntime + " --version") + assert cmd.rc == 0 + assert "youki" in cmd.stdout + + +def test_run_pod(host): + runtime = "youki" + + run_command = "/usr/local/bin/crictl run --with-pull --runtime {} /tmp/container.json /tmp/sandbox.json".format(runtime) + with host.sudo(): + cmd = host.command(run_command) + assert cmd.rc == 0 + + with host.sudo(): + log_f = host.file("/tmp/youki1.0.log") + + assert log_f.exists + assert b"Hello from Docker" in log_f.content diff --git a/kubespray/project/roles/container-engine/youki/tasks/main.yml b/kubespray/project/roles/container-engine/youki/tasks/main.yml new file mode 100644 index 0000000..e88f663 --- /dev/null +++ b/kubespray/project/roles/container-engine/youki/tasks/main.yml @@ -0,0 +1,12 @@ +--- +- name: Youki | Download youki + include_tasks: "../../../download/tasks/download_file.yml" + vars: + download: "{{ download_defaults | combine(downloads.youki) }}" + +- name: Youki | Copy youki binary from download dir + copy: + src: "{{ local_release_dir }}/youki_{{ youki_version | regex_replace('\\.', '_') }}_linux/youki-{{ youki_version }}/youki" + dest: "{{ youki_bin_dir }}/youki" + mode: 0755 + remote_src: true diff --git a/kubespray/project/roles/download/meta/main.yml b/kubespray/project/roles/download/meta/main.yml new file mode 100644 index 0000000..61d3ffe --- /dev/null +++ b/kubespray/project/roles/download/meta/main.yml @@ -0,0 +1,2 @@ +--- +allow_duplicates: true diff --git a/kubespray/project/roles/download/tasks/check_pull_required.yml b/kubespray/project/roles/download/tasks/check_pull_required.yml new file mode 100644 index 0000000..e5ae1dc --- /dev/null +++ b/kubespray/project/roles/download/tasks/check_pull_required.yml @@ -0,0 +1,25 @@ +--- +# The image_info_command depends on the Container Runtime and will output something like the following: +# nginx:1.15,gcr.io/google-containers/kube-proxy:v1.14.1,gcr.io/google-containers/kube-proxy@sha256:44af2833c6cbd9a7fc2e9d2f5244a39dfd2e31ad91bf9d4b7d810678db738ee9,gcr.io/google-containers/kube-apiserver:v1.14.1,etc... +- name: Check_pull_required | Generate a list of information about the images on a node # noqa command-instead-of-shell - image_info_command contains a pipe, therefore requiring shell + shell: "{{ image_info_command }}" + register: docker_images + changed_when: false + check_mode: no + when: not download_always_pull + +- name: Check_pull_required | Set pull_required if the desired image is not yet loaded + set_fact: + pull_required: >- + {%- if image_reponame | regex_replace('^docker\.io/(library/)?', '') in docker_images.stdout.split(',') %}false{%- else -%}true{%- endif -%} + when: not download_always_pull + +- name: Check_pull_required | Check that the local digest sha256 corresponds to the given image tag + assert: + that: "{{ download.repo }}:{{ download.tag }} in docker_images.stdout.split(',')" + when: + - not download_always_pull + - not pull_required + - pull_by_digest + tags: + - asserts diff --git a/kubespray/project/roles/download/tasks/download_container.yml b/kubespray/project/roles/download/tasks/download_container.yml new file mode 100644 index 0000000..f98adfa --- /dev/null +++ b/kubespray/project/roles/download/tasks/download_container.yml @@ -0,0 +1,125 @@ +--- +- tags: + - download + block: + - name: Set default values for flag variables + set_fact: + image_is_cached: false + image_changed: false + pull_required: "{{ download_always_pull }}" + tags: + - facts + + - name: Download_container | Set a few facts + import_tasks: set_container_facts.yml + tags: + - facts + + - name: Download_container | Prepare container download + include_tasks: check_pull_required.yml + when: + - not download_always_pull + + - debug: # noqa name[missing] + msg: "Pull {{ image_reponame }} required is: {{ pull_required }}" + + - name: Download_container | Determine if image is in cache + stat: + path: "{{ image_path_cached }}" + get_attributes: no + get_checksum: no + get_mime: no + delegate_to: localhost + connection: local + delegate_facts: no + register: cache_image + changed_when: false + become: false + when: + - download_force_cache + + - name: Download_container | Set fact indicating if image is in cache + set_fact: + image_is_cached: "{{ cache_image.stat.exists }}" + tags: + - facts + when: + - download_force_cache + + - name: Stop if image not in cache on ansible host when download_force_cache=true + assert: + that: image_is_cached + msg: "Image cache file {{ image_path_cached }} not found for {{ image_reponame }} on localhost" + when: + - download_force_cache + - not download_run_once + + - name: Download_container | Download image if required + command: "{{ image_pull_command_on_localhost if download_localhost else image_pull_command }} {{ image_reponame }}" + delegate_to: "{{ download_delegate if download_run_once else inventory_hostname }}" + delegate_facts: yes + run_once: "{{ download_run_once }}" + register: pull_task_result + until: pull_task_result is succeeded + delay: "{{ retry_stagger | random + 3 }}" + retries: "{{ download_retries }}" + become: "{{ user_can_become_root | default(false) or not download_localhost }}" + environment: "{{ proxy_env if container_manager == 'containerd' else omit }}" + when: + - pull_required or download_run_once + - not image_is_cached + + - name: Download_container | Save and compress image + shell: "{{ image_save_command_on_localhost if download_localhost else image_save_command }}" # noqa command-instead-of-shell - image_save_command_on_localhost contains a pipe, therefore requires shell + delegate_to: "{{ download_delegate }}" + delegate_facts: no + register: container_save_status + failed_when: container_save_status.stderr + run_once: true + become: "{{ user_can_become_root | default(false) or not download_localhost }}" + when: + - not image_is_cached + - download_run_once + + - name: Download_container | Copy image to ansible host cache + ansible.posix.synchronize: + src: "{{ image_path_final }}" + dest: "{{ image_path_cached }}" + use_ssh_args: true + mode: pull + when: + - not image_is_cached + - download_run_once + - not download_localhost + - download_delegate == inventory_hostname + + - name: Download_container | Upload image to node if it is cached + ansible.posix.synchronize: + src: "{{ image_path_cached }}" + dest: "{{ image_path_final }}" + use_ssh_args: true + mode: push + delegate_facts: no + register: upload_image + failed_when: not upload_image + until: upload_image is succeeded + retries: "{{ download_retries }}" + delay: "{{ retry_stagger | random + 3 }}" + when: + - pull_required + - download_force_cache + + - name: Download_container | Load image into the local container registry + shell: "{{ image_load_command }}" # noqa command-instead-of-shell - image_load_command uses pipes, therefore requires shell + register: container_load_status + failed_when: container_load_status is failed + when: + - pull_required + - download_force_cache + + - name: Download_container | Remove container image from cache + file: + state: absent + path: "{{ image_path_final }}" + when: + - not download_keep_remote_cache diff --git a/kubespray/project/roles/download/tasks/download_file.yml b/kubespray/project/roles/download/tasks/download_file.yml new file mode 100644 index 0000000..9773366 --- /dev/null +++ b/kubespray/project/roles/download/tasks/download_file.yml @@ -0,0 +1,115 @@ +--- +- name: "Download_file | download {{ download.dest }}" + tags: + - download + block: + - name: Prep_download | Set a few facts + set_fact: + download_force_cache: "{{ true if download_run_once else download_force_cache }}" + + - name: Download_file | Show url of file to dowload + debug: + msg: "{{ download.url }}" + run_once: "{{ download_run_once }}" + + - name: Download_file | Set pathname of cached file + set_fact: + file_path_cached: "{{ download_cache_dir }}/{{ download.dest | basename }}" + tags: + - facts + + - name: Download_file | Create dest directory on node + file: + path: "{{ download.dest | dirname }}" + owner: "{{ download.owner | default(omit) }}" + mode: 0755 + state: directory + recurse: yes + + - name: Download_file | Create local cache directory + file: + path: "{{ file_path_cached | dirname }}" + state: directory + recurse: yes + delegate_to: localhost + connection: local + delegate_facts: false + run_once: true + become: false + when: + - download_force_cache + tags: + - localhost + + - name: Download_file | Create cache directory on download_delegate host + file: + path: "{{ file_path_cached | dirname }}" + state: directory + recurse: yes + delegate_to: "{{ download_delegate }}" + delegate_facts: false + run_once: true + when: + - download_force_cache + - not download_localhost + + # This must always be called, to check if the checksum matches. On no-match the file is re-downloaded. + # This task will avoid logging it's parameters to not leak environment passwords in the log + - name: Download_file | Download item + get_url: + url: "{{ download.url }}" + dest: "{{ file_path_cached if download_force_cache else download.dest }}" + owner: "{{ omit if download_localhost else (download.owner | default(omit)) }}" + mode: "{{ omit if download_localhost else (download.mode | default(omit)) }}" + checksum: "{{ 'sha256:' + download.sha256 if download.sha256 else omit }}" + validate_certs: "{{ download_validate_certs }}" + url_username: "{{ download.username | default(omit) }}" + url_password: "{{ download.password | default(omit) }}" + force_basic_auth: "{{ download.force_basic_auth | default(omit) }}" + timeout: "{{ download.timeout | default(omit) }}" + delegate_to: "{{ download_delegate if download_force_cache else inventory_hostname }}" + run_once: "{{ download_force_cache }}" + register: get_url_result + become: "{{ not download_localhost }}" + until: "'OK' in get_url_result.msg or + 'file already exists' in get_url_result.msg or + get_url_result.status_code | default() == 304" + retries: "{{ download_retries }}" + delay: "{{ retry_stagger | default(5) }}" + environment: "{{ proxy_env }}" + no_log: "{{ not (unsafe_show_logs | bool) }}" + + - name: Download_file | Copy file back to ansible host file cache + ansible.posix.synchronize: + src: "{{ file_path_cached }}" + dest: "{{ file_path_cached }}" + use_ssh_args: true + mode: pull + when: + - download_force_cache + - not download_localhost + - download_delegate == inventory_hostname + + - name: Download_file | Copy file from cache to nodes, if it is available + ansible.posix.synchronize: + src: "{{ file_path_cached }}" + dest: "{{ download.dest }}" + use_ssh_args: true + mode: push + register: get_task + until: get_task is succeeded + delay: "{{ retry_stagger | random + 3 }}" + retries: "{{ download_retries }}" + when: + - download_force_cache + + - name: Download_file | Set mode and owner + file: + path: "{{ download.dest }}" + mode: "{{ download.mode | default(omit) }}" + owner: "{{ download.owner | default(omit) }}" + when: + - download_force_cache + + - name: "Download_file | Extract file archives" + include_tasks: "extract_file.yml" diff --git a/kubespray/project/roles/download/tasks/extract_file.yml b/kubespray/project/roles/download/tasks/extract_file.yml new file mode 100644 index 0000000..59d0531 --- /dev/null +++ b/kubespray/project/roles/download/tasks/extract_file.yml @@ -0,0 +1,11 @@ +--- +- name: Extract_file | Unpacking archive + unarchive: + src: "{{ download.dest }}" + dest: "{{ download.dest | dirname }}" + owner: "{{ download.owner | default(omit) }}" + mode: "{{ download.mode | default(omit) }}" + copy: no + extra_opts: "{{ download.unarchive_extra_opts | default(omit) }}" + when: + - download.unarchive | default(false) diff --git a/kubespray/project/roles/download/tasks/main.yml b/kubespray/project/roles/download/tasks/main.yml new file mode 100644 index 0000000..fe83e6c --- /dev/null +++ b/kubespray/project/roles/download/tasks/main.yml @@ -0,0 +1,66 @@ +--- +- name: Download | Prepare working directories and variables + import_tasks: prep_download.yml + when: + - not skip_downloads + tags: + - download + - upload + +- name: Download | Get kubeadm binary and list of required images + include_tasks: prep_kubeadm_images.yml + when: + - not skip_downloads + - inventory_hostname in groups['kube_control_plane'] + tags: + - download + - upload + +- name: Download | Download files / images + include_tasks: "{{ include_file }}" + loop: "{{ downloads | combine(kubeadm_images) | dict2items }}" + vars: + download: "{{ download_defaults | combine(item.value) }}" + include_file: "download_{% if download.container %}container{% else %}file{% endif %}.yml" + kubeadm_images: "{{ skip_kubeadm_images | ternary({}, _kubeadm_images) }}" + # The trick (converting list of tuples to list of dicts) below come from + # https://docs.ansible.com/ansible/latest/collections/community/general/dict_filter.html#examples + _kubeadm_images: "{{ dict(names | map('regex_replace', '^(.*)', 'kubeadm_\\1') | + zip( repos | zip(_tags, _groups) | + map('zip', keys) | map('map', 'reverse') | map('community.general.dict') | + map('combine', defaults))) | + dict2items | rejectattr('key', 'in', excluded) | items2dict }}" + keys: + - repo + - tag + - groups + images: "{{ kubeadm_images_raw.stdout_lines | map('split', ':') }}" + _tags: "{{ images | map(attribute=1) }}" + repos: "{{ images | map(attribute=0) }}" + names: "{{ repos | map('split', '/') | map(attribute=-1) }}" + _groups: "{{ names | map('extract', images_groups) }}" + defaults: + enabled: true + container: true + excluded: + - kubeadm_coredns + - kubeadm_pause + images_groups: + coredns: [] + pause: [] + kube-proxy: + - k8s_cluster + etcd: + - etcd + kube-scheduler: + - kube_control_plane + kube-controller-manager: + - kube_control_plane + kube-apiserver: + - kube_control_plane + when: + - not skip_downloads + - download.enabled + - item.value.enabled + - (not (item.value.container | default(false))) or (item.value.container and download_container) + - (download_run_once and inventory_hostname == download_delegate) or (group_names | intersect(download.groups) | length) diff --git a/kubespray/project/roles/download/tasks/prep_download.yml b/kubespray/project/roles/download/tasks/prep_download.yml new file mode 100644 index 0000000..58e5084 --- /dev/null +++ b/kubespray/project/roles/download/tasks/prep_download.yml @@ -0,0 +1,90 @@ +--- +- name: Prep_download | Set a few facts + set_fact: + download_force_cache: "{{ true if download_run_once else download_force_cache }}" + tags: + - facts + +- name: Prep_download | On localhost, check if passwordless root is possible + command: "true" + delegate_to: localhost + connection: local + run_once: true + register: test_become + changed_when: false + ignore_errors: true # noqa ignore-errors + become: true + when: + - download_localhost + tags: + - localhost + - asserts + +- name: Prep_download | On localhost, check if user has access to the container runtime without using sudo + shell: "{{ image_info_command_on_localhost }}" # noqa command-instead-of-shell - image_info_command_on_localhost contains pipe, therefore requires shell + delegate_to: localhost + connection: local + run_once: true + register: test_docker + changed_when: false + ignore_errors: true # noqa ignore-errors + become: false + when: + - download_localhost + tags: + - localhost + - asserts + +- name: Prep_download | Parse the outputs of the previous commands + set_fact: + user_in_docker_group: "{{ not test_docker.failed }}" + user_can_become_root: "{{ not test_become.failed }}" + when: + - download_localhost + tags: + - localhost + - asserts + +- name: Prep_download | Check that local user is in group or can become root + assert: + that: "user_in_docker_group or user_can_become_root" + msg: >- + Error: User is not in docker group and cannot become root. When download_localhost is true, at least one of these two conditions must be met. + when: + - download_localhost + tags: + - localhost + - asserts + +- name: Prep_download | Register docker images info + shell: "{{ image_info_command }}" # noqa command-instead-of-shell - image_info_command contains pipe therefore requires shell + no_log: "{{ not (unsafe_show_logs | bool) }}" + register: docker_images + failed_when: false + changed_when: false + check_mode: no + when: download_container + +- name: Prep_download | Create staging directory on remote node + file: + path: "{{ local_release_dir }}/images" + state: directory + mode: 0755 + owner: "{{ ansible_ssh_user | default(ansible_user_id) }}" + when: + - ansible_os_family not in ["Flatcar", "Flatcar Container Linux by Kinvolk"] + +- name: Prep_download | Create local cache for files and images on control node + file: + path: "{{ download_cache_dir }}/images" + state: directory + mode: 0755 + delegate_to: localhost + connection: local + delegate_facts: no + run_once: true + become: false + when: + - download_force_cache + tags: + - localhost diff --git a/kubespray/project/roles/download/tasks/prep_kubeadm_images.yml b/kubespray/project/roles/download/tasks/prep_kubeadm_images.yml new file mode 100644 index 0000000..67ac2f7 --- /dev/null +++ b/kubespray/project/roles/download/tasks/prep_kubeadm_images.yml @@ -0,0 +1,44 @@ +--- +- name: Prep_kubeadm_images | Check kubeadm version matches kubernetes version + fail: + msg: "Kubeadm version {{ kubeadm_version }} do not matches kubernetes {{ kube_version }}" + when: + - not skip_downloads | default(false) + - not kubeadm_version == downloads.kubeadm.version + +- name: Prep_kubeadm_images | Download kubeadm binary + include_tasks: "download_file.yml" + vars: + download: "{{ download_defaults | combine(downloads.kubeadm) }}" + when: + - not skip_downloads | default(false) + - downloads.kubeadm.enabled + +- name: Prep_kubeadm_images | Create kubeadm config + template: + src: "kubeadm-images.yaml.j2" + dest: "{{ kube_config_dir }}/kubeadm-images.yaml" + mode: 0644 + when: + - not skip_kubeadm_images + +- name: Prep_kubeadm_images | Copy kubeadm binary from download dir to system path + copy: + src: "{{ downloads.kubeadm.dest }}" + dest: "{{ bin_dir }}/kubeadm" + mode: 0755 + remote_src: true + +- name: Prep_kubeadm_images | Set kubeadm binary permissions + file: + path: "{{ bin_dir }}/kubeadm" + mode: "0755" + state: file + +- name: Prep_kubeadm_images | Generate list of required images + command: "{{ bin_dir }}/kubeadm config images list --config={{ kube_config_dir }}/kubeadm-images.yaml" + register: kubeadm_images_raw + run_once: true + changed_when: false + when: + - not skip_kubeadm_images diff --git a/kubespray/project/roles/download/tasks/set_container_facts.yml b/kubespray/project/roles/download/tasks/set_container_facts.yml new file mode 100644 index 0000000..5b93f29 --- /dev/null +++ b/kubespray/project/roles/download/tasks/set_container_facts.yml @@ -0,0 +1,55 @@ +--- +- name: Set_container_facts | Display the name of the image being processed + debug: + msg: "{{ download.repo }}" + +- name: Set_container_facts | Set if containers should be pulled by digest + set_fact: + pull_by_digest: "{{ download.sha256 is defined and download.sha256 }}" + +- name: Set_container_facts | Define by what name to pull the image + set_fact: + image_reponame: >- + {%- if pull_by_digest %}{{ download.repo }}@sha256:{{ download.sha256 }}{%- else -%}{{ download.repo }}:{{ download.tag }}{%- endif -%} + +- name: Set_container_facts | Define file name of image + set_fact: + image_filename: "{{ image_reponame | regex_replace('/|\0|:', '_') }}.tar" + +- name: Set_container_facts | Define path of image + set_fact: + image_path_cached: "{{ download_cache_dir }}/images/{{ image_filename }}" + image_path_final: "{{ local_release_dir }}/images/{{ image_filename }}" + +- name: Set image save/load command for docker + set_fact: + image_save_command: "{{ docker_bin_dir }}/docker save {{ image_reponame }} | gzip -{{ download_compress }} > {{ image_path_final }}" + image_load_command: "{{ docker_bin_dir }}/docker load < {{ image_path_final }}" + when: container_manager == 'docker' + +- name: Set image save/load command for containerd + set_fact: + image_save_command: "{{ bin_dir }}/nerdctl -n k8s.io image save -o {{ image_path_final }} {{ image_reponame }}" + image_load_command: "{{ bin_dir }}/nerdctl -n k8s.io image load < {{ image_path_final }}" + when: container_manager == 'containerd' + +- name: Set image save/load command for crio + set_fact: + image_save_command: "{{ bin_dir }}/skopeo copy containers-storage:{{ image_reponame }} docker-archive:{{ image_path_final }} 2>/dev/null" + image_load_command: "{{ bin_dir }}/skopeo copy docker-archive:{{ image_path_final }} containers-storage:{{ image_reponame }} 2>/dev/null" + when: container_manager == 'crio' + +- name: Set image save/load command for docker on localhost + set_fact: + image_save_command_on_localhost: "{{ docker_bin_dir }}/docker save {{ image_reponame }} | gzip -{{ download_compress }} > {{ image_path_cached }}" + when: container_manager_on_localhost == 'docker' + +- name: Set image save/load command for containerd on localhost + set_fact: + image_save_command_on_localhost: "{{ containerd_bin_dir }}/ctr -n k8s.io image export --platform linux/{{ image_arch }} {{ image_path_cached }} {{ image_reponame }}" + when: container_manager_on_localhost == 'containerd' + +- name: Set image save/load command for crio on localhost + set_fact: + image_save_command_on_localhost: "{{ bin_dir }}/skopeo copy containers-storage:{{ image_reponame }} docker-archive:{{ image_path_final }} 2>/dev/null" + when: container_manager_on_localhost == 'crio' diff --git a/kubespray/project/roles/download/templates/kubeadm-images.yaml.j2 b/kubespray/project/roles/download/templates/kubeadm-images.yaml.j2 new file mode 100644 index 0000000..452e164 --- /dev/null +++ b/kubespray/project/roles/download/templates/kubeadm-images.yaml.j2 @@ -0,0 +1,24 @@ +apiVersion: kubeadm.k8s.io/v1beta3 +kind: InitConfiguration +nodeRegistration: + criSocket: {{ cri_socket }} +--- +apiVersion: kubeadm.k8s.io/v1beta3 +kind: ClusterConfiguration +imageRepository: {{ kube_image_repo }} +kubernetesVersion: {{ kube_version }} +etcd: +{% if etcd_deployment_type == "kubeadm" %} + local: + imageRepository: "{{ etcd_image_repo | regex_replace("/etcd$","") }}" + imageTag: "{{ etcd_image_tag }}" +{% else %} + external: + endpoints: +{% for endpoint in etcd_access_addresses.split(',') %} + - {{ endpoint }} +{% endfor %} +{% endif %} +dns: + imageRepository: {{ coredns_image_repo | regex_replace('/coredns(?!/coredns).*$', '') }} + imageTag: {{ coredns_image_tag }} diff --git a/kubespray/project/roles/etcd/defaults/main.yml b/kubespray/project/roles/etcd/defaults/main.yml new file mode 100644 index 0000000..814caed --- /dev/null +++ b/kubespray/project/roles/etcd/defaults/main.yml @@ -0,0 +1,127 @@ +--- +# Set etcd user +etcd_owner: etcd + +# Set to false to only do certificate management +etcd_cluster_setup: true +etcd_events_cluster_setup: false + +# Set to true to separate k8s events to a different etcd cluster +etcd_events_cluster_enabled: false + +etcd_backup_prefix: "/var/backups" +etcd_data_dir: "/var/lib/etcd" + +# Number of etcd backups to retain. Set to a value < 0 to retain all backups +etcd_backup_retention_count: -1 + +force_etcd_cert_refresh: true +etcd_config_dir: /etc/ssl/etcd +etcd_cert_dir: "{{ etcd_config_dir }}/ssl" +etcd_cert_dir_mode: "0700" +etcd_cert_group: root +# Note: This does not set up DNS entries. It simply adds the following DNS +# entries to the certificate +etcd_cert_alt_names: + - "etcd.kube-system.svc.{{ dns_domain }}" + - "etcd.kube-system.svc" + - "etcd.kube-system" + - "etcd" +etcd_cert_alt_ips: [] + +etcd_script_dir: "{{ bin_dir }}/etcd-scripts" + +etcd_heartbeat_interval: "250" +etcd_election_timeout: "5000" + +# etcd_snapshot_count: "10000" + +etcd_metrics: "basic" + +# Define in inventory to set a separate port for etcd to expose metrics on +# etcd_metrics_port: 2381 + +## A dictionary of extra environment variables to add to etcd.env, formatted like: +## etcd_extra_vars: +## ETCD_VAR1: "value1" +## ETCD_VAR2: "value2" +etcd_extra_vars: {} + +# Limits +# Limit memory only if <4GB memory on host. 0=unlimited +# This value is only relevant when deploying etcd with `etcd_deployment_type: docker` +etcd_memory_limit: "{% if ansible_memtotal_mb < 4096 %}512M{% else %}0{% endif %}" + +# The default storage size limit is 2G. +# 8G is a suggested maximum size for normal environments and etcd warns at startup if the configured value exceeds it. +# etcd_quota_backend_bytes: "2147483648" + +# Maximum client request size in bytes the server will accept. +# etcd is designed to handle small key value pairs typical for metadata. +# Larger requests will work, but may increase the latency of other requests +# etcd_max_request_bytes: "1572864" + +# Uncomment to set CPU share for etcd +# etcd_cpu_limit: 300m + +etcd_blkio_weight: 1000 + +etcd_node_cert_hosts: "{{ groups['k8s_cluster'] }}" + +etcd_compaction_retention: "8" + +# Force clients like etcdctl to use TLS certs (different than peer security) +etcd_secure_client: true + +# Enable peer client cert authentication +etcd_peer_client_auth: true + +# Maximum number of snapshot files to retain (0 is unlimited) +# etcd_max_snapshots: 5 + +# Maximum number of wal files to retain (0 is unlimited) +# etcd_max_wals: 5 + +# Number of loop retries +etcd_retries: 4 + +## Support tls cipher suites. +# etcd_tls_cipher_suites: {} +# - TLS_RSA_WITH_RC4_128_SHA +# - TLS_RSA_WITH_3DES_EDE_CBC_SHA +# - TLS_RSA_WITH_AES_128_CBC_SHA +# - TLS_RSA_WITH_AES_256_CBC_SHA +# - TLS_RSA_WITH_AES_128_CBC_SHA256 +# - TLS_RSA_WITH_AES_128_GCM_SHA256 +# - TLS_RSA_WITH_AES_256_GCM_SHA384 +# - TLS_ECDHE_ECDSA_WITH_RC4_128_SHA +# - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA +# - TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA +# - TLS_ECDHE_RSA_WITH_RC4_128_SHA +# - TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA +# - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA +# - TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA +# - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256 +# - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 +# - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 +# - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 +# - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 +# - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 +# - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 +# - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256 + +# ETCD 3.5.x issue +# https://groups.google.com/a/kubernetes.io/g/dev/c/B7gJs88XtQc/m/rSgNOzV2BwAJ?utm_medium=email&utm_source=footer +etcd_experimental_initial_corrupt_check: true + +# If this is true, debug information will be displayed but +# may contain some private data, so it is recommended to set it to false +# in the production environment. +unsafe_show_logs: false + +# Enable distributed tracing +# https://etcd.io/docs/v3.5/op-guide/monitoring/#distributed-tracing +etcd_experimental_enable_distributed_tracing: false +etcd_experimental_distributed_tracing_sample_rate: 100 +etcd_experimental_distributed_tracing_address: "localhost:4317" +etcd_experimental_distributed_tracing_service_name: etcd \ No newline at end of file diff --git a/kubespray/project/roles/etcd/handlers/backup.yml b/kubespray/project/roles/etcd/handlers/backup.yml new file mode 100644 index 0000000..b79dd01 --- /dev/null +++ b/kubespray/project/roles/etcd/handlers/backup.yml @@ -0,0 +1,64 @@ +--- +- name: Refresh Time Fact + setup: + filter: ansible_date_time + listen: Restart etcd + when: etcd_cluster_is_healthy.rc == 0 + +- name: Set Backup Directory + set_fact: + etcd_backup_directory: "{{ etcd_backup_prefix }}/etcd-{{ ansible_date_time.date }}_{{ ansible_date_time.time }}" + listen: Restart etcd + +- name: Create Backup Directory + file: + path: "{{ etcd_backup_directory }}" + state: directory + owner: root + group: root + mode: 0600 + listen: Restart etcd + when: etcd_cluster_is_healthy.rc == 0 + +- name: Stat etcd v2 data directory + stat: + path: "{{ etcd_data_dir }}/member" + get_attributes: no + get_checksum: no + get_mime: no + register: etcd_data_dir_member + listen: Restart etcd + when: etcd_cluster_is_healthy.rc == 0 + +- name: Backup etcd v2 data + when: + - etcd_data_dir_member.stat.exists + - etcd_cluster_is_healthy.rc == 0 + command: >- + {{ bin_dir }}/etcdctl backup + --data-dir {{ etcd_data_dir }} + --backup-dir {{ etcd_backup_directory }} + environment: + ETCDCTL_API: "2" + retries: 3 + register: backup_v2_command + until: backup_v2_command.rc == 0 + delay: "{{ retry_stagger | random + 3 }}" + listen: Restart etcd + +- name: Backup etcd v3 data + command: >- + {{ bin_dir }}/etcdctl + snapshot save {{ etcd_backup_directory }}/snapshot.db + environment: + ETCDCTL_API: "3" + ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses.split(',') | first }}" + ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" + ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" + ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem" + retries: 3 + register: etcd_backup_v3_command + until: etcd_backup_v3_command.rc == 0 + delay: "{{ retry_stagger | random + 3 }}" + listen: Restart etcd + when: etcd_cluster_is_healthy.rc == 0 diff --git a/kubespray/project/roles/etcd/handlers/backup_cleanup.yml b/kubespray/project/roles/etcd/handlers/backup_cleanup.yml new file mode 100644 index 0000000..85b8d19 --- /dev/null +++ b/kubespray/project/roles/etcd/handlers/backup_cleanup.yml @@ -0,0 +1,18 @@ +--- +- name: Find old etcd backups + ansible.builtin.find: + file_type: directory + recurse: false + paths: "{{ etcd_backup_prefix }}" + patterns: "etcd-*" + register: _etcd_backups + when: etcd_backup_retention_count >= 0 + listen: Restart etcd + +- name: Remove old etcd backups + ansible.builtin.file: + state: absent + path: "{{ item }}" + loop: "{{ (_etcd_backups.files | sort(attribute='ctime', reverse=True))[etcd_backup_retention_count:] | map(attribute='path') }}" + when: etcd_backup_retention_count >= 0 + listen: Restart etcd diff --git a/kubespray/project/roles/etcd/handlers/main.yml b/kubespray/project/roles/etcd/handlers/main.yml new file mode 100644 index 0000000..3389061 --- /dev/null +++ b/kubespray/project/roles/etcd/handlers/main.yml @@ -0,0 +1,55 @@ +--- +- name: Backup etcd + import_tasks: backup.yml + +- name: Etcd | reload systemd + systemd: + daemon_reload: true + listen: + - Restart etcd + - Restart etcd-events + +- name: Reload etcd + service: + name: etcd + state: restarted + when: is_etcd_master + listen: Restart etcd + +- name: Reload etcd-events + service: + name: etcd-events + state: restarted + when: is_etcd_master + listen: Restart etcd-events + +- name: Wait for etcd up + uri: + url: "https://{% if is_etcd_master %}{{ etcd_address }}{% else %}127.0.0.1{% endif %}:2379/health" + validate_certs: no + client_cert: "{{ etcd_cert_dir }}/member-{{ inventory_hostname }}.pem" + client_key: "{{ etcd_cert_dir }}/member-{{ inventory_hostname }}-key.pem" + register: result + until: result.status is defined and result.status == 200 + retries: 60 + delay: 1 + listen: Restart etcd + +- name: Cleanup etcd backups + import_tasks: backup_cleanup.yml + +- name: Wait for etcd-events up + uri: + url: "https://{% if is_etcd_master %}{{ etcd_address }}{% else %}127.0.0.1{% endif %}:2383/health" + validate_certs: no + client_cert: "{{ etcd_cert_dir }}/member-{{ inventory_hostname }}.pem" + client_key: "{{ etcd_cert_dir }}/member-{{ inventory_hostname }}-key.pem" + register: result + until: result.status is defined and result.status == 200 + retries: 60 + delay: 1 + listen: Restart etcd-events + +- name: Set etcd_secret_changed + set_fact: + etcd_secret_changed: true diff --git a/kubespray/project/roles/etcd/meta/main.yml b/kubespray/project/roles/etcd/meta/main.yml new file mode 100644 index 0000000..e996646 --- /dev/null +++ b/kubespray/project/roles/etcd/meta/main.yml @@ -0,0 +1,8 @@ +--- +dependencies: + - role: adduser + user: "{{ addusers.etcd }}" + when: not (ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk", "ClearLinux"] or is_fedora_coreos) + - role: adduser + user: "{{ addusers.kube }}" + when: not (ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk", "ClearLinux"] or is_fedora_coreos) diff --git a/kubespray/project/roles/etcd/tasks/check_certs.yml b/kubespray/project/roles/etcd/tasks/check_certs.yml new file mode 100644 index 0000000..1611f9e --- /dev/null +++ b/kubespray/project/roles/etcd/tasks/check_certs.yml @@ -0,0 +1,134 @@ +--- +- name: "Check_certs | Register certs that have already been generated on first etcd node" + find: + paths: "{{ etcd_cert_dir }}" + patterns: "ca.pem,node*.pem,member*.pem,admin*.pem" + get_checksum: true + delegate_to: "{{ groups['etcd'][0] }}" + register: etcdcert_master + run_once: true + +- name: "Check_certs | Set default value for 'sync_certs', 'gen_certs' and 'etcd_secret_changed' to false" + set_fact: + sync_certs: false + gen_certs: false + etcd_secret_changed: false + +- name: "Check certs | Register ca and etcd admin/member certs on etcd hosts" + stat: + path: "{{ etcd_cert_dir }}/{{ item }}" + get_attributes: no + get_checksum: yes + get_mime: no + register: etcd_member_certs + when: inventory_hostname in groups['etcd'] + with_items: + - ca.pem + - member-{{ inventory_hostname }}.pem + - member-{{ inventory_hostname }}-key.pem + - admin-{{ inventory_hostname }}.pem + - admin-{{ inventory_hostname }}-key.pem + +- name: "Check certs | Register ca and etcd node certs on kubernetes hosts" + stat: + path: "{{ etcd_cert_dir }}/{{ item }}" + register: etcd_node_certs + when: inventory_hostname in groups['k8s_cluster'] + with_items: + - ca.pem + - node-{{ inventory_hostname }}.pem + - node-{{ inventory_hostname }}-key.pem + +- name: "Check_certs | Set 'gen_certs' to true if expected certificates are not on the first etcd node(1/2)" + set_fact: + gen_certs: true + when: force_etcd_cert_refresh or not item in etcdcert_master.files | map(attribute='path') | list + run_once: true + with_items: "{{ expected_files }}" + vars: + expected_files: >- + ['{{ etcd_cert_dir }}/ca.pem', + {% set etcd_members = groups['etcd'] %} + {% for host in etcd_members %} + '{{ etcd_cert_dir }}/admin-{{ host }}.pem', + '{{ etcd_cert_dir }}/admin-{{ host }}-key.pem', + '{{ etcd_cert_dir }}/member-{{ host }}.pem', + '{{ etcd_cert_dir }}/member-{{ host }}-key.pem', + {% endfor %} + {% set k8s_nodes = groups['kube_control_plane'] %} + {% for host in k8s_nodes %} + '{{ etcd_cert_dir }}/node-{{ host }}.pem', + '{{ etcd_cert_dir }}/node-{{ host }}-key.pem' + {% if not loop.last %}{{ ',' }}{% endif %} + {% endfor %}] + +- name: "Check_certs | Set 'gen_certs' to true if expected certificates are not on the first etcd node(2/2)" + set_fact: + gen_certs: true + run_once: true + with_items: "{{ expected_files }}" + vars: + expected_files: >- + ['{{ etcd_cert_dir }}/ca.pem', + {% set etcd_members = groups['etcd'] %} + {% for host in etcd_members %} + '{{ etcd_cert_dir }}/admin-{{ host }}.pem', + '{{ etcd_cert_dir }}/admin-{{ host }}-key.pem', + '{{ etcd_cert_dir }}/member-{{ host }}.pem', + '{{ etcd_cert_dir }}/member-{{ host }}-key.pem', + {% endfor %} + {% set k8s_nodes = groups['k8s_cluster'] | unique | sort %} + {% for host in k8s_nodes %} + '{{ etcd_cert_dir }}/node-{{ host }}.pem', + '{{ etcd_cert_dir }}/node-{{ host }}-key.pem' + {% if not loop.last %}{{ ',' }}{% endif %} + {% endfor %}] + when: + - kube_network_plugin in ["calico", "flannel", "cilium"] or cilium_deploy_additionally | default(false) | bool + - kube_network_plugin != "calico" or calico_datastore == "etcd" + - force_etcd_cert_refresh or not item in etcdcert_master.files | map(attribute='path') | list + +- name: "Check_certs | Set 'gen_*_certs' groups to track which nodes needs to have certs generated on first etcd node" + vars: + existing_certs: etcdcert_master.files | map(attribute='path') + ansible.builtin.group_by: + key: "gen_{{ item.node_type }}_certs_{{ force_etcd_cert_refresh or item.certs is not subset(existing_certs) }}" + loop: "{{ cert_files | dict2items(key_name='node_type', value_name='certs') }}" + +- name: "Check_certs | Set 'etcd_member_requires_sync' to true if ca or member/admin cert and key don't exist on etcd member or checksum doesn't match" + set_fact: + etcd_member_requires_sync: true + when: + - inventory_hostname in groups['etcd'] + - (not etcd_member_certs.results[0].stat.exists | default(false)) or + (not etcd_member_certs.results[1].stat.exists | default(false)) or + (not etcd_member_certs.results[2].stat.exists | default(false)) or + (not etcd_member_certs.results[3].stat.exists | default(false)) or + (not etcd_member_certs.results[4].stat.exists | default(false)) or + (etcd_member_certs.results[0].stat.checksum | default('') != etcdcert_master.files | selectattr("path", "equalto", etcd_member_certs.results[0].stat.path) | map(attribute="checksum") | first | default('')) or + (etcd_member_certs.results[1].stat.checksum | default('') != etcdcert_master.files | selectattr("path", "equalto", etcd_member_certs.results[1].stat.path) | map(attribute="checksum") | first | default('')) or + (etcd_member_certs.results[2].stat.checksum | default('') != etcdcert_master.files | selectattr("path", "equalto", etcd_member_certs.results[2].stat.path) | map(attribute="checksum") | first | default('')) or + (etcd_member_certs.results[3].stat.checksum | default('') != etcdcert_master.files | selectattr("path", "equalto", etcd_member_certs.results[3].stat.path) | map(attribute="checksum") | first | default('')) or + (etcd_member_certs.results[4].stat.checksum | default('') != etcdcert_master.files | selectattr("path", "equalto", etcd_member_certs.results[4].stat.path) | map(attribute="checksum") | first | default('')) + +- name: "Check_certs | Set 'kubernetes_host_requires_sync' to true if ca or node cert and key don't exist on kubernetes host or checksum doesn't match" + set_fact: + kubernetes_host_requires_sync: true + when: + - inventory_hostname in groups['k8s_cluster'] and + inventory_hostname not in groups['etcd'] + - (not etcd_node_certs.results[0].stat.exists | default(false)) or + (not etcd_node_certs.results[1].stat.exists | default(false)) or + (not etcd_node_certs.results[2].stat.exists | default(false)) or + (etcd_node_certs.results[0].stat.checksum | default('') != etcdcert_master.files | selectattr("path", "equalto", etcd_node_certs.results[0].stat.path) | map(attribute="checksum") | first | default('')) or + (etcd_node_certs.results[1].stat.checksum | default('') != etcdcert_master.files | selectattr("path", "equalto", etcd_node_certs.results[1].stat.path) | map(attribute="checksum") | first | default('')) or + (etcd_node_certs.results[2].stat.checksum | default('') != etcdcert_master.files | selectattr("path", "equalto", etcd_node_certs.results[2].stat.path) | map(attribute="checksum") | first | default('')) + +- name: "Check_certs | Set 'sync_certs' to true" + set_fact: + sync_certs: true + when: + - etcd_member_requires_sync | default(false) or + kubernetes_host_requires_sync | default(false) or + 'gen_master_certs_True' in group_names or + 'gen_node_certs_True' in group_names diff --git a/kubespray/project/roles/etcd/tasks/configure.yml b/kubespray/project/roles/etcd/tasks/configure.yml new file mode 100644 index 0000000..438dbc7 --- /dev/null +++ b/kubespray/project/roles/etcd/tasks/configure.yml @@ -0,0 +1,179 @@ +--- +- name: Configure | Check if etcd cluster is healthy + shell: "set -o pipefail && {{ bin_dir }}/etcdctl endpoint --cluster status && {{ bin_dir }}/etcdctl endpoint --cluster health 2>&1 | grep -v 'Error: unhealthy cluster' >/dev/null" + args: + executable: /bin/bash + register: etcd_cluster_is_healthy + failed_when: false + changed_when: false + check_mode: no + run_once: yes + when: + - is_etcd_master + - etcd_cluster_setup + tags: + - facts + environment: + ETCDCTL_API: "3" + ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" + ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" + ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem" + ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}" + +- name: Configure | Check if etcd-events cluster is healthy + shell: "set -o pipefail && {{ bin_dir }}/etcdctl endpoint --cluster status && {{ bin_dir }}/etcdctl endpoint --cluster health 2>&1 | grep -v 'Error: unhealthy cluster' >/dev/null" + args: + executable: /bin/bash + register: etcd_events_cluster_is_healthy + failed_when: false + changed_when: false + check_mode: no + run_once: yes + when: + - is_etcd_master + - etcd_events_cluster_setup + tags: + - facts + environment: + ETCDCTL_API: "3" + ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" + ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" + ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem" + ETCDCTL_ENDPOINTS: "{{ etcd_events_access_addresses }}" + +- name: Configure | Refresh etcd config + include_tasks: refresh_config.yml + when: is_etcd_master + +- name: Configure | Copy etcd.service systemd file + template: + src: "etcd-{{ etcd_deployment_type }}.service.j2" + dest: /etc/systemd/system/etcd.service + backup: yes + mode: 0644 + # FIXME: check that systemd version >= 250 (factory-reset.target was introduced in that release) + # Remove once we drop support for systemd < 250 + validate: "sh -c '[ -f /usr/bin/systemd/system/factory-reset.target ] || exit 0 && systemd-analyze verify %s:etcd-{{ etcd_deployment_type }}.service'" + when: is_etcd_master and etcd_cluster_setup + +- name: Configure | Copy etcd-events.service systemd file + template: + src: "etcd-events-{{ etcd_deployment_type }}.service.j2" + dest: /etc/systemd/system/etcd-events.service + backup: yes + mode: 0644 + validate: "sh -c '[ -f /usr/bin/systemd/system/factory-reset.target ] || exit 0 && systemd-analyze verify %s:etcd-events-{{ etcd_deployment_type }}.service'" + # FIXME: check that systemd version >= 250 (factory-reset.target was introduced in that release) + # Remove once we drop support for systemd < 250 + when: is_etcd_master and etcd_events_cluster_setup + +- name: Configure | reload systemd + systemd: + daemon_reload: true + when: is_etcd_master + +# when scaling new etcd will fail to start +- name: Configure | Ensure etcd is running + service: + name: etcd + state: started + enabled: yes + ignore_errors: "{{ etcd_cluster_is_healthy.rc == 0 }}" # noqa ignore-errors + when: is_etcd_master and etcd_cluster_setup + +# when scaling new etcd will fail to start +- name: Configure | Ensure etcd-events is running + service: + name: etcd-events + state: started + enabled: yes + ignore_errors: "{{ etcd_events_cluster_is_healthy.rc != 0 }}" # noqa ignore-errors + when: is_etcd_master and etcd_events_cluster_setup + +- name: Configure | Wait for etcd cluster to be healthy + shell: "set -o pipefail && {{ bin_dir }}/etcdctl endpoint --cluster status && {{ bin_dir }}/etcdctl endpoint --cluster health 2>&1 | grep -v 'Error: unhealthy cluster' >/dev/null" + args: + executable: /bin/bash + register: etcd_cluster_is_healthy + until: etcd_cluster_is_healthy.rc == 0 + retries: "{{ etcd_retries }}" + delay: "{{ retry_stagger | random + 3 }}" + changed_when: false + check_mode: no + run_once: yes + when: + - is_etcd_master + - etcd_cluster_setup + tags: + - facts + environment: + ETCDCTL_API: "3" + ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" + ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" + ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem" + ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}" + +- name: Configure | Wait for etcd-events cluster to be healthy + shell: "set -o pipefail && {{ bin_dir }}/etcdctl endpoint --cluster status && {{ bin_dir }}/etcdctl endpoint --cluster health 2>&1 | grep -v 'Error: unhealthy cluster' >/dev/null" + args: + executable: /bin/bash + register: etcd_events_cluster_is_healthy + until: etcd_events_cluster_is_healthy.rc == 0 + retries: "{{ etcd_retries }}" + delay: "{{ retry_stagger | random + 3 }}" + changed_when: false + check_mode: no + run_once: yes + when: + - is_etcd_master + - etcd_events_cluster_setup + tags: + - facts + environment: + ETCDCTL_API: "3" + ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" + ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" + ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem" + ETCDCTL_ENDPOINTS: "{{ etcd_events_access_addresses }}" + +- name: Configure | Check if member is in etcd cluster + shell: "{{ bin_dir }}/etcdctl member list | grep -w -q {{ etcd_access_address }}" + register: etcd_member_in_cluster + ignore_errors: true # noqa ignore-errors + changed_when: false + check_mode: no + when: is_etcd_master and etcd_cluster_setup + tags: + - facts + environment: + ETCDCTL_API: "3" + ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" + ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" + ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem" + ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}" + +- name: Configure | Check if member is in etcd-events cluster + shell: "{{ bin_dir }}/etcdctl member list | grep -w -q {{ etcd_access_address }}" + register: etcd_events_member_in_cluster + ignore_errors: true # noqa ignore-errors + changed_when: false + check_mode: no + when: is_etcd_master and etcd_events_cluster_setup + tags: + - facts + environment: + ETCDCTL_API: "3" + ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" + ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" + ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem" + ETCDCTL_ENDPOINTS: "{{ etcd_events_access_addresses }}" + +- name: Configure | Join member(s) to etcd cluster one at a time + include_tasks: join_etcd_member.yml + with_items: "{{ groups['etcd'] }}" + when: inventory_hostname == item and etcd_cluster_setup and etcd_member_in_cluster.rc != 0 and etcd_cluster_is_healthy.rc == 0 + +- name: Configure | Join member(s) to etcd-events cluster one at a time + include_tasks: join_etcd-events_member.yml + with_items: "{{ groups['etcd'] }}" + when: inventory_hostname == item and etcd_events_cluster_setup and etcd_events_member_in_cluster.rc != 0 and etcd_events_cluster_is_healthy.rc == 0 diff --git a/kubespray/project/roles/etcd/tasks/gen_certs_script.yml b/kubespray/project/roles/etcd/tasks/gen_certs_script.yml new file mode 100644 index 0000000..33e9d94 --- /dev/null +++ b/kubespray/project/roles/etcd/tasks/gen_certs_script.yml @@ -0,0 +1,156 @@ +--- +- name: Gen_certs | create etcd cert dir + file: + path: "{{ etcd_cert_dir }}" + group: "{{ etcd_cert_group }}" + state: directory + owner: "{{ etcd_owner }}" + mode: "{{ etcd_cert_dir_mode }}" + recurse: yes + +- name: "Gen_certs | create etcd script dir (on {{ groups['etcd'][0] }})" + file: + path: "{{ etcd_script_dir }}" + state: directory + owner: root + mode: 0700 + run_once: yes + when: inventory_hostname == groups['etcd'][0] + +- name: Gen_certs | write openssl config + template: + src: "openssl.conf.j2" + dest: "{{ etcd_config_dir }}/openssl.conf" + mode: 0640 + run_once: yes + delegate_to: "{{ groups['etcd'][0] }}" + when: + - gen_certs | default(false) + - inventory_hostname == groups['etcd'][0] + +- name: Gen_certs | copy certs generation script + template: + src: "make-ssl-etcd.sh.j2" + dest: "{{ etcd_script_dir }}/make-ssl-etcd.sh" + mode: 0700 + run_once: yes + when: + - gen_certs | default(false) + - inventory_hostname == groups['etcd'][0] + +- name: Gen_certs | run cert generation script for etcd and kube control plane nodes + command: "bash -x {{ etcd_script_dir }}/make-ssl-etcd.sh -f {{ etcd_config_dir }}/openssl.conf -d {{ etcd_cert_dir }}" + environment: + MASTERS: "{{ groups['gen_master_certs_True'] | ansible.builtin.intersect(groups['etcd']) | join(' ') }}" + HOSTS: "{{ groups['gen_node_certs_True'] | ansible.builtin.intersect(groups['kube_control_plane']) | join(' ') }}" + run_once: yes + delegate_to: "{{ groups['etcd'][0] }}" + when: gen_certs | default(false) + notify: Set etcd_secret_changed + +- name: Gen_certs | run cert generation script for all clients + command: "bash -x {{ etcd_script_dir }}/make-ssl-etcd.sh -f {{ etcd_config_dir }}/openssl.conf -d {{ etcd_cert_dir }}" + environment: + HOSTS: "{{ groups['gen_node_certs_True'] | ansible.builtin.intersect(groups['k8s_cluster']) | join(' ') }}" + run_once: yes + delegate_to: "{{ groups['etcd'][0] }}" + when: + - kube_network_plugin in ["calico", "flannel", "cilium"] or cilium_deploy_additionally | default(false) | bool + - kube_network_plugin != "calico" or calico_datastore == "etcd" + - gen_certs | default(false) + notify: Set etcd_secret_changed + +- name: Gen_certs | Gather etcd member/admin and kube_control_plane client certs from first etcd node + slurp: + src: "{{ item }}" + register: etcd_master_certs + with_items: + - "{{ etcd_cert_dir }}/ca.pem" + - "{{ etcd_cert_dir }}/ca-key.pem" + - "[{% for node in groups['etcd'] %} + '{{ etcd_cert_dir }}/admin-{{ node }}.pem', + '{{ etcd_cert_dir }}/admin-{{ node }}-key.pem', + '{{ etcd_cert_dir }}/member-{{ node }}.pem', + '{{ etcd_cert_dir }}/member-{{ node }}-key.pem', + {% endfor %}]" + - "[{% for node in (groups['kube_control_plane']) %} + '{{ etcd_cert_dir }}/node-{{ node }}.pem', + '{{ etcd_cert_dir }}/node-{{ node }}-key.pem', + {% endfor %}]" + delegate_to: "{{ groups['etcd'][0] }}" + when: + - inventory_hostname in groups['etcd'] + - sync_certs | default(false) + - inventory_hostname != groups['etcd'][0] + notify: Set etcd_secret_changed + +- name: Gen_certs | Write etcd member/admin and kube_control_plane client certs to other etcd nodes + copy: + dest: "{{ item.item }}" + content: "{{ item.content | b64decode }}" + group: "{{ etcd_cert_group }}" + owner: "{{ etcd_owner }}" + mode: 0640 + with_items: "{{ etcd_master_certs.results }}" + when: + - inventory_hostname in groups['etcd'] + - sync_certs | default(false) + - inventory_hostname != groups['etcd'][0] + loop_control: + label: "{{ item.item }}" + +- name: Gen_certs | Gather node certs from first etcd node + slurp: + src: "{{ item }}" + register: etcd_master_node_certs + with_items: + - "[{% for node in groups['k8s_cluster'] %} + '{{ etcd_cert_dir }}/node-{{ node }}.pem', + '{{ etcd_cert_dir }}/node-{{ node }}-key.pem', + {% endfor %}]" + delegate_to: "{{ groups['etcd'][0] }}" + when: + - inventory_hostname in groups['etcd'] + - inventory_hostname != groups['etcd'][0] + - kube_network_plugin in ["calico", "flannel", "cilium"] or cilium_deploy_additionally | default(false) | bool + - kube_network_plugin != "calico" or calico_datastore == "etcd" + notify: Set etcd_secret_changed + +- name: Gen_certs | Write node certs to other etcd nodes + copy: + dest: "{{ item.item }}" + content: "{{ item.content | b64decode }}" + group: "{{ etcd_cert_group }}" + owner: "{{ etcd_owner }}" + mode: 0640 + with_items: "{{ etcd_master_node_certs.results }}" + when: + - inventory_hostname in groups['etcd'] + - inventory_hostname != groups['etcd'][0] + - kube_network_plugin in ["calico", "flannel", "cilium"] or cilium_deploy_additionally | default(false) | bool + - kube_network_plugin != "calico" or calico_datastore == "etcd" + loop_control: + label: "{{ item.item }}" + +- name: Gen_certs | Generate etcd certs + include_tasks: gen_nodes_certs_script.yml + when: + - inventory_hostname in groups['kube_control_plane'] and + sync_certs | default(false) and inventory_hostname not in groups['etcd'] + +- name: Gen_certs | Generate etcd certs on nodes if needed + include_tasks: gen_nodes_certs_script.yml + when: + - kube_network_plugin in ["calico", "flannel", "cilium"] or cilium_deploy_additionally | default(false) | bool + - kube_network_plugin != "calico" or calico_datastore == "etcd" + - inventory_hostname in groups['k8s_cluster'] and + sync_certs | default(false) and inventory_hostname not in groups['etcd'] + +- name: Gen_certs | check certificate permissions + file: + path: "{{ etcd_cert_dir }}" + group: "{{ etcd_cert_group }}" + state: directory + owner: "{{ etcd_owner }}" + mode: "{{ etcd_cert_dir_mode }}" + recurse: yes diff --git a/kubespray/project/roles/etcd/tasks/gen_nodes_certs_script.yml b/kubespray/project/roles/etcd/tasks/gen_nodes_certs_script.yml new file mode 100644 index 0000000..2093bf8 --- /dev/null +++ b/kubespray/project/roles/etcd/tasks/gen_nodes_certs_script.yml @@ -0,0 +1,33 @@ +--- +- name: Gen_certs | Set cert names per node + set_fact: + my_etcd_node_certs: [ 'ca.pem', + 'node-{{ inventory_hostname }}.pem', + 'node-{{ inventory_hostname }}-key.pem'] + tags: + - facts + +- name: "Check_certs | Set 'sync_certs' to true on nodes" + set_fact: + sync_certs: true + with_items: + - "{{ my_etcd_node_certs }}" + +- name: Gen_certs | Gather node certs + vars: + ansible_ssh_retries: 10 + shell: "set -o pipefail && tar cfz - -C {{ etcd_cert_dir }} {{ my_etcd_node_certs | join(' ') }} | base64 --wrap=0" + args: + executable: /bin/bash + no_log: "{{ not (unsafe_show_logs | bool) }}" + register: etcd_node_certs + check_mode: no + delegate_to: "{{ groups['etcd'][0] }}" + changed_when: false + +- name: Gen_certs | Copy certs on nodes + shell: "set -o pipefail && base64 -d <<< '{{ etcd_node_certs.stdout | quote }}' | tar xz -C {{ etcd_cert_dir }}" + args: + executable: /bin/bash + no_log: "{{ not (unsafe_show_logs | bool) }}" + changed_when: false diff --git a/kubespray/project/roles/etcd/tasks/install_docker.yml b/kubespray/project/roles/etcd/tasks/install_docker.yml new file mode 100644 index 0000000..cc2fdec --- /dev/null +++ b/kubespray/project/roles/etcd/tasks/install_docker.yml @@ -0,0 +1,42 @@ +--- +- name: Get currently-deployed etcd version + shell: "{{ docker_bin_dir }}/docker ps --filter='name={{ etcd_member_name }}' --format='{{ '{{ .Image }}' }}'" + register: etcd_current_docker_image + when: etcd_cluster_setup + +- name: Get currently-deployed etcd-events version + shell: "{{ docker_bin_dir }}/docker ps --filter='name={{ etcd_member_name }}-events' --format='{{ '{{ .Image }}' }}'" + register: etcd_events_current_docker_image + when: etcd_events_cluster_setup + +- name: Restart etcd if necessary + command: /bin/true + notify: Restart etcd + when: + - etcd_cluster_setup + - etcd_image_tag not in etcd_current_docker_image.stdout | default('') + +- name: Restart etcd-events if necessary + command: /bin/true + notify: Restart etcd-events + when: + - etcd_events_cluster_setup + - etcd_image_tag not in etcd_events_current_docker_image.stdout | default('') + +- name: Install etcd launch script + template: + src: etcd.j2 + dest: "{{ bin_dir }}/etcd" + owner: 'root' + mode: 0750 + backup: yes + when: etcd_cluster_setup + +- name: Install etcd-events launch script + template: + src: etcd-events.j2 + dest: "{{ bin_dir }}/etcd-events" + owner: 'root' + mode: 0750 + backup: yes + when: etcd_events_cluster_setup diff --git a/kubespray/project/roles/etcd/tasks/install_host.yml b/kubespray/project/roles/etcd/tasks/install_host.yml new file mode 100644 index 0000000..d4baa2a --- /dev/null +++ b/kubespray/project/roles/etcd/tasks/install_host.yml @@ -0,0 +1,31 @@ +--- +- name: Get currently-deployed etcd version + command: "{{ bin_dir }}/etcd --version" + register: etcd_current_host_version + # There's a chance this play could run before etcd is installed at all + ignore_errors: true + when: etcd_cluster_setup + +- name: Restart etcd if necessary + command: /bin/true + notify: Restart etcd + when: + - etcd_cluster_setup + - etcd_version.lstrip('v') not in etcd_current_host_version.stdout | default('') + +- name: Restart etcd-events if necessary + command: /bin/true + notify: Restart etcd-events + when: + - etcd_events_cluster_setup + - etcd_version.lstrip('v') not in etcd_current_host_version.stdout | default('') + +- name: Install | Copy etcd binary from download dir + copy: + src: "{{ local_release_dir }}/etcd-{{ etcd_version }}-linux-{{ host_architecture }}/{{ item }}" + dest: "{{ bin_dir }}/{{ item }}" + mode: 0755 + remote_src: yes + with_items: + - etcd + when: etcd_cluster_setup diff --git a/kubespray/project/roles/etcd/tasks/join_etcd-events_member.yml b/kubespray/project/roles/etcd/tasks/join_etcd-events_member.yml new file mode 100644 index 0000000..0fad331 --- /dev/null +++ b/kubespray/project/roles/etcd/tasks/join_etcd-events_member.yml @@ -0,0 +1,49 @@ +--- +- name: Join Member | Add member to etcd-events cluster + command: "{{ bin_dir }}/etcdctl member add {{ etcd_member_name }} --peer-urls={{ etcd_events_peer_url }}" + register: member_add_result + until: member_add_result.rc == 0 + retries: "{{ etcd_retries }}" + delay: "{{ retry_stagger | random + 3 }}" + environment: + ETCDCTL_API: "3" + ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" + ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" + ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem" + ETCDCTL_ENDPOINTS: "{{ etcd_events_access_addresses }}" + +- name: Join Member | Refresh etcd config + include_tasks: refresh_config.yml + vars: + # noqa: jinja[spacing] + etcd_events_peer_addresses: >- + {% for host in groups['etcd'] -%} + {%- if hostvars[host]['etcd_events_member_in_cluster'].rc == 0 -%} + {{ "etcd" + loop.index | string }}=https://{{ hostvars[host].etcd_events_access_address | default(hostvars[host].ip | default(fallback_ips[host])) }}:2382, + {%- endif -%} + {%- if loop.last -%} + {{ etcd_member_name }}={{ etcd_events_peer_url }} + {%- endif -%} + {%- endfor -%} + +- name: Join Member | Ensure member is in etcd-events cluster + shell: "set -o pipefail && {{ bin_dir }}/etcdctl member list | grep -w {{ etcd_events_access_address }} >/dev/null" + args: + executable: /bin/bash + register: etcd_events_member_in_cluster + changed_when: false + check_mode: no + tags: + - facts + environment: + ETCDCTL_API: "3" + ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" + ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" + ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem" + ETCDCTL_ENDPOINTS: "{{ etcd_events_access_addresses }}" + +- name: Configure | Ensure etcd-events is running + service: + name: etcd-events + state: started + enabled: yes diff --git a/kubespray/project/roles/etcd/tasks/join_etcd_member.yml b/kubespray/project/roles/etcd/tasks/join_etcd_member.yml new file mode 100644 index 0000000..ee77d4b --- /dev/null +++ b/kubespray/project/roles/etcd/tasks/join_etcd_member.yml @@ -0,0 +1,53 @@ +--- +- name: Join Member | Add member to etcd cluster + command: "{{ bin_dir }}/etcdctl member add {{ etcd_member_name }} --peer-urls={{ etcd_peer_url }}" + register: member_add_result + until: member_add_result.rc == 0 or 'Peer URLs already exists' in member_add_result.stderr + failed_when: member_add_result.rc != 0 and 'Peer URLs already exists' not in member_add_result.stderr + retries: "{{ etcd_retries }}" + delay: "{{ retry_stagger | random + 3 }}" + environment: + ETCDCTL_API: "3" + ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" + ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" + ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem" + ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}" + +- name: Join Member | Refresh etcd config + include_tasks: refresh_config.yml + vars: + # noqa: jinja[spacing] + etcd_peer_addresses: >- + {% for host in groups['etcd'] -%} + {%- if hostvars[host]['etcd_member_in_cluster'].rc == 0 -%} + {{ "etcd" + loop.index | string }}=https://{{ hostvars[host].etcd_access_address | default(hostvars[host].ip | default(fallback_ips[host])) }}:2380, + {%- endif -%} + {%- if loop.last -%} + {{ etcd_member_name }}={{ etcd_peer_url }} + {%- endif -%} + {%- endfor -%} + +- name: Join Member | Ensure member is in etcd cluster + shell: "set -o pipefail && {{ bin_dir }}/etcdctl member list | grep -w {{ etcd_access_address }} >/dev/null" + args: + executable: /bin/bash + register: etcd_member_in_cluster + changed_when: false + check_mode: no + retries: "{{ etcd_retries }}" + delay: "{{ retry_stagger | random + 3 }}" + until: etcd_member_in_cluster.rc == 0 + tags: + - facts + environment: + ETCDCTL_API: "3" + ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" + ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" + ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem" + ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}" + +- name: Configure | Ensure etcd is running + service: + name: etcd + state: started + enabled: yes diff --git a/kubespray/project/roles/etcd/tasks/main.yml b/kubespray/project/roles/etcd/tasks/main.yml new file mode 100644 index 0000000..40ca3de --- /dev/null +++ b/kubespray/project/roles/etcd/tasks/main.yml @@ -0,0 +1,96 @@ +--- +- name: Check etcd certs + include_tasks: check_certs.yml + when: cert_management == "script" + tags: + - etcd-secrets + - facts + +- name: Generate etcd certs + include_tasks: "gen_certs_script.yml" + when: + - cert_management | d('script') == "script" + tags: + - etcd-secrets + +- name: Trust etcd CA + include_tasks: upd_ca_trust.yml + when: + - inventory_hostname in groups['etcd'] | union(groups['kube_control_plane']) | unique | sort + tags: + - etcd-secrets + +- name: Trust etcd CA on nodes if needed + include_tasks: upd_ca_trust.yml + when: + - kube_network_plugin in ["calico", "flannel", "cilium"] or cilium_deploy_additionally | default(false) | bool + - kube_network_plugin != "calico" or calico_datastore == "etcd" + - inventory_hostname in groups['k8s_cluster'] + tags: + - etcd-secrets + +- name: "Gen_certs | Get etcd certificate serials" + command: "openssl x509 -in {{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem -noout -serial" + register: "etcd_client_cert_serial_result" + changed_when: false + check_mode: no + when: + - kube_network_plugin in ["calico", "flannel", "cilium"] or cilium_deploy_additionally | default(false) | bool + - kube_network_plugin != "calico" or calico_datastore == "etcd" + - inventory_hostname in groups['k8s_cluster'] + tags: + - master + - network + +- name: Set etcd_client_cert_serial + set_fact: + etcd_client_cert_serial: "{{ etcd_client_cert_serial_result.stdout.split('=')[1] }}" + when: + - kube_network_plugin in ["calico", "flannel", "cilium"] or cilium_deploy_additionally | default(false) | bool + - kube_network_plugin != "calico" or calico_datastore == "etcd" + - inventory_hostname in groups['k8s_cluster'] + tags: + - master + - network + +- name: Install etcdctl and etcdutl binary + import_role: + name: etcdctl_etcdutl + tags: + - etcdctl + - etcdutl + - upgrade + when: + - inventory_hostname in groups['etcd'] + - etcd_cluster_setup + +- name: Install etcd + include_tasks: "install_{{ etcd_deployment_type }}.yml" + when: is_etcd_master + tags: + - upgrade + +- name: Configure etcd + include_tasks: configure.yml + when: is_etcd_master + +- name: Refresh etcd config + include_tasks: refresh_config.yml + when: is_etcd_master + +- name: Restart etcd if certs changed + command: /bin/true + notify: Restart etcd + when: is_etcd_master and etcd_cluster_setup and etcd_secret_changed | default(false) + +- name: Restart etcd-events if certs changed + command: /bin/true + notify: Restart etcd + when: is_etcd_master and etcd_events_cluster_setup and etcd_secret_changed | default(false) + +# After etcd cluster is assembled, make sure that +# initial state of the cluster is in `existing` +# state instead of `new`. +- name: Refresh etcd config again for idempotency + include_tasks: refresh_config.yml + when: is_etcd_master diff --git a/kubespray/project/roles/etcd/tasks/refresh_config.yml b/kubespray/project/roles/etcd/tasks/refresh_config.yml new file mode 100644 index 0000000..d5e0045 --- /dev/null +++ b/kubespray/project/roles/etcd/tasks/refresh_config.yml @@ -0,0 +1,16 @@ +--- +- name: Refresh config | Create etcd config file + template: + src: etcd.env.j2 + dest: /etc/etcd.env + mode: 0640 + notify: Restart etcd + when: is_etcd_master and etcd_cluster_setup + +- name: Refresh config | Create etcd-events config file + template: + src: etcd-events.env.j2 + dest: /etc/etcd-events.env + mode: 0640 + notify: Restart etcd-events + when: is_etcd_master and etcd_events_cluster_setup diff --git a/kubespray/project/roles/etcd/tasks/upd_ca_trust.yml b/kubespray/project/roles/etcd/tasks/upd_ca_trust.yml new file mode 100644 index 0000000..22c5901 --- /dev/null +++ b/kubespray/project/roles/etcd/tasks/upd_ca_trust.yml @@ -0,0 +1,37 @@ +--- +- name: Gen_certs | target ca-certificate store file + set_fact: + ca_cert_path: |- + {% if ansible_os_family == "Debian" -%} + /usr/local/share/ca-certificates/etcd-ca.crt + {%- elif ansible_os_family == "RedHat" -%} + /etc/pki/ca-trust/source/anchors/etcd-ca.crt + {%- elif ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] -%} + /etc/ssl/certs/etcd-ca.pem + {%- elif ansible_os_family == "Suse" -%} + /etc/pki/trust/anchors/etcd-ca.pem + {%- elif ansible_os_family == "ClearLinux" -%} + /usr/share/ca-certs/etcd-ca.pem + {%- endif %} + tags: + - facts + +- name: Gen_certs | add CA to trusted CA dir + copy: + src: "{{ etcd_cert_dir }}/ca.pem" + dest: "{{ ca_cert_path }}" + remote_src: true + mode: 0640 + register: etcd_ca_cert + +- name: Gen_certs | update ca-certificates (Debian/Ubuntu/SUSE/Flatcar) # noqa no-handler + command: update-ca-certificates + when: etcd_ca_cert.changed and ansible_os_family in ["Debian", "Flatcar", "Flatcar Container Linux by Kinvolk", "Suse"] + +- name: Gen_certs | update ca-certificates (RedHat) # noqa no-handler + command: update-ca-trust extract + when: etcd_ca_cert.changed and ansible_os_family == "RedHat" + +- name: Gen_certs | update ca-certificates (ClearLinux) # noqa no-handler + command: clrtrust add "{{ ca_cert_path }}" + when: etcd_ca_cert.changed and ansible_os_family == "ClearLinux" diff --git a/kubespray/project/roles/etcd/templates/etcd-docker.service.j2 b/kubespray/project/roles/etcd/templates/etcd-docker.service.j2 new file mode 100644 index 0000000..4dfbd72 --- /dev/null +++ b/kubespray/project/roles/etcd/templates/etcd-docker.service.j2 @@ -0,0 +1,18 @@ +[Unit] +Description=etcd docker wrapper +Wants=docker.socket +After=docker.service + +[Service] +User=root +PermissionsStartOnly=true +EnvironmentFile=-/etc/etcd.env +ExecStart={{ bin_dir }}/etcd +ExecStartPre=-{{ docker_bin_dir }}/docker rm -f {{ etcd_member_name | default("etcd") }} +ExecStop={{ docker_bin_dir }}/docker stop {{ etcd_member_name | default("etcd") }} +Restart=always +RestartSec=15s +TimeoutStartSec=30s + +[Install] +WantedBy=multi-user.target diff --git a/kubespray/project/roles/etcd/templates/etcd-events-docker.service.j2 b/kubespray/project/roles/etcd/templates/etcd-events-docker.service.j2 new file mode 100644 index 0000000..271980a --- /dev/null +++ b/kubespray/project/roles/etcd/templates/etcd-events-docker.service.j2 @@ -0,0 +1,18 @@ +[Unit] +Description=etcd docker wrapper +Wants=docker.socket +After=docker.service + +[Service] +User=root +PermissionsStartOnly=true +EnvironmentFile=-/etc/etcd-events.env +ExecStart={{ bin_dir }}/etcd-events +ExecStartPre=-{{ docker_bin_dir }}/docker rm -f {{ etcd_member_name }}-events +ExecStop={{ docker_bin_dir }}/docker stop {{ etcd_member_name }}-events +Restart=always +RestartSec=15s +TimeoutStartSec=30s + +[Install] +WantedBy=multi-user.target diff --git a/kubespray/project/roles/etcd/templates/etcd-events-host.service.j2 b/kubespray/project/roles/etcd/templates/etcd-events-host.service.j2 new file mode 100644 index 0000000..6e0167a --- /dev/null +++ b/kubespray/project/roles/etcd/templates/etcd-events-host.service.j2 @@ -0,0 +1,16 @@ +[Unit] +Description=etcd +After=network.target + +[Service] +Type=notify +User=root +EnvironmentFile=/etc/etcd-events.env +ExecStart={{ bin_dir }}/etcd +NotifyAccess=all +Restart=always +RestartSec=10s +LimitNOFILE=40000 + +[Install] +WantedBy=multi-user.target diff --git a/kubespray/project/roles/etcd/templates/etcd-events.env.j2 b/kubespray/project/roles/etcd/templates/etcd-events.env.j2 new file mode 100644 index 0000000..3abefd6 --- /dev/null +++ b/kubespray/project/roles/etcd/templates/etcd-events.env.j2 @@ -0,0 +1,43 @@ +ETCD_DATA_DIR={{ etcd_events_data_dir }} +ETCD_ADVERTISE_CLIENT_URLS={{ etcd_events_client_url }} +ETCD_INITIAL_ADVERTISE_PEER_URLS={{ etcd_events_peer_url }} +ETCD_INITIAL_CLUSTER_STATE={% if etcd_events_cluster_is_healthy.rc == 0 | bool %}existing{% else %}new{% endif %} + +ETCD_METRICS={{ etcd_metrics }} +ETCD_LISTEN_CLIENT_URLS=https://{{ etcd_address }}:2383,https://127.0.0.1:2383 +ETCD_ELECTION_TIMEOUT={{ etcd_election_timeout }} +ETCD_HEARTBEAT_INTERVAL={{ etcd_heartbeat_interval }} +ETCD_INITIAL_CLUSTER_TOKEN=k8s_events_etcd +ETCD_LISTEN_PEER_URLS=https://{{ etcd_address }}:2382 +ETCD_NAME={{ etcd_member_name }}-events +ETCD_PROXY=off +ETCD_INITIAL_CLUSTER={{ etcd_events_peer_addresses }} +ETCD_AUTO_COMPACTION_RETENTION={{ etcd_compaction_retention }} +{% if etcd_snapshot_count is defined %} +ETCD_SNAPSHOT_COUNT={{ etcd_snapshot_count }} +{% endif %} +{% if etcd_quota_backend_bytes is defined %} +ETCD_QUOTA_BACKEND_BYTES={{ etcd_quota_backend_bytes }} +{% endif %} +{% if etcd_max_request_bytes is defined %} +ETCD_MAX_REQUEST_BYTES={{ etcd_max_request_bytes }} +{% endif %} + +# TLS settings +ETCD_TRUSTED_CA_FILE={{ etcd_cert_dir }}/ca.pem +ETCD_CERT_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}.pem +ETCD_KEY_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}-key.pem +ETCD_CLIENT_CERT_AUTH={{ etcd_secure_client | lower}} + +ETCD_PEER_TRUSTED_CA_FILE={{ etcd_cert_dir }}/ca.pem +ETCD_PEER_CERT_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}.pem +ETCD_PEER_KEY_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}-key.pem +ETCD_PEER_CLIENT_CERT_AUTH={{ etcd_peer_client_auth }} + +{% if etcd_tls_cipher_suites is defined %} +ETCD_CIPHER_SUITES={% for tls in etcd_tls_cipher_suites %}{{ tls }}{{ "," if not loop.last else "" }}{% endfor %} +{% endif %} + +{% for key, value in etcd_extra_vars.items() %} +{{ key }}={{ value }} +{% endfor %} diff --git a/kubespray/project/roles/etcd/templates/etcd-events.j2 b/kubespray/project/roles/etcd/templates/etcd-events.j2 new file mode 100644 index 0000000..b268479 --- /dev/null +++ b/kubespray/project/roles/etcd/templates/etcd-events.j2 @@ -0,0 +1,21 @@ +#!/bin/bash +{{ docker_bin_dir }}/docker run \ + --restart=on-failure:5 \ + --env-file=/etc/etcd-events.env \ + --net=host \ + -v /etc/ssl/certs:/etc/ssl/certs:ro \ + -v {{ etcd_cert_dir }}:{{ etcd_cert_dir }}:ro \ + -v {{ etcd_events_data_dir }}:{{ etcd_events_data_dir }}:rw \ + {% if etcd_memory_limit is defined %} + --memory={{ etcd_memory_limit|regex_replace('Mi', 'M') }} \ + {% endif %} + {% if etcd_cpu_limit is defined %} + --cpu-shares={{ etcd_cpu_limit|regex_replace('m', '') }} \ + {% endif %} + {% if etcd_blkio_weight is defined %} + --blkio-weight={{ etcd_blkio_weight }} \ + {% endif %} + --name={{ etcd_member_name }}-events \ + {{ etcd_image_repo }}:{{ etcd_image_tag }} \ + /usr/local/bin/etcd \ + "$@" diff --git a/kubespray/project/roles/etcd/templates/etcd-host.service.j2 b/kubespray/project/roles/etcd/templates/etcd-host.service.j2 new file mode 100644 index 0000000..6bba805 --- /dev/null +++ b/kubespray/project/roles/etcd/templates/etcd-host.service.j2 @@ -0,0 +1,16 @@ +[Unit] +Description=etcd +After=network.target + +[Service] +Type=notify +User=root +EnvironmentFile=/etc/etcd.env +ExecStart={{ bin_dir }}/etcd +NotifyAccess=all +Restart=always +RestartSec=10s +LimitNOFILE=40000 + +[Install] +WantedBy=multi-user.target diff --git a/kubespray/project/roles/etcd/templates/etcd.env.j2 b/kubespray/project/roles/etcd/templates/etcd.env.j2 new file mode 100644 index 0000000..13a2437 --- /dev/null +++ b/kubespray/project/roles/etcd/templates/etcd.env.j2 @@ -0,0 +1,78 @@ +# Environment file for etcd {{ etcd_version }} +ETCD_DATA_DIR={{ etcd_data_dir }} +ETCD_ADVERTISE_CLIENT_URLS={{ etcd_client_url }} +ETCD_INITIAL_ADVERTISE_PEER_URLS={{ etcd_peer_url }} +ETCD_INITIAL_CLUSTER_STATE={% if etcd_cluster_is_healthy.rc == 0 | bool %}existing{% else %}new{% endif %} + +ETCD_METRICS={{ etcd_metrics }} +{% if etcd_listen_metrics_urls is defined %} +ETCD_LISTEN_METRICS_URLS={{ etcd_listen_metrics_urls }} +{% elif etcd_metrics_port is defined %} +ETCD_LISTEN_METRICS_URLS=http://{{ etcd_address }}:{{ etcd_metrics_port }},http://127.0.0.1:{{ etcd_metrics_port }} +{% endif %} +ETCD_LISTEN_CLIENT_URLS=https://{{ etcd_address }}:2379,https://127.0.0.1:2379 +ETCD_ELECTION_TIMEOUT={{ etcd_election_timeout }} +ETCD_HEARTBEAT_INTERVAL={{ etcd_heartbeat_interval }} +ETCD_INITIAL_CLUSTER_TOKEN=k8s_etcd +ETCD_LISTEN_PEER_URLS=https://{{ etcd_address }}:2380 +ETCD_NAME={{ etcd_member_name }} +ETCD_PROXY=off +ETCD_INITIAL_CLUSTER={{ etcd_peer_addresses }} +ETCD_AUTO_COMPACTION_RETENTION={{ etcd_compaction_retention }} +{% if etcd_snapshot_count is defined %} +ETCD_SNAPSHOT_COUNT={{ etcd_snapshot_count }} +{% endif %} +{% if etcd_quota_backend_bytes is defined %} +ETCD_QUOTA_BACKEND_BYTES={{ etcd_quota_backend_bytes }} +{% endif %} +{% if etcd_max_request_bytes is defined %} +ETCD_MAX_REQUEST_BYTES={{ etcd_max_request_bytes }} +{% endif %} +{% if etcd_log_level is defined %} +ETCD_LOG_LEVEL={{ etcd_log_level }} +{% endif %} +{% if etcd_max_snapshots is defined %} +ETCD_MAX_SNAPSHOTS={{ etcd_max_snapshots }} +{% endif %} +{% if etcd_max_wals is defined %} +ETCD_MAX_WALS={{ etcd_max_wals }} +{% endif %} +# Flannel need etcd v2 API +ETCD_ENABLE_V2=true + +# TLS settings +ETCD_TRUSTED_CA_FILE={{ etcd_cert_dir }}/ca.pem +ETCD_CERT_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}.pem +ETCD_KEY_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}-key.pem +ETCD_CLIENT_CERT_AUTH={{ etcd_secure_client | lower}} + +ETCD_PEER_TRUSTED_CA_FILE={{ etcd_cert_dir }}/ca.pem +ETCD_PEER_CERT_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}.pem +ETCD_PEER_KEY_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}-key.pem +ETCD_PEER_CLIENT_CERT_AUTH={{ etcd_peer_client_auth }} + +{% if etcd_tls_cipher_suites is defined %} +ETCD_CIPHER_SUITES={% for tls in etcd_tls_cipher_suites %}{{ tls }}{{ "," if not loop.last else "" }}{% endfor %} +{% endif %} + +{% for key, value in etcd_extra_vars.items() %} +{{ key }}={{ value }} +{% endfor %} + +# CLI settings +ETCDCTL_ENDPOINTS=https://127.0.0.1:2379 +ETCDCTL_CACERT={{ etcd_cert_dir }}/ca.pem +ETCDCTL_KEY={{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem +ETCDCTL_CERT={{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem + +# ETCD 3.5.x issue +# https://groups.google.com/a/kubernetes.io/g/dev/c/B7gJs88XtQc/m/rSgNOzV2BwAJ?utm_medium=email&utm_source=footer +ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK={{ etcd_experimental_initial_corrupt_check }} + +{% if etcd_experimental_enable_distributed_tracing %} +ETCD_EXPERIMENTAL_ENABLE_DISTRIBUTED_TRACING=true +ETCD_EXPERIMENTAL_DISTRIBUTED_TRACING_SAMPLE_RATE={{ etcd_experimental_distributed_tracing_sample_rate }} +ETCD_EXPERIMENTAL_DISTRIBUTED_TRACING_ADDRESS={{ etcd_experimental_distributed_tracing_address }} +ETCD_EXPERIMENTAL_DISTRIBUTED_TRACING_SERVICE_NAME={{ etcd_experimental_distributed_tracing_service_name }} +ETCD_EXPERIMENTAL_DISTRIBUTED_TRACING_INSTANCE_ID={{ etcd_member_name }} +{% endif %} \ No newline at end of file diff --git a/kubespray/project/roles/etcd/templates/etcd.j2 b/kubespray/project/roles/etcd/templates/etcd.j2 new file mode 100644 index 0000000..5374c70 --- /dev/null +++ b/kubespray/project/roles/etcd/templates/etcd.j2 @@ -0,0 +1,21 @@ +#!/bin/bash +{{ docker_bin_dir }}/docker run \ + --restart=on-failure:5 \ + --env-file=/etc/etcd.env \ + --net=host \ + -v /etc/ssl/certs:/etc/ssl/certs:ro \ + -v {{ etcd_cert_dir }}:{{ etcd_cert_dir }}:ro \ + -v {{ etcd_data_dir }}:{{ etcd_data_dir }}:rw \ +{% if etcd_memory_limit is defined %} + --memory={{ etcd_memory_limit|regex_replace('Mi', 'M') }} \ +{% endif %} +{% if etcd_cpu_limit is defined %} + --cpu-shares={{ etcd_cpu_limit|regex_replace('m', '') }} \ +{% endif %} +{% if etcd_blkio_weight is defined %} + --blkio-weight={{ etcd_blkio_weight }} \ +{% endif %} + --name={{ etcd_member_name | default("etcd") }} \ + {{ etcd_image_repo }}:{{ etcd_image_tag }} \ + /usr/local/bin/etcd \ + "$@" diff --git a/kubespray/project/roles/etcd/templates/make-ssl-etcd.sh.j2 b/kubespray/project/roles/etcd/templates/make-ssl-etcd.sh.j2 new file mode 100644 index 0000000..e0dde40 --- /dev/null +++ b/kubespray/project/roles/etcd/templates/make-ssl-etcd.sh.j2 @@ -0,0 +1,105 @@ +#!/bin/bash + +# Author: Smana smainklh@gmail.com +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o pipefail +usage() +{ + cat << EOF +Create self signed certificates + +Usage : $(basename $0) -f [-d ] + -h | --help : Show this message + -f | --config : Openssl configuration file + -d | --ssldir : Directory where the certificates will be installed + + ex : + $(basename $0) -f openssl.conf -d /srv/ssl +EOF +} + +# Options parsing +while (($#)); do + case "$1" in + -h | --help) usage; exit 0;; + -f | --config) CONFIG=${2}; shift 2;; + -d | --ssldir) SSLDIR="${2}"; shift 2;; + *) + usage + echo "ERROR : Unknown option" + exit 3 + ;; + esac +done + +if [ -z ${CONFIG} ]; then + echo "ERROR: the openssl configuration file is missing. option -f" + exit 1 +fi +if [ -z ${SSLDIR} ]; then + SSLDIR="/etc/ssl/etcd" +fi + +tmpdir=$(mktemp -d /tmp/etcd_cacert.XXXXXX) +trap 'rm -rf "${tmpdir}"' EXIT +cd "${tmpdir}" + +mkdir -p "${SSLDIR}" + +# Root CA +if [ -e "$SSLDIR/ca-key.pem" ]; then + # Reuse existing CA + cp $SSLDIR/{ca.pem,ca-key.pem} . +else + openssl genrsa -out ca-key.pem {{certificates_key_size}} > /dev/null 2>&1 + openssl req -x509 -new -nodes -key ca-key.pem -days {{certificates_duration}} -out ca.pem -subj "/CN=etcd-ca" > /dev/null 2>&1 +fi + +# ETCD member +if [ -n "$MASTERS" ]; then + for host in $MASTERS; do + cn="${host%%.*}" + # Member key + openssl genrsa -out member-${host}-key.pem {{certificates_key_size}} > /dev/null 2>&1 + openssl req -new -key member-${host}-key.pem -out member-${host}.csr -subj "/CN=etcd-member-${cn}" -config ${CONFIG} > /dev/null 2>&1 + openssl x509 -req -in member-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out member-${host}.pem -days {{certificates_duration}} -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1 + + # Admin key + openssl genrsa -out admin-${host}-key.pem {{certificates_key_size}} > /dev/null 2>&1 + openssl req -new -key admin-${host}-key.pem -out admin-${host}.csr -subj "/CN=etcd-admin-${cn}" > /dev/null 2>&1 + openssl x509 -req -in admin-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out admin-${host}.pem -days {{certificates_duration}} -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1 + done +fi + +# Node keys +if [ -n "$HOSTS" ]; then + for host in $HOSTS; do + cn="${host%%.*}" + openssl genrsa -out node-${host}-key.pem {{certificates_key_size}} > /dev/null 2>&1 + openssl req -new -key node-${host}-key.pem -out node-${host}.csr -subj "/CN=etcd-node-${cn}" > /dev/null 2>&1 + openssl x509 -req -in node-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out node-${host}.pem -days {{certificates_duration}} -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1 + done +fi + +# Install certs +if [ -e "$SSLDIR/ca-key.pem" ]; then + # No pass existing CA + rm -f ca.pem ca-key.pem +fi + +if [ -n "$(ls -A *.pem)" ]; then + mv *.pem ${SSLDIR}/ +fi diff --git a/kubespray/project/roles/etcd/templates/openssl.conf.j2 b/kubespray/project/roles/etcd/templates/openssl.conf.j2 new file mode 100644 index 0000000..f6681a1 --- /dev/null +++ b/kubespray/project/roles/etcd/templates/openssl.conf.j2 @@ -0,0 +1,45 @@ +{% set counter = {'dns': 2,'ip': 1,} %}{% macro increment(dct, key, inc=1)%}{% if dct.update({key: dct[key] + inc}) %} {% endif %}{% endmacro %}[req] +req_extensions = v3_req +distinguished_name = req_distinguished_name + +[req_distinguished_name] + +[ v3_req ] +basicConstraints = CA:FALSE +keyUsage = nonRepudiation, digitalSignature, keyEncipherment +subjectAltName = @alt_names + +[ ssl_client ] +extendedKeyUsage = clientAuth, serverAuth +basicConstraints = CA:FALSE +subjectKeyIdentifier=hash +authorityKeyIdentifier=keyid,issuer +subjectAltName = @alt_names + +[ v3_ca ] +basicConstraints = CA:TRUE +keyUsage = nonRepudiation, digitalSignature, keyEncipherment +subjectAltName = @alt_names +authorityKeyIdentifier=keyid:always,issuer + +[alt_names] +DNS.1 = localhost +{% for host in groups['etcd'] %} +DNS.{{ counter["dns"] }} = {{ host }}{{ increment(counter, 'dns') }} +{% endfor %} +{% if apiserver_loadbalancer_domain_name is defined %} +DNS.{{ counter["dns"] }} = {{ apiserver_loadbalancer_domain_name }}{{ increment(counter, 'dns') }} +{% endif %} +{% for etcd_alt_name in etcd_cert_alt_names %} +DNS.{{ counter["dns"] }} = {{ etcd_alt_name }}{{ increment(counter, 'dns') }} +{% endfor %} +{% for host in groups['etcd'] %} +{% if hostvars[host]['access_ip'] is defined %} +IP.{{ counter["ip"] }} = {{ hostvars[host]['access_ip'] }}{{ increment(counter, 'ip') }} +{% endif %} +IP.{{ counter["ip"] }} = {{ hostvars[host]['ip'] | default(fallback_ips[host]) }}{{ increment(counter, 'ip') }} +{% endfor %} +{% for cert_alt_ip in etcd_cert_alt_ips %} +IP.{{ counter["ip"] }} = {{ cert_alt_ip }}{{ increment(counter, 'ip') }} +{% endfor %} +IP.{{ counter["ip"] }} = 127.0.0.1 diff --git a/kubespray/project/roles/etcd/vars/main.yml b/kubespray/project/roles/etcd/vars/main.yml new file mode 100644 index 0000000..1e56904 --- /dev/null +++ b/kubespray/project/roles/etcd/vars/main.yml @@ -0,0 +1,10 @@ +--- +cert_files: + master: + - "{{ etcd_cert_dir }}/member-{{ inventory_hostname }}.pem" + - "{{ etcd_cert_dir }}/member-{{ inventory_hostname }}-key.pem" + - "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" + - "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" + node: + - "{{ etcd_cert_dir}}/node-{{ inventory_hostname }}.pem" + - "{{ etcd_cert_dir}}/node-{{ inventory_hostname }}-key.pem" diff --git a/kubespray/project/roles/etcdctl_etcdutl/tasks/main.yml b/kubespray/project/roles/etcdctl_etcdutl/tasks/main.yml new file mode 100644 index 0000000..be0eea4 --- /dev/null +++ b/kubespray/project/roles/etcdctl_etcdutl/tasks/main.yml @@ -0,0 +1,45 @@ +--- +- name: Copy etcdctl and etcdutl binary from docker container + command: sh -c "{{ docker_bin_dir }}/docker rm -f etcdxtl-binarycopy; + {{ docker_bin_dir }}/docker create --name etcdxtl-binarycopy {{ etcd_image_repo }}:{{ etcd_image_tag }} && + {{ docker_bin_dir }}/docker cp etcdxtl-binarycopy:/usr/local/bin/{{ item }} {{ bin_dir }}/{{ item }} && + {{ docker_bin_dir }}/docker rm -f etcdxtl-binarycopy" + with_items: + - etcdctl + - etcdutl + register: etcdxtl_install_result + until: etcdxtl_install_result.rc == 0 + retries: 4 + delay: "{{ retry_stagger | random + 3 }}" + changed_when: false + when: container_manager == "docker" + +- name: Download etcd binary + include_tasks: "../../download/tasks/download_file.yml" + vars: + download: "{{ download_defaults | combine(downloads.etcd) }}" + when: container_manager in ['crio', 'containerd'] + +- name: Copy etcd binary + unarchive: + src: "{{ downloads.etcd.dest }}" + dest: "{{ local_release_dir }}/" + remote_src: yes + when: container_manager in ['crio', 'containerd'] + +- name: Copy etcdctl and etcdutl binary from download dir + copy: + src: "{{ local_release_dir }}/etcd-{{ etcd_version }}-linux-{{ host_architecture }}/{{ item }}" + dest: "{{ bin_dir }}/{{ item }}" + mode: 0755 + remote_src: yes + with_items: + - etcdctl + - etcdutl + when: container_manager in ['crio', 'containerd'] + +- name: Create etcdctl wrapper script + template: + src: etcdctl.sh.j2 + dest: "{{ bin_dir }}/etcdctl.sh" + mode: 0755 diff --git a/kubespray/project/roles/etcdctl_etcdutl/templates/etcdctl.sh.j2 b/kubespray/project/roles/etcdctl_etcdutl/templates/etcdctl.sh.j2 new file mode 100644 index 0000000..e4ddfec --- /dev/null +++ b/kubespray/project/roles/etcdctl_etcdutl/templates/etcdctl.sh.j2 @@ -0,0 +1,14 @@ +#!/bin/bash +# {{ ansible_managed }} +# example invocation: etcdctl.sh get --keys-only --from-key "" + +etcdctl \ +{% if etcd_deployment_type == "kubeadm" %} + --cacert {{ kube_cert_dir }}/etcd/ca.crt \ + --cert {{ kube_cert_dir }}/etcd/server.crt \ + --key {{ kube_cert_dir }}/etcd/server.key "$@" +{% else %} + --cacert {{ etcd_cert_dir }}/ca.pem \ + --cert {{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem \ + --key {{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem "$@" +{% endif %} diff --git a/kubespray/project/roles/helm-apps/README.md b/kubespray/project/roles/helm-apps/README.md new file mode 100644 index 0000000..8619688 --- /dev/null +++ b/kubespray/project/roles/helm-apps/README.md @@ -0,0 +1,39 @@ +Role Name +========= + +This role is intended to be used to fetch and deploy Helm Charts as part of +cluster installation or upgrading with kubespray. + +Requirements +------------ + +The role needs to be executed on a host with access to the Kubernetes API, and +with the helm binary in place. + +Role Variables +-------------- + +See meta/argument_specs.yml + +Playbook example: + +```yaml +--- +- hosts: kube_control_plane[0] + gather_facts: no + roles: + - name: helm-apps + releases: + - name: app + namespace: app + chart_ref: simple-app/simple-app + - name: app2 + namespace: app + chart_ref: simple-app/simple-app + wait_timeout: "10m" # override the same option in `release_common_opts` + repositories: "{{ repos }}" + - name: simple-app + url: "https://blog.leiwang.info/simple-app" + release_common_opts: "{{ helm_params }}" + wait_timeout: "5m" +``` diff --git a/kubespray/project/roles/helm-apps/meta/argument_specs.yml b/kubespray/project/roles/helm-apps/meta/argument_specs.yml new file mode 100644 index 0000000..d8c2b5a --- /dev/null +++ b/kubespray/project/roles/helm-apps/meta/argument_specs.yml @@ -0,0 +1,95 @@ +--- +argument_specs: + main: + short_description: Install a list of Helm charts. + options: + releases: + type: list + elements: dict + required: true + description: | + List of dictionaries passed as arguments to kubernetes.core.helm. + Arguments passed here will override those in `helm_settings`. For + structure of the dictionary, see the documentation for + kubernetes.core.helm ansible module. + options: + chart_ref: + type: path + required: true + chart_version: + type: str + name: + type: str + required: true + namespace: + type: str + required: true + values: + type: dict + # Possibly general options + create_namespace: + type: bool + chart_repo_url: + type: str + disable_hook: + type: bool + history_max: + type: int + purge: + type: bool + replace: + type: bool + skip_crds: + type: bool + wait: + type: bool + default: true + wait_timeout: + type: str + release_state: + type: str + + repositories: + type: list + elements: dict + description: | + List of dictionaries passed as arguments to + kubernetes.core.helm_repository. + default: [] + options: + name: + type: str + required: true + password: + type: str + username: + type: str + url: + type: str + release_common_opts: + type: dict + description: | + Common arguments for every helm invocation. + default: {} + options: + create_namespace: + type: bool + default: true + chart_repo_url: + type: str + disable_hook: + type: bool + history_max: + type: int + purge: + type: bool + replace: + type: bool + skip_crds: + type: bool + wait: + type: bool + default: true + wait_timeout: + type: str + default: "5m" diff --git a/kubespray/project/roles/helm-apps/meta/main.yml b/kubespray/project/roles/helm-apps/meta/main.yml new file mode 100644 index 0000000..32ea6d4 --- /dev/null +++ b/kubespray/project/roles/helm-apps/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - role: kubernetes-apps/helm diff --git a/kubespray/project/roles/helm-apps/tasks/main.yml b/kubespray/project/roles/helm-apps/tasks/main.yml new file mode 100644 index 0000000..9515f16 --- /dev/null +++ b/kubespray/project/roles/helm-apps/tasks/main.yml @@ -0,0 +1,19 @@ +--- +- name: Add Helm repositories + kubernetes.core.helm_repository: "{{ helm_repository_defaults | combine(item) }}" # noqa args[module] + loop: "{{ repositories }}" + +- name: Update Helm repositories + kubernetes.core.helm: + state: absent + binary_path: "{{ bin_dir }}/helm" + release_name: dummy # trick needed to refresh in separate step + release_namespace: kube-system + update_repo_cache: true + when: + - repositories != [] + - helm_update + +- name: Install Helm Applications + kubernetes.core.helm: "{{ helm_defaults | combine(release_common_opts, item) }}" # noqa args[module] + loop: "{{ releases }}" diff --git a/kubespray/project/roles/helm-apps/vars/main.yml b/kubespray/project/roles/helm-apps/vars/main.yml new file mode 100644 index 0000000..acf82c2 --- /dev/null +++ b/kubespray/project/roles/helm-apps/vars/main.yml @@ -0,0 +1,12 @@ +--- +helm_update: true + +helm_defaults: + atomic: true + binary_path: "{{ bin_dir }}/helm" + +helm_repository_defaults: + binary_path: "{{ bin_dir }}/helm" + +release_common_opts: + wait_timeout: "5m" diff --git a/kubespray/project/roles/kubernetes-apps/ansible/defaults/main.yml b/kubespray/project/roles/kubernetes-apps/ansible/defaults/main.yml new file mode 100644 index 0000000..cb2317b --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ansible/defaults/main.yml @@ -0,0 +1,124 @@ +--- +# Limits for coredns + +# uncomment the line below to customize the DNS cpu limit value +# dns_cpu_limit: 300m +dns_memory_limit: 300Mi +dns_cpu_requests: 100m +dns_memory_requests: 70Mi +dns_min_replicas: "{{ [2, groups['k8s_cluster'] | length] | min }}" +dns_nodes_per_replica: 16 +dns_cores_per_replica: 256 +dns_prevent_single_point_failure: "{{ 'true' if dns_min_replicas | int > 1 else 'false' }}" +enable_coredns_reverse_dns_lookups: true +coredns_ordinal_suffix: "" +# dns_extra_tolerations: [{effect: NoSchedule, operator: "Exists"}] +coredns_deployment_nodeselector: "kubernetes.io/os: linux" +coredns_default_zone_cache_block: | + cache 30 +coredns_host_network: false +coredns_port: 53 + +coredns_pod_disruption_budget: false +# value for coredns pdb +coredns_pod_disruption_budget_max_unavailable: "30%" + +# coredns_additional_configs adds any extra configuration to coredns +# coredns_additional_configs: | +# whoami +# local + +# coredns_rewrite_block: | +# rewrite stop { +# name regex (.*)\.my\.domain {1}.svc.cluster.local +# answer name (.*)\.svc\.cluster\.local {1}.my.domain +# } + +# coredns_additional_error_config: | +# consolidate 5m ".* i/o timeout$" warning + +# Configure coredns and nodelocaldns to correctly answer DNS queries when you changed +# your 'dns_domain' and some workloads used it directly. +old_dns_domains: [] + +# dns_upstream_forward_extra_opts apply to coredns forward section as well as nodelocaldns upstream target forward section +# dns_upstream_forward_extra_opts: +# policy: sequential + +# Apply extra options to coredns kubernetes plugin +# coredns_kubernetes_extra_opts: +# - 'fallthrough example.local' + +# nodelocaldns +nodelocaldns_cpu_requests: 100m +nodelocaldns_memory_limit: 200Mi +nodelocaldns_memory_requests: 70Mi +nodelocaldns_ds_nodeselector: "kubernetes.io/os: linux" +nodelocaldns_prometheus_port: 9253 +nodelocaldns_secondary_prometheus_port: 9255 + +# Limits for dns-autoscaler +dns_autoscaler_cpu_requests: 20m +dns_autoscaler_memory_requests: 10Mi +dns_autoscaler_deployment_nodeselector: "kubernetes.io/os: linux" +# dns_autoscaler_extra_tolerations: [{effect: NoSchedule, operator: "Exists"}] + +# etcd metrics +# etcd_metrics_service_labels: +# k8s-app: etcd +# app.kubernetes.io/managed-by: Kubespray +# app: kube-prometheus-stack-kube-etcd +# release: prometheus-stack + +# Netchecker +deploy_netchecker: false +netchecker_port: 31081 +agent_report_interval: 15 +netcheck_namespace: default + +# Limits for netchecker apps +netchecker_agent_cpu_limit: 30m +netchecker_agent_memory_limit: 100M +netchecker_agent_cpu_requests: 15m +netchecker_agent_memory_requests: 64M +netchecker_server_cpu_limit: 100m +netchecker_server_memory_limit: 256M +netchecker_server_cpu_requests: 50m +netchecker_server_memory_requests: 64M +netchecker_etcd_cpu_limit: 200m +netchecker_etcd_memory_limit: 256M +netchecker_etcd_cpu_requests: 100m +netchecker_etcd_memory_requests: 128M + +# SecurityContext (user/group) +netchecker_agent_user: 1000 +netchecker_server_user: 1000 +netchecker_agent_group: 1000 +netchecker_server_group: 1000 + +# Dashboard +dashboard_replicas: 1 + +# Namespace for dashboard +dashboard_namespace: kube-system + +# Limits for dashboard +dashboard_cpu_limit: 100m +dashboard_memory_limit: 256M +dashboard_cpu_requests: 50m +dashboard_memory_requests: 64M + +# Set dashboard_use_custom_certs to true if overriding dashboard_certs_secret_name with a secret that +# contains dashboard_tls_key_file and dashboard_tls_cert_file instead of using the initContainer provisioned certs +dashboard_use_custom_certs: false +dashboard_certs_secret_name: kubernetes-dashboard-certs +dashboard_tls_key_file: dashboard.key +dashboard_tls_cert_file: dashboard.crt +dashboard_master_toleration: true + +# Override dashboard default settings +dashboard_token_ttl: 900 +dashboard_skip_login: false + +# Policy Controllers +# policy_controller_extra_tolerations: [{effect: NoSchedule, operator: "Exists"}] diff --git a/kubespray/project/roles/kubernetes-apps/ansible/tasks/coredns.yml b/kubespray/project/roles/kubernetes-apps/ansible/tasks/coredns.yml new file mode 100644 index 0000000..897c618 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ansible/tasks/coredns.yml @@ -0,0 +1,50 @@ +--- +- name: Kubernetes Apps | Lay Down CoreDNS templates + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/{{ item.file }}" + mode: 0644 + loop: + - { name: coredns, file: coredns-clusterrole.yml, type: clusterrole } + - { name: coredns, file: coredns-clusterrolebinding.yml, type: clusterrolebinding } + - { name: coredns, file: coredns-config.yml, type: configmap } + - { name: coredns, file: coredns-deployment.yml, type: deployment } + - { name: coredns, file: coredns-sa.yml, type: sa } + - { name: coredns, file: coredns-svc.yml, type: svc } + - { name: dns-autoscaler, file: dns-autoscaler.yml, type: deployment } + - { name: dns-autoscaler, file: dns-autoscaler-clusterrole.yml, type: clusterrole } + - { name: dns-autoscaler, file: dns-autoscaler-clusterrolebinding.yml, type: clusterrolebinding } + - { name: coredns, file: coredns-poddisruptionbudget.yml, type: poddisruptionbudget, condition: coredns_pod_disruption_budget } + - { name: dns-autoscaler, file: dns-autoscaler-sa.yml, type: sa } + register: coredns_manifests + vars: + clusterIP: "{{ skydns_server }}" + when: + - dns_mode in ['coredns', 'coredns_dual'] + - inventory_hostname == groups['kube_control_plane'][0] + - enable_dns_autoscaler or item.name != 'dns-autoscaler' + - item.condition | default(True) + tags: + - coredns + +- name: Kubernetes Apps | Lay Down Secondary CoreDNS Template + template: + src: "{{ item.src }}.j2" + dest: "{{ kube_config_dir }}/{{ item.file }}" + mode: 0644 + with_items: + - { name: coredns, src: coredns-deployment.yml, file: coredns-deployment-secondary.yml, type: deployment } + - { name: coredns, src: coredns-svc.yml, file: coredns-svc-secondary.yml, type: svc } + - { name: dns-autoscaler, src: dns-autoscaler.yml, file: coredns-autoscaler-secondary.yml, type: deployment } + - { name: coredns, src: coredns-poddisruptionbudget.yml, file: coredns-poddisruptionbudget-secondary.yml, type: poddisruptionbudget, condition: coredns_pod_disruption_budget } + register: coredns_secondary_manifests + vars: + clusterIP: "{{ skydns_server_secondary }}" + coredns_ordinal_suffix: "-secondary" + when: + - dns_mode == 'coredns_dual' + - inventory_hostname == groups['kube_control_plane'][0] + - enable_dns_autoscaler or item.name != 'dns-autoscaler' + - item.condition | default(True) + tags: + - coredns diff --git a/kubespray/project/roles/kubernetes-apps/ansible/tasks/dashboard.yml b/kubespray/project/roles/kubernetes-apps/ansible/tasks/dashboard.yml new file mode 100644 index 0000000..480b3db --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ansible/tasks/dashboard.yml @@ -0,0 +1,21 @@ +--- +- name: Kubernetes Apps | Lay down dashboard template + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/{{ item.file }}" + mode: 0644 + with_items: + - { file: dashboard.yml, type: deploy, name: kubernetes-dashboard } + register: manifests + when: inventory_hostname == groups['kube_control_plane'][0] + +- name: Kubernetes Apps | Start dashboard + kube: + name: "{{ item.item.name }}" + namespace: "{{ dashboard_namespace }}" + kubectl: "{{ bin_dir }}/kubectl" + resource: "{{ item.item.type }}" + filename: "{{ kube_config_dir }}/{{ item.item.file }}" + state: "latest" + with_items: "{{ manifests.results }}" + when: inventory_hostname == groups['kube_control_plane'][0] diff --git a/kubespray/project/roles/kubernetes-apps/ansible/tasks/etcd_metrics.yml b/kubespray/project/roles/kubernetes-apps/ansible/tasks/etcd_metrics.yml new file mode 100644 index 0000000..548de89 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ansible/tasks/etcd_metrics.yml @@ -0,0 +1,22 @@ +--- +- name: Kubernetes Apps | Lay down etcd_metrics templates + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/{{ item.file }}" + mode: 0644 + with_items: + - { file: etcd_metrics-endpoints.yml, type: endpoints, name: etcd-metrics } + - { file: etcd_metrics-service.yml, type: service, name: etcd-metrics } + register: manifests + when: inventory_hostname == groups['kube_control_plane'][0] + +- name: Kubernetes Apps | Start etcd_metrics + kube: + name: "{{ item.item.name }}" + namespace: kube-system + kubectl: "{{ bin_dir }}/kubectl" + resource: "{{ item.item.type }}" + filename: "{{ kube_config_dir }}/{{ item.item.file }}" + state: "latest" + with_items: "{{ manifests.results }}" + when: inventory_hostname == groups['kube_control_plane'][0] diff --git a/kubespray/project/roles/kubernetes-apps/ansible/tasks/main.yml b/kubespray/project/roles/kubernetes-apps/ansible/tasks/main.yml new file mode 100644 index 0000000..e3d82f1 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ansible/tasks/main.yml @@ -0,0 +1,73 @@ +--- +- name: Kubernetes Apps | Wait for kube-apiserver + uri: + url: "{{ kube_apiserver_endpoint }}/healthz" + validate_certs: no + client_cert: "{{ kube_apiserver_client_cert }}" + client_key: "{{ kube_apiserver_client_key }}" + register: result + until: result.status == 200 + retries: 20 + delay: 1 + when: inventory_hostname == groups['kube_control_plane'][0] + +- name: Kubernetes Apps | CoreDNS + import_tasks: "coredns.yml" + when: + - dns_mode in ['coredns', 'coredns_dual'] + - inventory_hostname == groups['kube_control_plane'][0] + tags: + - coredns + +- name: Kubernetes Apps | nodelocalDNS + import_tasks: "nodelocaldns.yml" + when: + - enable_nodelocaldns + - inventory_hostname == groups['kube_control_plane'] | first + tags: + - nodelocaldns + +- name: Kubernetes Apps | Start Resources + kube: + name: "{{ item.item.name }}" + namespace: "kube-system" + kubectl: "{{ bin_dir }}/kubectl" + resource: "{{ item.item.type }}" + filename: "{{ kube_config_dir }}/{{ item.item.file }}" + state: "latest" + with_items: + - "{{ coredns_manifests.results | default({}) }}" + - "{{ coredns_secondary_manifests.results | default({}) }}" + - "{{ nodelocaldns_manifests.results | default({}) }}" + - "{{ nodelocaldns_second_manifests.results | default({}) }}" + when: + - dns_mode != 'none' + - inventory_hostname == groups['kube_control_plane'][0] + - not item is skipped + register: resource_result + until: resource_result is succeeded + retries: 4 + delay: 5 + tags: + - coredns + - nodelocaldns + loop_control: + label: "{{ item.item.file }}" + +- name: Kubernetes Apps | Etcd metrics endpoints + import_tasks: etcd_metrics.yml + when: etcd_metrics_port is defined and etcd_metrics_service_labels is defined + tags: + - etcd_metrics + +- name: Kubernetes Apps | Netchecker + import_tasks: netchecker.yml + when: deploy_netchecker + tags: + - netchecker + +- name: Kubernetes Apps | Dashboard + import_tasks: dashboard.yml + when: dashboard_enabled + tags: + - dashboard diff --git a/kubespray/project/roles/kubernetes-apps/ansible/tasks/netchecker.yml b/kubespray/project/roles/kubernetes-apps/ansible/tasks/netchecker.yml new file mode 100644 index 0000000..0011e7f --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ansible/tasks/netchecker.yml @@ -0,0 +1,47 @@ +--- +- name: Kubernetes Apps | Check AppArmor status + command: which apparmor_parser + register: apparmor_status + when: + - inventory_hostname == groups['kube_control_plane'][0] + failed_when: false + +- name: Kubernetes Apps | Set apparmor_enabled + set_fact: + apparmor_enabled: "{{ apparmor_status.rc == 0 }}" + when: + - inventory_hostname == groups['kube_control_plane'][0] + +- name: Kubernetes Apps | Netchecker Templates list + set_fact: + netchecker_templates: + - {file: netchecker-ns.yml, type: ns, name: netchecker-namespace} + - {file: netchecker-agent-sa.yml, type: sa, name: netchecker-agent} + - {file: netchecker-agent-ds.yml, type: ds, name: netchecker-agent} + - {file: netchecker-agent-hostnet-ds.yml, type: ds, name: netchecker-agent-hostnet} + - {file: netchecker-server-sa.yml, type: sa, name: netchecker-server} + - {file: netchecker-server-clusterrole.yml, type: clusterrole, name: netchecker-server} + - {file: netchecker-server-clusterrolebinding.yml, type: clusterrolebinding, name: netchecker-server} + - {file: netchecker-server-deployment.yml, type: deployment, name: netchecker-server} + - {file: netchecker-server-svc.yml, type: svc, name: netchecker-service} + +- name: Kubernetes Apps | Lay Down Netchecker Template + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/{{ item.file }}" + mode: 0644 + with_items: "{{ netchecker_templates }}" + register: manifests + when: + - inventory_hostname == groups['kube_control_plane'][0] + +- name: Kubernetes Apps | Start Netchecker Resources + kube: + name: "{{ item.item.name }}" + namespace: "{{ netcheck_namespace }}" + kubectl: "{{ bin_dir }}/kubectl" + resource: "{{ item.item.type }}" + filename: "{{ kube_config_dir }}/{{ item.item.file }}" + state: "latest" + with_items: "{{ manifests.results }}" + when: inventory_hostname == groups['kube_control_plane'][0] and not item is skipped diff --git a/kubespray/project/roles/kubernetes-apps/ansible/tasks/nodelocaldns.yml b/kubespray/project/roles/kubernetes-apps/ansible/tasks/nodelocaldns.yml new file mode 100644 index 0000000..b438afb --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ansible/tasks/nodelocaldns.yml @@ -0,0 +1,79 @@ +--- +- name: Kubernetes Apps | set up necessary nodelocaldns parameters + set_fact: + # noqa: jinja[spacing] + primaryClusterIP: >- + {%- if dns_mode in ['coredns', 'coredns_dual'] -%} + {{ skydns_server }} + {%- elif dns_mode == 'manual' -%} + {{ manual_dns_server }} + {%- endif -%} + secondaryclusterIP: "{{ skydns_server_secondary }}" + when: + - enable_nodelocaldns + - inventory_hostname == groups['kube_control_plane'] | first + tags: + - nodelocaldns + - coredns + +- name: Kubernetes Apps | Lay Down nodelocaldns Template + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/{{ item.file }}" + mode: 0644 + with_items: + - { name: nodelocaldns, file: nodelocaldns-config.yml, type: configmap } + - { name: nodelocaldns, file: nodelocaldns-sa.yml, type: sa } + - { name: nodelocaldns, file: nodelocaldns-daemonset.yml, type: daemonset } + register: nodelocaldns_manifests + vars: + # noqa: jinja[spacing] + forwardTarget: >- + {%- if secondaryclusterIP is defined and dns_mode == 'coredns_dual' -%} + {{ primaryClusterIP }} {{ secondaryclusterIP }} + {%- else -%} + {{ primaryClusterIP }} + {%- endif -%} + upstreamForwardTarget: >- + {%- if upstream_dns_servers is defined and upstream_dns_servers | length > 0 -%} + {{ upstream_dns_servers | join(' ') }} + {%- else -%} + /etc/resolv.conf + {%- endif -%} + when: + - enable_nodelocaldns + - inventory_hostname == groups['kube_control_plane'] | first + tags: + - nodelocaldns + - coredns + +- name: Kubernetes Apps | Lay Down nodelocaldns-secondary Template + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/{{ item.file }}" + mode: 0644 + with_items: + - { name: nodelocaldns, file: nodelocaldns-second-daemonset.yml, type: daemonset } + register: nodelocaldns_second_manifests + vars: + # noqa: jinja[spacing] + forwardTarget: >- + {%- if secondaryclusterIP is defined and dns_mode == 'coredns_dual' -%} + {{ primaryClusterIP }} {{ secondaryclusterIP }} + {%- else -%} + {{ primaryClusterIP }} + {%- endif -%} + # noqa: jinja[spacing] + upstreamForwardTarget: >- + {%- if upstream_dns_servers is defined and upstream_dns_servers | length > 0 -%} + {{ upstream_dns_servers | join(' ') }} + {%- else -%} + /etc/resolv.conf + {%- endif -%} + when: + - enable_nodelocaldns + - enable_nodelocaldns_secondary + - inventory_hostname == groups['kube_control_plane'] | first + tags: + - nodelocaldns + - coredns diff --git a/kubespray/project/roles/kubernetes-apps/ansible/templates/coredns-clusterrole.yml.j2 b/kubespray/project/roles/kubernetes-apps/ansible/templates/coredns-clusterrole.yml.j2 new file mode 100644 index 0000000..d5f91ed --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ansible/templates/coredns-clusterrole.yml.j2 @@ -0,0 +1,32 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + kubernetes.io/bootstrapping: rbac-defaults + addonmanager.kubernetes.io/mode: Reconcile + name: system:coredns +rules: +- apiGroups: + - "" + resources: + - endpoints + - services + - pods + - namespaces + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch diff --git a/kubespray/project/roles/kubernetes-apps/ansible/templates/coredns-clusterrolebinding.yml.j2 b/kubespray/project/roles/kubernetes-apps/ansible/templates/coredns-clusterrolebinding.yml.j2 new file mode 100644 index 0000000..af7f684 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ansible/templates/coredns-clusterrolebinding.yml.j2 @@ -0,0 +1,18 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + kubernetes.io/bootstrapping: rbac-defaults + addonmanager.kubernetes.io/mode: EnsureExists + name: system:coredns +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:coredns +subjects: + - kind: ServiceAccount + name: coredns + namespace: kube-system diff --git a/kubespray/project/roles/kubernetes-apps/ansible/templates/coredns-config.yml.j2 b/kubespray/project/roles/kubernetes-apps/ansible/templates/coredns-config.yml.j2 new file mode 100644 index 0000000..587a4e6 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ansible/templates/coredns-config.yml.j2 @@ -0,0 +1,96 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: coredns + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: EnsureExists +data: + Corefile: | +{% if coredns_external_zones is defined and coredns_external_zones | length > 0 %} +{% for block in coredns_external_zones %} + {{ block['zones'] | join(' ') }} { + log + errors { +{% if coredns_additional_error_config is defined %} + {{ coredns_additional_error_config | indent(width=10, first=False) }} +{% endif %} + } +{% if block['rewrite'] is defined and block['rewrite'] | length > 0 %} +{% for rewrite_match in block['rewrite'] %} + rewrite {{ rewrite_match }} +{% endfor %} +{% endif %} + forward . {{ block['nameservers'] | join(' ') }} + loadbalance + cache {{ block['cache'] | default(5) }} + reload +{% if dns_etchosts | default(None) %} + hosts /etc/coredns/hosts { + fallthrough + } +{% endif %} + } +{% endfor %} +{% endif %} + .:{{ coredns_port }} { +{% if coredns_additional_configs is defined %} + {{ coredns_additional_configs | indent(width=8, first=False) }} +{% endif %} + errors { +{% if coredns_additional_error_config is defined %} + {{ coredns_additional_error_config | indent(width=10, first=False) }} +{% endif %} + } + health { + lameduck 5s + } +{% if coredns_rewrite_block is defined %} + {{ coredns_rewrite_block | indent(width=8, first=False) }} +{% endif %} +{% for old_dns_domain in old_dns_domains %} + rewrite name suffix {{ old_dns_domain }} {{ dns_domain }} answer auto +{% endfor %} + ready + kubernetes {{ dns_domain }} {% if coredns_kubernetes_extra_domains is defined %}{{ coredns_kubernetes_extra_domains }} {% endif %}{% if enable_coredns_reverse_dns_lookups %}in-addr.arpa ip6.arpa {% endif %}{ + pods insecure +{% if enable_coredns_k8s_endpoint_pod_names %} + endpoint_pod_names +{% endif %} +{% if enable_coredns_reverse_dns_lookups %} + fallthrough in-addr.arpa ip6.arpa +{% endif %} +{% if coredns_kubernetes_extra_opts is defined %} +{% for opt in coredns_kubernetes_extra_opts %} + {{ opt }} +{% endfor %} +{% endif %} + } + prometheus :9153 + forward . {{ upstream_dns_servers | join(' ') if upstream_dns_servers is defined and upstream_dns_servers | length > 0 else '/etc/resolv.conf' }} { + prefer_udp + max_concurrent 1000 +{% if dns_upstream_forward_extra_opts is defined %} +{% for optname, optvalue in dns_upstream_forward_extra_opts.items() %} + {{ optname }} {{ optvalue }} +{% endfor %} +{% endif %} + } +{% if enable_coredns_k8s_external %} + k8s_external {{ coredns_k8s_external_zone }} +{% endif %} + {{ coredns_default_zone_cache_block | indent(width=8, first=False) }} + loop + reload + loadbalance +{% if dns_etchosts | default(None) %} + hosts /etc/coredns/hosts { + fallthrough + } +{% endif %} + } +{% if dns_etchosts | default(None) %} + hosts: | + {{ dns_etchosts | indent(width=4, first=False) }} +{% endif %} diff --git a/kubespray/project/roles/kubernetes-apps/ansible/templates/coredns-deployment.yml.j2 b/kubespray/project/roles/kubernetes-apps/ansible/templates/coredns-deployment.yml.j2 new file mode 100644 index 0000000..cbdca57 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ansible/templates/coredns-deployment.yml.j2 @@ -0,0 +1,123 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: "coredns{{ coredns_ordinal_suffix }}" + namespace: kube-system + labels: + k8s-app: "kube-dns{{ coredns_ordinal_suffix }}" + addonmanager.kubernetes.io/mode: Reconcile + kubernetes.io/name: "coredns{{ coredns_ordinal_suffix }}" +spec: + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + maxSurge: 10% + selector: + matchLabels: + k8s-app: kube-dns{{ coredns_ordinal_suffix }} + template: + metadata: + labels: + k8s-app: kube-dns{{ coredns_ordinal_suffix }} + annotations: + createdby: 'kubespray' + spec: + hostNetwork: {{ coredns_host_network | default(false) }} + securityContext: + seccompProfile: + type: RuntimeDefault + nodeSelector: + {{ coredns_deployment_nodeselector }} + priorityClassName: system-cluster-critical + serviceAccountName: coredns + tolerations: + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule +{% if dns_extra_tolerations is defined %} + {{ dns_extra_tolerations | list | to_nice_yaml(indent=2) | indent(8) }} +{% endif %} + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: "kubernetes.io/hostname" + labelSelector: + matchLabels: + k8s-app: kube-dns{{ coredns_ordinal_suffix }} + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: In + values: + - "" + containers: + - name: coredns + image: "{{ coredns_image_repo }}:{{ coredns_image_tag }}" + imagePullPolicy: {{ k8s_image_pull_policy }} + resources: + # TODO: Set memory limits when we've profiled the container for large + # clusters, then set request = limit to keep this container in + # guaranteed class. Currently, this container falls into the + # "burstable" category so the kubelet doesn't backoff from restarting it. + limits: +{% if dns_cpu_limit is defined %} + cpu: {{ dns_cpu_limit }} +{% endif %} + memory: {{ dns_memory_limit }} + requests: + cpu: {{ dns_cpu_requests }} + memory: {{ dns_memory_requests }} + args: [ "-conf", "/etc/coredns/Corefile" ] + volumeMounts: + - name: config-volume + mountPath: /etc/coredns + ports: + - containerPort: {{ coredns_port }} + name: dns + protocol: UDP + - containerPort: {{ coredns_port }} + name: dns-tcp + protocol: TCP + - containerPort: 9153 + name: metrics + protocol: TCP + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - NET_BIND_SERVICE + drop: + - all + readOnlyRootFilesystem: true + livenessProbe: + httpGet: + path: /health + port: 8080 + scheme: HTTP + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 10 + readinessProbe: + httpGet: + path: /ready + port: 8181 + scheme: HTTP + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 10 + dnsPolicy: Default + volumes: + - name: config-volume + configMap: + name: coredns + items: + - key: Corefile + path: Corefile +{% if dns_etchosts | default(None) %} + - key: hosts + path: hosts +{% endif %} diff --git a/kubespray/project/roles/kubernetes-apps/ansible/templates/coredns-poddisruptionbudget.yml.j2 b/kubespray/project/roles/kubernetes-apps/ansible/templates/coredns-poddisruptionbudget.yml.j2 new file mode 100644 index 0000000..7df6b26 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ansible/templates/coredns-poddisruptionbudget.yml.j2 @@ -0,0 +1,9 @@ +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: coredns{{ coredns_ordinal_suffix }} +spec: + maxUnavailable: {{ coredns_pod_disruption_budget_max_unavailable }} + selector: + matchLabels: + k8s-app: kube-dns{{ coredns_ordinal_suffix }} diff --git a/kubespray/project/roles/kubernetes-apps/ansible/templates/coredns-sa.yml.j2 b/kubespray/project/roles/kubernetes-apps/ansible/templates/coredns-sa.yml.j2 new file mode 100644 index 0000000..64d9c4d --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ansible/templates/coredns-sa.yml.j2 @@ -0,0 +1,9 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: coredns + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile diff --git a/kubespray/project/roles/kubernetes-apps/ansible/templates/coredns-svc.yml.j2 b/kubespray/project/roles/kubernetes-apps/ansible/templates/coredns-svc.yml.j2 new file mode 100644 index 0000000..961e02a --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ansible/templates/coredns-svc.yml.j2 @@ -0,0 +1,30 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: coredns{{ coredns_ordinal_suffix }} + namespace: kube-system + labels: + k8s-app: kube-dns{{ coredns_ordinal_suffix }} + kubernetes.io/name: "coredns{{ coredns_ordinal_suffix }}" + addonmanager.kubernetes.io/mode: Reconcile + annotations: + prometheus.io/port: "9153" + prometheus.io/scrape: "true" + createdby: 'kubespray' +spec: + selector: + k8s-app: kube-dns{{ coredns_ordinal_suffix }} + clusterIP: {{ clusterIP }} + ports: + - name: dns + port: 53 + protocol: UDP + targetPort: "dns" + - name: dns-tcp + port: 53 + protocol: TCP + targetPort: "dns-tcp" + - name: metrics + port: 9153 + protocol: TCP diff --git a/kubespray/project/roles/kubernetes-apps/ansible/templates/dashboard.yml.j2 b/kubespray/project/roles/kubernetes-apps/ansible/templates/dashboard.yml.j2 new file mode 100644 index 0000000..7d28725 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ansible/templates/dashboard.yml.j2 @@ -0,0 +1,335 @@ +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Configuration to deploy release version of the Dashboard UI compatible with +# Kubernetes 1.8. +# +# Example usage: kubectl create -f + +{% if dashboard_namespace != "kube-system" %} +--- +apiVersion: v1 +kind: Namespace +metadata: + name: {{ dashboard_namespace }} + labels: + name: {{ dashboard_namespace }} +{% endif %} + +--- +# ------------------- Dashboard Secrets ------------------- # +apiVersion: v1 +kind: Secret +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard-certs + namespace: {{ dashboard_namespace }} +type: Opaque + +--- +apiVersion: v1 +kind: Secret +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard-csrf + namespace: {{ dashboard_namespace }} +type: Opaque +data: + csrf: "" + +--- +apiVersion: v1 +kind: Secret +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard-key-holder + namespace: {{ dashboard_namespace }} +type: Opaque + +--- +# ------------------- Dashboard ConfigMap ------------------- # +kind: ConfigMap +apiVersion: v1 +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard-settings + namespace: {{ dashboard_namespace }} + +--- +# ------------------- Dashboard Service Account ------------------- # + +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard + namespace: {{ dashboard_namespace }} + +--- +# ------------------- Dashboard Role & Role Binding ------------------- # +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard + namespace: {{ dashboard_namespace }} +rules: + # Allow Dashboard to get, update and delete Dashboard exclusive secrets. + - apiGroups: [""] + resources: ["secrets"] + resourceNames: ["kubernetes-dashboard-key-holder", "kubernetes-dashboard-certs", "kubernetes-dashboard-csrf"] + verbs: ["get", "update", "delete"] + # Allow Dashboard to get and update 'kubernetes-dashboard-settings' config map. + - apiGroups: [""] + resources: ["configmaps"] + resourceNames: ["kubernetes-dashboard-settings"] + verbs: ["get", "update"] + # Allow Dashboard to get metrics. + - apiGroups: [""] + resources: ["services"] + resourceNames: ["heapster", "dashboard-metrics-scraper"] + verbs: ["proxy"] + - apiGroups: [""] + resources: ["services/proxy"] + resourceNames: ["heapster", "http:heapster:", "https:heapster:", "dashboard-metrics-scraper", "http:dashboard-metrics-scraper"] + verbs: ["get"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard + namespace: {{ dashboard_namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kubernetes-dashboard +subjects: + - kind: ServiceAccount + name: kubernetes-dashboard + namespace: {{ dashboard_namespace }} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kubernetes-dashboard +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kubernetes-dashboard +subjects: + - kind: ServiceAccount + name: kubernetes-dashboard + namespace: {{ dashboard_namespace }} + +--- +# ------------------- Dashboard Deployment ------------------- # + +kind: Deployment +apiVersion: apps/v1 +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard + namespace: {{ dashboard_namespace }} +spec: + replicas: {{ dashboard_replicas }} + revisionHistoryLimit: 10 + selector: + matchLabels: + k8s-app: kubernetes-dashboard + template: + metadata: + labels: + k8s-app: kubernetes-dashboard + spec: + securityContext: + seccompProfile: + type: RuntimeDefault + priorityClassName: system-cluster-critical + containers: + - name: kubernetes-dashboard + image: {{ dashboard_image_repo }}:{{ dashboard_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + resources: + limits: + cpu: {{ dashboard_cpu_limit }} + memory: {{ dashboard_memory_limit }} + requests: + cpu: {{ dashboard_cpu_requests }} + memory: {{ dashboard_memory_requests }} + ports: + - containerPort: 8443 + protocol: TCP + args: + - --namespace={{ dashboard_namespace }} +{% if dashboard_use_custom_certs %} + - --tls-key-file={{ dashboard_tls_key_file }} + - --tls-cert-file={{ dashboard_tls_cert_file }} +{% else %} + - --auto-generate-certificates +{% endif %} +{% if dashboard_skip_login %} + - --enable-skip-login +{% endif %} + - --authentication-mode=token + # Uncomment the following line to manually specify Kubernetes API server Host + # If not specified, Dashboard will attempt to auto discover the API server and connect + # to it. Uncomment only if the default does not work. + # - --apiserver-host=http://my-address:port + - --token-ttl={{ dashboard_token_ttl }} + volumeMounts: + - name: kubernetes-dashboard-certs + mountPath: /certs + # Create on-disk volume to store exec logs + - mountPath: /tmp + name: tmp-volume + livenessProbe: + httpGet: + scheme: HTTPS + path: / + port: 8443 + initialDelaySeconds: 30 + timeoutSeconds: 30 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsUser: 1001 + runAsGroup: 2001 + volumes: + - name: kubernetes-dashboard-certs + secret: + secretName: {{ dashboard_certs_secret_name }} + - name: tmp-volume + emptyDir: {} + serviceAccountName: kubernetes-dashboard +{% if dashboard_master_toleration %} + tolerations: + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule +{% endif %} + +--- +# ------------------- Dashboard Service ------------------- # + +kind: Service +apiVersion: v1 +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard + namespace: {{ dashboard_namespace }} +spec: + ports: + - port: 443 + targetPort: 8443 + selector: + k8s-app: kubernetes-dashboard + +--- +# ------------------- Metrics Scrapper Service Account ------------------- # + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard +rules: + # Allow Metrics Scraper to get metrics from the Metrics server + - apiGroups: ["metrics.k8s.io"] + resources: ["pods", "nodes"] + verbs: ["get", "list", "watch"] + +--- + +# ------------------- Metrics Scrapper Service ------------------- # +kind: Service +apiVersion: v1 +metadata: + labels: + k8s-app: kubernetes-metrics-scraper + name: dashboard-metrics-scraper + namespace: {{ dashboard_namespace }} +spec: + ports: + - port: 8000 + targetPort: 8000 + selector: + k8s-app: kubernetes-metrics-scraper + +--- + +# ------------------- Metrics Scrapper Deployment ------------------- # +kind: Deployment +apiVersion: apps/v1 +metadata: + labels: + k8s-app: kubernetes-metrics-scraper + name: kubernetes-metrics-scraper + namespace: {{ dashboard_namespace }} +spec: + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + k8s-app: kubernetes-metrics-scraper + template: + metadata: + labels: + k8s-app: kubernetes-metrics-scraper + spec: + securityContext: + seccompProfile: + type: RuntimeDefault + priorityClassName: system-cluster-critical + containers: + - name: kubernetes-metrics-scraper + image: {{ dashboard_metrics_scraper_repo }}:{{ dashboard_metrics_scraper_tag }} + ports: + - containerPort: 8000 + protocol: TCP + livenessProbe: + httpGet: + scheme: HTTP + path: / + port: 8000 + initialDelaySeconds: 30 + timeoutSeconds: 30 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsUser: 1001 + runAsGroup: 2001 + volumeMounts: + - mountPath: /tmp + name: tmp-volume + serviceAccountName: kubernetes-dashboard + volumes: + - name: tmp-volume + emptyDir: {} +{% if dashboard_master_toleration %} + tolerations: + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule +{% endif %} diff --git a/kubespray/project/roles/kubernetes-apps/ansible/templates/dns-autoscaler-clusterrole.yml.j2 b/kubespray/project/roles/kubernetes-apps/ansible/templates/dns-autoscaler-clusterrole.yml.j2 new file mode 100644 index 0000000..ef642ce --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ansible/templates/dns-autoscaler-clusterrole.yml.j2 @@ -0,0 +1,34 @@ +--- +# Copyright 2016 The Kubernetes Authors. All rights reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: system:dns-autoscaler + labels: + addonmanager.kubernetes.io/mode: Reconcile +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["list", "watch"] + - apiGroups: [""] + resources: ["replicationcontrollers/scale"] + verbs: ["get", "update"] + - apiGroups: ["extensions", "apps"] + resources: ["deployments/scale", "replicasets/scale"] + verbs: ["get", "update"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "create"] diff --git a/kubespray/project/roles/kubernetes-apps/ansible/templates/dns-autoscaler-clusterrolebinding.yml.j2 b/kubespray/project/roles/kubernetes-apps/ansible/templates/dns-autoscaler-clusterrolebinding.yml.j2 new file mode 100644 index 0000000..da1a0a9 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ansible/templates/dns-autoscaler-clusterrolebinding.yml.j2 @@ -0,0 +1,29 @@ +--- +# Copyright 2016 The Kubernetes Authors. All rights reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: system:dns-autoscaler + labels: + addonmanager.kubernetes.io/mode: Reconcile +subjects: + - kind: ServiceAccount + name: dns-autoscaler + namespace: kube-system +roleRef: + kind: ClusterRole + name: system:dns-autoscaler + apiGroup: rbac.authorization.k8s.io diff --git a/kubespray/project/roles/kubernetes-apps/ansible/templates/dns-autoscaler-sa.yml.j2 b/kubespray/project/roles/kubernetes-apps/ansible/templates/dns-autoscaler-sa.yml.j2 new file mode 100644 index 0000000..3ce9b51 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ansible/templates/dns-autoscaler-sa.yml.j2 @@ -0,0 +1,22 @@ +--- +# Copyright 2016 The Kubernetes Authors. All rights reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +kind: ServiceAccount +apiVersion: v1 +metadata: + name: dns-autoscaler + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: Reconcile diff --git a/kubespray/project/roles/kubernetes-apps/ansible/templates/dns-autoscaler.yml.j2 b/kubespray/project/roles/kubernetes-apps/ansible/templates/dns-autoscaler.yml.j2 new file mode 100644 index 0000000..c10ebf9 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ansible/templates/dns-autoscaler.yml.j2 @@ -0,0 +1,86 @@ +--- +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: dns-autoscaler{{ coredns_ordinal_suffix }} + namespace: kube-system + labels: + k8s-app: dns-autoscaler{{ coredns_ordinal_suffix }} + addonmanager.kubernetes.io/mode: Reconcile +spec: + selector: + matchLabels: + k8s-app: dns-autoscaler{{ coredns_ordinal_suffix }} + template: + metadata: + labels: + k8s-app: dns-autoscaler{{ coredns_ordinal_suffix }} + annotations: + spec: + nodeSelector: + {{ dns_autoscaler_deployment_nodeselector }} + priorityClassName: system-cluster-critical + securityContext: + seccompProfile: + type: RuntimeDefault + supplementalGroups: [ 65534 ] + fsGroup: 65534 + nodeSelector: + kubernetes.io/os: linux + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane +{% if dns_autoscaler_extra_tolerations is defined %} + {{ dns_autoscaler_extra_tolerations | list | to_nice_yaml(indent=2) | indent(8) }} +{% endif %} + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: "kubernetes.io/hostname" + labelSelector: + matchLabels: + k8s-app: dns-autoscaler{{ coredns_ordinal_suffix }} + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: In + values: + - "" + containers: + - name: autoscaler + image: "{{ dnsautoscaler_image_repo }}:{{ dnsautoscaler_image_tag }}" + resources: + requests: + cpu: {{ dns_autoscaler_cpu_requests }} + memory: {{ dns_autoscaler_memory_requests }} + readinessProbe: + httpGet: + path: /healthz + port: 8080 + scheme: HTTP + command: + - /cluster-proportional-autoscaler + - --namespace=kube-system + - --default-params={"linear":{"preventSinglePointFailure":{{ dns_prevent_single_point_failure }},"coresPerReplica":{{ dns_cores_per_replica }},"nodesPerReplica":{{ dns_nodes_per_replica }},"min":{{ dns_min_replicas }}}} + - --logtostderr=true + - --v=2 + - --configmap=dns-autoscaler{{ coredns_ordinal_suffix }} + - --target=Deployment/coredns{{ coredns_ordinal_suffix }} + serviceAccountName: dns-autoscaler diff --git a/kubespray/project/roles/kubernetes-apps/ansible/templates/etcd_metrics-endpoints.yml.j2 b/kubespray/project/roles/kubernetes-apps/ansible/templates/etcd_metrics-endpoints.yml.j2 new file mode 100644 index 0000000..18f515d --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ansible/templates/etcd_metrics-endpoints.yml.j2 @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Endpoints +metadata: + name: etcd-metrics + namespace: kube-system + labels: + k8s-app: etcd + app.kubernetes.io/managed-by: Kubespray +subsets: +{% for etcd_metrics_address, etcd_host in etcd_metrics_addresses.split(',') | zip(etcd_hosts) %} + - addresses: + - ip: {{ etcd_metrics_address | urlsplit('hostname') }} + targetRef: + kind: Node + name: {{ etcd_host }} + ports: + - name: http-metrics + port: {{ etcd_metrics_address | urlsplit('port') }} + protocol: TCP +{% endfor %} diff --git a/kubespray/project/roles/kubernetes-apps/ansible/templates/etcd_metrics-service.yml.j2 b/kubespray/project/roles/kubernetes-apps/ansible/templates/etcd_metrics-service.yml.j2 new file mode 100644 index 0000000..5bd9254 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ansible/templates/etcd_metrics-service.yml.j2 @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + name: etcd-metrics + namespace: kube-system + labels: + {{ etcd_metrics_service_labels | to_yaml(indent=2, width=1337) | indent(width=4) }} +spec: + ports: + - name: http-metrics + protocol: TCP + port: {{ etcd_metrics_port }} + # targetPort: diff --git a/kubespray/project/roles/kubernetes-apps/ansible/templates/netchecker-agent-ds.yml.j2 b/kubespray/project/roles/kubernetes-apps/ansible/templates/netchecker-agent-ds.yml.j2 new file mode 100644 index 0000000..40dd199 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ansible/templates/netchecker-agent-ds.yml.j2 @@ -0,0 +1,56 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + app: netchecker-agent + name: netchecker-agent + namespace: {{ netcheck_namespace }} +spec: + selector: + matchLabels: + app: netchecker-agent + template: + metadata: + name: netchecker-agent + labels: + app: netchecker-agent + spec: + priorityClassName: {% if netcheck_namespace == 'kube-system' %}system-node-critical{% else %}k8s-cluster-critical{% endif %}{{ '' }} + tolerations: + - effect: NoSchedule + operator: Exists + nodeSelector: + kubernetes.io/os: linux + containers: + - name: netchecker-agent + image: "{{ netcheck_agent_image_repo }}:{{ netcheck_agent_image_tag }}" + imagePullPolicy: {{ k8s_image_pull_policy }} + env: + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: MY_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + args: + - "-v=5" + - "-alsologtostderr=true" + - "-serverendpoint=netchecker-service:8081" + - "-reportinterval={{ agent_report_interval }}" + resources: + limits: + cpu: {{ netchecker_agent_cpu_limit }} + memory: {{ netchecker_agent_memory_limit }} + requests: + cpu: {{ netchecker_agent_cpu_requests }} + memory: {{ netchecker_agent_memory_requests }} + securityContext: + runAsUser: {{ netchecker_agent_user | default('0') }} + runAsGroup: {{ netchecker_agent_group | default('0') }} + serviceAccountName: netchecker-agent + updateStrategy: + rollingUpdate: + maxUnavailable: 100% + type: RollingUpdate diff --git a/kubespray/project/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.yml.j2 b/kubespray/project/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.yml.j2 new file mode 100644 index 0000000..50e2793 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.yml.j2 @@ -0,0 +1,58 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + app: netchecker-agent-hostnet + name: netchecker-agent-hostnet + namespace: {{ netcheck_namespace }} +spec: + selector: + matchLabels: + app: netchecker-agent-hostnet + template: + metadata: + name: netchecker-agent-hostnet + labels: + app: netchecker-agent-hostnet + spec: + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + nodeSelector: + kubernetes.io/os: linux + priorityClassName: {% if netcheck_namespace == 'kube-system' %}system-node-critical{% else %}k8s-cluster-critical{% endif %}{{ '' }} + tolerations: + - effect: NoSchedule + operator: Exists + containers: + - name: netchecker-agent + image: "{{ netcheck_agent_image_repo }}:{{ netcheck_agent_image_tag }}" + imagePullPolicy: {{ k8s_image_pull_policy }} + env: + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: MY_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + args: + - "-v=5" + - "-alsologtostderr=true" + - "-serverendpoint=netchecker-service:8081" + - "-reportinterval={{ agent_report_interval }}" + resources: + limits: + cpu: {{ netchecker_agent_cpu_limit }} + memory: {{ netchecker_agent_memory_limit }} + requests: + cpu: {{ netchecker_agent_cpu_requests }} + memory: {{ netchecker_agent_memory_requests }} + securityContext: + runAsUser: {{ netchecker_agent_user | default('0') }} + runAsGroup: {{ netchecker_agent_group | default('0') }} + serviceAccountName: netchecker-agent + updateStrategy: + rollingUpdate: + maxUnavailable: 100% + type: RollingUpdate diff --git a/kubespray/project/roles/kubernetes-apps/ansible/templates/netchecker-agent-sa.yml.j2 b/kubespray/project/roles/kubernetes-apps/ansible/templates/netchecker-agent-sa.yml.j2 new file mode 100644 index 0000000..c544043 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ansible/templates/netchecker-agent-sa.yml.j2 @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: netchecker-agent + namespace: {{ netcheck_namespace }} diff --git a/kubespray/project/roles/kubernetes-apps/ansible/templates/netchecker-ns.yml.j2 b/kubespray/project/roles/kubernetes-apps/ansible/templates/netchecker-ns.yml.j2 new file mode 100644 index 0000000..3dd87aa --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ansible/templates/netchecker-ns.yml.j2 @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: "{{ netcheck_namespace }}" + labels: + name: "{{ netcheck_namespace }}" diff --git a/kubespray/project/roles/kubernetes-apps/ansible/templates/netchecker-server-clusterrole.yml.j2 b/kubespray/project/roles/kubernetes-apps/ansible/templates/netchecker-server-clusterrole.yml.j2 new file mode 100644 index 0000000..290dec3 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ansible/templates/netchecker-server-clusterrole.yml.j2 @@ -0,0 +1,9 @@ +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: netchecker-server + namespace: {{ netcheck_namespace }} +rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["list", "get"] diff --git a/kubespray/project/roles/kubernetes-apps/ansible/templates/netchecker-server-clusterrolebinding.yml.j2 b/kubespray/project/roles/kubernetes-apps/ansible/templates/netchecker-server-clusterrolebinding.yml.j2 new file mode 100644 index 0000000..55301b7 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ansible/templates/netchecker-server-clusterrolebinding.yml.j2 @@ -0,0 +1,13 @@ +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: netchecker-server + namespace: {{ netcheck_namespace }} +subjects: + - kind: ServiceAccount + name: netchecker-server + namespace: {{ netcheck_namespace }} +roleRef: + kind: ClusterRole + name: netchecker-server + apiGroup: rbac.authorization.k8s.io diff --git a/kubespray/project/roles/kubernetes-apps/ansible/templates/netchecker-server-deployment.yml.j2 b/kubespray/project/roles/kubernetes-apps/ansible/templates/netchecker-server-deployment.yml.j2 new file mode 100644 index 0000000..02fd6b6 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ansible/templates/netchecker-server-deployment.yml.j2 @@ -0,0 +1,83 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: netchecker-server + namespace: {{ netcheck_namespace }} + labels: + app: netchecker-server +spec: + replicas: 1 + selector: + matchLabels: + app: netchecker-server + template: + metadata: + name: netchecker-server + labels: + app: netchecker-server + spec: + priorityClassName: {% if netcheck_namespace == 'kube-system' %}system-cluster-critical{% else %}k8s-cluster-critical{% endif %}{{ '' }} + volumes: + - name: etcd-data + emptyDir: {} + containers: + - name: netchecker-server + image: "{{ netcheck_server_image_repo }}:{{ netcheck_server_image_tag }}" + imagePullPolicy: {{ k8s_image_pull_policy }} + resources: + limits: + cpu: {{ netchecker_server_cpu_limit }} + memory: {{ netchecker_server_memory_limit }} + requests: + cpu: {{ netchecker_server_cpu_requests }} + memory: {{ netchecker_server_memory_requests }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ['ALL'] + runAsUser: {{ netchecker_server_user | default('0') }} + runAsGroup: {{ netchecker_server_group | default('0') }} + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + ports: + - containerPort: 8081 + args: + - -v=5 + - -logtostderr + - -kubeproxyinit=false + - -endpoint=0.0.0.0:8081 + - -etcd-endpoints=http://127.0.0.1:2379 + - name: etcd + image: "{{ etcd_image_repo }}:{{ netcheck_etcd_image_tag }}" + imagePullPolicy: {{ k8s_image_pull_policy }} + command: + - etcd + - --listen-client-urls=http://127.0.0.1:2379 + - --advertise-client-urls=http://127.0.0.1:2379 + - --data-dir=/var/lib/etcd + - --enable-v2 + - --force-new-cluster + volumeMounts: + - mountPath: /var/lib/etcd + name: etcd-data + resources: + limits: + cpu: {{ netchecker_etcd_cpu_limit }} + memory: {{ netchecker_etcd_memory_limit }} + requests: + cpu: {{ netchecker_etcd_cpu_requests }} + memory: {{ netchecker_etcd_memory_requests }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ['ALL'] + runAsUser: {{ netchecker_server_user | default('0') }} + runAsGroup: {{ netchecker_server_group | default('0') }} + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + tolerations: + - effect: NoSchedule + operator: Exists + serviceAccountName: netchecker-server diff --git a/kubespray/project/roles/kubernetes-apps/ansible/templates/netchecker-server-sa.yml.j2 b/kubespray/project/roles/kubernetes-apps/ansible/templates/netchecker-server-sa.yml.j2 new file mode 100644 index 0000000..e3ec07f --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ansible/templates/netchecker-server-sa.yml.j2 @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: netchecker-server + namespace: {{ netcheck_namespace }} diff --git a/kubespray/project/roles/kubernetes-apps/ansible/templates/netchecker-server-svc.yml.j2 b/kubespray/project/roles/kubernetes-apps/ansible/templates/netchecker-server-svc.yml.j2 new file mode 100644 index 0000000..dc38946 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ansible/templates/netchecker-server-svc.yml.j2 @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: netchecker-service + namespace: {{ netcheck_namespace }} +spec: + selector: + app: netchecker-server + ports: + - + protocol: TCP + port: 8081 + targetPort: 8081 + nodePort: {{ netchecker_port }} + type: NodePort diff --git a/kubespray/project/roles/kubernetes-apps/ansible/templates/nodelocaldns-config.yml.j2 b/kubespray/project/roles/kubernetes-apps/ansible/templates/nodelocaldns-config.yml.j2 new file mode 100644 index 0000000..e8ce545 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ansible/templates/nodelocaldns-config.yml.j2 @@ -0,0 +1,182 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: nodelocaldns + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: EnsureExists + +data: + Corefile: | +{% if nodelocaldns_external_zones is defined and nodelocaldns_external_zones | length > 0 %} +{% for block in nodelocaldns_external_zones %} + {{ block['zones'] | join(' ') }} { + errors + cache {{ block['cache'] | default(30) }} + reload +{% if block['rewrite'] is defined and block['rewrite'] | length > 0 %} +{% for rewrite_match in block['rewrite'] %} + rewrite {{ rewrite_match }} +{% endfor %} +{% endif %} + loop + bind {{ nodelocaldns_ip }} + forward . {{ block['nameservers'] | join(' ') }} + prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:{{ nodelocaldns_prometheus_port }} + log +{% if dns_etchosts | default(None) %} + hosts /etc/coredns/hosts { + fallthrough + } +{% endif %} + } +{% endfor %} +{% endif %} + {{ ([dns_domain] + old_dns_domains) | join(' ') }}:53 { + errors + cache { + success 9984 30 + denial 9984 5 + } + reload + loop + bind {{ nodelocaldns_ip }} + forward . {{ forwardTarget }} { + force_tcp + } + prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:{{ nodelocaldns_prometheus_port }} + health {{ nodelocaldns_ip }}:{{ nodelocaldns_health_port }} +{% if dns_etchosts | default(None) %} + hosts /etc/coredns/hosts { + fallthrough + } +{% endif %} + } + in-addr.arpa:53 { + errors + cache 30 + reload + loop + bind {{ nodelocaldns_ip }} + forward . {{ forwardTarget }} { + force_tcp + } + prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:{{ nodelocaldns_prometheus_port }} + } + ip6.arpa:53 { + errors + cache 30 + reload + loop + bind {{ nodelocaldns_ip }} + forward . {{ forwardTarget }} { + force_tcp + } + prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:{{ nodelocaldns_prometheus_port }} + } + .:53 { + errors + cache 30 + reload + loop + bind {{ nodelocaldns_ip }} + forward . {{ upstreamForwardTarget }}{% if dns_upstream_forward_extra_opts is defined %} { +{% for optname, optvalue in dns_upstream_forward_extra_opts.items() %} + {{ optname }} {{ optvalue }} +{% endfor %} + }{% endif %} + + prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:{{ nodelocaldns_prometheus_port }} +{% if dns_etchosts | default(None) %} + hosts /etc/coredns/hosts { + fallthrough + } +{% endif %} + } +{% if enable_nodelocaldns_secondary %} + Corefile-second: | +{% if nodelocaldns_external_zones is defined and nodelocaldns_external_zones | length > 0 %} +{% for block in nodelocaldns_external_zones %} + {{ block['zones'] | join(' ') }} { + errors + cache {{ block['cache'] | default(30) }} + reload + loop + bind {{ nodelocaldns_ip }} + forward . {{ block['nameservers'] | join(' ') }} + prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:{{ nodelocaldns_secondary_prometheus_port }} + log +{% if dns_etchosts | default(None) %} + hosts /etc/coredns/hosts { + fallthrough + } +{% endif %} + } +{% endfor %} +{% endif %} + {{ dns_domain }}:53 { + errors + cache { + success 9984 30 + denial 9984 5 + } + reload + loop + bind {{ nodelocaldns_ip }} + forward . {{ forwardTarget }} { + force_tcp + } + prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:{{ nodelocaldns_secondary_prometheus_port }} + health {{ nodelocaldns_ip }}:{{ nodelocaldns_second_health_port }} +{% if dns_etchosts | default(None) %} + hosts /etc/coredns/hosts { + fallthrough + } +{% endif %} + } + in-addr.arpa:53 { + errors + cache 30 + reload + loop + bind {{ nodelocaldns_ip }} + forward . {{ forwardTarget }} { + force_tcp + } + prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:{{ nodelocaldns_secondary_prometheus_port }} + } + ip6.arpa:53 { + errors + cache 30 + reload + loop + bind {{ nodelocaldns_ip }} + forward . {{ forwardTarget }} { + force_tcp + } + prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:{{ nodelocaldns_secondary_prometheus_port }} + } + .:53 { + errors + cache 30 + reload + loop + bind {{ nodelocaldns_ip }} + forward . {{ upstreamForwardTarget }}{% if dns_upstream_forward_extra_opts is defined %} { +{% for optname, optvalue in dns_upstream_forward_extra_opts.items() %} + {{ optname }} {{ optvalue }} +{% endfor %} + }{% endif %} + + prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:{{ nodelocaldns_secondary_prometheus_port }} +{% if dns_etchosts | default(None) %} + hosts /etc/coredns/hosts { + fallthrough + } +{% endif %} + } +{% endif %} +{% if dns_etchosts | default(None) %} + hosts: | + {{ dns_etchosts | indent(width=4, first=False) }} +{% endif %} diff --git a/kubespray/project/roles/kubernetes-apps/ansible/templates/nodelocaldns-daemonset.yml.j2 b/kubespray/project/roles/kubernetes-apps/ansible/templates/nodelocaldns-daemonset.yml.j2 new file mode 100644 index 0000000..9ca15d7 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ansible/templates/nodelocaldns-daemonset.yml.j2 @@ -0,0 +1,115 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: nodelocaldns + namespace: kube-system + labels: + k8s-app: kube-dns + addonmanager.kubernetes.io/mode: Reconcile +spec: + selector: + matchLabels: + k8s-app: node-local-dns + template: + metadata: + labels: + k8s-app: node-local-dns + annotations: + prometheus.io/scrape: 'true' + prometheus.io/port: '{{ nodelocaldns_prometheus_port }}' + spec: + nodeSelector: + {{ nodelocaldns_ds_nodeselector }} + priorityClassName: system-cluster-critical + serviceAccountName: nodelocaldns + hostNetwork: true + dnsPolicy: Default # Don't use cluster DNS. + tolerations: + - effect: NoSchedule + operator: "Exists" + - effect: NoExecute + operator: "Exists" + containers: + - name: node-cache + image: "{{ nodelocaldns_image_repo }}:{{ nodelocaldns_image_tag }}" + resources: + limits: + memory: {{ nodelocaldns_memory_limit }} + requests: + cpu: {{ nodelocaldns_cpu_requests }} + memory: {{ nodelocaldns_memory_requests }} + args: + - -localip + - {{ nodelocaldns_ip }} + - -conf + - /etc/coredns/Corefile + - -upstreamsvc + - coredns +{% if enable_nodelocaldns_secondary %} + - -skipteardown +{% else %} + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + - containerPort: 9253 + name: metrics + protocol: TCP +{% endif %} + securityContext: + privileged: true +{% if nodelocaldns_bind_metrics_host_ip %} + env: + - name: MY_HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP +{% endif %} + livenessProbe: + httpGet: + host: {{ nodelocaldns_ip }} + path: /health + port: {{ nodelocaldns_health_port }} + scheme: HTTP + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 10 + readinessProbe: + httpGet: + host: {{ nodelocaldns_ip }} + path: /health + port: {{ nodelocaldns_health_port }} + scheme: HTTP + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 10 + volumeMounts: + - name: config-volume + mountPath: /etc/coredns + - name: xtables-lock + mountPath: /run/xtables.lock + volumes: + - name: config-volume + configMap: + name: nodelocaldns + items: + - key: Corefile + path: Corefile +{% if dns_etchosts | default(None) %} + - key: hosts + path: hosts +{% endif %} + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force + # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods. + terminationGracePeriodSeconds: 0 + updateStrategy: + rollingUpdate: + maxUnavailable: {{ serial | default('20%') }} + type: RollingUpdate diff --git a/kubespray/project/roles/kubernetes-apps/ansible/templates/nodelocaldns-sa.yml.j2 b/kubespray/project/roles/kubernetes-apps/ansible/templates/nodelocaldns-sa.yml.j2 new file mode 100644 index 0000000..bd962d8 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ansible/templates/nodelocaldns-sa.yml.j2 @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: nodelocaldns + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: Reconcile diff --git a/kubespray/project/roles/kubernetes-apps/ansible/templates/nodelocaldns-second-daemonset.yml.j2 b/kubespray/project/roles/kubernetes-apps/ansible/templates/nodelocaldns-second-daemonset.yml.j2 new file mode 100644 index 0000000..df9405e --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ansible/templates/nodelocaldns-second-daemonset.yml.j2 @@ -0,0 +1,103 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: nodelocaldns-second + namespace: kube-system + labels: + k8s-app: kube-dns + addonmanager.kubernetes.io/mode: Reconcile +spec: + selector: + matchLabels: + k8s-app: node-local-dns-second + template: + metadata: + labels: + k8s-app: node-local-dns-second + annotations: + prometheus.io/scrape: 'true' + prometheus.io/port: '{{ nodelocaldns_secondary_prometheus_port }}' + spec: + nodeSelector: + {{ nodelocaldns_ds_nodeselector }} + priorityClassName: system-cluster-critical + serviceAccountName: nodelocaldns + hostNetwork: true + dnsPolicy: Default # Don't use cluster DNS. + tolerations: + - effect: NoSchedule + operator: "Exists" + - effect: NoExecute + operator: "Exists" + containers: + - name: node-cache + image: "{{ nodelocaldns_image_repo }}:{{ nodelocaldns_image_tag }}" + resources: + limits: + memory: {{ nodelocaldns_memory_limit }} + requests: + cpu: {{ nodelocaldns_cpu_requests }} + memory: {{ nodelocaldns_memory_requests }} + args: [ "-localip", "{{ nodelocaldns_ip }}", "-conf", "/etc/coredns/Corefile", "-upstreamsvc", "coredns", "-skipteardown" ] + securityContext: + privileged: true +{% if nodelocaldns_bind_metrics_host_ip %} + env: + - name: MY_HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP +{% endif %} + livenessProbe: + httpGet: + host: {{ nodelocaldns_ip }} + path: /health + port: {{ nodelocaldns_health_port }} + scheme: HTTP + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 10 + readinessProbe: + httpGet: + host: {{ nodelocaldns_ip }} + path: /health + port: {{ nodelocaldns_health_port }} + scheme: HTTP + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 10 + volumeMounts: + - name: config-volume + mountPath: /etc/coredns + - name: xtables-lock + mountPath: /run/xtables.lock + lifecycle: + preStop: + exec: + command: + - sh + - -c + - sleep {{ nodelocaldns_secondary_skew_seconds }} && kill -9 1 + volumes: + - name: config-volume + configMap: + name: nodelocaldns + items: + - key: Corefile-second + path: Corefile +{% if dns_etchosts | default(None) %} + - key: hosts + path: hosts +{% endif %} + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + # Implement a time skew between the main nodelocaldns and this secondary. + # Since the two nodelocaldns instances share the :53 port, we want to keep + # at least one running at any time enven if the manifests are replaced simultaneously + terminationGracePeriodSeconds: {{ nodelocaldns_secondary_skew_seconds }} + updateStrategy: + rollingUpdate: + maxUnavailable: {{ serial | default('20%') }} + type: RollingUpdate diff --git a/kubespray/project/roles/kubernetes-apps/argocd/defaults/main.yml b/kubespray/project/roles/kubernetes-apps/argocd/defaults/main.yml new file mode 100644 index 0000000..f98a9d3 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/argocd/defaults/main.yml @@ -0,0 +1,6 @@ +--- +argocd_enabled: false +argocd_version: v2.8.4 +argocd_namespace: argocd +# argocd_admin_password: +argocd_install_url: "https://raw.githubusercontent.com/argoproj/argo-cd/{{ argocd_version }}/manifests/install.yaml" diff --git a/kubespray/project/roles/kubernetes-apps/argocd/tasks/main.yml b/kubespray/project/roles/kubernetes-apps/argocd/tasks/main.yml new file mode 100644 index 0000000..e11f097 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/argocd/tasks/main.yml @@ -0,0 +1,107 @@ +--- +- name: Kubernetes Apps | Download yq + include_tasks: "../../../download/tasks/download_file.yml" + vars: + download: "{{ download_defaults | combine(downloads.yq) }}" + +- name: Kubernetes Apps | Copy yq binary from download dir + ansible.posix.synchronize: + src: "{{ downloads.yq.dest }}" + dest: "{{ bin_dir }}/yq" + compress: no + perms: yes + owner: no + group: no + delegate_to: "{{ inventory_hostname }}" + +- name: Kubernetes Apps | Set ArgoCD template list + set_fact: + argocd_templates: + - name: namespace + file: argocd-namespace.yml + - name: install + file: argocd-install.yml + namespace: "{{ argocd_namespace }}" + url: "{{ argocd_install_url }}" + when: + - "inventory_hostname == groups['kube_control_plane'][0]" + +- name: Kubernetes Apps | Download ArgoCD remote manifests + include_tasks: "../../../download/tasks/download_file.yml" + vars: + download_argocd: + enabled: "{{ argocd_enabled }}" + file: true + dest: "{{ local_release_dir }}/{{ item.file }}" + url: "{{ item.url }}" + unarchive: false + owner: "root" + mode: 0644 + sha256: "" + download: "{{ download_defaults | combine(download_argocd) }}" + with_items: "{{ argocd_templates | selectattr('url', 'defined') | list }}" + loop_control: + label: "{{ item.file }}" + when: + - "inventory_hostname == groups['kube_control_plane'][0]" + +- name: Kubernetes Apps | Copy ArgoCD remote manifests from download dir + ansible.posix.synchronize: + src: "{{ local_release_dir }}/{{ item.file }}" + dest: "{{ kube_config_dir }}/{{ item.file }}" + compress: no + perms: yes + owner: no + group: no + delegate_to: "{{ inventory_hostname }}" + with_items: "{{ argocd_templates | selectattr('url', 'defined') | list }}" + when: + - "inventory_hostname == groups['kube_control_plane'][0]" + +- name: Kubernetes Apps | Set ArgoCD namespace for remote manifests + become: yes + command: | + {{ bin_dir }}/yq eval-all -i '.metadata.namespace="{{ argocd_namespace }}"' {{ kube_config_dir }}/{{ item.file }} + with_items: "{{ argocd_templates | selectattr('url', 'defined') | list }}" + loop_control: + label: "{{ item.file }}" + when: + - "inventory_hostname == groups['kube_control_plane'][0]" + +- name: Kubernetes Apps | Create ArgoCD manifests from templates + become: yes + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/{{ item.file }}" + mode: 0644 + with_items: "{{ argocd_templates | selectattr('url', 'undefined') | list }}" + loop_control: + label: "{{ item.file }}" + when: + - "inventory_hostname == groups['kube_control_plane'][0]" + +- name: Kubernetes Apps | Install ArgoCD + become: yes + kube: + name: ArgoCD + kubectl: "{{ bin_dir }}/kubectl" + filename: "{{ kube_config_dir }}/{{ item.file }}" + state: latest + with_items: "{{ argocd_templates }}" + when: + - "inventory_hostname == groups['kube_control_plane'][0]" + +# https://github.com/argoproj/argo-cd/blob/master/docs/faq.md#i-forgot-the-admin-password-how-do-i-reset-it +- name: Kubernetes Apps | Set ArgoCD custom admin password + become: yes + shell: | + {{ bin_dir }}/kubectl --kubeconfig /etc/kubernetes/admin.conf -n {{ argocd_namespace }} patch secret argocd-secret -p \ + '{ + "stringData": { + "admin.password": "{{ argocd_admin_password | password_hash('bcrypt') }}", + "admin.passwordMtime": "'$(date +%FT%T%Z)'" + } + }' + when: + - argocd_admin_password is defined + - "inventory_hostname == groups['kube_control_plane'][0]" diff --git a/kubespray/project/roles/kubernetes-apps/argocd/templates/argocd-namespace.yml.j2 b/kubespray/project/roles/kubernetes-apps/argocd/templates/argocd-namespace.yml.j2 new file mode 100644 index 0000000..5a3d40f --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/argocd/templates/argocd-namespace.yml.j2 @@ -0,0 +1,7 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: {{ argocd_namespace }} + labels: + app: argocd diff --git a/kubespray/project/roles/kubernetes-apps/cloud_controller/oci/defaults/main.yml b/kubespray/project/roles/kubernetes-apps/cloud_controller/oci/defaults/main.yml new file mode 100644 index 0000000..9d7ddf0 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/cloud_controller/oci/defaults/main.yml @@ -0,0 +1,6 @@ +--- + +oci_security_list_management: All +oci_use_instance_principals: false +oci_cloud_controller_version: 0.7.0 +oci_cloud_controller_pull_source: iad.ocir.io/oracle/cloud-provider-oci diff --git a/kubespray/project/roles/kubernetes-apps/cloud_controller/oci/tasks/credentials-check.yml b/kubespray/project/roles/kubernetes-apps/cloud_controller/oci/tasks/credentials-check.yml new file mode 100644 index 0000000..9eb8794 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/cloud_controller/oci/tasks/credentials-check.yml @@ -0,0 +1,67 @@ +--- + +- name: "OCI Cloud Controller | Credentials Check | oci_private_key" + fail: + msg: "oci_private_key is missing" + when: + - not oci_use_instance_principals + - oci_private_key is not defined or not oci_private_key + +- name: "OCI Cloud Controller | Credentials Check | oci_region_id" + fail: + msg: "oci_region_id is missing" + when: + - not oci_use_instance_principals + - oci_region_id is not defined or not oci_region_id + +- name: "OCI Cloud Controller | Credentials Check | oci_tenancy_id" + fail: + msg: "oci_tenancy_id is missing" + when: + - not oci_use_instance_principals + - oci_tenancy_id is not defined or not oci_tenancy_id + +- name: "OCI Cloud Controller | Credentials Check | oci_user_id" + fail: + msg: "oci_user_id is missing" + when: + - not oci_use_instance_principals + - oci_user_id is not defined or not oci_user_id + +- name: "OCI Cloud Controller | Credentials Check | oci_user_fingerprint" + fail: + msg: "oci_user_fingerprint is missing" + when: + - not oci_use_instance_principals + - oci_user_fingerprint is not defined or not oci_user_fingerprint + +- name: "OCI Cloud Controller | Credentials Check | oci_compartment_id" + fail: + msg: "oci_compartment_id is missing. This is the compartment in which the cluster resides" + when: + - oci_compartment_id is not defined or not oci_compartment_id + +- name: "OCI Cloud Controller | Credentials Check | oci_vnc_id" + fail: + msg: "oci_vnc_id is missing. This is the Virtual Cloud Network in which the cluster resides" + when: + - oci_vnc_id is not defined or not oci_vnc_id + +- name: "OCI Cloud Controller | Credentials Check | oci_subnet1_id" + fail: + msg: "oci_subnet1_id is missingg. This is the first subnet to which loadbalancers will be added" + when: + - oci_subnet1_id is not defined or not oci_subnet1_id + +- name: "OCI Cloud Controller | Credentials Check | oci_subnet2_id" + fail: + msg: "oci_subnet2_id is missing. Two subnets are required for load balancer high availability" + when: + - oci_cloud_controller_version is version_compare('0.7.0', '<') + - oci_subnet2_id is not defined or not oci_subnet2_id + +- name: "OCI Cloud Controller | Credentials Check | oci_security_list_management" + fail: + msg: "oci_security_list_management is missing, or not defined correctly. Valid options are (All, Frontend, None)." + when: + - oci_security_list_management is not defined or oci_security_list_management not in ["All", "Frontend", "None"] diff --git a/kubespray/project/roles/kubernetes-apps/cloud_controller/oci/tasks/main.yml b/kubespray/project/roles/kubernetes-apps/cloud_controller/oci/tasks/main.yml new file mode 100644 index 0000000..6bfcc25 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/cloud_controller/oci/tasks/main.yml @@ -0,0 +1,35 @@ +--- + +- name: OCI Cloud Controller | Check Oracle Cloud credentials + import_tasks: credentials-check.yml + +- name: "OCI Cloud Controller | Generate Cloud Provider Configuration" + template: + src: controller-manager-config.yml.j2 + dest: "{{ kube_config_dir }}/controller-manager-config.yml" + mode: 0644 + when: inventory_hostname == groups['kube_control_plane'][0] + +- name: "OCI Cloud Controller | Slurp Configuration" + slurp: + src: "{{ kube_config_dir }}/controller-manager-config.yml" + register: controller_manager_config + +- name: "OCI Cloud Controller | Encode Configuration" + set_fact: + controller_manager_config_base64: "{{ controller_manager_config.content }}" + when: inventory_hostname == groups['kube_control_plane'][0] + +- name: "OCI Cloud Controller | Generate Manifests" + template: + src: oci-cloud-provider.yml.j2 + dest: "{{ kube_config_dir }}/oci-cloud-provider.yml" + mode: 0644 + when: inventory_hostname == groups['kube_control_plane'][0] + +- name: "OCI Cloud Controller | Apply Manifests" + kube: + kubectl: "{{ bin_dir }}/kubectl" + filename: "{{ kube_config_dir }}/oci-cloud-provider.yml" + state: latest + when: inventory_hostname == groups['kube_control_plane'][0] diff --git a/kubespray/project/roles/kubernetes-apps/cloud_controller/oci/templates/controller-manager-config.yml.j2 b/kubespray/project/roles/kubernetes-apps/cloud_controller/oci/templates/controller-manager-config.yml.j2 new file mode 100644 index 0000000..d585de1 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/cloud_controller/oci/templates/controller-manager-config.yml.j2 @@ -0,0 +1,89 @@ +{% macro private_key() %}{{ oci_private_key }}{% endmacro %} + +{% if oci_use_instance_principals %} + # (https://docs.us-phoenix-1.oraclecloud.com/Content/Identity/Tasks/callingservicesfrominstances.htm). + # Ensure you have setup the following OCI policies and your kubernetes nodes are running within them + # allow dynamic-group [your dynamic group name] to read instance-family in compartment [your compartment name] + # allow dynamic-group [your dynamic group name] to use virtual-network-family in compartment [your compartment name] + # allow dynamic-group [your dynamic group name] to manage load-balancers in compartment [your compartment name] +useInstancePrincipals: true +{% else %} +useInstancePrincipals: false +{% endif %} + +auth: + +{% if oci_use_instance_principals %} + # This key is put here too for backwards compatibility + useInstancePrincipals: true +{% else %} + useInstancePrincipals: false + + region: {{ oci_region_id }} + tenancy: {{ oci_tenancy_id }} + user: {{ oci_user_id }} + key: | + {{ oci_private_key }} + + {% if oci_private_key_passphrase is defined %} + passphrase: {{ oci_private_key_passphrase }} + {% endif %} + + + fingerprint: {{ oci_user_fingerprint }} +{% endif %} + +# compartment configures Compartment within which the cluster resides. +compartment: {{ oci_compartment_id }} + +# vcn configures the Virtual Cloud Network (VCN) within which the cluster resides. +vcn: {{ oci_vnc_id }} + +loadBalancer: + # subnet1 configures one of two subnets to which load balancers will be added. + # OCI load balancers require two subnets to ensure high availability. + subnet1: {{ oci_subnet1_id }} +{% if oci_subnet2_id is defined %} + # subnet2 configures the second of two subnets to which load balancers will be + # added. OCI load balancers require two subnets to ensure high availability. + subnet2: {{ oci_subnet2_id }} +{% endif %} + # SecurityListManagementMode configures how security lists are managed by the CCM. + # "All" (default): Manage all required security list rules for load balancer services. + # "Frontend": Manage only security list rules for ingress to the load + # balancer. Requires that the user has setup a rule that + # allows inbound traffic to the appropriate ports for kube + # proxy health port, node port ranges, and health check port ranges. + # E.g. 10.82.0.0/16 30000-32000. + # "None": Disables all security list management. Requires that the + # user has setup a rule that allows inbound traffic to the + # appropriate ports for kube proxy health port, node port + # ranges, and health check port ranges. E.g. 10.82.0.0/16 30000-32000. + # Additionally requires the user to mange rules to allow + # inbound traffic to load balancers. + securityListManagementMode: {{ oci_security_list_management }} + +{% if oci_security_lists is defined and oci_security_lists | length > 0 %} + # Optional specification of which security lists to modify per subnet. This does not apply if security list management is off. + securityLists: +{% for subnet_ocid, list_ocid in oci_security_lists.items() %} + {{ subnet_ocid }}: {{ list_ocid }} +{% endfor %} +{% endif %} + +{% if oci_rate_limit is defined and oci_rate_limit | length > 0 %} +# Optional rate limit controls for accessing OCI API +rateLimiter: +{% if oci_rate_limit.rate_limit_qps_read %} + rateLimitQPSRead: {{ oci_rate_limit.rate_limit_qps_read }} +{% endif %} +{% if oci_rate_limit.rate_limit_qps_write %} + rateLimitQPSWrite: {{ oci_rate_limit.rate_limit_qps_write }} +{% endif %} +{% if oci_rate_limit.rate_limit_bucket_read %} + rateLimitBucketRead: {{ oci_rate_limit.rate_limit_bucket_read }} +{% endif %} +{% if oci_rate_limit.rate_limit_bucket_write %} + rateLimitBucketWrite: {{ oci_rate_limit.rate_limit_bucket_write }} +{% endif %} +{% endif %} diff --git a/kubespray/project/roles/kubernetes-apps/cloud_controller/oci/templates/oci-cloud-provider.yml.j2 b/kubespray/project/roles/kubernetes-apps/cloud_controller/oci/templates/oci-cloud-provider.yml.j2 new file mode 100644 index 0000000..e1fc11d --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/cloud_controller/oci/templates/oci-cloud-provider.yml.j2 @@ -0,0 +1,69 @@ +apiVersion: v1 +data: + cloud-provider.yaml: {{ controller_manager_config_base64 }} +kind: Secret +metadata: + name: oci-cloud-controller-manager + namespace: kube-system +type: Opaque + +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: oci-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: oci-cloud-controller-manager +spec: + selector: + matchLabels: + component: oci-cloud-controller-manager + tier: control-plane + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + component: oci-cloud-controller-manager + tier: control-plane + spec: +{% if oci_cloud_controller_pull_secret is defined %} + imagePullSecrets: + - name: {{ oci_cloud_controller_pull_secret }} +{% endif %} + serviceAccountName: cloud-controller-manager + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + volumes: + - name: cfg + secret: + secretName: oci-cloud-controller-manager + - name: kubernetes + hostPath: + path: /etc/kubernetes + containers: + - name: oci-cloud-controller-manager + image: {{ oci_cloud_controller_pull_source }}:{{ oci_cloud_controller_version }} + command: ["/usr/local/bin/oci-cloud-controller-manager"] + args: + - --cloud-config=/etc/oci/cloud-provider.yaml + - --cloud-provider=oci + - --leader-elect-resource-lock=configmaps + - -v=2 + volumeMounts: + - name: cfg + mountPath: /etc/oci + readOnly: true + - name: kubernetes + mountPath: /etc/kubernetes + readOnly: true diff --git a/kubespray/project/roles/kubernetes-apps/cluster_roles/files/k8s-cluster-critical-pc.yml b/kubespray/project/roles/kubernetes-apps/cluster_roles/files/k8s-cluster-critical-pc.yml new file mode 100644 index 0000000..479fb57 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/cluster_roles/files/k8s-cluster-critical-pc.yml @@ -0,0 +1,8 @@ +--- +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: k8s-cluster-critical +value: 1000000000 +globalDefault: false +description: "This priority class should only be used by the pods installed using kubespray." diff --git a/kubespray/project/roles/kubernetes-apps/cluster_roles/files/oci-rbac.yml b/kubespray/project/roles/kubernetes-apps/cluster_roles/files/oci-rbac.yml new file mode 100644 index 0000000..5e3b82b --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/cluster_roles/files/oci-rbac.yml @@ -0,0 +1,124 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: cloud-controller-manager + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:cloud-controller-manager +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - '*' + +- apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + +- apiGroups: + - "" + resources: + - services + verbs: + - list + - watch + - patch + +- apiGroups: + - "" + resources: + - services/status + verbs: + - update + +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + +# For leader election +- apiGroups: + - "" + resources: + - endpoints + verbs: + - create + +- apiGroups: + - "" + resources: + - endpoints + resourceNames: + - "cloud-controller-manager" + verbs: + - get + - list + - watch + - update + +- apiGroups: + - "" + resources: + - configmaps + verbs: + - create + +- apiGroups: + - "" + resources: + - configmaps + resourceNames: + - "cloud-controller-manager" + verbs: + - get + - update + +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + +# For the PVL +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - list + - watch + - patch +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: oci-cloud-controller-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager +subjects: +- kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system diff --git a/kubespray/project/roles/kubernetes-apps/cluster_roles/tasks/main.yml b/kubespray/project/roles/kubernetes-apps/cluster_roles/tasks/main.yml new file mode 100644 index 0000000..fdb3205 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/cluster_roles/tasks/main.yml @@ -0,0 +1,87 @@ +--- +- name: Kubernetes Apps | Wait for kube-apiserver + uri: + url: "{{ kube_apiserver_endpoint }}/healthz" + validate_certs: no + client_cert: "{{ kube_apiserver_client_cert }}" + client_key: "{{ kube_apiserver_client_key }}" + register: result + until: result.status == 200 + retries: 10 + delay: 6 + when: inventory_hostname == groups['kube_control_plane'][0] + +- name: Kubernetes Apps | Add ClusterRoleBinding to admit nodes + template: + src: "node-crb.yml.j2" + dest: "{{ kube_config_dir }}/node-crb.yml" + mode: 0640 + register: node_crb_manifest + when: + - rbac_enabled + - inventory_hostname == groups['kube_control_plane'][0] + +- name: Apply workaround to allow all nodes with cert O=system:nodes to register + kube: + name: "kubespray:system:node" + kubectl: "{{ bin_dir }}/kubectl" + resource: "clusterrolebinding" + filename: "{{ kube_config_dir }}/node-crb.yml" + state: latest + register: result + until: result is succeeded + retries: 10 + delay: 6 + when: + - rbac_enabled + - node_crb_manifest.changed + - inventory_hostname == groups['kube_control_plane'][0] + +- name: Kubernetes Apps | Remove old webhook ClusterRole + kube: + name: "system:node-webhook" + kubectl: "{{ bin_dir }}/kubectl" + resource: "clusterrole" + state: absent + when: + - rbac_enabled + - inventory_hostname == groups['kube_control_plane'][0] + tags: node-webhook + +- name: Kubernetes Apps | Remove old webhook ClusterRoleBinding + kube: + name: "system:node-webhook" + kubectl: "{{ bin_dir }}/kubectl" + resource: "clusterrolebinding" + state: absent + when: + - rbac_enabled + - inventory_hostname == groups['kube_control_plane'][0] + tags: node-webhook + +- name: Configure Oracle Cloud provider + include_tasks: oci.yml + tags: oci + when: + - cloud_provider is defined + - cloud_provider == 'oci' + +- name: PriorityClass | Copy k8s-cluster-critical-pc.yml file + copy: + src: k8s-cluster-critical-pc.yml + dest: "{{ kube_config_dir }}/k8s-cluster-critical-pc.yml" + mode: 0640 + when: inventory_hostname == groups['kube_control_plane'] | last + +- name: PriorityClass | Create k8s-cluster-critical + kube: + name: k8s-cluster-critical + kubectl: "{{ bin_dir }}/kubectl" + resource: "PriorityClass" + filename: "{{ kube_config_dir }}/k8s-cluster-critical-pc.yml" + state: latest + register: result + until: result is succeeded + retries: 10 + delay: 6 + when: inventory_hostname == groups['kube_control_plane'] | last diff --git a/kubespray/project/roles/kubernetes-apps/cluster_roles/tasks/oci.yml b/kubespray/project/roles/kubernetes-apps/cluster_roles/tasks/oci.yml new file mode 100644 index 0000000..eb07463 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/cluster_roles/tasks/oci.yml @@ -0,0 +1,19 @@ +--- +- name: Copy OCI RBAC Manifest + copy: + src: "oci-rbac.yml" + dest: "{{ kube_config_dir }}/oci-rbac.yml" + mode: 0640 + when: + - cloud_provider is defined + - cloud_provider == 'oci' + - inventory_hostname == groups['kube_control_plane'][0] + +- name: Apply OCI RBAC + kube: + kubectl: "{{ bin_dir }}/kubectl" + filename: "{{ kube_config_dir }}/oci-rbac.yml" + when: + - cloud_provider is defined + - cloud_provider == 'oci' + - inventory_hostname == groups['kube_control_plane'][0] diff --git a/kubespray/project/roles/kubernetes-apps/cluster_roles/templates/namespace.j2 b/kubespray/project/roles/kubernetes-apps/cluster_roles/templates/namespace.j2 new file mode 100644 index 0000000..f2e115a --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/cluster_roles/templates/namespace.j2 @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: "kube-system" diff --git a/kubespray/project/roles/kubernetes-apps/cluster_roles/templates/node-crb.yml.j2 b/kubespray/project/roles/kubernetes-apps/cluster_roles/templates/node-crb.yml.j2 new file mode 100644 index 0000000..9a4a3c4 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/cluster_roles/templates/node-crb.yml.j2 @@ -0,0 +1,17 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + kubernetes.io/bootstrapping: rbac-defaults + name: kubespray:system:node +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:node +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:nodes diff --git a/kubespray/project/roles/kubernetes-apps/cluster_roles/templates/vsphere-rbac.yml.j2 b/kubespray/project/roles/kubernetes-apps/cluster_roles/templates/vsphere-rbac.yml.j2 new file mode 100644 index 0000000..99da046 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/cluster_roles/templates/vsphere-rbac.yml.j2 @@ -0,0 +1,35 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:vsphere-cloud-provider +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:vsphere-cloud-provider +roleRef: + kind: ClusterRole + name: system:vsphere-cloud-provider + apiGroup: rbac.authorization.k8s.io +subjects: +- kind: ServiceAccount + name: vsphere-cloud-provider + namespace: kube-system diff --git a/kubespray/project/roles/kubernetes-apps/container_engine_accelerator/meta/main.yml b/kubespray/project/roles/kubernetes-apps/container_engine_accelerator/meta/main.yml new file mode 100644 index 0000000..c82c5d8 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/container_engine_accelerator/meta/main.yml @@ -0,0 +1,8 @@ +--- +dependencies: + - role: kubernetes-apps/container_engine_accelerator/nvidia_gpu + when: nvidia_accelerator_enabled + tags: + - apps + - nvidia_gpu + - container_engine_accelerator diff --git a/kubespray/project/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/defaults/main.yml b/kubespray/project/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/defaults/main.yml new file mode 100644 index 0000000..6e870e4 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/defaults/main.yml @@ -0,0 +1,14 @@ +--- +nvidia_accelerator_enabled: false +nvidia_driver_version: "390.87" +nvidia_gpu_tesla_base_url: https://us.download.nvidia.com/tesla/ +nvidia_gpu_gtx_base_url: http://us.download.nvidia.com/XFree86/Linux-x86_64/ +nvidia_gpu_flavor: tesla +nvidia_url_end: "{{ nvidia_driver_version }}/NVIDIA-Linux-x86_64-{{ nvidia_driver_version }}.run" +nvidia_driver_install_container: false +nvidia_driver_install_centos_container: atzedevries/nvidia-centos-driver-installer:2 +nvidia_driver_install_ubuntu_container: registry.k8s.io/ubuntu-nvidia-driver-installer@sha256:7df76a0f0a17294e86f691c81de6bbb7c04a1b4b3d4ea4e7e2cccdc42e1f6d63 +nvidia_driver_install_supported: false +nvidia_gpu_device_plugin_container: "registry.k8s.io/nvidia-gpu-device-plugin@sha256:0842734032018be107fa2490c98156992911e3e1f2a21e059ff0105b07dd8e9e" +nvidia_gpu_nodes: [] +nvidia_gpu_device_plugin_memory: 30Mi diff --git a/kubespray/project/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/tasks/main.yml b/kubespray/project/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/tasks/main.yml new file mode 100644 index 0000000..8cba9bf --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/tasks/main.yml @@ -0,0 +1,55 @@ +--- + +- name: Container Engine Acceleration Nvidia GPU | gather os specific variables + include_vars: "{{ item }}" + with_first_found: + - files: + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower | replace('/', '_') }}.yml" + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_release }}.yml" + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower | replace('/', '_') }}.yml" + - "{{ ansible_distribution | lower }}.yml" + - "{{ ansible_os_family | lower }}.yml" + skip: true + +- name: Container Engine Acceleration Nvidia GPU | Set fact of download url Tesla + set_fact: + nvidia_driver_download_url_default: "{{ nvidia_gpu_tesla_base_url }}{{ nvidia_url_end }}" + when: nvidia_gpu_flavor | lower == "tesla" + +- name: Container Engine Acceleration Nvidia GPU | Set fact of download url GTX + set_fact: + nvidia_driver_download_url_default: "{{ nvidia_gpu_gtx_base_url }}{{ nvidia_url_end }}" + when: nvidia_gpu_flavor | lower == "gtx" + +- name: Container Engine Acceleration Nvidia GPU | Create addon dir + file: + path: "{{ kube_config_dir }}/addons/container_engine_accelerator" + owner: root + group: root + mode: 0755 + recurse: true + +- name: Container Engine Acceleration Nvidia GPU | Create manifests for nvidia accelerators + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/addons/container_engine_accelerator/{{ item.file }}" + mode: 0644 + with_items: + - { name: nvidia-driver-install-daemonset, file: nvidia-driver-install-daemonset.yml, type: daemonset } + - { name: k8s-device-plugin-nvidia-daemonset, file: k8s-device-plugin-nvidia-daemonset.yml, type: daemonset } + register: container_engine_accelerator_manifests + when: + - inventory_hostname == groups['kube_control_plane'][0] and nvidia_driver_install_container + +- name: Container Engine Acceleration Nvidia GPU | Apply manifests for nvidia accelerators + kube: + name: "{{ item.item.name }}" + namespace: "kube-system" + kubectl: "{{ bin_dir }}/kubectl" + resource: "{{ item.item.type }}" + filename: "{{ kube_config_dir }}/addons/container_engine_accelerator/{{ item.item.file }}" + state: "latest" + with_items: + - "{{ container_engine_accelerator_manifests.results }}" + when: + - inventory_hostname == groups['kube_control_plane'][0] and nvidia_driver_install_container and nvidia_driver_install_supported diff --git a/kubespray/project/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/templates/k8s-device-plugin-nvidia-daemonset.yml.j2 b/kubespray/project/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/templates/k8s-device-plugin-nvidia-daemonset.yml.j2 new file mode 100644 index 0000000..c5a7f51 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/templates/k8s-device-plugin-nvidia-daemonset.yml.j2 @@ -0,0 +1,60 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: nvidia-gpu-device-plugin + namespace: kube-system + labels: + k8s-app: nvidia-gpu-device-plugin + addonmanager.kubernetes.io/mode: Reconcile +spec: + selector: + matchLabels: + k8s-app: nvidia-gpu-device-plugin + template: + metadata: + labels: + k8s-app: nvidia-gpu-device-plugin + spec: + priorityClassName: system-node-critical + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "nvidia.com/gpu" + operator: Exists + tolerations: + - operator: "Exists" + effect: "NoExecute" + - operator: "Exists" + effect: "NoSchedule" + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + hostPID: true + volumes: + - name: device-plugin + hostPath: + path: /var/lib/kubelet/device-plugins + - name: dev + hostPath: + path: /dev + containers: + - image: "{{ nvidia_gpu_device_plugin_container }}" + command: ["/usr/bin/nvidia-gpu-device-plugin", "-logtostderr"] + name: nvidia-gpu-device-plugin + resources: + requests: + cpu: 50m + memory: {{ nvidia_gpu_device_plugin_memory }} + limits: + cpu: 50m + memory: {{ nvidia_gpu_device_plugin_memory }} + securityContext: + privileged: true + volumeMounts: + - name: device-plugin + mountPath: /device-plugin + - name: dev + mountPath: /dev + updateStrategy: + type: RollingUpdate diff --git a/kubespray/project/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/templates/nvidia-driver-install-daemonset.yml.j2 b/kubespray/project/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/templates/nvidia-driver-install-daemonset.yml.j2 new file mode 100644 index 0000000..ea097ed --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/templates/nvidia-driver-install-daemonset.yml.j2 @@ -0,0 +1,82 @@ +# Copyright 2017 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: nvidia-driver-installer + namespace: kube-system +spec: + selector: + matchLabels: + name: nvidia-driver-installer + template: + metadata: + labels: + name: nvidia-driver-installer + spec: + priorityClassName: system-node-critical + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "nvidia.com/gpu" + operator: Exists + tolerations: + - key: "nvidia.com/gpu" + effect: "NoSchedule" + operator: "Exists" + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + hostPID: true + volumes: + - name: dev + hostPath: + path: /dev + - name: nvidia-install-dir-host + hostPath: + path: /home/kubernetes/bin/nvidia + - name: root-mount + hostPath: + path: / + initContainers: + - image: "{{ nvidia_driver_install_container }}" + name: nvidia-driver-installer + resources: + requests: + cpu: 0.15 + securityContext: + privileged: true + env: + - name: NVIDIA_INSTALL_DIR_HOST + value: /home/kubernetes/bin/nvidia + - name: NVIDIA_INSTALL_DIR_CONTAINER + value: /usr/local/nvidia + - name: ROOT_MOUNT_DIR + value: /root + - name: NVIDIA_DRIVER_VERSION + value: "{{ nvidia_driver_version }}" + - name: NVIDIA_DRIVER_DOWNLOAD_URL + value: "{{ nvidia_driver_download_url_default }}" + volumeMounts: + - name: nvidia-install-dir-host + mountPath: /usr/local/nvidia + - name: dev + mountPath: /dev + - name: root-mount + mountPath: /root + containers: + - image: "{{ pod_infra_image_repo }}:{{ pod_infra_image_tag }}" + name: pause diff --git a/kubespray/project/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/vars/centos-7.yml b/kubespray/project/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/vars/centos-7.yml new file mode 100644 index 0000000..b1ea65b --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/vars/centos-7.yml @@ -0,0 +1,3 @@ +--- +nvidia_driver_install_container: "{{ nvidia_driver_install_centos_container }}" +nvidia_driver_install_supported: true diff --git a/kubespray/project/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/vars/ubuntu-16.yml b/kubespray/project/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/vars/ubuntu-16.yml new file mode 100644 index 0000000..f1bfdfc --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/vars/ubuntu-16.yml @@ -0,0 +1,3 @@ +--- +nvidia_driver_install_container: "{{ nvidia_driver_install_ubuntu_container }}" +nvidia_driver_install_supported: true diff --git a/kubespray/project/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/vars/ubuntu-18.yml b/kubespray/project/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/vars/ubuntu-18.yml new file mode 100644 index 0000000..f1bfdfc --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/vars/ubuntu-18.yml @@ -0,0 +1,3 @@ +--- +nvidia_driver_install_container: "{{ nvidia_driver_install_ubuntu_container }}" +nvidia_driver_install_supported: true diff --git a/kubespray/project/roles/kubernetes-apps/container_runtimes/crun/tasks/main.yaml b/kubespray/project/roles/kubernetes-apps/container_runtimes/crun/tasks/main.yaml new file mode 100644 index 0000000..6690141 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/container_runtimes/crun/tasks/main.yaml @@ -0,0 +1,19 @@ +--- + +- name: Crun | Copy runtime class manifest + template: + src: runtimeclass-crun.yml + dest: "{{ kube_config_dir }}/runtimeclass-crun.yml" + mode: "0664" + when: + - inventory_hostname == groups['kube_control_plane'][0] + +- name: Crun | Apply manifests + kube: + name: "runtimeclass-crun" + kubectl: "{{ bin_dir }}/kubectl" + resource: "runtimeclass" + filename: "{{ kube_config_dir }}/runtimeclass-crun.yml" + state: "latest" + when: + - inventory_hostname == groups['kube_control_plane'][0] diff --git a/kubespray/project/roles/kubernetes-apps/container_runtimes/crun/templates/runtimeclass-crun.yml b/kubespray/project/roles/kubernetes-apps/container_runtimes/crun/templates/runtimeclass-crun.yml new file mode 100644 index 0000000..99d97e6 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/container_runtimes/crun/templates/runtimeclass-crun.yml @@ -0,0 +1,6 @@ +--- +kind: RuntimeClass +apiVersion: node.k8s.io/v1 +metadata: + name: crun +handler: crun diff --git a/kubespray/project/roles/kubernetes-apps/container_runtimes/gvisor/tasks/main.yaml b/kubespray/project/roles/kubernetes-apps/container_runtimes/gvisor/tasks/main.yaml new file mode 100644 index 0000000..90562f2 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/container_runtimes/gvisor/tasks/main.yaml @@ -0,0 +1,34 @@ +--- +- name: GVisor | Create addon dir + file: + path: "{{ kube_config_dir }}/addons/gvisor" + owner: root + group: root + mode: 0755 + recurse: true + +- name: GVisor | Templates List + set_fact: + gvisor_templates: + - { name: runtimeclass-gvisor, file: runtimeclass-gvisor.yml, type: runtimeclass } + +- name: GVisort | Create manifests + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/addons/gvisor/{{ item.file }}" + mode: 0644 + with_items: "{{ gvisor_templates }}" + register: gvisor_manifests + when: + - inventory_hostname == groups['kube_control_plane'][0] + +- name: GVisor | Apply manifests + kube: + name: "{{ item.item.name }}" + kubectl: "{{ bin_dir }}/kubectl" + resource: "{{ item.item.type }}" + filename: "{{ kube_config_dir }}/addons/gvisor/{{ item.item.file }}" + state: "latest" + with_items: "{{ gvisor_manifests.results }}" + when: + - inventory_hostname == groups['kube_control_plane'][0] diff --git a/kubespray/project/roles/kubernetes-apps/container_runtimes/gvisor/templates/runtimeclass-gvisor.yml.j2 b/kubespray/project/roles/kubernetes-apps/container_runtimes/gvisor/templates/runtimeclass-gvisor.yml.j2 new file mode 100644 index 0000000..64465fa --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/container_runtimes/gvisor/templates/runtimeclass-gvisor.yml.j2 @@ -0,0 +1,6 @@ +--- +kind: RuntimeClass +apiVersion: node.k8s.io/v1 +metadata: + name: gvisor +handler: runsc diff --git a/kubespray/project/roles/kubernetes-apps/container_runtimes/kata_containers/defaults/main.yaml b/kubespray/project/roles/kubernetes-apps/container_runtimes/kata_containers/defaults/main.yaml new file mode 100644 index 0000000..6eacb79 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/container_runtimes/kata_containers/defaults/main.yaml @@ -0,0 +1,5 @@ +--- + +kata_containers_qemu_overhead: true +kata_containers_qemu_overhead_fixed_cpu: 250m +kata_containers_qemu_overhead_fixed_memory: 160Mi diff --git a/kubespray/project/roles/kubernetes-apps/container_runtimes/kata_containers/tasks/main.yaml b/kubespray/project/roles/kubernetes-apps/container_runtimes/kata_containers/tasks/main.yaml new file mode 100644 index 0000000..a07c7c2 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/container_runtimes/kata_containers/tasks/main.yaml @@ -0,0 +1,35 @@ +--- + +- name: Kata Containers | Create addon dir + file: + path: "{{ kube_config_dir }}/addons/kata_containers" + owner: root + group: root + mode: 0755 + recurse: true + +- name: Kata Containers | Templates list + set_fact: + kata_containers_templates: + - { name: runtimeclass-kata-qemu, file: runtimeclass-kata-qemu.yml, type: runtimeclass } + +- name: Kata Containers | Create manifests + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/addons/kata_containers/{{ item.file }}" + mode: 0644 + with_items: "{{ kata_containers_templates }}" + register: kata_containers_manifests + when: + - inventory_hostname == groups['kube_control_plane'][0] + +- name: Kata Containers | Apply manifests + kube: + name: "{{ item.item.name }}" + kubectl: "{{ bin_dir }}/kubectl" + resource: "{{ item.item.type }}" + filename: "{{ kube_config_dir }}/addons/kata_containers/{{ item.item.file }}" + state: "latest" + with_items: "{{ kata_containers_manifests.results }}" + when: + - inventory_hostname == groups['kube_control_plane'][0] diff --git a/kubespray/project/roles/kubernetes-apps/container_runtimes/kata_containers/templates/runtimeclass-kata-qemu.yml.j2 b/kubespray/project/roles/kubernetes-apps/container_runtimes/kata_containers/templates/runtimeclass-kata-qemu.yml.j2 new file mode 100644 index 0000000..2240cdb --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/container_runtimes/kata_containers/templates/runtimeclass-kata-qemu.yml.j2 @@ -0,0 +1,12 @@ +--- +kind: RuntimeClass +apiVersion: node.k8s.io/v1 +metadata: + name: kata-qemu +handler: kata-qemu +{% if kata_containers_qemu_overhead %} +overhead: + podFixed: + cpu: {{ kata_containers_qemu_overhead_fixed_cpu }} + memory: {{ kata_containers_qemu_overhead_fixed_memory }} +{% endif %} diff --git a/kubespray/project/roles/kubernetes-apps/container_runtimes/meta/main.yml b/kubespray/project/roles/kubernetes-apps/container_runtimes/meta/main.yml new file mode 100644 index 0000000..8584117 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/container_runtimes/meta/main.yml @@ -0,0 +1,31 @@ +--- +dependencies: + - role: kubernetes-apps/container_runtimes/kata_containers + when: kata_containers_enabled + tags: + - apps + - kata-containers + - container-runtimes + + - role: kubernetes-apps/container_runtimes/gvisor + when: gvisor_enabled + tags: + - apps + - gvisor + - container-runtimes + + - role: kubernetes-apps/container_runtimes/crun + when: crun_enabled + tags: + - apps + - crun + - container-runtimes + + - role: kubernetes-apps/container_runtimes/youki + when: + - youki_enabled + - container_manager == 'crio' + tags: + - apps + - youki + - container-runtimes diff --git a/kubespray/project/roles/kubernetes-apps/container_runtimes/youki/tasks/main.yaml b/kubespray/project/roles/kubernetes-apps/container_runtimes/youki/tasks/main.yaml new file mode 100644 index 0000000..8ba7c7a --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/container_runtimes/youki/tasks/main.yaml @@ -0,0 +1,19 @@ +--- + +- name: Youki | Copy runtime class manifest + template: + src: runtimeclass-youki.yml + dest: "{{ kube_config_dir }}/runtimeclass-youki.yml" + mode: "0664" + when: + - inventory_hostname == groups['kube_control_plane'][0] + +- name: Youki | Apply manifests + kube: + name: "runtimeclass-youki" + kubectl: "{{ bin_dir }}/kubectl" + resource: "runtimeclass" + filename: "{{ kube_config_dir }}/runtimeclass-youki.yml" + state: "latest" + when: + - inventory_hostname == groups['kube_control_plane'][0] diff --git a/kubespray/project/roles/kubernetes-apps/container_runtimes/youki/templates/runtimeclass-youki.yml b/kubespray/project/roles/kubernetes-apps/container_runtimes/youki/templates/runtimeclass-youki.yml new file mode 100644 index 0000000..b68bd06 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/container_runtimes/youki/templates/runtimeclass-youki.yml @@ -0,0 +1,6 @@ +--- +kind: RuntimeClass +apiVersion: node.k8s.io/v1 +metadata: + name: youki +handler: youki diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/OWNERS b/kubespray/project/roles/kubernetes-apps/csi_driver/OWNERS new file mode 100644 index 0000000..6cfbaa8 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/OWNERS @@ -0,0 +1,6 @@ +# See the OWNERS docs at https://go.k8s.io/owners + +approvers: +reviewers: + - alijahnas + - luckySB diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/aws_ebs/defaults/main.yml b/kubespray/project/roles/kubernetes-apps/csi_driver/aws_ebs/defaults/main.yml new file mode 100644 index 0000000..33df37c --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/aws_ebs/defaults/main.yml @@ -0,0 +1,11 @@ +--- +aws_ebs_csi_enable_volume_scheduling: true +aws_ebs_csi_enable_volume_snapshot: false +aws_ebs_csi_enable_volume_resizing: false +aws_ebs_csi_controller_replicas: 1 +aws_ebs_csi_plugin_image_tag: latest + +# Add annotions to ebs_csi_controller. Useful if using kube2iam for role assumption +# aws_ebs_csi_annotations: +# - key: iam.amazonaws.com/role +# value: your-ebs-role-arn diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/aws_ebs/tasks/main.yml b/kubespray/project/roles/kubernetes-apps/csi_driver/aws_ebs/tasks/main.yml new file mode 100644 index 0000000..5570dcc --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/aws_ebs/tasks/main.yml @@ -0,0 +1,26 @@ +--- +- name: AWS CSI Driver | Generate Manifests + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/{{ item.file }}" + mode: 0644 + with_items: + - {name: aws-ebs-csi-driver, file: aws-ebs-csi-driver.yml} + - {name: aws-ebs-csi-controllerservice, file: aws-ebs-csi-controllerservice-rbac.yml} + - {name: aws-ebs-csi-controllerservice, file: aws-ebs-csi-controllerservice.yml} + - {name: aws-ebs-csi-nodeservice, file: aws-ebs-csi-nodeservice.yml} + register: aws_csi_manifests + when: inventory_hostname == groups['kube_control_plane'][0] + +- name: AWS CSI Driver | Apply Manifests + kube: + kubectl: "{{ bin_dir }}/kubectl" + filename: "{{ kube_config_dir }}/{{ item.item.file }}" + state: "latest" + with_items: + - "{{ aws_csi_manifests.results }}" + when: + - inventory_hostname == groups['kube_control_plane'][0] + - not item is skipped + loop_control: + label: "{{ item.item.file }}" diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-controllerservice-rbac.yml.j2 b/kubespray/project/roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-controllerservice-rbac.yml.j2 new file mode 100644 index 0000000..87bfa31 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-controllerservice-rbac.yml.j2 @@ -0,0 +1,180 @@ +# Controller Service +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ebs-csi-controller-sa + namespace: kube-system + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-external-provisioner-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-provisioner-binding +subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: ebs-external-provisioner-role + apiGroup: rbac.authorization.k8s.io + +--- + +# The permissions in this ClusterRole are tightly coupled with the version of csi-attacher used. More information about this can be found in kubernetes-csi/external-attacher. +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-external-attacher-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["patch"] + +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-attacher-binding +subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: ebs-external-attacher-role + apiGroup: rbac.authorization.k8s.io + +{% if aws_ebs_csi_enable_volume_snapshot %} +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-external-snapshotter-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create", "list", "watch", "delete"] + +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-snapshotter-binding +subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: ebs-external-snapshotter-role + apiGroup: rbac.authorization.k8s.io + +{% endif %} + +{% if aws_ebs_csi_enable_volume_resizing %} +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-external-resizer-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-resizer-binding +subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: ebs-external-resizer-role + apiGroup: rbac.authorization.k8s.io + +{% endif %} diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-controllerservice.yml.j2 b/kubespray/project/roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-controllerservice.yml.j2 new file mode 100644 index 0000000..d58490a --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-controllerservice.yml.j2 @@ -0,0 +1,131 @@ +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: ebs-csi-controller + namespace: kube-system +spec: + replicas: {{ aws_ebs_csi_controller_replicas }} + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + labels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver +{% if aws_ebs_csi_annotations is defined %} + annotations: +{% for annotation in aws_ebs_csi_annotations %} + {{ annotation.key }}: {{ annotation.value }} +{% endfor %} +{% endif %} + spec: + nodeSelector: + kubernetes.io/os: linux + serviceAccountName: ebs-csi-controller-sa + priorityClassName: system-cluster-critical + containers: + - name: ebs-plugin + image: {{ aws_ebs_csi_plugin_image_repo }}:{{ aws_ebs_csi_plugin_image_tag }} + args: + - --endpoint=$(CSI_ENDPOINT) +{% if aws_ebs_csi_extra_volume_tags is defined %} + - --extra-volume-tags={{ aws_ebs_csi_extra_volume_tags }} +{% endif %} + - --logtostderr + - --v=5 + env: + - name: CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: aws-secret + key: key_id + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: aws-secret + key: access_key + optional: true + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + ports: + - name: healthz + containerPort: 9808 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 10 + failureThreshold: 5 + - name: csi-provisioner + image: {{ csi_provisioner_image_repo }}:{{ csi_provisioner_image_tag }} + args: + - --csi-address=$(ADDRESS) + - --v=5 +{% if aws_ebs_csi_enable_volume_scheduling %} + - --feature-gates=Topology=true +{% endif %} + - --leader-election=true + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: csi-attacher + image: {{ csi_attacher_image_repo }}:{{ csi_attacher_image_tag }} + args: + - --csi-address=$(ADDRESS) + - --v=5 + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ +{% if aws_ebs_csi_enable_volume_snapshot %} + - name: csi-snapshotter + image: {{ csi_snapshotter_image_repo }}:{{ csi_snapshotter_image_tag }} + args: + - --csi-address=$(ADDRESS) + - --timeout=15s + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ +{% endif %} +{% if aws_ebs_csi_enable_volume_resizing %} + - name: csi-resizer + image: {{ csi_resizer_image_repo }}:{{ csi_resizer_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + args: + - --csi-address=$(ADDRESS) + - --v=5 + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ +{% endif %} + - name: liveness-probe + image: {{ csi_livenessprobe_image_repo }}:{{ csi_livenessprobe_image_tag }} + args: + - --csi-address=/csi/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /csi + volumes: + - name: socket-dir + emptyDir: {} diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-driver.yml.j2 b/kubespray/project/roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-driver.yml.j2 new file mode 100644 index 0000000..99c6c5b --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-driver.yml.j2 @@ -0,0 +1,8 @@ +--- +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: ebs.csi.aws.com +spec: + attachRequired: true + podInfoOnMount: false diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-nodeservice.yml.j2 b/kubespray/project/roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-nodeservice.yml.j2 new file mode 100644 index 0000000..1dc1925 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-nodeservice.yml.j2 @@ -0,0 +1,101 @@ +--- +# Node Service +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: ebs-csi-node + namespace: kube-system +spec: + selector: + matchLabels: + app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + labels: + app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver + spec: + nodeSelector: + kubernetes.io/os: linux + hostNetwork: true + priorityClassName: system-node-critical + containers: + - name: ebs-plugin + securityContext: + privileged: true + image: {{ aws_ebs_csi_plugin_image_repo }}:{{ aws_ebs_csi_plugin_image_tag }} + args: + - --endpoint=$(CSI_ENDPOINT) +{% if aws_ebs_csi_extra_volume_tags is defined %} + - --extra-volume-tags={{ aws_ebs_csi_extra_volume_tags }} +{% endif %} + - --logtostderr + - --v=5 + env: + - name: CSI_ENDPOINT + value: unix:/csi/csi.sock + volumeMounts: + - name: kubelet-dir + mountPath: /var/lib/kubelet + mountPropagation: "Bidirectional" + - name: plugin-dir + mountPath: /csi + - name: device-dir + mountPath: /dev + ports: + - name: healthz + containerPort: 9808 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 10 + failureThreshold: 5 + - name: node-driver-registrar + image: {{ csi_node_driver_registrar_image_repo }}:{{ csi_node_driver_registrar_image_tag }} + args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --v=5 + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "rm -rf /registration/ebs.csi.aws.com-reg.sock /csi/csi.sock"] + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + - name: liveness-probe + image: {{ csi_livenessprobe_image_repo }}:{{ csi_livenessprobe_image_tag }} + args: + - --csi-address=/csi/csi.sock + volumeMounts: + - name: plugin-dir + mountPath: /csi + volumes: + - name: kubelet-dir + hostPath: + path: /var/lib/kubelet + type: Directory + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/ebs.csi.aws.com/ + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + - name: device-dir + hostPath: + path: /dev + type: Directory diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/azuredisk/defaults/main.yml b/kubespray/project/roles/kubernetes-apps/csi_driver/azuredisk/defaults/main.yml new file mode 100644 index 0000000..341cc97 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/azuredisk/defaults/main.yml @@ -0,0 +1,6 @@ +--- +azure_csi_use_instance_metadata: true +azure_csi_controller_replicas: 2 +azure_csi_plugin_image_tag: latest +azure_csi_controller_affinity: {} +azure_csi_node_affinity: {} diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/azuredisk/tasks/azure-credential-check.yml b/kubespray/project/roles/kubernetes-apps/csi_driver/azuredisk/tasks/azure-credential-check.yml new file mode 100644 index 0000000..0a858ee --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/azuredisk/tasks/azure-credential-check.yml @@ -0,0 +1,54 @@ +--- +- name: Azure CSI Driver | check azure_csi_tenant_id value + fail: + msg: "azure_csi_tenant_id is missing" + when: azure_csi_tenant_id is not defined or not azure_csi_tenant_id + +- name: Azure CSI Driver | check azure_csi_subscription_id value + fail: + msg: "azure_csi_subscription_id is missing" + when: azure_csi_subscription_id is not defined or not azure_csi_subscription_id + +- name: Azure CSI Driver | check azure_csi_aad_client_id value + fail: + msg: "azure_csi_aad_client_id is missing" + when: azure_csi_aad_client_id is not defined or not azure_csi_aad_client_id + +- name: Azure CSI Driver | check azure_csi_aad_client_secret value + fail: + msg: "azure_csi_aad_client_secret is missing" + when: azure_csi_aad_client_secret is not defined or not azure_csi_aad_client_secret + +- name: Azure CSI Driver | check azure_csi_resource_group value + fail: + msg: "azure_csi_resource_group is missing" + when: azure_csi_resource_group is not defined or not azure_csi_resource_group + +- name: Azure CSI Driver | check azure_csi_location value + fail: + msg: "azure_csi_location is missing" + when: azure_csi_location is not defined or not azure_csi_location + +- name: Azure CSI Driver | check azure_csi_subnet_name value + fail: + msg: "azure_csi_subnet_name is missing" + when: azure_csi_subnet_name is not defined or not azure_csi_subnet_name + +- name: Azure CSI Driver | check azure_csi_security_group_name value + fail: + msg: "azure_csi_security_group_name is missing" + when: azure_csi_security_group_name is not defined or not azure_csi_security_group_name + +- name: Azure CSI Driver | check azure_csi_vnet_name value + fail: + msg: "azure_csi_vnet_name is missing" + when: azure_csi_vnet_name is not defined or not azure_csi_vnet_name + +- name: Azure CSI Driver | check azure_csi_vnet_resource_group value + fail: + msg: "azure_csi_vnet_resource_group is missing" + when: azure_csi_vnet_resource_group is not defined or not azure_csi_vnet_resource_group + +- name: "Azure CSI Driver | check azure_csi_use_instance_metadata is a bool" + assert: + that: azure_csi_use_instance_metadata | type_debug == 'bool' diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/azuredisk/tasks/main.yml b/kubespray/project/roles/kubernetes-apps/csi_driver/azuredisk/tasks/main.yml new file mode 100644 index 0000000..a94656f --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/azuredisk/tasks/main.yml @@ -0,0 +1,45 @@ +--- +- name: Azure CSI Driver | Check Azure credentials + include_tasks: azure-credential-check.yml + +- name: Azure CSI Driver | Write Azure CSI cloud-config + template: + src: "azure-csi-cloud-config.j2" + dest: "{{ kube_config_dir }}/azure_csi_cloud_config" + group: "{{ kube_cert_group }}" + mode: 0640 + when: inventory_hostname == groups['kube_control_plane'][0] + +- name: Azure CSI Driver | Get base64 cloud-config + slurp: + src: "{{ kube_config_dir }}/azure_csi_cloud_config" + register: cloud_config_secret + when: inventory_hostname == groups['kube_control_plane'][0] + +- name: Azure CSI Driver | Generate Manifests + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/{{ item.file }}" + mode: 0644 + with_items: + - {name: azure-csi-azuredisk-driver, file: azure-csi-azuredisk-driver.yml} + - {name: azure-csi-cloud-config-secret, file: azure-csi-cloud-config-secret.yml} + - {name: azure-csi-azuredisk-controller, file: azure-csi-azuredisk-controller-rbac.yml} + - {name: azure-csi-azuredisk-controller, file: azure-csi-azuredisk-controller.yml} + - {name: azure-csi-azuredisk-node-rbac, file: azure-csi-azuredisk-node-rbac.yml} + - {name: azure-csi-azuredisk-node, file: azure-csi-azuredisk-node.yml} + register: azure_csi_manifests + when: inventory_hostname == groups['kube_control_plane'][0] + +- name: Azure CSI Driver | Apply Manifests + kube: + kubectl: "{{ bin_dir }}/kubectl" + filename: "{{ kube_config_dir }}/{{ item.item.file }}" + state: "latest" + with_items: + - "{{ azure_csi_manifests.results }}" + when: + - inventory_hostname == groups['kube_control_plane'][0] + - not item is skipped + loop_control: + label: "{{ item.item.file }}" diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-controller-rbac.yml.j2 b/kubespray/project/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-controller-rbac.yml.j2 new file mode 100644 index 0000000..16f4c98 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-controller-rbac.yml.j2 @@ -0,0 +1,230 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-azuredisk-controller-sa + namespace: kube-system +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azuredisk-external-provisioner-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["get", "list"] +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azuredisk-csi-provisioner-binding +subjects: + - kind: ServiceAccount + name: csi-azuredisk-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: azuredisk-external-provisioner-role + apiGroup: rbac.authorization.k8s.io + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azuredisk-external-attacher-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csinodeinfos"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "update", "patch"] +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azuredisk-csi-attacher-binding +subjects: + - kind: ServiceAccount + name: csi-azuredisk-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: azuredisk-external-attacher-role + apiGroup: rbac.authorization.k8s.io + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azuredisk-cluster-driver-registrar-role +rules: + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create", "list", "watch", "delete"] + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csidrivers"] + verbs: ["create", "delete"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "update", "patch"] +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azuredisk-csi-driver-registrar-binding +subjects: + - kind: ServiceAccount + name: csi-azuredisk-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: azuredisk-cluster-driver-registrar-role + apiGroup: rbac.authorization.k8s.io + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azuredisk-external-snapshotter-role +rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create", "list", "watch", "delete"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azuredisk-csi-snapshotter-binding +subjects: + - kind: ServiceAccount + name: csi-azuredisk-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: azuredisk-external-snapshotter-role + apiGroup: rbac.authorization.k8s.io +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azuredisk-external-resizer-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azuredisk-csi-resizer-role +subjects: + - kind: ServiceAccount + name: csi-azuredisk-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: azuredisk-external-resizer-role + apiGroup: rbac.authorization.k8s.io + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-azuredisk-controller-secret-role +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-azuredisk-controller-secret-binding +subjects: + - kind: ServiceAccount + name: csi-azuredisk-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-azuredisk-controller-secret-role + apiGroup: rbac.authorization.k8s.io diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-controller.yml.j2 b/kubespray/project/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-controller.yml.j2 new file mode 100644 index 0000000..13e9b02 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-controller.yml.j2 @@ -0,0 +1,177 @@ +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: csi-azuredisk-controller + namespace: kube-system +spec: + replicas: {{ azure_csi_controller_replicas }} + selector: + matchLabels: + app: csi-azuredisk-controller + template: + metadata: + labels: + app: csi-azuredisk-controller + spec: + hostNetwork: true + serviceAccountName: csi-azuredisk-controller-sa + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + tolerations: + - key: "node-role.kubernetes.io/control-plane" + effect: "NoSchedule" +{% if azure_csi_controller_affinity %} + affinity: + {{ azure_csi_controller_affinity | to_nice_yaml | indent(width=8) }} +{% endif %} + containers: + - name: csi-provisioner + image: {{ azure_csi_image_repo }}/csi-provisioner:{{ azure_csi_provisioner_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + args: + - "--feature-gates=Topology=true" + - "--csi-address=$(ADDRESS)" + - "--v=2" + - "--timeout=15s" + - "--leader-election" + - "--worker-threads=40" + - "--extra-create-metadata=true" + - "--strict-topology=true" + env: + - name: ADDRESS + value: /csi/csi.sock + volumeMounts: + - mountPath: /csi + name: socket-dir + resources: + limits: + memory: 500Mi + requests: + cpu: 10m + memory: 20Mi + - name: csi-attacher + image: {{ azure_csi_image_repo }}/csi-attacher:{{ azure_csi_attacher_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + args: + - "-v=2" + - "-csi-address=$(ADDRESS)" + - "-timeout=600s" + - "-leader-election" + - "-worker-threads=500" + env: + - name: ADDRESS + value: /csi/csi.sock + volumeMounts: + - mountPath: /csi + name: socket-dir + resources: + limits: + memory: 500Mi + requests: + cpu: 10m + memory: 20Mi + - name: csi-snapshotter + image: {{ azure_csi_image_repo }}/csi-snapshotter:{{ azure_csi_snapshotter_image_tag }} + args: + - "-csi-address=$(ADDRESS)" + - "-leader-election" + - "-v=2" + env: + - name: ADDRESS + value: /csi/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: + limits: + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + - name: csi-resizer + image: {{ azure_csi_image_repo }}/csi-resizer:{{ azure_csi_resizer_image_tag }} + args: + - "-csi-address=$(ADDRESS)" + - "-v=2" + - "-leader-election" + - '-handle-volume-inuse-error=false' + - "-timeout=60s" + env: + - name: ADDRESS + value: /csi/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: + limits: + memory: 500Mi + requests: + cpu: 10m + memory: 20Mi + - name: liveness-probe + image: {{ azure_csi_image_repo }}/livenessprobe:{{ azure_csi_livenessprobe_image_tag }} + args: + - --csi-address=/csi/csi.sock + - --probe-timeout=3s + - --health-port=29602 + - --v=2 + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: + limits: + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + - name: azuredisk + image: {{ azure_csi_plugin_image_repo }}/azuredisk-csi:{{ azure_csi_plugin_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + args: + - "--v=5" + - "--endpoint=$(CSI_ENDPOINT)" + - "--metrics-address=0.0.0.0:29604" + - "--disable-avset-nodes=true" + - "--drivername=disk.csi.azure.com" + - "--cloud-config-secret-name=cloud-config" + - "--cloud-config-secret-namespace=kube-system" + ports: + - containerPort: 29602 + name: healthz + protocol: TCP + - containerPort: 29604 + name: metrics + protocol: TCP + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 30 + timeoutSeconds: 10 + periodSeconds: 30 + env: + - name: AZURE_CREDENTIAL_FILE + value: "/etc/kubernetes/azure.json" + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + volumeMounts: + - mountPath: /csi + name: socket-dir + - mountPath: /etc/kubernetes/ + name: azure-cred + readOnly: true + resources: + limits: + memory: 500Mi + requests: + cpu: 10m + memory: 20Mi + volumes: + - name: socket-dir + emptyDir: {} + - name: azure-cred + secret: + secretName: cloud-config diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-driver.yml.j2 b/kubespray/project/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-driver.yml.j2 new file mode 100644 index 0000000..c7cba34 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-driver.yml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: disk.csi.azure.com +spec: + attachRequired: true + podInfoOnMount: true + volumeLifecycleModes: # added in Kubernetes 1.16 + - Persistent diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-node-rbac.yml.j2 b/kubespray/project/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-node-rbac.yml.j2 new file mode 100644 index 0000000..d55ea0d --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-node-rbac.yml.j2 @@ -0,0 +1,30 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-azuredisk-node-sa + namespace: kube-system + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-azuredisk-node-secret-role +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-azuredisk-node-secret-binding +subjects: + - kind: ServiceAccount + name: csi-azuredisk-node-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-azuredisk-node-secret-role + apiGroup: rbac.authorization.k8s.io diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-node.yml.j2 b/kubespray/project/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-node.yml.j2 new file mode 100644 index 0000000..4d80319 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-node.yml.j2 @@ -0,0 +1,168 @@ +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: csi-azuredisk-node + namespace: kube-system +spec: + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate + selector: + matchLabels: + app: csi-azuredisk-node + template: + metadata: + labels: + app: csi-azuredisk-node + spec: + hostNetwork: true + dnsPolicy: Default + serviceAccountName: csi-azuredisk-node-sa + nodeSelector: + kubernetes.io/os: linux +{% if azure_csi_node_affinity %} + affinity: + {{ azure_csi_node_affinity | to_nice_yaml | indent(width=8) }} +{% endif %} + priorityClassName: system-node-critical + tolerations: + - operator: Exists + containers: + - name: liveness-probe + volumeMounts: + - mountPath: /csi + name: socket-dir + image: {{ azure_csi_image_repo }}/livenessprobe:{{ azure_csi_livenessprobe_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + args: + - --csi-address=/csi/csi.sock + - --probe-timeout=3s + - --health-port=29603 + - --v=2 + resources: + limits: + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + - name: node-driver-registrar + image: {{ azure_csi_image_repo }}/csi-node-driver-registrar:{{ azure_csi_node_registrar_image_tag }} + args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --v=2 + livenessProbe: + exec: + command: + - /csi-node-driver-registrar + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --mode=kubelet-registration-probe + initialDelaySeconds: 30 + timeoutSeconds: 15 + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/disk.csi.azure.com/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + resources: + limits: + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + - name: azuredisk + image: {{ azure_csi_plugin_image_repo }}/azuredisk-csi:{{ azure_csi_plugin_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + args: + - "--v=5" + - "--endpoint=$(CSI_ENDPOINT)" + - "--nodeid=$(KUBE_NODE_NAME)" + - "--metrics-address=0.0.0.0:29605" + - "--enable-perf-optimization=true" + - "--drivername=disk.csi.azure.com" + - "--volume-attach-limit=-1" + - "--cloud-config-secret-name=cloud-config" + - "--cloud-config-secret-namespace=kube-system" + ports: + - containerPort: 29603 + name: healthz + protocol: TCP + - containerPort: 29605 + name: metrics + protocol: TCP + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 30 + timeoutSeconds: 10 + periodSeconds: 30 + env: + - name: AZURE_CREDENTIAL_FILE + value: "/etc/kubernetes/azure.json" + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + securityContext: + privileged: true + volumeMounts: + - mountPath: /csi + name: socket-dir + - mountPath: /var/lib/kubelet/ + mountPropagation: Bidirectional + name: mountpoint-dir + - mountPath: /etc/kubernetes/ + name: azure-cred + - mountPath: /dev + name: device-dir + - mountPath: /sys/bus/scsi/devices + name: sys-devices-dir + - mountPath: /sys/class/scsi_host/ + name: scsi-host-dir + resources: + limits: + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + volumes: + - hostPath: + path: /var/lib/kubelet/plugins/disk.csi.azure.com + type: DirectoryOrCreate + name: socket-dir + - hostPath: + path: /var/lib/kubelet/ + type: DirectoryOrCreate + name: mountpoint-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: DirectoryOrCreate + name: registration-dir + - secret: + defaultMode: 0644 + secretName: cloud-config + name: azure-cred + - hostPath: + path: /dev + type: Directory + name: device-dir + - hostPath: + path: /sys/bus/scsi/devices + type: Directory + name: sys-devices-dir + - hostPath: + path: /sys/class/scsi_host/ + type: Directory + name: scsi-host-dir diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-cloud-config-secret.yml.j2 b/kubespray/project/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-cloud-config-secret.yml.j2 new file mode 100644 index 0000000..f259cec --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-cloud-config-secret.yml.j2 @@ -0,0 +1,7 @@ +kind: Secret +apiVersion: v1 +metadata: + name: cloud-config + namespace: kube-system +data: + azure.json: {{ cloud_config_secret.content }} diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-cloud-config.j2 b/kubespray/project/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-cloud-config.j2 new file mode 100644 index 0000000..d3932f5 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-cloud-config.j2 @@ -0,0 +1,14 @@ +{ + "cloud":"AzurePublicCloud", + "tenantId": "{{ azure_csi_tenant_id }}", + "subscriptionId": "{{ azure_csi_subscription_id }}", + "aadClientId": "{{ azure_csi_aad_client_id }}", + "aadClientSecret": "{{ azure_csi_aad_client_secret }}", + "location": "{{ azure_csi_location }}", + "resourceGroup": "{{ azure_csi_resource_group }}", + "vnetName": "{{ azure_csi_vnet_name }}", + "vnetResourceGroup": "{{ azure_csi_vnet_resource_group }}", + "subnetName": "{{ azure_csi_subnet_name }}", + "securityGroupName": "{{ azure_csi_security_group_name }}", + "useInstanceMetadata": {{ azure_csi_use_instance_metadata }}, +} diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/cinder/defaults/main.yml b/kubespray/project/roles/kubernetes-apps/csi_driver/cinder/defaults/main.yml new file mode 100644 index 0000000..8776c30 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/cinder/defaults/main.yml @@ -0,0 +1,37 @@ +--- + +cinder_csi_attacher_image_tag: "v4.4.2" +cinder_csi_provisioner_image_tag: "v3.6.2" +cinder_csi_snapshotter_image_tag: "v6.3.2" +cinder_csi_resizer_image_tag: "v1.9.2" +cinder_csi_livenessprobe_image_tag: "v2.11.0" + +# To access Cinder, the CSI controller will need credentials to access +# openstack apis. Per default this values will be +# read from the environment. +cinder_auth_url: "{{ lookup('env', 'OS_AUTH_URL') }}" +cinder_username: "{{ lookup('env', 'OS_USERNAME') }}" +cinder_password: "{{ lookup('env', 'OS_PASSWORD') }}" +cinder_application_credential_id: "{{ lookup('env', 'OS_APPLICATION_CREDENTIAL_ID') }}" +cinder_application_credential_name: "{{ lookup('env', 'OS_APPLICATION_CREDENTIAL_NAME') }}" +cinder_application_credential_secret: "{{ lookup('env', 'OS_APPLICATION_CREDENTIAL_SECRET') }}" +cinder_region: "{{ lookup('env', 'OS_REGION_NAME') }}" +cinder_tenant_id: "{{ lookup('env', 'OS_TENANT_ID') | default(lookup('env', 'OS_PROJECT_ID'), true) }}" +cinder_tenant_name: "{{ lookup('env', 'OS_TENANT_NAME') | default(lookup('env', 'OS_PROJECT_NAME'), true) }}" +cinder_domain_name: "{{ lookup('env', 'OS_USER_DOMAIN_NAME') }}" +cinder_domain_id: "{{ lookup('env', 'OS_USER_DOMAIN_ID') }}" +cinder_cacert: "{{ lookup('env', 'OS_CACERT') }}" + +# For now, only Cinder v3 is supported in Cinder CSI driver +cinder_blockstorage_version: "v3" +cinder_csi_controller_replicas: 1 + +# Optional. Set to true, to rescan block device and verify its size before expanding +# the filesystem. +# Not all hypervizors have a /sys/class/block/XXX/device/rescan location, therefore if +# you enable this option and your hypervizor doesn't support this, you'll get a warning +# log on resize event. It is recommended to disable this option in this case. +# Defaults to false +# cinder_csi_rescan_on_resize: true + +cinder_tolerations: [] diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/cinder/tasks/cinder-credential-check.yml b/kubespray/project/roles/kubernetes-apps/csi_driver/cinder/tasks/cinder-credential-check.yml new file mode 100644 index 0000000..d797732 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/cinder/tasks/cinder-credential-check.yml @@ -0,0 +1,59 @@ +--- +- name: Cinder CSI Driver | check cinder_auth_url value + fail: + msg: "cinder_auth_url is missing" + when: cinder_auth_url is not defined or not cinder_auth_url + +- name: Cinder CSI Driver | check cinder_username value cinder_application_credential_name value + fail: + msg: "you must either set cinder_username or cinder_application_credential_name" + when: + - cinder_username is not defined or not cinder_username + - cinder_application_credential_name is not defined or not cinder_application_credential_name + +- name: Cinder CSI Driver | check cinder_application_credential_id value + fail: + msg: "cinder_application_credential_id is missing" + when: + - cinder_application_credential_name is defined + - cinder_application_credential_name | length > 0 + - cinder_application_credential_id is not defined or not cinder_application_credential_id + +- name: Cinder CSI Driver | check cinder_application_credential_secret value + fail: + msg: "cinder_application_credential_secret is missing" + when: + - cinder_application_credential_name is defined + - cinder_application_credential_name | length > 0 + - cinder_application_credential_secret is not defined or not cinder_application_credential_secret + +- name: Cinder CSI Driver | check cinder_password value + fail: + msg: "cinder_password is missing" + when: + - cinder_username is defined + - cinder_username | length > 0 + - cinder_application_credential_name is not defined or not cinder_application_credential_name + - cinder_application_credential_secret is not defined or not cinder_application_credential_secret + - cinder_password is not defined or not cinder_password + +- name: Cinder CSI Driver | check cinder_region value + fail: + msg: "cinder_region is missing" + when: cinder_region is not defined or not cinder_region + +- name: Cinder CSI Driver | check cinder_tenant_id value + fail: + msg: "one of cinder_tenant_id or cinder_tenant_name must be specified" + when: + - cinder_tenant_id is not defined or not cinder_tenant_id + - cinder_tenant_name is not defined or not cinder_tenant_name + - cinder_application_credential_name is not defined or not cinder_application_credential_name + +- name: Cinder CSI Driver | check cinder_domain_id value + fail: + msg: "one of cinder_domain_id or cinder_domain_name must be specified" + when: + - cinder_domain_id is not defined or not cinder_domain_id + - cinder_domain_name is not defined or not cinder_domain_name + - cinder_application_credential_name is not defined or not cinder_application_credential_name diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/cinder/tasks/cinder-write-cacert.yml b/kubespray/project/roles/kubernetes-apps/csi_driver/cinder/tasks/cinder-write-cacert.yml new file mode 100644 index 0000000..c6d14a2 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/cinder/tasks/cinder-write-cacert.yml @@ -0,0 +1,11 @@ +--- +# include to workaround mitogen issue +# https://github.com/dw/mitogen/issues/663 + +- name: Cinder CSI Driver | Write cacert file + copy: + src: "{{ cinder_cacert }}" + dest: "{{ kube_config_dir }}/cinder-cacert.pem" + group: "{{ kube_cert_group }}" + mode: 0640 + delegate_to: "{{ delegate_host_to_write_cacert }}" diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/cinder/tasks/main.yml b/kubespray/project/roles/kubernetes-apps/csi_driver/cinder/tasks/main.yml new file mode 100644 index 0000000..47ce6cd --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/cinder/tasks/main.yml @@ -0,0 +1,57 @@ +--- +- name: Cinder CSI Driver | Check Cinder credentials + include_tasks: cinder-credential-check.yml + +- name: Cinder CSI Driver | Write cacert file + include_tasks: cinder-write-cacert.yml + run_once: true + loop: "{{ groups['k8s_cluster'] }}" + loop_control: + loop_var: delegate_host_to_write_cacert + when: + - inventory_hostname in groups['k8s_cluster'] + - cinder_cacert is defined + - cinder_cacert | length > 0 + +- name: Cinder CSI Driver | Write Cinder cloud-config + template: + src: "cinder-csi-cloud-config.j2" + dest: "{{ kube_config_dir }}/cinder_cloud_config" + group: "{{ kube_cert_group }}" + mode: 0640 + when: inventory_hostname == groups['kube_control_plane'][0] + +- name: Cinder CSI Driver | Get base64 cloud-config + slurp: + src: "{{ kube_config_dir }}/cinder_cloud_config" + register: cloud_config_secret + when: inventory_hostname == groups['kube_control_plane'][0] + +- name: Cinder CSI Driver | Generate Manifests + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/{{ item.file }}" + mode: 0644 + with_items: + - {name: cinder-csi-driver, file: cinder-csi-driver.yml} + - {name: cinder-csi-cloud-config-secret, file: cinder-csi-cloud-config-secret.yml} + - {name: cinder-csi-controllerplugin, file: cinder-csi-controllerplugin-rbac.yml} + - {name: cinder-csi-controllerplugin, file: cinder-csi-controllerplugin.yml} + - {name: cinder-csi-nodeplugin, file: cinder-csi-nodeplugin-rbac.yml} + - {name: cinder-csi-nodeplugin, file: cinder-csi-nodeplugin.yml} + - {name: cinder-csi-poddisruptionbudget, file: cinder-csi-poddisruptionbudget.yml} + register: cinder_csi_manifests + when: inventory_hostname == groups['kube_control_plane'][0] + +- name: Cinder CSI Driver | Apply Manifests + kube: + kubectl: "{{ bin_dir }}/kubectl" + filename: "{{ kube_config_dir }}/{{ item.item.file }}" + state: "latest" + with_items: + - "{{ cinder_csi_manifests.results }}" + when: + - inventory_hostname == groups['kube_control_plane'][0] + - not item is skipped + loop_control: + label: "{{ item.item.file }}" diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-cloud-config-secret.yml.j2 b/kubespray/project/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-cloud-config-secret.yml.j2 new file mode 100644 index 0000000..cb3cba6 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-cloud-config-secret.yml.j2 @@ -0,0 +1,10 @@ +# This YAML file contains secret objects, +# which are necessary to run csi cinder plugin. + +kind: Secret +apiVersion: v1 +metadata: + name: cloud-config + namespace: kube-system +data: + cloud.conf: {{ cloud_config_secret.content }} diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-cloud-config.j2 b/kubespray/project/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-cloud-config.j2 new file mode 100644 index 0000000..04d0c68 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-cloud-config.j2 @@ -0,0 +1,44 @@ +[Global] +auth-url="{{ cinder_auth_url }}" +{% if cinder_application_credential_id|length == 0 and cinder_application_credential_name|length == 0 %} +username="{{ cinder_username }}" +password="{{ cinder_password }}" +{% endif %} +{% if cinder_application_credential_id|length > 0 %} +application-credential-id={{ cinder_application_credential_id }} +{% endif %} +{% if cinder_application_credential_name|length > 0 %} +application-credential-name={{ cinder_application_credential_name }} +{% endif %} +{% if cinder_application_credential_secret|length > 0 %} +application-credential-secret={{ cinder_application_credential_secret }} +{% endif %} +region="{{ cinder_region }}" +{% if cinder_tenant_id|length > 0 %} +tenant-id="{{ cinder_tenant_id }}" +{% endif %} +{% if cinder_tenant_name|length > 0 %} +tenant-name="{{ cinder_tenant_name }}" +{% endif %} +{% if cinder_domain_name|length > 0 %} +domain-name="{{ cinder_domain_name }}" +{% elif cinder_domain_id|length > 0 %} +domain-id ="{{ cinder_domain_id }}" +{% endif %} +{% if cinder_cacert|length > 0 %} +ca-file="{{ kube_config_dir }}/cinder-cacert.pem" +{% endif %} + +[BlockStorage] +{% if cinder_blockstorage_version is defined %} +bs-version={{ cinder_blockstorage_version }} +{% endif %} +{% if cinder_csi_ignore_volume_az is defined %} +ignore-volume-az={{ cinder_csi_ignore_volume_az | bool }} +{% endif %} +{% if node_volume_attach_limit is defined and node_volume_attach_limit != "" %} +node-volume-attach-limit="{{ node_volume_attach_limit }}" +{% endif %} +{% if cinder_csi_rescan_on_resize is defined %} +rescan-on-resize={{ cinder_csi_rescan_on_resize | bool }} +{% endif %} diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-controllerplugin-rbac.yml.j2 b/kubespray/project/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-controllerplugin-rbac.yml.j2 new file mode 100644 index 0000000..09ecacb --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-controllerplugin-rbac.yml.j2 @@ -0,0 +1,179 @@ +# This YAML file contains RBAC API objects, +# which are necessary to run csi controller plugin + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-cinder-controller-sa + namespace: kube-system + +--- +# external attacher +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-attacher-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-attacher-binding +subjects: + - kind: ServiceAccount + name: csi-cinder-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-attacher-role + apiGroup: rbac.authorization.k8s.io + +--- +# external Provisioner +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-provisioner-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["get", "list"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-provisioner-binding +subjects: + - kind: ServiceAccount + name: csi-cinder-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-provisioner-role + apiGroup: rbac.authorization.k8s.io + +--- +# external snapshotter +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-snapshotter-role +rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "patch", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-snapshotter-binding +subjects: + - kind: ServiceAccount + name: csi-cinder-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-snapshotter-role + apiGroup: rbac.authorization.k8s.io +--- + +# External Resizer +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-resizer-role +rules: + # The following rule should be uncommented for plugins that require secrets + # for provisioning. + # - apiGroups: [""] + # resources: ["secrets"] + # verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["patch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-resizer-binding +subjects: + - kind: ServiceAccount + name: csi-cinder-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-resizer-role + apiGroup: rbac.authorization.k8s.io diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-controllerplugin.yml.j2 b/kubespray/project/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-controllerplugin.yml.j2 new file mode 100644 index 0000000..de27b76 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-controllerplugin.yml.j2 @@ -0,0 +1,173 @@ +# This YAML file contains CSI Controller Plugin Sidecars +# external-attacher, external-provisioner, external-snapshotter +# external-resize, liveness-probe + +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: csi-cinder-controllerplugin + namespace: kube-system +spec: + replicas: {{ cinder_csi_controller_replicas }} + selector: + matchLabels: + app: csi-cinder-controllerplugin + template: + metadata: + labels: + app: csi-cinder-controllerplugin + spec: + serviceAccount: csi-cinder-controller-sa + containers: + - name: csi-attacher + image: {{ csi_attacher_image_repo }}:{{ cinder_csi_attacher_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + args: + - "--csi-address=$(ADDRESS)" + - "--timeout=3m" +{% if cinder_csi_controller_replicas is defined and cinder_csi_controller_replicas > 1 %} + - --leader-election=true +{% endif %} + - "--default-fstype=ext4" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: csi-provisioner + image: {{ csi_provisioner_image_repo }}:{{ cinder_csi_provisioner_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + args: + - "--csi-address=$(ADDRESS)" + - "--timeout=3m" + - "--default-fstype=ext4" + - "--extra-create-metadata" +{% if cinder_topology is defined and cinder_topology %} + - --feature-gates=Topology=true +{% endif %} +{% if cinder_csi_controller_replicas is defined and cinder_csi_controller_replicas > 1 %} + - "--leader-election=true" +{% endif %} + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: csi-snapshotter + image: {{ csi_snapshotter_image_repo }}:{{ cinder_csi_snapshotter_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + args: + - "--csi-address=$(ADDRESS)" + - "--timeout=3m" + - "--extra-create-metadata" +{% if cinder_csi_controller_replicas is defined and cinder_csi_controller_replicas > 1 %} + - --leader-election=true +{% endif %} + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - name: csi-resizer + image: {{ csi_resizer_image_repo }}:{{ cinder_csi_resizer_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + args: + - "--csi-address=$(ADDRESS)" + - "--timeout=3m" + - "--handle-volume-inuse-error=false" +{% if cinder_csi_controller_replicas is defined and cinder_csi_controller_replicas > 1 %} + - --leader-election=true +{% endif %} + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: liveness-probe + image: {{ csi_livenessprobe_image_repo }}:{{ cinder_csi_livenessprobe_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + args: + - "--csi-address=$(ADDRESS)" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - name: cinder-csi-plugin + image: {{ cinder_csi_plugin_image_repo }}:{{ cinder_csi_plugin_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + args: + - /bin/cinder-csi-plugin + - "--endpoint=$(CSI_ENDPOINT)" + - "--cloud-config=$(CLOUD_CONFIG)" + - "--cluster=$(CLUSTER_NAME)" + env: + - name: CSI_ENDPOINT + value: unix://csi/csi.sock + - name: CLOUD_CONFIG + value: /etc/config/cloud.conf + - name: CLUSTER_NAME + value: {{ cluster_name }} + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 10 + periodSeconds: 60 + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: secret-cinderplugin + mountPath: /etc/config + readOnly: true + - name: ca-certs + mountPath: /etc/ssl/certs + readOnly: true +{% if ssl_ca_dirs | length %} +{% for dir in ssl_ca_dirs %} + - name: {{ dir | regex_replace('^/(.*)$', '\\1' ) | regex_replace('/', '-') }} + mountPath: {{ dir }} + readOnly: true +{% endfor %} +{% endif %} +{% if cinder_cacert is defined and cinder_cacert != "" %} + - name: cinder-cacert + mountPath: {{ kube_config_dir }}/cinder-cacert.pem + readOnly: true +{% endif %} + volumes: + - name: socket-dir + emptyDir: + - name: secret-cinderplugin + secret: + secretName: cloud-config + - name: ca-certs + hostPath: + path: /etc/ssl/certs + type: DirectoryOrCreate +{% if ssl_ca_dirs | length %} +{% for dir in ssl_ca_dirs %} + - name: {{ dir | regex_replace('^/(.*)$', '\\1' ) | regex_replace('/', '-') }} + hostPath: + path: {{ dir }} + type: DirectoryOrCreate +{% endfor %} +{% endif %} +{% if cinder_cacert is defined and cinder_cacert != "" %} + - name: cinder-cacert + hostPath: + path: {{ kube_config_dir }}/cinder-cacert.pem + type: FileOrCreate +{% endif %} diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-driver.yml.j2 b/kubespray/project/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-driver.yml.j2 new file mode 100644 index 0000000..5b681e4 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-driver.yml.j2 @@ -0,0 +1,10 @@ +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: cinder.csi.openstack.org +spec: + attachRequired: true + podInfoOnMount: true + volumeLifecycleModes: + - Persistent + - Ephemeral diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-nodeplugin-rbac.yml.j2 b/kubespray/project/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-nodeplugin-rbac.yml.j2 new file mode 100644 index 0000000..912923f --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-nodeplugin-rbac.yml.j2 @@ -0,0 +1,30 @@ +# This YAML defines all API objects to create RBAC roles for csi node plugin. + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-cinder-node-sa + namespace: kube-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-nodeplugin-role +rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-nodeplugin-binding +subjects: + - kind: ServiceAccount + name: csi-cinder-node-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-nodeplugin-role + apiGroup: rbac.authorization.k8s.io diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-nodeplugin.yml.j2 b/kubespray/project/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-nodeplugin.yml.j2 new file mode 100644 index 0000000..dd2736e --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-nodeplugin.yml.j2 @@ -0,0 +1,148 @@ +# This YAML file contains driver-registrar & csi driver nodeplugin API objects, +# which are necessary to run csi nodeplugin for cinder. + +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: csi-cinder-nodeplugin + namespace: kube-system +spec: + selector: + matchLabels: + app: csi-cinder-nodeplugin + template: + metadata: + labels: + app: csi-cinder-nodeplugin + spec: + tolerations: + - operator: Exists + serviceAccountName: csi-cinder-node-sa + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: node-driver-registrar + image: {{ csi_node_driver_registrar_image_repo }}:{{ csi_node_driver_registrar_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + args: + - "--csi-address=$(ADDRESS)" + - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)" + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/cinder.csi.openstack.org/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + - name: liveness-probe + image: {{ csi_livenessprobe_image_repo }}:{{ csi_livenessprobe_image_tag }} + args: + - "--csi-address=/csi/csi.sock" + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: cinder-csi-plugin + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + image: {{ cinder_csi_plugin_image_repo }}:{{ cinder_csi_plugin_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + args: + - /bin/cinder-csi-plugin + - "--endpoint=$(CSI_ENDPOINT)" + - "--cloud-config=$(CLOUD_CONFIG)" + env: + - name: CSI_ENDPOINT + value: unix://csi/csi.sock + - name: CLOUD_CONFIG + value: /etc/config/cloud.conf + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 10 + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: kubelet-dir + mountPath: /var/lib/kubelet + mountPropagation: "Bidirectional" + - name: pods-probe-dir + mountPath: /dev + mountPropagation: "HostToContainer" + - name: secret-cinderplugin + mountPath: /etc/config + readOnly: true + - name: ca-certs + mountPath: /etc/ssl/certs + readOnly: true +{% if ssl_ca_dirs | length %} +{% for dir in ssl_ca_dirs %} + - name: {{ dir | regex_replace('^/(.*)$', '\\1' ) | regex_replace('/', '-') }} + mountPath: {{ dir }} + readOnly: true +{% endfor %} +{% endif %} +{% if cinder_cacert is defined and cinder_cacert != "" %} + - name: cinder-cacert + mountPath: {{ kube_config_dir }}/cinder-cacert.pem + readOnly: true +{% endif %} + volumes: + - name: socket-dir + hostPath: + path: /var/lib/kubelet/plugins/cinder.csi.openstack.org + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + - name: kubelet-dir + hostPath: + path: /var/lib/kubelet + type: Directory + - name: pods-probe-dir + hostPath: + path: /dev + type: Directory + - name: secret-cinderplugin + secret: + secretName: cloud-config + - name: ca-certs + hostPath: + path: /etc/ssl/certs + type: DirectoryOrCreate +{% if ssl_ca_dirs | length %} +{% for dir in ssl_ca_dirs %} + - name: {{ dir | regex_replace('^/(.*)$', '\\1' ) | regex_replace('/', '-') }} + hostPath: + path: {{ dir }} + type: DirectoryOrCreate +{% endfor %} +{% endif %} +{% if cinder_cacert is defined and cinder_cacert != "" %} + - name: cinder-cacert + hostPath: + path: {{ kube_config_dir }}/cinder-cacert.pem + type: FileOrCreate +{% endif %} +{% if cinder_tolerations %} + tolerations: + {{ cinder_tolerations | to_nice_yaml(indent=2) | indent(width=8) }} +{% endif %} diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-poddisruptionbudget.yml.j2 b/kubespray/project/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-poddisruptionbudget.yml.j2 new file mode 100644 index 0000000..391d3b3 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-poddisruptionbudget.yml.j2 @@ -0,0 +1,14 @@ +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: cinder-csi-pdb + namespace: kube-system +spec: +{% if cinder_csi_controller_replicas is defined and cinder_csi_controller_replicas > 1 %} + minAvailable: 1 +{% else %} + minAvailable: 0 +{% endif %} + selector: + matchLabels: + app: csi-cinder-controllerplugin diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/csi_crd/tasks/main.yml b/kubespray/project/roles/kubernetes-apps/csi_driver/csi_crd/tasks/main.yml new file mode 100644 index 0000000..4790931 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/csi_crd/tasks/main.yml @@ -0,0 +1,26 @@ +--- +- name: CSI CRD | Generate Manifests + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/{{ item.file }}" + mode: 0644 + with_items: + - {name: volumesnapshotclasses, file: volumesnapshotclasses.yml} + - {name: volumesnapshotcontents, file: volumesnapshotcontents.yml} + - {name: volumesnapshots, file: volumesnapshots.yml} + register: csi_crd_manifests + when: inventory_hostname == groups['kube_control_plane'][0] + +- name: CSI CRD | Apply Manifests + kube: + kubectl: "{{ bin_dir }}/kubectl" + filename: "{{ kube_config_dir }}/{{ item.item.file }}" + state: "latest" + wait: true + with_items: + - "{{ csi_crd_manifests.results }}" + when: + - inventory_hostname == groups['kube_control_plane'][0] + - not item is skipped + loop_control: + label: "{{ item.item.file }}" diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/csi_crd/templates/volumesnapshotclasses.yml.j2 b/kubespray/project/roles/kubernetes-apps/csi_driver/csi_crd/templates/volumesnapshotclasses.yml.j2 new file mode 100644 index 0000000..47e5fd3 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/csi_crd/templates/volumesnapshotclasses.yml.j2 @@ -0,0 +1,116 @@ + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419" + creationTimestamp: null + name: volumesnapshotclasses.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotClass + listKind: VolumeSnapshotClassList + plural: volumesnapshotclasses + singular: volumesnapshotclass + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .driver + name: Driver + type: string + - description: Determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. + jsonPath: .deletionPolicy + name: DeletionPolicy + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: VolumeSnapshotClass specifies parameters that a underlying storage system uses when creating a volume snapshot. A specific VolumeSnapshotClass is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses are non-namespaced + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + deletionPolicy: + description: deletionPolicy determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the storage driver that handles this VolumeSnapshotClass. Required. + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + parameters: + additionalProperties: + type: string + description: parameters is a key-value map with storage driver specific parameters for creating snapshots. These values are opaque to Kubernetes. + type: object + required: + - deletionPolicy + - driver + type: object + served: true + storage: true + subresources: {} + - additionalPrinterColumns: + - jsonPath: .driver + name: Driver + type: string + - description: Determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. + jsonPath: .deletionPolicy + name: DeletionPolicy + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + # This indicates the v1beta1 version of the custom resource is deprecated. + # API requests to this version receive a warning in the server response. + deprecated: true + # This overrides the default warning returned to clients making v1beta1 API requests. + deprecationWarning: "snapshot.storage.k8s.io/v1beta1 VolumeSnapshotClass is deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshotClass" + schema: + openAPIV3Schema: + description: VolumeSnapshotClass specifies parameters that a underlying storage system uses when creating a volume snapshot. A specific VolumeSnapshotClass is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses are non-namespaced + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + deletionPolicy: + description: deletionPolicy determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the storage driver that handles this VolumeSnapshotClass. Required. + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + parameters: + additionalProperties: + type: string + description: parameters is a key-value map with storage driver specific parameters for creating snapshots. These values are opaque to Kubernetes. + type: object + required: + - deletionPolicy + - driver + type: object + served: true + storage: false + subresources: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/csi_crd/templates/volumesnapshotcontents.yml.j2 b/kubespray/project/roles/kubernetes-apps/csi_driver/csi_crd/templates/volumesnapshotcontents.yml.j2 new file mode 100644 index 0000000..c611221 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/csi_crd/templates/volumesnapshotcontents.yml.j2 @@ -0,0 +1,305 @@ + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419" + creationTimestamp: null + name: volumesnapshotcontents.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotContent + listKind: VolumeSnapshotContentList + plural: volumesnapshotcontents + singular: volumesnapshotcontent + scope: Cluster + versions: + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: Represents the complete size of the snapshot in bytes + jsonPath: .status.restoreSize + name: RestoreSize + type: integer + - description: Determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. + jsonPath: .spec.deletionPolicy + name: DeletionPolicy + type: string + - description: Name of the CSI driver used to create the physical snapshot on the underlying storage system. + jsonPath: .spec.driver + name: Driver + type: string + - description: Name of the VolumeSnapshotClass to which this snapshot belongs. + jsonPath: .spec.volumeSnapshotClassName + name: VolumeSnapshotClass + type: string + - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. + jsonPath: .spec.volumeSnapshotRef.name + name: VolumeSnapshot + type: string + - description: Namespace of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. + jsonPath: .spec.volumeSnapshotRef.namespace + name: VolumeSnapshotNamespace + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: VolumeSnapshotContent represents the actual "on-disk" snapshot object in the underlying storage system + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: spec defines properties of a VolumeSnapshotContent created by the underlying storage system. Required. + properties: + deletionPolicy: + description: deletionPolicy determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. For dynamically provisioned snapshots, this field will automatically be filled in by the CSI snapshotter sidecar with the "DeletionPolicy" field defined in the corresponding VolumeSnapshotClass. For pre-existing snapshots, users MUST specify this field when creating the VolumeSnapshotContent object. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the CSI driver used to create the physical snapshot on the underlying storage system. This MUST be the same as the name returned by the CSI GetPluginName() call for that driver. Required. + type: string + source: + description: source specifies whether the snapshot is (or should be) dynamically provisioned or already exists, and just requires a Kubernetes object representation. This field is immutable after creation. Required. + properties: + snapshotHandle: + description: snapshotHandle specifies the CSI "snapshot_id" of a pre-existing snapshot on the underlying storage system for which a Kubernetes object representation was (or should be) created. This field is immutable. + type: string + volumeHandle: + description: volumeHandle specifies the CSI "volume_id" of the volume from which a snapshot should be dynamically taken from. This field is immutable. + type: string + type: object + oneOf: + - required: ["snapshotHandle"] + - required: ["volumeHandle"] + volumeSnapshotClassName: + description: name of the VolumeSnapshotClass from which this snapshot was (or will be) created. Note that after provisioning, the VolumeSnapshotClass may be deleted or recreated with different set of values, and as such, should not be referenced post-snapshot creation. + type: string + volumeSnapshotRef: + description: volumeSnapshotRef specifies the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName field must reference to this VolumeSnapshotContent's name for the bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent object, name and namespace of the VolumeSnapshot object MUST be provided for binding to happen. This field is immutable after creation. Required. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. For example, if the object reference is to a container within a pod, this would take on a value like: "spec.containers{name}" (where "name" refers to the name of the container that triggered the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. TODO: this design is not final and this field is subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + required: + - deletionPolicy + - driver + - source + - volumeSnapshotRef + type: object + status: + description: status represents the current information of a snapshot. + properties: + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it indicates the creation time is unknown. The format of this field is a Unix nanoseconds time encoded as an int64. On Unix, the command `date +%s%N` returns the current time in nanoseconds since 1970-01-01 00:00:00 UTC. + format: int64 + type: integer + error: + description: error is the last observed error during snapshot creation, if any. Upon success after retry, this error field will be cleared. + properties: + message: + description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if a snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + description: restoreSize represents the complete size of the snapshot in bytes. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown. + format: int64 + minimum: 0 + type: integer + snapshotHandle: + description: snapshotHandle is the CSI "snapshot_id" of a snapshot on the underlying storage system. If not specified, it indicates that dynamic snapshot creation has either failed or it is still in progress. + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: Represents the complete size of the snapshot in bytes + jsonPath: .status.restoreSize + name: RestoreSize + type: integer + - description: Determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. + jsonPath: .spec.deletionPolicy + name: DeletionPolicy + type: string + - description: Name of the CSI driver used to create the physical snapshot on the underlying storage system. + jsonPath: .spec.driver + name: Driver + type: string + - description: Name of the VolumeSnapshotClass to which this snapshot belongs. + jsonPath: .spec.volumeSnapshotClassName + name: VolumeSnapshotClass + type: string + - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. + jsonPath: .spec.volumeSnapshotRef.name + name: VolumeSnapshot + type: string + - description: Namespace of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. + jsonPath: .spec.volumeSnapshotRef.namespace + name: VolumeSnapshotNamespace + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + # This indicates the v1beta1 version of the custom resource is deprecated. + # API requests to this version receive a warning in the server response. + deprecated: true + # This overrides the default warning returned to clients making v1beta1 API requests. + deprecationWarning: "snapshot.storage.k8s.io/v1beta1 VolumeSnapshotContent is deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshotContent" + schema: + openAPIV3Schema: + description: VolumeSnapshotContent represents the actual "on-disk" snapshot object in the underlying storage system + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: spec defines properties of a VolumeSnapshotContent created by the underlying storage system. Required. + properties: + deletionPolicy: + description: deletionPolicy determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. For dynamically provisioned snapshots, this field will automatically be filled in by the CSI snapshotter sidecar with the "DeletionPolicy" field defined in the corresponding VolumeSnapshotClass. For pre-existing snapshots, users MUST specify this field when creating the VolumeSnapshotContent object. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the CSI driver used to create the physical snapshot on the underlying storage system. This MUST be the same as the name returned by the CSI GetPluginName() call for that driver. Required. + type: string + source: + description: source specifies whether the snapshot is (or should be) dynamically provisioned or already exists, and just requires a Kubernetes object representation. This field is immutable after creation. Required. + properties: + snapshotHandle: + description: snapshotHandle specifies the CSI "snapshot_id" of a pre-existing snapshot on the underlying storage system for which a Kubernetes object representation was (or should be) created. This field is immutable. + type: string + volumeHandle: + description: volumeHandle specifies the CSI "volume_id" of the volume from which a snapshot should be dynamically taken from. This field is immutable. + type: string + type: object + volumeSnapshotClassName: + description: name of the VolumeSnapshotClass from which this snapshot was (or will be) created. Note that after provisioning, the VolumeSnapshotClass may be deleted or recreated with different set of values, and as such, should not be referenced post-snapshot creation. + type: string + volumeSnapshotRef: + description: volumeSnapshotRef specifies the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName field must reference to this VolumeSnapshotContent's name for the bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent object, name and namespace of the VolumeSnapshot object MUST be provided for binding to happen. This field is immutable after creation. Required. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. For example, if the object reference is to a container within a pod, this would take on a value like: "spec.containers{name}" (where "name" refers to the name of the container that triggered the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. TODO: this design is not final and this field is subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + required: + - deletionPolicy + - driver + - source + - volumeSnapshotRef + type: object + status: + description: status represents the current information of a snapshot. + properties: + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it indicates the creation time is unknown. The format of this field is a Unix nanoseconds time encoded as an int64. On Unix, the command `date +%s%N` returns the current time in nanoseconds since 1970-01-01 00:00:00 UTC. + format: int64 + type: integer + error: + description: error is the last observed error during snapshot creation, if any. Upon success after retry, this error field will be cleared. + properties: + message: + description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if a snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + description: restoreSize represents the complete size of the snapshot in bytes. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown. + format: int64 + minimum: 0 + type: integer + snapshotHandle: + description: snapshotHandle is the CSI "snapshot_id" of a snapshot on the underlying storage system. If not specified, it indicates that dynamic snapshot creation has either failed or it is still in progress. + type: string + type: object + required: + - spec + type: object + served: true + storage: false + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/csi_crd/templates/volumesnapshots.yml.j2 b/kubespray/project/roles/kubernetes-apps/csi_driver/csi_crd/templates/volumesnapshots.yml.j2 new file mode 100644 index 0000000..1b41ff8 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/csi_crd/templates/volumesnapshots.yml.j2 @@ -0,0 +1,231 @@ + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419" + creationTimestamp: null + name: volumesnapshots.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshot + listKind: VolumeSnapshotList + plural: volumesnapshots + singular: volumesnapshot + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: If a new snapshot needs to be created, this contains the name of the source PVC from which this snapshot was (or will be) created. + jsonPath: .spec.source.persistentVolumeClaimName + name: SourcePVC + type: string + - description: If a snapshot already exists, this contains the name of the existing VolumeSnapshotContent object representing the existing snapshot. + jsonPath: .spec.source.volumeSnapshotContentName + name: SourceSnapshotContent + type: string + - description: Represents the minimum size of volume required to rehydrate from this snapshot. + jsonPath: .status.restoreSize + name: RestoreSize + type: string + - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot. + jsonPath: .spec.volumeSnapshotClassName + name: SnapshotClass + type: string + - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot object intends to bind to. Please note that verification of binding actually requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure both are pointing at each other. Binding MUST be verified prior to usage of this object. + jsonPath: .status.boundVolumeSnapshotContentName + name: SnapshotContent + type: string + - description: Timestamp when the point-in-time snapshot was taken by the underlying storage system. + jsonPath: .status.creationTime + name: CreationTime + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: VolumeSnapshot is a user's request for either creating a point-in-time snapshot of a persistent volume, or binding to a pre-existing snapshot. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: 'spec defines the desired characteristics of a snapshot requested by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots Required.' + properties: + source: + description: source specifies where a snapshot will be created from. This field is immutable after creation. Required. + properties: + persistentVolumeClaimName: + description: persistentVolumeClaimName specifies the name of the PersistentVolumeClaim object representing the volume from which a snapshot should be created. This PVC is assumed to be in the same namespace as the VolumeSnapshot object. This field should be set if the snapshot does not exists, and needs to be created. This field is immutable. + type: string + volumeSnapshotContentName: + description: volumeSnapshotContentName specifies the name of a pre-existing VolumeSnapshotContent object representing an existing volume snapshot. This field should be set if the snapshot already exists and only needs a representation in Kubernetes. This field is immutable. + type: string + type: object + oneOf: + - required: ["persistentVolumeClaimName"] + - required: ["volumeSnapshotContentName"] + volumeSnapshotClassName: + description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass requested by the VolumeSnapshot. VolumeSnapshotClassName may be left nil to indicate that the default SnapshotClass should be used. A given cluster may have multiple default Volume SnapshotClasses: one default per CSI Driver. If a VolumeSnapshot does not specify a SnapshotClass, VolumeSnapshotSource will be checked to figure out what the associated CSI Driver is, and the default VolumeSnapshotClass associated with that CSI Driver will be used. If more than one VolumeSnapshotClass exist for a given CSI Driver and more than one have been marked as default, CreateSnapshot will fail and generate an event. Empty string is not allowed for this field.' + type: string + required: + - source + type: object + status: + description: status represents the current information of a snapshot. Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object. + properties: + boundVolumeSnapshotContentName: + description: 'boundVolumeSnapshotContentName is the name of the VolumeSnapshotContent object to which this VolumeSnapshot object intends to bind to. If not specified, it indicates that the VolumeSnapshot object has not been successfully bound to a VolumeSnapshotContent object yet. NOTE: To avoid possible security issues, consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object.' + type: string + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it may indicate that the creation time of the snapshot is unknown. + format: date-time + type: string + error: + description: error is the last observed error during snapshot creation, if any. This field could be helpful to upper level controllers(i.e., application controller) to decide whether they should continue on waiting for the snapshot to be created based on the type of error reported. The snapshot controller will keep retrying when an error occurrs during the snapshot creation. Upon success, this error field will be cleared. + properties: + message: + description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if the snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + type: string + description: restoreSize represents the minimum size of volume required to create a volume from this snapshot. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: If a new snapshot needs to be created, this contains the name of the source PVC from which this snapshot was (or will be) created. + jsonPath: .spec.source.persistentVolumeClaimName + name: SourcePVC + type: string + - description: If a snapshot already exists, this contains the name of the existing VolumeSnapshotContent object representing the existing snapshot. + jsonPath: .spec.source.volumeSnapshotContentName + name: SourceSnapshotContent + type: string + - description: Represents the minimum size of volume required to rehydrate from this snapshot. + jsonPath: .status.restoreSize + name: RestoreSize + type: string + - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot. + jsonPath: .spec.volumeSnapshotClassName + name: SnapshotClass + type: string + - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot object intends to bind to. Please note that verification of binding actually requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure both are pointing at each other. Binding MUST be verified prior to usage of this object. + jsonPath: .status.boundVolumeSnapshotContentName + name: SnapshotContent + type: string + - description: Timestamp when the point-in-time snapshot was taken by the underlying storage system. + jsonPath: .status.creationTime + name: CreationTime + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + # This indicates the v1beta1 version of the custom resource is deprecated. + # API requests to this version receive a warning in the server response. + deprecated: true + # This overrides the default warning returned to clients making v1beta1 API requests. + deprecationWarning: "snapshot.storage.k8s.io/v1beta1 VolumeSnapshot is deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshot" + schema: + openAPIV3Schema: + description: VolumeSnapshot is a user's request for either creating a point-in-time snapshot of a persistent volume, or binding to a pre-existing snapshot. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: 'spec defines the desired characteristics of a snapshot requested by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots Required.' + properties: + source: + description: source specifies where a snapshot will be created from. This field is immutable after creation. Required. + properties: + persistentVolumeClaimName: + description: persistentVolumeClaimName specifies the name of the PersistentVolumeClaim object representing the volume from which a snapshot should be created. This PVC is assumed to be in the same namespace as the VolumeSnapshot object. This field should be set if the snapshot does not exists, and needs to be created. This field is immutable. + type: string + volumeSnapshotContentName: + description: volumeSnapshotContentName specifies the name of a pre-existing VolumeSnapshotContent object representing an existing volume snapshot. This field should be set if the snapshot already exists and only needs a representation in Kubernetes. This field is immutable. + type: string + type: object + volumeSnapshotClassName: + description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass requested by the VolumeSnapshot. VolumeSnapshotClassName may be left nil to indicate that the default SnapshotClass should be used. A given cluster may have multiple default Volume SnapshotClasses: one default per CSI Driver. If a VolumeSnapshot does not specify a SnapshotClass, VolumeSnapshotSource will be checked to figure out what the associated CSI Driver is, and the default VolumeSnapshotClass associated with that CSI Driver will be used. If more than one VolumeSnapshotClass exist for a given CSI Driver and more than one have been marked as default, CreateSnapshot will fail and generate an event. Empty string is not allowed for this field.' + type: string + required: + - source + type: object + status: + description: status represents the current information of a snapshot. Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object. + properties: + boundVolumeSnapshotContentName: + description: 'boundVolumeSnapshotContentName is the name of the VolumeSnapshotContent object to which this VolumeSnapshot object intends to bind to. If not specified, it indicates that the VolumeSnapshot object has not been successfully bound to a VolumeSnapshotContent object yet. NOTE: To avoid possible security issues, consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object.' + type: string + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it may indicate that the creation time of the snapshot is unknown. + format: date-time + type: string + error: + description: error is the last observed error during snapshot creation, if any. This field could be helpful to upper level controllers(i.e., application controller) to decide whether they should continue on waiting for the snapshot to be created based on the type of error reported. The snapshot controller will keep retrying when an error occurrs during the snapshot creation. Upon success, this error field will be cleared. + properties: + message: + description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if the snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + type: string + description: restoreSize represents the minimum size of volume required to create a volume from this snapshot. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + required: + - spec + type: object + served: true + storage: false + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/gcp_pd/defaults/main.yml b/kubespray/project/roles/kubernetes-apps/csi_driver/gcp_pd/defaults/main.yml new file mode 100644 index 0000000..1ee662e --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/gcp_pd/defaults/main.yml @@ -0,0 +1,2 @@ +--- +gcp_pd_csi_controller_replicas: 1 diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/gcp_pd/tasks/main.yml b/kubespray/project/roles/kubernetes-apps/csi_driver/gcp_pd/tasks/main.yml new file mode 100644 index 0000000..be511ca --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/gcp_pd/tasks/main.yml @@ -0,0 +1,47 @@ +--- +- name: GCP PD CSI Driver | Check if cloud-sa.json exists + fail: + msg: "Credentials file cloud-sa.json is mandatory" + when: gcp_pd_csi_sa_cred_file is not defined or not gcp_pd_csi_sa_cred_file + +- name: GCP PD CSI Driver | Copy GCP credentials file + copy: + src: "{{ gcp_pd_csi_sa_cred_file }}" + dest: "{{ kube_config_dir }}/cloud-sa.json" + group: "{{ kube_cert_group }}" + mode: 0640 + when: inventory_hostname == groups['kube_control_plane'][0] + +- name: GCP PD CSI Driver | Get base64 cloud-sa.json + slurp: + src: "{{ kube_config_dir }}/cloud-sa.json" + register: gcp_cred_secret + when: inventory_hostname == groups['kube_control_plane'][0] + +- name: GCP PD CSI Driver | Generate Manifests + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/{{ item.file }}" + mode: 0644 + with_items: + - {name: gcp-pd-csi-cred-secret, file: gcp-pd-csi-cred-secret.yml} + - {name: gcp-pd-csi-setup, file: gcp-pd-csi-setup.yml} + - {name: gcp-pd-csi-controller, file: gcp-pd-csi-controller.yml} + - {name: gcp-pd-csi-node, file: gcp-pd-csi-node.yml} + - {name: gcp-pd-csi-sc-regional, file: gcp-pd-csi-sc-regional.yml} + - {name: gcp-pd-csi-sc-zonal, file: gcp-pd-csi-sc-zonal.yml} + register: gcp_pd_csi_manifests + when: inventory_hostname == groups['kube_control_plane'][0] + +- name: GCP PD CSI Driver | Apply Manifests + kube: + kubectl: "{{ bin_dir }}/kubectl" + filename: "{{ kube_config_dir }}/{{ item.item.file }}" + state: "latest" + with_items: + - "{{ gcp_pd_csi_manifests.results }}" + when: + - inventory_hostname == groups['kube_control_plane'][0] + - not item is skipped + loop_control: + label: "{{ item.item.file }}" diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-controller.yml.j2 b/kubespray/project/roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-controller.yml.j2 new file mode 100644 index 0000000..61157d8 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-controller.yml.j2 @@ -0,0 +1,165 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: csi-gce-pd-controller + namespace: kube-system +spec: + replicas: {{ gcp_pd_csi_controller_replicas }} + selector: + matchLabels: + app: gcp-compute-persistent-disk-csi-driver + template: + metadata: + labels: + app: gcp-compute-persistent-disk-csi-driver + spec: + # Host network must be used for interaction with Workload Identity in GKE + # since it replaces GCE Metadata Server with GKE Metadata Server. Remove + # this requirement when issue is resolved and before any exposure of + # metrics ports + hostNetwork: true + nodeSelector: + kubernetes.io/os: linux + serviceAccountName: csi-gce-pd-controller-sa + priorityClassName: csi-gce-pd-controller + containers: + - name: csi-provisioner + image: {{ csi_provisioner_image_repo }}:{{ csi_provisioner_image_tag }} + args: + - "--v=5" + - "--csi-address=/csi/csi.sock" + - "--feature-gates=Topology=true" + - "--http-endpoint=:22011" + - "--leader-election-namespace=$(PDCSI_NAMESPACE)" + - "--timeout=250s" + - "--extra-create-metadata" + # - "--run-controller-service=false" # disable the controller service of the CSI driver + # - "--run-node-service=false" # disable the node service of the CSI driver + - "--leader-election" + - "--default-fstype=ext4" + - "--controller-publish-readonly=true" + env: + - name: PDCSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - containerPort: 22011 + name: http-endpoint + protocol: TCP + livenessProbe: + failureThreshold: 1 + httpGet: + path: /healthz/leader-election + port: http-endpoint + initialDelaySeconds: 10 + timeoutSeconds: 10 + periodSeconds: 20 + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: csi-attacher + image: {{ csi_attacher_image_repo }}:{{ csi_attacher_image_tag }} + args: + - "--v=5" + - "--csi-address=/csi/csi.sock" + - "--http-endpoint=:22012" + - "--leader-election" + - "--leader-election-namespace=$(PDCSI_NAMESPACE)" + - "--timeout=250s" + env: + - name: PDCSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - containerPort: 22012 + name: http-endpoint + protocol: TCP + livenessProbe: + failureThreshold: 1 + httpGet: + path: /healthz/leader-election + port: http-endpoint + initialDelaySeconds: 10 + timeoutSeconds: 10 + periodSeconds: 20 + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: csi-resizer + image: {{ csi_resizer_image_repo }}:{{ csi_resizer_image_tag }} + args: + - "--v=5" + - "--csi-address=/csi/csi.sock" + - "--http-endpoint=:22013" + - "--leader-election" + - "--leader-election-namespace=$(PDCSI_NAMESPACE)" + - "--handle-volume-inuse-error=false" + env: + - name: PDCSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - containerPort: 22013 + name: http-endpoint + protocol: TCP + livenessProbe: + failureThreshold: 1 + httpGet: + path: /healthz/leader-election + port: http-endpoint + initialDelaySeconds: 10 + timeoutSeconds: 10 + periodSeconds: 20 + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: csi-snapshotter + image: {{ csi_snapshotter_image_repo }}:{{ csi_snapshotter_image_tag }} + args: + - "--v=5" + - "--csi-address=/csi/csi.sock" + - "--metrics-address=:22014" + - "--leader-election" + - "--leader-election-namespace=$(PDCSI_NAMESPACE)" + - "--timeout=300s" + env: + - name: PDCSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: gce-pd-driver + # Don't change base image without changing pdImagePlaceholder in + # test/k8s-integration/main.go + image: {{ gcp_pd_csi_plugin_image_repo }}:{{ gcp_pd_csi_plugin_image_tag }} + args: + - "--v=5" + - "--endpoint=unix:/csi/csi.sock" + env: + - name: GOOGLE_APPLICATION_CREDENTIALS + value: "/etc/cloud-sa/cloud-sa.json" + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: cloud-sa-volume + readOnly: true + mountPath: "/etc/cloud-sa" + volumes: + - name: socket-dir + emptyDir: {} + - name: cloud-sa-volume + secret: + secretName: cloud-sa +--- +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: pd.csi.storage.gke.io +spec: + attachRequired: true + podInfoOnMount: false \ No newline at end of file diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-cred-secret.yml.j2 b/kubespray/project/roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-cred-secret.yml.j2 new file mode 100644 index 0000000..f8291a4 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-cred-secret.yml.j2 @@ -0,0 +1,8 @@ +--- +kind: Secret +apiVersion: v1 +metadata: + name: cloud-sa + namespace: kube-system +data: + cloud-sa.json: {{ gcp_cred_secret.content }} diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-node.yml.j2 b/kubespray/project/roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-node.yml.j2 new file mode 100644 index 0000000..9aad620 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-node.yml.j2 @@ -0,0 +1,112 @@ +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: csi-gce-pd-node + namespace: kube-system +spec: + selector: + matchLabels: + app: gcp-compute-persistent-disk-csi-driver + template: + metadata: + labels: + app: gcp-compute-persistent-disk-csi-driver + spec: + # Host network must be used for interaction with Workload Identity in GKE + # since it replaces GCE Metadata Server with GKE Metadata Server. Remove + # this requirement when issue is resolved and before any exposure of + # metrics ports. + hostNetwork: true + priorityClassName: csi-gce-pd-node + serviceAccountName: csi-gce-pd-node-sa + containers: + - name: csi-driver-registrar + image: {{ csi_node_driver_registrar_image_repo }}:{{ csi_node_driver_registrar_image_tag }} + args: + - "--v=5" + - "--csi-address=/csi/csi.sock" + - "--kubelet-registration-path=/var/lib/kubelet/plugins/pd.csi.storage.gke.io/csi.sock" + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "rm -rf /registration/pd.csi.storage.gke.io /registration/pd.csi.storage.gke.io-reg.sock"] + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + - name: gce-pd-driver + securityContext: + privileged: true + # Don't change base image without changing pdImagePlaceholder in + # test/k8s-integration/main.go + image: {{ gcp_pd_csi_plugin_image_repo }}:{{ gcp_pd_csi_plugin_image_tag }} + args: + - "--v=5" + - "--endpoint=unix:/csi/csi.sock" + - "--run-controller-service=false" + volumeMounts: + - name: kubelet-dir + mountPath: /var/lib/kubelet + mountPropagation: "Bidirectional" + - name: plugin-dir + mountPath: /csi + - name: device-dir + mountPath: /dev + # The following mounts are required to trigger host udevadm from + # container + - name: udev-rules-etc + mountPath: /etc/udev + - name: udev-rules-lib + mountPath: /lib/udev + - name: udev-socket + mountPath: /run/udev + - name: sys + mountPath: /sys + nodeSelector: + kubernetes.io/os: linux + volumes: + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + - name: kubelet-dir + hostPath: + path: /var/lib/kubelet + type: Directory + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/pd.csi.storage.gke.io/ + type: DirectoryOrCreate + - name: device-dir + hostPath: + path: /dev + type: Directory + # The following mounts are required to trigger host udevadm from + # container + - name: udev-rules-etc + hostPath: + path: /etc/udev + type: Directory + - name: udev-rules-lib + hostPath: + path: /lib/udev + type: Directory + - name: udev-socket + hostPath: + path: /run/udev + type: Directory + - name: sys + hostPath: + path: /sys + type: Directory + # https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + # See "special case". This will tolerate everything. Node component should + # be scheduled on all nodes. + tolerations: + - operator: Exists \ No newline at end of file diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-sc-regional.yml.j2 b/kubespray/project/roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-sc-regional.yml.j2 new file mode 100644 index 0000000..57a8675 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-sc-regional.yml.j2 @@ -0,0 +1,9 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: csi-gce-pd-regional +provisioner: pd.csi.storage.gke.io +parameters: + type: pd-balanced + replication-type: regional-pd +volumeBindingMode: WaitForFirstConsumer \ No newline at end of file diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-sc-zonal.yml.j2 b/kubespray/project/roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-sc-zonal.yml.j2 new file mode 100644 index 0000000..e9bedaf --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-sc-zonal.yml.j2 @@ -0,0 +1,8 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: csi-gce-pd-zonal +provisioner: pd.csi.storage.gke.io +parameters: + type: pd-balanced +volumeBindingMode: WaitForFirstConsumer \ No newline at end of file diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-setup.yml.j2 b/kubespray/project/roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-setup.yml.j2 new file mode 100644 index 0000000..67ce7f6 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/gcp_pd/templates/gcp-pd-csi-setup.yml.j2 @@ -0,0 +1,241 @@ +##### Node Service Account, Roles, RoleBindings +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-gce-pd-node-sa + namespace: kube-system + +--- +##### Controller Service Account, Roles, Rolebindings +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-gce-pd-controller-sa + namespace: kube-system + +--- +# xref: https://github.com/kubernetes-csi/external-provisioner/blob/master/deploy/kubernetes/rbac.yaml +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-gce-pd-provisioner-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["get", "list"] + # Access to volumeattachments is only needed when the CSI driver + # has the PUBLISH_UNPUBLISH_VOLUME controller capability. + # In that case, external-provisioner will watch volumeattachments + # to determine when it is safe to delete a volume. + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch"] +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-gce-pd-controller-provisioner-binding +subjects: + - kind: ServiceAccount + name: csi-gce-pd-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-gce-pd-provisioner-role + apiGroup: rbac.authorization.k8s.io + +--- +# xref: https://github.com/kubernetes-csi/external-attacher/blob/master/deploy/kubernetes/rbac.yaml +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-gce-pd-attacher-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["patch"] +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-gce-pd-controller-attacher-binding +subjects: + - kind: ServiceAccount + name: csi-gce-pd-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-gce-pd-attacher-role + apiGroup: rbac.authorization.k8s.io + +--- + +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: csi-gce-pd-controller +value: 900000000 +globalDefault: false +description: "This priority class should be used for the GCE PD CSI driver controller deployment only." + +--- + +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: csi-gce-pd-node +value: 900001000 +globalDefault: false +description: "This priority class should be used for the GCE PD CSI driver node deployment only." + +--- + +# Resizer must be able to work with PVCs, PVs, SCs. +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-gce-pd-resizer-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + # If handle-volume-inuse-error=true, the pod specific rbac is needed + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-gce-pd-resizer-binding +subjects: + - kind: ServiceAccount + name: csi-gce-pd-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-gce-pd-resizer-role + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: csi-gce-pd-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: csi-gce-pd-node-deploy +subjects: +- kind: ServiceAccount + name: csi-gce-pd-controller-sa + namespace: kube-system + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: csi-gce-pd-snapshotter-role +rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + # Secrets resource omitted since GCE PD snapshots does not require them + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update", "patch"] +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-gce-pd-controller-snapshotter-binding +subjects: + - kind: ServiceAccount + name: csi-gce-pd-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-gce-pd-snapshotter-role + apiGroup: rbac.authorization.k8s.io +--- + +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-gce-pd-leaderelection-role + namespace: kube-system + labels: + k8s-app: gcp-compute-persistent-disk-csi-driver +rules: +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + +--- + +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-gce-pd-controller-leaderelection-binding + namespace: kube-system + labels: + k8s-app: gcp-compute-persistent-disk-csi-driver +subjects: +- kind: ServiceAccount + name: csi-gce-pd-controller-sa + namespace: kube-system +roleRef: + kind: Role + name: csi-gce-pd-leaderelection-role + apiGroup: rbac.authorization.k8s.io diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/upcloud/defaults/main.yml b/kubespray/project/roles/kubernetes-apps/csi_driver/upcloud/defaults/main.yml new file mode 100644 index 0000000..ea828f3 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/upcloud/defaults/main.yml @@ -0,0 +1,16 @@ +--- +upcloud_csi_controller_replicas: 1 +upcloud_csi_provisioner_image_tag: "v3.1.0" +upcloud_csi_attacher_image_tag: "v3.4.0" +upcloud_csi_resizer_image_tag: "v1.4.0" +upcloud_csi_plugin_image_tag: "v0.3.3" +upcloud_csi_node_image_tag: "v2.5.0" +upcloud_username: "{{ lookup('env', 'UPCLOUD_USERNAME') }}" +upcloud_password: "{{ lookup('env', 'UPCLOUD_PASSWORD') }}" +upcloud_tolerations: [] +upcloud_csi_enable_volume_snapshot: false +upcloud_csi_snapshot_controller_replicas: 2 +upcloud_csi_snapshotter_image_tag: "v4.2.1" +upcloud_csi_snapshot_controller_image_tag: "v4.2.1" +upcloud_csi_snapshot_validation_webhook_image_tag: "v4.2.1" +upcloud_cacert: "{{ lookup('env', 'OS_CACERT') }}" diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/upcloud/tasks/main.yml b/kubespray/project/roles/kubernetes-apps/csi_driver/upcloud/tasks/main.yml new file mode 100644 index 0000000..8f0b69f --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/upcloud/tasks/main.yml @@ -0,0 +1,40 @@ +--- +- name: UpCloud CSI Driver | Check if UPCLOUD_USERNAME exists + fail: + msg: "UpCloud username is missing. Env UPCLOUD_USERNAME is mandatory" + when: upcloud_username is not defined or not upcloud_username + +- name: UpCloud CSI Driver | Check if UPCLOUD_PASSWORD exists + fail: + msg: "UpCloud password is missing. Env UPCLOUD_PASSWORD is mandatory" + when: + - upcloud_username is defined + - upcloud_username | length > 0 + - upcloud_password is not defined or not upcloud_password + +- name: UpCloud CSI Driver | Generate Manifests + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/{{ item.file }}" + mode: 0644 + with_items: + - {name: upcloud-csi-cred-secret, file: upcloud-csi-cred-secret.yml} + - {name: upcloud-csi-setup, file: upcloud-csi-setup.yml} + - {name: upcloud-csi-controller, file: upcloud-csi-controller.yml} + - {name: upcloud-csi-node, file: upcloud-csi-node.yml} + - {name: upcloud-csi-driver, file: upcloud-csi-driver.yml} + register: upcloud_csi_manifests + when: inventory_hostname == groups['kube_control_plane'][0] + +- name: UpCloud CSI Driver | Apply Manifests + kube: + kubectl: "{{ bin_dir }}/kubectl" + filename: "{{ kube_config_dir }}/{{ item.item.file }}" + state: "latest" + with_items: + - "{{ upcloud_csi_manifests.results }}" + when: + - inventory_hostname == groups['kube_control_plane'][0] + - not item is skipped + loop_control: + label: "{{ item.item.file }}" diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/upcloud/templates/upcloud-csi-controller.yml.j2 b/kubespray/project/roles/kubernetes-apps/csi_driver/upcloud/templates/upcloud-csi-controller.yml.j2 new file mode 100644 index 0000000..1b8519d --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/upcloud/templates/upcloud-csi-controller.yml.j2 @@ -0,0 +1,93 @@ +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: csi-upcloud-controller + namespace: kube-system +spec: + serviceName: "csi-upcloud" + replicas: {{ upcloud_csi_controller_replicas }} + selector: + matchLabels: + app: csi-upcloud-controller + template: + metadata: + labels: + app: csi-upcloud-controller + role: csi-upcloud + spec: + priorityClassName: system-cluster-critical + serviceAccount: csi-upcloud-controller-sa + containers: + - name: csi-provisioner + image: registry.k8s.io/sig-storage/csi-provisioner:{{ upcloud_csi_provisioner_image_tag }} + args: + - "--csi-address=$(ADDRESS)" + - "--v=5" + - "--timeout=600s" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + imagePullPolicy: "Always" + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: csi-attacher + image: registry.k8s.io/sig-storage/csi-attacher:{{ upcloud_csi_attacher_image_tag }} + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + - "--timeout=120s" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + imagePullPolicy: "Always" + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: csi-resizer + image: registry.k8s.io/sig-storage/csi-resizer:{{ upcloud_csi_resizer_image_tag }} + args: + - "--v=5" + - "--timeout=120s" + - "--csi-address=$(ADDRESS)" + - "--handle-volume-inuse-error=true" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + imagePullPolicy: "Always" + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: csi-upcloud-plugin + image: ghcr.io/upcloudltd/upcloud-csi:{{ upcloud_csi_plugin_image_tag }} + args: + - "--endpoint=$(CSI_ENDPOINT)" + - "--nodehost=$(NODE_ID)" + - "--username=$(UPCLOUD_USERNAME)" + - "--password=$(UPCLOUD_PASSWORD)" + env: + - name: CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + - name: UPCLOUD_USERNAME + valueFrom: + secretKeyRef: + name: upcloud + key: username + - name: UPCLOUD_PASSWORD + valueFrom: + secretKeyRef: + name: upcloud + key: password + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + imagePullPolicy: "Always" + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + imagePullSecrets: + - name: regcred + volumes: + - name: socket-dir + emptyDir: {} diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/upcloud/templates/upcloud-csi-cred-secret.yml.j2 b/kubespray/project/roles/kubernetes-apps/csi_driver/upcloud/templates/upcloud-csi-cred-secret.yml.j2 new file mode 100644 index 0000000..5e91d88 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/upcloud/templates/upcloud-csi-cred-secret.yml.j2 @@ -0,0 +1,9 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: upcloud + namespace: kube-system +stringData: + username: {{ upcloud_username }} + password: {{ upcloud_password }} diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/upcloud/templates/upcloud-csi-driver.yml.j2 b/kubespray/project/roles/kubernetes-apps/csi_driver/upcloud/templates/upcloud-csi-driver.yml.j2 new file mode 100644 index 0000000..8f4c612 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/upcloud/templates/upcloud-csi-driver.yml.j2 @@ -0,0 +1,8 @@ +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: storage.csi.upcloud.com +spec: + attachRequired: true + podInfoOnMount: true + fsGroupPolicy: File diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/upcloud/templates/upcloud-csi-node.yml.j2 b/kubespray/project/roles/kubernetes-apps/csi_driver/upcloud/templates/upcloud-csi-node.yml.j2 new file mode 100644 index 0000000..7ed39be --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/upcloud/templates/upcloud-csi-node.yml.j2 @@ -0,0 +1,101 @@ +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: csi-upcloud-node + namespace: kube-system +spec: + selector: + matchLabels: + app: csi-upcloud-node + template: + metadata: + labels: + app: csi-upcloud-node + role: csi-upcloud + spec: + priorityClassName: system-node-critical + serviceAccount: csi-upcloud-node-sa + hostNetwork: true + containers: + - name: csi-node-driver-registrar + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:{{ upcloud_csi_node_image_tag }} + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)" + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/storage.csi.upcloud.com/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: plugin-dir + mountPath: /csi/ + - name: registration-dir + mountPath: /registration/ + - name: csi-upcloud-plugin + image: ghcr.io/upcloudltd/upcloud-csi:{{ upcloud_csi_plugin_image_tag }} + args: + - "--endpoint=$(CSI_ENDPOINT)" + - "--nodehost=$(NODE_ID)" + - "--username=$(UPCLOUD_USERNAME)" + - "--password=$(UPCLOUD_PASSWORD)" + env: + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: UPCLOUD_USERNAME + valueFrom: + secretKeyRef: + name: upcloud + key: username + - name: UPCLOUD_PASSWORD + valueFrom: + secretKeyRef: + name: upcloud + key: password + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + imagePullPolicy: "Always" + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: pods-mount-dir + mountPath: /var/lib/kubelet + # needed so that any mounts setup inside this container are + # propagated back to the host machine. + mountPropagation: "Bidirectional" + - name: device-dir + mountPath: /dev + imagePullSecrets: + - name: regcred + volumes: + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: DirectoryOrCreate + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/storage.csi.upcloud.com + type: DirectoryOrCreate + - name: pods-mount-dir + hostPath: + path: /var/lib/kubelet + type: Directory + - name: device-dir + hostPath: + path: /dev +{% if upcloud_tolerations %} + tolerations: + {{ upcloud_tolerations | to_nice_yaml(indent=2) | indent(width=8) }} +{% endif %} diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/upcloud/templates/upcloud-csi-setup.yml.j2 b/kubespray/project/roles/kubernetes-apps/csi_driver/upcloud/templates/upcloud-csi-setup.yml.j2 new file mode 100644 index 0000000..5af71d2 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/upcloud/templates/upcloud-csi-setup.yml.j2 @@ -0,0 +1,185 @@ +kind: ServiceAccount +apiVersion: v1 +metadata: + name: csi-upcloud-controller-sa + namespace: kube-system + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-upcloud-node-sa + namespace: kube-system + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-upcloud-node-driver-registrar-role + namespace: kube-system +rules: + - apiGroups: [ "" ] + resources: [ "events" ] + verbs: [ "get", "list", "watch", "create", "update", "patch" ] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-upcloud-node-driver-registrar-binding +subjects: + - kind: ServiceAccount + name: csi-upcloud-node-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-upcloud-node-driver-registrar-role + apiGroup: rbac.authorization.k8s.io + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-upcloud-provisioner-role +rules: + - apiGroups: [ "" ] + resources: [ "secrets" ] + verbs: [ "get", "list" ] + - apiGroups: [ "" ] + resources: [ "persistentvolumes" ] + verbs: [ "get", "list", "watch", "create", "delete" ] + - apiGroups: [ "" ] + resources: [ "persistentvolumeclaims" ] + verbs: [ "get", "list", "watch", "update" ] + - apiGroups: [ "storage.k8s.io" ] + resources: [ "storageclasses" ] + verbs: [ "get", "list", "watch" ] + - apiGroups: [ "storage.k8s.io" ] + resources: [ "csinodes" ] + verbs: [ "get", "list", "watch" ] + - apiGroups: [ "" ] + resources: [ "events" ] + verbs: [ "list", "watch", "create", "update", "patch" ] + - apiGroups: [ "" ] + resources: [ "nodes" ] + verbs: [ "get", "list", "watch" ] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-upcloud-provisioner-binding +subjects: + - kind: ServiceAccount + name: csi-upcloud-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-upcloud-provisioner-role + apiGroup: rbac.authorization.k8s.io + +--- +# Attacher must be able to work with PVs, nodes and VolumeAttachments +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-upcloud-attacher-role +rules: + - apiGroups: [ "" ] + resources: [ "persistentvolumes" ] + verbs: [ "get", "list", "watch", "update", "patch" ] + - apiGroups: [ "" ] + resources: [ "nodes" ] + verbs: [ "get", "list", "watch" ] + - apiGroups: [ "storage.k8s.io" ] + resources: [ "csinodes" ] + verbs: [ "get", "list", "watch" ] + - apiGroups: [ "storage.k8s.io" ] + resources: [ "volumeattachments" ] + verbs: [ "get", "list", "watch", "update", "patch" ] + - apiGroups: [ "storage.k8s.io" ] + resources: [ "volumeattachments/status" ] + verbs: [ "get", "list", "watch", "update", "patch" ] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-upcloud-attacher-binding +subjects: + - kind: ServiceAccount + name: csi-upcloud-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-upcloud-attacher-role + apiGroup: rbac.authorization.k8s.io + +--- +# Provisioner must be able to work with endpoints and leases in current namespace +# if (and only if) leadership election is enabled +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + namespace: kube-system + name: csi-upcloud-provisioner-cfg-role +rules: +- apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "watch", "list", "delete", "update", "create"] +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-provisioner-role-cfg-binding + namespace: kube-system +subjects: + - kind: ServiceAccount + name: csi-upcloud-controller-sa + namespace: kube-system +roleRef: + kind: Role + name: csi-upcloud-provisioner-cfg-role + apiGroup: rbac.authorization.k8s.io + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-upcloud-resizer-role +rules: + - apiGroups: [ "" ] + resources: [ "persistentvolumes" ] + verbs: [ "get", "list", "watch", "update", "patch" ] + - apiGroups: [ "" ] + resources: [ "persistentvolumeclaims" ] + verbs: [ "get", "list", "watch" ] + - apiGroups: [ "" ] + resources: [ "persistentvolumeclaims/status" ] + verbs: [ "update", "patch" ] + - apiGroups: [ "" ] + resources: [ "events" ] + verbs: [ "list", "watch", "create", "update", "patch" ] + - apiGroups: [ "" ] + resources: [ "pods" ] + verbs: [ "watch", "list" ] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-upcloud-resizer-binding +subjects: + - kind: ServiceAccount + name: csi-upcloud-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-upcloud-resizer-role + apiGroup: rbac.authorization.k8s.io diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/vsphere/defaults/main.yml b/kubespray/project/roles/kubernetes-apps/csi_driver/vsphere/defaults/main.yml new file mode 100644 index 0000000..0d41441 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/vsphere/defaults/main.yml @@ -0,0 +1,54 @@ +--- +external_vsphere_vcenter_port: "443" +external_vsphere_insecure: "true" +external_vsphere_kubernetes_cluster_id: "kubernetes-cluster-id" +external_vsphere_version: "7.0u1" + +vsphere_syncer_image_tag: "v3.1.0" +vsphere_csi_attacher_image_tag: "v4.3.0" +vsphere_csi_controller: "v3.1.0" +vsphere_csi_liveness_probe_image_tag: "v2.10.0" +vsphere_csi_provisioner_image_tag: "v3.5.0" +vsphere_csi_snapshotter_image_tag: "v6.2.2" +vsphere_csi_node_driver_registrar_image_tag: "v2.8.0" +vsphere_csi_driver_image_tag: "v3.1.0" +vsphere_csi_resizer_tag: "v1.8.0" + +# Set to kube-system for backward compatibility, should be change to vmware-system-csi on the long run +vsphere_csi_namespace: "kube-system" + +vsphere_csi_controller_replicas: 1 + +csi_endpoint: '{% if external_vsphere_version >= "7.0u1" %}/csi{% else %}/var/lib/csi/sockets/pluginproxy{% endif %}' + +vsphere_csi_aggressive_node_drain: False +vsphere_csi_aggressive_node_unreachable_timeout: 300 +vsphere_csi_aggressive_node_not_ready_timeout: 300 + +vsphere_csi_node_affinity: {} + +# If this is true, debug information will be displayed but +# may contain some private data, so it is recommended to set it to false +# in the production environment. +unsafe_show_logs: false + +# https://github.com/kubernetes-sigs/vsphere-csi-driver/blob/master/docs/book/features/volume_snapshot.md#how-to-enable-volume-snapshot--restore-feature-in-vsphere-csi- +# according to the above link , we can controler the block-volume-snapshot parameter +vsphere_csi_block_volume_snapshot: false + +external_vsphere_user: "{{ lookup('env', 'VSPHERE_USER') }}" +external_vsphere_password: "{{ lookup('env', 'VSPHERE_PASSWORD') }}" + +# Controller resources +vsphere_csi_snapshotter_resources: {} +vsphere_csi_provisioner_resources: {} +vsphere_syncer_resources: {} +vsphere_csi_liveness_probe_controller_resources: {} +vsphere_csi_resources: {} +vsphere_csi_resizer_resources: {} +vsphere_csi_attacher_resources: {} + +# DaemonSet node resources +vsphere_csi_node_driver_registrar_resources: {} +vsphere_csi_driver_resources: {} +vsphere_csi_liveness_probe_ds_resources: {} diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/vsphere/tasks/main.yml b/kubespray/project/roles/kubernetes-apps/csi_driver/vsphere/tasks/main.yml new file mode 100644 index 0000000..102dd8b --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/vsphere/tasks/main.yml @@ -0,0 +1,55 @@ +--- +- name: VSphere CSI Driver | Check vsphare credentials + include_tasks: vsphere-credentials-check.yml + +- name: VSphere CSI Driver | Generate CSI cloud-config + template: + src: "{{ item }}.j2" + dest: "{{ kube_config_dir }}/{{ item }}" + mode: 0640 + with_items: + - vsphere-csi-cloud-config + when: inventory_hostname == groups['kube_control_plane'][0] + +- name: VSphere CSI Driver | Generate Manifests + template: + src: "{{ item }}.j2" + dest: "{{ kube_config_dir }}/{{ item }}" + mode: 0644 + with_items: + - vsphere-csi-namespace.yml + - vsphere-csi-driver.yml + - vsphere-csi-controller-rbac.yml + - vsphere-csi-node-rbac.yml + - vsphere-csi-controller-config.yml + - vsphere-csi-controller-deployment.yml + - vsphere-csi-controller-service.yml + - vsphere-csi-node.yml + register: vsphere_csi_manifests + when: inventory_hostname == groups['kube_control_plane'][0] + +- name: VSphere CSI Driver | Apply Manifests + kube: + kubectl: "{{ bin_dir }}/kubectl" + filename: "{{ kube_config_dir }}/{{ item.item }}" + state: "latest" + with_items: + - "{{ vsphere_csi_manifests.results }}" + when: + - inventory_hostname == groups['kube_control_plane'][0] + - not item is skipped + loop_control: + label: "{{ item.item }}" + +- name: VSphere CSI Driver | Generate a CSI secret manifest + command: "{{ kubectl }} create secret generic vsphere-config-secret --from-file=csi-vsphere.conf={{ kube_config_dir }}/vsphere-csi-cloud-config -n {{ vsphere_csi_namespace }} --dry-run --save-config -o yaml" + register: vsphere_csi_secret_manifest + when: inventory_hostname == groups['kube_control_plane'][0] + no_log: "{{ not (unsafe_show_logs | bool) }}" + +- name: VSphere CSI Driver | Apply a CSI secret manifest + command: + cmd: "{{ kubectl }} apply -f -" + stdin: "{{ vsphere_csi_secret_manifest.stdout }}" + when: inventory_hostname == groups['kube_control_plane'][0] + no_log: "{{ not (unsafe_show_logs | bool) }}" diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/vsphere/tasks/vsphere-credentials-check.yml b/kubespray/project/roles/kubernetes-apps/csi_driver/vsphere/tasks/vsphere-credentials-check.yml new file mode 100644 index 0000000..3504f60 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/vsphere/tasks/vsphere-credentials-check.yml @@ -0,0 +1,38 @@ +--- +- name: External vSphere Cloud Provider | check external_vsphere_vcenter_ip value + fail: + msg: "external_vsphere_vcenter_ip is missing" + when: external_vsphere_vcenter_ip is not defined or not external_vsphere_vcenter_ip + +- name: External vSphere Cloud Provider | check external_vsphere_vcenter_port value + fail: + msg: "external_vsphere_vcenter_port is missing" + when: external_vsphere_vcenter_port is not defined or not external_vsphere_vcenter_port + +- name: External vSphere Cloud Provider | check external_vsphere_insecure value + fail: + msg: "external_vsphere_insecure is missing" + when: external_vsphere_insecure is not defined or not external_vsphere_insecure + +- name: External vSphere Cloud Provider | check external_vsphere_user value + fail: + msg: "external_vsphere_user is missing" + when: external_vsphere_user is not defined or not external_vsphere_user + +- name: External vSphere Cloud Provider | check external_vsphere_password value + fail: + msg: "external_vsphere_password is missing" + when: + - external_vsphere_password is not defined or not external_vsphere_password + +- name: External vSphere Cloud Provider | check external_vsphere_datacenter value + fail: + msg: "external_vsphere_datacenter is missing" + when: + - external_vsphere_datacenter is not defined or not external_vsphere_datacenter + +- name: External vSphere Cloud Provider | check external_vsphere_kubernetes_cluster_id value + fail: + msg: "external_vsphere_kubernetes_cluster_id is missing" + when: + - external_vsphere_kubernetes_cluster_id is not defined or not external_vsphere_kubernetes_cluster_id diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-cloud-config.j2 b/kubespray/project/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-cloud-config.j2 new file mode 100644 index 0000000..ee5033a --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-cloud-config.j2 @@ -0,0 +1,9 @@ +[Global] +cluster-id = "{{ external_vsphere_kubernetes_cluster_id }}" + +[VirtualCenter "{{ external_vsphere_vcenter_ip }}"] +insecure-flag = "{{ external_vsphere_insecure }}" +user = "{{ external_vsphere_user }}" +password = "{{ external_vsphere_password }}" +port = "{{ external_vsphere_vcenter_port }}" +datacenters = "{{ external_vsphere_datacenter }}" diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-config.yml.j2 b/kubespray/project/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-config.yml.j2 new file mode 100644 index 0000000..fb52d10 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-config.yml.j2 @@ -0,0 +1,31 @@ +apiVersion: v1 +data: +{% if external_vsphere_version >= "7.0" %} + "csi-auth-check": "true" +{% else %} + "csi-auth-check": "false" +{% endif %} + "csi-auth-check": "true" + "online-volume-extend": "true" + "trigger-csi-fullsync": "false" + "async-query-volume": "true" + "block-volume-snapshot": "true" + "csi-windows-support": "false" + "list-volumes": "true" + "pv-to-backingdiskobjectid-mapping": "false" + "cnsmgr-suspend-create-volume": "true" + "topology-preferential-datastores": "true" + "max-pvscsi-targets-per-vm": "true" + "multi-vcenter-csi-topology": "true" + "csi-internal-generated-cluster-id": "true" + "listview-tasks": "true" +{% if vsphere_csi_controller is version('v2.7.0', '>=') %} + "improved-csi-idempotency": "true" + "improved-volume-topology": "true" + "use-csinode-id": "true" + "list-volumes": "false" +{% endif %} +kind: ConfigMap +metadata: + name: internal-feature-states.csi.vsphere.vmware.com + namespace: "{{ vsphere_csi_namespace }}" diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-deployment.yml.j2 b/kubespray/project/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-deployment.yml.j2 new file mode 100644 index 0000000..00d6e6a --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-deployment.yml.j2 @@ -0,0 +1,260 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: vsphere-csi-controller + namespace: "{{ vsphere_csi_namespace }}" +spec: + replicas: {{ vsphere_csi_controller_replicas }} + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + maxSurge: 0 + selector: + matchLabels: + app: vsphere-csi-controller + template: + metadata: + labels: + app: vsphere-csi-controller + role: vsphere-csi + spec: + priorityClassName: system-cluster-critical # Guarantees scheduling for critical system pods + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app" + operator: In + values: + - vsphere-csi-controller + topologyKey: "kubernetes.io/hostname" + serviceAccountName: vsphere-csi-controller + nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: + - operator: "Exists" + key: node-role.kubernetes.io/control-plane + effect: NoSchedule +{% if vsphere_csi_aggressive_node_drain %} + # set below toleration if you need an aggressive pod eviction in case when + # node becomes not-ready or unreachable. Default is 300 seconds if not specified. + - key: node.kubernetes.io/not-ready + operator: Exists + effect: NoExecute + tolerationSeconds: {{ vsphere_csi_aggressive_node_not_ready_timeout }} + - key: node.kubernetes.io/unreachable + operator: Exists + effect: NoExecute + tolerationSeconds: {{ vsphere_csi_aggressive_node_unreachable_timeout }} +{% endif %} + dnsPolicy: "Default" + containers: + - name: csi-attacher + image: {{ kube_image_repo }}/sig-storage/csi-attacher:{{ vsphere_csi_attacher_image_tag }} + args: + - "--v=4" + - "--timeout=300s" + - "--csi-address=$(ADDRESS)" + - "--leader-election" + - "--leader-election-lease-duration=120s" + - "--leader-election-renew-deadline=60s" + - "--leader-election-retry-period=30s" + - "--kube-api-qps=100" + - "--kube-api-burst=100" +{% if vsphere_csi_attacher_resources | length > 0 %} + resources: + {{ vsphere_csi_attacher_resources | default({}) | to_nice_yaml | trim | indent(width=12) }} +{% endif %} + env: + - name: ADDRESS + value: /csi/csi.sock + volumeMounts: + - mountPath: /csi + name: socket-dir +{% if external_vsphere_version >= "7.0" %} + - name: csi-resizer + image: {{ kube_image_repo }}/sig-storage/csi-resizer:{{ vsphere_csi_resizer_tag }} + args: + - "--v=4" + - "--timeout=300s" + - "--csi-address=$(ADDRESS)" + - "--handle-volume-inuse-error=false" + - "--kube-api-qps=100" + - "--kube-api-burst=100" + - "--leader-election" + - "--leader-election-lease-duration=120s" + - "--leader-election-renew-deadline=60s" + - "--leader-election-retry-period=30s" +{% if vsphere_csi_resizer_resources | length > 0 %} + resources: + {{ vsphere_csi_resizer_resources | default({}) | to_nice_yaml | trim | indent(width=12) }} +{% endif %} + env: + - name: ADDRESS + value: /csi/csi.sock + volumeMounts: + - mountPath: /csi + name: socket-dir +{% endif %} + - name: vsphere-csi-controller + image: {{ gcr_image_repo }}/cloud-provider-vsphere/csi/release/driver:{{ vsphere_csi_controller }} + args: + - "--fss-name=internal-feature-states.csi.vsphere.vmware.com" + - "--fss-namespace={{ vsphere_csi_namespace }}" +{% if vsphere_csi_resources | length > 0 %} + resources: + {{ vsphere_csi_resources | default({}) | to_nice_yaml | trim | indent(width=12) }} +{% endif %} + imagePullPolicy: {{ k8s_image_pull_policy }} + env: + - name: CSI_ENDPOINT + value: unix://{{ csi_endpoint }}/csi.sock + - name: X_CSI_MODE + value: "controller" + - name: X_CSI_SPEC_DISABLE_LEN_CHECK + value: "true" + - name: X_CSI_SERIAL_VOL_ACCESS_TIMEOUT + value: 3m + - name: VSPHERE_CSI_CONFIG + value: "/etc/cloud/csi-vsphere.conf" + - name: LOGGER_LEVEL + value: "PRODUCTION" # Options: DEVELOPMENT, PRODUCTION +{% if external_vsphere_version >= "7.0u1" %} + - name: INCLUSTER_CLIENT_QPS + value: "100" + - name: INCLUSTER_CLIENT_BURST + value: "100" +{% endif %} + volumeMounts: + - mountPath: /etc/cloud + name: vsphere-config-volume + readOnly: true + - mountPath: {{ csi_endpoint }} + name: socket-dir + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + ports: + - name: healthz + containerPort: 9808 + protocol: TCP + - name: prometheus + containerPort: 2112 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 30 + timeoutSeconds: 10 + periodSeconds: 180 + failureThreshold: 3 + - name: liveness-probe + image: {{ kube_image_repo }}/sig-storage/livenessprobe:{{ vsphere_csi_liveness_probe_image_tag }} + args: + - "--v=4" + - "--csi-address=$(ADDRESS)" +{% if vsphere_csi_liveness_probe_controller_resources | length > 0 %} + resources: + {{ vsphere_csi_liveness_probe_controller_resources | default({}) | to_nice_yaml | trim | indent(width=12) }} +{% endif %} + env: + - name: ADDRESS + value: {{ csi_endpoint }}/csi.sock + volumeMounts: + - name: socket-dir + mountPath: {{ csi_endpoint }} + - name: vsphere-syncer + image: {{ gcr_image_repo }}/cloud-provider-vsphere/csi/release/syncer:{{ vsphere_syncer_image_tag }} + args: + - "--leader-election" + - "--leader-election-lease-duration=30s" + - "--leader-election-renew-deadline=20s" + - "--leader-election-retry-period=10s" + - "--fss-name=internal-feature-states.csi.vsphere.vmware.com" + - "--fss-namespace={{ vsphere_csi_namespace }}" + imagePullPolicy: {{ k8s_image_pull_policy }} + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + ports: + - containerPort: 2113 + name: prometheus + protocol: TCP +{% if vsphere_syncer_resources | length > 0 %} + resources: + {{ vsphere_syncer_resources | default({}) | to_nice_yaml | trim | indent(width=12) }} +{% endif %} + env: + - name: FULL_SYNC_INTERVAL_MINUTES + value: "30" + - name: VSPHERE_CSI_CONFIG + value: "/etc/cloud/csi-vsphere.conf" + - name: LOGGER_LEVEL + value: "PRODUCTION" # Options: DEVELOPMENT, PRODUCTION +{% if external_vsphere_version >= "7.0u1" %} + - name: INCLUSTER_CLIENT_QPS + value: "100" + - name: INCLUSTER_CLIENT_BURST + value: "100" +{% endif %} + volumeMounts: + - mountPath: /etc/cloud + name: vsphere-config-volume + readOnly: true + - name: csi-provisioner + image: {{ kube_image_repo }}/sig-storage/csi-provisioner:{{ vsphere_csi_provisioner_image_tag }} + args: + - "--v=4" + - "--timeout=300s" + - "--csi-address=$(ADDRESS)" + - "--kube-api-qps=100" + - "--kube-api-burst=100" + - "--leader-election" + - "--leader-election-lease-duration=120s" + - "--leader-election-renew-deadline=60s" + - "--leader-election-retry-period=30s" + - "--default-fstype=ext4" + - "--leader-election" + - "--default-fstype=ext4" + # needed only for topology aware setup + #- "--feature-gates=Topology=true" + #- "--strict-topology" + env: + - name: ADDRESS + value: /csi/csi.sock + volumeMounts: + - mountPath: /csi + name: socket-dir +{% if vsphere_csi_provisioner_resources | length > 0 %} + resources: + {{ vsphere_csi_provisioner_resources | default({}) | to_nice_yaml | trim | indent(width=12) }} +{% endif %} + - name: csi-snapshotter + image: {{ kube_image_repo }}/sig-storage/csi-snapshotter:{{ vsphere_csi_snapshotter_image_tag }} + args: + - "--v=4" + - "--kube-api-qps=100" + - "--kube-api-burst=100" + - "--timeout=300s" + - "--csi-address=$(ADDRESS)" + - "--leader-election" + - "--leader-election-lease-duration=120s" + - "--leader-election-renew-deadline=60s" + - "--leader-election-retry-period=30s" + env: + - name: ADDRESS + value: /csi/csi.sock + volumeMounts: + - mountPath: /csi + name: socket-dir + volumes: + - name: vsphere-config-volume + secret: + secretName: vsphere-config-secret + - name: socket-dir + emptyDir: {} diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-rbac.yml.j2 b/kubespray/project/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-rbac.yml.j2 new file mode 100644 index 0000000..013d3dc --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-rbac.yml.j2 @@ -0,0 +1,89 @@ +kind: ServiceAccount +apiVersion: v1 +metadata: + name: vsphere-csi-controller + namespace: "{{ vsphere_csi_namespace }}" +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: vsphere-csi-controller-role +rules: + - apiGroups: [""] + resources: ["nodes", "pods"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "create"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] +{% if external_vsphere_version >= "7.0" %} + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] +{% if external_vsphere_version >= "7.0u1" %} + verbs: ["patch"] +{% else %} + verbs: ["update", "patch"] +{% endif %} +{% endif %} + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "update", "delete", "patch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] +{% if vsphere_csi_controller is version('v2.0.0', '>=') %} + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] +{% endif %} + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses","csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "patch", "update"] + - apiGroups: ["cns.vmware.com"] + resources: ["triggercsifullsyncs"] + verbs: ["create", "get", "update", "watch", "list"] + - apiGroups: ["cns.vmware.com"] + resources: ["cnsvspherevolumemigrations"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["get", "create", "update"] + - apiGroups: ["cns.vmware.com"] + resources: ["cnsvolumeoperationrequests"] + verbs: ["create", "get", "list", "update", "delete"] + - apiGroups: [ "cns.vmware.com" ] + resources: [ "csinodetopologies" ] + verbs: ["get", "update", "watch", "list"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["patch"] + - apiGroups: [ "snapshot.storage.k8s.io" ] + resources: [ "volumesnapshots" ] + verbs: [ "get", "list" ] + - apiGroups: [ "snapshot.storage.k8s.io" ] + resources: [ "volumesnapshotclasses" ] + verbs: [ "watch", "get", "list" ] + - apiGroups: [ "snapshot.storage.k8s.io" ] + resources: [ "volumesnapshotcontents" ] + verbs: [ "create", "get", "list", "watch", "update", "delete", "patch" ] + - apiGroups: [ "snapshot.storage.k8s.io" ] + resources: [ "volumesnapshotcontents/status" ] + verbs: [ "update", "patch" ] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: vsphere-csi-controller-binding +subjects: + - kind: ServiceAccount + name: vsphere-csi-controller + namespace: "{{ vsphere_csi_namespace }}" +roleRef: + kind: ClusterRole + name: vsphere-csi-controller-role + apiGroup: rbac.authorization.k8s.io diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-service.yml.j2 b/kubespray/project/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-service.yml.j2 new file mode 100644 index 0000000..75967ba --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-service.yml.j2 @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: vsphere-csi-controller + namespace: "{{ vsphere_csi_namespace }}" + labels: + app: vsphere-csi-controller +spec: + ports: + - name: ctlr + port: 2112 + targetPort: 2112 + protocol: TCP + - name: syncer + port: 2113 + targetPort: 2113 + protocol: TCP + selector: + app: vsphere-csi-controller diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-driver.yml.j2 b/kubespray/project/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-driver.yml.j2 new file mode 100644 index 0000000..ad3260e --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-driver.yml.j2 @@ -0,0 +1,7 @@ +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: csi.vsphere.vmware.com +spec: + attachRequired: true + podInfoOnMount: false diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-namespace.yml.j2 b/kubespray/project/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-namespace.yml.j2 new file mode 100644 index 0000000..6cf3150 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-namespace.yml.j2 @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: "{{ vsphere_csi_namespace }}" diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-node-rbac.yml.j2 b/kubespray/project/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-node-rbac.yml.j2 new file mode 100644 index 0000000..42896e1 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-node-rbac.yml.j2 @@ -0,0 +1,55 @@ +--- +kind: ServiceAccount +apiVersion: v1 +metadata: + name: vsphere-csi-node + namespace: "{{ vsphere_csi_namespace }}" +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: vsphere-csi-node-cluster-role +rules: + - apiGroups: ["cns.vmware.com"] + resources: ["csinodetopologies"] + verbs: ["create", "watch", "get", "patch" ] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: vsphere-csi-node-cluster-role-binding +subjects: + - kind: ServiceAccount + name: vsphere-csi-node + namespace: "{{ vsphere_csi_namespace }}" +roleRef: + kind: ClusterRole + name: vsphere-csi-node-cluster-role + apiGroup: rbac.authorization.k8s.io +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: vsphere-csi-node-role + namespace: "{{ vsphere_csi_namespace }}" +rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: vsphere-csi-node-binding + namespace: "{{ vsphere_csi_namespace }}" +subjects: + - kind: ServiceAccount + name: vsphere-csi-node + namespace: "{{ vsphere_csi_namespace }}" +roleRef: + kind: Role + name: vsphere-csi-node-role + apiGroup: rbac.authorization.k8s.io diff --git a/kubespray/project/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-node.yml.j2 b/kubespray/project/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-node.yml.j2 new file mode 100644 index 0000000..e110ee3 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-node.yml.j2 @@ -0,0 +1,170 @@ +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: vsphere-csi-node + namespace: "{{ vsphere_csi_namespace }}" +spec: + selector: + matchLabels: + app: vsphere-csi-node + updateStrategy: + type: "RollingUpdate" + rollingUpdate: + maxUnavailable: 1 + template: + metadata: + labels: + app: vsphere-csi-node + role: vsphere-csi + spec: + priorityClassName: system-node-critical + nodeSelector: + kubernetes.io/os: linux +{% if vsphere_csi_node_affinity %} + affinity: + {{ vsphere_csi_node_affinity | to_nice_yaml | indent(width=8) }} +{% endif %} + serviceAccountName: vsphere-csi-node + hostNetwork: true + dnsPolicy: "ClusterFirstWithHostNet" + containers: + - name: node-driver-registrar + image: {{ kube_image_repo }}/sig-storage/csi-node-driver-registrar:{{ vsphere_csi_node_driver_registrar_image_tag }} +{% if external_vsphere_version < "7.0u1" %} + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "rm -rf /registration/csi.vsphere.vmware.com-reg.sock /csi/csi.sock"] +{% endif %} + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)" +{% if vsphere_csi_node_driver_registrar_resources | length > 0 %} + resources: + {{ vsphere_csi_node_driver_registrar_resources | default({}) | to_nice_yaml | trim | indent(width=10) }} +{% endif %} + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + livenessProbe: + exec: + command: + - /csi-node-driver-registrar + - --kubelet-registration-path=/var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock + - --mode=kubelet-registration-probe + initialDelaySeconds: 3 + - name: vsphere-csi-node + image: {{ gcr_image_repo }}/cloud-provider-vsphere/csi/release/driver:{{ vsphere_csi_driver_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + args: + - "--fss-name=internal-feature-states.csi.vsphere.vmware.com" + - "--fss-namespace={{ vsphere_csi_namespace }}" + imagePullPolicy: "Always" +{% if vsphere_csi_driver_resources | length > 0 %} + resources: + {{ vsphere_csi_driver_resources | default({}) | to_nice_yaml | trim | indent(width=10) }} +{% endif %} + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: MAX_VOLUMES_PER_NODE + value: "59" # Maximum number of volumes that controller can publish to the node. If value is not set or zero Kubernetes decide how many volumes can be published by the controller to the node. + - name: X_CSI_MODE + value: "node" + - name: X_CSI_SPEC_REQ_VALIDATION + value: "false" + - name: X_CSI_DEBUG + value: "true" + - name: X_CSI_SPEC_DISABLE_LEN_CHECK + value: "true" + - name: LOGGER_LEVEL + value: "PRODUCTION" # Options: DEVELOPMENT, PRODUCTION + - name: GODEBUG + value: x509sha1=1 + - name: NODEGETINFO_WATCH_TIMEOUT_MINUTES + value: "1" + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: pods-mount-dir + mountPath: /var/lib/kubelet + # needed so that any mounts setup inside this container are + # propagated back to the host machine. + mountPropagation: "Bidirectional" + - name: device-dir + mountPath: /dev + - name: blocks-dir + mountPath: /sys/block + - name: sys-devices-dir + mountPath: /sys/devices + ports: + - containerPort: 9808 + name: healthz + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 5 + periodSeconds: 5 + failureThreshold: 3 + - name: liveness-probe + image: {{ kube_image_repo }}/sig-storage/livenessprobe:{{ vsphere_csi_liveness_probe_image_tag }} + args: +{% if external_vsphere_version >= "7.0u1" %} + - "--v=4" +{% endif %} + - "--csi-address=/csi/csi.sock" +{% if vsphere_csi_liveness_probe_ds_resources | length > 0 %} + resources: + {{ vsphere_csi_liveness_probe_ds_resources | default({}) | to_nice_yaml | trim | indent(width=10) }} +{% endif %} + volumeMounts: + - name: plugin-dir + mountPath: /csi + volumes: + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry + type: Directory + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/csi.vsphere.vmware.com + type: DirectoryOrCreate + - name: pods-mount-dir + hostPath: + path: /var/lib/kubelet + type: Directory + - name: device-dir + hostPath: + path: /dev + - name: blocks-dir + hostPath: + path: /sys/block + type: Directory + - name: sys-devices-dir + hostPath: + path: /sys/devices + type: Directory + tolerations: + - effect: NoExecute + operator: Exists + - effect: NoSchedule + operator: Exists diff --git a/kubespray/project/roles/kubernetes-apps/external_cloud_controller/hcloud/defaults/main.yml b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/hcloud/defaults/main.yml new file mode 100644 index 0000000..5d9ba29 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/hcloud/defaults/main.yml @@ -0,0 +1,14 @@ +--- +external_hcloud_cloud: + hcloud_api_token: "" + token_secret_name: hcloud + + service_account_name: cloud-controller-manager + + controller_image_tag: "latest" + ## A dictionary of extra arguments to add to the openstack cloud controller manager daemonset + ## Format: + ## external_hcloud_cloud.controller_extra_args: + ## arg1: "value1" + ## arg2: "value2" + controller_extra_args: {} diff --git a/kubespray/project/roles/kubernetes-apps/external_cloud_controller/hcloud/tasks/main.yml b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/hcloud/tasks/main.yml new file mode 100644 index 0000000..c626e78 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/hcloud/tasks/main.yml @@ -0,0 +1,30 @@ +--- +- name: External Hcloud Cloud Controller | Generate Manifests + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/{{ item.file }}" + group: "{{ kube_cert_group }}" + mode: 0640 + with_items: + - {name: external-hcloud-cloud-secret, file: external-hcloud-cloud-secret.yml} + - {name: external-hcloud-cloud-service-account, file: external-hcloud-cloud-service-account.yml} + - {name: external-hcloud-cloud-role-bindings, file: external-hcloud-cloud-role-bindings.yml} + - {name: "{{ 'external-hcloud-cloud-controller-manager-ds-with-networks' if external_hcloud_cloud.with_networks else 'external-hcloud-cloud-controller-manager-ds' }}", file: "{{ 'external-hcloud-cloud-controller-manager-ds-with-networks.yml' if external_hcloud_cloud.with_networks else 'external-hcloud-cloud-controller-manager-ds.yml' }}"} + + register: external_hcloud_manifests + when: inventory_hostname == groups['kube_control_plane'][0] + tags: external-hcloud + +- name: External Hcloud Cloud Controller | Apply Manifests + kube: + kubectl: "{{ bin_dir }}/kubectl" + filename: "{{ kube_config_dir }}/{{ item.item.file }}" + state: "latest" + with_items: + - "{{ external_hcloud_manifests.results }}" + when: + - inventory_hostname == groups['kube_control_plane'][0] + - not item is skipped + loop_control: + label: "{{ item.item.file }}" + tags: external-hcloud diff --git a/kubespray/project/roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-controller-manager-ds-with-networks.yml.j2 b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-controller-manager-ds-with-networks.yml.j2 new file mode 100644 index 0000000..8279a44 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-controller-manager-ds-with-networks.yml.j2 @@ -0,0 +1,93 @@ +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: hcloud-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: hcloud-cloud-controller-manger +spec: + selector: + matchLabels: + app: hcloud-cloud-controller-manager + template: + metadata: + labels: + app: hcloud-cloud-controller-manager + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + serviceAccountName: {{ external_hcloud_cloud.service_account_name }} + dnsPolicy: Default + tolerations: + - key: "node.cloudprovider.kubernetes.io/uninitialized" + value: "true" + effect: "NoSchedule" + - key: "CriticalAddonsOnly" + operator: "Exists" + - key: "node-role.kubernetes.io/control-plane" + effect: NoSchedule + operator: Exists + - key: "node.kubernetes.io/not-ready" + effect: "NoSchedule" + hostNetwork: true + containers: + - image: {{ docker_image_repo }}/hetznercloud/hcloud-cloud-controller-manager:{{ external_hcloud_cloud.controller_image_tag }} + name: hcloud-cloud-controller-manager + command: + - "/bin/hcloud-cloud-controller-manager" + - "--cloud-provider=hcloud" + - "--leader-elect=false" + - "--allow-untagged-cloud" + - "--allocate-node-cidrs=true" + - "--cluster-cidr={{ kube_pods_subnet }}" +{% if external_hcloud_cloud.controller_extra_args is defined %} + + args: +{% for key, value in external_hcloud_cloud.controller_extra_args.items() %} + - "{{ '--' + key + '=' + value }}" +{% endfor %} +{% endif %} + resources: + requests: + cpu: 100m + memory: 50Mi + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: HCLOUD_TOKEN + valueFrom: + secretKeyRef: + name: {{ external_hcloud_cloud.token_secret_name }} + key: token + - name: HCLOUD_NETWORK + valueFrom: + secretKeyRef: + name: {{ external_hcloud_cloud.token_secret_name }} + key: network +{% if external_hcloud_cloud.network_routes_enabled is defined %} + - name: HCLOUD_NETWORK_ROUTES_ENABLED + value: "{{ external_hcloud_cloud.network_routes_enabled }}" +{% endif %} +{% if external_hcloud_cloud.load_balancers_location is defined %} + - name: HCLOUD_LOAD_BALANCERS_LOCATION + value: "{{ external_hcloud_cloud.load_balancers_location }}" +{% endif %} +{% if external_hcloud_cloud.load_balancers_network_zone is defined %} + - name: HCLOUD_LOAD_BALANCERS_NETWORK_ZONE + value: "{{ external_hcloud_cloud.load_balancers_network_zone }}" +{% endif %} +{% if external_hcloud_cloud.load_balancers_disable_private_ingress is defined %} + - name: HCLOUD_LOAD_BALANCERS_DISABLE_PRIVATE_INGRESS + value: "{{ external_hcloud_cloud.load_balancers_disable_private_ingress }}" +{% endif %} +{% if external_hcloud_cloud.load_balancers_use_private_ip is defined %} + - name: HCLOUD_LOAD_BALANCERS_USE_PRIVATE_IP + value: "{{ external_hcloud_cloud.load_balancers_use_private_ip }}" +{% endif %} +{% if external_hcloud_cloud.load_balancers_enabled is defined %} + - name: HCLOUD_LOAD_BALANCERS_ENABLED + value: "{{ external_hcloud_cloud.load_balancers_enabled }}" +{% endif %} diff --git a/kubespray/project/roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-controller-manager-ds.yml.j2 b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-controller-manager-ds.yml.j2 new file mode 100644 index 0000000..6442210 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-controller-manager-ds.yml.j2 @@ -0,0 +1,92 @@ +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: hcloud-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: hcloud-cloud-controller-manger +spec: + selector: + matchLabels: + app: hcloud-cloud-controller-manager + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + app: hcloud-cloud-controller-manager + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + serviceAccountName: {{ external_hcloud_cloud.service_account_name }} + dnsPolicy: Default + tolerations: + - key: "node.cloudprovider.kubernetes.io/uninitialized" + value: "true" + effect: "NoSchedule" + - key: "CriticalAddonsOnly" + operator: "Exists" + - key: "node-role.kubernetes.io/control-plane" + effect: NoSchedule + - key: "node.kubernetes.io/not-ready" + effect: "NoSchedule" + containers: + - image: {{ docker_image_repo }}/hetznercloud/hcloud-cloud-controller-manager:{{ external_hcloud_cloud.controller_image_tag }} + name: hcloud-cloud-controller-manager + command: + - "/bin/hcloud-cloud-controller-manager" + - "--cloud-provider=hcloud" + - "--leader-elect=false" + - "--allow-untagged-cloud" +{% if external_hcloud_cloud.controller_extra_args is defined %} + args: +{% for key, value in external_hcloud_cloud.controller_extra_args.items() %} + - "{{ '--' + key + '=' + value }}" +{% endfor %} +{% endif %} + resources: + requests: + cpu: 100m + memory: 50Mi + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: HCLOUD_TOKEN + valueFrom: + secretKeyRef: + name: {{ external_hcloud_cloud.token_secret_name }} + key: token +{% if external_hcloud_cloud.network_name is defined %} + - name: HCLOUD_NETWORK + valueFrom: + secretKeyRef: + name: {{ external_hcloud_cloud.token_secret_name }} + key: network +{% endif %} +{% if external_hcloud_cloud.network_routes_enabled is defined %} + - name: HCLOUD_NETWORK_ROUTES_ENABLED + value: "{{ external_hcloud_cloud.network_routes_enabled }}" +{% endif %} +{% if external_hcloud_cloud.load_balancers_location is defined %} + - name: HCLOUD_LOAD_BALANCERS_LOCATION + value: "{{ external_hcloud_cloud.load_balancers_location }}" +{% endif %} +{% if external_hcloud_cloud.load_balancers_network_zone is defined %} + - name: HCLOUD_LOAD_BALANCERS_NETWORK_ZONE + value: "{{ external_hcloud_cloud.load_balancers_network_zone }}" +{% endif %} +{% if external_hcloud_cloud.load_balancers_disable_private_ingress is defined %} + - name: HCLOUD_LOAD_BALANCERS_DISABLE_PRIVATE_INGRESS + value: "{{ external_hcloud_cloud.load_balancers_disable_private_ingress }}" +{% endif %} +{% if external_hcloud_cloud.load_balancers_use_private_ip is defined %} + - name: HCLOUD_LOAD_BALANCERS_USE_PRIVATE_IP + value: "{{ external_hcloud_cloud.load_balancers_use_private_ip }}" +{% endif %} +{% if external_hcloud_cloud.load_balancers_enabled is defined %} + - name: HCLOUD_LOAD_BALANCERS_ENABLED + value: "{{ external_hcloud_cloud.load_balancers_enabled }}" +{% endif %} diff --git a/kubespray/project/roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-role-bindings.yml.j2 b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-role-bindings.yml.j2 new file mode 100644 index 0000000..270c947 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-role-bindings.yml.j2 @@ -0,0 +1,13 @@ +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: system:cloud-controller-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: + - kind: ServiceAccount + name: {{ external_hcloud_cloud.service_account_name }} + namespace: kube-system diff --git a/kubespray/project/roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-secret.yml.j2 b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-secret.yml.j2 new file mode 100644 index 0000000..ab3df74 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-secret.yml.j2 @@ -0,0 +1,15 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: "{{ external_hcloud_cloud.token_secret_name }}" + namespace: kube-system +data: + token: "{{ external_hcloud_cloud.hcloud_api_token | b64encode }}" +{% if external_hcloud_cloud.with_networks or external_hcloud_cloud.network_name is defined %} +{% if network_id is defined%} + network: "{{ network_id | b64encode }}" +{% else %} + network: "{{ external_hcloud_cloud.network_name | b64encode }}" +{% endif %} +{% endif %} diff --git a/kubespray/project/roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-service-account.yml.j2 b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-service-account.yml.j2 new file mode 100644 index 0000000..93277dd --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-service-account.yml.j2 @@ -0,0 +1,6 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ external_hcloud_cloud.service_account_name }} + namespace: kube-system diff --git a/kubespray/project/roles/kubernetes-apps/external_cloud_controller/huaweicloud/defaults/main.yml b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/huaweicloud/defaults/main.yml new file mode 100644 index 0000000..f81bf1e --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/huaweicloud/defaults/main.yml @@ -0,0 +1,19 @@ +--- +# The external cloud controller will need credentials to access +# openstack apis. Per default these values will be +# read from the environment. +external_huaweicloud_auth_url: "{{ lookup('env','OS_AUTH_URL') }}" +external_huaweicloud_access_key: "{{ lookup('env','OS_ACCESS_KEY') }}" +external_huaweicloud_secret_key: "{{ lookup('env','OS_SECRET_KEY') }}" +external_huaweicloud_region: "{{ lookup('env','OS_REGION_NAME') }}" +external_huaweicloud_project_id: "{{ lookup('env','OS_TENANT_ID')| default(lookup('env','OS_PROJECT_ID'),true) }}" +external_huaweicloud_cloud: "{{ lookup('env','OS_CLOUD') }}" + +## A dictionary of extra arguments to add to the huawei cloud controller manager deployment +## Format: +## external_huawei_cloud_controller_extra_args: +## arg1: "value1" +## arg2: "value2" +external_huawei_cloud_controller_extra_args: {} +external_huawei_cloud_controller_image_repo: "swr.ap-southeast-1.myhuaweicloud.com" +external_huawei_cloud_controller_image_tag: "v0.26.6" diff --git a/kubespray/project/roles/kubernetes-apps/external_cloud_controller/huaweicloud/tasks/huaweicloud-credential-check.yml b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/huaweicloud/tasks/huaweicloud-credential-check.yml new file mode 100644 index 0000000..79172ff --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/huaweicloud/tasks/huaweicloud-credential-check.yml @@ -0,0 +1,33 @@ +--- +- name: External Huawei Cloud Controller | check external_huaweicloud_auth_url value + fail: + msg: "external_huaweicloud_auth_url is missing" + when: external_huaweicloud_auth_url is not defined or not external_huaweicloud_auth_url + + +- name: External Huawei Cloud Controller | check external_huaweicloud_access_key value + fail: + msg: "you must set external_huaweicloud_access_key" + when: + - external_huaweicloud_access_key is not defined or not external_huaweicloud_access_key + +- name: External Huawei Cloud Controller | check external_huaweicloud_secret_key value + fail: + msg: "external_huaweicloud_secret_key is missing" + when: + - external_huaweicloud_access_key is defined + - external_huaweicloud_access_key|length > 0 + - external_huaweicloud_secret_key is not defined or not external_huaweicloud_secret_key + + +- name: External Huawei Cloud Controller | check external_huaweicloud_region value + fail: + msg: "external_huaweicloud_region is missing" + when: external_huaweicloud_region is not defined or not external_huaweicloud_region + + +- name: External Huawei Cloud Controller | check external_huaweicloud_project_id value + fail: + msg: "one of external_huaweicloud_project_id must be specified" + when: + - external_huaweicloud_project_id is not defined or not external_huaweicloud_project_id diff --git a/kubespray/project/roles/kubernetes-apps/external_cloud_controller/huaweicloud/tasks/main.yml b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/huaweicloud/tasks/main.yml new file mode 100644 index 0000000..880be0d --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/huaweicloud/tasks/main.yml @@ -0,0 +1,49 @@ +--- +- name: External Huawei Cloud Controller | Check Huawei credentials + include_tasks: huaweicloud-credential-check.yml + tags: external-huaweicloud + +- name: External huaweicloud Cloud Controller | Get base64 cacert + slurp: + src: "{{ external_huaweicloud_cacert }}" + register: external_huaweicloud_cacert_b64 + when: + - inventory_hostname == groups['kube_control_plane'][0] + - external_huaweicloud_cacert is defined + - external_huaweicloud_cacert | length > 0 + tags: external-huaweicloud + +- name: External huaweicloud Cloud Controller | Get base64 cloud-config + set_fact: + external_huawei_cloud_config_secret: "{{ lookup('template', 'external-huawei-cloud-config.j2') | b64encode }}" + when: inventory_hostname == groups['kube_control_plane'][0] + tags: external-huaweicloud + +- name: External Huawei Cloud Controller | Generate Manifests + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/{{ item.file }}" + group: "{{ kube_cert_group }}" + mode: 0640 + with_items: + - {name: external-huawei-cloud-config-secret, file: external-huawei-cloud-config-secret.yml} + - {name: external-huawei-cloud-controller-manager-roles, file: external-huawei-cloud-controller-manager-roles.yml} + - {name: external-huawei-cloud-controller-manager-role-bindings, file: external-huawei-cloud-controller-manager-role-bindings.yml} + - {name: external-huawei-cloud-controller-manager-ds, file: external-huawei-cloud-controller-manager-ds.yml} + register: external_huaweicloud_manifests + when: inventory_hostname == groups['kube_control_plane'][0] + tags: external-huaweicloud + +- name: External Huawei Cloud Controller | Apply Manifests + kube: + kubectl: "{{ bin_dir }}/kubectl" + filename: "{{ kube_config_dir }}/{{ item.item.file }}" + state: "latest" + with_items: + - "{{ external_huaweicloud_manifests.results }}" + when: + - inventory_hostname == groups['kube_control_plane'][0] + - not item is skipped + loop_control: + label: "{{ item.item.file }}" + tags: external-huaweicloud diff --git a/kubespray/project/roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-config-secret.yml.j2 b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-config-secret.yml.j2 new file mode 100644 index 0000000..1f0bbf3 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-config-secret.yml.j2 @@ -0,0 +1,10 @@ +# This YAML file contains secret objects, +# which are necessary to run external huaweicloud cloud controller. + +kind: Secret +apiVersion: v1 +metadata: + name: external-huawei-cloud-config + namespace: kube-system +data: + cloud-config: {{ external_huawei_cloud_config_secret }} diff --git a/kubespray/project/roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-config.j2 b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-config.j2 new file mode 100644 index 0000000..07f1771 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-config.j2 @@ -0,0 +1,23 @@ +[Global] +auth-url="{{ external_huaweicloud_auth_url }}" +{% if external_huaweicloud_access_key is defined and external_huaweicloud_access_key != "" %} +access-key={{ external_huaweicloud_access_key }} +{% endif %} +{% if external_huaweicloud_secret_key is defined and external_huaweicloud_secret_key != "" %} +secret-key={{ external_huaweicloud_secret_key }} +{% endif %} +region="{{ external_huaweicloud_region }}" +{% if external_huaweicloud_project_id is defined and external_huaweicloud_project_id != "" %} +project-id="{{ external_huaweicloud_project_id }}" +{% endif %} +{% if external_huaweicloud_cloud is defined and external_huaweicloud_cloud != "" %} +cloud="{{ external_huaweicloud_cloud }}" +{% endif %} + +[VPC] +{% if external_huaweicloud_lbaas_subnet_id is defined %} +subnet-id={{ external_huaweicloud_lbaas_subnet_id }} +{% endif %} +{% if external_huaweicloud_lbaas_network_id is defined %} +id={{ external_huaweicloud_lbaas_network_id }} +{% endif %} diff --git a/kubespray/project/roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-controller-manager-ds.yml.j2 b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-controller-manager-ds.yml.j2 new file mode 100644 index 0000000..b9b2ec3 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-controller-manager-ds.yml.j2 @@ -0,0 +1,91 @@ +kind: Namespace +apiVersion: v1 +metadata: + name: huawei-cloud-provider +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: cloud-controller-manager + namespace: kube-system +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: huawei-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: huawei-cloud-controller-manager +spec: + selector: + matchLabels: + k8s-app: huawei-cloud-controller-manager + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + k8s-app: huawei-cloud-controller-manager + spec: + nodeSelector: + node-role.kubernetes.io/control-plane: "" + securityContext: + runAsUser: 1001 + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + serviceAccountName: cloud-controller-manager + containers: + - name: huawei-cloud-controller-manager + image: {{ external_huawei_cloud_controller_image_repo }}/k8s-cloudprovider/huawei-cloud-controller-manager:{{ external_huawei_cloud_controller_image_tag }} + args: + - /bin/huawei-cloud-controller-manager + - --v=1 + - --cloud-config=$(CLOUD_CONFIG) + - --cloud-provider=huaweicloud + - --use-service-account-credentials=true +{% for key, value in external_huawei_cloud_controller_extra_args.items() %} + - "{{ '--' + key + '=' + value }}" +{% endfor %} + volumeMounts: + - mountPath: /etc/kubernetes + name: k8s-certs + readOnly: true + - mountPath: /etc/ssl/certs + name: ca-certs + readOnly: true + - mountPath: /etc/config + name: cloud-config-volume + readOnly: true +{% if kubelet_flexvolumes_plugins_dir is defined %} + - mountPath: /usr/libexec/kubernetes/kubelet-plugins/volume/exec + name: flexvolume-dir +{% endif %} + resources: + requests: + cpu: 200m + env: + - name: CLOUD_CONFIG + value: /etc/config/cloud-config + hostNetwork: true + volumes: +{% if kubelet_flexvolumes_plugins_dir is defined %} + - name: flexvolume-dir + hostPath: + path: "{{ kubelet_flexvolumes_plugins_dir }}" + type: DirectoryOrCreate +{% endif %} + - name: k8s-certs + hostPath: + path: /etc/kubernetes + type: DirectoryOrCreate + - name: ca-certs + hostPath: + path: /etc/ssl/certs + type: DirectoryOrCreate + - name: cloud-config-volume + secret: + secretName: external-huawei-cloud-config diff --git a/kubespray/project/roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-controller-manager-role-bindings.yml.j2 b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-controller-manager-role-bindings.yml.j2 new file mode 100644 index 0000000..bbdf336 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-controller-manager-role-bindings.yml.j2 @@ -0,0 +1,16 @@ +apiVersion: v1 +items: +- apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system +kind: List +metadata: {} diff --git a/kubespray/project/roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-controller-manager-roles.yml.j2 b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-controller-manager-roles.yml.j2 new file mode 100644 index 0000000..2e2d8b6 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-controller-manager-roles.yml.j2 @@ -0,0 +1,117 @@ +apiVersion: v1 +items: +- apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + rules: + - resources: + - tokenreviews + verbs: + - get + - list + - watch + - create + - update + - patch + apiGroups: + - authentication.k8s.io + - resources: + - configmaps + - endpoints + - pods + - services + - secrets + - serviceaccounts + - serviceaccounts/token + verbs: + - get + - list + - watch + - create + - update + - patch + apiGroups: + - '' + - resources: + - nodes + verbs: + - get + - list + - watch + - delete + - patch + - update + apiGroups: + - '' + - resources: + - services/status + - pods/status + verbs: + - update + - patch + apiGroups: + - '' + - resources: + - nodes/status + verbs: + - patch + - update + apiGroups: + - '' + - resources: + - events + - endpoints + verbs: + - create + - patch + - update + apiGroups: + - '' + - resources: + - leases + verbs: + - get + - update + - create + - delete + apiGroups: + - coordination.k8s.io + - resources: + - customresourcedefinitions + verbs: + - get + - update + - create + - delete + apiGroups: + - apiextensions.k8s.io + - resources: + - ingresses + verbs: + - get + - list + - watch + - update + - create + - patch + - delete + apiGroups: + - networking.k8s.io + - resources: + - ingresses/status + verbs: + - update + - patch + apiGroups: + - networking.k8s.io + - resources: + - endpointslices + verbs: + - get + - list + - watch + apiGroups: + - discovery.k8s.io +kind: List +metadata: {} diff --git a/kubespray/project/roles/kubernetes-apps/external_cloud_controller/meta/main.yml b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/meta/main.yml new file mode 100644 index 0000000..b1fc4ad --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/meta/main.yml @@ -0,0 +1,42 @@ +--- +dependencies: + - role: kubernetes-apps/external_cloud_controller/openstack + when: + - cloud_provider is defined + - cloud_provider == "external" + - external_cloud_provider is defined + - external_cloud_provider == "openstack" + - inventory_hostname == groups['kube_control_plane'][0] + tags: + - external-cloud-controller + - external-openstack + - role: kubernetes-apps/external_cloud_controller/vsphere + when: + - cloud_provider is defined + - cloud_provider == "external" + - external_cloud_provider is defined + - external_cloud_provider == "vsphere" + - inventory_hostname == groups['kube_control_plane'][0] + tags: + - external-cloud-controller + - external-vsphere + - role: kubernetes-apps/external_cloud_controller/hcloud + when: + - cloud_provider is defined + - cloud_provider == "external" + - external_cloud_provider is defined + - external_cloud_provider == "hcloud" + - inventory_hostname == groups['kube_control_plane'][0] + tags: + - external-cloud-controller + - external-hcloud + - role: kubernetes-apps/external_cloud_controller/huaweicloud + when: + - cloud_provider is defined + - cloud_provider == "external" + - external_cloud_provider is defined + - external_cloud_provider == "huaweicloud" + - inventory_hostname == groups['kube_control_plane'][0] + tags: + - external-cloud-controller + - external-huaweicloud diff --git a/kubespray/project/roles/kubernetes-apps/external_cloud_controller/openstack/OWNERS b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/openstack/OWNERS new file mode 100644 index 0000000..6cfbaa8 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/openstack/OWNERS @@ -0,0 +1,6 @@ +# See the OWNERS docs at https://go.k8s.io/owners + +approvers: +reviewers: + - alijahnas + - luckySB diff --git a/kubespray/project/roles/kubernetes-apps/external_cloud_controller/openstack/defaults/main.yml b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/openstack/defaults/main.yml new file mode 100644 index 0000000..4bcf135 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/openstack/defaults/main.yml @@ -0,0 +1,25 @@ +--- +# The external cloud controller will need credentials to access +# openstack apis. Per default these values will be +# read from the environment. +external_openstack_auth_url: "{{ lookup('env', 'OS_AUTH_URL') }}" +external_openstack_username: "{{ lookup('env', 'OS_USERNAME') }}" +external_openstack_password: "{{ lookup('env', 'OS_PASSWORD') }}" +external_openstack_application_credential_id: "{{ lookup('env', 'OS_APPLICATION_CREDENTIAL_ID') }}" +external_openstack_application_credential_name: "{{ lookup('env', 'OS_APPLICATION_CREDENTIAL_NAME') }}" +external_openstack_application_credential_secret: "{{ lookup('env', 'OS_APPLICATION_CREDENTIAL_SECRET') }}" +external_openstack_region: "{{ lookup('env', 'OS_REGION_NAME') }}" +external_openstack_tenant_id: "{{ lookup('env', 'OS_TENANT_ID') | default(lookup('env', 'OS_PROJECT_ID'), true) }}" +external_openstack_tenant_name: "{{ lookup('env', 'OS_TENANT_NAME') | default(lookup('env', 'OS_PROJECT_NAME'), true) }}" +external_openstack_domain_name: "{{ lookup('env', 'OS_USER_DOMAIN_NAME') }}" +external_openstack_domain_id: "{{ lookup('env', 'OS_USER_DOMAIN_ID') }}" +external_openstack_cacert: "{{ lookup('env', 'OS_CACERT') }}" + +## A dictionary of extra arguments to add to the openstack cloud controller manager daemonset +## Format: +## external_openstack_cloud_controller_extra_args: +## arg1: "value1" +## arg2: "value2" +external_openstack_cloud_controller_extra_args: {} +external_openstack_cloud_controller_image_tag: "v1.25.3" +external_openstack_cloud_controller_bind_address: 127.0.0.1 diff --git a/kubespray/project/roles/kubernetes-apps/external_cloud_controller/openstack/tasks/main.yml b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/openstack/tasks/main.yml new file mode 100644 index 0000000..787dbb4 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/openstack/tasks/main.yml @@ -0,0 +1,49 @@ +--- +- name: External OpenStack Cloud Controller | Check OpenStack credentials + include_tasks: openstack-credential-check.yml + tags: external-openstack + +- name: External OpenStack Cloud Controller | Get base64 cacert + slurp: + src: "{{ external_openstack_cacert }}" + register: external_openstack_cacert_b64 + when: + - inventory_hostname == groups['kube_control_plane'][0] + - external_openstack_cacert is defined + - external_openstack_cacert | length > 0 + tags: external-openstack + +- name: External OpenStack Cloud Controller | Get base64 cloud-config + set_fact: + external_openstack_cloud_config_secret: "{{ lookup('template', 'external-openstack-cloud-config.j2') | b64encode }}" + when: inventory_hostname == groups['kube_control_plane'][0] + tags: external-openstack + +- name: External OpenStack Cloud Controller | Generate Manifests + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/{{ item.file }}" + group: "{{ kube_cert_group }}" + mode: 0640 + with_items: + - {name: external-openstack-cloud-config-secret, file: external-openstack-cloud-config-secret.yml} + - {name: external-openstack-cloud-controller-manager-roles, file: external-openstack-cloud-controller-manager-roles.yml} + - {name: external-openstack-cloud-controller-manager-role-bindings, file: external-openstack-cloud-controller-manager-role-bindings.yml} + - {name: external-openstack-cloud-controller-manager-ds, file: external-openstack-cloud-controller-manager-ds.yml} + register: external_openstack_manifests + when: inventory_hostname == groups['kube_control_plane'][0] + tags: external-openstack + +- name: External OpenStack Cloud Controller | Apply Manifests + kube: + kubectl: "{{ bin_dir }}/kubectl" + filename: "{{ kube_config_dir }}/{{ item.item.file }}" + state: "latest" + with_items: + - "{{ external_openstack_manifests.results }}" + when: + - inventory_hostname == groups['kube_control_plane'][0] + - not item is skipped + loop_control: + label: "{{ item.item.file }}" + tags: external-openstack diff --git a/kubespray/project/roles/kubernetes-apps/external_cloud_controller/openstack/tasks/openstack-credential-check.yml b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/openstack/tasks/openstack-credential-check.yml new file mode 100644 index 0000000..6a14658 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/openstack/tasks/openstack-credential-check.yml @@ -0,0 +1,66 @@ +--- +- name: External OpenStack Cloud Controller | check external_openstack_auth_url value + fail: + msg: "external_openstack_auth_url is missing" + when: external_openstack_auth_url is not defined or not external_openstack_auth_url + + +- name: External OpenStack Cloud Controller | check external_openstack_username or external_openstack_application_credential_name value + fail: + msg: "you must either set external_openstack_username or external_openstack_application_credential_name" + when: + - external_openstack_username is not defined or not external_openstack_username + - external_openstack_application_credential_name is not defined or not external_openstack_application_credential_name + + +- name: External OpenStack Cloud Controller | check external_openstack_application_credential_id value + fail: + msg: "external_openstack_application_credential_id is missing" + when: + - external_openstack_application_credential_name is defined + - external_openstack_application_credential_name | length > 0 + - external_openstack_application_credential_id is not defined or not external_openstack_application_credential_id + + +- name: External OpenStack Cloud Controller | check external_openstack_application_credential_secret value + fail: + msg: "external_openstack_application_credential_secret is missing" + when: + - external_openstack_application_credential_name is defined + - external_openstack_application_credential_name | length > 0 + - external_openstack_application_credential_secret is not defined or not external_openstack_application_credential_secret + + +- name: External OpenStack Cloud Controller | check external_openstack_password value + fail: + msg: "external_openstack_password is missing" + when: + - external_openstack_username is defined + - external_openstack_username | length > 0 + - external_openstack_application_credential_name is not defined or not external_openstack_application_credential_name + - external_openstack_application_credential_secret is not defined or not external_openstack_application_credential_secret + - external_openstack_password is not defined or not external_openstack_password + + +- name: External OpenStack Cloud Controller | check external_openstack_region value + fail: + msg: "external_openstack_region is missing" + when: external_openstack_region is not defined or not external_openstack_region + + +- name: External OpenStack Cloud Controller | check external_openstack_tenant_id value + fail: + msg: "one of external_openstack_tenant_id or external_openstack_tenant_name must be specified" + when: + - external_openstack_tenant_id is not defined or not external_openstack_tenant_id + - external_openstack_tenant_name is not defined or not external_openstack_tenant_name + - external_openstack_application_credential_name is not defined or not external_openstack_application_credential_name + + +- name: External OpenStack Cloud Controller | check external_openstack_domain_id value + fail: + msg: "one of external_openstack_domain_id or external_openstack_domain_name must be specified" + when: + - external_openstack_domain_id is not defined or not external_openstack_domain_id + - external_openstack_domain_name is not defined or not external_openstack_domain_name + - external_openstack_application_credential_name is not defined or not external_openstack_application_credential_name diff --git a/kubespray/project/roles/kubernetes-apps/external_cloud_controller/openstack/templates/external-openstack-cloud-config-secret.yml.j2 b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/openstack/templates/external-openstack-cloud-config-secret.yml.j2 new file mode 100644 index 0000000..2a6f6a8 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/openstack/templates/external-openstack-cloud-config-secret.yml.j2 @@ -0,0 +1,13 @@ +# This YAML file contains secret objects, +# which are necessary to run external openstack cloud controller. + +kind: Secret +apiVersion: v1 +metadata: + name: external-openstack-cloud-config + namespace: kube-system +data: + cloud.conf: {{ external_openstack_cloud_config_secret }} +{% if external_openstack_cacert_b64.content is defined %} + ca.cert: {{ external_openstack_cacert_b64.content }} +{% endif %} diff --git a/kubespray/project/roles/kubernetes-apps/external_cloud_controller/openstack/templates/external-openstack-cloud-config.j2 b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/openstack/templates/external-openstack-cloud-config.j2 new file mode 100644 index 0000000..08acd67 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/openstack/templates/external-openstack-cloud-config.j2 @@ -0,0 +1,92 @@ +[Global] +auth-url="{{ external_openstack_auth_url }}" +{% if external_openstack_application_credential_id == "" and external_openstack_application_credential_name == "" %} +username="{{ external_openstack_username }}" +password="{{ external_openstack_password }}" +{% endif %} +{% if external_openstack_application_credential_id is defined and external_openstack_application_credential_id != "" %} +application-credential-id={{ external_openstack_application_credential_id }} +{% endif %} +{% if external_openstack_application_credential_name is defined and external_openstack_application_credential_name != "" %} +application-credential-name={{ external_openstack_application_credential_name }} +{% endif %} +{% if external_openstack_application_credential_secret is defined and external_openstack_application_credential_secret != "" %} +application-credential-secret={{ external_openstack_application_credential_secret }} +{% endif %} +region="{{ external_openstack_region }}" +{% if external_openstack_tenant_id is defined and external_openstack_tenant_id != "" %} +tenant-id="{{ external_openstack_tenant_id }}" +{% endif %} +{% if external_openstack_tenant_name is defined and external_openstack_tenant_name != "" %} +tenant-name="{{ external_openstack_tenant_name }}" +{% endif %} +{% if external_openstack_domain_name is defined and external_openstack_domain_name != "" %} +domain-name="{{ external_openstack_domain_name }}" +{% elif external_openstack_domain_id is defined and external_openstack_domain_id != "" %} +domain-id ="{{ external_openstack_domain_id }}" +{% endif %} +{% if external_openstack_cacert is defined and external_openstack_cacert != "" %} +ca-file="{{ kube_config_dir }}/external-openstack-cacert.pem" +{% endif %} + +[LoadBalancer] +enabled={{ external_openstack_lbaas_enabled | string | lower }} +{% if external_openstack_lbaas_floating_network_id is defined %} +floating-network-id={{ external_openstack_lbaas_floating_network_id }} +{% endif %} +{% if external_openstack_lbaas_floating_subnet_id is defined %} +floating-subnet-id={{ external_openstack_lbaas_floating_subnet_id }} +{% endif %} +{% if external_openstack_lbaas_method is defined %} +lb-method={{ external_openstack_lbaas_method }} +{% endif %} +{% if external_openstack_lbaas_provider is defined %} +lb-provider={{ external_openstack_lbaas_provider }} +{% endif %} +{% if external_openstack_lbaas_subnet_id is defined %} +subnet-id={{ external_openstack_lbaas_subnet_id }} +{% endif %} +{% if external_openstack_lbaas_network_id is defined %} +network-id={{ external_openstack_lbaas_network_id }} +{% endif %} +{% if external_openstack_lbaas_manage_security_groups is defined %} +manage-security-groups={{ external_openstack_lbaas_manage_security_groups }} +{% endif %} +{% if external_openstack_lbaas_create_monitor is defined %} +create-monitor={{ external_openstack_lbaas_create_monitor }} +{% endif %} +{% if external_openstack_lbaas_monitor_delay is defined %} +monitor-delay={{ external_openstack_lbaas_monitor_delay }} +{% endif %} +{% if external_openstack_lbaas_monitor_max_retries is defined %} +monitor-max-retries={{ external_openstack_lbaas_monitor_max_retries }} +{% endif %} +{% if external_openstack_lbaas_monitor_timeout is defined %} +monitor-timeout={{ external_openstack_lbaas_monitor_timeout }} +{% endif %} +{% if external_openstack_lbaas_internal_lb is defined %} +internal-lb={{ external_openstack_lbaas_internal_lb }} +{% endif %} +{% if external_openstack_enable_ingress_hostname is defined %} +enable-ingress-hostname={{ external_openstack_enable_ingress_hostname | string | lower }} +{% endif %} +{% if external_openstack_ingress_hostname_suffix is defined %} +ingress-hostname-suffix={{ external_openstack_ingress_hostname_suffix | string | lower }} +{% endif %} +{% if external_openstack_max_shared_lb is defined %} +max-shared-lb={{ external_openstack_max_shared_lb }} +{% endif %} + +[Networking] +ipv6-support-disabled={{ external_openstack_network_ipv6_disabled | string | lower }} +{% for network_name in external_openstack_network_internal_networks %} +internal-network-name="{{ network_name }}" +{% endfor %} +{% for network_name in external_openstack_network_public_networks %} +public-network-name="{{ network_name }}" +{% endfor %} + +[Metadata] +{% if external_openstack_metadata_search_order is defined %} +search-order="{{ external_openstack_metadata_search_order }}" +{% endif %} diff --git a/kubespray/project/roles/kubernetes-apps/external_cloud_controller/openstack/templates/external-openstack-cloud-controller-manager-ds.yml.j2 b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/openstack/templates/external-openstack-cloud-controller-manager-ds.yml.j2 new file mode 100644 index 0000000..8bae2aa --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/openstack/templates/external-openstack-cloud-controller-manager-ds.yml.j2 @@ -0,0 +1,110 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: cloud-controller-manager + namespace: kube-system +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: openstack-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: openstack-cloud-controller-manager +spec: + selector: + matchLabels: + k8s-app: openstack-cloud-controller-manager + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + k8s-app: openstack-cloud-controller-manager + spec: + nodeSelector: + node-role.kubernetes.io/control-plane: "" + securityContext: + runAsUser: 999 + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + serviceAccountName: cloud-controller-manager + containers: + - name: openstack-cloud-controller-manager + image: {{ docker_image_repo }}/k8scloudprovider/openstack-cloud-controller-manager:{{ external_openstack_cloud_controller_image_tag }} + args: + - /bin/openstack-cloud-controller-manager + - --v=1 + - --cloud-config=$(CLOUD_CONFIG) + - --cloud-provider=openstack + - --cluster-name={{ cluster_name }} + - --use-service-account-credentials=true + - --bind-address={{ external_openstack_cloud_controller_bind_address }} +{% for key, value in external_openstack_cloud_controller_extra_args.items() %} + - "{{ '--' + key + '=' + value }}" +{% endfor %} + volumeMounts: + - mountPath: /etc/kubernetes/pki + name: k8s-certs + readOnly: true + - mountPath: /etc/ssl/certs + name: ca-certs + readOnly: true +{% if ssl_ca_dirs | length %} +{% for dir in ssl_ca_dirs %} + - name: {{ dir | regex_replace('^/(.*)$', '\\1' ) | regex_replace('/', '-') }} + mountPath: {{ dir }} + readOnly: true +{% endfor %} +{% endif %} + - mountPath: /etc/config/cloud.conf + name: cloud-config-volume + readOnly: true + subPath: cloud.conf + - mountPath: {{ kube_config_dir }}/external-openstack-cacert.pem + name: cloud-config-volume + readOnly: true + subPath: ca.cert +{% if kubelet_flexvolumes_plugins_dir is defined %} + - mountPath: /usr/libexec/kubernetes/kubelet-plugins/volume/exec + name: flexvolume-dir +{% endif %} + resources: + requests: + cpu: 200m + env: + - name: CLOUD_CONFIG + value: /etc/config/cloud.conf + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + volumes: +{% if kubelet_flexvolumes_plugins_dir is defined %} + - name: flexvolume-dir + hostPath: + path: "{{ kubelet_flexvolumes_plugins_dir }}" + type: DirectoryOrCreate +{% endif %} + - name: k8s-certs + hostPath: + path: /etc/kubernetes/pki + type: DirectoryOrCreate + - name: ca-certs + hostPath: + path: /etc/ssl/certs + type: DirectoryOrCreate +{% if ssl_ca_dirs | length %} +{% for dir in ssl_ca_dirs %} + - name: {{ dir | regex_replace('^/(.*)$', '\\1' ) | regex_replace('/', '-') }} + hostPath: + path: {{ dir }} + type: DirectoryOrCreate +{% endfor %} +{% endif %} + - name: cloud-config-volume + secret: + secretName: external-openstack-cloud-config diff --git a/kubespray/project/roles/kubernetes-apps/external_cloud_controller/openstack/templates/external-openstack-cloud-controller-manager-role-bindings.yml.j2 b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/openstack/templates/external-openstack-cloud-controller-manager-role-bindings.yml.j2 new file mode 100644 index 0000000..bbdf336 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/openstack/templates/external-openstack-cloud-controller-manager-role-bindings.yml.j2 @@ -0,0 +1,16 @@ +apiVersion: v1 +items: +- apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system +kind: List +metadata: {} diff --git a/kubespray/project/roles/kubernetes-apps/external_cloud_controller/openstack/templates/external-openstack-cloud-controller-manager-roles.yml.j2 b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/openstack/templates/external-openstack-cloud-controller-manager-roles.yml.j2 new file mode 100644 index 0000000..2ab3a5b --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/openstack/templates/external-openstack-cloud-controller-manager-roles.yml.j2 @@ -0,0 +1,109 @@ +apiVersion: v1 +items: +- apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + rules: + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - '*' + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - patch + - apiGroups: + - "" + resources: + - serviceaccounts/token + verbs: + - create + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - '*' + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - secrets + verbs: + - list + - get + - watch + - apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create + - apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +kind: List +metadata: {} diff --git a/kubespray/project/roles/kubernetes-apps/external_cloud_controller/vsphere/defaults/main.yml b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/vsphere/defaults/main.yml new file mode 100644 index 0000000..b6fb797 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/vsphere/defaults/main.yml @@ -0,0 +1,14 @@ +--- +external_vsphere_vcenter_port: "443" +external_vsphere_insecure: "true" + +## A dictionary of extra arguments to add to the vsphere cloud controller manager daemonset +## Format: +## external_vsphere_cloud_controller_extra_args: +## arg1: "value1" +## arg2: "value2" +external_vsphere_cloud_controller_extra_args: {} +external_vsphere_cloud_controller_image_tag: "latest" + +external_vsphere_user: "{{ lookup('env', 'VSPHERE_USER') }}" +external_vsphere_password: "{{ lookup('env', 'VSPHERE_PASSWORD') }}" diff --git a/kubespray/project/roles/kubernetes-apps/external_cloud_controller/vsphere/tasks/main.yml b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/vsphere/tasks/main.yml new file mode 100644 index 0000000..60b8ec8 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/vsphere/tasks/main.yml @@ -0,0 +1,49 @@ +--- +- name: External vSphere Cloud Controller | Check vsphere credentials + include_tasks: vsphere-credentials-check.yml + +- name: External vSphere Cloud Controller | Generate CPI cloud-config + template: + src: "{{ item }}.j2" + dest: "{{ kube_config_dir }}/{{ item }}" + mode: 0640 + with_items: + - external-vsphere-cpi-cloud-config + when: inventory_hostname == groups['kube_control_plane'][0] + +- name: External vSphere Cloud Controller | Generate Manifests + template: + src: "{{ item }}.j2" + dest: "{{ kube_config_dir }}/{{ item }}" + mode: 0644 + with_items: + - external-vsphere-cpi-cloud-config-secret.yml + - external-vsphere-cloud-controller-manager-roles.yml + - external-vsphere-cloud-controller-manager-role-bindings.yml + - external-vsphere-cloud-controller-manager-ds.yml + register: external_vsphere_manifests + when: inventory_hostname == groups['kube_control_plane'][0] + +- name: External vSphere Cloud Provider Interface | Create a CPI configMap manifest + command: "{{ bin_dir }}/kubectl create configmap cloud-config --from-file=vsphere.conf={{ kube_config_dir }}/external-vsphere-cpi-cloud-config -n kube-system --dry-run --save-config -o yaml" + register: external_vsphere_configmap_manifest + when: inventory_hostname == groups['kube_control_plane'][0] + +- name: External vSphere Cloud Provider Interface | Apply a CPI configMap manifest + command: + cmd: "{{ bin_dir }}/kubectl apply -f -" + stdin: "{{ external_vsphere_configmap_manifest.stdout }}" + when: inventory_hostname == groups['kube_control_plane'][0] + +- name: External vSphere Cloud Controller | Apply Manifests + kube: + kubectl: "{{ bin_dir }}/kubectl" + filename: "{{ kube_config_dir }}/{{ item.item }}" + state: "latest" + with_items: + - "{{ external_vsphere_manifests.results }}" + when: + - inventory_hostname == groups['kube_control_plane'][0] + - not item is skipped + loop_control: + label: "{{ item.item }}" diff --git a/kubespray/project/roles/kubernetes-apps/external_cloud_controller/vsphere/tasks/vsphere-credentials-check.yml b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/vsphere/tasks/vsphere-credentials-check.yml new file mode 100644 index 0000000..b6c12b8 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/vsphere/tasks/vsphere-credentials-check.yml @@ -0,0 +1,32 @@ +--- +- name: External vSphere Cloud Provider | check external_vsphere_vcenter_ip value + fail: + msg: "external_vsphere_vcenter_ip is missing" + when: external_vsphere_vcenter_ip is not defined or not external_vsphere_vcenter_ip + +- name: External vSphere Cloud Provider | check external_vsphere_vcenter_port value + fail: + msg: "external_vsphere_vcenter_port is missing" + when: external_vsphere_vcenter_port is not defined or not external_vsphere_vcenter_port + +- name: External vSphere Cloud Provider | check external_vsphere_insecure value + fail: + msg: "external_vsphere_insecure is missing" + when: external_vsphere_insecure is not defined or not external_vsphere_insecure + +- name: External vSphere Cloud Provider | check external_vsphere_user value + fail: + msg: "external_vsphere_user is missing" + when: external_vsphere_user is not defined or not external_vsphere_user + +- name: External vSphere Cloud Provider | check external_vsphere_password value + fail: + msg: "external_vsphere_password is missing" + when: + - external_vsphere_password is not defined or not external_vsphere_password + +- name: External vSphere Cloud Provider | check external_vsphere_datacenter value + fail: + msg: "external_vsphere_datacenter is missing" + when: + - external_vsphere_datacenter is not defined or not external_vsphere_datacenter diff --git a/kubespray/project/roles/kubernetes-apps/external_cloud_controller/vsphere/templates/external-vsphere-cloud-controller-manager-ds.yml.j2 b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/vsphere/templates/external-vsphere-cloud-controller-manager-ds.yml.j2 new file mode 100644 index 0000000..05e4ac3 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/vsphere/templates/external-vsphere-cloud-controller-manager-ds.yml.j2 @@ -0,0 +1,74 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: cloud-controller-manager + namespace: kube-system +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: vsphere-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: vsphere-cloud-controller-manager +spec: + selector: + matchLabels: + k8s-app: vsphere-cloud-controller-manager + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + k8s-app: vsphere-cloud-controller-manager + spec: + nodeSelector: + node-role.kubernetes.io/control-plane: "" + securityContext: + runAsUser: 0 + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + serviceAccountName: cloud-controller-manager + containers: + - name: vsphere-cloud-controller-manager + image: {{ gcr_image_repo }}/cloud-provider-vsphere/cpi/release/manager:{{ external_vsphere_cloud_controller_image_tag }} + args: + - --v=2 + - --cloud-provider=vsphere + - --cloud-config=/etc/cloud/vsphere.conf +{% for key, value in external_vsphere_cloud_controller_extra_args.items() %} + - "{{ '--' + key + '=' + value }}" +{% endfor %} + volumeMounts: + - mountPath: /etc/cloud + name: vsphere-config-volume + readOnly: true + resources: + requests: + cpu: 200m + hostNetwork: true + volumes: + - name: vsphere-config-volume + configMap: + name: cloud-config +--- +apiVersion: v1 +kind: Service +metadata: + labels: + component: cloud-controller-manager + name: vsphere-cloud-controller-manager + namespace: kube-system +spec: + type: NodePort + ports: + - port: 43001 + protocol: TCP + targetPort: 43001 + selector: + component: cloud-controller-manager diff --git a/kubespray/project/roles/kubernetes-apps/external_cloud_controller/vsphere/templates/external-vsphere-cloud-controller-manager-role-bindings.yml.j2 b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/vsphere/templates/external-vsphere-cloud-controller-manager-role-bindings.yml.j2 new file mode 100644 index 0000000..9f6107d --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/vsphere/templates/external-vsphere-cloud-controller-manager-role-bindings.yml.j2 @@ -0,0 +1,35 @@ +apiVersion: v1 +items: +- apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: servicecatalog.k8s.io:apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager +- apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager +kind: List +metadata: {} diff --git a/kubespray/project/roles/kubernetes-apps/external_cloud_controller/vsphere/templates/external-vsphere-cloud-controller-manager-roles.yml.j2 b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/vsphere/templates/external-vsphere-cloud-controller-manager-roles.yml.j2 new file mode 100644 index 0000000..2cd7ad0 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/vsphere/templates/external-vsphere-cloud-controller-manager-roles.yml.j2 @@ -0,0 +1,91 @@ +apiVersion: v1 +items: +- apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - '*' + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - patch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - "coordination.k8s.io" + resources: + - leases + verbs: + - get + - list + - watch + - create + - update +kind: List +metadata: {} diff --git a/kubespray/project/roles/kubernetes-apps/external_cloud_controller/vsphere/templates/external-vsphere-cpi-cloud-config-secret.yml.j2 b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/vsphere/templates/external-vsphere-cpi-cloud-config-secret.yml.j2 new file mode 100644 index 0000000..5364f42 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/vsphere/templates/external-vsphere-cpi-cloud-config-secret.yml.j2 @@ -0,0 +1,11 @@ +# This YAML file contains secret objects, +# which are necessary to run external vsphere cloud controller. + +apiVersion: v1 +kind: Secret +metadata: + name: cpi-global-secret + namespace: kube-system +stringData: + {{ external_vsphere_vcenter_ip }}.username: "{{ external_vsphere_user }}" + {{ external_vsphere_vcenter_ip }}.password: "{{ external_vsphere_password }}" diff --git a/kubespray/project/roles/kubernetes-apps/external_cloud_controller/vsphere/templates/external-vsphere-cpi-cloud-config.j2 b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/vsphere/templates/external-vsphere-cpi-cloud-config.j2 new file mode 100644 index 0000000..a32d876 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_cloud_controller/vsphere/templates/external-vsphere-cpi-cloud-config.j2 @@ -0,0 +1,8 @@ +[Global] +port = "{{ external_vsphere_vcenter_port }}" +insecure-flag = "{{ external_vsphere_insecure }}" +secret-name = "cpi-global-secret" +secret-namespace = "kube-system" + +[VirtualCenter "{{ external_vsphere_vcenter_ip }}"] +datacenters = "{{ external_vsphere_datacenter }}" diff --git a/kubespray/project/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/defaults/main.yml b/kubespray/project/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/defaults/main.yml new file mode 100644 index 0000000..577fbff --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/defaults/main.yml @@ -0,0 +1,10 @@ +--- +cephfs_provisioner_namespace: "cephfs-provisioner" +cephfs_provisioner_cluster: ceph +cephfs_provisioner_monitors: ~ +cephfs_provisioner_admin_id: admin +cephfs_provisioner_secret: secret +cephfs_provisioner_storage_class: cephfs +cephfs_provisioner_reclaim_policy: Delete +cephfs_provisioner_claim_root: /volumes +cephfs_provisioner_deterministic_names: true diff --git a/kubespray/project/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/tasks/main.yml b/kubespray/project/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/tasks/main.yml new file mode 100644 index 0000000..86cba2d --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/tasks/main.yml @@ -0,0 +1,71 @@ +--- + +- name: CephFS Provisioner | Remove legacy addon dir and manifests + file: + path: "{{ kube_config_dir }}/addons/cephfs_provisioner" + state: absent + when: + - inventory_hostname == groups['kube_control_plane'][0] + tags: + - upgrade + +- name: CephFS Provisioner | Remove legacy namespace + command: > + {{ kubectl }} delete namespace {{ cephfs_provisioner_namespace }} + ignore_errors: true # noqa ignore-errors + when: + - inventory_hostname == groups['kube_control_plane'][0] + tags: + - upgrade + +- name: CephFS Provisioner | Remove legacy storageclass + command: > + {{ kubectl }} delete storageclass {{ cephfs_provisioner_storage_class }} + ignore_errors: true # noqa ignore-errors + when: + - inventory_hostname == groups['kube_control_plane'][0] + tags: + - upgrade + +- name: CephFS Provisioner | Create addon dir + file: + path: "{{ kube_config_dir }}/addons/cephfs_provisioner" + state: directory + owner: root + group: root + mode: 0755 + when: + - inventory_hostname == groups['kube_control_plane'][0] + +- name: CephFS Provisioner | Templates list + set_fact: + cephfs_provisioner_templates: + - { name: 00-namespace, file: 00-namespace.yml, type: ns } + - { name: secret-cephfs-provisioner, file: secret-cephfs-provisioner.yml, type: secret } + - { name: sa-cephfs-provisioner, file: sa-cephfs-provisioner.yml, type: sa } + - { name: clusterrole-cephfs-provisioner, file: clusterrole-cephfs-provisioner.yml, type: clusterrole } + - { name: clusterrolebinding-cephfs-provisioner, file: clusterrolebinding-cephfs-provisioner.yml, type: clusterrolebinding } + - { name: role-cephfs-provisioner, file: role-cephfs-provisioner.yml, type: role } + - { name: rolebinding-cephfs-provisioner, file: rolebinding-cephfs-provisioner.yml, type: rolebinding } + - { name: deploy-cephfs-provisioner, file: deploy-cephfs-provisioner.yml, type: deploy } + - { name: sc-cephfs-provisioner, file: sc-cephfs-provisioner.yml, type: sc } + +- name: CephFS Provisioner | Create manifests + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/addons/cephfs_provisioner/{{ item.file }}" + mode: 0644 + with_items: "{{ cephfs_provisioner_templates }}" + register: cephfs_provisioner_manifests + when: inventory_hostname == groups['kube_control_plane'][0] + +- name: CephFS Provisioner | Apply manifests + kube: + name: "{{ item.item.name }}" + namespace: "{{ cephfs_provisioner_namespace }}" + kubectl: "{{ bin_dir }}/kubectl" + resource: "{{ item.item.type }}" + filename: "{{ kube_config_dir }}/addons/cephfs_provisioner/{{ item.item.file }}" + state: "latest" + with_items: "{{ cephfs_provisioner_manifests.results }}" + when: inventory_hostname == groups['kube_control_plane'][0] diff --git a/kubespray/project/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/00-namespace.yml.j2 b/kubespray/project/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/00-namespace.yml.j2 new file mode 100644 index 0000000..2a2a67c --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/00-namespace.yml.j2 @@ -0,0 +1,7 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: {{ cephfs_provisioner_namespace }} + labels: + name: {{ cephfs_provisioner_namespace }} diff --git a/kubespray/project/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/clusterrole-cephfs-provisioner.yml.j2 b/kubespray/project/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/clusterrole-cephfs-provisioner.yml.j2 new file mode 100644 index 0000000..c6a1490 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/clusterrole-cephfs-provisioner.yml.j2 @@ -0,0 +1,22 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cephfs-provisioner + namespace: {{ cephfs_provisioner_namespace }} +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "update", "patch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "create", "delete"] diff --git a/kubespray/project/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/clusterrolebinding-cephfs-provisioner.yml.j2 b/kubespray/project/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/clusterrolebinding-cephfs-provisioner.yml.j2 new file mode 100644 index 0000000..cc5d5ff --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/clusterrolebinding-cephfs-provisioner.yml.j2 @@ -0,0 +1,13 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cephfs-provisioner +subjects: + - kind: ServiceAccount + name: cephfs-provisioner + namespace: {{ cephfs_provisioner_namespace }} +roleRef: + kind: ClusterRole + name: cephfs-provisioner + apiGroup: rbac.authorization.k8s.io diff --git a/kubespray/project/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/deploy-cephfs-provisioner.yml.j2 b/kubespray/project/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/deploy-cephfs-provisioner.yml.j2 new file mode 100644 index 0000000..8d9eb08 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/deploy-cephfs-provisioner.yml.j2 @@ -0,0 +1,34 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cephfs-provisioner + namespace: {{ cephfs_provisioner_namespace }} + labels: + app: cephfs-provisioner + version: {{ cephfs_provisioner_image_tag }} +spec: + replicas: 1 + selector: + matchLabels: + app: cephfs-provisioner + version: {{ cephfs_provisioner_image_tag }} + template: + metadata: + labels: + app: cephfs-provisioner + version: {{ cephfs_provisioner_image_tag }} + spec: + priorityClassName: {% if cephfs_provisioner_namespace == 'kube-system' %}system-cluster-critical{% else %}k8s-cluster-critical{% endif %}{{ '' }} + serviceAccount: cephfs-provisioner + containers: + - name: cephfs-provisioner + image: {{ cephfs_provisioner_image_repo }}:{{ cephfs_provisioner_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + env: + - name: PROVISIONER_NAME + value: ceph.com/cephfs + command: + - "/usr/local/bin/cephfs-provisioner" + args: + - "-id=cephfs-provisioner-1" diff --git a/kubespray/project/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/role-cephfs-provisioner.yml.j2 b/kubespray/project/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/role-cephfs-provisioner.yml.j2 new file mode 100644 index 0000000..1fb80a1 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/role-cephfs-provisioner.yml.j2 @@ -0,0 +1,13 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cephfs-provisioner + namespace: {{ cephfs_provisioner_namespace }} +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["create", "get", "delete"] + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "list", "watch", "create", "update", "patch"] diff --git a/kubespray/project/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/rolebinding-cephfs-provisioner.yml.j2 b/kubespray/project/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/rolebinding-cephfs-provisioner.yml.j2 new file mode 100644 index 0000000..01ab87b --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/rolebinding-cephfs-provisioner.yml.j2 @@ -0,0 +1,14 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cephfs-provisioner + namespace: {{ cephfs_provisioner_namespace }} +subjects: + - kind: ServiceAccount + name: cephfs-provisioner + namespace: {{ cephfs_provisioner_namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cephfs-provisioner diff --git a/kubespray/project/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/sa-cephfs-provisioner.yml.j2 b/kubespray/project/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/sa-cephfs-provisioner.yml.j2 new file mode 100644 index 0000000..31f87bd --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/sa-cephfs-provisioner.yml.j2 @@ -0,0 +1,6 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: cephfs-provisioner + namespace: {{ cephfs_provisioner_namespace }} diff --git a/kubespray/project/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/sc-cephfs-provisioner.yml.j2 b/kubespray/project/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/sc-cephfs-provisioner.yml.j2 new file mode 100644 index 0000000..dd0e37e --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/sc-cephfs-provisioner.yml.j2 @@ -0,0 +1,15 @@ +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: {{ cephfs_provisioner_storage_class }} +provisioner: ceph.com/cephfs +reclaimPolicy: {{ cephfs_provisioner_reclaim_policy }} +parameters: + cluster: {{ cephfs_provisioner_cluster }} + monitors: {{ cephfs_provisioner_monitors }} + adminId: {{ cephfs_provisioner_admin_id }} + adminSecretName: cephfs-provisioner + adminSecretNamespace: {{ cephfs_provisioner_namespace }} + claimRoot: {{ cephfs_provisioner_claim_root }} + deterministicNames: "{{ cephfs_provisioner_deterministic_names | bool | lower }}" diff --git a/kubespray/project/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/secret-cephfs-provisioner.yml.j2 b/kubespray/project/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/secret-cephfs-provisioner.yml.j2 new file mode 100644 index 0000000..6d73c0c --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/secret-cephfs-provisioner.yml.j2 @@ -0,0 +1,9 @@ +--- +kind: Secret +apiVersion: v1 +metadata: + name: cephfs-provisioner + namespace: {{ cephfs_provisioner_namespace }} +type: Opaque +data: + secret: {{ cephfs_provisioner_secret | b64encode }} diff --git a/kubespray/project/roles/kubernetes-apps/external_provisioner/local_path_provisioner/defaults/main.yml b/kubespray/project/roles/kubernetes-apps/external_provisioner/local_path_provisioner/defaults/main.yml new file mode 100644 index 0000000..aacfb22 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_provisioner/local_path_provisioner/defaults/main.yml @@ -0,0 +1,10 @@ +--- +local_path_provisioner_enabled: false +local_path_provisioner_namespace: "local-path-storage" +local_path_provisioner_storage_class: "local-path" +local_path_provisioner_reclaim_policy: Delete +local_path_provisioner_claim_root: /opt/local-path-provisioner/ +local_path_provisioner_is_default_storageclass: "true" +local_path_provisioner_debug: false +local_path_provisioner_helper_image_repo: "busybox" +local_path_provisioner_helper_image_tag: "latest" diff --git a/kubespray/project/roles/kubernetes-apps/external_provisioner/local_path_provisioner/tasks/main.yml b/kubespray/project/roles/kubernetes-apps/external_provisioner/local_path_provisioner/tasks/main.yml new file mode 100644 index 0000000..71036ca --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_provisioner/local_path_provisioner/tasks/main.yml @@ -0,0 +1,47 @@ +--- +- name: Local Path Provisioner | Create addon dir + file: + path: "{{ kube_config_dir }}/addons/local_path_provisioner" + state: directory + owner: root + group: root + mode: 0755 + when: + - inventory_hostname == groups['kube_control_plane'][0] + +- name: Local Path Provisioner | Create claim root dir + file: + path: "{{ local_path_provisioner_claim_root }}" + state: directory + mode: 0755 + +- name: Local Path Provisioner | Render Template + set_fact: + local_path_provisioner_templates: + - { name: local-path-storage-ns, file: local-path-storage-ns.yml, type: ns } + - { name: local-path-storage-sa, file: local-path-storage-sa.yml, type: sa } + - { name: local-path-storage-cr, file: local-path-storage-cr.yml, type: cr } + - { name: local-path-storage-clusterrolebinding, file: local-path-storage-clusterrolebinding.yml, type: clusterrolebinding } + - { name: local-path-storage-cm, file: local-path-storage-cm.yml, type: cm } + - { name: local-path-storage-deployment, file: local-path-storage-deployment.yml, type: deployment } + - { name: local-path-storage-sc, file: local-path-storage-sc.yml, type: sc } + +- name: Local Path Provisioner | Create manifests + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/addons/local_path_provisioner/{{ item.file }}" + mode: 0644 + with_items: "{{ local_path_provisioner_templates }}" + register: local_path_provisioner_manifests + when: inventory_hostname == groups['kube_control_plane'][0] + +- name: Local Path Provisioner | Apply manifests + kube: + name: "{{ item.item.name }}" + namespace: "{{ local_path_provisioner_namespace }}" + kubectl: "{{ bin_dir }}/kubectl" + resource: "{{ item.item.type }}" + filename: "{{ kube_config_dir }}/addons/local_path_provisioner/{{ item.item.file }}" + state: "latest" + with_items: "{{ local_path_provisioner_manifests.results }}" + when: inventory_hostname == groups['kube_control_plane'][0] diff --git a/kubespray/project/roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-clusterrolebinding.yml.j2 b/kubespray/project/roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-clusterrolebinding.yml.j2 new file mode 100644 index 0000000..317a71f --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-clusterrolebinding.yml.j2 @@ -0,0 +1,13 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: local-path-provisioner-bind +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: local-path-provisioner-role +subjects: + - kind: ServiceAccount + name: local-path-provisioner-service-account + namespace: {{ local_path_provisioner_namespace }} diff --git a/kubespray/project/roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-cm.yml.j2 b/kubespray/project/roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-cm.yml.j2 new file mode 100644 index 0000000..9cd7fd3 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-cm.yml.j2 @@ -0,0 +1,35 @@ +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: local-path-config + namespace: {{ local_path_provisioner_namespace }} +data: + config.json: |- + { + "nodePathMap":[ + { + "node":"DEFAULT_PATH_FOR_NON_LISTED_NODES", + "paths":["{{ local_path_provisioner_claim_root }}"] + } + ] + } + setup: |- + #!/bin/sh + set -eu + mkdir -m 0777 -p "$VOL_DIR" + teardown: |- + #!/bin/sh + set -eu + rm -rf "$VOL_DIR" + helperPod.yaml: |- + apiVersion: v1 + kind: Pod + metadata: + name: helper-pod + spec: + containers: + - name: helper-pod + image: "{{ local_path_provisioner_helper_image_repo }}:{{ local_path_provisioner_helper_image_tag }}" + imagePullPolicy: IfNotPresent + diff --git a/kubespray/project/roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-cr.yml.j2 b/kubespray/project/roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-cr.yml.j2 new file mode 100644 index 0000000..299db6e --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-cr.yml.j2 @@ -0,0 +1,18 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: local-path-provisioner-role +rules: + - apiGroups: [ "" ] + resources: [ "nodes", "persistentvolumeclaims", "configmaps" ] + verbs: [ "get", "list", "watch" ] + - apiGroups: [ "" ] + resources: [ "endpoints", "persistentvolumes", "pods" ] + verbs: [ "*" ] + - apiGroups: [ "" ] + resources: [ "events" ] + verbs: [ "create", "patch" ] + - apiGroups: [ "storage.k8s.io" ] + resources: [ "storageclasses" ] + verbs: [ "get", "list", "watch" ] \ No newline at end of file diff --git a/kubespray/project/roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-deployment.yml.j2 b/kubespray/project/roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-deployment.yml.j2 new file mode 100644 index 0000000..6ce426a --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-deployment.yml.j2 @@ -0,0 +1,41 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: local-path-provisioner + namespace: {{ local_path_provisioner_namespace }} +spec: + replicas: 1 + selector: + matchLabels: + app: local-path-provisioner + template: + metadata: + labels: + app: local-path-provisioner + spec: + serviceAccountName: local-path-provisioner-service-account + containers: + - name: local-path-provisioner + image: {{ local_path_provisioner_image_repo }}:{{ local_path_provisioner_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + command: + - local-path-provisioner + - start + - --config + - /etc/config/config.json +{% if local_path_provisioner_debug | default(false) %} + - --debug +{% endif %} + volumeMounts: + - name: config-volume + mountPath: /etc/config/ + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumes: + - name: config-volume + configMap: + name: local-path-config diff --git a/kubespray/project/roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-ns.yml.j2 b/kubespray/project/roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-ns.yml.j2 new file mode 100644 index 0000000..1e8c6ce --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-ns.yml.j2 @@ -0,0 +1,5 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: {{ local_path_provisioner_namespace }} diff --git a/kubespray/project/roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-sa.yml.j2 b/kubespray/project/roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-sa.yml.j2 new file mode 100644 index 0000000..128a106 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-sa.yml.j2 @@ -0,0 +1,6 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: local-path-provisioner-service-account + namespace: {{ local_path_provisioner_namespace }} diff --git a/kubespray/project/roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-sc.yml.j2 b/kubespray/project/roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-sc.yml.j2 new file mode 100644 index 0000000..d662661 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_provisioner/local_path_provisioner/templates/local-path-storage-sc.yml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: {{ local_path_provisioner_storage_class }} + annotations: + storageclass.kubernetes.io/is-default-class: "{{ local_path_provisioner_is_default_storageclass }}" +provisioner: rancher.io/local-path +volumeBindingMode: WaitForFirstConsumer +reclaimPolicy: {{ local_path_provisioner_reclaim_policy }} diff --git a/kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/defaults/main.yml b/kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/defaults/main.yml new file mode 100644 index 0000000..38afefb --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/defaults/main.yml @@ -0,0 +1,20 @@ +--- +local_volume_provisioner_namespace: "kube-system" +# List of node labels to be copied to the PVs created by the provisioner +local_volume_provisioner_nodelabels: [] +# - kubernetes.io/hostname +# - topology.kubernetes.io/region +# - topology.kubernetes.io/zone +local_volume_provisioner_tolerations: [] +local_volume_provisioner_use_node_name_only: false +# Leverages Ansible's string to Python datatype casting. Otherwise the dict_key isn't substituted. +# see https://github.com/ansible/ansible/issues/17324 +local_volume_provisioner_storage_classes: | + { + "{{ local_volume_provisioner_storage_class | default('local-storage') }}": { + "host_dir": "{{ local_volume_provisioner_base_dir | default('/mnt/disks') }}", + "mount_dir": "{{ local_volume_provisioner_mount_dir | default('/mnt/disks') }}", + "volume_mode": "Filesystem", + "fs_type": "ext4" + } + } diff --git a/kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/tasks/basedirs.yml b/kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/tasks/basedirs.yml new file mode 100644 index 0000000..7add2da --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/tasks/basedirs.yml @@ -0,0 +1,12 @@ +--- +# include to workaround mitogen issue +# https://github.com/dw/mitogen/issues/663 + +- name: "Local Volume Provisioner | Ensure base dir {{ delegate_host_base_dir.1 }} is created on {{ delegate_host_base_dir.0 }}" + file: + path: "{{ local_volume_provisioner_storage_classes[delegate_host_base_dir.1].host_dir }}" + state: directory + owner: root + group: root + mode: "{{ local_volume_provisioner_directory_mode }}" + delegate_to: "{{ delegate_host_base_dir.0 }}" diff --git a/kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/tasks/main.yml b/kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/tasks/main.yml new file mode 100644 index 0000000..2308b5c --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/tasks/main.yml @@ -0,0 +1,48 @@ +--- + +- name: Local Volume Provisioner | Ensure base dir is created on all hosts + include_tasks: basedirs.yml + loop_control: + loop_var: delegate_host_base_dir + loop: "{{ groups['k8s_cluster'] | product(local_volume_provisioner_storage_classes.keys()) | list }}" + +- name: Local Volume Provisioner | Create addon dir + file: + path: "{{ kube_config_dir }}/addons/local_volume_provisioner" + state: directory + owner: root + group: root + mode: 0755 + +- name: Local Volume Provisioner | Templates list + set_fact: + local_volume_provisioner_templates: + - { name: local-volume-provisioner-ns, file: local-volume-provisioner-ns.yml, type: ns } + - { name: local-volume-provisioner-sa, file: local-volume-provisioner-sa.yml, type: sa } + - { name: local-volume-provisioner-clusterrole, file: local-volume-provisioner-clusterrole.yml, type: clusterrole } + - { name: local-volume-provisioner-clusterrolebinding, file: local-volume-provisioner-clusterrolebinding.yml, type: clusterrolebinding } + - { name: local-volume-provisioner-cm, file: local-volume-provisioner-cm.yml, type: cm } + - { name: local-volume-provisioner-ds, file: local-volume-provisioner-ds.yml, type: ds } + - { name: local-volume-provisioner-sc, file: local-volume-provisioner-sc.yml, type: sc } + +- name: Local Volume Provisioner | Create manifests + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/addons/local_volume_provisioner/{{ item.file }}" + mode: 0644 + with_items: "{{ local_volume_provisioner_templates }}" + register: local_volume_provisioner_manifests + when: inventory_hostname == groups['kube_control_plane'][0] + +- name: Local Volume Provisioner | Apply manifests + kube: + name: "{{ item.item.name }}" + namespace: "{{ local_volume_provisioner_namespace }}" + kubectl: "{{ bin_dir }}/kubectl" + resource: "{{ item.item.type }}" + filename: "{{ kube_config_dir }}/addons/local_volume_provisioner/{{ item.item.file }}" + state: "latest" + with_items: "{{ local_volume_provisioner_manifests.results }}" + when: inventory_hostname == groups['kube_control_plane'][0] + loop_control: + label: "{{ item.item.file }}" diff --git a/kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-clusterrole.yml.j2 b/kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-clusterrole.yml.j2 new file mode 100644 index 0000000..ada55dd --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-clusterrole.yml.j2 @@ -0,0 +1,22 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: local-volume-provisioner-node-clusterrole + namespace: {{ local_volume_provisioner_namespace }} +rules: +- apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] +- apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: ["events"] + verbs: ["watch"] +- apiGroups: ["", "events.k8s.io"] + resources: ["events"] + verbs: ["create", "update", "patch"] +- apiGroups: [""] + resources: ["nodes"] + verbs: ["get"] diff --git a/kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-clusterrolebinding.yml.j2 b/kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-clusterrolebinding.yml.j2 new file mode 100644 index 0000000..bc286b2 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-clusterrolebinding.yml.j2 @@ -0,0 +1,14 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: local-volume-provisioner-system-node + namespace: {{ local_volume_provisioner_namespace }} +subjects: +- kind: ServiceAccount + name: local-volume-provisioner + namespace: {{ local_volume_provisioner_namespace }} +roleRef: + kind: ClusterRole + name: local-volume-provisioner-node-clusterrole + apiGroup: rbac.authorization.k8s.io diff --git a/kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-cm.yml.j2 b/kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-cm.yml.j2 new file mode 100644 index 0000000..7e37283 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-cm.yml.j2 @@ -0,0 +1,33 @@ +# Macro to convert camelCase dictionary keys to snake_case keys +{% macro convert_keys(mydict) -%} + {% for key in mydict.keys() | list -%} + {% set key_split = key.split('_') -%} + {% set new_key = key_split[0] + key_split[1:] | map('capitalize') | join -%} + {% set value = mydict.pop(key) -%} + {{ mydict.__setitem__(new_key, value) -}} + {{ convert_keys(value) if value is mapping else None -}} + {% endfor -%} +{% endmacro -%} + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: local-volume-provisioner + namespace: {{ local_volume_provisioner_namespace }} +data: +{% if local_volume_provisioner_nodelabels | length > 0 %} + nodeLabelsForPV: | +{% for nodelabel in local_volume_provisioner_nodelabels %} + - {{ nodelabel }} +{% endfor %} +{% endif %} +{% if local_volume_provisioner_use_node_name_only %} + useNodeNameOnly: "true" +{% endif %} + storageClassMap: | +{% for class_name, storage_class in local_volume_provisioner_storage_classes.items() %} + {{ class_name }}: + {{- convert_keys(storage_class) }} + {{ storage_class | to_nice_yaml(indent=2) | indent(6) }} +{%- endfor %} diff --git a/kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-ds.yml.j2 b/kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-ds.yml.j2 new file mode 100644 index 0000000..90a4730 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-ds.yml.j2 @@ -0,0 +1,66 @@ +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: local-volume-provisioner + namespace: {{ local_volume_provisioner_namespace }} + labels: + k8s-app: local-volume-provisioner + version: {{ local_volume_provisioner_image_tag }} +spec: + selector: + matchLabels: + k8s-app: local-volume-provisioner + version: {{ local_volume_provisioner_image_tag }} + template: + metadata: + labels: + k8s-app: local-volume-provisioner + version: {{ local_volume_provisioner_image_tag }} + spec: + priorityClassName: {% if local_volume_provisioner_namespace == 'kube-system' %}system-node-critical{% else %}k8s-cluster-critical{% endif %}{{ '' }} + serviceAccountName: local-volume-provisioner + nodeSelector: + kubernetes.io/os: linux +{% if local_volume_provisioner_tolerations %} + tolerations: + {{ local_volume_provisioner_tolerations | to_nice_yaml(indent=2) | indent(width=8) }} +{% endif %} + containers: + - name: provisioner + image: {{ local_volume_provisioner_image_repo }}:{{ local_volume_provisioner_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + securityContext: + privileged: true + env: + - name: MY_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: MY_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: local-volume-provisioner + mountPath: /etc/provisioner/config + readOnly: true + - mountPath: /dev + name: provisioner-dev +{% for class_name, class_config in local_volume_provisioner_storage_classes.items() %} + - name: local-volume-provisioner-hostpath-{{ class_name }} + mountPath: {{ class_config.mount_dir }} + mountPropagation: "HostToContainer" +{% endfor %} + volumes: + - name: local-volume-provisioner + configMap: + name: local-volume-provisioner + - name: provisioner-dev + hostPath: + path: /dev +{% for class_name, class_config in local_volume_provisioner_storage_classes.items() %} + - name: local-volume-provisioner-hostpath-{{ class_name }} + hostPath: + path: {{ class_config.host_dir }} +{% endfor %} diff --git a/kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-ns.yml.j2 b/kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-ns.yml.j2 new file mode 100644 index 0000000..04a7910 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-ns.yml.j2 @@ -0,0 +1,7 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: {{ local_volume_provisioner_namespace }} + labels: + name: {{ local_volume_provisioner_namespace }} diff --git a/kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-sa.yml.j2 b/kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-sa.yml.j2 new file mode 100644 index 0000000..c78a16b --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-sa.yml.j2 @@ -0,0 +1,6 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: local-volume-provisioner + namespace: {{ local_volume_provisioner_namespace }} diff --git a/kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-sc.yml.j2 b/kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-sc.yml.j2 new file mode 100644 index 0000000..81e0260 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-sc.yml.j2 @@ -0,0 +1,12 @@ +{% for class_name, class_config in local_volume_provisioner_storage_classes.items() %} +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: {{ class_name }} +provisioner: kubernetes.io/no-provisioner +volumeBindingMode: WaitForFirstConsumer +{% if class_config.reclaim_policy is defined %} +reclaimPolicy: {{ class_config.reclaim_policy }} +{% endif %} +{% endfor %} diff --git a/kubespray/project/roles/kubernetes-apps/external_provisioner/meta/main.yml b/kubespray/project/roles/kubernetes-apps/external_provisioner/meta/main.yml new file mode 100644 index 0000000..13bc8b6 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_provisioner/meta/main.yml @@ -0,0 +1,30 @@ +--- +dependencies: + - role: kubernetes-apps/external_provisioner/local_volume_provisioner + when: + - local_volume_provisioner_enabled + - inventory_hostname == groups['kube_control_plane'][0] + tags: + - apps + - local-volume-provisioner + - external-provisioner + + - role: kubernetes-apps/external_provisioner/cephfs_provisioner + when: cephfs_provisioner_enabled + tags: + - apps + - cephfs-provisioner + - external-provisioner + + - role: kubernetes-apps/external_provisioner/rbd_provisioner + when: rbd_provisioner_enabled + tags: + - apps + - rbd-provisioner + - external-provisioner + - role: kubernetes-apps/external_provisioner/local_path_provisioner + when: local_path_provisioner_enabled + tags: + - apps + - local-path-provisioner + - external-provisioner diff --git a/kubespray/project/roles/kubernetes-apps/external_provisioner/rbd_provisioner/defaults/main.yml b/kubespray/project/roles/kubernetes-apps/external_provisioner/rbd_provisioner/defaults/main.yml new file mode 100644 index 0000000..f09e25a --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_provisioner/rbd_provisioner/defaults/main.yml @@ -0,0 +1,17 @@ +--- +rbd_provisioner_namespace: "rbd-provisioner" +rbd_provisioner_replicas: 2 +rbd_provisioner_monitors: ~ +rbd_provisioner_pool: kube +rbd_provisioner_admin_id: admin +rbd_provisioner_secret_name: ceph-secret-admin +rbd_provisioner_secret: ceph-key-admin +rbd_provisioner_user_id: kube +rbd_provisioner_user_secret_name: ceph-secret-user +rbd_provisioner_user_secret: ceph-key-user +rbd_provisioner_user_secret_namespace: rbd-provisioner +rbd_provisioner_fs_type: ext4 +rbd_provisioner_image_format: "2" +rbd_provisioner_image_features: layering +rbd_provisioner_storage_class: rbd +rbd_provisioner_reclaim_policy: Delete diff --git a/kubespray/project/roles/kubernetes-apps/external_provisioner/rbd_provisioner/tasks/main.yml b/kubespray/project/roles/kubernetes-apps/external_provisioner/rbd_provisioner/tasks/main.yml new file mode 100644 index 0000000..76445da --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_provisioner/rbd_provisioner/tasks/main.yml @@ -0,0 +1,71 @@ +--- + +- name: RBD Provisioner | Remove legacy addon dir and manifests + file: + path: "{{ kube_config_dir }}/addons/rbd_provisioner" + state: absent + when: + - inventory_hostname == groups['kube_control_plane'][0] + tags: + - upgrade + +- name: RBD Provisioner | Remove legacy namespace + command: > + {{ kubectl }} delete namespace {{ rbd_provisioner_namespace }} + ignore_errors: true # noqa ignore-errors + when: + - inventory_hostname == groups['kube_control_plane'][0] + tags: + - upgrade + +- name: RBD Provisioner | Remove legacy storageclass + command: > + {{ kubectl }} delete storageclass {{ rbd_provisioner_storage_class }} + ignore_errors: true # noqa ignore-errors + when: + - inventory_hostname == groups['kube_control_plane'][0] + tags: + - upgrade + +- name: RBD Provisioner | Create addon dir + file: + path: "{{ kube_config_dir }}/addons/rbd_provisioner" + state: directory + owner: root + group: root + mode: 0755 + when: + - inventory_hostname == groups['kube_control_plane'][0] + +- name: RBD Provisioner | Templates list + set_fact: + rbd_provisioner_templates: + - { name: 00-namespace, file: 00-namespace.yml, type: ns } + - { name: secret-rbd-provisioner, file: secret-rbd-provisioner.yml, type: secret } + - { name: sa-rbd-provisioner, file: sa-rbd-provisioner.yml, type: sa } + - { name: clusterrole-rbd-provisioner, file: clusterrole-rbd-provisioner.yml, type: clusterrole } + - { name: clusterrolebinding-rbd-provisioner, file: clusterrolebinding-rbd-provisioner.yml, type: clusterrolebinding } + - { name: role-rbd-provisioner, file: role-rbd-provisioner.yml, type: role } + - { name: rolebinding-rbd-provisioner, file: rolebinding-rbd-provisioner.yml, type: rolebinding } + - { name: deploy-rbd-provisioner, file: deploy-rbd-provisioner.yml, type: deploy } + - { name: sc-rbd-provisioner, file: sc-rbd-provisioner.yml, type: sc } + +- name: RBD Provisioner | Create manifests + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/addons/rbd_provisioner/{{ item.file }}" + mode: 0644 + with_items: "{{ rbd_provisioner_templates }}" + register: rbd_provisioner_manifests + when: inventory_hostname == groups['kube_control_plane'][0] + +- name: RBD Provisioner | Apply manifests + kube: + name: "{{ item.item.name }}" + namespace: "{{ rbd_provisioner_namespace }}" + kubectl: "{{ bin_dir }}/kubectl" + resource: "{{ item.item.type }}" + filename: "{{ kube_config_dir }}/addons/rbd_provisioner/{{ item.item.file }}" + state: "latest" + with_items: "{{ rbd_provisioner_manifests.results }}" + when: inventory_hostname == groups['kube_control_plane'][0] diff --git a/kubespray/project/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/00-namespace.yml.j2 b/kubespray/project/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/00-namespace.yml.j2 new file mode 100644 index 0000000..8bec2b5 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/00-namespace.yml.j2 @@ -0,0 +1,7 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: {{ rbd_provisioner_namespace }} + labels: + name: {{ rbd_provisioner_namespace }} diff --git a/kubespray/project/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/clusterrole-rbd-provisioner.yml.j2 b/kubespray/project/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/clusterrole-rbd-provisioner.yml.j2 new file mode 100644 index 0000000..9e319a3 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/clusterrole-rbd-provisioner.yml.j2 @@ -0,0 +1,26 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: rbd-provisioner + namespace: {{ rbd_provisioner_namespace }} +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "update", "patch"] + - apiGroups: [""] + resources: ["services"] + resourceNames: ["kube-dns","coredns"] + verbs: ["list", "get"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "create", "delete"] diff --git a/kubespray/project/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/clusterrolebinding-rbd-provisioner.yml.j2 b/kubespray/project/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/clusterrolebinding-rbd-provisioner.yml.j2 new file mode 100644 index 0000000..ae9e6c5 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/clusterrolebinding-rbd-provisioner.yml.j2 @@ -0,0 +1,13 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: rbd-provisioner +subjects: + - kind: ServiceAccount + name: rbd-provisioner + namespace: {{ rbd_provisioner_namespace }} +roleRef: + kind: ClusterRole + name: rbd-provisioner + apiGroup: rbac.authorization.k8s.io diff --git a/kubespray/project/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/deploy-rbd-provisioner.yml.j2 b/kubespray/project/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/deploy-rbd-provisioner.yml.j2 new file mode 100644 index 0000000..b8643db --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/deploy-rbd-provisioner.yml.j2 @@ -0,0 +1,40 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: rbd-provisioner + namespace: {{ rbd_provisioner_namespace }} + labels: + app: rbd-provisioner + version: {{ rbd_provisioner_image_tag }} +spec: + replicas: {{ rbd_provisioner_replicas }} + strategy: + type: Recreate + selector: + matchLabels: + app: rbd-provisioner + version: {{ rbd_provisioner_image_tag }} + template: + metadata: + labels: + app: rbd-provisioner + version: {{ rbd_provisioner_image_tag }} + spec: + priorityClassName: {% if rbd_provisioner_namespace == 'kube-system' %}system-cluster-critical{% else %}k8s-cluster-critical{% endif %}{{ '' }} + serviceAccount: rbd-provisioner + containers: + - name: rbd-provisioner + image: {{ rbd_provisioner_image_repo }}:{{ rbd_provisioner_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + env: + - name: PROVISIONER_NAME + value: ceph.com/rbd + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + command: + - "/usr/local/bin/rbd-provisioner" + args: + - "-id=${POD_NAME}" diff --git a/kubespray/project/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/role-rbd-provisioner.yml.j2 b/kubespray/project/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/role-rbd-provisioner.yml.j2 new file mode 100644 index 0000000..d8dbbf9 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/role-rbd-provisioner.yml.j2 @@ -0,0 +1,13 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: rbd-provisioner + namespace: {{ rbd_provisioner_namespace }} +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get"] + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "list", "watch", "create", "update", "patch"] diff --git a/kubespray/project/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/rolebinding-rbd-provisioner.yml.j2 b/kubespray/project/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/rolebinding-rbd-provisioner.yml.j2 new file mode 100644 index 0000000..fcae1cc --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/rolebinding-rbd-provisioner.yml.j2 @@ -0,0 +1,14 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: rbd-provisioner + namespace: {{ rbd_provisioner_namespace }} +subjects: + - kind: ServiceAccount + name: rbd-provisioner + namespace: {{ rbd_provisioner_namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: rbd-provisioner diff --git a/kubespray/project/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/sa-rbd-provisioner.yml.j2 b/kubespray/project/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/sa-rbd-provisioner.yml.j2 new file mode 100644 index 0000000..c4dce64 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/sa-rbd-provisioner.yml.j2 @@ -0,0 +1,6 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rbd-provisioner + namespace: {{ rbd_provisioner_namespace }} diff --git a/kubespray/project/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/sc-rbd-provisioner.yml.j2 b/kubespray/project/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/sc-rbd-provisioner.yml.j2 new file mode 100644 index 0000000..9fea17a --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/sc-rbd-provisioner.yml.j2 @@ -0,0 +1,19 @@ +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: {{ rbd_provisioner_storage_class }} +provisioner: ceph.com/rbd +reclaimPolicy: {{ rbd_provisioner_reclaim_policy }} +parameters: + monitors: {{ rbd_provisioner_monitors }} + adminId: {{ rbd_provisioner_admin_id }} + adminSecretNamespace: {{ rbd_provisioner_namespace }} + adminSecretName: {{ rbd_provisioner_secret_name }} + pool: {{ rbd_provisioner_pool }} + userId: {{ rbd_provisioner_user_id }} + userSecretNamespace: {{ rbd_provisioner_user_secret_namespace }} + userSecretName: {{ rbd_provisioner_user_secret_name }} + fsType: "{{ rbd_provisioner_fs_type }}" + imageFormat: "{{ rbd_provisioner_image_format }}" + imageFeatures: {{ rbd_provisioner_image_features }} diff --git a/kubespray/project/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/secret-rbd-provisioner.yml.j2 b/kubespray/project/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/secret-rbd-provisioner.yml.j2 new file mode 100644 index 0000000..a3b66d6 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/secret-rbd-provisioner.yml.j2 @@ -0,0 +1,18 @@ +--- +kind: Secret +apiVersion: v1 +metadata: + name: {{ rbd_provisioner_secret_name }} + namespace: {{ rbd_provisioner_namespace }} +type: Opaque +data: + secret: {{ rbd_provisioner_secret | b64encode }} +--- +kind: Secret +apiVersion: v1 +metadata: + name: {{ rbd_provisioner_user_secret_name }} + namespace: {{ rbd_provisioner_user_secret_namespace }} +type: Opaque +data: + key: {{ rbd_provisioner_user_secret | b64encode }} diff --git a/kubespray/project/roles/kubernetes-apps/helm/.gitkeep b/kubespray/project/roles/kubernetes-apps/helm/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/kubespray/project/roles/kubernetes-apps/helm/defaults/main.yml b/kubespray/project/roles/kubernetes-apps/helm/defaults/main.yml new file mode 100644 index 0000000..4dc1cca --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/helm/defaults/main.yml @@ -0,0 +1,2 @@ +--- +helm_enabled: false diff --git a/kubespray/project/roles/kubernetes-apps/helm/tasks/main.yml b/kubespray/project/roles/kubernetes-apps/helm/tasks/main.yml new file mode 100644 index 0000000..eae0e21 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/helm/tasks/main.yml @@ -0,0 +1,49 @@ +--- +- name: Helm | Gather os specific variables + include_vars: "{{ item }}" + with_first_found: + - files: + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower | replace('/', '_') }}.yml" + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_release }}.yml" + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower | replace('/', '_') }}.yml" + - "{{ ansible_distribution | lower }}.yml" + - "{{ ansible_os_family | lower }}.yml" + - defaults.yml + paths: + - ../vars + skip: true + +- name: Helm | Install PyYaml + package: + name: "{{ pyyaml_package }}" + state: present + when: pyyaml_package is defined + +- name: Helm | Install PyYaml [flatcar] + include_tasks: pyyaml-flatcar.yml + when: ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] + +- name: Helm | Download helm + include_tasks: "../../../download/tasks/download_file.yml" + vars: + download: "{{ download_defaults | combine(downloads.helm) }}" + +- name: Helm | Copy helm binary from download dir + copy: + src: "{{ local_release_dir }}/helm-{{ helm_version }}/linux-{{ image_arch }}/helm" + dest: "{{ bin_dir }}/helm" + mode: 0755 + remote_src: true + +- name: Helm | Get helm completion + command: "{{ bin_dir }}/helm completion bash" + changed_when: False + register: helm_completion + check_mode: False + +- name: Helm | Install helm completion + copy: + dest: /etc/bash_completion.d/helm.sh + content: "{{ helm_completion.stdout }}" + mode: 0755 + become: True diff --git a/kubespray/project/roles/kubernetes-apps/helm/tasks/pyyaml-flatcar.yml b/kubespray/project/roles/kubernetes-apps/helm/tasks/pyyaml-flatcar.yml new file mode 100644 index 0000000..ea0d63a --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/helm/tasks/pyyaml-flatcar.yml @@ -0,0 +1,22 @@ +--- +- name: Get installed pip version + command: "{{ ansible_python_interpreter if ansible_python_interpreter is defined else 'python' }} -m pip --version" + register: pip_version_output + ignore_errors: yes + changed_when: false + +- name: Get installed PyYAML version + command: "{{ ansible_python_interpreter if ansible_python_interpreter is defined else 'python' }} -m pip show PyYAML" + register: pyyaml_version_output + ignore_errors: yes + changed_when: false + +- name: Install pip + command: "{{ ansible_python_interpreter if ansible_python_interpreter is defined else 'python' }} -m ensurepip --upgrade" + when: (pyyaml_version_output is failed) and (pip_version_output is failed) + +- name: Install PyYAML + ansible.builtin.pip: + name: + - PyYAML + when: (pyyaml_version_output is failed) diff --git a/kubespray/project/roles/kubernetes-apps/helm/vars/amazon.yml b/kubespray/project/roles/kubernetes-apps/helm/vars/amazon.yml new file mode 100644 index 0000000..190d262 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/helm/vars/amazon.yml @@ -0,0 +1,2 @@ +--- +pyyaml_package: PyYAML diff --git a/kubespray/project/roles/kubernetes-apps/helm/vars/centos-7.yml b/kubespray/project/roles/kubernetes-apps/helm/vars/centos-7.yml new file mode 100644 index 0000000..190d262 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/helm/vars/centos-7.yml @@ -0,0 +1,2 @@ +--- +pyyaml_package: PyYAML diff --git a/kubespray/project/roles/kubernetes-apps/helm/vars/centos.yml b/kubespray/project/roles/kubernetes-apps/helm/vars/centos.yml new file mode 100644 index 0000000..ba3964d --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/helm/vars/centos.yml @@ -0,0 +1,2 @@ +--- +pyyaml_package: python3-pyyaml diff --git a/kubespray/project/roles/kubernetes-apps/helm/vars/debian.yml b/kubespray/project/roles/kubernetes-apps/helm/vars/debian.yml new file mode 100644 index 0000000..db0add5 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/helm/vars/debian.yml @@ -0,0 +1,2 @@ +--- +pyyaml_package: python3-yaml diff --git a/kubespray/project/roles/kubernetes-apps/helm/vars/fedora.yml b/kubespray/project/roles/kubernetes-apps/helm/vars/fedora.yml new file mode 100644 index 0000000..ba3964d --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/helm/vars/fedora.yml @@ -0,0 +1,2 @@ +--- +pyyaml_package: python3-pyyaml diff --git a/kubespray/project/roles/kubernetes-apps/helm/vars/redhat-7.yml b/kubespray/project/roles/kubernetes-apps/helm/vars/redhat-7.yml new file mode 100644 index 0000000..190d262 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/helm/vars/redhat-7.yml @@ -0,0 +1,2 @@ +--- +pyyaml_package: PyYAML diff --git a/kubespray/project/roles/kubernetes-apps/helm/vars/redhat.yml b/kubespray/project/roles/kubernetes-apps/helm/vars/redhat.yml new file mode 100644 index 0000000..ba3964d --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/helm/vars/redhat.yml @@ -0,0 +1,2 @@ +--- +pyyaml_package: python3-pyyaml diff --git a/kubespray/project/roles/kubernetes-apps/helm/vars/suse.yml b/kubespray/project/roles/kubernetes-apps/helm/vars/suse.yml new file mode 100644 index 0000000..c34bb69 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/helm/vars/suse.yml @@ -0,0 +1,2 @@ +--- +pyyaml_package: python3-PyYAML diff --git a/kubespray/project/roles/kubernetes-apps/helm/vars/ubuntu.yml b/kubespray/project/roles/kubernetes-apps/helm/vars/ubuntu.yml new file mode 100644 index 0000000..db0add5 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/helm/vars/ubuntu.yml @@ -0,0 +1,2 @@ +--- +pyyaml_package: python3-yaml diff --git a/kubespray/project/roles/kubernetes-apps/ingress_controller/alb_ingress_controller/OWNERS b/kubespray/project/roles/kubernetes-apps/ingress_controller/alb_ingress_controller/OWNERS new file mode 100644 index 0000000..a80e724 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ingress_controller/alb_ingress_controller/OWNERS @@ -0,0 +1,6 @@ +# See the OWNERS docs at https://go.k8s.io/owners + +approvers: + - kubespray-approvers +reviewers: + - kubespray-reviewers diff --git a/kubespray/project/roles/kubernetes-apps/ingress_controller/alb_ingress_controller/defaults/main.yml b/kubespray/project/roles/kubernetes-apps/ingress_controller/alb_ingress_controller/defaults/main.yml new file mode 100644 index 0000000..4c8d97e --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ingress_controller/alb_ingress_controller/defaults/main.yml @@ -0,0 +1,7 @@ +--- +alb_ingress_controller_namespace: kube-system +alb_ingress_aws_region: "us-east-1" + +# Enables logging on all outbound requests sent to the AWS API. +# If logging is desired, set to true. +alb_ingress_aws_debug: "false" diff --git a/kubespray/project/roles/kubernetes-apps/ingress_controller/alb_ingress_controller/tasks/main.yml b/kubespray/project/roles/kubernetes-apps/ingress_controller/alb_ingress_controller/tasks/main.yml new file mode 100644 index 0000000..8a188a4 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ingress_controller/alb_ingress_controller/tasks/main.yml @@ -0,0 +1,36 @@ +--- + +- name: ALB Ingress Controller | Create addon dir + file: + path: "{{ kube_config_dir }}/addons/alb_ingress" + state: directory + owner: root + group: root + mode: 0755 + +- name: ALB Ingress Controller | Create manifests + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/addons/alb_ingress/{{ item.file }}" + mode: 0644 + with_items: + - { name: alb-ingress-clusterrole, file: alb-ingress-clusterrole.yml, type: clusterrole } + - { name: alb-ingress-clusterrolebinding, file: alb-ingress-clusterrolebinding.yml, type: clusterrolebinding } + - { name: alb-ingress-ns, file: alb-ingress-ns.yml, type: ns } + - { name: alb-ingress-sa, file: alb-ingress-sa.yml, type: sa } + - { name: alb-ingress-deploy, file: alb-ingress-deploy.yml, type: deploy } + register: alb_ingress_manifests + when: + - inventory_hostname == groups['kube_control_plane'][0] + +- name: ALB Ingress Controller | Apply manifests + kube: + name: "{{ item.item.name }}" + namespace: "{{ alb_ingress_controller_namespace }}" + kubectl: "{{ bin_dir }}/kubectl" + resource: "{{ item.item.type }}" + filename: "{{ kube_config_dir }}/addons/alb_ingress/{{ item.item.file }}" + state: "latest" + with_items: "{{ alb_ingress_manifests.results }}" + when: + - inventory_hostname == groups['kube_control_plane'][0] diff --git a/kubespray/project/roles/kubernetes-apps/ingress_controller/alb_ingress_controller/templates/alb-ingress-clusterrole.yml.j2 b/kubespray/project/roles/kubernetes-apps/ingress_controller/alb_ingress_controller/templates/alb-ingress-clusterrole.yml.j2 new file mode 100644 index 0000000..bc03095 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ingress_controller/alb_ingress_controller/templates/alb-ingress-clusterrole.yml.j2 @@ -0,0 +1,13 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: alb-ingress + namespace: {{ alb_ingress_controller_namespace }} +rules: + - apiGroups: ["", "extensions"] + resources: ["configmaps", "endpoints", "nodes", "pods", "secrets", "events", "ingresses", "ingresses/status", "services"] + verbs: ["list", "create", "get", "update", "watch", "patch"] + - apiGroups: ["", "extensions"] + resources: ["nodes", "pods", "secrets", "services", "namespaces"] + verbs: ["get", "list", "watch"] diff --git a/kubespray/project/roles/kubernetes-apps/ingress_controller/alb_ingress_controller/templates/alb-ingress-clusterrolebinding.yml.j2 b/kubespray/project/roles/kubernetes-apps/ingress_controller/alb_ingress_controller/templates/alb-ingress-clusterrolebinding.yml.j2 new file mode 100644 index 0000000..71068f4 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ingress_controller/alb_ingress_controller/templates/alb-ingress-clusterrolebinding.yml.j2 @@ -0,0 +1,14 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: alb-ingress + namespace: {{ alb_ingress_controller_namespace }} +subjects: + - kind: ServiceAccount + name: alb-ingress + namespace: {{ alb_ingress_controller_namespace }} +roleRef: + kind: ClusterRole + name: alb-ingress + apiGroup: rbac.authorization.k8s.io diff --git a/kubespray/project/roles/kubernetes-apps/ingress_controller/alb_ingress_controller/templates/alb-ingress-deploy.yml.j2 b/kubespray/project/roles/kubernetes-apps/ingress_controller/alb_ingress_controller/templates/alb-ingress-deploy.yml.j2 new file mode 100644 index 0000000..a3d2834 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ingress_controller/alb_ingress_controller/templates/alb-ingress-deploy.yml.j2 @@ -0,0 +1,74 @@ +# Application Load Balancer (ALB) Ingress Controller Deployment Manifest. +# This manifest details sensible defaults for deploying an ALB Ingress Controller. +# GitHub: https://github.com/coreos/alb-ingress-controller +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: alb-ingress-controller + labels: + k8s-app: alb-ingress-controller + # Namespace the ALB Ingress Controller should run in. Does not impact which + # namespaces it's able to resolve ingress resource for. For limiting ingress + # namespace scope, see --watch-namespace. + namespace: {{ alb_ingress_controller_namespace }} +spec: + replicas: 1 + selector: + matchLabels: + k8s-app: alb-ingress-controller + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + creationTimestamp: null + labels: + k8s-app: alb-ingress-controller + spec: + containers: + - args: + # Limit the namespace where this ALB Ingress Controller deployment will + # resolve ingress resources. If left commented, all namespaces are used. + #- --watch-namespace=your-k8s-namespace + + # Setting the ingress-class flag below will ensure that only ingress resources with the + # annotation kubernetes.io/ingress.class: "alb" are respected by the controller. You may + # choose any class you'd like for this controller to respect. + - --ingress-class=alb + # Name of your cluster. Used when naming resources created + # by the ALB Ingress Controller, providing distinction between + # clusters. + - --cluster-name={{ cluster_name }} + + # Enables logging on all outbound requests sent to the AWS API. + # If logging is desired, set to true. + # - ---aws-api-debug +{% if alb_ingress_aws_debug %} + - --aws-api-debug +{% endif %} + # Maximum number of times to retry the aws calls. + # defaults to 10. + # - --aws-max-retries=10 + + # AWS region this ingress controller will operate in. + # If unspecified, it will be discovered from ec2metadata. + # List of regions: http://docs.aws.amazon.com/general/latest/gr/rande.html#vpc_region +{% if alb_ingress_aws_region is defined %} + - --aws-region={{ alb_ingress_aws_region }} +{% endif %} + + image: "{{ alb_ingress_image_repo }}:{{ alb_ingress_image_tag }}" + imagePullPolicy: {{ k8s_image_pull_policy }} + name: server + resources: {} + terminationMessagePath: /dev/termination-log + dnsPolicy: ClusterFirst + restartPolicy: Always + securityContext: {} + terminationGracePeriodSeconds: 30 +{% if rbac_enabled %} + serviceAccountName: alb-ingress +{% endif %} diff --git a/kubespray/project/roles/kubernetes-apps/ingress_controller/alb_ingress_controller/templates/alb-ingress-ns.yml.j2 b/kubespray/project/roles/kubernetes-apps/ingress_controller/alb_ingress_controller/templates/alb-ingress-ns.yml.j2 new file mode 100644 index 0000000..9f57537 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ingress_controller/alb_ingress_controller/templates/alb-ingress-ns.yml.j2 @@ -0,0 +1,7 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: {{ alb_ingress_controller_namespace }} + labels: + name: {{ alb_ingress_controller_namespace }} diff --git a/kubespray/project/roles/kubernetes-apps/ingress_controller/alb_ingress_controller/templates/alb-ingress-sa.yml.j2 b/kubespray/project/roles/kubernetes-apps/ingress_controller/alb_ingress_controller/templates/alb-ingress-sa.yml.j2 new file mode 100644 index 0000000..692e3e3 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ingress_controller/alb_ingress_controller/templates/alb-ingress-sa.yml.j2 @@ -0,0 +1,6 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: alb-ingress + namespace: {{ alb_ingress_controller_namespace }} diff --git a/kubespray/project/roles/kubernetes-apps/ingress_controller/cert_manager/defaults/main.yml b/kubespray/project/roles/kubernetes-apps/ingress_controller/cert_manager/defaults/main.yml new file mode 100644 index 0000000..0f58bd5 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ingress_controller/cert_manager/defaults/main.yml @@ -0,0 +1,19 @@ +--- +cert_manager_namespace: "cert-manager" +cert_manager_user: 1001 +cert_manager_tolerations: [] +cert_manager_affinity: {} +cert_manager_nodeselector: {} +cert_manager_dns_policy: "ClusterFirst" +cert_manager_dns_config: {} +cert_manager_controller_extra_args: [] + +## Allow http_proxy, https_proxy and no_proxy environment variables +## Details https://github.com/kubernetes-sigs/kubespray/blob/master/docs/proxy.md +cert_manager_http_proxy: "{{ http_proxy | default('') }}" +cert_manager_https_proxy: "{{ https_proxy | default('') }}" +cert_manager_no_proxy: "{{ no_proxy | default('') }}" + +## Change leader election namespace when deploying on GKE Autopilot that forbid the changes on kube-system namespace. +## See https://github.com/jetstack/cert-manager/issues/3717 +cert_manager_leader_election_namespace: kube-system diff --git a/kubespray/project/roles/kubernetes-apps/ingress_controller/cert_manager/tasks/main.yml b/kubespray/project/roles/kubernetes-apps/ingress_controller/cert_manager/tasks/main.yml new file mode 100644 index 0000000..4af64ad --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ingress_controller/cert_manager/tasks/main.yml @@ -0,0 +1,56 @@ +--- + +- name: Cert Manager | Remove legacy addon dir and manifests + file: + path: "{{ kube_config_dir }}/addons/cert_manager" + state: absent + when: + - inventory_hostname == groups['kube_control_plane'][0] + tags: + - upgrade + +- name: Cert Manager | Remove legacy namespace + command: > + {{ kubectl }} delete namespace {{ cert_manager_namespace }} + ignore_errors: true # noqa ignore-errors + when: + - inventory_hostname == groups['kube_control_plane'][0] + tags: + - upgrade + +- name: Cert Manager | Create addon dir + file: + path: "{{ kube_config_dir }}/addons/cert_manager" + state: directory + owner: root + group: root + mode: 0755 + when: + - inventory_hostname == groups['kube_control_plane'][0] + +- name: Cert Manager | Templates list + set_fact: + cert_manager_templates: + - { name: cert-manager, file: cert-manager.yml, type: all } + - { name: cert-manager.crds, file: cert-manager.crds.yml, type: crd } + +- name: Cert Manager | Create manifests + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/addons/cert_manager/{{ item.file }}" + mode: 0644 + with_items: "{{ cert_manager_templates }}" + register: cert_manager_manifests + when: + - inventory_hostname == groups['kube_control_plane'][0] + +- name: Cert Manager | Apply manifests + kube: + name: "{{ item.item.name }}" + kubectl: "{{ bin_dir }}/kubectl" + resource: "{{ item.item.type }}" + filename: "{{ kube_config_dir }}/addons/cert_manager/{{ item.item.file }}" + state: "latest" + with_items: "{{ cert_manager_manifests.results }}" + when: + - inventory_hostname == groups['kube_control_plane'][0] diff --git a/kubespray/project/roles/kubernetes-apps/ingress_controller/cert_manager/templates/cert-manager.crds.yml.j2 b/kubespray/project/roles/kubernetes-apps/ingress_controller/cert_manager/templates/cert-manager.crds.yml.j2 new file mode 100644 index 0000000..ec96dbb --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ingress_controller/cert_manager/templates/cert-manager.crds.yml.j2 @@ -0,0 +1,4482 @@ +# Copyright 2022 The cert-manager Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Source: cert-manager/deploy/crds/crd-clusterissuers.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: clusterissuers.cert-manager.io + labels: + app: 'cert-manager' + app.kubernetes.io/name: 'cert-manager' + app.kubernetes.io/instance: "cert-manager" + # Generated labels + app.kubernetes.io/version: "{{ cert_manager_version }}" +spec: + group: cert-manager.io + names: + kind: ClusterIssuer + listKind: ClusterIssuerList + plural: clusterissuers + singular: clusterissuer + categories: + - cert-manager + scope: Cluster + versions: + - name: v1 + subresources: + status: {} + additionalPrinterColumns: + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].message + name: Status + priority: 1 + type: string + - jsonPath: .metadata.creationTimestamp + description: CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC. + name: Age + type: date + schema: + openAPIV3Schema: + description: A ClusterIssuer represents a certificate issuing authority which can be referenced as part of `issuerRef` fields. It is similar to an Issuer, however it is cluster-scoped and therefore can be referenced by resources that exist in *any* namespace, not just the same namespace as the referent. + type: object + required: + - spec + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Desired state of the ClusterIssuer resource. + type: object + properties: + acme: + description: ACME configures this issuer to communicate with a RFC8555 (ACME) server to obtain signed x509 certificates. + type: object + required: + - privateKeySecretRef + - server + properties: + caBundle: + description: Base64-encoded bundle of PEM CAs which can be used to validate the certificate chain presented by the ACME server. Mutually exclusive with SkipTLSVerify; prefer using CABundle to prevent various kinds of security vulnerabilities. If CABundle and SkipTLSVerify are unset, the system certificate bundle inside the container is used to validate the TLS connection. + type: string + format: byte + disableAccountKeyGeneration: + description: Enables or disables generating a new ACME account key. If true, the Issuer resource will *not* request a new account but will expect the account key to be supplied via an existing secret. If false, the cert-manager system will generate a new ACME account key for the Issuer. Defaults to false. + type: boolean + email: + description: Email is the email address to be associated with the ACME account. This field is optional, but it is strongly recommended to be set. It will be used to contact you in case of issues with your account or certificates, including expiry notification emails. This field may be updated after the account is initially registered. + type: string + enableDurationFeature: + description: Enables requesting a Not After date on certificates that matches the duration of the certificate. This is not supported by all ACME servers like Let's Encrypt. If set to true when the ACME server does not support it it will create an error on the Order. Defaults to false. + type: boolean + externalAccountBinding: + description: ExternalAccountBinding is a reference to a CA external account of the ACME server. If set, upon registration cert-manager will attempt to associate the given external account credentials with the registered ACME account. + type: object + required: + - keyID + - keySecretRef + properties: + keyAlgorithm: + description: 'Deprecated: keyAlgorithm field exists for historical compatibility reasons and should not be used. The algorithm is now hardcoded to HS256 in golang/x/crypto/acme.' + type: string + enum: + - HS256 + - HS384 + - HS512 + keyID: + description: keyID is the ID of the CA key that the External Account is bound to. + type: string + keySecretRef: + description: keySecretRef is a Secret Key Selector referencing a data item in a Kubernetes Secret which holds the symmetric MAC key of the External Account Binding. The `key` is the index string that is paired with the key data in the Secret and should not be confused with the key data itself, or indeed with the External Account Binding keyID above. The secret key stored in the Secret **must** be un-padded, base64 URL encoded data. + type: object + required: + - name + properties: + key: + description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + type: string + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + preferredChain: + description: 'PreferredChain is the chain to use if the ACME server outputs multiple. PreferredChain is no guarantee that this one gets delivered by the ACME endpoint. For example, for Let''s Encrypt''s DST crosssign you would use: "DST Root CA X3" or "ISRG Root X1" for the newer Let''s Encrypt root CA. This value picks the first certificate bundle in the ACME alternative chains that has a certificate with this value as its issuer''s CN' + type: string + maxLength: 64 + privateKeySecretRef: + description: PrivateKey is the name of a Kubernetes Secret resource that will be used to store the automatically generated ACME account private key. Optionally, a `key` may be specified to select a specific entry within the named Secret resource. If `key` is not specified, a default of `tls.key` will be used. + type: object + required: + - name + properties: + key: + description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + type: string + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + server: + description: 'Server is the URL used to access the ACME server''s ''directory'' endpoint. For example, for Let''s Encrypt''s staging endpoint, you would use: "https://acme-staging-v02.api.letsencrypt.org/directory". Only ACME v2 endpoints (i.e. RFC 8555) are supported.' + type: string + skipTLSVerify: + description: 'INSECURE: Enables or disables validation of the ACME server TLS certificate. If true, requests to the ACME server will not have the TLS certificate chain validated. Mutually exclusive with CABundle; prefer using CABundle to prevent various kinds of security vulnerabilities. Only enable this option in development environments. If CABundle and SkipTLSVerify are unset, the system certificate bundle inside the container is used to validate the TLS connection. Defaults to false.' + type: boolean + solvers: + description: 'Solvers is a list of challenge solvers that will be used to solve ACME challenges for the matching domains. Solver configurations must be provided in order to obtain certificates from an ACME server. For more information, see: https://cert-manager.io/docs/configuration/acme/' + type: array + items: + description: An ACMEChallengeSolver describes how to solve ACME challenges for the issuer it is part of. A selector may be provided to use different solving strategies for different DNS names. Only one of HTTP01 or DNS01 must be provided. + type: object + properties: + dns01: + description: Configures cert-manager to attempt to complete authorizations by performing the DNS01 challenge flow. + type: object + properties: + acmeDNS: + description: Use the 'ACME DNS' (https://github.com/joohoi/acme-dns) API to manage DNS01 challenge records. + type: object + required: + - accountSecretRef + - host + properties: + accountSecretRef: + description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. + type: object + required: + - name + properties: + key: + description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + type: string + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + host: + type: string + akamai: + description: Use the Akamai DNS zone management API to manage DNS01 challenge records. + type: object + required: + - accessTokenSecretRef + - clientSecretSecretRef + - clientTokenSecretRef + - serviceConsumerDomain + properties: + accessTokenSecretRef: + description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. + type: object + required: + - name + properties: + key: + description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + type: string + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + clientSecretSecretRef: + description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. + type: object + required: + - name + properties: + key: + description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + type: string + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + clientTokenSecretRef: + description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. + type: object + required: + - name + properties: + key: + description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + type: string + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + serviceConsumerDomain: + type: string + azureDNS: + description: Use the Microsoft Azure DNS API to manage DNS01 challenge records. + type: object + required: + - resourceGroupName + - subscriptionID + properties: + clientID: + description: if both this and ClientSecret are left unset MSI will be used + type: string + clientSecretSecretRef: + description: if both this and ClientID are left unset MSI will be used + type: object + required: + - name + properties: + key: + description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + type: string + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + environment: + description: name of the Azure environment (default AzurePublicCloud) + type: string + enum: + - AzurePublicCloud + - AzureChinaCloud + - AzureGermanCloud + - AzureUSGovernmentCloud + hostedZoneName: + description: name of the DNS zone that should be used + type: string + managedIdentity: + description: managed identity configuration, can not be used at the same time as clientID, clientSecretSecretRef or tenantID + type: object + properties: + clientID: + description: client ID of the managed identity, can not be used at the same time as resourceID + type: string + resourceID: + description: resource ID of the managed identity, can not be used at the same time as clientID + type: string + resourceGroupName: + description: resource group the DNS zone is located in + type: string + subscriptionID: + description: ID of the Azure subscription + type: string + tenantID: + description: when specifying ClientID and ClientSecret then this field is also needed + type: string + cloudDNS: + description: Use the Google Cloud DNS API to manage DNS01 challenge records. + type: object + required: + - project + properties: + hostedZoneName: + description: HostedZoneName is an optional field that tells cert-manager in which Cloud DNS zone the challenge record has to be created. If left empty cert-manager will automatically choose a zone. + type: string + project: + type: string + serviceAccountSecretRef: + description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. + type: object + required: + - name + properties: + key: + description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + type: string + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + cloudflare: + description: Use the Cloudflare API to manage DNS01 challenge records. + type: object + properties: + apiKeySecretRef: + description: 'API key to use to authenticate with Cloudflare. Note: using an API token to authenticate is now the recommended method as it allows greater control of permissions.' + type: object + required: + - name + properties: + key: + description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + type: string + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + apiTokenSecretRef: + description: API token used to authenticate with Cloudflare. + type: object + required: + - name + properties: + key: + description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + type: string + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + email: + description: Email of the account, only required when using API key based authentication. + type: string + cnameStrategy: + description: CNAMEStrategy configures how the DNS01 provider should handle CNAME records when found in DNS zones. + type: string + enum: + - None + - Follow + digitalocean: + description: Use the DigitalOcean DNS API to manage DNS01 challenge records. + type: object + required: + - tokenSecretRef + properties: + tokenSecretRef: + description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. + type: object + required: + - name + properties: + key: + description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + type: string + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + rfc2136: + description: Use RFC2136 ("Dynamic Updates in the Domain Name System") (https://datatracker.ietf.org/doc/rfc2136/) to manage DNS01 challenge records. + type: object + required: + - nameserver + properties: + nameserver: + description: The IP address or hostname of an authoritative DNS server supporting RFC2136 in the form host:port. If the host is an IPv6 address it must be enclosed in square brackets (e.g [2001:db8::1]) ; port is optional. This field is required. + type: string + tsigAlgorithm: + description: 'The TSIG Algorithm configured in the DNS supporting RFC2136. Used only when ``tsigSecretSecretRef`` and ``tsigKeyName`` are defined. Supported values are (case-insensitive): ``HMACMD5`` (default), ``HMACSHA1``, ``HMACSHA256`` or ``HMACSHA512``.' + type: string + tsigKeyName: + description: The TSIG Key name configured in the DNS. If ``tsigSecretSecretRef`` is defined, this field is required. + type: string + tsigSecretSecretRef: + description: The name of the secret containing the TSIG value. If ``tsigKeyName`` is defined, this field is required. + type: object + required: + - name + properties: + key: + description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + type: string + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + route53: + description: Use the AWS Route53 API to manage DNS01 challenge records. + type: object + required: + - region + properties: + accessKeyID: + description: 'The AccessKeyID is used for authentication. Cannot be set when SecretAccessKeyID is set. If neither the Access Key nor Key ID are set, we fall-back to using env vars, shared credentials file or AWS Instance metadata, see: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials' + type: string + accessKeyIDSecretRef: + description: 'The SecretAccessKey is used for authentication. If set, pull the AWS access key ID from a key within a Kubernetes Secret. Cannot be set when AccessKeyID is set. If neither the Access Key nor Key ID are set, we fall-back to using env vars, shared credentials file or AWS Instance metadata, see: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials' + type: object + required: + - name + properties: + key: + description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + type: string + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + hostedZoneID: + description: If set, the provider will manage only this zone in Route53 and will not do an lookup using the route53:ListHostedZonesByName api call. + type: string + region: + description: Always set the region when using AccessKeyID and SecretAccessKey + type: string + role: + description: Role is a Role ARN which the Route53 provider will assume using either the explicit credentials AccessKeyID/SecretAccessKey or the inferred credentials from environment variables, shared credentials file or AWS Instance metadata + type: string + secretAccessKeySecretRef: + description: 'The SecretAccessKey is used for authentication. If neither the Access Key nor Key ID are set, we fall-back to using env vars, shared credentials file or AWS Instance metadata, see: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials' + type: object + required: + - name + properties: + key: + description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + type: string + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + webhook: + description: Configure an external webhook based DNS01 challenge solver to manage DNS01 challenge records. + type: object + required: + - groupName + - solverName + properties: + config: + description: Additional configuration that should be passed to the webhook apiserver when challenges are processed. This can contain arbitrary JSON data. Secret values should not be specified in this stanza. If secret values are needed (e.g. credentials for a DNS service), you should use a SecretKeySelector to reference a Secret resource. For details on the schema of this field, consult the webhook provider implementation's documentation. + x-kubernetes-preserve-unknown-fields: true + groupName: + description: The API group name that should be used when POSTing ChallengePayload resources to the webhook apiserver. This should be the same as the GroupName specified in the webhook provider implementation. + type: string + solverName: + description: The name of the solver to use, as defined in the webhook provider implementation. This will typically be the name of the provider, e.g. 'cloudflare'. + type: string + http01: + description: Configures cert-manager to attempt to complete authorizations by performing the HTTP01 challenge flow. It is not possible to obtain certificates for wildcard domain names (e.g. `*.example.com`) using the HTTP01 challenge mechanism. + type: object + properties: + gatewayHTTPRoute: + description: The Gateway API is a sig-network community API that models service networking in Kubernetes (https://gateway-api.sigs.k8s.io/). The Gateway solver will create HTTPRoutes with the specified labels in the same namespace as the challenge. This solver is experimental, and fields / behaviour may change in the future. + type: object + properties: + labels: + description: Custom labels that will be applied to HTTPRoutes created by cert-manager while solving HTTP-01 challenges. + type: object + additionalProperties: + type: string + parentRefs: + description: 'When solving an HTTP-01 challenge, cert-manager creates an HTTPRoute. cert-manager needs to know which parentRefs should be used when creating the HTTPRoute. Usually, the parentRef references a Gateway. See: https://gateway-api.sigs.k8s.io/api-types/httproute/#attaching-to-gateways' + type: array + items: + description: "ParentReference identifies an API object (usually a Gateway) that can be considered a parent of this resource (usually a route). There are two kinds of parent resources with \"Core\" support: \n * Gateway (Gateway conformance profile) * Service (Mesh conformance profile, experimental, ClusterIP Services only) \n This API may be extended in the future to support additional kinds of parent resources. \n The API object must be valid in the cluster; the Group and Kind must be registered in the cluster for this reference to be valid." + type: object + required: + - name + properties: + group: + description: "Group is the group of the referent. When unspecified, \"gateway.networking.k8s.io\" is inferred. To set the core API group (such as for a \"Service\" kind referent), Group must be explicitly set to \"\" (empty string). \n Support: Core" + type: string + default: gateway.networking.k8s.io + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + kind: + description: "Kind is kind of the referent. \n There are two kinds of parent resources with \"Core\" support: \n * Gateway (Gateway conformance profile) * Service (Mesh conformance profile, experimental, ClusterIP Services only) \n Support for other resources is Implementation-Specific." + type: string + default: Gateway + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + name: + description: "Name is the name of the referent. \n Support: Core" + type: string + maxLength: 253 + minLength: 1 + namespace: + description: "Namespace is the namespace of the referent. When unspecified, this refers to the local namespace of the Route. \n Note that there are specific rules for ParentRefs which cross namespace boundaries. Cross-namespace references are only valid if they are explicitly allowed by something in the namespace they are referring to. For example: Gateway has the AllowedRoutes field, and ReferenceGrant provides a generic way to enable any other kind of cross-namespace reference. \n ParentRefs from a Route to a Service in the same namespace are \"producer\" routes, which apply default routing rules to inbound connections from any namespace to the Service. \n ParentRefs from a Route to a Service in a different namespace are \"consumer\" routes, and these routing rules are only applied to outbound connections originating from the same namespace as the Route, for which the intended destination of the connections are a Service targeted as a ParentRef of the Route. \n Support: Core" + type: string + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + port: + description: "Port is the network port this Route targets. It can be interpreted differently based on the type of parent resource. \n When the parent resource is a Gateway, this targets all listeners listening on the specified port that also support this kind of Route(and select this Route). It's not recommended to set `Port` unless the networking behaviors specified in a Route must apply to a specific port as opposed to a listener(s) whose port(s) may be changed. When both Port and SectionName are specified, the name and port of the selected listener must match both specified values. \n When the parent resource is a Service, this targets a specific port in the Service spec. When both Port (experimental) and SectionName are specified, the name and port of the selected port must match both specified values. \n Implementations MAY choose to support other parent resources. Implementations supporting other types of parent resources MUST clearly document how/if Port is interpreted. \n For the purpose of status, an attachment is considered successful as long as the parent resource accepts it partially. For example, Gateway listeners can restrict which Routes can attach to them by Route kind, namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from the referencing Route, the Route MUST be considered successfully attached. If no Gateway listeners accept attachment from this Route, the Route MUST be considered detached from the Gateway. \n Support: Extended \n " + type: integer + format: int32 + maximum: 65535 + minimum: 1 + sectionName: + description: "SectionName is the name of a section within the target resource. In the following resources, SectionName is interpreted as the following: \n * Gateway: Listener Name. When both Port (experimental) and SectionName are specified, the name and port of the selected listener must match both specified values. * Service: Port Name. When both Port (experimental) and SectionName are specified, the name and port of the selected listener must match both specified values. Note that attaching Routes to Services as Parents is part of experimental Mesh support and is not supported for any other purpose. \n Implementations MAY choose to support attaching Routes to other resources. If that is the case, they MUST clearly document how SectionName is interpreted. \n When unspecified (empty string), this will reference the entire resource. For the purpose of status, an attachment is considered successful if at least one section in the parent resource accepts it. For example, Gateway listeners can restrict which Routes can attach to them by Route kind, namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from the referencing Route, the Route MUST be considered successfully attached. If no Gateway listeners accept attachment from this Route, the Route MUST be considered detached from the Gateway. \n Support: Core" + type: string + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + serviceType: + description: Optional service type for Kubernetes solver service. Supported values are NodePort or ClusterIP. If unset, defaults to NodePort. + type: string + ingress: + description: The ingress based HTTP01 challenge solver will solve challenges by creating or modifying Ingress resources in order to route requests for '/.well-known/acme-challenge/XYZ' to 'challenge solver' pods that are provisioned by cert-manager for each Challenge to be completed. + type: object + properties: + class: + description: This field configures the annotation `kubernetes.io/ingress.class` when creating Ingress resources to solve ACME challenges that use this challenge solver. Only one of `class`, `name` or `ingressClassName` may be specified. + type: string + ingressClassName: + description: This field configures the field `ingressClassName` on the created Ingress resources used to solve ACME challenges that use this challenge solver. This is the recommended way of configuring the ingress class. Only one of `class`, `name` or `ingressClassName` may be specified. + type: string + ingressTemplate: + description: Optional ingress template used to configure the ACME challenge solver ingress used for HTTP01 challenges. + type: object + properties: + metadata: + description: ObjectMeta overrides for the ingress used to solve HTTP01 challenges. Only the 'labels' and 'annotations' fields may be set. If labels or annotations overlap with in-built values, the values here will override the in-built values. + type: object + properties: + annotations: + description: Annotations that should be added to the created ACME HTTP01 solver ingress. + type: object + additionalProperties: + type: string + labels: + description: Labels that should be added to the created ACME HTTP01 solver ingress. + type: object + additionalProperties: + type: string + name: + description: The name of the ingress resource that should have ACME challenge solving routes inserted into it in order to solve HTTP01 challenges. This is typically used in conjunction with ingress controllers like ingress-gce, which maintains a 1:1 mapping between external IPs and ingress resources. Only one of `class`, `name` or `ingressClassName` may be specified. + type: string + podTemplate: + description: Optional pod template used to configure the ACME challenge solver pods used for HTTP01 challenges. + type: object + properties: + metadata: + description: ObjectMeta overrides for the pod used to solve HTTP01 challenges. Only the 'labels' and 'annotations' fields may be set. If labels or annotations overlap with in-built values, the values here will override the in-built values. + type: object + properties: + annotations: + description: Annotations that should be added to the create ACME HTTP01 solver pods. + type: object + additionalProperties: + type: string + labels: + description: Labels that should be added to the created ACME HTTP01 solver pods. + type: object + additionalProperties: + type: string + spec: + description: PodSpec defines overrides for the HTTP01 challenge solver pod. Check ACMEChallengeSolverHTTP01IngressPodSpec to find out currently supported fields. All other fields will be ignored. + type: object + properties: + affinity: + description: If specified, the pod's scheduling constraints + type: object + properties: + nodeAffinity: + description: Describes node affinity scheduling rules for the pod. + type: object + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node matches the corresponding matchExpressions; the node(s) with the highest sum are the most preferred. + type: array + items: + description: An empty preferred scheduling term matches all objects with implicit weight 0 (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + type: object + required: + - preference + - weight + properties: + preference: + description: A node selector term, associated with the corresponding weight. + type: object + properties: + matchExpressions: + description: A list of node selector requirements by node's labels. + type: array + items: + description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. + type: array + items: + type: string + matchFields: + description: A list of node selector requirements by node's fields. + type: array + items: + description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. + type: array + items: + type: string + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. + type: integer + format: int32 + requiredDuringSchedulingIgnoredDuringExecution: + description: If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to an update), the system may or may not try to eventually evict the pod from its node. + type: object + required: + - nodeSelectorTerms + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. The terms are ORed. + type: array + items: + description: A null or empty node selector term matches no objects. The requirements of them are ANDed. The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + type: object + properties: + matchExpressions: + description: A list of node selector requirements by node's labels. + type: array + items: + description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. + type: array + items: + type: string + matchFields: + description: A list of node selector requirements by node's fields. + type: array + items: + description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. + type: array + items: + type: string + x-kubernetes-map-type: atomic + x-kubernetes-map-type: atomic + podAffinity: + description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). + type: object + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. + type: array + items: + description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + type: object + required: + - podAffinityTerm + - weight + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated with the corresponding weight. + type: object + required: + - topologyKey + properties: + labelSelector: + description: A label query over a set of resources, in this case pods. + type: object + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + type: array + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + type: array + items: + type: string + matchLabels: + description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + additionalProperties: + type: string + x-kubernetes-map-type: atomic + namespaceSelector: + description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. + type: object + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + type: array + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + type: array + items: + type: string + matchLabels: + description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + additionalProperties: + type: string + x-kubernetes-map-type: atomic + namespaces: + description: namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace". + type: array + items: + type: string + topologyKey: + description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. + type: string + weight: + description: weight associated with matching the corresponding podAffinityTerm, in the range 1-100. + type: integer + format: int32 + requiredDuringSchedulingIgnoredDuringExecution: + description: If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied. + type: array + items: + description: Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running + type: object + required: + - topologyKey + properties: + labelSelector: + description: A label query over a set of resources, in this case pods. + type: object + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + type: array + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + type: array + items: + type: string + matchLabels: + description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + additionalProperties: + type: string + x-kubernetes-map-type: atomic + namespaceSelector: + description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. + type: object + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + type: array + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + type: array + items: + type: string + matchLabels: + description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + additionalProperties: + type: string + x-kubernetes-map-type: atomic + namespaces: + description: namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace". + type: array + items: + type: string + topologyKey: + description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. + type: string + podAntiAffinity: + description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). + type: object + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule pods to nodes that satisfy the anti-affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. + type: array + items: + description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + type: object + required: + - podAffinityTerm + - weight + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated with the corresponding weight. + type: object + required: + - topologyKey + properties: + labelSelector: + description: A label query over a set of resources, in this case pods. + type: object + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + type: array + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + type: array + items: + type: string + matchLabels: + description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + additionalProperties: + type: string + x-kubernetes-map-type: atomic + namespaceSelector: + description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. + type: object + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + type: array + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + type: array + items: + type: string + matchLabels: + description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + additionalProperties: + type: string + x-kubernetes-map-type: atomic + namespaces: + description: namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace". + type: array + items: + type: string + topologyKey: + description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. + type: string + weight: + description: weight associated with matching the corresponding podAffinityTerm, in the range 1-100. + type: integer + format: int32 + requiredDuringSchedulingIgnoredDuringExecution: + description: If the anti-affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the anti-affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied. + type: array + items: + description: Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running + type: object + required: + - topologyKey + properties: + labelSelector: + description: A label query over a set of resources, in this case pods. + type: object + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + type: array + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + type: array + items: + type: string + matchLabels: + description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + additionalProperties: + type: string + x-kubernetes-map-type: atomic + namespaceSelector: + description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. + type: object + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + type: array + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + type: array + items: + type: string + matchLabels: + description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + additionalProperties: + type: string + x-kubernetes-map-type: atomic + namespaces: + description: namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace". + type: array + items: + type: string + topologyKey: + description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. + type: string + imagePullSecrets: + description: If specified, the pod's imagePullSecrets + type: array + items: + description: LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace. + type: object + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + x-kubernetes-map-type: atomic + nodeSelector: + description: 'NodeSelector is a selector which must be true for the pod to fit on a node. Selector which must match a node''s labels for the pod to be scheduled on that node. More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/' + type: object + additionalProperties: + type: string + priorityClassName: + description: If specified, the pod's priorityClassName. + type: string + serviceAccountName: + description: If specified, the pod's service account + type: string + tolerations: + description: If specified, the pod's tolerations. + type: array + items: + description: The pod this Toleration is attached to tolerates any taint that matches the triple using the matching operator . + type: object + properties: + effect: + description: Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Key is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: Operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: TolerationSeconds represents the period of time the toleration (which must be of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) by the system. + type: integer + format: int64 + value: + description: Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + serviceType: + description: Optional service type for Kubernetes solver service. Supported values are NodePort or ClusterIP. If unset, defaults to NodePort. + type: string + selector: + description: Selector selects a set of DNSNames on the Certificate resource that should be solved using this challenge solver. If not specified, the solver will be treated as the 'default' solver with the lowest priority, i.e. if any other solver has a more specific match, it will be used instead. + type: object + properties: + dnsNames: + description: List of DNSNames that this solver will be used to solve. If specified and a match is found, a dnsNames selector will take precedence over a dnsZones selector. If multiple solvers match with the same dnsNames value, the solver with the most matching labels in matchLabels will be selected. If neither has more matches, the solver defined earlier in the list will be selected. + type: array + items: + type: string + dnsZones: + description: List of DNSZones that this solver will be used to solve. The most specific DNS zone match specified here will take precedence over other DNS zone matches, so a solver specifying sys.example.com will be selected over one specifying example.com for the domain www.sys.example.com. If multiple solvers match with the same dnsZones value, the solver with the most matching labels in matchLabels will be selected. If neither has more matches, the solver defined earlier in the list will be selected. + type: array + items: + type: string + matchLabels: + description: A label selector that is used to refine the set of certificate's that this challenge solver will apply to. + type: object + additionalProperties: + type: string + ca: + description: CA configures this issuer to sign certificates using a signing CA keypair stored in a Secret resource. This is used to build internal PKIs that are managed by cert-manager. + type: object + required: + - secretName + properties: + crlDistributionPoints: + description: The CRL distribution points is an X.509 v3 certificate extension which identifies the location of the CRL from which the revocation of this certificate can be checked. If not set, certificates will be issued without distribution points set. + type: array + items: + type: string + ocspServers: + description: The OCSP server list is an X.509 v3 extension that defines a list of URLs of OCSP responders. The OCSP responders can be queried for the revocation status of an issued certificate. If not set, the certificate will be issued with no OCSP servers set. For example, an OCSP server URL could be "http://ocsp.int-x3.letsencrypt.org". + type: array + items: + type: string + secretName: + description: SecretName is the name of the secret used to sign Certificates issued by this Issuer. + type: string + selfSigned: + description: SelfSigned configures this issuer to 'self sign' certificates using the private key used to create the CertificateRequest object. + type: object + properties: + crlDistributionPoints: + description: The CRL distribution points is an X.509 v3 certificate extension which identifies the location of the CRL from which the revocation of this certificate can be checked. If not set certificate will be issued without CDP. Values are strings. + type: array + items: + type: string + vault: + description: Vault configures this issuer to sign certificates using a HashiCorp Vault PKI backend. + type: object + required: + - auth + - path + - server + properties: + auth: + description: Auth configures how cert-manager authenticates with the Vault server. + type: object + properties: + appRole: + description: AppRole authenticates with Vault using the App Role auth mechanism, with the role and secret stored in a Kubernetes Secret resource. + type: object + required: + - path + - roleId + - secretRef + properties: + path: + description: 'Path where the App Role authentication backend is mounted in Vault, e.g: "approle"' + type: string + roleId: + description: RoleID configured in the App Role authentication backend when setting up the authentication backend in Vault. + type: string + secretRef: + description: Reference to a key in a Secret that contains the App Role secret used to authenticate with Vault. The `key` field must be specified and denotes which entry within the Secret resource is used as the app role secret. + type: object + required: + - name + properties: + key: + description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + type: string + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + kubernetes: + description: Kubernetes authenticates with Vault by passing the ServiceAccount token stored in the named Secret resource to the Vault server. + type: object + required: + - role + properties: + mountPath: + description: The Vault mountPath here is the mount path to use when authenticating with Vault. For example, setting a value to `/v1/auth/foo`, will use the path `/v1/auth/foo/login` to authenticate with Vault. If unspecified, the default value "/v1/auth/kubernetes" will be used. + type: string + role: + description: A required field containing the Vault Role to assume. A Role binds a Kubernetes ServiceAccount with a set of Vault policies. + type: string + secretRef: + description: The required Secret field containing a Kubernetes ServiceAccount JWT used for authenticating with Vault. Use of 'ambient credentials' is not supported. + type: object + required: + - name + properties: + key: + description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + type: string + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + serviceAccountRef: + description: A reference to a service account that will be used to request a bound token (also known as "projected token"). Compared to using "secretRef", using this field means that you don't rely on statically bound tokens. To use this field, you must configure an RBAC rule to let cert-manager request a token. + type: object + required: + - name + properties: + name: + description: Name of the ServiceAccount used to request a token. + type: string + tokenSecretRef: + description: TokenSecretRef authenticates with Vault by presenting a token. + type: object + required: + - name + properties: + key: + description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + type: string + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + caBundle: + description: Base64-encoded bundle of PEM CAs which will be used to validate the certificate chain presented by Vault. Only used if using HTTPS to connect to Vault and ignored for HTTP connections. Mutually exclusive with CABundleSecretRef. If neither CABundle nor CABundleSecretRef are defined, the certificate bundle in the cert-manager controller container is used to validate the TLS connection. + type: string + format: byte + caBundleSecretRef: + description: Reference to a Secret containing a bundle of PEM-encoded CAs to use when verifying the certificate chain presented by Vault when using HTTPS. Mutually exclusive with CABundle. If neither CABundle nor CABundleSecretRef are defined, the certificate bundle in the cert-manager controller container is used to validate the TLS connection. If no key for the Secret is specified, cert-manager will default to 'ca.crt'. + type: object + required: + - name + properties: + key: + description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + type: string + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Name of the vault namespace. Namespaces is a set of features within Vault Enterprise that allows Vault environments to support Secure Multi-tenancy. e.g: "ns1" More about namespaces can be found here https://www.vaultproject.io/docs/enterprise/namespaces' + type: string + path: + description: 'Path is the mount path of the Vault PKI backend''s `sign` endpoint, e.g: "my_pki_mount/sign/my-role-name".' + type: string + server: + description: 'Server is the connection address for the Vault server, e.g: "https://vault.example.com:8200".' + type: string + venafi: + description: Venafi configures this issuer to sign certificates using a Venafi TPP or Venafi Cloud policy zone. + type: object + required: + - zone + properties: + cloud: + description: Cloud specifies the Venafi cloud configuration settings. Only one of TPP or Cloud may be specified. + type: object + required: + - apiTokenSecretRef + properties: + apiTokenSecretRef: + description: APITokenSecretRef is a secret key selector for the Venafi Cloud API token. + type: object + required: + - name + properties: + key: + description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + type: string + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + url: + description: URL is the base URL for Venafi Cloud. Defaults to "https://api.venafi.cloud/v1". + type: string + tpp: + description: TPP specifies Trust Protection Platform configuration settings. Only one of TPP or Cloud may be specified. + type: object + required: + - credentialsRef + - url + properties: + caBundle: + description: Base64-encoded bundle of PEM CAs which will be used to validate the certificate chain presented by the TPP server. Only used if using HTTPS; ignored for HTTP. If undefined, the certificate bundle in the cert-manager controller container is used to validate the chain. + type: string + format: byte + credentialsRef: + description: CredentialsRef is a reference to a Secret containing the username and password for the TPP server. The secret must contain two keys, 'username' and 'password'. + type: object + required: + - name + properties: + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + url: + description: 'URL is the base URL for the vedsdk endpoint of the Venafi TPP instance, for example: "https://tpp.example.com/vedsdk".' + type: string + zone: + description: Zone is the Venafi Policy Zone to use for this issuer. All requests made to the Venafi platform will be restricted by the named zone policy. This field is required. + type: string + status: + description: Status of the ClusterIssuer. This is set and managed automatically. + type: object + properties: + acme: + description: ACME specific status options. This field should only be set if the Issuer is configured to use an ACME server to issue certificates. + type: object + properties: + lastPrivateKeyHash: + description: LastPrivateKeyHash is a hash of the private key associated with the latest registered ACME account, in order to track changes made to registered account associated with the Issuer + type: string + lastRegisteredEmail: + description: LastRegisteredEmail is the email associated with the latest registered ACME account, in order to track changes made to registered account associated with the Issuer + type: string + uri: + description: URI is the unique account identifier, which can also be used to retrieve account details from the CA + type: string + conditions: + description: List of status conditions to indicate the status of a CertificateRequest. Known condition types are `Ready`. + type: array + items: + description: IssuerCondition contains condition information for an Issuer. + type: object + required: + - status + - type + properties: + lastTransitionTime: + description: LastTransitionTime is the timestamp corresponding to the last status change of this condition. + type: string + format: date-time + message: + description: Message is a human readable description of the details of the last transition, complementing reason. + type: string + observedGeneration: + description: If set, this represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.condition[x].observedGeneration is 9, the condition is out of date with respect to the current state of the Issuer. + type: integer + format: int64 + reason: + description: Reason is a brief machine readable explanation for the condition's last transition. + type: string + status: + description: Status of the condition, one of (`True`, `False`, `Unknown`). + type: string + enum: + - "True" + - "False" + - Unknown + type: + description: Type of the condition, known values are (`Ready`). + type: string + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + served: true + storage: true +--- +# Source: cert-manager/deploy/crds/crd-challenges.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: challenges.acme.cert-manager.io + labels: + app: 'cert-manager' + app.kubernetes.io/name: 'cert-manager' + app.kubernetes.io/instance: 'cert-manager' + # Generated labels + app.kubernetes.io/version: "{{ cert_manager_version }}" +spec: + group: acme.cert-manager.io + names: + kind: Challenge + listKind: ChallengeList + plural: challenges + singular: challenge + categories: + - cert-manager + - cert-manager-acme + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.state + name: State + type: string + - jsonPath: .spec.dnsName + name: Domain + type: string + - jsonPath: .status.reason + name: Reason + priority: 1 + type: string + - description: CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC. + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: Challenge is a type to represent a Challenge request with an ACME server + type: object + required: + - metadata + - spec + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + type: object + required: + - authorizationURL + - dnsName + - issuerRef + - key + - solver + - token + - type + - url + properties: + authorizationURL: + description: The URL to the ACME Authorization resource that this challenge is a part of. + type: string + dnsName: + description: dnsName is the identifier that this challenge is for, e.g. example.com. If the requested DNSName is a 'wildcard', this field MUST be set to the non-wildcard domain, e.g. for `*.example.com`, it must be `example.com`. + type: string + issuerRef: + description: References a properly configured ACME-type Issuer which should be used to create this Challenge. If the Issuer does not exist, processing will be retried. If the Issuer is not an 'ACME' Issuer, an error will be returned and the Challenge will be marked as failed. + type: object + required: + - name + properties: + group: + description: Group of the resource being referred to. + type: string + kind: + description: Kind of the resource being referred to. + type: string + name: + description: Name of the resource being referred to. + type: string + key: + description: 'The ACME challenge key for this challenge For HTTP01 challenges, this is the value that must be responded with to complete the HTTP01 challenge in the format: `.`. For DNS01 challenges, this is the base64 encoded SHA256 sum of the `.` text that must be set as the TXT record content.' + type: string + solver: + description: Contains the domain solving configuration that should be used to solve this challenge resource. + type: object + properties: + dns01: + description: Configures cert-manager to attempt to complete authorizations by performing the DNS01 challenge flow. + type: object + properties: + acmeDNS: + description: Use the 'ACME DNS' (https://github.com/joohoi/acme-dns) API to manage DNS01 challenge records. + type: object + required: + - accountSecretRef + - host + properties: + accountSecretRef: + description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. + type: object + required: + - name + properties: + key: + description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + type: string + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + host: + type: string + akamai: + description: Use the Akamai DNS zone management API to manage DNS01 challenge records. + type: object + required: + - accessTokenSecretRef + - clientSecretSecretRef + - clientTokenSecretRef + - serviceConsumerDomain + properties: + accessTokenSecretRef: + description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. + type: object + required: + - name + properties: + key: + description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + type: string + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + clientSecretSecretRef: + description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. + type: object + required: + - name + properties: + key: + description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + type: string + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + clientTokenSecretRef: + description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. + type: object + required: + - name + properties: + key: + description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + type: string + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + serviceConsumerDomain: + type: string + azureDNS: + description: Use the Microsoft Azure DNS API to manage DNS01 challenge records. + type: object + required: + - resourceGroupName + - subscriptionID + properties: + clientID: + description: if both this and ClientSecret are left unset MSI will be used + type: string + clientSecretSecretRef: + description: if both this and ClientID are left unset MSI will be used + type: object + required: + - name + properties: + key: + description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + type: string + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + environment: + description: name of the Azure environment (default AzurePublicCloud) + type: string + enum: + - AzurePublicCloud + - AzureChinaCloud + - AzureGermanCloud + - AzureUSGovernmentCloud + hostedZoneName: + description: name of the DNS zone that should be used + type: string + managedIdentity: + description: managed identity configuration, can not be used at the same time as clientID, clientSecretSecretRef or tenantID + type: object + properties: + clientID: + description: client ID of the managed identity, can not be used at the same time as resourceID + type: string + resourceID: + description: resource ID of the managed identity, can not be used at the same time as clientID + type: string + resourceGroupName: + description: resource group the DNS zone is located in + type: string + subscriptionID: + description: ID of the Azure subscription + type: string + tenantID: + description: when specifying ClientID and ClientSecret then this field is also needed + type: string + cloudDNS: + description: Use the Google Cloud DNS API to manage DNS01 challenge records. + type: object + required: + - project + properties: + hostedZoneName: + description: HostedZoneName is an optional field that tells cert-manager in which Cloud DNS zone the challenge record has to be created. If left empty cert-manager will automatically choose a zone. + type: string + project: + type: string + serviceAccountSecretRef: + description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. + type: object + required: + - name + properties: + key: + description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + type: string + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + cloudflare: + description: Use the Cloudflare API to manage DNS01 challenge records. + type: object + properties: + apiKeySecretRef: + description: 'API key to use to authenticate with Cloudflare. Note: using an API token to authenticate is now the recommended method as it allows greater control of permissions.' + type: object + required: + - name + properties: + key: + description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + type: string + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + apiTokenSecretRef: + description: API token used to authenticate with Cloudflare. + type: object + required: + - name + properties: + key: + description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + type: string + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + email: + description: Email of the account, only required when using API key based authentication. + type: string + cnameStrategy: + description: CNAMEStrategy configures how the DNS01 provider should handle CNAME records when found in DNS zones. + type: string + enum: + - None + - Follow + digitalocean: + description: Use the DigitalOcean DNS API to manage DNS01 challenge records. + type: object + required: + - tokenSecretRef + properties: + tokenSecretRef: + description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. + type: object + required: + - name + properties: + key: + description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + type: string + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + rfc2136: + description: Use RFC2136 ("Dynamic Updates in the Domain Name System") (https://datatracker.ietf.org/doc/rfc2136/) to manage DNS01 challenge records. + type: object + required: + - nameserver + properties: + nameserver: + description: The IP address or hostname of an authoritative DNS server supporting RFC2136 in the form host:port. If the host is an IPv6 address it must be enclosed in square brackets (e.g [2001:db8::1]) ; port is optional. This field is required. + type: string + tsigAlgorithm: + description: 'The TSIG Algorithm configured in the DNS supporting RFC2136. Used only when ``tsigSecretSecretRef`` and ``tsigKeyName`` are defined. Supported values are (case-insensitive): ``HMACMD5`` (default), ``HMACSHA1``, ``HMACSHA256`` or ``HMACSHA512``.' + type: string + tsigKeyName: + description: The TSIG Key name configured in the DNS. If ``tsigSecretSecretRef`` is defined, this field is required. + type: string + tsigSecretSecretRef: + description: The name of the secret containing the TSIG value. If ``tsigKeyName`` is defined, this field is required. + type: object + required: + - name + properties: + key: + description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + type: string + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + route53: + description: Use the AWS Route53 API to manage DNS01 challenge records. + type: object + required: + - region + properties: + accessKeyID: + description: 'The AccessKeyID is used for authentication. Cannot be set when SecretAccessKeyID is set. If neither the Access Key nor Key ID are set, we fall-back to using env vars, shared credentials file or AWS Instance metadata, see: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials' + type: string + accessKeyIDSecretRef: + description: 'The SecretAccessKey is used for authentication. If set, pull the AWS access key ID from a key within a Kubernetes Secret. Cannot be set when AccessKeyID is set. If neither the Access Key nor Key ID are set, we fall-back to using env vars, shared credentials file or AWS Instance metadata, see: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials' + type: object + required: + - name + properties: + key: + description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + type: string + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + hostedZoneID: + description: If set, the provider will manage only this zone in Route53 and will not do an lookup using the route53:ListHostedZonesByName api call. + type: string + region: + description: Always set the region when using AccessKeyID and SecretAccessKey + type: string + role: + description: Role is a Role ARN which the Route53 provider will assume using either the explicit credentials AccessKeyID/SecretAccessKey or the inferred credentials from environment variables, shared credentials file or AWS Instance metadata + type: string + secretAccessKeySecretRef: + description: 'The SecretAccessKey is used for authentication. If neither the Access Key nor Key ID are set, we fall-back to using env vars, shared credentials file or AWS Instance metadata, see: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials' + type: object + required: + - name + properties: + key: + description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + type: string + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + webhook: + description: Configure an external webhook based DNS01 challenge solver to manage DNS01 challenge records. + type: object + required: + - groupName + - solverName + properties: + config: + description: Additional configuration that should be passed to the webhook apiserver when challenges are processed. This can contain arbitrary JSON data. Secret values should not be specified in this stanza. If secret values are needed (e.g. credentials for a DNS service), you should use a SecretKeySelector to reference a Secret resource. For details on the schema of this field, consult the webhook provider implementation's documentation. + x-kubernetes-preserve-unknown-fields: true + groupName: + description: The API group name that should be used when POSTing ChallengePayload resources to the webhook apiserver. This should be the same as the GroupName specified in the webhook provider implementation. + type: string + solverName: + description: The name of the solver to use, as defined in the webhook provider implementation. This will typically be the name of the provider, e.g. 'cloudflare'. + type: string + http01: + description: Configures cert-manager to attempt to complete authorizations by performing the HTTP01 challenge flow. It is not possible to obtain certificates for wildcard domain names (e.g. `*.example.com`) using the HTTP01 challenge mechanism. + type: object + properties: + gatewayHTTPRoute: + description: The Gateway API is a sig-network community API that models service networking in Kubernetes (https://gateway-api.sigs.k8s.io/). The Gateway solver will create HTTPRoutes with the specified labels in the same namespace as the challenge. This solver is experimental, and fields / behaviour may change in the future. + type: object + properties: + labels: + description: Custom labels that will be applied to HTTPRoutes created by cert-manager while solving HTTP-01 challenges. + type: object + additionalProperties: + type: string + parentRefs: + description: 'When solving an HTTP-01 challenge, cert-manager creates an HTTPRoute. cert-manager needs to know which parentRefs should be used when creating the HTTPRoute. Usually, the parentRef references a Gateway. See: https://gateway-api.sigs.k8s.io/api-types/httproute/#attaching-to-gateways' + type: array + items: + description: "ParentReference identifies an API object (usually a Gateway) that can be considered a parent of this resource (usually a route). There are two kinds of parent resources with \"Core\" support: \n * Gateway (Gateway conformance profile) * Service (Mesh conformance profile, experimental, ClusterIP Services only) \n This API may be extended in the future to support additional kinds of parent resources. \n The API object must be valid in the cluster; the Group and Kind must be registered in the cluster for this reference to be valid." + type: object + required: + - name + properties: + group: + description: "Group is the group of the referent. When unspecified, \"gateway.networking.k8s.io\" is inferred. To set the core API group (such as for a \"Service\" kind referent), Group must be explicitly set to \"\" (empty string). \n Support: Core" + type: string + default: gateway.networking.k8s.io + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + kind: + description: "Kind is kind of the referent. \n There are two kinds of parent resources with \"Core\" support: \n * Gateway (Gateway conformance profile) * Service (Mesh conformance profile, experimental, ClusterIP Services only) \n Support for other resources is Implementation-Specific." + type: string + default: Gateway + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + name: + description: "Name is the name of the referent. \n Support: Core" + type: string + maxLength: 253 + minLength: 1 + namespace: + description: "Namespace is the namespace of the referent. When unspecified, this refers to the local namespace of the Route. \n Note that there are specific rules for ParentRefs which cross namespace boundaries. Cross-namespace references are only valid if they are explicitly allowed by something in the namespace they are referring to. For example: Gateway has the AllowedRoutes field, and ReferenceGrant provides a generic way to enable any other kind of cross-namespace reference. \n ParentRefs from a Route to a Service in the same namespace are \"producer\" routes, which apply default routing rules to inbound connections from any namespace to the Service. \n ParentRefs from a Route to a Service in a different namespace are \"consumer\" routes, and these routing rules are only applied to outbound connections originating from the same namespace as the Route, for which the intended destination of the connections are a Service targeted as a ParentRef of the Route. \n Support: Core" + type: string + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + port: + description: "Port is the network port this Route targets. It can be interpreted differently based on the type of parent resource. \n When the parent resource is a Gateway, this targets all listeners listening on the specified port that also support this kind of Route(and select this Route). It's not recommended to set `Port` unless the networking behaviors specified in a Route must apply to a specific port as opposed to a listener(s) whose port(s) may be changed. When both Port and SectionName are specified, the name and port of the selected listener must match both specified values. \n When the parent resource is a Service, this targets a specific port in the Service spec. When both Port (experimental) and SectionName are specified, the name and port of the selected port must match both specified values. \n Implementations MAY choose to support other parent resources. Implementations supporting other types of parent resources MUST clearly document how/if Port is interpreted. \n For the purpose of status, an attachment is considered successful as long as the parent resource accepts it partially. For example, Gateway listeners can restrict which Routes can attach to them by Route kind, namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from the referencing Route, the Route MUST be considered successfully attached. If no Gateway listeners accept attachment from this Route, the Route MUST be considered detached from the Gateway. \n Support: Extended \n " + type: integer + format: int32 + maximum: 65535 + minimum: 1 + sectionName: + description: "SectionName is the name of a section within the target resource. In the following resources, SectionName is interpreted as the following: \n * Gateway: Listener Name. When both Port (experimental) and SectionName are specified, the name and port of the selected listener must match both specified values. * Service: Port Name. When both Port (experimental) and SectionName are specified, the name and port of the selected listener must match both specified values. Note that attaching Routes to Services as Parents is part of experimental Mesh support and is not supported for any other purpose. \n Implementations MAY choose to support attaching Routes to other resources. If that is the case, they MUST clearly document how SectionName is interpreted. \n When unspecified (empty string), this will reference the entire resource. For the purpose of status, an attachment is considered successful if at least one section in the parent resource accepts it. For example, Gateway listeners can restrict which Routes can attach to them by Route kind, namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from the referencing Route, the Route MUST be considered successfully attached. If no Gateway listeners accept attachment from this Route, the Route MUST be considered detached from the Gateway. \n Support: Core" + type: string + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + serviceType: + description: Optional service type for Kubernetes solver service. Supported values are NodePort or ClusterIP. If unset, defaults to NodePort. + type: string + ingress: + description: The ingress based HTTP01 challenge solver will solve challenges by creating or modifying Ingress resources in order to route requests for '/.well-known/acme-challenge/XYZ' to 'challenge solver' pods that are provisioned by cert-manager for each Challenge to be completed. + type: object + properties: + class: + description: This field configures the annotation `kubernetes.io/ingress.class` when creating Ingress resources to solve ACME challenges that use this challenge solver. Only one of `class`, `name` or `ingressClassName` may be specified. + type: string + ingressClassName: + description: This field configures the field `ingressClassName` on the created Ingress resources used to solve ACME challenges that use this challenge solver. This is the recommended way of configuring the ingress class. Only one of `class`, `name` or `ingressClassName` may be specified. + type: string + ingressTemplate: + description: Optional ingress template used to configure the ACME challenge solver ingress used for HTTP01 challenges. + type: object + properties: + metadata: + description: ObjectMeta overrides for the ingress used to solve HTTP01 challenges. Only the 'labels' and 'annotations' fields may be set. If labels or annotations overlap with in-built values, the values here will override the in-built values. + type: object + properties: + annotations: + description: Annotations that should be added to the created ACME HTTP01 solver ingress. + type: object + additionalProperties: + type: string + labels: + description: Labels that should be added to the created ACME HTTP01 solver ingress. + type: object + additionalProperties: + type: string + name: + description: The name of the ingress resource that should have ACME challenge solving routes inserted into it in order to solve HTTP01 challenges. This is typically used in conjunction with ingress controllers like ingress-gce, which maintains a 1:1 mapping between external IPs and ingress resources. Only one of `class`, `name` or `ingressClassName` may be specified. + type: string + podTemplate: + description: Optional pod template used to configure the ACME challenge solver pods used for HTTP01 challenges. + type: object + properties: + metadata: + description: ObjectMeta overrides for the pod used to solve HTTP01 challenges. Only the 'labels' and 'annotations' fields may be set. If labels or annotations overlap with in-built values, the values here will override the in-built values. + type: object + properties: + annotations: + description: Annotations that should be added to the create ACME HTTP01 solver pods. + type: object + additionalProperties: + type: string + labels: + description: Labels that should be added to the created ACME HTTP01 solver pods. + type: object + additionalProperties: + type: string + spec: + description: PodSpec defines overrides for the HTTP01 challenge solver pod. Check ACMEChallengeSolverHTTP01IngressPodSpec to find out currently supported fields. All other fields will be ignored. + type: object + properties: + affinity: + description: If specified, the pod's scheduling constraints + type: object + properties: + nodeAffinity: + description: Describes node affinity scheduling rules for the pod. + type: object + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node matches the corresponding matchExpressions; the node(s) with the highest sum are the most preferred. + type: array + items: + description: An empty preferred scheduling term matches all objects with implicit weight 0 (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + type: object + required: + - preference + - weight + properties: + preference: + description: A node selector term, associated with the corresponding weight. + type: object + properties: + matchExpressions: + description: A list of node selector requirements by node's labels. + type: array + items: + description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. + type: array + items: + type: string + matchFields: + description: A list of node selector requirements by node's fields. + type: array + items: + description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. + type: array + items: + type: string + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. + type: integer + format: int32 + requiredDuringSchedulingIgnoredDuringExecution: + description: If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to an update), the system may or may not try to eventually evict the pod from its node. + type: object + required: + - nodeSelectorTerms + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. The terms are ORed. + type: array + items: + description: A null or empty node selector term matches no objects. The requirements of them are ANDed. The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + type: object + properties: + matchExpressions: + description: A list of node selector requirements by node's labels. + type: array + items: + description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. + type: array + items: + type: string + matchFields: + description: A list of node selector requirements by node's fields. + type: array + items: + description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. + type: array + items: + type: string + x-kubernetes-map-type: atomic + x-kubernetes-map-type: atomic + podAffinity: + description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). + type: object + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. + type: array + items: + description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + type: object + required: + - podAffinityTerm + - weight + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated with the corresponding weight. + type: object + required: + - topologyKey + properties: + labelSelector: + description: A label query over a set of resources, in this case pods. + type: object + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + type: array + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + type: array + items: + type: string + matchLabels: + description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + additionalProperties: + type: string + x-kubernetes-map-type: atomic + namespaceSelector: + description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. + type: object + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + type: array + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + type: array + items: + type: string + matchLabels: + description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + additionalProperties: + type: string + x-kubernetes-map-type: atomic + namespaces: + description: namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace". + type: array + items: + type: string + topologyKey: + description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. + type: string + weight: + description: weight associated with matching the corresponding podAffinityTerm, in the range 1-100. + type: integer + format: int32 + requiredDuringSchedulingIgnoredDuringExecution: + description: If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied. + type: array + items: + description: Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running + type: object + required: + - topologyKey + properties: + labelSelector: + description: A label query over a set of resources, in this case pods. + type: object + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + type: array + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + type: array + items: + type: string + matchLabels: + description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + additionalProperties: + type: string + x-kubernetes-map-type: atomic + namespaceSelector: + description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. + type: object + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + type: array + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + type: array + items: + type: string + matchLabels: + description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + additionalProperties: + type: string + x-kubernetes-map-type: atomic + namespaces: + description: namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace". + type: array + items: + type: string + topologyKey: + description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. + type: string + podAntiAffinity: + description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). + type: object + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule pods to nodes that satisfy the anti-affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. + type: array + items: + description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + type: object + required: + - podAffinityTerm + - weight + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated with the corresponding weight. + type: object + required: + - topologyKey + properties: + labelSelector: + description: A label query over a set of resources, in this case pods. + type: object + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + type: array + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + type: array + items: + type: string + matchLabels: + description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + additionalProperties: + type: string + x-kubernetes-map-type: atomic + namespaceSelector: + description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. + type: object + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + type: array + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + type: array + items: + type: string + matchLabels: + description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + additionalProperties: + type: string + x-kubernetes-map-type: atomic + namespaces: + description: namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace". + type: array + items: + type: string + topologyKey: + description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. + type: string + weight: + description: weight associated with matching the corresponding podAffinityTerm, in the range 1-100. + type: integer + format: int32 + requiredDuringSchedulingIgnoredDuringExecution: + description: If the anti-affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the anti-affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied. + type: array + items: + description: Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running + type: object + required: + - topologyKey + properties: + labelSelector: + description: A label query over a set of resources, in this case pods. + type: object + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + type: array + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + type: array + items: + type: string + matchLabels: + description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + additionalProperties: + type: string + x-kubernetes-map-type: atomic + namespaceSelector: + description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. + type: object + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + type: array + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + type: array + items: + type: string + matchLabels: + description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + additionalProperties: + type: string + x-kubernetes-map-type: atomic + namespaces: + description: namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace". + type: array + items: + type: string + topologyKey: + description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. + type: string + imagePullSecrets: + description: If specified, the pod's imagePullSecrets + type: array + items: + description: LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace. + type: object + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + x-kubernetes-map-type: atomic + nodeSelector: + description: 'NodeSelector is a selector which must be true for the pod to fit on a node. Selector which must match a node''s labels for the pod to be scheduled on that node. More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/' + type: object + additionalProperties: + type: string + priorityClassName: + description: If specified, the pod's priorityClassName. + type: string + serviceAccountName: + description: If specified, the pod's service account + type: string + tolerations: + description: If specified, the pod's tolerations. + type: array + items: + description: The pod this Toleration is attached to tolerates any taint that matches the triple using the matching operator . + type: object + properties: + effect: + description: Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Key is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: Operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: TolerationSeconds represents the period of time the toleration (which must be of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) by the system. + type: integer + format: int64 + value: + description: Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + serviceType: + description: Optional service type for Kubernetes solver service. Supported values are NodePort or ClusterIP. If unset, defaults to NodePort. + type: string + selector: + description: Selector selects a set of DNSNames on the Certificate resource that should be solved using this challenge solver. If not specified, the solver will be treated as the 'default' solver with the lowest priority, i.e. if any other solver has a more specific match, it will be used instead. + type: object + properties: + dnsNames: + description: List of DNSNames that this solver will be used to solve. If specified and a match is found, a dnsNames selector will take precedence over a dnsZones selector. If multiple solvers match with the same dnsNames value, the solver with the most matching labels in matchLabels will be selected. If neither has more matches, the solver defined earlier in the list will be selected. + type: array + items: + type: string + dnsZones: + description: List of DNSZones that this solver will be used to solve. The most specific DNS zone match specified here will take precedence over other DNS zone matches, so a solver specifying sys.example.com will be selected over one specifying example.com for the domain www.sys.example.com. If multiple solvers match with the same dnsZones value, the solver with the most matching labels in matchLabels will be selected. If neither has more matches, the solver defined earlier in the list will be selected. + type: array + items: + type: string + matchLabels: + description: A label selector that is used to refine the set of certificate's that this challenge solver will apply to. + type: object + additionalProperties: + type: string + token: + description: The ACME challenge token for this challenge. This is the raw value returned from the ACME server. + type: string + type: + description: The type of ACME challenge this resource represents. One of "HTTP-01" or "DNS-01". + type: string + enum: + - HTTP-01 + - DNS-01 + url: + description: The URL of the ACME Challenge resource for this challenge. This can be used to lookup details about the status of this challenge. + type: string + wildcard: + description: wildcard will be true if this challenge is for a wildcard identifier, for example '*.example.com'. + type: boolean + status: + type: object + properties: + presented: + description: presented will be set to true if the challenge values for this challenge are currently 'presented'. This *does not* imply the self check is passing. Only that the values have been 'submitted' for the appropriate challenge mechanism (i.e. the DNS01 TXT record has been presented, or the HTTP01 configuration has been configured). + type: boolean + processing: + description: Used to denote whether this challenge should be processed or not. This field will only be set to true by the 'scheduling' component. It will only be set to false by the 'challenges' controller, after the challenge has reached a final state or timed out. If this field is set to false, the challenge controller will not take any more action. + type: boolean + reason: + description: Contains human readable information on why the Challenge is in the current state. + type: string + state: + description: Contains the current 'state' of the challenge. If not set, the state of the challenge is unknown. + type: string + enum: + - valid + - ready + - pending + - processing + - invalid + - expired + - errored + served: true + storage: true + subresources: + status: {} +--- +# Source: cert-manager/deploy/crds/crd-certificaterequests.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: certificaterequests.cert-manager.io + labels: + app: 'cert-manager' + app.kubernetes.io/name: 'cert-manager' + app.kubernetes.io/instance: 'cert-manager' + # Generated labels + app.kubernetes.io/version: "{{ cert_manager_version }}" +spec: + group: cert-manager.io + names: + kind: CertificateRequest + listKind: CertificateRequestList + plural: certificaterequests + shortNames: + - cr + - crs + singular: certificaterequest + categories: + - cert-manager + scope: Namespaced + versions: + - name: v1 + subresources: + status: {} + additionalPrinterColumns: + - jsonPath: .status.conditions[?(@.type=="Approved")].status + name: Approved + type: string + - jsonPath: .status.conditions[?(@.type=="Denied")].status + name: Denied + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .spec.issuerRef.name + name: Issuer + type: string + - jsonPath: .spec.username + name: Requestor + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].message + name: Status + priority: 1 + type: string + - jsonPath: .metadata.creationTimestamp + description: CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC. + name: Age + type: date + schema: + openAPIV3Schema: + description: "A CertificateRequest is used to request a signed certificate from one of the configured issuers. \n All fields within the CertificateRequest's `spec` are immutable after creation. A CertificateRequest will either succeed or fail, as denoted by its `Ready` status condition and its `status.failureTime` field. \n A CertificateRequest is a one-shot resource, meaning it represents a single point in time request for a certificate and cannot be re-used." + type: object + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Specification of the desired state of the CertificateRequest resource. https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + type: object + required: + - issuerRef + - request + properties: + duration: + description: Requested 'duration' (i.e. lifetime) of the Certificate. Note that the issuer may choose to ignore the requested duration, just like any other requested attribute. + type: string + extra: + description: Extra contains extra attributes of the user that created the CertificateRequest. Populated by the cert-manager webhook on creation and immutable. + type: object + additionalProperties: + type: array + items: + type: string + groups: + description: Groups contains group membership of the user that created the CertificateRequest. Populated by the cert-manager webhook on creation and immutable. + type: array + items: + type: string + x-kubernetes-list-type: atomic + isCA: + description: "Requested basic constraints isCA value. Note that the issuer may choose to ignore the requested isCA value, just like any other requested attribute. \n NOTE: If the CSR in the `Request` field has a BasicConstraints extension, it must have the same isCA value as specified here. \n If true, this will automatically add the `cert sign` usage to the list of requested `usages`." + type: boolean + issuerRef: + description: "Reference to the issuer responsible for issuing the certificate. If the issuer is namespace-scoped, it must be in the same namespace as the Certificate. If the issuer is cluster-scoped, it can be used from any namespace. \n The `name` field of the reference must always be specified." + type: object + required: + - name + properties: + group: + description: Group of the resource being referred to. + type: string + kind: + description: Kind of the resource being referred to. + type: string + name: + description: Name of the resource being referred to. + type: string + request: + description: "The PEM-encoded X.509 certificate signing request to be submitted to the issuer for signing. \n If the CSR has a BasicConstraints extension, its isCA attribute must match the `isCA` value of this CertificateRequest. If the CSR has a KeyUsage extension, its key usages must match the key usages in the `usages` field of this CertificateRequest. If the CSR has a ExtKeyUsage extension, its extended key usages must match the extended key usages in the `usages` field of this CertificateRequest." + type: string + format: byte + uid: + description: UID contains the uid of the user that created the CertificateRequest. Populated by the cert-manager webhook on creation and immutable. + type: string + usages: + description: "Requested key usages and extended key usages. \n NOTE: If the CSR in the `Request` field has uses the KeyUsage or ExtKeyUsage extension, these extensions must have the same values as specified here without any additional values. \n If unset, defaults to `digital signature` and `key encipherment`." + type: array + items: + description: "KeyUsage specifies valid usage contexts for keys. See: https://tools.ietf.org/html/rfc5280#section-4.2.1.3 https://tools.ietf.org/html/rfc5280#section-4.2.1.12 \n Valid KeyUsage values are as follows: \"signing\", \"digital signature\", \"content commitment\", \"key encipherment\", \"key agreement\", \"data encipherment\", \"cert sign\", \"crl sign\", \"encipher only\", \"decipher only\", \"any\", \"server auth\", \"client auth\", \"code signing\", \"email protection\", \"s/mime\", \"ipsec end system\", \"ipsec tunnel\", \"ipsec user\", \"timestamping\", \"ocsp signing\", \"microsoft sgc\", \"netscape sgc\"" + type: string + enum: + - signing + - digital signature + - content commitment + - key encipherment + - key agreement + - data encipherment + - cert sign + - crl sign + - encipher only + - decipher only + - any + - server auth + - client auth + - code signing + - email protection + - s/mime + - ipsec end system + - ipsec tunnel + - ipsec user + - timestamping + - ocsp signing + - microsoft sgc + - netscape sgc + username: + description: Username contains the name of the user that created the CertificateRequest. Populated by the cert-manager webhook on creation and immutable. + type: string + status: + description: 'Status of the CertificateRequest. This is set and managed automatically. Read-only. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status' + type: object + properties: + ca: + description: The PEM encoded X.509 certificate of the signer, also known as the CA (Certificate Authority). This is set on a best-effort basis by different issuers. If not set, the CA is assumed to be unknown/not available. + type: string + format: byte + certificate: + description: The PEM encoded X.509 certificate resulting from the certificate signing request. If not set, the CertificateRequest has either not been completed or has failed. More information on failure can be found by checking the `conditions` field. + type: string + format: byte + conditions: + description: List of status conditions to indicate the status of a CertificateRequest. Known condition types are `Ready`, `InvalidRequest`, `Approved` and `Denied`. + type: array + items: + description: CertificateRequestCondition contains condition information for a CertificateRequest. + type: object + required: + - status + - type + properties: + lastTransitionTime: + description: LastTransitionTime is the timestamp corresponding to the last status change of this condition. + type: string + format: date-time + message: + description: Message is a human readable description of the details of the last transition, complementing reason. + type: string + reason: + description: Reason is a brief machine readable explanation for the condition's last transition. + type: string + status: + description: Status of the condition, one of (`True`, `False`, `Unknown`). + type: string + enum: + - "True" + - "False" + - Unknown + type: + description: Type of the condition, known values are (`Ready`, `InvalidRequest`, `Approved`, `Denied`). + type: string + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + failureTime: + description: FailureTime stores the time that this CertificateRequest failed. This is used to influence garbage collection and back-off. + type: string + format: date-time + served: true + storage: true +--- +# Source: cert-manager/deploy/crds/crd-issuers.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: issuers.cert-manager.io + labels: + app: 'cert-manager' + app.kubernetes.io/name: 'cert-manager' + app.kubernetes.io/instance: "cert-manager" + # Generated labels + app.kubernetes.io/version: "{{ cert_manager_version }}" +spec: + group: cert-manager.io + names: + kind: Issuer + listKind: IssuerList + plural: issuers + singular: issuer + categories: + - cert-manager + scope: Namespaced + versions: + - name: v1 + subresources: + status: {} + additionalPrinterColumns: + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].message + name: Status + priority: 1 + type: string + - jsonPath: .metadata.creationTimestamp + description: CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC. + name: Age + type: date + schema: + openAPIV3Schema: + description: An Issuer represents a certificate issuing authority which can be referenced as part of `issuerRef` fields. It is scoped to a single namespace and can therefore only be referenced by resources within the same namespace. + type: object + required: + - spec + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Desired state of the Issuer resource. + type: object + properties: + acme: + description: ACME configures this issuer to communicate with a RFC8555 (ACME) server to obtain signed x509 certificates. + type: object + required: + - privateKeySecretRef + - server + properties: + caBundle: + description: Base64-encoded bundle of PEM CAs which can be used to validate the certificate chain presented by the ACME server. Mutually exclusive with SkipTLSVerify; prefer using CABundle to prevent various kinds of security vulnerabilities. If CABundle and SkipTLSVerify are unset, the system certificate bundle inside the container is used to validate the TLS connection. + type: string + format: byte + disableAccountKeyGeneration: + description: Enables or disables generating a new ACME account key. If true, the Issuer resource will *not* request a new account but will expect the account key to be supplied via an existing secret. If false, the cert-manager system will generate a new ACME account key for the Issuer. Defaults to false. + type: boolean + email: + description: Email is the email address to be associated with the ACME account. This field is optional, but it is strongly recommended to be set. It will be used to contact you in case of issues with your account or certificates, including expiry notification emails. This field may be updated after the account is initially registered. + type: string + enableDurationFeature: + description: Enables requesting a Not After date on certificates that matches the duration of the certificate. This is not supported by all ACME servers like Let's Encrypt. If set to true when the ACME server does not support it it will create an error on the Order. Defaults to false. + type: boolean + externalAccountBinding: + description: ExternalAccountBinding is a reference to a CA external account of the ACME server. If set, upon registration cert-manager will attempt to associate the given external account credentials with the registered ACME account. + type: object + required: + - keyID + - keySecretRef + properties: + keyAlgorithm: + description: 'Deprecated: keyAlgorithm field exists for historical compatibility reasons and should not be used. The algorithm is now hardcoded to HS256 in golang/x/crypto/acme.' + type: string + enum: + - HS256 + - HS384 + - HS512 + keyID: + description: keyID is the ID of the CA key that the External Account is bound to. + type: string + keySecretRef: + description: keySecretRef is a Secret Key Selector referencing a data item in a Kubernetes Secret which holds the symmetric MAC key of the External Account Binding. The `key` is the index string that is paired with the key data in the Secret and should not be confused with the key data itself, or indeed with the External Account Binding keyID above. The secret key stored in the Secret **must** be un-padded, base64 URL encoded data. + type: object + required: + - name + properties: + key: + description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + type: string + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + preferredChain: + description: 'PreferredChain is the chain to use if the ACME server outputs multiple. PreferredChain is no guarantee that this one gets delivered by the ACME endpoint. For example, for Let''s Encrypt''s DST crosssign you would use: "DST Root CA X3" or "ISRG Root X1" for the newer Let''s Encrypt root CA. This value picks the first certificate bundle in the ACME alternative chains that has a certificate with this value as its issuer''s CN' + type: string + maxLength: 64 + privateKeySecretRef: + description: PrivateKey is the name of a Kubernetes Secret resource that will be used to store the automatically generated ACME account private key. Optionally, a `key` may be specified to select a specific entry within the named Secret resource. If `key` is not specified, a default of `tls.key` will be used. + type: object + required: + - name + properties: + key: + description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + type: string + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + server: + description: 'Server is the URL used to access the ACME server''s ''directory'' endpoint. For example, for Let''s Encrypt''s staging endpoint, you would use: "https://acme-staging-v02.api.letsencrypt.org/directory". Only ACME v2 endpoints (i.e. RFC 8555) are supported.' + type: string + skipTLSVerify: + description: 'INSECURE: Enables or disables validation of the ACME server TLS certificate. If true, requests to the ACME server will not have the TLS certificate chain validated. Mutually exclusive with CABundle; prefer using CABundle to prevent various kinds of security vulnerabilities. Only enable this option in development environments. If CABundle and SkipTLSVerify are unset, the system certificate bundle inside the container is used to validate the TLS connection. Defaults to false.' + type: boolean + solvers: + description: 'Solvers is a list of challenge solvers that will be used to solve ACME challenges for the matching domains. Solver configurations must be provided in order to obtain certificates from an ACME server. For more information, see: https://cert-manager.io/docs/configuration/acme/' + type: array + items: + description: An ACMEChallengeSolver describes how to solve ACME challenges for the issuer it is part of. A selector may be provided to use different solving strategies for different DNS names. Only one of HTTP01 or DNS01 must be provided. + type: object + properties: + dns01: + description: Configures cert-manager to attempt to complete authorizations by performing the DNS01 challenge flow. + type: object + properties: + acmeDNS: + description: Use the 'ACME DNS' (https://github.com/joohoi/acme-dns) API to manage DNS01 challenge records. + type: object + required: + - accountSecretRef + - host + properties: + accountSecretRef: + description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. + type: object + required: + - name + properties: + key: + description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + type: string + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + host: + type: string + akamai: + description: Use the Akamai DNS zone management API to manage DNS01 challenge records. + type: object + required: + - accessTokenSecretRef + - clientSecretSecretRef + - clientTokenSecretRef + - serviceConsumerDomain + properties: + accessTokenSecretRef: + description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. + type: object + required: + - name + properties: + key: + description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + type: string + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + clientSecretSecretRef: + description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. + type: object + required: + - name + properties: + key: + description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + type: string + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + clientTokenSecretRef: + description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. + type: object + required: + - name + properties: + key: + description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + type: string + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + serviceConsumerDomain: + type: string + azureDNS: + description: Use the Microsoft Azure DNS API to manage DNS01 challenge records. + type: object + required: + - resourceGroupName + - subscriptionID + properties: + clientID: + description: if both this and ClientSecret are left unset MSI will be used + type: string + clientSecretSecretRef: + description: if both this and ClientID are left unset MSI will be used + type: object + required: + - name + properties: + key: + description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + type: string + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + environment: + description: name of the Azure environment (default AzurePublicCloud) + type: string + enum: + - AzurePublicCloud + - AzureChinaCloud + - AzureGermanCloud + - AzureUSGovernmentCloud + hostedZoneName: + description: name of the DNS zone that should be used + type: string + managedIdentity: + description: managed identity configuration, can not be used at the same time as clientID, clientSecretSecretRef or tenantID + type: object + properties: + clientID: + description: client ID of the managed identity, can not be used at the same time as resourceID + type: string + resourceID: + description: resource ID of the managed identity, can not be used at the same time as clientID + type: string + resourceGroupName: + description: resource group the DNS zone is located in + type: string + subscriptionID: + description: ID of the Azure subscription + type: string + tenantID: + description: when specifying ClientID and ClientSecret then this field is also needed + type: string + cloudDNS: + description: Use the Google Cloud DNS API to manage DNS01 challenge records. + type: object + required: + - project + properties: + hostedZoneName: + description: HostedZoneName is an optional field that tells cert-manager in which Cloud DNS zone the challenge record has to be created. If left empty cert-manager will automatically choose a zone. + type: string + project: + type: string + serviceAccountSecretRef: + description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. + type: object + required: + - name + properties: + key: + description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + type: string + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + cloudflare: + description: Use the Cloudflare API to manage DNS01 challenge records. + type: object + properties: + apiKeySecretRef: + description: 'API key to use to authenticate with Cloudflare. Note: using an API token to authenticate is now the recommended method as it allows greater control of permissions.' + type: object + required: + - name + properties: + key: + description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + type: string + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + apiTokenSecretRef: + description: API token used to authenticate with Cloudflare. + type: object + required: + - name + properties: + key: + description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + type: string + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + email: + description: Email of the account, only required when using API key based authentication. + type: string + cnameStrategy: + description: CNAMEStrategy configures how the DNS01 provider should handle CNAME records when found in DNS zones. + type: string + enum: + - None + - Follow + digitalocean: + description: Use the DigitalOcean DNS API to manage DNS01 challenge records. + type: object + required: + - tokenSecretRef + properties: + tokenSecretRef: + description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. + type: object + required: + - name + properties: + key: + description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + type: string + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + rfc2136: + description: Use RFC2136 ("Dynamic Updates in the Domain Name System") (https://datatracker.ietf.org/doc/rfc2136/) to manage DNS01 challenge records. + type: object + required: + - nameserver + properties: + nameserver: + description: The IP address or hostname of an authoritative DNS server supporting RFC2136 in the form host:port. If the host is an IPv6 address it must be enclosed in square brackets (e.g [2001:db8::1]) ; port is optional. This field is required. + type: string + tsigAlgorithm: + description: 'The TSIG Algorithm configured in the DNS supporting RFC2136. Used only when ``tsigSecretSecretRef`` and ``tsigKeyName`` are defined. Supported values are (case-insensitive): ``HMACMD5`` (default), ``HMACSHA1``, ``HMACSHA256`` or ``HMACSHA512``.' + type: string + tsigKeyName: + description: The TSIG Key name configured in the DNS. If ``tsigSecretSecretRef`` is defined, this field is required. + type: string + tsigSecretSecretRef: + description: The name of the secret containing the TSIG value. If ``tsigKeyName`` is defined, this field is required. + type: object + required: + - name + properties: + key: + description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + type: string + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + route53: + description: Use the AWS Route53 API to manage DNS01 challenge records. + type: object + required: + - region + properties: + accessKeyID: + description: 'The AccessKeyID is used for authentication. Cannot be set when SecretAccessKeyID is set. If neither the Access Key nor Key ID are set, we fall-back to using env vars, shared credentials file or AWS Instance metadata, see: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials' + type: string + accessKeyIDSecretRef: + description: 'The SecretAccessKey is used for authentication. If set, pull the AWS access key ID from a key within a Kubernetes Secret. Cannot be set when AccessKeyID is set. If neither the Access Key nor Key ID are set, we fall-back to using env vars, shared credentials file or AWS Instance metadata, see: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials' + type: object + required: + - name + properties: + key: + description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + type: string + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + hostedZoneID: + description: If set, the provider will manage only this zone in Route53 and will not do an lookup using the route53:ListHostedZonesByName api call. + type: string + region: + description: Always set the region when using AccessKeyID and SecretAccessKey + type: string + role: + description: Role is a Role ARN which the Route53 provider will assume using either the explicit credentials AccessKeyID/SecretAccessKey or the inferred credentials from environment variables, shared credentials file or AWS Instance metadata + type: string + secretAccessKeySecretRef: + description: 'The SecretAccessKey is used for authentication. If neither the Access Key nor Key ID are set, we fall-back to using env vars, shared credentials file or AWS Instance metadata, see: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials' + type: object + required: + - name + properties: + key: + description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + type: string + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + webhook: + description: Configure an external webhook based DNS01 challenge solver to manage DNS01 challenge records. + type: object + required: + - groupName + - solverName + properties: + config: + description: Additional configuration that should be passed to the webhook apiserver when challenges are processed. This can contain arbitrary JSON data. Secret values should not be specified in this stanza. If secret values are needed (e.g. credentials for a DNS service), you should use a SecretKeySelector to reference a Secret resource. For details on the schema of this field, consult the webhook provider implementation's documentation. + x-kubernetes-preserve-unknown-fields: true + groupName: + description: The API group name that should be used when POSTing ChallengePayload resources to the webhook apiserver. This should be the same as the GroupName specified in the webhook provider implementation. + type: string + solverName: + description: The name of the solver to use, as defined in the webhook provider implementation. This will typically be the name of the provider, e.g. 'cloudflare'. + type: string + http01: + description: Configures cert-manager to attempt to complete authorizations by performing the HTTP01 challenge flow. It is not possible to obtain certificates for wildcard domain names (e.g. `*.example.com`) using the HTTP01 challenge mechanism. + type: object + properties: + gatewayHTTPRoute: + description: The Gateway API is a sig-network community API that models service networking in Kubernetes (https://gateway-api.sigs.k8s.io/). The Gateway solver will create HTTPRoutes with the specified labels in the same namespace as the challenge. This solver is experimental, and fields / behaviour may change in the future. + type: object + properties: + labels: + description: Custom labels that will be applied to HTTPRoutes created by cert-manager while solving HTTP-01 challenges. + type: object + additionalProperties: + type: string + parentRefs: + description: 'When solving an HTTP-01 challenge, cert-manager creates an HTTPRoute. cert-manager needs to know which parentRefs should be used when creating the HTTPRoute. Usually, the parentRef references a Gateway. See: https://gateway-api.sigs.k8s.io/api-types/httproute/#attaching-to-gateways' + type: array + items: + description: "ParentReference identifies an API object (usually a Gateway) that can be considered a parent of this resource (usually a route). There are two kinds of parent resources with \"Core\" support: \n * Gateway (Gateway conformance profile) * Service (Mesh conformance profile, experimental, ClusterIP Services only) \n This API may be extended in the future to support additional kinds of parent resources. \n The API object must be valid in the cluster; the Group and Kind must be registered in the cluster for this reference to be valid." + type: object + required: + - name + properties: + group: + description: "Group is the group of the referent. When unspecified, \"gateway.networking.k8s.io\" is inferred. To set the core API group (such as for a \"Service\" kind referent), Group must be explicitly set to \"\" (empty string). \n Support: Core" + type: string + default: gateway.networking.k8s.io + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + kind: + description: "Kind is kind of the referent. \n There are two kinds of parent resources with \"Core\" support: \n * Gateway (Gateway conformance profile) * Service (Mesh conformance profile, experimental, ClusterIP Services only) \n Support for other resources is Implementation-Specific." + type: string + default: Gateway + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + name: + description: "Name is the name of the referent. \n Support: Core" + type: string + maxLength: 253 + minLength: 1 + namespace: + description: "Namespace is the namespace of the referent. When unspecified, this refers to the local namespace of the Route. \n Note that there are specific rules for ParentRefs which cross namespace boundaries. Cross-namespace references are only valid if they are explicitly allowed by something in the namespace they are referring to. For example: Gateway has the AllowedRoutes field, and ReferenceGrant provides a generic way to enable any other kind of cross-namespace reference. \n ParentRefs from a Route to a Service in the same namespace are \"producer\" routes, which apply default routing rules to inbound connections from any namespace to the Service. \n ParentRefs from a Route to a Service in a different namespace are \"consumer\" routes, and these routing rules are only applied to outbound connections originating from the same namespace as the Route, for which the intended destination of the connections are a Service targeted as a ParentRef of the Route. \n Support: Core" + type: string + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + port: + description: "Port is the network port this Route targets. It can be interpreted differently based on the type of parent resource. \n When the parent resource is a Gateway, this targets all listeners listening on the specified port that also support this kind of Route(and select this Route). It's not recommended to set `Port` unless the networking behaviors specified in a Route must apply to a specific port as opposed to a listener(s) whose port(s) may be changed. When both Port and SectionName are specified, the name and port of the selected listener must match both specified values. \n When the parent resource is a Service, this targets a specific port in the Service spec. When both Port (experimental) and SectionName are specified, the name and port of the selected port must match both specified values. \n Implementations MAY choose to support other parent resources. Implementations supporting other types of parent resources MUST clearly document how/if Port is interpreted. \n For the purpose of status, an attachment is considered successful as long as the parent resource accepts it partially. For example, Gateway listeners can restrict which Routes can attach to them by Route kind, namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from the referencing Route, the Route MUST be considered successfully attached. If no Gateway listeners accept attachment from this Route, the Route MUST be considered detached from the Gateway. \n Support: Extended \n " + type: integer + format: int32 + maximum: 65535 + minimum: 1 + sectionName: + description: "SectionName is the name of a section within the target resource. In the following resources, SectionName is interpreted as the following: \n * Gateway: Listener Name. When both Port (experimental) and SectionName are specified, the name and port of the selected listener must match both specified values. * Service: Port Name. When both Port (experimental) and SectionName are specified, the name and port of the selected listener must match both specified values. Note that attaching Routes to Services as Parents is part of experimental Mesh support and is not supported for any other purpose. \n Implementations MAY choose to support attaching Routes to other resources. If that is the case, they MUST clearly document how SectionName is interpreted. \n When unspecified (empty string), this will reference the entire resource. For the purpose of status, an attachment is considered successful if at least one section in the parent resource accepts it. For example, Gateway listeners can restrict which Routes can attach to them by Route kind, namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from the referencing Route, the Route MUST be considered successfully attached. If no Gateway listeners accept attachment from this Route, the Route MUST be considered detached from the Gateway. \n Support: Core" + type: string + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + serviceType: + description: Optional service type for Kubernetes solver service. Supported values are NodePort or ClusterIP. If unset, defaults to NodePort. + type: string + ingress: + description: The ingress based HTTP01 challenge solver will solve challenges by creating or modifying Ingress resources in order to route requests for '/.well-known/acme-challenge/XYZ' to 'challenge solver' pods that are provisioned by cert-manager for each Challenge to be completed. + type: object + properties: + class: + description: This field configures the annotation `kubernetes.io/ingress.class` when creating Ingress resources to solve ACME challenges that use this challenge solver. Only one of `class`, `name` or `ingressClassName` may be specified. + type: string + ingressClassName: + description: This field configures the field `ingressClassName` on the created Ingress resources used to solve ACME challenges that use this challenge solver. This is the recommended way of configuring the ingress class. Only one of `class`, `name` or `ingressClassName` may be specified. + type: string + ingressTemplate: + description: Optional ingress template used to configure the ACME challenge solver ingress used for HTTP01 challenges. + type: object + properties: + metadata: + description: ObjectMeta overrides for the ingress used to solve HTTP01 challenges. Only the 'labels' and 'annotations' fields may be set. If labels or annotations overlap with in-built values, the values here will override the in-built values. + type: object + properties: + annotations: + description: Annotations that should be added to the created ACME HTTP01 solver ingress. + type: object + additionalProperties: + type: string + labels: + description: Labels that should be added to the created ACME HTTP01 solver ingress. + type: object + additionalProperties: + type: string + name: + description: The name of the ingress resource that should have ACME challenge solving routes inserted into it in order to solve HTTP01 challenges. This is typically used in conjunction with ingress controllers like ingress-gce, which maintains a 1:1 mapping between external IPs and ingress resources. Only one of `class`, `name` or `ingressClassName` may be specified. + type: string + podTemplate: + description: Optional pod template used to configure the ACME challenge solver pods used for HTTP01 challenges. + type: object + properties: + metadata: + description: ObjectMeta overrides for the pod used to solve HTTP01 challenges. Only the 'labels' and 'annotations' fields may be set. If labels or annotations overlap with in-built values, the values here will override the in-built values. + type: object + properties: + annotations: + description: Annotations that should be added to the create ACME HTTP01 solver pods. + type: object + additionalProperties: + type: string + labels: + description: Labels that should be added to the created ACME HTTP01 solver pods. + type: object + additionalProperties: + type: string + spec: + description: PodSpec defines overrides for the HTTP01 challenge solver pod. Check ACMEChallengeSolverHTTP01IngressPodSpec to find out currently supported fields. All other fields will be ignored. + type: object + properties: + affinity: + description: If specified, the pod's scheduling constraints + type: object + properties: + nodeAffinity: + description: Describes node affinity scheduling rules for the pod. + type: object + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node matches the corresponding matchExpressions; the node(s) with the highest sum are the most preferred. + type: array + items: + description: An empty preferred scheduling term matches all objects with implicit weight 0 (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + type: object + required: + - preference + - weight + properties: + preference: + description: A node selector term, associated with the corresponding weight. + type: object + properties: + matchExpressions: + description: A list of node selector requirements by node's labels. + type: array + items: + description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. + type: array + items: + type: string + matchFields: + description: A list of node selector requirements by node's fields. + type: array + items: + description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. + type: array + items: + type: string + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. + type: integer + format: int32 + requiredDuringSchedulingIgnoredDuringExecution: + description: If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to an update), the system may or may not try to eventually evict the pod from its node. + type: object + required: + - nodeSelectorTerms + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. The terms are ORed. + type: array + items: + description: A null or empty node selector term matches no objects. The requirements of them are ANDed. The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + type: object + properties: + matchExpressions: + description: A list of node selector requirements by node's labels. + type: array + items: + description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. + type: array + items: + type: string + matchFields: + description: A list of node selector requirements by node's fields. + type: array + items: + description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. + type: array + items: + type: string + x-kubernetes-map-type: atomic + x-kubernetes-map-type: atomic + podAffinity: + description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). + type: object + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. + type: array + items: + description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + type: object + required: + - podAffinityTerm + - weight + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated with the corresponding weight. + type: object + required: + - topologyKey + properties: + labelSelector: + description: A label query over a set of resources, in this case pods. + type: object + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + type: array + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + type: array + items: + type: string + matchLabels: + description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + additionalProperties: + type: string + x-kubernetes-map-type: atomic + namespaceSelector: + description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. + type: object + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + type: array + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + type: array + items: + type: string + matchLabels: + description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + additionalProperties: + type: string + x-kubernetes-map-type: atomic + namespaces: + description: namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace". + type: array + items: + type: string + topologyKey: + description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. + type: string + weight: + description: weight associated with matching the corresponding podAffinityTerm, in the range 1-100. + type: integer + format: int32 + requiredDuringSchedulingIgnoredDuringExecution: + description: If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied. + type: array + items: + description: Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running + type: object + required: + - topologyKey + properties: + labelSelector: + description: A label query over a set of resources, in this case pods. + type: object + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + type: array + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + type: array + items: + type: string + matchLabels: + description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + additionalProperties: + type: string + x-kubernetes-map-type: atomic + namespaceSelector: + description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. + type: object + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + type: array + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + type: array + items: + type: string + matchLabels: + description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + additionalProperties: + type: string + x-kubernetes-map-type: atomic + namespaces: + description: namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace". + type: array + items: + type: string + topologyKey: + description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. + type: string + podAntiAffinity: + description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). + type: object + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule pods to nodes that satisfy the anti-affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. + type: array + items: + description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + type: object + required: + - podAffinityTerm + - weight + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated with the corresponding weight. + type: object + required: + - topologyKey + properties: + labelSelector: + description: A label query over a set of resources, in this case pods. + type: object + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + type: array + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + type: array + items: + type: string + matchLabels: + description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + additionalProperties: + type: string + x-kubernetes-map-type: atomic + namespaceSelector: + description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. + type: object + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + type: array + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + type: array + items: + type: string + matchLabels: + description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + additionalProperties: + type: string + x-kubernetes-map-type: atomic + namespaces: + description: namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace". + type: array + items: + type: string + topologyKey: + description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. + type: string + weight: + description: weight associated with matching the corresponding podAffinityTerm, in the range 1-100. + type: integer + format: int32 + requiredDuringSchedulingIgnoredDuringExecution: + description: If the anti-affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the anti-affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied. + type: array + items: + description: Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running + type: object + required: + - topologyKey + properties: + labelSelector: + description: A label query over a set of resources, in this case pods. + type: object + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + type: array + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + type: array + items: + type: string + matchLabels: + description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + additionalProperties: + type: string + x-kubernetes-map-type: atomic + namespaceSelector: + description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. + type: object + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + type: array + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + type: array + items: + type: string + matchLabels: + description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + additionalProperties: + type: string + x-kubernetes-map-type: atomic + namespaces: + description: namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace". + type: array + items: + type: string + topologyKey: + description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. + type: string + imagePullSecrets: + description: If specified, the pod's imagePullSecrets + type: array + items: + description: LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace. + type: object + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + x-kubernetes-map-type: atomic + nodeSelector: + description: 'NodeSelector is a selector which must be true for the pod to fit on a node. Selector which must match a node''s labels for the pod to be scheduled on that node. More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/' + type: object + additionalProperties: + type: string + priorityClassName: + description: If specified, the pod's priorityClassName. + type: string + serviceAccountName: + description: If specified, the pod's service account + type: string + tolerations: + description: If specified, the pod's tolerations. + type: array + items: + description: The pod this Toleration is attached to tolerates any taint that matches the triple using the matching operator . + type: object + properties: + effect: + description: Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Key is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: Operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: TolerationSeconds represents the period of time the toleration (which must be of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) by the system. + type: integer + format: int64 + value: + description: Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + serviceType: + description: Optional service type for Kubernetes solver service. Supported values are NodePort or ClusterIP. If unset, defaults to NodePort. + type: string + selector: + description: Selector selects a set of DNSNames on the Certificate resource that should be solved using this challenge solver. If not specified, the solver will be treated as the 'default' solver with the lowest priority, i.e. if any other solver has a more specific match, it will be used instead. + type: object + properties: + dnsNames: + description: List of DNSNames that this solver will be used to solve. If specified and a match is found, a dnsNames selector will take precedence over a dnsZones selector. If multiple solvers match with the same dnsNames value, the solver with the most matching labels in matchLabels will be selected. If neither has more matches, the solver defined earlier in the list will be selected. + type: array + items: + type: string + dnsZones: + description: List of DNSZones that this solver will be used to solve. The most specific DNS zone match specified here will take precedence over other DNS zone matches, so a solver specifying sys.example.com will be selected over one specifying example.com for the domain www.sys.example.com. If multiple solvers match with the same dnsZones value, the solver with the most matching labels in matchLabels will be selected. If neither has more matches, the solver defined earlier in the list will be selected. + type: array + items: + type: string + matchLabels: + description: A label selector that is used to refine the set of certificate's that this challenge solver will apply to. + type: object + additionalProperties: + type: string + ca: + description: CA configures this issuer to sign certificates using a signing CA keypair stored in a Secret resource. This is used to build internal PKIs that are managed by cert-manager. + type: object + required: + - secretName + properties: + crlDistributionPoints: + description: The CRL distribution points is an X.509 v3 certificate extension which identifies the location of the CRL from which the revocation of this certificate can be checked. If not set, certificates will be issued without distribution points set. + type: array + items: + type: string + ocspServers: + description: The OCSP server list is an X.509 v3 extension that defines a list of URLs of OCSP responders. The OCSP responders can be queried for the revocation status of an issued certificate. If not set, the certificate will be issued with no OCSP servers set. For example, an OCSP server URL could be "http://ocsp.int-x3.letsencrypt.org". + type: array + items: + type: string + secretName: + description: SecretName is the name of the secret used to sign Certificates issued by this Issuer. + type: string + selfSigned: + description: SelfSigned configures this issuer to 'self sign' certificates using the private key used to create the CertificateRequest object. + type: object + properties: + crlDistributionPoints: + description: The CRL distribution points is an X.509 v3 certificate extension which identifies the location of the CRL from which the revocation of this certificate can be checked. If not set certificate will be issued without CDP. Values are strings. + type: array + items: + type: string + vault: + description: Vault configures this issuer to sign certificates using a HashiCorp Vault PKI backend. + type: object + required: + - auth + - path + - server + properties: + auth: + description: Auth configures how cert-manager authenticates with the Vault server. + type: object + properties: + appRole: + description: AppRole authenticates with Vault using the App Role auth mechanism, with the role and secret stored in a Kubernetes Secret resource. + type: object + required: + - path + - roleId + - secretRef + properties: + path: + description: 'Path where the App Role authentication backend is mounted in Vault, e.g: "approle"' + type: string + roleId: + description: RoleID configured in the App Role authentication backend when setting up the authentication backend in Vault. + type: string + secretRef: + description: Reference to a key in a Secret that contains the App Role secret used to authenticate with Vault. The `key` field must be specified and denotes which entry within the Secret resource is used as the app role secret. + type: object + required: + - name + properties: + key: + description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + type: string + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + kubernetes: + description: Kubernetes authenticates with Vault by passing the ServiceAccount token stored in the named Secret resource to the Vault server. + type: object + required: + - role + properties: + mountPath: + description: The Vault mountPath here is the mount path to use when authenticating with Vault. For example, setting a value to `/v1/auth/foo`, will use the path `/v1/auth/foo/login` to authenticate with Vault. If unspecified, the default value "/v1/auth/kubernetes" will be used. + type: string + role: + description: A required field containing the Vault Role to assume. A Role binds a Kubernetes ServiceAccount with a set of Vault policies. + type: string + secretRef: + description: The required Secret field containing a Kubernetes ServiceAccount JWT used for authenticating with Vault. Use of 'ambient credentials' is not supported. + type: object + required: + - name + properties: + key: + description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + type: string + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + serviceAccountRef: + description: A reference to a service account that will be used to request a bound token (also known as "projected token"). Compared to using "secretRef", using this field means that you don't rely on statically bound tokens. To use this field, you must configure an RBAC rule to let cert-manager request a token. + type: object + required: + - name + properties: + name: + description: Name of the ServiceAccount used to request a token. + type: string + tokenSecretRef: + description: TokenSecretRef authenticates with Vault by presenting a token. + type: object + required: + - name + properties: + key: + description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + type: string + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + caBundle: + description: Base64-encoded bundle of PEM CAs which will be used to validate the certificate chain presented by Vault. Only used if using HTTPS to connect to Vault and ignored for HTTP connections. Mutually exclusive with CABundleSecretRef. If neither CABundle nor CABundleSecretRef are defined, the certificate bundle in the cert-manager controller container is used to validate the TLS connection. + type: string + format: byte + caBundleSecretRef: + description: Reference to a Secret containing a bundle of PEM-encoded CAs to use when verifying the certificate chain presented by Vault when using HTTPS. Mutually exclusive with CABundle. If neither CABundle nor CABundleSecretRef are defined, the certificate bundle in the cert-manager controller container is used to validate the TLS connection. If no key for the Secret is specified, cert-manager will default to 'ca.crt'. + type: object + required: + - name + properties: + key: + description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + type: string + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Name of the vault namespace. Namespaces is a set of features within Vault Enterprise that allows Vault environments to support Secure Multi-tenancy. e.g: "ns1" More about namespaces can be found here https://www.vaultproject.io/docs/enterprise/namespaces' + type: string + path: + description: 'Path is the mount path of the Vault PKI backend''s `sign` endpoint, e.g: "my_pki_mount/sign/my-role-name".' + type: string + server: + description: 'Server is the connection address for the Vault server, e.g: "https://vault.example.com:8200".' + type: string + venafi: + description: Venafi configures this issuer to sign certificates using a Venafi TPP or Venafi Cloud policy zone. + type: object + required: + - zone + properties: + cloud: + description: Cloud specifies the Venafi cloud configuration settings. Only one of TPP or Cloud may be specified. + type: object + required: + - apiTokenSecretRef + properties: + apiTokenSecretRef: + description: APITokenSecretRef is a secret key selector for the Venafi Cloud API token. + type: object + required: + - name + properties: + key: + description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + type: string + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + url: + description: URL is the base URL for Venafi Cloud. Defaults to "https://api.venafi.cloud/v1". + type: string + tpp: + description: TPP specifies Trust Protection Platform configuration settings. Only one of TPP or Cloud may be specified. + type: object + required: + - credentialsRef + - url + properties: + caBundle: + description: Base64-encoded bundle of PEM CAs which will be used to validate the certificate chain presented by the TPP server. Only used if using HTTPS; ignored for HTTP. If undefined, the certificate bundle in the cert-manager controller container is used to validate the chain. + type: string + format: byte + credentialsRef: + description: CredentialsRef is a reference to a Secret containing the username and password for the TPP server. The secret must contain two keys, 'username' and 'password'. + type: object + required: + - name + properties: + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + url: + description: 'URL is the base URL for the vedsdk endpoint of the Venafi TPP instance, for example: "https://tpp.example.com/vedsdk".' + type: string + zone: + description: Zone is the Venafi Policy Zone to use for this issuer. All requests made to the Venafi platform will be restricted by the named zone policy. This field is required. + type: string + status: + description: Status of the Issuer. This is set and managed automatically. + type: object + properties: + acme: + description: ACME specific status options. This field should only be set if the Issuer is configured to use an ACME server to issue certificates. + type: object + properties: + lastPrivateKeyHash: + description: LastPrivateKeyHash is a hash of the private key associated with the latest registered ACME account, in order to track changes made to registered account associated with the Issuer + type: string + lastRegisteredEmail: + description: LastRegisteredEmail is the email associated with the latest registered ACME account, in order to track changes made to registered account associated with the Issuer + type: string + uri: + description: URI is the unique account identifier, which can also be used to retrieve account details from the CA + type: string + conditions: + description: List of status conditions to indicate the status of a CertificateRequest. Known condition types are `Ready`. + type: array + items: + description: IssuerCondition contains condition information for an Issuer. + type: object + required: + - status + - type + properties: + lastTransitionTime: + description: LastTransitionTime is the timestamp corresponding to the last status change of this condition. + type: string + format: date-time + message: + description: Message is a human readable description of the details of the last transition, complementing reason. + type: string + observedGeneration: + description: If set, this represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.condition[x].observedGeneration is 9, the condition is out of date with respect to the current state of the Issuer. + type: integer + format: int64 + reason: + description: Reason is a brief machine readable explanation for the condition's last transition. + type: string + status: + description: Status of the condition, one of (`True`, `False`, `Unknown`). + type: string + enum: + - "True" + - "False" + - Unknown + type: + description: Type of the condition, known values are (`Ready`). + type: string + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + served: true + storage: true +--- +# Source: cert-manager/deploy/crds/crd-certificates.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: certificates.cert-manager.io + labels: + app: 'cert-manager' + app.kubernetes.io/name: 'cert-manager' + app.kubernetes.io/instance: 'cert-manager' + # Generated labels + app.kubernetes.io/version: "{{ cert_manager_version }}" +spec: + group: cert-manager.io + names: + kind: Certificate + listKind: CertificateList + plural: certificates + shortNames: + - cert + - certs + singular: certificate + categories: + - cert-manager + scope: Namespaced + versions: + - name: v1 + subresources: + status: {} + additionalPrinterColumns: + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .spec.secretName + name: Secret + type: string + - jsonPath: .spec.issuerRef.name + name: Issuer + priority: 1 + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].message + name: Status + priority: 1 + type: string + - jsonPath: .metadata.creationTimestamp + description: CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC. + name: Age + type: date + schema: + openAPIV3Schema: + description: "A Certificate resource should be created to ensure an up to date and signed X.509 certificate is stored in the Kubernetes Secret resource named in `spec.secretName`. \n The stored certificate will be renewed before it expires (as configured by `spec.renewBefore`)." + type: object + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Specification of the desired state of the Certificate resource. https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + type: object + required: + - issuerRef + - secretName + properties: + additionalOutputFormats: + description: "Defines extra output formats of the private key and signed certificate chain to be written to this Certificate's target Secret. \n This is an Alpha Feature and is only enabled with the `--feature-gates=AdditionalCertificateOutputFormats=true` option set on both the controller and webhook components." + type: array + items: + description: CertificateAdditionalOutputFormat defines an additional output format of a Certificate resource. These contain supplementary data formats of the signed certificate chain and paired private key. + type: object + required: + - type + properties: + type: + description: Type is the name of the format type that should be written to the Certificate's target Secret. + type: string + enum: + - DER + - CombinedPEM + commonName: + description: "Requested common name X509 certificate subject attribute. More info: https://datatracker.ietf.org/doc/html/rfc5280#section-4.1.2.6 NOTE: TLS clients will ignore this value when any subject alternative name is set (see https://tools.ietf.org/html/rfc6125#section-6.4.4). \n Should have a length of 64 characters or fewer to avoid generating invalid CSRs. Cannot be set if the `literalSubject` field is set." + type: string + dnsNames: + description: Requested DNS subject alternative names. + type: array + items: + type: string + duration: + description: "Requested 'duration' (i.e. lifetime) of the Certificate. Note that the issuer may choose to ignore the requested duration, just like any other requested attribute. \n If unset, this defaults to 90 days. Minimum accepted duration is 1 hour. Value must be in units accepted by Go time.ParseDuration https://golang.org/pkg/time/#ParseDuration." + type: string + emailAddresses: + description: Requested email subject alternative names. + type: array + items: + type: string + encodeUsagesInRequest: + description: "Whether the KeyUsage and ExtKeyUsage extensions should be set in the encoded CSR. \n This option defaults to true, and should only be disabled if the target issuer does not support CSRs with these X509 KeyUsage/ ExtKeyUsage extensions." + type: boolean + ipAddresses: + description: Requested IP address subject alternative names. + type: array + items: + type: string + isCA: + description: "Requested basic constraints isCA value. The isCA value is used to set the `isCA` field on the created CertificateRequest resources. Note that the issuer may choose to ignore the requested isCA value, just like any other requested attribute. \n If true, this will automatically add the `cert sign` usage to the list of requested `usages`." + type: boolean + issuerRef: + description: "Reference to the issuer responsible for issuing the certificate. If the issuer is namespace-scoped, it must be in the same namespace as the Certificate. If the issuer is cluster-scoped, it can be used from any namespace. \n The `name` field of the reference must always be specified." + type: object + required: + - name + properties: + group: + description: Group of the resource being referred to. + type: string + kind: + description: Kind of the resource being referred to. + type: string + name: + description: Name of the resource being referred to. + type: string + keystores: + description: Additional keystore output formats to be stored in the Certificate's Secret. + type: object + properties: + jks: + description: JKS configures options for storing a JKS keystore in the `spec.secretName` Secret resource. + type: object + required: + - create + - passwordSecretRef + properties: + create: + description: Create enables JKS keystore creation for the Certificate. If true, a file named `keystore.jks` will be created in the target Secret resource, encrypted using the password stored in `passwordSecretRef`. The keystore file will be updated immediately. If the issuer provided a CA certificate, a file named `truststore.jks` will also be created in the target Secret resource, encrypted using the password stored in `passwordSecretRef` containing the issuing Certificate Authority + type: boolean + passwordSecretRef: + description: PasswordSecretRef is a reference to a key in a Secret resource containing the password used to encrypt the JKS keystore. + type: object + required: + - name + properties: + key: + description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + type: string + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + pkcs12: + description: PKCS12 configures options for storing a PKCS12 keystore in the `spec.secretName` Secret resource. + type: object + required: + - create + - passwordSecretRef + properties: + create: + description: Create enables PKCS12 keystore creation for the Certificate. If true, a file named `keystore.p12` will be created in the target Secret resource, encrypted using the password stored in `passwordSecretRef`. The keystore file will be updated immediately. If the issuer provided a CA certificate, a file named `truststore.p12` will also be created in the target Secret resource, encrypted using the password stored in `passwordSecretRef` containing the issuing Certificate Authority + type: boolean + passwordSecretRef: + description: PasswordSecretRef is a reference to a key in a Secret resource containing the password used to encrypt the PKCS12 keystore. + type: object + required: + - name + properties: + key: + description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + type: string + name: + description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + literalSubject: + description: "Requested X.509 certificate subject, represented using the LDAP \"String Representation of a Distinguished Name\" [1]. Important: the LDAP string format also specifies the order of the attributes in the subject, this is important when issuing certs for LDAP authentication. Example: `CN=foo,DC=corp,DC=example,DC=com` More info [1]: https://datatracker.ietf.org/doc/html/rfc4514 More info: https://github.com/cert-manager/cert-manager/issues/3203 More info: https://github.com/cert-manager/cert-manager/issues/4424 \n Cannot be set if the `subject` or `commonName` field is set. This is an Alpha Feature and is only enabled with the `--feature-gates=LiteralCertificateSubject=true` option set on both the controller and webhook components." + type: string + privateKey: + description: Private key options. These include the key algorithm and size, the used encoding and the rotation policy. + type: object + properties: + algorithm: + description: "Algorithm is the private key algorithm of the corresponding private key for this certificate. \n If provided, allowed values are either `RSA`, `ECDSA` or `Ed25519`. If `algorithm` is specified and `size` is not provided, key size of 2048 will be used for `RSA` key algorithm and key size of 256 will be used for `ECDSA` key algorithm. key size is ignored when using the `Ed25519` key algorithm." + type: string + enum: + - RSA + - ECDSA + - Ed25519 + encoding: + description: "The private key cryptography standards (PKCS) encoding for this certificate's private key to be encoded in. \n If provided, allowed values are `PKCS1` and `PKCS8` standing for PKCS#1 and PKCS#8, respectively. Defaults to `PKCS1` if not specified." + type: string + enum: + - PKCS1 + - PKCS8 + rotationPolicy: + description: "RotationPolicy controls how private keys should be regenerated when a re-issuance is being processed. \n If set to `Never`, a private key will only be generated if one does not already exist in the target `spec.secretName`. If one does exists but it does not have the correct algorithm or size, a warning will be raised to await user intervention. If set to `Always`, a private key matching the specified requirements will be generated whenever a re-issuance occurs. Default is `Never` for backward compatibility." + type: string + enum: + - Never + - Always + size: + description: "Size is the key bit size of the corresponding private key for this certificate. \n If `algorithm` is set to `RSA`, valid values are `2048`, `4096` or `8192`, and will default to `2048` if not specified. If `algorithm` is set to `ECDSA`, valid values are `256`, `384` or `521`, and will default to `256` if not specified. If `algorithm` is set to `Ed25519`, Size is ignored. No other values are allowed." + type: integer + renewBefore: + description: "How long before the currently issued certificate's expiry cert-manager should renew the certificate. For example, if a certificate is valid for 60 minutes, and `renewBefore=10m`, cert-manager will begin to attempt to renew the certificate 50 minutes after it was issued (i.e. when there are 10 minutes remaining until the certificate is no longer valid). \n NOTE: The actual lifetime of the issued certificate is used to determine the renewal time. If an issuer returns a certificate with a different lifetime than the one requested, cert-manager will use the lifetime of the issued certificate. \n If unset, this defaults to 1/3 of the issued certificate's lifetime. Minimum accepted value is 5 minutes. Value must be in units accepted by Go time.ParseDuration https://golang.org/pkg/time/#ParseDuration." + type: string + revisionHistoryLimit: + description: "The maximum number of CertificateRequest revisions that are maintained in the Certificate's history. Each revision represents a single `CertificateRequest` created by this Certificate, either when it was created, renewed, or Spec was changed. Revisions will be removed by oldest first if the number of revisions exceeds this number. \n If set, revisionHistoryLimit must be a value of `1` or greater. If unset (`nil`), revisions will not be garbage collected. Default value is `nil`." + type: integer + format: int32 + secretName: + description: Name of the Secret resource that will be automatically created and managed by this Certificate resource. It will be populated with a private key and certificate, signed by the denoted issuer. The Secret resource lives in the same namespace as the Certificate resource. + type: string + secretTemplate: + description: Defines annotations and labels to be copied to the Certificate's Secret. Labels and annotations on the Secret will be changed as they appear on the SecretTemplate when added or removed. SecretTemplate annotations are added in conjunction with, and cannot overwrite, the base set of annotations cert-manager sets on the Certificate's Secret. + type: object + properties: + annotations: + description: Annotations is a key value map to be copied to the target Kubernetes Secret. + type: object + additionalProperties: + type: string + labels: + description: Labels is a key value map to be copied to the target Kubernetes Secret. + type: object + additionalProperties: + type: string + subject: + description: "Requested set of X509 certificate subject attributes. More info: https://datatracker.ietf.org/doc/html/rfc5280#section-4.1.2.6 \n The common name attribute is specified separately in the `commonName` field. Cannot be set if the `literalSubject` field is set." + type: object + properties: + countries: + description: Countries to be used on the Certificate. + type: array + items: + type: string + localities: + description: Cities to be used on the Certificate. + type: array + items: + type: string + organizationalUnits: + description: Organizational Units to be used on the Certificate. + type: array + items: + type: string + organizations: + description: Organizations to be used on the Certificate. + type: array + items: + type: string + postalCodes: + description: Postal codes to be used on the Certificate. + type: array + items: + type: string + provinces: + description: State/Provinces to be used on the Certificate. + type: array + items: + type: string + serialNumber: + description: Serial number to be used on the Certificate. + type: string + streetAddresses: + description: Street addresses to be used on the Certificate. + type: array + items: + type: string + uris: + description: Requested URI subject alternative names. + type: array + items: + type: string + usages: + description: "Requested key usages and extended key usages. These usages are used to set the `usages` field on the created CertificateRequest resources. If `encodeUsagesInRequest` is unset or set to `true`, the usages will additionally be encoded in the `request` field which contains the CSR blob. \n If unset, defaults to `digital signature` and `key encipherment`." + type: array + items: + description: "KeyUsage specifies valid usage contexts for keys. See: https://tools.ietf.org/html/rfc5280#section-4.2.1.3 https://tools.ietf.org/html/rfc5280#section-4.2.1.12 \n Valid KeyUsage values are as follows: \"signing\", \"digital signature\", \"content commitment\", \"key encipherment\", \"key agreement\", \"data encipherment\", \"cert sign\", \"crl sign\", \"encipher only\", \"decipher only\", \"any\", \"server auth\", \"client auth\", \"code signing\", \"email protection\", \"s/mime\", \"ipsec end system\", \"ipsec tunnel\", \"ipsec user\", \"timestamping\", \"ocsp signing\", \"microsoft sgc\", \"netscape sgc\"" + type: string + enum: + - signing + - digital signature + - content commitment + - key encipherment + - key agreement + - data encipherment + - cert sign + - crl sign + - encipher only + - decipher only + - any + - server auth + - client auth + - code signing + - email protection + - s/mime + - ipsec end system + - ipsec tunnel + - ipsec user + - timestamping + - ocsp signing + - microsoft sgc + - netscape sgc + status: + description: 'Status of the Certificate. This is set and managed automatically. Read-only. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status' + type: object + properties: + conditions: + description: List of status conditions to indicate the status of certificates. Known condition types are `Ready` and `Issuing`. + type: array + items: + description: CertificateCondition contains condition information for an Certificate. + type: object + required: + - status + - type + properties: + lastTransitionTime: + description: LastTransitionTime is the timestamp corresponding to the last status change of this condition. + type: string + format: date-time + message: + description: Message is a human readable description of the details of the last transition, complementing reason. + type: string + observedGeneration: + description: If set, this represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.condition[x].observedGeneration is 9, the condition is out of date with respect to the current state of the Certificate. + type: integer + format: int64 + reason: + description: Reason is a brief machine readable explanation for the condition's last transition. + type: string + status: + description: Status of the condition, one of (`True`, `False`, `Unknown`). + type: string + enum: + - "True" + - "False" + - Unknown + type: + description: Type of the condition, known values are (`Ready`, `Issuing`). + type: string + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + failedIssuanceAttempts: + description: The number of continuous failed issuance attempts up till now. This field gets removed (if set) on a successful issuance and gets set to 1 if unset and an issuance has failed. If an issuance has failed, the delay till the next issuance will be calculated using formula time.Hour * 2 ^ (failedIssuanceAttempts - 1). + type: integer + lastFailureTime: + description: LastFailureTime is set only if the lastest issuance for this Certificate failed and contains the time of the failure. If an issuance has failed, the delay till the next issuance will be calculated using formula time.Hour * 2 ^ (failedIssuanceAttempts - 1). If the latest issuance has succeeded this field will be unset. + type: string + format: date-time + nextPrivateKeySecretName: + description: The name of the Secret resource containing the private key to be used for the next certificate iteration. The keymanager controller will automatically set this field if the `Issuing` condition is set to `True`. It will automatically unset this field when the Issuing condition is not set or False. + type: string + notAfter: + description: The expiration time of the certificate stored in the secret named by this resource in `spec.secretName`. + type: string + format: date-time + notBefore: + description: The time after which the certificate stored in the secret named by this resource in `spec.secretName` is valid. + type: string + format: date-time + renewalTime: + description: RenewalTime is the time at which the certificate will be next renewed. If not set, no upcoming renewal is scheduled. + type: string + format: date-time + revision: + description: "The current 'revision' of the certificate as issued. \n When a CertificateRequest resource is created, it will have the `cert-manager.io/certificate-revision` set to one greater than the current value of this field. \n Upon issuance, this field will be set to the value of the annotation on the CertificateRequest resource used to issue the certificate. \n Persisting the value on the CertificateRequest resource allows the certificates controller to know whether a request is part of an old issuance or if it is part of the ongoing revision's issuance by checking if the revision value in the annotation is greater than this field." + type: integer + served: true + storage: true +--- +# Source: cert-manager/deploy/crds/crd-orders.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: orders.acme.cert-manager.io + labels: + app: 'cert-manager' + app.kubernetes.io/name: 'cert-manager' + app.kubernetes.io/instance: 'cert-manager' + # Generated labels + app.kubernetes.io/version: "{{ cert_manager_version }}" +spec: + group: acme.cert-manager.io + names: + kind: Order + listKind: OrderList + plural: orders + singular: order + categories: + - cert-manager + - cert-manager-acme + scope: Namespaced + versions: + - name: v1 + subresources: + status: {} + additionalPrinterColumns: + - jsonPath: .status.state + name: State + type: string + - jsonPath: .spec.issuerRef.name + name: Issuer + priority: 1 + type: string + - jsonPath: .status.reason + name: Reason + priority: 1 + type: string + - jsonPath: .metadata.creationTimestamp + description: CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC. + name: Age + type: date + schema: + openAPIV3Schema: + description: Order is a type to represent an Order with an ACME server + type: object + required: + - metadata + - spec + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + type: object + required: + - issuerRef + - request + properties: + commonName: + description: CommonName is the common name as specified on the DER encoded CSR. If specified, this value must also be present in `dnsNames` or `ipAddresses`. This field must match the corresponding field on the DER encoded CSR. + type: string + dnsNames: + description: DNSNames is a list of DNS names that should be included as part of the Order validation process. This field must match the corresponding field on the DER encoded CSR. + type: array + items: + type: string + duration: + description: Duration is the duration for the not after date for the requested certificate. this is set on order creation as pe the ACME spec. + type: string + ipAddresses: + description: IPAddresses is a list of IP addresses that should be included as part of the Order validation process. This field must match the corresponding field on the DER encoded CSR. + type: array + items: + type: string + issuerRef: + description: IssuerRef references a properly configured ACME-type Issuer which should be used to create this Order. If the Issuer does not exist, processing will be retried. If the Issuer is not an 'ACME' Issuer, an error will be returned and the Order will be marked as failed. + type: object + required: + - name + properties: + group: + description: Group of the resource being referred to. + type: string + kind: + description: Kind of the resource being referred to. + type: string + name: + description: Name of the resource being referred to. + type: string + request: + description: Certificate signing request bytes in DER encoding. This will be used when finalizing the order. This field must be set on the order. + type: string + format: byte + status: + type: object + properties: + authorizations: + description: Authorizations contains data returned from the ACME server on what authorizations must be completed in order to validate the DNS names specified on the Order. + type: array + items: + description: ACMEAuthorization contains data returned from the ACME server on an authorization that must be completed in order validate a DNS name on an ACME Order resource. + type: object + required: + - url + properties: + challenges: + description: Challenges specifies the challenge types offered by the ACME server. One of these challenge types will be selected when validating the DNS name and an appropriate Challenge resource will be created to perform the ACME challenge process. + type: array + items: + description: Challenge specifies a challenge offered by the ACME server for an Order. An appropriate Challenge resource can be created to perform the ACME challenge process. + type: object + required: + - token + - type + - url + properties: + token: + description: Token is the token that must be presented for this challenge. This is used to compute the 'key' that must also be presented. + type: string + type: + description: Type is the type of challenge being offered, e.g. 'http-01', 'dns-01', 'tls-sni-01', etc. This is the raw value retrieved from the ACME server. Only 'http-01' and 'dns-01' are supported by cert-manager, other values will be ignored. + type: string + url: + description: URL is the URL of this challenge. It can be used to retrieve additional metadata about the Challenge from the ACME server. + type: string + identifier: + description: Identifier is the DNS name to be validated as part of this authorization + type: string + initialState: + description: InitialState is the initial state of the ACME authorization when first fetched from the ACME server. If an Authorization is already 'valid', the Order controller will not create a Challenge resource for the authorization. This will occur when working with an ACME server that enables 'authz reuse' (such as Let's Encrypt's production endpoint). If not set and 'identifier' is set, the state is assumed to be pending and a Challenge will be created. + type: string + enum: + - valid + - ready + - pending + - processing + - invalid + - expired + - errored + url: + description: URL is the URL of the Authorization that must be completed + type: string + wildcard: + description: Wildcard will be true if this authorization is for a wildcard DNS name. If this is true, the identifier will be the *non-wildcard* version of the DNS name. For example, if '*.example.com' is the DNS name being validated, this field will be 'true' and the 'identifier' field will be 'example.com'. + type: boolean + certificate: + description: Certificate is a copy of the PEM encoded certificate for this Order. This field will be populated after the order has been successfully finalized with the ACME server, and the order has transitioned to the 'valid' state. + type: string + format: byte + failureTime: + description: FailureTime stores the time that this order failed. This is used to influence garbage collection and back-off. + type: string + format: date-time + finalizeURL: + description: FinalizeURL of the Order. This is used to obtain certificates for this order once it has been completed. + type: string + reason: + description: Reason optionally provides more information about a why the order is in the current state. + type: string + state: + description: State contains the current state of this Order resource. States 'success' and 'expired' are 'final' + type: string + enum: + - valid + - ready + - pending + - processing + - invalid + - expired + - errored + url: + description: URL of the Order. This will initially be empty when the resource is first created. The Order controller will populate this field when the Order is first processed. This field will be immutable after it is initially set. + type: string + served: true + storage: true diff --git a/kubespray/project/roles/kubernetes-apps/ingress_controller/cert_manager/templates/cert-manager.yml.j2 b/kubespray/project/roles/kubernetes-apps/ingress_controller/cert_manager/templates/cert-manager.yml.j2 new file mode 100644 index 0000000..2bbda11 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ingress_controller/cert_manager/templates/cert-manager.yml.j2 @@ -0,0 +1,1259 @@ +# Copyright 2022 The cert-manager Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v1 +kind: Namespace +metadata: + name: {{ cert_manager_namespace }} +--- +# Source: cert-manager/deploy/charts/cert-manager/templates/cainjector-serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +automountServiceAccountToken: true +metadata: + name: cert-manager-cainjector + namespace: {{ cert_manager_namespace }} + labels: + app: cainjector + app.kubernetes.io/name: cainjector + app.kubernetes.io/instance: cert-manager + app.kubernetes.io/component: "cainjector" + app.kubernetes.io/version: "{{ cert_manager_version }}" +--- +# Source: cert-manager/deploy/charts/cert-manager/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +automountServiceAccountToken: true +metadata: + name: cert-manager + namespace: {{ cert_manager_namespace }} + labels: + app: cert-manager + app.kubernetes.io/name: cert-manager + app.kubernetes.io/instance: cert-manager + app.kubernetes.io/component: "controller" + app.kubernetes.io/version: "{{ cert_manager_version }}" +--- +# Source: cert-manager/deploy/charts/cert-manager/templates/webhook-serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +automountServiceAccountToken: true +metadata: + name: cert-manager-webhook + namespace: {{ cert_manager_namespace }} + labels: + app: webhook + app.kubernetes.io/name: webhook + app.kubernetes.io/instance: cert-manager + app.kubernetes.io/component: "webhook" + app.kubernetes.io/version: "{{ cert_manager_version }}" +--- +# Source: cert-manager/deploy/charts/cert-manager/templates/controller-config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: cert-manager + namespace: {{ cert_manager_namespace }} + labels: + app: cert-manager + app.kubernetes.io/name: cert-manager + app.kubernetes.io/instance: cert-manager + app.kubernetes.io/component: "controller" + app.kubernetes.io/version: "{{ cert_manager_version }}" +data: +--- +# Source: cert-manager/deploy/charts/cert-manager/templates/webhook-config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: cert-manager-webhook + namespace: {{ cert_manager_namespace }} + labels: + app: webhook + app.kubernetes.io/name: webhook + app.kubernetes.io/instance: cert-manager + app.kubernetes.io/component: "webhook" + app.kubernetes.io/version: "{{ cert_manager_version }}" +data: +--- +# Source: cert-manager/deploy/charts/cert-manager/templates/cainjector-rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cert-manager-cainjector + labels: + app: cainjector + app.kubernetes.io/name: cainjector + app.kubernetes.io/instance: cert-manager + app.kubernetes.io/component: "cainjector" + app.kubernetes.io/version: "{{ cert_manager_version }}" +rules: + - apiGroups: ["cert-manager.io"] + resources: ["certificates"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "create", "update", "patch"] + - apiGroups: ["admissionregistration.k8s.io"] + resources: ["validatingwebhookconfigurations", "mutatingwebhookconfigurations"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["apiregistration.k8s.io"] + resources: ["apiservices"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["get", "list", "watch", "update", "patch"] +--- +# Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml +# Issuer controller role +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cert-manager-controller-issuers + labels: + app: cert-manager + app.kubernetes.io/name: cert-manager + app.kubernetes.io/instance: cert-manager + app.kubernetes.io/component: "controller" + app.kubernetes.io/version: "{{ cert_manager_version }}" +rules: + - apiGroups: ["cert-manager.io"] + resources: ["issuers", "issuers/status"] + verbs: ["update", "patch"] + - apiGroups: ["cert-manager.io"] + resources: ["issuers"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list", "watch", "create", "update", "delete"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "patch"] +--- +# Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml +# ClusterIssuer controller role +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cert-manager-controller-clusterissuers + labels: + app: cert-manager + app.kubernetes.io/name: cert-manager + app.kubernetes.io/instance: cert-manager + app.kubernetes.io/component: "controller" + app.kubernetes.io/version: "{{ cert_manager_version }}" +rules: + - apiGroups: ["cert-manager.io"] + resources: ["clusterissuers", "clusterissuers/status"] + verbs: ["update", "patch"] + - apiGroups: ["cert-manager.io"] + resources: ["clusterissuers"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list", "watch", "create", "update", "delete"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "patch"] +--- +# Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml +# Certificates controller role +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cert-manager-controller-certificates + labels: + app: cert-manager + app.kubernetes.io/name: cert-manager + app.kubernetes.io/instance: cert-manager + app.kubernetes.io/component: "controller" + app.kubernetes.io/version: "{{ cert_manager_version }}" +rules: + - apiGroups: ["cert-manager.io"] + resources: ["certificates", "certificates/status", "certificaterequests", "certificaterequests/status"] + verbs: ["update", "patch"] + - apiGroups: ["cert-manager.io"] + resources: ["certificates", "certificaterequests", "clusterissuers", "issuers"] + verbs: ["get", "list", "watch"] + # We require these rules to support users with the OwnerReferencesPermissionEnforcement + # admission controller enabled: + # https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#ownerreferencespermissionenforcement + - apiGroups: ["cert-manager.io"] + resources: ["certificates/finalizers", "certificaterequests/finalizers"] + verbs: ["update"] + - apiGroups: ["acme.cert-manager.io"] + resources: ["orders"] + verbs: ["create", "delete", "get", "list", "watch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list", "watch", "create", "update", "delete", "patch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "patch"] +--- +# Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml +# Orders controller role +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cert-manager-controller-orders + labels: + app: cert-manager + app.kubernetes.io/name: cert-manager + app.kubernetes.io/instance: cert-manager + app.kubernetes.io/component: "controller" + app.kubernetes.io/version: "{{ cert_manager_version }}" +rules: + - apiGroups: ["acme.cert-manager.io"] + resources: ["orders", "orders/status"] + verbs: ["update", "patch"] + - apiGroups: ["acme.cert-manager.io"] + resources: ["orders", "challenges"] + verbs: ["get", "list", "watch"] + - apiGroups: ["cert-manager.io"] + resources: ["clusterissuers", "issuers"] + verbs: ["get", "list", "watch"] + - apiGroups: ["acme.cert-manager.io"] + resources: ["challenges"] + verbs: ["create", "delete"] + # We require these rules to support users with the OwnerReferencesPermissionEnforcement + # admission controller enabled: + # https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#ownerreferencespermissionenforcement + - apiGroups: ["acme.cert-manager.io"] + resources: ["orders/finalizers"] + verbs: ["update"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "patch"] +--- +# Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml +# Challenges controller role +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cert-manager-controller-challenges + labels: + app: cert-manager + app.kubernetes.io/name: cert-manager + app.kubernetes.io/instance: cert-manager + app.kubernetes.io/component: "controller" + app.kubernetes.io/version: "{{ cert_manager_version }}" +rules: + # Use to update challenge resource status + - apiGroups: ["acme.cert-manager.io"] + resources: ["challenges", "challenges/status"] + verbs: ["update", "patch"] + # Used to watch challenge resources + - apiGroups: ["acme.cert-manager.io"] + resources: ["challenges"] + verbs: ["get", "list", "watch"] + # Used to watch challenges, issuer and clusterissuer resources + - apiGroups: ["cert-manager.io"] + resources: ["issuers", "clusterissuers"] + verbs: ["get", "list", "watch"] + # Need to be able to retrieve ACME account private key to complete challenges + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list", "watch"] + # Used to create events + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "patch"] + # HTTP01 rules + - apiGroups: [""] + resources: ["pods", "services"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: ["networking.k8s.io"] + resources: ["ingresses"] + verbs: ["get", "list", "watch", "create", "delete", "update"] + - apiGroups: [ "gateway.networking.k8s.io" ] + resources: [ "httproutes" ] + verbs: ["get", "list", "watch", "create", "delete", "update"] + # We require the ability to specify a custom hostname when we are creating + # new ingress resources. + # See: https://github.com/openshift/origin/blob/21f191775636f9acadb44fa42beeb4f75b255532/pkg/route/apiserver/admission/ingress_admission.go#L84-L148 + - apiGroups: ["route.openshift.io"] + resources: ["routes/custom-host"] + verbs: ["create"] + # We require these rules to support users with the OwnerReferencesPermissionEnforcement + # admission controller enabled: + # https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#ownerreferencespermissionenforcement + - apiGroups: ["acme.cert-manager.io"] + resources: ["challenges/finalizers"] + verbs: ["update"] + # DNS01 rules (duplicated above) + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list", "watch"] +--- +# Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml +# ingress-shim controller role +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cert-manager-controller-ingress-shim + labels: + app: cert-manager + app.kubernetes.io/name: cert-manager + app.kubernetes.io/instance: cert-manager + app.kubernetes.io/component: "controller" + app.kubernetes.io/version: "{{ cert_manager_version }}" +rules: + - apiGroups: ["cert-manager.io"] + resources: ["certificates", "certificaterequests"] + verbs: ["create", "update", "delete"] + - apiGroups: ["cert-manager.io"] + resources: ["certificates", "certificaterequests", "issuers", "clusterissuers"] + verbs: ["get", "list", "watch"] + - apiGroups: ["networking.k8s.io"] + resources: ["ingresses"] + verbs: ["get", "list", "watch"] + # We require these rules to support users with the OwnerReferencesPermissionEnforcement + # admission controller enabled: + # https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#ownerreferencespermissionenforcement + - apiGroups: ["networking.k8s.io"] + resources: ["ingresses/finalizers"] + verbs: ["update"] + - apiGroups: ["gateway.networking.k8s.io"] + resources: ["gateways", "httproutes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["gateway.networking.k8s.io"] + resources: ["gateways/finalizers", "httproutes/finalizers"] + verbs: ["update"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "patch"] +--- +# Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cert-manager-cluster-view + labels: + app: cert-manager + app.kubernetes.io/name: cert-manager + app.kubernetes.io/instance: cert-manager + app.kubernetes.io/component: "controller" + app.kubernetes.io/version: "{{ cert_manager_version }}" + rbac.authorization.k8s.io/aggregate-to-cluster-reader: "true" +rules: + - apiGroups: ["cert-manager.io"] + resources: ["clusterissuers"] + verbs: ["get", "list", "watch"] +--- +# Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cert-manager-view + labels: + app: cert-manager + app.kubernetes.io/name: cert-manager + app.kubernetes.io/instance: cert-manager + app.kubernetes.io/component: "controller" + app.kubernetes.io/version: "{{ cert_manager_version }}" + rbac.authorization.k8s.io/aggregate-to-view: "true" + rbac.authorization.k8s.io/aggregate-to-edit: "true" + rbac.authorization.k8s.io/aggregate-to-admin: "true" + rbac.authorization.k8s.io/aggregate-to-cluster-reader: "true" +rules: + - apiGroups: ["cert-manager.io"] + resources: ["certificates", "certificaterequests", "issuers"] + verbs: ["get", "list", "watch"] + - apiGroups: ["acme.cert-manager.io"] + resources: ["challenges", "orders"] + verbs: ["get", "list", "watch"] +--- +# Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cert-manager-edit + labels: + app: cert-manager + app.kubernetes.io/name: cert-manager + app.kubernetes.io/instance: cert-manager + app.kubernetes.io/component: "controller" + app.kubernetes.io/version: "{{ cert_manager_version }}" + rbac.authorization.k8s.io/aggregate-to-edit: "true" + rbac.authorization.k8s.io/aggregate-to-admin: "true" +rules: + - apiGroups: ["cert-manager.io"] + resources: ["certificates", "certificaterequests", "issuers"] + verbs: ["create", "delete", "deletecollection", "patch", "update"] + - apiGroups: ["cert-manager.io"] + resources: ["certificates/status"] + verbs: ["update"] + - apiGroups: ["acme.cert-manager.io"] + resources: ["challenges", "orders"] + verbs: ["create", "delete", "deletecollection", "patch", "update"] +--- +# Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml +# Permission to approve CertificateRequests referencing cert-manager.io Issuers and ClusterIssuers +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cert-manager-controller-approve:cert-manager-io + labels: + app: cert-manager + app.kubernetes.io/name: cert-manager + app.kubernetes.io/instance: cert-manager + app.kubernetes.io/component: "cert-manager" + app.kubernetes.io/version: "{{ cert_manager_version }}" +rules: + - apiGroups: ["cert-manager.io"] + resources: ["signers"] + verbs: ["approve"] + resourceNames: ["issuers.cert-manager.io/*", "clusterissuers.cert-manager.io/*"] +--- +# Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml +# Permission to: +# - Update and sign CertificatSigningeRequests referencing cert-manager.io Issuers and ClusterIssuers +# - Perform SubjectAccessReviews to test whether users are able to reference Namespaced Issuers +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cert-manager-controller-certificatesigningrequests + labels: + app: cert-manager + app.kubernetes.io/name: cert-manager + app.kubernetes.io/instance: cert-manager + app.kubernetes.io/component: "cert-manager" + app.kubernetes.io/version: "{{ cert_manager_version }}" +rules: + - apiGroups: ["certificates.k8s.io"] + resources: ["certificatesigningrequests"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["certificates.k8s.io"] + resources: ["certificatesigningrequests/status"] + verbs: ["update", "patch"] + - apiGroups: ["certificates.k8s.io"] + resources: ["signers"] + resourceNames: ["issuers.cert-manager.io/*", "clusterissuers.cert-manager.io/*"] + verbs: ["sign"] + - apiGroups: ["authorization.k8s.io"] + resources: ["subjectaccessreviews"] + verbs: ["create"] +--- +# Source: cert-manager/deploy/charts/cert-manager/templates/webhook-rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cert-manager-webhook:subjectaccessreviews + labels: + app: webhook + app.kubernetes.io/name: webhook + app.kubernetes.io/instance: cert-manager + app.kubernetes.io/component: "webhook" + app.kubernetes.io/version: "{{ cert_manager_version }}" +rules: +- apiGroups: ["authorization.k8s.io"] + resources: ["subjectaccessreviews"] + verbs: ["create"] +--- +# Source: cert-manager/deploy/charts/cert-manager/templates/cainjector-rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cert-manager-cainjector + labels: + app: cainjector + app.kubernetes.io/name: cainjector + app.kubernetes.io/instance: cert-manager + app.kubernetes.io/component: "cainjector" + app.kubernetes.io/version: "{{ cert_manager_version }}" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cert-manager-cainjector +subjects: + - name: cert-manager-cainjector + namespace: {{ cert_manager_namespace }} + kind: ServiceAccount +--- +# Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cert-manager-controller-issuers + labels: + app: cert-manager + app.kubernetes.io/name: cert-manager + app.kubernetes.io/instance: cert-manager + app.kubernetes.io/component: "controller" + app.kubernetes.io/version: "{{ cert_manager_version }}" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cert-manager-controller-issuers +subjects: + - name: cert-manager + namespace: {{ cert_manager_namespace }} + kind: ServiceAccount +--- +# Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cert-manager-controller-clusterissuers + labels: + app: cert-manager + app.kubernetes.io/name: cert-manager + app.kubernetes.io/instance: cert-manager + app.kubernetes.io/component: "controller" + app.kubernetes.io/version: "{{ cert_manager_version }}" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cert-manager-controller-clusterissuers +subjects: + - name: cert-manager + namespace: {{ cert_manager_namespace }} + kind: ServiceAccount +--- +# Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cert-manager-controller-certificates + labels: + app: cert-manager + app.kubernetes.io/name: cert-manager + app.kubernetes.io/instance: cert-manager + app.kubernetes.io/component: "controller" + app.kubernetes.io/version: "{{ cert_manager_version }}" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cert-manager-controller-certificates +subjects: + - name: cert-manager + namespace: {{ cert_manager_namespace }} + kind: ServiceAccount +--- +# Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cert-manager-controller-orders + labels: + app: cert-manager + app.kubernetes.io/name: cert-manager + app.kubernetes.io/instance: cert-manager + app.kubernetes.io/component: "controller" + app.kubernetes.io/version: "{{ cert_manager_version }}" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cert-manager-controller-orders +subjects: + - name: cert-manager + namespace: {{ cert_manager_namespace }} + kind: ServiceAccount +--- +# Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cert-manager-controller-challenges + labels: + app: cert-manager + app.kubernetes.io/name: cert-manager + app.kubernetes.io/instance: cert-manager + app.kubernetes.io/component: "controller" + app.kubernetes.io/version: "{{ cert_manager_version }}" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cert-manager-controller-challenges +subjects: + - name: cert-manager + namespace: {{ cert_manager_namespace }} + kind: ServiceAccount +--- +# Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cert-manager-controller-ingress-shim + labels: + app: cert-manager + app.kubernetes.io/name: cert-manager + app.kubernetes.io/instance: cert-manager + app.kubernetes.io/component: "controller" + app.kubernetes.io/version: "{{ cert_manager_version }}" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cert-manager-controller-ingress-shim +subjects: + - name: cert-manager + namespace: {{ cert_manager_namespace }} + kind: ServiceAccount +--- +# Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cert-manager-controller-approve:cert-manager-io + labels: + app: cert-manager + app.kubernetes.io/name: cert-manager + app.kubernetes.io/instance: cert-manager + app.kubernetes.io/component: "cert-manager" + app.kubernetes.io/version: "{{ cert_manager_version }}" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cert-manager-controller-approve:cert-manager-io +subjects: + - name: cert-manager + namespace: {{ cert_manager_namespace }} + kind: ServiceAccount +--- +# Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cert-manager-controller-certificatesigningrequests + labels: + app: cert-manager + app.kubernetes.io/name: cert-manager + app.kubernetes.io/instance: cert-manager + app.kubernetes.io/component: "cert-manager" + app.kubernetes.io/version: "{{ cert_manager_version }}" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cert-manager-controller-certificatesigningrequests +subjects: + - name: cert-manager + namespace: {{ cert_manager_namespace }} + kind: ServiceAccount +--- +# Source: cert-manager/deploy/charts/cert-manager/templates/webhook-rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cert-manager-webhook:subjectaccessreviews + labels: + app: webhook + app.kubernetes.io/name: webhook + app.kubernetes.io/instance: cert-manager + app.kubernetes.io/component: "webhook" + app.kubernetes.io/version: "{{ cert_manager_version }}" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cert-manager-webhook:subjectaccessreviews +subjects: +- apiGroup: "" + kind: ServiceAccount + name: cert-manager-webhook + namespace: {{ cert_manager_namespace }} +--- +# Source: cert-manager/deploy/charts/cert-manager/templates/cainjector-rbac.yaml +# leader election rules +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cert-manager-cainjector:leaderelection + namespace: {{ cert_manager_leader_election_namespace }} + labels: + app: cainjector + app.kubernetes.io/name: cainjector + app.kubernetes.io/instance: cert-manager + app.kubernetes.io/component: "cainjector" + app.kubernetes.io/version: "{{ cert_manager_version }}" +rules: + # Used for leader election by the controller + # cert-manager-cainjector-leader-election is used by the CertificateBased injector controller + # see cmd/cainjector/start.go#L113 + # cert-manager-cainjector-leader-election-core is used by the SecretBased injector controller + # see cmd/cainjector/start.go#L137 + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + resourceNames: ["cert-manager-cainjector-leader-election", "cert-manager-cainjector-leader-election-core"] + verbs: ["get", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["create"] +--- +# Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cert-manager:leaderelection + namespace: {{ cert_manager_leader_election_namespace }} + labels: + app: cert-manager + app.kubernetes.io/name: cert-manager + app.kubernetes.io/instance: cert-manager + app.kubernetes.io/component: "controller" + app.kubernetes.io/version: "{{ cert_manager_version }}" +rules: + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + resourceNames: ["cert-manager-controller"] + verbs: ["get", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["create"] +--- +# Source: cert-manager/deploy/charts/cert-manager/templates/webhook-rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cert-manager-webhook:dynamic-serving + namespace: {{ cert_manager_namespace }} + labels: + app: webhook + app.kubernetes.io/name: webhook + app.kubernetes.io/instance: cert-manager + app.kubernetes.io/component: "webhook" + app.kubernetes.io/version: "{{ cert_manager_version }}" +rules: +- apiGroups: [""] + resources: ["secrets"] + resourceNames: + - 'cert-manager-webhook-ca' + verbs: ["get", "list", "watch", "update"] +# It's not possible to grant CREATE permission on a single resourceName. +- apiGroups: [""] + resources: ["secrets"] + verbs: ["create"] +--- +# Source: cert-manager/deploy/charts/cert-manager/templates/cainjector-rbac.yaml +# grant cert-manager permission to manage the leaderelection configmap in the +# leader election namespace +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cert-manager-cainjector:leaderelection + namespace: {{ cert_manager_leader_election_namespace }} + labels: + app: cainjector + app.kubernetes.io/name: cainjector + app.kubernetes.io/instance: cert-manager + app.kubernetes.io/component: "cainjector" + app.kubernetes.io/version: "{{ cert_manager_version }}" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cert-manager-cainjector:leaderelection +subjects: + - kind: ServiceAccount + name: cert-manager-cainjector + namespace: {{ cert_manager_namespace }} +--- +# Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml +# grant cert-manager permission to manage the leaderelection configmap in the +# leader election namespace +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cert-manager:leaderelection + namespace: {{ cert_manager_leader_election_namespace }} + labels: + app: cert-manager + app.kubernetes.io/name: cert-manager + app.kubernetes.io/instance: cert-manager + app.kubernetes.io/component: "controller" + app.kubernetes.io/version: "{{ cert_manager_version }}" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cert-manager:leaderelection +subjects: + - apiGroup: "" + kind: ServiceAccount + name: cert-manager + namespace: {{ cert_manager_namespace }} +--- +# Source: cert-manager/deploy/charts/cert-manager/templates/webhook-rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cert-manager-webhook:dynamic-serving + namespace: {{ cert_manager_namespace }} + labels: + app: webhook + app.kubernetes.io/name: webhook + app.kubernetes.io/instance: cert-manager + app.kubernetes.io/component: "webhook" + app.kubernetes.io/version: "{{ cert_manager_version }}" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cert-manager-webhook:dynamic-serving +subjects: +- apiGroup: "" + kind: ServiceAccount + name: cert-manager-webhook + namespace: {{ cert_manager_namespace }} +--- +# Source: cert-manager/deploy/charts/cert-manager/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: cert-manager + namespace: {{ cert_manager_namespace }} + labels: + app: cert-manager + app.kubernetes.io/name: cert-manager + app.kubernetes.io/instance: cert-manager + app.kubernetes.io/component: "controller" + app.kubernetes.io/version: "{{ cert_manager_version }}" +spec: + type: ClusterIP + ports: + - protocol: TCP + port: 9402 + name: tcp-prometheus-servicemonitor + targetPort: 9402 + selector: + app.kubernetes.io/name: cert-manager + app.kubernetes.io/instance: cert-manager + app.kubernetes.io/component: "controller" +--- +# Source: cert-manager/deploy/charts/cert-manager/templates/webhook-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: cert-manager-webhook + namespace: {{ cert_manager_namespace }} + labels: + app: webhook + app.kubernetes.io/name: webhook + app.kubernetes.io/instance: cert-manager + app.kubernetes.io/component: "webhook" + app.kubernetes.io/version: "{{ cert_manager_version }}" +spec: + type: ClusterIP + ports: + - name: https + port: 443 + protocol: TCP + targetPort: "https" + selector: + app.kubernetes.io/name: webhook + app.kubernetes.io/instance: cert-manager + app.kubernetes.io/component: "webhook" +--- +# Source: cert-manager/deploy/charts/cert-manager/templates/cainjector-deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cert-manager-cainjector + namespace: {{ cert_manager_namespace }} + labels: + app: cainjector + app.kubernetes.io/name: cainjector + app.kubernetes.io/instance: cert-manager + app.kubernetes.io/component: "cainjector" + app.kubernetes.io/version: "{{ cert_manager_version }}" +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: cainjector + app.kubernetes.io/instance: cert-manager + app.kubernetes.io/component: "cainjector" + template: + metadata: + labels: + app: cainjector + app.kubernetes.io/name: cainjector + app.kubernetes.io/instance: cert-manager + app.kubernetes.io/component: "cainjector" + app.kubernetes.io/version: "{{ cert_manager_version }}" + spec: + serviceAccountName: cert-manager-cainjector + enableServiceLinks: false + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - name: cert-manager-cainjector + image: "{{ cert_manager_cainjector_image_repo }}:{{ cert_manager_cainjector_image_tag }}" + imagePullPolicy: {{ k8s_image_pull_policy }} + args: + - --v=2 + - --leader-election-namespace={{ cert_manager_leader_election_namespace }} + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace +{% if cert_manager_http_proxy is defined and cert_manager_http_proxy != "" %} + - name: HTTP_PROXY + value: "{{ cert_manager_http_proxy }}" +{% endif %} +{% if cert_manager_https_proxy is defined and cert_manager_https_proxy != "" %} + - name: HTTPS_PROXY + value: "{{ cert_manager_https_proxy }}" +{% endif %} +{% if cert_manager_no_proxy is defined and cert_manager_no_proxy != "" %} + - name: NO_PROXY + value: "{{ cert_manager_no_proxy }}" +{% endif %} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault +{% if cert_manager_tolerations %} + tolerations: + {{ cert_manager_tolerations | to_nice_yaml(indent=2) | indent(width=8) }} +{% endif %} +{% if cert_manager_nodeselector %} + nodeSelector: + {{ cert_manager_nodeselector | to_nice_yaml | indent(width=8) }} +{% endif %} +{% if cert_manager_affinity %} + affinity: + {{ cert_manager_affinity | to_nice_yaml | indent(width=8) }} +{% endif %} +--- +{% if cert_manager_trusted_internal_ca is defined %} +apiVersion: v1 +data: + internal-ca.pem: | + {{ cert_manager_trusted_internal_ca | indent(width=4, first=False) }} +kind: ConfigMap +metadata: + name: ca-internal-truststore + namespace: {{ cert_manager_namespace }} +--- +{% endif %} +# Source: cert-manager/deploy/charts/cert-manager/templates/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cert-manager + namespace: {{ cert_manager_namespace }} + labels: + app: cert-manager + app.kubernetes.io/name: cert-manager + app.kubernetes.io/instance: cert-manager + app.kubernetes.io/component: "controller" + app.kubernetes.io/version: "{{ cert_manager_version }}" +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: cert-manager + app.kubernetes.io/instance: cert-manager + app.kubernetes.io/component: "controller" + template: + metadata: + labels: + app: cert-manager + app.kubernetes.io/name: cert-manager + app.kubernetes.io/instance: cert-manager + app.kubernetes.io/component: "controller" + app.kubernetes.io/version: "{{ cert_manager_version }}" + annotations: + prometheus.io/path: "/metrics" + prometheus.io/scrape: 'true' + prometheus.io/port: '9402' + spec: + serviceAccountName: cert-manager + enableServiceLinks: false + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - name: cert-manager-controller + image: "{{ cert_manager_controller_image_repo }}:{{ cert_manager_controller_image_tag }}" + imagePullPolicy: {{ k8s_image_pull_policy }} + args: + - --v=2 + - --cluster-resource-namespace=$(POD_NAMESPACE) + - --leader-election-namespace={{ cert_manager_leader_election_namespace }} +{% for extra_arg in cert_manager_controller_extra_args %} + - {{ extra_arg }} +{% endfor %} + ports: + - containerPort: 9402 + name: http-metrics + protocol: TCP + - containerPort: 9403 + name: http-healthz + protocol: TCP + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace +{% if cert_manager_http_proxy is defined and cert_manager_http_proxy != "" %} + - name: HTTP_PROXY + value: "{{ cert_manager_http_proxy }}" +{% endif %} +{% if cert_manager_https_proxy is defined and cert_manager_https_proxy != "" %} + - name: HTTPS_PROXY + value: "{{ cert_manager_https_proxy }}" +{% endif %} +{% if cert_manager_no_proxy is defined and cert_manager_no_proxy != "" %} + - name: NO_PROXY + value: "{{ cert_manager_no_proxy }}" +{% endif %} +{% if cert_manager_trusted_internal_ca is defined %} + volumeMounts: + - mountPath: /etc/ssl/certs/internal-ca.pem + name: ca-internal-truststore + subPath: internal-ca.pem + volumes: + - configMap: + defaultMode: 420 + name: ca-internal-truststore + name: ca-internal-truststore +{% endif %} +{% if cert_manager_tolerations %} + tolerations: + {{ cert_manager_tolerations | to_nice_yaml(indent=2) | indent(width=8) }} +{% endif %} +{% if cert_manager_nodeselector %} + nodeSelector: + {{ cert_manager_nodeselector | to_nice_yaml | indent(width=8) }} +{% endif %} +{% if cert_manager_affinity %} + affinity: + {{ cert_manager_affinity | to_nice_yaml | indent(width=8) }} +{% endif %} +{% if cert_manager_dns_policy %} + dnsPolicy: {{ cert_manager_dns_policy }} +{% endif %} +{% if cert_manager_dns_config %} + dnsConfig: + {{ cert_manager_dns_config | to_nice_yaml | indent(width=8) }} +{% endif %} +--- +# Source: cert-manager/deploy/charts/cert-manager/templates/webhook-deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cert-manager-webhook + namespace: {{ cert_manager_namespace }} + labels: + app: webhook + app.kubernetes.io/name: webhook + app.kubernetes.io/instance: cert-manager + app.kubernetes.io/component: "webhook" + app.kubernetes.io/version: "{{ cert_manager_version }}" +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: webhook + app.kubernetes.io/instance: cert-manager + app.kubernetes.io/component: "webhook" + template: + metadata: + labels: + app: webhook + app.kubernetes.io/name: webhook + app.kubernetes.io/instance: cert-manager + app.kubernetes.io/component: "webhook" + app.kubernetes.io/version: "{{ cert_manager_version }}" + spec: + serviceAccountName: cert-manager-webhook + enableServiceLinks: false + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - name: cert-manager-webhook + image: "{{ cert_manager_webhook_image_repo }}:{{ cert_manager_webhook_image_tag }}" + imagePullPolicy: {{ k8s_image_pull_policy }} + args: + - --v=2 + - --secure-port=10250 + - --dynamic-serving-ca-secret-namespace=$(POD_NAMESPACE) + - --dynamic-serving-ca-secret-name=cert-manager-webhook-ca + - --dynamic-serving-dns-names=cert-manager-webhook + - --dynamic-serving-dns-names=cert-manager-webhook.$(POD_NAMESPACE) + - --dynamic-serving-dns-names=cert-manager-webhook.$(POD_NAMESPACE).svc + ports: + - name: https + protocol: TCP + containerPort: 10250 + - name: healthcheck + protocol: TCP + containerPort: 6080 + livenessProbe: + httpGet: + path: /livez + port: 6080 + scheme: HTTP + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /healthz + port: 6080 + scheme: HTTP + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace +{% if cert_manager_http_proxy is defined and cert_manager_http_proxy != "" %} + - name: HTTP_PROXY + value: "{{ cert_manager_http_proxy }}" +{% endif %} +{% if cert_manager_https_proxy is defined and cert_manager_https_proxy != "" %} + - name: HTTPS_PROXY + value: "{{ cert_manager_https_proxy }}" +{% endif %} +{% if cert_manager_no_proxy is defined and cert_manager_no_proxy != "" %} + - name: NO_PROXY + value: "{{ cert_manager_no_proxy }}" +{% endif %} +{% if cert_manager_tolerations %} + tolerations: + {{ cert_manager_tolerations | to_nice_yaml(indent=2) | indent(width=8) }} +{% endif %} +{% if cert_manager_nodeselector %} + nodeSelector: + {{ cert_manager_nodeselector | to_nice_yaml | indent(width=8) }} +{% endif %} +{% if cert_manager_affinity %} + affinity: + {{ cert_manager_affinity | to_nice_yaml | indent(width=8) }} +{% endif %} +--- +# Source: cert-manager/deploy/charts/cert-manager/templates/webhook-mutating-webhook.yaml +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: cert-manager-webhook + labels: + app: webhook + app.kubernetes.io/name: webhook + app.kubernetes.io/instance: cert-manager + app.kubernetes.io/component: "webhook" + app.kubernetes.io/version: "{{ cert_manager_version }}" + annotations: + cert-manager.io/inject-ca-from-secret: "{{ cert_manager_namespace }}/cert-manager-webhook-ca" +webhooks: + - name: webhook.cert-manager.io + rules: + - apiGroups: + - "cert-manager.io" + - "acme.cert-manager.io" + apiVersions: + - "v1" + operations: + - CREATE + - UPDATE + resources: + - "*/*" + admissionReviewVersions: ["v1"] + # This webhook only accepts v1 cert-manager resources. + # Equivalent matchPolicy ensures that non-v1 resource requests are sent to + # this webhook (after the resources have been converted to v1). + matchPolicy: Equivalent + timeoutSeconds: 10 + failurePolicy: Fail + # Only include 'sideEffects' field in Kubernetes 1.12+ + sideEffects: None + clientConfig: + service: + name: cert-manager-webhook + namespace: {{ cert_manager_namespace }} + path: /mutate +--- +# Source: cert-manager/deploy/charts/cert-manager/templates/webhook-validating-webhook.yaml +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: cert-manager-webhook + labels: + app: webhook + app.kubernetes.io/name: webhook + app.kubernetes.io/instance: cert-manager + app.kubernetes.io/component: "webhook" + app.kubernetes.io/version: "{{ cert_manager_version }}" + annotations: + cert-manager.io/inject-ca-from-secret: "{{ cert_manager_namespace }}/cert-manager-webhook-ca" +webhooks: + - name: webhook.cert-manager.io + namespaceSelector: + matchExpressions: + - key: "cert-manager.io/disable-validation" + operator: "NotIn" + values: + - "true" + rules: + - apiGroups: + - "cert-manager.io" + - "acme.cert-manager.io" + apiVersions: + - "v1" + operations: + - CREATE + - UPDATE + resources: + - "*/*" + admissionReviewVersions: ["v1"] + # This webhook only accepts v1 cert-manager resources. + # Equivalent matchPolicy ensures that non-v1 resource requests are sent to + # this webhook (after the resources have been converted to v1). + matchPolicy: Equivalent + timeoutSeconds: 10 + failurePolicy: Fail + sideEffects: None + clientConfig: + service: + name: cert-manager-webhook + namespace: {{ cert_manager_namespace }} + path: /validate diff --git a/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/defaults/main.yml b/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/defaults/main.yml new file mode 100644 index 0000000..c83c8c3 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/defaults/main.yml @@ -0,0 +1,23 @@ +--- +ingress_nginx_namespace: "ingress-nginx" +ingress_nginx_host_network: false +ingress_nginx_service_type: LoadBalancer +ingress_publish_status_address: "" +ingress_nginx_nodeselector: + kubernetes.io/os: "linux" +ingress_nginx_tolerations: [] +ingress_nginx_insecure_port: 80 +ingress_nginx_secure_port: 443 +ingress_nginx_metrics_port: 10254 +ingress_nginx_configmap: {} +ingress_nginx_configmap_tcp_services: {} +ingress_nginx_configmap_udp_services: {} +ingress_nginx_extra_args: [] +ingress_nginx_termination_grace_period_seconds: 300 +ingress_nginx_class: nginx +ingress_nginx_without_class: true +ingress_nginx_default: false +ingress_nginx_webhook_enabled: false +ingress_nginx_webhook_job_ttl: 1800 + +ingress_nginx_probe_initial_delay_seconds: 10 diff --git a/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/tasks/main.yml b/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/tasks/main.yml new file mode 100644 index 0000000..ac64c82 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/tasks/main.yml @@ -0,0 +1,68 @@ +--- + +- name: NGINX Ingress Controller | Create addon dir + file: + path: "{{ kube_config_dir }}/addons/ingress_nginx" + state: directory + owner: root + group: root + mode: 0755 + when: + - inventory_hostname == groups['kube_control_plane'][0] + +- name: NGINX Ingress Controller | Templates list + set_fact: + ingress_nginx_templates: + - { name: 00-namespace, file: 00-namespace.yml, type: ns } + - { name: cm-ingress-nginx, file: cm-ingress-nginx.yml, type: cm } + - { name: cm-tcp-services, file: cm-tcp-services.yml, type: cm } + - { name: cm-udp-services, file: cm-udp-services.yml, type: cm } + - { name: sa-ingress-nginx, file: sa-ingress-nginx.yml, type: sa } + - { name: clusterrole-ingress-nginx, file: clusterrole-ingress-nginx.yml, type: clusterrole } + - { name: clusterrolebinding-ingress-nginx, file: clusterrolebinding-ingress-nginx.yml, type: clusterrolebinding } + - { name: role-ingress-nginx, file: role-ingress-nginx.yml, type: role } + - { name: rolebinding-ingress-nginx, file: rolebinding-ingress-nginx.yml, type: rolebinding } + - { name: ingressclass-nginx, file: ingressclass-nginx.yml, type: ingressclass } + - { name: ds-ingress-nginx-controller, file: ds-ingress-nginx-controller.yml, type: ds } + ingress_nginx_template_for_service: + - { name: svc-ingress-nginx, file: svc-ingress-nginx.yml, type: svc } + ingress_nginx_templates_for_webhook: + - { name: admission-webhook-configuration, file: admission-webhook-configuration.yml, type: sa } + - { name: sa-admission-webhook, file: sa-admission-webhook.yml, type: sa } + - { name: clusterrole-admission-webhook, file: clusterrole-admission-webhook.yml, type: clusterrole } + - { name: clusterrolebinding-admission-webhook, file: clusterrolebinding-admission-webhook.yml, type: clusterrolebinding } + - { name: role-admission-webhook, file: role-admission-webhook.yml, type: role } + - { name: rolebinding-admission-webhook, file: rolebinding-admission-webhook.yml, type: rolebinding } + - { name: admission-webhook-job, file: admission-webhook-job.yml, type: job } + +- name: NGINX Ingress Controller | Append extra templates to NGINX Ingress Template list for service + set_fact: + ingress_nginx_templates: "{{ ingress_nginx_templates + ingress_nginx_template_for_service }}" + when: not ingress_nginx_host_network + +- name: NGINX Ingress Controller | Append extra templates to NGINX Ingress Templates list for webhook + set_fact: + ingress_nginx_templates: "{{ ingress_nginx_templates + ingress_nginx_templates_for_webhook }}" + when: ingress_nginx_webhook_enabled + +- name: NGINX Ingress Controller | Create manifests + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/addons/ingress_nginx/{{ item.file }}" + mode: 0644 + with_items: "{{ ingress_nginx_templates }}" + register: ingress_nginx_manifests + when: + - inventory_hostname == groups['kube_control_plane'][0] + +- name: NGINX Ingress Controller | Apply manifests + kube: + name: "{{ item.item.name }}" + namespace: "{{ ingress_nginx_namespace }}" + kubectl: "{{ bin_dir }}/kubectl" + resource: "{{ item.item.type }}" + filename: "{{ kube_config_dir }}/addons/ingress_nginx/{{ item.item.file }}" + state: "latest" + with_items: "{{ ingress_nginx_manifests.results }}" + when: + - inventory_hostname == groups['kube_control_plane'][0] diff --git a/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/00-namespace.yml.j2 b/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/00-namespace.yml.j2 new file mode 100644 index 0000000..1f12366 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/00-namespace.yml.j2 @@ -0,0 +1,7 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: {{ ingress_nginx_namespace }} + labels: + name: {{ ingress_nginx_namespace }} diff --git a/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/admission-webhook-configuration.yml.j2 b/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/admission-webhook-configuration.yml.j2 new file mode 100644 index 0000000..d6878a0 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/admission-webhook-configuration.yml.j2 @@ -0,0 +1,29 @@ +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + name: ingress-nginx-admission +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: ingress-nginx-controller-admission + namespace: {{ ingress_nginx_namespace }} + path: /networking/v1/ingresses + failurePolicy: Fail + matchPolicy: Equivalent + name: validate.nginx.ingress.kubernetes.io + rules: + - apiGroups: + - networking.k8s.io + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - ingresses + sideEffects: None diff --git a/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/admission-webhook-job.yml.j2 b/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/admission-webhook-job.yml.j2 new file mode 100644 index 0000000..258a7a1 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/admission-webhook-job.yml.j2 @@ -0,0 +1,86 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + name: ingress-nginx-admission-create + namespace: {{ ingress_nginx_namespace }} +spec: + template: + metadata: + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + name: ingress-nginx-admission-create + spec: + containers: + - args: + - create + - --host=ingress-nginx-controller-admission,ingress-nginx-controller-admission.$(POD_NAMESPACE).svc + - --namespace=$(POD_NAMESPACE) + - --secret-name=ingress-nginx-admission + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: "{{ ingress_nginx_kube_webhook_certgen_image_repo }}:{{ ingress_nginx_kube_webhook_certgen_image_tag }}" + imagePullPolicy: {{ k8s_image_pull_policy }} + name: create + securityContext: + allowPrivilegeEscalation: false + nodeSelector: + kubernetes.io/os: linux + restartPolicy: OnFailure + securityContext: + fsGroup: 2000 + runAsNonRoot: true + runAsUser: 2000 + serviceAccountName: ingress-nginx-admission + ttlSecondsAfterFinished: {{ ingress_nginx_webhook_job_ttl }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + name: ingress-nginx-admission-patch + namespace: {{ ingress_nginx_namespace }} +spec: + template: + metadata: + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + name: ingress-nginx-admission-patch + spec: + containers: + - args: + - patch + - --webhook-name=ingress-nginx-admission + - --namespace=$(POD_NAMESPACE) + - --patch-mutating=false + - --secret-name=ingress-nginx-admission + - --patch-failure-policy=Fail + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: "{{ ingress_nginx_kube_webhook_certgen_image_repo }}:{{ ingress_nginx_kube_webhook_certgen_image_tag }}" + imagePullPolicy: {{ k8s_image_pull_policy }} + name: patch + securityContext: + allowPrivilegeEscalation: false + nodeSelector: + kubernetes.io/os: linux + restartPolicy: OnFailure + securityContext: + fsGroup: 2000 + runAsNonRoot: true + runAsUser: 2000 + serviceAccountName: ingress-nginx-admission + ttlSecondsAfterFinished: {{ ingress_nginx_webhook_job_ttl }} diff --git a/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/clusterrole-admission-webhook.yml.j2 b/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/clusterrole-admission-webhook.yml.j2 new file mode 100644 index 0000000..daa4753 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/clusterrole-admission-webhook.yml.j2 @@ -0,0 +1,15 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + name: ingress-nginx-admission +rules: +- apiGroups: + - admissionregistration.k8s.io + resources: + - validatingwebhookconfigurations + verbs: + - get + - update diff --git a/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/clusterrole-ingress-nginx.yml.j2 b/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/clusterrole-ingress-nginx.yml.j2 new file mode 100644 index 0000000..38118bf --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/clusterrole-ingress-nginx.yml.j2 @@ -0,0 +1,36 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: ingress-nginx + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx +rules: + - apiGroups: [""] + resources: ["configmaps", "endpoints", "nodes", "pods", "secrets", "namespaces"] + verbs: ["list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get"] + - apiGroups: [""] + resources: ["services"] + verbs: ["get", "list", "watch"] + - apiGroups: ["networking.k8s.io"] + resources: ["ingresses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "patch"] + - apiGroups: ["extensions","networking.k8s.io"] + resources: ["ingresses/status"] + verbs: ["update"] + - apiGroups: ["networking.k8s.io"] + resources: ["ingressclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["list", "watch"] + - apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["get", "list", "watch"] diff --git a/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/clusterrolebinding-admission-webhook.yml.j2 b/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/clusterrolebinding-admission-webhook.yml.j2 new file mode 100644 index 0000000..8791594 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/clusterrolebinding-admission-webhook.yml.j2 @@ -0,0 +1,16 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + name: ingress-nginx-admission +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ingress-nginx-admission +subjects: + - kind: ServiceAccount + name: ingress-nginx-admission + namespace: {{ ingress_nginx_namespace }} diff --git a/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/clusterrolebinding-ingress-nginx.yml.j2 b/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/clusterrolebinding-ingress-nginx.yml.j2 new file mode 100644 index 0000000..ad83dc2 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/clusterrolebinding-ingress-nginx.yml.j2 @@ -0,0 +1,16 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: ingress-nginx + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ingress-nginx +subjects: + - kind: ServiceAccount + name: ingress-nginx + namespace: {{ ingress_nginx_namespace }} diff --git a/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/cm-ingress-nginx.yml.j2 b/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/cm-ingress-nginx.yml.j2 new file mode 100644 index 0000000..9f1e3bb --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/cm-ingress-nginx.yml.j2 @@ -0,0 +1,13 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ingress-nginx + namespace: {{ ingress_nginx_namespace }} + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx +{% if ingress_nginx_configmap %} +data: + {{ ingress_nginx_configmap | to_nice_yaml | indent(2) }} +{%- endif %} diff --git a/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/cm-tcp-services.yml.j2 b/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/cm-tcp-services.yml.j2 new file mode 100644 index 0000000..9752081 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/cm-tcp-services.yml.j2 @@ -0,0 +1,13 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: tcp-services + namespace: {{ ingress_nginx_namespace }} + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx +{% if ingress_nginx_configmap_tcp_services %} +data: + {{ ingress_nginx_configmap_tcp_services | to_nice_yaml | indent(2) }} +{%- endif %} diff --git a/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/cm-udp-services.yml.j2 b/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/cm-udp-services.yml.j2 new file mode 100644 index 0000000..a3f6613 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/cm-udp-services.yml.j2 @@ -0,0 +1,13 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: udp-services + namespace: {{ ingress_nginx_namespace }} + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx +{% if ingress_nginx_configmap_udp_services %} +data: + {{ ingress_nginx_configmap_udp_services | to_nice_yaml | indent(2) }} +{%- endif %} diff --git a/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ds-ingress-nginx-controller.yml.j2 b/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ds-ingress-nginx-controller.yml.j2 new file mode 100644 index 0000000..7f41935 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ds-ingress-nginx-controller.yml.j2 @@ -0,0 +1,141 @@ +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: ingress-nginx-controller + namespace: {{ ingress_nginx_namespace }} + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx +spec: + selector: + matchLabels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + template: + metadata: + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + annotations: + prometheus.io/port: "10254" + prometheus.io/scrape: "true" + spec: + serviceAccountName: ingress-nginx + terminationGracePeriodSeconds: {{ ingress_nginx_termination_grace_period_seconds }} +{% if ingress_nginx_host_network %} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet +{% endif %} +{% if ingress_nginx_nodeselector %} + nodeSelector: + {{ ingress_nginx_nodeselector | to_nice_yaml | indent(width=8) }} +{%- endif %} +{% if ingress_nginx_tolerations %} + tolerations: + {{ ingress_nginx_tolerations | to_nice_yaml(indent=2) | indent(width=8) }} +{% endif %} + priorityClassName: {% if ingress_nginx_namespace == 'kube-system' %}system-node-critical{% else %}k8s-cluster-critical{% endif %}{{ '' }} + containers: + - name: ingress-nginx-controller + image: {{ ingress_nginx_controller_image_repo }}:{{ ingress_nginx_controller_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + lifecycle: + preStop: + exec: + command: + - /wait-shutdown + args: + - /nginx-ingress-controller + - --configmap=$(POD_NAMESPACE)/ingress-nginx + - --election-id=ingress-controller-leader-{{ ingress_nginx_class }} + - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services + - --udp-services-configmap=$(POD_NAMESPACE)/udp-services + - --annotations-prefix=nginx.ingress.kubernetes.io + - --ingress-class={{ ingress_nginx_class }} +{% if ingress_nginx_without_class %} + - --watch-ingress-without-class=true +{% endif %} +{% if ingress_nginx_host_network %} + - --report-node-internal-ip-address +{% endif %} +{% if ingress_publish_status_address != "" %} + - --publish-status-address={{ ingress_publish_status_address }} +{% endif %} +{% for extra_arg in ingress_nginx_extra_args %} + - {{ extra_arg }} +{% endfor %} +{% if ingress_nginx_webhook_enabled %} + - --validating-webhook=:8443 + - --validating-webhook-certificate=/usr/local/certificates/cert + - --validating-webhook-key=/usr/local/certificates/key +{% endif %} + securityContext: + capabilities: + drop: + - ALL + add: + - NET_BIND_SERVICE + # www-data -> 101 + runAsUser: 101 + allowPrivilegeEscalation: true + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: LD_PRELOAD + value: /usr/local/lib/libmimalloc.so + ports: + - name: http + containerPort: 80 + hostPort: {{ ingress_nginx_insecure_port }} + - name: https + containerPort: 443 + hostPort: {{ ingress_nginx_secure_port }} + - name: metrics + containerPort: 10254 +{% if not ingress_nginx_host_network %} + hostPort: {{ ingress_nginx_metrics_port }} +{% endif %} +{% if ingress_nginx_webhook_enabled %} + - name: webhook + containerPort: 8443 + protocol: TCP +{% endif %} + livenessProbe: + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: {{ ingress_nginx_probe_initial_delay_seconds }} + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: {{ ingress_nginx_probe_initial_delay_seconds }} + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 +{% if ingress_nginx_webhook_enabled %} + volumeMounts: + - mountPath: /usr/local/certificates/ + name: webhook-cert + readOnly: true +{% endif %} +{% if ingress_nginx_webhook_enabled %} + volumes: + - name: webhook-cert + secret: + secretName: ingress-nginx-admission +{% endif %} diff --git a/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingressclass-nginx.yml.j2 b/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingressclass-nginx.yml.j2 new file mode 100644 index 0000000..0e5fa8c --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingressclass-nginx.yml.j2 @@ -0,0 +1,13 @@ +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + name: {{ ingress_nginx_class }} + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx +{% if ingress_nginx_default %} + annotations: + ingressclass.kubernetes.io/is-default-class: "true" +{% endif %} +spec: + controller: k8s.io/ingress-nginx diff --git a/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/role-admission-webhook.yml.j2 b/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/role-admission-webhook.yml.j2 new file mode 100644 index 0000000..5d1bb01 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/role-admission-webhook.yml.j2 @@ -0,0 +1,17 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + name: ingress-nginx-admission + namespace: {{ ingress_nginx_namespace }} +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - create diff --git a/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/role-ingress-nginx.yml.j2 b/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/role-ingress-nginx.yml.j2 new file mode 100644 index 0000000..695b07f --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/role-ingress-nginx.yml.j2 @@ -0,0 +1,47 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: ingress-nginx + namespace: {{ ingress_nginx_namespace }} + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx +rules: + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["get"] + - apiGroups: [""] + resources: ["configmaps", "pods", "secrets", "endpoints"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["services"] + verbs: ["get", "list", "watch"] + - apiGroups: ["networking.k8s.io"] + resources: ["ingresses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["networking.k8s.io"] + resources: ["ingresses/status"] + verbs: ["update"] + - apiGroups: ["networking.k8s.io"] + resources: ["ingressclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + # Defaults to "", defined in + # ds-ingress-nginx-controller.yml.js + # by a command-line argument. + # + # This is the correct behaviour for ingress-controller + # version 1.8.1 + resourceNames: ["ingress-controller-leader-{{ ingress_nginx_class }}"] + verbs: ["get", "update"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["create"] + - apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["get", "list", "watch"] diff --git a/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/rolebinding-admission-webhook.yml.j2 b/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/rolebinding-admission-webhook.yml.j2 new file mode 100644 index 0000000..671912d --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/rolebinding-admission-webhook.yml.j2 @@ -0,0 +1,17 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + name: ingress-nginx-admission + namespace: {{ ingress_nginx_namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ingress-nginx-admission +subjects: +- kind: ServiceAccount + name: ingress-nginx-admission + namespace: {{ ingress_nginx_namespace }} diff --git a/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/rolebinding-ingress-nginx.yml.j2 b/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/rolebinding-ingress-nginx.yml.j2 new file mode 100644 index 0000000..142d400 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/rolebinding-ingress-nginx.yml.j2 @@ -0,0 +1,17 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: ingress-nginx + namespace: {{ ingress_nginx_namespace }} + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ingress-nginx +subjects: + - kind: ServiceAccount + name: ingress-nginx + namespace: {{ ingress_nginx_namespace }} diff --git a/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/sa-admission-webhook.yml.j2 b/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/sa-admission-webhook.yml.j2 new file mode 100644 index 0000000..488a045 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/sa-admission-webhook.yml.j2 @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ingress-nginx-admission + namespace: {{ ingress_nginx_namespace }} + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx diff --git a/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/sa-ingress-nginx.yml.j2 b/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/sa-ingress-nginx.yml.j2 new file mode 100644 index 0000000..305d553 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/sa-ingress-nginx.yml.j2 @@ -0,0 +1,9 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ingress-nginx + namespace: {{ ingress_nginx_namespace }} + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx diff --git a/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/svc-ingress-nginx.yml.j2 b/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/svc-ingress-nginx.yml.j2 new file mode 100644 index 0000000..115232f --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/svc-ingress-nginx.yml.j2 @@ -0,0 +1,24 @@ +{% if not ingress_nginx_host_network %} +apiVersion: v1 +kind: Service +metadata: + name: ingress-nginx + namespace: {{ ingress_nginx_namespace }} + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx +spec: + type: {{ ingress_nginx_service_type }} + ports: + - name: http + port: 80 + targetPort: 80 + protocol: TCP + - name: https + port: 443 + targetPort: 443 + protocol: TCP + selector: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/port-of: ingress-nginx +{% endif %} diff --git a/kubespray/project/roles/kubernetes-apps/ingress_controller/meta/main.yml b/kubespray/project/roles/kubernetes-apps/ingress_controller/meta/main.yml new file mode 100644 index 0000000..b269607 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/ingress_controller/meta/main.yml @@ -0,0 +1,22 @@ +--- +dependencies: + - role: kubernetes-apps/ingress_controller/ingress_nginx + when: ingress_nginx_enabled + tags: + - apps + - ingress-controller + - ingress-nginx + + - role: kubernetes-apps/ingress_controller/cert_manager + when: cert_manager_enabled + tags: + - apps + - ingress-controller + - cert-manager + + - role: kubernetes-apps/ingress_controller/alb_ingress_controller + when: ingress_alb_enabled + tags: + - apps + - ingress-controller + - ingress_alb diff --git a/kubespray/project/roles/kubernetes-apps/krew/defaults/main.yml b/kubespray/project/roles/kubernetes-apps/krew/defaults/main.yml new file mode 100644 index 0000000..6878427 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/krew/defaults/main.yml @@ -0,0 +1,5 @@ +--- +krew_enabled: false +krew_root_dir: "/usr/local/krew" +krew_default_index_uri: https://github.com/kubernetes-sigs/krew-index.git +krew_no_upgrade_check: 0 diff --git a/kubespray/project/roles/kubernetes-apps/krew/tasks/krew.yml b/kubespray/project/roles/kubernetes-apps/krew/tasks/krew.yml new file mode 100644 index 0000000..a8b5201 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/krew/tasks/krew.yml @@ -0,0 +1,38 @@ +--- +- name: Krew | Download krew + include_tasks: "../../../download/tasks/download_file.yml" + vars: + download: "{{ download_defaults | combine(downloads.krew) }}" + +- name: Krew | krew env + template: + src: krew.j2 + dest: /etc/bash_completion.d/krew + mode: 0644 + +- name: Krew | Copy krew manifest + template: + src: krew.yml.j2 + dest: "{{ local_release_dir }}/krew.yml" + mode: 0644 + +- name: Krew | Install krew # noqa command-instead-of-shell + shell: "{{ local_release_dir }}/krew-{{ host_os }}_{{ image_arch }} install --archive={{ local_release_dir }}/krew-{{ host_os }}_{{ image_arch }}.tar.gz --manifest={{ local_release_dir }}/krew.yml" + environment: + KREW_ROOT: "{{ krew_root_dir }}" + KREW_DEFAULT_INDEX_URI: "{{ krew_default_index_uri | default('') }}" + +- name: Krew | Get krew completion + command: "{{ local_release_dir }}/krew-{{ host_os }}_{{ image_arch }} completion bash" + changed_when: False + register: krew_completion + check_mode: False + ignore_errors: yes # noqa ignore-errors + +- name: Krew | Install krew completion + copy: + dest: /etc/bash_completion.d/krew.sh + content: "{{ krew_completion.stdout }}" + mode: 0755 + become: True + when: krew_completion.rc == 0 diff --git a/kubespray/project/roles/kubernetes-apps/krew/tasks/main.yml b/kubespray/project/roles/kubernetes-apps/krew/tasks/main.yml new file mode 100644 index 0000000..40729e8 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/krew/tasks/main.yml @@ -0,0 +1,10 @@ +--- +- name: Krew | install krew on kube_control_plane + import_tasks: krew.yml + +- name: Krew | install krew on localhost + import_tasks: krew.yml + delegate_to: localhost + connection: local + run_once: true + when: kubectl_localhost diff --git a/kubespray/project/roles/kubernetes-apps/krew/templates/krew.j2 b/kubespray/project/roles/kubernetes-apps/krew/templates/krew.j2 new file mode 100644 index 0000000..62a744c --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/krew/templates/krew.j2 @@ -0,0 +1,7 @@ +# krew bash env(kubespray) +export KREW_ROOT="{{ krew_root_dir }}" +{% if krew_default_index_uri is defined %} +export KREW_DEFAULT_INDEX_URI='{{ krew_default_index_uri }}' +{% endif %} +export PATH="${KREW_ROOT:-$HOME/.krew}/bin:$PATH" +export KREW_NO_UPGRADE_CHECK={{ krew_no_upgrade_check }} diff --git a/kubespray/project/roles/kubernetes-apps/krew/templates/krew.yml.j2 b/kubespray/project/roles/kubernetes-apps/krew/templates/krew.yml.j2 new file mode 100644 index 0000000..b0c6152 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/krew/templates/krew.yml.j2 @@ -0,0 +1,100 @@ +apiVersion: krew.googlecontainertools.github.com/v1alpha2 +kind: Plugin +metadata: + name: krew +spec: + version: "{{ krew_version }}" + homepage: https://krew.sigs.k8s.io/ + shortDescription: Package manager for kubectl plugins. + caveats: | + krew is now installed! To start using kubectl plugins, you need to add + krew's installation directory to your PATH: + + * macOS/Linux: + - Add the following to your ~/.bashrc or ~/.zshrc: + export PATH="${KREW_ROOT:-$HOME/.krew}/bin:$PATH" + - Restart your shell. + + * Windows: Add %USERPROFILE%\.krew\bin to your PATH environment variable + + To list krew commands and to get help, run: + $ kubectl krew + For a full list of available plugins, run: + $ kubectl krew search + + You can find documentation at + https://krew.sigs.k8s.io/docs/user-guide/quickstart/. + + platforms: + - uri: {{ krew_download_url }} + sha256: {{ krew_archive_checksum }} + bin: krew + files: + - from: ./krew-darwin_amd64 + to: krew + - from: ./LICENSE + to: . + selector: + matchLabels: + os: darwin + arch: amd64 + - uri: {{ krew_download_url }} + sha256: {{ krew_archive_checksum }} + bin: krew + files: + - from: ./krew-darwin_arm64 + to: krew + - from: ./LICENSE + to: . + selector: + matchLabels: + os: darwin + arch: arm64 + - uri: {{ krew_download_url }} + sha256: {{ krew_archive_checksum }} + bin: krew + files: + - from: ./krew-linux_amd64 + to: krew + - from: ./LICENSE + to: . + selector: + matchLabels: + os: linux + arch: amd64 + - uri: {{ krew_download_url }} + sha256: {{ krew_archive_checksum }} + bin: krew + files: + - from: ./krew-linux_arm + to: krew + - from: ./LICENSE + to: . + selector: + matchLabels: + os: linux + arch: arm + - uri: {{ krew_download_url }} + sha256: {{ krew_archive_checksum }} + bin: krew + files: + - from: ./krew-linux_arm64 + to: krew + - from: ./LICENSE + to: . + selector: + matchLabels: + os: linux + arch: arm64 + - uri: {{ krew_download_url }} + sha256: {{ krew_archive_checksum }} + bin: krew.exe + files: + - from: ./krew-windows_amd64.exe + to: krew.exe + - from: ./LICENSE + to: . + selector: + matchLabels: + os: windows + arch: amd64 diff --git a/kubespray/project/roles/kubernetes-apps/kubelet-csr-approver/defaults/main.yml b/kubespray/project/roles/kubernetes-apps/kubelet-csr-approver/defaults/main.yml new file mode 100644 index 0000000..2edce70 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/kubelet-csr-approver/defaults/main.yml @@ -0,0 +1,12 @@ +--- +kubelet_csr_approver_enabled: "{{ kubelet_rotate_server_certificates }}" +kubelet_csr_approver_namespace: kube-system + +kubelet_csr_approver_repository_name: kubelet-csr-approver +kubelet_csr_approver_repository_url: https://postfinance.github.io/kubelet-csr-approver +kubelet_csr_approver_chart_ref: "{{ kubelet_csr_approver_repository_name }}/kubelet-csr-approver" +kubelet_csr_approver_chart_version: 0.2.8 + +# Fill values override here +# See upstream https://github.com/postfinance/kubelet-csr-approver +kubelet_csr_approver_values: {} diff --git a/kubespray/project/roles/kubernetes-apps/kubelet-csr-approver/meta/main.yml b/kubespray/project/roles/kubernetes-apps/kubelet-csr-approver/meta/main.yml new file mode 100644 index 0000000..93d1383 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/kubelet-csr-approver/meta/main.yml @@ -0,0 +1,20 @@ +--- +dependencies: + - role: helm-apps + when: + - inventory_hostname == groups['kube_control_plane'][0] + - kubelet_csr_approver_enabled + environment: + http_proxy: "{{ http_proxy | default('') }}" + https_proxy: "{{ https_proxy | default('') }}" + release_common_opts: {} + releases: + - name: kubelet-csr-approver + namespace: "{{ kubelet_csr_approver_namespace }}" + chart_ref: "{{ kubelet_csr_approver_chart_ref }}" + chart_version: "{{ kubelet_csr_approver_chart_version }}" + wait: true + values: "{{ kubelet_csr_approver_values }}" + repositories: + - name: "{{ kubelet_csr_approver_repository_name }}" + url: "{{ kubelet_csr_approver_repository_url }}" diff --git a/kubespray/project/roles/kubernetes-apps/meta/main.yml b/kubespray/project/roles/kubernetes-apps/meta/main.yml new file mode 100644 index 0000000..1b9cd6b --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/meta/main.yml @@ -0,0 +1,134 @@ +--- +dependencies: + - role: kubernetes-apps/ansible + when: + - inventory_hostname == groups['kube_control_plane'][0] + + - role: kubernetes-apps/helm + when: + - helm_enabled + tags: + - helm + + - role: kubernetes-apps/krew + when: + - krew_enabled + tags: + - krew + + - role: kubernetes-apps/registry + when: + - registry_enabled + - inventory_hostname == groups['kube_control_plane'][0] + tags: + - registry + + - role: kubernetes-apps/metrics_server + when: + - metrics_server_enabled + - inventory_hostname == groups['kube_control_plane'][0] + tags: + - metrics_server + + - role: kubernetes-apps/csi_driver/csi_crd + when: + - cinder_csi_enabled or csi_snapshot_controller_enabled + - inventory_hostname == groups['kube_control_plane'][0] + tags: + - csi-driver + + - role: kubernetes-apps/csi_driver/cinder + when: + - cinder_csi_enabled + tags: + - cinder-csi-driver + - csi-driver + + - role: kubernetes-apps/csi_driver/aws_ebs + when: + - aws_ebs_csi_enabled + tags: + - aws-ebs-csi-driver + - csi-driver + + - role: kubernetes-apps/csi_driver/azuredisk + when: + - azure_csi_enabled + tags: + - azure-csi-driver + - csi-driver + + - role: kubernetes-apps/csi_driver/gcp_pd + when: + - gcp_pd_csi_enabled + tags: + - gcp-pd-csi-driver + - csi-driver + + - role: kubernetes-apps/csi_driver/upcloud + when: + - upcloud_csi_enabled + tags: + - upcloud-csi-driver + - csi-driver + + - role: kubernetes-apps/csi_driver/vsphere + when: + - vsphere_csi_enabled + tags: + - vsphere-csi-driver + - csi-driver + + - role: kubernetes-apps/persistent_volumes + when: + - persistent_volumes_enabled + - inventory_hostname == groups['kube_control_plane'][0] + tags: + - persistent_volumes + + - role: kubernetes-apps/snapshots + when: inventory_hostname == groups['kube_control_plane'][0] + tags: + - snapshots + - csi-driver + + - role: kubernetes-apps/container_runtimes + when: + - inventory_hostname == groups['kube_control_plane'][0] + tags: + - container-runtimes + + - role: kubernetes-apps/container_engine_accelerator + when: nvidia_accelerator_enabled + tags: + - container_engine_accelerator + + - role: kubernetes-apps/cloud_controller/oci + when: + - cloud_provider is defined + - cloud_provider == "oci" + - inventory_hostname == groups['kube_control_plane'][0] + tags: + - oci + + - role: kubernetes-apps/metallb + when: + - metallb_enabled + - inventory_hostname == groups['kube_control_plane'][0] + tags: + - metallb + + - role: kubernetes-apps/argocd + when: + - argocd_enabled + - inventory_hostname == groups['kube_control_plane'][0] + tags: + - argocd + + - role: kubernetes-apps/scheduler_plugins + when: + - scheduler_plugins_enabled + - kube_major_version is version('v1.28', '<') + - inventory_hostname == groups['kube_control_plane'][0] + tags: + - scheduler_plugins diff --git a/kubespray/project/roles/kubernetes-apps/metallb/OWNERS b/kubespray/project/roles/kubernetes-apps/metallb/OWNERS new file mode 100644 index 0000000..b64c7bc --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/metallb/OWNERS @@ -0,0 +1,5 @@ +# See the OWNERS docs at https://go.k8s.io/owners + +approvers: +reviewers: + - oomichi diff --git a/kubespray/project/roles/kubernetes-apps/metallb/defaults/main.yml b/kubespray/project/roles/kubernetes-apps/metallb/defaults/main.yml new file mode 100644 index 0000000..02f4e3c --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/metallb/defaults/main.yml @@ -0,0 +1,16 @@ +--- +metallb_enabled: false +metallb_log_level: info +metallb_port: "7472" +metallb_memberlist_port: "7946" +metallb_speaker_enabled: "{{ metallb_enabled }}" +metallb_speaker_nodeselector: + kubernetes.io/os: "linux" +metallb_controller_nodeselector: + kubernetes.io/os: "linux" +metallb_speaker_tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists +metallb_controller_tolerations: [] +metallb_loadbalancer_class: "" \ No newline at end of file diff --git a/kubespray/project/roles/kubernetes-apps/metallb/tasks/main.yml b/kubespray/project/roles/kubernetes-apps/metallb/tasks/main.yml new file mode 100644 index 0000000..221fe3c --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/metallb/tasks/main.yml @@ -0,0 +1,108 @@ +--- +- name: Kubernetes Apps | Check cluster settings for MetalLB + fail: + msg: "MetalLB require kube_proxy_strict_arp = true, see https://github.com/danderson/metallb/issues/153#issuecomment-518651132" + when: + - "kube_proxy_mode == 'ipvs' and not kube_proxy_strict_arp" + +- name: Kubernetes Apps | Check that the deprecated 'matallb_auto_assign' variable is not used anymore + fail: + msg: "'matallb_auto_assign' configuration variable is deprecated, please use 'metallb_auto_assign' instead" + when: + - matallb_auto_assign is defined + +- name: Kubernetes Apps | Lay Down MetalLB + become: true + template: + src: "metallb.yaml.j2" + dest: "{{ kube_config_dir }}/metallb.yaml" + mode: 0644 + register: metallb_rendering + when: + - inventory_hostname == groups['kube_control_plane'][0] + +- name: Kubernetes Apps | Install and configure MetalLB + kube: + name: "MetalLB" + kubectl: "{{ bin_dir }}/kubectl" + filename: "{{ kube_config_dir }}/metallb.yaml" + state: "{{ metallb_rendering.changed | ternary('latest', 'present') }}" + wait: true + become: true + when: + - inventory_hostname == groups['kube_control_plane'][0] + +- name: Kubernetes Apps | Wait for MetalLB controller to be running + command: "{{ bin_dir }}/kubectl rollout status -n metallb-system deployment -l app=metallb,component=controller --timeout=2m" + become: true + when: + - inventory_hostname == groups['kube_control_plane'][0] + +- name: MetalLB | Address pools + when: + - inventory_hostname == groups['kube_control_plane'][0] + - metallb_config.address_pools is defined + block: + - name: MetalLB | Layout address pools template + ansible.builtin.template: + src: pools.yaml.j2 + dest: "{{ kube_config_dir }}/pools.yaml" + mode: 0644 + register: pools_rendering + + - name: MetalLB | Create address pools configuration + kube: + name: "MetalLB" + kubectl: "{{ bin_dir }}/kubectl" + filename: "{{ kube_config_dir }}/pools.yaml" + state: "{{ pools_rendering.changed | ternary('latest', 'present') }}" + become: true + +- name: MetalLB | Layer2 + when: + - inventory_hostname == groups['kube_control_plane'][0] + - metallb_config.layer2 is defined + block: + - name: MetalLB | Layout layer2 template + ansible.builtin.template: + src: layer2.yaml.j2 + dest: "{{ kube_config_dir }}/layer2.yaml" + mode: 0644 + register: layer2_rendering + + - name: MetalLB | Create layer2 configuration + kube: + name: "MetalLB" + kubectl: "{{ bin_dir }}/kubectl" + filename: "{{ kube_config_dir }}/layer2.yaml" + state: "{{ layer2_rendering.changed | ternary('latest', 'present') }}" + become: true + +- name: MetalLB | Layer3 + when: + - inventory_hostname == groups['kube_control_plane'][0] + - metallb_config.layer3 is defined + block: + - name: MetalLB | Layout layer3 template + ansible.builtin.template: + src: layer3.yaml.j2 + dest: "{{ kube_config_dir }}/layer3.yaml" + mode: 0644 + register: layer3_rendering + + - name: MetalLB | Create layer3 configuration + kube: + name: "MetalLB" + kubectl: "{{ bin_dir }}/kubectl" + filename: "{{ kube_config_dir }}/layer3.yaml" + state: "{{ layer3_rendering.changed | ternary('latest', 'present') }}" + become: true + + +- name: Kubernetes Apps | Delete MetalLB ConfigMap + kube: + name: config + kubectl: "{{ bin_dir }}/kubectl" + resource: ConfigMap + namespace: metallb-system + state: absent diff --git a/kubespray/project/roles/kubernetes-apps/metallb/templates/layer2.yaml.j2 b/kubespray/project/roles/kubernetes-apps/metallb/templates/layer2.yaml.j2 new file mode 100644 index 0000000..d249732 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/metallb/templates/layer2.yaml.j2 @@ -0,0 +1,19 @@ +#jinja2: trim_blocks: True, lstrip_blocks: True +# yamllint disable-file +--- + +# Create layer2 configuration +{% for entry in metallb_config.layer2 %} + +--- +# L2 Configuration +apiVersion: metallb.io/v1beta1 +kind: L2Advertisement +metadata: + name: "{{ entry }}" + namespace: metallb-system +spec: + ipAddressPools: + - "{{ entry }}" + +{% endfor %} diff --git a/kubespray/project/roles/kubernetes-apps/metallb/templates/layer3.yaml.j2 b/kubespray/project/roles/kubernetes-apps/metallb/templates/layer3.yaml.j2 new file mode 100644 index 0000000..490bae2 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/metallb/templates/layer3.yaml.j2 @@ -0,0 +1,125 @@ +#jinja2: trim_blocks: True, lstrip_blocks: True +# yamllint disable-file +--- +# Create layer3 configuration +{% if metallb_config.layer3.communities is defined %} +{% for community_name, community in metallb_config.layer3.communities.items() %} +--- +apiVersion: metallb.io/v1beta1 +kind: Community +metadata: + name: "{{ community_name }}" + namespace: metallb-system +spec: + communities: + - name: "{{ community_name }}" + value: "{{ community }}" +{% endfor %} +{% endif %} +--- +apiVersion: metallb.io/v1beta1 +kind: Community +metadata: + name: well-known + namespace: metallb-system +spec: + communities: + - name: no-export + value: 65535:65281 + - name: no-advertise + value: 65535:65282 + - name: local-as + value: 65535:65283 + - name: nopeer + value: 65535:65284 + +# BGPAdvertisement is used to advertise address pools to the BGP peer. Specific pools can be listed to be advertised. +# Local BGP Advertisement specifies that the IP specified in the address pool will be used as remote source address for traffic entering your cluster from the remote peer. +# When using this option, be sure to use a subnet and routable IP for your address pool. +# This is good: 10.0.0.10/24. This is also good: 10.0.0.129/25. This is bad: 10.0.0.0/24. This is also bad: 10.0.0.128/25. +# In this example, 10.0.0.10 will be used as the remote source address. +# This is also bad: 10.0.0.10-10.0.0.25. Remember: you are working with aggregationLength, which specifies a subnet, not an IP range! +# The no-advertise community is set on the local advertisement to prevent this route from being published to the BGP peer. +# Your aggregationLength ideally is the same size as your address pool. + +{% for peer_name, peer in metallb_config.layer3.metallb_peers.items() %} + +{% if peer.aggregation_length is defined and peer.aggregation_length <= 30 %} + +--- +apiVersion: metallb.io/v1beta1 +kind: BGPAdvertisement +metadata: + name: "{{ peer_name }}-local" + namespace: metallb-system +spec: + aggregationLength: 32 + aggregationLengthV6: 128 + communities: + - no-advertise + localpref: "{{ peer.localpref | default("100") }}" + ipAddressPools: + {% for address_pool in peer.address_pool %} + - "{{ address_pool }}" + {% endfor %} +{% endif %} + +# External BGP Advertisement. The IP range specied in the address pool is advertised to the BGP peer. +--- +apiVersion: metallb.io/v1beta1 +kind: BGPAdvertisement +metadata: + name: "{{ peer_name }}-external" + namespace: metallb-system +spec: + {% if peer.aggregation_length is defined and peer.aggregation_length <= 30 %} + aggregationLength: {{ peer.aggregation_length }} + {% endif %} + ipAddressPools: + {% for address_pool in peer.address_pool %} + - "{{ address_pool }}" + {% endfor %} + {% if peer.communities is defined %} + {% for community in peer.communities %} + communities: + - "{{ community }}" + {% endfor %} + {% endif %} + + +# Configuration for the BGP peer. +--- +apiVersion: metallb.io/v1beta2 +kind: BGPPeer +metadata: + name: "{{ peer_name }}" + namespace: metallb-system +spec: + myASN: {{ peer.my_asn }} + peerASN: {{ peer.peer_asn }} + peerAddress: {{ peer.peer_address }} + {% if peer.peer_port is defined %} + peerPort: {{ peer.peer_port }} + {% else %} + peerPort: {{ metallb_config.layer3.defaults.peer_port }} + {% endif -%} + + {% if peer.password is defined %} + password: "{{ peer.password }}" + {% endif -%} + + {% if peer.router_id is defined %} + routerID: "{{ peer.router_id }}" + {% endif -%} + + {% if peer.hold_time is defined %} + holdTime: {{ peer.hold_time }} + {% elif metallb_config.layer3.defaults.hold_time is defined %} + holdTime: {{ metallb_config.layer3.defaults.hold_time }} + {% endif -%} + + {% if peer.multihop is defined %} + ebgpMultiHop: {{ peer.multihop }} + {% endif -%} + +{% endfor %} diff --git a/kubespray/project/roles/kubernetes-apps/metallb/templates/metallb.yaml.j2 b/kubespray/project/roles/kubernetes-apps/metallb/templates/metallb.yaml.j2 new file mode 100644 index 0000000..af18a10 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/metallb/templates/metallb.yaml.j2 @@ -0,0 +1,2025 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + labels: + pod-security.kubernetes.io/audit: privileged + pod-security.kubernetes.io/enforce: privileged + pod-security.kubernetes.io/warn: privileged + name: metallb-system + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.1 + name: addresspools.metallb.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tDQpNSUlGWlRDQ0EwMmdBd0lCQWdJVU5GRW1XcTM3MVpKdGkrMmlSQzk1WmpBV1MxZ3dEUVlKS29aSWh2Y05BUUVMDQpCUUF3UWpFTE1Ba0dBMVVFQmhNQ1dGZ3hGVEFUQmdOVkJBY01ERVJsWm1GMWJIUWdRMmwwZVRFY01Cb0dBMVVFDQpDZ3dUUkdWbVlYVnNkQ0JEYjIxd1lXNTVJRXgwWkRBZUZ3MHlNakEzTVRrd09UTXlNek5hRncweU1qQTRNVGd3DQpPVE15TXpOYU1FSXhDekFKQmdOVkJBWVRBbGhZTVJVd0V3WURWUVFIREF4RVpXWmhkV3gwSUVOcGRIa3hIREFhDQpCZ05WQkFvTUUwUmxabUYxYkhRZ1EyOXRjR0Z1ZVNCTWRHUXdnZ0lpTUEwR0NTcUdTSWIzRFFFQkFRVUFBNElDDQpEd0F3Z2dJS0FvSUNBUUNxVFpxMWZRcC9vYkdlenhES0o3OVB3Ny94azJwellualNzMlkzb1ZYSm5sRmM4YjVlDQpma2ZZQnY2bndscW1keW5PL2phWFBaQmRQSS82aFdOUDBkdVhadEtWU0NCUUpyZzEyOGNXb3F0MGNTN3pLb1VpDQpvcU1tQ0QvRXVBeFFNZjhRZDF2c1gvVllkZ0poVTZBRXJLZEpIaXpFOUJtUkNkTDBGMW1OVW55Rk82UnRtWFZUDQpidkxsTDVYeTc2R0FaQVBLOFB4aVlDa0NtbDdxN0VnTWNiOXlLWldCYmlxQ3VkTXE5TGJLNmdKNzF6YkZnSXV4DQo1L1pXK2JraTB2RlplWk9ZODUxb1psckFUNzJvMDI4NHNTWW9uN0pHZVZkY3NoUnh5R1VpSFpSTzdkaXZVTDVTDQpmM2JmSDFYbWY1ZDQzT0NWTWRuUUV2NWVaOG8zeWVLa3ZrbkZQUGVJMU9BbjdGbDlFRVNNR2dhOGFaSG1URSttDQpsLzlMSmdDYjBnQmtPT0M0WnV4bWh2aERKV1EzWnJCS3pMQlNUZXN0NWlLNVlwcXRWVVk2THRyRW9FelVTK1lsDQpwWndXY2VQWHlHeHM5ZURsR3lNVmQraW15Y3NTU1UvVno2Mmx6MnZCS21NTXBkYldDQWhud0RsRTVqU2dyMjRRDQp0eGNXLys2N3d5KzhuQlI3UXdqVTFITndVRjBzeERWdEwrZ1NHVERnSEVZSlhZelYvT05zMy94TkpoVFNPSkxNDQpoeXNVdyttaGdackdhbUdXcHVIVU1DUitvTWJzMTc1UkcrQjJnUFFHVytPTjJnUTRyOXN2b0ZBNHBBQm8xd1dLDQpRYjRhY3pmeVVscElBOVFoSmFsZEY3S3dPSHVlV3gwRUNrNXg0T2tvVDBvWVp0dzFiR0JjRGtaSmF3SURBUUFCDQpvMU13VVRBZEJnTlZIUTRFRmdRVW90UlNIUm9IWTEyRFZ4R0NCdEhpb1g2ZmVFQXdId1lEVlIwakJCZ3dGb0FVDQpvdFJTSFJvSFkxMkRWeEdDQnRIaW9YNmZlRUF3RHdZRFZSMFRBUUgvQkFVd0F3RUIvekFOQmdrcWhraUc5dzBCDQpBUXNGQUFPQ0FnRUFSbkpsWWRjMTFHd0VxWnh6RDF2R3BDR2pDN2VWTlQ3aVY1d3IybXlybHdPYi9aUWFEa0xYDQpvVStaOVVXT1VlSXJTdzUydDdmQUpvVVAwSm5iYkMveVIrU1lqUGhvUXNiVHduOTc2ZldBWTduM3FMOXhCd1Y0DQphek41OXNjeUp0dlhMeUtOL2N5ak1ReDRLajBIMFg0bWJ6bzVZNUtzWWtYVU0vOEFPdWZMcEd0S1NGVGgrSEFDDQpab1Q5YnZHS25adnNHd0tYZFF0Wnh0akhaUjVqK3U3ZGtQOTJBT051RFNabS8rWVV4b2tBK09JbzdSR3BwSHNXDQo1ZTdNY0FTVXRtb1FORXd6dVFoVkJaRWQ1OGtKYjUrV0VWbGNzanlXNnRTbzErZ25tTWNqR1BsMWgxR2hVbjV4DQpFY0lWRnBIWXM5YWo1NmpBSjk1MVQvZjhMaWxmTlVnanBLQ0c1bnl0SUt3emxhOHNtdGlPdm1UNEpYbXBwSkI2DQo4bmdHRVluVjUrUTYwWFJ2OEhSSGp1VG9CRHVhaERrVDA2R1JGODU1d09FR2V4bkZpMXZYWUxLVllWb1V2MXRKDQo4dVdUR1pwNllDSVJldlBqbzg5ZytWTlJSaVFYUThJd0dybXE5c0RoVTlqTjA0SjdVL1RvRDFpNHE3VnlsRUc5DQorV1VGNkNLaEdBeTJIaEhwVncyTGFoOS9lUzdZMUZ1YURrWmhPZG1laG1BOCtqdHNZamJadnR5Mm1SWlF0UUZzDQpUU1VUUjREbUR2bVVPRVRmeStpRHdzK2RkWXVNTnJGeVVYV2dkMnpBQU4ydVl1UHFGY2pRcFNPODFzVTJTU3R3DQoxVzAyeUtYOGJEYmZFdjBzbUh3UzliQnFlSGo5NEM1Mjg0YXpsdTBmaUdpTm1OUEM4ckJLRmhBPQ0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ== + service: + name: webhook-service + namespace: metallb-system + path: /convert + conversionReviewVersions: + - v1alpha1 + - v1beta1 + group: metallb.io + names: + kind: AddressPool + listKind: AddressPoolList + plural: addresspools + singular: addresspool + scope: Namespaced + versions: + - deprecated: true + deprecationWarning: metallb.io v1alpha1 AddressPool is deprecated + name: v1alpha1 + schema: + openAPIV3Schema: + description: AddressPool is the Schema for the addresspools API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AddressPoolSpec defines the desired state of AddressPool. + properties: + addresses: + description: A list of IP address ranges over which MetalLB has authority. + You can list multiple ranges in a single pool, they will all share + the same settings. Each range can be either a CIDR prefix, or an + explicit start-end range of IPs. + items: + type: string + type: array + autoAssign: + default: true + description: AutoAssign flag used to prevent MetallB from automatic + allocation for a pool. + type: boolean + bgpAdvertisements: + description: When an IP is allocated from this pool, how should it + be translated into BGP announcements? + items: + properties: + aggregationLength: + default: 32 + description: The aggregation-length advertisement option lets + you “roll up” the /32s into a larger prefix. + format: int32 + minimum: 1 + type: integer + aggregationLengthV6: + default: 128 + description: Optional, defaults to 128 (i.e. no aggregation) + if not specified. + format: int32 + type: integer + communities: + description: BGP communities + items: + type: string + type: array + localPref: + description: BGP LOCAL_PREF attribute which is used by BGP best + path algorithm, Path with higher localpref is preferred over + one with lower localpref. + format: int32 + type: integer + type: object + type: array + protocol: + description: Protocol can be used to select how the announcement is + done. + enum: + - layer2 + - bgp + type: string + required: + - addresses + - protocol + type: object + status: + description: AddressPoolStatus defines the observed state of AddressPool. + type: object + required: + - spec + type: object + served: true + storage: false + subresources: + status: {} + - deprecated: true + deprecationWarning: metallb.io v1beta1 AddressPool is deprecated, consider using + IPAddressPool + name: v1beta1 + schema: + openAPIV3Schema: + description: AddressPool represents a pool of IP addresses that can be allocated + to LoadBalancer services. AddressPool is deprecated and being replaced by + IPAddressPool. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AddressPoolSpec defines the desired state of AddressPool. + properties: + addresses: + description: A list of IP address ranges over which MetalLB has authority. + You can list multiple ranges in a single pool, they will all share + the same settings. Each range can be either a CIDR prefix, or an + explicit start-end range of IPs. + items: + type: string + type: array + autoAssign: + default: true + description: AutoAssign flag used to prevent MetallB from automatic + allocation for a pool. + type: boolean + bgpAdvertisements: + description: Drives how an IP allocated from this pool should translated + into BGP announcements. + items: + properties: + aggregationLength: + default: 32 + description: The aggregation-length advertisement option lets + you “roll up” the /32s into a larger prefix. + format: int32 + minimum: 1 + type: integer + aggregationLengthV6: + default: 128 + description: Optional, defaults to 128 (i.e. no aggregation) + if not specified. + format: int32 + type: integer + communities: + description: BGP communities to be associated with the given + advertisement. + items: + type: string + type: array + localPref: + description: BGP LOCAL_PREF attribute which is used by BGP best + path algorithm, Path with higher localpref is preferred over + one with lower localpref. + format: int32 + type: integer + type: object + type: array + protocol: + description: Protocol can be used to select how the announcement is + done. + enum: + - layer2 + - bgp + type: string + required: + - addresses + - protocol + type: object + status: + description: AddressPoolStatus defines the observed state of AddressPool. + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.1 + creationTimestamp: null + name: bfdprofiles.metallb.io +spec: + group: metallb.io + names: + kind: BFDProfile + listKind: BFDProfileList + plural: bfdprofiles + singular: bfdprofile + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.passiveMode + name: Passive Mode + type: boolean + - jsonPath: .spec.transmitInterval + name: Transmit Interval + type: integer + - jsonPath: .spec.receiveInterval + name: Receive Interval + type: integer + - jsonPath: .spec.detectMultiplier + name: Multiplier + type: integer + name: v1beta1 + schema: + openAPIV3Schema: + description: BFDProfile represents the settings of the bfd session that can + be optionally associated with a BGP session. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BFDProfileSpec defines the desired state of BFDProfile. + properties: + detectMultiplier: + description: Configures the detection multiplier to determine packet + loss. The remote transmission interval will be multiplied by this + value to determine the connection loss detection timer. + format: int32 + maximum: 255 + minimum: 2 + type: integer + echoInterval: + description: Configures the minimal echo receive transmission interval + that this system is capable of handling in milliseconds. Defaults + to 50ms + format: int32 + maximum: 60000 + minimum: 10 + type: integer + echoMode: + description: Enables or disables the echo transmission mode. This + mode is disabled by default, and not supported on multi hops setups. + type: boolean + minimumTtl: + description: 'For multi hop sessions only: configure the minimum expected + TTL for an incoming BFD control packet.' + format: int32 + maximum: 254 + minimum: 1 + type: integer + passiveMode: + description: 'Mark session as passive: a passive session will not + attempt to start the connection and will wait for control packets + from peer before it begins replying.' + type: boolean + receiveInterval: + description: The minimum interval that this system is capable of receiving + control packets in milliseconds. Defaults to 300ms. + format: int32 + maximum: 60000 + minimum: 10 + type: integer + transmitInterval: + description: The minimum transmission interval (less jitter) that + this system wants to use to send BFD control packets in milliseconds. + Defaults to 300ms + format: int32 + maximum: 60000 + minimum: 10 + type: integer + type: object + status: + description: BFDProfileStatus defines the observed state of BFDProfile. + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.1 + creationTimestamp: null + name: bgpadvertisements.metallb.io +spec: + group: metallb.io + names: + kind: BGPAdvertisement + listKind: BGPAdvertisementList + plural: bgpadvertisements + singular: bgpadvertisement + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.ipAddressPools + name: IPAddressPools + type: string + - jsonPath: .spec.ipAddressPoolSelectors + name: IPAddressPool Selectors + type: string + - jsonPath: .spec.peers + name: Peers + type: string + - jsonPath: .spec.nodeSelectors + name: Node Selectors + priority: 10 + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: BGPAdvertisement allows to advertise the IPs coming from the + selected IPAddressPools via BGP, setting the parameters of the BGP Advertisement. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BGPAdvertisementSpec defines the desired state of BGPAdvertisement. + properties: + aggregationLength: + default: 32 + description: The aggregation-length advertisement option lets you + “roll up” the /32s into a larger prefix. Defaults to 32. Works for + IPv4 addresses. + format: int32 + minimum: 1 + type: integer + aggregationLengthV6: + default: 128 + description: The aggregation-length advertisement option lets you + “roll up” the /128s into a larger prefix. Defaults to 128. Works + for IPv6 addresses. + format: int32 + type: integer + communities: + description: The BGP communities to be associated with the announcement. + Each item can be a community of the form 1234:1234 or the name of + an alias defined in the Community CRD. + items: + type: string + type: array + ipAddressPoolSelectors: + description: A selector for the IPAddressPools which would get advertised + via this advertisement. If no IPAddressPool is selected by this + or by the list, the advertisement is applied to all the IPAddressPools. + items: + description: A label selector is a label query over a set of resources. + The result of matchLabels and matchExpressions are ANDed. An empty + label selector matches all objects. A null label selector matches + no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the + key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a + strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: array + ipAddressPools: + description: The list of IPAddressPools to advertise via this advertisement, + selected by name. + items: + type: string + type: array + localPref: + description: The BGP LOCAL_PREF attribute which is used by BGP best + path algorithm, Path with higher localpref is preferred over one + with lower localpref. + format: int32 + type: integer + nodeSelectors: + description: NodeSelectors allows to limit the nodes to announce as + next hops for the LoadBalancer IP. When empty, all the nodes having are + announced as next hops. + items: + description: A label selector is a label query over a set of resources. + The result of matchLabels and matchExpressions are ANDed. An empty + label selector matches all objects. A null label selector matches + no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the + key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a + strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: array + peers: + description: Peers limits the bgppeer to advertise the ips of the + selected pools to. When empty, the loadbalancer IP is announced + to all the BGPPeers configured. + items: + type: string + type: array + type: object + status: + description: BGPAdvertisementStatus defines the observed state of BGPAdvertisement. + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.1 + name: bgppeers.metallb.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tDQpNSUlGWlRDQ0EwMmdBd0lCQWdJVU5GRW1XcTM3MVpKdGkrMmlSQzk1WmpBV1MxZ3dEUVlKS29aSWh2Y05BUUVMDQpCUUF3UWpFTE1Ba0dBMVVFQmhNQ1dGZ3hGVEFUQmdOVkJBY01ERVJsWm1GMWJIUWdRMmwwZVRFY01Cb0dBMVVFDQpDZ3dUUkdWbVlYVnNkQ0JEYjIxd1lXNTVJRXgwWkRBZUZ3MHlNakEzTVRrd09UTXlNek5hRncweU1qQTRNVGd3DQpPVE15TXpOYU1FSXhDekFKQmdOVkJBWVRBbGhZTVJVd0V3WURWUVFIREF4RVpXWmhkV3gwSUVOcGRIa3hIREFhDQpCZ05WQkFvTUUwUmxabUYxYkhRZ1EyOXRjR0Z1ZVNCTWRHUXdnZ0lpTUEwR0NTcUdTSWIzRFFFQkFRVUFBNElDDQpEd0F3Z2dJS0FvSUNBUUNxVFpxMWZRcC9vYkdlenhES0o3OVB3Ny94azJwellualNzMlkzb1ZYSm5sRmM4YjVlDQpma2ZZQnY2bndscW1keW5PL2phWFBaQmRQSS82aFdOUDBkdVhadEtWU0NCUUpyZzEyOGNXb3F0MGNTN3pLb1VpDQpvcU1tQ0QvRXVBeFFNZjhRZDF2c1gvVllkZ0poVTZBRXJLZEpIaXpFOUJtUkNkTDBGMW1OVW55Rk82UnRtWFZUDQpidkxsTDVYeTc2R0FaQVBLOFB4aVlDa0NtbDdxN0VnTWNiOXlLWldCYmlxQ3VkTXE5TGJLNmdKNzF6YkZnSXV4DQo1L1pXK2JraTB2RlplWk9ZODUxb1psckFUNzJvMDI4NHNTWW9uN0pHZVZkY3NoUnh5R1VpSFpSTzdkaXZVTDVTDQpmM2JmSDFYbWY1ZDQzT0NWTWRuUUV2NWVaOG8zeWVLa3ZrbkZQUGVJMU9BbjdGbDlFRVNNR2dhOGFaSG1URSttDQpsLzlMSmdDYjBnQmtPT0M0WnV4bWh2aERKV1EzWnJCS3pMQlNUZXN0NWlLNVlwcXRWVVk2THRyRW9FelVTK1lsDQpwWndXY2VQWHlHeHM5ZURsR3lNVmQraW15Y3NTU1UvVno2Mmx6MnZCS21NTXBkYldDQWhud0RsRTVqU2dyMjRRDQp0eGNXLys2N3d5KzhuQlI3UXdqVTFITndVRjBzeERWdEwrZ1NHVERnSEVZSlhZelYvT05zMy94TkpoVFNPSkxNDQpoeXNVdyttaGdackdhbUdXcHVIVU1DUitvTWJzMTc1UkcrQjJnUFFHVytPTjJnUTRyOXN2b0ZBNHBBQm8xd1dLDQpRYjRhY3pmeVVscElBOVFoSmFsZEY3S3dPSHVlV3gwRUNrNXg0T2tvVDBvWVp0dzFiR0JjRGtaSmF3SURBUUFCDQpvMU13VVRBZEJnTlZIUTRFRmdRVW90UlNIUm9IWTEyRFZ4R0NCdEhpb1g2ZmVFQXdId1lEVlIwakJCZ3dGb0FVDQpvdFJTSFJvSFkxMkRWeEdDQnRIaW9YNmZlRUF3RHdZRFZSMFRBUUgvQkFVd0F3RUIvekFOQmdrcWhraUc5dzBCDQpBUXNGQUFPQ0FnRUFSbkpsWWRjMTFHd0VxWnh6RDF2R3BDR2pDN2VWTlQ3aVY1d3IybXlybHdPYi9aUWFEa0xYDQpvVStaOVVXT1VlSXJTdzUydDdmQUpvVVAwSm5iYkMveVIrU1lqUGhvUXNiVHduOTc2ZldBWTduM3FMOXhCd1Y0DQphek41OXNjeUp0dlhMeUtOL2N5ak1ReDRLajBIMFg0bWJ6bzVZNUtzWWtYVU0vOEFPdWZMcEd0S1NGVGgrSEFDDQpab1Q5YnZHS25adnNHd0tYZFF0Wnh0akhaUjVqK3U3ZGtQOTJBT051RFNabS8rWVV4b2tBK09JbzdSR3BwSHNXDQo1ZTdNY0FTVXRtb1FORXd6dVFoVkJaRWQ1OGtKYjUrV0VWbGNzanlXNnRTbzErZ25tTWNqR1BsMWgxR2hVbjV4DQpFY0lWRnBIWXM5YWo1NmpBSjk1MVQvZjhMaWxmTlVnanBLQ0c1bnl0SUt3emxhOHNtdGlPdm1UNEpYbXBwSkI2DQo4bmdHRVluVjUrUTYwWFJ2OEhSSGp1VG9CRHVhaERrVDA2R1JGODU1d09FR2V4bkZpMXZYWUxLVllWb1V2MXRKDQo4dVdUR1pwNllDSVJldlBqbzg5ZytWTlJSaVFYUThJd0dybXE5c0RoVTlqTjA0SjdVL1RvRDFpNHE3VnlsRUc5DQorV1VGNkNLaEdBeTJIaEhwVncyTGFoOS9lUzdZMUZ1YURrWmhPZG1laG1BOCtqdHNZamJadnR5Mm1SWlF0UUZzDQpUU1VUUjREbUR2bVVPRVRmeStpRHdzK2RkWXVNTnJGeVVYV2dkMnpBQU4ydVl1UHFGY2pRcFNPODFzVTJTU3R3DQoxVzAyeUtYOGJEYmZFdjBzbUh3UzliQnFlSGo5NEM1Mjg0YXpsdTBmaUdpTm1OUEM4ckJLRmhBPQ0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ== + service: + name: webhook-service + namespace: metallb-system + path: /convert + conversionReviewVersions: + - v1beta1 + - v1beta2 + group: metallb.io + names: + kind: BGPPeer + listKind: BGPPeerList + plural: bgppeers + singular: bgppeer + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.peerAddress + name: Address + type: string + - jsonPath: .spec.peerASN + name: ASN + type: string + - jsonPath: .spec.bfdProfile + name: BFD Profile + type: string + - jsonPath: .spec.ebgpMultiHop + name: Multi Hops + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: BGPPeer is the Schema for the peers API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BGPPeerSpec defines the desired state of Peer. + properties: + bfdProfile: + type: string + ebgpMultiHop: + description: EBGP peer is multi-hops away + type: boolean + holdTime: + description: Requested BGP hold time, per RFC4271. + type: string + keepaliveTime: + description: Requested BGP keepalive time, per RFC4271. + type: string + myASN: + description: AS number to use for the local end of the session. + format: int32 + maximum: 4294967295 + minimum: 0 + type: integer + nodeSelectors: + description: Only connect to this peer on nodes that match one of + these selectors. + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + minItems: 1 + type: array + required: + - key + - operator + - values + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + type: array + password: + description: Authentication password for routers enforcing TCP MD5 + authenticated sessions + type: string + peerASN: + description: AS number to expect from the remote end of the session. + format: int32 + maximum: 4294967295 + minimum: 0 + type: integer + peerAddress: + description: Address to dial when establishing the session. + type: string + peerPort: + description: Port to dial when establishing the session. + maximum: 16384 + minimum: 0 + type: integer + routerID: + description: BGP router ID to advertise to the peer + type: string + sourceAddress: + description: Source address to use when establishing the session. + type: string + required: + - myASN + - peerASN + - peerAddress + type: object + status: + description: BGPPeerStatus defines the observed state of Peer. + type: object + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - jsonPath: .spec.peerAddress + name: Address + type: string + - jsonPath: .spec.peerASN + name: ASN + type: string + - jsonPath: .spec.bfdProfile + name: BFD Profile + type: string + - jsonPath: .spec.ebgpMultiHop + name: Multi Hops + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: BGPPeer is the Schema for the peers API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BGPPeerSpec defines the desired state of Peer. + properties: + bfdProfile: + description: The name of the BFD Profile to be used for the BFD session + associated to the BGP session. If not set, the BFD session won't + be set up. + type: string + ebgpMultiHop: + description: To set if the BGPPeer is multi-hops away. Needed for + FRR mode only. + type: boolean + holdTime: + description: Requested BGP hold time, per RFC4271. + type: string + keepaliveTime: + description: Requested BGP keepalive time, per RFC4271. + type: string + myASN: + description: AS number to use for the local end of the session. + format: int32 + maximum: 4294967295 + minimum: 0 + type: integer + nodeSelectors: + description: Only connect to this peer on nodes that match one of + these selectors. + items: + description: A label selector is a label query over a set of resources. + The result of matchLabels and matchExpressions are ANDed. An empty + label selector matches all objects. A null label selector matches + no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the + key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a + strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: array + password: + description: Authentication password for routers enforcing TCP MD5 + authenticated sessions + type: string + passwordSecret: + description: passwordSecret is name of the authentication secret for + BGP Peer. the secret must be of type "kubernetes.io/basic-auth", + and created in the same namespace as the MetalLB deployment. The + password is stored in the secret as the key "password". + properties: + name: + description: name is unique within a namespace to reference a + secret resource. + type: string + namespace: + description: namespace defines the space within which the secret + name must be unique. + type: string + type: object + x-kubernetes-map-type: atomic + peerASN: + description: AS number to expect from the remote end of the session. + format: int32 + maximum: 4294967295 + minimum: 0 + type: integer + peerAddress: + description: Address to dial when establishing the session. + type: string + peerPort: + default: 179 + description: Port to dial when establishing the session. + maximum: 16384 + minimum: 0 + type: integer + routerID: + description: BGP router ID to advertise to the peer + type: string + sourceAddress: + description: Source address to use when establishing the session. + type: string + vrf: + description: To set if we want to peer with the BGPPeer using an interface + belonging to a host vrf + type: string + required: + - myASN + - peerASN + - peerAddress + type: object + status: + description: BGPPeerStatus defines the observed state of Peer. + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.1 + creationTimestamp: null + name: communities.metallb.io +spec: + group: metallb.io + names: + kind: Community + listKind: CommunityList + plural: communities + singular: community + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: Community is a collection of aliases for communities. Users can + define named aliases to be used in the BGPPeer CRD. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: CommunitySpec defines the desired state of Community. + properties: + communities: + items: + properties: + name: + description: The name of the alias for the community. + type: string + value: + description: The BGP community value corresponding to the given + name. + type: string + type: object + type: array + type: object + status: + description: CommunityStatus defines the observed state of Community. + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.1 + creationTimestamp: null + name: ipaddresspools.metallb.io +spec: + group: metallb.io + names: + kind: IPAddressPool + listKind: IPAddressPoolList + plural: ipaddresspools + singular: ipaddresspool + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.autoAssign + name: Auto Assign + type: boolean + - jsonPath: .spec.avoidBuggyIPs + name: Avoid Buggy IPs + type: boolean + - jsonPath: .spec.addresses + name: Addresses + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: IPAddressPool represents a pool of IP addresses that can be allocated + to LoadBalancer services. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: IPAddressPoolSpec defines the desired state of IPAddressPool. + properties: + addresses: + description: A list of IP address ranges over which MetalLB has authority. + You can list multiple ranges in a single pool, they will all share + the same settings. Each range can be either a CIDR prefix, or an + explicit start-end range of IPs. + items: + type: string + type: array + autoAssign: + default: true + description: AutoAssign flag used to prevent MetallB from automatic + allocation for a pool. + type: boolean + avoidBuggyIPs: + default: false + description: AvoidBuggyIPs prevents addresses ending with .0 and .255 + to be used by a pool. + type: boolean + serviceAllocation: + description: AllocateTo makes ip pool allocation to specific namespace + and/or service. The controller will use the pool with lowest value + of priority in case of multiple matches. A pool with no priority + set will be used only if the pools with priority can't be used. + If multiple matching IPAddressPools are available it will check + for the availability of IPs sorting the matching IPAddressPools + by priority, starting from the highest to the lowest. If multiple + IPAddressPools have the same priority, choice will be random. + properties: + namespaceSelectors: + description: NamespaceSelectors list of label selectors to select + namespace(s) for ip pool, an alternative to using namespace + list. + items: + description: A label selector is a label query over a set of + resources. The result of matchLabels and matchExpressions + are ANDed. An empty label selector matches all objects. A + null label selector matches no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values array + must be non-empty. If the operator is Exists or + DoesNotExist, the values array must be empty. This + array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field is + "key", the operator is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: array + namespaces: + description: Namespaces list of namespace(s) on which ip pool + can be attached. + items: + type: string + type: array + priority: + description: Priority priority given for ip pool while ip allocation + on a service. + type: integer + serviceSelectors: + description: ServiceSelectors list of label selector to select + service(s) for which ip pool can be used for ip allocation. + items: + description: A label selector is a label query over a set of + resources. The result of matchLabels and matchExpressions + are ANDed. An empty label selector matches all objects. A + null label selector matches no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values array + must be non-empty. If the operator is Exists or + DoesNotExist, the values array must be empty. This + array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field is + "key", the operator is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: array + type: object + required: + - addresses + type: object + status: + description: IPAddressPoolStatus defines the observed state of IPAddressPool. + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.1 + creationTimestamp: null + name: l2advertisements.metallb.io +spec: + group: metallb.io + names: + kind: L2Advertisement + listKind: L2AdvertisementList + plural: l2advertisements + singular: l2advertisement + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.ipAddressPools + name: IPAddressPools + type: string + - jsonPath: .spec.ipAddressPoolSelectors + name: IPAddressPool Selectors + type: string + - jsonPath: .spec.interfaces + name: Interfaces + type: string + - jsonPath: .spec.nodeSelectors + name: Node Selectors + priority: 10 + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: L2Advertisement allows to advertise the LoadBalancer IPs provided + by the selected pools via L2. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: L2AdvertisementSpec defines the desired state of L2Advertisement. + properties: + interfaces: + description: A list of interfaces to announce from. The LB IP will + be announced only from these interfaces. If the field is not set, + we advertise from all the interfaces on the host. + items: + type: string + type: array + ipAddressPoolSelectors: + description: A selector for the IPAddressPools which would get advertised + via this advertisement. If no IPAddressPool is selected by this + or by the list, the advertisement is applied to all the IPAddressPools. + items: + description: A label selector is a label query over a set of resources. + The result of matchLabels and matchExpressions are ANDed. An empty + label selector matches all objects. A null label selector matches + no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the + key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a + strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: array + ipAddressPools: + description: The list of IPAddressPools to advertise via this advertisement, + selected by name. + items: + type: string + type: array + nodeSelectors: + description: NodeSelectors allows to limit the nodes to announce as + next hops for the LoadBalancer IP. When empty, all the nodes having are + announced as next hops. + items: + description: A label selector is a label query over a set of resources. + The result of matchLabels and matchExpressions are ANDed. An empty + label selector matches all objects. A null label selector matches + no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the + key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a + strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: array + type: object + status: + description: L2AdvertisementStatus defines the observed state of L2Advertisement. + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app: metallb + pod-security.kubernetes.io/audit: privileged + pod-security.kubernetes.io/enforce: privileged + pod-security.kubernetes.io/warn: privileged + name: controller + namespace: metallb-system + +{% if metallb_speaker_enabled %} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app: metallb + name: speaker + namespace: metallb-system +{% endif %} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app: metallb + name: controller + namespace: metallb-system +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resourceNames: + - memberlist + resources: + - secrets + verbs: + - list +- apiGroups: + - apps + resourceNames: + - controller + resources: + - deployments + verbs: + - get +- apiGroups: + - metallb.io + resources: + - bgppeers + verbs: + - get + - list +- apiGroups: + - metallb.io + resources: + - addresspools + verbs: + - get + - list + - watch +- apiGroups: + - metallb.io + resources: + - bfdprofiles + verbs: + - get + - list + - watch +- apiGroups: + - metallb.io + resources: + - ipaddresspools + verbs: + - get + - list + - watch +- apiGroups: + - metallb.io + resources: + - bgpadvertisements + verbs: + - get + - list + - watch +- apiGroups: + - metallb.io + resources: + - l2advertisements + verbs: + - get + - list + - watch +- apiGroups: + - metallb.io + resources: + - communities + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app: metallb + name: pod-lister + namespace: metallb-system +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - list +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch +- apiGroups: + - metallb.io + resources: + - addresspools + verbs: + - get + - list + - watch +- apiGroups: + - metallb.io + resources: + - bfdprofiles + verbs: + - get + - list + - watch +- apiGroups: + - metallb.io + resources: + - bgppeers + verbs: + - get + - list + - watch +- apiGroups: + - metallb.io + resources: + - l2advertisements + verbs: + - get + - list + - watch +- apiGroups: + - metallb.io + resources: + - bgpadvertisements + verbs: + - get + - list + - watch +- apiGroups: + - metallb.io + resources: + - ipaddresspools + verbs: + - get + - list + - watch +- apiGroups: + - metallb.io + resources: + - communities + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app: metallb + name: metallb-system:controller +rules: +- apiGroups: + - "" + resources: + - services + - namespaces + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - services/status + verbs: + - update +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +- apiGroups: + - admissionregistration.k8s.io + resourceNames: + - metallb-webhook-configuration + resources: + - validatingwebhookconfigurations + - mutatingwebhookconfigurations + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - admissionregistration.k8s.io + resources: + - validatingwebhookconfigurations + - mutatingwebhookconfigurations + verbs: + - list + - watch +- apiGroups: + - apiextensions.k8s.io + resourceNames: + - addresspools.metallb.io + - bfdprofiles.metallb.io + - bgpadvertisements.metallb.io + - bgppeers.metallb.io + - ipaddresspools.metallb.io + - l2advertisements.metallb.io + - communities.metallb.io + resources: + - customresourcedefinitions + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - watch +--- +{% if metallb_speaker_enabled %} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app: metallb + name: metallb-system:speaker +rules: +- apiGroups: + - "" + resources: + - services + - endpoints + - nodes + - namespaces + verbs: + - get + - list + - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +{% endif %} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app: metallb + name: controller + namespace: metallb-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: controller +subjects: +- kind: ServiceAccount + name: controller + namespace: metallb-system + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app: metallb + name: pod-lister + namespace: metallb-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: pod-lister +subjects: +- kind: ServiceAccount + name: speaker + namespace: metallb-system + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app: metallb + name: metallb-system:controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: metallb-system:controller +subjects: +- kind: ServiceAccount + name: controller + namespace: metallb-system + +{% if metallb_speaker_enabled %} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app: metallb + name: metallb-system:speaker +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: metallb-system:speaker +subjects: +- kind: ServiceAccount + name: speaker + namespace: metallb-system +{% endif %} + +--- +apiVersion: v1 +kind: Secret +metadata: + name: webhook-server-cert + namespace: metallb-system + +--- +apiVersion: v1 +kind: Service +metadata: + name: webhook-service + namespace: metallb-system +spec: + ports: + - port: 443 + targetPort: 9443 + selector: + component: controller + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: metallb + component: controller + name: controller + namespace: metallb-system +spec: + revisionHistoryLimit: 3 + selector: + matchLabels: + app: metallb + component: controller + template: + metadata: + annotations: + prometheus.io/port: '{{ metallb_port }}' + prometheus.io/scrape: 'true' + labels: + app: metallb + component: controller + spec: + priorityClassName: system-cluster-critical + containers: + - args: + - --port={{ metallb_port }} + - --log-level={{ metallb_log_level }} +{% if metallb_loadbalancer_class != "" %} + - --lb-class={{ metallb_loadbalancer_class }} +{% endif %} + env: + - name: METALLB_ML_SECRET_NAME + value: memberlist + - name: METALLB_DEPLOYMENT + value: controller + image: "{{ metallb_controller_image_repo }}:{{ metallb_version }}" + livenessProbe: + failureThreshold: 3 + httpGet: + path: /metrics + port: monitoring + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: controller + ports: + - containerPort: {{ metallb_port }} + name: monitoring + - containerPort: 9443 + name: webhook-server + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /metrics + port: monitoring + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - all + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true +{% if metallb_config.controller is defined and metallb_config.controller.tolerations is defined %} + tolerations: + {{ metallb_config.controller.tolerations | to_nice_yaml(indent=2) | indent(width=8) }} +{%- endif %} + nodeSelector: + {{ metallb_controller_nodeselector | to_nice_yaml | indent(width=8) -}} + {% if metallb_config.controller is defined and metallb_config.controller.nodeselector is defined %} + {{ metallb_config.controller.nodeselector | to_nice_yaml | indent(width=8) -}} + {%- endif %} + securityContext: + fsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + serviceAccountName: controller + terminationGracePeriodSeconds: 0 + volumes: + - name: cert + secret: + defaultMode: 420 + secretName: webhook-server-cert + +--- +{% if metallb_speaker_enabled %} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + app: metallb + component: speaker + name: speaker + namespace: metallb-system +spec: + selector: + matchLabels: + app: metallb + component: speaker + template: + metadata: + annotations: + prometheus.io/port: '{{ metallb_port }}' + prometheus.io/scrape: 'true' + labels: + app: metallb + component: speaker + spec: + containers: + - args: + - --port={{ metallb_port }} + - --log-level={{ metallb_log_level }} +{% if metallb_loadbalancer_class != "" %} + - --lb-class={{ metallb_loadbalancer_class }} +{% endif %} + env: + - name: METALLB_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: METALLB_HOST + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: METALLB_ML_BIND_ADDR + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: METALLB_ML_LABELS + value: app=metallb,component=speaker + - name: METALLB_ML_SECRET_KEY + valueFrom: + secretKeyRef: + key: secretkey + name: memberlist + image: "{{ metallb_speaker_image_repo }}:{{ metallb_version }}" + livenessProbe: + failureThreshold: 3 + httpGet: + path: /metrics + port: monitoring + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: speaker + ports: + - containerPort: {{ metallb_port }} + name: monitoring + - containerPort: {{ metallb_memberlist_port }} + name: memberlist-tcp + - containerPort: {{ metallb_memberlist_port }} + name: memberlist-udp + protocol: UDP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /metrics + port: monitoring + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - NET_RAW + drop: + - ALL + readOnlyRootFilesystem: true + hostNetwork: true + nodeSelector: + {{ metallb_speaker_nodeselector | to_nice_yaml | indent(width=8) -}} + {% if metallb_config.speaker is defined and metallb_config.speaker.nodeselector is defined %} + {{ metallb_config.speaker.nodeselector | to_nice_yaml | indent(width=8) -}} + {%- endif %} + + serviceAccountName: speaker + terminationGracePeriodSeconds: 2 + tolerations: + {{ metallb_speaker_tolerations | to_nice_yaml(indent=2) | indent(width=8) -}} + {% if metallb_config.speaker is defined and metallb_config.speaker.tolerations is defined %} + {{ metallb_config.speaker.tolerations | to_nice_yaml(indent=2) | indent(width=8) -}} + {% endif %} +{% endif %} + +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + creationTimestamp: null + name: metallb-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: metallb-system + path: /validate-metallb-io-v1beta2-bgppeer + failurePolicy: Fail + name: bgppeersvalidationwebhook.metallb.io + rules: + - apiGroups: + - metallb.io + apiVersions: + - v1beta2 + operations: + - CREATE + - UPDATE + resources: + - bgppeers + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: metallb-system + path: /validate-metallb-io-v1beta1-addresspool + failurePolicy: Fail + name: addresspoolvalidationwebhook.metallb.io + rules: + - apiGroups: + - metallb.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - addresspools + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: metallb-system + path: /validate-metallb-io-v1beta1-bfdprofile + failurePolicy: Fail + name: bfdprofilevalidationwebhook.metallb.io + rules: + - apiGroups: + - metallb.io + apiVersions: + - v1beta1 + operations: + - CREATE + - DELETE + resources: + - bfdprofiles + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: metallb-system + path: /validate-metallb-io-v1beta1-bgpadvertisement + failurePolicy: Fail + name: bgpadvertisementvalidationwebhook.metallb.io + rules: + - apiGroups: + - metallb.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - bgpadvertisements + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: metallb-system + path: /validate-metallb-io-v1beta1-community + failurePolicy: Fail + name: communityvalidationwebhook.metallb.io + rules: + - apiGroups: + - metallb.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - communities + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: metallb-system + path: /validate-metallb-io-v1beta1-ipaddresspool + failurePolicy: Fail + name: ipaddresspoolvalidationwebhook.metallb.io + rules: + - apiGroups: + - metallb.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - ipaddresspools + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: metallb-system + path: /validate-metallb-io-v1beta1-l2advertisement + failurePolicy: Fail + name: l2advertisementvalidationwebhook.metallb.io + rules: + - apiGroups: + - metallb.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - l2advertisements + sideEffects: None diff --git a/kubespray/project/roles/kubernetes-apps/metallb/templates/pools.yaml.j2 b/kubespray/project/roles/kubernetes-apps/metallb/templates/pools.yaml.j2 new file mode 100644 index 0000000..f22a4e3 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/metallb/templates/pools.yaml.j2 @@ -0,0 +1,22 @@ +#jinja2: trim_blocks: True, lstrip_blocks: True +# yamllint disable-file +--- + +# Create all pools +{% for pool_name, pool in metallb_config.address_pools.items() %} + +--- +apiVersion: metallb.io/v1beta1 +kind: IPAddressPool +metadata: + namespace: metallb-system + name: "{{ pool_name }}" +spec: + addresses: +{% for ip_range in pool.ip_range %} + - "{{ ip_range }}" +{% endfor %} + autoAssign: {{ pool.auto_assign | default(true) }} + avoidBuggyIPs: {{ pool.avoid_buggy_ips | default(false) }} + +{% endfor %} diff --git a/kubespray/project/roles/kubernetes-apps/metrics_server/defaults/main.yml b/kubespray/project/roles/kubernetes-apps/metrics_server/defaults/main.yml new file mode 100644 index 0000000..934e67b --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/metrics_server/defaults/main.yml @@ -0,0 +1,14 @@ +--- +metrics_server_container_port: 10250 +metrics_server_kubelet_insecure_tls: true +metrics_server_kubelet_preferred_address_types: "InternalIP,ExternalIP,Hostname" +metrics_server_metric_resolution: 15s +metrics_server_limits_cpu: 100m +metrics_server_limits_memory: 200Mi +metrics_server_requests_cpu: 100m +metrics_server_requests_memory: 200Mi +metrics_server_host_network: false +metrics_server_replicas: 1 +# metrics_server_extra_tolerations: [] +# metrics_server_extra_affinity: {} +# metrics_server_nodeselector: {} diff --git a/kubespray/project/roles/kubernetes-apps/metrics_server/tasks/main.yml b/kubespray/project/roles/kubernetes-apps/metrics_server/tasks/main.yml new file mode 100644 index 0000000..1fe617d --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/metrics_server/tasks/main.yml @@ -0,0 +1,57 @@ +--- +# If all masters have node role, there are no tainted master and toleration should not be specified. +- name: Check all masters are node or not + set_fact: + masters_are_not_tainted: "{{ groups['kube_node'] | intersect(groups['kube_control_plane']) == groups['kube_control_plane'] }}" + +- name: Metrics Server | Delete addon dir + file: + path: "{{ kube_config_dir }}/addons/metrics_server" + state: absent + when: + - inventory_hostname == groups['kube_control_plane'][0] + tags: + - upgrade + +- name: Metrics Server | Create addon dir + file: + path: "{{ kube_config_dir }}/addons/metrics_server" + state: directory + owner: root + group: root + mode: 0755 + when: + - inventory_hostname == groups['kube_control_plane'][0] + +- name: Metrics Server | Templates list + set_fact: + metrics_server_templates: + - { name: auth-delegator, file: auth-delegator.yaml, type: clusterrolebinding } + - { name: auth-reader, file: auth-reader.yaml, type: rolebinding } + - { name: metrics-server-sa, file: metrics-server-sa.yaml, type: sa } + - { name: metrics-server-deployment, file: metrics-server-deployment.yaml, type: deploy } + - { name: metrics-server-service, file: metrics-server-service.yaml, type: service } + - { name: metrics-apiservice, file: metrics-apiservice.yaml, type: service } + - { name: resource-reader-clusterrolebinding, file: resource-reader-clusterrolebinding.yaml, type: clusterrolebinding } + - { name: resource-reader, file: resource-reader.yaml, type: clusterrole } + +- name: Metrics Server | Create manifests + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/addons/metrics_server/{{ item.file }}" + mode: 0644 + with_items: "{{ metrics_server_templates }}" + register: metrics_server_manifests + when: + - inventory_hostname == groups['kube_control_plane'][0] + +- name: Metrics Server | Apply manifests + kube: + name: "{{ item.item.name }}" + kubectl: "{{ bin_dir }}/kubectl" + resource: "{{ item.item.type }}" + filename: "{{ kube_config_dir }}/addons/metrics_server/{{ item.item.file }}" + state: "latest" + with_items: "{{ metrics_server_manifests.results }}" + when: + - inventory_hostname == groups['kube_control_plane'][0] diff --git a/kubespray/project/roles/kubernetes-apps/metrics_server/templates/auth-delegator.yaml.j2 b/kubespray/project/roles/kubernetes-apps/metrics_server/templates/auth-delegator.yaml.j2 new file mode 100644 index 0000000..92f8204 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/metrics_server/templates/auth-delegator.yaml.j2 @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: metrics-server:system:auth-delegator + labels: + addonmanager.kubernetes.io/mode: Reconcile +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system diff --git a/kubespray/project/roles/kubernetes-apps/metrics_server/templates/auth-reader.yaml.j2 b/kubespray/project/roles/kubernetes-apps/metrics_server/templates/auth-reader.yaml.j2 new file mode 100644 index 0000000..e02b8ea --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/metrics_server/templates/auth-reader.yaml.j2 @@ -0,0 +1,15 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: metrics-server-auth-reader + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: Reconcile +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system diff --git a/kubespray/project/roles/kubernetes-apps/metrics_server/templates/metrics-apiservice.yaml.j2 b/kubespray/project/roles/kubernetes-apps/metrics_server/templates/metrics-apiservice.yaml.j2 new file mode 100644 index 0000000..9341687 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/metrics_server/templates/metrics-apiservice.yaml.j2 @@ -0,0 +1,15 @@ +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + name: v1beta1.metrics.k8s.io + labels: + addonmanager.kubernetes.io/mode: Reconcile +spec: + service: + name: metrics-server + namespace: kube-system + group: metrics.k8s.io + version: v1beta1 + insecureSkipTLSVerify: {{ metrics_server_kubelet_insecure_tls }} + groupPriorityMinimum: 100 + versionPriority: 100 diff --git a/kubespray/project/roles/kubernetes-apps/metrics_server/templates/metrics-server-deployment.yaml.j2 b/kubespray/project/roles/kubernetes-apps/metrics_server/templates/metrics-server-deployment.yaml.j2 new file mode 100644 index 0000000..ce121fe --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/metrics_server/templates/metrics-server-deployment.yaml.j2 @@ -0,0 +1,118 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: metrics-server + namespace: kube-system + labels: + app.kubernetes.io/name: metrics-server + addonmanager.kubernetes.io/mode: Reconcile + version: {{ metrics_server_version }} +spec: + replicas: {{ metrics_server_replicas }} + selector: + matchLabels: + app.kubernetes.io/name: metrics-server + version: {{ metrics_server_version }} + strategy: + rollingUpdate: + maxUnavailable: 0 + template: + metadata: + name: metrics-server + labels: + app.kubernetes.io/name: metrics-server + version: {{ metrics_server_version }} + spec: + priorityClassName: system-cluster-critical + serviceAccountName: metrics-server + hostNetwork: {{ metrics_server_host_network | default(false) }} + containers: + - name: metrics-server + image: {{ metrics_server_image_repo }}:{{ metrics_server_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + args: + - --cert-dir=/tmp + - --secure-port={{ metrics_server_container_port }} +{% if metrics_server_kubelet_preferred_address_types %} + - --kubelet-preferred-address-types={{ metrics_server_kubelet_preferred_address_types }} +{% endif %} + - --kubelet-use-node-status-port +{% if metrics_server_kubelet_insecure_tls %} + - --kubelet-insecure-tls=true +{% endif %} + - --metric-resolution={{ metrics_server_metric_resolution }} + ports: + - containerPort: {{ metrics_server_container_port }} + name: https + protocol: TCP + volumeMounts: + - name: tmp + mountPath: /tmp + livenessProbe: + httpGet: + path: /livez + port: https + scheme: HTTPS + periodSeconds: 10 + failureThreshold: 3 + initialDelaySeconds: 40 + readinessProbe: + httpGet: + path: /readyz + port: https + scheme: HTTPS + periodSeconds: 10 + failureThreshold: 3 + initialDelaySeconds: 40 + securityContext: + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + allowPrivilegeEscalation: false + seccompProfile: + type: RuntimeDefault + capabilities: + drop: + - ALL + resources: + limits: + cpu: {{ metrics_server_limits_cpu }} + memory: {{ metrics_server_limits_memory }} + requests: + cpu: {{ metrics_server_requests_cpu }} + memory: {{ metrics_server_requests_memory }} + volumes: + - name: tmp + emptyDir: {} +{% if not masters_are_not_tainted or metrics_server_extra_tolerations is defined %} + tolerations: +{% if not masters_are_not_tainted %} + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule +{% endif %} +{% if metrics_server_extra_tolerations is defined %} + {{ metrics_server_extra_tolerations | list | to_nice_yaml(indent=2) | indent(8) }} +{% endif %} +{% endif %} + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: + - metrics-server + topologyKey: kubernetes.io/hostname + namespaces: + - kube-system +{% if metrics_server_extra_affinity is defined %} + {{ metrics_server_extra_affinity | to_nice_yaml | indent(width=8) }} +{% endif %} +{% if metrics_server_nodeselector is defined %} + nodeSelector: + {{ metrics_server_nodeselector | to_nice_yaml | indent(width=8) }} +{% endif %} diff --git a/kubespray/project/roles/kubernetes-apps/metrics_server/templates/metrics-server-sa.yaml.j2 b/kubespray/project/roles/kubernetes-apps/metrics_server/templates/metrics-server-sa.yaml.j2 new file mode 100644 index 0000000..94444ca --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/metrics_server/templates/metrics-server-sa.yaml.j2 @@ -0,0 +1,8 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: metrics-server + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: Reconcile diff --git a/kubespray/project/roles/kubernetes-apps/metrics_server/templates/metrics-server-service.yaml.j2 b/kubespray/project/roles/kubernetes-apps/metrics_server/templates/metrics-server-service.yaml.j2 new file mode 100644 index 0000000..f1c3691 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/metrics_server/templates/metrics-server-service.yaml.j2 @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: metrics-server + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: Reconcile + app.kubernetes.io/name: "metrics-server" +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: metrics-server + ports: + - name: https + port: 443 + protocol: TCP + targetPort: https diff --git a/kubespray/project/roles/kubernetes-apps/metrics_server/templates/resource-reader-clusterrolebinding.yaml.j2 b/kubespray/project/roles/kubernetes-apps/metrics_server/templates/resource-reader-clusterrolebinding.yaml.j2 new file mode 100644 index 0000000..038cfd8 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/metrics_server/templates/resource-reader-clusterrolebinding.yaml.j2 @@ -0,0 +1,15 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:metrics-server + labels: + addonmanager.kubernetes.io/mode: Reconcile +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:metrics-server +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system diff --git a/kubespray/project/roles/kubernetes-apps/metrics_server/templates/resource-reader.yaml.j2 b/kubespray/project/roles/kubernetes-apps/metrics_server/templates/resource-reader.yaml.j2 new file mode 100644 index 0000000..9c4a3b7 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/metrics_server/templates/resource-reader.yaml.j2 @@ -0,0 +1,22 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:metrics-server + labels: + addonmanager.kubernetes.io/mode: Reconcile +rules: +- apiGroups: + - "" + resources: + - nodes/metrics + verbs: + - get +- apiGroups: + - "" + resources: + - pods + - nodes + verbs: + - get + - list + - watch diff --git a/kubespray/project/roles/kubernetes-apps/network_plugin/calico/tasks/main.yml b/kubespray/project/roles/kubernetes-apps/network_plugin/calico/tasks/main.yml new file mode 100644 index 0000000..b8b4338 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/network_plugin/calico/tasks/main.yml @@ -0,0 +1,2 @@ +--- +# TODO: Handle Calico etcd -> kdd migration diff --git a/kubespray/project/roles/kubernetes-apps/network_plugin/flannel/tasks/main.yml b/kubespray/project/roles/kubernetes-apps/network_plugin/flannel/tasks/main.yml new file mode 100644 index 0000000..ff56d24 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/network_plugin/flannel/tasks/main.yml @@ -0,0 +1,17 @@ +--- +- name: Flannel | Start Resources + kube: + name: "{{ item.item.name }}" + namespace: "kube-system" + kubectl: "{{ bin_dir }}/kubectl" + resource: "{{ item.item.type }}" + filename: "{{ kube_config_dir }}/{{ item.item.file }}" + state: "latest" + with_items: "{{ flannel_node_manifests.results }}" + when: inventory_hostname == groups['kube_control_plane'][0] and not item is skipped + +- name: Flannel | Wait for flannel subnet.env file presence + wait_for: + path: /run/flannel/subnet.env + delay: 5 + timeout: 600 diff --git a/kubespray/project/roles/kubernetes-apps/network_plugin/kube-ovn/tasks/main.yml b/kubespray/project/roles/kubernetes-apps/network_plugin/kube-ovn/tasks/main.yml new file mode 100644 index 0000000..9f42501 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/network_plugin/kube-ovn/tasks/main.yml @@ -0,0 +1,9 @@ +--- +- name: Kube-OVN | Start Resources + kube: + name: "{{ item.item.name }}" + kubectl: "{{ bin_dir }}/kubectl" + filename: "{{ kube_config_dir }}/{{ item.item.file }}" + state: "latest" + with_items: "{{ kube_ovn_node_manifests.results }}" + when: inventory_hostname == groups['kube_control_plane'][0] and not item is skipped diff --git a/kubespray/project/roles/kubernetes-apps/network_plugin/kube-router/OWNERS b/kubespray/project/roles/kubernetes-apps/network_plugin/kube-router/OWNERS new file mode 100644 index 0000000..c40af3c --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/network_plugin/kube-router/OWNERS @@ -0,0 +1,6 @@ +# See the OWNERS docs at https://go.k8s.io/owners + +approvers: + - bozzo +reviewers: + - bozzo diff --git a/kubespray/project/roles/kubernetes-apps/network_plugin/kube-router/tasks/main.yml b/kubespray/project/roles/kubernetes-apps/network_plugin/kube-router/tasks/main.yml new file mode 100644 index 0000000..1d756a0 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/network_plugin/kube-router/tasks/main.yml @@ -0,0 +1,23 @@ +--- + +- name: Kube-router | Start Resources + kube: + name: "kube-router" + kubectl: "{{ bin_dir }}/kubectl" + filename: "{{ kube_config_dir }}/kube-router.yml" + resource: "ds" + namespace: "kube-system" + state: "latest" + delegate_to: "{{ groups['kube_control_plane'] | first }}" + run_once: true + +- name: Kube-router | Wait for kube-router pods to be ready + command: "{{ kubectl }} -n kube-system get pods -l k8s-app=kube-router -o jsonpath='{.items[?(@.status.containerStatuses[0].ready==false)].metadata.name}'" # noqa ignore-errors + register: pods_not_ready + until: pods_not_ready.stdout.find("kube-router")==-1 + retries: 30 + delay: 10 + ignore_errors: true + delegate_to: "{{ groups['kube_control_plane'] | first }}" + run_once: true + changed_when: false diff --git a/kubespray/project/roles/kubernetes-apps/network_plugin/meta/main.yml b/kubespray/project/roles/kubernetes-apps/network_plugin/meta/main.yml new file mode 100644 index 0000000..1128312 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/network_plugin/meta/main.yml @@ -0,0 +1,31 @@ +--- +dependencies: + - role: kubernetes-apps/network_plugin/calico + when: kube_network_plugin == 'calico' + tags: + - calico + + - role: kubernetes-apps/network_plugin/flannel + when: kube_network_plugin == 'flannel' + tags: + - flannel + + - role: kubernetes-apps/network_plugin/kube-ovn + when: kube_network_plugin == 'kube-ovn' + tags: + - kube-ovn + + - role: kubernetes-apps/network_plugin/weave + when: kube_network_plugin == 'weave' + tags: + - weave + + - role: kubernetes-apps/network_plugin/kube-router + when: kube_network_plugin == 'kube-router' + tags: + - kube-router + + - role: kubernetes-apps/network_plugin/multus + when: kube_network_plugin_multus + tags: + - multus diff --git a/kubespray/project/roles/kubernetes-apps/network_plugin/multus/tasks/main.yml b/kubespray/project/roles/kubernetes-apps/network_plugin/multus/tasks/main.yml new file mode 100644 index 0000000..fdbede5 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/network_plugin/multus/tasks/main.yml @@ -0,0 +1,18 @@ +--- +- name: Multus | Start resources + kube: + name: "{{ item.item.name }}" + namespace: "kube-system" + kubectl: "{{ bin_dir }}/kubectl" + resource: "{{ item.item.type }}" + filename: "{{ kube_config_dir }}/{{ item.item.file }}" + state: "latest" + delegate_to: "{{ groups['kube_control_plane'][0] }}" + run_once: true + with_items: "{{ multus_manifest_1.results + (multus_nodes_list | map('extract', hostvars, 'multus_manifest_2') | list | json_query('[].results')) }}" + loop_control: + label: "{{ item.item.name if item != None else 'skipped' }}" + vars: + multus_nodes_list: "{{ groups['k8s_cluster'] if ansible_play_batch | length == ansible_play_hosts_all | length else ansible_play_batch }}" + when: + - not item is skipped diff --git a/kubespray/project/roles/kubernetes-apps/network_plugin/weave/tasks/main.yml b/kubespray/project/roles/kubernetes-apps/network_plugin/weave/tasks/main.yml new file mode 100644 index 0000000..bc0f932 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/network_plugin/weave/tasks/main.yml @@ -0,0 +1,21 @@ +--- + +- name: Weave | Start Resources + kube: + name: "weave-net" + kubectl: "{{ bin_dir }}/kubectl" + filename: "{{ kube_config_dir }}/weave-net.yml" + resource: "ds" + namespace: "kube-system" + state: "latest" + when: inventory_hostname == groups['kube_control_plane'][0] + +- name: Weave | Wait for Weave to become available + uri: + url: http://127.0.0.1:6784/status + return_content: yes + register: weave_status + retries: 180 + delay: 5 + until: "weave_status.status == 200 and 'Status: ready' in weave_status.content" + when: inventory_hostname == groups['kube_control_plane'][0] diff --git a/kubespray/project/roles/kubernetes-apps/persistent_volumes/aws-ebs-csi/OWNERS b/kubespray/project/roles/kubernetes-apps/persistent_volumes/aws-ebs-csi/OWNERS new file mode 100644 index 0000000..6e44ceb --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/persistent_volumes/aws-ebs-csi/OWNERS @@ -0,0 +1,5 @@ +# See the OWNERS docs at https://go.k8s.io/owners + +approvers: + - alijahnas +reviewers: diff --git a/kubespray/project/roles/kubernetes-apps/persistent_volumes/aws-ebs-csi/defaults/main.yml b/kubespray/project/roles/kubernetes-apps/persistent_volumes/aws-ebs-csi/defaults/main.yml new file mode 100644 index 0000000..896d2d3 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/persistent_volumes/aws-ebs-csi/defaults/main.yml @@ -0,0 +1,8 @@ +--- +# To restrict which AZ the volume should be provisioned in +# set this value to true and set the list of relevant AZs +# For it to work, the flag aws_ebs_csi_enable_volume_scheduling +# in AWS EBS Driver must be true +restrict_az_provisioning: false +aws_ebs_availability_zones: + - eu-west-3c diff --git a/kubespray/project/roles/kubernetes-apps/persistent_volumes/aws-ebs-csi/tasks/main.yml b/kubespray/project/roles/kubernetes-apps/persistent_volumes/aws-ebs-csi/tasks/main.yml new file mode 100644 index 0000000..b49acdf --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/persistent_volumes/aws-ebs-csi/tasks/main.yml @@ -0,0 +1,20 @@ +--- +- name: Kubernetes Persistent Volumes | Copy AWS EBS CSI Storage Class template + template: + src: "aws-ebs-csi-storage-class.yml.j2" + dest: "{{ kube_config_dir }}/aws-ebs-csi-storage-class.yml" + mode: 0644 + register: manifests + when: + - inventory_hostname == groups['kube_control_plane'][0] + +- name: Kubernetes Persistent Volumes | Add AWS EBS CSI Storage Class + kube: + name: aws-ebs-csi + kubectl: "{{ bin_dir }}/kubectl" + resource: StorageClass + filename: "{{ kube_config_dir }}/aws-ebs-csi-storage-class.yml" + state: "latest" + when: + - inventory_hostname == groups['kube_control_plane'][0] + - manifests.changed diff --git a/kubespray/project/roles/kubernetes-apps/persistent_volumes/aws-ebs-csi/templates/aws-ebs-csi-storage-class.yml.j2 b/kubespray/project/roles/kubernetes-apps/persistent_volumes/aws-ebs-csi/templates/aws-ebs-csi-storage-class.yml.j2 new file mode 100644 index 0000000..1632646 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/persistent_volumes/aws-ebs-csi/templates/aws-ebs-csi-storage-class.yml.j2 @@ -0,0 +1,18 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: ebs-sc +provisioner: ebs.csi.aws.com +volumeBindingMode: WaitForFirstConsumer +parameters: + csi.storage.k8s.io/fstype: xfs + type: gp2 +{% if restrict_az_provisioning %} +allowedTopologies: +- matchLabelExpressions: + - key: topology.ebs.csi.aws.com/zone + values: +{% for value in aws_ebs_availability_zones %} + - {{ value }} +{% endfor %} +{% endif %} diff --git a/kubespray/project/roles/kubernetes-apps/persistent_volumes/azuredisk-csi/defaults/main.yml b/kubespray/project/roles/kubernetes-apps/persistent_volumes/azuredisk-csi/defaults/main.yml new file mode 100644 index 0000000..fc92e17 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/persistent_volumes/azuredisk-csi/defaults/main.yml @@ -0,0 +1,3 @@ +--- +## Available values: Standard_LRS, Premium_LRS, StandardSSD_LRS, UltraSSD_LRS +storage_account_type: StandardSSD_LRS diff --git a/kubespray/project/roles/kubernetes-apps/persistent_volumes/azuredisk-csi/tasks/main.yml b/kubespray/project/roles/kubernetes-apps/persistent_volumes/azuredisk-csi/tasks/main.yml new file mode 100644 index 0000000..9abffbe --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/persistent_volumes/azuredisk-csi/tasks/main.yml @@ -0,0 +1,20 @@ +--- +- name: Kubernetes Persistent Volumes | Copy Azure CSI Storage Class template + template: + src: "azure-csi-storage-class.yml.j2" + dest: "{{ kube_config_dir }}/azure-csi-storage-class.yml" + mode: 0644 + register: manifests + when: + - inventory_hostname == groups['kube_control_plane'][0] + +- name: Kubernetes Persistent Volumes | Add Azure CSI Storage Class + kube: + name: azure-csi + kubectl: "{{ bin_dir }}/kubectl" + resource: StorageClass + filename: "{{ kube_config_dir }}/azure-csi-storage-class.yml" + state: "latest" + when: + - inventory_hostname == groups['kube_control_plane'][0] + - manifests.changed diff --git a/kubespray/project/roles/kubernetes-apps/persistent_volumes/azuredisk-csi/templates/azure-csi-storage-class.yml.j2 b/kubespray/project/roles/kubernetes-apps/persistent_volumes/azuredisk-csi/templates/azure-csi-storage-class.yml.j2 new file mode 100644 index 0000000..be5cb38 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/persistent_volumes/azuredisk-csi/templates/azure-csi-storage-class.yml.j2 @@ -0,0 +1,14 @@ +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: disk.csi.azure.com +provisioner: disk.csi.azure.com +parameters: + skuname: {{ storage_account_type }} +{% if azure_csi_tags is defined %} + tags: {{ azure_csi_tags }} +{% endif %} +reclaimPolicy: Delete +volumeBindingMode: Immediate +allowVolumeExpansion: true diff --git a/kubespray/project/roles/kubernetes-apps/persistent_volumes/cinder-csi/defaults/main.yml b/kubespray/project/roles/kubernetes-apps/persistent_volumes/cinder-csi/defaults/main.yml new file mode 100644 index 0000000..5e35dd5 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/persistent_volumes/cinder-csi/defaults/main.yml @@ -0,0 +1,7 @@ +--- +storage_classes: + - name: cinder-csi + is_default: false + parameters: + availability: nova + allowVolumeExpansion: false diff --git a/kubespray/project/roles/kubernetes-apps/persistent_volumes/cinder-csi/tasks/main.yml b/kubespray/project/roles/kubernetes-apps/persistent_volumes/cinder-csi/tasks/main.yml new file mode 100644 index 0000000..52de1c5 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/persistent_volumes/cinder-csi/tasks/main.yml @@ -0,0 +1,20 @@ +--- +- name: Kubernetes Persistent Volumes | Copy Cinder CSI Storage Class template + template: + src: "cinder-csi-storage-class.yml.j2" + dest: "{{ kube_config_dir }}/cinder-csi-storage-class.yml" + mode: 0644 + register: manifests + when: + - inventory_hostname == groups['kube_control_plane'][0] + +- name: Kubernetes Persistent Volumes | Add Cinder CSI Storage Class + kube: + name: cinder-csi + kubectl: "{{ bin_dir }}/kubectl" + resource: StorageClass + filename: "{{ kube_config_dir }}/cinder-csi-storage-class.yml" + state: "latest" + when: + - inventory_hostname == groups['kube_control_plane'][0] + - manifests.changed diff --git a/kubespray/project/roles/kubernetes-apps/persistent_volumes/cinder-csi/templates/cinder-csi-storage-class.yml.j2 b/kubespray/project/roles/kubernetes-apps/persistent_volumes/cinder-csi/templates/cinder-csi-storage-class.yml.j2 new file mode 100644 index 0000000..be8ba13 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/persistent_volumes/cinder-csi/templates/cinder-csi-storage-class.yml.j2 @@ -0,0 +1,25 @@ +{% for class in storage_classes %} +--- +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: "{{ class.name }}" + annotations: + storageclass.kubernetes.io/is-default-class: "{{ class.is_default | default(false) | ternary("true","false") }}" +provisioner: cinder.csi.openstack.org +volumeBindingMode: WaitForFirstConsumer +parameters: +{% for key, value in (class.parameters | default({})).items() %} + "{{ key }}": "{{ value }}" +{% endfor %} +{% if cinder_topology is defined and cinder_topology is sameas true %} +allowedTopologies: +- matchLabelExpressions: + - key: topology.cinder.csi.openstack.org/zone + values: +{% for zone in cinder_topology_zones %} + - "{{ zone }}" +{% endfor %} +{% endif %} +allowVolumeExpansion: {{ expand_persistent_volumes }} +{% endfor %} diff --git a/kubespray/project/roles/kubernetes-apps/persistent_volumes/gcp-pd-csi/defaults/main.yml b/kubespray/project/roles/kubernetes-apps/persistent_volumes/gcp-pd-csi/defaults/main.yml new file mode 100644 index 0000000..d58706f --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/persistent_volumes/gcp-pd-csi/defaults/main.yml @@ -0,0 +1,8 @@ +--- +# Choose between pd-standard and pd-ssd +gcp_pd_csi_volume_type: pd-standard +gcp_pd_regional_replication_enabled: false +gcp_pd_restrict_zone_replication: false +gcp_pd_restricted_zones: + - europe-west1-b + - europe-west1-c diff --git a/kubespray/project/roles/kubernetes-apps/persistent_volumes/gcp-pd-csi/tasks/main.yml b/kubespray/project/roles/kubernetes-apps/persistent_volumes/gcp-pd-csi/tasks/main.yml new file mode 100644 index 0000000..29997e7 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/persistent_volumes/gcp-pd-csi/tasks/main.yml @@ -0,0 +1,20 @@ +--- +- name: Kubernetes Persistent Volumes | Copy GCP PD CSI Storage Class template + template: + src: "gcp-pd-csi-storage-class.yml.j2" + dest: "{{ kube_config_dir }}/gcp-pd-csi-storage-class.yml" + mode: 0644 + register: manifests + when: + - inventory_hostname == groups['kube_control_plane'][0] + +- name: Kubernetes Persistent Volumes | Add GCP PD CSI Storage Class + kube: + name: gcp-pd-csi + kubectl: "{{ bin_dir }}/kubectl" + resource: StorageClass + filename: "{{ kube_config_dir }}/gcp-pd-csi-storage-class.yml" + state: "latest" + when: + - inventory_hostname == groups['kube_control_plane'][0] + - manifests.changed diff --git a/kubespray/project/roles/kubernetes-apps/persistent_volumes/gcp-pd-csi/templates/gcp-pd-csi-storage-class.yml.j2 b/kubespray/project/roles/kubernetes-apps/persistent_volumes/gcp-pd-csi/templates/gcp-pd-csi-storage-class.yml.j2 new file mode 100644 index 0000000..475eb4f --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/persistent_volumes/gcp-pd-csi/templates/gcp-pd-csi-storage-class.yml.j2 @@ -0,0 +1,20 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: csi-gce-pd +provisioner: pd.csi.storage.gke.io +parameters: + type: {{ gcp_pd_csi_volume_type }} +{% if gcp_pd_regional_replication_enabled %} + replication-type: regional-pd +{% endif %} +volumeBindingMode: WaitForFirstConsumer +{% if gcp_pd_restrict_zone_replication %} +allowedTopologies: +- matchLabelExpressions: + - key: topology.gke.io/zone + values: +{% for value in gcp_pd_restricted_zones %} + - {{ value }} +{% endfor %} +{% endif %} diff --git a/kubespray/project/roles/kubernetes-apps/persistent_volumes/meta/main.yml b/kubespray/project/roles/kubernetes-apps/persistent_volumes/meta/main.yml new file mode 100644 index 0000000..e3066bb --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/persistent_volumes/meta/main.yml @@ -0,0 +1,43 @@ +--- +dependencies: + - role: kubernetes-apps/persistent_volumes/openstack + when: + - cloud_provider is defined + - cloud_provider in [ 'openstack' ] + tags: + - persistent_volumes_openstack + + - role: kubernetes-apps/persistent_volumes/cinder-csi + when: + - cinder_csi_enabled + tags: + - persistent_volumes_cinder_csi + - cinder-csi-driver + + - role: kubernetes-apps/persistent_volumes/aws-ebs-csi + when: + - aws_ebs_csi_enabled + tags: + - persistent_volumes_aws_ebs_csi + - aws-ebs-csi-driver + + - role: kubernetes-apps/persistent_volumes/azuredisk-csi + when: + - azure_csi_enabled + tags: + - persistent_volumes_azure_csi + - azure-csi-driver + + - role: kubernetes-apps/persistent_volumes/gcp-pd-csi + when: + - gcp_pd_csi_enabled + tags: + - persistent_volumes_gcp_pd_csi + - gcp-pd-csi-driver + + - role: kubernetes-apps/persistent_volumes/upcloud-csi + when: + - upcloud_csi_enabled + tags: + - persistent_volumes_upcloud_csi + - upcloud-csi-driver diff --git a/kubespray/project/roles/kubernetes-apps/persistent_volumes/openstack/defaults/main.yml b/kubespray/project/roles/kubernetes-apps/persistent_volumes/openstack/defaults/main.yml new file mode 100644 index 0000000..05a3d94 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/persistent_volumes/openstack/defaults/main.yml @@ -0,0 +1,7 @@ +--- +persistent_volumes_enabled: false +storage_classes: + - name: standard + is_default: true + parameters: + availability: nova diff --git a/kubespray/project/roles/kubernetes-apps/persistent_volumes/openstack/tasks/main.yml b/kubespray/project/roles/kubernetes-apps/persistent_volumes/openstack/tasks/main.yml new file mode 100644 index 0000000..3387e7f --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/persistent_volumes/openstack/tasks/main.yml @@ -0,0 +1,20 @@ +--- +- name: Kubernetes Persistent Volumes | Lay down OpenStack Cinder Storage Class template + template: + src: "openstack-storage-class.yml.j2" + dest: "{{ kube_config_dir }}/openstack-storage-class.yml" + mode: 0644 + register: manifests + when: + - inventory_hostname == groups['kube_control_plane'][0] + +- name: Kubernetes Persistent Volumes | Add OpenStack Cinder Storage Class + kube: + name: storage-class + kubectl: "{{ bin_dir }}/kubectl" + resource: StorageClass + filename: "{{ kube_config_dir }}/openstack-storage-class.yml" + state: "latest" + when: + - inventory_hostname == groups['kube_control_plane'][0] + - manifests.changed diff --git a/kubespray/project/roles/kubernetes-apps/persistent_volumes/openstack/templates/openstack-storage-class.yml.j2 b/kubespray/project/roles/kubernetes-apps/persistent_volumes/openstack/templates/openstack-storage-class.yml.j2 new file mode 100644 index 0000000..973353c --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/persistent_volumes/openstack/templates/openstack-storage-class.yml.j2 @@ -0,0 +1,27 @@ +{% for class in storage_classes %} +--- +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: "{{ class.name }}" + annotations: + storageclass.kubernetes.io/is-default-class: "{{ class.is_default | default(false) | ternary("true","false") }}" +provisioner: kubernetes.io/cinder +{% if class.mount_options is defined %} +mountOptions: +{% for option in class.mount_options | default([]) %} + - "{{ option }}" +{% endfor %} +{% endif %} +parameters: +{% for key, value in (class.parameters | default({})).items() %} + "{{ key }}": "{{ value }}" +{% endfor %} +{% if class.reclaim_policy is defined %} +reclaimPolicy: "{{ class.reclaim_policy }}" +{% endif %} +{% if class.volume_binding_mode is defined %} +volumeBindingMode: "{{ class.volume_binding_mode }}" +{% endif %} +allowVolumeExpansion: {{ expand_persistent_volumes }} +{% endfor %} diff --git a/kubespray/project/roles/kubernetes-apps/persistent_volumes/upcloud-csi/defaults/main.yml b/kubespray/project/roles/kubernetes-apps/persistent_volumes/upcloud-csi/defaults/main.yml new file mode 100644 index 0000000..5986e8c --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/persistent_volumes/upcloud-csi/defaults/main.yml @@ -0,0 +1,12 @@ +--- +storage_classes: + - name: standard + is_default: true + expand_persistent_volumes: true + parameters: + tier: maxiops + - name: hdd + is_default: false + expand_persistent_volumes: true + parameters: + tier: hdd diff --git a/kubespray/project/roles/kubernetes-apps/persistent_volumes/upcloud-csi/tasks/main.yml b/kubespray/project/roles/kubernetes-apps/persistent_volumes/upcloud-csi/tasks/main.yml new file mode 100644 index 0000000..26104a0 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/persistent_volumes/upcloud-csi/tasks/main.yml @@ -0,0 +1,20 @@ +--- +- name: Kubernetes Persistent Volumes | Copy UpCloud CSI Storage Class template + template: + src: "upcloud-csi-storage-class.yml.j2" + dest: "{{ kube_config_dir }}/upcloud-csi-storage-class.yml" + mode: 0644 + register: manifests + when: + - inventory_hostname == groups['kube_control_plane'][0] + +- name: Kubernetes Persistent Volumes | Add UpCloud CSI Storage Class + kube: + name: upcloud-csi + kubectl: "{{ bin_dir }}/kubectl" + resource: StorageClass + filename: "{{ kube_config_dir }}/upcloud-csi-storage-class.yml" + state: "latest" + when: + - inventory_hostname == groups['kube_control_plane'][0] + - manifests.changed diff --git a/kubespray/project/roles/kubernetes-apps/persistent_volumes/upcloud-csi/templates/upcloud-csi-storage-class.yml.j2 b/kubespray/project/roles/kubernetes-apps/persistent_volumes/upcloud-csi/templates/upcloud-csi-storage-class.yml.j2 new file mode 100644 index 0000000..a40df9b --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/persistent_volumes/upcloud-csi/templates/upcloud-csi-storage-class.yml.j2 @@ -0,0 +1,16 @@ +{% for class in storage_classes %} +--- +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: "{{ class.name }}" + annotations: + storageclass.kubernetes.io/is-default-class: "{{ class.is_default | default(false) | ternary("true","false") }}" +provisioner: storage.csi.upcloud.com +reclaimPolicy: Delete +parameters: +{% for key, value in (class.parameters | default({})).items() %} + "{{ key }}": "{{ value }}" +{% endfor %} +allowVolumeExpansion: {{ class.expand_persistent_volumes | default(true) | ternary("true","false") }} +{% endfor %} diff --git a/kubespray/project/roles/kubernetes-apps/policy_controller/calico/defaults/main.yml b/kubespray/project/roles/kubernetes-apps/policy_controller/calico/defaults/main.yml new file mode 100644 index 0000000..d3a780c --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/policy_controller/calico/defaults/main.yml @@ -0,0 +1,10 @@ +--- +# Limits for calico apps +calico_policy_controller_cpu_limit: 1000m +calico_policy_controller_memory_limit: 256M +calico_policy_controller_cpu_requests: 30m +calico_policy_controller_memory_requests: 64M +calico_policy_controller_deployment_nodeselector: "kubernetes.io/os: linux" + +# SSL +calico_cert_dir: "/etc/calico/certs" diff --git a/kubespray/project/roles/kubernetes-apps/policy_controller/calico/tasks/main.yml b/kubespray/project/roles/kubernetes-apps/policy_controller/calico/tasks/main.yml new file mode 100644 index 0000000..ba2eebb --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/policy_controller/calico/tasks/main.yml @@ -0,0 +1,34 @@ +--- +- name: Create calico-kube-controllers manifests + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/{{ item.file }}" + mode: 0644 + with_items: + - {name: calico-kube-controllers, file: calico-kube-controllers.yml, type: deployment} + - {name: calico-kube-controllers, file: calico-kube-sa.yml, type: sa} + - {name: calico-kube-controllers, file: calico-kube-cr.yml, type: clusterrole} + - {name: calico-kube-controllers, file: calico-kube-crb.yml, type: clusterrolebinding} + register: calico_kube_manifests + when: + - inventory_hostname == groups['kube_control_plane'][0] + - rbac_enabled or item.type not in rbac_resources + +- name: Start of Calico kube controllers + kube: + name: "{{ item.item.name }}" + namespace: "kube-system" + kubectl: "{{ bin_dir }}/kubectl" + resource: "{{ item.item.type }}" + filename: "{{ kube_config_dir }}/{{ item.item.file }}" + state: "latest" + with_items: + - "{{ calico_kube_manifests.results }}" + register: calico_kube_controller_start + until: calico_kube_controller_start is succeeded + retries: 4 + when: + - inventory_hostname == groups['kube_control_plane'][0] + - not item is skipped + loop_control: + label: "{{ item.item.file }}" diff --git a/kubespray/project/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-controllers.yml.j2 b/kubespray/project/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-controllers.yml.j2 new file mode 100644 index 0000000..490111c --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-controllers.yml.j2 @@ -0,0 +1,85 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: calico-kube-controllers + namespace: kube-system + labels: + k8s-app: calico-kube-controllers +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + k8s-app: calico-kube-controllers + template: + metadata: + name: calico-kube-controllers + namespace: kube-system + labels: + k8s-app: calico-kube-controllers + spec: + nodeSelector: + {{ calico_policy_controller_deployment_nodeselector }} +{% if calico_datastore == "etcd" %} + hostNetwork: true +{% endif %} + serviceAccountName: calico-kube-controllers + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule +{% if policy_controller_extra_tolerations is defined %} + {{ policy_controller_extra_tolerations | list | to_nice_yaml(indent=2) | indent(8) }} +{% endif %} + priorityClassName: system-cluster-critical + containers: + - name: calico-kube-controllers + image: {{ calico_policy_image_repo }}:{{ calico_policy_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + resources: + limits: + cpu: {{ calico_policy_controller_cpu_limit }} + memory: {{ calico_policy_controller_memory_limit }} + requests: + cpu: {{ calico_policy_controller_cpu_requests }} + memory: {{ calico_policy_controller_memory_requests }} + livenessProbe: + exec: + command: + - /usr/bin/check-status + - -l + periodSeconds: 10 + initialDelaySeconds: 10 + failureThreshold: 6 + readinessProbe: + exec: + command: + - /usr/bin/check-status + - -r + periodSeconds: 10 + env: +{% if calico_datastore == "kdd" %} + - name: ENABLED_CONTROLLERS + value: node + - name: DATASTORE_TYPE + value: kubernetes +{% else %} + - name: ETCD_ENDPOINTS + value: "{{ etcd_access_addresses }}" + - name: ETCD_CA_CERT_FILE + value: "{{ calico_cert_dir }}/ca_cert.crt" + - name: ETCD_CERT_FILE + value: "{{ calico_cert_dir }}/cert.crt" + - name: ETCD_KEY_FILE + value: "{{ calico_cert_dir }}/key.pem" + volumeMounts: + - mountPath: {{ calico_cert_dir }} + name: etcd-certs + readOnly: true + volumes: + - hostPath: + path: {{ calico_cert_dir }} + name: etcd-certs +{% endif %} diff --git a/kubespray/project/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-cr.yml.j2 b/kubespray/project/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-cr.yml.j2 new file mode 100644 index 0000000..f74b291 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-cr.yml.j2 @@ -0,0 +1,110 @@ +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: calico-kube-controllers + namespace: kube-system +rules: +{% if calico_datastore == "etcd" %} + - apiGroups: + - "" + - extensions + resources: + - pods + - namespaces + - networkpolicies + - nodes + - serviceaccounts + verbs: + - watch + - list + - get + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - watch + - list +{% elif calico_datastore == "kdd" %} + # Nodes are watched to monitor for deletions. + - apiGroups: [""] + resources: + - nodes + verbs: + - watch + - list + - get + # Pods are queried to check for existence. + - apiGroups: [""] + resources: + - pods + verbs: + - watch + - list + - get + # IPAM resources are manipulated when nodes are deleted. + - apiGroups: ["crd.projectcalico.org"] + resources: + - ipreservations + verbs: + - list + # Pools are watched to maintain a mapping of blocks to IP pools. + - apiGroups: ["crd.projectcalico.org"] + resources: + - ippools + verbs: + - list + - watch + - apiGroups: ["crd.projectcalico.org"] + resources: + - blockaffinities + - ipamblocks + - ipamhandles + verbs: + - get + - list + - create + - update + - delete + - watch + # kube-controllers manages hostendpoints. + - apiGroups: ["crd.projectcalico.org"] + resources: + - hostendpoints + verbs: + - get + - list + - create + - update + - delete + # Needs access to update clusterinformations. + - apiGroups: ["crd.projectcalico.org"] + resources: + - clusterinformations + verbs: + - get + - list + - create + - update + - watch + # KubeControllersConfiguration is where it gets its config + - apiGroups: ["crd.projectcalico.org"] + resources: + - kubecontrollersconfigurations + verbs: + # read its own config + - get + # create a default if none exists + - create + # update status + - update + # watch for changes + - watch +{% endif %} diff --git a/kubespray/project/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-crb.yml.j2 b/kubespray/project/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-crb.yml.j2 new file mode 100644 index 0000000..8168056 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-crb.yml.j2 @@ -0,0 +1,13 @@ +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: calico-kube-controllers +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-kube-controllers +subjects: +- kind: ServiceAccount + name: calico-kube-controllers + namespace: kube-system diff --git a/kubespray/project/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-sa.yml.j2 b/kubespray/project/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-sa.yml.j2 new file mode 100644 index 0000000..269d0a1 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-sa.yml.j2 @@ -0,0 +1,6 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: calico-kube-controllers + namespace: kube-system diff --git a/kubespray/project/roles/kubernetes-apps/policy_controller/meta/main.yml b/kubespray/project/roles/kubernetes-apps/policy_controller/meta/main.yml new file mode 100644 index 0000000..00fa041 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/policy_controller/meta/main.yml @@ -0,0 +1,8 @@ +--- +dependencies: + - role: policy_controller/calico + when: + - kube_network_plugin in ['calico'] + - enable_network_policy + tags: + - policy-controller diff --git a/kubespray/project/roles/kubernetes-apps/registry/defaults/main.yml b/kubespray/project/roles/kubernetes-apps/registry/defaults/main.yml new file mode 100644 index 0000000..6353b7c --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/registry/defaults/main.yml @@ -0,0 +1,48 @@ +--- +registry_namespace: "kube-system" +registry_storage_class: "" +registry_storage_access_mode: "ReadWriteOnce" +registry_disk_size: "10Gi" +registry_port: 5000 +registry_replica_count: 1 + +# type of service: ClusterIP, LoadBalancer or NodePort +registry_service_type: "ClusterIP" +# you can specify your cluster IP address when registry_service_type is ClusterIP +registry_service_cluster_ip: "" +# you can specify your cloud provider assigned loadBalancerIP when registry_service_type is LoadBalancer +registry_service_loadbalancer_ip: "" +# annotations for managing Cloud Load Balancers +registry_service_annotations: {} +# you can specify the node port when registry_service_type is NodePort +registry_service_nodeport: "" + +# name of kubernetes secret for registry TLS certs +registry_tls_secret: "" + +registry_htpasswd: "" + +# registry configuration +# see: https://docs.docker.com/registry/configuration/#list-of-configuration-options +registry_config: + version: 0.1 + log: + fields: + service: registry + storage: + cache: + blobdescriptor: inmemory + http: + addr: :{{ registry_port }} + headers: + X-Content-Type-Options: [nosniff] + health: + storagedriver: + enabled: true + interval: 10s + threshold: 3 + +registry_ingress_annotations: {} +registry_ingress_host: "" +# name of kubernetes secret for registry ingress TLS certs +registry_ingress_tls_secret: "" diff --git a/kubespray/project/roles/kubernetes-apps/registry/tasks/main.yml b/kubespray/project/roles/kubernetes-apps/registry/tasks/main.yml new file mode 100644 index 0000000..a915e07 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/registry/tasks/main.yml @@ -0,0 +1,98 @@ +--- +- name: Registry | check registry_service_type value + fail: + msg: "registry_service_type can only be 'ClusterIP', 'LoadBalancer' or 'NodePort'" + when: registry_service_type not in ['ClusterIP', 'LoadBalancer', 'NodePort'] + +- name: Registry | Stop if registry_service_cluster_ip is defined when registry_service_type is not 'ClusterIP' + fail: + msg: "registry_service_cluster_ip support only compatible with ClusterIP." + when: + - registry_service_cluster_ip is defined and registry_service_cluster_ip | length > 0 + - registry_service_type != "ClusterIP" + +- name: Registry | Stop if registry_service_loadbalancer_ip is defined when registry_service_type is not 'LoadBalancer' + fail: + msg: "registry_service_loadbalancer_ip support only compatible with LoadBalancer." + when: + - registry_service_loadbalancer_ip is defined and registry_service_loadbalancer_ip | length > 0 + - registry_service_type != "LoadBalancer" + +- name: Registry | Stop if registry_service_nodeport is defined when registry_service_type is not 'NodePort' + fail: + msg: "registry_service_nodeport support only compatible with NodePort." + when: + - registry_service_nodeport is defined and registry_service_nodeport | length > 0 + - registry_service_type != "NodePort" + +- name: Registry | Create addon dir + file: + path: "{{ kube_config_dir }}/addons/registry" + state: directory + owner: root + group: root + mode: 0755 + +- name: Registry | Templates list + set_fact: + registry_templates: + - { name: registry-ns, file: registry-ns.yml, type: ns } + - { name: registry-sa, file: registry-sa.yml, type: sa } + - { name: registry-svc, file: registry-svc.yml, type: svc } + - { name: registry-secrets, file: registry-secrets.yml, type: secrets } + - { name: registry-cm, file: registry-cm.yml, type: cm } + - { name: registry-rs, file: registry-rs.yml, type: rs } + +- name: Registry | Append nginx ingress templates to Registry Templates list when ingress enabled + set_fact: + registry_templates: "{{ registry_templates + [item] }}" + with_items: + - [{ name: registry-ing, file: registry-ing.yml, type: ing }] + when: ingress_nginx_enabled or ingress_alb_enabled + +- name: Registry | Create manifests + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/addons/registry/{{ item.file }}" + mode: 0644 + with_items: "{{ registry_templates }}" + register: registry_manifests + when: inventory_hostname == groups['kube_control_plane'][0] + +- name: Registry | Apply manifests + kube: + name: "{{ item.item.name }}" + namespace: "{{ registry_namespace }}" + kubectl: "{{ bin_dir }}/kubectl" + resource: "{{ item.item.type }}" + filename: "{{ kube_config_dir }}/addons/registry/{{ item.item.file }}" + state: "latest" + with_items: "{{ registry_manifests.results }}" + when: inventory_hostname == groups['kube_control_plane'][0] + +- name: Registry | Create PVC manifests + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/addons/registry/{{ item.file }}" + mode: 0644 + with_items: + - { name: registry-pvc, file: registry-pvc.yml, type: pvc } + register: registry_manifests + when: + - registry_storage_class != none and registry_storage_class + - registry_disk_size != none and registry_disk_size + - inventory_hostname == groups['kube_control_plane'][0] + +- name: Registry | Apply PVC manifests + kube: + name: "{{ item.item.name }}" + namespace: "{{ registry_namespace }}" + kubectl: "{{ bin_dir }}/kubectl" + resource: "{{ item.item.type }}" + filename: "{{ kube_config_dir }}/addons/registry/{{ item.item.file }}" + state: "latest" + with_items: "{{ registry_manifests.results }}" + when: + - registry_storage_class != none and registry_storage_class + - registry_disk_size != none and registry_disk_size + - inventory_hostname == groups['kube_control_plane'][0] diff --git a/kubespray/project/roles/kubernetes-apps/registry/templates/registry-cm.yml.j2 b/kubespray/project/roles/kubernetes-apps/registry/templates/registry-cm.yml.j2 new file mode 100644 index 0000000..b633dfd --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/registry/templates/registry-cm.yml.j2 @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: registry-config + namespace: {{ registry_namespace }} +{% if registry_config %} +data: + config.yml: |- + {{ registry_config | to_yaml(indent=2, width=1337) | indent(width=4) }} +{% endif %} diff --git a/kubespray/project/roles/kubernetes-apps/registry/templates/registry-ing.yml.j2 b/kubespray/project/roles/kubernetes-apps/registry/templates/registry-ing.yml.j2 new file mode 100644 index 0000000..29dfbba --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/registry/templates/registry-ing.yml.j2 @@ -0,0 +1,27 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: registry + namespace: {{ registry_namespace }} +{% if registry_ingress_annotations %} + annotations: + {{ registry_ingress_annotations | to_nice_yaml(indent=2, width=1337) | indent(width=4) }} +{% endif %} +spec: +{% if registry_ingress_tls_secret %} + tls: + - hosts: + - {{ registry_ingress_host }} + secretName: {{ registry_ingress_tls_secret }} +{% endif %} + rules: + - host: {{ registry_ingress_host }} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: registry + port: + number: {{ registry_port }} diff --git a/kubespray/project/roles/kubernetes-apps/registry/templates/registry-ns.yml.j2 b/kubespray/project/roles/kubernetes-apps/registry/templates/registry-ns.yml.j2 new file mode 100644 index 0000000..c224337 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/registry/templates/registry-ns.yml.j2 @@ -0,0 +1,7 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: {{ registry_namespace }} + labels: + name: {{ registry_namespace }} diff --git a/kubespray/project/roles/kubernetes-apps/registry/templates/registry-pvc.yml.j2 b/kubespray/project/roles/kubernetes-apps/registry/templates/registry-pvc.yml.j2 new file mode 100644 index 0000000..dc3fa5a --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/registry/templates/registry-pvc.yml.j2 @@ -0,0 +1,15 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: registry-pvc + namespace: {{ registry_namespace }} + labels: + addonmanager.kubernetes.io/mode: Reconcile +spec: + accessModes: + - {{ registry_storage_access_mode }} + storageClassName: {{ registry_storage_class }} + resources: + requests: + storage: {{ registry_disk_size }} diff --git a/kubespray/project/roles/kubernetes-apps/registry/templates/registry-rs.yml.j2 b/kubespray/project/roles/kubernetes-apps/registry/templates/registry-rs.yml.j2 new file mode 100644 index 0000000..3b51684 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/registry/templates/registry-rs.yml.j2 @@ -0,0 +1,115 @@ +--- +apiVersion: apps/v1 +kind: ReplicaSet +metadata: + name: registry + namespace: {{ registry_namespace }} + labels: + k8s-app: registry + version: v{{ registry_image_tag }} + addonmanager.kubernetes.io/mode: Reconcile +spec: +{% if registry_storage_class != "" and registry_storage_access_mode == "ReadWriteMany" %} + replicas: {{ registry_replica_count }} +{% else %} + replicas: 1 +{% endif %} + selector: + matchLabels: + k8s-app: registry + version: v{{ registry_image_tag }} + template: + metadata: + labels: + k8s-app: registry + version: v{{ registry_image_tag }} + spec: + priorityClassName: {% if registry_namespace == 'kube-system' %}system-cluster-critical{% else %}k8s-cluster-critical{% endif %}{{ '' }} + serviceAccountName: registry + securityContext: + fsGroup: 1000 + runAsUser: 1000 + containers: + - name: registry + image: {{ registry_image_repo }}:{{ registry_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + command: + - /bin/registry + - serve + - /etc/docker/registry/config.yml + env: + - name: REGISTRY_HTTP_ADDR + value: :{{ registry_port }} + - name: REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY + value: /var/lib/registry +{% if registry_htpasswd != "" %} + - name: REGISTRY_AUTH + value: "htpasswd" + - name: REGISTRY_AUTH_HTPASSWD_REALM + value: "Registry Realm" + - name: REGISTRY_AUTH_HTPASSWD_PATH + value: "/auth/htpasswd" +{% endif %} +{% if registry_tls_secret != "" %} + - name: REGISTRY_HTTP_TLS_CERTIFICATE + value: /etc/ssl/docker/tls.crt + - name: REGISTRY_HTTP_TLS_KEY + value: /etc/ssl/docker/tls.key +{% endif %} + volumeMounts: + - name: registry-pvc + mountPath: /var/lib/registry + - name: registry-config + mountPath: /etc/docker/registry +{% if registry_htpasswd != "" %} + - name: auth + mountPath: /auth + readOnly: true +{% endif %} +{% if registry_tls_secret != "" %} + - name: tls-cert + mountPath: /etc/ssl/docker + readOnly: true +{% endif %} + ports: + - containerPort: {{ registry_port }} + name: registry + protocol: TCP + livenessProbe: + httpGet: +{% if registry_tls_secret != "" %} + scheme: HTTPS +{% endif %} + path: / + port: {{ registry_port }} + readinessProbe: + httpGet: +{% if registry_tls_secret != "" %} + scheme: HTTPS +{% endif %} + path: / + port: {{ registry_port }} + volumes: + - name: registry-pvc +{% if registry_storage_class != "" %} + persistentVolumeClaim: + claimName: registry-pvc +{% else %} + emptyDir: {} +{% endif %} + - name: registry-config + configMap: + name: registry-config +{% if registry_htpasswd != "" %} + - name: auth + secret: + secretName: registry-secret + items: + - key: htpasswd + path: htpasswd +{% endif %} +{% if registry_tls_secret != "" %} + - name: tls-cert + secret: + secretName: {{ registry_tls_secret }} +{% endif %} diff --git a/kubespray/project/roles/kubernetes-apps/registry/templates/registry-sa.yml.j2 b/kubespray/project/roles/kubernetes-apps/registry/templates/registry-sa.yml.j2 new file mode 100644 index 0000000..20f9515 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/registry/templates/registry-sa.yml.j2 @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: registry + namespace: {{ registry_namespace }} diff --git a/kubespray/project/roles/kubernetes-apps/registry/templates/registry-secrets.yml.j2 b/kubespray/project/roles/kubernetes-apps/registry/templates/registry-secrets.yml.j2 new file mode 100644 index 0000000..80727d2 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/registry/templates/registry-secrets.yml.j2 @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Secret +metadata: + name: registry-secret + namespace: {{ registry_namespace }} +type: Opaque +data: +{% if registry_htpasswd != "" %} + htpasswd: {{ registry_htpasswd | b64encode }} +{% endif %} diff --git a/kubespray/project/roles/kubernetes-apps/registry/templates/registry-svc.yml.j2 b/kubespray/project/roles/kubernetes-apps/registry/templates/registry-svc.yml.j2 new file mode 100644 index 0000000..5485aa8 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/registry/templates/registry-svc.yml.j2 @@ -0,0 +1,32 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: registry + namespace: {{ registry_namespace }} + labels: + k8s-app: registry + addonmanager.kubernetes.io/mode: Reconcile + kubernetes.io/name: "KubeRegistry" +{% if registry_service_annotations %} + annotations: + {{ registry_service_annotations | to_nice_yaml(indent=2, width=1337) | indent(width=4) }} +{% endif %} +spec: + selector: + k8s-app: registry + type: {{ registry_service_type }} +{% if registry_service_type == "ClusterIP" and registry_service_cluster_ip != "" %} + clusterIP: {{ registry_service_cluster_ip }} +{% endif %} +{% if registry_service_type == "LoadBalancer" and registry_service_loadbalancer_ip != "" %} + loadBalancerIP: {{ registry_service_loadbalancer_ip }} +{% endif %} + ports: + - name: registry + port: {{ registry_port }} + protocol: TCP + targetPort: {{ registry_port }} +{% if registry_service_type == "NodePort" and registry_service_nodeport != "" %} + nodePort: {{ registry_service_nodeport }} +{% endif %} diff --git a/kubespray/project/roles/kubernetes-apps/scheduler_plugins/defaults/main.yml b/kubespray/project/roles/kubernetes-apps/scheduler_plugins/defaults/main.yml new file mode 100644 index 0000000..43f50f5 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/scheduler_plugins/defaults/main.yml @@ -0,0 +1,29 @@ +--- +scheduler_plugins_enabled: false + +scheduler_plugins_namespace: scheduler-plugins + +scheduler_plugins_controller_replicas: 1 + +scheduler_plugins_scheduler_replicas: 1 + +# The default is determined by the number of control plane nodes. +scheduler_plugins_scheduler_leader_elect: "{{ ((groups['kube_control_plane'] | length) > 1) }}" + +# Plugins to enable. See https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/manifests/install/charts/as-a-second-scheduler/README.md#configuration for more info. +scheduler_plugins_enabled_plugins: + - Coscheduling + - CapacityScheduling + - NodeResourceTopologyMatch + - NodeResourcesAllocatable + +# Plugins to disable. See https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/manifests/install/charts/as-a-second-scheduler/README.md#configuration for more info. +scheduler_plugins_disabled_plugins: + - PrioritySort + +# Customize the enabled plugins' config. +# Refer to the "pluginConfig" section of https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/manifests//scheduler-config.yaml. +scheduler_plugins_plugin_config: + - name: Coscheduling + args: + permitWaitingTimeSeconds: 10 # default is 60 diff --git a/kubespray/project/roles/kubernetes-apps/scheduler_plugins/tasks/main.yml b/kubespray/project/roles/kubernetes-apps/scheduler_plugins/tasks/main.yml new file mode 100644 index 0000000..d17b191 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/scheduler_plugins/tasks/main.yml @@ -0,0 +1,68 @@ +--- +- name: Scheduler Plugins | Ensure dir exists + file: + path: "{{ kube_config_dir }}/scheduler-plugins" + state: directory + owner: root + group: root + mode: 0755 + when: inventory_hostname == groups['kube_control_plane'][0] + tags: + - scheduler_plugins + +- name: Scheduler Plugins | Create manifests + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/scheduler-plugins/{{ item.file }}" + mode: 0644 + with_items: + - { name: appgroup, file: appgroup.diktyo.x-k8s.io_appgroups.yaml, type: crd } + - { name: networktopology, file: networktopology.diktyo.x-k8s.io_networktopologies.yaml, type: crd } + - { name: elasticquotas, file: scheduling.x-k8s.io_elasticquotas.yaml, type: crd } + - { name: podgroups, file: scheduling.x-k8s.io_podgroups.yaml, type: crd } + - { name: noderesourcetopologies, file: topology.node.k8s.io_noderesourcetopologies.yaml, type: crd } + - { name: namespace, file: namespace.yaml, type: namespace } + - { name: sa, file: sa-scheduler-plugins.yaml, type: serviceaccount } + - { name: rbac, file: rbac-scheduler-plugins.yaml, type: rbac } + - { name: cm, file: cm-scheduler-plugins.yaml, type: configmap } + - { name: deploy, file: deploy-scheduler-plugins.yaml, type: deployment } + register: scheduler_plugins_manifests + when: inventory_hostname == groups['kube_control_plane'][0] + tags: + - scheduler_plugins + +- name: Scheduler Plugins | Apply manifests + kube: + name: "{{ item.item.name }}" + kubectl: "{{ bin_dir }}/kubectl" + resource: "{{ item.item.type }}" + filename: "{{ kube_config_dir }}/scheduler-plugins/{{ item.item.file }}" + state: "latest" + with_items: "{{ scheduler_plugins_manifests.results }}" + when: inventory_hostname == groups['kube_control_plane'][0] + tags: + - scheduler_plugins + +- name: Scheduler Plugins | Wait for controller pods to be ready + command: "{{ kubectl }} -n {{ scheduler_plugins_namespace }} get pods -l app=scheduler-plugins-controller -o jsonpath='{.items[?(@.status.containerStatuses[0].ready==false)].metadata.name}'" # noqa ignore-errors + register: controller_pods_not_ready + until: controller_pods_not_ready.stdout.find("scheduler-plugins-controller")==-1 + retries: 30 + delay: 10 + ignore_errors: true + changed_when: false + when: inventory_hostname == groups['kube_control_plane'][0] + tags: + - scheduler_plugins + +- name: Scheduler Plugins | Wait for scheduler pods to be ready + command: "{{ kubectl }} -n {{ scheduler_plugins_namespace }} get pods -l component=scheduler -o jsonpath='{.items[?(@.status.containerStatuses[0].ready==false)].metadata.name}'" # noqa ignore-errors + register: scheduler_pods_not_ready + until: scheduler_pods_not_ready.stdout.find("scheduler-plugins-scheduler")==-1 + retries: 30 + delay: 10 + ignore_errors: true + changed_when: false + when: inventory_hostname == groups['kube_control_plane'][0] + tags: + - scheduler_plugins diff --git a/kubespray/project/roles/kubernetes-apps/scheduler_plugins/templates/appgroup.diktyo.x-k8s.io_appgroups.yaml.j2 b/kubespray/project/roles/kubernetes-apps/scheduler_plugins/templates/appgroup.diktyo.x-k8s.io_appgroups.yaml.j2 new file mode 100644 index 0000000..757a3b1 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/scheduler_plugins/templates/appgroup.diktyo.x-k8s.io_appgroups.yaml.j2 @@ -0,0 +1,197 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.kubernetes.io: https://github.com/kubernetes-sigs/scheduler-plugins/pull/432 # edited manually + controller-gen.kubebuilder.io/version: v0.11.1 + creationTimestamp: null + name: appgroups.appgroup.diktyo.x-k8s.io +spec: + group: appgroup.diktyo.x-k8s.io + names: + kind: AppGroup + listKind: AppGroupList + plural: appgroups + shortNames: + - ag + singular: appgroup + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: AppGroup is a collection of Pods belonging to the same application. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AppGroupSpec defines the number of Pods and which Pods belong + to the group. + properties: + numMembers: + description: NumMembers defines the number of Pods belonging to the + App Group + format: int32 + minimum: 1 + type: integer + topologySortingAlgorithm: + description: The preferred Topology Sorting Algorithm + type: string + workloads: + description: Workloads defines the workloads belonging to the group + items: + description: AppGroupWorkload represents the Workloads belonging + to the App Group. + properties: + dependencies: + description: Dependencies of the Workload. + items: + description: DependenciesInfo contains information about one + dependency. + properties: + maxNetworkCost: + description: Max Network Cost between workloads + format: int64 + maximum: 10000 + minimum: 0 + type: integer + minBandwidth: + anyOf: + - type: integer + - type: string + description: MinBandwidth between workloads + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + workload: + description: Workload reference Info. + properties: + apiVersion: + description: ApiVersion defines the versioned schema + of an object. + type: string + kind: + description: 'Kind of the workload, info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' + type: string + name: + description: 'Name represents the workload, info: + http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + namespace: + description: Namespace of the workload + type: string + selector: + description: Selector defines how to find Pods related + to the Workload (key = workload). (e.g., workload=w1) + type: string + required: + - kind + - name + - selector + type: object + required: + - workload + type: object + type: array + workload: + description: Workload reference Info. + properties: + apiVersion: + description: ApiVersion defines the versioned schema of + an object. + type: string + kind: + description: 'Kind of the workload, info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' + type: string + name: + description: 'Name represents the workload, info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + namespace: + description: Namespace of the workload + type: string + selector: + description: Selector defines how to find Pods related to + the Workload (key = workload). (e.g., workload=w1) + type: string + required: + - kind + - name + - selector + type: object + required: + - workload + type: object + type: array + required: + - numMembers + - topologySortingAlgorithm + - workloads + type: object + status: + description: AppGroupStatus defines the observed use. + properties: + runningWorkloads: + description: The number of actively running workloads (e.g., number + of pods). + format: int32 + minimum: 0 + type: integer + scheduleStartTime: + description: ScheduleStartTime of the group + format: date-time + type: string + topologyCalculationTime: + description: TopologyCalculationTime of the group + format: date-time + type: string + topologyOrder: + description: Topology order for TopSort plugin (QueueSort) + items: + description: AppGroupTopologyInfo represents the calculated order + for a given Workload. + properties: + index: + description: Topology index. + format: int32 + type: integer + workload: + description: Workload reference Info. + properties: + apiVersion: + description: ApiVersion defines the versioned schema of + an object. + type: string + kind: + description: 'Kind of the workload, info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' + type: string + name: + description: 'Name represents the workload, info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + namespace: + description: Namespace of the workload + type: string + selector: + description: Selector defines how to find Pods related to + the Workload (key = workload). (e.g., workload=w1) + type: string + required: + - kind + - name + - selector + type: object + type: object + type: array + type: object + type: object + served: true + storage: true \ No newline at end of file diff --git a/kubespray/project/roles/kubernetes-apps/scheduler_plugins/templates/cm-scheduler-plugins.yaml.j2 b/kubespray/project/roles/kubernetes-apps/scheduler_plugins/templates/cm-scheduler-plugins.yaml.j2 new file mode 100644 index 0000000..7e022e8 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/scheduler_plugins/templates/cm-scheduler-plugins.yaml.j2 @@ -0,0 +1,28 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: scheduler-config + namespace: {{ scheduler_plugins_namespace }} +data: + scheduler-config.yaml: | + apiVersion: kubescheduler.config.k8s.io/v1 + kind: KubeSchedulerConfiguration + leaderElection: + leaderElect: {{ scheduler_plugins_scheduler_leader_elect | bool | lower }} + profiles: + # Compose all plugins in one profile + - schedulerName: scheduler-plugins-scheduler + plugins: + multiPoint: + enabled: +{% for enabeld_plugin in scheduler_plugins_enabled_plugins %} + - name: {{ enabeld_plugin }} +{% endfor %} + disabled: +{% for disabled_plugin in scheduler_plugins_disabled_plugins %} + - name: {{ disabled_plugin }} +{% endfor %} +{% if scheduler_plugins_plugin_config is defined and scheduler_plugins_plugin_config | length != 0 %} + pluginConfig: +{{ scheduler_plugins_plugin_config | to_nice_yaml(indent=2, width=256) | indent(6, true) }} +{% endif %} \ No newline at end of file diff --git a/kubespray/project/roles/kubernetes-apps/scheduler_plugins/templates/deploy-scheduler-plugins.yaml.j2 b/kubespray/project/roles/kubernetes-apps/scheduler_plugins/templates/deploy-scheduler-plugins.yaml.j2 new file mode 100644 index 0000000..114698a --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/scheduler_plugins/templates/deploy-scheduler-plugins.yaml.j2 @@ -0,0 +1,74 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: scheduler-plugins-controller + namespace: {{ scheduler_plugins_namespace }} + labels: + app: scheduler-plugins-controller +spec: + replicas: {{ scheduler_plugins_controller_replicas }} + selector: + matchLabels: + app: scheduler-plugins-controller + template: + metadata: + labels: + app: scheduler-plugins-controller + spec: + serviceAccountName: scheduler-plugins-controller + containers: + - name: scheduler-plugins-controller + image: {{ scheduler_plugins_controller_image_repo }}:{{ scheduler_plugins_controller_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + component: scheduler + name: scheduler-plugins-scheduler + namespace: {{ scheduler_plugins_namespace }} +spec: + selector: + matchLabels: + component: scheduler + replicas: {{ scheduler_plugins_scheduler_replicas }} + template: + metadata: + labels: + component: scheduler + spec: + serviceAccountName: scheduler-plugins-scheduler + containers: + - command: + - /bin/kube-scheduler + - --config=/etc/kubernetes/scheduler-config.yaml + image: {{ scheduler_plugins_scheduler_image_repo }}:{{ scheduler_plugins_scheduler_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + livenessProbe: + httpGet: + path: /healthz + port: 10259 + scheme: HTTPS + initialDelaySeconds: 15 + name: scheduler-plugins-scheduler + readinessProbe: + httpGet: + path: /healthz + port: 10259 + scheme: HTTPS + resources: + requests: + cpu: '0.1' + securityContext: + privileged: false + volumeMounts: + - name: scheduler-config + mountPath: /etc/kubernetes + readOnly: true + hostNetwork: false + hostPID: false + volumes: + - name: scheduler-config + configMap: + name: scheduler-config \ No newline at end of file diff --git a/kubespray/project/roles/kubernetes-apps/scheduler_plugins/templates/namespace.yaml.j2 b/kubespray/project/roles/kubernetes-apps/scheduler_plugins/templates/namespace.yaml.j2 new file mode 100644 index 0000000..d54ae66 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/scheduler_plugins/templates/namespace.yaml.j2 @@ -0,0 +1,7 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: {{ scheduler_plugins_namespace }} + labels: + name: {{ scheduler_plugins_namespace }} \ No newline at end of file diff --git a/kubespray/project/roles/kubernetes-apps/scheduler_plugins/templates/networktopology.diktyo.x-k8s.io_networktopologies.yaml.j2 b/kubespray/project/roles/kubernetes-apps/scheduler_plugins/templates/networktopology.diktyo.x-k8s.io_networktopologies.yaml.j2 new file mode 100644 index 0000000..e33157c --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/scheduler_plugins/templates/networktopology.diktyo.x-k8s.io_networktopologies.yaml.j2 @@ -0,0 +1,148 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.kubernetes.io: https://github.com/kubernetes-sigs/scheduler-plugins/pull/432 # edited manually + controller-gen.kubebuilder.io/version: v0.11.1 + creationTimestamp: null + name: networktopologies.networktopology.diktyo.x-k8s.io +spec: + group: networktopology.diktyo.x-k8s.io + names: + kind: NetworkTopology + listKind: NetworkTopologyList + plural: networktopologies + shortNames: + - nt + singular: networktopology + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: NetworkTopology defines network costs in the cluster between + regions and zones + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: NetworkTopologySpec defines the zones and regions of the + cluster. + properties: + configmapName: + description: ConfigmapName to be used for cost calculation + type: string + weights: + description: The manual defined weights of the cluster + items: + description: WeightInfo contains information about all network costs + for a given algorithm. + properties: + name: + description: Algorithm Name for network cost calculation (e.g., + userDefined) + type: string + topologyList: + description: TopologyList owns Costs between origins + items: + description: TopologyInfo contains information about network + costs for a particular Topology Key. + properties: + originList: + description: OriginList for a particular origin. + items: + description: OriginInfo contains information about network + costs for a particular Origin. + properties: + costList: + description: Costs for the particular origin. + items: + description: CostInfo contains information about + networkCosts. + properties: + bandwidthAllocated: + anyOf: + - type: integer + - type: string + description: Bandwidth allocated between origin + and destination. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + bandwidthCapacity: + anyOf: + - type: integer + - type: string + description: Bandwidth capacity between origin + and destination. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + destination: + description: Name of the destination (e.g., + Region Name, Zone Name). + type: string + networkCost: + description: Network Cost between origin and + destination (e.g., Dijkstra shortest path, + etc) + format: int64 + minimum: 0 + type: integer + required: + - destination + - networkCost + type: object + type: array + origin: + description: Name of the origin (e.g., Region Name, + Zone Name). + type: string + required: + - origin + type: object + type: array + topologyKey: + description: Topology key (e.g., "topology.kubernetes.io/region", + "topology.kubernetes.io/zone"). + type: string + required: + - originList + - topologyKey + type: object + type: array + required: + - name + - topologyList + type: object + type: array + required: + - configmapName + - weights + type: object + status: + description: NetworkTopologyStatus defines the observed use. + properties: + nodeCount: + description: The total number of nodes in the cluster + format: int64 + minimum: 0 + type: integer + weightCalculationTime: + description: The calculation time for the weights in the network topology + CRD + format: date-time + type: string + type: object + type: object + served: true + storage: true \ No newline at end of file diff --git a/kubespray/project/roles/kubernetes-apps/scheduler_plugins/templates/rbac-scheduler-plugins.yaml.j2 b/kubespray/project/roles/kubernetes-apps/scheduler_plugins/templates/rbac-scheduler-plugins.yaml.j2 new file mode 100644 index 0000000..aa6f211 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/scheduler_plugins/templates/rbac-scheduler-plugins.yaml.j2 @@ -0,0 +1,140 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: scheduler-plugins-scheduler +rules: +- apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "list", "watch"] +- apiGroups: ["", "events.k8s.io"] + resources: ["events"] + verbs: ["create", "patch", "update"] +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["create"] +- apiGroups: ["coordination.k8s.io"] + resourceNames: ["kube-scheduler"] + resources: ["leases"] + verbs: ["get", "update"] +- apiGroups: [""] + resources: ["endpoints"] + verbs: ["create"] +- apiGroups: [""] + resourceNames: ["kube-scheduler"] + resources: ["endpoints"] + verbs: ["get", "update"] +- apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch", "patch"] +- apiGroups: [""] + resources: ["pods"] + verbs: ["delete", "get", "list", "watch", "update"] +- apiGroups: [""] + resources: ["bindings", "pods/binding"] + verbs: ["create"] +- apiGroups: [""] + resources: ["pods/status"] + verbs: ["patch", "update"] +- apiGroups: [""] + resources: ["replicationcontrollers", "services"] + verbs: ["get", "list", "watch"] +- apiGroups: ["apps", "extensions"] + resources: ["replicasets"] + verbs: ["get", "list", "watch"] +- apiGroups: ["apps"] + resources: ["statefulsets"] + verbs: ["get", "list", "watch"] +- apiGroups: ["policy"] + resources: ["poddisruptionbudgets"] + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: ["persistentvolumeclaims", "persistentvolumes"] + verbs: ["get", "list", "watch", "patch", "update"] +- apiGroups: ["authentication.k8s.io"] + resources: ["tokenreviews"] + verbs: ["create"] +- apiGroups: ["authorization.k8s.io"] + resources: ["subjectaccessreviews"] + verbs: ["create"] +- apiGroups: ["storage.k8s.io"] + resources: ["csinodes", "storageclasses" , "csidrivers" , "csistoragecapacities"] + verbs: ["get", "list", "watch"] +- apiGroups: ["topology.node.k8s.io"] + resources: ["noderesourcetopologies"] + verbs: ["get", "list", "watch"] +# resources need to be updated with the scheduler plugins used +- apiGroups: ["scheduling.x-k8s.io"] + resources: ["podgroups", "elasticquotas", "podgroups/status", "elasticquotas/status"] + verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] +# for network-aware plugins add the following lines (scheduler-plugins v0.27.8) +#- apiGroups: [ "appgroup.diktyo.x-k8s.io" ] +# resources: [ "appgroups" ] +# verbs: [ "get", "list", "watch", "create", "delete", "update", "patch" ] +#- apiGroups: [ "networktopology.diktyo.x-k8s.io" ] +# resources: [ "networktopologies" ] +# verbs: [ "get", "list", "watch", "create", "delete", "update", "patch" ] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: scheduler-plugins-scheduler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: scheduler-plugins-scheduler +subjects: +- kind: ServiceAccount + name: scheduler-plugins-scheduler + namespace: {{ scheduler_plugins_namespace }} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: scheduler-plugins-controller +rules: +- apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: ["events"] + verbs: ["create", "patch", "update"] +- apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch", "patch"] +- apiGroups: ["topology.node.k8s.io"] + resources: ["noderesourcetopologies"] + verbs: ["get", "list", "watch"] +# resources need to be updated with the scheduler plugins used +- apiGroups: ["scheduling.x-k8s.io"] + resources: ["podgroups", "elasticquotas", "podgroups/status", "elasticquotas/status"] + verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: scheduler-plugins-controller +subjects: +- kind: ServiceAccount + name: scheduler-plugins-controller + namespace: {{ scheduler_plugins_namespace }} +roleRef: + kind: ClusterRole + name: scheduler-plugins-controller + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: sched-plugins::extension-apiserver-authentication-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- kind: ServiceAccount + name: scheduler-plugins-scheduler + namespace: {{ scheduler_plugins_namespace }} +- kind: ServiceAccount + name: scheduler-plugins-controller + namespace: {{ scheduler_plugins_namespace }} \ No newline at end of file diff --git a/kubespray/project/roles/kubernetes-apps/scheduler_plugins/templates/sa-scheduler-plugins.yaml.j2 b/kubespray/project/roles/kubernetes-apps/scheduler_plugins/templates/sa-scheduler-plugins.yaml.j2 new file mode 100644 index 0000000..6c25e18 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/scheduler_plugins/templates/sa-scheduler-plugins.yaml.j2 @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: scheduler-plugins-scheduler + namespace: {{ scheduler_plugins_namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: scheduler-plugins-controller + namespace: {{ scheduler_plugins_namespace }} \ No newline at end of file diff --git a/kubespray/project/roles/kubernetes-apps/scheduler_plugins/templates/scheduling.x-k8s.io_elasticquotas.yaml.j2 b/kubespray/project/roles/kubernetes-apps/scheduler_plugins/templates/scheduling.x-k8s.io_elasticquotas.yaml.j2 new file mode 100644 index 0000000..d63f572 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/scheduler_plugins/templates/scheduling.x-k8s.io_elasticquotas.yaml.j2 @@ -0,0 +1,82 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.kubernetes.io: https://github.com/kubernetes-sigs/scheduler-plugins/pull/52 + controller-gen.kubebuilder.io/version: v0.11.1 + creationTimestamp: null + name: elasticquotas.scheduling.x-k8s.io +spec: + group: scheduling.x-k8s.io + names: + kind: ElasticQuota + listKind: ElasticQuotaList + plural: elasticquotas + shortNames: + - eq + - eqs + singular: elasticquota + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: ElasticQuota sets elastic quota restrictions per namespace + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ElasticQuotaSpec defines the Min and Max for Quota. + properties: + max: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Max is the set of desired max limits for each named resource. + The usage of max is based on the resource configurations of successfully + scheduled pods. + type: object + min: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Min is the set of desired guaranteed limits for each + named resource. + type: object + type: object + status: + description: ElasticQuotaStatus defines the observed use. + properties: + used: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Used is the current observed total usage of the resource + in the namespace. + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} \ No newline at end of file diff --git a/kubespray/project/roles/kubernetes-apps/scheduler_plugins/templates/scheduling.x-k8s.io_podgroups.yaml.j2 b/kubespray/project/roles/kubernetes-apps/scheduler_plugins/templates/scheduling.x-k8s.io_podgroups.yaml.j2 new file mode 100644 index 0000000..3767cf9 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/scheduler_plugins/templates/scheduling.x-k8s.io_podgroups.yaml.j2 @@ -0,0 +1,97 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.kubernetes.io: https://github.com/kubernetes-sigs/scheduler-plugins/pull/50 + controller-gen.kubebuilder.io/version: v0.11.1 + creationTimestamp: null + name: podgroups.scheduling.x-k8s.io +spec: + group: scheduling.x-k8s.io + names: + kind: PodGroup + listKind: PodGroupList + plural: podgroups + shortNames: + - pg + - pgs + singular: podgroup + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: PodGroup is a collection of Pod; used for batch workload. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Specification of the desired behavior of the pod group. + properties: + minMember: + description: MinMember defines the minimal number of members/tasks + to run the pod group; if there's not enough resources to start all + tasks, the scheduler will not start anyone. + format: int32 + type: integer + minResources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: MinResources defines the minimal resource of members/tasks + to run the pod group; if there's not enough resources to start all + tasks, the scheduler will not start anyone. + type: object + scheduleTimeoutSeconds: + description: ScheduleTimeoutSeconds defines the maximal time of members/tasks + to wait before run the pod group; + format: int32 + type: integer + type: object + status: + description: Status represents the current information about a pod group. + This data may not be up to date. + properties: + failed: + description: The number of pods which reached phase Failed. + format: int32 + type: integer + occupiedBy: + description: OccupiedBy marks the workload (e.g., deployment, statefulset) + UID that occupy the podgroup. It is empty if not initialized. + type: string + phase: + description: Current phase of PodGroup. + type: string + running: + description: The number of actively running pods. + format: int32 + type: integer + scheduleStartTime: + description: ScheduleStartTime of the group + format: date-time + type: string + succeeded: + description: The number of pods which reached phase Succeeded. + format: int32 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} \ No newline at end of file diff --git a/kubespray/project/roles/kubernetes-apps/scheduler_plugins/templates/topology.node.k8s.io_noderesourcetopologies.yaml.j2 b/kubespray/project/roles/kubernetes-apps/scheduler_plugins/templates/topology.node.k8s.io_noderesourcetopologies.yaml.j2 new file mode 100644 index 0000000..d83ef0b --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/scheduler_plugins/templates/topology.node.k8s.io_noderesourcetopologies.yaml.j2 @@ -0,0 +1,153 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.kubernetes.io: https://github.com/kubernetes/enhancements/pull/1870 + controller-gen.kubebuilder.io/version: v0.11.1 + creationTimestamp: null + name: noderesourcetopologies.topology.node.k8s.io +spec: + group: topology.node.k8s.io + names: + kind: NodeResourceTopology + listKind: NodeResourceTopologyList + plural: noderesourcetopologies + shortNames: + - node-res-topo + singular: noderesourcetopology + scope: Cluster + versions: + - name: v1alpha2 + schema: + openAPIV3Schema: + description: NodeResourceTopology describes node resources and their topology. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + attributes: + description: AttributeList contains an array of AttributeInfo objects. + items: + description: AttributeInfo contains one attribute of a Zone. + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + topologyPolicies: + description: 'DEPRECATED (to be removed in v1beta1): use top level attributes + if needed' + items: + type: string + type: array + zones: + description: ZoneList contains an array of Zone objects. + items: + description: Zone represents a resource topology zone, e.g. socket, + node, die or core. + properties: + attributes: + description: AttributeList contains an array of AttributeInfo objects. + items: + description: AttributeInfo contains one attribute of a Zone. + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + costs: + description: CostList contains an array of CostInfo objects. + items: + description: CostInfo describes the cost (or distance) between + two Zones. + properties: + name: + type: string + value: + format: int64 + type: integer + required: + - name + - value + type: object + type: array + name: + type: string + parent: + type: string + resources: + description: ResourceInfoList contains an array of ResourceInfo + objects. + items: + description: ResourceInfo contains information about one resource + type. + properties: + allocatable: + anyOf: + - type: integer + - type: string + description: Allocatable quantity of the resource, corresponding + to allocatable in node status, i.e. total amount of this + resource available to be used by pods. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + available: + anyOf: + - type: integer + - type: string + description: Available is the amount of this resource currently + available for new (to be scheduled) pods, i.e. Allocatable + minus the resources reserved by currently running pods. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + capacity: + anyOf: + - type: integer + - type: string + description: Capacity of the resource, corresponding to capacity + in node status, i.e. total amount of this resource that + the node has. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + name: + description: Name of the resource. + type: string + required: + - allocatable + - available + - capacity + - name + type: object + type: array + type: + type: string + required: + - name + - type + type: object + type: array + required: + - zones + type: object + served: true + storage: true \ No newline at end of file diff --git a/kubespray/project/roles/kubernetes-apps/snapshots/cinder-csi/defaults/main.yml b/kubespray/project/roles/kubernetes-apps/snapshots/cinder-csi/defaults/main.yml new file mode 100644 index 0000000..1186d98 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/snapshots/cinder-csi/defaults/main.yml @@ -0,0 +1,6 @@ +--- +snapshot_classes: + - name: cinder-csi-snapshot + is_default: false + force_create: true + deletionPolicy: Delete diff --git a/kubespray/project/roles/kubernetes-apps/snapshots/cinder-csi/tasks/main.yml b/kubespray/project/roles/kubernetes-apps/snapshots/cinder-csi/tasks/main.yml new file mode 100644 index 0000000..7e9116f --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/snapshots/cinder-csi/tasks/main.yml @@ -0,0 +1,18 @@ +--- +- name: Kubernetes Snapshots | Copy Cinder CSI Snapshot Class template + template: + src: "cinder-csi-snapshot-class.yml.j2" + dest: "{{ kube_config_dir }}/cinder-csi-snapshot-class.yml" + mode: 0644 + register: manifests + when: + - inventory_hostname == groups['kube_control_plane'][0] + +- name: Kubernetes Snapshots | Add Cinder CSI Snapshot Class + kube: + kubectl: "{{ bin_dir }}/kubectl" + filename: "{{ kube_config_dir }}/cinder-csi-snapshot-class.yml" + state: "latest" + when: + - inventory_hostname == groups['kube_control_plane'][0] + - manifests.changed diff --git a/kubespray/project/roles/kubernetes-apps/snapshots/cinder-csi/templates/cinder-csi-snapshot-class.yml.j2 b/kubespray/project/roles/kubernetes-apps/snapshots/cinder-csi/templates/cinder-csi-snapshot-class.yml.j2 new file mode 100644 index 0000000..86c73cc --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/snapshots/cinder-csi/templates/cinder-csi-snapshot-class.yml.j2 @@ -0,0 +1,13 @@ +{% for class in snapshot_classes %} +--- +kind: VolumeSnapshotClass +apiVersion: snapshot.storage.k8s.io/v1beta1 +metadata: + name: "{{ class.name }}" + annotations: + storageclass.kubernetes.io/is-default-class: "{{ class.is_default | default(false) | ternary("true","false") }}" +driver: cinder.csi.openstack.org +deletionPolicy: "{{ class.deletionPolicy | default("Delete") }}" +parameters: + force-create: "{{ class.force_create }}" +{% endfor %} diff --git a/kubespray/project/roles/kubernetes-apps/snapshots/meta/main.yml b/kubespray/project/roles/kubernetes-apps/snapshots/meta/main.yml new file mode 100644 index 0000000..0eed56c --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/snapshots/meta/main.yml @@ -0,0 +1,14 @@ +--- +dependencies: + - role: kubernetes-apps/snapshots/snapshot-controller + when: + - cinder_csi_enabled or csi_snapshot_controller_enabled + tags: + - snapshot-controller + + - role: kubernetes-apps/snapshots/cinder-csi + when: + - cinder_csi_enabled + tags: + - snapshot + - cinder-csi-driver diff --git a/kubespray/project/roles/kubernetes-apps/snapshots/snapshot-controller/defaults/main.yml b/kubespray/project/roles/kubernetes-apps/snapshots/snapshot-controller/defaults/main.yml new file mode 100644 index 0000000..c72dfb2 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/snapshots/snapshot-controller/defaults/main.yml @@ -0,0 +1,3 @@ +--- +snapshot_controller_replicas: 1 +snapshot_controller_namespace: kube-system diff --git a/kubespray/project/roles/kubernetes-apps/snapshots/snapshot-controller/tasks/main.yml b/kubespray/project/roles/kubernetes-apps/snapshots/snapshot-controller/tasks/main.yml new file mode 100644 index 0000000..e6da292 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/snapshots/snapshot-controller/tasks/main.yml @@ -0,0 +1,39 @@ +--- +- name: Check if snapshot namespace exists + register: snapshot_namespace_exists + kube: + kubectl: "{{ bin_dir }}/kubectl" + name: "{{ snapshot_controller_namespace }}" + resource: "namespace" + state: "exists" + when: inventory_hostname == groups['kube_control_plane'][0] + tags: snapshot-controller + +- name: Snapshot Controller | Generate Manifests + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/{{ item.file }}" + mode: 0644 + with_items: + - {name: snapshot-ns, file: snapshot-ns.yml, apply: not snapshot_namespace_exists} + - {name: rbac-snapshot-controller, file: rbac-snapshot-controller.yml} + - {name: snapshot-controller, file: snapshot-controller.yml} + register: snapshot_controller_manifests + when: + - inventory_hostname == groups['kube_control_plane'][0] + - item.apply | default(True) | bool + tags: snapshot-controller + +- name: Snapshot Controller | Apply Manifests + kube: + kubectl: "{{ bin_dir }}/kubectl" + filename: "{{ kube_config_dir }}/{{ item.item.file }}" + state: "latest" + with_items: + - "{{ snapshot_controller_manifests.results }}" + when: + - inventory_hostname == groups['kube_control_plane'][0] + - not item is skipped + loop_control: + label: "{{ item.item.file }}" + tags: snapshot-controller diff --git a/kubespray/project/roles/kubernetes-apps/snapshots/snapshot-controller/templates/rbac-snapshot-controller.yml.j2 b/kubespray/project/roles/kubernetes-apps/snapshots/snapshot-controller/templates/rbac-snapshot-controller.yml.j2 new file mode 100644 index 0000000..9413376 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/snapshots/snapshot-controller/templates/rbac-snapshot-controller.yml.j2 @@ -0,0 +1,85 @@ +# RBAC file for the snapshot controller. +# +# The snapshot controller implements the control loop for CSI snapshot functionality. +# It should be installed as part of the base Kubernetes distribution in an appropriate +# namespace for components implementing base system functionality. For installing with +# Vanilla Kubernetes, kube-system makes sense for the namespace. + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: snapshot-controller + namespace: {{ snapshot_controller_namespace }} + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + # rename if there are conflicts + name: snapshot-controller-runner +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["update"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: snapshot-controller-role +subjects: + - kind: ServiceAccount + name: snapshot-controller + namespace: {{ snapshot_controller_namespace }} +roleRef: + kind: ClusterRole + # change the name also here if the ClusterRole gets renamed + name: snapshot-controller-runner + apiGroup: rbac.authorization.k8s.io + +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + namespace: {{ snapshot_controller_namespace }} + name: snapshot-controller-leaderelection +rules: +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: snapshot-controller-leaderelection + namespace: {{ snapshot_controller_namespace }} +subjects: + - kind: ServiceAccount + name: snapshot-controller + namespace: {{ snapshot_controller_namespace }} +roleRef: + kind: Role + name: snapshot-controller-leaderelection + apiGroup: rbac.authorization.k8s.io diff --git a/kubespray/project/roles/kubernetes-apps/snapshots/snapshot-controller/templates/snapshot-controller.yml.j2 b/kubespray/project/roles/kubernetes-apps/snapshots/snapshot-controller/templates/snapshot-controller.yml.j2 new file mode 100644 index 0000000..d17ffb3 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/snapshots/snapshot-controller/templates/snapshot-controller.yml.j2 @@ -0,0 +1,40 @@ +# This YAML file shows how to deploy the snapshot controller + +# The snapshot controller implements the control loop for CSI snapshot functionality. +# It should be installed as part of the base Kubernetes distribution in an appropriate +# namespace for components implementing base system functionality. For installing with +# Vanilla Kubernetes, kube-system makes sense for the namespace. + +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: snapshot-controller + namespace: {{ snapshot_controller_namespace }} +spec: + replicas: {{ snapshot_controller_replicas }} + selector: + matchLabels: + app: snapshot-controller + # the snapshot controller won't be marked as ready if the v1 CRDs are unavailable + # in #504 the snapshot-controller will exit after around 7.5 seconds if it + # can't find the v1 CRDs so this value should be greater than that + minReadySeconds: 15 + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + labels: + app: snapshot-controller + spec: + serviceAccount: snapshot-controller + containers: + - name: snapshot-controller + image: {{ snapshot_controller_image_repo }}:{{ snapshot_controller_image_tag }} + args: + - "--v=5" + - "--leader-election=false" + imagePullPolicy: {{ k8s_image_pull_policy }} diff --git a/kubespray/project/roles/kubernetes-apps/snapshots/snapshot-controller/templates/snapshot-ns.yml.j2 b/kubespray/project/roles/kubernetes-apps/snapshots/snapshot-controller/templates/snapshot-ns.yml.j2 new file mode 100644 index 0000000..bb30d60 --- /dev/null +++ b/kubespray/project/roles/kubernetes-apps/snapshots/snapshot-controller/templates/snapshot-ns.yml.j2 @@ -0,0 +1,7 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: {{ snapshot_controller_namespace }} + labels: + name: {{ snapshot_controller_namespace }} diff --git a/kubespray/project/roles/kubernetes/client/defaults/main.yml b/kubespray/project/roles/kubernetes/client/defaults/main.yml new file mode 100644 index 0000000..83506a4 --- /dev/null +++ b/kubespray/project/roles/kubernetes/client/defaults/main.yml @@ -0,0 +1,8 @@ +--- +kubeconfig_localhost: false +kubeconfig_localhost_ansible_host: false +kubectl_localhost: false +artifacts_dir: "{{ inventory_dir }}/artifacts" + +kube_config_dir: "/etc/kubernetes" +kube_apiserver_port: "6443" diff --git a/kubespray/project/roles/kubernetes/client/tasks/main.yml b/kubespray/project/roles/kubernetes/client/tasks/main.yml new file mode 100644 index 0000000..e619761 --- /dev/null +++ b/kubespray/project/roles/kubernetes/client/tasks/main.yml @@ -0,0 +1,114 @@ +--- +- name: Set external kube-apiserver endpoint + set_fact: + # noqa: jinja[spacing] + external_apiserver_address: >- + {%- if loadbalancer_apiserver is defined and loadbalancer_apiserver.address is defined -%} + {{ loadbalancer_apiserver.address }} + {%- elif kubeconfig_localhost_ansible_host is defined and kubeconfig_localhost_ansible_host -%} + {{ hostvars[groups['kube_control_plane'][0]].ansible_host }} + {%- else -%} + {{ kube_apiserver_access_address }} + {%- endif -%} + # noqa: jinja[spacing] + external_apiserver_port: >- + {%- if loadbalancer_apiserver is defined and loadbalancer_apiserver.address is defined and loadbalancer_apiserver.port is defined -%} + {{ loadbalancer_apiserver.port | default(kube_apiserver_port) }} + {%- else -%} + {{ kube_apiserver_port }} + {%- endif -%} + tags: + - facts + +- name: Create kube config dir for current/ansible become user + file: + path: "{{ ansible_env.HOME | default('/root') }}/.kube" + mode: "0700" + state: directory + +- name: Copy admin kubeconfig to current/ansible become user home + copy: + src: "{{ kube_config_dir }}/admin.conf" + dest: "{{ ansible_env.HOME | default('/root') }}/.kube/config" + remote_src: yes + mode: "0600" + backup: yes + +- name: Create kube artifacts dir + file: + path: "{{ artifacts_dir }}" + mode: "0750" + state: directory + delegate_to: localhost + connection: local + become: no + run_once: yes + when: kubeconfig_localhost + +- name: Wait for k8s apiserver + wait_for: + host: "{{ kube_apiserver_access_address }}" + port: "{{ kube_apiserver_port }}" + timeout: 180 + +- name: Get admin kubeconfig from remote host + slurp: + src: "{{ kube_config_dir }}/admin.conf" + run_once: yes + register: raw_admin_kubeconfig + when: kubeconfig_localhost + +- name: Convert kubeconfig to YAML + set_fact: + admin_kubeconfig: "{{ raw_admin_kubeconfig.content | b64decode | from_yaml }}" + when: kubeconfig_localhost + +- name: Override username in kubeconfig + set_fact: + final_admin_kubeconfig: "{{ admin_kubeconfig | combine(override_cluster_name, recursive=true) | combine(override_context, recursive=true) | combine(override_user, recursive=true) }}" + vars: + cluster_infos: "{{ admin_kubeconfig['clusters'][0]['cluster'] }}" + user_certs: "{{ admin_kubeconfig['users'][0]['user'] }}" + username: "kubernetes-admin-{{ cluster_name }}" + context: "kubernetes-admin-{{ cluster_name }}@{{ cluster_name }}" + override_cluster_name: "{{ {'clusters': [{'cluster': (cluster_infos | combine({'server': 'https://' + external_apiserver_address + ':' + (external_apiserver_port | string)})), 'name': cluster_name}]} }}" + override_context: "{{ {'contexts': [{'context': {'user': username, 'cluster': cluster_name}, 'name': context}], 'current-context': context} }}" + override_user: "{{ {'users': [{'name': username, 'user': user_certs}]} }}" + when: kubeconfig_localhost + +- name: Write admin kubeconfig on ansible host + copy: + content: "{{ final_admin_kubeconfig | to_nice_yaml(indent=2) }}" + dest: "{{ artifacts_dir }}/admin.conf" + mode: 0600 + delegate_to: localhost + connection: local + become: no + run_once: yes + when: kubeconfig_localhost + +- name: Copy kubectl binary to ansible host + fetch: + src: "{{ bin_dir }}/kubectl" + dest: "{{ artifacts_dir }}/kubectl" + flat: yes + validate_checksum: no + register: copy_binary_result + until: copy_binary_result is not failed + retries: 20 + become: no + run_once: yes + when: kubectl_localhost + +- name: Create helper script kubectl.sh on ansible host + copy: + content: | + #!/bin/bash + ${BASH_SOURCE%/*}/kubectl --kubeconfig=${BASH_SOURCE%/*}/admin.conf "$@" + dest: "{{ artifacts_dir }}/kubectl.sh" + mode: 0755 + become: no + run_once: yes + delegate_to: localhost + connection: local + when: kubectl_localhost and kubeconfig_localhost diff --git a/kubespray/project/roles/kubernetes/control-plane/defaults/main/etcd.yml b/kubespray/project/roles/kubernetes/control-plane/defaults/main/etcd.yml new file mode 100644 index 0000000..344ce9b --- /dev/null +++ b/kubespray/project/roles/kubernetes/control-plane/defaults/main/etcd.yml @@ -0,0 +1,31 @@ +--- +# Set etcd user/group +etcd_owner: etcd + +# Note: This does not set up DNS entries. It simply adds the following DNS +# entries to the certificate +etcd_cert_alt_names: + - "etcd.kube-system.svc.{{ dns_domain }}" + - "etcd.kube-system.svc" + - "etcd.kube-system" + - "etcd" +etcd_cert_alt_ips: [] + +etcd_heartbeat_interval: "250" +etcd_election_timeout: "5000" + +# etcd_snapshot_count: "10000" + +etcd_metrics: "basic" + +## A dictionary of extra environment variables to add to etcd.env, formatted like: +## etcd_extra_vars: +## var1: "value1" +## var2: "value2" +## Note this is different from the etcd role with ETCD_ prfexi, caps, and underscores +etcd_extra_vars: {} + +# etcd_quota_backend_bytes: "2147483648" +# etcd_max_request_bytes: "1572864" + +etcd_compaction_retention: "8" diff --git a/kubespray/project/roles/kubernetes/control-plane/defaults/main/kube-proxy.yml b/kubespray/project/roles/kubernetes/control-plane/defaults/main/kube-proxy.yml new file mode 100644 index 0000000..24ebc6c --- /dev/null +++ b/kubespray/project/roles/kubernetes/control-plane/defaults/main/kube-proxy.yml @@ -0,0 +1,114 @@ +--- +# bind address for kube-proxy +kube_proxy_bind_address: '0.0.0.0' + +# acceptContentTypes defines the Accept header sent by clients when connecting to a server, overriding the +# default value of 'application/json'. This field will control all connections to the server used by a particular +# client. +kube_proxy_client_accept_content_types: '' + +# burst allows extra queries to accumulate when a client is exceeding its rate. +kube_proxy_client_burst: 10 + +# contentType is the content type used when sending data to the server from this client. +kube_proxy_client_content_type: application/vnd.kubernetes.protobuf + +# kubeconfig is the path to a KubeConfig file. +# Leave as empty string to generate from other fields +kube_proxy_client_kubeconfig: '' + +# qps controls the number of queries per second allowed for this connection. +kube_proxy_client_qps: 5 + +# How often configuration from the apiserver is refreshed. Must be greater than 0. +kube_proxy_config_sync_period: 15m0s + +### Conntrack +# maxPerCore is the maximum number of NAT connections to track +# per CPU core (0 to leave the limit as-is and ignore min). +kube_proxy_conntrack_max_per_core: 32768 + +# min is the minimum value of connect-tracking records to allocate, +# regardless of conntrackMaxPerCore (set maxPerCore=0 to leave the limit as-is). +kube_proxy_conntrack_min: 131072 + +# tcpCloseWaitTimeout is how long an idle conntrack entry +# in CLOSE_WAIT state will remain in the conntrack +# table. (e.g. '60s'). Must be greater than 0 to set. +kube_proxy_conntrack_tcp_close_wait_timeout: 1h0m0s + +# tcpEstablishedTimeout is how long an idle TCP connection will be kept open +# (e.g. '2s'). Must be greater than 0 to set. +kube_proxy_conntrack_tcp_established_timeout: 24h0m0s + +# Enables profiling via web interface on /debug/pprof handler. +# Profiling handlers will be handled by metrics server. +kube_proxy_enable_profiling: false + +# bind address for kube-proxy health check +kube_proxy_healthz_bind_address: 0.0.0.0:10256 + +# If using the pure iptables proxy, SNAT everything. Note that it breaks any +# policy engine. +kube_proxy_masquerade_all: false + +# If using the pure iptables proxy, the bit of the fwmark space to mark packets requiring SNAT with. +# Must be within the range [0, 31]. +kube_proxy_masquerade_bit: 14 + +# The minimum interval of how often the iptables or ipvs rules can be refreshed as +# endpoints and services change (e.g. '5s', '1m', '2h22m'). +kube_proxy_min_sync_period: 0s + +# The maximum interval of how often iptables or ipvs rules are refreshed (e.g. '5s', '1m', '2h22m'). +# Must be greater than 0. +kube_proxy_sync_period: 30s + +# A comma-separated list of CIDR's which the ipvs proxier should not touch when cleaning up IPVS rules. +kube_proxy_exclude_cidrs: [] + +# The ipvs scheduler type when proxy mode is ipvs +# rr: round-robin +# lc: least connection +# dh: destination hashing +# sh: source hashing +# sed: shortest expected delay +# nq: never queue +kube_proxy_scheduler: rr + +# configure arp_ignore and arp_announce to avoid answering ARP queries from kube-ipvs0 interface +# must be set to true for MetalLB, kube-vip(ARP enabled) to work +kube_proxy_strict_arp: false + +# kube_proxy_tcp_timeout is the timeout value used for idle IPVS TCP sessions. +# The default value is 0, which preserves the current timeout value on the system. +kube_proxy_tcp_timeout: 0s + +# kube_proxy_tcp_fin_timeout is the timeout value used for IPVS TCP sessions after receiving a FIN. +# The default value is 0, which preserves the current timeout value on the system. +kube_proxy_tcp_fin_timeout: 0s + +# kube_proxy_udp_timeout is the timeout value used for IPVS UDP packets. +# The default value is 0, which preserves the current timeout value on the system. +kube_proxy_udp_timeout: 0s + +# The IP address and port for the metrics server to serve on +# (set to 0.0.0.0 for all IPv4 interfaces and `::` for all IPv6 interfaces) +kube_proxy_metrics_bind_address: 127.0.0.1:10249 + +# A string slice of values which specify the addresses to use for NodePorts. +# Values may be valid IP blocks (e.g. 1.2.3.0/24, 1.2.3.4/32). +# The default empty string slice ([]) means to use all local addresses. +kube_proxy_nodeport_addresses: >- + {%- if kube_proxy_nodeport_addresses_cidr is defined -%} + [{{ kube_proxy_nodeport_addresses_cidr }}] + {%- else -%} + [] + {%- endif -%} + +# oom-score-adj value for kube-proxy process. Values must be within the range [-1000, 1000] +kube_proxy_oom_score_adj: -999 + +# portRange is the range of host ports (beginPort-endPort, inclusive) that may be consumed +# in order to proxy service traffic. If unspecified, 0, or (0-0) then ports will be randomly chosen. +kube_proxy_port_range: '' diff --git a/kubespray/project/roles/kubernetes/control-plane/defaults/main/kube-scheduler.yml b/kubespray/project/roles/kubernetes/control-plane/defaults/main/kube-scheduler.yml new file mode 100644 index 0000000..e61bcb7 --- /dev/null +++ b/kubespray/project/roles/kubernetes/control-plane/defaults/main/kube-scheduler.yml @@ -0,0 +1,33 @@ +--- +# Extra args passed by kubeadm +kube_kubeadm_scheduler_extra_args: {} + +# Associated interface must be reachable by the rest of the cluster, and by +# CLI/web clients. +kube_scheduler_bind_address: 0.0.0.0 + +# ClientConnection options (e.g. Burst, QPS) except from kubeconfig. +kube_scheduler_client_conn_extra_opts: {} + +# Additional KubeSchedulerConfiguration settings (e.g. metricsBindAddress). +kube_scheduler_config_extra_opts: {} + +# List of scheduler extenders (dicts), each holding the values of how to +# communicate with the extender. +kube_scheduler_extenders: [] + +# Leader Election options (e.g. ResourceName, RetryPerion) except from +# LeaseDuration and Renew deadline which are defined in following vars. +kube_scheduler_leader_elect_extra_opts: {} + +# Leader election lease duration +kube_scheduler_leader_elect_lease_duration: 15s + +# Leader election lease timeout +kube_scheduler_leader_elect_renew_deadline: 10s + +# Lisf of scheduling profiles (ditcs) supported by kube-scheduler +kube_scheduler_profiles: [] + +# Extra volume mounts +scheduler_extra_volumes: {} diff --git a/kubespray/project/roles/kubernetes/control-plane/defaults/main/main.yml b/kubespray/project/roles/kubernetes/control-plane/defaults/main/main.yml new file mode 100644 index 0000000..fd70477 --- /dev/null +++ b/kubespray/project/roles/kubernetes/control-plane/defaults/main/main.yml @@ -0,0 +1,242 @@ +--- +# disable upgrade cluster +upgrade_cluster_setup: false + +# By default the external API listens on all interfaces, this can be changed to +# listen on a specific address/interface. +# NOTE: If you specific address/interface and use loadbalancer_apiserver_localhost +# loadbalancer_apiserver_localhost (nginx/haproxy) will deploy on masters on 127.0.0.1:{{ loadbalancer_apiserver_port | default(kube_apiserver_port) }} too. +kube_apiserver_bind_address: 0.0.0.0 + +# A port range to reserve for services with NodePort visibility. +# Inclusive at both ends of the range. +kube_apiserver_node_port_range: "30000-32767" + +# ETCD backend for k8s data +kube_apiserver_storage_backend: etcd3 + +# The interval of compaction requests. If 0, the compaction request from apiserver is disabled. +kube_apiserver_etcd_compaction_interval: "5m0s" + +# CIS 1.2.26 +# Validate that the service account token +# in the request is actually present in etcd. +kube_apiserver_service_account_lookup: true + +kube_etcd_cacert_file: ca.pem +kube_etcd_cert_file: node-{{ inventory_hostname }}.pem +kube_etcd_key_file: node-{{ inventory_hostname }}-key.pem + +# Associated interfaces must be reachable by the rest of the cluster, and by +# CLI/web clients. +kube_controller_manager_bind_address: 0.0.0.0 + +# Leader election lease durations and timeouts for controller-manager +kube_controller_manager_leader_elect_lease_duration: 15s +kube_controller_manager_leader_elect_renew_deadline: 10s + +# discovery_timeout modifies the discovery timeout +discovery_timeout: 5m0s + +# Instruct first master to refresh kubeadm token +kubeadm_refresh_token: true + +# Scale down coredns replicas to 0 if not using coredns dns_mode +kubeadm_scale_down_coredns_enabled: true + +# audit support +kubernetes_audit: false +# path to audit log file +audit_log_path: /var/log/audit/kube-apiserver-audit.log +# num days +audit_log_maxage: 30 +# the num of audit logs to retain +audit_log_maxbackups: 10 +# the max size in MB to retain +audit_log_maxsize: 100 +# policy file +audit_policy_file: "{{ kube_config_dir }}/audit-policy/apiserver-audit-policy.yaml" +# custom audit policy rules (to replace the default ones) +# audit_policy_custom_rules: | +# - level: None +# users: [] +# verbs: [] +# resources: [] + +# audit log hostpath +audit_log_name: audit-logs +audit_log_hostpath: /var/log/kubernetes/audit +audit_log_mountpath: "{{ audit_log_path | dirname }}" + +# audit policy hostpath +audit_policy_name: audit-policy +audit_policy_hostpath: "{{ audit_policy_file | dirname }}" +audit_policy_mountpath: "{{ audit_policy_hostpath }}" + +# audit webhook support +kubernetes_audit_webhook: false + +# path to audit webhook config file +audit_webhook_config_file: "{{ kube_config_dir }}/audit-policy/apiserver-audit-webhook-config.yaml" +audit_webhook_server_url: "https://audit.app" +audit_webhook_server_extra_args: {} +audit_webhook_mode: batch +audit_webhook_batch_max_size: 100 +audit_webhook_batch_max_wait: 1s + +kube_controller_node_monitor_grace_period: 40s +kube_controller_node_monitor_period: 5s +kube_controller_terminated_pod_gc_threshold: 12500 +kube_apiserver_request_timeout: "1m0s" +kube_apiserver_pod_eviction_not_ready_timeout_seconds: "300" +kube_apiserver_pod_eviction_unreachable_timeout_seconds: "300" + +# 1.10+ admission plugins +kube_apiserver_enable_admission_plugins: [] + +# enable admission plugins configuration +kube_apiserver_admission_control_config_file: false + +# data structure to configure EventRateLimit admission plugin +# this should have the following structure: +# kube_apiserver_admission_event_rate_limits: +# : +# type: +# qps: +# burst: +# cache_size: +kube_apiserver_admission_event_rate_limits: {} + +kube_pod_security_use_default: false +kube_pod_security_default_enforce: baseline +kube_pod_security_default_enforce_version: "{{ kube_major_version }}" +kube_pod_security_default_audit: restricted +kube_pod_security_default_audit_version: "{{ kube_major_version }}" +kube_pod_security_default_warn: restricted +kube_pod_security_default_warn_version: "{{ kube_major_version }}" +kube_pod_security_exemptions_usernames: [] +kube_pod_security_exemptions_runtime_class_names: [] +kube_pod_security_exemptions_namespaces: + - kube-system + +# 1.10+ list of disabled admission plugins +kube_apiserver_disable_admission_plugins: [] + +# extra runtime config +kube_api_runtime_config: [] + +## Enable/Disable Kube API Server Authentication Methods +kube_token_auth: false +kube_oidc_auth: false + +## Variables for webhook token auth https://kubernetes.io/docs/reference/access-authn-authz/authentication/#webhook-token-authentication +kube_webhook_token_auth: false +kube_webhook_token_auth_url_skip_tls_verify: false +# kube_webhook_token_auth_url: https://... +## base64-encoded string of the webhook's CA certificate +# kube_webhook_token_auth_ca_data: "LS0t..." + +## Variables for webhook token authz https://kubernetes.io/docs/reference/access-authn-authz/webhook/ +# kube_webhook_authorization_url: https://... +kube_webhook_authorization: false +kube_webhook_authorization_url_skip_tls_verify: false + +# Default podnodeselector +kube_apiserver_admission_plugins_podnodeselector_default_node_selector: "" + +## Variables for OpenID Connect Configuration https://kubernetes.io/docs/admin/authentication/ +## To use OpenID you have to deploy additional an OpenID Provider (e.g Dex, Keycloak, ...) + +# kube_oidc_url: https:// ... +# kube_oidc_client_id: kubernetes +## Optional settings for OIDC +# kube_oidc_username_claim: sub +# kube_oidc_username_prefix: 'oidc:' +# kube_oidc_groups_claim: groups +# kube_oidc_groups_prefix: 'oidc:' +# Copy oidc CA file to the following path if needed +# kube_oidc_ca_file: {{ kube_cert_dir }}/ca.pem +# Optionally include a base64-encoded oidc CA cert +# kube_oidc_ca_cert: c3RhY2thYnVzZS5jb20... + +# List of the preferred NodeAddressTypes to use for kubelet connections. +kubelet_preferred_address_types: 'InternalDNS,InternalIP,Hostname,ExternalDNS,ExternalIP' + +## Extra args for k8s components passing by kubeadm +kube_kubeadm_apiserver_extra_args: {} +kube_kubeadm_controller_extra_args: {} + +## Extra control plane host volume mounts +## Example: +# apiserver_extra_volumes: +# - name: name +# hostPath: /host/path +# mountPath: /mount/path +# readOnly: true +apiserver_extra_volumes: {} +controller_manager_extra_volumes: {} + +## Encrypting Secret Data at Rest +kube_encrypt_secret_data: false +kube_encrypt_token: "{{ lookup('password', credentials_dir + '/kube_encrypt_token.creds length=32 chars=ascii_letters,digits') }}" +# Must be either: aescbc, secretbox or aesgcm +kube_encryption_algorithm: "secretbox" +# Which kubernetes resources to encrypt +kube_encryption_resources: [secrets] + +# If non-empty, will use this string as identification instead of the actual hostname +kube_override_hostname: >- + {%- if cloud_provider is defined and cloud_provider in ['aws'] -%} + {%- else -%} + {{ inventory_hostname }} + {%- endif -%} + +secrets_encryption_query: "resources[*].providers[0].{{ kube_encryption_algorithm }}.keys[0].secret" + +## Support tls min version, Possible values: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13. +# tls_min_version: "" + +## Support tls cipher suites. +# tls_cipher_suites: +# - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA +# - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256 +# - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 +# - TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA +# - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 +# - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256 +# - TLS_ECDHE_ECDSA_WITH_RC4_128_SHA +# - TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA +# - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA +# - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 +# - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 +# - TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA +# - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 +# - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 +# - TLS_ECDHE_RSA_WITH_RC4_128_SHA +# - TLS_RSA_WITH_3DES_EDE_CBC_SHA +# - TLS_RSA_WITH_AES_128_CBC_SHA +# - TLS_RSA_WITH_AES_128_CBC_SHA256 +# - TLS_RSA_WITH_AES_128_GCM_SHA256 +# - TLS_RSA_WITH_AES_256_CBC_SHA +# - TLS_RSA_WITH_AES_256_GCM_SHA384 +# - TLS_RSA_WITH_RC4_128_SHA + +## Amount of time to retain events. (default 1h0m0s) +event_ttl_duration: "1h0m0s" + +## Automatically renew K8S control plane certificates on first Monday of each month +auto_renew_certificates: false +# First Monday of each month +auto_renew_certificates_systemd_calendar: "Mon *-*-1,2,3,4,5,6,7 03:00:00" +# kubeadm renews all the certificates during control plane upgrade. +# If we have requirement like without renewing certs upgrade the cluster, +# we can opt out from the default behavior by setting kubeadm_upgrade_auto_cert_renewal to false +kubeadm_upgrade_auto_cert_renewal: true + +# Bash alias of kubectl to interact with Kubernetes cluster much easier +# kubectl_alias: k + +## Enable distributed tracing for kube-apiserver +kube_apiserver_tracing: false +kube_apiserver_tracing_endpoint: 0.0.0.0:4317 +kube_apiserver_tracing_sampling_rate_per_million: 100 diff --git a/kubespray/project/roles/kubernetes/control-plane/handlers/main.yml b/kubespray/project/roles/kubernetes/control-plane/handlers/main.yml new file mode 100644 index 0000000..1ee64f2 --- /dev/null +++ b/kubespray/project/roles/kubernetes/control-plane/handlers/main.yml @@ -0,0 +1,117 @@ +--- +- name: Master | reload systemd + systemd: + daemon_reload: true + listen: Master | restart kubelet + +- name: Master | reload kubelet + service: + name: kubelet + state: restarted + listen: Master | restart kubelet + +- name: Master | Remove apiserver container docker + shell: "set -o pipefail && docker ps -af name=k8s_kube-apiserver* -q | xargs --no-run-if-empty docker rm -f" + args: + executable: /bin/bash + register: remove_apiserver_container + retries: 10 + until: remove_apiserver_container.rc == 0 + delay: 1 + when: container_manager == "docker" + listen: Master | Restart apiserver + +- name: Master | Remove apiserver container containerd/crio + shell: "set -o pipefail && {{ bin_dir }}/crictl pods --name kube-apiserver* -q | xargs -I% --no-run-if-empty bash -c '{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %'" + args: + executable: /bin/bash + register: remove_apiserver_container + retries: 10 + until: remove_apiserver_container.rc == 0 + delay: 1 + when: container_manager in ['containerd', 'crio'] + listen: Master | Restart apiserver + +- name: Master | Remove scheduler container docker + shell: "set -o pipefail && {{ docker_bin_dir }}/docker ps -af name=k8s_kube-scheduler* -q | xargs --no-run-if-empty {{ docker_bin_dir }}/docker rm -f" + args: + executable: /bin/bash + register: remove_scheduler_container + retries: 10 + until: remove_scheduler_container.rc == 0 + delay: 1 + when: container_manager == "docker" + listen: Master | Restart kube-scheduler + +- name: Master | Remove scheduler container containerd/crio + shell: "set -o pipefail && {{ bin_dir }}/crictl pods --name kube-scheduler* -q | xargs -I% --no-run-if-empty bash -c '{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %'" + args: + executable: /bin/bash + register: remove_scheduler_container + retries: 10 + until: remove_scheduler_container.rc == 0 + delay: 1 + when: container_manager in ['containerd', 'crio'] + listen: Master | Restart kube-scheduler + +- name: Master | Remove controller manager container docker + shell: "set -o pipefail && {{ docker_bin_dir }}/docker ps -af name=k8s_kube-controller-manager* -q | xargs --no-run-if-empty {{ docker_bin_dir }}/docker rm -f" + args: + executable: /bin/bash + register: remove_cm_container + retries: 10 + until: remove_cm_container.rc == 0 + delay: 1 + when: container_manager == "docker" + listen: Master | Restart kube-controller-manager + +- name: Master | Remove controller manager container containerd/crio + shell: "set -o pipefail && {{ bin_dir }}/crictl pods --name kube-controller-manager* -q | xargs -I% --no-run-if-empty bash -c '{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %'" + args: + executable: /bin/bash + register: remove_cm_container + retries: 10 + until: remove_cm_container.rc == 0 + delay: 1 + when: container_manager in ['containerd', 'crio'] + listen: Master | Restart kube-controller-manager + +- name: Master | wait for kube-scheduler + vars: + endpoint: "{{ kube_scheduler_bind_address if kube_scheduler_bind_address != '0.0.0.0' else 'localhost' }}" + uri: + url: https://{{ endpoint }}:10259/healthz + validate_certs: no + register: scheduler_result + until: scheduler_result.status == 200 + retries: 60 + delay: 1 + listen: + - Master | restart kubelet + - Master | Restart kube-scheduler + +- name: Master | wait for kube-controller-manager + vars: + endpoint: "{{ kube_controller_manager_bind_address if kube_controller_manager_bind_address != '0.0.0.0' else 'localhost' }}" + uri: + url: https://{{ endpoint }}:10257/healthz + validate_certs: no + register: controller_manager_result + until: controller_manager_result.status == 200 + retries: 60 + delay: 1 + listen: + - Master | restart kubelet + - Master | Restart kube-controller-manager + +- name: Master | wait for the apiserver to be running + uri: + url: "{{ kube_apiserver_endpoint }}/healthz" + validate_certs: no + register: result + until: result.status == 200 + retries: 60 + delay: 1 + listen: + - Master | restart kubelet + - Master | Restart apiserver diff --git a/kubespray/project/roles/kubernetes/control-plane/meta/main.yml b/kubespray/project/roles/kubernetes/control-plane/meta/main.yml new file mode 100644 index 0000000..7d793f9 --- /dev/null +++ b/kubespray/project/roles/kubernetes/control-plane/meta/main.yml @@ -0,0 +1,12 @@ +--- +dependencies: + - role: kubernetes/tokens + when: kube_token_auth + tags: + - k8s-secrets + - role: adduser + user: "{{ addusers.etcd }}" + when: + - etcd_deployment_type == "kubeadm" + - not (ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk", "ClearLinux"] or is_fedora_coreos) + - role: network_plugin/calico_defaults diff --git a/kubespray/project/roles/kubernetes/control-plane/tasks/define-first-kube-control.yml b/kubespray/project/roles/kubernetes/control-plane/tasks/define-first-kube-control.yml new file mode 100644 index 0000000..36996da --- /dev/null +++ b/kubespray/project/roles/kubernetes/control-plane/tasks/define-first-kube-control.yml @@ -0,0 +1,19 @@ +--- + +- name: Check which kube-control nodes are already members of the cluster + command: "{{ bin_dir }}/kubectl get nodes --selector=node-role.kubernetes.io/control-plane -o json" + register: kube_control_planes_raw + ignore_errors: yes + changed_when: false + +- name: Set fact joined_control_planes + set_fact: + joined_control_planes: "{{ ((kube_control_planes_raw.stdout | from_json)['items']) | default([]) | map(attribute='metadata') | map(attribute='name') | list }}" + delegate_to: item + loop: "{{ groups['kube_control_plane'] }}" + when: kube_control_planes_raw is succeeded + run_once: yes + +- name: Set fact first_kube_control_plane + set_fact: + first_kube_control_plane: "{{ joined_control_planes | default([]) | first | default(groups['kube_control_plane'] | first) }}" diff --git a/kubespray/project/roles/kubernetes/control-plane/tasks/encrypt-at-rest.yml b/kubespray/project/roles/kubernetes/control-plane/tasks/encrypt-at-rest.yml new file mode 100644 index 0000000..209e4c7 --- /dev/null +++ b/kubespray/project/roles/kubernetes/control-plane/tasks/encrypt-at-rest.yml @@ -0,0 +1,40 @@ +--- +- name: Check if secret for encrypting data at rest already exist + stat: + path: "{{ kube_cert_dir }}/secrets_encryption.yaml" + get_attributes: no + get_checksum: no + get_mime: no + register: secrets_encryption_file + +- name: Slurp secrets_encryption file if it exists + slurp: + src: "{{ kube_cert_dir }}/secrets_encryption.yaml" + register: secret_file_encoded + when: secrets_encryption_file.stat.exists + +- name: Base 64 Decode slurped secrets_encryption.yaml file + set_fact: + secret_file_decoded: "{{ secret_file_encoded['content'] | b64decode | from_yaml }}" + when: secrets_encryption_file.stat.exists + +- name: Extract secret value from secrets_encryption.yaml + set_fact: + kube_encrypt_token_extracted: "{{ secret_file_decoded | json_query(secrets_encryption_query) | first | b64decode }}" + when: secrets_encryption_file.stat.exists + +- name: Set kube_encrypt_token across master nodes + set_fact: + kube_encrypt_token: "{{ kube_encrypt_token_extracted }}" + delegate_to: "{{ item }}" + delegate_facts: true + with_inventory_hostnames: kube_control_plane + when: kube_encrypt_token_extracted is defined + +- name: Write secrets for encrypting secret data at rest + template: + src: secrets_encryption.yaml.j2 + dest: "{{ kube_cert_dir }}/secrets_encryption.yaml" + owner: root + group: "{{ kube_cert_group }}" + mode: 0640 diff --git a/kubespray/project/roles/kubernetes/control-plane/tasks/kubeadm-backup.yml b/kubespray/project/roles/kubernetes/control-plane/tasks/kubeadm-backup.yml new file mode 100644 index 0000000..36bb627 --- /dev/null +++ b/kubespray/project/roles/kubernetes/control-plane/tasks/kubeadm-backup.yml @@ -0,0 +1,28 @@ +--- +- name: Backup old certs and keys + copy: + src: "{{ kube_cert_dir }}/{{ item }}" + dest: "{{ kube_cert_dir }}/{{ item }}.old" + mode: preserve + remote_src: yes + with_items: + - apiserver.crt + - apiserver.key + - apiserver-kubelet-client.crt + - apiserver-kubelet-client.key + - front-proxy-client.crt + - front-proxy-client.key + ignore_errors: true # noqa ignore-errors + +- name: Backup old confs + copy: + src: "{{ kube_config_dir }}/{{ item }}" + dest: "{{ kube_config_dir }}/{{ item }}.old" + mode: preserve + remote_src: yes + with_items: + - admin.conf + - controller-manager.conf + - kubelet.conf + - scheduler.conf + ignore_errors: true # noqa ignore-errors diff --git a/kubespray/project/roles/kubernetes/control-plane/tasks/kubeadm-etcd.yml b/kubespray/project/roles/kubernetes/control-plane/tasks/kubeadm-etcd.yml new file mode 100644 index 0000000..9de55c5 --- /dev/null +++ b/kubespray/project/roles/kubernetes/control-plane/tasks/kubeadm-etcd.yml @@ -0,0 +1,29 @@ +--- +- name: Calculate etcd cert serial + command: "openssl x509 -in {{ kube_cert_dir }}/apiserver-etcd-client.crt -noout -serial" + register: "etcd_client_cert_serial_result" + changed_when: false + tags: + - network + +- name: Set etcd_client_cert_serial + set_fact: + etcd_client_cert_serial: "{{ etcd_client_cert_serial_result.stdout.split('=')[1] }}" + tags: + - network + +- name: Ensure etcdctl and etcdutl script is installed + import_role: + name: etcdctl_etcdutl + when: etcd_deployment_type == "kubeadm" + tags: + - etcdctl + - etcdutl + +- name: Set ownership for etcd data directory + file: + path: "{{ etcd_data_dir }}" + owner: "{{ etcd_owner }}" + group: "{{ etcd_owner }}" + mode: 0700 + when: etcd_deployment_type == "kubeadm" diff --git a/kubespray/project/roles/kubernetes/control-plane/tasks/kubeadm-fix-apiserver.yml b/kubespray/project/roles/kubernetes/control-plane/tasks/kubeadm-fix-apiserver.yml new file mode 100644 index 0000000..5376aba --- /dev/null +++ b/kubespray/project/roles/kubernetes/control-plane/tasks/kubeadm-fix-apiserver.yml @@ -0,0 +1,17 @@ +--- + +- name: Update server field in component kubeconfigs + lineinfile: + dest: "{{ kube_config_dir }}/{{ item }}" + regexp: '^ server: https' + line: ' server: {{ kube_apiserver_endpoint }}' + backup: yes + with_items: + - admin.conf + - controller-manager.conf + - kubelet.conf + - scheduler.conf + notify: + - "Master | Restart kube-controller-manager" + - "Master | Restart kube-scheduler" + - "Master | reload kubelet" diff --git a/kubespray/project/roles/kubernetes/control-plane/tasks/kubeadm-secondary.yml b/kubespray/project/roles/kubernetes/control-plane/tasks/kubeadm-secondary.yml new file mode 100644 index 0000000..f3fd207 --- /dev/null +++ b/kubespray/project/roles/kubernetes/control-plane/tasks/kubeadm-secondary.yml @@ -0,0 +1,80 @@ +--- +- name: Set kubeadm_discovery_address + set_fact: + # noqa: jinja[spacing] + kubeadm_discovery_address: >- + {%- if "127.0.0.1" in kube_apiserver_endpoint or "localhost" in kube_apiserver_endpoint -%} + {{ first_kube_control_plane_address }}:{{ kube_apiserver_port }} + {%- else -%} + {{ kube_apiserver_endpoint | regex_replace('https://', '') }} + {%- endif %} + tags: + - facts + +- name: Upload certificates so they are fresh and not expired + command: >- + {{ bin_dir }}/kubeadm init phase + --config {{ kube_config_dir }}/kubeadm-config.yaml + upload-certs + --upload-certs + register: kubeadm_upload_cert + when: + - inventory_hostname == first_kube_control_plane + - not kube_external_ca_mode + +- name: Parse certificate key if not set + set_fact: + kubeadm_certificate_key: "{{ hostvars[groups['kube_control_plane'][0]]['kubeadm_upload_cert'].stdout_lines[-1] | trim }}" + run_once: yes + when: + - hostvars[groups['kube_control_plane'][0]]['kubeadm_upload_cert'] is defined + - hostvars[groups['kube_control_plane'][0]]['kubeadm_upload_cert'] is not skipped + +- name: Create kubeadm ControlPlane config + template: + src: "kubeadm-controlplane.{{ kubeadmConfig_api_version }}.yaml.j2" + dest: "{{ kube_config_dir }}/kubeadm-controlplane.yaml" + mode: 0640 + backup: yes + when: + - inventory_hostname != first_kube_control_plane + - not kubeadm_already_run.stat.exists + +- name: Wait for k8s apiserver + wait_for: + host: "{{ kubeadm_discovery_address.split(':')[0] }}" + port: "{{ kubeadm_discovery_address.split(':')[1] }}" + timeout: 180 + + +- name: Check already run + debug: + msg: "{{ kubeadm_already_run.stat.exists }}" + +- name: Reset cert directory + shell: >- + if [ -f /etc/kubernetes/manifests/kube-apiserver.yaml ]; then + {{ bin_dir }}/kubeadm reset -f --cert-dir {{ kube_cert_dir }}; + fi + environment: + PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}" + when: + - inventory_hostname != first_kube_control_plane + - kubeadm_already_run is not defined or not kubeadm_already_run.stat.exists + - not kube_external_ca_mode + +- name: Joining control plane node to the cluster. + command: >- + {{ bin_dir }}/kubeadm join + --config {{ kube_config_dir }}/kubeadm-controlplane.yaml + --ignore-preflight-errors=all + --skip-phases={{ kubeadm_join_phases_skip | join(',') }} + environment: + PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}" + register: kubeadm_join_control_plane + retries: 3 + throttle: 1 + until: kubeadm_join_control_plane is succeeded + when: + - inventory_hostname != first_kube_control_plane + - kubeadm_already_run is not defined or not kubeadm_already_run.stat.exists diff --git a/kubespray/project/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml b/kubespray/project/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml new file mode 100644 index 0000000..1f4ff20 --- /dev/null +++ b/kubespray/project/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml @@ -0,0 +1,265 @@ +--- +- name: Install OIDC certificate + copy: + content: "{{ kube_oidc_ca_cert | b64decode }}" + dest: "{{ kube_oidc_ca_file }}" + owner: root + group: root + mode: "0644" + when: + - kube_oidc_auth + - kube_oidc_ca_cert is defined + +- name: Kubeadm | Check if kubeadm has already run + stat: + path: "/var/lib/kubelet/config.yaml" + get_attributes: no + get_checksum: no + get_mime: no + register: kubeadm_already_run + +- name: Kubeadm | Backup kubeadm certs / kubeconfig + import_tasks: kubeadm-backup.yml + when: + - kubeadm_already_run.stat.exists + +- name: Kubeadm | aggregate all SANs + set_fact: + apiserver_sans: "{{ (sans_base + groups['kube_control_plane'] + sans_lb + sans_lb_ip + sans_supp + sans_access_ip + sans_ip + sans_address + sans_override + sans_hostname + sans_fqdn + sans_kube_vip_address) | unique }}" + vars: + sans_base: + - "kubernetes" + - "kubernetes.default" + - "kubernetes.default.svc" + - "kubernetes.default.svc.{{ dns_domain }}" + - "{{ kube_apiserver_ip }}" + - "localhost" + - "127.0.0.1" + sans_lb: "{{ [apiserver_loadbalancer_domain_name] if apiserver_loadbalancer_domain_name is defined else [] }}" + sans_lb_ip: "{{ [loadbalancer_apiserver.address] if loadbalancer_apiserver is defined and loadbalancer_apiserver.address is defined else [] }}" + sans_supp: "{{ supplementary_addresses_in_ssl_keys if supplementary_addresses_in_ssl_keys is defined else [] }}" + sans_access_ip: "{{ groups['kube_control_plane'] | map('extract', hostvars, 'access_ip') | list | select('defined') | list }}" + sans_ip: "{{ groups['kube_control_plane'] | map('extract', hostvars, 'ip') | list | select('defined') | list }}" + sans_address: "{{ groups['kube_control_plane'] | map('extract', hostvars, ['ansible_default_ipv4', 'address']) | list | select('defined') | list }}" + sans_override: "{{ [kube_override_hostname] if kube_override_hostname else [] }}" + sans_hostname: "{{ groups['kube_control_plane'] | map('extract', hostvars, ['ansible_hostname']) | list | select('defined') | list }}" + sans_fqdn: "{{ groups['kube_control_plane'] | map('extract', hostvars, ['ansible_fqdn']) | list | select('defined') | list }}" + sans_kube_vip_address: "{{ [kube_vip_address] if kube_vip_address is defined and kube_vip_address else [] }}" + tags: facts + +- name: Create audit-policy directory + file: + path: "{{ audit_policy_file | dirname }}" + state: directory + mode: 0640 + when: kubernetes_audit | default(false) or kubernetes_audit_webhook | default(false) + +- name: Write api audit policy yaml + template: + src: apiserver-audit-policy.yaml.j2 + dest: "{{ audit_policy_file }}" + mode: 0640 + when: kubernetes_audit | default(false) or kubernetes_audit_webhook | default(false) + +- name: Write api audit webhook config yaml + template: + src: apiserver-audit-webhook-config.yaml.j2 + dest: "{{ audit_webhook_config_file }}" + mode: 0640 + when: kubernetes_audit_webhook | default(false) + +- name: Create apiserver tracing config directory + file: + path: "{{ kube_config_dir }}/tracing" + state: directory + mode: 0640 + when: kube_apiserver_tracing + +- name: Write apiserver tracing config yaml + template: + src: apiserver-tracing.yaml.j2 + dest: "{{ kube_config_dir }}/tracing/apiserver-tracing.yaml" + mode: 0640 + when: kube_apiserver_tracing + +# Nginx LB(default), If kubeadm_config_api_fqdn is defined, use other LB by kubeadm controlPlaneEndpoint. +- name: Set kubeadm_config_api_fqdn define + set_fact: + kubeadm_config_api_fqdn: "{{ apiserver_loadbalancer_domain_name | default('lb-apiserver.kubernetes.local') }}" + when: loadbalancer_apiserver is defined + +- name: Set kubeadm api version to v1beta3 + set_fact: + kubeadmConfig_api_version: v1beta3 + +- name: Kubeadm | Create kubeadm config + template: + src: "kubeadm-config.{{ kubeadmConfig_api_version }}.yaml.j2" + dest: "{{ kube_config_dir }}/kubeadm-config.yaml" + mode: 0640 + +- name: Kubeadm | Create directory to store admission control configurations + file: + path: "{{ kube_config_dir }}/admission-controls" + state: directory + mode: 0640 + when: kube_apiserver_admission_control_config_file + +- name: Kubeadm | Push admission control config file + template: + src: "admission-controls.yaml.j2" + dest: "{{ kube_config_dir }}/admission-controls/admission-controls.yaml" + mode: 0640 + when: kube_apiserver_admission_control_config_file + +- name: Kubeadm | Push admission control config files + template: + src: "{{ item | lower }}.yaml.j2" + dest: "{{ kube_config_dir }}/admission-controls/{{ item | lower }}.yaml" + mode: 0640 + when: + - kube_apiserver_admission_control_config_file + - item in kube_apiserver_admission_plugins_needs_configuration + loop: "{{ kube_apiserver_enable_admission_plugins }}" + +- name: Kubeadm | Configure default cluster podnodeslector + template: + src: "podnodeselector.yaml.j2" + dest: "{{ kube_config_dir }}/admission-controls/podnodeselector.yaml" + mode: 0640 + when: + - kube_apiserver_admission_plugins_podnodeselector_default_node_selector is defined + - kube_apiserver_admission_plugins_podnodeselector_default_node_selector | length > 0 + +- name: Kubeadm | Check apiserver.crt SANs + vars: + apiserver_ips: "{{ apiserver_sans | map('ansible.utils.ipaddr') | reject('equalto', False) | list }}" + apiserver_hosts: "{{ apiserver_sans | difference(apiserver_ips) }}" + when: + - kubeadm_already_run.stat.exists + - not kube_external_ca_mode + block: + - name: Kubeadm | Check apiserver.crt SAN IPs + command: + cmd: "openssl x509 -noout -in {{ kube_cert_dir }}/apiserver.crt -checkip {{ item }}" + loop: "{{ apiserver_ips }}" + register: apiserver_sans_ip_check + changed_when: apiserver_sans_ip_check.stdout is not search('does match certificate') + - name: Kubeadm | Check apiserver.crt SAN hosts + command: + cmd: "openssl x509 -noout -in {{ kube_cert_dir }}/apiserver.crt -checkhost {{ item }}" + loop: "{{ apiserver_hosts }}" + register: apiserver_sans_host_check + changed_when: apiserver_sans_host_check.stdout is not search('does match certificate') + +- name: Kubeadm | regenerate apiserver cert 1/2 + file: + state: absent + path: "{{ kube_cert_dir }}/{{ item }}" + with_items: + - apiserver.crt + - apiserver.key + when: + - kubeadm_already_run.stat.exists + - apiserver_sans_ip_check.changed or apiserver_sans_host_check.changed + - not kube_external_ca_mode + +- name: Kubeadm | regenerate apiserver cert 2/2 + command: >- + {{ bin_dir }}/kubeadm + init phase certs apiserver + --config={{ kube_config_dir }}/kubeadm-config.yaml + when: + - kubeadm_already_run.stat.exists + - apiserver_sans_ip_check.changed or apiserver_sans_host_check.changed + - not kube_external_ca_mode + +- name: Kubeadm | Create directory to store kubeadm patches + file: + path: "{{ kubeadm_patches.dest_dir }}" + state: directory + mode: 0640 + when: kubeadm_patches is defined and kubeadm_patches.enabled + +- name: Kubeadm | Copy kubeadm patches from inventory files + copy: + src: "{{ kubeadm_patches.source_dir }}/" + dest: "{{ kubeadm_patches.dest_dir }}" + owner: "root" + mode: 0644 + when: kubeadm_patches is defined and kubeadm_patches.enabled + +- name: Kubeadm | Initialize first master + command: >- + timeout -k {{ kubeadm_init_timeout }} {{ kubeadm_init_timeout }} + {{ bin_dir }}/kubeadm init + --config={{ kube_config_dir }}/kubeadm-config.yaml + --ignore-preflight-errors=all + --skip-phases={{ kubeadm_init_phases_skip | join(',') }} + {{ kube_external_ca_mode | ternary('', '--upload-certs') }} + register: kubeadm_init + # Retry is because upload config sometimes fails + retries: 3 + until: kubeadm_init is succeeded or "field is immutable" in kubeadm_init.stderr + when: inventory_hostname == first_kube_control_plane and not kubeadm_already_run.stat.exists + failed_when: kubeadm_init.rc != 0 and "field is immutable" not in kubeadm_init.stderr + environment: + PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}" + notify: Master | restart kubelet + +- name: Set kubeadm certificate key + set_fact: + kubeadm_certificate_key: "{{ item | regex_search('--certificate-key ([^ ]+)', '\\1') | first }}" + with_items: "{{ hostvars[groups['kube_control_plane'][0]]['kubeadm_init'].stdout_lines | default([]) }}" + when: + - kubeadm_certificate_key is not defined + - (item | trim) is match('.*--certificate-key.*') + +- name: Create hardcoded kubeadm token for joining nodes with 24h expiration (if defined) + shell: >- + {{ bin_dir }}/kubeadm --kubeconfig {{ kube_config_dir }}/admin.conf token delete {{ kubeadm_token }} || :; + {{ bin_dir }}/kubeadm --kubeconfig {{ kube_config_dir }}/admin.conf token create {{ kubeadm_token }} + changed_when: false + when: + - inventory_hostname == first_kube_control_plane + - kubeadm_token is defined + - kubeadm_refresh_token + tags: + - kubeadm_token + +- name: Create kubeadm token for joining nodes with 24h expiration (default) + command: "{{ bin_dir }}/kubeadm --kubeconfig {{ kube_config_dir }}/admin.conf token create" + changed_when: false + register: temp_token + retries: 5 + delay: 5 + until: temp_token is succeeded + delegate_to: "{{ first_kube_control_plane }}" + when: kubeadm_token is not defined + tags: + - kubeadm_token + +- name: Set kubeadm_token + set_fact: + kubeadm_token: "{{ temp_token.stdout }}" + when: temp_token.stdout is defined + tags: + - kubeadm_token + +- name: Kubeadm | Join other masters + include_tasks: kubeadm-secondary.yml + +- name: Kubeadm | upgrade kubernetes cluster + include_tasks: kubeadm-upgrade.yml + when: + - upgrade_cluster_setup + - kubeadm_already_run.stat.exists + +# FIXME(mattymo): from docs: If you don't want to taint your control-plane node, set this field to an empty slice, i.e. `taints: {}` in the YAML file. +- name: Kubeadm | Remove taint for master with node role + command: "{{ kubectl }} taint node {{ inventory_hostname }} {{ item }}" + delegate_to: "{{ first_kube_control_plane }}" + with_items: + - "node-role.kubernetes.io/control-plane:NoSchedule-" + when: inventory_hostname in groups['kube_node'] + failed_when: false diff --git a/kubespray/project/roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml b/kubespray/project/roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml new file mode 100644 index 0000000..12ab0b9 --- /dev/null +++ b/kubespray/project/roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml @@ -0,0 +1,78 @@ +--- +- name: Kubeadm | Check api is up + uri: + url: "https://{{ ip | default(fallback_ips[inventory_hostname]) }}:{{ kube_apiserver_port }}/healthz" + validate_certs: false + when: inventory_hostname in groups['kube_control_plane'] + register: _result + retries: 60 + delay: 5 + until: _result.status == 200 + +- name: Kubeadm | Upgrade first master + command: >- + timeout -k 600s 600s + {{ bin_dir }}/kubeadm + upgrade apply -y {{ kube_version }} + --certificate-renewal={{ kubeadm_upgrade_auto_cert_renewal }} + --config={{ kube_config_dir }}/kubeadm-config.yaml + --ignore-preflight-errors=all + --allow-experimental-upgrades + --etcd-upgrade={{ (etcd_deployment_type == "kubeadm") | bool | lower }} + {% if kubeadm_patches is defined and kubeadm_patches.enabled %}--patches={{ kubeadm_patches.dest_dir }}{% endif %} + --force + register: kubeadm_upgrade + # Retry is because upload config sometimes fails + retries: 3 + until: kubeadm_upgrade.rc == 0 + when: inventory_hostname == first_kube_control_plane + failed_when: kubeadm_upgrade.rc != 0 and "field is immutable" not in kubeadm_upgrade.stderr + environment: + PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}" + notify: Master | restart kubelet + +- name: Kubeadm | Upgrade other masters + command: >- + timeout -k 600s 600s + {{ bin_dir }}/kubeadm + upgrade apply -y {{ kube_version }} + --certificate-renewal={{ kubeadm_upgrade_auto_cert_renewal }} + --config={{ kube_config_dir }}/kubeadm-config.yaml + --ignore-preflight-errors=all + --allow-experimental-upgrades + --etcd-upgrade={{ (etcd_deployment_type == "kubeadm") | bool | lower }} + {% if kubeadm_patches is defined and kubeadm_patches.enabled %}--patches={{ kubeadm_patches.dest_dir }}{% endif %} + --force + register: kubeadm_upgrade + # Retry is because upload config sometimes fails + retries: 3 + until: kubeadm_upgrade.rc == 0 + when: inventory_hostname != first_kube_control_plane + failed_when: kubeadm_upgrade.rc != 0 and "field is immutable" not in kubeadm_upgrade.stderr + environment: + PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}" + notify: Master | restart kubelet + +- name: Kubeadm | clean kubectl cache to refresh api types + file: + path: "{{ item }}" + state: absent + with_items: + - /root/.kube/cache + - /root/.kube/http-cache + +# FIXME: https://github.com/kubernetes/kubeadm/issues/1318 +- name: Kubeadm | scale down coredns replicas to 0 if not using coredns dns_mode + command: >- + {{ kubectl }} + -n kube-system + scale deployment/coredns --replicas 0 + register: scale_down_coredns + retries: 6 + delay: 5 + until: scale_down_coredns is succeeded + run_once: true + when: + - kubeadm_scale_down_coredns_enabled + - dns_mode not in ['coredns', 'coredns_dual'] + changed_when: false diff --git a/kubespray/project/roles/kubernetes/control-plane/tasks/kubelet-fix-client-cert-rotation.yml b/kubespray/project/roles/kubernetes/control-plane/tasks/kubelet-fix-client-cert-rotation.yml new file mode 100644 index 0000000..7d0c1a0 --- /dev/null +++ b/kubespray/project/roles/kubernetes/control-plane/tasks/kubelet-fix-client-cert-rotation.yml @@ -0,0 +1,18 @@ +--- +- name: Fixup kubelet client cert rotation 1/2 + lineinfile: + path: "{{ kube_config_dir }}/kubelet.conf" + regexp: '^ client-certificate-data: ' + line: ' client-certificate: /var/lib/kubelet/pki/kubelet-client-current.pem' + backup: yes + notify: + - "Master | reload kubelet" + +- name: Fixup kubelet client cert rotation 2/2 + lineinfile: + path: "{{ kube_config_dir }}/kubelet.conf" + regexp: '^ client-key-data: ' + line: ' client-key: /var/lib/kubelet/pki/kubelet-client-current.pem' + backup: yes + notify: + - "Master | reload kubelet" diff --git a/kubespray/project/roles/kubernetes/control-plane/tasks/main.yml b/kubespray/project/roles/kubernetes/control-plane/tasks/main.yml new file mode 100644 index 0000000..37f36ab --- /dev/null +++ b/kubespray/project/roles/kubernetes/control-plane/tasks/main.yml @@ -0,0 +1,126 @@ +--- +- name: Pre-upgrade control plane + import_tasks: pre-upgrade.yml + tags: + - k8s-pre-upgrade + +- name: Create webhook token auth config + template: + src: webhook-token-auth-config.yaml.j2 + dest: "{{ kube_config_dir }}/webhook-token-auth-config.yaml" + mode: 0640 + when: kube_webhook_token_auth | default(false) + +- name: Create webhook authorization config + template: + src: webhook-authorization-config.yaml.j2 + dest: "{{ kube_config_dir }}/webhook-authorization-config.yaml" + mode: 0640 + when: kube_webhook_authorization | default(false) + +- name: Create kube-scheduler config + template: + src: kubescheduler-config.yaml.j2 + dest: "{{ kube_config_dir }}/kubescheduler-config.yaml" + mode: 0644 + +- name: Apply Kubernetes encrypt at rest config + import_tasks: encrypt-at-rest.yml + when: + - kube_encrypt_secret_data + tags: + - kube-apiserver + +- name: Install | Copy kubectl binary from download dir + copy: + src: "{{ downloads.kubectl.dest }}" + dest: "{{ bin_dir }}/kubectl" + mode: 0755 + remote_src: true + tags: + - kubectl + - upgrade + +- name: Install kubectl bash completion + shell: "{{ bin_dir }}/kubectl completion bash >/etc/bash_completion.d/kubectl.sh" + when: ansible_os_family in ["Debian","RedHat"] + tags: + - kubectl + ignore_errors: true # noqa ignore-errors + +- name: Set kubectl bash completion file permissions + file: + path: /etc/bash_completion.d/kubectl.sh + owner: root + group: root + mode: 0755 + when: ansible_os_family in ["Debian","RedHat"] + tags: + - kubectl + - upgrade + ignore_errors: true # noqa ignore-errors + +- name: Set bash alias for kubectl + blockinfile: + path: /etc/bash_completion.d/kubectl.sh + block: |- + alias {{ kubectl_alias }}=kubectl + if [[ $(type -t compopt) = "builtin" ]]; then + complete -o default -F __start_kubectl {{ kubectl_alias }} + else + complete -o default -o nospace -F __start_kubectl {{ kubectl_alias }} + fi + state: present + marker: "# Ansible entries {mark}" + when: + - ansible_os_family in ["Debian","RedHat"] + - kubectl_alias is defined and kubectl_alias != "" + tags: + - kubectl + - upgrade + ignore_errors: true # noqa ignore-errors + +- name: Define nodes already joined to existing cluster and first_kube_control_plane + import_tasks: define-first-kube-control.yml + +- name: Include kubeadm setup + import_tasks: kubeadm-setup.yml + +- name: Include kubeadm etcd extra tasks + include_tasks: kubeadm-etcd.yml + when: etcd_deployment_type == "kubeadm" + +- name: Include kubeadm secondary server apiserver fixes + include_tasks: kubeadm-fix-apiserver.yml + +- name: Include kubelet client cert rotation fixes + include_tasks: kubelet-fix-client-cert-rotation.yml + when: kubelet_rotate_certificates + +- name: Install script to renew K8S control plane certificates + template: + src: k8s-certs-renew.sh.j2 + dest: "{{ bin_dir }}/k8s-certs-renew.sh" + mode: 0755 + +- name: Renew K8S control plane certificates monthly 1/2 + template: + src: "{{ item }}.j2" + dest: "/etc/systemd/system/{{ item }}" + mode: 0644 + validate: "sh -c '[ -f /usr/bin/systemd/system/factory-reset.target ] || exit 0 && systemd-analyze verify %s:{{item}}'" + # FIXME: check that systemd version >= 250 (factory-reset.target was introduced in that release) + # Remove once we drop support for systemd < 250 + with_items: + - k8s-certs-renew.service + - k8s-certs-renew.timer + register: k8s_certs_units + when: auto_renew_certificates + +- name: Renew K8S control plane certificates monthly 2/2 + systemd: + name: k8s-certs-renew.timer + enabled: yes + state: started + daemon_reload: "{{ k8s_certs_units is changed }}" + when: auto_renew_certificates diff --git a/kubespray/project/roles/kubernetes/control-plane/tasks/pre-upgrade.yml b/kubespray/project/roles/kubernetes/control-plane/tasks/pre-upgrade.yml new file mode 100644 index 0000000..2d7dce5 --- /dev/null +++ b/kubespray/project/roles/kubernetes/control-plane/tasks/pre-upgrade.yml @@ -0,0 +1,21 @@ +--- +- name: "Pre-upgrade | Delete master manifests if etcd secrets changed" + file: + path: "/etc/kubernetes/manifests/{{ item }}.manifest" + state: absent + with_items: + - ["kube-apiserver", "kube-controller-manager", "kube-scheduler"] + register: kube_apiserver_manifest_replaced + when: etcd_secret_changed | default(false) + +- name: "Pre-upgrade | Delete master containers forcefully" # noqa no-handler + shell: "set -o pipefail && docker ps -af name=k8s_{{ item }}* -q | xargs --no-run-if-empty docker rm -f" + args: + executable: /bin/bash + with_items: + - ["kube-apiserver", "kube-controller-manager", "kube-scheduler"] + when: kube_apiserver_manifest_replaced.changed + register: remove_master_container + retries: 10 + until: remove_master_container.rc == 0 + delay: 1 diff --git a/kubespray/project/roles/kubernetes/control-plane/templates/admission-controls.yaml.j2 b/kubespray/project/roles/kubernetes/control-plane/templates/admission-controls.yaml.j2 new file mode 100644 index 0000000..fc4d0ef --- /dev/null +++ b/kubespray/project/roles/kubernetes/control-plane/templates/admission-controls.yaml.j2 @@ -0,0 +1,9 @@ +apiVersion: apiserver.config.k8s.io/v1 +kind: AdmissionConfiguration +plugins: +{% for plugin in kube_apiserver_enable_admission_plugins %} +{% if plugin in kube_apiserver_admission_plugins_needs_configuration %} +- name: {{ plugin }} + path: {{ kube_config_dir }}/{{ plugin | lower }}.yaml +{% endif %} +{% endfor %} diff --git a/kubespray/project/roles/kubernetes/control-plane/templates/apiserver-audit-policy.yaml.j2 b/kubespray/project/roles/kubernetes/control-plane/templates/apiserver-audit-policy.yaml.j2 new file mode 100644 index 0000000..ca7bcf8 --- /dev/null +++ b/kubespray/project/roles/kubernetes/control-plane/templates/apiserver-audit-policy.yaml.j2 @@ -0,0 +1,129 @@ +apiVersion: audit.k8s.io/v1 +kind: Policy +rules: +{% if audit_policy_custom_rules is defined and audit_policy_custom_rules != "" %} +{{ audit_policy_custom_rules | indent(2, true) }} +{% else %} + # The following requests were manually identified as high-volume and low-risk, + # so drop them. + - level: None + users: ["system:kube-proxy"] + verbs: ["watch"] + resources: + - group: "" # core + resources: ["endpoints", "services", "services/status"] + - level: None + # Ingress controller reads `configmaps/ingress-uid` through the unsecured port. + # TODO(#46983): Change this to the ingress controller service account. + users: ["system:unsecured"] + namespaces: ["kube-system"] + verbs: ["get"] + resources: + - group: "" # core + resources: ["configmaps"] + - level: None + users: ["kubelet"] # legacy kubelet identity + verbs: ["get"] + resources: + - group: "" # core + resources: ["nodes", "nodes/status"] + - level: None + userGroups: ["system:nodes"] + verbs: ["get"] + resources: + - group: "" # core + resources: ["nodes", "nodes/status"] + - level: None + users: + - system:kube-controller-manager + - system:kube-scheduler + - system:serviceaccount:kube-system:endpoint-controller + verbs: ["get", "update"] + namespaces: ["kube-system"] + resources: + - group: "" # core + resources: ["endpoints"] + - level: None + users: ["system:apiserver"] + verbs: ["get"] + resources: + - group: "" # core + resources: ["namespaces", "namespaces/status", "namespaces/finalize"] + # Don't log HPA fetching metrics. + - level: None + users: + - system:kube-controller-manager + verbs: ["get", "list"] + resources: + - group: "metrics.k8s.io" + # Don't log these read-only URLs. + - level: None + nonResourceURLs: + - /healthz* + - /version + - /swagger* + # Don't log events requests. + - level: None + resources: + - group: "" # core + resources: ["events"] + # Secrets, ConfigMaps, TokenRequest and TokenReviews can contain sensitive & binary data, + # so only log at the Metadata level. + - level: Metadata + resources: + - group: "" # core + resources: ["secrets", "configmaps", "serviceaccounts/token"] + - group: authentication.k8s.io + resources: ["tokenreviews"] + omitStages: + - "RequestReceived" + # Get responses can be large; skip them. + - level: Request + verbs: ["get", "list", "watch"] + resources: + - group: "" # core + - group: "admissionregistration.k8s.io" + - group: "apiextensions.k8s.io" + - group: "apiregistration.k8s.io" + - group: "apps" + - group: "authentication.k8s.io" + - group: "authorization.k8s.io" + - group: "autoscaling" + - group: "batch" + - group: "certificates.k8s.io" + - group: "extensions" + - group: "metrics.k8s.io" + - group: "networking.k8s.io" + - group: "policy" + - group: "rbac.authorization.k8s.io" + - group: "settings.k8s.io" + - group: "storage.k8s.io" + omitStages: + - "RequestReceived" + # Default level for known APIs + - level: RequestResponse + resources: + - group: "" # core + - group: "admissionregistration.k8s.io" + - group: "apiextensions.k8s.io" + - group: "apiregistration.k8s.io" + - group: "apps" + - group: "authentication.k8s.io" + - group: "authorization.k8s.io" + - group: "autoscaling" + - group: "batch" + - group: "certificates.k8s.io" + - group: "extensions" + - group: "metrics.k8s.io" + - group: "networking.k8s.io" + - group: "policy" + - group: "rbac.authorization.k8s.io" + - group: "settings.k8s.io" + - group: "storage.k8s.io" + omitStages: + - "RequestReceived" + # Default level for all other requests. + - level: Metadata + omitStages: + - "RequestReceived" +{% endif %} diff --git a/kubespray/project/roles/kubernetes/control-plane/templates/apiserver-audit-webhook-config.yaml.j2 b/kubespray/project/roles/kubernetes/control-plane/templates/apiserver-audit-webhook-config.yaml.j2 new file mode 100644 index 0000000..cd8208e --- /dev/null +++ b/kubespray/project/roles/kubernetes/control-plane/templates/apiserver-audit-webhook-config.yaml.j2 @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Config +clusters: +- cluster: + server: {{ audit_webhook_server_url }} +{% for key in audit_webhook_server_extra_args %} + {{ key }}: "{{ audit_webhook_server_extra_args[key] }}" +{% endfor %} + name: auditsink +contexts: +- context: + cluster: auditsink + user: "" + name: default-context +current-context: default-context +preferences: {} +users: [] diff --git a/kubespray/project/roles/kubernetes/control-plane/templates/apiserver-tracing.yaml.j2 b/kubespray/project/roles/kubernetes/control-plane/templates/apiserver-tracing.yaml.j2 new file mode 100644 index 0000000..98decde --- /dev/null +++ b/kubespray/project/roles/kubernetes/control-plane/templates/apiserver-tracing.yaml.j2 @@ -0,0 +1,4 @@ +apiVersion: apiserver.config.k8s.io/v1beta1 +kind: TracingConfiguration +endpoint: {{ kube_apiserver_tracing_endpoint }} +samplingRatePerMillion: {{ kube_apiserver_tracing_sampling_rate_per_million }} \ No newline at end of file diff --git a/kubespray/project/roles/kubernetes/control-plane/templates/eventratelimit.yaml.j2 b/kubespray/project/roles/kubernetes/control-plane/templates/eventratelimit.yaml.j2 new file mode 100644 index 0000000..0d78670 --- /dev/null +++ b/kubespray/project/roles/kubernetes/control-plane/templates/eventratelimit.yaml.j2 @@ -0,0 +1,11 @@ +apiVersion: eventratelimit.admission.k8s.io/v1alpha1 +kind: Configuration +limits: +{% for limit in kube_apiserver_admission_event_rate_limits.values() %} +- type: {{ limit.type }} + qps: {{ limit.qps }} + burst: {{ limit.burst }} +{% if limit.cache_size is defined %} + cacheSize: {{ limit.cache_size }} +{% endif %} +{% endfor %} diff --git a/kubespray/project/roles/kubernetes/control-plane/templates/k8s-certs-renew.service.j2 b/kubespray/project/roles/kubernetes/control-plane/templates/k8s-certs-renew.service.j2 new file mode 100644 index 0000000..64610c2 --- /dev/null +++ b/kubespray/project/roles/kubernetes/control-plane/templates/k8s-certs-renew.service.j2 @@ -0,0 +1,6 @@ +[Unit] +Description=Renew K8S control plane certificates + +[Service] +Type=oneshot +ExecStart={{ bin_dir }}/k8s-certs-renew.sh diff --git a/kubespray/project/roles/kubernetes/control-plane/templates/k8s-certs-renew.sh.j2 b/kubespray/project/roles/kubernetes/control-plane/templates/k8s-certs-renew.sh.j2 new file mode 100644 index 0000000..b2c7c77 --- /dev/null +++ b/kubespray/project/roles/kubernetes/control-plane/templates/k8s-certs-renew.sh.j2 @@ -0,0 +1,23 @@ +#!/bin/bash + +echo "## Expiration before renewal ##" +{{ bin_dir }}/kubeadm certs check-expiration + +echo "## Renewing certificates managed by kubeadm ##" +{{ bin_dir }}/kubeadm certs renew all + +echo "## Restarting control plane pods managed by kubeadm ##" +{% if container_manager == "docker" %} +{{ docker_bin_dir }}/docker ps -af 'name=k8s_POD_(kube-apiserver|kube-controller-manager|kube-scheduler|etcd)-*' -q | /usr/bin/xargs {{ docker_bin_dir }}/docker rm -f +{% else %} +{{ bin_dir }}/crictl pods --namespace kube-system --name 'kube-scheduler-*|kube-controller-manager-*|kube-apiserver-*|etcd-*' -q | /usr/bin/xargs {{ bin_dir }}/crictl rmp -f +{% endif %} + +echo "## Updating /root/.kube/config ##" +cp {{ kube_config_dir }}/admin.conf /root/.kube/config + +echo "## Waiting for apiserver to be up again ##" +until printf "" 2>>/dev/null >>/dev/tcp/127.0.0.1/{{ kube_apiserver_port | default(6443) }}; do sleep 1; done + +echo "## Expiration after renewal ##" +{{ bin_dir }}/kubeadm certs check-expiration diff --git a/kubespray/project/roles/kubernetes/control-plane/templates/k8s-certs-renew.timer.j2 b/kubespray/project/roles/kubernetes/control-plane/templates/k8s-certs-renew.timer.j2 new file mode 100644 index 0000000..cca5aca --- /dev/null +++ b/kubespray/project/roles/kubernetes/control-plane/templates/k8s-certs-renew.timer.j2 @@ -0,0 +1,11 @@ +[Unit] +Description=Timer to renew K8S control plane certificates + +[Timer] +OnCalendar={{ auto_renew_certificates_systemd_calendar }} +RandomizedDelaySec={{ 10 * (groups['kube_control_plane'] | length) }}min +FixedRandomDelay=yes +Persistent=yes + +[Install] +WantedBy=multi-user.target diff --git a/kubespray/project/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2 b/kubespray/project/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2 new file mode 100644 index 0000000..b11fb33 --- /dev/null +++ b/kubespray/project/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2 @@ -0,0 +1,471 @@ +apiVersion: kubeadm.k8s.io/v1beta3 +kind: InitConfiguration +{% if kubeadm_token is defined %} +bootstrapTokens: +- token: "{{ kubeadm_token }}" + description: "kubespray kubeadm bootstrap token" + ttl: "24h" +{% endif %} +localAPIEndpoint: + advertiseAddress: {{ kube_apiserver_address }} + bindPort: {{ kube_apiserver_port }} +{% if kubeadm_certificate_key is defined %} +certificateKey: {{ kubeadm_certificate_key }} +{% endif %} +nodeRegistration: +{% if kube_override_hostname | default('') %} + name: {{ kube_override_hostname }} +{% endif %} +{% if inventory_hostname in groups['kube_control_plane'] and inventory_hostname not in groups['kube_node'] %} + taints: + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane +{% else %} + taints: [] +{% endif %} + criSocket: {{ cri_socket }} +{% if cloud_provider is defined and cloud_provider in ["external"] %} + kubeletExtraArgs: + cloud-provider: external +{% endif %} +{% if kubeadm_patches is defined and kubeadm_patches.enabled %} +patches: + directory: {{ kubeadm_patches.dest_dir }} +{% endif %} +--- +apiVersion: kubeadm.k8s.io/v1beta3 +kind: ClusterConfiguration +clusterName: {{ cluster_name }} +etcd: +{% if etcd_deployment_type != "kubeadm" %} + external: + endpoints: +{% for endpoint in etcd_access_addresses.split(',') %} + - {{ endpoint }} +{% endfor %} + caFile: {{ etcd_cert_dir }}/{{ kube_etcd_cacert_file }} + certFile: {{ etcd_cert_dir }}/{{ kube_etcd_cert_file }} + keyFile: {{ etcd_cert_dir }}/{{ kube_etcd_key_file }} +{% elif etcd_deployment_type == "kubeadm" %} + local: + imageRepository: "{{ etcd_image_repo | regex_replace("/etcd$","") }}" + imageTag: "{{ etcd_image_tag }}" + dataDir: "{{ etcd_data_dir }}" + extraArgs: + metrics: {{ etcd_metrics }} + election-timeout: "{{ etcd_election_timeout }}" + heartbeat-interval: "{{ etcd_heartbeat_interval }}" + auto-compaction-retention: "{{ etcd_compaction_retention }}" +{% if etcd_listen_metrics_urls is defined %} + listen-metrics-urls: "{{ etcd_listen_metrics_urls }}" +{% endif %} +{% if etcd_snapshot_count is defined %} + snapshot-count: "{{ etcd_snapshot_count }}" +{% endif %} +{% if etcd_quota_backend_bytes is defined %} + quota-backend-bytes: "{{ etcd_quota_backend_bytes }}" +{% endif %} +{% if etcd_max_request_bytes is defined %} + max-request-bytes: "{{ etcd_max_request_bytes }}" +{% endif %} +{% if etcd_log_level is defined %} + log-level: "{{ etcd_log_level }}" +{% endif %} +{% for key, value in etcd_extra_vars.items() %} + {{ key }}: "{{ value }}" +{% endfor %} + serverCertSANs: +{% for san in etcd_cert_alt_names %} + - {{ san }} +{% endfor %} +{% for san in etcd_cert_alt_ips %} + - {{ san }} +{% endfor %} + peerCertSANs: +{% for san in etcd_cert_alt_names %} + - {{ san }} +{% endfor %} +{% for san in etcd_cert_alt_ips %} + - {{ san }} +{% endfor %} +{% endif %} +dns: + imageRepository: {{ coredns_image_repo | regex_replace('/coredns(?!/coredns).*$', '') }} + imageTag: {{ coredns_image_tag }} +networking: + dnsDomain: {{ dns_domain }} + serviceSubnet: "{{ kube_service_addresses }}{{ ',' + kube_service_addresses_ipv6 if enable_dual_stack_networks else '' }}" +{% if kube_network_plugin is defined and kube_network_plugin not in ["kube-ovn"] %} + podSubnet: "{{ kube_pods_subnet }}{{ ',' + kube_pods_subnet_ipv6 if enable_dual_stack_networks else '' }}" +{% endif %} +{% if kubeadm_feature_gates %} +featureGates: +{% for feature in kubeadm_feature_gates %} + {{ feature | replace("=", ": ") }} +{% endfor %} +{% endif %} +kubernetesVersion: {{ kube_version }} +{% if kubeadm_config_api_fqdn is defined %} +controlPlaneEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }} +{% else %} +controlPlaneEndpoint: {{ ip | default(fallback_ips[inventory_hostname]) }}:{{ kube_apiserver_port }} +{% endif %} +certificatesDir: {{ kube_cert_dir }} +imageRepository: {{ kube_image_repo }} +apiServer: + extraArgs: +{% if kube_apiserver_etcd_compaction_interval is defined %} + etcd-compaction-interval: "{{ kube_apiserver_etcd_compaction_interval }}" +{% endif %} +{% if kube_apiserver_pod_eviction_not_ready_timeout_seconds is defined %} + default-not-ready-toleration-seconds: "{{ kube_apiserver_pod_eviction_not_ready_timeout_seconds }}" +{% endif %} +{% if kube_apiserver_pod_eviction_unreachable_timeout_seconds is defined %} + default-unreachable-toleration-seconds: "{{ kube_apiserver_pod_eviction_unreachable_timeout_seconds }}" +{% endif %} +{% if kube_api_anonymous_auth is defined %} + anonymous-auth: "{{ kube_api_anonymous_auth }}" +{% endif %} + authorization-mode: {{ authorization_modes | join(',') }} + bind-address: {{ kube_apiserver_bind_address }} +{% if kube_apiserver_enable_admission_plugins | length > 0 %} + enable-admission-plugins: {{ kube_apiserver_enable_admission_plugins | join(',') }} +{% endif %} +{% if kube_apiserver_admission_control_config_file %} + admission-control-config-file: {{ kube_config_dir }}/admission-controls.yaml +{% endif %} +{% if kube_apiserver_disable_admission_plugins | length > 0 %} + disable-admission-plugins: {{ kube_apiserver_disable_admission_plugins | join(',') }} +{% endif %} + apiserver-count: "{{ kube_apiserver_count }}" + endpoint-reconciler-type: lease +{% if etcd_events_cluster_enabled %} + etcd-servers-overrides: "/events#{{ etcd_events_access_addresses_semicolon }}" +{% endif %} + service-node-port-range: {{ kube_apiserver_node_port_range }} + service-cluster-ip-range: "{{ kube_service_addresses }}{{ ',' + kube_service_addresses_ipv6 if enable_dual_stack_networks else '' }}" + kubelet-preferred-address-types: "{{ kubelet_preferred_address_types }}" + profiling: "{{ kube_profiling }}" + request-timeout: "{{ kube_apiserver_request_timeout }}" + enable-aggregator-routing: "{{ kube_api_aggregator_routing }}" +{% if kube_token_auth | default(true) %} + token-auth-file: {{ kube_token_dir }}/known_tokens.csv +{% endif %} +{% if kube_apiserver_service_account_lookup %} + service-account-lookup: "{{ kube_apiserver_service_account_lookup }}" +{% endif %} +{% if kube_oidc_auth | default(false) and kube_oidc_url is defined and kube_oidc_client_id is defined %} + oidc-issuer-url: "{{ kube_oidc_url }}" + oidc-client-id: "{{ kube_oidc_client_id }}" +{% if kube_oidc_ca_file is defined %} + oidc-ca-file: "{{ kube_oidc_ca_file }}" +{% endif %} +{% if kube_oidc_username_claim is defined %} + oidc-username-claim: "{{ kube_oidc_username_claim }}" +{% endif %} +{% if kube_oidc_groups_claim is defined %} + oidc-groups-claim: "{{ kube_oidc_groups_claim }}" +{% endif %} +{% if kube_oidc_username_prefix is defined %} + oidc-username-prefix: "{{ kube_oidc_username_prefix }}" +{% endif %} +{% if kube_oidc_groups_prefix is defined %} + oidc-groups-prefix: "{{ kube_oidc_groups_prefix }}" +{% endif %} +{% endif %} +{% if kube_webhook_token_auth | default(false) %} + authentication-token-webhook-config-file: {{ kube_config_dir }}/webhook-token-auth-config.yaml +{% endif %} +{% if kube_webhook_authorization | default(false) %} + authorization-webhook-config-file: {{ kube_config_dir }}/webhook-authorization-config.yaml +{% endif %} +{% if kube_encrypt_secret_data %} + encryption-provider-config: {{ kube_cert_dir }}/secrets_encryption.yaml +{% endif %} + storage-backend: {{ kube_apiserver_storage_backend }} +{% if kube_api_runtime_config | length > 0 %} + runtime-config: {{ kube_api_runtime_config | join(',') }} +{% endif %} + allow-privileged: "true" +{% if kubernetes_audit or kubernetes_audit_webhook %} + audit-policy-file: {{ audit_policy_file }} +{% endif %} +{% if kubernetes_audit %} + audit-log-path: "{{ audit_log_path }}" + audit-log-maxage: "{{ audit_log_maxage }}" + audit-log-maxbackup: "{{ audit_log_maxbackups }}" + audit-log-maxsize: "{{ audit_log_maxsize }}" +{% endif %} +{% if kubernetes_audit_webhook %} + audit-webhook-config-file: {{ audit_webhook_config_file }} + audit-webhook-mode: {{ audit_webhook_mode }} +{% if audit_webhook_mode == "batch" %} + audit-webhook-batch-max-size: "{{ audit_webhook_batch_max_size }}" + audit-webhook-batch-max-wait: "{{ audit_webhook_batch_max_wait }}" +{% endif %} +{% endif %} +{% for key in kube_kubeadm_apiserver_extra_args %} + {{ key }}: "{{ kube_kubeadm_apiserver_extra_args[key] }}" +{% endfor %} +{% if kube_apiserver_feature_gates or kube_feature_gates %} + feature-gates: "{{ kube_apiserver_feature_gates | default(kube_feature_gates, true) | join(',') }}" +{% endif %} +{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "gce"] %} + cloud-provider: {{ cloud_provider }} + cloud-config: {{ kube_config_dir }}/cloud_config +{% endif %} +{% if tls_min_version is defined %} + tls-min-version: {{ tls_min_version }} +{% endif %} +{% if tls_cipher_suites is defined %} + tls-cipher-suites: {% for tls in tls_cipher_suites %}{{ tls }}{{ "," if not loop.last else "" }}{% endfor %} + +{% endif %} +{% if event_ttl_duration is defined %} + event-ttl: {{ event_ttl_duration }} +{% endif %} +{% if kubelet_rotate_server_certificates %} + kubelet-certificate-authority: {{ kube_cert_dir }}/ca.crt +{% endif %} +{% if kube_apiserver_tracing %} + tracing-config-file: {{ kube_config_dir }}/tracing/apiserver-tracing.yaml +{% endif %} +{% if kubernetes_audit or kube_token_auth | default(true) or kube_webhook_token_auth | default(false) or ( cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "gce"] ) or apiserver_extra_volumes or ssl_ca_dirs | length %} + extraVolumes: +{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "gce"] %} + - name: cloud-config + hostPath: {{ kube_config_dir }}/cloud_config + mountPath: {{ kube_config_dir }}/cloud_config +{% endif %} +{% if kube_token_auth | default(true) %} + - name: token-auth-config + hostPath: {{ kube_token_dir }} + mountPath: {{ kube_token_dir }} +{% endif %} +{% if kube_webhook_token_auth | default(false) %} + - name: webhook-token-auth-config + hostPath: {{ kube_config_dir }}/webhook-token-auth-config.yaml + mountPath: {{ kube_config_dir }}/webhook-token-auth-config.yaml +{% endif %} +{% if kube_webhook_authorization | default(false) %} + - name: webhook-authorization-config + hostPath: {{ kube_config_dir }}/webhook-authorization-config.yaml + mountPath: {{ kube_config_dir }}/webhook-authorization-config.yaml +{% endif %} +{% if kubernetes_audit or kubernetes_audit_webhook %} + - name: {{ audit_policy_name }} + hostPath: {{ audit_policy_hostpath }} + mountPath: {{ audit_policy_mountpath }} +{% if audit_log_path != "-" %} + - name: {{ audit_log_name }} + hostPath: {{ audit_log_hostpath }} + mountPath: {{ audit_log_mountpath }} + readOnly: false +{% endif %} +{% endif %} +{% if kube_apiserver_admission_control_config_file %} + - name: admission-control-configs + hostPath: {{ kube_config_dir }}/admission-controls + mountPath: {{ kube_config_dir }} + readOnly: false + pathType: DirectoryOrCreate +{% endif %} +{% if kube_apiserver_tracing %} + - name: tracing + hostPath: {{ kube_config_dir }}/tracing + mountPath: {{ kube_config_dir }}/tracing + readOnly: true + pathType: DirectoryOrCreate +{% endif %} +{% for volume in apiserver_extra_volumes %} + - name: {{ volume.name }} + hostPath: {{ volume.hostPath }} + mountPath: {{ volume.mountPath }} + readOnly: {{ volume.readOnly | d(not (volume.writable | d(false))) }} +{% endfor %} +{% if ssl_ca_dirs | length %} +{% for dir in ssl_ca_dirs %} + - name: {{ dir | regex_replace('^/(.*)$', '\\1' ) | regex_replace('/', '-') }} + hostPath: {{ dir }} + mountPath: {{ dir }} + readOnly: true +{% endfor %} +{% endif %} +{% endif %} + certSANs: +{% for san in apiserver_sans %} + - {{ san }} +{% endfor %} + timeoutForControlPlane: 5m0s +controllerManager: + extraArgs: + node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }} + node-monitor-period: {{ kube_controller_node_monitor_period }} +{% if kube_network_plugin is defined and kube_network_plugin not in ["kube-ovn"] %} + cluster-cidr: "{{ kube_pods_subnet }}{{ ',' + kube_pods_subnet_ipv6 if enable_dual_stack_networks else '' }}" +{% endif %} + service-cluster-ip-range: "{{ kube_service_addresses }}{{ ',' + kube_service_addresses_ipv6 if enable_dual_stack_networks else '' }}" +{% if kube_network_plugin is defined and kube_network_plugin == "calico" and not calico_ipam_host_local %} + allocate-node-cidrs: "false" +{% else %} +{% if enable_dual_stack_networks %} + node-cidr-mask-size-ipv4: "{{ kube_network_node_prefix }}" + node-cidr-mask-size-ipv6: "{{ kube_network_node_prefix_ipv6 }}" +{% else %} + node-cidr-mask-size: "{{ kube_network_node_prefix }}" +{% endif %} +{% endif %} + profiling: "{{ kube_profiling }}" + terminated-pod-gc-threshold: "{{ kube_controller_terminated_pod_gc_threshold }}" + bind-address: {{ kube_controller_manager_bind_address }} + leader-elect-lease-duration: {{ kube_controller_manager_leader_elect_lease_duration }} + leader-elect-renew-deadline: {{ kube_controller_manager_leader_elect_renew_deadline }} +{% if kube_controller_feature_gates or kube_feature_gates %} + feature-gates: "{{ kube_controller_feature_gates | default(kube_feature_gates, true) | join(',') }}" +{% endif %} +{% for key in kube_kubeadm_controller_extra_args %} + {{ key }}: "{{ kube_kubeadm_controller_extra_args[key] }}" +{% endfor %} +{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "gce"] %} + cloud-provider: {{ cloud_provider }} + cloud-config: {{ kube_config_dir }}/cloud_config +{% endif %} +{% if kube_network_plugin is defined and kube_network_plugin not in ["cloud"] %} + configure-cloud-routes: "false" +{% endif %} +{% if kubelet_flexvolumes_plugins_dir is defined %} + flex-volume-plugin-dir: {{ kubelet_flexvolumes_plugins_dir }} +{% endif %} +{% if tls_min_version is defined %} + tls-min-version: {{ tls_min_version }} +{% endif %} +{% if tls_cipher_suites is defined %} + tls-cipher-suites: {% for tls in tls_cipher_suites %}{{ tls }}{{ "," if not loop.last else "" }}{% endfor %} + +{% endif %} +{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "gce"] or controller_manager_extra_volumes %} + extraVolumes: +{% if cloud_provider is defined and cloud_provider in ["openstack"] and openstack_cacert is defined %} + - name: openstackcacert + hostPath: "{{ kube_config_dir }}/openstack-cacert.pem" + mountPath: "{{ kube_config_dir }}/openstack-cacert.pem" +{% endif %} +{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "gce"] %} + - name: cloud-config + hostPath: {{ kube_config_dir }}/cloud_config + mountPath: {{ kube_config_dir }}/cloud_config +{% endif %} +{% for volume in controller_manager_extra_volumes %} + - name: {{ volume.name }} + hostPath: {{ volume.hostPath }} + mountPath: {{ volume.mountPath }} + readOnly: {{ volume.readOnly | d(not (volume.writable | d(false))) }} +{% endfor %} +{% endif %} +scheduler: + extraArgs: + bind-address: {{ kube_scheduler_bind_address }} + config: {{ kube_config_dir }}/kubescheduler-config.yaml +{% if kube_scheduler_feature_gates or kube_feature_gates %} + feature-gates: "{{ kube_scheduler_feature_gates | default(kube_feature_gates, true) | join(',') }}" +{% endif %} + profiling: "{{ kube_profiling }}" +{% if kube_kubeadm_scheduler_extra_args | length > 0 %} +{% for key in kube_kubeadm_scheduler_extra_args %} + {{ key }}: "{{ kube_kubeadm_scheduler_extra_args[key] }}" +{% endfor %} +{% endif %} +{% if tls_min_version is defined %} + tls-min-version: {{ tls_min_version }} +{% endif %} +{% if tls_cipher_suites is defined %} + tls-cipher-suites: {% for tls in tls_cipher_suites %}{{ tls }}{{ "," if not loop.last else "" }}{% endfor %} + +{% endif %} + extraVolumes: + - name: kubescheduler-config + hostPath: {{ kube_config_dir }}/kubescheduler-config.yaml + mountPath: {{ kube_config_dir }}/kubescheduler-config.yaml + readOnly: true +{% if scheduler_extra_volumes %} +{% for volume in scheduler_extra_volumes %} + - name: {{ volume.name }} + hostPath: {{ volume.hostPath }} + mountPath: {{ volume.mountPath }} + readOnly: {{ volume.readOnly | d(not (volume.writable | d(false))) }} +{% endfor %} +{% endif %} +--- +apiVersion: kubeproxy.config.k8s.io/v1alpha1 +kind: KubeProxyConfiguration +bindAddress: {{ kube_proxy_bind_address }} +clientConnection: + acceptContentTypes: {{ kube_proxy_client_accept_content_types }} + burst: {{ kube_proxy_client_burst }} + contentType: {{ kube_proxy_client_content_type }} + kubeconfig: {{ kube_proxy_client_kubeconfig }} + qps: {{ kube_proxy_client_qps }} +{% if kube_network_plugin is defined and kube_network_plugin not in ["kube-ovn"] %} +clusterCIDR: "{{ kube_pods_subnet }}{{ ',' + kube_pods_subnet_ipv6 if enable_dual_stack_networks else '' }}" +{% endif %} +configSyncPeriod: {{ kube_proxy_config_sync_period }} +conntrack: + maxPerCore: {{ kube_proxy_conntrack_max_per_core }} + min: {{ kube_proxy_conntrack_min }} + tcpCloseWaitTimeout: {{ kube_proxy_conntrack_tcp_close_wait_timeout }} + tcpEstablishedTimeout: {{ kube_proxy_conntrack_tcp_established_timeout }} +enableProfiling: {{ kube_proxy_enable_profiling }} +healthzBindAddress: {{ kube_proxy_healthz_bind_address }} +hostnameOverride: {{ kube_override_hostname }} +iptables: + masqueradeAll: {{ kube_proxy_masquerade_all }} + masqueradeBit: {{ kube_proxy_masquerade_bit }} + minSyncPeriod: {{ kube_proxy_min_sync_period }} + syncPeriod: {{ kube_proxy_sync_period }} +ipvs: + excludeCIDRs: {{ kube_proxy_exclude_cidrs }} + minSyncPeriod: {{ kube_proxy_min_sync_period }} + scheduler: {{ kube_proxy_scheduler }} + syncPeriod: {{ kube_proxy_sync_period }} + strictARP: {{ kube_proxy_strict_arp }} + tcpTimeout: {{ kube_proxy_tcp_timeout }} + tcpFinTimeout: {{ kube_proxy_tcp_fin_timeout }} + udpTimeout: {{ kube_proxy_udp_timeout }} +metricsBindAddress: {{ kube_proxy_metrics_bind_address }} +mode: {{ kube_proxy_mode }} +nodePortAddresses: {{ kube_proxy_nodeport_addresses }} +oomScoreAdj: {{ kube_proxy_oom_score_adj }} +portRange: {{ kube_proxy_port_range }} +{% if kube_proxy_feature_gates or kube_feature_gates %} +{% set feature_gates = ( kube_proxy_feature_gates | default(kube_feature_gates, true) ) %} +featureGates: +{% for feature in feature_gates %} + {{ feature | replace("=", ": ") }} +{% endfor %} +{% endif %} +{# DNS settings for kubelet #} +{% if enable_nodelocaldns %} +{% set kubelet_cluster_dns = [nodelocaldns_ip] %} +{% elif dns_mode in ['coredns'] %} +{% set kubelet_cluster_dns = [skydns_server] %} +{% elif dns_mode == 'coredns_dual' %} +{% set kubelet_cluster_dns = [skydns_server,skydns_server_secondary] %} +{% elif dns_mode == 'manual' %} +{% set kubelet_cluster_dns = [manual_dns_server] %} +{% else %} +{% set kubelet_cluster_dns = [] %} +{% endif %} +--- +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +clusterDNS: +{% for dns_address in kubelet_cluster_dns %} +- {{ dns_address }} +{% endfor %} +{% if kubelet_feature_gates or kube_feature_gates %} +{% set feature_gates = ( kubelet_feature_gates | default(kube_feature_gates, true) ) %} +featureGates: +{% for feature in feature_gates %} + {{ feature | replace("=", ": ") }} +{% endfor %} +{% endif %} diff --git a/kubespray/project/roles/kubernetes/control-plane/templates/kubeadm-controlplane.v1beta3.yaml.j2 b/kubespray/project/roles/kubernetes/control-plane/templates/kubeadm-controlplane.v1beta3.yaml.j2 new file mode 100644 index 0000000..c950d00 --- /dev/null +++ b/kubespray/project/roles/kubernetes/control-plane/templates/kubeadm-controlplane.v1beta3.yaml.j2 @@ -0,0 +1,32 @@ +apiVersion: kubeadm.k8s.io/v1beta3 +kind: JoinConfiguration +discovery: + bootstrapToken: +{% if kubeadm_config_api_fqdn is defined %} + apiServerEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }} +{% else %} + apiServerEndpoint: {{ kubeadm_discovery_address }} +{% endif %} + token: {{ kubeadm_token }} + unsafeSkipCAVerification: true + timeout: {{ discovery_timeout }} + tlsBootstrapToken: {{ kubeadm_token }} +controlPlane: + localAPIEndpoint: + advertiseAddress: {{ kube_apiserver_address }} + bindPort: {{ kube_apiserver_port }} + certificateKey: {{ kubeadm_certificate_key }} +nodeRegistration: + name: {{ kube_override_hostname | default(inventory_hostname) }} + criSocket: {{ cri_socket }} +{% if inventory_hostname in groups['kube_control_plane'] and inventory_hostname not in groups['kube_node'] %} + taints: + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane +{% else %} + taints: [] +{% endif %} +{% if kubeadm_patches is defined and kubeadm_patches.enabled %} +patches: + directory: {{ kubeadm_patches.dest_dir }} +{% endif %} diff --git a/kubespray/project/roles/kubernetes/control-plane/templates/kubescheduler-config.yaml.j2 b/kubespray/project/roles/kubernetes/control-plane/templates/kubescheduler-config.yaml.j2 new file mode 100644 index 0000000..a517fe8 --- /dev/null +++ b/kubespray/project/roles/kubernetes/control-plane/templates/kubescheduler-config.yaml.j2 @@ -0,0 +1,24 @@ +apiVersion: kubescheduler.config.k8s.io/v1 +kind: KubeSchedulerConfiguration +clientConnection: + kubeconfig: "{{ kube_config_dir }}/scheduler.conf" +{% for key in kube_scheduler_client_conn_extra_opts %} + {{ key }}: {{ kube_scheduler_client_conn_extra_opts[key] }} +{% endfor %} +{% if kube_scheduler_extenders %} +extenders: +{{ kube_scheduler_extenders | to_nice_yaml(indent=2, width=256) }} +{% endif %} +leaderElection: + leaseDuration: {{ kube_scheduler_leader_elect_lease_duration }} + renewDeadline: {{ kube_scheduler_leader_elect_renew_deadline }} +{% for key in kube_scheduler_leader_elect_extra_opts %} + {{ key }}: {{ kube_scheduler_leader_elect_extra_opts[key] }} +{% endfor %} +{% if kube_scheduler_profiles %} +profiles: +{{ kube_scheduler_profiles | to_nice_yaml(indent=2, width=256) }} +{% endif %} +{% for key in kube_scheduler_config_extra_opts %} +{{ key }}: {{ kube_scheduler_config_extra_opts[key] }} +{% endfor %} diff --git a/kubespray/project/roles/kubernetes/control-plane/templates/podnodeselector.yaml.j2 b/kubespray/project/roles/kubernetes/control-plane/templates/podnodeselector.yaml.j2 new file mode 100644 index 0000000..a44457f --- /dev/null +++ b/kubespray/project/roles/kubernetes/control-plane/templates/podnodeselector.yaml.j2 @@ -0,0 +1,2 @@ +podNodeSelectorPluginConfig: + clusterDefaultNodeSelector: {{ kube_apiserver_admission_plugins_podnodeselector_default_node_selector }} diff --git a/kubespray/project/roles/kubernetes/control-plane/templates/podsecurity.yaml.j2 b/kubespray/project/roles/kubernetes/control-plane/templates/podsecurity.yaml.j2 new file mode 100644 index 0000000..c973733 --- /dev/null +++ b/kubespray/project/roles/kubernetes/control-plane/templates/podsecurity.yaml.j2 @@ -0,0 +1,17 @@ +{% if kube_pod_security_use_default %} +apiVersion: pod-security.admission.config.k8s.io/v1 +kind: PodSecurityConfiguration +defaults: + enforce: "{{ kube_pod_security_default_enforce }}" + enforce-version: "{{ kube_pod_security_default_enforce_version }}" + audit: "{{ kube_pod_security_default_audit }}" + audit-version: "{{ kube_pod_security_default_audit_version }}" + warn: "{{ kube_pod_security_default_warn }}" + warn-version: "{{ kube_pod_security_default_warn_version }}" +exemptions: + usernames: {{ kube_pod_security_exemptions_usernames | to_json }} + runtimeClasses: {{ kube_pod_security_exemptions_runtime_class_names | to_json }} + namespaces: {{ kube_pod_security_exemptions_namespaces | to_json }} +{% else %} +# This file is intentinally left empty as kube_pod_security_use_default={{ kube_pod_security_use_default }} +{% endif %} diff --git a/kubespray/project/roles/kubernetes/control-plane/templates/secrets_encryption.yaml.j2 b/kubespray/project/roles/kubernetes/control-plane/templates/secrets_encryption.yaml.j2 new file mode 100644 index 0000000..3c521ff --- /dev/null +++ b/kubespray/project/roles/kubernetes/control-plane/templates/secrets_encryption.yaml.j2 @@ -0,0 +1,11 @@ +apiVersion: apiserver.config.k8s.io/v1 +kind: EncryptionConfiguration +resources: + - resources: +{{ kube_encryption_resources | to_nice_yaml | indent(4, True) }} + providers: + - {{ kube_encryption_algorithm }}: + keys: + - name: key + secret: {{ kube_encrypt_token | b64encode }} + - identity: {} diff --git a/kubespray/project/roles/kubernetes/control-plane/templates/webhook-authorization-config.yaml.j2 b/kubespray/project/roles/kubernetes/control-plane/templates/webhook-authorization-config.yaml.j2 new file mode 100644 index 0000000..b5b5530 --- /dev/null +++ b/kubespray/project/roles/kubernetes/control-plane/templates/webhook-authorization-config.yaml.j2 @@ -0,0 +1,18 @@ +# clusters refers to the remote service. +clusters: +- name: webhook-token-authz-cluster + cluster: + server: {{ kube_webhook_authorization_url }} + insecure-skip-tls-verify: {{ kube_webhook_authorization_url_skip_tls_verify }} + +# users refers to the API server's webhook configuration. +users: +- name: webhook-token-authz-user + +# kubeconfig files require a context. Provide one for the API server. +current-context: webhook-token-authz +contexts: +- context: + cluster: webhook-token-authz-cluster + user: webhook-token-authz-user + name: webhook-token-authz diff --git a/kubespray/project/roles/kubernetes/control-plane/templates/webhook-token-auth-config.yaml.j2 b/kubespray/project/roles/kubernetes/control-plane/templates/webhook-token-auth-config.yaml.j2 new file mode 100644 index 0000000..f152d11 --- /dev/null +++ b/kubespray/project/roles/kubernetes/control-plane/templates/webhook-token-auth-config.yaml.j2 @@ -0,0 +1,21 @@ +# clusters refers to the remote service. +clusters: +- name: webhook-token-auth-cluster + cluster: + server: {{ kube_webhook_token_auth_url }} + insecure-skip-tls-verify: {{ kube_webhook_token_auth_url_skip_tls_verify }} +{% if kube_webhook_token_auth_ca_data is defined %} + certificate-authority-data: {{ kube_webhook_token_auth_ca_data }} +{% endif %} + +# users refers to the API server's webhook configuration. +users: +- name: webhook-token-auth-user + +# kubeconfig files require a context. Provide one for the API server. +current-context: webhook-token-auth +contexts: +- context: + cluster: webhook-token-auth-cluster + user: webhook-token-auth-user + name: webhook-token-auth diff --git a/kubespray/project/roles/kubernetes/control-plane/vars/main.yaml b/kubespray/project/roles/kubernetes/control-plane/vars/main.yaml new file mode 100644 index 0000000..f888d6b --- /dev/null +++ b/kubespray/project/roles/kubernetes/control-plane/vars/main.yaml @@ -0,0 +1,3 @@ +--- +# list of admission plugins that needs to be configured +kube_apiserver_admission_plugins_needs_configuration: [EventRateLimit, PodSecurity] diff --git a/kubespray/project/roles/kubernetes/kubeadm/defaults/main.yml b/kubespray/project/roles/kubernetes/kubeadm/defaults/main.yml new file mode 100644 index 0000000..61b132e --- /dev/null +++ b/kubespray/project/roles/kubernetes/kubeadm/defaults/main.yml @@ -0,0 +1,12 @@ +--- +# discovery_timeout modifies the discovery timeout +# This value must be smaller than kubeadm_join_timeout +discovery_timeout: 60s +kubeadm_join_timeout: 120s + +# If non-empty, will use this string as identification instead of the actual hostname +kube_override_hostname: >- + {%- if cloud_provider is defined and cloud_provider in ['aws'] -%} + {%- else -%} + {{ inventory_hostname }} + {%- endif -%} diff --git a/kubespray/project/roles/kubernetes/kubeadm/handlers/main.yml b/kubespray/project/roles/kubernetes/kubeadm/handlers/main.yml new file mode 100644 index 0000000..9f6d431 --- /dev/null +++ b/kubespray/project/roles/kubernetes/kubeadm/handlers/main.yml @@ -0,0 +1,11 @@ +--- +- name: Kubeadm | reload systemd + systemd: + daemon_reload: true + listen: Kubeadm | restart kubelet + +- name: Kubeadm | reload kubelet + service: + name: kubelet + state: restarted + listen: Kubeadm | restart kubelet diff --git a/kubespray/project/roles/kubernetes/kubeadm/tasks/kubeadm_etcd_node.yml b/kubespray/project/roles/kubernetes/kubeadm/tasks/kubeadm_etcd_node.yml new file mode 100644 index 0000000..d39ea2b --- /dev/null +++ b/kubespray/project/roles/kubernetes/kubeadm/tasks/kubeadm_etcd_node.yml @@ -0,0 +1,62 @@ +--- +- name: Parse certificate key if not set + set_fact: + kubeadm_certificate_key: "{{ hostvars[groups['kube_control_plane'][0]]['kubeadm_certificate_key'] }}" + when: kubeadm_certificate_key is undefined + +- name: Create kubeadm cert controlplane config + template: + src: "kubeadm-client.conf.{{ kubeadmConfig_api_version }}.j2" + dest: "{{ kube_config_dir }}/kubeadm-cert-controlplane.conf" + mode: 0640 + vars: + kubeadm_cert_controlplane: true + +- name: Pull control plane certs down + shell: >- + {{ bin_dir }}/kubeadm join phase + control-plane-prepare download-certs + --config {{ kube_config_dir }}/kubeadm-cert-controlplane.conf + && + {{ bin_dir }}/kubeadm join phase + control-plane-prepare certs + --config {{ kube_config_dir }}/kubeadm-cert-controlplane.conf + args: + creates: "{{ kube_cert_dir }}/apiserver-etcd-client.key" + +- name: Delete unneeded certificates + file: + path: "{{ item }}" + state: absent + with_items: + - "{{ kube_cert_dir }}/apiserver.crt" + - "{{ kube_cert_dir }}/apiserver.key" + - "{{ kube_cert_dir }}/ca.key" + - "{{ kube_cert_dir }}/etcd/ca.key" + - "{{ kube_cert_dir }}/etcd/healthcheck-client.crt" + - "{{ kube_cert_dir }}/etcd/healthcheck-client.key" + - "{{ kube_cert_dir }}/etcd/peer.crt" + - "{{ kube_cert_dir }}/etcd/peer.key" + - "{{ kube_cert_dir }}/etcd/server.crt" + - "{{ kube_cert_dir }}/etcd/server.key" + - "{{ kube_cert_dir }}/front-proxy-ca.crt" + - "{{ kube_cert_dir }}/front-proxy-ca.key" + - "{{ kube_cert_dir }}/front-proxy-client.crt" + - "{{ kube_cert_dir }}/front-proxy-client.key" + - "{{ kube_cert_dir }}/sa.key" + - "{{ kube_cert_dir }}/sa.pub" + +- name: Calculate etcd cert serial + command: "openssl x509 -in {{ kube_cert_dir }}/apiserver-etcd-client.crt -noout -serial" + register: "etcd_client_cert_serial_result" + changed_when: false + when: + - inventory_hostname in groups['k8s_cluster'] | union(groups['calico_rr'] | default([])) | unique | sort + tags: + - network + +- name: Set etcd_client_cert_serial + set_fact: + etcd_client_cert_serial: "{{ etcd_client_cert_serial_result.stdout.split('=')[1] }}" + tags: + - network diff --git a/kubespray/project/roles/kubernetes/kubeadm/tasks/main.yml b/kubespray/project/roles/kubernetes/kubeadm/tasks/main.yml new file mode 100644 index 0000000..4a65dbb --- /dev/null +++ b/kubespray/project/roles/kubernetes/kubeadm/tasks/main.yml @@ -0,0 +1,204 @@ +--- +- name: Set kubeadm_discovery_address + set_fact: + # noqa: jinja[spacing] + kubeadm_discovery_address: >- + {%- if "127.0.0.1" in kube_apiserver_endpoint or "localhost" in kube_apiserver_endpoint -%} + {{ first_kube_control_plane_address }}:{{ kube_apiserver_port }} + {%- else -%} + {{ kube_apiserver_endpoint | replace("https://", "") }} + {%- endif %} + tags: + - facts + +- name: Check if kubelet.conf exists + stat: + path: "{{ kube_config_dir }}/kubelet.conf" + get_attributes: no + get_checksum: no + get_mime: no + register: kubelet_conf + +- name: Check if kubeadm CA cert is accessible + stat: + path: "{{ kube_cert_dir }}/ca.crt" + get_attributes: no + get_checksum: no + get_mime: no + register: kubeadm_ca_stat + delegate_to: "{{ groups['kube_control_plane'][0] }}" + run_once: true + +- name: Calculate kubeadm CA cert hash + shell: set -o pipefail && openssl x509 -pubkey -in {{ kube_cert_dir }}/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //' + args: + executable: /bin/bash + register: kubeadm_ca_hash + when: + - kubeadm_ca_stat.stat is defined + - kubeadm_ca_stat.stat.exists + delegate_to: "{{ groups['kube_control_plane'][0] }}" + run_once: true + changed_when: false + +- name: Create kubeadm token for joining nodes with 24h expiration (default) + command: "{{ bin_dir }}/kubeadm token create" + register: temp_token + delegate_to: "{{ groups['kube_control_plane'][0] }}" + when: kubeadm_token is not defined + changed_when: false + +- name: Set kubeadm_token to generated token + set_fact: + kubeadm_token: "{{ temp_token.stdout }}" + when: kubeadm_token is not defined + +- name: Set kubeadm api version to v1beta3 + set_fact: + kubeadmConfig_api_version: v1beta3 + +- name: Create kubeadm client config + template: + src: "kubeadm-client.conf.{{ kubeadmConfig_api_version }}.j2" + dest: "{{ kube_config_dir }}/kubeadm-client.conf" + backup: yes + mode: 0640 + when: not is_kube_master + +- name: Kubeadm | Create directory to store kubeadm patches + file: + path: "{{ kubeadm_patches.dest_dir }}" + state: directory + mode: 0640 + when: kubeadm_patches is defined and kubeadm_patches.enabled + +- name: Kubeadm | Copy kubeadm patches from inventory files + copy: + src: "{{ kubeadm_patches.source_dir }}/" + dest: "{{ kubeadm_patches.dest_dir }}" + owner: "root" + mode: 0644 + when: kubeadm_patches is defined and kubeadm_patches.enabled + +- name: Join to cluster if needed + environment: + PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}:/sbin" + when: not is_kube_master and (not kubelet_conf.stat.exists) + block: + + - name: Join to cluster + command: >- + timeout -k {{ kubeadm_join_timeout }} {{ kubeadm_join_timeout }} + {{ bin_dir }}/kubeadm join + --config {{ kube_config_dir }}/kubeadm-client.conf + --ignore-preflight-errors=DirAvailable--etc-kubernetes-manifests + --skip-phases={{ kubeadm_join_phases_skip | join(',') }} + register: kubeadm_join + changed_when: kubeadm_join is success + + rescue: + + - name: Join to cluster with ignores + command: >- + timeout -k {{ kubeadm_join_timeout }} {{ kubeadm_join_timeout }} + {{ bin_dir }}/kubeadm join + --config {{ kube_config_dir }}/kubeadm-client.conf + --ignore-preflight-errors=all + --skip-phases={{ kubeadm_join_phases_skip | join(',') }} + register: kubeadm_join + changed_when: kubeadm_join is success + + always: + + - name: Display kubeadm join stderr if any + when: kubeadm_join is failed + debug: + msg: | + Joined with warnings + {{ kubeadm_join.stderr_lines }} + +- name: Update server field in kubelet kubeconfig + lineinfile: + dest: "{{ kube_config_dir }}/kubelet.conf" + regexp: 'server:' + line: ' server: {{ kube_apiserver_endpoint }}' + backup: yes + when: + - kubeadm_config_api_fqdn is not defined + - not is_kube_master + - kubeadm_discovery_address != kube_apiserver_endpoint | replace("https://", "") + notify: Kubeadm | restart kubelet + +- name: Update server field in kubelet kubeconfig - external lb + lineinfile: + dest: "{{ kube_config_dir }}/kubelet.conf" + regexp: '^ server: https' + line: ' server: {{ kube_apiserver_endpoint }}' + backup: yes + when: + - not is_kube_master + - loadbalancer_apiserver is defined + notify: Kubeadm | restart kubelet + +# FIXME(mattymo): Need to point to localhost, otherwise masters will all point +# incorrectly to first master, creating SPoF. +- name: Update server field in kube-proxy kubeconfig + shell: >- + set -o pipefail && {{ kubectl }} get configmap kube-proxy -n kube-system -o yaml + | sed 's#server:.*#server: https://127.0.0.1:{{ kube_apiserver_port }}#g' + | {{ kubectl }} replace -f - + args: + executable: /bin/bash + run_once: true + delegate_to: "{{ groups['kube_control_plane'] | first }}" + delegate_facts: false + when: + - kubeadm_config_api_fqdn is not defined + - kubeadm_discovery_address != kube_apiserver_endpoint | replace("https://", "") + - kube_proxy_deployed + - loadbalancer_apiserver_localhost + tags: + - kube-proxy + +- name: Update server field in kube-proxy kubeconfig - external lb + shell: >- + set -o pipefail && {{ kubectl }} get configmap kube-proxy -n kube-system -o yaml + | sed 's#server:.*#server: {{kube_apiserver_endpoint}}#g' + | {{ kubectl }} replace -f - + args: + executable: /bin/bash + run_once: true + delegate_to: "{{ groups['kube_control_plane'] | first }}" + delegate_facts: false + when: + - kube_proxy_deployed + - loadbalancer_apiserver is defined + tags: + - kube-proxy + +- name: Set ca.crt file permission + file: + path: "{{ kube_cert_dir }}/ca.crt" + owner: root + group: root + mode: "0644" + +- name: Restart all kube-proxy pods to ensure that they load the new configmap + command: "{{ kubectl }} delete pod -n kube-system -l k8s-app=kube-proxy --force --grace-period=0" + run_once: true + delegate_to: "{{ groups['kube_control_plane'] | first }}" + delegate_facts: false + when: + - kubeadm_config_api_fqdn is not defined or loadbalancer_apiserver is defined + - kubeadm_discovery_address != kube_apiserver_endpoint | replace("https://", "") or loadbalancer_apiserver is defined + - kube_proxy_deployed + tags: + - kube-proxy + +- name: Extract etcd certs from control plane if using etcd kubeadm mode + include_tasks: kubeadm_etcd_node.yml + when: + - etcd_deployment_type == "kubeadm" + - inventory_hostname not in groups['kube_control_plane'] + - kube_network_plugin in ["calico", "flannel", "cilium"] or cilium_deploy_additionally | default(false) | bool + - kube_network_plugin != "calico" or calico_datastore == "etcd" diff --git a/kubespray/project/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta3.j2 b/kubespray/project/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta3.j2 new file mode 100644 index 0000000..5104ecf --- /dev/null +++ b/kubespray/project/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta3.j2 @@ -0,0 +1,39 @@ +--- +apiVersion: kubeadm.k8s.io/v1beta3 +kind: JoinConfiguration +discovery: + bootstrapToken: +{% if kubeadm_config_api_fqdn is defined %} + apiServerEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }} +{% else %} + apiServerEndpoint: {{ kubeadm_discovery_address }} +{% endif %} + token: {{ kubeadm_token }} +{% if kubeadm_ca_hash.stdout is defined %} + caCertHashes: + - sha256:{{ kubeadm_ca_hash.stdout }} +{% else %} + unsafeSkipCAVerification: true +{% endif %} + timeout: {{ discovery_timeout }} + tlsBootstrapToken: {{ kubeadm_token }} +caCertPath: {{ kube_cert_dir }}/ca.crt +{% if kubeadm_cert_controlplane is defined and kubeadm_cert_controlplane %} +controlPlane: + localAPIEndpoint: + advertiseAddress: {{ kube_apiserver_address }} + bindPort: {{ kube_apiserver_port }} + certificateKey: {{ kubeadm_certificate_key }} +{% endif %} +nodeRegistration: + name: '{{ kube_override_hostname }}' + criSocket: {{ cri_socket }} +{% if 'calico_rr' in group_names and 'kube_node' not in group_names %} + taints: + - effect: NoSchedule + key: node-role.kubernetes.io/calico-rr +{% endif %} +{% if kubeadm_patches is defined and kubeadm_patches.enabled %} +patches: + directory: {{ kubeadm_patches.dest_dir }} +{% endif %} diff --git a/kubespray/project/roles/kubernetes/node-label/tasks/main.yml b/kubespray/project/roles/kubernetes/node-label/tasks/main.yml new file mode 100644 index 0000000..cda700c --- /dev/null +++ b/kubespray/project/roles/kubernetes/node-label/tasks/main.yml @@ -0,0 +1,49 @@ +--- +- name: Kubernetes Apps | Wait for kube-apiserver + uri: + url: "{{ kube_apiserver_endpoint }}/healthz" + validate_certs: no + client_cert: "{{ kube_apiserver_client_cert }}" + client_key: "{{ kube_apiserver_client_key }}" + register: result + until: result.status == 200 + retries: 10 + delay: 6 + when: inventory_hostname == groups['kube_control_plane'][0] + +- name: Set role node label to empty list + set_fact: + role_node_labels: [] + +- name: Node label for nvidia GPU nodes + set_fact: + role_node_labels: "{{ role_node_labels + ['nvidia.com/gpu=true'] }}" + when: + - nvidia_gpu_nodes is defined + - nvidia_accelerator_enabled | bool + - inventory_hostname in nvidia_gpu_nodes + +- name: Set inventory node label to empty list + set_fact: + inventory_node_labels: [] + +- name: Populate inventory node label + set_fact: + inventory_node_labels: "{{ inventory_node_labels + ['%s=%s' | format(item.key, item.value)] }}" + loop: "{{ node_labels | d({}) | dict2items }}" + when: + - node_labels is defined + - node_labels is mapping + +- debug: # noqa name[missing] + var: role_node_labels +- debug: # noqa name[missing] + var: inventory_node_labels + +- name: Set label to node + command: >- + {{ kubectl }} label node {{ kube_override_hostname | default(inventory_hostname) }} {{ item }} --overwrite=true + loop: "{{ role_node_labels + inventory_node_labels }}" + delegate_to: "{{ groups['kube_control_plane'][0] }}" + changed_when: false +... diff --git a/kubespray/project/roles/kubernetes/node-taint/tasks/main.yml b/kubespray/project/roles/kubernetes/node-taint/tasks/main.yml new file mode 100644 index 0000000..0766dc4 --- /dev/null +++ b/kubespray/project/roles/kubernetes/node-taint/tasks/main.yml @@ -0,0 +1,35 @@ +--- +- name: Set role and inventory node taint to empty list + set_fact: + role_node_taints: [] + inventory_node_taints: [] + +- name: Node taint for nvidia GPU nodes + set_fact: + role_node_taints: "{{ role_node_taints + ['nvidia.com/gpu=:NoSchedule'] }}" + when: + - nvidia_gpu_nodes is defined + - nvidia_accelerator_enabled | bool + - inventory_hostname in nvidia_gpu_nodes + +- name: Populate inventory node taint + set_fact: + inventory_node_taints: "{{ inventory_node_taints + ['%s' | format(item)] }}" + loop: "{{ node_taints | d([]) }}" + when: + - node_taints is defined + - node_taints is not string + - node_taints is not mapping + - node_taints is iterable +- debug: # noqa name[missing] + var: role_node_taints +- debug: # noqa name[missing] + var: inventory_node_taints + +- name: Set taint to node + command: >- + {{ kubectl }} taint node {{ kube_override_hostname | default(inventory_hostname) }} {{ (role_node_taints + inventory_node_taints) | join(' ') }} --overwrite=true + delegate_to: "{{ groups['kube_control_plane'][0] }}" + changed_when: false + when: + - (role_node_taints + inventory_node_taints) | length > 0 diff --git a/kubespray/project/roles/kubernetes/node/defaults/main.yml b/kubespray/project/roles/kubernetes/node/defaults/main.yml new file mode 100644 index 0000000..b3f8fbf --- /dev/null +++ b/kubespray/project/roles/kubernetes/node/defaults/main.yml @@ -0,0 +1,261 @@ +--- +# advertised host IP for kubelet. This affects network plugin config. Take caution +kubelet_address: "{{ ip | default(fallback_ips[inventory_hostname]) }}{{ (',' + ip6) if enable_dual_stack_networks and ip6 is defined else '' }}" + +# bind address for kubelet. Set to 0.0.0.0 to listen on all interfaces +kubelet_bind_address: "{{ ip | default('0.0.0.0') }}" + +# resolv.conf to base dns config +kube_resolv_conf: "/etc/resolv.conf" + +# Set to empty to avoid cgroup creation +kubelet_enforce_node_allocatable: "\"\"" + +# Set runtime and kubelet cgroups when using systemd as cgroup driver (default) +kube_service_cgroups: "{% if kube_reserved %}{{ kube_reserved_cgroups_for_service_slice }}{% else %}system.slice{% endif %}" +kubelet_runtime_cgroups: "/{{ kube_service_cgroups }}/{{ container_manager }}.service" +kubelet_kubelet_cgroups: "/{{ kube_service_cgroups }}/kubelet.service" + +# Set runtime and kubelet cgroups when using cgroupfs as cgroup driver +kubelet_runtime_cgroups_cgroupfs: "/system.slice/{{ container_manager }}.service" +kubelet_kubelet_cgroups_cgroupfs: "/system.slice/kubelet.service" + +# Set systemd service hardening features +kubelet_systemd_hardening: false + +# List of secure IPs for kubelet +kubelet_secure_addresses: >- + {%- for host in groups['kube_control_plane'] -%} + {{ hostvars[host]['ip'] | default(fallback_ips[host]) }}{{ ' ' if not loop.last else '' }} + {%- endfor -%} + +# Reserve this space for kube resources +# Set to true to reserve resources for kube daemons +kube_reserved: false +kube_reserved_cgroups_for_service_slice: kube.slice +kube_reserved_cgroups: "/{{ kube_reserved_cgroups_for_service_slice }}" +kube_memory_reserved: 256Mi +kube_cpu_reserved: 100m +# kube_ephemeral_storage_reserved: 2Gi +# kube_pid_reserved: "1000" +# Reservation for master hosts +kube_master_memory_reserved: 512Mi +kube_master_cpu_reserved: 200m +# kube_master_ephemeral_storage_reserved: 2Gi +# kube_master_pid_reserved: "1000" + +# Set to true to reserve resources for system daemons +system_reserved: false +system_reserved_cgroups_for_service_slice: system.slice +system_reserved_cgroups: "/{{ system_reserved_cgroups_for_service_slice }}" +system_memory_reserved: 512Mi +system_cpu_reserved: 500m +# system_ephemeral_storage_reserved: 2Gi +# system_pid_reserved: "1000" +# Reservation for master hosts +system_master_memory_reserved: 256Mi +system_master_cpu_reserved: 250m +# system_master_ephemeral_storage_reserved: 2Gi +# system_master_pid_reserved: "1000" + +## Eviction Thresholds to avoid system OOMs +# https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/#eviction-thresholds +eviction_hard: {} +eviction_hard_control_plane: {} + +kubelet_status_update_frequency: 10s + +# kube-vip +kube_vip_version: v0.5.12 + +kube_vip_arp_enabled: false +kube_vip_interface: +kube_vip_services_interface: +kube_vip_cidr: 32 +kube_vip_controlplane_enabled: false +kube_vip_ddns_enabled: false +kube_vip_services_enabled: false +kube_vip_leader_election_enabled: "{{ kube_vip_arp_enabled }}" +kube_vip_bgp_enabled: false +kube_vip_bgp_routerid: +kube_vip_local_as: 65000 +kube_vip_bgp_peeraddress: +kube_vip_bgp_peerpass: +kube_vip_bgp_peeras: 65000 +kube_vip_bgppeers: +kube_vip_address: +kube_vip_enableServicesElection: false +kube_vip_lb_enable: false +kube_vip_lb_fwdmethod: local + +# Requests for load balancer app +loadbalancer_apiserver_memory_requests: 32M +loadbalancer_apiserver_cpu_requests: 25m + +loadbalancer_apiserver_keepalive_timeout: 5m +loadbalancer_apiserver_pod_name: "{% if loadbalancer_apiserver_type == 'nginx' %}nginx-proxy{% else %}haproxy{% endif %}" + +# Uncomment if you need to enable deprecated runtimes +# kube_api_runtime_config: +# - apps/v1beta1=true +# - apps/v1beta2=true +# - extensions/v1beta1/daemonsets=true +# - extensions/v1beta1/deployments=true +# - extensions/v1beta1/replicasets=true +# - extensions/v1beta1/networkpolicies=true + +# A port range to reserve for services with NodePort visibility. +# Inclusive at both ends of the range. +kube_apiserver_node_port_range: "30000-32767" + +# Configure the amount of pods able to run on single node +# default is equal to application default +kubelet_max_pods: 110 + +# Sets the maximum number of processes running per Pod +# Default value -1 = unlimited +kubelet_pod_pids_limit: -1 + +## Support parameters to be passed to kubelet via kubelet-config.yaml +kubelet_config_extra_args: {} + +## Parameters to be passed to kubelet via kubelet-config.yaml when cgroupfs is used as cgroup driver +kubelet_config_extra_args_cgroupfs: + systemCgroups: /system.slice + cgroupRoot: / + +## Support parameters to be passed to kubelet via kubelet-config.yaml only on nodes, not masters +kubelet_node_config_extra_args: {} + +# Maximum number of container log files that can be present for a container. +kubelet_logfiles_max_nr: 5 + +# Maximum size of the container log file before it is rotated +kubelet_logfiles_max_size: 10Mi + +## Support custom flags to be passed to kubelet +kubelet_custom_flags: [] + +## Support custom flags to be passed to kubelet only on nodes, not masters +kubelet_node_custom_flags: [] + +# If non-empty, will use this string as identification instead of the actual hostname +kube_override_hostname: >- + {%- if cloud_provider is defined and cloud_provider in ['aws'] -%} + {%- else -%} + {{ inventory_hostname }} + {%- endif -%} + +# The read-only port for the Kubelet to serve on with no authentication/authorization. +kube_read_only_port: 0 + +# Port for healthz for Kubelet +kubelet_healthz_port: 10248 + +# Bind address for healthz for Kubelet +kubelet_healthz_bind_address: 127.0.0.1 + +# sysctl_file_path to add sysctl conf to +sysctl_file_path: "/etc/sysctl.d/99-sysctl.conf" + +# For the openstack integration kubelet will need credentials to access +# openstack apis like nova and cinder. Per default this values will be +# read from the environment. +openstack_auth_url: "{{ lookup('env', 'OS_AUTH_URL') }}" +openstack_username: "{{ lookup('env', 'OS_USERNAME') }}" +openstack_password: "{{ lookup('env', 'OS_PASSWORD') }}" +openstack_region: "{{ lookup('env', 'OS_REGION_NAME') }}" +openstack_tenant_id: "{{ lookup('env', 'OS_TENANT_ID') | default(lookup('env', 'OS_PROJECT_ID') | default(lookup('env', 'OS_PROJECT_NAME'), true), true) }}" +openstack_tenant_name: "{{ lookup('env', 'OS_TENANT_NAME') }}" +openstack_domain_name: "{{ lookup('env', 'OS_USER_DOMAIN_NAME') }}" +openstack_domain_id: "{{ lookup('env', 'OS_USER_DOMAIN_ID') }}" + +# For the vsphere integration, kubelet will need credentials to access +# vsphere apis +# Documentation regarding these values can be found +# https://github.com/kubernetes/kubernetes/blob/master/pkg/cloudprovider/providers/vsphere/vsphere.go#L105 +vsphere_vcenter_ip: "{{ lookup('env', 'VSPHERE_VCENTER') }}" +vsphere_vcenter_port: "{{ lookup('env', 'VSPHERE_VCENTER_PORT') }}" +vsphere_user: "{{ lookup('env', 'VSPHERE_USER') }}" +vsphere_password: "{{ lookup('env', 'VSPHERE_PASSWORD') }}" +vsphere_datacenter: "{{ lookup('env', 'VSPHERE_DATACENTER') }}" +vsphere_datastore: "{{ lookup('env', 'VSPHERE_DATASTORE') }}" +vsphere_working_dir: "{{ lookup('env', 'VSPHERE_WORKING_DIR') }}" +vsphere_insecure: "{{ lookup('env', 'VSPHERE_INSECURE') }}" +vsphere_resource_pool: "{{ lookup('env', 'VSPHERE_RESOURCE_POOL') }}" + +vsphere_scsi_controller_type: pvscsi +# vsphere_public_network is name of the network the VMs are joined to +vsphere_public_network: "{{ lookup('env', 'VSPHERE_PUBLIC_NETWORK') | default('') }}" + +## When azure is used, you need to also set the following variables. +## see docs/azure.md for details on how to get these values +# azure_tenant_id: +# azure_subscription_id: +# azure_aad_client_id: +# azure_aad_client_secret: +# azure_resource_group: +# azure_location: +# azure_subnet_name: +# azure_security_group_name: +# azure_vnet_name: +# azure_route_table_name: +# supported values are 'standard' or 'vmss' +# azure_vmtype: standard +# Sku of Load Balancer and Public IP. Candidate values are: basic and standard. +azure_loadbalancer_sku: basic +# excludes master nodes from standard load balancer. +azure_exclude_master_from_standard_lb: true +# disables the outbound SNAT for public load balancer rules +azure_disable_outbound_snat: false +# use instance metadata service where possible +azure_use_instance_metadata: true +# use specific Azure API endpoints +azure_cloud: AzurePublicCloud + +## Support tls min version, Possible values: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13. +# tls_min_version: "" + +## Support tls cipher suites. +# tls_cipher_suites: +# - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA +# - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256 +# - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 +# - TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA +# - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 +# - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256 +# - TLS_ECDHE_ECDSA_WITH_RC4_128_SHA +# - TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA +# - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA +# - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 +# - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 +# - TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA +# - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 +# - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 +# - TLS_ECDHE_RSA_WITH_RC4_128_SHA +# - TLS_RSA_WITH_3DES_EDE_CBC_SHA +# - TLS_RSA_WITH_AES_128_CBC_SHA +# - TLS_RSA_WITH_AES_128_CBC_SHA256 +# - TLS_RSA_WITH_AES_128_GCM_SHA256 +# - TLS_RSA_WITH_AES_256_CBC_SHA +# - TLS_RSA_WITH_AES_256_GCM_SHA384 +# - TLS_RSA_WITH_RC4_128_SHA + +kube_proxy_ipvs_modules: + - ip_vs + - ip_vs_rr + - ip_vs_wrr + - ip_vs_sh + - ip_vs_wlc + - ip_vs_lc + +# Kubespray will use the first module of this list which it can successfully modprobe +conntrack_modules: + - nf_conntrack + - nf_conntrack_ipv4 + + +## Enable distributed tracing for kubelet +kubelet_tracing: false +kubelet_tracing_endpoint: 0.0.0.0:4317 +kubelet_tracing_sampling_rate_per_million: 100 \ No newline at end of file diff --git a/kubespray/project/roles/kubernetes/node/handlers/main.yml b/kubespray/project/roles/kubernetes/node/handlers/main.yml new file mode 100644 index 0000000..8195b7d --- /dev/null +++ b/kubespray/project/roles/kubernetes/node/handlers/main.yml @@ -0,0 +1,11 @@ +--- +- name: Kubelet | reload systemd + systemd: + daemon_reload: true + listen: Node | restart kubelet + +- name: Kubelet | restart kubelet + service: + name: kubelet + state: restarted + listen: Node | restart kubelet diff --git a/kubespray/project/roles/kubernetes/node/tasks/cloud-credentials/azure-credential-check.yml b/kubespray/project/roles/kubernetes/node/tasks/cloud-credentials/azure-credential-check.yml new file mode 100644 index 0000000..c5d6030 --- /dev/null +++ b/kubespray/project/roles/kubernetes/node/tasks/cloud-credentials/azure-credential-check.yml @@ -0,0 +1,82 @@ +--- +- name: Check azure_tenant_id value + fail: + msg: "azure_tenant_id is missing" + when: azure_tenant_id is not defined or not azure_tenant_id + +- name: Check azure_subscription_id value + fail: + msg: "azure_subscription_id is missing" + when: azure_subscription_id is not defined or not azure_subscription_id + +- name: Check azure_aad_client_id value + fail: + msg: "azure_aad_client_id is missing" + when: azure_aad_client_id is not defined or not azure_aad_client_id + +- name: Check azure_aad_client_secret value + fail: + msg: "azure_aad_client_secret is missing" + when: azure_aad_client_secret is not defined or not azure_aad_client_secret + +- name: Check azure_resource_group value + fail: + msg: "azure_resource_group is missing" + when: azure_resource_group is not defined or not azure_resource_group + +- name: Check azure_location value + fail: + msg: "azure_location is missing" + when: azure_location is not defined or not azure_location + +- name: Check azure_subnet_name value + fail: + msg: "azure_subnet_name is missing" + when: azure_subnet_name is not defined or not azure_subnet_name + +- name: Check azure_security_group_name value + fail: + msg: "azure_security_group_name is missing" + when: azure_security_group_name is not defined or not azure_security_group_name + +- name: Check azure_vnet_name value + fail: + msg: "azure_vnet_name is missing" + when: azure_vnet_name is not defined or not azure_vnet_name + +- name: Check azure_vnet_resource_group value + fail: + msg: "azure_vnet_resource_group is missing" + when: azure_vnet_resource_group is not defined or not azure_vnet_resource_group + +- name: Check azure_route_table_name value + fail: + msg: "azure_route_table_name is missing" + when: azure_route_table_name is not defined or not azure_route_table_name + +- name: Check azure_loadbalancer_sku value + fail: + msg: "azure_loadbalancer_sku has an invalid value '{{ azure_loadbalancer_sku }}'. Supported values are 'basic', 'standard'" + when: azure_loadbalancer_sku not in ["basic", "standard"] + +- name: "Check azure_exclude_master_from_standard_lb is a bool" + assert: + that: azure_exclude_master_from_standard_lb | type_debug == 'bool' + +- name: "Check azure_disable_outbound_snat is a bool" + assert: + that: azure_disable_outbound_snat | type_debug == 'bool' + +- name: "Check azure_use_instance_metadata is a bool" + assert: + that: azure_use_instance_metadata | type_debug == 'bool' + +- name: Check azure_vmtype value + fail: + msg: "azure_vmtype is missing. Supported values are 'standard' or 'vmss'" + when: azure_vmtype is not defined or not azure_vmtype + +- name: Check azure_cloud value + fail: + msg: "azure_cloud has an invalid value '{{ azure_cloud }}'. Supported values are 'AzureChinaCloud', 'AzureGermanCloud', 'AzurePublicCloud', 'AzureUSGovernmentCloud'." + when: azure_cloud not in ["AzureChinaCloud", "AzureGermanCloud", "AzurePublicCloud", "AzureUSGovernmentCloud"] diff --git a/kubespray/project/roles/kubernetes/node/tasks/cloud-credentials/openstack-credential-check.yml b/kubespray/project/roles/kubernetes/node/tasks/cloud-credentials/openstack-credential-check.yml new file mode 100644 index 0000000..7354d43 --- /dev/null +++ b/kubespray/project/roles/kubernetes/node/tasks/cloud-credentials/openstack-credential-check.yml @@ -0,0 +1,34 @@ +--- +- name: Check openstack_auth_url value + fail: + msg: "openstack_auth_url is missing" + when: openstack_auth_url is not defined or not openstack_auth_url + +- name: Check openstack_username value + fail: + msg: "openstack_username is missing" + when: openstack_username is not defined or not openstack_username + +- name: Check openstack_password value + fail: + msg: "openstack_password is missing" + when: openstack_password is not defined or not openstack_password + +- name: Check openstack_region value + fail: + msg: "openstack_region is missing" + when: openstack_region is not defined or not openstack_region + +- name: Check openstack_tenant_id value + fail: + msg: "one of openstack_tenant_id or openstack_trust_id must be specified" + when: + - openstack_tenant_id is not defined or not openstack_tenant_id + - openstack_trust_id is not defined + +- name: Check openstack_trust_id value + fail: + msg: "one of openstack_tenant_id or openstack_trust_id must be specified" + when: + - openstack_trust_id is not defined or not openstack_trust_id + - openstack_tenant_id is not defined diff --git a/kubespray/project/roles/kubernetes/node/tasks/cloud-credentials/vsphere-credential-check.yml b/kubespray/project/roles/kubernetes/node/tasks/cloud-credentials/vsphere-credential-check.yml new file mode 100644 index 0000000..b18583a --- /dev/null +++ b/kubespray/project/roles/kubernetes/node/tasks/cloud-credentials/vsphere-credential-check.yml @@ -0,0 +1,22 @@ +--- +- name: Check vsphere environment variables + fail: + msg: "{{ item.name }} is missing" + when: item.value is not defined or not item.value + with_items: + - name: vsphere_vcenter_ip + value: "{{ vsphere_vcenter_ip }}" + - name: vsphere_vcenter_port + value: "{{ vsphere_vcenter_port }}" + - name: vsphere_user + value: "{{ vsphere_user }}" + - name: vsphere_password + value: "{{ vsphere_password }}" + - name: vsphere_datacenter + value: "{{ vsphere_datacenter }}" + - name: vsphere_datastore + value: "{{ vsphere_datastore }}" + - name: vsphere_working_dir + value: "{{ vsphere_working_dir }}" + - name: vsphere_insecure + value: "{{ vsphere_insecure }}" diff --git a/kubespray/project/roles/kubernetes/node/tasks/facts.yml b/kubespray/project/roles/kubernetes/node/tasks/facts.yml new file mode 100644 index 0000000..0aaa11d --- /dev/null +++ b/kubespray/project/roles/kubernetes/node/tasks/facts.yml @@ -0,0 +1,62 @@ +--- +- name: Gather cgroups facts for docker + when: container_manager == 'docker' + block: + - name: Look up docker cgroup driver + shell: "set -o pipefail && docker info | grep 'Cgroup Driver' | awk -F': ' '{ print $2; }'" + args: + executable: /bin/bash + register: docker_cgroup_driver_result + changed_when: false + check_mode: no + + - name: Set kubelet_cgroup_driver_detected fact for docker + set_fact: + kubelet_cgroup_driver_detected: "{{ docker_cgroup_driver_result.stdout }}" + +- name: Gather cgroups facts for crio + when: container_manager == 'crio' + block: + - name: Look up crio cgroup driver + shell: "set -o pipefail && {{ bin_dir }}/{{ crio_status_command }} info | grep 'cgroup driver' | awk -F': ' '{ print $2; }'" + args: + executable: /bin/bash + register: crio_cgroup_driver_result + changed_when: false + + - name: Set kubelet_cgroup_driver_detected fact for crio + set_fact: + kubelet_cgroup_driver_detected: "{{ crio_cgroup_driver_result.stdout }}" + +- name: Set kubelet_cgroup_driver_detected fact for containerd + when: container_manager == 'containerd' + set_fact: + kubelet_cgroup_driver_detected: >- + {%- if containerd_use_systemd_cgroup -%}systemd{%- else -%}cgroupfs{%- endif -%} + +- name: Set kubelet_cgroup_driver + set_fact: + kubelet_cgroup_driver: "{{ kubelet_cgroup_driver_detected }}" + when: kubelet_cgroup_driver is undefined + +- name: Set kubelet_cgroups options when cgroupfs is used + set_fact: + kubelet_runtime_cgroups: "{{ kubelet_runtime_cgroups_cgroupfs }}" + kubelet_kubelet_cgroups: "{{ kubelet_kubelet_cgroups_cgroupfs }}" + when: kubelet_cgroup_driver == 'cgroupfs' + +- name: Set kubelet_config_extra_args options when cgroupfs is used + set_fact: + kubelet_config_extra_args: "{{ kubelet_config_extra_args | combine(kubelet_config_extra_args_cgroupfs) }}" + when: kubelet_cgroup_driver == 'cgroupfs' + +- name: Os specific vars + include_vars: "{{ item }}" + with_first_found: + - files: + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower | replace('/', '_') }}.yml" + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_release }}.yml" + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower | replace('/', '_') }}.yml" + - "{{ ansible_distribution | lower }}.yml" + - "{{ ansible_os_family | lower }}.yml" + skip: true diff --git a/kubespray/project/roles/kubernetes/node/tasks/install.yml b/kubespray/project/roles/kubernetes/node/tasks/install.yml new file mode 100644 index 0000000..fb1e8ad --- /dev/null +++ b/kubespray/project/roles/kubernetes/node/tasks/install.yml @@ -0,0 +1,22 @@ +--- +- name: Install | Copy kubeadm binary from download dir + copy: + src: "{{ downloads.kubeadm.dest }}" + dest: "{{ bin_dir }}/kubeadm" + mode: 0755 + remote_src: true + tags: + - kubeadm + when: + - not inventory_hostname in groups['kube_control_plane'] + +- name: Install | Copy kubelet binary from download dir + copy: + src: "{{ downloads.kubelet.dest }}" + dest: "{{ bin_dir }}/kubelet" + mode: 0755 + remote_src: true + tags: + - kubelet + - upgrade + notify: Node | restart kubelet diff --git a/kubespray/project/roles/kubernetes/node/tasks/kubelet.yml b/kubespray/project/roles/kubernetes/node/tasks/kubelet.yml new file mode 100644 index 0000000..d8ff9e2 --- /dev/null +++ b/kubespray/project/roles/kubernetes/node/tasks/kubelet.yml @@ -0,0 +1,55 @@ +--- +- name: Set kubelet api version to v1beta1 + set_fact: + kubeletConfig_api_version: v1beta1 + tags: + - kubelet + - kubeadm + +- name: Write kubelet environment config file (kubeadm) + template: + src: "kubelet.env.{{ kubeletConfig_api_version }}.j2" + dest: "{{ kube_config_dir }}/kubelet.env" + setype: "{{ (preinstall_selinux_state != 'disabled') | ternary('etc_t', omit) }}" + backup: yes + mode: 0600 + notify: Node | restart kubelet + tags: + - kubelet + - kubeadm + +- name: Write kubelet config file + template: + src: "kubelet-config.{{ kubeletConfig_api_version }}.yaml.j2" + dest: "{{ kube_config_dir }}/kubelet-config.yaml" + mode: 0600 + notify: Kubelet | restart kubelet + tags: + - kubelet + - kubeadm + +- name: Write kubelet systemd init file + template: + src: "kubelet.service.j2" + dest: "/etc/systemd/system/kubelet.service" + backup: "yes" + mode: 0600 + validate: "sh -c '[ -f /usr/bin/systemd/system/factory-reset.target ] || exit 0 && systemd-analyze verify %s:kubelet.service'" + # FIXME: check that systemd version >= 250 (factory-reset.target was introduced in that release) + # Remove once we drop support for systemd < 250 + notify: Node | restart kubelet + tags: + - kubelet + - kubeadm + +- name: Flush_handlers and reload-systemd + meta: flush_handlers + +- name: Enable kubelet + service: + name: kubelet + enabled: yes + state: started + tags: + - kubelet + notify: Kubelet | restart kubelet diff --git a/kubespray/project/roles/kubernetes/node/tasks/loadbalancer/haproxy.yml b/kubespray/project/roles/kubernetes/node/tasks/loadbalancer/haproxy.yml new file mode 100644 index 0000000..7e5cfce --- /dev/null +++ b/kubespray/project/roles/kubernetes/node/tasks/loadbalancer/haproxy.yml @@ -0,0 +1,34 @@ +--- +- name: Haproxy | Cleanup potentially deployed nginx-proxy + file: + path: "{{ kube_manifest_dir }}/nginx-proxy.yml" + state: absent + +- name: Haproxy | Make haproxy directory + file: + path: "{{ haproxy_config_dir }}" + state: directory + mode: 0755 + owner: root + +- name: Haproxy | Write haproxy configuration + template: + src: "loadbalancer/haproxy.cfg.j2" + dest: "{{ haproxy_config_dir }}/haproxy.cfg" + owner: root + mode: 0755 + backup: yes + +- name: Haproxy | Get checksum from config + stat: + path: "{{ haproxy_config_dir }}/haproxy.cfg" + get_attributes: no + get_checksum: yes + get_mime: no + register: haproxy_stat + +- name: Haproxy | Write static pod + template: + src: manifests/haproxy.manifest.j2 + dest: "{{ kube_manifest_dir }}/haproxy.yml" + mode: 0640 diff --git a/kubespray/project/roles/kubernetes/node/tasks/loadbalancer/kube-vip.yml b/kubespray/project/roles/kubernetes/node/tasks/loadbalancer/kube-vip.yml new file mode 100644 index 0000000..f7b04a6 --- /dev/null +++ b/kubespray/project/roles/kubernetes/node/tasks/loadbalancer/kube-vip.yml @@ -0,0 +1,13 @@ +--- +- name: Kube-vip | Check cluster settings for kube-vip + fail: + msg: "kube-vip require kube_proxy_strict_arp = true, see https://github.com/kube-vip/kube-vip/blob/main/docs/kubernetes/arp/index.md" + when: + - kube_proxy_mode == 'ipvs' and not kube_proxy_strict_arp + - kube_vip_arp_enabled + +- name: Kube-vip | Write static pod + template: + src: manifests/kube-vip.manifest.j2 + dest: "{{ kube_manifest_dir }}/kube-vip.yml" + mode: 0640 diff --git a/kubespray/project/roles/kubernetes/node/tasks/loadbalancer/nginx-proxy.yml b/kubespray/project/roles/kubernetes/node/tasks/loadbalancer/nginx-proxy.yml new file mode 100644 index 0000000..5b82ff6 --- /dev/null +++ b/kubespray/project/roles/kubernetes/node/tasks/loadbalancer/nginx-proxy.yml @@ -0,0 +1,34 @@ +--- +- name: Haproxy | Cleanup potentially deployed haproxy + file: + path: "{{ kube_manifest_dir }}/haproxy.yml" + state: absent + +- name: Nginx-proxy | Make nginx directory + file: + path: "{{ nginx_config_dir }}" + state: directory + mode: 0700 + owner: root + +- name: Nginx-proxy | Write nginx-proxy configuration + template: + src: "loadbalancer/nginx.conf.j2" + dest: "{{ nginx_config_dir }}/nginx.conf" + owner: root + mode: 0755 + backup: yes + +- name: Nginx-proxy | Get checksum from config + stat: + path: "{{ nginx_config_dir }}/nginx.conf" + get_attributes: no + get_checksum: yes + get_mime: no + register: nginx_stat + +- name: Nginx-proxy | Write static pod + template: + src: manifests/nginx-proxy.manifest.j2 + dest: "{{ kube_manifest_dir }}/nginx-proxy.yml" + mode: 0640 diff --git a/kubespray/project/roles/kubernetes/node/tasks/main.yml b/kubespray/project/roles/kubernetes/node/tasks/main.yml new file mode 100644 index 0000000..9e15b16 --- /dev/null +++ b/kubespray/project/roles/kubernetes/node/tasks/main.yml @@ -0,0 +1,191 @@ +--- +- name: Fetch facts + import_tasks: facts.yml + tags: + - facts + - kubelet + +- name: Pre-upgrade kubelet + import_tasks: pre_upgrade.yml + tags: + - kubelet + +- name: Ensure /var/lib/cni exists + file: + path: /var/lib/cni + state: directory + mode: 0755 + +- name: Install kubelet binary + import_tasks: install.yml + tags: + - kubelet + +- name: Install kube-vip + import_tasks: loadbalancer/kube-vip.yml + when: + - is_kube_master + - kube_vip_enabled + tags: + - kube-vip + +- name: Install nginx-proxy + import_tasks: loadbalancer/nginx-proxy.yml + when: + - not is_kube_master or kube_apiserver_bind_address != '0.0.0.0' + - loadbalancer_apiserver_localhost + - loadbalancer_apiserver_type == 'nginx' + tags: + - nginx + +- name: Install haproxy + import_tasks: loadbalancer/haproxy.yml + when: + - not is_kube_master or kube_apiserver_bind_address != '0.0.0.0' + - loadbalancer_apiserver_localhost + - loadbalancer_apiserver_type == 'haproxy' + tags: + - haproxy + +- name: Ensure nodePort range is reserved + ansible.posix.sysctl: + name: net.ipv4.ip_local_reserved_ports + value: "{{ kube_apiserver_node_port_range }}" + sysctl_set: yes + sysctl_file: "{{ sysctl_file_path }}" + state: present + reload: yes + when: kube_apiserver_node_port_range is defined + tags: + - kube-proxy + +- name: Verify if br_netfilter module exists + command: "modinfo br_netfilter" + environment: + PATH: "{{ ansible_env.PATH }}:/sbin" # Make sure we can workaround RH's conservative path management + register: modinfo_br_netfilter + failed_when: modinfo_br_netfilter.rc not in [0, 1] + changed_when: false + check_mode: no + +# TODO: Remove once upstream issue is fixed +# https://github.com/ansible-collections/community.general/issues/7717 +- name: Verify br_netfilter module path exists + file: + path: "{{ item }}" + state: directory + mode: 0755 + loop: + - /etc/modules-load.d + - /etc/modprobe.d + +- name: Enable br_netfilter module + community.general.modprobe: + name: br_netfilter + state: present + when: modinfo_br_netfilter.rc == 0 + +- name: Persist br_netfilter module + copy: + dest: /etc/modules-load.d/kubespray-br_netfilter.conf + content: br_netfilter + mode: 0644 + when: modinfo_br_netfilter.rc == 0 + +# kube-proxy needs net.bridge.bridge-nf-call-iptables enabled when found if br_netfilter is not a module +- name: Check if bridge-nf-call-iptables key exists + command: "sysctl net.bridge.bridge-nf-call-iptables" + failed_when: false + changed_when: false + check_mode: no + register: sysctl_bridge_nf_call_iptables + +- name: Enable bridge-nf-call tables + ansible.posix.sysctl: + name: "{{ item }}" + state: present + sysctl_file: "{{ sysctl_file_path }}" + value: "1" + reload: yes + when: sysctl_bridge_nf_call_iptables.rc == 0 + with_items: + - net.bridge.bridge-nf-call-iptables + - net.bridge.bridge-nf-call-arptables + - net.bridge.bridge-nf-call-ip6tables + +- name: Modprobe Kernel Module for IPVS + community.general.modprobe: + name: "{{ item }}" + state: present + persistent: present + loop: "{{ kube_proxy_ipvs_modules }}" + when: kube_proxy_mode == 'ipvs' + tags: + - kube-proxy + +- name: Modprobe conntrack module + community.general.modprobe: + name: "{{ item }}" + state: present + persistent: present + register: modprobe_conntrack_module + ignore_errors: true # noqa ignore-errors + loop: "{{ conntrack_modules }}" + when: + - kube_proxy_mode == 'ipvs' + - "(modprobe_conntrack_module|default({'rc': 1})).rc != 0" # loop until first success + tags: + - kube-proxy + +- name: Check cloud provider credentials + include_tasks: "cloud-credentials/{{ cloud_provider }}-credential-check.yml" + when: + - cloud_provider is defined + - cloud_provider in [ 'openstack', 'azure', 'vsphere' ] + tags: + - cloud-provider + - facts + +- name: Test if openstack_cacert is a base64 string + set_fact: + openstack_cacert_is_base64: "{% if openstack_cacert is search('^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{3}= | [A-Za-z0-9+/]{2}==)?$') %}true{% else %}false{% endif %}" + when: + - cloud_provider is defined + - cloud_provider == 'openstack' + - openstack_cacert is defined + - openstack_cacert | length > 0 + + +- name: Write cacert file + copy: + src: "{{ openstack_cacert if not openstack_cacert_is_base64 else omit }}" + content: "{{ openstack_cacert | b64decode if openstack_cacert_is_base64 else omit }}" + dest: "{{ kube_config_dir }}/openstack-cacert.pem" + group: "{{ kube_cert_group }}" + mode: 0640 + when: + - cloud_provider is defined + - cloud_provider == 'openstack' + - openstack_cacert is defined + - openstack_cacert | length > 0 + tags: + - cloud-provider + +- name: Write cloud-config + template: + src: "cloud-configs/{{ cloud_provider }}-cloud-config.j2" + dest: "{{ kube_config_dir }}/cloud_config" + group: "{{ kube_cert_group }}" + mode: 0640 + when: + - cloud_provider is defined + - cloud_provider in [ 'openstack', 'azure', 'vsphere', 'aws', 'gce' ] + notify: Node | restart kubelet + tags: + - cloud-provider + +- name: Install kubelet + import_tasks: kubelet.yml + tags: + - kubelet + - kubeadm diff --git a/kubespray/project/roles/kubernetes/node/tasks/pre_upgrade.yml b/kubespray/project/roles/kubernetes/node/tasks/pre_upgrade.yml new file mode 100644 index 0000000..d9c2d07 --- /dev/null +++ b/kubespray/project/roles/kubernetes/node/tasks/pre_upgrade.yml @@ -0,0 +1,48 @@ +--- +- name: "Pre-upgrade | check if kubelet container exists" + shell: >- + set -o pipefail && + {% if container_manager in ['crio', 'docker'] %} + {{ docker_bin_dir }}/docker ps -af name=kubelet | grep kubelet + {% elif container_manager == 'containerd' %} + {{ bin_dir }}/crictl ps --all --name kubelet | grep kubelet + {% endif %} + args: + executable: /bin/bash + failed_when: false + changed_when: false + check_mode: no + register: kubelet_container_check + +- name: "Pre-upgrade | copy /var/lib/cni from kubelet" + command: >- + {% if container_manager in ['crio', 'docker'] %} + docker cp kubelet:/var/lib/cni /var/lib/cni + {% elif container_manager == 'containerd' %} + ctr run --rm --mount type=bind,src=/var/lib/cni,dst=/cnilibdir,options=rbind:rw kubelet kubelet-tmp sh -c 'cp /var/lib/cni/* /cnilibdir/' + {% endif %} + args: + creates: "/var/lib/cni" + failed_when: false + when: kubelet_container_check.rc == 0 + +- name: "Pre-upgrade | ensure kubelet container service is stopped if using host deployment" + service: + name: kubelet + state: stopped + when: kubelet_container_check.rc == 0 + +- name: "Pre-upgrade | ensure kubelet container is removed if using host deployment" + shell: >- + {% if container_manager in ['crio', 'docker'] %} + {{ docker_bin_dir }}/docker rm -fv kubelet + {% elif container_manager == 'containerd' %} + {{ bin_dir }}/crictl stop kubelet && {{ bin_dir }}/crictl rm kubelet + {% endif %} + failed_when: false + changed_when: false + register: remove_kubelet_container + retries: 4 + until: remove_kubelet_container.rc == 0 + delay: 5 + when: kubelet_container_check.rc == 0 diff --git a/kubespray/project/roles/kubernetes/node/templates/cloud-configs/aws-cloud-config.j2 b/kubespray/project/roles/kubernetes/node/templates/cloud-configs/aws-cloud-config.j2 new file mode 100644 index 0000000..f6d0c3d --- /dev/null +++ b/kubespray/project/roles/kubernetes/node/templates/cloud-configs/aws-cloud-config.j2 @@ -0,0 +1,11 @@ +[Global] +zone={{ aws_zone|default("") }} +vpc={{ aws_vpc|default("") }} +subnetId={{ aws_subnet_id|default("") }} +routeTableId={{ aws_route_table_id|default("") }} +roleArn={{ aws_role_arn|default("") }} +kubernetesClusterTag={{ aws_kubernetes_cluster_tag|default("") }} +kubernetesClusterId={{ aws_kubernetes_cluster_id|default("") }} +disableSecurityGroupIngress={{ "true" if aws_disable_security_group_ingress|default(False) else "false" }} +disableStrictZoneCheck={{ "true" if aws_disable_strict_zone_check|default(False) else "false" }} +elbSecurityGroup={{ aws_elb_security_group|default("") }} diff --git a/kubespray/project/roles/kubernetes/node/templates/cloud-configs/azure-cloud-config.j2 b/kubespray/project/roles/kubernetes/node/templates/cloud-configs/azure-cloud-config.j2 new file mode 100644 index 0000000..2b1c101 --- /dev/null +++ b/kubespray/project/roles/kubernetes/node/templates/cloud-configs/azure-cloud-config.j2 @@ -0,0 +1,26 @@ +{ + "cloud": "{{ azure_cloud }}", + "tenantId": "{{ azure_tenant_id }}", + "subscriptionId": "{{ azure_subscription_id }}", + "aadClientId": "{{ azure_aad_client_id }}", + "aadClientSecret": "{{ azure_aad_client_secret }}", + "resourceGroup": "{{ azure_resource_group }}", + "location": "{{ azure_location }}", + "subnetName": "{{ azure_subnet_name }}", + "securityGroupName": "{{ azure_security_group_name }}", + "securityGroupResourceGroup": "{{ azure_security_group_resource_group | default(azure_vnet_resource_group) }}", + "vnetName": "{{ azure_vnet_name }}", + "vnetResourceGroup": "{{ azure_vnet_resource_group }}", + "routeTableName": "{{ azure_route_table_name }}", + "routeTableResourceGroup": "{{ azure_route_table_resource_group | default(azure_vnet_resource_group) }}", + "vmType": "{{ azure_vmtype }}", +{% if azure_primary_availability_set_name is defined %} + "primaryAvailabilitySetName": "{{ azure_primary_availability_set_name }}", +{%endif%} + "useInstanceMetadata": {{azure_use_instance_metadata | lower }}, +{% if azure_loadbalancer_sku == "standard" %} + "excludeMasterFromStandardLB": {{ azure_exclude_master_from_standard_lb | lower }}, + "disableOutboundSNAT": {{ azure_disable_outbound_snat | lower }}, +{% endif%} + "loadBalancerSku": "{{ azure_loadbalancer_sku }}" +} diff --git a/kubespray/project/roles/kubernetes/node/templates/cloud-configs/gce-cloud-config.j2 b/kubespray/project/roles/kubernetes/node/templates/cloud-configs/gce-cloud-config.j2 new file mode 100644 index 0000000..3995126 --- /dev/null +++ b/kubespray/project/roles/kubernetes/node/templates/cloud-configs/gce-cloud-config.j2 @@ -0,0 +1,2 @@ +[global] +node-tags = {{ gce_node_tags }} diff --git a/kubespray/project/roles/kubernetes/node/templates/cloud-configs/openstack-cloud-config.j2 b/kubespray/project/roles/kubernetes/node/templates/cloud-configs/openstack-cloud-config.j2 new file mode 100644 index 0000000..b1f8e0a --- /dev/null +++ b/kubespray/project/roles/kubernetes/node/templates/cloud-configs/openstack-cloud-config.j2 @@ -0,0 +1,54 @@ +[Global] +auth-url="{{ openstack_auth_url }}" +username="{{ openstack_username }}" +password="{{ openstack_password }}" +region="{{ openstack_region }}" +{% if openstack_trust_id is defined and openstack_trust_id != "" %} +trust-id="{{ openstack_trust_id }}" +{% else %} +tenant-id="{{ openstack_tenant_id }}" +{% endif %} +{% if openstack_tenant_name is defined and openstack_tenant_name != "" %} +tenant-name="{{ openstack_tenant_name }}" +{% endif %} +{% if openstack_domain_name is defined and openstack_domain_name != "" %} +domain-name="{{ openstack_domain_name }}" +{% elif openstack_domain_id is defined and openstack_domain_id != "" %} +domain-id ="{{ openstack_domain_id }}" +{% endif %} +{% if openstack_cacert is defined and openstack_cacert != "" %} +ca-file="{{ kube_config_dir }}/openstack-cacert.pem" +{% endif %} + +[BlockStorage] +{% if openstack_blockstorage_version is defined %} +bs-version={{ openstack_blockstorage_version }} +{% endif %} +{% if openstack_blockstorage_ignore_volume_az is defined and openstack_blockstorage_ignore_volume_az|bool %} +ignore-volume-az={{ openstack_blockstorage_ignore_volume_az }} +{% endif %} +{% if node_volume_attach_limit is defined and node_volume_attach_limit != "" %} +node-volume-attach-limit="{{ node_volume_attach_limit }}" +{% endif %} + +{% if openstack_lbaas_enabled and openstack_lbaas_subnet_id is defined %} +[LoadBalancer] +subnet-id={{ openstack_lbaas_subnet_id }} +{% if openstack_lbaas_floating_network_id is defined %} +floating-network-id={{ openstack_lbaas_floating_network_id }} +{% endif %} +{% if openstack_lbaas_use_octavia is defined %} +use-octavia={{ openstack_lbaas_use_octavia }} +{% endif %} +{% if openstack_lbaas_method is defined %} +lb-method={{ openstack_lbaas_method }} +{% endif %} +{% if openstack_lbaas_provider is defined %} +lb-provider={{ openstack_lbaas_provider }} +{% endif %} + +create-monitor={{ openstack_lbaas_create_monitor }} +monitor-delay={{ openstack_lbaas_monitor_delay }} +monitor-timeout={{ openstack_lbaas_monitor_timeout }} +monitor-max-retries={{ openstack_lbaas_monitor_max_retries }} +{% endif %} diff --git a/kubespray/project/roles/kubernetes/node/templates/cloud-configs/vsphere-cloud-config.j2 b/kubespray/project/roles/kubernetes/node/templates/cloud-configs/vsphere-cloud-config.j2 new file mode 100644 index 0000000..2cda7f6 --- /dev/null +++ b/kubespray/project/roles/kubernetes/node/templates/cloud-configs/vsphere-cloud-config.j2 @@ -0,0 +1,36 @@ +[Global] +user = "{{ vsphere_user }}" +password = "{{ vsphere_password }}" +port = {{ vsphere_vcenter_port }} +insecure-flag = {{ vsphere_insecure }} + +datacenters = "{{ vsphere_datacenter }}" + +[VirtualCenter "{{ vsphere_vcenter_ip }}"] + + +[Workspace] +server = "{{ vsphere_vcenter_ip }}" +datacenter = "{{ vsphere_datacenter }}" +folder = "{{ vsphere_working_dir }}" +default-datastore = "{{ vsphere_datastore }}" +{% if vsphere_resource_pool is defined and vsphere_resource_pool != "" %} +resourcepool-path = "{{ vsphere_resource_pool }}" +{% endif %} + + +[Disk] +scsicontrollertype = {{ vsphere_scsi_controller_type }} + +{% if vsphere_public_network is defined and vsphere_public_network != "" %} +[Network] +public-network = {{ vsphere_public_network }} +{% endif %} + +[Labels] +{% if vsphere_zone_category is defined and vsphere_zone_category != "" %} +zone = {{ vsphere_zone_category }} +{% endif %} +{% if vsphere_region_category is defined and vsphere_region_category != "" %} +region = {{ vsphere_region_category }} +{% endif %} diff --git a/kubespray/project/roles/kubernetes/node/templates/http-proxy.conf.j2 b/kubespray/project/roles/kubernetes/node/templates/http-proxy.conf.j2 new file mode 100644 index 0000000..e790477 --- /dev/null +++ b/kubespray/project/roles/kubernetes/node/templates/http-proxy.conf.j2 @@ -0,0 +1,2 @@ +[Service] +Environment={% if http_proxy %}"HTTP_PROXY={{ http_proxy }}"{% endif %} {% if https_proxy %}"HTTPS_PROXY={{ https_proxy }}"{% endif %} {% if no_proxy %}"NO_PROXY={{ no_proxy }}"{% endif %} diff --git a/kubespray/project/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2 b/kubespray/project/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2 new file mode 100644 index 0000000..ba90fc9 --- /dev/null +++ b/kubespray/project/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2 @@ -0,0 +1,173 @@ +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +nodeStatusUpdateFrequency: "{{ kubelet_status_update_frequency }}" +failSwapOn: {{ kubelet_fail_swap_on }} +authentication: + anonymous: + enabled: false + webhook: + enabled: {{ kubelet_authentication_token_webhook }} + x509: + clientCAFile: {{ kube_cert_dir }}/ca.crt +authorization: +{% if kubelet_authorization_mode_webhook %} + mode: Webhook +{% else %} + mode: AlwaysAllow +{% endif %} +{% if kubelet_enforce_node_allocatable is defined and kubelet_enforce_node_allocatable != "\"\"" %} +{% set kubelet_enforce_node_allocatable_list = kubelet_enforce_node_allocatable.split(",") %} +enforceNodeAllocatable: +{% for item in kubelet_enforce_node_allocatable_list %} +- {{ item }} +{% endfor %} +{% endif %} +staticPodPath: {{ kube_manifest_dir }} +cgroupDriver: {{ kubelet_cgroup_driver | default('systemd') }} +containerLogMaxFiles: {{ kubelet_logfiles_max_nr }} +containerLogMaxSize: {{ kubelet_logfiles_max_size }} +maxPods: {{ kubelet_max_pods }} +podPidsLimit: {{ kubelet_pod_pids_limit }} +address: {{ kubelet_bind_address }} +readOnlyPort: {{ kube_read_only_port }} +healthzPort: {{ kubelet_healthz_port }} +healthzBindAddress: {{ kubelet_healthz_bind_address }} +kubeletCgroups: {{ kubelet_kubelet_cgroups }} +clusterDomain: {{ dns_domain }} +{% if kubelet_protect_kernel_defaults | bool %} +protectKernelDefaults: true +{% endif %} +{% if kubelet_rotate_certificates | bool %} +rotateCertificates: true +{% endif %} +{% if kubelet_rotate_server_certificates | bool %} +serverTLSBootstrap: true +{% endif %} +{# DNS settings for kubelet #} +{% if enable_nodelocaldns %} +{% set kubelet_cluster_dns = [nodelocaldns_ip] %} +{% elif dns_mode in ['coredns'] %} +{% set kubelet_cluster_dns = [skydns_server] %} +{% elif dns_mode == 'coredns_dual' %} +{% set kubelet_cluster_dns = [skydns_server,skydns_server_secondary] %} +{% elif dns_mode == 'manual' %} +{% set kubelet_cluster_dns = [manual_dns_server] %} +{% else %} +{% set kubelet_cluster_dns = [] %} +{% endif %} +clusterDNS: +{% for dns_address in kubelet_cluster_dns %} +- {{ dns_address }} +{% endfor %} +{# Node reserved CPU/memory #} +{% if kube_reserved | bool %} +kubeReservedCgroup: {{ kube_reserved_cgroups }} +kubeReserved: +{% if is_kube_master | bool %} + cpu: {{ kube_master_cpu_reserved }} + memory: {{ kube_master_memory_reserved }} +{% if kube_master_ephemeral_storage_reserved is defined %} + ephemeral-storage: {{ kube_master_ephemeral_storage_reserved }} +{% endif %} +{% if kube_master_pid_reserved is defined %} + pid: "{{ kube_master_pid_reserved }}" +{% endif %} +{% else %} + cpu: {{ kube_cpu_reserved }} + memory: {{ kube_memory_reserved }} +{% if kube_ephemeral_storage_reserved is defined %} + ephemeral-storage: {{ kube_ephemeral_storage_reserved }} +{% endif %} +{% if kube_pid_reserved is defined %} + pid: "{{ kube_pid_reserved }}" +{% endif %} +{% endif %} +{% endif %} +{% if system_reserved | bool %} +systemReservedCgroup: {{ system_reserved_cgroups }} +systemReserved: +{% if is_kube_master | bool %} + cpu: {{ system_master_cpu_reserved }} + memory: {{ system_master_memory_reserved }} +{% if system_master_ephemeral_storage_reserved is defined %} + ephemeral-storage: {{ system_master_ephemeral_storage_reserved }} +{% endif %} +{% if system_master_pid_reserved is defined %} + pid: "{{ system_master_pid_reserved }}" +{% endif %} +{% else %} + cpu: {{ system_cpu_reserved }} + memory: {{ system_memory_reserved }} +{% if system_ephemeral_storage_reserved is defined %} + ephemeral-storage: {{ system_ephemeral_storage_reserved }} +{% endif %} +{% if system_pid_reserved is defined %} + pid: "{{ system_pid_reserved }}" +{% endif %} +{% endif %} +{% endif %} +{% if is_kube_master | bool and eviction_hard_control_plane is defined and eviction_hard_control_plane %} +evictionHard: + {{ eviction_hard_control_plane | to_nice_yaml(indent=2) | indent(2) }} +{% elif not is_kube_master | bool and eviction_hard is defined and eviction_hard %} +evictionHard: + {{ eviction_hard | to_nice_yaml(indent=2) | indent(2) }} +{% endif %} +resolvConf: "{{ kube_resolv_conf }}" +{% if kubelet_config_extra_args %} +{{ kubelet_config_extra_args | to_nice_yaml(indent=2) }} +{% endif %} +{% if inventory_hostname in groups['kube_node'] and kubelet_node_config_extra_args %} +{{ kubelet_node_config_extra_args | to_nice_yaml(indent=2) }} +{% endif %} +{% if kubelet_feature_gates or kube_feature_gates %} +featureGates: +{% for feature in (kubelet_feature_gates | default(kube_feature_gates, true)) %} + {{ feature | replace("=", ": ") }} +{% endfor %} +{% endif %} +{% if tls_min_version is defined %} +tlsMinVersion: {{ tls_min_version }} +{% endif %} +{% if tls_cipher_suites is defined %} +tlsCipherSuites: +{% for tls in tls_cipher_suites %} +- {{ tls }} +{% endfor %} +{% endif %} +eventRecordQPS: {{ kubelet_event_record_qps }} +shutdownGracePeriod: {{ kubelet_shutdown_grace_period }} +shutdownGracePeriodCriticalPods: {{ kubelet_shutdown_grace_period_critical_pods }} +{% if not kubelet_fail_swap_on %} +memorySwap: + swapBehavior: {{ kubelet_swap_behavior }} +{% endif %} +{% if kubelet_streaming_connection_idle_timeout is defined %} +streamingConnectionIdleTimeout: {{ kubelet_streaming_connection_idle_timeout }} +{% endif %} +{% if kubelet_image_gc_high_threshold is defined %} +imageGCHighThresholdPercent: {{ kubelet_image_gc_high_threshold }} +{% endif %} +{% if kubelet_image_gc_low_threshold is defined %} +imageGCLowThresholdPercent: {{ kubelet_image_gc_low_threshold }} +{% endif %} +{% if kubelet_make_iptables_util_chains is defined %} +makeIPTablesUtilChains: {{ kubelet_make_iptables_util_chains | bool }} +{% endif %} +{% if kubelet_seccomp_default is defined %} +seccompDefault: {{ kubelet_seccomp_default | bool }} +{% endif %} +{% if kubelet_cpu_manager_policy is defined %} +cpuManagerPolicy: {{ kubelet_cpu_manager_policy }} +{% endif %} +{% if kubelet_topology_manager_policy is defined %} +topologyManagerPolicy: {{ kubelet_topology_manager_policy }} +{% endif %} +{% if kubelet_topology_manager_scope is defined %} +topologyManagerScope: {{ kubelet_topology_manager_scope }} +{% endif %} +{% if kubelet_tracing %} +tracing: + endpoint: {{ kubelet_tracing_endpoint }} + samplingRatePerMillion: {{ kubelet_tracing_sampling_rate_per_million }} +{% endif %} \ No newline at end of file diff --git a/kubespray/project/roles/kubernetes/node/templates/kubelet.env.v1beta1.j2 b/kubespray/project/roles/kubernetes/node/templates/kubelet.env.v1beta1.j2 new file mode 100644 index 0000000..a5aa369 --- /dev/null +++ b/kubespray/project/roles/kubernetes/node/templates/kubelet.env.v1beta1.j2 @@ -0,0 +1,33 @@ +KUBE_LOG_LEVEL="--v={{ kube_log_level }}" +KUBELET_ADDRESS="--node-ip={{ kubelet_address }}" +{% if kube_override_hostname|default('') %} +KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}" +{% endif %} + +{# Base kubelet args #} +{% set kubelet_args_base -%} +{# start kubeadm specific settings #} +--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ +--config={{ kube_config_dir }}/kubelet-config.yaml \ +--kubeconfig={{ kube_config_dir }}/kubelet.conf \ +{# end kubeadm specific settings #} +--container-runtime-endpoint={{ cri_socket }} \ +--runtime-cgroups={{ kubelet_runtime_cgroups }} \ +{% endset %} + +KUBELET_ARGS="{{ kubelet_args_base }} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}{% if inventory_hostname in groups['kube_node'] %}{% if kubelet_node_custom_flags is string %} {{kubelet_node_custom_flags}} {% else %}{% for flag in kubelet_node_custom_flags %} {{flag}} {% endfor %}{% endif %}{% endif %}" +{% if kubelet_flexvolumes_plugins_dir is defined %} +KUBELET_VOLUME_PLUGIN="--volume-plugin-dir={{ kubelet_flexvolumes_plugins_dir }}" +{% endif %} +{% if kube_network_plugin is defined and kube_network_plugin == "cloud" %} +KUBELET_NETWORK_PLUGIN="--hairpin-mode=promiscuous-bridge --network-plugin=kubenet" +{% endif %} +{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "gce"] %} +KUBELET_CLOUDPROVIDER="--cloud-provider={{ cloud_provider }} --cloud-config={{ kube_config_dir }}/cloud_config" +{% elif cloud_provider is defined and cloud_provider in ["external"] %} +KUBELET_CLOUDPROVIDER="--cloud-provider={{ cloud_provider }}" +{% else %} +KUBELET_CLOUDPROVIDER="" +{% endif %} + +PATH={{ bin_dir }}:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin diff --git a/kubespray/project/roles/kubernetes/node/templates/kubelet.service.j2 b/kubespray/project/roles/kubernetes/node/templates/kubelet.service.j2 new file mode 100644 index 0000000..9df98e0 --- /dev/null +++ b/kubespray/project/roles/kubernetes/node/templates/kubelet.service.j2 @@ -0,0 +1,52 @@ +[Unit] +Description=Kubernetes Kubelet Server +Documentation=https://github.com/GoogleCloudPlatform/kubernetes +After={{ container_manager }}.service +{% if container_manager == 'docker' %} +Wants=docker.socket +{% else %} +Wants={{ container_manager }}.service +{% endif %} + +[Service] +EnvironmentFile=-{{ kube_config_dir }}/kubelet.env +{% if system_reserved|bool %} +ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/cpu/{{ system_reserved_cgroups_for_service_slice }} +ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/cpuacct/{{ system_reserved_cgroups_for_service_slice }} +ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/cpuset/{{ system_reserved_cgroups_for_service_slice }} +ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/hugetlb/{{ system_reserved_cgroups_for_service_slice }} +ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/memory/{{ system_reserved_cgroups_for_service_slice }} +ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/pids/{{ system_reserved_cgroups_for_service_slice }} +ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/systemd/{{ system_reserved_cgroups_for_service_slice }} +{% endif %} +{% if kube_reserved|bool %} +ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/cpu/{{ kube_reserved_cgroups_for_service_slice }} +ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/cpuacct/{{ kube_reserved_cgroups_for_service_slice }} +ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/cpuset/{{ kube_reserved_cgroups_for_service_slice }} +ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/hugetlb/{{ kube_reserved_cgroups_for_service_slice }} +ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/memory/{{ kube_reserved_cgroups_for_service_slice }} +ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/pids/{{ kube_reserved_cgroups_for_service_slice }} +ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/systemd/{{ kube_reserved_cgroups_for_service_slice }} +{% endif %} +ExecStart={{ bin_dir }}/kubelet \ + $KUBE_LOGTOSTDERR \ + $KUBE_LOG_LEVEL \ + $KUBELET_API_SERVER \ + $KUBELET_ADDRESS \ + $KUBELET_PORT \ + $KUBELET_HOSTNAME \ + $KUBELET_ARGS \ + $DOCKER_SOCKET \ + $KUBELET_NETWORK_PLUGIN \ + $KUBELET_VOLUME_PLUGIN \ + $KUBELET_CLOUDPROVIDER +Restart=always +RestartSec=10s +{% if kubelet_systemd_hardening %} +# Hardening setup +IPAddressDeny=any +IPAddressAllow={{ kubelet_secure_addresses }} +{% endif %} + +[Install] +WantedBy=multi-user.target diff --git a/kubespray/project/roles/kubernetes/node/templates/loadbalancer/haproxy.cfg.j2 b/kubespray/project/roles/kubernetes/node/templates/loadbalancer/haproxy.cfg.j2 new file mode 100644 index 0000000..c629325 --- /dev/null +++ b/kubespray/project/roles/kubernetes/node/templates/loadbalancer/haproxy.cfg.j2 @@ -0,0 +1,49 @@ +global + maxconn 4000 + log 127.0.0.1 local0 + +defaults + mode http + log global + option httplog + option dontlognull + option http-server-close + option redispatch + retries 5 + timeout http-request 5m + timeout queue 5m + timeout connect 30s + timeout client {{ loadbalancer_apiserver_keepalive_timeout }} + timeout server 15m + timeout http-keep-alive 30s + timeout check 30s + maxconn 4000 + +{% if loadbalancer_apiserver_healthcheck_port is defined -%} +frontend healthz + bind 0.0.0.0:{{ loadbalancer_apiserver_healthcheck_port }} + {% if enable_dual_stack_networks -%} + bind :::{{ loadbalancer_apiserver_healthcheck_port }} + {% endif -%} + mode http + monitor-uri /healthz +{% endif %} + +frontend kube_api_frontend + bind 127.0.0.1:{{ loadbalancer_apiserver_port|default(kube_apiserver_port) }} + {% if enable_dual_stack_networks -%} + bind [::1]:{{ loadbalancer_apiserver_port|default(kube_apiserver_port) }}; + {% endif -%} + mode tcp + option tcplog + default_backend kube_api_backend + +backend kube_api_backend + mode tcp + balance leastconn + default-server inter 15s downinter 15s rise 2 fall 2 slowstart 60s maxconn 1000 maxqueue 256 weight 100 + option httpchk GET /healthz + http-check expect status 200 + {% for host in groups['kube_control_plane'] -%} + server {{ host }} {{ hostvars[host]['access_ip'] | default(hostvars[host]['ip'] | default(fallback_ips[host])) }}:{{ kube_apiserver_port }} check check-ssl verify none + {% endfor -%} diff --git a/kubespray/project/roles/kubernetes/node/templates/loadbalancer/nginx.conf.j2 b/kubespray/project/roles/kubernetes/node/templates/loadbalancer/nginx.conf.j2 new file mode 100644 index 0000000..07b9370 --- /dev/null +++ b/kubespray/project/roles/kubernetes/node/templates/loadbalancer/nginx.conf.j2 @@ -0,0 +1,60 @@ +error_log stderr notice; + +worker_processes 2; +worker_rlimit_nofile 130048; +worker_shutdown_timeout 10s; + +events { + multi_accept on; + use epoll; + worker_connections 16384; +} + +stream { + upstream kube_apiserver { + least_conn; + {% for host in groups['kube_control_plane'] -%} + server {{ hostvars[host]['access_ip'] | default(hostvars[host]['ip'] | default(fallback_ips[host])) }}:{{ kube_apiserver_port }}; + {% endfor -%} + } + + server { + listen 127.0.0.1:{{ loadbalancer_apiserver_port|default(kube_apiserver_port) }}; + {% if enable_dual_stack_networks -%} + listen [::1]:{{ loadbalancer_apiserver_port|default(kube_apiserver_port) }}; + {% endif -%} + proxy_pass kube_apiserver; + proxy_timeout 10m; + proxy_connect_timeout 1s; + } +} + +http { + aio threads; + aio_write on; + tcp_nopush on; + tcp_nodelay on; + + keepalive_timeout {{ loadbalancer_apiserver_keepalive_timeout }}; + keepalive_requests 100; + reset_timedout_connection on; + server_tokens off; + autoindex off; + + {% if loadbalancer_apiserver_healthcheck_port is defined -%} + server { + listen {{ loadbalancer_apiserver_healthcheck_port }}; + {% if enable_dual_stack_networks -%} + listen [::]:{{ loadbalancer_apiserver_healthcheck_port }}; + {% endif -%} + location /healthz { + access_log off; + return 200; + } + location /stub_status { + stub_status on; + access_log off; + } + } + {% endif %} +} diff --git a/kubespray/project/roles/kubernetes/node/templates/manifests/haproxy.manifest.j2 b/kubespray/project/roles/kubernetes/node/templates/manifests/haproxy.manifest.j2 new file mode 100644 index 0000000..7c5097c --- /dev/null +++ b/kubespray/project/roles/kubernetes/node/templates/manifests/haproxy.manifest.j2 @@ -0,0 +1,42 @@ +apiVersion: v1 +kind: Pod +metadata: + name: {{ loadbalancer_apiserver_pod_name }} + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: Reconcile + k8s-app: kube-haproxy + annotations: + haproxy-cfg-checksum: "{{ haproxy_stat.stat.checksum }}" +spec: + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + containers: + - name: haproxy + image: {{ haproxy_image_repo }}:{{ haproxy_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + resources: + requests: + cpu: {{ loadbalancer_apiserver_cpu_requests }} + memory: {{ loadbalancer_apiserver_memory_requests }} + {% if loadbalancer_apiserver_healthcheck_port is defined -%} + livenessProbe: + httpGet: + path: /healthz + port: {{ loadbalancer_apiserver_healthcheck_port }} + readinessProbe: + httpGet: + path: /healthz + port: {{ loadbalancer_apiserver_healthcheck_port }} + {% endif -%} + volumeMounts: + - mountPath: /usr/local/etc/haproxy/ + name: etc-haproxy + readOnly: true + volumes: + - name: etc-haproxy + hostPath: + path: {{ haproxy_config_dir }} diff --git a/kubespray/project/roles/kubernetes/node/templates/manifests/kube-vip.manifest.j2 b/kubespray/project/roles/kubernetes/node/templates/manifests/kube-vip.manifest.j2 new file mode 100644 index 0000000..936ad44 --- /dev/null +++ b/kubespray/project/roles/kubernetes/node/templates/manifests/kube-vip.manifest.j2 @@ -0,0 +1,106 @@ +# Inspired by https://github.com/kube-vip/kube-vip/blob/v0.5.11/pkg/kubevip/config_generator.go#L13 +apiVersion: v1 +kind: Pod +metadata: + name: kube-vip + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: Reconcile + k8s-app: kube-vip +spec: + containers: + - args: + - manager + env: + - name: vip_arp + value: {{ kube_vip_arp_enabled | string | to_json }} + - name: port + value: {{ kube_apiserver_port | string | to_json }} +{% if kube_vip_interface %} + - name: vip_interface + value: {{ kube_vip_interface | string | to_json }} +{% endif %} +{% if kube_vip_services_interface %} + - name: vip_servicesinterface + value: {{ kube_vip_services_interface | string | to_json }} +{% endif %} +{% if kube_vip_cidr %} + - name: vip_cidr + value: {{ kube_vip_cidr | string | to_json }} +{% endif %} +{% if kube_vip_controlplane_enabled %} + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: vip_ddns + value: {{ kube_vip_ddns_enabled | string | to_json }} +{% endif %} +{% if kube_vip_services_enabled %} + - name: svc_enable + value: "true" +{% endif %} +{% if kube_vip_enableServicesElection %} + - name: svc_election + value: "true" +{% endif %} +{% if kube_vip_leader_election_enabled %} + - name: vip_leaderelection + value: "true" + - name: vip_leaseduration + value: "5" + - name: vip_renewdeadline + value: "3" + - name: vip_retryperiod + value: "1" +{% endif %} +{% if kube_vip_bgp_enabled %} + - name: bgp_enable + value: "true" + - name: bgp_routerid + value: {{ kube_vip_bgp_routerid | string | to_json }} + - name: bgp_as + value: {{ kube_vip_local_as | string | to_json }} + - name: bgp_peeraddress + value: {{ kube_vip_bgp_peeraddress | to_json }} + - name: bgp_peerpass + value: {{ kube_vip_bgp_peerpass | to_json }} + - name: bgp_peeras + value: {{ kube_vip_bgp_peeras | string | to_json }} +{% if kube_vip_bgppeers %} + - name: bgp_peers + value: {{ kube_vip_bgppeers | join(',') | to_json }} +{% endif %} +{% endif %} + - name: address + value: {{ kube_vip_address | to_json }} +{% if kube_vip_lb_enable %} + - name: lb_enable + value: "true" +{% endif %} +{% if kube_vip_lb_fwdmethod %} + - name: lb_fwdmethod + value: {{ kube_vip_lb_fwdmethod | string | to_json }} +{% endif %} + image: {{ kube_vip_image_repo }}:{{ kube_vip_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + volumeMounts: + - mountPath: /etc/kubernetes/admin.conf + name: kubeconfig + hostAliases: + - hostnames: + - kubernetes + ip: 127.0.0.1 + hostNetwork: true + volumes: + - hostPath: + path: /etc/kubernetes/admin.conf + name: kubeconfig +status: {} diff --git a/kubespray/project/roles/kubernetes/node/templates/manifests/nginx-proxy.manifest.j2 b/kubespray/project/roles/kubernetes/node/templates/manifests/nginx-proxy.manifest.j2 new file mode 100644 index 0000000..16757ec --- /dev/null +++ b/kubespray/project/roles/kubernetes/node/templates/manifests/nginx-proxy.manifest.j2 @@ -0,0 +1,42 @@ +apiVersion: v1 +kind: Pod +metadata: + name: {{ loadbalancer_apiserver_pod_name }} + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: Reconcile + k8s-app: kube-nginx + annotations: + nginx-cfg-checksum: "{{ nginx_stat.stat.checksum }}" +spec: + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + containers: + - name: nginx-proxy + image: {{ nginx_image_repo }}:{{ nginx_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + resources: + requests: + cpu: {{ loadbalancer_apiserver_cpu_requests }} + memory: {{ loadbalancer_apiserver_memory_requests }} + {% if loadbalancer_apiserver_healthcheck_port is defined -%} + livenessProbe: + httpGet: + path: /healthz + port: {{ loadbalancer_apiserver_healthcheck_port }} + readinessProbe: + httpGet: + path: /healthz + port: {{ loadbalancer_apiserver_healthcheck_port }} + {% endif -%} + volumeMounts: + - mountPath: /etc/nginx + name: etc-nginx + readOnly: true + volumes: + - name: etc-nginx + hostPath: + path: {{ nginx_config_dir }} diff --git a/kubespray/project/roles/kubernetes/node/templates/node-kubeconfig.yaml.j2 b/kubespray/project/roles/kubernetes/node/templates/node-kubeconfig.yaml.j2 new file mode 100644 index 0000000..4b8af60 --- /dev/null +++ b/kubespray/project/roles/kubernetes/node/templates/node-kubeconfig.yaml.j2 @@ -0,0 +1,19 @@ +--- +apiVersion: v1 +kind: Config +clusters: +- name: local + cluster: + certificate-authority: {{ kube_cert_dir }}/ca.pem + server: {{ kube_apiserver_endpoint }} +users: +- name: kubelet + user: + client-certificate: {{ kube_cert_dir }}/node-{{ inventory_hostname }}.pem + client-key: {{ kube_cert_dir }}/node-{{ inventory_hostname }}-key.pem +contexts: +- context: + cluster: local + user: kubelet + name: kubelet-{{ cluster_name }} +current-context: kubelet-{{ cluster_name }} diff --git a/kubespray/project/roles/kubernetes/node/vars/fedora.yml b/kubespray/project/roles/kubernetes/node/vars/fedora.yml new file mode 100644 index 0000000..59bc55d --- /dev/null +++ b/kubespray/project/roles/kubernetes/node/vars/fedora.yml @@ -0,0 +1,2 @@ +--- +kube_resolv_conf: "/run/systemd/resolve/resolv.conf" diff --git a/kubespray/project/roles/kubernetes/node/vars/ubuntu-18.yml b/kubespray/project/roles/kubernetes/node/vars/ubuntu-18.yml new file mode 100644 index 0000000..59bc55d --- /dev/null +++ b/kubespray/project/roles/kubernetes/node/vars/ubuntu-18.yml @@ -0,0 +1,2 @@ +--- +kube_resolv_conf: "/run/systemd/resolve/resolv.conf" diff --git a/kubespray/project/roles/kubernetes/node/vars/ubuntu-20.yml b/kubespray/project/roles/kubernetes/node/vars/ubuntu-20.yml new file mode 100644 index 0000000..59bc55d --- /dev/null +++ b/kubespray/project/roles/kubernetes/node/vars/ubuntu-20.yml @@ -0,0 +1,2 @@ +--- +kube_resolv_conf: "/run/systemd/resolve/resolv.conf" diff --git a/kubespray/project/roles/kubernetes/node/vars/ubuntu-22.yml b/kubespray/project/roles/kubernetes/node/vars/ubuntu-22.yml new file mode 100644 index 0000000..59bc55d --- /dev/null +++ b/kubespray/project/roles/kubernetes/node/vars/ubuntu-22.yml @@ -0,0 +1,2 @@ +--- +kube_resolv_conf: "/run/systemd/resolve/resolv.conf" diff --git a/kubespray/project/roles/kubernetes/preinstall/defaults/main.yml b/kubespray/project/roles/kubernetes/preinstall/defaults/main.yml new file mode 100644 index 0000000..eb33ed3 --- /dev/null +++ b/kubespray/project/roles/kubernetes/preinstall/defaults/main.yml @@ -0,0 +1,155 @@ +--- +# Set to true to allow pre-checks to fail and continue deployment +ignore_assert_errors: false + +epel_enabled: false +# Kubespray sets this to true after clusterDNS is running to apply changes to the host resolv.conf +dns_late: false + +common_required_pkgs: + - "{{ (ansible_distribution == 'openSUSE Tumbleweed') | ternary('openssl-1_1', 'openssl') }}" + - curl + - rsync + - socat + - unzip + - e2fsprogs + - xfsprogs + - ebtables + - bash-completion + - tar + +# Set to true if your network does not support IPv6 +# This may be necessary for pulling Docker images from +# GCE docker repository +disable_ipv6_dns: false + +# Remove default cluster search domains (``default.svc.{{ dns_domain }}, svc.{{ dns_domain }}``). +remove_default_searchdomains: false + +kube_owner: kube +kube_cert_group: kube-cert +kube_config_dir: /etc/kubernetes +kube_cert_dir: "{{ kube_config_dir }}/ssl" +kube_cert_compat_dir: /etc/kubernetes/pki +kubelet_flexvolumes_plugins_dir: /usr/libexec/kubernetes/kubelet-plugins/volume/exec + +# Flatcar Container Linux by Kinvolk cloud init config file to define /etc/resolv.conf content +# for hostnet pods and infra needs +resolveconf_cloud_init_conf: /etc/resolveconf_cloud_init.conf + +# All inventory hostnames will be written into each /etc/hosts file. +populate_inventory_to_hosts_file: true +# K8S Api FQDN will be written into /etc/hosts file. +populate_loadbalancer_apiserver_to_hosts_file: true +# etc_hosts_localhost_entries will be written into /etc/hosts file. +populate_localhost_entries_to_hosts_file: true + +sysctl_file_path: "/etc/sysctl.d/99-sysctl.conf" + +etc_hosts_localhost_entries: + 127.0.0.1: + expected: + - localhost + - localhost.localdomain + ::1: + expected: + - localhost6 + - localhost6.localdomain + unexpected: + - localhost + - localhost.localdomain + +# Minimal memory requirement in MB for safety checks +minimal_node_memory_mb: 1024 +minimal_master_memory_mb: 1500 + +yum_repo_dir: /etc/yum.repos.d + +# number of times package install task should be retried +pkg_install_retries: 4 + +# Check if access_ip responds to ping. Set false if your firewall blocks ICMP. +ping_access_ip: true + +## NTP Settings +# Start the ntpd or chrony service and enable it at system boot. +ntp_enabled: false +# The package to install which provides NTP functionality. +# The default is ntp for most platforms, or chrony on RHEL/CentOS 7 and later. +# The ntp_package can be one of ['ntp', 'chrony'] +ntp_package: >- + {% if ansible_os_family == "RedHat" -%} + chrony + {%- else -%} + ntp + {%- endif -%} + +# Manage the NTP configuration file. +ntp_manage_config: false +# Specify the NTP servers +# Only takes effect when ntp_manage_config is true. +ntp_servers: + - "0.pool.ntp.org iburst" + - "1.pool.ntp.org iburst" + - "2.pool.ntp.org iburst" + - "3.pool.ntp.org iburst" +# Restrict NTP access to these hosts. +# Only takes effect when ntp_manage_config is true. +ntp_restrict: + - "127.0.0.1" + - "::1" +# The NTP driftfile path +# Only takes effect when ntp_manage_config is true. +ntp_driftfile: /var/lib/ntp/ntp.drift +# Enable tinker panic is useful when running NTP in a VM environment. +# Only takes effect when ntp_manage_config is true. +ntp_tinker_panic: false + +# Force sync time immediately after the ntp installed, which is useful in a newly installed system. +ntp_force_sync_immediately: false + +# Set the timezone for your server. eg: "Etc/UTC","Etc/GMT-8". If not set, the timezone will not change. +ntp_timezone: "" + +# Currently known os distributions +supported_os_distributions: + - 'RedHat' + - 'CentOS' + - 'Fedora' + - 'Ubuntu' + - 'Debian' + - 'Flatcar' + - 'Flatcar Container Linux by Kinvolk' + - 'Suse' + - 'openSUSE Leap' + - 'openSUSE Tumbleweed' + - 'ClearLinux' + - 'OracleLinux' + - 'AlmaLinux' + - 'Rocky' + - 'Amazon' + - 'Kylin Linux Advanced Server' + - 'UnionTech' + - 'UniontechOS' + - 'openEuler' + +# Extending some distributions into the redhat os family +redhat_os_family_extensions: + - "Kylin Linux Advanced Server" + - "openEuler" + - "UnionTech" + - "UniontechOS" + +# Extending some distributions into the debian os family +debian_os_family_extensions: + - "UnionTech OS Server 20" + +# Sets DNSStubListener=no, useful if you get "0.0.0.0:53: bind: address already in use" +systemd_resolved_disable_stub_listener: "{{ ansible_os_family in ['Flatcar', 'Flatcar Container Linux by Kinvolk'] }}" + +# Used to disable File Access Policy Daemon service. +# If service is enabled, the CNI plugin installation will fail +disable_fapolicyd: true + +# Enable 0120-growpart-azure-centos-7 tasks +growpart_azure_enabled: true diff --git a/kubespray/project/roles/kubernetes/preinstall/files/dhclient_nodnsupdate b/kubespray/project/roles/kubernetes/preinstall/files/dhclient_nodnsupdate new file mode 100644 index 0000000..03c7c99 --- /dev/null +++ b/kubespray/project/roles/kubernetes/preinstall/files/dhclient_nodnsupdate @@ -0,0 +1,4 @@ +#!/bin/sh +make_resolv_conf() { + : +} diff --git a/kubespray/project/roles/kubernetes/preinstall/gen-gitinfos.sh b/kubespray/project/roles/kubernetes/preinstall/gen-gitinfos.sh new file mode 100755 index 0000000..bfab5a4 --- /dev/null +++ b/kubespray/project/roles/kubernetes/preinstall/gen-gitinfos.sh @@ -0,0 +1,73 @@ +#!/bin/sh +set -e + +# Text color variables +txtbld=$(tput bold) # Bold +bldred=${txtbld}$(tput setaf 1) # red +bldgre=${txtbld}$(tput setaf 2) # green +bldylw=${txtbld}$(tput setaf 3) # yellow +txtrst=$(tput sgr0) # Reset +err=${bldred}ERROR${txtrst} +info=${bldgre}INFO${txtrst} +warn=${bldylw}WARNING${txtrst} + +usage() +{ + cat << EOF +Generates a file which contains useful git informations + +Usage : $(basename $0) [global|diff] + ex : + Generate git information + $(basename $0) global + Generate diff from latest tag + $(basename $0) diff +EOF +} + +if [ $# != 1 ]; then + printf "\n$err : Needs 1 argument\n" + usage + exit 2 +fi; + +current_commit=$(git rev-parse HEAD) +latest_tag=$(git describe --abbrev=0 --tags) +latest_tag_commit=$(git show-ref -s ${latest_tag}) +tags_list=$(git tag --points-at "${latest_tag}") + +case ${1} in + "global") +cat<0 + fail_msg: "nameserver should not empty in /etc/resolv.conf" + when: + - not ignore_assert_errors + - configured_nameservers is defined + - not (upstream_dns_servers is defined and upstream_dns_servers | length > 0) + - not (disable_host_nameservers | default(false)) + - dns_mode in ['coredns', 'coredns_dual'] + +- name: NetworkManager | Check if host has NetworkManager + # noqa command-instead-of-module - Should we use service_facts for this? + command: systemctl is-active --quiet NetworkManager.service + register: networkmanager_enabled + failed_when: false + changed_when: false + check_mode: false + +- name: Check systemd-resolved + # noqa command-instead-of-module - Should we use service_facts for this? + command: systemctl is-active systemd-resolved + register: systemd_resolved_enabled + failed_when: false + changed_when: false + check_mode: no + +- name: Set default dns if remove_default_searchdomains is false + set_fact: + default_searchdomains: ["default.svc.{{ dns_domain }}", "svc.{{ dns_domain }}"] + when: not remove_default_searchdomains | default() | bool or (remove_default_searchdomains | default() | bool and searchdomains | default([]) | length==0) + +- name: Set dns facts + set_fact: + resolvconf: >- + {%- if resolvconf.rc == 0 and resolvconfd_path.stat.isdir is defined and resolvconfd_path.stat.isdir -%}true{%- else -%}false{%- endif -%} + bogus_domains: |- + {% for d in default_searchdomains | default([]) + searchdomains | default([]) -%} + {{ dns_domain }}.{{ d }}./{{ d }}.{{ d }}./com.{{ d }}./ + {%- endfor %} + cloud_resolver: "{{ ['169.254.169.254'] if cloud_provider is defined and cloud_provider == 'gce' else + ['169.254.169.253'] if cloud_provider is defined and cloud_provider == 'aws' else + [] }}" + +- name: Check if kubelet is configured + stat: + path: "{{ kube_config_dir }}/kubelet.env" + get_attributes: no + get_checksum: no + get_mime: no + register: kubelet_configured + changed_when: false + +- name: Check if early DNS configuration stage + set_fact: + dns_early: "{{ not kubelet_configured.stat.exists }}" + +- name: Target resolv.conf files + set_fact: + resolvconffile: /etc/resolv.conf + base: >- + {%- if resolvconf | bool -%}/etc/resolvconf/resolv.conf.d/base{%- endif -%} + head: >- + {%- if resolvconf | bool -%}/etc/resolvconf/resolv.conf.d/head{%- endif -%} + when: not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] and not is_fedora_coreos + +- name: Target temporary resolvconf cloud init file (Flatcar Container Linux by Kinvolk / Fedora CoreOS) + set_fact: + resolvconffile: /tmp/resolveconf_cloud_init_conf + when: ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] or is_fedora_coreos + +- name: Check if /etc/dhclient.conf exists + stat: + path: /etc/dhclient.conf + get_attributes: no + get_checksum: no + get_mime: no + register: dhclient_stat + +- name: Target dhclient conf file for /etc/dhclient.conf + set_fact: + dhclientconffile: /etc/dhclient.conf + when: dhclient_stat.stat.exists + +- name: Check if /etc/dhcp/dhclient.conf exists + stat: + path: /etc/dhcp/dhclient.conf + get_attributes: no + get_checksum: no + get_mime: no + register: dhcp_dhclient_stat + +- name: Target dhclient conf file for /etc/dhcp/dhclient.conf + set_fact: + dhclientconffile: /etc/dhcp/dhclient.conf + when: dhcp_dhclient_stat.stat.exists + +- name: Target dhclient hook file for Red Hat family + set_fact: + dhclienthookfile: /etc/dhcp/dhclient.d/zdnsupdate.sh + when: ansible_os_family == "RedHat" + +- name: Target dhclient hook file for Debian family + set_fact: + dhclienthookfile: /etc/dhcp/dhclient-exit-hooks.d/zdnsupdate + when: ansible_os_family == "Debian" + +- name: Generate search domains to resolvconf + set_fact: + searchentries: + search {{ (default_searchdomains | default([]) + searchdomains | default([])) | join(' ') }} + domainentry: + domain {{ dns_domain }} + supersede_search: + supersede domain-search "{{ (default_searchdomains | default([]) + searchdomains | default([])) | join('", "') }}"; + supersede_domain: + supersede domain-name "{{ dns_domain }}"; + +- name: Pick coredns cluster IP or default resolver + set_fact: + coredns_server: |- + {%- if dns_mode == 'coredns' and not dns_early | bool -%} + {{ [skydns_server] }} + {%- elif dns_mode == 'coredns_dual' and not dns_early | bool -%} + {{ [skydns_server] + [skydns_server_secondary] }} + {%- elif dns_mode == 'manual' and not dns_early | bool -%} + {{ (manual_dns_server.split(',') | list) }} + {%- elif dns_mode == 'none' and not dns_early | bool -%} + [] + {%- elif dns_early | bool -%} + {{ upstream_dns_servers | default([]) }} + {%- endif -%} + +# This task should only run after cluster/nodelocal DNS is up, otherwise all DNS lookups will timeout +- name: Generate nameservers for resolvconf, including cluster DNS + set_fact: + nameserverentries: |- + {{ (([nodelocaldns_ip] if enable_nodelocaldns else []) + (coredns_server | d([]) if not enable_nodelocaldns else []) + nameservers | d([]) + cloud_resolver | d([]) + (configured_nameservers | d([]) if not disable_host_nameservers | d() | bool else [])) | unique | join(',') }} + supersede_nameserver: + supersede domain-name-servers {{ (([nodelocaldns_ip] if enable_nodelocaldns else []) + (coredns_server | d([]) if not enable_nodelocaldns else []) + nameservers | d([]) + cloud_resolver | d([]) + (configured_nameservers | d([]) if not disable_host_nameservers | d() | bool else [])) | unique | join(', ') }}; + when: not dns_early or dns_late + +# This task should run instead of the above task when cluster/nodelocal DNS hasn't +# been deployed yet (like scale.yml/cluster.yml) or when it's down (reset.yml) +- name: Generate nameservers for resolvconf, not including cluster DNS + set_fact: + nameserverentries: |- + {{ (nameservers | d([]) + cloud_resolver | d([]) + configured_nameservers | d([])) | unique | join(',') }} + supersede_nameserver: + supersede domain-name-servers {{ (nameservers | d([]) + cloud_resolver | d([])) | unique | join(', ') }}; + when: dns_early and not dns_late + +- name: Gather os specific variables + include_vars: "{{ item }}" + with_first_found: + - files: + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower | replace('/', '_') }}.yml" + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_release }}.yml" + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower | replace('/', '_') }}.yml" + - "{{ ansible_distribution | lower }}.yml" + - "{{ ansible_os_family | lower }}.yml" + - defaults.yml + paths: + - ../vars + skip: true + +- name: Set etcd vars if using kubeadm mode + set_fact: + etcd_cert_dir: "{{ kube_cert_dir }}" + kube_etcd_cacert_file: "etcd/ca.crt" + kube_etcd_cert_file: "apiserver-etcd-client.crt" + kube_etcd_key_file: "apiserver-etcd-client.key" + when: + - etcd_deployment_type == "kubeadm" + +- name: Check /usr readonly + stat: + path: "/usr" + get_attributes: no + get_checksum: no + get_mime: no + register: usr + +- name: Set alternate flexvolume path + set_fact: + kubelet_flexvolumes_plugins_dir: /var/lib/kubelet/volumeplugins + when: not usr.stat.writeable diff --git a/kubespray/project/roles/kubernetes/preinstall/tasks/0040-verify-settings.yml b/kubespray/project/roles/kubernetes/preinstall/tasks/0040-verify-settings.yml new file mode 100644 index 0000000..f2d40e9 --- /dev/null +++ b/kubespray/project/roles/kubernetes/preinstall/tasks/0040-verify-settings.yml @@ -0,0 +1,318 @@ +--- +- name: Stop if either kube_control_plane or kube_node group is empty + assert: + that: "groups.get( item )" + with_items: + - kube_control_plane + - kube_node + run_once: true + when: not ignore_assert_errors + +- name: Stop if etcd group is empty in external etcd mode + assert: + that: groups.get('etcd') + fail_msg: "Group 'etcd' cannot be empty in external etcd mode" + run_once: true + when: + - not ignore_assert_errors + - etcd_deployment_type != "kubeadm" + +- name: Stop if non systemd OS type + assert: + that: ansible_service_mgr == "systemd" + when: not ignore_assert_errors + +- name: Stop if the os does not support + assert: + that: (allow_unsupported_distribution_setup | default(false)) or ansible_distribution in supported_os_distributions + msg: "{{ ansible_distribution }} is not a known OS" + when: not ignore_assert_errors + +- name: Stop if unknown network plugin + assert: + that: kube_network_plugin in ['calico', 'flannel', 'weave', 'cloud', 'cilium', 'cni', 'kube-ovn', 'kube-router', 'macvlan', 'custom_cni'] + msg: "{{ kube_network_plugin }} is not supported" + when: + - kube_network_plugin is defined + - not ignore_assert_errors + +- name: Stop if unsupported version of Kubernetes + assert: + that: kube_version is version(kube_version_min_required, '>=') + msg: "The current release of Kubespray only support newer version of Kubernetes than {{ kube_version_min_required }} - You are trying to apply {{ kube_version }}" + when: not ignore_assert_errors + +# simplify this items-list when https://github.com/ansible/ansible/issues/15753 is resolved +- name: "Stop if known booleans are set as strings (Use JSON format on CLI: -e \"{'key': true }\")" + assert: + that: item.value | type_debug == 'bool' + msg: "{{ item.value }} isn't a bool" + run_once: yes + with_items: + - { name: download_run_once, value: "{{ download_run_once }}" } + - { name: deploy_netchecker, value: "{{ deploy_netchecker }}" } + - { name: download_always_pull, value: "{{ download_always_pull }}" } + - { name: helm_enabled, value: "{{ helm_enabled }}" } + - { name: openstack_lbaas_enabled, value: "{{ openstack_lbaas_enabled }}" } + when: not ignore_assert_errors + +- name: Stop if even number of etcd hosts + assert: + that: groups.etcd | length is not divisibleby 2 + when: + - not ignore_assert_errors + - inventory_hostname in groups.get('etcd',[]) + +- name: Stop if memory is too small for masters + assert: + that: ansible_memtotal_mb >= minimal_master_memory_mb + when: + - not ignore_assert_errors + - inventory_hostname in groups['kube_control_plane'] + +- name: Stop if memory is too small for nodes + assert: + that: ansible_memtotal_mb >= minimal_node_memory_mb + when: + - not ignore_assert_errors + - inventory_hostname in groups['kube_node'] + +# This assertion will fail on the safe side: One can indeed schedule more pods +# on a node than the CIDR-range has space for when additional pods use the host +# network namespace. It is impossible to ascertain the number of such pods at +# provisioning time, so to establish a guarantee, we factor these out. +# NOTICE: the check blatantly ignores the inet6-case +- name: Guarantee that enough network address space is available for all pods + assert: + that: "{{ (kubelet_max_pods | default(110)) | int <= (2 ** (32 - kube_network_node_prefix | int)) - 2 }}" + msg: "Do not schedule more pods on a node than inet addresses are available." + when: + - not ignore_assert_errors + - inventory_hostname in groups['k8s_cluster'] + - kube_network_node_prefix is defined + - kube_network_plugin != 'calico' + +- name: Stop if ip var does not match local ips + assert: + that: (ip in ansible_all_ipv4_addresses) or (ip in ansible_all_ipv6_addresses) + msg: "IPv4: '{{ ansible_all_ipv4_addresses }}' and IPv6: '{{ ansible_all_ipv6_addresses }}' do not contain '{{ ip }}'" + when: + - not ignore_assert_errors + - ip is defined + +- name: Ensure ping package + package: + # noqa: jinja[spacing] + name: >- + {%- if ansible_os_family == 'Debian' -%} + iputils-ping + {%- else -%} + iputils + {%- endif -%} + state: present + when: + - access_ip is defined + - not ignore_assert_errors + - ping_access_ip + - not is_fedora_coreos + - not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] + +- name: Stop if access_ip is not pingable + command: ping -c1 {{ access_ip }} + when: + - access_ip is defined + - not ignore_assert_errors + - ping_access_ip + changed_when: false + +- name: Stop if RBAC is not enabled when dashboard is enabled + assert: + that: rbac_enabled + when: + - dashboard_enabled + - not ignore_assert_errors + +- name: Stop if RBAC is not enabled when OCI cloud controller is enabled + assert: + that: rbac_enabled + when: + - cloud_provider is defined and cloud_provider == "oci" + - not ignore_assert_errors + +- name: Stop if kernel version is too low + assert: + that: ansible_kernel.split('-')[0] is version('4.9.17', '>=') + when: + - kube_network_plugin == 'cilium' or cilium_deploy_additionally | default(false) | bool + - not ignore_assert_errors + +- name: Stop if bad hostname + assert: + that: inventory_hostname is match("[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$") + msg: "Hostname must consist of lower case alphanumeric characters, '.' or '-', and must start and end with an alphanumeric character" + when: not ignore_assert_errors + +- name: Check cloud_provider value + assert: + that: cloud_provider in ['gce', 'aws', 'azure', 'openstack', 'vsphere', 'oci', 'external'] + msg: "If set the 'cloud_provider' var must be set either to 'gce', 'aws', 'azure', 'openstack', 'vsphere', 'oci' or 'external'" + when: + - cloud_provider is defined + - not ignore_assert_errors + tags: + - cloud-provider + - facts + +- name: "Check that kube_service_addresses is a network range" + assert: + that: + - kube_service_addresses | ansible.utils.ipaddr('net') + msg: "kube_service_addresses = '{{ kube_service_addresses }}' is not a valid network range" + run_once: yes + +- name: "Check that kube_pods_subnet is a network range" + assert: + that: + - kube_pods_subnet | ansible.utils.ipaddr('net') + msg: "kube_pods_subnet = '{{ kube_pods_subnet }}' is not a valid network range" + run_once: yes + +- name: "Check that kube_pods_subnet does not collide with kube_service_addresses" + assert: + that: + - kube_pods_subnet | ansible.utils.ipaddr(kube_service_addresses) | string == 'None' + msg: "kube_pods_subnet cannot be the same network segment as kube_service_addresses" + run_once: yes + +- name: "Check that IP range is enough for the nodes" + assert: + that: + - 2 ** (kube_network_node_prefix - kube_pods_subnet | ansible.utils.ipaddr('prefix')) >= groups['k8s_cluster'] | length + msg: "Not enough IPs are available for the desired node count." + when: kube_network_plugin != 'calico' + run_once: yes + +- name: Stop if unknown dns mode + assert: + that: dns_mode in ['coredns', 'coredns_dual', 'manual', 'none'] + msg: "dns_mode can only be 'coredns', 'coredns_dual', 'manual' or 'none'" + when: dns_mode is defined + run_once: true + +- name: Stop if unknown kube proxy mode + assert: + that: kube_proxy_mode in ['iptables', 'ipvs'] + msg: "kube_proxy_mode can only be 'iptables' or 'ipvs'" + when: kube_proxy_mode is defined + run_once: true + +- name: Stop if unknown cert_management + assert: + that: cert_management | d('script') in ['script', 'none'] + msg: "cert_management can only be 'script' or 'none'" + run_once: true + +- name: Stop if unknown resolvconf_mode + assert: + that: resolvconf_mode in ['docker_dns', 'host_resolvconf', 'none'] + msg: "resolvconf_mode can only be 'docker_dns', 'host_resolvconf' or 'none'" + when: resolvconf_mode is defined + run_once: true + +- name: Stop if etcd deployment type is not host, docker or kubeadm + assert: + that: etcd_deployment_type in ['host', 'docker', 'kubeadm'] + msg: "The etcd deployment type, 'etcd_deployment_type', must be host, docker or kubeadm" + when: + - inventory_hostname in groups.get('etcd',[]) + +- name: Stop if container manager is not docker, crio or containerd + assert: + that: container_manager in ['docker', 'crio', 'containerd'] + msg: "The container manager, 'container_manager', must be docker, crio or containerd" + run_once: true + +- name: Stop if etcd deployment type is not host or kubeadm when container_manager != docker + assert: + that: etcd_deployment_type in ['host', 'kubeadm'] + msg: "The etcd deployment type, 'etcd_deployment_type', must be host or kubeadm when container_manager is not docker" + when: + - inventory_hostname in groups.get('etcd',[]) + - container_manager != 'docker' + +# TODO: Clean this task up when we drop backward compatibility support for `etcd_kubeadm_enabled` +- name: Stop if etcd deployment type is not host or kubeadm when container_manager != docker and etcd_kubeadm_enabled is not defined + run_once: yes + when: etcd_kubeadm_enabled is defined + block: + - name: Warn the user if they are still using `etcd_kubeadm_enabled` + debug: + msg: > + "WARNING! => `etcd_kubeadm_enabled` is deprecated and will be removed in a future release. + You can set `etcd_deployment_type` to `kubeadm` instead of setting `etcd_kubeadm_enabled` to `true`." + changed_when: true + + - name: Stop if `etcd_kubeadm_enabled` is defined and `etcd_deployment_type` is not `kubeadm` or `host` + assert: + that: etcd_deployment_type == 'kubeadm' + msg: > + It is not possible to use `etcd_kubeadm_enabled` when `etcd_deployment_type` is set to {{ etcd_deployment_type }}. + Unset the `etcd_kubeadm_enabled` variable and set `etcd_deployment_type` to desired deployment type (`host`, `kubeadm`, `docker`) instead." + when: etcd_kubeadm_enabled + +- name: Stop if download_localhost is enabled but download_run_once is not + assert: + that: download_run_once + msg: "download_localhost requires enable download_run_once" + when: download_localhost + +- name: Stop if kata_containers_enabled is enabled when container_manager is docker + assert: + that: container_manager != 'docker' + msg: "kata_containers_enabled support only for containerd and crio-o. See https://github.com/kata-containers/documentation/blob/1.11.4/how-to/run-kata-with-k8s.md#install-a-cri-implementation for details" + when: kata_containers_enabled + +- name: Stop if gvisor_enabled is enabled when container_manager is not containerd + assert: + that: container_manager == 'containerd' + msg: "gvisor_enabled support only compatible with containerd. See https://github.com/kubernetes-sigs/kubespray/issues/7650 for details" + when: gvisor_enabled + +- name: Stop if download_localhost is enabled for Flatcar Container Linux + assert: + that: ansible_os_family not in ["Flatcar", "Flatcar Container Linux by Kinvolk"] + msg: "download_run_once not supported for Flatcar Container Linux" + when: download_run_once or download_force_cache + +- name: Ensure minimum containerd version + assert: + that: containerd_version is version(containerd_min_version_required, '>=') + msg: "containerd_version is too low. Minimum version {{ containerd_min_version_required }}" + run_once: yes + when: + - containerd_version not in ['latest', 'edge', 'stable'] + - container_manager == 'containerd' + +- name: Stop if using deprecated containerd_config variable + assert: + that: containerd_config is not defined + msg: "Variable containerd_config is now deprecated. See https://github.com/kubernetes-sigs/kubespray/blob/master/inventory/sample/group_vars/all/containerd.yml for details." + when: + - containerd_config is defined + - not ignore_assert_errors + +- name: Stop if auto_renew_certificates is enabled when certificates are managed externally (kube_external_ca_mode is true) + assert: + that: not auto_renew_certificates + msg: "Variable auto_renew_certificates must be disabled when CA are managed externally: kube_external_ca_mode = true" + when: + - kube_external_ca_mode + - not ignore_assert_errors + +- name: Stop if using deprecated comma separated list for admission plugins + assert: + that: "',' not in kube_apiserver_enable_admission_plugins[0]" + msg: "Comma-separated list for kube_apiserver_enable_admission_plugins is now deprecated, use separate list items for each plugin." + when: + - kube_apiserver_enable_admission_plugins is defined + - kube_apiserver_enable_admission_plugins | length > 0 diff --git a/kubespray/project/roles/kubernetes/preinstall/tasks/0050-create_directories.yml b/kubespray/project/roles/kubernetes/preinstall/tasks/0050-create_directories.yml new file mode 100644 index 0000000..f773989 --- /dev/null +++ b/kubespray/project/roles/kubernetes/preinstall/tasks/0050-create_directories.yml @@ -0,0 +1,119 @@ +--- +- name: Create kubernetes directories + file: + path: "{{ item }}" + state: directory + owner: "{{ kube_owner }}" + mode: 0755 + when: inventory_hostname in groups['k8s_cluster'] + become: true + tags: + - kubelet + - k8s-secrets + - kube-controller-manager + - kube-apiserver + - bootstrap-os + - apps + - network + - master + - node + with_items: + - "{{ kube_config_dir }}" + - "{{ kube_manifest_dir }}" + - "{{ kube_script_dir }}" + - "{{ kubelet_flexvolumes_plugins_dir }}" + +- name: Create other directories of root owner + file: + path: "{{ item }}" + state: directory + owner: root + mode: 0755 + when: inventory_hostname in groups['k8s_cluster'] + become: true + tags: + - kubelet + - k8s-secrets + - kube-controller-manager + - kube-apiserver + - bootstrap-os + - apps + - network + - master + - node + with_items: + - "{{ kube_cert_dir }}" + - "{{ bin_dir }}" + +- name: Check if kubernetes kubeadm compat cert dir exists + stat: + path: "{{ kube_cert_compat_dir }}" + get_attributes: no + get_checksum: no + get_mime: no + register: kube_cert_compat_dir_check + when: + - inventory_hostname in groups['k8s_cluster'] + - kube_cert_dir != kube_cert_compat_dir + +- name: Create kubernetes kubeadm compat cert dir (kubernetes/kubeadm issue 1498) + file: + src: "{{ kube_cert_dir }}" + dest: "{{ kube_cert_compat_dir }}" + state: link + mode: 0755 + when: + - inventory_hostname in groups['k8s_cluster'] + - kube_cert_dir != kube_cert_compat_dir + - not kube_cert_compat_dir_check.stat.exists + +- name: Create cni directories + file: + path: "{{ item }}" + state: directory + owner: "{{ kube_owner }}" + mode: 0755 + with_items: + - "/etc/cni/net.d" + - "/opt/cni/bin" + when: + - kube_network_plugin in ["calico", "weave", "flannel", "cilium", "kube-ovn", "kube-router", "macvlan"] + - inventory_hostname in groups['k8s_cluster'] + tags: + - network + - cilium + - calico + - weave + - kube-ovn + - kube-router + - bootstrap-os + +- name: Create calico cni directories + file: + path: "{{ item }}" + state: directory + owner: "{{ kube_owner }}" + mode: 0755 + with_items: + - "/var/lib/calico" + when: + - kube_network_plugin == "calico" + - inventory_hostname in groups['k8s_cluster'] + tags: + - network + - calico + - bootstrap-os + +- name: Create local volume provisioner directories + file: + path: "{{ local_volume_provisioner_storage_classes[item].host_dir }}" + state: directory + owner: root + group: root + mode: "{{ local_volume_provisioner_directory_mode }}" + with_items: "{{ local_volume_provisioner_storage_classes.keys() | list }}" + when: + - inventory_hostname in groups['k8s_cluster'] + - local_volume_provisioner_enabled + tags: + - persistent_volumes diff --git a/kubespray/project/roles/kubernetes/preinstall/tasks/0060-resolvconf.yml b/kubespray/project/roles/kubernetes/preinstall/tasks/0060-resolvconf.yml new file mode 100644 index 0000000..da5fc85 --- /dev/null +++ b/kubespray/project/roles/kubernetes/preinstall/tasks/0060-resolvconf.yml @@ -0,0 +1,58 @@ +--- +- name: Create temporary resolveconf cloud init file + command: cp -f /etc/resolv.conf "{{ resolvconffile }}" + when: ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] + +- name: Add domain/search/nameservers/options to resolv.conf + blockinfile: + path: "{{ resolvconffile }}" + block: |- + {% for item in [domainentry] + [searchentries] -%} + {{ item }} + {% endfor %} + {% for item in nameserverentries.split(',') %} + nameserver {{ item }} + {% endfor %} + options ndots:{{ ndots }} timeout:{{ dns_timeout | default('2') }} attempts:{{ dns_attempts | default('2') }} + state: present + insertbefore: BOF + create: yes + backup: "{{ not resolvconf_stat.stat.islnk }}" + marker: "# Ansible entries {mark}" + mode: 0644 + notify: Preinstall | propagate resolvconf to k8s components + +- name: Remove search/domain/nameserver options before block + replace: + path: "{{ item[0] }}" + regexp: '^{{ item[1] }}[^#]*(?=# Ansible entries BEGIN)' + backup: "{{ not resolvconf_stat.stat.islnk }}" + with_nested: + - "{{ [resolvconffile, base | default(''), head | default('')] | difference(['']) }}" + - [ 'search\s', 'nameserver\s', 'domain\s', 'options\s' ] + notify: Preinstall | propagate resolvconf to k8s components + +- name: Remove search/domain/nameserver options after block + replace: + path: "{{ item[0] }}" + regexp: '(# Ansible entries END\n(?:(?!^{{ item[1] }}).*\n)*)(?:^{{ item[1] }}.*\n?)+' + replace: '\1' + backup: "{{ not resolvconf_stat.stat.islnk }}" + with_nested: + - "{{ [resolvconffile, base | default(''), head | default('')] | difference(['']) }}" + - [ 'search\s', 'nameserver\s', 'domain\s', 'options\s' ] + notify: Preinstall | propagate resolvconf to k8s components + +- name: Get temporary resolveconf cloud init file content + command: cat {{ resolvconffile }} + register: cloud_config + when: ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] + +- name: Persist resolvconf cloud init file + template: + dest: "{{ resolveconf_cloud_init_conf }}" + src: resolvconf.j2 + owner: root + mode: 0644 + notify: Preinstall | update resolvconf for Flatcar Container Linux by Kinvolk + when: ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] diff --git a/kubespray/project/roles/kubernetes/preinstall/tasks/0061-systemd-resolved.yml b/kubespray/project/roles/kubernetes/preinstall/tasks/0061-systemd-resolved.yml new file mode 100644 index 0000000..9edec2e --- /dev/null +++ b/kubespray/project/roles/kubernetes/preinstall/tasks/0061-systemd-resolved.yml @@ -0,0 +1,15 @@ +--- +- name: Create systemd-resolved drop-in directory + file: + state: directory + name: /etc/systemd/resolved.conf.d/ + mode: 0755 + +- name: Write Kubespray DNS settings to systemd-resolved + template: + src: resolved.conf.j2 + dest: /etc/systemd/resolved.conf.d/kubespray.conf + owner: root + group: root + mode: 0644 + notify: Preinstall | Restart systemd-resolved diff --git a/kubespray/project/roles/kubernetes/preinstall/tasks/0062-networkmanager-unmanaged-devices.yml b/kubespray/project/roles/kubernetes/preinstall/tasks/0062-networkmanager-unmanaged-devices.yml new file mode 100644 index 0000000..44d6191 --- /dev/null +++ b/kubespray/project/roles/kubernetes/preinstall/tasks/0062-networkmanager-unmanaged-devices.yml @@ -0,0 +1,28 @@ +--- +- name: NetworkManager | Ensure NetworkManager conf.d dir + file: + path: "/etc/NetworkManager/conf.d" + state: directory + recurse: yes + +- name: NetworkManager | Prevent NetworkManager from managing Calico interfaces (cali*/tunl*/vxlan.calico) + copy: + content: | + [keyfile] + unmanaged-devices+=interface-name:cali*;interface-name:tunl*;interface-name:vxlan.calico;interface-name:vxlan-v6.calico + dest: /etc/NetworkManager/conf.d/calico.conf + mode: 0644 + when: + - kube_network_plugin == "calico" + notify: Preinstall | reload NetworkManager + +# TODO: add other network_plugin interfaces + +- name: NetworkManager | Prevent NetworkManager from managing K8S interfaces (kube-ipvs0/nodelocaldns) + copy: + content: | + [keyfile] + unmanaged-devices+=interface-name:kube-ipvs0;interface-name:nodelocaldns + dest: /etc/NetworkManager/conf.d/k8s.conf + mode: 0644 + notify: Preinstall | reload NetworkManager diff --git a/kubespray/project/roles/kubernetes/preinstall/tasks/0063-networkmanager-dns.yml b/kubespray/project/roles/kubernetes/preinstall/tasks/0063-networkmanager-dns.yml new file mode 100644 index 0000000..e155f0a --- /dev/null +++ b/kubespray/project/roles/kubernetes/preinstall/tasks/0063-networkmanager-dns.yml @@ -0,0 +1,37 @@ +--- +- name: NetworkManager | Add nameservers to NM configuration + community.general.ini_file: + path: /etc/NetworkManager/conf.d/dns.conf + section: global-dns-domain-* + option: servers + value: "{{ nameserverentries }}" + mode: '0600' + backup: yes + when: + - nameserverentries != "127.0.0.53" or systemd_resolved_enabled.rc != 0 + notify: Preinstall | update resolvconf for networkmanager + +- name: Set default dns if remove_default_searchdomains is false + set_fact: + default_searchdomains: ["default.svc.{{ dns_domain }}", "svc.{{ dns_domain }}"] + when: not remove_default_searchdomains | default() | bool or (remove_default_searchdomains | default() | bool and searchdomains | default([]) | length==0) + +- name: NetworkManager | Add DNS search to NM configuration + community.general.ini_file: + path: /etc/NetworkManager/conf.d/dns.conf + section: global-dns + option: searches + value: "{{ (default_searchdomains | default([]) + searchdomains | default([])) | join(',') }}" + mode: '0600' + backup: yes + notify: Preinstall | update resolvconf for networkmanager + +- name: NetworkManager | Add DNS options to NM configuration + community.general.ini_file: + path: /etc/NetworkManager/conf.d/dns.conf + section: global-dns + option: options + value: "ndots:{{ ndots }},timeout:{{ dns_timeout | default('2') }},attempts:{{ dns_attempts | default('2') }}" + mode: '0600' + backup: yes + notify: Preinstall | update resolvconf for networkmanager diff --git a/kubespray/project/roles/kubernetes/preinstall/tasks/0070-system-packages.yml b/kubespray/project/roles/kubernetes/preinstall/tasks/0070-system-packages.yml new file mode 100644 index 0000000..ccfb490 --- /dev/null +++ b/kubespray/project/roles/kubernetes/preinstall/tasks/0070-system-packages.yml @@ -0,0 +1,99 @@ +--- +- name: Update package management cache (zypper) - SUSE + command: zypper -n --gpg-auto-import-keys ref + register: make_cache_output + until: make_cache_output is succeeded + retries: 4 + delay: "{{ retry_stagger | random + 3 }}" + when: + - ansible_pkg_mgr == 'zypper' + tags: bootstrap-os + +- name: Add debian 10 required repos + when: + - ansible_distribution == "Debian" + - ansible_distribution_version == "10" + tags: + - bootstrap-os + block: + - name: Add Debian Backports apt repo + apt_repository: + repo: "deb http://deb.debian.org/debian {{ ansible_distribution_release }}-backports main" + state: present + filename: debian-backports + + - name: Set libseccomp2 pin priority to apt_preferences on Debian buster + copy: + content: | + Package: libseccomp2 + Pin: release a={{ ansible_distribution_release }}-backports + Pin-Priority: 1001 + dest: "/etc/apt/preferences.d/libseccomp2" + owner: "root" + mode: 0644 + +- name: Update package management cache (APT) + apt: + update_cache: yes + cache_valid_time: 3600 + when: ansible_os_family == "Debian" + tags: + - bootstrap-os + +- name: Remove legacy docker repo file + file: + path: "{{ yum_repo_dir }}/docker.repo" + state: absent + when: + - ansible_os_family == "RedHat" + - not is_fedora_coreos + +- name: Install python3-dnf for latest RedHat versions + command: dnf install -y python3-dnf + register: dnf_task_result + until: dnf_task_result is succeeded + retries: 4 + delay: "{{ retry_stagger | random + 3 }}" + when: + - ansible_distribution == "Fedora" + - ansible_distribution_major_version | int >= 30 + - not is_fedora_coreos + changed_when: False + tags: + - bootstrap-os + +- name: Install epel-release on RHEL derivatives + package: + name: epel-release + state: present + when: + - ansible_os_family == "RedHat" + - not is_fedora_coreos + - epel_enabled | bool + tags: + - bootstrap-os + +- name: Update common_required_pkgs with ipvsadm when kube_proxy_mode is ipvs + set_fact: + common_required_pkgs: "{{ common_required_pkgs | default([]) + ['ipvsadm', 'ipset'] }}" + when: kube_proxy_mode == 'ipvs' + +- name: Install packages requirements + package: + name: "{{ required_pkgs | default([]) | union(common_required_pkgs | default([])) }}" + state: present + register: pkgs_task_result + until: pkgs_task_result is succeeded + retries: "{{ pkg_install_retries }}" + delay: "{{ retry_stagger | random + 3 }}" + when: not (ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk", "ClearLinux"] or is_fedora_coreos) + tags: + - bootstrap-os + +- name: Install ipvsadm for ClearLinux + package: + name: ipvsadm + state: present + when: + - ansible_os_family in ["ClearLinux"] + - kube_proxy_mode == 'ipvs' diff --git a/kubespray/project/roles/kubernetes/preinstall/tasks/0080-system-configurations.yml b/kubespray/project/roles/kubernetes/preinstall/tasks/0080-system-configurations.yml new file mode 100644 index 0000000..8159dcc --- /dev/null +++ b/kubespray/project/roles/kubernetes/preinstall/tasks/0080-system-configurations.yml @@ -0,0 +1,145 @@ +--- +# Todo : selinux configuration +- name: Confirm selinux deployed + stat: + path: /etc/selinux/config + get_attributes: no + get_checksum: no + get_mime: no + when: + - ansible_os_family == "RedHat" + - "'Amazon' not in ansible_distribution" + register: slc + +- name: Set selinux policy + ansible.posix.selinux: + policy: targeted + state: "{{ preinstall_selinux_state }}" + when: + - ansible_os_family == "RedHat" + - "'Amazon' not in ansible_distribution" + - slc.stat.exists + tags: + - bootstrap-os + +- name: Disable IPv6 DNS lookup + lineinfile: + dest: /etc/gai.conf + line: "precedence ::ffff:0:0/96 100" + state: present + create: yes + backup: yes + mode: 0644 + when: + - disable_ipv6_dns + - not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] + tags: + - bootstrap-os + +- name: Clean previously used sysctl file locations + file: + path: "/etc/sysctl.d/{{ item }}" + state: absent + with_items: + - ipv4-ip_forward.conf + - bridge-nf-call.conf + +- name: Stat sysctl file configuration + stat: + path: "{{ sysctl_file_path }}" + get_attributes: no + get_checksum: no + get_mime: no + register: sysctl_file_stat + tags: + - bootstrap-os + +- name: Change sysctl file path to link source if linked + set_fact: + sysctl_file_path: "{{ sysctl_file_stat.stat.lnk_source }}" + when: + - sysctl_file_stat.stat.islnk is defined + - sysctl_file_stat.stat.islnk + tags: + - bootstrap-os + +- name: Make sure sysctl file path folder exists + file: + name: "{{ sysctl_file_path | dirname }}" + state: directory + mode: 0755 + +- name: Enable ip forwarding + ansible.posix.sysctl: + sysctl_file: "{{ sysctl_file_path }}" + name: net.ipv4.ip_forward + value: "1" + state: present + reload: yes + +- name: Enable ipv6 forwarding + ansible.posix.sysctl: + sysctl_file: "{{ sysctl_file_path }}" + name: net.ipv6.conf.all.forwarding + value: "1" + state: present + reload: yes + when: enable_dual_stack_networks | bool + +- name: Check if we need to set fs.may_detach_mounts + stat: + path: /proc/sys/fs/may_detach_mounts + get_attributes: no + get_checksum: no + get_mime: no + register: fs_may_detach_mounts + ignore_errors: true # noqa ignore-errors + +- name: Set fs.may_detach_mounts if needed + ansible.posix.sysctl: + sysctl_file: "{{ sysctl_file_path }}" + name: fs.may_detach_mounts + value: 1 + state: present + reload: yes + when: fs_may_detach_mounts.stat.exists | d(false) + +- name: Ensure kube-bench parameters are set + ansible.posix.sysctl: + sysctl_file: "{{ sysctl_file_path }}" + name: "{{ item.name }}" + value: "{{ item.value }}" + state: present + reload: yes + with_items: + - { name: kernel.keys.root_maxbytes, value: 25000000 } + - { name: kernel.keys.root_maxkeys, value: 1000000 } + - { name: kernel.panic, value: 10 } + - { name: kernel.panic_on_oops, value: 1 } + - { name: vm.overcommit_memory, value: 1 } + - { name: vm.panic_on_oom, value: 0 } + when: kubelet_protect_kernel_defaults | bool + +- name: Check dummy module + community.general.modprobe: + name: dummy + state: present + params: 'numdummies=0' + when: enable_nodelocaldns + +- name: Set additional sysctl variables + ansible.posix.sysctl: + sysctl_file: "{{ sysctl_file_path }}" + name: "{{ item.name }}" + value: "{{ item.value }}" + state: present + reload: yes + with_items: "{{ additional_sysctl }}" + +- name: Disable fapolicyd service + failed_when: false + systemd: + name: fapolicyd + state: stopped + enabled: false + when: disable_fapolicyd diff --git a/kubespray/project/roles/kubernetes/preinstall/tasks/0081-ntp-configurations.yml b/kubespray/project/roles/kubernetes/preinstall/tasks/0081-ntp-configurations.yml new file mode 100644 index 0000000..da4b312 --- /dev/null +++ b/kubespray/project/roles/kubernetes/preinstall/tasks/0081-ntp-configurations.yml @@ -0,0 +1,87 @@ +--- +- name: Ensure NTP package + package: + name: + - "{{ ntp_package }}" + state: present + when: + - not is_fedora_coreos + - not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] + +- name: Disable systemd-timesyncd + service: + name: systemd-timesyncd.service + enabled: false + state: stopped + failed_when: false + +- name: Set fact NTP settings + set_fact: + # noqa: jinja[spacing] + ntp_config_file: >- + {% if ntp_package == "ntp" -%} + /etc/ntp.conf + {%- elif ansible_os_family in ['RedHat', 'Suse'] -%} + /etc/chrony.conf + {%- else -%} + /etc/chrony/chrony.conf + {%- endif -%} + # noqa: jinja[spacing] + ntp_service_name: >- + {% if ntp_package == "chrony" -%} + chronyd + {%- elif ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk", "RedHat", "Suse"] -%} + ntpd + {%- else -%} + ntp + {%- endif %} + +- name: Generate NTP configuration file. + template: + src: "{{ ntp_config_file | basename }}.j2" + dest: "{{ ntp_config_file }}" + mode: 0644 + notify: Preinstall | restart ntp + when: + - ntp_manage_config + +- name: Stop the NTP Deamon For Sync Immediately # `ntpd -gq`,`chronyd -q` requires the ntp daemon stop + service: + name: "{{ ntp_service_name }}" + state: stopped + when: + - ntp_force_sync_immediately + +- name: Force Sync NTP Immediately + # noqa: jinja[spacing] + command: >- + timeout -k 60s 60s + {% if ntp_package == "ntp" -%} + ntpd -gq + {%- else -%} + chronyd -q + {%- endif -%} + when: + - ntp_force_sync_immediately + +- name: Ensure NTP service is started and enabled + service: + name: "{{ ntp_service_name }}" + state: started + enabled: true + +- name: Ensure tzdata package + package: + name: + - tzdata + state: present + when: + - ntp_timezone + - not is_fedora_coreos + - not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] + +- name: Set timezone + community.general.timezone: + name: "{{ ntp_timezone }}" + when: + - ntp_timezone diff --git a/kubespray/project/roles/kubernetes/preinstall/tasks/0090-etchosts.yml b/kubespray/project/roles/kubernetes/preinstall/tasks/0090-etchosts.yml new file mode 100644 index 0000000..6bec169 --- /dev/null +++ b/kubespray/project/roles/kubernetes/preinstall/tasks/0090-etchosts.yml @@ -0,0 +1,81 @@ +--- +- name: Hosts | create hosts list from inventory + set_fact: + etc_hosts_inventory_block: |- + {% for item in (groups['k8s_cluster'] + groups['etcd'] | default([]) + groups['calico_rr'] | default([])) | unique -%} + {% if 'access_ip' in hostvars[item] or 'ip' in hostvars[item] or 'ansible_default_ipv4' in hostvars[item] -%} + {{ hostvars[item]['access_ip'] | default(hostvars[item]['ip'] | default(hostvars[item]['ansible_default_ipv4']['address'])) }} + {%- if ('ansible_hostname' in hostvars[item] and item != hostvars[item]['ansible_hostname']) %} {{ hostvars[item]['ansible_hostname'] }}.{{ dns_domain }} {{ hostvars[item]['ansible_hostname'] }} {% else %} {{ item }}.{{ dns_domain }} {{ item }} {% endif %} + + {% endif %} + {% endfor %} + delegate_to: localhost + connection: local + delegate_facts: yes + run_once: yes + +- name: Hosts | populate inventory into hosts file + blockinfile: + path: /etc/hosts + block: "{{ hostvars.localhost.etc_hosts_inventory_block }}" + state: "{{ 'present' if populate_inventory_to_hosts_file else 'absent' }}" + create: yes + backup: yes + unsafe_writes: yes + marker: "# Ansible inventory hosts {mark}" + mode: 0644 + +- name: Hosts | populate kubernetes loadbalancer address into hosts file + lineinfile: + dest: /etc/hosts + regexp: ".*{{ apiserver_loadbalancer_domain_name }}$" + line: "{{ loadbalancer_apiserver.address }} {{ apiserver_loadbalancer_domain_name }}" + state: present + backup: yes + unsafe_writes: yes + when: + - populate_loadbalancer_apiserver_to_hosts_file + - loadbalancer_apiserver is defined + - loadbalancer_apiserver.address is defined + +- name: Hosts | Update localhost entries in hosts file + when: populate_localhost_entries_to_hosts_file + block: + - name: Hosts | Retrieve hosts file content + slurp: + src: /etc/hosts + register: etc_hosts_content + + - name: Hosts | Extract existing entries for localhost from hosts file + set_fact: + etc_hosts_localhosts_dict: >- + {%- set splitted = (item | regex_replace('[ \t]+', ' ') | regex_replace('#.*$') | trim).split(' ') -%} + {{ etc_hosts_localhosts_dict | default({}) | combine({splitted[0]: splitted[1::]}) }} + with_items: "{{ (etc_hosts_content['content'] | b64decode).splitlines() }}" + when: + - etc_hosts_content.content is defined + - (item is match('^::1 .*') or item is match('^127.0.0.1 .*')) + + - name: Hosts | Update target hosts file entries dict with required entries + set_fact: + etc_hosts_localhosts_dict_target: >- + {%- set target_entries = (etc_hosts_localhosts_dict | default({})).get(item.key, []) | difference(item.value.get('unexpected', [])) -%} + {{ etc_hosts_localhosts_dict_target | default({}) | combine({item.key: (target_entries + item.value.expected) | unique}) }} + loop: "{{ etc_hosts_localhost_entries | dict2items }}" + + - name: Hosts | Update (if necessary) hosts file + lineinfile: + dest: /etc/hosts + line: "{{ item.key }} {{ item.value | join(' ') }}" + regexp: "^{{ item.key }}.*$" + state: present + backup: yes + unsafe_writes: yes + loop: "{{ etc_hosts_localhosts_dict_target | default({}) | dict2items }}" + +# gather facts to update ansible_fqdn +- name: Update facts + setup: + gather_subset: min + when: + - not dns_late diff --git a/kubespray/project/roles/kubernetes/preinstall/tasks/0100-dhclient-hooks.yml b/kubespray/project/roles/kubernetes/preinstall/tasks/0100-dhclient-hooks.yml new file mode 100644 index 0000000..da38147 --- /dev/null +++ b/kubespray/project/roles/kubernetes/preinstall/tasks/0100-dhclient-hooks.yml @@ -0,0 +1,33 @@ +--- +- name: Configure dhclient to supersede search/domain/nameservers + blockinfile: + block: |- + {% for item in [supersede_domain, supersede_search, supersede_nameserver] -%} + {{ item }} + {% endfor %} + path: "{{ dhclientconffile }}" + create: yes + state: present + insertbefore: BOF + backup: yes + marker: "# Ansible entries {mark}" + mode: 0644 + notify: Preinstall | propagate resolvconf to k8s components + +- name: Configure dhclient hooks for resolv.conf (non-RH) + template: + src: dhclient_dnsupdate.sh.j2 + dest: "{{ dhclienthookfile }}" + owner: root + mode: 0755 + notify: Preinstall | propagate resolvconf to k8s components + when: ansible_os_family not in [ "RedHat", "Suse" ] + +- name: Configure dhclient hooks for resolv.conf (RH-only) + template: + src: dhclient_dnsupdate_rh.sh.j2 + dest: "{{ dhclienthookfile }}" + owner: root + mode: 0755 + notify: Preinstall | propagate resolvconf to k8s components + when: ansible_os_family == "RedHat" diff --git a/kubespray/project/roles/kubernetes/preinstall/tasks/0110-dhclient-hooks-undo.yml b/kubespray/project/roles/kubernetes/preinstall/tasks/0110-dhclient-hooks-undo.yml new file mode 100644 index 0000000..024e39f --- /dev/null +++ b/kubespray/project/roles/kubernetes/preinstall/tasks/0110-dhclient-hooks-undo.yml @@ -0,0 +1,18 @@ +--- + +# These tasks will undo changes done by kubespray in the past if needed (e.g. when upgrading from kubespray 2.0.x +# or when changing resolvconf_mode) + +- name: Remove kubespray specific config from dhclient config + blockinfile: + path: "{{ dhclientconffile }}" + state: absent + backup: yes + marker: "# Ansible entries {mark}" + notify: Preinstall | propagate resolvconf to k8s components + +- name: Remove kubespray specific dhclient hook + file: + path: "{{ dhclienthookfile }}" + state: absent + notify: Preinstall | propagate resolvconf to k8s components diff --git a/kubespray/project/roles/kubernetes/preinstall/tasks/0120-growpart-azure-centos-7.yml b/kubespray/project/roles/kubernetes/preinstall/tasks/0120-growpart-azure-centos-7.yml new file mode 100644 index 0000000..621629f --- /dev/null +++ b/kubespray/project/roles/kubernetes/preinstall/tasks/0120-growpart-azure-centos-7.yml @@ -0,0 +1,44 @@ +--- + +# Running growpart seems to be only required on Azure, as other Cloud Providers do this at boot time + +- name: Install growpart + package: + name: cloud-utils-growpart + state: present + +- name: Gather mounts facts + setup: + gather_subset: 'mounts' + +- name: Search root filesystem device + vars: + query: "[?mount=='/'].device" + _root_device: "{{ ansible_mounts | json_query(query) }}" + set_fact: + device: "{{ _root_device | first | regex_replace('([^0-9]+)[0-9]+', '\\1') }}" + partition: "{{ _root_device | first | regex_replace('[^0-9]+([0-9]+)', '\\1') }}" + root_device: "{{ _root_device }}" + +- name: Check if growpart needs to be run + command: growpart -N {{ device }} {{ partition }} + failed_when: False + changed_when: "'NOCHANGE:' not in growpart_needed.stdout" + register: growpart_needed + environment: + LC_ALL: C + +- name: Check fs type + command: file -Ls {{ root_device }} + changed_when: False + register: fs_type + +- name: Run growpart # noqa no-handler + command: growpart {{ device }} {{ partition }} + when: growpart_needed.changed + environment: + LC_ALL: C + +- name: Run xfs_growfs # noqa no-handler + command: xfs_growfs {{ root_device }} + when: growpart_needed.changed and 'XFS' in fs_type.stdout diff --git a/kubespray/project/roles/kubernetes/preinstall/tasks/main.yml b/kubespray/project/roles/kubernetes/preinstall/tasks/main.yml new file mode 100644 index 0000000..ee4de5d --- /dev/null +++ b/kubespray/project/roles/kubernetes/preinstall/tasks/main.yml @@ -0,0 +1,148 @@ +--- +# Disable swap +- name: Disable swap + import_tasks: 0010-swapoff.yml + when: + - not dns_late + - kubelet_fail_swap_on + +- name: Set facts + import_tasks: 0020-set_facts.yml + tags: + - resolvconf + - facts + +- name: Check settings + import_tasks: 0040-verify-settings.yml + when: + - not dns_late + tags: + - asserts + +- name: Create directories + import_tasks: 0050-create_directories.yml + when: + - not dns_late + +- name: Apply resolvconf settings + import_tasks: 0060-resolvconf.yml + when: + - dns_mode != 'none' + - resolvconf_mode == 'host_resolvconf' + - systemd_resolved_enabled.rc != 0 + - networkmanager_enabled.rc != 0 + tags: + - bootstrap-os + - resolvconf + +- name: Apply systemd-resolved settings + import_tasks: 0061-systemd-resolved.yml + when: + - dns_mode != 'none' + - resolvconf_mode == 'host_resolvconf' + - systemd_resolved_enabled.rc == 0 + tags: + - bootstrap-os + - resolvconf + +- name: Apply networkmanager unmanaged devices settings + import_tasks: 0062-networkmanager-unmanaged-devices.yml + when: + - networkmanager_enabled.rc == 0 + tags: + - bootstrap-os + +- name: Apply networkmanager DNS settings + import_tasks: 0063-networkmanager-dns.yml + when: + - dns_mode != 'none' + - resolvconf_mode == 'host_resolvconf' + - networkmanager_enabled.rc == 0 + tags: + - bootstrap-os + - resolvconf + +- name: Install required system packages + import_tasks: 0070-system-packages.yml + when: + - not dns_late + tags: + - bootstrap-os + +- name: Apply system configurations + import_tasks: 0080-system-configurations.yml + when: + - not dns_late + tags: + - bootstrap-os + +- name: Configure NTP + import_tasks: 0081-ntp-configurations.yml + when: + - not dns_late + - ntp_enabled + tags: + - bootstrap-os + +- name: Configure /etc/hosts + import_tasks: 0090-etchosts.yml + tags: + - bootstrap-os + - etchosts + +- name: Configure dhclient + import_tasks: 0100-dhclient-hooks.yml + when: + - dns_mode != 'none' + - resolvconf_mode == 'host_resolvconf' + - dhclientconffile is defined + - not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] + tags: + - bootstrap-os + - resolvconf + +- name: Configure dhclient dhclient hooks + import_tasks: 0110-dhclient-hooks-undo.yml + when: + - dns_mode != 'none' + - resolvconf_mode != 'host_resolvconf' + - dhclientconffile is defined + - not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] + tags: + - bootstrap-os + - resolvconf + +# We need to make sure the network is restarted early enough so that docker can later pick up the correct system +# nameservers and search domains +- name: Flush handlers + meta: flush_handlers + +- name: Check if we are running inside a Azure VM + stat: + path: /var/lib/waagent/ + get_attributes: no + get_checksum: no + get_mime: no + register: azure_check + when: + - not dns_late + tags: + - bootstrap-os + +- name: Grow partition on azure CentOS + import_tasks: 0120-growpart-azure-centos-7.yml + when: + - not dns_late + - azure_check.stat.exists + - ansible_os_family == "RedHat" + - growpart_azure_enabled + tags: + - bootstrap-os + +- name: Run calico checks + include_role: + name: network_plugin/calico + tasks_from: check + when: + - kube_network_plugin == 'calico' + - not ignore_assert_errors diff --git a/kubespray/project/roles/kubernetes/preinstall/templates/ansible_git.j2 b/kubespray/project/roles/kubernetes/preinstall/templates/ansible_git.j2 new file mode 100644 index 0000000..abf92a7 --- /dev/null +++ b/kubespray/project/roles/kubernetes/preinstall/templates/ansible_git.j2 @@ -0,0 +1,3 @@ +; This file contains the information which identifies the deployment state relative to the git repo +[default] +{{ gitinfo.stdout }} diff --git a/kubespray/project/roles/kubernetes/preinstall/templates/chrony.conf.j2 b/kubespray/project/roles/kubernetes/preinstall/templates/chrony.conf.j2 new file mode 100644 index 0000000..7931f43 --- /dev/null +++ b/kubespray/project/roles/kubernetes/preinstall/templates/chrony.conf.j2 @@ -0,0 +1,27 @@ +# {{ ansible_managed }} + +# Specify one or more NTP servers. +# Use public servers from the pool.ntp.org project. +# Please consider joining the pool (http://www.pool.ntp.org/join.html). +{% for server in ntp_servers %} +server {{ server }} +{% endfor %} + +# Record the rate at which the system clock gains/losses time. +driftfile /var/lib/chrony/drift + +{% if ntp_tinker_panic is sameas true %} +# Force time sync if the drift exceeds the threshold specified +# Useful for VMs that can be paused and much later resumed. +makestep 1.0 -1 +{% else %} +# Allow the system clock to be stepped in the first three updates +# if its offset is larger than 1 second. +makestep 1.0 3 +{% endif %} + +# Enable kernel synchronization of the real-time clock (RTC). +rtcsync + +# Specify directory for log files. +logdir /var/log/chrony diff --git a/kubespray/project/roles/kubernetes/preinstall/templates/dhclient_dnsupdate.sh.j2 b/kubespray/project/roles/kubernetes/preinstall/templates/dhclient_dnsupdate.sh.j2 new file mode 100644 index 0000000..8cf8b81 --- /dev/null +++ b/kubespray/project/roles/kubernetes/preinstall/templates/dhclient_dnsupdate.sh.j2 @@ -0,0 +1,13 @@ +#!/bin/sh +# +# Prepend resolver options to /etc/resolv.conf after dhclient` +# regenerates the file. See man (5) resolver for more details. +# +if [ $reason = "BOUND" ]; then + if [ -n "$new_domain_search" -o -n "$new_domain_name_servers" ]; then + RESOLV_CONF=$(cat /etc/resolv.conf | sed -r '/^options (timeout|attempts|ndots).*$/d') + OPTIONS="options timeout:{{ dns_timeout|default('2') }} attempts:{{ dns_attempts|default('2') }} ndots:{{ ndots }}" + + printf "%b\n" "$RESOLV_CONF\n$OPTIONS" > /etc/resolv.conf + fi +fi diff --git a/kubespray/project/roles/kubernetes/preinstall/templates/dhclient_dnsupdate_rh.sh.j2 b/kubespray/project/roles/kubernetes/preinstall/templates/dhclient_dnsupdate_rh.sh.j2 new file mode 100644 index 0000000..511839f --- /dev/null +++ b/kubespray/project/roles/kubernetes/preinstall/templates/dhclient_dnsupdate_rh.sh.j2 @@ -0,0 +1,17 @@ +#!/bin/sh +# +# Prepend resolver options to /etc/resolv.conf after dhclient` +# regenerates the file. See man (5) resolver for more details. +# +zdnsupdate_config() { + if [ -n "$new_domain_search" -o -n "$new_domain_name_servers" ]; then + RESOLV_CONF=$(cat /etc/resolv.conf | sed -r '/^options (timeout|attempts|ndots).*$/d') + OPTIONS="options timeout:{{ dns_timeout|default('2') }} attempts:{{ dns_attempts|default('2') }} ndots:{{ ndots }}" + + echo -e "$RESOLV_CONF\n$OPTIONS" > /etc/resolv.conf + fi +} + +zdnsupdate_restore() { + : +} diff --git a/kubespray/project/roles/kubernetes/preinstall/templates/ntp.conf.j2 b/kubespray/project/roles/kubernetes/preinstall/templates/ntp.conf.j2 new file mode 100644 index 0000000..abeb899 --- /dev/null +++ b/kubespray/project/roles/kubernetes/preinstall/templates/ntp.conf.j2 @@ -0,0 +1,45 @@ +# {{ ansible_managed }} + +# /etc/ntp.conf, configuration for ntpd; see ntp.conf(5) for help + +driftfile {{ ntp_driftfile }} + +{% if ntp_tinker_panic is sameas true %} +# Always reset the clock, even if the new time is more than 1000s away +# from the current system time. Useful for VMs that can be paused +# and much later resumed. +tinker panic 0 +{% endif %} + +# Specify one or more NTP servers. +# Use public servers from the pool.ntp.org project. +# Please consider joining the pool (http://www.pool.ntp.org/join.html). +{% for item in ntp_servers %} +pool {{ item }} +{% endfor %} + +# Access control configuration; see /usr/share/doc/ntp-doc/html/accopt.html for +# details. The web page +# might also be helpful. +# +# Note that "restrict" applies to both servers and clients, so a configuration +# that might be intended to block requests from certain clients could also end +# up blocking replies from your own upstream servers. + +# By default, exchange time with everybody, but don't allow configuration. +restrict -4 default kod notrap nomodify nopeer noquery limited +restrict -6 default kod notrap nomodify nopeer noquery limited + +# Local users may interrogate the ntp server more closely. +{% for item in ntp_restrict %} +restrict {{ item }} +{% endfor %} + +# Needed for adding pool entries +restrict source notrap nomodify noquery + +# Disable the monitoring facility to prevent amplification attacks using ntpdc +# monlist command when default restrict does not include the noquery flag. See +# CVE-2013-5211 for more details. +# Note: Monitoring will not be disabled with the limited restriction flag. +disable monitor diff --git a/kubespray/project/roles/kubernetes/preinstall/templates/resolvconf.j2 b/kubespray/project/roles/kubernetes/preinstall/templates/resolvconf.j2 new file mode 100644 index 0000000..807fdd0 --- /dev/null +++ b/kubespray/project/roles/kubernetes/preinstall/templates/resolvconf.j2 @@ -0,0 +1,10 @@ +#cloud-config +write_files: + - path: "/etc/resolv.conf" + permissions: "0644" + owner: "root" + content: | + {% for l in cloud_config.stdout_lines %} + {{ l }} + {% endfor %} + # diff --git a/kubespray/project/roles/kubernetes/preinstall/templates/resolved.conf.j2 b/kubespray/project/roles/kubernetes/preinstall/templates/resolved.conf.j2 new file mode 100644 index 0000000..edafbf9 --- /dev/null +++ b/kubespray/project/roles/kubernetes/preinstall/templates/resolved.conf.j2 @@ -0,0 +1,15 @@ +[Resolve] +{% if not dns_early and dns_late %} +DNS={{ ([nodelocaldns_ip] if enable_nodelocaldns else coredns_server )| list | join(' ') }} +{% endif %} +FallbackDNS={{ ( upstream_dns_servers|d([]) + nameservers|d([]) + cloud_resolver|d([])) | unique | join(' ') }} +{% if remove_default_searchdomains and searchdomains|default([])|length != 0 %} +Domains={{ searchdomains|default([]) | join(' ') }} +{% else %} +Domains={{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([])) | join(' ') }} +{% endif %} +DNSSEC=no +Cache=no-negative +{% if systemd_resolved_disable_stub_listener | bool %} +DNSStubListener=no +{% endif %} diff --git a/kubespray/project/roles/kubernetes/preinstall/vars/amazon.yml b/kubespray/project/roles/kubernetes/preinstall/vars/amazon.yml new file mode 100644 index 0000000..09c645f --- /dev/null +++ b/kubespray/project/roles/kubernetes/preinstall/vars/amazon.yml @@ -0,0 +1,7 @@ +--- +required_pkgs: + - libselinux-python + - device-mapper-libs + - nss + - conntrack-tools + - libseccomp diff --git a/kubespray/project/roles/kubernetes/preinstall/vars/centos.yml b/kubespray/project/roles/kubernetes/preinstall/vars/centos.yml new file mode 100644 index 0000000..9b1a874 --- /dev/null +++ b/kubespray/project/roles/kubernetes/preinstall/vars/centos.yml @@ -0,0 +1,8 @@ +--- +required_pkgs: + - "{{ ((ansible_distribution_major_version | int) < 8) | ternary('libselinux-python', 'python3-libselinux') }}" + - device-mapper-libs + - nss + - conntrack + - container-selinux + - libseccomp diff --git a/kubespray/project/roles/kubernetes/preinstall/vars/debian-11.yml b/kubespray/project/roles/kubernetes/preinstall/vars/debian-11.yml new file mode 100644 index 0000000..59cbc5a --- /dev/null +++ b/kubespray/project/roles/kubernetes/preinstall/vars/debian-11.yml @@ -0,0 +1,10 @@ +--- +required_pkgs: + - python3-apt + - gnupg + - apt-transport-https + - software-properties-common + - conntrack + - iptables + - apparmor + - libseccomp2 diff --git a/kubespray/project/roles/kubernetes/preinstall/vars/debian-12.yml b/kubespray/project/roles/kubernetes/preinstall/vars/debian-12.yml new file mode 100644 index 0000000..e0dca4d --- /dev/null +++ b/kubespray/project/roles/kubernetes/preinstall/vars/debian-12.yml @@ -0,0 +1,11 @@ +--- +required_pkgs: + - python3-apt + - gnupg + - apt-transport-https + - software-properties-common + - conntrack + - iptables + - apparmor + - libseccomp2 + - mergerfs diff --git a/kubespray/project/roles/kubernetes/preinstall/vars/debian.yml b/kubespray/project/roles/kubernetes/preinstall/vars/debian.yml new file mode 100644 index 0000000..51a2802 --- /dev/null +++ b/kubespray/project/roles/kubernetes/preinstall/vars/debian.yml @@ -0,0 +1,9 @@ +--- +required_pkgs: + - python-apt + - aufs-tools + - apt-transport-https + - software-properties-common + - conntrack + - apparmor + - libseccomp2 diff --git a/kubespray/project/roles/kubernetes/preinstall/vars/fedora.yml b/kubespray/project/roles/kubernetes/preinstall/vars/fedora.yml new file mode 100644 index 0000000..d69b111 --- /dev/null +++ b/kubespray/project/roles/kubernetes/preinstall/vars/fedora.yml @@ -0,0 +1,8 @@ +--- +required_pkgs: + - iptables + - libselinux-python3 + - device-mapper-libs + - conntrack + - container-selinux + - libseccomp diff --git a/kubespray/project/roles/kubernetes/preinstall/vars/redhat.yml b/kubespray/project/roles/kubernetes/preinstall/vars/redhat.yml new file mode 100644 index 0000000..9b1a874 --- /dev/null +++ b/kubespray/project/roles/kubernetes/preinstall/vars/redhat.yml @@ -0,0 +1,8 @@ +--- +required_pkgs: + - "{{ ((ansible_distribution_major_version | int) < 8) | ternary('libselinux-python', 'python3-libselinux') }}" + - device-mapper-libs + - nss + - conntrack + - container-selinux + - libseccomp diff --git a/kubespray/project/roles/kubernetes/preinstall/vars/suse.yml b/kubespray/project/roles/kubernetes/preinstall/vars/suse.yml new file mode 100644 index 0000000..d089ac1 --- /dev/null +++ b/kubespray/project/roles/kubernetes/preinstall/vars/suse.yml @@ -0,0 +1,5 @@ +--- +required_pkgs: + - device-mapper + - conntrack-tools + - libseccomp2 diff --git a/kubespray/project/roles/kubernetes/preinstall/vars/ubuntu.yml b/kubespray/project/roles/kubernetes/preinstall/vars/ubuntu.yml new file mode 100644 index 0000000..85b3f25 --- /dev/null +++ b/kubespray/project/roles/kubernetes/preinstall/vars/ubuntu.yml @@ -0,0 +1,8 @@ +--- +required_pkgs: + - python3-apt + - apt-transport-https + - software-properties-common + - conntrack + - apparmor + - libseccomp2 diff --git a/kubespray/project/roles/kubernetes/tokens/files/kube-gen-token.sh b/kubespray/project/roles/kubernetes/tokens/files/kube-gen-token.sh new file mode 100644 index 0000000..121b522 --- /dev/null +++ b/kubespray/project/roles/kubernetes/tokens/files/kube-gen-token.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +token_dir=${TOKEN_DIR:-/var/srv/kubernetes} +token_file="${token_dir}/known_tokens.csv" + +create_accounts=($@) + +if [ ! -e "${token_file}" ]; then + touch "${token_file}" +fi + +for account in "${create_accounts[@]}"; do + if grep ",${account}," "${token_file}" ; then + continue + fi + token=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) + echo "${token},${account},${account}" >> "${token_file}" + echo "${token}" > "${token_dir}/${account}.token" + echo "Added ${account}" +done diff --git a/kubespray/project/roles/kubernetes/tokens/tasks/check-tokens.yml b/kubespray/project/roles/kubernetes/tokens/tasks/check-tokens.yml new file mode 100644 index 0000000..a157a05 --- /dev/null +++ b/kubespray/project/roles/kubernetes/tokens/tasks/check-tokens.yml @@ -0,0 +1,41 @@ +--- +- name: "Check_tokens | check if the tokens have already been generated on first master" + stat: + path: "{{ kube_token_dir }}/known_tokens.csv" + get_attributes: no + get_checksum: yes + get_mime: no + delegate_to: "{{ groups['kube_control_plane'][0] }}" + register: known_tokens_master + run_once: true + +- name: "Check_tokens | Set default value for 'sync_tokens' and 'gen_tokens' to false" + set_fact: + sync_tokens: false + gen_tokens: false + +- name: "Check_tokens | Set 'sync_tokens' and 'gen_tokens' to true" + set_fact: + gen_tokens: true + when: not known_tokens_master.stat.exists and kube_token_auth | default(true) + run_once: true + +- name: "Check tokens | check if a cert already exists" + stat: + path: "{{ kube_token_dir }}/known_tokens.csv" + get_attributes: no + get_checksum: yes + get_mime: no + register: known_tokens + +- name: "Check_tokens | Set 'sync_tokens' to true" + set_fact: + sync_tokens: >- + {%- set tokens = {'sync': False} -%} + {%- for server in groups['kube_control_plane'] | intersect(ansible_play_batch) + if (not hostvars[server].known_tokens.stat.exists) or + (hostvars[server].known_tokens.stat.checksum | default('') != known_tokens_master.stat.checksum | default('')) -%} + {%- set _ = tokens.update({'sync': True}) -%} + {%- endfor -%} + {{ tokens.sync }} + run_once: true diff --git a/kubespray/project/roles/kubernetes/tokens/tasks/gen_tokens.yml b/kubespray/project/roles/kubernetes/tokens/tasks/gen_tokens.yml new file mode 100644 index 0000000..6ac6b49 --- /dev/null +++ b/kubespray/project/roles/kubernetes/tokens/tasks/gen_tokens.yml @@ -0,0 +1,63 @@ +--- +- name: Gen_tokens | copy tokens generation script + copy: + src: "kube-gen-token.sh" + dest: "{{ kube_script_dir }}/kube-gen-token.sh" + mode: 0700 + run_once: yes + delegate_to: "{{ groups['kube_control_plane'][0] }}" + when: gen_tokens | default(false) + +- name: Gen_tokens | generate tokens for master components + command: "{{ kube_script_dir }}/kube-gen-token.sh {{ item[0] }}-{{ item[1] }}" + environment: + TOKEN_DIR: "{{ kube_token_dir }}" + with_nested: + - [ "system:kubectl" ] + - "{{ groups['kube_control_plane'] }}" + register: gentoken_master + changed_when: "'Added' in gentoken_master.stdout" + run_once: yes + delegate_to: "{{ groups['kube_control_plane'][0] }}" + when: gen_tokens | default(false) + +- name: Gen_tokens | generate tokens for node components + command: "{{ kube_script_dir }}/kube-gen-token.sh {{ item[0] }}-{{ item[1] }}" + environment: + TOKEN_DIR: "{{ kube_token_dir }}" + with_nested: + - [ 'system:kubelet' ] + - "{{ groups['kube_node'] }}" + register: gentoken_node + changed_when: "'Added' in gentoken_node.stdout" + run_once: yes + delegate_to: "{{ groups['kube_control_plane'][0] }}" + when: gen_tokens | default(false) + +- name: Gen_tokens | Get list of tokens from first master + command: "find {{ kube_token_dir }} -maxdepth 1 -type f" + register: tokens_list + check_mode: no + delegate_to: "{{ groups['kube_control_plane'][0] }}" + run_once: true + when: sync_tokens | default(false) + +- name: Gen_tokens | Gather tokens + shell: "set -o pipefail && tar cfz - {{ tokens_list.stdout_lines | join(' ') }} | base64 --wrap=0" + args: + executable: /bin/bash + register: tokens_data + check_mode: no + delegate_to: "{{ groups['kube_control_plane'][0] }}" + run_once: true + when: sync_tokens | default(false) + +- name: Gen_tokens | Copy tokens on masters + shell: "set -o pipefail && echo '{{ tokens_data.stdout | quote }}' | base64 -d | tar xz -C /" + args: + executable: /bin/bash + when: + - inventory_hostname in groups['kube_control_plane'] + - sync_tokens | default(false) + - inventory_hostname != groups['kube_control_plane'][0] + - tokens_data.stdout diff --git a/kubespray/project/roles/kubernetes/tokens/tasks/main.yml b/kubespray/project/roles/kubernetes/tokens/tasks/main.yml new file mode 100644 index 0000000..c9dfd07 --- /dev/null +++ b/kubespray/project/roles/kubernetes/tokens/tasks/main.yml @@ -0,0 +1,21 @@ +--- + +- name: Check tokens + import_tasks: check-tokens.yml + tags: + - k8s-secrets + - k8s-gen-tokens + - facts + +- name: Make sure the tokens directory exits + file: + path: "{{ kube_token_dir }}" + state: directory + mode: 0644 + group: "{{ kube_cert_group }}" + +- name: Generate tokens + import_tasks: gen_tokens.yml + tags: + - k8s-secrets + - k8s-gen-tokens diff --git a/kubespray/project/roles/kubespray-defaults/defaults/main/checksums.yml b/kubespray/project/roles/kubespray-defaults/defaults/main/checksums.yml new file mode 100644 index 0000000..963ec23 --- /dev/null +++ b/kubespray/project/roles/kubespray-defaults/defaults/main/checksums.yml @@ -0,0 +1,1486 @@ +--- +crictl_checksums: + arm: + v1.29.0: 0019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5 + v1.28.0: 1ea267f3872f4b7f311963ab43ce6653ceeaf8727206c889b56587c95497e9dd + v1.27.1: ec24fb7e4d45b7f3f3df254b22333839f9bdbde585187a51c93d695abefbf147 + v1.27.0: 0b6983195cc62bfc98de1f3fc2ee297a7274fb79ccabf413b8a20765f12d522a + arm64: + v1.29.0: 0b615cfa00c331fb9c4524f3d4058a61cc487b33a3436d1269e7832cf283f925 + v1.28.0: 06e9224e42bc5e23085751e93cccdac89f7930ba6f7a45b8f8fc70ef663c37c4 + v1.27.1: 322bf64d12f9e5cd9540987d47446bf9b0545ceb1900ef93376418083ad88241 + v1.27.0: 9317560069ded8e7bf8b9488fdb110d9e62f0fbc0e33ed09fe972768b47752bd + amd64: + v1.29.0: d16a1ffb3938f5a19d5c8f45d363bd091ef89c0bc4d44ad16b933eede32fdcbb + v1.28.0: 8dc78774f7cbeaf787994d386eec663f0a3cf24de1ea4893598096cb39ef2508 + v1.27.1: b70e8d7bde8ec6ab77c737b6c69be8cb518ce446365734c6db95f15c74a93ce8 + v1.27.0: d335d6e16c309fbc3ff1a29a7e49bb253b5c9b4b030990bf7c6b48687f985cee + ppc64le: + v1.29.0: 2803a1865045077f29f798b9c569e1db7d44b5c329a546a0fd183e906925b99f + v1.28.0: b70fb7bee5982aa1318ba25088319f1d0d1415567f1f76cd69011b8a14da4daf + v1.27.1: c408bb5e797bf02215acf9604c43007bd09cf69353cefa8f20f2c16ab1728a85 + v1.27.0: 3e4301c2d4b561d861970004002fe15d49af907963de06c70d326f2af1f145e0 +crio_archive_checksums: + arm: + v1.29.2: 0 + v1.29.1: 0 + v1.29.0: 0 + v1.28.2: 0 + v1.28.1: 0 + v1.28.0: 0 + v1.27.1: 0 + v1.27.0: 0 + arm64: + v1.29.2: 9b4aa572d4cd51a41b1067161d961423d0d12b120fb636ea887a12a975d4b19a + v1.29.1: b6d3b502ba1474da370519bcdb9eefa07bb04a5a8e7073fbb7071d6e96f49411 + v1.29.0: 2bf11aeb85362ce4b25a7d9fc17bbe80659013425430e5efb922b4388031a027 + v1.28.2: 739923cb744a862039557f23823f4cc12feba121bd26ca3cc01d80cc8aaa1efb + v1.28.1: 98a96c6b6bdf20c60e1a7948847c28b57d9e6e47e396b2e405811ea2c24ab9dc + v1.28.0: c8ea800244d9e4ce74af85126afadea2939cd6f7ddd152d0f09fafbf294ef1cc + v1.27.1: ddf601e28dc22d878cdd34549402a236afaa47e0a08f39b09e65bab7034b1b97 + v1.27.0: c6615360311bff7fdfe1933e8d5030a2e9926b7196c4e7a07fcb10e51a676272 + amd64: + v1.29.2: f71a85039b71fe08f1c063a93d61a1c952dc8f9a8c6be9b13fbdac8f0d9ff960 + v1.29.1: 6ff5af9962bfc079275ecec2b6caf5dbd25bf037fff53e87d67f27c2617432c5 + v1.29.0: 79c161d8db8ee7f0f4807d6232283d481ef0c20c514b61289238258f66734ac6 + v1.28.2: c8002a622e268b73f8d45b0adbdff9422b832106a23be137fabdc8a233b3f787 + v1.28.1: 63cee2e67e283e29d790caa52531bcca7bc59473fb73bde75f4fd8daa169d4bf + v1.28.0: fa87497c12815766d18f332b38a4d823fa6ad6bb3d159e383a5557e6c912eb3b + v1.27.1: 23c0b26f9df65671f20c042466c0e6c543e16ba769bbf63aa26abef170f393ba + v1.27.0: 8f99db9aeea00299cb3f28ee61646472014cac91930e4c7551c9153f8f720093 + ppc64le: + v1.29.1: c79cc835b66fbf2d2dc8e43908475f0603714f688a1450d4736d1f9f86579b52 + v1.29.0: 8adddaf6cf0ed2905820dc162ca5ef541baa7b251368ee00c75435a872a886fb + v1.28.2: 0 + v1.28.1: 0 + v1.28.0: 0 + v1.27.1: 0 + v1.27.0: 0 +# Checksum +# Kubernetes versions above Kubespray's current target version are untested and should be used with caution. +kubelet_checksums: + arm: + v1.29.1: 0 + v1.29.0: 0 + v1.28.7: 0 + v1.28.6: 0 + v1.28.5: 0 + v1.28.4: 0 + v1.28.3: 0 + v1.28.2: 0 + v1.28.1: 0 + v1.28.0: 0 + v1.27.11: 0 + v1.27.10: 0 + v1.27.9: 0 + v1.27.8: 0 + v1.27.7: 0 + v1.27.6: 0 + v1.27.5: 0 + v1.27.4: 0 + v1.27.3: 0 + v1.27.2: 0 + v1.27.1: 0 + v1.27.0: 0 + arm64: + v1.29.2: 9b4aa572d4cd51a41b1067161d961423d0d12b120fb636ea887a12a975d4b19a + v1.29.1: e46417ab1ceae995f0e00d4177959a36ed34b807829422bc9dda70b263fe5c5d + v1.29.0: 0e0e4544c2a0a3475529154b7534d0d58683466efa04a2bb2e763b476db0bb16 + v1.28.7: e2c98b39b0b0745ef3e30febaeb8eaaf31ec721012405bd0dcf25e84026c221e + v1.28.6: ee2c060deff330d3338e24aec9734c9e5d5aea4fea1905c0795bccff6997a65e + v1.28.5: 28ddb696eb6e076f2a2f59ccaa2e409785a63346e5bda819717c6e0f58297702 + v1.28.4: bf203989dd9b3987b8a0d2331dcce6319f834b57df810fafba5a4805d54823ac + v1.28.3: 64f56e9c55183919153fe59df2c9015dff09c56de13a3cbccc0f04a95b76dab9 + v1.28.2: 32269e9ec38c561d028b65c3048ea6a100e1292cbe9e505565222455c8096577 + v1.28.1: 9b7fa64b2785da4a38768377961e227f8da629c56a5df43ca1b665dd07b56f3c + v1.28.0: 05dd12e35783cab4960e885ec0e7d0e461989b94297e7bea9018ccbd15c4dce9 + v1.27.11: e81987a864fb47afe14f65fa4e93760bc19c424335e0f0540c6c725b727ce22a + v1.27.10: 0edadc44ef36be8d8106cad9972360c0477540e2d8c0bbeb38fd97fd1d7801d5 + v1.27.9: 8a14bc3739f5ca3b23d08301c2e769ee58c8d1cecb7243b46b1c098ae77effd7 + v1.27.8: 71849182ceb018dc084f499ad28b7b1afb7f23e35ccaf8421941dd5dafef0d4c + v1.27.7: ed5bfa48ee64d5e6cf23ed9fc03ea0593021839429fdc1ea7cc2ebf3f11b6491 + v1.27.6: be579ef4e8fa3e1de9d40a77e4d35d99e535a293f66bf3038cbea9cf803d11e5 + v1.27.5: 4e78fafdeb5d61ab6ebcd6e75e968c47001c321bec169bb9bd9f001132de5321 + v1.27.4: c75ad8e7c7ef05c0c021b21a9fe86e92f64db1e4c1bc84e1baf45d8dbb8ba8d1 + v1.27.3: 2838fd55340d59f777d7bd7e5989fc72b7a0ca198cf4f3f723cd9956859ce942 + v1.27.2: 810cd9a611e9f084e57c9ee466e33c324b2228d4249ff38c2588a0cc3224f10d + v1.27.1: dbb09d297d924575654db38ed2fc627e35913c2d4000c34613ac6de4995457d0 + v1.27.0: 37aa2edc7c0c4b3e488518c6a4b44c8aade75a55010534ee2be291220c73d157 + amd64: + v1.29.2: f71a85039b71fe08f1c063a93d61a1c952dc8f9a8c6be9b13fbdac8f0d9ff960 + v1.29.1: 1b1975c58d38be1a99a8bcba4564ac489afd223b0abe9f2ab08bbde89d2412a3 + v1.29.0: e1c38137db8d8777eed8813646b59bf4d22d19b9011ab11dc28e2e34f6b80a05 + v1.28.7: 120b1495babc4364f7e16a9d0f8b8e6b6f78316d047e4f6de77b5569b05813c7 + v1.28.6: 8506df1f20a5f8bba0592f5a4cf5d0cc541047708e664cb88580735400d0b26f + v1.28.5: bf37335da58182783a8c63866ec1f895b4c436e3ed96bdd87fe3f8ae8004ba1d + v1.28.4: db2a473b73c3754d4011590f2f0aa877657608499590c6b0f8b40bec96a3e9ba + v1.28.3: a3a058b4ba30da01ffe1801cd38fcad58a9022a2d39e080b4b2e0e9749a75ad5 + v1.28.2: 17edb866636f14eceaad58c56eab12af7ab3be3c78400aff9680635d927f1185 + v1.28.1: 2bc22332f44f8fcd3fce57879fd873f977949ebd261571fbae31fbb2713a5dd3 + v1.28.0: bfb6b977100963f2879a33e5fbaa59a5276ba829a957a6819c936e9c1465f981 + v1.27.11: 2ce92a5d8985b93bd8ffc4f5519cd79bf2f844590aa38228a3d809c5bf5986e0 + v1.27.10: 25a34bf98bb8a296ea07f1ebbcb496b1e6b6c6da3247695288a7c99fc8c1be2c + v1.27.9: ede60eea3acbac3f35dbb23d7b148f45cf169ebbb20af102d3ce141fc0bac60c + v1.27.8: 2e0557b38c5b9a1263eed25a0b84d741453ed9c0c7bd916f80eadaf7edfb7784 + v1.27.7: 236bc8bc22c52e914d3364c23e273628c63e193365b6a43b8cb013716c1cd2f5 + v1.27.6: daa42f9b6f5e2176bbce0d24d89a05613000630bcddec1fafd2a8d42a523ce9d + v1.27.5: 66df07ab4f9d72028c97ec7e5eea23adc0ab62a209ba2285431456d7d75a5bb3 + v1.27.4: 385f65878dc8b48df0f2bd369535ff273390518b5ac2cc1a1684d65619324704 + v1.27.3: c0e18da6a55830cf4910ecd7261597c66ea3f8f58cf44d4adb6bdcb6e2e6f0bf + v1.27.2: a0d12afcab3b2836de4a427558d067bebdff040e9b306b0512c93d9d2a066579 + v1.27.1: cb2845fff0ce41c400489393da73925d28fbee54cfeb7834cd4d11e622cbd3a7 + v1.27.0: 0b4ed4fcd75d33f5dff3ba17776e6089847fc83064d3f7a3ad59a34e94e60a29 + ppc64le: + v1.29.2: b0eb5e0362a4e153ed1239c65b0abb02b2d9fbbca6846d0bab8b285de8c84fca + v1.29.1: 467d2b457205363f53f72081295ea390fc25215b0ccc29dc04c4f82925266067 + v1.29.0: 67f09f866d3e4aee8211ce9887ec8bc427b188474a882a7af999fc0fee939028 + v1.28.7: c67277445af9a97a375da3caa6a7dae7bb52fa454deb811c5bc89c2838b3322f + v1.28.6: 8f79f40bef88aaedfdf7256de48a972295b0069ae0ddefa90dff3f8690c825ce + v1.28.5: ae9fe81804ba67ee81e8a5fe1dc18fe285267764c61f831886a25245a11d8528 + v1.28.4: d79c97811fb10c1b1f48b69573f1164f108630631d9dba0d991fe924bd305f20 + v1.28.3: f20cfb8c9de73cdc66fbbecd03bb936ce57fe86ebced8ea93aa64ebda0235c21 + v1.28.2: 79f568ac700d29f88d669c6b6a09adb3b726bdd13c10aa0839cbc70b414372e5 + v1.28.1: 547fc76f0c1d78352fad841ebeacd387fe48750b2648565dfd49197621622fbb + v1.28.0: 22de59965f2d220afa24bf04f4c6d6b65a4bb1cd80756c13381973b1ac3b4578 + v1.27.11: ce4dc48a61399038eff680b952386ed9be471c5af1e43dc461e0fb5339de9356 + v1.27.10: c5014bed224347245fadec3d763846ec33ccd7a580d0c4ee19a45a948392f20c + v1.27.9: f270051c9b0f36da10a5d27011783be042edd396e8c729709c2396f29b72b6d2 + v1.27.8: 2354fdb19b5018cabe43fde1979965686afd3c95b75531e678a0064c4a30b4e9 + v1.27.7: 9154a7b5d5793fb8a05cba0d309ddfd975409d0e0686be19cf2feca172f06162 + v1.27.6: 1001da3586a3f868c371aefde991af94ca780ec1599c8a969390ba105aaf9dcb + v1.27.5: 3c643564bf07753c1388096aef9125811800fd28aa6a5faf3bfb1cef0e1637eb + v1.27.4: 16c69a941f2b67fef35d84062626622d205f9e2375a8daf3410fb1a42fc6e9e7 + v1.27.3: a8ea8f9e857d1140b569fff88d8d750dccaea0aa33d624befbb67f725b5340a0 + v1.27.2: 3af92edd687f7932e7fce877944dfe5efa437bf5f171fc8331725c631a1a86ef + v1.27.1: 7a800b9539beaba0b5d6357070a40fb3c4d216c2ad6693b15f9b1307b1c99e1f + v1.27.0: 17c061a9f7919697ac71c151c19337f65b86f59f59441687ac92e977d851c75b +kubectl_checksums: + arm: + v1.29.2: f1bab202f0ce0c4209af0a977fc3dd4076397b1983544e09942ca4f586dff900 + v1.29.1: a4b478cc0e9adaab0c5bb3627c20c5228ea0fe2aeff9e805d611eb3edb761972 + v1.29.0: a2388eb458d07ec734e4fa02fd0147456a1922a7d6b8e67a32db9d64a4d7621c + v1.28.7: d0c2e9228aff23bdcc62072ef9cebf5ebb0b14fb8638b6df8f7f6d5220c36bd9 + v1.28.6: 2358d98d4970c177a3af0ae1c2398f69922074a961a61cdff6ae4a7f13106dc1 + v1.28.5: 0819c9d0ea66a1e20d74d9a455090e1f67fe07d671866be342ab55532203f4bc + v1.28.4: 835ef8d72f8dec4493b855ddd8e4163f107053496d923c89c216489a45757df6 + v1.28.3: b252ec9e97abde80fe067eb215a1acb69a8c83022cba897fd2c4d387bd45f5ca + v1.28.2: 6576aa70413ff00c593a07b549b8b9d9e5ef73c42bb39ab4af475e0fdb540613 + v1.28.1: eaa05dab1bffb8593d8e5caa612530ee5c914ee2be73429b7ce36c3becad893f + v1.28.0: 372c4e7bbe98c7067c4b7820c4a440c931ad77f7cb83d3237b439ca3c14d3d37 + v1.27.11: 11586f333abaf2776e0d2f9e02c71ae5eeff3ad8e629815aa8bb7d7e9a406301 + v1.27.10: 4d81649935ec127f9aa21954697f82e0796f61e8e6406fd058b3a8b80e858c8e + v1.27.9: 89b76aa415018377f2c5fc33fc4d45f4997cc63677336f1768ee8a11593515ce + v1.27.8: 2f2936f950beb3f08ee0e45fbf80d020163829b95aa11c99ec726ee1a922329c + v1.27.7: 2a387ade64cd4cf90f002ae8c0e6f60250d26d1ee2cce6051f559430a44352ea + v1.27.6: 3a34a38908a9d0f85dc531cc1c49061ceeaa2ab742382d891d9fc7bf8dc53b8c + v1.27.5: c5e8a02102a93c84413ce8a029f194049429d27ad559061de267d84020a4594b + v1.27.4: 4269133eca9abd29c0a31e15ede2837713635893f1763eccba4b27e66a45abfb + v1.27.3: 1d51a48a0497e47f4f4036687cd337c53d297ea5322e8395432879570379d82e + v1.27.2: 7792f5630543c0af84f444521ee6113da5ae00f2b50872d57324aa725a5341c5 + v1.27.1: fe704e355bf2c5f69964cd12772687535a11a5e9ec0baf4f27e0a8fb156bc615 + v1.27.0: 288470e3eb89a2f55273d753ce6674dfb00e732f2971428acb964810aa726188 + arm64: + v1.29.2: 3507ecb4224cf05ae2151a98d4932253624e7762159936d5347b19fe037655ca + v1.29.1: 96d6dc7b2bdcd344ce58d17631c452225de5bbf59b83fd3c89c33c6298fb5d8b + v1.29.0: 8f7a4bd6bae900a4ddab12bd1399aa652c0d59ea508f39b910e111d248893ff7 + v1.28.7: 13d547495bdea49b223fe06bffb6d2bef96436634847f759107655aa80fc990e + v1.28.6: 0de705659a80c3fef01df43cc0926610fe31482f728b0f992818abd9bdcd2cb9 + v1.28.5: f87fe017ae3ccfd93df03bf17edd4089672528107f230563b8c9966909661ef2 + v1.28.4: edf1e17b41891ec15d59dd3cc62bcd2cdce4b0fd9c2ee058b0967b17534457d7 + v1.28.3: 06511f03e34d8ee350bd55717845e27ebec3116526db7c60092eeb33a475a337 + v1.28.2: ea6d89b677a8d9df331a82139bb90d9968131530b94eab26cee561531eff4c53 + v1.28.1: 46954a604b784a8b0dc16754cfc3fa26aabca9fd4ffd109cd028bfba99d492f6 + v1.28.0: f5484bd9cac66b183c653abed30226b561f537d15346c605cc81d98095f1717c + v1.27.11: d30e1aa873e78eb376ddee3c785aa78c44eddc56ce2ef901dac1ce0c2c4f50b0 + v1.27.10: 2e1996379d5a8b132e0606fcd3df3c8689e11882630b75cca3b7135126847871 + v1.27.9: bda475539fdeda9d8a85a84b967af361af264d0826c121b23b0b62ee9b00cd2d + v1.27.8: 97ed6739e2803e63fd2d9de78be22d5ba6205bb63179a16ec773063526525a8e + v1.27.7: 61fc334f2c0290270e43fb8a1d4ff07e8cec5642d5a123eb7ab66a134b04ae83 + v1.27.6: 7322a6f600de6d0d06cf333bdc24cd2a340bba12920b0c2385c97884c808c810 + v1.27.5: 0158955c59c775165937918f910380ed7b52fca4a26fb41a369734e83aa44874 + v1.27.4: 5178cbb51dcfff286c20bc847d64dd35cd5993b81a2e3609581377a520a6425d + v1.27.3: 7bb7fec4e28e0b50b603d64e47629e812408751bd1e0ce059b2fee83b0e3ff6f + v1.27.2: 1b0966692e398efe71fe59f913eaec44ffd4468cc1acd00bf91c29fa8ff8f578 + v1.27.1: fd3cb8f16e6ed8aee9955b76e3027ac423b6d1cc7356867310d128082e2db916 + v1.27.0: f8e09630211f2b7c6a8cc38835e7dea94708d401f5c84b23a37c70c604602ddc + amd64: + v1.29.2: 7816d067740f47f949be826ac76943167b7b3a38c4f0c18b902fffa8779a5afa + v1.29.1: 69ab3a931e826bf7ac14d38ba7ca637d66a6fcb1ca0e3333a2cafdf15482af9f + v1.29.0: 0e03ab096163f61ab610b33f37f55709d3af8e16e4dcc1eb682882ef80f96fd5 + v1.28.7: aff42d3167685e4d8e86fda0ad9c6ce6ec6c047bc24d608041d54717a18192ba + v1.28.6: c8351fe0611119fd36634dd3f53eb94ec1a2d43ef9e78b92b4846df5cc7aa7e3 + v1.28.5: 2a44c0841b794d85b7819b505da2ff3acd5950bd1bcd956863714acc80653574 + v1.28.4: 893c92053adea6edbbd4e959c871f5c21edce416988f968bec565d115383f7b8 + v1.28.3: 0c680c90892c43e5ce708e918821f92445d1d244f9b3d7513023bcae9a6246d1 + v1.28.2: c922440b043e5de1afa3c1382f8c663a25f055978cbc6e8423493ec157579ec5 + v1.28.1: e7a7d6f9d06fab38b4128785aa80f65c54f6675a0d2abef655259ddd852274e1 + v1.28.0: 4717660fd1466ec72d59000bb1d9f5cdc91fac31d491043ca62b34398e0799ce + v1.27.11: 7ae327978a1edb43700070c86f5fd77215792c6b58a7ea70192647e0da848e29 + v1.27.10: bfb219643c28d9842fceae51590776f06987835d93fc3cb9b0149c9111c741ac + v1.27.9: d0caae91072297b2915dd65f6ef3055d27646dce821ec67d18da35ba9a8dc85b + v1.27.8: 027b3161e99fa0a7fa529e8f17f73ee2c0807c81c721ca7cf307f6b41c17bc57 + v1.27.7: e5fe510ba6f421958358d3d43b3f0b04c2957d4bc3bb24cf541719af61a06d79 + v1.27.6: 2b7adb71c8630904da1b94e262c8c3c477e9609b3c0ed8ae1213a1e156ae38dd + v1.27.5: 9a091fb65e4cf4e8be3ce9a21c79210177dd7ce31a2998ec638c92f37f058bcd + v1.27.4: 4685bfcf732260f72fce58379e812e091557ef1dfc1bc8084226c7891dd6028f + v1.27.3: fba6c062e754a120bc8105cde1344de200452fe014a8759e06e4eec7ed258a09 + v1.27.2: 4f38ee903f35b300d3b005a9c6bfb9a46a57f92e89ae602ef9c129b91dc6c5a5 + v1.27.1: 7fe3a762d926fb068bae32c399880e946e8caf3d903078bea9b169dcd5c17f6d + v1.27.0: 71a78259d70da9c5540c4cf4cff121f443e863376f68f89a759d90cef3f51e87 + ppc64le: + v1.29.2: 382552d15a1aa7ec5a316b2a912e7fbdaaff2f3c714cd38b2b0c6a48b670fed8 + v1.29.1: b7780124ccfe9640f3a37d242d31e8dbb252bcd379bd0d7bf3776d15baf15ca3 + v1.29.0: ea926d8cf25e2ce982ff5c375da32b51ccbd122b721b1bc4a32f52a9a0d073ab + v1.28.7: 1394cc047551bbecffee7f1c28cccd0f3c9839a72344854362a08e98d6513c18 + v1.28.6: 60fdb4386b5499dd6a6e3a369f35eef63c99647f7a0436fdbeb4db8c052d14f6 + v1.28.5: 4448a9f95421cbe69726aa4d2967d706bc43466b9c656c7425b55431b1c20dd4 + v1.28.4: 816ca2cef39c0d1ac8ad60c05ae6f6ea5c4a0ca33748240bd1f019381244ca23 + v1.28.3: 2b7331a91f558a748167672c18458aa205d4d6d2794654dfd308942e9a376ca4 + v1.28.2: 87cca30846fec99a4fbea122b21e938717b309631bd2220de52049fce30d2e81 + v1.28.1: 81b45c27abbdf2be6c5203dfccfd76ded1ac273f9f7672e6dcdf3440aa191324 + v1.28.0: 7a9dcb4c75b33b9dac497c1a756b1f12c7c63f86fc0f321452360fbe1a79ce0f + v1.27.11: af736cbdb7ae42e696fa3543e483726c1728c95039b9520797511965caca56b6 + v1.27.10: 445928336932248cb104d99919e659696afa60f8dd8513821f92775e893d0dcb + v1.27.9: 2464d947370b8902e1245b0a75a4ecf55fe2aeee5bc87f2add7da00b73535a59 + v1.27.8: e25a09dea99192ff43ee13af61bfadd7c79eb538dc8e85376b6c590b4d471204 + v1.27.7: a60da56f856bc5b224f0be71b03443d5a0fb69424f31cd86f069cc79c13b2870 + v1.27.6: f3ed7752a20dbae271eeff9e9d109381e3ed6772853b5c84dc8a7476bbad847c + v1.27.5: 7ab5fe6eb51bd267b3156ef6e18f9e264e6c7c26ec0dafc2f55edcf3164bac99 + v1.27.4: dee25e38897b16ed9009ddcfd96b6635ab3097a051573c6c444209dc27e8ada5 + v1.27.3: b2da3d262e61ffc3e70511977a933b344b18efa5c238bfa388438bc321bc5e11 + v1.27.2: efee037a276f72c77cc230194d7dadf943a5778be46b7985edeb414d27894266 + v1.27.1: 440bcfd9611319f3d9e5d4fa4cdee2421cdf80c01fad223934d9a9b640673d75 + v1.27.0: daa9f1d4fe3f217de2546bca4ac14601f34b34a25c1f571f1e44eb313aee1385 +kubeadm_checksums: + arm: + v1.29.2: 0 + v1.29.1: 0 + v1.29.0: 0 + v1.28.7: 0 + v1.28.6: 0 + v1.28.5: 0 + v1.28.4: 0 + v1.28.3: 0 + v1.28.2: 0 + v1.28.1: 0 + v1.28.0: 0 + v1.27.11: 0 + v1.27.10: 0 + v1.27.9: 0 + v1.27.8: 0 + v1.27.7: 0 + v1.27.6: 0 + v1.27.5: 0 + v1.27.4: 0 + v1.27.3: 0 + v1.27.2: 0 + v1.27.1: 0 + v1.27.0: 0 + arm64: + v1.29.2: e05720feb9d2d67eff25b0156a5c22e2de37be2ffab4e1f4d31e8c526fafd0e1 + v1.29.1: 3bff8c50c104c45e416cce9991706c6ac46365f0defbcd54f8cf4ace0fa68dcf + v1.29.0: bbddee2d46d2e1643ae3623698b45b13aa2e858616d61c642f2f49e5bb14c980 + v1.28.7: f556e49494737f97a15bf15bb4b27d45f8747b477302cdfd22dd61816bc02203 + v1.28.6: 4298cad464e92eec19cdf3e6a607a82a1d626ae70fedba7956175152ab983457 + v1.28.5: 22bb6b3377204e93d008f33ac4924d77adca1478f1ae3b515c03476ba54f1adc + v1.28.4: a4422780020954436b8e76ab1c59b68c5581a54432dd3e566c4709bb40c8d4f9 + v1.28.3: dcb37d78ccdfe9d8dd6f100e188ddc6e3f5570d0c49db68470073683b453a1e7 + v1.28.2: 010789a94cf512d918ec4a3ef8ec734dea0061d89a8293059ef9101ca1bf6bff + v1.28.1: 7d2f68917470a5d66bd2a7d62897f59cb4afaeffb2f26c028afa119acd8c3fc8 + v1.28.0: b9b473d2d9136559b19eb465006af77df45c09862cd7ce6673a33aae517ff5ab + v1.27.11: b8452d6c3f1331beb3d5fa42466a9bc96638a76c40980dba9822300f230c0858 + v1.27.10: ed0447155a7e967ae23480b06b31b2c0aaa871e7c59dfd82ae25b03a1eccf6e6 + v1.27.9: d3d022842b0b8e4661222e8873249f5acafdbef52fd1bfb98152a582352b3c40 + v1.27.8: 0d0f5b2781d663d314e785d14361aa5a09cfaf6e1694aa3cc731f4f06342ec13 + v1.27.7: 46d7c43532233906919a53ee0e03ab04ab9e08514392d17a86f058e0364cda4b + v1.27.6: faec35315203913b835e9b789d89001a05e072943c960bcf4de1e331d08e10c8 + v1.27.5: 3023ef1d2eff885af860e13c8b9fcdb857d259728f16bf992d59c2be522cec82 + v1.27.4: b4ede8a18ef3d1cfa61e6fbca8fcab02f8eee3d0770d2329490fa7be90a4cae4 + v1.27.3: 495e2193ed779d25584b4b532796c2270df0f7139ef15fb89dc7980603615ef4 + v1.27.2: 8f01f363f7c7f92de2f2276124a895503cdc5a60ff549440170880f296b087eb + v1.27.1: 024a59cd6fc76784b597c0c1cf300526e856e8c9fefa5fa7948158929b739551 + v1.27.0: acd805c6783b678ee0068b9dd8165bbfd879c345fd9c25d6a978dbc965f48544 + amd64: + v1.29.2: 2d4e4fa8685bcbfb661cb41050cd4756f50a7aa147f68492d51a99f9cdfd69ac + v1.29.1: d4d81d9020b550c896376fb9e0586a9f15a332175890d061619b52b3e9bc6cbd + v1.29.0: 629d4630657caace9c819fd3797f4a70c397fbd41a2a7e464a0507dad675d52c + v1.28.7: 8aa005bdf6af43e47fc818b26f4cb9f361aae8ec4390519e8d4033be65fbef2b + v1.28.6: bda3eda8d51e8746a42b535b7eab7df52b091a796227c3212dc30909a8f1b431 + v1.28.5: 2b54078c5ea9e85b27f162f508e0bf834a2753e52a57e896812ec3dca92fe9cd + v1.28.4: b4d2531b7cddf782f59555436bc098485b5fa6c05afccdeecf0d62d21d84f5bd + v1.28.3: ce3848b1dfa562e0fa2f911a3d8e3bb07ba040eea76654d68e213315c8846ac0 + v1.28.2: 6a4808230661c69431143db2e200ea2d021c7f1b1085e6353583075471310d00 + v1.28.1: 6134dbc92dcb83c3bae1a8030f7bb391419b5d13ea94badd3a79b7ece75b2736 + v1.28.0: 12ea68bfef0377ccedc1a7c98a05ea76907decbcf1e1ec858a60a7b9b73211bb + v1.27.11: 31bf446a712fb08190838c35d1f4c93b0f975708c59634a5dc3d8915a241c83e + v1.27.10: 23985e958443ac1aabdbeeedc675358abc0638eb580707829fd42b0996a0aae5 + v1.27.9: 78dddac376fa2f04116022cb44ed39ccb9cb0104e05c5b21b220d5151e5c0f86 + v1.27.8: f8864769b8b2d7a14f53eb983f23317ff14d68ab76aba71e9de17ce84c38d4eb + v1.27.7: bc589219a003b3b94c114e4bcf20549a02657a0c6e5c73f588b37817148892d2 + v1.27.6: 2bcdd68957ec25d0689bb56f32b4ec86e38463d2691d5ea21cd109c7afa3aa7c + v1.27.5: 35df8efa6e1bc864ed3c48a665caed634a5c46cfd7f41cda5ad66defdfddb2aa + v1.27.4: 7be21d6fb3707fbbe8f0db0403db6234c8af773b941f931bf8248759ee988bcd + v1.27.3: 2cd663f25c2490bd614a6c0ad9089a47ef315caf0dbdf78efd787d5653b1c6e3 + v1.27.2: 95c4bfb7929900506a42de4d92280f06efe6b47e0a32cbc1f5a1ed737592977a + v1.27.1: c7d32d698e99b90f877025104cb4a9f3f8c707e99e6817940f260135b6d1ad0a + v1.27.0: 78d0e04705a7bdb76a514d60f60c073b16334b15f57ee87f064354ca8a233e80 + ppc64le: + v1.29.2: a0f8ffa8cbfa4bb061ff028df2f6dbb31a9527c561d8c0186d679559f9f347b4 + v1.29.1: 3ec6d90c05dd8e4c6bb1f42fd2fe0f091d85317efaf47d9baebd9af506b3878b + v1.29.0: 4c414a463ed4277e9062c797d1c0435aa7aec2fd1688c5d34e3161c898113cb5 + v1.28.7: 60aec330562326fe1ac4a26fe16053c976467fdbc5811c37a5b4a5c13379cac8 + v1.28.6: 71fc8af0f80599a991ece0c31b21ca85f3ce49322941a305048d9287c249446c + v1.28.5: a9bf8b18711639d9d002f63cebc22c8df1627737891c640f2229461d19b8c321 + v1.28.4: 24e4b42b1d0ec68fc291fcc57fa88ec34b9e8ba758e01639873ef2068222af4a + v1.28.3: 0ae62912b057f3228dd7a9fbe2492c4b8c3a661f27a1d46e70b0b6627ccf60fb + v1.28.2: fdc28482a4316c84d61b0997c29c4d4c7b11459af9c654fdee3b4a3031f0fcb7 + v1.28.1: 73e06f2b614ed5665951f7c059e225a7b0b31319c64a3f57e146fbe7a77fe54e + v1.28.0: 146fe9194486e46accd5054fa93939f9608fdbeefefc4bc68e4c40fb4a84ccc9 + v1.27.11: b7da90f29cae799d96e47aadb1f20b567fbded09d3d5ddcb88d3378098c6a1f3 + v1.27.10: c928ad330bae724b1ef9775e07285408727513a024e3d86e3d72e05768859db8 + v1.27.9: 92da9084fa9f8b8b55436b61ec3c697ef951b0b0416a3b3a7f0dd0e5e4d8cd88 + v1.27.8: d65b972cd661cb28972f0df731f9e5b65d959920275bad5ef44ff94d3bb8331d + v1.27.7: 6a95a8fb5560a6698d895b2a809485ace0cf5b8c6fb89c843ab20cf89d8b11ad + v1.27.6: f2b53fdcd0a71390e84d16facbcd7a581f1309cb8bd0501f9508ebefe5a3498c + v1.27.5: 3df86ca5de57a6c6b4043be3c050ed9ed39a50720364b399e12e9e52e87e377b + v1.27.4: 1635ba4269daf422be112ae8c3954332e69c2b1e50ecd285343f1f2d65955de8 + v1.27.3: 3f174f096a5aaa62fe0298e9a16b3af9031cb1d2a29fc3823f80f9a2144d5fd4 + v1.27.2: 412bccd310f4976201d359f0637745944944c0fb2ace315e5e07b180445530c7 + v1.27.1: d4c46dcc3d210b6eae0b8c34b3ece9f24b1bb2697175615c451db717a99430fb + v1.27.0: cf2860aef800496fee0d9fd8722bd7d17c6609e32d87ca380127151f2ce02bb0 +etcd_binary_checksums: + arm: + v3.5.10: 0 + v3.5.9: 0 + v3.5.8: 0 + v3.5.7: 0 + v3.5.6: 0 + v3.5.5: 0 + v3.5.4: 0 + v3.5.3: 0 + arm64: + v3.5.10: ff74a6018d9b2a1320bff30e5a11b4f2f5c2a3d147df8a8bad53c01b9f800ee1 + v3.5.9: bb201c106a61bbab59e2d9f37f4bdff99d50201f513c66b4578741eab581fb28 + v3.5.8: 3f4441b293a2d0d4d2f8b2cd9504376e15818f7b865ef4b436e8e6f865f895ff + v3.5.7: 1a35314900da7db006b198dd917e923459b462128101736c63a3cda57ecdbf51 + v3.5.6: 888e25c9c94702ac1254c7655709b44bb3711ebaabd3cb05439f3dd1f2b51a87 + v3.5.5: a8d177ae8ecfd1ef025c35ac8c444041d14e67028c1a7b4eda3a69a8dee5f9c3 + v3.5.4: 8e9c2c28ed6b35f36fd94300541da10e1385f335d677afd8efccdcba026f1fa7 + v3.5.3: 8b00f2f51568303799368ee4a3c9b9ff8a3dd9f8b7772c4f6589e46bc62f7115 + amd64: + v3.5.10: 26e90d024fa2310bc52bb40e7f2132e81640b55f8fc446c00ae07e30af2a44fd + v3.5.9: d59017044eb776597eca480432081c5bb26f318ad292967029af1f62b588b042 + v3.5.8: d4c1b8d90ad53658f12ffc293afc5694b7bc6cb093af609188649a799e1cc8dc + v3.5.7: a43119af79c592a874e8f59c4f23832297849d0c479338f9df36e196b86bc396 + v3.5.6: 4db32e3bc06dd0999e2171f76a87c1cffed8369475ec7aa7abee9023635670fb + v3.5.5: 7910a2fdb1863c80b885d06f6729043bff0540f2006bf6af34674df2636cb906 + v3.5.4: b1091166153df1ee0bb29b47fb1943ef0ddf0cd5d07a8fe69827580a08134def + v3.5.3: e13e119ff9b28234561738cd261c2a031eb1c8688079dcf96d8035b3ad19ca58 + ppc64le: + v3.5.10: 10cd8e4ecf6718b9712bf2edfac2e4924d7f21dbe58d368e6e10578c85bd8c01 + v3.5.9: 551539ebb344ebdc77f170ea51512a6cda35877ffdcbd8b3316b2495a8b2bd87 + v3.5.8: 20e28302c1424b1a3daf7d817f2662e4c64e395a82765d1696cb53cb6bc37a4e + v3.5.7: e861aa6acd4d326ec01bfa06fffb80d33f3f8c26e0eb8b73e4424578d149bd04 + v3.5.6: e235cb885996b8aac133975e0077eaf0a2f8dc7062ad052fa7395668a365906b + v3.5.5: 08422dffd5749f0a5f18bd820241d751e539a666af94251c3715cba8f4702c42 + v3.5.4: 2f0389caed87c2504ffc5a07592ca2a688dee45d599073e5f977d9ce75b5f941 + v3.5.3: f14154897ca5ad4698383b4c197001340fbe467525f6fab3b89ee8116246480f +cni_binary_checksums: + arm: + v1.3.0: 86c4c866a01a8073ad14f6feec74de1fd63669786850c7be47521433f9570902 + v1.2.0: fde5bf2da73995196d248177ee8deeafa8005f33cbe1ab33bd2d75c17ca5a99a + v1.1.1: 84f97baf80f9670a8cd0308dedcc8405d2bbc65166d670b48795e0d1262b4248 + v1.1.0: 91e03a9287dcf8d0249159c90357b0f871ecf7ef0ca5014b2e143f2b30ae9c6d + v1.0.1: d35e3e9fd71687fc7e165f7dc7b1e35654b8012995bbfd937946b0681926d62d + v1.0.0: 910c2ba8b6f50b1081b219d6db04459b555940973249fcf39a792932a91f6d39 + v0.9.1: 909e800d01cc61ffa26f2629e4a202a58d727e6ccaabd0310ef18d2b1e00943c + arm64: + v1.3.0: de7a666fd6ad83a228086bd55756db62ef335a193d1b143d910b69f079e30598 + v1.2.0: 525e2b62ba92a1b6f3dc9612449a84aa61652e680f7ebf4eff579795fe464b57 + v1.1.1: 16484966a46b4692028ba32d16afd994e079dc2cc63fbc2191d7bfaf5e11f3dd + v1.1.0: 33fc7b8d9d5be2d7f95e69e6a9e2af206879942f1e6b7615c04017dce5067f1a + v1.0.1: 2d4528c45bdd0a8875f849a75082bc4eafe95cb61f9bcc10a6db38a031f67226 + v1.0.0: 736335bc5923a37cfb6cc2305489ce6206bcc565004f525b5f7c3604f092aa3a + v0.9.1: ef17764ffd6cdcb16d76401bac1db6acc050c9b088f1be5efa0e094ea3b01df0 + amd64: + v1.3.0: 754a71ed60a4bd08726c3af705a7d55ee3df03122b12e389fdba4bea35d7dd7e + v1.2.0: f3a841324845ca6bf0d4091b4fc7f97e18a623172158b72fc3fdcdb9d42d2d37 + v1.1.1: b275772da4026d2161bf8a8b41ed4786754c8a93ebfb6564006d5da7f23831e5 + v1.1.0: 05d46ac19d01669d424ee57401c0deba101763ac494858064b4ea4ffdcc37c5d + v1.0.1: 5238fbb2767cbf6aae736ad97a7aa29167525dcd405196dfbc064672a730d3cf + v1.0.0: 5894883eebe3e38f4474810d334b00dc5ec59bd01332d1f92ca4eb142a67d2e8 + v0.9.1: 962100bbc4baeaaa5748cdbfce941f756b1531c2eadb290129401498bfac21e7 + ppc64le: + v1.3.0: 8ceff026f4eccf33c261b4153af6911e10784ac169d08c1d86cf6887b9f4e99b + v1.2.0: 4960283b88d53b8c45ff7a938a6b398724005313e0388e0a36bd6d0b2bb5acdc + v1.1.1: 1551259fbfe861d942846bee028d5a85f492393e04bcd6609ac8aaa7a3d71431 + v1.1.0: 98239a57452e93c0a27ba9f87bcbb80c7f982f225246f3fe4f3f5ac9b6b1becb + v1.0.1: f078e33067e6daaef3a3a5010d6440f2464b7973dec3ca0b5d5be22fdcb1fd96 + v1.0.0: 1a055924b1b859c54a97dc14894ecaa9b81d6d949530b9544f0af4173f5a8f2a + v0.9.1: 5bd3c82ef248e5c6cc388f25545aa5a7d318778e5f9bc0a31475361bb27acefe +calicoctl_binary_checksums: + arm: + v3.26.4: 0 + v3.26.3: 0 + v3.26.2: 0 + v3.26.1: 0 + v3.26.0: 0 + v3.25.2: 0 + v3.25.1: 0 + v3.25.0: 0 + v3.24.6: 0 + v3.24.5: 0 + v3.24.4: 0 + v3.24.3: 0 + v3.24.2: 0 + v3.24.1: 0 + v3.24.0: 0 + v3.23.5: 0 + v3.23.4: 0 + v3.23.3: 0 + v3.23.2: 0 + v3.23.1: 0 + v3.23.0: 0 + v3.22.5: 0 + v3.22.4: 0 + v3.22.3: 0 + arm64: + v3.26.4: d647d9443ce89df62da6619643375a4f577f5a7fa4e1162416403df521826c2d + v3.26.3: c50272a39658a3b358b33c03fe10d1dde894764413279fecc72d40b95535b398 + v3.26.2: 44de9118f481a1125e2d50cdfbb55073e744dd8e71d2be45eeb2757302910c67 + v3.26.1: bba2fbdd6d2998bca144ae12c2675d65c4fbf51c0944d69b1b2f20e08cd14c22 + v3.26.0: b88c4fd34293fa95d4291b7631502f6b9ad38b5f5a3889bb8012f36f001ff170 + v3.25.2: 1cf28599dc1d52ef7c888731f508662a187129ff7bb3294f58319d79c517085c + v3.25.1: 83084be5de90a94bfd7a10da5758acbf200ddd68fa24ee4e7e1dedc8935aa41d + v3.25.0: 6eda153187ab76821903cf6bb69fe11b016529c3344e2dd1a0f7f3cb3069ded0 + v3.24.6: 98eaeb3d75c7ebb41012641e393a442a509f00572981abcc758668ac0806e1e7 + v3.24.5: 2d56b768ed346129b0249261db27d97458cfb35f98bd028a0c817a23180ab2d2 + v3.24.4: 90ffaf6aab30d5e4c7227cf20a68c7254ec9d871f2e7a4a98ba86a855ee61040 + v3.24.3: dfd74167dd55677a54ac73fd1e3f9391d62cf7f4da210b267d437d4a9b7d4561 + v3.24.2: 6fe53f3ba1c7291e2b1cd15ccb72c393297a668cec46f4aa7137499f68fb37e6 + v3.24.1: b7b1a023ddb81ec32f385f4b90f9a3f415d7fce6242e1ae8ebe5c77b2015209c + v3.24.0: db306755fc9c6a746516eec33337bc102b0d546f6b9fc671795b47d1a878f05d + v3.23.5: 0941ad0deeb03d8fda96340948cdbc15d14062086438150cf3ec5ee2767b22c3 + v3.23.4: c54b7d122d9315bbab1a88707b7168a0934a80c4f2a94c9e871bcc8a8cf11c11 + v3.23.3: 741b222f9bb10b7b5e268e5362796061c8862d4f785bb6b9c4f623ea143f4682 + v3.23.2: 232b992e6767c68c8c832cc7027a0d9aacb29901a9b5e8871e25baedbbb9c64c + v3.23.1: 30f7e118c21ecba445b4fbb27f7ac8bc0d1525ab3c776641433e3b1a3388c65b + v3.23.0: 2afa5795c426faae1fdfd966249f8191929e43d2b94bea268fa9c7ab5a36f6b6 + v3.22.5: f0f6ba82d55c7faa5afb361eb76a78c8e2cf38cd06e0287e03821f77af0c7837 + v3.22.4: e84ba529091818282012fd460e7509995156e50854781c031c81e4f6c715a39a + v3.22.3: 3a3e70828c020efd911181102d21cb4390b7b68669898bd40c0c69b64d11bb63 + amd64: + v3.26.4: 9960357ef6d61eda7abf80bd397544c1952f89d61e5eaf9f6540dae379a3ef61 + v3.26.3: 82bd7d12b0f6973f9593fb62f5410ad6a81ff6b79e92f1afd3e664202e8387cf + v3.26.2: eba9bc34f44801a513c48f730a409dc1ece0ebfd9c1acc21fd3adf0eff93ecdc + v3.26.1: c8f61c1c8e2504410adaff4a7255c65785fe7805eebfd63340ccd3c472aa42cf + v3.26.0: 19ce069f121f9e245f785a7517521e20fe3294ce1add9d1b2bbcbb0a9b9de24e + v3.25.2: b6f6017b1c9520d8eaea101442d82020123d1efc622964b20d97d3e08e198eed + v3.25.1: 13565e5304209ffaa93df3ba722e6f623b66c76057ca8ff5c5864fa13176fe48 + v3.25.0: 5a464075ccbaa8715882de6b32fe82b41488e904fa66b19c48ee6388cf48b1b8 + v3.24.6: 52e8231d14f626c9b3273659697d95559c72e1b081e713b86eaa7f6910bda384 + v3.24.5: 01e6c8a2371050f9edd0ade9dcde89da054e84d8e96bd4ba8cf82806c8d3e8e7 + v3.24.4: 6d6448537d9abd827c01f289303cf66729578b0bd952c043228568af46000e49 + v3.24.3: 22d7ba5547aff1b4202ddd55952c1e5b6e45e416cd79e1721438aab54a23324a + v3.24.2: 185be69fffcaf46fea8328fcc1b73167021fe16548459148853d084ba8a4aac8 + v3.24.1: 10a36ebc7a4cf355b28e061f5a5f4b261daff4773a51ac73ca1071e7551a934a + v3.24.0: 0da282a6a7870fe25742799a921730343c57a1609c5e255e1bb06b5e85011ee2 + v3.23.5: 4c777881709ddaabcf4b56dcbe683125d7ed5743c036fee9273c5295e522082f + v3.23.4: 1ea0d3b6543645612e8239978878b6adefdb7619a16ecbdb8e6dc2687538f689 + v3.23.3: d9c04ab15bad9d8037192abd2aa4733a01b0b64a461c7b788118a0d6747c1737 + v3.23.2: 3784200cdfc0106c9987df2048d219bb91147f0cc3fa365b36279ac82ea37c7a + v3.23.1: e8fd04d776df5571917512560800bf77f3cdf36ca864c9cae966cb74d62ba4fe + v3.23.0: 38106fdd581ab30dc835efeaf83a88b49b21484f8ad33afbefdaf3c49e007550 + v3.22.5: ba75fa65be0e97555b37282e1ab469ad933866eed164b40513e835279bea7348 + v3.22.4: cc412783992abeba6dc01d7bc67bdb2e3a0cf2f27fc3334bdfc02d326c3c9e15 + v3.22.3: a9e5f6bad4ad8c543f6bdcd21d3665cdd23edc780860d8e52a87881a7b3e203c + ppc64le: + v3.26.4: 41cfa77cc27cfe89a046ddb033cf71a46512f4b81251e28c69fca2cee13617ff + v3.26.3: 30a32acbe71894a9783e350ed44294e739b3322f157b2c224ad3c058473e5701 + v3.26.2: c4d42a85afb67020e9cf9dcafe184af6ad60c5609d60001b9505b1a83959b246 + v3.26.1: 7f8baf18f4d7954b6f16b1ddcdadbc818cae2fe1f72137464ccc7b8e6fef03a0 + v3.26.0: b82b931e3aa53248d87b24f00969abfe5ea4518c56a85b5894187c7b47dc452e + v3.25.2: a5e19931ce50953a36387c67b596e56c4b4fc7903893f1ad25248177027ad0dd + v3.25.1: 43f7a19c3f81a658349d727283f201ce5a560dc9a9f7e56d70961755f4196135 + v3.25.0: 15545aa42dfafb12b68070253e649dfbfdb4b495935e4717d2f04c46500d1a9e + v3.24.6: 1d2e2d8ec1524c5fd9f9796bb6ec53e3351d1833c11eb312ca39b549dbccf188 + v3.24.5: 4c40d1703a31eb1d1786287fbf295d614eb9594a4748e505a03a2fbb6eda85b4 + v3.24.4: 2731382823179b49f1e9af7cddca7da191a54d5163a15b19b5ae75ed27dd30f9 + v3.24.3: 2811f71a9a31f8b2965109dc2bbbae24eb5425b4366a6dea8bed1fdb2abe5b60 + v3.24.2: 06c356c1ff741c7d2b49daade10e1bde49ef7db962adfea30dbc4bb314ac8abb + v3.24.1: f2b24bb1cb33795ceba2988ea89e78f25e8fb6283ea22a59a676e37a68a5771e + v3.24.0: cf63f4820e792c101940af3ed6422e1b8769ffcbafd0c3672f2e86675733b053 + v3.23.5: 1b352e73515cbe5746f9b9d7633d8317bd48f713b9b731837f7d79089463321c + v3.23.4: cdd6eace3dc2676b7eed79c665cb0b3dbdd9dcb3bf5b09d7ae20f4f015f75f9b + v3.23.3: f83efcd8d3d7c96dfe8e596dc9739eb5d9616626a6afba29b0af97e5c222575a + v3.23.2: d9ded02381a0fc1311561d0cc9eed9ea827462f3b823593d6ac8bd0591d2020f + v3.23.1: ef5e9b413fbe32da09023cdafc2c3977627dd64a0abcfc68398d3b3923cdd8a6 + v3.23.0: 8b4d40a4613cbc94540b1c7f3252b5924cb549085e73f49e4f84e7814bac7c06 + v3.22.5: 1b3ea734a474d4504c019a8b2213385c8c18cd334edcaefb877e59f8381d2b45 + v3.22.4: f8672ac27ab72c1b05b0f9ae5694881ef8e061bfbcf551f964e7f0a37090a243 + v3.22.3: 7c2fe391f2a18eccff65c64bf93133dc5c58c7322cbd31ea207bbfef5b563947 +ciliumcli_binary_checksums: + arm: + v0.15.0: c8c2d7e2564b1cb6bc82266132f584cb42a430930967ef1fced0b01c8384fedc + v0.14.8: d93dc926c795696f43e7f979ca3a1ed3e912d2c8fd5af305f78c8b08521ef939 + v0.14.7: f9ecd2c029f69d89cb418f461a6098039824ae841aaf9d213df698ccb6a1bfc6 + v0.14.6: 2481dd4edeb01de08c193a421e1b12068b215ee03ff77c3c8ed4514fb810a9d7 + v0.14.5: 08ba6fed412d8e0d1d8f2d9c402aea6c69ac69630f6bf0fd985ad1909b298aae + v0.14.4: 08decd0cca8e1dfcda4322e76c1dbb7eb2c32a2ee6fb8b78d4d6d5bf9cf06373 + v0.14.3: 172e8320ac42750b3a2f41cc6407e4a63c59b30c32ab6e0ff8d5c0695026a5f3 + v0.14.2: 294f27672ad32d065f8899899c68f2561293b68714a65229394769a209254bbd + v0.14.1: 78eae78564624b1346998da9b7da200e5f8489575da09af042bcde0532674264 + v0.14.0: f1873e41a14c380971ee418d60cf37bd8dd2ceefd7eeff6befbbe0768283a65b + arm64: + v0.15.0: 57daf587073147402421f5d8bf069018f73cb66a2da5b4393b742ef59ee15139 + v0.14.8: 25e568d7f26c2a0e83e125a98255ae2faa8ea9e0b0e6d34cd8c690238911e97b + v0.14.7: ad12a491a71185e9cf37665734bf529c5b992025f00c5f3cebfca9524af36472 + v0.14.6: c8218c246fe4c2c2fabfdaaa6f779ab3d8a20bdad9d7113289e9cf9aa0fe75ff + v0.14.5: 7cf13e10162f4ff6114e17c85377b96ed91f187e3131e0c6c35d8f6a181de07d + v0.14.4: 6b6326c178df30085da0f584ac380b1c81bdfbbb7c2a8df0862184444b8bd9eb + v0.14.3: 36df943449dde3eecf1e45cc42a244ef5163ac89f614791a0657cf03ff92273f + v0.14.2: bf94d6aaaf02a6bf728e4ea022e7e37b2dfdc49d5931245be298b2ff4d6af008 + v0.14.1: a73afb03a9815297e6f891aa8420ea04434b479e2777b04b49084d7a8d9cf062 + v0.14.0: e59bd6a38a9bc42f61e34907698ae5cb53a43d93bdec6e7327613f911cb8f205 + amd64: + v0.15.0: 504bbb94b55d4605157b78bf7747cca778888910f8c65729fe69cb94c3d37f5b + v0.14.8: b36014107cea29bdd1df34aa1f292eca9f966d0cc9255232891c0ac6956d421c + v0.14.7: 687b913840f6d54c80e540fac31dd22edbce8962fe8875810e7ed4abc874a45a + v0.14.6: 83fa27d0318f85df78ae6ca06f33c71b900e309cf7488c0db9b9ad5753f6560f + v0.14.5: e6d3b2d297129b10f5690558a85e97d2af407ac30d85758ff77dea686b9c1303 + v0.14.4: 6b3950f8c3b1e8cf7e2123bb4cb1fae4217d720b3353bf924c78d87824c9f1b0 + v0.14.3: 613ffc1cc62ce35b519feb6fc39d1cb2b46635511d365db0da5df498fc6bc001 + v0.14.2: 7ff65f0e85af5daa755c63851f85dea656a59ae4a306e1e9ae02abdf0014f564 + v0.14.1: 86c27fff43f99719271f54a330374ee23f4308aeb6decf7747b354e885a0fcfd + v0.14.0: 73bcbce6fac15c3a62d2a68629f292fa2787440a15998d8c868dae20a6e0e6ed + ppc64le: + v0.15.0: 0 + v0.14.8: 0 + v0.14.7: 0 + v0.14.6: 0 + v0.14.5: 0 + v0.14.4: 0 + v0.14.3: 0 + v0.14.2: 0 + v0.14.1: 0 + v0.14.0: 0 +calico_crds_archive_checksums: + v3.26.4: 481e52de684c049f3f7f7bac78f0f6f4ae424d643451adc9e3d3fa9d03fb6d57 + v3.26.3: b51817e7ae5189b0737ccc901b7b5950a4f84b6029eebfdcc3e3b851bd410d03 + v3.26.2: 8c15b29db525c4cab7bea304357c942a0d55483c03d9c2a0ed3303f66b8f9ff8 + v3.26.1: 6d0afbbd4bdfe4deb18d0ac30adb7165eb08b0114ec5a00d016f37a8caf88849 + v3.26.0: a263e507e79c0a131fdc2a49422d3bdf0456ea5786eb44ace2659aba879b5c7c + v3.25.2: 6a6e95a51a8ebf65d41d671f20854319cca1f26cd87fbcfc30d1382a06ecfee0 + v3.25.1: 4d6b6653499f24f80a85a0a7dac28d9571cabfa25356b08f3b438fd97e322e2d + v3.25.0: 117b4493ad933f24ea6fb82eabfad300da2dd926995bb8c55336595d38c72881 + v3.24.6: 71644374ae7f50bc17cd79544b07e59a3967d1d43b289ae62d750ce9167312e9 + v3.24.5: 10320b45ebcf4335703d692adacc96cdd3a27de62b4599238604bd7b0bedccc3 + v3.24.4: 7f8e54f50388784b5c17ba20ebfb0b65c6a87291771fe8be300646906aa1558d + v3.24.3: dbe3a48d602a3ac9073185a9e12e2452f4520b3e8c01f1af8603ef7af5e44fe9 + v3.24.2: eb0a57e6eb37c4658aa51ecfc078d1dfacb89da23a10b67fed8ea0e4e9c66eea + v3.24.1: 62c30e126c1595adc851f3df0a69926cc6bf97a7d0d38293f23d2232c6411a31 + v3.24.0: 3c6694779b916fa364592a8e19d45f509c67e7dec64fb4cf09c379e170de7720 + v3.23.5: aca591282d9e10a180a2afb05da6ca8db4dd02b886b4788f4962cf5b37ba1bda + v3.23.4: c8b6b033755416756b2b5ef248332b7c5b660618327cb7f83a80fb949fdc601a + v3.23.3: d25f5c9a3adeba63219f3c8425a8475ebfbca485376a78193ec1e4c74e7a6115 + v3.23.2: 37c429650723c5f12ffc20dd390ead1e10d2b8a955a199666d155115a49b4dcc + v3.23.1: a1754ae4bb158e3b46ba3fb326d8038d54cd0dc2c5c8527eadf2b0a6cf8ef2e3 + v3.23.0: 27dd12ff792eb8f680506566e8d99467673f859298fe93d4f23c2139cc3f0c96 + v3.22.5: f3b6a6861b7beae549b4cf0be5c4b954c0cc19e95adb89dd9d78e983f9f2a5d7 + v3.22.4: e72e7b8b26256950c1ce0042ac85fa83700154dae9723c8d007de88343f6a7e5 + v3.22.3: 55ece01da00f82c62619b82b6bfd6442a021acc6fd915a753735e6ebceabaa21 +krew_archive_checksums: + darwin: + arm: + v0.4.4: 0 + v0.4.3: 0 + v0.4.2: 0 + arm64: + v0.4.4: e6ac776140b228d6bdfda11247baf4e9b11068f42005d0975fc260c629954464 + v0.4.3: 22f29ce3c3c9c030e2eaf3939d2b00f0187dfdbbfaee37fba8ffaadc46e51372 + v0.4.2: a69d48f8cad7d87b379071129cde3ee4abcaaa1c3f3692bc80887178b2cc7d33 + amd64: + v0.4.4: 5f4d2f34868a87cf1188212cf7cb598e76a32f389054089aad1fa46e6daf1e1b + v0.4.3: 6f6a774f03ad4190a709d7d4dcbb4af956ca0eb308cb0d0a44abc90777b0b21a + v0.4.2: 47c6b5b647c5de679a2302444f75a36a70530fa4751cb655e0edd5da56a5f110 + ppc64le: + v0.4.4: 0 + v0.4.3: 0 + v0.4.2: 0 + linux: + arm: + v0.4.4: 4f3d550227e014f3ba7c72031108ffda0654cb755f70eb96be413a5102d23333 + v0.4.3: 68eb9e9f5bba29c7c19fb52bfc43a31300f92282a4e81f0c51ad26ed2c73eb03 + v0.4.2: 115f503e35ef7f63f00a9b01236d80a9f94862ec684010a81c3a3b51bdca1351 + arm64: + v0.4.4: f8f0cdbf698ed3e8cb46e7bd213754701341a10e11ccb69c90d4863e0cf5a16a + v0.4.3: 0994923848882ad0d4825d5af1dc227687a10a02688f785709b03549dd34d71d + v0.4.2: 7581be80d803536acc63cceb20065023b96f07fd7eb9f4ee495dce0294a866eb + amd64: + v0.4.4: e471396b0ed4f2be092b4854cc030dfcbb12b86197972e7bef0cb89ad9c72477 + v0.4.3: 5df32eaa0e888a2566439c4ccb2ef3a3e6e89522f2f2126030171e2585585e4f + v0.4.2: 203bfd8006b304c1e58d9e96f9afdc5f4a055e0fbd7ee397fac9f36bf202e721 + ppc64le: + v0.4.4: 0 + v0.4.3: 0 + v0.4.2: 0 + windows: + arm: + v0.4.4: 0 + v0.4.3: 0 + v0.4.2: 0 + arm64: + v0.4.4: 0 + v0.4.3: 0 + v0.4.2: 0 + amd64: + v0.4.4: da0dfeb2a598f11fb9ce871ee7f3b1a69beb371a45f531ee65a71b2201511d28 + v0.4.3: d1343a366a867e9de60b23cc3d8ee935ee185af25ff8f717a5e696ba3cae7c85 + v0.4.2: 3150ff0291ac876ebe4fe0e813ee90a18aa2bc0510c3adcfae6117dec44ef269 + ppc64le: + v0.4.4: 0 + v0.4.3: 0 + v0.4.2: 0 +helm_archive_checksums: + arm: + v3.13.2: 06e8436bde78d53ddb5095ba146fe6c7001297c7dceb9ef6b68992c3ecfde770 + v3.13.1: a9c188c1a79d2eb1721aece7c4e7cfcd56fa76d1e37bd7c9c05d3969bb0499b4 + v3.13.0: bb2cdde0d12c55f65e88e7c398e67463e74bc236f68b7f307a73174b35628c2e + v3.12.3: 6b67cf5fc441c1fcb4a860629b2ec613d0e6c8ac536600445f52a033671e985e + v3.12.2: 39cc63757901eaea5f0c30b464d3253a5d034ffefcb9b9d3c9e284887b9bb381 + v3.12.1: 6ae6d1cb3b9f7faf68d5cd327eaa53c432f01e8fd67edba4e4c744dcbd8a0883 + v3.12.0: 1d1d3b0b6397825c3f91ec5f5e66eb415a4199ccfaf063ca399d64854897f3f0 + v3.11.3: 0816db0efd033c78c3cc1c37506967947b01965b9c0739fe13ec2b1eea08f601 + v3.11.2: 444b65100e224beee0a3a3a54cb19dad37388fa9217ab2782ba63551c4a2e128 + v3.11.1: 77b797134ea9a121f2ede9d159a43a8b3895a9ff92cc24b71b77fb726d9eba6d + v3.11.0: cddbef72886c82a123038883f32b04e739cc4bd7b9e5f869740d51e50a38be01 + v3.10.3: dca718eb68c72c51fc7157c4c2ebc8ce7ac79b95fc9355c5427ded99e913ec4c + arm64: + v3.13.2: f5654aaed63a0da72852776e1d3f851b2ea9529cb5696337202703c2e1ed2321 + v3.13.1: 8c4a0777218b266a7b977394aaf0e9cef30ed2df6e742d683e523d75508d6efe + v3.13.0: d12a0e73a7dbff7d89d13e0c6eb73f5095f72d70faea30531941d320678904d2 + v3.12.3: 79ef06935fb47e432c0c91bdefd140e5b543ec46376007ca14a52e5ed3023088 + v3.12.2: cfafbae85c31afde88c69f0e5053610c8c455826081c1b2d665d9b44c31b3759 + v3.12.1: 50548d4fedef9d8d01d1ed5a2dd5c849271d1017127417dc4c7ef6777ae68f7e + v3.12.0: 658839fed8f9be2169f5df68e55cb2f0aa731a50df454caf183186766800bbd0 + v3.11.3: 9f58e707dcbe9a3b7885c4e24ef57edfb9794490d72705b33a93fa1f3572cce4 + v3.11.2: 0a60baac83c3106017666864e664f52a4e16fbd578ac009f9a85456a9241c5db + v3.11.1: 919173e8fb7a3b54d76af9feb92e49e86d5a80c5185020bae8c393fa0f0de1e8 + v3.11.0: 57d36ff801ce8c0201ce9917c5a2d3b4da33e5d4ea154320962c7d6fb13e1f2c + v3.10.3: 260cda5ff2ed5d01dd0fd6e7e09bc80126e00d8bdc55f3269d05129e32f6f99d + amd64: + v3.13.2: 55a8e6dce87a1e52c61e0ce7a89bf85b38725ba3e8deb51d4a08ade8a2c70b2d + v3.13.1: 98c363564d00afd0cc3088e8f830f2a0eeb5f28755b3d8c48df89866374a1ed0 + v3.13.0: 138676351483e61d12dfade70da6c03d471bbdcac84eaadeb5e1d06fa114a24f + v3.12.3: 1b2313cd198d45eab00cc37c38f6b1ca0a948ba279c29e322bdf426d406129b5 + v3.12.2: 2b6efaa009891d3703869f4be80ab86faa33fa83d9d5ff2f6492a8aebe97b219 + v3.12.1: 1a7074f58ef7190f74ce6db5db0b70e355a655e2013c4d5db2317e63fa9e3dea + v3.12.0: da36e117d6dbc57c8ec5bab2283222fbd108db86c83389eebe045ad1ef3e2c3b + v3.11.3: ca2d5d40d4cdfb9a3a6205dd803b5bc8def00bd2f13e5526c127e9b667974a89 + v3.11.2: 781d826daec584f9d50a01f0f7dadfd25a3312217a14aa2fbb85107b014ac8ca + v3.11.1: 0b1be96b66fab4770526f136f5f1a385a47c41923d33aab0dcb500e0f6c1bf7c + v3.11.0: 6c3440d829a56071a4386dd3ce6254eab113bc9b1fe924a6ee99f7ff869b9e0b + v3.10.3: 950439759ece902157cf915b209b8d694e6f675eaab5099fb7894f30eeaee9a2 + ppc64le: + v3.13.2: 11d96134cc4ec106c23cd8c163072e9aed6cd73e36a3da120e5876d426203f37 + v3.13.1: f0d4ae95b4db25d03ced987e30d424564bd4727af6a4a0b7fca41f14203306fb + v3.13.0: d9be0057c21ce5994885630340b4f2725a68510deca6e3c455030d83336e4797 + v3.12.3: 8f2182ae53dd129a176ee15a09754fa942e9e7e9adab41fd60a39833686fe5e6 + v3.12.2: fb0313bfd6ec5a08d8755efb7e603f76633726160040434fd885e74b6c10e387 + v3.12.1: 32b25dba14549a4097bf3dd62221cf6df06279ded391f7479144e3a215982aaf + v3.12.0: 252d952b0e1b4ed2013710ddedf687ed5545d9f95a4fd72de0ff9617ff69155c + v3.11.3: 9f0a8299152ec714cee7bdf61066ba83d34d614c63e97843d30815b55c942612 + v3.11.2: 04cbb8d053f2d8023e5cc6b771e9fa384fdd341eb7193a0fb592b7e2a036bf3d + v3.11.1: 6ab8f2e253c115b17eda1e10e96d1637047efd315e9807bcb1d0d0bcad278ab7 + v3.11.0: 6481a51095f408773212ab53edc2ead8a70e39eba67c2491e11c4229a251f9b5 + v3.10.3: 93cdf398abc68e388d1b46d49d8e1197544930ecd3e81cc58d0a87a4579d60ed +cri_dockerd_archive_checksums: + arm: + 0.3.9: 0 + 0.3.8: 0 + 0.3.7: 0 + 0.3.6: 0 + 0.3.5: 0 + 0.3.4: 0 + 0.3.3: 0 + 0.3.2: 0 + 0.3.1: 0 + 0.3.0: 0 + 0.2.6: 0 + 0.2.5: 0 + arm64: + 0.3.9: f5051002b4f95b0e8fe7fbd5f8de4493350e010834d2a8b647f2b26c45c6c203 + 0.3.8: 64286af171785f0facb72cf364867600b4db19f43a01db49b8b364f5d04aadae + 0.3.7: 8da54563ee7ddee36b1adf1f96b3b7b97ec2bc0ec23559b89d9af8eae5e62d9e + 0.3.6: 793b8f57cecf734c47bface10387a8e90994c570b516cb755900f21ebd0a663b + 0.3.5: c20014dc5a71e6991a3bd7e1667c744e3807b5675b1724b26bb7c70093582cfe + 0.3.4: 598709c96585936729140d31a76be778e86f9e31180ff3622a44b63806f37779 + 0.3.3: fa0aa587fc7615248f814930c2e0c9a252afb18dc37c8f4d6d0263faed45d5a7 + 0.3.2: b24ae82808bb5ee531348c952152746241ab9b1b7477466ba6c47a7698ef16ae + 0.3.1: dcaa2794ac23348c6d370717a68e70d1da1723a11a892d63459cd88fb5d82226 + 0.3.0: 2a7e5bb156b80f737ef07ae2e8050394ea3e47fb0b7055afac47a365eaa321fb + 0.2.6: 90122641e45e8ff81dbdd4d84c06fd9744b807b87bff5d0db7f826ded326a9fd + 0.2.5: 067242bf5e4b39fece10500a239612c7b0723ce9766ba309dbd22acaf1a2def2 + amd64: + 0.3.9: a6d9b4b796e9eff830311a2349d259507302cb3955dd07b78296b91e40e8b433 + 0.3.8: e12ea6df8228b7d0794c930d32117c4e5a3dcf25a56c3facdf7006289ec6383c + 0.3.7: 518c5d5345085f36d311f274208705d7fdb79337a80c256871ce941d5a7d47a1 + 0.3.6: cf271d65abee88c0c0a6d9dacb151913bf37d25d45913a7e04b09efe408eae18 + 0.3.5: 30d47bd89998526d51a8518f9e8ef10baed408ab273879ee0e30350702092938 + 0.3.4: b77a1fbd70d12e5b1dacfa24e5824619ec54184dbc655e721b8523572651adeb + 0.3.3: 169dce95e7252165c719e066a90b4a64af64119f9ee74fdca73bf9386bcf96c8 + 0.3.2: 93acc0b8c73c68720c9e40b89c2a220a2df315eb2cd3d162b294337c4dcb2193 + 0.3.1: 126431e7b207e013004311f5a21803cad44511616e7440157381476bdc6c5219 + 0.3.0: 8e6a445591e77b9570299d0afadeee26cb7aa23e4bfd7518baa6a3260b9ee889 + 0.2.6: 5d57b160d5a1f75333149823bec3e291a1a0960383ddc9ddd6e4ff177382c755 + 0.2.5: 1660052586390fd2668421d16265dfcc2bbdba79d923c7ede268cf91935657c1 + ppc64le: + 0.3.9: 0 + 0.3.8: 0 + 0.3.7: 0 + 0.3.6: 0 + 0.3.5: 0 + 0.3.4: 0 + 0.3.3: 0 + 0.3.2: 0 + 0.3.1: 0 + 0.3.0: 0 + 0.2.6: 0 + 0.2.5: 0 +runc_checksums: + arm: + v1.1.12: 0 + v1.1.11: 0 + v1.1.10: 0 + v1.1.9: 0 + v1.1.8: 0 + v1.1.7: 0 + v1.1.6: 0 + v1.1.5: 0 + v1.1.4: 0 + v1.1.3: 0 + arm64: + v1.1.12: 879f910a05c95c10c64ad8eb7d5e3aa8e4b30e65587b3d68e009a3565aed5bb8 + v1.1.11: 9f1ee53f06b78cc4a115ca6ae4eec10567999539ce828a22c5351edba043ed12 + v1.1.10: 4830afd426bdeacbdf9cb8729524aa2ed51790b8c4b28786995925593708f1c8 + v1.1.9: b43e9f561e85906f469eef5a7b7992fc586f750f44a0e011da4467e7008c33a0 + v1.1.8: 7c22cb618116d1d5216d79e076349f93a672253d564b19928a099c20e4acd658 + v1.1.7: 1b309c4d5aa4cc7b888b2f79c385ecee26ca3d55dae0852e7c4a692196d5faab + v1.1.6: da5b2ed26a173a69ea66eae7c369feebf59c1031e14985f512a0a293bb5f76fb + v1.1.5: 54e79e4d48b9e191767e4abc08be1a8476a1c757e9a9f8c45c6ded001226867f + v1.1.4: dbb71e737eaef454a406ce21fd021bd8f1b35afb7635016745992bbd7c17a223 + v1.1.3: 00c9ad161a77a01d9dcbd25b1d76fa9822e57d8e4abf26ba8907c98f6bcfcd0f + amd64: + v1.1.12: aadeef400b8f05645768c1476d1023f7875b78f52c7ff1967a6dbce236b8cbd8 + v1.1.11: 77ae134de014613c44d25e6310a57a219a7a91155cd47d069a0f22a2cad5caea + v1.1.10: 81f73a59be3d122ab484d7dfe9ddc81030f595cc59968f61c113a9a38a2c113a + v1.1.9: b9bfdd4cb27cddbb6172a442df165a80bfc0538a676fbca1a6a6c8f4c6933b43 + v1.1.8: 1d05ed79854efc707841dfc7afbf3b86546fc1d0b3a204435ca921c14af8385b + v1.1.7: c3aadb419e5872af49504b6de894055251d2e685fddddb981a79703e7f895cbd + v1.1.6: 868bee5b8dc2a01df0ca41d0accfad6a3372dc1165ebfb76143d2c6672e86115 + v1.1.5: f00b144e86f8c1db347a2e8f22caade07d55382c5f76dd5c0a5b1ab64eaec8bb + v1.1.4: db772be63147a4e747b4fe286c7c16a2edc4a8458bd3092ea46aaee77750e8ce + v1.1.3: 6e8b24be90fffce6b025d254846da9d2ca6d65125f9139b6354bab0272253d01 + ppc64le: + v1.1.12: 4069d1d57724126e116ad6dbd84409082d1b0afee1ee960b17558f146a742bb6 + v1.1.11: e3d1da41f97db1bb7e9a8d96c9092747c14ee53bc9f160048828e63f3a2d0896 + v1.1.10: 94a091c06c363e4af7be398dc31fa6e02576d5ecda6de1cbf3a08fe8662bf678 + v1.1.9: 065cf4f84b5acc0acdb017af2955743dfb5f5e1f49a493eea3e8206f33bf6fe6 + v1.1.8: a816cd654e804249c4f757cc6bf2aa2c128e4b8e6a993067d44c63c891c081ab + v1.1.7: eb0e76876d09fa8119dc6e6b037107e5d265d1cfa51f1fbed5418e5745ecf153 + v1.1.6: f98d585dd88d45a296a3f3adde39eaec84e0cfc75f75c50e5470d871e3538460 + v1.1.5: 4f06d25b46e11e6670bf38e638c9183bb6676787801f1226f0aa8e74e40169ea + v1.1.4: 0f7fb3d2426b6012d9b33c354c778c0ffbce02c329c4c16c1189433a958fd60d + v1.1.3: 3b1b7f953fc8402dec53dcf2de05b6b72d86850737efa9766f8ffefc7cae3c0a +crun_checksums: + arm: + 1.11.2: 0 + 1.11.1: 0 + 1.9.2: 0 + 1.9.1: 0 + 1.8.7: 0 + 1.8.6: 0 + 1.8.5: 0 + 1.8.4: 0 + 1.8.3: 0 + 1.8.2: 0 + 1.8.1: 0 + 1.7.2: 0 + 1.7.1: 0 + arm64: + 1.11.2: 9e1aeb86bce609eccff46a8b976ed06994bca27d639e564fd45756786c4d0123 + 1.11.1: c8b0d243f6ac4fb02665c157b5404e5184bdc9240dbdcdde0ccef2db352ce97a + 1.9.2: 1ad8bd3c1aa693f59133c480aa13bbdf6d81e4528e72ce955612c6bae8cb1720 + 1.9.1: fab460328d425a72cfd1a70f8fc25c888b6f17cfd95abdace61035a80c3dfe4a + 1.8.7: 004f40b48ec28e963eee79929002b9dfb88496be5699e6052358c67e47fdc88a + 1.8.6: 1f86d20292284f29593594df8d8556d5363a9e087e169626604cc212c77d1727 + 1.8.5: 77032341af7c201db03a53e46707ba8b1af11cdd788530426f2da6ccb9535202 + 1.8.4: 29bbb848881868c58908933bab252e73ee055672d00b7f40cea751441ca74fa4 + 1.8.3: 5394336630618c724274bf3e5e0c8a64c2e67e4723f671029c4f57f459359f73 + 1.8.2: d17970486fab69058e182c3322b7f9fe51561cc3ce28339a0d65b0c81acda933 + 1.8.1: c8382b91a52ac09797ff44990daf014803dde9487d1a41243bc9d8eaf07484e4 + 1.7.2: 576a39ca227a911e0e758db8381d2786f782bfbd40b54684be4af5e1fe67b018 + 1.7.1: 8d458c975f6bf754e86ebedda9927abc3942cbebe4c4cb34a2f1df5acd399690 + amd64: + 1.11.2: acb62839ab8615f0e2485e8d71272b5659cbe35182eb24c5e96bd213240567fe + 1.11.1: ca8c9cef23f4a3f7a635ee58a3d9fa35e768581fda89dc3b6baed219cc407a02 + 1.9.2: 2bb60bcd5652cb17e44f66f0b8ae48195434bd1d66593db97fba85c7778eac53 + 1.9.1: a2bc565c8bbcb1074b70cdec0c39ca93e4aa84f1188641d160531f4a8aae80f0 + 1.8.7: f26e90ab197df8b1cb81d70bcb2cd36a80299d6445470b3c1a84ceda59a34199 + 1.8.6: 23cd9901106ad7a8ebf33725a16b99a14b95368a085d6ffc2ede0b0c9b002bde + 1.8.5: 75062fa96a7cabd70e6f6baf1e11da00131584cc74a2ef682a172769178d8731 + 1.8.4: 99be7d3c9ba3196c35d64b63fa14e9f5c37d1e91b194cfdbfa92dbcbebd651bc + 1.8.3: f82ccdc575a72fe2d91ea8d68161746a0e28898bc86a2a6f55eed00aa1d79afa + 1.8.2: 9febf1dd7600d15db2ee9a6b8836a76db563bf715e009d0c5f662353e7fa6c29 + 1.8.1: b7f2150da473ed2d052df371244176aa96c9ad908fed06b81ebcb51a8a0f6b06 + 1.7.2: 2bd2640d43bc78be598e0e09dd5bb11631973fc79829c1b738b9a1d73fdc7997 + 1.7.1: 8e095f258eee554bb94b42af07aa5c54e0672a403d56b2cfecd49153a11d6760 + ppc64le: + 1.11.2: 0 + 1.11.1: 0 + 1.9.2: 0 + 1.9.1: 0 + 1.8.7: 0 + 1.8.6: 0 + 1.8.5: 0 + 1.8.4: 0 + 1.8.3: 0 + 1.8.2: 0 + 1.8.1: 0 + 1.7.2: 0 + 1.7.1: 0 +youki_checksums: + arm: + 0.3.0: 0 + 0.2.0: 0 + 0.1.0: 0 + 0.0.5: 0 + 0.0.4: 0 + 0.0.3: 0 + 0.0.2: 0 + 0.0.1: 0 + arm64: + 0.3.0: 0 + 0.2.0: 0 + 0.1.0: 0 + 0.0.5: 0 + 0.0.4: 0 + 0.0.3: 0 + 0.0.2: 0 + 0.0.1: 0 + amd64: + 0.3.0: 741ba3cd85d768bebba02598cedcf3b15a2160e4d6ce33a3d5c4e1b3080f9c1c + 0.2.0: b268689a91db07feebfd41d5806b10c7d051fbcbf7efb15076e2228763ac0762 + 0.1.0: f00677e9674215b44f140f0c0f4b79b0001c72c073d2c5bb514b7a9dcb13bdbc + 0.0.5: 8504f4c35a24b96782b9e0feb7813aba4e7262c55a39b8368e94c80c9a4ec564 + 0.0.4: c213376393cb16462ef56586e68fef9ec5b5dd80787e7152f911d7cfd72d952e + 0.0.3: 15df10c78f6a35e45a1dce92c827d91b9aef22dc926c619ff5befafc8543f1bb + 0.0.2: dd61f1c3af204ec8a29a52792897ca0d0f21dca0b0ec44a16d84511a19e4a569 + 0.0.1: 8bd712fe95c8a81194bfbc54c70516350f95153d67044579af95788fbafd943b + ppc64le: + 0.3.0: 0 + 0.2.0: 0 + 0.1.0: 0 + 0.0.5: 0 + 0.0.4: 0 + 0.0.3: 0 + 0.0.2: 0 + 0.0.1: 0 +kata_containers_binary_checksums: + arm: + 3.2.0: 0 + 3.1.3: 0 + 3.1.2: 0 + 3.1.1: 0 + 3.1.0: 0 + 3.0.2: 0 + 3.0.1: 0 + 3.0.0: 0 + 2.5.2: 0 + 2.5.1: 0 + 2.5.0: 0 + arm64: + 3.2.0: 0 + 3.1.3: 0 + 3.1.2: 0 + 3.1.1: 0 + 3.1.0: 0 + 3.0.2: 0 + 3.0.1: 0 + 3.0.0: 0 + 2.5.2: 0 + 2.5.1: 0 + 2.5.0: 0 + amd64: + 3.2.0: 21bb8484a060450d6522f29bed7d88d773c28520774eaa2c522b6f47fd12c4a1 + 3.1.3: 266c906222c85b67867dea3c9bdb58c6da0b656be3a29f9e0bed227c939f3f26 + 3.1.2: 11a2921242cdacf08a72bbce85418fc21c2772615cec6f3de7fd371e04188388 + 3.1.1: 999bab0b362cdf856be6448d1ac4c79fa8d33e79a7dfd1cadaafa544f22ade83 + 3.1.0: 452cc850e021539c14359d016aba18ddba128f59aa9ab637738296d9b5cd78a0 + 3.0.2: a32dc555ffae23f3caab3bc57b03d5ed7792f651221f6cb95cdfe906e18c4bd1 + 3.0.1: e2505482f68cc1b1417b8011f2755bf87171a8dd6daaace28531746118fbddaa + 3.0.0: ff475932f65936504f63ff087c81f89103df2a99e0ceb6571246f63f7a4f948e + 2.5.2: 2c7ce463b32d52b613c1b1ea3d89e83a59ca0fd0ee7fdd24eb854ab2de05ec10 + 2.5.1: 4e4fe5204ae9aea43aa9d9bee467a780d4ae9d52cd716edd7e28393a881377ad + 2.5.0: 044e257c16b8dfa1df92663bd8e4b7f62dbef3e431bc427cdd498ff1b2163515 + ppc64le: + 3.2.0: 0 + 3.1.3: 0 + 3.1.2: 0 + 3.1.1: 0 + 3.1.0: 0 + 3.0.2: 0 + 3.0.1: 0 + 3.0.0: 0 + 2.5.2: 0 + 2.5.1: 0 + 2.5.0: 0 +gvisor_runsc_binary_checksums: + arm: + 20230807: 0 + 20230801: 0 + 20230731: 0 + 20230724: 0 + 20230717: 0 + 20230710: 0 + 20230627: 0 + 20230621: 0 + 20230605: 0 + 20230529: 0 + 20230522: 0 + 20230517: 0 + 20230508: 0 + 20230501: 0 + 20230417: 0 + 20231030: 0 + 20231023: 0 + 20231016: 0 + 20231009: 0 + 20231003: 0 + 20230925: 0 + 20230920: 0 + 20230911: 0 + 20230904: 0 + arm64: + 20230807: 562c629abb6576d02a4b5a5c32cb4706e29122f72737c55a2bf87d012682117f + 20230801: 69f4b7fd068fcc9a30181657ae5dcdd259e5fe71111d86e7cb0065e190b82fc3 + 20230731: 228ad19507ed23f97d99a2ea19be355f57fa4fddc70d0c425879952bd2d2cd7d + 20230724: bbc929ade0211f1d4759db4d3b1e12942dbd198ec91e3e40a272a787856be6e9 + 20230717: 94e36ee1581b951ab328b097d8aff994e96c035462bd2ea5d67f0bc225996c7a + 20230710: 1af3f8640f517339e2b1d522c6ec7066bc32329d0d96d265af6fe7e6d966d4b3 + 20230627: b791646f8129542f110f5ed9d88c3c1fbe6a242207202a8fba2873fad4c6eca6 + 20230621: 7e57e36c146e4aeae736b777c13bcd077a60110e5f1db9e60b87199aed1b533f + 20230605: ef3a965ff6e585c5604f72172a03e6bf3c7511a04f86925eaad6b78b7b9cb4f1 + 20230529: d31e781026a0afa4e2864839993ab17cf4f581ec92419e7c263f4ed34958a2cd + 20230522: 873163cb0e850685efa2f8c98a3b57502e4af72e5a14edf15c81d8830afa3dc3 + 20230517: 9a107bed8a1184a6f1040e6893c0975e572966a3ecd7009e8b2be70482e4ab1d + 20230508: b1cffc3c3071fe92f2d6c14aa946d50f01b0650ce8a8ed51b240cebc2ae2d1f0 + 20230501: b0e0e74ca92efbb65cfa2de1fbb00f767056c2797ca1b1b091ecee9ae0be8122 + 20230417: 21d01bb86f31812d5bca09fa89129ceee6561e5dd2722afcc52e28649383f311 + 20231030: c4a11ed7066bff777db048167b01a8662d0e1a48672a8c78ab7c3d5e5a5297c7 + 20231023: 90572e057cc05360c052aa2a161038e65328b9860d85d1f5db6c24c4c6a2433e + 20231016: 2bd8aee1ca3563e08afdd7019783d02c8c701c63e67fac3be6ee1243c5b0ee21 + 20231009: c6730e8ba356dd763b451b00e0206d0c69ff9857fd0a7ad456546db192b3ca4b + 20231003: 5b18676dab77d2725da02489de61445336e590018678898b7d8ff0afcda4d9f8 + 20230925: 64b6f59a7ec247fa01db5b9ee0a66c5a2e4c5ddf891dda8c8db65cbf8a4f0ae2 + 20230920: 8db0d62d750c510e7c0458e7644926be5d8d11d099dde3ed97591f6ecc10e278 + 20230911: 34aa27e693666335c69d233747be2fccfa8665ef88381d451c0b1a33a2050ae1 + 20230904: 990734ea106aaac65bd97ed496f7f87115fc743896ee4ef897e12c10204aea9d + amd64: + 20230807: bb5055d820a3698181b593e3f6d2b44e8e957a6df91bea7776fee030c007814f + 20230801: 9df74be6ed44f4b35d5aa5ba1956bb3959680c6909748009a2f9476e04b0921e + 20230731: 50e586ce482bce290d893277644ca950a3ab09b53719170578b324666c6e0eec + 20230724: 859476ba858012724d845b3d417070dfb577b68640f0c0840712485710c975fd + 20230717: 4ab67b11728ab1f8d5a897c3346bfb2d725c73c358a588eab0ad3b3ddaa094c8 + 20230710: cd5648f0e32862a4a733840c565c112fe7b505970a10d3b2e882805158bd82b2 + 20230627: d2db10692a56c73cf2fb78776cb41a911db26f65d122922455786db63e25e22c + 20230621: d4e03fee422c87c8f533493de04286277def528e768b3fc90d22c15d8cc1ff7e + 20230605: 85c8feb5d71f45abe29f97548c5432966cd1e57feab560a923f2ea64395b63c1 + 20230529: 41eb6a5fc545e27c253a21b2d6881c7bfed25932cc34e042023600d17a5b3cd9 + 20230522: 5587bcd68e32432d596492c78e935463a530980c35c34abd215d84e93efd1e8f + 20230517: 20ec2fc2e5bc90840ae6540fcd83879e84c7495b0400bc377f9d502b9fdff591 + 20230508: 2a1385d3ef6e31058671e2d2a1ce83130e934081fa2c5c93589eebf7856f5681 + 20230501: b60dccad63a07553809065d1c4d094dfc5e862353cc61a933b05100ffd1f831c + 20230417: 7c0ccb6144861e45bd14e2ccd02f3fdb935219447256d16c71f6c8e42f90a73d + 20231030: 61467bc39f58109dd7a2115e7bcecff460565fc681b0e89436a88d5c316300c8 + 20231023: 99363d5d432bf466f2de2f9de6be140fa1eee0ffc3fceceec87ef0fad907015e + 20231016: dc06735fd3bb333a08294931c55c6867e679dc484713e9262ec5dc258e8a08d6 + 20231009: 9532413b235dace99911192ae82414fd20be4af7f1d3479624412e49be707500 + 20231003: e925c65f51c879ee29ceb9d346a400b2efd8e36deb68028845f3fea1ccb57710 + 20230925: 72a0ef23ae6d487e164aa9749c2e1f81fb0512a18a1f776d9adc7b0c156ed194 + 20230920: 1fe85e4a6963fea744cb70044d38063e4b884dccf7ed7308c1e75cb802fbc276 + 20230911: 2048af00625f45d7134a651cd836f0be29c2edfc8f061162ea3fde2b212c443b + 20230904: 5c107f15c5be5cfacd1be0e3745fb680bee516ee613a0c6d1e9314795c911458 + ppc64le: + 20230807: 0 + 20230801: 0 + 20230731: 0 + 20230724: 0 + 20230717: 0 + 20230710: 0 + 20230627: 0 + 20230621: 0 + 20230605: 0 + 20230529: 0 + 20230522: 0 + 20230517: 0 + 20230508: 0 + 20230501: 0 + 20230417: 0 + 20231030: 0 + 20231023: 0 + 20231016: 0 + 20231009: 0 + 20231003: 0 + 20230925: 0 + 20230920: 0 + 20230911: 0 + 20230904: 0 +gvisor_containerd_shim_binary_checksums: + arm: + 20230807: 0 + 20230801: 0 + 20230731: 0 + 20230724: 0 + 20230717: 0 + 20230710: 0 + 20230627: 0 + 20230621: 0 + 20230605: 0 + 20230529: 0 + 20230522: 0 + 20230517: 0 + 20230508: 0 + 20230501: 0 + 20230417: 0 + 20231030: 0 + 20231023: 0 + 20231016: 0 + 20231009: 0 + 20231003: 0 + 20230925: 0 + 20230920: 0 + 20230911: 0 + 20230904: 0 + arm64: + 20230807: 0b80fba82a7c492dc8c7d8585d172d399116970155c2d35f3a29d37aa5eeb80d + 20230801: 08323b1db170fe611f39306922bf56c8c4ee354d6881385fae0a27d84d6f5a62 + 20230731: 74e72b4c7f0f818f8f1a983072beca3c27f72536be808597fb1f9878f736c6ef + 20230724: 78d106ae19da764065e3e16573f347609206ebc72b48a678e14c2cf5ad04c901 + 20230717: 6425bc2285d8a6322bd9e18864da601b9ba56d7d1759a9a93805e270621036da + 20230710: 3ea3487f5d9ba87fc557c43d8f3b830780696048805883255757c940b669664c + 20230627: 5b0db73bcd6b051e7398fd7bb894bb9969ba7a1291d036aac3d47448e59c5aca + 20230621: bf5fb12a0aba6f3c1520f1b7b18d5bbdc05a5ceeb7143feb46063434c70253de + 20230605: 5ccd58ba24c1b44e6d0add79fd728d124e40b0ccfa40e9731f357719cc020bfc + 20230529: 3b1dd2904e31ab8da3aa518cdee2540caed67eecbe4fc87e143dbe90f294e31d + 20230522: 850bab454c134adefc20bc0f24015beb562ec289537b67c42e6ea3781e1fb763 + 20230517: e3ccb7aea2708d664e2b278b7c40dbf0156e2d2f773e2b13ba5766beb67b7ad2 + 20230508: d16d59d076b0856242d67eda95ee1b2301b04f14abd95ef4fe6c08509f304617 + 20230501: a5f4361897a634ac5832b269e1cc5bc1993825c06e4b0080770a948b36584754 + 20230417: 575163e65e1fda019cb34ee56120d5ecf63b0a3a00dda28c0fc138ce58a5bbff + 20231030: 6b2a6ff15d37e3cffe09cacf8dc22255759dc87f6c2290253c9bc82b4ba15909 + 20231023: ac80cb6a9be9697eefdbe0b1f60e00657f8b97552b9f7d3105b03fa30fb9e99b + 20231016: 3f513a2c042096f7636e6c2e1313ed43d2fae43b2428dcc3135f6f844e559f8d + 20231009: 638167817a34b73b3c737f4755a7564abc51ab42327334e652d7a70116e8fada + 20231003: ed8a6df203dcb80f44d239452c88ef4b86123e85443c3d7f019512d37611bab5 + 20230925: ed8a6df203dcb80f44d239452c88ef4b86123e85443c3d7f019512d37611bab5 + 20230920: a397cdf0f3a08d5f9b54ce40e01cc07373b270649caf065e2a21400f98c99687 + 20230911: f62538996f4680ace4c38dccb7fe433ad7c95622a65b1afad0f9cb35d249e1ab + 20230904: 24480f319a51d8d66a4617d80b113069ca56e1157c731fb39a0f412a1d3e4176 + amd64: + 20230807: fa16b92b3a36665783951fa6abb541bca52983322d135eb1d91ae03b93f4b477 + 20230801: 372a7c60054ab3351f5d32c925fb19bb0467b7eb9cd6032f846ba3f5b23451f8 + 20230731: 8d3f2a3c5634d56a2c2cf6017ccb2ef2764c1bceb3814a1276efe043b440d0b6 + 20230724: ff9c5e51f210c06a3e07a71f6214a0420651cd4c30fddd04d01b6d09ce24eb61 + 20230717: cf342881148b80ed95a8946f9f44781bb9323211e3c7cdc0254d27af5c5b215f + 20230710: db008c53e5a2a868d8b44df0faa051eae7fc5241f1488b53c6040a3665ac04b8 + 20230627: e55b22970b81ccee9b9d0837b11d2a20ad670fc4c028c7e87d03721257e928a9 + 20230621: 180a7320466b997114a903b034fe71aa3a65b9374b3565f7461df587b167e5de + 20230605: 4be04c512c72415b3456b0e94ca9d0c850ffd2c5fd06211f2515da70ac7f74b7 + 20230529: a8fcba918f06d984f4626acef700ab359350d631431564ca37a9ce8b4869eab1 + 20230522: 72587a1f458f20eb029f01a578fe018219688b2c4461453fdb15e97efc987e55 + 20230517: 8de99272600cfdc8b3319dc77f63c57ae592f1319b3c1b50245bc67a0740f2ae + 20230508: 7e4c74b8fc05738a7cdf7c8d99d8030007894cbc7dda774533482c5c459a8ba9 + 20230501: f951c2b8d017005787437d9f849c3edfff54c79d39eb172f92abe745652ef94d + 20230417: 61c3d75a46c8d2652306b5b3ab33e4fb519b03a3852bea738c2700ec650afe4e + 20231030: ea485191cd95d57d7e7fd1a59f7e42a624432a5634bd9b6e4f3d37e86ab0e935 + 20231023: 5045ac4983701aea469ec9934b9fd37c292259682c35e2e25f664633d183db93 + 20231016: a9f85a10b914526f78465142958c82c058484cbab1f6af5bf4cf1d95a0322ea3 + 20231009: 274eeb298a538d899ca19a659120b8249dd3169f3128a1e5794308a241ce3fcc + 20231003: c96d0f062d979249e03355421659363a353310169df86511861505f329ab1614 + 20230925: c96d0f062d979249e03355421659363a353310169df86511861505f329ab1614 + 20230920: 09eafdf9d37e51fa9719a3752c11564124a5ba509fa50c651f81967a0051781d + 20230911: 6f1f1fc33f0dc1255642ece98b909ebccf3a54a1a2c950d1d84928b78a7c3e22 + 20230904: 73b2e9c4761622ef82eefe74a3346196834728ad8b980101e2d87c6e3c48015c + ppc64le: + 20230807: 0 + 20230801: 0 + 20230731: 0 + 20230724: 0 + 20230717: 0 + 20230710: 0 + 20230627: 0 + 20230621: 0 + 20230605: 0 + 20230529: 0 + 20230522: 0 + 20230517: 0 + 20230508: 0 + 20230501: 0 + 20230417: 0 + 20231030: 0 + 20231023: 0 + 20231016: 0 + 20231009: 0 + 20231003: 0 + 20230925: 0 + 20230920: 0 + 20230911: 0 + 20230904: 0 +nerdctl_archive_checksums: + arm: + 1.7.1: 799d35de7a182da35d850308c7f1787cd7321404348ff2d5ba64ad43b06b395a + 1.7.0: 8b9e7cccbcc0a472685d1bc285f591f41005f8699e7265ea5438a3e06aefdcfd + 1.6.2: 69363f4dbf2616d5238647bfbff60525b7b59417a26de8eb255b6d6a09171175 + 1.6.1: 89187ff46c5a515a5635a4017a476d82cdc1fc3de906135273c64329189b906e + 1.6.0: 20dc5f6912de321d4b6aa8647ce77c261cd6d5726d0b5dfae56bd9cdbd7c28fb + 1.5.0: 36c44498b08a08b652d01812e5f857009373fba64ce9c8ff22e101b205bbc5fb + 1.4.0: b81bece6e8a10762a132f04f54df60d043df4856b5c5ce35d8e6c6936db0b6a0 + 1.3.1: ed24086dbea22612dbcc3d14ee6f1d152b0eb6905bd8c84d3413c1f4c8d45d10 + 1.3.0: 2fa1b0cdb95cafb9ea6293c86a164b2a00b342e02d7a9444794f48f002e187c3 + 1.2.1: 750e4788515779cb8e3ea18678c3125b5724e521214bb18a799903821c17d32c + 1.2.0: d42b3329c3c99d6243a4efa0f33e9324a26597e175dd7fdae574270bf3e26f28 + 1.1.0: cc3bc31b4df015806717149f13b3b329f8fb62e3631aa2abdbae71664ce5c40d + 1.0.0: 8fd283a2f2272b15f3df43cd79642c25f19f62c3c56ad58bb68afb7ed92904c2 + arm64: + 1.7.1: 46affa0564bb74f595a817e7d5060140099d9cfd9e00e1272b4dbe8b0b85c655 + 1.7.0: 1255eea5bc2dbac9339d0a9acfb0651dda117504d52cd52b38cf3c2251db4f39 + 1.6.2: ece848045290dd61f542942248587e91125563af46c0ea972a7c908d0d39c96c + 1.6.1: b91ec17a6f7bcb148ed7ad086da6c470ee33f7218c769d5d490e0a1d6a45fdb4 + 1.6.0: d5f1ed3cda151385d313f9007afc708cae0018c9da581088b092328db154d0c6 + 1.5.0: 1bb613049a91871614d407273e883057040e8393ef7be9508598a92b2efda4b7 + 1.4.0: 0edb064a7d68d0425152ed59472ce7566700b4e547afb300481498d4c7fc6cf1 + 1.3.1: 9e82a6a34c89d3e6a65dc8d77a3723d796d71e0784f54a0c762a2a1940294e3b + 1.3.0: e3405bbaadbee716e50ce4535d03f854129773152aab4876b14f117e1ed3b5ee + 1.2.1: 8dc3d918b44b3ea863a4bc8f121277389d3bdb952d549e44916502a0774ab1bb + 1.2.0: 79d71bbdd0b433838d64ef96d26eb9648911f8d5e5ab494359d32d0ff09abb34 + 1.1.0: a0b57b39341b9d67a3f0ae74e19985c72e930bad14291cbbd8479ed6a6a64e83 + 1.0.0: 27622c9d95efe6d807d5f3770d24ddd71719c6ae18f76b5fc89663a51bcd6208 + amd64: + 1.7.1: 5fc0a6e8c3a71cbba95fbdb6833fb8a7cd8e78f53de10988362d4029c14b905a + 1.7.0: 844c47b175a3d6bc8eaad0c51f23624a5ef10c09e55607803ec2bc846fb04df9 + 1.6.2: 67991fc144b03596f15be6c20ca112d10bd92ad467414e95b0f1d60d332ae34e + 1.6.1: 992e4ffd3d88cf197f78b78333ac345faf5f184a119d43ad8a106f560781fd89 + 1.6.0: fc3e7eef775eff85eb6c16b2761a574e83de444831312bc92e755a1f5577872d + 1.5.0: 6dc945e3dfdc38e77ceafd2ec491af753366a3cf83fefccb1debaed3459829f1 + 1.4.0: d8dcd4e270ae76ab294be3a451a2d8299010e69dce6ae559bc3193535610e4cc + 1.3.1: 3ab552877100b336ebe3167aa57f66f99db763742f2bce9e6233644ef77fb7c9 + 1.3.0: 28299050ed28ed78db4fed95daef1ce326ce0101569dc73cc49f8f7e0c17de25 + 1.2.1: 67aa5cf2a32a3dc0c335b96133daee4d2764d9c1a4d86a38398c4995d2df2176 + 1.2.0: 9d6f3427a1c0af0c38a0a707751b424d04cca13b82c62ad03ec3f4799c2de48c + 1.1.0: fcfd36b0b9441541aab0793c0f586599e6d774781c74f16468a3300026120c0e + 1.0.0: 3e993d714e6b88d1803a58d9ff5a00d121f0544c35efed3a3789e19d6ab36964 + ppc64le: + 1.7.1: 09fd0cbef25c98e08c5cc2d1e39da279cbf66c430fdf6c8738e56ce8f949dad9 + 1.7.0: e421ae655ff68461bad04b4a1a0ffe40c6f0fcfb0847d5730d66cd95a7fd10cd + 1.6.2: 3b0d6e4c42b99e2dd8059ded81cde69f42b065d9f486142f3c9b0861ba7effef + 1.6.1: 3924467d9430df991ebdf4e78211bac2b29e9a066d5000d98f8d4ebde2bb7b4c + 1.6.0: c47717ed176f55b291d2068ed6e2445481c391936bd322614e0ff9effe06eb4d + 1.5.0: 169d546d35ba3e6ef088cc81e101f58d5ecb08e71c5ed776c99482854ea3ef8a + 1.4.0: 306d5915b387637407db67ceb96cd89ff7069f0024fb1bbc948a6602638eceaa + 1.3.1: 21700f5fe8786ed7749b61b3dbd49e3f2345461e88fe2014b418a1bdeffbfb99 + 1.3.0: 8eda05b803da56772e38dd0970279e081ba797f6d36ad28a535ba8a98993074c + 1.2.1: d5c60700709e8c3908e9863df57303e43d343e38aadf96f2d06eaaac3bc2b06b + 1.2.0: 64fb56543ee69dafa57691a9dc3b756f9cd6fec021ba1c591680d16ebb8d109d + 1.1.0: 7e97d0a856439d07e82cc26a16dfe243c21da14b7099e330e4da11e825004478 + 1.0.0: 2fb02e629a4be16b194bbfc64819132a72ede1f52596bd8e1ec2beaf7c28c117 +containerd_archive_checksums: + arm: + 1.7.13: 0 + 1.7.12: 0 + 1.7.11: 0 + 1.7.10: 0 + 1.7.9: 0 + 1.7.8: 0 + 1.7.7: 0 + 1.7.6: 0 + 1.7.5: 0 + 1.7.4: 0 + 1.7.3: 0 + 1.7.2: 0 + 1.7.1: 0 + 1.7.0: 0 + 1.6.28: 0 + 1.6.25: 0 + 1.6.24: 0 + 1.6.23: 0 + 1.6.22: 0 + 1.6.21: 0 + 1.6.20: 0 + 1.6.19: 0 + 1.6.18: 0 + 1.6.17: 0 + 1.6.16: 0 + 1.6.15: 0 + 1.6.14: 0 + 1.6.13: 0 + 1.6.12: 0 + 1.6.11: 0 + 1.6.10: 0 + 1.6.9: 0 + 1.6.8: 0 + 1.6.7: 0 + 1.6.6: 0 + 1.6.5: 0 + 1.6.4: 0 + 1.6.3: 0 + 1.6.2: 0 + 1.6.1: 0 + 1.6.0: 0 + 1.5.18: 0 + 1.5.17: 0 + 1.5.16: 0 + 1.5.15: 0 + 1.5.14: 0 + arm64: + 1.7.13: 118759e398f35337109592b4d237538872dc12a207d38832b9d04515d0acbc4d + 1.7.12: 8a1b35a521d071a8828f63fe007a51e5b7ac863a1195f5dee32543b1a9d5f2b6 + 1.7.11: 5eae27cce38a14be5390d4035127aa11416bc5ae592a9ff25b11870872ce1159 + 1.7.10: 0667b12a04a896a61cf508a4a77190c280f4a1fa35f38c8a4ba63f605b5ec375 + 1.7.9: 09ca326dee14e00c439137071747c15cc280480e2c26c1e82698c992dd1889c6 + 1.7.8: 3fc551e8f51150804d80cc1958a271bd2252b6334f0355244d0faa5da7fa55d1 + 1.7.7: 0a104f487193665d2681fcb5ed83f2baa5f97849fe2661188da835c9d4eaf9e3 + 1.7.6: d844a1c8b993e7e9647f73b9814567004dce1287c0529ce55c50519490eafcce + 1.7.5: 98fc6990820d52d45b56ea2cda808157d4e61bb30ded96887634644c03025fa9 + 1.7.4: ea5a04379bd4252fc1e0b7b37f69cd516350c5269054483535d6eab7a0c79d2e + 1.7.3: 85d2eaedabff57ac1d7cd3884bf232155c4c46491f6b071982e4f7b684b74445 + 1.7.2: d75a4ca53d9addd0b2c50172d168b12957e18b2d8b802db2658f2767f15889a6 + 1.7.1: 1f828dc063e3c24b0840b284c5635b5a11b1197d564c97f9e873b220bab2b41b + 1.7.0: e7e5be2d9c92e076f1e2e15c9f0a6e0609ddb75f7616999b843cba92d01e4da2 + 1.6.28: 96a231f875ddf9cc7682b881d408ae993f2bd5d0a40402a74ec4fda672047427 + 1.6.26: 177bed65b6425255bacbe48d99ea7aa5209d381576962c0962dc8615ef16c5c5 + 1.6.25: 4948677cfc5f98a1d5d46cec90d6d6f84f6b27cd6d28fd87f7f5936d61580ceb + 1.6.24: 1d741e9e2d907f02a8b2a46034a28ff9aacdba88c485cef2f4bad18be9ea23ba + 1.6.23: ea7afb82dc5789307e684ef9b4a55ce1ee9a05dc02c2118df640b01207208c45 + 1.6.22: 7882d6e7f4e97dcba041c37592c4cb9e7a5b4d972380c74d959e388b12d57d01 + 1.6.21: d713d8fbec491705ffe8c33ecc9051a904f6eedc92574928e1d33616f291c583 + 1.6.20: c3e6a054b18b20fce06c7c3ed53f0989bb4b255c849bede446ebca955f07a9ce + 1.6.19: 25a0dd6cce4e1058824d6dc277fc01dc45da92539ccb39bb6c8a481c24d2476e + 1.6.18: 56b83a0bc955edc5ebaa3bd0f788e654b63395be00fcb1bd03ff4bdfe4b5e1e7 + 1.6.17: 7e110faa738bff2f5f0ffd54c4ec2c17c05fd2af6de4877c839794ca3dadd61c + 1.6.16: c2bf51fde02ec9cf8b9c18721bc4f53bd1f19fb2bb3251f41ece61af7347e082 + 1.6.15: d63e4d27c51e33cd10f8b5621c559f09ece8a65fec66d80551b36cac9e61a07d + 1.6.14: 3ccb61218e60cbba0e1bbe1e5e2bf809ac1ead8eafbbff36c3195d3edd0e4809 + 1.6.13: 8c7892ae7c2e96a4a9358b1064fb5519a5c0528b715beee67b72e74d7a644064 + 1.6.12: 0a0133336596b2d1dcafe3587eb91ab302afc28f273614e0e02300694b5457a0 + 1.6.11: 1b34d8ff067da482af021dac325dc4e993d7356c0bd9dc8e5a3bb8271c1532de + 1.6.10: 6d655e80a843f480e1c1cead18479185251581ff2d4a2e2e5eb88ad5b5e3d937 + 1.6.9: 140197aee930a8bd8a69ff8e0161e56305751be66e899dccd833c27d139f4f47 + 1.6.8: b114e36ecce78cef9d611416c01b784a420928c82766d6df7dc02b10d9da94cd + 1.6.7: 4167bf688a0ed08b76b3ac264b90aad7d9dd1424ad9c3911e9416b45e37b0be5 + 1.6.6: 807bf333df331d713708ead66919189d7b142a0cc21ec32debbc988f9069d5eb + 1.6.5: 2833e2f0e8f3cb5044566d64121fdd92bbdfe523e9fe912259e936af280da62a + 1.6.4: 0205bd1907154388dc85b1afeeb550cbb44c470ef4a290cb1daf91501c85cae6 + 1.6.3: 354e30d52ff94bd6cd7ceb8259bdf28419296b46cf5585e9492a87fdefcfe8b2 + 1.6.2: a4b24b3c38a67852daa80f03ec2bc94e31a0f4393477cd7dc1c1a7c2d3eb2a95 + 1.6.1: fbeec71f2d37e0e4ceaaac2bdf081295add940a7a5c7a6bcc125e5bbae067791 + 1.6.0: 6eff3e16d44c89e1e8480a9ca078f79bab82af602818455cc162be344f64686a + 1.5.18: 0 + 1.5.17: 0 + 1.5.16: 0 + 1.5.15: 0 + 1.5.14: 0 + amd64: + 1.7.13: c2371c009dd8b7738663333d91e5ab50d204f8bcae24201f45d59060d12c3a23 + 1.7.12: 6a24d8b996533fa1b0d7348fe9813a78cd01fa16cff865a961ad0d556f5cd665 + 1.7.11: d66161d54546fad502fd50a13fcb79efff033fcd895adc9c44762680dcde4e69 + 1.7.10: eacb0296bff2ae5225a18492dcb32fb28ad4a1fe0a39ea9073367c7e43dc5838 + 1.7.9: ccd5b434393666f6ebbc90eea959ffd3e61958a1e3e1cc830a678f040142d4b0 + 1.7.8: 5f1d017a5a7359514d6187d6656e88fb2a592d107e6298db7963dbddb9a111d9 + 1.7.7: 371de359d6102c51f6ee2361d08297948d134ce7379e01cb965ceeffa4365fba + 1.7.6: 58408cfa025003e671b0af72183b963363d519543d0d0ba186037e9c57489ffe + 1.7.5: 33609ae2d5838bc5798306a1ac30d7f2c6a8cff785ca6253d2be8a8b3ccbab25 + 1.7.4: fc070fabfe3539d46ae5db160d18381270928b3f912e2e800947e9fbd43f510c + 1.7.3: de7f61aacba88ee647a7dcde1ca77672ec44ab9fb3e58ae90c0efc9b2d8f3068 + 1.7.2: 2755c70152ab40856510b4549c2dd530e15f5355eb7bf82868e813c9380e22a7 + 1.7.1: 9504771bcb816d3b27fab37a6cf76928ee5e95a31eb41510a7d10ae726e01e85 + 1.7.0: b068b05d58025dc9f2fc336674cac0e377a478930f29b48e068f97c783a423f0 + 1.6.28: b2f15c722d1cc8b74ed643068e043b92bd031fc23d53488d1e837cf4b2777391 + 1.6.26: fa806d3e945a8ad25aa1f8123a98524768ead83af2ed1ab3d922d2dd5fe6b14c + 1.6.25: 878b331b5fa65df3d33c68ee355724de0044c25071486086409b374a9c62d145 + 1.6.24: a56fac5ba03c3d6f74ceae14abdc9fafabcba900105e9890c0ac895cc00164ad + 1.6.23: bcf16bb63a295721a2603e9a56602c5d18e5443df04a9f2c1ca5328f41556fcc + 1.6.22: 5671eb4eba97f0ec98223c84401c9aeb21d0ef16ac3ece3eb8fadd46174d7eab + 1.6.21: 04dcc1b99368492caee758583e531392683268197e58156888a3cea2941117b6 + 1.6.20: bb9a9ccd6517e2a54da748a9f60dc9aa9d79d19d4724663f2386812f083968e2 + 1.6.19: 3262454d9b3581f4d4da0948f77dde1be51cfc42347a1548bc9ab6870b055815 + 1.6.18: c4e516376a2392520a87abea94baf2045cc3a67e9e0c90c75fb6ed038170561e + 1.6.17: 5f0584d000769d0cf08fc0e25135614ef5bf52971a6069175c78437699f3b8d4 + 1.6.16: 2415b431a900275c14942f87f751e1e13d513c1c2f062322b5ca5a9a2190f22a + 1.6.15: 191bb4f6e4afc237efc5c85b5866b6fdfed731bde12cceaa6017a9c7f8aeda02 + 1.6.14: 7da626d46c4edcae1eefe6d48dc6521db3e594a402715afcddc6ac9e67e1bfcd + 1.6.13: 97f00411587512e62ec762828e581047b23199f8744754706d09976ec24a2736 + 1.6.12: a56c39795fd0d0ee356b4099a4dfa34689779f61afc858ef84c765c63e983a7d + 1.6.11: 21870d7022c52f5f74336d440deffb208ba747b332a88e6369e2aecb69382e48 + 1.6.10: dd1f4730daf728822aea3ba35a440e14b1dfa8f1db97288a59a8666676a13637 + 1.6.9: 9ee2644bfb95b23123f96b564df2035ec94a46f64060ae12322e09a8ec3c2b53 + 1.6.8: 3a1322c18ee5ff4b9bd5af6b7b30c923a3eab8af1df05554f530ef8e2b24ac5e + 1.6.7: 52e817b712d521b193773529ff33626f47507973040c02474a2db95a37da1c37 + 1.6.6: 0212869675742081d70600a1afc6cea4388435cc52bf5dc21f4efdcb9a92d2ef + 1.6.5: cf02a2da998bfcf61727c65ede6f53e89052a68190563a1799a7298b0cea86b4 + 1.6.4: f23c8ac914d748f85df94d3e82d11ca89ca9fe19a220ce61b99a05b070044de0 + 1.6.3: 306b3c77f0b5e28ed10d527edf3d73f56bf0a1fb296075af4483d8516b6975ed + 1.6.2: 3d94f887de5f284b0d6ee61fa17ba413a7d60b4bb27d756a402b713a53685c6a + 1.6.1: c1df0a12af2be019ca2d6c157f94e8ce7430484ab29948c9805882df40ec458b + 1.6.0: f77725e4f757523bf1472ec3b9e02b09303a5d99529173be0f11a6d39f5676e9 + 1.5.18: d132525d375bbafd3aee8e9aa5517203cef2bbf77197db7522c8730cc526b3db + 1.5.17: b676f56f43bf02782179320b5070fb210cbc1784a9b0875086bc15c3bcc546f3 + 1.5.16: f5326865c2b86aba794f590fd2ca479f817fa1f88c084d97a45816aec9d0ce32 + 1.5.15: 0d09043be08dcf6bf136aa78bfd719e836cf9f9679afa4db0b6e4d478e396528 + 1.5.14: 8513ead11aca164b7e70bcea0429b4e51dad836b6383b806322e128821aaebbd + ppc64le: + 1.7.13: 89605ed2365d5eb779477d11947101236eb44e5244f1e58bb162a9e68d242798 + 1.7.12: 80f16891b387d86712026234de7d4d0365a38106dbe5e51b65b1200b24822721 + 1.7.11: 6f91c5dabdccd1fc75aae8687381bb185b9eb4200beb29d0993dea8175f5fa61 + 1.7.10: 15a5191bf7c555956a8565d8786399d51b13f2718d59b1a5b2bd380fc420bf8a + 1.7.9: 174b8af2d878ad8410205b9ba44fa8d2a9683a521abf13f168f67b7f7375d5b3 + 1.7.8: 2b563df9e1bddc96a99a023963c99b5faf3066d3fcbc23ff44ba24229e939444 + 1.7.7: 0335e7447ed84757489337686a709e95ffa379a8780f238725abb10facaeaa7f + 1.7.6: 956fadb01b35c3214f2b6f82abc0dda3e1b754cb223cd24e818334b08cb09fb2 + 1.7.5: 2496e24a95fa74750363a8a7e2ac36acf8d41ee2e4b67a452154ad4c8efbc4bc + 1.7.4: c3397f67fb5756e6336ff76eeb34dfad7ad66235877a4186d82044c7a4caf482 + 1.7.3: d1977922e74147782dd5bb488f260ee14d758d29a7651cd97bc2e6c7cc1a3cce + 1.7.2: cbe7ec913cb603ca218bd8867efdce4bee3b0e0115e467e51c910467daf8184e + 1.7.1: 17d97ef55c6ce7af9778dbafb5e73f577d1b34220043a91cccde49dbcc610342 + 1.7.0: 051e897d3ee5b8c8097f65be447fea2d29226b583ca5d9ed78e9aebcf4e69889 + 1.6.28: 35411f9d1bafc9cae91c2e30d46a59d01bffc7e18ac7f0942dca9d1d5907ab38 + 1.6.26: 75fb01a4bd3bcd16263c2f833b8e7081356e2e390dd7eb0710232cb04dac5a01 + 1.6.25: 3ddcc1739ffeb3e1df786d45518a01d93a5cef243eb6dee61cbdd4cd110bc723 + 1.6.24: abff9e7ec4cc21d19150d2bc55fc89cf53dc03c002cdaf5016ee82aedead9b03 + 1.6.23: 1099fce4a4dfc78712cc1c19be6d9f80e9321f513834dba0b2418bd5c78ad398 + 1.6.22: 0f8647aedd96174704a63a17b1a7cf4c4c5c2fc066606b1a419b2a860b754bfb + 1.6.21: 196d91799070a5ff2f5f3b6efe8516c5377f299d38012b6c0cf4ae77fc8c22c5 + 1.6.20: f3ee666fdd31031f07cbb7bad24f0181fad1094ba36273dc61f8fa5a570b5311 + 1.6.19: 18cf11b6dfc980aca8792a2cd3ea7afed6379c2988ca6fe9e53a19a0bece5a2d + 1.6.18: b7083473061a61200d04f500ad4a96813d1b09f71b2d427019076836c2a49836 + 1.6.17: 2f689ff36ba41c3c86ce926f55b0101118a40dd7b741946386062fddaa287db0 + 1.6.16: 9cfd5dade6a1c2671f5c76496395afe0aa0ce902c13672b306d8d09fdbb99492 + 1.6.15: 502f3e4c8ea2018aaa285fe4f704bfd560fdf93193bb829dd9302d013bc38370 + 1.6.14: 73025da0666079fc3bbd48cf185da320955d323c7dc42d8a4ade0e7926d62bb0 + 1.6.13: f2508ada0c8bd7d3cb09b0e7f10416aba3d643c0da7adc27efe4e76d444322ae + 1.6.12: 088e4d1fe1787fc4a173de24a58da01880d1ead5a13f1ab55e1ade972d3907d4 + 1.6.11: e600a5714ffb29937b3710f9ae81bb7aa15b7b6661192f5e8d0b9b58ac6d5e66 + 1.6.10: 704b1affd306b807fe6b4701d778129283635c576ecedc6d0a9da5370a07d56a + 1.6.9: fe0046437cfe971ef0b3101ee69fcef5cf52e8868de708d35f8b82f998044f6e + 1.6.8: f18769721f614828f6b778030c72dc6969ce2108f2363ddc85f6c7a147df0fb8 + 1.6.7: 0db5cb6d5dd4f3b7369c6945d2ec29a9c10b106643948e3224e53885f56863a9 + 1.6.6: 0 + 1.6.5: 0 + 1.6.4: 0 + 1.6.3: 0 + 1.6.2: 0 + 1.6.1: 0 + 1.6.0: 0 + 1.5.18: 0 + 1.5.17: 0 + 1.5.16: 0 + 1.5.15: 0 + 1.5.14: 0 +skopeo_binary_checksums: + arm: + v1.13.3: 0 + v1.13.2: 0 + v1.13.1: 0 + v1.13.0: 0 + v1.12.0: 0 + v1.11.2: 0 + v1.11.1: 0 + v1.11.0: 0 + v1.10.0: 0 + v1.9.3: 0 + v1.9.2: 0 + arm64: + v1.13.3: 1f7726b020ff9bc931ce16caa13c29999738a231f1414028282cd8f8661eb747 + v1.13.2: 520cc31c15796405b82d01c78629d5b581eced3512ca0b6b184ed82f5e18dc86 + v1.13.1: 3b7db2b827fea432aa8a861b5caa250271c05da70bd240aa4045f692eba52e24 + v1.13.0: d23e43323c0a441d1825f9da483b07c7f265f2bd0a4728f7daac4239460600a3 + v1.12.0: f34476bd33d2ab3784675611b405cc0855ce5decedfa22287e261d23d17e7688 + v1.11.2: cd90552f7d4eb78ba032c885b47cd97ef015a958279d2a2b828b109d75d6c7e0 + v1.11.1: 693f7d2791e0549b173b2c16f1c3326328aa5e95bc2b4d71f5ecd35b6524b09d + v1.11.0: 3e510999ffb6544b11d339812df75d14a46261518b5c73f530242ebed679fb1b + v1.10.0: 3bfc344d4940df29358f8056de7b8dd488b88a5d777b3106748ba66851fa2c58 + v1.9.3: 27c88183de036ebd4ffa5bc5211329666e3c40ac69c5d938bcdab9b9ec248fd4 + v1.9.2: 1b7b4411c9723dbbdda4ae9dde23a33d8ab093b54c97d3323784b117d3e9413f + amd64: + v1.13.3: 65707992885b1a4a446af6342874749478a1af7e17ab3f4df8fb89509e8b1966 + v1.13.2: 2f00be6ee1c4cbfa7f2452be90a1a2ce88fd92a6d0f6a2e9d901bd2087bd9092 + v1.13.1: 8c15c56a6caffeb863c17d73a6361218c04c7763e020fffc8d5d6745cacfa901 + v1.13.0: 8cb477ee25010497fc9df53a6205dbd9fe264dd8a5ea4e934b9ec24d5bdc126c + v1.12.0: 38143238e945959e6b24dba1447ba49e0b79f10a3aef2634391d0205ab950003 + v1.11.2: c8641decb185f43bb49f3f2a68abbc22e051a497440beba28b10d25d0a856574 + v1.11.1: 9aad99e41533800ce08526a602de2f87b8ce123ea9547358e2cccfa2f9c3a9e0 + v1.11.0: 0a6c0a1b349d2efd2895d2ec9a1d9c5c4bbd59f10c3993acb0c92d31914fbd62 + v1.10.0: 20fbd1bac1d33768c3671e4fe9d90c5233d7e13a40e4935b4b24ebc083390604 + v1.9.3: 6e00cf4661c081fb1d010ce60904dccb880788a52bf10de16a40f32082415a87 + v1.9.2: 5c82f8fc2bcb2502cf7cdf9239f54468d52f5a2a8072893c75408b78173c4ba6 + ppc64le: + v1.13.3: 0 + v1.13.2: 0 + v1.13.1: 0 + v1.13.0: 0 + v1.12.0: 0 + v1.11.2: 0 + v1.11.1: 0 + v1.11.0: 0 + v1.10.0: 0 + v1.9.3: 0 + v1.9.2: 0 +yq_checksums: + arm: + v4.40.1: 5a005b89cb63994f999785716ab196160042516cad53a0535244f76aad468966 + v4.35.2: 000e1a8e82be5e99341c507a2abe93e104f0d4619dc7df742e88043206544c7e + v4.35.1: b2349bc220394329bc95865375feb5d777f5a5177bcdede272788b218f057a05 + v4.34.2: 161f2b64e7bf277614983014b2b842e9ae9c1f234a9ea12593b0e5ebe5a89681 + v4.34.1: dfda7fc51bdf44d3551c4bca78ecd52c13d7137d99ec3f7b466c50333e0a0b7c + v4.33.3: 77c239e17cb50a330da3c48af7dbd3b667af02c950a9ecae49b171cc0bf66c48 + v4.33.2: ee8d61975ebdfabf9b79ed2947ebfe39d62f290ab71d6eb53af183f06cfa1af2 + v4.33.1: 80ec1487b9497e0c5f182c8e44d73b5e9c437719230e114b9fce2fa25c23f95a + v4.32.2: 776b8dcfaf796255e60c0e9893d24ce9eb61cc7e787a1cc8b4a5dc37a84f02d5 + v4.32.1: 0fc6c3e41af7613dcd0e2af5c3c448e7d0a46eab8b38a22f682d71e35720daed + v4.31.2: 4873c86de5571487cb2dcfd68138fc9e0aa9a1382db958527fa8bc02349b5b26 + arm64: + v4.40.1: 11491c62fa0af9995f26a64e9cce97c8404018bb6b0acd7d7ac1be0f437ecf28 + v4.35.2: 6ea822bc966e7dc23bb7d675a1ff36bc2e7a9a9f88c402129eafbd6b19d8ff8a + v4.35.1: 1d830254fe5cc2fb046479e6c781032976f5cf88f9d01a6385898c29182f9bed + v4.34.2: 6ea70418755aa805b6d03e78a1c8a1bf236220f187ba3fb4f30663a35c43b4c1 + v4.34.1: c1410df7b1266d34a89a91dcfeaf8eb27cb1c3f69822d72040d167ec61917ba0 + v4.33.3: 15925a972d268bcb0a7aa2236c7e5925b7a3ba4b5569bb57e943db7e8c6f834f + v4.33.2: aa86e5f36850f9350a7393c7cf654ee726df99ae985b663eb3605ca2bdd24692 + v4.33.1: e3a47e60765322995f11422108829881d2166dcf9b13a3ae8ad2c002ee61f8a1 + v4.32.2: 2f855a9e616eb9c635269630666e7594931ab7524326ff02a8351d5762a28940 + v4.32.1: db4eba6ced2656e1c40e4d0f406ee189773bdda1054cbd097c1dba471e04dd4d + v4.31.2: 590f19f0a696103376e383c719fe0df28c62515627bf44e5e69403073ba83cbf + amd64: + v4.40.1: 97e931eb40791b7f0cf02363065684d807bdbc0c5973b97a37d60a7a71e8cf73 + v4.35.2: 8afd786b3b8ba8053409c5e7d154403e2d4ed4cf3e93c237462dc9ef75f38c8d + v4.35.1: bd695a6513f1196aeda17b174a15e9c351843fb1cef5f9be0af170f2dd744f08 + v4.34.2: 1952f93323e871700325a70610d2b33bafae5fe68e6eb4aec0621214f39a4c1e + v4.34.1: c5a92a572b3bd0024c7b1fe8072be3251156874c05f017c23f9db7b3254ae71a + v4.33.3: 4ee662847c588c3ef2fec8bfb304e8739e3dbaba87ccb9a608d691c88f5b64dc + v4.33.2: fbcc9551afd66622ffd68ad417837139741b2ad0eef9af1bb4b64e3596679ffa + v4.33.1: 5b9d60aa55e53fc06c9114aa5b9d5f1de9bdb231c91aed62b35d10d991831cda + v4.32.2: 0e5c6b5a74d4ccd6eed43180f60dd48a6e1d0e778f834dca33a312301674b628 + v4.32.1: e53b82caa86477bd96cf447138c72c9a0a857142a5bcdd34440b2644693ed18f + v4.31.2: 71ef4141dbd9aec3f7fb45963b92460568d044245c945a7390831a5a470623f7 + ppc64le: + v4.40.1: 7434cac727b3bc544e2e91abffb52bafa7180cc344cd84b5019ccbd2eebf9f0c + v4.35.2: 33242c57d1cab1b880b37ea7235c09966a8525319edc41ced2c70290c6a7c924 + v4.35.1: 713e2c40c5d659cbed7bf093f4c718674a75f9fe5b10ac96fd422372af198684 + v4.34.2: e149b36f93a1318414c0af971755a1488df4844356b6e9e052adf099a72e3a3a + v4.34.1: 3e629c51a07302920110893796f54f056a6ef232f791b9c67fdbe95362921a03 + v4.33.3: b5b7a59e72a5f603b819f50f2dc3e42e53127398c6f0d77da7a06d2f4d0952ea + v4.33.2: 682db3754da9f91ea447e9811e582c3f395e8d566b5afc43d1a67f6be56fa1e2 + v4.33.1: 22311b8726022963e50494d5510ffe5bb63bd0aa3f47feff445e35c4483ac68a + v4.32.2: a3650f6838a8a7a9b8914a8c27c16fb3a172ac3430836565c3927bf1a641d714 + v4.32.1: 43f1f5078a2fa7748cb5dab693538a9e634557ef2c8aad390f147beb727278cf + v4.31.2: 14e79e8eb6d36858adb3355d77ccd1d128ce74257d1358f53e1a46b2f252e28c diff --git a/kubespray/project/roles/kubespray-defaults/defaults/main/download.yml b/kubespray/project/roles/kubespray-defaults/defaults/main/download.yml new file mode 100644 index 0000000..273d4dd --- /dev/null +++ b/kubespray/project/roles/kubespray-defaults/defaults/main/download.yml @@ -0,0 +1,1191 @@ +--- +local_release_dir: /tmp/releases +download_cache_dir: /tmp/kubespray_cache + +# If this is true, debug information will be displayed but +# may contain some private data, so it is recommended to set it to false +# in the production environment. +unsafe_show_logs: false + +# do not delete remote cache files after using them +# NOTE: Setting this parameter to TRUE is only really useful when developing kubespray +download_keep_remote_cache: false + +# Only useful when download_run_once is false: Localy cached files and images are +# uploaded to kubernetes nodes. Also, images downloaded on those nodes are copied +# back to the ansible runner's cache, if they are not yet preset. +download_force_cache: false + +# Used to only evaluate vars from download role +skip_downloads: false + +# Optionally skip kubeadm images download +skip_kubeadm_images: false +kubeadm_images: {} + +# if this is set to true will only download files once. Doesn't work +# on Flatcar Container Linux by Kinvolk unless the download_localhost is true and localhost +# is running another OS type. Default compress level is 1 (fastest). +download_run_once: false +download_compress: 1 + +# if this is set to true will download container +download_container: true + +# if this is set to true, uses the localhost for download_run_once mode +# (requires docker and sudo to access docker). You may want this option for +# local caching of docker images or for Flatcar Container Linux by Kinvolk cluster nodes. +# Otherwise, uses the first node in the kube_control_plane group to store images +# in the download_run_once mode. +download_localhost: false + +# Always pull images if set to True. Otherwise check by the repo's tag/digest. +download_always_pull: false + +# Some problems may occur when downloading files over https proxy due to ansible bug +# https://github.com/ansible/ansible/issues/32750. Set this variable to False to disable +# SSL validation of get_url module. Note that kubespray will still be performing checksum validation. +download_validate_certs: true + +# Use the first kube_control_plane if download_localhost is not set +download_delegate: "{% if download_localhost %}localhost{% else %}{{ groups['kube_control_plane'][0] }}{% endif %}" + +# Allow control the times of download retries for files and containers +download_retries: 4 + +# The docker_image_info_command might seems weird but we are using raw/endraw and `{{ `{{` }}` to manage the double jinja2 processing +docker_image_pull_command: "{{ docker_bin_dir }}/docker pull" +docker_image_info_command: "{{ docker_bin_dir }}/docker images -q | xargs -i {{ '{{' }} docker_bin_dir }}/docker inspect -f {% raw %}'{{ '{{' }} if .RepoTags }}{{ '{{' }} join .RepoTags \",\" }}{{ '{{' }} end }}{{ '{{' }} if .RepoDigests }},{{ '{{' }} join .RepoDigests \",\" }}{{ '{{' }} end }}' {% endraw %} {} | tr '\n' ','" +nerdctl_image_info_command: "{{ bin_dir }}/nerdctl -n k8s.io images --format '{% raw %}{{ .Repository }}:{{ .Tag }}{% endraw %}' 2>/dev/null | grep -v ^:$ | tr '\n' ','" +# Using the ctr instead of nerdctl to workdaround the https://github.com/kubernetes-sigs/kubespray/issues/10670 +nerdctl_image_pull_command: "{{ bin_dir }}/ctr -n k8s.io images pull{% if containerd_registries_mirrors is defined %} --hosts-dir {{ containerd_cfg_dir }}/certs.d{%- endif -%}" +crictl_image_info_command: "{{ bin_dir }}/crictl images --verbose | awk -F ': ' '/RepoTags|RepoDigests/ {print $2}' | tr '\n' ','" +crictl_image_pull_command: "{{ bin_dir }}/crictl pull" + +image_command_tool: "{%- if container_manager == 'containerd' -%}nerdctl{%- elif container_manager == 'crio' -%}crictl{%- else -%}{{ container_manager }}{%- endif -%}" +image_command_tool_on_localhost: "{{ image_command_tool }}" + +image_pull_command: "{{ lookup('vars', image_command_tool + '_image_pull_command') }}" +image_info_command: "{{ lookup('vars', image_command_tool + '_image_info_command') }}" +image_pull_command_on_localhost: "{{ lookup('vars', image_command_tool_on_localhost + '_image_pull_command') }}" +image_info_command_on_localhost: "{{ lookup('vars', image_command_tool_on_localhost + '_image_info_command') }}" + +# Arch of Docker images and needed packages +image_arch: "{{ host_architecture | default('amd64') }}" + +# Versions +kubeadm_version: "{{ kube_version }}" +crun_version: 1.8.5 +runc_version: v1.1.12 +kata_containers_version: 3.1.3 +youki_version: 0.1.0 +gvisor_version: 20230807 +containerd_version: 1.7.13 +cri_dockerd_version: 0.3.9 + +# this is relevant when container_manager == 'docker' +docker_containerd_version: 1.6.28 + +# gcr and kubernetes image repo define +gcr_image_repo: "gcr.io" +kube_image_repo: "registry.k8s.io" + +# docker image repo define +docker_image_repo: "docker.io" + +# quay image repo define +quay_image_repo: "quay.io" + +# github image repo define (ex multus only use that) +github_image_repo: "ghcr.io" + +# TODO(mattymo): Move calico versions to roles/network_plugins/calico/defaults +# after migration to container download +calico_version: "v3.26.4" +calico_ctl_version: "{{ calico_version }}" +calico_cni_version: "{{ calico_version }}" +calico_flexvol_version: "{{ calico_version }}" +calico_policy_version: "{{ calico_version }}" +calico_typha_version: "{{ calico_version }}" +calico_apiserver_version: "{{ calico_version }}" +typha_enabled: false +calico_apiserver_enabled: false + +flannel_version: "v0.22.0" +flannel_cni_version: "v1.1.2" +cni_version: "v1.3.0" +weave_version: 2.8.1 + +cilium_version: "v1.13.4" +cilium_cli_version: "v0.15.0" +cilium_enable_hubble: false + +kube_ovn_version: "v1.11.5" +kube_ovn_dpdk_version: "19.11-{{ kube_ovn_version }}" +kube_router_version: "v2.0.0" +multus_version: "v3.8" +helm_version: "v3.13.1" +nerdctl_version: "1.7.1" +krew_version: "v0.4.4" +skopeo_version: "v1.13.2" + +# Get kubernetes major version (i.e. 1.17.4 => 1.17) +kube_major_version: "{{ kube_version | regex_replace('^v([0-9])+\\.([0-9]+)\\.[0-9]+', 'v\\1.\\2') }}" + +pod_infra_supported_versions: + v1.29: "3.9" + v1.28: "3.9" + v1.27: "3.9" +pod_infra_version: "{{ pod_infra_supported_versions[kube_major_version] }}" + +etcd_supported_versions: + v1.29: "v3.5.10" + v1.28: "v3.5.10" + v1.27: "v3.5.10" +etcd_version: "{{ etcd_supported_versions[kube_major_version] }}" + +crictl_supported_versions: + v1.29: "v1.29.0" + v1.28: "v1.28.0" + v1.27: "v1.27.1" +crictl_version: "{{ crictl_supported_versions[kube_major_version] }}" + +crio_supported_versions: + v1.29: v1.29.1 + v1.28: v1.28.1 + v1.27: v1.27.1 +crio_version: "{{ crio_supported_versions[kube_major_version] }}" + +# Scheduler plugins doesn't build for K8s 1.28 yet +scheduler_plugins_supported_versions: + v1.29: 0 + v1.28: 0 + v1.27: v0.27.8 +scheduler_plugins_version: "{{ scheduler_plugins_supported_versions[kube_major_version] }}" + +yq_version: "v4.35.2" + +# Download URLs +kubelet_download_url: "https://dl.k8s.io/release/{{ kube_version }}/bin/linux/{{ image_arch }}/kubelet" +kubectl_download_url: "https://dl.k8s.io/release/{{ kube_version }}/bin/linux/{{ image_arch }}/kubectl" +kubeadm_download_url: "https://dl.k8s.io/release/{{ kubeadm_version }}/bin/linux/{{ image_arch }}/kubeadm" +etcd_download_url: "https://github.com/etcd-io/etcd/releases/download/{{ etcd_version }}/etcd-{{ etcd_version }}-linux-{{ image_arch }}.tar.gz" +cni_download_url: "https://github.com/containernetworking/plugins/releases/download/{{ cni_version }}/cni-plugins-linux-{{ image_arch }}-{{ cni_version }}.tgz" +calicoctl_download_url: "https://github.com/projectcalico/calico/releases/download/{{ calico_ctl_version }}/calicoctl-linux-{{ image_arch }}" +calico_crds_download_url: "https://github.com/projectcalico/calico/archive/{{ calico_version }}.tar.gz" +ciliumcli_download_url: "https://github.com/cilium/cilium-cli/releases/download/{{ cilium_cli_version }}/cilium-linux-{{ image_arch }}.tar.gz" +crictl_download_url: "https://github.com/kubernetes-sigs/cri-tools/releases/download/{{ crictl_version }}/crictl-{{ crictl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz" +crio_download_url: "https://storage.googleapis.com/cri-o/artifacts/cri-o.{{ image_arch }}.{{ crio_version }}.tar.gz" +helm_download_url: "https://get.helm.sh/helm-{{ helm_version }}-linux-{{ image_arch }}.tar.gz" +runc_download_url: "https://github.com/opencontainers/runc/releases/download/{{ runc_version }}/runc.{{ image_arch }}" +crun_download_url: "https://github.com/containers/crun/releases/download/{{ crun_version }}/crun-{{ crun_version }}-linux-{{ image_arch }}" +youki_download_url: "https://github.com/containers/youki/releases/download/v{{ youki_version }}/youki_{{ youki_version | regex_replace('\\.', '_') }}_linux.tar.gz" +kata_containers_download_url: "https://github.com/kata-containers/kata-containers/releases/download/{{ kata_containers_version }}/kata-static-{{ kata_containers_version }}-{{ ansible_architecture }}.tar.xz" +# gVisor only supports amd64 and uses x86_64 to in the download link +gvisor_runsc_download_url: "https://storage.googleapis.com/gvisor/releases/release/{{ gvisor_version }}/{{ ansible_architecture }}/runsc" +gvisor_containerd_shim_runsc_download_url: "https://storage.googleapis.com/gvisor/releases/release/{{ gvisor_version }}/{{ ansible_architecture }}/containerd-shim-runsc-v1" +nerdctl_download_url: "https://github.com/containerd/nerdctl/releases/download/v{{ nerdctl_version }}/nerdctl-{{ nerdctl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz" +krew_download_url: "https://github.com/kubernetes-sigs/krew/releases/download/{{ krew_version }}/krew-{{ host_os }}_{{ image_arch }}.tar.gz" +containerd_download_url: "https://github.com/containerd/containerd/releases/download/v{{ containerd_version }}/containerd-{{ containerd_version }}-linux-{{ image_arch }}.tar.gz" +cri_dockerd_download_url: "https://github.com/Mirantis/cri-dockerd/releases/download/v{{ cri_dockerd_version }}/cri-dockerd-{{ cri_dockerd_version }}.{{ image_arch }}.tgz" +skopeo_download_url: "https://github.com/lework/skopeo-binary/releases/download/{{ skopeo_version }}/skopeo-linux-{{ image_arch }}" +yq_download_url: "https://github.com/mikefarah/yq/releases/download/{{ yq_version }}/yq_linux_{{ image_arch }}" + +etcd_binary_checksum: "{{ etcd_binary_checksums[image_arch][etcd_version] }}" +cni_binary_checksum: "{{ cni_binary_checksums[image_arch][cni_version] }}" +kubelet_binary_checksum: "{{ kubelet_checksums[image_arch][kube_version] }}" +kubectl_binary_checksum: "{{ kubectl_checksums[image_arch][kube_version] }}" +kubeadm_binary_checksum: "{{ kubeadm_checksums[image_arch][kubeadm_version] }}" +yq_binary_checksum: "{{ yq_checksums[image_arch][yq_version] }}" +calicoctl_binary_checksum: "{{ calicoctl_binary_checksums[image_arch][calico_ctl_version] }}" +calico_crds_archive_checksum: "{{ calico_crds_archive_checksums[calico_version] }}" +ciliumcli_binary_checksum: "{{ ciliumcli_binary_checksums[image_arch][cilium_cli_version] }}" +crictl_binary_checksum: "{{ crictl_checksums[image_arch][crictl_version] }}" +crio_archive_checksum: "{{ crio_archive_checksums[image_arch][crio_version] }}" +cri_dockerd_archive_checksum: "{{ cri_dockerd_archive_checksums[image_arch][cri_dockerd_version] }}" +helm_archive_checksum: "{{ helm_archive_checksums[image_arch][helm_version] }}" +runc_binary_checksum: "{{ runc_checksums[image_arch][runc_version] }}" +crun_binary_checksum: "{{ crun_checksums[image_arch][crun_version] }}" +youki_archive_checksum: "{{ youki_checksums[image_arch][youki_version] }}" +kata_containers_binary_checksum: "{{ kata_containers_binary_checksums[image_arch][kata_containers_version] }}" +gvisor_runsc_binary_checksum: "{{ gvisor_runsc_binary_checksums[image_arch][gvisor_version] }}" +gvisor_containerd_shim_binary_checksum: "{{ gvisor_containerd_shim_binary_checksums[image_arch][gvisor_version] }}" +nerdctl_archive_checksum: "{{ nerdctl_archive_checksums[image_arch][nerdctl_version] }}" +krew_archive_checksum: "{{ krew_archive_checksums[host_os][image_arch][krew_version] }}" +containerd_archive_checksum: "{{ containerd_archive_checksums[image_arch][containerd_version] }}" +skopeo_binary_checksum: "{{ skopeo_binary_checksums[image_arch][skopeo_version] }}" + +# Containers +# In some cases, we need a way to set --registry-mirror or --insecure-registry for docker, +# it helps a lot for local private development or bare metal environment. +# So you need define --registry-mirror or --insecure-registry, and modify the following url address. +# example: +# You need to deploy kubernetes cluster on local private development. +# Also provide the address of your own private registry. +# And use --insecure-registry options for docker +kube_proxy_image_repo: "{{ kube_image_repo }}/kube-proxy" +etcd_image_repo: "{{ quay_image_repo }}/coreos/etcd" +etcd_image_tag: "{{ etcd_version }}" +flannel_image_repo: "{{ docker_image_repo }}/flannel/flannel" +flannel_image_tag: "{{ flannel_version }}" +flannel_init_image_repo: "{{ docker_image_repo }}/flannel/flannel-cni-plugin" +flannel_init_image_tag: "{{ flannel_cni_version }}" +calico_node_image_repo: "{{ quay_image_repo }}/calico/node" +calico_node_image_tag: "{{ calico_version }}" +calico_cni_image_repo: "{{ quay_image_repo }}/calico/cni" +calico_cni_image_tag: "{{ calico_cni_version }}" +calico_flexvol_image_repo: "{{ quay_image_repo }}/calico/pod2daemon-flexvol" +calico_flexvol_image_tag: "{{ calico_flexvol_version }}" +calico_policy_image_repo: "{{ quay_image_repo }}/calico/kube-controllers" +calico_policy_image_tag: "{{ calico_policy_version }}" +calico_typha_image_repo: "{{ quay_image_repo }}/calico/typha" +calico_typha_image_tag: "{{ calico_typha_version }}" +calico_apiserver_image_repo: "{{ quay_image_repo }}/calico/apiserver" +calico_apiserver_image_tag: "{{ calico_apiserver_version }}" +pod_infra_image_repo: "{{ kube_image_repo }}/pause" +pod_infra_image_tag: "{{ pod_infra_version }}" +netcheck_version: "v1.2.2" +netcheck_agent_image_repo: "{{ docker_image_repo }}/mirantis/k8s-netchecker-agent" +netcheck_agent_image_tag: "{{ netcheck_version }}" +netcheck_server_image_repo: "{{ docker_image_repo }}/mirantis/k8s-netchecker-server" +netcheck_server_image_tag: "{{ netcheck_version }}" +netcheck_etcd_image_tag: "v3.4.17" +weave_kube_image_repo: "{{ docker_image_repo }}/weaveworks/weave-kube" +weave_kube_image_tag: "{{ weave_version }}" +weave_npc_image_repo: "{{ docker_image_repo }}/weaveworks/weave-npc" +weave_npc_image_tag: "{{ weave_version }}" +cilium_image_repo: "{{ quay_image_repo }}/cilium/cilium" +cilium_image_tag: "{{ cilium_version }}" +cilium_operator_image_repo: "{{ quay_image_repo }}/cilium/operator" +cilium_operator_image_tag: "{{ cilium_version }}" +cilium_hubble_relay_image_repo: "{{ quay_image_repo }}/cilium/hubble-relay" +cilium_hubble_relay_image_tag: "{{ cilium_version }}" +cilium_hubble_certgen_image_repo: "{{ quay_image_repo }}/cilium/certgen" +cilium_hubble_certgen_image_tag: "v0.1.8" +cilium_hubble_ui_image_repo: "{{ quay_image_repo }}/cilium/hubble-ui" +cilium_hubble_ui_image_tag: "v0.11.0" +cilium_hubble_ui_backend_image_repo: "{{ quay_image_repo }}/cilium/hubble-ui-backend" +cilium_hubble_ui_backend_image_tag: "v0.11.0" +cilium_hubble_envoy_image_repo: "{{ docker_image_repo }}/envoyproxy/envoy" +cilium_hubble_envoy_image_tag: "v1.22.5" +kube_ovn_container_image_repo: "{{ docker_image_repo }}/kubeovn/kube-ovn" +kube_ovn_container_image_tag: "{{ kube_ovn_version }}" +kube_ovn_dpdk_container_image_repo: "{{ docker_image_repo }}/kubeovn/kube-ovn-dpdk" +kube_ovn_dpdk_container_image_tag: "{{ kube_ovn_dpdk_version }}" +kube_router_image_repo: "{{ docker_image_repo }}/cloudnativelabs/kube-router" +kube_router_image_tag: "{{ kube_router_version }}" +multus_image_repo: "{{ github_image_repo }}/k8snetworkplumbingwg/multus-cni" +multus_image_tag: "{{ multus_version }}" + +kube_vip_image_repo: "{{ github_image_repo }}/kube-vip/kube-vip" +kube_vip_image_tag: v0.5.12 +nginx_image_repo: "{{ docker_image_repo }}/library/nginx" +nginx_image_tag: 1.25.2-alpine +haproxy_image_repo: "{{ docker_image_repo }}/library/haproxy" +haproxy_image_tag: 2.8.2-alpine + +# Coredns version should be supported by corefile-migration (or at least work with) +# bundle with kubeadm; if not 'basic' upgrade can sometimes fail + +coredns_version: "{{ 'v1.11.1' if (kube_version is version('v1.29.0', '>=')) else 'v1.10.1' }}" +coredns_image_is_namespaced: "{{ (coredns_version is version('v1.7.1', '>=')) }}" + +coredns_image_repo: "{{ kube_image_repo }}{{ '/coredns/coredns' if (coredns_image_is_namespaced | bool) else '/coredns' }}" +coredns_image_tag: "{{ coredns_version if (coredns_image_is_namespaced | bool) else (coredns_version | regex_replace('^v', '')) }}" + +nodelocaldns_version: "1.22.28" +nodelocaldns_image_repo: "{{ kube_image_repo }}/dns/k8s-dns-node-cache" +nodelocaldns_image_tag: "{{ nodelocaldns_version }}" + +dnsautoscaler_version: v1.8.8 +dnsautoscaler_image_repo: "{{ kube_image_repo }}/cpa/cluster-proportional-autoscaler" +dnsautoscaler_image_tag: "{{ dnsautoscaler_version }}" + +scheduler_plugins_controller_image_repo: "{{ kube_image_repo }}/scheduler-plugins/controller" +scheduler_plugins_controller_image_tag: "{{ scheduler_plugins_version }}" +scheduler_plugins_scheduler_image_repo: "{{ kube_image_repo }}/scheduler-plugins/kube-scheduler" +scheduler_plugins_scheduler_image_tag: "{{ scheduler_plugins_version }}" + +registry_version: "2.8.1" +registry_image_repo: "{{ docker_image_repo }}/library/registry" +registry_image_tag: "{{ registry_version }}" +metrics_server_version: "v0.7.0" +metrics_server_image_repo: "{{ kube_image_repo }}/metrics-server/metrics-server" +metrics_server_image_tag: "{{ metrics_server_version }}" +local_volume_provisioner_version: "v2.5.0" +local_volume_provisioner_image_repo: "{{ kube_image_repo }}/sig-storage/local-volume-provisioner" +local_volume_provisioner_image_tag: "{{ local_volume_provisioner_version }}" +cephfs_provisioner_version: "v2.1.0-k8s1.11" +cephfs_provisioner_image_repo: "{{ quay_image_repo }}/external_storage/cephfs-provisioner" +cephfs_provisioner_image_tag: "{{ cephfs_provisioner_version }}" +rbd_provisioner_version: "v2.1.1-k8s1.11" +rbd_provisioner_image_repo: "{{ quay_image_repo }}/external_storage/rbd-provisioner" +rbd_provisioner_image_tag: "{{ rbd_provisioner_version }}" +local_path_provisioner_version: "v0.0.24" +local_path_provisioner_image_repo: "{{ docker_image_repo }}/rancher/local-path-provisioner" +local_path_provisioner_image_tag: "{{ local_path_provisioner_version }}" +ingress_nginx_version: "v1.9.6" +ingress_nginx_controller_image_repo: "{{ kube_image_repo }}/ingress-nginx/controller" +ingress_nginx_controller_image_tag: "{{ ingress_nginx_version }}" +ingress_nginx_kube_webhook_certgen_image_repo: "{{ kube_image_repo }}/ingress-nginx/kube-webhook-certgen" +ingress_nginx_kube_webhook_certgen_image_tag: "v20231011-8b53cabe0" +alb_ingress_image_repo: "{{ docker_image_repo }}/amazon/aws-alb-ingress-controller" +alb_ingress_image_tag: "v1.1.9" +cert_manager_version: "v1.13.2" +cert_manager_controller_image_repo: "{{ quay_image_repo }}/jetstack/cert-manager-controller" +cert_manager_controller_image_tag: "{{ cert_manager_version }}" +cert_manager_cainjector_image_repo: "{{ quay_image_repo }}/jetstack/cert-manager-cainjector" +cert_manager_cainjector_image_tag: "{{ cert_manager_version }}" +cert_manager_webhook_image_repo: "{{ quay_image_repo }}/jetstack/cert-manager-webhook" +cert_manager_webhook_image_tag: "{{ cert_manager_version }}" + +csi_attacher_image_repo: "{{ kube_image_repo }}/sig-storage/csi-attacher" +csi_attacher_image_tag: "v3.3.0" +csi_provisioner_image_repo: "{{ kube_image_repo }}/sig-storage/csi-provisioner" +csi_provisioner_image_tag: "v3.0.0" +csi_snapshotter_image_repo: "{{ kube_image_repo }}/sig-storage/csi-snapshotter" +csi_snapshotter_image_tag: "v5.0.0" +csi_resizer_image_repo: "{{ kube_image_repo }}/sig-storage/csi-resizer" +csi_resizer_image_tag: "v1.3.0" +csi_node_driver_registrar_image_repo: "{{ kube_image_repo }}/sig-storage/csi-node-driver-registrar" +csi_node_driver_registrar_image_tag: "v2.4.0" +csi_livenessprobe_image_repo: "{{ kube_image_repo }}/sig-storage/livenessprobe" +csi_livenessprobe_image_tag: "v2.5.0" + +snapshot_controller_supported_versions: + v1.29: "v6.3.3" + v1.28: "v4.2.1" + v1.27: "v4.2.1" +snapshot_controller_image_repo: "{{ kube_image_repo }}/sig-storage/snapshot-controller" +snapshot_controller_image_tag: "{{ snapshot_controller_supported_versions[kube_major_version] }}" + +cinder_csi_plugin_version: "v1.29.0" +cinder_csi_plugin_image_repo: "{{ kube_image_repo }}/provider-os/cinder-csi-plugin" +cinder_csi_plugin_image_tag: "{{ cinder_csi_plugin_version }}" + +aws_ebs_csi_plugin_version: "v0.5.0" +aws_ebs_csi_plugin_image_repo: "{{ docker_image_repo }}/amazon/aws-ebs-csi-driver" +aws_ebs_csi_plugin_image_tag: "{{ aws_ebs_csi_plugin_version }}" + +gcp_pd_csi_plugin_version: "v1.9.2" +gcp_pd_csi_plugin_image_repo: "{{ kube_image_repo }}/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver" +gcp_pd_csi_plugin_image_tag: "{{ gcp_pd_csi_plugin_version }}" + +azure_csi_image_repo: "mcr.microsoft.com/oss/kubernetes-csi" +azure_csi_provisioner_image_tag: "v2.2.2" +azure_csi_attacher_image_tag: "v3.3.0" +azure_csi_resizer_image_tag: "v1.3.0" +azure_csi_livenessprobe_image_tag: "v2.5.0" +azure_csi_node_registrar_image_tag: "v2.4.0" +azure_csi_snapshotter_image_tag: "v3.0.3" +azure_csi_plugin_version: "v1.10.0" +azure_csi_plugin_image_repo: "mcr.microsoft.com/k8s/csi" +azure_csi_plugin_image_tag: "{{ azure_csi_plugin_version }}" + +gcp_pd_csi_image_repo: "gke.gcr.io" +gcp_pd_csi_driver_image_tag: "v0.7.0-gke.0" +gcp_pd_csi_provisioner_image_tag: "v1.5.0-gke.0" +gcp_pd_csi_attacher_image_tag: "v2.1.1-gke.0" +gcp_pd_csi_resizer_image_tag: "v0.4.0-gke.0" +gcp_pd_csi_registrar_image_tag: "v1.2.0-gke.0" + +dashboard_image_repo: "{{ docker_image_repo }}/kubernetesui/dashboard" +dashboard_image_tag: "v2.7.0" +dashboard_metrics_scraper_repo: "{{ docker_image_repo }}/kubernetesui/metrics-scraper" +dashboard_metrics_scraper_tag: "v1.0.8" + +metallb_speaker_image_repo: "{{ quay_image_repo }}/metallb/speaker" +metallb_controller_image_repo: "{{ quay_image_repo }}/metallb/controller" +metallb_version: v0.13.9 + +downloads: + netcheck_server: + enabled: "{{ deploy_netchecker }}" + container: true + repo: "{{ netcheck_server_image_repo }}" + tag: "{{ netcheck_server_image_tag }}" + sha256: "{{ netcheck_server_digest_checksum | default(None) }}" + groups: + - k8s_cluster + + netcheck_agent: + enabled: "{{ deploy_netchecker }}" + container: true + repo: "{{ netcheck_agent_image_repo }}" + tag: "{{ netcheck_agent_image_tag }}" + sha256: "{{ netcheck_agent_digest_checksum | default(None) }}" + groups: + - k8s_cluster + + etcd: + container: "{{ etcd_deployment_type != 'host' }}" + file: "{{ etcd_deployment_type == 'host' }}" + enabled: true + version: "{{ etcd_version }}" + dest: "{{ local_release_dir }}/etcd-{{ etcd_version }}-linux-{{ image_arch }}.tar.gz" + repo: "{{ etcd_image_repo }}" + tag: "{{ etcd_image_tag }}" + sha256: >- + {{ etcd_binary_checksum if (etcd_deployment_type == 'host') + else etcd_digest_checksum | d(None) }} + url: "{{ etcd_download_url }}" + unarchive: "{{ etcd_deployment_type == 'host' }}" + owner: "root" + mode: "0755" + groups: + - etcd + + cni: + enabled: true + file: true + version: "{{ cni_version }}" + dest: "{{ local_release_dir }}/cni-plugins-linux-{{ image_arch }}-{{ cni_version }}.tgz" + sha256: "{{ cni_binary_checksum }}" + url: "{{ cni_download_url }}" + unarchive: false + owner: "root" + mode: "0755" + groups: + - k8s_cluster + + kubeadm: + enabled: true + file: true + version: "{{ kubeadm_version }}" + dest: "{{ local_release_dir }}/kubeadm-{{ kubeadm_version }}-{{ image_arch }}" + sha256: "{{ kubeadm_binary_checksum }}" + url: "{{ kubeadm_download_url }}" + unarchive: false + owner: "root" + mode: "0755" + groups: + - k8s_cluster + + kubelet: + enabled: true + file: true + version: "{{ kube_version }}" + dest: "{{ local_release_dir }}/kubelet-{{ kube_version }}-{{ image_arch }}" + sha256: "{{ kubelet_binary_checksum }}" + url: "{{ kubelet_download_url }}" + unarchive: false + owner: "root" + mode: "0755" + groups: + - k8s_cluster + + kubectl: + enabled: true + file: true + version: "{{ kube_version }}" + dest: "{{ local_release_dir }}/kubectl-{{ kube_version }}-{{ image_arch }}" + sha256: "{{ kubectl_binary_checksum }}" + url: "{{ kubectl_download_url }}" + unarchive: false + owner: "root" + mode: "0755" + groups: + - kube_control_plane + + crictl: + file: true + enabled: true + version: "{{ crictl_version }}" + dest: "{{ local_release_dir }}/crictl-{{ crictl_version }}-linux-{{ image_arch }}.tar.gz" + sha256: "{{ crictl_binary_checksum }}" + url: "{{ crictl_download_url }}" + unarchive: true + owner: "root" + mode: "0755" + groups: + - k8s_cluster + + crio: + file: true + enabled: "{{ container_manager == 'crio' }}" + version: "{{ crio_version }}" + dest: "{{ local_release_dir }}/cri-o.{{ image_arch }}.{{ crio_version }}tar.gz" + sha256: "{{ crio_archive_checksum }}" + url: "{{ crio_download_url }}" + unarchive: true + owner: "root" + mode: "0755" + groups: + - k8s_cluster + + cri_dockerd: + file: true + enabled: "{{ container_manager == 'docker' }}" + version: "{{ cri_dockerd_version }}" + dest: "{{ local_release_dir }}/cri-dockerd-{{ cri_dockerd_version }}.{{ image_arch }}.tar.gz" + sha256: "{{ cri_dockerd_archive_checksum }}" + url: "{{ cri_dockerd_download_url }}" + unarchive: true + unarchive_extra_opts: + - --strip=1 + owner: "root" + mode: "0755" + groups: + - k8s_cluster + + crun: + file: true + enabled: "{{ crun_enabled }}" + version: "{{ crun_version }}" + dest: "{{ local_release_dir }}/crun-{{ crun_version }}-{{ image_arch }}" + sha256: "{{ crun_binary_checksum }}" + url: "{{ crun_download_url }}" + unarchive: false + owner: "root" + mode: "0755" + groups: + - k8s_cluster + + youki: + file: true + enabled: "{{ youki_enabled }}" + version: "{{ youki_version }}" + dest: "{{ local_release_dir }}/youki_{{ youki_version | regex_replace('\\.', '_') }}_linux.tar.gz" + sha256: "{{ youki_archive_checksum }}" + url: "{{ youki_download_url }}" + unarchive: true + owner: "root" + mode: "0755" + groups: + - k8s_cluster + + runc: + file: true + enabled: "{{ container_manager == 'containerd' }}" + version: "{{ runc_version }}" + dest: "{{ local_release_dir }}/runc-{{ runc_version }}.{{ image_arch }}" + sha256: "{{ runc_binary_checksum }}" + url: "{{ runc_download_url }}" + unarchive: false + owner: "root" + mode: "0755" + groups: + - k8s_cluster + + kata_containers: + enabled: "{{ kata_containers_enabled }}" + file: true + version: "{{ kata_containers_version }}" + dest: "{{ local_release_dir }}/kata-static-{{ kata_containers_version }}-{{ image_arch }}.tar.xz" + sha256: "{{ kata_containers_binary_checksum }}" + url: "{{ kata_containers_download_url }}" + unarchive: false + owner: "root" + mode: "0755" + groups: + - k8s_cluster + + containerd: + enabled: "{{ container_manager == 'containerd' }}" + file: true + version: "{{ containerd_version }}" + dest: "{{ local_release_dir }}/containerd-{{ containerd_version }}-linux-{{ image_arch }}.tar.gz" + sha256: "{{ containerd_archive_checksum }}" + url: "{{ containerd_download_url }}" + unarchive: false + owner: "root" + mode: "0755" + groups: + - k8s_cluster + + gvisor_runsc: + enabled: "{{ gvisor_enabled }}" + file: true + version: "{{ gvisor_version }}" + dest: "{{ local_release_dir }}/gvisor-runsc-{{ gvisor_version }}-{{ ansible_architecture }}" + sha256: "{{ gvisor_runsc_binary_checksum }}" + url: "{{ gvisor_runsc_download_url }}" + unarchive: false + owner: "root" + mode: 755 + groups: + - k8s_cluster + + gvisor_containerd_shim: + enabled: "{{ gvisor_enabled }}" + file: true + version: "{{ gvisor_version }}" + dest: "{{ local_release_dir }}/gvisor-containerd-shim-runsc-v1-{{ gvisor_version }}-{{ ansible_architecture }}" + sha256: "{{ gvisor_containerd_shim_binary_checksum }}" + url: "{{ gvisor_containerd_shim_runsc_download_url }}" + unarchive: false + owner: "root" + mode: 755 + groups: + - k8s_cluster + + nerdctl: + file: true + enabled: "{{ container_manager == 'containerd' }}" + version: "{{ nerdctl_version }}" + dest: "{{ local_release_dir }}/nerdctl-{{ nerdctl_version }}-linux-{{ image_arch }}.tar.gz" + sha256: "{{ nerdctl_archive_checksum }}" + url: "{{ nerdctl_download_url }}" + unarchive: true + owner: "root" + mode: "0755" + groups: + - k8s_cluster + + skopeo: + file: true + enabled: "{{ container_manager == 'crio' }}" + version: "{{ skopeo_version }}" + dest: "{{ local_release_dir }}/skopeo-{{ skopeo_version }}-{{ image_arch }}" + sha256: "{{ skopeo_binary_checksum }}" + url: "{{ skopeo_download_url }}" + unarchive: false + owner: "root" + mode: "0755" + groups: + - kube_control_plane + + cilium: + enabled: "{{ kube_network_plugin == 'cilium' or cilium_deploy_additionally | default(false) | bool }}" + container: true + repo: "{{ cilium_image_repo }}" + tag: "{{ cilium_image_tag }}" + sha256: "{{ cilium_digest_checksum | default(None) }}" + groups: + - k8s_cluster + + cilium_operator: + enabled: "{{ kube_network_plugin == 'cilium' or cilium_deploy_additionally | default(false) | bool }}" + container: true + repo: "{{ cilium_operator_image_repo }}" + tag: "{{ cilium_operator_image_tag }}" + sha256: "{{ cilium_operator_digest_checksum | default(None) }}" + groups: + - k8s_cluster + + cilium_hubble_relay: + enabled: "{{ cilium_enable_hubble }}" + container: true + repo: "{{ cilium_hubble_relay_image_repo }}" + tag: "{{ cilium_hubble_relay_image_tag }}" + sha256: "{{ cilium_hubble_relay_digest_checksum | default(None) }}" + groups: + - k8s_cluster + + cilium_hubble_certgen: + enabled: "{{ cilium_enable_hubble }}" + container: true + repo: "{{ cilium_hubble_certgen_image_repo }}" + tag: "{{ cilium_hubble_certgen_image_tag }}" + sha256: "{{ cilium_hubble_certgen_digest_checksum | default(None) }}" + groups: + - k8s_cluster + + cilium_hubble_ui: + enabled: "{{ cilium_enable_hubble }}" + container: true + repo: "{{ cilium_hubble_ui_image_repo }}" + tag: "{{ cilium_hubble_ui_image_tag }}" + sha256: "{{ cilium_hubble_ui_digest_checksum | default(None) }}" + groups: + - k8s_cluster + + cilium_hubble_ui_backend: + enabled: "{{ cilium_enable_hubble }}" + container: true + repo: "{{ cilium_hubble_ui_backend_image_repo }}" + tag: "{{ cilium_hubble_ui_backend_image_tag }}" + sha256: "{{ cilium_hubble_ui_backend_digest_checksum | default(None) }}" + groups: + - k8s_cluster + + cilium_hubble_envoy: + enabled: "{{ cilium_enable_hubble }}" + container: true + repo: "{{ cilium_hubble_envoy_image_repo }}" + tag: "{{ cilium_hubble_envoy_image_tag }}" + sha256: "{{ cilium_hubble_envoy_digest_checksum | default(None) }}" + groups: + - k8s_cluster + + ciliumcli: + enabled: "{{ kube_network_plugin == 'cilium' or cilium_deploy_additionally | default(false) | bool }}" + file: true + version: "{{ cilium_cli_version }}" + dest: "{{ local_release_dir }}/cilium-{{ cilium_cli_version }}-{{ image_arch }}.tar.gz" + sha256: "{{ ciliumcli_binary_checksum }}" + url: "{{ ciliumcli_download_url }}" + unarchive: true + owner: "root" + mode: "0755" + groups: + - k8s_cluster + + multus: + enabled: "{{ kube_network_plugin_multus }}" + container: true + repo: "{{ multus_image_repo }}" + tag: "{{ multus_image_tag }}" + sha256: "{{ multus_digest_checksum | default(None) }}" + groups: + - k8s_cluster + + flannel: + enabled: "{{ kube_network_plugin == 'flannel' }}" + container: true + repo: "{{ flannel_image_repo }}" + tag: "{{ flannel_image_tag }}" + sha256: "{{ flannel_digest_checksum | default(None) }}" + groups: + - k8s_cluster + + flannel_init: + enabled: "{{ kube_network_plugin == 'flannel' }}" + container: true + repo: "{{ flannel_init_image_repo }}" + tag: "{{ flannel_init_image_tag }}" + sha256: "{{ flannel_init_digest_checksum | default(None) }}" + groups: + - k8s_cluster + + calicoctl: + enabled: "{{ kube_network_plugin == 'calico' }}" + file: true + version: "{{ calico_ctl_version }}" + dest: "{{ local_release_dir }}/calicoctl-{{ calico_ctl_version }}-{{ image_arch }}" + sha256: "{{ calicoctl_binary_checksum }}" + url: "{{ calicoctl_download_url }}" + unarchive: false + owner: "root" + mode: "0755" + groups: + - k8s_cluster + + calico_node: + enabled: "{{ kube_network_plugin == 'calico' }}" + container: true + repo: "{{ calico_node_image_repo }}" + tag: "{{ calico_node_image_tag }}" + sha256: "{{ calico_node_digest_checksum | default(None) }}" + groups: + - k8s_cluster + + calico_cni: + enabled: "{{ kube_network_plugin == 'calico' }}" + container: true + repo: "{{ calico_cni_image_repo }}" + tag: "{{ calico_cni_image_tag }}" + sha256: "{{ calico_cni_digest_checksum | default(None) }}" + groups: + - k8s_cluster + + calico_flexvol: + enabled: "{{ kube_network_plugin == 'calico' }}" + container: true + repo: "{{ calico_flexvol_image_repo }}" + tag: "{{ calico_flexvol_image_tag }}" + sha256: "{{ calico_flexvol_digest_checksum | default(None) }}" + groups: + - k8s_cluster + + calico_policy: + enabled: "{{ enable_network_policy and kube_network_plugin in ['calico'] }}" + container: true + repo: "{{ calico_policy_image_repo }}" + tag: "{{ calico_policy_image_tag }}" + sha256: "{{ calico_policy_digest_checksum | default(None) }}" + groups: + - k8s_cluster + + calico_typha: + enabled: "{{ typha_enabled }}" + container: true + repo: "{{ calico_typha_image_repo }}" + tag: "{{ calico_typha_image_tag }}" + sha256: "{{ calico_typha_digest_checksum | default(None) }}" + groups: + - k8s_cluster + + calico_apiserver: + enabled: "{{ calico_apiserver_enabled }}" + container: true + repo: "{{ calico_apiserver_image_repo }}" + tag: "{{ calico_apiserver_image_tag }}" + sha256: "{{ calico_apiserver_digest_checksum | default(None) }}" + groups: + - k8s_cluster + + calico_crds: + file: true + enabled: "{{ kube_network_plugin == 'calico' and calico_datastore == 'kdd' }}" + version: "{{ calico_version }}" + dest: "{{ local_release_dir }}/calico-{{ calico_version }}-kdd-crds/{{ calico_version }}.tar.gz" + sha256: "{{ calico_crds_archive_checksum }}" + url: "{{ calico_crds_download_url }}" + unarchive: true + unarchive_extra_opts: + - "{{ '--strip=6' if (calico_version is version('v3.22.3', '<')) else '--strip=3' }}" + - "--wildcards" + - "{{ '*/_includes/charts/calico/crds/kdd/' if (calico_version is version('v3.22.3', '<')) else '*/libcalico-go/config/crd/' }}" + owner: "root" + mode: "0755" + groups: + - kube_control_plane + + weave_kube: + enabled: "{{ kube_network_plugin == 'weave' }}" + container: true + repo: "{{ weave_kube_image_repo }}" + tag: "{{ weave_kube_image_tag }}" + sha256: "{{ weave_kube_digest_checksum | default(None) }}" + groups: + - k8s_cluster + + weave_npc: + enabled: "{{ kube_network_plugin == 'weave' }}" + container: true + repo: "{{ weave_npc_image_repo }}" + tag: "{{ weave_npc_image_tag }}" + sha256: "{{ weave_npc_digest_checksum | default(None) }}" + groups: + - k8s_cluster + + kube_ovn: + enabled: "{{ kube_network_plugin == 'kube-ovn' }}" + container: true + repo: "{{ kube_ovn_container_image_repo }}" + tag: "{{ kube_ovn_container_image_tag }}" + sha256: "{{ kube_ovn_digest_checksum | default(None) }}" + groups: + - k8s_cluster + + kube_router: + enabled: "{{ kube_network_plugin == 'kube-router' }}" + container: true + repo: "{{ kube_router_image_repo }}" + tag: "{{ kube_router_image_tag }}" + sha256: "{{ kube_router_digest_checksum | default(None) }}" + groups: + - k8s_cluster + + pod_infra: + enabled: true + container: true + repo: "{{ pod_infra_image_repo }}" + tag: "{{ pod_infra_image_tag }}" + sha256: "{{ pod_infra_digest_checksum | default(None) }}" + groups: + - k8s_cluster + + kube-vip: + enabled: "{{ kube_vip_enabled }}" + container: true + repo: "{{ kube_vip_image_repo }}" + tag: "{{ kube_vip_image_tag }}" + sha256: "{{ kube_vip_digest_checksum | default(None) }}" + groups: + - kube_control_plane + + nginx: + enabled: "{{ loadbalancer_apiserver_localhost and loadbalancer_apiserver_type == 'nginx' }}" + container: true + repo: "{{ nginx_image_repo }}" + tag: "{{ nginx_image_tag }}" + sha256: "{{ nginx_digest_checksum | default(None) }}" + groups: + - kube_node + + haproxy: + enabled: "{{ loadbalancer_apiserver_localhost and loadbalancer_apiserver_type == 'haproxy' }}" + container: true + repo: "{{ haproxy_image_repo }}" + tag: "{{ haproxy_image_tag }}" + sha256: "{{ haproxy_digest_checksum | default(None) }}" + groups: + - kube_node + + coredns: + enabled: "{{ dns_mode in ['coredns', 'coredns_dual'] }}" + container: true + repo: "{{ coredns_image_repo }}" + tag: "{{ coredns_image_tag }}" + sha256: "{{ coredns_digest_checksum | default(None) }}" + groups: + - k8s_cluster + + nodelocaldns: + enabled: "{{ enable_nodelocaldns }}" + container: true + repo: "{{ nodelocaldns_image_repo }}" + tag: "{{ nodelocaldns_image_tag }}" + sha256: "{{ nodelocaldns_digest_checksum | default(None) }}" + groups: + - k8s_cluster + + dnsautoscaler: + enabled: "{{ dns_mode in ['coredns', 'coredns_dual'] }}" + container: true + repo: "{{ dnsautoscaler_image_repo }}" + tag: "{{ dnsautoscaler_image_tag }}" + sha256: "{{ dnsautoscaler_digest_checksum | default(None) }}" + groups: + - kube_control_plane + + helm: + enabled: "{{ helm_enabled }}" + file: true + version: "{{ helm_version }}" + dest: "{{ local_release_dir }}/helm-{{ helm_version }}/helm-{{ helm_version }}-linux-{{ image_arch }}.tar.gz" + sha256: "{{ helm_archive_checksum }}" + url: "{{ helm_download_url }}" + unarchive: true + owner: "root" + mode: "0755" + groups: + - kube_control_plane + + krew: + enabled: "{{ krew_enabled }}" + file: true + version: "{{ krew_version }}" + dest: "{{ local_release_dir }}/krew-{{ host_os }}_{{ image_arch }}.tar.gz" + sha256: "{{ krew_archive_checksum }}" + url: "{{ krew_download_url }}" + unarchive: true + owner: "root" + mode: "0755" + groups: + - kube_control_plane + + registry: + enabled: "{{ registry_enabled }}" + container: true + repo: "{{ registry_image_repo }}" + tag: "{{ registry_image_tag }}" + sha256: "{{ registry_digest_checksum | default(None) }}" + groups: + - kube_node + + metrics_server: + enabled: "{{ metrics_server_enabled }}" + container: true + repo: "{{ metrics_server_image_repo }}" + tag: "{{ metrics_server_image_tag }}" + sha256: "{{ metrics_server_digest_checksum | default(None) }}" + groups: + - kube_control_plane + + local_volume_provisioner: + enabled: "{{ local_volume_provisioner_enabled }}" + container: true + repo: "{{ local_volume_provisioner_image_repo }}" + tag: "{{ local_volume_provisioner_image_tag }}" + sha256: "{{ local_volume_provisioner_digest_checksum | default(None) }}" + groups: + - kube_node + + cephfs_provisioner: + enabled: "{{ cephfs_provisioner_enabled }}" + container: true + repo: "{{ cephfs_provisioner_image_repo }}" + tag: "{{ cephfs_provisioner_image_tag }}" + sha256: "{{ cephfs_provisioner_digest_checksum | default(None) }}" + groups: + - kube_node + + rbd_provisioner: + enabled: "{{ rbd_provisioner_enabled }}" + container: true + repo: "{{ rbd_provisioner_image_repo }}" + tag: "{{ rbd_provisioner_image_tag }}" + sha256: "{{ rbd_provisioner_digest_checksum | default(None) }}" + groups: + - kube_node + + local_path_provisioner: + enabled: "{{ local_path_provisioner_enabled }}" + container: true + repo: "{{ local_path_provisioner_image_repo }}" + tag: "{{ local_path_provisioner_image_tag }}" + sha256: "{{ local_path_provisioner_digest_checksum | default(None) }}" + groups: + - kube_node + + ingress_nginx_controller: + enabled: "{{ ingress_nginx_enabled }}" + container: true + repo: "{{ ingress_nginx_controller_image_repo }}" + tag: "{{ ingress_nginx_controller_image_tag }}" + sha256: "{{ ingress_nginx_controller_digest_checksum | default(None) }}" + groups: + - kube_node + + ingress_alb_controller: + enabled: "{{ ingress_alb_enabled }}" + container: true + repo: "{{ alb_ingress_image_repo }}" + tag: "{{ alb_ingress_image_tag }}" + sha256: "{{ ingress_alb_controller_digest_checksum | default(None) }}" + groups: + - kube_node + + cert_manager_controller: + enabled: "{{ cert_manager_enabled }}" + container: true + repo: "{{ cert_manager_controller_image_repo }}" + tag: "{{ cert_manager_controller_image_tag }}" + sha256: "{{ cert_manager_controller_digest_checksum | default(None) }}" + groups: + - kube_node + + cert_manager_cainjector: + enabled: "{{ cert_manager_enabled }}" + container: true + repo: "{{ cert_manager_cainjector_image_repo }}" + tag: "{{ cert_manager_cainjector_image_tag }}" + sha256: "{{ cert_manager_cainjector_digest_checksum | default(None) }}" + groups: + - kube_node + + cert_manager_webhook: + enabled: "{{ cert_manager_enabled }}" + container: true + repo: "{{ cert_manager_webhook_image_repo }}" + tag: "{{ cert_manager_webhook_image_tag }}" + sha256: "{{ cert_manager_webhook_digest_checksum | default(None) }}" + groups: + - kube_node + + csi_attacher: + enabled: "{{ cinder_csi_enabled or aws_ebs_csi_enabled }}" + container: true + repo: "{{ csi_attacher_image_repo }}" + tag: "{{ csi_attacher_image_tag }}" + sha256: "{{ csi_attacher_digest_checksum | default(None) }}" + groups: + - kube_node + + csi_provisioner: + enabled: "{{ cinder_csi_enabled or aws_ebs_csi_enabled }}" + container: true + repo: "{{ csi_provisioner_image_repo }}" + tag: "{{ csi_provisioner_image_tag }}" + sha256: "{{ csi_provisioner_digest_checksum | default(None) }}" + groups: + - kube_node + + csi_snapshotter: + enabled: "{{ cinder_csi_enabled or aws_ebs_csi_enabled }}" + container: true + repo: "{{ csi_snapshotter_image_repo }}" + tag: "{{ csi_snapshotter_image_tag }}" + sha256: "{{ csi_snapshotter_digest_checksum | default(None) }}" + groups: + - kube_node + + snapshot_controller: + enabled: "{{ csi_snapshot_controller_enabled }}" + container: true + repo: "{{ snapshot_controller_image_repo }}" + tag: "{{ snapshot_controller_image_tag }}" + sha256: "{{ snapshot_controller_digest_checksum | default(None) }}" + groups: + - kube_node + + csi_resizer: + enabled: "{{ cinder_csi_enabled or aws_ebs_csi_enabled }}" + container: true + repo: "{{ csi_resizer_image_repo }}" + tag: "{{ csi_resizer_image_tag }}" + sha256: "{{ csi_resizer_digest_checksum | default(None) }}" + groups: + - kube_node + + csi_node_driver_registrar: + enabled: "{{ cinder_csi_enabled or aws_ebs_csi_enabled }}" + container: true + repo: "{{ csi_node_driver_registrar_image_repo }}" + tag: "{{ csi_node_driver_registrar_image_tag }}" + sha256: "{{ csi_node_driver_registrar_digest_checksum | default(None) }}" + groups: + - kube_node + + cinder_csi_plugin: + enabled: "{{ cinder_csi_enabled }}" + container: true + repo: "{{ cinder_csi_plugin_image_repo }}" + tag: "{{ cinder_csi_plugin_image_tag }}" + sha256: "{{ cinder_csi_plugin_digest_checksum | default(None) }}" + groups: + - kube_node + + aws_ebs_csi_plugin: + enabled: "{{ aws_ebs_csi_enabled }}" + container: true + repo: "{{ aws_ebs_csi_plugin_image_repo }}" + tag: "{{ aws_ebs_csi_plugin_image_tag }}" + sha256: "{{ aws_ebs_csi_plugin_digest_checksum | default(None) }}" + groups: + - kube_node + + dashboard: + enabled: "{{ dashboard_enabled }}" + container: true + repo: "{{ dashboard_image_repo }}" + tag: "{{ dashboard_image_tag }}" + sha256: "{{ dashboard_digest_checksum | default(None) }}" + groups: + - kube_control_plane + + dashboard_metrics_scrapper: + enabled: "{{ dashboard_enabled }}" + container: true + repo: "{{ dashboard_metrics_scraper_repo }}" + tag: "{{ dashboard_metrics_scraper_tag }}" + sha256: "{{ dashboard_digest_checksum | default(None) }}" + groups: + - kube_control_plane + + metallb_speaker: + enabled: "{{ metallb_speaker_enabled }}" + container: true + repo: "{{ metallb_speaker_image_repo }}" + tag: "{{ metallb_version }}" + sha256: "{{ metallb_speaker_digest_checksum | default(None) }}" + groups: + - kube_control_plane + + metallb_controller: + enabled: "{{ metallb_enabled }}" + container: true + repo: "{{ metallb_controller_image_repo }}" + tag: "{{ metallb_version }}" + sha256: "{{ metallb_controller_digest_checksum | default(None) }}" + groups: + - kube_control_plane + + yq: + enabled: "{{ argocd_enabled }}" + file: true + version: "{{ yq_version }}" + dest: "{{ local_release_dir }}/yq-{{ yq_version }}-{{ image_arch }}" + sha256: "{{ yq_binary_checksum | default(None) }}" + url: "{{ yq_download_url }}" + unarchive: false + owner: "root" + mode: "0755" + groups: + - kube_control_plane + +download_defaults: + container: false + file: false + repo: None + tag: None + enabled: false + dest: None + version: None + url: None + unarchive: false + owner: "{{ kube_owner }}" + mode: None diff --git a/kubespray/project/roles/kubespray-defaults/defaults/main/main.yml b/kubespray/project/roles/kubespray-defaults/defaults/main/main.yml new file mode 100644 index 0000000..8966431 --- /dev/null +++ b/kubespray/project/roles/kubespray-defaults/defaults/main/main.yml @@ -0,0 +1,686 @@ +--- +# Use proxycommand if bastion host is in group all +# This change obseletes editing ansible.cfg file depending on bastion existence +ansible_ssh_common_args: "{% if 'bastion' in groups['all'] %} -o ProxyCommand='ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -W %h:%p -p {{ hostvars['bastion']['ansible_port'] | default(22) }} {{ hostvars['bastion']['ansible_user'] }}@{{ hostvars['bastion']['ansible_host'] }} {% if ansible_ssh_private_key_file is defined %}-i {{ ansible_ssh_private_key_file }}{% endif %} ' {% endif %}" + +# selinux state +preinstall_selinux_state: permissive + +kube_api_anonymous_auth: true + +# Default value, but will be set to true automatically if detected +is_fedora_coreos: false + +# Swap settings +kubelet_fail_swap_on: true +kubelet_swap_behavior: LimitedSwap + +## Change this to use another Kubernetes version, e.g. a current beta release +kube_version: v1.29.2 + +## The minimum version working +kube_version_min_required: v1.27.0 + +## Kube Proxy mode One of ['iptables', 'ipvs'] +kube_proxy_mode: ipvs + +## The timeout for init first control-plane +kubeadm_init_timeout: 300s + +## List of kubeadm init phases that should be skipped during control plane setup +## By default 'addon/coredns' is skipped +## 'addon/kube-proxy' gets skipped for some network plugins +kubeadm_init_phases_skip_default: [ "addon/coredns" ] +kubeadm_init_phases_skip: >- + {%- if kube_network_plugin == 'kube-router' and (kube_router_run_service_proxy is defined and kube_router_run_service_proxy) -%} + {{ kubeadm_init_phases_skip_default + ["addon/kube-proxy"] }} + {%- elif kube_network_plugin == 'cilium' and (cilium_kube_proxy_replacement is defined and cilium_kube_proxy_replacement == 'strict') -%} + {{ kubeadm_init_phases_skip_default + ["addon/kube-proxy"] }} + {%- elif kube_network_plugin == 'calico' and (calico_bpf_enabled is defined and calico_bpf_enabled) -%} + {{ kubeadm_init_phases_skip_default + ["addon/kube-proxy"] }} + {%- elif kube_proxy_remove is defined and kube_proxy_remove -%} + {{ kubeadm_init_phases_skip_default + ["addon/kube-proxy"] }} + {%- else -%} + {{ kubeadm_init_phases_skip_default }} + {%- endif -%} + +# List of kubeadm phases that should be skipped when joining a new node +# You may need to set this to ['preflight'] for air-gaped deployments to avoid failing connectivity tests. +kubeadm_join_phases_skip_default: [] +kubeadm_join_phases_skip: >- + {{ kubeadm_join_phases_skip_default }} + +# A string slice of values which specify the addresses to use for NodePorts. +# Values may be valid IP blocks (e.g. 1.2.3.0/24, 1.2.3.4/32). +# The default empty string slice ([]) means to use all local addresses. +# kube_proxy_nodeport_addresses_cidr is retained for legacy config +kube_proxy_nodeport_addresses: >- + {%- if kube_proxy_nodeport_addresses_cidr is defined -%} + [{{ kube_proxy_nodeport_addresses_cidr }}] + {%- else -%} + [] + {%- endif -%} + +# Set to true to allow pre-checks to fail and continue deployment +ignore_assert_errors: false + +kube_vip_enabled: false + +# nginx-proxy configure +nginx_config_dir: "/etc/nginx" + +# haproxy configure +haproxy_config_dir: "/etc/haproxy" + +# Directory where the binaries will be installed +bin_dir: /usr/local/bin +docker_bin_dir: /usr/bin +containerd_bin_dir: "{{ bin_dir }}" +etcd_data_dir: /var/lib/etcd +# Where the binaries will be downloaded. +# Note: ensure that you've enough disk space (about 1G) +local_release_dir: "/tmp/releases" +# Random shifts for retrying failed ops like pushing/downloading +retry_stagger: 5 + +# Install epel repo on Centos/RHEL +epel_enabled: false + +# DNS configuration. +# Kubernetes cluster name, also will be used as DNS domain +cluster_name: cluster.local +# Subdomains of DNS domain to be resolved via /etc/resolv.conf for hostnet pods +ndots: 2 +# Default resolv.conf options +docker_dns_options: +- ndots:{{ ndots }} +- timeout:2 +- attempts:2 +# Can be coredns, coredns_dual, manual, or none +dns_mode: coredns + +# Enable dns autoscaler +enable_dns_autoscaler: true + +# Enable nodelocal dns cache +enable_nodelocaldns: true +enable_nodelocaldns_secondary: false +nodelocaldns_ip: 169.254.25.10 +nodelocaldns_health_port: 9254 +nodelocaldns_second_health_port: 9256 +nodelocaldns_bind_metrics_host_ip: false +nodelocaldns_secondary_skew_seconds: 5 + +# Should be set to a cluster IP if using a custom cluster DNS +manual_dns_server: "" + +# Can be host_resolvconf, docker_dns or none +resolvconf_mode: host_resolvconf +# Deploy netchecker app to verify DNS resolve as an HTTP service +deploy_netchecker: false +# Ip address of the kubernetes DNS service (called skydns for historical reasons) +skydns_server: "{{ kube_service_addresses | ansible.utils.ipaddr('net') | ansible.utils.ipaddr(3) | ansible.utils.ipaddr('address') }}" +skydns_server_secondary: "{{ kube_service_addresses | ansible.utils.ipaddr('net') | ansible.utils.ipaddr(4) | ansible.utils.ipaddr('address') }}" +dns_domain: "{{ cluster_name }}" +docker_dns_search_domains: +- 'default.svc.{{ dns_domain }}' +- 'svc.{{ dns_domain }}' + +kube_dns_servers: + coredns: ["{{ skydns_server }}"] + coredns_dual: "{{ [skydns_server] + [skydns_server_secondary] }}" + manual: ["{{ manual_dns_server }}"] + +dns_servers: "{{ kube_dns_servers[dns_mode] }}" + +enable_coredns_k8s_external: false +coredns_k8s_external_zone: k8s_external.local + +enable_coredns_k8s_endpoint_pod_names: false + +# Kubernetes configuration dirs and system namespace. +# Those are where all the additional config stuff goes +# the kubernetes normally puts in /srv/kubernetes. +# This puts them in a sane location and namespace. +# Editing those values will almost surely break something. +kube_config_dir: /etc/kubernetes +kube_script_dir: "{{ bin_dir }}/kubernetes-scripts" +kube_manifest_dir: "{{ kube_config_dir }}/manifests" + +# Kubectl command +# This is for consistency when using kubectl command in roles, and ensure +kubectl: "{{ bin_dir }}/kubectl --kubeconfig {{ kube_config_dir }}/admin.conf" + +# This is where all the cert scripts and certs will be located +kube_cert_dir: "{{ kube_config_dir }}/ssl" + +# compatibility directory for kubeadm +kube_cert_compat_dir: "/etc/kubernetes/pki" + +# This is where all of the bearer tokens will be stored +kube_token_dir: "{{ kube_config_dir }}/tokens" + +# This is the user that owns the cluster installation. +kube_owner: kube + +# This is the group that the cert creation scripts chgrp the +# cert files to. Not really changeable... +kube_cert_group: kube-cert + +# Set to true when the CAs are managed externally. +# When true, disables all tasks manipulating certificates. Ensure before the kubespray run that: +# - Certificates and CAs are present in kube_cert_dir +# - Kubeconfig files are present in kube_config_dir +kube_external_ca_mode: false + +# Cluster Loglevel configuration +kube_log_level: 2 + +# Choose network plugin (cilium, calico, kube-ovn, weave or flannel. Use cni for generic cni plugin) +# Can also be set to 'cloud', which lets the cloud provider setup appropriate routing +kube_network_plugin: calico +kube_network_plugin_multus: false + +# Determines if calico_rr group exists +peer_with_calico_rr: "{{ 'calico_rr' in groups and groups['calico_rr'] | length > 0 }}" + +# Choose data store type for calico: "etcd" or "kdd" (kubernetes datastore) +calico_datastore: "kdd" + +# Kubernetes internal network for services, unused block of space. +kube_service_addresses: 10.233.0.0/18 + +# internal network. When used, it will assign IP +# addresses from this range to individual pods. +# This network must be unused in your network infrastructure! +kube_pods_subnet: 10.233.64.0/18 + +# internal network node size allocation (optional). This is the size allocated +# to each node for pod IP address allocation. Note that the number of pods per node is +# also limited by the kubelet_max_pods variable which defaults to 110. +# +# Example: +# Up to 64 nodes and up to 254 or kubelet_max_pods (the lowest of the two) pods per node: +# - kube_pods_subnet: 10.233.64.0/18 +# - kube_network_node_prefix: 24 +# - kubelet_max_pods: 110 +# +# Example: +# Up to 128 nodes and up to 126 or kubelet_max_pods (the lowest of the two) pods per node: +# - kube_pods_subnet: 10.233.64.0/18 +# - kube_network_node_prefix: 25 +# - kubelet_max_pods: 110 +kube_network_node_prefix: 24 + +# Configure Dual Stack networking (i.e. both IPv4 and IPv6) +enable_dual_stack_networks: false + +# Kubernetes internal network for IPv6 services, unused block of space. +# This is only used if enable_dual_stack_networks is set to true +# This provides 4096 IPv6 IPs +kube_service_addresses_ipv6: fd85:ee78:d8a6:8607::1000/116 + +# Internal network. When used, it will assign IPv6 addresses from this range to individual pods. +# This network must not already be in your network infrastructure! +# This is only used if enable_dual_stack_networks is set to true. +# This provides room for 256 nodes with 254 pods per node. +kube_pods_subnet_ipv6: fd85:ee78:d8a6:8607::1:0000/112 + +# IPv6 subnet size allocated to each for pods. +# This is only used if enable_dual_stack_networks is set to true +# This provides room for 254 pods per node. +kube_network_node_prefix_ipv6: 120 + +# The virtual cluster IP, real host IPs and ports the API Server will be +# listening on. +# NOTE: loadbalancer_apiserver_localhost somewhat alters the final API enpdoint +# access IP value (automatically evaluated below) +kube_apiserver_ip: "{{ kube_service_addresses | ansible.utils.ipaddr('net') | ansible.utils.ipaddr(1) | ansible.utils.ipaddr('address') }}" + +# NOTE: If you specific address/interface and use loadbalancer_apiserver_localhost +# loadbalancer_apiserver_localhost (nginx/haproxy) will deploy on masters on 127.0.0.1:{{ loadbalancer_apiserver_port | default(kube_apiserver_port) }} too. +kube_apiserver_bind_address: 0.0.0.0 + +# https +kube_apiserver_port: 6443 + +# If non-empty, will use this string as identification instead of the actual hostname +kube_override_hostname: >- + {%- if cloud_provider is defined and cloud_provider in ['aws'] -%} + {%- else -%} + {{ inventory_hostname }} + {%- endif -%} + +# define kubelet config dir for dynamic kubelet +# kubelet_config_dir: +default_kubelet_config_dir: "{{ kube_config_dir }}/dynamic_kubelet_dir" + +# Aggregator +kube_api_aggregator_routing: false + +# Profiling +kube_profiling: false + +# Graceful Node Shutdown +kubelet_shutdown_grace_period: 60s +# kubelet_shutdown_grace_period_critical_pods should be less than kubelet_shutdown_grace_period +# to give normal pods time to be gracefully evacuated +kubelet_shutdown_grace_period_critical_pods: 20s + +# Whether to deploy the container engine +deploy_container_engine: "{{ inventory_hostname in groups['k8s_cluster'] or etcd_deployment_type == 'docker' }}" + +# Container for runtime +container_manager: containerd + +# Enable Node Resource Interface in containerd or CRI-O. Requires crio_version >= v1.26.0 +# or containerd_version >= 1.7.0. +nri_enabled: false + +# Enable Kata Containers as additional container runtime +# When enabled, it requires `container_manager` different than Docker +kata_containers_enabled: false + +# Enable gVisor as an additional container runtime +# gVisor is only supported with container_manager Docker or containerd +gvisor_enabled: false + +# Enable crun as additional container runtime +# When enabled, it requires container_manager=crio +crun_enabled: false + +# Enable youki as additional container runtime +# When enabled, it requires container_manager=crio +youki_enabled: false + +# Container on localhost (download images when download_localhost is true) +container_manager_on_localhost: "{{ container_manager }}" + +# CRI socket path +cri_socket: >- + {%- if container_manager == 'crio' -%} + unix:///var/run/crio/crio.sock + {%- elif container_manager == 'containerd' -%} + unix:///var/run/containerd/containerd.sock + {%- elif container_manager == 'docker' -%} + unix:///var/run/cri-dockerd.sock + {%- endif -%} + +crio_insecure_registries: [] + +## Uncomment this if you want to force overlay/overlay2 as docker storage driver +## Please note that overlay2 is only supported on newer kernels +# docker_storage_options: -s overlay2 + +## Enable docker_container_storage_setup, it will configure devicemapper driver on Centos7 or RedHat7. +docker_container_storage_setup: false + +## It must be define a disk path for docker_container_storage_setup_devs. +## Otherwise docker-storage-setup will be executed incorrectly. +# docker_container_storage_setup_devs: /dev/vdb + +## Only set this if you have more than 3 nameservers: +## If true Kubespray will only use the first 3, otherwise it will fail +docker_dns_servers_strict: false + +# Path used to store Docker data +docker_daemon_graph: "/var/lib/docker" + +## Used to set docker daemon iptables options to true +docker_iptables_enabled: "false" + +# Docker log options +# Rotate container stderr/stdout logs at 50m and keep last 5 +docker_log_opts: "--log-opt max-size=50m --log-opt max-file=5" + +## A list of insecure docker registries (IP address or domain name), for example +## to allow insecure-registry access to self-hosted registries. Empty by default. +# docker_insecure_registries: +# - mirror.registry.io +# - 172.19.16.11 +docker_insecure_registries: [] + +## A list of additional registry mirrors, for example China registry mirror. Empty by default. +# docker_registry_mirrors: +# - https://registry.docker-cn.com +# - https://mirror.aliyuncs.com +docker_registry_mirrors: [] + +## If non-empty will override default system MounFlags value. +## This option takes a mount propagation flag: shared, slave +## or private, which control whether mounts in the file system +## namespace set up for docker will receive or propagate mounts +## and unmounts. Leave empty for system default +# docker_mount_flags: + +## A string of extra options to pass to the docker daemon. +# docker_options: "" + +## A list of plugins to install using 'docker plugin install --grant-all-permissions' +## Empty by default so no plugins will be installed. +docker_plugins: [] + +# Containerd options - thse are relevant when container_manager == 'containerd' +containerd_use_systemd_cgroup: true + +# Containerd conf default dir +containerd_storage_dir: "/var/lib/containerd" +containerd_state_dir: "/run/containerd" +containerd_systemd_dir: "/etc/systemd/system/containerd.service.d" +containerd_cfg_dir: "/etc/containerd" + +# Settings for containerized control plane (etcd/kubelet/secrets) +# deployment type for legacy etcd mode +etcd_deployment_type: host +cert_management: script + +# Make a copy of kubeconfig on the host that runs Ansible in {{ inventory_dir }}/artifacts +kubeconfig_localhost: false +# Download kubectl onto the host that runs Ansible in {{ bin_dir }} +kubectl_localhost: false + +# Define credentials_dir here so it can be overridden +credentials_dir: "{{ inventory_dir }}/credentials" + +# K8s image pull policy (imagePullPolicy) +k8s_image_pull_policy: IfNotPresent + +# Kubernetes dashboard +# RBAC required. see docs/getting-started.md for access details. +dashboard_enabled: false + +# Addons which can be enabled +helm_enabled: false +krew_enabled: false +registry_enabled: false +metrics_server_enabled: false +enable_network_policy: true +local_path_provisioner_enabled: false +local_volume_provisioner_enabled: false +local_volume_provisioner_directory_mode: 0700 +cinder_csi_enabled: false +aws_ebs_csi_enabled: false +azure_csi_enabled: false +gcp_pd_csi_enabled: false +vsphere_csi_enabled: false +upcloud_csi_enabled: false +csi_snapshot_controller_enabled: false +persistent_volumes_enabled: false +cephfs_provisioner_enabled: false +rbd_provisioner_enabled: false +ingress_nginx_enabled: false +ingress_alb_enabled: false +cert_manager_enabled: false +expand_persistent_volumes: false +metallb_enabled: false +metallb_speaker_enabled: "{{ metallb_enabled }}" +argocd_enabled: false + +## When OpenStack is used, Cinder version can be explicitly specified if autodetection fails (Fixed in 1.9: https://github.com/kubernetes/kubernetes/issues/50461) +# openstack_blockstorage_version: "v1/v2/auto (default)" +openstack_blockstorage_ignore_volume_az: "{{ volume_cross_zone_attachment | default('false') }}" +# set max volumes per node (cinder-csi), default not set +# node_volume_attach_limit: 25 +# Cinder CSI topology, when false volumes can be cross-mounted between availability zones +# cinder_topology: false +# Set Cinder topology zones (can be multiple zones, default not set) +# cinder_topology_zones: +# - nova +cinder_csi_ignore_volume_az: "{{ volume_cross_zone_attachment | default('false') }}" + +## When OpenStack is used, if LBaaSv2 is available you can enable it with the following 2 variables. +openstack_lbaas_enabled: false +# openstack_lbaas_subnet_id: "Neutron subnet ID (not network ID) to create LBaaS VIP" +## To enable automatic floating ip provisioning, specify a subnet. +# openstack_lbaas_floating_network_id: "Neutron network ID (not subnet ID) to get floating IP from, disabled by default" +## Override default LBaaS behavior +# openstack_lbaas_use_octavia: False +# openstack_lbaas_method: "ROUND_ROBIN" +# openstack_lbaas_provider: "haproxy" +openstack_lbaas_create_monitor: "yes" +openstack_lbaas_monitor_delay: "1m" +openstack_lbaas_monitor_timeout: "30s" +openstack_lbaas_monitor_max_retries: "3" +openstack_cacert: "{{ lookup('env', 'OS_CACERT') }}" + +# Default values for the external OpenStack Cloud Controller +external_openstack_lbaas_enabled: true +external_openstack_network_ipv6_disabled: false +external_openstack_network_internal_networks: [] +external_openstack_network_public_networks: [] + +# Default values for the external Hcloud Cloud Controller +external_hcloud_cloud: + hcloud_api_token: "" + token_secret_name: hcloud + + service_account_name: cloud-controller-manager + + controller_image_tag: "latest" + ## A dictionary of extra arguments to add to the openstack cloud controller manager daemonset + ## Format: + ## external_hcloud_cloud.controller_extra_args: + ## arg1: "value1" + ## arg2: "value2" + controller_extra_args: {} + +## List of authorization modes that must be configured for +## the k8s cluster. Only 'AlwaysAllow', 'AlwaysDeny', 'Node' and +## 'RBAC' modes are tested. Order is important. +authorization_modes: ['Node', 'RBAC'] +rbac_enabled: "{{ 'RBAC' in authorization_modes }}" + +# When enabled, API bearer tokens (including service account tokens) can be used to authenticate to the kubelet's HTTPS endpoint +kubelet_authentication_token_webhook: true + +# When enabled, access to the kubelet API requires authorization by delegation to the API server +kubelet_authorization_mode_webhook: true + +# kubelet uses certificates for authenticating to the Kubernetes API +# Automatically generate a new key and request a new certificate from the Kubernetes API as the current certificate approaches expiration +kubelet_rotate_certificates: true +# kubelet can also request a new server certificate from the Kubernetes API +kubelet_rotate_server_certificates: false + +# If set to true, kubelet errors if any of kernel tunables is different than kubelet defaults +kubelet_protect_kernel_defaults: true + +# Set additional sysctl variables to modify Linux kernel variables, for example: +# additional_sysctl: +# - { name: kernel.pid_max, value: 131072 } +# +additional_sysctl: [] + +## List of key=value pairs that describe feature gates for +## the k8s cluster. +kube_feature_gates: [] +kube_apiserver_feature_gates: [] +kube_controller_feature_gates: [] +kube_scheduler_feature_gates: [] +kube_proxy_feature_gates: [] +kubelet_feature_gates: [] +kubeadm_feature_gates: [] + +# Local volume provisioner storage classes +# Levarages Ansibles string to Python datatype casting. Otherwise the dict_key isn't substituted +# see https://github.com/ansible/ansible/issues/17324 +local_volume_provisioner_storage_classes: | + { + "{{ local_volume_provisioner_storage_class | default('local-storage') }}": { + "host_dir": "{{ local_volume_provisioner_base_dir | default('/mnt/disks') }}", + "mount_dir": "{{ local_volume_provisioner_mount_dir | default('/mnt/disks') }}", + "volume_mode": "Filesystem", + "fs_type": "ext4" + + } + } + +# weave's network password for encryption +# if null then no network encryption +# you can use --extra-vars to pass the password in command line +weave_password: EnterPasswordHere + +ssl_ca_dirs: |- + [ + {% if ansible_os_family in ['Flatcar', 'Flatcar Container Linux by Kinvolk'] -%} + '/usr/share/ca-certificates', + {% elif ansible_os_family == 'RedHat' -%} + '/etc/pki/tls', + '/etc/pki/ca-trust', + {% elif ansible_os_family == 'Debian' -%} + '/usr/share/ca-certificates', + {% endif -%} + ] + +# Vars for pointing to kubernetes api endpoints +is_kube_master: "{{ inventory_hostname in groups['kube_control_plane'] }}" +kube_apiserver_count: "{{ groups['kube_control_plane'] | length }}" +kube_apiserver_address: "{{ ip | default(fallback_ips[inventory_hostname]) }}" +kube_apiserver_access_address: "{{ access_ip | default(kube_apiserver_address) }}" +first_kube_control_plane_address: "{{ hostvars[groups['kube_control_plane'][0]]['access_ip'] | default(hostvars[groups['kube_control_plane'][0]]['ip'] | default(fallback_ips[groups['kube_control_plane'][0]])) }}" +loadbalancer_apiserver_localhost: "{{ loadbalancer_apiserver is not defined }}" +loadbalancer_apiserver_type: "nginx" +# applied if only external loadbalancer_apiserver is defined, otherwise ignored +apiserver_loadbalancer_domain_name: "lb-apiserver.kubernetes.local" +kube_apiserver_global_endpoint: |- + {% if loadbalancer_apiserver is defined -%} + https://{{ apiserver_loadbalancer_domain_name }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }} + {%- elif loadbalancer_apiserver_localhost and (loadbalancer_apiserver_port is not defined or loadbalancer_apiserver_port == kube_apiserver_port) -%} + https://localhost:{{ kube_apiserver_port }} + {%- else -%} + https://{{ first_kube_control_plane_address }}:{{ kube_apiserver_port }} + {%- endif %} +kube_apiserver_endpoint: |- + {% if loadbalancer_apiserver is defined -%} + https://{{ apiserver_loadbalancer_domain_name }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }} + {%- elif not is_kube_master and loadbalancer_apiserver_localhost -%} + https://localhost:{{ loadbalancer_apiserver_port | default(kube_apiserver_port) }} + {%- elif is_kube_master -%} + https://{{ kube_apiserver_bind_address | regex_replace('0\.0\.0\.0', '127.0.0.1') }}:{{ kube_apiserver_port }} + {%- else -%} + https://{{ first_kube_control_plane_address }}:{{ kube_apiserver_port }} + {%- endif %} +kube_apiserver_client_cert: "{{ kube_cert_dir }}/ca.crt" +kube_apiserver_client_key: "{{ kube_cert_dir }}/ca.key" + +# Set to true to deploy etcd-events cluster +etcd_events_cluster_enabled: false + +# etcd group can be empty when kubeadm manages etcd +etcd_hosts: "{{ groups['etcd'] | default(groups['kube_control_plane']) }}" + +# Vars for pointing to etcd endpoints +is_etcd_master: "{{ inventory_hostname in groups['etcd'] }}" +etcd_address: "{{ ip | default(fallback_ips[inventory_hostname]) }}" +etcd_access_address: "{{ access_ip | default(etcd_address) }}" +etcd_events_access_address: "{{ access_ip | default(etcd_address) }}" +etcd_peer_url: "https://{{ etcd_access_address }}:2380" +etcd_client_url: "https://{{ etcd_access_address }}:2379" +etcd_events_peer_url: "https://{{ etcd_events_access_address }}:2382" +etcd_events_client_url: "https://{{ etcd_events_access_address }}:2383" +etcd_access_addresses: |- + {% for item in etcd_hosts -%} + https://{{ hostvars[item]['etcd_access_address'] | default(hostvars[item]['ip'] | default(fallback_ips[item])) }}:2379{% if not loop.last %},{% endif %} + {%- endfor %} +etcd_events_access_addresses_list: |- + [ + {% for item in etcd_hosts -%} + 'https://{{ hostvars[item]['etcd_events_access_address'] | default(hostvars[item]['ip'] | default(fallback_ips[item])) }}:2383'{% if not loop.last %},{% endif %} + {%- endfor %} + ] +etcd_metrics_addresses: |- + {% for item in etcd_hosts -%} + https://{{ hostvars[item]['etcd_access_address'] | default(hostvars[item]['ip'] | default(fallback_ips[item])) }}:{{ etcd_metrics_port | default(2381) }}{% if not loop.last %},{% endif %} + {%- endfor %} +etcd_events_access_addresses: "{{ etcd_events_access_addresses_list | join(',') }}" +etcd_events_access_addresses_semicolon: "{{ etcd_events_access_addresses_list | join(';') }}" +# user should set etcd_member_name in inventory/mycluster/hosts.ini +etcd_member_name: |- + {% for host in groups['etcd'] %} + {% if inventory_hostname == host %}{{ hostvars[host].etcd_member_name | default("etcd" + loop.index | string) }}{% endif %} + {% endfor %} +etcd_peer_addresses: |- + {% for item in groups['etcd'] -%} + {{ hostvars[item].etcd_member_name | default("etcd" + loop.index | string) }}=https://{{ hostvars[item].etcd_access_address | default(hostvars[item].ip | default(fallback_ips[item])) }}:2380{% if not loop.last %},{% endif %} + {%- endfor %} +etcd_events_peer_addresses: |- + {% for item in groups['etcd'] -%} + {{ hostvars[item].etcd_member_name | default("etcd" + loop.index | string) }}-events=https://{{ hostvars[item].etcd_events_access_address | default(hostvars[item].ip | default(fallback_ips[item])) }}:2382{% if not loop.last %},{% endif %} + {%- endfor %} + +etcd_heartbeat_interval: "250" +etcd_election_timeout: "5000" +etcd_snapshot_count: "10000" + +certificates_key_size: 2048 +certificates_duration: 36500 + +etcd_config_dir: /etc/ssl/etcd +etcd_events_data_dir: "/var/lib/etcd-events" +etcd_cert_dir: "{{ etcd_config_dir }}/ssl" + +typha_enabled: false + +calico_apiserver_enabled: false + +_host_architecture_groups: + x86_64: amd64 + aarch64: arm64 + armv7l: arm +host_architecture: >- + {%- if ansible_architecture in _host_architecture_groups -%} + {{ _host_architecture_groups[ansible_architecture] }} + {%- else -%} + {{ ansible_architecture }} + {%- endif -%} + +_host_os_groups: + Linux: linux + Darwin: darwin + Win32NT: windows +host_os: >- + {%- if ansible_system in _host_os_groups -%} + {{ _host_os_groups[ansible_system] }} + {%- else -%} + {{ ansible_system }} + {%- endif -%} + +# Sets the eventRecordQPS parameter in kubelet-config.yaml. +# Setting it to 0 allows unlimited requests per second. +kubelet_event_record_qps: 50 + +proxy_env_defaults: + http_proxy: "{{ http_proxy | default('') }}" + HTTP_PROXY: "{{ http_proxy | default('') }}" + https_proxy: "{{ https_proxy | default('') }}" + HTTPS_PROXY: "{{ https_proxy | default('') }}" + no_proxy: "{{ no_proxy | default('') }}" + NO_PROXY: "{{ no_proxy | default('') }}" + +# If we use SSL_CERT_FILE: {{ omit }} it cause in value __omit_place_holder__ and break environments +# Combine dict is avoiding the problem with omit placeholder. Maybe it can be better solution? +proxy_env: "{{ proxy_env_defaults | combine({'SSL_CERT_FILE': https_proxy_cert_file}) if https_proxy_cert_file is defined else proxy_env_defaults }}" + +proxy_disable_env: + ALL_PROXY: '' + FTP_PROXY: '' + HTTPS_PROXY: '' + HTTP_PROXY: '' + NO_PROXY: '' + all_proxy: '' + ftp_proxy: '' + http_proxy: '' + https_proxy: '' + no_proxy: '' + +# krew root dir +krew_root_dir: "/usr/local/krew" + +# sysctl_file_path to add sysctl conf to +sysctl_file_path: "/etc/sysctl.d/99-sysctl.conf" + +system_upgrade: false +system_upgrade_reboot: on-upgrade # never, always + +# Enables or disables the scheduler plugins. +scheduler_plugins_enabled: false diff --git a/kubespray/project/roles/kubespray-defaults/tasks/fallback_ips.yml b/kubespray/project/roles/kubespray-defaults/tasks/fallback_ips.yml new file mode 100644 index 0000000..26b9447 --- /dev/null +++ b/kubespray/project/roles/kubespray-defaults/tasks/fallback_ips.yml @@ -0,0 +1,34 @@ +--- +# Set 127.0.0.1 as fallback IP if we do not have host facts for host +# ansible_default_ipv4 isn't what you think. +# Thanks https://medium.com/opsops/ansible-default-ipv4-is-not-what-you-think-edb8ab154b10 + +- name: Gather ansible_default_ipv4 from all hosts + setup: + gather_subset: '!all,network' + filter: "ansible_default_ipv4" + delegate_to: "{{ item }}" + delegate_facts: yes + when: hostvars[item].ansible_default_ipv4 is not defined + loop: "{{ (groups['k8s_cluster'] | default([]) + groups['etcd'] | default([]) + groups['calico_rr'] | default([])) | unique }}" + run_once: yes + ignore_unreachable: true + tags: always + +- name: Create fallback_ips_base + set_fact: + fallback_ips_base: | + --- + {% for item in (groups['k8s_cluster'] | default([]) + groups['etcd'] | default([]) + groups['calico_rr'] | default([])) | unique %} + {% set found = hostvars[item].get('ansible_default_ipv4') %} + {{ item }}: "{{ found.get('address', '127.0.0.1') }}" + {% endfor %} + delegate_to: localhost + connection: local + delegate_facts: yes + become: no + run_once: yes + +- name: Set fallback_ips + set_fact: + fallback_ips: "{{ hostvars.localhost.fallback_ips_base | from_yaml }}" diff --git a/kubespray/project/roles/kubespray-defaults/tasks/main.yaml b/kubespray/project/roles/kubespray-defaults/tasks/main.yaml new file mode 100644 index 0000000..0de87f1 --- /dev/null +++ b/kubespray/project/roles/kubespray-defaults/tasks/main.yaml @@ -0,0 +1,28 @@ +--- +- name: Set facts variables + # do not run gather facts when bootstrap-os in roles + when: > + ansible_play_role_names | + intersect(['bootstrap-os', 'kubernetes-sigs.kubespray.bootstrap-os']) | + length == 0 + tags: + - always + block: + - name: Set fallback_ips + import_tasks: fallback_ips.yml + when: fallback_ips is not defined + + - name: Set no_proxy + import_tasks: no_proxy.yml + when: + - http_proxy is defined or https_proxy is defined + - no_proxy is not defined + +# TODO: Clean this task up when we drop backward compatibility support for `etcd_kubeadm_enabled` +- name: Set `etcd_deployment_type` to "kubeadm" if `etcd_kubeadm_enabled` is true + set_fact: + etcd_deployment_type: kubeadm + when: + - etcd_kubeadm_enabled is defined and etcd_kubeadm_enabled + tags: + - always diff --git a/kubespray/project/roles/kubespray-defaults/tasks/no_proxy.yml b/kubespray/project/roles/kubespray-defaults/tasks/no_proxy.yml new file mode 100644 index 0000000..d2d5cc6 --- /dev/null +++ b/kubespray/project/roles/kubespray-defaults/tasks/no_proxy.yml @@ -0,0 +1,40 @@ +--- +- name: Set no_proxy to all assigned cluster IPs and hostnames + set_fact: + # noqa: jinja[spacing] + no_proxy_prepare: >- + {%- if loadbalancer_apiserver is defined -%} + {{ apiserver_loadbalancer_domain_name | default('') }}, + {{ loadbalancer_apiserver.address | default('') }}, + {%- endif -%} + {%- if no_proxy_exclude_workers | default(false) -%} + {% set cluster_or_master = 'kube_control_plane' %} + {%- else -%} + {% set cluster_or_master = 'k8s_cluster' %} + {%- endif -%} + {%- for item in (groups[cluster_or_master] + groups['etcd'] | default([]) + groups['calico_rr'] | default([])) | unique -%} + {{ hostvars[item]['access_ip'] | default(hostvars[item]['ip'] | default(fallback_ips[item])) }}, + {%- if item != hostvars[item].get('ansible_hostname', '') -%} + {{ hostvars[item]['ansible_hostname'] }}, + {{ hostvars[item]['ansible_hostname'] }}.{{ dns_domain }}, + {%- endif -%} + {{ item }},{{ item }}.{{ dns_domain }}, + {%- endfor -%} + {%- if additional_no_proxy is defined -%} + {{ additional_no_proxy }}, + {%- endif -%} + 127.0.0.1,localhost,{{ kube_service_addresses }},{{ kube_pods_subnet }},svc,svc.{{ dns_domain }} + delegate_to: localhost + connection: local + delegate_facts: yes + become: no + run_once: yes + +- name: Populates no_proxy to all hosts + set_fact: + no_proxy: "{{ hostvars.localhost.no_proxy_prepare }}" + # noqa: jinja[spacing] + proxy_env: "{{ proxy_env | combine({ + 'no_proxy': hostvars.localhost.no_proxy_prepare, + 'NO_PROXY': hostvars.localhost.no_proxy_prepare + }) }}" diff --git a/kubespray/project/roles/kubespray-defaults/vars/main.yml b/kubespray/project/roles/kubespray-defaults/vars/main.yml new file mode 100644 index 0000000..c79edf5 --- /dev/null +++ b/kubespray/project/roles/kubespray-defaults/vars/main.yml @@ -0,0 +1,9 @@ +--- +# Kubespray constants + +kube_proxy_deployed: "{{ 'addon/kube-proxy' not in kubeadm_init_phases_skip }}" + +# The lowest version allowed to upgrade from (same as calico_version in the previous branch) +calico_min_version_required: "v3.19.4" + +containerd_min_version_required: "1.3.7" diff --git a/kubespray/project/roles/network_plugin/calico/files/openssl.conf b/kubespray/project/roles/network_plugin/calico/files/openssl.conf new file mode 100644 index 0000000..f4ba47d --- /dev/null +++ b/kubespray/project/roles/network_plugin/calico/files/openssl.conf @@ -0,0 +1,27 @@ +req_extensions = v3_req +distinguished_name = req_distinguished_name + +[req_distinguished_name] + +[ v3_req ] +basicConstraints = CA:FALSE +keyUsage = digitalSignature, keyEncipherment + +[ ssl_client ] +extendedKeyUsage = clientAuth, serverAuth +basicConstraints = CA:FALSE +subjectKeyIdentifier=hash +authorityKeyIdentifier=keyid,issuer + +[ v3_ca ] +basicConstraints = CA:TRUE +keyUsage = cRLSign, digitalSignature, keyCertSign +subjectKeyIdentifier=hash +authorityKeyIdentifier=keyid:always,issuer + +[ ssl_client_apiserver ] +extendedKeyUsage = clientAuth, serverAuth +basicConstraints = CA:FALSE +subjectKeyIdentifier=hash +authorityKeyIdentifier=keyid,issuer +subjectAltName = DNS:calico-api.calico-apiserver.svc diff --git a/kubespray/project/roles/network_plugin/calico/handlers/main.yml b/kubespray/project/roles/network_plugin/calico/handlers/main.yml new file mode 100644 index 0000000..f5f5dc2 --- /dev/null +++ b/kubespray/project/roles/network_plugin/calico/handlers/main.yml @@ -0,0 +1,31 @@ +--- +- name: Delete 10-calico.conflist + file: + path: /etc/cni/net.d/10-calico.conflist + state: absent + listen: Reset_calico_cni + when: calico_cni_config is defined + +- name: Calico | delete calico-node docker containers + shell: "set -o pipefail && {{ docker_bin_dir }}/docker ps -af name=k8s_POD_calico-node* -q | xargs --no-run-if-empty {{ docker_bin_dir }}/docker rm -f" + args: + executable: /bin/bash + register: docker_calico_node_remove + until: docker_calico_node_remove is succeeded + retries: 5 + when: + - container_manager in ["docker"] + - calico_cni_config is defined + listen: Reset_calico_cni + +- name: Calico | delete calico-node crio/containerd containers + shell: 'set -o pipefail && {{ bin_dir }}/crictl pods --name calico-node-* -q | xargs -I% --no-run-if-empty bash -c "{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %"' + args: + executable: /bin/bash + register: crictl_calico_node_remove + until: crictl_calico_node_remove is succeeded + retries: 5 + when: + - container_manager in ["crio", "containerd"] + - calico_cni_config is defined + listen: Reset_calico_cni diff --git a/kubespray/project/roles/network_plugin/calico/meta/main.yml b/kubespray/project/roles/network_plugin/calico/meta/main.yml new file mode 100644 index 0000000..15e9b8c --- /dev/null +++ b/kubespray/project/roles/network_plugin/calico/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - role: network_plugin/calico_defaults diff --git a/kubespray/project/roles/network_plugin/calico/rr/defaults/main.yml b/kubespray/project/roles/network_plugin/calico/rr/defaults/main.yml new file mode 100644 index 0000000..dedda19 --- /dev/null +++ b/kubespray/project/roles/network_plugin/calico/rr/defaults/main.yml @@ -0,0 +1,5 @@ +--- +# Global as_num (/calico/bgp/v1/global/as_num) +# should be the same as in calico role +global_as_num: "64512" +calico_baremetal_nodename: "{{ kube_override_hostname | default(inventory_hostname) }}" diff --git a/kubespray/project/roles/network_plugin/calico/rr/tasks/main.yml b/kubespray/project/roles/network_plugin/calico/rr/tasks/main.yml new file mode 100644 index 0000000..471518d --- /dev/null +++ b/kubespray/project/roles/network_plugin/calico/rr/tasks/main.yml @@ -0,0 +1,16 @@ +--- +- name: Calico-rr | Pre-upgrade tasks + include_tasks: pre.yml + +- name: Calico-rr | Configuring node tasks + include_tasks: update-node.yml + +- name: Calico-rr | Set label for route reflector + command: >- + {{ bin_dir }}/calicoctl.sh label node {{ inventory_hostname }} + 'i-am-a-route-reflector=true' --overwrite + changed_when: false + register: calico_rr_label + until: calico_rr_label is succeeded + delay: "{{ retry_stagger | random + 3 }}" + retries: 10 diff --git a/kubespray/project/roles/network_plugin/calico/rr/tasks/pre.yml b/kubespray/project/roles/network_plugin/calico/rr/tasks/pre.yml new file mode 100644 index 0000000..d8dbd80 --- /dev/null +++ b/kubespray/project/roles/network_plugin/calico/rr/tasks/pre.yml @@ -0,0 +1,15 @@ +--- +- name: Calico-rr | Disable calico-rr service if it exists + service: + name: calico-rr + state: stopped + enabled: no + failed_when: false + +- name: Calico-rr | Delete obsolete files + file: + path: "{{ item }}" + state: absent + with_items: + - /etc/calico/calico-rr.env + - /etc/systemd/system/calico-rr.service diff --git a/kubespray/project/roles/network_plugin/calico/rr/tasks/update-node.yml b/kubespray/project/roles/network_plugin/calico/rr/tasks/update-node.yml new file mode 100644 index 0000000..fc873ba --- /dev/null +++ b/kubespray/project/roles/network_plugin/calico/rr/tasks/update-node.yml @@ -0,0 +1,50 @@ +--- +# Workaround to retry a block of tasks, ansible doesn't have a direct way to do it, +# you can follow the block loop request in: https://github.com/ansible/ansible/issues/46203 +- name: Calico-rr | Configure route reflector + block: + - name: Set the retry count + set_fact: + retry_count: "{{ 0 if retry_count is undefined else retry_count | int + 1 }}" + + - name: Calico | Set label for route reflector # noqa command-instead-of-shell + shell: "{{ bin_dir }}/calicoctl.sh label node {{ inventory_hostname }} calico-rr-id={{ calico_rr_id }} --overwrite" + changed_when: false + register: calico_rr_id_label + until: calico_rr_id_label is succeeded + delay: "{{ retry_stagger | random + 3 }}" + retries: 10 + when: calico_rr_id is defined + + - name: Calico-rr | Fetch current node object + command: "{{ bin_dir }}/calicoctl.sh get node {{ inventory_hostname }} -ojson" + changed_when: false + register: calico_rr_node + until: calico_rr_node is succeeded + delay: "{{ retry_stagger | random + 3 }}" + retries: 10 + + - name: Calico-rr | Set route reflector cluster ID + # noqa: jinja[spacing] + set_fact: + calico_rr_node_patched: >- + {{ calico_rr_node.stdout | from_json | combine({ 'spec': { 'bgp': + { 'routeReflectorClusterID': cluster_id }}}, recursive=True) }} + + - name: Calico-rr | Configure route reflector # noqa command-instead-of-shell + shell: "{{ bin_dir }}/calicoctl.sh replace -f-" + args: + stdin: "{{ calico_rr_node_patched | to_json }}" + + rescue: + - name: Fail if retry limit is reached + fail: + msg: Ended after 10 retries + when: retry_count | int == 10 + + - name: Retrying node configuration + debug: + msg: "Failed to configure route reflector - Retrying..." + + - name: Retry node configuration + include_tasks: update-node.yml diff --git a/kubespray/project/roles/network_plugin/calico/tasks/calico_apiserver_certs.yml b/kubespray/project/roles/network_plugin/calico/tasks/calico_apiserver_certs.yml new file mode 100644 index 0000000..fc336e4 --- /dev/null +++ b/kubespray/project/roles/network_plugin/calico/tasks/calico_apiserver_certs.yml @@ -0,0 +1,60 @@ +--- +- name: Calico | Check if calico apiserver exists + command: "{{ kubectl }} -n calico-apiserver get secret calico-apiserver-certs" + register: calico_apiserver_secret + changed_when: false + failed_when: false + +- name: Calico | Create ns manifests + template: + src: "calico-apiserver-ns.yml.j2" + dest: "{{ kube_config_dir }}/calico-apiserver-ns.yml" + mode: 0644 + +- name: Calico | Apply ns manifests + kube: + kubectl: "{{ bin_dir }}/kubectl" + filename: "{{ kube_config_dir }}/calico-apiserver-ns.yml" + state: "latest" + +- name: Calico | Ensure calico certs dir + file: + path: /etc/calico/certs + state: directory + mode: 0755 + when: calico_apiserver_secret.rc != 0 + +- name: Calico | Copy ssl script for apiserver certs + template: + src: make-ssl-calico.sh.j2 + dest: "{{ bin_dir }}/make-ssl-apiserver.sh" + mode: 0755 + when: calico_apiserver_secret.rc != 0 + +- name: Calico | Copy ssl config for apiserver certs + copy: + src: openssl.conf + dest: /etc/calico/certs/openssl.conf + mode: 0644 + when: calico_apiserver_secret.rc != 0 + +- name: Calico | Generate apiserver certs + command: >- + {{ bin_dir }}/make-ssl-apiserver.sh + -f /etc/calico/certs/openssl.conf + -c {{ kube_cert_dir }} + -d /etc/calico/certs + -s apiserver + when: calico_apiserver_secret.rc != 0 + +- name: Calico | Create calico apiserver generic secrets + command: >- + {{ kubectl }} -n calico-apiserver + create secret generic {{ item.name }} + --from-file={{ item.cert }} + --from-file={{ item.key }} + with_items: + - name: calico-apiserver-certs + cert: /etc/calico/certs/apiserver.crt + key: /etc/calico/certs/apiserver.key + when: calico_apiserver_secret.rc != 0 diff --git a/kubespray/project/roles/network_plugin/calico/tasks/check.yml b/kubespray/project/roles/network_plugin/calico/tasks/check.yml new file mode 100644 index 0000000..de67b15 --- /dev/null +++ b/kubespray/project/roles/network_plugin/calico/tasks/check.yml @@ -0,0 +1,207 @@ +--- +- name: Stop if legacy encapsulation variables are detected (ipip) + assert: + that: + - ipip is not defined + msg: "'ipip' configuration variable is deprecated, please configure your inventory with 'calico_ipip_mode' set to 'Always' or 'CrossSubnet' according to your specific needs" + run_once: True + delegate_to: "{{ groups['kube_control_plane'][0] }}" + +- name: Stop if legacy encapsulation variables are detected (ipip_mode) + assert: + that: + - ipip_mode is not defined + msg: "'ipip_mode' configuration variable is deprecated, please configure your inventory with 'calico_ipip_mode' set to 'Always' or 'CrossSubnet' according to your specific needs" + run_once: True + delegate_to: "{{ groups['kube_control_plane'][0] }}" + +- name: Stop if legacy encapsulation variables are detected (calcio_ipam_autoallocateblocks) + assert: + that: + - calcio_ipam_autoallocateblocks is not defined + msg: "'calcio_ipam_autoallocateblocks' configuration variable is deprecated, it's a typo, please configure your inventory with 'calico_ipam_autoallocateblocks' set to 'true' or 'false' according to your specific needs" + run_once: True + delegate_to: "{{ groups['kube_control_plane'][0] }}" + + +- name: Stop if incompatible network plugin and cloudprovider + assert: + that: + - calico_ipip_mode == 'Never' + - calico_vxlan_mode in ['Always', 'CrossSubnet'] + msg: "When using cloud_provider azure and network_plugin calico calico_ipip_mode must be 'Never' and calico_vxlan_mode 'Always' or 'CrossSubnet'" + when: + - cloud_provider is defined and cloud_provider == 'azure' + run_once: True + delegate_to: "{{ groups['kube_control_plane'][0] }}" + +- name: Stop if supported Calico versions + assert: + that: + - "calico_version in calico_crds_archive_checksums.keys()" + msg: "Calico version not supported {{ calico_version }} not in {{ calico_crds_archive_checksums.keys() }}" + run_once: True + delegate_to: "{{ groups['kube_control_plane'][0] }}" + +- name: Check if calicoctl.sh exists + stat: + path: "{{ bin_dir }}/calicoctl.sh" + register: calicoctl_sh_exists + run_once: True + delegate_to: "{{ groups['kube_control_plane'][0] }}" + +- name: Check if calico ready + command: "{{ bin_dir }}/calicoctl.sh get ClusterInformation default" + register: calico_ready + run_once: True + ignore_errors: True + retries: 5 + delay: 10 + until: calico_ready.rc == 0 + delegate_to: "{{ groups['kube_control_plane'][0] }}" + when: calicoctl_sh_exists.stat.exists + +- name: Check that current calico version is enough for upgrade + run_once: True + delegate_to: "{{ groups['kube_control_plane'][0] }}" + when: calicoctl_sh_exists.stat.exists and calico_ready.rc == 0 + block: + - name: Get current calico version + shell: "set -o pipefail && {{ bin_dir }}/calicoctl.sh version | grep 'Client Version:' | awk '{ print $3}'" + args: + executable: /bin/bash + register: calico_version_on_server + changed_when: false + + - name: Assert that current calico version is enough for upgrade + assert: + that: + - calico_version_on_server.stdout is version(calico_min_version_required, '>=') + msg: > + Your version of calico is not fresh enough for upgrade. + Minimum version is {{ calico_min_version_required }} supported by the previous kubespray release. + But current version is {{ calico_version_on_server.stdout }}. + +- name: "Check that cluster_id is set if calico_rr enabled" + assert: + that: + - cluster_id is defined + msg: "A unique cluster_id is required if using calico_rr" + when: + - peer_with_calico_rr + - inventory_hostname == groups['kube_control_plane'][0] + run_once: True + delegate_to: "{{ groups['kube_control_plane'][0] }}" + +- name: "Check that calico_rr nodes are in k8s_cluster group" + assert: + that: + - '"k8s_cluster" in group_names' + msg: "calico_rr must be a child group of k8s_cluster group" + when: + - '"calico_rr" in group_names' + run_once: True + delegate_to: "{{ groups['kube_control_plane'][0] }}" + +- name: "Check vars defined correctly" + assert: + that: + - "calico_pool_name is defined" + - "calico_pool_name is match('^[a-zA-Z0-9-_\\\\.]{2,63}$')" + msg: "calico_pool_name contains invalid characters" + run_once: True + delegate_to: "{{ groups['kube_control_plane'][0] }}" + +- name: "Check calico network backend defined correctly" + assert: + that: + - "calico_network_backend in ['bird', 'vxlan', 'none']" + msg: "calico network backend is not 'bird', 'vxlan' or 'none'" + run_once: True + delegate_to: "{{ groups['kube_control_plane'][0] }}" + +- name: "Check ipip and vxlan mode defined correctly" + run_once: True + delegate_to: "{{ groups['kube_control_plane'][0] }}" + assert: + that: + - "calico_ipip_mode in ['Always', 'CrossSubnet', 'Never']" + - "calico_vxlan_mode in ['Always', 'CrossSubnet', 'Never']" + msg: "calico inter host encapsulation mode is not 'Always', 'CrossSubnet' or 'Never'" + +- name: "Check ipip and vxlan mode if simultaneously enabled" + assert: + that: + - "calico_vxlan_mode in ['Never']" + msg: "IP in IP and VXLAN mode is mutualy exclusive modes" + when: + - "calico_ipip_mode in ['Always', 'CrossSubnet']" + run_once: True + delegate_to: "{{ groups['kube_control_plane'][0] }}" + +- name: "Check ipip and vxlan mode if simultaneously enabled" + assert: + that: + - "calico_ipip_mode in ['Never']" + msg: "IP in IP and VXLAN mode is mutualy exclusive modes" + when: + - "calico_vxlan_mode in ['Always', 'CrossSubnet']" + run_once: True + delegate_to: "{{ groups['kube_control_plane'][0] }}" + +- name: "Get Calico {{ calico_pool_name }} configuration" + command: "{{ bin_dir }}/calicoctl.sh get ipPool {{ calico_pool_name }} -o json" + failed_when: False + changed_when: False + check_mode: no + register: calico + run_once: True + delegate_to: "{{ groups['kube_control_plane'][0] }}" + +- name: "Set calico_pool_conf" + set_fact: + calico_pool_conf: '{{ calico.stdout | from_json }}' + when: calico.rc == 0 and calico.stdout + run_once: True + delegate_to: "{{ groups['kube_control_plane'][0] }}" + +- name: "Check if inventory match current cluster configuration" + assert: + that: + - calico_pool_conf.spec.blockSize | int == calico_pool_blocksize | int + - calico_pool_conf.spec.cidr == (calico_pool_cidr | default(kube_pods_subnet)) + - not calico_pool_conf.spec.ipipMode is defined or calico_pool_conf.spec.ipipMode == calico_ipip_mode + - not calico_pool_conf.spec.vxlanMode is defined or calico_pool_conf.spec.vxlanMode == calico_vxlan_mode + msg: "Your inventory doesn't match the current cluster configuration" + when: + - calico_pool_conf is defined + run_once: True + delegate_to: "{{ groups['kube_control_plane'][0] }}" + +- name: "Check kdd calico_datastore if calico_apiserver_enabled" + assert: + that: calico_datastore == "kdd" + msg: "When using calico apiserver you need to use the kubernetes datastore" + when: + - calico_apiserver_enabled + run_once: True + delegate_to: "{{ groups['kube_control_plane'][0] }}" + +- name: "Check kdd calico_datastore if typha_enabled" + assert: + that: calico_datastore == "kdd" + msg: "When using typha you need to use the kubernetes datastore" + when: + - typha_enabled + run_once: True + delegate_to: "{{ groups['kube_control_plane'][0] }}" + +- name: "Check ipip mode is Never for calico ipv6" + assert: + that: + - "calico_ipip_mode_ipv6 in ['Never']" + msg: "Calico doesn't support ipip tunneling for the IPv6" + when: + - enable_dual_stack_networks + run_once: True + delegate_to: "{{ groups['kube_control_plane'][0] }}" diff --git a/kubespray/project/roles/network_plugin/calico/tasks/install.yml b/kubespray/project/roles/network_plugin/calico/tasks/install.yml new file mode 100644 index 0000000..6b293dc --- /dev/null +++ b/kubespray/project/roles/network_plugin/calico/tasks/install.yml @@ -0,0 +1,499 @@ +--- +- name: Calico | Install Wireguard packages + package: + name: "{{ item }}" + state: present + with_items: "{{ calico_wireguard_packages }}" + register: calico_package_install + until: calico_package_install is succeeded + retries: 4 + when: calico_wireguard_enabled + +- name: Calico | Copy calicoctl binary from download dir + copy: + src: "{{ downloads.calicoctl.dest }}" + dest: "{{ bin_dir }}/calicoctl" + mode: 0755 + remote_src: yes + +- name: Calico | Create calico certs directory + file: + dest: "{{ calico_cert_dir }}" + state: directory + mode: 0750 + owner: root + group: root + when: calico_datastore == "etcd" + +- name: Calico | Link etcd certificates for calico-node + file: + src: "{{ etcd_cert_dir }}/{{ item.s }}" + dest: "{{ calico_cert_dir }}/{{ item.d }}" + state: hard + mode: 0640 + force: yes + with_items: + - {s: "{{ kube_etcd_cacert_file }}", d: "ca_cert.crt"} + - {s: "{{ kube_etcd_cert_file }}", d: "cert.crt"} + - {s: "{{ kube_etcd_key_file }}", d: "key.pem"} + when: calico_datastore == "etcd" + +- name: Calico | Generate typha certs + include_tasks: typha_certs.yml + when: + - typha_secure + - inventory_hostname == groups['kube_control_plane'][0] + +- name: Calico | Generate apiserver certs + include_tasks: calico_apiserver_certs.yml + when: + - calico_apiserver_enabled + - inventory_hostname == groups['kube_control_plane'][0] + +- name: Calico | Install calicoctl wrapper script + template: + src: "calicoctl.{{ calico_datastore }}.sh.j2" + dest: "{{ bin_dir }}/calicoctl.sh" + mode: 0755 + owner: root + group: root + +- name: Calico | wait for etcd + uri: + url: "{{ etcd_access_addresses.split(',') | first }}/health" + validate_certs: no + client_cert: "{{ calico_cert_dir }}/cert.crt" + client_key: "{{ calico_cert_dir }}/key.pem" + register: result + until: result.status == 200 or result.status == 401 + retries: 10 + delay: 5 + run_once: true + when: calico_datastore == "etcd" + +- name: Calico | Check if calico network pool has already been configured + # noqa risky-shell-pipe - grep will exit 1 if no match found + shell: > + {{ bin_dir }}/calicoctl.sh get ippool | grep -w "{{ calico_pool_cidr | default(kube_pods_subnet) }}" | wc -l + args: + executable: /bin/bash + register: calico_conf + retries: 4 + until: calico_conf.rc == 0 + delay: "{{ retry_stagger | random + 3 }}" + changed_when: false + when: + - inventory_hostname == groups['kube_control_plane'][0] + +- name: Calico | Ensure that calico_pool_cidr is within kube_pods_subnet when defined + assert: + that: "[calico_pool_cidr] | ansible.utils.ipaddr(kube_pods_subnet) | length == 1" + msg: "{{ calico_pool_cidr }} is not within or equal to {{ kube_pods_subnet }}" + when: + - inventory_hostname == groups['kube_control_plane'][0] + - 'calico_conf.stdout == "0"' + - calico_pool_cidr is defined + +- name: Calico | Check if calico IPv6 network pool has already been configured + # noqa risky-shell-pipe - grep will exit 1 if no match found + shell: > + {{ bin_dir }}/calicoctl.sh get ippool | grep -w "{{ calico_pool_cidr_ipv6 | default(kube_pods_subnet_ipv6) }}" | wc -l + args: + executable: /bin/bash + register: calico_conf_ipv6 + retries: 4 + until: calico_conf_ipv6.rc == 0 + delay: "{{ retry_stagger | random + 3 }}" + changed_when: false + when: + - inventory_hostname == groups['kube_control_plane'][0] + - enable_dual_stack_networks + +- name: Calico | Ensure that calico_pool_cidr_ipv6 is within kube_pods_subnet_ipv6 when defined + assert: + that: "[calico_pool_cidr_ipv6] | ansible.utils.ipaddr(kube_pods_subnet_ipv6) | length == 1" + msg: "{{ calico_pool_cidr_ipv6 }} is not within or equal to {{ kube_pods_subnet_ipv6 }}" + when: + - inventory_hostname == groups['kube_control_plane'][0] + - calico_conf_ipv6.stdout is defined and calico_conf_ipv6.stdout == "0" + - calico_pool_cidr_ipv6 is defined + - enable_dual_stack_networks + +- name: Calico | kdd specific configuration + when: + - inventory_hostname in groups['kube_control_plane'] + - calico_datastore == "kdd" + block: + - name: Calico | Check if extra directory is needed + stat: + path: "{{ local_release_dir }}/calico-{{ calico_version }}-kdd-crds/{{ 'kdd' if (calico_version is version('v3.22.3', '<')) else 'crd' }}" + register: kdd_path + - name: Calico | Set kdd path when calico < v3.22.3 + set_fact: + calico_kdd_path: "{{ local_release_dir }}/calico-{{ calico_version }}-kdd-crds{{ '/kdd' if kdd_path.stat.exists is defined and kdd_path.stat.exists }}" + when: + - calico_version is version('v3.22.3', '<') + - name: Calico | Set kdd path when calico > v3.22.2 + set_fact: + calico_kdd_path: "{{ local_release_dir }}/calico-{{ calico_version }}-kdd-crds{{ '/crd' if kdd_path.stat.exists is defined and kdd_path.stat.exists }}" + when: + - calico_version is version('v3.22.2', '>') + - name: Calico | Create calico manifests for kdd + assemble: + src: "{{ calico_kdd_path }}" + dest: "{{ kube_config_dir }}/kdd-crds.yml" + mode: 0644 + delimiter: "---\n" + regexp: ".*\\.yaml" + remote_src: true + + - name: Calico | Create Calico Kubernetes datastore resources + kube: + kubectl: "{{ bin_dir }}/kubectl" + filename: "{{ kube_config_dir }}/kdd-crds.yml" + state: "latest" + register: kubectl_result + until: kubectl_result is succeeded + retries: 5 + when: + - inventory_hostname == groups['kube_control_plane'][0] + +- name: Calico | Configure Felix + when: + - inventory_hostname == groups['kube_control_plane'][0] + block: + - name: Calico | Get existing FelixConfiguration + command: "{{ bin_dir }}/calicoctl.sh get felixconfig default -o json" + register: _felix_cmd + ignore_errors: True + changed_when: False + + - name: Calico | Set kubespray FelixConfiguration + set_fact: + _felix_config: > + { + "kind": "FelixConfiguration", + "apiVersion": "projectcalico.org/v3", + "metadata": { + "name": "default", + }, + "spec": { + "ipipEnabled": {{ calico_ipip_mode != 'Never' }}, + "reportingInterval": "{{ calico_felix_reporting_interval }}", + "bpfLogLevel": "{{ calico_bpf_log_level }}", + "bpfEnabled": {{ calico_bpf_enabled | bool }}, + "bpfExternalServiceMode": "{{ calico_bpf_service_mode }}", + "wireguardEnabled": {{ calico_wireguard_enabled | bool }}, + "logSeverityScreen": "{{ calico_felix_log_severity_screen }}", + "vxlanEnabled": {{ calico_vxlan_mode != 'Never' }}, + "featureDetectOverride": "{{ calico_feature_detect_override }}", + "floatingIPs": "{{ calico_felix_floating_ips }}" + } + } + + - name: Calico | Process FelixConfiguration + set_fact: + _felix_config: "{{ _felix_cmd.stdout | from_json | combine(_felix_config, recursive=True) }}" + when: + - _felix_cmd is success + + - name: Calico | Configure calico FelixConfiguration + command: + cmd: "{{ bin_dir }}/calicoctl.sh apply -f -" + stdin: "{{ _felix_config is string | ternary(_felix_config, _felix_config | to_json) }}" + changed_when: False + +- name: Calico | Configure Calico IP Pool + when: + - inventory_hostname == groups['kube_control_plane'][0] + block: + - name: Calico | Get existing calico network pool + command: "{{ bin_dir }}/calicoctl.sh get ippool {{ calico_pool_name }} -o json" + register: _calico_pool_cmd + ignore_errors: True + changed_when: False + + - name: Calico | Set kubespray calico network pool + set_fact: + _calico_pool: > + { + "kind": "IPPool", + "apiVersion": "projectcalico.org/v3", + "metadata": { + "name": "{{ calico_pool_name }}", + }, + "spec": { + "blockSize": {{ calico_pool_blocksize }}, + "cidr": "{{ calico_pool_cidr | default(kube_pods_subnet) }}", + "ipipMode": "{{ calico_ipip_mode }}", + "vxlanMode": "{{ calico_vxlan_mode }}", + "natOutgoing": {{ nat_outgoing | default(false) }} + } + } + + - name: Calico | Process calico network pool + when: + - _calico_pool_cmd is success + block: + - name: Calico | Get current calico network pool blocksize + set_fact: + _calico_blocksize: > + { + "spec": { + "blockSize": {{ (_calico_pool_cmd.stdout | from_json).spec.blockSize }} + } + } + - name: Calico | Merge calico network pool + set_fact: + _calico_pool: "{{ _calico_pool_cmd.stdout | from_json | combine(_calico_pool, _calico_blocksize, recursive=True) }}" + + - name: Calico | Configure calico network pool + command: + cmd: "{{ bin_dir }}/calicoctl.sh apply -f -" + stdin: "{{ _calico_pool is string | ternary(_calico_pool, _calico_pool | to_json) }}" + changed_when: False + +- name: Calico | Configure Calico IPv6 Pool + when: + - inventory_hostname == groups['kube_control_plane'][0] + - enable_dual_stack_networks | bool + block: + - name: Calico | Get existing calico ipv6 network pool + command: "{{ bin_dir }}/calicoctl.sh get ippool {{ calico_pool_name }}-ipv6 -o json" + register: _calico_pool_ipv6_cmd + ignore_errors: True + changed_when: False + + - name: Calico | Set kubespray calico network pool + set_fact: + _calico_pool_ipv6: > + { + "kind": "IPPool", + "apiVersion": "projectcalico.org/v3", + "metadata": { + "name": "{{ calico_pool_name }}-ipv6", + }, + "spec": { + "blockSize": {{ calico_pool_blocksize_ipv6 }}, + "cidr": "{{ calico_pool_cidr_ipv6 | default(kube_pods_subnet_ipv6) }}", + "ipipMode": "{{ calico_ipip_mode_ipv6 }}", + "vxlanMode": "{{ calico_vxlan_mode_ipv6 }}", + "natOutgoing": {{ nat_outgoing_ipv6 | default(false) }} + } + } + + - name: Calico | Process calico ipv6 network pool + when: + - _calico_pool_ipv6_cmd is success + block: + - name: Calico | Get current calico ipv6 network pool blocksize + set_fact: + _calico_blocksize_ipv6: > + { + "spec": { + "blockSize": {{ (_calico_pool_ipv6_cmd.stdout | from_json).spec.blockSize }} + } + } + - name: Calico | Merge calico ipv6 network pool + set_fact: + _calico_pool_ipv6: "{{ _calico_pool_ipv6_cmd.stdout | from_json | combine(_calico_pool_ipv6, _calico_blocksize_ipv6, recursive=True) }}" + + - name: Calico | Configure calico ipv6 network pool + command: + cmd: "{{ bin_dir }}/calicoctl.sh apply -f -" + stdin: "{{ _calico_pool_ipv6 is string | ternary(_calico_pool_ipv6, _calico_pool_ipv6 | to_json) }}" + changed_when: False + +- name: Populate Service External IPs + set_fact: + _service_external_ips: "{{ _service_external_ips | default([]) + [{'cidr': item}] }}" + with_items: "{{ calico_advertise_service_external_ips }}" + run_once: yes + +- name: Populate Service LoadBalancer IPs + set_fact: + _service_loadbalancer_ips: "{{ _service_loadbalancer_ips | default([]) + [{'cidr': item}] }}" + with_items: "{{ calico_advertise_service_loadbalancer_ips }}" + run_once: yes + +- name: "Determine nodeToNodeMesh needed state" + set_fact: + nodeToNodeMeshEnabled: "false" + when: + - peer_with_router | default(false) or peer_with_calico_rr | default(false) + - inventory_hostname in groups['k8s_cluster'] + run_once: yes + +- name: Calico | Configure Calico BGP + when: + - inventory_hostname == groups['kube_control_plane'][0] + block: + - name: Calico | Get existing BGP Configuration + command: "{{ bin_dir }}/calicoctl.sh get bgpconfig default -o json" + register: _bgp_config_cmd + ignore_errors: True + changed_when: False + + - name: Calico | Set kubespray BGP Configuration + set_fact: + # noqa: jinja[spacing] + _bgp_config: > + { + "kind": "BGPConfiguration", + "apiVersion": "projectcalico.org/v3", + "metadata": { + "name": "default", + }, + "spec": { + "listenPort": {{ calico_bgp_listen_port }}, + "logSeverityScreen": "Info", + {% if not calico_no_global_as_num | default(false) %}"asNumber": {{ global_as_num }},{% endif %} + "nodeToNodeMeshEnabled": {{ nodeToNodeMeshEnabled | default('true') }} , + {% if calico_advertise_cluster_ips | default(false) %} + "serviceClusterIPs": [{"cidr": "{{ kube_service_addresses }}" } {{ ',{"cidr":"' + kube_service_addresses_ipv6 + '"}' if enable_dual_stack_networks else '' }}],{% endif %} + {% if calico_advertise_service_loadbalancer_ips | length > 0 %}"serviceLoadBalancerIPs": {{ _service_loadbalancer_ips }},{% endif %} + "serviceExternalIPs": {{ _service_external_ips | default([]) }} + } + } + + - name: Calico | Process BGP Configuration + set_fact: + _bgp_config: "{{ _bgp_config_cmd.stdout | from_json | combine(_bgp_config, recursive=True) }}" + when: + - _bgp_config_cmd is success + + - name: Calico | Set up BGP Configuration + command: + cmd: "{{ bin_dir }}/calicoctl.sh apply -f -" + stdin: "{{ _bgp_config is string | ternary(_bgp_config, _bgp_config | to_json) }}" + changed_when: False + +- name: Calico | Create calico manifests + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/{{ item.file }}" + mode: 0644 + with_items: + - {name: calico-config, file: calico-config.yml, type: cm} + - {name: calico-node, file: calico-node.yml, type: ds} + - {name: calico, file: calico-node-sa.yml, type: sa} + - {name: calico, file: calico-cr.yml, type: clusterrole} + - {name: calico, file: calico-crb.yml, type: clusterrolebinding} + - {name: kubernetes-services-endpoint, file: kubernetes-services-endpoint.yml, type: cm } + register: calico_node_manifests + when: + - inventory_hostname in groups['kube_control_plane'] + - rbac_enabled or item.type not in rbac_resources + +- name: Calico | Create calico manifests for typha + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/{{ item.file }}" + mode: 0644 + with_items: + - {name: calico, file: calico-typha.yml, type: typha} + register: calico_node_typha_manifest + when: + - inventory_hostname in groups['kube_control_plane'] + - typha_enabled + +- name: Calico | get calico apiserver caBundle + command: "{{ bin_dir }}/kubectl get secret -n calico-apiserver calico-apiserver-certs -o jsonpath='{.data.apiserver\\.crt}'" + changed_when: false + register: calico_apiserver_cabundle + when: + - inventory_hostname == groups['kube_control_plane'][0] + - calico_apiserver_enabled + +- name: Calico | set calico apiserver caBundle fact + set_fact: + calico_apiserver_cabundle: "{{ calico_apiserver_cabundle.stdout }}" + when: + - inventory_hostname == groups['kube_control_plane'][0] + - calico_apiserver_enabled + +- name: Calico | Create calico manifests for apiserver + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/{{ item.file }}" + mode: 0644 + with_items: + - {name: calico, file: calico-apiserver.yml, type: calico-apiserver} + register: calico_apiserver_manifest + when: + - inventory_hostname in groups['kube_control_plane'] + - calico_apiserver_enabled + +- name: Start Calico resources + kube: + name: "{{ item.item.name }}" + namespace: "kube-system" + kubectl: "{{ bin_dir }}/kubectl" + resource: "{{ item.item.type }}" + filename: "{{ kube_config_dir }}/{{ item.item.file }}" + state: "latest" + with_items: + - "{{ calico_node_manifests.results }}" + - "{{ calico_node_typha_manifest.results }}" + when: + - inventory_hostname == groups['kube_control_plane'][0] + - not item is skipped + loop_control: + label: "{{ item.item.file }}" + +- name: Start Calico apiserver resources + kube: + name: "{{ item.item.name }}" + namespace: "calico-apiserver" + kubectl: "{{ bin_dir }}/kubectl" + resource: "{{ item.item.type }}" + filename: "{{ kube_config_dir }}/{{ item.item.file }}" + state: "latest" + with_items: + - "{{ calico_apiserver_manifest.results }}" + when: + - inventory_hostname == groups['kube_control_plane'][0] + - not item is skipped + loop_control: + label: "{{ item.item.file }}" + +- name: Wait for calico kubeconfig to be created + wait_for: + path: /etc/cni/net.d/calico-kubeconfig + timeout: "{{ calico_kubeconfig_wait_timeout }}" + when: + - inventory_hostname not in groups['kube_control_plane'] + - calico_datastore == "kdd" + +- name: Calico | Create Calico ipam manifests + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/{{ item.file }}" + mode: 0644 + with_items: + - {name: calico, file: calico-ipamconfig.yml, type: ipam} + when: + - inventory_hostname in groups['kube_control_plane'] + - calico_datastore == "kdd" + +- name: Calico | Create ipamconfig resources + kube: + kubectl: "{{ bin_dir }}/kubectl" + filename: "{{ kube_config_dir }}/calico-ipamconfig.yml" + state: "latest" + register: resource_result + until: resource_result is succeeded + retries: 4 + when: + - inventory_hostname == groups['kube_control_plane'][0] + - calico_datastore == "kdd" + +- name: Calico | Peer with Calico Route Reflector + include_tasks: peer_with_calico_rr.yml + when: + - peer_with_calico_rr | default(false) + +- name: Calico | Peer with the router + include_tasks: peer_with_router.yml + when: + - peer_with_router | default(false) diff --git a/kubespray/project/roles/network_plugin/calico/tasks/main.yml b/kubespray/project/roles/network_plugin/calico/tasks/main.yml new file mode 100644 index 0000000..5921a91 --- /dev/null +++ b/kubespray/project/roles/network_plugin/calico/tasks/main.yml @@ -0,0 +1,9 @@ +--- +- name: Calico Pre tasks + import_tasks: pre.yml + +- name: Calico repos + import_tasks: repos.yml + +- name: Calico install + include_tasks: install.yml diff --git a/kubespray/project/roles/network_plugin/calico/tasks/peer_with_calico_rr.yml b/kubespray/project/roles/network_plugin/calico/tasks/peer_with_calico_rr.yml new file mode 100644 index 0000000..9d216bd --- /dev/null +++ b/kubespray/project/roles/network_plugin/calico/tasks/peer_with_calico_rr.yml @@ -0,0 +1,86 @@ +--- +- name: Calico | Set label for groups nodes + command: "{{ bin_dir }}/calicoctl.sh label node {{ inventory_hostname }} calico-group-id={{ calico_group_id }} --overwrite" + changed_when: false + register: calico_group_id_label + until: calico_group_id_label is succeeded + delay: "{{ retry_stagger | random + 3 }}" + retries: 10 + when: + - calico_group_id is defined + +- name: Calico | Configure peering with route reflectors at global scope + command: + cmd: "{{ bin_dir }}/calicoctl.sh apply -f -" + # revert when it's already a string + stdin: "{{ stdin is string | ternary(stdin, stdin | to_json) }}" + vars: + stdin: > + {"apiVersion": "projectcalico.org/v3", + "kind": "BGPPeer", + "metadata": { + "name": "{{ calico_rr_id }}-to-node" + }, + "spec": { + "peerSelector": "calico-rr-id == '{{ calico_rr_id }}'", + "nodeSelector": "calico-group-id == '{{ calico_group_id }}'" + }} + register: output + retries: 4 + until: output.rc == 0 + delay: "{{ retry_stagger | random + 3 }}" + when: + - calico_rr_id is defined + - calico_group_id is defined + - inventory_hostname in groups['calico_rr'] + +- name: Calico | Configure peering with route reflectors at global scope + command: + cmd: "{{ bin_dir }}/calicoctl.sh apply -f -" + # revert when it's already a string + stdin: "{{ stdin is string | ternary(stdin, stdin | to_json) }}" + vars: + stdin: > + {"apiVersion": "projectcalico.org/v3", + "kind": "BGPPeer", + "metadata": { + "name": "peer-to-rrs" + }, + "spec": { + "nodeSelector": "!has(i-am-a-route-reflector)", + "peerSelector": "has(i-am-a-route-reflector)" + }} + register: output + retries: 4 + until: output.rc == 0 + delay: "{{ retry_stagger | random + 3 }}" + with_items: + - "{{ groups['calico_rr'] | default([]) }}" + when: + - inventory_hostname == groups['kube_control_plane'][0] + - calico_rr_id is not defined or calico_group_id is not defined + +- name: Calico | Configure route reflectors to peer with each other + command: + cmd: "{{ bin_dir }}/calicoctl.sh apply -f -" + # revert when it's already a string + stdin: "{{ stdin is string | ternary(stdin, stdin | to_json) }}" + vars: + stdin: > + {"apiVersion": "projectcalico.org/v3", + "kind": "BGPPeer", + "metadata": { + "name": "rr-mesh" + }, + "spec": { + "nodeSelector": "has(i-am-a-route-reflector)", + "peerSelector": "has(i-am-a-route-reflector)" + }} + register: output + retries: 4 + until: output.rc == 0 + delay: "{{ retry_stagger | random + 3 }}" + with_items: + - "{{ groups['calico_rr'] | default([]) }}" + when: + - inventory_hostname == groups['kube_control_plane'][0] diff --git a/kubespray/project/roles/network_plugin/calico/tasks/peer_with_router.yml b/kubespray/project/roles/network_plugin/calico/tasks/peer_with_router.yml new file mode 100644 index 0000000..a29ca36 --- /dev/null +++ b/kubespray/project/roles/network_plugin/calico/tasks/peer_with_router.yml @@ -0,0 +1,77 @@ +--- +- name: Calico | Configure peering with router(s) at global scope + command: + cmd: "{{ bin_dir }}/calicoctl.sh apply -f -" + stdin: "{{ stdin is string | ternary(stdin, stdin | to_json) }}" + vars: + stdin: > + {"apiVersion": "projectcalico.org/v3", + "kind": "BGPPeer", + "metadata": { + "name": "global-{{ item.name | default(item.router_id | replace(':', '-')) }}" + }, + "spec": { + "asNumber": "{{ item.as }}", + "peerIP": "{{ item.router_id }}" + }} + register: output + retries: 4 + until: output.rc == 0 + delay: "{{ retry_stagger | random + 3 }}" + with_items: + - "{{ peers | selectattr('scope', 'defined') | selectattr('scope', 'equalto', 'global') | list | default([]) }}" + when: + - inventory_hostname == groups['kube_control_plane'][0] + +- name: Calico | Configure node asNumber for per node peering + command: + cmd: "{{ bin_dir }}/calicoctl.sh apply -f -" + stdin: "{{ stdin is string | ternary(stdin, stdin | to_json) }}" + vars: + stdin: > + {"apiVersion": "projectcalico.org/v3", + "kind": "Node", + "metadata": { + "name": "{{ inventory_hostname }}" + }, + "spec": { + "bgp": { + "asNumber": "{{ local_as }}" + }, + "orchRefs":[{"nodeName":"{{ inventory_hostname }}","orchestrator":"k8s"}] + }} + register: output + retries: 4 + until: output.rc == 0 + delay: "{{ retry_stagger | random + 3 }}" + when: + - inventory_hostname in groups['k8s_cluster'] + - local_as is defined + - groups['calico_rr'] | default([]) | length == 0 + +- name: Calico | Configure peering with router(s) at node scope + command: + cmd: "{{ bin_dir }}/calicoctl.sh apply -f -" + stdin: "{{ stdin is string | ternary(stdin, stdin | to_json) }}" + vars: + stdin: > + {"apiVersion": "projectcalico.org/v3", + "kind": "BGPPeer", + "metadata": { + "name": "{{ inventory_hostname }}-{{ item.name | default(item.router_id | replace(':', '-')) }}" + }, + "spec": { + "asNumber": "{{ item.as }}", + "node": "{{ inventory_hostname }}", + "peerIP": "{{ item.router_id }}", + "sourceAddress": "{{ item.sourceaddress | default('UseNodeIP') }}" + }} + register: output + retries: 4 + until: output.rc == 0 + delay: "{{ retry_stagger | random + 3 }}" + with_items: + - "{{ peers | selectattr('scope', 'undefined') | list | default([]) | union(peers | selectattr('scope', 'defined') | selectattr('scope', 'equalto', 'node') | list | default([])) }}" + delegate_to: "{{ groups['kube_control_plane'][0] }}" + when: + - inventory_hostname in groups['k8s_cluster'] diff --git a/kubespray/project/roles/network_plugin/calico/tasks/pre.yml b/kubespray/project/roles/network_plugin/calico/tasks/pre.yml new file mode 100644 index 0000000..969699f --- /dev/null +++ b/kubespray/project/roles/network_plugin/calico/tasks/pre.yml @@ -0,0 +1,47 @@ +--- +- name: Slurp CNI config + slurp: + src: /etc/cni/net.d/10-calico.conflist + register: calico_cni_config_slurp + failed_when: false + +- name: Gather calico facts + tags: + - facts + when: calico_cni_config_slurp.content is defined + block: + - name: Set fact calico_cni_config from slurped CNI config + set_fact: + calico_cni_config: "{{ calico_cni_config_slurp['content'] | b64decode | from_json }}" + - name: Set fact calico_datastore to etcd if needed + set_fact: + calico_datastore: etcd + when: + - "'plugins' in calico_cni_config" + - "'etcd_endpoints' in calico_cni_config.plugins.0" + +- name: Calico | Get kubelet hostname + shell: >- + set -o pipefail && {{ kubectl }} get node -o custom-columns='NAME:.metadata.name,INTERNAL-IP:.status.addresses[?(@.type=="InternalIP")].address' + | egrep "{{ ansible_all_ipv4_addresses | join('$|') }}$" | cut -d" " -f1 + args: + executable: /bin/bash + register: calico_kubelet_name + delegate_to: "{{ groups['kube_control_plane'][0] }}" + when: + - "cloud_provider is defined" + +- name: Calico | Gather os specific variables + include_vars: "{{ item }}" + with_first_found: + - files: + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower | replace('/', '_') }}.yml" + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_release }}.yml" + - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower | replace('/', '_') }}.yml" + - "{{ ansible_distribution | lower }}.yml" + - "{{ ansible_os_family | lower }}-{{ ansible_architecture }}.yml" + - "{{ ansible_os_family | lower }}.yml" + - defaults.yml + paths: + - ../vars + skip: true diff --git a/kubespray/project/roles/network_plugin/calico/tasks/repos.yml b/kubespray/project/roles/network_plugin/calico/tasks/repos.yml new file mode 100644 index 0000000..dd29f45 --- /dev/null +++ b/kubespray/project/roles/network_plugin/calico/tasks/repos.yml @@ -0,0 +1,21 @@ +--- +- name: Calico | Add wireguard yum repo + when: + - calico_wireguard_enabled + block: + + - name: Calico | Add wireguard yum repo + yum_repository: + name: copr:copr.fedorainfracloud.org:jdoss:wireguard + file: _copr:copr.fedorainfracloud.org:jdoss:wireguard + description: Copr repo for wireguard owned by jdoss + baseurl: "{{ calico_wireguard_repo }}" + gpgcheck: yes + gpgkey: https://download.copr.fedorainfracloud.org/results/jdoss/wireguard/pubkey.gpg + skip_if_unavailable: yes + enabled: yes + repo_gpgcheck: no + when: + - ansible_os_family in ['RedHat'] + - ansible_distribution not in ['Fedora'] + - ansible_facts['distribution_major_version'] | int < 9 diff --git a/kubespray/project/roles/network_plugin/calico/tasks/reset.yml b/kubespray/project/roles/network_plugin/calico/tasks/reset.yml new file mode 100644 index 0000000..8dab214 --- /dev/null +++ b/kubespray/project/roles/network_plugin/calico/tasks/reset.yml @@ -0,0 +1,30 @@ +--- +- name: Reset | check vxlan.calico network device + stat: + path: /sys/class/net/vxlan.calico + get_attributes: no + get_checksum: no + get_mime: no + register: vxlan + +- name: Reset | remove the network vxlan.calico device created by calico + command: ip link del vxlan.calico + when: vxlan.stat.exists + +- name: Reset | check dummy0 network device + stat: + path: /sys/class/net/dummy0 + get_attributes: no + get_checksum: no + get_mime: no + register: dummy0 + +- name: Reset | remove the network device created by calico + command: ip link del dummy0 + when: dummy0.stat.exists + +- name: Reset | get and remove remaining routes set by bird + shell: set -o pipefail && ip route show proto bird | xargs -i bash -c "ip route del {} proto bird " + args: + executable: /bin/bash + changed_when: false diff --git a/kubespray/project/roles/network_plugin/calico/tasks/typha_certs.yml b/kubespray/project/roles/network_plugin/calico/tasks/typha_certs.yml new file mode 100644 index 0000000..5d3f279 --- /dev/null +++ b/kubespray/project/roles/network_plugin/calico/tasks/typha_certs.yml @@ -0,0 +1,51 @@ +--- +- name: Calico | Check if typha-server exists + command: "{{ kubectl }} -n kube-system get secret typha-server" + register: typha_server_secret + changed_when: false + failed_when: false + +- name: Calico | Ensure calico certs dir + file: + path: /etc/calico/certs + state: directory + mode: 0755 + when: typha_server_secret.rc != 0 + +- name: Calico | Copy ssl script for typha certs + template: + src: make-ssl-calico.sh.j2 + dest: "{{ bin_dir }}/make-ssl-typha.sh" + mode: 0755 + when: typha_server_secret.rc != 0 + +- name: Calico | Copy ssl config for typha certs + copy: + src: openssl.conf + dest: /etc/calico/certs/openssl.conf + mode: 0644 + when: typha_server_secret.rc != 0 + +- name: Calico | Generate typha certs + command: >- + {{ bin_dir }}/make-ssl-typha.sh + -f /etc/calico/certs/openssl.conf + -c {{ kube_cert_dir }} + -d /etc/calico/certs + -s typha + when: typha_server_secret.rc != 0 + +- name: Calico | Create typha tls secrets + command: >- + {{ kubectl }} -n kube-system + create secret tls {{ item.name }} + --cert {{ item.cert }} + --key {{ item.key }} + with_items: + - name: typha-server + cert: /etc/calico/certs/typha-server.crt + key: /etc/calico/certs/typha-server.key + - name: typha-client + cert: /etc/calico/certs/typha-client.crt + key: /etc/calico/certs/typha-client.key + when: typha_server_secret.rc != 0 diff --git a/kubespray/project/roles/network_plugin/calico/templates/calico-apiserver-ns.yml.j2 b/kubespray/project/roles/network_plugin/calico/templates/calico-apiserver-ns.yml.j2 new file mode 100644 index 0000000..a1bdfcb --- /dev/null +++ b/kubespray/project/roles/network_plugin/calico/templates/calico-apiserver-ns.yml.j2 @@ -0,0 +1,10 @@ +# This is a tech-preview manifest which installs the Calico API server. Note that this manifest is liable to change +# or be removed in future releases without further warning. +# +# Namespace and namespace-scoped resources. +apiVersion: v1 +kind: Namespace +metadata: + labels: + name: calico-apiserver + name: calico-apiserver diff --git a/kubespray/project/roles/network_plugin/calico/templates/calico-apiserver.yml.j2 b/kubespray/project/roles/network_plugin/calico/templates/calico-apiserver.yml.j2 new file mode 100644 index 0000000..49f5918 --- /dev/null +++ b/kubespray/project/roles/network_plugin/calico/templates/calico-apiserver.yml.j2 @@ -0,0 +1,280 @@ +# Policy to ensure the API server isn't cut off. Can be modified, but ensure +# that the main API server is always able to reach the Calico API server. +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: allow-apiserver + namespace: calico-apiserver +spec: + podSelector: + matchLabels: + apiserver: "true" + ingress: + - ports: + - protocol: TCP + port: 5443 + +--- + +apiVersion: v1 +kind: Service +metadata: + name: calico-api + namespace: calico-apiserver +spec: + ports: + - name: apiserver + port: 443 + protocol: TCP + targetPort: 5443 + selector: + apiserver: "true" + type: ClusterIP + +--- + +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + apiserver: "true" + k8s-app: calico-apiserver + name: calico-apiserver + namespace: calico-apiserver +spec: + replicas: 1 + selector: + matchLabels: + apiserver: "true" + strategy: + type: Recreate + template: + metadata: + labels: + apiserver: "true" + k8s-app: calico-apiserver + name: calico-apiserver + namespace: calico-apiserver + spec: + containers: + - args: + - --secure-port=5443 + env: + - name: DATASTORE_TYPE + value: kubernetes + image: {{ calico_apiserver_image_repo }}:{{ calico_apiserver_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + livenessProbe: + httpGet: + path: /version + port: 5443 + scheme: HTTPS + initialDelaySeconds: 90 + periodSeconds: 10 + name: calico-apiserver + readinessProbe: + exec: + command: + - /code/filecheck + failureThreshold: 5 + initialDelaySeconds: 5 + periodSeconds: 10 + securityContext: + privileged: false + runAsUser: 0 + volumeMounts: + - mountPath: /code/apiserver.local.config/certificates + name: calico-apiserver-certs + dnsPolicy: ClusterFirst + nodeSelector: + kubernetes.io/os: linux + restartPolicy: Always + serviceAccount: calico-apiserver + serviceAccountName: calico-apiserver + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + volumes: + - name: calico-apiserver-certs + secret: + secretName: calico-apiserver-certs + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: calico-apiserver + namespace: calico-apiserver + +--- + +# Cluster-scoped resources below here. +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + name: v3.projectcalico.org +spec: + group: projectcalico.org + groupPriorityMinimum: 1500 + caBundle: {{ calico_apiserver_cabundle }} + service: + name: calico-api + namespace: calico-apiserver + port: 443 + version: v3 + versionPriority: 200 + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: calico-crds +rules: +- apiGroups: + - extensions + - networking.k8s.io + - "" + resources: + - networkpolicies + - nodes + - namespaces + - pods + - serviceaccounts + verbs: + - get + - list + - watch +- apiGroups: + - crd.projectcalico.org + resources: + - globalnetworkpolicies + - networkpolicies + - clusterinformations + - hostendpoints + - globalnetworksets + - networksets + - bgpconfigurations + - bgppeers + - felixconfigurations + - kubecontrollersconfigurations + - ippools + - ipamconfigs + - ipreservations + - ipamblocks + - blockaffinities + - caliconodestatuses + verbs: + - get + - list + - watch + - create + - update + - delete + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: calico-extension-apiserver-auth-access +rules: +- apiGroups: + - "" + resourceNames: + - extension-apiserver-authentication + resources: + - configmaps + verbs: + - list + - watch + - get +- apiGroups: + - rbac.authorization.k8s.io + resources: + - clusterroles + - clusterrolebindings + - roles + - rolebindings + verbs: + - get + - list + - watch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: calico-webhook-reader +rules: +- apiGroups: + - admissionregistration.k8s.io + resources: + - mutatingwebhookconfigurations + - validatingwebhookconfigurations + verbs: + - get + - list + - watch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: calico-apiserver-access-crds +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-crds +subjects: +- kind: ServiceAccount + name: calico-apiserver + namespace: calico-apiserver + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: calico-apiserver-delegate-auth +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: calico-apiserver + namespace: calico-apiserver + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: calico-apiserver-webhook-reader +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-webhook-reader +subjects: +- kind: ServiceAccount + name: calico-apiserver + namespace: calico-apiserver + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: calico-extension-apiserver-auth-access +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-extension-apiserver-auth-access +subjects: +- kind: ServiceAccount + name: calico-apiserver + namespace: calico-apiserver diff --git a/kubespray/project/roles/network_plugin/calico/templates/calico-config.yml.j2 b/kubespray/project/roles/network_plugin/calico/templates/calico-config.yml.j2 new file mode 100644 index 0000000..26983ec --- /dev/null +++ b/kubespray/project/roles/network_plugin/calico/templates/calico-config.yml.j2 @@ -0,0 +1,105 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: calico-config + namespace: kube-system +data: +{% if calico_datastore == "etcd" %} + etcd_endpoints: "{{ etcd_access_addresses }}" + etcd_ca: "/calico-secrets/ca_cert.crt" + etcd_cert: "/calico-secrets/cert.crt" + etcd_key: "/calico-secrets/key.pem" +{% elif calico_datastore == "kdd" and typha_enabled %} + # To enable Typha, set this to "calico-typha" *and* set a non-zero value for Typha replicas + # below. We recommend using Typha if you have more than 50 nodes. Above 100 nodes it is + # essential. + typha_service_name: "calico-typha" +{% endif %} +{% if calico_network_backend == 'bird' %} + cluster_type: "kubespray,bgp" + calico_backend: "bird" +{% else %} + cluster_type: "kubespray" + calico_backend: "{{ calico_network_backend }}" +{% endif %} +{% if inventory_hostname in groups['k8s_cluster'] and peer_with_router | default(false) %} + as: "{{ local_as | default(global_as_num) }}" +{% endif -%} + # The CNI network configuration to install on each node. The special + # values in this config will be automatically populated. + cni_network_config: |- + { + "name": "{{ calico_cni_name }}", + "cniVersion":"0.3.1", + "plugins":[ + { + {% if calico_datastore == "kdd" %} + "datastore_type": "kubernetes", + "nodename": "__KUBERNETES_NODE_NAME__", + {% endif %} + "type": "calico", + "log_level": "info", + {% if calico_cni_log_file_path %} + "log_file_path": "{{ calico_cni_log_file_path }}", + {% endif %} + {% if calico_datastore == "etcd" %} + "etcd_endpoints": "{{ etcd_access_addresses }}", + "etcd_cert_file": "{{ calico_cert_dir }}/cert.crt", + "etcd_key_file": "{{ calico_cert_dir }}/key.pem", + "etcd_ca_cert_file": "{{ calico_cert_dir }}/ca_cert.crt", + {% endif %} + {% if calico_ipam_host_local %} + "ipam": { + "type": "host-local", + "subnet": "usePodCidr" + }, + {% else %} + "ipam": { + "type": "calico-ipam", + {% if enable_dual_stack_networks %} + "assign_ipv6": "true", + {% endif %} + "assign_ipv4": "true" + }, + {% endif %} + {% if calico_allow_ip_forwarding %} + "container_settings": { + "allow_ip_forwarding": true + }, + {% endif %} + {% if (calico_feature_control is defined) and (calico_feature_control | length > 0) %} + "feature_control": { + {% for fc in calico_feature_control -%} + {% set fcval = calico_feature_control[fc] -%} + "{{ fc }}": {{ (fcval | string | lower) if (fcval == true or fcval == false) else "\"" + fcval + "\"" }}{{ "," if not loop.last else "" }} + {% endfor -%} + {{- "" }} + }, + {% endif %} + {% if enable_network_policy %} + "policy": { + "type": "k8s" + }, + {% endif %} + {% if calico_mtu is defined and calico_mtu is number %} + "mtu": {{ calico_mtu }}, + {% endif %} + "kubernetes": { + "kubeconfig": "__KUBECONFIG_FILEPATH__" + } + }, + { + "type":"portmap", + "capabilities": { + "portMappings": true + } + }, + { + "type":"bandwidth", + "capabilities": { + "bandwidth": true + } + } + ] + } + diff --git a/kubespray/project/roles/network_plugin/calico/templates/calico-cr.yml.j2 b/kubespray/project/roles/network_plugin/calico/templates/calico-cr.yml.j2 new file mode 100644 index 0000000..ac0331f --- /dev/null +++ b/kubespray/project/roles/network_plugin/calico/templates/calico-cr.yml.j2 @@ -0,0 +1,194 @@ +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: calico-cni-plugin +rules: + - apiGroups: [""] + resources: + - pods + - nodes + - namespaces + verbs: + - get + - apiGroups: [""] + resources: + - pods/status + verbs: + - patch + - apiGroups: ["crd.projectcalico.org"] + resources: + - blockaffinities + - ipamblocks + - ipamhandles + - clusterinformations + - ippools + - ipreservations + - ipamconfigs + verbs: + - get + - list + - create + - update + - delete +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: calico-node + namespace: kube-system +rules: + - apiGroups: [""] + resources: + - pods + - nodes + - namespaces + - configmaps + verbs: + - get + # EndpointSlices are used for Service-based network policy rule + # enforcement. + - apiGroups: ["discovery.k8s.io"] + resources: + - endpointslices + verbs: + - watch + - list + - apiGroups: [""] + resources: + - endpoints + - services + verbs: + - watch + - list +{% if calico_datastore == "kdd" %} + # Used to discover Typhas. + - get +{% endif %} + - apiGroups: [""] + resources: + - nodes/status + verbs: + # Needed for clearing NodeNetworkUnavailable flag. + - patch +{% if calico_datastore == "kdd" %} + # Calico stores some configuration information in node annotations. + - update + # Watch for changes to Kubernetes NetworkPolicies. + - apiGroups: ["networking.k8s.io"] + resources: + - networkpolicies + verbs: + - watch + - list + # Used by Calico for policy information. + - apiGroups: [""] + resources: + - pods + - namespaces + - serviceaccounts + verbs: + - list + - watch + # The CNI plugin patches pods/status. + - apiGroups: [""] + resources: + - pods/status + verbs: + - patch + # Calico monitors various CRDs for config. + - apiGroups: ["crd.projectcalico.org"] + resources: + - globalfelixconfigs + - felixconfigurations + - bgppeers + - bgpfilters + - globalbgpconfigs + - bgpconfigurations + - ippools + - ipreservations + - ipamblocks + - globalnetworkpolicies + - globalnetworksets + - networkpolicies + - networksets + - clusterinformations + - hostendpoints + - blockaffinities + - caliconodestatuses + verbs: + - get + - list + - watch + # Calico must create and update some CRDs on startup. + - apiGroups: ["crd.projectcalico.org"] + resources: + - ippools + - felixconfigurations + - clusterinformations + verbs: + - create + - update + # Calico must update some CRDs. + - apiGroups: [ "crd.projectcalico.org" ] + resources: + - caliconodestatuses + verbs: + - update + # Calico stores some configuration information on the node. + - apiGroups: [""] + resources: + - nodes + verbs: + - get + - list + - watch + # These permissions are only required for upgrade from v2.6, and can + # be removed after upgrade or on fresh installations. + - apiGroups: ["crd.projectcalico.org"] + resources: + - bgpconfigurations + - bgppeers + verbs: + - create + - update + # These permissions are required for Calico CNI to perform IPAM allocations. + - apiGroups: ["crd.projectcalico.org"] + resources: + - blockaffinities + - ipamblocks + - ipamhandles + verbs: + - get + - list + - create + - update + - delete + - apiGroups: ["crd.projectcalico.org"] + resources: + - ipamconfigs + verbs: + - get + - create + # Block affinities must also be watchable by confd for route aggregation. + - apiGroups: ["crd.projectcalico.org"] + resources: + - blockaffinities + verbs: + - watch + # The Calico IPAM migration needs to get daemonsets. These permissions can be + # removed if not upgrading from an installation using host-local IPAM. + - apiGroups: ["apps"] + resources: + - daemonsets + verbs: + - get +{% endif %} + # Used for creating service account tokens to be used by the CNI plugin + - apiGroups: [""] + resources: + - serviceaccounts/token + resourceNames: + - calico-cni-plugin + verbs: + - create diff --git a/kubespray/project/roles/network_plugin/calico/templates/calico-crb.yml.j2 b/kubespray/project/roles/network_plugin/calico/templates/calico-crb.yml.j2 new file mode 100644 index 0000000..add99ba --- /dev/null +++ b/kubespray/project/roles/network_plugin/calico/templates/calico-crb.yml.j2 @@ -0,0 +1,28 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: calico-node +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-node +subjects: +- kind: ServiceAccount + name: calico-node + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: calico-cni-plugin +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-cni-plugin +subjects: +- kind: ServiceAccount + name: calico-cni-plugin + namespace: kube-system diff --git a/kubespray/project/roles/network_plugin/calico/templates/calico-ipamconfig.yml.j2 b/kubespray/project/roles/network_plugin/calico/templates/calico-ipamconfig.yml.j2 new file mode 100644 index 0000000..af7e211 --- /dev/null +++ b/kubespray/project/roles/network_plugin/calico/templates/calico-ipamconfig.yml.j2 @@ -0,0 +1,8 @@ +apiVersion: crd.projectcalico.org/v1 +kind: IPAMConfig +metadata: + name: default +spec: + autoAllocateBlocks: {{ calico_ipam_autoallocateblocks }} + strictAffinity: {{ calico_ipam_strictaffinity }} + maxBlocksPerHost: {{ calico_ipam_maxblocksperhost }} diff --git a/kubespray/project/roles/network_plugin/calico/templates/calico-node-sa.yml.j2 b/kubespray/project/roles/network_plugin/calico/templates/calico-node-sa.yml.j2 new file mode 100644 index 0000000..0743303 --- /dev/null +++ b/kubespray/project/roles/network_plugin/calico/templates/calico-node-sa.yml.j2 @@ -0,0 +1,13 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: calico-node + namespace: kube-system + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: calico-cni-plugin + namespace: kube-system diff --git a/kubespray/project/roles/network_plugin/calico/templates/calico-node.yml.j2 b/kubespray/project/roles/network_plugin/calico/templates/calico-node.yml.j2 new file mode 100644 index 0000000..5092011 --- /dev/null +++ b/kubespray/project/roles/network_plugin/calico/templates/calico-node.yml.j2 @@ -0,0 +1,475 @@ +--- +# This manifest installs the calico/node container, as well +# as the Calico CNI plugins and network config on +# each master and worker node in a Kubernetes cluster. +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: calico-node + namespace: kube-system + labels: + k8s-app: calico-node +spec: + selector: + matchLabels: + k8s-app: calico-node + template: + metadata: + labels: + k8s-app: calico-node + annotations: +{% if calico_datastore == "etcd" %} + kubespray.etcd-cert/serial: "{{ etcd_client_cert_serial }}" +{% endif %} +{% if calico_felix_prometheusmetricsenabled %} + prometheus.io/scrape: 'true' + prometheus.io/port: "{{ calico_felix_prometheusmetricsport }}" +{% endif %} + spec: + nodeSelector: + {{ calico_ds_nodeselector }} + priorityClassName: system-node-critical + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: calico-node + tolerations: + - operator: Exists + # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force + # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods. + terminationGracePeriodSeconds: 0 + initContainers: +{% if calico_datastore == "kdd" %} + # This container performs upgrade from host-local IPAM to calico-ipam. + # It can be deleted if this is a fresh installation, or if you have already + # upgraded to use calico-ipam. + - name: upgrade-ipam + image: {{ calico_cni_image_repo }}:{{ calico_cni_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + command: ["/opt/cni/bin/calico-ipam", "-upgrade"] + envFrom: + - configMapRef: + # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. + name: kubernetes-services-endpoint + optional: true + env: + - name: KUBERNETES_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CALICO_NETWORKING_BACKEND + valueFrom: + configMapKeyRef: + name: calico-config + key: calico_backend + volumeMounts: + - mountPath: /var/lib/cni/networks + name: host-local-net-dir + - mountPath: /host/opt/cni/bin + name: cni-bin-dir + securityContext: + privileged: true +{% endif %} + # This container installs the Calico CNI binaries + # and CNI network config file on each node. + - name: install-cni + image: {{ calico_cni_image_repo }}:{{ calico_cni_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + command: ["/opt/cni/bin/install"] + envFrom: + - configMapRef: + # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. + name: kubernetes-services-endpoint + optional: true + env: + # The CNI network config to install on each node. + - name: CNI_NETWORK_CONFIG + valueFrom: + configMapKeyRef: + name: calico-config + key: cni_network_config + # Name of the CNI config file to create. + - name: CNI_CONF_NAME + value: "10-calico.conflist" + # Install CNI binaries + - name: UPDATE_CNI_BINARIES + value: "true" + # Prevents the container from sleeping forever. + - name: SLEEP + value: "false" +{% if calico_datastore == "etcd" %} + - name: ETCD_ENDPOINTS + valueFrom: + configMapKeyRef: + name: calico-config + key: etcd_endpoints +{% endif %} +{% if calico_datastore == "kdd" %} + # Set the hostname based on the k8s node name. + - name: KUBERNETES_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName +{% endif %} + volumeMounts: + - mountPath: /host/etc/cni/net.d + name: cni-net-dir + - mountPath: /host/opt/cni/bin + name: cni-bin-dir + securityContext: + privileged: true + # Adds a Flex Volume Driver that creates a per-pod Unix Domain Socket to allow Dikastes + # to communicate with Felix over the Policy Sync API. + - name: flexvol-driver + image: {{ calico_flexvol_image_repo }}:{{ calico_flexvol_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + volumeMounts: + - name: flexvol-driver-host + mountPath: /host/driver + securityContext: + privileged: true + containers: + # Runs calico/node container on each Kubernetes node. This + # container programs network policy and routes on each + # host. + - name: calico-node + image: {{ calico_node_image_repo }}:{{ calico_node_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + envFrom: + - configMapRef: + # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. + name: kubernetes-services-endpoint + optional: true + env: + # The location of the Calico etcd cluster. +{% if calico_datastore == "etcd" %} + - name: ETCD_ENDPOINTS + valueFrom: + configMapKeyRef: + name: calico-config + key: etcd_endpoints + # Location of the CA certificate for etcd. + - name: ETCD_CA_CERT_FILE + valueFrom: + configMapKeyRef: + name: calico-config + key: etcd_ca + # Location of the client key for etcd. + - name: ETCD_KEY_FILE + valueFrom: + configMapKeyRef: + name: calico-config + key: etcd_key + # Location of the client certificate for etcd. + - name: ETCD_CERT_FILE + valueFrom: + configMapKeyRef: + name: calico-config + key: etcd_cert +{% elif calico_datastore == "kdd" %} + # Use Kubernetes API as the backing datastore. + - name: DATASTORE_TYPE + value: "kubernetes" +{% if typha_enabled %} + # Typha support: controlled by the ConfigMap. + - name: FELIX_TYPHAK8SSERVICENAME + valueFrom: + configMapKeyRef: + name: calico-config + key: typha_service_name +{% if typha_secure %} + - name: FELIX_TYPHACN + value: typha-server + - name: FELIX_TYPHACAFILE + value: /etc/typha-ca/ca.crt + - name: FELIX_TYPHACERTFILE + value: /etc/typha-client/typha-client.crt + - name: FELIX_TYPHAKEYFILE + value: /etc/typha-client/typha-client.key +{% endif %} +{% endif %} + # Wait for the datastore. + - name: WAIT_FOR_DATASTORE + value: "true" +{% endif %} +{% if calico_network_backend == 'vxlan' %} + - name: FELIX_VXLANVNI + value: "{{ calico_vxlan_vni }}" + - name: FELIX_VXLANPORT + value: "{{ calico_vxlan_port }}" +{% endif %} + # Choose the backend to use. + - name: CALICO_NETWORKING_BACKEND + valueFrom: + configMapKeyRef: + name: calico-config + key: calico_backend + # Cluster type to identify the deployment type + - name: CLUSTER_TYPE + valueFrom: + configMapKeyRef: + name: calico-config + key: cluster_type + # Set noderef for node controller. + - name: CALICO_K8S_NODE_REF + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # Disable file logging so `kubectl logs` works. + - name: CALICO_DISABLE_FILE_LOGGING + value: "true" + # Set Felix endpoint to host default action to ACCEPT. + - name: FELIX_DEFAULTENDPOINTTOHOSTACTION + value: "{{ calico_endpoint_to_host_action | default('RETURN') }}" + - name: FELIX_HEALTHHOST + value: "{{ calico_healthhost }}" +{% if kube_proxy_mode == 'ipvs' and kube_apiserver_node_port_range is defined %} + - name: FELIX_KUBENODEPORTRANGES + value: "{{ kube_apiserver_node_port_range.split('-')[0] }}:{{ kube_apiserver_node_port_range.split('-')[1] }}" +{% endif %} + - name: FELIX_IPTABLESBACKEND + value: "{{ calico_iptables_backend }}" + - name: FELIX_IPTABLESLOCKTIMEOUTSECS + value: "{{ calico_iptables_lock_timeout_secs }}" +# should be set in etcd before deployment +# # Configure the IP Pool from which Pod IPs will be chosen. +# - name: CALICO_IPV4POOL_CIDR +# value: "{{ calico_pool_cidr | default(kube_pods_subnet) }}" + - name: CALICO_IPV4POOL_IPIP + value: "{{ calico_ipv4pool_ipip }}" + - name: FELIX_IPV6SUPPORT + value: "{{ enable_dual_stack_networks | default(false) }}" + # Set Felix logging to "info" + - name: FELIX_LOGSEVERITYSCREEN + value: "{{ calico_loglevel }}" + # Set Calico startup logging to "error" + - name: CALICO_STARTUP_LOGLEVEL + value: "{{ calico_node_startup_loglevel }}" + # Enable or disable usage report + - name: FELIX_USAGEREPORTINGENABLED + value: "{{ calico_usage_reporting }}" + # Set MTU for tunnel device used if ipip is enabled +{% if calico_mtu is defined %} + # Set MTU for tunnel device used if ipip is enabled + - name: FELIX_IPINIPMTU + value: "{{ calico_veth_mtu | default(calico_mtu) }}" + # Set MTU for the VXLAN tunnel device. + - name: FELIX_VXLANMTU + value: "{{ calico_veth_mtu | default(calico_mtu) }}" + # Set MTU for the Wireguard tunnel device. + - name: FELIX_WIREGUARDMTU + value: "{{ calico_veth_mtu | default(calico_mtu) }}" +{% endif %} + - name: FELIX_CHAININSERTMODE + value: "{{ calico_felix_chaininsertmode }}" + - name: FELIX_PROMETHEUSMETRICSENABLED + value: "{{ calico_felix_prometheusmetricsenabled }}" + - name: FELIX_PROMETHEUSMETRICSPORT + value: "{{ calico_felix_prometheusmetricsport }}" + - name: FELIX_PROMETHEUSGOMETRICSENABLED + value: "{{ calico_felix_prometheusgometricsenabled }}" + - name: FELIX_PROMETHEUSPROCESSMETRICSENABLED + value: "{{ calico_felix_prometheusprocessmetricsenabled }}" +{% if calico_ip_auto_method is defined %} + - name: IP_AUTODETECTION_METHOD + value: "{{ calico_ip_auto_method }}" +{% else %} + - name: NODEIP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: IP_AUTODETECTION_METHOD + value: "can-reach=$(NODEIP)" +{% endif %} + - name: IP + value: "autodetect" +{% if calico_ip6_auto_method is defined and enable_dual_stack_networks %} + - name: IP6_AUTODETECTION_METHOD + value: "{{ calico_ip6_auto_method }}" +{% endif %} +{% if calico_felix_mtu_iface_pattern is defined %} + - name: FELIX_MTUIFACEPATTERN + value: "{{ calico_felix_mtu_iface_pattern }}" +{% endif %} +{% if enable_dual_stack_networks %} + - name: IP6 + value: autodetect +{% endif %} +{% if calico_use_default_route_src_ipaddr | default(false) %} + - name: FELIX_DEVICEROUTESOURCEADDRESS + valueFrom: + fieldRef: + fieldPath: status.hostIP +{% endif %} + - name: NODENAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: FELIX_HEALTHENABLED + value: "true" + - name: FELIX_IGNORELOOSERPF + value: "{{ calico_node_ignorelooserpf }}" + - name: CALICO_MANAGE_CNI + value: "true" +{% if calico_node_extra_envs is defined %} +{% for key in calico_node_extra_envs %} + - name: {{ key }} + value: "{{ calico_node_extra_envs[key] }}" +{% endfor %} +{% endif %} + securityContext: + privileged: true + resources: + limits: + cpu: {{ calico_node_cpu_limit }} + memory: {{ calico_node_memory_limit }} + requests: + cpu: {{ calico_node_cpu_requests }} + memory: {{ calico_node_memory_requests }} + lifecycle: + preStop: + exec: + command: + - /bin/calico-node + - -shutdown + livenessProbe: + exec: + command: + - /bin/calico-node + - -felix-live +{% if calico_network_backend == "bird" %} + - -bird-live +{% endif %} + periodSeconds: 10 + initialDelaySeconds: 10 + timeoutSeconds: {{ calico_node_livenessprobe_timeout | default(10) }} + failureThreshold: 6 + readinessProbe: + exec: + command: + - /bin/calico-node +{% if calico_network_backend == "bird" %} + - -bird-ready +{% endif %} + - -felix-ready + periodSeconds: 10 + timeoutSeconds: {{ calico_node_readinessprobe_timeout | default(10) }} + failureThreshold: 6 + volumeMounts: + - mountPath: /lib/modules + name: lib-modules + readOnly: true + - mountPath: /var/run/calico + name: var-run-calico + readOnly: false + - mountPath: /var/lib/calico + name: var-lib-calico + readOnly: false +{% if calico_datastore == "etcd" %} + - mountPath: /calico-secrets + name: etcd-certs + readOnly: true +{% endif %} + - name: xtables-lock + mountPath: /run/xtables.lock + readOnly: false + # For maintaining CNI plugin API credentials. + - mountPath: /host/etc/cni/net.d + name: cni-net-dir + readOnly: false +{% if typha_secure %} + - name: typha-client + mountPath: /etc/typha-client + readOnly: true + - name: typha-cacert + subPath: ca.crt + mountPath: /etc/typha-ca/ca.crt + readOnly: true +{% endif %} + - name: policysync + mountPath: /var/run/nodeagent +{% if calico_bpf_enabled %} + # For eBPF mode, we need to be able to mount the BPF filesystem at /sys/fs/bpf so we mount in the + # parent directory. + - name: sysfs + mountPath: /sys/fs/ + # Bidirectional means that, if we mount the BPF filesystem at /sys/fs/bpf it will propagate to the host. + # If the host is known to mount that filesystem already then Bidirectional can be omitted. + mountPropagation: Bidirectional +{% endif %} + - name: cni-log-dir + mountPath: /var/log/calico/cni + readOnly: true + volumes: + # Used by calico/node. + - name: lib-modules + hostPath: + path: /lib/modules + - name: var-run-calico + hostPath: + path: /var/run/calico + - name: var-lib-calico + hostPath: + path: /var/lib/calico + # Used to install CNI. + - name: cni-net-dir + hostPath: + path: /etc/cni/net.d + - name: cni-bin-dir + hostPath: + path: /opt/cni/bin +{% if calico_datastore == "etcd" %} + # Mount in the etcd TLS secrets. + - name: etcd-certs + hostPath: + path: "{{ calico_cert_dir }}" +{% endif %} + # Mount the global iptables lock file, used by calico/node + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate +{% if calico_datastore == "kdd" %} + # Mount in the directory for host-local IPAM allocations. This is + # used when upgrading from host-local to calico-ipam, and can be removed + # if not using the upgrade-ipam init container. + - name: host-local-net-dir + hostPath: + path: /var/lib/cni/networks +{% endif %} +{% if typha_enabled and typha_secure %} + - name: typha-client + secret: + secretName: typha-client + items: + - key: tls.crt + path: typha-client.crt + - key: tls.key + path: typha-client.key + - name: typha-cacert + hostPath: + path: "/etc/kubernetes/ssl/" +{% endif %} +{% if calico_bpf_enabled %} + - name: sysfs + hostPath: + path: /sys/fs/ + type: DirectoryOrCreate +{% endif %} + # Used to access CNI logs. + - name: cni-log-dir + hostPath: + path: /var/log/calico/cni + # Used to create per-pod Unix Domain Sockets + - name: policysync + hostPath: + type: DirectoryOrCreate + path: /var/run/nodeagent + # Used to install Flex Volume Driver + - name: flexvol-driver-host + hostPath: + type: DirectoryOrCreate + path: "{{ kubelet_flexvolumes_plugins_dir | default('/usr/libexec/kubernetes/kubelet-plugins/volume/exec') }}/nodeagent~uds" + updateStrategy: + rollingUpdate: + maxUnavailable: {{ serial | default('20%') }} + type: RollingUpdate diff --git a/kubespray/project/roles/network_plugin/calico/templates/calico-typha.yml.j2 b/kubespray/project/roles/network_plugin/calico/templates/calico-typha.yml.j2 new file mode 100644 index 0000000..2c6d510 --- /dev/null +++ b/kubespray/project/roles/network_plugin/calico/templates/calico-typha.yml.j2 @@ -0,0 +1,187 @@ +# This manifest creates a Service, which will be backed by Calico's Typha daemon. +# Typha sits in between Felix and the API server, reducing Calico's load on the API server. + +apiVersion: v1 +kind: Service +metadata: + name: calico-typha + namespace: kube-system + labels: + k8s-app: calico-typha +spec: + ports: + - port: 5473 + protocol: TCP + targetPort: calico-typha + name: calico-typha +{% if typha_prometheusmetricsenabled %} + - port: {{ typha_prometheusmetricsport }} + protocol: TCP + targetPort: http-metrics + name: metrics +{% endif %} + selector: + k8s-app: calico-typha + +--- + +# This manifest creates a Deployment of Typha to back the above service. + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: calico-typha + namespace: kube-system + labels: + k8s-app: calico-typha +spec: + # Number of Typha replicas. To enable Typha, set this to a non-zero value *and* set the + # typha_service_name variable in the calico-config ConfigMap above. + # + # We recommend using Typha if you have more than 50 nodes. Above 100 nodes it is essential + # (when using the Kubernetes datastore). Use one replica for every 100-200 nodes. In + # production, we recommend running at least 3 replicas to reduce the impact of rolling upgrade. + replicas: {{ typha_replicas }} + revisionHistoryLimit: 2 + selector: + matchLabels: + k8s-app: calico-typha + template: + metadata: + labels: + k8s-app: calico-typha + annotations: + cluster-autoscaler.kubernetes.io/safe-to-evict: 'true' +{% if typha_prometheusmetricsenabled %} + prometheus.io/scrape: 'true' + prometheus.io/port: "{{ typha_prometheusmetricsport }}" +{% endif %} + spec: + nodeSelector: + kubernetes.io/os: linux + hostNetwork: true + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + # Since Calico can't network a pod until Typha is up, we need to run Typha itself + # as a host-networked pod. + serviceAccountName: calico-node + priorityClassName: system-cluster-critical + # fsGroup allows using projected serviceaccount tokens as described here kubernetes/kubernetes#82573 + securityContext: + fsGroup: 65534 + containers: + - image: {{ calico_typha_image_repo }}:{{ calico_typha_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + name: calico-typha + ports: + - containerPort: 5473 + name: calico-typha + protocol: TCP +{% if typha_prometheusmetricsenabled %} + - containerPort: {{ typha_prometheusmetricsport }} + name: http-metrics + protocol: TCP +{% endif %} + envFrom: + - configMapRef: + # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. + name: kubernetes-services-endpoint + optional: true + env: + # Enable "info" logging by default. Can be set to "debug" to increase verbosity. + - name: TYPHA_LOGSEVERITYSCREEN + value: "info" + # Disable logging to file and syslog since those don't make sense in Kubernetes. + - name: TYPHA_LOGFILEPATH + value: "none" + - name: TYPHA_LOGSEVERITYSYS + value: "none" + # Monitor the Kubernetes API to find the number of running instances and rebalance + # connections. + - name: TYPHA_CONNECTIONREBALANCINGMODE + value: "kubernetes" + - name: TYPHA_DATASTORETYPE + value: "kubernetes" + - name: TYPHA_HEALTHENABLED + value: "true" + - name: TYPHA_MAXCONNECTIONSLOWERLIMIT + value: "{{ typha_max_connections_lower_limit }}" +{% if typha_secure %} + - name: TYPHA_CAFILE + value: /etc/ca/ca.crt + - name: TYPHA_CLIENTCN + value: typha-client + - name: TYPHA_SERVERCERTFILE + value: /etc/typha/server_certificate.pem + - name: TYPHA_SERVERKEYFILE + value: /etc/typha/server_key.pem +{% endif %} +{% if typha_prometheusmetricsenabled %} + # Since Typha is host-networked, + # this opens a port on the host, which may need to be secured. + - name: TYPHA_PROMETHEUSMETRICSENABLED + value: "true" + - name: TYPHA_PROMETHEUSMETRICSPORT + value: "{{ typha_prometheusmetricsport }}" +{% endif %} +{% if typha_secure %} + volumeMounts: + - mountPath: /etc/typha + name: typha-server + readOnly: true + - mountPath: /etc/ca/ca.crt + subPath: ca.crt + name: cacert + readOnly: true +{% endif %} + # Needed for version >=3.7 when the 'host-local' ipam is used + # Should never happen given templates/cni-calico.conflist.j2 + # Configure route aggregation based on pod CIDR. + # - name: USE_POD_CIDR + # value: "true" + livenessProbe: + httpGet: + path: /liveness + port: 9098 + host: localhost + periodSeconds: 30 + initialDelaySeconds: 30 + readinessProbe: + httpGet: + path: /readiness + port: 9098 + host: localhost + periodSeconds: 10 +{% if typha_secure %} + volumes: + - name: typha-server + secret: + secretName: typha-server + items: + - key: tls.crt + path: server_certificate.pem + - key: tls.key + path: server_key.pem + - name: cacert + hostPath: + path: "{{ kube_cert_dir }}" +{% endif %} + +--- + +# This manifest creates a Pod Disruption Budget for Typha to allow K8s Cluster Autoscaler to evict + +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: calico-typha + namespace: kube-system + labels: + k8s-app: calico-typha +spec: + maxUnavailable: 1 + selector: + matchLabels: + k8s-app: calico-typha diff --git a/kubespray/project/roles/network_plugin/calico/templates/calicoctl.etcd.sh.j2 b/kubespray/project/roles/network_plugin/calico/templates/calicoctl.etcd.sh.j2 new file mode 100644 index 0000000..fcde4a5 --- /dev/null +++ b/kubespray/project/roles/network_plugin/calico/templates/calicoctl.etcd.sh.j2 @@ -0,0 +1,6 @@ +#!/bin/bash +ETCD_ENDPOINTS={{ etcd_access_addresses }} \ +ETCD_CA_CERT_FILE={{ calico_cert_dir }}/ca_cert.crt \ +ETCD_CERT_FILE={{ calico_cert_dir }}/cert.crt \ +ETCD_KEY_FILE={{ calico_cert_dir }}/key.pem \ +{{ bin_dir }}/calicoctl --allow-version-mismatch "$@" diff --git a/kubespray/project/roles/network_plugin/calico/templates/calicoctl.kdd.sh.j2 b/kubespray/project/roles/network_plugin/calico/templates/calicoctl.kdd.sh.j2 new file mode 100644 index 0000000..ef89f39 --- /dev/null +++ b/kubespray/project/roles/network_plugin/calico/templates/calicoctl.kdd.sh.j2 @@ -0,0 +1,8 @@ +#!/bin/bash +DATASTORE_TYPE=kubernetes \ +{% if inventory_hostname in groups['kube_control_plane'] %} +KUBECONFIG=/etc/kubernetes/admin.conf \ +{% else %} +KUBECONFIG=/etc/cni/net.d/calico-kubeconfig \ +{% endif %} +{{ bin_dir }}/calicoctl --allow-version-mismatch "$@" diff --git a/kubespray/project/roles/network_plugin/calico/templates/kubernetes-services-endpoint.yml.j2 b/kubespray/project/roles/network_plugin/calico/templates/kubernetes-services-endpoint.yml.j2 new file mode 100644 index 0000000..f1e8177 --- /dev/null +++ b/kubespray/project/roles/network_plugin/calico/templates/kubernetes-services-endpoint.yml.j2 @@ -0,0 +1,11 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: kube-system + name: kubernetes-services-endpoint +data: +{% if calico_bpf_enabled %} + KUBERNETES_SERVICE_HOST: "{{ kube_apiserver_global_endpoint | urlsplit('hostname') }}" + KUBERNETES_SERVICE_PORT: "{{ kube_apiserver_global_endpoint | urlsplit('port') }}" +{% endif %} diff --git a/kubespray/project/roles/network_plugin/calico/templates/make-ssl-calico.sh.j2 b/kubespray/project/roles/network_plugin/calico/templates/make-ssl-calico.sh.j2 new file mode 100644 index 0000000..94b2022 --- /dev/null +++ b/kubespray/project/roles/network_plugin/calico/templates/make-ssl-calico.sh.j2 @@ -0,0 +1,102 @@ +#!/bin/bash + +# Author: Smana smainklh@gmail.com +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o pipefail +usage() +{ + cat << EOF +Create self signed certificates + +Usage : $(basename $0) -f [-d ] + -h | --help : Show this message + -f | --config : Openssl configuration file + -d | --ssldir : Directory where the certificates will be installed + -c | --cadir : Directory where the existing CA is located + -s | --service : Service for the ca + + ex : + $(basename $0) -f openssl.conf -d /srv/ssl +EOF +} + +# Options parsing +while (($#)); do + case "$1" in + -h | --help) usage; exit 0;; + -f | --config) CONFIG=${2}; shift 2;; + -d | --ssldir) SSLDIR="${2}"; shift 2;; + -c | --cadir) CADIR="${2}"; shift 2;; + -s | --service) SERVICE="${2}"; shift 2;; + *) + usage + echo "ERROR : Unknown option" + exit 3 + ;; + esac +done + +if [ -z ${CONFIG} ]; then + echo "ERROR: the openssl configuration file is missing. option -f" + exit 1 +fi +if [ -z ${SSLDIR} ]; then + SSLDIR="/etc/calico/certs" +fi + +tmpdir=$(mktemp -d /tmp/calico_${SERVICE}_certs.XXXXXX) +trap 'rm -rf "${tmpdir}"' EXIT +cd "${tmpdir}" + +mkdir -p ${SSLDIR} ${CADIR} + +# Root CA +if [ -e "$CADIR/ca.key" ]; then + # Reuse existing CA + cp $CADIR/{ca.crt,ca.key} . +else + openssl genrsa -out ca.key {{certificates_key_size}} > /dev/null 2>&1 + openssl req -x509 -new -nodes -key ca.key -days {{certificates_duration}} -out ca.crt -subj "/CN=calico-${SERVICE}-ca" > /dev/null 2>&1 +fi + +if [ $SERVICE == "typha" ]; then + # Typha server + openssl genrsa -out typha-server.key {{certificates_key_size}} > /dev/null 2>&1 + openssl req -new -key typha-server.key -out typha-server.csr -subj "/CN=typha-server" -config ${CONFIG} > /dev/null 2>&1 + openssl x509 -req -in typha-server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out typha-server.crt -days {{certificates_duration}} -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1 + + # Typha client + openssl genrsa -out typha-client.key {{certificates_key_size}} > /dev/null 2>&1 + openssl req -new -key typha-client.key -out typha-client.csr -subj "/CN=typha-client" -config ${CONFIG} > /dev/null 2>&1 + openssl x509 -req -in typha-client.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out typha-client.crt -days {{certificates_duration}} -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1 + +elif [ $SERVICE == "apiserver" ]; then + # calico-apiserver + openssl genrsa -out apiserver.key {{certificates_key_size}} > /dev/null 2>&1 + openssl req -new -key apiserver.key -out apiserver.csr -subj "/CN=calico-apiserver" -config ${CONFIG} > /dev/null 2>&1 + openssl x509 -req -in apiserver.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out apiserver.crt -days {{certificates_duration}} -extensions ssl_client_apiserver -extfile ${CONFIG} > /dev/null 2>&1 +else + echo "ERROR: the openssl configuration file is missing. option -s" + exit 1 +fi + +# Install certs +if [ -e "$CADIR/ca.key" ]; then + # No pass existing CA + rm -f ca.crt ca.key +fi + +mv {*.crt,*.key} ${SSLDIR}/ diff --git a/kubespray/project/roles/network_plugin/calico/vars/amazon.yml b/kubespray/project/roles/network_plugin/calico/vars/amazon.yml new file mode 100644 index 0000000..83efdcd --- /dev/null +++ b/kubespray/project/roles/network_plugin/calico/vars/amazon.yml @@ -0,0 +1,5 @@ +--- +calico_wireguard_repo: https://download.copr.fedorainfracloud.org/results/jdoss/wireguard/epel-7-$basearch/ +calico_wireguard_packages: + - wireguard-dkms + - wireguard-tools diff --git a/kubespray/project/roles/network_plugin/calico/vars/centos-9.yml b/kubespray/project/roles/network_plugin/calico/vars/centos-9.yml new file mode 100644 index 0000000..43df545 --- /dev/null +++ b/kubespray/project/roles/network_plugin/calico/vars/centos-9.yml @@ -0,0 +1,3 @@ +--- +calico_wireguard_packages: + - wireguard-tools diff --git a/kubespray/project/roles/network_plugin/calico/vars/debian.yml b/kubespray/project/roles/network_plugin/calico/vars/debian.yml new file mode 100644 index 0000000..baf603c --- /dev/null +++ b/kubespray/project/roles/network_plugin/calico/vars/debian.yml @@ -0,0 +1,3 @@ +--- +calico_wireguard_packages: + - wireguard diff --git a/kubespray/project/roles/network_plugin/calico/vars/fedora.yml b/kubespray/project/roles/network_plugin/calico/vars/fedora.yml new file mode 100644 index 0000000..43df545 --- /dev/null +++ b/kubespray/project/roles/network_plugin/calico/vars/fedora.yml @@ -0,0 +1,3 @@ +--- +calico_wireguard_packages: + - wireguard-tools diff --git a/kubespray/project/roles/network_plugin/calico/vars/opensuse.yml b/kubespray/project/roles/network_plugin/calico/vars/opensuse.yml new file mode 100644 index 0000000..43df545 --- /dev/null +++ b/kubespray/project/roles/network_plugin/calico/vars/opensuse.yml @@ -0,0 +1,3 @@ +--- +calico_wireguard_packages: + - wireguard-tools diff --git a/kubespray/project/roles/network_plugin/calico/vars/redhat-9.yml b/kubespray/project/roles/network_plugin/calico/vars/redhat-9.yml new file mode 100644 index 0000000..43df545 --- /dev/null +++ b/kubespray/project/roles/network_plugin/calico/vars/redhat-9.yml @@ -0,0 +1,3 @@ +--- +calico_wireguard_packages: + - wireguard-tools diff --git a/kubespray/project/roles/network_plugin/calico/vars/redhat.yml b/kubespray/project/roles/network_plugin/calico/vars/redhat.yml new file mode 100644 index 0000000..a83a8a5 --- /dev/null +++ b/kubespray/project/roles/network_plugin/calico/vars/redhat.yml @@ -0,0 +1,4 @@ +--- +calico_wireguard_packages: + - wireguard-dkms + - wireguard-tools diff --git a/kubespray/project/roles/network_plugin/calico/vars/rocky-9.yml b/kubespray/project/roles/network_plugin/calico/vars/rocky-9.yml new file mode 100644 index 0000000..43df545 --- /dev/null +++ b/kubespray/project/roles/network_plugin/calico/vars/rocky-9.yml @@ -0,0 +1,3 @@ +--- +calico_wireguard_packages: + - wireguard-tools diff --git a/kubespray/project/roles/network_plugin/calico_defaults/defaults/main.yml b/kubespray/project/roles/network_plugin/calico_defaults/defaults/main.yml new file mode 100644 index 0000000..8b909f0 --- /dev/null +++ b/kubespray/project/roles/network_plugin/calico_defaults/defaults/main.yml @@ -0,0 +1,176 @@ +--- +# the default value of name +calico_cni_name: k8s-pod-network + +# Enables Internet connectivity from containers +nat_outgoing: true +nat_outgoing_ipv6: false + +# add default ippool name +calico_pool_name: "default-pool" +calico_ipv4pool_ipip: "Off" + +# Change encapsulation mode, by default we enable vxlan which is the most mature and well tested mode +calico_ipip_mode: Never # valid values are 'Always', 'Never' and 'CrossSubnet' +calico_vxlan_mode: Always # valid values are 'Always', 'Never' and 'CrossSubnet' + +calico_cni_pool: true +calico_cni_pool_ipv6: true + +# add default ippool blockSize +calico_pool_blocksize: 26 + +# Calico doesn't support ipip tunneling for the IPv6. +calico_ipip_mode_ipv6: Never +calico_vxlan_mode_ipv6: Never + +# add default ipv6 ippool blockSize +calico_pool_blocksize_ipv6: 122 + +# Calico network backend can be 'bird', 'vxlan' and 'none' +calico_network_backend: vxlan + +calico_cert_dir: /etc/calico/certs + +# Global as_num (/calico/bgp/v1/global/as_num) +global_as_num: "64512" + +# You can set MTU value here. If left undefined or empty, it will +# not be specified in calico CNI config, so Calico will use built-in +# defaults. The value should be a number, not a string. +# calico_mtu: 1500 + +# Advertise Service External IPs +calico_advertise_service_external_ips: [] + +# Advertise Service LoadBalancer IPs +calico_advertise_service_loadbalancer_ips: [] + +# Calico eBPF support +calico_bpf_enabled: false +calico_bpf_log_level: "" +# Valid option for service mode: Tunnel (default), DSR=Direct Server Return +calico_bpf_service_mode: Tunnel + +# Calico floatingIPs support +# Valid option for floatingIPs: Disabled (default), Enabled +calico_felix_floating_ips: Disabled + +# Limits for apps +calico_node_memory_limit: 500M +calico_node_cpu_limit: 300m +calico_node_memory_requests: 64M +calico_node_cpu_requests: 150m +calico_felix_chaininsertmode: Insert + +# Calico daemonset nodeselector +calico_ds_nodeselector: "kubernetes.io/os: linux" + +# Virtual network ID to use for VXLAN traffic. A value of 0 means “use the kernel default”. +calico_vxlan_vni: 4096 + +# Port to use for VXLAN traffic. A value of 0 means “use the kernel default”. +calico_vxlan_port: 4789 + +# Enable Prometheus Metrics endpoint for felix +calico_felix_prometheusmetricsenabled: false +calico_felix_prometheusmetricsport: 9091 +calico_felix_prometheusgometricsenabled: true +calico_felix_prometheusprocessmetricsenabled: true + +# Set the agent log level. Can be debug, warning, info or fatal +calico_loglevel: info +calico_node_startup_loglevel: error + +# Set log path for calico CNI plugin. Set to false to disable logging to disk. +calico_cni_log_file_path: /var/log/calico/cni/cni.log + +# Enable or disable usage report to 'usage.projectcalico.org' +calico_usage_reporting: false + +# Should calico ignore kernel's RPF check setting, +# see https://github.com/projectcalico/felix/blob/ab8799eaea66627e5db7717e62fca61fd9c08646/python/calico/felix/config.py#L198 +calico_node_ignorelooserpf: false + +# Define address on which Felix will respond to health requests +calico_healthhost: "localhost" + +# Configure time in seconds that calico will wait for the iptables lock +calico_iptables_lock_timeout_secs: 10 + +# Choose Calico iptables backend: "Legacy", "Auto" or "NFT" (FELIX_IPTABLESBACKEND) +calico_iptables_backend: "Auto" + +# Calico Wireguard support +calico_wireguard_enabled: false +calico_wireguard_packages: [] +calico_wireguard_repo: https://download.copr.fedorainfracloud.org/results/jdoss/wireguard/epel-{{ ansible_distribution_major_version }}-$basearch/ + +# If you want to use non default IP_AUTODETECTION_METHOD, IP6_AUTODETECTION_METHOD for calico node set this option to one of: +# * can-reach=DESTINATION +# * interface=INTERFACE-REGEX +# see https://projectcalico.docs.tigera.io/reference/node/configuration#ip-autodetection-methods +# calico_ip_auto_method: "interface=eth.*" +# calico_ip6_auto_method: "interface=eth.*" + +# Set FELIX_MTUIFACEPATTERN, Pattern used to discover the host’s interface for MTU auto-detection. +# see https://projectcalico.docs.tigera.io/reference/felix/configuration +# calico_felix_mtu_iface_pattern: "^((en|wl|ww|sl|ib)[opsx].*|(eth|wlan|wwan).*)" + +calico_baremetal_nodename: "{{ kube_override_hostname | default(inventory_hostname) }}" + +kube_etcd_cacert_file: ca.pem +kube_etcd_cert_file: node-{{ inventory_hostname }}.pem +kube_etcd_key_file: node-{{ inventory_hostname }}-key.pem + +# Choose data store type for calico: "etcd" or "kdd" (kubernetes datastore) +# The default value for calico_datastore is set in role kubespray-default + +# Use typha (only with kdd) +typha_enabled: false +typha_prometheusmetricsenabled: false +typha_prometheusmetricsport: 9093 + +# Scaling typha: 1 replica per 100 nodes is adequate +# Number of typha replicas +typha_replicas: 1 + +# Set max typha connections +typha_max_connections_lower_limit: 300 + +# Generate certifcates for typha<->calico-node communication +typha_secure: false + +calico_feature_control: {} + +# Calico default BGP port +calico_bgp_listen_port: 179 + +# Calico FelixConfiguration options +calico_felix_reporting_interval: 0s +calico_felix_log_severity_screen: Info + +# Calico container settings +calico_allow_ip_forwarding: false + +# Calico IPAM strictAffinity +calico_ipam_strictaffinity: false + +# Calico IPAM autoAllocateBlocks +calico_ipam_autoallocateblocks: true + +# Calico IPAM maxBlocksPerHost, default 0 +calico_ipam_maxblocksperhost: 0 + +# Calico host local IPAM (use node .spec.podCIDR) + +calico_ipam_host_local: false + +# Calico apiserver (only with kdd) +calico_apiserver_enabled: false + +# Calico feature detect override +calico_feature_detect_override: "" + +# Calico kubeconfig wait timeout in seconds +calico_kubeconfig_wait_timeout: 300 diff --git a/kubespray/project/roles/network_plugin/cilium/defaults/main.yml b/kubespray/project/roles/network_plugin/cilium/defaults/main.yml new file mode 100644 index 0000000..f4c70e4 --- /dev/null +++ b/kubespray/project/roles/network_plugin/cilium/defaults/main.yml @@ -0,0 +1,311 @@ +--- +cilium_min_version_required: "1.10" +# Log-level +cilium_debug: false + +cilium_mtu: "" +cilium_enable_ipv4: true +cilium_enable_ipv6: false + +# Cilium agent health port +cilium_agent_health_port: "{%- if cilium_version | regex_replace('v') is version('1.11.6', '>=') -%}9879{%- else -%}9876{%- endif -%}" + +# Identity allocation mode selects how identities are shared between cilium +# nodes by setting how they are stored. The options are "crd" or "kvstore". +# - "crd" stores identities in kubernetes as CRDs (custom resource definition). +# These can be queried with: +# `kubectl get ciliumid` +# - "kvstore" stores identities in an etcd kvstore. +# - In order to support External Workloads, "crd" is required +# - Ref: https://docs.cilium.io/en/stable/gettingstarted/external-workloads/#setting-up-support-for-external-workloads-beta +# - KVStore operations are only required when cilium-operator is running with any of the below options: +# - --synchronize-k8s-services +# - --synchronize-k8s-nodes +# - --identity-allocation-mode=kvstore +# - Ref: https://docs.cilium.io/en/stable/internals/cilium_operator/#kvstore-operations +cilium_identity_allocation_mode: kvstore + +# Etcd SSL dirs +cilium_cert_dir: /etc/cilium/certs +kube_etcd_cacert_file: ca.pem +kube_etcd_cert_file: node-{{ inventory_hostname }}.pem +kube_etcd_key_file: node-{{ inventory_hostname }}-key.pem + +# Limits for apps +cilium_memory_limit: 500M +cilium_cpu_limit: 500m +cilium_memory_requests: 64M +cilium_cpu_requests: 100m + +# Overlay Network Mode +cilium_tunnel_mode: vxlan +# Optional features +cilium_enable_prometheus: false +# Enable if you want to make use of hostPort mappings +cilium_enable_portmap: false +# Monitor aggregation level (none/low/medium/maximum) +cilium_monitor_aggregation: medium +# Kube Proxy Replacement mode (strict/partial) +cilium_kube_proxy_replacement: partial + +# If upgrading from Cilium < 1.5, you may want to override some of these options +# to prevent service disruptions. See also: +# http://docs.cilium.io/en/stable/install/upgrade/#changes-that-may-require-action +cilium_preallocate_bpf_maps: false + +# `cilium_tofqdns_enable_poller` is deprecated in 1.8, removed in 1.9 +cilium_tofqdns_enable_poller: false + +# `cilium_enable_legacy_services` is deprecated in 1.6, removed in 1.9 +cilium_enable_legacy_services: false + +# Deploy cilium even if kube_network_plugin is not cilium. +# This enables to deploy cilium alongside another CNI to replace kube-proxy. +cilium_deploy_additionally: false + +# Auto direct nodes routes can be used to advertise pods routes in your cluster +# without any tunelling (with `cilium_tunnel_mode` sets to `disabled`). +# This works only if you have a L2 connectivity between all your nodes. +# You wil also have to specify the variable `cilium_native_routing_cidr` to +# make this work. Please refer to the cilium documentation for more +# information about this kind of setups. +cilium_auto_direct_node_routes: false + +# Allows to explicitly specify the IPv4 CIDR for native routing. +# When specified, Cilium assumes networking for this CIDR is preconfigured and +# hands traffic destined for that range to the Linux network stack without +# applying any SNAT. +# Generally speaking, specifying a native routing CIDR implies that Cilium can +# depend on the underlying networking stack to route packets to their +# destination. To offer a concrete example, if Cilium is configured to use +# direct routing and the Kubernetes CIDR is included in the native routing CIDR, +# the user must configure the routes to reach pods, either manually or by +# setting the auto-direct-node-routes flag. +cilium_native_routing_cidr: "" + +# Allows to explicitly specify the IPv6 CIDR for native routing. +cilium_native_routing_cidr_ipv6: "" + +# Enable transparent network encryption. +cilium_encryption_enabled: false + +# Encryption method. Can be either ipsec or wireguard. +# Only effective when `cilium_encryption_enabled` is set to true. +cilium_encryption_type: "ipsec" + +# Enable encryption for pure node to node traffic. +# This option is only effective when `cilium_encryption_type` is set to `ipsec`. +cilium_ipsec_node_encryption: false + +# If your kernel or distribution does not support WireGuard, Cilium agent can be configured to fall back on the user-space implementation. +# When this flag is enabled and Cilium detects that the kernel has no native support for WireGuard, +# it will fallback on the wireguard-go user-space implementation of WireGuard. +# This option is only effective when `cilium_encryption_type` is set to `wireguard`. +cilium_wireguard_userspace_fallback: false + +# Enable Bandwidth Manager +# Cilium’s bandwidth manager supports the kubernetes.io/egress-bandwidth Pod annotation. +# Bandwidth enforcement currently does not work in combination with L7 Cilium Network Policies. +# In case they select the Pod at egress, then the bandwidth enforcement will be disabled for those Pods. +# Bandwidth Manager requires a v5.1.x or more recent Linux kernel. +cilium_enable_bandwidth_manager: false + +# IP Masquerade Agent +# https://docs.cilium.io/en/stable/concepts/networking/masquerading/ +# By default, all packets from a pod destined to an IP address outside of the cilium_native_routing_cidr range are masqueraded +cilium_ip_masq_agent_enable: false + +### A packet sent from a pod to a destination which belongs to any CIDR from the nonMasqueradeCIDRs is not going to be masqueraded +cilium_non_masquerade_cidrs: + - 10.0.0.0/8 + - 172.16.0.0/12 + - 192.168.0.0/16 + - 100.64.0.0/10 + - 192.0.0.0/24 + - 192.0.2.0/24 + - 192.88.99.0/24 + - 198.18.0.0/15 + - 198.51.100.0/24 + - 203.0.113.0/24 + - 240.0.0.0/4 +### Indicates whether to masquerade traffic to the link local prefix. +### If the masqLinkLocal is not set or set to false, then 169.254.0.0/16 is appended to the non-masquerade CIDRs list. +cilium_masq_link_local: false +### A time interval at which the agent attempts to reload config from disk +cilium_ip_masq_resync_interval: 60s + +# Hubble +### Enable Hubble without install +cilium_enable_hubble: false +### Enable Hubble Metrics +cilium_enable_hubble_metrics: false +### if cilium_enable_hubble_metrics: true +cilium_hubble_metrics: {} +# - dns +# - drop +# - tcp +# - flow +# - icmp +# - http +### Enable Hubble install +cilium_hubble_install: false +### Enable auto generate certs if cilium_hubble_install: true +cilium_hubble_tls_generate: false + +# The default IP address management mode is "Cluster Scope". +# https://docs.cilium.io/en/stable/concepts/networking/ipam/ +cilium_ipam_mode: cluster-pool + +# Cluster Pod CIDRs use the kube_pods_subnet value by default. +# If your node network is in the same range you will lose connectivity to other nodes. +# Defaults to kube_pods_subnet if not set. +# cilium_pool_cidr: 10.233.64.0/18 + +# When cilium_enable_ipv6 is used, you need to set the IPV6 value. Defaults to kube_pods_subnet_ipv6 if not set. +# cilium_pool_cidr_ipv6: fd85:ee78:d8a6:8607::1:0000/112 + +# When cilium IPAM uses the "Cluster Scope" mode, it will pre-allocate a segment of IP to each node, +# schedule the Pod to this node, and then allocate IP from here. cilium_pool_mask_size Specifies +# the size allocated from cluster Pod CIDR to node.ipam.podCIDRs +# Defaults to kube_network_node_prefix if not set. +# cilium_pool_mask_size: "24" + +# cilium_pool_mask_size Specifies the size allocated to node.ipam.podCIDRs from cluster Pod IPV6 CIDR +# Defaults to kube_network_node_prefix_ipv6 if not set. +# cilium_pool_mask_size_ipv6: "120" + + +# Extra arguments for the Cilium agent +cilium_agent_custom_args: [] + +# For adding and mounting extra volumes to the cilium agent +cilium_agent_extra_volumes: [] +cilium_agent_extra_volume_mounts: [] + +cilium_agent_extra_env_vars: [] + +cilium_operator_replicas: 2 + +# The address at which the cillium operator bind health check api +cilium_operator_api_serve_addr: "127.0.0.1:9234" + +## A dictionary of extra config variables to add to cilium-config, formatted like: +## cilium_config_extra_vars: +## var1: "value1" +## var2: "value2" +cilium_config_extra_vars: {} + +# For adding and mounting extra volumes to the cilium operator +cilium_operator_extra_volumes: [] +cilium_operator_extra_volume_mounts: [] + +# Extra arguments for the Cilium Operator +cilium_operator_custom_args: [] + +# Name of the cluster. Only relevant when building a mesh of clusters. +cilium_cluster_name: default + +# Make Cilium take ownership over the `/etc/cni/net.d` directory on the node, renaming all non-Cilium CNI configurations to `*.cilium_bak`. +# This ensures no Pods can be scheduled using other CNI plugins during Cilium agent downtime. +# Available for Cilium v1.10 and up. +cilium_cni_exclusive: true + +# Configure the log file for CNI logging with retention policy of 7 days. +# Disable CNI file logging by setting this field to empty explicitly. +# Available for Cilium v1.12 and up. +cilium_cni_log_file: "/var/run/cilium/cilium-cni.log" + +# -- Configure cgroup related configuration +# -- Enable auto mount of cgroup2 filesystem. +# When `cilium_cgroup_auto_mount` is enabled, cgroup2 filesystem is mounted at +# `cilium_cgroup_host_root` path on the underlying host and inside the cilium agent pod. +# If users disable `cilium_cgroup_auto_mount`, it's expected that users have mounted +# cgroup2 filesystem at the specified `cilium_cgroup_auto_mount` volume, and then the +# volume will be mounted inside the cilium agent pod at the same path. +# Available for Cilium v1.11 and up +cilium_cgroup_auto_mount: true +# -- Configure cgroup root where cgroup2 filesystem is mounted on the host +cilium_cgroup_host_root: "/run/cilium/cgroupv2" + +# Specifies the ratio (0.0-1.0) of total system memory to use for dynamic +# sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps. +cilium_bpf_map_dynamic_size_ratio: "0.0025" + +# -- Enables masquerading of IPv4 traffic leaving the node from endpoints. +# Available for Cilium v1.10 and up +cilium_enable_ipv4_masquerade: true +# -- Enables masquerading of IPv6 traffic leaving the node from endpoints. +# Available for Cilium v1.10 and up +cilium_enable_ipv6_masquerade: true + +# -- Enable native IP masquerade support in eBPF +cilium_enable_bpf_masquerade: false + +# -- Configure whether direct routing mode should route traffic via +# host stack (true) or directly and more efficiently out of BPF (false) if +# the kernel supports it. The latter has the implication that it will also +# bypass netfilter in the host namespace. +cilium_enable_host_legacy_routing: true + +# -- Enable use of the remote node identity. +# ref: https://docs.cilium.io/en/v1.7/install/upgrade/#configmap-remote-node-identity +cilium_enable_remote_node_identity: true + +# -- Enable the use of well-known identities. +cilium_enable_well_known_identities: false + +# The monitor aggregation flags determine which TCP flags which, upon the +# first observation, cause monitor notifications to be generated. +# +# Only effective when monitor aggregation is set to "medium" or higher. +cilium_monitor_aggregation_flags: "all" + +cilium_enable_bpf_clock_probe: true + +# -- Whether to enable CNP status updates. +cilium_disable_cnp_status_updates: true + +# Configure how long to wait for the Cilium DaemonSet to be ready again +cilium_rolling_restart_wait_retries_count: 30 +cilium_rolling_restart_wait_retries_delay_seconds: 10 + +# Cilium changed the default metrics exporter ports in 1.12 +cilium_agent_scrape_port: "{{ cilium_version | regex_replace('v') is version('1.12', '>=') | ternary('9962', '9090') }}" +cilium_operator_scrape_port: "{{ cilium_version | regex_replace('v') is version('1.12', '>=') | ternary('9963', '6942') }}" +cilium_hubble_scrape_port: "{{ cilium_version | regex_replace('v') is version('1.12', '>=') | ternary('9965', '9091') }}" + +# Cilium certgen args for generate certificate for hubble mTLS +cilium_certgen_args: + cilium-namespace: kube-system + ca-reuse-secret: true + ca-secret-name: hubble-ca-secret + ca-generate: true + ca-validity-duration: 94608000s + hubble-server-cert-generate: true + hubble-server-cert-common-name: '*.{{ cilium_cluster_name }}.hubble-grpc.cilium.io' + hubble-server-cert-validity-duration: 94608000s + hubble-server-cert-secret-name: hubble-server-certs + hubble-relay-client-cert-generate: true + hubble-relay-client-cert-common-name: '*.{{ cilium_cluster_name }}.hubble-grpc.cilium.io' + hubble-relay-client-cert-validity-duration: 94608000s + hubble-relay-client-cert-secret-name: hubble-relay-client-certs + hubble-relay-server-cert-generate: false + +# A list of extra rules variables to add to clusterrole for cilium operator, formatted like: +# cilium_clusterrole_rules_operator_extra_vars: +# - apiGroups: +# - '""' +# resources: +# - pods +# verbs: +# - delete +# - apiGroups: +# - '""' +# resources: +# - nodes +# verbs: +# - list +# - watch +# resourceNames: +# - toto +cilium_clusterrole_rules_operator_extra_vars: [] diff --git a/kubespray/project/roles/network_plugin/cilium/tasks/apply.yml b/kubespray/project/roles/network_plugin/cilium/tasks/apply.yml new file mode 100644 index 0000000..75868ba --- /dev/null +++ b/kubespray/project/roles/network_plugin/cilium/tasks/apply.yml @@ -0,0 +1,33 @@ +--- +- name: Cilium | Start Resources + kube: + name: "{{ item.item.name }}" + namespace: "kube-system" + kubectl: "{{ bin_dir }}/kubectl" + resource: "{{ item.item.type }}" + filename: "{{ kube_config_dir }}/{{ item.item.name }}-{{ item.item.file }}" + state: "latest" + loop: "{{ cilium_node_manifests.results }}" + when: inventory_hostname == groups['kube_control_plane'][0] and not item is skipped + +- name: Cilium | Wait for pods to run + command: "{{ kubectl }} -n kube-system get pods -l k8s-app=cilium -o jsonpath='{.items[?(@.status.containerStatuses[0].ready==false)].metadata.name}'" # noqa literal-compare + register: pods_not_ready + until: pods_not_ready.stdout.find("cilium")==-1 + retries: "{{ cilium_rolling_restart_wait_retries_count | int }}" + delay: "{{ cilium_rolling_restart_wait_retries_delay_seconds | int }}" + failed_when: false + when: inventory_hostname == groups['kube_control_plane'][0] + +- name: Cilium | Hubble install + kube: + name: "{{ item.item.name }}" + namespace: "kube-system" + kubectl: "{{ bin_dir }}/kubectl" + resource: "{{ item.item.type }}" + filename: "{{ kube_config_dir }}/addons/hubble/{{ item.item.name }}-{{ item.item.file }}" + state: "latest" + loop: "{{ cilium_hubble_manifests.results }}" + when: + - inventory_hostname == groups['kube_control_plane'][0] and not item is skipped + - cilium_enable_hubble and cilium_hubble_install diff --git a/kubespray/project/roles/network_plugin/cilium/tasks/check.yml b/kubespray/project/roles/network_plugin/cilium/tasks/check.yml new file mode 100644 index 0000000..c65591f --- /dev/null +++ b/kubespray/project/roles/network_plugin/cilium/tasks/check.yml @@ -0,0 +1,63 @@ +--- +- name: Cilium | Check Cilium encryption `cilium_ipsec_key` for ipsec + assert: + that: + - "cilium_ipsec_key is defined" + msg: "cilium_ipsec_key should be defined to enable encryption using ipsec" + when: + - cilium_encryption_enabled + - cilium_encryption_type == "ipsec" + - cilium_tunnel_mode in ['vxlan'] + +# TODO: Clean this task up when we drop backward compatibility support for `cilium_ipsec_enabled` +- name: Stop if `cilium_ipsec_enabled` is defined and `cilium_encryption_type` is not `ipsec` + assert: + that: cilium_encryption_type == 'ipsec' + msg: > + It is not possible to use `cilium_ipsec_enabled` when `cilium_encryption_type` is set to {{ cilium_encryption_type }}. + when: + - cilium_ipsec_enabled is defined + - cilium_ipsec_enabled + - kube_network_plugin == 'cilium' or cilium_deploy_additionally | default(false) | bool + +- name: Stop if kernel version is too low for Cilium Wireguard encryption + assert: + that: ansible_kernel.split('-')[0] is version('5.6.0', '>=') + when: + - kube_network_plugin == 'cilium' or cilium_deploy_additionally | default(false) | bool + - cilium_encryption_enabled + - cilium_encryption_type == "wireguard" + - not ignore_assert_errors + +- name: Stop if bad Cilium identity allocation mode + assert: + that: cilium_identity_allocation_mode in ['crd', 'kvstore'] + msg: "cilium_identity_allocation_mode must be either 'crd' or 'kvstore'" + +- name: Stop if bad Cilium Cluster ID + assert: + that: + - cilium_cluster_id <= 255 + - cilium_cluster_id >= 0 + msg: "'cilium_cluster_id' must be between 1 and 255" + when: cilium_cluster_id is defined + +- name: Stop if bad encryption type + assert: + that: cilium_encryption_type in ['ipsec', 'wireguard'] + msg: "cilium_encryption_type must be either 'ipsec' or 'wireguard'" + when: cilium_encryption_enabled + +- name: Stop if cilium_version is < v1.10.0 + assert: + that: cilium_version | regex_replace('v') is version(cilium_min_version_required, '>=') + msg: "cilium_version is too low. Minimum version {{ cilium_min_version_required }}" + +# TODO: Clean this task up when we drop backward compatibility support for `cilium_ipsec_enabled` +- name: Set `cilium_encryption_type` to "ipsec" and if `cilium_ipsec_enabled` is true + set_fact: + cilium_encryption_type: ipsec + cilium_encryption_enabled: true + when: + - cilium_ipsec_enabled is defined + - cilium_ipsec_enabled diff --git a/kubespray/project/roles/network_plugin/cilium/tasks/install.yml b/kubespray/project/roles/network_plugin/cilium/tasks/install.yml new file mode 100644 index 0000000..d531d72 --- /dev/null +++ b/kubespray/project/roles/network_plugin/cilium/tasks/install.yml @@ -0,0 +1,97 @@ +--- +- name: Cilium | Ensure BPFFS mounted + ansible.posix.mount: + fstype: bpf + path: /sys/fs/bpf + src: bpffs + state: mounted + +- name: Cilium | Create Cilium certs directory + file: + dest: "{{ cilium_cert_dir }}" + state: directory + mode: 0750 + owner: root + group: root + when: + - cilium_identity_allocation_mode == "kvstore" + +- name: Cilium | Link etcd certificates for cilium + file: + src: "{{ etcd_cert_dir }}/{{ item.s }}" + dest: "{{ cilium_cert_dir }}/{{ item.d }}" + mode: 0644 + state: hard + force: yes + loop: + - {s: "{{ kube_etcd_cacert_file }}", d: "ca_cert.crt"} + - {s: "{{ kube_etcd_cert_file }}", d: "cert.crt"} + - {s: "{{ kube_etcd_key_file }}", d: "key.pem"} + when: + - cilium_identity_allocation_mode == "kvstore" + +- name: Cilium | Create hubble dir + file: + path: "{{ kube_config_dir }}/addons/hubble" + state: directory + owner: root + group: root + mode: 0755 + when: + - inventory_hostname == groups['kube_control_plane'][0] + - cilium_hubble_install + +- name: Cilium | Create Cilium node manifests + template: + src: "{{ item.name }}/{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/{{ item.name }}-{{ item.file }}" + mode: 0644 + loop: + - {name: cilium, file: config.yml, type: cm} + - {name: cilium-operator, file: crb.yml, type: clusterrolebinding} + - {name: cilium-operator, file: cr.yml, type: clusterrole} + - {name: cilium, file: crb.yml, type: clusterrolebinding} + - {name: cilium, file: cr.yml, type: clusterrole} + - {name: cilium, file: secret.yml, type: secret, when: "{{ cilium_encryption_enabled and cilium_encryption_type == 'ipsec' }}"} + - {name: cilium, file: ds.yml, type: ds} + - {name: cilium-operator, file: deploy.yml, type: deploy} + - {name: cilium-operator, file: sa.yml, type: sa} + - {name: cilium, file: sa.yml, type: sa} + register: cilium_node_manifests + when: + - inventory_hostname in groups['kube_control_plane'] + - item.when | default(True) | bool + +- name: Cilium | Create Cilium Hubble manifests + template: + src: "{{ item.name }}/{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/addons/hubble/{{ item.name }}-{{ item.file }}" + mode: 0644 + loop: + - {name: hubble, file: config.yml, type: cm} + - {name: hubble, file: crb.yml, type: clusterrolebinding} + - {name: hubble, file: cr.yml, type: clusterrole} + - {name: hubble, file: cronjob.yml, type: cronjob, when: "{{ cilium_hubble_tls_generate }}"} + - {name: hubble, file: deploy.yml, type: deploy} + - {name: hubble, file: job.yml, type: job, when: "{{ cilium_hubble_tls_generate }}"} + - {name: hubble, file: sa.yml, type: sa} + - {name: hubble, file: service.yml, type: service} + register: cilium_hubble_manifests + when: + - inventory_hostname == groups['kube_control_plane'][0] + - cilium_enable_hubble and cilium_hubble_install + - item.when | default(True) | bool + +- name: Cilium | Enable portmap addon + template: + src: 000-cilium-portmap.conflist.j2 + dest: /etc/cni/net.d/000-cilium-portmap.conflist + mode: 0644 + when: cilium_enable_portmap + +- name: Cilium | Copy Ciliumcli binary from download dir + copy: + src: "{{ local_release_dir }}/cilium" + dest: "{{ bin_dir }}/cilium" + mode: 0755 + remote_src: yes diff --git a/kubespray/project/roles/network_plugin/cilium/tasks/main.yml b/kubespray/project/roles/network_plugin/cilium/tasks/main.yml new file mode 100644 index 0000000..8123c5a --- /dev/null +++ b/kubespray/project/roles/network_plugin/cilium/tasks/main.yml @@ -0,0 +1,9 @@ +--- +- name: Cilium check + import_tasks: check.yml + +- name: Cilium install + include_tasks: install.yml + +- name: Cilium apply + include_tasks: apply.yml diff --git a/kubespray/project/roles/network_plugin/cilium/tasks/reset.yml b/kubespray/project/roles/network_plugin/cilium/tasks/reset.yml new file mode 100644 index 0000000..b578b07 --- /dev/null +++ b/kubespray/project/roles/network_plugin/cilium/tasks/reset.yml @@ -0,0 +1,9 @@ +--- +- name: Reset | check and remove devices if still present + include_tasks: reset_iface.yml + vars: + iface: "{{ item }}" + loop: + - cilium_host + - cilium_net + - cilium_vxlan diff --git a/kubespray/project/roles/network_plugin/cilium/tasks/reset_iface.yml b/kubespray/project/roles/network_plugin/cilium/tasks/reset_iface.yml new file mode 100644 index 0000000..e2f7c14 --- /dev/null +++ b/kubespray/project/roles/network_plugin/cilium/tasks/reset_iface.yml @@ -0,0 +1,12 @@ +--- +- name: "Reset | check if network device {{ iface }} is present" + stat: + path: "/sys/class/net/{{ iface }}" + get_attributes: no + get_checksum: no + get_mime: no + register: device_remains + +- name: "Reset | remove network device {{ iface }}" + command: "ip link del {{ iface }}" + when: device_remains.stat.exists diff --git a/kubespray/project/roles/network_plugin/cilium/templates/000-cilium-portmap.conflist.j2 b/kubespray/project/roles/network_plugin/cilium/templates/000-cilium-portmap.conflist.j2 new file mode 100644 index 0000000..982a7c9 --- /dev/null +++ b/kubespray/project/roles/network_plugin/cilium/templates/000-cilium-portmap.conflist.j2 @@ -0,0 +1,13 @@ +{ + "cniVersion": "0.3.1", + "name": "cilium-portmap", + "plugins": [ + { + "type": "cilium-cni" + }, + { + "type": "portmap", + "capabilities": { "portMappings": true } + } + ] +} diff --git a/kubespray/project/roles/network_plugin/cilium/templates/cilium-operator/cr.yml.j2 b/kubespray/project/roles/network_plugin/cilium/templates/cilium-operator/cr.yml.j2 new file mode 100644 index 0000000..642a667 --- /dev/null +++ b/kubespray/project/roles/network_plugin/cilium/templates/cilium-operator/cr.yml.j2 @@ -0,0 +1,169 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cilium-operator +rules: +- apiGroups: + - "" + resources: + # to automatically delete [core|kube]dns pods so that are starting to being + # managed by Cilium + - pods + verbs: + - get + - list + - watch + - delete +- apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch +- apiGroups: + - "" + resources: + # To remove node taints + - nodes + # To set NetworkUnavailable false on startup + - nodes/status + verbs: + - patch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + # to perform LB IP allocation for BGP + - services/status + verbs: + - update + - patch +- apiGroups: + - "" + resources: + # to perform the translation of a CNP that contains `ToGroup` to its endpoints + - services + - endpoints + # to check apiserver connectivity + - namespaces + verbs: + - get + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumnetworkpolicies + - ciliumnetworkpolicies/status + - ciliumnetworkpolicies/finalizers + - ciliumclusterwidenetworkpolicies + - ciliumclusterwidenetworkpolicies/status + - ciliumclusterwidenetworkpolicies/finalizers + - ciliumendpoints + - ciliumendpoints/status + - ciliumendpoints/finalizers + - ciliumnodes + - ciliumnodes/status + - ciliumnodes/finalizers + - ciliumidentities + - ciliumidentities/status + - ciliumidentities/finalizers + - ciliumlocalredirectpolicies + - ciliumlocalredirectpolicies/status + - ciliumlocalredirectpolicies/finalizers +{% if cilium_version | regex_replace('v') is version('1.11', '>=') %} + - ciliumendpointslices +{% endif %} +{% if cilium_version | regex_replace('v') is version('1.12', '>=') %} + - ciliumbgploadbalancerippools + - ciliumloadbalancerippools + - ciliumloadbalancerippools/status + - ciliumbgppeeringpolicies + - ciliumenvoyconfigs +{% endif %} + verbs: + - '*' +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - create + - get + - list + - update + - watch +# For cilium-operator running in HA mode. +# +# Cilium operator running in HA mode requires the use of ResourceLock for Leader Election +# between multiple running instances. +# The preferred way of doing this is to use LeasesResourceLock as edits to Leases are less +# common and fewer objects in the cluster watch "all Leases". +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - update +{% if cilium_version | regex_replace('v') is version('1.12', '>=') %} +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - update + resourceNames: + - ciliumbgploadbalancerippools.cilium.io + - ciliumbgppeeringpolicies.cilium.io + - ciliumclusterwideenvoyconfigs.cilium.io + - ciliumclusterwidenetworkpolicies.cilium.io + - ciliumegressgatewaypolicies.cilium.io + - ciliumegressnatpolicies.cilium.io + - ciliumendpoints.cilium.io + - ciliumendpointslices.cilium.io + - ciliumenvoyconfigs.cilium.io + - ciliumexternalworkloads.cilium.io + - ciliumidentities.cilium.io + - ciliumlocalredirectpolicies.cilium.io + - ciliumnetworkpolicies.cilium.io + - ciliumnodes.cilium.io +{% endif %} +{% for rules in cilium_clusterrole_rules_operator_extra_vars %} +- apiGroups: +{% for api in rules['apiGroups'] %} + - {{ api }} +{% endfor %} + resources: +{% for resource in rules['resources'] %} + - {{ resource }} +{% endfor %} + verbs: +{% for verb in rules['verbs'] %} + - {{ verb }} +{% endfor %} +{% if 'resourceNames' in rules %} + resourceNames: +{% for resourceName in rules['resourceNames'] %} + - {{ resourceName }} +{% endfor %} +{% endif %} +{% endfor %} diff --git a/kubespray/project/roles/network_plugin/cilium/templates/cilium-operator/crb.yml.j2 b/kubespray/project/roles/network_plugin/cilium/templates/cilium-operator/crb.yml.j2 new file mode 100644 index 0000000..00f0835 --- /dev/null +++ b/kubespray/project/roles/network_plugin/cilium/templates/cilium-operator/crb.yml.j2 @@ -0,0 +1,13 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cilium-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cilium-operator +subjects: +- kind: ServiceAccount + name: cilium-operator + namespace: kube-system diff --git a/kubespray/project/roles/network_plugin/cilium/templates/cilium-operator/deploy.yml.j2 b/kubespray/project/roles/network_plugin/cilium/templates/cilium-operator/deploy.yml.j2 new file mode 100644 index 0000000..1418965 --- /dev/null +++ b/kubespray/project/roles/network_plugin/cilium/templates/cilium-operator/deploy.yml.j2 @@ -0,0 +1,170 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cilium-operator + namespace: kube-system + labels: + io.cilium/app: operator + name: cilium-operator +spec: +{% if groups.k8s_cluster | length == 1 %} + replicas: 1 +{% else %} + replicas: {{ cilium_operator_replicas }} +{% endif %} + selector: + matchLabels: + io.cilium/app: operator + name: cilium-operator + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: +{% if cilium_enable_prometheus %} + annotations: + prometheus.io/port: "{{ cilium_operator_scrape_port }}" + prometheus.io/scrape: "true" +{% endif %} + labels: + io.cilium/app: operator + name: cilium-operator + spec: + containers: + - name: cilium-operator + image: "{{ cilium_operator_image_repo }}:{{ cilium_operator_image_tag }}" + imagePullPolicy: {{ k8s_image_pull_policy }} + command: + - cilium-operator + args: + - --config-dir=/tmp/cilium/config-map + - --debug=$(CILIUM_DEBUG) +{% if cilium_operator_custom_args is string %} + - {{ cilium_operator_custom_args }} +{% else %} +{% for flag in cilium_operator_custom_args %} + - {{ flag }} +{% endfor %} +{% endif %} + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: CILIUM_DEBUG + valueFrom: + configMapKeyRef: + key: debug + name: cilium-config + optional: true + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: cilium-aws + key: AWS_ACCESS_KEY_ID + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: cilium-aws + key: AWS_SECRET_ACCESS_KEY + optional: true + - name: AWS_DEFAULT_REGION + valueFrom: + secretKeyRef: + name: cilium-aws + key: AWS_DEFAULT_REGION + optional: true +{% if cilium_kube_proxy_replacement == 'strict' %} + - name: KUBERNETES_SERVICE_HOST + value: "{{ kube_apiserver_global_endpoint | urlsplit('hostname') }}" + - name: KUBERNETES_SERVICE_PORT + value: "{{ kube_apiserver_global_endpoint | urlsplit('port') }}" +{% endif %} +{% if cilium_enable_prometheus %} + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + ports: + - name: prometheus + containerPort: {{ cilium_operator_scrape_port }} + hostPort: {{ cilium_operator_scrape_port }} + protocol: TCP +{% endif %} + livenessProbe: + httpGet: +{% if cilium_enable_ipv4 %} + host: 127.0.0.1 +{% else %} + host: '::1' +{% endif %} + path: /healthz + port: 9234 + scheme: HTTP + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 3 + volumeMounts: + - name: cilium-config-path + mountPath: /tmp/cilium/config-map + readOnly: true +{% if cilium_identity_allocation_mode == "kvstore" %} + - name: etcd-config-path + mountPath: /var/lib/etcd-config + readOnly: true + - name: etcd-secrets + mountPath: "{{ cilium_cert_dir }}" + readOnly: true +{% endif %} +{% for volume_mount in cilium_operator_extra_volume_mounts %} + - {{ volume_mount | to_nice_yaml(indent=2) | indent(14) }} +{% endfor %} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + restartPolicy: Always + priorityClassName: system-node-critical + serviceAccount: cilium-operator + serviceAccountName: cilium-operator + # In HA mode, cilium-operator pods must not be scheduled on the same + # node as they will clash with each other. + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + io.cilium/app: operator + tolerations: + - operator: Exists + volumes: + - name: cilium-config-path + configMap: + name: cilium-config +{% if cilium_identity_allocation_mode == "kvstore" %} + # To read the etcd config stored in config maps + - name: etcd-config-path + configMap: + name: cilium-config + defaultMode: 420 + items: + - key: etcd-config + path: etcd.config + # To read the k8s etcd secrets in case the user might want to use TLS + - name: etcd-secrets + hostPath: + path: "{{ cilium_cert_dir }}" +{% endif %} +{% for volume in cilium_operator_extra_volumes %} + - {{ volume | to_nice_yaml(indent=2) | indent(10) }} +{% endfor %} diff --git a/kubespray/project/roles/network_plugin/cilium/templates/cilium-operator/sa.yml.j2 b/kubespray/project/roles/network_plugin/cilium/templates/cilium-operator/sa.yml.j2 new file mode 100644 index 0000000..c5d1893 --- /dev/null +++ b/kubespray/project/roles/network_plugin/cilium/templates/cilium-operator/sa.yml.j2 @@ -0,0 +1,6 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: cilium-operator + namespace: kube-system diff --git a/kubespray/project/roles/network_plugin/cilium/templates/cilium/config.yml.j2 b/kubespray/project/roles/network_plugin/cilium/templates/cilium/config.yml.j2 new file mode 100644 index 0000000..9fb5f28 --- /dev/null +++ b/kubespray/project/roles/network_plugin/cilium/templates/cilium/config.yml.j2 @@ -0,0 +1,263 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: cilium-config + namespace: kube-system +data: + identity-allocation-mode: {{ cilium_identity_allocation_mode }} + +{% if cilium_identity_allocation_mode == "kvstore" %} + # This etcd-config contains the etcd endpoints of your cluster. If you use + # TLS please make sure you follow the tutorial in https://cilium.link/etcd-config + etcd-config: |- + --- + endpoints: +{% for ip_addr in etcd_access_addresses.split(',') %} + - {{ ip_addr }} +{% endfor %} + + # In case you want to use TLS in etcd, uncomment the 'ca-file' line + # and create a kubernetes secret by following the tutorial in + # https://cilium.link/etcd-config + ca-file: "{{ cilium_cert_dir }}/ca_cert.crt" + + # In case you want client to server authentication, uncomment the following + # lines and create a kubernetes secret by following the tutorial in + # https://cilium.link/etcd-config + key-file: "{{ cilium_cert_dir }}/key.pem" + cert-file: "{{ cilium_cert_dir }}/cert.crt" + + # kvstore + # https://docs.cilium.io/en/latest/cmdref/kvstore/ + kvstore: etcd + kvstore-opt: '{"etcd.config": "/var/lib/etcd-config/etcd.config"}' +{% endif %} + + # If you want metrics enabled in all of your Cilium agents, set the port for + # which the Cilium agents will have their metrics exposed. + # This option deprecates the "prometheus-serve-addr" in the + # "cilium-metrics-config" ConfigMap + # NOTE that this will open the port on ALL nodes where Cilium pods are + # scheduled. +{% if cilium_enable_prometheus %} + prometheus-serve-addr: ":{{ cilium_agent_scrape_port }}" + operator-prometheus-serve-addr: ":{{ cilium_operator_scrape_port }}" + enable-metrics: "true" +{% endif %} + + # If you want to run cilium in debug mode change this value to true + debug: "{{ cilium_debug }}" + enable-ipv4: "{{ cilium_enable_ipv4 }}" + enable-ipv6: "{{ cilium_enable_ipv6 }}" + # If a serious issue occurs during Cilium startup, this + # invasive option may be set to true to remove all persistent + # state. Endpoints will not be restored using knowledge from a + # prior Cilium run, so they may receive new IP addresses upon + # restart. This also triggers clean-cilium-bpf-state. + clean-cilium-state: "false" + # If you want to clean cilium BPF state, set this to true; + # Removes all BPF maps from the filesystem. Upon restart, + # endpoints are restored with the same IP addresses, however + # any ongoing connections may be disrupted briefly. + # Loadbalancing decisions will be reset, so any ongoing + # connections via a service may be loadbalanced to a different + # backend after restart. + clean-cilium-bpf-state: "false" + + # Users who wish to specify their own custom CNI configuration file must set + # custom-cni-conf to "true", otherwise Cilium may overwrite the configuration. + custom-cni-conf: "false" + + # If you want cilium monitor to aggregate tracing for packets, set this level + # to "low", "medium", or "maximum". The higher the level, the less packets + # that will be seen in monitor output. + monitor-aggregation: "{{ cilium_monitor_aggregation }}" + + # ct-global-max-entries-* specifies the maximum number of connections + # supported across all endpoints, split by protocol: tcp or other. One pair + # of maps uses these values for IPv4 connections, and another pair of maps + # use these values for IPv6 connections. + # + # If these values are modified, then during the next Cilium startup the + # tracking of ongoing connections may be disrupted. This may lead to brief + # policy drops or a change in loadbalancing decisions for a connection. + # + # For users upgrading from Cilium 1.2 or earlier, to minimize disruption + # during the upgrade process, comment out these options. + bpf-ct-global-tcp-max: "524288" + bpf-ct-global-any-max: "262144" + + # Pre-allocation of map entries allows per-packet latency to be reduced, at + # the expense of up-front memory allocation for the entries in the maps. The + # default value below will minimize memory usage in the default installation; + # users who are sensitive to latency may consider setting this to "true". + # + # This option was introduced in Cilium 1.4. Cilium 1.3 and earlier ignore + # this option and behave as though it is set to "true". + # + # If this value is modified, then during the next Cilium startup the restore + # of existing endpoints and tracking of ongoing connections may be disrupted. + # This may lead to policy drops or a change in loadbalancing decisions for a + # connection for some time. Endpoints may need to be recreated to restore + # connectivity. + # + # If this option is set to "false" during an upgrade from 1.3 or earlier to + # 1.4 or later, then it may cause one-time disruptions during the upgrade. + preallocate-bpf-maps: "{{ cilium_preallocate_bpf_maps }}" + + # Regular expression matching compatible Istio sidecar istio-proxy + # container image names + sidecar-istio-proxy-image: "cilium/istio_proxy" + + # Encapsulation mode for communication between nodes + # Possible values: + # - disabled + # - vxlan (default) + # - geneve +{% if cilium_version | regex_replace('v') is version('1.14.0', '<') %} + tunnel: "{{ cilium_tunnel_mode }}" +{% elif cilium_version | regex_replace('v') is version('1.14.0', '>=') and cilium_tunnel_mode == 'disabled' %} + routing-mode: 'native' +{% elif cilium_version | regex_replace('v') is version('1.14.0', '>=') and cilium_tunnel_mode != 'disabled' %} + routing-mode: 'tunnel' + tunnel-protocol: "{{ cilium_tunnel_mode }}" +{% endif %} + + # Enable Bandwidth Manager + # Cilium’s bandwidth manager supports the kubernetes.io/egress-bandwidth Pod annotation. + # Bandwidth enforcement currently does not work in combination with L7 Cilium Network Policies. + # In case they select the Pod at egress, then the bandwidth enforcement will be disabled for those Pods. + # Bandwidth Manager requires a v5.1.x or more recent Linux kernel. +{% if cilium_enable_bandwidth_manager %} + enable-bandwidth-manager: "true" +{% endif %} + + # Name of the cluster. Only relevant when building a mesh of clusters. + cluster-name: "{{ cilium_cluster_name }}" + + # Unique ID of the cluster. Must be unique across all conneted clusters and + # in the range of 1 and 255. Only relevant when building a mesh of clusters. + #cluster-id: 1 +{% if cilium_cluster_id is defined %} + cluster-id: "{{ cilium_cluster_id }}" +{% endif %} + +# `wait-bpf-mount` is removed after v1.10.4 +# https://github.com/cilium/cilium/commit/d2217045cb3726a7f823174e086913b69b8090da +{% if cilium_version | regex_replace('v') is version('1.10.4', '<') %} + # wait-bpf-mount makes init container wait until bpf filesystem is mounted + wait-bpf-mount: "false" +{% endif %} + + kube-proxy-replacement: "{{ cilium_kube_proxy_replacement }}" + +# `native-routing-cidr` is deprecated in 1.10, removed in 1.12. +# Replaced by `ipv4-native-routing-cidr` +# https://github.com/cilium/cilium/pull/16695 +{% if cilium_version | regex_replace('v') is version('1.12', '<') %} + native-routing-cidr: "{{ cilium_native_routing_cidr }}" +{% else %} +{% if cilium_native_routing_cidr | length %} + ipv4-native-routing-cidr: "{{ cilium_native_routing_cidr }}" +{% endif %} +{% if cilium_native_routing_cidr_ipv6 | length %} + ipv6-native-routing-cidr: "{{ cilium_native_routing_cidr_ipv6 }}" +{% endif %} +{% endif %} + + auto-direct-node-routes: "{{ cilium_auto_direct_node_routes }}" + + operator-api-serve-addr: "{{ cilium_operator_api_serve_addr }}" + + # Hubble settings +{% if cilium_enable_hubble %} + enable-hubble: "true" +{% if cilium_enable_hubble_metrics %} + hubble-metrics-server: ":{{ cilium_hubble_scrape_port }}" + hubble-metrics: +{% for hubble_metrics_cycle in cilium_hubble_metrics %} + {{ hubble_metrics_cycle }} +{% endfor %} +{% endif %} + hubble-listen-address: ":4244" +{% if cilium_enable_hubble and cilium_hubble_install %} + hubble-disable-tls: "{% if cilium_hubble_tls_generate %}false{% else %}true{% endif %}" + hubble-tls-cert-file: /var/lib/cilium/tls/hubble/server.crt + hubble-tls-key-file: /var/lib/cilium/tls/hubble/server.key + hubble-tls-client-ca-files: /var/lib/cilium/tls/hubble/client-ca.crt +{% endif %} +{% endif %} + + # IP Masquerade Agent + enable-ip-masq-agent: "{{ cilium_ip_masq_agent_enable }}" + +{% for key, value in cilium_config_extra_vars.items() %} + {{ key }}: "{{ value }}" +{% endfor %} + + # Enable transparent network encryption +{% if cilium_encryption_enabled %} +{% if cilium_encryption_type == "ipsec" %} + enable-ipsec: "true" + ipsec-key-file: /etc/ipsec/keys + encrypt-node: "{{ cilium_ipsec_node_encryption }}" +{% endif %} + +{% if cilium_encryption_type == "wireguard" %} + enable-wireguard: "true" + enable-wireguard-userspace-fallback: "{{ cilium_wireguard_userspace_fallback }}" +{% endif %} +{% endif %} + + # IPAM settings + ipam: "{{ cilium_ipam_mode }}" +{% if cilium_ipam_mode == "cluster-pool" %} + cluster-pool-ipv4-cidr: "{{ cilium_pool_cidr | default(kube_pods_subnet) }}" + cluster-pool-ipv4-mask-size: "{{ cilium_pool_mask_size | default(kube_network_node_prefix) }}" +{% if cilium_enable_ipv6 %} + cluster-pool-ipv6-cidr: "{{ cilium_pool_cidr_ipv6 | default(kube_pods_subnet_ipv6) }}" + cluster-pool-ipv6-mask-size: "{{ cilium_pool_mask_size_ipv6 | default(kube_network_node_prefix_ipv6) }}" +{% endif %} +{% endif %} + + agent-health-port: "{{ cilium_agent_health_port }}" + +{% if cilium_version | regex_replace('v') is version('1.11', '>=') and cilium_cgroup_host_root != '' %} + cgroup-root: "{{ cilium_cgroup_host_root }}" +{% endif %} + + bpf-map-dynamic-size-ratio: "{{ cilium_bpf_map_dynamic_size_ratio }}" + + enable-ipv4-masquerade: "{{ cilium_enable_ipv4_masquerade }}" + enable-ipv6-masquerade: "{{ cilium_enable_ipv6_masquerade }}" + + enable-bpf-masquerade: "{{ cilium_enable_bpf_masquerade }}" + + enable-host-legacy-routing: "{{ cilium_enable_host_legacy_routing }}" + + enable-remote-node-identity: "{{ cilium_enable_remote_node_identity }}" + + enable-well-known-identities: "{{ cilium_enable_well_known_identities }}" + + monitor-aggregation-flags: "{{ cilium_monitor_aggregation_flags }}" + + enable-bpf-clock-probe: "{{ cilium_enable_bpf_clock_probe }}" + + disable-cnp-status-updates: "{{ cilium_disable_cnp_status_updates }}" +{% if cilium_ip_masq_agent_enable %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ip-masq-agent + namespace: kube-system +data: + config: | + nonMasqueradeCIDRs: +{% for cidr in cilium_non_masquerade_cidrs %} + - {{ cidr }} +{% endfor %} + masqLinkLocal: {{ cilium_masq_link_local | bool }} + resyncInterval: "{{ cilium_ip_masq_resync_interval }}" +{% endif %} diff --git a/kubespray/project/roles/network_plugin/cilium/templates/cilium/cr.yml.j2 b/kubespray/project/roles/network_plugin/cilium/templates/cilium/cr.yml.j2 new file mode 100644 index 0000000..4ce747f --- /dev/null +++ b/kubespray/project/roles/network_plugin/cilium/templates/cilium/cr.yml.j2 @@ -0,0 +1,131 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cilium +rules: +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - get + - list + - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - namespaces + - services + - pods + - endpoints + - nodes + verbs: + - get + - list + - watch +{% if cilium_version | regex_replace('v') is version('1.12', '<') %} +- apiGroups: + - "" + resources: + - pods + - pods/finalizers + verbs: + - get + - list + - watch + - update + - delete +- apiGroups: + - "" + resources: + - pods + - nodes + verbs: + - get + - list + - watch + - update +{% endif %} +- apiGroups: + - "" + resources: + - nodes + - nodes/status + verbs: + - patch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + # Deprecated for removal in v1.10 + - create + - list + - watch + - update + + # This is used when validating policies in preflight. This will need to stay + # until we figure out how to avoid "get" inside the preflight, and then + # should be removed ideally. + - get +- apiGroups: + - cilium.io + resources: + - ciliumnetworkpolicies + - ciliumnetworkpolicies/status + - ciliumclusterwidenetworkpolicies + - ciliumclusterwidenetworkpolicies/status + - ciliumendpoints + - ciliumendpoints/status + - ciliumnodes + - ciliumnodes/status + - ciliumidentities + - ciliumlocalredirectpolicies + - ciliumlocalredirectpolicies/status + - ciliumegressnatpolicies +{% if cilium_version | regex_replace('v') is version('1.11', '>=') %} + - ciliumendpointslices +{% endif %} +{% if cilium_version | regex_replace('v') is version('1.12', '>=') %} + - ciliumbgploadbalancerippools + - ciliumbgppeeringpolicies +{% endif %} +{% if cilium_version | regex_replace('v') is version('1.11.5', '<') %} + - ciliumnetworkpolicies/finalizers + - ciliumclusterwidenetworkpolicies/finalizers + - ciliumendpoints/finalizers + - ciliumnodes/finalizers + - ciliumidentities/finalizers + - ciliumlocalredirectpolicies/finalizers +{% endif %} + verbs: + - '*' +{% if cilium_version | regex_replace('v') is version('1.12', '>=') %} +- apiGroups: + - cilium.io + resources: + - ciliumclusterwideenvoyconfigs + - ciliumenvoyconfigs + - ciliumegressgatewaypolicies + verbs: + - list + - watch +{% endif %} +{% if cilium_version | regex_replace('v') is version('1.14', '>=') %} +- apiGroups: + - cilium.io + resources: + - ciliumcidrgroups + verbs: + - list + - watch +{% endif %} diff --git a/kubespray/project/roles/network_plugin/cilium/templates/cilium/crb.yml.j2 b/kubespray/project/roles/network_plugin/cilium/templates/cilium/crb.yml.j2 new file mode 100644 index 0000000..d23897f --- /dev/null +++ b/kubespray/project/roles/network_plugin/cilium/templates/cilium/crb.yml.j2 @@ -0,0 +1,13 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cilium +subjects: +- kind: ServiceAccount + name: cilium + namespace: kube-system diff --git a/kubespray/project/roles/network_plugin/cilium/templates/cilium/ds.yml.j2 b/kubespray/project/roles/network_plugin/cilium/templates/cilium/ds.yml.j2 new file mode 100644 index 0000000..658f53b --- /dev/null +++ b/kubespray/project/roles/network_plugin/cilium/templates/cilium/ds.yml.j2 @@ -0,0 +1,446 @@ +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: cilium + namespace: kube-system + labels: + k8s-app: cilium +spec: + selector: + matchLabels: + k8s-app: cilium + updateStrategy: + rollingUpdate: + # Specifies the maximum number of Pods that can be unavailable during the update process. + maxUnavailable: 2 + type: RollingUpdate + template: + metadata: + annotations: +{% if cilium_enable_prometheus %} + prometheus.io/port: "{{ cilium_agent_scrape_port }}" + prometheus.io/scrape: "true" +{% endif %} + scheduler.alpha.kubernetes.io/tolerations: '[{"key":"dedicated","operator":"Equal","value":"master","effect":"NoSchedule"}]' + labels: + k8s-app: cilium + spec: + containers: + - name: cilium-agent + image: "{{ cilium_image_repo }}:{{ cilium_image_tag }}" + imagePullPolicy: {{ k8s_image_pull_policy }} + command: + - cilium-agent + args: + - --config-dir=/tmp/cilium/config-map +{% if cilium_mtu != "" %} + - --mtu={{ cilium_mtu }} +{% endif %} +{% if cilium_agent_custom_args is string %} + - {{ cilium_agent_custom_args }} +{% else %} +{% for flag in cilium_agent_custom_args %} + - {{ flag }} +{% endfor %} +{% endif %} + startupProbe: + httpGet: + host: '127.0.0.1' + path: /healthz + port: {{ cilium_agent_health_port }} + scheme: HTTP + httpHeaders: + - name: "brief" + value: "true" + failureThreshold: 105 + periodSeconds: 2 + successThreshold: 1 + livenessProbe: + httpGet: + host: '127.0.0.1' + path: /healthz + port: {{ cilium_agent_health_port }} + scheme: HTTP + httpHeaders: + - name: "brief" + value: "true" + failureThreshold: 10 + periodSeconds: 30 + successThreshold: 1 + timeoutSeconds: 5 + readinessProbe: + httpGet: + host: 127.0.0.1 + path: /healthz + port: {{ cilium_agent_health_port }} + scheme: HTTP + httpHeaders: + - name: "brief" + value: "true" + initialDelaySeconds: 5 + periodSeconds: 30 + successThreshold: 1 + failureThreshold: 3 + timeoutSeconds: 5 + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: CILIUM_CLUSTERMESH_CONFIG + value: /var/lib/cilium/clustermesh/ +{% if cilium_kube_proxy_replacement == 'strict' %} + - name: KUBERNETES_SERVICE_HOST + value: "{{ kube_apiserver_global_endpoint | urlsplit('hostname') }}" + - name: KUBERNETES_SERVICE_PORT + value: "{{ kube_apiserver_global_endpoint | urlsplit('port') }}" +{% endif %} +{% for env_var in cilium_agent_extra_env_vars %} + - {{ env_var | to_nice_yaml(indent=2) | indent(10) }} +{% endfor %} + lifecycle: +{% if cilium_version | regex_replace('v') is version('1.14', '<') %} + postStart: + exec: + command: + - "/cni-install.sh" + - "--cni-exclusive={{ cilium_cni_exclusive | string | lower }}" +{% if cilium_version | regex_replace('v') is version('1.12', '>=') %} + - "--enable-debug={{ cilium_debug | string | lower }}" + - "--log-file={{ cilium_cni_log_file }}" +{% endif %} +{% endif %} + preStop: + exec: + command: + - /cni-uninstall.sh + resources: + limits: + cpu: {{ cilium_cpu_limit }} + memory: {{ cilium_memory_limit }} + requests: + cpu: {{ cilium_cpu_requests }} + memory: {{ cilium_memory_requests }} +{% if cilium_enable_prometheus or cilium_enable_hubble_metrics %} + ports: +{% endif %} +{% if cilium_enable_prometheus %} + - name: prometheus + containerPort: {{ cilium_agent_scrape_port }} + hostPort: {{ cilium_agent_scrape_port }} + protocol: TCP +{% endif %} +{% if cilium_enable_hubble_metrics %} + - name: hubble-metrics + containerPort: {{ cilium_hubble_scrape_port }} + hostPort: {{ cilium_hubble_scrape_port }} + protocol: TCP +{% endif %} + securityContext: + privileged: true + volumeMounts: + - name: bpf-maps + mountPath: /sys/fs/bpf + mountPropagation: Bidirectional + - name: cilium-run + mountPath: /var/run/cilium +{% if cilium_version | regex_replace('v') is version('1.13.1', '<') %} + - name: cni-path + mountPath: /host/opt/cni/bin +{% endif %} + - name: etc-cni-netd + mountPath: /host/etc/cni/net.d +{% if cilium_identity_allocation_mode == "kvstore" %} + - name: etcd-config-path + mountPath: /var/lib/etcd-config + readOnly: true + - name: etcd-secrets + mountPath: "{{ cilium_cert_dir }}" + readOnly: true +{% endif %} + - name: clustermesh-secrets + mountPath: /var/lib/cilium/clustermesh + readOnly: true + - name: cilium-config-path + mountPath: /tmp/cilium/config-map + readOnly: true +{% if cilium_ip_masq_agent_enable %} + - name: ip-masq-agent + mountPath: /etc/config + readOnly: true +{% endif %} + # Needed to be able to load kernel modules + - name: lib-modules + mountPath: /lib/modules + readOnly: true + - name: xtables-lock + mountPath: /run/xtables.lock +{% if cilium_encryption_enabled and cilium_encryption_type == "ipsec" %} + - name: cilium-ipsec-secrets + mountPath: /etc/ipsec + readOnly: true +{% endif %} +{% if cilium_hubble_install %} + - name: hubble-tls + mountPath: /var/lib/cilium/tls/hubble + readOnly: true +{% endif %} +{% for volume_mount in cilium_agent_extra_volume_mounts %} + - {{ volume_mount | to_nice_yaml(indent=2) | indent(10) }} +{% endfor %} +# In managed etcd mode, Cilium must be able to resolve the DNS name of the etcd service +{% if cilium_identity_allocation_mode == "kvstore" %} + dnsPolicy: ClusterFirstWithHostNet +{% endif %} + hostNetwork: true + initContainers: +{% if cilium_version | regex_replace('v') is version('1.11', '>=') and cilium_cgroup_auto_mount %} + - name: mount-cgroup + image: "{{ cilium_image_repo }}:{{ cilium_image_tag }}" + imagePullPolicy: {{ k8s_image_pull_policy }} + env: + - name: CGROUP_ROOT + value: {{ cilium_cgroup_host_root }} + - name: BIN_PATH + value: /opt/cni/bin + command: + - sh + - -ec + # The statically linked Go program binary is invoked to avoid any + # dependency on utilities like sh and mount that can be missing on certain + # distros installed on the underlying host. Copy the binary to the + # same directory where we install cilium cni plugin so that exec permissions + # are available. + - | + cp /usr/bin/cilium-mount /hostbin/cilium-mount; + nsenter --cgroup=/hostproc/1/ns/cgroup --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-mount" $CGROUP_ROOT; + rm /hostbin/cilium-mount + volumeMounts: + - name: hostproc + mountPath: /hostproc + - name: cni-path + mountPath: /hostbin + securityContext: + privileged: true +{% endif %} +{% if cilium_version | regex_replace('v') is version('1.11.7', '>=') %} + - name: apply-sysctl-overwrites + image: "{{ cilium_image_repo }}:{{ cilium_image_tag }}" + imagePullPolicy: {{ k8s_image_pull_policy }} + env: + - name: BIN_PATH + value: /opt/cni/bin + command: + - sh + - -ec + # The statically linked Go program binary is invoked to avoid any + # dependency on utilities like sh that can be missing on certain + # distros installed on the underlying host. Copy the binary to the + # same directory where we install cilium cni plugin so that exec permissions + # are available. + - | + cp /usr/bin/cilium-sysctlfix /hostbin/cilium-sysctlfix; + nsenter --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-sysctlfix"; + rm /hostbin/cilium-sysctlfix + volumeMounts: + - name: hostproc + mountPath: /hostproc + - name: cni-path + mountPath: /hostbin + securityContext: + privileged: true +{% endif %} + - name: clean-cilium-state + image: "{{ cilium_image_repo }}:{{ cilium_image_tag }}" + imagePullPolicy: {{ k8s_image_pull_policy }} + command: + - /init-container.sh + env: + - name: CILIUM_ALL_STATE + valueFrom: + configMapKeyRef: + name: cilium-config + key: clean-cilium-state + optional: true + - name: CILIUM_BPF_STATE + valueFrom: + configMapKeyRef: + name: cilium-config + key: clean-cilium-bpf-state + optional: true +# Removed in 1.11 and up. +# https://github.com/cilium/cilium/commit/f7a3f59fd74983c600bfce9cac364b76d20849d9 +{% if cilium_version | regex_replace('v') is version('1.11', '<') %} + - name: CILIUM_WAIT_BPF_MOUNT + valueFrom: + configMapKeyRef: + key: wait-bpf-mount + name: cilium-config + optional: true +{% endif %} +{% if cilium_kube_proxy_replacement == 'strict' %} + - name: KUBERNETES_SERVICE_HOST + value: "{{ kube_apiserver_global_endpoint | urlsplit('hostname') }}" + - name: KUBERNETES_SERVICE_PORT + value: "{{ kube_apiserver_global_endpoint | urlsplit('port') }}" +{% endif %} + securityContext: + privileged: true + volumeMounts: + - name: bpf-maps + mountPath: /sys/fs/bpf +{% if cilium_version | regex_replace('v') is version('1.11', '>=') %} + # Required to mount cgroup filesystem from the host to cilium agent pod + - name: cilium-cgroup + mountPath: {{ cilium_cgroup_host_root }} + mountPropagation: HostToContainer +{% endif %} + - name: cilium-run + mountPath: /var/run/cilium + resources: + requests: + cpu: 100m + memory: 100Mi +{% if cilium_version | regex_replace('v') is version('1.13.1', '>=') %} + # Install the CNI binaries in an InitContainer so we don't have a writable host mount in the agent + - name: install-cni-binaries + image: "{{ cilium_image_repo }}:{{ cilium_image_tag }}" + imagePullPolicy: {{ k8s_image_pull_policy }} + command: + - "/install-plugin.sh" + resources: + requests: + cpu: 100m + memory: 10Mi + securityContext: + privileged: true + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - name: cni-path + mountPath: /host/opt/cni/bin +{% endif %} + restartPolicy: Always + priorityClassName: system-node-critical + serviceAccount: cilium + serviceAccountName: cilium + terminationGracePeriodSeconds: 1 + hostNetwork: true +# In managed etcd mode, Cilium must be able to resolve the DNS name of the etcd service +{% if cilium_identity_allocation_mode == "kvstore" %} + dnsPolicy: ClusterFirstWithHostNet +{% endif %} + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + k8s-app: cilium + tolerations: + - operator: Exists + volumes: + # To keep state between restarts / upgrades + - name: cilium-run + hostPath: + path: /var/run/cilium + type: DirectoryOrCreate + # To keep state between restarts / upgrades for bpf maps + - name: bpf-maps + hostPath: + path: /sys/fs/bpf + type: DirectoryOrCreate +{% if cilium_version | regex_replace('v') is version('1.11', '>=') %} + # To mount cgroup2 filesystem on the host + - name: hostproc + hostPath: + path: /proc + type: Directory + # To keep state between restarts / upgrades for cgroup2 filesystem + - name: cilium-cgroup + hostPath: + path: {{ cilium_cgroup_host_root }} + type: DirectoryOrCreate +{% endif %} + # To install cilium cni plugin in the host + - name: cni-path + hostPath: + path: /opt/cni/bin + type: DirectoryOrCreate + # To install cilium cni configuration in the host + - name: etc-cni-netd + hostPath: + path: /etc/cni/net.d + type: DirectoryOrCreate + # To be able to load kernel modules + - name: lib-modules + hostPath: + path: /lib/modules + # To access iptables concurrently with other processes (e.g. kube-proxy) + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate +{% if cilium_identity_allocation_mode == "kvstore" %} + # To read the etcd config stored in config maps + - name: etcd-config-path + configMap: + name: cilium-config + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + items: + - key: etcd-config + path: etcd.config + # To read the k8s etcd secrets in case the user might want to use TLS + - name: etcd-secrets + hostPath: + path: "{{ cilium_cert_dir }}" +{% endif %} + # To read the clustermesh configuration + - name: clustermesh-secrets + secret: + secretName: cilium-clustermesh + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + optional: true + # To read the configuration from the config map + - name: cilium-config-path + configMap: + name: cilium-config +{% if cilium_ip_masq_agent_enable %} + - name: ip-masq-agent + configMap: + name: ip-masq-agent + optional: true + items: + - key: config + path: ip-masq-agent +{% endif %} +{% if cilium_encryption_enabled and cilium_encryption_type == "ipsec" %} + - name: cilium-ipsec-secrets + secret: + secretName: cilium-ipsec-keys +{% endif %} +{% if cilium_hubble_install %} + - name: hubble-tls + projected: + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + sources: + - secret: + name: hubble-server-certs + optional: true + items: + - key: ca.crt + path: client-ca.crt + - key: tls.crt + path: server.crt + - key: tls.key + path: server.key +{% endif %} diff --git a/kubespray/project/roles/network_plugin/cilium/templates/cilium/sa.yml.j2 b/kubespray/project/roles/network_plugin/cilium/templates/cilium/sa.yml.j2 new file mode 100644 index 0000000..c03ac59 --- /dev/null +++ b/kubespray/project/roles/network_plugin/cilium/templates/cilium/sa.yml.j2 @@ -0,0 +1,6 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: cilium + namespace: kube-system diff --git a/kubespray/project/roles/network_plugin/cilium/templates/cilium/secret.yml.j2 b/kubespray/project/roles/network_plugin/cilium/templates/cilium/secret.yml.j2 new file mode 100644 index 0000000..776c689 --- /dev/null +++ b/kubespray/project/roles/network_plugin/cilium/templates/cilium/secret.yml.j2 @@ -0,0 +1,9 @@ +--- +apiVersion: v1 +data: + keys: {{ cilium_ipsec_key }} +kind: Secret +metadata: + name: cilium-ipsec-keys + namespace: kube-system +type: Opaque diff --git a/kubespray/project/roles/network_plugin/cilium/templates/hubble/config.yml.j2 b/kubespray/project/roles/network_plugin/cilium/templates/hubble/config.yml.j2 new file mode 100644 index 0000000..888db41 --- /dev/null +++ b/kubespray/project/roles/network_plugin/cilium/templates/hubble/config.yml.j2 @@ -0,0 +1,71 @@ +#jinja2: trim_blocks:False +--- +# Source: cilium helm chart: cilium/templates/hubble-relay/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: hubble-relay-config + namespace: kube-system +data: + config.yaml: | + cluster-name: "{{ cilium_cluster_name }}" + peer-service: "hubble-peer.kube-system.svc.{{ dns_domain }}:443" + listen-address: :4245 + metrics-listen-address: ":9966" + dial-timeout: + retry-timeout: + sort-buffer-len-max: + sort-buffer-drain-timeout: + tls-client-cert-file: /var/lib/hubble-relay/tls/client.crt + tls-client-key-file: /var/lib/hubble-relay/tls/client.key + tls-server-cert-file: /var/lib/hubble-relay/tls/server.crt + tls-server-key-file: /var/lib/hubble-relay/tls/server.key + tls-hubble-server-ca-files: /var/lib/hubble-relay/tls/hubble-server-ca.crt + disable-server-tls: {% if cilium_hubble_tls_generate %}false{% else %}true{% endif %} + disable-client-tls: {% if cilium_hubble_tls_generate %}false{% else %}true{% endif %} +--- +# Source: cilium/templates/hubble-ui/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: hubble-ui-nginx + namespace: kube-system +data: + nginx.conf: | + server { + listen 8081; + {% if cilium_enable_ipv6 %} + listen [::]:8081; + {% endif %} + server_name localhost; + root /app; + index index.html; + client_max_body_size 1G; + + location / { + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + + # CORS + add_header Access-Control-Allow-Methods "GET, POST, PUT, HEAD, DELETE, OPTIONS"; + add_header Access-Control-Allow-Origin *; + add_header Access-Control-Max-Age 1728000; + add_header Access-Control-Expose-Headers content-length,grpc-status,grpc-message; + add_header Access-Control-Allow-Headers range,keep-alive,user-agent,cache-control,content-type,content-transfer-encoding,x-accept-content-transfer-encoding,x-accept-response-streaming,x-user-agent,x-grpc-web,grpc-timeout; + if ($request_method = OPTIONS) { + return 204; + } + # /CORS + + location /api { + proxy_http_version 1.1; + proxy_pass_request_headers on; + proxy_hide_header Access-Control-Allow-Origin; + proxy_pass http://127.0.0.1:8090; + } + + location / { + try_files $uri $uri/ /index.html; + } + } + } diff --git a/kubespray/project/roles/network_plugin/cilium/templates/hubble/cr.yml.j2 b/kubespray/project/roles/network_plugin/cilium/templates/hubble/cr.yml.j2 new file mode 100644 index 0000000..4a95565 --- /dev/null +++ b/kubespray/project/roles/network_plugin/cilium/templates/hubble/cr.yml.j2 @@ -0,0 +1,106 @@ +{% if cilium_hubble_tls_generate %} +--- +# Source: cilium/templates/hubble-generate-certs-clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: hubble-generate-certs +rules: + - apiGroups: + - "" + resources: + - secrets + - configmaps + verbs: + - create + - apiGroups: + - "" + resources: + - secrets + resourceNames: + - hubble-server-certs + - hubble-relay-client-certs + - hubble-relay-server-certs + verbs: + - update + - apiGroups: + - "" + resources: + - configmaps + resourceNames: + - hubble-ca-cert + verbs: + - update + - apiGroups: + - "" + resources: + - secrets + resourceNames: + - hubble-ca-secret + verbs: + - get +{% endif %} +--- +# Source: cilium/templates/hubble-relay-clusterrole.yaml +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: hubble-relay +rules: + - apiGroups: + - "" + resources: + - componentstatuses + - endpoints + - namespaces + - nodes + - pods + - services + verbs: + - get + - list + - watch +--- +# Source: cilium/templates/hubble-ui-clusterrole.yaml +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: hubble-ui +rules: + - apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - componentstatuses + - endpoints + - namespaces + - nodes + - pods + - services + verbs: + - get + - list + - watch + - apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list + - watch + - apiGroups: + - cilium.io + resources: + - "*" + verbs: + - get + - list + - watch diff --git a/kubespray/project/roles/network_plugin/cilium/templates/hubble/crb.yml.j2 b/kubespray/project/roles/network_plugin/cilium/templates/hubble/crb.yml.j2 new file mode 100644 index 0000000..f033429 --- /dev/null +++ b/kubespray/project/roles/network_plugin/cilium/templates/hubble/crb.yml.j2 @@ -0,0 +1,44 @@ +{% if cilium_hubble_tls_generate %} +--- +# Source: cilium/templates/hubble-generate-certs-clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: hubble-generate-certs +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: hubble-generate-certs +subjects: +- kind: ServiceAccount + name: hubble-generate-certs + namespace: kube-system +{% endif %} +--- +# Source: cilium/templates/hubble-relay-clusterrolebinding.yaml +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: hubble-relay +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: hubble-relay +subjects: +- kind: ServiceAccount + namespace: kube-system + name: hubble-relay +--- +# Source: cilium/templates/hubble-ui-clusterrolebinding.yaml +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: hubble-ui +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: hubble-ui +subjects: +- kind: ServiceAccount + namespace: kube-system + name: hubble-ui diff --git a/kubespray/project/roles/network_plugin/cilium/templates/hubble/cronjob.yml.j2 b/kubespray/project/roles/network_plugin/cilium/templates/hubble/cronjob.yml.j2 new file mode 100644 index 0000000..8010c52 --- /dev/null +++ b/kubespray/project/roles/network_plugin/cilium/templates/hubble/cronjob.yml.j2 @@ -0,0 +1,38 @@ +--- +# Source: cilium/templates/hubble-generate-certs-cronjob.yaml +apiVersion: batch/v1 +kind: CronJob +metadata: + name: hubble-generate-certs + namespace: kube-system + labels: + k8s-app: hubble-generate-certs +spec: + schedule: "0 0 1 */4 *" + concurrencyPolicy: Forbid + jobTemplate: + spec: + template: + metadata: + labels: + k8s-app: hubble-generate-certs + spec: + serviceAccount: hubble-generate-certs + serviceAccountName: hubble-generate-certs + containers: + - name: certgen + image: "{{ cilium_hubble_certgen_image_repo }}:{{ cilium_hubble_certgen_image_tag }}" + imagePullPolicy: {{ k8s_image_pull_policy }} + command: + - "/usr/bin/cilium-certgen" + # Because this is executed as a job, we pass the values as command + # line args instead of via config map. This allows users to inspect + # the values used in past runs by inspecting the completed pod. + args: + {% for key, value in cilium_certgen_args.items() -%} + - "--{{ key }}={{ value }}" + {% endfor %} + + hostNetwork: true + restartPolicy: OnFailure + ttlSecondsAfterFinished: 1800 diff --git a/kubespray/project/roles/network_plugin/cilium/templates/hubble/deploy.yml.j2 b/kubespray/project/roles/network_plugin/cilium/templates/hubble/deploy.yml.j2 new file mode 100644 index 0000000..f0153d6 --- /dev/null +++ b/kubespray/project/roles/network_plugin/cilium/templates/hubble/deploy.yml.j2 @@ -0,0 +1,197 @@ +--- +# Source: cilium/templates/hubble-relay-deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hubble-relay + labels: + k8s-app: hubble-relay + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + k8s-app: hubble-relay + strategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + annotations: + labels: + k8s-app: hubble-relay + spec: + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "k8s-app" + operator: In + values: + - cilium + topologyKey: "kubernetes.io/hostname" + containers: + - name: hubble-relay + image: "{{ cilium_hubble_relay_image_repo }}:{{ cilium_hubble_relay_image_tag }}" + imagePullPolicy: {{ k8s_image_pull_policy }} + command: + - hubble-relay + args: + - serve + ports: + - name: grpc + containerPort: 4245 +{% if cilium_enable_prometheus %} + - name: prometheus + containerPort: 9966 + protocol: TCP +{% endif %} + readinessProbe: + tcpSocket: + port: grpc + livenessProbe: + tcpSocket: + port: grpc + volumeMounts: + - mountPath: /var/run/cilium + name: hubble-sock-dir + readOnly: true + - mountPath: /etc/hubble-relay + name: config + readOnly: true + {% if cilium_hubble_tls_generate -%} + - mountPath: /var/lib/hubble-relay/tls + name: tls + readOnly: true + {%- endif %} + + restartPolicy: Always + serviceAccount: hubble-relay + serviceAccountName: hubble-relay + terminationGracePeriodSeconds: 0 + volumes: + - configMap: + name: hubble-relay-config + items: + - key: config.yaml + path: config.yaml + name: config + - hostPath: + path: /var/run/cilium + type: Directory + name: hubble-sock-dir + {% if cilium_hubble_tls_generate -%} + - projected: + sources: + - secret: + name: hubble-relay-client-certs + items: + - key: ca.crt + path: hubble-server-ca.crt + - key: tls.crt + path: client.crt + - key: tls.key + path: client.key + - secret: + name: hubble-server-certs + items: + - key: tls.crt + path: server.crt + - key: tls.key + path: server.key + name: tls + {%- endif %} + +--- +# Source: cilium/templates/hubble-ui/deployment.yaml +kind: Deployment +apiVersion: apps/v1 +metadata: + namespace: kube-system + labels: + k8s-app: hubble-ui + name: hubble-ui +spec: + replicas: 1 + selector: + matchLabels: + k8s-app: hubble-ui + template: + metadata: + annotations: + labels: + k8s-app: hubble-ui + spec: + securityContext: + runAsUser: 1001 + serviceAccount: hubble-ui + serviceAccountName: hubble-ui + containers: + - name: frontend + image: "{{ cilium_hubble_ui_image_repo }}:{{ cilium_hubble_ui_image_tag }}" + imagePullPolicy: {{ k8s_image_pull_policy }} + ports: + - containerPort: 8081 + name: http + volumeMounts: + - name: hubble-ui-nginx-conf + mountPath: /etc/nginx/conf.d/default.conf + subPath: nginx.conf + - name: tmp-dir + mountPath: /tmp + resources: + {} + - name: backend + image: "{{ cilium_hubble_ui_backend_image_repo }}:{{ cilium_hubble_ui_backend_image_tag }}" + imagePullPolicy: {{ k8s_image_pull_policy }} + env: + - name: EVENTS_SERVER_PORT + value: "8090" + {% if cilium_hubble_tls_generate -%} + - name: TLS_TO_RELAY_ENABLED + value: "true" + - name: FLOWS_API_ADDR + value: "hubble-relay:443" + - name: TLS_RELAY_SERVER_NAME + value: ui.{{ cilium_cluster_name }}.hubble-grpc.cilium.io + - name: TLS_RELAY_CA_CERT_FILES + value: /var/lib/hubble-ui/certs/hubble-server-ca.crt + - name: TLS_RELAY_CLIENT_CERT_FILE + value: /var/lib/hubble-ui/certs/client.crt + - name: TLS_RELAY_CLIENT_KEY_FILE + value: /var/lib/hubble-ui/certs/client.key + {% else -%} + - name: FLOWS_API_ADDR + value: "hubble-relay:80" + {% endif %} + + volumeMounts: + - name: tls + mountPath: /var/lib/hubble-ui/certs + readOnly: true + ports: + - containerPort: 8090 + name: grpc + resources: + {} + volumes: + - configMap: + defaultMode: 420 + name: hubble-ui-nginx + name: hubble-ui-nginx-conf + - projected: + sources: + - secret: + name: hubble-relay-client-certs + items: + - key: ca.crt + path: hubble-server-ca.crt + - key: tls.crt + path: client.crt + - key: tls.key + path: client.key + name: tls + - emptyDir: {} + name: tmp-dir diff --git a/kubespray/project/roles/network_plugin/cilium/templates/hubble/job.yml.j2 b/kubespray/project/roles/network_plugin/cilium/templates/hubble/job.yml.j2 new file mode 100644 index 0000000..9ad3ae3 --- /dev/null +++ b/kubespray/project/roles/network_plugin/cilium/templates/hubble/job.yml.j2 @@ -0,0 +1,34 @@ +--- +# Source: cilium/templates/hubble-generate-certs-job.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: hubble-generate-certs + namespace: kube-system + labels: + k8s-app: hubble-generate-certs +spec: + template: + metadata: + labels: + k8s-app: hubble-generate-certs + spec: + serviceAccount: hubble-generate-certs + serviceAccountName: hubble-generate-certs + containers: + - name: certgen + image: "{{ cilium_hubble_certgen_image_repo }}:{{ cilium_hubble_certgen_image_tag }}" + imagePullPolicy: {{ k8s_image_pull_policy }} + command: + - "/usr/bin/cilium-certgen" + # Because this is executed as a job, we pass the values as command + # line args instead of via config map. This allows users to inspect + # the values used in past runs by inspecting the completed pod. + args: + {% for key, value in cilium_certgen_args.items() -%} + - "--{{ key }}={{ value }}" + {% endfor %} + + hostNetwork: true + restartPolicy: OnFailure + ttlSecondsAfterFinished: 1800 diff --git a/kubespray/project/roles/network_plugin/cilium/templates/hubble/sa.yml.j2 b/kubespray/project/roles/network_plugin/cilium/templates/hubble/sa.yml.j2 new file mode 100644 index 0000000..9b3203d --- /dev/null +++ b/kubespray/project/roles/network_plugin/cilium/templates/hubble/sa.yml.j2 @@ -0,0 +1,23 @@ +{% if cilium_hubble_tls_generate %} +--- +# Source: cilium/templates/hubble-generate-certs-serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: hubble-generate-certs + namespace: kube-system +{% endif %} +--- +# Source: cilium/templates/hubble-relay-serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: hubble-relay + namespace: kube-system +--- +# Source: cilium/templates/hubble-ui-serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: hubble-ui + namespace: kube-system diff --git a/kubespray/project/roles/network_plugin/cilium/templates/hubble/service.yml.j2 b/kubespray/project/roles/network_plugin/cilium/templates/hubble/service.yml.j2 new file mode 100644 index 0000000..0f862a9 --- /dev/null +++ b/kubespray/project/roles/network_plugin/cilium/templates/hubble/service.yml.j2 @@ -0,0 +1,105 @@ +{% if cilium_enable_prometheus or cilium_enable_hubble_metrics %} +--- +# Source: cilium/templates/cilium-agent-service.yaml +kind: Service +apiVersion: v1 +metadata: + name: hubble-metrics + namespace: kube-system + annotations: + prometheus.io/scrape: 'true' + prometheus.io/port: "{{ cilium_hubble_scrape_port }}" + labels: + k8s-app: hubble +spec: + clusterIP: None + type: ClusterIP + ports: + - name: hubble-metrics + port: 9091 + protocol: TCP + targetPort: hubble-metrics + selector: + k8s-app: cilium +--- +# Source: cilium/templates/hubble-relay/metrics-service.yaml +# We use a separate service from hubble-relay which can be exposed externally +kind: Service +apiVersion: v1 +metadata: + name: hubble-relay-metrics + namespace: kube-system + labels: + k8s-app: hubble-relay + annotations: + prometheus.io/scrape: 'true' + prometheus.io/port: "9966" +spec: + clusterIP: None + type: ClusterIP + selector: + k8s-app: hubble-relay + ports: + - name: metrics + port: 9966 + protocol: TCP + targetPort: prometheus + +{% endif %} +--- +# Source: cilium/templates/hubble-relay-service.yaml +kind: Service +apiVersion: v1 +metadata: + name: hubble-relay + namespace: kube-system + labels: + k8s-app: hubble-relay +spec: + type: ClusterIP + selector: + k8s-app: hubble-relay + ports: + - protocol: TCP + {% if cilium_hubble_tls_generate -%} + port: 443 + {% else -%} + port: 80 + {% endif -%} + targetPort: 4245 +--- +# Source: cilium/templates/hubble-ui-service.yaml +kind: Service +apiVersion: v1 +metadata: + name: hubble-ui + labels: + k8s-app: hubble-ui + namespace: kube-system +spec: + selector: + k8s-app: hubble-ui + ports: + - name: http + port: 80 + targetPort: 8081 + type: ClusterIP +--- +# Source: cilium/templates/hubble/peer-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: hubble-peer + namespace: kube-system + labels: + k8s-app: cilium +spec: + selector: + k8s-app: cilium + ports: + - name: peer-service + port: 443 + protocol: TCP + targetPort: 4244 + internalTrafficPolicy: Local + diff --git a/kubespray/project/roles/network_plugin/cni/defaults/main.yml b/kubespray/project/roles/network_plugin/cni/defaults/main.yml new file mode 100644 index 0000000..5d11edf --- /dev/null +++ b/kubespray/project/roles/network_plugin/cni/defaults/main.yml @@ -0,0 +1,2 @@ +--- +cni_bin_owner: "{{ kube_owner }}" diff --git a/kubespray/project/roles/network_plugin/cni/tasks/main.yml b/kubespray/project/roles/network_plugin/cni/tasks/main.yml new file mode 100644 index 0000000..bcab4ef --- /dev/null +++ b/kubespray/project/roles/network_plugin/cni/tasks/main.yml @@ -0,0 +1,16 @@ +--- +- name: CNI | make sure /opt/cni/bin exists + file: + path: /opt/cni/bin + state: directory + mode: 0755 + owner: "{{ cni_bin_owner }}" + recurse: true + +- name: CNI | Copy cni plugins + unarchive: + src: "{{ downloads.cni.dest }}" + dest: "/opt/cni/bin" + mode: 0755 + owner: "{{ cni_bin_owner }}" + remote_src: yes diff --git a/kubespray/project/roles/network_plugin/custom_cni/defaults/main.yml b/kubespray/project/roles/network_plugin/custom_cni/defaults/main.yml new file mode 100644 index 0000000..0eab143 --- /dev/null +++ b/kubespray/project/roles/network_plugin/custom_cni/defaults/main.yml @@ -0,0 +1,11 @@ +--- + +custom_cni_manifests: [] + +custom_cni_chart_namespace: kube-system +custom_cni_chart_release_name: "" +custom_cni_chart_repository_name: "" +custom_cni_chart_repository_url: "" +custom_cni_chart_ref: "" +custom_cni_chart_version: "" +custom_cni_chart_values: {} diff --git a/kubespray/project/roles/network_plugin/custom_cni/meta/main.yml b/kubespray/project/roles/network_plugin/custom_cni/meta/main.yml new file mode 100644 index 0000000..361c406 --- /dev/null +++ b/kubespray/project/roles/network_plugin/custom_cni/meta/main.yml @@ -0,0 +1,20 @@ +--- +dependencies: + - role: helm-apps + when: + - inventory_hostname == groups['kube_control_plane'][0] + - custom_cni_chart_release_name | length > 0 + environment: + http_proxy: "{{ http_proxy | default('') }}" + https_proxy: "{{ https_proxy | default('') }}" + release_common_opts: {} + releases: + - name: "{{ custom_cni_chart_release_name }}" + namespace: "{{ custom_cni_chart_namespace }}" + chart_ref: "{{ custom_cni_chart_ref }}" + chart_version: "{{ custom_cni_chart_version }}" + wait: true + values: "{{ custom_cni_chart_values }}" + repositories: + - name: "{{ custom_cni_chart_repository_name }}" + url: "{{ custom_cni_chart_repository_url }}" diff --git a/kubespray/project/roles/network_plugin/custom_cni/tasks/main.yml b/kubespray/project/roles/network_plugin/custom_cni/tasks/main.yml new file mode 100644 index 0000000..8f5b419 --- /dev/null +++ b/kubespray/project/roles/network_plugin/custom_cni/tasks/main.yml @@ -0,0 +1,29 @@ +--- +- name: Custom CNI | Manifest deployment + when: not custom_cni_chart_release_name | length > 0 + block: + - name: Custom CNI | Check Custom CNI Manifests + assert: + that: + - "custom_cni_manifests | length > 0" + msg: "custom_cni_manifests should not be empty" + + - name: Custom CNI | Copy Custom manifests + template: + src: "{{ item }}" + dest: "{{ kube_config_dir }}/{{ item | basename | replace('.j2', '') }}" + mode: 0644 + loop: "{{ custom_cni_manifests }}" + delegate_to: "{{ groups['kube_control_plane'] | first }}" + run_once: true + + - name: Custom CNI | Start Resources + kube: + namespace: "kube-system" + kubectl: "{{ bin_dir }}/kubectl" + filename: "{{ kube_config_dir }}/{{ item | basename | replace('.j2', '') }}" + state: "latest" + wait: true + loop: "{{ custom_cni_manifests }}" + delegate_to: "{{ groups['kube_control_plane'] | first }}" + run_once: true diff --git a/kubespray/project/roles/network_plugin/flannel/defaults/main.yml b/kubespray/project/roles/network_plugin/flannel/defaults/main.yml new file mode 100644 index 0000000..8d7713b --- /dev/null +++ b/kubespray/project/roles/network_plugin/flannel/defaults/main.yml @@ -0,0 +1,28 @@ +--- +# Flannel public IP +# The address that flannel should advertise as how to access the system +# Disabled until https://github.com/coreos/flannel/issues/712 is fixed +# flannel_public_ip: "{{ access_ip | default(ip | default(fallback_ips[inventory_hostname])) }}" + +## interface that should be used for flannel operations +## This is actually an inventory cluster-level item +# flannel_interface: + +## Select interface that should be used for flannel operations by regexp on Name or IP +## This is actually an inventory cluster-level item +## example: select interface with ip from net 10.0.0.0/23 +## single quote and escape backslashes +# flannel_interface_regexp: '10\\.0\\.[0-2]\\.\\d{1,3}' + +# You can choose what type of flannel backend to use +# please refer to flannel's docs : https://github.com/coreos/flannel/blob/master/README.md +flannel_backend_type: "vxlan" +flannel_vxlan_vni: 1 +flannel_vxlan_port: 8472 +flannel_vxlan_direct_routing: false + +# Limits for apps +flannel_memory_limit: 500M +flannel_cpu_limit: 300m +flannel_memory_requests: 64M +flannel_cpu_requests: 150m diff --git a/kubespray/project/roles/network_plugin/flannel/meta/main.yml b/kubespray/project/roles/network_plugin/flannel/meta/main.yml new file mode 100644 index 0000000..9b7065f --- /dev/null +++ b/kubespray/project/roles/network_plugin/flannel/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - role: network_plugin/cni diff --git a/kubespray/project/roles/network_plugin/flannel/tasks/main.yml b/kubespray/project/roles/network_plugin/flannel/tasks/main.yml new file mode 100644 index 0000000..2fd82e9 --- /dev/null +++ b/kubespray/project/roles/network_plugin/flannel/tasks/main.yml @@ -0,0 +1,21 @@ +--- + +- name: Flannel | Stop if kernel version is too low for Flannel Wireguard encryption + assert: + that: ansible_kernel.split('-')[0] is version('5.6.0', '>=') + when: + - kube_network_plugin == 'flannel' + - flannel_backend_type == 'wireguard' + - not ignore_assert_errors + +- name: Flannel | Create Flannel manifests + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/{{ item.file }}" + mode: 0644 + with_items: + - {name: flannel, file: cni-flannel-rbac.yml, type: sa} + - {name: kube-flannel, file: cni-flannel.yml, type: ds} + register: flannel_node_manifests + when: + - inventory_hostname == groups['kube_control_plane'][0] diff --git a/kubespray/project/roles/network_plugin/flannel/tasks/reset.yml b/kubespray/project/roles/network_plugin/flannel/tasks/reset.yml new file mode 100644 index 0000000..03d40a0 --- /dev/null +++ b/kubespray/project/roles/network_plugin/flannel/tasks/reset.yml @@ -0,0 +1,24 @@ +--- +- name: Reset | check cni network device + stat: + path: /sys/class/net/cni0 + get_attributes: no + get_checksum: no + get_mime: no + register: cni + +- name: Reset | remove the network device created by the flannel + command: ip link del cni0 + when: cni.stat.exists + +- name: Reset | check flannel network device + stat: + path: /sys/class/net/flannel.1 + get_attributes: no + get_checksum: no + get_mime: no + register: flannel + +- name: Reset | remove the network device created by the flannel + command: ip link del flannel.1 + when: flannel.stat.exists diff --git a/kubespray/project/roles/network_plugin/flannel/templates/cni-flannel-rbac.yml.j2 b/kubespray/project/roles/network_plugin/flannel/templates/cni-flannel-rbac.yml.j2 new file mode 100644 index 0000000..631ec5e --- /dev/null +++ b/kubespray/project/roles/network_plugin/flannel/templates/cni-flannel-rbac.yml.j2 @@ -0,0 +1,52 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: flannel + namespace: kube-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: flannel +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch +- apiGroups: + - "networking.k8s.io" + resources: + - clustercidrs + verbs: + - list + - watch +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: flannel +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: flannel +subjects: +- kind: ServiceAccount + name: flannel + namespace: kube-system diff --git a/kubespray/project/roles/network_plugin/flannel/templates/cni-flannel.yml.j2 b/kubespray/project/roles/network_plugin/flannel/templates/cni-flannel.yml.j2 new file mode 100644 index 0000000..9c36d01 --- /dev/null +++ b/kubespray/project/roles/network_plugin/flannel/templates/cni-flannel.yml.j2 @@ -0,0 +1,170 @@ +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: kube-flannel-cfg + namespace: kube-system + labels: + tier: node + app: flannel +data: + cni-conf.json: | + { + "name": "cbr0", + "cniVersion": "0.3.1", + "plugins": [ + { + "type": "flannel", + "delegate": { + "hairpinMode": true, + "isDefaultGateway": true + } + }, + { + "type": "portmap", + "capabilities": { + "portMappings": true + } + } + ] + } + net-conf.json: | + { + "Network": "{{ kube_pods_subnet }}", + "EnableIPv4": true, +{% if enable_dual_stack_networks %} + "EnableIPv6": true, + "IPv6Network": "{{ kube_pods_subnet_ipv6 }}", +{% endif %} + "Backend": { + "Type": "{{ flannel_backend_type }}"{% if flannel_backend_type == "vxlan" %}, + "VNI": {{ flannel_vxlan_vni }}, + "Port": {{ flannel_vxlan_port }}, + "DirectRouting": {{ flannel_vxlan_direct_routing | to_json }} +{% endif %} + } + } +{% for arch in ['amd64', 'arm64', 'arm', 'ppc64le', 's390x'] %} +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: +{% if arch == 'amd64' %} + name: kube-flannel +{% else %} + name: kube-flannel-ds-{{ arch }} +{% endif %} + namespace: kube-system + labels: + tier: node + app: flannel +spec: + selector: + matchLabels: + app: flannel + template: + metadata: + labels: + tier: node + app: flannel + spec: + priorityClassName: system-node-critical + serviceAccountName: flannel + containers: + - name: kube-flannel + image: {{ flannel_image_repo }}:{{ flannel_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + resources: + limits: + cpu: {{ flannel_cpu_limit }} + memory: {{ flannel_memory_limit }} + requests: + cpu: {{ flannel_cpu_requests }} + memory: {{ flannel_memory_requests }} + command: [ "/opt/bin/flanneld", "--ip-masq", "--kube-subnet-mgr"{% if flannel_interface is defined %}, "--iface={{ flannel_interface }}"{% endif %}{% if flannel_interface_regexp is defined %}, "--iface-regex={{ flannel_interface_regexp }}"{% endif %} ] + securityContext: + privileged: false + capabilities: + add: ["NET_ADMIN", "NET_RAW"] + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: EVENT_QUEUE_DEPTH + value: "5000" + volumeMounts: + - name: run + mountPath: /run/flannel + - name: flannel-cfg + mountPath: /etc/kube-flannel/ + - name: xtables-lock + mountPath: /run/xtables.lock + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/os + operator: In + values: + - linux + - key: kubernetes.io/arch + operator: In + values: + - {{ arch }} + initContainers: + - name: install-cni-plugin + image: {{ flannel_init_image_repo }}:{{ flannel_init_image_tag }} + command: + - cp + args: + - -f + - /flannel + - /opt/cni/bin/flannel + volumeMounts: + - name: cni-plugin + mountPath: /opt/cni/bin + - name: install-cni + image: {{ flannel_image_repo }}:{{ flannel_image_tag }} + command: + - cp + args: + - -f + - /etc/kube-flannel/cni-conf.json + - /etc/cni/net.d/10-flannel.conflist + volumeMounts: + - name: cni + mountPath: /etc/cni/net.d + - name: flannel-cfg + mountPath: /etc/kube-flannel/ + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + tolerations: + - operator: Exists + volumes: + - name: run + hostPath: + path: /run/flannel + - name: cni + hostPath: + path: /etc/cni/net.d + - name: flannel-cfg + configMap: + name: kube-flannel-cfg + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + - name: cni-plugin + hostPath: + path: /opt/cni/bin + updateStrategy: + rollingUpdate: + maxUnavailable: {{ serial | default('20%') }} + type: RollingUpdate +{% endfor %} diff --git a/kubespray/project/roles/network_plugin/kube-ovn/OWNERS b/kubespray/project/roles/network_plugin/kube-ovn/OWNERS new file mode 100644 index 0000000..84256aa --- /dev/null +++ b/kubespray/project/roles/network_plugin/kube-ovn/OWNERS @@ -0,0 +1,4 @@ +# See the OWNERS docs at https://go.k8s.io/owners + +emeritus_approvers: +- oilbeater diff --git a/kubespray/project/roles/network_plugin/kube-ovn/defaults/main.yml b/kubespray/project/roles/network_plugin/kube-ovn/defaults/main.yml new file mode 100644 index 0000000..44850e5 --- /dev/null +++ b/kubespray/project/roles/network_plugin/kube-ovn/defaults/main.yml @@ -0,0 +1,118 @@ +--- +kube_ovn_db_cpu_request: 500m +kube_ovn_db_memory_request: 200Mi +kube_ovn_db_cpu_limit: 3000m +kube_ovn_db_memory_limit: 3000Mi +kube_ovn_node_cpu_request: 200m +kube_ovn_node_memory_request: 200Mi +kube_ovn_node_cpu_limit: 1000m +kube_ovn_node_memory_limit: 800Mi +kube_ovn_cni_server_cpu_request: 200m +kube_ovn_cni_server_memory_request: 200Mi +kube_ovn_cni_server_cpu_limit: 1000m +kube_ovn_cni_server_memory_limit: 1Gi +kube_ovn_controller_cpu_request: 200m +kube_ovn_controller_memory_request: 200Mi +kube_ovn_controller_cpu_limit: 1000m +kube_ovn_controller_memory_limit: 1Gi +kube_ovn_pinger_cpu_request: 100m +kube_ovn_pinger_memory_request: 200Mi +kube_ovn_pinger_cpu_limit: 200m +kube_ovn_pinger_memory_limit: 400Mi +kube_ovn_monitor_memory_request: 200Mi +kube_ovn_monitor_cpu_request: 200m +kube_ovn_monitor_memory_limit: 200Mi +kube_ovn_monitor_cpu_limit: 200m +kube_ovn_dpdk_node_cpu_request: 1000m +kube_ovn_dpdk_node_memory_request: 2Gi +kube_ovn_dpdk_node_cpu_limit: 1000m +kube_ovn_dpdk_node_memory_limit: 2Gi + +kube_ovn_central_hosts: "{{ groups['kube_control_plane'] }}" +kube_ovn_central_replics: "{{ kube_ovn_central_hosts | length }}" +kube_ovn_controller_replics: "{{ kube_ovn_central_hosts | length }}" +kube_ovn_central_ips: |- + {% for item in kube_ovn_central_hosts -%} + {{ hostvars[item]['ip'] | default(fallback_ips[item]) }}{% if not loop.last %},{% endif %} + {%- endfor %} + +kube_ovn_ic_enable: false +kube_ovn_ic_autoroute: true +kube_ovn_ic_dbhost: "127.0.0.1" +kube_ovn_ic_zone: "kubernetes" + +# geneve or vlan +kube_ovn_network_type: geneve + +# geneve, vxlan or stt. ATTENTION: some networkpolicy cannot take effect when using vxlan and stt need custom compile ovs kernel module +kube_ovn_tunnel_type: geneve + +## The nic to support container network can be a nic name or a group of regex separated by comma e.g: 'enp6s0f0,eth.*', if empty will use the nic that the default route use. +# kube_ovn_iface: eth1 +## The MTU used by pod iface in overlay networks (default iface MTU - 100) +# kube_ovn_mtu: 1333 + +## Enable hw-offload, disable traffic mirror and set the iface to the physical port. Make sure that there is an IP address bind to the physical port. +kube_ovn_hw_offload: false +# traffic mirror +kube_ovn_traffic_mirror: false + +# kube_ovn_pool_cidr_ipv6: fd85:ee78:d8a6:8607::1:0000/112 +# kube_ovn_default_interface_name: eth0 + +kube_ovn_external_address: 8.8.8.8 +kube_ovn_external_address_ipv6: 2400:3200::1 +kube_ovn_external_dns: alauda.cn + +# kube_ovn_default_gateway: 10.233.64.1,fd85:ee78:d8a6:8607::1:0 +kube_ovn_default_gateway_check: true +kube_ovn_default_logical_gateway: false + +# u2o_interconnection +kube_ovn_u2o_interconnection: false + +# kube_ovn_default_exclude_ips: 10.16.0.1 +kube_ovn_node_switch_cidr: 100.64.0.0/16 +kube_ovn_node_switch_cidr_ipv6: fd00:100:64::/64 + +## vlan config, set default interface name and vlan id +# kube_ovn_default_interface_name: eth0 +kube_ovn_default_vlan_id: 100 +kube_ovn_vlan_name: product + +## pod nic type, support: veth-pair or internal-port +kube_ovn_pod_nic_type: veth_pair + +## Enable load balancer +kube_ovn_enable_lb: true + +## Enable network policy support +kube_ovn_enable_np: true + +## Enable external vpc support +kube_ovn_enable_external_vpc: true + +## Enable checksum +kube_ovn_encap_checksum: true + +## enable ssl +kube_ovn_enable_ssl: false + +## dpdk +kube_ovn_dpdk_enabled: false +kube_ovn_dpdk_tunnel_iface: br-phy + +## bind local ip +kube_ovn_bind_local_ip_enabled: true + +## eip snat +kube_ovn_eip_snat_enabled: true + +# ls dnat mod dl dst +kube_ovn_ls_dnat_mod_dl_dst: true + +## keep vm ip +kube_ovn_keep_vm_ip: true + +## cni config priority, default: 01 +kube_ovn_cni_config_priority: 01 diff --git a/kubespray/project/roles/network_plugin/kube-ovn/tasks/main.yml b/kubespray/project/roles/network_plugin/kube-ovn/tasks/main.yml new file mode 100644 index 0000000..ab45b62 --- /dev/null +++ b/kubespray/project/roles/network_plugin/kube-ovn/tasks/main.yml @@ -0,0 +1,17 @@ +--- +- name: Kube-OVN | Label ovn-db node + command: "{{ kubectl }} label --overwrite node {{ item }} kube-ovn/role=master" + loop: "{{ kube_ovn_central_hosts }}" + when: + - inventory_hostname == groups['kube_control_plane'][0] + +- name: Kube-OVN | Create Kube-OVN manifests + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/{{ item.file }}" + mode: 0644 + with_items: + - {name: kube-ovn-crd, file: cni-kube-ovn-crd.yml} + - {name: ovn, file: cni-ovn.yml} + - {name: kube-ovn, file: cni-kube-ovn.yml} + register: kube_ovn_node_manifests diff --git a/kubespray/project/roles/network_plugin/kube-ovn/templates/cni-kube-ovn-crd.yml.j2 b/kubespray/project/roles/network_plugin/kube-ovn/templates/cni-kube-ovn-crd.yml.j2 new file mode 100644 index 0000000..379381d --- /dev/null +++ b/kubespray/project/roles/network_plugin/kube-ovn/templates/cni-kube-ovn-crd.yml.j2 @@ -0,0 +1,1533 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: vpc-dnses.kubeovn.io +spec: + group: kubeovn.io + names: + plural: vpc-dnses + singular: vpc-dns + shortNames: + - vpc-dns + kind: VpcDns + listKind: VpcDnsList + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .status.active + name: Active + type: boolean + - jsonPath: .spec.vpc + name: Vpc + type: string + - jsonPath: .spec.subnet + name: Subnet + type: string + name: v1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + vpc: + type: string + subnet: + type: string + status: + type: object + properties: + active: + type: boolean + conditions: + type: array + items: + type: object + properties: + type: + type: string + status: + type: string + reason: + type: string + message: + type: string + lastUpdateTime: + type: string + lastTransitionTime: + type: string +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: switch-lb-rules.kubeovn.io +spec: + group: kubeovn.io + names: + plural: switch-lb-rules + singular: switch-lb-rule + shortNames: + - slr + kind: SwitchLBRule + listKind: SwitchLBRuleList + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.vip + name: vip + type: string + - jsonPath: .status.ports + name: port(s) + type: string + - jsonPath: .status.service + name: service + type: string + - jsonPath: .metadata.creationTimestamp + name: age + type: date + name: v1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + namespace: + type: string + vip: + type: string + sessionAffinity: + type: string + ports: + items: + properties: + name: + type: string + port: + type: integer + minimum: 1 + maximum: 65535 + protocol: + type: string + targetPort: + type: integer + minimum: 1 + maximum: 65535 + type: object + type: array + selector: + items: + type: string + type: array + status: + type: object + properties: + ports: + type: string + service: + type: string +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: vpc-nat-gateways.kubeovn.io +spec: + group: kubeovn.io + names: + plural: vpc-nat-gateways + singular: vpc-nat-gateway + shortNames: + - vpc-nat-gw + kind: VpcNatGateway + listKind: VpcNatGatewayList + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.vpc + name: Vpc + type: string + - jsonPath: .spec.subnet + name: Subnet + type: string + - jsonPath: .spec.lanIp + name: LanIP + type: string + name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + lanIp: + type: string + subnet: + type: string + vpc: + type: string + selector: + type: array + items: + type: string + tolerations: + type: array + items: + type: object + properties: + key: + type: string + operator: + type: string + value: + type: string + effect: + type: string + tolerationSeconds: + type: integer +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: iptables-eips.kubeovn.io +spec: + group: kubeovn.io + names: + plural: iptables-eips + singular: iptables-eip + shortNames: + - eip + kind: IptablesEIP + listKind: IptablesEIPList + scope: Cluster + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + additionalPrinterColumns: + - jsonPath: .status.ip + name: IP + type: string + - jsonPath: .spec.macAddress + name: Mac + type: string + - jsonPath: .status.nat + name: Nat + type: string + - jsonPath: .spec.natGwDp + name: NatGwDp + type: string + - jsonPath: .status.ready + name: Ready + type: boolean + schema: + openAPIV3Schema: + type: object + properties: + status: + type: object + properties: + ready: + type: boolean + ip: + type: string + nat: + type: string + redo: + type: string + conditions: + type: array + items: + type: object + properties: + type: + type: string + status: + type: string + reason: + type: string + message: + type: string + lastUpdateTime: + type: string + lastTransitionTime: + type: string + spec: + type: object + properties: + v4ip: + type: string + v6ip: + type: string + macAddress: + type: string + natGwDp: + type: string +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: iptables-fip-rules.kubeovn.io +spec: + group: kubeovn.io + names: + plural: iptables-fip-rules + singular: iptables-fip-rule + shortNames: + - fip + kind: IptablesFIPRule + listKind: IptablesFIPRuleList + scope: Cluster + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + additionalPrinterColumns: + - jsonPath: .spec.eip + name: Eip + type: string + - jsonPath: .status.v4ip + name: V4ip + type: string + - jsonPath: .spec.internalIp + name: InternalIp + type: string + - jsonPath: .status.v6ip + name: V6ip + type: string + - jsonPath: .status.ready + name: Ready + type: boolean + - jsonPath: .status.natGwDp + name: NatGwDp + type: string + schema: + openAPIV3Schema: + type: object + properties: + status: + type: object + properties: + ready: + type: boolean + v4ip: + type: string + v6ip: + type: string + natGwDp: + type: string + redo: + type: string + conditions: + type: array + items: + type: object + properties: + type: + type: string + status: + type: string + reason: + type: string + message: + type: string + lastUpdateTime: + type: string + lastTransitionTime: + type: string + spec: + type: object + properties: + eip: + type: string + internalIp: + type: string +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: iptables-dnat-rules.kubeovn.io +spec: + group: kubeovn.io + names: + plural: iptables-dnat-rules + singular: iptables-dnat-rule + shortNames: + - dnat + kind: IptablesDnatRule + listKind: IptablesDnatRuleList + scope: Cluster + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + additionalPrinterColumns: + - jsonPath: .spec.eip + name: Eip + type: string + - jsonPath: .spec.protocol + name: Protocol + type: string + - jsonPath: .status.v4ip + name: V4ip + type: string + - jsonPath: .status.v6ip + name: V6ip + type: string + - jsonPath: .spec.internalIp + name: InternalIp + type: string + - jsonPath: .spec.externalPort + name: ExternalPort + type: string + - jsonPath: .spec.internalPort + name: InternalPort + type: string + - jsonPath: .status.natGwDp + name: NatGwDp + type: string + - jsonPath: .status.ready + name: Ready + type: boolean + schema: + openAPIV3Schema: + type: object + properties: + status: + type: object + properties: + ready: + type: boolean + v4ip: + type: string + v6ip: + type: string + natGwDp: + type: string + redo: + type: string + protocol: + type: string + internalIp: + type: string + internalPort: + type: string + externalPort: + type: string + conditions: + type: array + items: + type: object + properties: + type: + type: string + status: + type: string + reason: + type: string + message: + type: string + lastUpdateTime: + type: string + lastTransitionTime: + type: string + spec: + type: object + properties: + eip: + type: string + externalPort: + type: string + protocol: + type: string + internalIp: + type: string + internalPort: + type: string +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: iptables-snat-rules.kubeovn.io +spec: + group: kubeovn.io + names: + plural: iptables-snat-rules + singular: iptables-snat-rule + shortNames: + - snat + kind: IptablesSnatRule + listKind: IptablesSnatRuleList + scope: Cluster + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + additionalPrinterColumns: + - jsonPath: .spec.eip + name: EIP + type: string + - jsonPath: .status.v4ip + name: V4ip + type: string + - jsonPath: .status.v6ip + name: V6ip + type: string + - jsonPath: .spec.internalCIDR + name: InternalCIDR + type: string + - jsonPath: .status.natGwDp + name: NatGwDp + type: string + - jsonPath: .status.ready + name: Ready + type: boolean + schema: + openAPIV3Schema: + type: object + properties: + status: + type: object + properties: + ready: + type: boolean + v4ip: + type: string + v6ip: + type: string + natGwDp: + type: string + redo: + type: string + conditions: + type: array + items: + type: object + properties: + type: + type: string + status: + type: string + reason: + type: string + message: + type: string + lastUpdateTime: + type: string + lastTransitionTime: + type: string + spec: + type: object + properties: + eip: + type: string + internalCIDR: + type: string +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: ovn-eips.kubeovn.io +spec: + group: kubeovn.io + names: + plural: ovn-eips + singular: ovn-eip + shortNames: + - oeip + kind: OvnEip + listKind: OvnEipList + scope: Cluster + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + additionalPrinterColumns: + - jsonPath: .spec.v4ip + name: IP + type: string + - jsonPath: .spec.macAddress + name: Mac + type: string + - jsonPath: .spec.type + name: Type + type: string + schema: + openAPIV3Schema: + type: object + properties: + status: + type: object + properties: + v4Ip: + type: string + macAddress: + type: string + conditions: + type: array + items: + type: object + properties: + type: + type: string + status: + type: string + reason: + type: string + message: + type: string + lastUpdateTime: + type: string + lastTransitionTime: + type: string + spec: + type: object + properties: + externalSubnet: + type: string + type: + type: string + v4ip: + type: string + macAddress: + type: string +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: ovn-fips.kubeovn.io +spec: + group: kubeovn.io + names: + plural: ovn-fips + singular: ovn-fip + shortNames: + - ofip + kind: OvnFip + listKind: OvnFipList + scope: Cluster + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + additionalPrinterColumns: + - jsonPath: .status.vpc + name: Vpc + type: string + - jsonPath: .status.v4Eip + name: V4Eip + type: string + - jsonPath: .status.v4Ip + name: V4Ip + type: string + - jsonPath: .status.ready + name: Ready + type: boolean + schema: + openAPIV3Schema: + type: object + properties: + status: + type: object + properties: + ready: + type: boolean + v4Eip: + type: string + v4Ip: + type: string + macAddress: + type: string + vpc: + type: string + conditions: + type: array + items: + type: object + properties: + type: + type: string + status: + type: string + reason: + type: string + message: + type: string + lastUpdateTime: + type: string + lastTransitionTime: + type: string + spec: + type: object + properties: + ovnEip: + type: string + ipName: + type: string +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: ovn-snat-rules.kubeovn.io +spec: + group: kubeovn.io + names: + plural: ovn-snat-rules + singular: ovn-snat-rule + shortNames: + - osnat + kind: OvnSnatRule + listKind: OvnSnatRuleList + scope: Cluster + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + additionalPrinterColumns: + - jsonPath: .status.vpc + name: Vpc + type: string + - jsonPath: .status.v4Eip + name: V4Eip + type: string + - jsonPath: .status.v4ipCidr + name: V4Ip + type: string + - jsonPath: .status.ready + name: Ready + type: boolean + schema: + openAPIV3Schema: + type: object + properties: + status: + type: object + properties: + ready: + type: boolean + v4Eip: + type: string + v4ipCidr: + type: string + vpc: + type: string + conditions: + type: array + items: + type: object + properties: + type: + type: string + status: + type: string + reason: + type: string + message: + type: string + lastUpdateTime: + type: string + lastTransitionTime: + type: string + spec: + type: object + properties: + ovnEip: + type: string + vpcSubnet: + type: string + ipName: + type: string +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: vpcs.kubeovn.io +spec: + group: kubeovn.io + versions: + - additionalPrinterColumns: + - jsonPath: .status.enableExternal + name: EnableExternal + type: boolean + - jsonPath: .status.standby + name: Standby + type: boolean + - jsonPath: .status.subnets + name: Subnets + type: string + - jsonPath: .spec.namespaces + name: Namespaces + type: string + name: v1 + schema: + openAPIV3Schema: + properties: + spec: + properties: + enableExternal: + type: boolean + namespaces: + items: + type: string + type: array + staticRoutes: + items: + properties: + policy: + type: string + cidr: + type: string + nextHopIP: + type: string + type: object + type: array + policyRoutes: + items: + properties: + priority: + type: integer + action: + type: string + match: + type: string + nextHopIP: + type: string + type: object + type: array + vpcPeerings: + items: + properties: + remoteVpc: + type: string + localConnectIP: + type: string + type: object + type: array + type: object + status: + properties: + conditions: + items: + properties: + lastTransitionTime: + type: string + lastUpdateTime: + type: string + message: + type: string + reason: + type: string + status: + type: string + type: + type: string + type: object + type: array + default: + type: boolean + defaultLogicalSwitch: + type: string + router: + type: string + standby: + type: boolean + enableExternal: + type: boolean + subnets: + items: + type: string + type: array + vpcPeerings: + items: + type: string + type: array + tcpLoadBalancer: + type: string + tcpSessionLoadBalancer: + type: string + udpLoadBalancer: + type: string + udpSessionLoadBalancer: + type: string + sctpLoadBalancer: + type: string + sctpSessionLoadBalancer: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} + names: + kind: Vpc + listKind: VpcList + plural: vpcs + shortNames: + - vpc + singular: vpc + scope: Cluster +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: ips.kubeovn.io +spec: + group: kubeovn.io + versions: + - name: v1 + served: true + storage: true + additionalPrinterColumns: + - name: V4IP + type: string + jsonPath: .spec.v4IpAddress + - name: V6IP + type: string + jsonPath: .spec.v6IpAddress + - name: Mac + type: string + jsonPath: .spec.macAddress + - name: Node + type: string + jsonPath: .spec.nodeName + - name: Subnet + type: string + jsonPath: .spec.subnet + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + podName: + type: string + namespace: + type: string + subnet: + type: string + attachSubnets: + type: array + items: + type: string + nodeName: + type: string + ipAddress: + type: string + v4IpAddress: + type: string + v6IpAddress: + type: string + attachIps: + type: array + items: + type: string + macAddress: + type: string + attachMacs: + type: array + items: + type: string + containerID: + type: string + podType: + type: string + scope: Cluster + names: + plural: ips + singular: ip + kind: IP + shortNames: + - ip +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: vips.kubeovn.io +spec: + group: kubeovn.io + names: + plural: vips + singular: vip + shortNames: + - vip + kind: Vip + listKind: VipList + scope: Cluster + versions: + - name: v1 + served: true + storage: true + additionalPrinterColumns: + - name: V4IP + type: string + jsonPath: .status.v4ip + - name: PV4IP + type: string + jsonPath: .spec.parentV4ip + - name: Mac + type: string + jsonPath: .status.mac + - name: PMac + type: string + jsonPath: .spec.parentMac + - name: V6IP + type: string + jsonPath: .status.v6ip + - name: PV6IP + type: string + jsonPath: .spec.parentV6ip + - name: Subnet + type: string + jsonPath: .spec.subnet + - jsonPath: .status.ready + name: Ready + type: boolean + schema: + openAPIV3Schema: + type: object + properties: + status: + type: object + properties: + ready: + type: boolean + v4ip: + type: string + v6ip: + type: string + mac: + type: string + pv4ip: + type: string + pv6ip: + type: string + pmac: + type: string + conditions: + type: array + items: + type: object + properties: + type: + type: string + status: + type: string + reason: + type: string + message: + type: string + lastUpdateTime: + type: string + lastTransitionTime: + type: string + spec: + type: object + properties: + namespace: + type: string + subnet: + type: string + attachSubnets: + type: array + items: + type: string + v4ip: + type: string + macAddress: + type: string + v6ip: + type: string + parentV4ip: + type: string + parentMac: + type: string + parentV6ip: + type: string +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: subnets.kubeovn.io +spec: + group: kubeovn.io + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + additionalPrinterColumns: + - name: Provider + type: string + jsonPath: .spec.provider + - name: Vpc + type: string + jsonPath: .spec.vpc + - name: Protocol + type: string + jsonPath: .spec.protocol + - name: CIDR + type: string + jsonPath: .spec.cidrBlock + - name: Private + type: boolean + jsonPath: .spec.private + - name: NAT + type: boolean + jsonPath: .spec.natOutgoing + - name: Default + type: boolean + jsonPath: .spec.default + - name: GatewayType + type: string + jsonPath: .spec.gatewayType + - name: V4Used + type: number + jsonPath: .status.v4usingIPs + - name: V4Available + type: number + jsonPath: .status.v4availableIPs + - name: V6Used + type: number + jsonPath: .status.v6usingIPs + - name: V6Available + type: number + jsonPath: .status.v6availableIPs + - name: ExcludeIPs + type: string + jsonPath: .spec.excludeIps + - name: U2OInterconnectionIP + type: string + jsonPath: .status.u2oInterconnectionIP + schema: + openAPIV3Schema: + type: object + properties: + status: + type: object + properties: + v4availableIPs: + type: number + v4usingIPs: + type: number + v6availableIPs: + type: number + v6usingIPs: + type: number + activateGateway: + type: string + dhcpV4OptionsUUID: + type: string + dhcpV6OptionsUUID: + type: string + u2oInterconnectionIP: + type: string + conditions: + type: array + items: + type: object + properties: + type: + type: string + status: + type: string + reason: + type: string + message: + type: string + lastUpdateTime: + type: string + lastTransitionTime: + type: string + spec: + type: object + properties: + vpc: + type: string + default: + type: boolean + protocol: + type: string + enum: + - IPv4 + - IPv6 + - Dual + cidrBlock: + type: string + namespaces: + type: array + items: + type: string + gateway: + type: string + provider: + type: string + excludeIps: + type: array + items: + type: string + vips: + type: array + items: + type: string + gatewayType: + type: string + allowSubnets: + type: array + items: + type: string + gatewayNode: + type: string + natOutgoing: + type: boolean + u2oRouting: + type: boolean + externalEgressGateway: + type: string + policyRoutingPriority: + type: integer + minimum: 1 + maximum: 32765 + policyRoutingTableID: + type: integer + minimum: 1 + maximum: 2147483647 + not: + enum: + - 252 # compat + - 253 # default + - 254 # main + - 255 # local + private: + type: boolean + vlan: + type: string + logicalGateway: + type: boolean + disableGatewayCheck: + type: boolean + disableInterConnection: + type: boolean + enableDHCP: + type: boolean + dhcpV4Options: + type: string + dhcpV6Options: + type: string + enableIPv6RA: + type: boolean + ipv6RAConfigs: + type: string + acls: + type: array + items: + type: object + properties: + direction: + type: string + enum: + - from-lport + - to-lport + priority: + type: integer + minimum: 0 + maximum: 32767 + match: + type: string + action: + type: string + enum: + - allow-related + - allow-stateless + - allow + - drop + - reject + u2oInterconnection: + type: boolean + scope: Cluster + names: + plural: subnets + singular: subnet + kind: Subnet + shortNames: + - subnet +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: vlans.kubeovn.io +spec: + group: kubeovn.io + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + id: + type: integer + minimum: 0 + maximum: 4095 + provider: + type: string + vlanId: + type: integer + description: Deprecated in favor of id + providerInterfaceName: + type: string + description: Deprecated in favor of provider + required: + - provider + status: + type: object + properties: + subnets: + type: array + items: + type: string + additionalPrinterColumns: + - name: ID + type: string + jsonPath: .spec.id + - name: Provider + type: string + jsonPath: .spec.provider + scope: Cluster + names: + plural: vlans + singular: vlan + kind: Vlan + shortNames: + - vlan +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: provider-networks.kubeovn.io +spec: + group: kubeovn.io + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + properties: + metadata: + type: object + properties: + name: + type: string + maxLength: 12 + not: + enum: + - int + - external + spec: + type: object + properties: + defaultInterface: + type: string + maxLength: 15 + pattern: '^[^/\s]+$' + customInterfaces: + type: array + items: + type: object + properties: + interface: + type: string + maxLength: 15 + pattern: '^[^/\s]+$' + nodes: + type: array + items: + type: string + exchangeLinkName: + type: boolean + excludeNodes: + type: array + items: + type: string + required: + - defaultInterface + status: + type: object + properties: + ready: + type: boolean + readyNodes: + type: array + items: + type: string + notReadyNodes: + type: array + items: + type: string + vlans: + type: array + items: + type: string + conditions: + type: array + items: + type: object + properties: + node: + type: string + type: + type: string + status: + type: string + reason: + type: string + message: + type: string + lastUpdateTime: + type: string + lastTransitionTime: + type: string + additionalPrinterColumns: + - name: DefaultInterface + type: string + jsonPath: .spec.defaultInterface + - name: Ready + type: boolean + jsonPath: .status.ready + scope: Cluster + names: + plural: provider-networks + singular: provider-network + kind: ProviderNetwork + listKind: ProviderNetworkList +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: security-groups.kubeovn.io +spec: + group: kubeovn.io + names: + plural: security-groups + singular: security-group + shortNames: + - sg + kind: SecurityGroup + listKind: SecurityGroupList + scope: Cluster + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + ingressRules: + type: array + items: + type: object + properties: + ipVersion: + type: string + protocol: + type: string + priority: + type: integer + remoteType: + type: string + remoteAddress: + type: string + remoteSecurityGroup: + type: string + portRangeMin: + type: integer + portRangeMax: + type: integer + policy: + type: string + egressRules: + type: array + items: + type: object + properties: + ipVersion: + type: string + protocol: + type: string + priority: + type: integer + remoteType: + type: string + remoteAddress: + type: string + remoteSecurityGroup: + type: string + portRangeMin: + type: integer + portRangeMax: + type: integer + policy: + type: string + allowSameGroupTraffic: + type: boolean + status: + type: object + properties: + portGroup: + type: string + allowSameGroupTraffic: + type: boolean + ingressMd5: + type: string + egressMd5: + type: string + ingressLastSyncSuccess: + type: boolean + egressLastSyncSuccess: + type: boolean + subresources: + status: {} + conversion: + strategy: None \ No newline at end of file diff --git a/kubespray/project/roles/network_plugin/kube-ovn/templates/cni-kube-ovn.yml.j2 b/kubespray/project/roles/network_plugin/kube-ovn/templates/cni-kube-ovn.yml.j2 new file mode 100644 index 0000000..cee7ccb --- /dev/null +++ b/kubespray/project/roles/network_plugin/kube-ovn/templates/cni-kube-ovn.yml.j2 @@ -0,0 +1,673 @@ +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: kube-ovn-controller + namespace: kube-system + annotations: + kubernetes.io/description: | + kube-ovn controller +spec: + replicas: {{ kube_ovn_controller_replics }} + selector: + matchLabels: + app: kube-ovn-controller + strategy: + rollingUpdate: + maxSurge: 0% + maxUnavailable: 100% + type: RollingUpdate + template: + metadata: + labels: + app: kube-ovn-controller + component: network + type: infra + spec: + tolerations: + - operator: Exists + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app: kube-ovn-controller + topologyKey: kubernetes.io/hostname + priorityClassName: system-cluster-critical + serviceAccountName: ovn + hostNetwork: true + containers: + - name: kube-ovn-controller + image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + args: + - /kube-ovn/start-controller.sh + - --default-cidr={{ kube_pods_subnet }}{% if enable_dual_stack_networks %},{{ kube_ovn_pool_cidr_ipv6 | default(kube_pods_subnet_ipv6) }}{% endif %}{{ '' }} + - --default-gateway={% if kube_ovn_default_gateway is defined %}{{ kube_ovn_default_gateway }}{% endif %}{{ '' }} + - --default-gateway-check={{ kube_ovn_default_gateway_check | string }} + - --default-logical-gateway={{ kube_ovn_default_logical_gateway | string }} + - --default-u2o-interconnection={{ kube_ovn_u2o_interconnection }} + - --default-exclude-ips={% if kube_ovn_default_exclude_ips is defined %}{{ kube_ovn_default_exclude_ips }}{% endif %}{{ '' }} + - --node-switch-cidr={{ kube_ovn_node_switch_cidr }}{% if enable_dual_stack_networks %},{{ kube_ovn_node_switch_cidr_ipv6 }}{% endif %}{{ '' }} + - --service-cluster-ip-range={{ kube_service_addresses }}{% if enable_dual_stack_networks %},{{ kube_service_addresses_ipv6 }}{% endif %}{{ '' }} + - --network-type={{ kube_ovn_network_type }} + - --default-interface-name={{ kube_ovn_default_interface_name | default('') }} + - --default-vlan-id={{ kube_ovn_default_vlan_id }} + - --ls-dnat-mod-dl-dst={{ kube_ovn_ls_dnat_mod_dl_dst }} + - --pod-nic-type={{ kube_ovn_pod_nic_type }} + - --enable-lb={{ kube_ovn_enable_lb | string }} + - --enable-np={{ kube_ovn_enable_np | string }} + - --enable-eip-snat={{ kube_ovn_eip_snat_enabled }} + - --enable-external-vpc={{ kube_ovn_enable_external_vpc | string }} + - --logtostderr=false + - --alsologtostderr=true + - --gc-interval=360 + - --inspect-interval=20 + - --log_file=/var/log/kube-ovn/kube-ovn-controller.log + - --log_file_max_size=0 + - --enable-lb-svc=false + - --keep-vm-ip={{ kube_ovn_keep_vm_ip }} + - --pod-default-fip-type="" + env: + - name: ENABLE_SSL + value: "{{ kube_ovn_enable_ssl | lower }}" + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: KUBE_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: OVN_DB_IPS + value: "{{ kube_ovn_central_ips }}" + - name: POD_IPS + valueFrom: + fieldRef: + fieldPath: status.podIPs + - name: ENABLE_BIND_LOCAL_IP + value: "{{ kube_ovn_bind_local_ip_enabled }}" + volumeMounts: + - mountPath: /etc/localtime + name: localtime + - mountPath: /var/log/kube-ovn + name: kube-ovn-log + - mountPath: /var/run/tls + name: kube-ovn-tls + readinessProbe: + exec: + command: + - /kube-ovn/kube-ovn-controller-healthcheck + periodSeconds: 3 + timeoutSeconds: 45 + livenessProbe: + exec: + command: + - /kube-ovn/kube-ovn-controller-healthcheck + initialDelaySeconds: 300 + periodSeconds: 7 + failureThreshold: 5 + timeoutSeconds: 45 + resources: + requests: + cpu: {{ kube_ovn_controller_cpu_request }} + memory: {{ kube_ovn_controller_memory_request }} + limits: + cpu: {{ kube_ovn_controller_cpu_limit }} + memory: {{ kube_ovn_controller_memory_limit }} + nodeSelector: + kubernetes.io/os: "linux" + volumes: + - name: localtime + hostPath: + path: /etc/localtime + - name: kube-ovn-log + hostPath: + path: /var/log/kube-ovn + - name: kube-ovn-tls + secret: + optional: true + secretName: kube-ovn-tls + +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: kube-ovn-cni + namespace: kube-system + annotations: + kubernetes.io/description: | + This daemon set launches the kube-ovn cni daemon. +spec: + selector: + matchLabels: + app: kube-ovn-cni + template: + metadata: + labels: + app: kube-ovn-cni + component: network + type: infra + spec: + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + - key: CriticalAddonsOnly + operator: Exists + priorityClassName: system-node-critical + serviceAccountName: ovn + hostNetwork: true + hostPID: true + initContainers: + - name: install-cni + image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + command: ["/kube-ovn/install-cni.sh"] + securityContext: + runAsUser: 0 + privileged: true + volumeMounts: + - mountPath: /opt/cni/bin + name: cni-bin + - mountPath: /usr/local/bin + name: local-bin + containers: + - name: cni-server + image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + command: + - bash + - /kube-ovn/start-cniserver.sh + args: + - --enable-mirror={{ kube_ovn_traffic_mirror | lower }} + - --encap-checksum={{ kube_ovn_encap_checksum | lower }} + - --service-cluster-ip-range={{ kube_service_addresses }}{% if enable_dual_stack_networks %},{{ kube_service_addresses_ipv6 }}{% endif %}{{ '' }} + - --iface={{ kube_ovn_iface | default('') }} + - --dpdk-tunnel-iface={{ kube_ovn_dpdk_tunnel_iface }} + - --network-type={{ kube_ovn_network_type }} + - --default-interface-name={{ kube_ovn_default_interface_name | default('') }} + {% if kube_ovn_mtu is defined %} + - --mtu={{ kube_ovn_mtu }} +{% endif %} + - --cni-conf-name={{ kube_ovn_cni_config_priority }}-kube-ovn.conflist + - --logtostderr=false + - --alsologtostderr=true + - --log_file=/var/log/kube-ovn/kube-ovn-cni.log + - --log_file_max_size=0 + securityContext: + runAsUser: 0 + privileged: true + env: + - name: ENABLE_SSL + value: "{{ kube_ovn_enable_ssl | lower }}" + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: MODULES + value: kube_ovn_fastpath.ko + - name: RPMS + value: openvswitch-kmod + - name: POD_IPS + valueFrom: + fieldRef: + fieldPath: status.podIPs + - name: ENABLE_BIND_LOCAL_IP + value: "{{ kube_ovn_bind_local_ip_enabled }}" + - name: DBUS_SYSTEM_BUS_ADDRESS + value: "unix:path=/host/var/run/dbus/system_bus_socket" + volumeMounts: + - name: host-modules + mountPath: /lib/modules + readOnly: true + - name: shared-dir + mountPath: /var/lib/kubelet/pods + - mountPath: /etc/openvswitch + name: systemid + - mountPath: /etc/cni/net.d + name: cni-conf + - mountPath: /run/openvswitch + name: host-run-ovs + mountPropagation: Bidirectional + - mountPath: /run/ovn + name: host-run-ovn + - mountPath: /host/var/run/dbus + name: host-dbus + mountPropagation: HostToContainer + - mountPath: /var/run/netns + name: host-ns + mountPropagation: HostToContainer + - mountPath: /var/log/kube-ovn + name: kube-ovn-log + - mountPath: /var/log/openvswitch + name: host-log-ovs + - mountPath: /var/log/ovn + name: host-log-ovn + - mountPath: /etc/localtime + name: localtime + - mountPath: /tmp + name: tmp + livenessProbe: + failureThreshold: 3 + initialDelaySeconds: 30 + periodSeconds: 7 + successThreshold: 1 + tcpSocket: + port: 10665 + timeoutSeconds: 3 + readinessProbe: + failureThreshold: 3 + initialDelaySeconds: 30 + periodSeconds: 7 + successThreshold: 1 + tcpSocket: + port: 10665 + timeoutSeconds: 3 + resources: + requests: + cpu: {{ kube_ovn_cni_server_cpu_request }} + memory: {{ kube_ovn_cni_server_memory_request }} + limits: + cpu: {{ kube_ovn_cni_server_cpu_limit }} + memory: {{ kube_ovn_cni_server_memory_limit }} + nodeSelector: + kubernetes.io/os: "linux" + volumes: + - name: host-modules + hostPath: + path: /lib/modules + - name: shared-dir + hostPath: + path: /var/lib/kubelet/pods + - name: systemid + hostPath: + path: /etc/origin/openvswitch + - name: host-run-ovs + hostPath: + path: /run/openvswitch + - name: host-run-ovn + hostPath: + path: /run/ovn + - name: cni-conf + hostPath: + path: /etc/cni/net.d + - name: cni-bin + hostPath: + path: /opt/cni/bin + - name: host-ns + hostPath: + path: /var/run/netns + - name: host-dbus + hostPath: + path: /var/run/dbus + - name: host-log-ovs + hostPath: + path: /var/log/openvswitch + - name: kube-ovn-log + hostPath: + path: /var/log/kube-ovn + - name: host-log-ovn + hostPath: + path: /var/log/ovn + - name: localtime + hostPath: + path: /etc/localtime + - name: tmp + hostPath: + path: /tmp + - name: local-bin + hostPath: + path: /usr/local/bin +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: kube-ovn-pinger + namespace: kube-system + annotations: + kubernetes.io/description: | + This daemon set launches the openvswitch daemon. +spec: + selector: + matchLabels: + app: kube-ovn-pinger + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + app: kube-ovn-pinger + component: network + type: infra + spec: + priorityClassName: system-node-critical + serviceAccountName: ovn + hostPID: true + containers: + - name: pinger + image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }} + command: + - /kube-ovn/kube-ovn-pinger + args: + - --external-address={{ kube_ovn_external_address }}{% if enable_dual_stack_networks %},{{ kube_ovn_external_address_ipv6 }}{% endif %}{{ '' }} + - --external-dns={{ kube_ovn_external_dns }} + - --logtostderr=false + - --alsologtostderr=true + - --log_file=/var/log/kube-ovn/kube-ovn-pinger.log + - --log_file_max_size=0 + imagePullPolicy: {{ k8s_image_pull_policy }} + securityContext: + runAsUser: 0 + privileged: false + env: + - name: ENABLE_SSL + value: "{{ kube_ovn_enable_ssl | lower }}" + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - mountPath: /lib/modules + name: host-modules + readOnly: true + - mountPath: /run/openvswitch + name: host-run-ovs + - mountPath: /var/run/openvswitch + name: host-run-ovs + - mountPath: /var/run/ovn + name: host-run-ovn + - mountPath: /sys + name: host-sys + readOnly: true + - mountPath: /etc/openvswitch + name: host-config-openvswitch + - mountPath: /var/log/openvswitch + name: host-log-ovs + - mountPath: /var/log/ovn + name: host-log-ovn + - mountPath: /var/log/kube-ovn + name: kube-ovn-log + - mountPath: /etc/localtime + name: localtime + - mountPath: /var/run/tls + name: kube-ovn-tls + resources: + requests: + cpu: {{ kube_ovn_pinger_cpu_request }} + memory: {{ kube_ovn_pinger_memory_request }} + limits: + cpu: {{ kube_ovn_pinger_cpu_limit }} + memory: {{ kube_ovn_pinger_memory_limit }} + nodeSelector: + kubernetes.io/os: "linux" + volumes: + - name: host-modules + hostPath: + path: /lib/modules + - name: host-run-ovs + hostPath: + path: /run/openvswitch + - name: host-run-ovn + hostPath: + path: /run/ovn + - name: host-sys + hostPath: + path: /sys + - name: host-config-openvswitch + hostPath: + path: /etc/origin/openvswitch + - name: host-log-ovs + hostPath: + path: /var/log/openvswitch + - name: kube-ovn-log + hostPath: + path: /var/log/kube-ovn + - name: host-log-ovn + hostPath: + path: /var/log/ovn + - name: localtime + hostPath: + path: /etc/localtime + - name: kube-ovn-tls + secret: + optional: true + secretName: kube-ovn-tls +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: kube-ovn-monitor + namespace: kube-system + annotations: + kubernetes.io/description: | + Metrics for OVN components: northd, nb and sb. +spec: + replicas: 1 + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: RollingUpdate + selector: + matchLabels: + app: kube-ovn-monitor + template: + metadata: + labels: + app: kube-ovn-monitor + component: network + type: infra + spec: + tolerations: + - effect: NoSchedule + operator: Exists + - key: CriticalAddonsOnly + operator: Exists + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app: kube-ovn-monitor + topologyKey: kubernetes.io/hostname + priorityClassName: system-cluster-critical + serviceAccountName: ovn + hostNetwork: true + containers: + - name: kube-ovn-monitor + image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + command: ["/kube-ovn/start-ovn-monitor.sh"] + securityContext: + runAsUser: 0 + privileged: false + env: + - name: ENABLE_SSL + value: "{{ kube_ovn_enable_ssl | lower }}" + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_IPS + valueFrom: + fieldRef: + fieldPath: status.podIPs + - name: ENABLE_BIND_LOCAL_IP + value: "{{ kube_ovn_bind_local_ip_enabled }}" + resources: + requests: + cpu: {{ kube_ovn_monitor_cpu_request }} + memory: {{ kube_ovn_monitor_memory_request }} + limits: + cpu: {{ kube_ovn_monitor_cpu_limit }} + memory: {{ kube_ovn_monitor_memory_limit }} + volumeMounts: + - mountPath: /var/run/openvswitch + name: host-run-ovs + - mountPath: /var/run/ovn + name: host-run-ovn + - mountPath: /etc/openvswitch + name: host-config-openvswitch + - mountPath: /etc/ovn + name: host-config-ovn + - mountPath: /var/log/openvswitch + name: host-log-ovs + - mountPath: /var/log/ovn + name: host-log-ovn + - mountPath: /etc/localtime + name: localtime + - mountPath: /var/run/tls + name: kube-ovn-tls + readinessProbe: + exec: + command: + - cat + - /var/run/ovn/ovn-controller.pid + periodSeconds: 10 + timeoutSeconds: 45 + livenessProbe: + exec: + command: + - cat + - /var/run/ovn/ovn-controller.pid + initialDelaySeconds: 30 + periodSeconds: 10 + failureThreshold: 5 + timeoutSeconds: 45 + nodeSelector: + kubernetes.io/os: "linux" + kube-ovn/role: "master" + volumes: + - name: host-run-ovs + hostPath: + path: /run/openvswitch + - name: host-run-ovn + hostPath: + path: /run/ovn + - name: host-config-openvswitch + hostPath: + path: /etc/origin/openvswitch + - name: host-config-ovn + hostPath: + path: /etc/origin/ovn + - name: host-log-ovs + hostPath: + path: /var/log/openvswitch + - name: host-log-ovn + hostPath: + path: /var/log/ovn + - name: localtime + hostPath: + path: /etc/localtime + - name: kube-ovn-tls + secret: + optional: true + secretName: kube-ovn-tls +--- +kind: Service +apiVersion: v1 +metadata: + name: kube-ovn-monitor + namespace: kube-system + labels: + app: kube-ovn-monitor +spec: + ports: + - name: metrics + port: 10661 + type: ClusterIP +{% if enable_dual_stack_networks %} + ipFamilyPolicy: PreferDualStack +{% endif %} + selector: + app: kube-ovn-monitor + sessionAffinity: None +--- +kind: Service +apiVersion: v1 +metadata: + name: kube-ovn-pinger + namespace: kube-system + labels: + app: kube-ovn-pinger +spec: +{% if enable_dual_stack_networks %} + ipFamilyPolicy: PreferDualStack +{% endif %} + selector: + app: kube-ovn-pinger + ports: + - port: 8080 + name: metrics +--- +kind: Service +apiVersion: v1 +metadata: + name: kube-ovn-controller + namespace: kube-system + labels: + app: kube-ovn-controller +spec: +{% if enable_dual_stack_networks %} + ipFamilyPolicy: PreferDualStack +{% endif %} + selector: + app: kube-ovn-controller + ports: + - port: 10660 + name: metrics +--- +kind: Service +apiVersion: v1 +metadata: + name: kube-ovn-cni + namespace: kube-system + labels: + app: kube-ovn-cni +spec: +{% if enable_dual_stack_networks %} + ipFamilyPolicy: PreferDualStack +{% endif %} + selector: + app: kube-ovn-cni + ports: + - port: 10665 + name: metrics + {% if kube_ovn_ic_enable %} +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: ovn-ic-config + namespace: kube-system +data: + enable-ic: "{{ kube_ovn_ic_enable | lower }}" + az-name: "{{ kube_ovn_ic_zone }}" + ic-db-host: "{{ kube_ovn_ic_dbhost }}" + ic-nb-port: "6645" + ic-sb-port: "6646" + gw-nodes: "{{ kube_ovn_central_hosts | join(',') }}" + auto-route: "{{ kube_ovn_ic_autoroute | lower }}" +{% endif %} diff --git a/kubespray/project/roles/network_plugin/kube-ovn/templates/cni-ovn.yml.j2 b/kubespray/project/roles/network_plugin/kube-ovn/templates/cni-ovn.yml.j2 new file mode 100644 index 0000000..d632f3b --- /dev/null +++ b/kubespray/project/roles/network_plugin/kube-ovn/templates/cni-ovn.yml.j2 @@ -0,0 +1,517 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ovn + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + annotations: + rbac.authorization.k8s.io/system-only: "true" + name: system:ovn +rules: + - apiGroups: + - "kubeovn.io" + resources: + - vpcs + - vpcs/status + - vpc-nat-gateways + - subnets + - subnets/status + - ips + - vips + - vips/status + - vlans + - vlans/status + - provider-networks + - provider-networks/status + - security-groups + - security-groups/status + - iptables-eips + - iptables-fip-rules + - iptables-dnat-rules + - iptables-snat-rules + - iptables-eips/status + - iptables-fip-rules/status + - iptables-dnat-rules/status + - iptables-snat-rules/status + - ovn-eips + - ovn-fips + - ovn-snat-rules + - ovn-eips/status + - ovn-fips/status + - ovn-snat-rules/status + - switch-lb-rules + - switch-lb-rules/status + - vpc-dnses + - vpc-dnses/status + verbs: + - "*" + - apiGroups: + - "" + resources: + - pods + - pods/exec + - namespaces + - nodes + - configmaps + verbs: + - create + - get + - list + - watch + - patch + - update + - apiGroups: + - "k8s.cni.cncf.io" + resources: + - network-attachment-definitions + verbs: + - create + - delete + - get + - list + - update + - apiGroups: + - "" + - networking.k8s.io + - apps + - extensions + resources: + - networkpolicies + - services + - services/status + - endpoints + - statefulsets + - daemonsets + - deployments + - deployments/scale + verbs: + - create + - delete + - update + - patch + - get + - list + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - "*" + - apiGroups: + - "kubevirt.io" + resources: + - virtualmachines + - virtualmachineinstances + verbs: + - get + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: ovn +roleRef: + name: system:ovn + kind: ClusterRole + apiGroup: rbac.authorization.k8s.io +subjects: + - kind: ServiceAccount + name: ovn + namespace: kube-system +--- +kind: Service +apiVersion: v1 +metadata: + name: ovn-nb + namespace: kube-system +spec: + ports: + - name: ovn-nb + protocol: TCP + port: 6641 + targetPort: 6641 + type: ClusterIP +{% if enable_dual_stack_networks %} + ipFamilyPolicy: PreferDualStack +{% endif %} + selector: + app: ovn-central + ovn-nb-leader: "true" + sessionAffinity: None +--- +kind: Service +apiVersion: v1 +metadata: + name: ovn-sb + namespace: kube-system +spec: + ports: + - name: ovn-sb + protocol: TCP + port: 6642 + targetPort: 6642 + type: ClusterIP +{% if enable_dual_stack_networks %} + ipFamilyPolicy: PreferDualStack +{% endif %} + selector: + app: ovn-central + ovn-sb-leader: "true" + sessionAffinity: None +--- +kind: Service +apiVersion: v1 +metadata: + name: ovn-northd + namespace: kube-system +spec: + ports: + - name: ovn-northd + protocol: TCP + port: 6643 + targetPort: 6643 + type: ClusterIP +{% if enable_dual_stack_networks %} + ipFamilyPolicy: PreferDualStack +{% endif %} + selector: + app: ovn-central + ovn-northd-leader: "true" + sessionAffinity: None +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: ovn-central + namespace: kube-system + annotations: + kubernetes.io/description: | + OVN components: northd, nb and sb. +spec: + replicas: {{ kube_ovn_central_replics }} + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + type: RollingUpdate + selector: + matchLabels: + app: ovn-central + template: + metadata: + labels: + app: ovn-central + component: network + type: infra + spec: + tolerations: + - operator: Exists + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app: ovn-central + topologyKey: kubernetes.io/hostname + priorityClassName: system-cluster-critical + serviceAccountName: ovn + hostNetwork: true + containers: + - name: ovn-central + image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + command: ["/kube-ovn/start-db.sh"] + securityContext: + capabilities: + add: ["SYS_NICE"] + env: + - name: ENABLE_SSL + value: "{{ kube_ovn_enable_ssl | lower }}" + - name: NODE_IPS + value: "{{ kube_ovn_central_ips }}" + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_IPS + valueFrom: + fieldRef: + fieldPath: status.podIPs + - name: ENABLE_BIND_LOCAL_IP + value: "{{ kube_ovn_bind_local_ip_enabled }}" + resources: + requests: + cpu: {{ kube_ovn_db_cpu_request }} + memory: {{ kube_ovn_db_memory_request }} + limits: + cpu: {{ kube_ovn_db_cpu_limit }} + memory: {{ kube_ovn_db_memory_limit }} + volumeMounts: + - mountPath: /var/run/openvswitch + name: host-run-ovs + - mountPath: /var/run/ovn + name: host-run-ovn + - mountPath: /sys + name: host-sys + readOnly: true + - mountPath: /etc/openvswitch + name: host-config-openvswitch + - mountPath: /etc/ovn + name: host-config-ovn + - mountPath: /var/log/openvswitch + name: host-log-ovs + - mountPath: /var/log/ovn + name: host-log-ovn + - mountPath: /etc/localtime + name: localtime + - mountPath: /var/run/tls + name: kube-ovn-tls + readinessProbe: + exec: + command: + - bash + - /kube-ovn/ovn-healthcheck.sh + periodSeconds: 15 + timeoutSeconds: 45 + livenessProbe: + exec: + command: + - bash + - /kube-ovn/ovn-healthcheck.sh + initialDelaySeconds: 30 + periodSeconds: 15 + failureThreshold: 5 + timeoutSeconds: 45 + nodeSelector: + kubernetes.io/os: "linux" + kube-ovn/role: "master" + volumes: + - name: host-run-ovs + hostPath: + path: /run/openvswitch + - name: host-run-ovn + hostPath: + path: /run/ovn + - name: host-sys + hostPath: + path: /sys + - name: host-config-openvswitch + hostPath: + path: /etc/origin/openvswitch + - name: host-config-ovn + hostPath: + path: /etc/origin/ovn + - name: host-log-ovs + hostPath: + path: /var/log/openvswitch + - name: host-log-ovn + hostPath: + path: /var/log/ovn + - name: localtime + hostPath: + path: /etc/localtime + - name: kube-ovn-tls + secret: + optional: true + secretName: kube-ovn-tls +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: ovs-ovn + namespace: kube-system + annotations: + kubernetes.io/description: | + This daemon set launches the openvswitch daemon. +spec: + selector: + matchLabels: + app: ovs + updateStrategy: + type: OnDelete + template: + metadata: + labels: + app: ovs + component: network + type: infra + spec: + tolerations: + - operator: Exists + priorityClassName: system-node-critical + serviceAccountName: ovn + hostNetwork: true + hostPID: true + containers: + - name: openvswitch + image: {% if kube_ovn_dpdk_enabled %}{{ kube_ovn_dpdk_container_image_repo }}:{{ kube_ovn_dpdk_container_image_tag }}{% else %}{{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }}{% endif %} + + imagePullPolicy: {{ k8s_image_pull_policy }} + command: [{% if kube_ovn_dpdk_enabled %}"/kube-ovn/start-ovs-dpdk.sh"{% else %}"/kube-ovn/start-ovs.sh"{% endif %}] + securityContext: + runAsUser: 0 + privileged: true + env: + - name: ENABLE_SSL + value: "{{ kube_ovn_enable_ssl | lower }}" + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP +{% if not kube_ovn_dpdk_enabled %} + - name: HW_OFFLOAD + value: "{{ kube_ovn_hw_offload | string | lower }}" + - name: TUNNEL_TYPE + value: "{{ kube_ovn_tunnel_type }}" +{% endif %} + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: OVN_DB_IPS + value: "{{ kube_ovn_central_ips }}" + volumeMounts: + - mountPath: /var/run/netns + name: host-ns + mountPropagation: HostToContainer + - mountPath: /lib/modules + name: host-modules + readOnly: true + - mountPath: /var/run/openvswitch + name: host-run-ovs + - mountPath: /var/run/ovn + name: host-run-ovn + - mountPath: /sys + name: host-sys + readOnly: true + - mountPath: /etc/cni/net.d + name: cni-conf + - mountPath: /etc/openvswitch + name: host-config-openvswitch + - mountPath: /etc/ovn + name: host-config-ovn + - mountPath: /var/log/openvswitch + name: host-log-ovs + - mountPath: /var/log/ovn + name: host-log-ovn +{% if kube_ovn_dpdk_enabled %} + - mountPath: /opt/ovs-config + name: host-config-ovs + - mountPath: /dev/hugepages + name: hugepage +{% endif %} + - mountPath: /etc/localtime + name: localtime + - mountPath: /var/run/tls + name: kube-ovn-tls + readinessProbe: + exec: + command: + - bash +{% if kube_ovn_dpdk_enabled %} + - /kube-ovn/ovs-dpdk-healthcheck.sh +{% else %} + - /kube-ovn/ovs-healthcheck.sh +{% endif %} + periodSeconds: 5 + timeoutSeconds: 45 + livenessProbe: + exec: + command: + - bash +{% if kube_ovn_dpdk_enabled %} + - /kube-ovn/ovs-dpdk-healthcheck.sh +{% else %} + - /kube-ovn/ovs-healthcheck.sh +{% endif %} + initialDelaySeconds: 60 + periodSeconds: 5 + failureThreshold: 5 + timeoutSeconds: 45 + resources: +{% if kube_ovn_dpdk_enabled %} + requests: + cpu: {{ kube_ovn_dpdk_node_cpu_request }} + memory: {{ kube_ovn_dpdk_node_memory_request }} + limits: + cpu: {{ kube_ovn_dpdk_node_cpu_limit }} + memory: {{ kube_ovn_dpdk_node_memory_limit }} + hugepages-1Gi: 1Gi +{% else %} + requests: + cpu: {{ kube_ovn_node_cpu_request }} + memory: {{ kube_ovn_node_memory_request }} + limits: + cpu: {{ kube_ovn_node_cpu_limit }} + memory: {{ kube_ovn_node_memory_limit }} +{% endif %} + nodeSelector: + kubernetes.io/os: "linux" + volumes: + - name: host-modules + hostPath: + path: /lib/modules + - name: host-run-ovs + hostPath: + path: /run/openvswitch + - name: host-run-ovn + hostPath: + path: /run/ovn + - name: host-sys + hostPath: + path: /sys + - name: host-ns + hostPath: + path: /var/run/netns + - name: cni-conf + hostPath: + path: /etc/cni/net.d + - name: host-config-openvswitch + hostPath: + path: /etc/origin/openvswitch + - name: host-config-ovn + hostPath: + path: /etc/origin/ovn + - name: host-log-ovs + hostPath: + path: /var/log/openvswitch + - name: host-log-ovn + hostPath: + path: /var/log/ovn +{% if kube_ovn_dpdk_enabled %} + - name: host-config-ovs + hostPath: + path: /opt/ovs-config + type: DirectoryOrCreate + - name: hugepage + emptyDir: + medium: HugePages +{% endif %} + - name: localtime + hostPath: + path: /etc/localtime + - name: kube-ovn-tls + secret: + optional: true + secretName: kube-ovn-tls diff --git a/kubespray/project/roles/network_plugin/kube-router/OWNERS b/kubespray/project/roles/network_plugin/kube-router/OWNERS new file mode 100644 index 0000000..c40af3c --- /dev/null +++ b/kubespray/project/roles/network_plugin/kube-router/OWNERS @@ -0,0 +1,6 @@ +# See the OWNERS docs at https://go.k8s.io/owners + +approvers: + - bozzo +reviewers: + - bozzo diff --git a/kubespray/project/roles/network_plugin/kube-router/defaults/main.yml b/kubespray/project/roles/network_plugin/kube-router/defaults/main.yml new file mode 100644 index 0000000..c01a353 --- /dev/null +++ b/kubespray/project/roles/network_plugin/kube-router/defaults/main.yml @@ -0,0 +1,69 @@ +--- +# Enables Pod Networking -- Advertises and learns the routes to Pods via iBGP +kube_router_run_router: true + +# Enables Network Policy -- sets up iptables to provide ingress firewall for pods +kube_router_run_firewall: true + +# Enables Service Proxy -- sets up IPVS for Kubernetes Services +# see docs/kube-router.md "Caveats" section +kube_router_run_service_proxy: false + +# Add Cluster IP of the service to the RIB so that it gets advertises to the BGP peers. +kube_router_advertise_cluster_ip: false + +# Add External IP of service to the RIB so that it gets advertised to the BGP peers. +kube_router_advertise_external_ip: false + +# Add LoadBalancer IP of service status as set by the LB provider to the RIB so that it gets advertised to the BGP peers. +kube_router_advertise_loadbalancer_ip: false + +# Enables BGP graceful restarts +kube_router_bgp_graceful_restart: true + +# Adjust manifest of kube-router daemonset template with DSR needed changes +kube_router_enable_dsr: false + +# Array of arbitrary extra arguments to kube-router, see +# https://github.com/cloudnativelabs/kube-router/blob/master/docs/user-guide.md +kube_router_extra_args: [] + +# ASN number of the cluster, used when communicating with external BGP routers +kube_router_cluster_asn: ~ + +# ASN numbers of the BGP peer to which cluster nodes will advertise cluster ip and node's pod cidr. +kube_router_peer_router_asns: ~ + +# The ip address of the external router to which all nodes will peer and advertise the cluster ip and pod cidr's. +kube_router_peer_router_ips: ~ + +# The remote port of the external BGP to which all nodes will peer. If not set, default BGP port (179) will be used. +kube_router_peer_router_ports: ~ + +# Setups node CNI to allow hairpin mode, requires node reboots, see +# https://github.com/cloudnativelabs/kube-router/blob/master/docs/user-guide.md#hairpin-mode +kube_router_support_hairpin_mode: false + +# Select DNS Policy ClusterFirstWithHostNet, ClusterFirst, etc. +kube_router_dns_policy: ClusterFirstWithHostNet + +# Adds annotations to kubernetes nodes for advanced configuration of BGP Peers. +# https://github.com/cloudnativelabs/kube-router/blob/master/docs/bgp.md + +# Array of annotations for master +kube_router_annotations_master: [] + +# Array of annotations for every node +kube_router_annotations_node: [] + +# Array of common annotations for every node +kube_router_annotations_all: [] + +# Enables scraping kube-router metrics with Prometheus +kube_router_enable_metrics: false + +# Path to serve Prometheus metrics on +kube_router_metrics_path: /metrics + +# Prometheus metrics port to use +kube_router_metrics_port: 9255 diff --git a/kubespray/project/roles/network_plugin/kube-router/handlers/main.yml b/kubespray/project/roles/network_plugin/kube-router/handlers/main.yml new file mode 100644 index 0000000..ad5eb21 --- /dev/null +++ b/kubespray/project/roles/network_plugin/kube-router/handlers/main.yml @@ -0,0 +1,20 @@ +--- +- name: Kube-router | delete kube-router docker containers + shell: "set -o pipefail && {{ docker_bin_dir }}/docker ps -af name=k8s_POD_kube-router* -q | xargs --no-run-if-empty docker rm -f" + args: + executable: /bin/bash + register: docker_kube_router_remove + until: docker_kube_router_remove is succeeded + retries: 5 + when: container_manager in ["docker"] + listen: Reset_kube_router + +- name: Kube-router | delete kube-router crio/containerd containers + shell: 'set -o pipefail && {{ bin_dir }}/crictl pods --name kube-router* -q | xargs -I% --no-run-if-empty bash -c "{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %"' + args: + executable: /bin/bash + register: crictl_kube_router_remove + until: crictl_kube_router_remove is succeeded + retries: 5 + when: container_manager in ["crio", "containerd"] + listen: Reset_kube_router diff --git a/kubespray/project/roles/network_plugin/kube-router/meta/main.yml b/kubespray/project/roles/network_plugin/kube-router/meta/main.yml new file mode 100644 index 0000000..9b7065f --- /dev/null +++ b/kubespray/project/roles/network_plugin/kube-router/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - role: network_plugin/cni diff --git a/kubespray/project/roles/network_plugin/kube-router/tasks/annotate.yml b/kubespray/project/roles/network_plugin/kube-router/tasks/annotate.yml new file mode 100644 index 0000000..67d57a2 --- /dev/null +++ b/kubespray/project/roles/network_plugin/kube-router/tasks/annotate.yml @@ -0,0 +1,21 @@ +--- +- name: Kube-router | Add annotations on kube_control_plane + command: "{{ kubectl }} annotate --overwrite node {{ ansible_hostname }} {{ item }}" + with_items: + - "{{ kube_router_annotations_master }}" + delegate_to: "{{ groups['kube_control_plane'][0] }}" + when: kube_router_annotations_master is defined and inventory_hostname in groups['kube_control_plane'] + +- name: Kube-router | Add annotations on kube_node + command: "{{ kubectl }} annotate --overwrite node {{ ansible_hostname }} {{ item }}" + with_items: + - "{{ kube_router_annotations_node }}" + delegate_to: "{{ groups['kube_control_plane'][0] }}" + when: kube_router_annotations_node is defined and inventory_hostname in groups['kube_node'] + +- name: Kube-router | Add common annotations on all servers + command: "{{ kubectl }} annotate --overwrite node {{ ansible_hostname }} {{ item }}" + with_items: + - "{{ kube_router_annotations_all }}" + delegate_to: "{{ groups['kube_control_plane'][0] }}" + when: kube_router_annotations_all is defined and inventory_hostname in groups['k8s_cluster'] diff --git a/kubespray/project/roles/network_plugin/kube-router/tasks/main.yml b/kubespray/project/roles/network_plugin/kube-router/tasks/main.yml new file mode 100644 index 0000000..b6367f0 --- /dev/null +++ b/kubespray/project/roles/network_plugin/kube-router/tasks/main.yml @@ -0,0 +1,62 @@ +--- +- name: Kube-router | Create annotations + import_tasks: annotate.yml + tags: annotate + +- name: Kube-router | Create config directory + file: + path: /var/lib/kube-router + state: directory + owner: "{{ kube_owner }}" + recurse: true + mode: 0755 + +- name: Kube-router | Create kubeconfig + template: + src: kubeconfig.yml.j2 + dest: /var/lib/kube-router/kubeconfig + mode: 0644 + owner: "{{ kube_owner }}" + notify: + - Reset_kube_router + +- name: Kube-router | Slurp cni config + slurp: + src: /etc/cni/net.d/10-kuberouter.conflist + register: cni_config_slurp + ignore_errors: true # noqa ignore-errors + +- name: Kube-router | Set cni_config variable + set_fact: + cni_config: "{{ cni_config_slurp.content | b64decode | from_json }}" + when: + - not cni_config_slurp.failed + +- name: Kube-router | Set host_subnet variable + when: + - cni_config is defined + - cni_config | json_query('plugins[?bridge==`kube-bridge`].ipam.subnet') | length > 0 + set_fact: + host_subnet: "{{ cni_config | json_query('plugins[?bridge==`kube-bridge`].ipam.subnet') | first }}" + +- name: Kube-router | Create cni config + template: + src: cni-conf.json.j2 + dest: /etc/cni/net.d/10-kuberouter.conflist + mode: 0644 + owner: "{{ kube_owner }}" + notify: + - Reset_kube_router + +- name: Kube-router | Delete old configuration + file: + path: /etc/cni/net.d/10-kuberouter.conf + state: absent + +- name: Kube-router | Create manifest + template: + src: kube-router.yml.j2 + dest: "{{ kube_config_dir }}/kube-router.yml" + mode: 0644 + delegate_to: "{{ groups['kube_control_plane'] | first }}" + run_once: true diff --git a/kubespray/project/roles/network_plugin/kube-router/tasks/reset.yml b/kubespray/project/roles/network_plugin/kube-router/tasks/reset.yml new file mode 100644 index 0000000..ae9ee55 --- /dev/null +++ b/kubespray/project/roles/network_plugin/kube-router/tasks/reset.yml @@ -0,0 +1,28 @@ +--- +- name: Reset | check kube-dummy-if network device + stat: + path: /sys/class/net/kube-dummy-if + get_attributes: no + get_checksum: no + get_mime: no + register: kube_dummy_if + +- name: Reset | remove the network device created by kube-router + command: ip link del kube-dummy-if + when: kube_dummy_if.stat.exists + +- name: Check kube-bridge exists + stat: + path: /sys/class/net/kube-bridge + get_attributes: no + get_checksum: no + get_mime: no + register: kube_bridge_if + +- name: Reset | donw the network bridge create by kube-router + command: ip link set kube-bridge down + when: kube_bridge_if.stat.exists + +- name: Reset | remove the network bridge create by kube-router + command: ip link del kube-bridge + when: kube_bridge_if.stat.exists diff --git a/kubespray/project/roles/network_plugin/kube-router/templates/cni-conf.json.j2 b/kubespray/project/roles/network_plugin/kube-router/templates/cni-conf.json.j2 new file mode 100644 index 0000000..91fafac --- /dev/null +++ b/kubespray/project/roles/network_plugin/kube-router/templates/cni-conf.json.j2 @@ -0,0 +1,27 @@ +{ + "cniVersion":"0.3.0", + "name":"kubernetes", + "plugins":[ + { + "name":"kubernetes", + "type":"bridge", + "bridge":"kube-bridge", + "isDefaultGateway":true, +{% if kube_router_support_hairpin_mode %} + "hairpinMode":true, +{% endif %} + "ipam":{ +{% if host_subnet is defined %} + "subnet": "{{ host_subnet }}", +{% endif %} + "type":"host-local" + } + }, + { + "type":"portmap", + "capabilities":{ + "portMappings":true + } + } + ] +} diff --git a/kubespray/project/roles/network_plugin/kube-router/templates/kube-router.yml.j2 b/kubespray/project/roles/network_plugin/kube-router/templates/kube-router.yml.j2 new file mode 100644 index 0000000..d868287 --- /dev/null +++ b/kubespray/project/roles/network_plugin/kube-router/templates/kube-router.yml.j2 @@ -0,0 +1,228 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + k8s-app: kube-router + tier: node + name: kube-router + namespace: kube-system +spec: + minReadySeconds: 3 + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate + selector: + matchLabels: + k8s-app: kube-router + tier: node + template: + metadata: + labels: + k8s-app: kube-router + tier: node + annotations: +{% if kube_router_enable_metrics %} + prometheus.io/path: {{ kube_router_metrics_path }} + prometheus.io/port: "{{ kube_router_metrics_port }}" + prometheus.io/scrape: "true" +{% endif %} + spec: + priorityClassName: system-node-critical + serviceAccountName: kube-router + containers: + - name: kube-router + image: {{ kube_router_image_repo }}:{{ kube_router_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + args: + - --run-router={{ kube_router_run_router | bool }} + - --run-firewall={{ kube_router_run_firewall | bool }} + - --run-service-proxy={{ kube_router_run_service_proxy | bool }} + - --kubeconfig=/var/lib/kube-router/kubeconfig + - --bgp-graceful-restart={{ kube_router_bgp_graceful_restart }} +{% if kube_router_advertise_cluster_ip %} + - --advertise-cluster-ip +{% endif %} +{% if kube_router_advertise_external_ip %} + - --advertise-external-ip +{% endif %} +{% if kube_router_advertise_loadbalancer_ip %} + - --advertise-loadbalancer-ip +{% endif %} +{% if kube_router_cluster_asn %} + - --cluster-asn={{ kube_router_cluster_asn }} +{% endif %} +{% if kube_router_peer_router_asns %} + - --peer-router-asns={{ kube_router_peer_router_asns }} +{% endif %} +{% if kube_router_peer_router_ips %} + - --peer-router-ips={{ kube_router_peer_router_ips }} +{% endif %} +{% if kube_router_peer_router_ports %} + - --peer-router-ports={{ kube_router_peer_router_ports }} +{% endif %} +{% if kube_router_enable_metrics %} + - --metrics-path={{ kube_router_metrics_path }} + - --metrics-port={{ kube_router_metrics_port }} +{% endif %} +{% if kube_router_enable_dsr %} +{% if container_manager == "docker" %} + - --runtime-endpoint=unix:///var/run/docker.sock +{% endif %} +{% if container_manager == "containerd" %} +{% endif %} + - --runtime-endpoint=unix:///run/containerd/containerd.sock +{% endif %} +{% for arg in kube_router_extra_args %} + - "{{ arg }}" +{% endfor %} + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: KUBE_ROUTER_CNI_CONF_FILE + value: /etc/cni/net.d/10-kuberouter.conflist + livenessProbe: + httpGet: + path: /healthz + port: 20244 + initialDelaySeconds: 10 + periodSeconds: 3 + resources: + requests: + cpu: 250m + memory: 250Mi + securityContext: + privileged: true + volumeMounts: +{% if kube_router_enable_dsr %} +{% if container_manager == "docker" %} + - name: docker-socket + mountPath: /var/run/docker.sock + readOnly: true +{% endif %} +{% if container_manager == "containerd" %} + - name: containerd-socket + mountPath: /run/containerd/containerd.sock + readOnly: true +{% endif %} +{% endif %} + - name: lib-modules + mountPath: /lib/modules + readOnly: true + - name: cni-conf-dir + mountPath: /etc/cni/net.d + - name: kubeconfig + mountPath: /var/lib/kube-router + readOnly: true + - name: xtables-lock + mountPath: /run/xtables.lock + readOnly: false +{% if kube_router_enable_metrics %} + ports: + - containerPort: {{ kube_router_metrics_port }} + hostPort: {{ kube_router_metrics_port }} + name: metrics + protocol: TCP +{% endif %} + hostNetwork: true + dnsPolicy: {{ kube_router_dns_policy }} +{% if kube_router_enable_dsr %} + hostIPC: true + hostPID: true +{% endif %} + tolerations: + - operator: Exists + volumes: +{% if kube_router_enable_dsr %} +{% if container_manager == "docker" %} + - name: docker-socket + hostPath: + path: /var/run/docker.sock + type: Socket +{% endif %} +{% if container_manager == "containerd" %} + - name: containerd-socket + hostPath: + path: /run/containerd/containerd.sock + type: Socket +{% endif %} +{% endif %} + - name: lib-modules + hostPath: + path: /lib/modules + - name: cni-conf-dir + hostPath: + path: /etc/cni/net.d + - name: kubeconfig + hostPath: + path: /var/lib/kube-router + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-router + namespace: kube-system + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kube-router + namespace: kube-system +rules: + - apiGroups: + - "" + resources: + - namespaces + - pods + - services + - nodes + - endpoints + verbs: + - list + - get + - watch + - apiGroups: + - "networking.k8s.io" + resources: + - networkpolicies + verbs: + - list + - get + - watch + - apiGroups: + - extensions + resources: + - networkpolicies + verbs: + - get + - list + - watch + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kube-router +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kube-router +subjects: +- kind: ServiceAccount + name: kube-router + namespace: kube-system diff --git a/kubespray/project/roles/network_plugin/kube-router/templates/kubeconfig.yml.j2 b/kubespray/project/roles/network_plugin/kube-router/templates/kubeconfig.yml.j2 new file mode 100644 index 0000000..42fd317 --- /dev/null +++ b/kubespray/project/roles/network_plugin/kube-router/templates/kubeconfig.yml.j2 @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Config +clusterCIDR: {{ kube_pods_subnet }} +clusters: +- name: cluster + cluster: + certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + server: {{ kube_apiserver_endpoint }} +users: +- name: kube-router + user: + tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +contexts: +- context: + cluster: cluster + user: kube-router + name: kube-router-context +current-context: kube-router-context diff --git a/kubespray/project/roles/network_plugin/macvlan/OWNERS b/kubespray/project/roles/network_plugin/macvlan/OWNERS new file mode 100644 index 0000000..c5dfbc7 --- /dev/null +++ b/kubespray/project/roles/network_plugin/macvlan/OWNERS @@ -0,0 +1,6 @@ +# See the OWNERS docs at https://go.k8s.io/owners + +approvers: + - simon +reviewers: + - simon diff --git a/kubespray/project/roles/network_plugin/macvlan/defaults/main.yml b/kubespray/project/roles/network_plugin/macvlan/defaults/main.yml new file mode 100644 index 0000000..70a8dd0 --- /dev/null +++ b/kubespray/project/roles/network_plugin/macvlan/defaults/main.yml @@ -0,0 +1,6 @@ +--- +macvlan_interface: eth0 +enable_nat_default_gateway: true + +# sysctl_file_path to add sysctl conf to +sysctl_file_path: "/etc/sysctl.d/99-sysctl.conf" diff --git a/kubespray/project/roles/network_plugin/macvlan/files/ifdown-local b/kubespray/project/roles/network_plugin/macvlan/files/ifdown-local new file mode 100644 index 0000000..003b8a1 --- /dev/null +++ b/kubespray/project/roles/network_plugin/macvlan/files/ifdown-local @@ -0,0 +1,6 @@ +#!/bin/bash + +POSTDOWNNAME="/etc/sysconfig/network-scripts/post-down-$1" +if [ -x $POSTDOWNNAME ]; then + exec $POSTDOWNNAME +fi diff --git a/kubespray/project/roles/network_plugin/macvlan/files/ifdown-macvlan b/kubespray/project/roles/network_plugin/macvlan/files/ifdown-macvlan new file mode 100755 index 0000000..b79b9c1 --- /dev/null +++ b/kubespray/project/roles/network_plugin/macvlan/files/ifdown-macvlan @@ -0,0 +1,40 @@ +#!/bin/bash +# +# initscripts-macvlan +# Copyright (C) 2014 Lars Kellogg-Stedman +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /etc/init.d/functions + +cd /etc/sysconfig/network-scripts +. ./network-functions + +[ -f ../network ] && . ../network + +CONFIG=${1} + +need_config ${CONFIG} + +source_config + +OTHERSCRIPT="/etc/sysconfig/network-scripts/ifdown-${REAL_DEVICETYPE}" + +if [ ! -x ${OTHERSCRIPT} ]; then + OTHERSCRIPT="/etc/sysconfig/network-scripts/ifdown-eth" +fi + +${OTHERSCRIPT} ${CONFIG} + +ip link del ${DEVICE} type ${TYPE:-macvlan} diff --git a/kubespray/project/roles/network_plugin/macvlan/files/ifup-local b/kubespray/project/roles/network_plugin/macvlan/files/ifup-local new file mode 100755 index 0000000..3b6891e --- /dev/null +++ b/kubespray/project/roles/network_plugin/macvlan/files/ifup-local @@ -0,0 +1,6 @@ +#!/bin/bash + +POSTUPNAME="/etc/sysconfig/network-scripts/post-up-$1" +if [ -x $POSTUPNAME ]; then + exec $POSTUPNAME +fi diff --git a/kubespray/project/roles/network_plugin/macvlan/files/ifup-macvlan b/kubespray/project/roles/network_plugin/macvlan/files/ifup-macvlan new file mode 100755 index 0000000..97daec0 --- /dev/null +++ b/kubespray/project/roles/network_plugin/macvlan/files/ifup-macvlan @@ -0,0 +1,43 @@ +#!/bin/bash +# +# initscripts-macvlan +# Copyright (C) 2014 Lars Kellogg-Stedman +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /etc/init.d/functions + +cd /etc/sysconfig/network-scripts +. ./network-functions + +[ -f ../network ] && . ../network + +CONFIG=${1} + +need_config ${CONFIG} + +source_config + +OTHERSCRIPT="/etc/sysconfig/network-scripts/ifup-${REAL_DEVICETYPE}" + +if [ ! -x ${OTHERSCRIPT} ]; then + OTHERSCRIPT="/etc/sysconfig/network-scripts/ifup-eth" +fi + +ip link add \ + link ${MACVLAN_PARENT} \ + name ${DEVICE} \ + type ${TYPE:-macvlan} mode ${MACVLAN_MODE:-private} + +${OTHERSCRIPT} ${CONFIG} diff --git a/kubespray/project/roles/network_plugin/macvlan/handlers/main.yml b/kubespray/project/roles/network_plugin/macvlan/handlers/main.yml new file mode 100644 index 0000000..e4844c2 --- /dev/null +++ b/kubespray/project/roles/network_plugin/macvlan/handlers/main.yml @@ -0,0 +1,15 @@ +--- +- name: Macvlan | reload network + service: + # noqa: jinja[spacing] + name: >- + {% if ansible_os_family == "RedHat" -%} + network + {%- elif ansible_distribution == "Ubuntu" and ansible_distribution_release == "bionic" -%} + systemd-networkd + {%- elif ansible_os_family == "Debian" -%} + networking + {%- endif %} + state: restarted + when: not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] and kube_network_plugin not in ['calico'] + listen: Macvlan | restart network diff --git a/kubespray/project/roles/network_plugin/macvlan/meta/main.yml b/kubespray/project/roles/network_plugin/macvlan/meta/main.yml new file mode 100644 index 0000000..9b7065f --- /dev/null +++ b/kubespray/project/roles/network_plugin/macvlan/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - role: network_plugin/cni diff --git a/kubespray/project/roles/network_plugin/macvlan/tasks/main.yml b/kubespray/project/roles/network_plugin/macvlan/tasks/main.yml new file mode 100644 index 0000000..f7c3027 --- /dev/null +++ b/kubespray/project/roles/network_plugin/macvlan/tasks/main.yml @@ -0,0 +1,110 @@ +--- +- name: Macvlan | Retrieve Pod Cidr + command: "{{ kubectl }} get nodes {{ kube_override_hostname | default(inventory_hostname) }} -o jsonpath='{.spec.podCIDR}'" + changed_when: false + register: node_pod_cidr_cmd + delegate_to: "{{ groups['kube_control_plane'][0] }}" + +- name: Macvlan | set node_pod_cidr + set_fact: + node_pod_cidr: "{{ node_pod_cidr_cmd.stdout }}" + +- name: Macvlan | Retrieve default gateway network interface + become: false + raw: ip -4 route list 0/0 | sed 's/.*dev \([[:alnum:]]*\).*/\1/' + changed_when: false + register: node_default_gateway_interface_cmd + +- name: Macvlan | set node_default_gateway_interface + set_fact: + node_default_gateway_interface: "{{ node_default_gateway_interface_cmd.stdout | trim }}" + +- name: Macvlan | Install network gateway interface on debian + template: + src: debian-network-macvlan.cfg.j2 + dest: /etc/network/interfaces.d/60-mac0.cfg + mode: 0644 + notify: Macvlan | restart network + when: ansible_os_family in ["Debian"] + +- name: Install macvlan config on RH distros + when: ansible_os_family == "RedHat" + block: + - name: Macvlan | Install macvlan script on centos + copy: + src: "{{ item }}" + dest: /etc/sysconfig/network-scripts/ + owner: root + group: root + mode: "0755" + with_fileglob: + - files/* + + - name: Macvlan | Install post-up script on centos + copy: + src: "files/ifup-local" + dest: /sbin/ + owner: root + group: root + mode: "0755" + when: enable_nat_default_gateway + + - name: Macvlan | Install network gateway interface on centos + template: + src: "{{ item.src }}.j2" + dest: "/etc/sysconfig/network-scripts/{{ item.dst }}" + mode: 0644 + with_items: + - {src: centos-network-macvlan.cfg, dst: ifcfg-mac0 } + - {src: centos-routes-macvlan.cfg, dst: route-mac0 } + - {src: centos-postup-macvlan.cfg, dst: post-up-mac0 } + notify: Macvlan | restart network + +- name: Install macvlan config on Flatcar + when: ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] + block: + - name: Macvlan | Install service nat via gateway on Flatcar Container Linux + template: + src: coreos-service-nat_ouside.j2 + dest: /etc/systemd/system/enable_nat_ouside.service + mode: 0644 + when: enable_nat_default_gateway + + - name: Macvlan | Enable service nat via gateway on Flatcar Container Linux + command: "{{ item }}" + with_items: + - systemctl daemon-reload + - systemctl enable enable_nat_ouside.service + when: enable_nat_default_gateway + + - name: Macvlan | Install network gateway interface on Flatcar Container Linux + template: + src: "{{ item.src }}.j2" + dest: "/etc/systemd/network/{{ item.dst }}" + mode: 0644 + with_items: + - {src: coreos-device-macvlan.cfg, dst: macvlan.netdev } + - {src: coreos-interface-macvlan.cfg, dst: output.network } + - {src: coreos-network-macvlan.cfg, dst: macvlan.network } + notify: Macvlan | restart network + +- name: Macvlan | Install cni definition for Macvlan + template: + src: 10-macvlan.conf.j2 + dest: /etc/cni/net.d/10-macvlan.conf + mode: 0644 + +- name: Macvlan | Install loopback definition for Macvlan + template: + src: 99-loopback.conf.j2 + dest: /etc/cni/net.d/99-loopback.conf + mode: 0644 + +- name: Enable net.ipv4.conf.all.arp_notify in sysctl + ansible.posix.sysctl: + name: net.ipv4.conf.all.arp_notify + value: 1 + sysctl_set: yes + sysctl_file: "{{ sysctl_file_path }}" + state: present + reload: yes diff --git a/kubespray/project/roles/network_plugin/macvlan/templates/10-macvlan.conf.j2 b/kubespray/project/roles/network_plugin/macvlan/templates/10-macvlan.conf.j2 new file mode 100644 index 0000000..8924547 --- /dev/null +++ b/kubespray/project/roles/network_plugin/macvlan/templates/10-macvlan.conf.j2 @@ -0,0 +1,15 @@ +{ + "cniVersion": "0.4.0", + "name": "mynet", + "type": "macvlan", + "master": "{{ macvlan_interface }}", + "hairpinMode": true, + "ipam": { + "type": "host-local", + "subnet": "{{ node_pod_cidr }}", + "routes": [ + { "dst": "0.0.0.0/0" } + ], + "gateway": "{{ node_pod_cidr|ansible.utils.ipaddr('net')|ansible.utils.ipaddr(1)|ansible.utils.ipaddr('address') }}" + } +} diff --git a/kubespray/project/roles/network_plugin/macvlan/templates/99-loopback.conf.j2 b/kubespray/project/roles/network_plugin/macvlan/templates/99-loopback.conf.j2 new file mode 100644 index 0000000..b41ab65 --- /dev/null +++ b/kubespray/project/roles/network_plugin/macvlan/templates/99-loopback.conf.j2 @@ -0,0 +1,5 @@ +{ + "cniVersion": "0.2.0", + "name": "lo", + "type": "loopback" +} diff --git a/kubespray/project/roles/network_plugin/macvlan/templates/centos-network-macvlan.cfg.j2 b/kubespray/project/roles/network_plugin/macvlan/templates/centos-network-macvlan.cfg.j2 new file mode 100644 index 0000000..1e6c0aa --- /dev/null +++ b/kubespray/project/roles/network_plugin/macvlan/templates/centos-network-macvlan.cfg.j2 @@ -0,0 +1,13 @@ +DEVICE=mac0 +DEVICETYPE=macvlan +TYPE=macvlan +BOOTPROTO=none +ONBOOT=yes +NM_CONTROLLED=no + +MACVLAN_PARENT={{ macvlan_interface }} +MACVLAN_MODE=bridge + +IPADDR={{ node_pod_cidr|ansible.utils.ipaddr('net')|ansible.utils.ipaddr(1)|ansible.utils.ipaddr('address') }} +NETMASK={{ node_pod_cidr|ansible.utils.ipaddr('netmask') }} +NETWORK={{ node_pod_cidr|ansible.utils.ipaddr('network') }} diff --git a/kubespray/project/roles/network_plugin/macvlan/templates/centos-postdown-macvlan.cfg.j2 b/kubespray/project/roles/network_plugin/macvlan/templates/centos-postdown-macvlan.cfg.j2 new file mode 100644 index 0000000..87f1f56 --- /dev/null +++ b/kubespray/project/roles/network_plugin/macvlan/templates/centos-postdown-macvlan.cfg.j2 @@ -0,0 +1,3 @@ +{% if enable_nat_default_gateway %} +iptables -t nat -D POSTROUTING -s {{ node_pod_cidr|ansible.utils.ipaddr('net') }} -o {{ node_default_gateway_interface }} -j MASQUERADE +{% endif %} diff --git a/kubespray/project/roles/network_plugin/macvlan/templates/centos-postup-macvlan.cfg.j2 b/kubespray/project/roles/network_plugin/macvlan/templates/centos-postup-macvlan.cfg.j2 new file mode 100644 index 0000000..254827e --- /dev/null +++ b/kubespray/project/roles/network_plugin/macvlan/templates/centos-postup-macvlan.cfg.j2 @@ -0,0 +1,3 @@ +{% if enable_nat_default_gateway %} +iptables -t nat -I POSTROUTING -s {{ node_pod_cidr|ansible.utils.ipaddr('net') }} -o {{ node_default_gateway_interface }} -j MASQUERADE +{% endif %} diff --git a/kubespray/project/roles/network_plugin/macvlan/templates/centos-routes-macvlan.cfg.j2 b/kubespray/project/roles/network_plugin/macvlan/templates/centos-routes-macvlan.cfg.j2 new file mode 100644 index 0000000..60400dd --- /dev/null +++ b/kubespray/project/roles/network_plugin/macvlan/templates/centos-routes-macvlan.cfg.j2 @@ -0,0 +1,7 @@ +{% for host in groups['kube_node'] %} +{% if hostvars[host]['access_ip'] is defined %} +{% if hostvars[host]['node_pod_cidr'] != node_pod_cidr %} +{{ hostvars[host]['node_pod_cidr'] }} via {{ hostvars[host]['access_ip'] }} +{% endif %} +{% endif %} +{% endfor %} diff --git a/kubespray/project/roles/network_plugin/macvlan/templates/coreos-device-macvlan.cfg.j2 b/kubespray/project/roles/network_plugin/macvlan/templates/coreos-device-macvlan.cfg.j2 new file mode 100644 index 0000000..2418dac --- /dev/null +++ b/kubespray/project/roles/network_plugin/macvlan/templates/coreos-device-macvlan.cfg.j2 @@ -0,0 +1,6 @@ +[NetDev] +Name=mac0 +Kind=macvlan + +[MACVLAN] +Mode=bridge diff --git a/kubespray/project/roles/network_plugin/macvlan/templates/coreos-interface-macvlan.cfg.j2 b/kubespray/project/roles/network_plugin/macvlan/templates/coreos-interface-macvlan.cfg.j2 new file mode 100644 index 0000000..342f680 --- /dev/null +++ b/kubespray/project/roles/network_plugin/macvlan/templates/coreos-interface-macvlan.cfg.j2 @@ -0,0 +1,6 @@ +[Match] +Name={{ macvlan_interface }} + +[Network] +MACVLAN=mac0 +DHCP=yes diff --git a/kubespray/project/roles/network_plugin/macvlan/templates/coreos-network-macvlan.cfg.j2 b/kubespray/project/roles/network_plugin/macvlan/templates/coreos-network-macvlan.cfg.j2 new file mode 100644 index 0000000..0c4c33b --- /dev/null +++ b/kubespray/project/roles/network_plugin/macvlan/templates/coreos-network-macvlan.cfg.j2 @@ -0,0 +1,17 @@ +[Match] +Name=mac0 + +[Network] +Address={{ node_pod_cidr|ansible.utils.ipaddr('net')|ansible.utils.ipaddr(1)|ansible.utils.ipaddr('address') }}/{{ node_pod_cidr|ansible.utils.ipaddr('prefix') }} + +{% for host in groups['kube_node'] %} +{% if hostvars[host]['access_ip'] is defined %} +{% if hostvars[host]['node_pod_cidr'] != node_pod_cidr %} +[Route] +Gateway={{ hostvars[host]['access_ip'] }} +Destination={{ hostvars[host]['node_pod_cidr'] }} +GatewayOnlink=yes + +{% endif %} +{% endif %} +{% endfor %} diff --git a/kubespray/project/roles/network_plugin/macvlan/templates/coreos-service-nat_ouside.j2 b/kubespray/project/roles/network_plugin/macvlan/templates/coreos-service-nat_ouside.j2 new file mode 100644 index 0000000..1d8df03 --- /dev/null +++ b/kubespray/project/roles/network_plugin/macvlan/templates/coreos-service-nat_ouside.j2 @@ -0,0 +1,6 @@ +[Service] +Type=oneshot +ExecStart=/bin/bash -c "iptables -t nat -I POSTROUTING -s {{ node_pod_cidr|ansible.utils.ipaddr('net') }} -o {{ node_default_gateway_interface }} -j MASQUERADE" + +[Install] +WantedBy=sys-subsystem-net-devices-mac0.device diff --git a/kubespray/project/roles/network_plugin/macvlan/templates/debian-network-macvlan.cfg.j2 b/kubespray/project/roles/network_plugin/macvlan/templates/debian-network-macvlan.cfg.j2 new file mode 100644 index 0000000..cbd4325 --- /dev/null +++ b/kubespray/project/roles/network_plugin/macvlan/templates/debian-network-macvlan.cfg.j2 @@ -0,0 +1,26 @@ +auto mac0 +iface mac0 inet static + address {{ node_pod_cidr|ansible.utils.ipaddr('net')|ansible.utils.ipaddr(1)|ansible.utils.ipaddr('address') }} + network {{ node_pod_cidr|ansible.utils.ipaddr('network') }} + netmask {{ node_pod_cidr|ansible.utils.ipaddr('netmask') }} + broadcast {{ node_pod_cidr|ansible.utils.ipaddr('broadcast') }} + pre-up ip link add link {{ macvlan_interface }} mac0 type macvlan mode bridge +{% for host in groups['kube_node'] %} +{% if hostvars[host]['access_ip'] is defined %} +{% if hostvars[host]['node_pod_cidr'] != node_pod_cidr %} + post-up ip route add {{ hostvars[host]['node_pod_cidr'] }} via {{ hostvars[host]['access_ip'] }} +{% endif %} +{% endif %} +{% endfor %} +{% if enable_nat_default_gateway %} + post-up iptables -t nat -I POSTROUTING -s {{ node_pod_cidr|ansible.utils.ipaddr('net') }} -o {{ node_default_gateway_interface }} -j MASQUERADE +{% endif %} +{% for host in groups['kube_node'] %} +{% if hostvars[host]['access_ip'] is defined %} +{% if hostvars[host]['node_pod_cidr'] != node_pod_cidr %} + post-down ip route del {{ hostvars[host]['node_pod_cidr'] }} via {{ hostvars[host]['access_ip'] }} +{% endif %} +{% endif %} +{% endfor %} + post-down iptables -t nat -D POSTROUTING -s {{ node_pod_cidr|ansible.utils.ipaddr('net') }} -o {{ node_default_gateway_interface }} -j MASQUERADE + post-down ip link delete mac0 diff --git a/kubespray/project/roles/network_plugin/meta/main.yml b/kubespray/project/roles/network_plugin/meta/main.yml new file mode 100644 index 0000000..dd2c362 --- /dev/null +++ b/kubespray/project/roles/network_plugin/meta/main.yml @@ -0,0 +1,48 @@ +--- +dependencies: + - role: network_plugin/cni + + - role: network_plugin/cilium + when: kube_network_plugin == 'cilium' or cilium_deploy_additionally | default(false) | bool + tags: + - cilium + + - role: network_plugin/calico + when: kube_network_plugin == 'calico' + tags: + - calico + + - role: network_plugin/flannel + when: kube_network_plugin == 'flannel' + tags: + - flannel + + - role: network_plugin/weave + when: kube_network_plugin == 'weave' + tags: + - weave + + - role: network_plugin/macvlan + when: kube_network_plugin == 'macvlan' + tags: + - macvlan + + - role: network_plugin/kube-ovn + when: kube_network_plugin == 'kube-ovn' + tags: + - kube-ovn + + - role: network_plugin/kube-router + when: kube_network_plugin == 'kube-router' + tags: + - kube-router + + - role: network_plugin/custom_cni + when: kube_network_plugin == 'custom_cni' + tags: + - custom_cni + + - role: network_plugin/multus + when: kube_network_plugin_multus + tags: + - multus diff --git a/kubespray/project/roles/network_plugin/multus/defaults/main.yml b/kubespray/project/roles/network_plugin/multus/defaults/main.yml new file mode 100644 index 0000000..c6b7ecd --- /dev/null +++ b/kubespray/project/roles/network_plugin/multus/defaults/main.yml @@ -0,0 +1,10 @@ +--- +multus_conf_file: "auto" +multus_cni_conf_dir_host: "/etc/cni/net.d" +multus_cni_bin_dir_host: "/opt/cni/bin" +multus_cni_run_dir_host: "/run" +multus_cni_conf_dir: "{{ ('/host', multus_cni_conf_dir_host) | join }}" +multus_cni_bin_dir: "{{ ('/host', multus_cni_bin_dir_host) | join }}" +multus_cni_run_dir: "{{ ('/host', multus_cni_run_dir_host) | join }}" +multus_cni_version: "0.4.0" +multus_kubeconfig_file_host: "{{ (multus_cni_conf_dir_host, '/multus.d/multus.kubeconfig') | join }}" diff --git a/kubespray/project/roles/network_plugin/multus/files/multus-clusterrole.yml b/kubespray/project/roles/network_plugin/multus/files/multus-clusterrole.yml new file mode 100644 index 0000000..b574069 --- /dev/null +++ b/kubespray/project/roles/network_plugin/multus/files/multus-clusterrole.yml @@ -0,0 +1,28 @@ +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: multus +rules: + - apiGroups: ["k8s.cni.cncf.io"] + resources: + - '*' + verbs: + - '*' + - apiGroups: + - "" + resources: + - pods + - pods/status + verbs: + - get + - update + - apiGroups: + - "" + - events.k8s.io + resources: + - events + verbs: + - create + - patch + - update diff --git a/kubespray/project/roles/network_plugin/multus/files/multus-clusterrolebinding.yml b/kubespray/project/roles/network_plugin/multus/files/multus-clusterrolebinding.yml new file mode 100644 index 0000000..2d1e1a4 --- /dev/null +++ b/kubespray/project/roles/network_plugin/multus/files/multus-clusterrolebinding.yml @@ -0,0 +1,13 @@ +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: multus +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: multus +subjects: +- kind: ServiceAccount + name: multus + namespace: kube-system diff --git a/kubespray/project/roles/network_plugin/multus/files/multus-crd.yml b/kubespray/project/roles/network_plugin/multus/files/multus-crd.yml new file mode 100644 index 0000000..24b2c58 --- /dev/null +++ b/kubespray/project/roles/network_plugin/multus/files/multus-crd.yml @@ -0,0 +1,45 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: network-attachment-definitions.k8s.cni.cncf.io +spec: + group: k8s.cni.cncf.io + scope: Namespaced + names: + plural: network-attachment-definitions + singular: network-attachment-definition + kind: NetworkAttachmentDefinition + shortNames: + - net-attach-def + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + description: 'NetworkAttachmentDefinition is a CRD schema specified by the Network Plumbing + Working Group to express the intent for attaching pods to one or more logical or physical + networks. More information available at: https://github.com/k8snetworkplumbingwg/multi-net-spec' + type: object + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this represen + tation of an object. Servers should convert recognized schemas to the + latest internal value, and may reject unrecognized values. More info: + https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: 'NetworkAttachmentDefinition spec defines the desired state of a network attachment' + type: object + properties: + config: + description: 'NetworkAttachmentDefinition config is a JSON-formatted CNI configuration' + type: string diff --git a/kubespray/project/roles/network_plugin/multus/files/multus-serviceaccount.yml b/kubespray/project/roles/network_plugin/multus/files/multus-serviceaccount.yml new file mode 100644 index 0000000..6242308 --- /dev/null +++ b/kubespray/project/roles/network_plugin/multus/files/multus-serviceaccount.yml @@ -0,0 +1,6 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: multus + namespace: kube-system diff --git a/kubespray/project/roles/network_plugin/multus/meta/main.yml b/kubespray/project/roles/network_plugin/multus/meta/main.yml new file mode 100644 index 0000000..9b7065f --- /dev/null +++ b/kubespray/project/roles/network_plugin/multus/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - role: network_plugin/cni diff --git a/kubespray/project/roles/network_plugin/multus/tasks/main.yml b/kubespray/project/roles/network_plugin/multus/tasks/main.yml new file mode 100644 index 0000000..1428929 --- /dev/null +++ b/kubespray/project/roles/network_plugin/multus/tasks/main.yml @@ -0,0 +1,36 @@ +--- +- name: Multus | Copy manifest files + copy: + src: "{{ item.file }}" + dest: "{{ kube_config_dir }}" + mode: 0644 + with_items: + - {name: multus-crd, file: multus-crd.yml, type: customresourcedefinition} + - {name: multus-serviceaccount, file: multus-serviceaccount.yml, type: serviceaccount} + - {name: multus-clusterrole, file: multus-clusterrole.yml, type: clusterrole} + - {name: multus-clusterrolebinding, file: multus-clusterrolebinding.yml, type: clusterrolebinding} + register: multus_manifest_1 + when: inventory_hostname == groups['kube_control_plane'][0] + +- name: Multus | Check container engine type + set_fact: + container_manager_types: "{{ ansible_play_hosts_all | map('extract', hostvars, ['container_manager']) | list | unique }}" + +- name: Multus | Copy manifest templates + template: + src: multus-daemonset.yml.j2 + dest: "{{ kube_config_dir }}/{{ item.file }}" + mode: 0644 + with_items: + - {name: multus-daemonset-containerd, file: multus-daemonset-containerd.yml, type: daemonset, engine: containerd } + - {name: multus-daemonset-docker, file: multus-daemonset-docker.yml, type: daemonset, engine: docker } + - {name: multus-daemonset-crio, file: multus-daemonset-crio.yml, type: daemonset, engine: crio } + register: multus_manifest_2 + vars: + query: "*|[?container_manager=='{{ container_manager }}']|[0].inventory_hostname" + vars_from_node: "{{ hostvars | json_query(query) }}" + delegate_to: "{{ groups['kube_control_plane'][0] }}" + when: + - item.engine in container_manager_types + - hostvars[inventory_hostname].container_manager == item.engine + - inventory_hostname == vars_from_node diff --git a/kubespray/project/roles/network_plugin/multus/templates/multus-daemonset.yml.j2 b/kubespray/project/roles/network_plugin/multus/templates/multus-daemonset.yml.j2 new file mode 100644 index 0000000..10c42c1 --- /dev/null +++ b/kubespray/project/roles/network_plugin/multus/templates/multus-daemonset.yml.j2 @@ -0,0 +1,79 @@ +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: +{% if container_manager_types | length >= 2 %} + name: kube-multus-{{ container_manager }}-{{ image_arch }} +{% else %} + name: kube-multus-ds-{{ image_arch }} +{% endif %} + namespace: kube-system + labels: + tier: node + app: multus +spec: + selector: + matchLabels: + tier: node + app: multus + template: + metadata: + labels: + tier: node + app: multus + spec: + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + nodeSelector: + kubernetes.io/arch: {{ image_arch }} +{% if container_manager_types | length >= 2 %} + kubespray.io/container_manager: {{ container_manager }} +{% endif %} + tolerations: + - operator: Exists + serviceAccountName: multus + containers: + - name: kube-multus + image: {{ multus_image_repo }}:{{ multus_image_tag }} + command: ["/entrypoint.sh"] + args: + - "--cni-conf-dir={{ multus_cni_conf_dir }}" + - "--cni-bin-dir={{ multus_cni_bin_dir }}" + - "--multus-conf-file={{ multus_conf_file }}" + - "--multus-kubeconfig-file-host={{ multus_kubeconfig_file_host }}" + - "--cni-version={{ multus_cni_version }}" + resources: + requests: + cpu: "100m" + memory: "90Mi" + limits: + cpu: "100m" + memory: "90Mi" + securityContext: + privileged: true +{% if container_manager == 'crio' %} + capabilities: + add: ["SYS_ADMIN"] +{% endif %} + volumeMounts: +{% if container_manager == 'crio' %} + - name: run + mountPath: {{ multus_cni_run_dir }} + mountPropagation: HostToContainer +{% endif %} + - name: cni + mountPath: {{ multus_cni_conf_dir }} + - name: cnibin + mountPath: {{ multus_cni_bin_dir }} + volumes: +{% if container_manager == 'crio' %} + - name: run + hostPath: + path: {{ multus_cni_run_dir_host }} +{% endif %} + - name: cni + hostPath: + path: {{ multus_cni_conf_dir_host }} + - name: cnibin + hostPath: + path: {{ multus_cni_bin_dir_host }} diff --git a/kubespray/project/roles/network_plugin/ovn4nfv/tasks/main.yml b/kubespray/project/roles/network_plugin/ovn4nfv/tasks/main.yml new file mode 100644 index 0000000..777fd9a --- /dev/null +++ b/kubespray/project/roles/network_plugin/ovn4nfv/tasks/main.yml @@ -0,0 +1,16 @@ +--- +- name: Ovn4nfv | Label control-plane node + command: >- + {{ kubectl }} label --overwrite node {{ groups['kube_control_plane'] | first }} ovn4nfv-k8s-plugin=ovn-control-plane + when: + - inventory_hostname == groups['kube_control_plane'][0] + +- name: Ovn4nfv | Create ovn4nfv-k8s manifests + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/{{ item.file }}" + mode: 0644 + with_items: + - {name: ovn-daemonset, file: ovn-daemonset.yml} + - {name: ovn4nfv-k8s-plugin, file: ovn4nfv-k8s-plugin.yml} + register: ovn4nfv_node_manifests diff --git a/kubespray/project/roles/network_plugin/weave/defaults/main.yml b/kubespray/project/roles/network_plugin/weave/defaults/main.yml new file mode 100644 index 0000000..47469ae --- /dev/null +++ b/kubespray/project/roles/network_plugin/weave/defaults/main.yml @@ -0,0 +1,64 @@ +--- + +# Weave's network password for encryption, if null then no network encryption. +weave_password: ~ + +# If set to 1, disable checking for new Weave Net versions (default is blank, +# i.e. check is enabled) +weave_checkpoint_disable: false + +# Soft limit on the number of connections between peers. Defaults to 100. +weave_conn_limit: 100 + +# Weave Net defaults to enabling hairpin on the bridge side of the veth pair +# for containers attached. If you need to disable hairpin, e.g. your kernel is +# one of those that can panic if hairpin is enabled, then you can disable it by +# setting `HAIRPIN_MODE=false`. +weave_hairpin_mode: true + +# The range of IP addresses used by Weave Net and the subnet they are placed in +# (CIDR format; default 10.32.0.0/12) +weave_ipalloc_range: "{{ kube_pods_subnet }}" + +# Set to 0 to disable Network Policy Controller (default is on) +weave_expect_npc: "{{ enable_network_policy }}" + +# List of addresses of peers in the Kubernetes cluster (default is to fetch the +# list from the api-server) +weave_kube_peers: ~ + +# Set the initialization mode of the IP Address Manager (defaults to consensus +# amongst the KUBE_PEERS) +weave_ipalloc_init: ~ + +# Set the IP address used as a gateway from the Weave network to the host +# network - this is useful if you are configuring the addon as a static pod. +weave_expose_ip: ~ + +# Address and port that the Weave Net daemon will serve Prometheus-style +# metrics on (defaults to 0.0.0.0:6782) +weave_metrics_addr: ~ + +# Address and port that the Weave Net daemon will serve status requests on +# (defaults to disabled) +weave_status_addr: ~ + +# Weave Net defaults to 1376 bytes, but you can set a smaller size if your +# underlying network has a tighter limit, or set a larger size for better +# performance if your network supports jumbo frames (e.g. 8916) +weave_mtu: 1376 + +# Set to 1 to preserve the client source IP address when accessing Service +# annotated with `service.spec.externalTrafficPolicy=Local`. The feature works +# only with Weave IPAM (default). +weave_no_masq_local: true + +# set to nft to use nftables backend for iptables (default is iptables) +weave_iptables_backend: ~ + +# Extra variables that passing to launch.sh, useful for enabling seed mode, see +# https://www.weave.works/docs/net/latest/tasks/ipam/ipam/ +weave_extra_args: ~ + +# Extra variables for weave_npc that passing to launch.sh, useful for change log level, ex --log-level=error +weave_npc_extra_args: ~ diff --git a/kubespray/project/roles/network_plugin/weave/meta/main.yml b/kubespray/project/roles/network_plugin/weave/meta/main.yml new file mode 100644 index 0000000..9b7065f --- /dev/null +++ b/kubespray/project/roles/network_plugin/weave/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - role: network_plugin/cni diff --git a/kubespray/project/roles/network_plugin/weave/tasks/main.yml b/kubespray/project/roles/network_plugin/weave/tasks/main.yml new file mode 100644 index 0000000..ae4a5a4 --- /dev/null +++ b/kubespray/project/roles/network_plugin/weave/tasks/main.yml @@ -0,0 +1,12 @@ +--- +- name: Weave | Create manifest + template: + src: weave-net.yml.j2 + dest: "{{ kube_config_dir }}/weave-net.yml" + mode: 0644 + +- name: Weave | Fix nodePort for Weave + template: + src: 10-weave.conflist.j2 + dest: /etc/cni/net.d/10-weave.conflist + mode: 0644 diff --git a/kubespray/project/roles/network_plugin/weave/templates/10-weave.conflist.j2 b/kubespray/project/roles/network_plugin/weave/templates/10-weave.conflist.j2 new file mode 100644 index 0000000..9aab7e9 --- /dev/null +++ b/kubespray/project/roles/network_plugin/weave/templates/10-weave.conflist.j2 @@ -0,0 +1,16 @@ +{ + "cniVersion": "0.3.0", + "name": "weave", + "plugins": [ + { + "name": "weave", + "type": "weave-net", + "hairpinMode": {{ weave_hairpin_mode | bool | lower }} + }, + { + "type": "portmap", + "capabilities": {"portMappings": true}, + "snat": true + } + ] +} diff --git a/kubespray/project/roles/network_plugin/weave/templates/weave-net.yml.j2 b/kubespray/project/roles/network_plugin/weave/templates/weave-net.yml.j2 new file mode 100644 index 0000000..3a38865 --- /dev/null +++ b/kubespray/project/roles/network_plugin/weave/templates/weave-net.yml.j2 @@ -0,0 +1,297 @@ +--- +apiVersion: v1 +kind: List +items: + - apiVersion: v1 + kind: ServiceAccount + metadata: + name: weave-net + labels: + name: weave-net + namespace: kube-system + - apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: weave-net + labels: + name: weave-net + rules: + - apiGroups: + - '' + resources: + - pods + - namespaces + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - extensions + resources: + - networkpolicies + verbs: + - get + - list + - watch + - apiGroups: + - 'networking.k8s.io' + resources: + - networkpolicies + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - nodes/status + verbs: + - patch + - update + - apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: weave-net + labels: + name: weave-net + roleRef: + kind: ClusterRole + name: weave-net + apiGroup: rbac.authorization.k8s.io + subjects: + - kind: ServiceAccount + name: weave-net + namespace: kube-system + - apiVersion: rbac.authorization.k8s.io/v1 + kind: Role + metadata: + name: weave-net + namespace: kube-system + labels: + name: weave-net + rules: + - apiGroups: + - '' + resources: + - configmaps + resourceNames: + - weave-net + verbs: + - get + - update + - apiGroups: + - '' + resources: + - configmaps + verbs: + - create + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: weave-net + namespace: kube-system + labels: + name: weave-net + roleRef: + kind: Role + name: weave-net + apiGroup: rbac.authorization.k8s.io + subjects: + - kind: ServiceAccount + name: weave-net + namespace: kube-system + - apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: weave-net + labels: + name: weave-net + namespace: kube-system + spec: + # Wait 5 seconds to let pod connect before rolling next pod + selector: + matchLabels: + name: weave-net + minReadySeconds: 5 + template: + metadata: + labels: + name: weave-net + spec: + initContainers: + - name: weave-init + image: {{ weave_kube_image_repo }}:{{ weave_kube_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + command: + - /home/weave/init.sh + env: + securityContext: + privileged: true + volumeMounts: + - name: cni-bin + mountPath: /host/opt + - name: cni-bin2 + mountPath: /host/home + - name: cni-conf + mountPath: /host/etc + - name: lib-modules + mountPath: /lib/modules + - name: xtables-lock + mountPath: /run/xtables.lock + readOnly: false + containers: + - name: weave + command: + - /home/weave/launch.sh + env: + - name: INIT_CONTAINER + value: "true" + - name: HOSTNAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: WEAVE_PASSWORD + valueFrom: + secretKeyRef: + name: weave-net + key: WEAVE_PASSWORD + - name: CHECKPOINT_DISABLE + value: "{{ weave_checkpoint_disable | bool | int }}" + - name: CONN_LIMIT + value: "{{ weave_conn_limit | int }}" + - name: HAIRPIN_MODE + value: "{{ weave_hairpin_mode | bool | lower }}" + - name: IPALLOC_RANGE + value: "{{ weave_ipalloc_range }}" + - name: EXPECT_NPC + value: "{{ weave_expect_npc | bool | int }}" +{% if weave_kube_peers %} + - name: KUBE_PEERS + value: "{{ weave_kube_peers }}" +{% endif %} +{% if weave_ipalloc_init %} + - name: IPALLOC_INIT + value: "{{ weave_ipalloc_init }}" +{% endif %} +{% if weave_expose_ip %} + - name: WEAVE_EXPOSE_IP + value: "{{ weave_expose_ip }}" +{% endif %} +{% if weave_metrics_addr %} + - name: WEAVE_METRICS_ADDR + value: "{{ weave_metrics_addr }}" +{% endif %} +{% if weave_status_addr %} + - name: WEAVE_STATUS_ADDR + value: "{{ weave_status_addr }}" +{% endif %} +{% if weave_iptables_backend %} + - name: IPTABLES_BACKEND + value: "{{ weave_iptables_backend }}" +{% endif %} + - name: WEAVE_MTU + value: "{{ weave_mtu | int }}" + - name: NO_MASQ_LOCAL + value: "{{ weave_no_masq_local | bool | int }}" +{% if weave_extra_args %} + - name: EXTRA_ARGS + value: "{{ weave_extra_args }}" +{% endif %} + image: {{ weave_kube_image_repo }}:{{ weave_kube_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + readinessProbe: + httpGet: + host: 127.0.0.1 + path: /status + port: 6784 + resources: + requests: + cpu: 50m + securityContext: + privileged: true + volumeMounts: + - name: weavedb + mountPath: /weavedb + - name: dbus + mountPath: /host/var/lib/dbus + readOnly: true + - mountPath: /host/etc/machine-id + name: cni-machine-id + readOnly: true + - name: xtables-lock + mountPath: /run/xtables.lock + readOnly: false + - name: weave-npc + env: + - name: HOSTNAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName +{% if weave_npc_extra_args %} + - name: EXTRA_ARGS + value: "{{ weave_npc_extra_args }}" +{% endif %} + image: {{ weave_npc_image_repo }}:{{ weave_npc_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + resources: + requests: + cpu: 50m + securityContext: + privileged: true + volumeMounts: + - name: xtables-lock + mountPath: /run/xtables.lock + readOnly: false + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + hostPID: false + restartPolicy: Always + securityContext: + seLinuxOptions: {} + serviceAccountName: weave-net + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + volumes: + - name: weavedb + hostPath: + path: /var/lib/weave + - name: cni-bin + hostPath: + path: /opt + - name: cni-bin2 + hostPath: + path: /home + - name: cni-conf + hostPath: + path: /etc + - name: cni-machine-id + hostPath: + path: /etc/machine-id + - name: dbus + hostPath: + path: /var/lib/dbus + - name: lib-modules + hostPath: + path: /lib/modules + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + priorityClassName: system-node-critical + updateStrategy: + rollingUpdate: + maxUnavailable: {{ serial | default('20%') }} + type: RollingUpdate + - apiVersion: v1 + kind: Secret + metadata: + name: weave-net + namespace: kube-system + data: + WEAVE_PASSWORD: "{{ weave_password | default("") | b64encode }}" diff --git a/kubespray/project/roles/recover_control_plane/OWNERS b/kubespray/project/roles/recover_control_plane/OWNERS new file mode 100644 index 0000000..cb814a1 --- /dev/null +++ b/kubespray/project/roles/recover_control_plane/OWNERS @@ -0,0 +1,8 @@ +# See the OWNERS docs at https://go.k8s.io/owners + +approvers: + - qvicksilver + - yujunz +reviewers: + - qvicksilver + - yujunz diff --git a/kubespray/project/roles/recover_control_plane/control-plane/defaults/main.yml b/kubespray/project/roles/recover_control_plane/control-plane/defaults/main.yml new file mode 100644 index 0000000..229514b --- /dev/null +++ b/kubespray/project/roles/recover_control_plane/control-plane/defaults/main.yml @@ -0,0 +1,2 @@ +--- +bin_dir: /usr/local/bin diff --git a/kubespray/project/roles/recover_control_plane/control-plane/tasks/main.yml b/kubespray/project/roles/recover_control_plane/control-plane/tasks/main.yml new file mode 100644 index 0000000..ec50f3f --- /dev/null +++ b/kubespray/project/roles/recover_control_plane/control-plane/tasks/main.yml @@ -0,0 +1,29 @@ +--- +- name: Wait for apiserver + command: "{{ kubectl }} get nodes" + environment: + KUBECONFIG: "{{ ansible_env.HOME | default('/root') }}/.kube/config" + register: apiserver_is_ready + until: apiserver_is_ready.rc == 0 + retries: 6 + delay: 10 + changed_when: false + when: groups['broken_kube_control_plane'] + +- name: Delete broken kube_control_plane nodes from cluster + command: "{{ kubectl }} delete node {{ item }}" + environment: + KUBECONFIG: "{{ ansible_env.HOME | default('/root') }}/.kube/config" + with_items: "{{ groups['broken_kube_control_plane'] }}" + register: delete_broken_kube_masters + failed_when: false + when: groups['broken_kube_control_plane'] + +- name: Fail if unable to delete broken kube_control_plane nodes from cluster + fail: + msg: "Unable to delete broken kube_control_plane node: {{ item.item }}" + loop: "{{ delete_broken_kube_masters.results }}" + changed_when: false + when: + - groups['broken_kube_control_plane'] + - "item.rc != 0 and not 'NotFound' in item.stderr" diff --git a/kubespray/project/roles/recover_control_plane/etcd/tasks/main.yml b/kubespray/project/roles/recover_control_plane/etcd/tasks/main.yml new file mode 100644 index 0000000..599f56b --- /dev/null +++ b/kubespray/project/roles/recover_control_plane/etcd/tasks/main.yml @@ -0,0 +1,94 @@ +--- +- name: Get etcd endpoint health + command: "{{ bin_dir }}/etcdctl endpoint health" + register: etcd_endpoint_health + ignore_errors: true # noqa ignore-errors + changed_when: false + check_mode: no + environment: + ETCDCTL_API: "3" + ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}" + ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" + ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" + ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem" + when: + - groups['broken_etcd'] + +- name: Set healthy fact + set_fact: + healthy: "{{ etcd_endpoint_health.stderr is match('Error: unhealthy cluster') }}" + when: + - groups['broken_etcd'] + +- name: Set has_quorum fact + set_fact: + has_quorum: "{{ etcd_endpoint_health.stdout_lines | select('match', '.*is healthy.*') | list | length >= etcd_endpoint_health.stderr_lines | select('match', '.*is unhealthy.*') | list | length }}" + when: + - groups['broken_etcd'] + +- name: Recover lost etcd quorum + include_tasks: recover_lost_quorum.yml + when: + - groups['broken_etcd'] + - not has_quorum + +- name: Remove etcd data dir + file: + path: "{{ etcd_data_dir }}" + state: absent + delegate_to: "{{ item }}" + with_items: "{{ groups['broken_etcd'] }}" + ignore_errors: true # noqa ignore-errors + ignore_unreachable: true + when: + - groups['broken_etcd'] + - has_quorum + +- name: Delete old certificates + shell: "rm {{ etcd_cert_dir }}/*{{ item }}*" + with_items: "{{ groups['broken_etcd'] }}" + register: delete_old_cerificates + ignore_errors: true + when: groups['broken_etcd'] + +- name: Fail if unable to delete old certificates + fail: + msg: "Unable to delete old certificates for: {{ item.item }}" + loop: "{{ delete_old_cerificates.results }}" + changed_when: false + when: + - groups['broken_etcd'] + - "item.rc != 0 and not 'No such file or directory' in item.stderr" + +- name: Get etcd cluster members + command: "{{ bin_dir }}/etcdctl member list" + register: member_list + changed_when: false + check_mode: no + environment: + ETCDCTL_API: "3" + ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}" + ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" + ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" + ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem" + when: + - groups['broken_etcd'] + - not healthy + - has_quorum + +- name: Remove broken cluster members + command: "{{ bin_dir }}/etcdctl member remove {{ item[1].replace(' ', '').split(',')[0] }}" + environment: + ETCDCTL_API: "3" + ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}" + ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" + ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" + ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem" + with_nested: + - "{{ groups['broken_etcd'] }}" + - "{{ member_list.stdout_lines }}" + when: + - groups['broken_etcd'] + - not healthy + - has_quorum + - hostvars[item[0]]['etcd_member_name'] == item[1].replace(' ', '').split(',')[2] diff --git a/kubespray/project/roles/recover_control_plane/etcd/tasks/recover_lost_quorum.yml b/kubespray/project/roles/recover_control_plane/etcd/tasks/recover_lost_quorum.yml new file mode 100644 index 0000000..3889628 --- /dev/null +++ b/kubespray/project/roles/recover_control_plane/etcd/tasks/recover_lost_quorum.yml @@ -0,0 +1,59 @@ +--- +- name: Save etcd snapshot + command: "{{ bin_dir }}/etcdctl snapshot save /tmp/snapshot.db" + environment: + ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" + ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" + ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem" + ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses.split(',') | first }}" + ETCDCTL_API: "3" + when: etcd_snapshot is not defined + +- name: Transfer etcd snapshot to host + copy: + src: "{{ etcd_snapshot }}" + dest: /tmp/snapshot.db + mode: 0640 + when: etcd_snapshot is defined + +- name: Stop etcd + systemd: + name: etcd + state: stopped + +- name: Remove etcd data-dir + file: + path: "{{ etcd_data_dir }}" + state: absent + +- name: Restore etcd snapshot # noqa command-instead-of-shell + shell: "{{ bin_dir }}/etcdctl snapshot restore /tmp/snapshot.db --name {{ etcd_member_name }} --initial-cluster {{ etcd_member_name }}={{ etcd_peer_url }} --initial-cluster-token k8s_etcd --initial-advertise-peer-urls {{ etcd_peer_url }} --data-dir {{ etcd_data_dir }}" + environment: + ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" + ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" + ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem" + ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}" + ETCDCTL_API: "3" + +- name: Remove etcd snapshot + file: + path: /tmp/snapshot.db + state: absent + +- name: Change etcd data-dir owner + file: + path: "{{ etcd_data_dir }}" + owner: etcd + group: etcd + recurse: true + +- name: Reconfigure etcd + replace: + path: /etc/etcd.env + regexp: "^(ETCD_INITIAL_CLUSTER=).*" + replace: '\1{{ etcd_member_name }}={{ etcd_peer_url }}' + +- name: Start etcd + systemd: + name: etcd + state: started diff --git a/kubespray/project/roles/recover_control_plane/post-recover/tasks/main.yml b/kubespray/project/roles/recover_control_plane/post-recover/tasks/main.yml new file mode 100644 index 0000000..a62f912 --- /dev/null +++ b/kubespray/project/roles/recover_control_plane/post-recover/tasks/main.yml @@ -0,0 +1,20 @@ +--- +# TODO: Figure out why kubeadm does not fix this +- name: Set etcd-servers fact + set_fact: + # noqa: jinja[spacing] + etcd_servers: >- + {% for host in groups['etcd'] -%} + {% if not loop.last -%} + https://{{ hostvars[host].access_ip | default(hostvars[host].ip | default(hostvars[host].ansible_default_ipv4['address'])) }}:2379, + {%- endif -%} + {%- if loop.last -%} + https://{{ hostvars[host].access_ip | default(hostvars[host].ip | default(hostvars[host].ansible_default_ipv4['address'])) }}:2379 + {%- endif -%} + {%- endfor -%} + +- name: Update apiserver etcd-servers list + replace: + path: /etc/kubernetes/manifests/kube-apiserver.yaml + regexp: "(etcd-servers=).*" + replace: "\\1{{ etcd_servers }}" diff --git a/kubespray/project/roles/remove-node/post-remove/defaults/main.yml b/kubespray/project/roles/remove-node/post-remove/defaults/main.yml new file mode 100644 index 0000000..11298b9 --- /dev/null +++ b/kubespray/project/roles/remove-node/post-remove/defaults/main.yml @@ -0,0 +1,3 @@ +--- +delete_node_retries: 10 +delete_node_delay_seconds: 3 diff --git a/kubespray/project/roles/remove-node/post-remove/tasks/main.yml b/kubespray/project/roles/remove-node/post-remove/tasks/main.yml new file mode 100644 index 0000000..bc8bfd6 --- /dev/null +++ b/kubespray/project/roles/remove-node/post-remove/tasks/main.yml @@ -0,0 +1,13 @@ +--- +- name: Remove-node | Delete node + command: "{{ kubectl }} delete node {{ kube_override_hostname | default(inventory_hostname) }}" + delegate_to: "{{ groups['kube_control_plane'] | first }}" + when: + - groups['kube_control_plane'] | length > 0 + # ignore servers that are not nodes + - inventory_hostname in groups['k8s_cluster'] and kube_override_hostname | default(inventory_hostname) in nodes.stdout_lines + retries: "{{ delete_node_retries }}" + # Sometimes the api-server can have a short window of indisponibility when we delete a master node + delay: "{{ delete_node_delay_seconds }}" + register: result + until: result is not failed diff --git a/kubespray/project/roles/remove-node/pre-remove/defaults/main.yml b/kubespray/project/roles/remove-node/pre-remove/defaults/main.yml new file mode 100644 index 0000000..deaa8af --- /dev/null +++ b/kubespray/project/roles/remove-node/pre-remove/defaults/main.yml @@ -0,0 +1,6 @@ +--- +allow_ungraceful_removal: false +drain_grace_period: 300 +drain_timeout: 360s +drain_retries: 3 +drain_retry_delay_seconds: 10 diff --git a/kubespray/project/roles/remove-node/pre-remove/tasks/main.yml b/kubespray/project/roles/remove-node/pre-remove/tasks/main.yml new file mode 100644 index 0000000..6f6c314 --- /dev/null +++ b/kubespray/project/roles/remove-node/pre-remove/tasks/main.yml @@ -0,0 +1,43 @@ +--- +- name: Remove-node | List nodes + command: >- + {{ kubectl }} get nodes -o go-template={% raw %}'{{ range .items }}{{ .metadata.name }}{{ "\n" }}{{ end }}'{% endraw %} + register: nodes + when: + - groups['kube_control_plane'] | length > 0 + delegate_to: "{{ groups['kube_control_plane'] | first }}" + changed_when: false + run_once: true + +- name: Remove-node | Drain node except daemonsets resource + command: >- + {{ kubectl }} drain + --force + --ignore-daemonsets + --grace-period {{ drain_grace_period }} + --timeout {{ drain_timeout }} + --delete-emptydir-data {{ kube_override_hostname | default(inventory_hostname) }} + when: + - groups['kube_control_plane'] | length > 0 + # ignore servers that are not nodes + - kube_override_hostname | default(inventory_hostname) in nodes.stdout_lines + register: result + failed_when: result.rc != 0 and not allow_ungraceful_removal + delegate_to: "{{ groups['kube_control_plane'] | first }}" + until: result.rc == 0 or allow_ungraceful_removal + retries: "{{ drain_retries }}" + delay: "{{ drain_retry_delay_seconds }}" + +- name: Remove-node | Wait until Volumes will be detached from the node + command: >- + {{ kubectl }} get volumeattachments -o go-template={% raw %}'{{ range .items }}{{ .spec.nodeName }}{{ "\n" }}{{ end }}'{% endraw %} + register: nodes_with_volumes + delegate_to: "{{ groups['kube_control_plane'] | first }}" + changed_when: false + until: not (kube_override_hostname | default(inventory_hostname) in nodes_with_volumes.stdout_lines) + retries: 3 + delay: "{{ drain_grace_period }}" + when: + - groups['kube_control_plane'] | length > 0 + - not allow_ungraceful_removal + - kube_override_hostname | default(inventory_hostname) in nodes.stdout_lines diff --git a/kubespray/project/roles/remove-node/remove-etcd-node/tasks/main.yml b/kubespray/project/roles/remove-node/remove-etcd-node/tasks/main.yml new file mode 100644 index 0000000..0279018 --- /dev/null +++ b/kubespray/project/roles/remove-node/remove-etcd-node/tasks/main.yml @@ -0,0 +1,58 @@ +--- +- name: Lookup node IP in kubernetes + command: > + {{ kubectl }} get nodes {{ node }} + -o jsonpath='{range .status.addresses[?(@.type=="InternalIP")]}{@.address}{"\n"}{end}' + register: remove_node_ip + when: + - groups['kube_control_plane'] | length > 0 + - inventory_hostname in groups['etcd'] + - ip is not defined + - access_ip is not defined + delegate_to: "{{ groups['etcd'] | first }}" + failed_when: false + +- name: Set node IP + set_fact: + node_ip: "{{ ip | default(access_ip | default(remove_node_ip.stdout)) | trim }}" + when: + - inventory_hostname in groups['etcd'] + +- name: Make sure node_ip is set + assert: + that: node_ip is defined and node_ip | length > 0 + msg: "Etcd node ip is not set !" + when: + - inventory_hostname in groups['etcd'] + +- name: Lookup etcd member id + shell: "set -o pipefail && {{ bin_dir }}/etcdctl member list | grep {{ node_ip }} | cut -d, -f1" + args: + executable: /bin/bash + register: etcd_member_id + ignore_errors: true # noqa ignore-errors + changed_when: false + check_mode: no + tags: + - facts + environment: + ETCDCTL_API: "3" + ETCDCTL_CERT: "{{ kube_cert_dir + '/etcd/server.crt' if etcd_deployment_type == 'kubeadm' else etcd_cert_dir + '/admin-' + groups['etcd'] | first + '.pem' }}" + ETCDCTL_KEY: "{{ kube_cert_dir + '/etcd/server.key' if etcd_deployment_type == 'kubeadm' else etcd_cert_dir + '/admin-' + groups['etcd'] | first + '-key.pem' }}" + ETCDCTL_CACERT: "{{ kube_cert_dir + '/etcd/ca.crt' if etcd_deployment_type == 'kubeadm' else etcd_cert_dir + '/ca.pem' }}" + ETCDCTL_ENDPOINTS: "https://127.0.0.1:2379" + delegate_to: "{{ groups['etcd'] | first }}" + when: inventory_hostname in groups['etcd'] + +- name: Remove etcd member from cluster + command: "{{ bin_dir }}/etcdctl member remove {{ etcd_member_id.stdout }}" + environment: + ETCDCTL_API: "3" + ETCDCTL_CERT: "{{ kube_cert_dir + '/etcd/server.crt' if etcd_deployment_type == 'kubeadm' else etcd_cert_dir + '/admin-' + groups['etcd'] | first + '.pem' }}" + ETCDCTL_KEY: "{{ kube_cert_dir + '/etcd/server.key' if etcd_deployment_type == 'kubeadm' else etcd_cert_dir + '/admin-' + groups['etcd'] | first + '-key.pem' }}" + ETCDCTL_CACERT: "{{ kube_cert_dir + '/etcd/ca.crt' if etcd_deployment_type == 'kubeadm' else etcd_cert_dir + '/ca.pem' }}" + ETCDCTL_ENDPOINTS: "https://127.0.0.1:2379" + delegate_to: "{{ groups['etcd'] | first }}" + when: + - inventory_hostname in groups['etcd'] + - etcd_member_id.stdout | length > 0 diff --git a/kubespray/project/roles/reset/defaults/main.yml b/kubespray/project/roles/reset/defaults/main.yml new file mode 100644 index 0000000..b63f2e2 --- /dev/null +++ b/kubespray/project/roles/reset/defaults/main.yml @@ -0,0 +1,21 @@ +--- +flush_iptables: true +reset_restart_network: true + +reset_restart_network_service_name: >- + {% if ansible_os_family == "RedHat" -%} + {%- + if ansible_distribution_major_version | int >= 8 + or is_fedora_coreos or ansible_distribution == "Fedora" -%} + NetworkManager + {%- else -%} + network + {%- endif -%} + {%- elif ansible_distribution == "Ubuntu" -%} + systemd-networkd + {%- elif ansible_os_family == "Debian" -%} + networking + {%- endif %} + +# crictl stop container grace period +cri_stop_containers_grace_period: 0 diff --git a/kubespray/project/roles/reset/tasks/main.yml b/kubespray/project/roles/reset/tasks/main.yml new file mode 100644 index 0000000..a0fa409 --- /dev/null +++ b/kubespray/project/roles/reset/tasks/main.yml @@ -0,0 +1,440 @@ +--- +- name: Reset | stop services + service: + name: "{{ item }}" + state: stopped + with_items: + - kubelet.service + - cri-dockerd.service + - cri-dockerd.socket + failed_when: false + tags: + - services + +- name: Reset | remove services + file: + path: "/etc/systemd/system/{{ item }}" + state: absent + with_items: + - kubelet.service + - cri-dockerd.service + - cri-dockerd.socket + - calico-node.service + - containerd.service.d/http-proxy.conf + - crio.service.d/http-proxy.conf + - k8s-certs-renew.service + - k8s-certs-renew.timer + register: services_removed + tags: + - services + - containerd + - crio + +- name: Reset | Remove Docker + include_role: + name: container-engine/docker + tasks_from: reset + when: container_manager == 'docker' + tags: + - docker + +- name: Reset | systemctl daemon-reload # noqa no-handler + systemd: + daemon_reload: true + when: services_removed.changed + +- name: Reset | check if crictl is present + stat: + path: "{{ bin_dir }}/crictl" + get_attributes: no + get_checksum: no + get_mime: no + register: crictl + +- name: Reset | stop all cri containers + shell: "set -o pipefail && {{ bin_dir }}/crictl ps -q | xargs -r {{ bin_dir }}/crictl -t 60s stop -t {{ cri_stop_containers_grace_period }}" + args: + executable: /bin/bash + register: remove_all_cri_containers + retries: 5 + until: remove_all_cri_containers.rc == 0 + delay: 5 + tags: + - crio + - containerd + when: + - crictl.stat.exists + - container_manager in ["crio", "containerd"] + - ansible_facts.services['containerd.service'] is defined or ansible_facts.services['cri-o.service'] is defined + ignore_errors: true # noqa ignore-errors + +- name: Reset | force remove all cri containers + command: "{{ bin_dir }}/crictl rm -a -f" + register: remove_all_cri_containers + retries: 5 + until: remove_all_cri_containers.rc == 0 + delay: 5 + tags: + - crio + - containerd + when: + - crictl.stat.exists + - container_manager in ["crio", "containerd"] + - deploy_container_engine + - ansible_facts.services['containerd.service'] is defined or ansible_facts.services['cri-o.service'] is defined + ignore_errors: true # noqa ignore-errors + +- name: Reset | stop and disable crio service + service: + name: crio + state: stopped + enabled: false + failed_when: false + tags: [ crio ] + when: container_manager == "crio" + +- name: Reset | forcefully wipe CRI-O's container and image storage + command: "crio wipe -f" + failed_when: false + tags: [ crio ] + when: container_manager == "crio" + +- name: Reset | stop all cri pods + shell: "set -o pipefail && {{ bin_dir }}/crictl pods -q | xargs -r {{ bin_dir }}/crictl -t 60s stopp" + args: + executable: /bin/bash + register: remove_all_cri_containers + retries: 5 + until: remove_all_cri_containers.rc == 0 + delay: 5 + tags: [ containerd ] + when: + - crictl.stat.exists + - container_manager == "containerd" + - ansible_facts.services['containerd.service'] is defined or ansible_facts.services['cri-o.service'] is defined + ignore_errors: true # noqa ignore-errors + +- name: Reset | force remove all cri pods + block: + - name: Reset | force remove all cri pods + command: "{{ bin_dir }}/crictl rmp -a -f" + register: remove_all_cri_containers + retries: 5 + until: remove_all_cri_containers.rc == 0 + delay: 5 + tags: [ containerd ] + when: + - crictl.stat.exists + - container_manager == "containerd" + - ansible_facts.services['containerd.service'] is defined or ansible_facts.services['cri-o.service'] is defined + + rescue: + - name: Reset | force remove all cri pods (rescue) + shell: "ip netns list | cut -d' ' -f 1 | xargs -n1 ip netns delete && {{ bin_dir }}/crictl rmp -a -f" + ignore_errors: true # noqa ignore-errors + changed_when: true + +- name: Reset | stop etcd services + service: + name: "{{ item }}" + state: stopped + with_items: + - etcd + - etcd-events + failed_when: false + tags: + - services + +- name: Reset | remove etcd services + file: + path: "/etc/systemd/system/{{ item }}.service" + state: absent + with_items: + - etcd + - etcd-events + register: services_removed + tags: + - services + +- name: Reset | remove containerd + when: container_manager == 'containerd' + block: + - name: Reset | stop containerd service + service: + name: containerd + state: stopped + failed_when: false + tags: + - services + + - name: Reset | remove containerd service + file: + path: /etc/systemd/system/containerd.service + state: absent + register: services_removed + tags: + - services + +- name: Reset | gather mounted kubelet dirs + shell: set -o pipefail && mount | grep /var/lib/kubelet/ | awk '{print $3}' | tac + args: + executable: /bin/bash + check_mode: no + register: mounted_dirs + failed_when: false + changed_when: false + tags: + - mounts + +- name: Reset | unmount kubelet dirs + command: umount -f {{ item }} + with_items: "{{ mounted_dirs.stdout_lines }}" + register: umount_dir + when: mounted_dirs + retries: 4 + until: umount_dir.rc == 0 + delay: 5 + tags: + - mounts + +- name: Flush iptables + iptables: + table: "{{ item }}" + flush: yes + with_items: + - filter + - nat + - mangle + - raw + when: flush_iptables | bool + tags: + - iptables + +- name: Flush ip6tables + iptables: + table: "{{ item }}" + flush: yes + ip_version: ipv6 + with_items: + - filter + - nat + - mangle + - raw + when: flush_iptables | bool and enable_dual_stack_networks + tags: + - ip6tables + +- name: Clear IPVS virtual server table + command: "ipvsadm -C" + ignore_errors: true # noqa ignore-errors + when: + - kube_proxy_mode == 'ipvs' and inventory_hostname in groups['k8s_cluster'] + +- name: Reset | check kube-ipvs0 network device + stat: + path: /sys/class/net/kube-ipvs0 + get_attributes: no + get_checksum: no + get_mime: no + register: kube_ipvs0 + +- name: Reset | Remove kube-ipvs0 + command: "ip link del kube-ipvs0" + when: + - kube_proxy_mode == 'ipvs' + - kube_ipvs0.stat.exists + +- name: Reset | check nodelocaldns network device + stat: + path: /sys/class/net/nodelocaldns + get_attributes: no + get_checksum: no + get_mime: no + register: nodelocaldns_device + +- name: Reset | Remove nodelocaldns + command: "ip link del nodelocaldns" + when: + - enable_nodelocaldns | default(false) | bool + - nodelocaldns_device.stat.exists + +- name: Reset | Check whether /var/lib/kubelet directory exists + stat: + path: /var/lib/kubelet + get_attributes: no + get_checksum: no + get_mime: no + register: var_lib_kubelet_directory + +- name: Reset | Find files/dirs with immutable flag in /var/lib/kubelet + command: lsattr -laR /var/lib/kubelet + become: true + register: var_lib_kubelet_files_dirs_w_attrs + changed_when: false + no_log: true + when: var_lib_kubelet_directory.stat.exists + +- name: Reset | Remove immutable flag from files/dirs in /var/lib/kubelet + file: + path: "{{ filedir_path }}" + state: touch + attributes: "-i" + mode: 0644 + loop: "{{ var_lib_kubelet_files_dirs_w_attrs.stdout_lines | select('search', 'Immutable') | list }}" + loop_control: + loop_var: file_dir_line + label: "{{ filedir_path }}" + vars: + filedir_path: "{{ file_dir_line.split(' ')[0] }}" + when: var_lib_kubelet_directory.stat.exists + +- name: Reset | delete some files and directories + file: + path: "{{ item }}" + state: absent + with_items: + - "{{ kube_config_dir }}" + - /var/lib/kubelet + - "{{ containerd_storage_dir }}" + - "{{ ansible_env.HOME | default('/root') }}/.kube" + - "{{ ansible_env.HOME | default('/root') }}/.helm" + - "{{ ansible_env.HOME | default('/root') }}/.config/helm" + - "{{ ansible_env.HOME | default('/root') }}/.cache/helm" + - "{{ ansible_env.HOME | default('/root') }}/.local/share/helm" + - "{{ etcd_data_dir }}" + - "{{ etcd_events_data_dir }}" + - "{{ etcd_config_dir }}" + - /var/log/calico + - /var/log/openvswitch + - /var/log/ovn + - /var/log/kube-ovn + - /etc/cni + - /etc/nerdctl + - "{{ nginx_config_dir }}" + - /etc/dnsmasq.d + - /etc/dnsmasq.conf + - /etc/dnsmasq.d-available + - /etc/systemd/resolved.conf.d/kubespray.conf + - /etc/etcd.env + - /etc/calico + - /etc/NetworkManager/conf.d/calico.conf + - /etc/NetworkManager/conf.d/k8s.conf + - /etc/weave.env + - /opt/cni + - /etc/dhcp/dhclient.d/zdnsupdate.sh + - /etc/dhcp/dhclient-exit-hooks.d/zdnsupdate + - /run/flannel + - /etc/flannel + - /run/kubernetes + - /usr/local/share/ca-certificates/etcd-ca.crt + - /usr/local/share/ca-certificates/kube-ca.crt + - /etc/ssl/certs/etcd-ca.pem + - /etc/ssl/certs/kube-ca.pem + - /etc/pki/ca-trust/source/anchors/etcd-ca.crt + - /etc/pki/ca-trust/source/anchors/kube-ca.crt + - /var/log/pods/ + - "{{ bin_dir }}/kubelet" + - "{{ bin_dir }}/cri-dockerd" + - "{{ bin_dir }}/etcd-scripts" + - "{{ bin_dir }}/etcd" + - "{{ bin_dir }}/etcd-events" + - "{{ bin_dir }}/etcdctl" + - "{{ bin_dir }}/etcdctl.sh" + - "{{ bin_dir }}/kubernetes-scripts" + - "{{ bin_dir }}/kubectl" + - "{{ bin_dir }}/kubeadm" + - "{{ bin_dir }}/helm" + - "{{ bin_dir }}/calicoctl" + - "{{ bin_dir }}/calicoctl.sh" + - "{{ bin_dir }}/calico-upgrade" + - "{{ bin_dir }}/weave" + - "{{ bin_dir }}/crictl" + - "{{ bin_dir }}/nerdctl" + - "{{ bin_dir }}/netctl" + - "{{ bin_dir }}/k8s-certs-renew.sh" + - /var/lib/cni + - /etc/openvswitch + - /run/openvswitch + - /var/lib/kube-router + - /var/lib/calico + - /etc/cilium + - /run/calico + - /etc/bash_completion.d/kubectl.sh + - /etc/bash_completion.d/crictl + - /etc/bash_completion.d/nerdctl + - /etc/bash_completion.d/krew + - /etc/bash_completion.d/krew.sh + - "{{ krew_root_dir }}" + - /etc/modules-load.d/kube_proxy-ipvs.conf + - /etc/modules-load.d/kubespray-br_netfilter.conf + - /etc/modules-load.d/kubespray-kata-containers.conf + - /usr/libexec/kubernetes + - /etc/origin/openvswitch + - /etc/origin/ovn + - "{{ sysctl_file_path }}" + - /etc/crictl.yaml + ignore_errors: true # noqa ignore-errors + tags: + - files + +- name: Reset | remove containerd binary files + file: + path: "{{ containerd_bin_dir }}/{{ item }}" + state: absent + with_items: + - containerd + - containerd-shim + - containerd-shim-runc-v1 + - containerd-shim-runc-v2 + - containerd-stress + - crictl + - critest + - ctd-decoder + - ctr + - runc + ignore_errors: true # noqa ignore-errors + when: container_manager == 'containerd' + tags: + - files + +- name: Reset | remove dns settings from dhclient.conf + blockinfile: + path: "{{ item }}" + state: absent + marker: "# Ansible entries {mark}" + failed_when: false + with_items: + - /etc/dhclient.conf + - /etc/dhcp/dhclient.conf + tags: + - files + - dns + +- name: Reset | remove host entries from /etc/hosts + blockinfile: + path: "/etc/hosts" + state: absent + marker: "# Ansible inventory hosts {mark}" + tags: + - files + - dns + +- name: Reset | include file with reset tasks specific to the network_plugin if exists + include_role: + name: "network_plugin/{{ kube_network_plugin }}" + tasks_from: reset + when: + - kube_network_plugin in ['flannel', 'cilium', 'kube-router', 'calico'] + tags: + - network + +- name: Reset | Restart network + service: + name: "{{ reset_restart_network_service_name }}" + state: restarted + when: + - ansible_os_family not in ["Flatcar", "Flatcar Container Linux by Kinvolk"] + - reset_restart_network | bool + tags: + - services + - network diff --git a/kubespray/project/roles/upgrade/post-upgrade/defaults/main.yml b/kubespray/project/roles/upgrade/post-upgrade/defaults/main.yml new file mode 100644 index 0000000..aa72843 --- /dev/null +++ b/kubespray/project/roles/upgrade/post-upgrade/defaults/main.yml @@ -0,0 +1,5 @@ +--- +# how long to wait for cilium after upgrade before uncordoning +upgrade_post_cilium_wait_timeout: 120s +upgrade_node_post_upgrade_confirm: false +upgrade_node_post_upgrade_pause_seconds: 0 diff --git a/kubespray/project/roles/upgrade/post-upgrade/tasks/main.yml b/kubespray/project/roles/upgrade/post-upgrade/tasks/main.yml new file mode 100644 index 0000000..434ef1e --- /dev/null +++ b/kubespray/project/roles/upgrade/post-upgrade/tasks/main.yml @@ -0,0 +1,32 @@ +--- +- name: Wait for cilium + when: + - needs_cordoning | default(false) + - kube_network_plugin == 'cilium' + command: > + {{ kubectl }} + wait pod -n kube-system -l k8s-app=cilium + --field-selector 'spec.nodeName=={{ kube_override_hostname | default(inventory_hostname) }}' + --for=condition=Ready + --timeout={{ upgrade_post_cilium_wait_timeout }} + delegate_to: "{{ groups['kube_control_plane'][0] }}" + +- name: Confirm node uncordon + pause: + echo: yes + prompt: "Ready to uncordon node?" + when: + - upgrade_node_post_upgrade_confirm + +- name: Wait before uncordoning node + pause: + seconds: "{{ upgrade_node_post_upgrade_pause_seconds }}" + when: + - not upgrade_node_post_upgrade_confirm + - upgrade_node_post_upgrade_pause_seconds != 0 + +- name: Uncordon node + command: "{{ kubectl }} uncordon {{ kube_override_hostname | default(inventory_hostname) }}" + delegate_to: "{{ groups['kube_control_plane'][0] }}" + when: + - needs_cordoning | default(false) diff --git a/kubespray/project/roles/upgrade/pre-upgrade/defaults/main.yml b/kubespray/project/roles/upgrade/pre-upgrade/defaults/main.yml new file mode 100644 index 0000000..900b834 --- /dev/null +++ b/kubespray/project/roles/upgrade/pre-upgrade/defaults/main.yml @@ -0,0 +1,20 @@ +--- +drain_grace_period: 300 +drain_timeout: 360s +drain_pod_selector: "" +drain_nodes: true +drain_retries: 3 +drain_retry_delay_seconds: 10 + +drain_fallback_enabled: false +drain_fallback_grace_period: 300 +drain_fallback_timeout: 360s +drain_fallback_retries: 0 +drain_fallback_retry_delay_seconds: 10 + +upgrade_node_always_cordon: false +upgrade_node_uncordon_after_drain_failure: true +upgrade_node_fail_if_drain_fails: true + +upgrade_node_confirm: false +upgrade_node_pause_seconds: 0 diff --git a/kubespray/project/roles/upgrade/pre-upgrade/tasks/main.yml b/kubespray/project/roles/upgrade/pre-upgrade/tasks/main.yml new file mode 100644 index 0000000..8d5d99c --- /dev/null +++ b/kubespray/project/roles/upgrade/pre-upgrade/tasks/main.yml @@ -0,0 +1,98 @@ +--- +# Wait for upgrade +- name: Confirm node upgrade + pause: + echo: yes + prompt: "Ready to upgrade node? (Press Enter to continue or Ctrl+C for other options)" + when: + - upgrade_node_confirm + +- name: Wait before upgrade node + pause: + seconds: "{{ upgrade_node_pause_seconds }}" + when: + - not upgrade_node_confirm + - upgrade_node_pause_seconds != 0 + +# Node Ready: type = ready, status = True +# Node NotReady: type = ready, status = Unknown +- name: See if node is in ready state + command: > + {{ kubectl }} get node {{ kube_override_hostname | default(inventory_hostname) }} + -o jsonpath='{ range .status.conditions[?(@.type == "Ready")].status }{ @ }{ end }' + register: kubectl_node_ready + delegate_to: "{{ groups['kube_control_plane'][0] }}" + failed_when: false + changed_when: false + +# SchedulingDisabled: unschedulable = true +# else unschedulable key doesn't exist +- name: See if node is schedulable + command: > + {{ kubectl }} get node {{ kube_override_hostname | default(inventory_hostname) }} + -o jsonpath='{ .spec.unschedulable }' + register: kubectl_node_schedulable + delegate_to: "{{ groups['kube_control_plane'][0] }}" + failed_when: false + changed_when: false + +- name: Set if node needs cordoning + set_fact: + needs_cordoning: "{{ (kubectl_node_ready.stdout == 'True' and not kubectl_node_schedulable.stdout) or upgrade_node_always_cordon }}" + +- name: Node draining + delegate_to: "{{ groups['kube_control_plane'][0] }}" + when: + - needs_cordoning + block: + - name: Cordon node + command: "{{ kubectl }} cordon {{ kube_override_hostname | default(inventory_hostname) }}" + delegate_to: "{{ groups['kube_control_plane'][0] }}" + changed_when: true + + - name: Drain node + command: >- + {{ kubectl }} drain + --force + --ignore-daemonsets + --grace-period {{ drain_grace_period }} + --timeout {{ drain_timeout }} + --delete-emptydir-data {{ kube_override_hostname | default(inventory_hostname) }} + {% if drain_pod_selector %}--pod-selector '{{ drain_pod_selector }}'{% endif %} + when: drain_nodes + register: result + failed_when: + - result.rc != 0 + - not drain_fallback_enabled + until: result.rc == 0 + retries: "{{ drain_retries }}" + delay: "{{ drain_retry_delay_seconds }}" + + - name: Drain node - fallback with disabled eviction + when: + - drain_nodes + - drain_fallback_enabled + - result.rc != 0 + command: >- + {{ kubectl }} drain + --force + --ignore-daemonsets + --grace-period {{ drain_fallback_grace_period }} + --timeout {{ drain_fallback_timeout }} + --delete-emptydir-data {{ kube_override_hostname | default(inventory_hostname) }} + {% if drain_pod_selector %}--pod-selector '{{ drain_pod_selector }}'{% endif %} + --disable-eviction + register: drain_fallback_result + until: drain_fallback_result.rc == 0 + retries: "{{ drain_fallback_retries }}" + delay: "{{ drain_fallback_retry_delay_seconds }}" + changed_when: drain_fallback_result.rc == 0 + + rescue: + - name: Set node back to schedulable + command: "{{ kubectl }} uncordon {{ kube_override_hostname | default(inventory_hostname) }}" + when: upgrade_node_uncordon_after_drain_failure + - name: Fail after rescue + fail: + msg: "Failed to drain node {{ kube_override_hostname | default(inventory_hostname) }}" + when: upgrade_node_fail_if_drain_fails diff --git a/kubespray/project/roles/upgrade/system-upgrade/tasks/apt.yml b/kubespray/project/roles/upgrade/system-upgrade/tasks/apt.yml new file mode 100644 index 0000000..992bbce --- /dev/null +++ b/kubespray/project/roles/upgrade/system-upgrade/tasks/apt.yml @@ -0,0 +1,13 @@ +--- +- name: APT Dist-Upgrade + apt: + upgrade: dist + autoremove: true + dpkg_options: force-confold,force-confdef + register: apt_upgrade + +- name: Reboot after APT Dist-Upgrade # noqa no-handler + when: + - apt_upgrade.changed or system_upgrade_reboot == 'always' + - system_upgrade_reboot != 'never' + reboot: diff --git a/kubespray/project/roles/upgrade/system-upgrade/tasks/main.yml b/kubespray/project/roles/upgrade/system-upgrade/tasks/main.yml new file mode 100644 index 0000000..61561b1 --- /dev/null +++ b/kubespray/project/roles/upgrade/system-upgrade/tasks/main.yml @@ -0,0 +1,17 @@ +--- +- name: APT upgrade + when: + - system_upgrade + - ansible_os_family == "Debian" + include_tasks: apt.yml + tags: + - system-upgrade-apt + +- name: YUM upgrade + when: + - system_upgrade + - ansible_os_family == "RedHat" + - not is_fedora_coreos + include_tasks: yum.yml + tags: + - system-upgrade-yum diff --git a/kubespray/project/roles/upgrade/system-upgrade/tasks/yum.yml b/kubespray/project/roles/upgrade/system-upgrade/tasks/yum.yml new file mode 100644 index 0000000..6a27177 --- /dev/null +++ b/kubespray/project/roles/upgrade/system-upgrade/tasks/yum.yml @@ -0,0 +1,12 @@ +--- +- name: YUM upgrade all packages # noqa package-latest + yum: + name: '*' + state: latest + register: yum_upgrade + +- name: Reboot after YUM upgrade # noqa no-handler + when: + - yum_upgrade.changed or system_upgrade_reboot == 'always' + - system_upgrade_reboot != 'never' + reboot: diff --git a/kubespray/project/roles/win_nodes/kubernetes_patch/defaults/main.yml b/kubespray/project/roles/win_nodes/kubernetes_patch/defaults/main.yml new file mode 100644 index 0000000..954cb51 --- /dev/null +++ b/kubespray/project/roles/win_nodes/kubernetes_patch/defaults/main.yml @@ -0,0 +1,4 @@ +--- + +kubernetes_user_manifests_path: "{{ ansible_env.HOME }}/kube-manifests" +kube_proxy_nodeselector: "kubernetes.io/os" diff --git a/kubespray/project/roles/win_nodes/kubernetes_patch/tasks/main.yml b/kubespray/project/roles/win_nodes/kubernetes_patch/tasks/main.yml new file mode 100644 index 0000000..880c58c --- /dev/null +++ b/kubespray/project/roles/win_nodes/kubernetes_patch/tasks/main.yml @@ -0,0 +1,41 @@ +--- + +- name: Ensure that user manifests directory exists + file: + path: "{{ kubernetes_user_manifests_path }}/kubernetes" + state: directory + recurse: yes + tags: [init, cni] + +- name: Apply kube-proxy nodeselector + tags: init + when: + - kube_proxy_deployed + block: + # Due to https://github.com/kubernetes/kubernetes/issues/58212 we cannot rely on exit code for "kubectl patch" + - name: Check current nodeselector for kube-proxy daemonset + command: >- + {{ kubectl }} + get ds kube-proxy --namespace=kube-system + -o jsonpath={.spec.template.spec.nodeSelector.{{ kube_proxy_nodeselector | regex_replace('\.', '\\.') }}} + register: current_kube_proxy_state + retries: 60 + delay: 5 + until: current_kube_proxy_state is succeeded + changed_when: false + + - name: Apply nodeselector patch for kube-proxy daemonset + command: > + {{ kubectl }} + patch ds kube-proxy --namespace=kube-system --type=strategic -p + '{"spec":{"template":{"spec":{"nodeSelector":{"{{ kube_proxy_nodeselector }}":"linux"} }}}}' + register: patch_kube_proxy_state + when: current_kube_proxy_state.stdout | trim | lower != "linux" + + - debug: # noqa name[missing] + msg: "{{ patch_kube_proxy_state.stdout_lines }}" + when: patch_kube_proxy_state is not skipped + + - debug: # noqa name[missing] + msg: "{{ patch_kube_proxy_state.stderr_lines }}" + when: patch_kube_proxy_state is not skipped diff --git a/kubespray/project/scale.yml b/kubespray/project/scale.yml new file mode 100644 index 0000000..171e378 --- /dev/null +++ b/kubespray/project/scale.yml @@ -0,0 +1,104 @@ +--- +- name: Common tasks for every playbooks + import_playbook: boilerplate.yml + +- name: Gather facts + import_playbook: facts.yml + +- name: Generate the etcd certificates beforehand + hosts: etcd:kube_control_plane + gather_facts: False + any_errors_fatal: "{{ any_errors_fatal | default(true) }}" + environment: "{{ proxy_disable_env }}" + roles: + - { role: kubespray-defaults } + - role: etcd + tags: etcd + vars: + etcd_cluster_setup: false + etcd_events_cluster_setup: false + when: + - etcd_deployment_type != "kubeadm" + - kube_network_plugin in ["calico", "flannel", "canal", "cilium"] or cilium_deploy_additionally | default(false) | bool + - kube_network_plugin != "calico" or calico_datastore == "etcd" + +- name: Download images to ansible host cache via first kube_control_plane node + hosts: kube_control_plane[0] + gather_facts: False + any_errors_fatal: "{{ any_errors_fatal | default(true) }}" + environment: "{{ proxy_disable_env }}" + roles: + - { role: kubespray-defaults, when: "not skip_downloads and download_run_once and not download_localhost" } + - { role: kubernetes/preinstall, tags: preinstall, when: "not skip_downloads and download_run_once and not download_localhost" } + - { role: download, tags: download, when: "not skip_downloads and download_run_once and not download_localhost" } + +- name: Target only workers to get kubelet installed and checking in on any new nodes(engine) + hosts: kube_node + gather_facts: False + any_errors_fatal: "{{ any_errors_fatal | default(true) }}" + environment: "{{ proxy_disable_env }}" + roles: + - { role: kubespray-defaults } + - { role: kubernetes/preinstall, tags: preinstall } + - { role: container-engine, tags: "container-engine", when: deploy_container_engine } + - { role: download, tags: download, when: "not skip_downloads" } + - role: etcd + tags: etcd + vars: + etcd_cluster_setup: false + when: + - etcd_deployment_type != "kubeadm" + - kube_network_plugin in ["calico", "flannel", "canal", "cilium"] or cilium_deploy_additionally | default(false) | bool + - kube_network_plugin != "calico" or calico_datastore == "etcd" + +- name: Target only workers to get kubelet installed and checking in on any new nodes(node) + hosts: kube_node + gather_facts: False + any_errors_fatal: "{{ any_errors_fatal | default(true) }}" + environment: "{{ proxy_disable_env }}" + roles: + - { role: kubespray-defaults } + - { role: kubernetes/node, tags: node } + +- name: Upload control plane certs and retrieve encryption key + hosts: kube_control_plane | first + environment: "{{ proxy_disable_env }}" + gather_facts: False + tags: kubeadm + roles: + - { role: kubespray-defaults } + tasks: + - name: Upload control plane certificates + command: >- + {{ bin_dir }}/kubeadm init phase + --config {{ kube_config_dir }}/kubeadm-config.yaml + upload-certs + --upload-certs + environment: "{{ proxy_disable_env }}" + register: kubeadm_upload_cert + changed_when: false + - name: Set fact 'kubeadm_certificate_key' for later use + set_fact: + kubeadm_certificate_key: "{{ kubeadm_upload_cert.stdout_lines[-1] | trim }}" + when: kubeadm_certificate_key is not defined + +- name: Target only workers to get kubelet installed and checking in on any new nodes(network) + hosts: kube_node + gather_facts: False + any_errors_fatal: "{{ any_errors_fatal | default(true) }}" + environment: "{{ proxy_disable_env }}" + roles: + - { role: kubespray-defaults } + - { role: kubernetes/kubeadm, tags: kubeadm } + - { role: kubernetes/node-label, tags: node-label } + - { role: kubernetes/node-taint, tags: node-taint } + - { role: network_plugin, tags: network } + +- name: Apply resolv.conf changes now that cluster DNS is up + hosts: k8s_cluster + gather_facts: False + any_errors_fatal: "{{ any_errors_fatal | default(true) }}" + environment: "{{ proxy_disable_env }}" + roles: + - { role: kubespray-defaults } + - { role: kubernetes/preinstall, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf, dns_late: true } diff --git a/kubespray/project/upgrade_cluster.yml b/kubespray/project/upgrade_cluster.yml new file mode 100644 index 0000000..3180fec --- /dev/null +++ b/kubespray/project/upgrade_cluster.yml @@ -0,0 +1,130 @@ +--- +- name: Common tasks for every playbooks + import_playbook: boilerplate.yml + +- name: Gather facts + import_playbook: facts.yml + +- name: Download images to ansible host cache via first kube_control_plane node + hosts: kube_control_plane[0] + gather_facts: False + any_errors_fatal: "{{ any_errors_fatal | default(true) }}" + environment: "{{ proxy_disable_env }}" + roles: + - { role: kubespray-defaults, when: "not skip_downloads and download_run_once and not download_localhost"} + - { role: kubernetes/preinstall, tags: preinstall, when: "not skip_downloads and download_run_once and not download_localhost" } + - { role: download, tags: download, when: "not skip_downloads and download_run_once and not download_localhost" } + +- name: Prepare nodes for upgrade + hosts: k8s_cluster:etcd:calico_rr + gather_facts: False + any_errors_fatal: "{{ any_errors_fatal | default(true) }}" + environment: "{{ proxy_disable_env }}" + roles: + - { role: kubespray-defaults } + - { role: kubernetes/preinstall, tags: preinstall } + - { role: download, tags: download, when: "not skip_downloads" } + +- name: Upgrade container engine on non-cluster nodes + hosts: etcd:calico_rr:!k8s_cluster + gather_facts: False + any_errors_fatal: "{{ any_errors_fatal | default(true) }}" + environment: "{{ proxy_disable_env }}" + serial: "{{ serial | default('20%') }}" + roles: + - { role: kubespray-defaults } + - { role: container-engine, tags: "container-engine", when: deploy_container_engine } + +- name: Install etcd + import_playbook: install_etcd.yml + +- name: Handle upgrades to master components first to maintain backwards compat. + gather_facts: False + hosts: kube_control_plane + any_errors_fatal: "{{ any_errors_fatal | default(true) }}" + environment: "{{ proxy_disable_env }}" + serial: 1 + roles: + - { role: kubespray-defaults } + - { role: upgrade/pre-upgrade, tags: pre-upgrade } + - { role: upgrade/system-upgrade, tags: system-upgrade } + - { role: download, tags: download, when: "system_upgrade and system_upgrade_reboot != 'never' and not skip_downloads" } + - { role: kubernetes-apps/kubelet-csr-approver, tags: kubelet-csr-approver } + - { role: container-engine, tags: "container-engine", when: deploy_container_engine } + - { role: kubernetes/node, tags: node } + - { role: kubernetes/control-plane, tags: master, upgrade_cluster_setup: true } + - { role: kubernetes/client, tags: client } + - { role: kubernetes/node-label, tags: node-label } + - { role: kubernetes/node-taint, tags: node-taint } + - { role: kubernetes-apps/cluster_roles, tags: cluster-roles } + - { role: kubernetes-apps, tags: csi-driver } + - { role: upgrade/post-upgrade, tags: post-upgrade } + +- name: Upgrade calico and external cloud provider on all masters, calico-rrs, and nodes + hosts: kube_control_plane:calico_rr:kube_node + gather_facts: False + any_errors_fatal: "{{ any_errors_fatal | default(true) }}" + serial: "{{ serial | default('20%') }}" + environment: "{{ proxy_disable_env }}" + roles: + - { role: kubespray-defaults } + - { role: kubernetes-apps/external_cloud_controller, tags: external-cloud-controller } + - { role: network_plugin, tags: network } + - { role: kubernetes-apps/network_plugin, tags: network } + - { role: kubernetes-apps/policy_controller, tags: policy-controller } + +- name: Finally handle worker upgrades, based on given batch size + hosts: kube_node:calico_rr:!kube_control_plane + gather_facts: False + any_errors_fatal: "{{ any_errors_fatal | default(true) }}" + environment: "{{ proxy_disable_env }}" + serial: "{{ serial | default('20%') }}" + roles: + - { role: kubespray-defaults } + - { role: upgrade/pre-upgrade, tags: pre-upgrade } + - { role: upgrade/system-upgrade, tags: system-upgrade } + - { role: download, tags: download, when: "system_upgrade and system_upgrade_reboot != 'never' and not skip_downloads" } + - { role: container-engine, tags: "container-engine", when: deploy_container_engine } + - { role: kubernetes/node, tags: node } + - { role: kubernetes/kubeadm, tags: kubeadm } + - { role: kubernetes/node-label, tags: node-label } + - { role: kubernetes/node-taint, tags: node-taint } + - { role: upgrade/post-upgrade, tags: post-upgrade } + +- name: Patch Kubernetes for Windows + hosts: kube_control_plane[0] + gather_facts: False + any_errors_fatal: true + environment: "{{ proxy_disable_env }}" + roles: + - { role: kubespray-defaults } + - { role: win_nodes/kubernetes_patch, tags: ["master", "win_nodes"] } + +- name: Install Calico Route Reflector + hosts: calico_rr + gather_facts: False + any_errors_fatal: "{{ any_errors_fatal | default(true) }}" + environment: "{{ proxy_disable_env }}" + roles: + - { role: kubespray-defaults } + - { role: network_plugin/calico/rr, tags: network } + +- name: Install Kubernetes apps + hosts: kube_control_plane + gather_facts: False + any_errors_fatal: "{{ any_errors_fatal | default(true) }}" + environment: "{{ proxy_disable_env }}" + roles: + - { role: kubespray-defaults } + - { role: kubernetes-apps/ingress_controller, tags: ingress-controller } + - { role: kubernetes-apps/external_provisioner, tags: external-provisioner } + - { role: kubernetes-apps, tags: apps } + +- name: Apply resolv.conf changes now that cluster DNS is up + hosts: k8s_cluster + gather_facts: False + any_errors_fatal: "{{ any_errors_fatal | default(true) }}" + environment: "{{ proxy_disable_env }}" + roles: + - { role: kubespray-defaults } + - { role: kubernetes/preinstall, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf, dns_late: true } diff --git a/pkg/config/ansible_playbook_config.go b/pkg/config/ansible_playbook_config.go new file mode 100644 index 0000000..047cc8b --- /dev/null +++ b/pkg/config/ansible_playbook_config.go @@ -0,0 +1,12 @@ +package config + +import ( + "github.com/apenella/go-ansible/pkg/options" + "github.com/apenella/go-ansible/pkg/playbook" +) + +type AnsiblePlaybookConfig struct { + ConnectionOptions options.AnsibleConnectionOptions + PrivilegeEscalationOptions options.AnsiblePrivilegeEscalationOptions + PlaybookOptions playbook.AnsiblePlaybookOptions +} diff --git a/pkg/config/config.go b/pkg/config/config.go new file mode 100644 index 0000000..2d068ff --- /dev/null +++ b/pkg/config/config.go @@ -0,0 +1,66 @@ +package config + +import ( + "fmt" + + "github.com/ilyakaznacheev/cleanenv" +) + +type Host struct { + Hostname string `yaml:"hostname"` + Ip string `yaml:"ip"` + Roles []string `yaml:"roles"` + User string `yaml:"user"` + Password string `yaml:"password"` +} + +type Config struct { + DataDir string + Credentials struct { + User string `yaml:"user"` + Password string `yaml:"password"` + AskSudoPassword bool `yaml:"ask_sudo_password"` + PrivateKeyFile string `yaml:"private_key_file"` + } `yaml:"credentials"` + + Hosts []Host `yaml:"hosts"` + + Orchestrator Orchestrator `yaml:"orchestrator"` + + Modules struct { + AdminPassword string `yaml:"admin_password"` + AdditionalRepositories interface{} `yaml:"additional_repositories"` + Additional Additional `yaml:"additional"` + Observability Observability `yaml:"observability"` + Registry Registry `yaml:"registry"` + Cicd Cicd `yaml:"cicd"` + SecretsStorage SecretsStorage `yaml:"secrets_storage"` + } `yaml:"modules"` + + Repositories string + Releases string +} + +var instance *Config + +func CreateConfig(configPath string, dataDir string, password string) *Config { + instance = &Config{} + instance.DataDir = dataDir + + if err := cleanenv.ReadConfig(configPath, instance); err != nil { + helper, _ := cleanenv.GetDescription(instance, nil) + panic(fmt.Sprintf("%s\n%s", helper, err)) + } + + if password != "" { + instance.Credentials.Password = password + } + + generateCreds(instance) + + return instance +} + +func GetConfig() *Config { + return instance +} diff --git a/pkg/config/modules_additional.go b/pkg/config/modules_additional.go new file mode 100644 index 0000000..7f21009 --- /dev/null +++ b/pkg/config/modules_additional.go @@ -0,0 +1,37 @@ +package config + +type Additional struct { + CertManager struct { + Install bool `yaml:"install"` + DnsServers []string `yaml:"dns_servers" env-default:"[8.8.8.8,1.1.1.1]"` + AccountEmail string `yaml:"account_email"` + } `yaml:"cert_manager"` + + Ingress struct { + Type string `yaml:"type" env-default:"nginx"` + Install bool `yaml:"install" env-default:"false"` + InsecurePort int `yaml:"insecure_port" env-default:"80"` + SecurePort int `yaml:"secure_port" env-default:"443"` + IngressClassName string `yaml:"ingress_class_name" env-default:"nginx"` + HostNetwork string `yaml:"host_network" env-default:"false"` + } `yaml:"ingress"` + + LoadBalancer struct { + Type string `yaml:"type" env-default:"metallb"` + Install bool `yaml:"install"` + } `yaml:"load_balancer"` + + Storage struct { + LocalPathProvisioner struct { + Enabled bool `yaml:"enabled"` + StorageClassName string `yaml:"storage_class_name" env-default:"local-path"` + ReclaimPolicy string `yaml:"reclaim_policy" env-default:"Delete"` + } `yaml:"local_path_provisioner"` + Longhorn struct { + Enabled bool `yaml:"enabled"` + } `yaml:"longhorn"` + SecretsStoreCsiDriver struct { + Enabled bool `yaml:"enabled"` + } `yaml:"secrets_store_csi_driver"` + } `yaml:"storage"` +} diff --git a/pkg/config/modules_cicd.go b/pkg/config/modules_cicd.go new file mode 100644 index 0000000..0f0ddbf --- /dev/null +++ b/pkg/config/modules_cicd.go @@ -0,0 +1,29 @@ +package config + +type Cicd struct { + Enabled bool `yaml:"enabled"` + ArgoCd struct { + AdminPassword string + Ha struct { + Enabled bool `yaml:"enabled"` + Autoscaling bool `yaml:"autoscaling"` + } `yaml:"ha"` + Expose struct { + Type string `yaml:"type"` + Domain string `yaml:"domain"` + Tls struct { + Enabled bool `yaml:"enabled"` + } `yaml:"tls"` + } `yaml:"expose"` + Repositories interface{} `yaml:"repositories"` + Rbac struct { + AdditionalPolicies string `yaml:"additional_policies"` + } `yaml:"argo_cd"` + } `yaml:"argo_cd"` + UpdatesOperator struct { + Enabled bool `yaml:"enabled"` + } `yaml:"updates_operator"` + Rollouts struct { + Enabled bool `yaml:"enabled"` + } `yaml:"rollouts"` +} diff --git a/pkg/config/modules_observability.go b/pkg/config/modules_observability.go new file mode 100644 index 0000000..0de2459 --- /dev/null +++ b/pkg/config/modules_observability.go @@ -0,0 +1,154 @@ +package config + +type Observability struct { + Enabled bool `yaml:"enabled"` + Logging Logging `yaml:"logging"` + Tracing Tracing `yaml:"tracing"` + Monitoring Monitoring `yaml:"monitoring"` + Visualization Visualization `yaml:"visualization"` +} + +type Logging struct { + Enabled bool `yaml:"enabled"` + Operator struct { + Image string `yaml:"image" env-default:"kubesphere/fluent-operator"` + Tag string `yaml:"tag" env-default:"v2.7.0"` + InitContainer struct { + Image string `yaml:"image" env-default:"docker"` + Tag string `yaml:"tag" env-default:"20.10"` + } `yaml:"initcontainer"` + } `yaml:"operator"` + Fluentd struct { + Image string `yaml:"image" env-default:"kubesphere/fluentd"` + Tag string `yaml:"tag" env-default:"v1.15.3"` + } `yaml:"fluentd"` + FluentBit struct { + Image string `yaml:"image" env-default:"kubesphere/fluent-bit"` + Tag string `yaml:"tag" env-default:"v2.2.2"` + } `yaml:"fluent_bit"` + Loki struct { + Registry string `yaml:"registry" env-default:"docker.io"` + Image string `yaml:"image" env-default:"grafana/loki"` + Tag string `yaml:"tag" env-default:"null"` + Persistence struct { + StorageClass string `yaml:"storage_class" env-default:"local-path"` + StorageSize string `yaml:"storage_size" env-default:"10Gi"` + Retention string `yaml:"retention" env-default:"168h"` + } `yaml:"persistence"` + AlertManagerUrl string `yaml:"alert_manager_url" env-default:"http://observability-alert-manager:9093"` + AdditionalRulesGroups string `yaml:"additional_rules_groups" env-default:""` + } `yaml:"loki"` + Events struct { + Enabled bool `yaml:"enabled"` + Exporter struct { + Image string `yaml:"image" env-default:"ghcr.io/resmoio/kubernetes-event-exporter"` + Tag string `yaml:"tag" env-default:"v1.4"` + } `yaml:"exporter"` + Cron struct { + Image string `yaml:"image" env-default:"bitnami/kubectl"` + Tag string `yaml:"tag" env-default:"1.27.5-debian-11-r8"` + Schedule string `yaml:"schedule" env-default:"*/2 * * * *"` + } `yaml:"cron"` + } `yaml:"events"` +} + +type Tracing struct { + Enabled bool `yaml:"enabled"` + Operator struct { + Image string `yaml:"image" env-default:"ghcr.io/open-telemetry/opentelemetry-operator/opentelemetry-operator"` + Tag string `yaml:"tag" env-default:""` + } `yaml:"operator"` + Collector struct { + Image string `yaml:"image" env-default:"otel/opentelemetry-collector-contrib"` + Tag string `yaml:"tag" env-default:"0.95.0"` + } `yaml:"collector"` + Tempo struct { + Image string `yaml:"image" env-default:"grafana/tempo"` + Tag string `yaml:"tag" env-default:""` + Retention string `yaml:"retention" env-default:"24h"` + ListenPort int `yaml:"listen_port" env-default:"3100"` + Persistence struct { + StorageClass string `yaml:"storage_class" env-default:"local-path"` + StorageSize string `yaml:"storage_size" env-default:"10Gi"` + } `yaml:"persistence"` + TempoQuery struct { + Image string `yaml:"image" env-default:"grafana/tempo-query"` + Tag string `yaml:"tag" env-default:"null"` + ListenPort int `yaml:"listen_port" env-default:"16686"` + } `yaml:"tempo_query"` + } `yaml:"tempo"` +} + +type Monitoring struct { + Enabled bool `yaml:"enabled"` + Prometheus struct { + Image string `yaml:"image" env-default:"prom/prometheus"` + Tag string `yaml:"tag" env-default:"v2.45.0"` + ScrapeInterval string `yaml:"scrape_interval" env-default:"15s"` + Persistence struct { + StorageClass string `yaml:"storage_class" env-default:"local-path"` + StorageSize string `yaml:"storage_size" env-default:"3Gi"` + Retention string `yaml:"retention" env-default:"7d"` + } `yaml:"persistence"` + Operator struct { + Image string `yaml:"image" env-default:"ghcr.io/prometheus-operator/prometheus-operator"` + Tag string `yaml:"tag" env-default:"v0.65.2"` + ConfigReloader struct { + Image string `yaml:"image" env-default:"ghcr.io/prometheus-operator/prometheus-config-reloader"` + Tag string `yaml:"tag" env-default:"v0.65.2"` + } `yaml:"config_reloader"` + KubeRbacProxy struct { + Image string `yaml:"image" env-default:"bitnami/kube-rbac-proxy"` + Tag string `yaml:"tag" env-default:"0.14.1"` + } `yaml:"kube_rbac_proxy"` + } `yaml:"operator"` + } `yaml:"prometheus"` + AlertManager struct { + Enabled bool `yaml:"enabled"` + Image string `yaml:"image" env-default:"prom/alertmanager"` + Tag string `yaml:"tag" env-default:"v0.26.0"` + Route interface{} `yaml:"route"` + Receivers interface{} `yaml:"receivers"` + } `yaml:"alert_manager"` + Blackbox struct { + Enabled bool `yaml:"enabled"` + Image string `yaml:"image" env-default:"prom/blackbox-exporter"` + Tag string `yaml:"tag" env-default:"v0.24.0"` + additionalModules string `yaml:"routes" env-default:""` + } `yaml:"blackbox"` + KubeState struct { + Enabled bool `yaml:"enabled"` + Image string `yaml:"image" env-default:"bitnami/kube-state-metrics"` + Tag string `yaml:"tag" env-default:"2.9.2"` + } `yaml:"kube_state"` + Node struct { + Enabled bool `yaml:"enabled"` + Image string `yaml:"image" env-default:"prom/node-exporter"` + Tag string `yaml:"tag" env-default:"v1.5.0"` + } `yaml:"node"` +} + +type Visualization struct { + Enabled bool `yaml:"enabled"` + Grafana struct { + Enabled bool `yaml:"enabled"` + Image string `yaml:"image" env-default:"grafana/grafana"` + Tag string `yaml:"tag" env-default:"10.4.1"` + Expose struct { + Type string `yaml:"type" env-default:"ingress"` + Domain string `yaml:"domain" env-default:""` + Tls struct { + Enabled bool `yaml:"enabled"` + } + } `yaml:"expose"` + Persistence struct { + StorageClass string `yaml:"storage_class" env-default:"local-path"` + StorageSize string `yaml:"storage_size" env-default:"2Gi"` + } `yaml:"persistence"` + Config struct { + Auth string `yaml:"auth" env-default:""` + AuthGenericAuth string `yaml:"auth_generic_auth" env-default:""` + AdditionalDatasources interface{} `yaml:"additional_datasources"` + } `yaml:"config"` + } `yaml:"grafana"` +} diff --git a/pkg/config/modules_registry.go b/pkg/config/modules_registry.go new file mode 100644 index 0000000..6267546 --- /dev/null +++ b/pkg/config/modules_registry.go @@ -0,0 +1,26 @@ +package config + +type Registry struct { + Enabled bool `yaml:"enabled"` + AdminPassword string + Namespace string `yaml:"namespace" env-default:"registry"` + Version string `yaml:"version" env-default:"v2.10.1"` + Expose struct { + Type string `yaml:"type" env-default:"nodePort"` + Domain string `yaml:"domain" env-default:""` + NodePortHttp int `yaml:"node_port_http" env-default:"30002"` + NodePortHttps int `yaml:"node_port_https" env-default:"30003"` + } `yaml:"expose"` + Tls struct { + Enabled bool `yaml:"enabled"` + } `yaml:"tls"` + Persistence struct { + StorageClass string `yaml:"storage_class" env-default:"local-path"` + RegistrySize string `yaml:"registry_size" env-default:"10Gi"` + JobserviceSize string `yaml:"jobservice_size" env-default:"1Gi"` + DatabaseSize string `yaml:"database_size" env-default:"2Gi"` + RedisSize string `yaml:"redis_size" env-default:"1Gi"` + TrivySize string `yaml:"trivy_size" env-default:"5Gi"` + } `yaml:"persistence"` + EnabledScanner bool `yaml:"enabled_scanner"` +} diff --git a/pkg/config/modules_secrets_storage.go b/pkg/config/modules_secrets_storage.go new file mode 100644 index 0000000..d2edb47 --- /dev/null +++ b/pkg/config/modules_secrets_storage.go @@ -0,0 +1,42 @@ +package config + +type SecretsStorage struct { + Enabled bool `yaml:"enabled"` + Image string `yaml:"image" env-default:"kubesphere/fluent-operator"` + Tag string `yaml:"tag" env-default:"v2.7.0"` + Expose struct { + Type string `yaml:"type"` + Domain string `yaml:"domain"` + NodePort int `yaml:"node_port"` + Tls struct { + Enabled bool `yaml:"enabled"` + } `yaml:"tls"` + } `yaml:"expose"` + CsiIntegration struct { + Enabled bool `yaml:"enabled"` + Image string `yaml:"image" env-default:"hashicorp/vault-csi-provider"` + Tag string `yaml:"tag" env-default:"1.4.1"` + } `yaml:"csi_integration"` + Injector struct { + Image string `yaml:"image" env-default:"hashicorp/vault-k8s"` + Tag string `yaml:"tag" env-default:"1.3.1"` + } `yaml:"injector"` + Server struct { + Image string `yaml:"image" env-default:"hashicorp/vault"` + Tag string `yaml:"tag" env-default:"1.15.6"` + Persistence struct { + DataStorage struct { + StorageClass string `yaml:"storage_class" env-default:"local-path"` + Size string `yaml:"size" env-default:"10Gi"` + } `yaml:"data_storage"` + AuditStorage struct { + StorageClass string `yaml:"storage_class" env-default:"local-path"` + Size string `yaml:"size" env-default:"10Gi"` + } `yaml:"audit_storage"` + } `yaml:"persistence"` + } `yaml:"server"` + Agent struct { + Image string `yaml:"image" env-default:"hashicorp/vault"` + Tag string `yaml:"tag" env-default:"1.15.6"` + } `yaml:"agent"` +} diff --git a/pkg/config/orchestrator.go b/pkg/config/orchestrator.go new file mode 100644 index 0000000..77015e6 --- /dev/null +++ b/pkg/config/orchestrator.go @@ -0,0 +1,31 @@ +package config + +type Dns struct { + Servers []string `yaml:"servers" env-default:"[8.8.8.8,8.8.4.4]"` + DisableHostNameservers bool `yaml:"disable_host_nameservers" env-default:"false"` +} + +type Orchestrator struct { + Version string `yaml:"version" env-default:"v1.28.0"` + ClusterName string `yaml:"cluster_name" env-default:"k8s-cluster.local"` + BinDir string `yaml:"bin_dir" env-default:"/usr/local/bin"` + SysctlFilePath string `yaml:"sysctl_file_path" env-default:"/etc/sysctl.d/99-sysctl.conf"` + LoadbalancerApiserverPort int `yaml:"loadbalancer_apiserver_port" env-default:"6443"` + Dns Dns `yaml:"dns"` + CloudProvider string `yaml:"cloud_provider"` + ExgernalCloudProvider string `yaml:"external_cloud_provider"` + ContainerEngine struct { + Type string `yaml:"type" env-default:"containerd"` + Install bool `yaml:"install" env-default:"true"` + } `yaml:"container_engine"` + PingAccessIp bool `yaml:"ping_access_ip" env-default:"true"` + AutoRenewCertificates bool `yaml:"auto_renew_certificates" env-default:"false"` + EventTtl string `yaml:"event_ttl" env-default:"1h0m0s"` + PodSecurotyPolicyEnabled bool `yaml:"pod_security_policy_enabled" env-default:"false"` + + Network struct { + Plugin string `yaml:"plugin" env-default:"calico"` + ServiceAddresses string `yaml:"service_addresses" env-default:"10.233.0.0/18"` + PodsSubnet string `yaml:"pods_subnet" env-default:"10.233.64.0/18"` + } `yaml:"network"` +} diff --git a/pkg/config/utility.go b/pkg/config/utility.go new file mode 100644 index 0000000..0c40e0c --- /dev/null +++ b/pkg/config/utility.go @@ -0,0 +1,18 @@ +package config + +import ( + "golang.org/x/crypto/bcrypt" +) + +func getBcryptHash(input string) string { + hashedPassword, err := bcrypt.GenerateFromPassword([]byte(input), bcrypt.DefaultCost) + if err != nil { + panic(err) + } + return string(hashedPassword) +} + +func generateCreds(config *Config) { + config.Modules.Cicd.ArgoCd.AdminPassword = getBcryptHash(config.Modules.AdminPassword) + config.Modules.Registry.AdminPassword = config.Modules.AdminPassword +} diff --git a/pkg/kubespray/cluster.go b/pkg/kubespray/cluster.go new file mode 100644 index 0000000..e0f4776 --- /dev/null +++ b/pkg/kubespray/cluster.go @@ -0,0 +1,16 @@ +package kubespray + +func InstallCluster(tags string) { + playbookConfig := getPlaybookParameters(tags) + runPlaybook("kubespray/project/cluster.yml", playbookConfig) +} + +func UpgradeCluster(tags string) { + playbookConfig := getPlaybookParameters(tags) + runPlaybook("kubespray/project/upgrade_cluster.yml", playbookConfig) +} + +func ResetCluster() { + playbookConfig := getPlaybookParameters("reset") + runPlaybook("kubespray/project/reset.yml", playbookConfig) +} diff --git a/pkg/kubespray/utility.go b/pkg/kubespray/utility.go new file mode 100644 index 0000000..2e33e92 --- /dev/null +++ b/pkg/kubespray/utility.go @@ -0,0 +1,91 @@ +package kubespray + +import ( + "context" + "io" + "kube-forge/pkg/config" + "os" + "path/filepath" + + "github.com/apenella/go-ansible/pkg/execute" + "github.com/apenella/go-ansible/pkg/inventory" + "github.com/apenella/go-ansible/pkg/options" + "github.com/apenella/go-ansible/pkg/playbook" +) + +func getPlaybookParameters(tags string) config.AnsiblePlaybookConfig { + cfg := config.GetConfig() + ansibleInventoryOptions := inventory.AnsibleInventoryOptions{ + Graph: true, + Inventory: "kubespray/inventory/hosts", + Vars: true, + Yaml: false, + } + + ansiblePlaybookOptions := playbook.AnsiblePlaybookOptions{ + Inventory: ansibleInventoryOptions.Inventory, + Tags: tags, + } + + ansiblePlaybookConnectionOptions := options.AnsibleConnectionOptions{ + User: cfg.Credentials.User, + } + + if cfg.Credentials.PrivateKeyFile != "" { + ansiblePlaybookConnectionOptions.PrivateKey = cfg.Credentials.PrivateKeyFile + } + + ansiblePlaybookPrivilegeEscalationOptions := options.AnsiblePrivilegeEscalationOptions{ + Become: true, + } + + if cfg.Credentials.AskSudoPassword { + ansiblePlaybookPrivilegeEscalationOptions.AskBecomePass = true + } + + return config.AnsiblePlaybookConfig{ + ConnectionOptions: ansiblePlaybookConnectionOptions, + PrivilegeEscalationOptions: ansiblePlaybookPrivilegeEscalationOptions, + PlaybookOptions: ansiblePlaybookOptions, + } +} + +func CopyK8SAdminConfig(pathInDataDir string) { + config := config.GetConfig() + dataDir := config.DataDir + var adminDefaultConfigPath = filepath.Join(dataDir, "kubespray/inventory/artifacts/admin.conf") + var adminOutConfigPath = filepath.Join(dataDir, pathInDataDir) + source, err := os.Open(adminDefaultConfigPath) + if err != nil { + panic(err) + } + defer source.Close() + + destination, err := os.Create(adminOutConfigPath) + if err != nil { + panic(err) + } + defer destination.Close() + _, err = io.Copy(destination, source) + if err != nil { + panic(err) + } +} + +func runPlaybook(playbookPath string, playbookConfig config.AnsiblePlaybookConfig) { + playbook := &playbook.AnsiblePlaybookCmd{ + Playbooks: []string{playbookPath}, + ConnectionOptions: &playbookConfig.ConnectionOptions, + PrivilegeEscalationOptions: &playbookConfig.PrivilegeEscalationOptions, + Options: &playbookConfig.PlaybookOptions, + Exec: execute.NewDefaultExecute( + execute.WithEnvVar("ANSIBLE_FORCE_COLOR", "true"), + // execute.WithEnvVar("ANSIBLE_STDOUT_CALLBACK", "true"), + ), + } + + err := playbook.Run(context.TODO()) + if err != nil { + panic(err) + } +} diff --git a/pkg/resources/resources.go b/pkg/resources/resources.go new file mode 100644 index 0000000..9fd078f --- /dev/null +++ b/pkg/resources/resources.go @@ -0,0 +1,6 @@ +package resources + +import "embed" + +//go:embed templates +var Templates embed.FS diff --git a/pkg/resources/templates/helm-apps/releases/additional-modules/cert-manager.yml.tmpl b/pkg/resources/templates/helm-apps/releases/additional-modules/cert-manager.yml.tmpl new file mode 100644 index 0000000..e69de29 diff --git a/pkg/resources/templates/helm-apps/releases/additional-modules/ingress-nginx.yml.tmpl b/pkg/resources/templates/helm-apps/releases/additional-modules/ingress-nginx.yml.tmpl new file mode 100644 index 0000000..e69de29 diff --git a/pkg/resources/templates/helm-apps/releases/additional-modules/longhorn.yml.tmpl b/pkg/resources/templates/helm-apps/releases/additional-modules/longhorn.yml.tmpl new file mode 100644 index 0000000..1f06ce1 --- /dev/null +++ b/pkg/resources/templates/helm-apps/releases/additional-modules/longhorn.yml.tmpl @@ -0,0 +1,487 @@ +{{- if .Modules.Additional.Storage.Longhorn.Enabled }} +- name: longhorn + namespace: longhorn-system + create_namespace: true + chart_ref: kube-forge/longhorn + chart_version: 1.6.1 + values: + global: + cattle: + # -- Default system registry. + systemDefaultRegistry: "" + windowsCluster: + # -- Setting that allows Longhorn to run on a Rancher Windows cluster. + enabled: false + # -- Toleration for Linux nodes that can run user-deployed Longhorn components. + tolerations: + - key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" + # -- Node selector for Linux nodes that can run user-deployed Longhorn components. + nodeSelector: + kubernetes.io/os: "linux" + defaultSetting: + # -- Toleration for system-managed Longhorn components. + taintToleration: cattle.io/os=linux:NoSchedule + # -- Node selector for system-managed Longhorn components. + systemManagedComponentsNodeSelector: kubernetes.io/os:linux + + networkPolicies: + # -- Setting that allows you to enable network policies that control access to Longhorn pods. + enabled: false + # -- Distribution that determines the policy for allowing access for an ingress. (Options: "k3s", "rke2", "rke1") + type: "k3s" + + image: + longhorn: + engine: + # -- Repository for the Longhorn Engine image. + repository: longhornio/longhorn-engine + # -- Specify Longhorn engine image tag + tag: v1.6.1 + manager: + # -- Repository for the Longhorn Manager image. + repository: longhornio/longhorn-manager + # -- Specify Longhorn manager image tag + tag: v1.6.1 + ui: + # -- Repository for the Longhorn UI image. + repository: longhornio/longhorn-ui + # -- Specify Longhorn ui image tag + tag: v1.6.1 + instanceManager: + # -- Repository for the Longhorn Instance Manager image. + repository: longhornio/longhorn-instance-manager + # -- Specify Longhorn instance manager image tag + tag: v1.6.1 + shareManager: + # -- Repository for the Longhorn Share Manager image. + repository: longhornio/longhorn-share-manager + # -- Specify Longhorn share manager image tag + tag: v1.6.1 + backingImageManager: + # -- Repository for the Backing Image Manager image. When unspecified, Longhorn uses the default value. + repository: longhornio/backing-image-manager + # -- Specify Longhorn backing image manager image tag + tag: v1.6.1 + supportBundleKit: + # -- Repository for the Longhorn Support Bundle Manager image. + repository: longhornio/support-bundle-kit + # -- Tag for the Longhorn Support Bundle Manager image. + tag: v0.0.36 + csi: + attacher: + # -- Repository for the CSI attacher image. When unspecified, Longhorn uses the default value. + repository: longhornio/csi-attacher + # -- Tag for the CSI attacher image. When unspecified, Longhorn uses the default value. + tag: v4.4.2 + provisioner: + # -- Repository for the CSI Provisioner image. When unspecified, Longhorn uses the default value. + repository: longhornio/csi-provisioner + # -- Tag for the CSI Provisioner image. When unspecified, Longhorn uses the default value. + tag: v3.6.2 + nodeDriverRegistrar: + # -- Repository for the CSI Node Driver Registrar image. When unspecified, Longhorn uses the default value. + repository: longhornio/csi-node-driver-registrar + # -- Tag for the CSI Node Driver Registrar image. When unspecified, Longhorn uses the default value. + tag: v2.9.2 + resizer: + # -- Repository for the CSI Resizer image. When unspecified, Longhorn uses the default value. + repository: longhornio/csi-resizer + # -- Tag for the CSI Resizer image. When unspecified, Longhorn uses the default value. + tag: v1.9.2 + snapshotter: + # -- Repository for the CSI Snapshotter image. When unspecified, Longhorn uses the default value. + repository: longhornio/csi-snapshotter + # -- Tag for the CSI Snapshotter image. When unspecified, Longhorn uses the default value. + tag: v6.3.2 + livenessProbe: + # -- Repository for the CSI liveness probe image. When unspecified, Longhorn uses the default value. + repository: longhornio/livenessprobe + # -- Tag for the CSI liveness probe image. When unspecified, Longhorn uses the default value. + tag: v2.12.0 + openshift: + oauthProxy: + # -- Repository for the OAuth Proxy image. This setting applies only to OpenShift users. + repository: quay.io/openshift/origin-oauth-proxy + # -- Tag for the OAuth Proxy image. This setting applies only to OpenShift users. Specify OCP/OKD version 4.1 or later. The latest stable version is 4.14. + tag: 4.14 + # -- Image pull policy that applies to all user-deployed Longhorn components, such as Longhorn Manager, Longhorn driver, and Longhorn UI. + pullPolicy: IfNotPresent + + service: + ui: + # -- Service type for Longhorn UI. (Options: "ClusterIP", "NodePort", "LoadBalancer", "Rancher-Proxy") + type: ClusterIP + # -- NodePort port number for Longhorn UI. When unspecified, Longhorn selects a free port between 30000 and 32767. + nodePort: null + manager: + # -- Service type for Longhorn Manager. + type: ClusterIP + # -- NodePort port number for Longhorn Manager. When unspecified, Longhorn selects a free port between 30000 and 32767. + nodePort: "" + + persistence: + # -- Setting that allows you to specify the default Longhorn StorageClass. + defaultClass: true + # -- Filesystem type of the default Longhorn StorageClass. + defaultFsType: ext4 + # -- mkfs parameters of the default Longhorn StorageClass. + defaultMkfsParams: "" + # -- Replica count of the default Longhorn StorageClass. + defaultClassReplicaCount: 3 + # -- Data locality of the default Longhorn StorageClass. (Options: "disabled", "best-effort") + defaultDataLocality: disabled + # -- Reclaim policy that provides instructions for handling of a volume after its claim is released. (Options: "Retain", "Delete") + reclaimPolicy: Delete + # -- Setting that allows you to enable live migration of a Longhorn volume from one node to another. + migratable: false + # -- Set NFS mount options for Longhorn StorageClass for RWX volumes + nfsOptions: "" + recurringJobSelector: + # -- Setting that allows you to enable the recurring job selector for a Longhorn StorageClass. + enable: false + # -- Recurring job selector for a Longhorn StorageClass. Ensure that quotes are used correctly when specifying job parameters. (Example: `[{"name":"backup", "isGroup":true}]`) + jobList: [] + backingImage: + # -- Setting that allows you to use a backing image in a Longhorn StorageClass. + enable: false + # -- Backing image to be used for creating and restoring volumes in a Longhorn StorageClass. When no backing images are available, specify the data source type and parameters that Longhorn can use to create a backing image. + name: ~ + # -- Data source type of a backing image used in a Longhorn StorageClass. + # If the backing image exists in the cluster, Longhorn uses this setting to verify the image. + # If the backing image does not exist, Longhorn creates one using the specified data source type. + dataSourceType: ~ + # -- Data source parameters of a backing image used in a Longhorn StorageClass. + # You can specify a JSON string of a map. (Example: `'{\"url\":\"https://backing-image-example.s3-region.amazonaws.com/test-backing-image\"}'`) + dataSourceParameters: ~ + # -- Expected SHA-512 checksum of a backing image used in a Longhorn StorageClass. + expectedChecksum: ~ + defaultNodeSelector: + # -- Setting that allows you to enable the node selector for the default Longhorn StorageClass. + enable: false + # -- Node selector for the default Longhorn StorageClass. Longhorn uses only nodes with the specified tags for storing volume data. (Examples: "storage,fast") + selector: "" + # -- Setting that allows you to enable automatic snapshot removal during filesystem trim for a Longhorn StorageClass. (Options: "ignored", "enabled", "disabled") + removeSnapshotsDuringFilesystemTrim: ignored + + preUpgradeChecker: + # -- Setting that allows Longhorn to perform pre-upgrade checks. Disable this setting when installing Longhorn using Argo CD or other GitOps solutions. + jobEnabled: true + # -- Setting that allows Longhorn to perform upgrade version checks after starting the Longhorn Manager DaemonSet Pods. Disabling this setting also disables `preUpgradeChecker.jobEnabled`. Longhorn recommends keeping this setting enabled. + upgradeVersionCheck: true + + csi: + # -- kubelet root directory. When unspecified, Longhorn uses the default value. + kubeletRootDir: ~ + # -- Replica count of the CSI Attacher. When unspecified, Longhorn uses the default value ("3"). + attacherReplicaCount: ~ + # -- Replica count of the CSI Provisioner. When unspecified, Longhorn uses the default value ("3"). + provisionerReplicaCount: ~ + # -- Replica count of the CSI Resizer. When unspecified, Longhorn uses the default value ("3"). + resizerReplicaCount: ~ + # -- Replica count of the CSI Snapshotter. When unspecified, Longhorn uses the default value ("3"). + snapshotterReplicaCount: ~ + + defaultSettings: + # -- Endpoint used to access the backupstore. (Options: "NFS", "CIFS", "AWS", "GCP", "AZURE") + backupTarget: ~ + # -- Name of the Kubernetes secret associated with the backup target. + backupTargetCredentialSecret: ~ + # -- Setting that allows Longhorn to automatically attach a volume and create snapshots or backups when recurring jobs are run. + allowRecurringJobWhileVolumeDetached: ~ + # -- Setting that allows Longhorn to automatically create a default disk only on nodes with the label "node.longhorn.io/create-default-disk=true" (if no other disks exist). When this setting is disabled, Longhorn creates a default disk on each node that is added to the cluster. + createDefaultDiskLabeledNodes: ~ + # -- Default path for storing data on a host. The default value is "/var/lib/longhorn/". + defaultDataPath: ~ + # -- Default data locality. A Longhorn volume has data locality if a local replica of the volume exists on the same node as the pod that is using the volume. + defaultDataLocality: ~ + # -- Setting that allows scheduling on nodes with healthy replicas of the same volume. This setting is disabled by default. + replicaSoftAntiAffinity: ~ + # -- Setting that automatically rebalances replicas when an available node is discovered. + replicaAutoBalance: ~ + # -- Percentage of storage that can be allocated relative to hard drive capacity. The default value is "100". + storageOverProvisioningPercentage: ~ + # -- Percentage of minimum available disk capacity. When the minimum available capacity exceeds the total available capacity, the disk becomes unschedulable until more space is made available for use. The default value is "25". + storageMinimalAvailablePercentage: ~ + # -- Percentage of disk space that is not allocated to the default disk on each new Longhorn node. + storageReservedPercentageForDefaultDisk: ~ + # -- Upgrade Checker that periodically checks for new Longhorn versions. When a new version is available, a notification appears on the Longhorn UI. This setting is enabled by default + upgradeChecker: ~ + # -- Default number of replicas for volumes created using the Longhorn UI. For Kubernetes configuration, modify the `numberOfReplicas` field in the StorageClass. The default value is "3". + defaultReplicaCount: ~ + # -- Default Longhorn StorageClass. "storageClassName" is assigned to PVs and PVCs that are created for an existing Longhorn volume. "storageClassName" can also be used as a label, so it is possible to use a Longhorn StorageClass to bind a workload to an existing PV without creating a Kubernetes StorageClass object. The default value is "longhorn-static". + defaultLonghornStaticStorageClass: ~ + # -- Number of seconds that Longhorn waits before checking the backupstore for new backups. The default value is "300". When the value is "0", polling is disabled. + backupstorePollInterval: ~ + # -- Number of minutes that Longhorn keeps a failed backup resource. When the value is "0", automatic deletion is disabled. + failedBackupTTL: ~ + # -- Setting that restores recurring jobs from a backup volume on a backup target and creates recurring jobs if none exist during backup restoration. + restoreVolumeRecurringJobs: ~ + # -- Maximum number of successful recurring backup and snapshot jobs to be retained. When the value is "0", a history of successful recurring jobs is not retained. + recurringSuccessfulJobsHistoryLimit: ~ + # -- Maximum number of failed recurring backup and snapshot jobs to be retained. When the value is "0", a history of failed recurring jobs is not retained. + recurringFailedJobsHistoryLimit: ~ + # -- Maximum number of snapshots or backups to be retained. + recurringJobMaxRetention: ~ + # -- Maximum number of failed support bundles that can exist in the cluster. When the value is "0", Longhorn automatically purges all failed support bundles. + supportBundleFailedHistoryLimit: ~ + # -- Taint or toleration for system-managed Longhorn components. + taintToleration: ~ + # -- Node selector for system-managed Longhorn components. + systemManagedComponentsNodeSelector: ~ + # -- PriorityClass for system-managed Longhorn components. + # This setting can help prevent Longhorn components from being evicted under Node Pressure. + # Notice that this will be applied to Longhorn user-deployed components by default if there are no priority class values set yet, such as `longhornManager.priorityClass`. + priorityClass: &defaultPriorityClassNameRef "longhorn-critical" + # -- Setting that allows Longhorn to automatically salvage volumes when all replicas become faulty (for example, when the network connection is interrupted). Longhorn determines which replicas are usable and then uses these replicas for the volume. This setting is enabled by default. + autoSalvage: ~ + # -- Setting that allows Longhorn to automatically delete a workload pod that is managed by a controller (for example, daemonset) whenever a Longhorn volume is detached unexpectedly (for example, during Kubernetes upgrades). After deletion, the controller restarts the pod and then Kubernetes handles volume reattachment and remounting. + autoDeletePodWhenVolumeDetachedUnexpectedly: ~ + # -- Setting that prevents Longhorn Manager from scheduling replicas on a cordoned Kubernetes node. This setting is enabled by default. + disableSchedulingOnCordonedNode: ~ + # -- Setting that allows Longhorn to schedule new replicas of a volume to nodes in the same zone as existing healthy replicas. Nodes that do not belong to any zone are treated as existing in the zone that contains healthy replicas. When identifying zones, Longhorn relies on the label "topology.kubernetes.io/zone=" in the Kubernetes node object. + replicaZoneSoftAntiAffinity: ~ + # -- Setting that allows scheduling on disks with existing healthy replicas of the same volume. This setting is enabled by default. + replicaDiskSoftAntiAffinity: ~ + # -- Policy that defines the action Longhorn takes when a volume is stuck with a StatefulSet or Deployment pod on a node that failed. + nodeDownPodDeletionPolicy: ~ + # -- Policy that defines the action Longhorn takes when a node with the last healthy replica of a volume is drained. + nodeDrainPolicy: ~ + # -- Setting that allows automatic detaching of manually-attached volumes when a node is cordoned. + detachManuallyAttachedVolumesWhenCordoned: ~ + # -- Number of seconds that Longhorn waits before reusing existing data on a failed replica instead of creating a new replica of a degraded volume. + replicaReplenishmentWaitInterval: ~ + # -- Maximum number of replicas that can be concurrently rebuilt on each node. + concurrentReplicaRebuildPerNodeLimit: ~ + # -- Maximum number of volumes that can be concurrently restored on each node using a backup. When the value is "0", restoration of volumes using a backup is disabled. + concurrentVolumeBackupRestorePerNodeLimit: ~ + # -- Setting that disables the revision counter and thereby prevents Longhorn from tracking all write operations to a volume. When salvaging a volume, Longhorn uses properties of the "volume-head-xxx.img" file (the last file size and the last time the file was modified) to select the replica to be used for volume recovery. This setting applies only to volumes created using the Longhorn UI. + disableRevisionCounter: ~ + # -- Image pull policy for system-managed pods, such as Instance Manager, engine images, and CSI Driver. Changes to the image pull policy are applied only after the system-managed pods restart. + systemManagedPodsImagePullPolicy: ~ + # -- Setting that allows you to create and attach a volume without having all replicas scheduled at the time of creation. + allowVolumeCreationWithDegradedAvailability: ~ + # -- Setting that allows Longhorn to automatically clean up the system-generated snapshot after replica rebuilding is completed. + autoCleanupSystemGeneratedSnapshot: ~ + # -- Setting that allows Longhorn to automatically clean up the snapshot generated by a recurring backup job. + autoCleanupRecurringJobBackupSnapshot: ~ + # -- Maximum number of engines that are allowed to concurrently upgrade on each node after Longhorn Manager is upgraded. When the value is "0", Longhorn does not automatically upgrade volume engines to the new default engine image version. + concurrentAutomaticEngineUpgradePerNodeLimit: ~ + # -- Number of minutes that Longhorn waits before cleaning up the backing image file when no replicas in the disk are using it. + backingImageCleanupWaitInterval: ~ + # -- Number of seconds that Longhorn waits before downloading a backing image file again when the status of all image disk files changes to "failed" or "unknown". + backingImageRecoveryWaitInterval: ~ + # -- Percentage of the total allocatable CPU resources on each node to be reserved for each instance manager pod when the V1 Data Engine is enabled. The default value is "12". + guaranteedInstanceManagerCPU: ~ + # -- Setting that notifies Longhorn that the cluster is using the Kubernetes Cluster Autoscaler. + kubernetesClusterAutoscalerEnabled: ~ + # -- Setting that allows Longhorn to automatically delete an orphaned resource and the corresponding data (for example, stale replicas). Orphaned resources on failed or unknown nodes are not automatically cleaned up. + orphanAutoDeletion: ~ + # -- Storage network for in-cluster traffic. When unspecified, Longhorn uses the Kubernetes cluster network. + storageNetwork: ~ + # -- Flag that prevents accidental uninstallation of Longhorn. + deletingConfirmationFlag: ~ + # -- Timeout between the Longhorn Engine and replicas. Specify a value between "8" and "30" seconds. The default value is "8". + engineReplicaTimeout: ~ + # -- Setting that allows you to enable and disable snapshot hashing and data integrity checks. + snapshotDataIntegrity: ~ + # -- Setting that allows disabling of snapshot hashing after snapshot creation to minimize impact on system performance. + snapshotDataIntegrityImmediateCheckAfterSnapshotCreation: ~ + # -- Setting that defines when Longhorn checks the integrity of data in snapshot disk files. You must use the Unix cron expression format. + snapshotDataIntegrityCronjob: ~ + # -- Setting that allows Longhorn to automatically mark the latest snapshot and its parent files as removed during a filesystem trim. Longhorn does not remove snapshots containing multiple child files. + removeSnapshotsDuringFilesystemTrim: ~ + # -- Setting that allows fast rebuilding of replicas using the checksum of snapshot disk files. Before enabling this setting, you must set the snapshot-data-integrity value to "enable" or "fast-check". + fastReplicaRebuildEnabled: ~ + # -- Number of seconds that an HTTP client waits for a response from a File Sync server before considering the connection to have failed. + replicaFileSyncHttpClientTimeout: ~ + # -- Log levels that indicate the type and severity of logs in Longhorn Manager. The default value is "Info". (Options: "Panic", "Fatal", "Error", "Warn", "Info", "Debug", "Trace") + logLevel: ~ + # -- Setting that allows you to specify a backup compression method. + backupCompressionMethod: ~ + # -- Maximum number of worker threads that can concurrently run for each backup. + backupConcurrentLimit: ~ + # -- Maximum number of worker threads that can concurrently run for each restore operation. + restoreConcurrentLimit: ~ + # -- Setting that allows you to enable the V1 Data Engine. + v1DataEngine: ~ + # -- Setting that allows you to enable the V2 Data Engine, which is based on the Storage Performance Development Kit (SPDK). The V2 Data Engine is a preview feature and should not be used in production environments. + v2DataEngine: ~ + # -- Setting that allows you to configure maximum huge page size (in MiB) for the V2 Data Engine. + v2DataEngineHugepageLimit: ~ + # -- Setting that allows rebuilding of offline replicas for volumes using the V2 Data Engine. + offlineReplicaRebuilding: ~ + # -- Number of millicpus on each node to be reserved for each Instance Manager pod when the V2 Data Engine is enabled. The default value is "1250". + v2DataEngineGuaranteedInstanceManagerCPU: ~ + # -- Setting that allows scheduling of empty node selector volumes to any node. + allowEmptyNodeSelectorVolume: ~ + # -- Setting that allows scheduling of empty disk selector volumes to any disk. + allowEmptyDiskSelectorVolume: ~ + # -- Setting that allows Longhorn to periodically collect anonymous usage data for product improvement purposes. Longhorn sends collected data to the [Upgrade Responder](https://github.com/longhorn/upgrade-responder) server, which is the data source of the Longhorn Public Metrics Dashboard (https://metrics.longhorn.io). The Upgrade Responder server does not store data that can be used to identify clients, including IP addresses. + allowCollectingLonghornUsageMetrics: ~ + # -- Setting that temporarily prevents all attempts to purge volume snapshots. + disableSnapshotPurge: ~ + # -- Maximum snapshot count for a volume. The value should be between 2 to 250 + snapshotMaxCount: ~ + + privateRegistry: + # -- Setting that allows you to create a private registry secret. + createSecret: ~ + # -- URL of a private registry. When unspecified, Longhorn uses the default system registry. + registryUrl: ~ + # -- User account used for authenticating with a private registry. + registryUser: ~ + # -- Password for authenticating with a private registry. + registryPasswd: ~ + # -- Kubernetes secret that allows you to pull images from a private registry. This setting applies only when creation of private registry secrets is enabled. You must include the private registry name in the secret name. + registrySecret: ~ + + longhornManager: + log: + # -- Format of Longhorn Manager logs. (Options: "plain", "json") + format: plain + # -- PriorityClass for Longhorn Manager. + priorityClass: *defaultPriorityClassNameRef + # -- Toleration for Longhorn Manager on nodes allowed to run Longhorn Manager. + tolerations: [] + ## If you want to set tolerations for Longhorn Manager DaemonSet, delete the `[]` in the line above + ## and uncomment this example block + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + # -- Node selector for Longhorn Manager. Specify the nodes allowed to run Longhorn Manager. + nodeSelector: {} + ## If you want to set node selector for Longhorn Manager DaemonSet, delete the `{}` in the line above + ## and uncomment this example block + # label-key1: "label-value1" + # label-key2: "label-value2" + # -- Annotation for the Longhorn Manager service. + serviceAnnotations: {} + ## If you want to set annotations for the Longhorn Manager service, delete the `{}` in the line above + ## and uncomment this example block + # annotation-key1: "annotation-value1" + # annotation-key2: "annotation-value2" + + longhornDriver: + # -- PriorityClass for Longhorn Driver. + priorityClass: *defaultPriorityClassNameRef + # -- Toleration for Longhorn Driver on nodes allowed to run Longhorn components. + tolerations: [] + ## If you want to set tolerations for Longhorn Driver Deployer Deployment, delete the `[]` in the line above + ## and uncomment this example block + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + # -- Node selector for Longhorn Driver. Specify the nodes allowed to run Longhorn Driver. + nodeSelector: {} + ## If you want to set node selector for Longhorn Driver Deployer Deployment, delete the `{}` in the line above + ## and uncomment this example block + # label-key1: "label-value1" + # label-key2: "label-value2" + + longhornUI: + # -- Replica count for Longhorn UI. + replicas: 2 + # -- PriorityClass for Longhorn UI. + priorityClass: *defaultPriorityClassNameRef + # -- Toleration for Longhorn UI on nodes allowed to run Longhorn components. + tolerations: [] + ## If you want to set tolerations for Longhorn UI Deployment, delete the `[]` in the line above + ## and uncomment this example block + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + # -- Node selector for Longhorn UI. Specify the nodes allowed to run Longhorn UI. + nodeSelector: {} + ## If you want to set node selector for Longhorn UI Deployment, delete the `{}` in the line above + ## and uncomment this example block + # label-key1: "label-value1" + # label-key2: "label-value2" + + ingress: + # -- Setting that allows Longhorn to generate ingress records for the Longhorn UI service. + enabled: false + + # -- IngressClass resource that contains ingress configuration, including the name of the Ingress controller. + # ingressClassName can replace the kubernetes.io/ingress.class annotation used in earlier Kubernetes releases. + ingressClassName: ~ + + # -- Hostname of the Layer 7 load balancer. + host: sslip.io + + # -- Setting that allows you to enable TLS on ingress records. + tls: false + + # -- Setting that allows you to enable secure connections to the Longhorn UI service via port 443. + secureBackends: false + + # -- TLS secret that contains the private key and certificate to be used for TLS. This setting applies only when TLS is enabled on ingress records. + tlsSecret: longhorn.local-tls + + path: / + + ## If you're using kube-lego, you will want to add: + ## kubernetes.io/tls-acme: true + ## + ## For a full list of possible ingress annotations, please see + ## ref: https://github.com/kubernetes/ingress-nginx/blob/master/docs/annotations.md + ## + ## If tls is set to true, annotation ingress.kubernetes.io/secure-backends: "true" will automatically be set + # -- Ingress annotations in the form of key-value pairs. + annotations: + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: true + + # -- Secret that contains a TLS private key and certificate. Use secrets if you want to use your own certificates to secure ingresses. + secrets: + ## If you're providing your own certificates, please use this to add the certificates as secrets + ## key and certificate should start with -----BEGIN CERTIFICATE----- or + ## -----BEGIN RSA PRIVATE KEY----- + ## + ## name should line up with a tlsSecret set further up + ## If you're using kube-lego, this is unneeded, as it will create the secret for you if it is not set + ## + ## It is also possible to create and manage the certificates outside of this helm chart + ## Please see README.md for more information + # - name: longhorn.local-tls + # key: + # certificate: + + # -- Setting that allows you to enable pod security policies (PSPs) that allow privileged Longhorn pods to start. This setting applies only to clusters running Kubernetes 1.25 and earlier, and with the built-in Pod Security admission controller enabled. + enablePSP: false + + # -- Specify override namespace, specifically this is useful for using longhorn as sub-chart and its release namespace is not the `longhorn-system`. + namespaceOverride: "" + + # -- Annotation for the Longhorn Manager DaemonSet pods. This setting is optional. + annotations: {} + + serviceAccount: + # -- Annotations to add to the service account + annotations: {} + + metrics: + serviceMonitor: + # -- Setting that allows the creation of a Prometheus ServiceMonitor resource for Longhorn Manager components. + enabled: false + + ## openshift settings + openshift: + # -- Setting that allows Longhorn to integrate with OpenShift. + enabled: false + ui: + # -- Route for connections between Longhorn and the OpenShift web console. + route: "longhorn-ui" + # -- Port for accessing the OpenShift web console. + port: 443 + # -- Port for proxy that provides access to the OpenShift web console. + proxy: 8443 + + # -- Setting that allows Longhorn to generate code coverage profiles. + enableGoCoverDir: false +{{- end }} diff --git a/pkg/resources/templates/helm-apps/releases/additional-modules/secrets-store-csi-driver.yml.tmpl b/pkg/resources/templates/helm-apps/releases/additional-modules/secrets-store-csi-driver.yml.tmpl new file mode 100644 index 0000000..bb3b07a --- /dev/null +++ b/pkg/resources/templates/helm-apps/releases/additional-modules/secrets-store-csi-driver.yml.tmpl @@ -0,0 +1,6 @@ +{{- if .Modules.Additional.Storage.SecretsStoreCsiDriver.Enabled }} +- name: csi-secrets-store + namespace: kube-system + chart_ref: kube-forge/secrets-store-csi-driver + chart_version: 1.4.3 +{{- end }} \ No newline at end of file diff --git a/pkg/resources/templates/helm-apps/releases/cicd/argo-cd-ingress.yml.tmpl b/pkg/resources/templates/helm-apps/releases/cicd/argo-cd-ingress.yml.tmpl new file mode 100644 index 0000000..f55b97e --- /dev/null +++ b/pkg/resources/templates/helm-apps/releases/cicd/argo-cd-ingress.yml.tmpl @@ -0,0 +1,34 @@ +- name: argo-cd-ingress + namespace: cicd + create_namespace: true + chart_ref: kube-forge/service-ingress + chart_version: 0.1.0 + {{- if and .Modules.Cicd.Enabled (eq .Modules.Cicd.ArgoCd.Expose.Type "ingress") }} + release_state: "present" + {{- else }} + release_state: "absent" + {{- end }} + values: + services: + - domain: {{ .Modules.Cicd.ArgoCd.Expose.Domain }} + address: argo-cd-argocd-server + port: 80 + secretName: argo-cd-server-tls + + ingress: + accountEmail: {{ .Modules.Additional.CertManager.AccountEmail }} + class: {{ .Modules.Additional.Ingress.Type }} + annotations: + {{- if eq .Modules.Additional.Ingress.Type "nginx" }} + nginx.ingress.kubernetes.io/proxy-buffer-size: "128k" + nginx.ingress.kubernetes.io/proxy-buffers: "4 256k" + nginx.ingress.kubernetes.io/proxy-busy-buffers-size: "256k" + nginx.ingress.kubernetes.io/proxy-body-size: "0" + {{- end }} + tls: + enabled: {{ .Modules.Cicd.ArgoCd.Expose.Tls.Enabled }} + useCertManager: true + + # used if "useCertManager" is false + crt: "" + key: "" diff --git a/pkg/resources/templates/helm-apps/releases/cicd/argo-cd.yml.tmpl b/pkg/resources/templates/helm-apps/releases/cicd/argo-cd.yml.tmpl new file mode 100644 index 0000000..3698ff8 --- /dev/null +++ b/pkg/resources/templates/helm-apps/releases/cicd/argo-cd.yml.tmpl @@ -0,0 +1,144 @@ +- name: argo-cd + namespace: cicd + create_namespace: true + chart_ref: kube-forge/argo-cd + chart_version: 6.7.12 + {{- if .Modules.Cicd.Enabled }} + release_state: "present" + {{- else }} + release_state: "absent" + {{- end }} + values: + crds: + install: true + + server: + certificateSecret: + enabled: false + + {{- if .Modules.Cicd.ArgoCd.Ha.Enabled }} + {{- if .Modules.Cicd.ArgoCd.Ha.Autoscaling }} + autoscaling: + enabled: true + minReplicas: 2 + {{- else }} + replicas: 2 + {{- end }} + {{- end }} + + metrics: + enabled: {{ and .Modules.Observability.Enabled .Modules.Observability.Monitoring.Enabled }} + serviceMonitor: + enabled: {{ and .Modules.Observability.Enabled .Modules.Observability.Monitoring.Enabled }} + + redis: + metrics: + enabled: {{ and .Modules.Observability.Enabled .Modules.Observability.Monitoring.Enabled }} + serviceMonitor: + enabled: {{ and .Modules.Observability.Enabled .Modules.Observability.Monitoring.Enabled }} + + controller: + replicas: 1 + metrics: + enabled: {{ and .Modules.Observability.Enabled .Modules.Observability.Monitoring.Enabled }} + serviceMonitor: + enabled: {{ and .Modules.Observability.Enabled .Modules.Observability.Monitoring.Enabled }} + + applicationSet: + {{- if .Modules.Cicd.ArgoCd.Ha.Enabled }} + replicas: 2 + {{- end }} + metrics: + enabled: {{ and .Modules.Observability.Enabled .Modules.Observability.Monitoring.Enabled }} + serviceMonitor: + enabled: {{ and .Modules.Observability.Enabled .Modules.Observability.Monitoring.Enabled }} + + dex: + metrics: + enabled: {{ and .Modules.Observability.Enabled .Modules.Observability.Monitoring.Enabled }} + ## check later + serviceMonitor: + enabled: false + + repoServer: + {{- if .Modules.Cicd.ArgoCd.Ha.Enabled }} + {{- if .Modules.Cicd.ArgoCd.Ha.Autoscaling }} + autoscaling: + enabled: true + minReplicas: 2 + {{- else }} + replicas: 2 + {{- end }} + {{- end }} + + metrics: + enabled: {{ and .Modules.Observability.Enabled .Modules.Observability.Monitoring.Enabled }} + serviceMonitor: + enabled: {{ and .Modules.Observability.Enabled .Modules.Observability.Monitoring.Enabled }} + + notifications: + metrics: + enabled: {{ and .Modules.Observability.Enabled .Modules.Observability.Monitoring.Enabled }} + serviceMonitor: + enabled: {{ and .Modules.Observability.Enabled .Modules.Observability.Monitoring.Enabled }} + + configs: + params: + server.insecure: true + + secret: + argocdServerAdminPassword: {{ .Modules.Cicd.ArgoCd.AdminPassword }} + + repositories: + # add default helm-repository from harbor + {{- .Modules.Cicd.ArgoCd.Repositories | toYaml | nindent 8 }} + + cm: + create: true + url: "{{ if .Modules.Cicd.ArgoCd.Expose.Tls.Enabled }}https{{ else }}http{{ end }}://{{ .Modules.Cicd.ArgoCd.Expose.Domain }}" + + accounts.developer: login + accounts.guest: login + + # oidc.config: "" + + rbac: + create: true + policy.csv: | + p, role:admin, applications, create, */*, allow + p, role:admin, applications, update, */*, allow + p, role:admin, applications, delete, */*, allow + p, role:admin, applications, sync, */*, allow + p, role:admin, applications, override, */*, allow + p, role:admin, applications, action/*, */*, allow + p, role:admin, applicationsets, get, */*, allow + p, role:admin, applicationsets, create, */*, allow + p, role:admin, applicationsets, update, */*, allow + p, role:admin, applicationsets, delete, */*, allow + p, role:admin, certificates, create, *, allow + p, role:admin, certificates, update, *, allow + p, role:admin, certificates, delete, *, allow + p, role:admin, clusters, create, *, allow + p, role:admin, clusters, update, *, allow + p, role:admin, clusters, delete, *, allow + p, role:admin, repositories, create, *, allow + p, role:admin, repositories, update, *, allow + p, role:admin, repositories, delete, *, allow + p, role:admin, projects, create, *, allow + p, role:admin, projects, update, *, allow + p, role:admin, projects, delete, *, allow + p, role:admin, accounts, update, *, allow + p, role:admin, gpgkeys, create, *, allow + p, role:admin, gpgkeys, delete, *, allow + p, role:admin, exec, create, */*, allow + + {{- .Modules.Cicd.ArgoCd.Rbac.AdditionalPolicies }} + + + policy.default: role:'' + # scopes: "[roles,email,groups]" + + {{- if .Modules.Cicd.ArgoCd.Ha.Enabled }} + redis-ha: + enabled: true + {{- end }} diff --git a/pkg/resources/templates/helm-apps/releases/cicd/argo-rollouts.yml.tmpl b/pkg/resources/templates/helm-apps/releases/cicd/argo-rollouts.yml.tmpl new file mode 100644 index 0000000..398e279 --- /dev/null +++ b/pkg/resources/templates/helm-apps/releases/cicd/argo-rollouts.yml.tmpl @@ -0,0 +1,421 @@ +- name: argo-rollouts + namespace: cicd + create_namespace: true + chart_ref: kube-forge/argo-rollouts + {{- if and .Modules.Cicd.Enabled .Modules.Cicd.Rollouts.Enabled }} + release_state: "present" + {{- else }} + release_state: "absent" + {{- end }} + chart_version: 2.35.1 + values: + installCRDs: true + keepCRDs: false + clusterInstall: true + createClusterAggregateRoles: true + + apiVersionOverrides: + # -- String to override apiVersion of ingresses rendered by this helm chart + ingress: "" # networking.k8s.io/v1beta1 + + # -- Override the Kubernetes version, which is used to evaluate certain manifests + kubeVersionOverride: "" + + # -- Additional manifests to deploy within the chart. A list of objects. + ## Can be used to add secrets for Analysis with 3rd-party monitoring solutions. + extraObjects: [] + # - apiVersion: v1 + # kind: Secret + # metadata: + # name: datadog + # type: Opaque + # data: + # address: https://api.datadoghq.com + # api-key: + # app-key: + + global: + # -- Annotations for all deployed Deployments + deploymentAnnotations: {} + + controller: + # -- Value of label `app.kubernetes.io/component` + component: rollouts-controller + # -- Annotations to be added to the controller deployment + deploymentAnnotations: {} + # -- Annotations to be added to application controller pods + podAnnotations: {} + # -- [Node selector] + nodeSelector: {} + # -- [Tolerations] for use with node taints + tolerations: [] + # -- Assign custom [affinity] rules to the deployment + affinity: {} + logging: + # -- Set the logging level (one of: `debug`, `info`, `warn`, `error`) + level: info + # -- Set the klog logging level + kloglevel: "0" + # -- Set the logging format (one of: `text`, `json`) + format: "text" + + # -- Assign custom [TopologySpreadConstraints] rules to the controller + ## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ + ## If labelSelector is left out, it will default to the labelSelector configuration of the deployment + topologySpreadConstraints: [] + # - maxSkew: 1 + # topologyKey: topology.kubernetes.io/zone + # whenUnsatisfiable: DoNotSchedule + + # -- [priorityClassName] for the controller + priorityClassName: "" + # -- The number of controller pods to run + replicas: 2 + image: + # -- Registry to use + registry: quay.io + # -- Repository to use + repository: argoproj/argo-rollouts + # -- Overrides the image tag (default is the chart appVersion) + tag: "" + # -- Image pull policy + pullPolicy: IfNotPresent + + # -- flag to enable creation of cluster controller role (requires cluster RBAC) + createClusterRole: true + + # Controller container ports + containerPorts: + # -- Metrics container port + metrics: 8090 + # -- Healthz container port + healthz: 8080 + {{- if and .Modules.Observability.Enabled .Modules.Observability.Monitoring.Enabled }} + metrics: + # -- Deploy metrics service + enabled: true + service: + # -- Metrics service port name + portName: metrics + # -- Metrics service port + port: 8090 + # -- Service annotations + annotations: {} + serviceMonitor: + # -- Enable a prometheus ServiceMonitor + enabled: true + # -- Namespace to be used for the ServiceMonitor + namespace: "" + # -- Labels to be added to the ServiceMonitor + additionalLabels: {} + # -- Annotations to be added to the ServiceMonitor + additionalAnnotations: {} + # -- RelabelConfigs to apply to samples before scraping + relabelings: [] + # -- MetricRelabelConfigs to apply to samples before ingestion + metricRelabelings: [] + {{- end }} + + # -- Configure liveness [probe] for the controller + # @default -- See [values.yaml] + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 30 + periodSeconds: 20 + failureThreshold: 3 + successThreshold: 1 + timeoutSeconds: 10 + + # -- Configure readiness [probe] for the controller + # @default -- See [values.yaml] + readinessProbe: + httpGet: + path: /metrics + port: metrics + initialDelaySeconds: 15 + periodSeconds: 5 + failureThreshold: 3 + successThreshold: 1 + timeoutSeconds: 4 + + ## Configure Pod Disruption Budget for the controller + pdb: + # -- Labels to be added to controller [Pod Disruption Budget] + labels: {} + # -- Annotations to be added to controller [Pod Disruption Budget] + annotations: {} + # -- Deploy a [Pod Disruption Budget] for the controller + enabled: false + # -- Minimum number / percentage of pods that should remain scheduled + minAvailable: # 1 + # -- Maximum number / percentage of pods that may be made unavailable + maxUnavailable: # 0 + + # -- Additional volumes to add to the controller pod + volumes: [] + # - configMap: + # name: my-certs-cm + # name: my-certs + + # -- Additional volumeMounts to add to the controller container + volumeMounts: [] + # - mountPath: /etc/ssl/certs + # name: my-certs + + # -- Configures 3rd party metric providers for controller + ## Ref: https://argo-rollouts.readthedocs.io/en/stable/analysis/plugins/ + metricProviderPlugins: {} + # metricProviderPlugins: |- + # - name: "argoproj-labs/sample-prometheus" # name of the plugin, it must match the name required by the plugin so that it can find its configuration + # location: "file://./my-custom-plugin" # supports http(s):// urls and file:// + + # -- Configures 3rd party traffic router plugins for controller + ## Ref: https://argo-rollouts.readthedocs.io/en/stable/features/traffic-management/plugins/ + trafficRouterPlugins: {} + # trafficRouterPlugins: |- + # - name: "argoproj-labs/sample-nginx" # name of the plugin, it must match the name required by the plugin so it can find it's configuration + # location: "file://./my-custom-plugin" # supports http(s):// urls and file:// + + serviceAccount: + # -- Specifies whether a service account should be created + create: true + # -- Annotations to add to the service account + annotations: {} + # -- The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + + # -- Annotations to be added to all CRDs + crdAnnotations: {} + + # -- Annotations for the all deployed pods + podAnnotations: {} + + # -- Security Context to set on pod level + podSecurityContext: + runAsNonRoot: true + + # -- Security Context to set on container level + containerSecurityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + + # -- Annotations to be added to the Rollout service + serviceAnnotations: {} + + # -- Labels to be added to the Rollout pods + podLabels: {} + + # -- Secrets with credentials to pull images from a private registry. Registry secret names as an array. + imagePullSecrets: [] + # - name: argo-pull-secret + + providerRBAC: + # -- Toggles addition of provider-specific RBAC rules to the controller Role and ClusterRole + enabled: true + # providerRBAC.enabled must be true in order to toggle the individual providers + providers: + # -- Adds RBAC rules for the Istio provider + istio: true + # -- Adds RBAC rules for the SMI provider + smi: true + # -- Adds RBAC rules for the Ambassador provider + ambassador: true + # -- Adds RBAC rules for the AWS Load Balancer Controller provider + awsLoadBalancerController: true + # -- Adds RBAC rules for the AWS App Mesh provider + awsAppMesh: true + # -- Adds RBAC rules for the Traefik provider + traefik: true + # -- Adds RBAC rules for the Apisix provider + apisix: true + # -- Adds RBAC rules for the Contour provider, see `https://github.com/argoproj-labs/rollouts-plugin-trafficrouter-contour/blob/main/README.md` + contour: true + # -- Adds RBAC rules for the Gloo Platform provider, see `https://github.com/argoproj-labs/rollouts-plugin-trafficrouter-glooplatform/blob/main/README.md` + glooPlatform: true + # -- Additional RBAC rules for others providers + additionalRules: [] + + dashboard: + # -- Deploy dashboard server + enabled: true + # -- Set cluster role to readonly + readonly: false + # -- Value of label `app.kubernetes.io/component` + component: rollouts-dashboard + # -- Annotations to be added to the dashboard deployment + deploymentAnnotations: {} + # -- Annotations to be added to application dashboard pods + podAnnotations: {} + # -- [Node selector] + nodeSelector: {} + # -- [Tolerations] for use with node taints + tolerations: [] + # -- Assign custom [affinity] rules to the deployment + affinity: {} + logging: + # -- Set the logging level (one of: `debug`, `info`, `warn`, `error`) + level: info + # -- Set the klog logging level + kloglevel: "0" + + # -- Assign custom [TopologySpreadConstraints] rules to the dashboard server + ## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ + ## If labelSelector is left out, it will default to the labelSelector configuration of the deployment + topologySpreadConstraints: [] + # - maxSkew: 1 + # topologyKey: topology.kubernetes.io/zone + # whenUnsatisfiable: DoNotSchedule + + # -- [priorityClassName] for the dashboard server + priorityClassName: "" + + # -- flag to enable creation of dashbord cluster role (requires cluster RBAC) + createClusterRole: true + + # -- The number of dashboard pods to run + replicas: 1 + image: + # -- Registry to use + registry: quay.io + # -- Repository to use + repository: argoproj/kubectl-argo-rollouts + # -- Overrides the image tag (default is the chart appVersion) + tag: "" + # -- Image pull policy + pullPolicy: IfNotPresent + # -- Additional command line arguments to pass to rollouts-dashboard. A list of flags. + extraArgs: [] + # -- Additional environment variables for rollouts-dashboard. A list of name/value maps. + extraEnv: [] + # - name: FOO + # value: bar + # -- Resource limits and requests for the dashboard pods. + resources: {} + # -- Security Context to set on pod level + podSecurityContext: + runAsNonRoot: true + # -- Security Context to set on container level + containerSecurityContext: {} + service: + # -- Sets the type of the Service + type: ClusterIP + # -- LoadBalancer will get created with the IP specified in this field + loadBalancerIP: "" + # -- Source IP ranges to allow access to service from + loadBalancerSourceRanges: [] + # -- Dashboard service external IPs + externalIPs: [] + # -- Service annotations + annotations: {} + # -- Service labels + labels: {} + # -- Service port name + portName: dashboard + # -- Service port + port: 3100 + # -- Service target port + targetPort: 3100 + # -- (int) Service nodePort + nodePort: + serviceAccount: + # -- Specifies whether a dashboard service account should be created + create: true + # -- Annotations to add to the dashboard service account + annotations: {} + # -- The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + + ## Configure Pod Disruption Budget for the dashboard + pdb: + # -- Labels to be added to dashboard [Pod Disruption Budget] + labels: {} + # -- Annotations to be added to dashboard [Pod Disruption Budget] + annotations: {} + # -- Deploy a [Pod Disruption Budget] for the dashboard + enabled: false + # -- Minimum number / percentage of pods that should remain scheduled + minAvailable: # 1 + # -- Maximum number / percentage of pods that may be made unavailable + maxUnavailable: # 0 + + ## Ingress configuration. + ## ref: https://kubernetes.io/docs/user-guide/ingress/ + ## + ingress: + # -- Enable dashboard ingress support + enabled: false + # -- Dashboard ingress annotations + annotations: {} + # -- Dashboard ingress labels + labels: {} + # -- Dashboard ingress class name + ingressClassName: "" + + # -- Dashboard ingress hosts + ## Argo Rollouts Dashboard Ingress. + ## Hostnames must be provided if Ingress is enabled. + ## Secrets must be manually created in the namespace + hosts: [] + # - argorollouts.example.com + + # -- Dashboard ingress paths + paths: + - / + # -- Dashboard ingress path type + pathType: Prefix + # -- Dashboard ingress extra paths + extraPaths: [] + # - path: /* + # backend: + # serviceName: ssl-redirect + # servicePort: use-annotation + ## for Kubernetes >=1.19 (when "networking.k8s.io/v1" is used) + # - path: /* + # pathType: Prefix + # backend: + # service + # name: ssl-redirect + # port: + # name: use-annotation + + # -- Dashboard ingress tls + tls: [] + # - secretName: argorollouts-example-tls + # hosts: + # - argorollouts.example.com + + # -- Additional volumes to add to the dashboard pod + volumes: [] + + # -- Additional volumeMounts to add to the dashboard container + volumeMounts: [] + + notifications: + secret: + # -- Whether to create notifications secret + create: false + # -- Generic key:value pairs to be inserted into the notifications secret + items: {} + # slack-token: + + # -- Configures notification services + notifiers: {} + # service.slack: | + # token: $slack-token + + # -- Notification templates + templates: {} + + # -- The trigger defines the condition when the notification should be sent + triggers: {} + # trigger.on-purple: | + # - send: [my-purple-template] + # when: rollout.spec.template.spec.containers[0].image == 'argoproj/rollouts-demo:purple' diff --git a/pkg/resources/templates/helm-apps/releases/cicd/keel.yml.tmpl b/pkg/resources/templates/helm-apps/releases/cicd/keel.yml.tmpl new file mode 100644 index 0000000..75da976 --- /dev/null +++ b/pkg/resources/templates/helm-apps/releases/cicd/keel.yml.tmpl @@ -0,0 +1,9 @@ +- name: keel + namespace: kube-system + chart_ref: kube-forge/keel + chart_version: 1.0.3 + {{- if and .Modules.Cicd.Enabled .Modules.Cicd.UpdatesOperator.Enabled }} + release_state: "present" + {{- else }} + release_state: "absent" + {{- end }} diff --git a/pkg/resources/templates/helm-apps/releases/observability/fluent-operator.yml.tmpl b/pkg/resources/templates/helm-apps/releases/observability/fluent-operator.yml.tmpl new file mode 100644 index 0000000..f3da9b4 --- /dev/null +++ b/pkg/resources/templates/helm-apps/releases/observability/fluent-operator.yml.tmpl @@ -0,0 +1,66 @@ +- name: fluent-operator + namespace: observability + create_namespace: true + chart_ref: kube-forge/fluent-operator + chart_version: 2.7.0 + {{- if and .Modules.Observability.Enabled .Modules.Observability.Logging.Enabled }} + release_state: "present" + {{- else }} + release_state: "absent" + {{- end }} + values: + # Set this to containerd or crio if you want to collect CRI format logs + containerRuntime: {{ .Orchestrator.ContainerEngine.Type }} + Kubernetes: false + + operator: + initcontainer: + repository: "{{ .Modules.Observability.Logging.Operator.InitContainer.Image }}" + tag: "{{ .Modules.Observability.Logging.Operator.InitContainer.Tag }}" + + resources: + limits: + cpu: 100m + memory: 100Mi + + requests: + cpu: 50m + memory: 64Mi + container: + repository: "{{ .Modules.Observability.Logging.Operator.Image }}" + tag: "{{ .Modules.Observability.Logging.Operator.Tag }}" + resources: + limits: + cpu: 100m + memory: 100Mi + requests: + cpu: 100m + memory: 60Mi + imagePullSecrets: [] + labels: {} + logPath: + # The operator currently assumes a Docker container runtime path for the logs as the default, for other container runtimes you can set the location explicitly below. + # crio: /var/log + containerd: /var/log + disableComponentControllers: "" + + fluentbit: + crdsEnable: true + enable: false + + fluentd: + crdsEnable: true + enable: false + name: fluentd + # Valid modes include "collector" and "agent". + # The "collector" mode will deploy Fluentd as a StatefulSet as before. + # The new "agent" mode will deploy Fluentd as a DaemonSet. + mode: "agent" + port: 24224 + image: + repository: "{{ .Modules.Observability.Logging.Fluentd.Image }}" + tag: "{{ .Modules.Observability.Logging.Fluentd.Tag }}" + + nameOverride: "" + fullnameOverride: "" + namespaceOverride: "" diff --git a/pkg/resources/templates/helm-apps/releases/observability/loki.yml.tmpl b/pkg/resources/templates/helm-apps/releases/observability/loki.yml.tmpl new file mode 100644 index 0000000..bf048ba --- /dev/null +++ b/pkg/resources/templates/helm-apps/releases/observability/loki.yml.tmpl @@ -0,0 +1,200 @@ +- name: loki + namespace: observability + create_namespace: true + chart_ref: kube-forge/loki + chart_version: 5.47.2 + {{- if and .Modules.Observability.Enabled .Modules.Observability.Logging.Enabled }} + release_state: "present" + {{- else }} + release_state: "absent" + {{- end }} + values: + loki: + image: + registry: {{ .Modules.Observability.Logging.Loki.Registry }} + repository: {{ .Modules.Observability.Logging.Loki.Image }} + tag: {{ .Modules.Observability.Logging.Loki.Tag }} + podAnnotations: + app.kubernetes.io/component: "loki" + auth_enabled: false + commonConfig: + replication_factor: 1 + storage: + type: 'filesystem' + + frontend: + max_outstanding_per_tenant: 10000 + + limits_config: + reject_old_samples: false + split_queries_by_interval: 15m + max_query_parallelism: 32 + max_query_series: 10000 + retention_period: {{ .Modules.Observability.Logging.Loki.Persistence.Retention }} + + compactor: + compaction_interval: 10m + retention_enabled: true + retention_delete_delay: 2h + + querier: + max_concurrent: 2048 + + query_scheduler: + max_outstanding_requests_per_tenant: 10000 + + rulerConfig: + storage: + type: local + local: + directory: /var/loki/rules + rule_path: /tmp/rules + + alertmanager_url: {{ .Modules.Observability.Logging.Loki.AlertManagerUrl }} + + + singleBinary: + replicas: 1 + + extraVolumes: + - name: loki-default-rules + configMap: + name: loki-default-alerting-rules + + extraVolumeMounts: + - name: loki-default-rules + mountPath: /var/loki/rules + + + write: + persistence: + volumeClaimsEnabled: true + storageClass: "{{ .Modules.Observability.Logging.Loki.Persistence.StorageClass }}" + size: {{ .Modules.Observability.Logging.Loki.Persistence.StorageSize }} + + test: + enabled: false + + gateway: + enabled: false + + monitoring: + selfMonitoring: + enabled: false + grafanaAgent: + installOperator: false + lokiCanary: + enabled: false + rules: + enabled: true + alerting: true + additionalGroups: {} + + extraObjects: + - apiVersion: v1 + kind: ConfigMap + metadata: + name: loki-default-alerting-rules + labels: + loki_rule: "" + + data: + loki-default-alerting-rules.yaml: |- + groups: + {{- .Modules.Observability.Logging.Loki.AdditionalRulesGroups | toString | nindent 14 -}} + - name: kube-events-alerts + rules: + - alert: FailedEventsOccured + expr: | + count(rate({logs_type="kube-events"} | json reason="reason", event_type="event_type" | event_type = `Warning` | reason = `Failed` [1h])) > 0 + for: 1m + labels: + severity: critical + annotations: + alertname: FailedEventsOccured + instance: kube-cluster + jobName: kube_events + summary: Failed events occured in cluster + addDefaultUrl: "true" + + - alert: OOMKilledEventsOccured + expr: | + count(rate({logs_type="kube-events"} | json reason="reason", event_type="event_type" | event_type = `Warning` | reason = `OOMKilled` [1h])) > 0 + for: 1m + labels: + severity: critical + annotations: + alertname: OOMKilledEventsOccured + instance: kube-cluster + jobName: kube_events + summary: OOMKilled events occured in cluster + addDefaultUrl: "true" + + - alert: EvictedEventsOccured + expr: | + count(rate({logs_type="kube-events"} | json reason="reason", event_type="event_type" | event_type = `Warning` | reason = `Evicted` [1h])) > 0 + for: 1m + labels: + severity: critical + annotations: + alertname: EvictedEventsOccured + instance: kube-cluster + jobName: kube_events + summary: Evicted events occured in cluster + addDefaultUrl: "true" + + - alert: ImagePullBackOffEventsOccured + expr: | + count(rate({logs_type="kube-events"} | json reason="reason", event_type="event_type" | event_type = `Warning` | reason = `ImagePullBackOff` [1h])) > 0 + for: 1m + labels: + severity: critical + annotations: + alertname: ImagePullBackOffEventsOccured + instance: kube-cluster + jobName: kube_events + summary: ImagePullBackOff events occured in cluster + addDefaultUrl: "true" + + - alert: BackOffEventsOccured + expr: | + count(rate({logs_type="kube-events"} | json reason="reason", event_type="event_type" | event_type = `Warning` | reason = `BackOff` [1h])) > 0 + for: 1m + labels: + severity: critical + annotations: + alertname: BackOffEventsOccured + instance: kube-cluster + jobName: kube_events + summary: BackOff events occured in cluster + addDefaultUrl: "true" + sidecar: + rules: + enabled: true + # -- Label that the configmaps/secrets with rules will be marked with. + label: loki_rule + # -- Label value that the configmaps/secrets with rules will be set to. + labelValue: "" + # -- Folder into which the rules will be placed. + folder: /var/loki/rules + # -- Comma separated list of namespaces. If specified, the sidecar will search for config-maps/secrets inside these namespaces. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify 'ALL' to search in all namespaces. + searchNamespace: 'ALL' + # -- Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH request, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # -- Search in configmap, secret, or both. + resource: both + # -- Absolute path to the shell script to execute after a configmap or secret has been reloaded. + script: null + # -- WatchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S. + watchServerTimeout: 60 + # + # -- WatchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # Defaults to 66sec. + watchClientTimeout: 60 + # -- Log level of the sidecar container. + logLevel: INFO diff --git a/pkg/resources/templates/helm-apps/releases/observability/observability.yml.tmpl b/pkg/resources/templates/helm-apps/releases/observability/observability.yml.tmpl new file mode 100644 index 0000000..f98cdb2 --- /dev/null +++ b/pkg/resources/templates/helm-apps/releases/observability/observability.yml.tmpl @@ -0,0 +1,315 @@ +- name: observability + namespace: observability + create_namespace: true + chart_ref: kube-forge/observability + chart_version: 0.1.0 + {{- if .Modules.Observability.Enabled }} + release_state: "present" + {{- else }} + release_state: "absent" + {{- end }} + values: + prometheus: + enabled: {{ .Modules.Observability.Monitoring.Enabled }} + serviceMonitor: true + image: + repository: {{ .Modules.Observability.Monitoring.Prometheus.Image }} + tag: {{ .Modules.Observability.Monitoring.Prometheus.Tag }} + pullPolicy: IfNotPresent + + clustering: + enabled: false + replicas: 3 + shards: 1 + + persistence: + enabled: true + + storageClassName: "{{ .Modules.Observability.Monitoring.Prometheus.Persistence.StorageClass }}" + storageResources: + requests: + storage: {{ .Modules.Observability.Monitoring.Prometheus.Persistence.StorageSize }} + + scrapeInterval: {{ .Modules.Observability.Monitoring.Prometheus.ScrapeInterval }} + retention: {{ .Modules.Observability.Monitoring.Prometheus.Persistence.Retention }} + # serviceNodePort: 30008 + + alertManager: + enabled: {{ .Modules.Observability.Monitoring.AlertManager.Enabled }} + serviceMonitor: {{ .Modules.Observability.Monitoring.Enabled }} + enableDefaultRules: true + image: + repository: {{ .Modules.Observability.Monitoring.AlertManager.Image }} + tag: "{{ .Modules.Observability.Monitoring.AlertManager.Tag }}" + pullPolicy: IfNotPresent + + # serviceNodePort: 30009 + + configPath: /etc/alertmanager + + route: + {{- .Modules.Observability.Monitoring.AlertManager.Route | toYaml | nindent 8 }} + + receivers: + {{- .Modules.Observability.Monitoring.AlertManager.Receivers | toYaml | nindent 8 }} + + + blackboxExporter: + enabled: {{ .Modules.Observability.Monitoring.Blackbox.Enabled }} + serviceMonitor: {{ .Modules.Observability.Monitoring.Enabled }} + image: + repository: {{ .Modules.Observability.Monitoring.Blackbox.Image }} + tag: "{{ .Modules.Observability.Monitoring.Blackbox.Tag }}" + pullPolicy: IfNotPresent + + # serviceNodePort: 30012 + + configPath: /etc/blackbox_exporter + additionalModules: + + + kubeStateMetrics: + enabled: {{ .Modules.Observability.Monitoring.KubeState.Enabled }} + image: + repository: {{ .Modules.Observability.Monitoring.KubeState.Image }} + tag: "{{ .Modules.Observability.Monitoring.KubeState.Tag }}" + pullPolicy: IfNotPresent + resources: + requests: + cpu: 30m + memory: 120Mi + limits: + memory: 240Mi + cpu: 60m + + prometheusOperator: + enabled: {{ .Modules.Observability.Monitoring.Enabled }} + image: + repository: {{ .Modules.Observability.Monitoring.Prometheus.Operator.Image }} + tag: {{ .Modules.Observability.Monitoring.Prometheus.Operator.Tag }} + pullPolicy: IfNotPresent + + prometheusConfigReloader: + image: + repository: {{ .Modules.Observability.Monitoring.Prometheus.Operator.ConfigReloader.Image }} + tag: {{ .Modules.Observability.Monitoring.Prometheus.Operator.ConfigReloader.Tag }} + pullPolicy: IfNotPresent + + kubeRbacProxy: + image: + repository: {{ .Modules.Observability.Monitoring.Prometheus.Operator.KubeRbacProxy.Image }} + tag: {{ .Modules.Observability.Monitoring.Prometheus.Operator.KubeRbacProxy.Tag }} + pullPolicy: IfNotPresent + + nodeExporter: + enabled: {{ .Modules.Observability.Monitoring.Node.Enabled }} + image: + repository: {{ .Modules.Observability.Monitoring.Node.Image }} + tag: {{ .Modules.Observability.Monitoring.Node.Tag }} + pullPolicy: IfNotPresent + + kubeEventsExporter: + enabled: {{ and .Modules.Observability.Logging.Enabled .Modules.Observability.Logging.Events.Enabled }} + image: + repository: {{ .Modules.Observability.Logging.Events.Exporter.Image }} + tag: {{ .Modules.Observability.Logging.Events.Exporter.Tag }} + pullPolicy: IfNotPresent + lokiAddress: http://loki.observability.svc.cluster.local:3100 + logLevel: warn + logFormat: json + kubeQPS: 100 + kubeBurst: 500 + maxEventAgeSeconds: 120 + metricsNamePrefix: event_exporter_ + + cron: + restartSchedule: "{{ .Modules.Observability.Logging.Events.Cron.Schedule }}" + image: + repository: {{ .Modules.Observability.Logging.Events.Cron.Image }} + tag: {{ .Modules.Observability.Logging.Events.Cron.Tag }} + pullPolicy: IfNotPresent + additionalRoutes: + additionalReceivers: + + grafana: + enabled: {{ and .Modules.Observability.Enabled .Modules.Observability.Visualization.Grafana.Enabled }} + serviceMonitor: {{ .Modules.Observability.Monitoring.Enabled }} + domain: &grafanaDomain {{ .Modules.Observability.Visualization.Grafana.Expose.Domain }} + image: + repository: {{ .Modules.Observability.Visualization.Grafana.Image }} + tag: {{ .Modules.Observability.Visualization.Grafana.Tag }} + pullPolicy: IfNotPresent + + storageClassName: "{{ .Modules.Observability.Visualization.Grafana.Persistence.StorageClass }}" + storageResources: + requests: + storage: {{ .Modules.Observability.Visualization.Grafana.Persistence.StorageSize }} + + config: + server: | + root_url = {{ if .Modules.Observability.Visualization.Grafana.Expose.Tls.Enabled }}https{{ else }}http{{ end }}://{{ .Modules.Observability.Visualization.Grafana.Expose.Domain }} + + security: | + admin_user = admin + admin_password = {{ .Modules.AdminPassword }} + + auth: | + {{- .Modules.Observability.Visualization.Grafana.Config.Auth | toString | nindent 10 }} + + authGenericAuth: | + {{- .Modules.Observability.Visualization.Grafana.Config.AuthGenericAuth | toString | nindent 10 }} + + additionalDatasources: + {{- if and .Modules.Observability.Enabled .Modules.Observability.Logging.Enabled }} + - name: Kube-loki + type: loki + access: proxy + url: http://loki:3100 + editable: false + basicAuth: false + isDefault: false + jsonData: + maxLines: 1000 + {{- end }} + {{- if and .Modules.Observability.Enabled .Modules.Observability.Tracing.Enabled }} + - name: Kube-jaeger-query + type: jaeger + access: proxy + url: http://tempo:16686 + editable: false + basicAuth: false + isDefault: false + {{- end }} + {{- if .Modules.Observability.Visualization.Grafana.Config.AdditionalDatasources }} + {{- .Modules.Observability.Visualization.Grafana.Config.AdditionalDatasources | toYaml | nindent 10 }} + {{- end }} + + opentelemetryCollector: + enabled: {{ and .Modules.Observability.Enabled .Modules.Observability.Tracing.Enabled }} + serviceMonitor: true + config: | + receivers: + otlp: + protocols: + grpc: + http: + exporters: + otlphttp: + endpoint: http://tempo:4318 + service: + telemetry: + logs: + level: "debug" + pipelines: + traces: + receivers: [otlp] + exporters: [otlphttp] + + ingress: + {{- if eq .Modules.Observability.Visualization.Grafana.Expose.Type "ingress" }} + enabled: true + {{- end }} + accountEmail: {{ .Modules.Additional.CertManager.AccountEmail }} + class: {{ .Modules.Additional.Ingress.Type }} + annotations: + {{- if eq .Modules.Additional.Ingress.Type "nginx" }} + nginx.ingress.kubernetes.io/proxy-buffer-size: "128k" + nginx.ingress.kubernetes.io/proxy-buffers: "4 256k" + nginx.ingress.kubernetes.io/proxy-busy-buffers-size: "256k" + {{- end }} + tls: + {{- if .Modules.Observability.Visualization.Grafana.Expose.Tls.Enabled }} + enabled: true + {{- end }} + hosts: + - host: {{ .Modules.Observability.Visualization.Grafana.Expose.Domain }} + secretName: grafana-tls + + containerRuntime: {{ .Orchestrator.ContainerEngine.Type }} + + fluentbit: + enable: {{ .Modules.Observability.Logging.Enabled }} + serviceMonitor: true + image: + repository: "{{ .Modules.Observability.Logging.FluentBit.Image }}" + tag: "{{ .Modules.Observability.Logging.FluentBit.Tag }}" + + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/edge + operator: DoesNotExist + tolerations: + - operator: Exists + + input: + tail: + enable: true + refreshIntervalSeconds: 10 + memBufLimit: 100MB + bufferMaxSize: "" + path: "/var/log/containers/*.log" + skipLongLines: true + readFromHead: false + storageType: memory + pauseOnChunksOverlimit: "off" + systemd: + enable: true + systemdFilter: + enable: true + filters: [] + path: "/var/log/journal" + includeKubelet: true + stripUnderscores: "off" + storageType: memory + pauseOnChunksOverlimit: "off" + + nodeExporterMetrics: {} + fluentBitMetrics: {} + + output: + es: + enable: false + host: "" + port: 9200 + logstashPrefix: ks-logstash-log + bufferSize: 20MB + traceError: true + kafka: + enable: false + brokers: "" + topics: ks-log + opentelemetry: {} + opensearch: + enable: false + stdout: + enable: false + loki: + enable: true + host: loki + port: 3100 + + stackdriver: {} + + service: + storage: {} + + filter: + kubernetes: + enable: true + labels: true + annotations: true + containerd: + enable: true + systemd: + enable: true + + kubeedge: + enable: false + prometheusRemoteWrite: + # Change the host to the address of a cloud-side Prometheus-compatible server that can receive Prometheus remote write data + host: "" + # Change the port to the port of a cloud-side Prometheus-compatible server that can receive Prometheus remote write data + port: "" diff --git a/pkg/resources/templates/helm-apps/releases/observability/opentelemetry-operator.yml.tmpl b/pkg/resources/templates/helm-apps/releases/observability/opentelemetry-operator.yml.tmpl new file mode 100644 index 0000000..54e01f4 --- /dev/null +++ b/pkg/resources/templates/helm-apps/releases/observability/opentelemetry-operator.yml.tmpl @@ -0,0 +1,145 @@ +- name: opentelemetry-operator + namespace: observability + create_namespace: true + chart_ref: kube-forge/opentelemetry-operator + chart_version: 0.55.0 + {{- if and .Modules.Observability.Enabled .Modules.Observability.Tracing.Enabled }} + release_state: "present" + {{- else }} + release_state: "absent" + {{- end }} + values: + replicaCount: 1 + nameOverride: "" + imagePullSecrets: [] + pdb: + create: false + minAvailable: 1 + maxUnavailable: "" + + manager: + image: + repository: {{ .Modules.Observability.Tracing.Operator.Image }} + tag: "{{ .Modules.Observability.Tracing.Operator.Tag }}" + collectorImage: + repository: {{ .Modules.Observability.Tracing.Collector.Image }} + tag: {{ .Modules.Observability.Tracing.Collector.Tag }} + + featureGates: "" + ports: + metricsPort: 8080 + webhookPort: 9443 + healthzPort: 8081 + resources: + limits: + cpu: 100m + memory: 128Mi + requests: + cpu: 100m + memory: 64Mi + env: + ENABLE_WEBHOOKS: "true" + + serviceAccount: + create: true + annotations: {} + + serviceMonitor: + enabled: true + metricsEndpoints: + - port: metrics + + prometheusRule: + enabled: true + groups: [] + defaultRules: + enabled: true + + extraArgs: [] + + leaderElection: + enabled: true + + verticalPodAutoscaler: + enabled: false + controlledResources: [] + maxAllowed: {} + minAllowed: {} + + updatePolicy: + updateMode: Auto + minReplicas: 2 + rolling: false + + securityContext: {} + + kubeRBACProxy: + enabled: true + image: + repository: quay.io/brancz/kube-rbac-proxy + tag: v0.15.0 + ports: + proxyPort: 8443 + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 5m + memory: 64Mi + + extraArgs: [] + + securityContext: {} + + admissionWebhooks: + create: true + servicePort: 443 + failurePolicy: Fail + secretName: "" + pods: + failurePolicy: Ignore + + namePrefix: "" + + timeoutSeconds: 10 + + namespaceSelector: {} + objectSelector: {} + certManager: + enabled: true + issuerRef: {} + certificateAnnotations: {} + issuerAnnotations: {} + + autoGenerateCert: + enabled: true + recreate: true + + secretAnnotations: {} + secretLabels: {} + + role: + create: true + + clusterRole: + create: true + + affinity: {} + tolerations: [] + nodeSelector: {} + topologySpreadConstraints: [] + hostNetwork: false + + priorityClassName: "" + + securityContext: + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 + fsGroup: 65532 + + testFramework: + image: + repository: busybox + tag: latest diff --git a/pkg/resources/templates/helm-apps/releases/observability/tempo.yml.tmpl b/pkg/resources/templates/helm-apps/releases/observability/tempo.yml.tmpl new file mode 100644 index 0000000..b261a75 --- /dev/null +++ b/pkg/resources/templates/helm-apps/releases/observability/tempo.yml.tmpl @@ -0,0 +1,79 @@ +- name: tempo + namespace: observability + create_namespace: true + chart_ref: kube-forge/tempo + chart_version: 1.7.2 + {{- if and .Modules.Observability.Enabled .Modules.Observability.Tracing.Enabled }} + release_state: "present" + {{- else }} + release_state: "absent" + {{- end }} + values: + replicas: 1 + + tempo: + repository: {{ .Modules.Observability.Tracing.Tempo.Image }} + tag: "{{ .Modules.Observability.Tracing.Tempo.Tag }}" + pullPolicy: IfNotPresent + + updateStrategy: RollingUpdate + + memBallastSizeMbs: 1024 + multitenancyEnabled: false + reportingEnabled: false + + metricsGenerator: + enabled: false + remoteWriteUrl: "http://prometheus.monitoring:9090/api/v1/write" + retention: {{ .Modules.Observability.Tracing.Tempo.Retention }} + global_overrides: + per_tenant_override_config: /conf/overrides.yaml + + server: + http_listen_port: {{ .Modules.Observability.Tracing.Tempo.ListenPort }} + storage: + trace: + backend: local + local: + path: /var/tempo/traces + wal: + path: /var/tempo/wal + receivers: + otlp: + protocols: + grpc: + endpoint: "0.0.0.0:4317" + http: + endpoint: "0.0.0.0:4318" + + tempoQuery: + repository: {{ .Modules.Observability.Tracing.Tempo.TempoQuery.Image }} + tag: {{ .Modules.Observability.Tracing.Tempo.TempoQuery.Tag }} + pullPolicy: IfNotPresent + + enabled: true + + service: + port: {{ .Modules.Observability.Tracing.Tempo.TempoQuery.ListenPort }} + + ingress: + enabled: false + + serviceAccount: + create: true + automountServiceAccountToken: true + + service: + type: ClusterIP + + serviceMonitor: + enabled: true + + persistence: + enabled: true + storageClassName: {{ .Modules.Observability.Tracing.Tempo.Persistence.StorageClass }} + accessModes: + - ReadWriteOnce + size: {{ .Modules.Observability.Tracing.Tempo.Persistence.StorageSize }} + + priorityClassName: null diff --git a/pkg/resources/templates/helm-apps/releases/registry/harbor-certificate-generator.yml.tmpl b/pkg/resources/templates/helm-apps/releases/registry/harbor-certificate-generator.yml.tmpl new file mode 100644 index 0000000..716bc7a --- /dev/null +++ b/pkg/resources/templates/helm-apps/releases/registry/harbor-certificate-generator.yml.tmpl @@ -0,0 +1,17 @@ +- name: harbor-certificate-generator + namespace: {{ .Modules.Registry.Namespace }} + create_namespace: true + chart_ref: kube-forge/certificate-generator + chart_version: 0.1.0 + {{- if and .Modules.Registry.Enabled (eq .Modules.Registry.Expose.Type "ingress") }} + release_state: "present" + {{- else }} + release_state: "absent" + {{- end }} + values: + issuer_email: {{ .Modules.Additional.CertManager.AccountEmail }} + solver_ingress_class: {{ .Modules.Additional.Ingress.Type }} + + certificates: + - name: harbor-tls + domain: {{ .Modules.Registry.Expose.Domain }} diff --git a/pkg/resources/templates/helm-apps/releases/registry/harbor.yml.tmpl b/pkg/resources/templates/helm-apps/releases/registry/harbor.yml.tmpl new file mode 100644 index 0000000..276e2c3 --- /dev/null +++ b/pkg/resources/templates/helm-apps/releases/registry/harbor.yml.tmpl @@ -0,0 +1,382 @@ +- name: harbor + namespace: {{ .Modules.Registry.Namespace }} + create_namespace: true + chart_ref: kube-forge/harbor + chart_version: 1.14.2 + {{- if .Modules.Registry.Enabled }} + release_state: "present" + {{- else }} + release_state: "absent" + {{- end }} + values: + expose: + type: {{ .Modules.Registry.Expose.Type }} + tls: + enabled: {{ .Modules.Registry.Tls.Enabled }} + certSource: secret + secret: + secretName: harbor-tls + ingress: + hosts: + core: {{ .Modules.Registry.Expose.Domain }} + controller: default + kubeVersionOverride: "" + className: "{{ .Modules.Additional.Ingress.Type }}" + annotations: + ingress.kubernetes.io/ssl-redirect: "true" + ingress.kubernetes.io/proxy-body-size: "0" + {{- if eq .Modules.Additional.Ingress.Type "nginx" }} + nginx.ingress.kubernetes.io/ssl-redirect: "true" + nginx.ingress.kubernetes.io/proxy-body-size: "0" + {{- end }} + labels: {} + + nodePort: + name: harbor + ports: + http: + port: 80 + nodePort: {{ .Modules.Registry.Expose.NodePortHttp }} + https: + port: 443 + nodePort: {{ .Modules.Registry.Expose.NodePortHttps }} + + externalURL: {{ if .Modules.Registry.Tls.Enabled }}https{{ else }}http{{ end }}://{{ .Modules.Registry.Expose.Domain }} + persistence: + resourcePolicy: "keep" + persistentVolumeClaim: + registry: + existingClaim: "" + storageClass: "{{ .Modules.Registry.Persistence.StorageClass }}" + subPath: "" + accessMode: ReadWriteOnce + size: {{ .Modules.Registry.Persistence.RegistrySize }} + annotations: {} + jobservice: + jobLog: + existingClaim: "" + storageClass: "{{ .Modules.Registry.Persistence.StorageClass }}" + subPath: "" + accessMode: ReadWriteOnce + size: {{ .Modules.Registry.Persistence.JobserviceSize }} + annotations: {} + database: + existingClaim: "" + storageClass: "{{ .Modules.Registry.Persistence.StorageClass }}" + subPath: "" + accessMode: ReadWriteOnce + size: {{ .Modules.Registry.Persistence.DatabaseSize }} + annotations: {} + redis: + existingClaim: "" + storageClass: "{{ .Modules.Registry.Persistence.StorageClass }}" + subPath: "" + accessMode: ReadWriteOnce + size: {{ .Modules.Registry.Persistence.RedisSize }} + annotations: {} + trivy: + existingClaim: "" + storageClass: "{{ .Modules.Registry.Persistence.StorageClass }}" + subPath: "" + accessMode: ReadWriteOnce + size: {{ .Modules.Registry.Persistence.TrivySize }} + annotations: {} + + imageChartStorage: + disableredirect: false + + type: filesystem + filesystem: + rootdirectory: /storage + #maxthreads: 100 + + imagePullPolicy: IfNotPresent + + updateStrategy: + type: RollingUpdate + + harborAdminPassword: "{{ .Modules.Registry.AdminPassword }}" + + logLevel: info + + metrics: + enabled: {{ and .Modules.Observability.Enabled .Modules.Observability.Monitoring.Enabled }} + core: + path: /metrics + port: 8001 + registry: + path: /metrics + port: 8001 + jobservice: + path: /metrics + port: 8001 + exporter: + path: /metrics + port: 8001 + + serviceMonitor: + enabled: {{ and .Modules.Observability.Enabled .Modules.Observability.Monitoring.Enabled }} + + trace: + enabled: {{ and .Modules.Observability.Enabled .Modules.Observability.Tracing.Enabled }} + provider: otel + sample_rate: 1 + attributes: + application: harbor + jaeger: + endpoint: http://hostname:14268/api/traces + otel: + endpoint: observability-opentelemetry-collector-collector.observability.svc.{{ .Orchestrator.ClusterName }}:4318 + url_path: /v1/traces + compression: false + insecure: true + timeout: 10 + + portal: + image: + repository: goharbor/harbor-portal + tag: {{ .Modules.Registry.Version }} + serviceAccountName: "" + automountServiceAccountToken: false + replicas: 1 + revisionHistoryLimit: 10 + + topologySpreadConstraints: [] + + podLabels: + "app.kubernetes.io/component": "harbor-portal" + priorityClassName: + + core: + image: + repository: goharbor/harbor-core + tag: {{ .Modules.Registry.Version }} + serviceAccountName: "" + automountServiceAccountToken: false + replicas: 1 + revisionHistoryLimit: 10 + startupProbe: + enabled: true + initialDelaySeconds: 10 + extraEnvVars: [] + nodeSelector: {} + tolerations: [] + affinity: {} + topologySpreadConstraints: [] + podLabels: + "app.kubernetes.io/component": "harbor-core" + serviceAnnotations: {} + priorityClassName: + configureUserSettings: + quotaUpdateProvider: db # Or redis + secret: "" + existingSecret: "" + secretName: "" + tokenKey: "" + + tokenCert: "" + + xsrfKey: "" + existingXsrfSecret: "" + existingXsrfSecretKey: CSRF_KEY + artifactPullAsyncFlushDuration: + gdpr: + deleteUser: false + auditLogsCompliant: false + + + jobservice: + image: + repository: goharbor/harbor-jobservice + tag: {{ .Modules.Registry.Version }} + serviceAccountName: "" + automountServiceAccountToken: false + replicas: 1 + revisionHistoryLimit: 10 + topologySpreadConstraints: + podLabels: + "app.kubernetes.io/component": "harbor-jobservice" + priorityClassName: + maxJobWorkers: 10 + jobLoggers: + - file + # - database + # - stdout + loggerSweeperDuration: 14 #days + notification: + webhook_job_max_retry: 3 + webhook_job_http_client_timeout: 3 # in seconds + reaper: + max_update_hours: 24 + max_dangling_hours: 168 + secret: "" + existingSecret: "" + existingSecretKey: JOBSERVICE_SECRET + + registry: + registry: + image: + repository: goharbor/registry-photon + tag: {{ .Modules.Registry.Version }} + extraEnvVars: [] + controller: + image: + repository: goharbor/harbor-registryctl + tag: {{ .Modules.Registry.Version }} + extraEnvVars: [] + serviceAccountName: "" + automountServiceAccountToken: false + replicas: 1 + revisionHistoryLimit: 10 + topologySpreadConstraints: [] + podLabels: + "app.kubernetes.io/component": "harbor-registry" + priorityClassName: + secret: "" + existingSecret: "" + existingSecretKey: REGISTRY_HTTP_SECRET + relativeurls: false + credentials: + # If using existingSecret, the key must be REGISTRY_PASSWD and REGISTRY_HTPASSWD + existingSecret: "" + # Login and password in htpasswd string format. Excludes `registry.credentials.username` and `registry.credentials.password`. May come in handy when integrating with tools like argocd or flux. This allows the same line to be generated each time the template is rendered, instead of the `htpasswd` function from helm, which generates different lines each time because of the salt. + # htpasswdString: $apr1$XLefHzeG$Xl4.s00sMSCCcMyJljSZb0 # example string + # htpasswdString: "" + middleware: + enabled: false + type: cloudFront + cloudFront: + baseurl: example.cloudfront.net + keypairid: KEYPAIRID + duration: 3000s + ipfilteredby: none + # The secret key that should be present is CLOUDFRONT_KEY_DATA, which should be the encoded private key + # that allows access to CloudFront + privateKeySecret: "my-secret" + # enable purge _upload directories + upload_purging: + enabled: true + # remove files in _upload directories which exist for a period of time, default is one week. + age: 168h + # the interval of the purge operations + interval: 24h + dryrun: false + + trivy: + enabled: {{ .Modules.Registry.EnabledScanner }} + image: + repository: goharbor/trivy-adapter-photon + tag: {{ .Modules.Registry.Version }} + serviceAccountName: "" + automountServiceAccountToken: false + resources: + requests: + cpu: 200m + memory: 512Mi + limits: + cpu: 1 + memory: 1Gi + + + database: + # if external database is used, set "type" to "external" + # and fill the connection information in "external" section + type: internal + internal: + image: + repository: goharbor/harbor-db + tag: {{ .Modules.Registry.Version }} + serviceAccountName: "" + automountServiceAccountToken: false + livenessProbe: + timeoutSeconds: 1 + readinessProbe: + timeoutSeconds: 1 + priorityClassName: + # The initial superuser password for internal database + # password: "changeit" + # The size limit for Shared memory, pgSQL use it for shared_buffer + # More details see: + # https://github.com/goharbor/harbor/issues/15034 + shmSizeLimit: 512Mi + initContainer: + migrator: {} + # resources: + # requests: + # memory: 128Mi + # cpu: 100m + permissions: {} + # resources: + # requests: + # memory: 128Mi + # cpu: 100m + external: + host: "192.168.0.1" + port: "5432" + username: "user" + password: "password" + coreDatabase: "registry" + # if using existing secret, the key must be "password" + existingSecret: "" + # "disable" - No SSL + # "require" - Always SSL (skip verification) + # "verify-ca" - Always SSL (verify that the certificate presented by the + # server was signed by a trusted CA) + # "verify-full" - Always SSL (verify that the certification presented by the + # server was signed by a trusted CA and the server host name matches the one + # in the certificate) + sslmode: "disable" + # The maximum number of connections in the idle connection pool per pod (core+exporter). + # If it <=0, no idle connections are retained. + maxIdleConns: 100 + # The maximum number of open connections to the database per pod (core+exporter). + # If it <= 0, then there is no limit on the number of open connections. + # Note: the default number of connections is 1024 for postgre of harbor. + maxOpenConns: 900 + ## Additional deployment annotations + podAnnotations: {} + ## Additional deployment labels + podLabels: {} + + + redis: + type: internal + internal: + image: + repository: goharbor/redis-photon + tag: {{ .Modules.Registry.Version }} + serviceAccountName: "" + automountServiceAccountToken: false + extraEnvVars: [] + nodeSelector: {} + tolerations: [] + affinity: {} + priorityClassName: + jobserviceDatabaseIndex: "1" + registryDatabaseIndex: "2" + trivyAdapterIndex: "5" + # harborDatabaseIndex: "6" + # cacheLayerDatabaseIndex: "7" + external: + # support redis, redis+sentinel + # addr for redis: : + # addr for redis+sentinel: :,:,: + addr: "192.168.0.2:6379" + # The name of the set of Redis instances to monitor, it must be set to support redis+sentinel + sentinelMasterSet: "" + # The "coreDatabaseIndex" must be "0" as the library Harbor + # used doesn't support configuring it + # harborDatabaseIndex defaults to "0", but it can be configured to "6", this config is optional + # cacheLayerDatabaseIndex defaults to "0", but it can be configured to "7", this config is optional + coreDatabaseIndex: "0" + jobserviceDatabaseIndex: "1" + registryDatabaseIndex: "2" + trivyAdapterIndex: "5" + # harborDatabaseIndex: "6" + # cacheLayerDatabaseIndex: "7" + # username field can be an empty string, and it will be authenticated against the default user + username: "" + password: "" + existingSecret: "" + podAnnotations: {} + podLabels: {} diff --git a/pkg/resources/templates/helm-apps/releases/secrets-storage/vault.yml.tmpl b/pkg/resources/templates/helm-apps/releases/secrets-storage/vault.yml.tmpl new file mode 100644 index 0000000..787ce09 --- /dev/null +++ b/pkg/resources/templates/helm-apps/releases/secrets-storage/vault.yml.tmpl @@ -0,0 +1,883 @@ +- name: vault + namespace: secrets-storage + create_namespace: true + chart_ref: kube-forge/vault + chart_version: 0.1.0 + {{- if .Modules.SecretsStorage.Enabled }} + release_state: "present" + {{- else }} + release_state: "absent" + {{- end }} + values: + global: + enabled: true + + imagePullSecrets: [] + tlsDisable: true + + externalVaultAddr: "" + + openshift: false + + # Create PodSecurityPolicy for pods + psp: + enable: false + # Annotation for PodSecurityPolicy. + # This is a multi-line templated string map, and can also be set as YAML. + annotations: | + seccomp.security.alpha.kubernetes.io/allowedProfileNames: docker/default,runtime/default + apparmor.security.beta.kubernetes.io/allowedProfileNames: runtime/default + seccomp.security.alpha.kubernetes.io/defaultProfileName: runtime/default + apparmor.security.beta.kubernetes.io/defaultProfileName: runtime/default + + serverTelemetry: + # Enable integration with the Prometheus Operator + # See the top level serverTelemetry section below before enabling this feature. + prometheusOperator: false + + injector: + enabled: true + + replicas: 1 + + # Configures the port the injector should listen on + port: 8080 + + # If multiple replicas are specified, by default a leader will be determined + # so that only one injector attempts to create TLS certificates. + leaderElector: + enabled: true + + # If true, will enable a node exporter metrics endpoint at /metrics. + metrics: + enabled: false + + # Deprecated: Please use global.externalVaultAddr instead. + externalVaultAddr: "" + + # image sets the repo and tag of the vault-k8s image to use for the injector. + image: + repository: "{{ .Modules.SecretsStorage.Injector.Image }}" + tag: "{{ .Modules.SecretsStorage.Injector.Tag }}" + pullPolicy: IfNotPresent + + # agentImage sets the repo and tag of the Vault image to use for the Vault Agent + # containers. This should be set to the official Vault image. Vault 1.3.1+ is + # required. + agentImage: + repository: "{{ .Modules.SecretsStorage.Agent.Image }}" + tag: "{{ .Modules.SecretsStorage.Agent.Tag }}" + agentDefaults: + cpuLimit: "500m" + cpuRequest: "250m" + memLimit: "128Mi" + memRequest: "64Mi" + # ephemeralLimit: "128Mi" + # ephemeralRequest: "64Mi" + + # Default template type for secrets when no custom template is specified. + # Possible values include: "json" and "map". + template: "map" + + # Default values within Agent's template_config stanza. + templateConfig: + exitOnRetryFailure: true + staticSecretRenderInterval: "" + + # Used to define custom livenessProbe settings + livenessProbe: + # When a probe fails, Kubernetes will try failureThreshold times before giving up + failureThreshold: 2 + # Number of seconds after the container has started before probe initiates + initialDelaySeconds: 5 + # How often (in seconds) to perform the probe + periodSeconds: 2 + # Minimum consecutive successes for the probe to be considered successful after having failed + successThreshold: 1 + # Number of seconds after which the probe times out. + timeoutSeconds: 5 + # Used to define custom readinessProbe settings + readinessProbe: + # When a probe fails, Kubernetes will try failureThreshold times before giving up + failureThreshold: 2 + # Number of seconds after the container has started before probe initiates + initialDelaySeconds: 5 + # How often (in seconds) to perform the probe + periodSeconds: 2 + # Minimum consecutive successes for the probe to be considered successful after having failed + successThreshold: 1 + # Number of seconds after which the probe times out. + timeoutSeconds: 5 + # Used to define custom startupProbe settings + startupProbe: + # When a probe fails, Kubernetes will try failureThreshold times before giving up + failureThreshold: 12 + # Number of seconds after the container has started before probe initiates + initialDelaySeconds: 5 + # How often (in seconds) to perform the probe + periodSeconds: 5 + # Minimum consecutive successes for the probe to be considered successful after having failed + successThreshold: 1 + # Number of seconds after which the probe times out. + timeoutSeconds: 5 + + # Mount Path of the Vault Kubernetes Auth Method. + authPath: "auth/kubernetes" + + # Configures the log verbosity of the injector. + # Supported log levels include: trace, debug, info, warn, error + logLevel: "info" + + # Configures the log format of the injector. Supported log formats: "standard", "json". + logFormat: "standard" + + # Configures all Vault Agent sidecars to revoke their token when shutting down + revokeOnShutdown: false + + webhook: + # Configures failurePolicy of the webhook. The "unspecified" default behaviour depends on the + # API Tag of the WebHook. + # To block pod creation while the webhook is unavailable, set the policy to `Fail` below. + # See https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy + # + failurePolicy: Ignore + + # matchPolicy specifies the approach to accepting changes based on the rules of + # the MutatingWebhookConfiguration. + # See https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-matchpolicy + # for more details. + # + matchPolicy: Exact + + # timeoutSeconds is the amount of seconds before the webhook request will be ignored + # or fails. + # If it is ignored or fails depends on the failurePolicy + # See https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#timeouts + # for more details. + # + timeoutSeconds: 30 + + # namespaceSelector is the selector for restricting the webhook to only + # specific namespaces. + # See https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-namespaceselector + # for more details. + # Example: + # namespaceSelector: + # matchLabels: + # sidecar-injector: enabled + namespaceSelector: {} + + # objectSelector is the selector for restricting the webhook to only + # specific labels. + # See https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector + # for more details. + # Example: + # objectSelector: + # matchLabels: + # vault-sidecar-injector: enabled + + # Extra annotations to attach to the webhook + annotations: {} + + # Deprecated: please use 'webhook.failurePolicy' instead + # Configures failurePolicy of the webhook. The "unspecified" default behaviour depends on the + # API Tag of the WebHook. + # To block pod creation while webhook is unavailable, set the policy to `Fail` below. + # See https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy + # + failurePolicy: Ignore + + # Deprecated: please use 'webhook.namespaceSelector' instead + # namespaceSelector is the selector for restricting the webhook to only + # specific namespaces. + # See https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-namespaceselector + # for more details. + # Example: + # namespaceSelector: + # matchLabels: + # sidecar-injector: enabled + namespaceSelector: {} + + # Deprecated: please use 'webhook.objectSelector' instead + # objectSelector is the selector for restricting the webhook to only + # specific labels. + # See https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#matching-requests-objectselector + # for more details. + # Example: + # objectSelector: + # matchLabels: + # vault-sidecar-injector: enabled + objectSelector: {} + + # Deprecated: please use 'webhook.annotations' instead + # Extra annotations to attach to the webhook + webhookAnnotations: {} + + certs: + # secretName is the name of the secret that has the TLS certificate and + # private key to serve the injector webhook. If this is null, then the + # injector will default to its automatic management mode that will assign + # a service account to the injector to generate its own certificates. + secretName: null + + # caBundle is a base64-encoded PEM-encoded certificate bundle for the CA + # that signed the TLS certificate that the webhook serves. This must be set + # if secretName is non-null unless an external service like cert-manager is + # keeping the caBundle updated. + caBundle: "" + + # certName and keyName are the names of the files within the secret for + # the TLS cert and private key, respectively. These have reasonable + # defaults but can be customized if necessary. + certName: tls.crt + keyName: tls.key + + securityContext: + pod: {} + container: {} + + resources: {} + + # extraEnvironmentVars is a list of extra environment variables to set in the + # injector deployment. + extraEnvironmentVars: {} + # KUBERNETES_SERVICE_HOST: kubernetes.default.svc + + topologySpreadConstraints: [] + + tolerations: [] + + nodeSelector: {} + + priorityClassName: "" + + annotations: {} + + extraLabels: {} + + hostNetwork: false + + + service: + # Extra annotations to attach to the injector service + annotations: {} + + # Injector serviceAccount specific config + serviceAccount: + # Extra annotations to attach to the injector serviceAccount + annotations: {} + + # A disruption budget limits the number of pods of a replicated application + # that are down simultaneously from voluntary disruptions + podDisruptionBudget: {} + # podDisruptionBudget: + # maxUnavailable: 1 + + # strategy for updating the deployment. This can be a multi-line string or a + # YAML map. + strategy: {} + # strategy: | + # rollingUpdate: + # maxSurge: 25% + # maxUnavailable: 25% + # type: RollingUpdate + + server: + enabled: true + enterpriseLicense: + # The name of the Kubernetes secret that holds the enterprise license. The + # secret must be in the same namespace that Vault is installed into. + secretName: "" + # The key within the Kubernetes secret that holds the enterprise license. + secretKey: "license" + + image: + repository: "{{ .Modules.SecretsStorage.Server.Image }}" + tag: "{{ .Modules.SecretsStorage.Server.Tag }}" + # Overrides the default Image Pull Policy + pullPolicy: IfNotPresent + + updateStrategyType: "RollingUpdate" + + # Supported log levels include: trace, debug, info, warn, error + logLevel: "" + + # Supported log formats include: standard, json + logFormat: "" + + resources: {} + + hostAliases: [] + # - ip: 127.0.0.1 + # hostnames: + # - chart-example.local + + route: + enabled: false + + # When HA mode is enabled and K8s service registration is being used, + # configure the route to point to the Vault active service. + activeService: true + + labels: {} + annotations: {} + host: chart-example.local + # tls will be passed directly to the route's TLS config, which + # can be used to configure other termination methods that terminate + # TLS at the router + tls: + termination: passthrough + + # authDelegator enables a cluster role binding to be attached to the service + # account. This cluster role binding can be used to setup Kubernetes auth + # method. See https://developer.hashicorp.com/vault/docs/auth/kubernetes + authDelegator: + enabled: true + + extraInitContainers: null + extraContainers: null + shareProcessNamespace: false + extraArgs: "" + + extraPorts: null + # - containerPort: 8300 + # name: http-monitoring + + readinessProbe: + enabled: true + # If you need to use a http path instead of the default exec + # path: /v1/sys/health?standbyok=true + + # Port number on which readinessProbe will be checked. + port: 8200 + # When a probe fails, Kubernetes will try failureThreshold times before giving up + failureThreshold: 2 + # Number of seconds after the container has started before probe initiates + initialDelaySeconds: 5 + # How often (in seconds) to perform the probe + periodSeconds: 5 + # Minimum consecutive successes for the probe to be considered successful after having failed + successThreshold: 1 + # Number of seconds after which the probe times out. + timeoutSeconds: 3 + # Used to enable a livenessProbe for the pods + livenessProbe: + enabled: false + # Used to define a liveness exec command. If provided, exec is preferred to httpGet (path) as the livenessProbe handler. + execCommand: [] + # - /bin/sh + # - -c + # - /vault/userconfig/mylivenessscript/run.sh + # Path for the livenessProbe to use httpGet as the livenessProbe handler + path: "/v1/sys/health?standbyok=true" + # Port number on which livenessProbe will be checked if httpGet is used as the livenessProbe handler + port: 8200 + # When a probe fails, Kubernetes will try failureThreshold times before giving up + failureThreshold: 2 + # Number of seconds after the container has started before probe initiates + initialDelaySeconds: 60 + # How often (in seconds) to perform the probe + periodSeconds: 5 + # Minimum consecutive successes for the probe to be considered successful after having failed + successThreshold: 1 + # Number of seconds after which the probe times out. + timeoutSeconds: 3 + + terminationGracePeriodSeconds: 10 + + # Used to set the sleep time during the preStop step + preStopSleepSeconds: 5 + + postStart: + # - /bin/sh + # - -c + # - /vault/userconfig/myscript/run.sh + + extraEnvironmentVars: {} + + extraSecretEnvironmentVars: [] + + extraVolumes: [] + + volumes: null + + volumeMounts: null + + topologySpreadConstraints: [] + + tolerations: [] + nodeSelector: {} + + # Enables network policy for server pods + networkPolicy: + enabled: false + egress: [] + # egress: + # - to: + # - ipBlock: + # cidr: 10.0.0.0/24 + # ports: + # - protocol: TCP + # port: 443 + ingress: + - from: + - namespaceSelector: {} + ports: + - port: 8200 + protocol: TCP + - port: 8201 + protocol: TCP + + priorityClassName: "" + extraLabels: {} + + annotations: {} + + service: + enabled: true + # Enable or disable the vault-active service, which selects Vault pods that + # have labeled themselves as the cluster leader with `vault-active: "true"`. + active: + enabled: true + # Extra annotations for the service definition. This can either be YAML or a + # YAML-formatted multi-line templated string map of the annotations to apply + # to the active service. + annotations: {} + # Enable or disable the vault-standby service, which selects Vault pods that + # have labeled themselves as a cluster follower with `vault-active: "false"`. + standby: + enabled: true + # Extra annotations for the service definition. This can either be YAML or a + # YAML-formatted multi-line templated string map of the annotations to apply + # to the standby service. + annotations: {} + # When disabled, services may select Vault pods not deployed from the chart. + # Does not affect the headless vault-internal service with `ClusterIP: None` + instanceSelector: + enabled: true + # clusterIP controls whether a Cluster IP address is attached to the + # Vault service within Kubernetes. By default, the Vault service will + # be given a Cluster IP address, set to None to disable. When disabled + # Kubernetes will create a "headless" service. Headless services can be + # used to communicate with pods directly through DNS instead of a round-robin + # load balancer. + # clusterIP: None + + # Configures the service type for the main Vault service. Can be ClusterIP + # or NodePort. + #type: ClusterIP + + # The IP family and IP families options are to set the behaviour in a dual-stack environment. + # Omitting these values will let the service fall back to whatever the CNI dictates the defaults + # should be. + # These are only supported for kubernetes versions >=1.23.0 + # + # Configures the service's supported IP family policy, can be either: + # SingleStack: Single-stack service. The control plane allocates a cluster IP for the Service, using the first configured service cluster IP range. + # PreferDualStack: Allocates IPv4 and IPv6 cluster IPs for the Service. + # RequireDualStack: Allocates Service .spec.ClusterIPs from both IPv4 and IPv6 address ranges. + ipFamilyPolicy: "" + + # Sets the families that should be supported and the order in which they should be applied to ClusterIP as well. + # Can be IPv4 and/or IPv6. + ipFamilies: [] + + # Do not wait for pods to be ready before including them in the services' + # targets. Does not apply to the headless service, which is used for + # cluster-internal communication. + publishNotReadyAddresses: true + + # The externalTrafficPolicy can be set to either Cluster or Local + # and is only valid for LoadBalancer and NodePort service types. + # The default value is Cluster. + # ref: https://kubernetes.io/docs/concepts/services-networking/service/#external-traffic-policy + externalTrafficPolicy: Cluster + + # If type is set to "NodePort", a specific nodePort value can be configured, + # will be random if left blank. + #nodePort: 30000 + + # When HA mode is enabled + # If type is set to "NodePort", a specific nodePort value can be configured, + # will be random if left blank. + #activeNodePort: 30001 + + # When HA mode is enabled + # If type is set to "NodePort", a specific nodePort value can be configured, + # will be random if left blank. + #standbyNodePort: 30002 + + # Port on which Vault server is listening + port: 8200 + # Target port to which the service should be mapped to + targetPort: 8200 + # Extra annotations for the service definition. This can either be YAML or a + # YAML-formatted multi-line templated string map of the annotations to apply + # to the service. + annotations: {} + + dataStorage: + enabled: true + size: {{ .Modules.SecretsStorage.Server.Persistence.DataStorage.Size }} + mountPath: "/vault/data" + storageClass: {{ .Modules.SecretsStorage.Server.Persistence.DataStorage.StorageClass }} + accessMode: ReadWriteOnce + annotations: {} + labels: {} + + persistentVolumeClaimRetentionPolicy: {} + + # required for ha installation + auditStorage: + enabled: false + # Size of the PVC created + size: {{ .Modules.SecretsStorage.Server.Persistence.AuditStorage.Size }} + # Location where the PVC will be mounted. + mountPath: "/vault/audit" + # Name of the storage class to use. If null it will use the + # configured default Storage Class. + storageClass: {{ .Modules.SecretsStorage.Server.Persistence.AuditStorage.StorageClass }} + # Access Mode of the storage device being used for the PVC + accessMode: ReadWriteOnce + # Annotations to apply to the PVC + annotations: {} + # Labels to apply to the PVC + labels: {} + + dev: + enabled: false + + # Set VAULT_DEV_ROOT_TOKEN_ID value + devRootToken: "root" + + # Run Vault in "standalone" mode. This is the default mode that will deploy if + # no arguments are given to helm. This requires a PVC for data storage to use + # the "file" backend. This mode is not highly available and should not be scaled + # past a single replica. + standalone: + enabled: "-" + + # config is a raw string of default configuration when using a Stateful + # deployment. Default is to use a PersistentVolumeClaim mounted at /vault/data + # and store data there. This is only used when using a Replica count of 1, and + # using a stateful set. This should be HCL. + + # Note: Configuration files are stored in ConfigMaps so sensitive data + # such as passwords should be either mounted through extraSecretEnvironmentVars + # or through a Kube secret. For more information see: + # https://developer.hashicorp.com/vault/docs/platform/k8s/helm/run#protecting-sensitive-vault-configurations + config: | + ui = true + + listener "tcp" { + tls_disable = 1 + address = "[::]:8200" + cluster_address = "[::]:8201" + + {{- if and .Modules.Observability.Enabled .Modules.Observability.Monitoring.Enabled }} + telemetry { + unauthenticated_metrics_access = "true" + } + {{- end }} + } + storage "file" { + path = "/vault/data" + } + + {{- if and .Modules.Observability.Enabled .Modules.Observability.Monitoring.Enabled }} + telemetry { + prometheus_retention_time = "30s" + disable_hostname = true + } + {{- end }} + + # Run Vault in "HA" mode. There are no storage requirements unless the audit log + # persistence is required. In HA mode Vault will configure itself to use Consul + # for its storage backend. The default configuration provided will work the Consul + # Helm project by default. It is possible to manually configure Vault to use a + # different HA backend. + ha: + enabled: false + replicas: 3 + + # Set the api_addr configuration for Vault HA + # See https://developer.hashicorp.com/vault/docs/configuration#api_addr + # If set to null, this will be set to the Pod IP Address + apiAddr: null + + # Set the cluster_addr confuguration for Vault HA + # See https://developer.hashicorp.com/vault/docs/configuration#cluster_addr + clusterAddr: null + + # Enables Vault's integrated Raft storage. Unlike the typical HA modes where + # Vault's persistence is external (such as Consul), enabling Raft mode will create + # persistent volumes for Vault to store data according to the configuration under server.dataStorage. + # The Vault cluster will coordinate leader elections and failovers internally. + raft: + # Enables Raft integrated storage + enabled: false + # Set the Node Raft ID to the name of the pod + setNodeId: false + + config: | + ui = true + + listener "tcp" { + tls_disable = 1 + address = "[::]:8200" + cluster_address = "[::]:8201" + + {{- if and .Modules.Observability.Enabled .Modules.Observability.Monitoring.Enabled }} + telemetry { + unauthenticated_metrics_access = "true" + } + {{- end }} + } + + storage "raft" { + path = "/vault/data" + } + + service_registration "kubernetes" {} + + # config is a raw string of default configuration when using a Stateful + # deployment. Default is to use a Consul for its HA storage backend. + # This should be HCL. + + # Note: Configuration files are stored in ConfigMaps so sensitive data + # such as passwords should be either mounted through extraSecretEnvironmentVars + # or through a Kube secret. For more information see: + # https://developer.hashicorp.com/vault/docs/platform/k8s/helm/run#protecting-sensitive-vault-configurations + config: | + ui = true + + listener "tcp" { + tls_disable = 1 + address = "[::]:8200" + cluster_address = "[::]:8201" + {{- if and .Modules.Observability.Enabled .Modules.Observability.Monitoring.Enabled }} + telemetry { + unauthenticated_metrics_access = "true" + } + {{- end }} + } + storage "consul" { + path = "vault" + address = "HOST_IP:8500" + } + + service_registration "kubernetes" {} + + # Example configuration for using auto-unseal, using Google Cloud KMS. The + # GKMS keys must already exist, and the cluster must have a service account + # that is authorized to access GCP KMS. + #seal "gcpckms" { + # project = "vault-helm-dev-246514" + # region = "global" + # key_ring = "vault-helm-unseal-kr" + # crypto_key = "vault-helm-unseal-key" + #} + + {{- if and .Modules.Observability.Enabled .Modules.Observability.Monitoring.Enabled }} + telemetry { + prometheus_retention_time = "30s" + disable_hostname = true + } + {{- end }} + + # A disruption budget limits the number of pods of a replicated application + # that are down simultaneously from voluntary disruptions + disruptionBudget: + enabled: true + + # maxUnavailable will default to (n/2)-1 where n is the number of + # replicas. If you'd like a custom value, you can specify an override here. + maxUnavailable: null + + serviceAccount: + create: true + name: "" + createSecret: false + annotations: {} + extraLabels: {} + serviceDiscovery: + enabled: true + + statefulSet: + annotations: {} + securityContext: + pod: {} + container: {} + + hostNetwork: false + + # Vault UI + ui: + enabled: true + domain: {{ .Modules.SecretsStorage.Expose.Domain }} + publishNotReadyAddresses: true + # The service should only contain selectors for active Vault pod + activeVaultPodOnly: false + {{- if eq .Modules.SecretsStorage.Expose.Type "NodePort" }} + serviceType: "NodePort" + serviceNodePort: {{ .Modules.SecretsStorage.Expose.NodePort }} + {{- else }} + serviceType: "ClusterIP" + serviceNodePort: null + {{- end }} + externalPort: 8200 + targetPort: 8200 + + serviceIPFamilyPolicy: "" + + serviceIPFamilies: [] + + externalTrafficPolicy: Cluster + + #loadBalancerSourceRanges: + # - 10.0.0.0/16 + # - 1.78.23.3/32 + + # loadBalancerIP: + + annotations: {} + + csi: + # True if you want to install a secrets-store-csi-driver-provider-vault daemonset. + # + # Requires installing the secrets-store-csi-driver separately, see: + # https://github.com/kubernetes-sigs/secrets-store-csi-driver#install-the-secrets-store-csi-driver + # + # With the driver and provider installed, you can mount Vault secrets into volumes + # similar to the Vault Agent injector, and you can also sync those secrets into + # Kubernetes secrets. + enabled: {{ .Modules.SecretsStorage.CsiIntegration.Enabled }} + + image: + repository: "{{ .Modules.SecretsStorage.CsiIntegration.Image }}" + tag: "{{ .Modules.SecretsStorage.CsiIntegration.Tag }}" + pullPolicy: IfNotPresent + + volumes: null + + volumeMounts: null + + resources: {} + + # Override the default secret name for the CSI Provider's HMAC key used for + # generating secret versions. + hmacSecretName: "" + + daemonSet: + updateStrategy: + type: RollingUpdate + maxUnavailable: "" + # Extra annotations for the daemonSet. This can either be YAML or a + # YAML-formatted multi-line templated string map of the annotations to apply + # to the daemonSet. + annotations: {} + # Provider host path (must match the CSI provider's path) + providersDir: "/etc/kubernetes/secrets-store-csi-providers" + # Kubelet host path + kubeletRootDir: "/var/lib/kubelet" + # Extra labels to attach to the vault-csi-provider daemonSet + # This should be a YAML map of the labels to apply to the csi provider daemonSet + extraLabels: {} + # security context for the pod template and container in the csi provider daemonSet + securityContext: + pod: {} + container: {} + + pod: + annotations: {} + tolerations: [] + nodeSelector: {} + affinity: {} + extraLabels: {} + + agent: + enabled: true + extraArgs: [] + + image: + repository: "{{ .Modules.SecretsStorage.Agent.Image }}" + tag: "{{ .Modules.SecretsStorage.Agent.Tag }}" + pullPolicy: IfNotPresent + + logFormat: standard + logLevel: info + + resources: {} + + priorityClassName: "" + + serviceAccount: + annotations: {} + extraLabels: {} + + readinessProbe: + failureThreshold: 2 + initialDelaySeconds: 5 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 3 + + livenessProbe: + failureThreshold: 2 + initialDelaySeconds: 5 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 3 + + debug: false + extraArgs: [] + + serverTelemetry: + # Enable support for the Prometheus Operator. Currently, this chart does not support + # authenticating to Vault's metrics endpoint, so the following `telemetry{}` must be included + # in the `listener "tcp"{}` stanza + # telemetry { + # unauthenticated_metrics_access = "true" + # } + # + # See the `standalone.config` for a more complete example of this. + # + # In addition, a top level `telemetry{}` stanza must also be included in the Vault configuration: + # + # example: + # telemetry { + # prometheus_retention_time = "30s" + # disable_hostname = true + # } + # + # Configuration for monitoring the Vault server. + serviceMonitor: + enabled: {{ and .Modules.Observability.Enabled .Modules.Observability.Monitoring.Enabled }} + selectors: {} + interval: 30s + scrapeTimeout: 10s + + prometheusRules: + enabled: {{ and .Modules.Observability.Enabled .Modules.Observability.Monitoring.Enabled }} + selectors: {} + rules: [] + + ingress: + {{- if eq .Modules.SecretsStorage.Expose.Type "ingress" }} + enabled: true + {{- end }} + accountEmail: {{ .Modules.Additional.CertManager.AccountEmail }} + class: {{ .Modules.Additional.Ingress.Type }} + annotations: + {{- if eq .Modules.Additional.Ingress.Type "nginx" }} + nginx.ingress.kubernetes.io/proxy-buffer-size: "128k" + nginx.ingress.kubernetes.io/proxy-buffers: "4 256k" + nginx.ingress.kubernetes.io/proxy-busy-buffers-size: "256k" + {{- end }} + tls: + {{- if .Modules.SecretsStorage.Expose.Tls.Enabled }} + enabled: true + {{- end }} + hosts: + - host: {{ .Modules.SecretsStorage.Expose.Domain }} + secretName: vault-tls diff --git a/pkg/resources/templates/helm-apps/repositories/repositories.yml.tmpl b/pkg/resources/templates/helm-apps/repositories/repositories.yml.tmpl new file mode 100644 index 0000000..a2c16d0 --- /dev/null +++ b/pkg/resources/templates/helm-apps/repositories/repositories.yml.tmpl @@ -0,0 +1,5 @@ +- name: kube-forge + url: "https://git.kvazaric.ru/api/v4/projects/41/packages/helm/stable" +{{ if .Modules.AdditionalRepositories }} +{{ .Modules.AdditionalRepositories | toYaml }} +{{- end }} \ No newline at end of file diff --git a/pkg/resources/templates/kubespray/inventory/group_vars/all.yml.tmpl b/pkg/resources/templates/kubespray/inventory/group_vars/all.yml.tmpl new file mode 100644 index 0000000..9026b7f --- /dev/null +++ b/pkg/resources/templates/kubespray/inventory/group_vars/all.yml.tmpl @@ -0,0 +1,136 @@ +--- +bin_dir: {{ .Orchestrator.BinDir }} + +## The access_ip variable is used to define how other nodes should access +## the node. This is used in flannel to allow other flannel nodes to see +## this node for example. The access_ip is really useful AWS and Google +## environments where the nodes are accessed remotely by the "public" ip, +## but don't know about that address themselves. +# access_ip: 1.1.1.1 + + +## External LB example config +## apiserver_loadbalancer_domain_name: "elb.some.domain" +# loadbalancer_apiserver: +# address: 1.2.3.4 +# port: 1234 + +## Internal loadbalancers for apiservers +# loadbalancer_apiserver_localhost: true +# valid options are "nginx" or "haproxy" +# loadbalancer_apiserver_type: nginx # valid values "nginx" or "haproxy" + +## Local loadbalancer should use this port +## And must be set port 6443 +loadbalancer_apiserver_port: {{ .Orchestrator.LoadbalancerApiserverPort }} + +## If loadbalancer_apiserver_healthcheck_port variable defined, enables proxy liveness check for nginx. +loadbalancer_apiserver_healthcheck_port: 8081 + +disable_host_nameservers: {{ .Orchestrator.Dns.DisableHostNameservers }} + +upstream_dns_servers: +{{- range .Orchestrator.Dns.Servers }} + - {{ . }} +{{- end }} + +## There are some changes specific to the cloud providers +## for instance we need to encapsulate packets with some network plugins +## If set the possible values are either 'gce', 'aws', 'azure', 'openstack', 'vsphere', 'oci', or 'external' +## When openstack is used make sure to source in the openstack credentials +## like you would do when using openstack-client before starting the playbook. +{{- if not (eq .Orchestrator.CloudProvider "") }} +cloud_provider: {{ .Orchestrator.CloudProvider }} +{{- end }} + +## When cloud_provider is set to 'external', you can set the cloud controller to deploy +## Supported cloud controllers are: 'openstack', 'vsphere' and 'hcloud' +## When openstack or vsphere are used make sure to source in the required fields +{{- if not (eq .Orchestrator.ExgernalCloudProvider "") }} +external_cloud_provider: {{ .Orchestrator.ExgernalCloudProvider }} +{{- end }} + +## Set these proxy values in order to update package manager and docker daemon to use proxies +# http_proxy: "" +# https_proxy: "" + +## Refer to roles/kubespray-defaults/defaults/main.yml before modifying no_proxy +# no_proxy: "" + +## Some problems may occur when downloading files over https proxy due to ansible bug +## https://github.com/ansible/ansible/issues/32750. Set this variable to False to disable +## SSL validation of get_url module. Note that kubespray will still be performing checksum validation. +# download_validate_certs: False + +## If you need exclude all cluster nodes from proxy and other resources, add other resources here. +# additional_no_proxy: "" + +## If you need to disable proxying of os package repositories but are still behind an http_proxy set +## skip_http_proxy_on_os_packages to true +## This will cause kubespray not to set proxy environment in /etc/yum.conf for centos and in /etc/apt/apt.conf for debian/ubuntu +## Special information for debian/ubuntu - you have to set the no_proxy variable, then apt package will install from your source of wish +# skip_http_proxy_on_os_packages: false + +## Since workers are included in the no_proxy variable by default, docker engine will be restarted on all nodes (all +## pods will restart) when adding or removing workers. To override this behaviour by only including master nodes in the +## no_proxy variable, set below to true: +no_proxy_exclude_workers: false + +## Certificate Management +## This setting determines whether certs are generated via scripts. +## Chose 'none' if you provide your own certificates. +## Option is "script", "none" +# cert_management: script + +## Set to true to allow pre-checks to fail and continue deployment +# ignore_assert_errors: false + +## The read-only port for the Kubelet to serve on with no authentication/authorization. Uncomment to enable. +# kube_read_only_port: 10255 + +## Set true to download and cache container +# download_container: true + +## Deploy container engine +# Set false if you want to deploy container engine manually. +deploy_container_engine: {{ .Orchestrator.ContainerEngine.Install }} + +## Red Hat Enterprise Linux subscription registration +## Add either RHEL subscription Username/Password or Organization ID/Activation Key combination +## Update RHEL subscription purpose usage, role and SLA if necessary +# rh_subscription_username: "" +# rh_subscription_password: "" +# rh_subscription_org_id: "" +# rh_subscription_activation_key: "" +# rh_subscription_usage: "Development" +# rh_subscription_role: "Red Hat Enterprise Server" +# rh_subscription_sla: "Self-Support" + +## Check if access_ip responds to ping. Set false if your firewall blocks ICMP. +ping_access_ip: {{ .Orchestrator.PingAccessIp }} + +# sysctl_file_path to add sysctl conf to +sysctl_file_path: {{ .Orchestrator.SysctlFilePath }} + +## Variables for webhook token auth https://kubernetes.io/docs/reference/access-authn-authz/authentication/#webhook-token-authentication +kube_webhook_token_auth: false +kube_webhook_token_auth_url_skip_tls_verify: false +# kube_webhook_token_auth_url: https://... +## base64-encoded string of the webhook's CA certificate +# kube_webhook_token_auth_ca_data: "LS0t..." + +## NTP Settings +# Start the ntpd or chrony service and enable it at system boot. +ntp_enabled: false +ntp_manage_config: false +ntp_servers: + - "0.pool.ntp.org iburst" + - "1.pool.ntp.org iburst" + - "2.pool.ntp.org iburst" + - "3.pool.ntp.org iburst" + +## Used to control no_log attribute +unsafe_show_logs: false + +## If enabled it will allow kubespray to attempt setup even if the distribution is not supported. For unsupported distributions this can lead to unexpected failures in some cases. +allow_unsupported_distribution_setup: false diff --git a/pkg/resources/templates/kubespray/inventory/group_vars/k8s_cluster/addons.yml.tmpl b/pkg/resources/templates/kubespray/inventory/group_vars/k8s_cluster/addons.yml.tmpl new file mode 100644 index 0000000..400248b --- /dev/null +++ b/pkg/resources/templates/kubespray/inventory/group_vars/k8s_cluster/addons.yml.tmpl @@ -0,0 +1,247 @@ +--- +dashboard_enabled: false + +helm_enabled: true + +registry_enabled: false + +metrics_server_enabled: false + +# Rancher Local Path Provisioner +local_path_provisioner_enabled: true +# local_path_provisioner_namespace: "local-path-storage" +local_path_provisioner_storage_class: {{ .Modules.Additional.Storage.LocalPathProvisioner.StorageClassName }} +local_path_provisioner_reclaim_policy: {{ .Modules.Additional.Storage.LocalPathProvisioner.ReclaimPolicy }} +# local_path_provisioner_claim_root: /opt/local-path-provisioner/ +# local_path_provisioner_debug: false +# local_path_provisioner_image_repo: "rancher/local-path-provisioner" +# local_path_provisioner_image_tag: "v0.0.23" +# local_path_provisioner_helper_image_repo: "busybox" +# local_path_provisioner_helper_image_tag: "latest" + +# Local volume provisioner deployment +local_volume_provisioner_enabled: false +# local_volume_provisioner_namespace: kube-system +# local_volume_provisioner_nodelabels: +# - kubernetes.io/hostname +# - topology.kubernetes.io/region +# - topology.kubernetes.io/zone +# local_volume_provisioner_storage_classes: +# local-storage: +# host_dir: /mnt/disks +# mount_dir: /mnt/disks +# volume_mode: Filesystem +# fs_type: ext4 +# fast-disks: +# host_dir: /mnt/fast-disks +# mount_dir: /mnt/fast-disks +# block_cleaner_command: +# - "/scripts/shred.sh" +# - "2" +# volume_mode: Filesystem +# fs_type: ext4 +# local_volume_provisioner_tolerations: +# - effect: NoSchedule +# operator: Exists + +# CSI Volume Snapshot Controller deployment, set this to true if your CSI is able to manage snapshots +# currently, setting cinder_csi_enabled=true would automatically enable the snapshot controller +# Longhorn is an extenal CSI that would also require setting this to true but it is not included in kubespray +# csi_snapshot_controller_enabled: false +# csi snapshot namespace +# snapshot_controller_namespace: kube-system + +# CephFS provisioner deployment +cephfs_provisioner_enabled: false +# cephfs_provisioner_namespace: "cephfs-provisioner" +# cephfs_provisioner_cluster: ceph +# cephfs_provisioner_monitors: "172.24.0.1:6789,172.24.0.2:6789,172.24.0.3:6789" +# cephfs_provisioner_admin_id: admin +# cephfs_provisioner_secret: secret +# cephfs_provisioner_storage_class: cephfs +# cephfs_provisioner_reclaim_policy: Delete +# cephfs_provisioner_claim_root: /volumes +# cephfs_provisioner_deterministic_names: true + +# RBD provisioner deployment +rbd_provisioner_enabled: false +# rbd_provisioner_namespace: rbd-provisioner +# rbd_provisioner_replicas: 2 +# rbd_provisioner_monitors: "172.24.0.1:6789,172.24.0.2:6789,172.24.0.3:6789" +# rbd_provisioner_pool: kube +# rbd_provisioner_admin_id: admin +# rbd_provisioner_secret_name: ceph-secret-admin +# rbd_provisioner_secret: ceph-key-admin +# rbd_provisioner_user_id: kube +# rbd_provisioner_user_secret_name: ceph-secret-user +# rbd_provisioner_user_secret: ceph-key-user +# rbd_provisioner_user_secret_namespace: rbd-provisioner +# rbd_provisioner_fs_type: ext4 +# rbd_provisioner_image_format: "2" +# rbd_provisioner_image_features: layering +# rbd_provisioner_storage_class: rbd +# rbd_provisioner_reclaim_policy: Delete + +# Nginx ingress controller deployment +{{- if eq .Modules.Additional.Ingress.Type "nginx" }} +ingress_nginx_enabled: {{ .Modules.Additional.Ingress.Install }} +ingress_nginx_host_network: {{ .Modules.Additional.Ingress.HostNetwork }} +ingress_publish_status_address: "" +# ingress_nginx_nodeselector: +# kubernetes.io/os: "linux" +# ingress_nginx_tolerations: +# - key: "node-role.kubernetes.io/master" +# operator: "Equal" +# value: "" +# effect: "NoSchedule" +# - key: "node-role.kubernetes.io/control-plane" +# operator: "Equal" +# value: "" +# effect: "NoSchedule" +# ingress_nginx_namespace: "ingress-nginx" +ingress_nginx_insecure_port: {{ .Modules.Additional.Ingress.InsecurePort }} +ingress_nginx_secure_port: {{ .Modules.Additional.Ingress.SecurePort }} +# ingress_nginx_configmap: +# map-hash-bucket-size: "128" +# ssl-protocols: "TLSv1.2 TLSv1.3" +# ingress_nginx_configmap_tcp_services: +# 9000: "default/example-go:8080" +# ingress_nginx_configmap_udp_services: +# 53: "kube-system/coredns:53" +# ingress_nginx_extra_args: +# - --default-ssl-certificate=default/foo-tls +# ingress_nginx_termination_grace_period_seconds: 300 +ingress_nginx_class: {{ .Modules.Additional.Ingress.IngressClassName }} +{{- end }} + +# ALB ingress controller deployment +ingress_alb_enabled: false +# alb_ingress_aws_region: "us-east-1" +# alb_ingress_restrict_scheme: "false" +# Enables logging on all outbound requests sent to the AWS API. +# If logging is desired, set to true. +# alb_ingress_aws_debug: "false" + +# Cert manager deployment +cert_manager_enabled: {{ .Modules.Additional.CertManager.Install }} +# cert_manager_namespace: "cert-manager" +# cert_manager_tolerations: +# - key: node-role.kubernetes.io/master +# effect: NoSchedule +# - key: node-role.kubernetes.io/control-plane +# effect: NoSchedule +# cert_manager_affinity: +# nodeAffinity: +# preferredDuringSchedulingIgnoredDuringExecution: +# - weight: 100 +# preference: +# matchExpressions: +# - key: node-role.kubernetes.io/control-plane +# operator: In +# values: +# - "" +# cert_manager_nodeselector: +# kubernetes.io/os: "linux" + +# cert_manager_trusted_internal_ca: | +# -----BEGIN CERTIFICATE----- +# [REPLACE with your CA certificate] +# -----END CERTIFICATE----- +# cert_manager_leader_election_namespace: kube-system + +# cert_manager_dns_policy: "ClusterFirst" +cert_manager_dns_config: + nameservers: + {{- range .Modules.Additional.CertManager.DnsServers }} + - "{{ . }}" + {{- end }} + +# MetalLB deployment +{{- if and (eq .Modules.Additional.LoadBalancer.Type "metallb") .Modules.Additional.LoadBalancer.Install }} +metallb_enabled: true +{{- else }} +metallb_enabled: false +{{- end }} +metallb_speaker_enabled: {{ `"{{ metallb_enabled }}"` }} +# metallb_speaker_nodeselector: +# kubernetes.io/os: "linux" +# metallb_controller_nodeselector: +# kubernetes.io/os: "linux" +# metallb_speaker_tolerations: +# - key: "node-role.kubernetes.io/master" +# operator: "Equal" +# value: "" +# effect: "NoSchedule" +# - key: "node-role.kubernetes.io/control-plane" +# operator: "Equal" +# value: "" +# effect: "NoSchedule" +# metallb_controller_tolerations: +# - key: "node-role.kubernetes.io/master" +# operator: "Equal" +# value: "" +# effect: "NoSchedule" +# - key: "node-role.kubernetes.io/control-plane" +# operator: "Equal" +# value: "" +# effect: "NoSchedule" +# metallb_version: v0.13.9 +# metallb_protocol: "layer2" +# metallb_port: "7472" +# metallb_memberlist_port: "7946" +# metallb_config: +# address_pools: +# primary: +# ip_range: +# - 10.5.0.0/16 +# auto_assign: true +# pool1: +# ip_range: +# - 10.6.0.0/16 +# auto_assign: true +# pool2: +# ip_range: +# - 10.10.0.0/16 +# auto_assign: true +# layer2: +# - primary +# layer3: +# defaults: +# peer_port: 179 +# hold_time: 120s +# communities: +# vpn-only: "1234:1" +# NO_ADVERTISE: "65535:65282" +# metallb_peers: +# peer1: +# peer_address: 10.6.0.1 +# peer_asn: 64512 +# my_asn: 4200000000 +# communities: +# - vpn-only +# address_pool: +# - pool1 +# peer2: +# peer_address: 10.10.0.1 +# peer_asn: 64513 +# my_asn: 4200000000 +# communities: +# - NO_ADVERTISE +# address_pool: +# - pool2 + +argocd_enabled: false + +# The plugin manager for kubectl +krew_enabled: false +krew_root_dir: "/usr/local/krew" + + +######################################## +# Helm apps configuration +######################################## +repositories: + {{- .Repositories | nindent 2 }} + +releases: + {{- .Releases | nindent 2 }} diff --git a/pkg/resources/templates/kubespray/inventory/group_vars/k8s_cluster/k8s-cluster.yml.tmpl b/pkg/resources/templates/kubespray/inventory/group_vars/k8s_cluster/k8s-cluster.yml.tmpl new file mode 100644 index 0000000..fea46b2 --- /dev/null +++ b/pkg/resources/templates/kubespray/inventory/group_vars/k8s_cluster/k8s-cluster.yml.tmpl @@ -0,0 +1,384 @@ +--- +# Kubernetes configuration dirs and system namespace. +# Those are where all the additional config stuff goes +# the kubernetes normally puts in /srv/kubernetes. +# This puts them in a sane location and namespace. +# Editing those values will almost surely break something. +kube_config_dir: /etc/kubernetes +kube_script_dir: {{ `"{{ bin_dir }}/kubernetes-scripts"` }} +kube_manifest_dir: {{ `"{{ kube_config_dir }}/manifests"` }} + +# This is where all the cert scripts and certs will be located +kube_cert_dir: {{ `"{{ kube_config_dir }}/ssl"` }} + +# This is where all of the bearer tokens will be stored +kube_token_dir: {{ `"{{ kube_config_dir }}/tokens"` }} + +kube_api_anonymous_auth: true + +## Change this to use another Kubernetes version, e.g. a current beta release +kube_version: {{ .Orchestrator.Version }} + +# Where the binaries will be downloaded. +# Note: ensure that you've enough disk space (about 1G) +local_release_dir: "/tmp/releases" +# Random shifts for retrying failed ops like pushing/downloading +retry_stagger: 5 + +# This is the user that owns tha cluster installation. +kube_owner: kube + +# This is the group that the cert creation scripts chgrp the +# cert files to. Not really changeable... +kube_cert_group: kube-cert + +# Cluster Loglevel configuration +kube_log_level: 2 + +# Directory where credentials will be stored +credentials_dir: {{ `"{{ inventory_dir }}/credentials"` }} + +## It is possible to activate / deactivate selected authentication methods (oidc, static token auth) +# kube_oidc_auth: false +# kube_token_auth: false + + +## Variables for OpenID Connect Configuration https://kubernetes.io/docs/admin/authentication/ +## To use OpenID you have to deploy additional an OpenID Provider (e.g Dex, Keycloak, ...) + +# kube_oidc_url: https:// ... +# kube_oidc_client_id: kubernetes +## Optional settings for OIDC +# kube_oidc_ca_file: {{ `"{{ kube_cert_dir }}/ca.pem"` }} +# kube_oidc_username_claim: sub +# kube_oidc_username_prefix: 'oidc:' +# kube_oidc_groups_claim: groups +# kube_oidc_groups_prefix: 'oidc:' + +## Variables to control webhook authn/authz +# kube_webhook_token_auth: false +# kube_webhook_token_auth_url: https://... +# kube_webhook_token_auth_url_skip_tls_verify: false + +## For webhook authorization, authorization_modes must include Webhook +# kube_webhook_authorization: false +# kube_webhook_authorization_url: https://... +# kube_webhook_authorization_url_skip_tls_verify: false + +# Choose network plugin (cilium, calico, kube-ovn, weave or flannel. Use cni for generic cni plugin) +# Can also be set to 'cloud', which lets the cloud provider setup appropriate routing +kube_network_plugin: {{ .Orchestrator.Network.Plugin }} + +# Setting multi_networking to true will install Multus: https://github.com/k8snetworkplumbingwg/multus-cni +kube_network_plugin_multus: false + +# Kubernetes internal network for services, unused block of space. +kube_service_addresses: {{ .Orchestrator.Network.ServiceAddresses }} + +# internal network. When used, it will assign IP +# addresses from this range to individual pods. +# This network must be unused in your network infrastructure! +kube_pods_subnet: {{ .Orchestrator.Network.PodsSubnet }} + +# internal network node size allocation (optional). This is the size allocated +# to each node for pod IP address allocation. Note that the number of pods per node is +# also limited by the kubelet_max_pods variable which defaults to 110. +# +# Example: +# Up to 64 nodes and up to 254 or kubelet_max_pods (the lowest of the two) pods per node: +# - kube_pods_subnet: 10.233.64.0/18 +# - kube_network_node_prefix: 24 +# - kubelet_max_pods: 110 +# +# Example: +# Up to 128 nodes and up to 126 or kubelet_max_pods (the lowest of the two) pods per node: +# - kube_pods_subnet: 10.233.64.0/18 +# - kube_network_node_prefix: 25 +# - kubelet_max_pods: 110 +kube_network_node_prefix: 24 + +# Configure Dual Stack networking (i.e. both IPv4 and IPv6) +enable_dual_stack_networks: false + +# Kubernetes internal network for IPv6 services, unused block of space. +# This is only used if enable_dual_stack_networks is set to true +# This provides 4096 IPv6 IPs +kube_service_addresses_ipv6: fd85:ee78:d8a6:8607::1000/116 + +# Internal network. When used, it will assign IPv6 addresses from this range to individual pods. +# This network must not already be in your network infrastructure! +# This is only used if enable_dual_stack_networks is set to true. +# This provides room for 256 nodes with 254 pods per node. +kube_pods_subnet_ipv6: fd85:ee78:d8a6:8607::1:0000/112 + +# IPv6 subnet size allocated to each for pods. +# This is only used if enable_dual_stack_networks is set to true +# This provides room for 254 pods per node. +kube_network_node_prefix_ipv6: 120 + +# The port the API Server will be listening on. +kube_apiserver_ip: {{ `"{{ kube_service_addresses|ipaddr('net')|ipaddr(1)|ipaddr('address') }}"` }} +kube_apiserver_port: 6443 # (https) + +# Kube-proxy proxyMode configuration. +# Can be ipvs, iptables +kube_proxy_mode: ipvs + +# configure arp_ignore and arp_announce to avoid answering ARP queries from kube-ipvs0 interface +# must be set to true for MetalLB, kube-vip(ARP enabled) to work +{{- if and (eq .Modules.Additional.LoadBalancer.Type "metallb") .Modules.Additional.LoadBalancer.Install }} +kube_proxy_strict_arp: true +{{- else }} +kube_proxy_strict_arp: false +{{- end }} + +# A string slice of values which specify the addresses to use for NodePorts. +# Values may be valid IP blocks (e.g. 1.2.3.0/24, 1.2.3.4/32). +# The default empty string slice ([]) means to use all local addresses. +# kube_proxy_nodeport_addresses_cidr is retained for legacy config +kube_proxy_nodeport_addresses: >- {{ ` + {%- if kube_proxy_nodeport_addresses_cidr is defined -%} + [{{ kube_proxy_nodeport_addresses_cidr }}] + {%- else -%} + [] + {%- endif -%} ` }} + +# If non-empty, will use this string as identification instead of the actual hostname +# kube_override_hostname: >- {{ ` +# {%- if cloud_provider is defined and cloud_provider in [ 'aws' ] -%} +# {%- else -%} +# {{ inventory_hostname }} +# {%- endif -%}` }} + +## Encrypting Secret Data at Rest +kube_encrypt_secret_data: false + +# Graceful Node Shutdown (Kubernetes >= 1.21.0), see https://kubernetes.io/blog/2021/04/21/graceful-node-shutdown-beta/ +# kubelet_shutdown_grace_period had to be greater than kubelet_shutdown_grace_period_critical_pods to allow +# non-critical podsa to also terminate gracefully +# kubelet_shutdown_grace_period: 60s +# kubelet_shutdown_grace_period_critical_pods: 20s + +# DNS configuration. +# Kubernetes cluster name, also will be used as DNS domain +cluster_name: {{ .Orchestrator.ClusterName }} +# Subdomains of DNS domain to be resolved via /etc/resolv.conf for hostnet pods +ndots: 2 +# dns_timeout: 2 +# dns_attempts: 2 +# Custom search domains to be added in addition to the default cluster search domains +# searchdomains: +# - {{ `"svc.{{ cluster_name }}"` }} +# - {{ `"default.svc.{{ cluster_name }}"` }} +# remove_default_searchdomains: false +# Can be coredns, coredns_dual, manual or none +dns_mode: coredns +# Set manual server if using a custom cluster DNS server +# manual_dns_server: 10.x.x.x +# Enable nodelocal dns cache +enable_nodelocaldns: true +enable_nodelocaldns_secondary: false +nodelocaldns_ip: 169.254.25.10 +nodelocaldns_health_port: 9254 +nodelocaldns_second_health_port: 9256 +nodelocaldns_bind_metrics_host_ip: false +nodelocaldns_secondary_skew_seconds: 5 +# nodelocaldns_external_zones: +# - zones: +# - example.com +# - example.io:1053 +# nameservers: +# - 1.1.1.1 +# - 2.2.2.2 +# cache: 5 +# - zones: +# - https://mycompany.local:4453 +# nameservers: +# - 192.168.0.53 +# cache: 0 +# - zones: +# - mydomain.tld +# nameservers: +# - 10.233.0.3 +# cache: 5 +# rewrite: +# - name website.tld website.namespace.svc.cluster.local +# Enable k8s_external plugin for CoreDNS +enable_coredns_k8s_external: false +coredns_k8s_external_zone: k8s_external.local +# Enable endpoint_pod_names option for kubernetes plugin +enable_coredns_k8s_endpoint_pod_names: false +# Set forward options for upstream DNS servers in coredns (and nodelocaldns) config +# dns_upstream_forward_extra_opts: +# policy: sequential +# Apply extra options to coredns kubernetes plugin +# coredns_kubernetes_extra_opts: +# - 'fallthrough example.local' +# Forward extra domains to the coredns kubernetes plugin +# coredns_kubernetes_extra_domains: '' + +# Can be docker_dns, host_resolvconf or none +resolvconf_mode: host_resolvconf +# Deploy netchecker app to verify DNS resolve as an HTTP service +deploy_netchecker: false +# Ip address of the kubernetes skydns service +skydns_server: {{ `"{{ kube_service_addresses|ipaddr('net')|ipaddr(3)|ipaddr('address') }}"` }} +skydns_server_secondary: {{ `"{{ kube_service_addresses|ipaddr('net')|ipaddr(4)|ipaddr('address') }}"` }} +dns_domain: {{ `"{{ cluster_name }}"` }} + +## Container runtime +## docker for docker, crio for cri-o and containerd for containerd. +## Default: containerd +container_manager: {{ .Orchestrator.ContainerEngine.Type }} + +# Additional container runtimes +kata_containers_enabled: false + +kubeadm_certificate_key: {{ `"{{ lookup('password', credentials_dir + '/kubeadm_certificate_key.creds length=64 chars=hexdigits') | lower }}"` }} + +# K8s image pull policy (imagePullPolicy) +k8s_image_pull_policy: IfNotPresent + +# audit log for kubernetes +kubernetes_audit: false + +# define kubelet config dir for dynamic kubelet +# kubelet_config_dir: +default_kubelet_config_dir: {{ `"{{ kube_config_dir }}/dynamic_kubelet_dir"` }} + +# pod security policy (RBAC must be enabled either by having 'RBAC' in authorization_modes or kubeadm enabled) +podsecuritypolicy_enabled: {{ .Orchestrator.PodSecurotyPolicyEnabled }} + +# Custom PodSecurityPolicySpec for restricted policy +# podsecuritypolicy_restricted_spec: {} + +# Custom PodSecurityPolicySpec for privileged policy +# podsecuritypolicy_privileged_spec: {} + +# Make a copy of kubeconfig on the host that runs Ansible in {{ `{{ inventory_dir }}/artifacts`}} +kubeconfig_localhost: true +# Use ansible_host as external api ip when copying over kubeconfig. +kubeconfig_localhost_ansible_host: true +# kubectl_localhost: false + +# A comma separated list of levels of node allocatable enforcement to be enforced by kubelet. +# Acceptable options are 'pods', 'system-reserved', 'kube-reserved' and ''. Default is "". +# kubelet_enforce_node_allocatable: pods + +## Set runtime and kubelet cgroups when using systemd as cgroup driver (default) +# kubelet_runtime_cgroups: {{ `"/{{ kube_service_cgroups }}/{{ container_manager }}.service"` }} +# kubelet_kubelet_cgroups: {{ `"/{{ kube_service_cgroups }}/kubelet.service"` }} + +## Set runtime and kubelet cgroups when using cgroupfs as cgroup driver +# kubelet_runtime_cgroups_cgroupfs: {{ `"/system.slice/{{ container_manager }}.service"` }} +# kubelet_kubelet_cgroups_cgroupfs: "/system.slice/kubelet.service" + +# Optionally reserve this space for kube daemons. +# kube_reserved: false +## Uncomment to override default values +## The following two items need to be set when kube_reserved is true +# kube_reserved_cgroups_for_service_slice: kube.slice +# kube_reserved_cgroups: {{ `"/{{ kube_reserved_cgroups_for_service_slice }}"` }} +# kube_memory_reserved: 256Mi +# kube_cpu_reserved: 100m +# kube_ephemeral_storage_reserved: 2Gi +# kube_pid_reserved: "1000" +# Reservation for master hosts +# kube_master_memory_reserved: 512Mi +# kube_master_cpu_reserved: 200m +# kube_master_ephemeral_storage_reserved: 2Gi +# kube_master_pid_reserved: "1000" + +## Optionally reserve resources for OS system daemons. +# system_reserved: true +## Uncomment to override default values +## The following two items need to be set when system_reserved is true +# system_reserved_cgroups_for_service_slice: system.slice +# system_reserved_cgroups: {{ `"/{{ system_reserved_cgroups_for_service_slice }}"` }} +# system_memory_reserved: 512Mi +# system_cpu_reserved: 500m +# system_ephemeral_storage_reserved: 2Gi +## Reservation for master hosts +# system_master_memory_reserved: 256Mi +# system_master_cpu_reserved: 250m +# system_master_ephemeral_storage_reserved: 2Gi + +## Eviction Thresholds to avoid system OOMs +# https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/#eviction-thresholds +# eviction_hard: {} +# eviction_hard_control_plane: {} + +# An alternative flexvolume plugin directory +# kubelet_flexvolumes_plugins_dir: /usr/libexec/kubernetes/kubelet-plugins/volume/exec + +## Supplementary addresses that can be added in kubernetes ssl keys. +## That can be useful for example to setup a keepalived virtual IP +# supplementary_addresses_in_ssl_keys: [10.0.0.1, 10.0.0.2, 10.0.0.3] + +## Running on top of openstack vms with cinder enabled may lead to unschedulable pods due to NoVolumeZoneConflict restriction in kube-scheduler. +## See https://github.com/kubernetes-sigs/kubespray/issues/2141 +## Set this variable to true to get rid of this issue +volume_cross_zone_attachment: false +## Add Persistent Volumes Storage Class for corresponding cloud provider (supported: in-tree OpenStack, Cinder CSI, +## AWS EBS CSI, Azure Disk CSI, GCP Persistent Disk CSI) +persistent_volumes_enabled: false + +## Container Engine Acceleration +## Enable container acceleration feature, for example use gpu acceleration in containers +# nvidia_accelerator_enabled: true +## Nvidia GPU driver install. Install will by done by a (init) pod running as a daemonset. +## Important: if you use Ubuntu then you should set in all.yml 'docker_storage_options: -s overlay2' +## Array with nvida_gpu_nodes, leave empty or comment if you don't want to install drivers. +## Labels and taints won't be set to nodes if they are not in the array. +# nvidia_gpu_nodes: +# - kube-gpu-001 +# nvidia_driver_version: "384.111" +## flavor can be tesla or gtx +# nvidia_gpu_flavor: gtx +## NVIDIA driver installer images. Change them if you have trouble accessing gcr.io. +# nvidia_driver_install_centos_container: atzedevries/nvidia-centos-driver-installer:2 +# nvidia_driver_install_ubuntu_container: gcr.io/google-containers/ubuntu-nvidia-driver-installer@sha256:7df76a0f0a17294e86f691c81de6bbb7c04a1b4b3d4ea4e7e2cccdc42e1f6d63 +## NVIDIA GPU device plugin image. +# nvidia_gpu_device_plugin_container: "registry.k8s.io/nvidia-gpu-device-plugin@sha256:0842734032018be107fa2490c98156992911e3e1f2a21e059ff0105b07dd8e9e" + +## Support tls min version, Possible values: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13. +# tls_min_version: "" + +## Support tls cipher suites. +# tls_cipher_suites: {} +# - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA +# - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256 +# - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 +# - TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA +# - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 +# - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 +# - TLS_ECDHE_ECDSA_WITH_RC4_128_SHA +# - TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA +# - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA +# - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 +# - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 +# - TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA +# - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 +# - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 +# - TLS_ECDHE_RSA_WITH_RC4_128_SHA +# - TLS_RSA_WITH_3DES_EDE_CBC_SHA +# - TLS_RSA_WITH_AES_128_CBC_SHA +# - TLS_RSA_WITH_AES_128_CBC_SHA256 +# - TLS_RSA_WITH_AES_128_GCM_SHA256 +# - TLS_RSA_WITH_AES_256_CBC_SHA +# - TLS_RSA_WITH_AES_256_GCM_SHA384 +# - TLS_RSA_WITH_RC4_128_SHA + +## Amount of time to retain events. (default 1h0m0s) +event_ttl_duration: "{{ .Orchestrator.EventTtl }}" + +## Automatically renew K8S control plane certificates on first Monday of each month +auto_renew_certificates: {{ .Orchestrator.AutoRenewCertificates }} +# First Monday of each month +# auto_renew_certificates_systemd_calendar: {{ `"Mon *-*-1,2,3,4,5,6,7 03:{{ groups['kube_control_plane'].index(inventory_hostname) }}0:00"` }} + +# kubeadm patches path +kubeadm_patches: + enabled: false + source_dir: {{ `"{{ inventory_dir }}/patches"` }} + dest_dir: {{ `"{{ kube_config_dir }}/patches"` }} diff --git a/pkg/resources/templates/kubespray/inventory/hosts.tmpl b/pkg/resources/templates/kubespray/inventory/hosts.tmpl new file mode 100644 index 0000000..d210f36 --- /dev/null +++ b/pkg/resources/templates/kubespray/inventory/hosts.tmpl @@ -0,0 +1,61 @@ +{{- range .Hosts }} +{{ .Hostname }} ansible_host={{ .Ip }} ip={{ .Ip }} {{ if .User }}ansible_user={{ .User }}{{- end }} {{ if .Password }}ansible_ssh_password={{ .Password }}{{- end }} +{{- end }} + +[kube_control_plane] +{{- range .Hosts }} +{{- $search := "control_plane" -}} +{{- $found := false -}} +{{- range .Roles }} +{{- if eq . $search }} +{{- $found = true }} +{{- end }} +{{- end }} +{{- if $found }} +{{ .Hostname }} +{{- end }} +{{- end }} + +[etcd] +{{- range .Hosts }} +{{- $search := "etcd" -}} +{{- $found := false -}} +{{- range .Roles }} +{{- if eq . $search }} +{{- $found = true }} +{{- end }} +{{- end }} +{{- if $found }} +{{ .Hostname }} +{{- end }} +{{- end }} + +[kube_node] +{{- range .Hosts }} +{{- $search := "node" -}} +{{- $found := false -}} +{{- range .Roles }} +{{- if eq . $search }} +{{- $found = true }} +{{- end }} +{{- end }} +{{- if $found }} +{{ .Hostname }} +{{- end }} +{{- end }} + +[k8s_cluster:children] +kube_control_plane +kube_node + +[all:vars] +ansible_connection=ssh +{{- if not (eq .Credentials.User "") }} +ansible_user={{ .Credentials.User }} +{{- end }} +{{- if not (eq .Credentials.Password "") }} +ansible_ssh_password={{ .Credentials.Password }} +{{- end }} +{{- if not (eq .Credentials.PrivateKeyFile "") }} +ansible_ssh_private_key_file={{ .Credentials.PrivateKeyFile }} +{{- end }} diff --git a/pkg/templates/funcs.go b/pkg/templates/funcs.go new file mode 100644 index 0000000..7d03ebf --- /dev/null +++ b/pkg/templates/funcs.go @@ -0,0 +1,105 @@ +package templates + +import ( + "bytes" + "encoding/json" + "strings" + "text/template" + + "github.com/BurntSushi/toml" + "github.com/Masterminds/sprig/v3" + "sigs.k8s.io/yaml" +) + +func funcMap() template.FuncMap { + f := sprig.TxtFuncMap() + delete(f, "env") + delete(f, "expandenv") + + extra := template.FuncMap{ + "toToml": toTOML, + "toYaml": toYAML, + "fromYaml": fromYAML, + "fromYamlArray": fromYAMLArray, + "toJson": toJSON, + "fromJson": fromJSON, + "fromJsonArray": fromJSONArray, + + "include": func(string, interface{}) string { return "not implemented" }, + "tpl": func(string, interface{}) interface{} { return "not implemented" }, + "required": func(string, interface{}) (interface{}, error) { return "not implemented", nil }, + "lookup": func(string, string, string, string) (map[string]interface{}, error) { + return map[string]interface{}{}, nil + }, + } + + for k, v := range extra { + f[k] = v + } + + return f +} + +func toYAML(v interface{}) string { + data, err := yaml.Marshal(v) + if err != nil { + // Swallow errors inside of a template. + return "" + } + return strings.TrimSuffix(string(data), "\n") +} + +func fromYAML(str string) map[string]interface{} { + m := map[string]interface{}{} + + if err := yaml.Unmarshal([]byte(str), &m); err != nil { + m["Error"] = err.Error() + } + return m +} + +func fromYAMLArray(str string) []interface{} { + a := []interface{}{} + + if err := yaml.Unmarshal([]byte(str), &a); err != nil { + a = []interface{}{err.Error()} + } + return a +} + +func toTOML(v interface{}) string { + b := bytes.NewBuffer(nil) + e := toml.NewEncoder(b) + err := e.Encode(v) + if err != nil { + return err.Error() + } + return b.String() +} + +func toJSON(v interface{}) string { + data, err := json.Marshal(v) + if err != nil { + // Swallow errors inside of a template. + return "" + } + return string(data) +} + +func fromJSON(str string) map[string]interface{} { + m := make(map[string]interface{}) + + if err := json.Unmarshal([]byte(str), &m); err != nil { + m["Error"] = err.Error() + } + return m +} + +func fromJSONArray(str string) []interface{} { + a := []interface{}{} + + if err := json.Unmarshal([]byte(str), &a); err != nil { + a = []interface{}{err.Error()} + } + return a +} diff --git a/pkg/templates/templates.go b/pkg/templates/templates.go new file mode 100644 index 0000000..8965c84 --- /dev/null +++ b/pkg/templates/templates.go @@ -0,0 +1,101 @@ +package templates + +import ( + "bytes" + "embed" + "kube-forge/pkg/config" + "kube-forge/pkg/resources" + "os" + "path/filepath" + "strings" + "text/template" +) + +var K8S_TEMPLATES = [...][2]string{ + {"templates/kubespray/inventory/hosts.tmpl", "kubespray/inventory/hosts"}, + {"templates/kubespray/inventory/group_vars/all.yml.tmpl", "kubespray/inventory/group_vars/all.yml"}, + {"templates/kubespray/inventory/group_vars/k8s_cluster/addons.yml.tmpl", "kubespray/inventory/group_vars/k8s_cluster/addons.yml"}, + {"templates/kubespray/inventory/group_vars/k8s_cluster/k8s-cluster.yml.tmpl", "kubespray/inventory/group_vars/k8s_cluster/k8s-cluster.yml"}, +} + +var HELM_APPS_TEMPLATES = [...]string{ + "templates/helm-apps/releases/additional-modules/longhorn.yml.tmpl", + "templates/helm-apps/releases/additional-modules/secrets-store-csi-driver.yml.tmpl", + "templates/helm-apps/releases/observability/fluent-operator.yml.tmpl", + "templates/helm-apps/releases/observability/opentelemetry-operator.yml.tmpl", + "templates/helm-apps/releases/observability/tempo.yml.tmpl", + "templates/helm-apps/releases/observability/loki.yml.tmpl", + "templates/helm-apps/releases/observability/observability.yml.tmpl", + "templates/helm-apps/releases/registry/harbor-certificate-generator.yml.tmpl", + "templates/helm-apps/releases/registry/harbor.yml.tmpl", + "templates/helm-apps/releases/cicd/argo-cd.yml.tmpl", + "templates/helm-apps/releases/cicd/argo-rollouts.yml.tmpl", + "templates/helm-apps/releases/cicd/keel.yml.tmpl", + "templates/helm-apps/releases/cicd/argo-cd-ingress.yml.tmpl", + "templates/helm-apps/releases/secrets-storage/vault.yml.tmpl", +} +var HELM_REPOSITORIES_TEMPLATES = [...]string{ + "templates/helm-apps/repositories/repositories.yml.tmpl", +} + +func executeTemplateToString(template *template.Template, config *config.Config) string { + templateResult := &bytes.Buffer{} + err := template.Execute(templateResult, config) + if err != nil { + panic(err) + } + templateResultString := templateResult.String() + return templateResultString +} + +func getTemplateFromEmbedFSFolder(embedFS embed.FS, templateFile string) *template.Template { + templateData, err := embedFS.ReadFile(templateFile) + if err != nil { + panic(err) + } + templateDataString := string(templateData) + template, err := template.New("tmpl").Funcs(funcMap()).Parse(templateDataString) + if err != nil { + panic(err) + } + return template +} + +func applyTemplates() { + config := config.GetConfig() + for _, templateData := range K8S_TEMPLATES { + var templateFile = templateData[0] + var outFile = filepath.Join(config.DataDir, templateData[1]) + + tmpl := getTemplateFromEmbedFSFolder(resources.Templates, templateFile) + + file, err := os.Create(outFile) + if err != nil { + panic(err) + } + defer file.Close() + err = tmpl.Execute(file, config) + if err != nil { + panic(err) + } + } +} + +func GetHelmAppsConfigData() (string, string) { + cfg := config.GetConfig() + helmAppsTemplateResults := []string{} + for _, templateFile := range HELM_APPS_TEMPLATES { + template := getTemplateFromEmbedFSFolder(resources.Templates, templateFile) + helmAppsTemplateResults = append(helmAppsTemplateResults, executeTemplateToString(template, cfg)) + } + repositoriesTemplateResults := []string{} + for _, templateFile := range HELM_REPOSITORIES_TEMPLATES { + template := getTemplateFromEmbedFSFolder(resources.Templates, templateFile) + repositoriesTemplateResults = append(repositoriesTemplateResults, executeTemplateToString(template, cfg)) + } + return strings.Join(repositoriesTemplateResults, "\n"), strings.Join(helmAppsTemplateResults, "\n") +} + +func ApplyTemplates() { + applyTemplates() +}