gstykalin/kube-forge
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

actualized for kubespray version 2.28.0

Browse Source
This commit is contained in:
George Stykalin
2025-06-15 20:25:45 +03:00
parent c819d4f51f
commit c2df2abc78
366 changed files with 9939 additions and 40388 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
kubespray/project/roles/kubernetes/client/tasks/main.yml
View File

@@ -71,7 +71,7 @@
user_certs: "{{ admin_kubeconfig['users'][0]['user'] }}"
username: "kubernetes-admin-{{ cluster_name }}"
context: "kubernetes-admin-{{ cluster_name }}@{{ cluster_name }}"
override_cluster_name: "{{ {'clusters': [{'cluster': (cluster_infos | combine({'server': 'https://' + external_apiserver_address + ':' + (external_apiserver_port | string)})), 'name': cluster_name}]} }}"
override_cluster_name: "{{ {'clusters': [{'cluster': (cluster_infos | combine({'server': 'https://' + (external_apiserver_address | ansible.utils.ipwrap) + ':' + (external_apiserver_port | string)})), 'name': cluster_name}]} }}"
override_context: "{{ {'contexts': [{'context': {'user': username, 'cluster': cluster_name}, 'name': context}], 'current-context': context} }}"
override_user: "{{ {'users': [{'name': username, 'user': user_certs}]} }}"
when: kubeconfig_localhost

2
kubespray/project/roles/kubernetes/control-plane/defaults/main/etcd.yml
View File

@@ -14,8 +14,6 @@ etcd_cert_alt_ips: []
etcd_heartbeat_interval: "250"
etcd_election_timeout: "5000"
# etcd_snapshot_count: "10000"
etcd_metrics: "basic"
## A dictionary of extra environment variables to add to etcd.env, formatted like:

2
kubespray/project/roles/kubernetes/control-plane/defaults/main/kube-scheduler.yml
View File

@@ -4,7 +4,7 @@ kube_kubeadm_scheduler_extra_args: {}
# Associated interface must be reachable by the rest of the cluster, and by
# CLI/web clients.
kube_scheduler_bind_address: 0.0.0.0
kube_scheduler_bind_address: "::"
# ClientConnection options (e.g. Burst, QPS) except from kubeconfig.
kube_scheduler_client_conn_extra_opts: {}

42
kubespray/project/roles/kubernetes/control-plane/defaults/main/main.yml
View File

@@ -5,8 +5,8 @@ upgrade_cluster_setup: false
# By default the external API listens on all interfaces, this can be changed to
# listen on a specific address/interface.
# NOTE: If you specific address/interface and use loadbalancer_apiserver_localhost
# loadbalancer_apiserver_localhost (nginx/haproxy) will deploy on masters on 127.0.0.1:{{ loadbalancer_apiserver_port | default(kube_apiserver_port) }} too.
kube_apiserver_bind_address: 0.0.0.0
# loadbalancer_apiserver_localhost (nginx/haproxy) will deploy on control plane nodes on 127.0.0.1:{{ loadbalancer_apiserver_port | default(kube_apiserver_port) }} too.
kube_apiserver_bind_address: "::"
# A port range to reserve for services with NodePort visibility.
# Inclusive at both ends of the range.
@@ -29,7 +29,7 @@ kube_etcd_key_file: node-{{ inventory_hostname }}-key.pem
# Associated interfaces must be reachable by the rest of the cluster, and by
# CLI/web clients.
kube_controller_manager_bind_address: 0.0.0.0
kube_controller_manager_bind_address: "::"
# Leader election lease durations and timeouts for controller-manager
kube_controller_manager_leader_elect_lease_duration: 15s
@@ -38,7 +38,7 @@ kube_controller_manager_leader_elect_renew_deadline: 10s
# discovery_timeout modifies the discovery timeout
discovery_timeout: 5m0s
# Instruct first master to refresh kubeadm token
# Instruct first control plane node to refresh kubeadm token
kubeadm_refresh_token: true
# Scale down coredns replicas to 0 if not using coredns dns_mode
@@ -107,18 +107,29 @@ kube_apiserver_admission_control_config_file: false
# cache_size: <cache_size_value>
kube_apiserver_admission_event_rate_limits: {}
## PodSecurityAdmission plugin configuration
kube_pod_security_use_default: false
kube_pod_security_default_enforce: baseline
kube_pod_security_default_enforce_version: "{{ kube_major_version }}"
kube_pod_security_default_enforce_version: "v{{ kube_major_version }}"
kube_pod_security_default_audit: restricted
kube_pod_security_default_audit_version: "{{ kube_major_version }}"
kube_pod_security_default_audit_version: "v{{ kube_major_version }}"
kube_pod_security_default_warn: restricted
kube_pod_security_default_warn_version: "{{ kube_major_version }}"
kube_pod_security_default_warn_version: "v{{ kube_major_version }}"
kube_pod_security_exemptions_usernames: []
kube_pod_security_exemptions_runtime_class_names: []
kube_pod_security_exemptions_namespaces:
- kube-system
## ResourceQuota plugin configuration
## Resources that ResourceQuota should limit by default if no quota exists
## Example below enforces quota on all storage classes
# kube_resource_quota_limited_resources:
# - apiGroup: ""
# resource: persistentvolumeclaims
# matchContains:
# - .storageclass.storage.k8s.io/requests.storage
kube_resource_quota_limited_resources: []
# 1.10+ list of disabled admission plugins
kube_apiserver_disable_admission_plugins: []
@@ -184,13 +195,6 @@ kube_encryption_algorithm: "secretbox"
# Which kubernetes resources to encrypt
kube_encryption_resources: [secrets]
# If non-empty, will use this string as identification instead of the actual hostname
kube_override_hostname: >-
{%- if cloud_provider is defined and cloud_provider in ['aws'] -%}
{%- else -%}
{{ inventory_hostname }}
{%- endif -%}
secrets_encryption_query: "resources[*].providers[0].{{ kube_encryption_algorithm }}.keys[0].secret"
## Support tls min version, Possible values: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13.
@@ -238,8 +242,16 @@ kubeadm_upgrade_auto_cert_renewal: true
## Enable distributed tracing for kube-apiserver
kube_apiserver_tracing: false
kube_apiserver_tracing_endpoint: 0.0.0.0:4317
kube_apiserver_tracing_endpoint: "[::]:4317"
kube_apiserver_tracing_sampling_rate_per_million: 100
# Enable kubeadm file discovery if anonymous access has been removed
kubeadm_use_file_discovery: "{{ remove_anonymous_access }}"
# imagePullSerial specifies if image pulling performed by kubeadm must be done serially or in parallel. Default: true
kubeadm_image_pull_serial: true
# Supported asymmetric encryption algorithm types for the cluster's keys and certificates.
# can be one of RSA-2048(default), RSA-3072, RSA-4096, ECDSA-P256
# ref: https://kubernetes.io/docs/reference/config-api/kubeadm-config.v1beta4/#kubeadm-k8s-io-v1beta4-ClusterConfiguration
kube_asymmetric_encryption_algorithm: "RSA-2048"

60
kubespray/project/roles/kubernetes/control-plane/handlers/main.yml
View File

@@ -1,16 +1,16 @@
---
- name: Master | reload systemd
- name: Control plane | reload systemd
systemd_service:
daemon_reload: true
listen: Master | restart kubelet
listen: Control plane | restart kubelet
- name: Master | reload kubelet
- name: Control plane | reload kubelet
service:
name: kubelet
state: restarted
listen: Master | restart kubelet
listen: Control plane | restart kubelet
- name: Master | Remove apiserver container docker
- name: Control plane | Remove apiserver container docker
shell: "set -o pipefail && docker ps -af name=k8s_kube-apiserver* -q | xargs --no-run-if-empty docker rm -f"
args:
executable: /bin/bash
@@ -19,10 +19,10 @@
until: remove_apiserver_container.rc == 0
delay: 1
when: container_manager == "docker"
listen: Master | Restart apiserver
listen: Control plane | Restart apiserver
- name: Master | Remove apiserver container containerd/crio
shell: "set -o pipefail && {{ bin_dir }}/crictl pods --name kube-apiserver* -q | xargs -I% --no-run-if-empty bash -c '{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %'"
- name: Control plane | Remove apiserver container containerd/crio
shell: "set -o pipefail && {{ bin_dir }}/crictl pods --name 'kube-apiserver*' -q | xargs -I% --no-run-if-empty bash -c '{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %'"
args:
executable: /bin/bash
register: remove_apiserver_container
@@ -30,9 +30,9 @@
until: remove_apiserver_container.rc == 0
delay: 1
when: container_manager in ['containerd', 'crio']
listen: Master | Restart apiserver
listen: Control plane | Restart apiserver
- name: Master | Remove scheduler container docker
- name: Control plane | Remove scheduler container docker
shell: "set -o pipefail && {{ docker_bin_dir }}/docker ps -af name=k8s_kube-scheduler* -q | xargs --no-run-if-empty {{ docker_bin_dir }}/docker rm -f"
args:
executable: /bin/bash
@@ -41,10 +41,10 @@
until: remove_scheduler_container.rc == 0
delay: 1
when: container_manager == "docker"
listen: Master | Restart kube-scheduler
listen: Control plane | Restart kube-scheduler
- name: Master | Remove scheduler container containerd/crio
shell: "set -o pipefail && {{ bin_dir }}/crictl pods --name kube-scheduler* -q | xargs -I% --no-run-if-empty bash -c '{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %'"
- name: Control plane | Remove scheduler container containerd/crio
shell: "set -o pipefail && {{ bin_dir }}/crictl pods --name 'kube-scheduler*' -q | xargs -I% --no-run-if-empty bash -c '{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %'"
args:
executable: /bin/bash
register: remove_scheduler_container
@@ -52,9 +52,9 @@
until: remove_scheduler_container.rc == 0
delay: 1
when: container_manager in ['containerd', 'crio']
listen: Master | Restart kube-scheduler
listen: Control plane | Restart kube-scheduler
- name: Master | Remove controller manager container docker
- name: Control plane | Remove controller manager container docker
shell: "set -o pipefail && {{ docker_bin_dir }}/docker ps -af name=k8s_kube-controller-manager* -q | xargs --no-run-if-empty {{ docker_bin_dir }}/docker rm -f"
args:
executable: /bin/bash
@@ -63,10 +63,10 @@
until: remove_cm_container.rc == 0
delay: 1
when: container_manager == "docker"
listen: Master | Restart kube-controller-manager
listen: Control plane | Restart kube-controller-manager
- name: Master | Remove controller manager container containerd/crio
shell: "set -o pipefail && {{ bin_dir }}/crictl pods --name kube-controller-manager* -q | xargs -I% --no-run-if-empty bash -c '{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %'"
- name: Control plane | Remove controller manager container containerd/crio
shell: "set -o pipefail && {{ bin_dir }}/crictl pods --name 'kube-controller-manager*' -q | xargs -I% --no-run-if-empty bash -c '{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %'"
args:
executable: /bin/bash
register: remove_cm_container
@@ -74,11 +74,11 @@
until: remove_cm_container.rc == 0
delay: 1
when: container_manager in ['containerd', 'crio']
listen: Master | Restart kube-controller-manager
listen: Control plane | Restart kube-controller-manager
- name: Master | wait for kube-scheduler
- name: Control plane | wait for kube-scheduler
vars:
endpoint: "{{ kube_scheduler_bind_address if kube_scheduler_bind_address != '0.0.0.0' else 'localhost' }}"
endpoint: "{{ kube_scheduler_bind_address if kube_scheduler_bind_address != '::' else 'localhost' }}"
uri:
url: https://{{ endpoint }}:10259/healthz
validate_certs: false
@@ -87,12 +87,12 @@
retries: 60
delay: 1
listen:
- Master | restart kubelet
- Master | Restart kube-scheduler
- Control plane | restart kubelet
- Control plane | Restart kube-scheduler
- name: Master | wait for kube-controller-manager
- name: Control plane | wait for kube-controller-manager
vars:
endpoint: "{{ kube_controller_manager_bind_address if kube_controller_manager_bind_address != '0.0.0.0' else 'localhost' }}"
endpoint: "{{ kube_controller_manager_bind_address if kube_controller_manager_bind_address != '::' else 'localhost' }}"
uri:
url: https://{{ endpoint }}:10257/healthz
validate_certs: false
@@ -101,10 +101,10 @@
retries: 60
delay: 1
listen:
- Master | restart kubelet
- Master | Restart kube-controller-manager
- Control plane | restart kubelet
- Control plane | Restart kube-controller-manager
- name: Master | wait for the apiserver to be running
- name: Control plane | wait for the apiserver to be running
uri:
url: "{{ kube_apiserver_endpoint }}/healthz"
validate_certs: false
@@ -113,5 +113,5 @@
retries: 60
delay: 1
listen:
- Master | restart kubelet
- Master | Restart apiserver
- Control plane | restart kubelet
- Control plane | Restart apiserver

5
kubespray/project/roles/kubernetes/control-plane/meta/main.yml
View File

@@ -1,9 +1,6 @@
---
dependencies:
- role: kubernetes/tokens
when: kube_token_auth
tags:
- k8s-secrets
- role: kubernetes/kubeadm_common
- role: adduser
user: "{{ addusers.etcd }}"
when:

10
kubespray/project/roles/kubernetes/control-plane/tasks/check-api.yml Normal file
View File

@@ -0,0 +1,10 @@
---
- name: Kubeadm | Check api is up
uri:
url: "https://{{ ip | default(fallback_ip) }}:{{ kube_apiserver_port }}/healthz"
validate_certs: false
when: ('kube_control_plane' in group_names)
register: _result
retries: 60
delay: 5
until: _result.status == 200

2
kubespray/project/roles/kubernetes/control-plane/tasks/encrypt-at-rest.yml
View File

@@ -23,7 +23,7 @@
kube_encrypt_token_extracted: "{{ secret_file_decoded | json_query(secrets_encryption_query) | first | b64decode }}"
when: secrets_encryption_file.stat.exists
- name: Set kube_encrypt_token across master nodes
- name: Set kube_encrypt_token across control plane nodes
set_fact:
kube_encrypt_token: "{{ kube_encrypt_token_extracted }}"
delegate_to: "{{ item }}"

6
kubespray/project/roles/kubernetes/control-plane/tasks/kubeadm-fix-apiserver.yml
View File

@@ -12,6 +12,6 @@
- kubelet.conf
- scheduler.conf
notify:
- "Master | Restart kube-controller-manager"
- "Master | Restart kube-scheduler"
- "Master | reload kubelet"
- "Control plane | Restart kube-controller-manager"
- "Control plane | Restart kube-scheduler"
- "Control plane | reload kubelet"

17
kubespray/project/roles/kubernetes/control-plane/tasks/kubeadm-secondary.yml
View File

@@ -4,7 +4,7 @@
# noqa: jinja[spacing]
kubeadm_discovery_address: >-
{%- if "127.0.0.1" in kube_apiserver_endpoint or "localhost" in kube_apiserver_endpoint -%}
{{ first_kube_control_plane_address }}:{{ kube_apiserver_port }}
{{ first_kube_control_plane_address | ansible.utils.ipwrap }}:{{ kube_apiserver_port }}
{%- else -%}
{{ kube_apiserver_endpoint | regex_replace('https://', '') }}
{%- endif %}
@@ -24,26 +24,27 @@
- name: Parse certificate key if not set
set_fact:
kubeadm_certificate_key: "{{ hostvars[groups['kube_control_plane'][0]]['kubeadm_upload_cert'].stdout_lines[-1] | trim }}"
kubeadm_certificate_key: "{{ hostvars[first_kube_control_plane]['kubeadm_upload_cert'].stdout_lines[-1] | trim }}"
run_once: true
when:
- hostvars[groups['kube_control_plane'][0]]['kubeadm_upload_cert'] is defined
- hostvars[groups['kube_control_plane'][0]]['kubeadm_upload_cert'] is not skipped
- hostvars[first_kube_control_plane]['kubeadm_upload_cert'] is defined
- hostvars[first_kube_control_plane]['kubeadm_upload_cert'] is not skipped
- name: Create kubeadm ControlPlane config
template:
src: "kubeadm-controlplane.{{ kubeadmConfig_api_version }}.yaml.j2"
src: "kubeadm-controlplane.yaml.j2"
dest: "{{ kube_config_dir }}/kubeadm-controlplane.yaml"
mode: "0640"
backup: true
validate: "{{ kubeadm_config_validate_enabled | ternary(bin_dir + '/kubeadm config validate --config %s', omit) }}"
when:
- inventory_hostname != first_kube_control_plane
- not kubeadm_already_run.stat.exists
- name: Wait for k8s apiserver
wait_for:
host: "{{ kubeadm_discovery_address.split(':')[0] }}"
port: "{{ kubeadm_discovery_address.split(':')[1] }}"
host: "{{ kubeadm_discovery_address | regex_replace('\\]?:\\d+$', '') | regex_replace('^\\[', '') }}"
port: "{{ kubeadm_discovery_address.split(':')[-1] }}"
timeout: 180
@@ -87,7 +88,7 @@
command: >-
{{ bin_dir }}/kubeadm join
--config {{ kube_config_dir }}/kubeadm-controlplane.yaml
--ignore-preflight-errors=all
--ignore-preflight-errors={{ kubeadm_ignore_preflight_errors | join(',') }}
--skip-phases={{ kubeadm_join_phases_skip | join(',') }}
environment:
PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}"

97
kubespray/project/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml
View File

@@ -35,12 +35,13 @@
- "{{ kube_apiserver_ip }}"
- "localhost"
- "127.0.0.1"
- "::1"
sans_lb: "{{ [apiserver_loadbalancer_domain_name] if apiserver_loadbalancer_domain_name is defined else [] }}"
sans_lb_ip: "{{ [loadbalancer_apiserver.address] if loadbalancer_apiserver is defined and loadbalancer_apiserver.address is defined else [] }}"
sans_supp: "{{ supplementary_addresses_in_ssl_keys if supplementary_addresses_in_ssl_keys is defined else [] }}"
sans_access_ip: "{{ groups['kube_control_plane'] | map('extract', hostvars, 'access_ip') | list | select('defined') | list }}"
sans_ip: "{{ groups['kube_control_plane'] | map('extract', hostvars, 'ip') | list | select('defined') | list }}"
sans_address: "{{ groups['kube_control_plane'] | map('extract', hostvars, ['ansible_default_ipv4', 'address']) | list | select('defined') | list }}"
sans_access_ip: "{{ groups['kube_control_plane'] | map('extract', hostvars, 'main_access_ip') | list | select('defined') | list }}"
sans_ip: "{{ groups['kube_control_plane'] | map('extract', hostvars, 'main_ip') | list | select('defined') | list }}"
sans_address: "{{ groups['kube_control_plane'] | map('extract', hostvars, ['ansible_default_ipv6', 'ansible_default_ipv4', 'address']) | list | select('defined') | list }}"
sans_override: "{{ [kube_override_hostname] if kube_override_hostname else [] }}"
sans_hostname: "{{ groups['kube_control_plane'] | map('extract', hostvars, ['ansible_hostname']) | list | select('defined') | list }}"
sans_fqdn: "{{ groups['kube_control_plane'] | map('extract', hostvars, ['ansible_fqdn']) | list | select('defined') | list }}"
@@ -52,21 +53,23 @@
path: "{{ audit_policy_file | dirname }}"
state: directory
mode: "0640"
when: kubernetes_audit | default(false) or kubernetes_audit_webhook | default(false)
when: kubernetes_audit or kubernetes_audit_webhook
- name: Write api audit policy yaml
template:
src: apiserver-audit-policy.yaml.j2
dest: "{{ audit_policy_file }}"
mode: "0640"
when: kubernetes_audit | default(false) or kubernetes_audit_webhook | default(false)
when: kubernetes_audit or kubernetes_audit_webhook
notify: Control plane | Restart apiserver
- name: Write api audit webhook config yaml
template:
src: apiserver-audit-webhook-config.yaml.j2
dest: "{{ audit_webhook_config_file }}"
mode: "0640"
when: kubernetes_audit_webhook | default(false)
when: kubernetes_audit_webhook
notify: Control plane | Restart apiserver
- name: Create apiserver tracing config directory
file:
@@ -81,6 +84,7 @@
dest: "{{ kube_config_dir }}/tracing/apiserver-tracing.yaml"
mode: "0640"
when: kube_apiserver_tracing
notify: Control plane | Restart apiserver
# Nginx LB(default), If kubeadm_config_api_fqdn is defined, use other LB by kubeadm controlPlaneEndpoint.
- name: Set kubeadm_config_api_fqdn define
@@ -88,15 +92,12 @@
kubeadm_config_api_fqdn: "{{ apiserver_loadbalancer_domain_name | default('lb-apiserver.kubernetes.local') }}"
when: loadbalancer_apiserver is defined
- name: Set kubeadm api version to v1beta3
set_fact:
kubeadmConfig_api_version: v1beta3
- name: Kubeadm | Create kubeadm config
template:
src: "kubeadm-config.{{ kubeadmConfig_api_version }}.yaml.j2"
src: "kubeadm-config.{{ kubeadm_config_api_version }}.yaml.j2"
dest: "{{ kube_config_dir }}/kubeadm-config.yaml"
mode: "0640"
validate: "{{ kubeadm_config_validate_enabled | ternary(bin_dir + '/kubeadm config validate --config %s', omit) }}"
- name: Kubeadm | Create directory to store admission control configurations
file:
@@ -111,6 +112,7 @@
dest: "{{ kube_config_dir }}/admission-controls/admission-controls.yaml"
mode: "0640"
when: kube_apiserver_admission_control_config_file
notify: Control plane | Restart apiserver
- name: Kubeadm | Push admission control config files
template:
@@ -121,15 +123,7 @@
- kube_apiserver_admission_control_config_file
- item in kube_apiserver_admission_plugins_needs_configuration
loop: "{{ kube_apiserver_enable_admission_plugins }}"
- name: Kubeadm | Configure default cluster podnodeslector
template:
src: "podnodeselector.yaml.j2"
dest: "{{ kube_config_dir }}/admission-controls/podnodeselector.yaml"
mode: "0640"
when:
- kube_apiserver_admission_plugins_podnodeselector_default_node_selector is defined
- kube_apiserver_admission_plugins_podnodeselector_default_node_selector | length > 0
notify: Control plane | Restart apiserver
- name: Kubeadm | Check apiserver.crt SANs
vars:
@@ -145,12 +139,14 @@
loop: "{{ apiserver_ips }}"
register: apiserver_sans_ip_check
changed_when: apiserver_sans_ip_check.stdout is not search('does match certificate')
failed_when: apiserver_sans_ip_check.rc != 0 and apiserver_sans_ip_check.stdout is not search('does NOT match certificate')
- name: Kubeadm | Check apiserver.crt SAN hosts
command:
cmd: "openssl x509 -noout -in {{ kube_cert_dir }}/apiserver.crt -checkhost {{ item }}"
loop: "{{ apiserver_hosts }}"
register: apiserver_sans_host_check
changed_when: apiserver_sans_host_check.stdout is not search('does match certificate')
failed_when: apiserver_sans_host_check.rc != 0 and apiserver_sans_host_check.stdout is not search('does NOT match certificate')
- name: Kubeadm | regenerate apiserver cert 1/2
file:
@@ -174,38 +170,33 @@
- apiserver_sans_ip_check.changed or apiserver_sans_host_check.changed
- not kube_external_ca_mode
- name: Kubeadm | Create directory to store kubeadm patches
file:
path: "{{ kubeadm_patches.dest_dir }}"
state: directory
mode: "0640"
when: kubeadm_patches is defined and kubeadm_patches.enabled
- name: Kubeadm | Copy kubeadm patches from inventory files
copy:
src: "{{ kubeadm_patches.source_dir }}/"
dest: "{{ kubeadm_patches.dest_dir }}"
owner: "root"
mode: "0644"
when: kubeadm_patches is defined and kubeadm_patches.enabled
- name: Kubeadm | Initialize first master
command: >-
timeout -k {{ kubeadm_init_timeout }} {{ kubeadm_init_timeout }}
{{ bin_dir }}/kubeadm init
--config={{ kube_config_dir }}/kubeadm-config.yaml
--ignore-preflight-errors=all
--skip-phases={{ kubeadm_init_phases_skip | join(',') }}
{{ kube_external_ca_mode | ternary('', '--upload-certs') }}
register: kubeadm_init
# Retry is because upload config sometimes fails
retries: 3
until: kubeadm_init is succeeded or "field is immutable" in kubeadm_init.stderr
- name: Kubeadm | Initialize first control plane node
when: inventory_hostname == first_kube_control_plane and not kubeadm_already_run.stat.exists
failed_when: kubeadm_init.rc != 0 and "field is immutable" not in kubeadm_init.stderr
vars:
kubeadm_init_first_control_plane_cmd: >-
timeout -k {{ kubeadm_init_timeout }} {{ kubeadm_init_timeout }}
{{ bin_dir }}/kubeadm init
--config={{ kube_config_dir }}/kubeadm-config.yaml
--ignore-preflight-errors={{ kubeadm_ignore_preflight_errors | join(',') }}
--skip-phases={{ kubeadm_init_phases_skip | join(',') }}
{{ kube_external_ca_mode | ternary('', '--upload-certs') }}
environment:
PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}"
notify: Master | restart kubelet
notify: Control plane | restart kubelet
block:
- name: Kubeadm | Initialize first control plane node (1st try)
command: "{{ kubeadm_init_first_control_plane_cmd }}"
register: kubeadm_init
failed_when: kubeadm_init.rc != 0 and "field is immutable" not in kubeadm_init.stderr
rescue:
# Retry is because upload config sometimes fails
# This retry task is separated from 1st task to show log of failure of 1st task.
- name: Kubeadm | Initialize first control plane node (retry)
command: "{{ kubeadm_init_first_control_plane_cmd }}"
register: kubeadm_init
retries: 2
until: kubeadm_init is succeeded or "field is immutable" in kubeadm_init.stderr
failed_when: kubeadm_init.rc != 0 and "field is immutable" not in kubeadm_init.stderr
- name: Set kubeadm certificate key
set_fact:
@@ -250,20 +241,20 @@
tags:
- kubeadm_token
- name: Kubeadm | Join other masters
- name: Kubeadm | Join other control plane nodes
include_tasks: kubeadm-secondary.yml
- name: Kubeadm | upgrade kubernetes cluster
- name: Kubeadm | upgrade kubernetes cluster to {{ kube_version }}
include_tasks: kubeadm-upgrade.yml
when:
- upgrade_cluster_setup
- kubeadm_already_run.stat.exists
# FIXME(mattymo): from docs: If you don't want to taint your control-plane node, set this field to an empty slice, i.e. `taints: {}` in the YAML file.
- name: Kubeadm | Remove taint for master with node role
- name: Kubeadm | Remove taint for control plane node with node role
command: "{{ kubectl }} taint node {{ inventory_hostname }} {{ item }}"
delegate_to: "{{ first_kube_control_plane }}"
with_items:
- "node-role.kubernetes.io/control-plane:NoSchedule-"
when: inventory_hostname in groups['kube_node']
when: ('kube_node' in group_names)
failed_when: false

97
kubespray/project/roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml
View File

@@ -1,55 +1,80 @@
---
- name: Kubeadm | Check api is up
uri:
url: "https://{{ ip | default(fallback_ips[inventory_hostname]) }}:{{ kube_apiserver_port }}/healthz"
validate_certs: false
when: inventory_hostname in groups['kube_control_plane']
register: _result
retries: 60
delay: 5
until: _result.status == 200
- name: Ensure kube-apiserver is up before upgrade
import_tasks: check-api.yml
- name: Kubeadm | Upgrade first master
# kubeadm-config.v1beta4 with UpgradeConfiguration requires some values that were previously allowed as args to be specified in the config file
- name: Kubeadm | Upgrade first control plane node
command: >-
timeout -k 600s 600s
{{ bin_dir }}/kubeadm
upgrade apply -y {{ kube_version }}
{{ bin_dir }}/kubeadm upgrade apply -y v{{ kube_version }}
{%- if kubeadm_config_api_version == 'v1beta3' %}
--certificate-renewal={{ kubeadm_upgrade_auto_cert_renewal }}
--ignore-preflight-errors=all
--ignore-preflight-errors={{ kubeadm_ignore_preflight_errors | join(',') }}
--allow-experimental-upgrades
--etcd-upgrade={{ (etcd_deployment_type == "kubeadm") | bool | lower }}
{% if kubeadm_patches is defined and kubeadm_patches.enabled %}--patches={{ kubeadm_patches.dest_dir }}{% endif %}
--etcd-upgrade={{ (etcd_deployment_type == "kubeadm") | lower }}
{% if kubeadm_patches | length > 0 %}--patches={{ kubeadm_patches_dir }}{% endif %}
--force
{%- else %}
--config={{ kube_config_dir }}/kubeadm-config.yaml
{%- endif -%}
register: kubeadm_upgrade
# Retry is because upload config sometimes fails
retries: 3
until: kubeadm_upgrade.rc == 0
when: inventory_hostname == first_kube_control_plane
failed_when: kubeadm_upgrade.rc != 0 and "field is immutable" not in kubeadm_upgrade.stderr
environment:
PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}"
notify: Master | restart kubelet
- name: Kubeadm | Upgrade other masters
- name: Kubeadm | Upgrade other control plane nodes
command: >-
timeout -k 600s 600s
{{ bin_dir }}/kubeadm
upgrade apply -y {{ kube_version }}
{{ bin_dir }}/kubeadm upgrade node
{%- if kubeadm_config_api_version == 'v1beta3' %}
--certificate-renewal={{ kubeadm_upgrade_auto_cert_renewal }}
--ignore-preflight-errors=all
--allow-experimental-upgrades
--etcd-upgrade={{ (etcd_deployment_type == "kubeadm") | bool | lower }}
{% if kubeadm_patches is defined and kubeadm_patches.enabled %}--patches={{ kubeadm_patches.dest_dir }}{% endif %}
--force
--ignore-preflight-errors={{ kubeadm_ignore_preflight_errors | join(',') }}
--etcd-upgrade={{ (etcd_deployment_type == "kubeadm") | lower }}
{% if kubeadm_patches | length > 0 %}--patches={{ kubeadm_patches_dir }}{% endif %}
{%- else %}
--config={{ kube_config_dir }}/kubeadm-config.yaml
{%- endif -%}
register: kubeadm_upgrade
# Retry is because upload config sometimes fails
retries: 3
until: kubeadm_upgrade.rc == 0
when: inventory_hostname != first_kube_control_plane
failed_when: kubeadm_upgrade.rc != 0 and "field is immutable" not in kubeadm_upgrade.stderr
environment:
PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}"
notify: Master | restart kubelet
# kubeadm upgrade no longer reconciles ClusterConfiguration and KubeProxyConfiguration changes, this must be done separately after upgrade to ensure the latest config is applied
- name: Update kubeadm and kubelet configmaps after upgrade
command: "{{ bin_dir }}/kubeadm init phase upload-config all --config {{ kube_config_dir }}/kubeadm-config.yaml"
register: kubeadm_upload_config
# Retry is because upload config sometimes fails
retries: 3
until: kubeadm_upload_config.rc == 0
when:
- inventory_hostname == first_kube_control_plane
- name: Update kube-proxy configmap after upgrade
command: "{{ bin_dir }}/kubeadm init phase addon kube-proxy --config {{ kube_config_dir }}/kubeadm-config.yaml"
register: kube_proxy_upload_config
# Retry is because upload config sometimes fails
retries: 3
until: kube_proxy_upload_config.rc == 0
when:
- inventory_hostname == first_kube_control_plane
- ('addon/kube-proxy' not in kubeadm_init_phases_skip)
- name: Rewrite kubeadm managed etcd static pod manifests with updated configmap
command: "{{ bin_dir }}/kubeadm init phase etcd local --config {{ kube_config_dir }}/kubeadm-config.yaml"
when:
- etcd_deployment_type == "kubeadm"
notify: Control plane | restart kubelet
- name: Rewrite kubernetes control plane static pod manifests with updated configmap
command: "{{ bin_dir }}/kubeadm init phase control-plane all --config {{ kube_config_dir }}/kubeadm-config.yaml"
notify: Control plane | restart kubelet
- name: Flush kubelet handlers
meta: flush_handlers
- name: Ensure kube-apiserver is up after upgrade and control plane configuration updates
import_tasks: check-api.yml
- name: Kubeadm | Remove binding to anonymous user
command: "{{ kubectl }} -n kube-public delete rolebinding kubeadm:bootstrap-signer-clusterinfo --ignore-not-found"
@@ -60,8 +85,8 @@
path: "{{ item }}"
state: absent
with_items:
- /root/.kube/cache
- /root/.kube/http-cache
- /root/.kube/cache
- /root/.kube/http-cache
# FIXME: https://github.com/kubernetes/kubeadm/issues/1318
- name: Kubeadm | scale down coredns replicas to 0 if not using coredns dns_mode
@@ -75,6 +100,6 @@
until: scale_down_coredns is succeeded
run_once: true
when:
- kubeadm_scale_down_coredns_enabled
- dns_mode not in ['coredns', 'coredns_dual']
- kubeadm_scale_down_coredns_enabled
- dns_mode not in ['coredns', 'coredns_dual']
changed_when: false

4
kubespray/project/roles/kubernetes/control-plane/tasks/kubelet-fix-client-cert-rotation.yml
View File

@@ -6,7 +6,7 @@
line: ' client-certificate: /var/lib/kubelet/pki/kubelet-client-current.pem'
backup: true
notify:
- "Master | reload kubelet"
- "Control plane | reload kubelet"
- name: Fixup kubelet client cert rotation 2/2
lineinfile:
@@ -15,4 +15,4 @@
line: ' client-key: /var/lib/kubelet/pki/kubelet-client-current.pem'
backup: true
notify:
- "Master | reload kubelet"
- "Control plane | reload kubelet"

25
kubespray/project/roles/kubernetes/control-plane/tasks/main.yml
View File

@@ -18,6 +18,18 @@
mode: "0640"
when: kube_webhook_authorization | default(false)
- name: Create structured AuthorizationConfiguration file
copy:
content: "{{ authz_config | to_nice_yaml(indent=2, sort_keys=false) }}"
dest: "{{ kube_config_dir }}/apiserver-authorization-config-{{ kube_apiserver_authorization_config_api_version }}.yaml"
mode: "0640"
vars:
authz_config:
apiVersion: apiserver.config.k8s.io/{{ kube_apiserver_authorization_config_api_version }}
kind: AuthorizationConfiguration
authorizers: "{{ kube_apiserver_authorization_config_authorizers }}"
when: kube_apiserver_use_authorization_config_file
- name: Create kube-scheduler config
template:
src: kubescheduler-config.yaml.j2
@@ -43,7 +55,7 @@
- name: Install kubectl bash completion
shell: "{{ bin_dir }}/kubectl completion bash >/etc/bash_completion.d/kubectl.sh"
when: ansible_os_family in ["Debian","RedHat"]
when: ansible_os_family in ["Debian","RedHat", "Suse"]
tags:
- kubectl
ignore_errors: true # noqa ignore-errors
@@ -54,7 +66,7 @@
owner: root
group: root
mode: "0755"
when: ansible_os_family in ["Debian","RedHat"]
when: ansible_os_family in ["Debian","RedHat", "Suse"]
tags:
- kubectl
- upgrade
@@ -73,7 +85,7 @@
state: present
marker: "# Ansible entries {mark}"
when:
- ansible_os_family in ["Debian","RedHat"]
- ansible_os_family in ["Debian","RedHat", "Suse"]
- kubectl_alias is defined and kubectl_alias != ""
tags:
- kubectl
@@ -93,6 +105,13 @@
- name: Include kubeadm secondary server apiserver fixes
include_tasks: kubeadm-fix-apiserver.yml
- name: Cleanup unused AuthorizationConfiguration file versions
file:
path: "{{ kube_config_dir }}/apiserver-authorization-config-{{ item }}.yaml"
state: absent
loop: "{{ ['v1alpha1', 'v1beta1', 'v1'] | reject('equalto', kube_apiserver_authorization_config_api_version) | list }}"
when: kube_apiserver_use_authorization_config_file
- name: Include kubelet client cert rotation fixes
include_tasks: kubelet-fix-client-cert-rotation.yml
when: kubelet_rotate_certificates

8
kubespray/project/roles/kubernetes/control-plane/tasks/pre-upgrade.yml
View File

@@ -1,5 +1,5 @@
---
- name: "Pre-upgrade | Delete master manifests if etcd secrets changed"
- name: "Pre-upgrade | Delete control plane manifests if etcd secrets changed"
file:
path: "/etc/kubernetes/manifests/{{ item }}.manifest"
state: absent
@@ -8,14 +8,14 @@
register: kube_apiserver_manifest_replaced
when: etcd_secret_changed | default(false)
- name: "Pre-upgrade | Delete master containers forcefully" # noqa no-handler
- name: "Pre-upgrade | Delete control plane containers forcefully" # noqa no-handler
shell: "set -o pipefail && docker ps -af name=k8s_{{ item }}* -q | xargs --no-run-if-empty docker rm -f"
args:
executable: /bin/bash
with_items:
- ["kube-apiserver", "kube-controller-manager", "kube-scheduler"]
when: kube_apiserver_manifest_replaced.changed
register: remove_master_container
register: remove_control_plane_container
retries: 10
until: remove_master_container.rc == 0
until: remove_control_plane_container.rc == 0
delay: 1

92
kubespray/project/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2
View File

@@ -7,7 +7,7 @@ bootstrapTokens:
ttl: "24h"
{% endif %}
localAPIEndpoint:
advertiseAddress: {{ kube_apiserver_address }}
advertiseAddress: "{{ kube_apiserver_address }}"
bindPort: {{ kube_apiserver_port }}
{% if kubeadm_certificate_key is defined %}
certificateKey: {{ kubeadm_certificate_key }}
@@ -16,7 +16,7 @@ nodeRegistration:
{% if kube_override_hostname | default('') %}
name: "{{ kube_override_hostname }}"
{% endif %}
{% if inventory_hostname in groups['kube_control_plane'] and inventory_hostname not in groups['kube_node'] %}
{% if 'kube_control_plane' in group_names and 'kube_node' not in group_names %}
taints:
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
@@ -24,13 +24,13 @@ nodeRegistration:
taints: []
{% endif %}
criSocket: {{ cri_socket }}
{% if cloud_provider is defined and cloud_provider in ["external"] %}
{% if cloud_provider == "external" %}
kubeletExtraArgs:
cloud-provider: external
{% endif %}
{% if kubeadm_patches is defined and kubeadm_patches.enabled %}
{% if kubeadm_patches | length > 0 %}
patches:
directory: {{ kubeadm_patches.dest_dir }}
directory: {{ kubeadm_patches_dir }}
{% endif %}
---
apiVersion: kubeadm.k8s.io/v1beta3
@@ -41,7 +41,7 @@ etcd:
external:
endpoints:
{% for endpoint in etcd_access_addresses.split(',') %}
- {{ endpoint }}
- "{{ endpoint }}"
{% endfor %}
caFile: {{ etcd_cert_dir }}/{{ kube_etcd_cacert_file }}
certFile: {{ etcd_cert_dir }}/{{ kube_etcd_cert_file }}
@@ -94,9 +94,9 @@ dns:
imageTag: {{ coredns_image_tag }}
networking:
dnsDomain: {{ dns_domain }}
serviceSubnet: "{{ kube_service_addresses }}{{ ',' + kube_service_addresses_ipv6 if enable_dual_stack_networks else '' }}"
serviceSubnet: "{{ kube_service_subnets }}"
{% if kube_network_plugin is defined and kube_network_plugin not in ["kube-ovn"] %}
podSubnet: "{{ kube_pods_subnet }}{{ ',' + kube_pods_subnet_ipv6 if enable_dual_stack_networks else '' }}"
podSubnet: "{{ kube_pods_subnets }}"
{% endif %}
{% if kubeadm_feature_gates %}
featureGates:
@@ -104,11 +104,11 @@ featureGates:
{{ feature | replace("=", ": ") }}
{% endfor %}
{% endif %}
kubernetesVersion: {{ kube_version }}
kubernetesVersion: v{{ kube_version }}
{% if kubeadm_config_api_fqdn is defined %}
controlPlaneEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}
controlPlaneEndpoint: "{{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}"
{% else %}
controlPlaneEndpoint: {{ ip | default(fallback_ips[inventory_hostname]) }}:{{ kube_apiserver_port }}
controlPlaneEndpoint: "{{ main_ip | ansible.utils.ipwrap }}:{{ kube_apiserver_port }}"
{% endif %}
certificatesDir: {{ kube_cert_dir }}
imageRepository: {{ kube_image_repo }}
@@ -126,8 +126,12 @@ apiServer:
{% if kube_api_anonymous_auth is defined %}
anonymous-auth: "{{ kube_api_anonymous_auth }}"
{% endif %}
{% if kube_apiserver_use_authorization_config_file %}
authorization-config: "{{ kube_config_dir }}/apiserver-authorization-config-{{ kube_apiserver_authorization_config_api_version }}.yaml"
{% else %}
authorization-mode: {{ authorization_modes | join(',') }}
bind-address: {{ kube_apiserver_bind_address }}
{% endif %}
bind-address: "{{ kube_apiserver_bind_address }}"
{% if kube_apiserver_enable_admission_plugins | length > 0 %}
enable-admission-plugins: {{ kube_apiserver_enable_admission_plugins | join(',') }}
{% endif %}
@@ -143,12 +147,12 @@ apiServer:
etcd-servers-overrides: "/events#{{ etcd_events_access_addresses_semicolon }}"
{% endif %}
service-node-port-range: {{ kube_apiserver_node_port_range }}
service-cluster-ip-range: "{{ kube_service_addresses }}{{ ',' + kube_service_addresses_ipv6 if enable_dual_stack_networks else '' }}"
service-cluster-ip-range: "{{ kube_service_subnets }}"
kubelet-preferred-address-types: "{{ kubelet_preferred_address_types }}"
profiling: "{{ kube_profiling }}"
request-timeout: "{{ kube_apiserver_request_timeout }}"
enable-aggregator-routing: "{{ kube_api_aggregator_routing }}"
{% if kube_token_auth | default(true) %}
{% if kube_token_auth %}
token-auth-file: {{ kube_token_dir }}/known_tokens.csv
{% endif %}
{% if kube_apiserver_service_account_lookup %}
@@ -176,7 +180,7 @@ apiServer:
{% if kube_webhook_token_auth | default(false) %}
authentication-token-webhook-config-file: {{ kube_config_dir }}/webhook-token-auth-config.yaml
{% endif %}
{% if kube_webhook_authorization | default(false) %}
{% if kube_webhook_authorization and not kube_apiserver_use_authorization_config_file %}
authorization-webhook-config-file: {{ kube_config_dir }}/webhook-authorization-config.yaml
{% endif %}
{% if kube_encrypt_secret_data %}
@@ -210,10 +214,6 @@ apiServer:
{% if kube_apiserver_feature_gates or kube_feature_gates %}
feature-gates: "{{ kube_apiserver_feature_gates | default(kube_feature_gates, true) | join(',') }}"
{% endif %}
{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "gce"] %}
cloud-provider: {{ cloud_provider }}
cloud-config: {{ kube_config_dir }}/cloud_config
{% endif %}
{% if tls_min_version is defined %}
tls-min-version: {{ tls_min_version }}
{% endif %}
@@ -230,14 +230,9 @@ apiServer:
{% if kube_apiserver_tracing %}
tracing-config-file: {{ kube_config_dir }}/tracing/apiserver-tracing.yaml
{% endif %}
{% if kubernetes_audit or kube_token_auth | default(true) or kube_webhook_token_auth | default(false) or ( cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "gce"] ) or apiserver_extra_volumes or ssl_ca_dirs | length %}
{% if kubernetes_audit or kube_token_auth or kube_webhook_token_auth or apiserver_extra_volumes or ssl_ca_dirs | length %}
extraVolumes:
{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "gce"] %}
- name: cloud-config
hostPath: {{ kube_config_dir }}/cloud_config
mountPath: {{ kube_config_dir }}/cloud_config
{% endif %}
{% if kube_token_auth | default(true) %}
{% if kube_token_auth %}
- name: token-auth-config
hostPath: {{ kube_token_dir }}
mountPath: {{ kube_token_dir }}
@@ -252,6 +247,11 @@ apiServer:
hostPath: {{ kube_config_dir }}/webhook-authorization-config.yaml
mountPath: {{ kube_config_dir }}/webhook-authorization-config.yaml
{% endif %}
{% if kube_apiserver_use_authorization_config_file %}
- name: authorization-config
hostPath: {{ kube_config_dir }}/apiserver-authorization-config-{{ kube_apiserver_authorization_config_api_version }}.yaml
mountPath: {{ kube_config_dir }}/apiserver-authorization-config-{{ kube_apiserver_authorization_config_api_version }}.yaml
{% endif %}
{% if kubernetes_audit or kubernetes_audit_webhook %}
- name: {{ audit_policy_name }}
hostPath: {{ audit_policy_hostpath }}
@@ -294,7 +294,7 @@ apiServer:
{% endif %}
certSANs:
{% for san in apiserver_sans %}
- "{{ san }}"
- {{ san }}
{% endfor %}
timeoutForControlPlane: 5m0s
controllerManager:
@@ -302,22 +302,22 @@ controllerManager:
node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }}
node-monitor-period: {{ kube_controller_node_monitor_period }}
{% if kube_network_plugin is defined and kube_network_plugin not in ["kube-ovn"] %}
cluster-cidr: "{{ kube_pods_subnet }}{{ ',' + kube_pods_subnet_ipv6 if enable_dual_stack_networks else '' }}"
cluster-cidr: "{{ kube_pods_subnets }}"
{% endif %}
service-cluster-ip-range: "{{ kube_service_addresses }}{{ ',' + kube_service_addresses_ipv6 if enable_dual_stack_networks else '' }}"
service-cluster-ip-range: "{{ kube_service_subnets }}"
{% if kube_network_plugin is defined and kube_network_plugin == "calico" and not calico_ipam_host_local %}
allocate-node-cidrs: "false"
{% else %}
{% if enable_dual_stack_networks %}
{% if ipv4_stack %}
node-cidr-mask-size-ipv4: "{{ kube_network_node_prefix }}"
{% endif %}
{% if ipv6_stack %}
node-cidr-mask-size-ipv6: "{{ kube_network_node_prefix_ipv6 }}"
{% else %}
node-cidr-mask-size: "{{ kube_network_node_prefix }}"
{% endif %}
{% endif %}
profiling: "{{ kube_profiling }}"
terminated-pod-gc-threshold: "{{ kube_controller_terminated_pod_gc_threshold }}"
bind-address: {{ kube_controller_manager_bind_address }}
bind-address: "{{ kube_controller_manager_bind_address }}"
leader-elect-lease-duration: {{ kube_controller_manager_leader_elect_lease_duration }}
leader-elect-renew-deadline: {{ kube_controller_manager_leader_elect_renew_deadline }}
{% if kube_controller_feature_gates or kube_feature_gates %}
@@ -326,10 +326,6 @@ controllerManager:
{% for key in kube_kubeadm_controller_extra_args %}
{{ key }}: "{{ kube_kubeadm_controller_extra_args[key] }}"
{% endfor %}
{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "gce"] %}
cloud-provider: {{ cloud_provider }}
cloud-config: {{ kube_config_dir }}/cloud_config
{% endif %}
{% if kube_network_plugin is defined and kube_network_plugin not in ["cloud"] %}
configure-cloud-routes: "false"
{% endif %}
@@ -343,18 +339,8 @@ controllerManager:
tls-cipher-suites: {% for tls in tls_cipher_suites %}{{ tls }}{{ "," if not loop.last else "" }}{% endfor %}
{% endif %}
{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "gce"] or controller_manager_extra_volumes %}
{% if controller_manager_extra_volumes %}
extraVolumes:
{% if cloud_provider is defined and cloud_provider in ["openstack"] and openstack_cacert is defined %}
- name: openstackcacert
hostPath: "{{ kube_config_dir }}/openstack-cacert.pem"
mountPath: "{{ kube_config_dir }}/openstack-cacert.pem"
{% endif %}
{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "gce"] %}
- name: cloud-config
hostPath: {{ kube_config_dir }}/cloud_config
mountPath: {{ kube_config_dir }}/cloud_config
{% endif %}
{% for volume in controller_manager_extra_volumes %}
- name: {{ volume.name }}
hostPath: {{ volume.hostPath }}
@@ -364,7 +350,7 @@ controllerManager:
{% endif %}
scheduler:
extraArgs:
bind-address: {{ kube_scheduler_bind_address }}
bind-address: "{{ kube_scheduler_bind_address }}"
config: {{ kube_config_dir }}/kubescheduler-config.yaml
{% if kube_scheduler_feature_gates or kube_feature_gates %}
feature-gates: "{{ kube_scheduler_feature_gates | default(kube_feature_gates, true) | join(',') }}"
@@ -398,7 +384,7 @@ scheduler:
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
bindAddress: {{ kube_proxy_bind_address }}
bindAddress: "{{ kube_proxy_bind_address }}"
clientConnection:
acceptContentTypes: {{ kube_proxy_client_accept_content_types }}
burst: {{ kube_proxy_client_burst }}
@@ -406,7 +392,7 @@ clientConnection:
kubeconfig: {{ kube_proxy_client_kubeconfig }}
qps: {{ kube_proxy_client_qps }}
{% if kube_network_plugin is defined and kube_network_plugin not in ["kube-ovn"] %}
clusterCIDR: "{{ kube_pods_subnet }}{{ ',' + kube_pods_subnet_ipv6 if enable_dual_stack_networks else '' }}"
clusterCIDR: "{{ kube_pods_subnets }}"
{% endif %}
configSyncPeriod: {{ kube_proxy_config_sync_period }}
conntrack:
@@ -415,7 +401,7 @@ conntrack:
tcpCloseWaitTimeout: {{ kube_proxy_conntrack_tcp_close_wait_timeout }}
tcpEstablishedTimeout: {{ kube_proxy_conntrack_tcp_established_timeout }}
enableProfiling: {{ kube_proxy_enable_profiling }}
healthzBindAddress: {{ kube_proxy_healthz_bind_address }}
healthzBindAddress: "{{ kube_proxy_healthz_bind_address }}"
hostnameOverride: "{{ kube_override_hostname }}"
iptables:
masqueradeAll: {{ kube_proxy_masquerade_all }}
@@ -431,7 +417,7 @@ ipvs:
tcpTimeout: {{ kube_proxy_tcp_timeout }}
tcpFinTimeout: {{ kube_proxy_tcp_fin_timeout }}
udpTimeout: {{ kube_proxy_udp_timeout }}
metricsBindAddress: {{ kube_proxy_metrics_bind_address }}
metricsBindAddress: "{{ kube_proxy_metrics_bind_address }}"
mode: {{ kube_proxy_mode }}
nodePortAddresses: {{ kube_proxy_nodeport_addresses }}
oomScoreAdj: {{ kube_proxy_oom_score_adj }}

593
kubespray/project/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta4.yaml.j2 Normal file
View File

@@ -0,0 +1,593 @@
apiVersion: kubeadm.k8s.io/v1beta4
kind: InitConfiguration
{% if kubeadm_token is defined %}
bootstrapTokens:
- token: "{{ kubeadm_token }}"
description: "kubespray kubeadm bootstrap token"
ttl: "24h"
{% endif %}
localAPIEndpoint:
advertiseAddress: "{{ kube_apiserver_address }}"
bindPort: {{ kube_apiserver_port }}
{% if kubeadm_certificate_key is defined %}
certificateKey: {{ kubeadm_certificate_key }}
{% endif %}
nodeRegistration:
{% if kube_override_hostname | default('') %}
name: "{{ kube_override_hostname }}"
{% endif %}
{% if 'kube_control_plane' in group_names and 'kube_node' not in group_names %}
taints:
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
{% else %}
taints: []
{% endif %}
criSocket: {{ cri_socket }}
{% if cloud_provider == "external" %}
kubeletExtraArgs:
- name: cloud-provider
value: external
{% endif %}
imagePullPolicy: {{ k8s_image_pull_policy }}
imagePullSerial: {{ kubeadm_image_pull_serial | lower }}
{% if kubeadm_patches | length > 0 %}
patches:
directory: {{ kubeadm_patches_dir }}
{% endif %}
---
apiVersion: kubeadm.k8s.io/v1beta4
kind: ClusterConfiguration
clusterName: {{ cluster_name }}
encryptionAlgorithm: {{ kube_asymmetric_encryption_algorithm }}
etcd:
{% if etcd_deployment_type != "kubeadm" %}
external:
endpoints:
{% for endpoint in etcd_access_addresses.split(',') %}
- "{{ endpoint }}"
{% endfor %}
caFile: {{ etcd_cert_dir }}/{{ kube_etcd_cacert_file }}
certFile: {{ etcd_cert_dir }}/{{ kube_etcd_cert_file }}
keyFile: {{ etcd_cert_dir }}/{{ kube_etcd_key_file }}
{% elif etcd_deployment_type == "kubeadm" %}
local:
imageRepository: "{{ etcd_image_repo | regex_replace("/etcd$","") }}"
imageTag: "{{ etcd_image_tag }}"
dataDir: "{{ etcd_data_dir }}"
extraArgs:
- name: metrics
value: {{ etcd_metrics }}
- name: election-timeout
value: "{{ etcd_election_timeout }}"
- name: heartbeat-interval
value: "{{ etcd_heartbeat_interval }}"
- name: auto-compaction-retention
value: "{{ etcd_compaction_retention }}"
{% if etcd_listen_metrics_urls is defined %}
- name: listen-metrics-urls
value: "{{ etcd_listen_metrics_urls }}"
{% endif %}
{% if etcd_snapshot_count is defined %}
- name: snapshot-count
value: "{{ etcd_snapshot_count }}"
{% endif %}
{% if etcd_quota_backend_bytes is defined %}
- name: quota-backend-bytes
value: "{{ etcd_quota_backend_bytes }}"
{% endif %}
{% if etcd_max_request_bytes is defined %}
- name: max-request-bytes
value: "{{ etcd_max_request_bytes }}"
{% endif %}
{% if etcd_log_level is defined %}
- name: log-level
value: "{{ etcd_log_level }}"
{% endif %}
{% for key, value in etcd_extra_vars.items() %}
- name: {{ key }}
value: "{{ value }}"
{% endfor %}
serverCertSANs:
{% for san in etcd_cert_alt_names %}
- "{{ san }}"
{% endfor %}
{% for san in etcd_cert_alt_ips %}
- "{{ san }}"
{% endfor %}
peerCertSANs:
{% for san in etcd_cert_alt_names %}
- "{{ san }}"
{% endfor %}
{% for san in etcd_cert_alt_ips %}
- "{{ san }}"
{% endfor %}
{% endif %}
dns:
{% if 'addon/coredns' in kubeadm_init_phases_skip %}
disabled: true
{% endif %}
imageRepository: {{ coredns_image_repo | regex_replace('/coredns(?!/coredns).*$', '') }}
imageTag: {{ coredns_image_tag }}
networking:
dnsDomain: {{ dns_domain }}
serviceSubnet: "{{ kube_service_subnets }}"
{% if kube_network_plugin is defined and kube_network_plugin not in ["kube-ovn"] %}
podSubnet: "{{ kube_pods_subnets }}"
{% endif %}
{% if kubeadm_feature_gates %}
featureGates:
{% for feature in kubeadm_feature_gates %}
{{ feature | replace("=", ": ") }}
{% endfor %}
{% endif %}
kubernetesVersion: v{{ kube_version }}
{% if kubeadm_config_api_fqdn is defined %}
controlPlaneEndpoint: "{{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}"
{% else %}
controlPlaneEndpoint: "{{ main_ip | ansible.utils.ipwrap }}:{{ kube_apiserver_port }}"
{% endif %}
certificatesDir: {{ kube_cert_dir }}
imageRepository: {{ kube_image_repo }}
apiServer:
extraArgs:
{% if kube_apiserver_etcd_compaction_interval is defined %}
- name: etcd-compaction-interval
value: "{{ kube_apiserver_etcd_compaction_interval }}"
{% endif %}
{% if kube_apiserver_pod_eviction_not_ready_timeout_seconds is defined %}
- name: default-not-ready-toleration-seconds
value: "{{ kube_apiserver_pod_eviction_not_ready_timeout_seconds }}"
{% endif %}
{% if kube_apiserver_pod_eviction_unreachable_timeout_seconds is defined %}
- name: default-unreachable-toleration-seconds
value: "{{ kube_apiserver_pod_eviction_unreachable_timeout_seconds }}"
{% endif %}
{% if kube_api_anonymous_auth is defined %}
- name: anonymous-auth
value: "{{ kube_api_anonymous_auth }}"
{% endif %}
{% if kube_apiserver_use_authorization_config_file %}
- name: authorization-config
value: "{{ kube_config_dir }}/apiserver-authorization-config-{{ kube_apiserver_authorization_config_api_version }}.yaml"
{% else %}
- name: authorization-mode
value: "{{ authorization_modes | join(',') }}"
{% endif %}
- name: bind-address
value: "{{ kube_apiserver_bind_address }}"
{% if kube_apiserver_enable_admission_plugins | length > 0 %}
- name: enable-admission-plugins
value: "{{ kube_apiserver_enable_admission_plugins | join(',') }}"
{% endif %}
{% if kube_apiserver_admission_control_config_file %}
- name: admission-control-config-file
value: "{{ kube_config_dir }}/admission-controls.yaml"
{% endif %}
{% if kube_apiserver_disable_admission_plugins | length > 0 %}
- name: disable-admission-plugins
value: "{{ kube_apiserver_disable_admission_plugins | join(',') }}"
{% endif %}
- name: apiserver-count
value: "{{ kube_apiserver_count }}"
- name: endpoint-reconciler-type
value: lease
{% if etcd_events_cluster_enabled %}
- name: etcd-servers-overrides
value: "/events#{{ etcd_events_access_addresses_semicolon }}"
{% endif %}
- name: service-node-port-range
value: "{{ kube_apiserver_node_port_range }}"
- name: service-cluster-ip-range
value: "{{ kube_service_subnets }}"
- name: kubelet-preferred-address-types
value: "{{ kubelet_preferred_address_types }}"
- name: profiling
value: "{{ kube_profiling }}"
- name: request-timeout
value: "{{ kube_apiserver_request_timeout }}"
- name: enable-aggregator-routing
value: "{{ kube_api_aggregator_routing }}"
{% if kube_apiserver_service_account_lookup %}
- name: service-account-lookup
value: "{{ kube_apiserver_service_account_lookup }}"
{% endif %}
{% if kube_oidc_auth | default(false) and kube_oidc_url is defined and kube_oidc_client_id is defined %}
- name: oidc-issuer-url
value: "{{ kube_oidc_url }}"
- name: oidc-client-id
value: "{{ kube_oidc_client_id }}"
{% if kube_oidc_ca_file is defined %}
- name: oidc-ca-file
value: "{{ kube_oidc_ca_file }}"
{% endif %}
{% if kube_oidc_username_claim is defined %}
- name: oidc-username-claim
value: "{{ kube_oidc_username_claim }}"
{% endif %}
{% if kube_oidc_groups_claim is defined %}
- name: oidc-groups-claim
value: "{{ kube_oidc_groups_claim }}"
{% endif %}
{% if kube_oidc_username_prefix is defined %}
- name: oidc-username-prefix
value: "{{ kube_oidc_username_prefix }}"
{% endif %}
{% if kube_oidc_groups_prefix is defined %}
- name: oidc-groups-prefix
value: "{{ kube_oidc_groups_prefix }}"
{% endif %}
{% endif %}
{% if kube_webhook_token_auth | default(false) %}
- name: authentication-token-webhook-config-file
value: "{{ kube_config_dir }}/webhook-token-auth-config.yaml"
{% endif %}
{% if kube_webhook_authorization and not kube_apiserver_use_authorization_config_file %}
- name: authorization-webhook-config-file
value: "{{ kube_config_dir }}/webhook-authorization-config.yaml"
{% endif %}
{% if kube_encrypt_secret_data %}
- name: encryption-provider-config
value: "{{ kube_cert_dir }}/secrets_encryption.yaml"
{% endif %}
- name: storage-backend
value: "{{ kube_apiserver_storage_backend }}"
{% if kube_api_runtime_config | length > 0 %}
- name: runtime-config
value: "{{ kube_api_runtime_config | join(',') }}"
{% endif %}
- name: allow-privileged
value: "true"
{% if kubernetes_audit or kubernetes_audit_webhook %}
- name: audit-policy-file
value: "{{ audit_policy_file }}"
{% endif %}
{% if kubernetes_audit %}
- name: audit-log-path
value: "{{ audit_log_path }}"
- name: audit-log-maxage
value: "{{ audit_log_maxage }}"
- name: audit-log-maxbackup
value: "{{ audit_log_maxbackups }}"
- name: audit-log-maxsize
value: "{{ audit_log_maxsize }}"
{% endif %}
{% if kubernetes_audit_webhook %}
- name: audit-webhook-config-file
value: "{{ audit_webhook_config_file }}"
- name: audit-webhook-mode
value: "{{ audit_webhook_mode }}"
{% if audit_webhook_mode == "batch" %}
- name: audit-webhook-batch-max-size
value: "{{ audit_webhook_batch_max_size }}"
- name: audit-webhook-batch-max-wait
value: "{{ audit_webhook_batch_max_wait }}"
{% endif %}
{% endif %}
{% for key in kube_kubeadm_apiserver_extra_args %}
- name: "{{ key }}"
value: "{{ kube_kubeadm_apiserver_extra_args[key] }}"
{% endfor %}
{% if kube_apiserver_feature_gates or kube_feature_gates %}
- name: feature-gates
value: "{{ kube_apiserver_feature_gates | default(kube_feature_gates, true) | join(',') }}"
{% endif %}
{% if tls_min_version is defined %}
- name: tls-min-version
value: "{{ tls_min_version }}"
{% endif %}
{% if tls_cipher_suites is defined %}
- name: tls-cipher-suites
value: "{% for tls in tls_cipher_suites %}{{ tls }}{{ ',' if not loop.last else '' }}{% endfor %}"
{% endif %}
{% if event_ttl_duration is defined %}
- name: event-ttl
value: "{{ event_ttl_duration }}"
{% endif %}
{% if kubelet_rotate_server_certificates %}
- name: kubelet-certificate-authority
value: "{{ kube_cert_dir }}/ca.crt"
{% endif %}
{% if kube_apiserver_tracing %}
- name: tracing-config-file
value: "{{ kube_config_dir }}/tracing/apiserver-tracing.yaml"
{% endif %}
{% if kubernetes_audit or kube_token_auth or kube_webhook_token_auth or apiserver_extra_volumes or ssl_ca_dirs | length %}
extraVolumes:
{% if kube_token_auth %}
- name: token-auth-config
hostPath: {{ kube_token_dir }}
mountPath: {{ kube_token_dir }}
{% endif %}
{% if kube_webhook_token_auth | default(false) %}
- name: webhook-token-auth-config
hostPath: {{ kube_config_dir }}/webhook-token-auth-config.yaml
mountPath: {{ kube_config_dir }}/webhook-token-auth-config.yaml
{% endif %}
{% if kube_webhook_authorization | default(false) %}
- name: webhook-authorization-config
hostPath: {{ kube_config_dir }}/webhook-authorization-config.yaml
mountPath: {{ kube_config_dir }}/webhook-authorization-config.yaml
{% endif %}
{% if kube_apiserver_use_authorization_config_file %}
- name: authorization-config
hostPath: {{ kube_config_dir }}/apiserver-authorization-config-{{ kube_apiserver_authorization_config_api_version }}.yaml
mountPath: {{ kube_config_dir }}/apiserver-authorization-config-{{ kube_apiserver_authorization_config_api_version }}.yaml
{% endif %}
{% if kubernetes_audit or kubernetes_audit_webhook %}
- name: {{ audit_policy_name }}
hostPath: {{ audit_policy_hostpath }}
mountPath: {{ audit_policy_mountpath }}
{% if audit_log_path != "-" %}
- name: {{ audit_log_name }}
hostPath: {{ audit_log_hostpath }}
mountPath: {{ audit_log_mountpath }}
readOnly: false
{% endif %}
{% endif %}
{% if kube_apiserver_admission_control_config_file %}
- name: admission-control-configs
hostPath: {{ kube_config_dir }}/admission-controls
mountPath: {{ kube_config_dir }}
readOnly: false
pathType: DirectoryOrCreate
{% endif %}
{% if kube_apiserver_tracing %}
- name: tracing
hostPath: {{ kube_config_dir }}/tracing
mountPath: {{ kube_config_dir }}/tracing
readOnly: true
pathType: DirectoryOrCreate
{% endif %}
{% for volume in apiserver_extra_volumes %}
- name: {{ volume.name }}
hostPath: {{ volume.hostPath }}
mountPath: {{ volume.mountPath }}
readOnly: {{ volume.readOnly | d(not (volume.writable | d(false))) }}
{% endfor %}
{% if ssl_ca_dirs | length %}
{% for dir in ssl_ca_dirs %}
- name: {{ dir | regex_replace('^/(.*)$', '\\1' ) | regex_replace('/', '-') }}
hostPath: {{ dir }}
mountPath: {{ dir }}
readOnly: true
{% endfor %}
{% endif %}
{% endif %}
certSANs:
{% for san in apiserver_sans %}
- {{ san }}
{% endfor %}
controllerManager:
extraArgs:
- name: node-monitor-grace-period
value: "{{ kube_controller_node_monitor_grace_period }}"
- name: node-monitor-period
value: "{{ kube_controller_node_monitor_period }}"
{% if kube_network_plugin is defined and kube_network_plugin not in ["kube-ovn"] %}
- name: cluster-cidr
value: "{{ kube_pods_subnets }}"
{% endif %}
- name: service-cluster-ip-range
value: "{{ kube_service_subnets }}"
{% if kube_network_plugin is defined and kube_network_plugin == "calico" and not calico_ipam_host_local %}
- name: allocate-node-cidrs
value: "false"
{% else %}
{% if ipv4_stack %}
- name: node-cidr-mask-size-ipv4
value: "{{ kube_network_node_prefix }}"
{% endif %}
{% if ipv6_stack %}
- name: node-cidr-mask-size-ipv6
value: "{{ kube_network_node_prefix_ipv6 }}"
{% endif %}
{% endif %}
- name: profiling
value: "{{ kube_profiling }}"
- name: terminated-pod-gc-threshold
value: "{{ kube_controller_terminated_pod_gc_threshold }}"
- name: bind-address
value: "{{ kube_controller_manager_bind_address }}"
- name: leader-elect-lease-duration
value: "{{ kube_controller_manager_leader_elect_lease_duration }}"
- name: leader-elect-renew-deadline
value: "{{ kube_controller_manager_leader_elect_renew_deadline }}"
{% if kube_controller_feature_gates or kube_feature_gates %}
- name: feature-gates
value: "{{ kube_controller_feature_gates | default(kube_feature_gates, true) | join(',') }}"
{% endif %}
{% for key in kube_kubeadm_controller_extra_args %}
- name: "{{ key }}"
value: "{{ kube_kubeadm_controller_extra_args[key] }}"
{% endfor %}
{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "gce"] %}
- name: cloud-provider
value: "{{ cloud_provider }}"
- name: cloud-config
value: "{{ kube_config_dir }}/cloud_config"
{% endif %}
{% if kube_network_plugin is defined and kube_network_plugin not in ["cloud"] %}
- name: configure-cloud-routes
value: "false"
{% endif %}
{% if kubelet_flexvolumes_plugins_dir is defined %}
- name: flex-volume-plugin-dir
value: "{{ kubelet_flexvolumes_plugins_dir }}"
{% endif %}
{% if tls_min_version is defined %}
- name: tls-min-version
value: "{{ tls_min_version }}"
{% endif %}
{% if tls_cipher_suites is defined %}
- name: tls-cipher-suites
value: "{% for tls in tls_cipher_suites %}{{ tls }}{{ ',' if not loop.last else '' }}{% endfor %}"
{% endif %}
{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "gce"] or controller_manager_extra_volumes %}
extraVolumes:
{% if cloud_provider is defined and cloud_provider in ["openstack"] and openstack_cacert is defined %}
- name: openstackcacert
hostPath: "{{ kube_config_dir }}/openstack-cacert.pem"
mountPath: "{{ kube_config_dir }}/openstack-cacert.pem"
{% endif %}
{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "gce"] %}
- name: cloud-config
hostPath: {{ kube_config_dir }}/cloud_config
mountPath: {{ kube_config_dir }}/cloud_config
{% endif %}
{% for volume in controller_manager_extra_volumes %}
- name: {{ volume.name }}
hostPath: {{ volume.hostPath }}
mountPath: {{ volume.mountPath }}
readOnly: {{ volume.readOnly | d(not (volume.writable | d(false))) }}
{% endfor %}
{% endif %}
scheduler:
extraArgs:
- name: bind-address
value: "{{ kube_scheduler_bind_address }}"
- name: config
value: "{{ kube_config_dir }}/kubescheduler-config.yaml"
{% if kube_scheduler_feature_gates or kube_feature_gates %}
- name: feature-gates
value: "{{ kube_scheduler_feature_gates | default(kube_feature_gates, true) | join(',') }}"
{% endif %}
- name: profiling
value: "{{ kube_profiling }}"
{% if kube_kubeadm_scheduler_extra_args | length > 0 %}
{% for key in kube_kubeadm_scheduler_extra_args %}
- name: "{{ key }}"
value: "{{ kube_kubeadm_scheduler_extra_args[key] }}"
{% endfor %}
{% endif %}
{% if tls_min_version is defined %}
- name: tls-min-version
value: "{{ tls_min_version }}"
{% endif %}
{% if tls_cipher_suites is defined %}
- name: tls-cipher-suites
value: "{% for tls in tls_cipher_suites %}{{ tls }}{{ ',' if not loop.last else '' }}{% endfor %}"
{% endif %}
extraVolumes:
- name: kubescheduler-config
hostPath: {{ kube_config_dir }}/kubescheduler-config.yaml
mountPath: {{ kube_config_dir }}/kubescheduler-config.yaml
readOnly: true
{% if scheduler_extra_volumes %}
{% for volume in scheduler_extra_volumes %}
- name: {{ volume.name }}
hostPath: {{ volume.hostPath }}
mountPath: {{ volume.mountPath }}
readOnly: {{ volume.readOnly | d(not (volume.writable | d(false))) }}
{% endfor %}
{% endif %}
---
apiVersion: kubeadm.k8s.io/v1beta4
kind: UpgradeConfiguration
apply:
kubernetesVersion: v{{ kube_version }}
allowExperimentalUpgrades: true
certificateRenewal: {{ kubeadm_upgrade_auto_cert_renewal | lower }}
etcdUpgrade: {{ (etcd_deployment_type == "kubeadm") | lower }}
forceUpgrade: true
{% if kubeadm_ignore_preflight_errors | length > 0 %}
ignorePreflightErrors:
{% for ignore_error in kubeadm_ignore_preflight_errors %}
- "{{ ignore_error }}"
{% endfor %}
{% endif %}
{% if kubeadm_patches | length > 0 %}
patches:
directory: {{ kubeadm_patches_dir }}
{% endif %}
imagePullPolicy: {{ k8s_image_pull_policy }}
imagePullSerial: {{ kubeadm_image_pull_serial | lower }}
node:
certificateRenewal: {{ kubeadm_upgrade_auto_cert_renewal | lower }}
etcdUpgrade: {{ (etcd_deployment_type == "kubeadm") | lower }}
{% if kubeadm_ignore_preflight_errors | length > 0 %}
ignorePreflightErrors:
{% for ignore_error in kubeadm_ignore_preflight_errors %}
- "{{ ignore_error }}"
{% endfor %}
{% endif %}
{% if kubeadm_patches | length > 0 %}
patches:
directory: {{ kubeadm_patches_dir }}
{% endif %}
imagePullPolicy: {{ k8s_image_pull_policy }}
imagePullSerial: {{ kubeadm_image_pull_serial | lower }}
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
bindAddress: "{{ kube_proxy_bind_address }}"
clientConnection:
acceptContentTypes: {{ kube_proxy_client_accept_content_types }}
burst: {{ kube_proxy_client_burst }}
contentType: {{ kube_proxy_client_content_type }}
kubeconfig: {{ kube_proxy_client_kubeconfig }}
qps: {{ kube_proxy_client_qps }}
{% if kube_network_plugin is defined and kube_network_plugin not in ["kube-ovn"] %}
clusterCIDR: "{{ kube_pods_subnets }}"
{% endif %}
configSyncPeriod: {{ kube_proxy_config_sync_period }}
conntrack:
maxPerCore: {{ kube_proxy_conntrack_max_per_core }}
min: {{ kube_proxy_conntrack_min }}
tcpCloseWaitTimeout: {{ kube_proxy_conntrack_tcp_close_wait_timeout }}
tcpEstablishedTimeout: {{ kube_proxy_conntrack_tcp_established_timeout }}
enableProfiling: {{ kube_proxy_enable_profiling }}
healthzBindAddress: "{{ kube_proxy_healthz_bind_address }}"
hostnameOverride: "{{ kube_override_hostname }}"
iptables:
masqueradeAll: {{ kube_proxy_masquerade_all }}
masqueradeBit: {{ kube_proxy_masquerade_bit }}
minSyncPeriod: {{ kube_proxy_min_sync_period }}
syncPeriod: {{ kube_proxy_sync_period }}
ipvs:
excludeCIDRs: {{ kube_proxy_exclude_cidrs }}
minSyncPeriod: {{ kube_proxy_min_sync_period }}
scheduler: {{ kube_proxy_scheduler }}
syncPeriod: {{ kube_proxy_sync_period }}
strictARP: {{ kube_proxy_strict_arp }}
tcpTimeout: {{ kube_proxy_tcp_timeout }}
tcpFinTimeout: {{ kube_proxy_tcp_fin_timeout }}
udpTimeout: {{ kube_proxy_udp_timeout }}
metricsBindAddress: "{{ kube_proxy_metrics_bind_address }}"
mode: {{ kube_proxy_mode }}
nodePortAddresses: {{ kube_proxy_nodeport_addresses }}
oomScoreAdj: {{ kube_proxy_oom_score_adj }}
portRange: {{ kube_proxy_port_range }}
{% if kube_proxy_feature_gates or kube_feature_gates %}
{% set feature_gates = ( kube_proxy_feature_gates | default(kube_feature_gates, true) ) %}
featureGates:
{% for feature in feature_gates %}
{{ feature | replace("=", ": ") }}
{% endfor %}
{% endif %}
{# DNS settings for kubelet #}
{% if enable_nodelocaldns %}
{% set kubelet_cluster_dns = [nodelocaldns_ip] %}
{% elif dns_mode in ['coredns'] %}
{% set kubelet_cluster_dns = [skydns_server] %}
{% elif dns_mode == 'coredns_dual' %}
{% set kubelet_cluster_dns = [skydns_server,skydns_server_secondary] %}
{% elif dns_mode == 'manual' %}
{% set kubelet_cluster_dns = [manual_dns_server] %}
{% else %}
{% set kubelet_cluster_dns = [] %}
{% endif %}
---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
clusterDNS:
{% for dns_address in kubelet_cluster_dns %}
- {{ dns_address }}
{% endfor %}
{% if kubelet_feature_gates or kube_feature_gates %}
{% set feature_gates = ( kubelet_feature_gates | default(kube_feature_gates, true) ) %}
featureGates:
{% for feature in feature_gates %}
{{ feature | replace("=", ": ") }}
{% endfor %}
{% endif %}

20
kubespray/project/roles/kubernetes/control-plane/templates/kubeadm-controlplane.v1beta3.yaml.j2 → kubespray/project/roles/kubernetes/control-plane/templates/kubeadm-controlplane.yaml.j2
View File

@@ -1,4 +1,4 @@
apiVersion: kubeadm.k8s.io/v1beta3
apiVersion: kubeadm.k8s.io/{{ kubeadm_config_api_version }}
kind: JoinConfiguration
discovery:
{% if kubeadm_use_file_discovery %}
@@ -9,29 +9,35 @@ discovery:
{% if kubeadm_config_api_fqdn is defined %}
apiServerEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}
{% else %}
apiServerEndpoint: {{ kubeadm_discovery_address }}
apiServerEndpoint: "{{ kubeadm_discovery_address }}"
{% endif %}
token: {{ kubeadm_token }}
unsafeSkipCAVerification: true
{% endif %}
timeout: {{ discovery_timeout }}
tlsBootstrapToken: {{ kubeadm_token }}
{# TODO: drop the if when we drop support for k8s<1.31 #}
{% if kubeadm_config_api_version == 'v1beta3' %}
timeout: {{ discovery_timeout }}
{% else %}
timeouts:
discovery: {{ discovery_timeout }}
{% endif %}
controlPlane:
localAPIEndpoint:
advertiseAddress: {{ kube_apiserver_address }}
advertiseAddress: "{{ kube_apiserver_address }}"
bindPort: {{ kube_apiserver_port }}
certificateKey: {{ kubeadm_certificate_key }}
nodeRegistration:
name: {{ kube_override_hostname | default(inventory_hostname) }}
criSocket: {{ cri_socket }}
{% if inventory_hostname in groups['kube_control_plane'] and inventory_hostname not in groups['kube_node'] %}
{% if 'kube_control_plane' in group_names and 'kube_node' not in group_names %}
taints:
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
{% else %}
taints: []
{% endif %}
{% if kubeadm_patches is defined and kubeadm_patches.enabled %}
{% if kubeadm_patches | length > 0 %}
patches:
directory: {{ kubeadm_patches.dest_dir }}
directory: {{ kubeadm_patches_dir }}
{% endif %}

8
kubespray/project/roles/kubernetes/control-plane/templates/resourcequota.yaml.j2 Normal file
View File

@@ -0,0 +1,8 @@
apiVersion: apiserver.config.k8s.io/v1
kind: ResourceQuotaConfiguration
{% if kube_resource_quota_limited_resources | d(false) -%}
limitedResources:
{{ kube_resource_quota_limited_resources | to_nice_yaml(indent=2, sort_keys=false) }}
{% else %}
# No limitedResources configured. If limitedResources are required, please set kube_resource_quota_limited_resources.
{%- endif %}

8
kubespray/project/roles/kubernetes/control-plane/vars/main.yaml
View File

@@ -1,3 +1,9 @@
---
# list of admission plugins that needs to be configured
kube_apiserver_admission_plugins_needs_configuration: [EventRateLimit, PodSecurity]
# https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/
kube_apiserver_admission_plugins_needs_configuration:
- EventRateLimit
- ImagePolicyWebhook
- PodSecurity
- PodNodeSelector
- ResourceQuota

7
kubespray/project/roles/kubernetes/kubeadm/defaults/main.yml
View File

@@ -6,10 +6,3 @@ kubeadm_join_timeout: 120s
# Enable kubeadm file discovery if anonymous access has been removed
kubeadm_use_file_discovery: "{{ remove_anonymous_access }}"
# If non-empty, will use this string as identification instead of the actual hostname
kube_override_hostname: >-
{%- if cloud_provider is defined and cloud_provider in ['aws'] -%}
{%- else -%}
{{ inventory_hostname }}
{%- endif -%}

3
kubespray/project/roles/kubernetes/kubeadm/meta/main.yml Normal file
View File

@@ -0,0 +1,3 @@
---
dependencies:
- role: kubernetes/kubeadm_common

5
kubespray/project/roles/kubernetes/kubeadm/tasks/kubeadm_etcd_node.yml
View File

@@ -6,9 +6,10 @@
- name: Create kubeadm cert controlplane config
template:
src: "kubeadm-client.conf.{{ kubeadmConfig_api_version }}.j2"
src: "kubeadm-client.conf.j2"
dest: "{{ kube_config_dir }}/kubeadm-cert-controlplane.conf"
mode: "0640"
validate: "{{ kubeadm_config_validate_enabled | ternary(bin_dir + '/kubeadm config validate --config %s', omit) }}"
vars:
kubeadm_cert_controlplane: true
@@ -51,7 +52,7 @@
register: "etcd_client_cert_serial_result"
changed_when: false
when:
- inventory_hostname in groups['k8s_cluster'] | union(groups['calico_rr'] | default([])) | unique | sort
- group_names | intersect(['k8s_cluster', 'calico_rr']) | length > 0
tags:
- network

94
kubespray/project/roles/kubernetes/kubeadm/tasks/main.yml
View File

@@ -4,7 +4,7 @@
# noqa: jinja[spacing]
kubeadm_discovery_address: >-
{%- if "127.0.0.1" in kube_apiserver_endpoint or "localhost" in kube_apiserver_endpoint -%}
{{ first_kube_control_plane_address }}:{{ kube_apiserver_port }}
{{ first_kube_control_plane_address | ansible.utils.ipwrap }}:{{ kube_apiserver_port }}
{%- else -%}
{{ kube_apiserver_endpoint | replace("https://", "") }}
{%- endif %}
@@ -29,17 +29,15 @@
delegate_to: "{{ groups['kube_control_plane'][0] }}"
run_once: true
- name: Calculate kubeadm CA cert hash
shell: set -o pipefail && openssl x509 -pubkey -in {{ kube_cert_dir }}/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //'
args:
executable: /bin/bash
register: kubeadm_ca_hash
- name: Fetch CA certificate from control plane node
slurp:
src: "{{ kube_cert_dir }}/ca.crt"
register: ca_cert_content
when:
- kubeadm_ca_stat.stat is defined
- kubeadm_ca_stat.stat.exists
delegate_to: "{{ groups['kube_control_plane'][0] }}"
run_once: true
changed_when: false
- name: Create kubeadm token for joining nodes with 24h expiration (default)
command: "{{ bin_dir }}/kubeadm token create"
@@ -53,10 +51,6 @@
kubeadm_token: "{{ temp_token.stdout }}"
when: kubeadm_token is not defined
- name: Set kubeadm api version to v1beta3
set_fact:
kubeadmConfig_api_version: v1beta3
- name: Get kubeconfig for join discovery process
command: "{{ kubectl }} -n kube-public get cm cluster-info -o jsonpath='{.data.kubeconfig}'"
register: kubeconfig_file_discovery
@@ -71,69 +65,35 @@
owner: "root"
mode: "0644"
when:
- not is_kube_master
- ('kube_control_plane' not in group_names)
- not kubelet_conf.stat.exists
- kubeadm_use_file_discovery
- name: Create kubeadm client config
template:
src: "kubeadm-client.conf.{{ kubeadmConfig_api_version }}.j2"
src: "kubeadm-client.conf.j2"
dest: "{{ kube_config_dir }}/kubeadm-client.conf"
backup: true
mode: "0640"
when: not is_kube_master
- name: Kubeadm | Create directory to store kubeadm patches
file:
path: "{{ kubeadm_patches.dest_dir }}"
state: directory
mode: "0640"
when: kubeadm_patches is defined and kubeadm_patches.enabled
- name: Kubeadm | Copy kubeadm patches from inventory files
copy:
src: "{{ kubeadm_patches.source_dir }}/"
dest: "{{ kubeadm_patches.dest_dir }}"
owner: "root"
mode: "0644"
when: kubeadm_patches is defined and kubeadm_patches.enabled
validate: "{{ kubeadm_config_validate_enabled | ternary(bin_dir + '/kubeadm config validate --config %s', omit) }}"
when: ('kube_control_plane' not in group_names)
- name: Join to cluster if needed
environment:
PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}:/sbin"
when: not is_kube_master and (not kubelet_conf.stat.exists)
block:
- name: Join to cluster
command: >-
timeout -k {{ kubeadm_join_timeout }} {{ kubeadm_join_timeout }}
{{ bin_dir }}/kubeadm join
--config {{ kube_config_dir }}/kubeadm-client.conf
--ignore-preflight-errors=DirAvailable--etc-kubernetes-manifests
--skip-phases={{ kubeadm_join_phases_skip | join(',') }}
register: kubeadm_join
changed_when: kubeadm_join is success
rescue:
- name: Join to cluster with ignores
command: >-
timeout -k {{ kubeadm_join_timeout }} {{ kubeadm_join_timeout }}
{{ bin_dir }}/kubeadm join
--config {{ kube_config_dir }}/kubeadm-client.conf
--ignore-preflight-errors=all
--skip-phases={{ kubeadm_join_phases_skip | join(',') }}
register: kubeadm_join
changed_when: kubeadm_join is success
always:
- name: Display kubeadm join stderr if any
when: kubeadm_join is failed
debug:
msg: |
Joined with warnings
{{ kubeadm_join.stderr_lines }}
when:
- ('kube_control_plane' not in group_names)
- not kubelet_conf.stat.exists
vars:
ignored:
- DirAvailable--etc-kubernetes-manifests
- "{{ kubeadm_ignore_preflight_errors }}"
command: >-
timeout -k {{ kubeadm_join_timeout }} {{ kubeadm_join_timeout }}
{{ bin_dir }}/kubeadm join
--config {{ kube_config_dir }}/kubeadm-client.conf
--ignore-preflight-errors={{ ignored | flatten | join(',') }}
--skip-phases={{ kubeadm_join_phases_skip | join(',') }}
- name: Update server field in kubelet kubeconfig
lineinfile:
@@ -143,7 +103,7 @@
backup: true
when:
- kubeadm_config_api_fqdn is not defined
- not is_kube_master
- ('kube_control_plane' not in group_names)
- kubeadm_discovery_address != kube_apiserver_endpoint | replace("https://", "")
notify: Kubeadm | restart kubelet
@@ -154,7 +114,7 @@
line: ' server: {{ kube_apiserver_endpoint }}'
backup: true
when:
- not is_kube_master
- ('kube_control_plane' not in group_names)
- loadbalancer_apiserver is defined
notify: Kubeadm | restart kubelet
@@ -169,8 +129,8 @@
tags:
- kube-proxy
# FIXME(mattymo): Need to point to localhost, otherwise masters will all point
# incorrectly to first master, creating SPoF.
# FIXME(mattymo): Need to point to localhost, otherwise control plane nodes will all point
# incorrectly to first control plane node, creating SPoF.
- name: Update server field in kube-proxy kubeconfig
shell: >-
set -o pipefail && {{ kubectl }} get configmap kube-proxy -n kube-system -o yaml
@@ -241,5 +201,5 @@
when:
- etcd_deployment_type == "kubeadm"
- inventory_hostname not in groups['kube_control_plane']
- kube_network_plugin in ["calico", "flannel", "cilium"] or cilium_deploy_additionally | default(false) | bool
- kube_network_plugin in ["calico", "flannel", "cilium"] or cilium_deploy_additionally
- kube_network_plugin != "calico" or calico_datastore == "etcd"

24
kubespray/project/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta3.j2 → kubespray/project/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.j2
View File

@@ -1,5 +1,5 @@
---
apiVersion: kubeadm.k8s.io/v1beta3
apiVersion: kubeadm.k8s.io/{{ kubeadm_config_api_version }}
kind: JoinConfiguration
discovery:
{% if kubeadm_use_file_discovery %}
@@ -8,25 +8,31 @@ discovery:
{% else %}
bootstrapToken:
{% if kubeadm_config_api_fqdn is defined %}
apiServerEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}
apiServerEndpoint: "{{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}"
{% else %}
apiServerEndpoint: {{ kubeadm_discovery_address }}
apiServerEndpoint: "{{ kubeadm_discovery_address }}"
{% endif %}
token: {{ kubeadm_token }}
{% if kubeadm_ca_hash.stdout is defined %}
{% if ca_cert_content is defined %}
caCertHashes:
- sha256:{{ kubeadm_ca_hash.stdout }}
- sha256:{{ (ca_cert_content.content | b64decode | community.crypto.x509_certificate_info).public_key_fingerprints.sha256.replace(':', '') }}
{% else %}
unsafeSkipCAVerification: true
{% endif %}
{% endif %}
timeout: {{ discovery_timeout }}
tlsBootstrapToken: {{ kubeadm_token }}
{# TODO: drop the if when we drop support for k8s<1.31 #}
{% if kubeadm_config_api_version == 'v1beta3' %}
timeout: {{ discovery_timeout }}
{% else %}
timeouts:
discovery: {{ discovery_timeout }}
{% endif %}
caCertPath: {{ kube_cert_dir }}/ca.crt
{% if kubeadm_cert_controlplane is defined and kubeadm_cert_controlplane %}
controlPlane:
localAPIEndpoint:
advertiseAddress: {{ kube_apiserver_address }}
advertiseAddress: "{{ kube_apiserver_address }}"
bindPort: {{ kube_apiserver_port }}
certificateKey: {{ kubeadm_certificate_key }}
{% endif %}
@@ -38,7 +44,7 @@ nodeRegistration:
- effect: NoSchedule
key: node-role.kubernetes.io/calico-rr
{% endif %}
{% if kubeadm_patches is defined and kubeadm_patches.enabled %}
{% if kubeadm_patches | length > 0 %}
patches:
directory: {{ kubeadm_patches.dest_dir }}
directory: {{ kubeadm_patches_dir }}
{% endif %}

23
kubespray/project/roles/kubernetes/kubeadm_common/defaults/main.yml Normal file
View File

@@ -0,0 +1,23 @@
---
kubeadm_patches_dir: "{{ kube_config_dir }}/patches"
kubeadm_patches: []
# See https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/control-plane-flags/#patches
# Correspondance with this link
# patchtype = type
# target = target
# suffix -> managed automatically
# extension -> always "yaml"
# kubeadm_patches:
# - target: kube-apiserver|kube-controller-manager|kube-scheduler|etcd|kubeletconfiguration
# type: strategic(default)|json|merge
# patch:
# metadata:
# annotations:
# example.com/test: "true"
# labels:
# example.com/prod_level: "{{ prod_level }}"
# - ...
# Patches are applied in the order they are specified.
# List of errors to ignore during kubeadm preflight checks
kubeadm_ignore_preflight_errors: []

17
kubespray/project/roles/kubernetes/kubeadm_common/tasks/main.yml Normal file
View File

@@ -0,0 +1,17 @@
---
- name: Kubeadm | Create directory to store kubeadm patches
file:
path: "{{ kubeadm_patches_dir }}"
state: directory
mode: "0640"
when: kubeadm_patches | length > 0
- name: Kubeadm | Copy kubeadm patches from inventory files
copy:
content: "{{ item.patch | to_yaml }}"
dest: "{{ kubeadm_patches_dir }}/{{ item.target }}{{ suffix }}+{{ item.type | d('strategic') }}.yaml"
owner: "root"
mode: "0644"
loop: "{{ kubeadm_patches }}"
loop_control:
index_var: suffix

118
kubespray/project/roles/kubernetes/node/defaults/main.yml
View File

@@ -1,15 +1,16 @@
---
# advertised host IP for kubelet. This affects network plugin config. Take caution
kubelet_address: "{{ ip | default(fallback_ips[inventory_hostname]) }}{{ (',' + ip6) if enable_dual_stack_networks and ip6 is defined else '' }}"
# add ipv6 manual for dualstack mode because ipv4 priority in main_ip for dualstack
kubelet_address: "{{ main_ips | join(',') }}"
# bind address for kubelet. Set to 0.0.0.0 to listen on all interfaces
kubelet_bind_address: "{{ ip | default('0.0.0.0') }}"
# bind address for kubelet. Set to :: to listen on all interfaces
kubelet_bind_address: "{{ main_ip | default('::') }}"
# resolv.conf to base dns config
kube_resolv_conf: "/etc/resolv.conf"
# Set to empty to avoid cgroup creation
kubelet_enforce_node_allocatable: '""'
kubelet_enforce_node_allocatable: "\"\""
# Set runtime and kubelet cgroups when using systemd as cgroup driver (default)
kube_service_cgroups: "{% if kube_reserved %}{{ kube_reserved_cgroups_for_service_slice }}{% else %}system.slice{% endif %}"
@@ -27,39 +28,30 @@ kubelet_systemd_hardening: false
kubelet_systemd_wants_dependencies: []
# List of secure IPs for kubelet
# don't forget ipv6 addresses for dualstack(because "main_ip" prioritizes ipv4)
kube_node_addresses: >-
{%- for host in (groups['kube_control_plane'] + groups['kube_node'] + groups['etcd']) | unique -%}
{{ hostvars[host]['ip'] | default(fallback_ips[host]) }}{{ ' ' if not loop.last else '' }}
{%- for host in (groups['k8s_cluster'] | union(groups['etcd'])) -%}
{{ hostvars[host]['main_ips'] | join(' ') }}{{ ' ' if not loop.last else '' }}
{%- endfor -%}
kubelet_secure_addresses: "localhost link-local {{ kube_pods_subnet }} {{ kube_node_addresses }}"
kubelet_secure_addresses: "localhost link-local {{ kube_pods_subnets | regex_replace(',', ' ') }} {{ kube_node_addresses }}"
# Reserve this space for kube resources
# Whether to run kubelet and container-engine daemons in a dedicated cgroup. (Not required for resource reservations).
kube_reserved: false
kube_reserved_cgroups_for_service_slice: kube.slice
kube_reserved_cgroups: "/{{ kube_reserved_cgroups_for_service_slice }}"
kube_memory_reserved: 256Mi
kube_cpu_reserved: 100m
# kube_ephemeral_storage_reserved: 2Gi
# kube_pid_reserved: "1000"
# Reservation for master hosts
kube_master_memory_reserved: 512Mi
kube_master_cpu_reserved: 200m
# kube_master_ephemeral_storage_reserved: 2Gi
# kube_master_pid_reserved: "1000"
kube_memory_reserved: "256Mi"
kube_cpu_reserved: "100m"
kube_ephemeral_storage_reserved: "500Mi"
kube_pid_reserved: "1000"
# Set to true to reserve resources for system daemons
system_reserved: false
system_reserved_cgroups_for_service_slice: system.slice
system_reserved_cgroups: "/{{ system_reserved_cgroups_for_service_slice }}"
system_memory_reserved: 512Mi
system_cpu_reserved: 500m
# system_ephemeral_storage_reserved: 2Gi
# system_pid_reserved: "1000"
# Reservation for master hosts
system_master_memory_reserved: 256Mi
system_master_cpu_reserved: 250m
# system_master_ephemeral_storage_reserved: 2Gi
# system_master_pid_reserved: "1000"
system_memory_reserved: "512Mi"
system_cpu_reserved: "500m"
system_ephemeral_storage_reserved: "500Mi"
system_pid_reserved: 1000
## Eviction Thresholds to avoid system OOMs
# https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/#eviction-thresholds
@@ -69,7 +61,7 @@ eviction_hard_control_plane: {}
kubelet_status_update_frequency: 10s
# kube-vip
kube_vip_version: v0.8.0
kube_vip_version: 0.8.0
kube_vip_arp_enabled: false
kube_vip_interface:
@@ -91,7 +83,6 @@ kube_vip_bgppeers:
kube_vip_address:
kube_vip_enableServicesElection: false
kube_vip_lb_enable: false
kube_vip_lb_fwdmethod: local
kube_vip_leasename: plndr-cp-lock
kube_vip_svc_leasename: plndr-svcs-lock
kube_vip_leaseduration: 5
@@ -135,9 +126,6 @@ kubelet_config_extra_args_cgroupfs:
systemCgroups: /system.slice
cgroupRoot: /
## Support parameters to be passed to kubelet via kubelet-config.yaml only on nodes, not masters
kubelet_node_config_extra_args: {}
# Maximum number of container log files that can be present for a container.
kubelet_logfiles_max_nr: 5
@@ -147,16 +135,6 @@ kubelet_logfiles_max_size: 10Mi
## Support custom flags to be passed to kubelet
kubelet_custom_flags: []
## Support custom flags to be passed to kubelet only on nodes, not masters
kubelet_node_custom_flags: []
# If non-empty, will use this string as identification instead of the actual hostname
kube_override_hostname: >-
{%- if cloud_provider is defined and cloud_provider in ['aws'] -%}
{%- else -%}
{{ inventory_hostname }}
{%- endif -%}
# The read-only port for the Kubelet to serve on with no authentication/authorization.
kube_read_only_port: 0
@@ -169,61 +147,6 @@ kubelet_healthz_bind_address: 127.0.0.1
# sysctl_file_path to add sysctl conf to
sysctl_file_path: "/etc/sysctl.d/99-sysctl.conf"
# For the openstack integration kubelet will need credentials to access
# openstack apis like nova and cinder. Per default this values will be
# read from the environment.
openstack_auth_url: "{{ lookup('env', 'OS_AUTH_URL') }}"
openstack_username: "{{ lookup('env', 'OS_USERNAME') }}"
openstack_password: "{{ lookup('env', 'OS_PASSWORD') }}"
openstack_region: "{{ lookup('env', 'OS_REGION_NAME') }}"
openstack_tenant_id: "{{ lookup('env', 'OS_TENANT_ID') | default(lookup('env', 'OS_PROJECT_ID') | default(lookup('env', 'OS_PROJECT_NAME'), true), true) }}"
openstack_tenant_name: "{{ lookup('env', 'OS_TENANT_NAME') }}"
openstack_domain_name: "{{ lookup('env', 'OS_USER_DOMAIN_NAME') }}"
openstack_domain_id: "{{ lookup('env', 'OS_USER_DOMAIN_ID') }}"
# For the vsphere integration, kubelet will need credentials to access
# vsphere apis
# Documentation regarding these values can be found
# https://github.com/kubernetes/kubernetes/blob/master/pkg/cloudprovider/providers/vsphere/vsphere.go#L105
vsphere_vcenter_ip: "{{ lookup('env', 'VSPHERE_VCENTER') }}"
vsphere_vcenter_port: "{{ lookup('env', 'VSPHERE_VCENTER_PORT') }}"
vsphere_user: "{{ lookup('env', 'VSPHERE_USER') }}"
vsphere_password: "{{ lookup('env', 'VSPHERE_PASSWORD') }}"
vsphere_datacenter: "{{ lookup('env', 'VSPHERE_DATACENTER') }}"
vsphere_datastore: "{{ lookup('env', 'VSPHERE_DATASTORE') }}"
vsphere_working_dir: "{{ lookup('env', 'VSPHERE_WORKING_DIR') }}"
vsphere_insecure: "{{ lookup('env', 'VSPHERE_INSECURE') }}"
vsphere_resource_pool: "{{ lookup('env', 'VSPHERE_RESOURCE_POOL') }}"
vsphere_scsi_controller_type: pvscsi
# vsphere_public_network is name of the network the VMs are joined to
vsphere_public_network: "{{ lookup('env', 'VSPHERE_PUBLIC_NETWORK') | default('') }}"
## When azure is used, you need to also set the following variables.
## see docs/azure.md for details on how to get these values
# azure_tenant_id:
# azure_subscription_id:
# azure_aad_client_id:
# azure_aad_client_secret:
# azure_resource_group:
# azure_location:
# azure_subnet_name:
# azure_security_group_name:
# azure_vnet_name:
# azure_route_table_name:
# supported values are 'standard' or 'vmss'
# azure_vmtype: standard
# Sku of Load Balancer and Public IP. Candidate values are: basic and standard.
azure_loadbalancer_sku: basic
# excludes master nodes from standard load balancer.
azure_exclude_master_from_standard_lb: true
# disables the outbound SNAT for public load balancer rules
azure_disable_outbound_snat: false
# use instance metadata service where possible
azure_use_instance_metadata: true
# use specific Azure API endpoints
azure_cloud: AzurePublicCloud
## Support tls min version, Possible values: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13.
# tls_min_version: ""
@@ -265,9 +188,10 @@ conntrack_modules:
- nf_conntrack
- nf_conntrack_ipv4
## Enable distributed tracing for kubelet
kubelet_tracing: false
kubelet_tracing_endpoint: 0.0.0.0:4317
kubelet_tracing_endpoint: "[::]:4317"
kubelet_tracing_sampling_rate_per_million: 100
# The maximum number of image pulls in parallel. Set it to a integer great than 1 to enable image pulling in parallel.

82
kubespray/project/roles/kubernetes/node/tasks/cloud-credentials/azure-credential-check.yml
View File

@@ -1,82 +0,0 @@
---
- name: Check azure_tenant_id value
fail:
msg: "azure_tenant_id is missing"
when: azure_tenant_id is not defined or not azure_tenant_id
- name: Check azure_subscription_id value
fail:
msg: "azure_subscription_id is missing"
when: azure_subscription_id is not defined or not azure_subscription_id
- name: Check azure_aad_client_id value
fail:
msg: "azure_aad_client_id is missing"
when: azure_aad_client_id is not defined or not azure_aad_client_id
- name: Check azure_aad_client_secret value
fail:
msg: "azure_aad_client_secret is missing"
when: azure_aad_client_secret is not defined or not azure_aad_client_secret
- name: Check azure_resource_group value
fail:
msg: "azure_resource_group is missing"
when: azure_resource_group is not defined or not azure_resource_group
- name: Check azure_location value
fail:
msg: "azure_location is missing"
when: azure_location is not defined or not azure_location
- name: Check azure_subnet_name value
fail:
msg: "azure_subnet_name is missing"
when: azure_subnet_name is not defined or not azure_subnet_name
- name: Check azure_security_group_name value
fail:
msg: "azure_security_group_name is missing"
when: azure_security_group_name is not defined or not azure_security_group_name
- name: Check azure_vnet_name value
fail:
msg: "azure_vnet_name is missing"
when: azure_vnet_name is not defined or not azure_vnet_name
- name: Check azure_vnet_resource_group value
fail:
msg: "azure_vnet_resource_group is missing"
when: azure_vnet_resource_group is not defined or not azure_vnet_resource_group
- name: Check azure_route_table_name value
fail:
msg: "azure_route_table_name is missing"
when: azure_route_table_name is not defined or not azure_route_table_name
- name: Check azure_loadbalancer_sku value
fail:
msg: "azure_loadbalancer_sku has an invalid value '{{ azure_loadbalancer_sku }}'. Supported values are 'basic', 'standard'"
when: azure_loadbalancer_sku not in ["basic", "standard"]
- name: "Check azure_exclude_master_from_standard_lb is a bool"
assert:
that: azure_exclude_master_from_standard_lb | type_debug == 'bool'
- name: "Check azure_disable_outbound_snat is a bool"
assert:
that: azure_disable_outbound_snat | type_debug == 'bool'
- name: "Check azure_use_instance_metadata is a bool"
assert:
that: azure_use_instance_metadata | type_debug == 'bool'
- name: Check azure_vmtype value
fail:
msg: "azure_vmtype is missing. Supported values are 'standard' or 'vmss'"
when: azure_vmtype is not defined or not azure_vmtype
- name: Check azure_cloud value
fail:
msg: "azure_cloud has an invalid value '{{ azure_cloud }}'. Supported values are 'AzureChinaCloud', 'AzureGermanCloud', 'AzurePublicCloud', 'AzureUSGovernmentCloud'."
when: azure_cloud not in ["AzureChinaCloud", "AzureGermanCloud", "AzurePublicCloud", "AzureUSGovernmentCloud"]

34
kubespray/project/roles/kubernetes/node/tasks/cloud-credentials/openstack-credential-check.yml
View File

@@ -1,34 +0,0 @@
---
- name: Check openstack_auth_url value
fail:
msg: "openstack_auth_url is missing"
when: openstack_auth_url is not defined or not openstack_auth_url
- name: Check openstack_username value
fail:
msg: "openstack_username is missing"
when: openstack_username is not defined or not openstack_username
- name: Check openstack_password value
fail:
msg: "openstack_password is missing"
when: openstack_password is not defined or not openstack_password
- name: Check openstack_region value
fail:
msg: "openstack_region is missing"
when: openstack_region is not defined or not openstack_region
- name: Check openstack_tenant_id value
fail:
msg: "one of openstack_tenant_id or openstack_trust_id must be specified"
when:
- openstack_tenant_id is not defined or not openstack_tenant_id
- openstack_trust_id is not defined
- name: Check openstack_trust_id value
fail:
msg: "one of openstack_tenant_id or openstack_trust_id must be specified"
when:
- openstack_trust_id is not defined or not openstack_trust_id
- openstack_tenant_id is not defined

22
kubespray/project/roles/kubernetes/node/tasks/cloud-credentials/vsphere-credential-check.yml
View File

@@ -1,22 +0,0 @@
---
- name: Check vsphere environment variables
fail:
msg: "{{ item.name }} is missing"
when: item.value is not defined or not item.value
with_items:
- name: vsphere_vcenter_ip
value: "{{ vsphere_vcenter_ip }}"
- name: vsphere_vcenter_port
value: "{{ vsphere_vcenter_port }}"
- name: vsphere_user
value: "{{ vsphere_user }}"
- name: vsphere_password
value: "{{ vsphere_password }}"
- name: vsphere_datacenter
value: "{{ vsphere_datacenter }}"
- name: vsphere_datastore
value: "{{ vsphere_datastore }}"
- name: vsphere_working_dir
value: "{{ vsphere_working_dir }}"
- name: vsphere_insecure
value: "{{ vsphere_insecure }}"

2
kubespray/project/roles/kubernetes/node/tasks/install.yml
View File

@@ -8,7 +8,7 @@
tags:
- kubeadm
when:
- not inventory_hostname in groups['kube_control_plane']
- not ('kube_control_plane' in group_names)
- name: Install | Copy kubelet binary from download dir
copy:

63
kubespray/project/roles/kubernetes/node/tasks/main.yml
View File

@@ -5,11 +5,6 @@
- facts
- kubelet
- name: Pre-upgrade kubelet
import_tasks: pre_upgrade.yml
tags:
- kubelet
- name: Ensure /var/lib/cni exists
file:
path: /var/lib/cni
@@ -24,7 +19,7 @@
- name: Install kube-vip
import_tasks: loadbalancer/kube-vip.yml
when:
- is_kube_master
- ('kube_control_plane' in group_names)
- kube_vip_enabled
tags:
- kube-vip
@@ -32,7 +27,7 @@
- name: Install nginx-proxy
import_tasks: loadbalancer/nginx-proxy.yml
when:
- not is_kube_master or kube_apiserver_bind_address != '0.0.0.0'
- ('kube_control_plane' not in group_names) or (kube_apiserver_bind_address != '::')
- loadbalancer_apiserver_localhost
- loadbalancer_apiserver_type == 'nginx'
tags:
@@ -41,7 +36,7 @@
- name: Install haproxy
import_tasks: loadbalancer/haproxy.yml
when:
- not is_kube_master or kube_apiserver_bind_address != '0.0.0.0'
- ('kube_control_plane' not in group_names) or (kube_apiserver_bind_address != '::')
- loadbalancer_apiserver_localhost
- loadbalancer_apiserver_type == 'haproxy'
tags:
@@ -137,52 +132,14 @@
tags:
- kube-proxy
- name: Check cloud provider credentials
include_tasks: "cloud-credentials/{{ cloud_provider }}-credential-check.yml"
when:
- cloud_provider is defined
- cloud_provider in [ 'openstack', 'azure', 'vsphere' ]
- name: Modprobe Kernel Module for nftables
community.general.modprobe:
name: "nf_tables"
state: present
persistent: present
when: kube_proxy_mode == 'nftables'
tags:
- cloud-provider
- facts
- name: Test if openstack_cacert is a base64 string
set_fact:
openstack_cacert_is_base64: "{% if openstack_cacert is search('^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{3}= | [A-Za-z0-9+/]{2}==)?$') %}true{% else %}false{% endif %}"
when:
- cloud_provider is defined
- cloud_provider == 'openstack'
- openstack_cacert is defined
- openstack_cacert | length > 0
- name: Write cacert file
copy:
src: "{{ openstack_cacert if not openstack_cacert_is_base64 else omit }}"
content: "{{ openstack_cacert | b64decode if openstack_cacert_is_base64 else omit }}"
dest: "{{ kube_config_dir }}/openstack-cacert.pem"
group: "{{ kube_cert_group }}"
mode: "0640"
when:
- cloud_provider is defined
- cloud_provider == 'openstack'
- openstack_cacert is defined
- openstack_cacert | length > 0
tags:
- cloud-provider
- name: Write cloud-config
template:
src: "cloud-configs/{{ cloud_provider }}-cloud-config.j2"
dest: "{{ kube_config_dir }}/cloud_config"
group: "{{ kube_cert_group }}"
mode: "0640"
when:
- cloud_provider is defined
- cloud_provider in [ 'openstack', 'azure', 'vsphere', 'aws', 'gce' ]
notify: Node | restart kubelet
tags:
- cloud-provider
- kube-proxy
- name: Install kubelet
import_tasks: kubelet.yml

48
kubespray/project/roles/kubernetes/node/tasks/pre_upgrade.yml
View File

@@ -1,48 +0,0 @@
---
- name: "Pre-upgrade | check if kubelet container exists"
shell: >-
set -o pipefail &&
{% if container_manager in ['crio', 'docker'] %}
{{ docker_bin_dir }}/docker ps -af name=kubelet | grep kubelet
{% elif container_manager == 'containerd' %}
{{ bin_dir }}/crictl ps --all --name kubelet | grep kubelet
{% endif %}
args:
executable: /bin/bash
failed_when: false
changed_when: false
check_mode: false
register: kubelet_container_check
- name: "Pre-upgrade | copy /var/lib/cni from kubelet"
command: >-
{% if container_manager in ['crio', 'docker'] %}
docker cp kubelet:/var/lib/cni /var/lib/cni
{% elif container_manager == 'containerd' %}
ctr run --rm --mount type=bind,src=/var/lib/cni,dst=/cnilibdir,options=rbind:rw kubelet kubelet-tmp sh -c 'cp /var/lib/cni/* /cnilibdir/'
{% endif %}
args:
creates: "/var/lib/cni"
failed_when: false
when: kubelet_container_check.rc == 0
- name: "Pre-upgrade | ensure kubelet container service is stopped if using host deployment"
service:
name: kubelet
state: stopped
when: kubelet_container_check.rc == 0
- name: "Pre-upgrade | ensure kubelet container is removed if using host deployment"
shell: >-
{% if container_manager in ['crio', 'docker'] %}
{{ docker_bin_dir }}/docker rm -fv kubelet
{% elif container_manager == 'containerd' %}
{{ bin_dir }}/crictl stop kubelet && {{ bin_dir }}/crictl rm kubelet
{% endif %}
failed_when: false
changed_when: false
register: remove_kubelet_container
retries: 4
until: remove_kubelet_container.rc == 0
delay: 5
when: kubelet_container_check.rc == 0

11
kubespray/project/roles/kubernetes/node/templates/cloud-configs/aws-cloud-config.j2
View File

@@ -1,11 +0,0 @@
[Global]
zone={{ aws_zone|default("") }}
vpc={{ aws_vpc|default("") }}
subnetId={{ aws_subnet_id|default("") }}
routeTableId={{ aws_route_table_id|default("") }}
roleArn={{ aws_role_arn|default("") }}
kubernetesClusterTag={{ aws_kubernetes_cluster_tag|default("") }}
kubernetesClusterId={{ aws_kubernetes_cluster_id|default("") }}
disableSecurityGroupIngress={{ "true" if aws_disable_security_group_ingress|default(False) else "false" }}
disableStrictZoneCheck={{ "true" if aws_disable_strict_zone_check|default(False) else "false" }}
elbSecurityGroup={{ aws_elb_security_group|default("") }}

26
kubespray/project/roles/kubernetes/node/templates/cloud-configs/azure-cloud-config.j2
View File

@@ -1,26 +0,0 @@
{
"cloud": "{{ azure_cloud }}",
"tenantId": "{{ azure_tenant_id }}",
"subscriptionId": "{{ azure_subscription_id }}",
"aadClientId": "{{ azure_aad_client_id }}",
"aadClientSecret": "{{ azure_aad_client_secret }}",
"resourceGroup": "{{ azure_resource_group }}",
"location": "{{ azure_location }}",
"subnetName": "{{ azure_subnet_name }}",
"securityGroupName": "{{ azure_security_group_name }}",
"securityGroupResourceGroup": "{{ azure_security_group_resource_group | default(azure_vnet_resource_group) }}",
"vnetName": "{{ azure_vnet_name }}",
"vnetResourceGroup": "{{ azure_vnet_resource_group }}",
"routeTableName": "{{ azure_route_table_name }}",
"routeTableResourceGroup": "{{ azure_route_table_resource_group | default(azure_vnet_resource_group) }}",
"vmType": "{{ azure_vmtype }}",
{% if azure_primary_availability_set_name is defined %}
"primaryAvailabilitySetName": "{{ azure_primary_availability_set_name }}",
{%endif%}
"useInstanceMetadata": {{azure_use_instance_metadata | lower }},
{% if azure_loadbalancer_sku == "standard" %}
"excludeMasterFromStandardLB": {{ azure_exclude_master_from_standard_lb | lower }},
"disableOutboundSNAT": {{ azure_disable_outbound_snat | lower }},
{% endif%}
"loadBalancerSku": "{{ azure_loadbalancer_sku }}"
}

2
kubespray/project/roles/kubernetes/node/templates/cloud-configs/gce-cloud-config.j2
View File

@@ -1,2 +0,0 @@
[global]
node-tags = {{ gce_node_tags }}

54
kubespray/project/roles/kubernetes/node/templates/cloud-configs/openstack-cloud-config.j2
View File

@@ -1,54 +0,0 @@
[Global]
auth-url="{{ openstack_auth_url }}"
username="{{ openstack_username }}"
password="{{ openstack_password }}"
region="{{ openstack_region }}"
{% if openstack_trust_id is defined and openstack_trust_id != "" %}
trust-id="{{ openstack_trust_id }}"
{% else %}
tenant-id="{{ openstack_tenant_id }}"
{% endif %}
{% if openstack_tenant_name is defined and openstack_tenant_name != "" %}
tenant-name="{{ openstack_tenant_name }}"
{% endif %}
{% if openstack_domain_name is defined and openstack_domain_name != "" %}
domain-name="{{ openstack_domain_name }}"
{% elif openstack_domain_id is defined and openstack_domain_id != "" %}
domain-id ="{{ openstack_domain_id }}"
{% endif %}
{% if openstack_cacert is defined and openstack_cacert != "" %}
ca-file="{{ kube_config_dir }}/openstack-cacert.pem"
{% endif %}
[BlockStorage]
{% if openstack_blockstorage_version is defined %}
bs-version={{ openstack_blockstorage_version }}
{% endif %}
{% if openstack_blockstorage_ignore_volume_az is defined and openstack_blockstorage_ignore_volume_az|bool %}
ignore-volume-az={{ openstack_blockstorage_ignore_volume_az }}
{% endif %}
{% if node_volume_attach_limit is defined and node_volume_attach_limit != "" %}
node-volume-attach-limit="{{ node_volume_attach_limit }}"
{% endif %}
{% if openstack_lbaas_enabled and openstack_lbaas_subnet_id is defined %}
[LoadBalancer]
subnet-id={{ openstack_lbaas_subnet_id }}
{% if openstack_lbaas_floating_network_id is defined %}
floating-network-id={{ openstack_lbaas_floating_network_id }}
{% endif %}
{% if openstack_lbaas_use_octavia is defined %}
use-octavia={{ openstack_lbaas_use_octavia }}
{% endif %}
{% if openstack_lbaas_method is defined %}
lb-method={{ openstack_lbaas_method }}
{% endif %}
{% if openstack_lbaas_provider is defined %}
lb-provider={{ openstack_lbaas_provider }}
{% endif %}
create-monitor={{ openstack_lbaas_create_monitor }}
monitor-delay={{ openstack_lbaas_monitor_delay }}
monitor-timeout={{ openstack_lbaas_monitor_timeout }}
monitor-max-retries={{ openstack_lbaas_monitor_max_retries }}
{% endif %}

36
kubespray/project/roles/kubernetes/node/templates/cloud-configs/vsphere-cloud-config.j2
View File

@@ -1,36 +0,0 @@
[Global]
user = "{{ vsphere_user }}"
password = "{{ vsphere_password }}"
port = {{ vsphere_vcenter_port }}
insecure-flag = {{ vsphere_insecure }}
datacenters = "{{ vsphere_datacenter }}"
[VirtualCenter "{{ vsphere_vcenter_ip }}"]
[Workspace]
server = "{{ vsphere_vcenter_ip }}"
datacenter = "{{ vsphere_datacenter }}"
folder = "{{ vsphere_working_dir }}"
default-datastore = "{{ vsphere_datastore }}"
{% if vsphere_resource_pool is defined and vsphere_resource_pool != "" %}
resourcepool-path = "{{ vsphere_resource_pool }}"
{% endif %}
[Disk]
scsicontrollertype = {{ vsphere_scsi_controller_type }}
{% if vsphere_public_network is defined and vsphere_public_network != "" %}
[Network]
public-network = {{ vsphere_public_network }}
{% endif %}
[Labels]
{% if vsphere_zone_category is defined and vsphere_zone_category != "" %}
zone = {{ vsphere_zone_category }}
{% endif %}
{% if vsphere_region_category is defined and vsphere_region_category != "" %}
region = {{ vsphere_region_category }}
{% endif %}

70
kubespray/project/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2
View File

@@ -26,12 +26,13 @@ staticPodPath: {{ kube_manifest_dir }}
cgroupDriver: {{ kubelet_cgroup_driver | default('systemd') }}
containerLogMaxFiles: {{ kubelet_logfiles_max_nr }}
containerLogMaxSize: {{ kubelet_logfiles_max_size }}
containerRuntimeEndpoint : {{ cri_socket }}
maxPods: {{ kubelet_max_pods }}
podPidsLimit: {{ kubelet_pod_pids_limit }}
address: {{ kubelet_bind_address }}
address: "{{ kubelet_bind_address }}"
readOnlyPort: {{ kube_read_only_port }}
healthzPort: {{ kubelet_healthz_port }}
healthzBindAddress: {{ kubelet_healthz_bind_address }}
healthzBindAddress: "{{ kubelet_healthz_bind_address }}"
kubeletCgroups: {{ kubelet_kubelet_cgroups }}
clusterDomain: {{ dns_domain }}
{% if kubelet_protect_kernel_defaults | bool %}
@@ -60,56 +61,16 @@ clusterDNS:
- {{ dns_address }}
{% endfor %}
{# Node reserved CPU/memory #}
{% if kube_reserved | bool %}
kubeReservedCgroup: {{ kube_reserved_cgroups }}
{% for scope in "kube", "system" %}
{% if lookup('ansible.builtin.vars', scope + "_reserved") | bool %}
{{ scope }}ReservedCgroup: {{ lookup('ansible.builtin.vars', scope + '_reserved_cgroups') }}
{% endif %}
kubeReserved:
{% if is_kube_master | bool %}
cpu: "{{ kube_master_cpu_reserved }}"
memory: {{ kube_master_memory_reserved }}
{% if kube_master_ephemeral_storage_reserved is defined %}
ephemeral-storage: {{ kube_master_ephemeral_storage_reserved }}
{% endif %}
{% if kube_master_pid_reserved is defined %}
pid: "{{ kube_master_pid_reserved }}"
{% endif %}
{% else %}
cpu: "{{ kube_cpu_reserved }}"
memory: {{ kube_memory_reserved }}
{% if kube_ephemeral_storage_reserved is defined %}
ephemeral-storage: {{ kube_ephemeral_storage_reserved }}
{% endif %}
{% if kube_pid_reserved is defined %}
pid: "{{ kube_pid_reserved }}"
{% endif %}
{% endif %}
{% if system_reserved | bool %}
systemReservedCgroup: {{ system_reserved_cgroups }}
systemReserved:
{% if is_kube_master | bool %}
cpu: "{{ system_master_cpu_reserved }}"
memory: {{ system_master_memory_reserved }}
{% if system_master_ephemeral_storage_reserved is defined %}
ephemeral-storage: {{ system_master_ephemeral_storage_reserved }}
{% endif %}
{% if system_master_pid_reserved is defined %}
pid: "{{ system_master_pid_reserved }}"
{% endif %}
{% else %}
cpu: "{{ system_cpu_reserved }}"
memory: {{ system_memory_reserved }}
{% if system_ephemeral_storage_reserved is defined %}
ephemeral-storage: {{ system_ephemeral_storage_reserved }}
{% endif %}
{% if system_pid_reserved is defined %}
pid: "{{ system_pid_reserved }}"
{% endif %}
{% endif %}
{% endif %}
{% if is_kube_master | bool and eviction_hard_control_plane is defined and eviction_hard_control_plane %}
evictionHard:
{{ eviction_hard_control_plane | to_nice_yaml(indent=2) | indent(2) }}
{% elif not is_kube_master | bool and eviction_hard is defined and eviction_hard %}
{{ scope }}Reserved:
{% for resource in "cpu", "memory", "ephemeral-storage", "pid" %}
{{ resource }}: "{{ lookup('ansible.builtin.vars', scope + '_' ~ (resource | replace('-', '_')) + '_reserved') }}"
{% endfor %}
{% endfor %}
{% if eviction_hard is defined and eviction_hard %}
evictionHard:
{{ eviction_hard | to_nice_yaml(indent=2) | indent(2) }}
{% endif %}
@@ -117,9 +78,6 @@ resolvConf: "{{ kube_resolv_conf }}"
{% if kubelet_config_extra_args %}
{{ kubelet_config_extra_args | to_nice_yaml(indent=2) }}
{% endif %}
{% if inventory_hostname in groups['kube_node'] and kubelet_node_config_extra_args %}
{{ kubelet_node_config_extra_args | to_nice_yaml(indent=2) }}
{% endif %}
{% if kubelet_feature_gates or kube_feature_gates %}
featureGates:
{% for feature in (kubelet_feature_gates | default(kube_feature_gates, true)) %}
@@ -162,7 +120,7 @@ cpuManagerPolicy: {{ kubelet_cpu_manager_policy }}
{% endif %}
{% if kubelet_cpu_manager_policy_options is defined %}
cpuManagerPolicyOptions:
{{ kubelet_cpu_manager_policy_options | to_nice_yaml(indent=2) }}
{{ kubelet_cpu_manager_policy_options | to_nice_yaml(indent=2) | indent(width=2) }}
{% endif %}
{% if kubelet_topology_manager_policy is defined %}
topologyManagerPolicy: {{ kubelet_topology_manager_policy }}
@@ -172,7 +130,7 @@ topologyManagerScope: {{ kubelet_topology_manager_scope }}
{% endif %}
{% if kubelet_tracing %}
tracing:
endpoint: {{ kubelet_tracing_endpoint }}
endpoint: "{{ kubelet_tracing_endpoint }}"
samplingRatePerMillion: {{ kubelet_tracing_sampling_rate_per_million }}
{% endif %}
maxParallelImagePulls: {{ kubelet_max_parallel_image_pulls }}

7
kubespray/project/roles/kubernetes/node/templates/kubelet.env.v1beta1.j2
View File

@@ -11,20 +11,17 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
--config={{ kube_config_dir }}/kubelet-config.yaml \
--kubeconfig={{ kube_config_dir }}/kubelet.conf \
{# end kubeadm specific settings #}
--container-runtime-endpoint={{ cri_socket }} \
--runtime-cgroups={{ kubelet_runtime_cgroups }} \
{% endset %}
KUBELET_ARGS="{{ kubelet_args_base }} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}{% if inventory_hostname in groups['kube_node'] %}{% if kubelet_node_custom_flags is string %} {{kubelet_node_custom_flags}} {% else %}{% for flag in kubelet_node_custom_flags %} {{flag}} {% endfor %}{% endif %}{% endif %}"
KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_custom_flags | join(' ') }}"
{% if kubelet_flexvolumes_plugins_dir is defined %}
KUBELET_VOLUME_PLUGIN="--volume-plugin-dir={{ kubelet_flexvolumes_plugins_dir }}"
{% endif %}
{% if kube_network_plugin is defined and kube_network_plugin == "cloud" %}
KUBELET_NETWORK_PLUGIN="--hairpin-mode=promiscuous-bridge --network-plugin=kubenet"
{% endif %}
{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "gce"] %}
KUBELET_CLOUDPROVIDER="--cloud-provider={{ cloud_provider }} --cloud-config={{ kube_config_dir }}/cloud_config"
{% elif cloud_provider is defined and cloud_provider in ["external"] %}
{% if cloud_provider == "external" %}
KUBELET_CLOUDPROVIDER="--cloud-provider={{ cloud_provider }}"
{% else %}
KUBELET_CLOUDPROVIDER=""

6
kubespray/project/roles/kubernetes/node/templates/loadbalancer/haproxy.cfg.j2
View File

@@ -22,7 +22,7 @@ defaults
{% if loadbalancer_apiserver_healthcheck_port is defined -%}
frontend healthz
bind 0.0.0.0:{{ loadbalancer_apiserver_healthcheck_port }}
{% if enable_dual_stack_networks -%}
{% if ipv6_stack -%}
bind :::{{ loadbalancer_apiserver_healthcheck_port }}
{% endif -%}
mode http
@@ -31,7 +31,7 @@ frontend healthz
frontend kube_api_frontend
bind 127.0.0.1:{{ loadbalancer_apiserver_port|default(kube_apiserver_port) }}
{% if enable_dual_stack_networks -%}
{% if ipv6_stack -%}
bind [::1]:{{ loadbalancer_apiserver_port|default(kube_apiserver_port) }};
{% endif -%}
mode tcp
@@ -45,5 +45,5 @@ backend kube_api_backend
option httpchk GET /healthz
http-check expect status 200
{% for host in groups['kube_control_plane'] -%}
server {{ host }} {{ hostvars[host]['access_ip'] | default(hostvars[host]['ip'] | default(fallback_ips[host])) }}:{{ kube_apiserver_port }} check check-ssl verify none
server {{ host }} {{ hostvars[host]['main_access_ip'] | ansible.utils.ipwrap }}:{{ kube_apiserver_port }} check check-ssl verify none
{% endfor -%}

6
kubespray/project/roles/kubernetes/node/templates/loadbalancer/nginx.conf.j2
View File

@@ -14,13 +14,13 @@ stream {
upstream kube_apiserver {
least_conn;
{% for host in groups['kube_control_plane'] -%}
server {{ hostvars[host]['access_ip'] | default(hostvars[host]['ip'] | default(fallback_ips[host])) }}:{{ kube_apiserver_port }};
server {{ hostvars[host]['main_access_ip'] | ansible.utils.ipwrap }}:{{ kube_apiserver_port }};
{% endfor -%}
}
server {
listen 127.0.0.1:{{ loadbalancer_apiserver_port|default(kube_apiserver_port) }};
{% if enable_dual_stack_networks -%}
{% if ipv6_stack -%}
listen [::1]:{{ loadbalancer_apiserver_port|default(kube_apiserver_port) }};
{% endif -%}
proxy_pass kube_apiserver;
@@ -44,7 +44,7 @@ http {
{% if loadbalancer_apiserver_healthcheck_port is defined -%}
server {
listen {{ loadbalancer_apiserver_healthcheck_port }};
{% if enable_dual_stack_networks -%}
{% if ipv6_stack -%}
listen [::]:{{ loadbalancer_apiserver_healthcheck_port }};
{% endif -%}
location /healthz {

5
kubespray/project/roles/kubernetes/node/templates/manifests/kube-vip.manifest.j2
View File

@@ -104,11 +104,16 @@ spec:
imagePullPolicy: {{ k8s_image_pull_policy }}
name: kube-vip
resources: {}
{% if kube_vip_lb_fwdmethod == "masquerade" %}
securityContext:
privileged: true
{% else %}
securityContext:
capabilities:
add:
- NET_ADMIN
- NET_RAW
{% endif %}
volumeMounts:
- mountPath: /etc/kubernetes/admin.conf
name: kubeconfig

2
kubespray/project/roles/kubernetes/node/templates/node-kubeconfig.yaml.j2
View File

@@ -5,7 +5,7 @@ clusters:
- name: local
cluster:
certificate-authority: {{ kube_cert_dir }}/ca.pem
server: {{ kube_apiserver_endpoint }}
server: "{{ kube_apiserver_endpoint }}"
users:
- name: kubelet
user:

35
kubespray/project/roles/kubernetes/preinstall/defaults/main.yml
View File

@@ -1,8 +1,11 @@
---
# Set to true to allow pre-checks to fail and continue deployment
ignore_assert_errors: false
epel_enabled: false
# Set to false to disable the backup parameter, set to true to accumulate backups of config files.
leave_etc_backup_files: true
nameservers: []
cloud_resolver: []
disable_host_nameservers: false
# Kubespray sets this to true after clusterDNS is running to apply changes to the host resolv.conf
dns_late: false
@@ -51,26 +54,7 @@ etc_hosts_localhost_entries:
minimal_node_memory_mb: 1024
minimal_master_memory_mb: 1500
yum_repo_dir: /etc/yum.repos.d
# number of times package install task should be retried
pkg_install_retries: 4
# Check if access_ip responds to ping. Set false if your firewall blocks ICMP.
ping_access_ip: true
## NTP Settings
# Start the ntpd or chrony service and enable it at system boot.
ntp_enabled: false
# The package to install which provides NTP functionality.
# The default is ntp for most platforms, or chrony on RHEL/CentOS 7 and later.
# The ntp_package can be one of ['ntp', 'chrony']
ntp_package: >-
{% if ansible_os_family == "RedHat" -%}
chrony
{%- else -%}
ntp
{%- endif -%}
# Manage the NTP configuration file.
ntp_manage_config: false
@@ -95,8 +79,13 @@ ntp_filter_interface: false
# - listen xxx
# The NTP driftfile path
# Only takes effect when ntp_manage_config is true.
ntp_driftfile: /var/lib/ntp/ntp.drift
# Enable tinker panic is useful when running NTP in a VM environment.
# Default value is `/var/lib/ntp/ntp.drift`, for ntpsec use '/var/lib/ntpsec/ntp.drift'
ntp_driftfile: >-
{% if ntp_package == "ntpsec" -%}
/var/lib/ntpsec/ntp.drift
{%- else -%}
/var/lib/ntp/ntp.drift
{%- endif -%}
# Only takes effect when ntp_manage_config is true.
ntp_tinker_panic: false

80
kubespray/project/roles/kubernetes/preinstall/files/pkgs-schema.json
View File

@@ -1,80 +0,0 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://kubespray.io/internal/os_packages.schema.json",
"title": "Os packages",
"description": "Criteria for selecting packages to install on Kubernetes nodes during installation by Kubespray",
"type": "object",
"patternProperties": {
".*": {
"type": "object",
"additionalProperties": false,
"properties": {
"enabled": {
"description": "Escape hatch to filter packages. The value is expected to be pre-resolved to a boolean by Jinja",
"type": "boolean",
"default": true
},
"groups": {
"description": "Match if the host is in one of these groups. If not specified match any host.",
"type": "array",
"minItems": 1,
"items":{
"type": "string",
"pattern": "^[0-9A-Za-z_]*$"
}
},
"os": {
"type": "object",
"description": "If not specified match any OS. Otherwise, must match by 'families' or 'distributions' to be included.",
"additionalProperties": false,
"minProperties": 1,
"properties": {
"families": {
"description": "Match if ansible_os_family is part of the list.",
"type": "array",
"minItems": 1,
"items": {
"type": "string"
}
},
"distributions": {
"type": "object",
"description": "Match if ansible_distribution match one of defined keys.",
"minProperties": 1,
"patternProperties": {
".*": {
"description": "Match if either the value is the empty hash, or one major_versions/versions/releases contains the corresponding variable ('ansible_distrbution_*')",
"type": "object",
"additionalProperties": false,
"properties": {
"major_versions": {
"type": "array",
"minItems": 1,
"items": {
"type": "string"
}
},
"versions": {
"type": "array",
"minItems": 1,
"items": {
"type": "string"
}
},
"releases": {
"type": "array",
"minItems": 1,
"items": {
"type": "string"
}
}
}
}
}
}
}
}
}
}
}
}

14
kubespray/project/roles/kubernetes/preinstall/handlers/main.yml
View File

@@ -35,7 +35,7 @@
get_checksum: false
get_mime: false
register: kube_apiserver_set
when: inventory_hostname in groups['kube_control_plane'] and dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'
when: ('kube_control_plane' in group_names) and dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'
listen: Preinstall | propagate resolvconf to k8s components
# FIXME(mattymo): Also restart for kubeadm mode
@@ -46,7 +46,7 @@
get_checksum: false
get_mime: false
register: kube_controller_set
when: inventory_hostname in groups['kube_control_plane'] and dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'
when: ('kube_control_plane' in group_names) and dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'
listen: Preinstall | propagate resolvconf to k8s components
- name: Preinstall | restart kube-controller-manager docker
@@ -55,7 +55,7 @@
executable: /bin/bash
when:
- container_manager == "docker"
- inventory_hostname in groups['kube_control_plane']
- ('kube_control_plane' in group_names)
- dns_mode != 'none'
- resolvconf_mode == 'host_resolvconf'
- kube_controller_set.stat.exists
@@ -71,7 +71,7 @@
until: preinstall_restart_controller_manager.rc == 0
when:
- container_manager in ['crio', 'containerd']
- inventory_hostname in groups['kube_control_plane']
- ('kube_control_plane' in group_names)
- dns_mode != 'none'
- resolvconf_mode == 'host_resolvconf'
- kube_controller_set.stat.exists
@@ -83,7 +83,7 @@
executable: /bin/bash
when:
- container_manager == "docker"
- inventory_hostname in groups['kube_control_plane']
- ('kube_control_plane' in group_names)
- dns_mode != 'none'
- resolvconf_mode == 'host_resolvconf'
- kube_apiserver_set.stat.exists
@@ -99,7 +99,7 @@
delay: 1
when:
- container_manager in ['crio', 'containerd']
- inventory_hostname in groups['kube_control_plane']
- ('kube_control_plane' in group_names)
- dns_mode != 'none'
- resolvconf_mode == 'host_resolvconf'
- kube_apiserver_set.stat.exists
@@ -116,7 +116,7 @@
delay: 1
when:
- dns_late
- inventory_hostname in groups['kube_control_plane']
- ('kube_control_plane' in group_names)
- dns_mode != 'none'
- resolvconf_mode == 'host_resolvconf'
- not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] and not is_fedora_coreos

91
kubespray/project/roles/kubernetes/preinstall/tasks/0020-set_facts.yml
View File

@@ -33,30 +33,12 @@
failed_when: false
register: resolvconf_stat
- name: Fetch resolvconf
when: resolvconf_stat.stat.exists is defined and resolvconf_stat.stat.exists
block:
- name: Get content of /etc/resolv.conf
slurp:
src: /etc/resolv.conf
register: resolvconf_slurp
- name: Get currently configured nameservers
set_fact:
configured_nameservers: "{{ resolvconf_slurp.content | b64decode | regex_findall('^nameserver\\s*(\\S*)', multiline=True) | ansible.utils.ipaddr }}"
when: resolvconf_slurp.content is defined
- name: Stop if /etc/resolv.conf not configured nameservers
assert:
that: configured_nameservers | length>0
fail_msg: "nameserver should not empty in /etc/resolv.conf"
when:
- not ignore_assert_errors
- configured_nameservers is defined
- not (upstream_dns_servers is defined and upstream_dns_servers | length > 0)
- not (disable_host_nameservers | default(false))
- dns_mode in ['coredns', 'coredns_dual']
# Used in vars/
- name: Fetch resolv.conf
when: resolvconf_stat.stat.exists
slurp:
src: /etc/resolv.conf
register: resolvconf_slurp
- name: NetworkManager | Check if host has NetworkManager
# noqa command-instead-of-module - Should we use service_facts for this?
@@ -77,19 +59,12 @@
- name: Set default dns if remove_default_searchdomains is false
set_fact:
default_searchdomains: ["default.svc.{{ dns_domain }}", "svc.{{ dns_domain }}"]
when: not remove_default_searchdomains | default() | bool or (remove_default_searchdomains | default() | bool and searchdomains | default([]) | length==0)
when: not remove_default_searchdomains | default() | bool or (remove_default_searchdomains | default() | bool and searchdomains | length == 0)
- name: Set dns facts
set_fact:
resolvconf: >-
{%- if resolvconf.rc == 0 and resolvconfd_path.stat.isdir is defined and resolvconfd_path.stat.isdir -%}true{%- else -%}false{%- endif -%}
bogus_domains: |-
{% for d in default_searchdomains | default([]) + searchdomains | default([]) -%}
{{ dns_domain }}.{{ d }}./{{ d }}.{{ d }}./com.{{ d }}./
{%- endfor %}
cloud_resolver: "{{ ['169.254.169.254'] if cloud_provider is defined and cloud_provider == 'gce' else
['169.254.169.253'] if cloud_provider is defined and cloud_provider == 'aws' else
[] }}"
- name: Check if kubelet is configured
stat:
@@ -154,58 +129,6 @@
dhclienthookfile: /etc/dhcp/dhclient-exit-hooks.d/zdnsupdate
when: ansible_os_family == "Debian"
- name: Generate search domains to resolvconf
set_fact:
searchentries:
search {{ (default_searchdomains | default([]) + searchdomains | default([])) | join(' ') }}
domainentry:
domain {{ dns_domain }}
supersede_search:
supersede domain-search "{{ (default_searchdomains | default([]) + searchdomains | default([])) | join('", "') }}";
supersede_domain:
supersede domain-name "{{ dns_domain }}";
- name: Pick coredns cluster IP or default resolver
set_fact:
coredns_server: |-
{%- if dns_mode == 'coredns' and not dns_early | bool -%}
{{ [skydns_server] }}
{%- elif dns_mode == 'coredns_dual' and not dns_early | bool -%}
{{ [skydns_server] + [skydns_server_secondary] }}
{%- elif dns_mode == 'manual' and not dns_early | bool -%}
{{ (manual_dns_server.split(',') | list) }}
{%- elif dns_mode == 'none' and not dns_early | bool -%}
[]
{%- elif dns_early | bool -%}
{{ upstream_dns_servers | default([]) }}
{%- endif -%}
# This task should only run after cluster/nodelocal DNS is up, otherwise all DNS lookups will timeout
- name: Generate nameservers for resolvconf, including cluster DNS
set_fact:
nameserverentries: |-
{{ (([nodelocaldns_ip] if enable_nodelocaldns else []) + (coredns_server | d([]) if not enable_nodelocaldns else []) + nameservers | d([]) + cloud_resolver | d([]) + (configured_nameservers | d([]) if not disable_host_nameservers | d() | bool else [])) | unique | join(',') }}
dhclient_supersede_nameserver_entries_list: |-
{{ (([nodelocaldns_ip] if enable_nodelocaldns else []) + (coredns_server | d([]) if not enable_nodelocaldns else []) + nameservers | d([]) + cloud_resolver | d([]) + (configured_nameservers | d([]) if not disable_host_nameservers | d() | bool else [])) | unique }}
when: not dns_early or dns_late
# This task should run instead of the above task when cluster/nodelocal DNS hasn't
# been deployed yet (like scale.yml/cluster.yml) or when it's down (reset.yml)
- name: Generate nameservers for resolvconf, not including cluster DNS
set_fact:
nameserverentries: |-
{{ (nameservers | d([]) + cloud_resolver | d([]) + configured_nameservers | d([])) | unique | join(',') }}
dhclient_supersede_nameserver_entries_list: |-
{{ (nameservers | d([]) + cloud_resolver | d([])) | unique }}
when: dns_early and not dns_late
- name: Generate supersede_nameserver from dhclient_supersede_nameserver_entries_list
set_fact:
supersede_nameserver: |-
{%- if dhclient_supersede_nameserver_entries_list | length > 0 -%}
supersede domain-name-servers {{ dhclient_supersede_nameserver_entries_list | join(', ') }};
{%- endif -%}
- name: Set etcd vars if using kubeadm mode
set_fact:
etcd_cert_dir: "{{ kube_cert_dir }}"

297
kubespray/project/roles/kubernetes/preinstall/tasks/0040-verify-settings.yml
View File

@@ -1,19 +1,18 @@
---
- name: Stop if kube_control_plane group is empty
- name: Stop if any host not in '--limit' does not have a fact cache
vars:
uncached_hosts: "{{ hostvars | dict2items | selectattr('value.ansible_default_ipv6', 'undefined') | selectattr('value.ansible_default_ipv4', 'undefined') | map(attribute='key') }}"
excluded_hosts: "{{ groups['k8s_cluster'] | difference(query('inventory_hostnames', ansible_limit)) }}"
assert:
that: groups.get( 'kube_control_plane' )
run_once: true
when: not ignore_assert_errors
- name: Stop if etcd group is empty in external etcd mode
assert:
that: groups.get('etcd')
fail_msg: "Group 'etcd' cannot be empty in external etcd mode"
that: uncached_hosts | intersect(excluded_hosts) == []
fail_msg: |
Kubespray does not support '--limit' without a populated facts cache for the excluded hosts.
Please run the facts.yml playbook first without '--limit'.
The following excluded hosts are not cached: {{ uncached_hosts | intersect(excluded_hosts) }}
run_once: true
when:
- ansible_limit is defined
- not ignore_assert_errors
- etcd_deployment_type != "kubeadm"
- name: Stop if non systemd OS type
assert:
that: ansible_service_mgr == "systemd"
@@ -25,54 +24,19 @@
msg: "{{ ansible_distribution }} is not a known OS"
when: not ignore_assert_errors
- name: Stop if unknown network plugin
assert:
that: kube_network_plugin in ['calico', 'flannel', 'weave', 'cloud', 'cilium', 'cni', 'kube-ovn', 'kube-router', 'macvlan', 'custom_cni']
msg: "{{ kube_network_plugin }} is not supported"
when:
- kube_network_plugin is defined
- not ignore_assert_errors
- name: Stop if unsupported version of Kubernetes
assert:
that: kube_version is version(kube_version_min_required, '>=')
msg: "The current release of Kubespray only support newer version of Kubernetes than {{ kube_version_min_required }} - You are trying to apply {{ kube_version }}"
when: not ignore_assert_errors
# simplify this items-list when https://github.com/ansible/ansible/issues/15753 is resolved
- name: "Stop if known booleans are set as strings (Use JSON format on CLI: -e \"{'key': true }\")"
assert:
that: item.value | type_debug == 'bool'
msg: "{{ item.value }} isn't a bool"
run_once: true
with_items:
- { name: download_run_once, value: "{{ download_run_once }}" }
- { name: deploy_netchecker, value: "{{ deploy_netchecker }}" }
- { name: download_always_pull, value: "{{ download_always_pull }}" }
- { name: helm_enabled, value: "{{ helm_enabled }}" }
- { name: openstack_lbaas_enabled, value: "{{ openstack_lbaas_enabled }}" }
when: not ignore_assert_errors
- name: Stop if even number of etcd hosts
assert:
that: groups.etcd | length is not divisibleby 2
when:
- not ignore_assert_errors
- inventory_hostname in groups.get('etcd',[])
- name: Stop if memory is too small for masters
- name: Stop if memory is too small for control plane nodes
assert:
that: ansible_memtotal_mb >= minimal_master_memory_mb
when:
- not ignore_assert_errors
- inventory_hostname in groups['kube_control_plane']
- ('kube_control_plane' in group_names)
- name: Stop if memory is too small for nodes
assert:
that: ansible_memtotal_mb >= minimal_node_memory_mb
when:
- not ignore_assert_errors
- inventory_hostname in groups['kube_node']
- ('kube_node' in group_names)
# This command will fail if cgroups are not enabled on the node.
# For reference: https://kubernetes.io/docs/concepts/architecture/cgroups/#check-cgroup-version
@@ -81,21 +45,6 @@
changed_when: false
when: not ignore_assert_errors
# This assertion will fail on the safe side: One can indeed schedule more pods
# on a node than the CIDR-range has space for when additional pods use the host
# network namespace. It is impossible to ascertain the number of such pods at
# provisioning time, so to establish a guarantee, we factor these out.
# NOTICE: the check blatantly ignores the inet6-case
- name: Guarantee that enough network address space is available for all pods
assert:
that: "{{ (kubelet_max_pods | default(110)) | int <= (2 ** (32 - kube_network_node_prefix | int)) - 2 }}"
msg: "Do not schedule more pods on a node than inet addresses are available."
when:
- not ignore_assert_errors
- inventory_hostname in groups['k8s_cluster']
- kube_network_node_prefix is defined
- kube_network_plugin != 'calico'
- name: Stop if ip var does not match local ips
assert:
that: (ip in ansible_all_ipv4_addresses) or (ip in ansible_all_ipv6_addresses)
@@ -104,50 +53,26 @@
- not ignore_assert_errors
- ip is defined
- name: Ensure ping package
package:
# noqa: jinja[spacing]
name: >-
{%- if ansible_os_family == 'Debian' -%}
iputils-ping
{%- else -%}
iputils
{%- endif -%}
state: present
when:
- access_ip is defined
- not ignore_assert_errors
- ping_access_ip
- not is_fedora_coreos
- not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"]
- name: Stop if access_ip is not pingable
command: ping -c1 {{ access_ip }}
command: ping -c1 {{ main_access_ip }}
when:
- access_ip is defined
- main_access_ip is defined
- not ignore_assert_errors
- ping_access_ip
changed_when: false
- name: Stop if RBAC is not enabled when dashboard is enabled
assert:
that: rbac_enabled
when:
- dashboard_enabled
- not ignore_assert_errors
- name: Stop if RBAC is not enabled when OCI cloud controller is enabled
assert:
that: rbac_enabled
when:
- cloud_provider is defined and cloud_provider == "oci"
- not ignore_assert_errors
- name: Stop if kernel version is too low
- name: Stop if kernel version is too low for cilium
assert:
that: ansible_kernel.split('-')[0] is version('4.9.17', '>=')
when:
- kube_network_plugin == 'cilium' or cilium_deploy_additionally | default(false) | bool
- kube_network_plugin == 'cilium' or cilium_deploy_additionally
- not ignore_assert_errors
- name: Stop if kernel version is too low for nftables
assert:
that: ansible_kernel.split('-')[0] is version('5.13', '>=')
when:
- kube_proxy_mode == 'nftables'
- not ignore_assert_errors
- name: Stop if bad hostname
@@ -156,179 +81,17 @@
msg: "Hostname must consist of lower case alphanumeric characters, '.' or '-', and must start and end with an alphanumeric character"
when: not ignore_assert_errors
- name: Check cloud_provider value
- name: Stop if /etc/resolv.conf has no configured nameservers
assert:
that: cloud_provider in ['gce', 'aws', 'azure', 'openstack', 'vsphere', 'oci', 'external']
msg: "If set the 'cloud_provider' var must be set either to 'gce', 'aws', 'azure', 'openstack', 'vsphere', 'oci' or 'external'"
that: configured_nameservers | length>0
fail_msg: "nameserver should not be empty in /etc/resolv.conf"
when:
- cloud_provider is defined
- not ignore_assert_errors
tags:
- cloud-provider
- facts
- name: "Check that kube_service_addresses is a network range"
assert:
that:
- kube_service_addresses | ansible.utils.ipaddr('net')
msg: "kube_service_addresses = '{{ kube_service_addresses }}' is not a valid network range"
run_once: true
- name: "Check that kube_pods_subnet is a network range"
assert:
that:
- kube_pods_subnet | ansible.utils.ipaddr('net')
msg: "kube_pods_subnet = '{{ kube_pods_subnet }}' is not a valid network range"
run_once: true
- name: "Check that kube_pods_subnet does not collide with kube_service_addresses"
assert:
that:
- kube_pods_subnet | ansible.utils.ipaddr(kube_service_addresses) | string == 'None'
msg: "kube_pods_subnet cannot be the same network segment as kube_service_addresses"
run_once: true
- name: "Check that IP range is enough for the nodes"
assert:
that:
- 2 ** (kube_network_node_prefix - kube_pods_subnet | ansible.utils.ipaddr('prefix')) >= groups['k8s_cluster'] | length
msg: "Not enough IPs are available for the desired node count."
when: kube_network_plugin != 'calico'
run_once: true
- name: Stop if unknown dns mode
assert:
that: dns_mode in ['coredns', 'coredns_dual', 'manual', 'none']
msg: "dns_mode can only be 'coredns', 'coredns_dual', 'manual' or 'none'"
when: dns_mode is defined
run_once: true
- name: Stop if unknown kube proxy mode
assert:
that: kube_proxy_mode in ['iptables', 'ipvs']
msg: "kube_proxy_mode can only be 'iptables' or 'ipvs'"
when: kube_proxy_mode is defined
run_once: true
- name: Stop if unknown cert_management
assert:
that: cert_management | d('script') in ['script', 'none']
msg: "cert_management can only be 'script' or 'none'"
run_once: true
- name: Stop if unknown resolvconf_mode
assert:
that: resolvconf_mode in ['docker_dns', 'host_resolvconf', 'none']
msg: "resolvconf_mode can only be 'docker_dns', 'host_resolvconf' or 'none'"
when: resolvconf_mode is defined
run_once: true
- name: Stop if etcd deployment type is not host, docker or kubeadm
assert:
that: etcd_deployment_type in ['host', 'docker', 'kubeadm']
msg: "The etcd deployment type, 'etcd_deployment_type', must be host, docker or kubeadm"
when:
- inventory_hostname in groups.get('etcd',[])
- name: Stop if container manager is not docker, crio or containerd
assert:
that: container_manager in ['docker', 'crio', 'containerd']
msg: "The container manager, 'container_manager', must be docker, crio or containerd"
run_once: true
- name: Stop if etcd deployment type is not host or kubeadm when container_manager != docker
assert:
that: etcd_deployment_type in ['host', 'kubeadm']
msg: "The etcd deployment type, 'etcd_deployment_type', must be host or kubeadm when container_manager is not docker"
when:
- inventory_hostname in groups.get('etcd',[])
- container_manager != 'docker'
# TODO: Clean this task up when we drop backward compatibility support for `etcd_kubeadm_enabled`
- name: Stop if etcd deployment type is not host or kubeadm when container_manager != docker and etcd_kubeadm_enabled is not defined
run_once: true
when: etcd_kubeadm_enabled is defined
block:
- name: Warn the user if they are still using `etcd_kubeadm_enabled`
debug:
msg: >
"WARNING! => `etcd_kubeadm_enabled` is deprecated and will be removed in a future release.
You can set `etcd_deployment_type` to `kubeadm` instead of setting `etcd_kubeadm_enabled` to `true`."
changed_when: true
- name: Stop if `etcd_kubeadm_enabled` is defined and `etcd_deployment_type` is not `kubeadm` or `host`
assert:
that: etcd_deployment_type == 'kubeadm'
msg: >
It is not possible to use `etcd_kubeadm_enabled` when `etcd_deployment_type` is set to {{ etcd_deployment_type }}.
Unset the `etcd_kubeadm_enabled` variable and set `etcd_deployment_type` to desired deployment type (`host`, `kubeadm`, `docker`) instead."
when: etcd_kubeadm_enabled
- name: Stop if download_localhost is enabled but download_run_once is not
assert:
that: download_run_once
msg: "download_localhost requires enable download_run_once"
when: download_localhost
- name: Stop if kata_containers_enabled is enabled when container_manager is docker
assert:
that: container_manager != 'docker'
msg: "kata_containers_enabled support only for containerd and crio-o. See https://github.com/kata-containers/documentation/blob/1.11.4/how-to/run-kata-with-k8s.md#install-a-cri-implementation for details"
when: kata_containers_enabled
- name: Stop if gvisor_enabled is enabled when container_manager is not containerd
assert:
that: container_manager == 'containerd'
msg: "gvisor_enabled support only compatible with containerd. See https://github.com/kubernetes-sigs/kubespray/issues/7650 for details"
when: gvisor_enabled
- upstream_dns_servers | length == 0
- not disable_host_nameservers
- dns_mode in ['coredns', 'coredns_dual']
- name: Stop if download_localhost is enabled for Flatcar Container Linux
assert:
that: ansible_os_family not in ["Flatcar", "Flatcar Container Linux by Kinvolk"]
msg: "download_run_once not supported for Flatcar Container Linux"
when: download_run_once or download_force_cache
- name: Ensure minimum containerd version
assert:
that: containerd_version is version(containerd_min_version_required, '>=')
msg: "containerd_version is too low. Minimum version {{ containerd_min_version_required }}"
run_once: true
when:
- containerd_version not in ['latest', 'edge', 'stable']
- container_manager == 'containerd'
- name: Stop if using deprecated containerd_config variable
assert:
that: containerd_config is not defined
msg: "Variable containerd_config is now deprecated. See https://github.com/kubernetes-sigs/kubespray/blob/master/inventory/sample/group_vars/all/containerd.yml for details."
when:
- containerd_config is defined
- not ignore_assert_errors
- name: Stop if auto_renew_certificates is enabled when certificates are managed externally (kube_external_ca_mode is true)
assert:
that: not auto_renew_certificates
msg: "Variable auto_renew_certificates must be disabled when CA are managed externally: kube_external_ca_mode = true"
when:
- kube_external_ca_mode
- not ignore_assert_errors
- name: Stop if using deprecated comma separated list for admission plugins
assert:
that: "',' not in kube_apiserver_enable_admission_plugins[0]"
msg: "Comma-separated list for kube_apiserver_enable_admission_plugins is now deprecated, use separate list items for each plugin."
when:
- kube_apiserver_enable_admission_plugins is defined
- kube_apiserver_enable_admission_plugins | length > 0
- name: Verify that the packages list structure is valid
ansible.utils.validate:
criteria: "{{ lookup('file', 'pkgs-schema.json') }}"
data: "{{ pkgs }}"
- name: Verify that the packages list is sorted
vars:
pkgs_lists: "{{ pkgs.keys() | list }}"
assert:
that: "pkgs_lists | sort == pkgs_lists"
fail_msg: "pkgs is not sorted: {{ pkgs_lists | ansible.utils.fact_diff(pkgs_lists | sort) }}"

30
kubespray/project/roles/kubernetes/preinstall/tasks/0050-create_directories.yml
View File

@@ -5,17 +5,17 @@
state: directory
owner: "{{ kube_owner }}"
mode: "0755"
when: inventory_hostname in groups['k8s_cluster']
when: ('k8s_cluster' in group_names)
become: true
tags:
- kubelet
- k8s-secrets
- kube-controller-manager
- kube-apiserver
- bootstrap-os
- bootstrap_os
- apps
- network
- master
- master # master tag is deprecated and replaced by control-plane
- control-plane
- node
with_items:
- "{{ kube_config_dir }}"
@@ -29,17 +29,17 @@
state: directory
owner: root
mode: "0755"
when: inventory_hostname in groups['k8s_cluster']
when: ('k8s_cluster' in group_names)
become: true
tags:
- kubelet
- k8s-secrets
- kube-controller-manager
- kube-apiserver
- bootstrap-os
- bootstrap_os
- apps
- network
- master
- master # master tag is deprecated and replaced by control-plane
- control-plane
- node
with_items:
- "{{ kube_cert_dir }}"
@@ -53,7 +53,7 @@
get_mime: false
register: kube_cert_compat_dir_check
when:
- inventory_hostname in groups['k8s_cluster']
- ('k8s_cluster' in group_names)
- kube_cert_dir != kube_cert_compat_dir
- name: Create kubernetes kubeadm compat cert dir (kubernetes/kubeadm issue 1498)
@@ -63,7 +63,7 @@
state: link
mode: "0755"
when:
- inventory_hostname in groups['k8s_cluster']
- ('k8s_cluster' in group_names)
- kube_cert_dir != kube_cert_compat_dir
- not kube_cert_compat_dir_check.stat.exists
@@ -78,7 +78,7 @@
- "/opt/cni/bin"
when:
- kube_network_plugin in ["calico", "weave", "flannel", "cilium", "kube-ovn", "kube-router", "macvlan"]
- inventory_hostname in groups['k8s_cluster']
- ('k8s_cluster' in group_names)
tags:
- network
- cilium
@@ -86,7 +86,7 @@
- weave
- kube-ovn
- kube-router
- bootstrap-os
- bootstrap_os
- name: Create calico cni directories
file:
@@ -98,11 +98,11 @@
- "/var/lib/calico"
when:
- kube_network_plugin == "calico"
- inventory_hostname in groups['k8s_cluster']
- ('k8s_cluster' in group_names)
tags:
- network
- calico
- bootstrap-os
- bootstrap_os
- name: Create local volume provisioner directories
file:
@@ -113,7 +113,7 @@
mode: "{{ local_volume_provisioner_directory_mode }}"
with_items: "{{ local_volume_provisioner_storage_classes.keys() | list }}"
when:
- inventory_hostname in groups['k8s_cluster']
- ('k8s_cluster' in group_names)
- local_volume_provisioner_enabled
tags:
- persistent_volumes

7
kubespray/project/roles/kubernetes/preinstall/tasks/0060-resolvconf.yml
View File

@@ -7,10 +7,9 @@
blockinfile:
path: "{{ resolvconffile }}"
block: |-
{% for item in [domainentry] + [searchentries] -%}
{{ item }}
{% endfor %}
{% for item in nameserverentries.split(',') %}
domain {{ dns_domain }}
search {{ (default_searchdomains + searchdomains) | join(' ') }}
{% for item in nameserverentries %}
nameserver {{ item }}
{% endfor %}
options ndots:{{ ndots }} timeout:{{ dns_timeout | default('2') }} attempts:{{ dns_attempts | default('2') }}

13
kubespray/project/roles/kubernetes/preinstall/tasks/0063-networkmanager-dns.yml
View File

@@ -4,11 +4,12 @@
path: /etc/NetworkManager/conf.d/dns.conf
section: global-dns-domain-*
option: servers
value: "{{ nameserverentries }}"
value: "{{ nameserverentries | join(',') }}"
mode: '0600'
backup: true
backup: "{{ leave_etc_backup_files }}"
when:
- nameserverentries != "127.0.0.53" or systemd_resolved_enabled.rc != 0
- ('127.0.0.53' not in nameserverentries
or systemd_resolved_enabled.rc != 0)
notify: Preinstall | update resolvconf for networkmanager
- name: Set default dns if remove_default_searchdomains is false
@@ -21,9 +22,9 @@
path: /etc/NetworkManager/conf.d/dns.conf
section: global-dns
option: searches
value: "{{ (default_searchdomains | default([]) + searchdomains | default([])) | join(',') }}"
value: "{{ (default_searchdomains | default([]) + searchdomains) | join(',') }}"
mode: '0600'
backup: true
backup: "{{ leave_etc_backup_files }}"
notify: Preinstall | update resolvconf for networkmanager
- name: NetworkManager | Add DNS options to NM configuration
@@ -33,5 +34,5 @@
option: options
value: "ndots:{{ ndots }},timeout:{{ dns_timeout | default('2') }},attempts:{{ dns_attempts | default('2') }}"
mode: '0600'
backup: true
backup: "{{ leave_etc_backup_files }}"
notify: Preinstall | update resolvconf for networkmanager

87
kubespray/project/roles/kubernetes/preinstall/tasks/0070-system-packages.yml
View File

@@ -1,87 +0,0 @@
---
- name: Update package management cache (zypper) - SUSE
command: zypper -n --gpg-auto-import-keys ref
register: make_cache_output
until: make_cache_output is succeeded
retries: 4
delay: "{{ retry_stagger | random + 3 }}"
when:
- ansible_pkg_mgr == 'zypper'
tags: bootstrap-os
- name: Add debian 10 required repos
when:
- ansible_distribution == "Debian"
- ansible_distribution_version == "10"
tags:
- bootstrap-os
block:
- name: Add Debian Backports apt repo
apt_repository:
repo: "deb http://deb.debian.org/debian {{ ansible_distribution_release }}-backports main"
state: present
filename: debian-backports
- name: Set libseccomp2 pin priority to apt_preferences on Debian buster
copy:
content: |
Package: libseccomp2
Pin: release a={{ ansible_distribution_release }}-backports
Pin-Priority: 1001
dest: "/etc/apt/preferences.d/libseccomp2"
owner: "root"
mode: "0644"
- name: Update package management cache (APT)
apt:
update_cache: true
cache_valid_time: 3600
when: ansible_os_family == "Debian"
tags:
- bootstrap-os
- name: Remove legacy docker repo file
file:
path: "{{ yum_repo_dir }}/docker.repo"
state: absent
when:
- ansible_os_family == "RedHat"
- not is_fedora_coreos
- name: Install epel-release on RHEL derivatives
package:
name: epel-release
state: present
when:
- ansible_os_family == "RedHat"
- not is_fedora_coreos
- epel_enabled | bool
tags:
- bootstrap-os
- name: Install packages requirements
vars:
# The json_query for selecting packages name is split for readability
# see files/pkgs-schema.json for the structure of `pkgs`
# and the matching semantics
full_query: "[? value | (enabled == null || enabled) && ( {{ filters_os }} ) && ( {{ filters_groups }} ) ].key"
filters_groups: "groups | @ == null || [? contains(`{{ group_names }}`, @)]"
filters_os: "os == null || (os | ( {{ filters_family }} ) || ( {{ filters_distro }} ))"
dquote: !unsafe '"'
# necessary to workaround Ansible escaping
filters_distro: "distributions.{{ dquote }}{{ ansible_distribution }}{{ dquote }} |
@ == `{}` ||
contains(not_null(major_versions, `[]`), '{{ ansible_distribution_major_version }}') ||
contains(not_null(versions, `[]`), '{{ ansible_distribution_version }}') ||
contains(not_null(releases, `[]`), '{{ ansible_distribution_release }}')"
filters_family: "families && contains(families, '{{ ansible_os_family }}')"
package:
name: "{{ pkgs | dict2items | to_json|from_json | community.general.json_query(full_query) }}"
state: present
register: pkgs_task_result
until: pkgs_task_result is succeeded
retries: "{{ pkg_install_retries }}"
delay: "{{ retry_stagger | random + 3 }}"
when: not (ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] or is_fedora_coreos)
tags:
- bootstrap-os

13
kubespray/project/roles/kubernetes/preinstall/tasks/0080-system-configurations.yml
View File

@@ -20,7 +20,7 @@
- "'Amazon' not in ansible_distribution"
- slc.stat.exists
tags:
- bootstrap-os
- bootstrap_os
- name: Disable IPv6 DNS lookup
lineinfile:
@@ -28,13 +28,13 @@
line: "precedence ::ffff:0:0/96 100"
state: present
create: true
backup: true
backup: "{{ leave_etc_backup_files }}"
mode: "0644"
when:
- disable_ipv6_dns
- not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"]
tags:
- bootstrap-os
- bootstrap_os
- name: Clean previously used sysctl file locations
file:
@@ -52,7 +52,7 @@
get_mime: false
register: sysctl_file_stat
tags:
- bootstrap-os
- bootstrap_os
- name: Change sysctl file path to link source if linked
set_fact:
@@ -61,7 +61,7 @@
- sysctl_file_stat.stat.islnk is defined
- sysctl_file_stat.stat.islnk
tags:
- bootstrap-os
- bootstrap_os
- name: Make sure sysctl file path folder exists
file:
@@ -76,6 +76,7 @@
value: "1"
state: present
reload: true
when: ipv4_stack | bool
- name: Enable ipv6 forwarding
ansible.posix.sysctl:
@@ -84,7 +85,7 @@
value: "1"
state: present
reload: true
when: enable_dual_stack_networks | bool
when: ipv6_stack | bool
- name: Check if we need to set fs.may_detach_mounts
stat:

16
kubespray/project/roles/kubernetes/preinstall/tasks/0081-ntp-configurations.yml
View File

@@ -1,12 +1,4 @@
---
- name: Ensure NTP package
package:
name:
- "{{ ntp_package }}"
state: present
when:
- not is_fedora_coreos
- not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"]
- name: Disable systemd-timesyncd
service:
@@ -21,6 +13,8 @@
ntp_config_file: >-
{% if ntp_package == "ntp" -%}
/etc/ntp.conf
{%- elif ntp_package == "ntpsec" -%}
/etc/ntpsec/ntp.conf
{%- elif ansible_os_family in ['RedHat', 'Suse'] -%}
/etc/chrony.conf
{%- else -%}
@@ -56,10 +50,10 @@
# noqa: jinja[spacing]
command: >-
timeout -k 60s 60s
{% if ntp_package == "ntp" -%}
ntpd -gq
{%- else -%}
{% if ntp_package == "chrony" -%}
chronyd -q
{%- else -%}
ntpd -gq
{%- endif -%}
when:
- ntp_force_sync_immediately

15
kubespray/project/roles/kubernetes/preinstall/tasks/0090-etchosts.yml
View File

@@ -2,11 +2,10 @@
- name: Hosts | create hosts list from inventory
set_fact:
etc_hosts_inventory_block: |-
{% for item in (groups['k8s_cluster'] + groups['etcd'] | default([]) + groups['calico_rr'] | default([])) | unique -%}
{% if 'access_ip' in hostvars[item] or 'ip' in hostvars[item] or 'ansible_default_ipv4' in hostvars[item] -%}
{{ hostvars[item]['access_ip'] | default(hostvars[item]['ip'] | default(hostvars[item]['ansible_default_ipv4']['address'])) }}
{%- if ('ansible_hostname' in hostvars[item] and item != hostvars[item]['ansible_hostname']) %} {{ hostvars[item]['ansible_hostname'] }}.{{ dns_domain }} {{ hostvars[item]['ansible_hostname'] }} {% else %} {{ item }}.{{ dns_domain }} {{ item }} {% endif %}
{% for item in (groups['k8s_cluster'] + groups['etcd'] | default([]) + groups['calico_rr'] | default([])) | unique %}
{{ hostvars[item]['main_access_ip'] }} {{ hostvars[item]['ansible_hostname'] | default(item) }}.{{ dns_domain }} {{ hostvars[item]['ansible_hostname'] | default(item) }}
{% if ipv4_stack and ipv6_stack %}
{{ hostvars[item]['access_ip6'] | default(hostvars[item]['ip6'] | default(hostvars[item]['ansible_default_ipv6']['address'])) }} {{ hostvars[item]['ansible_hostname'] | default(item) }}.{{ dns_domain }} {{ hostvars[item]['ansible_hostname'] | default(item) }}
{% endif %}
{% endfor %}
delegate_to: localhost
@@ -20,7 +19,7 @@
block: "{{ hostvars.localhost.etc_hosts_inventory_block }}"
state: "{{ 'present' if populate_inventory_to_hosts_file else 'absent' }}"
create: true
backup: true
backup: "{{ leave_etc_backup_files }}"
unsafe_writes: true
marker: "# Ansible inventory hosts {mark}"
mode: "0644"
@@ -31,7 +30,7 @@
regexp: ".*{{ apiserver_loadbalancer_domain_name }}$"
line: "{{ loadbalancer_apiserver.address }} {{ apiserver_loadbalancer_domain_name }}"
state: present
backup: true
backup: "{{ leave_etc_backup_files }}"
unsafe_writes: true
when:
- populate_loadbalancer_apiserver_to_hosts_file
@@ -69,7 +68,7 @@
line: "{{ item.key }} {{ item.value | join(' ') }}"
regexp: "^{{ item.key }}.*$"
state: present
backup: true
backup: "{{ leave_etc_backup_files }}"
unsafe_writes: true
loop: "{{ etc_hosts_localhosts_dict_target | default({}) | dict2items }}"

11
kubespray/project/roles/kubernetes/preinstall/tasks/0100-dhclient-hooks.yml
View File

@@ -1,15 +1,20 @@
---
- name: Configure dhclient to supersede search/domain/nameservers
blockinfile:
# 1 is the 2nd item of a tuple in items()
block: |-
{% for item in [supersede_domain, supersede_search, supersede_nameserver] | reject('equalto', '') -%}
{{ item }}
{% for key, val in dhclient_supersede.items() | rejectattr(1, '==', []) -%}
{% if key == "domain-name-servers" -%}
supersede {{ key }} {{ val | join(',') }};
{% else -%}
supersede {{ key }} "{{ val | join('","') }}";
{% endif -%}
{% endfor %}
path: "{{ dhclientconffile }}"
create: true
state: present
insertbefore: BOF
backup: true
backup: "{{ leave_etc_backup_files }}"
marker: "# Ansible entries {mark}"
mode: "0644"
notify: Preinstall | propagate resolvconf to k8s components

2
kubespray/project/roles/kubernetes/preinstall/tasks/0110-dhclient-hooks-undo.yml
View File

@@ -7,7 +7,7 @@
blockinfile:
path: "{{ dhclientconffile }}"
state: absent
backup: true
backup: "{{ leave_etc_backup_files }}"
marker: "# Ansible entries {mark}"
notify: Preinstall | propagate resolvconf to k8s components

28
kubespray/project/roles/kubernetes/preinstall/tasks/main.yml
View File

@@ -32,7 +32,7 @@
- systemd_resolved_enabled.rc != 0
- networkmanager_enabled.rc != 0
tags:
- bootstrap-os
- bootstrap_os
- resolvconf
- name: Apply systemd-resolved settings
@@ -42,7 +42,7 @@
- resolvconf_mode == 'host_resolvconf'
- systemd_resolved_enabled.rc == 0
tags:
- bootstrap-os
- bootstrap_os
- resolvconf
- name: Apply networkmanager unmanaged devices settings
@@ -50,7 +50,7 @@
when:
- networkmanager_enabled.rc == 0
tags:
- bootstrap-os
- bootstrap_os
- name: Apply networkmanager DNS settings
import_tasks: 0063-networkmanager-dns.yml
@@ -59,23 +59,15 @@
- resolvconf_mode == 'host_resolvconf'
- networkmanager_enabled.rc == 0
tags:
- bootstrap-os
- bootstrap_os
- resolvconf
- name: Install required system packages
import_tasks: 0070-system-packages.yml
when:
- not dns_late
tags:
- bootstrap-os
- system-packages
- name: Apply system configurations
import_tasks: 0080-system-configurations.yml
when:
- not dns_late
tags:
- bootstrap-os
- bootstrap_os
- name: Configure NTP
import_tasks: 0081-ntp-configurations.yml
@@ -83,12 +75,12 @@
- not dns_late
- ntp_enabled
tags:
- bootstrap-os
- bootstrap_os
- name: Configure /etc/hosts
import_tasks: 0090-etchosts.yml
tags:
- bootstrap-os
- bootstrap_os
- etchosts
- name: Configure dhclient
@@ -99,7 +91,7 @@
- dhclientconffile is defined
- not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"]
tags:
- bootstrap-os
- bootstrap_os
- resolvconf
- name: Configure dhclient dhclient hooks
@@ -110,7 +102,7 @@
- dhclientconffile is defined
- not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"]
tags:
- bootstrap-os
- bootstrap_os
- resolvconf
# We need to make sure the network is restarted early enough so that docker can later pick up the correct system
@@ -128,7 +120,7 @@
when:
- not dns_late
tags:
- bootstrap-os
- bootstrap_os
- name: Run calico checks
include_role:

10
kubespray/project/roles/kubernetes/preinstall/templates/resolved.conf.j2
View File

@@ -1,12 +1,12 @@
[Resolve]
{% if not dns_early and dns_late %}
DNS={{ ([nodelocaldns_ip] if enable_nodelocaldns else coredns_server )| list | join(' ') }}
DNS={{ ([nodelocaldns_ip] if enable_nodelocaldns else coredns_server) | list | join(' ') }}
{% endif %}
FallbackDNS={{ ( upstream_dns_servers|d([]) + nameservers|d([]) + cloud_resolver|d([])) | unique | join(' ') }}
{% if remove_default_searchdomains and searchdomains|default([])|length != 0 %}
Domains={{ searchdomains|default([]) | join(' ') }}
FallbackDNS={{ ( upstream_dns_servers + nameservers + cloud_resolver) | unique | join(' ') }}
{% if remove_default_searchdomains and searchdomains | length != 0 %}
Domains={{ searchdomains | join(' ') }}
{% else %}
Domains={{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([])) | join(' ') }}
Domains={{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains) | join(' ') }}
{% endif %}
DNSSEC=no
Cache=no-negative

123
kubespray/project/roles/kubernetes/preinstall/vars/main.yml
View File

@@ -1,103 +1,22 @@
---
pkgs:
apparmor: &debian_family_base
os:
families:
- Debian
apt-transport-https: *debian_family_base
aufs-tools: &deb_10
groups:
- k8s_cluster
os:
distributions:
Debian:
major_versions:
- "10"
bash-completion: {}
conntrack: &deb_redhat
groups:
- k8s_cluster
os:
families:
- Debian
- RedHat
conntrack-tools:
groups:
- k8s_cluster
os:
families:
- Suse
distributions:
Amazon: {}
container-selinux: &redhat_family
groups:
- k8s_cluster
os:
families:
- RedHat
curl: {}
device-mapper:
groups:
- k8s_cluster
os:
families:
- Suse
device-mapper-libs: *redhat_family
e2fsprogs: {}
ebtables: {}
gnupg: &debian
groups:
- k8s_cluster
os:
distributions:
Debian:
major_versions:
- "11"
- "12"
ipset:
enabled: "{{ kube_proxy_mode != 'ipvs' }}"
groups:
- k8s_cluster
iptables: *deb_redhat
ipvsadm:
enabled: "{{ kube_proxy_mode == 'ipvs' }}"
groups:
- k8s_cluster
libseccomp: *redhat_family
libseccomp2:
groups:
- k8s_cluster
os:
families:
- Suse
- Debian
libselinux-python: # TODO: Handle rehat_family + major < 8
os:
distributions:
Amazon: {}
libselinux-python3:
os:
distributions:
Fedora: {}
mergerfs:
os:
distributions:
Debian:
major_versions:
- "12"
nss: *redhat_family
openssl: {}
python-apt: *deb_10
# TODO: not for debian 10
python3-apt: *debian_family_base
python3-libselinux:
os:
distributions:
RedHat: {}
CentOS: {}
rsync: {}
socat: {}
software-properties-common: *debian_family_base
tar: {}
unzip: {}
xfsprogs: {}
coredns_server_by_mode:
coredns: "{{ [skydns_server] }}"
coredns_dual: "{{ [skydns_server, skydns_server_secondary] }}"
manual: "{{ manual_dns_server.split(',') }}"
none: []
coredns_server: "{{ upstream_dns_servers if dns_early else coredns_server_by_mode[dns_mode] }}"
_nameserverentries:
late:
- "{{ nodelocaldns_ip if enable_nodelocaldns else coredns_server }}"
early:
- "{{ nameservers }}"
- "{{ cloud_resolver }}"
- "{{ configured_nameservers if not disable_host_nameservers else [] }}"
nameserverentries: "{{ ((_nameserverentries['late'] if not dns_early else []) + _nameserverentries['early']) | flatten | unique }}"
dhclient_supersede:
domain-name-servers: "{{ ([nameservers, cloud_resolver] | flatten | unique) if dns_early else nameserverentries }}"
domain-name: "{{ [dns_domain] }}"
domain-search: "{{ default_searchdomains + searchdomains }}"
configured_nameservers: "{{ (resolvconf_slurp.content | b64decode | regex_findall('^nameserver\\s*(\\S*)', multiline=True) | ansible.utils.ipaddr)
if resolvconf_stat.stat.exists else [] }}"

34
kubespray/project/roles/kubernetes/tokens/files/kube-gen-token.sh
View File

@@ -1,34 +0,0 @@
#!/bin/bash
# Copyright 2015 The Kubernetes Authors All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
token_dir=${TOKEN_DIR:-/var/srv/kubernetes}
token_file="${token_dir}/known_tokens.csv"
create_accounts=($@)
if [ ! -e "${token_file}" ]; then
touch "${token_file}"
fi
for account in "${create_accounts[@]}"; do
if grep ",${account}," "${token_file}" ; then
continue
fi
token=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
echo "${token},${account},${account}" >> "${token_file}"
echo "${token}" > "${token_dir}/${account}.token"
echo "Added ${account}"
done

41
kubespray/project/roles/kubernetes/tokens/tasks/check-tokens.yml
View File

@@ -1,41 +0,0 @@
---
- name: "Check_tokens | check if the tokens have already been generated on first master"
stat:
path: "{{ kube_token_dir }}/known_tokens.csv"
get_attributes: false
get_checksum: true
get_mime: false
delegate_to: "{{ groups['kube_control_plane'][0] }}"
register: known_tokens_master
run_once: true
- name: "Check_tokens | Set default value for 'sync_tokens' and 'gen_tokens' to false"
set_fact:
sync_tokens: false
gen_tokens: false
- name: "Check_tokens | Set 'sync_tokens' and 'gen_tokens' to true"
set_fact:
gen_tokens: true
when: not known_tokens_master.stat.exists and kube_token_auth | default(true)
run_once: true
- name: "Check tokens | check if a cert already exists"
stat:
path: "{{ kube_token_dir }}/known_tokens.csv"
get_attributes: false
get_checksum: true
get_mime: false
register: known_tokens
- name: "Check_tokens | Set 'sync_tokens' to true"
set_fact:
sync_tokens: >-
{%- set tokens = {'sync': False} -%}
{%- for server in groups['kube_control_plane'] | intersect(ansible_play_batch)
if (not hostvars[server].known_tokens.stat.exists) or
(hostvars[server].known_tokens.stat.checksum | default('') != known_tokens_master.stat.checksum | default('')) -%}
{%- set _ = tokens.update({'sync': True}) -%}
{%- endfor -%}
{{ tokens.sync }}
run_once: true

63
kubespray/project/roles/kubernetes/tokens/tasks/gen_tokens.yml
View File

@@ -1,63 +0,0 @@
---
- name: Gen_tokens | copy tokens generation script
copy:
src: "kube-gen-token.sh"
dest: "{{ kube_script_dir }}/kube-gen-token.sh"
mode: "0700"
run_once: true
delegate_to: "{{ groups['kube_control_plane'][0] }}"
when: gen_tokens | default(false)
- name: Gen_tokens | generate tokens for master components
command: "{{ kube_script_dir }}/kube-gen-token.sh {{ item[0] }}-{{ item[1] }}"
environment:
TOKEN_DIR: "{{ kube_token_dir }}"
with_nested:
- [ "system:kubectl" ]
- "{{ groups['kube_control_plane'] }}"
register: gentoken_master
changed_when: "'Added' in gentoken_master.stdout"
run_once: true
delegate_to: "{{ groups['kube_control_plane'][0] }}"
when: gen_tokens | default(false)
- name: Gen_tokens | generate tokens for node components
command: "{{ kube_script_dir }}/kube-gen-token.sh {{ item[0] }}-{{ item[1] }}"
environment:
TOKEN_DIR: "{{ kube_token_dir }}"
with_nested:
- [ 'system:kubelet' ]
- "{{ groups['kube_node'] }}"
register: gentoken_node
changed_when: "'Added' in gentoken_node.stdout"
run_once: true
delegate_to: "{{ groups['kube_control_plane'][0] }}"
when: gen_tokens | default(false)
- name: Gen_tokens | Get list of tokens from first master
command: "find {{ kube_token_dir }} -maxdepth 1 -type f"
register: tokens_list
check_mode: false
delegate_to: "{{ groups['kube_control_plane'][0] }}"
run_once: true
when: sync_tokens | default(false)
- name: Gen_tokens | Gather tokens
shell: "set -o pipefail && tar cfz - {{ tokens_list.stdout_lines | join(' ') }} | base64 --wrap=0"
args:
executable: /bin/bash
register: tokens_data
check_mode: false
delegate_to: "{{ groups['kube_control_plane'][0] }}"
run_once: true
when: sync_tokens | default(false)
- name: Gen_tokens | Copy tokens on masters
shell: "set -o pipefail && echo '{{ tokens_data.stdout | quote }}' | base64 -d | tar xz -C /"
args:
executable: /bin/bash
when:
- inventory_hostname in groups['kube_control_plane']
- sync_tokens | default(false)
- inventory_hostname != groups['kube_control_plane'][0]
- tokens_data.stdout

21
kubespray/project/roles/kubernetes/tokens/tasks/main.yml
View File

@@ -1,21 +0,0 @@
---
- name: Check tokens
import_tasks: check-tokens.yml
tags:
- k8s-secrets
- k8s-gen-tokens
- facts
- name: Make sure the tokens directory exits
file:
path: "{{ kube_token_dir }}"
state: directory
mode: "0644"
group: "{{ kube_cert_group }}"
- name: Generate tokens
import_tasks: gen_tokens.yml
tags:
- k8s-secrets
- k8s-gen-tokens