kube-forge v2: embede python with ansible (kubespray) and cobra as cli building tool

This commit is contained in:
2025-06-22 00:37:47 +03:00
parent c2df2abc78
commit 4142a9db61
830 changed files with 874 additions and 2047 deletions

View File

@@ -0,0 +1,2 @@
[Service]
Environment={% if http_proxy %}"HTTP_PROXY={{ http_proxy }}"{% endif %} {% if https_proxy %}"HTTPS_PROXY={{ https_proxy }}"{% endif %} {% if no_proxy %}"NO_PROXY={{ no_proxy }}"{% endif %}

View File

@@ -0,0 +1,136 @@
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
nodeStatusUpdateFrequency: "{{ kubelet_status_update_frequency }}"
failSwapOn: {{ kubelet_fail_swap_on }}
authentication:
anonymous:
enabled: false
webhook:
enabled: {{ kubelet_authentication_token_webhook }}
x509:
clientCAFile: {{ kube_cert_dir }}/ca.crt
authorization:
{% if kubelet_authorization_mode_webhook %}
mode: Webhook
{% else %}
mode: AlwaysAllow
{% endif %}
{% if kubelet_enforce_node_allocatable is defined and kubelet_enforce_node_allocatable != "\"\"" %}
{% set kubelet_enforce_node_allocatable_list = kubelet_enforce_node_allocatable.split(",") %}
enforceNodeAllocatable:
{% for item in kubelet_enforce_node_allocatable_list %}
- {{ item }}
{% endfor %}
{% endif %}
staticPodPath: {{ kube_manifest_dir }}
cgroupDriver: {{ kubelet_cgroup_driver | default('systemd') }}
containerLogMaxFiles: {{ kubelet_logfiles_max_nr }}
containerLogMaxSize: {{ kubelet_logfiles_max_size }}
containerRuntimeEndpoint : {{ cri_socket }}
maxPods: {{ kubelet_max_pods }}
podPidsLimit: {{ kubelet_pod_pids_limit }}
address: "{{ kubelet_bind_address }}"
readOnlyPort: {{ kube_read_only_port }}
healthzPort: {{ kubelet_healthz_port }}
healthzBindAddress: "{{ kubelet_healthz_bind_address }}"
kubeletCgroups: {{ kubelet_kubelet_cgroups }}
clusterDomain: {{ dns_domain }}
{% if kubelet_protect_kernel_defaults | bool %}
protectKernelDefaults: true
{% endif %}
{% if kubelet_rotate_certificates | bool %}
rotateCertificates: true
{% endif %}
{% if kubelet_rotate_server_certificates | bool %}
serverTLSBootstrap: true
{% endif %}
{# DNS settings for kubelet #}
{% if enable_nodelocaldns %}
{% set kubelet_cluster_dns = [nodelocaldns_ip] %}
{% elif dns_mode in ['coredns'] %}
{% set kubelet_cluster_dns = [skydns_server] %}
{% elif dns_mode == 'coredns_dual' %}
{% set kubelet_cluster_dns = [skydns_server,skydns_server_secondary] %}
{% elif dns_mode == 'manual' %}
{% set kubelet_cluster_dns = [manual_dns_server] %}
{% else %}
{% set kubelet_cluster_dns = [] %}
{% endif %}
clusterDNS:
{% for dns_address in kubelet_cluster_dns %}
- {{ dns_address }}
{% endfor %}
{# Node reserved CPU/memory #}
{% for scope in "kube", "system" %}
{% if lookup('ansible.builtin.vars', scope + "_reserved") | bool %}
{{ scope }}ReservedCgroup: {{ lookup('ansible.builtin.vars', scope + '_reserved_cgroups') }}
{% endif %}
{{ scope }}Reserved:
{% for resource in "cpu", "memory", "ephemeral-storage", "pid" %}
{{ resource }}: "{{ lookup('ansible.builtin.vars', scope + '_' ~ (resource | replace('-', '_')) + '_reserved') }}"
{% endfor %}
{% endfor %}
{% if eviction_hard is defined and eviction_hard %}
evictionHard:
{{ eviction_hard | to_nice_yaml(indent=2) | indent(2) }}
{% endif %}
resolvConf: "{{ kube_resolv_conf }}"
{% if kubelet_config_extra_args %}
{{ kubelet_config_extra_args | to_nice_yaml(indent=2) }}
{% endif %}
{% if kubelet_feature_gates or kube_feature_gates %}
featureGates:
{% for feature in (kubelet_feature_gates | default(kube_feature_gates, true)) %}
{{ feature | replace("=", ": ") }}
{% endfor %}
{% endif %}
{% if tls_min_version is defined %}
tlsMinVersion: {{ tls_min_version }}
{% endif %}
{% if tls_cipher_suites is defined %}
tlsCipherSuites:
{% for tls in tls_cipher_suites %}
- {{ tls }}
{% endfor %}
{% endif %}
eventRecordQPS: {{ kubelet_event_record_qps }}
shutdownGracePeriod: {{ kubelet_shutdown_grace_period }}
shutdownGracePeriodCriticalPods: {{ kubelet_shutdown_grace_period_critical_pods }}
{% if not kubelet_fail_swap_on %}
memorySwap:
swapBehavior: {{ kubelet_swap_behavior }}
{% endif %}
{% if kubelet_streaming_connection_idle_timeout is defined %}
streamingConnectionIdleTimeout: {{ kubelet_streaming_connection_idle_timeout }}
{% endif %}
{% if kubelet_image_gc_high_threshold is defined %}
imageGCHighThresholdPercent: {{ kubelet_image_gc_high_threshold }}
{% endif %}
{% if kubelet_image_gc_low_threshold is defined %}
imageGCLowThresholdPercent: {{ kubelet_image_gc_low_threshold }}
{% endif %}
{% if kubelet_make_iptables_util_chains is defined %}
makeIPTablesUtilChains: {{ kubelet_make_iptables_util_chains | bool }}
{% endif %}
{% if kubelet_seccomp_default is defined %}
seccompDefault: {{ kubelet_seccomp_default | bool }}
{% endif %}
{% if kubelet_cpu_manager_policy is defined %}
cpuManagerPolicy: {{ kubelet_cpu_manager_policy }}
{% endif %}
{% if kubelet_cpu_manager_policy_options is defined %}
cpuManagerPolicyOptions:
{{ kubelet_cpu_manager_policy_options | to_nice_yaml(indent=2) | indent(width=2) }}
{% endif %}
{% if kubelet_topology_manager_policy is defined %}
topologyManagerPolicy: {{ kubelet_topology_manager_policy }}
{% endif %}
{% if kubelet_topology_manager_scope is defined %}
topologyManagerScope: {{ kubelet_topology_manager_scope }}
{% endif %}
{% if kubelet_tracing %}
tracing:
endpoint: "{{ kubelet_tracing_endpoint }}"
samplingRatePerMillion: {{ kubelet_tracing_sampling_rate_per_million }}
{% endif %}
maxParallelImagePulls: {{ kubelet_max_parallel_image_pulls }}

View File

@@ -0,0 +1,30 @@
KUBE_LOG_LEVEL="--v={{ kube_log_level }}"
KUBELET_ADDRESS="--node-ip={{ kubelet_address }}"
{% if kube_override_hostname|default('') %}
KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
{% endif %}
{# Base kubelet args #}
{% set kubelet_args_base -%}
{# start kubeadm specific settings #}
--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \
--config={{ kube_config_dir }}/kubelet-config.yaml \
--kubeconfig={{ kube_config_dir }}/kubelet.conf \
{# end kubeadm specific settings #}
--runtime-cgroups={{ kubelet_runtime_cgroups }} \
{% endset %}
KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_custom_flags | join(' ') }}"
{% if kubelet_flexvolumes_plugins_dir is defined %}
KUBELET_VOLUME_PLUGIN="--volume-plugin-dir={{ kubelet_flexvolumes_plugins_dir }}"
{% endif %}
{% if kube_network_plugin is defined and kube_network_plugin == "cloud" %}
KUBELET_NETWORK_PLUGIN="--hairpin-mode=promiscuous-bridge --network-plugin=kubenet"
{% endif %}
{% if cloud_provider == "external" %}
KUBELET_CLOUDPROVIDER="--cloud-provider={{ cloud_provider }}"
{% else %}
KUBELET_CLOUDPROVIDER=""
{% endif %}
PATH={{ bin_dir }}:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin

View File

@@ -0,0 +1,57 @@
[Unit]
Description=Kubernetes Kubelet Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After={{ container_manager }}.service
{% if container_manager == 'docker' %}
Wants=docker.socket
{% else %}
Wants={{ container_manager }}.service
{% endif %}
{% for kubelet_dependency in kubelet_systemd_wants_dependencies|default([]) %}
{% if kubelet_dependency|length > 0 %}
Wants={{ kubelet_dependency }}
{% endif %}
{% endfor %}
[Service]
EnvironmentFile=-{{ kube_config_dir }}/kubelet.env
{% if system_reserved|bool %}
ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/cpu/{{ system_reserved_cgroups_for_service_slice }}
ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/cpuacct/{{ system_reserved_cgroups_for_service_slice }}
ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/cpuset/{{ system_reserved_cgroups_for_service_slice }}
ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/hugetlb/{{ system_reserved_cgroups_for_service_slice }}
ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/memory/{{ system_reserved_cgroups_for_service_slice }}
ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/pids/{{ system_reserved_cgroups_for_service_slice }}
ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/systemd/{{ system_reserved_cgroups_for_service_slice }}
{% endif %}
{% if kube_reserved|bool %}
ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/cpu/{{ kube_reserved_cgroups_for_service_slice }}
ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/cpuacct/{{ kube_reserved_cgroups_for_service_slice }}
ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/cpuset/{{ kube_reserved_cgroups_for_service_slice }}
ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/hugetlb/{{ kube_reserved_cgroups_for_service_slice }}
ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/memory/{{ kube_reserved_cgroups_for_service_slice }}
ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/pids/{{ kube_reserved_cgroups_for_service_slice }}
ExecStartPre=/bin/mkdir -p /sys/fs/cgroup/systemd/{{ kube_reserved_cgroups_for_service_slice }}
{% endif %}
ExecStart={{ bin_dir }}/kubelet \
$KUBE_LOGTOSTDERR \
$KUBE_LOG_LEVEL \
$KUBELET_API_SERVER \
$KUBELET_ADDRESS \
$KUBELET_PORT \
$KUBELET_HOSTNAME \
$KUBELET_ARGS \
$DOCKER_SOCKET \
$KUBELET_NETWORK_PLUGIN \
$KUBELET_VOLUME_PLUGIN \
$KUBELET_CLOUDPROVIDER
Restart=always
RestartSec=10s
{% if kubelet_systemd_hardening %}
# Hardening setup
IPAddressDeny=any
IPAddressAllow={{ kubelet_secure_addresses }}
{% endif %}
[Install]
WantedBy=multi-user.target

View File

@@ -0,0 +1,49 @@
global
maxconn 4000
log 127.0.0.1 local0
defaults
mode http
log global
option httplog
option dontlognull
option http-server-close
option redispatch
retries 5
timeout http-request 5m
timeout queue 5m
timeout connect 30s
timeout client {{ loadbalancer_apiserver_keepalive_timeout }}
timeout server 15m
timeout http-keep-alive 30s
timeout check 30s
maxconn 4000
{% if loadbalancer_apiserver_healthcheck_port is defined -%}
frontend healthz
bind 0.0.0.0:{{ loadbalancer_apiserver_healthcheck_port }}
{% if ipv6_stack -%}
bind :::{{ loadbalancer_apiserver_healthcheck_port }}
{% endif -%}
mode http
monitor-uri /healthz
{% endif %}
frontend kube_api_frontend
bind 127.0.0.1:{{ loadbalancer_apiserver_port|default(kube_apiserver_port) }}
{% if ipv6_stack -%}
bind [::1]:{{ loadbalancer_apiserver_port|default(kube_apiserver_port) }};
{% endif -%}
mode tcp
option tcplog
default_backend kube_api_backend
backend kube_api_backend
mode tcp
balance leastconn
default-server inter 15s downinter 15s rise 2 fall 2 slowstart 60s maxconn 1000 maxqueue 256 weight 100
option httpchk GET /healthz
http-check expect status 200
{% for host in groups['kube_control_plane'] -%}
server {{ host }} {{ hostvars[host]['main_access_ip'] | ansible.utils.ipwrap }}:{{ kube_apiserver_port }} check check-ssl verify none
{% endfor -%}

View File

@@ -0,0 +1,60 @@
error_log stderr notice;
worker_processes 2;
worker_rlimit_nofile 130048;
worker_shutdown_timeout 10s;
events {
multi_accept on;
use epoll;
worker_connections 16384;
}
stream {
upstream kube_apiserver {
least_conn;
{% for host in groups['kube_control_plane'] -%}
server {{ hostvars[host]['main_access_ip'] | ansible.utils.ipwrap }}:{{ kube_apiserver_port }};
{% endfor -%}
}
server {
listen 127.0.0.1:{{ loadbalancer_apiserver_port|default(kube_apiserver_port) }};
{% if ipv6_stack -%}
listen [::1]:{{ loadbalancer_apiserver_port|default(kube_apiserver_port) }};
{% endif -%}
proxy_pass kube_apiserver;
proxy_timeout 10m;
proxy_connect_timeout 1s;
}
}
http {
aio threads;
aio_write on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout {{ loadbalancer_apiserver_keepalive_timeout }};
keepalive_requests 100;
reset_timedout_connection on;
server_tokens off;
autoindex off;
{% if loadbalancer_apiserver_healthcheck_port is defined -%}
server {
listen {{ loadbalancer_apiserver_healthcheck_port }};
{% if ipv6_stack -%}
listen [::]:{{ loadbalancer_apiserver_healthcheck_port }};
{% endif -%}
location /healthz {
access_log off;
return 200;
}
location /stub_status {
stub_status on;
access_log off;
}
}
{% endif %}
}

View File

@@ -0,0 +1,42 @@
apiVersion: v1
kind: Pod
metadata:
name: {{ loadbalancer_apiserver_pod_name }}
namespace: kube-system
labels:
addonmanager.kubernetes.io/mode: Reconcile
k8s-app: kube-haproxy
annotations:
haproxy-cfg-checksum: "{{ haproxy_stat.stat.checksum }}"
spec:
hostNetwork: true
dnsPolicy: ClusterFirstWithHostNet
nodeSelector:
kubernetes.io/os: linux
priorityClassName: system-node-critical
containers:
- name: haproxy
image: {{ haproxy_image_repo }}:{{ haproxy_image_tag }}
imagePullPolicy: {{ k8s_image_pull_policy }}
resources:
requests:
cpu: {{ loadbalancer_apiserver_cpu_requests }}
memory: {{ loadbalancer_apiserver_memory_requests }}
{% if loadbalancer_apiserver_healthcheck_port is defined -%}
livenessProbe:
httpGet:
path: /healthz
port: {{ loadbalancer_apiserver_healthcheck_port }}
readinessProbe:
httpGet:
path: /healthz
port: {{ loadbalancer_apiserver_healthcheck_port }}
{% endif -%}
volumeMounts:
- mountPath: /usr/local/etc/haproxy/
name: etc-haproxy
readOnly: true
volumes:
- name: etc-haproxy
hostPath:
path: {{ haproxy_config_dir }}

View File

@@ -0,0 +1,129 @@
# Inspired by https://github.com/kube-vip/kube-vip/blob/v0.8.0/pkg/kubevip/config_generator.go#L103
apiVersion: v1
kind: Pod
metadata:
name: kube-vip
namespace: kube-system
labels:
addonmanager.kubernetes.io/mode: Reconcile
k8s-app: kube-vip
spec:
containers:
- args:
- manager
env:
- name: vip_arp
value: {{ kube_vip_arp_enabled | string | to_json }}
- name: port
value: {{ kube_apiserver_port | string | to_json }}
- name: vip_nodename
value: {{ inventory_hostname }}
{% if kube_vip_interface %}
- name: vip_interface
value: {{ kube_vip_interface | string | to_json }}
{% endif %}
{% if kube_vip_services_interface %}
- name: vip_servicesinterface
value: {{ kube_vip_services_interface | string | to_json }}
{% endif %}
{% if kube_vip_cidr %}
- name: vip_cidr
value: {{ kube_vip_cidr | string | to_json }}
{% endif %}
{% if kube_vip_dns_mode %}
- name: dns_mode
value: {{ kube_vip_dns_mode | string | to_json }}
{% endif %}
{% if kube_vip_controlplane_enabled %}
- name: cp_enable
value: "true"
- name: cp_namespace
value: kube-system
- name: vip_ddns
value: {{ kube_vip_ddns_enabled | string | to_json }}
- name: cp_detect
value: {{ kube_vip_cp_detect | string | to_json }}
{% endif %}
{% if kube_vip_services_enabled %}
- name: svc_enable
value: "true"
{% endif %}
{% if kube_vip_svc_leasename %}
- name: svc_leasename
value: {{ kube_vip_svc_leasename | string | to_json }}
{% endif %}
{% if kube_vip_enableServicesElection %}
- name: svc_election
value: "true"
{% endif %}
{% if kube_vip_leader_election_enabled %}
- name: vip_leaderelection
value: "true"
- name: vip_leasename
value: {{ kube_vip_leasename | string | to_json }}
- name: vip_leaseduration
value: {{ kube_vip_leaseduration | string | to_json }}
- name: vip_renewdeadline
value: {{ kube_vip_renewdeadline | string | to_json }}
- name: vip_retryperiod
value: {{ kube_vip_retryperiod | string | to_json }}
{% endif %}
{% if kube_vip_enable_node_labeling %}
- name: enable_node_labeling
value: {{ kube_vip_enable_node_labeling | string | to_json }}
{% endif %}
{% if kube_vip_bgp_enabled %}
- name: bgp_enable
value: "true"
- name: bgp_routerid
value: {{ kube_vip_bgp_routerid | string | to_json }}
- name: bgp_as
value: {{ kube_vip_local_as | string | to_json }}
- name: bgp_peeraddress
value: {{ kube_vip_bgp_peeraddress | to_json }}
- name: bgp_peerpass
value: {{ kube_vip_bgp_peerpass | to_json }}
- name: bgp_peeras
value: {{ kube_vip_bgp_peeras | string | to_json }}
{% if kube_vip_bgppeers %}
- name: bgp_peers
value: {{ kube_vip_bgppeers | join(',') | to_json }}
{% endif %}
{% endif %}
- name: address
value: {{ kube_vip_address | to_json }}
{% if kube_vip_lb_enable %}
- name: lb_enable
value: "true"
{% endif %}
{% if kube_vip_lb_fwdmethod %}
- name: lb_fwdmethod
value: {{ kube_vip_lb_fwdmethod | string | to_json }}
{% endif %}
image: {{ kube_vip_image_repo }}:{{ kube_vip_image_tag }}
imagePullPolicy: {{ k8s_image_pull_policy }}
name: kube-vip
resources: {}
{% if kube_vip_lb_fwdmethod == "masquerade" %}
securityContext:
privileged: true
{% else %}
securityContext:
capabilities:
add:
- NET_ADMIN
- NET_RAW
{% endif %}
volumeMounts:
- mountPath: /etc/kubernetes/admin.conf
name: kubeconfig
hostAliases:
- hostnames:
- kubernetes
ip: 127.0.0.1
hostNetwork: true
volumes:
- hostPath:
path: /etc/kubernetes/{{kube_vip_admin_conf}}
name: kubeconfig
status: {}

View File

@@ -0,0 +1,42 @@
apiVersion: v1
kind: Pod
metadata:
name: {{ loadbalancer_apiserver_pod_name }}
namespace: kube-system
labels:
addonmanager.kubernetes.io/mode: Reconcile
k8s-app: kube-nginx
annotations:
nginx-cfg-checksum: "{{ nginx_stat.stat.checksum }}"
spec:
hostNetwork: true
dnsPolicy: ClusterFirstWithHostNet
nodeSelector:
kubernetes.io/os: linux
priorityClassName: system-node-critical
containers:
- name: nginx-proxy
image: {{ nginx_image_repo }}:{{ nginx_image_tag }}
imagePullPolicy: {{ k8s_image_pull_policy }}
resources:
requests:
cpu: {{ loadbalancer_apiserver_cpu_requests }}
memory: {{ loadbalancer_apiserver_memory_requests }}
{% if loadbalancer_apiserver_healthcheck_port is defined -%}
livenessProbe:
httpGet:
path: /healthz
port: {{ loadbalancer_apiserver_healthcheck_port }}
readinessProbe:
httpGet:
path: /healthz
port: {{ loadbalancer_apiserver_healthcheck_port }}
{% endif -%}
volumeMounts:
- mountPath: /etc/nginx
name: etc-nginx
readOnly: true
volumes:
- name: etc-nginx
hostPath:
path: {{ nginx_config_dir }}

View File

@@ -0,0 +1,19 @@
---
apiVersion: v1
kind: Config
clusters:
- name: local
cluster:
certificate-authority: {{ kube_cert_dir }}/ca.pem
server: "{{ kube_apiserver_endpoint }}"
users:
- name: kubelet
user:
client-certificate: {{ kube_cert_dir }}/node-{{ inventory_hostname }}.pem
client-key: {{ kube_cert_dir }}/node-{{ inventory_hostname }}-key.pem
contexts:
- context:
cluster: local
user: kubelet
name: kubelet-{{ cluster_name }}
current-context: kubelet-{{ cluster_name }}