kube-forge v2: embede python with ansible (kubespray) and cobra as cli building tool

internal/resources/kubespray/project/roles/etcd/handlers/backup.yml

@@ -0,0 +1,64 @@
+---
+- name: Refresh Time Fact
+  setup:
+    filter: ansible_date_time
+  listen: Restart etcd
+  when: etcd_cluster_is_healthy.rc == 0
+
+- name: Set Backup Directory
+  set_fact:
+    etcd_backup_directory: "{{ etcd_backup_prefix }}/etcd-{{ ansible_date_time.date }}_{{ ansible_date_time.time }}"
+  listen: Restart etcd
+
+- name: Create Backup Directory
+  file:
+    path: "{{ etcd_backup_directory }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "0600"
+  listen: Restart etcd
+  when: etcd_cluster_is_healthy.rc == 0
+
+- name: Stat etcd v2 data directory
+  stat:
+    path: "{{ etcd_data_dir }}/member"
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: etcd_data_dir_member
+  listen: Restart etcd
+  when: etcd_cluster_is_healthy.rc == 0
+
+- name: Backup etcd v2 data
+  when:
+    - etcd_data_dir_member.stat.exists
+    - etcd_cluster_is_healthy.rc == 0
+  command: >-
+    {{ bin_dir }}/etcdctl backup
+      --data-dir {{ etcd_data_dir }}
+      --backup-dir {{ etcd_backup_directory }}
+  environment:
+    ETCDCTL_API: "2"
+  retries: 3
+  register: backup_v2_command
+  until: backup_v2_command.rc == 0
+  delay: "{{ retry_stagger | random + 3 }}"
+  listen: Restart etcd
+
+- name: Backup etcd v3 data
+  command: >-
+    {{ bin_dir }}/etcdctl
+      snapshot save {{ etcd_backup_directory }}/snapshot.db
+  environment:
+    ETCDCTL_API: "3"
+    ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses.split(',') | first }}"
+    ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
+    ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
+    ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem"
+  retries: 3
+  register: etcd_backup_v3_command
+  until: etcd_backup_v3_command.rc == 0
+  delay: "{{ retry_stagger | random + 3 }}"
+  listen: Restart etcd
+  when: etcd_cluster_is_healthy.rc == 0

internal/resources/kubespray/project/roles/etcd/handlers/backup_cleanup.yml

@@ -0,0 +1,18 @@
+---
+- name: Find old etcd backups
+  ansible.builtin.find:
+    file_type: directory
+    recurse: false
+    paths: "{{ etcd_backup_prefix }}"
+    patterns: "etcd-*"
+  register: _etcd_backups
+  when: etcd_backup_retention_count >= 0
+  listen: Restart etcd
+
+- name: Remove old etcd backups
+  ansible.builtin.file:
+    state: absent
+    path: "{{ item }}"
+  loop: "{{ (_etcd_backups.files | sort(attribute='ctime', reverse=True))[etcd_backup_retention_count:] | map(attribute='path') }}"
+  when: etcd_backup_retention_count >= 0
+  listen: Restart etcd

internal/resources/kubespray/project/roles/etcd/handlers/main.yml

@@ -0,0 +1,54 @@
+---
+- name: Backup etcd
+  import_tasks: backup.yml
+
+- name: Restart etcd
+  systemd_service:
+    name: etcd
+    state: restarted
+    daemon_reload: true
+  when: ('etcd' in group_names)
+  throttle: "{{ groups['etcd'] | length // 2 }}"
+  # Etcd cluster MUST have an odd number of members
+  # Truncated integer division by 2 will always return (majority - 1) which
+  # means the cluster will keep quorum and stay available
+
+- name: Restart etcd-events
+  systemd_service:
+    name: etcd-events
+    state: restarted
+    daemon_reload: true
+  # TODO: this seems odd. etcd-events should be a different group possibly ?
+  when: ('etcd' in group_names)
+  throttle: "{{ groups['etcd'] | length // 2 }}"
+
+- name: Wait for etcd up
+  uri:
+    url: "https://{% if 'etcd' in group_names %}{{ etcd_address | ansible.utils.ipwrap }}{% else %}127.0.0.1{% endif %}:2379/health"
+    validate_certs: false
+    client_cert: "{{ etcd_cert_dir }}/member-{{ inventory_hostname }}.pem"
+    client_key: "{{ etcd_cert_dir }}/member-{{ inventory_hostname }}-key.pem"
+  register: result
+  until: result.status is defined and result.status == 200
+  retries: 60
+  delay: 1
+  listen: Restart etcd
+
+- name: Cleanup etcd backups
+  import_tasks: backup_cleanup.yml
+
+- name: Wait for etcd-events up
+  uri:
+    url: "https://{% if 'etcd' in group_names %}{{ etcd_address | ansible.utils.ipwrap }}{% else %}127.0.0.1{% endif %}:2383/health"
+    validate_certs: false
+    client_cert: "{{ etcd_cert_dir }}/member-{{ inventory_hostname }}.pem"
+    client_key: "{{ etcd_cert_dir }}/member-{{ inventory_hostname }}-key.pem"
+  register: result
+  until: result.status is defined and result.status == 200
+  retries: 60
+  delay: 1
+  listen: Restart etcd-events
+
+- name: Set etcd_secret_changed
+  set_fact:
+    etcd_secret_changed: true