--- # Defragment etcd on a single-control-plane cluster. # # cluster-etcd-operator disables its defrag controller on SingleReplica topology # (Etcd/cluster condition DefragControllerDisabled=True), so fragmentation grows # until etcdDatabaseHighFragmentationRatio fires. This role follows the documented # manual procedure: defrag the local member, then clear any NOSPACE alarm. # # Defragmenting the only member blocks etcd reads/writes for the duration (seconds # for a few hundred MiB) — schedule it outside busy periods. # # Requires pods/exec in openshift-etcd (cluster-admin kubeconfig). - name: Find the etcd pod kubernetes.core.k8s_info: api_version: v1 kind: Pod namespace: openshift-etcd label_selectors: - app=etcd field_selectors: - status.phase=Running register: __sno_etcd_defrag_pods failed_when: __sno_etcd_defrag_pods.resources | length != 1 - name: Set etcd pod name ansible.builtin.set_fact: __sno_etcd_defrag_pod: "{{ __sno_etcd_defrag_pods.resources[0].metadata.name }}" - name: Read etcd database size before defrag kubernetes.core.k8s_exec: namespace: openshift-etcd pod: "{{ __sno_etcd_defrag_pod }}" container: etcdctl command: etcdctl endpoint status -w json register: __sno_etcd_defrag_before changed_when: false - name: Evaluate fragmentation ansible.builtin.set_fact: __sno_etcd_defrag_status_before: "{{ (__sno_etcd_defrag_before.stdout | from_json)[0].Status }}" - name: Decide whether to defragment ansible.builtin.set_fact: __sno_etcd_defrag_needed: >- {{ sno_etcd_defrag_force | bool or ( __sno_etcd_defrag_status_before.dbSize | int > sno_etcd_defrag_min_db_size_bytes | int and (__sno_etcd_defrag_status_before.dbSizeInUse | float / __sno_etcd_defrag_status_before.dbSize | float) < sno_etcd_defrag_max_in_use_ratio | float) }} - name: Defragment etcd when: __sno_etcd_defrag_needed | bool block: - name: Defragment the local etcd member kubernetes.core.k8s_exec: namespace: openshift-etcd pod: "{{ __sno_etcd_defrag_pod }}" container: etcdctl command: >- sh -c 'unset ETCDCTL_ENDPOINTS; etcdctl --command-timeout={{ sno_etcd_defrag_command_timeout }}s --endpoints=https://localhost:2379 defrag' register: __sno_etcd_defrag_result changed_when: true failed_when: __sno_etcd_defrag_result.rc != 0 - name: List etcd alarms kubernetes.core.k8s_exec: namespace: openshift-etcd pod: "{{ __sno_etcd_defrag_pod }}" container: etcdctl command: etcdctl alarm list register: __sno_etcd_defrag_alarms changed_when: false retries: 6 delay: 10 until: __sno_etcd_defrag_alarms.rc | default(1) == 0 - name: Disarm NOSPACE alarm kubernetes.core.k8s_exec: namespace: openshift-etcd pod: "{{ __sno_etcd_defrag_pod }}" container: etcdctl command: etcdctl alarm disarm changed_when: true when: "'NOSPACE' in __sno_etcd_defrag_alarms.stdout" - name: Read etcd database size after defrag kubernetes.core.k8s_exec: namespace: openshift-etcd pod: "{{ __sno_etcd_defrag_pod }}" container: etcdctl command: etcdctl endpoint status -w json register: __sno_etcd_defrag_after changed_when: false - name: Display etcd defrag summary ansible.builtin.debug: msg: - "etcd {{ 'defragmented' if __sno_etcd_defrag_needed | bool else 'not defragmented (below thresholds)' }}" - " Before : {{ (__sno_etcd_defrag_status_before.dbSize / 1048576) | round(0) | int }} MiB allocated, {{ (__sno_etcd_defrag_status_before.dbSizeInUse / 1048576) | round(0) | int }} MiB in use" - " After : {{ ((__sno_etcd_defrag_after.stdout | from_json)[0].Status.dbSize / 1048576) | round(0) | int ~ ' MiB allocated' if __sno_etcd_defrag_needed | bool else 'n/a' }}"