Automation Orchestrator install and tweaks

roles/sno_etcd_defrag/defaults/main.yml

@@ -0,0 +1,9 @@
+---
+# Defragment only when the in-use fraction of the etcd database drops below this
+# ratio and the database exceeds the minimum size (mirrors the thresholds of the
+# etcdDatabaseHighFragmentationRatio alert: < 50% in use and > 100 MiB).
+sno_etcd_defrag_max_in_use_ratio: 0.5
+sno_etcd_defrag_min_db_size_bytes: 104857600
+sno_etcd_defrag_command_timeout: 60
+# Set true to defragment regardless of the thresholds.
+sno_etcd_defrag_force: false

roles/sno_etcd_defrag/meta/argument_specs.yml

@@ -0,0 +1,26 @@
+---
+argument_specs:
+  main:
+    short_description: Defragment etcd on a single-control-plane OpenShift cluster
+    description:
+      - cluster-etcd-operator disables automatic defragmentation on SingleReplica topology.
+        This role defragments the local member when fragmentation exceeds the thresholds of
+        the etcdDatabaseHighFragmentationRatio alert, then clears any NOSPACE alarm.
+      - Blocks etcd for the duration of the defrag. Requires pods/exec in openshift-etcd.
+    options:
+      sno_etcd_defrag_max_in_use_ratio:
+        description: Defragment when dbSizeInUse / dbSize is below this ratio.
+        type: float
+        default: 0.5
+      sno_etcd_defrag_min_db_size_bytes:
+        description: Skip defragmentation when the database is smaller than this.
+        type: int
+        default: 104857600
+      sno_etcd_defrag_command_timeout:
+        description: etcdctl --command-timeout for the defrag, in seconds.
+        type: int
+        default: 60
+      sno_etcd_defrag_force:
+        description: Defragment regardless of the thresholds.
+        type: bool
+        default: false

roles/sno_etcd_defrag/meta/main.yml

@@ -0,0 +1,16 @@
+---
+galaxy_info:
+  author: ptoal
+  description: Defragment etcd on single-node OpenShift (automatic defrag is disabled there)
+  license: MIT
+  min_ansible_version: "2.16"
+  platforms:
+    - name: GenericLinux
+      versions:
+        - all
+  galaxy_tags:
+    - openshift
+    - etcd
+    - sno
+
+dependencies: []

roles/sno_etcd_defrag/tasks/main.yml

@@ -0,0 +1,103 @@
+---
+# Defragment etcd on a single-control-plane cluster.
+#
+# cluster-etcd-operator disables its defrag controller on SingleReplica topology
+# (Etcd/cluster condition DefragControllerDisabled=True), so fragmentation grows
+# until etcdDatabaseHighFragmentationRatio fires. This role follows the documented
+# manual procedure: defrag the local member, then clear any NOSPACE alarm.
+#
+# Defragmenting the only member blocks etcd reads/writes for the duration (seconds
+# for a few hundred MiB) — schedule it outside busy periods.
+#
+# Requires pods/exec in openshift-etcd (cluster-admin kubeconfig).
+
+- name: Find the etcd pod
+  kubernetes.core.k8s_info:
+    api_version: v1
+    kind: Pod
+    namespace: openshift-etcd
+    label_selectors:
+      - app=etcd
+    field_selectors:
+      - status.phase=Running
+  register: __sno_etcd_defrag_pods
+  failed_when: __sno_etcd_defrag_pods.resources | length != 1
+
+- name: Set etcd pod name
+  ansible.builtin.set_fact:
+    __sno_etcd_defrag_pod: "{{ __sno_etcd_defrag_pods.resources[0].metadata.name }}"
+
+- name: Read etcd database size before defrag
+  kubernetes.core.k8s_exec:
+    namespace: openshift-etcd
+    pod: "{{ __sno_etcd_defrag_pod }}"
+    container: etcdctl
+    command: etcdctl endpoint status -w json
+  register: __sno_etcd_defrag_before
+  changed_when: false
+
+- name: Evaluate fragmentation
+  ansible.builtin.set_fact:
+    __sno_etcd_defrag_status_before: "{{ (__sno_etcd_defrag_before.stdout | from_json)[0].Status }}"
+
+- name: Decide whether to defragment
+  ansible.builtin.set_fact:
+    __sno_etcd_defrag_needed: >-
+      {{ sno_etcd_defrag_force | bool or (
+           __sno_etcd_defrag_status_before.dbSize | int > sno_etcd_defrag_min_db_size_bytes | int and
+           (__sno_etcd_defrag_status_before.dbSizeInUse | float / __sno_etcd_defrag_status_before.dbSize | float)
+             < sno_etcd_defrag_max_in_use_ratio | float) }}
+
+- name: Defragment etcd
+  when: __sno_etcd_defrag_needed | bool
+  block:
+    - name: Defragment the local etcd member
+      kubernetes.core.k8s_exec:
+        namespace: openshift-etcd
+        pod: "{{ __sno_etcd_defrag_pod }}"
+        container: etcdctl
+        command: >-
+          sh -c 'unset ETCDCTL_ENDPOINTS;
+          etcdctl --command-timeout={{ sno_etcd_defrag_command_timeout }}s --endpoints=https://localhost:2379 defrag'
+      register: __sno_etcd_defrag_result
+      changed_when: true
+      failed_when: __sno_etcd_defrag_result.rc != 0
+
+    - name: List etcd alarms
+      kubernetes.core.k8s_exec:
+        namespace: openshift-etcd
+        pod: "{{ __sno_etcd_defrag_pod }}"
+        container: etcdctl
+        command: etcdctl alarm list
+      register: __sno_etcd_defrag_alarms
+      changed_when: false
+      retries: 6
+      delay: 10
+      until: __sno_etcd_defrag_alarms.rc | default(1) == 0
+
+    - name: Disarm NOSPACE alarm
+      kubernetes.core.k8s_exec:
+        namespace: openshift-etcd
+        pod: "{{ __sno_etcd_defrag_pod }}"
+        container: etcdctl
+        command: etcdctl alarm disarm
+      changed_when: true
+      when: "'NOSPACE' in __sno_etcd_defrag_alarms.stdout"
+
+    - name: Read etcd database size after defrag
+      kubernetes.core.k8s_exec:
+        namespace: openshift-etcd
+        pod: "{{ __sno_etcd_defrag_pod }}"
+        container: etcdctl
+        command: etcdctl endpoint status -w json
+      register: __sno_etcd_defrag_after
+      changed_when: false
+
+- name: Display etcd defrag summary
+  ansible.builtin.debug:
+    msg:
+      - "etcd {{ 'defragmented' if __sno_etcd_defrag_needed | bool else 'not defragmented (below thresholds)' }}"
+      - "  Before : {{ (__sno_etcd_defrag_status_before.dbSize / 1048576) | round(0) | int }} MiB allocated,
+         {{ (__sno_etcd_defrag_status_before.dbSizeInUse / 1048576) | round(0) | int }} MiB in use"
+      - "  After  : {{ ((__sno_etcd_defrag_after.stdout | from_json)[0].Status.dbSize / 1048576) | round(0) | int ~ ' MiB allocated'
+                       if __sno_etcd_defrag_needed | bool else 'n/a' }}"