Automation Orchestrator install and tweaks

playbooks/deploy_openshift.yml

@@ -33,6 +33,9 @@
 #   Play 7: sno_deploy_lvms            — Install LVM Storage for persistent volumes
 #   Play 8: sno_deploy_nfs             — Deploy in-cluster NFS provisioner (RWX StorageClass)
 #   Play 9: sno_deploy_service_accounts — Provision ServiceAccounts for app deployers
+#   Play 10: sno_deploy_alert_tuning    — Drop single-control-plane false-positive alerts
+#
+# Recurring maintenance (separate playbook, schedule in AAP): maintain_sno_etcd.yml
 #
 # AAP deployment is in a separate playbook: deploy_aap.yml
 #
@@ -46,6 +49,7 @@
 #   ansible-navigator run playbooks/deploy_openshift.yml --tags sno_deploy_lvms
 #   ansible-navigator run playbooks/deploy_openshift.yml --tags sno_deploy_nfs
 #   ansible-navigator run playbooks/deploy_openshift.yml --tags sno_deploy_service_accounts
+#   ansible-navigator run playbooks/deploy_openshift.yml --tags sno_deploy_alert_tuning
 
 # ---------------------------------------------------------------------------
 # Play 1: Create SNO VM in Proxmox
@@ -438,3 +442,20 @@
         data:
           token: "{{ __ocp_service_account_token }}"
       no_log: true
+
+# ---------------------------------------------------------------------------
+# Play 10: Drop platform alerts that are false positives on a single
+# control-plane cluster (AlertRelabelConfig in openshift-monitoring)
+# ---------------------------------------------------------------------------
+- name: Tune platform alerts for single-node OpenShift
+  hosts: sno.openshift.toal.ca
+  gather_facts: false
+  connection: local
+  tags: sno_deploy_alert_tuning
+
+  environment:
+    KUBECONFIG: "{{ ocp_kubeconfig | default('~/.kube/config') }}"
+    K8S_AUTH_VERIFY_SSL: "false"
+
+  roles:
+    - role: sno_alert_tuning

playbooks/maintain_sno_etcd.yml

@@ -0,0 +1,23 @@
+---
+# Defragment etcd on the single-node OpenShift cluster.
+#
+# OpenShift disables automatic etcd defragmentation on single-control-plane
+# clusters, so run this on a schedule (e.g. weekly AAP schedule, off-hours).
+# It only defragments when fragmentation exceeds the thresholds of the
+# etcdDatabaseHighFragmentationRatio alert; otherwise it reports and exits.
+#
+# Usage:
+#   ansible-navigator run playbooks/maintain_sno_etcd.yml --mode stdout
+#   ansible-navigator run playbooks/maintain_sno_etcd.yml --mode stdout -e sno_etcd_defrag_force=true
+
+- name: Defragment etcd on single-node OpenShift
+  hosts: sno.openshift.toal.ca
+  gather_facts: false
+  connection: local
+
+  environment:
+    KUBECONFIG: "{{ ocp_kubeconfig | default('~/.kube/config') }}"
+    K8S_AUTH_VERIFY_SSL: "false"
+
+  roles:
+    - role: sno_etcd_defrag

roles/cloudnativepg_operator/defaults/main.yml

@@ -15,6 +15,9 @@ cloudnativepg_operator_cluster_namespace: automation-orchestrator
 cloudnativepg_operator_cluster_name: orchestrator-postgres
 cloudnativepg_operator_owner_user: orchestrator
 cloudnativepg_operator_instances: 1
+# A primary PDB on a single instance is permanently at its limit (PodDisruptionBudgetAtLimit)
+# and blocks node drains; CNPG recommends disabling it for single-instance dev clusters.
+cloudnativepg_operator_enable_pdb: "{{ cloudnativepg_operator_instances | int > 1 }}"
 cloudnativepg_operator_image_name: ghcr.io/cloudnative-pg/postgresql:15
 cloudnativepg_operator_storage_size: 5Gi
 cloudnativepg_operator_storage_class: ""

roles/cloudnativepg_operator/meta/argument_specs.yml

@@ -49,6 +49,11 @@ argument_specs:
         description: Number of Postgres instances in the Cluster (1 = no HA).
         type: int
         default: 1
+      cloudnativepg_operator_enable_pdb:
+        description: >-
+          Create the CNPG primary PodDisruptionBudget. Defaults to true only when instances > 1;
+          a single-instance PDB is always at its limit and blocks node drains.
+        type: bool
       cloudnativepg_operator_image_name:
         description: PostgreSQL container image for the Cluster.
         type: str

roles/cloudnativepg_operator/tasks/main.yml

@@ -166,6 +166,7 @@
         {{
           {
             'instances': cloudnativepg_operator_instances,
+            'enablePDB': cloudnativepg_operator_enable_pdb | bool,
             'imageName': cloudnativepg_operator_image_name,
             'bootstrap': {
               'initdb': {

roles/sno_alert_tuning/defaults/main.yml

@@ -0,0 +1,13 @@
+---
+# Platform alerts that are false positives on a single-control-plane cluster.
+# Dropped via an AlertRelabelConfig (monitoring.openshift.io/v1), so they never
+# reach Alertmanager. Alerts are matched by exact alertname.
+#
+# KubeCPUOvercommit / KubeMemoryOvercommit: the HA branch of these rules assumes
+# the cluster survives losing its largest node. With one control-plane node plus
+# smaller workers, that node is master-0 — losing it loses the cluster anyway.
+sno_alert_tuning_dropped_alerts:
+  - KubeCPUOvercommit
+  - KubeMemoryOvercommit
+
+sno_alert_tuning_config_name: sno-single-control-plane

roles/sno_alert_tuning/meta/argument_specs.yml

@@ -0,0 +1,17 @@
+---
+argument_specs:
+  main:
+    short_description: Drop platform alerts that are false positives on a single-control-plane cluster
+    description:
+      - Creates an AlertRelabelConfig in openshift-monitoring that drops the listed alerts
+        before they reach Alertmanager. An empty list removes the AlertRelabelConfig.
+    options:
+      sno_alert_tuning_dropped_alerts:
+        description: Exact alertnames to drop.
+        type: list
+        elements: str
+        default: [KubeCPUOvercommit, KubeMemoryOvercommit]
+      sno_alert_tuning_config_name:
+        description: Name of the AlertRelabelConfig.
+        type: str
+        default: sno-single-control-plane

roles/sno_alert_tuning/meta/main.yml

@@ -0,0 +1,16 @@
+---
+galaxy_info:
+  author: ptoal
+  description: Drop platform alerts that are false positives on single-node OpenShift
+  license: MIT
+  min_ansible_version: "2.16"
+  platforms:
+    - name: GenericLinux
+      versions:
+        - all
+  galaxy_tags:
+    - openshift
+    - monitoring
+    - sno
+
+dependencies: []

roles/sno_alert_tuning/tasks/main.yml

@@ -0,0 +1,19 @@
+---
+# Drop platform alerts that only fire because the cluster has a single
+# control-plane node. Uses the supported AlertRelabelConfig API rather than
+# silences (which expire) or editing cluster-monitoring-operator rules (reverted).
+
+- name: Drop single-control-plane false-positive alerts
+  kubernetes.core.k8s:
+    state: "{{ 'present' if sno_alert_tuning_dropped_alerts | length > 0 else 'absent' }}"
+    definition:
+      apiVersion: monitoring.openshift.io/v1
+      kind: AlertRelabelConfig
+      metadata:
+        name: "{{ sno_alert_tuning_config_name }}"
+        namespace: openshift-monitoring
+      spec:
+        configs:
+          - sourceLabels: [alertname]
+            regex: "{{ sno_alert_tuning_dropped_alerts | map('regex_escape') | join('|') }}"
+            action: Drop

roles/sno_etcd_defrag/defaults/main.yml

@@ -0,0 +1,9 @@
+---
+# Defragment only when the in-use fraction of the etcd database drops below this
+# ratio and the database exceeds the minimum size (mirrors the thresholds of the
+# etcdDatabaseHighFragmentationRatio alert: < 50% in use and > 100 MiB).
+sno_etcd_defrag_max_in_use_ratio: 0.5
+sno_etcd_defrag_min_db_size_bytes: 104857600
+sno_etcd_defrag_command_timeout: 60
+# Set true to defragment regardless of the thresholds.
+sno_etcd_defrag_force: false

roles/sno_etcd_defrag/meta/argument_specs.yml

@@ -0,0 +1,26 @@
+---
+argument_specs:
+  main:
+    short_description: Defragment etcd on a single-control-plane OpenShift cluster
+    description:
+      - cluster-etcd-operator disables automatic defragmentation on SingleReplica topology.
+        This role defragments the local member when fragmentation exceeds the thresholds of
+        the etcdDatabaseHighFragmentationRatio alert, then clears any NOSPACE alarm.
+      - Blocks etcd for the duration of the defrag. Requires pods/exec in openshift-etcd.
+    options:
+      sno_etcd_defrag_max_in_use_ratio:
+        description: Defragment when dbSizeInUse / dbSize is below this ratio.
+        type: float
+        default: 0.5
+      sno_etcd_defrag_min_db_size_bytes:
+        description: Skip defragmentation when the database is smaller than this.
+        type: int
+        default: 104857600
+      sno_etcd_defrag_command_timeout:
+        description: etcdctl --command-timeout for the defrag, in seconds.
+        type: int
+        default: 60
+      sno_etcd_defrag_force:
+        description: Defragment regardless of the thresholds.
+        type: bool
+        default: false

roles/sno_etcd_defrag/meta/main.yml

@@ -0,0 +1,16 @@
+---
+galaxy_info:
+  author: ptoal
+  description: Defragment etcd on single-node OpenShift (automatic defrag is disabled there)
+  license: MIT
+  min_ansible_version: "2.16"
+  platforms:
+    - name: GenericLinux
+      versions:
+        - all
+  galaxy_tags:
+    - openshift
+    - etcd
+    - sno
+
+dependencies: []

roles/sno_etcd_defrag/tasks/main.yml

@@ -0,0 +1,103 @@
+---
+# Defragment etcd on a single-control-plane cluster.
+#
+# cluster-etcd-operator disables its defrag controller on SingleReplica topology
+# (Etcd/cluster condition DefragControllerDisabled=True), so fragmentation grows
+# until etcdDatabaseHighFragmentationRatio fires. This role follows the documented
+# manual procedure: defrag the local member, then clear any NOSPACE alarm.
+#
+# Defragmenting the only member blocks etcd reads/writes for the duration (seconds
+# for a few hundred MiB) — schedule it outside busy periods.
+#
+# Requires pods/exec in openshift-etcd (cluster-admin kubeconfig).
+
+- name: Find the etcd pod
+  kubernetes.core.k8s_info:
+    api_version: v1
+    kind: Pod
+    namespace: openshift-etcd
+    label_selectors:
+      - app=etcd
+    field_selectors:
+      - status.phase=Running
+  register: __sno_etcd_defrag_pods
+  failed_when: __sno_etcd_defrag_pods.resources | length != 1
+
+- name: Set etcd pod name
+  ansible.builtin.set_fact:
+    __sno_etcd_defrag_pod: "{{ __sno_etcd_defrag_pods.resources[0].metadata.name }}"
+
+- name: Read etcd database size before defrag
+  kubernetes.core.k8s_exec:
+    namespace: openshift-etcd
+    pod: "{{ __sno_etcd_defrag_pod }}"
+    container: etcdctl
+    command: etcdctl endpoint status -w json
+  register: __sno_etcd_defrag_before
+  changed_when: false
+
+- name: Evaluate fragmentation
+  ansible.builtin.set_fact:
+    __sno_etcd_defrag_status_before: "{{ (__sno_etcd_defrag_before.stdout | from_json)[0].Status }}"
+
+- name: Decide whether to defragment
+  ansible.builtin.set_fact:
+    __sno_etcd_defrag_needed: >-
+      {{ sno_etcd_defrag_force | bool or (
+           __sno_etcd_defrag_status_before.dbSize | int > sno_etcd_defrag_min_db_size_bytes | int and
+           (__sno_etcd_defrag_status_before.dbSizeInUse | float / __sno_etcd_defrag_status_before.dbSize | float)
+             < sno_etcd_defrag_max_in_use_ratio | float) }}
+
+- name: Defragment etcd
+  when: __sno_etcd_defrag_needed | bool
+  block:
+    - name: Defragment the local etcd member
+      kubernetes.core.k8s_exec:
+        namespace: openshift-etcd
+        pod: "{{ __sno_etcd_defrag_pod }}"
+        container: etcdctl
+        command: >-
+          sh -c 'unset ETCDCTL_ENDPOINTS;
+          etcdctl --command-timeout={{ sno_etcd_defrag_command_timeout }}s --endpoints=https://localhost:2379 defrag'
+      register: __sno_etcd_defrag_result
+      changed_when: true
+      failed_when: __sno_etcd_defrag_result.rc != 0
+
+    - name: List etcd alarms
+      kubernetes.core.k8s_exec:
+        namespace: openshift-etcd
+        pod: "{{ __sno_etcd_defrag_pod }}"
+        container: etcdctl
+        command: etcdctl alarm list
+      register: __sno_etcd_defrag_alarms
+      changed_when: false
+      retries: 6
+      delay: 10
+      until: __sno_etcd_defrag_alarms.rc | default(1) == 0
+
+    - name: Disarm NOSPACE alarm
+      kubernetes.core.k8s_exec:
+        namespace: openshift-etcd
+        pod: "{{ __sno_etcd_defrag_pod }}"
+        container: etcdctl
+        command: etcdctl alarm disarm
+      changed_when: true
+      when: "'NOSPACE' in __sno_etcd_defrag_alarms.stdout"
+
+    - name: Read etcd database size after defrag
+      kubernetes.core.k8s_exec:
+        namespace: openshift-etcd
+        pod: "{{ __sno_etcd_defrag_pod }}"
+        container: etcdctl
+        command: etcdctl endpoint status -w json
+      register: __sno_etcd_defrag_after
+      changed_when: false
+
+- name: Display etcd defrag summary
+  ansible.builtin.debug:
+    msg:
+      - "etcd {{ 'defragmented' if __sno_etcd_defrag_needed | bool else 'not defragmented (below thresholds)' }}"
+      - "  Before : {{ (__sno_etcd_defrag_status_before.dbSize / 1048576) | round(0) | int }} MiB allocated,
+         {{ (__sno_etcd_defrag_status_before.dbSizeInUse / 1048576) | round(0) | int }} MiB in use"
+      - "  After  : {{ ((__sno_etcd_defrag_after.stdout | from_json)[0].Status.dbSize / 1048576) | round(0) | int ~ ' MiB allocated'
+                       if __sno_etcd_defrag_needed | bool else 'n/a' }}"