diff --git a/playbooks/deploy_openshift.yml b/playbooks/deploy_openshift.yml index edadde2..475d661 100644 --- a/playbooks/deploy_openshift.yml +++ b/playbooks/deploy_openshift.yml @@ -33,6 +33,9 @@ # Play 7: sno_deploy_lvms — Install LVM Storage for persistent volumes # Play 8: sno_deploy_nfs — Deploy in-cluster NFS provisioner (RWX StorageClass) # Play 9: sno_deploy_service_accounts — Provision ServiceAccounts for app deployers +# Play 10: sno_deploy_alert_tuning — Drop single-control-plane false-positive alerts +# +# Recurring maintenance (separate playbook, schedule in AAP): maintain_sno_etcd.yml # # AAP deployment is in a separate playbook: deploy_aap.yml # @@ -46,6 +49,7 @@ # ansible-navigator run playbooks/deploy_openshift.yml --tags sno_deploy_lvms # ansible-navigator run playbooks/deploy_openshift.yml --tags sno_deploy_nfs # ansible-navigator run playbooks/deploy_openshift.yml --tags sno_deploy_service_accounts +# ansible-navigator run playbooks/deploy_openshift.yml --tags sno_deploy_alert_tuning # --------------------------------------------------------------------------- # Play 1: Create SNO VM in Proxmox @@ -438,3 +442,20 @@ data: token: "{{ __ocp_service_account_token }}" no_log: true + +# --------------------------------------------------------------------------- +# Play 10: Drop platform alerts that are false positives on a single +# control-plane cluster (AlertRelabelConfig in openshift-monitoring) +# --------------------------------------------------------------------------- +- name: Tune platform alerts for single-node OpenShift + hosts: sno.openshift.toal.ca + gather_facts: false + connection: local + tags: sno_deploy_alert_tuning + + environment: + KUBECONFIG: "{{ ocp_kubeconfig | default('~/.kube/config') }}" + K8S_AUTH_VERIFY_SSL: "false" + + roles: + - role: sno_alert_tuning diff --git a/playbooks/maintain_sno_etcd.yml b/playbooks/maintain_sno_etcd.yml new file mode 100644 index 0000000..035792d --- /dev/null +++ b/playbooks/maintain_sno_etcd.yml @@ -0,0 +1,23 @@ +--- +# Defragment etcd on the single-node OpenShift cluster. +# +# OpenShift disables automatic etcd defragmentation on single-control-plane +# clusters, so run this on a schedule (e.g. weekly AAP schedule, off-hours). +# It only defragments when fragmentation exceeds the thresholds of the +# etcdDatabaseHighFragmentationRatio alert; otherwise it reports and exits. +# +# Usage: +# ansible-navigator run playbooks/maintain_sno_etcd.yml --mode stdout +# ansible-navigator run playbooks/maintain_sno_etcd.yml --mode stdout -e sno_etcd_defrag_force=true + +- name: Defragment etcd on single-node OpenShift + hosts: sno.openshift.toal.ca + gather_facts: false + connection: local + + environment: + KUBECONFIG: "{{ ocp_kubeconfig | default('~/.kube/config') }}" + K8S_AUTH_VERIFY_SSL: "false" + + roles: + - role: sno_etcd_defrag diff --git a/roles/cloudnativepg_operator/defaults/main.yml b/roles/cloudnativepg_operator/defaults/main.yml index ff2506a..657908e 100644 --- a/roles/cloudnativepg_operator/defaults/main.yml +++ b/roles/cloudnativepg_operator/defaults/main.yml @@ -15,6 +15,9 @@ cloudnativepg_operator_cluster_namespace: automation-orchestrator cloudnativepg_operator_cluster_name: orchestrator-postgres cloudnativepg_operator_owner_user: orchestrator cloudnativepg_operator_instances: 1 +# A primary PDB on a single instance is permanently at its limit (PodDisruptionBudgetAtLimit) +# and blocks node drains; CNPG recommends disabling it for single-instance dev clusters. +cloudnativepg_operator_enable_pdb: "{{ cloudnativepg_operator_instances | int > 1 }}" cloudnativepg_operator_image_name: ghcr.io/cloudnative-pg/postgresql:15 cloudnativepg_operator_storage_size: 5Gi cloudnativepg_operator_storage_class: "" diff --git a/roles/cloudnativepg_operator/meta/argument_specs.yml b/roles/cloudnativepg_operator/meta/argument_specs.yml index cc424b9..6e60812 100644 --- a/roles/cloudnativepg_operator/meta/argument_specs.yml +++ b/roles/cloudnativepg_operator/meta/argument_specs.yml @@ -49,6 +49,11 @@ argument_specs: description: Number of Postgres instances in the Cluster (1 = no HA). type: int default: 1 + cloudnativepg_operator_enable_pdb: + description: >- + Create the CNPG primary PodDisruptionBudget. Defaults to true only when instances > 1; + a single-instance PDB is always at its limit and blocks node drains. + type: bool cloudnativepg_operator_image_name: description: PostgreSQL container image for the Cluster. type: str diff --git a/roles/cloudnativepg_operator/tasks/main.yml b/roles/cloudnativepg_operator/tasks/main.yml index 233b208..81ce369 100644 --- a/roles/cloudnativepg_operator/tasks/main.yml +++ b/roles/cloudnativepg_operator/tasks/main.yml @@ -166,6 +166,7 @@ {{ { 'instances': cloudnativepg_operator_instances, + 'enablePDB': cloudnativepg_operator_enable_pdb | bool, 'imageName': cloudnativepg_operator_image_name, 'bootstrap': { 'initdb': { diff --git a/roles/sno_alert_tuning/defaults/main.yml b/roles/sno_alert_tuning/defaults/main.yml new file mode 100644 index 0000000..9deec80 --- /dev/null +++ b/roles/sno_alert_tuning/defaults/main.yml @@ -0,0 +1,13 @@ +--- +# Platform alerts that are false positives on a single-control-plane cluster. +# Dropped via an AlertRelabelConfig (monitoring.openshift.io/v1), so they never +# reach Alertmanager. Alerts are matched by exact alertname. +# +# KubeCPUOvercommit / KubeMemoryOvercommit: the HA branch of these rules assumes +# the cluster survives losing its largest node. With one control-plane node plus +# smaller workers, that node is master-0 — losing it loses the cluster anyway. +sno_alert_tuning_dropped_alerts: + - KubeCPUOvercommit + - KubeMemoryOvercommit + +sno_alert_tuning_config_name: sno-single-control-plane diff --git a/roles/sno_alert_tuning/meta/argument_specs.yml b/roles/sno_alert_tuning/meta/argument_specs.yml new file mode 100644 index 0000000..6b13974 --- /dev/null +++ b/roles/sno_alert_tuning/meta/argument_specs.yml @@ -0,0 +1,17 @@ +--- +argument_specs: + main: + short_description: Drop platform alerts that are false positives on a single-control-plane cluster + description: + - Creates an AlertRelabelConfig in openshift-monitoring that drops the listed alerts + before they reach Alertmanager. An empty list removes the AlertRelabelConfig. + options: + sno_alert_tuning_dropped_alerts: + description: Exact alertnames to drop. + type: list + elements: str + default: [KubeCPUOvercommit, KubeMemoryOvercommit] + sno_alert_tuning_config_name: + description: Name of the AlertRelabelConfig. + type: str + default: sno-single-control-plane diff --git a/roles/sno_alert_tuning/meta/main.yml b/roles/sno_alert_tuning/meta/main.yml new file mode 100644 index 0000000..9704f7c --- /dev/null +++ b/roles/sno_alert_tuning/meta/main.yml @@ -0,0 +1,16 @@ +--- +galaxy_info: + author: ptoal + description: Drop platform alerts that are false positives on single-node OpenShift + license: MIT + min_ansible_version: "2.16" + platforms: + - name: GenericLinux + versions: + - all + galaxy_tags: + - openshift + - monitoring + - sno + +dependencies: [] diff --git a/roles/sno_alert_tuning/tasks/main.yml b/roles/sno_alert_tuning/tasks/main.yml new file mode 100644 index 0000000..78f5493 --- /dev/null +++ b/roles/sno_alert_tuning/tasks/main.yml @@ -0,0 +1,19 @@ +--- +# Drop platform alerts that only fire because the cluster has a single +# control-plane node. Uses the supported AlertRelabelConfig API rather than +# silences (which expire) or editing cluster-monitoring-operator rules (reverted). + +- name: Drop single-control-plane false-positive alerts + kubernetes.core.k8s: + state: "{{ 'present' if sno_alert_tuning_dropped_alerts | length > 0 else 'absent' }}" + definition: + apiVersion: monitoring.openshift.io/v1 + kind: AlertRelabelConfig + metadata: + name: "{{ sno_alert_tuning_config_name }}" + namespace: openshift-monitoring + spec: + configs: + - sourceLabels: [alertname] + regex: "{{ sno_alert_tuning_dropped_alerts | map('regex_escape') | join('|') }}" + action: Drop diff --git a/roles/sno_etcd_defrag/defaults/main.yml b/roles/sno_etcd_defrag/defaults/main.yml new file mode 100644 index 0000000..1033a74 --- /dev/null +++ b/roles/sno_etcd_defrag/defaults/main.yml @@ -0,0 +1,9 @@ +--- +# Defragment only when the in-use fraction of the etcd database drops below this +# ratio and the database exceeds the minimum size (mirrors the thresholds of the +# etcdDatabaseHighFragmentationRatio alert: < 50% in use and > 100 MiB). +sno_etcd_defrag_max_in_use_ratio: 0.5 +sno_etcd_defrag_min_db_size_bytes: 104857600 +sno_etcd_defrag_command_timeout: 60 +# Set true to defragment regardless of the thresholds. +sno_etcd_defrag_force: false diff --git a/roles/sno_etcd_defrag/meta/argument_specs.yml b/roles/sno_etcd_defrag/meta/argument_specs.yml new file mode 100644 index 0000000..e776e60 --- /dev/null +++ b/roles/sno_etcd_defrag/meta/argument_specs.yml @@ -0,0 +1,26 @@ +--- +argument_specs: + main: + short_description: Defragment etcd on a single-control-plane OpenShift cluster + description: + - cluster-etcd-operator disables automatic defragmentation on SingleReplica topology. + This role defragments the local member when fragmentation exceeds the thresholds of + the etcdDatabaseHighFragmentationRatio alert, then clears any NOSPACE alarm. + - Blocks etcd for the duration of the defrag. Requires pods/exec in openshift-etcd. + options: + sno_etcd_defrag_max_in_use_ratio: + description: Defragment when dbSizeInUse / dbSize is below this ratio. + type: float + default: 0.5 + sno_etcd_defrag_min_db_size_bytes: + description: Skip defragmentation when the database is smaller than this. + type: int + default: 104857600 + sno_etcd_defrag_command_timeout: + description: etcdctl --command-timeout for the defrag, in seconds. + type: int + default: 60 + sno_etcd_defrag_force: + description: Defragment regardless of the thresholds. + type: bool + default: false diff --git a/roles/sno_etcd_defrag/meta/main.yml b/roles/sno_etcd_defrag/meta/main.yml new file mode 100644 index 0000000..9e03016 --- /dev/null +++ b/roles/sno_etcd_defrag/meta/main.yml @@ -0,0 +1,16 @@ +--- +galaxy_info: + author: ptoal + description: Defragment etcd on single-node OpenShift (automatic defrag is disabled there) + license: MIT + min_ansible_version: "2.16" + platforms: + - name: GenericLinux + versions: + - all + galaxy_tags: + - openshift + - etcd + - sno + +dependencies: [] diff --git a/roles/sno_etcd_defrag/tasks/main.yml b/roles/sno_etcd_defrag/tasks/main.yml new file mode 100644 index 0000000..1fe12d9 --- /dev/null +++ b/roles/sno_etcd_defrag/tasks/main.yml @@ -0,0 +1,103 @@ +--- +# Defragment etcd on a single-control-plane cluster. +# +# cluster-etcd-operator disables its defrag controller on SingleReplica topology +# (Etcd/cluster condition DefragControllerDisabled=True), so fragmentation grows +# until etcdDatabaseHighFragmentationRatio fires. This role follows the documented +# manual procedure: defrag the local member, then clear any NOSPACE alarm. +# +# Defragmenting the only member blocks etcd reads/writes for the duration (seconds +# for a few hundred MiB) — schedule it outside busy periods. +# +# Requires pods/exec in openshift-etcd (cluster-admin kubeconfig). + +- name: Find the etcd pod + kubernetes.core.k8s_info: + api_version: v1 + kind: Pod + namespace: openshift-etcd + label_selectors: + - app=etcd + field_selectors: + - status.phase=Running + register: __sno_etcd_defrag_pods + failed_when: __sno_etcd_defrag_pods.resources | length != 1 + +- name: Set etcd pod name + ansible.builtin.set_fact: + __sno_etcd_defrag_pod: "{{ __sno_etcd_defrag_pods.resources[0].metadata.name }}" + +- name: Read etcd database size before defrag + kubernetes.core.k8s_exec: + namespace: openshift-etcd + pod: "{{ __sno_etcd_defrag_pod }}" + container: etcdctl + command: etcdctl endpoint status -w json + register: __sno_etcd_defrag_before + changed_when: false + +- name: Evaluate fragmentation + ansible.builtin.set_fact: + __sno_etcd_defrag_status_before: "{{ (__sno_etcd_defrag_before.stdout | from_json)[0].Status }}" + +- name: Decide whether to defragment + ansible.builtin.set_fact: + __sno_etcd_defrag_needed: >- + {{ sno_etcd_defrag_force | bool or ( + __sno_etcd_defrag_status_before.dbSize | int > sno_etcd_defrag_min_db_size_bytes | int and + (__sno_etcd_defrag_status_before.dbSizeInUse | float / __sno_etcd_defrag_status_before.dbSize | float) + < sno_etcd_defrag_max_in_use_ratio | float) }} + +- name: Defragment etcd + when: __sno_etcd_defrag_needed | bool + block: + - name: Defragment the local etcd member + kubernetes.core.k8s_exec: + namespace: openshift-etcd + pod: "{{ __sno_etcd_defrag_pod }}" + container: etcdctl + command: >- + sh -c 'unset ETCDCTL_ENDPOINTS; + etcdctl --command-timeout={{ sno_etcd_defrag_command_timeout }}s --endpoints=https://localhost:2379 defrag' + register: __sno_etcd_defrag_result + changed_when: true + failed_when: __sno_etcd_defrag_result.rc != 0 + + - name: List etcd alarms + kubernetes.core.k8s_exec: + namespace: openshift-etcd + pod: "{{ __sno_etcd_defrag_pod }}" + container: etcdctl + command: etcdctl alarm list + register: __sno_etcd_defrag_alarms + changed_when: false + retries: 6 + delay: 10 + until: __sno_etcd_defrag_alarms.rc | default(1) == 0 + + - name: Disarm NOSPACE alarm + kubernetes.core.k8s_exec: + namespace: openshift-etcd + pod: "{{ __sno_etcd_defrag_pod }}" + container: etcdctl + command: etcdctl alarm disarm + changed_when: true + when: "'NOSPACE' in __sno_etcd_defrag_alarms.stdout" + + - name: Read etcd database size after defrag + kubernetes.core.k8s_exec: + namespace: openshift-etcd + pod: "{{ __sno_etcd_defrag_pod }}" + container: etcdctl + command: etcdctl endpoint status -w json + register: __sno_etcd_defrag_after + changed_when: false + +- name: Display etcd defrag summary + ansible.builtin.debug: + msg: + - "etcd {{ 'defragmented' if __sno_etcd_defrag_needed | bool else 'not defragmented (below thresholds)' }}" + - " Before : {{ (__sno_etcd_defrag_status_before.dbSize / 1048576) | round(0) | int }} MiB allocated, + {{ (__sno_etcd_defrag_status_before.dbSizeInUse / 1048576) | round(0) | int }} MiB in use" + - " After : {{ ((__sno_etcd_defrag_after.stdout | from_json)[0].Status.dbSize / 1048576) | round(0) | int ~ ' MiB allocated' + if __sno_etcd_defrag_needed | bool else 'n/a' }}"