From 5b46ba7c74be57332467772e1dc3f237adaf34d4 Mon Sep 17 00:00:00 2001 From: Patrick Toal Date: Tue, 20 Oct 2026 14:08:57 -0400 Subject: [PATCH] Add OPA and Openshift AI --- OPA_POLICY_ENFORCEMENT.md | 317 +++++++++++++++ RHOAI_MAAS.md | 372 ++++++++++++++++++ playbooks/deploy_opa.yml | 49 +++ playbooks/deploy_openshift.yml | 91 ++++- playbooks/deploy_rhoai_maas.yml | 55 +++ roles/opa_server/defaults/main.yml | 58 +++ .../opa_server/files/policies/aap_policy.rego | 65 +++ .../opa_server/files/policies/aap_smoke.rego | 8 + roles/opa_server/files/policies/authz.rego | 11 + .../files/tests/aap_policy_test.rego | 87 ++++ roles/opa_server/meta/argument_specs.yml | 103 +++++ roles/opa_server/meta/main.yml | 17 + roles/opa_server/tasks/aap.yml | 95 +++++ roles/opa_server/tasks/deploy.yml | 172 ++++++++ roles/opa_server/tasks/main.yml | 32 ++ roles/opa_server/tasks/vault_token.yml | 28 ++ roles/opa_server/vars/main.yml | 9 + roles/rhoai_maas/defaults/main.yml | 111 ++++++ roles/rhoai_maas/meta/argument_specs.yml | 90 +++++ roles/rhoai_maas/meta/main.yml | 18 + roles/rhoai_maas/tasks/database.yml | 49 +++ roles/rhoai_maas/tasks/main.yml | 41 ++ roles/rhoai_maas/tasks/models.yml | 165 ++++++++ roles/rhoai_maas/tasks/operator.yml | 69 ++++ roles/rhoai_maas/tasks/platform.yml | 361 +++++++++++++++++ roles/rhoai_maas/tasks/rhoai.yml | 143 +++++++ roles/rhoai_maas/vars/main.yml | 19 + 27 files changed, 2634 insertions(+), 1 deletion(-) create mode 100644 OPA_POLICY_ENFORCEMENT.md create mode 100644 RHOAI_MAAS.md create mode 100644 playbooks/deploy_opa.yml create mode 100644 playbooks/deploy_rhoai_maas.yml create mode 100644 roles/opa_server/defaults/main.yml create mode 100644 roles/opa_server/files/policies/aap_policy.rego create mode 100644 roles/opa_server/files/policies/aap_smoke.rego create mode 100644 roles/opa_server/files/policies/authz.rego create mode 100644 roles/opa_server/files/tests/aap_policy_test.rego create mode 100644 roles/opa_server/meta/argument_specs.yml create mode 100644 roles/opa_server/meta/main.yml create mode 100644 roles/opa_server/tasks/aap.yml create mode 100644 roles/opa_server/tasks/deploy.yml create mode 100644 roles/opa_server/tasks/main.yml create mode 100644 roles/opa_server/tasks/vault_token.yml create mode 100644 roles/opa_server/vars/main.yml create mode 100644 roles/rhoai_maas/defaults/main.yml create mode 100644 roles/rhoai_maas/meta/argument_specs.yml create mode 100644 roles/rhoai_maas/meta/main.yml create mode 100644 roles/rhoai_maas/tasks/database.yml create mode 100644 roles/rhoai_maas/tasks/main.yml create mode 100644 roles/rhoai_maas/tasks/models.yml create mode 100644 roles/rhoai_maas/tasks/operator.yml create mode 100644 roles/rhoai_maas/tasks/platform.yml create mode 100644 roles/rhoai_maas/tasks/rhoai.yml create mode 100644 roles/rhoai_maas/vars/main.yml diff --git a/OPA_POLICY_ENFORCEMENT.md b/OPA_POLICY_ENFORCEMENT.md new file mode 100644 index 0000000..ac09b9e --- /dev/null +++ b/OPA_POLICY_ENFORCEMENT.md @@ -0,0 +1,317 @@ +# OPA Policy Enforcement for AAP + +This deploys an [Open Policy Agent](https://www.openpolicyagent.org/) (OPA) server on the +OpenShift cluster and connects Ansible Automation Platform 2.7 to it. Once connected, AAP +asks OPA whether each job may run before launching it. + +- Playbook: [playbooks/deploy_opa.yml](playbooks/deploy_opa.yml) +- Role: [roles/opa_server/](roles/opa_server/) +- Policies: [roles/opa_server/files/policies/](roles/opa_server/files/policies/) +- Policy tests: [roles/opa_server/files/tests/](roles/opa_server/files/tests/) + +## How it works + +``` + job launch ──► AAP controller ──HTTPS + bearer token──► OPA (opa.opa.svc:8443) + │ │ + │ POST /v1/data/ │ Rego policies + │ {"input": } │ + data.aap_policy_config + ◄──── {"allowed": bool, "violations": [...]} +``` + +1. A job launches for a job template. That template, its inventory, or its organization has an + **OPA query path** set (for example `aap/policy/decision`). +2. AAP sends the job's details to OPA once for each of those resources that has a query path. +3. OPA evaluates the Rego policy at that path and returns `allowed` plus a list of + `violations`. +4. If any policy denies the job, AAP blocks it and shows the violation messages on the job. + +**AAP fails closed.** Once a resource has a query path, the job is also blocked when: +- OPA can't be reached or times out; +- the path doesn't exist in OPA; +- the response isn't in the `{allowed, violations}` shape. + +Resources with no query path are never checked. + +### What gets deployed + +| Resource | Purpose | +|---|---| +| Namespace `opa` | Holds the OPA server | +| Deployment `opa` | OPA v1.21.1 (`docker.io/openpolicyagent/opa:1.21.1-static`), non-root, read-only filesystem | +| Service `opa` (port 8443) | HTTPS, using a certificate issued by OpenShift's internal service CA | +| ConfigMap `opa-policies` | Every `*.rego` file in the policy directory, plus `aap_policy_config.json` | +| Secret `opa-authz` | The bearer token, which the server's authorization policy checks | +| AAP setting changes | The `OPA_*` settings under **Settings → Policy as Code** | + +### Security model + +- **TLS:** OPA serves TLS with a certificate from OpenShift's internal service CA. AAP is given + that CA bundle so it can verify the server. +- **Authentication:** every request except `GET /health` needs the bearer token. This is + enforced by `opa run --authentication=token --authorization=basic` together with + [authz.rego](roles/opa_server/files/policies/authz.rego). +- **Token storage:** the token is generated on the first run and stored in Vault at + `kv/toallab/opa-server` (key `token`). Later runs reuse it. +- **Decision logs:** every decision is written to the OPA pod log for auditing, including the + job's extra vars. Turn this off with `opa_server_decision_logs: false`. + +## Prerequisites + +- AAP is deployed on the cluster ([deploy_aap.yml](playbooks/deploy_aap.yml)). +- The `aap` host has `aap_gateway_url` and `aap_k8s_api_url` set in the inventory. +- `vault_aap_deployer_token` resolves, and the `aap-deployer` ServiceAccount has the current + permissions. The playbook needs to create ConfigMaps, Services, and Deployments, which were + added for this. Re-run this after any change to those permissions: + + ```bash + ansible-navigator run playbooks/deploy_openshift.yml --tags sno_deploy_service_accounts + ``` + +## Deploy + +```bash +source ~/.venv/ansible/bin/activate +op run --env-file=$HOME/.ansible.zshenv -- \ + ansible-navigator run playbooks/deploy_opa.yml --mode stdout +``` + +The playbook: + +1. Reads the token from Vault, or generates and stores one. +2. Applies the namespace, policy ConfigMap, token Secret, Service, and Deployment. +3. Waits for the TLS certificate and for the pod to become available. +4. Updates AAP's `OPA_*` settings. +5. Attaches any query paths listed in `opa_server_aap_query_paths`. This list is empty by + default. + +Re-running is safe. Pods restart only when the policies, policy config, or token change. + +### Verify the deployment + +```bash +oc get pods -n opa +oc logs -n opa deploy/opa | head + +# Query OPA directly +oc port-forward -n opa svc/opa 8443:8443 & +TOKEN=$(vault kv get -field=token kv/toallab/opa-server) + +curl -sk https://localhost:8443/health # {} (no token needed) +curl -sk -o /dev/null -w '%{http_code}\n' https://localhost:8443/v1/data # 401 +curl -sk -H "Authorization: Bearer $TOKEN" \ + -d '{"input":{"job_type":"run","execution_environment":{"image":"quay.io/x/ee:latest"},"extra_vars":{}}}' \ + https://localhost:8443/v1/data/aap/policy/decision +``` + +In AAP, open **Settings → Policy as Code**. You should see: +- OPA host `opa.opa.svc` +- port `8443` +- SSL enabled +- auth type `Token` + +## Use it + +### Available query paths + +| Query path | Behavior | +|---|---| +| `aap/smoke/allow_all` | Always allows. Use it to attach safely and confirm AAP can reach OPA. | +| `aap/smoke/deny_all` | Always denies. Use it to confirm enforcement is active. | +| `aap/policy/decision` | The baseline policy described below. | + +### Baseline policy (`aap/policy/decision`) + +[aap_policy.rego](roles/opa_server/files/policies/aap_policy.rego) reports these violations: + +| Rule | Denies when | Setting in `opa_server_policy_config` | +|---|---|---| +| Approved registries | The execution environment image doesn't start with an allowed prefix | `allowed_ee_registries` (default: `aap.toal.ca/`, `registry.redhat.io/`) | +| No credentials in extra vars | A launch-time extra var matches a forbidden name (case-insensitive) | `forbidden_extra_vars` (default: `ansible_password`, `ansible_become_password`, `ansible_ssh_pass`, `ansible_become_pass`) | +| Change freeze | `change_freeze` is true and the job runs in `run` mode (not check mode), unless the launcher is a superuser or in an exempt team | `change_freeze`, `change_freeze_exempt_teams` | + +An empty `allowed_ee_registries` list turns the registry check off. + +Scheduled jobs have no launching user, so a change freeze blocks them too. + +The **Cloud Services Execution Environment** (`quay.io/scottharwell/cloud-ee`) fails the +registry rule as configured. That makes it a convenient demo. + +### Attach a policy + +Attach the smoke test first. Then switch to the real policy once AAP shows it reaching OPA. + +1. Confirm AAP can reach OPA. The job should run normally: + + ```bash + ansible-navigator run playbooks/deploy_opa.yml --mode stdout -e '{ + "opa_server_aap_query_paths": [ + {"type": "job_templates", "name": "Demo Job Template", "query_path": "aap/smoke/allow_all"} + ]}' + ``` + +2. Confirm enforcement. Switch the path to `aap/smoke/deny_all`, then launch the template. + The job should fail with + `aap.smoke.deny_all: policy enforcement is active`. + +3. Turn on the real policy by switching the path to `aap/policy/decision`. + +`type` is one of `organizations`, `inventories`, or `job_templates`. `name` must match +exactly one resource. + +To keep the attachments permanently, put `opa_server_aap_query_paths` in the `aap` host's +`host_vars` instead of passing `-e`. You can also set the path by hand on the resource's edit form in the AAP UI, or with the +API field `opa_query_path` on `/api/controller/v2/{organizations,inventories,job_templates}//`. + +### Detach a policy + +Set the query path to an empty string: + +```bash +-e '{"opa_server_aap_query_paths": [{"type": "job_templates", "name": "Demo Job Template", "query_path": ""}]}' +``` + +Removing an entry from `opa_server_aap_query_paths` does **not** detach it. The role only +sets the paths it's given. + +### Change the policy config + +Override `opa_server_policy_config` in `host_vars` or with `-e`, then re-run the playbook. +For example, to start a change freeze: + +```yaml +opa_server_policy_config: + allowed_ee_registries: [aap.toal.ca/, registry.redhat.io/] + forbidden_extra_vars: [ansible_password, ansible_become_password, ansible_ssh_pass, ansible_become_pass] + change_freeze: true + change_freeze_exempt_teams: [Platform] +``` + +Lists replace the defaults rather than merging with them, so give the whole object. + +## Write your own policies + +AAP needs the rule at the query path to return exactly this shape: + +```json +{"allowed": true, "violations": []} +``` + +`violations` is a list of strings shown to the user. It's ignored when `allowed` is true. + +### The input AAP sends + +`input` is the serialized job ([AWX policy.py](https://github.com/ansible/awx/blob/devel/awx/main/tasks/policy.py)): + +| Field | Contents | +|---|---| +| `id`, `name`, `created`, `job_type` (`run`/`check`), `launch_type`, `launched_by` | Job basics | +| `limit`, `forks`, `playbook`, `scm_branch`, `scm_revision`, `hosts_count` | Run details | +| `extra_vars` | Launch extra vars (survey passwords masked) | +| `created_by` | `{id, username, is_superuser, teams: [{id, name}]}`; `null` for scheduled jobs | +| `execution_environment` | `{id, name, image, pull}` | +| `inventory` | `{id, name, kind, total_hosts, has_active_failures, inventory_sources, ...}` | +| `job_template` | `{id, name, job_type}` | +| `organization` | `{id, name}` | +| `project` | `{id, name, scm_type, scm_url, scm_branch, ...}` | +| `credentials` | `[{id, name, credential_type, kind, cloud, kubernetes, organization, ...}]` | +| `labels` | `[{id, name, organization}]` | +| `instance_group` | `{id, name, capacity, max_forks, ...}` | +| `workflow_job`, `workflow_job_template` | `{id, name, ...}`, or `null` if not run from a workflow | + +### Add a policy + +1. Add a `.rego` file to [roles/opa_server/files/policies/](roles/opa_server/files/policies/). + Use `import rego.v1`; OPA 1.x requires Rego v1 syntax. The package determines the query + path: `package aap.network` maps to `aap/network/`. + + ```rego + package aap.network + + import rego.v1 + + decision := {"allowed": count(violations) == 0, "violations": [m | some m in violations]} + + violations contains "Production jobs require a --limit" if { + input.inventory.name == "Production" + input.limit == "" + } + ``` + +2. Add tests to [roles/opa_server/files/tests/](roles/opa_server/files/tests/) and run them: + + ```bash + podman run --rm -v ./roles/opa_server/files:/files:Z \ + docker.io/openpolicyagent/opa:1.21.1-static test /files/policies /files/tests -v + + podman run --rm -v ./roles/opa_server/files:/files:Z \ + docker.io/openpolicyagent/opa:1.21.1-static check --strict /files/policies + ``` + +3. Re-run the playbook. The ConfigMap is replaced in full, so deleted `.rego` files are + removed too, and the pod restarts to load the changes. + +Reference data you need in a policy through `data.aap_policy_config`. To read data, define a +defaulted rule such as `default user_config := {}` and assign `data.aap_policy_config` to it. +Don't use `object.get(data, ...)`: referencing all of `data` from inside a policy is treated +as recursion. + +## Configuration reference + +All variables are in [roles/opa_server/defaults/main.yml](roles/opa_server/defaults/main.yml). +[meta/argument_specs.yml](roles/opa_server/meta/argument_specs.yml) documents each one. + +| Variable | Default | Notes | +|---|---|---| +| `opa_server_namespace` | `opa` | | +| `opa_server_image` | `docker.io/openpolicyagent/opa:1.21.1-static` | Pin exact versions | +| `opa_server_replicas` | `1` | Increase for availability (AAP fails closed) | +| `opa_server_port` | `8443` | | +| `opa_server_decision_logs` | `true` | Logs inputs, including extra vars | +| `opa_server_policy_dir` | `files/policies` in the role | | +| `opa_server_policy_config` | see above | Exposed as `data.aap_policy_config` | +| `opa_server_vault_path` | `toallab/opa-server` | Mount `kv` | +| `opa_server_configure_aap` | `true` | `false` deploys OPA only | +| `opa_server_aap_request_timeout` / `_retries` | `1.5` / `2` | AAP's `OPA_REQUEST_TIMEOUT` / `OPA_REQUEST_RETRIES` | +| `opa_server_aap_query_paths` | `[]` | Attachments | + +## Operations + +### Rotate the token + +```bash +vault kv delete kv/toallab/opa-server +ansible-navigator run playbooks/deploy_opa.yml --mode stdout +``` + +The playbook generates a new token, updates the Secret (which restarts the pod), and updates +AAP's `OPA_AUTH_TOKEN`. + +> **Warning:** for a few seconds AAP and OPA hold different tokens. Jobs on resources with a +> policy attached fail during that window, so rotate outside busy periods. + +### Troubleshooting + +| Symptom on the AAP job | Cause | +|---|---| +| `Call to OPA failed ... 401` | AAP's token doesn't match the Secret. Re-run the playbook. If that doesn't fix it, delete the Vault key to force a new token. | +| `Call to OPA failed ... Connection` or a timeout | The OPA pod is down. Check `oc get pods -n opa`. | +| `did not return a "result" property` | The query path doesn't exist. Check the package and rule name, and use `/` or `.` as separators. | +| `OPA policy returned invalid result` | The rule doesn't return `{allowed: bool, violations: [string]}`. | +| A job runs when it should be blocked | No query path is attached to its organization, inventory, or template, or the rule returned `allowed: true`. | + +See exactly what OPA decided: + +```bash +oc logs -n opa deploy/opa | grep '"decision_id"' | tail -5 +``` + +### Remove + +Detach every query path first. Otherwise jobs on those resources fail once OPA is gone. + +```bash +# In AAP: Settings → Policy as Code → clear OPA host (an empty host disables evaluation) +oc delete namespace opa +vault kv metadata delete kv/toallab/opa-server +``` diff --git a/RHOAI_MAAS.md b/RHOAI_MAAS.md new file mode 100644 index 0000000..41a25a8 --- /dev/null +++ b/RHOAI_MAAS.md @@ -0,0 +1,372 @@ +# OpenShift AI Models-as-a-Service + +This deploys Red Hat OpenShift AI (RHOAI) 3.5 with **Models-as-a-Service (MaaS)** and +publishes the existing `llama-coder` server (llama.cpp running Qwen3.5-9B) through it. + +MaaS puts one authenticated, rate-limited, OpenAI-compatible endpoint in front of the +cluster's models: +- users get their own API keys; +- token quotas apply per user; +- access is granted by group. + +Nothing about llama-coder itself changes. Clients that call it directly keep working. + +- Playbook: [playbooks/deploy_rhoai_maas.yml](playbooks/deploy_rhoai_maas.yml) +- Role: [roles/rhoai_maas/](roles/rhoai_maas/) (database: [roles/cloudnativepg_operator/](roles/cloudnativepg_operator/)) +- Official docs: [Govern LLM access with Models-as-a-Service (RHOAI 3.5)](https://docs.redhat.com/en/documentation/red_hat_openshift_ai_self-managed/3.5/pdf/govern_llm_access_with_models-as-a-service/Red_Hat_OpenShift_AI_Self-Managed-3.5-Govern_LLM_access_with_Models-as-a-Service-en-US.pdf), + plus the [companion guide](https://github.com/jharmison-redhat/rhoai-maas-guide) + +> **Support status:** MaaS itself is GA in RHOAI 3.5. **External models**, which this uses +> to front llama.cpp instead of a KServe `LLMInferenceService`, are **Technology Preview**. + +## How it works + +``` + client ──HTTPS──► Route maas.apps.openshift.toal.ca (passthrough) + (sk-oai-… key) │ + ▼ + Gateway maas-default-gateway (openshift-ingress, Istio, ClusterIP) + │ Authorino: validate API key + MaaSAuthPolicy + │ Limitador: enforce MaaSSubscription token limits + ▼ + ExternalModel qwen35-9b-coder ──HTTP:8080 + llama-coder API key──► + Service llama-coder (llm-coding-assistant) ──► llama.cpp on the desktop GPU node +``` + +1. A user creates a MaaS API key (prefix `sk-oai-`) for a **subscription**. +2. The client sends OpenAI-format requests to `https://maas.apps.openshift.toal.ca/v1/...`, + using the `model` field to pick the model. +3. The gateway checks three things: + - the key is valid; + - a **MaaSAuthPolicy** grants the user's groups access to that model; + - the user's subscription still has **token quota** left. +4. MaaS forwards the request to llama-coder over in-cluster HTTP, adding llama-coder's own API + key. Users never see that key. + +### What gets deployed + +| Layer | Resources | +|---|---| +| Operators (OLM) | Red Hat Connectivity Link 1.4 (Kuadrant, Authorino, Limitador, DNS) in `openshift-operators`, Leader Worker Set in `openshift-lws-operator`, RHOAI `stable-3.5` in `redhat-ods-operator` | +| Monitoring | User Workload Monitoring turned on (`cluster-monitoring-config`). MaaS reports *Degraded* without it. | +| Gateway | GatewayClass `openshift-default` (makes the cluster install Service Mesh 3/Istio), Certificate `maas-default-gateway-cert` (Let's Encrypt, `maas.apps…` only), Gateway `maas-default-gateway` using it, Route `maas-default-gateway-https` (passthrough) | +| Auth and limits | `Kuadrant` and `Authorino` in `kuadrant-system`, with Authorino TLS using a certificate from OpenShift's internal service CA | +| Database | Single-instance CloudNativePG cluster `maas-postgres` in `rhoai-maas-db`, plus the `maas-db-config` secret in `redhat-ai-gateway-infra` | +| RHOAI | `DSCInitialization default-dsci` and `DataScienceCluster default-dsc` (minimal component set, below), with the dashboard's MaaS and external-models views turned on | +| Model | ExternalProvider `llama-coder-provider`, ExternalModel and MaaSModelRef `qwen35-9b-coder` in `llm-coding-assistant` | +| Access | MaaSAuthPolicy `llama-coder-access` and MaaSSubscription `llama-coder-default` in `models-as-a-service` | + +**DataScienceCluster components:** +- **Managed:** `dashboard`, `kserve`, `aigateway` (with `modelsAsAService`), and `ogx` (Gen AI Studio). +- **Removed:** everything else (pipelines, workbenches, model registry, Ray, TrustyAI, MLflow, + Feast, Kueue, trainer, training operator). + +### Why an external model, not KServe + +The cluster has one GPU: an RTX 4060 with 8 GB of VRAM and no sharing, already in use by +llama-coder. Serving through KServe/vLLM would mean shutting down llama.cpp and dropping to a +much smaller model. Using `ExternalModel` with `maas.opendatahub.io/tls: "false"` and +`maas.opendatahub.io/port: "8080"` lets MaaS govern the existing server in place. + +## Prerequisites + +- OpenShift 4.19.9 or later (the cluster runs 4.22) with cert-manager installed. +- The `letsencrypt-production` ClusterIssuer (DNS-01). The role issues a dedicated cert for + `maas.apps.openshift.toal.ca`. Don't put the `*.apps` wildcard on the gateway: browsers reuse an + HTTP/2 connection for any host its certificate covers, so after you visit MaaS, the console + and other `*.apps` sites would be sent to the MaaS gateway and get 404s. +- llama-coder is running in `llm-coding-assistant`, with secret `llama-coder-api-key` (key `api-key`). +- A `rhoai-deployer` ServiceAccount, provisioned with the permissions in + [deploy_openshift.yml](playbooks/deploy_openshift.yml). Re-run this after any change to those + permissions: + + ```bash + ansible-navigator run playbooks/deploy_openshift.yml --tags sno_deploy_service_accounts + ``` + +- Its token is referenced in the inventory at + `host_vars/sno.openshift.toal.ca/hashivault_secrets.yml`: + + ```yaml + vault_rhoai_deployer_token: "{{ lookup('community.hashi_vault.hashi_vault', 'kv/data/toallab/rhoai-deployer:token') }}" + ``` + +## Deploy + +```bash +source ~/.venv/ansible/bin/activate +op run --env-file=$HOME/.ansible.zshenv -- \ + ansible-navigator run playbooks/deploy_rhoai_maas.yml --mode stdout +``` + +A first run takes roughly 15–25 minutes, mostly waiting on operators. It runs these steps in +order: + +1. **CloudNativePG:** creates the `maas-postgres` cluster and its `maas` database. The password + is generated once and stored in Vault at `kv/cloudnativepg/maas-postgres`. +2. **Operators:** installs Connectivity Link, Leader Worker Set, and RHOAI, waiting for each + to finish installing. +3. **Platform:** + - turns on User Workload Monitoring; + - creates the GatewayClass, Gateway, and Route; + - creates Kuadrant and restarts its operator once Istio exists; + - turns on Authorino TLS. +4. **Database secret:** builds `maas-db-config` from the CloudNativePG credentials. +5. **RHOAI:** + - applies the DSCInitialization and DataScienceCluster; + - waits for the MaaS resource types to be registered and for `maas-api` to become available; + - turns on the dashboard's MaaS views. +6. **Models:** + - labels the model namespace and API-key secret; + - creates the ExternalProvider, ExternalModel, and MaaSModelRef; + - applies the auth policy and subscription. + +Each step is safe to re-run. If a run fails, fix the cause and run the playbook again; it +continues from where things stand. + +### Verify the deployment + +```bash +oc get datasciencecluster default-dsc -o jsonpath='{.status.phase}{"\n"}' # Ready +oc get kuadrant kuadrant -n kuadrant-system \ + -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}{"\n"}' # True +oc get gateway maas-default-gateway -n openshift-ingress # PROGRAMMED True +oc get deploy maas-api -n redhat-ai-gateway-infra # 1/1 +oc get externalprovider,externalmodel,maasmodelref -n llm-coding-assistant +oc get maasauthpolicy,maassubscription -n models-as-a-service +curl -s -o /dev/null -w '%{http_code}\n' https://maas.apps.openshift.toal.ca/maas-api/health +``` + +## Use it + +### 1. Get an API key + +You must belong to a group named in a MaaSSubscription. The default subscription covers every +authenticated user. + +**From the dashboard:** + +1. Open **OpenShift AI → Gen AI studio → API keys → Create API key**. +2. Pick the `llama-coder default` subscription and an expiry (the default is 90 days). +3. Copy the key. It's shown **only once**. + +**From the CLI**, using your OpenShift login token: + +```bash +MAAS=https://maas.apps.openshift.toal.ca +curl -s -X POST "$MAAS/maas-api/v1/api-keys" \ + -H "Authorization: Bearer $(oc whoami -t)" \ + -H "Content-Type: application/json" \ + -d '{"name": "laptop", "expiresIn": "720h", "subscription": "llama-coder-default"}' | jq -r .key +``` + +An API key records your group memberships **when it is created**. If your groups change, +revoke the key and create a new one. + +### 2. Call the model + +The endpoint is OpenAI-compatible. Use the model name `qwen35-9b-coder`. + +```bash +export MAAS_URL=https://maas.apps.openshift.toal.ca +export MAAS_API_KEY=sk-oai-... + +curl -s "$MAAS_URL/v1/models" -H "Authorization: Bearer $MAAS_API_KEY" + +curl -s "$MAAS_URL/v1/chat/completions" \ + -H "Authorization: Bearer $MAAS_API_KEY" -H "Content-Type: application/json" \ + -d '{"model": "qwen35-9b-coder", "messages": [{"role": "user", "content": "Write a haiku about Ansible"}]}' +``` + +Any OpenAI SDK or coding assistant works with these values: + +| Setting | Value | +|---|---| +| Base URL | `https://maas.apps.openshift.toal.ca/v1` | +| API key | your `sk-oai-…` key | +| Model | `qwen35-9b-coder` | + +```python +from openai import OpenAI + +client = OpenAI(base_url="https://maas.apps.openshift.toal.ca/v1", api_key="sk-oai-...") +print(client.chat.completions.create( + model="qwen35-9b-coder", + messages=[{"role": "user", "content": "Hello"}], +).choices[0].message.content) +``` + +### Limits + +- The default subscription allows each user **200,000 tokens per hour**. A user who exceeds + it gets HTTP 429 until the hour window resets. +- Token limits are counted only for OpenAI Chat Completions requests (`openai-chat`). +- llama.cpp handles one model with a 48k-token context and queues requests on a single GPU. + MaaS adds governance, not capacity. + +## Administer + +All access settings live in role variables. Change them in the inventory or with `-e`, then +re-run the playbook. The role creates and updates resources but never deletes them; see +[Remove a model or policy](#remove-a-model-or-policy). + +### Grant access to specific groups + +The OpenShift groups come from the Keycloak OIDC login. For example, to replace "all +authenticated users" with two tiers: + +```yaml +rhoai_maas_auth_policies: + - name: llama-coder-access + groups: [developers, platform-admins] + model_refs: + - {name: qwen35-9b-coder, namespace: llm-coding-assistant} + +rhoai_maas_subscriptions: + - name: llama-coder-standard + display_name: Standard + priority: 10 + groups: [developers] + model_refs: + - name: qwen35-9b-coder + namespace: llm-coding-assistant + tokenRateLimits: [{limit: 100000, window: 1h}] + - name: llama-coder-premium + display_name: Premium + priority: 20 # wins for users in both groups + groups: [platform-admins] + model_refs: + - name: qwen35-9b-coder + namespace: llm-coding-assistant + tokenRateLimits: [{limit: 1000000, window: 1h}] +``` + +- An **auth policy** decides *who may call* a model. +- A **subscription** decides *how many tokens* they get. + +A user needs both. `users:` (a list of usernames) works alongside `groups:` in both resources. + +### Add another model + +Any OpenAI-compatible endpoint can be added to `rhoai_maas_external_models`. That includes +another in-cluster server or a cloud provider. Then reference it from an auth policy and a +subscription. + +```yaml +rhoai_maas_external_models: + - name: qwen35-9b-coder + # ... existing entry ... + - name: gpt-4o-mini + namespace: llm-coding-assistant + display_name: GPT-4o mini (OpenAI) + provider_name: openai-provider + provider_type: openai # openai | anthropic | azure | vertex | bedrock + endpoint: api.openai.com # hostname only, no scheme + port: 443 + tls: true + api_key_secret: openai-api-key # create first: key "api-key" + target_model: gpt-4o-mini + api_format: openai-chat + path: /v1/chat/completions +``` + +Rules for model entries: +- **Names:** must be DNS-safe and must not collide with an existing Service in the namespace. + The controller creates a Service for each provider and model. That's why the provider is + called `llama-coder-provider` rather than `llama-coder`. +- **Secrets:** the API-key secret must exist in the model's namespace before you run the + playbook. The role only adds the `inference.llm-d.ai/ipp-managed` label to it. +- **Shared provider limits:** for a cloud provider, every user shares the provider's key, so + its rate limits apply to all MaaS users combined. + +### Revoke keys + +Use **Gen AI studio → API keys** in the dashboard. Administrators can revoke one key or all of +a user's keys. Revoke a user's keys after removing them from a group, because existing keys +keep the old memberships. + +### Remove a model or policy + +The role never deletes resources. Delete them by hand, then remove the entry from the +variables: + +```bash +oc delete maassubscription -n models-as-a-service +oc delete maasauthpolicy -n models-as-a-service +oc delete maasmodelref,externalmodel -n llm-coding-assistant +oc delete externalprovider -n llm-coding-assistant +``` + +## Configuration reference + +All variables are in [roles/rhoai_maas/defaults/main.yml](roles/rhoai_maas/defaults/main.yml). +[meta/argument_specs.yml](roles/rhoai_maas/meta/argument_specs.yml) documents each one. + +| Variable | Default | Notes | +|---|---|---| +| `rhoai_maas_operators` | RHCL `stable-1.4`, LWS `stable-v1.0`, RHOAI `stable-3.5` | RHOAI is pinned because the MaaS settings layout changed between 3.4 and 3.5 | +| `rhoai_maas_hostname` | `""` → `maas.` | | +| `rhoai_maas_gateway_cert_secret` | `maas-gateway-tls` | Single-host cert in `openshift-ingress`; never the `*.apps` wildcard | +| `rhoai_maas_gateway_cert_issuer` | `letsencrypt-production` | ClusterIssuer for that cert; `""` to bring your own secret | +| `rhoai_maas_gateway_service_type` | `ClusterIP` | Use `LoadBalancer` if MetalLB is added | +| `rhoai_maas_gateway_create_route` | `true` | Passthrough Route on the default router | +| `rhoai_maas_db_secret_name` / `_namespace` | `maas-postgres-secret` / `rhoai-maas-db` | Basic-auth secret from CloudNativePG | +| `rhoai_maas_db_sslmode` | `require` | | +| `rhoai_maas_dsc_managed_components` / `_removed_components` | see above | | +| `rhoai_maas_external_models` | llama-coder | | +| `rhoai_maas_auth_policies` | all authenticated users | | +| `rhoai_maas_subscriptions` | 200k tokens/h per user | | +| `rhoai_maas_wait_timeout` | `900` | Seconds, per wait | + +The database is sized in the playbook's `cloudnativepg_operator` variables: 1 instance, 5 Gi, +100 connections. + +## Operations + +### Troubleshooting + +| Symptom | Cause and fix | +|---|---| +| Run hangs at **Wait for MaaS CRDs**, and the DSC shows `ModelsAsAServiceReady=False … Removed` | MaaS isn't turned on. In 3.5 the field is `aigateway.modelsAsAService` (note "AsA"). The older `modelsAsService` spelling is silently dropped by the API server. Check with `oc get dsc default-dsc -o jsonpath='{.spec.components.aigateway}'`. | +| Run stops at **Wait for Kuadrant to be ready**, and Kuadrant shows `MissingDependency` | The Kuadrant operator started before Istio existed. The role restarts it automatically. To fix it by hand: `oc delete pod -n openshift-operators -l app=kuadrant,control-plane=controller-manager` | +| Gateway not `Programmed` | Check `oc describe gateway maas-default-gateway -n openshift-ingress`. `LoadBalancerPending` and `DNSReady=False` are expected with ClusterIP plus a Route and do no harm. | +| `maas-api` never appears | Check `oc get datasciencecluster default-dsc -o yaml` (status conditions) and that `maas-db-config` exists in `redhat-ai-gateway-infra`. | +| DSC shows *Degraded*, mentioning monitoring | User Workload Monitoring is off. Check `cluster-monitoring-config`. | +| `401` from `/v1/...` | The key is missing, expired, or revoked, or you used an OpenShift token instead of an `sk-oai-` key. | +| `403` | No MaaSAuthPolicy grants your groups that model. Check that `model_refs` names and namespaces match. | +| `429` | Subscription token limit reached. Wait for the window to reset, or raise `tokenRateLimits`. | +| `ExternalProvider` not `Ready` | Check the API-key secret exists with key `api-key`, and that the Service, ServiceEntry, and DestinationRule were created in the model namespace. | +| Long prompts fail or are cut off | Requests over 16 KB can be truncated on Service Mesh before 3.3.1. Check the installed version with `oc get csv -n openshift-operators \| grep servicemesh`. | +| Model timeouts under load | llama.cpp is serving on one GPU. Check `oc logs -n llm-coding-assistant deploy/llama-coder`. | + +### Usage metrics + +MaaS exports token and request metrics to User Workload Monitoring. Query them under +**Observe → Metrics** in the console, or use the dashboard's MaaS views. The full +observability stack (Tempo, OpenTelemetry, Loki) isn't deployed here. See phase 7 of the +[companion guide](https://github.com/jharmison-redhat/rhoai-maas-guide). + +### Upgrades + +RHOAI follows the `stable-3.5` channel with Automatic approval, so 3.5.z updates install on +their own. Moving to 3.6 or later means changing the channel in `rhoai_maas_operators`, +after checking the release notes. MaaS settings have changed between minor versions before: +3.4 configured MaaS under `kserve`, 3.5 moved it to `aigateway`. + +### Remove + +Removing MaaS deletes the database and every API key. Work in this order: + +```bash +oc delete maassubscription,maasauthpolicy --all -n models-as-a-service +oc delete maasmodelref,externalmodel,externalprovider --all -n llm-coding-assistant +oc delete datasciencecluster default-dsc +oc delete dscinitialization default-dsci +oc delete gateway maas-default-gateway -n openshift-ingress +oc delete route maas-default-gateway-https -n openshift-ingress +oc delete kuadrant kuadrant -n kuadrant-system +oc delete cluster.postgresql.cnpg.io maas-postgres -n rhoai-maas-db +# then uninstall the operators (Subscriptions + CSVs) from the console +``` + +Leave the `openshift-default` GatewayClass in place if anything else uses Gateway API. diff --git a/playbooks/deploy_opa.yml b/playbooks/deploy_opa.yml new file mode 100644 index 0000000..bf4c93f --- /dev/null +++ b/playbooks/deploy_opa.yml @@ -0,0 +1,49 @@ +--- +# Deploy an Open Policy Agent server on OpenShift and wire it into AAP's +# policy enforcement (policy as code). +# +# Authenticates via the aap-deployer ServiceAccount token used by deploy_aap.yml +# (same cluster). The token is stored in 1Password as vault_aap_deployer_token. +# +# Prerequisites: +# - AAP deployed (deploy_aap.yml) and aap_gateway_url set for the aap host +# - aap-deployer ServiceAccount RBAC current (re-run after RBAC changes): +# ansible-navigator run playbooks/deploy_openshift.yml --tags sno_deploy_service_accounts +# +# Policies live in roles/opa_server/files/policies; test them with: +# podman run --rm -v ./roles/opa_server/files:/files:Z docker.io/openpolicyagent/opa:1.21.1-static \ +# test /files/policies /files/tests -v +# +# Attaching a policy (AAP fails closed — start with aap/smoke/allow_all): +# -e '{"opa_server_aap_query_paths": [{"type": "job_templates", "name": "Demo Job Template", +# "query_path": "aap/policy/decision"}]}' +# +# Usage: +# ansible-navigator run playbooks/deploy_opa.yml --mode stdout + +- name: Deploy OPA server for AAP policy enforcement + hosts: aap + gather_facts: false + connection: local + + environment: + K8S_AUTH_HOST: "{{ aap_k8s_api_url }}" + K8S_AUTH_API_KEY: "{{ vault_aap_deployer_token }}" + + pre_tasks: + - name: Verify aap-deployer token and AAP URL are available + ansible.builtin.assert: + that: + - vault_aap_deployer_token is defined + - vault_aap_deployer_token | length > 0 + - aap_gateway_url is defined + fail_msg: >- + vault_aap_deployer_token or aap_gateway_url is not set. Provision the ServiceAccount with: + ansible-navigator run playbooks/deploy_openshift.yml --tags sno_deploy_service_accounts + + roles: + - role: opa_server + vars: + opa_server_aap_namespace: "{{ aap_operator_namespace | default('aap') }}" + opa_server_aap_admin_secret: "{{ aap_operator_platform_name | default('aap') }}-admin-password" + opa_server_aap_username: "{{ aap_operator_admin_user | default('admin') }}" diff --git a/playbooks/deploy_openshift.yml b/playbooks/deploy_openshift.yml index 475d661..e1d0809 100644 --- a/playbooks/deploy_openshift.yml +++ b/playbooks/deploy_openshift.yml @@ -400,10 +400,15 @@ - apiGroups: [""] resources: ["serviceaccounts"] verbs: ["get", "list", "watch"] + # OPA server (deploy_opa.yml): policy ConfigMap, Service, service CA bundle + - apiGroups: [""] + resources: ["configmaps", "services"] + verbs: ["get", "list", "watch", "create", "patch"] # patch: Automation Orchestrator backend env (APP_OIDC_ALLOW_PRIVATE_NETWORKS) + # create: OPA server Deployment - apiGroups: ["apps"] resources: ["deployments"] - verbs: ["get", "list", "watch", "patch"] + verbs: ["get", "list", "watch", "create", "patch"] - apiGroups: ["route.openshift.io"] resources: ["routes"] verbs: ["get", "list"] @@ -443,6 +448,90 @@ token: "{{ __ocp_service_account_token }}" no_log: true + # ---- rhoai-deployer (deploy_rhoai_maas.yml) ---- + - name: Provision rhoai-deployer ServiceAccount + ansible.builtin.include_role: + name: ocp_service_account + vars: + ocp_service_account_name: rhoai-deployer + ocp_service_account_namespace: redhat-ods-operator + ocp_service_account_cluster_role_rules: + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "list", "create", "patch"] + # configmaps: cluster-monitoring-config (UWM), gateway options + # services: Authorino serving-cert annotation + - apiGroups: [""] + resources: ["secrets", "configmaps", "services"] + verbs: ["get", "list", "watch", "create", "patch"] + - apiGroups: [""] + resources: ["serviceaccounts", "endpoints"] + verbs: ["get", "list", "watch"] + # delete: restart the Kuadrant operator after Istio is installed + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch", "delete"] + # patch: Authorino service-CA trust env + - apiGroups: ["apps"] + resources: ["deployments"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: ["route.openshift.io"] + resources: ["routes", "routes/custom-host"] + verbs: ["get", "list", "create", "patch"] + - apiGroups: ["config.openshift.io"] + resources: ["ingresses"] + verbs: ["get", "list"] + - apiGroups: ["operators.coreos.com"] + resources: ["operatorgroups", "subscriptions", "clusterserviceversions", + "catalogsources"] + verbs: ["get", "list", "create", "patch", "watch"] + - apiGroups: ["operators.coreos.com"] + resources: ["installplans"] + verbs: ["get", "list", "patch", "watch"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["get", "list", "watch"] + # CloudNativePG Cluster/Database CRs backing the MaaS database + - apiGroups: ["postgresql.cnpg.io"] + resources: ["clusters", "databases"] + verbs: ["get", "list", "create", "patch", "watch"] + - apiGroups: ["gateway.networking.k8s.io"] + resources: ["gatewayclasses", "gateways"] + verbs: ["get", "list", "create", "patch", "watch"] + # Dedicated MaaS gateway certificate (avoids HTTP/2 coalescing on the wildcard) + - apiGroups: ["cert-manager.io"] + resources: ["certificates"] + verbs: ["get", "list", "create", "patch", "watch"] + - apiGroups: ["kuadrant.io"] + resources: ["kuadrants"] + verbs: ["get", "list", "create", "patch", "watch"] + - apiGroups: ["operator.authorino.kuadrant.io"] + resources: ["authorinos"] + verbs: ["get", "list", "patch", "watch"] + - apiGroups: ["dscinitialization.opendatahub.io"] + resources: ["dscinitializations"] + verbs: ["get", "list", "create", "patch", "watch"] + - apiGroups: ["datasciencecluster.opendatahub.io"] + resources: ["datascienceclusters"] + verbs: ["get", "list", "create", "patch", "watch"] + - apiGroups: ["opendatahub.io"] + resources: ["odhdashboardconfigs"] + verbs: ["get", "list", "patch"] + - apiGroups: ["inference.opendatahub.io"] + resources: ["externalproviders", "externalmodels"] + verbs: ["get", "list", "create", "patch", "watch"] + - apiGroups: ["maas.opendatahub.io"] + resources: ["maasmodelrefs", "maasauthpolicies", "maassubscriptions"] + verbs: ["get", "list", "create", "patch", "watch"] + + - name: Store rhoai-deployer token in Vault + community.hashi_vault.vault_kv2_write: + path: toallab/rhoai-deployer + engine_mount_point: kv + data: + token: "{{ __ocp_service_account_token }}" + no_log: true + # --------------------------------------------------------------------------- # Play 10: Drop platform alerts that are false positives on a single # control-plane cluster (AlertRelabelConfig in openshift-monitoring) diff --git a/playbooks/deploy_rhoai_maas.yml b/playbooks/deploy_rhoai_maas.yml new file mode 100644 index 0000000..53081ea --- /dev/null +++ b/playbooks/deploy_rhoai_maas.yml @@ -0,0 +1,55 @@ +--- +# Deploy Red Hat OpenShift AI 3.5 Models-as-a-Service and publish the existing +# llama-coder (llama.cpp) server through it as an ExternalModel. +# +# WARNING: the MaaS database is a single-instance CloudNativePG cluster intended +# for demo/dev use — not production. +# +# Technology Preview: MaaS is GA in 3.5, but ExternalModel (used here to front +# llama.cpp instead of a KServe LLMInferenceService) is Technology Preview. +# +# Prerequisites: +# - OpenShift cluster deployed (deploy_openshift.yml) with cert-manager +# - rhoai-deployer ServiceAccount provisioned (re-run after RBAC changes): +# ansible-navigator run playbooks/deploy_openshift.yml --tags sno_deploy_service_accounts +# - vault_rhoai_deployer_token set in host_vars (see hashivault_secrets.yml) +# - llama-coder running in llm-coding-assistant with secret llama-coder-api-key +# +# Usage: +# ansible-navigator run playbooks/deploy_rhoai_maas.yml --mode stdout + +- name: Deploy OpenShift AI Models-as-a-Service + hosts: sno.openshift.toal.ca + gather_facts: false + connection: local + + environment: + K8S_AUTH_HOST: "{{ rhoai_maas_k8s_api_url | default('https://api.' ~ ocp_cluster_name ~ '.' ~ ocp_base_domain ~ ':6443') }}" + K8S_AUTH_API_KEY: "{{ vault_rhoai_deployer_token }}" + + pre_tasks: + - name: Verify rhoai-deployer token is available + ansible.builtin.assert: + that: + - vault_rhoai_deployer_token is defined + - vault_rhoai_deployer_token | length > 0 + fail_msg: >- + vault_rhoai_deployer_token is not set. Provision the ServiceAccount with: + ansible-navigator run playbooks/deploy_openshift.yml --tags sno_deploy_service_accounts + Then reference kv/data/toallab/rhoai-deployer:token as vault_rhoai_deployer_token. + + roles: + - role: cloudnativepg_operator + vars: + cloudnativepg_operator_cluster_namespace: rhoai-maas-db + cloudnativepg_operator_cluster_name: maas-postgres + cloudnativepg_operator_owner_user: maas + cloudnativepg_operator_max_connections: 100 + cloudnativepg_operator_databases: + - name: maas + dbname: maas + cloudnativepg_operator_secrets: + - name: maas-postgres-secret + database: maas + cloudnativepg_operator_vault_path: cloudnativepg/maas-postgres + - role: rhoai_maas diff --git a/roles/opa_server/defaults/main.yml b/roles/opa_server/defaults/main.yml new file mode 100644 index 0000000..65e6a6d --- /dev/null +++ b/roles/opa_server/defaults/main.yml @@ -0,0 +1,58 @@ +--- +# --- Deployment --- +opa_server_namespace: opa +opa_server_name: opa +opa_server_image: docker.io/openpolicyagent/opa:1.21.1-static +opa_server_replicas: 1 +opa_server_port: 8443 +opa_server_log_level: info +# Log every decision (including the job's extra_vars) to the pod log for auditing. +opa_server_decision_logs: true +opa_server_resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 500m + memory: 256Mi +opa_server_wait_timeout: 300 + +# --- Policies --- +# Every *.rego file in this directory is loaded into the server. +opa_server_policy_dir: "{{ role_path }}/files/policies" +# Exposed to policies as data.aap_policy_config (see files/policies/aap_policy.rego). +opa_server_policy_config: + allowed_ee_registries: + - aap.toal.ca/ + - registry.redhat.io/ + forbidden_extra_vars: + - ansible_password + - ansible_become_password + - ansible_ssh_pass + - ansible_become_pass + change_freeze: false + change_freeze_exempt_teams: [] + +# --- Bearer token (OPA --authentication=token) --- +# Generated on first run and stored in Vault; reused afterwards. +opa_server_vault_mount: kv +opa_server_vault_path: toallab/opa-server + +# --- AAP integration --- +opa_server_configure_aap: true +opa_server_aap_url: "{{ aap_gateway_url }}" +opa_server_aap_username: admin +opa_server_aap_namespace: aap +opa_server_aap_admin_secret: aap-admin-password +opa_server_aap_validate_certs: true +opa_server_aap_request_timeout: 1.5 +opa_server_aap_request_retries: 2 +# Attach a policy to AAP resources. Jobs fail closed: any OPA error or violation +# blocks the launch, so start with aap/smoke/allow_all and a test resource. +# - type: organizations | inventories | job_templates +# name: Default +# query_path: aap/policy/decision +opa_server_aap_query_paths: [] + +# --- Debugging --- +opa_server_no_log: true diff --git a/roles/opa_server/files/policies/aap_policy.rego b/roles/opa_server/files/policies/aap_policy.rego new file mode 100644 index 0000000..a79bcdb --- /dev/null +++ b/roles/opa_server/files/policies/aap_policy.rego @@ -0,0 +1,65 @@ +# AAP policy-as-code baseline. +# Set opa_query_path to aap/policy/decision on an organization, inventory, or +# job template. Input is the serialized Job (awx/main/tasks/policy.py). +# Tunables come from data.aap_policy_config (opa_server_policy_config). +package aap.policy + +import rego.v1 + +decision := { + "allowed": count(violations) == 0, + "violations": sort([msg | some msg in violations]), +} + +default_config := { + "allowed_ee_registries": [], + "forbidden_extra_vars": [], + "change_freeze": false, + "change_freeze_exempt_teams": [], +} + +default user_config := {} + +user_config := data.aap_policy_config + +config := object.union(default_config, user_config) + +# Execution environment must come from an approved registry +violations contains msg if { + count(config.allowed_ee_registries) > 0 + image := input.execution_environment.image + not approved_registry(image) + msg := sprintf( + "Execution environment image %q is not from an approved registry (%s)", + [image, concat(", ", config.allowed_ee_registries)], + ) +} + +approved_registry(image) if { + some prefix in config.allowed_ee_registries + startswith(image, prefix) +} + +# Credentials belong in AAP credentials, not launch-time extra vars +violations contains msg if { + extra_vars := object.get(input, "extra_vars", {}) + is_object(extra_vars) + some key, _ in extra_vars + lower(key) in {lower(k) | some k in config.forbidden_extra_vars} + msg := sprintf("Extra variable %q may not be passed at launch; use a credential", [key]) +} + +# Change freeze: only check-mode jobs unless the launcher is exempt +violations contains msg if { + config.change_freeze + input.job_type == "run" + not freeze_exempt + msg := "A change freeze is in effect: only check-mode jobs are allowed" +} + +freeze_exempt if input.created_by.is_superuser + +freeze_exempt if { + some team in input.created_by.teams + team.name in config.change_freeze_exempt_teams +} diff --git a/roles/opa_server/files/policies/aap_smoke.rego b/roles/opa_server/files/policies/aap_smoke.rego new file mode 100644 index 0000000..3026163 --- /dev/null +++ b/roles/opa_server/files/policies/aap_smoke.rego @@ -0,0 +1,8 @@ +# Wiring checks for AAP policy enforcement. +# aap/smoke/allow_all - always passes (safe first attachment) +# aap/smoke/deny_all - always blocks, proving enforcement is active +package aap.smoke + +allow_all := {"allowed": true, "violations": []} + +deny_all := {"allowed": false, "violations": ["aap.smoke.deny_all: policy enforcement is active"]} diff --git a/roles/opa_server/files/policies/authz.rego b/roles/opa_server/files/policies/authz.rego new file mode 100644 index 0000000..b49e6af --- /dev/null +++ b/roles/opa_server/files/policies/authz.rego @@ -0,0 +1,11 @@ +# Server authorization (opa run --authorization=basic). +# Health checks are open; everything else needs the bearer token. +package system.authz + +import rego.v1 + +default allow := false + +allow if input.path == ["health"] + +allow if input.identity == data.opa_authz.token diff --git a/roles/opa_server/files/tests/aap_policy_test.rego b/roles/opa_server/files/tests/aap_policy_test.rego new file mode 100644 index 0000000..c18a4d6 --- /dev/null +++ b/roles/opa_server/files/tests/aap_policy_test.rego @@ -0,0 +1,87 @@ +# Unit tests (not deployed). Run with: +# podman run --rm -v ./roles/opa_server/files:/files:Z docker.io/openpolicyagent/opa:1.21.1-static \ +# test /files/policies /files/tests -v +package aap.policy_test + +import rego.v1 + +import data.aap.policy + +cfg := { + "allowed_ee_registries": ["aap.toal.ca/", "registry.redhat.io/"], + "forbidden_extra_vars": ["ansible_password"], + "change_freeze": false, + "change_freeze_exempt_teams": ["Platform"], +} + +job := { + "job_type": "run", + "execution_environment": {"image": "aap.toal.ca/toallab/ee-demo:latest"}, + "extra_vars": {"target": "web"}, + "created_by": {"username": "dev", "is_superuser": false, "teams": [{"id": 1, "name": "Dev"}]}, +} + +test_compliant_job_allowed if { + d := policy.decision with input as job with data.aap_policy_config as cfg + d.allowed + d.violations == [] +} + +test_unapproved_registry_denied if { + j := object.union(job, {"execution_environment": {"image": "quay.io/scottharwell/cloud-ee:latest"}}) + d := policy.decision with input as j with data.aap_policy_config as cfg + not d.allowed + count(d.violations) == 1 +} + +test_forbidden_extra_var_denied_case_insensitive if { + j := object.union(job, {"extra_vars": {"Ansible_Password": "x"}}) + d := policy.decision with input as j with data.aap_policy_config as cfg + not d.allowed +} + +test_null_extra_vars_and_ee_allowed if { + j := object.union(job, {"extra_vars": null, "execution_environment": null}) + d := policy.decision with input as j with data.aap_policy_config as cfg + d.allowed +} + +test_freeze_blocks_run if { + d := policy.decision with input as job with data.aap_policy_config as object.union(cfg, {"change_freeze": true}) + not d.allowed +} + +test_freeze_allows_check if { + j := object.union(job, {"job_type": "check"}) + d := policy.decision with input as j with data.aap_policy_config as object.union(cfg, {"change_freeze": true}) + d.allowed +} + +test_freeze_exempt_team if { + j := object.union(job, {"created_by": {"is_superuser": false, "teams": [{"name": "Platform"}]}}) + d := policy.decision with input as j with data.aap_policy_config as object.union(cfg, {"change_freeze": true}) + d.allowed +} + +test_freeze_scheduled_job_without_creator if { + j := object.union(job, {"created_by": null}) + d := policy.decision with input as j with data.aap_policy_config as object.union(cfg, {"change_freeze": true}) + not d.allowed +} + +test_no_config_allows_everything if { + j := object.union(job, {"execution_environment": {"image": "quay.io/anything:latest"}}) + d := policy.decision with input as j + d.allowed +} + +test_authz_health_open if { + data.system.authz.allow with input as {"path": ["health"]} +} + +test_authz_requires_token if { + not data.system.authz.allow with input as {"path": ["v1", "data"], "identity": "bad"} + with data.opa_authz.token as "good" + data.system.authz.allow with input as {"path": ["v1", "data"], "identity": "good"} + with data.opa_authz.token as "good" +} diff --git a/roles/opa_server/meta/argument_specs.yml b/roles/opa_server/meta/argument_specs.yml new file mode 100644 index 0000000..d37374e --- /dev/null +++ b/roles/opa_server/meta/argument_specs.yml @@ -0,0 +1,103 @@ +--- +argument_specs: + main: + short_description: Deploy an OPA server for AAP policy enforcement + description: + - Deploys Open Policy Agent with TLS (service-ca) and bearer-token authentication, loading + the Rego policies in opa_server_policy_dir and opa_server_policy_config as + data.aap_policy_config. + - Configures AAP's PolicyAsCode settings to use it and optionally attaches opa_query_path + to organizations, inventories, or job templates. AAP fails closed once a path is attached. + - The bearer token is generated on first run and stored in Vault at opa_server_vault_path. + options: + opa_server_namespace: + description: Namespace for the OPA server. + type: str + default: opa + opa_server_name: + description: Name for the Deployment, Service, and derived resources. + type: str + default: opa + opa_server_image: + description: OPA container image (pin an exact version). + type: str + default: docker.io/openpolicyagent/opa:1.21.1-static + opa_server_replicas: + description: Number of OPA replicas. + type: int + default: 1 + opa_server_port: + description: HTTPS port OPA listens on and the Service exposes. + type: int + default: 8443 + opa_server_log_level: + description: OPA log level. + type: str + choices: [debug, info, warn, error] + default: info + opa_server_decision_logs: + description: Log every decision (including job extra_vars) to the pod log. + type: bool + default: true + opa_server_resources: + description: Container resource requests and limits. + type: dict + opa_server_wait_timeout: + description: Seconds to wait for the OPA Deployment to become available. + type: int + default: 300 + opa_server_policy_dir: + description: Directory whose *.rego files are loaded into OPA. + type: path + opa_server_policy_config: + description: Policy tunables exposed as data.aap_policy_config. + type: dict + opa_server_vault_mount: + description: Vault KV2 mount for the bearer token. + type: str + default: kv + opa_server_vault_path: + description: Vault KV2 path for the bearer token. + type: str + default: toallab/opa-server + opa_server_configure_aap: + description: Configure AAP's PolicyAsCode settings to use this server. + type: bool + default: true + opa_server_aap_url: + description: AAP gateway URL. + type: str + opa_server_aap_username: + description: AAP admin username. + type: str + default: admin + opa_server_aap_namespace: + description: Namespace of the AAP instance (holds the admin password secret). + type: str + default: aap + opa_server_aap_admin_secret: + description: Secret holding the AAP admin password (key password). + type: str + default: aap-admin-password + opa_server_aap_validate_certs: + description: Validate the AAP gateway certificate. + type: bool + default: true + opa_server_aap_request_timeout: + description: OPA_REQUEST_TIMEOUT in seconds. + type: float + default: 1.5 + opa_server_aap_request_retries: + description: OPA_REQUEST_RETRIES. + type: int + default: 2 + opa_server_aap_query_paths: + description: Resources to attach a policy to (type organizations|inventories|job_templates, + name, query_path). + type: list + elements: dict + default: [] + opa_server_no_log: + description: Set to false to expose secret values in task output. + type: bool + default: true diff --git a/roles/opa_server/meta/main.yml b/roles/opa_server/meta/main.yml new file mode 100644 index 0000000..3fcde29 --- /dev/null +++ b/roles/opa_server/meta/main.yml @@ -0,0 +1,17 @@ +--- +galaxy_info: + author: ptoal + description: Deploy an Open Policy Agent server on OpenShift for AAP policy enforcement + license: MIT + min_ansible_version: "2.16" + platforms: + - name: GenericLinux + versions: + - all + galaxy_tags: + - openshift + - opa + - policy + - aap + +dependencies: [] diff --git a/roles/opa_server/tasks/aap.yml b/roles/opa_server/tasks/aap.yml new file mode 100644 index 0000000..472b3f3 --- /dev/null +++ b/roles/opa_server/tasks/aap.yml @@ -0,0 +1,95 @@ +--- +# Point the AAP controller at OPA (PolicyAsCode settings) and optionally attach +# query paths to organizations, inventories, or job templates. + +- name: Read AAP admin password + kubernetes.core.k8s_info: + api_version: v1 + kind: Secret + namespace: "{{ opa_server_aap_namespace }}" + name: "{{ opa_server_aap_admin_secret }}" + register: __opa_server_aap_admin + no_log: true + +- name: Read the service CA bundle + kubernetes.core.k8s_info: + api_version: v1 + kind: ConfigMap + namespace: "{{ opa_server_namespace }}" + name: openshift-service-ca.crt + register: __opa_server_service_ca + +- name: Resolve AAP API connection + ansible.builtin.set_fact: + __opa_server_aap_api: "{{ opa_server_aap_url | regex_replace('/$', '') }}/api/controller/v2" + __opa_server_aap_password: "{{ __opa_server_aap_admin.resources[0].data.password | b64decode }}" + __opa_server_aap_settings: + OPA_HOST: "{{ __opa_server_host }}" + OPA_PORT: "{{ opa_server_port | int }}" + OPA_SSL: true + OPA_AUTH_TYPE: Token + OPA_AUTH_CA_CERT: "{{ __opa_server_service_ca.resources[0].data['service-ca.crt'] }}" + OPA_REQUEST_TIMEOUT: "{{ opa_server_aap_request_timeout | float }}" + OPA_REQUEST_RETRIES: "{{ opa_server_aap_request_retries | int }}" + no_log: true + +- name: Read AAP policy-as-code settings + ansible.builtin.uri: + url: "{{ __opa_server_aap_api }}/settings/policyascode/" + user: "{{ opa_server_aap_username }}" + password: "{{ __opa_server_aap_password }}" + force_basic_auth: true + validate_certs: "{{ opa_server_aap_validate_certs }}" + register: __opa_server_aap_current + no_log: "{{ opa_server_no_log }}" + +# The token reads back as $encrypted$, so it is only resent when unset or rotated +- name: Configure AAP policy-as-code settings + ansible.builtin.uri: + url: "{{ __opa_server_aap_api }}/settings/policyascode/" + method: PATCH + user: "{{ opa_server_aap_username }}" + password: "{{ __opa_server_aap_password }}" + force_basic_auth: true + validate_certs: "{{ opa_server_aap_validate_certs }}" + body_format: json + body: "{{ __opa_server_aap_settings | combine({'OPA_AUTH_TOKEN': __opa_server_token}) }}" + changed_when: true + when: >- + (__opa_server_aap_current.json | dict2items + | selectattr('key', 'in', __opa_server_aap_settings.keys() | list) + | items2dict) != __opa_server_aap_settings + or (__opa_server_aap_current.json.OPA_AUTH_TOKEN | default('')) | length == 0 + or __opa_server_token_generated | bool + no_log: "{{ opa_server_no_log }}" + +- name: Look up resources to attach policies to + ansible.builtin.uri: + url: "{{ __opa_server_aap_api }}/{{ item.type }}/?name={{ item.name | urlencode }}" + user: "{{ opa_server_aap_username }}" + password: "{{ __opa_server_aap_password }}" + force_basic_auth: true + validate_certs: "{{ opa_server_aap_validate_certs }}" + register: __opa_server_aap_targets + failed_when: >- + __opa_server_aap_targets.status != 200 or __opa_server_aap_targets.json.count != 1 + loop: "{{ opa_server_aap_query_paths }}" + loop_control: + label: "{{ item.type }}/{{ item.name }}" + +- name: Attach OPA query paths + ansible.builtin.uri: + url: "{{ __opa_server_aap_api }}/{{ item.item.type }}/{{ item.json.results[0].id }}/" + method: PATCH + user: "{{ opa_server_aap_username }}" + password: "{{ __opa_server_aap_password }}" + force_basic_auth: true + validate_certs: "{{ opa_server_aap_validate_certs }}" + body_format: json + body: + opa_query_path: "{{ item.item.query_path }}" + changed_when: true + when: item.json.results[0].opa_query_path | default('') != item.item.query_path + loop: "{{ __opa_server_aap_targets.results }}" + loop_control: + label: "{{ item.item.type }}/{{ item.item.name }} -> {{ item.item.query_path }}" diff --git a/roles/opa_server/tasks/deploy.yml b/roles/opa_server/tasks/deploy.yml new file mode 100644 index 0000000..adea946 --- /dev/null +++ b/roles/opa_server/tasks/deploy.yml @@ -0,0 +1,172 @@ +--- +- name: Create OPA namespace + kubernetes.core.k8s: + state: present + definition: + apiVersion: v1 + kind: Namespace + metadata: + name: "{{ opa_server_namespace }}" + +- name: Create OPA policy ConfigMap + kubernetes.core.k8s: + state: present + apply: true + definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: "{{ opa_server_name }}-policies" + namespace: "{{ opa_server_namespace }}" + data: >- + {{ __opa_server_policies + | combine({'aap_policy_config.json': {'aap_policy_config': opa_server_policy_config} | to_json}) }} + +- name: Create OPA authorization token Secret + kubernetes.core.k8s: + state: present + definition: + apiVersion: v1 + kind: Secret + metadata: + name: "{{ opa_server_name }}-authz" + namespace: "{{ opa_server_namespace }}" + type: Opaque + stringData: + opa_authz.json: "{{ {'opa_authz': {'token': __opa_server_token}} | to_json }}" + no_log: "{{ opa_server_no_log }}" + +- name: Create OPA Service with a service-ca serving certificate + kubernetes.core.k8s: + state: present + definition: + apiVersion: v1 + kind: Service + metadata: + name: "{{ opa_server_name }}" + namespace: "{{ opa_server_namespace }}" + annotations: + service.beta.openshift.io/serving-cert-secret-name: "{{ __opa_server_tls_secret }}" + spec: + selector: + app: "{{ opa_server_name }}" + ports: + - name: https + port: "{{ opa_server_port | int }}" + targetPort: https + +- name: Wait for OPA serving certificate + kubernetes.core.k8s_info: + api_version: v1 + kind: Secret + namespace: "{{ opa_server_namespace }}" + name: "{{ __opa_server_tls_secret }}" + register: __opa_server_cert + until: __opa_server_cert.resources | length > 0 + retries: 30 + delay: 10 + no_log: true + +- name: Create OPA Deployment + kubernetes.core.k8s: + state: present + definition: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: "{{ opa_server_name }}" + namespace: "{{ opa_server_namespace }}" + labels: + app: "{{ opa_server_name }}" + spec: + replicas: "{{ opa_server_replicas | int }}" + selector: + matchLabels: + app: "{{ opa_server_name }}" + template: + metadata: + labels: + app: "{{ opa_server_name }}" + annotations: + # Roll pods when policies, config, or the token change + checksum/policies: "{{ (__opa_server_policies | to_json ~ opa_server_policy_config | to_json) | hash('sha256') }}" + checksum/authz: "{{ __opa_server_token | hash('sha256') }}" + spec: + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - name: opa + image: "{{ opa_server_image }}" + args: >- + {{ [ + 'run', '--server', + '--addr=0.0.0.0:' ~ opa_server_port, + '--tls-cert-file=/certs/tls.crt', + '--tls-private-key-file=/certs/tls.key', + '--authentication=token', + '--authorization=basic', + '--ignore=.*', + '--log-format=json', + '--log-level=' ~ opa_server_log_level, + ] + + (['--set=decision_logs.console=true'] if opa_server_decision_logs | bool else []) + + ['/policies', '/authz'] }} + ports: + - name: https + containerPort: "{{ opa_server_port | int }}" + readinessProbe: + httpGet: + path: /health + port: https + scheme: HTTPS + initialDelaySeconds: 3 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /health + port: https + scheme: HTTPS + initialDelaySeconds: 10 + periodSeconds: 20 + resources: "{{ opa_server_resources }}" + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: [ALL] + volumeMounts: + - name: policies + mountPath: /policies + readOnly: true + - name: authz + mountPath: /authz + readOnly: true + - name: certs + mountPath: /certs + readOnly: true + volumes: + - name: policies + configMap: + name: "{{ opa_server_name }}-policies" + - name: authz + secret: + secretName: "{{ opa_server_name }}-authz" + - name: certs + secret: + secretName: "{{ __opa_server_tls_secret }}" + +- name: Wait for OPA to be available + kubernetes.core.k8s_info: + api_version: apps/v1 + kind: Deployment + namespace: "{{ opa_server_namespace }}" + name: "{{ opa_server_name }}" + register: __opa_server_deploy + until: >- + __opa_server_deploy.resources | length > 0 and + (__opa_server_deploy.resources[0].status.updatedReplicas | default(0)) == opa_server_replicas | int and + (__opa_server_deploy.resources[0].status.availableReplicas | default(0)) == opa_server_replicas | int + retries: "{{ __opa_server_wait_retries }}" + delay: 10 diff --git a/roles/opa_server/tasks/main.yml b/roles/opa_server/tasks/main.yml new file mode 100644 index 0000000..b6d3296 --- /dev/null +++ b/roles/opa_server/tasks/main.yml @@ -0,0 +1,32 @@ +--- +# Deploy an Open Policy Agent server for AAP policy enforcement (policy as code) +# and point AAP's controller at it. +# +# OPA serves TLS with a service-ca certificate and requires a bearer token +# (--authentication=token, system.authz policy). AAP connects in-cluster with +# OPA_SSL=true, OPA_AUTH_TYPE=Token and the service CA bundle. +# +# AAP fails closed: once a resource has an opa_query_path, any OPA error or +# violation blocks its jobs. Nothing is attached unless opa_server_aap_query_paths +# lists it. +# +# All tasks are idempotent (kubernetes.core.k8s state: present). + +- name: Resolve OPA bearer token from Vault + ansible.builtin.include_tasks: vault_token.yml + +- name: Deploy OPA server + ansible.builtin.include_tasks: deploy.yml + +- name: Configure AAP policy enforcement + ansible.builtin.include_tasks: aap.yml + when: opa_server_configure_aap | bool + +- name: Display OPA deployment summary + ansible.builtin.debug: + msg: + - "OPA server ready!" + - " Endpoint : https://{{ __opa_server_host }}:{{ opa_server_port }} (in-cluster)" + - " Policies : {{ __opa_server_policies.keys() | list | join(', ') }}" + - " Query paths : aap/policy/decision, aap/smoke/allow_all, aap/smoke/deny_all" + - " Token : Vault {{ opa_server_vault_mount }}/{{ opa_server_vault_path }}" diff --git a/roles/opa_server/tasks/vault_token.yml b/roles/opa_server/tasks/vault_token.yml new file mode 100644 index 0000000..815c3cf --- /dev/null +++ b/roles/opa_server/tasks/vault_token.yml @@ -0,0 +1,28 @@ +--- +# Read the OPA bearer token from Vault; generate and store it if missing. +# Sets __opa_server_token and __opa_server_token_generated. + +- name: Read OPA token from Vault + community.hashi_vault.vault_kv2_get: + path: "{{ opa_server_vault_path }}" + engine_mount_point: "{{ opa_server_vault_mount }}" + register: __opa_server_vault_read + failed_when: false + no_log: "{{ opa_server_no_log }}" + +- name: Resolve OPA token (generate if missing) + ansible.builtin.set_fact: + __opa_server_token_generated: "{{ ((__opa_server_vault_read.secret | default({})).token | default('')) | length == 0 }}" + __opa_server_token: >- + {{ (__opa_server_vault_read.secret | default({})).token + | default(lookup('community.general.random_string', length=48, special=false)) }} + no_log: "{{ opa_server_no_log }}" + +- name: Write OPA token to Vault + community.hashi_vault.vault_kv2_write: + path: "{{ opa_server_vault_path }}" + engine_mount_point: "{{ opa_server_vault_mount }}" + data: + token: "{{ __opa_server_token }}" + when: __opa_server_token_generated | bool + no_log: "{{ opa_server_no_log }}" diff --git a/roles/opa_server/vars/main.yml b/roles/opa_server/vars/main.yml new file mode 100644 index 0000000..0c45302 --- /dev/null +++ b/roles/opa_server/vars/main.yml @@ -0,0 +1,9 @@ +--- +# Computed internal variables - do not override +__opa_server_wait_retries: "{{ (opa_server_wait_timeout / 10) | int }}" +__opa_server_host: "{{ opa_server_name }}.{{ opa_server_namespace }}.svc" +__opa_server_tls_secret: "{{ opa_server_name }}-tls" +__opa_server_policy_files: "{{ query('ansible.builtin.fileglob', opa_server_policy_dir ~ '/*.rego') | sort }}" +__opa_server_policies: >- + {{ dict(__opa_server_policy_files | map('basename') + | zip(query('ansible.builtin.file', *__opa_server_policy_files))) }} diff --git a/roles/rhoai_maas/defaults/main.yml b/roles/rhoai_maas/defaults/main.yml new file mode 100644 index 0000000..26fa3ad --- /dev/null +++ b/roles/rhoai_maas/defaults/main.yml @@ -0,0 +1,111 @@ +--- +# --- OLM operators --- +# Installed in order. RHOAI is pinned to stable-3.5: the MaaS DSC layout +# (aigateway.modelsAsAService) changed between 3.4 and 3.5, so don't float on stable-3.x. +# RHCL must be 1.4.x (Kuadrant >= 1.4.2 for AuthPolicy spec.defaults.rules). +# cert-manager is a prerequisite but already installed by deploy_openshift.yml. +rhoai_maas_operators: + - name: rhcl-operator + namespace: openshift-operators + channel: stable-1.4 + - name: leader-worker-set + namespace: openshift-lws-operator + channel: stable-v1.0 + operator_group_target_namespaces: + - openshift-lws-operator + - name: rhods-operator + namespace: redhat-ods-operator + channel: stable-3.5 + namespace_labels: + openshift.io/cluster-monitoring: "true" +rhoai_maas_wait_timeout: 900 + +# --- Gateway --- +# Empty = maas.. +rhoai_maas_hostname: "" +# The gateway gets its own single-host certificate. Don't use the *.apps wildcard: +# browsers would reuse the MaaS HTTP/2 connection for every other *.apps host +# (console, dashboards), and the MaaS gateway answers those with 404. +rhoai_maas_gateway_cert_secret: maas-gateway-tls +# cert-manager ClusterIssuer for the gateway certificate. Empty = use the existing +# rhoai_maas_gateway_cert_secret as-is and don't create a Certificate. +rhoai_maas_gateway_cert_issuer: letsencrypt-production +# No cloud LB on this cluster: run the gateway Service as ClusterIP and expose it +# through a passthrough Route on the default router. Set to LoadBalancer with MetalLB. +rhoai_maas_gateway_service_type: ClusterIP +rhoai_maas_gateway_create_route: true + +# --- MaaS database --- +# PostgreSQL is provisioned by the cloudnativepg_operator role (see the playbook); +# this secret holds its credentials (type kubernetes.io/basic-auth). +rhoai_maas_db_secret_name: maas-postgres-secret +rhoai_maas_db_secret_namespace: rhoai-maas-db +rhoai_maas_db_sslmode: require +rhoai_maas_infra_namespace: redhat-ai-gateway-infra + +# --- DataScienceCluster --- +# Minimal component set for MaaS. Anything not listed here is set to Removed. +rhoai_maas_dsc_name: default-dsc +rhoai_maas_dsc_managed_components: + - dashboard + - kserve + - aigateway + - ogx +rhoai_maas_dsc_removed_components: + - aipipelines + - feastoperator + - kueue + - mlflowoperator + - modelregistry + - ray + - trainer + - trainingoperator + - trustyai + - workbenches + +# --- Models (ExternalModel fronting in-cluster OpenAI-compatible servers) --- +# The api_key_secret must already exist in the model namespace with key `api-key`; +# the role only labels it for MaaS. Names must not collide with existing Services +# in the namespace (the controller creates a Service per provider/model). +rhoai_maas_external_models: + - name: qwen35-9b-coder + namespace: llm-coding-assistant + display_name: Qwen3.5 9B Coder (llama.cpp) + description: Qwen3.5-9B Q5_K_M served by llama.cpp on the desktop GPU node + provider_name: llama-coder-provider + provider_type: openai + endpoint: llama-coder.llm-coding-assistant.svc.cluster.local + port: 8080 + tls: false + api_key_secret: llama-coder-api-key + # llama-server serves a single model and ignores the request's model field. + target_model: Qwen3.5-9B-Q5_K_M.gguf + api_format: openai-chat + path: /v1/chat/completions + +# --- Access policy and quota (CR spec shapes, models-as-a-service namespace) --- +rhoai_maas_auth_policies: + - name: llama-coder-access + display_name: llama-coder access + description: All authenticated users can call the llama-coder model + groups: + - system:authenticated + users: [] + model_refs: + - name: qwen35-9b-coder + namespace: llm-coding-assistant + +rhoai_maas_subscriptions: + - name: llama-coder-default + display_name: llama-coder default + description: 200k tokens/hour per user for all authenticated users + priority: 10 + groups: + - system:authenticated + users: [] + model_refs: + - name: qwen35-9b-coder + namespace: llm-coding-assistant + tokenRateLimits: + - limit: 200000 + window: 1h diff --git a/roles/rhoai_maas/meta/argument_specs.yml b/roles/rhoai_maas/meta/argument_specs.yml new file mode 100644 index 0000000..c498571 --- /dev/null +++ b/roles/rhoai_maas/meta/argument_specs.yml @@ -0,0 +1,90 @@ +--- +argument_specs: + main: + short_description: Install Red Hat OpenShift AI 3.5 with Models-as-a-Service + description: + - Installs Red Hat Connectivity Link, Leader Worker Set and the RHOAI operator via OLM, + configures User Workload Monitoring, the MaaS Gateway, Kuadrant and Authorino TLS, + creates the maas-db-config secret, and enables a minimal DataScienceCluster with + aigateway.modelsAsAService Managed. + - Registers in-cluster OpenAI-compatible servers as ExternalModels (Technology Preview) + and applies MaaSAuthPolicy / MaaSSubscription resources. + - PostgreSQL is not provisioned here; run cloudnativepg_operator first (see + playbooks/deploy_rhoai_maas.yml). + options: + rhoai_maas_operators: + description: OLM operators to install, in order (name, namespace, channel, optional + operator_group_target_namespaces and namespace_labels). + type: list + elements: dict + rhoai_maas_wait_timeout: + description: Seconds to wait for each operator, CRD, and deployment to become ready. + type: int + default: 900 + rhoai_maas_hostname: + description: MaaS gateway hostname. Empty uses maas.. + type: str + default: "" + rhoai_maas_gateway_cert_secret: + description: >- + TLS Secret in openshift-ingress used by the Gateway HTTPS listener. Must be + a single-host cert, not the *.apps wildcard (browser HTTP/2 coalescing). + type: str + default: maas-gateway-tls + rhoai_maas_gateway_cert_issuer: + description: >- + cert-manager ClusterIssuer that issues rhoai_maas_gateway_cert_secret for the + MaaS hostname. Empty skips creating the Certificate. + type: str + default: letsencrypt-production + rhoai_maas_gateway_service_type: + description: Service type for the Gateway's generated Service. + type: str + choices: [ClusterIP, LoadBalancer, NodePort] + default: ClusterIP + rhoai_maas_gateway_create_route: + description: Create a passthrough Route on the default router to the Gateway Service. + type: bool + default: true + rhoai_maas_db_secret_name: + description: basic-auth Secret (username/password/host/port/database) for the MaaS database. + type: str + default: maas-postgres-secret + rhoai_maas_db_secret_namespace: + description: Namespace of rhoai_maas_db_secret_name. + type: str + default: rhoai-maas-db + rhoai_maas_db_sslmode: + description: libpq sslmode used in DB_CONNECTION_URL. + type: str + default: require + rhoai_maas_infra_namespace: + description: MaaS infrastructure namespace that holds maas-db-config (3.5+). + type: str + default: redhat-ai-gateway-infra + rhoai_maas_dsc_name: + description: DataScienceCluster name. + type: str + default: default-dsc + rhoai_maas_dsc_managed_components: + description: DSC components set to Managed. + type: list + elements: str + rhoai_maas_dsc_removed_components: + description: DSC components set to Removed. + type: list + elements: str + rhoai_maas_external_models: + description: In-cluster or external OpenAI-compatible endpoints to register as + ExternalProvider + ExternalModel + MaaSModelRef. + type: list + elements: dict + rhoai_maas_auth_policies: + description: MaaSAuthPolicy definitions (name, groups, users, model_refs). + type: list + elements: dict + rhoai_maas_subscriptions: + description: MaaSSubscription definitions (name, priority, groups, users, model_refs + with tokenRateLimits). + type: list + elements: dict diff --git a/roles/rhoai_maas/meta/main.yml b/roles/rhoai_maas/meta/main.yml new file mode 100644 index 0000000..4639d58 --- /dev/null +++ b/roles/rhoai_maas/meta/main.yml @@ -0,0 +1,18 @@ +--- +galaxy_info: + author: ptoal + description: Install Red Hat OpenShift AI with Models-as-a-Service and register in-cluster models + license: MIT + min_ansible_version: "2.16" + platforms: + - name: GenericLinux + versions: + - all + galaxy_tags: + - openshift + - rhoai + - maas + - operator + - olm + +dependencies: [] diff --git a/roles/rhoai_maas/tasks/database.yml b/roles/rhoai_maas/tasks/database.yml new file mode 100644 index 0000000..77a0159 --- /dev/null +++ b/roles/rhoai_maas/tasks/database.yml @@ -0,0 +1,49 @@ +--- +# Build maas-db-config (DB_CONNECTION_URL) from the CloudNativePG basic-auth secret. +# RHOAI 3.5+ reads it from the infrastructure namespace; it must exist before the +# DSC enables modelsAsAService. + +- name: Read MaaS database credentials + kubernetes.core.k8s_info: + api_version: v1 + kind: Secret + namespace: "{{ rhoai_maas_db_secret_namespace }}" + name: "{{ rhoai_maas_db_secret_name }}" + register: __rhoai_maas_db_secret + no_log: true + +- name: Verify MaaS database credentials exist + ansible.builtin.assert: + that: __rhoai_maas_db_secret.resources | length > 0 + fail_msg: >- + Secret {{ rhoai_maas_db_secret_namespace }}/{{ rhoai_maas_db_secret_name }} not found. + Run the cloudnativepg_operator role first (playbooks/deploy_rhoai_maas.yml does this). + +- name: Create MaaS infrastructure namespace + kubernetes.core.k8s: + state: present + definition: + apiVersion: v1 + kind: Namespace + metadata: + name: "{{ rhoai_maas_infra_namespace }}" + +- name: Create maas-db-config secret + vars: + __db: "{{ __rhoai_maas_db_secret.resources[0].data }}" + kubernetes.core.k8s: + state: present + definition: + apiVersion: v1 + kind: Secret + metadata: + name: maas-db-config + namespace: "{{ rhoai_maas_infra_namespace }}" + type: Opaque + stringData: + DB_CONNECTION_URL: >- + postgresql://{{ __db.username | b64decode | urlencode | replace('/', '%2F') }}:{{ + __db.password | b64decode | urlencode | replace('/', '%2F') }}@{{ + __db.host | b64decode }}:{{ __db.port | b64decode }}/{{ __db.database | b64decode + }}?sslmode={{ rhoai_maas_db_sslmode }} + no_log: true diff --git a/roles/rhoai_maas/tasks/main.yml b/roles/rhoai_maas/tasks/main.yml new file mode 100644 index 0000000..3b638a2 --- /dev/null +++ b/roles/rhoai_maas/tasks/main.yml @@ -0,0 +1,41 @@ +--- +# Install Red Hat OpenShift AI 3.5 with Models-as-a-Service and register +# in-cluster OpenAI-compatible servers (llama-coder) as MaaS ExternalModels. +# +# Follows "Govern LLM access with Models-as-a-Service" (RHOAI Self-Managed 3.5) +# and its companion guide https://github.com/jharmison-redhat/rhoai-maas-guide. +# +# Order matters: +# - GatewayClass before Kuadrant, or Kuadrant reports MissingDependency +# (Istio is provisioned by the GatewayClass). +# - maas-db-config before the DSC enables modelsAsAService. +# +# All tasks are idempotent (kubernetes.core.k8s state: present / patched). + +- name: Install operators + ansible.builtin.include_tasks: operator.yml + loop: "{{ rhoai_maas_operators }}" + loop_control: + loop_var: __rhoai_maas_operator + label: "{{ __rhoai_maas_operator.name }}" + +- name: Configure platform (monitoring, gateway, Kuadrant) + ansible.builtin.include_tasks: platform.yml + +- name: Configure MaaS database secret + ansible.builtin.include_tasks: database.yml + +- name: Configure DataScienceCluster + ansible.builtin.include_tasks: rhoai.yml + +- name: Register models and access policies + ansible.builtin.include_tasks: models.yml + +- name: Display MaaS deployment summary + ansible.builtin.debug: + msg: + - "Models-as-a-Service ready!" + - " Endpoint : https://{{ __rhoai_maas_hostname }}/v1" + - " Models : {{ rhoai_maas_external_models | map(attribute='name') | join(', ') }}" + - " API keys : RHOAI dashboard, or POST https://{{ __rhoai_maas_hostname }}/maas-api/v1/api-keys with an OpenShift token" + - " Test : curl -H \"Authorization: Bearer $MAAS_API_KEY\" https://{{ __rhoai_maas_hostname }}/v1/models" diff --git a/roles/rhoai_maas/tasks/models.yml b/roles/rhoai_maas/tasks/models.yml new file mode 100644 index 0000000..6af170b --- /dev/null +++ b/roles/rhoai_maas/tasks/models.yml @@ -0,0 +1,165 @@ +--- +# ------------------------------------------------------------------ +# ExternalProvider + ExternalModel + MaaSModelRef per model +# ------------------------------------------------------------------ +- name: Allow model namespaces to attach to the Gateway + kubernetes.core.k8s: + state: patched + api_version: v1 + kind: Namespace + name: "{{ item }}" + definition: + metadata: + labels: + maas.opendatahub.io/gateway-access: "true" + opendatahub.io/dashboard: "true" + loop: "{{ rhoai_maas_external_models | map(attribute='namespace') | unique }}" + +- name: Mark provider API key secrets as MaaS-managed + kubernetes.core.k8s: + state: patched + api_version: v1 + kind: Secret + namespace: "{{ item.namespace }}" + name: "{{ item.api_key_secret }}" + definition: + metadata: + labels: + inference.llm-d.ai/ipp-managed: "true" + loop: "{{ rhoai_maas_external_models }}" + loop_control: + label: "{{ item.namespace }}/{{ item.api_key_secret }}" + +- name: Create ExternalProviders + kubernetes.core.k8s: + state: present + definition: + apiVersion: inference.opendatahub.io/v1alpha1 + kind: ExternalProvider + metadata: + name: "{{ item.provider_name }}" + namespace: "{{ item.namespace }}" + spec: + provider: "{{ item.provider_type }}" + endpoint: "{{ item.endpoint }}" + auth: + type: apikey + secretRef: + name: "{{ item.api_key_secret }}" + loop: "{{ rhoai_maas_external_models }}" + loop_control: + label: "{{ item.namespace }}/{{ item.provider_name }}" + +- name: Wait for ExternalProviders to be ready + kubernetes.core.k8s_info: + api_version: inference.opendatahub.io/v1alpha1 + kind: ExternalProvider + namespace: "{{ item.namespace }}" + name: "{{ item.provider_name }}" + register: __rhoai_maas_provider + until: (__rhoai_maas_provider.resources[0].status.phase | default('')) == 'Ready' + retries: 30 + delay: 10 + loop: "{{ rhoai_maas_external_models }}" + loop_control: + label: "{{ item.namespace }}/{{ item.provider_name }}" + +- name: Create ExternalModels + kubernetes.core.k8s: + state: present + definition: + apiVersion: inference.opendatahub.io/v1alpha1 + kind: ExternalModel + metadata: + name: "{{ item.name }}" + namespace: "{{ item.namespace }}" + annotations: + maas.opendatahub.io/tls: "{{ item.tls | default(true) | string | lower }}" + maas.opendatahub.io/port: "{{ item.port | default(443) | string }}" + spec: + externalProviderRefs: + - ref: + name: "{{ item.provider_name }}" + targetModel: "{{ item.target_model }}" + apiFormat: "{{ item.api_format | default('openai-chat') }}" + path: "{{ item.path | default('/v1/chat/completions') }}" + weight: 100 + loop: "{{ rhoai_maas_external_models }}" + loop_control: + label: "{{ item.namespace }}/{{ item.name }}" + +- name: Publish models to MaaS + kubernetes.core.k8s: + state: present + definition: + apiVersion: maas.opendatahub.io/v1alpha1 + kind: MaaSModelRef + metadata: + name: "{{ item.name }}" + namespace: "{{ item.namespace }}" + annotations: + openshift.io/display-name: "{{ item.display_name | default(item.name) }}" + openshift.io/description: "{{ item.description | default('') }}" + spec: + modelRef: + kind: ExternalModel + name: "{{ item.name }}" + loop: "{{ rhoai_maas_external_models }}" + loop_control: + label: "{{ item.namespace }}/{{ item.name }}" + +# ------------------------------------------------------------------ +# Access policies and quota (models-as-a-service namespace) +# ------------------------------------------------------------------ +- name: Wait for the models-as-a-service namespace + kubernetes.core.k8s_info: + api_version: v1 + kind: Namespace + name: "{{ __rhoai_maas_namespace }}" + register: __rhoai_maas_ns + until: __rhoai_maas_ns.resources | length > 0 + retries: "{{ __rhoai_maas_wait_retries }}" + delay: 10 + +- name: Create MaaSAuthPolicies + kubernetes.core.k8s: + state: present + definition: + apiVersion: maas.opendatahub.io/v1alpha1 + kind: MaaSAuthPolicy + metadata: + name: "{{ item.name }}" + namespace: "{{ __rhoai_maas_namespace }}" + annotations: + openshift.io/display-name: "{{ item.display_name | default(item.name) }}" + openshift.io/description: "{{ item.description | default('') }}" + spec: + modelRefs: "{{ item.model_refs }}" + subjects: + groups: "{{ item.groups | default([]) | map('community.general.dict_kv', 'name') | list }}" + users: "{{ item.users | default([]) }}" + loop: "{{ rhoai_maas_auth_policies }}" + loop_control: + label: "{{ item.name }}" + +- name: Create MaaSSubscriptions + kubernetes.core.k8s: + state: present + definition: + apiVersion: maas.opendatahub.io/v1alpha1 + kind: MaaSSubscription + metadata: + name: "{{ item.name }}" + namespace: "{{ __rhoai_maas_namespace }}" + annotations: + openshift.io/display-name: "{{ item.display_name | default(item.name) }}" + openshift.io/description: "{{ item.description | default('') }}" + spec: + owner: + groups: "{{ item.groups | default([]) | map('community.general.dict_kv', 'name') | list }}" + users: "{{ item.users | default([]) }}" + modelRefs: "{{ item.model_refs }}" + priority: "{{ item.priority | default(10) | int }}" + loop: "{{ rhoai_maas_subscriptions }}" + loop_control: + label: "{{ item.name }}" diff --git a/roles/rhoai_maas/tasks/operator.yml b/roles/rhoai_maas/tasks/operator.yml new file mode 100644 index 0000000..5aaa694 --- /dev/null +++ b/roles/rhoai_maas/tasks/operator.yml @@ -0,0 +1,69 @@ +--- +# Install one OLM operator (Automatic approval) and wait for its CSV. +# Expects __rhoai_maas_operator: {name, namespace, channel, +# operator_group_target_namespaces?, namespace_labels?} + +- name: Create operator namespace {{ __rhoai_maas_operator.namespace }} + kubernetes.core.k8s: + state: present + definition: + apiVersion: v1 + kind: Namespace + metadata: + name: "{{ __rhoai_maas_operator.namespace }}" + labels: "{{ __rhoai_maas_operator.namespace_labels | default({}) }}" + +# openshift-operators already has the global-operators OperatorGroup +- name: Create OperatorGroup for {{ __rhoai_maas_operator.name }} + kubernetes.core.k8s: + state: present + definition: + apiVersion: operators.coreos.com/v1 + kind: OperatorGroup + metadata: + name: "{{ __rhoai_maas_operator.name }}" + namespace: "{{ __rhoai_maas_operator.namespace }}" + spec: >- + {{ {'targetNamespaces': __rhoai_maas_operator.operator_group_target_namespaces} + if __rhoai_maas_operator.operator_group_target_namespaces is defined else {} }} + when: __rhoai_maas_operator.namespace != 'openshift-operators' + +- name: Subscribe to {{ __rhoai_maas_operator.name }} + kubernetes.core.k8s: + state: present + definition: + apiVersion: operators.coreos.com/v1alpha1 + kind: Subscription + metadata: + name: "{{ __rhoai_maas_operator.name }}" + namespace: "{{ __rhoai_maas_operator.namespace }}" + spec: + channel: "{{ __rhoai_maas_operator.channel }}" + installPlanApproval: Automatic + name: "{{ __rhoai_maas_operator.name }}" + source: "{{ __rhoai_maas_operator.source | default('redhat-operators') }}" + sourceNamespace: openshift-marketplace + +- name: Wait for Subscription to report an installed CSV - {{ __rhoai_maas_operator.name }} + kubernetes.core.k8s_info: + api_version: operators.coreos.com/v1alpha1 + kind: Subscription + namespace: "{{ __rhoai_maas_operator.namespace }}" + name: "{{ __rhoai_maas_operator.name }}" + register: __rhoai_maas_sub + until: (__rhoai_maas_sub.resources[0].status.installedCSV | default('')) | length > 0 + retries: "{{ __rhoai_maas_wait_retries }}" + delay: 10 + +- name: Wait for CSV to succeed - {{ __rhoai_maas_operator.name }} + kubernetes.core.k8s_info: + api_version: operators.coreos.com/v1alpha1 + kind: ClusterServiceVersion + namespace: "{{ __rhoai_maas_operator.namespace }}" + name: "{{ __rhoai_maas_sub.resources[0].status.installedCSV }}" + register: __rhoai_maas_csv + until: >- + __rhoai_maas_csv.resources | length > 0 and + __rhoai_maas_csv.resources[0].status.phase | default('') == 'Succeeded' + retries: "{{ __rhoai_maas_wait_retries }}" + delay: 10 diff --git a/roles/rhoai_maas/tasks/platform.yml b/roles/rhoai_maas/tasks/platform.yml new file mode 100644 index 0000000..58f8edb --- /dev/null +++ b/roles/rhoai_maas/tasks/platform.yml @@ -0,0 +1,361 @@ +--- +# ------------------------------------------------------------------ +# User Workload Monitoring (MaaS reports Degraded without it) +# ------------------------------------------------------------------ +- name: Read cluster monitoring config + kubernetes.core.k8s_info: + api_version: v1 + kind: ConfigMap + namespace: openshift-monitoring + name: cluster-monitoring-config + register: __rhoai_maas_cmo + +- name: Resolve cluster monitoring config + ansible.builtin.set_fact: + __rhoai_maas_cmo_config: >- + {{ ((__rhoai_maas_cmo.resources[0].data['config.yaml'] | default('{}')) if __rhoai_maas_cmo.resources else '{}') + | from_yaml | default({}, true) }} + +- name: Enable User Workload Monitoring + kubernetes.core.k8s: + state: present + definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: cluster-monitoring-config + namespace: openshift-monitoring + data: + config.yaml: "{{ __rhoai_maas_cmo_config | combine({'enableUserWorkload': true}) | to_nice_yaml }}" + when: not (__rhoai_maas_cmo_config.enableUserWorkload | default(false) | bool) + +# ------------------------------------------------------------------ +# Gateway API: GatewayClass, Gateway, passthrough Route +# ------------------------------------------------------------------ +- name: Read cluster ingress domain + kubernetes.core.k8s_info: + api_version: config.openshift.io/v1 + kind: Ingress + name: cluster + register: __rhoai_maas_ingress + when: rhoai_maas_hostname | length == 0 + +- name: Resolve MaaS hostname + ansible.builtin.set_fact: + __rhoai_maas_hostname: >- + {{ rhoai_maas_hostname if rhoai_maas_hostname | length > 0 + else 'maas.' ~ __rhoai_maas_ingress.resources[0].spec.domain }} + +- name: Create MaaS gateway certificate + kubernetes.core.k8s: + state: present + definition: + apiVersion: cert-manager.io/v1 + kind: Certificate + metadata: + name: "{{ __rhoai_maas_gateway_name }}-cert" + namespace: "{{ __rhoai_maas_gateway_namespace }}" + spec: + secretName: "{{ rhoai_maas_gateway_cert_secret }}" + issuerRef: + name: "{{ rhoai_maas_gateway_cert_issuer }}" + kind: ClusterIssuer + dnsNames: + - "{{ __rhoai_maas_hostname }}" + duration: 2160h + renewBefore: 720h + when: rhoai_maas_gateway_cert_issuer | length > 0 + +- name: Wait for MaaS gateway certificate to be issued + kubernetes.core.k8s_info: + api_version: cert-manager.io/v1 + kind: Certificate + namespace: "{{ __rhoai_maas_gateway_namespace }}" + name: "{{ __rhoai_maas_gateway_name }}-cert" + register: __rhoai_maas_gw_cert + until: >- + (__rhoai_maas_gw_cert.resources[0].status.conditions | default([]) + | selectattr('type', 'equalto', 'Ready') | selectattr('status', 'equalto', 'True') | list | length) > 0 + retries: 30 + delay: 10 + when: rhoai_maas_gateway_cert_issuer | length > 0 + +# Creating the GatewayClass makes the ingress operator install Service Mesh 3 (Istio) +- name: Create openshift-default GatewayClass + kubernetes.core.k8s: + state: present + definition: + apiVersion: gateway.networking.k8s.io/v1 + kind: GatewayClass + metadata: + name: openshift-default + spec: + controllerName: openshift.io/gateway-controller/v1 + +- name: Wait for GatewayClass to be accepted + kubernetes.core.k8s_info: + api_version: gateway.networking.k8s.io/v1 + kind: GatewayClass + name: openshift-default + register: __rhoai_maas_gwc + until: >- + __rhoai_maas_gwc.resources | length > 0 and + (__rhoai_maas_gwc.resources[0].status.conditions | default([]) + | selectattr('type', 'equalto', 'Accepted') | selectattr('status', 'equalto', 'True') | list | length) > 0 + retries: "{{ __rhoai_maas_wait_retries }}" + delay: 10 + +- name: Create Gateway deployment options + kubernetes.core.k8s: + state: present + definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: maas-gateway-options + namespace: "{{ __rhoai_maas_gateway_namespace }}" + data: + deployment: | + spec: + template: + spec: + containers: + - name: istio-proxy + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: "2" + memory: 2Gi + +- name: Create MaaS Gateway + kubernetes.core.k8s: + state: present + definition: + apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + name: "{{ __rhoai_maas_gateway_name }}" + namespace: "{{ __rhoai_maas_gateway_namespace }}" + annotations: + opendatahub.io/managed: "false" + security.opendatahub.io/authorino-tls-bootstrap: "true" + networking.istio.io/service-type: "{{ rhoai_maas_gateway_service_type }}" + spec: + gatewayClassName: openshift-default + infrastructure: + parametersRef: + group: "" + kind: ConfigMap + name: maas-gateway-options + listeners: + - name: http + hostname: "{{ __rhoai_maas_hostname }}" + port: 80 + protocol: HTTP + allowedRoutes: + namespaces: + from: Selector + selector: + matchLabels: + maas.opendatahub.io/gateway-access: "true" + - name: https + hostname: "{{ __rhoai_maas_hostname }}" + port: 443 + protocol: HTTPS + allowedRoutes: + namespaces: + from: Selector + selector: + matchLabels: + maas.opendatahub.io/gateway-access: "true" + tls: + mode: Terminate + certificateRefs: + - group: "" + kind: Secret + name: "{{ rhoai_maas_gateway_cert_secret }}" + +- name: Wait for MaaS Gateway to be programmed + kubernetes.core.k8s_info: + api_version: gateway.networking.k8s.io/v1 + kind: Gateway + namespace: "{{ __rhoai_maas_gateway_namespace }}" + name: "{{ __rhoai_maas_gateway_name }}" + register: __rhoai_maas_gw + until: >- + (__rhoai_maas_gw.resources[0].status.conditions | default([]) + | selectattr('type', 'equalto', 'Programmed') | selectattr('status', 'equalto', 'True') | list | length) > 0 + retries: "{{ __rhoai_maas_wait_retries }}" + delay: 10 + +- name: Expose MaaS Gateway through the default router + kubernetes.core.k8s: + state: present + definition: + apiVersion: route.openshift.io/v1 + kind: Route + metadata: + name: "{{ __rhoai_maas_gateway_name }}-https" + namespace: "{{ __rhoai_maas_gateway_namespace }}" + spec: + host: "{{ __rhoai_maas_hostname }}" + port: + targetPort: https + tls: + termination: passthrough + insecureEdgeTerminationPolicy: Redirect + to: + kind: Service + name: "{{ __rhoai_maas_gateway_service }}" + weight: 100 + when: rhoai_maas_gateway_create_route | bool + +# ------------------------------------------------------------------ +# Kuadrant (Authorino + Limitador) and Authorino TLS +# ------------------------------------------------------------------ +- name: Create kuadrant-system namespace + kubernetes.core.k8s: + state: present + definition: + apiVersion: v1 + kind: Namespace + metadata: + name: kuadrant-system + +- name: Create Kuadrant instance + kubernetes.core.k8s: + state: present + definition: + apiVersion: kuadrant.io/v1beta1 + kind: Kuadrant + metadata: + name: kuadrant + namespace: kuadrant-system + spec: + observability: + enable: true + +# The Kuadrant operator only detects the Gateway API provider (Istio) at startup. +# RHCL is installed before the GatewayClass provisions Istio, so on a fresh +# install Kuadrant reports MissingDependency until the operator pod restarts. +- name: Read Kuadrant status + kubernetes.core.k8s_info: + api_version: kuadrant.io/v1beta1 + kind: Kuadrant + namespace: kuadrant-system + name: kuadrant + register: __rhoai_maas_kuadrant + until: >- + (__rhoai_maas_kuadrant.resources[0].status.conditions | default([]) + | selectattr('type', 'equalto', 'Ready') | list | length) > 0 + retries: 12 + delay: 5 + failed_when: false + +- name: Restart the Kuadrant operator to detect Istio + kubernetes.core.k8s: + state: absent + api_version: v1 + kind: Pod + namespace: openshift-operators + label_selectors: + - app=kuadrant + - control-plane=controller-manager + when: >- + (__rhoai_maas_kuadrant.resources[0].status.conditions | default([]) + | selectattr('type', 'equalto', 'Ready') | map(attribute='reason') | first | default('')) + == 'MissingDependency' + +- name: Wait for Kuadrant to be ready + kubernetes.core.k8s_info: + api_version: kuadrant.io/v1beta1 + kind: Kuadrant + namespace: kuadrant-system + name: kuadrant + register: __rhoai_maas_kuadrant + until: >- + (__rhoai_maas_kuadrant.resources[0].status.conditions | default([]) + | selectattr('type', 'equalto', 'Ready') | selectattr('status', 'equalto', 'True') | list | length) > 0 + retries: "{{ __rhoai_maas_wait_retries }}" + delay: 10 + +- name: Wait for Authorino service + kubernetes.core.k8s_info: + api_version: v1 + kind: Service + namespace: kuadrant-system + name: authorino-authorino-authorization + register: __rhoai_maas_authorino_svc + until: __rhoai_maas_authorino_svc.resources | length > 0 + retries: "{{ __rhoai_maas_wait_retries }}" + delay: 10 + +- name: Request a service-ca serving certificate for Authorino + kubernetes.core.k8s: + state: patched + api_version: v1 + kind: Service + namespace: kuadrant-system + name: authorino-authorino-authorization + definition: + metadata: + annotations: + service.beta.openshift.io/serving-cert-secret-name: authorino-server-cert + +- name: Wait for Authorino serving certificate + kubernetes.core.k8s_info: + api_version: v1 + kind: Secret + namespace: kuadrant-system + name: authorino-server-cert + register: __rhoai_maas_authorino_cert + until: __rhoai_maas_authorino_cert.resources | length > 0 + retries: 30 + delay: 10 + no_log: true + +- name: Enable Authorino TLS listener + kubernetes.core.k8s: + state: patched + api_version: operator.authorino.kuadrant.io/v1beta1 + kind: Authorino + namespace: kuadrant-system + name: authorino + merge_type: merge + definition: + spec: + listener: + tls: + enabled: true + certSecretRef: + name: authorino-server-cert + +- name: Read Authorino deployment + kubernetes.core.k8s_info: + api_version: apps/v1 + kind: Deployment + namespace: kuadrant-system + name: authorino + register: __rhoai_maas_authorino_deploy + +- name: Trust the service CA in Authorino + kubernetes.core.k8s: + state: patched + api_version: apps/v1 + kind: Deployment + namespace: kuadrant-system + name: authorino + merge_type: strategic-merge + definition: + spec: + template: + spec: + containers: + - name: "{{ __rhoai_maas_authorino_deploy.resources[0].spec.template.spec.containers[0].name }}" + env: + - name: SSL_CERT_FILE + value: /etc/ssl/certs/openshift-service-ca/service-ca-bundle.crt + - name: REQUESTS_CA_BUNDLE + value: /etc/ssl/certs/openshift-service-ca/service-ca-bundle.crt diff --git a/roles/rhoai_maas/tasks/rhoai.yml b/roles/rhoai_maas/tasks/rhoai.yml new file mode 100644 index 0000000..86920ac --- /dev/null +++ b/roles/rhoai_maas/tasks/rhoai.yml @@ -0,0 +1,143 @@ +--- +- name: Wait for RHOAI CRDs + kubernetes.core.k8s_info: + api_version: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + name: "{{ item }}" + register: __rhoai_maas_crd + until: >- + __rhoai_maas_crd.resources | length > 0 and + (__rhoai_maas_crd.resources[0].status.conditions | default([]) + | selectattr('type', 'equalto', 'Established') | selectattr('status', 'equalto', 'True') | list | length) > 0 + retries: "{{ __rhoai_maas_wait_retries }}" + delay: 10 + loop: + - dscinitializations.dscinitialization.opendatahub.io + - datascienceclusters.datasciencecluster.opendatahub.io + +- name: Configure DSCInitialization + kubernetes.core.k8s: + state: present + definition: + apiVersion: dscinitialization.opendatahub.io/v2 + kind: DSCInitialization + metadata: + name: default-dsci + spec: + applicationsNamespace: "{{ __rhoai_maas_apps_namespace }}" + monitoring: + managementState: Managed + trustedCABundle: + managementState: Managed + customCABundle: "" + register: __rhoai_maas_dsci_result + # The operator webhook may not be serving yet right after the CSV succeeds + until: __rhoai_maas_dsci_result is not failed + retries: 30 + delay: 10 + +- name: Resolve DataScienceCluster components + ansible.builtin.set_fact: + __rhoai_maas_dsc_components: >- + {{ dict(rhoai_maas_dsc_managed_components | product([{'managementState': 'Managed'}])) + | combine(dict(rhoai_maas_dsc_removed_components | product([{'managementState': 'Removed'}]))) + | combine(__rhoai_maas_dsc_component_overrides, recursive=True) }} + +- name: Configure DataScienceCluster + kubernetes.core.k8s: + state: present + definition: + apiVersion: datasciencecluster.opendatahub.io/v2 + kind: DataScienceCluster + metadata: + name: "{{ rhoai_maas_dsc_name }}" + spec: + components: "{{ __rhoai_maas_dsc_components }}" + register: __rhoai_maas_dsc_result + until: __rhoai_maas_dsc_result is not failed + retries: 30 + delay: 10 + +# The API server silently prunes unknown fields, so a misspelled component +# setting applies cleanly but leaves MaaS Removed and its CRDs never appear. +- name: Verify MaaS is enabled in the DataScienceCluster + ansible.builtin.assert: + that: + - >- + (__rhoai_maas_dsc_result.result.spec.components.aigateway.modelsAsAService.managementState + | default('')) == 'Managed' + fail_msg: >- + DataScienceCluster {{ rhoai_maas_dsc_name }} does not have + aigateway.modelsAsAService Managed after apply (field pruned or overridden): + {{ __rhoai_maas_dsc_result.result.spec.components.aigateway | default({}) }} + +- name: Wait for MaaS CRDs + kubernetes.core.k8s_info: + api_version: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + name: "{{ item }}" + register: __rhoai_maas_crd + until: >- + __rhoai_maas_crd.resources | length > 0 and + (__rhoai_maas_crd.resources[0].status.conditions | default([]) + | selectattr('type', 'equalto', 'Established') | selectattr('status', 'equalto', 'True') | list | length) > 0 + retries: "{{ __rhoai_maas_wait_retries }}" + delay: 10 + loop: + - maasmodelrefs.maas.opendatahub.io + - maasauthpolicies.maas.opendatahub.io + - maassubscriptions.maas.opendatahub.io + - externalproviders.inference.opendatahub.io + - externalmodels.inference.opendatahub.io + - odhdashboardconfigs.opendatahub.io + +# The operator normally labels this itself; keep it as a safety net +- name: Allow the MaaS infrastructure namespace to attach to the Gateway + kubernetes.core.k8s: + state: patched + api_version: v1 + kind: Namespace + name: "{{ rhoai_maas_infra_namespace }}" + definition: + metadata: + labels: + maas.opendatahub.io/gateway-access: "true" + +- name: Wait for maas-api to be available + kubernetes.core.k8s_info: + api_version: apps/v1 + kind: Deployment + namespace: "{{ rhoai_maas_infra_namespace }}" + name: maas-api + register: __rhoai_maas_api + until: >- + __rhoai_maas_api.resources | length > 0 and + (__rhoai_maas_api.resources[0].status.availableReplicas | default(0)) >= 1 + retries: "{{ __rhoai_maas_wait_retries }}" + delay: 10 + +- name: Wait for the dashboard config + kubernetes.core.k8s_info: + api_version: opendatahub.io/v1alpha + kind: OdhDashboardConfig + namespace: "{{ __rhoai_maas_apps_namespace }}" + name: odh-dashboard-config + register: __rhoai_maas_dashboard + until: __rhoai_maas_dashboard.resources | length > 0 + retries: "{{ __rhoai_maas_wait_retries }}" + delay: 10 + +- name: Enable MaaS and external models in the dashboard + kubernetes.core.k8s: + state: patched + api_version: opendatahub.io/v1alpha + kind: OdhDashboardConfig + namespace: "{{ __rhoai_maas_apps_namespace }}" + name: odh-dashboard-config + merge_type: merge + definition: + spec: + dashboardConfig: + modelAsService: true + externalModels: true + genAiStudio: true diff --git a/roles/rhoai_maas/vars/main.yml b/roles/rhoai_maas/vars/main.yml new file mode 100644 index 0000000..286969c --- /dev/null +++ b/roles/rhoai_maas/vars/main.yml @@ -0,0 +1,19 @@ +--- +# Computed internal variables - do not override +__rhoai_maas_wait_retries: "{{ (rhoai_maas_wait_timeout / 10) | int }}" +__rhoai_maas_apps_namespace: redhat-ods-applications +__rhoai_maas_namespace: models-as-a-service +__rhoai_maas_gateway_name: maas-default-gateway +__rhoai_maas_gateway_namespace: openshift-ingress +# Service name the OpenShift gateway controller derives from - +__rhoai_maas_gateway_service: "{{ __rhoai_maas_gateway_name }}-openshift-default" + +# Component settings layered on top of Managed/Removed in the DataScienceCluster +__rhoai_maas_dsc_component_overrides: + aigateway: + modelsAsAService: + managementState: Managed + kserve: + rawDeploymentServiceConfig: Headed + nim: + managementState: Removed