From e11dc229995b89cd6dafd3fc766bb1d5b54ffd63 Mon Sep 17 00:00:00 2001 From: beatz174-bit Date: Tue, 21 Apr 2026 12:07:29 +1000 Subject: [PATCH] Add phase-1 Ansible foundation and validation scaffolding --- README.md | 14 +++- docs/ansible-workflows.md | 72 +++++++++++++++++++ docs/infrastructure-inventory.md | 14 +++- docs/repo-structure.md | 5 +- docs/source-of-truth.md | 12 ++++ infrastructure/ansible/README.md | 46 ++++++++++++ infrastructure/ansible/ansible.cfg | 9 +++ .../ansible/collections/requirements.yml | 4 ++ .../ansible/inventory/group_vars/all.yml | 14 ++++ .../ansible/inventory/host_vars/.gitkeep | 0 infrastructure/ansible/inventory/hosts.yml | 17 +++++ infrastructure/ansible/playbooks/ping.yml | 7 ++ infrastructure/ansible/roles/.gitkeep | 0 scripts/codex-maintenance.sh | 36 +++++++++- scripts/codex-setup.sh | 36 +++++++++- 15 files changed, 277 insertions(+), 9 deletions(-) create mode 100644 docs/ansible-workflows.md create mode 100644 infrastructure/ansible/README.md create mode 100644 infrastructure/ansible/ansible.cfg create mode 100644 infrastructure/ansible/collections/requirements.yml create mode 100644 infrastructure/ansible/inventory/group_vars/all.yml create mode 100644 infrastructure/ansible/inventory/host_vars/.gitkeep create mode 100644 infrastructure/ansible/inventory/hosts.yml create mode 100644 infrastructure/ansible/playbooks/ping.yml create mode 100644 infrastructure/ansible/roles/.gitkeep diff --git a/README.md b/README.md index a725d16..94b96c7 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ If you only read one section, read **[Source-of-truth boundaries](docs/source-of - Docker environment composition and `services-up.sh`: [docs/docker-environment.md](docs/docker-environment.md) - Terraform workflows (brownfield import/reconciliation): [docs/terraform-workflows.md](docs/terraform-workflows.md) - Infrastructure inventory intent and outputs: [docs/infrastructure-inventory.md](docs/infrastructure-inventory.md) +- Ansible bootstrap workflows: [docs/ansible-workflows.md](docs/ansible-workflows.md) - Deployment prerequisites and secrets setup: [docs/deployment-prerequisites.md](docs/deployment-prerequisites.md) - Secrets inventory: [docs/security-secrets.md](docs/security-secrets.md) @@ -25,8 +26,9 @@ Codex helper scripts: - Initial Codex environment/bootstrap setup: [scripts/codex-setup.sh](scripts/codex-setup.sh) - Codex environment maintenance/refresh: [scripts/codex-maintenance.sh](scripts/codex-maintenance.sh) -Terraform subtrees: +Infrastructure subtrees: +- Ansible foundation docs: [infrastructure/ansible/README.md](infrastructure/ansible/README.md) - Terraform root docs: [infrastructure/terraform/README.md](infrastructure/terraform/README.md) - Terraform Docker mirror: [infrastructure/terraform/docker/README.md](infrastructure/terraform/docker/README.md) - Terraform Proxmox inventory: [infrastructure/terraform/proxmox/README.md](infrastructure/terraform/proxmox/README.md) @@ -41,6 +43,13 @@ Terraform subtrees: - `services-up.sh` composes the environment by discovering compose files and applying common env/network inputs. - For service runtime behavior, start from Compose files and `services-up.sh` (not Terraform). + +### Ansible (bootstrap foundation) + +- Ansible under `infrastructure/ansible/` is a phase-1 foundation for inventory/configuration scaffolding. +- It supports safe validation (inventory parsing and playbook syntax checks) while hosts/devices are onboarded gradually. +- It does not replace Compose runtime authority or Terraform reconciliation authority at this stage. + ### Terraform (inventory and reconciliation authority) - Terraform under `infrastructure/terraform/` is used to codify and reconcile existing infrastructure. @@ -118,11 +127,14 @@ The repository includes helper scripts for Codex sessions that need local toolin - `scripts/codex-setup.sh` - Installs baseline CLI dependencies (shell/yaml/terraform/ansible tooling). - Prepares `secrets/stack-secrets.env` from templates and creates dummy file-based secret placeholders based on `secrets/inventory.json`. + - Installs/refreshed baseline Ansible collections when `infrastructure/ansible/collections/requirements.yml` is present. + - Runs safe Ansible bootstrap checks (version, inventory parse, playbook syntax check) without live connectivity operations. - Prints installed tool versions for quick verification. - `scripts/codex-maintenance.sh` - Refreshes Python-based linting/automation tooling. - Reconciles placeholder secret files against current `secrets/inventory.json` (creates missing, removes stale). - Rebuilds `secrets/stack-secrets.env` with dummy values for compose-config validation. + - Refreshes Ansible collections and repeats safe inventory/syntax validation checks. Both scripts are intended for local validation environments and should not be treated as production provisioning automation. diff --git a/docs/ansible-workflows.md b/docs/ansible-workflows.md new file mode 100644 index 0000000..ef43777 --- /dev/null +++ b/docs/ansible-workflows.md @@ -0,0 +1,72 @@ +# Ansible Workflows (Bootstrap / Phase 1) + +Ansible is being introduced as a minimal, maintainable foundation for host/device inventory and future configuration workflows. + +## Why introduce Ansible now + +- The repository already has strong runtime and infrastructure boundaries (Compose + Terraform). +- A small Ansible baseline allows gradual host onboarding without forcing immediate large-scale automation. +- It enables safe validation workflows (`inventory --list`, playbook syntax checks) before real execution. + +## What Ansible is for in this repository (right now) + +- YAML inventory structure for hosts/devices to be onboarded over time. +- Group and host variable scaffolding for future incremental adoption. +- Validation-oriented starter playbook and local tooling checks. + +## What Ansible is not for yet + +- Replacing Docker Compose runtime authority. +- Replacing Terraform inventory/reconciliation authority. +- Becoming the current source of truth for NixOS host management. +- Becoming the current source of truth for all network automation. + +## Directory layout + +- `infrastructure/ansible/ansible.cfg` +- `infrastructure/ansible/inventory/hosts.yml` +- `infrastructure/ansible/inventory/group_vars/` +- `infrastructure/ansible/inventory/host_vars/` +- `infrastructure/ansible/playbooks/ping.yml` +- `infrastructure/ansible/collections/requirements.yml` + +## Add a host (gradual onboarding) + +1. Open `infrastructure/ansible/inventory/hosts.yml`. +2. Add the host under an appropriate group (`linux`, `network`, `virtualization`, or `nixos`). +3. Add non-sensitive defaults under group vars only when shared across hosts. +4. Add host-specific values in `inventory/host_vars/.yml`. +5. Keep secrets out of committed files. + +Example pattern: + +```yaml +linux: + hosts: + my-host: + ansible_host: my-host.local +``` + +## Validation commands + +Run from repository root: + +```bash +ansible --version +ansible-lint --version +ansible-inventory -i infrastructure/ansible/inventory/hosts.yml --list +ansible-playbook -i infrastructure/ansible/inventory/hosts.yml infrastructure/ansible/playbooks/ping.yml --syntax-check +``` + +Install/update baseline collections: + +```bash +ansible-galaxy collection install -r infrastructure/ansible/collections/requirements.yml -p infrastructure/ansible/collections +``` + +## Guardrails for future expansion + +- Keep changes incremental (one host/group/playbook change at a time). +- Prefer simple playbooks before introducing roles. +- Add network-platform/NixOS-specific logic only when those boundaries are explicitly adopted. +- Keep documentation aligned with source-of-truth boundaries when Ansible authority evolves. diff --git a/docs/infrastructure-inventory.md b/docs/infrastructure-inventory.md index 6cb62d7..799658f 100644 --- a/docs/infrastructure-inventory.md +++ b/docs/infrastructure-inventory.md @@ -25,11 +25,18 @@ This is currently the most structured host/VM inventory in the repo. These resources should match existing running containers, not redefine runtime composition strategy. -### 3) Compose runtime definitions + +### 3) Ansible bootstrap layer + +`infrastructure/ansible/` provides an emerging inventory/configuration scaffold for hosts and devices. + +Current scope is intentionally limited to structure, variables scaffolding, and safe validation workflows. + +### 4) Compose runtime definitions Compose files define intended service runtime composition, networking, labels, and integration. -### 4) Architecture docs +### 5) Architecture docs `docs/architecture.md` provides a human-readable topology view based on repository configuration and observed runtime signals. @@ -46,6 +53,7 @@ When adding Terraform outputs for documentation/tooling: - No full generated inventory document pipeline is present yet. - Some Terraform files still include generated boilerplate comments requiring ongoing cleanup. -- Ansible/NixOS operational layers are not yet implemented in a way that provides authoritative inventory in this repo. +- Ansible is currently a bootstrap inventory/configuration layer and is not authoritative for full operations yet. +- NixOS operational management is not yet implemented as an Ansible authority in this repo. These limitations are expected for the current adoption stage. diff --git a/docs/repo-structure.md b/docs/repo-structure.md index 0e039e0..d85e376 100644 --- a/docs/repo-structure.md +++ b/docs/repo-structure.md @@ -8,6 +8,7 @@ This page explains where to find authoritative files quickly. - `apps/` — user/business applications (Nextcloud, Passbolt, Gitea, Gramps, SearXNG). - `monitoring/` — observability and operational tooling (Prometheus, Grafana, InfluxDB, Node-RED, etc.). - `infrastructure/terraform/` — brownfield Terraform inventory/reconciliation layers. +- `infrastructure/ansible/` — phase-1 Ansible inventory/configuration scaffold and validation playbooks. - `docs/` — repository-level architecture and workflow documentation. - `archive/` — historical compose/config artifacts not part of active runtime composition. - `secrets/` — local secret material and templates; never commit real values. @@ -18,7 +19,7 @@ This page explains where to find authoritative files quickly. - `default-network.yml` — shared docker network definitions used across compose files. - `default-environment.env` — non-secret default env values for compose rendering. - `scripts/codex-setup.sh` — Codex/bootstrap helper to install validation tooling and prepare dummy secret material. -- `scripts/codex-maintenance.sh` — Codex maintenance helper to refresh tooling and reconcile dummy secret material. +- `scripts/codex-maintenance.sh` — Codex maintenance helper to refresh tooling, reconcile dummy secret material, and run safe Ansible validation checks. - `docs/deployment-prerequisites.md` — prerequisite setup before runtime operations. - `docs/security-secrets.md` — secrets documentation and inventory model. @@ -37,3 +38,5 @@ This page explains where to find authoritative files quickly. 3. Read [docs/docker-environment.md](docker-environment.md). 4. Read [docs/terraform-workflows.md](terraform-workflows.md). 5. Only then edit Compose/Terraform files. + +6. For Ansible bootstrap changes, validate inventory and playbook syntax checks only. diff --git a/docs/source-of-truth.md b/docs/source-of-truth.md index 57ab7e5..f38d54b 100644 --- a/docs/source-of-truth.md +++ b/docs/source-of-truth.md @@ -10,6 +10,7 @@ This repository has multiple layers. Knowing the authority for each layer preven | Docker shared baseline inputs | `default-network.yml`, `default-environment.env`, `secrets/stack-secrets.env` | Shared network/env material applied during compose rendering. | | Infrastructure inventory and reconciliation | Terraform under `infrastructure/terraform/` | Codified inventory of existing infrastructure and relationships, especially Proxmox VMs and selected Docker mirrors. | | Secret policy and inventory | `docs/security-secrets.md` + `secrets/inventory.json` + local secret files in `secrets/` | What secrets exist, where they are expected, and what automation should parse. | +| Host/device configuration bootstrap (emerging) | Ansible under `infrastructure/ansible/` | Gradual inventory/configuration layer for hosts/devices; validation-first at current stage. | ## Practical meaning @@ -29,6 +30,17 @@ Use Terraform when documenting/reconciling existing: Do **not** treat Terraform as a full replacement for Compose operations in this repo. + +### Ansible bootstrap decisions + +Use Ansible under `infrastructure/ansible/` to build inventory and configuration structure incrementally. + +At the current stage: + +- Do **not** treat Ansible as replacement authority for Docker runtime operations. +- Do **not** treat Ansible as replacement authority for Terraform inventory/reconciliation. +- NixOS remains outside Ansible authority unless explicitly adopted in a later phase. + ## Declared config vs observed/runtime state - **Declared config**: files in this repository (Compose, Terraform, docs). diff --git a/infrastructure/ansible/README.md b/infrastructure/ansible/README.md new file mode 100644 index 0000000..b73155d --- /dev/null +++ b/infrastructure/ansible/README.md @@ -0,0 +1,46 @@ +# Ansible Foundation (Phase 1) + +This directory provides a minimal Ansible bootstrap for this repository. + +## Purpose + +- Establish a maintainable inventory/configuration foundation for hosts and devices. +- Support gradual host onboarding and validation workflows. +- Keep boundaries clear with existing Compose and Terraform authorities. + +This is intentionally a **foundation stage**, not full production automation. + +## Boundaries + +- Docker runtime authority remains in Compose files and `services-up.sh`. +- Terraform remains the primary structured infrastructure inventory/reconciliation layer. +- Ansible here is a complementary configuration/inventory layer. +- NixOS and network gear management are not authoritative through Ansible yet. + +## Structure + +- `ansible.cfg` - local defaults for inventory, collections, and output behavior. +- `inventory/hosts.yml` - YAML inventory scaffold with starter groups. +- `inventory/group_vars/` - shared/group variables. +- `inventory/host_vars/` - per-host variables. +- `playbooks/ping.yml` - minimal syntax/connection test playbook. +- `collections/requirements.yml` - lightweight baseline collections. +- `roles/` - reserved for future incremental role adoption. + +## Basic commands + +Run from repository root: + +```bash +ansible --version +ansible-lint --version +ansible-galaxy collection install -r infrastructure/ansible/collections/requirements.yml -p infrastructure/ansible/collections +ansible-inventory -i infrastructure/ansible/inventory/hosts.yml --list +ansible-playbook -i infrastructure/ansible/inventory/hosts.yml infrastructure/ansible/playbooks/ping.yml --syntax-check +``` + +## Secrets and safety + +- Do not commit real credentials or private keys. +- Put sensitive per-host variables in local, untracked files or a future vault approach. +- Keep host and device entries factual; avoid speculative production entries. diff --git a/infrastructure/ansible/ansible.cfg b/infrastructure/ansible/ansible.cfg new file mode 100644 index 0000000..57525e5 --- /dev/null +++ b/infrastructure/ansible/ansible.cfg @@ -0,0 +1,9 @@ +[defaults] +inventory = ./inventory/hosts.yml +collections_path = ./collections +retry_files_enabled = False +stdout_callback = yaml +host_key_checking = True + +[inventory] +enable_plugins = yaml diff --git a/infrastructure/ansible/collections/requirements.yml b/infrastructure/ansible/collections/requirements.yml new file mode 100644 index 0000000..92a0244 --- /dev/null +++ b/infrastructure/ansible/collections/requirements.yml @@ -0,0 +1,4 @@ +--- +collections: + - name: ansible.posix + - name: community.general diff --git a/infrastructure/ansible/inventory/group_vars/all.yml b/infrastructure/ansible/inventory/group_vars/all.yml new file mode 100644 index 0000000..4d32fd1 --- /dev/null +++ b/infrastructure/ansible/inventory/group_vars/all.yml @@ -0,0 +1,14 @@ +--- +# Bootstrap defaults for the Ansible foundation in this repository. +# Keep secrets and environment-specific auth details out of version control. + +# Common interpreter hint for modern Linux hosts. Override per-host if needed. +ansible_python_interpreter: /usr/bin/python3 + +# Placeholders for future connection/auth settings: +# ansible_user: "" +# ansible_port: 22 +# ansible_ssh_private_key_file: "" + +# Add group-specific settings under inventory/group_vars/.yml +# and host-specific settings under inventory/host_vars/.yml. diff --git a/infrastructure/ansible/inventory/host_vars/.gitkeep b/infrastructure/ansible/inventory/host_vars/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/infrastructure/ansible/inventory/hosts.yml b/infrastructure/ansible/inventory/hosts.yml new file mode 100644 index 0000000..92eeb86 --- /dev/null +++ b/infrastructure/ansible/inventory/hosts.yml @@ -0,0 +1,17 @@ +--- +all: + children: + linux: + hosts: {} + network: + hosts: {} + virtualization: + hosts: {} + nixos: + hosts: {} + examples: + hosts: + example-managed-host: + ansible_host: example-host.local + ansible_connection: ssh + # Example only: replace/remove before real operations. diff --git a/infrastructure/ansible/playbooks/ping.yml b/infrastructure/ansible/playbooks/ping.yml new file mode 100644 index 0000000..486621f --- /dev/null +++ b/infrastructure/ansible/playbooks/ping.yml @@ -0,0 +1,7 @@ +--- +- name: Basic inventory and connectivity check + hosts: all + gather_facts: false + tasks: + - name: Ping managed hosts + ansible.builtin.ping: diff --git a/infrastructure/ansible/roles/.gitkeep b/infrastructure/ansible/roles/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/scripts/codex-maintenance.sh b/scripts/codex-maintenance.sh index a5cd11a..e7b82c3 100755 --- a/scripts/codex-maintenance.sh +++ b/scripts/codex-maintenance.sh @@ -41,7 +41,8 @@ dummy_value_for_key() { local key="$1" case "$key" in *EMAIL* ) echo "dummy@example.com" ;; - *USER*|*USERNAME* ) echo "dummy-user" ;; + *DB_USER* ) echo "dummyuser" ;; + *USERNAME*|*USER* ) echo "dummy-user" ;; *DOMAIN* ) echo "example.lan.ddnsgeek.com" ;; *TZ ) echo "Australia/Brisbane" ;; *URL* ) echo "https://example.lan.ddnsgeek.com" ;; @@ -49,7 +50,6 @@ dummy_value_for_key() { *PASSWORD*|*PASS*|*TOKEN*|*SECRET*|*KEY*|*JWT* ) echo "dummy-${key,,}" ;; *FINGERPRINT* ) echo "0000000000000000000000000000000000000000" ;; *DB_NAME* ) echo "dummydb" ;; - *DB_USER* ) echo "dummyuser" ;; *NAME* ) echo "dummy-name" ;; *ADDRESS* ) echo "dummy" ;; * ) echo "dummy-value" ;; @@ -120,3 +120,35 @@ reconcile_file_based_secrets echo "== Dummy secret reconciliation complete ==" echo "stack env: $STACK_ENV" jq -r '.file_based_secrets[].path' "$INVENTORY_JSON" | sed 's/^/file secret: /' + + +REPO_ROOT="${CODEX_REPO_DIR:-$PWD}" +ANSIBLE_DIR="$REPO_ROOT/infrastructure/ansible" +ANSIBLE_CONFIG="$ANSIBLE_DIR/ansible.cfg" +ANSIBLE_COLLECTIONS_REQ="$ANSIBLE_DIR/collections/requirements.yml" +ANSIBLE_INVENTORY="$ANSIBLE_DIR/inventory/hosts.yml" +ANSIBLE_PING_PLAYBOOK="$ANSIBLE_DIR/playbooks/ping.yml" + +if [[ -f "$ANSIBLE_COLLECTIONS_REQ" ]]; then + echo "== Refresh Ansible collections (bootstrap) ==" + ansible-galaxy collection install -r "$ANSIBLE_COLLECTIONS_REQ" -p "$ANSIBLE_DIR/collections" || true +fi + +if command -v ansible >/dev/null 2>&1; then + echo "== Ansible bootstrap validation ==" + ANSIBLE_CONFIG="$ANSIBLE_CONFIG" ansible --version | head -n 1 || true + + if command -v ansible-lint >/dev/null 2>&1; then + ansible-lint --version || true + fi + + if [[ -f "$ANSIBLE_INVENTORY" ]]; then + ANSIBLE_CONFIG="$ANSIBLE_CONFIG" \ + ansible-inventory -i "$ANSIBLE_INVENTORY" --list > /dev/null || true + fi + + if [[ -f "$ANSIBLE_PING_PLAYBOOK" && -f "$ANSIBLE_INVENTORY" ]]; then + ANSIBLE_CONFIG="$ANSIBLE_CONFIG" \ + ansible-playbook -i "$ANSIBLE_INVENTORY" "$ANSIBLE_PING_PLAYBOOK" --syntax-check || true + fi +fi diff --git a/scripts/codex-setup.sh b/scripts/codex-setup.sh index 8c90f8a..67ca9e8 100755 --- a/scripts/codex-setup.sh +++ b/scripts/codex-setup.sh @@ -104,7 +104,8 @@ dummy_value_for_key() { local key="$1" case "$key" in *EMAIL* ) echo "dummy@example.com" ;; - *USER*|*USERNAME* ) echo "dummy-user" ;; + *DB_USER* ) echo "dummyuser" ;; + *USERNAME*|*USER* ) echo "dummy-user" ;; *DOMAIN* ) echo "example.lan.ddnsgeek.com" ;; *TZ ) echo "Australia/Brisbane" ;; *URL* ) echo "https://example.lan.ddnsgeek.com" ;; @@ -112,7 +113,6 @@ dummy_value_for_key() { *PASSWORD*|*PASS*|*TOKEN*|*SECRET*|*KEY*|*JWT* ) echo "dummy-${key,,}" ;; *FINGERPRINT* ) echo "0000000000000000000000000000000000000000" ;; *DB_NAME* ) echo "dummydb" ;; - *DB_USER* ) echo "dummyuser" ;; *NAME* ) echo "dummy-name" ;; *ADDRESS* ) echo "dummy" ;; * ) echo "dummy-value" ;; @@ -152,6 +152,38 @@ ensure_dummy_secret_files() { render_dummy_stack_env ensure_dummy_secret_files + +ANSIBLE_DIR="$REPO_ROOT/infrastructure/ansible" +ANSIBLE_CONFIG="$ANSIBLE_DIR/ansible.cfg" +ANSIBLE_COLLECTIONS_REQ="$ANSIBLE_DIR/collections/requirements.yml" +ANSIBLE_INVENTORY="$ANSIBLE_DIR/inventory/hosts.yml" +ANSIBLE_PING_PLAYBOOK="$ANSIBLE_DIR/playbooks/ping.yml" + +if [[ -f "$ANSIBLE_COLLECTIONS_REQ" ]]; then + echo "== Ansible collections (bootstrap) ==" + ansible-galaxy collection install -r "$ANSIBLE_COLLECTIONS_REQ" -p "$ANSIBLE_DIR/collections" || true +fi + +if command -v ansible >/dev/null 2>&1; then + echo "== Ansible bootstrap validation ==" + ANSIBLE_CONFIG="$ANSIBLE_CONFIG" ansible --version | head -n 1 || true + + if command -v ansible-lint >/dev/null 2>&1; then + ansible-lint --version || true + else + echo "ansible-lint not available; skipping version check" + fi + + if [[ -f "$ANSIBLE_INVENTORY" ]]; then + ANSIBLE_CONFIG="$ANSIBLE_CONFIG" \ + ansible-inventory -i "$ANSIBLE_INVENTORY" --list > /dev/null || true + fi + + if [[ -f "$ANSIBLE_PING_PLAYBOOK" && -f "$ANSIBLE_INVENTORY" ]]; then + ANSIBLE_CONFIG="$ANSIBLE_CONFIG" \ + ansible-playbook -i "$ANSIBLE_INVENTORY" "$ANSIBLE_PING_PLAYBOOK" --syntax-check || true + fi +fi echo echo "== Installed versions ==" bash --version | head -n 1 || true