improve health checks across all nomad job specs

- traefik: TCP → HTTP check on /ping (enable ping entrypoint) - gitea: check path → /api/healthz - jellyfin: TCP → HTTP check on /health - glance: TCP → HTTP check on / - sonarr/prowlarr: check path / → /ping (×2 checks each) - ntfy/transfer/deluge/openreader/authelia/pgadmin: add name and port to existing checks - postgres: remove invalid TCP check (Connect-enabled service) - unifi: TCP → script check via curl (macvlan host isolation workaround)
ci: add Docker image pull validation job (Phase 2)
2026-05-26 20:12:23 +10:00 · 2026-04-19 18:06:17 +10:00
19 changed files with 118 additions and 54 deletions
--- a/.gitea/workflows/ci.yml
+++ b/.gitea/workflows/ci.yml
@@ -52,3 +52,39 @@ jobs:
            echo "==> $f"
            nomad job validate "$f"
          done
+
+  image-pull:
+    name: Docker image pull validation
+    runs-on: ubuntu-latest
+    # Only run on PRs that touch nomad job specs
+    if: github.event_name == 'pull_request'
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Pull changed images
+        run: |
+          git fetch origin ${{ github.base_ref }}
+          IMAGES=$(git diff origin/${{ github.base_ref }}...HEAD -- '*.nomad.hcl' \
+            | grep '^+\s*image\s*=' \
+            | grep -oP '"[^"]+:[^"]+"' \
+            | tr -d '"' \
+            | sort -u || true)
+
+          if [ -z "$IMAGES" ]; then
+            echo "No image changes detected, skipping pull."
+            exit 0
+          fi
+
+          FAILED=0
+          while IFS= read -r image; do
+            echo "==> Pulling $image"
+            if ! docker pull "$image"; then
+              echo "ERROR: Failed to pull $image"
+              FAILED=1
+            fi
+          done <<< "$IMAGES"
+
+          exit $FAILED
--- a/1-nixos-node/.terraform.lock.hcl
+++ b/1-nixos-node/.terraform.lock.hcl
@@ -2,32 +2,22 @@
 # Manual edits may be lost in future updates.

 provider "registry.terraform.io/hashicorp/null" {
-  version     = "3.2.4"
-  constraints = "3.2.4"
+  version     = "3.2.1"
+  constraints = "3.2.1"
  hashes = [
-    "h1:+Ag4hSb4qQjNtAS6gj2+gsGl7v0iB/Bif6zZZU8lXsw=",
-    "h1:127ts0CG8hFk1bHIfrBsKxcnt9bAYQCq3udWM+AACH8=",
-    "h1:1fD1DCOTkei4uxrLwszCkQapuZcw5sq4CatKFtRg58E=",
-    "h1:DskWuCIvJ4FBUpngJsiRCtVFiKZMQbT3WQgn9GBnFPc=",
-    "h1:L5V05xwp/Gto1leRryuesxjMfgZwjb7oool4WS1UEFQ=",
-    "h1:Oz/OcdizPzjor4pnGHNvcXURwyS6uDDXAZccg4R6iR4=",
-    "h1:TpiL9vwc23kFuJgQlFsgIlaI4WjAsX9H3LLzcZ36Yfs=",
-    "h1:VMNuSHZMkfsbrzvhpp6lzm2vWdmT/1vuUdW0x+Dsa0Q=",
-    "h1:dNVrmZwFvVPlL2FqTMDasI6mbDIr9pcn7tHexkZU9z8=",
-    "h1:hkf5w5B6q8e2A42ND2CjAvgvSN3puAosDmOJb3zCVQM=",
-    "h1:wTNrZnwQdOOT/TW9pa+7GgJeFK2OvTvDmx78VmUmZXM=",
-    "zh:59f6b52ab4ff35739647f9509ee6d93d7c032985d9f8c6237d1f8a59471bbbe2",
+    "h1:FbGfc+muBsC17Ohy5g806iuI1hQc4SIexpYCrQHQd8w=",
+    "zh:58ed64389620cc7b82f01332e27723856422820cfd302e304b5f6c3436fb9840",
+    "zh:62a5cc82c3b2ddef7ef3a6f2fedb7b9b3deff4ab7b414938b08e51d6e8be87cb",
+    "zh:63cff4de03af983175a7e37e52d4bd89d990be256b16b5c7f919aff5ad485aa5",
+    "zh:74cb22c6700e48486b7cabefa10b33b801dfcab56f1a6ac9b6624531f3d36ea3",
    "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
-    "zh:795c897119ff082133150121d39ff26cb5f89a730a2c8c26f3a9c1abf81a9c43",
-    "zh:7b9c7b16f118fbc2b05a983817b8ce2f86df125857966ad356353baf4bff5c0a",
-    "zh:85e33ab43e0e1726e5f97a874b8e24820b6565ff8076523cc2922ba671492991",
-    "zh:9d32ac3619cfc93eb3c4f423492a8e0f79db05fec58e449dee9b2d5873d5f69f",
-    "zh:9e15c3c9dd8e0d1e3731841d44c34571b6c97f5b95e8296a45318b94e5287a6e",
-    "zh:b4c2ab35d1b7696c30b64bf2c0f3a62329107bd1a9121ce70683dec58af19615",
-    "zh:c43723e8cc65bcdf5e0c92581dcbbdcbdcf18b8d2037406a5f2033b1e22de442",
-    "zh:ceb5495d9c31bfb299d246ab333f08c7fb0d67a4f82681fbf47f2a21c3e11ab5",
-    "zh:e171026b3659305c558d9804062762d168f50ba02b88b231d20ec99578a6233f",
-    "zh:ed0fe2acdb61330b01841fa790be00ec6beaac91d41f311fb8254f74eb6a711f",
+    "zh:79e553aff77f1cfa9012a2218b8238dd672ea5e1b2924775ac9ac24d2a75c238",
+    "zh:a1e06ddda0b5ac48f7e7c7d59e1ab5a4073bbcf876c73c0299e4610ed53859dc",
+    "zh:c37a97090f1a82222925d45d84483b2aa702ef7ab66532af6cbcfb567818b970",
+    "zh:e4453fbebf90c53ca3323a92e7ca0f9961427d2f0ce0d2b65523cc04d5d999c2",
+    "zh:e80a746921946d8b6761e77305b752ad188da60688cfd2059322875d363be5f5",
+    "zh:fbdb892d9822ed0e4cb60f2fedbdbb556e4da0d88d3b942ae963ed6ff091e48f",
+    "zh:fca01a623d90d0cad0843102f9b8b9fe0d3ff8244593bd817f126582b52dd694",
  ]
 }

--- a/1-nixos-node/node.tf
+++ b/1-nixos-node/node.tf
@@ -2,7 +2,7 @@ terraform {
  required_providers {
    null = {
      source  = "hashicorp/null"
-      version = "3.2.4"
+      version = "3.2.1"
    }
    template = {
      source  = "hashicorp/template"
--- a/2-nomad-config/1-data/pgadmin.nomad.hcl
+++ b/2-nomad-config/1-data/pgadmin.nomad.hcl
@@ -58,8 +58,10 @@ job "pgadmin" {
        ]

        check {
+          name     = "alive"
          type     = "http"
          path     = "/"
+          port     = "http"
          interval = "10s"
          timeout  = "2s"
        }
--- a/2-nomad-config/1-data/postgres.nomad.hcl
+++ b/2-nomad-config/1-data/postgres.nomad.hcl
@@ -7,6 +7,10 @@ job "postgres" {
      connect {
        sidecar_service {}
      }
+
+      # Note: TCP checks are not valid for Connect-enabled services (runs through
+      # Envoy sidecar). Postgres is a single-writer DB that we never canary, so
+      # observable health via Consul is lower priority than other services.
    }

    task "postgres" {
--- a/2-nomad-config/2-ingress/authelia.nomad.hcl
+++ b/2-nomad-config/2-ingress/authelia.nomad.hcl
@@ -33,8 +33,10 @@ job "authelia" {
      }

      check {
+        name     = "alive"
        type     = "http"
        path     = "/health"
+        port     = "http"
        interval = "10s"
        timeout  = "2s"
      }
--- a/2-nomad-config/2-ingress/traefik.nomad.hcl
+++ b/2-nomad-config/2-ingress/traefik.nomad.hcl
@@ -35,7 +35,8 @@ job "traefik" {

      check {
        name     = "alive"
-        type     = "tcp"
+        type     = "http"
+        path     = "/ping"
        port     = "api"
        interval = "10s"
        timeout  = "2s"
@@ -85,6 +86,9 @@ api:
  dashboard: true
  insecure: true

+ping:
+  entryPoint: traefik
+
 providers:
  file:
    directory: "/etc/traefik/configs/"
--- a/2-nomad-config/act-runner.nomad.hcl
+++ b/2-nomad-config/act-runner.nomad.hcl
@@ -29,7 +29,6 @@ job "act-runner" {

      env = {
        GITEA_INSTANCE_URL = "https://gitea-1ef0bea6b75a4fd3e9393a9f7f7e4b02.othrayte.one"
-        CONFIG_FILE        = "/secrets/runner-config.yml"
      }

      # Required SOPS key:
@@ -43,19 +42,6 @@ EOF
        env         = true
      }

-      # Limit which images/labels the runner will accept so it doesn't pick up
-      # unrelated workloads if more runners are added later.
-      template {
-        data        = <<EOF
-runner:
-  labels:
-    - "ubuntu-latest:docker://node:20-bookworm"
-    - "ubuntu-22.04:docker://node:20-bookworm"
-    - "ubuntu-24.04:docker://node:20-bookworm"
-EOF
-        destination = "secrets/runner-config.yml"
-      }
-
      resources {
        cpu        = 200
        memory     = 256
--- a/2-nomad-config/deluge.nomad.hcl
+++ b/2-nomad-config/deluge.nomad.hcl
@@ -54,8 +54,10 @@ EOH
      ]

      check {
+        name     = "alive"
        type     = "http"
        path     = "/"
+        port     = "http"
        interval = "10s"
        timeout  = "2s"
      }
@@ -78,8 +80,10 @@ EOH
      }

      check {
+        name     = "alive"
        type     = "http"
        path     = "/"
+        port     = "http"
        interval = "10s"
        timeout  = "2s"
      }
--- a/2-nomad-config/gitea.nomad.hcl
+++ b/2-nomad-config/gitea.nomad.hcl
@@ -32,8 +32,10 @@ job "gitea" {
      ]

      check {
+        name     = "alive"
        type     = "http"
-        path     = "/"
+        path     = "/api/healthz"
+        port     = "http"
        interval = "10s"
        timeout  = "2s"
      }
--- a/2-nomad-config/glance.nomad.hcl
+++ b/2-nomad-config/glance.nomad.hcl
@@ -12,7 +12,7 @@ job "glance" {
      driver = "docker"

      config {
-        image = "glanceapp/glance:latest"
+        image = "glanceapp/glance:v0.7.8"
        ports = ["http"]
        volumes = [
          "local/glance.yml:/app/config/glance.yml",
@@ -30,7 +30,8 @@ job "glance" {

        check {
          name     = "alive"
-          type     = "tcp"
+          type     = "http"
+          path     = "/"
          port     = "http"
          interval = "10s"
          timeout  = "2s"
--- a/2-nomad-config/jellyfin.nomad.hcl
+++ b/2-nomad-config/jellyfin.nomad.hcl
@@ -28,10 +28,11 @@ job "jellyfin" {

        check {
          name     = "alive"
-          type     = "tcp"
+          type     = "http"
+          path     = "/health"
          port     = "http"
          interval = "10s"
-          timeout  = "2s"
+          timeout  = "5s"
        }
      }

--- a/2-nomad-config/ntfy.nomad.hcl
+++ b/2-nomad-config/ntfy.nomad.hcl
@@ -33,8 +33,10 @@ job "ntfy" {
      ]

      check {
+        name     = "alive"
        type     = "http"
        path     = "/healthz"
+        port     = "http"
        interval = "10s"
        timeout  = "2s"
      }
--- a/2-nomad-config/openreader.nomad.hcl
+++ b/2-nomad-config/openreader.nomad.hcl
@@ -31,8 +31,10 @@ job "openreader" {
      ]

      check {
+        name     = "alive"
        type     = "http"
        path     = "/"
+        port     = "http"
        interval = "10s"
        timeout  = "2s"
      }
@@ -48,8 +50,10 @@ job "openreader" {
      }

      check {
+        name     = "alive"
        type     = "http"
        path     = "/"
+        port     = "http"
        interval = "10s"
        timeout  = "2s"
      }
--- a/2-nomad-config/prowlarr.nomad.hcl
+++ b/2-nomad-config/prowlarr.nomad.hcl
@@ -34,8 +34,9 @@ job "prowlarr" {
      ]

      check {
+        name     = "alive"
        type     = "http"
-        path     = "/"
+        path     = "/ping"
        interval = "10s"
        timeout  = "2s"
      }
@@ -51,8 +52,9 @@ job "prowlarr" {
      }

      check {
+        name     = "alive"
        type     = "http"
-        path     = "/"
+        path     = "/ping"
        interval = "10s"
        timeout  = "2s"
      }
--- a/2-nomad-config/sonarr.nomad.hcl
+++ b/2-nomad-config/sonarr.nomad.hcl
@@ -37,8 +37,9 @@ job "sonarr" {
      ]

      check {
+        name     = "alive"
        type     = "http"
-        path     = "/"
+        path     = "/ping"
        interval = "10s"
        timeout  = "2s"
      }
@@ -54,8 +55,9 @@ job "sonarr" {
      }

      check {
+        name     = "alive"
        type     = "http"
-        path     = "/"
+        path     = "/ping"
        interval = "10s"
        timeout  = "2s"
      }
--- a/2-nomad-config/transfer.nomad.hcl
+++ b/2-nomad-config/transfer.nomad.hcl
@@ -16,8 +16,10 @@ job "transfer" {
      ]

      check {
+        name     = "alive"
        type     = "http"
        path     = "/"
+        port     = "http"
        interval = "10s"
        timeout  = "2s"
      }
--- a/2-nomad-config/unifi.nomad.hcl
+++ b/2-nomad-config/unifi.nomad.hcl
@@ -21,6 +21,26 @@ job "unifi-network" {
        UNIFI_STDOUT       = "true"
      }

+      # Register in Consul so Traefik and health checks can find it.
+      # address_mode=driver uses the macvlan IP (192.168.1.50) rather than the host IP.
+      service {
+        name         = "unifi-network"
+        port         = 8443
+        address_mode = "driver"
+
+        # TCP/HTTP checks from the Consul agent can't reach the macvlan IP (host↔macvlan
+        # isolation). Use a script check instead — it runs inside the container via
+        # docker exec and connects to localhost:8443 directly.
+        check {
+          name     = "alive"
+          type     = "script"
+          command  = "/usr/bin/curl"
+          args     = ["-sk", "--max-time", "5", "-o", "/dev/null", "https://localhost:8443"]
+          interval = "30s"
+          timeout  = "10s"
+        }
+      }
+
      volume_mount {
        volume      = "unraid_appdata_unifi_network"
        destination = "/unifi" # Expected root directory (contains data, log, cert subdirs)
--- a/cicd-plan.md
+++ b/cicd-plan.md
@@ -188,8 +188,8 @@ Most jobs already have Consul health checks — these can use `health_check = "c
 | frigate    | ✅            | ✅ `single-node-writer` | ⚠️ same — rolling                                                                 |
 | glance     | ✅            | no                      | ✅ yes                                                                            |
 | transfer   | ✅            | ✅ `single-node-writer` | ⚠️ rolling                                                                        |
-| openreader | ❌            | ✅ `single-node-writer` | ⚠️ add check first, then rolling                                                  |
-| unifi      | ❌            | ✅ `single-node-writer` | ⚠️ add check first, then rolling                                                  |
+| openreader | ✅ `/`        | ✅ `single-node-writer` | ⚠️ rolling                                                                        |
+| unifi      | ✅ script     | ✅ `single-node-writer` | ⚠️ rolling                                                                        |
 | traefik    | (ingress)     | ✅                      | ⚠️ rolling — downtime risk, promote quickly                                       |
 | authelia   | (ingress)     | ✅                      | ✅ stateless config, canary fine                                                  |
 | renovate   | batch job     | n/a                     | n/a — no deployment model                                                         |
@@ -296,10 +296,10 @@ exit 1
 - [x] **Phase 1a**: Create `act-runner.nomad.hcl` + Terraform wrapper, register runner token in Gitea, get a hello-world workflow green
 - [x] **Phase 1b**: Add `terraform fmt` + `terraform validate -backend=false` workflow — no secrets needed
 - [x] **Phase 1c**: Add Nomad validate step — add `NOMAD_ADDR` + read-only `NOMAD_TOKEN` to Gitea secrets
- [ ] **Phase 2**: Add image pull validation step to the workflow
+- [x] **Phase 2**: Add image pull validation step to the workflow
 - [ ] **Phase 3a**: Add `update` stanzas to ntfy and glance (simplest, no volume conflict)
- [ ] **Phase 3b**: Add rolling `update` stanzas to remaining service jobs (jellyfin, sonarr, etc.)
- [ ] **Phase 3c**: Add health checks to openreader and unifi before adding update stanzas
+- [ ] **Phase 3b**: Add rolling `update` stanzas to remaining service jobs (jellyfin, sonarr, prowlarr, deluge, gitea, immich, transfer, frigate, openreader, unifi, authelia, traefik)
+- [x] **Phase 3c**: Add health checks to openreader and unifi before adding update stanzas
 - [ ] **Phase 4a**: Add on-push workflow that runs `terraform apply -auto-approve` using full credential set
 - [ ] **Phase 4b**: Add deployment promotion/revert polling script
 - [ ] **Phase 4c**: Wire ntfy notifications for promote/revert outcomes
Author	SHA1	Message	Date
Adrian Cowan	e695485353	improve health checks across all nomad job specs All checks were successful CI / Terraform fmt + validate (pull_request) Successful in 27s Details CI / Nomad job spec validate (pull_request) Successful in 22s Details CI / Docker image pull validation (pull_request) Successful in 16s Details CI / Terraform fmt + validate (push) Successful in 23s Details CI / Nomad job spec validate (push) Successful in 22s Details CI / Docker image pull validation (push) Has been skipped Details - traefik: TCP → HTTP check on /ping (enable ping entrypoint) - gitea: check path → /api/healthz - jellyfin: TCP → HTTP check on /health - glance: TCP → HTTP check on / - sonarr/prowlarr: check path / → /ping (×2 checks each) - ntfy/transfer/deluge/openreader/authelia/pgadmin: add name and port to existing checks - postgres: remove invalid TCP check (Connect-enabled service) - unifi: TCP → script check via curl (macvlan host isolation workaround)	2026-05-26 20:12:23 +10:00
Adrian Cowan	8e32d00d90	ci: add Docker image pull validation job (Phase 2) All checks were successful CI / Docker image pull validation (pull_request) Successful in 14s Details CI / Nomad job spec validate (push) Successful in 30s Details CI / Docker image pull validation (push) Has been skipped Details CI / Terraform fmt + validate (pull_request) Successful in 20s Details CI / Nomad job spec validate (pull_request) Successful in 24s Details CI / Terraform fmt + validate (push) Successful in 21s Details - Add image-pull job to .gitea/workflows/ci.yml that detects image changes in *.nomad.hcl files on PRs and pulls each changed image - Remove act-runner labels config — default runner already uses docker.gitea.com/runner-images:ubuntu-latest which has docker CLI - Remove CONFIG_FILE env var from act-runner (no longer needed) - Mark Phase 2 complete in cicd-plan.md	2026-04-19 18:06:17 +10:00