Complete migration now that the changes are applied.

This should make finding things easier

0-hardware/cameras.md

@@ -0,0 +1,5 @@
+| Location  | IP            |
+| --------- | ------------- |
+| Doorbell  | 192.168.1.108 |
+| Side Gate | 192.168.1.109 |
+| Ceiling   | 192.168.1.110 |

0-hardware/servernames.md

@@ -1,33 +1,40 @@
 # Server Names
+
 From https://namingschemes.com/Hitchhikers_Guide_Planets.  
 Was [google sheet](https://docs.google.com/spreadsheets/d/1aLb14nme9gprYzWOEo5IadG1vKEtf8VQk0tkCBGz9KA/edit#gid=0).
 
-
-| Name                   | Hostname                | Use                                                           |
-| ---------------------- | ----------------------- | ------------------------------------------------------------- |
-| Antares                | antares-win             | Win11 PC                                                      |
-| Barteldan              | barteldan-linux         | Fedora Linux PC                                               |
-| Betelgeuse Seven       | betelgeuse-seven-unraid | Unraid Storage Host                                           |
-| Bethselamin            |                         |                                                               |
-| Damogran               |                         |                                                               |
-| Eadrax                 |                         |                                                               |
-| Earth                  |                         | Home Assistant Server?                                        |
-| Eroticon 6             |                         |                                                               |
-| Frogstar A & B         |                         |                                                               |
-| Golgafrincham          |                         |                                                               |
-| Han Wavel              |                         |                                                               |
-| Jaglan Beta            |                         | Compute Cluster Hosts                                         |
-|  - Moon 1              | jaglan-beta-m01         | NUC Cluster Host<br>(Intel \| N5105 [4/4], 16GB ram, 1TB SSD) |
-|  - Moon 42 (Test moon) | jaglan-beta-m42         | Test server (VM)                                              |
-| Krikkit                |                         |                                                               |
-| Lamuella               |                         |                                                               |
-| Magrathea              |                         |                                                               |
-| Maximegalon            |                         |                                                               |
-| Oglarun                |                         |                                                               |
-| Rupert (Persephone)    |                         |                                                               |
-| Saqua Pilia Hensha     |                         |                                                               |
-| Ursa Minor Beta        |                         |                                                               |
-| Viltvodl VI            |                         |                                                               |
-| Vogosphere             |                         |                                                               |
-| Ysolldins              |                         |                                                               |
-|                        |                         |                                                               |
+| Name                  | Hostname                | Use                                                                     |
+| --------------------- | ----------------------- | ----------------------------------------------------------------------- |
+| Antares               | antares-win             | Win11 PC                                                                |
+| Barteldan             | barteldan-linux         | Fedora Linux PC                                                         |
+| Betelgeuse Seven      | betelgeuse-seven-unraid | Unraid Storage Host                                                     |
+| Bethselamin           |                         |                                                                         |
+| Damogran              |                         |                                                                         |
+| Eadrax                |                         |                                                                         |
+| Earth                 |                         | Home Assistant Server?                                                  |
+| Eroticon 6            |                         |                                                                         |
+| Frogstar A & B        |                         |                                                                         |
+| Golgafrincham         |                         |                                                                         |
+| Han Wavel             |                         |                                                                         |
+| Jaglan Beta           |                         | Compute Cluster Hosts                                                   |
+| - Moon 1              | jaglan-beta-m01         | NUC Cluster Host<br>(Intel \| N5105 [4/4], 16GB, 1TB SSD)               |
+| - Moon 2              | jaglan-beta-m02         | LattePanda IOTA Cluster Host<br>(Intel \| N150 [4/4], 16GB, 129GB eMMC) |
+| - Moon 3              | jaglan-beta-m03         | LattePanda IOTA Cluster Host<br>(Intel \| N150 [4/4], 16GB, 129GB eMMC) |
+| - Moon 4              | jaglan-beta-m04         | LattePanda IOTA Cluster Host<br>(Intel \| N150 [4/4], 16GB, 129GB eMMC) |
+| - Moon 5              | jaglan-beta-m05         | LattePanda IOTA Cluster Host<br>(Intel \| N150 [4/4], 16GB, 129GB eMMC) |
+| - Moon 20             | jaglan-beta-m20         | VM on unraid                                                            |
+| - Moon 21             | jaglan-beta-m21         | VM on unraid                                                            |
+| - Moon 22             | jaglan-beta-m22         | VM on unraid                                                            |
+| - Moon 42 (Test moon) | jaglan-beta-m42         | Test server (VM)                                                        |
+| Krikkit               |                         |                                                                         |
+| Lamuella              |                         |                                                                         |
+| Magrathea             |                         |                                                                         |
+| Maximegalon           |                         |                                                                         |
+| Oglarun               |                         |                                                                         |
+| Rupert (Persephone)   |                         |                                                                         |
+| Saqua Pilia Hensha    |                         |                                                                         |
+| Ursa Minor Beta       |                         |                                                                         |
+| Viltvodl VI           |                         |                                                                         |
+| Vogosphere            |                         |                                                                         |
+| Ysolldins             |                         |                                                                         |
+|                       |                         |                                                                         |

1-nixos-node/0-install-nixos.md

@@ -1,13 +1,45 @@
-Follow steps at https://nixos.org/manual/nixos/stable/#sec-installation-manual-summary
-
-Note: run `lsblk` to determine correct device to install to.
-
-Set the hostname when editing `/mnt/etc/nixos/configuration.nix` before the installation:
-`networking.hostName = "jaglan-beta-mNN";`
-
-Once `nixos-install` is complete and the root password is set and saved edit the `/etc/nixos/configuration.nix` again to enable ssh and run `nixos-rebuild switch`
+Determine the correct device to install to
 
+```sh
+lsblk
 ```
-services.openssh.enable = true;
-services.openssh.settings.PermitRootLogin = "yes";
+
+Steps based on https://nixos.org/manual/nixos/stable/#sec-installation-manual-summary for UEFI
+
+```sh
+parted /dev/mmcblk0 -- mklabel gpt
+# If there is an existing OS you will have to type yes to proceed
+parted /dev/mmcblk0 -- mkpart root ext4 512MB -8GB
+parted /dev/mmcblk0 -- mkpart swap linux-swap -8GB 100%
+parted /dev/mmcblk0 -- mkpart ESP fat32 1MB 512MB
+parted /dev/mmcblk0 -- set 3 esp on
+
+mkfs.ext4 -L nixos /dev/mmcblk0p1
+
+mkswap -L swap /dev/mmcblk0p2
+swapon /dev/mmcblk0p2
+mkfs.fat -F 32 -n boot /dev/mmcblk0p3
+mount /dev/disk/by-label/nixos /mnt
+mkdir -p /mnt/boot
+mount -o umask=077 /dev/disk/by-label/boot /mnt/boot
+nixos-generate-config --root /mnt
+nano /mnt/etc/nixos/configuration.nix
+# Set hostname networking.hostName = "jaglan-beta-mNN";
+nixos-install
+# Set the root password
+reboot
+nano /etc/nixos/configuration.nix
+# Enable ssh access
+# services.openssh.enable = true;
+# services.openssh.settings.PermitRootLogin = "yes";
+nixos-rebuild switch
+```
+
+If starting from older nixos, upgrade the OS
+
+```sh
+nix-channel --list
+nix-channel --add https://channels.nixos.org/nixos-25.05 nixos
+nix-channel --list
+nixos-rebuild switch --upgrade
 ```

1-nixos-node/1-apply-changes.md

@@ -0,0 +1,8 @@
+Avoid applying change to multiple hosts at once as this could take done the cluster, instead apply to each on one at a time
+terraform apply -target='null_resource.deploy_nixos["jaglan-beta-m02"]'
+terraform apply -target='null_resource.deploy_nixos["jaglan-beta-m03"]'
+terraform apply -target='null_resource.deploy_nixos["jaglan-beta-m04"]'
+terraform apply -target='null_resource.deploy_nixos["jaglan-beta-m05"]'
+terraform apply -target='null_resource.deploy_nixos["jaglan-beta-m20"]'
+terraform apply -target='null_resource.deploy_nixos["jaglan-beta-m21"]'
+terraform apply -target='null_resource.deploy_nixos["jaglan-beta-m22"]'

1-nixos-node/configuration.nix

@@ -46,6 +46,10 @@
         };
         client = {
           enabled = true;
+          preferred_address_family = "ipv4";
+%{if cpu_total_compute != null ~}
+          cpu_total_compute = ${cpu_total_compute};
+%{endif ~}
           host_volume = {
 %{ for volume in host_volumes ~}
             ${volume} = {
@@ -68,6 +72,7 @@
       webUi = true;
       interface.bind = "${bind_interface}";
       interface.advertise = "${bind_interface}";
+      forceAddrFamily = "ipv4";
       extraConfig  = {
         client_addr = "{{ GetPrivateInterfaces | exclude \"type\" \"ipv6\" | join \"address\" \" \" }} {{ GetAllInterfaces | include \"flags\" \"loopback\" | join \"address\" \" \" }}";
 %{if bootstrap ~}
@@ -76,7 +81,13 @@
         server = true;
         retry_join = [
           "jaglan-beta-m01"
+          "jaglan-beta-m02"
+          "jaglan-beta-m03"
+          "jaglan-beta-m04"
+          "jaglan-beta-m05"
           "jaglan-beta-m20"
+          "jaglan-beta-m21"
+          "jaglan-beta-m22"
         ];
         datacenter = "jaglan-beta";
         connect.enabled = true;
@@ -99,7 +110,7 @@
   ];
 
   # Open ports in the firewall. 80/443 are for HTTP/HTTPS (terraform), 464X are the default ports for Nomad, 830X are the default ports for Consul.
-  networking.firewall.allowedTCPPorts = [ 80 443 8081 4646 4647 4648 8300 8301 ];
+  networking.firewall.allowedTCPPorts = [ 80 443 8081 4646 4647 4648 8300 8301 8500 ];
   networking.firewall.allowedUDPPorts = [ 8301 ];
 
   # Copy the NixOS configuration file and link it from the resulting system

1-nixos-node/node.tf

@@ -16,21 +16,23 @@ provider "template" {}
 variable "nodes" {
   description = "Map of nodes with host, password, bind interface, and host volumes"
   type = map(object({
-    host           = string
-    password       = string
-    bind_interface = string
-    bootstrap      = optional(bool, false) # Optional field for bootstrap nodes
-    host_volumes   = list(string)
+    host              = string
+    password          = string
+    bind_interface    = string
+    bootstrap         = optional(bool, false)  # Optional field for bootstrap nodes
+    cpu_total_compute = optional(number, null) # Optional field for CPU total compute
+    host_volumes      = list(string)
   }))
 }
 
 locals {
   config_files = { for k, v in var.nodes :
     k => templatefile("${path.module}/configuration.nix", {
-      hostname       = v.host
-      bind_interface = v.bind_interface
-      bootstrap      = v.bootstrap
-      host_volumes   = v.host_volumes
+      hostname          = v.host
+      bind_interface    = v.bind_interface
+      bootstrap         = v.bootstrap
+      cpu_total_compute = v.cpu_total_compute
+      host_volumes      = v.host_volumes
     })
   }
 }

1-nixos-node/terraform.tfstate

@@ -1,7 +1,7 @@
 {
   "version": 4,
-  "terraform_version": "1.11.4",
-  "serial": 251,
+  "terraform_version": "1.13.0",
+  "serial": 325,
   "lineage": "db7dcf21-a255-0ec4-c8b8-d4a7559b3768",
   "outputs": {},
   "resources": [
@@ -12,26 +12,88 @@
       "provider": "provider[\"registry.terraform.io/hashicorp/null\"]",
       "instances": [
         {
-          "index_key": "jaglan-beta-m01",
+          "index_key": "jaglan-beta-m02",
           "schema_version": 0,
           "attributes": {
-            "id": "6048275408611354621",
+            "id": "4799650720916837801",
             "triggers": {
-              "configuration_content": "{ config, lib, pkgs, ... }:\n{\n  imports =\n    [ # Include the results of the hardware scan.\n      ./hardware-configuration.nix\n    ];\n\n  nixpkgs.config.allowUnfree = true;\n\n  # Use the systemd-boot EFI boot loader.\n  boot.loader.systemd-boot.enable = true;\n  boot.loader.efi.canTouchEfiVariables = true;\n\n  networking.hostName = \"jaglan-beta-m01\"; # Define your hostname.\n\n  time.timeZone = \"Australia/Melbourne\";\n\n  # List packages installed in system profile. To search, run:\n  # $ nix search wget\n  # environment.systemPackages = with pkgs; [\n  #   vim # Do not forget to add an editor to edit configuration.nix! The Nano editor is also installed by default.\n  #   wget\n  # ];\n\n  # Some programs need SUID wrappers, can be configured further or are\n  # started in user sessions.\n  # programs.mtr.enable = true;\n  # programs.gnupg.agent = {\n  #   enable = true;\n  #   enableSSHSupport = true;\n  # };\n\n  # List services that you want to enable:\n  services = {\n    nomad = {\n      enable = true;\n      enableDocker = true;\n      dropPrivileges = false;\n      settings = {\n        datacenter = \"jaglan-beta\";\n        server = {\n          enabled = true;\n          bootstrap_expect = 1;\n        };\n        client = {\n          enabled = true;\n          host_volume = {\n            traefik = {\n              path = \"/opt/traefik\";\n              read_only = false;\n            };\n            postgres = {\n              path = \"/opt/postgres\";\n              read_only = false;\n            };\n          };\n          cni_path = \"${pkgs.cni-plugins}/bin\";\n        };\n        plugin.docker.config.allow_privileged = true;\n      };\n      extraPackages = with pkgs; [\n        cni-plugins\n        consul\n      ];\n    };\n    consul = {\n      enable = true;\n      webUi = true;\n      interface.bind = \"eno1\";\n      interface.advertise = \"eno1\";\n      extraConfig  = {\n        client_addr = \"{{ GetPrivateInterfaces | exclude \\\"type\\\" \\\"ipv6\\\" | join \\\"address\\\" \\\" \\\" }} {{ GetAllInterfaces | include \\\"flags\\\" \\\"loopback\\\" | join \\\"address\\\" \\\" \\\" }}\";\n        bootstrap_expect = 1;\n        server = true;\n        retry_join = [\n          \"jaglan-beta-m01\"\n          \"jaglan-beta-m20\"\n        ];\n        datacenter = \"jaglan-beta\";\n        connect.enabled = true;\n        ports.grpc = 8502;\n      };\n    };\n    openssh = {\n      enable = true;\n      settings.PermitRootLogin = \"yes\";\n    };\n  };\n\n  systemd.tmpfiles.rules = [\n    # Fix issue where nomad needs alloc_mounts to be writable\n    \"d /var/lib/alloc_mounts 0755 root root -\"\n    # Create a directory for traefik to store its data\n    \"d /opt/traefik 0755 root root -\"\n    # Create a directory for postgres to store its data\n    \"d /opt/postgres 0755 root root -\"\n  ];\n\n  # Open ports in the firewall. 80/443 are for HTTP/HTTPS (terraform), 464X are the default ports for Nomad, 830X are the default ports for Consul.\n  networking.firewall.allowedTCPPorts = [ 80 443 8081 4646 4647 4648 8300 8301 ];\n  networking.firewall.allowedUDPPorts = [ 8301 ];\n\n  # Copy the NixOS configuration file and link it from the resulting system\n  # (/run/current-system/configuration.nix). This is useful in case you\n  # accidentally delete configuration.nix.\n  system.copySystemConfiguration = true;\n \n  # Defines the initial NixOS version for compatibility with older application data.\n  # Do NOT change this value after installation without careful consideration.\n  system.stateVersion = \"24.11\"; # Did you read the comment?\n}\n"
+              "configuration_content": "{ config, lib, pkgs, ... }:\n{\n  imports =\n    [ # Include the results of the hardware scan.\n      ./hardware-configuration.nix\n    ];\n\n  nixpkgs.config.allowUnfree = true;\n\n  # Use the systemd-boot EFI boot loader.\n  boot.loader.systemd-boot.enable = true;\n  boot.loader.efi.canTouchEfiVariables = true;\n\n  networking.hostName = \"jaglan-beta-m02\"; # Define your hostname.\n\n  time.timeZone = \"Australia/Melbourne\";\n\n  # List packages installed in system profile. To search, run:\n  # $ nix search wget\n  # environment.systemPackages = with pkgs; [\n  #   vim # Do not forget to add an editor to edit configuration.nix! The Nano editor is also installed by default.\n  #   wget\n  # ];\n\n  # Some programs need SUID wrappers, can be configured further or are\n  # started in user sessions.\n  # programs.mtr.enable = true;\n  # programs.gnupg.agent = {\n  #   enable = true;\n  #   enableSSHSupport = true;\n  # };\n\n  # List services that you want to enable:\n  services = {\n    nomad = {\n      enable = true;\n      enableDocker = true;\n      dropPrivileges = false;\n      settings = {\n        datacenter = \"jaglan-beta\";\n        server = {\n          enabled = true;\n        };\n        client = {\n          enabled = true;\n          preferred_address_family = \"ipv4\";\n          cpu_total_compute = 7200;\n          host_volume = {\n          };\n          cni_path = \"${pkgs.cni-plugins}/bin\";\n        };\n        plugin.docker.config.allow_privileged = true;\n      };\n      extraPackages = with pkgs; [\n        cni-plugins\n        consul\n      ];\n    };\n    consul = {\n      enable = true;\n      webUi = true;\n      interface.bind = \"enp1s0\";\n      interface.advertise = \"enp1s0\";\n      forceAddrFamily = \"ipv4\";\n      extraConfig  = {\n        client_addr = \"{{ GetPrivateInterfaces | exclude \\\"type\\\" \\\"ipv6\\\" | join \\\"address\\\" \\\" \\\" }} {{ GetAllInterfaces | include \\\"flags\\\" \\\"loopback\\\" | join \\\"address\\\" \\\" \\\" }}\";\n        server = true;\n        retry_join = [\n          \"jaglan-beta-m01\"\n          \"jaglan-beta-m02\"\n          \"jaglan-beta-m03\"\n          \"jaglan-beta-m04\"\n          \"jaglan-beta-m05\"\n          \"jaglan-beta-m20\"\n          \"jaglan-beta-m21\"\n          \"jaglan-beta-m22\"\n        ];\n        datacenter = \"jaglan-beta\";\n        connect.enabled = true;\n        ports.grpc = 8502;\n      };\n    };\n    openssh = {\n      enable = true;\n      settings.PermitRootLogin = \"yes\";\n    };\n  };\n\n  systemd.tmpfiles.rules = [\n    # Fix issue where nomad needs alloc_mounts to be writable\n    \"d /var/lib/alloc_mounts 0755 root root -\"\n  ];\n\n  # Open ports in the firewall. 80/443 are for HTTP/HTTPS (terraform), 464X are the default ports for Nomad, 830X are the default ports for Consul.\n  networking.firewall.allowedTCPPorts = [ 80 443 8081 4646 4647 4648 8300 8301 8500 ];\n  networking.firewall.allowedUDPPorts = [ 8301 ];\n\n  # Copy the NixOS configuration file and link it from the resulting system\n  # (/run/current-system/configuration.nix). This is useful in case you\n  # accidentally delete configuration.nix.\n  system.copySystemConfiguration = true;\n \n  # Defines the initial NixOS version for compatibility with older application data.\n  # Do NOT change this value after installation without careful consideration.\n  system.stateVersion = \"24.11\"; # Did you read the comment?\n}\n"
             }
           },
-          "sensitive_attributes": []
+          "sensitive_attributes": [],
+          "identity_schema_version": 0
+        },
+        {
+          "index_key": "jaglan-beta-m03",
+          "schema_version": 0,
+          "attributes": {
+            "id": "3940670083197547320",
+            "triggers": {
+              "configuration_content": "{ config, lib, pkgs, ... }:\n{\n  imports =\n    [ # Include the results of the hardware scan.\n      ./hardware-configuration.nix\n    ];\n\n  nixpkgs.config.allowUnfree = true;\n\n  # Use the systemd-boot EFI boot loader.\n  boot.loader.systemd-boot.enable = true;\n  boot.loader.efi.canTouchEfiVariables = true;\n\n  networking.hostName = \"jaglan-beta-m03\"; # Define your hostname.\n\n  time.timeZone = \"Australia/Melbourne\";\n\n  # List packages installed in system profile. To search, run:\n  # $ nix search wget\n  # environment.systemPackages = with pkgs; [\n  #   vim # Do not forget to add an editor to edit configuration.nix! The Nano editor is also installed by default.\n  #   wget\n  # ];\n\n  # Some programs need SUID wrappers, can be configured further or are\n  # started in user sessions.\n  # programs.mtr.enable = true;\n  # programs.gnupg.agent = {\n  #   enable = true;\n  #   enableSSHSupport = true;\n  # };\n\n  # List services that you want to enable:\n  services = {\n    nomad = {\n      enable = true;\n      enableDocker = true;\n      dropPrivileges = false;\n      settings = {\n        datacenter = \"jaglan-beta\";\n        server = {\n          enabled = true;\n        };\n        client = {\n          enabled = true;\n          preferred_address_family = \"ipv4\";\n          cpu_total_compute = 7200;\n          host_volume = {\n          };\n          cni_path = \"${pkgs.cni-plugins}/bin\";\n        };\n        plugin.docker.config.allow_privileged = true;\n      };\n      extraPackages = with pkgs; [\n        cni-plugins\n        consul\n      ];\n    };\n    consul = {\n      enable = true;\n      webUi = true;\n      interface.bind = \"enp1s0\";\n      interface.advertise = \"enp1s0\";\n      forceAddrFamily = \"ipv4\";\n      extraConfig  = {\n        client_addr = \"{{ GetPrivateInterfaces | exclude \\\"type\\\" \\\"ipv6\\\" | join \\\"address\\\" \\\" \\\" }} {{ GetAllInterfaces | include \\\"flags\\\" \\\"loopback\\\" | join \\\"address\\\" \\\" \\\" }}\";\n        server = true;\n        retry_join = [\n          \"jaglan-beta-m01\"\n          \"jaglan-beta-m02\"\n          \"jaglan-beta-m03\"\n          \"jaglan-beta-m04\"\n          \"jaglan-beta-m05\"\n          \"jaglan-beta-m20\"\n          \"jaglan-beta-m21\"\n          \"jaglan-beta-m22\"\n        ];\n        datacenter = \"jaglan-beta\";\n        connect.enabled = true;\n        ports.grpc = 8502;\n      };\n    };\n    openssh = {\n      enable = true;\n      settings.PermitRootLogin = \"yes\";\n    };\n  };\n\n  systemd.tmpfiles.rules = [\n    # Fix issue where nomad needs alloc_mounts to be writable\n    \"d /var/lib/alloc_mounts 0755 root root -\"\n  ];\n\n  # Open ports in the firewall. 80/443 are for HTTP/HTTPS (terraform), 464X are the default ports for Nomad, 830X are the default ports for Consul.\n  networking.firewall.allowedTCPPorts = [ 80 443 8081 4646 4647 4648 8300 8301 8500 ];\n  networking.firewall.allowedUDPPorts = [ 8301 ];\n\n  # Copy the NixOS configuration file and link it from the resulting system\n  # (/run/current-system/configuration.nix). This is useful in case you\n  # accidentally delete configuration.nix.\n  system.copySystemConfiguration = true;\n \n  # Defines the initial NixOS version for compatibility with older application data.\n  # Do NOT change this value after installation without careful consideration.\n  system.stateVersion = \"24.11\"; # Did you read the comment?\n}\n"
+            }
+          },
+          "sensitive_attributes": [],
+          "identity_schema_version": 0
+        },
+        {
+          "index_key": "jaglan-beta-m04",
+          "schema_version": 0,
+          "attributes": {
+            "id": "5130090292645542538",
+            "triggers": {
+              "configuration_content": "{ config, lib, pkgs, ... }:\n{\n  imports =\n    [ # Include the results of the hardware scan.\n      ./hardware-configuration.nix\n    ];\n\n  nixpkgs.config.allowUnfree = true;\n\n  # Use the systemd-boot EFI boot loader.\n  boot.loader.systemd-boot.enable = true;\n  boot.loader.efi.canTouchEfiVariables = true;\n\n  networking.hostName = \"jaglan-beta-m04\"; # Define your hostname.\n\n  time.timeZone = \"Australia/Melbourne\";\n\n  # List packages installed in system profile. To search, run:\n  # $ nix search wget\n  # environment.systemPackages = with pkgs; [\n  #   vim # Do not forget to add an editor to edit configuration.nix! The Nano editor is also installed by default.\n  #   wget\n  # ];\n\n  # Some programs need SUID wrappers, can be configured further or are\n  # started in user sessions.\n  # programs.mtr.enable = true;\n  # programs.gnupg.agent = {\n  #   enable = true;\n  #   enableSSHSupport = true;\n  # };\n\n  # List services that you want to enable:\n  services = {\n    nomad = {\n      enable = true;\n      enableDocker = true;\n      dropPrivileges = false;\n      settings = {\n        datacenter = \"jaglan-beta\";\n        server = {\n          enabled = true;\n        };\n        client = {\n          enabled = true;\n          preferred_address_family = \"ipv4\";\n          cpu_total_compute = 7200;\n          host_volume = {\n          };\n          cni_path = \"${pkgs.cni-plugins}/bin\";\n        };\n        plugin.docker.config.allow_privileged = true;\n      };\n      extraPackages = with pkgs; [\n        cni-plugins\n        consul\n      ];\n    };\n    consul = {\n      enable = true;\n      webUi = true;\n      interface.bind = \"enp1s0\";\n      interface.advertise = \"enp1s0\";\n      forceAddrFamily = \"ipv4\";\n      extraConfig  = {\n        client_addr = \"{{ GetPrivateInterfaces | exclude \\\"type\\\" \\\"ipv6\\\" | join \\\"address\\\" \\\" \\\" }} {{ GetAllInterfaces | include \\\"flags\\\" \\\"loopback\\\" | join \\\"address\\\" \\\" \\\" }}\";\n        server = true;\n        retry_join = [\n          \"jaglan-beta-m01\"\n          \"jaglan-beta-m02\"\n          \"jaglan-beta-m03\"\n          \"jaglan-beta-m04\"\n          \"jaglan-beta-m05\"\n          \"jaglan-beta-m20\"\n          \"jaglan-beta-m21\"\n          \"jaglan-beta-m22\"\n        ];\n        datacenter = \"jaglan-beta\";\n        connect.enabled = true;\n        ports.grpc = 8502;\n      };\n    };\n    openssh = {\n      enable = true;\n      settings.PermitRootLogin = \"yes\";\n    };\n  };\n\n  systemd.tmpfiles.rules = [\n    # Fix issue where nomad needs alloc_mounts to be writable\n    \"d /var/lib/alloc_mounts 0755 root root -\"\n  ];\n\n  # Open ports in the firewall. 80/443 are for HTTP/HTTPS (terraform), 464X are the default ports for Nomad, 830X are the default ports for Consul.\n  networking.firewall.allowedTCPPorts = [ 80 443 8081 4646 4647 4648 8300 8301 8500 ];\n  networking.firewall.allowedUDPPorts = [ 8301 ];\n\n  # Copy the NixOS configuration file and link it from the resulting system\n  # (/run/current-system/configuration.nix). This is useful in case you\n  # accidentally delete configuration.nix.\n  system.copySystemConfiguration = true;\n \n  # Defines the initial NixOS version for compatibility with older application data.\n  # Do NOT change this value after installation without careful consideration.\n  system.stateVersion = \"24.11\"; # Did you read the comment?\n}\n"
+            }
+          },
+          "sensitive_attributes": [],
+          "identity_schema_version": 0
+        },
+        {
+          "index_key": "jaglan-beta-m05",
+          "schema_version": 0,
+          "attributes": {
+            "id": "1855172426704636197",
+            "triggers": {
+              "configuration_content": "{ config, lib, pkgs, ... }:\n{\n  imports =\n    [ # Include the results of the hardware scan.\n      ./hardware-configuration.nix\n    ];\n\n  nixpkgs.config.allowUnfree = true;\n\n  # Use the systemd-boot EFI boot loader.\n  boot.loader.systemd-boot.enable = true;\n  boot.loader.efi.canTouchEfiVariables = true;\n\n  networking.hostName = \"jaglan-beta-m05\"; # Define your hostname.\n\n  time.timeZone = \"Australia/Melbourne\";\n\n  # List packages installed in system profile. To search, run:\n  # $ nix search wget\n  # environment.systemPackages = with pkgs; [\n  #   vim # Do not forget to add an editor to edit configuration.nix! The Nano editor is also installed by default.\n  #   wget\n  # ];\n\n  # Some programs need SUID wrappers, can be configured further or are\n  # started in user sessions.\n  # programs.mtr.enable = true;\n  # programs.gnupg.agent = {\n  #   enable = true;\n  #   enableSSHSupport = true;\n  # };\n\n  # List services that you want to enable:\n  services = {\n    nomad = {\n      enable = true;\n      enableDocker = true;\n      dropPrivileges = false;\n      settings = {\n        datacenter = \"jaglan-beta\";\n        server = {\n          enabled = true;\n        };\n        client = {\n          enabled = true;\n          preferred_address_family = \"ipv4\";\n          cpu_total_compute = 7200;\n          host_volume = {\n          };\n          cni_path = \"${pkgs.cni-plugins}/bin\";\n        };\n        plugin.docker.config.allow_privileged = true;\n      };\n      extraPackages = with pkgs; [\n        cni-plugins\n        consul\n      ];\n    };\n    consul = {\n      enable = true;\n      webUi = true;\n      interface.bind = \"enp1s0\";\n      interface.advertise = \"enp1s0\";\n      forceAddrFamily = \"ipv4\";\n      extraConfig  = {\n        client_addr = \"{{ GetPrivateInterfaces | exclude \\\"type\\\" \\\"ipv6\\\" | join \\\"address\\\" \\\" \\\" }} {{ GetAllInterfaces | include \\\"flags\\\" \\\"loopback\\\" | join \\\"address\\\" \\\" \\\" }}\";\n        server = true;\n        retry_join = [\n          \"jaglan-beta-m01\"\n          \"jaglan-beta-m02\"\n          \"jaglan-beta-m03\"\n          \"jaglan-beta-m04\"\n          \"jaglan-beta-m05\"\n          \"jaglan-beta-m20\"\n          \"jaglan-beta-m21\"\n          \"jaglan-beta-m22\"\n        ];\n        datacenter = \"jaglan-beta\";\n        connect.enabled = true;\n        ports.grpc = 8502;\n      };\n    };\n    openssh = {\n      enable = true;\n      settings.PermitRootLogin = \"yes\";\n    };\n  };\n\n  systemd.tmpfiles.rules = [\n    # Fix issue where nomad needs alloc_mounts to be writable\n    \"d /var/lib/alloc_mounts 0755 root root -\"\n  ];\n\n  # Open ports in the firewall. 80/443 are for HTTP/HTTPS (terraform), 464X are the default ports for Nomad, 830X are the default ports for Consul.\n  networking.firewall.allowedTCPPorts = [ 80 443 8081 4646 4647 4648 8300 8301 8500 ];\n  networking.firewall.allowedUDPPorts = [ 8301 ];\n\n  # Copy the NixOS configuration file and link it from the resulting system\n  # (/run/current-system/configuration.nix). This is useful in case you\n  # accidentally delete configuration.nix.\n  system.copySystemConfiguration = true;\n \n  # Defines the initial NixOS version for compatibility with older application data.\n  # Do NOT change this value after installation without careful consideration.\n  system.stateVersion = \"24.11\"; # Did you read the comment?\n}\n"
+            }
+          },
+          "sensitive_attributes": [],
+          "identity_schema_version": 0
         },
         {
           "index_key": "jaglan-beta-m20",
           "schema_version": 0,
           "attributes": {
-            "id": "4501962519001291201",
+            "id": "2979501464234412040",
             "triggers": {
-              "configuration_content": "{ config, lib, pkgs, ... }:\n{\n  imports =\n    [ # Include the results of the hardware scan.\n      ./hardware-configuration.nix\n    ];\n\n  nixpkgs.config.allowUnfree = true;\n\n  # Use the systemd-boot EFI boot loader.\n  boot.loader.systemd-boot.enable = true;\n  boot.loader.efi.canTouchEfiVariables = true;\n\n  networking.hostName = \"jaglan-beta-m20\"; # Define your hostname.\n\n  time.timeZone = \"Australia/Melbourne\";\n\n  # List packages installed in system profile. To search, run:\n  # $ nix search wget\n  # environment.systemPackages = with pkgs; [\n  #   vim # Do not forget to add an editor to edit configuration.nix! The Nano editor is also installed by default.\n  #   wget\n  # ];\n\n  # Some programs need SUID wrappers, can be configured further or are\n  # started in user sessions.\n  # programs.mtr.enable = true;\n  # programs.gnupg.agent = {\n  #   enable = true;\n  #   enableSSHSupport = true;\n  # };\n\n  # List services that you want to enable:\n  services = {\n    nomad = {\n      enable = true;\n      enableDocker = true;\n      dropPrivileges = false;\n      settings = {\n        datacenter = \"jaglan-beta\";\n        server = {\n          enabled = true;\n        };\n        client = {\n          enabled = true;\n          host_volume = {\n          };\n          cni_path = \"${pkgs.cni-plugins}/bin\";\n        };\n        plugin.docker.config.allow_privileged = true;\n      };\n      extraPackages = with pkgs; [\n        cni-plugins\n        consul\n      ];\n    };\n    consul = {\n      enable = true;\n      webUi = true;\n      interface.bind = \"ens2\";\n      interface.advertise = \"ens2\";\n      extraConfig  = {\n        client_addr = \"{{ GetPrivateInterfaces | exclude \\\"type\\\" \\\"ipv6\\\" | join \\\"address\\\" \\\" \\\" }} {{ GetAllInterfaces | include \\\"flags\\\" \\\"loopback\\\" | join \\\"address\\\" \\\" \\\" }}\";\n        server = true;\n        retry_join = [\n          \"jaglan-beta-m01\"\n          \"jaglan-beta-m20\"\n        ];\n        datacenter = \"jaglan-beta\";\n        connect.enabled = true;\n        ports.grpc = 8502;\n      };\n    };\n    openssh = {\n      enable = true;\n      settings.PermitRootLogin = \"yes\";\n    };\n  };\n\n  systemd.tmpfiles.rules = [\n    # Fix issue where nomad needs alloc_mounts to be writable\n    \"d /var/lib/alloc_mounts 0755 root root -\"\n  ];\n\n  # Open ports in the firewall. 80/443 are for HTTP/HTTPS (terraform), 464X are the default ports for Nomad, 830X are the default ports for Consul.\n  networking.firewall.allowedTCPPorts = [ 80 443 8081 4646 4647 4648 8300 8301 ];\n  networking.firewall.allowedUDPPorts = [ 8301 ];\n\n  # Copy the NixOS configuration file and link it from the resulting system\n  # (/run/current-system/configuration.nix). This is useful in case you\n  # accidentally delete configuration.nix.\n  system.copySystemConfiguration = true;\n \n  # Defines the initial NixOS version for compatibility with older application data.\n  # Do NOT change this value after installation without careful consideration.\n  system.stateVersion = \"24.11\"; # Did you read the comment?\n}\n"
+              "configuration_content": "{ config, lib, pkgs, ... }:\n{\n  imports =\n    [ # Include the results of the hardware scan.\n      ./hardware-configuration.nix\n    ];\n\n  nixpkgs.config.allowUnfree = true;\n\n  # Use the systemd-boot EFI boot loader.\n  boot.loader.systemd-boot.enable = true;\n  boot.loader.efi.canTouchEfiVariables = true;\n\n  networking.hostName = \"jaglan-beta-m20\"; # Define your hostname.\n\n  time.timeZone = \"Australia/Melbourne\";\n\n  # List packages installed in system profile. To search, run:\n  # $ nix search wget\n  # environment.systemPackages = with pkgs; [\n  #   vim # Do not forget to add an editor to edit configuration.nix! The Nano editor is also installed by default.\n  #   wget\n  # ];\n\n  # Some programs need SUID wrappers, can be configured further or are\n  # started in user sessions.\n  # programs.mtr.enable = true;\n  # programs.gnupg.agent = {\n  #   enable = true;\n  #   enableSSHSupport = true;\n  # };\n\n  # List services that you want to enable:\n  services = {\n    nomad = {\n      enable = true;\n      enableDocker = true;\n      dropPrivileges = false;\n      settings = {\n        datacenter = \"jaglan-beta\";\n        server = {\n          enabled = true;\n          bootstrap_expect = 1;\n        };\n        client = {\n          enabled = true;\n          preferred_address_family = \"ipv4\";\n          cpu_total_compute = 10000;\n          host_volume = {\n          };\n          cni_path = \"${pkgs.cni-plugins}/bin\";\n        };\n        plugin.docker.config.allow_privileged = true;\n      };\n      extraPackages = with pkgs; [\n        cni-plugins\n        consul\n      ];\n    };\n    consul = {\n      enable = true;\n      webUi = true;\n      interface.bind = \"ens2\";\n      interface.advertise = \"ens2\";\n      forceAddrFamily = \"ipv4\";\n      extraConfig  = {\n        client_addr = \"{{ GetPrivateInterfaces | exclude \\\"type\\\" \\\"ipv6\\\" | join \\\"address\\\" \\\" \\\" }} {{ GetAllInterfaces | include \\\"flags\\\" \\\"loopback\\\" | join \\\"address\\\" \\\" \\\" }}\";\n        bootstrap_expect = 1;\n        server = true;\n        retry_join = [\n          \"jaglan-beta-m01\"\n          \"jaglan-beta-m02\"\n          \"jaglan-beta-m03\"\n          \"jaglan-beta-m04\"\n          \"jaglan-beta-m05\"\n          \"jaglan-beta-m20\"\n          \"jaglan-beta-m21\"\n          \"jaglan-beta-m22\"\n        ];\n        datacenter = \"jaglan-beta\";\n        connect.enabled = true;\n        ports.grpc = 8502;\n      };\n    };\n    openssh = {\n      enable = true;\n      settings.PermitRootLogin = \"yes\";\n    };\n  };\n\n  systemd.tmpfiles.rules = [\n    # Fix issue where nomad needs alloc_mounts to be writable\n    \"d /var/lib/alloc_mounts 0755 root root -\"\n  ];\n\n  # Open ports in the firewall. 80/443 are for HTTP/HTTPS (terraform), 464X are the default ports for Nomad, 830X are the default ports for Consul.\n  networking.firewall.allowedTCPPorts = [ 80 443 8081 4646 4647 4648 8300 8301 8500 ];\n  networking.firewall.allowedUDPPorts = [ 8301 ];\n\n  # Copy the NixOS configuration file and link it from the resulting system\n  # (/run/current-system/configuration.nix). This is useful in case you\n  # accidentally delete configuration.nix.\n  system.copySystemConfiguration = true;\n \n  # Defines the initial NixOS version for compatibility with older application data.\n  # Do NOT change this value after installation without careful consideration.\n  system.stateVersion = \"24.11\"; # Did you read the comment?\n}\n"
             }
           },
-          "sensitive_attributes": []
+          "sensitive_attributes": [],
+          "identity_schema_version": 0
+        },
+        {
+          "index_key": "jaglan-beta-m21",
+          "schema_version": 0,
+          "attributes": {
+            "id": "6363236121611066492",
+            "triggers": {
+              "configuration_content": "{ config, lib, pkgs, ... }:\n{\n  imports =\n    [ # Include the results of the hardware scan.\n      ./hardware-configuration.nix\n    ];\n\n  nixpkgs.config.allowUnfree = true;\n\n  # Use the systemd-boot EFI boot loader.\n  boot.loader.systemd-boot.enable = true;\n  boot.loader.efi.canTouchEfiVariables = true;\n\n  networking.hostName = \"jaglan-beta-m21\"; # Define your hostname.\n\n  time.timeZone = \"Australia/Melbourne\";\n\n  # List packages installed in system profile. To search, run:\n  # $ nix search wget\n  # environment.systemPackages = with pkgs; [\n  #   vim # Do not forget to add an editor to edit configuration.nix! The Nano editor is also installed by default.\n  #   wget\n  # ];\n\n  # Some programs need SUID wrappers, can be configured further or are\n  # started in user sessions.\n  # programs.mtr.enable = true;\n  # programs.gnupg.agent = {\n  #   enable = true;\n  #   enableSSHSupport = true;\n  # };\n\n  # List services that you want to enable:\n  services = {\n    nomad = {\n      enable = true;\n      enableDocker = true;\n      dropPrivileges = false;\n      settings = {\n        datacenter = \"jaglan-beta\";\n        server = {\n          enabled = true;\n        };\n        client = {\n          enabled = true;\n          preferred_address_family = \"ipv4\";\n          cpu_total_compute = 10000;\n          host_volume = {\n            traefik = {\n              path = \"/opt/traefik\";\n              read_only = false;\n            };\n            postgres = {\n              path = \"/opt/postgres\";\n              read_only = false;\n            };\n          };\n          cni_path = \"${pkgs.cni-plugins}/bin\";\n        };\n        plugin.docker.config.allow_privileged = true;\n      };\n      extraPackages = with pkgs; [\n        cni-plugins\n        consul\n      ];\n    };\n    consul = {\n      enable = true;\n      webUi = true;\n      interface.bind = \"ens2\";\n      interface.advertise = \"ens2\";\n      forceAddrFamily = \"ipv4\";\n      extraConfig  = {\n        client_addr = \"{{ GetPrivateInterfaces | exclude \\\"type\\\" \\\"ipv6\\\" | join \\\"address\\\" \\\" \\\" }} {{ GetAllInterfaces | include \\\"flags\\\" \\\"loopback\\\" | join \\\"address\\\" \\\" \\\" }}\";\n        server = true;\n        retry_join = [\n          \"jaglan-beta-m01\"\n          \"jaglan-beta-m02\"\n          \"jaglan-beta-m03\"\n          \"jaglan-beta-m04\"\n          \"jaglan-beta-m05\"\n          \"jaglan-beta-m20\"\n          \"jaglan-beta-m21\"\n          \"jaglan-beta-m22\"\n        ];\n        datacenter = \"jaglan-beta\";\n        connect.enabled = true;\n        ports.grpc = 8502;\n      };\n    };\n    openssh = {\n      enable = true;\n      settings.PermitRootLogin = \"yes\";\n    };\n  };\n\n  systemd.tmpfiles.rules = [\n    # Fix issue where nomad needs alloc_mounts to be writable\n    \"d /var/lib/alloc_mounts 0755 root root -\"\n    # Create a directory for traefik to store its data\n    \"d /opt/traefik 0755 root root -\"\n    # Create a directory for postgres to store its data\n    \"d /opt/postgres 0755 root root -\"\n  ];\n\n  # Open ports in the firewall. 80/443 are for HTTP/HTTPS (terraform), 464X are the default ports for Nomad, 830X are the default ports for Consul.\n  networking.firewall.allowedTCPPorts = [ 80 443 8081 4646 4647 4648 8300 8301 8500 ];\n  networking.firewall.allowedUDPPorts = [ 8301 ];\n\n  # Copy the NixOS configuration file and link it from the resulting system\n  # (/run/current-system/configuration.nix). This is useful in case you\n  # accidentally delete configuration.nix.\n  system.copySystemConfiguration = true;\n \n  # Defines the initial NixOS version for compatibility with older application data.\n  # Do NOT change this value after installation without careful consideration.\n  system.stateVersion = \"24.11\"; # Did you read the comment?\n}\n"
+            }
+          },
+          "sensitive_attributes": [],
+          "identity_schema_version": 0
+        },
+        {
+          "index_key": "jaglan-beta-m22",
+          "schema_version": 0,
+          "attributes": {
+            "id": "2272269092899111201",
+            "triggers": {
+              "configuration_content": "{ config, lib, pkgs, ... }:\n{\n  imports =\n    [ # Include the results of the hardware scan.\n      ./hardware-configuration.nix\n    ];\n\n  nixpkgs.config.allowUnfree = true;\n\n  # Use the systemd-boot EFI boot loader.\n  boot.loader.systemd-boot.enable = true;\n  boot.loader.efi.canTouchEfiVariables = true;\n\n  networking.hostName = \"jaglan-beta-m22\"; # Define your hostname.\n\n  time.timeZone = \"Australia/Melbourne\";\n\n  # List packages installed in system profile. To search, run:\n  # $ nix search wget\n  # environment.systemPackages = with pkgs; [\n  #   vim # Do not forget to add an editor to edit configuration.nix! The Nano editor is also installed by default.\n  #   wget\n  # ];\n\n  # Some programs need SUID wrappers, can be configured further or are\n  # started in user sessions.\n  # programs.mtr.enable = true;\n  # programs.gnupg.agent = {\n  #   enable = true;\n  #   enableSSHSupport = true;\n  # };\n\n  # List services that you want to enable:\n  services = {\n    nomad = {\n      enable = true;\n      enableDocker = true;\n      dropPrivileges = false;\n      settings = {\n        datacenter = \"jaglan-beta\";\n        server = {\n          enabled = true;\n        };\n        client = {\n          enabled = true;\n          preferred_address_family = \"ipv4\";\n          cpu_total_compute = 10000;\n          host_volume = {\n          };\n          cni_path = \"${pkgs.cni-plugins}/bin\";\n        };\n        plugin.docker.config.allow_privileged = true;\n      };\n      extraPackages = with pkgs; [\n        cni-plugins\n        consul\n      ];\n    };\n    consul = {\n      enable = true;\n      webUi = true;\n      interface.bind = \"ens2\";\n      interface.advertise = \"ens2\";\n      forceAddrFamily = \"ipv4\";\n      extraConfig  = {\n        client_addr = \"{{ GetPrivateInterfaces | exclude \\\"type\\\" \\\"ipv6\\\" | join \\\"address\\\" \\\" \\\" }} {{ GetAllInterfaces | include \\\"flags\\\" \\\"loopback\\\" | join \\\"address\\\" \\\" \\\" }}\";\n        server = true;\n        retry_join = [\n          \"jaglan-beta-m01\"\n          \"jaglan-beta-m02\"\n          \"jaglan-beta-m03\"\n          \"jaglan-beta-m04\"\n          \"jaglan-beta-m05\"\n          \"jaglan-beta-m20\"\n          \"jaglan-beta-m21\"\n          \"jaglan-beta-m22\"\n        ];\n        datacenter = \"jaglan-beta\";\n        connect.enabled = true;\n        ports.grpc = 8502;\n      };\n    };\n    openssh = {\n      enable = true;\n      settings.PermitRootLogin = \"yes\";\n    };\n  };\n\n  systemd.tmpfiles.rules = [\n    # Fix issue where nomad needs alloc_mounts to be writable\n    \"d /var/lib/alloc_mounts 0755 root root -\"\n  ];\n\n  # Open ports in the firewall. 80/443 are for HTTP/HTTPS (terraform), 464X are the default ports for Nomad, 830X are the default ports for Consul.\n  networking.firewall.allowedTCPPorts = [ 80 443 8081 4646 4647 4648 8300 8301 8500 ];\n  networking.firewall.allowedUDPPorts = [ 8301 ];\n\n  # Copy the NixOS configuration file and link it from the resulting system\n  # (/run/current-system/configuration.nix). This is useful in case you\n  # accidentally delete configuration.nix.\n  system.copySystemConfiguration = true;\n \n  # Defines the initial NixOS version for compatibility with older application data.\n  # Do NOT change this value after installation without careful consideration.\n  system.stateVersion = \"24.11\"; # Did you read the comment?\n}\n"
+            }
+          },
+          "sensitive_attributes": [],
+          "identity_schema_version": 0
         }
       ]
     }

1-nixos-node/terraform.tfstate.backup

@@ -1,7 +1,7 @@
 {
   "version": 4,
-  "terraform_version": "1.11.4",
-  "serial": 246,
+  "terraform_version": "1.13.0",
+  "serial": 322,
   "lineage": "db7dcf21-a255-0ec4-c8b8-d4a7559b3768",
   "outputs": {},
   "resources": [
@@ -12,26 +12,88 @@
       "provider": "provider[\"registry.terraform.io/hashicorp/null\"]",
       "instances": [
         {
-          "index_key": "jaglan-beta-m01",
+          "index_key": "jaglan-beta-m02",
           "schema_version": 0,
           "attributes": {
-            "id": "375315072123105414",
+            "id": "4799650720916837801",
             "triggers": {
-              "configuration_content": "{ config, lib, pkgs, ... }:\n{\n  imports =\n    [ # Include the results of the hardware scan.\n      ./hardware-configuration.nix\n    ];\n\n  nixpkgs.config.allowUnfree = true;\n\n  # Use the systemd-boot EFI boot loader.\n  boot.loader.systemd-boot.enable = true;\n  boot.loader.efi.canTouchEfiVariables = true;\n\n  networking.hostName = \"jaglan-beta-m01\"; # Define your hostname.\n\n  time.timeZone = \"Australia/Melbourne\";\n\n  # List packages installed in system profile. To search, run:\n  # $ nix search wget\n  # environment.systemPackages = with pkgs; [\n  #   vim # Do not forget to add an editor to edit configuration.nix! The Nano editor is also installed by default.\n  #   wget\n  # ];\n\n  # Some programs need SUID wrappers, can be configured further or are\n  # started in user sessions.\n  # programs.mtr.enable = true;\n  # programs.gnupg.agent = {\n  #   enable = true;\n  #   enableSSHSupport = true;\n  # };\n\n  # List services that you want to enable:\n  services = {\n    nomad = {\n      enable = true;\n      enableDocker = true;\n      dropPrivileges = false;\n      settings = {\n        datacenter = \"jaglan-beta\";\n        server = {\n          enabled = true;\n          bootstrap_expect = 1;\n        };\n        client = {\n          enabled = true;\n          host_volume = {\n            traefik = {\n              path = \"/opt/traefik\";\n              read_only = false;\n            };\n            postgres = {\n              path = \"/opt/postgres\";\n              read_only = false;\n            };\n          };\n          cni_path = \"${pkgs.cni-plugins}/bin\";\n        };\n        plugin.docker.config.allow_privileged = true;\n      };\n      extraPackages = with pkgs; [\n        cni-plugins\n        consul\n      ];\n    };\n    consul = {\n      enable = true;\n      webUi = true;\n      interface.bind = \"eno1\";\n      interface.advertise = \"eno1\";\n      extraConfig  = {\n        client_addr = \"{{ GetPrivateInterfaces | exclude \\\"type\\\" \\\"ipv6\\\" | join \\\"address\\\" \\\" \\\" }} {{ GetAllInterfaces | include \\\"flags\\\" \\\"loopback\\\" | join \\\"address\\\" \\\" \\\" }}\";\n        bootstrap_expect = 1;\n        server = true;\n        retry_join = [\n          \"jaglan-beta-m01\"\n          \"jaglan-beta-m20\"\n        ];\n        datacenter = \"jaglan-beta\";\n        connect.enabled = true;\n        ports.grpc = 8502;\n      };\n    };\n    openssh = {\n      enable = true;\n      settings.PermitRootLogin = \"yes\";\n    };\n  };\n\n  systemd.tmpfiles.rules = [\n    # Fix issue where nomad needs alloc_mounts to be writable\n    \"d /var/lib/alloc_mounts 0755 root root -\"\n    # Create a directory for traefik to store its data\n    \"d /opt/traefik 0755 root root -\"\n    # Create a directory for postgres to store its data\n    \"d /opt/postgres 0755 root root -\"\n  ];\n\n  # Open ports in the firewall. 80/443 are for HTTP/HTTPS (terraform), 464X are the default ports for Nomad, 830X are the default ports for Consul.\n  networking.firewall.allowedTCPPorts = [ 80 443 4646 4647 4648 8300 8301 ];\n  networking.firewall.allowedUDPPorts = [ 8301 ];\n\n  # Copy the NixOS configuration file and link it from the resulting system\n  # (/run/current-system/configuration.nix). This is useful in case you\n  # accidentally delete configuration.nix.\n  system.copySystemConfiguration = true;\n \n  # Defines the initial NixOS version for compatibility with older application data.\n  # Do NOT change this value after installation without careful consideration.\n  system.stateVersion = \"24.11\"; # Did you read the comment?\n}\n"
+              "configuration_content": "{ config, lib, pkgs, ... }:\n{\n  imports =\n    [ # Include the results of the hardware scan.\n      ./hardware-configuration.nix\n    ];\n\n  nixpkgs.config.allowUnfree = true;\n\n  # Use the systemd-boot EFI boot loader.\n  boot.loader.systemd-boot.enable = true;\n  boot.loader.efi.canTouchEfiVariables = true;\n\n  networking.hostName = \"jaglan-beta-m02\"; # Define your hostname.\n\n  time.timeZone = \"Australia/Melbourne\";\n\n  # List packages installed in system profile. To search, run:\n  # $ nix search wget\n  # environment.systemPackages = with pkgs; [\n  #   vim # Do not forget to add an editor to edit configuration.nix! The Nano editor is also installed by default.\n  #   wget\n  # ];\n\n  # Some programs need SUID wrappers, can be configured further or are\n  # started in user sessions.\n  # programs.mtr.enable = true;\n  # programs.gnupg.agent = {\n  #   enable = true;\n  #   enableSSHSupport = true;\n  # };\n\n  # List services that you want to enable:\n  services = {\n    nomad = {\n      enable = true;\n      enableDocker = true;\n      dropPrivileges = false;\n      settings = {\n        datacenter = \"jaglan-beta\";\n        server = {\n          enabled = true;\n        };\n        client = {\n          enabled = true;\n          preferred_address_family = \"ipv4\";\n          cpu_total_compute = 7200;\n          host_volume = {\n          };\n          cni_path = \"${pkgs.cni-plugins}/bin\";\n        };\n        plugin.docker.config.allow_privileged = true;\n      };\n      extraPackages = with pkgs; [\n        cni-plugins\n        consul\n      ];\n    };\n    consul = {\n      enable = true;\n      webUi = true;\n      interface.bind = \"enp1s0\";\n      interface.advertise = \"enp1s0\";\n      forceAddrFamily = \"ipv4\";\n      extraConfig  = {\n        client_addr = \"{{ GetPrivateInterfaces | exclude \\\"type\\\" \\\"ipv6\\\" | join \\\"address\\\" \\\" \\\" }} {{ GetAllInterfaces | include \\\"flags\\\" \\\"loopback\\\" | join \\\"address\\\" \\\" \\\" }}\";\n        server = true;\n        retry_join = [\n          \"jaglan-beta-m01\"\n          \"jaglan-beta-m02\"\n          \"jaglan-beta-m03\"\n          \"jaglan-beta-m04\"\n          \"jaglan-beta-m05\"\n          \"jaglan-beta-m20\"\n          \"jaglan-beta-m21\"\n          \"jaglan-beta-m22\"\n        ];\n        datacenter = \"jaglan-beta\";\n        connect.enabled = true;\n        ports.grpc = 8502;\n      };\n    };\n    openssh = {\n      enable = true;\n      settings.PermitRootLogin = \"yes\";\n    };\n  };\n\n  systemd.tmpfiles.rules = [\n    # Fix issue where nomad needs alloc_mounts to be writable\n    \"d /var/lib/alloc_mounts 0755 root root -\"\n  ];\n\n  # Open ports in the firewall. 80/443 are for HTTP/HTTPS (terraform), 464X are the default ports for Nomad, 830X are the default ports for Consul.\n  networking.firewall.allowedTCPPorts = [ 80 443 8081 4646 4647 4648 8300 8301 8500 ];\n  networking.firewall.allowedUDPPorts = [ 8301 ];\n\n  # Copy the NixOS configuration file and link it from the resulting system\n  # (/run/current-system/configuration.nix). This is useful in case you\n  # accidentally delete configuration.nix.\n  system.copySystemConfiguration = true;\n \n  # Defines the initial NixOS version for compatibility with older application data.\n  # Do NOT change this value after installation without careful consideration.\n  system.stateVersion = \"24.11\"; # Did you read the comment?\n}\n"
             }
           },
-          "sensitive_attributes": []
+          "sensitive_attributes": [],
+          "identity_schema_version": 0
+        },
+        {
+          "index_key": "jaglan-beta-m03",
+          "schema_version": 0,
+          "attributes": {
+            "id": "3940670083197547320",
+            "triggers": {
+              "configuration_content": "{ config, lib, pkgs, ... }:\n{\n  imports =\n    [ # Include the results of the hardware scan.\n      ./hardware-configuration.nix\n    ];\n\n  nixpkgs.config.allowUnfree = true;\n\n  # Use the systemd-boot EFI boot loader.\n  boot.loader.systemd-boot.enable = true;\n  boot.loader.efi.canTouchEfiVariables = true;\n\n  networking.hostName = \"jaglan-beta-m03\"; # Define your hostname.\n\n  time.timeZone = \"Australia/Melbourne\";\n\n  # List packages installed in system profile. To search, run:\n  # $ nix search wget\n  # environment.systemPackages = with pkgs; [\n  #   vim # Do not forget to add an editor to edit configuration.nix! The Nano editor is also installed by default.\n  #   wget\n  # ];\n\n  # Some programs need SUID wrappers, can be configured further or are\n  # started in user sessions.\n  # programs.mtr.enable = true;\n  # programs.gnupg.agent = {\n  #   enable = true;\n  #   enableSSHSupport = true;\n  # };\n\n  # List services that you want to enable:\n  services = {\n    nomad = {\n      enable = true;\n      enableDocker = true;\n      dropPrivileges = false;\n      settings = {\n        datacenter = \"jaglan-beta\";\n        server = {\n          enabled = true;\n        };\n        client = {\n          enabled = true;\n          preferred_address_family = \"ipv4\";\n          cpu_total_compute = 7200;\n          host_volume = {\n          };\n          cni_path = \"${pkgs.cni-plugins}/bin\";\n        };\n        plugin.docker.config.allow_privileged = true;\n      };\n      extraPackages = with pkgs; [\n        cni-plugins\n        consul\n      ];\n    };\n    consul = {\n      enable = true;\n      webUi = true;\n      interface.bind = \"enp1s0\";\n      interface.advertise = \"enp1s0\";\n      forceAddrFamily = \"ipv4\";\n      extraConfig  = {\n        client_addr = \"{{ GetPrivateInterfaces | exclude \\\"type\\\" \\\"ipv6\\\" | join \\\"address\\\" \\\" \\\" }} {{ GetAllInterfaces | include \\\"flags\\\" \\\"loopback\\\" | join \\\"address\\\" \\\" \\\" }}\";\n        server = true;\n        retry_join = [\n          \"jaglan-beta-m01\"\n          \"jaglan-beta-m02\"\n          \"jaglan-beta-m03\"\n          \"jaglan-beta-m04\"\n          \"jaglan-beta-m05\"\n          \"jaglan-beta-m20\"\n          \"jaglan-beta-m21\"\n          \"jaglan-beta-m22\"\n        ];\n        datacenter = \"jaglan-beta\";\n        connect.enabled = true;\n        ports.grpc = 8502;\n      };\n    };\n    openssh = {\n      enable = true;\n      settings.PermitRootLogin = \"yes\";\n    };\n  };\n\n  systemd.tmpfiles.rules = [\n    # Fix issue where nomad needs alloc_mounts to be writable\n    \"d /var/lib/alloc_mounts 0755 root root -\"\n  ];\n\n  # Open ports in the firewall. 80/443 are for HTTP/HTTPS (terraform), 464X are the default ports for Nomad, 830X are the default ports for Consul.\n  networking.firewall.allowedTCPPorts = [ 80 443 8081 4646 4647 4648 8300 8301 8500 ];\n  networking.firewall.allowedUDPPorts = [ 8301 ];\n\n  # Copy the NixOS configuration file and link it from the resulting system\n  # (/run/current-system/configuration.nix). This is useful in case you\n  # accidentally delete configuration.nix.\n  system.copySystemConfiguration = true;\n \n  # Defines the initial NixOS version for compatibility with older application data.\n  # Do NOT change this value after installation without careful consideration.\n  system.stateVersion = \"24.11\"; # Did you read the comment?\n}\n"
+            }
+          },
+          "sensitive_attributes": [],
+          "identity_schema_version": 0
+        },
+        {
+          "index_key": "jaglan-beta-m04",
+          "schema_version": 0,
+          "attributes": {
+            "id": "5130090292645542538",
+            "triggers": {
+              "configuration_content": "{ config, lib, pkgs, ... }:\n{\n  imports =\n    [ # Include the results of the hardware scan.\n      ./hardware-configuration.nix\n    ];\n\n  nixpkgs.config.allowUnfree = true;\n\n  # Use the systemd-boot EFI boot loader.\n  boot.loader.systemd-boot.enable = true;\n  boot.loader.efi.canTouchEfiVariables = true;\n\n  networking.hostName = \"jaglan-beta-m04\"; # Define your hostname.\n\n  time.timeZone = \"Australia/Melbourne\";\n\n  # List packages installed in system profile. To search, run:\n  # $ nix search wget\n  # environment.systemPackages = with pkgs; [\n  #   vim # Do not forget to add an editor to edit configuration.nix! The Nano editor is also installed by default.\n  #   wget\n  # ];\n\n  # Some programs need SUID wrappers, can be configured further or are\n  # started in user sessions.\n  # programs.mtr.enable = true;\n  # programs.gnupg.agent = {\n  #   enable = true;\n  #   enableSSHSupport = true;\n  # };\n\n  # List services that you want to enable:\n  services = {\n    nomad = {\n      enable = true;\n      enableDocker = true;\n      dropPrivileges = false;\n      settings = {\n        datacenter = \"jaglan-beta\";\n        server = {\n          enabled = true;\n        };\n        client = {\n          enabled = true;\n          preferred_address_family = \"ipv4\";\n          cpu_total_compute = 7200;\n          host_volume = {\n          };\n          cni_path = \"${pkgs.cni-plugins}/bin\";\n        };\n        plugin.docker.config.allow_privileged = true;\n      };\n      extraPackages = with pkgs; [\n        cni-plugins\n        consul\n      ];\n    };\n    consul = {\n      enable = true;\n      webUi = true;\n      interface.bind = \"enp1s0\";\n      interface.advertise = \"enp1s0\";\n      forceAddrFamily = \"ipv4\";\n      extraConfig  = {\n        client_addr = \"{{ GetPrivateInterfaces | exclude \\\"type\\\" \\\"ipv6\\\" | join \\\"address\\\" \\\" \\\" }} {{ GetAllInterfaces | include \\\"flags\\\" \\\"loopback\\\" | join \\\"address\\\" \\\" \\\" }}\";\n        server = true;\n        retry_join = [\n          \"jaglan-beta-m01\"\n          \"jaglan-beta-m02\"\n          \"jaglan-beta-m03\"\n          \"jaglan-beta-m04\"\n          \"jaglan-beta-m05\"\n          \"jaglan-beta-m20\"\n          \"jaglan-beta-m21\"\n          \"jaglan-beta-m22\"\n        ];\n        datacenter = \"jaglan-beta\";\n        connect.enabled = true;\n        ports.grpc = 8502;\n      };\n    };\n    openssh = {\n      enable = true;\n      settings.PermitRootLogin = \"yes\";\n    };\n  };\n\n  systemd.tmpfiles.rules = [\n    # Fix issue where nomad needs alloc_mounts to be writable\n    \"d /var/lib/alloc_mounts 0755 root root -\"\n  ];\n\n  # Open ports in the firewall. 80/443 are for HTTP/HTTPS (terraform), 464X are the default ports for Nomad, 830X are the default ports for Consul.\n  networking.firewall.allowedTCPPorts = [ 80 443 8081 4646 4647 4648 8300 8301 8500 ];\n  networking.firewall.allowedUDPPorts = [ 8301 ];\n\n  # Copy the NixOS configuration file and link it from the resulting system\n  # (/run/current-system/configuration.nix). This is useful in case you\n  # accidentally delete configuration.nix.\n  system.copySystemConfiguration = true;\n \n  # Defines the initial NixOS version for compatibility with older application data.\n  # Do NOT change this value after installation without careful consideration.\n  system.stateVersion = \"24.11\"; # Did you read the comment?\n}\n"
+            }
+          },
+          "sensitive_attributes": [],
+          "identity_schema_version": 0
+        },
+        {
+          "index_key": "jaglan-beta-m05",
+          "schema_version": 0,
+          "attributes": {
+            "id": "1855172426704636197",
+            "triggers": {
+              "configuration_content": "{ config, lib, pkgs, ... }:\n{\n  imports =\n    [ # Include the results of the hardware scan.\n      ./hardware-configuration.nix\n    ];\n\n  nixpkgs.config.allowUnfree = true;\n\n  # Use the systemd-boot EFI boot loader.\n  boot.loader.systemd-boot.enable = true;\n  boot.loader.efi.canTouchEfiVariables = true;\n\n  networking.hostName = \"jaglan-beta-m05\"; # Define your hostname.\n\n  time.timeZone = \"Australia/Melbourne\";\n\n  # List packages installed in system profile. To search, run:\n  # $ nix search wget\n  # environment.systemPackages = with pkgs; [\n  #   vim # Do not forget to add an editor to edit configuration.nix! The Nano editor is also installed by default.\n  #   wget\n  # ];\n\n  # Some programs need SUID wrappers, can be configured further or are\n  # started in user sessions.\n  # programs.mtr.enable = true;\n  # programs.gnupg.agent = {\n  #   enable = true;\n  #   enableSSHSupport = true;\n  # };\n\n  # List services that you want to enable:\n  services = {\n    nomad = {\n      enable = true;\n      enableDocker = true;\n      dropPrivileges = false;\n      settings = {\n        datacenter = \"jaglan-beta\";\n        server = {\n          enabled = true;\n        };\n        client = {\n          enabled = true;\n          preferred_address_family = \"ipv4\";\n          cpu_total_compute = 7200;\n          host_volume = {\n          };\n          cni_path = \"${pkgs.cni-plugins}/bin\";\n        };\n        plugin.docker.config.allow_privileged = true;\n      };\n      extraPackages = with pkgs; [\n        cni-plugins\n        consul\n      ];\n    };\n    consul = {\n      enable = true;\n      webUi = true;\n      interface.bind = \"enp1s0\";\n      interface.advertise = \"enp1s0\";\n      forceAddrFamily = \"ipv4\";\n      extraConfig  = {\n        client_addr = \"{{ GetPrivateInterfaces | exclude \\\"type\\\" \\\"ipv6\\\" | join \\\"address\\\" \\\" \\\" }} {{ GetAllInterfaces | include \\\"flags\\\" \\\"loopback\\\" | join \\\"address\\\" \\\" \\\" }}\";\n        server = true;\n        retry_join = [\n          \"jaglan-beta-m01\"\n          \"jaglan-beta-m02\"\n          \"jaglan-beta-m03\"\n          \"jaglan-beta-m04\"\n          \"jaglan-beta-m05\"\n          \"jaglan-beta-m20\"\n          \"jaglan-beta-m21\"\n          \"jaglan-beta-m22\"\n        ];\n        datacenter = \"jaglan-beta\";\n        connect.enabled = true;\n        ports.grpc = 8502;\n      };\n    };\n    openssh = {\n      enable = true;\n      settings.PermitRootLogin = \"yes\";\n    };\n  };\n\n  systemd.tmpfiles.rules = [\n    # Fix issue where nomad needs alloc_mounts to be writable\n    \"d /var/lib/alloc_mounts 0755 root root -\"\n  ];\n\n  # Open ports in the firewall. 80/443 are for HTTP/HTTPS (terraform), 464X are the default ports for Nomad, 830X are the default ports for Consul.\n  networking.firewall.allowedTCPPorts = [ 80 443 8081 4646 4647 4648 8300 8301 8500 ];\n  networking.firewall.allowedUDPPorts = [ 8301 ];\n\n  # Copy the NixOS configuration file and link it from the resulting system\n  # (/run/current-system/configuration.nix). This is useful in case you\n  # accidentally delete configuration.nix.\n  system.copySystemConfiguration = true;\n \n  # Defines the initial NixOS version for compatibility with older application data.\n  # Do NOT change this value after installation without careful consideration.\n  system.stateVersion = \"24.11\"; # Did you read the comment?\n}\n"
+            }
+          },
+          "sensitive_attributes": [],
+          "identity_schema_version": 0
         },
         {
           "index_key": "jaglan-beta-m20",
           "schema_version": 0,
           "attributes": {
-            "id": "7569759743571304983",
+            "id": "2979501464234412040",
             "triggers": {
-              "configuration_content": "{ config, lib, pkgs, ... }:\n{\n  imports =\n    [ # Include the results of the hardware scan.\n      ./hardware-configuration.nix\n    ];\n\n  nixpkgs.config.allowUnfree = true;\n\n  # Use the systemd-boot EFI boot loader.\n  boot.loader.systemd-boot.enable = true;\n  boot.loader.efi.canTouchEfiVariables = true;\n\n  networking.hostName = \"jaglan-beta-m20\"; # Define your hostname.\n\n  time.timeZone = \"Australia/Melbourne\";\n\n  # List packages installed in system profile. To search, run:\n  # $ nix search wget\n  # environment.systemPackages = with pkgs; [\n  #   vim # Do not forget to add an editor to edit configuration.nix! The Nano editor is also installed by default.\n  #   wget\n  # ];\n\n  # Some programs need SUID wrappers, can be configured further or are\n  # started in user sessions.\n  # programs.mtr.enable = true;\n  # programs.gnupg.agent = {\n  #   enable = true;\n  #   enableSSHSupport = true;\n  # };\n\n  # List services that you want to enable:\n  services = {\n    nomad = {\n      enable = true;\n      enableDocker = true;\n      dropPrivileges = false;\n      settings = {\n        datacenter = \"jaglan-beta\";\n        server = {\n          enabled = true;\n        };\n        client = {\n          enabled = true;\n          host_volume = {\n          };\n          cni_path = \"${pkgs.cni-plugins}/bin\";\n        };\n        plugin.docker.config.allow_privileged = true;\n      };\n      extraPackages = with pkgs; [\n        cni-plugins\n        consul\n      ];\n    };\n    consul = {\n      enable = true;\n      webUi = true;\n      interface.bind = \"ens2\";\n      interface.advertise = \"ens2\";\n      extraConfig  = {\n        client_addr = \"{{ GetPrivateInterfaces | exclude \\\"type\\\" \\\"ipv6\\\" | join \\\"address\\\" \\\" \\\" }} {{ GetAllInterfaces | include \\\"flags\\\" \\\"loopback\\\" | join \\\"address\\\" \\\" \\\" }}\";\n        server = true;\n        retry_join = [\n          \"jaglan-beta-m01\"\n          \"jaglan-beta-m20\"\n        ];\n        datacenter = \"jaglan-beta\";\n        connect.enabled = true;\n        ports.grpc = 8502;\n      };\n    };\n    openssh = {\n      enable = true;\n      settings.PermitRootLogin = \"yes\";\n    };\n  };\n\n  systemd.tmpfiles.rules = [\n    # Fix issue where nomad needs alloc_mounts to be writable\n    \"d /var/lib/alloc_mounts 0755 root root -\"\n  ];\n\n  # Open ports in the firewall. 80/443 are for HTTP/HTTPS (terraform), 464X are the default ports for Nomad, 830X are the default ports for Consul.\n  networking.firewall.allowedTCPPorts = [ 80 443 4646 4647 4648 8300 8301 ];\n  networking.firewall.allowedUDPPorts = [ 8301 ];\n\n  # Copy the NixOS configuration file and link it from the resulting system\n  # (/run/current-system/configuration.nix). This is useful in case you\n  # accidentally delete configuration.nix.\n  system.copySystemConfiguration = true;\n \n  # Defines the initial NixOS version for compatibility with older application data.\n  # Do NOT change this value after installation without careful consideration.\n  system.stateVersion = \"24.11\"; # Did you read the comment?\n}\n"
+              "configuration_content": "{ config, lib, pkgs, ... }:\n{\n  imports =\n    [ # Include the results of the hardware scan.\n      ./hardware-configuration.nix\n    ];\n\n  nixpkgs.config.allowUnfree = true;\n\n  # Use the systemd-boot EFI boot loader.\n  boot.loader.systemd-boot.enable = true;\n  boot.loader.efi.canTouchEfiVariables = true;\n\n  networking.hostName = \"jaglan-beta-m20\"; # Define your hostname.\n\n  time.timeZone = \"Australia/Melbourne\";\n\n  # List packages installed in system profile. To search, run:\n  # $ nix search wget\n  # environment.systemPackages = with pkgs; [\n  #   vim # Do not forget to add an editor to edit configuration.nix! The Nano editor is also installed by default.\n  #   wget\n  # ];\n\n  # Some programs need SUID wrappers, can be configured further or are\n  # started in user sessions.\n  # programs.mtr.enable = true;\n  # programs.gnupg.agent = {\n  #   enable = true;\n  #   enableSSHSupport = true;\n  # };\n\n  # List services that you want to enable:\n  services = {\n    nomad = {\n      enable = true;\n      enableDocker = true;\n      dropPrivileges = false;\n      settings = {\n        datacenter = \"jaglan-beta\";\n        server = {\n          enabled = true;\n          bootstrap_expect = 1;\n        };\n        client = {\n          enabled = true;\n          preferred_address_family = \"ipv4\";\n          cpu_total_compute = 10000;\n          host_volume = {\n          };\n          cni_path = \"${pkgs.cni-plugins}/bin\";\n        };\n        plugin.docker.config.allow_privileged = true;\n      };\n      extraPackages = with pkgs; [\n        cni-plugins\n        consul\n      ];\n    };\n    consul = {\n      enable = true;\n      webUi = true;\n      interface.bind = \"ens2\";\n      interface.advertise = \"ens2\";\n      forceAddrFamily = \"ipv4\";\n      extraConfig  = {\n        client_addr = \"{{ GetPrivateInterfaces | exclude \\\"type\\\" \\\"ipv6\\\" | join \\\"address\\\" \\\" \\\" }} {{ GetAllInterfaces | include \\\"flags\\\" \\\"loopback\\\" | join \\\"address\\\" \\\" \\\" }}\";\n        bootstrap_expect = 1;\n        server = true;\n        retry_join = [\n          \"jaglan-beta-m01\"\n          \"jaglan-beta-m02\"\n          \"jaglan-beta-m03\"\n          \"jaglan-beta-m04\"\n          \"jaglan-beta-m05\"\n          \"jaglan-beta-m20\"\n          \"jaglan-beta-m21\"\n          \"jaglan-beta-m22\"\n        ];\n        datacenter = \"jaglan-beta\";\n        connect.enabled = true;\n        ports.grpc = 8502;\n      };\n    };\n    openssh = {\n      enable = true;\n      settings.PermitRootLogin = \"yes\";\n    };\n  };\n\n  systemd.tmpfiles.rules = [\n    # Fix issue where nomad needs alloc_mounts to be writable\n    \"d /var/lib/alloc_mounts 0755 root root -\"\n  ];\n\n  # Open ports in the firewall. 80/443 are for HTTP/HTTPS (terraform), 464X are the default ports for Nomad, 830X are the default ports for Consul.\n  networking.firewall.allowedTCPPorts = [ 80 443 8081 4646 4647 4648 8300 8301 8500 ];\n  networking.firewall.allowedUDPPorts = [ 8301 ];\n\n  # Copy the NixOS configuration file and link it from the resulting system\n  # (/run/current-system/configuration.nix). This is useful in case you\n  # accidentally delete configuration.nix.\n  system.copySystemConfiguration = true;\n \n  # Defines the initial NixOS version for compatibility with older application data.\n  # Do NOT change this value after installation without careful consideration.\n  system.stateVersion = \"24.11\"; # Did you read the comment?\n}\n"
             }
           },
-          "sensitive_attributes": []
+          "sensitive_attributes": [],
+          "identity_schema_version": 0
+        },
+        {
+          "index_key": "jaglan-beta-m21",
+          "schema_version": 0,
+          "attributes": {
+            "id": "6363236121611066492",
+            "triggers": {
+              "configuration_content": "{ config, lib, pkgs, ... }:\n{\n  imports =\n    [ # Include the results of the hardware scan.\n      ./hardware-configuration.nix\n    ];\n\n  nixpkgs.config.allowUnfree = true;\n\n  # Use the systemd-boot EFI boot loader.\n  boot.loader.systemd-boot.enable = true;\n  boot.loader.efi.canTouchEfiVariables = true;\n\n  networking.hostName = \"jaglan-beta-m21\"; # Define your hostname.\n\n  time.timeZone = \"Australia/Melbourne\";\n\n  # List packages installed in system profile. To search, run:\n  # $ nix search wget\n  # environment.systemPackages = with pkgs; [\n  #   vim # Do not forget to add an editor to edit configuration.nix! The Nano editor is also installed by default.\n  #   wget\n  # ];\n\n  # Some programs need SUID wrappers, can be configured further or are\n  # started in user sessions.\n  # programs.mtr.enable = true;\n  # programs.gnupg.agent = {\n  #   enable = true;\n  #   enableSSHSupport = true;\n  # };\n\n  # List services that you want to enable:\n  services = {\n    nomad = {\n      enable = true;\n      enableDocker = true;\n      dropPrivileges = false;\n      settings = {\n        datacenter = \"jaglan-beta\";\n        server = {\n          enabled = true;\n        };\n        client = {\n          enabled = true;\n          preferred_address_family = \"ipv4\";\n          cpu_total_compute = 10000;\n          host_volume = {\n            traefik = {\n              path = \"/opt/traefik\";\n              read_only = false;\n            };\n            postgres = {\n              path = \"/opt/postgres\";\n              read_only = false;\n            };\n          };\n          cni_path = \"${pkgs.cni-plugins}/bin\";\n        };\n        plugin.docker.config.allow_privileged = true;\n      };\n      extraPackages = with pkgs; [\n        cni-plugins\n        consul\n      ];\n    };\n    consul = {\n      enable = true;\n      webUi = true;\n      interface.bind = \"ens2\";\n      interface.advertise = \"ens2\";\n      forceAddrFamily = \"ipv4\";\n      extraConfig  = {\n        client_addr = \"{{ GetPrivateInterfaces | exclude \\\"type\\\" \\\"ipv6\\\" | join \\\"address\\\" \\\" \\\" }} {{ GetAllInterfaces | include \\\"flags\\\" \\\"loopback\\\" | join \\\"address\\\" \\\" \\\" }}\";\n        server = true;\n        retry_join = [\n          \"jaglan-beta-m01\"\n          \"jaglan-beta-m02\"\n          \"jaglan-beta-m03\"\n          \"jaglan-beta-m04\"\n          \"jaglan-beta-m05\"\n          \"jaglan-beta-m20\"\n          \"jaglan-beta-m21\"\n          \"jaglan-beta-m22\"\n        ];\n        datacenter = \"jaglan-beta\";\n        connect.enabled = true;\n        ports.grpc = 8502;\n      };\n    };\n    openssh = {\n      enable = true;\n      settings.PermitRootLogin = \"yes\";\n    };\n  };\n\n  systemd.tmpfiles.rules = [\n    # Fix issue where nomad needs alloc_mounts to be writable\n    \"d /var/lib/alloc_mounts 0755 root root -\"\n    # Create a directory for traefik to store its data\n    \"d /opt/traefik 0755 root root -\"\n    # Create a directory for postgres to store its data\n    \"d /opt/postgres 0755 root root -\"\n  ];\n\n  # Open ports in the firewall. 80/443 are for HTTP/HTTPS (terraform), 464X are the default ports for Nomad, 830X are the default ports for Consul.\n  networking.firewall.allowedTCPPorts = [ 80 443 8081 4646 4647 4648 8300 8301 8500 ];\n  networking.firewall.allowedUDPPorts = [ 8301 ];\n\n  # Copy the NixOS configuration file and link it from the resulting system\n  # (/run/current-system/configuration.nix). This is useful in case you\n  # accidentally delete configuration.nix.\n  system.copySystemConfiguration = true;\n \n  # Defines the initial NixOS version for compatibility with older application data.\n  # Do NOT change this value after installation without careful consideration.\n  system.stateVersion = \"24.11\"; # Did you read the comment?\n}\n"
+            }
+          },
+          "sensitive_attributes": [],
+          "identity_schema_version": 0
+        },
+        {
+          "index_key": "jaglan-beta-m22",
+          "schema_version": 0,
+          "attributes": {
+            "id": "1381056892788237526",
+            "triggers": {
+              "configuration_content": "{ config, lib, pkgs, ... }:\n{\n  imports =\n    [ # Include the results of the hardware scan.\n      ./hardware-configuration.nix\n    ];\n\n  nixpkgs.config.allowUnfree = true;\n\n  # Use the systemd-boot EFI boot loader.\n  boot.loader.systemd-boot.enable = true;\n  boot.loader.efi.canTouchEfiVariables = true;\n\n  networking.hostName = \"jaglan-beta-m22\"; # Define your hostname.\n\n  time.timeZone = \"Australia/Melbourne\";\n\n  # List packages installed in system profile. To search, run:\n  # $ nix search wget\n  # environment.systemPackages = with pkgs; [\n  #   vim # Do not forget to add an editor to edit configuration.nix! The Nano editor is also installed by default.\n  #   wget\n  # ];\n\n  # Some programs need SUID wrappers, can be configured further or are\n  # started in user sessions.\n  # programs.mtr.enable = true;\n  # programs.gnupg.agent = {\n  #   enable = true;\n  #   enableSSHSupport = true;\n  # };\n\n  # List services that you want to enable:\n  services = {\n    nomad = {\n      enable = true;\n      enableDocker = true;\n      dropPrivileges = false;\n      settings = {\n        datacenter = \"jaglan-beta\";\n        server = {\n          enabled = true;\n        };\n        client = {\n          enabled = true;\n          cpu_total_compute = 10000;\n          host_volume = {\n          };\n          cni_path = \"${pkgs.cni-plugins}/bin\";\n        };\n        plugin.docker.config.allow_privileged = true;\n      };\n      extraPackages = with pkgs; [\n        cni-plugins\n        consul\n      ];\n    };\n    consul = {\n      enable = true;\n      webUi = true;\n      interface.bind = \"ens2\";\n      interface.advertise = \"ens2\";\n      extraConfig  = {\n        client_addr = \"{{ GetPrivateInterfaces | exclude \\\"type\\\" \\\"ipv6\\\" | join \\\"address\\\" \\\" \\\" }} {{ GetAllInterfaces | include \\\"flags\\\" \\\"loopback\\\" | join \\\"address\\\" \\\" \\\" }}\";\n        server = true;\n        retry_join = [\n          \"jaglan-beta-m01\"\n          \"jaglan-beta-m20\"\n          \"jaglan-beta-m21\"\n          \"jaglan-beta-m22\"\n        ];\n        datacenter = \"jaglan-beta\";\n        connect.enabled = true;\n        ports.grpc = 8502;\n      };\n    };\n    openssh = {\n      enable = true;\n      settings.PermitRootLogin = \"yes\";\n    };\n  };\n\n  systemd.tmpfiles.rules = [\n    # Fix issue where nomad needs alloc_mounts to be writable\n    \"d /var/lib/alloc_mounts 0755 root root -\"\n  ];\n\n  # Open ports in the firewall. 80/443 are for HTTP/HTTPS (terraform), 464X are the default ports for Nomad, 830X are the default ports for Consul.\n  networking.firewall.allowedTCPPorts = [ 80 443 8081 4646 4647 4648 8300 8301 8500 ];\n  networking.firewall.allowedUDPPorts = [ 8301 ];\n\n  # Copy the NixOS configuration file and link it from the resulting system\n  # (/run/current-system/configuration.nix). This is useful in case you\n  # accidentally delete configuration.nix.\n  system.copySystemConfiguration = true;\n \n  # Defines the initial NixOS version for compatibility with older application data.\n  # Do NOT change this value after installation without careful consideration.\n  system.stateVersion = \"24.11\"; # Did you read the comment?\n}\n"
+            }
+          },
+          "sensitive_attributes": [],
+          "identity_schema_version": 0
         }
       ]
     }

2-nomad-config/1-data.tf

@@ -0,0 +1,19 @@
+
+module "data" {
+  source = "./1-data"
+}
+
+data "nomad_plugin" "smb" {
+  plugin_id        = "smb"
+  wait_for_healthy = true
+}
+
+provider "postgresql" {
+  host            = "jaglan-beta-m21.lan"
+  port            = 5432
+  database        = "postgres"
+  username        = "postgres"
+  password        = data.sops_file.secrets.data["postgres.postgres"]
+  sslmode         = "disable"
+  connect_timeout = 15
+}

2-nomad-config/1-data/csi-smb.tf

@@ -0,0 +1,5 @@
+
+resource "nomad_job" "csi-smb" {
+  jobspec = file("${path.module}/csi-smb.nomad.hcl")
+}
+

2-nomad-config/1-data/main.tf

@@ -0,0 +1,25 @@
+terraform {
+  required_providers {
+    sops = {
+      source  = "carlpett/sops"
+      version = "~> 0.5"
+    }
+    postgresql = {
+      source = "cyrilgdn/postgresql"
+    }
+  }
+}
+
+provider "nomad" {
+  address = "http://jaglan-beta-m20.lan:4646"
+}
+
+data "sops_file" "secrets" {
+  source_file = "secrets/secrets.enc.json"
+}
+
+data "nomad_plugin" "smb" {
+  plugin_id        = "smb"
+  wait_for_healthy = true
+}
+

2-nomad-config/1-data/postgres.nomad.hcl

@@ -13,7 +13,10 @@ job "postgres" {
       driver = "docker"
 
       config {
-        image = "postgres:latest"
+        # Temporarily pin to v17 as v18 moved the default data directory and immich doesn't officially support it yet
+        # immich also needs >= 0.3.0, < 0.5.0. https://docs.immich.app/administration/postgres-standalone/#prerequisites
+        #image = "postgres:17"  
+        image = "tensorchord/vchord-postgres:pg17-v0.4.3"
         ports = ["db"]
 
         volumes = [
@@ -35,7 +38,7 @@ job "postgres" {
 
       resources {
         cpu    = 500
-        memory = 512
+        memory = 1024
       }
 
       template {

2-nomad-config/1-data/postgres.tf

@@ -1,14 +1,14 @@
 resource "nomad_job" "postgres" {
-  jobspec       = file("postgres.nomad.hcl")
+  jobspec       = file("${path.module}/postgres.nomad.hcl")
   rerun_if_dead = true
 }
 
 resource "nomad_job" "pgadmin" {
-  jobspec = file("pgadmin.nomad.hcl")
+  jobspec = file("${path.module}/pgadmin.nomad.hcl")
 }
 
 resource "nomad_job" "pgbackup" {
-  jobspec = file("pgbackup.nomad.hcl")
+  jobspec = file("${path.module}/pgbackup.nomad.hcl")
 }
 
 resource "nomad_variable" "postgres" {
@@ -18,16 +18,6 @@ resource "nomad_variable" "postgres" {
   }
 }
 
-provider "postgresql" {
-  host            = "jaglan-beta-m01.lan"
-  port            = 5432
-  database        = "postgres"
-  username        = "postgres"
-  password        = data.sops_file.secrets.data["postgres.postgres"]
-  sslmode         = "disable"
-  connect_timeout = 15
-}
-
 resource "nomad_csi_volume_registration" "unraid_database_dump" {
   #Note: Before chaning the definition of this volume you need to stop the jobs that are using it
   depends_on = [data.nomad_plugin.smb]

2-nomad-config/2-ingress.tf

@@ -0,0 +1,31 @@
+
+module "ingress" {
+  source = "./2-ingress"
+}
+
+# traefik.tf
+
+moved {
+  from = cloudflare_dns_record.othrayte-one
+  to   = module.ingress.cloudflare_dns_record.othrayte-one
+}
+
+moved {
+  from = cloudflare_dns_record.star-othrayte-one
+  to   = module.ingress.cloudflare_dns_record.star-othrayte-one
+}
+
+moved {
+  from = nomad_variable.traefik
+  to   = module.ingress.nomad_variable.traefik
+}
+
+moved {
+  from = nomad_job.traefik
+  to   = module.ingress.nomad_job.traefik
+}
+
+moved {
+  from = nomad_csi_volume_registration.unraid_appdata_traefik
+  to   = module.ingress.nomad_csi_volume_registration.unraid_appdata_traefik
+}

2-nomad-config/2-ingress/authelia.nomad.hcl

@@ -28,6 +28,10 @@ job "authelia" {
         "traefik.enable=true",
       ]
 
+      connect {
+        sidecar_service {}
+      }
+
       check {
         type     = "http"
         path     = "/health"
@@ -77,7 +81,9 @@ access_control:
 session:
   name: authelia_session
   secret: "{{ with nomadVar "nomad/jobs/authelia" }}{{ .session_secret }}{{ end }}"
-  expiration: "1 day and 9 hours"
+  inactivity: '2 days and 9 hours'
+  expiration: '1 hour'
+  remember_me: '90 days'
   cookies:
     - domain: othrayte.one
       authelia_url: "https://auth.othrayte.one"

2-nomad-config/2-ingress/authelia.tf

@@ -1,5 +1,5 @@
 resource "nomad_job" "authelia" {
-  jobspec = file("authelia.nomad.hcl")
+  jobspec = file("${path.module}/authelia.nomad.hcl")
 }
 
 resource "postgresql_role" "authelia" {

2-nomad-config/2-ingress/main.tf

@@ -0,0 +1,33 @@
+terraform {
+  required_providers {
+    sops = {
+      source  = "carlpett/sops"
+      version = "~> 0.5"
+    }
+    cloudflare = {
+      source  = "cloudflare/cloudflare"
+      version = "~> 5"
+    }
+    postgresql = {
+      source = "cyrilgdn/postgresql"
+    }
+  }
+}
+
+provider "nomad" {
+  address = "http://jaglan-beta-m20.lan:4646"
+}
+
+data "sops_file" "secrets" {
+  source_file = "secrets/secrets.enc.json"
+}
+
+provider "cloudflare" {
+  api_token = data.sops_file.secrets.data["cloudflare.api_token"]
+}
+
+data "nomad_plugin" "smb" {
+  plugin_id        = "smb"
+  wait_for_healthy = true
+}
+

2-nomad-config/2-ingress/traefik.nomad.hcl

@@ -0,0 +1,248 @@
+job "traefik" {
+  group "traefik" {
+    count = 2
+
+    network {
+      mode = "bridge"
+      port "http" {
+        static = 80
+      }
+
+      port "https" {
+        static = 443
+      }
+
+      port "api" {
+        static = 8081
+      }
+    }
+    service {
+      connect {
+        sidecar_service {
+          proxy {
+            upstreams {
+              destination_name = "auth"
+              local_bind_port  = 9091
+            }
+          }
+        }
+      }
+    }
+
+    service {
+      name = "traefik"
+      port = "api"
+
+      check {
+        name     = "alive"
+        type     = "tcp"
+        port     = "api"
+        interval = "10s"
+        timeout  = "2s"
+      }
+    }
+
+    task "traefik" {
+      driver = "docker"
+
+      config {
+        image = "traefik:v3.3"
+        ports = ["http", "https", "api"]
+
+        volumes = [
+          "local/traefik.yml:/etc/traefik/traefik.yml",
+          "local/configs/:/etc/traefik/configs/"
+        ]
+      }
+
+      volume_mount {
+        volume      = "unraid_appdata_traefik"
+        destination = "/opt/traefik"
+        read_only   = false
+      }
+
+      template {
+        data = <<EOF
+log:
+  level: INFO
+entryPoints:
+  web:
+    address: ":80"
+    http:
+      redirections:
+        entryPoint:
+          to: websecure
+          scheme: https
+  websecure:
+    address: ":443"
+    http:
+      tls:
+        certResolver: letsencrypt
+  traefik:
+    address: ":8081"
+
+api:
+  dashboard: true
+  insecure: true
+
+providers:
+  file:
+    directory: "/etc/traefik/configs/"
+
+  consulCatalog:
+    prefix: "traefik"
+    exposedByDefault: false
+    defaultRule: {{"Host(`{{ .Name }}.othrayte.one`)"}}
+    endpoint:
+      address: "{{ env "NOMAD_HOST_IP_http" }}:8500"
+      scheme: "http"
+
+certificatesResolvers:
+  letsencrypt:
+    acme:
+      email: "othrayte@gmail.com"
+      storage: "/opt/traefik/acme.json"
+      httpChallenge:
+        entryPoint: web
+EOF
+
+        destination = "local/traefik.yml"
+      }
+
+      template {
+        data = <<EOF
+http:
+  middlewares:
+    auth:
+      forwardAuth:
+        address: "http://localhost:9091/api/authz/forward-auth"
+        trustForwardHeader: true
+    auth-allow-token:
+      chain:
+        middlewares:
+          - auth
+    inject-kopia-basic-auth:
+      headers:
+        customRequestHeaders:
+          Authorization: "Basic {{ with nomadVar "nomad/jobs/traefik" }}{{ .kopia_basic_auth }}{{ end }}"
+  routers:
+    fallback:
+      rule: "HostRegexp(`^.+$`)"
+      entryPoints:
+        - websecure
+      middlewares:
+        - auth
+      service: noop@internal  # This router just applies middleware
+      priority: 1
+    traefik:
+      rule: "Host(`traefik.othrayte.one`)"
+      service: traefik
+      middlewares:
+        - auth
+    nomad-ui:
+      rule: "Host(`nomad.othrayte.one`)"
+      service: nomad-ui
+      middlewares:
+        - auth
+    consul-ui:
+      rule: "Host(`consul.othrayte.one`)"
+      service: consul-ui
+      middlewares: 
+        - auth
+    unraid:
+      rule: "Host(`unraid.othrayte.one`)"
+      service: unraid
+      middlewares:
+        - auth
+    frigate:
+      rule: "Host(`frigate.othrayte.one`)"
+      service: frigate
+      middlewares:
+        - auth
+    kopia:
+      rule: "Host(`kopia.othrayte.one`)"
+      service: kopia
+      middlewares:
+        - auth
+        - inject-kopia-basic-auth
+    hass:
+      rule: "Host(`hass.othrayte.one`)"
+      service: hass
+      middlewares:
+        - auth
+    hass-token:
+      rule: "Host(`${hass_magic_token}-hass.othrayte.one`)"
+      service: hass
+
+  services:
+    traefik:
+      loadBalancer:
+        servers:
+          - url: "http://localhost:8081"
+    nomad-ui:
+      loadBalancer:
+        servers:
+          - url: "http://{{ env "NOMAD_HOST_IP_http" }}:4646"
+    consul-ui:
+      loadBalancer:
+        servers:
+          - url: "http://{{ env "NOMAD_HOST_IP_http" }}:8500"
+    unraid:
+      loadBalancer:
+        servers:
+          - url: "http://192.168.1.192:80"
+    frigate:
+      loadBalancer:
+        servers:
+          - url: "http://192.168.1.192:5000"
+    kopia:
+      loadBalancer:
+        servers:
+          - url: "http://192.168.1.192:51515"
+    hass:
+      loadBalancer:
+        servers:
+          - url: "http://192.168.1.234:8123"
+EOF
+
+        destination = "local/configs/nomad.yml"
+      }
+
+      resources {
+        cpu    = 100
+        memory = 128
+      }
+    }
+
+    volume "unraid_appdata_traefik" {
+      type            = "csi"
+      read_only       = false
+      source          = "unraid_appdata_traefik"
+      access_mode     = "multi-node-multi-writer"
+      attachment_mode = "file-system"
+
+      mount_options {
+        mount_flags = ["file_mode=0600", "uid=1000", "gid=1000"]
+      }
+    }
+
+    task "cloudflared" {
+      driver = "docker"
+
+      config {
+        image = "cloudflare/cloudflared:latest"
+        args = [
+          "tunnel", "--no-autoupdate", "run"
+        ]
+      }
+
+      template {
+        data        = <<EOH
+TUNNEL_TOKEN="{{ with nomadVar "nomad/jobs/traefik" }}{{ .cf_tunnel_token }}{{ end }}"
+EOH
+        destination = "secrets/tunnel.env"
+        env         = true # Load the file as environment variables
+      }
+    }
+  }
+}

2-nomad-config/2-ingress/traefik.tf

@@ -0,0 +1,59 @@
+resource "cloudflare_dns_record" "othrayte-one" {
+  comment = "othrayte.one proxy via cloudflared tunnel to traefik"
+  zone_id = "2616ab2a44d0645b03fbc3106c79bd99"
+  type    = "CNAME"
+  name    = "othrayte.one"
+  content = "59ca3eb1-5f0b-45e1-97ff-e373569c6689.cfargotunnel.com"
+  proxied = true
+  ttl     = 1 # Auto
+}
+
+resource "cloudflare_dns_record" "star-othrayte-one" {
+  comment = "*.othrayte.one proxy via cloudflared tunnel to traefik"
+  zone_id = "2616ab2a44d0645b03fbc3106c79bd99"
+  type    = "CNAME"
+  name    = "*.othrayte.one"
+  content = "59ca3eb1-5f0b-45e1-97ff-e373569c6689.cfargotunnel.com"
+  proxied = true
+  ttl     = 1 # Auto
+}
+
+resource "nomad_variable" "traefik" {
+  path = "nomad/jobs/traefik"
+  items = {
+    cf_tunnel_token  = data.sops_file.secrets.data["traefik.cf_tunnel_token"]
+    kopia_basic_auth = data.sops_file.secrets.data["traefik.kopia_basic_auth"]
+  }
+}
+
+resource "nomad_job" "traefik" {
+  jobspec = templatefile("${path.module}/traefik.nomad.hcl", {
+    hass_magic_token = nonsensitive(data.sops_file.secrets.data["hass.magic-token"])
+  })
+}
+
+resource "nomad_csi_volume_registration" "unraid_appdata_traefik" {
+  #Note: Before chaning the definition of this volume you need to stop the jobs that are using it
+  depends_on = [data.nomad_plugin.smb]
+  plugin_id  = "smb"
+
+  volume_id = "unraid_appdata_traefik"
+  name      = "unraid_appdata_traefik"
+
+  external_id = "unraid_appdata_traefik"
+
+  capability {
+    access_mode     = "multi-node-multi-writer"
+    attachment_mode = "file-system"
+  }
+
+  context = {
+    source = "//192.168.1.192/appdata"
+    subDir = "traefik" # Note: Needs to be manually created on the share
+  }
+
+  secrets = {
+    "username" = "nomad"
+    "password" = data.sops_file.secrets.data["unraid.nomad"]
+  }
+}

2-nomad-config/immich.nomad.hcl

@@ -0,0 +1,209 @@
+job "immich" {
+  group "immich" {
+    network {
+      mode = "bridge"
+      port "http" {
+        to = 2283
+      }
+    }
+    service {
+      connect {
+        sidecar_service {
+          proxy {
+            # TODO https://docs.immich.app/administration/postgres-standalone#prerequisites
+            upstreams {
+              destination_name = "postgres"
+              local_bind_port  = 5432
+            }
+          }
+        }
+      }
+    }
+
+    task "immich-server" {
+      driver = "docker"
+
+      config {
+        image = "ghcr.io/immich-app/immich-server:release"
+        ports = ["http"]
+      }
+
+      service {
+        name = "immich"
+        port = "http"
+
+        tags = [
+          "traefik.enable=true",
+          "traefik.http.routers.immich.middlewares=auth@file",
+          "traefik.http.routers.immich-token.rule=Host(`c3ll7nbevl5j4j8rcnfxnr95q48fuayz-immich.othrayte.one`)",
+        ]
+
+        # Use the Immich server ping endpoint
+        # See: https://api.immich.app/endpoints/server/pingServer
+        check {
+          name     = "alive"
+          type     = "http"
+          path     = "/api/server/ping"
+          method   = "GET"
+          port     = "http"
+          interval = "10s"
+          timeout  = "2s"
+        }
+      }
+
+      env {
+        IMMICH_LOG_LEVEL                 = "log"
+        TZ                               = "Australia/Melbourne"
+        REDIS_HOSTNAME                   = "localhost"
+        DB_HOSTNAME                      = "localhost"
+        IMMICH_IGNORE_MOUNT_CHECK_ERRORS = "true" # Let immich start whilst we figure out what's wrong with the mount permissions
+      }
+
+      volume_mount {
+        volume      = "unraid_appdata_immich"
+        destination = "/data"
+        read_only   = false
+      }
+
+      volume_mount {
+        volume      = "unraid_media_photosvideos"
+        destination = "/data/library"
+        read_only   = false
+      }
+
+      volume_mount {
+        volume      = "unraid_media_immich_encodedvideo"
+        destination = "/data/encoded-video"
+        read_only   = false
+      }
+
+      volume_mount {
+        volume      = "unraid_mediadump_photosvideos"
+        destination = "/data/upload"
+        read_only   = false
+      }
+
+      resources {
+        cpu        = 200
+        memory     = 512
+        memory_max = 1500
+      }
+
+      template {
+        data        = <<EOH
+DB_USERNAME="immich"
+DB_PASSWORD="{{ with nomadVar "nomad/jobs/immich" }}{{ .database_pw }}{{ end }}"
+EOH
+        destination = "secrets/db.env"
+        env         = true # Load the file as environment variables
+      }
+    }
+
+    volume "unraid_appdata_immich" {
+      type            = "csi"
+      read_only       = false
+      source          = "unraid_appdata_immich"
+      access_mode     = "single-node-writer"
+      attachment_mode = "file-system"
+
+      mount_options {
+        mount_flags = ["uid=1000", "gid=1000"]
+      }
+    }
+
+    volume "unraid_media_photosvideos" {
+      type            = "csi"
+      read_only       = false
+      source          = "unraid_media_photosvideos"
+      access_mode     = "single-node-writer"
+      attachment_mode = "file-system"
+
+      mount_options {
+        mount_flags = ["uid=1000", "gid=1000"]
+      }
+    }
+
+    volume "unraid_media_immich_encodedvideo" {
+      type            = "csi"
+      read_only       = false
+      source          = "unraid_media_immich_encodedvideo"
+      access_mode     = "single-node-writer"
+      attachment_mode = "file-system"
+
+      mount_options {
+        mount_flags = ["uid=1000", "gid=1000"]
+      }
+    }
+
+    volume "unraid_mediadump_photosvideos" {
+      type            = "csi"
+      read_only       = false
+      source          = "unraid_mediadump_photosvideos"
+      access_mode     = "single-node-writer"
+      attachment_mode = "file-system"
+
+      mount_options {
+        mount_flags = ["uid=1000", "gid=1000"]
+      }
+    }
+
+    task "immich-machine-learning" {
+      driver = "docker"
+      config {
+        image = "ghcr.io/immich-app/immich-machine-learning:release"
+      }
+
+      resources {
+        cpu        = 200
+        memory     = 500
+        memory_max = 2000
+      }
+    }
+
+    task "redis" {
+      driver = "docker"
+
+      config {
+        image = "docker.io/valkey/valkey:8-bookworm@sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571"
+      }
+
+      resources {
+        cpu        = 30
+        memory     = 10
+        memory_max = 50
+      }
+    }
+
+    task "tailscale" {
+      driver = "docker"
+
+      config {
+        image = "tailscale/tailscale:latest"
+      }
+
+      env = {
+        TS_HOSTNAME   = "immich"
+        TS_EXTRA_ARGS = "--advertise-tags=tag:nomad"
+      }
+
+      resources {
+        cpu        = 100
+        memory     = 100
+        memory_max = 300
+      }
+
+      lifecycle {
+        hook    = "prestart"
+        sidecar = true
+      }
+
+      template {
+        data        = <<EOH
+TS_AUTHKEY="{{ with nomadVar "nomad/jobs/immich" }}{{ .tailscale_oauthsecret }}{{ end }}?ephemeral=true"
+EOH
+        destination = "secrets/ts_oauth.env"
+        env         = true # Load the file as environment variables
+      }
+    }
+  }
+}

2-nomad-config/immich.tf

@@ -0,0 +1,140 @@
+
+resource "nomad_job" "immich" {
+  jobspec = file("immich.nomad.hcl")
+}
+
+resource "postgresql_role" "immich" {
+  name     = "immich"
+  password = data.sops_file.secrets.data["immich.database_pw"]
+  login    = true
+}
+
+resource "postgresql_database" "immich" {
+  name  = "immich"
+  owner = postgresql_role.immich.name
+}
+
+resource "postgresql_extension" "immich_vchord" {
+  name           = "vchord"
+  database       = postgresql_database.immich.name
+  create_cascade = true
+}
+
+resource "postgresql_extension" "immich_earthdistance" {
+  name           = "earthdistance"
+  database       = postgresql_database.immich.name
+  create_cascade = true
+}
+
+resource "nomad_variable" "immich" {
+  path = "nomad/jobs/immich"
+  items = {
+    database_pw           = data.sops_file.secrets.data["immich.database_pw"]
+    tailscale_oauthsecret = data.sops_file.secrets.data["tailscale.oauthsecret"]
+  }
+}
+
+resource "nomad_csi_volume_registration" "unraid_appdata_immich" {
+  #Note: Before chaning the definition of this volume you need to stop the jobs that are using it
+  depends_on = [data.nomad_plugin.smb]
+  plugin_id  = "smb"
+
+  volume_id = "unraid_appdata_immich"
+  name      = "unraid_appdata_immich"
+
+  external_id = "unraid_appdata_immich"
+
+  capability {
+    access_mode     = "single-node-writer"
+    attachment_mode = "file-system"
+  }
+
+  context = {
+    source = "//192.168.1.192/appdata"
+    subDir = "immich" # Note: Needs to be manually created on the share
+  }
+
+  secrets = {
+    "username" = "nomad"
+    "password" = data.sops_file.secrets.data["unraid.nomad"]
+  }
+}
+
+resource "nomad_csi_volume_registration" "unraid_media_photosvideos" {
+  #Note: Before chaning the definition of this volume you need to stop the jobs that are using it
+  depends_on = [data.nomad_plugin.smb]
+  plugin_id  = "smb"
+
+  volume_id = "unraid_media_photosvideos"
+  name      = "unraid_media_photosvideos"
+
+  external_id = "unraid_media_photosvideos"
+
+  capability {
+    access_mode     = "single-node-writer"
+    attachment_mode = "file-system"
+  }
+
+  context = {
+    source = "//192.168.1.192/media"
+    subDir = "Photos and Videos" # Note: Needs to be manually created on the share
+  }
+
+  secrets = {
+    "username" = "nomad"
+    "password" = data.sops_file.secrets.data["unraid.nomad"]
+  }
+}
+
+resource "nomad_csi_volume_registration" "unraid_media_immich_encodedvideo" {
+  #Note: Before chaning the definition of this volume you need to stop the jobs that are using it
+  depends_on = [data.nomad_plugin.smb]
+  plugin_id  = "smb"
+
+  volume_id = "unraid_media_immich_encodedvideo"
+  name      = "unraid_media_immich_encodedvideo"
+
+  external_id = "unraid_media_immich_encodedvideo"
+
+  capability {
+    access_mode     = "single-node-writer"
+    attachment_mode = "file-system"
+  }
+
+  context = {
+    source = "//192.168.1.192/media"
+    subDir = "immich/encoded-video" # Note: Needs to be manually created on the share
+  }
+
+  secrets = {
+    "username" = "nomad"
+    "password" = data.sops_file.secrets.data["unraid.nomad"]
+  }
+}
+
+resource "nomad_csi_volume_registration" "unraid_mediadump_photosvideos" {
+  #Note: Before chaning the definition of this volume you need to stop the jobs that are using it
+  depends_on = [data.nomad_plugin.smb]
+  plugin_id  = "smb"
+
+  volume_id = "unraid_mediadump_photosvideos"
+  name      = "unraid_mediadump_photosvideos"
+
+  external_id = "unraid_mediadump_photosvideos"
+
+  capability {
+    access_mode     = "single-node-writer"
+    attachment_mode = "file-system"
+  }
+
+  context = {
+    source = "//192.168.1.192/media-dump"
+    subDir = "Photos and Videos" # Note: Needs to be manually created on the share
+  }
+
+  secrets = {
+    "username" = "nomad"
+    "password" = data.sops_file.secrets.data["unraid.nomad"]
+  }
+}
+

2-nomad-config/main.tf

@@ -21,22 +21,24 @@ terraform {
 }
 
 provider "nomad" {
-  address = "http://jaglan-beta-m01.lan:4646"
+  address = "http://jaglan-beta-m20.lan:4646"
 }
 
 data "sops_file" "secrets" {
-  source_file = "secrets.enc.json"
+  source_file = "secrets/secrets.enc.json"
 }
 
 provider "cloudflare" {
   api_token = data.sops_file.secrets.data["cloudflare.api_token"]
 }
 
-resource "nomad_job" "csi-smb" {
-  jobspec = file("csi-smb.nomad.hcl")
+resource "nomad_scheduler_config" "config" {
+  memory_oversubscription_enabled = true
+  preemption_config = {
+    batch_scheduler_enabled    = false
+    service_scheduler_enabled  = false
+    sysbatch_scheduler_enabled = false
+    system_scheduler_enabled   = false
+  }
 }
 
-data "nomad_plugin" "smb" {
-  plugin_id        = "smb"
-  wait_for_healthy = true
-}

2-nomad-config/readme.md

@@ -15,4 +15,17 @@ Put the age keys in /home/<user>/.config/sops/age/keys.txt
 
 ## Adding Secrets
 
-Edit the secrets using `sops secrets.enc.json`
+Edit the secrets using `sops secrets/secrets.enc.json`
+
+# Bootstrapping (starting without PostgreSQL running)
+
+terraform apply -target=module.data
+
+## Restoring PostgreSQL DBs
+
+`psql -h jaglan-beta-m21 -p 5432 -U postgres -f ~/Downloads/all_databases.sql postgres`
+
+# Deploying and testing changes
+
+Sometimes the nomad job fails but the solution is to fix another job and so we need to tell nomad to retry the unchanged job.
+`nomad job eval -force-reschedule glance`

2-nomad-config/secrets/secrets.enc.json

@@ -23,6 +23,16 @@
 		"jwt_secret": "ENC[AES256_GCM,data:/dPDqJdn4Af3Wo005V7lU9b8RbN/wyF0Tx66827cdyaZfi4QPOSj23wNqw==,iv:yJW2PiAGGr97q0DoBr64X88eFNpuVPZX0SPyNDp5QjQ=,tag:p27XTUbMC0WDMTNJCscmGQ==,type:str]",
 		"database_pw": "ENC[AES256_GCM,data:EzGPKdsX3Ib2zWrz09kUdegIxGNwg1j4msbOKUmvCGy6R9/EG1nvOC9Z5Oo=,iv:msek112FxmVAwFume6b7RnSICL/sw5CK3XzgCq9Sp1s=,tag:UcxUi2hySv54liN+Ddodpw==,type:str]"
 	},
+	"hass": {
+		"magic-token": "ENC[AES256_GCM,data:3mKbPFgvtX+hWYEZ0q4jBjnR8KM+E/1DqmkVzoV6ROY=,iv:9L748apqK1TcsW0Y0HvU9QHVD/eSh56c/uN/K4KNct4=,tag:ZmXiaPz7MEvaQ0yu3byiKQ==,type:str]"
+	},
+	"traefik": {
+		"cf_tunnel_token": "ENC[AES256_GCM,data:IgrmKwdeipix1dIXNuXnTWN5rCZjClbKZQJfgr5c2IP/n8bcc/nG5Wb42WL2C4hTeVqhG5p62ZXoz0j4dNAjxvuzcW/P0XeSYaiDRXMNWKhNIcK7jOexgswio0sUC+F7f3fa6HH4C02Mx8dWoFZChYtM5EhGdcEwVwspyBlMhTSHTz+/w5T9OqH18o132ZTM6kMQY85sgH36azWoSw73N+aC4ANhgybuok06z6R5D2jMdDX47Bo5bg==,iv:yOcUDTYHh58iejbl0wxNJO1hcDypcBq6KlHKyqnMSVk=,tag:CMyHKgahkIGdXItMJ1/hOg==,type:str]",
+		"kopia_basic_auth": "ENC[AES256_GCM,data:GKJKTtFqW2f8L6VYsBIuNsssUk8vBn74A4TIFw==,iv:rjV0o+CKUtZi8nVsVv0m17OPkYW5ymje9QoWvlRHa7g=,tag:CqEf6n5xgc2RWddbZoNqBQ==,type:str]"
+	},
+	"immich": {
+		"database_pw": "ENC[AES256_GCM,data:SUyMGqu7deZyZpVt,iv:asZehOvn/JamwFyS+Xl9Xpr4JFkKlJjHVw7LywYOxTc=,tag:plRvuv7+ievfEhxurBl7YQ==,type:str]"
+	},
 	"sops": {
 		"age": [
 			{
@@ -30,8 +40,8 @@
 				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSByUWM4ZDVVbGFrUGdMRHBX\nUFBmU3Nlc0RBSzhFK0tHNHpkQXUvUVdiZUZJCmpRN1lFdENpWW0rcThjVlVQNUl6\nWnlLU0RnQ3FZby81Ly8xTFBrek9nMncKLS0tIFQ4UTRNOC9CRmx4OFJWem1wckZz\nUDFTSzdWZldFK3FqcTNWTWRyNDhHQ2MKS811mR5xn7qiC/aVgPFYJ5c6Q3zxRfcr\nHcvxUvB01vNJKZpRg92vvKPkV6lQO3DXCT98OdfwiymlEOvYxg71Pg==\n-----END AGE ENCRYPTED FILE-----\n"
 			}
 		],
-		"lastmodified": "2025-09-06T12:15:59Z",
-		"mac": "ENC[AES256_GCM,data:kiyEudOTWXnF485QoODePBGNACuS6bY7KVZZe9oSPe2jnyyNn4oI3ukxsgZDEN48k4sESvSLN+yCCKx4I14oRYHMFRhLSN4YLivQOEp0XcR3w7wx3ONmNdiyMG+UgEquaCX4/lWDFUVfWkoWQeq8y+ap5LY1ocqZ9zJ+yCilCA4=,iv:qyQJi7Uf+JGDiPt0C6Ww4A7Fa6NGL0aD3B/CfB4pEG0=,tag:ci+amgE24/uiEPIT0aoc+A==,type:str]",
+		"lastmodified": "2025-10-13T12:19:46Z",
+		"mac": "ENC[AES256_GCM,data:QJ1Prqf37xMZbvyMvjBVxZOiOr07CmCYrWmr+5hwDsEmG4eEC9sPF/UY+/Cy2OTzsMp+cHb6C3maAo09O171wj6nJIZucg3B9fjEW2+4AoO217G4vmauMl3FFkut2CuvVV9zt2B/fLAskRg/yeYYOhjzPkWA6lyeV31sV5ZQ6Kw=,iv:5WfkmNr5vdfTqp6+INjQN/Zmc7/iJNc/2auO9h3En08=,tag:snBgJyMzBXVAkV3zERkK8g==,type:str]",
 		"encrypted_regex": "^(.*)$",
 		"version": "3.10.2"
 	}

2-nomad-config/teamsstatus.nomad.hcl

@@ -234,7 +234,7 @@ def build_message(distance, start_loc, end_loc):
     message = "On leave"
     if distance > 13144:
         message += f", driving my EV back from WA"
-    if distance > 2118:
+    elif distance > 2118:
         message += f", driving my EV around WA"
     elif distance > 0:
         message += f", driving my EV to WA"

2-nomad-config/teamsstatus.tf

@@ -1,6 +1,8 @@
-resource "nomad_job" "teamsstatus" {
-  jobspec = file("${path.module}/teamsstatus.nomad.hcl")
-}
+# Disabled
+
+# resource "nomad_job" "teamsstatus" {
+#   jobspec = file("${path.module}/teamsstatus.nomad.hcl")
+# }
 
 # resource "nomad_variable" "teamsstatus" {
 #   path = "nomad/jobs/teamsstatus"

2-nomad-config/traefik.nomad.hcl

@@ -1,166 +0,0 @@
-job "traefik" {
-  group "traefik" {
-    network {
-      port "http" {
-        static = 80
-      }
-
-      port "https" {
-        static = 443
-      }
-
-      port "api" {
-        static = 8081
-      }
-    }
-
-    service {
-      name = "traefik"
-      port = "api"
-
-      tags = [
-        "traefik.enable=true",
-        "traefik.http.routers.traefik.rule=Host(`traefik.othrayte.one`)",
-        "traefik.http.routers.traefik.service=traefik",
-        "traefik.http.routers.traefik.middlewares=auth@file",
-        "traefik.http.services.traefik.loadbalancer.server.port=8081",
-      ]
-
-      check {
-        name     = "alive"
-        type     = "tcp"
-        port     = "api"
-        interval = "10s"
-        timeout  = "2s"
-      }
-    }
-
-    volume "traefik" {
-      type      = "host"
-      read_only = false
-      source    = "traefik"
-    }
-
-    task "traefik" {
-      driver = "docker"
-
-      config {
-        image        = "traefik:v3.3"
-        network_mode = "host"
-
-        volumes = [
-          "local/traefik.yml:/etc/traefik/traefik.yml",
-          "local/configs/:/etc/traefik/configs/"
-        ]
-      }
-
-      volume_mount {
-        volume      = "traefik"
-        destination = "/opt/traefik"
-        read_only   = false
-      }
-
-      template {
-        data = <<EOF
-entryPoints:
-  web:
-    address: ":80"
-    http:
-      redirections:
-        entryPoint:
-          to: websecure
-          scheme: https
-  websecure:
-    address: ":443"
-    http:
-      tls:
-        certResolver: letsencrypt
-  traefik:
-    address: ":8081"
-
-api:
-  dashboard: true
-  insecure: true
-
-providers:
-  file:
-    directory: "/etc/traefik/configs/"
-
-  consulCatalog:
-    prefix: "traefik"
-    exposedByDefault: false
-    defaultRule: {{"Host(`{{ .Name }}.othrayte.one`)"}}
-    endpoint:
-      address: "127.0.0.1:8500"
-      scheme: "http"
-
-certificatesResolvers:
-  letsencrypt:
-    acme:
-      email: "othrayte@gmail.com"
-      storage: "/opt/traefik/acme.json"
-      httpChallenge:
-        entryPoint: web
-EOF
-
-        destination = "local/traefik.yml"
-      }
-
-      template {
-        data = <<EOF
-http:
-  middlewares:
-    auth:
-      forwardAuth:
-        address: "http://192.168.1.235:9091/api/authz/forward-auth"
-        trustForwardHeader: true
-  routers:
-    fallback:
-      rule: "HostRegexp(`^.+$`)"
-      entryPoints:
-        - websecure
-      middlewares:
-        - auth
-      service: noop@internal  # This router just applies middleware
-      priority: 1
-    nomad-ui:
-      rule: "Host(`nomad.othrayte.one`)"
-      service: nomad-ui
-      middlewares:
-        - auth
-    consul-ui:
-      rule: "Host(`consul.othrayte.one`)"
-      service: consul-ui
-      middlewares: 
-        - auth
-    unraid:
-      rule: "Host(`unraid.othrayte.one`)"
-      service: unraid
-      middlewares:
-        - auth
-
-  services:
-    nomad-ui:
-      loadBalancer:
-        servers:
-          - url: "http://127.0.0.1:4646"
-    consul-ui:
-      loadBalancer:
-        servers:
-          - url: "http://127.0.0.1:8500"
-    unraid:
-      loadBalancer:
-        servers:
-          - url: "http://192.168.1.192:80"
-EOF
-
-        destination = "local/configs/nomad.yml"
-      }
-
-      resources {
-        cpu    = 100
-        memory = 128
-      }
-    }
-  }
-}

2-nomad-config/traefik.tf

@@ -1,23 +0,0 @@
-resource "cloudflare_dns_record" "othrayte-one" {
-  comment = "othrayte.one proxy to internal IP for traefik"
-  zone_id = "2616ab2a44d0645b03fbc3106c79bd99"
-  type    = "AAAA"
-  name    = "othrayte.one"
-  content = data.sops_file.secrets.data["cloudflare.direct_ip6"]
-  proxied = true
-  ttl     = 1 # Auto
-}
-
-resource "cloudflare_dns_record" "star-othrayte-one" {
-  comment = "*.othrayte.one proxy to internal IP for traefik"
-  zone_id = "2616ab2a44d0645b03fbc3106c79bd99"
-  type    = "AAAA"
-  name    = "*.othrayte.one"
-  content = data.sops_file.secrets.data["cloudflare.direct_ip6"]
-  proxied = true
-  ttl     = 1 # Auto
-}
-
-resource "nomad_job" "traefik" {
-  jobspec = file("traefik.nomad.hcl")
-}

2-nomad-config/transfer.nomad.hcl

@@ -86,10 +86,7 @@ job "transfer" {
   "address": "",
   "log": "stdout",
   "database": "/database/filebrowser.db",
-  "root": "/srv",
-  "auth": {
-    "method": "noauth"
-  }
+  "root": "/srv"
 }
 EOF