diff --git a/1-nixos-node/0-install-nixos.md b/1-nixos-node/0-install-nixos.md index d79d2b6..c792e70 100644 --- a/1-nixos-node/0-install-nixos.md +++ b/1-nixos-node/0-install-nixos.md @@ -5,9 +5,15 @@ Note: run `lsblk` to determine correct device to install to. Set the hostname when editing `/mnt/etc/nixos/configuration.nix` before the installation: `networking.hostName = "jaglan-beta-mNN";` -Once `nixos-install` is complete and the root password is set and saved edit the `/etc/nixos/configuration.nix` again to enable ssh and run `nixos-rebuild switch` +Once `nixos-install` is complete and the root password is set and saved edit the `(/mnt)/etc/nixos/configuration.nix` again to enable ssh and run `nixos-rebuild switch` ``` services.openssh.enable = true; services.openssh.settings.PermitRootLogin = "yes"; ``` + +Upgrade the OS +`nix-channel --list` +`nix-channel --add https://channels.nixos.org/nixos-25.05 nixos` +`nix-channel --list` +`nixos-rebuild switch --upgrade` diff --git a/1-nixos-node/1-apply-changes.md b/1-nixos-node/1-apply-changes.md new file mode 100644 index 0000000..bcb4911 --- /dev/null +++ b/1-nixos-node/1-apply-changes.md @@ -0,0 +1,4 @@ +Avoid applying change to multiple hosts at once as this could take done the cluster, instead apply to each on one at a time +terraform apply -target='null_resource.deploy_nixos["jaglan-beta-m20"]' +terraform apply -target='null_resource.deploy_nixos["jaglan-beta-m21"]' +terraform apply -target='null_resource.deploy_nixos["jaglan-beta-m22"]' diff --git a/1-nixos-node/configuration.nix b/1-nixos-node/configuration.nix index bd9fde4..e184679 100644 --- a/1-nixos-node/configuration.nix +++ b/1-nixos-node/configuration.nix @@ -77,6 +77,8 @@ retry_join = [ "jaglan-beta-m01" "jaglan-beta-m20" + "jaglan-beta-m21" + "jaglan-beta-m22" ]; datacenter = "jaglan-beta"; connect.enabled = true; @@ -99,7 +101,7 @@ ]; # Open ports in the firewall. 80/443 are for HTTP/HTTPS (terraform), 464X are the default ports for Nomad, 830X are the default ports for Consul. - networking.firewall.allowedTCPPorts = [ 80 443 8081 4646 4647 4648 8300 8301 ]; + networking.firewall.allowedTCPPorts = [ 80 443 8081 4646 4647 4648 8300 8301 8500 ]; networking.firewall.allowedUDPPorts = [ 8301 ]; # Copy the NixOS configuration file and link it from the resulting system diff --git a/1-nixos-node/terraform.tfstate b/1-nixos-node/terraform.tfstate index a4f6acc..83a2642 100644 --- a/1-nixos-node/terraform.tfstate +++ b/1-nixos-node/terraform.tfstate @@ -1,7 +1,7 @@ { "version": 4, - "terraform_version": "1.11.4", - "serial": 251, + "terraform_version": "1.13.0", + "serial": 278, "lineage": "db7dcf21-a255-0ec4-c8b8-d4a7559b3768", "outputs": {}, "resources": [ @@ -11,27 +11,41 @@ "name": "deploy_nixos", "provider": "provider[\"registry.terraform.io/hashicorp/null\"]", "instances": [ - { - "index_key": "jaglan-beta-m01", - "schema_version": 0, - "attributes": { - "id": "6048275408611354621", - "triggers": { - "configuration_content": "{ config, lib, pkgs, ... }:\n{\n imports =\n [ # Include the results of the hardware scan.\n ./hardware-configuration.nix\n ];\n\n nixpkgs.config.allowUnfree = true;\n\n # Use the systemd-boot EFI boot loader.\n boot.loader.systemd-boot.enable = true;\n boot.loader.efi.canTouchEfiVariables = true;\n\n networking.hostName = \"jaglan-beta-m01\"; # Define your hostname.\n\n time.timeZone = \"Australia/Melbourne\";\n\n # List packages installed in system profile. To search, run:\n # $ nix search wget\n # environment.systemPackages = with pkgs; [\n # vim # Do not forget to add an editor to edit configuration.nix! The Nano editor is also installed by default.\n # wget\n # ];\n\n # Some programs need SUID wrappers, can be configured further or are\n # started in user sessions.\n # programs.mtr.enable = true;\n # programs.gnupg.agent = {\n # enable = true;\n # enableSSHSupport = true;\n # };\n\n # List services that you want to enable:\n services = {\n nomad = {\n enable = true;\n enableDocker = true;\n dropPrivileges = false;\n settings = {\n datacenter = \"jaglan-beta\";\n server = {\n enabled = true;\n bootstrap_expect = 1;\n };\n client = {\n enabled = true;\n host_volume = {\n traefik = {\n path = \"/opt/traefik\";\n read_only = false;\n };\n postgres = {\n path = \"/opt/postgres\";\n read_only = false;\n };\n };\n cni_path = \"${pkgs.cni-plugins}/bin\";\n };\n plugin.docker.config.allow_privileged = true;\n };\n extraPackages = with pkgs; [\n cni-plugins\n consul\n ];\n };\n consul = {\n enable = true;\n webUi = true;\n interface.bind = \"eno1\";\n interface.advertise = \"eno1\";\n extraConfig = {\n client_addr = \"{{ GetPrivateInterfaces | exclude \\\"type\\\" \\\"ipv6\\\" | join \\\"address\\\" \\\" \\\" }} {{ GetAllInterfaces | include \\\"flags\\\" \\\"loopback\\\" | join \\\"address\\\" \\\" \\\" }}\";\n bootstrap_expect = 1;\n server = true;\n retry_join = [\n \"jaglan-beta-m01\"\n \"jaglan-beta-m20\"\n ];\n datacenter = \"jaglan-beta\";\n connect.enabled = true;\n ports.grpc = 8502;\n };\n };\n openssh = {\n enable = true;\n settings.PermitRootLogin = \"yes\";\n };\n };\n\n systemd.tmpfiles.rules = [\n # Fix issue where nomad needs alloc_mounts to be writable\n \"d /var/lib/alloc_mounts 0755 root root -\"\n # Create a directory for traefik to store its data\n \"d /opt/traefik 0755 root root -\"\n # Create a directory for postgres to store its data\n \"d /opt/postgres 0755 root root -\"\n ];\n\n # Open ports in the firewall. 80/443 are for HTTP/HTTPS (terraform), 464X are the default ports for Nomad, 830X are the default ports for Consul.\n networking.firewall.allowedTCPPorts = [ 80 443 8081 4646 4647 4648 8300 8301 ];\n networking.firewall.allowedUDPPorts = [ 8301 ];\n\n # Copy the NixOS configuration file and link it from the resulting system\n # (/run/current-system/configuration.nix). This is useful in case you\n # accidentally delete configuration.nix.\n system.copySystemConfiguration = true;\n \n # Defines the initial NixOS version for compatibility with older application data.\n # Do NOT change this value after installation without careful consideration.\n system.stateVersion = \"24.11\"; # Did you read the comment?\n}\n" - } - }, - "sensitive_attributes": [] - }, { "index_key": "jaglan-beta-m20", "schema_version": 0, "attributes": { - "id": "4501962519001291201", + "id": "1521310958736582307", "triggers": { - "configuration_content": "{ config, lib, pkgs, ... }:\n{\n imports =\n [ # Include the results of the hardware scan.\n ./hardware-configuration.nix\n ];\n\n nixpkgs.config.allowUnfree = true;\n\n # Use the systemd-boot EFI boot loader.\n boot.loader.systemd-boot.enable = true;\n boot.loader.efi.canTouchEfiVariables = true;\n\n networking.hostName = \"jaglan-beta-m20\"; # Define your hostname.\n\n time.timeZone = \"Australia/Melbourne\";\n\n # List packages installed in system profile. To search, run:\n # $ nix search wget\n # environment.systemPackages = with pkgs; [\n # vim # Do not forget to add an editor to edit configuration.nix! The Nano editor is also installed by default.\n # wget\n # ];\n\n # Some programs need SUID wrappers, can be configured further or are\n # started in user sessions.\n # programs.mtr.enable = true;\n # programs.gnupg.agent = {\n # enable = true;\n # enableSSHSupport = true;\n # };\n\n # List services that you want to enable:\n services = {\n nomad = {\n enable = true;\n enableDocker = true;\n dropPrivileges = false;\n settings = {\n datacenter = \"jaglan-beta\";\n server = {\n enabled = true;\n };\n client = {\n enabled = true;\n host_volume = {\n };\n cni_path = \"${pkgs.cni-plugins}/bin\";\n };\n plugin.docker.config.allow_privileged = true;\n };\n extraPackages = with pkgs; [\n cni-plugins\n consul\n ];\n };\n consul = {\n enable = true;\n webUi = true;\n interface.bind = \"ens2\";\n interface.advertise = \"ens2\";\n extraConfig = {\n client_addr = \"{{ GetPrivateInterfaces | exclude \\\"type\\\" \\\"ipv6\\\" | join \\\"address\\\" \\\" \\\" }} {{ GetAllInterfaces | include \\\"flags\\\" \\\"loopback\\\" | join \\\"address\\\" \\\" \\\" }}\";\n server = true;\n retry_join = [\n \"jaglan-beta-m01\"\n \"jaglan-beta-m20\"\n ];\n datacenter = \"jaglan-beta\";\n connect.enabled = true;\n ports.grpc = 8502;\n };\n };\n openssh = {\n enable = true;\n settings.PermitRootLogin = \"yes\";\n };\n };\n\n systemd.tmpfiles.rules = [\n # Fix issue where nomad needs alloc_mounts to be writable\n \"d /var/lib/alloc_mounts 0755 root root -\"\n ];\n\n # Open ports in the firewall. 80/443 are for HTTP/HTTPS (terraform), 464X are the default ports for Nomad, 830X are the default ports for Consul.\n networking.firewall.allowedTCPPorts = [ 80 443 8081 4646 4647 4648 8300 8301 ];\n networking.firewall.allowedUDPPorts = [ 8301 ];\n\n # Copy the NixOS configuration file and link it from the resulting system\n # (/run/current-system/configuration.nix). This is useful in case you\n # accidentally delete configuration.nix.\n system.copySystemConfiguration = true;\n \n # Defines the initial NixOS version for compatibility with older application data.\n # Do NOT change this value after installation without careful consideration.\n system.stateVersion = \"24.11\"; # Did you read the comment?\n}\n" + "configuration_content": "{ config, lib, pkgs, ... }:\n{\n imports =\n [ # Include the results of the hardware scan.\n ./hardware-configuration.nix\n ];\n\n nixpkgs.config.allowUnfree = true;\n\n # Use the systemd-boot EFI boot loader.\n boot.loader.systemd-boot.enable = true;\n boot.loader.efi.canTouchEfiVariables = true;\n\n networking.hostName = \"jaglan-beta-m20\"; # Define your hostname.\n\n time.timeZone = \"Australia/Melbourne\";\n\n # List packages installed in system profile. To search, run:\n # $ nix search wget\n # environment.systemPackages = with pkgs; [\n # vim # Do not forget to add an editor to edit configuration.nix! The Nano editor is also installed by default.\n # wget\n # ];\n\n # Some programs need SUID wrappers, can be configured further or are\n # started in user sessions.\n # programs.mtr.enable = true;\n # programs.gnupg.agent = {\n # enable = true;\n # enableSSHSupport = true;\n # };\n\n # List services that you want to enable:\n services = {\n nomad = {\n enable = true;\n enableDocker = true;\n dropPrivileges = false;\n settings = {\n datacenter = \"jaglan-beta\";\n server = {\n enabled = true;\n bootstrap_expect = 1;\n };\n client = {\n enabled = true;\n host_volume = {\n };\n cni_path = \"${pkgs.cni-plugins}/bin\";\n };\n plugin.docker.config.allow_privileged = true;\n };\n extraPackages = with pkgs; [\n cni-plugins\n consul\n ];\n };\n consul = {\n enable = true;\n webUi = true;\n interface.bind = \"ens2\";\n interface.advertise = \"ens2\";\n extraConfig = {\n client_addr = \"{{ GetPrivateInterfaces | exclude \\\"type\\\" \\\"ipv6\\\" | join \\\"address\\\" \\\" \\\" }} {{ GetAllInterfaces | include \\\"flags\\\" \\\"loopback\\\" | join \\\"address\\\" \\\" \\\" }}\";\n bootstrap_expect = 1;\n server = true;\n retry_join = [\n \"jaglan-beta-m01\"\n \"jaglan-beta-m20\"\n \"jaglan-beta-m21\"\n \"jaglan-beta-m22\"\n ];\n datacenter = \"jaglan-beta\";\n connect.enabled = true;\n ports.grpc = 8502;\n };\n };\n openssh = {\n enable = true;\n settings.PermitRootLogin = \"yes\";\n };\n };\n\n systemd.tmpfiles.rules = [\n # Fix issue where nomad needs alloc_mounts to be writable\n \"d /var/lib/alloc_mounts 0755 root root -\"\n ];\n\n # Open ports in the firewall. 80/443 are for HTTP/HTTPS (terraform), 464X are the default ports for Nomad, 830X are the default ports for Consul.\n networking.firewall.allowedTCPPorts = [ 80 443 8081 4646 4647 4648 8300 8301 8500 ];\n networking.firewall.allowedUDPPorts = [ 8301 ];\n\n # Copy the NixOS configuration file and link it from the resulting system\n # (/run/current-system/configuration.nix). This is useful in case you\n # accidentally delete configuration.nix.\n system.copySystemConfiguration = true;\n \n # Defines the initial NixOS version for compatibility with older application data.\n # Do NOT change this value after installation without careful consideration.\n system.stateVersion = \"24.11\"; # Did you read the comment?\n}\n" } }, - "sensitive_attributes": [] + "sensitive_attributes": [], + "identity_schema_version": 0 + }, + { + "index_key": "jaglan-beta-m21", + "schema_version": 0, + "attributes": { + "id": "8863603003911062019", + "triggers": { + "configuration_content": "{ config, lib, pkgs, ... }:\n{\n imports =\n [ # Include the results of the hardware scan.\n ./hardware-configuration.nix\n ];\n\n nixpkgs.config.allowUnfree = true;\n\n # Use the systemd-boot EFI boot loader.\n boot.loader.systemd-boot.enable = true;\n boot.loader.efi.canTouchEfiVariables = true;\n\n networking.hostName = \"jaglan-beta-m21\"; # Define your hostname.\n\n time.timeZone = \"Australia/Melbourne\";\n\n # List packages installed in system profile. To search, run:\n # $ nix search wget\n # environment.systemPackages = with pkgs; [\n # vim # Do not forget to add an editor to edit configuration.nix! The Nano editor is also installed by default.\n # wget\n # ];\n\n # Some programs need SUID wrappers, can be configured further or are\n # started in user sessions.\n # programs.mtr.enable = true;\n # programs.gnupg.agent = {\n # enable = true;\n # enableSSHSupport = true;\n # };\n\n # List services that you want to enable:\n services = {\n nomad = {\n enable = true;\n enableDocker = true;\n dropPrivileges = false;\n settings = {\n datacenter = \"jaglan-beta\";\n server = {\n enabled = true;\n };\n client = {\n enabled = true;\n host_volume = {\n traefik = {\n path = \"/opt/traefik\";\n read_only = false;\n };\n postgres = {\n path = \"/opt/postgres\";\n read_only = false;\n };\n };\n cni_path = \"${pkgs.cni-plugins}/bin\";\n };\n plugin.docker.config.allow_privileged = true;\n };\n extraPackages = with pkgs; [\n cni-plugins\n consul\n ];\n };\n consul = {\n enable = true;\n webUi = true;\n interface.bind = \"ens2\";\n interface.advertise = \"ens2\";\n extraConfig = {\n client_addr = \"{{ GetPrivateInterfaces | exclude \\\"type\\\" \\\"ipv6\\\" | join \\\"address\\\" \\\" \\\" }} {{ GetAllInterfaces | include \\\"flags\\\" \\\"loopback\\\" | join \\\"address\\\" \\\" \\\" }}\";\n server = true;\n retry_join = [\n \"jaglan-beta-m01\"\n \"jaglan-beta-m20\"\n \"jaglan-beta-m21\"\n \"jaglan-beta-m22\"\n ];\n datacenter = \"jaglan-beta\";\n connect.enabled = true;\n ports.grpc = 8502;\n };\n };\n openssh = {\n enable = true;\n settings.PermitRootLogin = \"yes\";\n };\n };\n\n systemd.tmpfiles.rules = [\n # Fix issue where nomad needs alloc_mounts to be writable\n \"d /var/lib/alloc_mounts 0755 root root -\"\n # Create a directory for traefik to store its data\n \"d /opt/traefik 0755 root root -\"\n # Create a directory for postgres to store its data\n \"d /opt/postgres 0755 root root -\"\n ];\n\n # Open ports in the firewall. 80/443 are for HTTP/HTTPS (terraform), 464X are the default ports for Nomad, 830X are the default ports for Consul.\n networking.firewall.allowedTCPPorts = [ 80 443 8081 4646 4647 4648 8300 8301 8500 ];\n networking.firewall.allowedUDPPorts = [ 8301 ];\n\n # Copy the NixOS configuration file and link it from the resulting system\n # (/run/current-system/configuration.nix). This is useful in case you\n # accidentally delete configuration.nix.\n system.copySystemConfiguration = true;\n \n # Defines the initial NixOS version for compatibility with older application data.\n # Do NOT change this value after installation without careful consideration.\n system.stateVersion = \"24.11\"; # Did you read the comment?\n}\n" + } + }, + "sensitive_attributes": [], + "identity_schema_version": 0 + }, + { + "index_key": "jaglan-beta-m22", + "schema_version": 0, + "attributes": { + "id": "6319172431738426418", + "triggers": { + "configuration_content": "{ config, lib, pkgs, ... }:\n{\n imports =\n [ # Include the results of the hardware scan.\n ./hardware-configuration.nix\n ];\n\n nixpkgs.config.allowUnfree = true;\n\n # Use the systemd-boot EFI boot loader.\n boot.loader.systemd-boot.enable = true;\n boot.loader.efi.canTouchEfiVariables = true;\n\n networking.hostName = \"jaglan-beta-m22\"; # Define your hostname.\n\n time.timeZone = \"Australia/Melbourne\";\n\n # List packages installed in system profile. To search, run:\n # $ nix search wget\n # environment.systemPackages = with pkgs; [\n # vim # Do not forget to add an editor to edit configuration.nix! The Nano editor is also installed by default.\n # wget\n # ];\n\n # Some programs need SUID wrappers, can be configured further or are\n # started in user sessions.\n # programs.mtr.enable = true;\n # programs.gnupg.agent = {\n # enable = true;\n # enableSSHSupport = true;\n # };\n\n # List services that you want to enable:\n services = {\n nomad = {\n enable = true;\n enableDocker = true;\n dropPrivileges = false;\n settings = {\n datacenter = \"jaglan-beta\";\n server = {\n enabled = true;\n };\n client = {\n enabled = true;\n host_volume = {\n };\n cni_path = \"${pkgs.cni-plugins}/bin\";\n };\n plugin.docker.config.allow_privileged = true;\n };\n extraPackages = with pkgs; [\n cni-plugins\n consul\n ];\n };\n consul = {\n enable = true;\n webUi = true;\n interface.bind = \"ens2\";\n interface.advertise = \"ens2\";\n extraConfig = {\n client_addr = \"{{ GetPrivateInterfaces | exclude \\\"type\\\" \\\"ipv6\\\" | join \\\"address\\\" \\\" \\\" }} {{ GetAllInterfaces | include \\\"flags\\\" \\\"loopback\\\" | join \\\"address\\\" \\\" \\\" }}\";\n server = true;\n retry_join = [\n \"jaglan-beta-m01\"\n \"jaglan-beta-m20\"\n \"jaglan-beta-m21\"\n \"jaglan-beta-m22\"\n ];\n datacenter = \"jaglan-beta\";\n connect.enabled = true;\n ports.grpc = 8502;\n };\n };\n openssh = {\n enable = true;\n settings.PermitRootLogin = \"yes\";\n };\n };\n\n systemd.tmpfiles.rules = [\n # Fix issue where nomad needs alloc_mounts to be writable\n \"d /var/lib/alloc_mounts 0755 root root -\"\n ];\n\n # Open ports in the firewall. 80/443 are for HTTP/HTTPS (terraform), 464X are the default ports for Nomad, 830X are the default ports for Consul.\n networking.firewall.allowedTCPPorts = [ 80 443 8081 4646 4647 4648 8300 8301 8500 ];\n networking.firewall.allowedUDPPorts = [ 8301 ];\n\n # Copy the NixOS configuration file and link it from the resulting system\n # (/run/current-system/configuration.nix). This is useful in case you\n # accidentally delete configuration.nix.\n system.copySystemConfiguration = true;\n \n # Defines the initial NixOS version for compatibility with older application data.\n # Do NOT change this value after installation without careful consideration.\n system.stateVersion = \"24.11\"; # Did you read the comment?\n}\n" + } + }, + "sensitive_attributes": [], + "identity_schema_version": 0 } ] } diff --git a/1-nixos-node/terraform.tfstate.backup b/1-nixos-node/terraform.tfstate.backup index e2e479d..61709c5 100644 --- a/1-nixos-node/terraform.tfstate.backup +++ b/1-nixos-node/terraform.tfstate.backup @@ -1,7 +1,7 @@ { "version": 4, - "terraform_version": "1.11.4", - "serial": 246, + "terraform_version": "1.13.0", + "serial": 275, "lineage": "db7dcf21-a255-0ec4-c8b8-d4a7559b3768", "outputs": {}, "resources": [ @@ -11,27 +11,41 @@ "name": "deploy_nixos", "provider": "provider[\"registry.terraform.io/hashicorp/null\"]", "instances": [ - { - "index_key": "jaglan-beta-m01", - "schema_version": 0, - "attributes": { - "id": "375315072123105414", - "triggers": { - "configuration_content": "{ config, lib, pkgs, ... }:\n{\n imports =\n [ # Include the results of the hardware scan.\n ./hardware-configuration.nix\n ];\n\n nixpkgs.config.allowUnfree = true;\n\n # Use the systemd-boot EFI boot loader.\n boot.loader.systemd-boot.enable = true;\n boot.loader.efi.canTouchEfiVariables = true;\n\n networking.hostName = \"jaglan-beta-m01\"; # Define your hostname.\n\n time.timeZone = \"Australia/Melbourne\";\n\n # List packages installed in system profile. To search, run:\n # $ nix search wget\n # environment.systemPackages = with pkgs; [\n # vim # Do not forget to add an editor to edit configuration.nix! The Nano editor is also installed by default.\n # wget\n # ];\n\n # Some programs need SUID wrappers, can be configured further or are\n # started in user sessions.\n # programs.mtr.enable = true;\n # programs.gnupg.agent = {\n # enable = true;\n # enableSSHSupport = true;\n # };\n\n # List services that you want to enable:\n services = {\n nomad = {\n enable = true;\n enableDocker = true;\n dropPrivileges = false;\n settings = {\n datacenter = \"jaglan-beta\";\n server = {\n enabled = true;\n bootstrap_expect = 1;\n };\n client = {\n enabled = true;\n host_volume = {\n traefik = {\n path = \"/opt/traefik\";\n read_only = false;\n };\n postgres = {\n path = \"/opt/postgres\";\n read_only = false;\n };\n };\n cni_path = \"${pkgs.cni-plugins}/bin\";\n };\n plugin.docker.config.allow_privileged = true;\n };\n extraPackages = with pkgs; [\n cni-plugins\n consul\n ];\n };\n consul = {\n enable = true;\n webUi = true;\n interface.bind = \"eno1\";\n interface.advertise = \"eno1\";\n extraConfig = {\n client_addr = \"{{ GetPrivateInterfaces | exclude \\\"type\\\" \\\"ipv6\\\" | join \\\"address\\\" \\\" \\\" }} {{ GetAllInterfaces | include \\\"flags\\\" \\\"loopback\\\" | join \\\"address\\\" \\\" \\\" }}\";\n bootstrap_expect = 1;\n server = true;\n retry_join = [\n \"jaglan-beta-m01\"\n \"jaglan-beta-m20\"\n ];\n datacenter = \"jaglan-beta\";\n connect.enabled = true;\n ports.grpc = 8502;\n };\n };\n openssh = {\n enable = true;\n settings.PermitRootLogin = \"yes\";\n };\n };\n\n systemd.tmpfiles.rules = [\n # Fix issue where nomad needs alloc_mounts to be writable\n \"d /var/lib/alloc_mounts 0755 root root -\"\n # Create a directory for traefik to store its data\n \"d /opt/traefik 0755 root root -\"\n # Create a directory for postgres to store its data\n \"d /opt/postgres 0755 root root -\"\n ];\n\n # Open ports in the firewall. 80/443 are for HTTP/HTTPS (terraform), 464X are the default ports for Nomad, 830X are the default ports for Consul.\n networking.firewall.allowedTCPPorts = [ 80 443 4646 4647 4648 8300 8301 ];\n networking.firewall.allowedUDPPorts = [ 8301 ];\n\n # Copy the NixOS configuration file and link it from the resulting system\n # (/run/current-system/configuration.nix). This is useful in case you\n # accidentally delete configuration.nix.\n system.copySystemConfiguration = true;\n \n # Defines the initial NixOS version for compatibility with older application data.\n # Do NOT change this value after installation without careful consideration.\n system.stateVersion = \"24.11\"; # Did you read the comment?\n}\n" - } - }, - "sensitive_attributes": [] - }, { "index_key": "jaglan-beta-m20", "schema_version": 0, "attributes": { - "id": "7569759743571304983", + "id": "1521310958736582307", "triggers": { - "configuration_content": "{ config, lib, pkgs, ... }:\n{\n imports =\n [ # Include the results of the hardware scan.\n ./hardware-configuration.nix\n ];\n\n nixpkgs.config.allowUnfree = true;\n\n # Use the systemd-boot EFI boot loader.\n boot.loader.systemd-boot.enable = true;\n boot.loader.efi.canTouchEfiVariables = true;\n\n networking.hostName = \"jaglan-beta-m20\"; # Define your hostname.\n\n time.timeZone = \"Australia/Melbourne\";\n\n # List packages installed in system profile. To search, run:\n # $ nix search wget\n # environment.systemPackages = with pkgs; [\n # vim # Do not forget to add an editor to edit configuration.nix! The Nano editor is also installed by default.\n # wget\n # ];\n\n # Some programs need SUID wrappers, can be configured further or are\n # started in user sessions.\n # programs.mtr.enable = true;\n # programs.gnupg.agent = {\n # enable = true;\n # enableSSHSupport = true;\n # };\n\n # List services that you want to enable:\n services = {\n nomad = {\n enable = true;\n enableDocker = true;\n dropPrivileges = false;\n settings = {\n datacenter = \"jaglan-beta\";\n server = {\n enabled = true;\n };\n client = {\n enabled = true;\n host_volume = {\n };\n cni_path = \"${pkgs.cni-plugins}/bin\";\n };\n plugin.docker.config.allow_privileged = true;\n };\n extraPackages = with pkgs; [\n cni-plugins\n consul\n ];\n };\n consul = {\n enable = true;\n webUi = true;\n interface.bind = \"ens2\";\n interface.advertise = \"ens2\";\n extraConfig = {\n client_addr = \"{{ GetPrivateInterfaces | exclude \\\"type\\\" \\\"ipv6\\\" | join \\\"address\\\" \\\" \\\" }} {{ GetAllInterfaces | include \\\"flags\\\" \\\"loopback\\\" | join \\\"address\\\" \\\" \\\" }}\";\n server = true;\n retry_join = [\n \"jaglan-beta-m01\"\n \"jaglan-beta-m20\"\n ];\n datacenter = \"jaglan-beta\";\n connect.enabled = true;\n ports.grpc = 8502;\n };\n };\n openssh = {\n enable = true;\n settings.PermitRootLogin = \"yes\";\n };\n };\n\n systemd.tmpfiles.rules = [\n # Fix issue where nomad needs alloc_mounts to be writable\n \"d /var/lib/alloc_mounts 0755 root root -\"\n ];\n\n # Open ports in the firewall. 80/443 are for HTTP/HTTPS (terraform), 464X are the default ports for Nomad, 830X are the default ports for Consul.\n networking.firewall.allowedTCPPorts = [ 80 443 4646 4647 4648 8300 8301 ];\n networking.firewall.allowedUDPPorts = [ 8301 ];\n\n # Copy the NixOS configuration file and link it from the resulting system\n # (/run/current-system/configuration.nix). This is useful in case you\n # accidentally delete configuration.nix.\n system.copySystemConfiguration = true;\n \n # Defines the initial NixOS version for compatibility with older application data.\n # Do NOT change this value after installation without careful consideration.\n system.stateVersion = \"24.11\"; # Did you read the comment?\n}\n" + "configuration_content": "{ config, lib, pkgs, ... }:\n{\n imports =\n [ # Include the results of the hardware scan.\n ./hardware-configuration.nix\n ];\n\n nixpkgs.config.allowUnfree = true;\n\n # Use the systemd-boot EFI boot loader.\n boot.loader.systemd-boot.enable = true;\n boot.loader.efi.canTouchEfiVariables = true;\n\n networking.hostName = \"jaglan-beta-m20\"; # Define your hostname.\n\n time.timeZone = \"Australia/Melbourne\";\n\n # List packages installed in system profile. To search, run:\n # $ nix search wget\n # environment.systemPackages = with pkgs; [\n # vim # Do not forget to add an editor to edit configuration.nix! The Nano editor is also installed by default.\n # wget\n # ];\n\n # Some programs need SUID wrappers, can be configured further or are\n # started in user sessions.\n # programs.mtr.enable = true;\n # programs.gnupg.agent = {\n # enable = true;\n # enableSSHSupport = true;\n # };\n\n # List services that you want to enable:\n services = {\n nomad = {\n enable = true;\n enableDocker = true;\n dropPrivileges = false;\n settings = {\n datacenter = \"jaglan-beta\";\n server = {\n enabled = true;\n bootstrap_expect = 1;\n };\n client = {\n enabled = true;\n host_volume = {\n };\n cni_path = \"${pkgs.cni-plugins}/bin\";\n };\n plugin.docker.config.allow_privileged = true;\n };\n extraPackages = with pkgs; [\n cni-plugins\n consul\n ];\n };\n consul = {\n enable = true;\n webUi = true;\n interface.bind = \"ens2\";\n interface.advertise = \"ens2\";\n extraConfig = {\n client_addr = \"{{ GetPrivateInterfaces | exclude \\\"type\\\" \\\"ipv6\\\" | join \\\"address\\\" \\\" \\\" }} {{ GetAllInterfaces | include \\\"flags\\\" \\\"loopback\\\" | join \\\"address\\\" \\\" \\\" }}\";\n bootstrap_expect = 1;\n server = true;\n retry_join = [\n \"jaglan-beta-m01\"\n \"jaglan-beta-m20\"\n \"jaglan-beta-m21\"\n \"jaglan-beta-m22\"\n ];\n datacenter = \"jaglan-beta\";\n connect.enabled = true;\n ports.grpc = 8502;\n };\n };\n openssh = {\n enable = true;\n settings.PermitRootLogin = \"yes\";\n };\n };\n\n systemd.tmpfiles.rules = [\n # Fix issue where nomad needs alloc_mounts to be writable\n \"d /var/lib/alloc_mounts 0755 root root -\"\n ];\n\n # Open ports in the firewall. 80/443 are for HTTP/HTTPS (terraform), 464X are the default ports for Nomad, 830X are the default ports for Consul.\n networking.firewall.allowedTCPPorts = [ 80 443 8081 4646 4647 4648 8300 8301 8500 ];\n networking.firewall.allowedUDPPorts = [ 8301 ];\n\n # Copy the NixOS configuration file and link it from the resulting system\n # (/run/current-system/configuration.nix). This is useful in case you\n # accidentally delete configuration.nix.\n system.copySystemConfiguration = true;\n \n # Defines the initial NixOS version for compatibility with older application data.\n # Do NOT change this value after installation without careful consideration.\n system.stateVersion = \"24.11\"; # Did you read the comment?\n}\n" } }, - "sensitive_attributes": [] + "sensitive_attributes": [], + "identity_schema_version": 0 + }, + { + "index_key": "jaglan-beta-m21", + "schema_version": 0, + "attributes": { + "id": "8793571600250298680", + "triggers": { + "configuration_content": "{ config, lib, pkgs, ... }:\n{\n imports =\n [ # Include the results of the hardware scan.\n ./hardware-configuration.nix\n ];\n\n nixpkgs.config.allowUnfree = true;\n\n # Use the systemd-boot EFI boot loader.\n boot.loader.systemd-boot.enable = true;\n boot.loader.efi.canTouchEfiVariables = true;\n\n networking.hostName = \"jaglan-beta-m21\"; # Define your hostname.\n\n time.timeZone = \"Australia/Melbourne\";\n\n # List packages installed in system profile. To search, run:\n # $ nix search wget\n # environment.systemPackages = with pkgs; [\n # vim # Do not forget to add an editor to edit configuration.nix! The Nano editor is also installed by default.\n # wget\n # ];\n\n # Some programs need SUID wrappers, can be configured further or are\n # started in user sessions.\n # programs.mtr.enable = true;\n # programs.gnupg.agent = {\n # enable = true;\n # enableSSHSupport = true;\n # };\n\n # List services that you want to enable:\n services = {\n nomad = {\n enable = true;\n enableDocker = true;\n dropPrivileges = false;\n settings = {\n datacenter = \"jaglan-beta\";\n server = {\n enabled = true;\n };\n client = {\n enabled = true;\n host_volume = {\n traefik = {\n path = \"/opt/traefik\";\n read_only = false;\n };\n };\n cni_path = \"${pkgs.cni-plugins}/bin\";\n };\n plugin.docker.config.allow_privileged = true;\n };\n extraPackages = with pkgs; [\n cni-plugins\n consul\n ];\n };\n consul = {\n enable = true;\n webUi = true;\n interface.bind = \"ens2\";\n interface.advertise = \"ens2\";\n extraConfig = {\n client_addr = \"{{ GetPrivateInterfaces | exclude \\\"type\\\" \\\"ipv6\\\" | join \\\"address\\\" \\\" \\\" }} {{ GetAllInterfaces | include \\\"flags\\\" \\\"loopback\\\" | join \\\"address\\\" \\\" \\\" }}\";\n server = true;\n retry_join = [\n \"jaglan-beta-m01\"\n \"jaglan-beta-m20\"\n \"jaglan-beta-m21\"\n \"jaglan-beta-m22\"\n ];\n datacenter = \"jaglan-beta\";\n connect.enabled = true;\n ports.grpc = 8502;\n };\n };\n openssh = {\n enable = true;\n settings.PermitRootLogin = \"yes\";\n };\n };\n\n systemd.tmpfiles.rules = [\n # Fix issue where nomad needs alloc_mounts to be writable\n \"d /var/lib/alloc_mounts 0755 root root -\"\n # Create a directory for traefik to store its data\n \"d /opt/traefik 0755 root root -\"\n ];\n\n # Open ports in the firewall. 80/443 are for HTTP/HTTPS (terraform), 464X are the default ports for Nomad, 830X are the default ports for Consul.\n networking.firewall.allowedTCPPorts = [ 80 443 8081 4646 4647 4648 8300 8301 8500 ];\n networking.firewall.allowedUDPPorts = [ 8301 ];\n\n # Copy the NixOS configuration file and link it from the resulting system\n # (/run/current-system/configuration.nix). This is useful in case you\n # accidentally delete configuration.nix.\n system.copySystemConfiguration = true;\n \n # Defines the initial NixOS version for compatibility with older application data.\n # Do NOT change this value after installation without careful consideration.\n system.stateVersion = \"24.11\"; # Did you read the comment?\n}\n" + } + }, + "sensitive_attributes": [], + "identity_schema_version": 0 + }, + { + "index_key": "jaglan-beta-m22", + "schema_version": 0, + "attributes": { + "id": "6319172431738426418", + "triggers": { + "configuration_content": "{ config, lib, pkgs, ... }:\n{\n imports =\n [ # Include the results of the hardware scan.\n ./hardware-configuration.nix\n ];\n\n nixpkgs.config.allowUnfree = true;\n\n # Use the systemd-boot EFI boot loader.\n boot.loader.systemd-boot.enable = true;\n boot.loader.efi.canTouchEfiVariables = true;\n\n networking.hostName = \"jaglan-beta-m22\"; # Define your hostname.\n\n time.timeZone = \"Australia/Melbourne\";\n\n # List packages installed in system profile. To search, run:\n # $ nix search wget\n # environment.systemPackages = with pkgs; [\n # vim # Do not forget to add an editor to edit configuration.nix! The Nano editor is also installed by default.\n # wget\n # ];\n\n # Some programs need SUID wrappers, can be configured further or are\n # started in user sessions.\n # programs.mtr.enable = true;\n # programs.gnupg.agent = {\n # enable = true;\n # enableSSHSupport = true;\n # };\n\n # List services that you want to enable:\n services = {\n nomad = {\n enable = true;\n enableDocker = true;\n dropPrivileges = false;\n settings = {\n datacenter = \"jaglan-beta\";\n server = {\n enabled = true;\n };\n client = {\n enabled = true;\n host_volume = {\n };\n cni_path = \"${pkgs.cni-plugins}/bin\";\n };\n plugin.docker.config.allow_privileged = true;\n };\n extraPackages = with pkgs; [\n cni-plugins\n consul\n ];\n };\n consul = {\n enable = true;\n webUi = true;\n interface.bind = \"ens2\";\n interface.advertise = \"ens2\";\n extraConfig = {\n client_addr = \"{{ GetPrivateInterfaces | exclude \\\"type\\\" \\\"ipv6\\\" | join \\\"address\\\" \\\" \\\" }} {{ GetAllInterfaces | include \\\"flags\\\" \\\"loopback\\\" | join \\\"address\\\" \\\" \\\" }}\";\n server = true;\n retry_join = [\n \"jaglan-beta-m01\"\n \"jaglan-beta-m20\"\n \"jaglan-beta-m21\"\n \"jaglan-beta-m22\"\n ];\n datacenter = \"jaglan-beta\";\n connect.enabled = true;\n ports.grpc = 8502;\n };\n };\n openssh = {\n enable = true;\n settings.PermitRootLogin = \"yes\";\n };\n };\n\n systemd.tmpfiles.rules = [\n # Fix issue where nomad needs alloc_mounts to be writable\n \"d /var/lib/alloc_mounts 0755 root root -\"\n ];\n\n # Open ports in the firewall. 80/443 are for HTTP/HTTPS (terraform), 464X are the default ports for Nomad, 830X are the default ports for Consul.\n networking.firewall.allowedTCPPorts = [ 80 443 8081 4646 4647 4648 8300 8301 8500 ];\n networking.firewall.allowedUDPPorts = [ 8301 ];\n\n # Copy the NixOS configuration file and link it from the resulting system\n # (/run/current-system/configuration.nix). This is useful in case you\n # accidentally delete configuration.nix.\n system.copySystemConfiguration = true;\n \n # Defines the initial NixOS version for compatibility with older application data.\n # Do NOT change this value after installation without careful consideration.\n system.stateVersion = \"24.11\"; # Did you read the comment?\n}\n" + } + }, + "sensitive_attributes": [], + "identity_schema_version": 0 } ] } diff --git a/2-nomad-config/main.tf b/2-nomad-config/main.tf index fd6efec..f5ab553 100644 --- a/2-nomad-config/main.tf +++ b/2-nomad-config/main.tf @@ -21,7 +21,7 @@ terraform { } provider "nomad" { - address = "http://jaglan-beta-m01.lan:4646" + address = "http://jaglan-beta-m20.lan:4646" } data "sops_file" "secrets" { diff --git a/2-nomad-config/postgres.tf b/2-nomad-config/postgres.tf index 5ab752b..2ff6cf6 100644 --- a/2-nomad-config/postgres.tf +++ b/2-nomad-config/postgres.tf @@ -19,7 +19,7 @@ resource "nomad_variable" "postgres" { } provider "postgresql" { - host = "jaglan-beta-m01.lan" + host = "jaglan-beta-m21.lan" port = 5432 database = "postgres" username = "postgres" diff --git a/2-nomad-config/readme.md b/2-nomad-config/readme.md index b847050..5ce3cba 100644 --- a/2-nomad-config/readme.md +++ b/2-nomad-config/readme.md @@ -16,3 +16,16 @@ Put the age keys in /home//.config/sops/age/keys.txt ## Adding Secrets Edit the secrets using `sops secrets.enc.json` + +# Bootstrapping (starting without PostgreSQL running) + +terraform apply -target=nomad_job.postgres + +## Restoring PostgreSQL DBs + +`psql -h jaglan-beta-m21 -p 5432 -U postgres -f ~/Downloads/all_databases.sql postgres` + +# Deploying and testing changes + +Sometimes the nomad job fails but the solution is to fix another job and so we need to tell nomad to retry the unchanged job. +`nomad job eval -force-reschedule glance` diff --git a/2-nomad-config/secrets.enc.json b/2-nomad-config/secrets.enc.json index ed35079..8286fbd 100644 --- a/2-nomad-config/secrets.enc.json +++ b/2-nomad-config/secrets.enc.json @@ -26,6 +26,9 @@ "hass": { "magic-token": "ENC[AES256_GCM,data:3mKbPFgvtX+hWYEZ0q4jBjnR8KM+E/1DqmkVzoV6ROY=,iv:9L748apqK1TcsW0Y0HvU9QHVD/eSh56c/uN/K4KNct4=,tag:ZmXiaPz7MEvaQ0yu3byiKQ==,type:str]" }, + "traefik": { + "cf_tunnel_token": "ENC[AES256_GCM,data:IgrmKwdeipix1dIXNuXnTWN5rCZjClbKZQJfgr5c2IP/n8bcc/nG5Wb42WL2C4hTeVqhG5p62ZXoz0j4dNAjxvuzcW/P0XeSYaiDRXMNWKhNIcK7jOexgswio0sUC+F7f3fa6HH4C02Mx8dWoFZChYtM5EhGdcEwVwspyBlMhTSHTz+/w5T9OqH18o132ZTM6kMQY85sgH36azWoSw73N+aC4ANhgybuok06z6R5D2jMdDX47Bo5bg==,iv:yOcUDTYHh58iejbl0wxNJO1hcDypcBq6KlHKyqnMSVk=,tag:CMyHKgahkIGdXItMJ1/hOg==,type:str]" + }, "sops": { "age": [ { @@ -33,8 +36,8 @@ "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSByUWM4ZDVVbGFrUGdMRHBX\nUFBmU3Nlc0RBSzhFK0tHNHpkQXUvUVdiZUZJCmpRN1lFdENpWW0rcThjVlVQNUl6\nWnlLU0RnQ3FZby81Ly8xTFBrek9nMncKLS0tIFQ4UTRNOC9CRmx4OFJWem1wckZz\nUDFTSzdWZldFK3FqcTNWTWRyNDhHQ2MKS811mR5xn7qiC/aVgPFYJ5c6Q3zxRfcr\nHcvxUvB01vNJKZpRg92vvKPkV6lQO3DXCT98OdfwiymlEOvYxg71Pg==\n-----END AGE ENCRYPTED FILE-----\n" } ], - "lastmodified": "2025-10-04T04:09:12Z", - "mac": "ENC[AES256_GCM,data:+NnopVex61fOpxTSMhkrBQXB2Zq1Vj4a5kNrdFI2o947NCMkRxtTyYYP+7xEsk97P0z7eUCRE0xG5vMU0u+w+i+wgV5OtlSlJMoLJaXA2Rtxvd+THmzk9CEWHpxRzyZFGB5r124LANiMXb+YWEH2HYEqmk0Y0TiOGAqnN2Z0kzo=,iv:Xf49WoI27nJf3RdIaDqRdxITpizXFT3Uht/MWxjJInE=,tag:o/WS1Nk0Q9o/fB881saaOw==,type:str]", + "lastmodified": "2025-10-11T03:45:00Z", + "mac": "ENC[AES256_GCM,data:zLaCBYKSnXc81LOSuKHlBkiclsvV2B88SUFvy5oW2kL+Sbu3x058oE0P6T+9BlKkfKWLuTKjMwzAQGPeY0wjHtS4htxpcnMl+uVni888jr3lDLe7IPDReOp7Liv7UddQnLP9ftLIxZEOCeFtCP3/jkCEFbD8t30tLM2hSO6MjgM=,iv:IuMsWEQWZT537vMgfOFxdw2kwJO2rB5PmhsjJsnE934=,tag:8j5hVJCsvffebuj2Ay7zgg==,type:str]", "encrypted_regex": "^(.*)$", "version": "3.10.2" } diff --git a/2-nomad-config/traefik.nomad.hcl b/2-nomad-config/traefik.nomad.hcl index b6e6669..a8402b3 100644 --- a/2-nomad-config/traefik.nomad.hcl +++ b/2-nomad-config/traefik.nomad.hcl @@ -112,7 +112,7 @@ http: middlewares: auth: forwardAuth: - address: "http://192.168.1.235:9091/api/authz/forward-auth" + address: "http://jaglan-beta-m21:9091/api/authz/forward-auth" trustForwardHeader: true auth-allow-token: chain: @@ -192,5 +192,24 @@ EOF memory = 128 } } + + task "cloudflared" { + driver = "docker" + + config { + image = "cloudflare/cloudflared:latest" + args = [ + "tunnel", "--no-autoupdate", "run" + ] + } + + template { + data = <