add: playbook to update OPNsense HA cluster

Reviewed-on: #6

ansible/opnsense/requirements.yml

@@ -0,0 +1,4 @@
+---
+collections:
+  - name: community.proxmox
+    version: "2.0.0"

ansible/opnsense/update_opnsense_ha_cluster.yml

@@ -0,0 +1,432 @@
+---
+# OPNsense HA Cluster Update
+# Order: backup (TrueNAS) first, then master (Proxmox)
+###############################################################
+- name: "Phase 1 | Firmware check on all nodes"
+  hosts: opnsense
+  any_errors_fatal: true
+  gather_facts: false
+  tasks:
+
+    - block:
+
+        - name: Check node availability
+          ansible.builtin.uri:
+            url: "https://{{ opnsense_host }}/api/core/system/status"
+            method: GET
+            user: "{{ opnsense_api_key }}"
+            password: "{{ opnsense_api_secret }}"
+            force_basic_auth: true
+            validate_certs: false
+
+        - name: Trigger firmware check
+          ansible.builtin.uri:
+            url: "https://{{ opnsense_host }}/api/core/firmware/check"
+            method: POST
+            user: "{{ opnsense_api_key }}"
+            password: "{{ opnsense_api_secret }}"
+            force_basic_auth: true
+            validate_certs: false
+
+        - name: Check VIP status 
+          ansible.builtin.uri:
+            url: "https://{{ opnsense_host }}/api/diagnostics/interface/get_vip_status"
+            method: GET
+            user: "{{ opnsense_api_key }}"
+            password: "{{ opnsense_api_secret }}"
+            force_basic_auth: true
+            validate_certs: false
+          register: _vip_status
+
+        - name: Wait for check to complete
+          ansible.builtin.uri:
+            url: "https://{{ opnsense_host }}/api/core/firmware/status"
+            method: GET
+            user: "{{ opnsense_api_key }}"
+            password: "{{ opnsense_api_secret }}"
+            force_basic_auth: true
+            validate_certs: false
+          register: _firmware_status 
+          until: _firmware_status.json.status_msg != "Firmware status requires to check for update first to provide more information."
+          retries: 10
+          delay: 2
+
+        - name: Store node facts
+          ansible.builtin.set_fact:
+            firmware_action: "{{ opnsense_action | default('update')}}"
+            firmware_status: "{{ _firmware_status.json.status }}"
+            firmware_status_msg: "{{ _firmware_status.json.status_msg }}"
+            firmware_current_version: "{{ _firmware_status.json.product.product_version | default('unknown') }}"
+            firmware_product_series: "{{ _firmware_status.json.product.product_series | default('unknown') }}"
+            firmware_update_version: "{{ _firmware_status.json.upgrade_packages | selectattr('name', 'equalto', 'opnsense') | map(attribute='new_version') | first | default('') }}"
+            firmware_upgrade_version: "{{ _firmware_status.json.upgrade_major_version | default('unknown') }}"
+            firmware_upgrade_message: "{{ _firmware_status.json.upgrade_major_message | regex_replace('<[^>]+>', ' ') | regex_replace('\\s{2,}', ' ') | trim | default('') }}"
+            firmware_up_to_date: "{{ _firmware_status.json.status_msg == 'There are no updates available on the selected mirror.' }}"                   
+            needs_reboot: "{{ (_firmware_status.json.upgrade_needs_reboot == '1') if _firmware_status.json.status == 'upgrade' else (_firmware_status.json.needs_reboot == '1') }}"
+            total_vips: "{{ _vip_status.json.rowCount }}"
+            mismatched_vips: "{{ _vip_status.json.rows | rejectattr('status', 'equalto', main_role) | list | length }}"
+            in_maintenance: "{{ _vip_status.json.carp.maintenancemode }}"
+
+        - name: Resolve update-vs-upgrade specifics
+          ansible.builtin.set_fact:
+            firmware_target_kind: "{{ 'series' if firmware_status == 'upgrade' else 'version' }}"
+            firmware_target_value: "{{ firmware_upgrade_version if firmware_status == 'upgrade' else firmware_update_version }}"
+
+        - name: Backup already updated
+          ansible.builtin.set_fact:
+            skip_update: true
+            firmware_status: "skipped"
+          delegate_to: "{{ groups['opnsense_backup'][0] }}"
+          delegate_facts: true
+          run_once: true
+          when: >-
+            (hostvars[groups['opnsense_backup'][0]].firmware_current_version
+            if hostvars[groups['opnsense_master'][0]].firmware_target_kind == 'version'
+            else hostvars[groups['opnsense_backup'][0]].firmware_product_series)
+            == hostvars[groups['opnsense_master'][0]].firmware_target_value
+
+        - name: Skip node {{ firmware_action }}
+          ansible.builtin.set_fact:
+            skip_update: true
+            firmware_status: "skipped"
+          when: firmware_up_to_date or firmware_action != firmware_status
+        
+        - name: "Assert"
+          ansible.builtin.assert:
+            that: 
+              - not in_maintenance
+              - mismatched_vips == 0
+              - total_vips > 0
+            quiet: true
+
+      rescue:
+
+        - name: Send Ntfy notification
+          ansible.builtin.uri:
+            url: "{{  ntfy_url }}/{{ ntfy_topic }}"
+            method: POST
+            user: "{{ ntfy_user }}"
+            password: "{{ lookup('env', 'NTFY_PASSWORD') }}"
+            force_basic_auth: true
+            body: |
+              Current version: {{ firmware_current_version }}
+              {% if in_maintenance %}
+              The node {{ inventory_hostname }} is in CARP maintenance mode.
+              {% elif mismatched_vips > 0 %}
+              Some VIP are not {{ main_role }} on {{ ansible_host }}.
+              {% elif total_vips == 0 %}
+              No VIPs are managed.
+              {% else %}
+              An error occured during the {{ firmware_action }}.
+              {% endif %}
+            headers:
+              Title: "OPNsense {{ firmware_action }} report"
+              Priority: "default"
+              Tags: "x"
+          when: ntfy_url is defined
+
+        - ansible.builtin.fail:
+            msg: "Update aborted"
+
+
+- name: "Phase 2 | Update backup node (TrueNAS, no snapshot)"
+  hosts: opnsense_backup
+  any_errors_fatal: true
+  gather_facts: false
+  tasks:
+
+    - when: not skip_update | default(false)
+      block:
+
+        - name: Enable CARP maintenance mode
+          ansible.builtin.uri:
+            url: "https://{{ opnsense_host }}/api/diagnostics/interface/carp_status/maintenance"
+            method: POST
+            user: "{{ opnsense_api_key }}"
+            password: "{{ opnsense_api_secret }}"
+            force_basic_auth: true
+            validate_certs: false
+          register: _enable_maintenance
+          changed_when: _enable_maintenance.status == "ok"
+
+        - name: Trigger firmware {{ firmware_action }}
+          ansible.builtin.uri:
+            url: "https://{{ opnsense_host }}/api/core/firmware/{{ firmware_action }}"
+            method: POST
+            user: "{{ opnsense_api_key }}"
+            password: "{{ opnsense_api_secret }}"
+            force_basic_auth: true
+            validate_certs: false
+
+        - name: Check if the {{ firmware_action }} is running
+          ansible.builtin.uri:
+            url: "https://{{ opnsense_host }}/api/core/firmware/running"
+            method: GET
+            user: "{{ opnsense_api_key }}"
+            password: "{{ opnsense_api_secret }}"
+            force_basic_auth: true
+            validate_certs: false
+          register: _update_running
+          until: _update_running.json.status == 'busy'
+          retries: 2
+          delay: 2
+
+        - name: Wait for node to reboot after the {{ firmware_action }}
+          ansible.builtin.wait_for:
+            host: "{{ ansible_host }}"
+            port: "{{ opnsense_https_port }}"
+            state: stopped
+            timeout: 3600
+          when: needs_reboot
+
+        - name: Wait for node to come back online
+          ansible.builtin.wait_for:
+            host: "{{ ansible_host }}"
+            port: "{{ opnsense_https_port }}"
+            state: started
+            timeout: 5400
+            delay: 30
+          when: needs_reboot
+
+        - name: Check firmware version
+          ansible.builtin.uri:
+            url: "https://{{ opnsense_host }}/api/core/firmware/status"
+            method: GET
+            user: "{{ opnsense_api_key }}"
+            password: "{{ opnsense_api_secret }}"
+            force_basic_auth: true
+            validate_certs: false
+          register: _post_firmware_status
+          until: _post_firmware_status.json.product['product_' ~ firmware_target_kind] | default('unknown') == firmware_target_value
+          retries: 240
+          delay: 15
+
+        - name: Check VIP status 
+          ansible.builtin.uri:
+            url: "https://{{ opnsense_host }}/api/diagnostics/interface/get_vip_status"
+            method: GET
+            user: "{{ opnsense_api_key }}"
+            password: "{{ opnsense_api_secret }}"
+            force_basic_auth: true
+            validate_certs: false
+          register: _carp_post_udapte
+
+        - name: Disable CARP maintenance mode
+          ansible.builtin.uri:
+            url: "https://{{ opnsense_host }}/api/diagnostics/interface/carp_status/maintenance"
+            method: POST
+            user: "{{ opnsense_api_key }}"
+            password: "{{ opnsense_api_secret }}"
+            force_basic_auth: true
+            validate_certs: false
+          when: _carp_post_udapte.json.carp.maintenancemode
+          ignore_errors: true
+
+      rescue:
+
+        - name: Send Ntfy notification
+          ansible.builtin.uri:
+            url: "{{  ntfy_url }}/{{ ntfy_topic }}"
+            method: POST
+            user: "{{ ntfy_user }}"
+            password: "{{ lookup('env', 'NTFY_PASSWORD') }}"
+            force_basic_auth: true
+            body: |
+              Current version: {{ firmware_current_version }}
+              {{ firmware_action | capitalize }} to {{ firmware_target_value }} failed on {{ inventory_hostname }} ({{ main_role }}).
+              No snapshot have been taken before the {{ firmware_action }}.
+            headers:
+              Title: "OPNsense {{ firmware_action }} failed on {{ inventory_hostname }}"
+              Priority: "high"
+              Tags: "x"
+          when: ntfy_url is defined
+
+        - ansible.builtin.fail:
+            msg: "Update failed"
+
+- name: "Phase 3 | Update master node (Proxmox, with snapshot)"
+  hosts: opnsense_master
+  any_errors_fatal: true
+  gather_facts: false
+  vars:
+    proxmox_snap_name: "preupdate-{{ now().strftime('%Y%m%d') }}"
+
+  tasks:
+
+    - when: not skip_update | default(false)
+      block:
+
+        - name: Take Proxmox VM snapshot
+          community.proxmox.proxmox_snap:
+            vmid: "{{ proxmox_vmid }}"
+            state: present
+            snapname: "{{ proxmox_snap_name }}"
+            description: "Pre-firmware-{{ firmware_action }}: {{ firmware_current_version }} → {{ firmware_target_value }}"
+
+        - name: Enable CARP maintenance mode
+          ansible.builtin.uri:
+            url: "https://{{ opnsense_host }}/api/diagnostics/interface/carp_status/maintenance"
+            method: POST
+            user: "{{ opnsense_api_key }}"
+            password: "{{ opnsense_api_secret }}"
+            force_basic_auth: true
+            validate_certs: false
+          register: _enable_maintenance
+          changed_when: _enable_maintenance.status == "ok"
+
+        - name: Wait for CARP failover to settle
+          ansible.builtin.pause:
+            seconds: 5
+
+        - name: Trigger firmware {{ firmware_action }}
+          ansible.builtin.uri:
+            url: "https://{{ opnsense_host }}/api/core/firmware/{{ firmware_action }}"
+            method: POST
+            user: "{{ opnsense_api_key }}"
+            password: "{{ opnsense_api_secret }}"
+            force_basic_auth: true
+            validate_certs: false
+
+        - name: Check if the {{ firmware_action }} is running
+          ansible.builtin.uri:
+            url: "https://{{ opnsense_host }}/api/core/firmware/running"
+            method: GET
+            user: "{{ opnsense_api_key }}"
+            password: "{{ opnsense_api_secret }}"
+            force_basic_auth: true
+            validate_certs: false
+          register: _update_running
+          until: _update_running.json.status == 'busy'
+          retries: 2
+          delay: 2
+
+        - name: Wait for node to reboot after the {{ firmware_action }}
+          ansible.builtin.wait_for:
+            host: "{{ ansible_host }}"
+            port: "{{ opnsense_https_port }}"
+            state: stopped
+            timeout: 3600
+          when: needs_reboot
+
+        - name: Wait for node to come back online
+          ansible.builtin.wait_for:
+            host: "{{ ansible_host }}"
+            port: "{{ opnsense_https_port }}"
+            state: started
+            timeout: 5400
+            delay: 30
+          when: needs_reboot
+
+        - name: Check firmware version
+          ansible.builtin.uri:
+            url: "https://{{ opnsense_host }}/api/core/firmware/status"
+            method: GET
+            user: "{{ opnsense_api_key }}"
+            password: "{{ opnsense_api_secret }}"
+            force_basic_auth: true
+            validate_certs: false
+          register: _post_firmware_status
+          until: _post_firmware_status.json.product['product_' ~ firmware_target_kind] | default('unknown') == firmware_target_value
+          retries: 240
+          delay: 15
+
+        - name: Check VIP status 
+          ansible.builtin.uri:
+            url: "https://{{ opnsense_host }}/api/diagnostics/interface/get_vip_status"
+            method: GET
+            user: "{{ opnsense_api_key }}"
+            password: "{{ opnsense_api_secret }}"
+            force_basic_auth: true
+            validate_certs: false
+          register: _carp_post_udapte
+
+        - name: Disable CARP maintenance mode
+          ansible.builtin.uri:
+            url: "https://{{ opnsense_host }}/api/diagnostics/interface/carp_status/maintenance"
+            method: POST
+            user: "{{ opnsense_api_key }}"
+            password: "{{ opnsense_api_secret }}"
+            force_basic_auth: true
+            validate_certs: false
+          when: _carp_post_udapte.json.carp.maintenancemode
+          ignore_errors: true
+
+      rescue:
+
+        - name: Rollback VM to the pre-{{ firmware_action }} snapshot
+          community.proxmox.proxmox_snap:
+            vmid: "{{ proxmox_vmid }}"
+            state: rollback
+            snapname: "{{ proxmox_snap_name }}"
+          register: _proxmox_snapshot_rollback
+          ignore_errors: true
+
+        - name: Send Ntfy notification
+          ansible.builtin.uri:
+            url: "{{  ntfy_url }}/{{ ntfy_topic }}"
+            method: POST
+            user: "{{ ntfy_user }}"
+            password: "{{ lookup('env', 'NTFY_PASSWORD') }}"
+            force_basic_auth: true
+            body: |
+              Current version: {{ firmware_current_version }}
+              {{ firmware_action | capitalize }} to {{ firmware_target_value }} failed on {{ inventory_hostname }} ({{ main_role }}).
+              A snapshot have been taken before the {{ firmware_action }} and the VM has been rollback.
+            headers:
+              Title: "OPNsense {{ firmware_action }} failed on {{ inventory_hostname }}"
+              Priority: "high"
+              Tags: "x"
+          when: ntfy_url is defined
+
+        - ansible.builtin.fail:
+            msg: "Update aborted"
+
+
+- name: "Phase 4 | Notification"
+  hosts: localhost
+  gather_facts: false
+  tasks:
+
+    - name: Send Ntfy notification
+      vars:
+        master: "{{ groups['opnsense_master'][0] }}"
+        backup: "{{ groups['opnsense_backup'][0] }}"
+        m: "{{ hostvars[master] }}"
+        b: "{{ hostvars[backup] }}"
+        same_operation: "{{ m.firmware_status == b.firmware_status }}"
+
+      ansible.builtin.uri:
+        url: "{{ ntfy_url }}/{{ ntfy_topic }}"
+        method: POST
+        user: "{{ ntfy_user }}"
+        password: "{{ lookup('env', 'NTFY_PASSWORD') }}"
+        force_basic_auth: true
+        body: |
+          {% if same_operation %}
+          {% if m.skip_update | default(false) %}
+          Both nodes already on {{ m.firmware_current_version }}, no action taken.
+          {% else %}
+          OPNsense cluster: {{ m.firmware_current_version }} → {{ m.firmware_target_value }} ({{ m.firmware_status }})
+          {% endif %}
+          {% else %}
+          {% if m.skip_update | default(false) %}
+          MASTER ({{ master }}): already on {{ m.firmware_current_version }}, no action taken.
+          {% else %}
+          MASTER ({{ master }}): {{ m.firmware_current_version }} → {{ m.firmware_target_value }} ({{ m.firmware_status }})
+          {% endif %}
+          {% if b.skip_update | default(false) %}
+          BACKUP ({{ backup }}): already on {{ b.firmware_current_version }}, no action taken.
+          {% else %}
+          BACKUP ({{ backup }}): {{ b.firmware_current_version }} → {{ b.firmware_target_value }} ({{ b.firmware_status }})
+          {% endif %}
+          {% endif %}
+        headers:
+          Title: >-
+            {%- if m.skip_update | default(false) and b.skip_update | default(false) -%}
+            OPNsense cluster - no {{m.firmware_action }} available
+            {%- else -%}
+            OPNsense cluster {{m.firmware_action }} completed
+            {%- endif -%}
+          Priority: "{{ 'min' if (m.skip_update | default(false) and b.skip_update | default(false)) else 'default' }}"
+          Tags: "{{ 'information_source' if (m.skip_update | default(false) and b.skip_update | default(false)) else 'white_check_mark' }}"
+      when: ntfy_url is defined

ansible/proxmox/cloudinit_template.yml

@@ -0,0 +1,120 @@
+---
+- name: Create or update a cloud-init Ubuntu template
+  hosts: nodes
+  become: true
+  vars:
+    template_vmid: 900
+    template_os: ubuntu
+    os_release: noble
+
+    template_name: "{{template_os}}-{{ os_release }}-cloud"
+ 
+    image_filename: "{{ os_release }}-server-cloudimg-amd64.img"
+    image_url: "https://cloud-images.ubuntu.com/{{ os_release }}/current/{{ image_filename }}"
+    image_dest: "/var/lib/vz/template/iso/{{ image_filename }}"
+ 
+    iso_storage: local
+    vm_storage: ceph-workload
+ 
+    template_memory: 2048
+    template_cores: 1
+ 
+  tasks:
+    - name: Check whether the template VMID already exists
+      ansible.builtin.command: qm status {{ template_vmid }}
+      register: template_status
+      changed_when: false
+      failed_when: false
+
+    - name: Check if the cloud image file exists
+      ansible.builtin.stat:
+        path: "{{ image_dest }}"
+      register: image_exists
+      changed_when: false
+      failed_when: false
+
+    - name: Set candidate information
+      ansible.builtin.set_fact:
+        has_image_file: "{{ image_exists.stat.exists }}"
+        root_free_space: "{{ (ansible_facts['mounts'] | selectattr('mount', 'equalto', '/') | first).size_available }}"
+
+    - name: Select target host
+      ansible.builtin.set_fact:
+        target_host: >-
+          {{ (ansible_play_hosts
+              | map('extract', hostvars)
+              | selectattr('has_image_file', 'equalto', true)
+              | sort(attribute='root_free_space', reverse=true)
+              | map(attribute='inventory_hostname')
+              | first)
+              | default(ansible_play_hosts
+              | map('extract', hostvars)
+              | sort(attribute='root_free_space', reverse=true)
+              | map(attribute='inventory_hostname')
+              | first,
+              true) }}
+      run_once: true
+
+    - name: Download the current Ubuntu cloud image
+      ansible.builtin.get_url:
+        url: "{{ image_url }}"
+        dest: "{{ image_dest }}"
+        force: false 
+        mode: "0644"
+      register: image_download
+      when: inventory_hostname == target_host
+
+    - name: Destroy existing template if present
+      ansible.builtin.command: qm destroy {{ template_vmid }} --purge
+      when: 
+        - template_status.rc == 0
+        - image_download.changed
+ 
+    - name: Rebuild the template
+      when: 
+        - inventory_hostname == target_host
+        - image_download.changed or 
+          template_status.rc != 0
+      block:
+ 
+        - name: Create the base VM shell
+          ansible.builtin.command: >
+            qm create {{ template_vmid }}
+            --memory {{ template_memory }}
+            --core {{ template_cores }}
+            --net0 virtio,bridge=vmbr0
+            --scsihw virtio-scsi-pci
+            --bios ovmf
+            --machine q35
+            --efidisk0 {{ vm_storage }}:0,pre-enrolled-keys=0
+            --name {{ template_name }}
+ 
+        - name: Import the cloud image as the primary disk
+          ansible.builtin.command: qm set {{ template_vmid }} --scsi0 {{ vm_storage }}:0,import-from={{ image_dest }}
+ 
+        - name: Attach a cloud-init drive
+          ansible.builtin.command: qm set {{ template_vmid }} --scsi1 {{ vm_storage }}:cloudinit
+ 
+        - name: Set boot order to the primary disk
+          ansible.builtin.command: qm set {{ template_vmid }} --boot order=scsi0
+ 
+        - name: Add serial console for cloud-init consoles
+          ansible.builtin.command: qm set {{ template_vmid }} --serial0 socket --vga serial0
+ 
+        - name: Convert the VM to a template
+          ansible.builtin.command: qm template {{ template_vmid }}
+
+        - name: Send notification
+          ansible.builtin.uri:
+            url: "{{  ntfy_url }}/{{ ntfy_topic }}"
+            method: POST
+            user: "{{ ntfy_user }}"
+            password: "{{ lookup('env', 'NTFY_PASSWORD') }}"
+            force_basic_auth: true
+            body: |
+              The {{ template_os | capitalize }} {{ os_release }} cloud-init template has been successfully updated on {{ inventory_hostname }}.
+            headers:
+              Title: "Cloud-init template updated"
+              Priority: "min"
+              Tags: "white_check_mark"
+          when: ntfy_url is defined

opnsense/10-wan

@@ -84,7 +84,7 @@ if ($type === "MASTER") {
         write_config("disable interface '$ifkey' due CARP event '$type'", false);
         interface_configure(false, $ifkey, false, false);
         // Remove WAN default gateway
-        mwexec("/sbin/route delete default");
+        mwexecf("/sbin/route delete default");
         foreach ($config['OPNsense']['Gateways']['gateway_item'] as $gw) {
             if ($gw['name'] === $gw_name) {
                 $gw_ip = $gw['gateway'];
@@ -92,9 +92,9 @@ if ($type === "MASTER") {
             }
         }
         // Shutdown WAN interface
-        mwexec("/sbin/ifconfig {$real_if} down")
+        mwexecf("/sbin/ifconfig {$real_if} down")
         // Add fallback default gateway
-        mwexec("/sbin/route add default {$gw_ip}");
+        mwexecf("/sbin/route add default {$gw_ip}");
         // Create lock file
         file_put_contents($lock_file, time());
         sleep($lock_default_age);

terraform/modules/pve_vm/variables.tf

@@ -12,7 +12,7 @@ variable "node_datastore" {
 variable "vm_template" {
   description = "Template of the VM"
   type        = string
-  default     = "ubuntu-cloud"
+  default     = "ubuntu-noble-cloud"
 }
 
 variable "vm_name" {

terraform/projects/simple-vm-telmate/main.tf

@@ -6,7 +6,7 @@ resource "proxmox_vm_qemu" "cloudinit-test" {
 
   target_node = "zenith"
 
-  clone = "ubuntu-cloud"
+  clone = "ubuntu-noble-cloud"
 
   agent = 0
 

terraform/projects/simple-vm/variables.tf

@@ -23,7 +23,7 @@ variable "node_datastore" {
 variable "vm_template" {
   description = "Template of the VM"
   type        = string
-  default     = "ubuntu-cloud"
+  default     = "ubuntu-noble-cloud"
 }
 
 variable "vm_name" {