add: get vip status

This reverts commit 3c8375d6f4.

ansible/opnsense/inventory.yml

@@ -17,4 +17,3 @@ all:
               ansible_host: 192.168.88.2
               main_role: MASTER
               hypervisor: Proxmox
-              proxmox_vmid: 122

ansible/opnsense/requirements.yml

@@ -1,4 +0,0 @@
----
-collections:
-  - name: community.proxmox
-    version: "2.0.0"

ansible/opnsense/update_opnsense_ha_cluster.yml

@@ -6,6 +6,7 @@
   hosts: opnsense
   any_errors_fatal: true
   gather_facts: false
+
   tasks:
 
     - block:
@@ -47,386 +48,455 @@
             force_basic_auth: true
             validate_certs: false
           register: _firmware_status
-          until: _firmware_status.json.status_msg != "Firmware status requires to check for update first to provide more information."
+          until: _firmware_status.json.status != 'none'
-          retries: 10
+          retries: 12
-          delay: 2
+          delay: 5
 
-        - name: Store node facts
+        - name: Store firmware facts
           ansible.builtin.set_fact:
-            firmware_action: "{{ opnsense_action | default('update')}}"
             firmware_status: "{{ _firmware_status.json.status }}"
-            firmware_status_msg: "{{ _firmware_status.json.status_msg }}"
+            fw_current_version: "{{ _firmware_status.json.product_version | default('unknown') }}"
-            firmware_current_version: "{{ _firmware_status.json.product.product_version | default('unknown') }}"
+            fw_new_version: "{{ _firmware_status.json.upgrade_packages | selectattr('name', 'equalto', 'opnsense') | map(attribute='new_version') | first | default('') }}"
-            firmware_product_series: "{{ _firmware_status.json.product.product_series | default('unknown') }}"
+            needs_reboot: "{{ true if _firmware_status.json.needs_reboot == '1' else false }}"
-            firmware_update_version: "{{ _firmware_status.json.upgrade_packages | selectattr('name', 'equalto', 'opnsense') | map(attribute='new_version') | first | default('') }}"
+            # 'pending' means updates exist; will be overwritten to success/rolled_back
-            firmware_upgrade_version: "{{ _firmware_status.json.upgrade_major_version | default('unknown') }}"
+            # If it stays 'pending' after the run, a pre-update check failed
-            firmware_upgrade_message: "{{ _firmware_status.json.upgrade_major_message | regex_replace('<[^>]+>', ' ') | regex_replace('\\s{2,}', ' ') | trim | default('') }}"
+            update_result: "{{ 'pending' if _firmware_status.json.status == 'update' else 'skipped' }}"
-            firmware_up_to_date: "{{ _firmware_status.json.status_msg == 'There are no updates available on the selected mirror.' }}"                   
-            needs_reboot: "{{ (_firmware_status.json.upgrade_needs_reboot == '1') if _firmware_status.json.status == 'upgrade' else (_firmware_status.json.needs_reboot == '1') }}"
-            total_vips: "{{ _vip_status.json.rowCount }}"
-            mismatched_vips: "{{ _vip_status.json.rows | rejectattr('status', 'equalto', main_role) | list | length }}"
-            in_maintenance: "{{ _vip_status.json.carp.maintenancemode }}"
 
-        - name: Resolve update-vs-upgrade specifics
+        - name: Display firmware status
-          ansible.builtin.set_fact:
+          ansible.builtin.debug:
-            firmware_target_kind: "{{ 'series' if firmware_status == 'upgrade' else 'version' }}"
+            msg: >-
-            firmware_target_value: "{{ firmware_upgrade_version if firmware_status == 'upgrade' else firmware_update_version }}"
+              {{ inventory_hostname }}: {{ firmware_status }}
+              ({{ fw_current_version }}
+              {% if fw_new_version %} → {{ fw_new_version }} (reboot: {{ needs_reboot }}){% endif %})
 
-        - name: Backup already updated
+        - name: Warn — major upgrade skipped, handle manually
-          ansible.builtin.set_fact:
+          ansible.builtin.debug:
-            skip_update: true
+            msg: >-
-            firmware_status: "skipped"
+              NOTICE: {{ inventory_hostname }} has a MAJOR upgrade available
-          delegate_to: "{{ groups['opnsense_backup'][0] }}"
+              ({{ fw_new_version }}). This playbook handles minor updates only.
-          delegate_facts: true
+          when: firmware_status == 'upgrade'
-          run_once: true
-          when: >-
-            (hostvars[groups['opnsense_backup'][0]].firmware_current_version
-            if hostvars[groups['opnsense_master'][0]].firmware_target_kind == 'version'
-            else hostvars[groups['opnsense_backup'][0]].firmware_product_series)
-            == hostvars[groups['opnsense_master'][0]].firmware_target_value
-
-        - name: Skip node {{ firmware_action }}
-          ansible.builtin.set_fact:
-            skip_update: true
-            firmware_status: "skipped"
-          when: firmware_up_to_date or firmware_action != firmware_status
-        
-        - name: "Assert"
-          ansible.builtin.assert:
-            that: 
-              - not in_maintenance
-              - mismatched_vips == 0
-              - total_vips > 0
-            quiet: true
-
-      rescue:
-
-        - name: Send Ntfy notification
-          ansible.builtin.uri:
-            url: "{{  ntfy_url }}/{{ ntfy_topic }}"
-            method: POST
-            user: "{{ ntfy_user }}"
-            password: "{{ lookup('env', 'NTFY_PASSWORD') }}"
-            force_basic_auth: true
-            body: |
-              Current version: {{ firmware_current_version }}
-              {% if in_maintenance %}
-              The node {{ inventory_hostname }} is in CARP maintenance mode.
-              {% elif mismatched_vips > 0 %}
-              Some VIP are not {{ main_role }} on {{ ansible_host }}.
-              {% elif total_vips == 0 %}
-              No VIPs are managed.
-              {% else %}
-              An error occured during the {{ firmware_action }}.
-              {% endif %}
-            headers:
-              Title: "OPNsense {{ firmware_action }} report"
-              Priority: "default"
-              Tags: "x"
-          when: ntfy_url is defined
-
-        - ansible.builtin.fail:
-            msg: "Update aborted"
 
 
-- name: "Phase 2 | Update backup node (TrueNAS, no snapshot)"
+# ###############################################################
-  hosts: opnsense_backup
+# - name: "Phase 2 | Update backup node (TrueNAS, no snapshot)"
-  any_errors_fatal: true
+#   hosts: opnsense_backup
-  gather_facts: false
+#   gather_facts: false
-  tasks:
 
-    - when: not skip_update | default(false)
+#   tasks:
-      block:
+#     - name: Skip — no minor updates available on backup
+#       ansible.builtin.meta: end_host
+#       when: firmware_status != 'update'
 
-        - name: Enable CARP maintenance mode
+#     # Safety: confirm backup is actually in BACKUP role before touching it
-          ansible.builtin.uri:
+#     - name: Verify backup node CARP role
-            url: "https://{{ opnsense_host }}/api/diagnostics/interface/carp_status/maintenance"
+#       ansible.builtin.uri:
-            method: POST
+#         url: "https://{{ opnsense_host }}/api/carp/status/get"
-            user: "{{ opnsense_api_key }}"
+#         method: GET
-            password: "{{ opnsense_api_secret }}"
+#         user: "{{ opnsense_api_key }}"
-            force_basic_auth: true
+#         password: "{{ opnsense_api_secret }}"
-            validate_certs: false
+#         force_basic_auth: true
-          register: _enable_maintenance
+#         validate_certs: false
-          changed_when: _enable_maintenance.status == "ok"
+#       register: _carp
+#       # Abort if node is unexpectedly MASTER — updating it would disrupt traffic
+#       failed_when: _carp.json.status | upper != 'BACKUP'
 
-        - name: Trigger firmware {{ firmware_action }}
+#     # Maintenance mode prevents the backup from attempting to take over
-          ansible.builtin.uri:
+#     # during its own reboot (avoids CARP flapping)
-            url: "https://{{ opnsense_host }}/api/core/firmware/{{ firmware_action }}"
+#     - name: Enable CARP maintenance mode
-            method: POST
+#       ansible.builtin.uri:
-            user: "{{ opnsense_api_key }}"
+#         url: "https://{{ opnsense_host }}/api/carp/status/maintenancemode"
-            password: "{{ opnsense_api_secret }}"
+#         method: POST
-            force_basic_auth: true
+#         user: "{{ opnsense_api_key }}"
-            validate_certs: false
+#         password: "{{ opnsense_api_secret }}"
+#         force_basic_auth: true
+#         validate_certs: false
+#         body_format: json
+#         body: {}
 
-        - name: Check if the {{ firmware_action }} is running
+#     - name: Trigger firmware update
-          ansible.builtin.uri:
+#       ansible.builtin.uri:
-            url: "https://{{ opnsense_host }}/api/core/firmware/running"
+#         url: "https://{{ opnsense_host }}/api/core/firmware/update"
-            method: GET
+#         method: POST
-            user: "{{ opnsense_api_key }}"
+#         user: "{{ opnsense_api_key }}"
-            password: "{{ opnsense_api_secret }}"
+#         password: "{{ opnsense_api_secret }}"
-            force_basic_auth: true
+#         force_basic_auth: true
-            validate_certs: false
+#         validate_certs: false
-          register: _update_running
+#         body_format: json
-          until: _update_running.json.status == 'busy'
+#         body: {}
-          retries: 2
-          delay: 2
 
-        - name: Wait for node to reboot after the {{ firmware_action }}
+#     - name: Wait for packages to install
-          ansible.builtin.wait_for:
+#       ansible.builtin.uri:
-            host: "{{ ansible_host }}"
+#         url: "https://{{ opnsense_host }}/api/core/firmware/running"
-            port: "{{ opnsense_https_port }}"
+#         method: GET
-            state: stopped
+#         user: "{{ opnsense_api_key }}"
-            timeout: 3600
+#         password: "{{ opnsense_api_secret }}"
-          when: needs_reboot
+#         force_basic_auth: true
+#         validate_certs: false
+#       register: _running
+#       until: _running.json.status != 'running'
+#       retries: 30
+#       delay: 15
 
-        - name: Wait for node to come back online
+#     # Reboot is a separate explicit step — update does NOT auto-reboot
-          ansible.builtin.wait_for:
+#     - name: Reboot node
-            host: "{{ ansible_host }}"
+#       ansible.builtin.uri:
-            port: "{{ opnsense_https_port }}"
+#         url: "https://{{ opnsense_host }}/api/core/firmware/reboot"
-            state: started
+#         method: POST
-            timeout: 5400
+#         user: "{{ opnsense_api_key }}"
-            delay: 30
+#         password: "{{ opnsense_api_secret }}"
-          when: needs_reboot
+#         force_basic_auth: true
+#         validate_certs: false
+#         body_format: json
+#         body: {}
+#       ignore_errors: true  # Node drops the connection immediately
 
-        - name: Check firmware version
+#     - name: Wait for node to go offline
-          ansible.builtin.uri:
+#       ansible.builtin.wait_for:
-            url: "https://{{ opnsense_host }}/api/core/firmware/status"
+#         host: "{{ ansible_host }}"
-            method: GET
+#         port: 443
-            user: "{{ opnsense_api_key }}"
+#         state: stopped
-            password: "{{ opnsense_api_secret }}"
+#         timeout: 60
-            force_basic_auth: true
+#       delegate_to: localhost
-            validate_certs: false
-          register: _post_firmware_status
-          until: _post_firmware_status.json.product['product_' ~ firmware_target_kind] | default('unknown') == firmware_target_value
-          retries: 240
-          delay: 15
 
-        - name: Check VIP status 
+#     - name: Wait for node to come back online
-          ansible.builtin.uri:
+#       ansible.builtin.wait_for:
-            url: "https://{{ opnsense_host }}/api/diagnostics/interface/get_vip_status"
+#         host: "{{ ansible_host }}"
-            method: GET
+#         port: 443
-            user: "{{ opnsense_api_key }}"
+#         state: started
-            password: "{{ opnsense_api_secret }}"
+#         timeout: 300
-            force_basic_auth: true
+#         delay: 30
-            validate_certs: false
+#       delegate_to: localhost
-          register: _carp_post_udapte
 
-        - name: Disable CARP maintenance mode
+#     - name: Wait for API to be fully responsive
-          ansible.builtin.uri:
+#       ansible.builtin.uri:
-            url: "https://{{ opnsense_host }}/api/diagnostics/interface/carp_status/maintenance"
+#         url: "https://{{ opnsense_host }}/api/core/firmware/status"
-            method: POST
+#         method: GET
-            user: "{{ opnsense_api_key }}"
+#         user: "{{ opnsense_api_key }}"
-            password: "{{ opnsense_api_secret }}"
+#         password: "{{ opnsense_api_secret }}"
-            force_basic_auth: true
+#         force_basic_auth: true
-            validate_certs: false
+#         validate_certs: false
-          when: _carp_post_udapte.json.carp.maintenancemode
+#       register: _post_status
-          ignore_errors: true
+#       until: _post_status.status == 200
+#       retries: 20
+#       delay: 10
 
-      rescue:
+#     - name: Health check — no remaining updates
+#       ansible.builtin.assert:
+#         that:
+#           - _post_status.json.status == 'none'
+#         fail_msg: >-
+#           Post-update firmware status is '{{ _post_status.json.status }}'
+#           on {{ inventory_hostname }}. Expected 'none'.
 
-        - name: Send Ntfy notification
+#     - name: Health check — CARP role is BACKUP
-          ansible.builtin.uri:
+#       ansible.builtin.uri:
-            url: "{{  ntfy_url }}/{{ ntfy_topic }}"
+#         url: "https://{{ opnsense_host }}/api/carp/status/get"
-            method: POST
+#         method: GET
-            user: "{{ ntfy_user }}"
+#         user: "{{ opnsense_api_key }}"
-            password: "{{ lookup('env', 'NTFY_PASSWORD') }}"
+#         password: "{{ opnsense_api_secret }}"
-            force_basic_auth: true
+#         force_basic_auth: true
-            body: |
+#         validate_certs: false
-              Current version: {{ firmware_current_version }}
+#       register: _carp_post
-              {{ firmware_action | capitalize }} to {{ firmware_target_value }} failed on {{ inventory_hostname }} ({{ main_role }}).
+#       failed_when: _carp_post.json.status | upper not in ['BACKUP', 'MAINTENANCE']
-              No snapshot have been taken before the {{ firmware_action }}.
-            headers:
-              Title: "OPNsense {{ firmware_action }} failed on {{ inventory_hostname }}"
-              Priority: "high"
-              Tags: "x"
-          when: ntfy_url is defined
 
-        - ansible.builtin.fail:
+#     - name: Disable CARP maintenance mode
-            msg: "Update failed"
+#       ansible.builtin.uri:
+#         url: "https://{{ opnsense_host }}/api/carp/status/maintenancemode"
+#         method: POST
+#         user: "{{ opnsense_api_key }}"
+#         password: "{{ opnsense_api_secret }}"
+#         force_basic_auth: true
+#         validate_certs: false
+#         body_format: json
+#         body: {}
 
-- name: "Phase 3 | Update master node (Proxmox, with snapshot)"
+#     - name: Record backup update result
-  hosts: opnsense_master
+#       ansible.builtin.set_fact:
-  any_errors_fatal: true
+#         update_result: "success"
-  gather_facts: false
+#         fw_post_version: "{{ _post_status.json.product_version | default('unknown') }}"
-  vars:
-    proxmox_snap_name: "preupdate-{{ now().strftime('%Y%m%d') }}"
-
-  tasks:
-
-    - when: not skip_update | default(false)
-      block:
-
-        - name: Take Proxmox VM snapshot
-          community.proxmox.proxmox_snap:
-            vmid: "{{ proxmox_vmid }}"
-            state: present
-            snapname: "{{ proxmox_snap_name }}"
-            description: "Pre-firmware-{{ firmware_action }}: {{ firmware_current_version }} → {{ firmware_target_value }}"
-
-        - name: Enable CARP maintenance mode
-          ansible.builtin.uri:
-            url: "https://{{ opnsense_host }}/api/diagnostics/interface/carp_status/maintenance"
-            method: POST
-            user: "{{ opnsense_api_key }}"
-            password: "{{ opnsense_api_secret }}"
-            force_basic_auth: true
-            validate_certs: false
-          register: _enable_maintenance
-          changed_when: _enable_maintenance.status == "ok"
-
-        - name: Wait for CARP failover to settle
-          ansible.builtin.pause:
-            seconds: 5
-
-        - name: Trigger firmware {{ firmware_action }}
-          ansible.builtin.uri:
-            url: "https://{{ opnsense_host }}/api/core/firmware/{{ firmware_action }}"
-            method: POST
-            user: "{{ opnsense_api_key }}"
-            password: "{{ opnsense_api_secret }}"
-            force_basic_auth: true
-            validate_certs: false
-
-        - name: Check if the {{ firmware_action }} is running
-          ansible.builtin.uri:
-            url: "https://{{ opnsense_host }}/api/core/firmware/running"
-            method: GET
-            user: "{{ opnsense_api_key }}"
-            password: "{{ opnsense_api_secret }}"
-            force_basic_auth: true
-            validate_certs: false
-          register: _update_running
-          until: _update_running.json.status == 'busy'
-          retries: 2
-          delay: 2
-
-        - name: Wait for node to reboot after the {{ firmware_action }}
-          ansible.builtin.wait_for:
-            host: "{{ ansible_host }}"
-            port: "{{ opnsense_https_port }}"
-            state: stopped
-            timeout: 3600
-          when: needs_reboot
-
-        - name: Wait for node to come back online
-          ansible.builtin.wait_for:
-            host: "{{ ansible_host }}"
-            port: "{{ opnsense_https_port }}"
-            state: started
-            timeout: 5400
-            delay: 30
-          when: needs_reboot
-
-        - name: Check firmware version
-          ansible.builtin.uri:
-            url: "https://{{ opnsense_host }}/api/core/firmware/status"
-            method: GET
-            user: "{{ opnsense_api_key }}"
-            password: "{{ opnsense_api_secret }}"
-            force_basic_auth: true
-            validate_certs: false
-          register: _post_firmware_status
-          until: _post_firmware_status.json.product['product_' ~ firmware_target_kind] | default('unknown') == firmware_target_value
-          retries: 240
-          delay: 15
-
-        - name: Check VIP status 
-          ansible.builtin.uri:
-            url: "https://{{ opnsense_host }}/api/diagnostics/interface/get_vip_status"
-            method: GET
-            user: "{{ opnsense_api_key }}"
-            password: "{{ opnsense_api_secret }}"
-            force_basic_auth: true
-            validate_certs: false
-          register: _carp_post_udapte
-
-        - name: Disable CARP maintenance mode
-          ansible.builtin.uri:
-            url: "https://{{ opnsense_host }}/api/diagnostics/interface/carp_status/maintenance"
-            method: POST
-            user: "{{ opnsense_api_key }}"
-            password: "{{ opnsense_api_secret }}"
-            force_basic_auth: true
-            validate_certs: false
-          when: _carp_post_udapte.json.carp.maintenancemode
-          ignore_errors: true
-
-      rescue:
-
-        - name: Rollback VM to the pre-{{ firmware_action }} snapshot
-          community.proxmox.proxmox_snap:
-            vmid: "{{ proxmox_vmid }}"
-            state: rollback
-            snapname: "{{ proxmox_snap_name }}"
-          register: _proxmox_snapshot_rollback
-          ignore_errors: true
-
-        - name: Send Ntfy notification
-          ansible.builtin.uri:
-            url: "{{  ntfy_url }}/{{ ntfy_topic }}"
-            method: POST
-            user: "{{ ntfy_user }}"
-            password: "{{ lookup('env', 'NTFY_PASSWORD') }}"
-            force_basic_auth: true
-            body: |
-              Current version: {{ firmware_current_version }}
-              {{ firmware_action | capitalize }} to {{ firmware_target_value }} failed on {{ inventory_hostname }} ({{ main_role }}).
-              A snapshot have been taken before the {{ firmware_action }} and the VM has been rollback.
-            headers:
-              Title: "OPNsense {{ firmware_action }} failed on {{ inventory_hostname }}"
-              Priority: "high"
-              Tags: "x"
-          when: ntfy_url is defined
-
-        - ansible.builtin.fail:
-            msg: "Update aborted"
 
 
-- name: "Phase 4 | Notification"
+# ###############################################################
-  hosts: localhost
+# - name: "Phase 3 | Update master node (Proxmox, with snapshot)"
-  gather_facts: false
+#   hosts: opnsense_master
-  tasks:
+#   gather_facts: false
 
-    - name: Send Ntfy notification
+#   vars:
-      vars:
+#     snap_name: "preupdate-{{ now().strftime('%Y%m%d-%H%M') }}"
-        master: "{{ groups['opnsense_master'][0] }}"
+#     _pve_base: "https://{{ proxmox_api_host }}/api2/json/nodes/{{ proxmox_node }}/qemu/{{ proxmox_vmid }}"
-        backup: "{{ groups['opnsense_backup'][0] }}"
+#     _pve_auth:
-        m: "{{ hostvars[master] }}"
+#       Authorization: "PVEAPIToken={{ proxmox_api_token_id }}={{ proxmox_api_token_secret }}"
-        b: "{{ hostvars[backup] }}"
-        same_operation: "{{ m.firmware_status == b.firmware_status }}"
 
-      ansible.builtin.uri:
+#   tasks:
-        url: "{{ ntfy_url }}/{{ ntfy_topic }}"
+#     - name: Skip — no minor updates available on master
-        method: POST
+#       ansible.builtin.meta: end_host
-        user: "{{ ntfy_user }}"
+#       when: firmware_status != 'update'
-        password: "{{ lookup('env', 'NTFY_PASSWORD') }}"
+
-        force_basic_auth: true
+#     - name: Verify master node CARP role
-        body: |
+#       ansible.builtin.uri:
-          {% if same_operation %}
+#         url: "https://{{ opnsense_host }}/api/carp/status/get"
-          {% if m.skip_update | default(false) %}
+#         method: GET
-          Both nodes already on {{ m.firmware_current_version }}, no action taken.
+#         user: "{{ opnsense_api_key }}"
-          {% else %}
+#         password: "{{ opnsense_api_secret }}"
-          OPNsense cluster: {{ m.firmware_current_version }} → {{ m.firmware_target_value }} ({{ m.firmware_status }})
+#         force_basic_auth: true
-          {% endif %}
+#         validate_certs: false
-          {% else %}
+#       register: _carp
-          {% if m.skip_update | default(false) %}
+#       failed_when: _carp.json.status | upper != 'MASTER'
-          MASTER ({{ master }}): already on {{ m.firmware_current_version }}, no action taken.
+
-          {% else %}
+#     # Snapshot BEFORE maintenance mode — the restore point should be
-          MASTER ({{ master }}): {{ m.firmware_current_version }} → {{ m.firmware_target_value }} ({{ m.firmware_status }})
+#     # a fully functioning MASTER node, not one mid-failover
-          {% endif %}
+#     - name: Take Proxmox VM snapshot
-          {% if b.skip_update | default(false) %}
+#       community.general.proxmox_snap:
-          BACKUP ({{ backup }}): already on {{ b.firmware_current_version }}, no action taken.
+#         api_host: "{{ proxmox_api_host }}"
-          {% else %}
+#         api_token_id: "{{ proxmox_api_token_id }}"
-          BACKUP ({{ backup }}): {{ b.firmware_current_version }} → {{ b.firmware_target_value }} ({{ b.firmware_status }})
+#         api_token_secret: "{{ proxmox_api_token_secret }}"
-          {% endif %}
+#         vmid: "{{ proxmox_vmid }}"
-          {% endif %}
+#         state: present
-        headers:
+#         snapname: "{{ snap_name }}"
-          Title: >-
+#         description: "Pre-firmware-update: {{ fw_current_version }} → {{ fw_new_version }}"
-            {%- if m.skip_update | default(false) and b.skip_update | default(false) -%}
+#         timeout: 60
-            OPNsense cluster - no {{m.firmware_action }} available
+#       delegate_to: localhost
-            {%- else -%}
+
-            OPNsense cluster {{m.firmware_action }} completed
+#     - name: Enable CARP maintenance mode (triggers controlled failover to backup)
-            {%- endif -%}
+#       ansible.builtin.uri:
-          Priority: "{{ 'min' if (m.skip_update | default(false) and b.skip_update | default(false)) else 'default' }}"
+#         url: "https://{{ opnsense_host }}/api/carp/status/maintenancemode"
-          Tags: "{{ 'information_source' if (m.skip_update | default(false) and b.skip_update | default(false)) else 'white_check_mark' }}"
+#         method: POST
-      when: ntfy_url is defined
+#         user: "{{ opnsense_api_key }}"
+#         password: "{{ opnsense_api_secret }}"
+#         force_basic_auth: true
+#         validate_certs: false
+#         body_format: json
+#         body: {}
+
+#     - name: Wait for CARP failover to settle
+#       ansible.builtin.pause:
+#         seconds: 10
+
+#     # Do not proceed if the backup has not actually taken over.
+#     # A failed failover means no traffic coverage during master update.
+#     - name: Confirm backup node is now MASTER
+#       ansible.builtin.uri:
+#         url: "https://{{ hostvars[groups['opnsense_backup'][0]].ansible_host }}/api/carp/status/get"
+#         method: GET
+#         user: "{{ opnsense_api_key }}"
+#         password: "{{ opnsense_api_secret }}"
+#         force_basic_auth: true
+#         validate_certs: false
+#       register: _backup_carp
+#       delegate_to: localhost
+#       failed_when: _backup_carp.json.status | upper != 'MASTER'
+
+#     - name: Update and verify
+#       block:
+
+#         - name: Trigger firmware update
+#           ansible.builtin.uri:
+#             url: "https://{{ opnsense_host }}/api/core/firmware/update"
+#             method: POST
+#             user: "{{ opnsense_api_key }}"
+#             password: "{{ opnsense_api_secret }}"
+#             force_basic_auth: true
+#             validate_certs: false
+#             body_format: json
+#             body: {}
+
+#         - name: Wait for packages to install
+#           ansible.builtin.uri:
+#             url: "https://{{ opnsense_host }}/api/core/firmware/running"
+#             method: GET
+#             user: "{{ opnsense_api_key }}"
+#             password: "{{ opnsense_api_secret }}"
+#             force_basic_auth: true
+#             validate_certs: false
+#           register: _running
+#           until: _running.json.status != 'running'
+#           retries: 30
+#           delay: 15
+
+#         - name: Reboot node
+#           ansible.builtin.uri:
+#             url: "https://{{ opnsense_host }}/api/core/firmware/reboot"
+#             method: POST
+#             user: "{{ opnsense_api_key }}"
+#             password: "{{ opnsense_api_secret }}"
+#             force_basic_auth: true
+#             validate_certs: false
+#             body_format: json
+#             body: {}
+#           ignore_errors: true
+
+#         - name: Wait for node to go offline
+#           ansible.builtin.wait_for:
+#             host: "{{ ansible_host }}"
+#             port: 443
+#             state: stopped
+#             timeout: 60
+#           delegate_to: localhost
+
+#         - name: Wait for node to come back online
+#           ansible.builtin.wait_for:
+#             host: "{{ ansible_host }}"
+#             port: 443
+#             state: started
+#             timeout: 300
+#             delay: 30
+#           delegate_to: localhost
+
+#         - name: Wait for API to be fully responsive
+#           ansible.builtin.uri:
+#             url: "https://{{ opnsense_host }}/api/core/firmware/status"
+#             method: GET
+#             user: "{{ opnsense_api_key }}"
+#             password: "{{ opnsense_api_secret }}"
+#             force_basic_auth: true
+#             validate_certs: false
+#           register: _post_status
+#           until: _post_status.status == 200
+#           retries: 20
+#           delay: 10
+
+#         - name: Health check — no remaining updates
+#           ansible.builtin.assert:
+#             that:
+#               - _post_status.json.status == 'none'
+#             fail_msg: >-
+#               Post-update firmware status is '{{ _post_status.json.status }}'.
+#               Expected 'none'. Triggering rollback.
+
+#         # After maintenance mode failover, master should rejoin as BACKUP
+#         - name: Health check — CARP role is BACKUP
+#           ansible.builtin.uri:
+#             url: "https://{{ opnsense_host }}/api/carp/status/get"
+#             method: GET
+#             user: "{{ opnsense_api_key }}"
+#             password: "{{ opnsense_api_secret }}"
+#             force_basic_auth: true
+#             validate_certs: false
+#           register: _carp_post
+#           failed_when: _carp_post.json.status | upper not in ['BACKUP', 'MAINTENANCE']
+
+#         - name: Record master update result
+#           ansible.builtin.set_fact:
+#             update_result: "success"
+#             fw_post_version: "{{ _post_status.json.product_version | default('unknown') }}"
+
+#       rescue:
+#         # Snapshot was taken before maintenance mode, so the restored VM
+#         # will come back as a healthy MASTER on the previous version.
+#         # Backup (already updated) continues serving traffic during rollback.
+
+#         - name: "ROLLBACK | Stop VM"
+#           ansible.builtin.uri:
+#             url: "{{ _pve_base }}/status/stop"
+#             method: POST
+#             headers: "{{ _pve_auth }}"
+#             validate_certs: false
+#           delegate_to: localhost
+
+#         - name: "ROLLBACK | Wait for VM to stop"
+#           ansible.builtin.uri:
+#             url: "{{ _pve_base }}/status/current"
+#             method: GET
+#             headers: "{{ _pve_auth }}"
+#             validate_certs: false
+#           register: _vm_state
+#           until: _vm_state.json.data.status == 'stopped'
+#           retries: 12
+#           delay: 5
+#           delegate_to: localhost
+
+#         - name: "ROLLBACK | Restore snapshot {{ snap_name }}"
+#           ansible.builtin.uri:
+#             url: "{{ _pve_base }}/snapshot/{{ snap_name }}/rollback"
+#             method: POST
+#             headers: "{{ _pve_auth }}"
+#             validate_certs: false
+#           delegate_to: localhost
+
+#         # Proxmox rollback tasks run async; wait for completion
+#         - name: "ROLLBACK | Wait for snapshot restore to complete"
+#           ansible.builtin.pause:
+#             seconds: 30
+
+#         - name: "ROLLBACK | Start VM"
+#           ansible.builtin.uri:
+#             url: "{{ _pve_base }}/status/start"
+#             method: POST
+#             headers: "{{ _pve_auth }}"
+#             validate_certs: false
+#           delegate_to: localhost
+
+#         - name: "ROLLBACK | Wait for VM to come back online"
+#           ansible.builtin.wait_for:
+#             host: "{{ ansible_host }}"
+#             port: 443
+#             state: started
+#             timeout: 300
+#             delay: 30
+#           delegate_to: localhost
+
+#         - name: "ROLLBACK | Record result"
+#           ansible.builtin.set_fact:
+#             update_result: "rolled_back"
+#             fw_post_version: "{{ fw_current_version }}"
+
+#       always:
+#         # Best-effort: disable maintenance mode whether update succeeded or rolled back.
+#         # On rollback, the restored snapshot predates maintenance mode so this
+#         # is effectively a no-op, but run it anyway as a safety net.
+#         - name: Disable CARP maintenance mode
+#           ansible.builtin.uri:
+#             url: "https://{{ opnsense_host }}/api/carp/status/maintenancemode"
+#             method: POST
+#             user: "{{ opnsense_api_key }}"
+#             password: "{{ opnsense_api_secret }}"
+#             force_basic_auth: true
+#             validate_certs: false
+#             body_format: json
+#             body: {}
+#           ignore_errors: true
+
+
+# ###############################################################
+# # Phase 4 targets localhost — it runs even if Phase 3 failed,
+# # because localhost was never marked as a failed host.
+# - name: "Phase 4 | Send ntfy notification"
+#   hosts: localhost
+#   gather_facts: false
+
+#   tasks:
+#     - name: Build report
+#       ansible.builtin.set_fact:
+#         _report: |
+#           {% for host in groups['opnsense'] %}
+#           {% set h = hostvars[host] %}
+#           {{ host }}
+#             fw check : {{ h.firmware_status | default('unknown') }}
+#             result   : {{ h.update_result | default('unknown') }}
+#             version  : {{ h.fw_post_version | default(h.fw_current_version | default('?')) }}
+#           {% endfor %}
+#         _any_issue: >-
+#           {{ groups['opnsense']
+#              | map('extract', hostvars, 'update_result')
+#              | select('in', ['rolled_back', 'pending'])
+#              | list
+#              | length > 0 }}
+
+#     - name: Send ntfy notification
+#       ansible.builtin.uri:
+#         url: "{{ ntfy_url }}/{{ ntfy_topic }}"
+#         method: POST
+#         headers:
+#           Authorization: "Bearer {{ ntfy_token }}"
+#           Title: "OPNsense HA Update Report"
+#           Priority: "{{ 'high' if _any_issue else 'default' }}"
+#           Tags: "{{ 'warning' if _any_issue else 'white_check_mark' }},shield"
+#           Content-Type: "text/plain"
+#         body: "{{ _report }}"
+
+#     - name: Fail playbook if any node was rolled back or did not complete
+#       ansible.builtin.fail:
+#         msg: >-
+#           One or more nodes did not update successfully.
+#           Check ntfy report and Ansible output for details.
+#       when: _any_issue | bool

ansible/proxmox/update_proxmox.yml

@@ -94,10 +94,6 @@
             retries: 60
             delay: 15
 
-          - name: Pause 15s 
-            ansible.builtin.pause:
-              seconds: 15
-
           - name: Update packages
             ansible.builtin.apt:
               upgrade: full

opnsense/10-wan

@@ -84,7 +84,7 @@ if ($type === "MASTER") {
         write_config("disable interface '$ifkey' due CARP event '$type'", false);
         interface_configure(false, $ifkey, false, false);
         // Remove WAN default gateway
-        mwexecf("/sbin/route delete default");
+        mwexec("/sbin/route delete default");
         foreach ($config['OPNsense']['Gateways']['gateway_item'] as $gw) {
             if ($gw['name'] === $gw_name) {
                 $gw_ip = $gw['gateway'];
@@ -92,9 +92,9 @@ if ($type === "MASTER") {
             }
         }
         // Shutdown WAN interface
-        mwexecf("/sbin/ifconfig {$real_if} down")
+        mwexec("/sbin/ifconfig {$real_if} down")
         // Add fallback default gateway
-        mwexecf("/sbin/route add default {$gw_ip}");
+        mwexec("/sbin/route add default {$gw_ip}");
         // Create lock file
         file_put_contents($lock_file, time());
         sleep($lock_default_age);