From f35bd094c7f215661561d92fa3ed9522161d896d Mon Sep 17 00:00:00 2001 From: Vezpi Date: Wed, 17 Jun 2026 09:13:55 +0000 Subject: [PATCH] add: OPNsense HA cluster update playbook --- .../opnsense/update_opnsense_ha_cluster.yml | 495 ++++++++++++++++++ 1 file changed, 495 insertions(+) create mode 100644 ansible/opnsense/update_opnsense_ha_cluster.yml diff --git a/ansible/opnsense/update_opnsense_ha_cluster.yml b/ansible/opnsense/update_opnsense_ha_cluster.yml new file mode 100644 index 0000000..7482be4 --- /dev/null +++ b/ansible/opnsense/update_opnsense_ha_cluster.yml @@ -0,0 +1,495 @@ +--- +# OPNsense HA Cluster Update +# Order: backup (TrueNAS) first, then master (Proxmox) +############################################################### +- name: "Phase 1 | Firmware check on all nodes" + hosts: opnsense + gather_facts: false + vars: + opnsense_api_key: "{{ lookup('env', 'OPNSENSE_API_KEY') }}" + opnsense_api_secret: "{{ lookup('env', 'OPNSENSE_API_SECRET') }}" + + tasks: + - name: Check node availability + ansible.builtin.uri: + url: "https://{{ ansible_host }}/api/core/system/status" + method: GET + user: "{{ opnsense_api_key }}" + password: "{{ opnsense_api_secret }}" + force_basic_auth: true + # validate_certs: false + # body_format: json + # body: {} + + + # - name: Trigger firmware check + # ansible.builtin.uri: + # url: "https://{{ ansible_host }}/api/core/firmware/check" + # method: POST + # user: "{{ opnsense_api_key }}" + # password: "{{ opnsense_api_secret }}" + # force_basic_auth: true + # validate_certs: false + # body_format: json + # body: {} + + # - name: Wait for check to complete + # ansible.builtin.uri: + # url: "https://{{ ansible_host }}/api/core/firmware/status" + # method: GET + # user: "{{ opnsense_api_key }}" + # password: "{{ opnsense_api_secret }}" + # force_basic_auth: true + # validate_certs: false + # register: _fw_status + # until: _fw_status.json.status != 'running' + # retries: 12 + # delay: 10 + +# - name: Store firmware facts +# ansible.builtin.set_fact: +# fw_status: "{{ _fw_status.json.status }}" +# fw_current_version: "{{ _fw_status.json.product_version | default('unknown') }}" +# fw_new_version: "{{ _fw_status.json.new_version | default('') }}" +# # 'pending' means updates exist; will be overwritten to success/rolled_back +# # If it stays 'pending' after the run, a pre-update check failed +# update_result: "{{ 'pending' if _fw_status.json.status == 'update' else 'skipped' }}" + +# - name: Display firmware status +# ansible.builtin.debug: +# msg: >- +# {{ inventory_hostname }}: {{ fw_status }} +# ({{ fw_current_version }} +# {% if fw_new_version %} → {{ fw_new_version }}{% endif %}) + +# - name: Warn — major upgrade skipped, handle manually +# ansible.builtin.debug: +# msg: >- +# NOTICE: {{ inventory_hostname }} has a MAJOR upgrade available +# ({{ fw_new_version }}). This playbook handles minor updates only. +# when: fw_status == 'upgrade' + + +# ############################################################### +# - name: "Phase 2 | Update backup node (TrueNAS, no snapshot)" +# hosts: opnsense_backup +# gather_facts: false + +# tasks: +# - name: Skip — no minor updates available on backup +# ansible.builtin.meta: end_host +# when: fw_status != 'update' + +# # Safety: confirm backup is actually in BACKUP role before touching it +# - name: Verify backup node CARP role +# ansible.builtin.uri: +# url: "https://{{ ansible_host }}/api/carp/status/get" +# method: GET +# user: "{{ opnsense_api_key }}" +# password: "{{ opnsense_api_secret }}" +# force_basic_auth: true +# validate_certs: false +# register: _carp +# # Abort if node is unexpectedly MASTER — updating it would disrupt traffic +# failed_when: _carp.json.status | upper != 'BACKUP' + +# # Maintenance mode prevents the backup from attempting to take over +# # during its own reboot (avoids CARP flapping) +# - name: Enable CARP maintenance mode +# ansible.builtin.uri: +# url: "https://{{ ansible_host }}/api/carp/status/maintenancemode" +# method: POST +# user: "{{ opnsense_api_key }}" +# password: "{{ opnsense_api_secret }}" +# force_basic_auth: true +# validate_certs: false +# body_format: json +# body: {} + +# - name: Trigger firmware update +# ansible.builtin.uri: +# url: "https://{{ ansible_host }}/api/core/firmware/update" +# method: POST +# user: "{{ opnsense_api_key }}" +# password: "{{ opnsense_api_secret }}" +# force_basic_auth: true +# validate_certs: false +# body_format: json +# body: {} + +# - name: Wait for packages to install +# ansible.builtin.uri: +# url: "https://{{ ansible_host }}/api/core/firmware/running" +# method: GET +# user: "{{ opnsense_api_key }}" +# password: "{{ opnsense_api_secret }}" +# force_basic_auth: true +# validate_certs: false +# register: _running +# until: _running.json.status != 'running' +# retries: 30 +# delay: 15 + +# # Reboot is a separate explicit step — update does NOT auto-reboot +# - name: Reboot node +# ansible.builtin.uri: +# url: "https://{{ ansible_host }}/api/core/firmware/reboot" +# method: POST +# user: "{{ opnsense_api_key }}" +# password: "{{ opnsense_api_secret }}" +# force_basic_auth: true +# validate_certs: false +# body_format: json +# body: {} +# ignore_errors: true # Node drops the connection immediately + +# - name: Wait for node to go offline +# ansible.builtin.wait_for: +# host: "{{ ansible_host }}" +# port: 443 +# state: stopped +# timeout: 60 +# delegate_to: localhost + +# - name: Wait for node to come back online +# ansible.builtin.wait_for: +# host: "{{ ansible_host }}" +# port: 443 +# state: started +# timeout: 300 +# delay: 30 +# delegate_to: localhost + +# - name: Wait for API to be fully responsive +# ansible.builtin.uri: +# url: "https://{{ ansible_host }}/api/core/firmware/status" +# method: GET +# user: "{{ opnsense_api_key }}" +# password: "{{ opnsense_api_secret }}" +# force_basic_auth: true +# validate_certs: false +# register: _post_status +# until: _post_status.status == 200 +# retries: 20 +# delay: 10 + +# - name: Health check — no remaining updates +# ansible.builtin.assert: +# that: +# - _post_status.json.status == 'none' +# fail_msg: >- +# Post-update firmware status is '{{ _post_status.json.status }}' +# on {{ inventory_hostname }}. Expected 'none'. + +# - name: Health check — CARP role is BACKUP +# ansible.builtin.uri: +# url: "https://{{ ansible_host }}/api/carp/status/get" +# method: GET +# user: "{{ opnsense_api_key }}" +# password: "{{ opnsense_api_secret }}" +# force_basic_auth: true +# validate_certs: false +# register: _carp_post +# failed_when: _carp_post.json.status | upper not in ['BACKUP', 'MAINTENANCE'] + +# - name: Disable CARP maintenance mode +# ansible.builtin.uri: +# url: "https://{{ ansible_host }}/api/carp/status/maintenancemode" +# method: POST +# user: "{{ opnsense_api_key }}" +# password: "{{ opnsense_api_secret }}" +# force_basic_auth: true +# validate_certs: false +# body_format: json +# body: {} + +# - name: Record backup update result +# ansible.builtin.set_fact: +# update_result: "success" +# fw_post_version: "{{ _post_status.json.product_version | default('unknown') }}" + + +# ############################################################### +# - name: "Phase 3 | Update master node (Proxmox, with snapshot)" +# hosts: opnsense_master +# gather_facts: false + +# vars: +# snap_name: "preupdate-{{ now().strftime('%Y%m%d-%H%M') }}" +# _pve_base: "https://{{ proxmox_api_host }}/api2/json/nodes/{{ proxmox_node }}/qemu/{{ proxmox_vmid }}" +# _pve_auth: +# Authorization: "PVEAPIToken={{ proxmox_api_token_id }}={{ proxmox_api_token_secret }}" + +# tasks: +# - name: Skip — no minor updates available on master +# ansible.builtin.meta: end_host +# when: fw_status != 'update' + +# - name: Verify master node CARP role +# ansible.builtin.uri: +# url: "https://{{ ansible_host }}/api/carp/status/get" +# method: GET +# user: "{{ opnsense_api_key }}" +# password: "{{ opnsense_api_secret }}" +# force_basic_auth: true +# validate_certs: false +# register: _carp +# failed_when: _carp.json.status | upper != 'MASTER' + +# # Snapshot BEFORE maintenance mode — the restore point should be +# # a fully functioning MASTER node, not one mid-failover +# - name: Take Proxmox VM snapshot +# community.general.proxmox_snap: +# api_host: "{{ proxmox_api_host }}" +# api_token_id: "{{ proxmox_api_token_id }}" +# api_token_secret: "{{ proxmox_api_token_secret }}" +# vmid: "{{ proxmox_vmid }}" +# state: present +# snapname: "{{ snap_name }}" +# description: "Pre-firmware-update: {{ fw_current_version }} → {{ fw_new_version }}" +# timeout: 60 +# delegate_to: localhost + +# - name: Enable CARP maintenance mode (triggers controlled failover to backup) +# ansible.builtin.uri: +# url: "https://{{ ansible_host }}/api/carp/status/maintenancemode" +# method: POST +# user: "{{ opnsense_api_key }}" +# password: "{{ opnsense_api_secret }}" +# force_basic_auth: true +# validate_certs: false +# body_format: json +# body: {} + +# - name: Wait for CARP failover to settle +# ansible.builtin.pause: +# seconds: 10 + +# # Do not proceed if the backup has not actually taken over. +# # A failed failover means no traffic coverage during master update. +# - name: Confirm backup node is now MASTER +# ansible.builtin.uri: +# url: "https://{{ hostvars[groups['opnsense_backup'][0]].ansible_host }}/api/carp/status/get" +# method: GET +# user: "{{ opnsense_api_key }}" +# password: "{{ opnsense_api_secret }}" +# force_basic_auth: true +# validate_certs: false +# register: _backup_carp +# delegate_to: localhost +# failed_when: _backup_carp.json.status | upper != 'MASTER' + +# - name: Update and verify +# block: + +# - name: Trigger firmware update +# ansible.builtin.uri: +# url: "https://{{ ansible_host }}/api/core/firmware/update" +# method: POST +# user: "{{ opnsense_api_key }}" +# password: "{{ opnsense_api_secret }}" +# force_basic_auth: true +# validate_certs: false +# body_format: json +# body: {} + +# - name: Wait for packages to install +# ansible.builtin.uri: +# url: "https://{{ ansible_host }}/api/core/firmware/running" +# method: GET +# user: "{{ opnsense_api_key }}" +# password: "{{ opnsense_api_secret }}" +# force_basic_auth: true +# validate_certs: false +# register: _running +# until: _running.json.status != 'running' +# retries: 30 +# delay: 15 + +# - name: Reboot node +# ansible.builtin.uri: +# url: "https://{{ ansible_host }}/api/core/firmware/reboot" +# method: POST +# user: "{{ opnsense_api_key }}" +# password: "{{ opnsense_api_secret }}" +# force_basic_auth: true +# validate_certs: false +# body_format: json +# body: {} +# ignore_errors: true + +# - name: Wait for node to go offline +# ansible.builtin.wait_for: +# host: "{{ ansible_host }}" +# port: 443 +# state: stopped +# timeout: 60 +# delegate_to: localhost + +# - name: Wait for node to come back online +# ansible.builtin.wait_for: +# host: "{{ ansible_host }}" +# port: 443 +# state: started +# timeout: 300 +# delay: 30 +# delegate_to: localhost + +# - name: Wait for API to be fully responsive +# ansible.builtin.uri: +# url: "https://{{ ansible_host }}/api/core/firmware/status" +# method: GET +# user: "{{ opnsense_api_key }}" +# password: "{{ opnsense_api_secret }}" +# force_basic_auth: true +# validate_certs: false +# register: _post_status +# until: _post_status.status == 200 +# retries: 20 +# delay: 10 + +# - name: Health check — no remaining updates +# ansible.builtin.assert: +# that: +# - _post_status.json.status == 'none' +# fail_msg: >- +# Post-update firmware status is '{{ _post_status.json.status }}'. +# Expected 'none'. Triggering rollback. + +# # After maintenance mode failover, master should rejoin as BACKUP +# - name: Health check — CARP role is BACKUP +# ansible.builtin.uri: +# url: "https://{{ ansible_host }}/api/carp/status/get" +# method: GET +# user: "{{ opnsense_api_key }}" +# password: "{{ opnsense_api_secret }}" +# force_basic_auth: true +# validate_certs: false +# register: _carp_post +# failed_when: _carp_post.json.status | upper not in ['BACKUP', 'MAINTENANCE'] + +# - name: Record master update result +# ansible.builtin.set_fact: +# update_result: "success" +# fw_post_version: "{{ _post_status.json.product_version | default('unknown') }}" + +# rescue: +# # Snapshot was taken before maintenance mode, so the restored VM +# # will come back as a healthy MASTER on the previous version. +# # Backup (already updated) continues serving traffic during rollback. + +# - name: "ROLLBACK | Stop VM" +# ansible.builtin.uri: +# url: "{{ _pve_base }}/status/stop" +# method: POST +# headers: "{{ _pve_auth }}" +# validate_certs: false +# delegate_to: localhost + +# - name: "ROLLBACK | Wait for VM to stop" +# ansible.builtin.uri: +# url: "{{ _pve_base }}/status/current" +# method: GET +# headers: "{{ _pve_auth }}" +# validate_certs: false +# register: _vm_state +# until: _vm_state.json.data.status == 'stopped' +# retries: 12 +# delay: 5 +# delegate_to: localhost + +# - name: "ROLLBACK | Restore snapshot {{ snap_name }}" +# ansible.builtin.uri: +# url: "{{ _pve_base }}/snapshot/{{ snap_name }}/rollback" +# method: POST +# headers: "{{ _pve_auth }}" +# validate_certs: false +# delegate_to: localhost + +# # Proxmox rollback tasks run async; wait for completion +# - name: "ROLLBACK | Wait for snapshot restore to complete" +# ansible.builtin.pause: +# seconds: 30 + +# - name: "ROLLBACK | Start VM" +# ansible.builtin.uri: +# url: "{{ _pve_base }}/status/start" +# method: POST +# headers: "{{ _pve_auth }}" +# validate_certs: false +# delegate_to: localhost + +# - name: "ROLLBACK | Wait for VM to come back online" +# ansible.builtin.wait_for: +# host: "{{ ansible_host }}" +# port: 443 +# state: started +# timeout: 300 +# delay: 30 +# delegate_to: localhost + +# - name: "ROLLBACK | Record result" +# ansible.builtin.set_fact: +# update_result: "rolled_back" +# fw_post_version: "{{ fw_current_version }}" + +# always: +# # Best-effort: disable maintenance mode whether update succeeded or rolled back. +# # On rollback, the restored snapshot predates maintenance mode so this +# # is effectively a no-op, but run it anyway as a safety net. +# - name: Disable CARP maintenance mode +# ansible.builtin.uri: +# url: "https://{{ ansible_host }}/api/carp/status/maintenancemode" +# method: POST +# user: "{{ opnsense_api_key }}" +# password: "{{ opnsense_api_secret }}" +# force_basic_auth: true +# validate_certs: false +# body_format: json +# body: {} +# ignore_errors: true + + +# ############################################################### +# # Phase 4 targets localhost — it runs even if Phase 3 failed, +# # because localhost was never marked as a failed host. +# - name: "Phase 4 | Send ntfy notification" +# hosts: localhost +# gather_facts: false + +# tasks: +# - name: Build report +# ansible.builtin.set_fact: +# _report: | +# {% for host in groups['opnsense'] %} +# {% set h = hostvars[host] %} +# {{ host }} +# fw check : {{ h.fw_status | default('unknown') }} +# result : {{ h.update_result | default('unknown') }} +# version : {{ h.fw_post_version | default(h.fw_current_version | default('?')) }} +# {% endfor %} +# _any_issue: >- +# {{ groups['opnsense'] +# | map('extract', hostvars, 'update_result') +# | select('in', ['rolled_back', 'pending']) +# | list +# | length > 0 }} + +# - name: Send ntfy notification +# ansible.builtin.uri: +# url: "{{ ntfy_url }}/{{ ntfy_topic }}" +# method: POST +# headers: +# Authorization: "Bearer {{ ntfy_token }}" +# Title: "OPNsense HA Update Report" +# Priority: "{{ 'high' if _any_issue else 'default' }}" +# Tags: "{{ 'warning' if _any_issue else 'white_check_mark' }},shield" +# Content-Type: "text/plain" +# body: "{{ _report }}" + +# - name: Fail playbook if any node was rolled back or did not complete +# ansible.builtin.fail: +# msg: >- +# One or more nodes did not update successfully. +# Check ntfy report and Ansible output for details. +# when: _any_issue | bool \ No newline at end of file