492 lines
18 KiB
YAML
492 lines
18 KiB
YAML
---
|
|
# OPNsense HA Cluster Update
|
|
# Order: backup (TrueNAS) first, then master (Proxmox)
|
|
###############################################################
|
|
- name: "Phase 1 | Firmware check on all nodes"
|
|
hosts: opnsense
|
|
any_errors_fatal: true
|
|
gather_facts: false
|
|
|
|
tasks:
|
|
|
|
- block:
|
|
|
|
- name: Check node availability
|
|
ansible.builtin.uri:
|
|
url: "https://{{ opnsense_host }}/api/core/system/status"
|
|
method: GET
|
|
user: "{{ opnsense_api_key }}"
|
|
password: "{{ opnsense_api_secret }}"
|
|
force_basic_auth: true
|
|
validate_certs: false
|
|
|
|
- name: Trigger firmware check
|
|
ansible.builtin.uri:
|
|
url: "https://{{ opnsense_host }}/api/core/firmware/check"
|
|
method: POST
|
|
user: "{{ opnsense_api_key }}"
|
|
password: "{{ opnsense_api_secret }}"
|
|
force_basic_auth: true
|
|
validate_certs: false
|
|
|
|
- name: Wait for check to complete
|
|
ansible.builtin.uri:
|
|
url: "https://{{ opnsense_host }}/api/core/firmware/status"
|
|
method: GET
|
|
user: "{{ opnsense_api_key }}"
|
|
password: "{{ opnsense_api_secret }}"
|
|
force_basic_auth: true
|
|
validate_certs: false
|
|
register: _firmware_status
|
|
until: _firmware_status.json.status != 'none'
|
|
retries: 12
|
|
delay: 5
|
|
|
|
- name: Store firmware facts
|
|
ansible.builtin.set_fact:
|
|
firmware_status: "{{ _firmware_status.json.status }}"
|
|
fw_current_version: "{{ _firmware_status.json.product_version | default('unknown') }}"
|
|
fw_new_version: "{{ _firmware_status.json.upgrade_packages | selectattr('name', 'equalto', 'opnsense') | map(attribute='new_version') | first | default('') }}"
|
|
needs_reboot: "{{ true if _firmware_status.json.needs_reboot == '1' else false }}"
|
|
# 'pending' means updates exist; will be overwritten to success/rolled_back
|
|
# If it stays 'pending' after the run, a pre-update check failed
|
|
update_result: "{{ 'pending' if _firmware_status.json.status == 'update' else 'skipped' }}"
|
|
|
|
- name: Display firmware status
|
|
ansible.builtin.debug:
|
|
msg: >-
|
|
{{ inventory_hostname }}: {{ firmware_status }}
|
|
({{ fw_current_version }}
|
|
{% if fw_new_version %} → {{ fw_new_version }} (reboot: {{ needs_reboot }}){% endif %})
|
|
|
|
- name: Warn — major upgrade skipped, handle manually
|
|
ansible.builtin.debug:
|
|
msg: >-
|
|
NOTICE: {{ inventory_hostname }} has a MAJOR upgrade available
|
|
({{ fw_new_version }}). This playbook handles minor updates only.
|
|
when: firmware_status == 'upgrade'
|
|
|
|
|
|
# ###############################################################
|
|
# - name: "Phase 2 | Update backup node (TrueNAS, no snapshot)"
|
|
# hosts: opnsense_backup
|
|
# gather_facts: false
|
|
|
|
# tasks:
|
|
# - name: Skip — no minor updates available on backup
|
|
# ansible.builtin.meta: end_host
|
|
# when: firmware_status != 'update'
|
|
|
|
# # Safety: confirm backup is actually in BACKUP role before touching it
|
|
# - name: Verify backup node CARP role
|
|
# ansible.builtin.uri:
|
|
# url: "https://{{ opnsense_host }}/api/carp/status/get"
|
|
# method: GET
|
|
# user: "{{ opnsense_api_key }}"
|
|
# password: "{{ opnsense_api_secret }}"
|
|
# force_basic_auth: true
|
|
# validate_certs: false
|
|
# register: _carp
|
|
# # Abort if node is unexpectedly MASTER — updating it would disrupt traffic
|
|
# failed_when: _carp.json.status | upper != 'BACKUP'
|
|
|
|
# # Maintenance mode prevents the backup from attempting to take over
|
|
# # during its own reboot (avoids CARP flapping)
|
|
# - name: Enable CARP maintenance mode
|
|
# ansible.builtin.uri:
|
|
# url: "https://{{ opnsense_host }}/api/carp/status/maintenancemode"
|
|
# method: POST
|
|
# user: "{{ opnsense_api_key }}"
|
|
# password: "{{ opnsense_api_secret }}"
|
|
# force_basic_auth: true
|
|
# validate_certs: false
|
|
# body_format: json
|
|
# body: {}
|
|
|
|
# - name: Trigger firmware update
|
|
# ansible.builtin.uri:
|
|
# url: "https://{{ opnsense_host }}/api/core/firmware/update"
|
|
# method: POST
|
|
# user: "{{ opnsense_api_key }}"
|
|
# password: "{{ opnsense_api_secret }}"
|
|
# force_basic_auth: true
|
|
# validate_certs: false
|
|
# body_format: json
|
|
# body: {}
|
|
|
|
# - name: Wait for packages to install
|
|
# ansible.builtin.uri:
|
|
# url: "https://{{ opnsense_host }}/api/core/firmware/running"
|
|
# method: GET
|
|
# user: "{{ opnsense_api_key }}"
|
|
# password: "{{ opnsense_api_secret }}"
|
|
# force_basic_auth: true
|
|
# validate_certs: false
|
|
# register: _running
|
|
# until: _running.json.status != 'running'
|
|
# retries: 30
|
|
# delay: 15
|
|
|
|
# # Reboot is a separate explicit step — update does NOT auto-reboot
|
|
# - name: Reboot node
|
|
# ansible.builtin.uri:
|
|
# url: "https://{{ opnsense_host }}/api/core/firmware/reboot"
|
|
# method: POST
|
|
# user: "{{ opnsense_api_key }}"
|
|
# password: "{{ opnsense_api_secret }}"
|
|
# force_basic_auth: true
|
|
# validate_certs: false
|
|
# body_format: json
|
|
# body: {}
|
|
# ignore_errors: true # Node drops the connection immediately
|
|
|
|
# - name: Wait for node to go offline
|
|
# ansible.builtin.wait_for:
|
|
# host: "{{ ansible_host }}"
|
|
# port: 443
|
|
# state: stopped
|
|
# timeout: 60
|
|
# delegate_to: localhost
|
|
|
|
# - name: Wait for node to come back online
|
|
# ansible.builtin.wait_for:
|
|
# host: "{{ ansible_host }}"
|
|
# port: 443
|
|
# state: started
|
|
# timeout: 300
|
|
# delay: 30
|
|
# delegate_to: localhost
|
|
|
|
# - name: Wait for API to be fully responsive
|
|
# ansible.builtin.uri:
|
|
# url: "https://{{ opnsense_host }}/api/core/firmware/status"
|
|
# method: GET
|
|
# user: "{{ opnsense_api_key }}"
|
|
# password: "{{ opnsense_api_secret }}"
|
|
# force_basic_auth: true
|
|
# validate_certs: false
|
|
# register: _post_status
|
|
# until: _post_status.status == 200
|
|
# retries: 20
|
|
# delay: 10
|
|
|
|
# - name: Health check — no remaining updates
|
|
# ansible.builtin.assert:
|
|
# that:
|
|
# - _post_status.json.status == 'none'
|
|
# fail_msg: >-
|
|
# Post-update firmware status is '{{ _post_status.json.status }}'
|
|
# on {{ inventory_hostname }}. Expected 'none'.
|
|
|
|
# - name: Health check — CARP role is BACKUP
|
|
# ansible.builtin.uri:
|
|
# url: "https://{{ opnsense_host }}/api/carp/status/get"
|
|
# method: GET
|
|
# user: "{{ opnsense_api_key }}"
|
|
# password: "{{ opnsense_api_secret }}"
|
|
# force_basic_auth: true
|
|
# validate_certs: false
|
|
# register: _carp_post
|
|
# failed_when: _carp_post.json.status | upper not in ['BACKUP', 'MAINTENANCE']
|
|
|
|
# - name: Disable CARP maintenance mode
|
|
# ansible.builtin.uri:
|
|
# url: "https://{{ opnsense_host }}/api/carp/status/maintenancemode"
|
|
# method: POST
|
|
# user: "{{ opnsense_api_key }}"
|
|
# password: "{{ opnsense_api_secret }}"
|
|
# force_basic_auth: true
|
|
# validate_certs: false
|
|
# body_format: json
|
|
# body: {}
|
|
|
|
# - name: Record backup update result
|
|
# ansible.builtin.set_fact:
|
|
# update_result: "success"
|
|
# fw_post_version: "{{ _post_status.json.product_version | default('unknown') }}"
|
|
|
|
|
|
# ###############################################################
|
|
# - name: "Phase 3 | Update master node (Proxmox, with snapshot)"
|
|
# hosts: opnsense_master
|
|
# gather_facts: false
|
|
|
|
# vars:
|
|
# snap_name: "preupdate-{{ now().strftime('%Y%m%d-%H%M') }}"
|
|
# _pve_base: "https://{{ proxmox_api_host }}/api2/json/nodes/{{ proxmox_node }}/qemu/{{ proxmox_vmid }}"
|
|
# _pve_auth:
|
|
# Authorization: "PVEAPIToken={{ proxmox_api_token_id }}={{ proxmox_api_token_secret }}"
|
|
|
|
# tasks:
|
|
# - name: Skip — no minor updates available on master
|
|
# ansible.builtin.meta: end_host
|
|
# when: firmware_status != 'update'
|
|
|
|
# - name: Verify master node CARP role
|
|
# ansible.builtin.uri:
|
|
# url: "https://{{ opnsense_host }}/api/carp/status/get"
|
|
# method: GET
|
|
# user: "{{ opnsense_api_key }}"
|
|
# password: "{{ opnsense_api_secret }}"
|
|
# force_basic_auth: true
|
|
# validate_certs: false
|
|
# register: _carp
|
|
# failed_when: _carp.json.status | upper != 'MASTER'
|
|
|
|
# # Snapshot BEFORE maintenance mode — the restore point should be
|
|
# # a fully functioning MASTER node, not one mid-failover
|
|
# - name: Take Proxmox VM snapshot
|
|
# community.general.proxmox_snap:
|
|
# api_host: "{{ proxmox_api_host }}"
|
|
# api_token_id: "{{ proxmox_api_token_id }}"
|
|
# api_token_secret: "{{ proxmox_api_token_secret }}"
|
|
# vmid: "{{ proxmox_vmid }}"
|
|
# state: present
|
|
# snapname: "{{ snap_name }}"
|
|
# description: "Pre-firmware-update: {{ fw_current_version }} → {{ fw_new_version }}"
|
|
# timeout: 60
|
|
# delegate_to: localhost
|
|
|
|
# - name: Enable CARP maintenance mode (triggers controlled failover to backup)
|
|
# ansible.builtin.uri:
|
|
# url: "https://{{ opnsense_host }}/api/carp/status/maintenancemode"
|
|
# method: POST
|
|
# user: "{{ opnsense_api_key }}"
|
|
# password: "{{ opnsense_api_secret }}"
|
|
# force_basic_auth: true
|
|
# validate_certs: false
|
|
# body_format: json
|
|
# body: {}
|
|
|
|
# - name: Wait for CARP failover to settle
|
|
# ansible.builtin.pause:
|
|
# seconds: 10
|
|
|
|
# # Do not proceed if the backup has not actually taken over.
|
|
# # A failed failover means no traffic coverage during master update.
|
|
# - name: Confirm backup node is now MASTER
|
|
# ansible.builtin.uri:
|
|
# url: "https://{{ hostvars[groups['opnsense_backup'][0]].ansible_host }}/api/carp/status/get"
|
|
# method: GET
|
|
# user: "{{ opnsense_api_key }}"
|
|
# password: "{{ opnsense_api_secret }}"
|
|
# force_basic_auth: true
|
|
# validate_certs: false
|
|
# register: _backup_carp
|
|
# delegate_to: localhost
|
|
# failed_when: _backup_carp.json.status | upper != 'MASTER'
|
|
|
|
# - name: Update and verify
|
|
# block:
|
|
|
|
# - name: Trigger firmware update
|
|
# ansible.builtin.uri:
|
|
# url: "https://{{ opnsense_host }}/api/core/firmware/update"
|
|
# method: POST
|
|
# user: "{{ opnsense_api_key }}"
|
|
# password: "{{ opnsense_api_secret }}"
|
|
# force_basic_auth: true
|
|
# validate_certs: false
|
|
# body_format: json
|
|
# body: {}
|
|
|
|
# - name: Wait for packages to install
|
|
# ansible.builtin.uri:
|
|
# url: "https://{{ opnsense_host }}/api/core/firmware/running"
|
|
# method: GET
|
|
# user: "{{ opnsense_api_key }}"
|
|
# password: "{{ opnsense_api_secret }}"
|
|
# force_basic_auth: true
|
|
# validate_certs: false
|
|
# register: _running
|
|
# until: _running.json.status != 'running'
|
|
# retries: 30
|
|
# delay: 15
|
|
|
|
# - name: Reboot node
|
|
# ansible.builtin.uri:
|
|
# url: "https://{{ opnsense_host }}/api/core/firmware/reboot"
|
|
# method: POST
|
|
# user: "{{ opnsense_api_key }}"
|
|
# password: "{{ opnsense_api_secret }}"
|
|
# force_basic_auth: true
|
|
# validate_certs: false
|
|
# body_format: json
|
|
# body: {}
|
|
# ignore_errors: true
|
|
|
|
# - name: Wait for node to go offline
|
|
# ansible.builtin.wait_for:
|
|
# host: "{{ ansible_host }}"
|
|
# port: 443
|
|
# state: stopped
|
|
# timeout: 60
|
|
# delegate_to: localhost
|
|
|
|
# - name: Wait for node to come back online
|
|
# ansible.builtin.wait_for:
|
|
# host: "{{ ansible_host }}"
|
|
# port: 443
|
|
# state: started
|
|
# timeout: 300
|
|
# delay: 30
|
|
# delegate_to: localhost
|
|
|
|
# - name: Wait for API to be fully responsive
|
|
# ansible.builtin.uri:
|
|
# url: "https://{{ opnsense_host }}/api/core/firmware/status"
|
|
# method: GET
|
|
# user: "{{ opnsense_api_key }}"
|
|
# password: "{{ opnsense_api_secret }}"
|
|
# force_basic_auth: true
|
|
# validate_certs: false
|
|
# register: _post_status
|
|
# until: _post_status.status == 200
|
|
# retries: 20
|
|
# delay: 10
|
|
|
|
# - name: Health check — no remaining updates
|
|
# ansible.builtin.assert:
|
|
# that:
|
|
# - _post_status.json.status == 'none'
|
|
# fail_msg: >-
|
|
# Post-update firmware status is '{{ _post_status.json.status }}'.
|
|
# Expected 'none'. Triggering rollback.
|
|
|
|
# # After maintenance mode failover, master should rejoin as BACKUP
|
|
# - name: Health check — CARP role is BACKUP
|
|
# ansible.builtin.uri:
|
|
# url: "https://{{ opnsense_host }}/api/carp/status/get"
|
|
# method: GET
|
|
# user: "{{ opnsense_api_key }}"
|
|
# password: "{{ opnsense_api_secret }}"
|
|
# force_basic_auth: true
|
|
# validate_certs: false
|
|
# register: _carp_post
|
|
# failed_when: _carp_post.json.status | upper not in ['BACKUP', 'MAINTENANCE']
|
|
|
|
# - name: Record master update result
|
|
# ansible.builtin.set_fact:
|
|
# update_result: "success"
|
|
# fw_post_version: "{{ _post_status.json.product_version | default('unknown') }}"
|
|
|
|
# rescue:
|
|
# # Snapshot was taken before maintenance mode, so the restored VM
|
|
# # will come back as a healthy MASTER on the previous version.
|
|
# # Backup (already updated) continues serving traffic during rollback.
|
|
|
|
# - name: "ROLLBACK | Stop VM"
|
|
# ansible.builtin.uri:
|
|
# url: "{{ _pve_base }}/status/stop"
|
|
# method: POST
|
|
# headers: "{{ _pve_auth }}"
|
|
# validate_certs: false
|
|
# delegate_to: localhost
|
|
|
|
# - name: "ROLLBACK | Wait for VM to stop"
|
|
# ansible.builtin.uri:
|
|
# url: "{{ _pve_base }}/status/current"
|
|
# method: GET
|
|
# headers: "{{ _pve_auth }}"
|
|
# validate_certs: false
|
|
# register: _vm_state
|
|
# until: _vm_state.json.data.status == 'stopped'
|
|
# retries: 12
|
|
# delay: 5
|
|
# delegate_to: localhost
|
|
|
|
# - name: "ROLLBACK | Restore snapshot {{ snap_name }}"
|
|
# ansible.builtin.uri:
|
|
# url: "{{ _pve_base }}/snapshot/{{ snap_name }}/rollback"
|
|
# method: POST
|
|
# headers: "{{ _pve_auth }}"
|
|
# validate_certs: false
|
|
# delegate_to: localhost
|
|
|
|
# # Proxmox rollback tasks run async; wait for completion
|
|
# - name: "ROLLBACK | Wait for snapshot restore to complete"
|
|
# ansible.builtin.pause:
|
|
# seconds: 30
|
|
|
|
# - name: "ROLLBACK | Start VM"
|
|
# ansible.builtin.uri:
|
|
# url: "{{ _pve_base }}/status/start"
|
|
# method: POST
|
|
# headers: "{{ _pve_auth }}"
|
|
# validate_certs: false
|
|
# delegate_to: localhost
|
|
|
|
# - name: "ROLLBACK | Wait for VM to come back online"
|
|
# ansible.builtin.wait_for:
|
|
# host: "{{ ansible_host }}"
|
|
# port: 443
|
|
# state: started
|
|
# timeout: 300
|
|
# delay: 30
|
|
# delegate_to: localhost
|
|
|
|
# - name: "ROLLBACK | Record result"
|
|
# ansible.builtin.set_fact:
|
|
# update_result: "rolled_back"
|
|
# fw_post_version: "{{ fw_current_version }}"
|
|
|
|
# always:
|
|
# # Best-effort: disable maintenance mode whether update succeeded or rolled back.
|
|
# # On rollback, the restored snapshot predates maintenance mode so this
|
|
# # is effectively a no-op, but run it anyway as a safety net.
|
|
# - name: Disable CARP maintenance mode
|
|
# ansible.builtin.uri:
|
|
# url: "https://{{ opnsense_host }}/api/carp/status/maintenancemode"
|
|
# method: POST
|
|
# user: "{{ opnsense_api_key }}"
|
|
# password: "{{ opnsense_api_secret }}"
|
|
# force_basic_auth: true
|
|
# validate_certs: false
|
|
# body_format: json
|
|
# body: {}
|
|
# ignore_errors: true
|
|
|
|
|
|
# ###############################################################
|
|
# # Phase 4 targets localhost — it runs even if Phase 3 failed,
|
|
# # because localhost was never marked as a failed host.
|
|
# - name: "Phase 4 | Send ntfy notification"
|
|
# hosts: localhost
|
|
# gather_facts: false
|
|
|
|
# tasks:
|
|
# - name: Build report
|
|
# ansible.builtin.set_fact:
|
|
# _report: |
|
|
# {% for host in groups['opnsense'] %}
|
|
# {% set h = hostvars[host] %}
|
|
# {{ host }}
|
|
# fw check : {{ h.firmware_status | default('unknown') }}
|
|
# result : {{ h.update_result | default('unknown') }}
|
|
# version : {{ h.fw_post_version | default(h.fw_current_version | default('?')) }}
|
|
# {% endfor %}
|
|
# _any_issue: >-
|
|
# {{ groups['opnsense']
|
|
# | map('extract', hostvars, 'update_result')
|
|
# | select('in', ['rolled_back', 'pending'])
|
|
# | list
|
|
# | length > 0 }}
|
|
|
|
# - name: Send ntfy notification
|
|
# ansible.builtin.uri:
|
|
# url: "{{ ntfy_url }}/{{ ntfy_topic }}"
|
|
# method: POST
|
|
# headers:
|
|
# Authorization: "Bearer {{ ntfy_token }}"
|
|
# Title: "OPNsense HA Update Report"
|
|
# Priority: "{{ 'high' if _any_issue else 'default' }}"
|
|
# Tags: "{{ 'warning' if _any_issue else 'white_check_mark' }},shield"
|
|
# Content-Type: "text/plain"
|
|
# body: "{{ _report }}"
|
|
|
|
# - name: Fail playbook if any node was rolled back or did not complete
|
|
# ansible.builtin.fail:
|
|
# msg: >-
|
|
# One or more nodes did not update successfully.
|
|
# Check ntfy report and Ansible output for details.
|
|
# when: _any_issue | bool |