Compare commits
27 Commits
2918ea4578
...
bbb2e0f024
| Author | SHA1 | Date | |
|---|---|---|---|
| bbb2e0f024 | |||
| fbf38991d7 | |||
| f672517cc8 | |||
| 03a1b243bc | |||
| de745d74dc | |||
| 9c49b131e2 | |||
| 0912c0a253 | |||
| e2a1e53082 | |||
| 80d9befa6a | |||
| fcf5753f57 | |||
| 413694f47d | |||
| 2e912c7ad5 | |||
| 755e736ba9 | |||
| 12f3c65c39 | |||
| 02b908c4cd | |||
| d059b86dfc | |||
| 6e3814ce3f | |||
| fe24fec3a3 | |||
| 7d51740a1a | |||
| ca01e8f17e | |||
| b7cf1499a7 | |||
| 63db96d1bd | |||
| e1b7a88d1f | |||
| aaf111919d | |||
| f35bd094c7 | |||
| 98758f4fa7 | |||
| e4771be50c |
@@ -16,4 +16,5 @@ all:
|
|||||||
cerbere-head1:
|
cerbere-head1:
|
||||||
ansible_host: 192.168.88.2
|
ansible_host: 192.168.88.2
|
||||||
main_role: MASTER
|
main_role: MASTER
|
||||||
hypervisor: Proxmox
|
hypervisor: Proxmox
|
||||||
|
proxmox_vmid: 122
|
||||||
4
ansible/opnsense/requirements.yml
Normal file
4
ansible/opnsense/requirements.yml
Normal file
@@ -0,0 +1,4 @@
|
|||||||
|
---
|
||||||
|
collections:
|
||||||
|
- name: community.proxmox
|
||||||
|
version: "2.0.0"
|
||||||
390
ansible/opnsense/update_opnsense_ha_cluster.yml
Normal file
390
ansible/opnsense/update_opnsense_ha_cluster.yml
Normal file
@@ -0,0 +1,390 @@
|
|||||||
|
---
|
||||||
|
# OPNsense HA Cluster Update
|
||||||
|
# Order: backup (TrueNAS) first, then master (Proxmox)
|
||||||
|
###############################################################
|
||||||
|
- name: "Phase 1 | Firmware check on all nodes"
|
||||||
|
hosts: opnsense
|
||||||
|
any_errors_fatal: true
|
||||||
|
gather_facts: false
|
||||||
|
tasks:
|
||||||
|
|
||||||
|
- block:
|
||||||
|
|
||||||
|
- name: Check node availability
|
||||||
|
ansible.builtin.uri:
|
||||||
|
url: "https://{{ opnsense_host }}/api/core/system/status"
|
||||||
|
method: GET
|
||||||
|
user: "{{ opnsense_api_key }}"
|
||||||
|
password: "{{ opnsense_api_secret }}"
|
||||||
|
force_basic_auth: true
|
||||||
|
validate_certs: false
|
||||||
|
|
||||||
|
- name: Trigger firmware check
|
||||||
|
ansible.builtin.uri:
|
||||||
|
url: "https://{{ opnsense_host }}/api/core/firmware/check"
|
||||||
|
method: POST
|
||||||
|
user: "{{ opnsense_api_key }}"
|
||||||
|
password: "{{ opnsense_api_secret }}"
|
||||||
|
force_basic_auth: true
|
||||||
|
validate_certs: false
|
||||||
|
|
||||||
|
- name: Check VIP status
|
||||||
|
ansible.builtin.uri:
|
||||||
|
url: "https://{{ opnsense_host }}/api/diagnostics/interface/get_vip_status"
|
||||||
|
method: GET
|
||||||
|
user: "{{ opnsense_api_key }}"
|
||||||
|
password: "{{ opnsense_api_secret }}"
|
||||||
|
force_basic_auth: true
|
||||||
|
validate_certs: false
|
||||||
|
register: _vip_status
|
||||||
|
|
||||||
|
- name: Wait for check to complete
|
||||||
|
ansible.builtin.uri:
|
||||||
|
url: "https://{{ opnsense_host }}/api/core/firmware/status"
|
||||||
|
method: GET
|
||||||
|
user: "{{ opnsense_api_key }}"
|
||||||
|
password: "{{ opnsense_api_secret }}"
|
||||||
|
force_basic_auth: true
|
||||||
|
validate_certs: false
|
||||||
|
register: _firmware_status
|
||||||
|
until: _firmware_status.json.status != 'none'
|
||||||
|
retries: 12
|
||||||
|
delay: 5
|
||||||
|
|
||||||
|
- name: Store firmware facts
|
||||||
|
ansible.builtin.set_fact:
|
||||||
|
firmware_status: "{{ _firmware_status.json.status }}"
|
||||||
|
firmware_current_version: "{{ _firmware_status.json.product_version | default('unknown') }}"
|
||||||
|
firmware_new_version: "{{ _firmware_status.json.upgrade_packages | selectattr('name', 'equalto', 'opnsense') | map(attribute='new_version') | first | default('') }}"
|
||||||
|
firmware_upgrade_version: "{{ _firmware_status.json.upgrade_major_version | default('unknown') }}"
|
||||||
|
needs_reboot: "{{ true if _firmware_status.json.needs_reboot == '1' else false }}"
|
||||||
|
total_vips: "{{ _vip_status.json.rowCount }}"
|
||||||
|
mismatched_vips: "{{ _vip_status.json.rows | rejectattr('status', 'equalto', main_role) | list | length }}"
|
||||||
|
in_maintenance: "{{ _vip_status.json.carp.maintenancemode }}"
|
||||||
|
# update_result: "{{ 'pending' if _firmware_status.json.status == 'update' else 'skipped' }}"
|
||||||
|
|
||||||
|
- name: Skip backup update
|
||||||
|
ansible.builtin.set_fact:
|
||||||
|
skip_update: true
|
||||||
|
delegate_to: "{{ groups['opnsense_backup'][0] }}"
|
||||||
|
delegate_facts: true
|
||||||
|
run_once: true
|
||||||
|
when: hostvars[groups['opnsense_backup'][0]].firmware_current_version == hostvars[groups['opnsense_master'][0]].firmware_new_version
|
||||||
|
|
||||||
|
- name: "Assert"
|
||||||
|
ansible.builtin.assert:
|
||||||
|
that:
|
||||||
|
- firmware_status == 'update' or skip_update
|
||||||
|
- not in_maintenance
|
||||||
|
- mismatched_vips == 0
|
||||||
|
- total_vips > 0
|
||||||
|
quiet: true
|
||||||
|
|
||||||
|
- name: Display firmware status
|
||||||
|
ansible.builtin.debug:
|
||||||
|
msg: >-
|
||||||
|
{{ inventory_hostname }} ({{ main_role }}): {{ firmware_status }}
|
||||||
|
({{ firmware_current_version }}
|
||||||
|
{% if firmware_new_version %} → {{ firmware_new_version }} (reboot: {{ needs_reboot }}){% endif %})
|
||||||
|
|
||||||
|
rescue:
|
||||||
|
|
||||||
|
- name: Send Ntfy notification
|
||||||
|
ansible.builtin.uri:
|
||||||
|
url: "{{ ntfy_url }}/{{ ntfy_topic }}"
|
||||||
|
method: POST
|
||||||
|
user: "{{ ntfy_user }}"
|
||||||
|
password: "{{ lookup('env', 'NTFY_PASSWORD') }}"
|
||||||
|
force_basic_auth: true
|
||||||
|
body: |
|
||||||
|
Current version: {{ firmware_current_version }}
|
||||||
|
{% if firmware_status != "update" %}
|
||||||
|
{% if firmware_status == "upgrade" %}
|
||||||
|
Upgrade to {{ firmware_upgrade_version }} available, relaunch the playbook manually to proceed.
|
||||||
|
{% else %}
|
||||||
|
No updates available on the cluster.
|
||||||
|
{% endif %}
|
||||||
|
{% elif in_maintenance %}
|
||||||
|
The node {{ inventory_hostname }} is in CARP maintenance mode.
|
||||||
|
{% elif mismatched_vips > 0 %}
|
||||||
|
Some VIP are not {{ main_role }} on {{ ansible_host }}.
|
||||||
|
{% else %}
|
||||||
|
No VIPs are managed.
|
||||||
|
{% endif %}
|
||||||
|
headers:
|
||||||
|
Title: "OPNsense update aborted"
|
||||||
|
Priority: "default"
|
||||||
|
Tags: "x"
|
||||||
|
run_once: true
|
||||||
|
when: ntfy_url is defined
|
||||||
|
|
||||||
|
- ansible.builtin.fail:
|
||||||
|
msg: "Update aborted"
|
||||||
|
|
||||||
|
|
||||||
|
- name: "Phase 2 | Update backup node (TrueNAS, no snapshot)"
|
||||||
|
hosts: opnsense_backup
|
||||||
|
gather_facts: false
|
||||||
|
tasks:
|
||||||
|
|
||||||
|
- when: not skip_update | default(false)
|
||||||
|
block:
|
||||||
|
|
||||||
|
- name: Enable CARP maintenance mode
|
||||||
|
ansible.builtin.uri:
|
||||||
|
url: "https://{{ opnsense_host }}/api/diagnostics/interface/carp_status/maintenance"
|
||||||
|
method: POST
|
||||||
|
user: "{{ opnsense_api_key }}"
|
||||||
|
password: "{{ opnsense_api_secret }}"
|
||||||
|
force_basic_auth: true
|
||||||
|
validate_certs: false
|
||||||
|
register: _enable_maintenance
|
||||||
|
changed_when: _enable_maintenance.status == "ok"
|
||||||
|
|
||||||
|
- name: Trigger firmware update
|
||||||
|
ansible.builtin.uri:
|
||||||
|
url: "https://{{ opnsense_host }}/api/core/firmware/update"
|
||||||
|
method: POST
|
||||||
|
user: "{{ opnsense_api_key }}"
|
||||||
|
password: "{{ opnsense_api_secret }}"
|
||||||
|
force_basic_auth: true
|
||||||
|
validate_certs: false
|
||||||
|
|
||||||
|
- name: Wait for packages to update
|
||||||
|
ansible.builtin.uri:
|
||||||
|
url: "https://{{ opnsense_host }}/api/core/firmware/running"
|
||||||
|
method: GET
|
||||||
|
user: "{{ opnsense_api_key }}"
|
||||||
|
password: "{{ opnsense_api_secret }}"
|
||||||
|
force_basic_auth: true
|
||||||
|
validate_certs: false
|
||||||
|
register: _update_running
|
||||||
|
until: _update_running.json.status != 'busy'
|
||||||
|
retries: 300
|
||||||
|
delay: 15
|
||||||
|
|
||||||
|
- name: Wait for node to go offline
|
||||||
|
ansible.builtin.wait_for:
|
||||||
|
host: "{{ ansible_host }}"
|
||||||
|
port: "{{ opnsense_https_port }}"
|
||||||
|
state: stopped
|
||||||
|
timeout: 60
|
||||||
|
when: needs_reboot
|
||||||
|
|
||||||
|
- name: Wait for node to come back online
|
||||||
|
ansible.builtin.wait_for:
|
||||||
|
host: "{{ ansible_host }}"
|
||||||
|
port: "{{ opnsense_https_port }}"
|
||||||
|
state: started
|
||||||
|
timeout: 300
|
||||||
|
delay: 30
|
||||||
|
when: needs_reboot
|
||||||
|
|
||||||
|
- name: Wait for API to be fully responsive
|
||||||
|
ansible.builtin.uri:
|
||||||
|
url: "https://{{ opnsense_host }}/api/core/firmware/status"
|
||||||
|
method: GET
|
||||||
|
user: "{{ opnsense_api_key }}"
|
||||||
|
password: "{{ opnsense_api_secret }}"
|
||||||
|
force_basic_auth: true
|
||||||
|
validate_certs: false
|
||||||
|
register: _post_status
|
||||||
|
until: _post_status.status == 200
|
||||||
|
retries: 20
|
||||||
|
delay: 10
|
||||||
|
|
||||||
|
- name: Check VIP status
|
||||||
|
ansible.builtin.uri:
|
||||||
|
url: "https://{{ opnsense_host }}/api/diagnostics/interface/get_vip_status"
|
||||||
|
method: GET
|
||||||
|
user: "{{ opnsense_api_key }}"
|
||||||
|
password: "{{ opnsense_api_secret }}"
|
||||||
|
force_basic_auth: true
|
||||||
|
validate_certs: false
|
||||||
|
register: _carp_post_udapte
|
||||||
|
|
||||||
|
- name: Disable CARP maintenance mode
|
||||||
|
ansible.builtin.uri:
|
||||||
|
url: "https://{{ opnsense_host }}/api/carp/status/maintenancemode"
|
||||||
|
method: POST
|
||||||
|
user: "{{ opnsense_api_key }}"
|
||||||
|
password: "{{ opnsense_api_secret }}"
|
||||||
|
force_basic_auth: true
|
||||||
|
validate_certs: false
|
||||||
|
when: _carp_post_udapte.json.carp.maintenancemode
|
||||||
|
|
||||||
|
rescue:
|
||||||
|
|
||||||
|
- name: Send Ntfy notification
|
||||||
|
ansible.builtin.uri:
|
||||||
|
url: "{{ ntfy_url }}/{{ ntfy_topic }}"
|
||||||
|
method: POST
|
||||||
|
user: "{{ ntfy_user }}"
|
||||||
|
password: "{{ lookup('env', 'NTFY_PASSWORD') }}"
|
||||||
|
force_basic_auth: true
|
||||||
|
body: |
|
||||||
|
TODO
|
||||||
|
# Current version: {{ firmware_current_version }}
|
||||||
|
# {% if firmware_status != "update" %}
|
||||||
|
# {% if firmware_status == "upgrade" %}
|
||||||
|
# Upgrade to {{ firmware_upgrade_version }} available, relaunch the playbook manually to proceed.
|
||||||
|
# {% else %}
|
||||||
|
# No updates available on the cluster.
|
||||||
|
# {% endif %}
|
||||||
|
# {% elif in_maintenance %}
|
||||||
|
# The node {{ inventory_hostname }} is in CARP maintenance mode.
|
||||||
|
# {% elif mismatched_vips > 0 %}
|
||||||
|
# Some VIP are not {{ main_role }} on {{ ansible_host }}.
|
||||||
|
# {% else %}
|
||||||
|
# No VIPs are managed.
|
||||||
|
# {% endif %}
|
||||||
|
headers:
|
||||||
|
Title: "OPNsense update failed on {{ inventory_hostname }}"
|
||||||
|
Priority: "default"
|
||||||
|
Tags: "x"
|
||||||
|
run_once: true
|
||||||
|
when: ntfy_url is defined
|
||||||
|
|
||||||
|
- ansible.builtin.fail:
|
||||||
|
msg: "Update aborted"
|
||||||
|
|
||||||
|
- name: "Phase 3 | Update master node (Proxmox, with snapshot)"
|
||||||
|
hosts: opnsense_master
|
||||||
|
gather_facts: false
|
||||||
|
vars:
|
||||||
|
proxmox_snap_name: "preupdate-{{ now().strftime('%Y%m%d-%H%M') }}"
|
||||||
|
|
||||||
|
tasks:
|
||||||
|
|
||||||
|
- block:
|
||||||
|
|
||||||
|
- name: Take Proxmox VM snapshot
|
||||||
|
community.general.proxmox_snap:
|
||||||
|
vmid: "{{ proxmox_vmid }}"
|
||||||
|
state: present
|
||||||
|
snapname: "{{ proxmox_snap_name }}"
|
||||||
|
description: "Pre-firmware-update: {{ firmware_current_version }} → {{ firmware_new_version }}"
|
||||||
|
|
||||||
|
- name: Enable CARP maintenance mode
|
||||||
|
ansible.builtin.uri:
|
||||||
|
url: "https://{{ opnsense_host }}/api/diagnostics/interface/carp_status/maintenance"
|
||||||
|
method: POST
|
||||||
|
user: "{{ opnsense_api_key }}"
|
||||||
|
password: "{{ opnsense_api_secret }}"
|
||||||
|
force_basic_auth: true
|
||||||
|
validate_certs: false
|
||||||
|
register: _enable_maintenance
|
||||||
|
changed_when: _enable_maintenance.status == "ok"
|
||||||
|
|
||||||
|
- name: Trigger firmware update
|
||||||
|
ansible.builtin.uri:
|
||||||
|
url: "https://{{ opnsense_host }}/api/core/firmware/update"
|
||||||
|
method: POST
|
||||||
|
user: "{{ opnsense_api_key }}"
|
||||||
|
password: "{{ opnsense_api_secret }}"
|
||||||
|
force_basic_auth: true
|
||||||
|
validate_certs: false
|
||||||
|
|
||||||
|
- name: Wait for packages to update
|
||||||
|
ansible.builtin.uri:
|
||||||
|
url: "https://{{ opnsense_host }}/api/core/firmware/running"
|
||||||
|
method: GET
|
||||||
|
user: "{{ opnsense_api_key }}"
|
||||||
|
password: "{{ opnsense_api_secret }}"
|
||||||
|
force_basic_auth: true
|
||||||
|
validate_certs: false
|
||||||
|
register: _update_running
|
||||||
|
until: _update_running.json.status != 'busy'
|
||||||
|
retries: 300
|
||||||
|
delay: 15
|
||||||
|
|
||||||
|
- name: Wait for node to go offline
|
||||||
|
ansible.builtin.wait_for:
|
||||||
|
host: "{{ ansible_host }}"
|
||||||
|
port: "{{ opnsense_https_port }}"
|
||||||
|
state: stopped
|
||||||
|
timeout: 60
|
||||||
|
when: needs_reboot
|
||||||
|
|
||||||
|
- name: Wait for node to come back online
|
||||||
|
ansible.builtin.wait_for:
|
||||||
|
host: "{{ ansible_host }}"
|
||||||
|
port: "{{ opnsense_https_port }}"
|
||||||
|
state: started
|
||||||
|
timeout: 300
|
||||||
|
delay: 30
|
||||||
|
when: needs_reboot
|
||||||
|
|
||||||
|
- name: Wait for API to be fully responsive
|
||||||
|
ansible.builtin.uri:
|
||||||
|
url: "https://{{ opnsense_host }}/api/core/firmware/status"
|
||||||
|
method: GET
|
||||||
|
user: "{{ opnsense_api_key }}"
|
||||||
|
password: "{{ opnsense_api_secret }}"
|
||||||
|
force_basic_auth: true
|
||||||
|
validate_certs: false
|
||||||
|
register: _post_status
|
||||||
|
until: _post_status.status == 200
|
||||||
|
retries: 20
|
||||||
|
delay: 10
|
||||||
|
|
||||||
|
- name: Check VIP status
|
||||||
|
ansible.builtin.uri:
|
||||||
|
url: "https://{{ opnsense_host }}/api/diagnostics/interface/get_vip_status"
|
||||||
|
method: GET
|
||||||
|
user: "{{ opnsense_api_key }}"
|
||||||
|
password: "{{ opnsense_api_secret }}"
|
||||||
|
force_basic_auth: true
|
||||||
|
validate_certs: false
|
||||||
|
register: _carp_post_udapte
|
||||||
|
|
||||||
|
- name: Disable CARP maintenance mode
|
||||||
|
ansible.builtin.uri:
|
||||||
|
url: "https://{{ opnsense_host }}/api/carp/status/maintenancemode"
|
||||||
|
method: POST
|
||||||
|
user: "{{ opnsense_api_key }}"
|
||||||
|
password: "{{ opnsense_api_secret }}"
|
||||||
|
force_basic_auth: true
|
||||||
|
validate_certs: false
|
||||||
|
when: _carp_post_udapte.json.carp.maintenancemode
|
||||||
|
|
||||||
|
rescue:
|
||||||
|
|
||||||
|
- name: Rollback VM to the pre-update snapshot
|
||||||
|
community.proxmox.proxmox_snap:
|
||||||
|
vmid: "{{ proxmox_vmid }}"
|
||||||
|
state: rollback
|
||||||
|
snapname: "{{ proxmox_snap_name }}"
|
||||||
|
|
||||||
|
- name: Send Ntfy notification
|
||||||
|
ansible.builtin.uri:
|
||||||
|
url: "{{ ntfy_url }}/{{ ntfy_topic }}"
|
||||||
|
method: POST
|
||||||
|
user: "{{ ntfy_user }}"
|
||||||
|
password: "{{ lookup('env', 'NTFY_PASSWORD') }}"
|
||||||
|
force_basic_auth: true
|
||||||
|
body: |
|
||||||
|
TODO
|
||||||
|
# Current version: {{ firmware_current_version }}
|
||||||
|
# {% if firmware_status != "update" %}
|
||||||
|
# {% if firmware_status == "upgrade" %}
|
||||||
|
# Upgrade to {{ firmware_upgrade_version }} available, relaunch the playbook manually to proceed.
|
||||||
|
# {% else %}
|
||||||
|
# No updates available on the cluster.
|
||||||
|
# {% endif %}
|
||||||
|
# {% elif in_maintenance %}
|
||||||
|
# The node {{ inventory_hostname }} is in CARP maintenance mode.
|
||||||
|
# {% elif mismatched_vips > 0 %}
|
||||||
|
# Some VIP are not {{ main_role }} on {{ ansible_host }}.
|
||||||
|
# {% else %}
|
||||||
|
# No VIPs are managed.
|
||||||
|
# {% endif %}
|
||||||
|
headers:
|
||||||
|
Title: "OPNsense update failed on {{ inventory_hostname }}"
|
||||||
|
Priority: "default"
|
||||||
|
Tags: "x"
|
||||||
|
run_once: true
|
||||||
|
when: ntfy_url is defined
|
||||||
|
|
||||||
|
- ansible.builtin.fail:
|
||||||
|
msg: "Update aborted"
|
||||||
@@ -94,6 +94,10 @@
|
|||||||
retries: 60
|
retries: 60
|
||||||
delay: 15
|
delay: 15
|
||||||
|
|
||||||
|
- name: Pause 15s
|
||||||
|
ansible.builtin.pause:
|
||||||
|
seconds: 15
|
||||||
|
|
||||||
- name: Update packages
|
- name: Update packages
|
||||||
ansible.builtin.apt:
|
ansible.builtin.apt:
|
||||||
upgrade: full
|
upgrade: full
|
||||||
|
|||||||
Reference in New Issue
Block a user