Compare commits
81 Commits
d8b6e9a91d
...
lab
| Author | SHA1 | Date | |
|---|---|---|---|
| 2d1c4c9934 | |||
| 29ba93e096 | |||
| 09862a53c1 | |||
| 336f1fd466 | |||
| 2937ba4344 | |||
| 52eed8aade | |||
| 6acf547df1 | |||
| d7f3a7ef06 | |||
| eccd940727 | |||
| 52eaeaae93 | |||
| c99279cb49 | |||
| 2d9c6ffb1d | |||
| 240791d22e | |||
| 86e0b4a23f | |||
| 1f5d99b5ae | |||
| cd0bfd956c | |||
| 6cf28240df | |||
| 697ac311a7 | |||
| d08f207fc4 | |||
| 74fa6432db | |||
| 8ab282eb95 | |||
| 3d18c01d5a | |||
| 2404692a30 | |||
| 557f80979b | |||
| fac4a27b75 | |||
| 2d80a76353 | |||
| d5c36e3118 | |||
| ff19c2faae | |||
| 4a7e8ef7ca | |||
| 3f2b208651 | |||
| fa334f90d4 | |||
| 43e535143b | |||
| f0ded8ac17 | |||
| f78c8a6cd7 | |||
| cbcbc45d00 | |||
| 869faf720a | |||
| 5d39cf3172 | |||
| 137b13d94e | |||
| b1008ec425 | |||
| 22de950cf5 | |||
| fd0a9a83e8 | |||
| 79302e8139 | |||
| 1ca4807d4b | |||
| 4c086cfad5 | |||
| b91ea08874 | |||
| ac69a7a5c6 | |||
| 0a576b7ec0 | |||
| e46f78b971 | |||
| a8b0fca263 | |||
| bbb2e0f024 | |||
| fbf38991d7 | |||
| f672517cc8 | |||
| 03a1b243bc | |||
| de745d74dc | |||
| 9c49b131e2 | |||
| 0912c0a253 | |||
| e2a1e53082 | |||
| 80d9befa6a | |||
| fcf5753f57 | |||
| 413694f47d | |||
| 2e912c7ad5 | |||
| 755e736ba9 | |||
| 12f3c65c39 | |||
| 02b908c4cd | |||
| d059b86dfc | |||
| 6e3814ce3f | |||
| fe24fec3a3 | |||
| 7d51740a1a | |||
| ca01e8f17e | |||
| b7cf1499a7 | |||
| 63db96d1bd | |||
| e1b7a88d1f | |||
| aaf111919d | |||
| f35bd094c7 | |||
| 98758f4fa7 | |||
| e4771be50c | |||
| 130db47180 | |||
| 8284a5fb66 | |||
| 0b7af06fb3 | |||
| eaef6ad961 | |||
| 20ff3707e6 |
2
.gitignore
vendored
2
.gitignore
vendored
@@ -2,3 +2,5 @@
|
|||||||
**/.terraform*
|
**/.terraform*
|
||||||
*.tfstate*
|
*.tfstate*
|
||||||
*credentials.auto.tfvars
|
*credentials.auto.tfvars
|
||||||
|
# Bruno
|
||||||
|
.bruno
|
||||||
20
ansible/opnsense/inventory.yml
Normal file
20
ansible/opnsense/inventory.yml
Normal file
@@ -0,0 +1,20 @@
|
|||||||
|
---
|
||||||
|
all:
|
||||||
|
children:
|
||||||
|
opnsense:
|
||||||
|
vars:
|
||||||
|
ansible_connection: local
|
||||||
|
children:
|
||||||
|
opnsense_backup:
|
||||||
|
hosts:
|
||||||
|
cerbere-head2:
|
||||||
|
ansible_host: 192.168.88.3
|
||||||
|
main_role: BACKUP
|
||||||
|
hypervisor: TrueNAS
|
||||||
|
opnsense_master:
|
||||||
|
hosts:
|
||||||
|
cerbere-head1:
|
||||||
|
ansible_host: 192.168.88.2
|
||||||
|
main_role: MASTER
|
||||||
|
hypervisor: Proxmox
|
||||||
|
proxmox_vmid: 122
|
||||||
4
ansible/opnsense/requirements.yml
Normal file
4
ansible/opnsense/requirements.yml
Normal file
@@ -0,0 +1,4 @@
|
|||||||
|
---
|
||||||
|
collections:
|
||||||
|
- name: community.proxmox
|
||||||
|
version: "2.0.0"
|
||||||
432
ansible/opnsense/update_opnsense_ha_cluster.yml
Normal file
432
ansible/opnsense/update_opnsense_ha_cluster.yml
Normal file
@@ -0,0 +1,432 @@
|
|||||||
|
---
|
||||||
|
# OPNsense HA Cluster Update
|
||||||
|
# Order: backup (TrueNAS) first, then master (Proxmox)
|
||||||
|
###############################################################
|
||||||
|
- name: "Phase 1 | Firmware check on all nodes"
|
||||||
|
hosts: opnsense
|
||||||
|
any_errors_fatal: true
|
||||||
|
gather_facts: false
|
||||||
|
tasks:
|
||||||
|
|
||||||
|
- block:
|
||||||
|
|
||||||
|
- name: Check node availability
|
||||||
|
ansible.builtin.uri:
|
||||||
|
url: "https://{{ opnsense_host }}/api/core/system/status"
|
||||||
|
method: GET
|
||||||
|
user: "{{ opnsense_api_key }}"
|
||||||
|
password: "{{ opnsense_api_secret }}"
|
||||||
|
force_basic_auth: true
|
||||||
|
validate_certs: false
|
||||||
|
|
||||||
|
- name: Trigger firmware check
|
||||||
|
ansible.builtin.uri:
|
||||||
|
url: "https://{{ opnsense_host }}/api/core/firmware/check"
|
||||||
|
method: POST
|
||||||
|
user: "{{ opnsense_api_key }}"
|
||||||
|
password: "{{ opnsense_api_secret }}"
|
||||||
|
force_basic_auth: true
|
||||||
|
validate_certs: false
|
||||||
|
|
||||||
|
- name: Check VIP status
|
||||||
|
ansible.builtin.uri:
|
||||||
|
url: "https://{{ opnsense_host }}/api/diagnostics/interface/get_vip_status"
|
||||||
|
method: GET
|
||||||
|
user: "{{ opnsense_api_key }}"
|
||||||
|
password: "{{ opnsense_api_secret }}"
|
||||||
|
force_basic_auth: true
|
||||||
|
validate_certs: false
|
||||||
|
register: _vip_status
|
||||||
|
|
||||||
|
- name: Wait for check to complete
|
||||||
|
ansible.builtin.uri:
|
||||||
|
url: "https://{{ opnsense_host }}/api/core/firmware/status"
|
||||||
|
method: GET
|
||||||
|
user: "{{ opnsense_api_key }}"
|
||||||
|
password: "{{ opnsense_api_secret }}"
|
||||||
|
force_basic_auth: true
|
||||||
|
validate_certs: false
|
||||||
|
register: _firmware_status
|
||||||
|
until: _firmware_status.json.status_msg != "Firmware status requires to check for update first to provide more information."
|
||||||
|
retries: 10
|
||||||
|
delay: 2
|
||||||
|
|
||||||
|
- name: Store node facts
|
||||||
|
ansible.builtin.set_fact:
|
||||||
|
firmware_action: "{{ opnsense_action | default('update')}}"
|
||||||
|
firmware_status: "{{ _firmware_status.json.status }}"
|
||||||
|
firmware_status_msg: "{{ _firmware_status.json.status_msg }}"
|
||||||
|
firmware_current_version: "{{ _firmware_status.json.product.product_version | default('unknown') }}"
|
||||||
|
firmware_product_series: "{{ _firmware_status.json.product.product_series | default('unknown') }}"
|
||||||
|
firmware_update_version: "{{ _firmware_status.json.upgrade_packages | selectattr('name', 'equalto', 'opnsense') | map(attribute='new_version') | first | default('') }}"
|
||||||
|
firmware_upgrade_version: "{{ _firmware_status.json.upgrade_major_version | default('unknown') }}"
|
||||||
|
firmware_upgrade_message: "{{ _firmware_status.json.upgrade_major_message | regex_replace('<[^>]+>', ' ') | regex_replace('\\s{2,}', ' ') | trim | default('') }}"
|
||||||
|
firmware_up_to_date: "{{ _firmware_status.json.status_msg == 'There are no updates available on the selected mirror.' }}"
|
||||||
|
needs_reboot: "{{ (_firmware_status.json.upgrade_needs_reboot == '1') if _firmware_status.json.status == 'upgrade' else (_firmware_status.json.needs_reboot == '1') }}"
|
||||||
|
total_vips: "{{ _vip_status.json.rowCount }}"
|
||||||
|
mismatched_vips: "{{ _vip_status.json.rows | rejectattr('status', 'equalto', main_role) | list | length }}"
|
||||||
|
in_maintenance: "{{ _vip_status.json.carp.maintenancemode }}"
|
||||||
|
|
||||||
|
- name: Resolve update-vs-upgrade specifics
|
||||||
|
ansible.builtin.set_fact:
|
||||||
|
firmware_target_kind: "{{ 'series' if firmware_status == 'upgrade' else 'version' }}"
|
||||||
|
firmware_target_value: "{{ firmware_upgrade_version if firmware_status == 'upgrade' else firmware_update_version }}"
|
||||||
|
|
||||||
|
- name: Backup already updated
|
||||||
|
ansible.builtin.set_fact:
|
||||||
|
skip_update: true
|
||||||
|
firmware_status: "skipped"
|
||||||
|
delegate_to: "{{ groups['opnsense_backup'][0] }}"
|
||||||
|
delegate_facts: true
|
||||||
|
run_once: true
|
||||||
|
when: >-
|
||||||
|
(hostvars[groups['opnsense_backup'][0]].firmware_current_version
|
||||||
|
if hostvars[groups['opnsense_master'][0]].firmware_target_kind == 'version'
|
||||||
|
else hostvars[groups['opnsense_backup'][0]].firmware_product_series)
|
||||||
|
== hostvars[groups['opnsense_master'][0]].firmware_target_value
|
||||||
|
|
||||||
|
- name: Skip node {{ firmware_action }}
|
||||||
|
ansible.builtin.set_fact:
|
||||||
|
skip_update: true
|
||||||
|
firmware_status: "skipped"
|
||||||
|
when: firmware_up_to_date or firmware_action != firmware_status
|
||||||
|
|
||||||
|
- name: "Assert"
|
||||||
|
ansible.builtin.assert:
|
||||||
|
that:
|
||||||
|
- not in_maintenance
|
||||||
|
- mismatched_vips == 0
|
||||||
|
- total_vips > 0
|
||||||
|
quiet: true
|
||||||
|
|
||||||
|
rescue:
|
||||||
|
|
||||||
|
- name: Send Ntfy notification
|
||||||
|
ansible.builtin.uri:
|
||||||
|
url: "{{ ntfy_url }}/{{ ntfy_topic }}"
|
||||||
|
method: POST
|
||||||
|
user: "{{ ntfy_user }}"
|
||||||
|
password: "{{ lookup('env', 'NTFY_PASSWORD') }}"
|
||||||
|
force_basic_auth: true
|
||||||
|
body: |
|
||||||
|
Current version: {{ firmware_current_version }}
|
||||||
|
{% if in_maintenance %}
|
||||||
|
The node {{ inventory_hostname }} is in CARP maintenance mode.
|
||||||
|
{% elif mismatched_vips > 0 %}
|
||||||
|
Some VIP are not {{ main_role }} on {{ ansible_host }}.
|
||||||
|
{% elif total_vips == 0 %}
|
||||||
|
No VIPs are managed.
|
||||||
|
{% else %}
|
||||||
|
An error occured during the {{ firmware_action }}.
|
||||||
|
{% endif %}
|
||||||
|
headers:
|
||||||
|
Title: "OPNsense {{ firmware_action }} report"
|
||||||
|
Priority: "default"
|
||||||
|
Tags: "x"
|
||||||
|
when: ntfy_url is defined
|
||||||
|
|
||||||
|
- ansible.builtin.fail:
|
||||||
|
msg: "Update aborted"
|
||||||
|
|
||||||
|
|
||||||
|
- name: "Phase 2 | Update backup node (TrueNAS, no snapshot)"
|
||||||
|
hosts: opnsense_backup
|
||||||
|
any_errors_fatal: true
|
||||||
|
gather_facts: false
|
||||||
|
tasks:
|
||||||
|
|
||||||
|
- when: not skip_update | default(false)
|
||||||
|
block:
|
||||||
|
|
||||||
|
- name: Enable CARP maintenance mode
|
||||||
|
ansible.builtin.uri:
|
||||||
|
url: "https://{{ opnsense_host }}/api/diagnostics/interface/carp_status/maintenance"
|
||||||
|
method: POST
|
||||||
|
user: "{{ opnsense_api_key }}"
|
||||||
|
password: "{{ opnsense_api_secret }}"
|
||||||
|
force_basic_auth: true
|
||||||
|
validate_certs: false
|
||||||
|
register: _enable_maintenance
|
||||||
|
changed_when: _enable_maintenance.status == "ok"
|
||||||
|
|
||||||
|
- name: Trigger firmware {{ firmware_action }}
|
||||||
|
ansible.builtin.uri:
|
||||||
|
url: "https://{{ opnsense_host }}/api/core/firmware/{{ firmware_action }}"
|
||||||
|
method: POST
|
||||||
|
user: "{{ opnsense_api_key }}"
|
||||||
|
password: "{{ opnsense_api_secret }}"
|
||||||
|
force_basic_auth: true
|
||||||
|
validate_certs: false
|
||||||
|
|
||||||
|
- name: Check if the {{ firmware_action }} is running
|
||||||
|
ansible.builtin.uri:
|
||||||
|
url: "https://{{ opnsense_host }}/api/core/firmware/running"
|
||||||
|
method: GET
|
||||||
|
user: "{{ opnsense_api_key }}"
|
||||||
|
password: "{{ opnsense_api_secret }}"
|
||||||
|
force_basic_auth: true
|
||||||
|
validate_certs: false
|
||||||
|
register: _update_running
|
||||||
|
until: _update_running.json.status == 'busy'
|
||||||
|
retries: 2
|
||||||
|
delay: 2
|
||||||
|
|
||||||
|
- name: Wait for node to reboot after the {{ firmware_action }}
|
||||||
|
ansible.builtin.wait_for:
|
||||||
|
host: "{{ ansible_host }}"
|
||||||
|
port: "{{ opnsense_https_port }}"
|
||||||
|
state: stopped
|
||||||
|
timeout: 3600
|
||||||
|
when: needs_reboot
|
||||||
|
|
||||||
|
- name: Wait for node to come back online
|
||||||
|
ansible.builtin.wait_for:
|
||||||
|
host: "{{ ansible_host }}"
|
||||||
|
port: "{{ opnsense_https_port }}"
|
||||||
|
state: started
|
||||||
|
timeout: 5400
|
||||||
|
delay: 30
|
||||||
|
when: needs_reboot
|
||||||
|
|
||||||
|
- name: Check firmware version
|
||||||
|
ansible.builtin.uri:
|
||||||
|
url: "https://{{ opnsense_host }}/api/core/firmware/status"
|
||||||
|
method: GET
|
||||||
|
user: "{{ opnsense_api_key }}"
|
||||||
|
password: "{{ opnsense_api_secret }}"
|
||||||
|
force_basic_auth: true
|
||||||
|
validate_certs: false
|
||||||
|
register: _post_firmware_status
|
||||||
|
until: _post_firmware_status.json.product['product_' ~ firmware_target_kind] | default('unknown') == firmware_target_value
|
||||||
|
retries: 240
|
||||||
|
delay: 15
|
||||||
|
|
||||||
|
- name: Check VIP status
|
||||||
|
ansible.builtin.uri:
|
||||||
|
url: "https://{{ opnsense_host }}/api/diagnostics/interface/get_vip_status"
|
||||||
|
method: GET
|
||||||
|
user: "{{ opnsense_api_key }}"
|
||||||
|
password: "{{ opnsense_api_secret }}"
|
||||||
|
force_basic_auth: true
|
||||||
|
validate_certs: false
|
||||||
|
register: _carp_post_udapte
|
||||||
|
|
||||||
|
- name: Disable CARP maintenance mode
|
||||||
|
ansible.builtin.uri:
|
||||||
|
url: "https://{{ opnsense_host }}/api/diagnostics/interface/carp_status/maintenance"
|
||||||
|
method: POST
|
||||||
|
user: "{{ opnsense_api_key }}"
|
||||||
|
password: "{{ opnsense_api_secret }}"
|
||||||
|
force_basic_auth: true
|
||||||
|
validate_certs: false
|
||||||
|
when: _carp_post_udapte.json.carp.maintenancemode
|
||||||
|
ignore_errors: true
|
||||||
|
|
||||||
|
rescue:
|
||||||
|
|
||||||
|
- name: Send Ntfy notification
|
||||||
|
ansible.builtin.uri:
|
||||||
|
url: "{{ ntfy_url }}/{{ ntfy_topic }}"
|
||||||
|
method: POST
|
||||||
|
user: "{{ ntfy_user }}"
|
||||||
|
password: "{{ lookup('env', 'NTFY_PASSWORD') }}"
|
||||||
|
force_basic_auth: true
|
||||||
|
body: |
|
||||||
|
Current version: {{ firmware_current_version }}
|
||||||
|
{{ firmware_action | capitalize }} to {{ firmware_target_value }} failed on {{ inventory_hostname }} ({{ main_role }}).
|
||||||
|
No snapshot have been taken before the {{ firmware_action }}.
|
||||||
|
headers:
|
||||||
|
Title: "OPNsense {{ firmware_action }} failed on {{ inventory_hostname }}"
|
||||||
|
Priority: "high"
|
||||||
|
Tags: "x"
|
||||||
|
when: ntfy_url is defined
|
||||||
|
|
||||||
|
- ansible.builtin.fail:
|
||||||
|
msg: "Update failed"
|
||||||
|
|
||||||
|
- name: "Phase 3 | Update master node (Proxmox, with snapshot)"
|
||||||
|
hosts: opnsense_master
|
||||||
|
any_errors_fatal: true
|
||||||
|
gather_facts: false
|
||||||
|
vars:
|
||||||
|
proxmox_snap_name: "preupdate-{{ now().strftime('%Y%m%d') }}"
|
||||||
|
|
||||||
|
tasks:
|
||||||
|
|
||||||
|
- when: not skip_update | default(false)
|
||||||
|
block:
|
||||||
|
|
||||||
|
- name: Take Proxmox VM snapshot
|
||||||
|
community.proxmox.proxmox_snap:
|
||||||
|
vmid: "{{ proxmox_vmid }}"
|
||||||
|
state: present
|
||||||
|
snapname: "{{ proxmox_snap_name }}"
|
||||||
|
description: "Pre-firmware-{{ firmware_action }}: {{ firmware_current_version }} → {{ firmware_target_value }}"
|
||||||
|
|
||||||
|
- name: Enable CARP maintenance mode
|
||||||
|
ansible.builtin.uri:
|
||||||
|
url: "https://{{ opnsense_host }}/api/diagnostics/interface/carp_status/maintenance"
|
||||||
|
method: POST
|
||||||
|
user: "{{ opnsense_api_key }}"
|
||||||
|
password: "{{ opnsense_api_secret }}"
|
||||||
|
force_basic_auth: true
|
||||||
|
validate_certs: false
|
||||||
|
register: _enable_maintenance
|
||||||
|
changed_when: _enable_maintenance.status == "ok"
|
||||||
|
|
||||||
|
- name: Wait for CARP failover to settle
|
||||||
|
ansible.builtin.pause:
|
||||||
|
seconds: 5
|
||||||
|
|
||||||
|
- name: Trigger firmware {{ firmware_action }}
|
||||||
|
ansible.builtin.uri:
|
||||||
|
url: "https://{{ opnsense_host }}/api/core/firmware/{{ firmware_action }}"
|
||||||
|
method: POST
|
||||||
|
user: "{{ opnsense_api_key }}"
|
||||||
|
password: "{{ opnsense_api_secret }}"
|
||||||
|
force_basic_auth: true
|
||||||
|
validate_certs: false
|
||||||
|
|
||||||
|
- name: Check if the {{ firmware_action }} is running
|
||||||
|
ansible.builtin.uri:
|
||||||
|
url: "https://{{ opnsense_host }}/api/core/firmware/running"
|
||||||
|
method: GET
|
||||||
|
user: "{{ opnsense_api_key }}"
|
||||||
|
password: "{{ opnsense_api_secret }}"
|
||||||
|
force_basic_auth: true
|
||||||
|
validate_certs: false
|
||||||
|
register: _update_running
|
||||||
|
until: _update_running.json.status == 'busy'
|
||||||
|
retries: 2
|
||||||
|
delay: 2
|
||||||
|
|
||||||
|
- name: Wait for node to reboot after the {{ firmware_action }}
|
||||||
|
ansible.builtin.wait_for:
|
||||||
|
host: "{{ ansible_host }}"
|
||||||
|
port: "{{ opnsense_https_port }}"
|
||||||
|
state: stopped
|
||||||
|
timeout: 3600
|
||||||
|
when: needs_reboot
|
||||||
|
|
||||||
|
- name: Wait for node to come back online
|
||||||
|
ansible.builtin.wait_for:
|
||||||
|
host: "{{ ansible_host }}"
|
||||||
|
port: "{{ opnsense_https_port }}"
|
||||||
|
state: started
|
||||||
|
timeout: 5400
|
||||||
|
delay: 30
|
||||||
|
when: needs_reboot
|
||||||
|
|
||||||
|
- name: Check firmware version
|
||||||
|
ansible.builtin.uri:
|
||||||
|
url: "https://{{ opnsense_host }}/api/core/firmware/status"
|
||||||
|
method: GET
|
||||||
|
user: "{{ opnsense_api_key }}"
|
||||||
|
password: "{{ opnsense_api_secret }}"
|
||||||
|
force_basic_auth: true
|
||||||
|
validate_certs: false
|
||||||
|
register: _post_firmware_status
|
||||||
|
until: _post_firmware_status.json.product['product_' ~ firmware_target_kind] | default('unknown') == firmware_target_value
|
||||||
|
retries: 240
|
||||||
|
delay: 15
|
||||||
|
|
||||||
|
- name: Check VIP status
|
||||||
|
ansible.builtin.uri:
|
||||||
|
url: "https://{{ opnsense_host }}/api/diagnostics/interface/get_vip_status"
|
||||||
|
method: GET
|
||||||
|
user: "{{ opnsense_api_key }}"
|
||||||
|
password: "{{ opnsense_api_secret }}"
|
||||||
|
force_basic_auth: true
|
||||||
|
validate_certs: false
|
||||||
|
register: _carp_post_udapte
|
||||||
|
|
||||||
|
- name: Disable CARP maintenance mode
|
||||||
|
ansible.builtin.uri:
|
||||||
|
url: "https://{{ opnsense_host }}/api/diagnostics/interface/carp_status/maintenance"
|
||||||
|
method: POST
|
||||||
|
user: "{{ opnsense_api_key }}"
|
||||||
|
password: "{{ opnsense_api_secret }}"
|
||||||
|
force_basic_auth: true
|
||||||
|
validate_certs: false
|
||||||
|
when: _carp_post_udapte.json.carp.maintenancemode
|
||||||
|
ignore_errors: true
|
||||||
|
|
||||||
|
rescue:
|
||||||
|
|
||||||
|
- name: Rollback VM to the pre-{{ firmware_action }} snapshot
|
||||||
|
community.proxmox.proxmox_snap:
|
||||||
|
vmid: "{{ proxmox_vmid }}"
|
||||||
|
state: rollback
|
||||||
|
snapname: "{{ proxmox_snap_name }}"
|
||||||
|
register: _proxmox_snapshot_rollback
|
||||||
|
ignore_errors: true
|
||||||
|
|
||||||
|
- name: Send Ntfy notification
|
||||||
|
ansible.builtin.uri:
|
||||||
|
url: "{{ ntfy_url }}/{{ ntfy_topic }}"
|
||||||
|
method: POST
|
||||||
|
user: "{{ ntfy_user }}"
|
||||||
|
password: "{{ lookup('env', 'NTFY_PASSWORD') }}"
|
||||||
|
force_basic_auth: true
|
||||||
|
body: |
|
||||||
|
Current version: {{ firmware_current_version }}
|
||||||
|
{{ firmware_action | capitalize }} to {{ firmware_target_value }} failed on {{ inventory_hostname }} ({{ main_role }}).
|
||||||
|
A snapshot have been taken before the {{ firmware_action }} and the VM has been rollback.
|
||||||
|
headers:
|
||||||
|
Title: "OPNsense {{ firmware_action }} failed on {{ inventory_hostname }}"
|
||||||
|
Priority: "high"
|
||||||
|
Tags: "x"
|
||||||
|
when: ntfy_url is defined
|
||||||
|
|
||||||
|
- ansible.builtin.fail:
|
||||||
|
msg: "Update aborted"
|
||||||
|
|
||||||
|
|
||||||
|
- name: "Phase 4 | Notification"
|
||||||
|
hosts: localhost
|
||||||
|
gather_facts: false
|
||||||
|
tasks:
|
||||||
|
|
||||||
|
- name: Send Ntfy notification
|
||||||
|
vars:
|
||||||
|
master: "{{ groups['opnsense_master'][0] }}"
|
||||||
|
backup: "{{ groups['opnsense_backup'][0] }}"
|
||||||
|
m: "{{ hostvars[master] }}"
|
||||||
|
b: "{{ hostvars[backup] }}"
|
||||||
|
same_operation: "{{ m.firmware_status == b.firmware_status }}"
|
||||||
|
|
||||||
|
ansible.builtin.uri:
|
||||||
|
url: "{{ ntfy_url }}/{{ ntfy_topic }}"
|
||||||
|
method: POST
|
||||||
|
user: "{{ ntfy_user }}"
|
||||||
|
password: "{{ lookup('env', 'NTFY_PASSWORD') }}"
|
||||||
|
force_basic_auth: true
|
||||||
|
body: |
|
||||||
|
{% if same_operation %}
|
||||||
|
{% if m.skip_update | default(false) %}
|
||||||
|
Both nodes already on {{ m.firmware_current_version }}, no action taken.
|
||||||
|
{% else %}
|
||||||
|
OPNsense cluster: {{ m.firmware_current_version }} → {{ m.firmware_target_value }} ({{ m.firmware_status }})
|
||||||
|
{% endif %}
|
||||||
|
{% else %}
|
||||||
|
{% if m.skip_update | default(false) %}
|
||||||
|
MASTER ({{ master }}): already on {{ m.firmware_current_version }}, no action taken.
|
||||||
|
{% else %}
|
||||||
|
MASTER ({{ master }}): {{ m.firmware_current_version }} → {{ m.firmware_target_value }} ({{ m.firmware_status }})
|
||||||
|
{% endif %}
|
||||||
|
{% if b.skip_update | default(false) %}
|
||||||
|
BACKUP ({{ backup }}): already on {{ b.firmware_current_version }}, no action taken.
|
||||||
|
{% else %}
|
||||||
|
BACKUP ({{ backup }}): {{ b.firmware_current_version }} → {{ b.firmware_target_value }} ({{ b.firmware_status }})
|
||||||
|
{% endif %}
|
||||||
|
{% endif %}
|
||||||
|
headers:
|
||||||
|
Title: >-
|
||||||
|
{%- if m.skip_update | default(false) and b.skip_update | default(false) -%}
|
||||||
|
OPNsense cluster - no {{m.firmware_action }} available
|
||||||
|
{%- else -%}
|
||||||
|
OPNsense cluster {{m.firmware_action }} completed
|
||||||
|
{%- endif -%}
|
||||||
|
Priority: "{{ 'min' if (m.skip_update | default(false) and b.skip_update | default(false)) else 'default' }}"
|
||||||
|
Tags: "{{ 'information_source' if (m.skip_update | default(false) and b.skip_update | default(false)) else 'white_check_mark' }}"
|
||||||
|
when: ntfy_url is defined
|
||||||
204
ansible/proxmox/update_proxmox.yml
Normal file
204
ansible/proxmox/update_proxmox.yml
Normal file
@@ -0,0 +1,204 @@
|
|||||||
|
---
|
||||||
|
- name: Check Proxmox VE cluster health
|
||||||
|
hosts: nodes
|
||||||
|
any_errors_fatal: true
|
||||||
|
become: true
|
||||||
|
tasks:
|
||||||
|
|
||||||
|
- delegate_to: "{{ groups['nodes'][0] }}"
|
||||||
|
run_once: true
|
||||||
|
block:
|
||||||
|
|
||||||
|
- name: Verify cluster quorum
|
||||||
|
ansible.builtin.command: pvecm status
|
||||||
|
register: quorum_status
|
||||||
|
changed_when: false
|
||||||
|
failed_when: quorum_status.stdout is not search('Quorate:\\s*Yes')
|
||||||
|
|
||||||
|
- name: Verify Ceph health
|
||||||
|
ansible.builtin.command: ceph health
|
||||||
|
register: ceph_health
|
||||||
|
changed_when: false
|
||||||
|
failed_when: "'HEALTH_OK' not in ceph_health.stdout"
|
||||||
|
|
||||||
|
rescue:
|
||||||
|
|
||||||
|
- name: Send no ready notification
|
||||||
|
ansible.builtin.uri:
|
||||||
|
url: "{{ ntfy_url }}/{{ ntfy_topic }}"
|
||||||
|
method: POST
|
||||||
|
user: "{{ ntfy_user }}"
|
||||||
|
password: "{{ lookup('env', 'NTFY_PASSWORD') }}"
|
||||||
|
force_basic_auth: true
|
||||||
|
body: No updates have been rolled out
|
||||||
|
headers:
|
||||||
|
Title: "Proxmox VE Not Ready for Updates"
|
||||||
|
Priority: "default"
|
||||||
|
Tags: "x"
|
||||||
|
delegate_to: localhost
|
||||||
|
become: false
|
||||||
|
run_once: true
|
||||||
|
when: ntfy_url is defined
|
||||||
|
|
||||||
|
- ansible.builtin.fail:
|
||||||
|
msg: "Update aborted"
|
||||||
|
|
||||||
|
|
||||||
|
- name: Rolling update of Proxmox VE cluster
|
||||||
|
hosts: nodes
|
||||||
|
serial: 1
|
||||||
|
any_errors_fatal: true
|
||||||
|
become: true
|
||||||
|
tasks:
|
||||||
|
|
||||||
|
- block:
|
||||||
|
|
||||||
|
- name: Refresh repositories
|
||||||
|
ansible.builtin.apt:
|
||||||
|
update_cache: true
|
||||||
|
|
||||||
|
- name: Check if updates are available
|
||||||
|
ansible.builtin.apt:
|
||||||
|
upgrade: dist
|
||||||
|
check_mode: true
|
||||||
|
register: apt_check
|
||||||
|
|
||||||
|
- name: Proceed if updates are available
|
||||||
|
when: apt_check.changed
|
||||||
|
block:
|
||||||
|
|
||||||
|
- name: Get version before upgrade
|
||||||
|
ansible.builtin.shell: pveversion | awk -F'/' '{print $2}'
|
||||||
|
register: pve_old_version
|
||||||
|
changed_when: false
|
||||||
|
|
||||||
|
- name: Enable maintenance mode
|
||||||
|
ansible.builtin.command: >
|
||||||
|
ha-manager crm-command node-maintenance enable {{ inventory_hostname_short }}
|
||||||
|
|
||||||
|
- name: Wait for LXCs to leave node
|
||||||
|
ansible.builtin.shell: |
|
||||||
|
pct list | awk 'NR>1 && $2=="running" {count++} END {print count+0}'
|
||||||
|
register: lxc_count
|
||||||
|
changed_when: false
|
||||||
|
until: lxc_count.stdout | int == 0
|
||||||
|
retries: 60
|
||||||
|
delay: 15
|
||||||
|
|
||||||
|
- name: Wait for VMs to leave node
|
||||||
|
ansible.builtin.shell: |
|
||||||
|
qm list | awk 'NR>1 && $3=="running" {count++} END {print count+0}'
|
||||||
|
register: vm_count
|
||||||
|
changed_when: false
|
||||||
|
until: vm_count.stdout | int == 0
|
||||||
|
retries: 60
|
||||||
|
delay: 15
|
||||||
|
|
||||||
|
- name: Pause 15s
|
||||||
|
ansible.builtin.pause:
|
||||||
|
seconds: 15
|
||||||
|
|
||||||
|
- name: Update packages
|
||||||
|
ansible.builtin.apt:
|
||||||
|
upgrade: full
|
||||||
|
autoremove: true
|
||||||
|
autoclean: true
|
||||||
|
|
||||||
|
- name: Disable Ceph rebalancing
|
||||||
|
ansible.builtin.command: ceph osd set noout
|
||||||
|
|
||||||
|
- name: Reboot node
|
||||||
|
ansible.builtin.reboot:
|
||||||
|
reboot_timeout: 900
|
||||||
|
post_reboot_delay: 30
|
||||||
|
|
||||||
|
- name: Enable Ceph rebalancing
|
||||||
|
ansible.builtin.command: ceph osd unset noout
|
||||||
|
|
||||||
|
- name: Disable maintenance mode
|
||||||
|
ansible.builtin.command: >
|
||||||
|
ha-manager crm-command node-maintenance disable {{ inventory_hostname_short }}
|
||||||
|
|
||||||
|
- name: Get version after upgrade
|
||||||
|
ansible.builtin.shell: pveversion | awk -F'/' '{print $2}'
|
||||||
|
register: pve_new_version
|
||||||
|
changed_when: false
|
||||||
|
|
||||||
|
- name: Save update report
|
||||||
|
ansible.builtin.set_fact:
|
||||||
|
update_report:
|
||||||
|
old: "{{ pve_old_version.stdout }}"
|
||||||
|
new: "{{ pve_new_version.stdout }}"
|
||||||
|
|
||||||
|
- name: Wait for Ceph to be healthy
|
||||||
|
ansible.builtin.command: ceph health
|
||||||
|
register: ceph_status
|
||||||
|
changed_when: false
|
||||||
|
until: "'HEALTH_OK' in ceph_status.stdout"
|
||||||
|
retries: 60
|
||||||
|
delay: 15
|
||||||
|
delegate_to: "{{ groups['nodes'][0] }}"
|
||||||
|
|
||||||
|
rescue:
|
||||||
|
|
||||||
|
- name: Send failure notification
|
||||||
|
ansible.builtin.uri:
|
||||||
|
url: "{{ ntfy_url }}/{{ ntfy_topic }}"
|
||||||
|
method: POST
|
||||||
|
user: "{{ ntfy_user }}"
|
||||||
|
password: "{{ lookup('env', 'NTFY_PASSWORD') }}"
|
||||||
|
force_basic_auth: true
|
||||||
|
body: Update failed on {{ inventory_hostname_short }}
|
||||||
|
headers:
|
||||||
|
Title: "Proxmox VE Update Failed"
|
||||||
|
Priority: "high"
|
||||||
|
Tags: "x"
|
||||||
|
delegate_to: localhost
|
||||||
|
become: false
|
||||||
|
run_once: true
|
||||||
|
when: ntfy_url is defined
|
||||||
|
|
||||||
|
- ansible.builtin.fail:
|
||||||
|
msg: "Update aborted"
|
||||||
|
|
||||||
|
|
||||||
|
- name: Send notification
|
||||||
|
hosts: localhost
|
||||||
|
tasks:
|
||||||
|
|
||||||
|
- name: Determine if updates occurred
|
||||||
|
ansible.builtin.set_fact:
|
||||||
|
updates_performed: "{{ groups['nodes'] | map('extract', hostvars) | selectattr('update_report', 'defined') | list | length > 0 }}"
|
||||||
|
|
||||||
|
- name: Send success notification
|
||||||
|
ansible.builtin.uri:
|
||||||
|
url: "{{ ntfy_url }}/{{ ntfy_topic }}"
|
||||||
|
method: POST
|
||||||
|
user: "{{ ntfy_user }}"
|
||||||
|
password: "{{ lookup('env', 'NTFY_PASSWORD') }}"
|
||||||
|
force_basic_auth: true
|
||||||
|
body: |
|
||||||
|
{% set updated_nodes = [] %}
|
||||||
|
{% for node in groups['nodes'] %}
|
||||||
|
{% if hostvars[node].update_report is defined %}
|
||||||
|
{% set _ = updated_nodes.append(node) %}
|
||||||
|
{% endif %}
|
||||||
|
{% endfor %}
|
||||||
|
{% if not updates_performed %}
|
||||||
|
No updates available on the cluster.
|
||||||
|
{% else %}
|
||||||
|
The following nodes were updated:
|
||||||
|
{% for node in updated_nodes %}
|
||||||
|
{% if hostvars[node].update_report.old == hostvars[node].update_report.new %}
|
||||||
|
- {{ hostvars[node].inventory_hostname_short }}: version {{ hostvars[node].update_report.old }} (unchanged)
|
||||||
|
{% else %}
|
||||||
|
- {{ hostvars[node].inventory_hostname_short }}: version {{ hostvars[node].update_report.old }} → {{ hostvars[node].update_report.new }}
|
||||||
|
{% endif %}
|
||||||
|
{% endfor %}
|
||||||
|
{% endif %}
|
||||||
|
headers:
|
||||||
|
Title: "Proxmox VE Update Report"
|
||||||
|
Priority: "{{ 'min' if not updates_performed else 'default' }}"
|
||||||
|
Tags: "white_check_mark"
|
||||||
|
when: ntfy_url is defined
|
||||||
|
|
||||||
105
opnsense/10-wan
Normal file
105
opnsense/10-wan
Normal file
@@ -0,0 +1,105 @@
|
|||||||
|
#!/usr/local/bin/php
|
||||||
|
<?php
|
||||||
|
/**
|
||||||
|
* Author 2025 Etienne Girault <etienne.girault@gmail.com>
|
||||||
|
* OPNsense CARP event script
|
||||||
|
* - Enables/disables the WAN interface only when needed
|
||||||
|
* - Avoids reapplying config when CARP triggers multiple times
|
||||||
|
*/
|
||||||
|
|
||||||
|
require_once("config.inc");
|
||||||
|
require_once("interfaces.inc");
|
||||||
|
require_once("util.inc");
|
||||||
|
require_once("system.inc");
|
||||||
|
|
||||||
|
// Read CARP event arguments
|
||||||
|
$subsystem = !empty($argv[1]) ? $argv[1] : '';
|
||||||
|
$type = !empty($argv[2]) ? $argv[2] : '';
|
||||||
|
|
||||||
|
// Accept only MASTER/BACKUP events
|
||||||
|
if (!in_array($type, ['MASTER', 'BACKUP'])) {
|
||||||
|
// Ignore CARP INIT, DEMOTED, etc.
|
||||||
|
exit(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate subsystem name format, expected pattern: <ifname>@<vhid>
|
||||||
|
if (!preg_match('/^[a-z0-9_]+@\S+$/i', $subsystem)) {
|
||||||
|
log_error("Malformed subsystem argument: '{$subsystem}'.");
|
||||||
|
exit(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Only react to the primary VHID
|
||||||
|
list($vhid, $iface) = explode('@', $subsystem);
|
||||||
|
$primary_vhid = '1'; //
|
||||||
|
|
||||||
|
if ($vhid !== $primary_vhid) {
|
||||||
|
exit(0); // ignore events from other VHIDs
|
||||||
|
}
|
||||||
|
|
||||||
|
// Interface key to manage
|
||||||
|
$ifkey = 'wan';
|
||||||
|
$real_if = get_real_interface('wan');
|
||||||
|
// Fallback gateway name
|
||||||
|
$gw_name = 'LAN_GW';
|
||||||
|
// Determine whether WAN interface is currently enabled
|
||||||
|
$ifkey_enabled = !empty($config['interfaces'][$ifkey]['enable']) ? true : false;
|
||||||
|
// Lock file to prevent interface flapping
|
||||||
|
$lock_file = '/tmp/carp_wan_disable_lock';
|
||||||
|
$lock_default_age = 5;
|
||||||
|
$lock_max_age = 10;
|
||||||
|
|
||||||
|
// MASTER event
|
||||||
|
if ($type === "MASTER") {
|
||||||
|
// Enable WAN only if it's currently disabled
|
||||||
|
if (!$ifkey_enabled) {
|
||||||
|
// Check if lock file is present
|
||||||
|
if (file_exists($lock_file)) {
|
||||||
|
$lock_age = time() - (int)file_get_contents($lock_file);
|
||||||
|
if ($lock_age < $lock_max_age) {
|
||||||
|
log_msg("CARP event: WAN disable lock present ({$lock_age}s old), waiting...");
|
||||||
|
$elapsed = 0;
|
||||||
|
while (file_exists($lock_file) && $elapsed < 5000) {
|
||||||
|
usleep(500000);
|
||||||
|
$elapsed += 500;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
log_msg("CARP event: removing stale WAN disable lock.");
|
||||||
|
@unlink($lock_file);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
log_msg("CARP event: switching to '$type', enabling interface '$ifkey'.", LOG_WARNING);
|
||||||
|
$config['interfaces'][$ifkey]['enable'] = '1';
|
||||||
|
write_config("enable interface '$ifkey' due CARP event '$type'", false);
|
||||||
|
interface_configure(false, $ifkey, false, false);
|
||||||
|
} else {
|
||||||
|
log_msg("CARP event: already 'MASTER' for interface '$ifkey', nothing to do.");
|
||||||
|
}
|
||||||
|
|
||||||
|
// BACKUP event
|
||||||
|
} else {
|
||||||
|
// Disable WAN only if it's currently enabled
|
||||||
|
if ($ifkey_enabled) {
|
||||||
|
log_msg("CARP event: switching to '$type', disabling interface '$ifkey'.", LOG_WARNING);
|
||||||
|
unset($config['interfaces'][$ifkey]['enable']);
|
||||||
|
write_config("disable interface '$ifkey' due CARP event '$type'", false);
|
||||||
|
interface_configure(false, $ifkey, false, false);
|
||||||
|
// Remove WAN default gateway
|
||||||
|
mwexecf("/sbin/route delete default");
|
||||||
|
foreach ($config['OPNsense']['Gateways']['gateway_item'] as $gw) {
|
||||||
|
if ($gw['name'] === $gw_name) {
|
||||||
|
$gw_ip = $gw['gateway'];
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Shutdown WAN interface
|
||||||
|
mwexecf("/sbin/ifconfig {$real_if} down")
|
||||||
|
// Add fallback default gateway
|
||||||
|
mwexecf("/sbin/route add default {$gw_ip}");
|
||||||
|
// Create lock file
|
||||||
|
file_put_contents($lock_file, time());
|
||||||
|
sleep($lock_default_age);
|
||||||
|
@unlink($lock_file);
|
||||||
|
} else {
|
||||||
|
log_msg("CARP event: already '$type' for interface '$ifkey', nothing to do.");
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user