Compare commits
81 Commits
d8b6e9a91d
...
lab
| Author | SHA1 | Date | |
|---|---|---|---|
| 2d1c4c9934 | |||
| 29ba93e096 | |||
| 09862a53c1 | |||
| 336f1fd466 | |||
| 2937ba4344 | |||
| 52eed8aade | |||
| 6acf547df1 | |||
| d7f3a7ef06 | |||
| eccd940727 | |||
| 52eaeaae93 | |||
| c99279cb49 | |||
| 2d9c6ffb1d | |||
| 240791d22e | |||
| 86e0b4a23f | |||
| 1f5d99b5ae | |||
| cd0bfd956c | |||
| 6cf28240df | |||
| 697ac311a7 | |||
| d08f207fc4 | |||
| 74fa6432db | |||
| 8ab282eb95 | |||
| 3d18c01d5a | |||
| 2404692a30 | |||
| 557f80979b | |||
| fac4a27b75 | |||
| 2d80a76353 | |||
| d5c36e3118 | |||
| ff19c2faae | |||
| 4a7e8ef7ca | |||
| 3f2b208651 | |||
| fa334f90d4 | |||
| 43e535143b | |||
| f0ded8ac17 | |||
| f78c8a6cd7 | |||
| cbcbc45d00 | |||
| 869faf720a | |||
| 5d39cf3172 | |||
| 137b13d94e | |||
| b1008ec425 | |||
| 22de950cf5 | |||
| fd0a9a83e8 | |||
| 79302e8139 | |||
| 1ca4807d4b | |||
| 4c086cfad5 | |||
| b91ea08874 | |||
| ac69a7a5c6 | |||
| 0a576b7ec0 | |||
| e46f78b971 | |||
| a8b0fca263 | |||
| bbb2e0f024 | |||
| fbf38991d7 | |||
| f672517cc8 | |||
| 03a1b243bc | |||
| de745d74dc | |||
| 9c49b131e2 | |||
| 0912c0a253 | |||
| e2a1e53082 | |||
| 80d9befa6a | |||
| fcf5753f57 | |||
| 413694f47d | |||
| 2e912c7ad5 | |||
| 755e736ba9 | |||
| 12f3c65c39 | |||
| 02b908c4cd | |||
| d059b86dfc | |||
| 6e3814ce3f | |||
| fe24fec3a3 | |||
| 7d51740a1a | |||
| ca01e8f17e | |||
| b7cf1499a7 | |||
| 63db96d1bd | |||
| e1b7a88d1f | |||
| aaf111919d | |||
| f35bd094c7 | |||
| 98758f4fa7 | |||
| e4771be50c | |||
| 130db47180 | |||
| 8284a5fb66 | |||
| 0b7af06fb3 | |||
| eaef6ad961 | |||
| 20ff3707e6 |
4
.gitignore
vendored
4
.gitignore
vendored
@@ -1,4 +1,6 @@
|
||||
# Terraform
|
||||
**/.terraform*
|
||||
*.tfstate*
|
||||
*credentials.auto.tfvars
|
||||
*credentials.auto.tfvars
|
||||
# Bruno
|
||||
.bruno
|
||||
20
ansible/opnsense/inventory.yml
Normal file
20
ansible/opnsense/inventory.yml
Normal file
@@ -0,0 +1,20 @@
|
||||
---
|
||||
all:
|
||||
children:
|
||||
opnsense:
|
||||
vars:
|
||||
ansible_connection: local
|
||||
children:
|
||||
opnsense_backup:
|
||||
hosts:
|
||||
cerbere-head2:
|
||||
ansible_host: 192.168.88.3
|
||||
main_role: BACKUP
|
||||
hypervisor: TrueNAS
|
||||
opnsense_master:
|
||||
hosts:
|
||||
cerbere-head1:
|
||||
ansible_host: 192.168.88.2
|
||||
main_role: MASTER
|
||||
hypervisor: Proxmox
|
||||
proxmox_vmid: 122
|
||||
4
ansible/opnsense/requirements.yml
Normal file
4
ansible/opnsense/requirements.yml
Normal file
@@ -0,0 +1,4 @@
|
||||
---
|
||||
collections:
|
||||
- name: community.proxmox
|
||||
version: "2.0.0"
|
||||
432
ansible/opnsense/update_opnsense_ha_cluster.yml
Normal file
432
ansible/opnsense/update_opnsense_ha_cluster.yml
Normal file
@@ -0,0 +1,432 @@
|
||||
---
|
||||
# OPNsense HA Cluster Update
|
||||
# Order: backup (TrueNAS) first, then master (Proxmox)
|
||||
###############################################################
|
||||
- name: "Phase 1 | Firmware check on all nodes"
|
||||
hosts: opnsense
|
||||
any_errors_fatal: true
|
||||
gather_facts: false
|
||||
tasks:
|
||||
|
||||
- block:
|
||||
|
||||
- name: Check node availability
|
||||
ansible.builtin.uri:
|
||||
url: "https://{{ opnsense_host }}/api/core/system/status"
|
||||
method: GET
|
||||
user: "{{ opnsense_api_key }}"
|
||||
password: "{{ opnsense_api_secret }}"
|
||||
force_basic_auth: true
|
||||
validate_certs: false
|
||||
|
||||
- name: Trigger firmware check
|
||||
ansible.builtin.uri:
|
||||
url: "https://{{ opnsense_host }}/api/core/firmware/check"
|
||||
method: POST
|
||||
user: "{{ opnsense_api_key }}"
|
||||
password: "{{ opnsense_api_secret }}"
|
||||
force_basic_auth: true
|
||||
validate_certs: false
|
||||
|
||||
- name: Check VIP status
|
||||
ansible.builtin.uri:
|
||||
url: "https://{{ opnsense_host }}/api/diagnostics/interface/get_vip_status"
|
||||
method: GET
|
||||
user: "{{ opnsense_api_key }}"
|
||||
password: "{{ opnsense_api_secret }}"
|
||||
force_basic_auth: true
|
||||
validate_certs: false
|
||||
register: _vip_status
|
||||
|
||||
- name: Wait for check to complete
|
||||
ansible.builtin.uri:
|
||||
url: "https://{{ opnsense_host }}/api/core/firmware/status"
|
||||
method: GET
|
||||
user: "{{ opnsense_api_key }}"
|
||||
password: "{{ opnsense_api_secret }}"
|
||||
force_basic_auth: true
|
||||
validate_certs: false
|
||||
register: _firmware_status
|
||||
until: _firmware_status.json.status_msg != "Firmware status requires to check for update first to provide more information."
|
||||
retries: 10
|
||||
delay: 2
|
||||
|
||||
- name: Store node facts
|
||||
ansible.builtin.set_fact:
|
||||
firmware_action: "{{ opnsense_action | default('update')}}"
|
||||
firmware_status: "{{ _firmware_status.json.status }}"
|
||||
firmware_status_msg: "{{ _firmware_status.json.status_msg }}"
|
||||
firmware_current_version: "{{ _firmware_status.json.product.product_version | default('unknown') }}"
|
||||
firmware_product_series: "{{ _firmware_status.json.product.product_series | default('unknown') }}"
|
||||
firmware_update_version: "{{ _firmware_status.json.upgrade_packages | selectattr('name', 'equalto', 'opnsense') | map(attribute='new_version') | first | default('') }}"
|
||||
firmware_upgrade_version: "{{ _firmware_status.json.upgrade_major_version | default('unknown') }}"
|
||||
firmware_upgrade_message: "{{ _firmware_status.json.upgrade_major_message | regex_replace('<[^>]+>', ' ') | regex_replace('\\s{2,}', ' ') | trim | default('') }}"
|
||||
firmware_up_to_date: "{{ _firmware_status.json.status_msg == 'There are no updates available on the selected mirror.' }}"
|
||||
needs_reboot: "{{ (_firmware_status.json.upgrade_needs_reboot == '1') if _firmware_status.json.status == 'upgrade' else (_firmware_status.json.needs_reboot == '1') }}"
|
||||
total_vips: "{{ _vip_status.json.rowCount }}"
|
||||
mismatched_vips: "{{ _vip_status.json.rows | rejectattr('status', 'equalto', main_role) | list | length }}"
|
||||
in_maintenance: "{{ _vip_status.json.carp.maintenancemode }}"
|
||||
|
||||
- name: Resolve update-vs-upgrade specifics
|
||||
ansible.builtin.set_fact:
|
||||
firmware_target_kind: "{{ 'series' if firmware_status == 'upgrade' else 'version' }}"
|
||||
firmware_target_value: "{{ firmware_upgrade_version if firmware_status == 'upgrade' else firmware_update_version }}"
|
||||
|
||||
- name: Backup already updated
|
||||
ansible.builtin.set_fact:
|
||||
skip_update: true
|
||||
firmware_status: "skipped"
|
||||
delegate_to: "{{ groups['opnsense_backup'][0] }}"
|
||||
delegate_facts: true
|
||||
run_once: true
|
||||
when: >-
|
||||
(hostvars[groups['opnsense_backup'][0]].firmware_current_version
|
||||
if hostvars[groups['opnsense_master'][0]].firmware_target_kind == 'version'
|
||||
else hostvars[groups['opnsense_backup'][0]].firmware_product_series)
|
||||
== hostvars[groups['opnsense_master'][0]].firmware_target_value
|
||||
|
||||
- name: Skip node {{ firmware_action }}
|
||||
ansible.builtin.set_fact:
|
||||
skip_update: true
|
||||
firmware_status: "skipped"
|
||||
when: firmware_up_to_date or firmware_action != firmware_status
|
||||
|
||||
- name: "Assert"
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- not in_maintenance
|
||||
- mismatched_vips == 0
|
||||
- total_vips > 0
|
||||
quiet: true
|
||||
|
||||
rescue:
|
||||
|
||||
- name: Send Ntfy notification
|
||||
ansible.builtin.uri:
|
||||
url: "{{ ntfy_url }}/{{ ntfy_topic }}"
|
||||
method: POST
|
||||
user: "{{ ntfy_user }}"
|
||||
password: "{{ lookup('env', 'NTFY_PASSWORD') }}"
|
||||
force_basic_auth: true
|
||||
body: |
|
||||
Current version: {{ firmware_current_version }}
|
||||
{% if in_maintenance %}
|
||||
The node {{ inventory_hostname }} is in CARP maintenance mode.
|
||||
{% elif mismatched_vips > 0 %}
|
||||
Some VIP are not {{ main_role }} on {{ ansible_host }}.
|
||||
{% elif total_vips == 0 %}
|
||||
No VIPs are managed.
|
||||
{% else %}
|
||||
An error occured during the {{ firmware_action }}.
|
||||
{% endif %}
|
||||
headers:
|
||||
Title: "OPNsense {{ firmware_action }} report"
|
||||
Priority: "default"
|
||||
Tags: "x"
|
||||
when: ntfy_url is defined
|
||||
|
||||
- ansible.builtin.fail:
|
||||
msg: "Update aborted"
|
||||
|
||||
|
||||
- name: "Phase 2 | Update backup node (TrueNAS, no snapshot)"
|
||||
hosts: opnsense_backup
|
||||
any_errors_fatal: true
|
||||
gather_facts: false
|
||||
tasks:
|
||||
|
||||
- when: not skip_update | default(false)
|
||||
block:
|
||||
|
||||
- name: Enable CARP maintenance mode
|
||||
ansible.builtin.uri:
|
||||
url: "https://{{ opnsense_host }}/api/diagnostics/interface/carp_status/maintenance"
|
||||
method: POST
|
||||
user: "{{ opnsense_api_key }}"
|
||||
password: "{{ opnsense_api_secret }}"
|
||||
force_basic_auth: true
|
||||
validate_certs: false
|
||||
register: _enable_maintenance
|
||||
changed_when: _enable_maintenance.status == "ok"
|
||||
|
||||
- name: Trigger firmware {{ firmware_action }}
|
||||
ansible.builtin.uri:
|
||||
url: "https://{{ opnsense_host }}/api/core/firmware/{{ firmware_action }}"
|
||||
method: POST
|
||||
user: "{{ opnsense_api_key }}"
|
||||
password: "{{ opnsense_api_secret }}"
|
||||
force_basic_auth: true
|
||||
validate_certs: false
|
||||
|
||||
- name: Check if the {{ firmware_action }} is running
|
||||
ansible.builtin.uri:
|
||||
url: "https://{{ opnsense_host }}/api/core/firmware/running"
|
||||
method: GET
|
||||
user: "{{ opnsense_api_key }}"
|
||||
password: "{{ opnsense_api_secret }}"
|
||||
force_basic_auth: true
|
||||
validate_certs: false
|
||||
register: _update_running
|
||||
until: _update_running.json.status == 'busy'
|
||||
retries: 2
|
||||
delay: 2
|
||||
|
||||
- name: Wait for node to reboot after the {{ firmware_action }}
|
||||
ansible.builtin.wait_for:
|
||||
host: "{{ ansible_host }}"
|
||||
port: "{{ opnsense_https_port }}"
|
||||
state: stopped
|
||||
timeout: 3600
|
||||
when: needs_reboot
|
||||
|
||||
- name: Wait for node to come back online
|
||||
ansible.builtin.wait_for:
|
||||
host: "{{ ansible_host }}"
|
||||
port: "{{ opnsense_https_port }}"
|
||||
state: started
|
||||
timeout: 5400
|
||||
delay: 30
|
||||
when: needs_reboot
|
||||
|
||||
- name: Check firmware version
|
||||
ansible.builtin.uri:
|
||||
url: "https://{{ opnsense_host }}/api/core/firmware/status"
|
||||
method: GET
|
||||
user: "{{ opnsense_api_key }}"
|
||||
password: "{{ opnsense_api_secret }}"
|
||||
force_basic_auth: true
|
||||
validate_certs: false
|
||||
register: _post_firmware_status
|
||||
until: _post_firmware_status.json.product['product_' ~ firmware_target_kind] | default('unknown') == firmware_target_value
|
||||
retries: 240
|
||||
delay: 15
|
||||
|
||||
- name: Check VIP status
|
||||
ansible.builtin.uri:
|
||||
url: "https://{{ opnsense_host }}/api/diagnostics/interface/get_vip_status"
|
||||
method: GET
|
||||
user: "{{ opnsense_api_key }}"
|
||||
password: "{{ opnsense_api_secret }}"
|
||||
force_basic_auth: true
|
||||
validate_certs: false
|
||||
register: _carp_post_udapte
|
||||
|
||||
- name: Disable CARP maintenance mode
|
||||
ansible.builtin.uri:
|
||||
url: "https://{{ opnsense_host }}/api/diagnostics/interface/carp_status/maintenance"
|
||||
method: POST
|
||||
user: "{{ opnsense_api_key }}"
|
||||
password: "{{ opnsense_api_secret }}"
|
||||
force_basic_auth: true
|
||||
validate_certs: false
|
||||
when: _carp_post_udapte.json.carp.maintenancemode
|
||||
ignore_errors: true
|
||||
|
||||
rescue:
|
||||
|
||||
- name: Send Ntfy notification
|
||||
ansible.builtin.uri:
|
||||
url: "{{ ntfy_url }}/{{ ntfy_topic }}"
|
||||
method: POST
|
||||
user: "{{ ntfy_user }}"
|
||||
password: "{{ lookup('env', 'NTFY_PASSWORD') }}"
|
||||
force_basic_auth: true
|
||||
body: |
|
||||
Current version: {{ firmware_current_version }}
|
||||
{{ firmware_action | capitalize }} to {{ firmware_target_value }} failed on {{ inventory_hostname }} ({{ main_role }}).
|
||||
No snapshot have been taken before the {{ firmware_action }}.
|
||||
headers:
|
||||
Title: "OPNsense {{ firmware_action }} failed on {{ inventory_hostname }}"
|
||||
Priority: "high"
|
||||
Tags: "x"
|
||||
when: ntfy_url is defined
|
||||
|
||||
- ansible.builtin.fail:
|
||||
msg: "Update failed"
|
||||
|
||||
- name: "Phase 3 | Update master node (Proxmox, with snapshot)"
|
||||
hosts: opnsense_master
|
||||
any_errors_fatal: true
|
||||
gather_facts: false
|
||||
vars:
|
||||
proxmox_snap_name: "preupdate-{{ now().strftime('%Y%m%d') }}"
|
||||
|
||||
tasks:
|
||||
|
||||
- when: not skip_update | default(false)
|
||||
block:
|
||||
|
||||
- name: Take Proxmox VM snapshot
|
||||
community.proxmox.proxmox_snap:
|
||||
vmid: "{{ proxmox_vmid }}"
|
||||
state: present
|
||||
snapname: "{{ proxmox_snap_name }}"
|
||||
description: "Pre-firmware-{{ firmware_action }}: {{ firmware_current_version }} → {{ firmware_target_value }}"
|
||||
|
||||
- name: Enable CARP maintenance mode
|
||||
ansible.builtin.uri:
|
||||
url: "https://{{ opnsense_host }}/api/diagnostics/interface/carp_status/maintenance"
|
||||
method: POST
|
||||
user: "{{ opnsense_api_key }}"
|
||||
password: "{{ opnsense_api_secret }}"
|
||||
force_basic_auth: true
|
||||
validate_certs: false
|
||||
register: _enable_maintenance
|
||||
changed_when: _enable_maintenance.status == "ok"
|
||||
|
||||
- name: Wait for CARP failover to settle
|
||||
ansible.builtin.pause:
|
||||
seconds: 5
|
||||
|
||||
- name: Trigger firmware {{ firmware_action }}
|
||||
ansible.builtin.uri:
|
||||
url: "https://{{ opnsense_host }}/api/core/firmware/{{ firmware_action }}"
|
||||
method: POST
|
||||
user: "{{ opnsense_api_key }}"
|
||||
password: "{{ opnsense_api_secret }}"
|
||||
force_basic_auth: true
|
||||
validate_certs: false
|
||||
|
||||
- name: Check if the {{ firmware_action }} is running
|
||||
ansible.builtin.uri:
|
||||
url: "https://{{ opnsense_host }}/api/core/firmware/running"
|
||||
method: GET
|
||||
user: "{{ opnsense_api_key }}"
|
||||
password: "{{ opnsense_api_secret }}"
|
||||
force_basic_auth: true
|
||||
validate_certs: false
|
||||
register: _update_running
|
||||
until: _update_running.json.status == 'busy'
|
||||
retries: 2
|
||||
delay: 2
|
||||
|
||||
- name: Wait for node to reboot after the {{ firmware_action }}
|
||||
ansible.builtin.wait_for:
|
||||
host: "{{ ansible_host }}"
|
||||
port: "{{ opnsense_https_port }}"
|
||||
state: stopped
|
||||
timeout: 3600
|
||||
when: needs_reboot
|
||||
|
||||
- name: Wait for node to come back online
|
||||
ansible.builtin.wait_for:
|
||||
host: "{{ ansible_host }}"
|
||||
port: "{{ opnsense_https_port }}"
|
||||
state: started
|
||||
timeout: 5400
|
||||
delay: 30
|
||||
when: needs_reboot
|
||||
|
||||
- name: Check firmware version
|
||||
ansible.builtin.uri:
|
||||
url: "https://{{ opnsense_host }}/api/core/firmware/status"
|
||||
method: GET
|
||||
user: "{{ opnsense_api_key }}"
|
||||
password: "{{ opnsense_api_secret }}"
|
||||
force_basic_auth: true
|
||||
validate_certs: false
|
||||
register: _post_firmware_status
|
||||
until: _post_firmware_status.json.product['product_' ~ firmware_target_kind] | default('unknown') == firmware_target_value
|
||||
retries: 240
|
||||
delay: 15
|
||||
|
||||
- name: Check VIP status
|
||||
ansible.builtin.uri:
|
||||
url: "https://{{ opnsense_host }}/api/diagnostics/interface/get_vip_status"
|
||||
method: GET
|
||||
user: "{{ opnsense_api_key }}"
|
||||
password: "{{ opnsense_api_secret }}"
|
||||
force_basic_auth: true
|
||||
validate_certs: false
|
||||
register: _carp_post_udapte
|
||||
|
||||
- name: Disable CARP maintenance mode
|
||||
ansible.builtin.uri:
|
||||
url: "https://{{ opnsense_host }}/api/diagnostics/interface/carp_status/maintenance"
|
||||
method: POST
|
||||
user: "{{ opnsense_api_key }}"
|
||||
password: "{{ opnsense_api_secret }}"
|
||||
force_basic_auth: true
|
||||
validate_certs: false
|
||||
when: _carp_post_udapte.json.carp.maintenancemode
|
||||
ignore_errors: true
|
||||
|
||||
rescue:
|
||||
|
||||
- name: Rollback VM to the pre-{{ firmware_action }} snapshot
|
||||
community.proxmox.proxmox_snap:
|
||||
vmid: "{{ proxmox_vmid }}"
|
||||
state: rollback
|
||||
snapname: "{{ proxmox_snap_name }}"
|
||||
register: _proxmox_snapshot_rollback
|
||||
ignore_errors: true
|
||||
|
||||
- name: Send Ntfy notification
|
||||
ansible.builtin.uri:
|
||||
url: "{{ ntfy_url }}/{{ ntfy_topic }}"
|
||||
method: POST
|
||||
user: "{{ ntfy_user }}"
|
||||
password: "{{ lookup('env', 'NTFY_PASSWORD') }}"
|
||||
force_basic_auth: true
|
||||
body: |
|
||||
Current version: {{ firmware_current_version }}
|
||||
{{ firmware_action | capitalize }} to {{ firmware_target_value }} failed on {{ inventory_hostname }} ({{ main_role }}).
|
||||
A snapshot have been taken before the {{ firmware_action }} and the VM has been rollback.
|
||||
headers:
|
||||
Title: "OPNsense {{ firmware_action }} failed on {{ inventory_hostname }}"
|
||||
Priority: "high"
|
||||
Tags: "x"
|
||||
when: ntfy_url is defined
|
||||
|
||||
- ansible.builtin.fail:
|
||||
msg: "Update aborted"
|
||||
|
||||
|
||||
- name: "Phase 4 | Notification"
|
||||
hosts: localhost
|
||||
gather_facts: false
|
||||
tasks:
|
||||
|
||||
- name: Send Ntfy notification
|
||||
vars:
|
||||
master: "{{ groups['opnsense_master'][0] }}"
|
||||
backup: "{{ groups['opnsense_backup'][0] }}"
|
||||
m: "{{ hostvars[master] }}"
|
||||
b: "{{ hostvars[backup] }}"
|
||||
same_operation: "{{ m.firmware_status == b.firmware_status }}"
|
||||
|
||||
ansible.builtin.uri:
|
||||
url: "{{ ntfy_url }}/{{ ntfy_topic }}"
|
||||
method: POST
|
||||
user: "{{ ntfy_user }}"
|
||||
password: "{{ lookup('env', 'NTFY_PASSWORD') }}"
|
||||
force_basic_auth: true
|
||||
body: |
|
||||
{% if same_operation %}
|
||||
{% if m.skip_update | default(false) %}
|
||||
Both nodes already on {{ m.firmware_current_version }}, no action taken.
|
||||
{% else %}
|
||||
OPNsense cluster: {{ m.firmware_current_version }} → {{ m.firmware_target_value }} ({{ m.firmware_status }})
|
||||
{% endif %}
|
||||
{% else %}
|
||||
{% if m.skip_update | default(false) %}
|
||||
MASTER ({{ master }}): already on {{ m.firmware_current_version }}, no action taken.
|
||||
{% else %}
|
||||
MASTER ({{ master }}): {{ m.firmware_current_version }} → {{ m.firmware_target_value }} ({{ m.firmware_status }})
|
||||
{% endif %}
|
||||
{% if b.skip_update | default(false) %}
|
||||
BACKUP ({{ backup }}): already on {{ b.firmware_current_version }}, no action taken.
|
||||
{% else %}
|
||||
BACKUP ({{ backup }}): {{ b.firmware_current_version }} → {{ b.firmware_target_value }} ({{ b.firmware_status }})
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
headers:
|
||||
Title: >-
|
||||
{%- if m.skip_update | default(false) and b.skip_update | default(false) -%}
|
||||
OPNsense cluster - no {{m.firmware_action }} available
|
||||
{%- else -%}
|
||||
OPNsense cluster {{m.firmware_action }} completed
|
||||
{%- endif -%}
|
||||
Priority: "{{ 'min' if (m.skip_update | default(false) and b.skip_update | default(false)) else 'default' }}"
|
||||
Tags: "{{ 'information_source' if (m.skip_update | default(false) and b.skip_update | default(false)) else 'white_check_mark' }}"
|
||||
when: ntfy_url is defined
|
||||
204
ansible/proxmox/update_proxmox.yml
Normal file
204
ansible/proxmox/update_proxmox.yml
Normal file
@@ -0,0 +1,204 @@
|
||||
---
|
||||
- name: Check Proxmox VE cluster health
|
||||
hosts: nodes
|
||||
any_errors_fatal: true
|
||||
become: true
|
||||
tasks:
|
||||
|
||||
- delegate_to: "{{ groups['nodes'][0] }}"
|
||||
run_once: true
|
||||
block:
|
||||
|
||||
- name: Verify cluster quorum
|
||||
ansible.builtin.command: pvecm status
|
||||
register: quorum_status
|
||||
changed_when: false
|
||||
failed_when: quorum_status.stdout is not search('Quorate:\\s*Yes')
|
||||
|
||||
- name: Verify Ceph health
|
||||
ansible.builtin.command: ceph health
|
||||
register: ceph_health
|
||||
changed_when: false
|
||||
failed_when: "'HEALTH_OK' not in ceph_health.stdout"
|
||||
|
||||
rescue:
|
||||
|
||||
- name: Send no ready notification
|
||||
ansible.builtin.uri:
|
||||
url: "{{ ntfy_url }}/{{ ntfy_topic }}"
|
||||
method: POST
|
||||
user: "{{ ntfy_user }}"
|
||||
password: "{{ lookup('env', 'NTFY_PASSWORD') }}"
|
||||
force_basic_auth: true
|
||||
body: No updates have been rolled out
|
||||
headers:
|
||||
Title: "Proxmox VE Not Ready for Updates"
|
||||
Priority: "default"
|
||||
Tags: "x"
|
||||
delegate_to: localhost
|
||||
become: false
|
||||
run_once: true
|
||||
when: ntfy_url is defined
|
||||
|
||||
- ansible.builtin.fail:
|
||||
msg: "Update aborted"
|
||||
|
||||
|
||||
- name: Rolling update of Proxmox VE cluster
|
||||
hosts: nodes
|
||||
serial: 1
|
||||
any_errors_fatal: true
|
||||
become: true
|
||||
tasks:
|
||||
|
||||
- block:
|
||||
|
||||
- name: Refresh repositories
|
||||
ansible.builtin.apt:
|
||||
update_cache: true
|
||||
|
||||
- name: Check if updates are available
|
||||
ansible.builtin.apt:
|
||||
upgrade: dist
|
||||
check_mode: true
|
||||
register: apt_check
|
||||
|
||||
- name: Proceed if updates are available
|
||||
when: apt_check.changed
|
||||
block:
|
||||
|
||||
- name: Get version before upgrade
|
||||
ansible.builtin.shell: pveversion | awk -F'/' '{print $2}'
|
||||
register: pve_old_version
|
||||
changed_when: false
|
||||
|
||||
- name: Enable maintenance mode
|
||||
ansible.builtin.command: >
|
||||
ha-manager crm-command node-maintenance enable {{ inventory_hostname_short }}
|
||||
|
||||
- name: Wait for LXCs to leave node
|
||||
ansible.builtin.shell: |
|
||||
pct list | awk 'NR>1 && $2=="running" {count++} END {print count+0}'
|
||||
register: lxc_count
|
||||
changed_when: false
|
||||
until: lxc_count.stdout | int == 0
|
||||
retries: 60
|
||||
delay: 15
|
||||
|
||||
- name: Wait for VMs to leave node
|
||||
ansible.builtin.shell: |
|
||||
qm list | awk 'NR>1 && $3=="running" {count++} END {print count+0}'
|
||||
register: vm_count
|
||||
changed_when: false
|
||||
until: vm_count.stdout | int == 0
|
||||
retries: 60
|
||||
delay: 15
|
||||
|
||||
- name: Pause 15s
|
||||
ansible.builtin.pause:
|
||||
seconds: 15
|
||||
|
||||
- name: Update packages
|
||||
ansible.builtin.apt:
|
||||
upgrade: full
|
||||
autoremove: true
|
||||
autoclean: true
|
||||
|
||||
- name: Disable Ceph rebalancing
|
||||
ansible.builtin.command: ceph osd set noout
|
||||
|
||||
- name: Reboot node
|
||||
ansible.builtin.reboot:
|
||||
reboot_timeout: 900
|
||||
post_reboot_delay: 30
|
||||
|
||||
- name: Enable Ceph rebalancing
|
||||
ansible.builtin.command: ceph osd unset noout
|
||||
|
||||
- name: Disable maintenance mode
|
||||
ansible.builtin.command: >
|
||||
ha-manager crm-command node-maintenance disable {{ inventory_hostname_short }}
|
||||
|
||||
- name: Get version after upgrade
|
||||
ansible.builtin.shell: pveversion | awk -F'/' '{print $2}'
|
||||
register: pve_new_version
|
||||
changed_when: false
|
||||
|
||||
- name: Save update report
|
||||
ansible.builtin.set_fact:
|
||||
update_report:
|
||||
old: "{{ pve_old_version.stdout }}"
|
||||
new: "{{ pve_new_version.stdout }}"
|
||||
|
||||
- name: Wait for Ceph to be healthy
|
||||
ansible.builtin.command: ceph health
|
||||
register: ceph_status
|
||||
changed_when: false
|
||||
until: "'HEALTH_OK' in ceph_status.stdout"
|
||||
retries: 60
|
||||
delay: 15
|
||||
delegate_to: "{{ groups['nodes'][0] }}"
|
||||
|
||||
rescue:
|
||||
|
||||
- name: Send failure notification
|
||||
ansible.builtin.uri:
|
||||
url: "{{ ntfy_url }}/{{ ntfy_topic }}"
|
||||
method: POST
|
||||
user: "{{ ntfy_user }}"
|
||||
password: "{{ lookup('env', 'NTFY_PASSWORD') }}"
|
||||
force_basic_auth: true
|
||||
body: Update failed on {{ inventory_hostname_short }}
|
||||
headers:
|
||||
Title: "Proxmox VE Update Failed"
|
||||
Priority: "high"
|
||||
Tags: "x"
|
||||
delegate_to: localhost
|
||||
become: false
|
||||
run_once: true
|
||||
when: ntfy_url is defined
|
||||
|
||||
- ansible.builtin.fail:
|
||||
msg: "Update aborted"
|
||||
|
||||
|
||||
- name: Send notification
|
||||
hosts: localhost
|
||||
tasks:
|
||||
|
||||
- name: Determine if updates occurred
|
||||
ansible.builtin.set_fact:
|
||||
updates_performed: "{{ groups['nodes'] | map('extract', hostvars) | selectattr('update_report', 'defined') | list | length > 0 }}"
|
||||
|
||||
- name: Send success notification
|
||||
ansible.builtin.uri:
|
||||
url: "{{ ntfy_url }}/{{ ntfy_topic }}"
|
||||
method: POST
|
||||
user: "{{ ntfy_user }}"
|
||||
password: "{{ lookup('env', 'NTFY_PASSWORD') }}"
|
||||
force_basic_auth: true
|
||||
body: |
|
||||
{% set updated_nodes = [] %}
|
||||
{% for node in groups['nodes'] %}
|
||||
{% if hostvars[node].update_report is defined %}
|
||||
{% set _ = updated_nodes.append(node) %}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{% if not updates_performed %}
|
||||
No updates available on the cluster.
|
||||
{% else %}
|
||||
The following nodes were updated:
|
||||
{% for node in updated_nodes %}
|
||||
{% if hostvars[node].update_report.old == hostvars[node].update_report.new %}
|
||||
- {{ hostvars[node].inventory_hostname_short }}: version {{ hostvars[node].update_report.old }} (unchanged)
|
||||
{% else %}
|
||||
- {{ hostvars[node].inventory_hostname_short }}: version {{ hostvars[node].update_report.old }} → {{ hostvars[node].update_report.new }}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
headers:
|
||||
Title: "Proxmox VE Update Report"
|
||||
Priority: "{{ 'min' if not updates_performed else 'default' }}"
|
||||
Tags: "white_check_mark"
|
||||
when: ntfy_url is defined
|
||||
|
||||
105
opnsense/10-wan
Normal file
105
opnsense/10-wan
Normal file
@@ -0,0 +1,105 @@
|
||||
#!/usr/local/bin/php
|
||||
<?php
|
||||
/**
|
||||
* Author 2025 Etienne Girault <etienne.girault@gmail.com>
|
||||
* OPNsense CARP event script
|
||||
* - Enables/disables the WAN interface only when needed
|
||||
* - Avoids reapplying config when CARP triggers multiple times
|
||||
*/
|
||||
|
||||
require_once("config.inc");
|
||||
require_once("interfaces.inc");
|
||||
require_once("util.inc");
|
||||
require_once("system.inc");
|
||||
|
||||
// Read CARP event arguments
|
||||
$subsystem = !empty($argv[1]) ? $argv[1] : '';
|
||||
$type = !empty($argv[2]) ? $argv[2] : '';
|
||||
|
||||
// Accept only MASTER/BACKUP events
|
||||
if (!in_array($type, ['MASTER', 'BACKUP'])) {
|
||||
// Ignore CARP INIT, DEMOTED, etc.
|
||||
exit(0);
|
||||
}
|
||||
|
||||
// Validate subsystem name format, expected pattern: <ifname>@<vhid>
|
||||
if (!preg_match('/^[a-z0-9_]+@\S+$/i', $subsystem)) {
|
||||
log_error("Malformed subsystem argument: '{$subsystem}'.");
|
||||
exit(0);
|
||||
}
|
||||
|
||||
// Only react to the primary VHID
|
||||
list($vhid, $iface) = explode('@', $subsystem);
|
||||
$primary_vhid = '1'; //
|
||||
|
||||
if ($vhid !== $primary_vhid) {
|
||||
exit(0); // ignore events from other VHIDs
|
||||
}
|
||||
|
||||
// Interface key to manage
|
||||
$ifkey = 'wan';
|
||||
$real_if = get_real_interface('wan');
|
||||
// Fallback gateway name
|
||||
$gw_name = 'LAN_GW';
|
||||
// Determine whether WAN interface is currently enabled
|
||||
$ifkey_enabled = !empty($config['interfaces'][$ifkey]['enable']) ? true : false;
|
||||
// Lock file to prevent interface flapping
|
||||
$lock_file = '/tmp/carp_wan_disable_lock';
|
||||
$lock_default_age = 5;
|
||||
$lock_max_age = 10;
|
||||
|
||||
// MASTER event
|
||||
if ($type === "MASTER") {
|
||||
// Enable WAN only if it's currently disabled
|
||||
if (!$ifkey_enabled) {
|
||||
// Check if lock file is present
|
||||
if (file_exists($lock_file)) {
|
||||
$lock_age = time() - (int)file_get_contents($lock_file);
|
||||
if ($lock_age < $lock_max_age) {
|
||||
log_msg("CARP event: WAN disable lock present ({$lock_age}s old), waiting...");
|
||||
$elapsed = 0;
|
||||
while (file_exists($lock_file) && $elapsed < 5000) {
|
||||
usleep(500000);
|
||||
$elapsed += 500;
|
||||
}
|
||||
} else {
|
||||
log_msg("CARP event: removing stale WAN disable lock.");
|
||||
@unlink($lock_file);
|
||||
}
|
||||
}
|
||||
log_msg("CARP event: switching to '$type', enabling interface '$ifkey'.", LOG_WARNING);
|
||||
$config['interfaces'][$ifkey]['enable'] = '1';
|
||||
write_config("enable interface '$ifkey' due CARP event '$type'", false);
|
||||
interface_configure(false, $ifkey, false, false);
|
||||
} else {
|
||||
log_msg("CARP event: already 'MASTER' for interface '$ifkey', nothing to do.");
|
||||
}
|
||||
|
||||
// BACKUP event
|
||||
} else {
|
||||
// Disable WAN only if it's currently enabled
|
||||
if ($ifkey_enabled) {
|
||||
log_msg("CARP event: switching to '$type', disabling interface '$ifkey'.", LOG_WARNING);
|
||||
unset($config['interfaces'][$ifkey]['enable']);
|
||||
write_config("disable interface '$ifkey' due CARP event '$type'", false);
|
||||
interface_configure(false, $ifkey, false, false);
|
||||
// Remove WAN default gateway
|
||||
mwexecf("/sbin/route delete default");
|
||||
foreach ($config['OPNsense']['Gateways']['gateway_item'] as $gw) {
|
||||
if ($gw['name'] === $gw_name) {
|
||||
$gw_ip = $gw['gateway'];
|
||||
break;
|
||||
}
|
||||
}
|
||||
// Shutdown WAN interface
|
||||
mwexecf("/sbin/ifconfig {$real_if} down")
|
||||
// Add fallback default gateway
|
||||
mwexecf("/sbin/route add default {$gw_ip}");
|
||||
// Create lock file
|
||||
file_put_contents($lock_file, time());
|
||||
sleep($lock_default_age);
|
||||
@unlink($lock_file);
|
||||
} else {
|
||||
log_msg("CARP event: already '$type' for interface '$ifkey', nothing to do.");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user