diff --git a/ansible/proxmox/update_proxmox.yml b/ansible/proxmox/update_proxmox.yml new file mode 100644 index 0000000..3886b3c --- /dev/null +++ b/ansible/proxmox/update_proxmox.yml @@ -0,0 +1,200 @@ +--- +- name: Check Proxmox VE cluster health + hosts: nodes + any_errors_fatal: true + become: true + tasks: + + - delegate_to: "{{ groups['nodes'][0] }}" + run_once: true + block: + + - name: Verify cluster quorum + ansible.builtin.command: pvecm status + register: quorum_status + changed_when: false + failed_when: quorum_status.stdout is not search('Quorate:\\s*Yes') + + - name: Verify Ceph health + ansible.builtin.command: ceph health + register: ceph_health + changed_when: false + failed_when: "'HEALTH_OK' not in ceph_health.stdout" + + rescue: + + - name: Send no ready notification + ansible.builtin.uri: + url: "{{ ntfy_url }}/{{ ntfy_topic }}" + method: POST + user: "{{ ntfy_user }}" + password: "{{ lookup('env', 'NTFY_PASSWORD') }}" + force_basic_auth: true + body: No updates have been rolled out + headers: + Title: "Proxmox VE Not Ready for Updates" + Priority: "default" + Tags: "x" + delegate_to: localhost + become: false + run_once: true + when: ntfy_url is defined + + - ansible.builtin.fail: + msg: "Update aborted" + + +- name: Rolling update of Proxmox VE cluster + hosts: nodes + serial: 1 + any_errors_fatal: true + become: true + tasks: + + - block: + + - name: Refresh repositories + ansible.builtin.apt: + update_cache: true + + - name: Check if updates are available + ansible.builtin.apt: + upgrade: dist + check_mode: true + register: apt_check + + - name: Proceed if updates are available + when: apt_check.changed + block: + + - name: Get version before upgrade + ansible.builtin.shell: pveversion | awk -F'/' '{print $2}' + register: pve_old_version + changed_when: false + + - name: Enable maintenance mode + ansible.builtin.command: > + ha-manager crm-command node-maintenance enable {{ inventory_hostname_short }} + + - name: Wait for LXCs to leave node + ansible.builtin.shell: | + pct list | awk 'NR>1 && $2=="running" {count++} END {print count+0}' + register: lxc_count + changed_when: false + until: lxc_count.stdout | int == 0 + retries: 60 + delay: 15 + + - name: Wait for VMs to leave node + ansible.builtin.shell: | + qm list | awk 'NR>1 && $3=="running" {count++} END {print count+0}' + register: vm_count + changed_when: false + until: vm_count.stdout | int == 0 + retries: 60 + delay: 15 + + - name: Update packages + ansible.builtin.apt: + upgrade: full + autoremove: true + autoclean: true + + - name: Disable Ceph rebalancing + ansible.builtin.command: ceph osd set noout + + - name: Reboot node + ansible.builtin.reboot: + reboot_timeout: 900 + post_reboot_delay: 30 + + - name: Enable Ceph rebalancing + ansible.builtin.command: ceph osd unset noout + + - name: Disable maintenance mode + ansible.builtin.command: > + ha-manager crm-command node-maintenance disable {{ inventory_hostname_short }} + + - name: Get version after upgrade + ansible.builtin.shell: pveversion | awk -F'/' '{print $2}' + register: pve_new_version + changed_when: false + + - name: Save update report + ansible.builtin.set_fact: + update_report: + old: "{{ pve_old_version.stdout }}" + new: "{{ pve_new_version.stdout }}" + + - name: Wait for Ceph to be healthy + ansible.builtin.command: ceph health + register: ceph_status + changed_when: false + until: "'HEALTH_OK' in ceph_status.stdout" + retries: 60 + delay: 15 + delegate_to: "{{ groups['nodes'][0] }}" + + rescue: + + - name: Send failure notification + ansible.builtin.uri: + url: "{{ ntfy_url }}/{{ ntfy_topic }}" + method: POST + user: "{{ ntfy_user }}" + password: "{{ lookup('env', 'NTFY_PASSWORD') }}" + force_basic_auth: true + body: Update failed on {{ inventory_hostname_short }} + headers: + Title: "Proxmox VE Update Failed" + Priority: "high" + Tags: "x" + delegate_to: localhost + become: false + run_once: true + when: ntfy_url is defined + + - ansible.builtin.fail: + msg: "Update aborted" + + +- name: Send notification + hosts: localhost + tasks: + + - name: Determine if updates occurred + ansible.builtin.set_fact: + updates_performed: "{{ groups['nodes'] | map('extract', hostvars) | selectattr('update_report', 'defined') | list | length > 0 }}" + + - name: Send success notification + ansible.builtin.uri: + url: "{{ ntfy_url }}/{{ ntfy_topic }}" + method: POST + user: "{{ ntfy_user }}" + password: "{{ lookup('env', 'NTFY_PASSWORD') }}" + force_basic_auth: true + body: | + {% set updated_nodes = [] %} + {% for node in groups['nodes'] %} + {% if hostvars[node].update_report is defined %} + {% set _ = updated_nodes.append(node) %} + {% endif %} + {% endfor %} + {% if not updates_performed %} + No updates available on the cluster. + {% else %} + The following nodes were updated: + {% for node in updated_nodes %} + {% if hostvars[node].update_report.old == hostvars[node].update_report.new %} + - {{ hostvars[node].inventory_hostname_short }}: version {{ hostvars[node].update_report.old }} (unchanged) + {% else %} + - {{ hostvars[node].inventory_hostname_short }}: version {{ hostvars[node].update_report.old }} → {{ hostvars[node].update_report.new }} + {% endif %} + {% endfor %} + {% endif %} + headers: + Title: "Proxmox VE Update Report" + Priority: "{{ 'min' if not updates_performed else 'default' }}" + Tags: "white_check_mark" + when: ntfy_url is defined + diff --git a/opnsense/10-wan b/opnsense/10-wan new file mode 100644 index 0000000..0373bb1 --- /dev/null +++ b/opnsense/10-wan @@ -0,0 +1,90 @@ +#!/usr/local/bin/php + + * OPNsense CARP event script + * - Enables/disables the WAN interface only when needed + * - Avoids reapplying config when CARP triggers multiple times + */ + +require_once("config.inc"); +require_once("interfaces.inc"); +require_once("util.inc"); +require_once("system.inc"); + +// Read CARP event arguments +$subsystem = !empty($argv[1]) ? $argv[1] : ''; +$type = !empty($argv[2]) ? $argv[2] : ''; + +// Accept only MASTER/BACKUP events +if (!in_array($type, ['MASTER', 'BACKUP'])) { + // Ignore CARP INIT, DEMOTED, etc. + exit(0); +} + +// Validate subsystem name format, expected pattern: @ +if (!preg_match('/^[a-z0-9_]+@\S+$/i', $subsystem)) { + log_error("Malformed subsystem argument: '{$subsystem}'."); + exit(0); +} + +// Only react to the primary VHID +list($vhid, $iface) = explode('@', $subsystem); +$primary_vhid = '1'; // + +if ($vhid !== $primary_vhid) { + exit(0); // ignore events from other VHIDs +} + +// Interface key to manage +$ifkey = 'wan'; +// Determine whether WAN interface is currently enabled +$ifkey_enabled = !empty($config['interfaces'][$ifkey]['enable']) ? true : false; +// Lock file to prevent interface flapping +$lock_file = '/tmp/carp_wan_disable_lock'; +$lock_default_age = 5; +$lock_max_age = 10; + +// MASTER event +if ($type === "MASTER") { + // Enable WAN only if it's currently disabled + if (!$ifkey_enabled) { + // Check if lock file is present + if (file_exists($lock_file)) { + $lock_age = time() - (int)file_get_contents($lock_file); + if ($lock_age < $lock_max_age) { + log_msg("CARP event: WAN disable lock present ({$lock_age}s old), waiting..."); + $elapsed = 0; + while (file_exists($lock_file) && $elapsed < 5000) { + usleep(500000); + $elapsed += 500; + } + } else { + log_msg("CARP event: removing stale WAN disable lock."); + @unlink($lock_file); + } + } + log_msg("CARP event: switching to '$type', enabling interface '$ifkey'.", LOG_WARNING); + $config['interfaces'][$ifkey]['enable'] = '1'; + write_config("enable interface '$ifkey' due CARP event '$type'", false); + interface_configure(false, $ifkey, false, false); + } else { + log_msg("CARP event: already 'MASTER' for interface '$ifkey', nothing to do."); + } + +// BACKUP event +} else { + // Disable WAN only if it's currently enabled + if ($ifkey_enabled) { + log_msg("CARP event: switching to '$type', disabling interface '$ifkey'.", LOG_WARNING); + unset($config['interfaces'][$ifkey]['enable']); + write_config("disable interface '$ifkey' due CARP event '$type'", false); + interface_configure(false, $ifkey, false, false); + // Create lock file + file_put_contents($lock_file, time()); + sleep($lock_default_age); + @unlink($lock_file); + } else { + log_msg("CARP event: already '$type' for interface '$ifkey', nothing to do."); + } +} \ No newline at end of file