diff --git a/ansible/proxmox/update_proxmox.yml b/ansible/proxmox/update_proxmox.yml index 7cfeb81..4e8cdaa 100644 --- a/ansible/proxmox/update_proxmox.yml +++ b/ansible/proxmox/update_proxmox.yml @@ -1,68 +1,68 @@ --- -- name: Rolling update Proxmox VE cluster +- name: Check Proxmox VE cluster health hosts: nodes any_errors_fatal: true become: true - serial: 1 - vars: - ntfy_password: "{{ lookup('env', 'NTFY_PASSWORD') }}" - - pre_tasks: - - ########################################################## - # Verify input variables - ########################################################## - - - name: Assert ntfy variables - ansible.builtin.assert: - that: - - ntfy_topic is defined - - ntfy_user is defined - - ntfy_password != "" - quiet: true - when: ntfy_url is defined + tasks: ########################################################## # Verify cluster health ########################################################## - - name: Verify quorum - ansible.builtin.command: pvecm status - register: quorum_status - changed_when: false - delegate_to: "{{ groups['nodes'][0] }}" + - delegate_to: "{{ groups['nodes'][0] }}" run_once: true + block: - - name: Abort if cluster has no quorum - ansible.builtin.fail: - msg: "Cluster quorum not available" - when: quorum_status.stdout is not search('Quorate:\\s*Yes') + - name: Verify cluster quorum + ansible.builtin.command: pvecm status + register: quorum_status + changed_when: false + failed_when: quorum_status.stdout is not search('Quorate:\\s*Yes') + + - name: Verify Ceph health + ansible.builtin.command: ceph health + register: ceph_health + changed_when: false + failed_when: "'HEALTH_OK' not in ceph_health.stdout" - - name: Check Ceph health - ansible.builtin.command: ceph health - register: ceph_health - changed_when: false - delegate_to: "{{ groups['nodes'][0] }}" - run_once: true + ########################################################## + # Capture current versions + ########################################################## - - name: Abort if Ceph unhealthy - ansible.builtin.fail: - msg: "{{ ceph_health.stdout }}" - when: "'HEALTH_OK' not in ceph_health.stdout" - - tasks: - - - block: - - ########################################################## - # Capture current versions - ########################################################## - - - name: Capture version after upgrade + - name: Capture version before upgrade ansible.builtin.shell: pveversion | awk -F'/' '{print $2}' register: pve_current_version changed_when: false + rescue: + + - name: Send no ready notification + ansible.builtin.uri: + url: "{{ ntfy_url }}/{{ ntfy_topic }}" + method: POST + user: "{{ ntfy_user }}" + password: "{{ lookup('env', 'NTFY_PASSWORD') }}" + force_basic_auth: true + body: Proxmox cluster not ready for updates + headers: + Title: "Proxmox cluster update failed" + Priority: "5" + Tags: "x" + delegate_to: localhost + become: false + run_once: true + when: ntfy_url is defined + + +- name: Rolling update of Proxmox VE cluster + hosts: nodes + serial: 1 + any_errors_fatal: true + become: true + tasks: + + - block: + ########################################################## # Enable maintenance mode ########################################################## @@ -75,15 +75,6 @@ # Wait until node empty ########################################################## - - name: Wait for VMs to leave node - ansible.builtin.shell: | - qm list | awk 'NR>1 && $3=="running" {count++} END {print count+0}' - register: vm_count - changed_when: false - until: vm_count.stdout | int == 0 - retries: 60 - delay: 15 - - name: Wait for LXCs to leave node ansible.builtin.shell: | pct list | awk 'NR>1 && $2=="running" {count++} END {print count+0}' @@ -93,6 +84,15 @@ retries: 60 delay: 15 + - name: Wait for VMs to leave node + ansible.builtin.shell: | + qm list | awk 'NR>1 && $3=="running" {count++} END {print count+0}' + register: vm_count + changed_when: false + until: vm_count.stdout | int == 0 + retries: 60 + delay: 15 + ########################################################## # Upgrade packages ########################################################## @@ -101,7 +101,7 @@ ansible.builtin.apt: update_cache: true - - name: Dist upgrade + - name: Update packages ansible.builtin.apt: upgrade: dist autoremove: true @@ -114,6 +114,7 @@ - name: Capture version after upgrade ansible.builtin.shell: pveversion | awk -F'/' '{print $2}' register: pve_new_version + delegate_to: "{{ groups['nodes'][0] }}" changed_when: false ########################################################## @@ -147,7 +148,7 @@ # Ceph validation ########################################################## - - name: Wait for HEALTH_OK + - name: Wait for Ceph to be healthy ansible.builtin.command: ceph health register: ceph_status changed_when: false @@ -167,11 +168,11 @@ url: "{{ ntfy_url }}/{{ ntfy_topic }}" method: POST user: "{{ ntfy_user }}" - password: "{{ ntfy_password }}" + password: "{{ lookup('env', 'NTFY_PASSWORD') }}" force_basic_auth: true body: Update failed on {{ inventory_hostname_short }} headers: - Title: "Proxmox cluster update failed" + Title: "Proxmox Update Failed" Priority: "5" Tags: "x" delegate_to: localhost @@ -182,25 +183,27 @@ - ansible.builtin.fail: msg: "Update aborted" - post_tasks: + +- name: Send notification + hosts: localhost + tasks: - name: Send success notification ansible.builtin.uri: url: "{{ ntfy_url }}/{{ ntfy_topic }}" method: POST user: "{{ ntfy_user }}" - password: "{{ ntfy_password }}" + password: "{{ lookup('env', 'NTFY_PASSWORD') }}" force_basic_auth: true body: | Proxmox cluster update completed successfully. + {% if hostvars[groups['nodes'][0]].pve_current_version.stdout == hostvars[groups['nodes'][0]].pve_new_version.stdout %} Old version: {{ pve_current_version.stdout }} New version: {{ pve_new_version.stdout }} headers: - Title: "Proxmox Update" + Title: "Proxmox Update Completed" Priority: "3" Tags: "white_check_mark" delegate_to: localhost - become: false - run_once: true when: ntfy_url is defined