Homelab/ansible/proxmox/update_proxmox.yml

---
- name: Check Proxmox VE cluster health
  hosts: nodes
  any_errors_fatal: true
  become: true
  tasks:

    - delegate_to: "{{ groups['nodes'][0] }}"
      run_once: true
      block:

        - name: Verify cluster quorum
          ansible.builtin.command: pvecm status
          register: quorum_status
          changed_when: false
          failed_when: quorum_status.stdout is not search('Quorate:\\s*Yes')
          
        - name: Verify Ceph health
          ansible.builtin.command: ceph health
          register: ceph_health
          changed_when: false
          failed_when: "'HEALTH_OK' not in ceph_health.stdout"

        - name: Get version before upgrade
          ansible.builtin.shell: pveversion | awk -F'/' '{print $2}'
          register: pve_current_version
          changed_when: false

      rescue:

        - name: Send no ready notification
          ansible.builtin.uri:
            url: "{{  ntfy_url }}/{{ ntfy_topic }}"
            method: POST
            user: "{{ ntfy_user }}"
            password: "{{ lookup('env', 'NTFY_PASSWORD') }}"
            force_basic_auth: true
            body: No updates have been rolled out
            headers:
              Title: "Proxmox Not Ready for Updates"
              Priority: "5"
              Tags: "x"
          delegate_to: localhost
          become: false
          run_once: true
          when: ntfy_url is defined


- name: Rolling update of Proxmox VE cluster
  hosts: nodes
  serial: 1
  any_errors_fatal: true
  become: true
  tasks:

    - block:

        - name: Refresh repositories
          ansible.builtin.apt:
            update_cache: true

        - name: Check if upgrades are available
          ansible.builtin.apt:
            upgrade: dist
          check_mode: true
          register: apt_check

        - name: Proceed if updates are available
          when: apt_check.changed
          block:         

          - name: Enable maintenance mode
            ansible.builtin.command: >
              ha-manager crm-command node-maintenance enable {{ inventory_hostname_short }}

          - name: Wait for LXCs to leave node
            ansible.builtin.shell: |
              pct list | awk 'NR>1 && $2=="running" {count++} END {print count+0}'
            register: lxc_count
            changed_when: false
            until: lxc_count.stdout | int == 0
            retries: 60
            delay: 15

          - name: Wait for VMs to leave node
            ansible.builtin.shell: |
              qm list | awk 'NR>1 && $3=="running" {count++} END {print count+0}'
            register: vm_count
            changed_when: false
            until: vm_count.stdout | int == 0
            retries: 60
            delay: 15

          - name: Update packages
            ansible.builtin.apt:
              upgrade: full
              autoremove: true
              autoclean: true

          - name: Get version after upgrade
            ansible.builtin.shell: pveversion | awk -F'/' '{print $2}'
            register: pve_new_version
            delegate_to: "{{ groups['nodes'][0] }}"
            changed_when: false

          - name: Disable Ceph rebalancing
            ansible.builtin.command: ceph osd set noout

          - name: Reboot node
            ansible.builtin.reboot:
              reboot_timeout: 900
              post_reboot_delay: 30
              
          - name: Enable Ceph rebalancing
            ansible.builtin.command: ceph osd unset noout

          - name: Disable maintenance mode
            ansible.builtin.command: >
              ha-manager crm-command node-maintenance disable {{ inventory_hostname_short }}

          - name: Wait for Ceph to be healthy
            ansible.builtin.command: ceph health
            register: ceph_status
            changed_when: false
            until: "'HEALTH_OK' in ceph_status.stdout"
            retries: 60
            delay: 15
            delegate_to: "{{ groups['nodes'][0] }}"
          
      rescue:

        - name: Send failure notification
          ansible.builtin.uri:
            url: "{{  ntfy_url }}/{{ ntfy_topic }}"
            method: POST
            user: "{{ ntfy_user }}"
            password: "{{ lookup('env', 'NTFY_PASSWORD') }}"
            force_basic_auth: true
            body: Update failed on {{ inventory_hostname_short }}
            headers:
              Title: "Proxmox Update Failed"
              Priority: "5"
              Tags: "x"
          delegate_to: localhost
          become: false
          run_once: true
          when: ntfy_url is defined

        - ansible.builtin.fail:
            msg: "Update aborted"


- name: Send notification
  hosts: localhost
  tasks:

    - name: Send success notification
      ansible.builtin.uri:
        url: "{{  ntfy_url }}/{{ ntfy_topic }}"
        method: POST
        user: "{{ ntfy_user }}"
        password: "{{ lookup('env', 'NTFY_PASSWORD') }}"
        force_basic_auth: true
        body: |
          {% if hostvars[groups['nodes'][0]].pve_current_version.stdout == hostvars[groups['nodes'][0]].pve_new_version.stdout %}
          Version unchanged ({{ hostvars[groups['nodes'][0]].pve_new_version.stdout }})
          {% else %}
          Old version: {{ pve_current_version.stdout }}
          New version: {{ pve_new_version.stdout }}
          {% endif %}
        headers:
          Title: "Proxmox Update Completed"
          Priority: "3"
          Tags: "white_check_mark"
      when: ntfy_url is defined