Homelab/ansible/proxmox/update_proxmox.yml

---
- name: Check Proxmox VE cluster health
  hosts: nodes
  any_errors_fatal: true
  become: true
  tasks:

    ##########################################################
    # Verify cluster health
    ##########################################################

    - delegate_to: "{{ groups['nodes'][0] }}"
      run_once: true
      block:

        - name: Verify cluster quorum
          ansible.builtin.command: pvecm status
          register: quorum_status
          changed_when: false
          failed_when: quorum_status.stdout is not search('Quorate:\\s*Yes')
          
        - name: Verify Ceph health
          ansible.builtin.command: ceph health
          register: ceph_health
          changed_when: false
          failed_when: "'HEALTH_OK' not in ceph_health.stdout"

        ##########################################################
        # Capture current versions
        ##########################################################

        - name: Capture version before upgrade
          ansible.builtin.shell: pveversion | awk -F'/' '{print $2}'
          register: pve_current_version
          changed_when: false

      rescue:

        - name: Send no ready notification
          ansible.builtin.uri:
            url: "{{  ntfy_url }}/{{ ntfy_topic }}"
            method: POST
            user: "{{ ntfy_user }}"
            password: "{{ lookup('env', 'NTFY_PASSWORD') }}"
            force_basic_auth: true
            body: Proxmox cluster not ready for updates
            headers:
              Title: "Proxmox cluster update failed"
              Priority: "5"
              Tags: "x"
          delegate_to: localhost
          become: false
          run_once: true
          when: ntfy_url is defined


- name: Rolling update of Proxmox VE cluster
  hosts: nodes
  serial: 1
  any_errors_fatal: true
  become: true
  tasks:

    - block:

    ##########################################################
    # Enable maintenance mode
    ##########################################################

        - name: Enable maintenance mode
          ansible.builtin.command: >
            ha-manager crm-command node-maintenance enable {{ inventory_hostname_short }}

    ##########################################################
    # Wait until node empty
    ##########################################################

        - name: Wait for LXCs to leave node
          ansible.builtin.shell: |
            pct list | awk 'NR>1 && $2=="running" {count++} END {print count+0}'
          register: lxc_count
          changed_when: false
          until: lxc_count.stdout | int == 0
          retries: 60
          delay: 15

        - name: Wait for VMs to leave node
          ansible.builtin.shell: |
            qm list | awk 'NR>1 && $3=="running" {count++} END {print count+0}'
          register: vm_count
          changed_when: false
          until: vm_count.stdout | int == 0
          retries: 60
          delay: 15

    ##########################################################
    # Upgrade packages
    ##########################################################

        - name: Refresh repositories
          ansible.builtin.apt:
            update_cache: true

        - name: Update packages
          ansible.builtin.apt:
            upgrade: dist
            autoremove: true
            autoclean: true

    ##########################################################
    # Capture new versions
    ##########################################################

        - name: Capture version after upgrade
          ansible.builtin.shell: pveversion | awk -F'/' '{print $2}'
          register: pve_new_version
          delegate_to: "{{ groups['nodes'][0] }}"
          changed_when: false

    ##########################################################
    # Reboot with Ceph rebalancing disabled
    ##########################################################

        - name: Disable Ceph rebalancing
          ansible.builtin.command: ceph osd set noout

        - name: Reboot node
          ansible.builtin.reboot:
            reboot_timeout: 900
            post_reboot_delay: 30

        - name: Wait for connection
          ansible.builtin.wait_for_connection:
            timeout: 600
            
        - name: Enable Ceph rebalancing
          ansible.builtin.command: ceph osd unset noout

    ##########################################################
    # Disable maintenance mode
    ##########################################################

        - name: Disable maintenance mode
          ansible.builtin.command: >
            ha-manager crm-command node-maintenance disable {{ inventory_hostname_short }}

    ##########################################################
    # Ceph validation
    ##########################################################

        - name: Wait for Ceph to be healthy
          ansible.builtin.command: ceph health
          register: ceph_status
          changed_when: false
          until: "'HEALTH_OK' in ceph_status.stdout"
          retries: 60
          delay: 15
          delegate_to: "{{ groups['nodes'][0] }}"
          
      rescue:

    ##########################################################
    # Send notification
    ##########################################################

        - name: Send failure notification
          ansible.builtin.uri:
            url: "{{  ntfy_url }}/{{ ntfy_topic }}"
            method: POST
            user: "{{ ntfy_user }}"
            password: "{{ lookup('env', 'NTFY_PASSWORD') }}"
            force_basic_auth: true
            body: Update failed on {{ inventory_hostname_short }}
            headers:
              Title: "Proxmox Update Failed"
              Priority: "5"
              Tags: "x"
          delegate_to: localhost
          become: false
          run_once: true
          when: ntfy_url is defined

        - ansible.builtin.fail:
            msg: "Update aborted"


- name: Send notification
  hosts: localhost
  tasks:

    - name: Send success notification
      ansible.builtin.uri:
        url: "{{  ntfy_url }}/{{ ntfy_topic }}"
        method: POST
        user: "{{ ntfy_user }}"
        password: "{{ lookup('env', 'NTFY_PASSWORD') }}"
        force_basic_auth: true
        body: |
          Proxmox cluster update completed successfully.
          {% if hostvars[groups['nodes'][0]].pve_current_version.stdout == hostvars[groups['nodes'][0]].pve_new_version.stdout %}
          Old version: {{ pve_current_version.stdout }}
          New version: {{ pve_new_version.stdout }}
        headers:
          Title: "Proxmox Update Completed"
          Priority: "3"
          Tags: "white_check_mark"
      delegate_to: localhost
      when: ntfy_url is defined