From dfadd55977838363b577633720de61ec5af80394 Mon Sep 17 00:00:00 2001 From: Vezpi Date: Fri, 18 Sep 2026 15:37:00 +0200 Subject: [PATCH] feat: deploy k8s vms with Terraform and Ansible(#8) Reviewed-on: https://git.vezpi.com/Vezpi/Homelab/pulls/8 --- ansible/kubernetes/README.md | 97 +++++++++ .../kubernetes/collections/requirements.yml | 4 + ansible/kubernetes/inventories/tf_k8s_vms.yml | 4 + ansible/kubernetes/k8s_node.yml | 6 + ansible/kubernetes/ping.yml | 5 + .../roles/k8s_node/defaults/main.yml | 32 +++ .../roles/k8s_node/tasks/containerd.yml | 34 +++ .../roles/k8s_node/tasks/firewall.yml | 62 ++++++ .../kubernetes/roles/k8s_node/tasks/main.yml | 5 + .../roles/k8s_node/tasks/packages.yml | 36 ++++ .../roles/k8s_node/tasks/system.yml | 51 +++++ terraform/projects/kubernetes_vms/README.md | 93 ++++++++ terraform/projects/kubernetes_vms/main.tf | 198 ++++++++++++++++++ terraform/projects/kubernetes_vms/provider.tf | 21 ++ .../projects/kubernetes_vms/variables.tf | 135 ++++++++++++ 15 files changed, 783 insertions(+) create mode 100644 ansible/kubernetes/README.md create mode 100644 ansible/kubernetes/collections/requirements.yml create mode 100644 ansible/kubernetes/inventories/tf_k8s_vms.yml create mode 100644 ansible/kubernetes/k8s_node.yml create mode 100644 ansible/kubernetes/ping.yml create mode 100644 ansible/kubernetes/roles/k8s_node/defaults/main.yml create mode 100644 ansible/kubernetes/roles/k8s_node/tasks/containerd.yml create mode 100644 ansible/kubernetes/roles/k8s_node/tasks/firewall.yml create mode 100644 ansible/kubernetes/roles/k8s_node/tasks/main.yml create mode 100644 ansible/kubernetes/roles/k8s_node/tasks/packages.yml create mode 100644 ansible/kubernetes/roles/k8s_node/tasks/system.yml create mode 100644 terraform/projects/kubernetes_vms/README.md create mode 100644 terraform/projects/kubernetes_vms/main.tf create mode 100644 terraform/projects/kubernetes_vms/provider.tf create mode 100644 terraform/projects/kubernetes_vms/variables.tf diff --git a/ansible/kubernetes/README.md b/ansible/kubernetes/README.md new file mode 100644 index 0000000..0768604 --- /dev/null +++ b/ansible/kubernetes/README.md @@ -0,0 +1,97 @@ +# Ansible – Kubernetes nodes + +Prepares the Kubernetes VM nodes provisioned by the +[`kubernetes_vms` Terraform project](../../terraform/projects/kubernetes_vms/). + +## What it does + +The `k8s_node` playbook configures each node for `kubeadm`/`kubelet`: + +1. **System** – disables swap (runtime + fstab), loads `overlay` and `br_netfilter`, + persists kernel modules, applies the Kubernetes `sysctl` settings. +2. **Packages** – installs prerequisites, adds the official Kubernetes apt repo/key, + installs and holds `kubeadm` and `kubelet`. +3. **containerd** – installs it, generates the default config, enables the + `SystemdCgroup` driver, starts and enables the service. +4. **Firewall** – configures UFW per role: SSH + common Kubernetes ports from the admin + subnet, control-plane ports (`2379:2380`, `10257`, `10259`) on masters, worker ports + (`10256`, `30000:32767`) on workers, default deny incoming. + +Group membership (who is a `master` / `worker`) is resolved from the Terraform inventory, +so the role-specific firewall rules apply automatically. + +> Cluster bootstrap (`kubeadm init` / `kubeadm join`) is **not** part of this playbook +> and is done manually per the migration roadmap. + +## Inventory + +The inventory is generated from Terraform state via the `cloud.terraform` inventory +plugin: + +```yaml +# inventories/tf_k8s_vms.yml +--- +plugin: cloud.terraform.terraform_provider +project_path: + - /tmp/semaphore/project_1/repository_1_template_7/terraform/projects/kubernetes_vms +``` + +> The `project_path` is a Semaphore runtime checkout path. If you run locally, point it +> at your own `terraform/projects/kubernetes_vms` directory. The same `ansible` provider +> that writes the inventory is used by Terraform, so hosts/groups are always in sync with +> the applied VMs. + +## Requirements + +Install the collections referenced by `collections/requirements.yml`: + +```bash +ansible-galaxy collection install -r collections/requirements.yml +``` + +- `cloud.terraform` – Terraform inventory plugin +- `community.general` – `ufw` module + +## Usage + +```bash +# Connectivity check +ansible-playbook -i inventories/tf_k8s_vms.yml ping.yml + +# Prepare all Kubernetes nodes +ansible-playbook -i inventories/tf_k8s_vms.yml k8s_node.yml +``` + +Both playbooks target `all`; the inventory plugin runs from the Terraform project path. + +## Role variables + +Defined in `roles/k8s_node/defaults/main.yml` (override as needed): + +| Variable | Default | Description | +| ------------------------------- | ---------------------------- | -------------------------------------- | +| `k8s_version` | `1.37` | Kubernetes minor version | +| `k8s_repo` | `pkgs.k8s.io` core v | Kubernetes apt repository | +| `k8s_repo_key` | matching `Release.key` URL | Repository signing key | +| `k8s_packages` | `kubelet`, `kubeadm` | Packages installed and held | +| `k8s_prerequisite_packages` | `apt-transport-https`, `ca-certificates`, `curl`, `gnupg` | Build deps for the repo | +| `k8s_firewall_admin_subnet` | `192.168.66.0/24` | Subnet allowed through UFW | +| `k8s_firewall_ssh_port` | `22` | SSH port | +| `k8s_firewall_common_ports` | `6443`, `10250` | Ports open on all nodes | +| `k8s_firewall_master_ports` | `2379:2380`, `10257`, `10259` | Ports open on masters | +| `k8s_firewall_worker_ports` | `10256`, `30000:32767` | Ports open on workers | + +## File reference + +| Path | Purpose | +| ---------------------------------------- | ------------------------------------ | +| `k8s_node.yml` | Main playbook (role `k8s_node`) | +| `ping.yml` | Connectivity smoke-test playbook | +| `inventories/tf_k8s_vms.yml` | Terraform-backed inventory | +| `collections/requirements.yml` | Required Ansible collections | +| `roles/k8s_node/tasks/main.yml` | Task dispatcher | +| `roles/k8s_node/tasks/system.yml` | Swap, kernel modules, sysctl | +| `roles/k8s_node/tasks/packages.yml` | Kubernetes repo + kubeadm/kubelet | +| `roles/k8s_node/tasks/containerd.yml` | containerd install/config | +| `roles/k8s_node/tasks/firewall.yml` | Per-role UFW rules | +| `roles/k8s_node/defaults/main.yml` | Role defaults | \ No newline at end of file diff --git a/ansible/kubernetes/collections/requirements.yml b/ansible/kubernetes/collections/requirements.yml new file mode 100644 index 0000000..fc84c67 --- /dev/null +++ b/ansible/kubernetes/collections/requirements.yml @@ -0,0 +1,4 @@ +--- +collections: + - name: cloud.terraform + - name: community.general \ No newline at end of file diff --git a/ansible/kubernetes/inventories/tf_k8s_vms.yml b/ansible/kubernetes/inventories/tf_k8s_vms.yml new file mode 100644 index 0000000..c6b4cc3 --- /dev/null +++ b/ansible/kubernetes/inventories/tf_k8s_vms.yml @@ -0,0 +1,4 @@ +--- +plugin: cloud.terraform.terraform_provider +project_path: + - /tmp/semaphore/project_1/repository_1_template_7/terraform/projects/kubernetes_vms \ No newline at end of file diff --git a/ansible/kubernetes/k8s_node.yml b/ansible/kubernetes/k8s_node.yml new file mode 100644 index 0000000..c999c49 --- /dev/null +++ b/ansible/kubernetes/k8s_node.yml @@ -0,0 +1,6 @@ +--- +- name: Prepare Kubernetes nodes + hosts: all + + roles: + - role: k8s_node \ No newline at end of file diff --git a/ansible/kubernetes/ping.yml b/ansible/kubernetes/ping.yml new file mode 100644 index 0000000..ecd6527 --- /dev/null +++ b/ansible/kubernetes/ping.yml @@ -0,0 +1,5 @@ +--- +- name: Prepare Kubernetes nodes + hosts: all + tasks: + - ansible.builtin.ping: \ No newline at end of file diff --git a/ansible/kubernetes/roles/k8s_node/defaults/main.yml b/ansible/kubernetes/roles/k8s_node/defaults/main.yml new file mode 100644 index 0000000..37471eb --- /dev/null +++ b/ansible/kubernetes/roles/k8s_node/defaults/main.yml @@ -0,0 +1,32 @@ +--- +k8s_version: "1.37" + +k8s_repo: "https://pkgs.k8s.io/core:/stable:/v{{ k8s_version }}/deb" +k8s_repo_key: "https://pkgs.k8s.io/core:/stable:/v{{ k8s_version }}/deb/Release.key" + +k8s_packages: + - kubelet + - kubeadm + +k8s_prerequisite_packages: + - apt-transport-https + - ca-certificates + - curl + - gnupg + +k8s_firewall_admin_subnet: "192.168.88.0/24" +k8s_firewall_ssh_port: "22" +k8s_firewall_common_ports: + - "6443" + - "10250" +k8s_firewall_master_ports: + - "2379:2380" + - "10257" + - "10259" +k8s_firewall_worker_ports: + - "10256" + - "30000:32767" +k8s_firewall_cilium_udp_ports: + - "8472" +k8s_firewall_cilium_tcp_ports: + - "4240" \ No newline at end of file diff --git a/ansible/kubernetes/roles/k8s_node/tasks/containerd.yml b/ansible/kubernetes/roles/k8s_node/tasks/containerd.yml new file mode 100644 index 0000000..5f0d65d --- /dev/null +++ b/ansible/kubernetes/roles/k8s_node/tasks/containerd.yml @@ -0,0 +1,34 @@ +--- +- name: Install containerd + ansible.builtin.apt: + name: containerd + state: present + update_cache: true + become: yes + +- name: Create containerd configuration directory + ansible.builtin.file: + path: /etc/containerd + state: directory + mode: "0755" + become: yes + +- name: Generate default containerd configuration + ansible.builtin.shell: + cmd: containerd config default > /etc/containerd/config.toml + creates: /etc/containerd/config.toml + become: yes + +- name: Enable systemd cgroup driver in containerd + ansible.builtin.replace: + path: /etc/containerd/config.toml + regexp: '^(\s*)SystemdCgroup(\s*=\s*)false' + replace: '\1SystemdCgroup\2true' + become: yes + +- name: Enable and start containerd + ansible.builtin.systemd: + name: containerd + state: restarted + enabled: true + become: yes \ No newline at end of file diff --git a/ansible/kubernetes/roles/k8s_node/tasks/firewall.yml b/ansible/kubernetes/roles/k8s_node/tasks/firewall.yml new file mode 100644 index 0000000..a8306b2 --- /dev/null +++ b/ansible/kubernetes/roles/k8s_node/tasks/firewall.yml @@ -0,0 +1,62 @@ +--- +- name: Allow SSH from admin subnet + community.general.ufw: + rule: allow + port: "{{ k8s_firewall_ssh_port }}" + proto: tcp + from_ip: "{{ k8s_firewall_admin_subnet }}" + become: yes + +- name: Allow common Kubernetes ports from admin subnet + community.general.ufw: + rule: allow + port: "{{ item }}" + proto: tcp + from_ip: "{{ k8s_firewall_admin_subnet }}" + become: yes + loop: "{{ k8s_firewall_common_ports }}" + +- name: Allow etcd and control plane ports on control-plane nodes + community.general.ufw: + rule: allow + port: "{{ item }}" + proto: tcp + from_ip: "{{ k8s_firewall_admin_subnet }}" + become: yes + loop: "{{ k8s_firewall_master_ports }}" + when: inventory_hostname in groups['master'] + +- name: Allow kube-proxy and NodePort ports on worker nodes + community.general.ufw: + rule: allow + port: "{{ item }}" + proto: tcp + from_ip: "{{ k8s_firewall_admin_subnet }}" + become: yes + loop: "{{ k8s_firewall_worker_ports }}" + when: inventory_hostname in groups['worker'] + +- name: Allow Cilium VXLAN and health check UDP ports from admin subnet + community.general.ufw: + rule: allow + port: "{{ item }}" + proto: udp + from_ip: "{{ k8s_firewall_admin_subnet }}" + become: yes + loop: "{{ k8s_firewall_cilium_udp_ports }}" + +- name: Allow Cilium health check TCP ports from admin subnet + community.general.ufw: + rule: allow + port: "{{ item }}" + proto: tcp + from_ip: "{{ k8s_firewall_admin_subnet }}" + become: yes + loop: "{{ k8s_firewall_cilium_tcp_ports }}" + +- name: Enable ufw and deny incoming by default + community.general.ufw: + state: enabled + policy: deny + direction: incoming + become: yes \ No newline at end of file diff --git a/ansible/kubernetes/roles/k8s_node/tasks/main.yml b/ansible/kubernetes/roles/k8s_node/tasks/main.yml new file mode 100644 index 0000000..1b0af9f --- /dev/null +++ b/ansible/kubernetes/roles/k8s_node/tasks/main.yml @@ -0,0 +1,5 @@ +--- +- ansible.builtin.import_tasks: system.yml +- ansible.builtin.import_tasks: packages.yml +- ansible.builtin.import_tasks: containerd.yml +- ansible.builtin.import_tasks: firewall.yml \ No newline at end of file diff --git a/ansible/kubernetes/roles/k8s_node/tasks/packages.yml b/ansible/kubernetes/roles/k8s_node/tasks/packages.yml new file mode 100644 index 0000000..a38b7c0 --- /dev/null +++ b/ansible/kubernetes/roles/k8s_node/tasks/packages.yml @@ -0,0 +1,36 @@ +--- +- name: Install prerequisite packages + ansible.builtin.apt: + name: "{{ k8s_prerequisite_packages }}" + state: present + update_cache: true + become: yes + +- name: Add Kubernetes apt key + ansible.builtin.apt_key: + url: "{{ k8s_repo_key }}" + keyring: /etc/apt/keyrings/kubernetes-apt-keyring.gpg + state: present + become: yes + +- name: Add Kubernetes apt repository + ansible.builtin.apt_repository: + repo: "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] {{ k8s_repo }} /" + state: present + filename: kubernetes + become: yes + +- name: Install kubelet and kubeadm + ansible.builtin.apt: + name: "{{ k8s_packages }}" + state: present + update_cache: true + become: yes + +- name: Hold kubelet and kubeadm packages + ansible.builtin.command: + cmd: apt-mark hold {{ item }} + loop: "{{ k8s_packages }}" + register: _kube_hold + become: yes + changed_when: "'already set on hold' not in _kube_hold.stdout" \ No newline at end of file diff --git a/ansible/kubernetes/roles/k8s_node/tasks/system.yml b/ansible/kubernetes/roles/k8s_node/tasks/system.yml new file mode 100644 index 0000000..83dada1 --- /dev/null +++ b/ansible/kubernetes/roles/k8s_node/tasks/system.yml @@ -0,0 +1,51 @@ +--- +- name: Disable swap + ansible.builtin.shell: + cmd: swapoff -a + become: yes + changed_when: false + +- name: Comment out swap entries in fstab + ansible.builtin.replace: + path: /etc/fstab + regexp: '^(\s*[^#\s][^\s]*\s+swap\s+.*)$' + replace: '# \1' + become: yes + +- name: Load overlay kernel module + ansible.builtin.shell: + cmd: modprobe overlay + become: yes + changed_when: false + failed_when: false + +- name: Load br_netfilter kernel module + ansible.builtin.shell: + cmd: modprobe br_netfilter + become: yes + changed_when: false + +- name: Persist kernel modules + ansible.builtin.copy: + dest: /etc/modules-load.d/k8s.conf + content: | + overlay + br_netfilter + mode: "0644" + become: yes + +- name: Configure kernel network settings + ansible.builtin.copy: + dest: /etc/sysctl.d/k8s.conf + content: | + net.bridge.bridge-nf-call-iptables = 1 + net.bridge.bridge-nf-call-ip6tables = 1 + net.ipv4.ip_forward = 1 + mode: "0644" + become: yes + +- name: Apply kernel network settings + ansible.builtin.command: + cmd: sysctl --system + become: yes + changed_when: false \ No newline at end of file diff --git a/terraform/projects/kubernetes_vms/README.md b/terraform/projects/kubernetes_vms/README.md new file mode 100644 index 0000000..3e65e20 --- /dev/null +++ b/terraform/projects/kubernetes_vms/README.md @@ -0,0 +1,93 @@ +# Terraform – Kubernetes VMs + +Provisions the Kubernetes node VMs on Proxmox for the homelab Kubernetes cluster. + +## Overview + +This Terraform project clones the `ubuntu-noble-cloud` template into virtual machines +on the proxmox nodes, following a capacity-aware placement algorithm: + +1. **Control plane (master)** VMs are spread one per node, on the nodes with the most + free memory that still fit the role's `ram`. +2. **Pack roles (worker)** are water-filled over the remaining capacity, most-available + node first, without a per-node limit. + +Each VM is tagged with the environment (`vm_env`) and registered in a Terraform-backed +Ansible inventory (via the `ansible` provider), so the `k8s_node` playbook can target +them by role group. + +## Placement details + +- Online Proxmox nodes are ordered by available memory (most available first, node name + as tie-break). +- The control-plane role must fit entirely on distinct nodes with `free_mib >= ram`. +- Workers are packed into residual capacity (`free_mib - ram_per_cp_vm_on_node`), + again most-available first. +- Naming: `kub--m` for masters and `kub--w` for workers, e.g. + `kub-prod-m01`, `kub-prod-w01`. + +Deployment is blocked by `check` blocks when the fleet cannot be placed: + +- no online node is available, +- not enough nodes have the free memory required for the control-plane role, +- residual capacity cannot fit all worker VMs. + +## Prerequisites + +- Proxmox with: + - a cloneable template named `ubuntu-noble-cloud` (from the `pve_vm` module), + - a `ceph-workload` datastore, + - VLAN bridge `vlan` (default `vlan66`), + - an API token and an SSH user/password with rights to create VMs on the nodes. +- Terraform with the `bpg/proxmox` and `ansible/ansible` providers (init downloads them). + +## Input variables + +| Variable | Type | Required | Default | Description | +| ---------------------------- | -------- | -------- | ---------------------------- | -------------------------------------------------- | +| `proxmox_endpoint` | string | yes | – | Proxmox API URL | +| `proxmox_api_token` | string | yes | – | Proxmox API token (secret) | +| `proxmox_ssh_username` | string | yes | – | Proxmox SSH user for placement (secret) | +| `proxmox_ssh_password` | string | yes | – | Proxmox SSH password (secret) | +| `vm_env` | string | no | `"prod"` | Environment, part of the VM name and tags | +| `master_count` | number | no | `null` → `vm_attr` default | Number of master VMs | +| `master_cpu` | number | no | `null` → `vm_attr` default | vCPU cores for masters | +| `master_ram` | number | no | `null` → `vm_attr` default | Master RAM in **GiB** (converted to MiB internally)| +| `master_vlan` | number | no | `null` → `vm_attr` default | VLAN for masters | +| `worker_count` | number | no | `null` → `vm_attr` default | Number of worker VMs | +| `worker_cpu` | number | no | `null` → `vm_attr` default | vCPU cores for workers | +| `worker_ram` | number | no | `null` → `vm_attr` default | Worker RAM in **GiB** | +| `worker_vlan` | number | no | `null` → `vm_attr` default | VLAN for workers | +| `vm_attr` | map(… ) | no | see below | Full per-role VM defaults | + +The `vm_attr` map holds the fallback attributes used when the corresponding +`master_*`/`worker_*` override is `null`: + +```hcl +vm_attr = { + master = { ram = 2048, cpu = 2, vlan = 66, count = 3 } + worker = { ram = 4096, cpu = 2, vlan = 66, count = 3 } +} +``` + +> `ram` values inside `vm_attr` are MiB; the `master_ram` / `worker_ram` overrides are +> set in GiB and converted internally (`× 1024`). The placement math always works in MiB. + +## Usage + +```bash +terraform init +terraform plan +terraform apply +``` + +The `vm_ip` output maps VM name to assigned IP and feeds the Ansible inventory +(`inventories/`) of the `k8s_node` playbook. + +## File reference + +| File | Purpose | +| ------------ | --------------------------------------------------------- | +| `main.tf` | Placement algorithm, `pve_vm` module wiring, checks | +| `variables.tf` | Clusters, Proxmox credentials, per-role defaults/overrides | +| `provider.tf` | `proxmox` and `ansible` provider configuration | \ No newline at end of file diff --git a/terraform/projects/kubernetes_vms/main.tf b/terraform/projects/kubernetes_vms/main.tf new file mode 100644 index 0000000..a78a3b3 --- /dev/null +++ b/terraform/projects/kubernetes_vms/main.tf @@ -0,0 +1,198 @@ +module "pve_vm" { + source = "../../modules/pve_vm" + for_each = local.vm_list + node_name = each.value.node_name + vm_name = each.key + vm_cpu = each.value.vm_cpu + vm_ram = each.value.vm_ram + vm_vlan = each.value.vm_vlan + vm_tags = ["${var.vm_env}"] +} + +locals { + node_stats = [ + for i, name in data.proxmox_virtual_environment_nodes.pve_nodes.names : { + name = name + free_mib = floor(data.proxmox_virtual_environment_nodes.pve_nodes.memory_available[i] / 1024 / 1024) + } + if data.proxmox_virtual_environment_nodes.pve_nodes.online[i] + ] + + max_free_mib = length(local.node_stats) > 0 ? max([for node in local.node_stats : node.free_mib]...) : 0 + + # Online nodes ordered by available memory, most available first (name as tie-break) + node_order = [ + for key in sort([ + for node in local.node_stats : "${format("%012d", local.max_free_mib - node.free_mib)}|${node.name}" + ]) : split("|", key)[1] + ] + + node_free = { for node in local.node_stats : node.name => node.free_mib } + + # Per-role simple overrides merged over vm_attr defaults (null keeps the default) + role_overrides = { + for role, cfg in var.vm_attr : role => lookup({ + "master" = { count = var.master_count, ram = var.master_ram != null ? var.master_ram * 1024 : null, cpu = var.master_cpu, vlan = var.master_vlan } + "worker" = { count = var.worker_count, ram = var.worker_ram != null ? var.worker_ram * 1024 : null, cpu = var.worker_cpu, vlan = var.worker_vlan } + }, role, {}) + } + + # Internal placement rule: the control-plane role (master) sits one per node, pack roles are unlimited + cp_role_name = "master" + role_max_per_node = { for role, cfg in var.vm_attr : role => role == local.cp_role_name ? 1 : 0 } + + effective_vm_attr = { + for role, cfg in var.vm_attr : role => merge(cfg, { + for attr, value in local.role_overrides[role] : attr => value if value != null + }, { max_per_node = local.role_max_per_node[role] }) + } + + # Control plane (max_per_node = 1): one VM per node, only on nodes with enough free memory + cp_role = [for role, cfg in local.effective_vm_attr : role if cfg.max_per_node == 1][0] + cp_attr = local.effective_vm_attr[local.cp_role] + + cp_feasible = [ + for node in local.node_order : node + if local.node_free[node] >= local.cp_attr.ram + ] + + cp_nodes = length(local.cp_feasible) >= local.cp_attr.count ? [ + for i in range(local.cp_attr.count) : local.cp_feasible[i] + ] : [] + + cp_hosts = toset(local.cp_nodes) + + # Residual capacity per node after control-plane VMs are reserved + residual_mib = { + for node in local.node_order : + node => local.node_free[node] - (contains(local.cp_hosts, node) ? local.cp_attr.ram : 0) + } + + max_residual_mib = length(local.node_order) > 0 ? max([for node in local.node_order : local.residual_mib[node]]...) : 0 + + # Remaining roles (max_per_node != 1): pack nodes with residual capacity, most available first + pack_roles = [for role, cfg in local.effective_vm_attr : role if cfg.max_per_node != 1] + + pack_slots = { + for role in local.pack_roles : role => { + for node in local.node_order : + node => min( + floor(local.residual_mib[node] / local.effective_vm_attr[role].ram), + local.effective_vm_attr[role].max_per_node == 0 + ? floor(local.residual_mib[node] / local.effective_vm_attr[role].ram) + : local.effective_vm_attr[role].max_per_node + ) + } + } + + pack_precedence = { + for role in local.pack_roles : role => [ + for key in sort([ + for node in local.node_order : + "${format("%012d", local.max_residual_mib - local.residual_mib[node])}|${node}" + if local.pack_slots[role][node] >= 1 + ]) : split("|", key)[1] + ] + } + + pack_slots_list = { + for role in local.pack_roles : role => + [for node in local.pack_precedence[role] : local.pack_slots[role][node]] + } + + # Water-fill: each node takes VMs until it runs out of capacity or demand is met + pack_placed = { + for role in local.pack_roles : role => [ + for i in range(length(local.pack_precedence[role])) : + min(local.pack_slots_list[role][i], max(0, local.effective_vm_attr[role].count - try(sum([for j in range(i) : local.pack_slots_list[role][j]]), 0))) + ] + } + + pack_prefix = { + for role in local.pack_roles : role => [ + for i in range(length(local.pack_placed[role])) : + try(sum([for j in range(i) : local.pack_placed[role][j]]), 0) + ] + } + + cp_placements = [ + for i in range(length(local.cp_nodes)) : { + vm_name = "kub-${var.vm_env}-${substr(local.cp_role, 0, 1)}${format("%01d", i + 1)}" + vm_role = local.cp_role + vm_node = local.cp_nodes[i] + } + ] + + pack_placements = flatten([ + for role in local.pack_roles : [ + for p in range(length(local.pack_precedence[role])) : [ + for k in range(local.pack_placed[role][p]) : { + vm_name = "kub-${var.vm_env}-${substr(role, 0, 1)}${format("%01d", local.pack_prefix[role][p] + k + 1)}" + vm_role = role + vm_node = local.pack_precedence[role][p] + } + ] + ] + ]) + + vm_list = { + for vm in concat(local.cp_placements, local.pack_placements) : + vm.vm_name => { + node_name = vm.vm_node + vm_cpu = local.effective_vm_attr[vm.vm_role].cpu + vm_ram = local.effective_vm_attr[vm.vm_role].ram + vm_vlan = local.effective_vm_attr[vm.vm_role].vlan + vm_role = vm.vm_role + } + } + + roles = toset([for vm in local.vm_list : vm.vm_role]) +} + +data "proxmox_virtual_environment_nodes" "pve_nodes" {} + +check "proxmox_nodes" { + assert { + condition = length(local.node_order) > 0 + error_message = "No online Proxmox node available for deployment." + } +} + +check "control_plane_capacity" { + assert { + condition = length(local.cp_nodes) == local.cp_attr.count + error_message = "Cannot place ${local.cp_attr.count} ${local.cp_role} VM(s): only ${length(local.cp_feasible)} online node(s) have at least ${local.cp_attr.ram} MiB free memory." + } +} + +check "pack_capacity" { + assert { + condition = length(local.pack_roles) == 0 ? true : alltrue([ + for role in local.pack_roles : + try(sum(local.pack_placed[role]), 0) == local.effective_vm_attr[role].count + ]) + error_message = "Cannot place all pack VMs: ${join(", ", [ + for role in local.pack_roles : + "${role} needs ${local.effective_vm_attr[role].count} VM(s) (${local.effective_vm_attr[role].ram} MiB each), residual free capacity fits only ${try(sum(local.pack_placed[role]), 0)} VM(s)." + if try(sum(local.pack_placed[role]), 0) != local.effective_vm_attr[role].count + ])}" + } +} + +output "vm_ip" { + value = { for k, v in module.pve_vm : k => v.vm_ip } +} + +resource "ansible_group" "vm_groups" { + for_each = local.roles + name = each.key +} + +resource "ansible_host" "vm_hosts" { + for_each = module.pve_vm + name = each.key + variables = { + ansible_host = each.value.vm_ip + } + groups = [local.vm_list[each.key].vm_role] +} \ No newline at end of file diff --git a/terraform/projects/kubernetes_vms/provider.tf b/terraform/projects/kubernetes_vms/provider.tf new file mode 100644 index 0000000..5a0be7a --- /dev/null +++ b/terraform/projects/kubernetes_vms/provider.tf @@ -0,0 +1,21 @@ +terraform { + required_providers { + proxmox = { + source = "bpg/proxmox" + } + ansible = { + source = "ansible/ansible" + } + } +} + +provider "proxmox" { + endpoint = var.proxmox_endpoint + api_token = var.proxmox_api_token + insecure = false + ssh { + agent = false + username = var.proxmox_ssh_username + password = var.proxmox_ssh_password + } +} \ No newline at end of file diff --git a/terraform/projects/kubernetes_vms/variables.tf b/terraform/projects/kubernetes_vms/variables.tf new file mode 100644 index 0000000..fff808f --- /dev/null +++ b/terraform/projects/kubernetes_vms/variables.tf @@ -0,0 +1,135 @@ +variable "proxmox_endpoint" { + description = "Proxmox URL endpoint" + type = string +} + +variable "proxmox_api_token" { + description = "Proxmox API token" + type = string + sensitive = true +} + +variable "proxmox_ssh_username" { + description = "Proxmox SSH username" + type = string + sensitive = true +} + +variable "proxmox_ssh_password" { + description = "Proxmox SSH password" + type = string + sensitive = true +} + +variable "vm_attr" { + description = "VM attributes per role. The control-plane role (master) is placed one per node; other roles are packed without a per-node limit." + type = map(object({ + ram = number + cpu = number + vlan = number + count = number + })) + default = { + "master" = { ram = 2048, cpu = 2, vlan = 66, count = 3 } + "worker" = { ram = 4096, cpu = 2, vlan = 66, count = 3 } + } + + validation { + condition = alltrue([for v in values(var.vm_attr) : v.count >= 1]) + error_message = "Each role must deploy at least 1 VM." + } +} + +variable "master_count" { + description = "Master role VM count override. null keeps the vm_attr default." + type = number + default = null + + validation { + condition = var.master_count == null || var.master_count >= 1 + error_message = "master_count must be null or at least 1." + } +} + +variable "master_cpu" { + description = "Master role CPU override. null keeps the vm_attr default." + type = number + default = null + + validation { + condition = var.master_cpu == null || var.master_cpu >= 1 + error_message = "master_cpu must be null or at least 1." + } +} + +variable "master_ram" { + description = "Master role RAM override in GiB (converted to MiB internally). null keeps the vm_attr default." + type = number + default = null + + validation { + condition = var.master_ram == null || var.master_ram >= 1 + error_message = "master_ram must be null or at least 1 GiB." + } +} + +variable "master_vlan" { + description = "Master role VLAN override. null keeps the vm_attr default." + type = number + default = null + + validation { + condition = var.master_vlan == null || var.master_vlan >= 1 + error_message = "master_vlan must be null or at least 1." + } +} + +variable "worker_count" { + description = "Worker role VM count override. null keeps the vm_attr default." + type = number + default = null + + validation { + condition = var.worker_count == null || var.worker_count >= 1 + error_message = "worker_count must be null or at least 1." + } +} + +variable "worker_cpu" { + description = "Worker role CPU override. null keeps the vm_attr default." + type = number + default = null + + validation { + condition = var.worker_cpu == null || var.worker_cpu >= 1 + error_message = "worker_cpu must be null or at least 1." + } +} + +variable "worker_ram" { + description = "Worker role RAM override in GiB (converted to MiB internally). null keeps the vm_attr default." + type = number + default = null + + validation { + condition = var.worker_ram == null || var.worker_ram >= 1 + error_message = "worker_ram must be null or at least 1 GiB." + } +} + +variable "worker_vlan" { + description = "Worker role VLAN override. null keeps the vm_attr default." + type = number + default = null + + validation { + condition = var.worker_vlan == null || var.worker_vlan >= 1 + error_message = "worker_vlan must be null or at least 1." + } +} + +variable "vm_env" { + description = "VM environment" + type = string + default = "prod" +} \ No newline at end of file