From 57ddf083f711730fc431b9b08663e6ac649d57cc Mon Sep 17 00:00:00 2001 From: Francesco Zimbolo Date: Thu, 23 Apr 2026 07:49:11 +0000 Subject: [PATCH] Initial commit --- .devcontainer/devcontainer.json | 35 ++++ README.md | 24 +++ ansible/deploy.yml | 153 ++++++++++++++++++ ansible/group_vars/all.yml | 37 +++++ ansible/inventory/hosts.yml | 13 ++ ansible/roles/linstor_cluster/tasks/main.yml | 61 +++++++ .../templates/linstor-client.conf.j2 | 3 + ansible/roles/linstor_ha/tasks/main.yml | 134 +++++++++++++++ .../linstor_ha/templates/linstor_db.toml.j2 | 9 ++ ansible/roles/linstor_install/tasks/main.yml | 63 ++++++++ ansible/roles/linstor_repo/tasks/main.yml | 24 +++ ansible/roles/linstor_storage/tasks/main.yml | 69 ++++++++ ansible/roles/proxmox_storage/tasks/main.yml | 29 ++++ docs/01-prerequisites.md | 21 +++ docs/02-configuration.md | 34 ++++ docs/03-execution.md | 21 +++ docs/04-architecture.md | 15 ++ docs/05-linstor-storage.md | 18 +++ docs/06-troubleshooting.md | 39 +++++ 19 files changed, 802 insertions(+) create mode 100644 .devcontainer/devcontainer.json create mode 100644 README.md create mode 100644 ansible/deploy.yml create mode 100644 ansible/group_vars/all.yml create mode 100644 ansible/inventory/hosts.yml create mode 100644 ansible/roles/linstor_cluster/tasks/main.yml create mode 100644 ansible/roles/linstor_cluster/templates/linstor-client.conf.j2 create mode 100644 ansible/roles/linstor_ha/tasks/main.yml create mode 100644 ansible/roles/linstor_ha/templates/linstor_db.toml.j2 create mode 100644 ansible/roles/linstor_install/tasks/main.yml create mode 100644 ansible/roles/linstor_repo/tasks/main.yml create mode 100644 ansible/roles/linstor_storage/tasks/main.yml create mode 100644 ansible/roles/proxmox_storage/tasks/main.yml create mode 100644 docs/01-prerequisites.md create mode 100644 docs/02-configuration.md create mode 100644 docs/03-execution.md create mode 100644 docs/04-architecture.md create mode 100644 docs/05-linstor-storage.md create mode 100644 docs/06-troubleshooting.md diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..05fde44 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,35 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the +// README at: https://github.com/devcontainers/templates/tree/main/src/python +{ + "name": "Python 3", + // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile + "image": "mcr.microsoft.com/devcontainers/python:3-3.14-trixie", + + // Features to add to the dev container. More info: https://containers.dev/features. + "features": { + "ghcr.io/devcontainers-extra/features/ansible": {}, + "ghcr.io/hspaans/devcontainer-features/ansible-lint": {} + }, + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [], + + // Use 'postCreateCommand' to run commands after the container is created. + // "postCreateCommand": "pip3 install --user -r requirements.txt", + + // Configure tool-specific properties. + "customizations": { + "vscode": { + "settings": { + "ansible.python.interpreterPath": "/usr/local/bin/python" + }, + "extensions": [ + "redhat.ansible", + "ms-python.python" + ] + } + } + + // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root. + // "remoteUser": "root" +} diff --git a/README.md b/README.md new file mode 100644 index 0000000..f632cbe --- /dev/null +++ b/README.md @@ -0,0 +1,24 @@ +# Proxmox & Linstor HCI Automated Provisioning + +This repository contains an Infrastructure as Code (IaC) pipeline to automatically bootstrap a Highly Available Proxmox Virtual Environment (PVE) cluster, with an optional High Availability Hyperconverged Linstor/DRBD storage stack. + +## 🚀 Quick Start (For Beginners) + +You do not need to install Ansible locally. This repository includes a DevContainer pre-configured with all necessary tools, linters, and extensions. + +1. Install[Visual Studio Code](https://code.visualstudio.com/) and [Docker](https://docs.docker.com/get-docker/) (or [Podman](https://podman.io/)). +2. Install the VS Code[Dev Containers Extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers). +3. Clone this repository and open the folder in VS Code. +4. Click **"Reopen in Container"** when prompted in the bottom right corner (or press `Ctrl+Shift+P` and type `Dev Containers: Reopen in Container`). +5. Proceed to the [Documentation](docs/01-prerequisites.md) to configure your cluster variables and run your first deployment. + +## 📚 Documentation + +Detailed documentation is available in the `docs/` directory. If you are new to Ansible or this stack, please read them in order: + +*[01 - Prerequisites & Node Preparation](docs/01-prerequisites.md) +*[02 - Configuring the Cluster (Inventory & Vars)](docs/02-configuration.md) +* [03 - Running the Playbook](docs/03-execution.md) +* [04 - Architecture & Phase Breakdown](docs/04-architecture.md) +* [05 - Linstor & Storage Deep Dive](docs/05-linstor-storage.md) +* [06 - Troubleshooting & FAQ](docs/06-troubleshooting.md) diff --git a/ansible/deploy.yml b/ansible/deploy.yml new file mode 100644 index 0000000..52e4e49 --- /dev/null +++ b/ansible/deploy.yml @@ -0,0 +1,153 @@ +--- +- name: Converged Proxmox & Optional Linstor Deployment + hosts: proxmox + become: true + gather_facts: true + any_errors_fatal: true + + vars: + corosync_ip: "{{ priv_ip | default(ansible_host) }}" + master_corosync_ip: "{{ hostvars[groups['proxmox'][0]]['priv_ip'] | default(hostvars[groups['proxmox'][0]]['ansible_host']) }}" + is_master: "{{ inventory_hostname == groups['proxmox'][0] }}" + + tasks: + # ========================================== + # PHASE 1: REPOSITORIES & LICENSING + # ========================================== + - name: Apply Proxmox Enterprise Key + ansible.builtin.command: pvesubscription set -k {{ pve_enterprise_key }} + when: pve_enterprise_key | length > 0 + register: pve_key_result + changed_when: "'successfully' in pve_key_result.stdout" + + - name: Manage Proxmox Enterprise Repository + ansible.builtin.lineinfile: + path: /etc/apt/sources.list.d/pve-enterprise.list + regexp: '^#?deb https://enterprise.proxmox.com/debian/pve' + line: "{{ '' if pve_enterprise_key | length > 0 else '#' }}deb https://enterprise.proxmox.com/debian/pve bookworm pve-enterprise" + state: present + + - name: Manage Proxmox No-Subscription Repository + ansible.builtin.apt_repository: + repo: 'deb http://download.proxmox.com/debian/pve bookworm pve-no-subscription' + state: "{{ 'absent' if pve_enterprise_key | length > 0 else 'present' }}" + filename: pve-no-subscription + + - name: Add Linstor Repositories + ansible.builtin.include_role: + name: linstor_repo + when: deploy_linstor | bool + + # ========================================== + # PHASE 2: UPDATES & NAG REMOVAL + # ========================================== + - name: Update APT cache and run dist-upgrade + ansible.builtin.apt: + update_cache: yes + upgrade: dist + autoremove: yes + register: apt_upgrade + + - name: Remove Proxmox Subscription Nag (JS Patch) + ansible.builtin.replace: + path: /usr/share/javascript/proxmox-widget-toolkit/proxmoxlib.js + regexp: "(Ext.Msg.show\\(\\{\\s+title: gettext\\('No valid subscription'\\),)" + replace: "void({ //\\1" + notify: Restart pveproxy + + - name: Install Linstor Packages + ansible.builtin.include_role: + name: linstor_install + when: deploy_linstor | bool + + - name: Reboot if kernel was updated + ansible.builtin.reboot: + msg: "Rebooting to apply new PVE Kernel updates" + when: apt_upgrade.changed + + # ========================================== + # PHASE 3: PROXMOX CLUSTERING + # ========================================== + - name: Ensure SSH keypair exists for root + ansible.builtin.user: + name: root + generate_ssh_key: yes + ssh_key_bits: 4096 + + - name: Fetch public SSH keys from all nodes + ansible.builtin.slurp: + src: /root/.ssh/id_rsa.pub + register: ssh_pub_keys + + - name: Distribute SSH keys to build full-mesh trust + ansible.posix.authorized_key: + user: root + key: "{{ hostvars[item]['ssh_pub_keys']['content'] | b64decode }}" + state: present + loop: "{{ ansible_play_hosts }}" + + - name: Automatically accept host keys for cluster subnets + ansible.builtin.blockinfile: + path: /root/.ssh/config + create: yes + mode: '0600' + block: | + Host {{ master_corosync_ip | regex_replace('\.[0-9]+$', '.*') }} {{ hostvars[groups['proxmox'][0]]['ansible_host'] | regex_replace('\.[0-9]+$', '.*') }} + StrictHostKeyChecking no + UserKnownHostsFile=/dev/null + + - name: Check current Proxmox cluster status + ansible.builtin.command: pvecm status + register: pvecm_status + ignore_errors: yes + changed_when: false + + - name: Initialize the Proxmox Cluster (Master Node) + ansible.builtin.command: pvecm create {{ pve_cluster_name }} --link0 {{ corosync_ip }} + when: + - pvecm_status.rc != 0 + - is_master | bool + + - name: Wait for Corosync to stabilize + ansible.builtin.pause: + seconds: 10 + when: + - pvecm_status.rc != 0 + - is_master | bool + + - name: Join the Proxmox Cluster (Worker Nodes) + ansible.builtin.command: > + pvecm add {{ master_corosync_ip }} --link0 {{ corosync_ip }} --use_ssh yes + when: + - pvecm_status.rc != 0 + - not is_master | bool + async: 60 + poll: 10 + + # ========================================== + # PHASE 4: LINSTOR CONFIGURATION + # ========================================== + - name: Configure Linstor (Skipped if deploy_linstor=false) + when: deploy_linstor | bool + block: + - name: Configure Linstor Cluster + ansible.builtin.include_role: + name: linstor_cluster + + - name: Configure Linstor Storage + ansible.builtin.include_role: + name: linstor_storage + + - name: Configure Linstor HA + ansible.builtin.include_role: + name: linstor_ha + + - name: Configure Proxmox Storage Plugin for Linstor + ansible.builtin.include_role: + name: proxmox_storage + + handlers: + - name: Restart pveproxy + ansible.builtin.service: + name: pveproxy + state: restarted \ No newline at end of file diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml new file mode 100644 index 0000000..44b7157 --- /dev/null +++ b/ansible/group_vars/all.yml @@ -0,0 +1,37 @@ +--- +# Deployment Toggles +deploy_linstor: false # Set to true to install and configure Linstor/DRBD + +# Proxmox Cluster Configuration +pve_cluster_name: "prod-cluster" +pve_enterprise_key: "" # Leave empty ("") to use no-subscription, or insert "PVE..." key + +# High Availability Controller Configuration +ha_pool: "fast_pool_1" +ha_vip: "10.191.125.4" +ha_vip_cidr: "29" + +# Linstor Storage Pools Configuration +linstor_storage_pools: + - pool_name: "fast_pool_1" + rg_name: "fast_pool_1" + vg_name: "pve" + thin_name: "data" + # TARGET RULES: + # Use raw disk (/dev/sdb) to wipe/create LVM + # Use existing thin-pool path (pve/data) to register safely + targets: + pve-node-01: "pve/data" + pve-node-02: "pve/data" + pve-node-03: "pve/data" + - pool_name: "slow_pool_1" + rg_name: "slow_pool_1" + vg_name: "sp1" + thin_name: "data" + # TARGET RULES: + # Use raw disk (/dev/sdb) to wipe/create LVM + # Use existing thin-pool path (pve/data) to register safely + targets: + pve-node-01: "/dev/sda" + pve-node-02: "/dev/sda" + pve-node-03: "/dev/sda" diff --git a/ansible/inventory/hosts.yml b/ansible/inventory/hosts.yml new file mode 100644 index 0000000..1dc25c3 --- /dev/null +++ b/ansible/inventory/hosts.yml @@ -0,0 +1,13 @@ +all: + children: + proxmox: + hosts: + pve-node-01: + ansible_host: 10.191.25.21 + priv_ip: 10.191.125.1 + pve-node-02: + ansible_host: 10.191.25.22 + priv_ip: 10.191.125.2 + pve-node-03: + ansible_host: 10.191.25.23 + priv_ip: 10.191.125.3 \ No newline at end of file diff --git a/ansible/roles/linstor_cluster/tasks/main.yml b/ansible/roles/linstor_cluster/tasks/main.yml new file mode 100644 index 0000000..9363b24 --- /dev/null +++ b/ansible/roles/linstor_cluster/tasks/main.yml @@ -0,0 +1,61 @@ +--- +# tasks file for linstor_cluster + +- name: Safely manage LINSTOR cluster nodes in /etc/hosts + ansible.builtin.blockinfile: + path: /etc/hosts + marker: "# {mark} ANSIBLE MANAGED BLOCK - LINSTOR CLUSTER" + block: | + # Linstor Cluster Nodes (Primary: Private Network) + {% for host in groups['proxmox'] %} + {{ hostvars[host].priv_ip }} {{ host }} + {% endfor %} + + # Linstor Cluster Nodes (Fallback: Public Network) + {% for host in groups['proxmox'] %} + {{ hostvars[host].ansible_host }} {{ host }}-public + {% endfor %} + +- name: Ensure /etc/linstor directory exists + ansible.builtin.file: + path: /etc/linstor + state: directory + mode: '0755' + +- name: Deploy linstor-client.conf + ansible.builtin.template: + src: linstor-client.conf.j2 + dest: /etc/linstor/linstor-client.conf + mode: '0644' + +# --------------------------------------------------------- +# LINSTOR NODE REGISTRATION (Executed ONLY on Node 1) +# --------------------------------------------------------- + +- name: Register nodes in LINSTOR + ansible.builtin.shell: | + set -o pipefail + if ! linstor node list | grep -q " {{ item }} "; then + linstor node create "{{ item }}" "{{ hostvars[item].priv_ip }}" --node-type Combined > /dev/null + echo "changed" + fi + args: + executable: /bin/bash + register: linstor_cluster_node_create + changed_when: "'changed' in linstor_cluster_node_create.stdout" + loop: "{{ groups['proxmox'] }}" + when: inventory_hostname == groups['proxmox'][0] + +- name: Configure Multipath Fallback Interfaces + ansible.builtin.shell: | + set -o pipefail + if ! linstor node interface list "{{ item }}" | grep -q " fallback "; then + linstor node interface create "{{ item }}" fallback "{{ hostvars[item].ansible_host }}" > /dev/null + echo "changed" + fi + args: + executable: /bin/bash + register: linstor_cluster_int_create + changed_when: "'changed' in linstor_cluster_int_create.stdout" + loop: "{{ groups['proxmox'] }}" + when: inventory_hostname == groups['proxmox'][0] diff --git a/ansible/roles/linstor_cluster/templates/linstor-client.conf.j2 b/ansible/roles/linstor_cluster/templates/linstor-client.conf.j2 new file mode 100644 index 0000000..c1d1316 --- /dev/null +++ b/ansible/roles/linstor_cluster/templates/linstor-client.conf.j2 @@ -0,0 +1,3 @@ +[global] +# Ansible dynamically grabs all hosts in the 'proxmox' group and joins them with commas +controllers={{ groups['proxmox'] | join(',') }} diff --git a/ansible/roles/linstor_ha/tasks/main.yml b/ansible/roles/linstor_ha/tasks/main.yml new file mode 100644 index 0000000..19be453 --- /dev/null +++ b/ansible/roles/linstor_ha/tasks/main.yml @@ -0,0 +1,134 @@ +--- +# tasks file for linstor_ha + +# --------------------------------------------------------- +# STEP 1: CREATE HA RESOURCES (Executed ONLY on Node 1) +# --------------------------------------------------------- +- name: Create HA Resource Group and Spawn Database Volume + ansible.builtin.shell: | + set -o pipefail + if ! linstor resource-group list | grep -q " linstor-db-grp "; then + linstor resource-group create --storage-pool "{{ ha_pool }}" --place-count 3 --diskless-on-remaining true linstor-db-grp > /dev/null + linstor resource-group drbd-options --auto-promote=no --quorum=majority \ + --on-suspended-primary-outdated=force-secondary --on-no-quorum=io-error --on-no-data-accessible=io-error linstor-db-grp > /dev/null + linstor volume-group create linstor-db-grp > /dev/null + linstor resource-group spawn-resources linstor-db-grp linstor_db 200M > /dev/null + echo "changed" + fi + args: + executable: /bin/bash + register: linstor_ha_rg_create + changed_when: "'changed' in linstor_ha_rg_create.stdout" + when: inventory_hostname == groups['proxmox'][0] + +- name: Wait for DRBD block device to appear locally + ansible.builtin.wait_for: + path: /dev/drbd/by-res/linstor_db/0 + state: present + timeout: 60 + when: inventory_hostname == groups['proxmox'][0] + +# --------------------------------------------------------- +# STEP 2: PREPARE ALL NODES +# --------------------------------------------------------- +- name: Install resource-agents + ansible.builtin.apt: + name: resource-agents + state: present + +- name: Ensure local Linstor Controller is stopped and disabled + ansible.builtin.systemd: + name: linstor-controller + enabled: false + state: stopped + +- name: Deploy var-lib-linstor.mount unit + ansible.builtin.copy: + dest: /etc/systemd/system/var-lib-linstor.mount + content: | + [Unit] + Description=Filesystem for the LINSTOR controller + + [Mount] + What=/dev/drbd/by-res/linstor_db/0 + Where=/var/lib/linstor + mode: '0644' + +- name: Ensure drbd-reactor directory exists + ansible.builtin.file: + path: /etc/drbd-reactor.d + state: directory + mode: '0755' + +- name: Deploy DRBD Reactor TOML config + ansible.builtin.template: + src: linstor_db.toml.j2 + dest: /etc/drbd-reactor.d/linstor_db.toml + mode: '0644' + +# Notice how the stat task is gone, replaced by 'creates' and 'removes' +- name: Move existing local database to .orig safely + ansible.builtin.command: mv /var/lib/linstor /var/lib/linstor.orig + args: + creates: /var/lib/linstor.orig + removes: /var/lib/linstor + +- name: Create empty mount point and set immutable flag (+i) + ansible.builtin.file: + path: /var/lib/linstor + state: directory + mode: '0755' + attributes: '+i' + +- name: Ensure Satellite Override directory exists + ansible.builtin.file: + path: /etc/systemd/system/linstor-satellite.service.d + state: directory + mode: '0755' + +- name: Deploy Satellite Keep-Resource Override + ansible.builtin.copy: + dest: /etc/systemd/system/linstor-satellite.service.d/override.conf + content: | + [Service] + Environment=LS_KEEP_RES=linstor_db + mode: '0644' + +- name: Reload daemon and restart Satellite + ansible.builtin.systemd: + daemon_reload: true + name: linstor-satellite + state: restarted + +# --------------------------------------------------------- +# STEP 3: FORMAT & MIGRATE DB (Executed ONLY on Node 1) +# --------------------------------------------------------- +- name: Format HA Volume and Migrate Database + ansible.builtin.shell: | + set -o pipefail + if ! blkid /dev/drbd/by-res/linstor_db/0 | grep -q "ext4"; then + drbdadm primary linstor_db + mkfs.ext4 /dev/drbd/by-res/linstor_db/0 > /dev/null + + chattr -i /var/lib/linstor + mount /dev/drbd/by-res/linstor_db/0 /var/lib/linstor + cp -a /var/lib/linstor.orig/* /var/lib/linstor/ + umount /var/lib/linstor + chattr +i /var/lib/linstor + drbdadm secondary linstor_db + echo "changed" + fi + args: + executable: /bin/bash + register: linstor_ha_migrate + changed_when: "'changed' in linstor_ha_migrate.stdout" + when: inventory_hostname == groups['proxmox'][0] + +# --------------------------------------------------------- +# STEP 4: ACTIVATE HA REACTOR +# --------------------------------------------------------- +- name: Enable and start DRBD Reactor + ansible.builtin.systemd: + name: drbd-reactor + enabled: true + state: started diff --git a/ansible/roles/linstor_ha/templates/linstor_db.toml.j2 b/ansible/roles/linstor_ha/templates/linstor_db.toml.j2 new file mode 100644 index 0000000..7c4ff6c --- /dev/null +++ b/ansible/roles/linstor_ha/templates/linstor_db.toml.j2 @@ -0,0 +1,9 @@ +[[promoter]] +[promoter.resources.linstor_db] +start = [ + "ocf:heartbeat:IPaddr2 service_ip cidr_netmask={{ ha_vip_cidr }} ip={{ ha_vip }}", + "var-lib-linstor.mount", + "linstor-controller.service" +] +on-drbd-demote-failure = "reboot" +secondary-force = true \ No newline at end of file diff --git a/ansible/roles/linstor_install/tasks/main.yml b/ansible/roles/linstor_install/tasks/main.yml new file mode 100644 index 0000000..05475b2 --- /dev/null +++ b/ansible/roles/linstor_install/tasks/main.yml @@ -0,0 +1,63 @@ +--- +# tasks file for linstor_install + +- name: Install Proxmox kernel headers + ansible.builtin.apt: + name: + - "proxmox-headers-{{ ansible_facts['kernel'] }}" + - proxmox-default-headers + state: present + update_cache: true + +- name: Install LINSTOR, DRBD, and LVM dependencies + ansible.builtin.apt: + name: + - drbd-dkms + - drbd-utils + - lvm2 + - thin-provisioning-tools + - linstor-satellite + - linstor-proxmox + - linstor-controller + - linstor-client + - drbd-reactor + state: present + +- name: Check currently loaded DRBD version + ansible.builtin.shell: cat /proc/drbd || true + register: linstor_install_drbd_version + changed_when: false + +- name: Eject legacy DRBD 8.4 module if present + community.general.modprobe: + name: drbd + state: absent + when: "'version: 8' in linstor_install_drbd_version.stdout" + +- name: Ensure DRBD kernel module is loaded + community.general.modprobe: + name: drbd + state: present + +- name: Enable and start base services (Satellite & Reactor) + ansible.builtin.systemd: + name: "{{ item }}" + enabled: true + state: started + loop: + - linstor-satellite + - drbd-reactor + +- name: Ensure LINSTOR Controller is disabled safely on all nodes + ansible.builtin.systemd: + name: linstor-controller + enabled: false + state: stopped + +- name: Enable and start LINSTOR Controller strictly on Node 1 + ansible.builtin.systemd: + name: linstor-controller + enabled: true + state: started + # Ansible Magic: This dynamically targets the very first node listed in your inventory under the 'proxmox' group! + when: inventory_hostname == groups['proxmox'][0] diff --git a/ansible/roles/linstor_repo/tasks/main.yml b/ansible/roles/linstor_repo/tasks/main.yml new file mode 100644 index 0000000..294da69 --- /dev/null +++ b/ansible/roles/linstor_repo/tasks/main.yml @@ -0,0 +1,24 @@ +--- +# tasks file for linstor_repo + +- name: Download LINBIT keyring package + ansible.builtin.get_url: + url: https://packages.linbit.com/public/linbit-keyring.deb + dest: /tmp/linbit-keyring.deb + mode: '0644' + +- name: Install LINBIT keyring + ansible.builtin.apt: + deb: /tmp/linbit-keyring.deb + state: present + +- name: Add LINBIT APT repository for Proxmox + ansible.builtin.apt_repository: + repo: >- + deb [signed-by=/etc/apt/trusted.gpg.d/linbit-keyring.gpg] + https://packages.linbit.com/public/ + proxmox-{{ ansible_facts['distribution_major_version'] | int - 4 }} + drbd-9 + filename: linbit + state: present + update_cache: true diff --git a/ansible/roles/linstor_storage/tasks/main.yml b/ansible/roles/linstor_storage/tasks/main.yml new file mode 100644 index 0000000..4ccb4f8 --- /dev/null +++ b/ansible/roles/linstor_storage/tasks/main.yml @@ -0,0 +1,69 @@ +--- +# tasks file for linstor_storage + +- name: Configure LVM global_filter for DRBD safely + ansible.builtin.lineinfile: + path: /etc/lvm/lvm.conf + insertafter: '^[ \t]*devices[ \t]*\{' + # This regex ensures we only ever have ONE DRBD filter line, replacing the old one if it exists + regexp: '^[ \t]*global_filter[ \t]*=.*r\|\^/dev/drbd\|' + line: ' global_filter = [ "r|^/dev/drbd|", "r|^/dev/mapper/[lL]instor|" ]' + state: present + +- name: Ensure Volume Groups exist for raw disks + community.general.lvg: + vg: "{{ item.vg_name }}" + pvs: "{{ item.targets[inventory_hostname] }}" + loop: "{{ linstor_storage_pools }}" + # Jinja2 magic: Only run this LVM task if the target string starts with '/dev/' + when: item.targets[inventory_hostname].startswith('/dev/') + +- name: Ensure LVM Thin Pools exist for raw disks + community.general.lvol: + vg: "{{ item.vg_name }}" + lv: "{{ item.thin_name }}" + size: "100%FREE" + opts: "-T -Zn" + shrink: false + loop: "{{ linstor_storage_pools }}" + when: item.targets[inventory_hostname].startswith('/dev/') + +# --------------------------------------------------------- +# LINSTOR STORAGE REGISTRATION (Executed ONLY on Node 1) +# --------------------------------------------------------- + +- name: Register Storage Pools in LINSTOR + ansible.builtin.shell: | + set -o pipefail + + # We use Jinja2 to write a custom bash loop for exactly the nodes in our inventory + {% for host in groups['proxmox'] %} + {% set target = item.targets[host] %} + {% set lvm_path = (item.vg_name + '/' + item.thin_name) if target.startswith('/dev/') else target %} + + if ! linstor storage-pool list | grep -q " {{ host }} .* {{ item.pool_name }} "; then + linstor storage-pool create lvmthin "{{ host }}" "{{ item.pool_name }}" "{{ lvm_path }}" > /dev/null + echo "changed" + fi + {% endfor %} + args: + executable: /bin/bash + register: linstor_storage_pool_create + changed_when: "'changed' in linstor_storage_pool_create.stdout" + loop: "{{ linstor_storage_pools }}" + when: inventory_hostname == groups['proxmox'][0] + +- name: Create Resource Groups and Volume Groups + ansible.builtin.shell: | + set -o pipefail + if ! linstor resource-group list | grep -q " {{ item.rg_name }} "; then + linstor resource-group create "{{ item.rg_name }}" --storage-pool "{{ item.pool_name }}" --place-count 3 > /dev/null + linstor volume-group create "{{ item.rg_name }}" > /dev/null + echo "changed" + fi + args: + executable: /bin/bash + register: linstor_storage_rg_create + changed_when: "'changed' in linstor_storage_rg_create.stdout" + loop: "{{ linstor_storage_pools }}" + when: inventory_hostname == groups['proxmox'][0] diff --git a/ansible/roles/proxmox_storage/tasks/main.yml b/ansible/roles/proxmox_storage/tasks/main.yml new file mode 100644 index 0000000..33ce2f8 --- /dev/null +++ b/ansible/roles/proxmox_storage/tasks/main.yml @@ -0,0 +1,29 @@ +--- +# tasks file for proxmox_storage + +- name: Create a safety backup of Proxmox storage.cfg + ansible.builtin.copy: + src: /etc/pve/storage.cfg + dest: "/root/pve_storage.cfg.bak.{{ ansible_facts['date_time']['epoch'] }}" + remote_src: true + mode: '0600' + when: inventory_hostname == groups['proxmox'][0] + +- name: Register LINSTOR resource groups in Proxmox GUI (pvesm) + ansible.builtin.shell: | + set -o pipefail + + # We use the ha_vip variable, but gracefully fallback to Node 1's public IP if it's undefined + CONTROLLER="{{ ha_vip | default(hostvars[groups['proxmox'][0]].ansible_host) }}" + STORAGE_ID="ls-{{ item.rg_name }}" + + if ! pvesm status | grep -q "^${STORAGE_ID} "; then + pvesm add drbd "${STORAGE_ID}" --controller "${CONTROLLER}" --resourcegroup "{{ item.rg_name }}" --content images,rootdir > /dev/null + echo "changed" + fi + args: + executable: /bin/bash + register: proxmox_storage_pvesm_add + changed_when: "'changed' in proxmox_storage_pvesm_add.stdout" + loop: "{{ linstor_storage_pools }}" + when: inventory_hostname == groups['proxmox'][0] diff --git a/docs/01-prerequisites.md b/docs/01-prerequisites.md new file mode 100644 index 0000000..8df27ed --- /dev/null +++ b/docs/01-prerequisites.md @@ -0,0 +1,21 @@ +# 01 - Prerequisites & Node Preparation + +This documentation outlines the required state for bare-metal servers before executing the Ansible provisioning pipeline. + +## Node State +* **OS:** Proxmox VE 8.x freshly installed. +* **Authentication:** Root password known. SSH key exchange is handled automatically by Ansible. +* **Network:** Static IPs assigned. If deploying Linstor, the management interface and Corosync/DRBD replication should ideally be physically separated. If utilizing a single NIC, QoS/traffic shaping must be handled at the switch level to prioritize Corosync UDP (5404/5405) over DRBD replication traffic. +* **Storage:** Target block devices intended for Linstor raw LVM provisioning must contain no existing partition tables, LVM metadata, or filesystems. + +## Execution Environment +Local Ansible installation is not required. This repository enforces the use of an isolated VS Code DevContainer. + +1. Open this repository in Visual Studio Code. +2. Ensure Docker/Podman and the "Dev Containers" extension are installed. +3. Select `Reopen in Container`. +4. Open a terminal within VS Code. All paths referenced in this documentation originate from the `ansible/` directory. + +```bash +cd ansible/ +``` \ No newline at end of file diff --git a/docs/02-configuration.md b/docs/02-configuration.md new file mode 100644 index 0000000..0d82e1e --- /dev/null +++ b/docs/02-configuration.md @@ -0,0 +1,34 @@ +# 02 - Configuring the Cluster + +Cluster configurations are segmented per project/customer using Ansible inventories. Do not edit variables in the root directory. + +## 1. Create the Environment +Copy the template directory for your specific project: + +```bash +cp -r inventories/_template inventories/ +mv inventories//hosts.yml.example inventories//hosts.yml +mv inventories//group_vars/all.yml.example inventories//group_vars/all.yml +``` + +## 2. Configure `hosts.yml` +Define the target PVE nodes. Ansible defaults to the `ansible_host` IP for Corosync and DRBD replication. If a dedicated backend network is available, define `priv_ip`. Ansible uses conditional fallback logic automatically. + +```yaml +all: + children: + proxmox: + hosts: + pve-node-01: + ansible_host: 192.168.1.11 + priv_ip: 10.0.0.11 +``` + +## 3. Configure `group_vars/all.yml` + +* **Deployment Toggles:** Set `deploy_linstor: true` for HCI setups, or `false` for traditional SAN integration. +* **Enterprise Licensing:** Supply a key in `pve_enterprise_key`. If empty (`""`), the playbook applies the `pve-no-subscription` repository. +* **HA Controller:** Define `ha_pool`, `ha_vip` (Virtual IP), and the CIDR prefix for the Linstor DB promoter. +* **Storage Pools:** + * If mapping a raw disk (`/dev/sda`), the playbook invokes `community.general.lvg` to create the VG/Thin Pool. + * If mapping an existing path (`pve/data`), LVM tasks are skipped, and the target is registered directly in Linstor. diff --git a/docs/03-execution.md b/docs/03-execution.md new file mode 100644 index 0000000..8ecba50 --- /dev/null +++ b/docs/03-execution.md @@ -0,0 +1,21 @@ +# 03 - Execution + +Ensure you are operating within the `ansible/` directory inside the DevContainer. + +## 1. Test Connectivity +Verify SSH access to the nodes before triggering the pipeline. Because SSH keys are not yet distributed, Ansible must prompt for the root password (`-k`). + +```bash +ansible all -i inventories//hosts.yml -m ping -u root -k +``` + +## 2. Execute the Playbook +Run the converged deployment playbook. You will be prompted for the SSH password once. + +```bash +ansible-playbook -i inventories//hosts.yml deploy.yml -u root -k +``` + +## Expected Behavior +* **Idempotency:** The playbook is declarative. If execution fails due to a transient network issue, re-running the exact command will bypass successful tasks and resume from the failure point. +* **Reboots:** If Phase 2 upgrades the running PVE kernel, the playbook forces a synchronized node reboot to ensure DRBD module compatibility against the active kernel headers. Ansible will pause and wait for the nodes to return online. diff --git a/docs/04-architecture.md b/docs/04-architecture.md new file mode 100644 index 0000000..9d16b24 --- /dev/null +++ b/docs/04-architecture.md @@ -0,0 +1,15 @@ +# 04 - Pipeline Architecture + +The `deploy.yml` playbook orchestrates the initialization via sequential phases utilizing `include_role` blocks to guarantee procedural execution order. + +## Phase 1: Repositories & Licensing +Evaluates the `pve_enterprise_key` variable. Modifies `/etc/apt/sources.list.d/`, drops the ceph/enterprise configurations as needed, and applies Linstor repository keys. + +## Phase 2: System Updates & JS Patches +Forces a `dist-upgrade`. Implements a regex replacement targeting `proxmoxlib.js` to nullify the ExtJS subscription prompt. Installs required DRBD headers and modules. Issues a conditional reboot. + +## Phase 3: Proxmox Clustering +Generates 4096-bit RSA keys, constructs a full-mesh authorized_keys trust, and nullifies StrictHostKeyChecking for the defined subnets. Bootstraps Corosync via `pvecm create` on Node 01, followed by asynchronous `pvecm add` operations on worker nodes. + +## Phase 4: Linstor/DRBD Configuration (Conditional) +Wrapped in a feature flag (`deploy_linstor`). Executes standard DRBD module loading, configures global LVM filters, sets up High Availability components, and injects backend parameters into `/etc/pve/storage.cfg`. diff --git a/docs/05-linstor-storage.md b/docs/05-linstor-storage.md new file mode 100644 index 0000000..3274bd3 --- /dev/null +++ b/docs/05-linstor-storage.md @@ -0,0 +1,18 @@ +# 05 - Linstor & Storage Implementation + +This document details the configuration enforcement executed by the Linstor roles. + +## LVM Global Filter +To prevent PVE from detecting double LVM signatures (raw block vs DRBD layer), the playbook injects the following global filter into `/etc/lvm/lvm.conf`. +`global_filter =[ "r|^/dev/drbd|", "r|^/dev/mapper/[lL]instor|" ]` + +## Storage Provisioning +The `community.general.lvg` task targets raw block devices strictly when the target string starts with `/dev/`. **The playbook does not invoke `wipefs`.** If residual partition tables exist on the target block device, the LVM module will purposefully fail to prevent data destruction. + +## HA Controller Topology +Linstor Controller HA is handled via `drbd-reactor`. +1. A 200M DRBD resource (`linstor_db`) is spawned across all nodes. +2. The default `linstor-controller` systemd service is disabled. +3. A `var-lib-linstor.mount` unit is deployed. +4. The local `/var/lib/linstor` mount point is secured with the immutable flag (`chattr +i`). This guarantees that if the DRBD resource fails to mount, the local filesystem rejects writes, preventing database split-brains. +5. `drbd-reactor` manages the floating VIP and promoter plugin logic. diff --git a/docs/06-troubleshooting.md b/docs/06-troubleshooting.md new file mode 100644 index 0000000..3dcb146 --- /dev/null +++ b/docs/06-troubleshooting.md @@ -0,0 +1,39 @@ +# 06 - Troubleshooting + +Common interventions for pipeline failures or hardware reuse. + +## LVM Provisioning Failures (Dirty Disks) +If Ansible halts at the LVM Volume Group creation phase, the target block device likely retains partition data. Ensure you are targeting the correct device and wipe the signatures manually: + +```bash +# Execute on the specific Proxmox node +wipefs -a /dev/sdX +``` +Rerun the playbook after clearing the disk. + +## Corosync Quorum Loss +If execution hangs on `pvecm status` or clustering commands, Corosync has lost quorum. In a 3-node cluster, 2 nodes must be online. To force a single active node to regain quorum and unblock operations (for recovery only): + +```bash +pvecm expected 1 +``` + +## Linstor HA VIP Unreachable +If the controller does not initialize, verify the promoter plugin state: +```bash +systemctl status drbd-reactor +journalctl -u drbd-reactor -n 50 +``` +Verify switch layer allows IP spoofing / MAC address floating for the defined `ha_vip`. + +## Complete Cluster Reset +To obliterate Corosync configuration and reset the node to a standalone state: +```bash +systemctl stop pve-cluster +pmxcfs -l +rm /etc/pve/corosync.conf +rm -rf /etc/corosync/* +killall pmxcfs +systemctl start pve-cluster +``` +*Note: If DRBD configurations are active, an OS reinstall is significantly faster and safer than manual DRBD block device teardown.*