diff --git a/ansible/ansible.cfg b/ansible/ansible.cfg new file mode 100644 index 0000000..bea26a3 --- /dev/null +++ b/ansible/ansible.cfg @@ -0,0 +1,15 @@ +[defaults] +remote_user = root +forks = 20 +pipelining = True +result_format = yaml +bin_ansible_callbacks = True +callback_whitelist = profile_tasks + +# SSH connection timeout (opening the connection) +timeout = 120 + +[ssh_connection] +ssh_args = -o ServerAliveInterval=60 -o ServerAliveCountMax=10 -o ControlMaster=auto -o ControlPersist=60s +# Timeout for individual commands on remote host (apt upgrade, DRBD, etc.) +command_timeout = 3600 diff --git a/ansible/deploy.yml b/ansible/deploy.yml index 8b58290..db92240 100644 --- a/ansible/deploy.yml +++ b/ansible/deploy.yml @@ -8,16 +8,44 @@ vars: # Dynamically detect if we are in Single-Node or Cluster mode is_single_node: "{{ groups['proxmox'] | length == 1 }}" - - # Existing clustering variables + # standalone_mode can be set manually in group_vars to bypass cluster/linstor tasks + # even with multiple nodes in inventory (e.g. independent nodes for the same customer) + _skip_cluster: "{{ (is_single_node | bool) or (standalone_mode | default(false) | bool) }}" + + # Corosync clustering IPs corosync_ip: "{{ priv_ip | default(ansible_host) }}" master_corosync_ip: "{{ hostvars[groups['proxmox'][0]]['priv_ip'] | default(hostvars[groups['proxmox'][0]]['ansible_host']) }}" is_master: "{{ inventory_hostname == groups['proxmox'][0] }}" tasks: + # ========================================== + # PHASE 0: P2P PRIVATE NETWORK + # ========================================== + - name: Configure P2P Private Network Bridge + ansible.builtin.include_role: + name: p2p_network + when: + - not _skip_cluster | bool + - priv_nic_1 is defined + - priv_nic_2 is defined + - priv_ip is defined + # ========================================== # PHASE 1: REPOSITORIES & LICENSING (PVE 9 / Trixie) # ========================================== + - name: Disable IPv6 system-wide + ansible.posix.sysctl: + name: "{{ item }}" + value: "1" + state: present + sysctl_file: /etc/sysctl.d/99-disable-ipv6.conf + reload: true + loop: + - net.ipv6.conf.all.disable_ipv6 + - net.ipv6.conf.default.disable_ipv6 + - net.ipv6.conf.lo.disable_ipv6 + when: disable_ipv6 | default(false) | bool + - name: Apply Proxmox Enterprise Key ansible.builtin.command: pvesubscription set -k {{ pve_enterprise_key }} when: pve_enterprise_key | default('') | length > 0 @@ -53,27 +81,36 @@ signed_by: /usr/share/keyrings/proxmox-archive-keyring.gpg state: "{{ 'absent' if pve_enterprise_key | default('') | length > 0 else 'present' }}" - - name: Remove Ceph Enterprise Repository (Unused in Linstor) + - name: Remove Ceph Enterprise Repository (Unused in Linstor setups) ansible.builtin.file: path: /etc/apt/sources.list.d/ceph.sources state: absent when: pve_enterprise_key | default('') | length == 0 - - name: Add Linstor Repositories (Skipped in Standalone Node) + - name: Add Linstor Repositories ansible.builtin.include_role: name: linstor_repo - when: + when: - deploy_linstor | bool - - not is_standalone | bool + - not _skip_cluster | bool # ========================================== # PHASE 2: UPDATES & NAG REMOVAL # ========================================== + + # NOTE: async/poll used here to prevent SSH timeout during long dist-upgrade. + # The task is submitted asynchronously (up to 1h) and polled every 30s. - name: Update APT cache and run dist-upgrade ansible.builtin.apt: - update_cache: yes + update_cache: true upgrade: dist - autoremove: yes + autoremove: true + lock_timeout: 300 # Wait up to 5 min for dpkg lock from other apt processes + dpkg_options: 'force-confnew,force-confdef' # Non-interactive config file handling + environment: + DEBIAN_FRONTEND: noninteractive # Prevent debconf from blocking on prompts + async: 3600 # Allow up to 1 hour for the full upgrade + poll: 30 # Check progress every 30 seconds - name: Remove Proxmox Subscription Nag (JS Patch) ansible.builtin.replace: @@ -82,12 +119,12 @@ replace: "void({ //\\1" notify: Restart pveproxy - - name: Install Linstor Packages (Skipped in Single Node) + - name: Install Linstor Packages ansible.builtin.include_role: name: linstor_install - when: + when: - deploy_linstor | bool - - not is_single_node | bool + - not _skip_cluster | bool - name: Check if a reboot is required by APT ansible.builtin.stat: @@ -97,19 +134,30 @@ - name: Reboot the node gracefully ansible.builtin.reboot: msg: "Rebooting node to apply new kernel/system updates" - reboot_timeout: 600 # Wait up to 10 minutes for the server to return + reboot_timeout: 600 when: reboot_required_file.stat.exists + # Explicit barrier: ensure ALL nodes are back online before proceeding to Phase 3. + # Without this, any_errors_fatal would abort if one node reboots and another doesn't, + # causing them to arrive at Phase 3 at different times. + - name: Wait for all nodes to be reachable after (optional) reboot + ansible.builtin.wait_for_connection: + timeout: 300 + sleep: 5 + + - name: Sync barrier — wait for all nodes before clustering phase + ansible.builtin.meta: clear_host_errors + # ========================================== # PHASE 3: PROXMOX CLUSTERING # ========================================== - - name: PROXMOX CLUSTERING (Skipped in Single Node Mode) - when: not is_single_node | bool + - name: PROXMOX CLUSTERING (Skipped in Single Node / Standalone Mode) + when: not _skip_cluster | bool block: - name: Ensure SSH keypair exists for root ansible.builtin.user: name: root - generate_ssh_key: yes + generate_ssh_key: true ssh_key_bits: 4096 - name: Fetch public SSH keys from all nodes @@ -124,51 +172,63 @@ state: present loop: "{{ ansible_play_hosts }}" + - name: Compute SSH wildcard host patterns for cluster subnets + ansible.builtin.set_fact: + _corosync_subnet: "{{ master_corosync_ip | regex_replace('\\.[0-9]+$', '.*') }}" + _mgmt_subnet: >- + {{ hostvars[groups['proxmox'][0]]['ansible_host'] + | regex_replace('\\.[0-9]+$', '.*') }} + - name: Automatically accept host keys for cluster subnets ansible.builtin.blockinfile: path: /root/.ssh/config - create: yes + create: true mode: '0600' block: | - Host {{ master_corosync_ip | regex_replace('\.[0-9]+$', '.*') }} {{ hostvars[groups['proxmox'][0]]['ansible_host'] | regex_replace('\.[0-9]+$', '.*') }} + Host {{ _corosync_subnet }} {{ _mgmt_subnet }} StrictHostKeyChecking no UserKnownHostsFile=/dev/null - name: Check current Proxmox cluster status ansible.builtin.command: pvecm status register: pvecm_status - ignore_errors: yes + ignore_errors: true changed_when: false - name: Initialize the Proxmox Cluster (Master Node) ansible.builtin.command: pvecm create {{ pve_cluster_name }} --link0 {{ corosync_ip }} - when: + changed_when: true + when: - pvecm_status.rc != 0 - is_master | bool - name: Wait for Corosync to stabilize ansible.builtin.pause: seconds: 10 - when: + when: - pvecm_status.rc != 0 - is_master | bool - - name: Join the Proxmox Cluster (Worker Nodes) + - name: Join the Proxmox Cluster (Worker Nodes — one at a time) ansible.builtin.command: > pvecm add {{ master_corosync_ip }} --link0 {{ corosync_ip }} --use_ssh yes - when: + register: pvecm_join + changed_when: true + failed_when: pvecm_join.rc != 0 or 'TASK ERROR' in pvecm_join.stdout or 'TASK ERROR' in pvecm_join.stderr + when: - pvecm_status.rc != 0 - not is_master | bool - async: 60 + throttle: 1 # Prevent race condition: pvecm add does not support parallel joins + async: 120 poll: 10 # ========================================== # PHASE 4: LINSTOR CONFIGURATION # ========================================== - - name: LINSTOR CONFIGURATION (Skipped if deploy_linstor=false or Single Node) - when: + - name: LINSTOR CONFIGURATION (Skipped if deploy_linstor=false or Single Node / Standalone) + when: - deploy_linstor | bool - - not is_single_node | bool + - not _skip_cluster | bool block: - name: Configure Linstor Cluster ansible.builtin.include_role: diff --git a/ansible/inventories/_template/group_vars/all.yml b/ansible/inventories/_template/group_vars/all.yml new file mode 100644 index 0000000..7faefcc --- /dev/null +++ b/ansible/inventories/_template/group_vars/all.yml @@ -0,0 +1,70 @@ +--- +# ============================================================ +# Proxmox Post-Install - Group Variables Template +# Copy this to your inventory folder as group_vars/all.yml +# ============================================================ + +# --- Deployment Toggles --- +# standalone_mode: Set to true to skip ALL cluster and Linstor tasks, +# even when the inventory contains multiple nodes (useful for independent nodes per customer). +standalone_mode: false + +# deploy_linstor: Set to true to install and configure Linstor + DRBD storage. +# Requires at least 3 nodes and standalone_mode: false. +deploy_linstor: false + +# disable_ipv6: Set to true to disable IPv6 system-wide via sysctl. +# Useful when IPv6 is unrouted and causes connection timeouts (e.g. apt, curl). +disable_ipv6: false + +# wipe_linstor_disks: Set to true to wipe partition tables and signatures from raw block devices +# before creating LVM VGs. Only affects targets defined as /dev/... paths. +# WARNING: destructive — only set on first deployment, never on a running cluster. +wipe_linstor_disks: false + +# --- Proxmox Cluster --- +pve_cluster_name: "example-cluster" +# pve_enterprise_key: "" # Optional: global key (prefer per-host key in hosts.ini) + +# --- Linstor / LINBIT Repository --- +# Auto-calculated as: Debian major version - 4 (e.g. Debian 13 = proxmox-9). +# Override manually if LINBIT publishes a differently-named repo for new Proxmox versions. +# linbit_proxmox_version: 9 + +# --- Linstor HA Controller --- +# ha_pool: Name of the Linstor storage pool to use for the controller HA volume. +ha_pool: "fast_pool_1" +# ha_vip: Virtual IP that follows the active Linstor controller across nodes. +ha_vip: "192.168.2.4" +# ha_vip_cidr: Prefix length for the VIP address (e.g. 24 for /24). +ha_vip_cidr: "24" + +# --- Linstor Storage Pools --- +# Each entry defines one Resource Group, composed of one storage pool per node. +# All pools in the same entry share the same pool_name across nodes. +# +# TARGET RULES: +# - Use a raw block device path (/dev/sdb) to auto-create VG + LVM Thin Pool. +# - Use an existing thin-pool path (vg_name/thin_name, e.g. "pve/data") to register +# an already-existing LVM thin pool WITHOUT touching existing data. +# +# Keys under "targets" must match the Ansible inventory_hostname of each node +# (i.e. the left-hand label in hosts.ini, e.g. "pve-node-01"). +linstor_storage_pools: + - pool_name: "fast_pool_1" + rg_name: "fast_pool_1" + vg_name: "pve" + thin_name: "data" + targets: + pve-node-01: "pve/data" # Register existing pve/data thin pool + pve-node-02: "pve/data" + pve-node-03: "pve/data" + + - pool_name: "slow_pool_1" + rg_name: "slow_pool_1" + vg_name: "sp1" + thin_name: "data" + targets: + pve-node-01: "/dev/sdb" # Format /dev/sdb as VG sp1 + thin pool data + pve-node-02: "/dev/sdb" + pve-node-03: "/dev/sdb" diff --git a/ansible/inventories/_template/group_vars/all.yml.example b/ansible/inventories/_template/group_vars/all.yml.example deleted file mode 100644 index 76f37ea..0000000 --- a/ansible/inventories/_template/group_vars/all.yml.example +++ /dev/null @@ -1,37 +0,0 @@ ---- -# Deployment Toggles -standalone_mode: true # Set to true to bypass all Corosync/Linstor tasks for independent nodes -deploy_linstor: false # Set to true to install and configure Linstor/DRBD - -# Proxmox Cluster Configuration -pve_cluster_name: "example-cluster" - -# High Availability Controller Configuration -ha_pool: "fast_pool_1" -ha_vip: "192.168.2.4" -ha_vip_cidr: "29" - -# Linstor Storage Pools Configuration -# You can add or remove pools as needed - -# TARGET RULES: -# Use raw disk (/dev/sdb) to wipe/create LVM -# Use existing thin-pool path (pve/data) to register safely (data won't be deleted) -linstor_storage_pools: - - pool_name: "fast_pool_1" - rg_name: "fast_pool_1" - vg_name: "pve" - thin_name: "data" - targets: - pve-node-01: "pve/data" - pve-node-02: "pve/data" - pve-node-03: "pve/data" - - - pool_name: "slow_pool_1" - rg_name: "slow_pool_1" - vg_name: "sp1" - thin_name: "data" - targets: - pve-node-01: "/dev/sda" - pve-node-02: "/dev/sda" - pve-node-03: "/dev/sda" diff --git a/ansible/inventories/_template/hosts.ini b/ansible/inventories/_template/hosts.ini new file mode 100644 index 0000000..4be12f1 --- /dev/null +++ b/ansible/inventories/_template/hosts.ini @@ -0,0 +1,25 @@ +# ============================================================ +# Proxmox Cluster Inventory - INI Format +# Copy this file to your inventory folder as "hosts.ini" +# and adjust IPs, hostnames, and optional keys. +# ============================================================ + +[proxmox] +# Format: ansible_host= [priv_ip=] [priv_nic_1=] [priv_nic_2=] [pve_enterprise_key=] +# +# - inventory_alias : Arbitrary label used by Ansible. Does NOT need to match the real hostname. +# The real system hostname (ansible_hostname) is used for Proxmox & Linstor nodes. +# - ansible_host : IP used by Ansible to SSH into the node (management/public IP). +# - priv_ip : IP used for Corosync cluster traffic and Linstor replication (private/dedicated NIC). +# If omitted, ansible_host is used as fallback. +# - priv_nic_1/2 : (Optional) The two NICs forming the full-mesh p2p bridge (named "p2p"). +# Required only when running network.yml to configure the private bridge. +# NIC names differ per node — check with: ip link show +# - pve_enterprise_key : (Optional) Per-node Proxmox enterprise subscription key. + +pve-node-01 ansible_host=192.168.1.1 priv_ip=192.168.2.1 priv_nic_1=enp2s0 priv_nic_2=enp3s0 +pve-node-02 ansible_host=192.168.1.2 priv_ip=192.168.2.2 priv_nic_1=enp2s0 priv_nic_2=enp3s0 +pve-node-03 ansible_host=192.168.1.3 priv_ip=192.168.2.3 priv_nic_1=enp2s0 priv_nic_2=enp3s0 + +# Single-node example (no cluster, no Linstor): +# pve-standalone ansible_host=10.0.0.50 diff --git a/ansible/inventories/_template/hosts.yml.example b/ansible/inventories/_template/hosts.yml.example deleted file mode 100644 index 0414bea..0000000 --- a/ansible/inventories/_template/hosts.yml.example +++ /dev/null @@ -1,16 +0,0 @@ -all: - children: - proxmox: - hosts: - pve-node-01: - ansible_host: 192.168.1.1 - priv_ip: 192.168.2.1 - pve_enterprise_key: "pve1c-111111111111" - pve-node-02: - ansible_host: 192.168.1.2 - priv_ip: 192.168.2.1 - pve_enterprise_key: "pve1c-222222222222" - pve-node-03: - ansible_host: 192.168.1.3 - priv_ip: 192.168.2.3 - pve_enterprise_key: "pve1c-333333333333" diff --git a/ansible/network.yml b/ansible/network.yml new file mode 100644 index 0000000..01f6494 --- /dev/null +++ b/ansible/network.yml @@ -0,0 +1,14 @@ +--- +- name: Configure P2P Private Network Bridge + hosts: proxmox + become: true + gather_facts: true + + tasks: + - name: Configure p2p bridge + ansible.builtin.include_role: + name: p2p_network + when: + - priv_nic_1 is defined + - priv_nic_2 is defined + - priv_ip is defined diff --git a/ansible/roles/linstor_cluster/tasks/main.yml b/ansible/roles/linstor_cluster/tasks/main.yml index 9363b24..4575def 100644 --- a/ansible/roles/linstor_cluster/tasks/main.yml +++ b/ansible/roles/linstor_cluster/tasks/main.yml @@ -1,5 +1,8 @@ --- # tasks file for linstor_cluster +# IMPORTANT: All Linstor node registrations use ansible_hostname (the real system hostname) +# to ensure they match Proxmox node names, which are also derived from the system hostname. +# The Ansible inventory_hostname (e.g. "pve1") may differ from the real hostname (e.g. "pve-node-01"). - name: Safely manage LINSTOR cluster nodes in /etc/hosts ansible.builtin.blockinfile: @@ -8,12 +11,12 @@ block: | # Linstor Cluster Nodes (Primary: Private Network) {% for host in groups['proxmox'] %} - {{ hostvars[host].priv_ip }} {{ host }} + {{ hostvars[host]['priv_ip'] | default(hostvars[host]['ansible_host']) }} {{ hostvars[host]['ansible_facts']['hostname'] }} {% endfor %} # Linstor Cluster Nodes (Fallback: Public Network) {% for host in groups['proxmox'] %} - {{ hostvars[host].ansible_host }} {{ host }}-public + {{ hostvars[host]['ansible_host'] }} {{ hostvars[host]['ansible_facts']['hostname'] }}-public {% endfor %} - name: Ensure /etc/linstor directory exists @@ -30,13 +33,17 @@ # --------------------------------------------------------- # LINSTOR NODE REGISTRATION (Executed ONLY on Node 1) +# Uses ansible_hostname (real system hostname) for Linstor node names +# to guarantee compatibility with Proxmox node names. # --------------------------------------------------------- - name: Register nodes in LINSTOR ansible.builtin.shell: | set -o pipefail - if ! linstor node list | grep -q " {{ item }} "; then - linstor node create "{{ item }}" "{{ hostvars[item].priv_ip }}" --node-type Combined > /dev/null + NODE_NAME="{{ hostvars[item]['ansible_facts']['hostname'] }}" + NODE_IP="{{ hostvars[item]['priv_ip'] | default(hostvars[item]['ansible_host']) }}" + if ! linstor node list | grep -q " ${NODE_NAME} "; then + linstor node create "${NODE_NAME}" "${NODE_IP}" --node-type Combined > /dev/null echo "changed" fi args: @@ -49,8 +56,10 @@ - name: Configure Multipath Fallback Interfaces ansible.builtin.shell: | set -o pipefail - if ! linstor node interface list "{{ item }}" | grep -q " fallback "; then - linstor node interface create "{{ item }}" fallback "{{ hostvars[item].ansible_host }}" > /dev/null + NODE_NAME="{{ hostvars[item]['ansible_facts']['hostname'] }}" + NODE_PUBLIC_IP="{{ hostvars[item]['ansible_host'] }}" + if ! linstor node interface list "${NODE_NAME}" | grep -q " fallback "; then + linstor node interface create "${NODE_NAME}" fallback "${NODE_PUBLIC_IP}" > /dev/null echo "changed" fi args: @@ -59,3 +68,22 @@ changed_when: "'changed' in linstor_cluster_int_create.stdout" loop: "{{ groups['proxmox'] }}" when: inventory_hostname == groups['proxmox'][0] + +- name: Wait for all Linstor satellite nodes to appear as Online + ansible.builtin.shell: | + set -o pipefail + if linstor node list | sed 's/\x1b\[[0-9;]*m//g' | grep " {{ hostvars[item]['ansible_facts']['hostname'] }} " | grep -q "Online"; then + echo "online" + else + echo "not_ready" + exit 1 + fi + args: + executable: /bin/bash + register: linstor_cluster_node_status + until: "'online' in linstor_cluster_node_status.stdout" + retries: 12 + delay: 10 + changed_when: false + loop: "{{ groups['proxmox'] }}" + when: inventory_hostname == groups['proxmox'][0] diff --git a/ansible/roles/linstor_cluster/templates/linstor-client.conf.j2 b/ansible/roles/linstor_cluster/templates/linstor-client.conf.j2 index c1d1316..5ada27e 100644 --- a/ansible/roles/linstor_cluster/templates/linstor-client.conf.j2 +++ b/ansible/roles/linstor_cluster/templates/linstor-client.conf.j2 @@ -1,3 +1,5 @@ [global] -# Ansible dynamically grabs all hosts in the 'proxmox' group and joins them with commas -controllers={{ groups['proxmox'] | join(',') }} +# Controllers list uses private IPs of all nodes (tries each in order). +# After linstor_ha setup, the VIP becomes the active controller endpoint. +# The drbd-reactor automatically manages controller failover via the VIP. +controllers={% for host in groups['proxmox'] %}{{ hostvars[host]['priv_ip'] | default(hostvars[host]['ansible_host']) }}{% if not loop.last %},{% endif %}{% endfor %} diff --git a/ansible/roles/linstor_ha/handlers/main.yml b/ansible/roles/linstor_ha/handlers/main.yml new file mode 100644 index 0000000..9a3259b --- /dev/null +++ b/ansible/roles/linstor_ha/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: Reload drbd-reactor + ansible.builtin.systemd: + name: drbd-reactor + state: reloaded diff --git a/ansible/roles/linstor_ha/tasks/main.yml b/ansible/roles/linstor_ha/tasks/main.yml index 19be453..4c5d391 100644 --- a/ansible/roles/linstor_ha/tasks/main.yml +++ b/ansible/roles/linstor_ha/tasks/main.yml @@ -65,6 +65,7 @@ src: linstor_db.toml.j2 dest: /etc/drbd-reactor.d/linstor_db.toml mode: '0644' + notify: Reload drbd-reactor # Notice how the stat task is gone, replaced by 'creates' and 'removes' - name: Move existing local database to .orig safely diff --git a/ansible/roles/linstor_repo/handlers/main.yml b/ansible/roles/linstor_repo/handlers/main.yml new file mode 100644 index 0000000..5476b97 --- /dev/null +++ b/ansible/roles/linstor_repo/handlers/main.yml @@ -0,0 +1,4 @@ +--- +- name: Update APT cache + ansible.builtin.apt: + update_cache: true diff --git a/ansible/roles/linstor_repo/tasks/main.yml b/ansible/roles/linstor_repo/tasks/main.yml index 294da69..f6594ed 100644 --- a/ansible/roles/linstor_repo/tasks/main.yml +++ b/ansible/roles/linstor_repo/tasks/main.yml @@ -1,24 +1,35 @@ --- # tasks file for linstor_repo +# Target: Proxmox 9 (Debian 13 / Trixie) with DRBD 9 + +# linbit_proxmox_version defaults to: Debian major version minus 4 +# (Debian 13 = PVE 9, Debian 12 = PVE 8, etc.) +# Override this variable in group_vars if LINBIT releases a new repo name. - name: Download LINBIT keyring package ansible.builtin.get_url: url: https://packages.linbit.com/public/linbit-keyring.deb dest: /tmp/linbit-keyring.deb mode: '0644' + check_mode: false - name: Install LINBIT keyring ansible.builtin.apt: deb: /tmp/linbit-keyring.deb state: present -- name: Add LINBIT APT repository for Proxmox - ansible.builtin.apt_repository: - repo: >- - deb [signed-by=/etc/apt/trusted.gpg.d/linbit-keyring.gpg] - https://packages.linbit.com/public/ - proxmox-{{ ansible_facts['distribution_major_version'] | int - 4 }} - drbd-9 - filename: linbit +- name: Remove legacy LINBIT .list file (old APT format, prevents duplicates) + ansible.builtin.file: + path: /etc/apt/sources.list.d/linbit.list + state: absent + +- name: Add LINBIT APT repository for Proxmox (DEB822 format) + ansible.builtin.deb822_repository: + name: linbit + types: deb + uris: https://packages.linbit.com/public/ + suites: "proxmox-{{ linbit_proxmox_version | default(ansible_facts['distribution_major_version'] | int - 4) }}" + components: drbd-9 + signed_by: /etc/apt/trusted.gpg.d/linbit-keyring.gpg state: present - update_cache: true + notify: Update APT cache diff --git a/ansible/roles/linstor_storage/tasks/main.yml b/ansible/roles/linstor_storage/tasks/main.yml index 4ccb4f8..5ff71fe 100644 --- a/ansible/roles/linstor_storage/tasks/main.yml +++ b/ansible/roles/linstor_storage/tasks/main.yml @@ -1,22 +1,66 @@ --- # tasks file for linstor_storage +# NOTE: linstor_storage_pools[].targets keys are Ansible inventory_hostname values. +# All Linstor registrations use ansible_hostname (real system hostname) instead, +# to match Proxmox node names. The mapping is done per-host at registration time. - name: Configure LVM global_filter for DRBD safely ansible.builtin.lineinfile: path: /etc/lvm/lvm.conf insertafter: '^[ \t]*devices[ \t]*\{' - # This regex ensures we only ever have ONE DRBD filter line, replacing the old one if it exists - regexp: '^[ \t]*global_filter[ \t]*=.*r\|\^/dev/drbd\|' + regexp: '^[ \t]*global_filter[ \t]*=' line: ' global_filter = [ "r|^/dev/drbd|", "r|^/dev/mapper/[lL]instor|" ]' state: present +- name: Remove duplicate global_filter entries from lvm.conf + ansible.builtin.shell: | + count=$(grep -c '^[[:space:]]*global_filter[[:space:]]*=' /etc/lvm/lvm.conf) + if [ "$count" -gt 1 ]; then + awk '/^[[:space:]]*global_filter[[:space:]]*=/{if(++n>1)next}1' /etc/lvm/lvm.conf > /tmp/lvm.conf.dedup + mv /tmp/lvm.conf.dedup /etc/lvm/lvm.conf + echo "changed: removed $((count - 1)) duplicate(s)" + fi + args: + executable: /bin/bash + register: lvm_dedup_result + changed_when: "'changed' in lvm_dedup_result.stdout" + +- name: Wipe existing signatures and partition tables from raw disks + ansible.builtin.shell: | + DEV="{{ item.targets[inventory_hostname] }}" + VG="{{ item.vg_name }}" + + # Deactivate VG properly if LVM still knows about it + vgchange -an "$VG" 2>/dev/null || true + + # Forcibly remove any stale dm-mapper devices for this VG (handles the case + # where the VG metadata was already wiped but kernel devices are still active) + dmsetup ls 2>/dev/null \ + | awk -v vg="$VG" 'index($1, vg"-") == 1 { print $1 }' \ + | xargs -r dmsetup remove --force 2>/dev/null || true + + wipefs -a "$DEV" + sgdisk --zap-all "$DEV" + partprobe "$DEV" 2>/dev/null || true + args: + executable: /bin/bash + changed_when: true + loop: "{{ linstor_storage_pools }}" + when: + - wipe_linstor_disks | default(false) | bool + - item.targets[inventory_hostname] is defined + - item.targets[inventory_hostname].startswith('/dev/') + - name: Ensure Volume Groups exist for raw disks community.general.lvg: vg: "{{ item.vg_name }}" pvs: "{{ item.targets[inventory_hostname] }}" + pv_options: "{{ '--force --force' if wipe_linstor_disks | default(false) | bool else '--force' }}" loop: "{{ linstor_storage_pools }}" - # Jinja2 magic: Only run this LVM task if the target string starts with '/dev/' - when: item.targets[inventory_hostname].startswith('/dev/') + # Only run for pools that target this node AND use a raw block device + when: + - item.targets[inventory_hostname] is defined + - item.targets[inventory_hostname].startswith('/dev/') - name: Ensure LVM Thin Pools exist for raw disks community.general.lvol: @@ -26,25 +70,31 @@ opts: "-T -Zn" shrink: false loop: "{{ linstor_storage_pools }}" - when: item.targets[inventory_hostname].startswith('/dev/') + when: + - item.targets[inventory_hostname] is defined + - item.targets[inventory_hostname].startswith('/dev/') # --------------------------------------------------------- # LINSTOR STORAGE REGISTRATION (Executed ONLY on Node 1) +# Uses ansible_hostname (real system hostname) for Linstor node names # --------------------------------------------------------- - name: Register Storage Pools in LINSTOR ansible.builtin.shell: | set -o pipefail - # We use Jinja2 to write a custom bash loop for exactly the nodes in our inventory + # Iterate over all inventory hosts for this pool. + # Use ansible_hostname (real system hostname) as Linstor node name. {% for host in groups['proxmox'] %} + {% if item.targets[host] is defined %} {% set target = item.targets[host] %} {% set lvm_path = (item.vg_name + '/' + item.thin_name) if target.startswith('/dev/') else target %} - - if ! linstor storage-pool list | grep -q " {{ host }} .* {{ item.pool_name }} "; then - linstor storage-pool create lvmthin "{{ host }}" "{{ item.pool_name }}" "{{ lvm_path }}" > /dev/null + NODE_NAME="{{ hostvars[host]['ansible_facts']['hostname'] }}" + if ! linstor storage-pool list | grep -q " ${NODE_NAME} .* {{ item.pool_name }} "; then + linstor storage-pool create lvmthin "${NODE_NAME}" "{{ item.pool_name }}" "{{ lvm_path }}" > /dev/null echo "changed" fi + {% endif %} {% endfor %} args: executable: /bin/bash @@ -57,7 +107,9 @@ ansible.builtin.shell: | set -o pipefail if ! linstor resource-group list | grep -q " {{ item.rg_name }} "; then - linstor resource-group create "{{ item.rg_name }}" --storage-pool "{{ item.pool_name }}" --place-count 3 > /dev/null + linstor resource-group create "{{ item.rg_name }}" \ + --storage-pool "{{ item.pool_name }}" \ + --place-count {{ groups['proxmox'] | length }} > /dev/null linstor volume-group create "{{ item.rg_name }}" > /dev/null echo "changed" fi diff --git a/ansible/roles/p2p_network/handlers/main.yml b/ansible/roles/p2p_network/handlers/main.yml new file mode 100644 index 0000000..ed97d53 --- /dev/null +++ b/ansible/roles/p2p_network/handlers/main.yml @@ -0,0 +1 @@ +--- diff --git a/ansible/roles/p2p_network/tasks/main.yml b/ansible/roles/p2p_network/tasks/main.yml new file mode 100644 index 0000000..b2a73b6 --- /dev/null +++ b/ansible/roles/p2p_network/tasks/main.yml @@ -0,0 +1,28 @@ +--- +- name: Validate p2p NICs exist on this node + ansible.builtin.assert: + that: + - priv_nic_1 in ansible_facts['interfaces'] + - priv_nic_2 in ansible_facts['interfaces'] + fail_msg: >- + One or both p2p NICs not found on {{ inventory_hostname }}. + Expected: {{ priv_nic_1 }}, {{ priv_nic_2 }}. + Available interfaces: {{ ansible_facts['interfaces'] | join(', ') }} + +- name: Configure p2p bridge in /etc/network/interfaces + ansible.builtin.blockinfile: + path: /etc/network/interfaces + marker: "# {mark} ANSIBLE MANAGED BLOCK - p2p bridge" + block: | + auto p2p + iface p2p inet static + address {{ priv_ip }}/{{ ha_vip_cidr }} + bridge-ports {{ priv_nic_1 }} {{ priv_nic_2 }} + bridge-stp on + bridge-fd 0 + register: p2p_network_p2p_bridge_config + +- name: Apply network config # noqa: no-handler + ansible.builtin.command: ifreload -a + changed_when: true + when: p2p_network_p2p_bridge_config is changed