Initial commit
This commit is contained in:
35
.devcontainer/devcontainer.json
Normal file
35
.devcontainer/devcontainer.json
Normal file
@@ -0,0 +1,35 @@
|
|||||||
|
// For format details, see https://aka.ms/devcontainer.json. For config options, see the
|
||||||
|
// README at: https://github.com/devcontainers/templates/tree/main/src/python
|
||||||
|
{
|
||||||
|
"name": "Python 3",
|
||||||
|
// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
|
||||||
|
"image": "mcr.microsoft.com/devcontainers/python:3-3.14-trixie",
|
||||||
|
|
||||||
|
// Features to add to the dev container. More info: https://containers.dev/features.
|
||||||
|
"features": {
|
||||||
|
"ghcr.io/devcontainers-extra/features/ansible": {},
|
||||||
|
"ghcr.io/hspaans/devcontainer-features/ansible-lint": {}
|
||||||
|
},
|
||||||
|
|
||||||
|
// Use 'forwardPorts' to make a list of ports inside the container available locally.
|
||||||
|
// "forwardPorts": [],
|
||||||
|
|
||||||
|
// Use 'postCreateCommand' to run commands after the container is created.
|
||||||
|
// "postCreateCommand": "pip3 install --user -r requirements.txt",
|
||||||
|
|
||||||
|
// Configure tool-specific properties.
|
||||||
|
"customizations": {
|
||||||
|
"vscode": {
|
||||||
|
"settings": {
|
||||||
|
"ansible.python.interpreterPath": "/usr/local/bin/python"
|
||||||
|
},
|
||||||
|
"extensions": [
|
||||||
|
"redhat.ansible",
|
||||||
|
"ms-python.python"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
|
||||||
|
// "remoteUser": "root"
|
||||||
|
}
|
||||||
24
README.md
Normal file
24
README.md
Normal file
@@ -0,0 +1,24 @@
|
|||||||
|
# Proxmox & Linstor HCI Automated Provisioning
|
||||||
|
|
||||||
|
This repository contains an Infrastructure as Code (IaC) pipeline to automatically bootstrap a Highly Available Proxmox Virtual Environment (PVE) cluster, with an optional High Availability Hyperconverged Linstor/DRBD storage stack.
|
||||||
|
|
||||||
|
## 🚀 Quick Start (For Beginners)
|
||||||
|
|
||||||
|
You do not need to install Ansible locally. This repository includes a DevContainer pre-configured with all necessary tools, linters, and extensions.
|
||||||
|
|
||||||
|
1. Install[Visual Studio Code](https://code.visualstudio.com/) and [Docker](https://docs.docker.com/get-docker/) (or [Podman](https://podman.io/)).
|
||||||
|
2. Install the VS Code[Dev Containers Extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers).
|
||||||
|
3. Clone this repository and open the folder in VS Code.
|
||||||
|
4. Click **"Reopen in Container"** when prompted in the bottom right corner (or press `Ctrl+Shift+P` and type `Dev Containers: Reopen in Container`).
|
||||||
|
5. Proceed to the [Documentation](docs/01-prerequisites.md) to configure your cluster variables and run your first deployment.
|
||||||
|
|
||||||
|
## 📚 Documentation
|
||||||
|
|
||||||
|
Detailed documentation is available in the `docs/` directory. If you are new to Ansible or this stack, please read them in order:
|
||||||
|
|
||||||
|
*[01 - Prerequisites & Node Preparation](docs/01-prerequisites.md)
|
||||||
|
*[02 - Configuring the Cluster (Inventory & Vars)](docs/02-configuration.md)
|
||||||
|
* [03 - Running the Playbook](docs/03-execution.md)
|
||||||
|
* [04 - Architecture & Phase Breakdown](docs/04-architecture.md)
|
||||||
|
* [05 - Linstor & Storage Deep Dive](docs/05-linstor-storage.md)
|
||||||
|
* [06 - Troubleshooting & FAQ](docs/06-troubleshooting.md)
|
||||||
153
ansible/deploy.yml
Normal file
153
ansible/deploy.yml
Normal file
@@ -0,0 +1,153 @@
|
|||||||
|
---
|
||||||
|
- name: Converged Proxmox & Optional Linstor Deployment
|
||||||
|
hosts: proxmox
|
||||||
|
become: true
|
||||||
|
gather_facts: true
|
||||||
|
any_errors_fatal: true
|
||||||
|
|
||||||
|
vars:
|
||||||
|
corosync_ip: "{{ priv_ip | default(ansible_host) }}"
|
||||||
|
master_corosync_ip: "{{ hostvars[groups['proxmox'][0]]['priv_ip'] | default(hostvars[groups['proxmox'][0]]['ansible_host']) }}"
|
||||||
|
is_master: "{{ inventory_hostname == groups['proxmox'][0] }}"
|
||||||
|
|
||||||
|
tasks:
|
||||||
|
# ==========================================
|
||||||
|
# PHASE 1: REPOSITORIES & LICENSING
|
||||||
|
# ==========================================
|
||||||
|
- name: Apply Proxmox Enterprise Key
|
||||||
|
ansible.builtin.command: pvesubscription set -k {{ pve_enterprise_key }}
|
||||||
|
when: pve_enterprise_key | length > 0
|
||||||
|
register: pve_key_result
|
||||||
|
changed_when: "'successfully' in pve_key_result.stdout"
|
||||||
|
|
||||||
|
- name: Manage Proxmox Enterprise Repository
|
||||||
|
ansible.builtin.lineinfile:
|
||||||
|
path: /etc/apt/sources.list.d/pve-enterprise.list
|
||||||
|
regexp: '^#?deb https://enterprise.proxmox.com/debian/pve'
|
||||||
|
line: "{{ '' if pve_enterprise_key | length > 0 else '#' }}deb https://enterprise.proxmox.com/debian/pve bookworm pve-enterprise"
|
||||||
|
state: present
|
||||||
|
|
||||||
|
- name: Manage Proxmox No-Subscription Repository
|
||||||
|
ansible.builtin.apt_repository:
|
||||||
|
repo: 'deb http://download.proxmox.com/debian/pve bookworm pve-no-subscription'
|
||||||
|
state: "{{ 'absent' if pve_enterprise_key | length > 0 else 'present' }}"
|
||||||
|
filename: pve-no-subscription
|
||||||
|
|
||||||
|
- name: Add Linstor Repositories
|
||||||
|
ansible.builtin.include_role:
|
||||||
|
name: linstor_repo
|
||||||
|
when: deploy_linstor | bool
|
||||||
|
|
||||||
|
# ==========================================
|
||||||
|
# PHASE 2: UPDATES & NAG REMOVAL
|
||||||
|
# ==========================================
|
||||||
|
- name: Update APT cache and run dist-upgrade
|
||||||
|
ansible.builtin.apt:
|
||||||
|
update_cache: yes
|
||||||
|
upgrade: dist
|
||||||
|
autoremove: yes
|
||||||
|
register: apt_upgrade
|
||||||
|
|
||||||
|
- name: Remove Proxmox Subscription Nag (JS Patch)
|
||||||
|
ansible.builtin.replace:
|
||||||
|
path: /usr/share/javascript/proxmox-widget-toolkit/proxmoxlib.js
|
||||||
|
regexp: "(Ext.Msg.show\\(\\{\\s+title: gettext\\('No valid subscription'\\),)"
|
||||||
|
replace: "void({ //\\1"
|
||||||
|
notify: Restart pveproxy
|
||||||
|
|
||||||
|
- name: Install Linstor Packages
|
||||||
|
ansible.builtin.include_role:
|
||||||
|
name: linstor_install
|
||||||
|
when: deploy_linstor | bool
|
||||||
|
|
||||||
|
- name: Reboot if kernel was updated
|
||||||
|
ansible.builtin.reboot:
|
||||||
|
msg: "Rebooting to apply new PVE Kernel updates"
|
||||||
|
when: apt_upgrade.changed
|
||||||
|
|
||||||
|
# ==========================================
|
||||||
|
# PHASE 3: PROXMOX CLUSTERING
|
||||||
|
# ==========================================
|
||||||
|
- name: Ensure SSH keypair exists for root
|
||||||
|
ansible.builtin.user:
|
||||||
|
name: root
|
||||||
|
generate_ssh_key: yes
|
||||||
|
ssh_key_bits: 4096
|
||||||
|
|
||||||
|
- name: Fetch public SSH keys from all nodes
|
||||||
|
ansible.builtin.slurp:
|
||||||
|
src: /root/.ssh/id_rsa.pub
|
||||||
|
register: ssh_pub_keys
|
||||||
|
|
||||||
|
- name: Distribute SSH keys to build full-mesh trust
|
||||||
|
ansible.posix.authorized_key:
|
||||||
|
user: root
|
||||||
|
key: "{{ hostvars[item]['ssh_pub_keys']['content'] | b64decode }}"
|
||||||
|
state: present
|
||||||
|
loop: "{{ ansible_play_hosts }}"
|
||||||
|
|
||||||
|
- name: Automatically accept host keys for cluster subnets
|
||||||
|
ansible.builtin.blockinfile:
|
||||||
|
path: /root/.ssh/config
|
||||||
|
create: yes
|
||||||
|
mode: '0600'
|
||||||
|
block: |
|
||||||
|
Host {{ master_corosync_ip | regex_replace('\.[0-9]+$', '.*') }} {{ hostvars[groups['proxmox'][0]]['ansible_host'] | regex_replace('\.[0-9]+$', '.*') }}
|
||||||
|
StrictHostKeyChecking no
|
||||||
|
UserKnownHostsFile=/dev/null
|
||||||
|
|
||||||
|
- name: Check current Proxmox cluster status
|
||||||
|
ansible.builtin.command: pvecm status
|
||||||
|
register: pvecm_status
|
||||||
|
ignore_errors: yes
|
||||||
|
changed_when: false
|
||||||
|
|
||||||
|
- name: Initialize the Proxmox Cluster (Master Node)
|
||||||
|
ansible.builtin.command: pvecm create {{ pve_cluster_name }} --link0 {{ corosync_ip }}
|
||||||
|
when:
|
||||||
|
- pvecm_status.rc != 0
|
||||||
|
- is_master | bool
|
||||||
|
|
||||||
|
- name: Wait for Corosync to stabilize
|
||||||
|
ansible.builtin.pause:
|
||||||
|
seconds: 10
|
||||||
|
when:
|
||||||
|
- pvecm_status.rc != 0
|
||||||
|
- is_master | bool
|
||||||
|
|
||||||
|
- name: Join the Proxmox Cluster (Worker Nodes)
|
||||||
|
ansible.builtin.command: >
|
||||||
|
pvecm add {{ master_corosync_ip }} --link0 {{ corosync_ip }} --use_ssh yes
|
||||||
|
when:
|
||||||
|
- pvecm_status.rc != 0
|
||||||
|
- not is_master | bool
|
||||||
|
async: 60
|
||||||
|
poll: 10
|
||||||
|
|
||||||
|
# ==========================================
|
||||||
|
# PHASE 4: LINSTOR CONFIGURATION
|
||||||
|
# ==========================================
|
||||||
|
- name: Configure Linstor (Skipped if deploy_linstor=false)
|
||||||
|
when: deploy_linstor | bool
|
||||||
|
block:
|
||||||
|
- name: Configure Linstor Cluster
|
||||||
|
ansible.builtin.include_role:
|
||||||
|
name: linstor_cluster
|
||||||
|
|
||||||
|
- name: Configure Linstor Storage
|
||||||
|
ansible.builtin.include_role:
|
||||||
|
name: linstor_storage
|
||||||
|
|
||||||
|
- name: Configure Linstor HA
|
||||||
|
ansible.builtin.include_role:
|
||||||
|
name: linstor_ha
|
||||||
|
|
||||||
|
- name: Configure Proxmox Storage Plugin for Linstor
|
||||||
|
ansible.builtin.include_role:
|
||||||
|
name: proxmox_storage
|
||||||
|
|
||||||
|
handlers:
|
||||||
|
- name: Restart pveproxy
|
||||||
|
ansible.builtin.service:
|
||||||
|
name: pveproxy
|
||||||
|
state: restarted
|
||||||
37
ansible/group_vars/all.yml
Normal file
37
ansible/group_vars/all.yml
Normal file
@@ -0,0 +1,37 @@
|
|||||||
|
---
|
||||||
|
# Deployment Toggles
|
||||||
|
deploy_linstor: false # Set to true to install and configure Linstor/DRBD
|
||||||
|
|
||||||
|
# Proxmox Cluster Configuration
|
||||||
|
pve_cluster_name: "prod-cluster"
|
||||||
|
pve_enterprise_key: "" # Leave empty ("") to use no-subscription, or insert "PVE..." key
|
||||||
|
|
||||||
|
# High Availability Controller Configuration
|
||||||
|
ha_pool: "fast_pool_1"
|
||||||
|
ha_vip: "10.191.125.4"
|
||||||
|
ha_vip_cidr: "29"
|
||||||
|
|
||||||
|
# Linstor Storage Pools Configuration
|
||||||
|
linstor_storage_pools:
|
||||||
|
- pool_name: "fast_pool_1"
|
||||||
|
rg_name: "fast_pool_1"
|
||||||
|
vg_name: "pve"
|
||||||
|
thin_name: "data"
|
||||||
|
# TARGET RULES:
|
||||||
|
# Use raw disk (/dev/sdb) to wipe/create LVM
|
||||||
|
# Use existing thin-pool path (pve/data) to register safely
|
||||||
|
targets:
|
||||||
|
pve-node-01: "pve/data"
|
||||||
|
pve-node-02: "pve/data"
|
||||||
|
pve-node-03: "pve/data"
|
||||||
|
- pool_name: "slow_pool_1"
|
||||||
|
rg_name: "slow_pool_1"
|
||||||
|
vg_name: "sp1"
|
||||||
|
thin_name: "data"
|
||||||
|
# TARGET RULES:
|
||||||
|
# Use raw disk (/dev/sdb) to wipe/create LVM
|
||||||
|
# Use existing thin-pool path (pve/data) to register safely
|
||||||
|
targets:
|
||||||
|
pve-node-01: "/dev/sda"
|
||||||
|
pve-node-02: "/dev/sda"
|
||||||
|
pve-node-03: "/dev/sda"
|
||||||
13
ansible/inventory/hosts.yml
Normal file
13
ansible/inventory/hosts.yml
Normal file
@@ -0,0 +1,13 @@
|
|||||||
|
all:
|
||||||
|
children:
|
||||||
|
proxmox:
|
||||||
|
hosts:
|
||||||
|
pve-node-01:
|
||||||
|
ansible_host: 10.191.25.21
|
||||||
|
priv_ip: 10.191.125.1
|
||||||
|
pve-node-02:
|
||||||
|
ansible_host: 10.191.25.22
|
||||||
|
priv_ip: 10.191.125.2
|
||||||
|
pve-node-03:
|
||||||
|
ansible_host: 10.191.25.23
|
||||||
|
priv_ip: 10.191.125.3
|
||||||
61
ansible/roles/linstor_cluster/tasks/main.yml
Normal file
61
ansible/roles/linstor_cluster/tasks/main.yml
Normal file
@@ -0,0 +1,61 @@
|
|||||||
|
---
|
||||||
|
# tasks file for linstor_cluster
|
||||||
|
|
||||||
|
- name: Safely manage LINSTOR cluster nodes in /etc/hosts
|
||||||
|
ansible.builtin.blockinfile:
|
||||||
|
path: /etc/hosts
|
||||||
|
marker: "# {mark} ANSIBLE MANAGED BLOCK - LINSTOR CLUSTER"
|
||||||
|
block: |
|
||||||
|
# Linstor Cluster Nodes (Primary: Private Network)
|
||||||
|
{% for host in groups['proxmox'] %}
|
||||||
|
{{ hostvars[host].priv_ip }} {{ host }}
|
||||||
|
{% endfor %}
|
||||||
|
|
||||||
|
# Linstor Cluster Nodes (Fallback: Public Network)
|
||||||
|
{% for host in groups['proxmox'] %}
|
||||||
|
{{ hostvars[host].ansible_host }} {{ host }}-public
|
||||||
|
{% endfor %}
|
||||||
|
|
||||||
|
- name: Ensure /etc/linstor directory exists
|
||||||
|
ansible.builtin.file:
|
||||||
|
path: /etc/linstor
|
||||||
|
state: directory
|
||||||
|
mode: '0755'
|
||||||
|
|
||||||
|
- name: Deploy linstor-client.conf
|
||||||
|
ansible.builtin.template:
|
||||||
|
src: linstor-client.conf.j2
|
||||||
|
dest: /etc/linstor/linstor-client.conf
|
||||||
|
mode: '0644'
|
||||||
|
|
||||||
|
# ---------------------------------------------------------
|
||||||
|
# LINSTOR NODE REGISTRATION (Executed ONLY on Node 1)
|
||||||
|
# ---------------------------------------------------------
|
||||||
|
|
||||||
|
- name: Register nodes in LINSTOR
|
||||||
|
ansible.builtin.shell: |
|
||||||
|
set -o pipefail
|
||||||
|
if ! linstor node list | grep -q " {{ item }} "; then
|
||||||
|
linstor node create "{{ item }}" "{{ hostvars[item].priv_ip }}" --node-type Combined > /dev/null
|
||||||
|
echo "changed"
|
||||||
|
fi
|
||||||
|
args:
|
||||||
|
executable: /bin/bash
|
||||||
|
register: linstor_cluster_node_create
|
||||||
|
changed_when: "'changed' in linstor_cluster_node_create.stdout"
|
||||||
|
loop: "{{ groups['proxmox'] }}"
|
||||||
|
when: inventory_hostname == groups['proxmox'][0]
|
||||||
|
|
||||||
|
- name: Configure Multipath Fallback Interfaces
|
||||||
|
ansible.builtin.shell: |
|
||||||
|
set -o pipefail
|
||||||
|
if ! linstor node interface list "{{ item }}" | grep -q " fallback "; then
|
||||||
|
linstor node interface create "{{ item }}" fallback "{{ hostvars[item].ansible_host }}" > /dev/null
|
||||||
|
echo "changed"
|
||||||
|
fi
|
||||||
|
args:
|
||||||
|
executable: /bin/bash
|
||||||
|
register: linstor_cluster_int_create
|
||||||
|
changed_when: "'changed' in linstor_cluster_int_create.stdout"
|
||||||
|
loop: "{{ groups['proxmox'] }}"
|
||||||
|
when: inventory_hostname == groups['proxmox'][0]
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
[global]
|
||||||
|
# Ansible dynamically grabs all hosts in the 'proxmox' group and joins them with commas
|
||||||
|
controllers={{ groups['proxmox'] | join(',') }}
|
||||||
134
ansible/roles/linstor_ha/tasks/main.yml
Normal file
134
ansible/roles/linstor_ha/tasks/main.yml
Normal file
@@ -0,0 +1,134 @@
|
|||||||
|
---
|
||||||
|
# tasks file for linstor_ha
|
||||||
|
|
||||||
|
# ---------------------------------------------------------
|
||||||
|
# STEP 1: CREATE HA RESOURCES (Executed ONLY on Node 1)
|
||||||
|
# ---------------------------------------------------------
|
||||||
|
- name: Create HA Resource Group and Spawn Database Volume
|
||||||
|
ansible.builtin.shell: |
|
||||||
|
set -o pipefail
|
||||||
|
if ! linstor resource-group list | grep -q " linstor-db-grp "; then
|
||||||
|
linstor resource-group create --storage-pool "{{ ha_pool }}" --place-count 3 --diskless-on-remaining true linstor-db-grp > /dev/null
|
||||||
|
linstor resource-group drbd-options --auto-promote=no --quorum=majority \
|
||||||
|
--on-suspended-primary-outdated=force-secondary --on-no-quorum=io-error --on-no-data-accessible=io-error linstor-db-grp > /dev/null
|
||||||
|
linstor volume-group create linstor-db-grp > /dev/null
|
||||||
|
linstor resource-group spawn-resources linstor-db-grp linstor_db 200M > /dev/null
|
||||||
|
echo "changed"
|
||||||
|
fi
|
||||||
|
args:
|
||||||
|
executable: /bin/bash
|
||||||
|
register: linstor_ha_rg_create
|
||||||
|
changed_when: "'changed' in linstor_ha_rg_create.stdout"
|
||||||
|
when: inventory_hostname == groups['proxmox'][0]
|
||||||
|
|
||||||
|
- name: Wait for DRBD block device to appear locally
|
||||||
|
ansible.builtin.wait_for:
|
||||||
|
path: /dev/drbd/by-res/linstor_db/0
|
||||||
|
state: present
|
||||||
|
timeout: 60
|
||||||
|
when: inventory_hostname == groups['proxmox'][0]
|
||||||
|
|
||||||
|
# ---------------------------------------------------------
|
||||||
|
# STEP 2: PREPARE ALL NODES
|
||||||
|
# ---------------------------------------------------------
|
||||||
|
- name: Install resource-agents
|
||||||
|
ansible.builtin.apt:
|
||||||
|
name: resource-agents
|
||||||
|
state: present
|
||||||
|
|
||||||
|
- name: Ensure local Linstor Controller is stopped and disabled
|
||||||
|
ansible.builtin.systemd:
|
||||||
|
name: linstor-controller
|
||||||
|
enabled: false
|
||||||
|
state: stopped
|
||||||
|
|
||||||
|
- name: Deploy var-lib-linstor.mount unit
|
||||||
|
ansible.builtin.copy:
|
||||||
|
dest: /etc/systemd/system/var-lib-linstor.mount
|
||||||
|
content: |
|
||||||
|
[Unit]
|
||||||
|
Description=Filesystem for the LINSTOR controller
|
||||||
|
|
||||||
|
[Mount]
|
||||||
|
What=/dev/drbd/by-res/linstor_db/0
|
||||||
|
Where=/var/lib/linstor
|
||||||
|
mode: '0644'
|
||||||
|
|
||||||
|
- name: Ensure drbd-reactor directory exists
|
||||||
|
ansible.builtin.file:
|
||||||
|
path: /etc/drbd-reactor.d
|
||||||
|
state: directory
|
||||||
|
mode: '0755'
|
||||||
|
|
||||||
|
- name: Deploy DRBD Reactor TOML config
|
||||||
|
ansible.builtin.template:
|
||||||
|
src: linstor_db.toml.j2
|
||||||
|
dest: /etc/drbd-reactor.d/linstor_db.toml
|
||||||
|
mode: '0644'
|
||||||
|
|
||||||
|
# Notice how the stat task is gone, replaced by 'creates' and 'removes'
|
||||||
|
- name: Move existing local database to .orig safely
|
||||||
|
ansible.builtin.command: mv /var/lib/linstor /var/lib/linstor.orig
|
||||||
|
args:
|
||||||
|
creates: /var/lib/linstor.orig
|
||||||
|
removes: /var/lib/linstor
|
||||||
|
|
||||||
|
- name: Create empty mount point and set immutable flag (+i)
|
||||||
|
ansible.builtin.file:
|
||||||
|
path: /var/lib/linstor
|
||||||
|
state: directory
|
||||||
|
mode: '0755'
|
||||||
|
attributes: '+i'
|
||||||
|
|
||||||
|
- name: Ensure Satellite Override directory exists
|
||||||
|
ansible.builtin.file:
|
||||||
|
path: /etc/systemd/system/linstor-satellite.service.d
|
||||||
|
state: directory
|
||||||
|
mode: '0755'
|
||||||
|
|
||||||
|
- name: Deploy Satellite Keep-Resource Override
|
||||||
|
ansible.builtin.copy:
|
||||||
|
dest: /etc/systemd/system/linstor-satellite.service.d/override.conf
|
||||||
|
content: |
|
||||||
|
[Service]
|
||||||
|
Environment=LS_KEEP_RES=linstor_db
|
||||||
|
mode: '0644'
|
||||||
|
|
||||||
|
- name: Reload daemon and restart Satellite
|
||||||
|
ansible.builtin.systemd:
|
||||||
|
daemon_reload: true
|
||||||
|
name: linstor-satellite
|
||||||
|
state: restarted
|
||||||
|
|
||||||
|
# ---------------------------------------------------------
|
||||||
|
# STEP 3: FORMAT & MIGRATE DB (Executed ONLY on Node 1)
|
||||||
|
# ---------------------------------------------------------
|
||||||
|
- name: Format HA Volume and Migrate Database
|
||||||
|
ansible.builtin.shell: |
|
||||||
|
set -o pipefail
|
||||||
|
if ! blkid /dev/drbd/by-res/linstor_db/0 | grep -q "ext4"; then
|
||||||
|
drbdadm primary linstor_db
|
||||||
|
mkfs.ext4 /dev/drbd/by-res/linstor_db/0 > /dev/null
|
||||||
|
|
||||||
|
chattr -i /var/lib/linstor
|
||||||
|
mount /dev/drbd/by-res/linstor_db/0 /var/lib/linstor
|
||||||
|
cp -a /var/lib/linstor.orig/* /var/lib/linstor/
|
||||||
|
umount /var/lib/linstor
|
||||||
|
chattr +i /var/lib/linstor
|
||||||
|
drbdadm secondary linstor_db
|
||||||
|
echo "changed"
|
||||||
|
fi
|
||||||
|
args:
|
||||||
|
executable: /bin/bash
|
||||||
|
register: linstor_ha_migrate
|
||||||
|
changed_when: "'changed' in linstor_ha_migrate.stdout"
|
||||||
|
when: inventory_hostname == groups['proxmox'][0]
|
||||||
|
|
||||||
|
# ---------------------------------------------------------
|
||||||
|
# STEP 4: ACTIVATE HA REACTOR
|
||||||
|
# ---------------------------------------------------------
|
||||||
|
- name: Enable and start DRBD Reactor
|
||||||
|
ansible.builtin.systemd:
|
||||||
|
name: drbd-reactor
|
||||||
|
enabled: true
|
||||||
|
state: started
|
||||||
9
ansible/roles/linstor_ha/templates/linstor_db.toml.j2
Normal file
9
ansible/roles/linstor_ha/templates/linstor_db.toml.j2
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
[[promoter]]
|
||||||
|
[promoter.resources.linstor_db]
|
||||||
|
start = [
|
||||||
|
"ocf:heartbeat:IPaddr2 service_ip cidr_netmask={{ ha_vip_cidr }} ip={{ ha_vip }}",
|
||||||
|
"var-lib-linstor.mount",
|
||||||
|
"linstor-controller.service"
|
||||||
|
]
|
||||||
|
on-drbd-demote-failure = "reboot"
|
||||||
|
secondary-force = true
|
||||||
63
ansible/roles/linstor_install/tasks/main.yml
Normal file
63
ansible/roles/linstor_install/tasks/main.yml
Normal file
@@ -0,0 +1,63 @@
|
|||||||
|
---
|
||||||
|
# tasks file for linstor_install
|
||||||
|
|
||||||
|
- name: Install Proxmox kernel headers
|
||||||
|
ansible.builtin.apt:
|
||||||
|
name:
|
||||||
|
- "proxmox-headers-{{ ansible_facts['kernel'] }}"
|
||||||
|
- proxmox-default-headers
|
||||||
|
state: present
|
||||||
|
update_cache: true
|
||||||
|
|
||||||
|
- name: Install LINSTOR, DRBD, and LVM dependencies
|
||||||
|
ansible.builtin.apt:
|
||||||
|
name:
|
||||||
|
- drbd-dkms
|
||||||
|
- drbd-utils
|
||||||
|
- lvm2
|
||||||
|
- thin-provisioning-tools
|
||||||
|
- linstor-satellite
|
||||||
|
- linstor-proxmox
|
||||||
|
- linstor-controller
|
||||||
|
- linstor-client
|
||||||
|
- drbd-reactor
|
||||||
|
state: present
|
||||||
|
|
||||||
|
- name: Check currently loaded DRBD version
|
||||||
|
ansible.builtin.shell: cat /proc/drbd || true
|
||||||
|
register: linstor_install_drbd_version
|
||||||
|
changed_when: false
|
||||||
|
|
||||||
|
- name: Eject legacy DRBD 8.4 module if present
|
||||||
|
community.general.modprobe:
|
||||||
|
name: drbd
|
||||||
|
state: absent
|
||||||
|
when: "'version: 8' in linstor_install_drbd_version.stdout"
|
||||||
|
|
||||||
|
- name: Ensure DRBD kernel module is loaded
|
||||||
|
community.general.modprobe:
|
||||||
|
name: drbd
|
||||||
|
state: present
|
||||||
|
|
||||||
|
- name: Enable and start base services (Satellite & Reactor)
|
||||||
|
ansible.builtin.systemd:
|
||||||
|
name: "{{ item }}"
|
||||||
|
enabled: true
|
||||||
|
state: started
|
||||||
|
loop:
|
||||||
|
- linstor-satellite
|
||||||
|
- drbd-reactor
|
||||||
|
|
||||||
|
- name: Ensure LINSTOR Controller is disabled safely on all nodes
|
||||||
|
ansible.builtin.systemd:
|
||||||
|
name: linstor-controller
|
||||||
|
enabled: false
|
||||||
|
state: stopped
|
||||||
|
|
||||||
|
- name: Enable and start LINSTOR Controller strictly on Node 1
|
||||||
|
ansible.builtin.systemd:
|
||||||
|
name: linstor-controller
|
||||||
|
enabled: true
|
||||||
|
state: started
|
||||||
|
# Ansible Magic: This dynamically targets the very first node listed in your inventory under the 'proxmox' group!
|
||||||
|
when: inventory_hostname == groups['proxmox'][0]
|
||||||
24
ansible/roles/linstor_repo/tasks/main.yml
Normal file
24
ansible/roles/linstor_repo/tasks/main.yml
Normal file
@@ -0,0 +1,24 @@
|
|||||||
|
---
|
||||||
|
# tasks file for linstor_repo
|
||||||
|
|
||||||
|
- name: Download LINBIT keyring package
|
||||||
|
ansible.builtin.get_url:
|
||||||
|
url: https://packages.linbit.com/public/linbit-keyring.deb
|
||||||
|
dest: /tmp/linbit-keyring.deb
|
||||||
|
mode: '0644'
|
||||||
|
|
||||||
|
- name: Install LINBIT keyring
|
||||||
|
ansible.builtin.apt:
|
||||||
|
deb: /tmp/linbit-keyring.deb
|
||||||
|
state: present
|
||||||
|
|
||||||
|
- name: Add LINBIT APT repository for Proxmox
|
||||||
|
ansible.builtin.apt_repository:
|
||||||
|
repo: >-
|
||||||
|
deb [signed-by=/etc/apt/trusted.gpg.d/linbit-keyring.gpg]
|
||||||
|
https://packages.linbit.com/public/
|
||||||
|
proxmox-{{ ansible_facts['distribution_major_version'] | int - 4 }}
|
||||||
|
drbd-9
|
||||||
|
filename: linbit
|
||||||
|
state: present
|
||||||
|
update_cache: true
|
||||||
69
ansible/roles/linstor_storage/tasks/main.yml
Normal file
69
ansible/roles/linstor_storage/tasks/main.yml
Normal file
@@ -0,0 +1,69 @@
|
|||||||
|
---
|
||||||
|
# tasks file for linstor_storage
|
||||||
|
|
||||||
|
- name: Configure LVM global_filter for DRBD safely
|
||||||
|
ansible.builtin.lineinfile:
|
||||||
|
path: /etc/lvm/lvm.conf
|
||||||
|
insertafter: '^[ \t]*devices[ \t]*\{'
|
||||||
|
# This regex ensures we only ever have ONE DRBD filter line, replacing the old one if it exists
|
||||||
|
regexp: '^[ \t]*global_filter[ \t]*=.*r\|\^/dev/drbd\|'
|
||||||
|
line: ' global_filter = [ "r|^/dev/drbd|", "r|^/dev/mapper/[lL]instor|" ]'
|
||||||
|
state: present
|
||||||
|
|
||||||
|
- name: Ensure Volume Groups exist for raw disks
|
||||||
|
community.general.lvg:
|
||||||
|
vg: "{{ item.vg_name }}"
|
||||||
|
pvs: "{{ item.targets[inventory_hostname] }}"
|
||||||
|
loop: "{{ linstor_storage_pools }}"
|
||||||
|
# Jinja2 magic: Only run this LVM task if the target string starts with '/dev/'
|
||||||
|
when: item.targets[inventory_hostname].startswith('/dev/')
|
||||||
|
|
||||||
|
- name: Ensure LVM Thin Pools exist for raw disks
|
||||||
|
community.general.lvol:
|
||||||
|
vg: "{{ item.vg_name }}"
|
||||||
|
lv: "{{ item.thin_name }}"
|
||||||
|
size: "100%FREE"
|
||||||
|
opts: "-T -Zn"
|
||||||
|
shrink: false
|
||||||
|
loop: "{{ linstor_storage_pools }}"
|
||||||
|
when: item.targets[inventory_hostname].startswith('/dev/')
|
||||||
|
|
||||||
|
# ---------------------------------------------------------
|
||||||
|
# LINSTOR STORAGE REGISTRATION (Executed ONLY on Node 1)
|
||||||
|
# ---------------------------------------------------------
|
||||||
|
|
||||||
|
- name: Register Storage Pools in LINSTOR
|
||||||
|
ansible.builtin.shell: |
|
||||||
|
set -o pipefail
|
||||||
|
|
||||||
|
# We use Jinja2 to write a custom bash loop for exactly the nodes in our inventory
|
||||||
|
{% for host in groups['proxmox'] %}
|
||||||
|
{% set target = item.targets[host] %}
|
||||||
|
{% set lvm_path = (item.vg_name + '/' + item.thin_name) if target.startswith('/dev/') else target %}
|
||||||
|
|
||||||
|
if ! linstor storage-pool list | grep -q " {{ host }} .* {{ item.pool_name }} "; then
|
||||||
|
linstor storage-pool create lvmthin "{{ host }}" "{{ item.pool_name }}" "{{ lvm_path }}" > /dev/null
|
||||||
|
echo "changed"
|
||||||
|
fi
|
||||||
|
{% endfor %}
|
||||||
|
args:
|
||||||
|
executable: /bin/bash
|
||||||
|
register: linstor_storage_pool_create
|
||||||
|
changed_when: "'changed' in linstor_storage_pool_create.stdout"
|
||||||
|
loop: "{{ linstor_storage_pools }}"
|
||||||
|
when: inventory_hostname == groups['proxmox'][0]
|
||||||
|
|
||||||
|
- name: Create Resource Groups and Volume Groups
|
||||||
|
ansible.builtin.shell: |
|
||||||
|
set -o pipefail
|
||||||
|
if ! linstor resource-group list | grep -q " {{ item.rg_name }} "; then
|
||||||
|
linstor resource-group create "{{ item.rg_name }}" --storage-pool "{{ item.pool_name }}" --place-count 3 > /dev/null
|
||||||
|
linstor volume-group create "{{ item.rg_name }}" > /dev/null
|
||||||
|
echo "changed"
|
||||||
|
fi
|
||||||
|
args:
|
||||||
|
executable: /bin/bash
|
||||||
|
register: linstor_storage_rg_create
|
||||||
|
changed_when: "'changed' in linstor_storage_rg_create.stdout"
|
||||||
|
loop: "{{ linstor_storage_pools }}"
|
||||||
|
when: inventory_hostname == groups['proxmox'][0]
|
||||||
29
ansible/roles/proxmox_storage/tasks/main.yml
Normal file
29
ansible/roles/proxmox_storage/tasks/main.yml
Normal file
@@ -0,0 +1,29 @@
|
|||||||
|
---
|
||||||
|
# tasks file for proxmox_storage
|
||||||
|
|
||||||
|
- name: Create a safety backup of Proxmox storage.cfg
|
||||||
|
ansible.builtin.copy:
|
||||||
|
src: /etc/pve/storage.cfg
|
||||||
|
dest: "/root/pve_storage.cfg.bak.{{ ansible_facts['date_time']['epoch'] }}"
|
||||||
|
remote_src: true
|
||||||
|
mode: '0600'
|
||||||
|
when: inventory_hostname == groups['proxmox'][0]
|
||||||
|
|
||||||
|
- name: Register LINSTOR resource groups in Proxmox GUI (pvesm)
|
||||||
|
ansible.builtin.shell: |
|
||||||
|
set -o pipefail
|
||||||
|
|
||||||
|
# We use the ha_vip variable, but gracefully fallback to Node 1's public IP if it's undefined
|
||||||
|
CONTROLLER="{{ ha_vip | default(hostvars[groups['proxmox'][0]].ansible_host) }}"
|
||||||
|
STORAGE_ID="ls-{{ item.rg_name }}"
|
||||||
|
|
||||||
|
if ! pvesm status | grep -q "^${STORAGE_ID} "; then
|
||||||
|
pvesm add drbd "${STORAGE_ID}" --controller "${CONTROLLER}" --resourcegroup "{{ item.rg_name }}" --content images,rootdir > /dev/null
|
||||||
|
echo "changed"
|
||||||
|
fi
|
||||||
|
args:
|
||||||
|
executable: /bin/bash
|
||||||
|
register: proxmox_storage_pvesm_add
|
||||||
|
changed_when: "'changed' in proxmox_storage_pvesm_add.stdout"
|
||||||
|
loop: "{{ linstor_storage_pools }}"
|
||||||
|
when: inventory_hostname == groups['proxmox'][0]
|
||||||
21
docs/01-prerequisites.md
Normal file
21
docs/01-prerequisites.md
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
# 01 - Prerequisites & Node Preparation
|
||||||
|
|
||||||
|
This documentation outlines the required state for bare-metal servers before executing the Ansible provisioning pipeline.
|
||||||
|
|
||||||
|
## Node State
|
||||||
|
* **OS:** Proxmox VE 8.x freshly installed.
|
||||||
|
* **Authentication:** Root password known. SSH key exchange is handled automatically by Ansible.
|
||||||
|
* **Network:** Static IPs assigned. If deploying Linstor, the management interface and Corosync/DRBD replication should ideally be physically separated. If utilizing a single NIC, QoS/traffic shaping must be handled at the switch level to prioritize Corosync UDP (5404/5405) over DRBD replication traffic.
|
||||||
|
* **Storage:** Target block devices intended for Linstor raw LVM provisioning must contain no existing partition tables, LVM metadata, or filesystems.
|
||||||
|
|
||||||
|
## Execution Environment
|
||||||
|
Local Ansible installation is not required. This repository enforces the use of an isolated VS Code DevContainer.
|
||||||
|
|
||||||
|
1. Open this repository in Visual Studio Code.
|
||||||
|
2. Ensure Docker/Podman and the "Dev Containers" extension are installed.
|
||||||
|
3. Select `Reopen in Container`.
|
||||||
|
4. Open a terminal within VS Code. All paths referenced in this documentation originate from the `ansible/` directory.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd ansible/
|
||||||
|
```
|
||||||
34
docs/02-configuration.md
Normal file
34
docs/02-configuration.md
Normal file
@@ -0,0 +1,34 @@
|
|||||||
|
# 02 - Configuring the Cluster
|
||||||
|
|
||||||
|
Cluster configurations are segmented per project/customer using Ansible inventories. Do not edit variables in the root directory.
|
||||||
|
|
||||||
|
## 1. Create the Environment
|
||||||
|
Copy the template directory for your specific project:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cp -r inventories/_template inventories/<customer_name>
|
||||||
|
mv inventories/<customer_name>/hosts.yml.example inventories/<customer_name>/hosts.yml
|
||||||
|
mv inventories/<customer_name>/group_vars/all.yml.example inventories/<customer_name>/group_vars/all.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
## 2. Configure `hosts.yml`
|
||||||
|
Define the target PVE nodes. Ansible defaults to the `ansible_host` IP for Corosync and DRBD replication. If a dedicated backend network is available, define `priv_ip`. Ansible uses conditional fallback logic automatically.
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
all:
|
||||||
|
children:
|
||||||
|
proxmox:
|
||||||
|
hosts:
|
||||||
|
pve-node-01:
|
||||||
|
ansible_host: 192.168.1.11
|
||||||
|
priv_ip: 10.0.0.11
|
||||||
|
```
|
||||||
|
|
||||||
|
## 3. Configure `group_vars/all.yml`
|
||||||
|
|
||||||
|
* **Deployment Toggles:** Set `deploy_linstor: true` for HCI setups, or `false` for traditional SAN integration.
|
||||||
|
* **Enterprise Licensing:** Supply a key in `pve_enterprise_key`. If empty (`""`), the playbook applies the `pve-no-subscription` repository.
|
||||||
|
* **HA Controller:** Define `ha_pool`, `ha_vip` (Virtual IP), and the CIDR prefix for the Linstor DB promoter.
|
||||||
|
* **Storage Pools:**
|
||||||
|
* If mapping a raw disk (`/dev/sda`), the playbook invokes `community.general.lvg` to create the VG/Thin Pool.
|
||||||
|
* If mapping an existing path (`pve/data`), LVM tasks are skipped, and the target is registered directly in Linstor.
|
||||||
21
docs/03-execution.md
Normal file
21
docs/03-execution.md
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
# 03 - Execution
|
||||||
|
|
||||||
|
Ensure you are operating within the `ansible/` directory inside the DevContainer.
|
||||||
|
|
||||||
|
## 1. Test Connectivity
|
||||||
|
Verify SSH access to the nodes before triggering the pipeline. Because SSH keys are not yet distributed, Ansible must prompt for the root password (`-k`).
|
||||||
|
|
||||||
|
```bash
|
||||||
|
ansible all -i inventories/<customer_name>/hosts.yml -m ping -u root -k
|
||||||
|
```
|
||||||
|
|
||||||
|
## 2. Execute the Playbook
|
||||||
|
Run the converged deployment playbook. You will be prompted for the SSH password once.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
ansible-playbook -i inventories/<customer_name>/hosts.yml deploy.yml -u root -k
|
||||||
|
```
|
||||||
|
|
||||||
|
## Expected Behavior
|
||||||
|
* **Idempotency:** The playbook is declarative. If execution fails due to a transient network issue, re-running the exact command will bypass successful tasks and resume from the failure point.
|
||||||
|
* **Reboots:** If Phase 2 upgrades the running PVE kernel, the playbook forces a synchronized node reboot to ensure DRBD module compatibility against the active kernel headers. Ansible will pause and wait for the nodes to return online.
|
||||||
15
docs/04-architecture.md
Normal file
15
docs/04-architecture.md
Normal file
@@ -0,0 +1,15 @@
|
|||||||
|
# 04 - Pipeline Architecture
|
||||||
|
|
||||||
|
The `deploy.yml` playbook orchestrates the initialization via sequential phases utilizing `include_role` blocks to guarantee procedural execution order.
|
||||||
|
|
||||||
|
## Phase 1: Repositories & Licensing
|
||||||
|
Evaluates the `pve_enterprise_key` variable. Modifies `/etc/apt/sources.list.d/`, drops the ceph/enterprise configurations as needed, and applies Linstor repository keys.
|
||||||
|
|
||||||
|
## Phase 2: System Updates & JS Patches
|
||||||
|
Forces a `dist-upgrade`. Implements a regex replacement targeting `proxmoxlib.js` to nullify the ExtJS subscription prompt. Installs required DRBD headers and modules. Issues a conditional reboot.
|
||||||
|
|
||||||
|
## Phase 3: Proxmox Clustering
|
||||||
|
Generates 4096-bit RSA keys, constructs a full-mesh authorized_keys trust, and nullifies StrictHostKeyChecking for the defined subnets. Bootstraps Corosync via `pvecm create` on Node 01, followed by asynchronous `pvecm add` operations on worker nodes.
|
||||||
|
|
||||||
|
## Phase 4: Linstor/DRBD Configuration (Conditional)
|
||||||
|
Wrapped in a feature flag (`deploy_linstor`). Executes standard DRBD module loading, configures global LVM filters, sets up High Availability components, and injects backend parameters into `/etc/pve/storage.cfg`.
|
||||||
18
docs/05-linstor-storage.md
Normal file
18
docs/05-linstor-storage.md
Normal file
@@ -0,0 +1,18 @@
|
|||||||
|
# 05 - Linstor & Storage Implementation
|
||||||
|
|
||||||
|
This document details the configuration enforcement executed by the Linstor roles.
|
||||||
|
|
||||||
|
## LVM Global Filter
|
||||||
|
To prevent PVE from detecting double LVM signatures (raw block vs DRBD layer), the playbook injects the following global filter into `/etc/lvm/lvm.conf`.
|
||||||
|
`global_filter =[ "r|^/dev/drbd|", "r|^/dev/mapper/[lL]instor|" ]`
|
||||||
|
|
||||||
|
## Storage Provisioning
|
||||||
|
The `community.general.lvg` task targets raw block devices strictly when the target string starts with `/dev/`. **The playbook does not invoke `wipefs`.** If residual partition tables exist on the target block device, the LVM module will purposefully fail to prevent data destruction.
|
||||||
|
|
||||||
|
## HA Controller Topology
|
||||||
|
Linstor Controller HA is handled via `drbd-reactor`.
|
||||||
|
1. A 200M DRBD resource (`linstor_db`) is spawned across all nodes.
|
||||||
|
2. The default `linstor-controller` systemd service is disabled.
|
||||||
|
3. A `var-lib-linstor.mount` unit is deployed.
|
||||||
|
4. The local `/var/lib/linstor` mount point is secured with the immutable flag (`chattr +i`). This guarantees that if the DRBD resource fails to mount, the local filesystem rejects writes, preventing database split-brains.
|
||||||
|
5. `drbd-reactor` manages the floating VIP and promoter plugin logic.
|
||||||
39
docs/06-troubleshooting.md
Normal file
39
docs/06-troubleshooting.md
Normal file
@@ -0,0 +1,39 @@
|
|||||||
|
# 06 - Troubleshooting
|
||||||
|
|
||||||
|
Common interventions for pipeline failures or hardware reuse.
|
||||||
|
|
||||||
|
## LVM Provisioning Failures (Dirty Disks)
|
||||||
|
If Ansible halts at the LVM Volume Group creation phase, the target block device likely retains partition data. Ensure you are targeting the correct device and wipe the signatures manually:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Execute on the specific Proxmox node
|
||||||
|
wipefs -a /dev/sdX
|
||||||
|
```
|
||||||
|
Rerun the playbook after clearing the disk.
|
||||||
|
|
||||||
|
## Corosync Quorum Loss
|
||||||
|
If execution hangs on `pvecm status` or clustering commands, Corosync has lost quorum. In a 3-node cluster, 2 nodes must be online. To force a single active node to regain quorum and unblock operations (for recovery only):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
pvecm expected 1
|
||||||
|
```
|
||||||
|
|
||||||
|
## Linstor HA VIP Unreachable
|
||||||
|
If the controller does not initialize, verify the promoter plugin state:
|
||||||
|
```bash
|
||||||
|
systemctl status drbd-reactor
|
||||||
|
journalctl -u drbd-reactor -n 50
|
||||||
|
```
|
||||||
|
Verify switch layer allows IP spoofing / MAC address floating for the defined `ha_vip`.
|
||||||
|
|
||||||
|
## Complete Cluster Reset
|
||||||
|
To obliterate Corosync configuration and reset the node to a standalone state:
|
||||||
|
```bash
|
||||||
|
systemctl stop pve-cluster
|
||||||
|
pmxcfs -l
|
||||||
|
rm /etc/pve/corosync.conf
|
||||||
|
rm -rf /etc/corosync/*
|
||||||
|
killall pmxcfs
|
||||||
|
systemctl start pve-cluster
|
||||||
|
```
|
||||||
|
*Note: If DRBD configurations are active, an OS reinstall is significantly faster and safer than manual DRBD block device teardown.*
|
||||||
Reference in New Issue
Block a user