Compare commits
3 Commits
7b271d07fb
...
e091584f79
| Author | SHA1 | Date | |
|---|---|---|---|
| e091584f79 | |||
| 65c8592aa1 | |||
| 7d7b17f945 |
229
README.md
229
README.md
@@ -29,18 +29,29 @@ Pool: IaC
|
|||||||
|
|
||||||
### 1.1 Install tools
|
### 1.1 Install tools
|
||||||
|
|
||||||
|
**Terraform:**
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Terraform
|
|
||||||
yum install -y yum-utils
|
yum install -y yum-utils
|
||||||
|
```
|
||||||
|
|
||||||
|
```bash
|
||||||
yum-config-manager --add-repo https://rpm.releases.hashicorp.com/RHEL/hashicorp.repo
|
yum-config-manager --add-repo https://rpm.releases.hashicorp.com/RHEL/hashicorp.repo
|
||||||
|
```
|
||||||
|
|
||||||
|
```bash
|
||||||
yum install -y terraform
|
yum install -y terraform
|
||||||
terraform version
|
```
|
||||||
|
|
||||||
# Ansible
|
**Ansible:**
|
||||||
|
|
||||||
|
```bash
|
||||||
pip3 install ansible
|
pip3 install ansible
|
||||||
# or: dnf install ansible
|
```
|
||||||
|
|
||||||
# Git
|
**Git:**
|
||||||
|
|
||||||
|
```bash
|
||||||
yum install -y git
|
yum install -y git
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -48,17 +59,15 @@ yum install -y git
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
git clone <your-gitea-repo-url> ~/k8s
|
git clone <your-gitea-repo-url> ~/k8s
|
||||||
cd ~/k8s
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### 1.3 Set up Terraform credentials
|
### 1.3 Set up Terraform credentials
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cp cluster-init/terraform/.env.example cluster-init/terraform/.env
|
cp cluster-init/terraform/.env.example cluster-init/terraform/.env
|
||||||
# edit cluster-init/terraform/.env — fill in endpoint + token
|
|
||||||
```
|
```
|
||||||
|
|
||||||
`cluster-init/terraform/.env` is gitignored — never commit it.
|
Edit `cluster-init/terraform/.env` — fill in endpoint + token. This file is gitignored — never commit it.
|
||||||
|
|
||||||
### 1.4 Generate SSH keypair for VM access
|
### 1.4 Generate SSH keypair for VM access
|
||||||
|
|
||||||
@@ -68,7 +77,7 @@ cp cluster-init/terraform/.env.example cluster-init/terraform/.env
|
|||||||
ssh-keygen -t ed25519 -f ~/k8s/cluster-init/terraform/k8s_terraform -N "" -C "terraform-k8s"
|
ssh-keygen -t ed25519 -f ~/k8s/cluster-init/terraform/k8s_terraform -N "" -C "terraform-k8s"
|
||||||
```
|
```
|
||||||
|
|
||||||
Private key: `cluster-init/terraform/k8s_terraform` (gitignored — do not commit)
|
Private key: `cluster-init/terraform/k8s_terraform` (gitignored)
|
||||||
Public key: `cluster-init/terraform/k8s_terraform.pub` (gitignored — injected into VMs via cloud-init)
|
Public key: `cluster-init/terraform/k8s_terraform.pub` (gitignored — injected into VMs via cloud-init)
|
||||||
|
|
||||||
---
|
---
|
||||||
@@ -85,37 +94,66 @@ Enable:
|
|||||||
|
|
||||||
### 2.2 Create Terraform user, role, token
|
### 2.2 Create Terraform user, role, token
|
||||||
|
|
||||||
SSH into any Proxmox node as root:
|
SSH into any Proxmox node as root, then run each command:
|
||||||
|
|
||||||
|
**Create role:**
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Create role
|
pveum role add TerraformRole --privs "VM.Allocate,VM.Clone,VM.Config.Disk,VM.Config.CPU,VM.Config.Memory,VM.Config.Network,VM.Config.Options,VM.Config.CloudInit,VM.Config.CDROM,VM.PowerMgmt,VM.Audit,Datastore.AllocateSpace,Datastore.AllocateTemplate,Datastore.Allocate,Datastore.Audit,Sys.Audit,Sys.Modify,Pool.Allocate,SDN.Use"
|
||||||
pveum role add TerraformRole --privs \
|
```
|
||||||
"VM.Allocate,VM.Clone,VM.Config.Disk,VM.Config.CPU,\
|
|
||||||
VM.Config.Memory,VM.Config.Network,VM.Config.Options,\
|
|
||||||
VM.Config.CloudInit,VM.Config.CDROM,VM.PowerMgmt,VM.Audit,\
|
|
||||||
Datastore.AllocateSpace,Datastore.AllocateTemplate,\
|
|
||||||
Datastore.Allocate,Datastore.Audit,Sys.Audit,Sys.Modify,\
|
|
||||||
Pool.Allocate,SDN.Use"
|
|
||||||
|
|
||||||
# Create user + token — SAVE the secret, shown only once
|
**Create user:**
|
||||||
|
|
||||||
|
```bash
|
||||||
pveum user add terraform@pve --comment "Terraform IaC"
|
pveum user add terraform@pve --comment "Terraform IaC"
|
||||||
|
```
|
||||||
|
|
||||||
|
**Create token** — SAVE the secret, shown only once:
|
||||||
|
|
||||||
|
```bash
|
||||||
pveum user token add terraform@pve terraform --privsep 0
|
pveum user token add terraform@pve terraform --privsep 0
|
||||||
|
```
|
||||||
|
|
||||||
# ACLs
|
**ACLs:**
|
||||||
pveum acl modify /nodes --user terraform@pve --role TerraformRole
|
|
||||||
pveum acl modify /nodes/proxmox03 --user terraform@pve --role TerraformRole
|
```bash
|
||||||
pveum acl modify /storage/local --user terraform@pve --role TerraformRole
|
pveum acl modify /nodes --user terraform@pve --role TerraformRole
|
||||||
pveum acl modify /storage/xpen --user terraform@pve --role TerraformRole
|
```
|
||||||
pveum acl modify /pool/IaC --user terraform@pve --role TerraformRole
|
|
||||||
|
```bash
|
||||||
|
pveum acl modify /nodes/proxmox03 --user terraform@pve --role TerraformRole
|
||||||
|
```
|
||||||
|
|
||||||
|
```bash
|
||||||
|
pveum acl modify /storage/local --user terraform@pve --role TerraformRole
|
||||||
|
```
|
||||||
|
|
||||||
|
```bash
|
||||||
|
pveum acl modify /storage/xpen --user terraform@pve --role TerraformRole
|
||||||
|
```
|
||||||
|
|
||||||
|
```bash
|
||||||
|
pveum acl modify /pool/IaC --user terraform@pve --role TerraformRole
|
||||||
|
```
|
||||||
|
|
||||||
|
```bash
|
||||||
pveum acl modify /sdn/zones/localnetwork/vmbr0 --user terraform@pve --role TerraformRole
|
pveum acl modify /sdn/zones/localnetwork/vmbr0 --user terraform@pve --role TerraformRole
|
||||||
|
```
|
||||||
|
|
||||||
# Required for proxmox_download_file — Sys.Audit + Sys.Modify must be at root
|
**Required for proxmox_download_file — must be at root:**
|
||||||
|
|
||||||
|
```bash
|
||||||
pveum role add SysDownloadRole --privs "Sys.Audit,Sys.Modify"
|
pveum role add SysDownloadRole --privs "Sys.Audit,Sys.Modify"
|
||||||
pveum acl modify / --user terraform@pve --role SysDownloadRole
|
```
|
||||||
|
|
||||||
# SDN — token needs explicit ACL (separate from user ACL)
|
```bash
|
||||||
pveum acl modify /sdn/zones/localnetwork/vmbr0 \
|
pveum acl modify / --user terraform@pve --role SysDownloadRole
|
||||||
--token terraform@pve!terraform --role TerraformRole
|
```
|
||||||
|
|
||||||
|
**SDN — token needs explicit ACL (separate from user ACL):**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
pveum acl modify /sdn/zones/localnetwork/vmbr0 --token terraform@pve!terraform --role TerraformRole
|
||||||
```
|
```
|
||||||
|
|
||||||
Update `cluster-init/terraform/.env` with the token secret.
|
Update `cluster-init/terraform/.env` with the token secret.
|
||||||
@@ -128,7 +166,13 @@ Update `cluster-init/terraform/.env` with the token secret.
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd ~/k8s/cluster-init/terraform
|
cd ~/k8s/cluster-init/terraform
|
||||||
|
```
|
||||||
|
|
||||||
|
```bash
|
||||||
source .env
|
source .env
|
||||||
|
```
|
||||||
|
|
||||||
|
```bash
|
||||||
terraform init
|
terraform init
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -136,6 +180,9 @@ terraform init
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
source .env
|
source .env
|
||||||
|
```
|
||||||
|
|
||||||
|
```bash
|
||||||
terraform plan
|
terraform plan
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -148,22 +195,23 @@ Expected: 5 resources to create
|
|||||||
|
|
||||||
### 3.3 Apply
|
### 3.3 Apply
|
||||||
|
|
||||||
|
> Must use `-parallelism=1` — parallel disk clones to xpen NFS cause storage lock timeout.
|
||||||
|
> Takes ~10-15 min (cloud image download ~1 min + 4 VMs × ~3 min each).
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
terraform apply -parallelism=1
|
terraform apply -parallelism=1
|
||||||
```
|
```
|
||||||
|
|
||||||
> Must use `-parallelism=1` — parallel disk clones to xpen NFS cause storage lock timeout.
|
|
||||||
|
|
||||||
Takes ~10-15 min (cloud image download ~1 min + 4 VMs × ~3 min each).
|
|
||||||
|
|
||||||
### 3.4 Post-apply state sync
|
### 3.4 Post-apply state sync
|
||||||
|
|
||||||
After first apply, run once to sync pool membership state:
|
After first apply, run once to sync pool membership state:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
terraform apply -refresh-only
|
terraform apply -refresh-only
|
||||||
```
|
```
|
||||||
|
|
||||||
Type `yes`. Then verify no further changes:
|
Type `yes`. Then verify no further changes:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
terraform plan
|
terraform plan
|
||||||
```
|
```
|
||||||
@@ -171,45 +219,80 @@ terraform plan
|
|||||||
### 3.5 Verify VMs
|
### 3.5 Verify VMs
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
ssh -i cluster-init/terraform/k8s_terraform cloud-user@192.168.1.31
|
ssh -i ~/k8s/cluster-init/terraform/k8s_terraform cloud-user@192.168.1.31
|
||||||
hostname # master01
|
```
|
||||||
free -h # Swap: 0 (disabled by cloud-init)
|
|
||||||
|
```bash
|
||||||
|
hostname
|
||||||
|
```
|
||||||
|
|
||||||
|
```bash
|
||||||
|
free -h
|
||||||
```
|
```
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Phase 4 — Configure k8s with Ansible
|
## Phase 4 — Configure k8s with Ansible
|
||||||
|
|
||||||
> Playbooks in `cluster-init/ansible/`
|
### 4.1 Run all playbooks (one command)
|
||||||
|
|
||||||
### 4.1 Run all playbooks
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd ~/k8s/cluster-init/ansible
|
cd ~/k8s/cluster-init/ansible
|
||||||
|
```
|
||||||
|
|
||||||
# All nodes — OS prerequisites (swap, kernel modules, sysctl)
|
```bash
|
||||||
|
ansible-playbook -i inventory.ini cluster.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
### 4.2 Or run individual playbooks
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd ~/k8s/cluster-init/ansible
|
||||||
|
```
|
||||||
|
|
||||||
|
All nodes — OS prerequisites (swap, kernel modules, sysctl):
|
||||||
|
|
||||||
|
```bash
|
||||||
ansible-playbook -i inventory.ini playbooks/01-common.yml
|
ansible-playbook -i inventory.ini playbooks/01-common.yml
|
||||||
|
```
|
||||||
|
|
||||||
# All nodes — /etc/hosts (nodes know each other by hostname)
|
All nodes — /etc/hosts:
|
||||||
|
|
||||||
|
```bash
|
||||||
ansible-playbook -i inventory.ini playbooks/02-hosts.yml
|
ansible-playbook -i inventory.ini playbooks/02-hosts.yml
|
||||||
|
```
|
||||||
|
|
||||||
# All nodes — container runtime (containerd)
|
All nodes — container runtime (containerd):
|
||||||
|
|
||||||
|
```bash
|
||||||
ansible-playbook -i inventory.ini playbooks/03-containerd.yml
|
ansible-playbook -i inventory.ini playbooks/03-containerd.yml
|
||||||
|
```
|
||||||
|
|
||||||
# All nodes — kubeadm/kubelet/kubectl
|
All nodes — kubeadm/kubelet/kubectl:
|
||||||
|
|
||||||
|
```bash
|
||||||
ansible-playbook -i inventory.ini playbooks/04-kube-tools.yml
|
ansible-playbook -i inventory.ini playbooks/04-kube-tools.yml
|
||||||
|
```
|
||||||
|
|
||||||
# master01 only — init cluster + CNI
|
master01 only — init cluster + CNI:
|
||||||
|
|
||||||
|
```bash
|
||||||
ansible-playbook -i inventory.ini playbooks/05-master-init.yml
|
ansible-playbook -i inventory.ini playbooks/05-master-init.yml
|
||||||
|
```
|
||||||
|
|
||||||
# worker01/02/03 — join cluster
|
worker01/02/03 — join cluster:
|
||||||
|
|
||||||
|
```bash
|
||||||
ansible-playbook -i inventory.ini playbooks/06-worker-join.yml
|
ansible-playbook -i inventory.ini playbooks/06-worker-join.yml
|
||||||
```
|
```
|
||||||
|
|
||||||
### 4.2 Verify cluster (from master01)
|
### 4.3 Verify cluster (from master01)
|
||||||
|
|
||||||
|
```bash
|
||||||
|
ssh -i ~/k8s/cluster-init/terraform/k8s_terraform cloud-user@192.168.1.31
|
||||||
|
```
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
ssh -i ../terraform/k8s_terraform cloud-user@192.168.1.31
|
|
||||||
kubectl get nodes
|
kubectl get nodes
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -222,10 +305,18 @@ All 4 nodes should be `Ready`.
|
|||||||
### 5.1 Install kubectl
|
### 5.1 Install kubectl
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Match the cluster version (1.32)
|
|
||||||
curl -LO "https://dl.k8s.io/release/v1.32.0/bin/linux/amd64/kubectl"
|
curl -LO "https://dl.k8s.io/release/v1.32.0/bin/linux/amd64/kubectl"
|
||||||
|
```
|
||||||
|
|
||||||
|
```bash
|
||||||
chmod +x kubectl
|
chmod +x kubectl
|
||||||
|
```
|
||||||
|
|
||||||
|
```bash
|
||||||
sudo mv kubectl /usr/local/bin/kubectl
|
sudo mv kubectl /usr/local/bin/kubectl
|
||||||
|
```
|
||||||
|
|
||||||
|
```bash
|
||||||
kubectl version --client
|
kubectl version --client
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -233,9 +324,10 @@ kubectl version --client
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
mkdir -p ~/.kube
|
mkdir -p ~/.kube
|
||||||
scp -i ~/k8s/cluster-init/terraform/k8s_terraform \
|
```
|
||||||
cloud-user@192.168.1.31:/home/cloud-user/.kube/config \
|
|
||||||
~/.kube/config
|
```bash
|
||||||
|
scp -i ~/k8s/cluster-init/terraform/k8s_terraform cloud-user@192.168.1.31:/home/cloud-user/.kube/config ~/.kube/config
|
||||||
```
|
```
|
||||||
|
|
||||||
### 5.3 Add cluster nodes to client /etc/hosts
|
### 5.3 Add cluster nodes to client /etc/hosts
|
||||||
@@ -255,6 +347,9 @@ EOF
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
kubectl get nodes
|
kubectl get nodes
|
||||||
|
```
|
||||||
|
|
||||||
|
```bash
|
||||||
kubectl get pods -A
|
kubectl get pods -A
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -264,7 +359,13 @@ kubectl get pods -A
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd ~/k8s/cluster-init/terraform
|
cd ~/k8s/cluster-init/terraform
|
||||||
|
```
|
||||||
|
|
||||||
|
```bash
|
||||||
source .env
|
source .env
|
||||||
|
```
|
||||||
|
|
||||||
|
```bash
|
||||||
terraform destroy -parallelism=1
|
terraform destroy -parallelism=1
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -275,13 +376,22 @@ Destroys all 4 VMs + cloud image file. Re-run Phase 3 to rebuild from scratch.
|
|||||||
## SSH access
|
## SSH access
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
ssh -i ~/k8s/cluster-init/terraform/k8s_terraform cloud-user@192.168.1.31 # master01
|
ssh -i ~/k8s/cluster-init/terraform/k8s_terraform cloud-user@192.168.1.31
|
||||||
ssh -i ~/k8s/cluster-init/terraform/k8s_terraform cloud-user@192.168.1.35 # worker01
|
|
||||||
ssh -i ~/k8s/cluster-init/terraform/k8s_terraform cloud-user@192.168.1.36 # worker02
|
|
||||||
ssh -i ~/k8s/cluster-init/terraform/k8s_terraform cloud-user@192.168.1.37 # worker03
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Or add to `~/.ssh/config`:
|
```bash
|
||||||
|
ssh -i ~/k8s/cluster-init/terraform/k8s_terraform cloud-user@192.168.1.35
|
||||||
|
```
|
||||||
|
|
||||||
|
```bash
|
||||||
|
ssh -i ~/k8s/cluster-init/terraform/k8s_terraform cloud-user@192.168.1.36
|
||||||
|
```
|
||||||
|
|
||||||
|
```bash
|
||||||
|
ssh -i ~/k8s/cluster-init/terraform/k8s_terraform cloud-user@192.168.1.37
|
||||||
|
```
|
||||||
|
|
||||||
|
Or add to `~/.ssh/config` for shorthand (`ssh master01`, `ssh worker01`, etc.):
|
||||||
|
|
||||||
```
|
```
|
||||||
Host master01
|
Host master01
|
||||||
@@ -305,15 +415,13 @@ Host worker03
|
|||||||
IdentityFile ~/k8s/cluster-init/terraform/k8s_terraform
|
IdentityFile ~/k8s/cluster-init/terraform/k8s_terraform
|
||||||
```
|
```
|
||||||
|
|
||||||
Then: `ssh master01`, `ssh worker01`, etc.
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## File reference
|
## File reference
|
||||||
|
|
||||||
```
|
```
|
||||||
k8s/cluster-init/
|
k8s/cluster-init/
|
||||||
├── RUNBOOK.md
|
├── README.md
|
||||||
├── terraform/
|
├── terraform/
|
||||||
│ ├── .env # Terraform credentials (gitignored)
|
│ ├── .env # Terraform credentials (gitignored)
|
||||||
│ ├── .env.example # template — safe to commit
|
│ ├── .env.example # template — safe to commit
|
||||||
@@ -327,6 +435,7 @@ k8s/cluster-init/
|
|||||||
│ ├── k8s_terraform.pub # SSH public key (gitignored)
|
│ ├── k8s_terraform.pub # SSH public key (gitignored)
|
||||||
│ └── .gitignore
|
│ └── .gitignore
|
||||||
└── ansible/
|
└── ansible/
|
||||||
|
├── cluster.yml # run all playbooks in one command
|
||||||
├── inventory.ini
|
├── inventory.ini
|
||||||
├── ansible.cfg
|
├── ansible.cfg
|
||||||
└── playbooks/
|
└── playbooks/
|
||||||
|
|||||||
7
ansible/cluster.yml
Normal file
7
ansible/cluster.yml
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
---
|
||||||
|
- import_playbook: playbooks/01-common.yml
|
||||||
|
- import_playbook: playbooks/02-hosts.yml
|
||||||
|
- import_playbook: playbooks/03-containerd.yml
|
||||||
|
- import_playbook: playbooks/04-kube-tools.yml
|
||||||
|
- import_playbook: playbooks/05-master-init.yml
|
||||||
|
- import_playbook: playbooks/06-worker-join.yml
|
||||||
Reference in New Issue
Block a user