# Install Network UPS Tools
apt install -y nut nut-client
 
# Configure UPS (CyberPower example)
cat > /etc/nut/ups.conf << 'EOF'
[myups]
  driver = usbhid-ups
  port = auto
  desc = "CyberPower UPS"
EOF
 
# Configure NUT mode
cat > /etc/nut/nut.conf << 'EOF'
MODE=netserver
EOF
 
# Configure monitoring
cat > /etc/nut/upsmon.conf << 'EOF'
MONITOR myups@localhost 1 admin secret master
SHUTDOWNCMD "/sbin/shutdown -h now"
POLLFREQ 5
POLLFREQALERT 2
HOSTSYNC 15
DEADTIME 15
FINALDELAY 5
EOF
 
# Start NUT
systemctl enable nut-server nut-monitor
systemctl start nut-server nut-monitor

# In TrueNAS shell - create encrypted dataset
zfs create -o encryption=aes-256-gcm -o keylocation=file:///root/tank-documents.key -o keyformat=raw -o recordsize=128K -o quota=500G tank/documents-encrypted
 
# Or via TrueNAS UI: Storage -> tank -> Add Dataset -> Encryption Options -> Enable
# Key type: Passphrase or Auto-generated key
# Auto-unlock: Yes (key stored on TrueNAS boot disk - unlocks at boot)

# On Proxmox host, verify NFSv4 is in use
mount | grep nfs
# Should show: type nfs4
 
# On TrueNAS, verify NFSv4 is active
nfsstat -s | head -5

  pool: tank
 state: ONLINE
 
        NAME                                    STATE     READ WRITE CKSUM
        tank                                    ONLINE       0     0     0
          mirror-0                              ONLINE       0     0     0
            ata-WDC_WD80EFAX_<SERIAL1>          ONLINE       0     0     0    <- SATA3
            ata-WDC_WD80EFAX_<SERIAL2>          ONLINE       0     0     0    <- SATA4
        spares
          ata-WDC_WD80EFAX_<SERIAL3>            AVAIL                         <- SATA5 (idle)

  pool: tank
 state: DEGRADED
status: One or more devices has been removed by the FMA.
 
        NAME                                    STATE     READ WRITE CKSUM
        tank                                    DEGRADED     0     0     0
          mirror-0                              DEGRADED     0     0     0
            spare-0                             DEGRADED     0     0     0
              ata-WDC_WD80EFAX_<SERIAL1>        FAULTED      0     0     0    <- SATA3 (failed)
              ata-WDC_WD80EFAX_<SERIAL3>        ONLINE       0     0     0    <- SATA5 (resilvering)
            ata-WDC_WD80EFAX_<SERIAL2>          ONLINE       0     0     0    <- SATA4
        spares
          ata-WDC_WD80EFAX_<SERIAL3>            INUSE     currently in use    <- SATA5 (auto-activated)

# Option A: Replace the failed drive in-place (detaches spare automatically)
zpool replace tank ata-WDC_WD80EFAX_<SERIAL1_OLD> ata-WDC_WD80EFAX_<SERIAL_NEW>
 
# Wait for resilver to complete
zpool status tank
 
# The spare auto-detaches and returns to AVAIL status
 
# Option B: Detach the failed drive, add new drive as spare
zpool detach tank ata-WDC_WD80EFAX_<SERIAL1_OLD>
zpool add tank spare ata-WDC_WD80EFAX_<SERIAL_NEW>

# 1. Pool health - must show ONLINE, mirror-0 + spare AVAIL
ssh root@192.168.50.50
zpool status tank
zpool list tank
 
# 2. All datasets exist
zfs list -r tank
 
# 3. Compression enabled
zfs get compression tank

# From Proxmox host (192.168.50.20):
 
# 4. NFS exports visible
showmount -e 192.168.50.50
 
# 5. NFS mounts working
df -h | grep truenas
ls -la /mnt/truenas-models/
ls -la /mnt/truenas-media/

# 6. Models accessible inside GPU containers
pct exec 100 -- ls -la /models/
pct exec 102 -- ls -la /models/
 
# 7. vzdump backup works - test with a small container
vzdump 105 --storage truenas-vzdump --mode snapshot --compress zstd
ls -la /mnt/truenas-vzdump/  # or check in Proxmox UI

# 8. Snapshot schedules - check in TrueNAS UI: Data Protection -> Periodic Snapshots
# Verify at least one snapshot exists after scheduled time:
zfs list -t snapshot -r tank | head -20
 
# 9. SMART tests running - check in TrueNAS UI: Storage -> Disks -> each disk -> S.M.A.R.T. Results
 
# 10. Email alerts working - send test alert from TrueNAS UI: System -> Alert Services -> test

# Simulate drive failure by offlining one mirror leg
# From TrueNAS shell:
zpool offline tank ata-WDC_WD80EFAX_<SERIAL1>
 
# Watch spare auto-activate
watch -n 5 'zpool status tank'
# Expected: spare-0 appears, resilver begins
 
# After confirming spare activated, bring the drive back
zpool online tank ata-WDC_WD80EFAX_<SERIAL1>
 
# Detach the spare and re-add it (this reverses the test)
zpool detach tank ata-WDC_WD80EFAX_<SERIAL3>
zpool add tank spare ata-WDC_WD80EFAX_<SERIAL3>
 
# Verify clean state
zpool status tank
# Should show: mirror-0 (SERIAL1 + SERIAL2 ONLINE), spare (SERIAL3 AVAIL)

# 11. Full reboot test
# Reboot Proxmox host:
reboot
 
# After boot, verify:
# a) TrueNAS VM starts first (check VM 300 uptime in Proxmox UI)
# b) NFS mounts auto-recover:
df -h | grep truenas
# c) All containers are running:
pct list
# d) Models still accessible:
pct exec 100 -- ls /models/

# Check VM status
qm status 300
 
# Check for errors in VM log
tail -50 /var/log/pve/qemu-server/300.log
 
# Common issue: EFI disk missing
qm config 300 | grep efidisk
 
# If missing, re-add:
qm set 300 --efidisk0 local-zfs:1,efitype=4m,pre-enrolled-keys=0

# From Proxmox host:
 
# Check if TrueNAS is reachable
ping 192.168.50.50
 
# Check if NFS service is running on TrueNAS
ssh root@192.168.50.50 systemctl status nfs-server
 
# Check exports
showmount -e 192.168.50.50
 
# If mount hangs, try mounting manually with verbose output
mount -v -t nfs4 192.168.50.50:/mnt/tank/models /mnt/truenas-models
 
# If boot fails due to NFS not ready, add 'nofail' to fstab:
# 192.168.50.50:/mnt/tank/models /mnt/truenas-models nfs4 rw,hard,intr,_netdev,nofail 0 0

# Check pool status inside TrueNAS
zpool status tank
 
# If a drive went offline during reboot:
zpool online tank ata-WDC_WD80EFAX_<SERIAL>
 
# If drive passthrough changed (wrong /dev/sdX mapping):
# Check Proxmox host - verify by-id paths are still valid
ls -la /dev/disk/by-id/ | grep WD
 
# If disk was removed or moved, re-add passthrough:
qm set 300 -scsiN /dev/disk/by-id/ata-WDC_WD80EFAX_<SERIAL>

# Verify SMART passthrough is working
ssh root@192.168.50.50 smartctl -a /dev/sdX  # for each data drive
 
# If SMART fails: the issue is likely virtio abstraction
# Verify drives were passed through as SCSI (not virtio):
qm config 300 | grep scsi
 
# Re-add with explicit SCSI if needed
qm set 300 -scsi1 /dev/disk/by-id/ata-WDC_WD80EFAX_<SERIAL>

# Check if NFS is mounted on host
df -h | grep truenas
 
# If not, manually mount
mount -a
 
# If TrueNAS wasn't ready yet (boot timing issue):
# Increase TrueNAS startup delay
qm set 300 --startup order=1,up=180  # 3 minutes instead of 2
 
# Or add a systemd mount dependency for containers

# Check which process is consuming RAM
ps aux --sort=-%mem | head -20
 
# Check ZFS ARC sizes
# On Proxmox host:
grep "^size" /proc/spl/kstat/zfs/arcstats
 
# On TrueNAS:
ssh root@192.168.50.50 arc_summary | head -30
 
# If host ARC is too high:
echo 4294967296 > /sys/module/zfs/parameters/zfs_arc_max
 
# If TrueNAS ARC is too aggressive:
# TrueNAS UI -> System -> Advanced -> Sysctl -> add:
# vfs.zfs.arc_max = 12884901888  (12GB, leaving 4GB for OS)

# Install rclone on TrueNAS
apt install rclone  # or download from rclone.io
 
# Configure Backblaze B2 (cheapest cloud storage - about $6/TB/month)
rclone config
# Follow prompts to add B2 remote
 
# Sync critical datasets
rclone sync /mnt/tank/documents b2:my-bucket/documents --transfers 8 --progress
rclone sync /mnt/tank/backups/vzdump b2:my-bucket/vzdump --transfers 4 --progress
 
# Schedule via crontab (weekly offsite sync)
0 4 * * 0 /usr/local/bin/rclone sync /mnt/tank/documents b2:my-bucket/documents --log-file /var/log/rclone.log

# On Proxmox host - create VLAN-aware bridge
# /etc/network/interfaces:
auto vmbr0.100
iface vmbr0.100 inet static
  address 10.100.0.1/24
  vlan-raw-device vmbr0
 
# Add second NIC to TrueNAS VM on VLAN 100
qm set 300 --net1 virtio,bridge=vmbr0,tag=100
 
# Configure TrueNAS: second NIC with IP 10.100.0.50
# NFS traffic goes over VLAN 100 (isolated from general LAN)

# Option B: Proxmox LXC (CT 106)
pct create 106 local:vztmpl/debian-12-standard_12.1-1_amd64.tar.zst \
  --hostname jellyfin \
  --memory 4096 \
  --cores 2 \
  --net0 name=eth0,bridge=vmbr0,ip=192.168.50.55/24,gw=192.168.50.1 \
  --onboot 1 \
  --startup order=2,up=30
 
# Mount TrueNAS media via NFS inside container
# Add to /etc/pve/lxc/106.conf:
# mp0: /mnt/truenas-media,mp=/media,ro=1

# Add second mirror vdev - doubles capacity with full redundancy
zpool add tank mirror ata-WDC_WD80EFAX_<SERIAL4> ata-WDC_WD80EFAX_<SERIAL3>
# Converts former spare (SERIAL3) to active + new drive (SERIAL4) as mirror partner
# Result: 2 mirror vdevs = about 16TB usable
# Tradeoff: no hot spare

# Or keep mirror + 2 hot spares
zpool add tank spare ata-WDC_WD80EFAX_<SERIAL4>
# Result: 2-disk mirror + 2 hot spares = about 8TB usable, extreme fault tolerance