Zabbix Server Installation and Configuration: Linux & Windows Agents, Security, Backup and Recovery

sudo apt update
sudo apt full-upgrade
sudo hostnamectl set-hostname zabbix.example.com
hostnamectl
getent ahostsv4 zabbix.example.com
ip -br address
ip route
sudo apt install -y chrony ufw curl ca-certificates openssl netcat-openbsd
sudo systemctl enable --now chrony
chronyc tracking
chronyc sources -v
timedatectl
test -e /var/run/reboot-required && cat /var/run/reboot-required || true
free -h
df -hT
lsblk -f
sudo aa-status

# Confirm actual SSH port, VPN access and console recovery before enabling UFW.
sudo ufw default deny incoming
sudo ufw default allow outgoing
sudo ufw allow from 192.0.2.0/24 to any port 22 proto tcp
sudo ufw allow from 192.0.2.0/24 to any port 443 proto tcp
sudo ufw allow from 198.51.100.21 to any port 10051 proto tcp
sudo ufw allow from 198.51.100.22 to any port 10051 proto tcp
sudo ufw allow from 203.0.113.20 to any port 10051 proto tcp
sudo ufw enable
sudo ufw status numbered
sudo ss -lntp

cd /tmp
curl --fail --location --proto '=https' --tlsv1.2 -O \
  https://repo.zabbix.com/zabbix/7.0/ubuntu/pool/main/z/zabbix-release/zabbix-release_latest_7.0+ubuntu24.04_all.deb
dpkg-deb --info zabbix-release_latest_7.0+ubuntu24.04_all.deb
sudo dpkg -i zabbix-release_latest_7.0+ubuntu24.04_all.deb
sudo apt update
apt-cache policy zabbix-server-pgsql zabbix-agent2 zabbix-frontend-php postgresql-16 nginx php8.3-fpm
# Verify official stable 7.0.x candidates, no beta/RC/devel.
sudo apt install -y postgresql-16 postgresql-client-16 nginx php8.3-fpm \
  php8.3-pgsql php8.3-bcmath php8.3-mbstring php8.3-xml php8.3-gd php8.3-curl \
  zabbix-server-pgsql zabbix-frontend-php zabbix-nginx-conf zabbix-sql-scripts zabbix-agent2 zabbix-get
sudo systemctl stop zabbix-server zabbix-agent2 nginx
sudo systemctl enable --now postgresql
pg_lsclusters
dpkg-query -W zabbix-server-pgsql zabbix-agent2 zabbix-frontend-php postgresql-16 nginx php8.3-fpm
zabbix_server -V
zabbix_agent2 -V
php -v
nginx -v

sudo -u postgres psql -X -v ON_ERROR_STOP=1 <<'SQL'
SET password_encryption = 'scram-sha-256';
CREATE ROLE zabbix LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION;
SQL
# Interactive prompt: no password in shell history.
sudo -u postgres psql -X -c '\password zabbix'
sudo -u postgres createdb -O zabbix -E UTF8 -T template0 zabbix
sudo -u postgres psql -X -d zabbix -c 'REVOKE ALL ON DATABASE zabbix FROM PUBLIC;'
sudo -u postgres psql -X -d zabbix -c 'REVOKE CREATE ON SCHEMA public FROM PUBLIC;'
sudo -u postgres psql -X -d zabbix -c 'GRANT ALL ON SCHEMA public TO zabbix;'
sudoedit /etc/postgresql/16/main/postgresql.conf
sudoedit /etc/postgresql/16/main/pg_hba.conf

# postgresql.conf
listen_addresses = '127.0.0.1'
password_encryption = 'scram-sha-256'
# pg_hba.conf: place before broader host rules; preserve local admin peer access
local all     postgres                   peer
local zabbix  zabbix                     peer
host  zabbix  zabbix  127.0.0.1/32        scram-sha-256
host  all     all     0.0.0.0/0           reject
host  all     all     ::/0                reject

sudo pg_ctlcluster 16 main restart
sudo -u postgres psql -X -c "SELECT line_number,error FROM pg_hba_file_rules WHERE error IS NOT NULL;"
sudo -u postgres psql -X -c 'SHOW listen_addresses;'
sudo -u postgres psql -X -c '\du zabbix'
# INITIALIZATION ONLY: NEW EMPTY database, never before a database restore.
set -o pipefail
zcat /usr/share/zabbix-sql-scripts/postgresql/server.sql.gz | sudo -u zabbix psql -X -v ON_ERROR_STOP=1 -d zabbix
sudo -u zabbix psql -X -d zabbix -c 'SELECT mandatory,optional FROM dbversion;'
psql -h 127.0.0.1 -U zabbix -W -d zabbix -c 'SELECT current_user,current_database();'
sudoedit /etc/zabbix/zabbix_server.conf

# Merge into package file; avoid duplicate active parameters.
DBHost=127.0.0.1
DBName=zabbix
DBUser=zabbix
# Set the actual DBPassword using sudoedit; never include it in downloads.
ListenIP=127.0.0.1,192.0.2.10
ListenPort=10051
# Preserve package LogFile, PidFile, SocketDir and script paths.

sudoedit /etc/zabbix/nginx.conf
sudoedit /etc/zabbix/php-fpm.conf
# Pool: php_value[date.timezone] = Asia/Tehran
# Verify memory_limit >=128M, post_max_size >=16M, max_execution_time >=300,
# max_input_time >=300, upload_max_filesize >=2M.
sudo chown root:zabbix /etc/zabbix/zabbix_server.conf
sudo chmod 0640 /etc/zabbix/zabbix_server.conf
sudo php-fpm8.3 -t
sudo nginx -t
sudo systemctl enable --now php8.3-fpm nginx zabbix-server
sudo systemctl status zabbix-server php8.3-fpm nginx --no-pager
sudo journalctl -u zabbix-server -n 50 --no-pager
sudo tail -n 50 /var/log/zabbix/zabbix_server.log
curl -I -H 'Host: zabbix.example.com' http://127.0.0.1:8080/
# Operator workstation: encrypted tunnel for setup
ssh -L 8080:127.0.0.1:8080 admin@192.0.2.10

sudo apt update
sudo apt install -y zabbix-agent2 openssl
sudo systemctl stop zabbix-agent2
sudo install -d -o root -g zabbix -m 0750 /etc/zabbix/keys
sudo sh -c 'umask 027; openssl rand -hex 32 > /etc/zabbix/keys/agent2.psk'
sudo chown root:zabbix /etc/zabbix/keys/agent2.psk
sudo chmod 0640 /etc/zabbix/keys/agent2.psk
# Adapt the downloaded configuration BEFORE starting the service.
sudo install -o root -g zabbix -m 0640 zabbix-agent2-linux.conf /etc/zabbix/zabbix_agent2.conf
sudoedit /etc/zabbix/zabbix_agent2.conf
sudo ufw allow from 192.0.2.10 to any port 10050 proto tcp
sudo -u zabbix zabbix_agent2 -c /etc/zabbix/zabbix_agent2.conf -t agent.ping
sudo systemctl enable --now zabbix-agent2
sudo systemctl status zabbix-agent2 --no-pager
sudo journalctl -u zabbix-agent2 -n 50 --no-pager
sudo ss -lntp | grep ':10050'
nc -vz 192.0.2.10 10051

# Ubuntu 24.04, Zabbix Agent 2 7.0 LTS. Replace documentation IP/name/identity.
LogType=file
LogFile=/var/log/zabbix/zabbix_agent2.log
LogFileSize=10
Server=192.0.2.10
ServerActive=192.0.2.10:10051
Hostname=linux-app-01
ListenIP=198.51.100.21
ListenPort=10050
TLSConnect=psk
TLSAccept=psk
TLSPSKIdentity=linux-app-01-psk
TLSPSKFile=/etc/zabbix/keys/agent2.psk
DenyKey=system.run[*]
UnsafeUserParameters=0
Include=/etc/zabbix/zabbix_agent2.d/*.conf
Include=/etc/zabbix/zabbix_agent2.d/plugins.d/*.conf
# Explicit read-only key for one approved service; no wildcard shell execution.
UserParameter=custom.service.nginx,/usr/bin/systemctl is-active nginx || true

sudo -u zabbix zabbix_agent2 -c /etc/zabbix/zabbix_agent2.conf -t 'system.uptime'
sudo -u zabbix zabbix_agent2 -c /etc/zabbix/zabbix_agent2.conf -t 'vfs.fs.size[/,pused]'
sudo -u zabbix zabbix_agent2 -c /etc/zabbix/zabbix_agent2.conf -t 'custom.service.nginx'
# From the authorized Server: provision this per-host PSK copy securely, root-only.
sudo zabbix_get -s 198.51.100.21 -p 10050 -k agent.ping \
  --tls-connect psk --tls-psk-identity linux-app-01-psk --tls-psk-file /root/linux-app-01.psk

# Windows Server 2022/2025 x64, Agent 2 7.0 LTS; default MSI destination.
LogType=file
LogFile=C:\ProgramData\Zabbix\zabbix_agent2.log
LogFileSize=10
Server=192.0.2.10
ServerActive=192.0.2.10:10051
Hostname=windows-app-01
ListenPort=10050
TLSConnect=psk
TLSAccept=psk
TLSPSKIdentity=windows-app-01-psk
TLSPSKFile=C:\ProgramData\Zabbix\agent2.psk
DenyKey=system.run[*]
UnsafeUserParameters=0
Include=C:\Program Files\Zabbix Agent 2\zabbix_agent2.d\plugins.d\*.conf

# Elevated PowerShell; approved values come from the deployment record.
$msi = 'C:\Staging\approved-agent2-7.0-windows-amd64-openssl.msi'
Get-FileHash -LiteralPath $msi -Algorithm SHA256
Get-AuthenticodeSignature -LiteralPath $msi | Format-List Status,SignerCertificate
# Run the script with MsiPath, ExpectedSha256, ApprovedSignerThumbprint,
# ConfigPath, PskSourcePath and ServerAddress parameters.
# Silent MSI step AFTER protected PSK/configuration provisioning:
msiexec.exe /i "C:\Staging\approved-agent2.msi" /qn /norestart CONF="C:\ProgramData\Zabbix\install.conf" DONOTSTART=1 /l*v "C:\ProgramData\Zabbix\install.log"

#requires -RunAsAdministrator
<# Fresh-install workflow for Windows Server 2022/2025, Agent 2 7.0.22+.
Download the current stable 7.0 x64 OpenSSL MSI from the official site first.
Requires approved SHA256 AND valid Zabbix Authenticode publisher signature.
Never pass PSK values to msiexec arguments or logs. Existing installs are refused.
#>
[CmdletBinding()]
param(
    [Parameter(Mandatory)][string]$MsiPath,
    [Parameter(Mandatory)][ValidatePattern('^[A-Fa-f0-9]{64}$')][string]$ExpectedSha256,
    [Parameter(Mandatory)][string]$ApprovedSignerThumbprint,
    [Parameter(Mandatory)][string]$ConfigPath,
    [Parameter(Mandatory)][string]$PskSourcePath,
    [string]$ServerAddress = '192.0.2.10'
)
Set-StrictMode -Version Latest
$ErrorActionPreference = 'Stop'
if (Get-Service -Name 'Zabbix Agent 2' -ErrorAction SilentlyContinue) { throw 'Existing Agent 2 found; use a reviewed upgrade workflow.' }
if (Get-Service -Name 'Zabbix Agent' -ErrorAction SilentlyContinue) { throw 'Classic agent exists; review port and service migration first.' }
$msi = (Resolve-Path -LiteralPath $MsiPath).Path
$config = (Resolve-Path -LiteralPath $ConfigPath).Path
$pskSource = (Resolve-Path -LiteralPath $PskSourcePath).Path
foreach ($path in @($msi, $config, $pskSource)) {
    if ((Get-Item -LiteralPath $path).Attributes -band [IO.FileAttributes]::ReparsePoint) { throw 'Reparse-point input refused.' }
    if ($path.Contains('"')) { throw 'Invalid path.' }
}
if ((Get-FileHash -LiteralPath $msi -Algorithm SHA256).Hash -ne $ExpectedSha256) { throw 'MSI hash mismatch.' }
$signature = Get-AuthenticodeSignature -LiteralPath $msi
if ($signature.Status -ne 'Valid' -or $signature.SignerCertificate.Thumbprint -ne $ApprovedSignerThumbprint) { throw 'MSI signature/publisher mismatch.' }
$content = Get-Content -LiteralPath $config -Raw
$parsedAddress = $null
if (-not [System.Net.IPAddress]::TryParse($ServerAddress, [ref]$parsedAddress)) { throw 'ServerAddress must be an approved IP address for the firewall allowlist.' }
if ($content -notmatch ('(?m)^Server=' + [regex]::Escape($ServerAddress) + '\r?$')) { throw 'Configuration Server must match the firewall ServerAddress.' }
if ($content -notmatch ('(?m)^ServerActive=' + [regex]::Escape($ServerAddress) + ':10051\r?$')) { throw 'Configuration ServerActive must match the selected monitoring server.' }
foreach ($setting in @('TLSConnect=psk','TLSAccept=psk','TLSPSKFile=C:\ProgramData\Zabbix\agent2.psk','DenyKey=system.run[*]')) {
    if (-not $content.Contains($setting)) { throw "Required configuration missing: $setting" }
}
$psk = (Get-Content -LiteralPath $pskSource -Raw).Trim()
if ($psk -notmatch '^[a-fA-F0-9]{64}$') { throw 'Expected a unique 256-bit hex PSK.' }
$data = 'C:\ProgramData\Zabbix'
if (Test-Path -LiteralPath $data) { throw 'Existing data directory refused; inspect before installation.' }
New-Item -ItemType Directory -Path $data | Out-Null
& icacls.exe $data /inheritance:r /grant:r '*S-1-5-18:(OI)(CI)F' '*S-1-5-32-544:(OI)(CI)F' | Out-Null
if ($LASTEXITCODE -ne 0) { throw 'ACL setup failed.' }
[IO.File]::WriteAllText("$data\agent2.psk", $psk + "`r`n", [Text.Encoding]::ASCII)
$psk = $null
Copy-Item -LiteralPath $config -Destination "$data\install.conf"
$arguments = "/i `"$msi`" /qn /norestart /l*v `"$data\install.log`" CONF=`"$data\install.conf`" DONOTSTART=1"
$process = Start-Process -FilePath msiexec.exe -ArgumentList $arguments -Wait -PassThru -WindowStyle Hidden
if ($process.ExitCode -notin @(0,3010)) { throw "MSI installation failed: $($process.ExitCode)" }
$target = 'C:\Program Files\Zabbix Agent 2\zabbix_agent2.conf'
Copy-Item -LiteralPath "$data\install.conf" -Destination $target -Force
& icacls.exe $target /inheritance:r /grant:r '*S-1-5-18:F' '*S-1-5-32-544:F' | Out-Null
if ($LASTEXITCODE -ne 0) { throw 'Configuration ACL setup failed.' }
$service = Get-CimInstance Win32_Service -Filter "Name='Zabbix Agent 2'"
if ($service.StartName -notin @('LocalSystem','NT AUTHORITY\SYSTEM')) { throw 'Adjust ACLs for the actual service account before startup.' }
New-NetFirewallRule -Name 'Zabbix-Agent2-TLS' -DisplayName 'Zabbix Agent 2 TLS from monitoring server' -Direction Inbound -Action Allow -Protocol TCP -LocalPort 10050 -RemoteAddress $ServerAddress -Profile Domain,Private | Out-Null
# Audit and remove any broad installer-created Agent rule before enabling the service.
Get-NetFirewallRule | Where-Object { $_.DisplayName -like '*Zabbix*' -and $_.Name -ne 'Zabbix-Agent2-TLS' } | Disable-NetFirewallRule
Set-Service -Name 'Zabbix Agent 2' -StartupType Automatic
Start-Service -Name 'Zabbix Agent 2'
Get-Service -Name 'Zabbix Agent 2'
Test-NetConnection -ComputerName $ServerAddress -Port 10051
if ($process.ExitCode -eq 3010) { Write-Warning 'MSI requests a reboot; schedule it and validate service startup afterwards.' }

Get-Service -Name 'Zabbix Agent 2'
Get-CimInstance Win32_Service -Filter "Name='Zabbix Agent 2'" | Select-Object Name,State,StartMode,StartName,PathName
Set-Service -Name 'Zabbix Agent 2' -StartupType Automatic
Restart-Service -Name 'Zabbix Agent 2'
Get-NetTCPConnection -LocalPort 10050 -State Listen
Test-NetConnection -ComputerName 192.0.2.10 -Port 10051
Get-NetFirewallRule -Name 'Zabbix-Agent2-TLS' | Get-NetFirewallAddressFilter
Get-Content 'C:\ProgramData\Zabbix\zabbix_agent2.log' -Tail 50
& 'C:\Program Files\Zabbix Agent 2\zabbix_agent2.exe' -c 'C:\Program Files\Zabbix Agent 2\zabbix_agent2.conf' -t agent.ping
& 'C:\Program Files\Zabbix Agent 2\zabbix_agent2.exe' -c 'C:\Program Files\Zabbix Agent 2\zabbix_agent2.conf' -t 'service.info[Spooler,state]'
Get-WinEvent -FilterHashtable @{LogName='System'; Level=2} -MaxEvents 5

# Inside existing /etc/zabbix/nginx.conf server block:
listen 192.0.2.10:443 ssl;
server_name zabbix.example.com;
ssl_certificate /etc/ssl/zabbix/fullchain.pem;
ssl_certificate_key /etc/ssl/zabbix/privkey.pem;
ssl_protocols TLSv1.2 TLSv1.3;
ssl_session_cache shared:ZabbixTLS:10m;
ssl_session_timeout 1d;
add_header Strict-Transport-Security "max-age=31536000" always;
add_header X-Content-Type-Options "nosniff" always;
add_header Referrer-Policy "same-origin" always;
# Keep packaged root/index/PHP location/deny rules/fastcgi_pass.
# Enable HSTS only after HTTPS and renewal validation.

sudo install -d -o root -g root -m 0700 /etc/ssl/zabbix
# Securely install approved certificate and key before testing Nginx.
sudo chmod 0600 /etc/ssl/zabbix/privkey.pem
sudo chmod 0644 /etc/ssl/zabbix/fullchain.pem
sudo nginx -t
sudo php-fpm8.3 -t
sudo systemctl reload nginx
curl -I --cacert /path/to/enterprise-ca.pem https://zabbix.example.com/
openssl s_client -connect zabbix.example.com:443 -servername zabbix.example.com \
  -CAfile /path/to/enterprise-ca.pem -verify_return_error </dev/null
sudo ss -lntp
sudo ufw status numbered

# Replace PSK settings for a reviewed certificate deployment:
TLSConnect=cert
TLSAccept=cert
TLSCAFile=/etc/zabbix/keys/ca.pem
TLSCertFile=/etc/zabbix/keys/agent-chain.pem
TLSKeyFile=/etc/zabbix/keys/agent.key
TLSServerCertIssuer=CN=Monitoring CA,O=Example
TLSServerCertSubject=CN=zabbix.example.com,O=Example

# Data collection -> Hosts -> Triggers
# Linux disk Warning; use separate Recovery expression below.
min(/linux-app-01/vfs.fs.size[/,pused],5m)>80
# Recovery:
max(/linux-app-01/vfs.fs.size[/,pused],5m)<75
# Windows disk High:
min(/windows-app-01/vfs.fs.size[C:,pused],5m)>90
# Linux CPU High (idle percent below 10 for the whole window):
max(/linux-app-01/system.cpu.util[,idle],5m)<10
# Recovery:
min(/linux-app-01/system.cpu.util[,idle],5m)>20
# Memory Average, less than 10% available:
max(/linux-app-01/vm.memory.size[available],5m)/last(/linux-app-01/vm.memory.size[total])<0.10
# Monitoring path unavailable, High:
nodata(/linux-app-01/agent.ping,5m)=1
# Windows service stopped for three samples, Average:
min(/windows-app-01/service.info[Spooler,state],#3)<>0
# Linux service stopped, Average:
last(/linux-app-01/custom.service.nginx)<>"active"

# Protected mounted /backup volume, on the database host.
sudo install -d -o root -g root -m 0700 /backup/zabbix/manual
sudo bash <<'BASH'
set -Eeuo pipefail
umask 077
stamp=$(date -u +%Y%m%dT%H%M%SZ)
target="/backup/zabbix/manual/$stamp"
mkdir -m 0700 "$target"
runuser -u postgres -- pg_dump -h /var/run/postgresql --role=zabbix \
  -d zabbix -Fc -Z 6 --no-owner --no-acl > "$target/zabbix.dump"
pg_restore --list "$target/zabbix.dump" > "$target/database-toc.txt"
pg_restore --file=/dev/null "$target/zabbix.dump"
cd "$target"
sha256sum zabbix.dump database-toc.txt > SHA256SUMS
sha256sum --check SHA256SUMS
BASH

# DESIGN starting point for a separately reviewed pgBackRest deployment:
# /etc/pgbackrest/pgbackrest.conf
[global]
repo1-path=/srv/pgbackrest
repo1-retention-full=2
[zabbix]
pg1-path=/var/lib/postgresql/16/main
# postgresql.conf: archive_mode change requires restart
wal_level = replica
archive_mode = on
archive_command = 'pgbackrest --stanza=zabbix archive-push %p'
# AFTER dedicated repository ownership, remote access and encryption setup:
# sudo -u postgres pgbackrest --stanza=zabbix stanza-create
# sudo -u postgres pgbackrest --stanza=zabbix check
# sudo -u postgres pgbackrest --stanza=zabbix --type=full backup
# sudo -u postgres pgbackrest --stanza=zabbix info
# ISOLATED EMPTY CLUSTER ONLY, destructive physical restore workflow:
# pgbackrest --stanza=zabbix --type=time --target='2026-10-09 01:30:00+00' --target-action=pause restore

#!/usr/bin/env bash
# Ubuntu 24.04 / PostgreSQL 16. Run as root through the supplied systemd unit.
# EnvironmentFile supplies settings; this script never sources executable config.
set -Eeuo pipefail
umask 077
export PATH=/usr/sbin:/usr/bin:/sbin:/bin
BACKUP_ROOT=${BACKUP_ROOT:-/backup/zabbix}
RETENTION_DAYS=${RETENTION_DAYS:-14}
MIN_FREE_MB=${MIN_FREE_MB:-4096}
SPACE_FACTOR=${SPACE_FACTOR:-2}
PGDATABASE=${PGDATABASE:-zabbix}
PGROLE=${PGROLE:-zabbix}
OFFSITE_REQUIRED=${OFFSITE_REQUIRED:-yes}
FAILURE_HOOK=${FAILURE_HOOK:-/usr/local/sbin/zabbix-backup-notify}
stage=''
log_file=''
log() { printf '%s %s\n' "$(date -u +%FT%TZ)" "$*"; }
finish() {
    local rc=$?
    trap - EXIT
    if (( rc != 0 )); then
        log "FAILED exit=$rc; incomplete backups retained for diagnosis; retention skipped"
        if [[ -n "$log_file" && -x "$FAILURE_HOOK" ]]; then
            timeout 30 "$FAILURE_HOOK" "$rc" "$log_file" || log 'Failure notification hook failed'
        fi
    fi
    exit "$rc"
}
trap finish EXIT
[[ $EUID -eq 0 ]] || { log 'Root is required'; exit 77; }
[[ "$BACKUP_ROOT" =~ ^/backup/[a-zA-Z0-9/_-]+$ && "$BACKUP_ROOT" != *'..'* ]] || exit 64
for number in "$RETENTION_DAYS" "$MIN_FREE_MB" "$SPACE_FACTOR"; do
    [[ "$number" =~ ^[1-9][0-9]*$ ]] || exit 64
done
[[ "$PGDATABASE" =~ ^[a-zA-Z0-9_]+$ && "$PGROLE" =~ ^[a-zA-Z0-9_]+$ ]] || exit 64
[[ "$OFFSITE_REQUIRED" == yes || "$OFFSITE_REQUIRED" == no ]] || exit 64
for command in flock runuser pg_dump pg_restore psql tar gzip sha256sum df du find timeout realpath mountpoint; do
    command -v "$command" >/dev/null || { log "Missing command: $command"; exit 69; }
done
install -d -o root -g root -m 0700 /var/lib/zabbix-backup
exec 9>/var/lib/zabbix-backup/backup.lock
flock -n 9 || { log 'Another backup is running'; exit 75; }
# A dedicated mounted backup volume prevents filling the OS root filesystem.
mountpoint -q /backup || { log '/backup must be a mounted backup volume'; exit 73; }
install -d -m 0700 "$BACKUP_ROOT" /var/log/zabbix-backup
[[ "$(realpath -e "$BACKUP_ROOT")" == "$BACKUP_ROOT" ]] || exit 64
[[ "$(stat -c %u "$BACKUP_ROOT")" == 0 && "$(stat -c %a "$BACKUP_ROOT")" == 700 ]] || exit 77
stamp=$(date -u +%Y%m%dT%H%M%SZ)
log_file="/var/log/zabbix-backup/$stamp.log"
exec > >(tee -a "$log_file") 2>&1
log 'Backup started'
if [[ "$OFFSITE_REQUIRED" == yes ]]; then
    command -v restic >/dev/null || exit 69
    : "${RESTIC_REPOSITORY:?Set an encrypted off-site restic repository}"
    : "${RESTIC_PASSWORD_FILE:?Set a protected restic password file}"
    [[ -f "$RESTIC_PASSWORD_FILE" && ! -L "$RESTIC_PASSWORD_FILE" ]] || exit 77
    [[ "$(stat -c %u "$RESTIC_PASSWORD_FILE")" == 0 && "$(stat -c %a "$RESTIC_PASSWORD_FILE")" == 600 ]] || exit 77
fi
# These directories contain secrets. Do not publish this archive or put it in Git.
paths=(etc/zabbix etc/nginx etc/php/8.3/fpm etc/postgresql/16/main etc/ssl/zabbix
       etc/zabbix-backup etc/systemd/system/zabbix-backup.service
       etc/systemd/system/zabbix-backup.timer etc/systemd/system/zabbix-backup-failure.service
       usr/local/sbin/zabbix-backup.sh)
for optional in etc/letsencrypt usr/lib/zabbix/alertscripts usr/lib/zabbix/externalscripts usr/local/lib/zabbix usr/local/sbin/zabbix-backup-notify etc/logrotate.d/zabbix-backup; do
    [[ ! -e "/$optional" ]] || paths+=("$optional")
done
for path in "${paths[@]}"; do [[ -e "/$path" ]] || { log "Missing required path: /$path"; exit 66; }; done
db_bytes=$(runuser -u postgres -- psql -X -At -v ON_ERROR_STOP=1 -d "$PGDATABASE" -c 'SELECT pg_database_size(current_database())')
[[ "$db_bytes" =~ ^[0-9]+$ ]] || exit 65
config_bytes=$(du -scb "${paths[@]/#//}" | tail -n 1 | cut -f 1)
required_mb=$(( (db_bytes * SPACE_FACTOR + config_bytes) / 1048576 + MIN_FREE_MB ))
free_mb=$(df -Pm "$BACKUP_ROOT" | awk 'NR==2 {print $4}')
(( free_mb >= required_mb )) || { log "Insufficient space: free=${free_mb}MiB required=${required_mb}MiB"; exit 73; }
stage=$(mktemp -d "$BACKUP_ROOT/.incomplete-$stamp-XXXXXX")
# The database role owns the schema but has no superuser/createdb/createrole rights.
# pg_dump's MVCC snapshot is consistent while ordinary monitoring writes continue.
runuser -u postgres -- pg_dump -h /var/run/postgresql --role="$PGROLE" \
    -d "$PGDATABASE" --format=custom --compress=6 --no-owner --no-acl > "$stage/zabbix.dump"
# tar exits nonzero if selected files change. Schedule configuration changes separately.
tar --acls --xattrs --numeric-owner -czf "$stage/configuration.tar.gz" -C / "${paths[@]}"
dpkg-query -W -f='${Package}\t${Version}\n' 'zabbix*' 'postgresql*' 'nginx*' 'php8.3*' > "$stage/packages.tsv"
pg_restore --list "$stage/zabbix.dump" > "$stage/database-toc.txt"
# Decode all archive data, not just its table of contents; still not a restore test.
pg_restore --file=/dev/null "$stage/zabbix.dump"
gzip -t "$stage/configuration.tar.gz"
tar -tzf "$stage/configuration.tar.gz" > "$stage/configuration-files.txt"
[[ -s "$stage/zabbix.dump" && -s "$stage/configuration.tar.gz" ]] || exit 65
(
    cd "$stage"
    sha256sum zabbix.dump configuration.tar.gz packages.tsv database-toc.txt configuration-files.txt > SHA256SUMS
    sha256sum --check SHA256SUMS
)
date -u +%FT%TZ > "$stage/VERIFIED"
final="$BACKUP_ROOT/$stamp"
[[ ! -e "$final" ]] || exit 73
mv -- "$stage" "$final"
stage=''
if [[ "$OFFSITE_REQUIRED" == yes ]]; then
    # Configure transport authentication separately (SSH agent or restic credentials).
    # Bounded operation fails the job instead of leaving retention running on timeout.
    timeout 6h restic backup --tag zabbix -- "$final"
    timeout 1h restic check
    date -u +%FT%TZ > "$final/OFFSITE_OK"
fi
date -u +%FT%TZ > "$final/SUCCESS"
# Delete only completed, real timestamp directories AFTER a verified successful backup.
# Exact cutoff uses directory names; altered directory mtimes cannot shorten retention.
cutoff=$(date -u -d "$RETENTION_DAYS days ago" +%Y%m%dT%H%M%SZ)
while IFS= read -r -d '' old; do
    name=${old##*/}
    [[ "$name" =~ ^[0-9]{8}T[0-9]{6}Z$ && "$name" < "$cutoff" ]] || continue
    [[ "$old" != "$final" && -f "$old/SUCCESS" && ! -L "$old" ]] || continue
    [[ "$(realpath -e "$old")" == "$BACKUP_ROOT/$name" ]] || exit 64
    rm -rf --one-file-system -- "$old"
done < <(find "$BACKUP_ROOT" -mindepth 1 -maxdepth 1 -type d -print0)
date -u +%FT%TZ > "$BACKUP_ROOT/last-success.txt"
# Log retention is independent of backup retention and runs only on success.
find /var/log/zabbix-backup -maxdepth 1 -type f -name '*.log' -mtime +90 -delete
log "SUCCESS backup=$final offsite=$OFFSITE_REQUIRED"

# Install as /etc/zabbix-backup/backup.env, root:root 0600.
# Plain assignments only; compatible with systemd EnvironmentFile.
BACKUP_ROOT=/backup/zabbix
PGDATABASE=zabbix
PGROLE=zabbix
RETENTION_DAYS=14
MIN_FREE_MB=4096
SPACE_FACTOR=2
OFFSITE_REQUIRED=yes
# Replace with your actual off-site server; RFC 5737 address is documentation only.
RESTIC_REPOSITORY=sftp:backup@203.0.113.30:/srv/restic/zabbix
RESTIC_PASSWORD_FILE=/etc/zabbix-backup/restic-password
FAILURE_HOOK=/usr/local/sbin/zabbix-backup-notify

[Unit]
Description=Verified PostgreSQL and Zabbix configuration backup
Wants=network-online.target
After=network-online.target postgresql.service
RequiresMountsFor=/backup
OnFailure=zabbix-backup-failure.service

[Service]
Type=oneshot
User=root
Group=root
UMask=0077
StateDirectory=zabbix-backup
StateDirectoryMode=0700
EnvironmentFile=/etc/zabbix-backup/backup.env
ExecStart=/usr/local/sbin/zabbix-backup.sh
TimeoutStartSec=8h
Nice=10
IOSchedulingClass=best-effort
IOSchedulingPriority=7
NoNewPrivileges=true
PrivateTmp=true
ProtectSystem=strict
ProtectHome=read-only
ReadWritePaths=/backup /var/log/zabbix-backup /var/lib/zabbix-backup /var/cache/restic
ProtectKernelTunables=true
ProtectKernelModules=true
ProtectControlGroups=true
RestrictSUIDSGID=true
LockPersonality=true

[Unit]
Description=Daily Zabbix backup at 02:15 UTC

[Timer]
OnCalendar=*-*-* 02:15:00 UTC
RandomizedDelaySec=10m
Persistent=true
Unit=zabbix-backup.service

[Install]
WantedBy=timers.target

[Unit]
Description=Notify independent operations channel of Zabbix backup failure

[Service]
Type=oneshot
User=root
UMask=0077
ExecStart=/usr/local/sbin/zabbix-backup-notify 1 journal:zabbix-backup.service
NoNewPrivileges=true
PrivateTmp=true
ProtectSystem=strict
ProtectHome=true

#!/usr/bin/env bash
# Requires mailutils and a separately configured authenticated TLS mail relay.
# Must be root-owned 0750; edit the destination before operational acceptance.
set -euo pipefail
umask 077
rc=${1:-1}
reference=${2:-journal:zabbix-backup.service}
command -v mail >/dev/null || { logger -p daemon.err 'Zabbix backup: notification transport unavailable'; exit 69; }
printf 'Zabbix backup failed on %s. Exit: %s. Evidence: %s\nInspect locally; no credentials or logs are attached.\n' \
    "$(hostname)" "$rc" "$reference" | mail -s 'Zabbix backup failure' ops@example.com

sudo apt install -y restic mailutils
# Mount encrypted /backup through a reviewed fstab entry first.
mountpoint /backup
sudo install -d -m 0700 /backup/zabbix /etc/zabbix-backup
sudo install -d -m 0700 /var/log/zabbix-backup /var/cache/restic
sudo install -o root -g root -m 0750 zabbix-backup.sh /usr/local/sbin/zabbix-backup.sh
sudo install -o root -g root -m 0750 zabbix-backup-notify /usr/local/sbin/zabbix-backup-notify
sudo install -o root -g root -m 0600 zabbix-backup.env /etc/zabbix-backup/backup.env
sudo install -o root -g root -m 0644 zabbix-backup.service zabbix-backup.timer zabbix-backup-failure.service /etc/systemd/system/
sudoedit /etc/zabbix-backup/backup.env
sudoedit /usr/local/sbin/zabbix-backup-notify
# Provision restic-password root:root 0600, SSH key and verified known_hosts.
# Never disable host-key verification; keep recovery keys separately in a vault.
sudo systemctl daemon-reload
sudo systemd-analyze verify /etc/systemd/system/zabbix-backup.service /etc/systemd/system/zabbix-backup.timer /etc/systemd/system/zabbix-backup-failure.service
sudo systemctl start zabbix-backup.service
sudo systemctl show zabbix-backup.service -p Result -p ExecMainStatus
sudo journalctl -u zabbix-backup.service -n 100 --no-pager
sudo cat /backup/zabbix/last-success.txt
sudo systemctl enable --now zabbix-backup.timer
systemctl list-timers --all zabbix-backup.timer

# Protected root session; use the SAME non-secret settings as backup.env.
export RESTIC_REPOSITORY='sftp:backup@203.0.113.30:/srv/restic/zabbix'
export RESTIC_PASSWORD_FILE=/etc/zabbix-backup/restic-password
restic init       # ONLY for a newly provisioned empty repository, once
restic snapshots
restic check
# Schedule a separate full-data verification:
restic check --read-data
# Separate approved remote retention policy AFTER verified snapshots:
# restic forget --tag zabbix --keep-daily 14 --keep-weekly 8 --keep-monthly 12 --prune

# ISOLATED REPLACEMENT ONLY; adapt selected-set to retrieved snapshot.
sudo systemctl stop zabbix-server zabbix-agent2 nginx
sudo systemctl disable --now zabbix-backup.timer 2>/dev/null || true
sudo install -d -m 0700 /restore/zabbix
# With protected repository settings, select reviewed snapshot ID:
# restic restore SELECTED_SNAPSHOT_ID --target /restore/zabbix
sudo bash -c 'cd /restore/zabbix/selected-set && sha256sum --check SHA256SUMS'
sudo pg_restore --list /restore/zabbix/selected-set/zabbix.dump
sudo pg_restore --file=/dev/null /restore/zabbix/selected-set/zabbix.dump
sudo gzip -t /restore/zabbix/selected-set/configuration.tar.gz
sudo tar -tzf /restore/zabbix/selected-set/configuration.tar.gz
# 3. Recreate role only if absent, then new EMPTY recovery database.
sudo -u postgres createuser --no-superuser --no-createdb --no-createrole zabbix
sudo -u postgres psql -X -c '\password zabbix'
sudo -u postgres createdb -O zabbix -E UTF8 -T template0 zabbix_recovery
sudo -u postgres psql -X -d zabbix_recovery -c 'REVOKE ALL ON DATABASE zabbix_recovery FROM PUBLIC;'
sudo -u postgres psql -X -d zabbix_recovery -c 'REVOKE CREATE ON SCHEMA public FROM PUBLIC;'
sudo -u postgres psql -X -d zabbix_recovery -c 'GRANT ALL ON SCHEMA public TO zabbix;'
# Pipe stdin because postgres cannot traverse root-only backup directories.
set -o pipefail
sudo cat /restore/zabbix/selected-set/zabbix.dump | sudo -u postgres pg_restore --exit-on-error --no-owner --no-acl --role=zabbix -d zabbix_recovery
sudo -u postgres psql -X -d zabbix_recovery -c 'SELECT mandatory,optional FROM dbversion;'
sudo -u postgres psql -X -d zabbix_recovery -c 'SELECT count(*) FROM hosts;'
sudo -u postgres psql -X -d zabbix_recovery -c 'SELECT count(*) FROM triggers;'
sudo -u postgres psql -X -d zabbix_recovery -c 'ANALYZE;'
# 4–6. Extract to staging, not directly over /etc.
sudo install -d -m 0700 /restore/configuration
sudo tar --acls --xattrs --numeric-owner -xzf /restore/zabbix/selected-set/configuration.tar.gz -C /restore/configuration

# OVERWRITES REPLACEMENT CONFIG: review staged files and destination host first.
sudo cp -a /restore/configuration/etc/zabbix/. /etc/zabbix/
sudo cp -a /restore/configuration/etc/nginx/. /etc/nginx/
sudo cp -a /restore/configuration/etc/php/8.3/fpm/. /etc/php/8.3/fpm/
sudo install -d -m 0700 /etc/ssl/zabbix
sudo cp -a /restore/configuration/etc/ssl/zabbix/. /etc/ssl/zabbix/
sudo chown root:root /etc/ssl/zabbix/privkey.pem
sudo chmod 0600 /etc/ssl/zabbix/privkey.pem
sudo chown root:zabbix /etc/zabbix/zabbix_server.conf
sudo chmod 0640 /etc/zabbix/zabbix_server.conf
# Reconcile PHP pool group before protecting zabbix.conf.php.
# Optional custom scripts, only if they were in the selected set:
for relative in usr/lib/zabbix/alertscripts usr/lib/zabbix/externalscripts usr/local/lib/zabbix; do
  if sudo test -d "/restore/configuration/$relative"; then
    sudo install -d -o root -g zabbix -m 0750 "/$relative"
    sudo cp -a "/restore/configuration/$relative/." "/$relative/"
  fi
done
# Restore /etc/letsencrypt and renewal hooks only if that CA workflow was used.
# Restore backup scripts/units from the set, but keep its timer disabled during drill.
sudoedit /etc/zabbix/zabbix_server.conf
sudoedit /etc/zabbix/web/zabbix.conf.php
sudoedit /etc/zabbix/nginx.conf
sudoedit /etc/postgresql/16/main/pg_hba.conf
# Add BEFORE rejection rules:
# host zabbix_recovery zabbix 127.0.0.1/32 scram-sha-256

# 7. Start required services only after configuration review.
sudo pg_ctlcluster 16 main reload
sudo php-fpm8.3 -t
sudo nginx -t
sudo systemctl start php8.3-fpm zabbix-server nginx
sudo systemctl status zabbix-server php8.3-fpm nginx --no-pager
sudo tail -n 100 /var/log/zabbix/zabbix_server.log
curl -I --cacert /path/to/enterprise-ca.pem https://zabbix.example.com/
# Start only disposable lab agents after name/network/TLS validation.

# DESTRUCTIVE, intentionally commented: confirmed replacement host only.
sudo systemctl stop nginx zabbix-server
# sudo -u postgres dropdb --if-exists zabbix
# sudo -u postgres createdb -O zabbix -E UTF8 -T template0 zabbix
# Restore the verified dump into this EMPTY DB as above, no initial schema import.

sudo systemctl status zabbix-server --no-pager
sudo journalctl -u zabbix-server -n 100 --no-pager
sudo tail -n 100 /var/log/zabbix/zabbix_server.log
sudo ss -lntp

pg_lsclusters
pg_isready -h 127.0.0.1
psql -h 127.0.0.1 -U zabbix -W -d zabbix -c 'SELECT 1;'
sudo -u postgres psql -X -c 'SELECT line_number,error FROM pg_hba_file_rules;'

sudo systemctl status zabbix-agent2 --no-pager
sudo tail -n 50 /var/log/zabbix/zabbix_agent2.log
sudo ufw status numbered
# Windows: Get-Service -Name "Zabbix Agent 2"

sudo grep '^Hostname=' /etc/zabbix/zabbix_agent2.conf
sudo tail -n 50 /var/log/zabbix/zabbix_agent2.log

sudo grep -E '^(ServerActive|Hostname|TLSConnect)=' /etc/zabbix/zabbix_agent2.conf
nc -vz 192.0.2.10 10051
sudo tail -n 50 /var/log/zabbix/zabbix_agent2.log

# On agent:
sudo ss -lntp | grep ':10050'
sudo ufw status numbered
# On assigned Server/Proxy:
nc -vz 198.51.100.21 10050

sudo ss -lntp | grep ':10051'
# From agent:
nc -vz 192.0.2.10 10051
ip route get 192.0.2.10

sudo grep -E '^(TLSConnect|TLSAccept|TLSPSKIdentity|TLSPSKFile)=' /etc/zabbix/zabbix_agent2.conf
sudo -u zabbix test -r /etc/zabbix/keys/agent2.psk
sudo tail -n 50 /var/log/zabbix/zabbix_agent2.log

sudo -u zabbix zabbix_agent2 -c /etc/zabbix/zabbix_agent2.conf -t 'custom.service.nginx'
# Read item error under Hosts -> Items.

sudo tail -n 100 /var/log/zabbix/zabbix_server.log
chronyc tracking
# Frontend: host/item status, Maintenance and Administration -> Queue

df -hT
df -i
sudo -u postgres psql -X -d zabbix -c 'SELECT pg_size_pretty(pg_database_size(current_database()));'
sudo -u postgres psql -X -c 'SELECT * FROM pg_stat_archiver;'

sudo systemctl show zabbix-backup.service -p Result -p ExecMainStatus
sudo journalctl -u zabbix-backup.service -n 100 --no-pager
mountpoint /backup
df -h /backup
systemctl list-timers --all zabbix-backup.timer