groups: - name: host rules: - alert: TargetDown expr: up == 0 for: 2m labels: {severity: critical} annotations: summary: "Таргет {{ $labels.job }} на {{ $labels.instance }} недоступен" - alert: HostHighCPU expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90 for: 10m labels: {severity: warning} annotations: summary: "Высокий CPU на {{ $labels.instance }}: {{ printf \"%.0f\" $value }}%" - alert: HostLowMemory expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10 for: 5m labels: {severity: warning} annotations: summary: "Мало памяти на {{ $labels.instance }}: доступно {{ printf \"%.0f\" $value }}%" - alert: HostDiskLow expr: node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs"} * 100 < 15 for: 5m labels: {severity: warning} annotations: summary: "Кончается диск на {{ $labels.instance }} ({{ $labels.mountpoint }}): свободно {{ printf \"%.0f\" $value }}%" - alert: HostDiskCritical expr: node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs"} * 100 < 5 for: 5m labels: {severity: critical} annotations: summary: "КРИТИЧНО: диск почти полон на {{ $labels.instance }} ({{ $labels.mountpoint }}): свободно {{ printf \"%.0f\" $value }}%" # Метрики от rpi приходят через pushgateway (SSH-пуш, см. servers/rpi.md). # push_time_seconds — когда группа последний раз обновлялась. - name: rpi rules: - alert: PushMetricsStale expr: time() - push_time_seconds{job=~"smart|backup_.*"} > 3 * 3600 for: 10m labels: {severity: warning} annotations: summary: "{{ $labels.instance }}: метрики {{ $labels.job }} не приходят {{ $value | humanizeDuration }} — хост/пуш лежит" - alert: SmartHealthFailed expr: smart_health_passed == 0 labels: {severity: critical} annotations: summary: "{{ $labels.instance }}: SMART FAILED на диске {{ $labels.model }} ({{ $labels.serial }}) — менять диск" - alert: SmartBadSectors expr: smart_attribute_raw{id=~"5|197|198"} > 0 labels: {severity: critical} annotations: summary: "{{ $labels.instance }}: {{ $labels.name }} = {{ $value }} на {{ $labels.model }} ({{ $labels.serial }}) — диск начал сыпаться" - alert: SmartBadSectorsGrowing expr: increase(smart_attribute_raw{id=~"5|197|198"}[1d]) > 0 labels: {severity: critical} annotations: summary: "{{ $labels.instance }}: {{ $labels.name }} растёт на {{ $labels.model }} — срочно вторую копию" - alert: SmartUdmaCrcErrors expr: increase(smart_attribute_raw{id="199"}[1d]) > 0 labels: {severity: warning} annotations: summary: "{{ $labels.instance }}: CRC-ошибки на {{ $labels.model }} — кабель/USB/питание" - alert: SmartSelftestFailed expr: smart_selftest_last_passed == 0 labels: {severity: critical} annotations: summary: "{{ $labels.instance }}: самотест SMART провален на {{ $labels.model }} ({{ $labels.serial }})" - alert: SmartTemperatureHigh expr: smart_temperature_celsius > 55 for: 15m labels: {severity: warning} annotations: summary: "{{ $labels.instance }}: диск {{ $labels.model }} греется: {{ $value }}°C" - alert: BackupFailed expr: backup_last_exit_code != 0 labels: {severity: warning} annotations: summary: "{{ $labels.instance }}: {{ $labels.job }} завершился с кодом {{ $value }} — см. journalctl на хосте" # sync каждые 4ч → 12ч = три пропуска подряд; verify раз в неделю → 8 дней; бэкап серверов ежедневно → 26ч; копия на роутер ежедневно → 30ч (первичная ~13ч) - alert: BackupStale expr: | (time() - backup_last_success_timestamp{job="backup_onedrive"} > 12 * 3600) or (time() - backup_last_success_timestamp{job="backup_onedrive_verify"} > 8 * 86400) or (time() - backup_last_success_timestamp{job="backup_servers"} > 26 * 3600) or (time() - backup_last_success_timestamp{job="backup_router"} > 30 * 3600) for: 10m labels: {severity: critical} annotations: summary: "{{ $labels.instance }}: {{ $labels.job }} не было успешного прогона {{ $value | humanizeDuration }}" # /mnt/backup (HDD 4 ТБ) и /mnt/router (1 ТБ диск в роутере) — заполнение - alert: BackupDiskFull expr: 1 - backup_disk_avail_bytes / backup_disk_size_bytes > 0.9 for: 15m labels: {severity: warning} annotations: summary: "{{ $labels.instance }}: диск бэкапа {{ $labels.mountpoint }} заполнен на {{ $value | humanizePercentage }}" - alert: RouterDiskMountDown expr: router_mount_up == 0 for: 30m labels: {severity: warning} annotations: summary: "{{ $labels.instance }}: диск в роутере (/mnt/router, SFTP) недоступен 30+ мин — вторая копия бэкапа не пишется"