gilfanovrandClaude Opus 5.5
d1269162e7
- servers/rpi.md: новая домашняя Raspberry Pi 4 (Trixie, cloud-init), HDD 4 ТБ, грабли с питанием двух HDD от USB - stacks/rpi-backup: rclone sync с вечным архивом версий, еженедельная сверка хэшей, smartd, пуш SMART и статуса бэкапов - fr1: pushgateway (127.0.0.1:9091), приём по SSH через forced command ~/bin/pushgw-ingest, алерты группы rpi Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
102 lines
4.7 KiB
YAML
102 lines
4.7 KiB
YAML
groups:
|
||
- name: host
|
||
rules:
|
||
- alert: TargetDown
|
||
expr: up == 0
|
||
for: 2m
|
||
labels: {severity: critical}
|
||
annotations:
|
||
summary: "Таргет {{ $labels.job }} на {{ $labels.instance }} недоступен"
|
||
|
||
- alert: HostHighCPU
|
||
expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90
|
||
for: 10m
|
||
labels: {severity: warning}
|
||
annotations:
|
||
summary: "Высокий CPU на {{ $labels.instance }}: {{ printf \"%.0f\" $value }}%"
|
||
|
||
- alert: HostLowMemory
|
||
expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10
|
||
for: 5m
|
||
labels: {severity: warning}
|
||
annotations:
|
||
summary: "Мало памяти на {{ $labels.instance }}: доступно {{ printf \"%.0f\" $value }}%"
|
||
|
||
- alert: HostDiskLow
|
||
expr: node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs"} * 100 < 15
|
||
for: 5m
|
||
labels: {severity: warning}
|
||
annotations:
|
||
summary: "Кончается диск на {{ $labels.instance }} ({{ $labels.mountpoint }}): свободно {{ printf \"%.0f\" $value }}%"
|
||
|
||
- alert: HostDiskCritical
|
||
expr: node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs"} * 100 < 5
|
||
for: 5m
|
||
labels: {severity: critical}
|
||
annotations:
|
||
summary: "КРИТИЧНО: диск почти полон на {{ $labels.instance }} ({{ $labels.mountpoint }}): свободно {{ printf \"%.0f\" $value }}%"
|
||
|
||
# Метрики от rpi приходят через pushgateway (SSH-пуш, см. servers/rpi.md).
|
||
# push_time_seconds — когда группа последний раз обновлялась.
|
||
- name: rpi
|
||
rules:
|
||
- alert: PushMetricsStale
|
||
expr: time() - push_time_seconds{job=~"smart|backup_.*"} > 3 * 3600
|
||
for: 10m
|
||
labels: {severity: warning}
|
||
annotations:
|
||
summary: "{{ $labels.instance }}: метрики {{ $labels.job }} не приходят {{ $value | humanizeDuration }} — хост/пуш лежит"
|
||
|
||
- alert: SmartHealthFailed
|
||
expr: smart_health_passed == 0
|
||
labels: {severity: critical}
|
||
annotations:
|
||
summary: "{{ $labels.instance }}: SMART FAILED на диске {{ $labels.model }} ({{ $labels.serial }}) — менять диск"
|
||
|
||
- alert: SmartBadSectors
|
||
expr: smart_attribute_raw{id=~"5|197|198"} > 0
|
||
labels: {severity: critical}
|
||
annotations:
|
||
summary: "{{ $labels.instance }}: {{ $labels.name }} = {{ $value }} на {{ $labels.model }} ({{ $labels.serial }}) — диск начал сыпаться"
|
||
|
||
- alert: SmartBadSectorsGrowing
|
||
expr: increase(smart_attribute_raw{id=~"5|197|198"}[1d]) > 0
|
||
labels: {severity: critical}
|
||
annotations:
|
||
summary: "{{ $labels.instance }}: {{ $labels.name }} растёт на {{ $labels.model }} — срочно вторую копию"
|
||
|
||
- alert: SmartUdmaCrcErrors
|
||
expr: increase(smart_attribute_raw{id="199"}[1d]) > 0
|
||
labels: {severity: warning}
|
||
annotations:
|
||
summary: "{{ $labels.instance }}: CRC-ошибки на {{ $labels.model }} — кабель/USB/питание"
|
||
|
||
- alert: SmartSelftestFailed
|
||
expr: smart_selftest_last_passed == 0
|
||
labels: {severity: critical}
|
||
annotations:
|
||
summary: "{{ $labels.instance }}: самотест SMART провален на {{ $labels.model }} ({{ $labels.serial }})"
|
||
|
||
- alert: SmartTemperatureHigh
|
||
expr: smart_temperature_celsius > 55
|
||
for: 15m
|
||
labels: {severity: warning}
|
||
annotations:
|
||
summary: "{{ $labels.instance }}: диск {{ $labels.model }} греется: {{ $value }}°C"
|
||
|
||
- alert: BackupFailed
|
||
expr: backup_last_exit_code != 0
|
||
labels: {severity: warning}
|
||
annotations:
|
||
summary: "{{ $labels.instance }}: {{ $labels.job }} завершился с кодом {{ $value }} — см. journalctl на хосте"
|
||
|
||
# sync каждые 4ч → 12ч = три пропуска подряд; verify раз в неделю → 8 дней
|
||
- alert: BackupStale
|
||
expr: |
|
||
(time() - backup_last_success_timestamp{job="backup_onedrive"} > 12 * 3600)
|
||
or (time() - backup_last_success_timestamp{job="backup_onedrive_verify"} > 8 * 86400)
|
||
for: 10m
|
||
labels: {severity: critical}
|
||
annotations:
|
||
summary: "{{ $labels.instance }}: {{ $labels.job }} не было успешного прогона {{ $value | humanizeDuration }}"
|