|
/etc/prometheus-rules/alert.rules > Application Metrics
|
|
|
|
/etc/prometheus-rules/alert.rules > BemobQueueProdEbIpSync
|
|
|
|
|
|
|
|
/etc/prometheus-rules/alert.rules > Clickhouse
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/etc/prometheus-rules/alert.rules > Deployment
|
|
|
|
|
|
|
|
|
|
/etc/prometheus-rules/alert.rules > Hosts
|
|
|
|
|
|
|
|
|
|
|
|
|
alert: HostEdacUncorrectableErrorsDetected
expr: node_edac_uncorrectable_errors_total
> 0
labels:
severity: warning
team: devops
annotations:
description: |-
Host {{ $labels.instance }} has had {{ printf "%.0f" $value }} uncorrectable memory errors reported by EDAC in the last 5 minutes.
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host EDAC Uncorrectable Errors detected (instance {{ $labels.instance }})
|
|
|
|
|
alert: HostMemoryUnderMemoryPressure
expr: rate(node_vmstat_pgmajfault[1m])
> 1000
for: 2m
labels:
severity: warning
team: devops
annotations:
description: |-
The node is under heavy memory pressure. High rate of major page faults
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host memory under memory pressure (instance {{ $labels.instance }})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
alert: HostPhysicalComponentTooHot
expr: node_hwmon_temp_celsius
> 75
for: 5m
labels:
severity: warning
team: devops
annotations:
description: |-
Physical hardware component too hot
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host physical component too hot (instance {{ $labels.instance }})
|
alert: HostRaidArrayGotInactive
expr: node_md_state{state="inactive"}
> 0
labels:
severity: critical
team: devops
annotations:
description: |-
RAID array {{ $labels.device }} is in degraded state due to one or more disks failures. Number of spare drives is insufficient to fix issue automatically.
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host RAID array got inactive (instance {{ $labels.instance }})
|
alert: HostRaidDiskFailure
expr: node_md_disks{state="failed"}
> 0
for: 2m
labels:
severity: warning
team: devops
annotations:
description: |-
At least one device in RAID array on {{ $labels.instance }} failed. Array {{ $labels.md_device }} needs attention and possibly a disk swap
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host RAID disk failure (instance {{ $labels.instance }})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/etc/prometheus-rules/alert.rules > MySQL
|
alert: MysqlDown
expr: mysql_up == 0
labels:
severity: critical
annotations:
description: |-
MySQL instance is down on {{ $labels.instance }}
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: MySQL down (instance {{ $labels.instance }})
|
|
|
|
|
alert: MysqlRestarted
expr: mysql_global_status_uptime
< 60
labels:
severity: info
annotations:
description: |-
MySQL has just been restarted, less than one minute ago on {{ $labels.instance }}.
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: MySQL restarted (instance {{ $labels.instance }})
|
|
|
|
|
|
|
|
|
|
|
|
/etc/prometheus-rules/alert.rules > PangolinHost
|
|
|
alert: PangolinHostDown
expr: up{job="pangolin-host"}
== 0
for: 2m
labels:
severity: critical
team: devops
annotations:
description: |-
Prometheus has not been able to scrape node_exporter on the Pangolin server (us-west-1b, EIP 54.151.74.26) for 2+ minutes. Either the host is down, node_exporter has stopped, or SG sg-0a4bbe4d623465337 was changed and blocks the EKS NAT IPs (52.86.29.40, 54.85.142.68). Pangolin carries the public ingress for 30+ bemobcloud.com hostnames, so if the host is actually down, expect a platform-wide outage.
LABELS = {{ $labels }}
summary: Pangolin host node_exporter unreachable
|
alert: PangolinHostMemoryCritical
expr: node_memory_MemAvailable_bytes{job="pangolin-host"}
< 256 * 1024 * 1024
for: 1m
labels:
severity: critical
team: devops
annotations:
description: |-
Available memory on the Pangolin server is {{ $value | humanize1024 }}B for 1+ minute. The 2026-05-11 outage cascaded from exactly this condition: docker daemon wedges, sshd cannot fork, traffic stops. ACT IMMEDIATELY — SSH in and restart the heaviest container, or reboot via `aws ec2 reboot-instances --region us-west-1 --instance-ids i-09be446891581e4eb`.
LABELS = {{ $labels }}
summary: Pangolin host has less than 256 MiB available memory
|
alert: PangolinHostMemoryLow
expr: node_memory_MemAvailable_bytes{job="pangolin-host"}
< 1 * 1024 * 1024 * 1024
for: 5m
labels:
severity: warning
team: devops
annotations:
description: |-
Available memory on the Pangolin server has been below 1 GiB for 5+ minutes (current {{ $value | humanize1024 }}B). Investigate which container is growing. The host has 4 GiB swap — sustained pressure here will start using it and may trip PangolinHostSwapHigh next.
LABELS = {{ $labels }}
summary: Pangolin host has less than 1 GiB available memory
|
|
|
|
/etc/prometheus-rules/alert.rules > Pods
|
|
|
|
/etc/prometheus-rules/alert.rules > Redis
|
alert: RedisClusterFlapping
expr: changes(redis_connected_slaves[1m])
> 1
for: 2m
labels:
severity: critical
team: devops
annotations:
description: |-
Changes have been detected in Redis replica connection. This can occur when replica nodes lose connection to the master and reconnect (a.k.a flapping).
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Redis cluster flapping (instance {{ $labels.instance }})
|
alert: RedisDown
expr: redis_up == 0
labels:
severity: critical
team: devops
annotations:
description: |-
Redis instance is down
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Redis down (instance {{ $labels.instance }})
|
|
|
|
|
|
|
|
|
alert: RedisTooManyConnections
expr: redis_connected_clients
> 2000
for: 2m
labels:
severity: warning
team: devops
annotations:
description: |-
Redis instance has too many connections
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Redis too many connections (instance {{ $labels.instance }})
|
|
/etc/prometheus-rules/alert.rules > Zookeeper
|
alert: ZookeeperDown
expr: zk_up == 0
labels:
severity: critical
team: devops
annotations:
description: |-
Zookeeper down on instance {{ $labels.instance }}
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Zookeeper Down (instance {{ $labels.instance }})
|
alert: ZookeeperMissingLeader
expr: sum(zk_server_leader)
== 0
labels:
severity: critical
team: devops
annotations:
description: |-
Zookeeper cluster has no node marked as leader
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Zookeeper missing leader (instance {{ $labels.instance }})
|
alert: ZookeeperNotOk
expr: zk_ruok == 0
for: 3m
labels:
severity: warning
team: devops
annotations:
description: |-
Zookeeper instance is not ok
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Zookeeper Not Ok (instance {{ $labels.instance }})
|
alert: ZookeeperTooManyLeaders
expr: sum(zk_server_leader)
> 1
labels:
severity: critical
team: devops
annotations:
description: |-
Zookeeper cluster has too many nodes marked as leader
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Zookeeper Too Many Leaders (instance {{ $labels.instance }})
|