Skip to content

fix(grafana): alert on Ogmios tip lag #78

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
wolf31o2 merged 1 commit into from
Oct 9, 2026
Merged

fix(grafana): alert on Ogmios tip lag #78

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
138 changes: 7 additions & 131 deletions grafana/alerts/demeter/ogmios.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,134 +7,10 @@
"folder": "Demeter",
"interval": "1m",
"rules": [
{
"uid": "cec6tlsnq9z40e",
"title": "Ogmios is down",
"condition": "C",
"data": [
{
"refId": "A",
"relativeTimeRange": {
"from": 600,
"to": 0
},
"datasourceUid": "${datasource_uid_map.gke_blinklabs-demeter_us-central1_dmtr-cluster-prometheus}",
"model": {
"datasource": {
"type": "prometheus",
"uid": "${datasource_uid_map.gke_blinklabs-demeter_us-central1_dmtr-cluster-prometheus}"
},
"editorMode": "code",
"expr": "count(avg_over_time(ogmios_connected[10m] offset 1h)) by (pod) unless count(avg_over_time(ogmios_connected[10m])) by (pod)",
"instant": true,
"intervalMs": 1000,
"legendFormat": "__auto",
"maxDataPoints": 43200,
"range": false,
"refId": "A"
}
},
{
"refId": "B",
"relativeTimeRange": {
"from": 0,
"to": 0
},
"datasourceUid": "__expr__",
"model": {
"conditions": [
{
"evaluator": {
"params": [
0,
0
],
"type": "gt"
},
"operator": {
"type": "and"
},
"query": {
"params": []
},
"reducer": {
"params": [],
"type": "avg"
},
"type": "query"
}
],
"datasource": {
"name": "Expression",
"type": "__expr__",
"uid": "__expr__"
},
"expression": "A",
"hide": false,
"reducer": "count",
"refId": "B",
"type": "reduce"
}
},
{
"refId": "C",
"relativeTimeRange": {
"from": 0,
"to": 0
},
"datasourceUid": "__expr__",
"model": {
"conditions": [
{
"evaluator": {
"params": [
0,
0
],
"type": "gt"
},
"operator": {
"type": "and"
},
"query": {
"params": []
},
"reducer": {
"params": [],
"type": "avg"
},
"type": "query"
}
],
"datasource": {
"name": "Expression",
"type": "__expr__",
"uid": "__expr__"
},
"expression": "B",
"hide": false,
"refId": "C",
"type": "threshold"
}
}
],
"noDataState": "OK",
"execErrState": "KeepLast",
"annotations": {
"description": "We are not receiving metrics from a particular Ogmios instance.",
"summary": "{{ range $k, $v := $values -}}\n{{ if (match \"C[0-9]+\" $k) -}}\nPod: {{ $v.Labels.pod }}\n{{ end }}\n{{ end }}"
},
"labels": {},
"isPaused": false,
"notification_settings": {
"receiver": "#infra-alerts"
}
},
{
"uid": "bkl0ogtipstale",
"title": "Ogmios tip is stale",
"title": "Ogmios is behind Cardano node",
"condition": "C",
"for": "10m",
"data": [
{
"refId": "A",
Expand All @@ -149,7 +25,7 @@
"uid": "${datasource_uid_map.gke_blinklabs-demeter_us-central1_dmtr-cluster-prometheus}"
},
"editorMode": "code",
"expr": "changes(ogmios_tip_slot[15m]) and on (pod) (ogmios_connected == 1)",
"expr": "max by (network) (cardano_node_metrics_slotNum_int{namespace=\"ext-nodes-m1\", network=~\"mainnet|preprod|preview\"}) - on (network) group_right() max by (network, pod, ogmios_version) (label_replace(ogmios_tip_slot{namespace=\"ext-ogmios-m1\", pod=~\".*(mainnet|preprod|preview).*\"}, \"network\", \"$1\", \"pod\", \".*(mainnet|preprod|preview).*\"))",
"instant": true,
"intervalMs": 1000,
"legendFormat": "__auto",
Expand Down Expand Up @@ -212,10 +88,10 @@
{
"evaluator": {
"params": [
1,
120,
0
],
"type": "lt"
"type": "gt"
},
"operator": {
"type": "and"
Expand Down Expand Up @@ -245,8 +121,8 @@
"noDataState": "OK",
"execErrState": "KeepLast",
"annotations": {
"description": "An Ogmios instance is connected to its cardano-node backend but its chain tip (ogmios_tip_slot) has not advanced for at least 15 minutes, meaning it is serving a stale tip to clients. This usually indicates a wedged chain-sync session that a pod restart will clear.",
"summary": "{{ range $k, $v := $values -}}\n{{ if (match \"C[0-9]+\" $k) -}}\nPod: {{ $v.Labels.pod }} tip slot is not advancing\n{{ end }}\n{{ end }}"
"description": "The Ogmios tip is more than 120 Cardano slots (about 2 minutes) behind the freshest Cardano node tip on the same network. Restart the identified Ogmios pod after checking its logs.",
"summary": "Ogmios pod {{ $labels.pod }} on {{ $labels.network }} is over 2 minutes behind the Cardano node tip"
},
"labels": {},
"isPaused": false,
Expand All @@ -257,4 +133,4 @@
]
}
]
}
}
2 changes: 1 addition & 1 deletion grafana/dashboards/shared/ogmios.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1642,7 +1642,7 @@
"type": "prometheus",
"uid": "$datasource"
},
"description": "Number of times each Ogmios instance's chain tip (ogmios_tip_slot) advanced in the last 15 minutes. A healthy, synced instance advances continuously; a value of 0 means the tip is frozen (wedged chain-sync) and the pod is serving a stale tip. This is the signal behind the \"Ogmios tip is stale\" alert.",
"description": "Number of times each Ogmios instance's chain tip (ogmios_tip_slot) advanced in the last 15 minutes. A value of 0 can help diagnose a frozen tip; the alert compares each pod's tip with the freshest Cardano node tip on its network.",
"fieldConfig": {
"defaults": {
"color": {
Expand Down
Loading