From 6286b62f311acc73dc00adfd304232330591ba78 Mon Sep 17 00:00:00 2001 From: Hazem Nureldin Date: Wed, 21 Oct 2026 12:46:50 +0100 Subject: [PATCH 1/3] feat(workflow-observability): add workflow latency alerts --- .../alert-rules/alert-rules.tpl.yaml | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/charts/monitoring/alert-rules/alert-rules.tpl.yaml b/charts/monitoring/alert-rules/alert-rules.tpl.yaml index 9a260a592..16359172d 100644 --- a/charts/monitoring/alert-rules/alert-rules.tpl.yaml +++ b/charts/monitoring/alert-rules/alert-rules.tpl.yaml @@ -399,6 +399,18 @@ groups: increase(kube_pod_container_status_restarts_total[15m]) ) >= 3 for: 5m + + - name: argo-workflows-latency + rules: + - alert: ArgoWorkflowHighRuntimeLatency + expr: | + histogram_quantile( + 0.95, + sum by (namespace, name, le) ( + rate(argo_workflows_workflowtemplate_runtime_bucket[1h]) + ) + ) > 60 + for: 10m labels: severity: warning priority: P1 @@ -606,6 +618,22 @@ groups: expr: | up == 0 for: 5m + summary: "High runtime for workflow template {{`{{ $labels.name }}`}}" + description: 'P95 runtime for template {{`{{ $labels.name }}`}} in namespace {{`{{ $labels.namespace }}`}} has exceeded 60 for 10 minutes. Confirm the metric unit is seconds.' + + - alert: ArgoWorkflowHighQueueLatency + expr: | + histogram_quantile( + 0.95, + sum by (namespace, queue_name, le) ( + rate( + argo_workflows_queue_latency_bucket{ + queue_name="workflow_queue" + }[5m] + ) + ) + ) > 30 + for: 10m labels: severity: warning priority: P1 @@ -635,3 +663,21 @@ groups: Investigate target availability, exporter health, network connectivity, and scrape configuration. dashboard: "https://prometheus.workflows.diamond.ac.uk/targets" + summary: "High Argo Workflow queue latency" + description: 'P95 queue latency for queue {{`{{ $labels.queue_name }}`}} in namespace {{`{{ $labels.namespace }}`}} has exceeded 30. Confirm the metric unit and threshold.' + + - alert: ArgoWorkflowHighQueueDuration + expr: | + histogram_quantile( + 0.95, + sum by (namespace, queue_name, le) ( + rate(argo_workflows_queue_duration_bucket[5m]) + ) + ) > 60 + for: 10m + labels: + severity: warning + priority: P1 + annotations: + summary: "High Argo Workflow queue duration" + description: 'P95 queue duration for queue {{`{{ $labels.queue_name }}`}} in namespace {{`{{ $labels.namespace }}`}} has exceeded 60. Confirm the metric unit and threshold.' From 4426301ed47126d6d199797bbb087d61a47047bd Mon Sep 17 00:00:00 2001 From: Hazem Nureldin Date: Thu, 22 Oct 2026 10:33:15 +0100 Subject: [PATCH 2/3] fix(monitoring): increase workflow template runtime threshold --- charts/monitoring/alert-rules/alert-rules.tpl.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/monitoring/alert-rules/alert-rules.tpl.yaml b/charts/monitoring/alert-rules/alert-rules.tpl.yaml index 16359172d..8bbf36d82 100644 --- a/charts/monitoring/alert-rules/alert-rules.tpl.yaml +++ b/charts/monitoring/alert-rules/alert-rules.tpl.yaml @@ -409,7 +409,7 @@ groups: sum by (namespace, name, le) ( rate(argo_workflows_workflowtemplate_runtime_bucket[1h]) ) - ) > 60 + ) > 3600 for: 10m labels: severity: warning @@ -619,7 +619,7 @@ groups: up == 0 for: 5m summary: "High runtime for workflow template {{`{{ $labels.name }}`}}" - description: 'P95 runtime for template {{`{{ $labels.name }}`}} in namespace {{`{{ $labels.namespace }}`}} has exceeded 60 for 10 minutes. Confirm the metric unit is seconds.' + description: 'P95 runtime for template {{`{{ $labels.name }}`}} in namespace {{`{{ $labels.namespace }}`}} has exceeded 3600 for 10 minutes. Confirm the metric unit is seconds.' - alert: ArgoWorkflowHighQueueLatency expr: | @@ -632,7 +632,7 @@ groups: }[5m] ) ) - ) > 30 + ) > 3600 for: 10m labels: severity: warning @@ -664,7 +664,7 @@ groups: and scrape configuration. dashboard: "https://prometheus.workflows.diamond.ac.uk/targets" summary: "High Argo Workflow queue latency" - description: 'P95 queue latency for queue {{`{{ $labels.queue_name }}`}} in namespace {{`{{ $labels.namespace }}`}} has exceeded 30. Confirm the metric unit and threshold.' + description: 'P95 queue latency for queue {{`{{ $labels.queue_name }}`}} in namespace {{`{{ $labels.namespace }}`}} has exceeded 3600. Confirm the metric unit and threshold.' - alert: ArgoWorkflowHighQueueDuration expr: | @@ -673,11 +673,11 @@ groups: sum by (namespace, queue_name, le) ( rate(argo_workflows_queue_duration_bucket[5m]) ) - ) > 60 + ) > 3600 for: 10m labels: severity: warning priority: P1 annotations: summary: "High Argo Workflow queue duration" - description: 'P95 queue duration for queue {{`{{ $labels.queue_name }}`}} in namespace {{`{{ $labels.namespace }}`}} has exceeded 60. Confirm the metric unit and threshold.' + description: 'P95 queue duration for queue {{`{{ $labels.queue_name }}`}} in namespace {{`{{ $labels.namespace }}`}} has exceeded 360. Confirm the metric unit and threshold.' From 37c5b2ae541a2a034d38bd8c1178757ba0952112 Mon Sep 17 00:00:00 2001 From: Hazem Nureldin Date: Tue, 27 Oct 2026 14:50:57 +0000 Subject: [PATCH 3/3] fix(workflows-observations): alert mix up --- charts/monitoring/Chart.yaml | 2 +- .../alert-rules/alert-rules.tpl.yaml | 95 ++++++++++--------- 2 files changed, 53 insertions(+), 44 deletions(-) diff --git a/charts/monitoring/Chart.yaml b/charts/monitoring/Chart.yaml index 4e2a5d762..70e20d9f4 100644 --- a/charts/monitoring/Chart.yaml +++ b/charts/monitoring/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: monitoring description: A monitoring stack for the workflows deployment type: application -version: 0.1.48 +version: 0.1.49 dependencies: - name: grafana repository: https://grafana-community.github.io/helm-charts diff --git a/charts/monitoring/alert-rules/alert-rules.tpl.yaml b/charts/monitoring/alert-rules/alert-rules.tpl.yaml index 8bbf36d82..7683f1706 100644 --- a/charts/monitoring/alert-rules/alert-rules.tpl.yaml +++ b/charts/monitoring/alert-rules/alert-rules.tpl.yaml @@ -391,15 +391,6 @@ groups: summary: "Workflow container memory usage critical on {{`{{ $labels.pod }}`}}" description: 'Container {{`{{ $labels.container }}`}} in pod {{`{{ $labels.pod }}`}} is using {{`{{ printf "%.1f" $value }}`}}% of its memory limit.' - - name: kubernetes-pod-restarts - rules: - - alert: KubernetesPodRestartWarning - expr: | - sum by (namespace, pod, container) ( - increase(kube_pod_container_status_restarts_total[15m]) - ) >= 3 - for: 5m - - name: argo-workflows-latency rules: - alert: ArgoWorkflowHighRuntimeLatency @@ -411,6 +402,58 @@ groups: ) ) > 3600 for: 10m + labels: + severity: warning + priority: P1 + annotations: + summary: "High runtime for workflow template {{`{{ $labels.name }}`}}" + description: 'P95 runtime for template {{`{{ $labels.name }}`}} in namespace {{`{{ $labels.namespace }}`}} has exceeded 60 for 10 minutes.' + + - alert: ArgoWorkflowHighQueueLatency + expr: | + histogram_quantile( + 0.95, + sum by (namespace, queue_name, le) ( + rate( + argo_workflows_queue_latency_bucket{ + queue_name="workflow_queue" + }[5m] + ) + ) + ) > 30 + for: 10m + labels: + severity: warning + priority: P1 + annotations: + summary: "High Argo Workflow queue latency" + description: 'P95 queue latency for queue {{`{{ $labels.queue_name }}`}} in namespace {{`{{ $labels.namespace }}`}} has exceeded 30.' + + - alert: ArgoWorkflowHighQueueDuration + expr: | + histogram_quantile( + 0.95, + sum by (namespace, queue_name, le) ( + rate(argo_workflows_queue_duration_bucket[5m]) + ) + ) > 60 + for: 10m + labels: + severity: warning + priority: P1 + annotations: + summary: "High Argo Workflow queue duration" + description: 'P95 queue duration for queue {{`{{ $labels.queue_name }}`}} in namespace {{`{{ $labels.namespace }}`}} has exceeded 60.' + + + - name: kubernetes-pod-restarts + rules: + - alert: KubernetesPodRestartWarning + expr: | + sum by (namespace, pod, container) ( + increase(kube_pod_container_status_restarts_total[15m]) + ) >= 3 + for: 5m labels: severity: warning priority: P1 @@ -618,22 +661,6 @@ groups: expr: | up == 0 for: 5m - summary: "High runtime for workflow template {{`{{ $labels.name }}`}}" - description: 'P95 runtime for template {{`{{ $labels.name }}`}} in namespace {{`{{ $labels.namespace }}`}} has exceeded 3600 for 10 minutes. Confirm the metric unit is seconds.' - - - alert: ArgoWorkflowHighQueueLatency - expr: | - histogram_quantile( - 0.95, - sum by (namespace, queue_name, le) ( - rate( - argo_workflows_queue_latency_bucket{ - queue_name="workflow_queue" - }[5m] - ) - ) - ) > 3600 - for: 10m labels: severity: warning priority: P1 @@ -663,21 +690,3 @@ groups: Investigate target availability, exporter health, network connectivity, and scrape configuration. dashboard: "https://prometheus.workflows.diamond.ac.uk/targets" - summary: "High Argo Workflow queue latency" - description: 'P95 queue latency for queue {{`{{ $labels.queue_name }}`}} in namespace {{`{{ $labels.namespace }}`}} has exceeded 3600. Confirm the metric unit and threshold.' - - - alert: ArgoWorkflowHighQueueDuration - expr: | - histogram_quantile( - 0.95, - sum by (namespace, queue_name, le) ( - rate(argo_workflows_queue_duration_bucket[5m]) - ) - ) > 3600 - for: 10m - labels: - severity: warning - priority: P1 - annotations: - summary: "High Argo Workflow queue duration" - description: 'P95 queue duration for queue {{`{{ $labels.queue_name }}`}} in namespace {{`{{ $labels.namespace }}`}} has exceeded 360. Confirm the metric unit and threshold.'