Add Argo Workflow latency alerts

charts/monitoring/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: monitoring
 description: A monitoring stack for the workflows deployment
 type: application
-version: 0.1.48
+version: 0.1.49
 dependencies:
   - name: grafana
     repository: https://grafana-community.github.io/helm-charts

charts/monitoring/alert-rules/alert-rules.tpl.yaml

@@ -391,6 +391,61 @@ groups:
           summary: "Workflow container memory usage critical on {{`{{ $labels.pod }}`}}"
           description: 'Container {{`{{ $labels.container }}`}} in pod {{`{{ $labels.pod }}`}} is using {{`{{ printf "%.1f" $value }}`}}% of its memory limit.'
 
+  - name: argo-workflows-latency
+    rules:
+      - alert: ArgoWorkflowHighRuntimeLatency
+        expr: |
+          histogram_quantile(
+            0.95,
+            sum by (namespace, name, le) (
+              rate(argo_workflows_workflowtemplate_runtime_bucket[1h])
+            )
+          ) > 3600
+        for: 10m
+        labels:
+          severity: warning
+          priority: P1
+        annotations:
+          summary: "High runtime for workflow template {{`{{ $labels.name }}`}}"
+          description: 'P95 runtime for template {{`{{ $labels.name }}`}} in namespace {{`{{ $labels.namespace }}`}} has exceeded 60 for 10 minutes.'
+
+      - alert: ArgoWorkflowHighQueueLatency
+        expr: |
+          histogram_quantile(
+            0.95,
+            sum by (namespace, queue_name, le) (
+              rate(
+                argo_workflows_queue_latency_bucket{
+                  queue_name="workflow_queue"
+                }[5m]
+              )
+            )
+          ) > 30
+        for: 10m
+        labels:
+          severity: warning
+          priority: P1
+        annotations:
+          summary: "High Argo Workflow queue latency"
+          description: 'P95 queue latency for queue {{`{{ $labels.queue_name }}`}} in namespace {{`{{ $labels.namespace }}`}} has exceeded 30.'
+
+      - alert: ArgoWorkflowHighQueueDuration
+        expr: |
+          histogram_quantile(
+            0.95,
+            sum by (namespace, queue_name, le) (
+              rate(argo_workflows_queue_duration_bucket[5m])
+            )
+          ) > 60
+        for: 10m
+        labels:
+          severity: warning
+          priority: P1
+        annotations:
+          summary: "High Argo Workflow queue duration"
+          description: 'P95 queue duration for queue {{`{{ $labels.queue_name }}`}} in namespace {{`{{ $labels.namespace }}`}} has exceeded 60.'
+
+
   - name: kubernetes-pod-restarts
     rules:
       - alert: KubernetesPodRestartWarning