diff --git a/charts/monitoring/Chart.yaml b/charts/monitoring/Chart.yaml index 4e2a5d762..70e20d9f4 100644 --- a/charts/monitoring/Chart.yaml +++ b/charts/monitoring/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: monitoring description: A monitoring stack for the workflows deployment type: application -version: 0.1.48 +version: 0.1.49 dependencies: - name: grafana repository: https://grafana-community.github.io/helm-charts diff --git a/charts/monitoring/alert-rules/alert-rules.tpl.yaml b/charts/monitoring/alert-rules/alert-rules.tpl.yaml index 9a260a592..7683f1706 100644 --- a/charts/monitoring/alert-rules/alert-rules.tpl.yaml +++ b/charts/monitoring/alert-rules/alert-rules.tpl.yaml @@ -391,6 +391,61 @@ groups: summary: "Workflow container memory usage critical on {{`{{ $labels.pod }}`}}" description: 'Container {{`{{ $labels.container }}`}} in pod {{`{{ $labels.pod }}`}} is using {{`{{ printf "%.1f" $value }}`}}% of its memory limit.' + - name: argo-workflows-latency + rules: + - alert: ArgoWorkflowHighRuntimeLatency + expr: | + histogram_quantile( + 0.95, + sum by (namespace, name, le) ( + rate(argo_workflows_workflowtemplate_runtime_bucket[1h]) + ) + ) > 3600 + for: 10m + labels: + severity: warning + priority: P1 + annotations: + summary: "High runtime for workflow template {{`{{ $labels.name }}`}}" + description: 'P95 runtime for template {{`{{ $labels.name }}`}} in namespace {{`{{ $labels.namespace }}`}} has exceeded 60 for 10 minutes.' + + - alert: ArgoWorkflowHighQueueLatency + expr: | + histogram_quantile( + 0.95, + sum by (namespace, queue_name, le) ( + rate( + argo_workflows_queue_latency_bucket{ + queue_name="workflow_queue" + }[5m] + ) + ) + ) > 30 + for: 10m + labels: + severity: warning + priority: P1 + annotations: + summary: "High Argo Workflow queue latency" + description: 'P95 queue latency for queue {{`{{ $labels.queue_name }}`}} in namespace {{`{{ $labels.namespace }}`}} has exceeded 30.' + + - alert: ArgoWorkflowHighQueueDuration + expr: | + histogram_quantile( + 0.95, + sum by (namespace, queue_name, le) ( + rate(argo_workflows_queue_duration_bucket[5m]) + ) + ) > 60 + for: 10m + labels: + severity: warning + priority: P1 + annotations: + summary: "High Argo Workflow queue duration" + description: 'P95 queue duration for queue {{`{{ $labels.queue_name }}`}} in namespace {{`{{ $labels.namespace }}`}} has exceeded 60.' + + - name: kubernetes-pod-restarts rules: - alert: KubernetesPodRestartWarning