Skip to content

feat(config): add reclaimed CPU PSI eviction settings #238

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
cheney-lin wants to merge 1 commit into from
Closed

feat(config): add reclaimed CPU PSI eviction settings #238

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
104 changes: 104 additions & 0 deletions config/crd/bases/config.katalyst.kubewharf.io_adminqosconfigurations.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,110 @@ spec:
evictionConfig:
description: EvictionConfig is a configuration for eviction
properties:
beCPUPSIEvictionConfig:
description: BECPUPSIEvictionConfig controls reclaimed cgroup
CPU PSI eviction.
properties:
budgetWindowSeconds:
description: BudgetWindowSeconds bounds the rolling proposal
budget and must be at least CooldownSeconds.
format: int64
maximum: 86400
minimum: 1
type: integer
cooldownSeconds:
description: CooldownSeconds is the minimum interval between
eviction proposals.
format: int64
maximum: 86400
minimum: 1
type: integer
enable:
description: Enable activates sampling and evaluation.
It defaults to false.
type: boolean
enableEviction:
description: EnableEviction allows soft eviction proposals
and requires EnableTaint.
type: boolean
enableTaint:
description: EnableTaint allows a reclaimed QoS NoSchedule
condition after sustained pressure.
type: boolean
evictionDurationSeconds:
description: EvictionDurationSeconds is the continuous
eligible pressure dwell time; it must be at least TaintDurationSeconds.
format: int64
maximum: 86400
minimum: 1
type: integer
evictionThreshold:
description: EvictionThreshold is the full fraction that
starts the eviction dwell timer; it must exceed TaintThreshold.
maximum: 1
minimum: 0
type: number
gracePeriodSeconds:
description: GracePeriodSeconds is the requested Pod termination
grace period for a soft eviction.
format: int64
maximum: 86400
minimum: 0
type: integer
maxEvictionsPerWindow:
description: MaxEvictionsPerWindow limits proposals, including
retries, per agent instance within BudgetWindowSeconds.
format: int64
minimum: 1
type: integer
maxSampleGapSeconds:
description: MaxSampleGapSeconds resets history after
a collection gap; it must not exceed WindowSeconds.
format: int64
maximum: 86400
minimum: 1
type: integer
minOutsideUsageRatio:
description: MinOutsideUsageRatio is the minimum estimated
usage outside BE on its effective CPUs required for
eviction.
maximum: 1
minimum: 0
type: number
minRecoveryIdleRatio:
description: MinRecoveryIdleRatio is the minimum spare
CPU fraction required for recovery.
maximum: 1
minimum: 0
type: number
recoveryThreshold:
description: RecoveryThreshold is the full fraction below
which spare CPU can permit recovery; it must be below
TaintThreshold.
maximum: 1
minimum: 0
type: number
taintDurationSeconds:
description: TaintDurationSeconds is the continuous pressure
dwell time after window warmup.
format: int64
maximum: 86400
minimum: 1
type: integer
taintThreshold:
description: TaintThreshold is the CPU PSI full fraction
that starts the taint dwell timer.
maximum: 1
minimum: 0
type: number
windowSeconds:
description: WindowSeconds is the minimum observation
window for delta(full.total) / elapsed time.
format: int64
maximum: 86400
minimum: 1
type: integer
type: object
cpuPressureEvictionConfig:
description: CPUPressureEvictionConfig is the config for cpu
pressure eviction
Expand Down
82 changes: 82 additions & 0 deletions pkg/apis/config/v1alpha1/adminqos.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -638,6 +638,10 @@ type HostWatermarkConfig struct {
}

type EvictionConfig struct {
// BECPUPSIEvictionConfig controls reclaimed cgroup CPU PSI eviction.
// +optional
BECPUPSIEvictionConfig *BECPUPSIEvictionConfig `json:"beCPUPSIEvictionConfig,omitempty"`

// DryRun is the list of eviction plugins to dryRun
// '*' means "all dry-run by default"
// 'foo' means "dry-run 'foo'"
Expand Down Expand Up @@ -1402,3 +1406,81 @@ type CPUBurstConfig struct {
// +optional
DefaultCPUBurstPercent *int64 `json:"defaultCPUBurstPercent,omitempty"`
}

// BECPUPSIEvictionConfig controls CPU starvation of reclaimed cgroups.
// Thresholds are fractions; all durations are seconds. Omitted fields retain agent defaults.
type BECPUPSIEvictionConfig struct {
// Enable activates sampling and evaluation. It defaults to false.
// +optional
Enable *bool `json:"enable,omitempty"`
// EnableTaint allows a reclaimed QoS NoSchedule condition after sustained pressure.
// +optional
EnableTaint *bool `json:"enableTaint,omitempty"`
// EnableEviction allows soft eviction proposals and requires EnableTaint.
// +optional
EnableEviction *bool `json:"enableEviction,omitempty"`
// WindowSeconds is the minimum observation window for delta(full.total) / elapsed time.
// +optional
// +kubebuilder:validation:Minimum=1
// +kubebuilder:validation:Maximum=86400
WindowSeconds *int64 `json:"windowSeconds,omitempty"`
// TaintThreshold is the CPU PSI full fraction that starts the taint dwell timer.
// +optional
// +kubebuilder:validation:Minimum=0
// +kubebuilder:validation:Maximum=1
TaintThreshold *float64 `json:"taintThreshold,omitempty"`
// EvictionThreshold is the full fraction that starts the eviction dwell timer; it must exceed TaintThreshold.
// +optional
// +kubebuilder:validation:Minimum=0
// +kubebuilder:validation:Maximum=1
EvictionThreshold *float64 `json:"evictionThreshold,omitempty"`
// RecoveryThreshold is the full fraction below which spare CPU can permit recovery; it must be below TaintThreshold.
// +optional
// +kubebuilder:validation:Minimum=0
// +kubebuilder:validation:Maximum=1
RecoveryThreshold *float64 `json:"recoveryThreshold,omitempty"`
// TaintDurationSeconds is the continuous pressure dwell time after window warmup.
// +optional
// +kubebuilder:validation:Minimum=1
// +kubebuilder:validation:Maximum=86400
TaintDurationSeconds *int64 `json:"taintDurationSeconds,omitempty"`
// EvictionDurationSeconds is the continuous eligible pressure dwell time; it must be at least TaintDurationSeconds.
// +optional
// +kubebuilder:validation:Minimum=1
// +kubebuilder:validation:Maximum=86400
EvictionDurationSeconds *int64 `json:"evictionDurationSeconds,omitempty"`
// CooldownSeconds is the minimum interval between eviction proposals.
// +optional
// +kubebuilder:validation:Minimum=1
// +kubebuilder:validation:Maximum=86400
CooldownSeconds *int64 `json:"cooldownSeconds,omitempty"`
// BudgetWindowSeconds bounds the rolling proposal budget and must be at least CooldownSeconds.
// +optional
// +kubebuilder:validation:Minimum=1
// +kubebuilder:validation:Maximum=86400
BudgetWindowSeconds *int64 `json:"budgetWindowSeconds,omitempty"`
// MaxEvictionsPerWindow limits proposals, including retries, per agent instance within BudgetWindowSeconds.
// +optional
// +kubebuilder:validation:Minimum=1
MaxEvictionsPerWindow *int64 `json:"maxEvictionsPerWindow,omitempty"`
// MaxSampleGapSeconds resets history after a collection gap; it must not exceed WindowSeconds.
// +optional
// +kubebuilder:validation:Minimum=1
// +kubebuilder:validation:Maximum=86400
MaxSampleGapSeconds *int64 `json:"maxSampleGapSeconds,omitempty"`
// MinOutsideUsageRatio is the minimum estimated usage outside BE on its effective CPUs required for eviction.
// +optional
// +kubebuilder:validation:Minimum=0
// +kubebuilder:validation:Maximum=1
MinOutsideUsageRatio *float64 `json:"minOutsideUsageRatio,omitempty"`
// MinRecoveryIdleRatio is the minimum spare CPU fraction required for recovery.
// +optional
// +kubebuilder:validation:Minimum=0
// +kubebuilder:validation:Maximum=1
MinRecoveryIdleRatio *float64 `json:"minRecoveryIdleRatio,omitempty"`
// GracePeriodSeconds is the requested Pod termination grace period for a soft eviction.
// +optional
// +kubebuilder:validation:Minimum=0
// +kubebuilder:validation:Maximum=86400
GracePeriodSeconds *int64 `json:"gracePeriodSeconds,omitempty"`
}
63 changes: 63 additions & 0 deletions pkg/apis/config/v1alpha1/be_cpu_psi_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
/*
Copyright 2022 The Katalyst Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1alpha1

import (
"encoding/json"
"reflect"
"testing"
)

func TestBECPUPSIEvictionConfigRoundTripAndDeepCopy(t *testing.T) {
t.Parallel()
// Explicit false and zero must survive JSON; nil must remain omitted.
input := []byte(`{"spec":{"config":{"evictionConfig":{"beCPUPSIEvictionConfig":{"enable":false,"enableTaint":true,"enableEviction":false,"windowSeconds":30,"taintThreshold":0.2,"evictionThreshold":0.5,"recoveryThreshold":0.1,"taintDurationSeconds":60,"evictionDurationSeconds":120,"cooldownSeconds":120,"budgetWindowSeconds":600,"maxEvictionsPerWindow":3,"maxSampleGapSeconds":30,"minOutsideUsageRatio":0.2,"minRecoveryIdleRatio":0.2,"gracePeriodSeconds":0}}}}}`)
var original AdminQoSConfiguration
if err := json.Unmarshal(input, &original); err != nil {
t.Fatal(err)
}
copy := original.DeepCopy()
if !reflect.DeepEqual(&original, copy) {
t.Fatal("deep copy changed configuration")
}
a := reflect.ValueOf(original.Spec.Config.EvictionConfig.BECPUPSIEvictionConfig).Elem()
b := reflect.ValueOf(copy.Spec.Config.EvictionConfig.BECPUPSIEvictionConfig).Elem()
for i := 0; i < a.NumField(); i++ {
if a.Field(i).IsNil() || b.Field(i).IsNil() {
t.Fatalf("field %s was lost", a.Type().Field(i).Name)
}
if a.Field(i).Pointer() == b.Field(i).Pointer() {
t.Fatalf("field %s aliases original", a.Type().Field(i).Name)
}
}
data, err := json.Marshal(copy)
if err != nil {
t.Fatal(err)
}
var decoded AdminQoSConfiguration
if err = json.Unmarshal(data, &decoded); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(&original, &decoded) {
t.Fatal("JSON round trip changed configuration")
}
empty := &BECPUPSIEvictionConfig{}
data, err = json.Marshal(empty.DeepCopy())
if err != nil || string(data) != "{}" {
t.Fatalf("nil fields should remain omitted: %s, %v", data, err)
}
}
Loading
Loading