Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions cmd/worker.go
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ func (t *TemporalWorker) NewServer(userConfig legacy.UserConfig, config legacy.C
Metrics: globalCfg.Metrics,
LyftAuditJobsSnsTopicArn: userConfig.LyftAuditJobsSnsTopicArn,
RevisionSetter: globalCfg.RevisionSetter,
LockSweeper: globalCfg.LockSweeper,
}
return temporalworker.NewServer(cfg)
}
3 changes: 3 additions & 0 deletions server/config/raw/global_cfg.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ type GlobalCfg struct {
RevisionSetter RevisionSetter `yaml:"revision_setter" json:"revision_setter"`
Admin Admin `yaml:"admin" json:"admin"`
AdhocMode AdhocMode `yaml:"adhoc_mode" json:"adhoc_mode"`
LockSweeper LockSweeper `yaml:"lock_sweeper" json:"lock_sweeper"`
}

type AdhocMode struct {
Expand Down Expand Up @@ -132,6 +133,7 @@ func (g GlobalCfg) Validate() error {
validation.Field(&g.Github),
validation.Field(&g.TerraformLogFilters),
validation.Field(&g.Persistence),
validation.Field(&g.LockSweeper),
)
if err != nil {
return err
Expand Down Expand Up @@ -215,6 +217,7 @@ func (g GlobalCfg) ToValid(defaultCfg valid.GlobalCfg) valid.GlobalCfg {
Admin: g.Admin.ToValid(),
RevisionSetter: g.RevisionSetter.ToValid(),
AdhocMode: g.AdhocMode.ToValid(),
LockSweeper: g.LockSweeper.ToValid(),
}
}

Expand Down
74 changes: 74 additions & 0 deletions server/config/raw/lock_sweeper.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
package raw

import (
"errors"

validation "github.com/go-ozzo/ozzo-validation"
"github.com/runatlantis/atlantis/server/config/valid"
)

const (
defaultLockSweeperMinAgeMinutes = 120
// A Terraform activity can run for up to 60 minutes, so a shorter threshold could delete the
// lock of a run that is still going.
minLockSweeperMinAgeMinutes = 60
)

// LockSweeper configures the job that clears Terraform state locks orphaned by killed workers.
// It is off unless enabled, and runs in dry-run mode unless dry_run is explicitly false.
type LockSweeper struct {
Enabled bool `yaml:"enabled" json:"enabled"`
DryRun *bool `yaml:"dry_run" json:"dry_run"`
Bucket string `yaml:"bucket" json:"bucket"`
KeyPrefix string `yaml:"key_prefix" json:"key_prefix"`
Repository string `yaml:"repository" json:"repository"`
Table string `yaml:"table" json:"table"`
// OwnerPrefix limits automatic deletes to locks whose holder (the lock's Who field) starts with it,
// so a lock taken by a person running Terraform locally is never deleted. Required when enabled.
OwnerPrefix string `yaml:"owner_prefix" json:"owner_prefix"`
MinAgeMinutes int `yaml:"min_age_minutes" json:"min_age_minutes"`
}

func (l LockSweeper) Validate() error {
if !l.Enabled {
return nil
}
return validation.ValidateStruct(&l,
validation.Field(&l.Bucket, validation.Required),
validation.Field(&l.KeyPrefix, validation.Required),
validation.Field(&l.Repository, validation.Required),
validation.Field(&l.Table, validation.Required),
validation.Field(&l.OwnerPrefix, validation.Required),
// 0 means "use the default"
validation.Field(&l.MinAgeMinutes, validation.By(func(v interface{}) error {
if n := v.(int); n != 0 && n < minLockSweeperMinAgeMinutes {
return errors.New("must be at least 60 minutes, the longest a Terraform activity can run")
}
return nil
})),
)
}

func (l LockSweeper) ToValid() valid.LockSweeper {
if !l.Enabled {
return valid.LockSweeper{}
}
minAge := l.MinAgeMinutes
if minAge == 0 {
minAge = defaultLockSweeperMinAgeMinutes
}
dryRun := true
if l.DryRun != nil {
dryRun = *l.DryRun
}
return valid.LockSweeper{
Enabled: l.Enabled,
DryRun: dryRun,
Bucket: l.Bucket,
KeyPrefix: l.KeyPrefix,
Repository: l.Repository,
Table: l.Table,
OwnerPrefix: l.OwnerPrefix,
MinAgeMinutes: minAge,
}
}
84 changes: 84 additions & 0 deletions server/config/raw/lock_sweeper_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
package raw_test

import (
"testing"

"github.com/runatlantis/atlantis/server/config/raw"
"github.com/runatlantis/atlantis/server/config/valid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

func enabledSweeper() raw.LockSweeper {
return raw.LockSweeper{
Enabled: true,
Bucket: "atlantis-production-states",
KeyPrefix: "example-org/terraform-roots",
Repository: "example-org/terraform-roots",
Table: "atlantis-production-states-locks",
OwnerPrefix: "atlantis-svc@worker",
}
}

func TestLockSweeper_Validate(t *testing.T) {
t.Run("disabled needs nothing", func(t *testing.T) {
require.NoError(t, raw.LockSweeper{}.Validate())
})

t.Run("enabled with all fields is valid", func(t *testing.T) {
require.NoError(t, enabledSweeper().Validate())
})

t.Run("enabled requires every target field", func(t *testing.T) {
for name, mutate := range map[string]func(*raw.LockSweeper){
"bucket": func(l *raw.LockSweeper) { l.Bucket = "" },
"key_prefix": func(l *raw.LockSweeper) { l.KeyPrefix = "" },
"repository": func(l *raw.LockSweeper) { l.Repository = "" },
"table": func(l *raw.LockSweeper) { l.Table = "" },
"owner_prefix": func(l *raw.LockSweeper) { l.OwnerPrefix = "" },
} {
l := enabledSweeper()
mutate(&l)
assert.Error(t, l.Validate(), name)
}
})

t.Run("min age below the longest activity is rejected", func(t *testing.T) {
l := enabledSweeper()
l.MinAgeMinutes = 30
assert.Error(t, l.Validate())
})

t.Run("min age at the floor is accepted", func(t *testing.T) {
l := enabledSweeper()
l.MinAgeMinutes = 60
assert.NoError(t, l.Validate())
})
}

func TestLockSweeper_ToValid(t *testing.T) {
t.Run("disabled converts to the zero value", func(t *testing.T) {
assert.Equal(t, valid.LockSweeper{}, raw.LockSweeper{Bucket: "ignored"}.ToValid())
})

t.Run("defaults to dry run with a 2 hour threshold", func(t *testing.T) {
v := enabledSweeper().ToValid()
assert.True(t, v.DryRun)
assert.Equal(t, 120, v.MinAgeMinutes)
assert.Equal(t, "atlantis-svc@worker", v.OwnerPrefix)
assert.True(t, v.Enabled)
})

t.Run("only an explicit false turns dry run off", func(t *testing.T) {
off := false
l := enabledSweeper()
l.DryRun = &off
assert.False(t, l.ToValid().DryRun)
})

t.Run("an explicit threshold is kept", func(t *testing.T) {
l := enabledSweeper()
l.MinAgeMinutes = 240
assert.Equal(t, 240, l.ToValid().MinAgeMinutes)
})
}
13 changes: 13 additions & 0 deletions server/config/valid/global_cfg.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,19 @@ type GlobalCfg struct {
RevisionSetter RevisionSetter
Admin Admin
AdhocMode AdhocMode
LockSweeper LockSweeper
}

// LockSweeper configures the job that clears Terraform state locks orphaned by killed workers.
type LockSweeper struct {
Enabled bool
DryRun bool
Bucket string
KeyPrefix string
Repository string
Table string
OwnerPrefix string
MinAgeMinutes int
}

type AdhocMode struct {
Expand Down
Loading
Loading