Merge pull request #3 from pjdubya/feat/delay-days-updates-from-pr

Rename param to defer-days, add reporting for deferred containers, general simplifications
pull/1884/head
pjdubya 11 months ago committed by GitHub
commit 692513043b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -39,7 +39,7 @@ var (
disableContainers []string
notifier t.Notifier
timeout time.Duration
delayDays int
deferDays int
lifecycleHooks bool
rollingRestart bool
scope string
@ -97,7 +97,7 @@ func PreRun(cmd *cobra.Command, _ []string) {
enableLabel, _ = f.GetBool("label-enable")
disableContainers, _ = f.GetStringSlice("disable-containers")
delayDays, _ = f.GetInt("delay-days")
deferDays, _ = f.GetInt("defer-days")
lifecycleHooks, _ = f.GetBool("enable-lifecycle-hooks")
rollingRestart, _ = f.GetBool("rolling-restart")
scope, _ = f.GetString("scope")
@ -290,9 +290,9 @@ func writeStartupMessage(c *cobra.Command, sched time.Time, filtering string) {
until := formatDuration(time.Until(sched))
startupLog.Info("Scheduling first run: " + sched.Format("2006-01-02 15:04:05 -0700 MST"))
startupLog.Info("Note that the first check will be performed in " + until)
delayDays, _ = c.PersistentFlags().GetInt("delay-days")
if delayDays > 0 {
startupLog.Infof("Container updates will be delayed until %d day(s) after image creation.", delayDays)
deferDays, _ = c.PersistentFlags().GetInt("defer-days")
if deferDays > 0 {
startupLog.Infof("Container updates will be deferred until %d day(s) after image creation.", deferDays)
}
} else if runOnce, _ := c.PersistentFlags().GetBool("run-once"); runOnce {
startupLog.Info("Running a one time update.")
@ -370,7 +370,7 @@ func runUpdatesWithNotifications(filter t.Filter) *metrics.Metric {
NoRestart: noRestart,
Timeout: timeout,
MonitorOnly: monitorOnly,
DelayDays: delayDays,
DeferDays: deferDays,
LifecycleHooks: lifecycleHooks,
RollingRestart: rollingRestart,
LabelPrecedence: labelPrecedence,
@ -383,9 +383,10 @@ func runUpdatesWithNotifications(filter t.Filter) *metrics.Metric {
notifier.SendNotification(result)
metricResults := metrics.NewMetric(result)
notifications.LocalLog.WithFields(log.Fields{
"Scanned": metricResults.Scanned,
"Updated": metricResults.Updated,
"Failed": metricResults.Failed,
"Scanned": metricResults.Scanned,
"Updated": metricResults.Updated,
"Deferred": metricResults.Deferred,
"Failed": metricResults.Failed,
}).Info("Session done")
return metricResults
}

@ -381,12 +381,12 @@ Environment Variable: WATCHTOWER_HTTP_API_METRICS
Default: false
```
## Delayed Update
## Deferred Update
Only update container to latest version of image if some number of days have passed since it has been published. This option may be useful for those who wish to avoid updating prior to the new version having some time in the field prior to updating in case there are critical defects found and released in a subsequent version.
```text
Argument: --delay-days
Environment Variable: WATCHTOWER_DELAY_DAYS
Argument: --defer-days
Environment Variable: WATCHTOWER_DEFER_DAYS
Type: Integer
Default: false
```

@ -79,6 +79,13 @@ func CreateMockContainerWithDigest(id string, name string, image string, created
return c
}
// CreateMockContainerWithDigest should only be used for testing
func CreateMockContainerWithImageCreatedTime(id string, name string, image string, created time.Time, imageCreated time.Time) wt.Container {
c := CreateMockContainer(id, name, image, created)
c.ImageInfo().Created = imageCreated.UTC().Format(time.RFC3339Nano)
return c
}
// CreateMockContainerWithConfig creates a container substitute valid for testing
func CreateMockContainerWithConfig(id string, name string, image string, running bool, restarting bool, created time.Time, config *dockerContainer.Config) wt.Container {
content := types.ContainerJSON{

@ -2,7 +2,6 @@ package actions
import (
"errors"
"strings"
"time"
"github.com/containrrr/watchtower/internal/util"
@ -38,16 +37,17 @@ func Update(client container.Client, params types.UpdateParams) (types.Report, e
// stale will be true if there is a more recent image than the current container is using
stale, newestImage, err := client.IsContainerStale(targetContainer, params)
shouldUpdate := stale && !params.NoRestart && !targetContainer.IsMonitorOnly(params)
imageUpdateDelayResolved := true
imageUpdateDeferred := false
imageAgeDays := 0
if err == nil && shouldUpdate {
// Check to make sure we have all the necessary information for recreating the container, including ImageInfo
err = targetContainer.VerifyConfiguration()
if err == nil {
if params.DelayDays > 0 {
imageAgeDays, err := getImageAgeDays(targetContainer.ImageInfo().Created)
if params.DeferDays > 0 {
imageAgeDays, imageErr := getImageAgeDays(targetContainer.ImageInfo().Created)
err = imageErr
if err == nil {
imageUpdateDelayResolved = imageAgeDays >= params.DelayDays
imageUpdateDeferred = imageAgeDays < params.DeferDays
}
}
} else if log.IsLevelEnabled(log.TraceLevel) {
@ -66,11 +66,12 @@ func Update(client container.Client, params types.UpdateParams) (types.Report, e
stale = false
staleCheckFailed++
progress.AddSkipped(targetContainer, err)
} else if !imageUpdateDelayResolved {
log.Infof("New image found for %s that was created %d day(s) ago but update delayed until %d day(s) after creation", targetContainer.Name(), imageAgeDays, params.DelayDays)
} else if imageUpdateDeferred {
log.Infof("New image found for %s that was created %d day(s) ago but update deferred until %d day(s) after creation", targetContainer.Name(), imageAgeDays, params.DeferDays)
// technically the container is stale but we set it to false here because it is this stale flag that tells downstream methods whether to perform the update
stale = false
progress.AddScanned(targetContainer, newestImage)
progress.MarkDeferred(targetContainer.ID())
} else {
progress.AddScanned(targetContainer, newestImage)
}
@ -90,9 +91,11 @@ func Update(client container.Client, params types.UpdateParams) (types.Report, e
// containersToUpdate will contain all containers, not just those that need to be updated. The "stale" flag is checked via container.ToRestart()
// within stopContainersInReversedOrder and restartContainersInSortedOrder to skip containers with stale set to false (unless LinkedToRestarting set)
// NOTE: This logic is changing with latest PR on main repo
var containersToUpdate []types.Container
for _, c := range containers {
if !c.IsMonitorOnly(params) {
// pulling this change in from PR 1895 for now to avoid updating status incorrectly
if c.ToRestart() && !c.IsMonitorOnly(params) {
containersToUpdate = append(containersToUpdate, c)
progress.MarkForUpdate(c.ID())
}
@ -286,19 +289,10 @@ func linkedContainerMarkedForRestart(links []string, containers []types.Containe
}
// Finds the difference between now and a given date, in full days. Input date is expected to originate
// from an image's Created attribute in ISO 8601, but since these do not always contain the same number of
// digits for milliseconds, the function also accounts for variations.
// from an image's Created attribute which will follow ISO 3339/8601 format.
// Reference: https://docs.docker.com/engine/api/v1.43/#tag/Image/operation/ImageInspect
func getImageAgeDays(imageCreatedDateTime string) (int, error) {
// Date strings sometimes vary in how many digits after the decimal point are present. If present, drop millisecond portion to standardize.
dotIndex := strings.Index(imageCreatedDateTime, ".")
if dotIndex != -1 {
imageCreatedDateTime = imageCreatedDateTime[:dotIndex] + "Z"
}
// Define the layout string for the date format without milliseconds
layout := "2006-01-02T15:04:05Z"
imageCreatedDate, error := time.Parse(layout, imageCreatedDateTime)
imageCreatedDate, error := time.Parse(time.RFC3339Nano, imageCreatedDateTime)
if error != nil {
log.Errorf("Error parsing imageCreatedDateTime date (%s). Error: %s", imageCreatedDateTime, error)

@ -65,6 +65,35 @@ func getLinkedTestData(withImageInfo bool) *TestData {
}
}
func getMixedAgeTestData(keepContainer string) *TestData {
return &TestData{
NameOfContainerToKeep: keepContainer,
Containers: []types.Container{
// new container with 5 day old image
CreateMockContainerWithImageCreatedTime(
"test-container-01",
"test-container-01",
"fake-image-01:latest",
time.Now(),
time.Now().AddDate(0, 0, -5)),
// new container with 1 day old image
CreateMockContainerWithImageCreatedTime(
"test-container-02",
"test-container-02",
"fake-image-02:latest",
time.Now(),
time.Now().AddDate(0, 0, -1)),
// new container with 1 hour old image
CreateMockContainerWithImageCreatedTime(
"test-container-03",
"test-container-03",
"fake-image-03:latest",
time.Now(),
time.Now().Add(-1*time.Hour)),
},
}
}
var _ = Describe("the update action", func() {
When("watchtower has been instructed to clean up", func() {
When("there are multiple containers using the same image", func() {
@ -258,6 +287,37 @@ var _ = Describe("the update action", func() {
})
})
When("watchtower has been instructed to defer updates by some number of days", func() {
It("should only update the 1 container with image at least 2 days old when DeferDays is 2", func() {
client := CreateMockClient(getMixedAgeTestData(""), false, false)
report, err := actions.Update(client, types.UpdateParams{DeferDays: 2})
Expect(err).NotTo(HaveOccurred())
Expect(report.Updated()).To(HaveLen(1))
Expect(report.Deferred()).To(HaveLen(2))
})
It("should only update the 2 containers with image at least 1 day old when DeferDays is 1", func() {
client := CreateMockClient(getMixedAgeTestData(""), false, false)
report, err := actions.Update(client, types.UpdateParams{DeferDays: 1})
Expect(err).NotTo(HaveOccurred())
Expect(report.Updated()).To(HaveLen(2))
Expect(report.Deferred()).To(HaveLen(1))
})
It("should update all containers when DeferDays is 0", func() {
client := CreateMockClient(getMixedAgeTestData(""), false, false)
report, err := actions.Update(client, types.UpdateParams{DeferDays: 0})
Expect(err).NotTo(HaveOccurred())
Expect(report.Updated()).To(HaveLen(3))
Expect(report.Deferred()).To(HaveLen(0))
})
It("should update all containers when DeferDays is not specified", func() {
client := CreateMockClient(getMixedAgeTestData(""), false, false)
report, err := actions.Update(client, types.UpdateParams{})
Expect(err).NotTo(HaveOccurred())
Expect(report.Updated()).To(HaveLen(3))
Expect(report.Deferred()).To(HaveLen(0))
})
})
When("watchtower has been instructed to run lifecycle hooks", func() {
When("pre-update script returns 1", func() {

@ -148,9 +148,9 @@ func RegisterSystemFlags(rootCmd *cobra.Command) {
"Enable the execution of commands triggered by pre- and post-update lifecycle hooks")
flags.IntP(
"delay-days",
"defer-days",
"0",
envInt("WATCHTOWER_DELAY_DAYS"),
envInt("WATCHTOWER_DEFER_DAYS"),
"Number of days to wait for new image version to be in place prior to installing it")
flags.BoolP(

@ -10,19 +10,21 @@ var metrics *Metrics
// Metric is the data points of a single scan
type Metric struct {
Scanned int
Updated int
Failed int
Scanned int
Updated int
Deferred int
Failed int
}
// Metrics is the handler processing all individual scan metrics
type Metrics struct {
channel chan *Metric
scanned prometheus.Gauge
updated prometheus.Gauge
failed prometheus.Gauge
total prometheus.Counter
skipped prometheus.Counter
channel chan *Metric
scanned prometheus.Gauge
updated prometheus.Gauge
deferred prometheus.Gauge
failed prometheus.Gauge
total prometheus.Counter
skipped prometheus.Counter
}
// NewMetric returns a Metric with the counts taken from the appropriate types.Report fields
@ -30,8 +32,9 @@ func NewMetric(report types.Report) *Metric {
return &Metric{
Scanned: len(report.Scanned()),
// Note: This is for backwards compatibility. ideally, stale containers should be counted separately
Updated: len(report.Updated()) + len(report.Stale()),
Failed: len(report.Failed()),
Updated: len(report.Updated()) + len(report.Stale()),
Deferred: len(report.Deferred()),
Failed: len(report.Failed()),
}
}
@ -60,6 +63,10 @@ func Default() *Metrics {
Name: "watchtower_containers_updated",
Help: "Number of containers updated by watchtower during the last scan",
}),
deferred: promauto.NewGauge(prometheus.GaugeOpts{
Name: "watchtower_containers_deferred",
Help: "Number of containers deferred by watchtower during the last scan",
}),
failed: promauto.NewGauge(prometheus.GaugeOpts{
Name: "watchtower_containers_failed",
Help: "Number of containers where update failed during the last scan",
@ -95,6 +102,7 @@ func (metrics *Metrics) HandleUpdate(channel <-chan *Metric) {
metrics.skipped.Inc()
metrics.scanned.Set(0)
metrics.updated.Set(0)
metrics.deferred.Set(0)
metrics.failed.Set(0)
continue
}
@ -102,6 +110,7 @@ func (metrics *Metrics) HandleUpdate(channel <-chan *Metric) {
metrics.total.Inc()
metrics.scanned.Set(float64(change.Scanned))
metrics.updated.Set(float64(change.Updated))
metrics.deferred.Set(float64(change.Deferred))
metrics.failed.Set(float64(change.Failed))
}
}

@ -23,12 +23,13 @@ func (d Data) MarshalJSON() ([]byte, error) {
var report jsonMap
if d.Report != nil {
report = jsonMap{
`scanned`: marshalReports(d.Report.Scanned()),
`updated`: marshalReports(d.Report.Updated()),
`failed`: marshalReports(d.Report.Failed()),
`skipped`: marshalReports(d.Report.Skipped()),
`stale`: marshalReports(d.Report.Stale()),
`fresh`: marshalReports(d.Report.Fresh()),
`scanned`: marshalReports(d.Report.Scanned()),
`updated`: marshalReports(d.Report.Updated()),
`deferred`: marshalReports(d.Report.Deferred()),
`failed`: marshalReports(d.Report.Failed()),
`skipped`: marshalReports(d.Report.Skipped()),
`stale`: marshalReports(d.Report.Stale()),
`fresh`: marshalReports(d.Report.Fresh()),
}
}

@ -21,6 +21,7 @@ var _ = Describe("JSON template", func() {
],
"host": "Mock",
"report": {
"deferred": [],
"failed": [
{
"currentImageId": "01d210000000",
@ -110,7 +111,7 @@ var _ = Describe("JSON template", func() {
},
"title": "Watchtower updates on Mock"
}`
data := mockDataFromStates(s.UpdatedState, s.FreshState, s.FailedState, s.SkippedState, s.UpdatedState)
data := mockDataFromStates(s.UpdatedState, s.DeferredState, s.FreshState, s.FailedState, s.SkippedState, s.UpdatedState)
Expect(getTemplatedResult(`json.v1`, false, data)).To(MatchJSON(expected))
})
})

@ -72,6 +72,8 @@ func (pb *previewData) addContainer(c containerStatus) {
pb.report.scanned = append(pb.report.scanned, &c)
case UpdatedState:
pb.report.updated = append(pb.report.updated, &c)
case DeferredState:
pb.report.deferred = append(pb.report.deferred, &c)
case FailedState:
pb.report.failed = append(pb.report.failed, &c)
case SkippedState:

@ -10,12 +10,13 @@ import (
type State string
const (
ScannedState State = "scanned"
UpdatedState State = "updated"
FailedState State = "failed"
SkippedState State = "skipped"
StaleState State = "stale"
FreshState State = "fresh"
ScannedState State = "scanned"
UpdatedState State = "updated"
DeferredState State = "deferred"
FailedState State = "failed"
SkippedState State = "skipped"
StaleState State = "stale"
FreshState State = "fresh"
)
// StatesFromString parses a string of state characters and returns a slice of the corresponding report states
@ -27,6 +28,8 @@ func StatesFromString(str string) []State {
states = append(states, ScannedState)
case 'u':
states = append(states, UpdatedState)
case 'd':
states = append(states, DeferredState)
case 'e':
states = append(states, FailedState)
case 'k':
@ -43,12 +46,13 @@ func StatesFromString(str string) []State {
}
type report struct {
scanned []types.ContainerReport
updated []types.ContainerReport
failed []types.ContainerReport
skipped []types.ContainerReport
stale []types.ContainerReport
fresh []types.ContainerReport
scanned []types.ContainerReport
updated []types.ContainerReport
deferred []types.ContainerReport
failed []types.ContainerReport
skipped []types.ContainerReport
stale []types.ContainerReport
fresh []types.ContainerReport
}
func (r *report) Scanned() []types.ContainerReport {
@ -57,6 +61,9 @@ func (r *report) Scanned() []types.ContainerReport {
func (r *report) Updated() []types.ContainerReport {
return r.updated
}
func (r *report) Deferred() []types.ContainerReport {
return r.deferred
}
func (r *report) Failed() []types.ContainerReport {
return r.failed
}
@ -87,6 +94,7 @@ func (r *report) All() []types.ContainerReport {
}
appendUnique(r.updated)
appendUnique(r.deferred)
appendUnique(r.failed)
appendUnique(r.skipped)
appendUnique(r.stale)

@ -12,6 +12,7 @@ const (
SkippedState
ScannedState
UpdatedState
DeferredState
FailedState
FreshState
StaleState
@ -70,6 +71,8 @@ func (u *ContainerStatus) State() string {
return "Scanned"
case UpdatedState:
return "Updated"
case DeferredState:
return "Deferred"
case FailedState:
return "Failed"
case FreshState:

@ -50,6 +50,11 @@ func (m Progress) MarkForUpdate(containerID types.ContainerID) {
m[containerID].state = UpdatedState
}
// MarkForUpdate marks the container identified by containerID for deferral
func (m Progress) MarkDeferred(containerID types.ContainerID) {
m[containerID].state = DeferredState
}
// Report creates a new Report from a Progress instance
func (m Progress) Report() types.Report {
return NewReport(m)

@ -7,12 +7,13 @@ import (
)
type report struct {
scanned []types.ContainerReport
updated []types.ContainerReport
failed []types.ContainerReport
skipped []types.ContainerReport
stale []types.ContainerReport
fresh []types.ContainerReport
scanned []types.ContainerReport
updated []types.ContainerReport
deferred []types.ContainerReport
failed []types.ContainerReport
skipped []types.ContainerReport
stale []types.ContainerReport
fresh []types.ContainerReport
}
func (r *report) Scanned() []types.ContainerReport {
@ -21,6 +22,9 @@ func (r *report) Scanned() []types.ContainerReport {
func (r *report) Updated() []types.ContainerReport {
return r.updated
}
func (r *report) Deferred() []types.ContainerReport {
return r.deferred
}
func (r *report) Failed() []types.ContainerReport {
return r.failed
}
@ -50,6 +54,7 @@ func (r *report) All() []types.ContainerReport {
}
appendUnique(r.updated)
appendUnique(r.deferred)
appendUnique(r.failed)
appendUnique(r.skipped)
appendUnique(r.stale)
@ -64,12 +69,13 @@ func (r *report) All() []types.ContainerReport {
// NewReport creates a types.Report from the supplied Progress
func NewReport(progress Progress) types.Report {
report := &report{
scanned: []types.ContainerReport{},
updated: []types.ContainerReport{},
failed: []types.ContainerReport{},
skipped: []types.ContainerReport{},
stale: []types.ContainerReport{},
fresh: []types.ContainerReport{},
scanned: []types.ContainerReport{},
updated: []types.ContainerReport{},
deferred: []types.ContainerReport{},
failed: []types.ContainerReport{},
skipped: []types.ContainerReport{},
stale: []types.ContainerReport{},
fresh: []types.ContainerReport{},
}
for _, update := range progress {
@ -88,9 +94,13 @@ func NewReport(progress Progress) types.Report {
switch update.state {
case UpdatedState:
report.updated = append(report.updated, update)
case DeferredState:
report.deferred = append(report.deferred, update)
case FailedState:
report.failed = append(report.failed, update)
default:
// TODO: should this be changed to something lke UnknownState since it shouldn't be possible for a container
// to be stale but its state to not be either UpdatedState, DeferredState, or FailedState?
update.state = StaleState
report.stale = append(report.stale, update)
}
@ -98,6 +108,7 @@ func NewReport(progress Progress) types.Report {
sort.Sort(sortableContainers(report.scanned))
sort.Sort(sortableContainers(report.updated))
sort.Sort(sortableContainers(report.deferred))
sort.Sort(sortableContainers(report.failed))
sort.Sort(sortableContainers(report.skipped))
sort.Sort(sortableContainers(report.stale))

@ -4,6 +4,7 @@ package types
type Report interface {
Scanned() []ContainerReport
Updated() []ContainerReport
Deferred() []ContainerReport
Failed() []ContainerReport
Skipped() []ContainerReport
Stale() []ContainerReport

@ -12,7 +12,7 @@ type UpdateParams struct {
Timeout time.Duration
MonitorOnly bool
NoPull bool
DelayDays int
DeferDays int
LifecycleHooks bool
RollingRestart bool
LabelPrecedence bool

@ -18,7 +18,7 @@ func main() {
var states string
var entries string
flag.StringVar(&states, "states", "cccuuueeekkktttfff", "sCanned, Updated, failEd, sKipped, sTale, Fresh")
flag.StringVar(&states, "states", "cccuuudddeeekkktttfff", "sCanned, Updated, Deferred, failEd, sKipped, sTale, Fresh")
flag.StringVar(&entries, "entries", "ewwiiidddd", "Fatal,Error,Warn,Info,Debug,Trace")
flag.Parse()

Loading…
Cancel
Save