feat: add timeout override for pre-update lifecycle hook

pull/480/head
Simon Aronsson 5 years ago committed by Simon Aronsson
parent 7e7d4bf9ce
commit 1d1c630f7a

@ -46,6 +46,19 @@ docker run -d \
--label=com.centurylinklabs.watchtower.lifecycle.post-check="/send-heartbeat.sh" \ --label=com.centurylinklabs.watchtower.lifecycle.post-check="/send-heartbeat.sh" \
``` ```
### Timeouts
The timeout for all lifecycle commands is 60 seconds. After that, a timeout will
occur, forcing Watchtower to continue the update loop.
#### Pre-update timeouts
For the `pre-update` lifecycle command, it is possible to override this timeout to
allow the script to finish before forcefully killing it. This is done by adding the
label `com.centurylinklabs.watchtower.lifecycle.pre-update-timeout` followed by
the timeout expressed in minutes.
If the label value is explicitly set to `0`, the timeout will be disabled.
### Execution failure ### Execution failure
The failure of a command to execute, identified by an exit code different than The failure of a command to execute, identified by an exit code different than

@ -61,6 +61,7 @@ github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BU
github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
github.com/docker/docker v0.0.0-20190404075923-dbe4a30928d4 h1:34LfsqlE2kEvmGP9qbRoPvOWkmluYGzmlvWVTzwvT0A= github.com/docker/docker v0.0.0-20190404075923-dbe4a30928d4 h1:34LfsqlE2kEvmGP9qbRoPvOWkmluYGzmlvWVTzwvT0A=
github.com/docker/docker v0.0.0-20190404075923-dbe4a30928d4/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v0.0.0-20190404075923-dbe4a30928d4/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/docker v1.13.1 h1:IkZjBSIc8hBjLpqeAbeE5mca5mNgeatLHBy3GO78BWo=
github.com/docker/docker-credential-helpers v0.6.1 h1:Dq4iIfcM7cNtddhLVWe9h4QDjsi4OER3Z8voPu/I52g= github.com/docker/docker-credential-helpers v0.6.1 h1:Dq4iIfcM7cNtddhLVWe9h4QDjsi4OER3Z8voPu/I52g=
github.com/docker/docker-credential-helpers v0.6.1/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= github.com/docker/docker-credential-helpers v0.6.1/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y=
github.com/docker/go v1.5.1-1 h1:hr4w35acWBPhGBXlzPoHpmZ/ygPjnmFVxGxxGnMyP7k= github.com/docker/go v1.5.1-1 h1:hr4w35acWBPhGBXlzPoHpmZ/ygPjnmFVxGxxGnMyP7k=
@ -284,6 +285,7 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190522155817-f3200d17e092 h1:4QSRKanuywn15aTZvI/mIDEgPQpswuFndXpOj3rKEco= golang.org/x/net v0.0.0-20190522155817-f3200d17e092 h1:4QSRKanuywn15aTZvI/mIDEgPQpswuFndXpOj3rKEco=
golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20191116160921-f9c825593386 h1:ktbWvQrW08Txdxno1PiDpSxPXG6ndGsfnJjRRtkM0LQ=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=

@ -75,7 +75,6 @@ func stopStaleContainer(container container.Container, client container.Client,
} }
if params.LifecycleHooks { if params.LifecycleHooks {
lifecycle.ExecutePreUpdateCommand(client, container) lifecycle.ExecutePreUpdateCommand(client, container)
} }
if err := client.StopContainer(container, params.Timeout); err != nil { if err := client.StopContainer(container, params.Timeout); err != nil {
@ -140,4 +139,4 @@ func checkDependencies(containers []container.Container) {
} }
} }
} }
} }

@ -30,12 +30,14 @@ func RegisterSystemFlags(rootCmd *cobra.Command) {
viper.GetInt("WATCHTOWER_POLL_INTERVAL"), viper.GetInt("WATCHTOWER_POLL_INTERVAL"),
"poll interval (in seconds)") "poll interval (in seconds)")
flags.StringP("schedule", flags.StringP(
"schedule",
"s", "s",
viper.GetString("WATCHTOWER_SCHEDULE"), viper.GetString("WATCHTOWER_SCHEDULE"),
"the cron expression which defines when to update") "the cron expression which defines when to update")
flags.DurationP("stop-timeout", flags.DurationP(
"stop-timeout",
"t", "t",
viper.GetDuration("WATCHTOWER_TIMEOUT"), viper.GetDuration("WATCHTOWER_TIMEOUT"),
"timeout before a container is forcefully stopped") "timeout before a container is forcefully stopped")

@ -29,9 +29,8 @@ type Client interface {
StartContainer(Container) (string, error) StartContainer(Container) (string, error)
RenameContainer(Container, string) error RenameContainer(Container, string) error
IsContainerStale(Container) (bool, error) IsContainerStale(Container) (bool, error)
ExecuteCommand(containerID string, command string) error ExecuteCommand(containerID string, command string, timeout int) error
RemoveImageByID(string) error RemoveImageByID(string) error
} }
// NewClient returns a new Client instance which can be used to interact with // NewClient returns a new Client instance which can be used to interact with
@ -301,7 +300,7 @@ func (client dockerClient) RemoveImageByID(id string) error {
return err return err
} }
func (client dockerClient) ExecuteCommand(containerID string, command string) error { func (client dockerClient) ExecuteCommand(containerID string, command string, timeout int) error {
bg := context.Background() bg := context.Background()
// Create the exec // Create the exec
@ -331,7 +330,7 @@ func (client dockerClient) ExecuteCommand(containerID string, command string) er
return err return err
} }
var execOutput string var output string
if attachErr == nil { if attachErr == nil {
defer response.Close() defer response.Close()
var writer bytes.Buffer var writer bytes.Buffer
@ -339,26 +338,56 @@ func (client dockerClient) ExecuteCommand(containerID string, command string) er
if err != nil { if err != nil {
log.Error(err) log.Error(err)
} else if written > 0 { } else if written > 0 {
execOutput = strings.TrimSpace(writer.String()) output = strings.TrimSpace(writer.String())
} }
} }
// Inspect the exec to get the exit code and print a message if the // Inspect the exec to get the exit code and print a message if the
// exit code is not success. // exit code is not success.
execInspect, err := client.api.ContainerExecInspect(bg, exec.ID) err = client.waitForExecOrTimeout(bg, exec.ID, output, timeout)
if err != nil { if err != nil {
return err return err
} }
if execInspect.ExitCode > 0 { return nil
log.Errorf("Command exited with code %v.", execInspect.ExitCode) }
log.Error(execOutput)
func (client dockerClient) waitForExecOrTimeout(bg context.Context, ID string, execOutput string, timeout int) error {
var ctx context.Context
var cancel context.CancelFunc
if timeout > 0 {
ctx, cancel = context.WithTimeout(bg, time.Duration(timeout)*time.Minute)
defer cancel()
} else { } else {
ctx = bg
}
for {
execInspect, err := client.api.ContainerExecInspect(ctx, ID)
log.WithFields(log.Fields{
"exit-code": execInspect.ExitCode,
"exec-id": execInspect.ExecID,
"running": execInspect.Running,
}).Debug("Awaiting timeout or completion")
if err != nil {
return err
}
if execInspect.Running == true {
time.Sleep(1 * time.Second)
continue
}
if len(execOutput) > 0 { if len(execOutput) > 0 {
log.Infof("Command output:\n%v", execOutput) log.Infof("Command output:\n%v", execOutput)
} }
if execInspect.ExitCode > 0 {
log.Errorf("Command exited with code %v.", execInspect.ExitCode)
log.Error(execOutput)
}
break
} }
return nil return nil
} }
@ -377,7 +406,6 @@ func (client dockerClient) waitForStopOrTimeout(c Container, waitTime time.Durat
return nil return nil
} }
} }
time.Sleep(1 * time.Second) time.Sleep(1 * time.Second)
} }
} }

@ -2,10 +2,11 @@ package container
import ( import (
"fmt" "fmt"
"github.com/containrrr/watchtower/internal/util"
"strconv" "strconv"
"strings" "strings"
"github.com/containrrr/watchtower/internal/util"
"github.com/docker/docker/api/types" "github.com/docker/docker/api/types"
dockercontainer "github.com/docker/docker/api/types/container" dockercontainer "github.com/docker/docker/api/types/container"
) )
@ -118,6 +119,25 @@ func (c Container) IsWatchtower() bool {
return ContainsWatchtowerLabel(c.containerInfo.Config.Labels) return ContainsWatchtowerLabel(c.containerInfo.Config.Labels)
} }
// PreUpdateTimeout checks whether a container has a specific timeout set
// for how long the pre-update command is allowed to run. This value is expressed
// either as an integer, in minutes, or as "off" which will allow the command/script
// to run indefinitely. Users should be cautious with the off option, as that
// could result in watchtower waiting forever.
func (c Container) PreUpdateTimeout() int {
var minutes int
var err error
val := c.getLabelValueOrEmpty(preUpdateTimeoutLabel)
minutes, err = strconv.Atoi(val)
if err != nil || val == "" {
return 1
}
return minutes
}
// StopSignal returns the custom stop signal (if any) that is encoded in the // StopSignal returns the custom stop signal (if any) that is encoded in the
// container's metadata. If the container has not specified a custom stop // container's metadata. If the container has not specified a custom stop
// signal, the empty string "" is returned. // signal, the empty string "" is returned.

@ -1,14 +1,15 @@
package container package container
const ( const (
watchtowerLabel = "com.centurylinklabs.watchtower" watchtowerLabel = "com.centurylinklabs.watchtower"
signalLabel = "com.centurylinklabs.watchtower.stop-signal" signalLabel = "com.centurylinklabs.watchtower.stop-signal"
enableLabel = "com.centurylinklabs.watchtower.enable" enableLabel = "com.centurylinklabs.watchtower.enable"
zodiacLabel = "com.centurylinklabs.zodiac.original-image" zodiacLabel = "com.centurylinklabs.zodiac.original-image"
preCheckLabel = "com.centurylinklabs.watchtower.lifecycle.pre-check" preCheckLabel = "com.centurylinklabs.watchtower.lifecycle.pre-check"
postCheckLabel = "com.centurylinklabs.watchtower.lifecycle.post-check" postCheckLabel = "com.centurylinklabs.watchtower.lifecycle.post-check"
preUpdateLabel = "com.centurylinklabs.watchtower.lifecycle.pre-update" preUpdateLabel = "com.centurylinklabs.watchtower.lifecycle.pre-update"
postUpdateLabel = "com.centurylinklabs.watchtower.lifecycle.post-update" postUpdateLabel = "com.centurylinklabs.watchtower.lifecycle.post-update"
preUpdateTimeoutLabel = "com.centurylinklabs.watchtower.lifecycle.pre-update-timeout"
) )
// GetLifecyclePreCheckCommand returns the pre-check command set in the container metadata or an empty string // GetLifecyclePreCheckCommand returns the pre-check command set in the container metadata or an empty string

Loading…
Cancel
Save