@ -1,22 +1,19 @@
package cmd
package cmd
import (
import (
"errors"
"math"
"net/http"
"os"
"os"
"os/signal"
"os/signal"
"strconv"
"strings"
"strings"
"sync"
"syscall"
"syscall"
"time"
"time"
"github.com/containrrr/watchtower/internal/actions"
"github.com/containrrr/watchtower/internal/actions"
"github.com/containrrr/watchtower/internal/flags"
"github.com/containrrr/watchtower/internal/flags"
"github.com/containrrr/watchtower/internal/meta"
"github.com/containrrr/watchtower/internal/meta"
"github.com/containrrr/watchtower/internal/util"
"github.com/containrrr/watchtower/pkg/api"
"github.com/containrrr/watchtower/pkg/api"
apiMetrics "github.com/containrrr/watchtower/pkg/api/metrics"
"github.com/containrrr/watchtower/pkg/api/updates"
"github.com/containrrr/watchtower/pkg/api/update"
"github.com/containrrr/watchtower/pkg/container"
"github.com/containrrr/watchtower/pkg/container"
"github.com/containrrr/watchtower/pkg/filters"
"github.com/containrrr/watchtower/pkg/filters"
"github.com/containrrr/watchtower/pkg/metrics"
"github.com/containrrr/watchtower/pkg/metrics"
@ -31,20 +28,16 @@ import (
var (
var (
client container . Client
client container . Client
scheduleSpec string
scheduleSpec string
cleanup bool
noRestart bool
monitorOnly bool
enableLabel bool
enableLabel bool
disableContainers [ ] string
disableContainers [ ] string
notifier t . Notifier
notifier t . Notifier
timeout time . Duration
lifecycleHooks bool
rollingRestart bool
scope string
scope string
labelPrecedence bool
up = t . UpdateParams { }
)
)
var rootCmd = NewRootCommand ( )
var rootCmd = NewRootCommand ( )
var localLog = notifications . LocalLog
// NewRootCommand creates the root command for watchtower
// NewRootCommand creates the root command for watchtower
func NewRootCommand ( ) * cobra . Command {
func NewRootCommand ( ) * cobra . Command {
@ -87,18 +80,18 @@ func PreRun(cmd *cobra.Command, _ []string) {
scheduleSpec , _ = f . GetString ( "schedule" )
scheduleSpec , _ = f . GetString ( "schedule" )
flags . GetSecretsFromFiles ( cmd )
flags . GetSecretsFromFiles ( cmd )
cleanup, noRestart , monitorOnly , t imeout = flags . ReadFlags ( cmd )
up. Cleanup , up . NoRestart , up . MonitorOnly , up . T imeout = flags . ReadFlags ( cmd )
if t imeout < 0 {
if up. T imeout < 0 {
log . Fatal ( "Please specify a positive value for timeout value." )
log . Fatal ( "Please specify a positive value for timeout value." )
}
}
enableLabel , _ = f . GetBool ( "label-enable" )
enableLabel , _ = f . GetBool ( "label-enable" )
disableContainers , _ = f . GetStringSlice ( "disable-containers" )
disableContainers , _ = f . GetStringSlice ( "disable-containers" )
l ifecycleHooks, _ = f . GetBool ( "enable-lifecycle-hooks" )
up. L ifecycleHooks, _ = f . GetBool ( "enable-lifecycle-hooks" )
r ollingRestart, _ = f . GetBool ( "rolling-restart" )
up. R ollingRestart, _ = f . GetBool ( "rolling-restart" )
scope , _ = f . GetString ( "scope" )
scope , _ = f . GetString ( "scope" )
l abelPrecedence, _ = f . GetBool ( "label-take-precedence" )
up. L abelPrecedence, _ = f . GetBool ( "label-take-precedence" )
if scope != "" {
if scope != "" {
log . Debugf ( ` Using scope %q ` , scope )
log . Debugf ( ` Using scope %q ` , scope )
@ -110,25 +103,22 @@ func PreRun(cmd *cobra.Command, _ []string) {
log . Fatal ( err )
log . Fatal ( err )
}
}
var clientOpts = container . ClientOptions { }
noPull , _ := f . GetBool ( "no-pull" )
noPull , _ := f . GetBool ( "no-pull" )
includeStopped , _ := f . GetBool ( "include-stopped" )
clientOpts . PullImages = ! noPull
includeRestarting , _ := f . GetBool ( "include-restarting" )
clientOpts . IncludeStopped , _ = f . GetBool ( "include-stopped" )
reviveStopped , _ := f . GetBool ( "revive-stopped" )
clientOpts . IncludeRestarting , _ = f . GetBool ( "include-restarting" )
removeVolumes , _ := f . GetBool ( "remove-volumes" )
clientOpts . ReviveStopped , _ = f . GetBool ( "revive-stopped" )
clientOpts . RemoveVolumes , _ = f . GetBool ( "remove-volumes" )
warnOnHeadPullFailed , _ := f . GetString ( "warn-on-head-failure" )
warnOnHeadPullFailed , _ := f . GetString ( "warn-on-head-failure" )
clientOpts . WarnOnHeadFailed = container . WarningStrategy ( warnOnHeadPullFailed )
if m onitorOnly && noPull {
if up. M onitorOnly && noPull {
log . Warn ( "Using `WATCHTOWER_NO_PULL` and `WATCHTOWER_MONITOR_ONLY` simultaneously might lead to no action being taken at all. If this is intentional, you may safely ignore this message." )
log . Warn ( "Using `WATCHTOWER_NO_PULL` and `WATCHTOWER_MONITOR_ONLY` simultaneously might lead to no action being taken at all. If this is intentional, you may safely ignore this message." )
}
}
client = container . NewClient ( container . ClientOptions {
client = container . NewClient ( clientOpts )
PullImages : ! noPull ,
IncludeStopped : includeStopped ,
ReviveStopped : reviveStopped ,
RemoveVolumes : removeVolumes ,
IncludeRestarting : includeRestarting ,
WarnOnHeadFailed : container . WarningStrategy ( warnOnHeadPullFailed ) ,
} )
notifier = notifications . NewNotifier ( cmd )
notifier = notifications . NewNotifier ( cmd )
notifier . AddLogHook ( )
notifier . AddLogHook ( )
@ -137,13 +127,16 @@ func PreRun(cmd *cobra.Command, _ []string) {
// Run is the main execution flow of the command
// Run is the main execution flow of the command
func Run ( c * cobra . Command , names [ ] string ) {
func Run ( c * cobra . Command , names [ ] string ) {
filter , filterDesc := filters . BuildFilter ( names , disableContainers , enableLabel , scope )
filter , filterDesc := filters . BuildFilter ( names , disableContainers , enableLabel , scope )
up . Filter = filter
runOnce , _ := c . PersistentFlags ( ) . GetBool ( "run-once" )
runOnce , _ := c . PersistentFlags ( ) . GetBool ( "run-once" )
enableUpdateAPI , _ := c . PersistentFlags ( ) . GetBool ( "http-api-update ")
enableUpdateAPI , _ := c . PersistentFlags ( ) . GetBool ( "http-api-update s ")
enableMetricsAPI , _ := c . PersistentFlags ( ) . GetBool ( "http-api-metrics" )
enableMetricsAPI , _ := c . PersistentFlags ( ) . GetBool ( "http-api-metrics" )
unblockHTTPAPI , _ := c . PersistentFlags ( ) . GetBool ( "http-api-periodic-polls" )
unblockHTTPAPI , _ := c . PersistentFlags ( ) . GetBool ( "http-api-periodic-polls" )
apiToken , _ := c . PersistentFlags ( ) . GetString ( "http-api-token" )
apiToken , _ := c . PersistentFlags ( ) . GetString ( "http-api-token" )
healthCheck , _ := c . PersistentFlags ( ) . GetBool ( "health-check" )
healthCheck , _ := c . PersistentFlags ( ) . GetBool ( "health-check" )
enableScheduler := ! enableUpdateAPI || unblockHTTPAPI
if healthCheck {
if healthCheck {
// health check should not have pid 1
// health check should not have pid 1
if os . Getpid ( ) == 1 {
if os . Getpid ( ) == 1 {
@ -153,61 +146,97 @@ func Run(c *cobra.Command, names []string) {
os . Exit ( 0 )
os . Exit ( 0 )
}
}
if rollingRestart && m onitorOnly {
if up. RollingRestart && up . M onitorOnly {
log . Fatal ( "Rolling restarts is not compatible with the global monitor only flag" )
log . Fatal ( "Rolling restarts is not compatible with the global monitor only flag" )
}
}
awaitDockerClient ( )
awaitDockerClient ( )
if err := actions . CheckForSanity ( client , filter, r ollingRestart) ; err != nil {
if err := actions . CheckForSanity ( client , up. Filter , up . R ollingRestart) ; err != nil {
logNotifyExit ( err )
logNotifyExit ( err )
}
}
if runOnce {
if runOnce {
writeStartupMessage ( c , time . Time { } , filterDesc )
writeStartupMessage ( c , time . Time { } , filterDesc )
runUpdatesWithNotifications ( filter )
runUpdatesWithNotifications ( up )
notifier . Close ( )
notifier . Close ( )
os . Exit ( 0 )
os . Exit ( 0 )
return
return
}
}
if err := actions . CheckForMultipleWatchtowerInstances ( client , c leanup, scope ) ; err != nil {
if err := actions . CheckForMultipleWatchtowerInstances ( client , up. C leanup, scope ) ; err != nil {
logNotifyExit ( err )
logNotifyExit ( err )
}
}
// The lock is shared between the scheduler and the HTTP API. It only allows one update to run at a time.
// The lock is shared between the scheduler and the HTTP API. It only allows one updates to run at a time.
updateLock := make ( chan bool , 1 )
updateLock := sync . Mutex { }
updateLock <- true
httpAPI := api . New ( apiToken )
httpAPI := api . New ( apiToken )
if enableUpdateAPI {
if enableUpdateAPI {
updateHandler := update . New ( func ( images [ ] string ) {
httpAPI . EnableUpdates ( func ( paramsFunc updates . ModifyParamsFunc ) t . Report {
metric := runUpdatesWithNotifications ( filters . FilterByImage ( images , filter ) )
apiUpdateParams := up
metrics . RegisterScan ( metric )
paramsFunc ( & apiUpdateParams )
} , updateLock )
if up . MonitorOnly && ! apiUpdateParams . MonitorOnly {
httpAPI . RegisterFunc ( updateHandler . Path , updateHandler . Handle )
apiUpdateParams . MonitorOnly = true
// If polling isn't enabled the scheduler is never started and
localLog . Warn ( "Ignoring request to disable monitor only through API" )
// we need to trigger the startup messages manually.
}
if ! unblockHTTPAPI {
report := runUpdatesWithNotifications ( apiUpdateParams )
writeStartupMessage ( c , time . Time { } , filterDesc )
metrics . RegisterScan ( metrics . NewMetric ( report ) )
}
return report
} , & updateLock )
}
}
if enableMetricsAPI {
if enableMetricsAPI {
metricsHandler := apiMetrics . New ( )
httpAPI . EnableMetrics ( )
httpAPI . RegisterHandler ( metricsHandler . Path , metricsHandler . Handle )
}
}
if err := httpAPI . Start ( enableUpdateAPI && ! unblockHTTPAPI ) ; err != nil && ! errors . Is ( err , http . ErrServerClosed ) {
if err := httpAPI . Start ( ) ; err != nil {
log . Error ( "failed to start API" , err )
log . Error ( "failed to start API" , err )
}
}
if err := runUpgradesOnSchedule ( c , filter , filterDesc , updateLock ) ; err != nil {
var firstScan time . Time
log . Error ( err )
var scheduler * cron . Cron
if enableScheduler {
var err error
scheduler , err = runUpgradesOnSchedule ( up , & updateLock )
if err != nil {
log . Errorf ( "Failed to start scheduler: %v" , err )
} else {
firstScan = scheduler . Entries ( ) [ 0 ] . Schedule . Next ( time . Now ( ) )
}
}
}
os . Exit ( 1 )
writeStartupMessage ( c , firstScan , filterDesc )
// Graceful shut-down on SIGINT/SIGTERM
interrupt := make ( chan os . Signal , 1 )
signal . Notify ( interrupt , os . Interrupt )
signal . Notify ( interrupt , syscall . SIGTERM )
recievedSignal := <- interrupt
localLog . WithField ( "signal" , recievedSignal ) . Infof ( "Got shutdown signal. Gracefully shutting down..." )
if scheduler != nil {
scheduler . Stop ( )
}
updateLock . Lock ( )
go func ( ) {
time . Sleep ( time . Second * 3 )
updateLock . Unlock ( )
} ( )
waitFor ( httpAPI . Stop ( ) , "Waiting for HTTP API requests to complete..." )
waitFor ( & updateLock , "Waiting for running updates to be finished..." )
localLog . Info ( "Shutdown completed" )
}
func waitFor ( waitLock * sync . Mutex , delayMessage string ) {
if ! waitLock . TryLock ( ) {
log . Info ( delayMessage )
waitLock . Lock ( )
}
}
}
func logNotifyExit ( err error ) {
func logNotifyExit ( err error ) {
@ -221,48 +250,9 @@ func awaitDockerClient() {
time . Sleep ( 1 * time . Second )
time . Sleep ( 1 * time . Second )
}
}
func formatDuration ( d time . Duration ) string {
sb := strings . Builder { }
hours := int64 ( d . Hours ( ) )
minutes := int64 ( math . Mod ( d . Minutes ( ) , 60 ) )
seconds := int64 ( math . Mod ( d . Seconds ( ) , 60 ) )
if hours == 1 {
sb . WriteString ( "1 hour" )
} else if hours != 0 {
sb . WriteString ( strconv . FormatInt ( hours , 10 ) )
sb . WriteString ( " hours" )
}
if hours != 0 && ( seconds != 0 || minutes != 0 ) {
sb . WriteString ( ", " )
}
if minutes == 1 {
sb . WriteString ( "1 minute" )
} else if minutes != 0 {
sb . WriteString ( strconv . FormatInt ( minutes , 10 ) )
sb . WriteString ( " minutes" )
}
if minutes != 0 && ( seconds != 0 ) {
sb . WriteString ( ", " )
}
if seconds == 1 {
sb . WriteString ( "1 second" )
} else if seconds != 0 || ( hours == 0 && minutes == 0 ) {
sb . WriteString ( strconv . FormatInt ( seconds , 10 ) )
sb . WriteString ( " seconds" )
}
return sb . String ( )
}
func writeStartupMessage ( c * cobra . Command , sched time . Time , filtering string ) {
func writeStartupMessage ( c * cobra . Command , sched time . Time , filtering string ) {
noStartupMessage , _ := c . PersistentFlags ( ) . GetBool ( "no-startup-message" )
noStartupMessage , _ := c . PersistentFlags ( ) . GetBool ( "no-startup-message" )
enableUpdateAPI , _ := c . PersistentFlags ( ) . GetBool ( "http-api-update ")
enableUpdateAPI , _ := c . PersistentFlags ( ) . GetBool ( "http-api-updates" )
var startupLog * log . Entry
var startupLog * log . Entry
if noStartupMessage {
if noStartupMessage {
@ -285,11 +275,11 @@ func writeStartupMessage(c *cobra.Command, sched time.Time, filtering string) {
startupLog . Info ( filtering )
startupLog . Info ( filtering )
if ! sched . IsZero ( ) {
if ! sched . IsZero ( ) {
until := f ormatDuration( time . Until ( sched ) )
until := util . F ormatDuration( time . Until ( sched ) )
startupLog . Info ( "Scheduling first run: " + sched . Format ( "2006-01-02 15:04:05 -0700 MST" ) )
startupLog . Info ( "Scheduling first run: " + sched . Format ( "2006-01-02 15:04:05 -0700 MST" ) )
startupLog . Info ( "Note that the first check will be performed in " + until )
startupLog . Info ( "Note that the first check will be performed in " + until )
} else if runOnce , _ := c . PersistentFlags ( ) . GetBool ( "run-once" ) ; runOnce {
} else if runOnce , _ := c . PersistentFlags ( ) . GetBool ( "run-once" ) ; runOnce {
startupLog . Info ( "Running a one time update .")
startupLog . Info ( "Running a one time update s .")
} else {
} else {
startupLog . Info ( "Periodic runs are not enabled." )
startupLog . Info ( "Periodic runs are not enabled." )
}
}
@ -309,25 +299,19 @@ func writeStartupMessage(c *cobra.Command, sched time.Time, filtering string) {
}
}
}
}
func runUpgradesOnSchedule ( c * cobra . Command , filter t . Filter , filtering string , lock chan bool ) error {
func runUpgradesOnSchedule ( updateParams t . UpdateParams , updateLock * sync . Mutex ) ( * cron . Cron , error ) {
if lock == nil {
lock = make ( chan bool , 1 )
lock <- true
}
scheduler := cron . New ( )
scheduler := cron . New ( )
err := scheduler . AddFunc (
err := scheduler . AddFunc (
scheduleSpec ,
scheduleSpec ,
func ( ) {
func ( ) {
select {
if updateLock . TryLock ( ) {
case v := <- lock :
defer updateLock . Unlock ( )
defer func ( ) { lock <- v } ( )
result := runUpdatesWithNotifications ( updateParams )
metric := runUpdatesWithNotifications ( filter )
metrics . RegisterScan ( metrics . NewMetric ( result ) )
metrics . RegisterScan ( metric )
} else {
default :
// Update was skipped
// Update was skipped
metrics . RegisterScan ( nil )
metrics . RegisterScan ( nil )
log . Debug ( "Skipped another update already running.")
log . Debug ( "Skipped another update s already running.")
}
}
nextRuns := scheduler . Entries ( )
nextRuns := scheduler . Entries ( )
@ -337,47 +321,28 @@ func runUpgradesOnSchedule(c *cobra.Command, filter t.Filter, filtering string,
} )
} )
if err != nil {
if err != nil {
return err
return nil , err
}
}
writeStartupMessage ( c , scheduler . Entries ( ) [ 0 ] . Schedule . Next ( time . Now ( ) ) , filtering )
scheduler . Start ( )
scheduler . Start ( )
// Graceful shut-down on SIGINT/SIGTERM
return scheduler , nil
interrupt := make ( chan os . Signal , 1 )
signal . Notify ( interrupt , os . Interrupt )
signal . Notify ( interrupt , syscall . SIGTERM )
<- interrupt
scheduler . Stop ( )
log . Info ( "Waiting for running update to be finished..." )
<- lock
return nil
}
}
func runUpdatesWithNotifications ( filter t . Filter ) * metrics . Metric {
func runUpdatesWithNotifications ( updateParams t . UpdateParams ) t . Report {
notifier . StartNotification ( )
notifier . StartNotification ( )
updateParams := t . UpdateParams {
Filter : filter ,
Cleanup : cleanup ,
NoRestart : noRestart ,
Timeout : timeout ,
MonitorOnly : monitorOnly ,
LifecycleHooks : lifecycleHooks ,
RollingRestart : rollingRestart ,
LabelPrecedence : labelPrecedence ,
}
result , err := actions . Update ( client , updateParams )
result , err := actions . Update ( client , updateParams )
if err != nil {
if err != nil {
log . Error ( err )
log . Error ( err )
}
}
notifier . SendNotification ( result )
notifier . SendNotification ( result )
metricResults := metrics . NewMetric ( result )
notifications. L ocalLog. WithFields ( log . Fields {
localLog . WithFields ( log . Fields {
"Scanned" : metricResults . Scanned ,
"Scanned" : len ( result . Scanned ( ) ) ,
"Updated" : metricResults . Updated ,
"Updated" : len ( result . Updated ( ) ) ,
"Failed" : metricResults . Failed ,
"Failed" : len ( result . Failed ( ) ) ,
} ) . Info ( "Session done" )
} ) . Info ( "Session done" )
return metricResults
return result
}
}