Adds scopeUID config to enable multiple instances of Watchtower (#511)

* Adds scopeUID config to enable multiple instances of Watchtower

* Adds tests for multiple instance support with scopeuid

* Adds docs on scope monitoring and multiple instance support

* Adds multiple instances docs to mkdocs config file

* Changes multiple instances check and refactors naming for scope feature

* Applies linter suggestions

* Fixes documentation on Watchtower monitoring scope

cmd/root.go

@@ -30,6 +30,7 @@ var (
 	notifier       *notifications.Notifier
 	timeout        time.Duration
 	lifecycleHooks bool
+	scope	   string
 )
 
 var rootCmd = &cobra.Command{
@@ -90,6 +91,9 @@ func PreRun(cmd *cobra.Command, args []string) {
 
 	enableLabel, _ = f.GetBool("label-enable")
 	lifecycleHooks, _ = f.GetBool("enable-lifecycle-hooks")
+	scope, _ = f.GetString("scope")
+
+	log.Debug(scope)
 
 	// configure environment vars for client
 	err := flags.EnvConfig(cmd)
@@ -118,21 +122,10 @@ func PreRun(cmd *cobra.Command, args []string) {
 
 // Run is the main execution flow of the command
 func Run(c *cobra.Command, names []string) {
-	filter := filters.BuildFilter(names, enableLabel)
+	filter := filters.BuildFilter(names, enableLabel, scope)
 	runOnce, _ := c.PersistentFlags().GetBool("run-once")
 	httpAPI, _ := c.PersistentFlags().GetBool("http-api")
 
-	if httpAPI {
-		apiToken, _ := c.PersistentFlags().GetString("http-api-token")
-
-		if err := api.SetupHTTPUpdates(apiToken, func() { runUpdatesWithNotifications(filter) }); err != nil {
-			log.Fatal(err)
-			os.Exit(1)
-		}
-
-		api.WaitForHTTPUpdates()
-	}
-
 	if runOnce {
 		if noStartupMessage, _ := c.PersistentFlags().GetBool("no-startup-message"); !noStartupMessage {
 			log.Info("Running a one time update.")
@@ -143,10 +136,21 @@ func Run(c *cobra.Command, names []string) {
 		return
 	}
 
-	if err := actions.CheckForMultipleWatchtowerInstances(client, cleanup); err != nil {
+	if err := actions.CheckForMultipleWatchtowerInstances(client, cleanup, scope); err != nil {
 		log.Fatal(err)
 	}
 
+	if httpAPI {
+		apiToken, _ := c.PersistentFlags().GetString("http-api-token")
+
+		if err := api.SetupHTTPUpdates(apiToken, func() { runUpdatesWithNotifications(filter) }); err != nil {
+			log.Fatal(err)
+			os.Exit(1)
+		}
+
+		api.WaitForHTTPUpdates()
+	}
+
 	if err := runUpgradesOnSchedule(c, filter); err != nil {
 		log.Error(err)
 	}

docs/arguments.md

@@ -228,6 +228,16 @@ Environment Variable: WATCHTOWER_HTTP_API_TOKEN
              Default: -
 ``` 
 
+## Filter by scope
+Update containers that have a `com.centurylinklabs.watchtower.scope` label set with the same value as the given argument. This enables [running multiple instances](https://containrrr.github.io/watchtower/running-multiple-instances).
+
+```
+            Argument: --scope
+Environment Variable: WATCHTOWER_SCOPE
+                Type: String
+             Default: -
+``` 
+
 ## Scheduling
 [Cron expression](https://pkg.go.dev/github.com/robfig/cron@v1.2.0?tab=doc#hdr-CRON_Expression_Format) in 6 fields (rather than the traditional 5) which defines when and how often to check for new images. Either `--interval` or the schedule expression 
 can be defined, but not both. An example: `--schedule "0 0 4 * * *"`

docs/container-selection.md

@@ -23,3 +23,9 @@ Or, it can be specified as part of the `docker run` command line:
 ```bash
 docker run -d --label=com.centurylinklabs.watchtower.enable=true someimage
 ```
+
+If you wish to create a monitoring scope, you will need to [run multiple instances and set a scope for each of them](https://containrrr.github.io/watchtower/running-multiple-instances).
+
+Watchtower filters running containers by testing them against each configured criteria. A container is monitored if all criteria are met. For example:
+- If a container's name is on the monitoring name list (not empty `--name` argument) but it is not enabled (_centurylinklabs.watchtower.enable=false_), it won't be monitored;
+- If a container's name is not on the monitoring name list (not empty `--name` argument), even if it is enabled (_centurylinklabs.watchtower.enable=true_ and `--label-enable` flag is set), it won't be monitored;

docs/running-multiple-instances.md

@@ -0,0 +1,27 @@
+By default, Watchtower will clean up other instances and won't allow multiple instances running on the same Docker host or swarm. It is possible to override this behavior by defining a [scope](https://containrrr.github.io/watchtower/arguments/#filter_by_scope) to each running instance. 
+
+Notice that:
+- Multiple instances can't run with the same scope;
+- An instance without a scope will clean up other running instances, even if they have a defined scope;
+
+To define an instance monitoring scope, use the `--scope` argument or the `WATCHTOWER_SCOPE` environment variable on startup and set the _com.centurylinklabs.watchtower.scope_ label with the same value for the containers you want to include in this instance's scope (including the instance itself).
+
+For example, in a Docker Compose config file:
+
+```json
+version: '3'
+
+services:
+  app-monitored-by-watchtower:
+    image: myapps/monitored-by-watchtower
+    labels:
+      - "com.centurylinklabs.watchtower.scope=myscope"
+
+  watchtower:
+    image: containrrr/watchtower
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    command: --interval 30 --scope myscope
+    labels:
+      - "com.centurylinklabs.watchtower.scope=myscope"
+```

internal/actions/actions_suite_test.go

@@ -46,7 +46,7 @@ var _ = Describe("the actions package", func() {
 		When("given an empty array", func() {
 			It("should not do anything", func() {
 				client.TestData.Containers = []container.Container{}
-				err := actions.CheckForMultipleWatchtowerInstances(client, false)
+				err := actions.CheckForMultipleWatchtowerInstances(client, false, "")
 				Expect(err).NotTo(HaveOccurred())
 			})
 		})
@@ -59,7 +59,7 @@ var _ = Describe("the actions package", func() {
 						"watchtower",
 						time.Now()),
 				}
-				err := actions.CheckForMultipleWatchtowerInstances(client, false)
+				err := actions.CheckForMultipleWatchtowerInstances(client, false, "")
 				Expect(err).NotTo(HaveOccurred())
 			})
 		})
@@ -90,7 +90,7 @@ var _ = Describe("the actions package", func() {
 			})
 
 			It("should stop all but the latest one", func() {
-				err := actions.CheckForMultipleWatchtowerInstances(client, false)
+				err := actions.CheckForMultipleWatchtowerInstances(client, false, "")
 				Expect(err).NotTo(HaveOccurred())
 			})
 		})
@@ -120,12 +120,12 @@ var _ = Describe("the actions package", func() {
 				)
 			})
 			It("should try to delete the image if the cleanup flag is true", func() {
-				err := actions.CheckForMultipleWatchtowerInstances(client, true)
+				err := actions.CheckForMultipleWatchtowerInstances(client, true, "")
 				Expect(err).NotTo(HaveOccurred())
 				Expect(client.TestData.TriedToRemoveImage()).To(BeTrue())
 			})
 			It("should not try to delete the image if the cleanup flag is false", func() {
-				err := actions.CheckForMultipleWatchtowerInstances(client, false)
+				err := actions.CheckForMultipleWatchtowerInstances(client, false, "")
 				Expect(err).NotTo(HaveOccurred())
 				Expect(client.TestData.TriedToRemoveImage()).To(BeFalse())
 			})

internal/actions/check.go

@@ -19,10 +19,11 @@ import (
 
 // CheckForMultipleWatchtowerInstances will ensure that there are not multiple instances of the
 // watchtower running simultaneously. If multiple watchtower containers are detected, this function
-// will stop and remove all but the most recently started container.
+// will stop and remove all but the most recently started container. This behaviour can be bypassed
-func CheckForMultipleWatchtowerInstances(client container.Client, cleanup bool) error {
+// if a scope UID is defined.
+func CheckForMultipleWatchtowerInstances(client container.Client, cleanup bool, scope string) error {
 	awaitDockerClient()
-	containers, err := client.ListContainers(filters.WatchtowerContainersFilter)
+	containers, err := client.ListContainers(filters.FilterByScope(scope, filters.WatchtowerContainersFilter))
 
 	if err != nil {
 		log.Fatal(err)

internal/flags/flags.go

@@ -134,6 +134,12 @@ func RegisterSystemFlags(rootCmd *cobra.Command) {
 		"",
 		viper.GetString("WATCHTOWER_HTTP_API_TOKEN"),
 		"Sets an authentication token to HTTP API requests.")
+	
+	flags.StringP(
+		"scope",
+		"",
+		viper.GetString("WATCHTOWER_SCOPE"),
+		"Defines a monitoring scope for the Watchtower instance.")
 }
 
 // RegisterNotificationFlags that are used by watchtower to send notifications

mkdocs.yml

@@ -20,5 +20,6 @@ nav:
    - 'Secure connections': 'secure-connections.md'
    - 'Stop signals': 'stop-signals.md'
    - 'Lifecycle hooks': 'lifecycle-hooks.md'
+   - 'Running multiple instances': 'running-multiple-instances.md'
 plugins:
     - search

pkg/container/container.go

@@ -90,6 +90,17 @@ func (c Container) Enabled() (bool, bool) {
 	return parsedBool, true
 }
 
+// Scope returns the value of the scope UID label and if the label
+// was set.
+func (c Container) Scope() (string, bool) {
+	rawString, ok := c.getLabelValue(scope)
+	if !ok {
+		return "", false
+	}
+
+	return rawString, true
+}
+
 // Links returns a list containing the names of all the containers to which
 // this container is linked.
 func (c Container) Links() []string {

pkg/container/metadata.go

@@ -6,6 +6,7 @@ const (
 	enableLabel           = "com.centurylinklabs.watchtower.enable"
 	dependsOnLabel        = "com.centurylinklabs.watchtower.depends-on"
 	zodiacLabel           = "com.centurylinklabs.zodiac.original-image"
+	scope                 = "com.centurylinklabs.watchtower.scope"
 	preCheckLabel         = "com.centurylinklabs.watchtower.lifecycle.pre-check"
 	postCheckLabel        = "com.centurylinklabs.watchtower.lifecycle.post-check"
 	preUpdateLabel        = "com.centurylinklabs.watchtower.lifecycle.pre-update"

pkg/container/mocks/FilterableContainer.go

@@ -55,3 +55,27 @@ func (_m *FilterableContainer) Name() string {
 
 	return r0
 }
+
+// Scope provides a mock function with given fields:
+func (_m *FilterableContainer) Scope() (string, bool) {
+	ret := _m.Called()
+
+	var r0 string
+
+	if rf, ok := ret.Get(0).(func() string); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Get(0).(string)
+	}
+
+	var r1 bool
+
+	if rf, ok := ret.Get(1).(func() bool); ok {
+		r1 = rf()
+	} else {
+		r1 = ret.Get(1).(bool)
+	}
+
+	return r0, r1
+}
+

pkg/filters/filters.go

@@ -51,8 +51,24 @@ func FilterByDisabledLabel(baseFilter t.Filter) t.Filter {
 	}
 }
 
+// FilterByScope returns all containers that belongs to a specific scope
+func FilterByScope(scope string, baseFilter t.Filter) t.Filter {
+	if scope == "" {
+		return baseFilter
+	}
+	
+	return func(c t.FilterableContainer) bool {
+		containerScope, ok := c.Scope()
+		if ok && containerScope == scope {
+			return baseFilter(c)
+		}
+
+		return false
+	}
+}
+
 // BuildFilter creates the needed filter of containers
-func BuildFilter(names []string, enableLabel bool) t.Filter {
+func BuildFilter(names []string, enableLabel bool, scope string) t.Filter {
 	filter := NoFilter
 	filter = FilterByNames(names, filter)
 	if enableLabel {
@@ -60,6 +76,11 @@ func BuildFilter(names []string, enableLabel bool) t.Filter {
 		// if the label is specifically set.
 		filter = FilterByEnableLabel(filter)
 	}
+	if scope != "" {
+		// If a scope has been defined, containers should only be considered
+		// if the scope is specifically set.
+		filter = FilterByScope(scope, filter)
+	}
 	filter = FilterByDisabledLabel(filter)
 	return filter
 }

pkg/filters/filters_test.go

@@ -67,6 +67,29 @@ func TestFilterByEnableLabel(t *testing.T) {
 	container.AssertExpectations(t)
 }
 
+func TestFilterByScope(t *testing.T) {
+	var scope string
+	scope = "testscope"
+
+	filter := FilterByScope(scope, NoFilter)
+	assert.NotNil(t, filter)
+
+	container := new(mocks.FilterableContainer)
+	container.On("Scope").Return("testscope", true)
+	assert.True(t, filter(container))
+	container.AssertExpectations(t)
+
+	container = new(mocks.FilterableContainer)
+	container.On("Scope").Return("nottestscope", true)
+	assert.False(t, filter(container))
+	container.AssertExpectations(t)
+
+	container = new(mocks.FilterableContainer)
+	container.On("Scope").Return("", false)
+	assert.False(t, filter(container))
+	container.AssertExpectations(t)
+}
+
 func TestFilterByDisabledLabel(t *testing.T) {
 	filter := FilterByDisabledLabel(NoFilter)
 	assert.NotNil(t, filter)
@@ -91,7 +114,7 @@ func TestBuildFilter(t *testing.T) {
 	var names []string
 	names = append(names, "test")
 
-	filter := BuildFilter(names, false)
+	filter := BuildFilter(names, false, "")
 
 	container := new(mocks.FilterableContainer)
 	container.On("Name").Return("Invalid")
@@ -127,7 +150,7 @@ func TestBuildFilterEnableLabel(t *testing.T) {
 	var names []string
 	names = append(names, "test")
 
-	filter := BuildFilter(names, true)
+	filter := BuildFilter(names, true, "")
 
 	container := new(mocks.FilterableContainer)
 	container.On("Enabled").Return(false, false)

pkg/types/filterable_container.go

@@ -6,4 +6,5 @@ type FilterableContainer interface {
 	Name() string
 	IsWatchtower() bool
 	Enabled() (bool, bool)
+	Scope() (string, bool)
 }