Enable watchtower to update itself

Dockerfile

@@ -1,5 +1,6 @@
 FROM centurylink/ca-certs
 MAINTAINER CenturyLink Labs <ctl-labs-futuretech@centurylink.com>
+LABEL "com.centurylinklabs.watchtower"="true"
 
 COPY watchtower /
 

docker/client.go

@@ -17,11 +17,17 @@ func init() {
 	pullImages = true
 }
 
+type ContainerFilter func(Container) bool
+
+func AllContainersFilter(Container) bool          { return true }
+func WatchtowerContainersFilter(c Container) bool { return c.IsWatchtower() }
+
 type Client interface {
-	ListContainers() ([]Container, error)
+	ListContainers(ContainerFilter) ([]Container, error)
-	RefreshImage(container *Container) error
+	RefreshImage(*Container) error
-	Stop(container Container) error
+	Stop(Container, time.Duration) error
-	Start(container Container) error
+	Start(Container) error
+	Rename(Container, string) error
 }
 
 func NewClient() Client {
@@ -38,7 +44,7 @@ type DockerClient struct {
 	api dockerclient.Client
 }
 
-func (client DockerClient) ListContainers() ([]Container, error) {
+func (client DockerClient) ListContainers(fn ContainerFilter) ([]Container, error) {
 	cs := []Container{}
 
 	runningContainers, err := client.api.ListContainers(false, false, "")
@@ -57,7 +63,10 @@ func (client DockerClient) ListContainers() ([]Container, error) {
 			return nil, err
 		}
 
-		cs = append(cs, Container{containerInfo: containerInfo, imageInfo: imageInfo})
+		c := Container{containerInfo: containerInfo, imageInfo: imageInfo}
+		if fn(c) {
+			cs = append(cs, Container{containerInfo: containerInfo, imageInfo: imageInfo})
+		}
 	}
 
 	return cs, nil
@@ -92,7 +101,7 @@ func (client DockerClient) RefreshImage(c *Container) error {
 	return nil
 }
 
-func (client DockerClient) Stop(c Container) error {
+func (client DockerClient) Stop(c Container, timeout time.Duration) error {
 	signal := "SIGTERM"
 
 	if sig, ok := c.containerInfo.Config.Labels["com.centurylinklabs.watchtower.stop-signal"]; ok {
@@ -106,20 +115,7 @@ func (client DockerClient) Stop(c Container) error {
 	}
 
 	// Wait for container to exit, but proceed anyway after 10 seconds
-	timeout := time.After(10 * time.Second)
+	client.waitForStop(c, timeout)
-PollLoop:
-	for {
-		select {
-		case <-timeout:
-			break PollLoop
-		default:
-			ci, err := client.api.InspectContainer(c.containerInfo.Id)
-			if err != nil || !ci.State.Running {
-				break PollLoop
-			}
-			time.Sleep(1 * time.Second)
-		}
-	}
 
 	return client.api.RemoveContainer(c.containerInfo.Id, true, false)
 }
@@ -127,13 +123,41 @@ PollLoop:
 func (client DockerClient) Start(c Container) error {
 	config := c.runtimeConfig()
 	hostConfig := c.hostConfig()
+	name := c.Name()
 
-	log.Printf("Starting: %s\n", c.Name())
+	if name == "" {
+		log.Printf("Starting new container from %s", c.containerInfo.Config.Image)
+	} else {
+		log.Printf("Starting %s\n", name)
+	}
 
-	newContainerId, err := client.api.CreateContainer(config, c.Name())
+	newContainerId, err := client.api.CreateContainer(config, name)
 	if err != nil {
 		return err
 	}
 
 	return client.api.StartContainer(newContainerId, hostConfig)
 }
+
+func (client DockerClient) Rename(c Container, newName string) error {
+	return client.api.RenameContainer(c.containerInfo.Id, newName)
+}
+
+func (client DockerClient) waitForStop(c Container, waitTime time.Duration) error {
+	timeout := time.After(waitTime)
+
+	for {
+		select {
+		case <-timeout:
+			return nil
+		default:
+			if ci, err := client.api.InspectContainer(c.containerInfo.Id); err != nil {
+				return err
+			} else if !ci.State.Running {
+				return nil
+			}
+
+			time.Sleep(1 * time.Second)
+		}
+	}
+}

docker/client_test.go

@@ -3,6 +3,7 @@ package docker
 import (
 	"errors"
 	"testing"
+	"time"
 
 	"github.com/samalba/dockerclient"
 	"github.com/samalba/dockerclient/mockclient"
@@ -10,6 +11,9 @@ import (
 	"github.com/stretchr/testify/mock"
 )
 
+func allContainers(Container) bool { return true }
+func noContainers(Container) bool  { return false }
+
 func TestListContainers_Success(t *testing.T) {
 	ci := &dockerclient.ContainerInfo{Image: "abc123"}
 	ii := &dockerclient.ImageInfo{}
@@ -19,7 +23,7 @@ func TestListContainers_Success(t *testing.T) {
 	api.On("InspectImage", "abc123").Return(ii, nil)
 
 	client := DockerClient{api: api}
-	cs, err := client.ListContainers()
+	cs, err := client.ListContainers(allContainers)
 
 	assert.NoError(t, err)
 	assert.Len(t, cs, 1)
@@ -28,12 +32,28 @@ func TestListContainers_Success(t *testing.T) {
 	api.AssertExpectations(t)
 }
 
+func TestListContainers_Filter(t *testing.T) {
+	ci := &dockerclient.ContainerInfo{Image: "abc123"}
+	ii := &dockerclient.ImageInfo{}
+	api := mockclient.NewMockClient()
+	api.On("ListContainers", false, false, "").Return([]dockerclient.Container{{Id: "foo"}}, nil)
+	api.On("InspectContainer", "foo").Return(ci, nil)
+	api.On("InspectImage", "abc123").Return(ii, nil)
+
+	client := DockerClient{api: api}
+	cs, err := client.ListContainers(noContainers)
+
+	assert.NoError(t, err)
+	assert.Len(t, cs, 0)
+	api.AssertExpectations(t)
+}
+
 func TestListContainers_ListError(t *testing.T) {
 	api := mockclient.NewMockClient()
 	api.On("ListContainers", false, false, "").Return([]dockerclient.Container{}, errors.New("oops"))
 
 	client := DockerClient{api: api}
-	_, err := client.ListContainers()
+	_, err := client.ListContainers(allContainers)
 
 	assert.Error(t, err)
 	assert.EqualError(t, err, "oops")
@@ -46,7 +66,7 @@ func TestListContainers_InspectContainerError(t *testing.T) {
 	api.On("InspectContainer", "foo").Return(&dockerclient.ContainerInfo{}, errors.New("uh-oh"))
 
 	client := DockerClient{api: api}
-	_, err := client.ListContainers()
+	_, err := client.ListContainers(allContainers)
 
 	assert.Error(t, err)
 	assert.EqualError(t, err, "uh-oh")
@@ -62,7 +82,7 @@ func TestListContainers_InspectImageError(t *testing.T) {
 	api.On("InspectImage", "abc123").Return(ii, errors.New("whoops"))
 
 	client := DockerClient{api: api}
-	_, err := client.ListContainers()
+	_, err := client.ListContainers(allContainers)
 
 	assert.Error(t, err)
 	assert.EqualError(t, err, "whoops")
@@ -176,7 +196,7 @@ func TestStop_DefaultSuccess(t *testing.T) {
 	api.On("RemoveContainer", "abc123", true, false).Return(nil)
 
 	client := DockerClient{api: api}
-	err := client.Stop(c)
+	err := client.Stop(c, time.Second)
 
 	assert.NoError(t, err)
 	api.AssertExpectations(t)
@@ -204,7 +224,7 @@ func TestStop_CustomSignalSuccess(t *testing.T) {
 	api.On("RemoveContainer", "abc123", true, false).Return(nil)
 
 	client := DockerClient{api: api}
-	err := client.Stop(c)
+	err := client.Stop(c, time.Second)
 
 	assert.NoError(t, err)
 	api.AssertExpectations(t)
@@ -223,7 +243,7 @@ func TestStop_KillContainerError(t *testing.T) {
 	api.On("KillContainer", "abc123", "SIGTERM").Return(errors.New("oops"))
 
 	client := DockerClient{api: api}
-	err := client.Stop(c)
+	err := client.Stop(c, time.Second)
 
 	assert.Error(t, err)
 	assert.EqualError(t, err, "oops")
@@ -245,7 +265,7 @@ func TestStop_RemoveContainerError(t *testing.T) {
 	api.On("RemoveContainer", "abc123", true, false).Return(errors.New("whoops"))
 
 	client := DockerClient{api: api}
-	err := client.Stop(c)
+	err := client.Stop(c, time.Second)
 
 	assert.Error(t, err)
 	assert.EqualError(t, err, "whoops")
@@ -321,3 +341,38 @@ func TestStart_StartContainerError(t *testing.T) {
 	assert.EqualError(t, err, "whoops")
 	api.AssertExpectations(t)
 }
+
+func TestRename_Success(t *testing.T) {
+	c := Container{
+		containerInfo: &dockerclient.ContainerInfo{
+			Id: "abc123",
+		},
+	}
+
+	api := mockclient.NewMockClient()
+	api.On("RenameContainer", "abc123", "foo").Return(nil)
+
+	client := DockerClient{api: api}
+	err := client.Rename(c, "foo")
+
+	assert.NoError(t, err)
+	api.AssertExpectations(t)
+}
+
+func TestRename_Error(t *testing.T) {
+	c := Container{
+		containerInfo: &dockerclient.ContainerInfo{
+			Id: "abc123",
+		},
+	}
+
+	api := mockclient.NewMockClient()
+	api.On("RenameContainer", "abc123", "foo").Return(errors.New("oops"))
+
+	client := DockerClient{api: api}
+	err := client.Rename(c, "foo")
+
+	assert.Error(t, err)
+	assert.EqualError(t, err, "oops")
+	api.AssertExpectations(t)
+}

docker/container.go

@@ -3,6 +3,7 @@ package docker
 import (
 	"fmt"
 	"strings"
+	"time"
 
 	"github.com/samalba/dockerclient"
 )
@@ -31,6 +32,11 @@ func (c Container) Links() []string {
 	return links
 }
 
+func (c Container) IsWatchtower() bool {
+	val, ok := c.containerInfo.Config.Labels["com.centurylinklabs.watchtower"]
+	return ok && val == "true"
+}
+
 // Ideally, we'd just be able to take the ContainerConfig from the old container
 // and use it as the starting point for creating the new container; however,
 // the ContainerConfig that comes back from the Inspect call merges the default
@@ -55,11 +61,11 @@ func (c Container) runtimeConfig() *dockerclient.ContainerConfig {
 	}
 
 	if sliceEqual(config.Cmd, imageConfig.Cmd) {
-		config.Cmd = []string{}
+		config.Cmd = nil
 	}
 
 	if sliceEqual(config.Entrypoint, imageConfig.Entrypoint) {
-		config.Entrypoint = []string{}
+		config.Entrypoint = nil
 	}
 
 	config.Env = sliceSubtract(config.Env, imageConfig.Env)
@@ -91,6 +97,26 @@ func (c Container) hostConfig() *dockerclient.HostConfig {
 	return hostConfig
 }
 
+// Sort containers by Created date
+type ByCreated []Container
+
+func (c ByCreated) Len() int      { return len(c) }
+func (c ByCreated) Swap(i, j int) { c[i], c[j] = c[j], c[i] }
+
+func (c ByCreated) Less(i, j int) bool {
+	t1, err := time.Parse(time.RFC3339Nano, c[i].containerInfo.Created)
+	if err != nil {
+		t1 = time.Now()
+	}
+
+	t2, _ := time.Parse(time.RFC3339Nano, c[j].containerInfo.Created)
+	if err != nil {
+		t1 = time.Now()
+	}
+
+	return t1.Before(t2)
+}
+
 func NewTestContainer(name string, links []string) Container {
 	return Container{
 		containerInfo: &dockerclient.ContainerInfo{

docker/container_test.go

@@ -1,6 +1,7 @@
 package docker
 
 import (
+	"sort"
 	"testing"
 
 	"github.com/samalba/dockerclient"
@@ -30,3 +31,62 @@ func TestLinks(t *testing.T) {
 
 	assert.Equal(t, []string{"foo", "bar"}, links)
 }
+
+func TestIsWatchtower_True(t *testing.T) {
+	c := Container{
+		containerInfo: &dockerclient.ContainerInfo{
+			Config: &dockerclient.ContainerConfig{
+				Labels: map[string]string{"com.centurylinklabs.watchtower": "true"},
+			},
+		},
+	}
+
+	assert.True(t, c.IsWatchtower())
+}
+
+func TestIsWatchtower_WrongLabelValue(t *testing.T) {
+	c := Container{
+		containerInfo: &dockerclient.ContainerInfo{
+			Config: &dockerclient.ContainerConfig{
+				Labels: map[string]string{"com.centurylinklabs.watchtower": "false"},
+			},
+		},
+	}
+
+	assert.False(t, c.IsWatchtower())
+}
+
+func TestIsWatchtower_NoLabel(t *testing.T) {
+	c := Container{
+		containerInfo: &dockerclient.ContainerInfo{
+			Config: &dockerclient.ContainerConfig{
+				Labels: map[string]string{},
+			},
+		},
+	}
+
+	assert.False(t, c.IsWatchtower())
+}
+
+func TestByCreated(t *testing.T) {
+	c1 := Container{
+		containerInfo: &dockerclient.ContainerInfo{
+			Created: "2015-07-01T12:00:01.000000000Z",
+		},
+	}
+	c2 := Container{
+		containerInfo: &dockerclient.ContainerInfo{
+			Created: "2015-07-01T12:00:02.000000000Z",
+		},
+	}
+	c3 := Container{
+		containerInfo: &dockerclient.ContainerInfo{
+			Created: "2015-07-01T12:00:02.000000001Z",
+		},
+	}
+	cs := []Container{c3, c2, c1}
+
+	sort.Sort(ByCreated(cs))
+
+	assert.Equal(t, []Container{c1, c2, c3}, cs)
+}

main.go

@@ -20,6 +20,7 @@ func main() {
 	app := cli.NewApp()
 	app.Name = "watchtower"
 	app.Usage = "Automatically update running Docker containers"
+	app.Before = before
 	app.Action = start
 	app.Flags = []cli.Flag{
 		cli.IntFlag{
@@ -46,6 +47,10 @@ func handleSignals() {
 	}()
 }
 
+func before(c *cli.Context) error {
+	return updater.CheckPrereqs()
+}
+
 func start(c *cli.Context) {
 	secs := time.Duration(c.Int("interval")) * time.Second
 

updater/updater.go

@@ -1,12 +1,37 @@
 package updater
 
 import (
+	"math/rand"
+	"sort"
+
 	"github.com/CenturyLinkLabs/watchtower/docker"
 )
 
+var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
+
+func CheckPrereqs() error {
+	client := docker.NewClient()
+
+	containers, err := client.ListContainers(docker.WatchtowerContainersFilter)
+	if err != nil {
+		return err
+	}
+
+	if len(containers) > 1 {
+		sort.Sort(docker.ByCreated(containers))
+
+		// Iterate over all containers execept the last one
+		for _, c := range containers[0 : len(containers)-1] {
+			client.Stop(c, 60)
+		}
+	}
+
+	return nil
+}
+
 func Run() error {
 	client := docker.NewClient()
-	containers, err := client.ListContainers()
+	containers, err := client.ListContainers(docker.AllContainersFilter)
 	if err != nil {
 		return err
 	}
@@ -27,8 +52,13 @@ func Run() error {
 	// Stop stale containers in reverse order
 	for i := len(containers) - 1; i >= 0; i-- {
 		container := containers[i]
+
+		if container.IsWatchtower() {
+			break
+		}
+
 		if container.Stale {
-			if err := client.Stop(container); err != nil {
+			if err := client.Stop(container, 10); err != nil {
 				return err
 			}
 		}
@@ -37,6 +67,16 @@ func Run() error {
 	// Restart stale containers in sorted order
 	for _, container := range containers {
 		if container.Stale {
+			// Since we can't shutdown a watchtower container immediately, we need to
+			// start the new one while the old one is still running. This prevents us
+			// from re-using the same container name so we first rename the current
+			// instance so that the new one can adopt the old name.
+			if container.IsWatchtower() {
+				if err := client.Rename(container, randName()); err != nil {
+					return err
+				}
+			}
+
 			if err := client.Start(container); err != nil {
 				return err
 			}
@@ -69,3 +109,11 @@ func checkDependencies(containers []docker.Container) {
 		}
 	}
 }
+
+func randName() string {
+	b := make([]rune, 32)
+	for i := range b {
+		b[i] = letters[rand.Intn(len(letters))]
+	}
+	return string(b)
+}