Feat/lifecycle hooks (#351)

* feat(update): add lifecycle hooks to the update action

* fix(ci): add bash tests for lifecycle-hooks to the ci workflow

* fix(ci): move integration tests to an isolated step

* fix(ci): fix malformed all-contributors json

* fix(ci): disable automatic bash test until we figure out a reasonable way to run it in circleci
pull/353/head
Simon Aronsson 5 years ago committed by GitHub
parent 874180a518
commit bfae38dbf8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -302,7 +302,7 @@
"code",
"doc"
]
}
},
{
"login": "zoispag",
"name": "Zois Pagoulatos",

@ -36,9 +36,18 @@ workflows:
only: /.*/
tags:
only: /.*/
# - integration_testing:
# requires:
# - checkout
# filters:
# branches:
# only: /.*/
# tags:
# only: /.*/
- build:
requires:
- testing
# - integration_testing
- linting
filters:
branches:
@ -90,6 +99,14 @@ jobs:
- run: go get -u github.com/haya14busa/goverage
- run: goverage -v -coverprofile=coverage.out ./...
- run: godacov -t $CODACY_TOKEN -r ./coverage.out -c $CIRCLE_SHA1
#integration_testing:
# executor: go
# steps:
# - attach_workspace:
# at: .
# - run: go build .
# - setup_remote_docker
# - run: ./scripts/lifecycle-tests.sh
build:
executor: go
steps:

@ -23,14 +23,15 @@ import (
const DockerAPIMinVersion string = "1.24"
var (
client container.Client
scheduleSpec string
cleanup bool
noRestart bool
monitorOnly bool
enableLabel bool
notifier *notifications.Notifier
timeout time.Duration
client container.Client
scheduleSpec string
cleanup bool
noRestart bool
monitorOnly bool
enableLabel bool
notifier *notifications.Notifier
timeout time.Duration
lifecycleHooks bool
)
var rootCmd = &cobra.Command{
@ -84,7 +85,9 @@ func PreRun(cmd *cobra.Command, args []string) {
if timeout < 0 {
log.Fatal("Please specify a positive value for timeout value.")
}
enableLabel, _ = f.GetBool("label-enable")
lifecycleHooks, _ = f.GetBool("enable-lifecycle-hooks")
// configure environment vars for client
err := flags.EnvConfig(cmd, DockerAPIMinVersion)
@ -95,6 +98,7 @@ func PreRun(cmd *cobra.Command, args []string) {
noPull, _ := f.GetBool("no-pull")
includeStopped, _ := f.GetBool("include-stopped")
removeVolumes, _ := f.GetBool("remove-volumes")
client = container.NewClient(
!noPull,
includeStopped,
@ -171,11 +175,12 @@ func runUpgradesOnSchedule(filter t.Filter) error {
func runUpdatesWithNotifications(filter t.Filter) {
notifier.StartNotification()
updateParams := actions.UpdateParams{
Filter: filter,
Cleanup: cleanup,
NoRestart: noRestart,
Timeout: timeout,
MonitorOnly: monitorOnly,
Filter: filter,
Cleanup: cleanup,
NoRestart: noRestart,
Timeout: timeout,
MonitorOnly: monitorOnly,
LifecycleHooks: lifecycleHooks,
}
err := actions.Update(client, updateParams)
if err != nil {

@ -0,0 +1,45 @@
## Executing commands before and after updating
> **DO NOTE**: Both commands are shell commands executed with `sh`, and therefore require the
> container to provide the `sh` executable.
It is possible to execute a *pre-update* command and a *post-update* command
**inside** every container updated by watchtower. The *pre-update* command is
executed before stopping the container, and the *post-update* command is
executed after restarting the container.
This feature is disabled by default. To enable it, you need to set the option
`--enable-lifecycle-hooks` on the command line, or set the environment variable
`WATCHTOWER_LIFECYCLE_HOOKS` to true.
### Specifying update commands
The commands are specified using docker container labels, with
`com.centurylinklabs.watchtower.pre-update-command` for the *pre-update*
command and `com.centurylinklabs.watchtower.lifecycle.post-update` for the
*post-update* command.
These labels can be declared as instructions in a Dockerfile:
```docker
LABEL com.centurylinklabs.watchtower.lifecycle.pre-update="/dump-data.sh"
LABEL com.centurylinklabs.watchtower.lifecycle.post-update="/restore-data.sh"
```
Or be specified as part of the `docker run` command line:
```bash
docker run -d \
--label=com.centurylinklabs.watchtower.lifecycle.pre-update="/dump-data.sh" \
--label=com.centurylinklabs.watchtower.lifecycle.post-update="/restore-data.sh" \
someimage
```
### Execution failure
The failure of a command to execute, identified by an exit code different than
0, will not prevent watchtower from updating the container. Only an error
log statement containing the exit code will be reported.

@ -160,7 +160,7 @@ func (client mockClient) StopContainer(c container.Container, d time.Duration) e
}
return nil
}
func (client mockClient) StartContainer(c container.Container) error {
func (client mockClient) StartContainer(c container.Container) (string, error) {
panic("Not implemented")
}
@ -173,6 +173,14 @@ func (client mockClient) RemoveImage(c container.Container) error {
return nil
}
func (client mockClient) GetContainer(containerID string) (container.Container, error) {
return container.Container{}, nil
}
func (client mockClient) ExecuteCommand(containerID string, command string) error {
return nil
}
func (client mockClient) IsContainerStale(c container.Container) (bool, error) {
panic("Not implemented")
}

@ -61,8 +61,9 @@ func stopStaleContainer(container container.Container, client container.Client,
return
}
err := client.StopContainer(container, params.Timeout)
if err != nil {
executePreUpdateCommand(client, container)
if err := client.StopContainer(container, params.Timeout); err != nil {
log.Error(err)
}
}
@ -89,8 +90,10 @@ func restartStaleContainer(container container.Container, client container.Clien
}
if !params.NoRestart {
if err := client.StartContainer(container); err != nil {
if newContainerID, err := client.StartContainer(container); err != nil {
log.Error(err)
} else if container.Stale && params.LifecycleHooks {
executePostUpdateCommand(client, newContainerID)
}
}
@ -104,18 +107,49 @@ func restartStaleContainer(container container.Container, client container.Clien
func checkDependencies(containers []container.Container) {
for i, parent := range containers {
if parent.Stale {
if parent.ToRestart() {
continue
}
LinkLoop:
for _, linkName := range parent.Links() {
for _, child := range containers {
if child.Name() == linkName && child.Stale {
containers[i].Stale = true
if child.Name() == linkName && child.ToRestart() {
containers[i].Linked = true
break LinkLoop
}
}
}
}
}
func executePreUpdateCommand(client container.Client, container container.Container) {
command := container.GetLifecyclePreUpdateCommand()
if len(command) == 0 {
log.Debug("No pre-update command supplied. Skipping")
}
log.Info("Executing pre-update command.")
if err := client.ExecuteCommand(container.ID(), command); err != nil {
log.Error(err)
}
}
func executePostUpdateCommand(client container.Client, newContainerID string) {
newContainer, err := client.GetContainer(newContainerID)
if err != nil {
log.Error(err)
return
}
command := newContainer.GetLifecyclePostUpdateCommand()
if len(command) == 0 {
log.Debug("No post-update command supplied. Skipping")
}
log.Info("Executing post-update command.")
if err := client.ExecuteCommand(newContainerID, command); err != nil {
log.Error(err)
}
}

@ -7,9 +7,10 @@ import (
// UpdateParams contains all different options available to alter the behavior of the Update func
type UpdateParams struct {
Filter t.Filter
Cleanup bool
NoRestart bool
Timeout time.Duration
MonitorOnly bool
Filter t.Filter
Cleanup bool
NoRestart bool
Timeout time.Duration
MonitorOnly bool
LifecycleHooks bool
}

@ -88,6 +88,12 @@ func RegisterSystemFlags(rootCmd *cobra.Command) {
"S",
viper.GetBool("WATCHTOWER_INCLUDE_STOPPED"),
"Will also include created and exited containers")
flags.BoolP(
"enable-lifecycle-hooks",
"",
viper.GetBool("WATCHTOWER_LIFECYCLE_HOOKS"),
"Enable the execution of commands triggered by pre- and post-update lifecycle hooks")
}
// RegisterNotificationFlags that are used by watchtower to send notifications

@ -1,8 +1,10 @@
package container
import (
"bytes"
"fmt"
"io/ioutil"
"strings"
"time"
t "github.com/containrrr/watchtower/pkg/types"
@ -15,18 +17,18 @@ import (
"golang.org/x/net/context"
)
const (
defaultStopSignal = "SIGTERM"
)
const defaultStopSignal = "SIGTERM"
// A Client is the interface through which watchtower interacts with the
// Docker API.
type Client interface {
ListContainers(t.Filter) ([]Container, error)
GetContainer(containerID string) (Container, error)
StopContainer(Container, time.Duration) error
StartContainer(Container) error
StartContainer(Container) (string, error)
RenameContainer(Container, string) error
IsContainerStale(Container) (bool, error)
ExecuteCommand(containerID string, command string) error
RemoveImage(Container) error
}
@ -80,18 +82,12 @@ func (client dockerClient) ListContainers(fn t.Filter) ([]Container, error) {
}
for _, runningContainer := range containers {
containerInfo, err := client.api.ContainerInspect(bg, runningContainer.ID)
if err != nil {
return nil, err
}
imageInfo, _, err := client.api.ImageInspectWithRaw(bg, containerInfo.Image)
c, err := client.GetContainer(runningContainer.ID)
if err != nil {
return nil, err
}
c := Container{containerInfo: &containerInfo, imageInfo: &imageInfo}
if fn(c) {
cs = append(cs, c)
}
@ -112,6 +108,23 @@ func (client dockerClient) createListFilter() filters.Args {
return filterArgs
}
func (client dockerClient) GetContainer(containerID string) (Container, error) {
bg := context.Background()
containerInfo, err := client.api.ContainerInspect(bg, containerID)
if err != nil {
return Container{}, err
}
imageInfo, _, err := client.api.ImageInspectWithRaw(bg, containerInfo.Image)
if err != nil {
return Container{}, err
}
container := Container{containerInfo: &containerInfo, imageInfo: &imageInfo}
return container, nil
}
func (client dockerClient) StopContainer(c Container, timeout time.Duration) error {
bg := context.Background()
signal := c.StopSignal()
@ -147,7 +160,7 @@ func (client dockerClient) StopContainer(c Container, timeout time.Duration) err
return nil
}
func (client dockerClient) StartContainer(c Container) error {
func (client dockerClient) StartContainer(c Container) (string, error) {
bg := context.Background()
config := c.runtimeConfig()
hostConfig := c.hostConfig()
@ -167,40 +180,40 @@ func (client dockerClient) StartContainer(c Container) error {
name := c.Name()
log.Infof("Creating %s", name)
creation, err := client.api.ContainerCreate(bg, config, hostConfig, simpleNetworkConfig, name)
createdContainer, err := client.api.ContainerCreate(bg, config, hostConfig, simpleNetworkConfig, name)
if err != nil {
return err
return "", err
}
if !(hostConfig.NetworkMode.IsHost()) {
for k := range simpleNetworkConfig.EndpointsConfig {
err = client.api.NetworkDisconnect(bg, k, creation.ID, true)
err = client.api.NetworkDisconnect(bg, k, createdContainer.ID, true)
if err != nil {
return err
return "", err
}
}
for k, v := range networkConfig.EndpointsConfig {
err = client.api.NetworkConnect(bg, k, creation.ID, v)
err = client.api.NetworkConnect(bg, k, createdContainer.ID, v)
if err != nil {
return err
return "", err
}
}
}
return client.startContainerIfPreviouslyRunning(bg, c, creation)
if !c.IsRunning() {
return createdContainer.ID, nil
}
return createdContainer.ID, client.doStartContainer(bg, c, createdContainer)
}
func (client dockerClient) startContainerIfPreviouslyRunning(bg context.Context, c Container, creation container.ContainerCreateCreatedBody) error {
func (client dockerClient) doStartContainer(bg context.Context, c Container, creation container.ContainerCreateCreatedBody) error {
name := c.Name()
if !c.IsRunning() {
return nil
}
log.Debugf("Starting container %s (%s)", name, creation.ID)
err := client.api.ContainerStart(bg, creation.ID, types.ContainerStartOptions{})
if err != nil {
@ -271,6 +284,67 @@ func (client dockerClient) RemoveImage(c Container) error {
return err
}
func (client dockerClient) ExecuteCommand(containerID string, command string) error {
bg := context.Background()
// Create the exec
execConfig := types.ExecConfig{
Tty: true,
Detach: false,
Cmd: []string{"sh", "-c", command},
}
exec, err := client.api.ContainerExecCreate(bg, containerID, execConfig)
if err != nil {
return err
}
response, attachErr := client.api.ContainerExecAttach(bg, exec.ID, types.ExecStartCheck{
Tty: true,
Detach: false,
})
if attachErr != nil {
log.Errorf("Failed to extract command exec logs: %v", attachErr)
}
// Run the exec
execStartCheck := types.ExecStartCheck{Detach: false, Tty: true}
err = client.api.ContainerExecStart(bg, exec.ID, execStartCheck)
if err != nil {
return err
}
var execOutput string
if attachErr == nil {
defer response.Close()
var writer bytes.Buffer
written, err := writer.ReadFrom(response.Reader)
if err != nil {
log.Error(err)
} else if written > 0 {
execOutput = strings.TrimSpace(writer.String())
}
}
// Inspect the exec to get the exit code and print a message if the
// exit code is not success.
execInspect, err := client.api.ContainerExecInspect(bg, exec.ID)
if err != nil {
return err
}
if execInspect.ExitCode > 0 {
log.Errorf("Command exited with code %v.", execInspect.ExitCode)
log.Error(execOutput)
} else {
if len(execOutput) > 0 {
log.Infof("Command output:\n%v", execOutput)
}
}
return nil
}
func (client dockerClient) waitForStopOrTimeout(c Container, waitTime time.Duration) error {
bg := context.Background()
timeout := time.After(waitTime)

@ -10,13 +10,6 @@ import (
dockercontainer "github.com/docker/docker/api/types/container"
)
const (
watchtowerLabel = "com.centurylinklabs.watchtower"
signalLabel = "com.centurylinklabs.watchtower.stop-signal"
enableLabel = "com.centurylinklabs.watchtower.enable"
zodiacLabel = "com.centurylinklabs.zodiac.original-image"
)
// NewContainer returns a new Container instance instantiated with the
// specified ContainerInfo and ImageInfo structs.
func NewContainer(containerInfo *types.ContainerJSON, imageInfo *types.ImageInspect) *Container {
@ -28,7 +21,8 @@ func NewContainer(containerInfo *types.ContainerJSON, imageInfo *types.ImageInsp
// Container represents a running Docker container.
type Container struct {
Stale bool
Linked bool
Stale bool
containerInfo *types.ContainerJSON
imageInfo *types.ImageInspect
@ -62,7 +56,7 @@ func (c Container) ImageID() string {
// "latest" tag is assumed.
func (c Container) ImageName() string {
// Compatibility w/ Zodiac deployments
imageName, ok := c.containerInfo.Config.Labels[zodiacLabel]
imageName, ok := c.getLabelValue(zodiacLabel)
if !ok {
imageName = c.containerInfo.Config.Image
}
@ -77,7 +71,7 @@ func (c Container) ImageName() string {
// Enabled returns the value of the container enabled label and if the label
// was set.
func (c Container) Enabled() (bool, bool) {
rawBool, ok := c.containerInfo.Config.Labels[enableLabel]
rawBool, ok := c.getLabelValue(enableLabel)
if !ok {
return false, false
}
@ -105,6 +99,12 @@ func (c Container) Links() []string {
return links
}
// ToRestart return whether the container should be restarted, either because
// is stale or linked to another stale container.
func (c Container) ToRestart() bool {
return c.Stale || c.Linked
}
// IsWatchtower returns a boolean flag indicating whether or not the current
// container is the watchtower container itself. The watchtower container is
// identified by the presence of the "com.centurylinklabs.watchtower" label in
@ -117,11 +117,7 @@ func (c Container) IsWatchtower() bool {
// container's metadata. If the container has not specified a custom stop
// signal, the empty string "" is returned.
func (c Container) StopSignal() string {
if val, ok := c.containerInfo.Config.Labels[signalLabel]; ok {
return val
}
return ""
return c.getLabelValueOrEmpty(signalLabel)
}
// Ideally, we'd just be able to take the ContainerConfig from the old container
@ -189,10 +185,3 @@ func (c Container) hostConfig() *dockercontainer.HostConfig {
return hostConfig
}
// ContainsWatchtowerLabel takes a map of labels and values and tells
// the consumer whether it contains a valid watchtower instance label
func ContainsWatchtowerLabel(labels map[string]string) bool {
val, ok := labels[watchtowerLabel]
return ok && val == "true"
}

@ -0,0 +1,39 @@
package container
const (
watchtowerLabel = "com.centurylinklabs.watchtower"
signalLabel = "com.centurylinklabs.watchtower.stop-signal"
enableLabel = "com.centurylinklabs.watchtower.enable"
zodiacLabel = "com.centurylinklabs.zodiac.original-image"
preUpdateLabel = "com.centurylinklabs.watchtower.lifecycle.pre-update"
postUpdateLabel = "com.centurylinklabs.watchtower.lifecycle.post-update"
)
// GetLifecyclePreUpdateCommand returns the pre-update command set in the container metadata or an empty string
func (c Container) GetLifecyclePreUpdateCommand() string {
return c.getLabelValueOrEmpty(preUpdateLabel)
}
// GetLifecyclePostUpdateCommand returns the post-update command set in the container metadata or an empty string
func (c Container) GetLifecyclePostUpdateCommand() string {
return c.getLabelValueOrEmpty(postUpdateLabel)
}
// ContainsWatchtowerLabel takes a map of labels and values and tells
// the consumer whether it contains a valid watchtower instance label
func ContainsWatchtowerLabel(labels map[string]string) bool {
val, ok := labels[watchtowerLabel]
return ok && val == "true"
}
func (c Container) getLabelValueOrEmpty(label string) string {
if val, ok := c.containerInfo.Config.Labels[label]; ok {
return val
}
return ""
}
func (c Container) getLabelValue(label string) (string, bool) {
val, ok := c.containerInfo.Config.Labels[label]
return val, ok
}

@ -0,0 +1,208 @@
#!/usr/bin/env bash
set -e
IMAGE=server
CONTAINER=server
LINKED_IMAGE=linked
LINKED_CONTAINER=linked
WATCHTOWER_INTERVAL=2
function remove_container {
docker kill $1 >> /dev/null || true && docker rm -v $1 >> /dev/null || true
}
function cleanup {
# Do cleanup on exit or error
echo "Final cleanup"
sleep 2
remove_container $CONTAINER
remove_container $LINKED_CONTAINER
pkill -9 -f watchtower >> /dev/null || true
}
trap cleanup EXIT
DEFAULT_WATCHTOWER="$(dirname "${BASH_SOURCE[0]}")/../watchtower"
WATCHTOWER=$1
WATCHTOWER=${WATCHTOWER:-$DEFAULT_WATCHTOWER}
echo "watchtower path is $WATCHTOWER"
##################################################################################
##### PREPARATION ################################################################
##################################################################################
# Create Dockerfile template
DOCKERFILE=$(cat << EOF
FROM node:alpine
LABEL com.centurylinklabs.watchtower.lifecycle.pre-update="cat /opt/test/value.txt"
LABEL com.centurylinklabs.watchtower.lifecycle.post-update="echo image > /opt/test/value.txt"
ENV IMAGE_TIMESTAMP=TIMESTAMP
WORKDIR /opt/test
ENTRYPOINT ["/usr/local/bin/node", "/opt/test/server.js"]
EXPOSE 8888
RUN mkdir -p /opt/test && echo "default" > /opt/test/value.txt
COPY server.js /opt/test/server.js
EOF
)
# Create temporary directory to build docker image
TMP_DIR="/tmp/watchtower-commands-test"
mkdir -p $TMP_DIR
# Create simple http server
cat > $TMP_DIR/server.js << EOF
const http = require("http");
const fs = require("fs");
http.createServer(function(request, response) {
const fileContent = fs.readFileSync("/opt/test/value.txt");
response.writeHead(200, {"Content-Type": "text/plain"});
response.write(fileContent);
response.end();
}).listen(8888, () => { console.log('server is listening on 8888'); });
EOF
function builddocker {
TIMESTAMP=$(date +%s)
echo "Building image $TIMESTAMP"
echo "${DOCKERFILE/TIMESTAMP/$TIMESTAMP}" > $TMP_DIR/Dockerfile
docker build $TMP_DIR -t $IMAGE >> /dev/null
}
# Start watchtower
echo "Starting watchtower"
$WATCHTOWER -i $WATCHTOWER_INTERVAL --no-pull --stop-timeout 2s --enable-lifecycle-hooks $CONTAINER $LINKED_CONTAINER &
sleep 3
echo "#################################################################"
echo "##### TEST CASE 1: Execute commands from base image"
echo "#################################################################"
# Build base image
builddocker
# Run container
docker run -d -p 0.0.0.0:8888:8888 --name $CONTAINER $IMAGE:latest >> /dev/null
sleep 1
echo "Container $CONTAINER is runnning"
# Test default value
RESP=$(curl -s http://localhost:8888)
if [ $RESP != "default" ]; then
echo "Default value of container response is invalid" 1>&2
exit 1
fi
# Build updated image to trigger watchtower update
builddocker
WAIT_AMOUNT=$(($WATCHTOWER_INTERVAL * 3))
echo "Wait for $WAIT_AMOUNT seconds"
sleep $WAIT_AMOUNT
# Test value after post-update-command
RESP=$(curl -s http://localhost:8888)
if [[ $RESP != "image" ]]; then
echo "Value of container response is invalid. Expected: image. Actual: $RESP"
exit 1
fi
remove_container $CONTAINER
echo "#################################################################"
echo "##### TEST CASE 2: Execute commands from container and base image"
echo "#################################################################"
# Build base image
builddocker
# Run container
docker run -d -p 0.0.0.0:8888:8888 \
--label=com.centurylinklabs.watchtower.lifecycle.post-update="echo container > /opt/test/value.txt" \
--name $CONTAINER $IMAGE:latest >> /dev/null
sleep 1
echo "Container $CONTAINER is runnning"
# Test default value
RESP=$(curl -s http://localhost:8888)
if [ $RESP != "default" ]; then
echo "Default value of container response is invalid" 1>&2
exit 1
fi
# Build updated image to trigger watchtower update
builddocker
WAIT_AMOUNT=$(($WATCHTOWER_INTERVAL * 3))
echo "Wait for $WAIT_AMOUNT seconds"
sleep $WAIT_AMOUNT
# Test value after post-update-command
RESP=$(curl -s http://localhost:8888)
if [[ $RESP != "container" ]]; then
echo "Value of container response is invalid. Expected: container. Actual: $RESP"
exit 1
fi
remove_container $CONTAINER
echo "#################################################################"
echo "##### TEST CASE 3: Execute commands with a linked container"
echo "#################################################################"
# Tag the current image to keep a version for the linked container
docker tag $IMAGE:latest $LINKED_IMAGE:latest
# Build base image
builddocker
# Run container
docker run -d -p 0.0.0.0:8888:8888 \
--label=com.centurylinklabs.watchtower.lifecycle.post-update="echo container > /opt/test/value.txt" \
--name $CONTAINER $IMAGE:latest >> /dev/null
docker run -d -p 0.0.0.0:8989:8888 \
--label=com.centurylinklabs.watchtower.lifecycle.post-update="echo container > /opt/test/value.txt" \
--link $CONTAINER \
--name $LINKED_CONTAINER $LINKED_IMAGE:latest >> /dev/null
sleep 1
echo "Container $CONTAINER and $LINKED_CONTAINER are runnning"
# Test default value
RESP=$(curl -s http://localhost:8888)
if [ $RESP != "default" ]; then
echo "Default value of container response is invalid" 1>&2
exit 1
fi
# Test default value for linked container
RESP=$(curl -s http://localhost:8989)
if [ $RESP != "default" ]; then
echo "Default value of linked container response is invalid" 1>&2
exit 1
fi
# Build updated image to trigger watchtower update
builddocker
WAIT_AMOUNT=$(($WATCHTOWER_INTERVAL * 3))
echo "Wait for $WAIT_AMOUNT seconds"
sleep $WAIT_AMOUNT
# Test value after post-update-command
RESP=$(curl -s http://localhost:8888)
if [[ $RESP != "container" ]]; then
echo "Value of container response is invalid. Expected: container. Actual: $RESP"
exit 1
fi
# Test that linked container did not execute pre/post-update-command
RESP=$(curl -s http://localhost:8989)
if [[ $RESP != "default" ]]; then
echo "Value of linked container response is invalid. Expected: default. Actual: $RESP"
exit 1
fi
Loading…
Cancel
Save