fix tests, simplify and integrate credentials properly

pull/674/head
Simon Aronsson 4 years ago
parent 3d21ea683c
commit 24cf0fd6a3
No known key found for this signature in database
GPG Key ID: 8DA57A5FD341605B

@ -17,6 +17,7 @@ func CreateMockContainer(id string, name string, image string, created time.Time
Created: created.String(),
},
Config: &container2.Config{
Image: image,
Labels: make(map[string]string),
},
}
@ -24,9 +25,38 @@ func CreateMockContainer(id string, name string, image string, created time.Time
&content,
&types.ImageInspect{
ID: image,
RepoDigests: []string{
image,
},
},
)
}
// CreateMockContainerWithImageInfo should only be used for testing
func CreateMockContainerWithImageInfo(id string, name string, image string, created time.Time, imageInfo types.ImageInspect) container.Container {
content := types.ContainerJSON{
ContainerJSONBase: &types.ContainerJSONBase{
ID: id,
Image: image,
Name: name,
Created: created.String(),
},
Config: &container2.Config{
Image: image,
Labels: make(map[string]string),
},
}
return *container.NewContainer(
&content,
&imageInfo,
)
}
// CreateMockContainerWithDigest should only be used for testing
func CreateMockContainerWithDigest(id string, name string, image string, created time.Time, digest string) container.Container {
c := CreateMockContainer(id, name, image, created)
c.ImageInfo().RepoDigests = []string{digest}
return c
}
// CreateMockContainerWithConfig creates a container substitute valid for testing
func CreateMockContainerWithConfig(id string, name string, image string, created time.Time, config *container2.Config) container.Container {

@ -294,7 +294,7 @@ func (client dockerClient) PullImage(ctx context.Context, container Container) e
log.WithFields(fields).Debugf("Checking if pull is needed")
if match, err := digest.CompareDigest(ctx, *container.ImageInfo(), nil); err != nil {
if match, err := digest.CompareDigest(ctx, container, opts.RegistryAuth); err != nil {
log.Info("Could not do a head request, falling back to regulara pull.")
log.Debugf("Reason: %s", err.Error())
} else if match {

@ -1,42 +0,0 @@
package logger
import (
"context"
"github.com/sirupsen/logrus"
)
type contextKeyType string
const contextKey = contextKeyType("LogrusLoggerContext")
// GetLogger returns a logger from the context if one is available, otherwise a default logger
func GetLogger(ctx context.Context) *logrus.Logger {
if logger, ok := ctx.Value(contextKey).(logrus.Logger); ok {
return &logger
}
return newLogger(&logrus.JSONFormatter{}, logrus.InfoLevel)
}
// AddLogger adds a logger to the passed context
func AddLogger(ctx context.Context) context.Context {
return setLogger(ctx, &logrus.JSONFormatter{}, logrus.InfoLevel)
}
// AddDebugLogger adds a text-formatted debug logger to the passed context
func AddDebugLogger(ctx context.Context) context.Context {
return setLogger(ctx, &logrus.TextFormatter{}, logrus.DebugLevel)
}
// SetLogger adds a logger to the supplied context
func setLogger(ctx context.Context, fmt logrus.Formatter, level logrus.Level) context.Context {
log := newLogger(fmt, level)
return context.WithValue(ctx, contextKey, log)
}
func newLogger(fmt logrus.Formatter, level logrus.Level) *logrus.Logger {
log := logrus.New()
log.SetFormatter(fmt)
log.SetLevel(level)
return log
}

@ -5,11 +5,9 @@ import (
"encoding/json"
"errors"
"fmt"
"github.com/containrrr/watchtower/pkg/logger"
"github.com/containrrr/watchtower/pkg/registry/helpers"
"github.com/containrrr/watchtower/pkg/types"
"github.com/docker/distribution/reference"
apiTypes "github.com/docker/docker/api/types"
"github.com/sirupsen/logrus"
"io/ioutil"
"net/http"
@ -21,22 +19,21 @@ import (
const ChallengeHeader = "WWW-Authenticate"
// GetToken fetches a token for the registry hosting the provided image
func GetToken(ctx context.Context, image apiTypes.ImageInspect, credentials *types.RegistryCredentials) (string, error) {
func GetToken(ctx context.Context, container types.Container, registryAuth string) (string, error) {
var err error
log := logger.GetLogger(ctx)
img := strings.Split(image.RepoTags[0], ":")[0]
var url url2.URL
if url, err = GetChallengeURL(img); err != nil {
if url, err = GetChallengeURL(container.ImageName()); err != nil {
return "", err
}
logrus.WithField("url", url.String()).Debug("Building challenge URL")
var req *http.Request
if req, err = GetChallengeRequest(url); err != nil {
return "", err
}
var client = http.Client{}
client := &http.Client{}
var res *http.Response
if res, err = client.Do(req); err != nil {
return "", err
@ -44,17 +41,21 @@ func GetToken(ctx context.Context, image apiTypes.ImageInspect, credentials *typ
v := res.Header.Get(ChallengeHeader)
log.WithFields(logrus.Fields{
logrus.WithFields(logrus.Fields{
"status": res.Status,
"header": v,
}).Debug("Got response to challenge request")
challenge := strings.ToLower(v)
if strings.HasPrefix(challenge, "basic") {
return "", errors.New("basic auth not implemented yet")
if registryAuth == "" {
return "", fmt.Errorf("no credentials available")
}
return fmt.Sprintf("Basic %s", registryAuth), nil
}
if strings.HasPrefix(challenge, "bearer") {
log.Debug("Fetching bearer token")
return GetBearerToken(ctx, challenge, img, err, credentials)
return GetBearerHeader(ctx, challenge, container.ImageName(), err, registryAuth)
}
return "", errors.New("unsupported challenge type from registry")
@ -62,7 +63,6 @@ func GetToken(ctx context.Context, image apiTypes.ImageInspect, credentials *typ
// GetChallengeRequest creates a request for getting challenge instructions
func GetChallengeRequest(url url2.URL) (*http.Request, error) {
req, err := http.NewRequest("GET", url.String(), nil)
if err != nil {
return nil, err
@ -72,12 +72,14 @@ func GetChallengeRequest(url url2.URL) (*http.Request, error) {
return req, nil
}
// GetBearerToken tries to fetch a bearer token from the registry based on the challenge instructions
func GetBearerToken(ctx context.Context, challenge string, img string, err error, credentials *types.RegistryCredentials) (string, error) {
log := logger.GetLogger(ctx)
// GetBearerHeader tries to fetch a bearer token from the registry based on the challenge instructions
func GetBearerHeader(ctx context.Context, challenge string, img string, err error, registryAuth string) (string, error) {
client := http.Client{}
if strings.Contains(img, ":") {
img = strings.Split(img, ":")[0]
}
authURL, err := GetAuthURL(challenge, img)
if err != nil {
return "", err
}
@ -87,11 +89,11 @@ func GetBearerToken(ctx context.Context, challenge string, img string, err error
return "", err
}
if credentials != nil && credentials.Username != "" && credentials.Password != "" {
log.WithField("credentials", credentials).Debug("Found credentials. Adding basic auth.")
r.SetBasicAuth(credentials.Username, credentials.Password)
if registryAuth != "" {
logrus.WithField("credentials", registryAuth).Debug("Credentials found.")
r.Header.Add("Authorization", fmt.Sprintf("Basic %s", registryAuth))
} else {
log.Debug("No credentials found. Doing an anonymous request.")
logrus.Debug("No credentials found.")
}
var authResponse *http.Response
@ -107,7 +109,7 @@ func GetBearerToken(ctx context.Context, challenge string, img string, err error
return "", err
}
return tokenResponse.Token, nil
return fmt.Sprintf("Bearer %s", tokenResponse.Token), nil
}
// GetAuthURL from the instructions in the challenge
@ -124,11 +126,12 @@ func GetAuthURL(challenge string, img string) (*url2.URL, error) {
val := strings.Trim(kv[1], "\"")
values[key] = val
}
logrus.WithFields(logrus.Fields{
"realm": values["realm"],
"service": values["service"],
}).Debug("Checking challenge header content")
if values["realm"] == "" || values["service"] == "" {
logrus.WithFields(logrus.Fields{
"realm": values["realm"],
"service": values["service"],
}).Debug("Checking challenge header content")
return nil, fmt.Errorf("challenge header did not include all values needed to construct an auth url")
}
@ -140,6 +143,7 @@ func GetAuthURL(challenge string, img string) (*url2.URL, error) {
scopeImage = "library/" + scopeImage
}
scope := fmt.Sprintf("repository:%s:pull", scopeImage)
logrus.WithFields(logrus.Fields{"scope": scope, "image": img}).Debug("Setting scope for auth token")
q.Add("scope", scope)
authURL.RawQuery = q.Encode()
@ -148,8 +152,9 @@ func GetAuthURL(challenge string, img string) (*url2.URL, error) {
// GetChallengeURL creates a URL object based on the image info
func GetChallengeURL(img string) (url2.URL, error) {
normalizedNamed, _ := reference.ParseNormalizedNamed(img)
host, err := helpers.NormalizeRegistry(normalizedNamed.Name())
host, err := helpers.NormalizeRegistry(normalizedNamed.String())
if err != nil {
return url2.URL{}, err
}

@ -1,14 +1,16 @@
package auth
package auth_test
import (
"context"
"fmt"
"github.com/containrrr/watchtower/internal/actions/mocks"
"github.com/containrrr/watchtower/pkg/registry/auth"
"net/url"
"os"
"testing"
"time"
"github.com/containrrr/watchtower/pkg/logger"
wtTypes "github.com/containrrr/watchtower/pkg/types"
"github.com/docker/docker/api/types"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
@ -37,21 +39,24 @@ var GHCRCredentials = &wtTypes.RegistryCredentials{
}
var _ = Describe("the auth module", func() {
var ctx = logger.AddDebugLogger(context.Background())
var ghImage = types.ImageInspect{
ID: "sha256:6972c414f322dfa40324df3c503d4b217ccdec6d576e408ed10437f508f4181b",
RepoTags: []string{
"ghcr.io/k6io/operator:latest",
},
RepoDigests: []string{
"ghcr.io/k6io/operator@sha256:d68e1e532088964195ad3a0a71526bc2f11a78de0def85629beb75e2265f0547",
},
}
mockId := "mock-id"
mockName := "mock-container"
mockImage := "ghcr.io/k6io/operator:latest"
mockCreated := time.Now()
mockDigest := "ghcr.io/k6io/operator@sha256:d68e1e532088964195ad3a0a71526bc2f11a78de0def85629beb75e2265f0547"
mockContainer := mocks.CreateMockContainerWithDigest(
mockId,
mockName,
mockImage,
mockCreated,
mockDigest)
When("getting an auth url", func() {
It("should parse the token from the response",
SkipIfCredentialsEmpty(GHCRCredentials, func() {
token, err := GetToken(ctx, ghImage, GHCRCredentials)
creds := fmt.Sprintf("%s:%s", GHCRCredentials.Username, GHCRCredentials.Password)
token, err := auth.GetToken(context.Background(), mockContainer, creds)
Expect(err).NotTo(HaveOccurred())
Expect(token).NotTo(Equal(""))
}),
@ -65,13 +70,13 @@ var _ = Describe("the auth module", func() {
Path: "/token",
RawQuery: "scope=repository%3Acontainrrr%2Fwatchtower%3Apull&service=ghcr.io",
}
res, err := GetAuthURL(input, "containrrr/watchtower")
res, err := auth.GetAuthURL(input, "containrrr/watchtower")
Expect(err).NotTo(HaveOccurred())
Expect(res).To(Equal(expected))
})
It("should create a valid auth url object based on the challenge header supplied", func() {
input := `bearer realm="https://ghcr.io/token"`
res, err := GetAuthURL(input, "containrrr/watchtower")
res, err := auth.GetAuthURL(input, "containrrr/watchtower")
Expect(err).To(HaveOccurred())
Expect(res).To(BeNil())
})
@ -79,16 +84,16 @@ var _ = Describe("the auth module", func() {
When("getting a challenge url", func() {
It("should create a valid challenge url object based on the image ref supplied", func() {
expected := url.URL{Host: "ghcr.io", Scheme: "https", Path: "/v2/"}
Expect(GetChallengeURL("ghcr.io/containrrr/watchtower:latest")).To(Equal(expected))
Expect(auth.GetChallengeURL("ghcr.io/containrrr/watchtower:latest")).To(Equal(expected))
})
It("should assume dockerhub if the image ref is not fully qualified", func() {
expected := url.URL{Host: "index.docker.io", Scheme: "https", Path: "/v2/"}
Expect(GetChallengeURL("containrrr/watchtower:latest")).To(Equal(expected))
Expect(auth.GetChallengeURL("containrrr/watchtower:latest")).To(Equal(expected))
})
It("should convert legacy dockerhub hostnames to index.docker.io", func() {
expected := url.URL{Host: "index.docker.io", Scheme: "https", Path: "/v2/"}
Expect(GetChallengeURL("docker.io/containrrr/watchtower:latest")).To(Equal(expected))
Expect(GetChallengeURL("registry-1.docker.io/containrrr/watchtower:latest")).To(Equal(expected))
Expect(auth.GetChallengeURL("docker.io/containrrr/watchtower:latest")).To(Equal(expected))
Expect(auth.GetChallengeURL("registry-1.docker.io/containrrr/watchtower:latest")).To(Equal(expected))
})
})
})

@ -2,13 +2,14 @@ package digest
import (
"context"
"crypto/tls"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"github.com/containrrr/watchtower/pkg/logger"
"github.com/containrrr/watchtower/pkg/registry/auth"
"github.com/containrrr/watchtower/pkg/registry/manifest"
"github.com/containrrr/watchtower/pkg/types"
apiTypes "github.com/docker/docker/api/types"
"github.com/sirupsen/logrus"
"net/http"
"strings"
@ -18,34 +19,31 @@ import (
const ContentDigestHeader = "Docker-Content-Digest"
// CompareDigest ...
func CompareDigest(ctx context.Context, image apiTypes.ImageInspect, credentials *types.RegistryCredentials) (bool, error) {
func CompareDigest(ctx context.Context, container types.Container, registryAuth string) (bool, error) {
var digest string
token, err := auth.GetToken(ctx, image, credentials)
registryAuth = TransformAuth(registryAuth)
token, err := auth.GetToken(ctx, container, registryAuth)
if err != nil {
return false, err
}
digestURL, err := manifest.BuildManifestURL(image)
digestURL, err := manifest.BuildManifestURL(container)
if err != nil {
return false, err
}
if digest, err = GetDigest(ctx, digestURL, token); err != nil {
if digest, err = GetDigest(digestURL, token); err != nil {
return false, err
}
logrus.WithField("remote", digest).Debug("Found a remote digest to compare with")
if image.ID == digest {
return true, nil
}
for _, dig := range image.RepoDigests {
for _, dig := range container.ImageInfo().RepoDigests {
localDigest := strings.Split(dig, "@")[1]
logrus.WithFields(logrus.Fields{
"local": localDigest,
"remote": digest,
}).Debug("Comparing")
fields := logrus.Fields{"local": localDigest, "remote": digest}
logrus.WithFields(fields).Debug("Comparing")
if localDigest == digest {
logrus.Debug("Found a match")
return true, nil
@ -55,18 +53,36 @@ func CompareDigest(ctx context.Context, image apiTypes.ImageInspect, credentials
return false, nil
}
// TransformAuth from a base64 encoded json object to base64 encoded string
func TransformAuth(registryAuth string) string {
b, _ := base64.StdEncoding.DecodeString(registryAuth)
credentials := &types.RegistryCredentials{}
_ = json.Unmarshal(b, credentials)
if credentials.Username != "" && credentials.Password != "" {
ba := []byte(fmt.Sprintf("%s:%s", credentials.Username, credentials.Password))
registryAuth = base64.StdEncoding.EncodeToString(ba)
}
return registryAuth
}
// GetDigest from registry using a HEAD request to prevent rate limiting
func GetDigest(ctx context.Context, url string, token string) (string, error) {
client := &http.Client{}
log := logger.GetLogger(ctx).WithField("fun", "GetDigest")
func GetDigest(url string, token string) (string, error) {
tr := &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
client := &http.Client{Transport: tr}
if token != "" {
log.WithField("token", token).Debug("Setting request bearer token")
logrus.WithField("token", token).Trace("Setting request token")
} else {
return "", errors.New("could not fetch token")
}
req, _ := http.NewRequest("HEAD", url, nil)
req.Header.Add("Authorization", "Bearer "+token)
req.Header.Add("Authorization", token)
req.Header.Add("Accept", "application/vnd.docker.distribution.manifest.v2+json")
req.Header.Add("Accept", "*/*")
logrus.WithField("url", url).Debug("Doing a HEAD request to fetch a digest")

@ -1,15 +1,16 @@
package digest
package digest_test
import (
"context"
"fmt"
"github.com/containrrr/watchtower/pkg/logger"
"github.com/containrrr/watchtower/internal/actions/mocks"
"github.com/containrrr/watchtower/pkg/registry/digest"
wtTypes "github.com/containrrr/watchtower/pkg/types"
"github.com/docker/docker/api/types"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"os"
"testing"
"time"
)
func TestDigest(t *testing.T) {
@ -18,16 +19,6 @@ func TestDigest(t *testing.T) {
RunSpecs(GinkgoT(), "Digest Suite")
}
var ghImage = types.ImageInspect{
ID: "sha256:6972c414f322dfa40324df3c503d4b217ccdec6d576e408ed10437f508f4181b",
RepoTags: []string{
"ghcr.io/k6io/operator:latest",
},
RepoDigests: []string{
"ghcr.io/k6io/operator@sha256:d68e1e532088964195ad3a0a71526bc2f11a78de0def85629beb75e2265f0547",
},
}
var DockerHubCredentials = &wtTypes.RegistryCredentials{
Username: os.Getenv("CI_INTEGRATION_TEST_REGISTRY_DH_USERNAME"),
Password: os.Getenv("CI_INTEGRATION_TEST_REGISTRY_DH_PASSWORD"),
@ -52,12 +43,24 @@ func SkipIfCredentialsEmpty(credentials *wtTypes.RegistryCredentials, fn func())
}
var _ = Describe("Digests", func() {
var ctx = logger.AddDebugLogger(context.Background())
mockId := "mock-id"
mockName := "mock-container"
mockImage := "ghcr.io/k6io/operator:latest"
mockCreated := time.Now()
mockDigest := "ghcr.io/k6io/operator@sha256:d68e1e532088964195ad3a0a71526bc2f11a78de0def85629beb75e2265f0547"
mockContainer := mocks.CreateMockContainerWithDigest(
mockId,
mockName,
mockImage,
mockCreated,
mockDigest)
When("a digest comparison is done", func() {
It("should return true if digests match",
SkipIfCredentialsEmpty(GHCRCredentials, func() {
matches, err := CompareDigest(ctx, ghImage, GHCRCredentials)
creds := fmt.Sprintf("%s:%s", GHCRCredentials.Username, GHCRCredentials.Password)
matches, err := digest.CompareDigest(context.Background(), mockContainer, creds)
Expect(err).NotTo(HaveOccurred())
Expect(matches).To(Equal(true))
}),

@ -3,21 +3,31 @@ package manifest
import (
"fmt"
"github.com/containrrr/watchtower/pkg/registry/helpers"
"github.com/containrrr/watchtower/pkg/types"
ref "github.com/docker/distribution/reference"
apiTypes "github.com/docker/docker/api/types"
"github.com/sirupsen/logrus"
url2 "net/url"
"strings"
)
// BuildManifestURL from raw image data
func BuildManifestURL(image apiTypes.ImageInspect) (string, error) {
img, tag := extractImageAndTag(image)
hostName, err := ref.ParseNormalizedNamed(img)
func BuildManifestURL(container types.Container) (string, error) {
normalizedName, err := ref.ParseNormalizedNamed(container.ImageName())
if err != nil {
return "", err
}
host, err := helpers.NormalizeRegistry(hostName.Name())
host, err := helpers.NormalizeRegistry(normalizedName.String())
img, tag := extractImageAndTag(strings.TrimPrefix(container.ImageName(), host+"/"))
logrus.WithFields(logrus.Fields{
"image": img,
"tag": tag,
"normalized": normalizedName,
"host": host,
}).Debug("Parsing image ref")
if err != nil {
return "", err
}
@ -33,15 +43,21 @@ func BuildManifestURL(image apiTypes.ImageInspect) (string, error) {
return url.String(), nil
}
func extractImageAndTag(image apiTypes.ImageInspect) (string, string) {
func extractImageAndTag(imageName string) (string, string) {
var img string
var tag string
if strings.Contains(image.RepoTags[0], ":") {
parts := strings.Split(image.RepoTags[0], ":")
img = parts[0]
tag = parts[1]
if strings.Contains(imageName, ":") {
parts := strings.Split(imageName, ":")
if len(parts) > 2 {
img = fmt.Sprintf("%s%s", parts[0], parts[1])
tag = parts[3]
} else {
img = parts[0]
tag = parts[1]
}
} else {
img = image.RepoTags[0]
img = imageName
tag = "latest"
}
return img, tag

@ -1,11 +1,13 @@
package manifest_test
import (
"github.com/containrrr/watchtower/internal/actions/mocks"
"github.com/containrrr/watchtower/pkg/registry/manifest"
apiTypes "github.com/docker/docker/api/types"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"testing"
"time"
)
func TestManifest(t *testing.T) {
@ -14,17 +16,20 @@ func TestManifest(t *testing.T) {
}
var _ = Describe("the manifest module", func() {
mockId := "mock-id"
mockName := "mock-container"
mockCreated := time.Now()
When("building a manifest url", func() {
It("should return a valid url given a fully qualified image", func() {
expected := "https://ghcr.io/v2/containrrr/watchtower/manifests/latest"
imageInfo := apiTypes.ImageInspect{
RepoTags: []string{
"ghcr.io/containrrr/watchtower:latest",
"ghcr.io/k6io/operator:latest",
},
}
res, err := manifest.BuildManifestURL(imageInfo)
mock := mocks.CreateMockContainerWithImageInfo(mockId, mockName, "ghcr.io/containrrr/watchtower:latest", mockCreated, imageInfo)
res, err := manifest.BuildManifestURL(mock)
Expect(err).NotTo(HaveOccurred())
Expect(res).To(Equal(expected))
})
@ -36,19 +41,23 @@ var _ = Describe("the manifest module", func() {
},
}
res, err := manifest.BuildManifestURL(imageInfo)
mock := mocks.CreateMockContainerWithImageInfo(mockId, mockName, "containrrr/watchtower:latest", mockCreated, imageInfo)
res, err := manifest.BuildManifestURL(mock)
Expect(err).NotTo(HaveOccurred())
Expect(res).To(Equal(expected))
})
It("should assume latest for images that lack an explicit tag", func() {
expected := "https://index.docker.io/v2/containrrr/watchtower/manifests/latest"
imageInfo := apiTypes.ImageInspect{
RepoTags: []string{
"containrrr/watchtower",
},
}
res, err := manifest.BuildManifestURL(imageInfo)
mock := mocks.CreateMockContainerWithImageInfo(mockId, mockName, "containrrr/watchtower", mockCreated, imageInfo)
res, err := manifest.BuildManifestURL(mock)
Expect(err).NotTo(HaveOccurred())
Expect(res).To(Equal(expected))
})

@ -66,7 +66,7 @@ func EncodedConfigAuth(ref string) (string, error) {
auth, _ := credStore.Get(server) // returns (types.AuthConfig{}) if server not in credStore
if auth == (types.AuthConfig{}) {
log.Debugf("No credentials for %s in %s", server, configFile.Filename)
log.WithField("config_file", configFile.Filename).Debugf("No credentials for %s found", server)
return "", nil
}
log.Debugf("Loaded auth credentials for user %s, on registry %s, from file %s", auth.Username, ref, configFile.Filename)

@ -0,0 +1,26 @@
package types
import "github.com/docker/docker/api/types"
// Container is a docker container running an image
type Container interface {
ContainerInfo() *types.ContainerJSON
ID() string
IsRunning() bool
Name() string
ImageID() string
ImageName() string
Enabled() (bool, bool)
IsMonitorOnly() bool
Scope() (string, bool)
Links() []string
ToRestart() bool
IsWatchtower() bool
StopSignal() string
HasImageInfo() bool
ImageInfo() *types.ImageInspect
GetLifecyclePreCheckCommand() string
GetLifecyclePostCheckCommand() string
GetLifecyclePreUpdateCommand() string
GetLifecyclePostUpdateCommand() string
}
Loading…
Cancel
Save