archive
/
tailscale
mirror of https://github.com/tailscale/tailscale/
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

cmd/k8s-operator: Allow specifying cluster ips for nameservers (#16477)

Browse Source 
This commit modifies the kubernetes operator's `DNSConfig` resource
with the addition of a new field at `nameserver.service.clusterIP`.

This field allows users to specify a static in-cluster IP address of
the nameserver when deployed.

Fixes #14305

Signed-off-by: David Bond <davidsbond93@gmail.com>
pull/16595/merge
David Bond 5 months ago committed by GitHub
parent 0d03a3746a
commit c989824aac
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 179 additions and 75 deletions
Show Stats Download Patch File Download Diff File

9
cmd/k8s-operator/deploy/crds/tailscale.com_dnsconfigs.yaml
Unescape Escape View File

@ -101,6 +101,13 @@ spec:
tag:
description: Tag defaults to unstable.
type: string
service:
description: Service configuration.
type: object
properties:
clusterIP:
description: ClusterIP sets the static IP of the service used by the nameserver.
type: string
status:
description: |-
Status describes the status of the DNSConfig. This is set
@ -172,7 +179,7 @@ spec:
ip:
description: |-
IP is the ClusterIP of the Service fronting the deployed ts.net nameserver.
Currently you must manually update your cluster DNS config to add
Currently, you must manually update your cluster DNS config to add
this address as a stub nameserver for ts.net for cluster workloads to be
able to resolve MagicDNS names associated with egress or Ingress
proxies.

9
cmd/k8s-operator/deploy/manifests/operator.yaml
Unescape Escape View File

@ -389,6 +389,13 @@ spec:
description: Tag defaults to unstable.
type: string
type: object
service:
description: Service configuration.
properties:
clusterIP:
description: ClusterIP sets the static IP of the service used by the nameserver.
type: string
type: object
type: object
required:
- nameserver
@ -462,7 +469,7 @@ spec:
ip:
description: |-
IP is the ClusterIP of the Service fronting the deployed ts.net nameserver.
Currently you must manually update your cluster DNS config to add
Currently, you must manually update your cluster DNS config to add
this address as a stub nameserver for ts.net for cluster workloads to be
able to resolve MagicDNS names associated with egress or Ingress
proxies.

9
cmd/k8s-operator/nameserver.go
Unescape Escape View File

@ -7,14 +7,13 @@ package main
import (
"context"
_ "embed"
"errors"
"fmt"
"slices"
"strings"
"sync"
_ "embed"
"go.uber.org/zap"
xslices "golang.org/x/exp/slices"
appsv1 "k8s.io/api/apps/v1"
@ -183,6 +182,10 @@ func (a *NameserverReconciler) maybeProvision(ctx context.Context, tsDNSCfg *tsa
if tsDNSCfg.Spec.Nameserver.Image != nil && tsDNSCfg.Spec.Nameserver.Image.Tag != "" {
dCfg.imageTag = tsDNSCfg.Spec.Nameserver.Image.Tag
}
if tsDNSCfg.Spec.Nameserver.Service != nil {
dCfg.clusterIP = tsDNSCfg.Spec.Nameserver.Service.ClusterIP
}
for _, deployable := range []deployable{saDeployable, deployDeployable, svcDeployable, cmDeployable} {
if err := deployable.updateObj(ctx, dCfg, a.Client); err != nil {
return fmt.Errorf("error reconciling %s: %w", deployable.kind, err)
@ -213,6 +216,7 @@ type deployConfig struct {
labels map[string]string
ownerRefs []metav1.OwnerReference
namespace string
clusterIP string
}
var (
@ -267,6 +271,7 @@ var (
svc.ObjectMeta.Labels = cfg.labels
svc.ObjectMeta.OwnerReferences = cfg.ownerRefs
svc.ObjectMeta.Namespace = cfg.namespace
svc.Spec.ClusterIP = cfg.clusterIP
_, err := createOrUpdate[corev1.Service](ctx, kubeClient, cfg.namespace, svc, func(*corev1.Service) {})
return err
},

177
cmd/k8s-operator/nameserver_test.go
Unescape Escape View File

@ -26,7 +26,7 @@ import (
)
func TestNameserverReconciler(t *testing.T) {
dnsCfg := &tsapi.DNSConfig{
dnsConfig := &tsapi.DNSConfig{
TypeMeta: metav1.TypeMeta{Kind: "DNSConfig", APIVersion: "tailscale.com/v1alpha1"},
ObjectMeta: metav1.ObjectMeta{
Name: "test",
@ -37,91 +37,130 @@ func TestNameserverReconciler(t *testing.T) {
Repo: "test",
Tag: "v0.0.1",
},
Service: &tsapi.NameserverService{
ClusterIP: "5.4.3.2",
},
},
},
}
fc := fake.NewClientBuilder().
WithScheme(tsapi.GlobalScheme).
WithObjects(dnsCfg).
WithStatusSubresource(dnsCfg).
WithObjects(dnsConfig).
WithStatusSubresource(dnsConfig).
Build()
zl, err := zap.NewDevelopment()
logger, err := zap.NewDevelopment()
if err != nil {
t.Fatal(err)
}
cl := tstest.NewClock(tstest.ClockOpts{})
nr := &NameserverReconciler{
clock := tstest.NewClock(tstest.ClockOpts{})
reconciler := &NameserverReconciler{
Client: fc,
clock: cl,
logger: zl.Sugar(),
tsNamespace: "tailscale",
clock: clock,
logger: logger.Sugar(),
tsNamespace: tsNamespace,
}
expectReconciled(t, nr, "", "test")
// Verify that nameserver Deployment has been created and has the expected fields.
wantsDeploy := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "nameserver", Namespace: "tailscale"}, TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: appsv1.SchemeGroupVersion.Identifier()}}
if err := yaml.Unmarshal(deployYaml, wantsDeploy); err != nil {
t.Fatalf("unmarshalling yaml: %v", err)
}
dnsCfgOwnerRef := metav1.NewControllerRef(dnsCfg, tsapi.SchemeGroupVersion.WithKind("DNSConfig"))
wantsDeploy.OwnerReferences = []metav1.OwnerReference{*dnsCfgOwnerRef}
wantsDeploy.Spec.Template.Spec.Containers[0].Image = "test:v0.0.1"
wantsDeploy.Namespace = "tailscale"
labels := nameserverResourceLabels("test", "tailscale")
wantsDeploy.ObjectMeta.Labels = labels
expectEqual(t, fc, wantsDeploy)
// Verify that DNSConfig advertizes the nameserver's Service IP address,
// has the ready status condition and tailscale finalizer.
mustUpdate(t, fc, "tailscale", "nameserver", func(svc *corev1.Service) {
svc.Spec.ClusterIP = "1.2.3.4"
expectReconciled(t, reconciler, "", "test")
ownerReference := metav1.NewControllerRef(dnsConfig, tsapi.SchemeGroupVersion.WithKind("DNSConfig"))
nameserverLabels := nameserverResourceLabels(dnsConfig.Name, tsNamespace)
wantsDeploy := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "nameserver", Namespace: tsNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: appsv1.SchemeGroupVersion.Identifier()}}
t.Run("deployment has expected fields", func(t *testing.T) {
if err = yaml.Unmarshal(deployYaml, wantsDeploy); err != nil {
t.Fatalf("unmarshalling yaml: %v", err)
}
wantsDeploy.OwnerReferences = []metav1.OwnerReference{*ownerReference}
wantsDeploy.Spec.Template.Spec.Containers[0].Image = "test:v0.0.1"
wantsDeploy.Namespace = tsNamespace
wantsDeploy.ObjectMeta.Labels = nameserverLabels
expectEqual(t, fc, wantsDeploy)
})
expectReconciled(t, nr, "", "test")
dnsCfg.Status.Nameserver = &tsapi.NameserverStatus{
IP: "1.2.3.4",
}
dnsCfg.Finalizers = []string{FinalizerName}
dnsCfg.Status.Conditions = append(dnsCfg.Status.Conditions, metav1.Condition{
Type: string(tsapi.NameserverReady),
Status: metav1.ConditionTrue,
Reason: reasonNameserverCreated,
Message: reasonNameserverCreated,
LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)},
wantsSvc := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "nameserver", Namespace: tsNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: corev1.SchemeGroupVersion.Identifier()}}
t.Run("service has expected fields", func(t *testing.T) {
if err = yaml.Unmarshal(svcYaml, wantsSvc); err != nil {
t.Fatalf("unmarshalling yaml: %v", err)
}
wantsSvc.Spec.ClusterIP = dnsConfig.Spec.Nameserver.Service.ClusterIP
wantsSvc.OwnerReferences = []metav1.OwnerReference{*ownerReference}
wantsSvc.Namespace = tsNamespace
wantsSvc.ObjectMeta.Labels = nameserverLabels
expectEqual(t, fc, wantsSvc)
})
expectEqual(t, fc, dnsCfg)
// // Verify that nameserver image gets updated to match DNSConfig spec.
mustUpdate(t, fc, "", "test", func(dnsCfg *tsapi.DNSConfig) {
dnsCfg.Spec.Nameserver.Image.Tag = "v0.0.2"
t.Run("dns config status is set", func(t *testing.T) {
// Verify that DNSConfig advertizes the nameserver's Service IP address,
// has the ready status condition and tailscale finalizer.
mustUpdate(t, fc, "tailscale", "nameserver", func(svc *corev1.Service) {
svc.Spec.ClusterIP = "1.2.3.4"
})
expectReconciled(t, reconciler, "", "test")
dnsConfig.Finalizers = []string{FinalizerName}
dnsConfig.Status.Nameserver = &tsapi.NameserverStatus{
IP: "1.2.3.4",
}
dnsConfig.Status.Conditions = append(dnsConfig.Status.Conditions, metav1.Condition{
Type: string(tsapi.NameserverReady),
Status: metav1.ConditionTrue,
Reason: reasonNameserverCreated,
Message: reasonNameserverCreated,
LastTransitionTime: metav1.Time{Time: clock.Now().Truncate(time.Second)},
})
expectEqual(t, fc, dnsConfig)
})
expectReconciled(t, nr, "", "test")
wantsDeploy.Spec.Template.Spec.Containers[0].Image = "test:v0.0.2"
expectEqual(t, fc, wantsDeploy)
// Verify that when another actor sets ConfigMap data, it does not get
// overwritten by nameserver reconciler.
dnsRecords := &operatorutils.Records{Version: "v1alpha1", IP4: map[string][]string{"foo.ts.net": {"1.2.3.4"}}}
bs, err := json.Marshal(dnsRecords)
if err != nil {
t.Fatalf("error marshalling ConfigMap contents: %v", err)
}
mustUpdate(t, fc, "tailscale", "dnsrecords", func(cm *corev1.ConfigMap) {
mak.Set(&cm.Data, "records.json", string(bs))
t.Run("nameserver image can be updated", func(t *testing.T) {
// Verify that nameserver image gets updated to match DNSConfig spec.
mustUpdate(t, fc, "", "test", func(dnsCfg *tsapi.DNSConfig) {
dnsCfg.Spec.Nameserver.Image.Tag = "v0.0.2"
})
expectReconciled(t, reconciler, "", "test")
wantsDeploy.Spec.Template.Spec.Containers[0].Image = "test:v0.0.2"
expectEqual(t, fc, wantsDeploy)
})
t.Run("reconciler does not overwrite custom configuration", func(t *testing.T) {
// Verify that when another actor sets ConfigMap data, it does not get
// overwritten by nameserver reconciler.
dnsRecords := &operatorutils.Records{Version: "v1alpha1", IP4: map[string][]string{"foo.ts.net": {"1.2.3.4"}}}
bs, err := json.Marshal(dnsRecords)
if err != nil {
t.Fatalf("error marshalling ConfigMap contents: %v", err)
}
mustUpdate(t, fc, "tailscale", "dnsrecords", func(cm *corev1.ConfigMap) {
mak.Set(&cm.Data, "records.json", string(bs))
})
expectReconciled(t, reconciler, "", "test")
wantCm := &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "dnsrecords",
Namespace: "tailscale",
Labels: nameserverLabels,
OwnerReferences: []metav1.OwnerReference{*ownerReference},
},
TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"},
Data: map[string]string{"records.json": string(bs)},
}
expectEqual(t, fc, wantCm)
})
expectReconciled(t, nr, "", "test")
wantCm := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "dnsrecords",
Namespace: "tailscale", Labels: labels, OwnerReferences: []metav1.OwnerReference{*dnsCfgOwnerRef}},
TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"},
Data: map[string]string{"records.json": string(bs)},
}
expectEqual(t, fc, wantCm)
// Verify that if dnsconfig.spec.nameserver.image.{repo,tag} are unset,
// the nameserver image defaults to tailscale/k8s-nameserver:unstable.
mustUpdate(t, fc, "", "test", func(dnsCfg *tsapi.DNSConfig) {
dnsCfg.Spec.Nameserver.Image = nil
t.Run("uses default nameserver image", func(t *testing.T) {
// Verify that if dnsconfig.spec.nameserver.image.{repo,tag} are unset,
// the nameserver image defaults to tailscale/k8s-nameserver:unstable.
mustUpdate(t, fc, "", "test", func(dnsCfg *tsapi.DNSConfig) {
dnsCfg.Spec.Nameserver.Image = nil
})
expectReconciled(t, reconciler, "", "test")
wantsDeploy.Spec.Template.Spec.Containers[0].Image = "tailscale/k8s-nameserver:unstable"
expectEqual(t, fc, wantsDeploy)
})
expectReconciled(t, nr, "", "test")
wantsDeploy.Spec.Template.Spec.Containers[0].Image = "tailscale/k8s-nameserver:unstable"
expectEqual(t, fc, wantsDeploy)
}

19
k8s-operator/api.md
Unescape Escape View File

@ -422,6 +422,7 @@ _Appears in:_
| Field | Description | Default | Validation |
| --- | --- | --- | --- |
| `image` _[NameserverImage](#nameserverimage)_ | Nameserver image. Defaults to tailscale/k8s-nameserver:unstable. | | |
| `service` _[NameserverService](#nameserverservice)_ | Service configuration. | | |
#### NameserverImage
@ -441,6 +442,22 @@ _Appears in:_
| `tag` _string_ | Tag defaults to unstable. | | |
#### NameserverService
_Appears in:_
- [Nameserver](#nameserver)
| Field | Description | Default | Validation |
| --- | --- | --- | --- |
| `clusterIP` _string_ | ClusterIP sets the static IP of the service used by the nameserver. | | |
#### NameserverStatus
@ -454,7 +471,7 @@ _Appears in:_
| Field | Description | Default | Validation |
| --- | --- | --- | --- |
| `ip` _string_ | IP is the ClusterIP of the Service fronting the deployed ts.net nameserver.<br />Currently you must manually update your cluster DNS config to add<br />this address as a stub nameserver for ts.net for cluster workloads to be<br />able to resolve MagicDNS names associated with egress or Ingress<br />proxies.<br />The IP address will change if you delete and recreate the DNSConfig. | | |
| `ip` _string_ | IP is the ClusterIP of the Service fronting the deployed ts.net nameserver.<br />Currently, you must manually update your cluster DNS config to add<br />this address as a stub nameserver for ts.net for cluster workloads to be<br />able to resolve MagicDNS names associated with egress or Ingress<br />proxies.<br />The IP address will change if you delete and recreate the DNSConfig. | | |
#### NodePortConfig

11
k8s-operator/apis/v1alpha1/types_tsdnsconfig.go
Unescape Escape View File

@ -82,6 +82,9 @@ type Nameserver struct {
// Nameserver image. Defaults to tailscale/k8s-nameserver:unstable.
// +optional
Image *NameserverImage `json:"image,omitempty"`
// Service configuration.
// +optional
Service *NameserverService `json:"service,omitempty"`
}
type NameserverImage struct {
@ -93,6 +96,12 @@ type NameserverImage struct {
Tag string `json:"tag,omitempty"`
}
type NameserverService struct {
// ClusterIP sets the static IP of the service used by the nameserver.
// +optional
ClusterIP string `json:"clusterIP,omitempty"`
}
type DNSConfigStatus struct {
// +listType=map
// +listMapKey=type
@ -105,7 +114,7 @@ type DNSConfigStatus struct {
type NameserverStatus struct {
// IP is the ClusterIP of the Service fronting the deployed ts.net nameserver.
// Currently you must manually update your cluster DNS config to add
// Currently, you must manually update your cluster DNS config to add
// this address as a stub nameserver for ts.net for cluster workloads to be
// able to resolve MagicDNS names associated with egress or Ingress
// proxies.

20
k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go
Unescape Escape View File

@ -385,6 +385,11 @@ func (in *Nameserver) DeepCopyInto(out *Nameserver) {
*out = new(NameserverImage)
**out = **in
}
if in.Service != nil {
in, out := &in.Service, &out.Service
*out = new(NameserverService)
**out = **in
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Nameserver.
@ -412,6 +417,21 @@ func (in *NameserverImage) DeepCopy() *NameserverImage {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *NameserverService) DeepCopyInto(out *NameserverService) {
*out = *in
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NameserverService.
func (in *NameserverService) DeepCopy() *NameserverService {
if in == nil {
return nil
}
out := new(NameserverService)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *NameserverStatus) DeepCopyInto(out *NameserverStatus) {
*out = *in

Write Preview
Loading…
Cancel
Save