mirror of https://github.com/tailscale/tailscale/
cmd/{k8s-operator,k8s-nameserver},k8s-operator: update nameserver config with records for ingress/egress proxies (#11019)
cmd/k8s-operator: optionally update dnsrecords Configmap with DNS records for proxies. This commit adds functionality to automatically populate DNS records for the in-cluster ts.net nameserver to allow cluster workloads to resolve MagicDNS names associated with operator's proxies. The records are created as follows: * For tailscale Ingress proxies there will be a record mapping the MagicDNS name of the Ingress device and each proxy Pod's IP address. * For cluster egress proxies, configured via tailscale.com/tailnet-fqdn annotation, there will be a record for each proxy Pod, mapping the MagicDNS name of the exposed tailnet workload to the proxy Pod's IP. No records will be created for any other proxy types. Records will only be created if users have configured the operator to deploy an in-cluster ts.net nameserver by applying tailscale.com/v1alpha1.DNSConfig. It is user's responsibility to add the ts.net nameserver as a stub nameserver for ts.net DNS names. https://kubernetes.io/docs/tasks/administer-cluster/dns-custom-nameservers/#configuration-of-stub-domain-and-upstream-nameserver-using-coredns https://cloud.google.com/kubernetes-engine/docs/how-to/kube-dns#upstream_nameservers See also https://github.com/tailscale/tailscale/pull/11017 Updates tailscale/tailscale#10499 Signed-off-by: Irbe Krumina <irbe@tailscale.com>pull/11979/head
Irbe Krumina
7 months ago
committed by
GitHub
parent
a49ed2e145
commit
19b31ac9a6
@ -0,0 +1,9 @@
|
|||||||
|
apiVersion: tailscale.com/v1alpha1
|
||||||
|
kind: DNSConfig
|
||||||
|
metadata:
|
||||||
|
name: ts-dns
|
||||||
|
spec:
|
||||||
|
nameserver:
|
||||||
|
image:
|
||||||
|
repo: tailscale/k8s-nameserver
|
||||||
|
tag: unstable-v1.65
|
||||||
@ -1,4 +1,4 @@
|
|||||||
apiVersion: v1
|
apiVersion: v1
|
||||||
kind: ConfigMap
|
kind: ConfigMap
|
||||||
metadata:
|
metadata:
|
||||||
name: dnsconfig
|
name: dnsrecords
|
||||||
|
|||||||
@ -0,0 +1,337 @@
|
|||||||
|
// Copyright (c) Tailscale Inc & AUTHORS
|
||||||
|
// SPDX-License-Identifier: BSD-3-Clause
|
||||||
|
|
||||||
|
//go:build !plan9
|
||||||
|
|
||||||
|
// tailscale-operator provides a way to expose services running in a Kubernetes
|
||||||
|
// cluster to your Tailnet and to make Tailscale nodes available to cluster
|
||||||
|
// workloads
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"slices"
|
||||||
|
|
||||||
|
"go.uber.org/zap"
|
||||||
|
corev1 "k8s.io/api/core/v1"
|
||||||
|
discoveryv1 "k8s.io/api/discovery/v1"
|
||||||
|
networkingv1 "k8s.io/api/networking/v1"
|
||||||
|
apiequality "k8s.io/apimachinery/pkg/api/equality"
|
||||||
|
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||||
|
"k8s.io/apimachinery/pkg/types"
|
||||||
|
"k8s.io/utils/net"
|
||||||
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||||
|
"sigs.k8s.io/controller-runtime/pkg/reconcile"
|
||||||
|
operatorutils "tailscale.com/k8s-operator"
|
||||||
|
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
|
||||||
|
"tailscale.com/util/mak"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
dnsRecordsRecocilerFinalizer = "tailscale.com/dns-records-reconciler"
|
||||||
|
annotationTSMagicDNSName = "tailscale.com/magic-dnsname"
|
||||||
|
)
|
||||||
|
|
||||||
|
// dnsRecordsReconciler knows how to update dnsrecords ConfigMap with DNS
|
||||||
|
// records.
|
||||||
|
// The records that it creates are:
|
||||||
|
// - For tailscale Ingress, a mapping of the Ingress's MagicDNSName to the IP address of
|
||||||
|
// the ingress proxy Pod.
|
||||||
|
// - For egress proxies configured via tailscale.com/tailnet-fqdn annotation, a
|
||||||
|
// mapping of the tailnet FQDN to the IP address of the egress proxy Pod.
|
||||||
|
//
|
||||||
|
// Records will only be created if there is exactly one ready
|
||||||
|
// tailscale.com/v1alpha1.DNSConfig instance in the cluster (so that we know
|
||||||
|
// that there is a ts.net nameserver deployed in the cluster).
|
||||||
|
type dnsRecordsReconciler struct {
|
||||||
|
client.Client
|
||||||
|
tsNamespace string // namespace in which we provision tailscale resources
|
||||||
|
logger *zap.SugaredLogger
|
||||||
|
isDefaultLoadBalancer bool // true if operator is the default ingress controller in this cluster
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reconcile takes a reconcile.Request for a headless Service fronting a
|
||||||
|
// tailscale proxy and updates DNS Records in dnsrecords ConfigMap for the
|
||||||
|
// in-cluster ts.net nameserver if required.
|
||||||
|
func (dnsRR *dnsRecordsReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {
|
||||||
|
logger := dnsRR.logger.With("Service", req.NamespacedName)
|
||||||
|
logger.Debugf("starting reconcile")
|
||||||
|
defer logger.Debugf("reconcile finished")
|
||||||
|
|
||||||
|
headlessSvc := new(corev1.Service)
|
||||||
|
err = dnsRR.Client.Get(ctx, req.NamespacedName, headlessSvc)
|
||||||
|
if apierrors.IsNotFound(err) {
|
||||||
|
logger.Debugf("Service not found")
|
||||||
|
return reconcile.Result{}, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return reconcile.Result{}, fmt.Errorf("failed to get Service: %w", err)
|
||||||
|
}
|
||||||
|
if !(isManagedByType(headlessSvc, "svc") || isManagedByType(headlessSvc, "ingress")) {
|
||||||
|
logger.Debugf("Service is not a headless Service for a tailscale ingress or egress proxy; do nothing")
|
||||||
|
return reconcile.Result{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if !headlessSvc.DeletionTimestamp.IsZero() {
|
||||||
|
logger.Debug("Service is being deleted, clean up resources")
|
||||||
|
return reconcile.Result{}, dnsRR.maybeCleanup(ctx, headlessSvc, logger)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check that there is a ts.net nameserver deployed to the cluster by
|
||||||
|
// checking that there is tailscale.com/v1alpha1.DNSConfig resource in a
|
||||||
|
// Ready state.
|
||||||
|
dnsCfgLst := new(tsapi.DNSConfigList)
|
||||||
|
if err = dnsRR.List(ctx, dnsCfgLst); err != nil {
|
||||||
|
return reconcile.Result{}, fmt.Errorf("error listing DNSConfigs: %w", err)
|
||||||
|
}
|
||||||
|
if len(dnsCfgLst.Items) == 0 {
|
||||||
|
logger.Debugf("DNSConfig does not exist, not creating DNS records")
|
||||||
|
return reconcile.Result{}, nil
|
||||||
|
}
|
||||||
|
if len(dnsCfgLst.Items) > 1 {
|
||||||
|
logger.Errorf("Invalid cluster state - more than one DNSConfig found in cluster. Please ensure no more than one exists")
|
||||||
|
return reconcile.Result{}, nil
|
||||||
|
}
|
||||||
|
dnsCfg := dnsCfgLst.Items[0]
|
||||||
|
if !operatorutils.DNSCfgIsReady(&dnsCfg) {
|
||||||
|
logger.Info("DNSConfig is not ready yet, waiting...")
|
||||||
|
return reconcile.Result{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return reconcile.Result{}, dnsRR.maybeProvision(ctx, headlessSvc, logger)
|
||||||
|
}
|
||||||
|
|
||||||
|
// maybeProvision ensures that dnsrecords ConfigMap contains a record for the
|
||||||
|
// proxy associated with the headless Service.
|
||||||
|
// The record is only provisioned if the proxy is for a tailscale Ingress or
|
||||||
|
// egress configured via tailscale.com/tailnet-fqdn annotation.
|
||||||
|
//
|
||||||
|
// For Ingress, the record is a mapping between the MagicDNSName of the Ingress, retrieved from
|
||||||
|
// ingress.status.loadBalancer.ingress.hostname field and the proxy Pod IP addresses
|
||||||
|
// retrieved from the EndpoinSlice associated with this headless Service, i.e
|
||||||
|
// Records{IP4: <MagicDNS name of the Ingress>: <[IPs of the ingress proxy Pods]>}
|
||||||
|
//
|
||||||
|
// For egress, the record is a mapping between tailscale.com/tailnet-fqdn
|
||||||
|
// annotation and the proxy Pod IP addresses, retrieved from the EndpointSlice
|
||||||
|
// associated with this headless Service, i.e
|
||||||
|
// Records{IP4: {<tailscale.com/tailnet-fqdn>: <[IPs of the egress proxy Pods]>}
|
||||||
|
//
|
||||||
|
// If records need to be created for this proxy, maybeProvision will also:
|
||||||
|
// - update the headless Service with a tailscale.com/magic-dnsname annotation
|
||||||
|
// - update the headless Service with a finalizer
|
||||||
|
func (dnsRR *dnsRecordsReconciler) maybeProvision(ctx context.Context, headlessSvc *corev1.Service, logger *zap.SugaredLogger) error {
|
||||||
|
if headlessSvc == nil {
|
||||||
|
logger.Info("[unexpected] maybeProvision called with a nil Service")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
isEgressFQDNSvc, err := dnsRR.isSvcForFQDNEgressProxy(ctx, headlessSvc)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("error checking whether the Service is for an egress proxy: %w", err)
|
||||||
|
}
|
||||||
|
if !(isEgressFQDNSvc || isManagedByType(headlessSvc, "ingress")) {
|
||||||
|
logger.Debug("Service is not fronting a proxy that we create DNS records for; do nothing")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
fqdn, err := dnsRR.fqdnForDNSRecord(ctx, headlessSvc, logger)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("error determining DNS name for record: %w", err)
|
||||||
|
}
|
||||||
|
if fqdn == "" {
|
||||||
|
logger.Debugf("MagicDNS name does not (yet) exist, not provisioning DNS record")
|
||||||
|
return nil // a new reconcile will be triggered once it's added
|
||||||
|
}
|
||||||
|
|
||||||
|
oldHeadlessSvc := headlessSvc.DeepCopy()
|
||||||
|
// Ensure that headless Service is annotated with a finalizer to help
|
||||||
|
// with records cleanup when proxy resources are deleted.
|
||||||
|
if !slices.Contains(headlessSvc.Finalizers, dnsRecordsRecocilerFinalizer) {
|
||||||
|
headlessSvc.Finalizers = append(headlessSvc.Finalizers, dnsRecordsRecocilerFinalizer)
|
||||||
|
}
|
||||||
|
// Ensure that headless Service is annotated with the current MagicDNS
|
||||||
|
// name to help with records cleanup when proxy resources are deleted or
|
||||||
|
// MagicDNS name changes.
|
||||||
|
oldFqdn := headlessSvc.Annotations[annotationTSMagicDNSName]
|
||||||
|
if oldFqdn != "" && oldFqdn != fqdn { // i.e user has changed the value of tailscale.com/tailnet-fqdn annotation
|
||||||
|
logger.Debugf("MagicDNS name has changed, remvoving record for %s", oldFqdn)
|
||||||
|
updateFunc := func(rec *operatorutils.Records) {
|
||||||
|
delete(rec.IP4, oldFqdn)
|
||||||
|
}
|
||||||
|
if err = dnsRR.updateDNSConfig(ctx, updateFunc); err != nil {
|
||||||
|
return fmt.Errorf("error removing record for %s: %w", oldFqdn, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
mak.Set(&headlessSvc.Annotations, annotationTSMagicDNSName, fqdn)
|
||||||
|
if !apiequality.Semantic.DeepEqual(oldHeadlessSvc, headlessSvc) {
|
||||||
|
logger.Infof("provisioning DNS record for MagicDNS name: %s", fqdn) // this will be printed exactly once
|
||||||
|
if err := dnsRR.Update(ctx, headlessSvc); err != nil {
|
||||||
|
return fmt.Errorf("error updating proxy headless Service metadata: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get the Pod IP addresses for the proxy from the EndpointSlice for the
|
||||||
|
// headless Service.
|
||||||
|
labels := map[string]string{discoveryv1.LabelServiceName: headlessSvc.Name} // https://kubernetes.io/docs/concepts/services-networking/endpoint-slices/#ownership
|
||||||
|
eps, err := getSingleObject[discoveryv1.EndpointSlice](ctx, dnsRR.Client, dnsRR.tsNamespace, labels)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("error getting the EndpointSlice for the proxy's headless Service: %w", err)
|
||||||
|
}
|
||||||
|
if eps == nil {
|
||||||
|
logger.Debugf("proxy's headless Service EndpointSlice does not yet exist. We will reconcile again once it's created")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
// An EndpointSlice for a Service can have a list of endpoints that each
|
||||||
|
// can have multiple addresses - these are the IP addresses of any Pods
|
||||||
|
// selected by that Service. Pick all the IPv4 addresses.
|
||||||
|
ips := make([]string, 0)
|
||||||
|
for _, ep := range eps.Endpoints {
|
||||||
|
for _, ip := range ep.Addresses {
|
||||||
|
if !net.IsIPv4String(ip) {
|
||||||
|
logger.Infof("EndpointSlice contains IP address %q that is not IPv4, ignoring. Currently only IPv4 is supported", ip)
|
||||||
|
} else {
|
||||||
|
ips = append(ips, ip)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(ips) == 0 {
|
||||||
|
logger.Debugf("EndpointSlice for the Service contains no IPv4 addresses. We will reconcile again once they are created.")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
updateFunc := func(rec *operatorutils.Records) {
|
||||||
|
mak.Set(&rec.IP4, fqdn, ips)
|
||||||
|
}
|
||||||
|
if err = dnsRR.updateDNSConfig(ctx, updateFunc); err != nil {
|
||||||
|
return fmt.Errorf("error updating DNS records: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// maybeCleanup ensures that the DNS record for the proxy has been removed from
|
||||||
|
// dnsrecords ConfigMap and the tailscale.com/dns-records-reconciler finalizer
|
||||||
|
// has been removed from the Service. If the record is not found in the
|
||||||
|
// ConfigMap, the ConfigMap does not exist, or the Service does not have
|
||||||
|
// tailscale.com/magic-dnsname annotation, just remove the finalizer.
|
||||||
|
func (h *dnsRecordsReconciler) maybeCleanup(ctx context.Context, headlessSvc *corev1.Service, logger *zap.SugaredLogger) error {
|
||||||
|
ix := slices.Index(headlessSvc.Finalizers, dnsRecordsRecocilerFinalizer)
|
||||||
|
if ix == -1 {
|
||||||
|
logger.Debugf("no finalizer, nothing to do")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
cm := &corev1.ConfigMap{}
|
||||||
|
err := h.Client.Get(ctx, types.NamespacedName{Name: operatorutils.DNSRecordsCMName, Namespace: h.tsNamespace}, cm)
|
||||||
|
if apierrors.IsNotFound(err) {
|
||||||
|
logger.Debug("'dsnrecords' ConfigMap not found")
|
||||||
|
return h.removeHeadlessSvcFinalizer(ctx, headlessSvc)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("error retrieving 'dnsrecords' ConfigMap: %w", err)
|
||||||
|
}
|
||||||
|
if cm.Data == nil {
|
||||||
|
logger.Debug("'dnsrecords' ConfigMap contains no records")
|
||||||
|
return h.removeHeadlessSvcFinalizer(ctx, headlessSvc)
|
||||||
|
}
|
||||||
|
_, ok := cm.Data[operatorutils.DNSRecordsCMKey]
|
||||||
|
if !ok {
|
||||||
|
logger.Debug("'dnsrecords' ConfigMap contains no records")
|
||||||
|
return h.removeHeadlessSvcFinalizer(ctx, headlessSvc)
|
||||||
|
}
|
||||||
|
fqdn, _ := headlessSvc.GetAnnotations()[annotationTSMagicDNSName]
|
||||||
|
if fqdn == "" {
|
||||||
|
return h.removeHeadlessSvcFinalizer(ctx, headlessSvc)
|
||||||
|
}
|
||||||
|
logger.Infof("removing DNS record for MagicDNS name %s", fqdn)
|
||||||
|
updateFunc := func(rec *operatorutils.Records) {
|
||||||
|
delete(rec.IP4, fqdn)
|
||||||
|
}
|
||||||
|
if err = h.updateDNSConfig(ctx, updateFunc); err != nil {
|
||||||
|
return fmt.Errorf("error updating DNS config: %w", err)
|
||||||
|
}
|
||||||
|
return h.removeHeadlessSvcFinalizer(ctx, headlessSvc)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (dnsRR *dnsRecordsReconciler) removeHeadlessSvcFinalizer(ctx context.Context, headlessSvc *corev1.Service) error {
|
||||||
|
idx := slices.Index(headlessSvc.Finalizers, dnsRecordsRecocilerFinalizer)
|
||||||
|
if idx == -1 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
headlessSvc.Finalizers = append(headlessSvc.Finalizers[:idx], headlessSvc.Finalizers[idx+1:]...)
|
||||||
|
return dnsRR.Update(ctx, headlessSvc)
|
||||||
|
}
|
||||||
|
|
||||||
|
// fqdnForDNSRecord returns MagicDNS name associated with a given headless Service.
|
||||||
|
// If the headless Service is for a tailscale Ingress proxy, returns ingress.status.loadBalancer.ingress.hostname.
|
||||||
|
// If the headless Service is for an tailscale egress proxy configured via tailscale.com/tailnet-fqdn annotation, returns the annotation value.
|
||||||
|
// This function is not expected to be called with headless Services for other
|
||||||
|
// proxy types, or any other Services, but it just returns an empty string if
|
||||||
|
// that happens.
|
||||||
|
func (dnsRR *dnsRecordsReconciler) fqdnForDNSRecord(ctx context.Context, headlessSvc *corev1.Service, logger *zap.SugaredLogger) (string, error) {
|
||||||
|
parentName := parentFromObjectLabels(headlessSvc)
|
||||||
|
if isManagedByType(headlessSvc, "ingress") {
|
||||||
|
ing := new(networkingv1.Ingress)
|
||||||
|
if err := dnsRR.Get(ctx, parentName, ing); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if len(ing.Status.LoadBalancer.Ingress) == 0 {
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
return ing.Status.LoadBalancer.Ingress[0].Hostname, nil
|
||||||
|
}
|
||||||
|
if isManagedByType(headlessSvc, "svc") {
|
||||||
|
svc := new(corev1.Service)
|
||||||
|
if err := dnsRR.Get(ctx, parentName, svc); apierrors.IsNotFound(err) {
|
||||||
|
logger.Info("[unexpected] parent Service for egress proxy %s not found", headlessSvc.Name)
|
||||||
|
return "", nil
|
||||||
|
} else if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return svc.Annotations[AnnotationTailnetTargetFQDN], nil
|
||||||
|
}
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// updateDNSConfig runs the provided update function against dnsrecords
|
||||||
|
// ConfigMap. At this point the in-cluster ts.net nameserver is expected to be
|
||||||
|
// successfully created together with the ConfigMap.
|
||||||
|
func (dnsRR *dnsRecordsReconciler) updateDNSConfig(ctx context.Context, update func(*operatorutils.Records)) error {
|
||||||
|
cm := &corev1.ConfigMap{}
|
||||||
|
err := dnsRR.Get(ctx, types.NamespacedName{Name: operatorutils.DNSRecordsCMName, Namespace: dnsRR.tsNamespace}, cm)
|
||||||
|
if apierrors.IsNotFound(err) {
|
||||||
|
dnsRR.logger.Info("[unexpected] dnsrecords ConfigMap not found in cluster. Not updating DNS records. Please open an isue and attach operator logs.")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("error retrieving dnsrecords ConfigMap: %w", err)
|
||||||
|
}
|
||||||
|
dnsRecords := operatorutils.Records{Version: operatorutils.Alpha1Version, IP4: map[string][]string{}}
|
||||||
|
if cm.Data != nil && cm.Data[operatorutils.DNSRecordsCMKey] != "" {
|
||||||
|
if err := json.Unmarshal([]byte(cm.Data[operatorutils.DNSRecordsCMKey]), &dnsRecords); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
update(&dnsRecords)
|
||||||
|
dnsRecordsBs, err := json.Marshal(dnsRecords)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("error marshalling DNS records: %w", err)
|
||||||
|
}
|
||||||
|
mak.Set(&cm.Data, operatorutils.DNSRecordsCMKey, string(dnsRecordsBs))
|
||||||
|
return dnsRR.Update(ctx, cm)
|
||||||
|
}
|
||||||
|
|
||||||
|
// isSvcForFQDNEgressProxy returns true if the Service is a headless Service
|
||||||
|
// created for a proxy for a tailscale egress Service configured via
|
||||||
|
// tailscale.com/tailnet-fqdn annotation.
|
||||||
|
func (dnsRR *dnsRecordsReconciler) isSvcForFQDNEgressProxy(ctx context.Context, svc *corev1.Service) (bool, error) {
|
||||||
|
if !isManagedByType(svc, "svc") {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
parentName := parentFromObjectLabels(svc)
|
||||||
|
parentSvc := new(corev1.Service)
|
||||||
|
if err := dnsRR.Get(ctx, parentName, parentSvc); apierrors.IsNotFound(err) {
|
||||||
|
return false, nil
|
||||||
|
} else if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
annots := parentSvc.Annotations
|
||||||
|
return annots != nil && annots[AnnotationTailnetTargetFQDN] != "", nil
|
||||||
|
}
|
||||||
@ -0,0 +1,198 @@
|
|||||||
|
// Copyright (c) Tailscale Inc & AUTHORS
|
||||||
|
// SPDX-License-Identifier: BSD-3-Clause
|
||||||
|
|
||||||
|
//go:build !plan9
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/google/go-cmp/cmp"
|
||||||
|
"go.uber.org/zap"
|
||||||
|
corev1 "k8s.io/api/core/v1"
|
||||||
|
discoveryv1 "k8s.io/api/discovery/v1"
|
||||||
|
networkingv1 "k8s.io/api/networking/v1"
|
||||||
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||||
|
"k8s.io/apimachinery/pkg/types"
|
||||||
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||||
|
"sigs.k8s.io/controller-runtime/pkg/client/fake"
|
||||||
|
operatorutils "tailscale.com/k8s-operator"
|
||||||
|
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
|
||||||
|
"tailscale.com/tstest"
|
||||||
|
"tailscale.com/types/ptr"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestDNSRecordsReconciler(t *testing.T) {
|
||||||
|
// Preconfigure a cluster with a DNSConfig
|
||||||
|
dnsConfig := &tsapi.DNSConfig{
|
||||||
|
ObjectMeta: metav1.ObjectMeta{
|
||||||
|
Name: "test",
|
||||||
|
},
|
||||||
|
TypeMeta: metav1.TypeMeta{Kind: "DNSConfig"},
|
||||||
|
Spec: tsapi.DNSConfigSpec{
|
||||||
|
Nameserver: &tsapi.Nameserver{},
|
||||||
|
}}
|
||||||
|
ing := &networkingv1.Ingress{
|
||||||
|
ObjectMeta: metav1.ObjectMeta{
|
||||||
|
Name: "ts-ingress",
|
||||||
|
Namespace: "test",
|
||||||
|
},
|
||||||
|
Spec: networkingv1.IngressSpec{
|
||||||
|
IngressClassName: ptr.To("tailscale"),
|
||||||
|
},
|
||||||
|
Status: networkingv1.IngressStatus{
|
||||||
|
LoadBalancer: networkingv1.IngressLoadBalancerStatus{
|
||||||
|
Ingress: []networkingv1.IngressLoadBalancerIngress{{
|
||||||
|
Hostname: "cluster.ingress.ts.net"}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
cm := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "dnsrecords", Namespace: "tailscale"}}
|
||||||
|
fc := fake.NewClientBuilder().
|
||||||
|
WithScheme(tsapi.GlobalScheme).
|
||||||
|
WithObjects(cm).
|
||||||
|
WithObjects(dnsConfig).
|
||||||
|
WithObjects(ing).
|
||||||
|
WithStatusSubresource(dnsConfig, ing).
|
||||||
|
Build()
|
||||||
|
zl, err := zap.NewDevelopment()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
cl := tstest.NewClock(tstest.ClockOpts{})
|
||||||
|
// Set the ready condition of the DNSConfig
|
||||||
|
mustUpdateStatus[tsapi.DNSConfig](t, fc, "", "test", func(c *tsapi.DNSConfig) {
|
||||||
|
operatorutils.SetDNSConfigCondition(c, tsapi.NameserverReady, metav1.ConditionTrue, reasonNameserverCreated, reasonNameserverCreated, 0, cl, zl.Sugar())
|
||||||
|
})
|
||||||
|
dnsRR := &dnsRecordsReconciler{
|
||||||
|
Client: fc,
|
||||||
|
logger: zl.Sugar(),
|
||||||
|
tsNamespace: "tailscale",
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1. DNS record is created for an egress proxy configured via
|
||||||
|
// tailscale.com/tailnet-fqdn annotation
|
||||||
|
egressSvcFQDN := &corev1.Service{
|
||||||
|
ObjectMeta: metav1.ObjectMeta{
|
||||||
|
Name: "egress-fqdn",
|
||||||
|
Namespace: "test",
|
||||||
|
Annotations: map[string]string{"tailscale.com/tailnet-fqdn": "foo.bar.ts.net"},
|
||||||
|
},
|
||||||
|
Spec: corev1.ServiceSpec{
|
||||||
|
ExternalName: "unused",
|
||||||
|
Type: corev1.ServiceTypeExternalName,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
headlessForEgressSvcFQDN := headlessSvcForParent(egressSvcFQDN, "svc") // create the proxy headless Service
|
||||||
|
ep := endpointSliceForService(headlessForEgressSvcFQDN, "10.9.8.7")
|
||||||
|
mustCreate(t, fc, egressSvcFQDN)
|
||||||
|
mustCreate(t, fc, headlessForEgressSvcFQDN)
|
||||||
|
mustCreate(t, fc, ep)
|
||||||
|
expectReconciled(t, dnsRR, "tailscale", "egress-fqdn") // dns-records-reconciler reconcile the headless Service
|
||||||
|
// ConfigMap should now have a record for foo.bar.ts.net -> 10.8.8.7
|
||||||
|
wantHosts := map[string][]string{"foo.bar.ts.net": {"10.9.8.7"}}
|
||||||
|
expectHostsRecords(t, fc, wantHosts)
|
||||||
|
|
||||||
|
// 2. DNS record is updated if tailscale.com/tailnet-fqdn annotation's
|
||||||
|
// value changes
|
||||||
|
mustUpdate(t, fc, "test", "egress-fqdn", func(svc *corev1.Service) {
|
||||||
|
svc.Annotations["tailscale.com/tailnet-fqdn"] = "baz.bar.ts.net"
|
||||||
|
})
|
||||||
|
expectReconciled(t, dnsRR, "tailscale", "egress-fqdn") // dns-records-reconciler reconcile the headless Service
|
||||||
|
wantHosts = map[string][]string{"baz.bar.ts.net": {"10.9.8.7"}}
|
||||||
|
expectHostsRecords(t, fc, wantHosts)
|
||||||
|
|
||||||
|
// 3. DNS record is updated if the IP address of the proxy Pod changes.
|
||||||
|
ep = endpointSliceForService(headlessForEgressSvcFQDN, "10.6.5.4")
|
||||||
|
mustUpdate(t, fc, ep.Namespace, ep.Name, func(ep *discoveryv1.EndpointSlice) {
|
||||||
|
ep.Endpoints[0].Addresses = []string{"10.6.5.4"}
|
||||||
|
})
|
||||||
|
expectReconciled(t, dnsRR, "tailscale", "egress-fqdn") // dns-records-reconciler reconcile the headless Service
|
||||||
|
wantHosts = map[string][]string{"baz.bar.ts.net": {"10.6.5.4"}}
|
||||||
|
expectHostsRecords(t, fc, wantHosts)
|
||||||
|
|
||||||
|
// 4. DNS record is created for an ingress proxy configured via Ingress
|
||||||
|
headlessForIngress := headlessSvcForParent(ing, "ingress")
|
||||||
|
ep = endpointSliceForService(headlessForIngress, "10.9.8.7")
|
||||||
|
mustCreate(t, fc, headlessForIngress)
|
||||||
|
mustCreate(t, fc, ep)
|
||||||
|
expectReconciled(t, dnsRR, "tailscale", "ts-ingress") // dns-records-reconciler should reconcile the headless Service
|
||||||
|
wantHosts["cluster.ingress.ts.net"] = []string{"10.9.8.7"}
|
||||||
|
expectHostsRecords(t, fc, wantHosts)
|
||||||
|
|
||||||
|
// 5. DNS records are updated if Ingress's MagicDNS name changes (i.e users changed spec.tls.hosts[0])
|
||||||
|
t.Log("test case 5")
|
||||||
|
mustUpdateStatus(t, fc, "test", "ts-ingress", func(ing *networkingv1.Ingress) {
|
||||||
|
ing.Status.LoadBalancer.Ingress[0].Hostname = "another.ingress.ts.net"
|
||||||
|
})
|
||||||
|
expectReconciled(t, dnsRR, "tailscale", "ts-ingress") // dns-records-reconciler should reconcile the headless Service
|
||||||
|
delete(wantHosts, "cluster.ingress.ts.net")
|
||||||
|
wantHosts["another.ingress.ts.net"] = []string{"10.9.8.7"}
|
||||||
|
expectHostsRecords(t, fc, wantHosts)
|
||||||
|
|
||||||
|
// 6. DNS records are updated if Ingress proxy's Pod IP changes
|
||||||
|
mustUpdate(t, fc, ep.Namespace, ep.Name, func(ep *discoveryv1.EndpointSlice) {
|
||||||
|
ep.Endpoints[0].Addresses = []string{"7.8.9.10"}
|
||||||
|
})
|
||||||
|
expectReconciled(t, dnsRR, "tailscale", "ts-ingress")
|
||||||
|
wantHosts["another.ingress.ts.net"] = []string{"7.8.9.10"}
|
||||||
|
expectHostsRecords(t, fc, wantHosts)
|
||||||
|
}
|
||||||
|
|
||||||
|
func headlessSvcForParent(o client.Object, typ string) *corev1.Service {
|
||||||
|
return &corev1.Service{
|
||||||
|
ObjectMeta: metav1.ObjectMeta{
|
||||||
|
Name: o.GetName(),
|
||||||
|
Namespace: "tailscale",
|
||||||
|
Labels: map[string]string{
|
||||||
|
LabelManaged: "true",
|
||||||
|
LabelParentName: o.GetName(),
|
||||||
|
LabelParentNamespace: o.GetNamespace(),
|
||||||
|
LabelParentType: typ,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
Spec: corev1.ServiceSpec{
|
||||||
|
ClusterIP: "None",
|
||||||
|
Type: corev1.ServiceTypeClusterIP,
|
||||||
|
Selector: map[string]string{"foo": "bar"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func endpointSliceForService(svc *corev1.Service, ip string) *discoveryv1.EndpointSlice {
|
||||||
|
return &discoveryv1.EndpointSlice{
|
||||||
|
ObjectMeta: metav1.ObjectMeta{
|
||||||
|
Name: svc.Name,
|
||||||
|
Namespace: svc.Namespace,
|
||||||
|
Labels: map[string]string{discoveryv1.LabelServiceName: svc.Name},
|
||||||
|
},
|
||||||
|
Endpoints: []discoveryv1.Endpoint{{
|
||||||
|
Addresses: []string{ip},
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func expectHostsRecords(t *testing.T, cl client.Client, wantsHosts map[string][]string) {
|
||||||
|
t.Helper()
|
||||||
|
cm := new(corev1.ConfigMap)
|
||||||
|
if err := cl.Get(context.Background(), types.NamespacedName{Name: "dnsrecords", Namespace: "tailscale"}, cm); err != nil {
|
||||||
|
t.Fatalf("getting dnsconfig ConfigMap: %v", err)
|
||||||
|
}
|
||||||
|
if cm.Data == nil {
|
||||||
|
t.Fatal("dnsconfig ConfigMap has no data")
|
||||||
|
}
|
||||||
|
dnsConfigString, ok := cm.Data[operatorutils.DNSRecordsCMKey]
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("dnsconfig ConfigMap does not contain dnsconfig")
|
||||||
|
}
|
||||||
|
dnsConfig := &operatorutils.Records{}
|
||||||
|
if err := json.Unmarshal([]byte(dnsConfigString), dnsConfig); err != nil {
|
||||||
|
t.Fatalf("unmarshaling dnsconfig: %v", err)
|
||||||
|
}
|
||||||
|
if diff := cmp.Diff(dnsConfig.IP4, wantsHosts); diff != "" {
|
||||||
|
t.Fatalf("unexpected dns config (-got +want):\n%s", diff)
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue