mirror of https://github.com/tailscale/tailscale/
cmd/k8s-operator,k8s-operator/apis: set a readiness condition on egress Services for ProxyGroup (#13746)
cmd/k8s-operator,k8s-operator/apis: set a readiness condition on egress Services Set a readiness condition on ExternalName Services that define a tailnet target to route cluster traffic to via a ProxyGroup's proxies. The condition is set to true if at least one proxy is currently set up to route. Updates tailscale/tailscale#13406 Signed-off-by: Irbe Krumina <irbe@tailscale.com>pull/13755/head
Irbe Krumina
2 months ago
committed by
GitHub
parent
94c79659fa
commit
89ee6bbdae
@ -0,0 +1,179 @@
|
||||
// Copyright (c) Tailscale Inc & AUTHORS
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
//go:build !plan9
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"go.uber.org/zap"
|
||||
appsv1 "k8s.io/api/apps/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
discoveryv1 "k8s.io/api/discovery/v1"
|
||||
apiequality "k8s.io/apimachinery/pkg/api/equality"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/reconcile"
|
||||
tsoperator "tailscale.com/k8s-operator"
|
||||
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
|
||||
"tailscale.com/tstime"
|
||||
)
|
||||
|
||||
const (
|
||||
reasonReadinessCheckFailed = "ReadinessCheckFailed"
|
||||
reasonClusterResourcesNotReady = "ClusterResourcesNotReady"
|
||||
reasonNoProxies = "NoProxiesConfigured"
|
||||
reasonNotReady = "NotReadyToRouteTraffic"
|
||||
reasonReady = "ReadyToRouteTraffic"
|
||||
reasonPartiallyReady = "PartiallyReadyToRouteTraffic"
|
||||
msgReadyToRouteTemplate = "%d out of %d replicas are ready to route traffic"
|
||||
)
|
||||
|
||||
type egressSvcsReadinessReconciler struct {
|
||||
client.Client
|
||||
logger *zap.SugaredLogger
|
||||
clock tstime.Clock
|
||||
tsNamespace string
|
||||
}
|
||||
|
||||
// Reconcile reconciles an ExternalName Service that defines a tailnet target to be exposed on a ProxyGroup and sets the
|
||||
// EgressSvcReady condition on it. The condition gets set to true if at least one of the proxies is currently ready to
|
||||
// route traffic to the target. It compares proxy Pod IPs with the endpoints set on the EndpointSlice for the egress
|
||||
// service to determine how many replicas are currently able to route traffic.
|
||||
func (esrr *egressSvcsReadinessReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {
|
||||
l := esrr.logger.With("Service", req.NamespacedName)
|
||||
defer l.Info("reconcile finished")
|
||||
|
||||
svc := new(corev1.Service)
|
||||
if err = esrr.Get(ctx, req.NamespacedName, svc); apierrors.IsNotFound(err) {
|
||||
l.Info("Service not found")
|
||||
return res, nil
|
||||
} else if err != nil {
|
||||
return res, fmt.Errorf("failed to get Service: %w", err)
|
||||
}
|
||||
var (
|
||||
reason, msg string
|
||||
st metav1.ConditionStatus = metav1.ConditionUnknown
|
||||
)
|
||||
oldStatus := svc.Status.DeepCopy()
|
||||
defer func() {
|
||||
tsoperator.SetServiceCondition(svc, tsapi.EgressSvcReady, st, reason, msg, esrr.clock, l)
|
||||
if !apiequality.Semantic.DeepEqual(oldStatus, svc.Status) {
|
||||
err = errors.Join(err, esrr.Status().Update(ctx, svc))
|
||||
}
|
||||
}()
|
||||
|
||||
crl := egressSvcChildResourceLabels(svc)
|
||||
eps, err := getSingleObject[discoveryv1.EndpointSlice](ctx, esrr.Client, esrr.tsNamespace, crl)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("error getting EndpointSlice: %w", err)
|
||||
reason = reasonReadinessCheckFailed
|
||||
msg = err.Error()
|
||||
return res, err
|
||||
}
|
||||
if eps == nil {
|
||||
l.Infof("EndpointSlice for Service does not yet exist, waiting...")
|
||||
reason, msg = reasonClusterResourcesNotReady, reasonClusterResourcesNotReady
|
||||
st = metav1.ConditionFalse
|
||||
return res, nil
|
||||
}
|
||||
pg := &tsapi.ProxyGroup{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: svc.Annotations[AnnotationProxyGroup],
|
||||
},
|
||||
}
|
||||
err = esrr.Get(ctx, client.ObjectKeyFromObject(pg), pg)
|
||||
if apierrors.IsNotFound(err) {
|
||||
l.Infof("ProxyGroup for Service does not exist, waiting...")
|
||||
reason, msg = reasonClusterResourcesNotReady, reasonClusterResourcesNotReady
|
||||
st = metav1.ConditionFalse
|
||||
return res, nil
|
||||
}
|
||||
if err != nil {
|
||||
err = fmt.Errorf("error retrieving ProxyGroup: %w", err)
|
||||
reason = reasonReadinessCheckFailed
|
||||
msg = err.Error()
|
||||
return res, err
|
||||
}
|
||||
if !tsoperator.ProxyGroupIsReady(pg) {
|
||||
l.Infof("ProxyGroup for Service is not ready, waiting...")
|
||||
reason, msg = reasonClusterResourcesNotReady, reasonClusterResourcesNotReady
|
||||
st = metav1.ConditionFalse
|
||||
return res, nil
|
||||
}
|
||||
|
||||
replicas := pgReplicas(pg)
|
||||
if replicas == 0 {
|
||||
l.Infof("ProxyGroup replicas set to 0")
|
||||
reason, msg = reasonNoProxies, reasonNoProxies
|
||||
st = metav1.ConditionFalse
|
||||
return res, nil
|
||||
}
|
||||
podLabels := pgLabels(pg.Name, nil)
|
||||
var readyReplicas int32
|
||||
for i := range replicas {
|
||||
podLabels[appsv1.PodIndexLabel] = fmt.Sprintf("%d", i)
|
||||
pod, err := getSingleObject[corev1.Pod](ctx, esrr.Client, esrr.tsNamespace, podLabels)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("error retrieving ProxyGroup Pod: %w", err)
|
||||
reason = reasonReadinessCheckFailed
|
||||
msg = err.Error()
|
||||
return res, err
|
||||
}
|
||||
if pod == nil {
|
||||
l.Infof("[unexpected] ProxyGroup is ready, but replica %d was not found", i)
|
||||
reason, msg = reasonClusterResourcesNotReady, reasonClusterResourcesNotReady
|
||||
return res, nil
|
||||
}
|
||||
l.Infof("looking at Pod with IPs %v", pod.Status.PodIPs)
|
||||
ready := false
|
||||
for _, ep := range eps.Endpoints {
|
||||
l.Infof("looking at endpoint with addresses %v", ep.Addresses)
|
||||
if endpointReadyForPod(&ep, pod, l) {
|
||||
l.Infof("endpoint is ready for Pod")
|
||||
ready = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if ready {
|
||||
readyReplicas++
|
||||
}
|
||||
}
|
||||
msg = fmt.Sprintf(msgReadyToRouteTemplate, readyReplicas, replicas)
|
||||
if readyReplicas == 0 {
|
||||
reason = reasonNotReady
|
||||
st = metav1.ConditionFalse
|
||||
return res, nil
|
||||
}
|
||||
st = metav1.ConditionTrue
|
||||
if readyReplicas < replicas {
|
||||
reason = reasonPartiallyReady
|
||||
} else {
|
||||
reason = reasonReady
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// endpointReadyForPod returns true if the endpoint is for the Pod's IPv4 address and is ready to serve traffic.
|
||||
// Endpoint must not be nil.
|
||||
func endpointReadyForPod(ep *discoveryv1.Endpoint, pod *corev1.Pod, l *zap.SugaredLogger) bool {
|
||||
podIP, err := podIPv4(pod)
|
||||
if err != nil {
|
||||
l.Infof("[unexpected] error retrieving Pod's IPv4 address: %v", err)
|
||||
return false
|
||||
}
|
||||
// Currently we only ever set a single address on and Endpoint and nothing else is meant to modify this.
|
||||
if len(ep.Addresses) != 1 {
|
||||
return false
|
||||
}
|
||||
return strings.EqualFold(ep.Addresses[0], podIP) &&
|
||||
*ep.Conditions.Ready &&
|
||||
*ep.Conditions.Serving &&
|
||||
!*ep.Conditions.Terminating
|
||||
}
|
||||
@ -0,0 +1,169 @@
|
||||
// Copyright (c) Tailscale Inc & AUTHORS
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
//go:build !plan9
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/AlekSi/pointer"
|
||||
"go.uber.org/zap"
|
||||
appsv1 "k8s.io/api/apps/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
discoveryv1 "k8s.io/api/discovery/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client/fake"
|
||||
tsoperator "tailscale.com/k8s-operator"
|
||||
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
|
||||
"tailscale.com/tstest"
|
||||
"tailscale.com/tstime"
|
||||
)
|
||||
|
||||
func TestEgressServiceReadiness(t *testing.T) {
|
||||
// We need to pass a ProxyGroup object to WithStatusSubresource because of some quirks in how the fake client
|
||||
// works. Without this code further down would not be able to update ProxyGroup status.
|
||||
fc := fake.NewClientBuilder().
|
||||
WithScheme(tsapi.GlobalScheme).
|
||||
WithStatusSubresource(&tsapi.ProxyGroup{}).
|
||||
Build()
|
||||
zl, _ := zap.NewDevelopment()
|
||||
cl := tstest.NewClock(tstest.ClockOpts{})
|
||||
rec := &egressSvcsReadinessReconciler{
|
||||
tsNamespace: "operator-ns",
|
||||
Client: fc,
|
||||
logger: zl.Sugar(),
|
||||
clock: cl,
|
||||
}
|
||||
tailnetFQDN := "my-app.tailnetxyz.ts.net"
|
||||
egressSvc := &corev1.Service{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "my-app",
|
||||
Namespace: "dev",
|
||||
Annotations: map[string]string{
|
||||
AnnotationProxyGroup: "dev",
|
||||
AnnotationTailnetTargetFQDN: tailnetFQDN,
|
||||
},
|
||||
},
|
||||
}
|
||||
fakeClusterIPSvc := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "my-app", Namespace: "operator-ns"}}
|
||||
l := egressSvcEpsLabels(egressSvc, fakeClusterIPSvc)
|
||||
eps := &discoveryv1.EndpointSlice{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "my-app",
|
||||
Namespace: "operator-ns",
|
||||
Labels: l,
|
||||
},
|
||||
AddressType: discoveryv1.AddressTypeIPv4,
|
||||
}
|
||||
pg := &tsapi.ProxyGroup{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "dev",
|
||||
},
|
||||
}
|
||||
mustCreate(t, fc, egressSvc)
|
||||
setClusterNotReady(egressSvc, cl, zl.Sugar())
|
||||
t.Run("endpointslice_does_not_exist", func(t *testing.T) {
|
||||
expectReconciled(t, rec, "dev", "my-app")
|
||||
expectEqual(t, fc, egressSvc, nil) // not ready
|
||||
})
|
||||
t.Run("proxy_group_does_not_exist", func(t *testing.T) {
|
||||
mustCreate(t, fc, eps)
|
||||
expectReconciled(t, rec, "dev", "my-app")
|
||||
expectEqual(t, fc, egressSvc, nil) // still not ready
|
||||
})
|
||||
t.Run("proxy_group_not_ready", func(t *testing.T) {
|
||||
mustCreate(t, fc, pg)
|
||||
expectReconciled(t, rec, "dev", "my-app")
|
||||
expectEqual(t, fc, egressSvc, nil) // still not ready
|
||||
})
|
||||
t.Run("no_ready_replicas", func(t *testing.T) {
|
||||
setPGReady(pg, cl, zl.Sugar())
|
||||
mustUpdateStatus(t, fc, pg.Namespace, pg.Name, func(p *tsapi.ProxyGroup) {
|
||||
p.Status = pg.Status
|
||||
})
|
||||
expectEqual(t, fc, pg, nil)
|
||||
for i := range pgReplicas(pg) {
|
||||
p := pod(pg, i)
|
||||
mustCreate(t, fc, p)
|
||||
mustUpdateStatus(t, fc, p.Namespace, p.Name, func(existing *corev1.Pod) {
|
||||
existing.Status.PodIPs = p.Status.PodIPs
|
||||
})
|
||||
}
|
||||
expectReconciled(t, rec, "dev", "my-app")
|
||||
setNotReady(egressSvc, cl, zl.Sugar(), pgReplicas(pg))
|
||||
expectEqual(t, fc, egressSvc, nil) // still not ready
|
||||
})
|
||||
t.Run("one_ready_replica", func(t *testing.T) {
|
||||
setEndpointForReplica(pg, 0, eps)
|
||||
mustUpdate(t, fc, eps.Namespace, eps.Name, func(e *discoveryv1.EndpointSlice) {
|
||||
e.Endpoints = eps.Endpoints
|
||||
})
|
||||
setReady(egressSvc, cl, zl.Sugar(), pgReplicas(pg), 1)
|
||||
expectReconciled(t, rec, "dev", "my-app")
|
||||
expectEqual(t, fc, egressSvc, nil) // partially ready
|
||||
})
|
||||
t.Run("all_replicas_ready", func(t *testing.T) {
|
||||
for i := range pgReplicas(pg) {
|
||||
setEndpointForReplica(pg, i, eps)
|
||||
}
|
||||
mustUpdate(t, fc, eps.Namespace, eps.Name, func(e *discoveryv1.EndpointSlice) {
|
||||
e.Endpoints = eps.Endpoints
|
||||
})
|
||||
setReady(egressSvc, cl, zl.Sugar(), pgReplicas(pg), pgReplicas(pg))
|
||||
expectReconciled(t, rec, "dev", "my-app")
|
||||
expectEqual(t, fc, egressSvc, nil) // ready
|
||||
})
|
||||
}
|
||||
|
||||
func setClusterNotReady(svc *corev1.Service, cl tstime.Clock, l *zap.SugaredLogger) {
|
||||
tsoperator.SetServiceCondition(svc, tsapi.EgressSvcReady, metav1.ConditionFalse, reasonClusterResourcesNotReady, reasonClusterResourcesNotReady, cl, l)
|
||||
}
|
||||
|
||||
func setNotReady(svc *corev1.Service, cl tstime.Clock, l *zap.SugaredLogger, replicas int32) {
|
||||
msg := fmt.Sprintf(msgReadyToRouteTemplate, 0, replicas)
|
||||
tsoperator.SetServiceCondition(svc, tsapi.EgressSvcReady, metav1.ConditionFalse, reasonNotReady, msg, cl, l)
|
||||
}
|
||||
|
||||
func setReady(svc *corev1.Service, cl tstime.Clock, l *zap.SugaredLogger, replicas, readyReplicas int32) {
|
||||
reason := reasonPartiallyReady
|
||||
if readyReplicas == replicas {
|
||||
reason = reasonReady
|
||||
}
|
||||
msg := fmt.Sprintf(msgReadyToRouteTemplate, readyReplicas, replicas)
|
||||
tsoperator.SetServiceCondition(svc, tsapi.EgressSvcReady, metav1.ConditionTrue, reason, msg, cl, l)
|
||||
}
|
||||
|
||||
func setPGReady(pg *tsapi.ProxyGroup, cl tstime.Clock, l *zap.SugaredLogger) {
|
||||
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, "foo", "foo", pg.Generation, cl, l)
|
||||
}
|
||||
|
||||
func setEndpointForReplica(pg *tsapi.ProxyGroup, ordinal int32, eps *discoveryv1.EndpointSlice) {
|
||||
p := pod(pg, ordinal)
|
||||
eps.Endpoints = append(eps.Endpoints, discoveryv1.Endpoint{
|
||||
Addresses: []string{p.Status.PodIPs[0].IP},
|
||||
Conditions: discoveryv1.EndpointConditions{
|
||||
Ready: pointer.ToBool(true),
|
||||
Serving: pointer.ToBool(true),
|
||||
Terminating: pointer.ToBool(false),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func pod(pg *tsapi.ProxyGroup, ordinal int32) *corev1.Pod {
|
||||
l := pgLabels(pg.Name, nil)
|
||||
l[appsv1.PodIndexLabel] = fmt.Sprintf("%d", ordinal)
|
||||
ip := fmt.Sprintf("10.0.0.%d", ordinal)
|
||||
return &corev1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: fmt.Sprintf("%s-%d", pg.Name, ordinal),
|
||||
Namespace: "operator-ns",
|
||||
Labels: l,
|
||||
},
|
||||
Status: corev1.PodStatus{
|
||||
PodIPs: []corev1.PodIP{{IP: ip}},
|
||||
},
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue