@ -10,7 +10,6 @@ import (
"fmt"
"net/netip"
"slices"
"strings"
"sync"
"time"
@ -33,17 +32,16 @@ import (
)
const (
reasonSubnetRouterCreationFailed = "SubnetRouterCreationFailed"
reasonSubnetRouterCreated = "SubnetRouterCreated"
reasonSubnetRouterCleanupFailed = "SubnetRouterCleanupFailed"
reasonSubnetRouterCleanupInProgress = "SubnetRouterCleanupInProgress"
reasonSubnetRouterInvalid = "SubnetRouterInvalid"
messageSubnetRouterCreationFailed = "Failed creating subnet router for routes %s: %v"
messageSubnetRouterInvalid = "Subnet router is invalid: %v"
messageSubnetRouterCreated = "Created subnet router for routes %s"
messageSubnetRouterCleanupFailed = "Failed cleaning up subnet router resources: %v"
msgSubnetRouterCleanupInProgress = "SubnetRouterCleanupInProgress"
reasonConnectorCreationFailed = "ConnectorCreationFailed"
reasonConnectorCreated = "ConnectorCreated"
reasonConnectorCleanupFailed = "ConnectorCleanupFailed"
reasonConnectorCleanupInProgress = "ConnectorCleanupInProgress"
reasonConnectorInvalid = "ConnectorInvalid"
messageConnectorCreationFailed = "Failed creating Connector: %v"
messageConnectorInvalid = "Connector is invalid: %v"
messageSubnetRouterCleanupFailed = "Failed cleaning up Connector resources: %v"
shortRequeue = time . Second * 5
)
@ -61,42 +59,39 @@ type ConnectorReconciler struct {
mu sync . Mutex // protects following
// subnetRouters tracks the subnet routers managed by this Tailscale
// Operator instance.
subnetRouters set . Slice [ types . UID ]
connectors set . Slice [ types . UID ] // for connectors gauge
}
var (
// gaugeIngressResources tracks the number of subnet routers that we're
// currently managing.
gaugeSubnetRouterResources = clientmetric . NewGauge ( "k8s_subnet_router_resources" )
// gaugeConnectorResources tracks the number of Connectors currently managed by this operator instance
gaugeConnectorResources = clientmetric . NewGauge ( "k8s_connector_resources" )
)
func ( a * ConnectorReconciler ) Reconcile ( ctx context . Context , req reconcile . Request ) ( _ reconcile . Result , err error ) {
logger := a . logger . With ( " c onnector", req . Name )
logger := a . logger . With ( " C onnector", req . Name )
logger . Debugf ( "starting reconcile" )
defer logger . Debugf ( "reconcile finished" )
cn := new ( tsapi . Connector )
err = a . Get ( ctx , req . NamespacedName , cn )
if apierrors . IsNotFound ( err ) {
logger . Debugf ( " c onnector not found, assuming it was deleted")
logger . Debugf ( " C onnector not found, assuming it was deleted")
return reconcile . Result { } , nil
} else if err != nil {
return reconcile . Result { } , fmt . Errorf ( "failed to get tailscale.com Connector: %w" , err )
}
if ! cn . DeletionTimestamp . IsZero ( ) {
logger . Debugf ( " c onnector is being deleted or should not be exposed, cleaning up compon ent s")
logger . Debugf ( " C onnector is being deleted or should not be exposed, cleaning up resour ces")
ix := xslices . Index ( cn . Finalizers , FinalizerName )
if ix < 0 {
logger . Debugf ( "no finalizer, nothing to do" )
return reconcile . Result { } , nil
}
if done , err := a . maybeCleanup SubnetRoute r( ctx , logger , cn ) ; err != nil {
if done , err := a . maybeCleanup Connecto r( ctx , logger , cn ) ; err != nil {
return reconcile . Result { } , err
} else if ! done {
logger . Debugf ( " cleanup not finished, will retry...")
logger . Debugf ( " Connector resource cleanup not ye t finished, will retry...")
return reconcile . Result { RequeueAfter : shortRequeue } , nil
}
@ -104,21 +99,20 @@ func (a *ConnectorReconciler) Reconcile(ctx context.Context, req reconcile.Reque
if err := a . Update ( ctx , cn ) ; err != nil {
return reconcile . Result { } , err
}
logger . Infof ( " c onnector resources cleaned up")
logger . Infof ( " C onnector resources cleaned up")
return reconcile . Result { } , nil
}
var (
reason , message string
readyStatus metav1 . ConditionStatus
)
oldCnStatus := cn . Status . DeepCopy ( )
defer func ( ) {
if cn . Status . SubnetRouter == nil {
tsoperator . SetConnectorCondition ( cn , tsapi . ConnectorReady , metav1 . ConditionUnknown , "" , "" , cn . Generation , a . clock , logger )
} else if cn . Status . SubnetRouter . Ready == metav1 . ConditionTrue {
tsoperator . SetConnectorCondition ( cn , tsapi . ConnectorReady , metav1 . ConditionTrue , reasonSubnetRouterCreated , reasonSubnetRouterCreated , cn . Generation , a . clock , logger )
} else {
tsoperator . SetConnectorCondition ( cn , tsapi . ConnectorReady , metav1 . ConditionFalse , cn . Status . SubnetRouter . Reason , cn . Status . SubnetRouter . Reason , cn . Generation , a . clock , logger )
}
tsoperator . SetConnectorCondition ( cn , tsapi . ConnectorReady , readyStatus , reason , message , cn . Generation , a . clock , logger )
if ! apiequality . Semantic . DeepEqual ( oldCnStatus , cn . Status ) {
// a n error encountered here should get returned by the Reconcile function
// An error encountered here should get returned by the Reconcile function.
if updateErr := a . Client . Status ( ) . Update ( ctx , cn ) ; updateErr != nil {
err = updateErr
}
@ -130,67 +124,85 @@ func (a *ConnectorReconciler) Reconcile(ctx context.Context, req reconcile.Reque
// because once the finalizer is in place this block gets skipped. So,
// this is a nice place to tell the operator that the high level,
// multi-reconcile operation is underway.
logger . Infof ( "ensuring c onnector is set up")
logger . Infof ( "ensuring C onnector is set up")
cn . Finalizers = append ( cn . Finalizers , FinalizerName )
if err := a . Update ( ctx , cn ) ; err != nil {
err = fmt . Errorf ( "failed to add finalizer: %w" , err )
logger . Errorf ( "error adding finalizer: %v" , err )
reason = reasonConnectorCreationFailed
message = fmt . Sprintf ( messageConnectorCreationFailed , err )
readyStatus = metav1 . ConditionFalse
return reconcile . Result { } , err
}
}
// A Connector with unset .spec.subnetRouter and unset
// cn.spec.subnetRouter.Routes will be rejected at apply time (because
// these fields are set as required by our CRD validation). This check
// is here for if our CRD validation breaks unnoticed we don't crash the
// operator with nil pointer exception.
if cn . Spec . SubnetRouter == nil || len ( cn . Spec . SubnetRouter . Routes ) < 1 {
if err := a . validate ( cn ) ; err != nil {
logger . Errorf ( "error validating Connector spec: %w" , err )
reason = reasonConnectorInvalid
message = fmt . Sprintf ( messageConnectorInvalid , err )
readyStatus = metav1 . ConditionFalse
a . recorder . Eventf ( cn , corev1 . EventTypeWarning , reasonConnectorInvalid , message )
return reconcile . Result { } , nil
}
if err := validateSubnetRouter ( * cn . Spec . SubnetRouter ) ; err != nil {
msg := fmt . Sprintf ( messageSubnetRouterInvalid , err )
cn . Status . SubnetRouter = & tsapi . SubnetRouterStatus {
Ready : metav1 . ConditionFalse ,
Reason : reasonSubnetRouterInvalid ,
Message : msg ,
if err = a . maybeProvisionConnector ( ctx , logger , cn ) ; err != nil {
logger . Errorf ( "error creating Connector resources: %w" , err )
reason = reasonConnectorCreationFailed
message = fmt . Sprintf ( messageConnectorCreationFailed , err )
readyStatus = metav1 . ConditionFalse
a . recorder . Eventf ( cn , corev1 . EventTypeWarning , reason , message )
} else {
logger . Info ( "Connector resources synced" )
reason = reasonConnectorCreated
message = reasonConnectorCreated
readyStatus = metav1 . ConditionTrue
cn . Status . IsExitNode = cn . Spec . IsExitNode
if cn . Spec . SubnetRouter != nil {
cn . Status . SubnetRoutes = cn . Spec . SubnetRouter . Routes . Stringify ( )
} else {
cn . Status . SubnetRoutes = ""
}
a . recorder . Eventf ( cn , corev1 . EventTypeWarning , reasonSubnetRouterInvalid , msg )
return reconcile . Result { } , nil
}
return reconcile . Result { } , err
}
var sb strings . Builder
sb . WriteString ( string ( cn . Spec . SubnetRouter . Routes [ 0 ] ) )
for _ , r := range cn . Spec . SubnetRouter . Routes [ 1 : ] {
sb . WriteString ( fmt . Sprintf ( ",%s" , r ) )
// maybeProvisionConnector ensures that any new resources required for this
// Connector instance are deployed to the cluster.
func ( a * ConnectorReconciler ) maybeProvisionConnector ( ctx context . Context , logger * zap . SugaredLogger , cn * tsapi . Connector ) error {
hostname := cn . Name + "-connector"
if cn . Spec . Hostname != "" {
hostname = string ( cn . Spec . Hostname )
}
cidrsS := sb . String ( )
logger . Debugf ( "ensuring a subnet router is deployed" )
err = a . maybeProvisionSubnetRouter ( ctx , logger , cn , cidrsS )
if err != nil {
msg := fmt . Sprintf ( messageSubnetRouterCreationFailed , cidrsS , err )
cn . Status . SubnetRouter = & tsapi . SubnetRouterStatus {
Ready : metav1 . ConditionFalse ,
Reason : reasonSubnetRouterCreationFailed ,
Message : msg ,
}
a . recorder . Eventf ( cn , corev1 . EventTypeWarning , reasonSubnetRouterCreationFailed , msg )
return reconcile . Result { } , err
crl := childResourceLabels ( cn . Name , a . tsnamespace , "connector" )
sts := & tailscaleSTSConfig {
ParentResourceName : cn . Name ,
ParentResourceUID : string ( cn . UID ) ,
Hostname : hostname ,
ChildResourceLabels : crl ,
Tags : cn . Spec . Tags . Stringify ( ) ,
Connector : & connector {
isExitNode : cn . Spec . IsExitNode ,
} ,
}
cn . Status . SubnetRouter = & tsapi . SubnetRouterStatus {
Routes : cidrsS ,
Ready : metav1 . ConditionTrue ,
Reason : reasonSubnetRouterCreated ,
Message : fmt . Sprintf ( messageSubnetRouterCreated , cidrsS ) ,
if cn . Spec . SubnetRouter != nil && len ( cn . Spec . SubnetRouter . Routes ) > 0 {
sts . Connector . routes = cn . Spec . SubnetRouter . Routes . Stringify ( )
}
return reconcile . Result { } , nil
a . mu . Lock ( )
a . connectors . Add ( cn . UID )
gaugeConnectorResources . Set ( int64 ( a . connectors . Len ( ) ) )
a . mu . Unlock ( )
_ , err := a . ssr . Provision ( ctx , logger , sts )
return err
}
func ( a * ConnectorReconciler ) maybeCleanupSubnetRouter ( ctx context . Context , logger * zap . SugaredLogger , cn * tsapi . Connector ) ( bool , error ) {
if done , err := a . ssr . Cleanup ( ctx , logger , childResourceLabels ( cn . Name , a . tsnamespace , "subnetrouter" ) ) ; err != nil {
return false , fmt . Errorf ( "failed to cleanup: %w" , err )
func ( a * ConnectorReconciler ) maybeCleanup Connecto r( ctx context . Context , logger * zap . SugaredLogger , cn * tsapi . Connector ) ( bool , error ) {
if done , err := a . ssr . Cleanup ( ctx , logger , childResourceLabels ( cn . Name , a . tsnamespace , " connecto r") ) ; err != nil {
return false , fmt . Errorf ( "failed to cleanup Connector resources : %w", err )
} else if ! done {
logger . Debugf ( "cleanup not done yet, waiting for next reconcile" )
logger . Debugf ( " Connector cleanup not done yet, waiting for next reconcile")
return false , nil
}
@ -198,42 +210,31 @@ func (a *ConnectorReconciler) maybeCleanupSubnetRouter(ctx context.Context, logg
// exactly once at the very end of cleanup, because the final step of
// cleanup removes the tailscale finalizer, which will make all future
// reconciles exit early.
logger . Infof ( "cleaned up subnet router ")
logger . Infof ( "cleaned up Connector resources ")
a . mu . Lock ( )
defer a . mu . Unlock ( )
a . subnetRoute rs. Remove ( cn . UID )
gauge SubnetRouterResources. Set ( int64 ( a . subnetRoute rs. Len ( ) ) )
a . connecto rs. Remove ( cn . UID )
gauge ConnectorResources. Set ( int64 ( a . connecto rs. Len ( ) ) )
return true , nil
}
// maybeProvisionSubnetRouter maybe deploys subnet router that exposes a subset of cluster cidrs to the tailnet
func ( a * ConnectorReconciler ) maybeProvisionSubnetRouter ( ctx context . Context , logger * zap . SugaredLogger , cn * tsapi . Connector , cidrs string ) error {
if cn . Spec . SubnetRouter == nil || len ( cn . Spec . SubnetRouter . Routes ) < 1 {
func ( a * ConnectorReconciler ) validate ( cn * tsapi . Connector ) error {
// Connector fields are already validated at apply time with CEL validation
// on custom resource fields. The checks here are a backup in case the
// CEL validation breaks without us noticing.
if ! ( cn . Spec . SubnetRouter != nil || cn . Spec . IsExitNode ) {
return errors . New ( "invalid Connector spec- a Connector must be either expose subnet routes or act as exit node (or both)" )
}
if cn . Spec . SubnetRouter == nil {
return nil
}
a . mu . Lock ( )
a . subnetRouters . Add ( cn . UID )
gaugeSubnetRouterResources . Set ( int64 ( a . subnetRouters . Len ( ) ) )
a . mu . Unlock ( )
return validateSubnetRouter ( cn . Spec . SubnetRouter )
}
crl := childResourceLabels ( cn . Name , a . tsnamespace , "subnetrouter" )
hostname := hostnameForSubnetRouter ( cn )
sts := & tailscaleSTSConfig {
ParentResourceName : cn . Name ,
ParentResourceUID : string ( cn . UID ) ,
Hostname : hostname ,
ChildResourceLabels : crl ,
Routes : cidrs ,
}
for _ , tag := range cn . Spec . SubnetRouter . Tags {
sts . Tags = append ( sts . Tags , string ( tag ) )
func validateSubnetRouter ( sb * tsapi . SubnetRouter ) error {
if len ( sb . Routes ) < 1 {
return errors . New ( "invalid subnet router spec: no routes defined" )
}
_ , err := a . ssr . Provision ( ctx , logger , sts )
return err
}
func validateSubnetRouter ( sb tsapi . SubnetRouter ) error {
var err error
for _ , route := range sb . Routes {
pfx , e := netip . ParsePrefix ( string ( route ) )
@ -247,13 +248,3 @@ func validateSubnetRouter(sb tsapi.SubnetRouter) error {
}
return err
}
func hostnameForSubnetRouter ( cn * tsapi . Connector ) string {
if cn . Spec . SubnetRouter == nil {
return ""
}
if cn . Spec . SubnetRouter . Hostname != "" {
return string ( cn . Spec . SubnetRouter . Hostname )
}
return cn . Name + "-" + "subnetrouter"
}