cmd/k8s-operator,k8s-operator: create ConfigMap for egress services + small fixes for egress services (#13715)

cmd/k8s-operator, k8s-operator: create ConfigMap for egress services + small reconciler fixes

Updates tailscale/tailscale#13406

Signed-off-by: Irbe Krumina <irbe@tailscale.com>

cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml

@@ -85,10 +85,7 @@ spec:
                     type: string
                     pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$
                 type:
-                  description: |-
+                  description: Type of the ProxyGroup proxies. Currently the only supported type is egress.
-                    Type of the ProxyGroup, either ingress or egress. Each set of proxies
-                    managed by a single ProxyGroup definition operate as only ingress or
-                    only egress proxies.
                   type: string
                   enum:
                     - egress

cmd/k8s-operator/deploy/manifests/operator.yaml

@@ -2497,10 +2497,7 @@ spec:
                                     type: string
                                 type: array
                             type:
-                                description: |-
+                                description: Type of the ProxyGroup proxies. Currently the only supported type is egress.
-                                    Type of the ProxyGroup, either ingress or egress. Each set of proxies
-                                    managed by a single ProxyGroup definition operate as only ingress or
-                                    only egress proxies.
                                 enum:
                                     - egress
                                 type: string

cmd/k8s-operator/egress-eps.go

@@ -58,8 +58,8 @@ func (er *egressEpsReconciler) Reconcile(ctx context.Context, req reconcile.Requ
 	// resources are set up for this tailnet service.
 	svc := &corev1.Service{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      eps.Labels[labelExternalSvcName],
+			Name:      eps.Labels[LabelParentName],
-			Namespace: eps.Labels[labelExternalSvcNamespace],
+			Namespace: eps.Labels[LabelParentNamespace],
 		},
 	}
 	err = er.Get(ctx, client.ObjectKeyFromObject(svc), svc)
@@ -98,7 +98,7 @@ func (er *egressEpsReconciler) Reconcile(ctx context.Context, req reconcile.Requ
 	// Check which Pods in ProxyGroup are ready to route traffic to this
 	// egress service.
 	podList := &corev1.PodList{}
-	if err := er.List(ctx, podList, client.MatchingLabels(map[string]string{labelProxyGroup: proxyGroupName})); err != nil {
+	if err := er.List(ctx, podList, client.MatchingLabels(pgLabels(proxyGroupName, nil))); err != nil {
 		return res, fmt.Errorf("error listing Pods for ProxyGroup %s: %w", proxyGroupName, err)
 	}
 	newEndpoints := make([]discoveryv1.Endpoint, 0)

cmd/k8s-operator/egress-eps_test.go

@@ -75,7 +75,11 @@ func TestTailscaleEgressEndpointSlices(t *testing.T) {
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "foo",
 			Namespace: "operator-ns",
-			Labels:    map[string]string{labelExternalSvcName: "test", labelExternalSvcNamespace: "default", labelProxyGroup: "foo"},
+			Labels: map[string]string{
+				LabelParentName:      "test",
+				LabelParentNamespace: "default",
+				labelSvcType:         typeEgress,
+				labelProxyGroup:      "foo"},
 		},
 		AddressType: discoveryv1.AddressTypeIPv4,
 	}
@@ -135,7 +139,7 @@ func configMapForSvc(t *testing.T, svc *corev1.Service, p uint16) *corev1.Config
 	}
 	cm := &corev1.ConfigMap{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      fmt.Sprintf(egressSvcsCMNameTemplate, svc.Annotations[AnnotationProxyGroup]),
+			Name:      pgEgressCMName(svc.Annotations[AnnotationProxyGroup]),
 			Namespace: "operator-ns",
 		},
 		BinaryData: map[string][]byte{egressservices.KeyEgressServices: bs},
@@ -173,7 +177,7 @@ func podAndSecretForProxyGroup(pg string) (*corev1.Pod, *corev1.Secret) {
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      fmt.Sprintf("%s-0", pg),
 			Namespace: "operator-ns",
-			Labels:    map[string]string{labelProxyGroup: pg},
+			Labels:    pgLabels(pg, nil),
 			UID:       "foo",
 		},
 		Status: corev1.PodStatus{
@@ -184,7 +188,7 @@ func podAndSecretForProxyGroup(pg string) (*corev1.Pod, *corev1.Secret) {
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      fmt.Sprintf("%s-0", pg),
 			Namespace: "operator-ns",
-			Labels:    map[string]string{labelProxyGroup: pg},
+			Labels:    pgSecretLabels(pg, "state"),
 		},
 	}
 	return p, s

cmd/k8s-operator/egress-services.go

@@ -46,10 +46,7 @@ const (
 	reasonEgressSvcCreationFailed = "EgressSvcCreationFailed"
 	reasonProxyGroupNotReady      = "ProxyGroupNotReady"
 
-	labelProxyGroup           = "tailscale.com/proxy-group"
+	labelProxyGroup = "tailscale.com/proxy-group"
-	labelProxyGroupType       = "tailscale.com/proxy-group-type"
-	labelExternalSvcName      = "tailscale.com/external-service-name"
-	labelExternalSvcNamespace = "tailscale.com/external-service-namespace"
 
 	labelSvcType = "tailscale.com/svc-type" // ingress or egress
 	typeEgress   = "egress"
@@ -62,8 +59,6 @@ const (
 	maxPorts = 10000
 
 	indexEgressProxyGroup = ".metadata.annotations.egress-proxy-group"
-
-	egressSvcsCMNameTemplate = "proxy-cfg-%s"
 )
 
 var gaugeEgressServices = clientmetric.NewGauge(kubetypes.MetricEgressServiceCount)
@@ -416,7 +411,7 @@ func (esr *egressSvcsReconciler) usedPortsForPG(ctx context.Context, pg string)
 func (esr *egressSvcsReconciler) clusterIPSvcForEgress(crl map[string]string) *corev1.Service {
 	return &corev1.Service{
 		ObjectMeta: metav1.ObjectMeta{
-			GenerateName: svcNameBase(crl[labelExternalSvcName]),
+			GenerateName: svcNameBase(crl[LabelParentName]),
 			Namespace:    esr.tsNamespace,
 			Labels:       crl,
 		},
@@ -428,7 +423,7 @@ func (esr *egressSvcsReconciler) clusterIPSvcForEgress(crl map[string]string) *c
 
 func (esr *egressSvcsReconciler) ensureEgressSvcCfgDeleted(ctx context.Context, svc *corev1.Service, logger *zap.SugaredLogger) error {
 	crl := egressSvcChildResourceLabels(svc)
-	cmName := fmt.Sprintf(egressSvcsCMNameTemplate, crl[labelProxyGroup])
+	cmName := pgEgressCMName(crl[labelProxyGroup])
 	cm := &corev1.ConfigMap{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      cmName,
@@ -479,15 +474,18 @@ func (esr *egressSvcsReconciler) validateClusterResources(ctx context.Context, s
 	if err := esr.Get(ctx, client.ObjectKeyFromObject(pg), pg); apierrors.IsNotFound(err) {
 		l.Infof("ProxyGroup %q not found, waiting...", proxyGroupName)
 		tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, esr.clock, l)
+		tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
 		return false, nil
 	} else if err != nil {
 		err := fmt.Errorf("unable to retrieve ProxyGroup %s: %w", proxyGroupName, err)
 		tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, err.Error(), esr.clock, l)
+		tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
 		return false, err
 	}
 	if !tsoperator.ProxyGroupIsReady(pg) {
 		l.Infof("ProxyGroup %s is not ready, waiting...", proxyGroupName)
 		tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, esr.clock, l)
+		tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
 		return false, nil
 	}
 
@@ -496,6 +494,7 @@ func (esr *egressSvcsReconciler) validateClusterResources(ctx context.Context, s
 		esr.recorder.Event(svc, corev1.EventTypeWarning, "INVALIDSERVICE", msg)
 		l.Info(msg)
 		tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionFalse, reasonEgressSvcInvalid, msg, esr.clock, l)
+		tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
 		return false, nil
 	}
 	l.Debugf("egress service is valid")
@@ -599,15 +598,15 @@ func isEgressSvcForProxyGroup(obj client.Object) bool {
 // egressSvcConfig returns a ConfigMap that contains egress services configuration for the provided ProxyGroup as well
 // as unmarshalled configuration from the ConfigMap.
 func egressSvcsConfigs(ctx context.Context, cl client.Client, proxyGroupName, tsNamespace string) (cm *corev1.ConfigMap, cfgs *egressservices.Configs, err error) {
-	cmName := fmt.Sprintf(egressSvcsCMNameTemplate, proxyGroupName)
+	name := pgEgressCMName(proxyGroupName)
 	cm = &corev1.ConfigMap{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      cmName,
+			Name:      name,
 			Namespace: tsNamespace,
 		},
 	}
 	if err := cl.Get(ctx, client.ObjectKeyFromObject(cm), cm); err != nil {
-		return nil, nil, fmt.Errorf("error retrieving egress services ConfigMap %s: %v", cmName, err)
+		return nil, nil, fmt.Errorf("error retrieving egress services ConfigMap %s: %v", name, err)
 	}
 	cfgs = &egressservices.Configs{}
 	if len(cm.BinaryData[egressservices.KeyEgressServices]) != 0 {
@@ -626,11 +625,12 @@ func egressSvcsConfigs(ctx context.Context, cl client.Client, proxyGroupName, ts
 // should probably validate and truncate (?) the names is they are too long.
 func egressSvcChildResourceLabels(svc *corev1.Service) map[string]string {
 	return map[string]string{
-		LabelManaged:              "true",
+		LabelManaged:         "true",
-		labelProxyGroup:           svc.Annotations[AnnotationProxyGroup],
+		LabelParentType:      "svc",
-		labelExternalSvcName:      svc.Name,
+		LabelParentName:      svc.Name,
-		labelExternalSvcNamespace: svc.Namespace,
+		LabelParentNamespace: svc.Namespace,
-		labelSvcType:              typeEgress,
+		labelProxyGroup:      svc.Annotations[AnnotationProxyGroup],
+		labelSvcType:         typeEgress,
 	}
 }
 

cmd/k8s-operator/egress-services_test.go

@@ -40,7 +40,7 @@ func TestTailscaleEgressServices(t *testing.T) {
 	}
 	cm := &corev1.ConfigMap{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      fmt.Sprintf(egressSvcsCMNameTemplate, "foo"),
+			Name:      pgEgressCMName("foo"),
 			Namespace: "operator-ns",
 		},
 	}

cmd/k8s-operator/operator.go

@@ -377,15 +377,16 @@ func runReconcilers(opts reconcilerOpts) {
 	}
 
 	epsFilter := handler.EnqueueRequestsFromMapFunc(egressEpsHandler)
-	podsSecretsFilter := handler.EnqueueRequestsFromMapFunc(egressEpsFromEgressPGChildResources(mgr.GetClient(), opts.log, opts.tailscaleNamespace))
+	podsFilter := handler.EnqueueRequestsFromMapFunc(egressEpsFromPGPods(mgr.GetClient(), opts.tailscaleNamespace))
-	epsFromExtNSvcFilter := handler.EnqueueRequestsFromMapFunc(epsFromExternalNameService(mgr.GetClient(), opts.log))
+	secretsFilter := handler.EnqueueRequestsFromMapFunc(egressEpsFromPGStateSecrets(mgr.GetClient(), opts.tailscaleNamespace))
+	epsFromExtNSvcFilter := handler.EnqueueRequestsFromMapFunc(epsFromExternalNameService(mgr.GetClient(), opts.log, opts.tailscaleNamespace))
 
 	err = builder.
 		ControllerManagedBy(mgr).
 		Named("egress-eps-reconciler").
 		Watches(&discoveryv1.EndpointSlice{}, epsFilter).
-		Watches(&corev1.Pod{}, podsSecretsFilter).
+		Watches(&corev1.Pod{}, podsFilter).
-		Watches(&corev1.Secret{}, podsSecretsFilter).
+		Watches(&corev1.Secret{}, secretsFilter).
 		Watches(&corev1.Service{}, epsFromExtNSvcFilter).
 		Complete(&egressEpsReconciler{
 			Client:      mgr.GetClient(),
@@ -841,40 +842,70 @@ func egressEpsHandler(_ context.Context, o client.Object) []reconcile.Request {
 	}
 }
 
-// egressEpsFromEgressPGChildResources returns a handler that checks if an
+// egressEpsFromEgressPods returns a Pod event handler that checks if Pod is a replica for a ProxyGroup and if it is,
-// object is a child resource for an egress ProxyGroup (a Pod or a state Secret)
+// returns reconciler requests for all egress EndpointSlices for that ProxyGroup.
-// and if it is, returns reconciler requests for all egress EndpointSlices for
+func egressEpsFromPGPods(cl client.Client, ns string) handler.MapFunc {
-// that ProxyGroup.
-func egressEpsFromEgressPGChildResources(cl client.Client, logger *zap.SugaredLogger, ns string) handler.MapFunc {
 	return func(_ context.Context, o client.Object) []reconcile.Request {
-		pg, ok := o.GetLabels()[labelProxyGroup]
+		if _, ok := o.GetLabels()[LabelManaged]; !ok {
+			return nil
+		}
+		// TODO(irbekrm): for now this is good enough as all ProxyGroups are egress. Add a type check once we
+		// have ingress ProxyGroups.
+		if typ := o.GetLabels()[LabelParentType]; typ != "proxygroup" {
+			return nil
+		}
+		pg, ok := o.GetLabels()[LabelParentName]
 		if !ok {
 			return nil
 		}
-		// TODO(irbekrm): depending on what labels we add to ProxyGroup
+		return reconcileRequestsForPG(pg, cl, ns)
-		// resources and which resources, this might need some extra
+	}
-		// checks.
+}
-		if typ, ok := o.GetLabels()[labelProxyGroupType]; !ok || typ != typeEgress {
+
+// egressEpsFromPGStateSecrets returns a Secret event handler that checks if Secret is a state Secret for a ProxyGroup and if it is,
+// returns reconciler requests for all egress EndpointSlices for that ProxyGroup.
+func egressEpsFromPGStateSecrets(cl client.Client, ns string) handler.MapFunc {
+	return func(_ context.Context, o client.Object) []reconcile.Request {
+		if _, ok := o.GetLabels()[LabelManaged]; !ok {
 			return nil
 		}
-		epsList := discoveryv1.EndpointSliceList{}
+		// TODO(irbekrm): for now this is good enough as all ProxyGroups are egress. Add a type check once we
-		if err := cl.List(context.Background(), &epsList, client.InNamespace(ns), client.MatchingLabels(map[string]string{labelProxyGroup: pg})); err != nil {
+		// have ingress ProxyGroups.
-			logger.Infof("error listing EndpointSlices: %v, skipping a reconcile for event on %s %s", err, o.GetName(), o.GetObjectKind().GroupVersionKind().Kind)
+		if parentType := o.GetLabels()[LabelParentType]; parentType != "proxygroup" {
 			return nil
 		}
-		reqs := make([]reconcile.Request, 0)
+		if secretType := o.GetLabels()[labelSecretType]; secretType != "state" {
-		for _, ep := range epsList.Items {
+			return nil
-			reqs = append(reqs, reconcile.Request{
-				NamespacedName: types.NamespacedName{
-					Namespace: ep.Namespace,
-					Name:      ep.Name,
-				},
-			})
 		}
-		return reqs
+		pg, ok := o.GetLabels()[LabelParentName]
+		if !ok {
+			return nil
+		}
+		return reconcileRequestsForPG(pg, cl, ns)
+	}
+}
+
+func reconcileRequestsForPG(pg string, cl client.Client, ns string) []reconcile.Request {
+	epsList := discoveryv1.EndpointSliceList{}
+	if err := cl.List(context.Background(), &epsList,
+		client.InNamespace(ns),
+		client.MatchingLabels(map[string]string{labelProxyGroup: pg})); err != nil {
+		return nil
+	}
+	reqs := make([]reconcile.Request, 0)
+	for _, ep := range epsList.Items {
+		reqs = append(reqs, reconcile.Request{
+			NamespacedName: types.NamespacedName{
+				Namespace: ep.Namespace,
+				Name:      ep.Name,
+			},
+		})
 	}
+	return reqs
 }
 
+// egressSvcsFromEgressProxyGroup is an event handler for egress ProxyGroups. It returns reconcile requests for all
+// user-created ExternalName Services that should be exposed on this ProxyGroup.
 func egressSvcsFromEgressProxyGroup(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {
 	return func(_ context.Context, o client.Object) []reconcile.Request {
 		pg, ok := o.(*tsapi.ProxyGroup)
@@ -903,7 +934,9 @@ func egressSvcsFromEgressProxyGroup(cl client.Client, logger *zap.SugaredLogger)
 	}
 }
 
-func epsFromExternalNameService(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {
+// epsFromExternalNameService is an event handler for ExternalName Services that define a Tailscale egress service that
+// should be exposed on a ProxyGroup. It returns reconcile requests for EndpointSlices created for this Service.
+func epsFromExternalNameService(cl client.Client, logger *zap.SugaredLogger, ns string) handler.MapFunc {
 	return func(_ context.Context, o client.Object) []reconcile.Request {
 		svc, ok := o.(*corev1.Service)
 		if !ok {
@@ -914,10 +947,8 @@ func epsFromExternalNameService(cl client.Client, logger *zap.SugaredLogger) han
 			return nil
 		}
 		epsList := &discoveryv1.EndpointSliceList{}
-		if err := cl.List(context.Background(), epsList, client.MatchingLabels(map[string]string{
+		if err := cl.List(context.Background(), epsList, client.InNamespace(ns),
-			labelExternalSvcName:      svc.Name,
+			client.MatchingLabels(egressSvcChildResourceLabels(svc))); err != nil {
-			labelExternalSvcNamespace: svc.Namespace,
-		})); err != nil {
 			logger.Infof("error listing EndpointSlices: %v, skipping a reconcile for event on Service %s", err, svc.Name)
 			return nil
 		}
@@ -934,6 +965,8 @@ func epsFromExternalNameService(cl client.Client, logger *zap.SugaredLogger) han
 	}
 }
 
+// indexEgressServices adds a local index to a cached Tailscale egress Services meant to be exposed on a ProxyGroup. The
+// index is used a list filter.
 func indexEgressServices(o client.Object) []string {
 	if !isEgressSvcForProxyGroup(o) {
 		return nil

cmd/k8s-operator/proxygroup.go

@@ -223,6 +223,15 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
 	}); err != nil {
 		return fmt.Errorf("error provisioning RoleBinding: %w", err)
 	}
+	if pg.Spec.Type == tsapi.ProxyGroupTypeEgress {
+		cm := pgEgressCM(pg, r.tsNamespace)
+		if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, cm, func(existing *corev1.ConfigMap) {
+			existing.ObjectMeta.Labels = cm.ObjectMeta.Labels
+			existing.ObjectMeta.OwnerReferences = cm.ObjectMeta.OwnerReferences
+		}); err != nil {
+			return fmt.Errorf("error provisioning ConfigMap: %w", err)
+		}
+	}
 	ss := pgStatefulSet(pg, r.tsNamespace, r.proxyImage, r.tsFirewallMode, cfgHash)
 	ss = applyProxyClassToStatefulSet(proxyClass, ss, nil, logger)
 	if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, ss, func(s *appsv1.StatefulSet) {

cmd/k8s-operator/proxygroup_specs.go

@@ -13,6 +13,7 @@ import (
 	rbacv1 "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/kube/egressservices"
 	"tailscale.com/types/ptr"
 )
 
@@ -80,6 +81,13 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode, cfgHa
 									})
 								}
 
+								if pg.Spec.Type == tsapi.ProxyGroupTypeEgress {
+									mounts = append(mounts, corev1.VolumeMount{
+										Name:      pgEgressCMName(pg.Name),
+										MountPath: "/etc/proxies",
+										ReadOnly:  true,
+									})
+								}
 								return mounts
 							}(),
 							Env: func() []corev1.EnvVar {
@@ -118,6 +126,12 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode, cfgHa
 										Value: "false",
 									},
 								}
+								if pg.Spec.Type == tsapi.ProxyGroupTypeEgress {
+									envs = append(envs, corev1.EnvVar{
+										Name:  "TS_EGRESS_SERVICES_CONFIG_PATH",
+										Value: fmt.Sprintf("/etc/proxies/%s", egressservices.KeyEgressServices),
+									})
+								}
 
 								if tsFirewallMode != "" {
 									envs = append(envs, corev1.EnvVar{
@@ -142,6 +156,18 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode, cfgHa
 								},
 							})
 						}
+						if pg.Spec.Type == tsapi.ProxyGroupTypeEgress {
+							volumes = append(volumes, corev1.Volume{
+								Name: pgEgressCMName(pg.Name),
+								VolumeSource: corev1.VolumeSource{
+									ConfigMap: &corev1.ConfigMapVolumeSource{
+										LocalObjectReference: corev1.LocalObjectReference{
+											Name: pgEgressCMName(pg.Name),
+										},
+									},
+								},
+							})
+						}
 
 						return volumes
 					}(),
@@ -230,6 +256,17 @@ func pgStateSecrets(pg *tsapi.ProxyGroup, namespace string) (secrets []*corev1.S
 	return secrets
 }
 
+func pgEgressCM(pg *tsapi.ProxyGroup, namespace string) *corev1.ConfigMap {
+	return &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            pgEgressCMName(pg.Name),
+			Namespace:       namespace,
+			Labels:          pgLabels(pg.Name, nil),
+			OwnerReferences: pgOwnerReference(pg),
+		},
+	}
+}
+
 func pgSecretLabels(pgName, typ string) map[string]string {
 	return pgLabels(pgName, map[string]string{
 		labelSecretType: typ, // "config" or "state".
@@ -260,3 +297,7 @@ func pgReplicas(pg *tsapi.ProxyGroup) int32 {
 
 	return 2
 }
+
+func pgEgressCMName(pg string) string {
+	return fmt.Sprintf("%s-egress-config", pg)
+}

cmd/k8s-operator/sts.go

@@ -49,10 +49,9 @@ const (
 	LabelParentNamespace = "tailscale.com/parent-resource-ns"
 	labelSecretType      = "tailscale.com/secret-type" // "config" or "state".
 
-	// LabelProxyClass can be set by users on Connectors, tailscale
+	// LabelProxyClass can be set by users on tailscale Ingresses and Services that define cluster ingress or
-	// Ingresses and Services that define cluster ingress or cluster egress,
+	// cluster egress, to specify that configuration in this ProxyClass should be applied to resources created for
-	// to specify that configuration in this ProxyClass should be applied to
+	// the Ingress or Service.
-	// resources created for the Connector, Ingress or Service.
 	LabelProxyClass = "tailscale.com/proxy-class"
 
 	FinalizerName = "tailscale.com/finalizer"

cmd/k8s-operator/svc.go

@@ -112,6 +112,10 @@ func (a *ServiceReconciler) Reconcile(ctx context.Context, req reconcile.Request
 		return reconcile.Result{}, fmt.Errorf("failed to get svc: %w", err)
 	}
 
+	if _, ok := svc.Annotations[AnnotationProxyGroup]; ok {
+		return reconcile.Result{}, nil // this reconciler should not look at Services for ProxyGroup
+	}
+
 	if !svc.DeletionTimestamp.IsZero() || !a.isTailscaleService(svc) {
 		logger.Debugf("service is being deleted or is (no longer) referring to Tailscale ingress/egress, ensuring any created resources are cleaned up")
 		return reconcile.Result{}, a.maybeCleanup(ctx, logger, svc)

k8s-operator/api.md

@@ -522,7 +522,7 @@ _Appears in:_
 
 | Field | Description | Default | Validation |
 | --- | --- | --- | --- |
-| `type` _[ProxyGroupType](#proxygrouptype)_ | Type of the ProxyGroup, either ingress or egress. Each set of proxies<br />managed by a single ProxyGroup definition operate as only ingress or<br />only egress proxies. |  | Enum: [egress] <br />Type: string <br /> |
+| `type` _[ProxyGroupType](#proxygrouptype)_ | Type of the ProxyGroup proxies. Currently the only supported type is egress. |  | Enum: [egress] <br />Type: string <br /> |
 | `tags` _[Tags](#tags)_ | Tags that the Tailscale devices will be tagged with. Defaults to [tag:k8s].<br />If you specify custom tags here, make sure you also make the operator<br />an owner of these tags.<br />See  https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator.<br />Tags cannot be changed once a ProxyGroup device has been created.<br />Tag values must be in form ^tag:[a-zA-Z][a-zA-Z0-9-]*$. |  | Pattern: `^tag:[a-zA-Z][a-zA-Z0-9-]*$` <br />Type: string <br /> |
 | `replicas` _integer_ | Replicas specifies how many replicas to create the StatefulSet with.<br />Defaults to 2. |  |  |
 | `hostnamePrefix` _[HostnamePrefix](#hostnameprefix)_ | HostnamePrefix is the hostname prefix to use for tailnet devices created<br />by the ProxyGroup. Each device will have the integer number from its<br />StatefulSet pod appended to this prefix to form the full hostname.<br />HostnamePrefix can contain lower case letters, numbers and dashes, it<br />must not start with a dash and must be between 1 and 62 characters long. |  | Pattern: `^[a-z0-9][a-z0-9-]{0,61}$` <br />Type: string <br /> |

k8s-operator/apis/v1alpha1/types_proxygroup.go

@@ -37,9 +37,7 @@ type ProxyGroupList struct {
 }
 
 type ProxyGroupSpec struct {
-	// Type of the ProxyGroup, either ingress or egress. Each set of proxies
+	// Type of the ProxyGroup proxies. Currently the only supported type is egress.
-	// managed by a single ProxyGroup definition operate as only ingress or
-	// only egress proxies.
 	Type ProxyGroupType `json:"type"`
 
 	// Tags that the Tailscale devices will be tagged with. Defaults to [tag:k8s].