From 7f016baa8743a6d66f61e5fc81d55e5147dd898c Mon Sep 17 00:00:00 2001 From: Irbe Krumina Date: Mon, 7 Oct 2024 20:12:56 +0100 Subject: [PATCH] cmd/k8s-operator,k8s-operator: create ConfigMap for egress services + small fixes for egress services (#13715) cmd/k8s-operator, k8s-operator: create ConfigMap for egress services + small reconciler fixes Updates tailscale/tailscale#13406 Signed-off-by: Irbe Krumina --- .../crds/tailscale.com_proxygroups.yaml | 5 +- .../deploy/manifests/operator.yaml | 5 +- cmd/k8s-operator/egress-eps.go | 6 +- cmd/k8s-operator/egress-eps_test.go | 12 ++- cmd/k8s-operator/egress-services.go | 32 +++---- cmd/k8s-operator/egress-services_test.go | 2 +- cmd/k8s-operator/operator.go | 95 +++++++++++++------ cmd/k8s-operator/proxygroup.go | 9 ++ cmd/k8s-operator/proxygroup_specs.go | 41 ++++++++ cmd/k8s-operator/sts.go | 7 +- cmd/k8s-operator/svc.go | 4 + k8s-operator/api.md | 2 +- .../apis/v1alpha1/types_proxygroup.go | 4 +- 13 files changed, 153 insertions(+), 71 deletions(-) diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml index 32e2ab450..035d04786 100644 --- a/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml +++ b/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml @@ -85,10 +85,7 @@ spec: type: string pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$ type: - description: |- - Type of the ProxyGroup, either ingress or egress. Each set of proxies - managed by a single ProxyGroup definition operate as only ingress or - only egress proxies. + description: Type of the ProxyGroup proxies. Currently the only supported type is egress. type: string enum: - egress diff --git a/cmd/k8s-operator/deploy/manifests/operator.yaml b/cmd/k8s-operator/deploy/manifests/operator.yaml index e6358708b..14166fed9 100644 --- a/cmd/k8s-operator/deploy/manifests/operator.yaml +++ b/cmd/k8s-operator/deploy/manifests/operator.yaml @@ -2497,10 +2497,7 @@ spec: type: string type: array type: - description: |- - Type of the ProxyGroup, either ingress or egress. Each set of proxies - managed by a single ProxyGroup definition operate as only ingress or - only egress proxies. + description: Type of the ProxyGroup proxies. Currently the only supported type is egress. enum: - egress type: string diff --git a/cmd/k8s-operator/egress-eps.go b/cmd/k8s-operator/egress-eps.go index 510d58783..fa13c525f 100644 --- a/cmd/k8s-operator/egress-eps.go +++ b/cmd/k8s-operator/egress-eps.go @@ -58,8 +58,8 @@ func (er *egressEpsReconciler) Reconcile(ctx context.Context, req reconcile.Requ // resources are set up for this tailnet service. svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: eps.Labels[labelExternalSvcName], - Namespace: eps.Labels[labelExternalSvcNamespace], + Name: eps.Labels[LabelParentName], + Namespace: eps.Labels[LabelParentNamespace], }, } err = er.Get(ctx, client.ObjectKeyFromObject(svc), svc) @@ -98,7 +98,7 @@ func (er *egressEpsReconciler) Reconcile(ctx context.Context, req reconcile.Requ // Check which Pods in ProxyGroup are ready to route traffic to this // egress service. podList := &corev1.PodList{} - if err := er.List(ctx, podList, client.MatchingLabels(map[string]string{labelProxyGroup: proxyGroupName})); err != nil { + if err := er.List(ctx, podList, client.MatchingLabels(pgLabels(proxyGroupName, nil))); err != nil { return res, fmt.Errorf("error listing Pods for ProxyGroup %s: %w", proxyGroupName, err) } newEndpoints := make([]discoveryv1.Endpoint, 0) diff --git a/cmd/k8s-operator/egress-eps_test.go b/cmd/k8s-operator/egress-eps_test.go index a2e95e5d3..00d13b2a7 100644 --- a/cmd/k8s-operator/egress-eps_test.go +++ b/cmd/k8s-operator/egress-eps_test.go @@ -75,7 +75,11 @@ func TestTailscaleEgressEndpointSlices(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "foo", Namespace: "operator-ns", - Labels: map[string]string{labelExternalSvcName: "test", labelExternalSvcNamespace: "default", labelProxyGroup: "foo"}, + Labels: map[string]string{ + LabelParentName: "test", + LabelParentNamespace: "default", + labelSvcType: typeEgress, + labelProxyGroup: "foo"}, }, AddressType: discoveryv1.AddressTypeIPv4, } @@ -135,7 +139,7 @@ func configMapForSvc(t *testing.T, svc *corev1.Service, p uint16) *corev1.Config } cm := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf(egressSvcsCMNameTemplate, svc.Annotations[AnnotationProxyGroup]), + Name: pgEgressCMName(svc.Annotations[AnnotationProxyGroup]), Namespace: "operator-ns", }, BinaryData: map[string][]byte{egressservices.KeyEgressServices: bs}, @@ -173,7 +177,7 @@ func podAndSecretForProxyGroup(pg string) (*corev1.Pod, *corev1.Secret) { ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-0", pg), Namespace: "operator-ns", - Labels: map[string]string{labelProxyGroup: pg}, + Labels: pgLabels(pg, nil), UID: "foo", }, Status: corev1.PodStatus{ @@ -184,7 +188,7 @@ func podAndSecretForProxyGroup(pg string) (*corev1.Pod, *corev1.Secret) { ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-0", pg), Namespace: "operator-ns", - Labels: map[string]string{labelProxyGroup: pg}, + Labels: pgSecretLabels(pg, "state"), }, } return p, s diff --git a/cmd/k8s-operator/egress-services.go b/cmd/k8s-operator/egress-services.go index 1c4f70a96..20bafe8ec 100644 --- a/cmd/k8s-operator/egress-services.go +++ b/cmd/k8s-operator/egress-services.go @@ -46,10 +46,7 @@ const ( reasonEgressSvcCreationFailed = "EgressSvcCreationFailed" reasonProxyGroupNotReady = "ProxyGroupNotReady" - labelProxyGroup = "tailscale.com/proxy-group" - labelProxyGroupType = "tailscale.com/proxy-group-type" - labelExternalSvcName = "tailscale.com/external-service-name" - labelExternalSvcNamespace = "tailscale.com/external-service-namespace" + labelProxyGroup = "tailscale.com/proxy-group" labelSvcType = "tailscale.com/svc-type" // ingress or egress typeEgress = "egress" @@ -62,8 +59,6 @@ const ( maxPorts = 10000 indexEgressProxyGroup = ".metadata.annotations.egress-proxy-group" - - egressSvcsCMNameTemplate = "proxy-cfg-%s" ) var gaugeEgressServices = clientmetric.NewGauge(kubetypes.MetricEgressServiceCount) @@ -416,7 +411,7 @@ func (esr *egressSvcsReconciler) usedPortsForPG(ctx context.Context, pg string) func (esr *egressSvcsReconciler) clusterIPSvcForEgress(crl map[string]string) *corev1.Service { return &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - GenerateName: svcNameBase(crl[labelExternalSvcName]), + GenerateName: svcNameBase(crl[LabelParentName]), Namespace: esr.tsNamespace, Labels: crl, }, @@ -428,7 +423,7 @@ func (esr *egressSvcsReconciler) clusterIPSvcForEgress(crl map[string]string) *c func (esr *egressSvcsReconciler) ensureEgressSvcCfgDeleted(ctx context.Context, svc *corev1.Service, logger *zap.SugaredLogger) error { crl := egressSvcChildResourceLabels(svc) - cmName := fmt.Sprintf(egressSvcsCMNameTemplate, crl[labelProxyGroup]) + cmName := pgEgressCMName(crl[labelProxyGroup]) cm := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: cmName, @@ -479,15 +474,18 @@ func (esr *egressSvcsReconciler) validateClusterResources(ctx context.Context, s if err := esr.Get(ctx, client.ObjectKeyFromObject(pg), pg); apierrors.IsNotFound(err) { l.Infof("ProxyGroup %q not found, waiting...", proxyGroupName) tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, esr.clock, l) + tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured) return false, nil } else if err != nil { err := fmt.Errorf("unable to retrieve ProxyGroup %s: %w", proxyGroupName, err) tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, err.Error(), esr.clock, l) + tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured) return false, err } if !tsoperator.ProxyGroupIsReady(pg) { l.Infof("ProxyGroup %s is not ready, waiting...", proxyGroupName) tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, esr.clock, l) + tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured) return false, nil } @@ -496,6 +494,7 @@ func (esr *egressSvcsReconciler) validateClusterResources(ctx context.Context, s esr.recorder.Event(svc, corev1.EventTypeWarning, "INVALIDSERVICE", msg) l.Info(msg) tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionFalse, reasonEgressSvcInvalid, msg, esr.clock, l) + tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured) return false, nil } l.Debugf("egress service is valid") @@ -599,15 +598,15 @@ func isEgressSvcForProxyGroup(obj client.Object) bool { // egressSvcConfig returns a ConfigMap that contains egress services configuration for the provided ProxyGroup as well // as unmarshalled configuration from the ConfigMap. func egressSvcsConfigs(ctx context.Context, cl client.Client, proxyGroupName, tsNamespace string) (cm *corev1.ConfigMap, cfgs *egressservices.Configs, err error) { - cmName := fmt.Sprintf(egressSvcsCMNameTemplate, proxyGroupName) + name := pgEgressCMName(proxyGroupName) cm = &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ - Name: cmName, + Name: name, Namespace: tsNamespace, }, } if err := cl.Get(ctx, client.ObjectKeyFromObject(cm), cm); err != nil { - return nil, nil, fmt.Errorf("error retrieving egress services ConfigMap %s: %v", cmName, err) + return nil, nil, fmt.Errorf("error retrieving egress services ConfigMap %s: %v", name, err) } cfgs = &egressservices.Configs{} if len(cm.BinaryData[egressservices.KeyEgressServices]) != 0 { @@ -626,11 +625,12 @@ func egressSvcsConfigs(ctx context.Context, cl client.Client, proxyGroupName, ts // should probably validate and truncate (?) the names is they are too long. func egressSvcChildResourceLabels(svc *corev1.Service) map[string]string { return map[string]string{ - LabelManaged: "true", - labelProxyGroup: svc.Annotations[AnnotationProxyGroup], - labelExternalSvcName: svc.Name, - labelExternalSvcNamespace: svc.Namespace, - labelSvcType: typeEgress, + LabelManaged: "true", + LabelParentType: "svc", + LabelParentName: svc.Name, + LabelParentNamespace: svc.Namespace, + labelProxyGroup: svc.Annotations[AnnotationProxyGroup], + labelSvcType: typeEgress, } } diff --git a/cmd/k8s-operator/egress-services_test.go b/cmd/k8s-operator/egress-services_test.go index 1adde4e90..ac7733985 100644 --- a/cmd/k8s-operator/egress-services_test.go +++ b/cmd/k8s-operator/egress-services_test.go @@ -40,7 +40,7 @@ func TestTailscaleEgressServices(t *testing.T) { } cm := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf(egressSvcsCMNameTemplate, "foo"), + Name: pgEgressCMName("foo"), Namespace: "operator-ns", }, } diff --git a/cmd/k8s-operator/operator.go b/cmd/k8s-operator/operator.go index f744c9f5e..28895269d 100644 --- a/cmd/k8s-operator/operator.go +++ b/cmd/k8s-operator/operator.go @@ -377,15 +377,16 @@ func runReconcilers(opts reconcilerOpts) { } epsFilter := handler.EnqueueRequestsFromMapFunc(egressEpsHandler) - podsSecretsFilter := handler.EnqueueRequestsFromMapFunc(egressEpsFromEgressPGChildResources(mgr.GetClient(), opts.log, opts.tailscaleNamespace)) - epsFromExtNSvcFilter := handler.EnqueueRequestsFromMapFunc(epsFromExternalNameService(mgr.GetClient(), opts.log)) + podsFilter := handler.EnqueueRequestsFromMapFunc(egressEpsFromPGPods(mgr.GetClient(), opts.tailscaleNamespace)) + secretsFilter := handler.EnqueueRequestsFromMapFunc(egressEpsFromPGStateSecrets(mgr.GetClient(), opts.tailscaleNamespace)) + epsFromExtNSvcFilter := handler.EnqueueRequestsFromMapFunc(epsFromExternalNameService(mgr.GetClient(), opts.log, opts.tailscaleNamespace)) err = builder. ControllerManagedBy(mgr). Named("egress-eps-reconciler"). Watches(&discoveryv1.EndpointSlice{}, epsFilter). - Watches(&corev1.Pod{}, podsSecretsFilter). - Watches(&corev1.Secret{}, podsSecretsFilter). + Watches(&corev1.Pod{}, podsFilter). + Watches(&corev1.Secret{}, secretsFilter). Watches(&corev1.Service{}, epsFromExtNSvcFilter). Complete(&egressEpsReconciler{ Client: mgr.GetClient(), @@ -841,40 +842,70 @@ func egressEpsHandler(_ context.Context, o client.Object) []reconcile.Request { } } -// egressEpsFromEgressPGChildResources returns a handler that checks if an -// object is a child resource for an egress ProxyGroup (a Pod or a state Secret) -// and if it is, returns reconciler requests for all egress EndpointSlices for -// that ProxyGroup. -func egressEpsFromEgressPGChildResources(cl client.Client, logger *zap.SugaredLogger, ns string) handler.MapFunc { +// egressEpsFromEgressPods returns a Pod event handler that checks if Pod is a replica for a ProxyGroup and if it is, +// returns reconciler requests for all egress EndpointSlices for that ProxyGroup. +func egressEpsFromPGPods(cl client.Client, ns string) handler.MapFunc { return func(_ context.Context, o client.Object) []reconcile.Request { - pg, ok := o.GetLabels()[labelProxyGroup] + if _, ok := o.GetLabels()[LabelManaged]; !ok { + return nil + } + // TODO(irbekrm): for now this is good enough as all ProxyGroups are egress. Add a type check once we + // have ingress ProxyGroups. + if typ := o.GetLabels()[LabelParentType]; typ != "proxygroup" { + return nil + } + pg, ok := o.GetLabels()[LabelParentName] if !ok { return nil } - // TODO(irbekrm): depending on what labels we add to ProxyGroup - // resources and which resources, this might need some extra - // checks. - if typ, ok := o.GetLabels()[labelProxyGroupType]; !ok || typ != typeEgress { + return reconcileRequestsForPG(pg, cl, ns) + } +} + +// egressEpsFromPGStateSecrets returns a Secret event handler that checks if Secret is a state Secret for a ProxyGroup and if it is, +// returns reconciler requests for all egress EndpointSlices for that ProxyGroup. +func egressEpsFromPGStateSecrets(cl client.Client, ns string) handler.MapFunc { + return func(_ context.Context, o client.Object) []reconcile.Request { + if _, ok := o.GetLabels()[LabelManaged]; !ok { return nil } - epsList := discoveryv1.EndpointSliceList{} - if err := cl.List(context.Background(), &epsList, client.InNamespace(ns), client.MatchingLabels(map[string]string{labelProxyGroup: pg})); err != nil { - logger.Infof("error listing EndpointSlices: %v, skipping a reconcile for event on %s %s", err, o.GetName(), o.GetObjectKind().GroupVersionKind().Kind) + // TODO(irbekrm): for now this is good enough as all ProxyGroups are egress. Add a type check once we + // have ingress ProxyGroups. + if parentType := o.GetLabels()[LabelParentType]; parentType != "proxygroup" { return nil } - reqs := make([]reconcile.Request, 0) - for _, ep := range epsList.Items { - reqs = append(reqs, reconcile.Request{ - NamespacedName: types.NamespacedName{ - Namespace: ep.Namespace, - Name: ep.Name, - }, - }) + if secretType := o.GetLabels()[labelSecretType]; secretType != "state" { + return nil } - return reqs + pg, ok := o.GetLabels()[LabelParentName] + if !ok { + return nil + } + return reconcileRequestsForPG(pg, cl, ns) + } +} + +func reconcileRequestsForPG(pg string, cl client.Client, ns string) []reconcile.Request { + epsList := discoveryv1.EndpointSliceList{} + if err := cl.List(context.Background(), &epsList, + client.InNamespace(ns), + client.MatchingLabels(map[string]string{labelProxyGroup: pg})); err != nil { + return nil + } + reqs := make([]reconcile.Request, 0) + for _, ep := range epsList.Items { + reqs = append(reqs, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: ep.Namespace, + Name: ep.Name, + }, + }) } + return reqs } +// egressSvcsFromEgressProxyGroup is an event handler for egress ProxyGroups. It returns reconcile requests for all +// user-created ExternalName Services that should be exposed on this ProxyGroup. func egressSvcsFromEgressProxyGroup(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc { return func(_ context.Context, o client.Object) []reconcile.Request { pg, ok := o.(*tsapi.ProxyGroup) @@ -903,7 +934,9 @@ func egressSvcsFromEgressProxyGroup(cl client.Client, logger *zap.SugaredLogger) } } -func epsFromExternalNameService(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc { +// epsFromExternalNameService is an event handler for ExternalName Services that define a Tailscale egress service that +// should be exposed on a ProxyGroup. It returns reconcile requests for EndpointSlices created for this Service. +func epsFromExternalNameService(cl client.Client, logger *zap.SugaredLogger, ns string) handler.MapFunc { return func(_ context.Context, o client.Object) []reconcile.Request { svc, ok := o.(*corev1.Service) if !ok { @@ -914,10 +947,8 @@ func epsFromExternalNameService(cl client.Client, logger *zap.SugaredLogger) han return nil } epsList := &discoveryv1.EndpointSliceList{} - if err := cl.List(context.Background(), epsList, client.MatchingLabels(map[string]string{ - labelExternalSvcName: svc.Name, - labelExternalSvcNamespace: svc.Namespace, - })); err != nil { + if err := cl.List(context.Background(), epsList, client.InNamespace(ns), + client.MatchingLabels(egressSvcChildResourceLabels(svc))); err != nil { logger.Infof("error listing EndpointSlices: %v, skipping a reconcile for event on Service %s", err, svc.Name) return nil } @@ -934,6 +965,8 @@ func epsFromExternalNameService(cl client.Client, logger *zap.SugaredLogger) han } } +// indexEgressServices adds a local index to a cached Tailscale egress Services meant to be exposed on a ProxyGroup. The +// index is used a list filter. func indexEgressServices(o client.Object) []string { if !isEgressSvcForProxyGroup(o) { return nil diff --git a/cmd/k8s-operator/proxygroup.go b/cmd/k8s-operator/proxygroup.go index f19339059..99f48f323 100644 --- a/cmd/k8s-operator/proxygroup.go +++ b/cmd/k8s-operator/proxygroup.go @@ -223,6 +223,15 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro }); err != nil { return fmt.Errorf("error provisioning RoleBinding: %w", err) } + if pg.Spec.Type == tsapi.ProxyGroupTypeEgress { + cm := pgEgressCM(pg, r.tsNamespace) + if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, cm, func(existing *corev1.ConfigMap) { + existing.ObjectMeta.Labels = cm.ObjectMeta.Labels + existing.ObjectMeta.OwnerReferences = cm.ObjectMeta.OwnerReferences + }); err != nil { + return fmt.Errorf("error provisioning ConfigMap: %w", err) + } + } ss := pgStatefulSet(pg, r.tsNamespace, r.proxyImage, r.tsFirewallMode, cfgHash) ss = applyProxyClassToStatefulSet(proxyClass, ss, nil, logger) if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, ss, func(s *appsv1.StatefulSet) { diff --git a/cmd/k8s-operator/proxygroup_specs.go b/cmd/k8s-operator/proxygroup_specs.go index bf2adcbf5..a1ec9ccde 100644 --- a/cmd/k8s-operator/proxygroup_specs.go +++ b/cmd/k8s-operator/proxygroup_specs.go @@ -13,6 +13,7 @@ import ( rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/kube/egressservices" "tailscale.com/types/ptr" ) @@ -80,6 +81,13 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode, cfgHa }) } + if pg.Spec.Type == tsapi.ProxyGroupTypeEgress { + mounts = append(mounts, corev1.VolumeMount{ + Name: pgEgressCMName(pg.Name), + MountPath: "/etc/proxies", + ReadOnly: true, + }) + } return mounts }(), Env: func() []corev1.EnvVar { @@ -118,6 +126,12 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode, cfgHa Value: "false", }, } + if pg.Spec.Type == tsapi.ProxyGroupTypeEgress { + envs = append(envs, corev1.EnvVar{ + Name: "TS_EGRESS_SERVICES_CONFIG_PATH", + Value: fmt.Sprintf("/etc/proxies/%s", egressservices.KeyEgressServices), + }) + } if tsFirewallMode != "" { envs = append(envs, corev1.EnvVar{ @@ -142,6 +156,18 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode, cfgHa }, }) } + if pg.Spec.Type == tsapi.ProxyGroupTypeEgress { + volumes = append(volumes, corev1.Volume{ + Name: pgEgressCMName(pg.Name), + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: pgEgressCMName(pg.Name), + }, + }, + }, + }) + } return volumes }(), @@ -230,6 +256,17 @@ func pgStateSecrets(pg *tsapi.ProxyGroup, namespace string) (secrets []*corev1.S return secrets } +func pgEgressCM(pg *tsapi.ProxyGroup, namespace string) *corev1.ConfigMap { + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: pgEgressCMName(pg.Name), + Namespace: namespace, + Labels: pgLabels(pg.Name, nil), + OwnerReferences: pgOwnerReference(pg), + }, + } +} + func pgSecretLabels(pgName, typ string) map[string]string { return pgLabels(pgName, map[string]string{ labelSecretType: typ, // "config" or "state". @@ -260,3 +297,7 @@ func pgReplicas(pg *tsapi.ProxyGroup) int32 { return 2 } + +func pgEgressCMName(pg string) string { + return fmt.Sprintf("%s-egress-config", pg) +} diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go index 19c98100f..6378a8263 100644 --- a/cmd/k8s-operator/sts.go +++ b/cmd/k8s-operator/sts.go @@ -49,10 +49,9 @@ const ( LabelParentNamespace = "tailscale.com/parent-resource-ns" labelSecretType = "tailscale.com/secret-type" // "config" or "state". - // LabelProxyClass can be set by users on Connectors, tailscale - // Ingresses and Services that define cluster ingress or cluster egress, - // to specify that configuration in this ProxyClass should be applied to - // resources created for the Connector, Ingress or Service. + // LabelProxyClass can be set by users on tailscale Ingresses and Services that define cluster ingress or + // cluster egress, to specify that configuration in this ProxyClass should be applied to resources created for + // the Ingress or Service. LabelProxyClass = "tailscale.com/proxy-class" FinalizerName = "tailscale.com/finalizer" diff --git a/cmd/k8s-operator/svc.go b/cmd/k8s-operator/svc.go index e47fcae7f..22487ee26 100644 --- a/cmd/k8s-operator/svc.go +++ b/cmd/k8s-operator/svc.go @@ -112,6 +112,10 @@ func (a *ServiceReconciler) Reconcile(ctx context.Context, req reconcile.Request return reconcile.Result{}, fmt.Errorf("failed to get svc: %w", err) } + if _, ok := svc.Annotations[AnnotationProxyGroup]; ok { + return reconcile.Result{}, nil // this reconciler should not look at Services for ProxyGroup + } + if !svc.DeletionTimestamp.IsZero() || !a.isTailscaleService(svc) { logger.Debugf("service is being deleted or is (no longer) referring to Tailscale ingress/egress, ensuring any created resources are cleaned up") return reconcile.Result{}, a.maybeCleanup(ctx, logger, svc) diff --git a/k8s-operator/api.md b/k8s-operator/api.md index 82a3476ae..fd0a4e6ce 100644 --- a/k8s-operator/api.md +++ b/k8s-operator/api.md @@ -522,7 +522,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `type` _[ProxyGroupType](#proxygrouptype)_ | Type of the ProxyGroup, either ingress or egress. Each set of proxies
managed by a single ProxyGroup definition operate as only ingress or
only egress proxies. | | Enum: [egress]
Type: string
| +| `type` _[ProxyGroupType](#proxygrouptype)_ | Type of the ProxyGroup proxies. Currently the only supported type is egress. | | Enum: [egress]
Type: string
| | `tags` _[Tags](#tags)_ | Tags that the Tailscale devices will be tagged with. Defaults to [tag:k8s].
If you specify custom tags here, make sure you also make the operator
an owner of these tags.
See https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator.
Tags cannot be changed once a ProxyGroup device has been created.
Tag values must be in form ^tag:[a-zA-Z][a-zA-Z0-9-]*$. | | Pattern: `^tag:[a-zA-Z][a-zA-Z0-9-]*$`
Type: string
| | `replicas` _integer_ | Replicas specifies how many replicas to create the StatefulSet with.
Defaults to 2. | | | | `hostnamePrefix` _[HostnamePrefix](#hostnameprefix)_ | HostnamePrefix is the hostname prefix to use for tailnet devices created
by the ProxyGroup. Each device will have the integer number from its
StatefulSet pod appended to this prefix to form the full hostname.
HostnamePrefix can contain lower case letters, numbers and dashes, it
must not start with a dash and must be between 1 and 62 characters long. | | Pattern: `^[a-z0-9][a-z0-9-]{0,61}$`
Type: string
| diff --git a/k8s-operator/apis/v1alpha1/types_proxygroup.go b/k8s-operator/apis/v1alpha1/types_proxygroup.go index 9b0e4215e..ef1e8c8c1 100644 --- a/k8s-operator/apis/v1alpha1/types_proxygroup.go +++ b/k8s-operator/apis/v1alpha1/types_proxygroup.go @@ -37,9 +37,7 @@ type ProxyGroupList struct { } type ProxyGroupSpec struct { - // Type of the ProxyGroup, either ingress or egress. Each set of proxies - // managed by a single ProxyGroup definition operate as only ingress or - // only egress proxies. + // Type of the ProxyGroup proxies. Currently the only supported type is egress. Type ProxyGroupType `json:"type"` // Tags that the Tailscale devices will be tagged with. Defaults to [tag:k8s].