diff --git a/cmd/k8s-operator/connector.go b/cmd/k8s-operator/connector.go index 7fa311532..f4d518faa 100644 --- a/cmd/k8s-operator/connector.go +++ b/cmd/k8s-operator/connector.go @@ -25,6 +25,7 @@ import ( "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" + tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/kube/kubetypes" @@ -207,6 +208,7 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge ProxyClassName: proxyClass, proxyType: proxyTypeConnector, LoginServer: a.ssr.loginServer, + Tailnet: cn.Spec.Tailnet, } if cn.Spec.SubnetRouter != nil && len(cn.Spec.SubnetRouter.AdvertiseRoutes) > 0 { @@ -276,7 +278,7 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge } func (a *ConnectorReconciler) maybeCleanupConnector(ctx context.Context, logger *zap.SugaredLogger, cn *tsapi.Connector) (bool, error) { - if done, err := a.ssr.Cleanup(ctx, logger, childResourceLabels(cn.Name, a.tsnamespace, "connector"), proxyTypeConnector); err != nil { + if done, err := a.ssr.Cleanup(ctx, cn.Spec.Tailnet, logger, childResourceLabels(cn.Name, a.tsnamespace, "connector"), proxyTypeConnector); err != nil { return false, fmt.Errorf("failed to cleanup Connector resources: %w", err) } else if !done { logger.Debugf("Connector cleanup not done yet, waiting for next reconcile") diff --git a/cmd/k8s-operator/depaware.txt b/cmd/k8s-operator/depaware.txt index b809c85b9..f635dd99e 100644 --- a/cmd/k8s-operator/depaware.txt +++ b/cmd/k8s-operator/depaware.txt @@ -654,7 +654,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ k8s.io/utils/net from k8s.io/apimachinery/pkg/util/net+ k8s.io/utils/ptr from k8s.io/client-go/tools/cache+ k8s.io/utils/trace from k8s.io/client-go/tools/cache - sigs.k8s.io/controller-runtime/pkg/builder from tailscale.com/cmd/k8s-operator + sigs.k8s.io/controller-runtime/pkg/builder from tailscale.com/cmd/k8s-operator+ sigs.k8s.io/controller-runtime/pkg/cache from sigs.k8s.io/controller-runtime/pkg/cluster+ sigs.k8s.io/controller-runtime/pkg/cache/internal from sigs.k8s.io/controller-runtime/pkg/cache sigs.k8s.io/controller-runtime/pkg/certwatcher from sigs.k8s.io/controller-runtime/pkg/metrics/server+ @@ -750,10 +750,12 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ tailscale.com/ipn/store from tailscale.com/ipn/ipnlocal+ tailscale.com/ipn/store/kubestore from tailscale.com/cmd/k8s-operator tailscale.com/ipn/store/mem from tailscale.com/ipn/ipnlocal+ - tailscale.com/k8s-operator from tailscale.com/cmd/k8s-operator + tailscale.com/k8s-operator from tailscale.com/cmd/k8s-operator+ tailscale.com/k8s-operator/api-proxy from tailscale.com/cmd/k8s-operator tailscale.com/k8s-operator/apis from tailscale.com/k8s-operator/apis/v1alpha1 tailscale.com/k8s-operator/apis/v1alpha1 from tailscale.com/cmd/k8s-operator+ + tailscale.com/k8s-operator/reconciler from tailscale.com/k8s-operator/reconciler/tailnet + tailscale.com/k8s-operator/reconciler/tailnet from tailscale.com/cmd/k8s-operator tailscale.com/k8s-operator/sessionrecording from tailscale.com/k8s-operator/api-proxy tailscale.com/k8s-operator/sessionrecording/spdy from tailscale.com/k8s-operator/sessionrecording tailscale.com/k8s-operator/sessionrecording/tsrecorder from tailscale.com/k8s-operator/sessionrecording+ diff --git a/cmd/k8s-operator/deploy/chart/templates/.gitignore b/cmd/k8s-operator/deploy/chart/templates/.gitignore index ae7c682d9..f480bb57d 100644 --- a/cmd/k8s-operator/deploy/chart/templates/.gitignore +++ b/cmd/k8s-operator/deploy/chart/templates/.gitignore @@ -8,3 +8,4 @@ /proxyclass.yaml /proxygroup.yaml /recorder.yaml +/tailnet.yaml diff --git a/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml b/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml index 5eb920a6f..930eef852 100644 --- a/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml +++ b/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml @@ -37,6 +37,9 @@ rules: - apiGroups: ["tailscale.com"] resources: ["dnsconfigs", "dnsconfigs/status"] verbs: ["get", "list", "watch", "update"] +- apiGroups: ["tailscale.com"] + resources: ["tailnets", "tailnets/status"] + verbs: ["get", "list", "watch", "update"] - apiGroups: ["tailscale.com"] resources: ["recorders", "recorders/status"] verbs: ["get", "list", "watch", "update"] diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_connectors.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_connectors.yaml index 74d32d53d..03c51c755 100644 --- a/cmd/k8s-operator/deploy/crds/tailscale.com_connectors.yaml +++ b/cmd/k8s-operator/deploy/crds/tailscale.com_connectors.yaml @@ -181,6 +181,14 @@ spec: items: type: string pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$ + tailnet: + description: |- + Tailnet specifies the tailnet this Connector should join. If blank, the default tailnet is used. When set, this + name must match that of a valid Tailnet resource. This field is immutable and cannot be changed once set. + type: string + x-kubernetes-validations: + - rule: self == oldSelf + message: Connector tailnet is immutable x-kubernetes-validations: - rule: has(self.subnetRouter) || (has(self.exitNode) && self.exitNode == true) || has(self.appConnector) message: A Connector needs to have at least one of exit node, subnet router or app connector configured. diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml index 98ca1c378..0254f01b8 100644 --- a/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml +++ b/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml @@ -139,6 +139,14 @@ spec: items: type: string pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$ + tailnet: + description: |- + Tailnet specifies the tailnet this ProxyGroup should join. If blank, the default tailnet is used. When set, this + name must match that of a valid Tailnet resource. This field is immutable and cannot be changed once set. + type: string + x-kubernetes-validations: + - rule: self == oldSelf + message: ProxyGroup tailnet is immutable type: description: |- Type of the ProxyGroup proxies. Supported types are egress, ingress, and kube-apiserver. diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_recorders.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_recorders.yaml index 3d80c55e1..28d2be78e 100644 --- a/cmd/k8s-operator/deploy/crds/tailscale.com_recorders.yaml +++ b/cmd/k8s-operator/deploy/crds/tailscale.com_recorders.yaml @@ -1680,6 +1680,14 @@ spec: items: type: string pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$ + tailnet: + description: |- + Tailnet specifies the tailnet this Recorder should join. If blank, the default tailnet is used. When set, this + name must match that of a valid Tailnet resource. This field is immutable and cannot be changed once set. + type: string + x-kubernetes-validations: + - rule: self == oldSelf + message: Recorder tailnet is immutable x-kubernetes-validations: - rule: '!(self.replicas > 1 && (!has(self.storage) || !has(self.storage.s3)))' message: S3 storage must be used when deploying multiple Recorder replicas diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_tailnets.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_tailnets.yaml new file mode 100644 index 000000000..642fc3d4b --- /dev/null +++ b/cmd/k8s-operator/deploy/crds/tailscale.com_tailnets.yaml @@ -0,0 +1,137 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.0 + name: tailnets.tailscale.com +spec: + group: tailscale.com + names: + kind: Tailnet + listKind: TailnetList + plural: tailnets + shortNames: + - tn + singular: tailnet + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + type: object + required: + - metadata + - spec + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + Spec describes the desired state of the Tailnet. + More info: + https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + type: object + required: + - credentials + properties: + credentials: + description: Denotes the location of the OAuth credentials to use for authenticating with this Tailnet. + type: object + required: + - secretName + properties: + secretName: + description: |- + The name of the secret containing the OAuth credentials. This secret must contain two fields "client_id" and + "client_secret". + type: string + loginUrl: + description: URL of the control plane to be used by all resources managed by the operator using this Tailnet. + type: string + status: + description: |- + Status describes the status of the Tailnet. This is set + and managed by the Tailscale operator. + type: object + properties: + conditions: + type: array + items: + description: Condition contains details for one aspect of the current state of this API Resource. + type: object + required: + - lastTransitionTime + - message + - reason + - status + - type + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + type: string + format: date-time + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + type: string + maxLength: 32768 + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + type: integer + format: int64 + minimum: 0 + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + type: string + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + status: + description: status of the condition, one of True, False, Unknown. + type: string + enum: + - "True" + - "False" + - Unknown + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + type: string + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + served: true + storage: true + subresources: + status: {} diff --git a/cmd/k8s-operator/deploy/manifests/operator.yaml b/cmd/k8s-operator/deploy/manifests/operator.yaml index c53f50492..5e89a8dff 100644 --- a/cmd/k8s-operator/deploy/manifests/operator.yaml +++ b/cmd/k8s-operator/deploy/manifests/operator.yaml @@ -206,6 +206,14 @@ spec: pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$ type: string type: array + tailnet: + description: |- + Tailnet specifies the tailnet this Connector should join. If blank, the default tailnet is used. When set, this + name must match that of a valid Tailnet resource. This field is immutable and cannot be changed once set. + type: string + x-kubernetes-validations: + - message: Connector tailnet is immutable + rule: self == oldSelf type: object x-kubernetes-validations: - message: A Connector needs to have at least one of exit node, subnet router or app connector configured. @@ -3135,6 +3143,14 @@ spec: pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$ type: string type: array + tailnet: + description: |- + Tailnet specifies the tailnet this ProxyGroup should join. If blank, the default tailnet is used. When set, this + name must match that of a valid Tailnet resource. This field is immutable and cannot be changed once set. + type: string + x-kubernetes-validations: + - message: ProxyGroup tailnet is immutable + rule: self == oldSelf type: description: |- Type of the ProxyGroup proxies. Supported types are egress, ingress, and kube-apiserver. @@ -4950,6 +4966,14 @@ spec: pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$ type: string type: array + tailnet: + description: |- + Tailnet specifies the tailnet this Recorder should join. If blank, the default tailnet is used. When set, this + name must match that of a valid Tailnet resource. This field is immutable and cannot be changed once set. + type: string + x-kubernetes-validations: + - message: Recorder tailnet is immutable + rule: self == oldSelf type: object x-kubernetes-validations: - message: S3 storage must be used when deploying multiple Recorder replicas @@ -5059,6 +5083,144 @@ spec: subresources: status: {} --- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.0 + name: tailnets.tailscale.com +spec: + group: tailscale.com + names: + kind: Tailnet + listKind: TailnetList + plural: tailnets + shortNames: + - tn + singular: tailnet + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + Spec describes the desired state of the Tailnet. + More info: + https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + properties: + credentials: + description: Denotes the location of the OAuth credentials to use for authenticating with this Tailnet. + properties: + secretName: + description: |- + The name of the secret containing the OAuth credentials. This secret must contain two fields "client_id" and + "client_secret". + type: string + required: + - secretName + type: object + loginUrl: + description: URL of the control plane to be used by all resources managed by the operator using this Tailnet. + type: string + required: + - credentials + type: object + status: + description: |- + Status describes the status of the Tailnet. This is set + and managed by the Tailscale operator. + properties: + conditions: + items: + description: Condition contains details for one aspect of the current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -5141,6 +5303,16 @@ rules: - list - watch - update + - apiGroups: + - tailscale.com + resources: + - tailnets + - tailnets/status + verbs: + - get + - list + - watch + - update - apiGroups: - tailscale.com resources: diff --git a/cmd/k8s-operator/generate/main.go b/cmd/k8s-operator/generate/main.go index 08bdc350d..ca54e9090 100644 --- a/cmd/k8s-operator/generate/main.go +++ b/cmd/k8s-operator/generate/main.go @@ -26,12 +26,14 @@ const ( dnsConfigCRDPath = operatorDeploymentFilesPath + "/crds/tailscale.com_dnsconfigs.yaml" recorderCRDPath = operatorDeploymentFilesPath + "/crds/tailscale.com_recorders.yaml" proxyGroupCRDPath = operatorDeploymentFilesPath + "/crds/tailscale.com_proxygroups.yaml" + tailnetCRDPath = operatorDeploymentFilesPath + "/crds/tailscale.com_tailnets.yaml" helmTemplatesPath = operatorDeploymentFilesPath + "/chart/templates" connectorCRDHelmTemplatePath = helmTemplatesPath + "/connector.yaml" proxyClassCRDHelmTemplatePath = helmTemplatesPath + "/proxyclass.yaml" dnsConfigCRDHelmTemplatePath = helmTemplatesPath + "/dnsconfig.yaml" recorderCRDHelmTemplatePath = helmTemplatesPath + "/recorder.yaml" proxyGroupCRDHelmTemplatePath = helmTemplatesPath + "/proxygroup.yaml" + tailnetCRDHelmTemplatePath = helmTemplatesPath + "/tailnet.yaml" helmConditionalStart = "{{ if .Values.installCRDs -}}\n" helmConditionalEnd = "{{- end -}}" @@ -154,6 +156,7 @@ func generate(baseDir string) error { {dnsConfigCRDPath, dnsConfigCRDHelmTemplatePath}, {recorderCRDPath, recorderCRDHelmTemplatePath}, {proxyGroupCRDPath, proxyGroupCRDHelmTemplatePath}, + {tailnetCRDPath, tailnetCRDHelmTemplatePath}, } { if err := addCRDToHelm(crd.crdPath, crd.templatePath); err != nil { return fmt.Errorf("error adding %s CRD to Helm templates: %w", crd.crdPath, err) diff --git a/cmd/k8s-operator/ingress.go b/cmd/k8s-operator/ingress.go index 050b03f55..9ef173ece 100644 --- a/cmd/k8s-operator/ingress.go +++ b/cmd/k8s-operator/ingress.go @@ -102,7 +102,7 @@ func (a *IngressReconciler) maybeCleanup(ctx context.Context, logger *zap.Sugare return nil } - if done, err := a.ssr.Cleanup(ctx, logger, childResourceLabels(ing.Name, ing.Namespace, "ingress"), proxyTypeIngressResource); err != nil { + if done, err := a.ssr.Cleanup(ctx, operatorTailnet, logger, childResourceLabels(ing.Name, ing.Namespace, "ingress"), proxyTypeIngressResource); err != nil { return fmt.Errorf("failed to cleanup: %w", err) } else if !done { logger.Debugf("cleanup not done yet, waiting for next reconcile") diff --git a/cmd/k8s-operator/operator.go b/cmd/k8s-operator/operator.go index b50be8ce7..7bb8b95f0 100644 --- a/cmd/k8s-operator/operator.go +++ b/cmd/k8s-operator/operator.go @@ -54,6 +54,7 @@ import ( "tailscale.com/ipn/store/kubestore" apiproxy "tailscale.com/k8s-operator/api-proxy" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler/tailnet" "tailscale.com/kube/kubetypes" "tailscale.com/tsnet" "tailscale.com/tstime" @@ -325,6 +326,17 @@ func runReconcilers(opts reconcilerOpts) { startlog.Fatalf("could not create manager: %v", err) } + tailnetOptions := tailnet.ReconcilerOptions{ + Client: mgr.GetClient(), + TailscaleNamespace: opts.tailscaleNamespace, + Clock: tstime.DefaultClock{}, + Logger: opts.log, + } + + if err = tailnet.NewReconciler(tailnetOptions).Register(mgr); err != nil { + startlog.Fatalf("could not register tailnet reconciler: %v", err) + } + svcFilter := handler.EnqueueRequestsFromMapFunc(serviceHandler) svcChildFilter := handler.EnqueueRequestsFromMapFunc(managedResourceHandlerForType("svc")) // If a ProxyClass changes, enqueue all Services labeled with that diff --git a/cmd/k8s-operator/proxygroup.go b/cmd/k8s-operator/proxygroup.go index 946e017a2..3a50ed8fb 100644 --- a/cmd/k8s-operator/proxygroup.go +++ b/cmd/k8s-operator/proxygroup.go @@ -49,11 +49,12 @@ import ( ) const ( - reasonProxyGroupCreationFailed = "ProxyGroupCreationFailed" - reasonProxyGroupReady = "ProxyGroupReady" - reasonProxyGroupAvailable = "ProxyGroupAvailable" - reasonProxyGroupCreating = "ProxyGroupCreating" - reasonProxyGroupInvalid = "ProxyGroupInvalid" + reasonProxyGroupCreationFailed = "ProxyGroupCreationFailed" + reasonProxyGroupReady = "ProxyGroupReady" + reasonProxyGroupAvailable = "ProxyGroupAvailable" + reasonProxyGroupCreating = "ProxyGroupCreating" + reasonProxyGroupInvalid = "ProxyGroupInvalid" + reasonProxyGroupTailnetUnavailable = "ProxyGroupTailnetUnavailable" // Copied from k8s.io/apiserver/pkg/registry/generic/registry/store.go@cccad306d649184bf2a0e319ba830c53f65c445c optimisticLockErrorMsg = "the object has been modified; please apply your changes to the latest version and try again" @@ -117,6 +118,23 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ } else if err != nil { return reconcile.Result{}, fmt.Errorf("failed to get tailscale.com ProxyGroup: %w", err) } + + tailscaleClient := r.tsClient + if pg.Spec.Tailnet != "" { + tc, err := clientForTailnet(ctx, r.Client, r.tsNamespace, pg.Spec.Tailnet) + if err != nil { + oldPGStatus := pg.Status.DeepCopy() + nrr := ¬ReadyReason{ + reason: reasonProxyGroupTailnetUnavailable, + message: err.Error(), + } + + return reconcile.Result{}, errors.Join(err, r.maybeUpdateStatus(ctx, logger, pg, oldPGStatus, nrr, make(map[string][]netip.AddrPort))) + } + + tailscaleClient = tc + } + if markedForDeletion(pg) { logger.Debugf("ProxyGroup is being deleted, cleaning up resources") ix := xslices.Index(pg.Finalizers, FinalizerName) @@ -125,7 +143,7 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ return reconcile.Result{}, nil } - if done, err := r.maybeCleanup(ctx, pg); err != nil { + if done, err := r.maybeCleanup(ctx, tailscaleClient, pg); err != nil { if strings.Contains(err.Error(), optimisticLockErrorMsg) { logger.Infof("optimistic lock error, retrying: %s", err) return reconcile.Result{}, nil @@ -144,7 +162,7 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ } oldPGStatus := pg.Status.DeepCopy() - staticEndpoints, nrr, err := r.reconcilePG(ctx, pg, logger) + staticEndpoints, nrr, err := r.reconcilePG(ctx, tailscaleClient, pg, logger) return reconcile.Result{}, errors.Join(err, r.maybeUpdateStatus(ctx, logger, pg, oldPGStatus, nrr, staticEndpoints)) } @@ -152,7 +170,7 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ // for deletion. It is separated out from Reconcile to make a clear separation // between reconciling the ProxyGroup, and posting the status of its created // resources onto the ProxyGroup status field. -func (r *ProxyGroupReconciler) reconcilePG(ctx context.Context, pg *tsapi.ProxyGroup, logger *zap.SugaredLogger) (map[string][]netip.AddrPort, *notReadyReason, error) { +func (r *ProxyGroupReconciler) reconcilePG(ctx context.Context, tailscaleClient tsClient, pg *tsapi.ProxyGroup, logger *zap.SugaredLogger) (map[string][]netip.AddrPort, *notReadyReason, error) { if !slices.Contains(pg.Finalizers, FinalizerName) { // This log line is printed exactly once during initial provisioning, // because once the finalizer is in place this block gets skipped. So, @@ -193,7 +211,7 @@ func (r *ProxyGroupReconciler) reconcilePG(ctx context.Context, pg *tsapi.ProxyG return notReady(reasonProxyGroupInvalid, fmt.Sprintf("invalid ProxyGroup spec: %v", err)) } - staticEndpoints, nrr, err := r.maybeProvision(ctx, pg, proxyClass) + staticEndpoints, nrr, err := r.maybeProvision(ctx, tailscaleClient, pg, proxyClass) if err != nil { return nil, nrr, err } @@ -279,7 +297,7 @@ func (r *ProxyGroupReconciler) validate(ctx context.Context, pg *tsapi.ProxyGrou return errors.Join(errs...) } -func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.ProxyGroup, proxyClass *tsapi.ProxyClass) (map[string][]netip.AddrPort, *notReadyReason, error) { +func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, tailscaleClient tsClient, pg *tsapi.ProxyGroup, proxyClass *tsapi.ProxyClass) (map[string][]netip.AddrPort, *notReadyReason, error) { logger := r.logger(pg.Name) r.mu.Lock() r.ensureAddedToGaugeForProxyGroup(pg) @@ -302,7 +320,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro } } - staticEndpoints, err := r.ensureConfigSecretsCreated(ctx, pg, proxyClass, svcToNodePorts) + staticEndpoints, err := r.ensureConfigSecretsCreated(ctx, tailscaleClient, pg, proxyClass, svcToNodePorts) if err != nil { var selectorErr *FindStaticEndpointErr if errors.As(err, &selectorErr) { @@ -414,7 +432,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro return r.notReadyErrf(pg, logger, "error reconciling metrics resources: %w", err) } - if err := r.cleanupDanglingResources(ctx, pg, proxyClass); err != nil { + if err := r.cleanupDanglingResources(ctx, tailscaleClient, pg, proxyClass); err != nil { return r.notReadyErrf(pg, logger, "error cleaning up dangling resources: %w", err) } @@ -611,7 +629,7 @@ func (r *ProxyGroupReconciler) ensureNodePortServiceCreated(ctx context.Context, // cleanupDanglingResources ensures we don't leak config secrets, state secrets, and // tailnet devices when the number of replicas specified is reduced. -func (r *ProxyGroupReconciler) cleanupDanglingResources(ctx context.Context, pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass) error { +func (r *ProxyGroupReconciler) cleanupDanglingResources(ctx context.Context, tailscaleClient tsClient, pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass) error { logger := r.logger(pg.Name) metadata, err := r.getNodeMetadata(ctx, pg) if err != nil { @@ -625,7 +643,7 @@ func (r *ProxyGroupReconciler) cleanupDanglingResources(ctx context.Context, pg // Dangling resource, delete the config + state Secrets, as well as // deleting the device from the tailnet. - if err := r.deleteTailnetDevice(ctx, m.tsID, logger); err != nil { + if err := r.deleteTailnetDevice(ctx, tailscaleClient, m.tsID, logger); err != nil { return err } if err := r.Delete(ctx, m.stateSecret); err != nil && !apierrors.IsNotFound(err) { @@ -668,7 +686,7 @@ func (r *ProxyGroupReconciler) cleanupDanglingResources(ctx context.Context, pg // maybeCleanup just deletes the device from the tailnet. All the kubernetes // resources linked to a ProxyGroup will get cleaned up via owner references // (which we can use because they are all in the same namespace). -func (r *ProxyGroupReconciler) maybeCleanup(ctx context.Context, pg *tsapi.ProxyGroup) (bool, error) { +func (r *ProxyGroupReconciler) maybeCleanup(ctx context.Context, tailscaleClient tsClient, pg *tsapi.ProxyGroup) (bool, error) { logger := r.logger(pg.Name) metadata, err := r.getNodeMetadata(ctx, pg) @@ -677,7 +695,7 @@ func (r *ProxyGroupReconciler) maybeCleanup(ctx context.Context, pg *tsapi.Proxy } for _, m := range metadata { - if err := r.deleteTailnetDevice(ctx, m.tsID, logger); err != nil { + if err := r.deleteTailnetDevice(ctx, tailscaleClient, m.tsID, logger); err != nil { return false, err } } @@ -698,9 +716,9 @@ func (r *ProxyGroupReconciler) maybeCleanup(ctx context.Context, pg *tsapi.Proxy return true, nil } -func (r *ProxyGroupReconciler) deleteTailnetDevice(ctx context.Context, id tailcfg.StableNodeID, logger *zap.SugaredLogger) error { +func (r *ProxyGroupReconciler) deleteTailnetDevice(ctx context.Context, tailscaleClient tsClient, id tailcfg.StableNodeID, logger *zap.SugaredLogger) error { logger.Debugf("deleting device %s from control", string(id)) - if err := r.tsClient.DeleteDevice(ctx, string(id)); err != nil { + if err := tailscaleClient.DeleteDevice(ctx, string(id)); err != nil { errResp := &tailscale.ErrResponse{} if ok := errors.As(err, errResp); ok && errResp.Status == http.StatusNotFound { logger.Debugf("device %s not found, likely because it has already been deleted from control", string(id)) @@ -714,7 +732,13 @@ func (r *ProxyGroupReconciler) deleteTailnetDevice(ctx context.Context, id tailc return nil } -func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, pg *tsapi.ProxyGroup, proxyClass *tsapi.ProxyClass, svcToNodePorts map[string]uint16) (endpoints map[string][]netip.AddrPort, err error) { +func (r *ProxyGroupReconciler) ensureConfigSecretsCreated( + ctx context.Context, + tailscaleClient tsClient, + pg *tsapi.ProxyGroup, + proxyClass *tsapi.ProxyClass, + svcToNodePorts map[string]uint16, +) (endpoints map[string][]netip.AddrPort, err error) { logger := r.logger(pg.Name) endpoints = make(map[string][]netip.AddrPort, pgReplicas(pg)) // keyed by Service name. for i := range pgReplicas(pg) { @@ -728,7 +752,7 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p } var existingCfgSecret *corev1.Secret // unmodified copy of secret - if err := r.Get(ctx, client.ObjectKeyFromObject(cfgSecret), cfgSecret); err == nil { + if err = r.Get(ctx, client.ObjectKeyFromObject(cfgSecret), cfgSecret); err == nil { logger.Debugf("Secret %s/%s already exists", cfgSecret.GetNamespace(), cfgSecret.GetName()) existingCfgSecret = cfgSecret.DeepCopy() } else if !apierrors.IsNotFound(err) { @@ -742,7 +766,7 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p if len(tags) == 0 { tags = r.defaultTags } - key, err := newAuthKey(ctx, r.tsClient, tags) + key, err := newAuthKey(ctx, tailscaleClient, tags) if err != nil { return nil, err } @@ -757,7 +781,7 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p Namespace: r.tsNamespace, }, } - if err := r.Get(ctx, client.ObjectKeyFromObject(stateSecret), stateSecret); err != nil && !apierrors.IsNotFound(err) { + if err = r.Get(ctx, client.ObjectKeyFromObject(stateSecret), stateSecret); err != nil && !apierrors.IsNotFound(err) { return nil, err } diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go index 2b6d1290e..2919e535c 100644 --- a/cmd/k8s-operator/sts.go +++ b/cmd/k8s-operator/sts.go @@ -107,6 +107,7 @@ const ( letsEncryptStagingEndpoint = "https://acme-staging-v02.api.letsencrypt.org/directory" mainContainerName = "tailscale" + operatorTailnet = "" ) var ( @@ -152,6 +153,9 @@ type tailscaleSTSConfig struct { // HostnamePrefix specifies the desired prefix for the device's hostname. The hostname will be suffixed with the // ordinal number generated by the StatefulSet. HostnamePrefix string + + // Tailnet specifies the Tailnet resource to use for producing auth keys. + Tailnet string } type connector struct { @@ -194,6 +198,16 @@ func IsHTTPSEnabledOnTailnet(tsnetServer tsnetServer) bool { // Provision ensures that the StatefulSet for the given service is running and // up to date. func (a *tailscaleSTSReconciler) Provision(ctx context.Context, logger *zap.SugaredLogger, sts *tailscaleSTSConfig) (*corev1.Service, error) { + tailscaleClient := a.tsClient + if sts.Tailnet != "" { + tc, err := clientForTailnet(ctx, a.Client, a.operatorNamespace, sts.Tailnet) + if err != nil { + return nil, err + } + + tailscaleClient = tc + } + // Do full reconcile. // TODO (don't create Service for the Connector) hsvc, err := a.reconcileHeadlessService(ctx, logger, sts) @@ -213,7 +227,7 @@ func (a *tailscaleSTSReconciler) Provision(ctx context.Context, logger *zap.Suga } sts.ProxyClass = proxyClass - secretNames, err := a.provisionSecrets(ctx, logger, sts, hsvc) + secretNames, err := a.provisionSecrets(ctx, tailscaleClient, logger, sts, hsvc) if err != nil { return nil, fmt.Errorf("failed to create or get API key secret: %w", err) } @@ -237,7 +251,18 @@ func (a *tailscaleSTSReconciler) Provision(ctx context.Context, logger *zap.Suga // Cleanup removes all resources associated that were created by Provision with // the given labels. It returns true when all resources have been removed, // otherwise it returns false and the caller should retry later. -func (a *tailscaleSTSReconciler) Cleanup(ctx context.Context, logger *zap.SugaredLogger, labels map[string]string, typ string) (done bool, _ error) { +func (a *tailscaleSTSReconciler) Cleanup(ctx context.Context, tailnet string, logger *zap.SugaredLogger, labels map[string]string, typ string) (done bool, _ error) { + tailscaleClient := a.tsClient + if tailnet != "" { + tc, err := clientForTailnet(ctx, a.Client, a.operatorNamespace, tailnet) + if err != nil { + logger.Errorf("failed to get tailscale client: %v", err) + return false, nil + } + + tailscaleClient = tc + } + // Need to delete the StatefulSet first, and delete it with foreground // cascading deletion. That way, the pod that's writing to the Secret will // stop running before we start looking at the Secret's contents, and @@ -279,7 +304,7 @@ func (a *tailscaleSTSReconciler) Cleanup(ctx context.Context, logger *zap.Sugare for _, dev := range devices { if dev.id != "" { logger.Debugf("deleting device %s from control", string(dev.id)) - if err = a.tsClient.DeleteDevice(ctx, string(dev.id)); err != nil { + if err = tailscaleClient.DeleteDevice(ctx, string(dev.id)); err != nil { errResp := &tailscale.ErrResponse{} if ok := errors.As(err, errResp); ok && errResp.Status == http.StatusNotFound { logger.Debugf("device %s not found, likely because it has already been deleted from control", string(dev.id)) @@ -360,7 +385,7 @@ func (a *tailscaleSTSReconciler) reconcileHeadlessService(ctx context.Context, l return createOrUpdate(ctx, a.Client, a.operatorNamespace, hsvc, func(svc *corev1.Service) { svc.Spec = hsvc.Spec }) } -func (a *tailscaleSTSReconciler) provisionSecrets(ctx context.Context, logger *zap.SugaredLogger, stsC *tailscaleSTSConfig, hsvc *corev1.Service) ([]string, error) { +func (a *tailscaleSTSReconciler) provisionSecrets(ctx context.Context, tailscaleClient tsClient, logger *zap.SugaredLogger, stsC *tailscaleSTSConfig, hsvc *corev1.Service) ([]string, error) { secretNames := make([]string, stsC.Replicas) // Start by ensuring we have Secrets for the desired number of replicas. This will handle both creating and scaling @@ -403,7 +428,7 @@ func (a *tailscaleSTSReconciler) provisionSecrets(ctx context.Context, logger *z if len(tags) == 0 { tags = a.defaultTags } - authKey, err = newAuthKey(ctx, a.tsClient, tags) + authKey, err = newAuthKey(ctx, tailscaleClient, tags) if err != nil { return nil, err } @@ -477,7 +502,7 @@ func (a *tailscaleSTSReconciler) provisionSecrets(ctx context.Context, logger *z if dev != nil && dev.id != "" { var errResp *tailscale.ErrResponse - err = a.tsClient.DeleteDevice(ctx, string(dev.id)) + err = tailscaleClient.DeleteDevice(ctx, string(dev.id)) switch { case errors.As(err, &errResp) && errResp.Status == http.StatusNotFound: // This device has possibly already been deleted in the admin console. So we can ignore this diff --git a/cmd/k8s-operator/svc.go b/cmd/k8s-operator/svc.go index 5c163e081..ec7bb8080 100644 --- a/cmd/k8s-operator/svc.go +++ b/cmd/k8s-operator/svc.go @@ -23,6 +23,7 @@ import ( "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" + tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/kube/kubetypes" @@ -167,7 +168,7 @@ func (a *ServiceReconciler) maybeCleanup(ctx context.Context, logger *zap.Sugare proxyTyp = proxyTypeIngressService } - if done, err := a.ssr.Cleanup(ctx, logger, childResourceLabels(svc.Name, svc.Namespace, "svc"), proxyTyp); err != nil { + if done, err := a.ssr.Cleanup(ctx, operatorTailnet, logger, childResourceLabels(svc.Name, svc.Namespace, "svc"), proxyTyp); err != nil { return fmt.Errorf("failed to cleanup: %w", err) } else if !done { logger.Debugf("cleanup not done yet, waiting for next reconcile") diff --git a/cmd/k8s-operator/tailnet.go b/cmd/k8s-operator/tailnet.go new file mode 100644 index 000000000..3226ab023 --- /dev/null +++ b/cmd/k8s-operator/tailnet.go @@ -0,0 +1,58 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package main + +import ( + "context" + "fmt" + + "golang.org/x/oauth2" + "golang.org/x/oauth2/clientcredentials" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "tailscale.com/internal/client/tailscale" + "tailscale.com/ipn" + operatorutils "tailscale.com/k8s-operator" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" +) + +func clientForTailnet(ctx context.Context, cl client.Client, namespace, name string) (tsClient, error) { + var tn tsapi.Tailnet + if err := cl.Get(ctx, client.ObjectKey{Name: name}, &tn); err != nil { + return nil, fmt.Errorf("failed to get Tailnet %q: %w", name, err) + } + + if !operatorutils.TailnetIsReady(&tn) { + return nil, fmt.Errorf("tailnet %q is not ready", name) + } + + var secret corev1.Secret + if err := cl.Get(ctx, client.ObjectKey{Name: tn.Spec.Credentials.SecretName, Namespace: namespace}, &secret); err != nil { + return nil, fmt.Errorf("failed to get Secret %q in namespace %q: %w", tn.Spec.Credentials.SecretName, namespace, err) + } + + baseURL := ipn.DefaultControlURL + if tn.Spec.LoginURL != "" { + baseURL = tn.Spec.LoginURL + } + + credentials := clientcredentials.Config{ + ClientID: string(secret.Data["client_id"]), + ClientSecret: string(secret.Data["client_secret"]), + TokenURL: baseURL + "/api/v2/oauth/token", + } + + source := credentials.TokenSource(ctx) + httpClient := oauth2.NewClient(ctx, source) + + ts := tailscale.NewClient(defaultTailnet, nil) + ts.UserAgent = "tailscale-k8s-operator" + ts.HTTPClient = httpClient + ts.BaseURL = baseURL + + return ts, nil +} diff --git a/cmd/k8s-operator/tsrecorder.go b/cmd/k8s-operator/tsrecorder.go index bfb01fa86..3e8608bc8 100644 --- a/cmd/k8s-operator/tsrecorder.go +++ b/cmd/k8s-operator/tsrecorder.go @@ -42,10 +42,11 @@ import ( ) const ( - reasonRecorderCreationFailed = "RecorderCreationFailed" - reasonRecorderCreating = "RecorderCreating" - reasonRecorderCreated = "RecorderCreated" - reasonRecorderInvalid = "RecorderInvalid" + reasonRecorderCreationFailed = "RecorderCreationFailed" + reasonRecorderCreating = "RecorderCreating" + reasonRecorderCreated = "RecorderCreated" + reasonRecorderInvalid = "RecorderInvalid" + reasonRecorderTailnetUnavailable = "RecorderTailnetUnavailable" currentProfileKey = "_current-profile" ) @@ -84,6 +85,30 @@ func (r *RecorderReconciler) Reconcile(ctx context.Context, req reconcile.Reques } else if err != nil { return reconcile.Result{}, fmt.Errorf("failed to get tailscale.com Recorder: %w", err) } + + oldTSRStatus := tsr.Status.DeepCopy() + setStatusReady := func(tsr *tsapi.Recorder, status metav1.ConditionStatus, reason, message string) (reconcile.Result, error) { + tsoperator.SetRecorderCondition(tsr, tsapi.RecorderReady, status, reason, message, tsr.Generation, r.clock, logger) + if !apiequality.Semantic.DeepEqual(oldTSRStatus, &tsr.Status) { + // An error encountered here should get returned by the Reconcile function. + if updateErr := r.Client.Status().Update(ctx, tsr); updateErr != nil { + return reconcile.Result{}, errors.Join(err, updateErr) + } + } + + return reconcile.Result{}, nil + } + + tailscaleClient := r.tsClient + if tsr.Spec.Tailnet != "" { + tc, err := clientForTailnet(ctx, r.Client, r.tsNamespace, tsr.Spec.Tailnet) + if err != nil { + return setStatusReady(tsr, metav1.ConditionFalse, reasonRecorderTailnetUnavailable, err.Error()) + } + + tailscaleClient = tc + } + if markedForDeletion(tsr) { logger.Debugf("Recorder is being deleted, cleaning up resources") ix := xslices.Index(tsr.Finalizers, FinalizerName) @@ -92,7 +117,7 @@ func (r *RecorderReconciler) Reconcile(ctx context.Context, req reconcile.Reques return reconcile.Result{}, nil } - if done, err := r.maybeCleanup(ctx, tsr); err != nil { + if done, err := r.maybeCleanup(ctx, tsr, tailscaleClient); err != nil { return reconcile.Result{}, err } else if !done { logger.Debugf("Recorder resource cleanup not yet finished, will retry...") @@ -106,19 +131,6 @@ func (r *RecorderReconciler) Reconcile(ctx context.Context, req reconcile.Reques return reconcile.Result{}, nil } - oldTSRStatus := tsr.Status.DeepCopy() - setStatusReady := func(tsr *tsapi.Recorder, status metav1.ConditionStatus, reason, message string) (reconcile.Result, error) { - tsoperator.SetRecorderCondition(tsr, tsapi.RecorderReady, status, reason, message, tsr.Generation, r.clock, logger) - if !apiequality.Semantic.DeepEqual(oldTSRStatus, &tsr.Status) { - // An error encountered here should get returned by the Reconcile function. - if updateErr := r.Client.Status().Update(ctx, tsr); updateErr != nil { - return reconcile.Result{}, errors.Join(err, updateErr) - } - } - - return reconcile.Result{}, nil - } - if !slices.Contains(tsr.Finalizers, FinalizerName) { // This log line is printed exactly once during initial provisioning, // because once the finalizer is in place this block gets skipped. So, @@ -137,7 +149,7 @@ func (r *RecorderReconciler) Reconcile(ctx context.Context, req reconcile.Reques return setStatusReady(tsr, metav1.ConditionFalse, reasonRecorderInvalid, message) } - if err = r.maybeProvision(ctx, tsr); err != nil { + if err = r.maybeProvision(ctx, tailscaleClient, tsr); err != nil { reason := reasonRecorderCreationFailed message := fmt.Sprintf("failed creating Recorder: %s", err) if strings.Contains(err.Error(), optimisticLockErrorMsg) { @@ -155,7 +167,7 @@ func (r *RecorderReconciler) Reconcile(ctx context.Context, req reconcile.Reques return setStatusReady(tsr, metav1.ConditionTrue, reasonRecorderCreated, reasonRecorderCreated) } -func (r *RecorderReconciler) maybeProvision(ctx context.Context, tsr *tsapi.Recorder) error { +func (r *RecorderReconciler) maybeProvision(ctx context.Context, tailscaleClient tsClient, tsr *tsapi.Recorder) error { logger := r.logger(tsr.Name) r.mu.Lock() @@ -163,7 +175,7 @@ func (r *RecorderReconciler) maybeProvision(ctx context.Context, tsr *tsapi.Reco gaugeRecorderResources.Set(int64(r.recorders.Len())) r.mu.Unlock() - if err := r.ensureAuthSecretsCreated(ctx, tsr); err != nil { + if err := r.ensureAuthSecretsCreated(ctx, tailscaleClient, tsr); err != nil { return fmt.Errorf("error creating secrets: %w", err) } @@ -241,13 +253,13 @@ func (r *RecorderReconciler) maybeProvision(ctx context.Context, tsr *tsapi.Reco // If we have scaled the recorder down, we will have dangling state secrets // that we need to clean up. - if err = r.maybeCleanupSecrets(ctx, tsr); err != nil { + if err = r.maybeCleanupSecrets(ctx, tailscaleClient, tsr); err != nil { return fmt.Errorf("error cleaning up Secrets: %w", err) } var devices []tsapi.RecorderTailnetDevice for replica := range replicas { - dev, ok, err := r.getDeviceInfo(ctx, tsr.Name, replica) + dev, ok, err := r.getDeviceInfo(ctx, tailscaleClient, tsr.Name, replica) switch { case err != nil: return fmt.Errorf("failed to get device info: %w", err) @@ -312,7 +324,7 @@ func (r *RecorderReconciler) maybeCleanupServiceAccounts(ctx context.Context, ts return nil } -func (r *RecorderReconciler) maybeCleanupSecrets(ctx context.Context, tsr *tsapi.Recorder) error { +func (r *RecorderReconciler) maybeCleanupSecrets(ctx context.Context, tailscaleClient tsClient, tsr *tsapi.Recorder) error { options := []client.ListOption{ client.InNamespace(r.tsNamespace), client.MatchingLabels(tsrLabels("recorder", tsr.Name, nil)), @@ -354,7 +366,7 @@ func (r *RecorderReconciler) maybeCleanupSecrets(ctx context.Context, tsr *tsapi var errResp *tailscale.ErrResponse r.log.Debugf("deleting device %s", devicePrefs.Config.NodeID) - err = r.tsClient.DeleteDevice(ctx, string(devicePrefs.Config.NodeID)) + err = tailscaleClient.DeleteDevice(ctx, string(devicePrefs.Config.NodeID)) switch { case errors.As(err, &errResp) && errResp.Status == http.StatusNotFound: // This device has possibly already been deleted in the admin console. So we can ignore this @@ -375,7 +387,7 @@ func (r *RecorderReconciler) maybeCleanupSecrets(ctx context.Context, tsr *tsapi // maybeCleanup just deletes the device from the tailnet. All the kubernetes // resources linked to a Recorder will get cleaned up via owner references // (which we can use because they are all in the same namespace). -func (r *RecorderReconciler) maybeCleanup(ctx context.Context, tsr *tsapi.Recorder) (bool, error) { +func (r *RecorderReconciler) maybeCleanup(ctx context.Context, tsr *tsapi.Recorder, tailscaleClient tsClient) (bool, error) { logger := r.logger(tsr.Name) var replicas int32 = 1 @@ -399,7 +411,7 @@ func (r *RecorderReconciler) maybeCleanup(ctx context.Context, tsr *tsapi.Record nodeID := string(devicePrefs.Config.NodeID) logger.Debugf("deleting device %s from control", nodeID) - if err = r.tsClient.DeleteDevice(ctx, nodeID); err != nil { + if err = tailscaleClient.DeleteDevice(ctx, nodeID); err != nil { errResp := &tailscale.ErrResponse{} if errors.As(err, errResp) && errResp.Status == http.StatusNotFound { logger.Debugf("device %s not found, likely because it has already been deleted from control", nodeID) @@ -425,7 +437,7 @@ func (r *RecorderReconciler) maybeCleanup(ctx context.Context, tsr *tsapi.Record return true, nil } -func (r *RecorderReconciler) ensureAuthSecretsCreated(ctx context.Context, tsr *tsapi.Recorder) error { +func (r *RecorderReconciler) ensureAuthSecretsCreated(ctx context.Context, tailscaleClient tsClient, tsr *tsapi.Recorder) error { var replicas int32 = 1 if tsr.Spec.Replicas != nil { replicas = *tsr.Spec.Replicas @@ -453,7 +465,7 @@ func (r *RecorderReconciler) ensureAuthSecretsCreated(ctx context.Context, tsr * return fmt.Errorf("failed to get Secret %q: %w", key.Name, err) } - authKey, err := newAuthKey(ctx, r.tsClient, tags.Stringify()) + authKey, err := newAuthKey(ctx, tailscaleClient, tags.Stringify()) if err != nil { return err } @@ -555,7 +567,7 @@ func getDevicePrefs(secret *corev1.Secret) (prefs prefs, ok bool, err error) { return prefs, ok, nil } -func (r *RecorderReconciler) getDeviceInfo(ctx context.Context, tsrName string, replica int32) (d tsapi.RecorderTailnetDevice, ok bool, err error) { +func (r *RecorderReconciler) getDeviceInfo(ctx context.Context, tailscaleClient tsClient, tsrName string, replica int32) (d tsapi.RecorderTailnetDevice, ok bool, err error) { secret, err := r.getStateSecret(ctx, tsrName, replica) if err != nil || secret == nil { return tsapi.RecorderTailnetDevice{}, false, err @@ -569,7 +581,7 @@ func (r *RecorderReconciler) getDeviceInfo(ctx context.Context, tsrName string, // TODO(tomhjp): The profile info doesn't include addresses, which is why we // need the API. Should maybe update tsrecorder to write IPs to the state // Secret like containerboot does. - device, err := r.tsClient.Device(ctx, string(prefs.Config.NodeID), nil) + device, err := tailscaleClient.Device(ctx, string(prefs.Config.NodeID), nil) if err != nil { return tsapi.RecorderTailnetDevice{}, false, fmt.Errorf("failed to get device info from API: %w", err) } diff --git a/k8s-operator/api.md b/k8s-operator/api.md index 3a4e692d9..31f351013 100644 --- a/k8s-operator/api.md +++ b/k8s-operator/api.md @@ -18,6 +18,8 @@ - [ProxyGroupList](#proxygrouplist) - [Recorder](#recorder) - [RecorderList](#recorderlist) +- [Tailnet](#tailnet) +- [TailnetList](#tailnetlist) @@ -139,6 +141,7 @@ _Appears in:_ | `appConnector` _[AppConnector](#appconnector)_ | AppConnector defines whether the Connector device should act as a Tailscale app connector. A Connector that is
configured as an app connector cannot be a subnet router or an exit node. If this field is unset, the
Connector does not act as an app connector.
Note that you will need to manually configure the permissions and the domains for the app connector via the
Admin panel.
Note also that the main tested and supported use case of this config option is to deploy an app connector on
Kubernetes to access SaaS applications available on the public internet. Using the app connector to expose
cluster workloads or other internal workloads to tailnet might work, but this is not a use case that we have
tested or optimised for.
If you are using the app connector to access SaaS applications because you need a predictable egress IP that
can be whitelisted, it is also your responsibility to ensure that cluster traffic from the connector flows
via that predictable IP, for example by enforcing that cluster egress traffic is routed via an egress NAT
device with a static IP address.
https://tailscale.com/kb/1281/app-connectors | | | | `exitNode` _boolean_ | ExitNode defines whether the Connector device should act as a Tailscale exit node. Defaults to false.
This field is mutually exclusive with the appConnector field.
https://tailscale.com/kb/1103/exit-nodes | | | | `replicas` _integer_ | Replicas specifies how many devices to create. Set this to enable
high availability for app connectors, subnet routers, or exit nodes.
https://tailscale.com/kb/1115/high-availability. Defaults to 1. | | Minimum: 0
| +| `tailnet` _string_ | Tailnet specifies the tailnet this Connector should join. If blank, the default tailnet is used. When set, this
name must match that of a valid Tailnet resource. This field is immutable and cannot be changed once set. | | | #### ConnectorStatus @@ -741,6 +744,7 @@ _Appears in:_ | `hostnamePrefix` _[HostnamePrefix](#hostnameprefix)_ | HostnamePrefix is the hostname prefix to use for tailnet devices created
by the ProxyGroup. Each device will have the integer number from its
StatefulSet pod appended to this prefix to form the full hostname.
HostnamePrefix can contain lower case letters, numbers and dashes, it
must not start with a dash and must be between 1 and 62 characters long. | | Pattern: `^[a-z0-9][a-z0-9-]{0,61}$`
Type: string
| | `proxyClass` _string_ | ProxyClass is the name of the ProxyClass custom resource that contains
configuration options that should be applied to the resources created
for this ProxyGroup. If unset, and there is no default ProxyClass
configured, the operator will create resources with the default
configuration. | | | | `kubeAPIServer` _[KubeAPIServerConfig](#kubeapiserverconfig)_ | KubeAPIServer contains configuration specific to the kube-apiserver
ProxyGroup type. This field is only used when Type is set to "kube-apiserver". | | | +| `tailnet` _string_ | Tailnet specifies the tailnet this ProxyGroup should join. If blank, the default tailnet is used. When set, this
name must match that of a valid Tailnet resource. This field is immutable and cannot be changed once set. | | | #### ProxyGroupStatus @@ -901,6 +905,7 @@ _Appears in:_ | `enableUI` _boolean_ | Set to true to enable the Recorder UI. The UI lists and plays recorded sessions.
The UI will be served at :443. Defaults to false.
Corresponds to --ui tsrecorder flag https://tailscale.com/kb/1246/tailscale-ssh-session-recording#deploy-a-recorder-node.
Required if S3 storage is not set up, to ensure that recordings are accessible. | | | | `storage` _[Storage](#storage)_ | Configure where to store session recordings. By default, recordings will
be stored in a local ephemeral volume, and will not be persisted past the
lifetime of a specific pod. | | | | `replicas` _integer_ | Replicas specifies how many instances of tsrecorder to run. Defaults to 1. | | Minimum: 0
| +| `tailnet` _string_ | Tailnet specifies the tailnet this Recorder should join. If blank, the default tailnet is used. When set, this
name must match that of a valid Tailnet resource. This field is immutable and cannot be changed once set. | | | #### RecorderStatefulSet @@ -1154,6 +1159,44 @@ _Appears in:_ +#### Tailnet + + + + + + + +_Appears in:_ +- [TailnetList](#tailnetlist) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `apiVersion` _string_ | `tailscale.com/v1alpha1` | | | +| `kind` _string_ | `Tailnet` | | | +| `kind` _string_ | Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | | | +| `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources | | | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `spec` _[TailnetSpec](#tailnetspec)_ | Spec describes the desired state of the Tailnet.
More info:
https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status | | | +| `status` _[TailnetStatus](#tailnetstatus)_ | Status describes the status of the Tailnet. This is set
and managed by the Tailscale operator. | | | + + +#### TailnetCredentials + + + + + + + +_Appears in:_ +- [TailnetSpec](#tailnetspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `secretName` _string_ | The name of the secret containing the OAuth credentials. This secret must contain two fields "client_id" and
"client_secret". | | | + + #### TailnetDevice @@ -1172,6 +1215,59 @@ _Appears in:_ | `staticEndpoints` _string array_ | StaticEndpoints are user configured, 'static' endpoints by which tailnet peers can reach this device. | | | +#### TailnetList + + + + + + + + + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `apiVersion` _string_ | `tailscale.com/v1alpha1` | | | +| `kind` _string_ | `TailnetList` | | | +| `kind` _string_ | Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | | | +| `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources | | | +| `metadata` _[ListMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#listmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `items` _[Tailnet](#tailnet) array_ | | | | + + +#### TailnetSpec + + + + + + + +_Appears in:_ +- [Tailnet](#tailnet) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `loginUrl` _string_ | URL of the control plane to be used by all resources managed by the operator using this Tailnet. | | | +| `credentials` _[TailnetCredentials](#tailnetcredentials)_ | Denotes the location of the OAuth credentials to use for authenticating with this Tailnet. | | | + + +#### TailnetStatus + + + + + + + +_Appears in:_ +- [Tailnet](#tailnet) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#condition-v1-meta) array_ | | | | + + #### TailscaleConfig diff --git a/k8s-operator/apis/v1alpha1/register.go b/k8s-operator/apis/v1alpha1/register.go index 0880ac975..993a119fa 100644 --- a/k8s-operator/apis/v1alpha1/register.go +++ b/k8s-operator/apis/v1alpha1/register.go @@ -67,6 +67,8 @@ func addKnownTypes(scheme *runtime.Scheme) error { &RecorderList{}, &ProxyGroup{}, &ProxyGroupList{}, + &Tailnet{}, + &TailnetList{}, ) metav1.AddToGroupVersion(scheme, SchemeGroupVersion) diff --git a/k8s-operator/apis/v1alpha1/types_connector.go b/k8s-operator/apis/v1alpha1/types_connector.go index 58457500f..ebedea18f 100644 --- a/k8s-operator/apis/v1alpha1/types_connector.go +++ b/k8s-operator/apis/v1alpha1/types_connector.go @@ -133,6 +133,12 @@ type ConnectorSpec struct { // +optional // +kubebuilder:validation:Minimum=0 Replicas *int32 `json:"replicas,omitempty"` + + // Tailnet specifies the tailnet this Connector should join. If blank, the default tailnet is used. When set, this + // name must match that of a valid Tailnet resource. This field is immutable and cannot be changed once set. + // +optional + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Connector tailnet is immutable" + Tailnet string `json:"tailnet,omitempty"` } // SubnetRouter defines subnet routes that should be exposed to tailnet via a diff --git a/k8s-operator/apis/v1alpha1/types_proxygroup.go b/k8s-operator/apis/v1alpha1/types_proxygroup.go index 28fd9e009..8cbcc2d19 100644 --- a/k8s-operator/apis/v1alpha1/types_proxygroup.go +++ b/k8s-operator/apis/v1alpha1/types_proxygroup.go @@ -97,6 +97,12 @@ type ProxyGroupSpec struct { // ProxyGroup type. This field is only used when Type is set to "kube-apiserver". // +optional KubeAPIServer *KubeAPIServerConfig `json:"kubeAPIServer,omitempty"` + + // Tailnet specifies the tailnet this ProxyGroup should join. If blank, the default tailnet is used. When set, this + // name must match that of a valid Tailnet resource. This field is immutable and cannot be changed once set. + // +optional + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="ProxyGroup tailnet is immutable" + Tailnet string `json:"tailnet,omitempty"` } type ProxyGroupStatus struct { diff --git a/k8s-operator/apis/v1alpha1/types_recorder.go b/k8s-operator/apis/v1alpha1/types_recorder.go index 67cffbf09..d5a22e82c 100644 --- a/k8s-operator/apis/v1alpha1/types_recorder.go +++ b/k8s-operator/apis/v1alpha1/types_recorder.go @@ -81,6 +81,12 @@ type RecorderSpec struct { // +optional // +kubebuilder:validation:Minimum=0 Replicas *int32 `json:"replicas,omitzero"` + + // Tailnet specifies the tailnet this Recorder should join. If blank, the default tailnet is used. When set, this + // name must match that of a valid Tailnet resource. This field is immutable and cannot be changed once set. + // +optional + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Recorder tailnet is immutable" + Tailnet string `json:"tailnet,omitempty"` } type RecorderStatefulSet struct { diff --git a/k8s-operator/apis/v1alpha1/types_tailnet.go b/k8s-operator/apis/v1alpha1/types_tailnet.go new file mode 100644 index 000000000..921d9bab5 --- /dev/null +++ b/k8s-operator/apis/v1alpha1/types_tailnet.go @@ -0,0 +1,68 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// Code comments on these types should be treated as user facing documentation- +// they will appear on the Tailnet CRD i.e. if someone runs kubectl explain tailnet. + +var TailnetKind = "Tailnet" + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:scope=Cluster,shortName=tn +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" + +type Tailnet struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitzero"` + + // Spec describes the desired state of the Tailnet. + // More info: + // https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + Spec TailnetSpec `json:"spec"` + + // Status describes the status of the Tailnet. This is set + // and managed by the Tailscale operator. + // +optional + Status TailnetStatus `json:"status"` +} + +// +kubebuilder:object:root=true + +type TailnetList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata"` + + Items []Tailnet `json:"items"` +} + +type TailnetSpec struct { + // URL of the control plane to be used by all resources managed by the operator using this Tailnet. + // +optional + LoginURL string `json:"loginUrl,omitempty"` + // Denotes the location of the OAuth credentials to use for authenticating with this Tailnet. + Credentials TailnetCredentials `json:"credentials"` +} + +type TailnetCredentials struct { + // The name of the secret containing the OAuth credentials. This secret must contain two fields "client_id" and + // "client_secret". + SecretName string `json:"secretName"` +} + +type TailnetStatus struct { + // +listType=map + // +listMapKey=type + // +optional + Conditions []metav1.Condition `json:"conditions"` +} + +// TailnetReady is set to True if the Tailnet is available for use by operator workloads. +const TailnetReady ConditionType = `TailnetReady` diff --git a/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go b/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go index ff0f3f6ac..4743a5156 100644 --- a/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go +++ b/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go @@ -1365,6 +1365,48 @@ func (in Tags) DeepCopy() Tags { return *out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Tailnet) DeepCopyInto(out *Tailnet) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Tailnet. +func (in *Tailnet) DeepCopy() *Tailnet { + if in == nil { + return nil + } + out := new(Tailnet) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Tailnet) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TailnetCredentials) DeepCopyInto(out *TailnetCredentials) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TailnetCredentials. +func (in *TailnetCredentials) DeepCopy() *TailnetCredentials { + if in == nil { + return nil + } + out := new(TailnetCredentials) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TailnetDevice) DeepCopyInto(out *TailnetDevice) { *out = *in @@ -1390,6 +1432,76 @@ func (in *TailnetDevice) DeepCopy() *TailnetDevice { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TailnetList) DeepCopyInto(out *TailnetList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Tailnet, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TailnetList. +func (in *TailnetList) DeepCopy() *TailnetList { + if in == nil { + return nil + } + out := new(TailnetList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *TailnetList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TailnetSpec) DeepCopyInto(out *TailnetSpec) { + *out = *in + out.Credentials = in.Credentials +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TailnetSpec. +func (in *TailnetSpec) DeepCopy() *TailnetSpec { + if in == nil { + return nil + } + out := new(TailnetSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TailnetStatus) DeepCopyInto(out *TailnetStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TailnetStatus. +func (in *TailnetStatus) DeepCopy() *TailnetStatus { + if in == nil { + return nil + } + out := new(TailnetStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TailscaleConfig) DeepCopyInto(out *TailscaleConfig) { *out = *in diff --git a/k8s-operator/conditions.go b/k8s-operator/conditions.go index ae465a728..bce6e39bd 100644 --- a/k8s-operator/conditions.go +++ b/k8s-operator/conditions.go @@ -13,6 +13,7 @@ import ( xslices "golang.org/x/exp/slices" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/tstime" ) @@ -91,6 +92,14 @@ func SetProxyGroupCondition(pg *tsapi.ProxyGroup, conditionType tsapi.ConditionT pg.Status.Conditions = conds } +// SetTailnetCondition ensures that Tailnet status has a condition with the +// given attributes. LastTransitionTime gets set every time condition's status +// changes. +func SetTailnetCondition(tn *tsapi.Tailnet, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string, clock tstime.Clock, logger *zap.SugaredLogger) { + conds := updateCondition(tn.Status.Conditions, conditionType, status, reason, message, tn.Generation, clock, logger) + tn.Status.Conditions = conds +} + func updateCondition(conds []metav1.Condition, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string, gen int64, clock tstime.Clock, logger *zap.SugaredLogger) []metav1.Condition { newCondition := metav1.Condition{ Type: string(conditionType), @@ -187,3 +196,14 @@ func SvcIsReady(svc *corev1.Service) bool { cond := svc.Status.Conditions[idx] return cond.Status == metav1.ConditionTrue } + +func TailnetIsReady(tn *tsapi.Tailnet) bool { + idx := xslices.IndexFunc(tn.Status.Conditions, func(cond metav1.Condition) bool { + return cond.Type == string(tsapi.TailnetReady) + }) + if idx == -1 { + return false + } + cond := tn.Status.Conditions[idx] + return cond.Status == metav1.ConditionTrue && cond.ObservedGeneration == tn.Generation +} diff --git a/k8s-operator/reconciler/reconciler.go b/k8s-operator/reconciler/reconciler.go new file mode 100644 index 000000000..275179096 --- /dev/null +++ b/k8s-operator/reconciler/reconciler.go @@ -0,0 +1,39 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +// Package reconciler provides utilities for working with Kubernetes resources within controller reconciliation +// loops. +package reconciler + +import ( + "slices" + + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + // FinalizerName is the common finalizer used across all Tailscale Kubernetes resources. + FinalizerName = "tailscale.com/finalizer" +) + +// SetFinalizer adds the finalizer to the resource if not already present. +func SetFinalizer(obj client.Object) { + if idx := slices.Index(obj.GetFinalizers(), FinalizerName); idx >= 0 { + return + } + + obj.SetFinalizers(append(obj.GetFinalizers(), FinalizerName)) +} + +// RemoveFinalizer removes the finalizer from the resource if present. +func RemoveFinalizer(obj client.Object) { + idx := slices.Index(obj.GetFinalizers(), FinalizerName) + if idx < 0 { + return + } + + finalizers := obj.GetFinalizers() + obj.SetFinalizers(append(finalizers[:idx], finalizers[idx+1:]...)) +} diff --git a/k8s-operator/reconciler/reconciler_test.go b/k8s-operator/reconciler/reconciler_test.go new file mode 100644 index 000000000..573cd4d9d --- /dev/null +++ b/k8s-operator/reconciler/reconciler_test.go @@ -0,0 +1,42 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package reconciler_test + +import ( + "slices" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "tailscale.com/k8s-operator/reconciler" +) + +func TestFinalizers(t *testing.T) { + t.Parallel() + + object := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "test", + }, + StringData: map[string]string{ + "hello": "world", + }, + } + + reconciler.SetFinalizer(object) + + if !slices.Contains(object.Finalizers, reconciler.FinalizerName) { + t.Fatalf("object does not have finalizer %q: %v", reconciler.FinalizerName, object.Finalizers) + } + + reconciler.RemoveFinalizer(object) + + if slices.Contains(object.Finalizers, reconciler.FinalizerName) { + t.Fatalf("object still has finalizer %q: %v", reconciler.FinalizerName, object.Finalizers) + } +} diff --git a/k8s-operator/reconciler/tailnet/mocks_test.go b/k8s-operator/reconciler/tailnet/mocks_test.go new file mode 100644 index 000000000..7f3f2ddb9 --- /dev/null +++ b/k8s-operator/reconciler/tailnet/mocks_test.go @@ -0,0 +1,45 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package tailnet_test + +import ( + "context" + "io" + + "tailscale.com/internal/client/tailscale" +) + +type ( + MockTailnetClient struct { + ErrorOnDevices bool + ErrorOnKeys bool + ErrorOnServices bool + } +) + +func (m MockTailnetClient) Devices(_ context.Context, _ *tailscale.DeviceFieldsOpts) ([]*tailscale.Device, error) { + if m.ErrorOnDevices { + return nil, io.EOF + } + + return nil, nil +} + +func (m MockTailnetClient) Keys(_ context.Context) ([]string, error) { + if m.ErrorOnKeys { + return nil, io.EOF + } + + return nil, nil +} + +func (m MockTailnetClient) ListVIPServices(_ context.Context) (*tailscale.VIPServiceList, error) { + if m.ErrorOnServices { + return nil, io.EOF + } + + return nil, nil +} diff --git a/k8s-operator/reconciler/tailnet/tailnet.go b/k8s-operator/reconciler/tailnet/tailnet.go new file mode 100644 index 000000000..f574c2848 --- /dev/null +++ b/k8s-operator/reconciler/tailnet/tailnet.go @@ -0,0 +1,304 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +// Package tailnet provides reconciliation logic for the Tailnet custom resource definition. It is responsible for +// ensuring the referenced OAuth credentials are valid and have the required scopes to be able to generate authentication +// keys, manage devices & manage VIP services. +package tailnet + +import ( + "context" + "errors" + "fmt" + "time" + + "go.uber.org/zap" + "golang.org/x/oauth2" + "golang.org/x/oauth2/clientcredentials" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "tailscale.com/internal/client/tailscale" + "tailscale.com/ipn" + operatorutils "tailscale.com/k8s-operator" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler" + "tailscale.com/tstime" +) + +type ( + // The Reconciler type is a reconcile.TypedReconciler implementation used to manage the reconciliation of + // Tailnet custom resources. + Reconciler struct { + client.Client + + tailscaleNamespace string + clock tstime.Clock + logger *zap.SugaredLogger + clientFunc func(*tsapi.Tailnet, *corev1.Secret) TailscaleClient + } + + // The ReconcilerOptions type contains configuration values for the Reconciler. + ReconcilerOptions struct { + // The client for interacting with the Kubernetes API. + Client client.Client + // The namespace the operator is installed in. This reconciler expects Tailnet OAuth credentials to be stored + // in Secret resources within this namespace. + TailscaleNamespace string + // Controls which clock to use for performing time-based functions. This is typically modified for use + // in tests. + Clock tstime.Clock + // The logger to use for this Reconciler. + Logger *zap.SugaredLogger + // ClientFunc is a function that takes tailscale credentials and returns an implementation for the Tailscale + // HTTP API. This should generally be nil unless needed for testing. + ClientFunc func(*tsapi.Tailnet, *corev1.Secret) TailscaleClient + } + + // The TailscaleClient interface describes types that interact with the Tailscale HTTP API. + TailscaleClient interface { + Devices(context.Context, *tailscale.DeviceFieldsOpts) ([]*tailscale.Device, error) + Keys(ctx context.Context) ([]string, error) + ListVIPServices(ctx context.Context) (*tailscale.VIPServiceList, error) + } +) + +const reconcilerName = "tailnet-reconciler" + +// NewReconciler returns a new instance of the Reconciler type. It watches specifically for changes to Tailnet custom +// resources. The ReconcilerOptions can be used to modify the behaviour of the Reconciler. +func NewReconciler(options ReconcilerOptions) *Reconciler { + return &Reconciler{ + Client: options.Client, + tailscaleNamespace: options.TailscaleNamespace, + clock: options.Clock, + logger: options.Logger.Named(reconcilerName), + clientFunc: options.ClientFunc, + } +} + +// Register the Reconciler onto the given manager.Manager implementation. +func (r *Reconciler) Register(mgr manager.Manager) error { + return builder. + ControllerManagedBy(mgr). + For(&tsapi.Tailnet{}). + Named(reconcilerName). + Complete(r) +} + +// Reconcile is invoked when a change occurs to Tailnet resources within the cluster. On create/update, the Tailnet +// resource is validated ensuring that the specified Secret exists and contains valid OAuth credentials that have +// required permissions to perform all necessary functions by the operator. +func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { + var tailnet tsapi.Tailnet + err := r.Get(ctx, req.NamespacedName, &tailnet) + switch { + case apierrors.IsNotFound(err): + return reconcile.Result{}, nil + case err != nil: + return reconcile.Result{}, fmt.Errorf("failed to get Tailnet %q: %w", req.NamespacedName, err) + } + + if !tailnet.DeletionTimestamp.IsZero() { + return r.delete(ctx, &tailnet) + } + + return r.createOrUpdate(ctx, &tailnet) +} + +func (r *Reconciler) delete(ctx context.Context, tailnet *tsapi.Tailnet) (reconcile.Result, error) { + reconciler.RemoveFinalizer(tailnet) + if err := r.Update(ctx, tailnet); err != nil { + return reconcile.Result{}, fmt.Errorf("failed to remove finalizer from Tailnet %q: %w", tailnet.Name, err) + } + + return reconcile.Result{}, nil +} + +// Constants for condition reasons. +const ( + ReasonInvalidOAuth = "InvalidOAuth" + ReasonInvalidSecret = "InvalidSecret" + ReasonValid = "TailnetValid" +) + +func (r *Reconciler) createOrUpdate(ctx context.Context, tailnet *tsapi.Tailnet) (reconcile.Result, error) { + name := types.NamespacedName{Name: tailnet.Spec.Credentials.SecretName, Namespace: r.tailscaleNamespace} + + var secret corev1.Secret + err := r.Get(ctx, name, &secret) + + // The referenced Secret does not exist within the tailscale namespace, so we'll mark the Tailnet as not ready + // for use. + if apierrors.IsNotFound(err) { + operatorutils.SetTailnetCondition( + tailnet, + tsapi.TailnetReady, + metav1.ConditionFalse, + ReasonInvalidSecret, + fmt.Sprintf("referenced secret %q does not exist in namespace %q", name.Name, r.tailscaleNamespace), + r.clock, + r.logger, + ) + + if err = r.Status().Update(ctx, tailnet); err != nil { + return reconcile.Result{}, fmt.Errorf("failed to update Tailnet status for %q: %w", tailnet.Name, err) + } + + return reconcile.Result{}, nil + } + + if err != nil { + return reconcile.Result{}, fmt.Errorf("failed to get secret %q: %w", name, err) + } + + // We first ensure that the referenced secret contains the required fields. Otherwise, we set the Tailnet as + // invalid. The operator will not allow the use of this Tailnet while it is in an invalid state. + if ok := r.ensureSecret(tailnet, &secret); !ok { + if err = r.Status().Update(ctx, tailnet); err != nil { + return reconcile.Result{}, fmt.Errorf("failed to update Tailnet status for %q: %w", tailnet.Name, err) + } + + return reconcile.Result{RequeueAfter: time.Minute / 2}, nil + } + + tsClient := r.createClient(ctx, tailnet, &secret) + + // Second, we ensure the OAuth credentials supplied in the secret are valid and have the required scopes to access + // the various API endpoints required by the operator. + if ok := r.ensurePermissions(ctx, tsClient, tailnet); !ok { + if err = r.Status().Update(ctx, tailnet); err != nil { + return reconcile.Result{}, fmt.Errorf("failed to update Tailnet status for %q: %w", tailnet.Name, err) + } + + // We provide a requeue duration here as a user will likely want to go and modify their scopes and come back. + // This should save them having to delete and recreate the resource. + return reconcile.Result{RequeueAfter: time.Minute / 2}, nil + } + + operatorutils.SetTailnetCondition( + tailnet, + tsapi.TailnetReady, + metav1.ConditionTrue, + ReasonValid, + ReasonValid, + r.clock, + r.logger, + ) + + if err = r.Status().Update(ctx, tailnet); err != nil { + return reconcile.Result{}, fmt.Errorf("failed to update Tailnet status for %q: %w", tailnet.Name, err) + } + + reconciler.SetFinalizer(tailnet) + if err = r.Update(ctx, tailnet); err != nil { + return reconcile.Result{}, fmt.Errorf("failed to add finalizer to Tailnet %q: %w", tailnet.Name, err) + } + + return reconcile.Result{}, nil +} + +// Constants for OAuth credential fields within the Secret referenced by the Tailnet. +const ( + clientIDKey = "client_id" + clientSecretKey = "client_secret" +) + +func (r *Reconciler) createClient(ctx context.Context, tailnet *tsapi.Tailnet, secret *corev1.Secret) TailscaleClient { + if r.clientFunc != nil { + return r.clientFunc(tailnet, secret) + } + + baseURL := ipn.DefaultControlURL + if tailnet.Spec.LoginURL != "" { + baseURL = tailnet.Spec.LoginURL + } + + credentials := clientcredentials.Config{ + ClientID: string(secret.Data[clientIDKey]), + ClientSecret: string(secret.Data[clientSecretKey]), + TokenURL: baseURL + "/api/v2/oauth/token", + } + + source := credentials.TokenSource(ctx) + httpClient := oauth2.NewClient(ctx, source) + + tsClient := tailscale.NewClient("-", nil) + tsClient.UserAgent = "tailscale-k8s-operator" + tsClient.HTTPClient = httpClient + tsClient.BaseURL = baseURL + + return tsClient +} + +func (r *Reconciler) ensurePermissions(ctx context.Context, tsClient TailscaleClient, tailnet *tsapi.Tailnet) bool { + // Perform basic list requests here to confirm that the OAuth credentials referenced on the Tailnet resource + // can perform the basic operations required for the operator to function. This has a caveat of only performing + // read actions, as we don't want to create arbitrary keys and VIP services. However, it will catch when a user + // has completely forgotten an entire scope that's required. + var errs error + if _, err := tsClient.Devices(ctx, nil); err != nil { + errs = errors.Join(errs, fmt.Errorf("failed to list devices: %w", err)) + } + + if _, err := tsClient.Keys(ctx); err != nil { + errs = errors.Join(errs, fmt.Errorf("failed to list auth keys: %w", err)) + } + + if _, err := tsClient.ListVIPServices(ctx); err != nil { + errs = errors.Join(errs, fmt.Errorf("failed to list VIP services: %w", err)) + } + + if errs != nil { + operatorutils.SetTailnetCondition( + tailnet, + tsapi.TailnetReady, + metav1.ConditionFalse, + ReasonInvalidOAuth, + errs.Error(), + r.clock, + r.logger, + ) + + return false + } + + return true +} + +func (r *Reconciler) ensureSecret(tailnet *tsapi.Tailnet, secret *corev1.Secret) bool { + var message string + + switch { + case len(secret.Data) == 0: + message = fmt.Sprintf("Secret %q is empty", secret.Name) + case len(secret.Data[clientIDKey]) == 0: + message = fmt.Sprintf("Secret %q is missing the client_id field", secret.Name) + case len(secret.Data[clientSecretKey]) == 0: + message = fmt.Sprintf("Secret %q is missing the client_secret field", secret.Name) + } + + if message == "" { + return true + } + + operatorutils.SetTailnetCondition( + tailnet, + tsapi.TailnetReady, + metav1.ConditionFalse, + ReasonInvalidSecret, + message, + r.clock, + r.logger, + ) + + return false +} diff --git a/k8s-operator/reconciler/tailnet/tailnet_test.go b/k8s-operator/reconciler/tailnet/tailnet_test.go new file mode 100644 index 000000000..c3e4d62cf --- /dev/null +++ b/k8s-operator/reconciler/tailnet/tailnet_test.go @@ -0,0 +1,411 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package tailnet_test + +import ( + "testing" + + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler/tailnet" + "tailscale.com/tstest" +) + +func TestReconciler_Reconcile(t *testing.T) { + t.Parallel() + clock := tstest.NewClock(tstest.ClockOpts{}) + logger, err := zap.NewDevelopment() + if err != nil { + t.Fatal(err) + } + + tt := []struct { + Name string + Request reconcile.Request + Tailnet *tsapi.Tailnet + Secret *corev1.Secret + ExpectsError bool + ExpectedConditions []metav1.Condition + ClientFunc func(*tsapi.Tailnet, *corev1.Secret) tailnet.TailscaleClient + }{ + { + Name: "ignores unknown tailnet requests", + Request: reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: "test", + }, + }, + }, + { + Name: "invalid status for missing secret", + Request: reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: "test", + }, + }, + Tailnet: &tsapi.Tailnet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: tsapi.TailnetSpec{ + Credentials: tsapi.TailnetCredentials{ + SecretName: "test", + }, + }, + }, + ExpectedConditions: []metav1.Condition{ + { + Type: string(tsapi.TailnetReady), + Status: metav1.ConditionFalse, + Reason: tailnet.ReasonInvalidSecret, + Message: `referenced secret "test" does not exist in namespace "tailscale"`, + }, + }, + }, + { + Name: "invalid status for empty secret", + Request: reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: "test", + }, + }, + Tailnet: &tsapi.Tailnet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: tsapi.TailnetSpec{ + Credentials: tsapi.TailnetCredentials{ + SecretName: "test", + }, + }, + }, + Secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "tailscale", + }, + }, + ExpectedConditions: []metav1.Condition{ + { + Type: string(tsapi.TailnetReady), + Status: metav1.ConditionFalse, + Reason: tailnet.ReasonInvalidSecret, + Message: `Secret "test" is empty`, + }, + }, + }, + { + Name: "invalid status for missing client id", + Request: reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: "test", + }, + }, + Tailnet: &tsapi.Tailnet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: tsapi.TailnetSpec{ + Credentials: tsapi.TailnetCredentials{ + SecretName: "test", + }, + }, + }, + Secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "tailscale", + }, + Data: map[string][]byte{ + "client_secret": []byte("test"), + }, + }, + ExpectedConditions: []metav1.Condition{ + { + Type: string(tsapi.TailnetReady), + Status: metav1.ConditionFalse, + Reason: tailnet.ReasonInvalidSecret, + Message: `Secret "test" is missing the client_id field`, + }, + }, + }, + { + Name: "invalid status for missing client secret", + Request: reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: "test", + }, + }, + Tailnet: &tsapi.Tailnet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: tsapi.TailnetSpec{ + Credentials: tsapi.TailnetCredentials{ + SecretName: "test", + }, + }, + }, + Secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "tailscale", + }, + Data: map[string][]byte{ + "client_id": []byte("test"), + }, + }, + ExpectedConditions: []metav1.Condition{ + { + Type: string(tsapi.TailnetReady), + Status: metav1.ConditionFalse, + Reason: tailnet.ReasonInvalidSecret, + Message: `Secret "test" is missing the client_secret field`, + }, + }, + }, + { + Name: "invalid status for bad devices scope", + Request: reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: "test", + }, + }, + Tailnet: &tsapi.Tailnet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: tsapi.TailnetSpec{ + Credentials: tsapi.TailnetCredentials{ + SecretName: "test", + }, + }, + }, + Secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "tailscale", + }, + Data: map[string][]byte{ + "client_id": []byte("test"), + "client_secret": []byte("test"), + }, + }, + ClientFunc: func(_ *tsapi.Tailnet, _ *corev1.Secret) tailnet.TailscaleClient { + return &MockTailnetClient{ErrorOnDevices: true} + }, + ExpectedConditions: []metav1.Condition{ + { + Type: string(tsapi.TailnetReady), + Status: metav1.ConditionFalse, + Reason: tailnet.ReasonInvalidOAuth, + Message: `failed to list devices: EOF`, + }, + }, + }, + { + Name: "invalid status for bad services scope", + Request: reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: "test", + }, + }, + Tailnet: &tsapi.Tailnet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: tsapi.TailnetSpec{ + Credentials: tsapi.TailnetCredentials{ + SecretName: "test", + }, + }, + }, + Secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "tailscale", + }, + Data: map[string][]byte{ + "client_id": []byte("test"), + "client_secret": []byte("test"), + }, + }, + ClientFunc: func(_ *tsapi.Tailnet, _ *corev1.Secret) tailnet.TailscaleClient { + return &MockTailnetClient{ErrorOnServices: true} + }, + ExpectedConditions: []metav1.Condition{ + { + Type: string(tsapi.TailnetReady), + Status: metav1.ConditionFalse, + Reason: tailnet.ReasonInvalidOAuth, + Message: `failed to list VIP services: EOF`, + }, + }, + }, + { + Name: "invalid status for bad keys scope", + Request: reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: "test", + }, + }, + Tailnet: &tsapi.Tailnet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: tsapi.TailnetSpec{ + Credentials: tsapi.TailnetCredentials{ + SecretName: "test", + }, + }, + }, + Secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "tailscale", + }, + Data: map[string][]byte{ + "client_id": []byte("test"), + "client_secret": []byte("test"), + }, + }, + ClientFunc: func(_ *tsapi.Tailnet, _ *corev1.Secret) tailnet.TailscaleClient { + return &MockTailnetClient{ErrorOnKeys: true} + }, + ExpectedConditions: []metav1.Condition{ + { + Type: string(tsapi.TailnetReady), + Status: metav1.ConditionFalse, + Reason: tailnet.ReasonInvalidOAuth, + Message: `failed to list auth keys: EOF`, + }, + }, + }, + { + Name: "ready when valid and scopes are correct", + Request: reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: "default", + }, + }, + Tailnet: &tsapi.Tailnet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "default", + }, + Spec: tsapi.TailnetSpec{ + Credentials: tsapi.TailnetCredentials{ + SecretName: "test", + }, + }, + }, + Secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "tailscale", + }, + Data: map[string][]byte{ + "client_id": []byte("test"), + "client_secret": []byte("test"), + }, + }, + ClientFunc: func(_ *tsapi.Tailnet, _ *corev1.Secret) tailnet.TailscaleClient { + return &MockTailnetClient{} + }, + ExpectedConditions: []metav1.Condition{ + { + Type: string(tsapi.TailnetReady), + Status: metav1.ConditionTrue, + Reason: tailnet.ReasonValid, + Message: tailnet.ReasonValid, + }, + }, + }, + } + + for _, tc := range tt { + t.Run(tc.Name, func(t *testing.T) { + builder := fake.NewClientBuilder().WithScheme(tsapi.GlobalScheme) + if tc.Tailnet != nil { + builder = builder.WithObjects(tc.Tailnet).WithStatusSubresource(tc.Tailnet) + } + if tc.Secret != nil { + builder = builder.WithObjects(tc.Secret) + } + + fc := builder.Build() + opts := tailnet.ReconcilerOptions{ + Client: fc, + Clock: clock, + Logger: logger.Sugar(), + ClientFunc: tc.ClientFunc, + TailscaleNamespace: "tailscale", + } + + reconciler := tailnet.NewReconciler(opts) + _, err = reconciler.Reconcile(t.Context(), tc.Request) + if tc.ExpectsError && err == nil { + t.Fatalf("expected error, got none") + } + + if !tc.ExpectsError && err != nil { + t.Fatalf("expected no error, got %v", err) + } + + if len(tc.ExpectedConditions) == 0 { + return + } + + var tn tsapi.Tailnet + if err = fc.Get(t.Context(), tc.Request.NamespacedName, &tn); err != nil { + t.Fatal(err) + } + + if len(tn.Status.Conditions) != len(tc.ExpectedConditions) { + t.Fatalf("expected %v condition(s), got %v", len(tc.ExpectedConditions), len(tn.Status.Conditions)) + } + + for i, expected := range tc.ExpectedConditions { + actual := tn.Status.Conditions[i] + + if actual.Type != expected.Type { + t.Errorf("expected %v, got %v", expected.Type, actual.Type) + } + + if actual.Status != expected.Status { + t.Errorf("expected %v, got %v", expected.Status, actual.Status) + } + + if actual.Reason != expected.Reason { + t.Errorf("expected %v, got %v", expected.Reason, actual.Reason) + } + + if actual.Message != expected.Message { + t.Errorf("expected %v, got %v", expected.Message, actual.Message) + } + } + + if err = fc.Delete(t.Context(), &tn); err != nil { + t.Fatal(err) + } + + if _, err = reconciler.Reconcile(t.Context(), tc.Request); err != nil { + t.Fatal(err) + } + + err = fc.Get(t.Context(), tc.Request.NamespacedName, &tn) + if !apierrors.IsNotFound(err) { + t.Fatalf("expected not found error, got %v", err) + } + }) + } +}