mirror of https://github.com/tailscale/tailscale/
cmd/k8s-operator,k8s-operator: Allow the use of multiple tailnets (#18344)
This commit contains the implementation of multi-tailnet support within the Kubernetes Operator
Each of our custom resources now expose the `spec.tailnet` field. This field is a string that must match the name of an existing `Tailnet` resource. A `Tailnet` resource looks like this:
```yaml
apiVersion: tailscale.com/v1alpha1
kind: Tailnet
metadata:
name: example # This is the name that must be referenced by other resources
spec:
credentials:
secretName: example-oauth
```
Each `Tailnet` references a `Secret` resource that contains a set of oauth credentials. This secret must be created in the same namespace as the operator:
```yaml
apiVersion: v1
kind: Secret
metadata:
name: example-oauth # This is the name that's referenced by the Tailnet resource.
namespace: tailscale
stringData:
client_id: "client-id"
client_secret: "client-secret"
```
When created, the operator performs a basic check that the oauth client has access to all required scopes. This is done using read actions on devices, keys & services. While this doesn't capture a missing "write" permission, it catches completely missing permissions. Once this check passes, the `Tailnet` moves into a ready state and can be referenced. Attempting to use a `Tailnet` in a non-ready state will stall the deployment of `Connector`s, `ProxyGroup`s and `Recorder`s until the `Tailnet` becomes ready.
The `spec.tailnet` field informs the operator that a `Connector`, `ProxyGroup`, or `Recorder` must be given an auth key generated using the specified oauth client. For backwards compatibility, the set of credentials the operator is configured with are considered the default. That is, where `spec.tailnet` is not set, the resource will be deployed in the same tailnet as the operator.
Updates https://github.com/tailscale/corp/issues/34561
main
David Bond
14 hours ago
committed by
GitHub
parent
e30626c480
commit
2cb86cf65e
@ -0,0 +1,141 @@
|
||||
apiVersion: apiextensions.k8s.io/v1
|
||||
kind: CustomResourceDefinition
|
||||
metadata:
|
||||
annotations:
|
||||
controller-gen.kubebuilder.io/version: v0.17.0
|
||||
name: tailnets.tailscale.com
|
||||
spec:
|
||||
group: tailscale.com
|
||||
names:
|
||||
kind: Tailnet
|
||||
listKind: TailnetList
|
||||
plural: tailnets
|
||||
shortNames:
|
||||
- tn
|
||||
singular: tailnet
|
||||
scope: Cluster
|
||||
versions:
|
||||
- additionalPrinterColumns:
|
||||
- jsonPath: .metadata.creationTimestamp
|
||||
name: Age
|
||||
type: date
|
||||
- description: Status of the deployed Tailnet resources.
|
||||
jsonPath: .status.conditions[?(@.type == "TailnetReady")].reason
|
||||
name: Status
|
||||
type: string
|
||||
name: v1alpha1
|
||||
schema:
|
||||
openAPIV3Schema:
|
||||
type: object
|
||||
required:
|
||||
- metadata
|
||||
- spec
|
||||
properties:
|
||||
apiVersion:
|
||||
description: |-
|
||||
APIVersion defines the versioned schema of this representation of an object.
|
||||
Servers should convert recognized schemas to the latest internal value, and
|
||||
may reject unrecognized values.
|
||||
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
|
||||
type: string
|
||||
kind:
|
||||
description: |-
|
||||
Kind is a string value representing the REST resource this object represents.
|
||||
Servers may infer this from the endpoint the client submits requests to.
|
||||
Cannot be updated.
|
||||
In CamelCase.
|
||||
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
|
||||
type: string
|
||||
metadata:
|
||||
type: object
|
||||
spec:
|
||||
description: |-
|
||||
Spec describes the desired state of the Tailnet.
|
||||
More info:
|
||||
https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status
|
||||
type: object
|
||||
required:
|
||||
- credentials
|
||||
properties:
|
||||
credentials:
|
||||
description: Denotes the location of the OAuth credentials to use for authenticating with this Tailnet.
|
||||
type: object
|
||||
required:
|
||||
- secretName
|
||||
properties:
|
||||
secretName:
|
||||
description: |-
|
||||
The name of the secret containing the OAuth credentials. This secret must contain two fields "client_id" and
|
||||
"client_secret".
|
||||
type: string
|
||||
loginUrl:
|
||||
description: URL of the control plane to be used by all resources managed by the operator using this Tailnet.
|
||||
type: string
|
||||
status:
|
||||
description: |-
|
||||
Status describes the status of the Tailnet. This is set
|
||||
and managed by the Tailscale operator.
|
||||
type: object
|
||||
properties:
|
||||
conditions:
|
||||
type: array
|
||||
items:
|
||||
description: Condition contains details for one aspect of the current state of this API Resource.
|
||||
type: object
|
||||
required:
|
||||
- lastTransitionTime
|
||||
- message
|
||||
- reason
|
||||
- status
|
||||
- type
|
||||
properties:
|
||||
lastTransitionTime:
|
||||
description: |-
|
||||
lastTransitionTime is the last time the condition transitioned from one status to another.
|
||||
This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
|
||||
type: string
|
||||
format: date-time
|
||||
message:
|
||||
description: |-
|
||||
message is a human readable message indicating details about the transition.
|
||||
This may be an empty string.
|
||||
type: string
|
||||
maxLength: 32768
|
||||
observedGeneration:
|
||||
description: |-
|
||||
observedGeneration represents the .metadata.generation that the condition was set based upon.
|
||||
For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
|
||||
with respect to the current state of the instance.
|
||||
type: integer
|
||||
format: int64
|
||||
minimum: 0
|
||||
reason:
|
||||
description: |-
|
||||
reason contains a programmatic identifier indicating the reason for the condition's last transition.
|
||||
Producers of specific condition types may define expected values and meanings for this field,
|
||||
and whether the values are considered a guaranteed API.
|
||||
The value should be a CamelCase string.
|
||||
This field may not be empty.
|
||||
type: string
|
||||
maxLength: 1024
|
||||
minLength: 1
|
||||
pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
|
||||
status:
|
||||
description: status of the condition, one of True, False, Unknown.
|
||||
type: string
|
||||
enum:
|
||||
- "True"
|
||||
- "False"
|
||||
- Unknown
|
||||
type:
|
||||
description: type of condition in CamelCase or in foo.example.com/CamelCase.
|
||||
type: string
|
||||
maxLength: 316
|
||||
pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
|
||||
x-kubernetes-list-map-keys:
|
||||
- type
|
||||
x-kubernetes-list-type: map
|
||||
served: true
|
||||
storage: true
|
||||
subresources:
|
||||
status: {}
|
||||
@ -0,0 +1,58 @@
|
||||
// Copyright (c) Tailscale Inc & AUTHORS
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
//go:build !plan9
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"golang.org/x/oauth2"
|
||||
"golang.org/x/oauth2/clientcredentials"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
|
||||
"tailscale.com/internal/client/tailscale"
|
||||
"tailscale.com/ipn"
|
||||
operatorutils "tailscale.com/k8s-operator"
|
||||
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
|
||||
)
|
||||
|
||||
func clientForTailnet(ctx context.Context, cl client.Client, namespace, name string) (tsClient, error) {
|
||||
var tn tsapi.Tailnet
|
||||
if err := cl.Get(ctx, client.ObjectKey{Name: name}, &tn); err != nil {
|
||||
return nil, fmt.Errorf("failed to get tailnet %q: %w", name, err)
|
||||
}
|
||||
|
||||
if !operatorutils.TailnetIsReady(&tn) {
|
||||
return nil, fmt.Errorf("tailnet %q is not ready", name)
|
||||
}
|
||||
|
||||
var secret corev1.Secret
|
||||
if err := cl.Get(ctx, client.ObjectKey{Name: tn.Spec.Credentials.SecretName, Namespace: namespace}, &secret); err != nil {
|
||||
return nil, fmt.Errorf("failed to get Secret %q in namespace %q: %w", tn.Spec.Credentials.SecretName, namespace, err)
|
||||
}
|
||||
|
||||
baseURL := ipn.DefaultControlURL
|
||||
if tn.Spec.LoginURL != "" {
|
||||
baseURL = tn.Spec.LoginURL
|
||||
}
|
||||
|
||||
credentials := clientcredentials.Config{
|
||||
ClientID: string(secret.Data["client_id"]),
|
||||
ClientSecret: string(secret.Data["client_secret"]),
|
||||
TokenURL: baseURL + "/api/v2/oauth/token",
|
||||
}
|
||||
|
||||
source := credentials.TokenSource(ctx)
|
||||
httpClient := oauth2.NewClient(ctx, source)
|
||||
|
||||
ts := tailscale.NewClient(defaultTailnet, nil)
|
||||
ts.UserAgent = "tailscale-k8s-operator"
|
||||
ts.HTTPClient = httpClient
|
||||
ts.BaseURL = baseURL
|
||||
|
||||
return ts, nil
|
||||
}
|
||||
@ -0,0 +1,69 @@
|
||||
// Copyright (c) Tailscale Inc & AUTHORS
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
//go:build !plan9
|
||||
|
||||
package v1alpha1
|
||||
|
||||
import (
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
// Code comments on these types should be treated as user facing documentation-
|
||||
// they will appear on the Tailnet CRD i.e. if someone runs kubectl explain tailnet.
|
||||
|
||||
var TailnetKind = "Tailnet"
|
||||
|
||||
// +kubebuilder:object:root=true
|
||||
// +kubebuilder:subresource:status
|
||||
// +kubebuilder:resource:scope=Cluster,shortName=tn
|
||||
// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp"
|
||||
// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=`.status.conditions[?(@.type == "TailnetReady")].reason`,description="Status of the deployed Tailnet resources."
|
||||
|
||||
type Tailnet struct {
|
||||
metav1.TypeMeta `json:",inline"`
|
||||
metav1.ObjectMeta `json:"metadata,omitzero"`
|
||||
|
||||
// Spec describes the desired state of the Tailnet.
|
||||
// More info:
|
||||
// https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status
|
||||
Spec TailnetSpec `json:"spec"`
|
||||
|
||||
// Status describes the status of the Tailnet. This is set
|
||||
// and managed by the Tailscale operator.
|
||||
// +optional
|
||||
Status TailnetStatus `json:"status"`
|
||||
}
|
||||
|
||||
// +kubebuilder:object:root=true
|
||||
|
||||
type TailnetList struct {
|
||||
metav1.TypeMeta `json:",inline"`
|
||||
metav1.ListMeta `json:"metadata"`
|
||||
|
||||
Items []Tailnet `json:"items"`
|
||||
}
|
||||
|
||||
type TailnetSpec struct {
|
||||
// URL of the control plane to be used by all resources managed by the operator using this Tailnet.
|
||||
// +optional
|
||||
LoginURL string `json:"loginUrl,omitempty"`
|
||||
// Denotes the location of the OAuth credentials to use for authenticating with this Tailnet.
|
||||
Credentials TailnetCredentials `json:"credentials"`
|
||||
}
|
||||
|
||||
type TailnetCredentials struct {
|
||||
// The name of the secret containing the OAuth credentials. This secret must contain two fields "client_id" and
|
||||
// "client_secret".
|
||||
SecretName string `json:"secretName"`
|
||||
}
|
||||
|
||||
type TailnetStatus struct {
|
||||
// +listType=map
|
||||
// +listMapKey=type
|
||||
// +optional
|
||||
Conditions []metav1.Condition `json:"conditions"`
|
||||
}
|
||||
|
||||
// TailnetReady is set to True if the Tailnet is available for use by operator workloads.
|
||||
const TailnetReady ConditionType = `TailnetReady`
|
||||
@ -0,0 +1,39 @@
|
||||
// Copyright (c) Tailscale Inc & AUTHORS
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
//go:build !plan9
|
||||
|
||||
// Package reconciler provides utilities for working with Kubernetes resources within controller reconciliation
|
||||
// loops.
|
||||
package reconciler
|
||||
|
||||
import (
|
||||
"slices"
|
||||
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
)
|
||||
|
||||
const (
|
||||
// FinalizerName is the common finalizer used across all Tailscale Kubernetes resources.
|
||||
FinalizerName = "tailscale.com/finalizer"
|
||||
)
|
||||
|
||||
// SetFinalizer adds the finalizer to the resource if not already present.
|
||||
func SetFinalizer(obj client.Object) {
|
||||
if idx := slices.Index(obj.GetFinalizers(), FinalizerName); idx >= 0 {
|
||||
return
|
||||
}
|
||||
|
||||
obj.SetFinalizers(append(obj.GetFinalizers(), FinalizerName))
|
||||
}
|
||||
|
||||
// RemoveFinalizer removes the finalizer from the resource if present.
|
||||
func RemoveFinalizer(obj client.Object) {
|
||||
idx := slices.Index(obj.GetFinalizers(), FinalizerName)
|
||||
if idx < 0 {
|
||||
return
|
||||
}
|
||||
|
||||
finalizers := obj.GetFinalizers()
|
||||
obj.SetFinalizers(append(finalizers[:idx], finalizers[idx+1:]...))
|
||||
}
|
||||
@ -0,0 +1,42 @@
|
||||
// Copyright (c) Tailscale Inc & AUTHORS
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
//go:build !plan9
|
||||
|
||||
package reconciler_test
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
"tailscale.com/k8s-operator/reconciler"
|
||||
)
|
||||
|
||||
func TestFinalizers(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
object := &corev1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "test",
|
||||
Namespace: "test",
|
||||
},
|
||||
StringData: map[string]string{
|
||||
"hello": "world",
|
||||
},
|
||||
}
|
||||
|
||||
reconciler.SetFinalizer(object)
|
||||
|
||||
if !slices.Contains(object.Finalizers, reconciler.FinalizerName) {
|
||||
t.Fatalf("object does not have finalizer %q: %v", reconciler.FinalizerName, object.Finalizers)
|
||||
}
|
||||
|
||||
reconciler.RemoveFinalizer(object)
|
||||
|
||||
if slices.Contains(object.Finalizers, reconciler.FinalizerName) {
|
||||
t.Fatalf("object still has finalizer %q: %v", reconciler.FinalizerName, object.Finalizers)
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,45 @@
|
||||
// Copyright (c) Tailscale Inc & AUTHORS
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
//go:build !plan9
|
||||
|
||||
package tailnet_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
|
||||
"tailscale.com/internal/client/tailscale"
|
||||
)
|
||||
|
||||
type (
|
||||
MockTailnetClient struct {
|
||||
ErrorOnDevices bool
|
||||
ErrorOnKeys bool
|
||||
ErrorOnServices bool
|
||||
}
|
||||
)
|
||||
|
||||
func (m MockTailnetClient) Devices(_ context.Context, _ *tailscale.DeviceFieldsOpts) ([]*tailscale.Device, error) {
|
||||
if m.ErrorOnDevices {
|
||||
return nil, io.EOF
|
||||
}
|
||||
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (m MockTailnetClient) Keys(_ context.Context) ([]string, error) {
|
||||
if m.ErrorOnKeys {
|
||||
return nil, io.EOF
|
||||
}
|
||||
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (m MockTailnetClient) ListVIPServices(_ context.Context) (*tailscale.VIPServiceList, error) {
|
||||
if m.ErrorOnServices {
|
||||
return nil, io.EOF
|
||||
}
|
||||
|
||||
return nil, nil
|
||||
}
|
||||
@ -0,0 +1,327 @@
|
||||
// Copyright (c) Tailscale Inc & AUTHORS
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
//go:build !plan9
|
||||
|
||||
// Package tailnet provides reconciliation logic for the Tailnet custom resource definition. It is responsible for
|
||||
// ensuring the referenced OAuth credentials are valid and have the required scopes to be able to generate authentication
|
||||
// keys, manage devices & manage VIP services.
|
||||
package tailnet
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
"golang.org/x/oauth2"
|
||||
"golang.org/x/oauth2/clientcredentials"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"sigs.k8s.io/controller-runtime/pkg/builder"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/manager"
|
||||
"sigs.k8s.io/controller-runtime/pkg/reconcile"
|
||||
|
||||
"tailscale.com/internal/client/tailscale"
|
||||
"tailscale.com/ipn"
|
||||
operatorutils "tailscale.com/k8s-operator"
|
||||
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
|
||||
"tailscale.com/k8s-operator/reconciler"
|
||||
"tailscale.com/kube/kubetypes"
|
||||
"tailscale.com/tstime"
|
||||
"tailscale.com/util/clientmetric"
|
||||
"tailscale.com/util/set"
|
||||
)
|
||||
|
||||
type (
|
||||
// The Reconciler type is a reconcile.TypedReconciler implementation used to manage the reconciliation of
|
||||
// Tailnet custom resources.
|
||||
Reconciler struct {
|
||||
client.Client
|
||||
|
||||
tailscaleNamespace string
|
||||
clock tstime.Clock
|
||||
logger *zap.SugaredLogger
|
||||
clientFunc func(*tsapi.Tailnet, *corev1.Secret) TailscaleClient
|
||||
|
||||
// Metrics related fields
|
||||
mu sync.Mutex
|
||||
tailnets set.Slice[types.UID]
|
||||
}
|
||||
|
||||
// The ReconcilerOptions type contains configuration values for the Reconciler.
|
||||
ReconcilerOptions struct {
|
||||
// The client for interacting with the Kubernetes API.
|
||||
Client client.Client
|
||||
// The namespace the operator is installed in. This reconciler expects Tailnet OAuth credentials to be stored
|
||||
// in Secret resources within this namespace.
|
||||
TailscaleNamespace string
|
||||
// Controls which clock to use for performing time-based functions. This is typically modified for use
|
||||
// in tests.
|
||||
Clock tstime.Clock
|
||||
// The logger to use for this Reconciler.
|
||||
Logger *zap.SugaredLogger
|
||||
// ClientFunc is a function that takes tailscale credentials and returns an implementation for the Tailscale
|
||||
// HTTP API. This should generally be nil unless needed for testing.
|
||||
ClientFunc func(*tsapi.Tailnet, *corev1.Secret) TailscaleClient
|
||||
}
|
||||
|
||||
// The TailscaleClient interface describes types that interact with the Tailscale HTTP API.
|
||||
TailscaleClient interface {
|
||||
Devices(context.Context, *tailscale.DeviceFieldsOpts) ([]*tailscale.Device, error)
|
||||
Keys(ctx context.Context) ([]string, error)
|
||||
ListVIPServices(ctx context.Context) (*tailscale.VIPServiceList, error)
|
||||
}
|
||||
)
|
||||
|
||||
const reconcilerName = "tailnet-reconciler"
|
||||
|
||||
// NewReconciler returns a new instance of the Reconciler type. It watches specifically for changes to Tailnet custom
|
||||
// resources. The ReconcilerOptions can be used to modify the behaviour of the Reconciler.
|
||||
func NewReconciler(options ReconcilerOptions) *Reconciler {
|
||||
return &Reconciler{
|
||||
Client: options.Client,
|
||||
tailscaleNamespace: options.TailscaleNamespace,
|
||||
clock: options.Clock,
|
||||
logger: options.Logger.Named(reconcilerName),
|
||||
clientFunc: options.ClientFunc,
|
||||
}
|
||||
}
|
||||
|
||||
// Register the Reconciler onto the given manager.Manager implementation.
|
||||
func (r *Reconciler) Register(mgr manager.Manager) error {
|
||||
return builder.
|
||||
ControllerManagedBy(mgr).
|
||||
For(&tsapi.Tailnet{}).
|
||||
Named(reconcilerName).
|
||||
Complete(r)
|
||||
}
|
||||
|
||||
var (
|
||||
// gaugeTailnetResources tracks the overall number of Tailnet resources currently managed by this operator instance.
|
||||
gaugeTailnetResources = clientmetric.NewGauge(kubetypes.MetricTailnetCount)
|
||||
)
|
||||
|
||||
// Reconcile is invoked when a change occurs to Tailnet resources within the cluster. On create/update, the Tailnet
|
||||
// resource is validated ensuring that the specified Secret exists and contains valid OAuth credentials that have
|
||||
// required permissions to perform all necessary functions by the operator.
|
||||
func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) {
|
||||
var tailnet tsapi.Tailnet
|
||||
err := r.Get(ctx, req.NamespacedName, &tailnet)
|
||||
switch {
|
||||
case apierrors.IsNotFound(err):
|
||||
return reconcile.Result{}, nil
|
||||
case err != nil:
|
||||
return reconcile.Result{}, fmt.Errorf("failed to get Tailnet %q: %w", req.NamespacedName, err)
|
||||
}
|
||||
|
||||
if !tailnet.DeletionTimestamp.IsZero() {
|
||||
return r.delete(ctx, &tailnet)
|
||||
}
|
||||
|
||||
return r.createOrUpdate(ctx, &tailnet)
|
||||
}
|
||||
|
||||
func (r *Reconciler) delete(ctx context.Context, tailnet *tsapi.Tailnet) (reconcile.Result, error) {
|
||||
reconciler.RemoveFinalizer(tailnet)
|
||||
if err := r.Update(ctx, tailnet); err != nil {
|
||||
return reconcile.Result{}, fmt.Errorf("failed to remove finalizer from Tailnet %q: %w", tailnet.Name, err)
|
||||
}
|
||||
|
||||
r.mu.Lock()
|
||||
r.tailnets.Remove(tailnet.UID)
|
||||
r.mu.Unlock()
|
||||
gaugeTailnetResources.Set(int64(r.tailnets.Len()))
|
||||
|
||||
return reconcile.Result{}, nil
|
||||
}
|
||||
|
||||
// Constants for condition reasons.
|
||||
const (
|
||||
ReasonInvalidOAuth = "InvalidOAuth"
|
||||
ReasonInvalidSecret = "InvalidSecret"
|
||||
ReasonValid = "TailnetValid"
|
||||
)
|
||||
|
||||
func (r *Reconciler) createOrUpdate(ctx context.Context, tailnet *tsapi.Tailnet) (reconcile.Result, error) {
|
||||
r.mu.Lock()
|
||||
r.tailnets.Add(tailnet.UID)
|
||||
r.mu.Unlock()
|
||||
gaugeTailnetResources.Set(int64(r.tailnets.Len()))
|
||||
|
||||
name := types.NamespacedName{Name: tailnet.Spec.Credentials.SecretName, Namespace: r.tailscaleNamespace}
|
||||
|
||||
var secret corev1.Secret
|
||||
err := r.Get(ctx, name, &secret)
|
||||
|
||||
// The referenced Secret does not exist within the tailscale namespace, so we'll mark the Tailnet as not ready
|
||||
// for use.
|
||||
if apierrors.IsNotFound(err) {
|
||||
operatorutils.SetTailnetCondition(
|
||||
tailnet,
|
||||
tsapi.TailnetReady,
|
||||
metav1.ConditionFalse,
|
||||
ReasonInvalidSecret,
|
||||
fmt.Sprintf("referenced secret %q does not exist in namespace %q", name.Name, r.tailscaleNamespace),
|
||||
r.clock,
|
||||
r.logger,
|
||||
)
|
||||
|
||||
if err = r.Status().Update(ctx, tailnet); err != nil {
|
||||
return reconcile.Result{}, fmt.Errorf("failed to update Tailnet status for %q: %w", tailnet.Name, err)
|
||||
}
|
||||
|
||||
return reconcile.Result{}, nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return reconcile.Result{}, fmt.Errorf("failed to get secret %q: %w", name, err)
|
||||
}
|
||||
|
||||
// We first ensure that the referenced secret contains the required fields. Otherwise, we set the Tailnet as
|
||||
// invalid. The operator will not allow the use of this Tailnet while it is in an invalid state.
|
||||
if ok := r.ensureSecret(tailnet, &secret); !ok {
|
||||
if err = r.Status().Update(ctx, tailnet); err != nil {
|
||||
return reconcile.Result{}, fmt.Errorf("failed to update Tailnet status for %q: %w", tailnet.Name, err)
|
||||
}
|
||||
|
||||
return reconcile.Result{RequeueAfter: time.Minute / 2}, nil
|
||||
}
|
||||
|
||||
tsClient := r.createClient(ctx, tailnet, &secret)
|
||||
|
||||
// Second, we ensure the OAuth credentials supplied in the secret are valid and have the required scopes to access
|
||||
// the various API endpoints required by the operator.
|
||||
if ok := r.ensurePermissions(ctx, tsClient, tailnet); !ok {
|
||||
if err = r.Status().Update(ctx, tailnet); err != nil {
|
||||
return reconcile.Result{}, fmt.Errorf("failed to update Tailnet status for %q: %w", tailnet.Name, err)
|
||||
}
|
||||
|
||||
// We provide a requeue duration here as a user will likely want to go and modify their scopes and come back.
|
||||
// This should save them having to delete and recreate the resource.
|
||||
return reconcile.Result{RequeueAfter: time.Minute / 2}, nil
|
||||
}
|
||||
|
||||
operatorutils.SetTailnetCondition(
|
||||
tailnet,
|
||||
tsapi.TailnetReady,
|
||||
metav1.ConditionTrue,
|
||||
ReasonValid,
|
||||
ReasonValid,
|
||||
r.clock,
|
||||
r.logger,
|
||||
)
|
||||
|
||||
if err = r.Status().Update(ctx, tailnet); err != nil {
|
||||
return reconcile.Result{}, fmt.Errorf("failed to update Tailnet status for %q: %w", tailnet.Name, err)
|
||||
}
|
||||
|
||||
reconciler.SetFinalizer(tailnet)
|
||||
if err = r.Update(ctx, tailnet); err != nil {
|
||||
return reconcile.Result{}, fmt.Errorf("failed to add finalizer to Tailnet %q: %w", tailnet.Name, err)
|
||||
}
|
||||
|
||||
return reconcile.Result{}, nil
|
||||
}
|
||||
|
||||
// Constants for OAuth credential fields within the Secret referenced by the Tailnet.
|
||||
const (
|
||||
clientIDKey = "client_id"
|
||||
clientSecretKey = "client_secret"
|
||||
)
|
||||
|
||||
func (r *Reconciler) createClient(ctx context.Context, tailnet *tsapi.Tailnet, secret *corev1.Secret) TailscaleClient {
|
||||
if r.clientFunc != nil {
|
||||
return r.clientFunc(tailnet, secret)
|
||||
}
|
||||
|
||||
baseURL := ipn.DefaultControlURL
|
||||
if tailnet.Spec.LoginURL != "" {
|
||||
baseURL = tailnet.Spec.LoginURL
|
||||
}
|
||||
|
||||
credentials := clientcredentials.Config{
|
||||
ClientID: string(secret.Data[clientIDKey]),
|
||||
ClientSecret: string(secret.Data[clientSecretKey]),
|
||||
TokenURL: baseURL + "/api/v2/oauth/token",
|
||||
}
|
||||
|
||||
source := credentials.TokenSource(ctx)
|
||||
httpClient := oauth2.NewClient(ctx, source)
|
||||
|
||||
tsClient := tailscale.NewClient("-", nil)
|
||||
tsClient.UserAgent = "tailscale-k8s-operator"
|
||||
tsClient.HTTPClient = httpClient
|
||||
tsClient.BaseURL = baseURL
|
||||
|
||||
return tsClient
|
||||
}
|
||||
|
||||
func (r *Reconciler) ensurePermissions(ctx context.Context, tsClient TailscaleClient, tailnet *tsapi.Tailnet) bool {
|
||||
// Perform basic list requests here to confirm that the OAuth credentials referenced on the Tailnet resource
|
||||
// can perform the basic operations required for the operator to function. This has a caveat of only performing
|
||||
// read actions, as we don't want to create arbitrary keys and VIP services. However, it will catch when a user
|
||||
// has completely forgotten an entire scope that's required.
|
||||
var errs error
|
||||
if _, err := tsClient.Devices(ctx, nil); err != nil {
|
||||
errs = errors.Join(errs, fmt.Errorf("failed to list devices: %w", err))
|
||||
}
|
||||
|
||||
if _, err := tsClient.Keys(ctx); err != nil {
|
||||
errs = errors.Join(errs, fmt.Errorf("failed to list auth keys: %w", err))
|
||||
}
|
||||
|
||||
if _, err := tsClient.ListVIPServices(ctx); err != nil {
|
||||
errs = errors.Join(errs, fmt.Errorf("failed to list tailscale services: %w", err))
|
||||
}
|
||||
|
||||
if errs != nil {
|
||||
operatorutils.SetTailnetCondition(
|
||||
tailnet,
|
||||
tsapi.TailnetReady,
|
||||
metav1.ConditionFalse,
|
||||
ReasonInvalidOAuth,
|
||||
errs.Error(),
|
||||
r.clock,
|
||||
r.logger,
|
||||
)
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func (r *Reconciler) ensureSecret(tailnet *tsapi.Tailnet, secret *corev1.Secret) bool {
|
||||
var message string
|
||||
|
||||
switch {
|
||||
case len(secret.Data) == 0:
|
||||
message = fmt.Sprintf("Secret %q is empty", secret.Name)
|
||||
case len(secret.Data[clientIDKey]) == 0:
|
||||
message = fmt.Sprintf("Secret %q is missing the client_id field", secret.Name)
|
||||
case len(secret.Data[clientSecretKey]) == 0:
|
||||
message = fmt.Sprintf("Secret %q is missing the client_secret field", secret.Name)
|
||||
}
|
||||
|
||||
if message == "" {
|
||||
return true
|
||||
}
|
||||
|
||||
operatorutils.SetTailnetCondition(
|
||||
tailnet,
|
||||
tsapi.TailnetReady,
|
||||
metav1.ConditionFalse,
|
||||
ReasonInvalidSecret,
|
||||
message,
|
||||
r.clock,
|
||||
r.logger,
|
||||
)
|
||||
|
||||
return false
|
||||
}
|
||||
@ -0,0 +1,411 @@
|
||||
// Copyright (c) Tailscale Inc & AUTHORS
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
//go:build !plan9
|
||||
|
||||
package tailnet_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"go.uber.org/zap"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client/fake"
|
||||
"sigs.k8s.io/controller-runtime/pkg/reconcile"
|
||||
|
||||
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
|
||||
"tailscale.com/k8s-operator/reconciler/tailnet"
|
||||
"tailscale.com/tstest"
|
||||
)
|
||||
|
||||
func TestReconciler_Reconcile(t *testing.T) {
|
||||
t.Parallel()
|
||||
clock := tstest.NewClock(tstest.ClockOpts{})
|
||||
logger, err := zap.NewDevelopment()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
tt := []struct {
|
||||
Name string
|
||||
Request reconcile.Request
|
||||
Tailnet *tsapi.Tailnet
|
||||
Secret *corev1.Secret
|
||||
ExpectsError bool
|
||||
ExpectedConditions []metav1.Condition
|
||||
ClientFunc func(*tsapi.Tailnet, *corev1.Secret) tailnet.TailscaleClient
|
||||
}{
|
||||
{
|
||||
Name: "ignores unknown tailnet requests",
|
||||
Request: reconcile.Request{
|
||||
NamespacedName: types.NamespacedName{
|
||||
Name: "test",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "invalid status for missing secret",
|
||||
Request: reconcile.Request{
|
||||
NamespacedName: types.NamespacedName{
|
||||
Name: "test",
|
||||
},
|
||||
},
|
||||
Tailnet: &tsapi.Tailnet{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "test",
|
||||
},
|
||||
Spec: tsapi.TailnetSpec{
|
||||
Credentials: tsapi.TailnetCredentials{
|
||||
SecretName: "test",
|
||||
},
|
||||
},
|
||||
},
|
||||
ExpectedConditions: []metav1.Condition{
|
||||
{
|
||||
Type: string(tsapi.TailnetReady),
|
||||
Status: metav1.ConditionFalse,
|
||||
Reason: tailnet.ReasonInvalidSecret,
|
||||
Message: `referenced secret "test" does not exist in namespace "tailscale"`,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "invalid status for empty secret",
|
||||
Request: reconcile.Request{
|
||||
NamespacedName: types.NamespacedName{
|
||||
Name: "test",
|
||||
},
|
||||
},
|
||||
Tailnet: &tsapi.Tailnet{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "test",
|
||||
},
|
||||
Spec: tsapi.TailnetSpec{
|
||||
Credentials: tsapi.TailnetCredentials{
|
||||
SecretName: "test",
|
||||
},
|
||||
},
|
||||
},
|
||||
Secret: &corev1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "test",
|
||||
Namespace: "tailscale",
|
||||
},
|
||||
},
|
||||
ExpectedConditions: []metav1.Condition{
|
||||
{
|
||||
Type: string(tsapi.TailnetReady),
|
||||
Status: metav1.ConditionFalse,
|
||||
Reason: tailnet.ReasonInvalidSecret,
|
||||
Message: `Secret "test" is empty`,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "invalid status for missing client id",
|
||||
Request: reconcile.Request{
|
||||
NamespacedName: types.NamespacedName{
|
||||
Name: "test",
|
||||
},
|
||||
},
|
||||
Tailnet: &tsapi.Tailnet{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "test",
|
||||
},
|
||||
Spec: tsapi.TailnetSpec{
|
||||
Credentials: tsapi.TailnetCredentials{
|
||||
SecretName: "test",
|
||||
},
|
||||
},
|
||||
},
|
||||
Secret: &corev1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "test",
|
||||
Namespace: "tailscale",
|
||||
},
|
||||
Data: map[string][]byte{
|
||||
"client_secret": []byte("test"),
|
||||
},
|
||||
},
|
||||
ExpectedConditions: []metav1.Condition{
|
||||
{
|
||||
Type: string(tsapi.TailnetReady),
|
||||
Status: metav1.ConditionFalse,
|
||||
Reason: tailnet.ReasonInvalidSecret,
|
||||
Message: `Secret "test" is missing the client_id field`,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "invalid status for missing client secret",
|
||||
Request: reconcile.Request{
|
||||
NamespacedName: types.NamespacedName{
|
||||
Name: "test",
|
||||
},
|
||||
},
|
||||
Tailnet: &tsapi.Tailnet{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "test",
|
||||
},
|
||||
Spec: tsapi.TailnetSpec{
|
||||
Credentials: tsapi.TailnetCredentials{
|
||||
SecretName: "test",
|
||||
},
|
||||
},
|
||||
},
|
||||
Secret: &corev1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "test",
|
||||
Namespace: "tailscale",
|
||||
},
|
||||
Data: map[string][]byte{
|
||||
"client_id": []byte("test"),
|
||||
},
|
||||
},
|
||||
ExpectedConditions: []metav1.Condition{
|
||||
{
|
||||
Type: string(tsapi.TailnetReady),
|
||||
Status: metav1.ConditionFalse,
|
||||
Reason: tailnet.ReasonInvalidSecret,
|
||||
Message: `Secret "test" is missing the client_secret field`,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "invalid status for bad devices scope",
|
||||
Request: reconcile.Request{
|
||||
NamespacedName: types.NamespacedName{
|
||||
Name: "test",
|
||||
},
|
||||
},
|
||||
Tailnet: &tsapi.Tailnet{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "test",
|
||||
},
|
||||
Spec: tsapi.TailnetSpec{
|
||||
Credentials: tsapi.TailnetCredentials{
|
||||
SecretName: "test",
|
||||
},
|
||||
},
|
||||
},
|
||||
Secret: &corev1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "test",
|
||||
Namespace: "tailscale",
|
||||
},
|
||||
Data: map[string][]byte{
|
||||
"client_id": []byte("test"),
|
||||
"client_secret": []byte("test"),
|
||||
},
|
||||
},
|
||||
ClientFunc: func(_ *tsapi.Tailnet, _ *corev1.Secret) tailnet.TailscaleClient {
|
||||
return &MockTailnetClient{ErrorOnDevices: true}
|
||||
},
|
||||
ExpectedConditions: []metav1.Condition{
|
||||
{
|
||||
Type: string(tsapi.TailnetReady),
|
||||
Status: metav1.ConditionFalse,
|
||||
Reason: tailnet.ReasonInvalidOAuth,
|
||||
Message: `failed to list devices: EOF`,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "invalid status for bad services scope",
|
||||
Request: reconcile.Request{
|
||||
NamespacedName: types.NamespacedName{
|
||||
Name: "test",
|
||||
},
|
||||
},
|
||||
Tailnet: &tsapi.Tailnet{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "test",
|
||||
},
|
||||
Spec: tsapi.TailnetSpec{
|
||||
Credentials: tsapi.TailnetCredentials{
|
||||
SecretName: "test",
|
||||
},
|
||||
},
|
||||
},
|
||||
Secret: &corev1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "test",
|
||||
Namespace: "tailscale",
|
||||
},
|
||||
Data: map[string][]byte{
|
||||
"client_id": []byte("test"),
|
||||
"client_secret": []byte("test"),
|
||||
},
|
||||
},
|
||||
ClientFunc: func(_ *tsapi.Tailnet, _ *corev1.Secret) tailnet.TailscaleClient {
|
||||
return &MockTailnetClient{ErrorOnServices: true}
|
||||
},
|
||||
ExpectedConditions: []metav1.Condition{
|
||||
{
|
||||
Type: string(tsapi.TailnetReady),
|
||||
Status: metav1.ConditionFalse,
|
||||
Reason: tailnet.ReasonInvalidOAuth,
|
||||
Message: `failed to list tailscale services: EOF`,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "invalid status for bad keys scope",
|
||||
Request: reconcile.Request{
|
||||
NamespacedName: types.NamespacedName{
|
||||
Name: "test",
|
||||
},
|
||||
},
|
||||
Tailnet: &tsapi.Tailnet{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "test",
|
||||
},
|
||||
Spec: tsapi.TailnetSpec{
|
||||
Credentials: tsapi.TailnetCredentials{
|
||||
SecretName: "test",
|
||||
},
|
||||
},
|
||||
},
|
||||
Secret: &corev1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "test",
|
||||
Namespace: "tailscale",
|
||||
},
|
||||
Data: map[string][]byte{
|
||||
"client_id": []byte("test"),
|
||||
"client_secret": []byte("test"),
|
||||
},
|
||||
},
|
||||
ClientFunc: func(_ *tsapi.Tailnet, _ *corev1.Secret) tailnet.TailscaleClient {
|
||||
return &MockTailnetClient{ErrorOnKeys: true}
|
||||
},
|
||||
ExpectedConditions: []metav1.Condition{
|
||||
{
|
||||
Type: string(tsapi.TailnetReady),
|
||||
Status: metav1.ConditionFalse,
|
||||
Reason: tailnet.ReasonInvalidOAuth,
|
||||
Message: `failed to list auth keys: EOF`,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "ready when valid and scopes are correct",
|
||||
Request: reconcile.Request{
|
||||
NamespacedName: types.NamespacedName{
|
||||
Name: "default",
|
||||
},
|
||||
},
|
||||
Tailnet: &tsapi.Tailnet{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "default",
|
||||
},
|
||||
Spec: tsapi.TailnetSpec{
|
||||
Credentials: tsapi.TailnetCredentials{
|
||||
SecretName: "test",
|
||||
},
|
||||
},
|
||||
},
|
||||
Secret: &corev1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "test",
|
||||
Namespace: "tailscale",
|
||||
},
|
||||
Data: map[string][]byte{
|
||||
"client_id": []byte("test"),
|
||||
"client_secret": []byte("test"),
|
||||
},
|
||||
},
|
||||
ClientFunc: func(_ *tsapi.Tailnet, _ *corev1.Secret) tailnet.TailscaleClient {
|
||||
return &MockTailnetClient{}
|
||||
},
|
||||
ExpectedConditions: []metav1.Condition{
|
||||
{
|
||||
Type: string(tsapi.TailnetReady),
|
||||
Status: metav1.ConditionTrue,
|
||||
Reason: tailnet.ReasonValid,
|
||||
Message: tailnet.ReasonValid,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tt {
|
||||
t.Run(tc.Name, func(t *testing.T) {
|
||||
builder := fake.NewClientBuilder().WithScheme(tsapi.GlobalScheme)
|
||||
if tc.Tailnet != nil {
|
||||
builder = builder.WithObjects(tc.Tailnet).WithStatusSubresource(tc.Tailnet)
|
||||
}
|
||||
if tc.Secret != nil {
|
||||
builder = builder.WithObjects(tc.Secret)
|
||||
}
|
||||
|
||||
fc := builder.Build()
|
||||
opts := tailnet.ReconcilerOptions{
|
||||
Client: fc,
|
||||
Clock: clock,
|
||||
Logger: logger.Sugar(),
|
||||
ClientFunc: tc.ClientFunc,
|
||||
TailscaleNamespace: "tailscale",
|
||||
}
|
||||
|
||||
reconciler := tailnet.NewReconciler(opts)
|
||||
_, err = reconciler.Reconcile(t.Context(), tc.Request)
|
||||
if tc.ExpectsError && err == nil {
|
||||
t.Fatalf("expected error, got none")
|
||||
}
|
||||
|
||||
if !tc.ExpectsError && err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
|
||||
if len(tc.ExpectedConditions) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
var tn tsapi.Tailnet
|
||||
if err = fc.Get(t.Context(), tc.Request.NamespacedName, &tn); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if len(tn.Status.Conditions) != len(tc.ExpectedConditions) {
|
||||
t.Fatalf("expected %v condition(s), got %v", len(tc.ExpectedConditions), len(tn.Status.Conditions))
|
||||
}
|
||||
|
||||
for i, expected := range tc.ExpectedConditions {
|
||||
actual := tn.Status.Conditions[i]
|
||||
|
||||
if actual.Type != expected.Type {
|
||||
t.Errorf("expected %v, got %v", expected.Type, actual.Type)
|
||||
}
|
||||
|
||||
if actual.Status != expected.Status {
|
||||
t.Errorf("expected %v, got %v", expected.Status, actual.Status)
|
||||
}
|
||||
|
||||
if actual.Reason != expected.Reason {
|
||||
t.Errorf("expected %v, got %v", expected.Reason, actual.Reason)
|
||||
}
|
||||
|
||||
if actual.Message != expected.Message {
|
||||
t.Errorf("expected %v, got %v", expected.Message, actual.Message)
|
||||
}
|
||||
}
|
||||
|
||||
if err = fc.Delete(t.Context(), &tn); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if _, err = reconciler.Reconcile(t.Context(), tc.Request); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = fc.Get(t.Context(), tc.Request.NamespacedName, &tn)
|
||||
if !apierrors.IsNotFound(err) {
|
||||
t.Fatalf("expected not found error, got %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue