From 1a08ea5990c30caef5ad2c207c7e9ff2a94e8859 Mon Sep 17 00:00:00 2001 From: Irbe Krumina Date: Thu, 14 Dec 2023 13:51:59 +0000 Subject: [PATCH] cmd/k8s-operator: operator can create subnetrouter (#9505) * k8s-operator,cmd/k8s-operator,Makefile,scripts,.github/workflows: add Connector kube CRD. Connector CRD allows users to configure the Tailscale Kubernetes operator to deploy a subnet router to expose cluster CIDRs or other CIDRs available from within the cluster to their tailnet. Also adds various CRD related machinery to generate CRD YAML, deep copy implementations etc. Engineers will now have to run 'make kube-generate-all` after changing kube files to ensure that all generated files are up to date. * cmd/k8s-operator,k8s-operator: reconcile Connector resources Reconcile Connector resources, create/delete subnetrouter resources in response to changes to Connector(s). Connector reconciler will not be started unless ENABLE_CONNECTOR env var is set to true. This means that users who don't want to use the alpha Connector custom resource don't have to install the Connector CRD to their cluster. For users who do want to use it the flow is: - install the CRD - install the operator (via Helm chart or using static manifests). For Helm users set .values.enableConnector to true, for static manifest users, set ENABLE_CONNECTOR to true in the static manifest. Updates tailscale/tailscale#502 Signed-off-by: Irbe Krumina --- .github/workflows/kubemanifests.yaml | 5 +- Makefile | 15 + cmd/k8s-operator/connector.go | 259 ++++++++++++++++++ cmd/k8s-operator/connector_test.go | 164 +++++++++++ .../deploy/chart/templates/deployment.yaml | 2 + .../deploy/chart/templates/operator-rbac.yaml | 3 + cmd/k8s-operator/deploy/chart/values.yaml | 6 + .../deploy/crds/tailscale.com_connectors.yaml | 134 +++++++++ .../deploy/examples/subnetrouter.yaml | 17 ++ .../deploy/manifests/operator.yaml | 12 + cmd/k8s-operator/operator.go | 39 ++- cmd/k8s-operator/operator_test.go | 61 ++--- cmd/k8s-operator/sts.go | 10 + go.mod | 4 +- go.sum | 10 + header.txt | 2 + k8s-operator/apis/doc.go | 8 + k8s-operator/apis/v1alpha1/doc.go | 8 + k8s-operator/apis/v1alpha1/register.go | 56 ++++ k8s-operator/apis/v1alpha1/types_connector.go | 155 +++++++++++ .../apis/v1alpha1/zz_generated.deepcopy.go | 177 ++++++++++++ k8s-operator/conditions.go | 60 ++++ k8s-operator/conditions_test.go | 102 +++++++ scripts/check_license_headers.sh | 12 +- scripts/kube-deepcopy.sh | 11 + tstest/tools/tools.go | 1 + 26 files changed, 1291 insertions(+), 42 deletions(-) create mode 100644 cmd/k8s-operator/connector.go create mode 100644 cmd/k8s-operator/connector_test.go create mode 100644 cmd/k8s-operator/deploy/crds/tailscale.com_connectors.yaml create mode 100644 cmd/k8s-operator/deploy/examples/subnetrouter.yaml create mode 100644 header.txt create mode 100644 k8s-operator/apis/doc.go create mode 100644 k8s-operator/apis/v1alpha1/doc.go create mode 100644 k8s-operator/apis/v1alpha1/register.go create mode 100644 k8s-operator/apis/v1alpha1/types_connector.go create mode 100644 k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go create mode 100644 k8s-operator/conditions.go create mode 100644 k8s-operator/conditions_test.go create mode 100755 scripts/kube-deepcopy.sh diff --git a/.github/workflows/kubemanifests.yaml b/.github/workflows/kubemanifests.yaml index 6f51fb98c..c6b04cbd7 100644 --- a/.github/workflows/kubemanifests.yaml +++ b/.github/workflows/kubemanifests.yaml @@ -3,6 +3,7 @@ on: pull_request: paths: - './cmd/k8s-operator/' + - './k8s-operator/' - '.github/workflows/kubemanifests.yaml' # Cancel workflow run if there is a newer push to the same PR for which it is @@ -24,7 +25,7 @@ jobs: ./tool/helm lint "tailscale-operator-${VERSION_SHORT}.tgz" - name: Verify that static manifests are up to date run: | - ./tool/go generate tailscale.com/cmd/k8s-operator + make kube-generate-all echo echo - git diff --name-only --exit-code || (echo "Static manifests for Tailscale Kubernetes operator are out of date. Please run 'go generate tailscale.com/cmd/k8s-operator' and commit the diff."; exit 1) + git diff --name-only --exit-code || (echo "Generated files for Tailscale Kubernetes operator are out of date. Please run 'make kube-generate-all' and commit the diff."; exit 1) diff --git a/Makefile b/Makefile index 8149ca1d9..6a14bbc47 100644 --- a/Makefile +++ b/Makefile @@ -54,6 +54,21 @@ check: staticcheck vet depaware buildwindows build386 buildlinuxarm buildwasm ## staticcheck: ## Run staticcheck.io checks ./tool/go run honnef.co/go/tools/cmd/staticcheck -- $$(./tool/go list ./... | grep -v tempfork) +kube-generate-all: kube-generate-deepcopy ## Refresh generated files for Tailscale Kubernetes Operator + ./tool/go generate ./cmd/k8s-operator + +# Tailscale operator watches Connector custom resources in a Kubernetes cluster +# and caches them locally. Caching is done implicitly by controller-runtime +# library (the middleware used by Tailscale operator to create kube control +# loops). When a Connector resource is GET/LIST-ed from within our control loop, +# the request goes through the cache. To ensure that cache contents don't get +# modified by control loops, controller-runtime deep copies the requested +# object. In order for this to work, Connector must implement deep copy +# functionality so we autogenerate it here. +# https://github.com/kubernetes-sigs/controller-runtime/blob/v0.16.3/pkg/cache/internal/cache_reader.go#L86-L89 +kube-generate-deepcopy: ## Refresh generated deepcopy functionality for Tailscale kube API types + ./scripts/kube-deepcopy.sh + spk: ## Build synology package for ${SYNO_ARCH} architecture and ${SYNO_DSM} DSM version ./tool/go run ./cmd/dist build synology/dsm${SYNO_DSM}/${SYNO_ARCH} diff --git a/cmd/k8s-operator/connector.go b/cmd/k8s-operator/connector.go new file mode 100644 index 000000000..2062741d7 --- /dev/null +++ b/cmd/k8s-operator/connector.go @@ -0,0 +1,259 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package main + +import ( + "context" + "fmt" + "net/netip" + "slices" + "strings" + "sync" + "time" + + "github.com/pkg/errors" + "go.uber.org/zap" + xslices "golang.org/x/exp/slices" + corev1 "k8s.io/api/core/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + tsoperator "tailscale.com/k8s-operator" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/tstime" + "tailscale.com/util/clientmetric" + "tailscale.com/util/set" +) + +const ( + reasonSubnetRouterCreationFailed = "SubnetRouterCreationFailed" + reasonSubnetRouterCreated = "SubnetRouterCreated" + reasonSubnetRouterCleanupFailed = "SubnetRouterCleanupFailed" + reasonSubnetRouterCleanupInProgress = "SubnetRouterCleanupInProgress" + reasonSubnetRouterInvalid = "SubnetRouterInvalid" + + messageSubnetRouterCreationFailed = "Failed creating subnet router for routes %s: %v" + messageSubnetRouterInvalid = "Subnet router is invalid: %v" + messageSubnetRouterCreated = "Created subnet router for routes %s" + messageSubnetRouterCleanupFailed = "Failed cleaning up subnet router resources: %v" + msgSubnetRouterCleanupInProgress = "SubnetRouterCleanupInProgress" + + shortRequeue = time.Second * 5 +) + +type ConnectorReconciler struct { + client.Client + + recorder record.EventRecorder + ssr *tailscaleSTSReconciler + logger *zap.SugaredLogger + + tsnamespace string + + clock tstime.Clock + + mu sync.Mutex // protects following + + // subnetRouters tracks the subnet routers managed by this Tailscale + // Operator instance. + subnetRouters set.Slice[types.UID] +} + +var ( + // gaugeIngressResources tracks the number of subnet routers that we're + // currently managing. + gaugeSubnetRouterResources = clientmetric.NewGauge("k8s_subnet_router_resources") +) + +func (a *ConnectorReconciler) Reconcile(ctx context.Context, req reconcile.Request) (_ reconcile.Result, err error) { + logger := a.logger.With("connector", req.Name) + logger.Debugf("starting reconcile") + defer logger.Debugf("reconcile finished") + + cn := new(tsapi.Connector) + err = a.Get(ctx, req.NamespacedName, cn) + if apierrors.IsNotFound(err) { + logger.Debugf("connector not found, assuming it was deleted") + return reconcile.Result{}, nil + } else if err != nil { + return reconcile.Result{}, fmt.Errorf("failed to get tailscale.com Connector: %w", err) + } + if !cn.DeletionTimestamp.IsZero() { + logger.Debugf("connector is being deleted or should not be exposed, cleaning up components") + ix := xslices.Index(cn.Finalizers, FinalizerName) + if ix < 0 { + logger.Debugf("no finalizer, nothing to do") + return reconcile.Result{}, nil + } + + if done, err := a.maybeCleanupSubnetRouter(ctx, logger, cn); err != nil { + return reconcile.Result{}, err + } else if !done { + logger.Debugf("cleanup not finished, will retry...") + return reconcile.Result{RequeueAfter: shortRequeue}, nil + } + + cn.Finalizers = append(cn.Finalizers[:ix], cn.Finalizers[ix+1:]...) + if err := a.Update(ctx, cn); err != nil { + return reconcile.Result{}, err + } + logger.Infof("connector resources cleaned up") + return reconcile.Result{}, nil + } + + oldCnStatus := cn.Status.DeepCopy() + defer func() { + if cn.Status.SubnetRouter == nil { + tsoperator.SetConnectorCondition(cn, tsapi.ConnectorReady, metav1.ConditionUnknown, "", "", cn.Generation, a.clock, logger) + } else if cn.Status.SubnetRouter.Ready == metav1.ConditionTrue { + tsoperator.SetConnectorCondition(cn, tsapi.ConnectorReady, metav1.ConditionTrue, reasonSubnetRouterCreated, reasonSubnetRouterCreated, cn.Generation, a.clock, logger) + } else { + tsoperator.SetConnectorCondition(cn, tsapi.ConnectorReady, metav1.ConditionFalse, cn.Status.SubnetRouter.Reason, cn.Status.SubnetRouter.Reason, cn.Generation, a.clock, logger) + } + if !apiequality.Semantic.DeepEqual(oldCnStatus, cn.Status) { + // an error encountered here should get returned by the Reconcile function + if updateErr := a.Client.Status().Update(ctx, cn); updateErr != nil { + err = updateErr + } + } + }() + + if !slices.Contains(cn.Finalizers, FinalizerName) { + // This log line is printed exactly once during initial provisioning, + // because once the finalizer is in place this block gets skipped. So, + // this is a nice place to tell the operator that the high level, + // multi-reconcile operation is underway. + logger.Infof("ensuring connector is set up") + cn.Finalizers = append(cn.Finalizers, FinalizerName) + if err := a.Update(ctx, cn); err != nil { + err = fmt.Errorf("failed to add finalizer: %w", err) + logger.Errorf("error adding finalizer: %v", err) + return reconcile.Result{}, err + } + } + + // A Connector with unset .spec.subnetRouter and unset + // cn.spec.subnetRouter.Routes will be rejected at apply time (because + // these fields are set as required by our CRD validation). This check + // is here for if our CRD validation breaks unnoticed we don't crash the + // operator with nil pointer exception. + if cn.Spec.SubnetRouter == nil || len(cn.Spec.SubnetRouter.Routes) < 1 { + return reconcile.Result{}, nil + } + + if err := validateSubnetRouter(*cn.Spec.SubnetRouter); err != nil { + msg := fmt.Sprintf(messageSubnetRouterInvalid, err) + cn.Status.SubnetRouter = &tsapi.SubnetRouterStatus{ + Ready: metav1.ConditionFalse, + Reason: reasonSubnetRouterInvalid, + Message: msg, + } + a.recorder.Eventf(cn, corev1.EventTypeWarning, reasonSubnetRouterInvalid, msg) + return reconcile.Result{}, nil + } + + var sb strings.Builder + sb.WriteString(string(cn.Spec.SubnetRouter.Routes[0])) + for _, r := range cn.Spec.SubnetRouter.Routes[1:] { + sb.WriteString(fmt.Sprintf(",%s", r)) + } + cidrsS := sb.String() + logger.Debugf("ensuring a subnet router is deployed") + err = a.maybeProvisionSubnetRouter(ctx, logger, cn, cidrsS) + if err != nil { + msg := fmt.Sprintf(messageSubnetRouterCreationFailed, cidrsS, err) + cn.Status.SubnetRouter = &tsapi.SubnetRouterStatus{ + Ready: metav1.ConditionFalse, + Reason: reasonSubnetRouterCreationFailed, + Message: msg, + } + a.recorder.Eventf(cn, corev1.EventTypeWarning, reasonSubnetRouterCreationFailed, msg) + return reconcile.Result{}, err + } + cn.Status.SubnetRouter = &tsapi.SubnetRouterStatus{ + Routes: cidrsS, + Ready: metav1.ConditionTrue, + Reason: reasonSubnetRouterCreated, + Message: fmt.Sprintf(messageSubnetRouterCreated, cidrsS), + } + return reconcile.Result{}, nil +} + +func (a *ConnectorReconciler) maybeCleanupSubnetRouter(ctx context.Context, logger *zap.SugaredLogger, cn *tsapi.Connector) (bool, error) { + if done, err := a.ssr.Cleanup(ctx, logger, childResourceLabels(cn.Name, a.tsnamespace, "subnetrouter")); err != nil { + return false, fmt.Errorf("failed to cleanup: %w", err) + } else if !done { + logger.Debugf("cleanup not done yet, waiting for next reconcile") + return false, nil + } + + // Unlike most log entries in the reconcile loop, this will get printed + // exactly once at the very end of cleanup, because the final step of + // cleanup removes the tailscale finalizer, which will make all future + // reconciles exit early. + logger.Infof("cleaned up subnet router") + a.mu.Lock() + defer a.mu.Unlock() + a.subnetRouters.Remove(cn.UID) + gaugeSubnetRouterResources.Set(int64(a.subnetRouters.Len())) + return true, nil +} + +// maybeProvisionSubnetRouter maybe deploys subnet router that exposes a subset of cluster cidrs to the tailnet +func (a *ConnectorReconciler) maybeProvisionSubnetRouter(ctx context.Context, logger *zap.SugaredLogger, cn *tsapi.Connector, cidrs string) error { + if cn.Spec.SubnetRouter == nil || len(cn.Spec.SubnetRouter.Routes) < 1 { + return nil + } + a.mu.Lock() + a.subnetRouters.Add(cn.UID) + gaugeSubnetRouterResources.Set(int64(a.subnetRouters.Len())) + a.mu.Unlock() + + crl := childResourceLabels(cn.Name, a.tsnamespace, "subnetrouter") + hostname := hostnameForSubnetRouter(cn) + sts := &tailscaleSTSConfig{ + ParentResourceName: cn.Name, + ParentResourceUID: string(cn.UID), + Hostname: hostname, + ChildResourceLabels: crl, + Routes: cidrs, + } + for _, tag := range cn.Spec.SubnetRouter.Tags { + sts.Tags = append(sts.Tags, string(tag)) + } + + _, err := a.ssr.Provision(ctx, logger, sts) + + return err +} +func validateSubnetRouter(sb tsapi.SubnetRouter) error { + var err error + for _, route := range sb.Routes { + pfx, e := netip.ParsePrefix(string(route)) + if e != nil { + err = errors.Wrap(err, fmt.Sprintf("route %s is invalid: %v", route, err)) + continue + } + if pfx.Masked() != pfx { + err = errors.Wrap(err, fmt.Sprintf("route %s has non-address bits set; expected %s", pfx, pfx.Masked())) + } + } + return err +} + +func hostnameForSubnetRouter(cn *tsapi.Connector) string { + if cn.Spec.SubnetRouter == nil { + return "" + } + if cn.Spec.SubnetRouter.Hostname != "" { + return string(cn.Spec.SubnetRouter.Hostname) + } + return cn.Name + "-" + "subnetrouter" +} diff --git a/cmd/k8s-operator/connector_test.go b/cmd/k8s-operator/connector_test.go new file mode 100644 index 000000000..754eb7bdd --- /dev/null +++ b/cmd/k8s-operator/connector_test.go @@ -0,0 +1,164 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package main + +import ( + "context" + "testing" + + "go.uber.org/zap" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/tstest" + "tailscale.com/types/ptr" +) + +func TestConnector(t *testing.T) { + cn := &tsapi.Connector{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + UID: types.UID("1234-UID"), + }, + TypeMeta: metav1.TypeMeta{ + Kind: tsapi.ConnectorKind, + APIVersion: "tailscale.io/v1alpha1", + }, + Spec: tsapi.ConnectorSpec{ + SubnetRouter: &tsapi.SubnetRouter{ + Routes: []tsapi.Route{"10.40.0.0/14"}, + }, + }, + } + fc := fake.NewClientBuilder(). + WithScheme(tsapi.GlobalScheme). + WithObjects(cn). + WithStatusSubresource(cn). + Build() + ft := &fakeTSClient{} + zl, err := zap.NewDevelopment() + if err != nil { + t.Fatal(err) + } + + cl := tstest.NewClock(tstest.ClockOpts{}) + // Create a Connector with a subnet router definition + cr := &ConnectorReconciler{ + Client: fc, + ssr: &tailscaleSTSReconciler{ + Client: fc, + tsClient: ft, + defaultTags: []string{"tag:k8s"}, + operatorNamespace: "operator-ns", + proxyImage: "tailscale/tailscale", + }, + clock: cl, + logger: zl.Sugar(), + } + + expectReconciled(t, cr, "", "test") + fullName, shortName := findGenName(t, fc, "", "test", "subnetrouter") + + expectEqual(t, fc, expectedSecret(fullName, "", "subnetrouter")) + expectEqual(t, fc, expectedConnectorSTS(shortName, fullName, "10.40.0.0/14")) + + // Add another CIDR + mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) { + conn.Spec.SubnetRouter.Routes = []tsapi.Route{"10.40.0.0/14", "10.44.0.0/20"} + }) + expectReconciled(t, cr, "", "test") + expectEqual(t, fc, expectedConnectorSTS(shortName, fullName, "10.40.0.0/14,10.44.0.0/20")) + + // Remove a CIDR + mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) { + conn.Spec.SubnetRouter.Routes = []tsapi.Route{"10.44.0.0/20"} + }) + expectReconciled(t, cr, "", "test") + expectEqual(t, fc, expectedConnectorSTS(shortName, fullName, "10.44.0.0/20")) + + // Delete the Connector + if err = fc.Delete(context.Background(), cn); err != nil { + t.Fatalf("error deleting Connector: %v", err) + } + + expectRequeue(t, cr, "", "test") + expectReconciled(t, cr, "", "test") + + expectMissing[appsv1.StatefulSet](t, fc, "operator-ns", shortName) + expectMissing[corev1.Secret](t, fc, "operator-ns", fullName) + +} + +func expectedConnectorSTS(stsName, secretName, routes string) *appsv1.StatefulSet { + return &appsv1.StatefulSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "StatefulSet", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: stsName, + Namespace: "operator-ns", + Labels: map[string]string{ + "tailscale.com/managed": "true", + "tailscale.com/parent-resource": "test", + "tailscale.com/parent-resource-ns": "", + "tailscale.com/parent-resource-type": "subnetrouter", + }, + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: ptr.To[int32](1), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "1234-UID"}, + }, + ServiceName: stsName, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + DeletionGracePeriodSeconds: ptr.To[int64](10), + Labels: map[string]string{"app": "1234-UID"}, + Annotations: map[string]string{ + "tailscale.com/operator-last-set-hostname": "test-subnetrouter", + }, + }, + Spec: corev1.PodSpec{ + ServiceAccountName: "proxies", + InitContainers: []corev1.Container{ + { + Name: "sysctler", + Image: "tailscale/tailscale", + Command: []string{"/bin/sh"}, + Args: []string{"-c", "sysctl -w net.ipv4.ip_forward=1 net.ipv6.conf.all.forwarding=1"}, + SecurityContext: &corev1.SecurityContext{ + Privileged: ptr.To(true), + }, + }, + }, + Containers: []corev1.Container{ + { + Name: "tailscale", + Image: "tailscale/tailscale", + Env: []corev1.EnvVar{ + {Name: "TS_USERSPACE", Value: "false"}, + {Name: "TS_AUTH_ONCE", Value: "true"}, + {Name: "TS_KUBE_SECRET", Value: secretName}, + {Name: "TS_HOSTNAME", Value: "test-subnetrouter"}, + {Name: "TS_ROUTES", Value: routes}, + }, + SecurityContext: &corev1.SecurityContext{ + Capabilities: &corev1.Capabilities{ + Add: []corev1.Capability{"NET_ADMIN"}, + }, + }, + ImagePullPolicy: "Always", + }, + }, + }, + }, + }, + } +} diff --git a/cmd/k8s-operator/deploy/chart/templates/deployment.yaml b/cmd/k8s-operator/deploy/chart/templates/deployment.yaml index a451cf27f..75a53b51e 100644 --- a/cmd/k8s-operator/deploy/chart/templates/deployment.yaml +++ b/cmd/k8s-operator/deploy/chart/templates/deployment.yaml @@ -59,6 +59,8 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: ENABLE_CONNECTOR + value: "{{ .Values.enableConnector }}" - name: CLIENT_ID_FILE value: /oauth/client_id - name: CLIENT_SECRET_FILE diff --git a/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml b/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml index 5fc7ad203..fbd83e7e1 100644 --- a/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml +++ b/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml @@ -18,6 +18,9 @@ rules: - apiGroups: ["networking.k8s.io"] resources: ["ingresses", "ingresses/status"] verbs: ["*"] +- apiGroups: ["tailscale.com"] + resources: ["connectors", "connectors/status"] + verbs: ["get", "list", "watch", "update"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/cmd/k8s-operator/deploy/chart/values.yaml b/cmd/k8s-operator/deploy/chart/values.yaml index 4cd9ce750..3bb0b7e23 100644 --- a/cmd/k8s-operator/deploy/chart/values.yaml +++ b/cmd/k8s-operator/deploy/chart/values.yaml @@ -8,6 +8,12 @@ oauth: {} # clientId: "" # clientSecret: "" +# enableConnector determines whether the operator should reconcile +# connector.tailscale.com custom resources. If set to true you have to install +# connector CRD in a separate step. +# You can do so by running 'kubectl apply -f ./cmd/k8s-operator/deploy/crds'. +enableConnector: "false" + operatorConfig: image: repo: tailscale/k8s-operator diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_connectors.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_connectors.yaml new file mode 100644 index 000000000..9a162bb4c --- /dev/null +++ b/cmd/k8s-operator/deploy/crds/tailscale.com_connectors.yaml @@ -0,0 +1,134 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.13.0 + name: connectors.tailscale.com +spec: + group: tailscale.com + names: + kind: Connector + listKind: ConnectorList + plural: connectors + shortNames: + - cn + singular: connector + scope: Cluster + versions: + - additionalPrinterColumns: + - description: Cluster CIDR ranges exposed to tailnet via subnet router + jsonPath: .status.subnetRouter.routes + name: SubnetRoutes + type: string + - description: Status of the components deployed by the connector + jsonPath: .status.conditions[?(@.type == "ConnectorReady")].reason + name: Status + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + type: object + required: + - spec + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Desired state of the Connector resource. + type: object + required: + - subnetRouter + properties: + subnetRouter: + description: SubnetRouter configures a Tailscale subnet router to be deployed in the cluster. If unset no subnet router will be deployed. https://tailscale.com/kb/1019/subnets/ + type: object + required: + - routes + properties: + hostname: + description: Hostname is the tailnet hostname that should be assigned to the subnet router node. If unset hostname is defaulted to -subnetrouter. Hostname can contain lower case letters, numbers and dashes, it must not start or end with a dash and must be between 2 and 63 characters long. + type: string + pattern: ^[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$ + routes: + description: Routes refer to in-cluster CIDRs that the subnet router should make available. Route values must be strings that represent a valid IPv4 or IPv6 CIDR range. Values can be Tailscale 4via6 subnet routes. https://tailscale.com/kb/1201/4via6-subnets/ + type: array + items: + type: string + format: cidr + tags: + description: Tags that the Tailscale node will be tagged with. If you want the subnet router to be autoapproved, you can configure Tailscale ACLs to autoapprove the subnetrouter's CIDRs for these tags. See https://tailscale.com/kb/1018/acls/#auto-approvers-for-routes-and-exit-nodes Defaults to tag:k8s. If you specify custom tags here, you must also make tag:k8s-operator owner of the custom tag. See https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator. Tags cannot be changed once a Connector has been created. Tag values must be in form ^tag:[a-zA-Z][a-zA-Z0-9-]*$. + type: array + items: + type: string + pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$ + x-kubernetes-validations: + - rule: has(self.tags) == has(oldSelf.tags) + message: Subnetrouter tags cannot be changed. Delete and redeploy the Connector if you need to change it. + status: + description: Status of the Connector. This is set and managed by the Tailscale operator. + type: object + properties: + conditions: + description: List of status conditions to indicate the status of the Connector. Known condition types are `ConnectorReady`. + type: array + items: + description: ConnectorCondition contains condition information for a Connector. + type: object + required: + - status + - type + properties: + lastTransitionTime: + description: LastTransitionTime is the timestamp corresponding to the last status change of this condition. + type: string + format: date-time + message: + description: Message is a human readable description of the details of the last transition, complementing reason. + type: string + observedGeneration: + description: If set, this represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.condition[x].observedGeneration is 9, the condition is out of date with respect to the current state of the Connector. + type: integer + format: int64 + reason: + description: Reason is a brief machine readable explanation for the condition's last transition. + type: string + status: + description: Status of the condition, one of ('True', 'False', 'Unknown'). + type: string + type: + description: Type of the condition, known values are (`SubnetRouterReady`). + type: string + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + subnetRouter: + description: SubnetRouter status is the current status of a subnet router + type: object + required: + - message + - ready + - reason + - routes + properties: + message: + description: Message is a more verbose reason for the current subnet router status + type: string + ready: + description: Ready is the ready status of the subnet router + type: string + reason: + description: Reason is the reason for the subnet router status + type: string + routes: + description: Routes are the CIDRs currently exposed via subnet router + type: string + served: true + storage: true + subresources: + status: {} diff --git a/cmd/k8s-operator/deploy/examples/subnetrouter.yaml b/cmd/k8s-operator/deploy/examples/subnetrouter.yaml new file mode 100644 index 000000000..6604622a7 --- /dev/null +++ b/cmd/k8s-operator/deploy/examples/subnetrouter.yaml @@ -0,0 +1,17 @@ +# Before applyong this ensure that the operator is owner of tag:subnet. +# https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator. +# To set up autoapproval set tag:subnet as approver for 10.40.0.0/14 route +# otherwise you will need to approve it manually in control panel once the +# subnet router has been created. +# https://tailscale.com/kb/1019/subnets/#advertise-subnet-routes +apiVersion: tailscale.com/v1alpha1 +kind: Connector +metadata: + name: exposepods +spec: + subnetRouter: + routes: + - "10.40.0.0/14" + tags: + - "tag:subnet" + hostname: pods-subnetrouter diff --git a/cmd/k8s-operator/deploy/manifests/operator.yaml b/cmd/k8s-operator/deploy/manifests/operator.yaml index 73a3895f6..f385a8966 100644 --- a/cmd/k8s-operator/deploy/manifests/operator.yaml +++ b/cmd/k8s-operator/deploy/manifests/operator.yaml @@ -47,6 +47,16 @@ rules: - ingresses/status verbs: - '*' + - apiGroups: + - tailscale.com + resources: + - connectors + - connectors/status + verbs: + - get + - list + - watch + - update --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -150,6 +160,8 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: ENABLE_CONNECTOR + value: "false" - name: CLIENT_ID_FILE value: /oauth/client_id - name: CLIENT_SECRET_FILE diff --git a/cmd/k8s-operator/operator.go b/cmd/k8s-operator/operator.go index 73b039b4d..d762acd9a 100644 --- a/cmd/k8s-operator/operator.go +++ b/cmd/k8s-operator/operator.go @@ -37,13 +37,19 @@ import ( "tailscale.com/hostinfo" "tailscale.com/ipn" "tailscale.com/ipn/store/kubestore" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/tsnet" + "tailscale.com/tstime" "tailscale.com/types/logger" "tailscale.com/version" ) +// Generate static manifests for deploying Tailscale operator on Kubernetes from the operator's Helm chart. //go:generate go run tailscale.com/cmd/k8s-operator/generate +// Generate Connector CustomResourceDefinition yaml from its Go types. +//go:generate go run sigs.k8s.io/controller-tools/cmd/controller-gen crd schemapatch:manifests=./deploy/crds output:dir=./deploy/crds paths=../../k8s-operator/apis/... + func main() { // Required to use our client API. We're fine with the instability since the // client lives in the same repo as this code. @@ -56,6 +62,7 @@ func main() { priorityClassName = defaultEnv("PROXY_PRIORITY_CLASS_NAME", "") tags = defaultEnv("PROXY_TAGS", "tag:k8s") tsFirewallMode = defaultEnv("PROXY_FIREWALL_MODE", "") + tsEnableConnector = defaultBool("ENABLE_CONNECTOR", false) ) var opts []kzap.Opts @@ -84,7 +91,9 @@ func main() { defer s.Close() restConfig := config.GetConfigOrDie() maybeLaunchAPIServerProxy(zlog, restConfig, s, mode) - runReconcilers(zlog, s, tsNamespace, restConfig, tsClient, image, priorityClassName, tags, tsFirewallMode) + // TODO (irbekrm): gather the reconciler options into an opts struct + // rather than passing a million of them in one by one. + runReconcilers(zlog, s, tsNamespace, restConfig, tsClient, image, priorityClassName, tags, tsFirewallMode, tsEnableConnector) } // initTSNet initializes the tsnet.Server and logs in to Tailscale. It uses the @@ -192,7 +201,7 @@ waitOnline: // runReconcilers starts the controller-runtime manager and registers the // ServiceReconciler. It blocks forever. -func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string, restConfig *rest.Config, tsClient *tailscale.Client, image, priorityClassName, tags, tsFirewallMode string) { +func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string, restConfig *rest.Config, tsClient *tailscale.Client, image, priorityClassName, tags, tsFirewallMode string, enableConnector bool) { var ( isDefaultLoadBalancer = defaultBool("OPERATOR_DEFAULT_LOAD_BALANCER", false) ) @@ -206,20 +215,25 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string nsFilter := cache.ByObject{ Field: client.InNamespace(tsNamespace).AsSelector(), } - mgr, err := manager.New(restConfig, manager.Options{ + mgrOpts := manager.Options{ Cache: cache.Options{ ByObject: map[client.Object]cache.ByObject{ &corev1.Secret{}: nsFilter, &appsv1.StatefulSet{}: nsFilter, }, }, - }) + } + if enableConnector { + mgrOpts.Scheme = tsapi.GlobalScheme + } + mgr, err := manager.New(restConfig, mgrOpts) if err != nil { startlog.Fatalf("could not create manager: %v", err) } svcFilter := handler.EnqueueRequestsFromMapFunc(serviceHandler) svcChildFilter := handler.EnqueueRequestsFromMapFunc(managedResourceHandlerForType("svc")) + eventRecorder := mgr.GetEventRecorderFor("tailscale-operator") ssr := &tailscaleSTSReconciler{ Client: mgr.GetClient(), @@ -264,6 +278,23 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string startlog.Fatalf("could not create controller: %v", err) } + if enableConnector { + connectorFilter := handler.EnqueueRequestsFromMapFunc(managedResourceHandlerForType("subnetrouter")) + err = builder.ControllerManagedBy(mgr). + For(&tsapi.Connector{}). + Watches(&appsv1.StatefulSet{}, connectorFilter). + Watches(&corev1.Secret{}, connectorFilter). + Complete(&ConnectorReconciler{ + ssr: ssr, + recorder: eventRecorder, + Client: mgr.GetClient(), + logger: zlog.Named("connector-reconciler"), + clock: tstime.DefaultClock{}, + }) + if err != nil { + startlog.Fatal("could not create connector reconciler: %v", err) + } + } startlog.Infof("Startup complete, operator running, version: %s", version.Long()) if err := mgr.Start(signals.SetupSignalHandler()); err != nil { startlog.Fatalf("could not start manager: %v", err) diff --git a/cmd/k8s-operator/operator_test.go b/cmd/k8s-operator/operator_test.go index 881f8c959..2f1ea5a57 100644 --- a/cmd/k8s-operator/operator_test.go +++ b/cmd/k8s-operator/operator_test.go @@ -66,9 +66,9 @@ func TestLoadBalancerClass(t *testing.T) { expectReconciled(t, sr, "default", "test") - fullName, shortName := findGenName(t, fc, "default", "test") + fullName, shortName := findGenName(t, fc, "default", "test", "svc") - expectEqual(t, fc, expectedSecret(fullName)) + expectEqual(t, fc, expectedSecret(fullName, "default", "svc")) expectEqual(t, fc, expectedHeadlessService(shortName)) o := stsOpts{ name: shortName, @@ -203,9 +203,9 @@ func TestTailnetTargetFQDNAnnotation(t *testing.T) { expectReconciled(t, sr, "default", "test") - fullName, shortName := findGenName(t, fc, "default", "test") + fullName, shortName := findGenName(t, fc, "default", "test", "svc") - expectEqual(t, fc, expectedSecret(fullName)) + expectEqual(t, fc, expectedSecret(fullName, "default", "svc")) expectEqual(t, fc, expectedHeadlessService(shortName)) o := stsOpts{ name: shortName, @@ -235,7 +235,7 @@ func TestTailnetTargetFQDNAnnotation(t *testing.T) { }, } expectEqual(t, fc, want) - expectEqual(t, fc, expectedSecret(fullName)) + expectEqual(t, fc, expectedSecret(fullName, "default", "svc")) expectEqual(t, fc, expectedHeadlessService(shortName)) o = stsOpts{ name: shortName, @@ -316,9 +316,9 @@ func TestTailnetTargetIPAnnotation(t *testing.T) { expectReconciled(t, sr, "default", "test") - fullName, shortName := findGenName(t, fc, "default", "test") + fullName, shortName := findGenName(t, fc, "default", "test", "svc") - expectEqual(t, fc, expectedSecret(fullName)) + expectEqual(t, fc, expectedSecret(fullName, "default", "svc")) expectEqual(t, fc, expectedHeadlessService(shortName)) o := stsOpts{ name: shortName, @@ -348,7 +348,7 @@ func TestTailnetTargetIPAnnotation(t *testing.T) { }, } expectEqual(t, fc, want) - expectEqual(t, fc, expectedSecret(fullName)) + expectEqual(t, fc, expectedSecret(fullName, "default", "svc")) expectEqual(t, fc, expectedHeadlessService(shortName)) o = stsOpts{ name: shortName, @@ -427,9 +427,9 @@ func TestAnnotations(t *testing.T) { expectReconciled(t, sr, "default", "test") - fullName, shortName := findGenName(t, fc, "default", "test") + fullName, shortName := findGenName(t, fc, "default", "test", "svc") - expectEqual(t, fc, expectedSecret(fullName)) + expectEqual(t, fc, expectedSecret(fullName, "default", "svc")) expectEqual(t, fc, expectedHeadlessService(shortName)) o := stsOpts{ name: shortName, @@ -532,9 +532,9 @@ func TestAnnotationIntoLB(t *testing.T) { expectReconciled(t, sr, "default", "test") - fullName, shortName := findGenName(t, fc, "default", "test") + fullName, shortName := findGenName(t, fc, "default", "test", "svc") - expectEqual(t, fc, expectedSecret(fullName)) + expectEqual(t, fc, expectedSecret(fullName, "default", "svc")) expectEqual(t, fc, expectedHeadlessService(shortName)) o := stsOpts{ name: shortName, @@ -665,9 +665,9 @@ func TestLBIntoAnnotation(t *testing.T) { expectReconciled(t, sr, "default", "test") - fullName, shortName := findGenName(t, fc, "default", "test") + fullName, shortName := findGenName(t, fc, "default", "test", "svc") - expectEqual(t, fc, expectedSecret(fullName)) + expectEqual(t, fc, expectedSecret(fullName, "default", "svc")) expectEqual(t, fc, expectedHeadlessService(shortName)) o := stsOpts{ name: shortName, @@ -808,9 +808,9 @@ func TestCustomHostname(t *testing.T) { expectReconciled(t, sr, "default", "test") - fullName, shortName := findGenName(t, fc, "default", "test") + fullName, shortName := findGenName(t, fc, "default", "test", "svc") - expectEqual(t, fc, expectedSecret(fullName)) + expectEqual(t, fc, expectedSecret(fullName, "default", "svc")) expectEqual(t, fc, expectedHeadlessService(shortName)) o := stsOpts{ name: shortName, @@ -919,7 +919,7 @@ func TestCustomPriorityClassName(t *testing.T) { expectReconciled(t, sr, "default", "test") - fullName, shortName := findGenName(t, fc, "default", "test") + fullName, shortName := findGenName(t, fc, "default", "test", "svc") o := stsOpts{ name: shortName, secretName: fullName, @@ -969,9 +969,9 @@ func TestDefaultLoadBalancer(t *testing.T) { expectReconciled(t, sr, "default", "test") - fullName, shortName := findGenName(t, fc, "default", "test") + fullName, shortName := findGenName(t, fc, "default", "test", "svc") - expectEqual(t, fc, expectedSecret(fullName)) + // expectEqual(t, fc, expectedSecret(fullName, "default", "svc")) expectEqual(t, fc, expectedHeadlessService(shortName)) o := stsOpts{ name: shortName, @@ -1021,7 +1021,7 @@ func TestProxyFirewallMode(t *testing.T) { expectReconciled(t, sr, "default", "test") - fullName, shortName := findGenName(t, fc, "default", "test") + fullName, shortName := findGenName(t, fc, "default", "test", "svc") o := stsOpts{ name: shortName, secretName: fullName, @@ -1032,7 +1032,7 @@ func TestProxyFirewallMode(t *testing.T) { } -func expectedSecret(name string) *corev1.Secret { +func expectedSecret(name, parentNamespace, typ string) *corev1.Secret { return &corev1.Secret{ TypeMeta: metav1.TypeMeta{ Kind: "Secret", @@ -1044,8 +1044,8 @@ func expectedSecret(name string) *corev1.Secret { Labels: map[string]string{ "tailscale.com/managed": "true", "tailscale.com/parent-resource": "test", - "tailscale.com/parent-resource-ns": "default", - "tailscale.com/parent-resource-type": "svc", + "tailscale.com/parent-resource-ns": parentNamespace, + "tailscale.com/parent-resource-type": typ, }, }, StringData: map[string]string{ @@ -1178,20 +1178,20 @@ func expectedSTS(opts stsOpts) *appsv1.StatefulSet { } } -func findGenName(t *testing.T, client client.Client, ns, name string) (full, noSuffix string) { +func findGenName(t *testing.T, client client.Client, ns, name, typ string) (full, noSuffix string) { t.Helper() labels := map[string]string{ LabelManaged: "true", LabelParentName: name, LabelParentNamespace: ns, - LabelParentType: "svc", + LabelParentType: typ, } s, err := getSingleObject[corev1.Secret](context.Background(), client, "operator-ns", labels) if err != nil { t.Fatalf("finding secret for %q: %v", name, err) } if s == nil { - t.Fatalf("no secret found for %q", name) + t.Fatalf("no secret found for %q %s %+#v", name, ns, labels) } return s.GetName(), strings.TrimSuffix(s.GetName(), "-0") } @@ -1263,12 +1263,12 @@ func expectMissing[T any, O ptrObject[T]](t *testing.T, client client.Client, ns } } -func expectReconciled(t *testing.T, sr *ServiceReconciler, ns, name string) { +func expectReconciled(t *testing.T, sr reconcile.Reconciler, ns, name string) { t.Helper() req := reconcile.Request{ NamespacedName: types.NamespacedName{ - Name: name, Namespace: ns, + Name: name, }, } res, err := sr.Reconcile(context.Background(), req) @@ -1283,7 +1283,7 @@ func expectReconciled(t *testing.T, sr *ServiceReconciler, ns, name string) { } } -func expectRequeue(t *testing.T, sr *ServiceReconciler, ns, name string) { +func expectRequeue(t *testing.T, sr reconcile.Reconciler, ns, name string) { t.Helper() req := reconcile.Request{ NamespacedName: types.NamespacedName{ @@ -1295,9 +1295,6 @@ func expectRequeue(t *testing.T, sr *ServiceReconciler, ns, name string) { if err != nil { t.Fatalf("Reconcile: unexpected error: %v", err) } - if res.Requeue { - t.Fatalf("unexpected immediate requeue") - } if res.RequeueAfter == 0 { t.Fatalf("expected timed requeue, got success") } diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go index 042490e92..6e54d9741 100644 --- a/cmd/k8s-operator/sts.go +++ b/cmd/k8s-operator/sts.go @@ -78,6 +78,10 @@ type tailscaleSTSConfig struct { Hostname string Tags []string // if empty, use defaultTags + + // Routes is a list of CIDRs to pass via --advertise-routes flag + // Should only be set if this is config for subnetRouter + Routes string } type tailscaleSTSReconciler struct { @@ -415,6 +419,12 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S }, }, }) + } else if len(sts.Routes) > 0 { + container.Env = append(container.Env, corev1.EnvVar{ + Name: "TS_ROUTES", + Value: sts.Routes, + }) + } if a.tsFirewallMode != "" { container.Env = append(container.Env, corev1.EnvVar{ diff --git a/go.mod b/go.mod index ab1492d62..5f426d0fe 100644 --- a/go.mod +++ b/go.mod @@ -103,6 +103,7 @@ require ( k8s.io/client-go v0.28.2 nhooyr.io/websocket v1.8.7 sigs.k8s.io/controller-runtime v0.16.2 + sigs.k8s.io/controller-tools v0.13.0 sigs.k8s.io/yaml v1.3.0 software.sslmate.com/src/go-pkcs12 v0.2.1 ) @@ -111,6 +112,7 @@ require ( github.com/Microsoft/go-winio v0.6.1 // indirect github.com/dave/astrid v0.0.0-20170323122508-8c2895878b14 // indirect github.com/dave/brenda v1.1.0 // indirect + github.com/gobuffalo/flect v1.0.2 // indirect github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 // indirect github.com/gorilla/securecookie v1.1.1 // indirect ) @@ -325,7 +327,7 @@ require ( github.com/ssgreg/nlreturn/v2 v2.2.1 // indirect github.com/stbenjam/no-sprintf-host-port v0.1.1 // indirect github.com/stretchr/objx v0.5.0 // indirect - github.com/stretchr/testify v1.8.4 // indirect + github.com/stretchr/testify v1.8.4 github.com/subosito/gotenv v1.4.2 // indirect github.com/t-yuki/gocover-cobertura v0.0.0-20180217150009-aaee18c8195c // indirect github.com/tailscale/go-winio v0.0.0-20231025203758-c4f33415bf55 diff --git a/go.sum b/go.sum index b33551c1a..a0e74f6ab 100644 --- a/go.sum +++ b/go.sum @@ -363,6 +363,8 @@ github.com/go-toolsmith/typep v1.1.0 h1:fIRYDyF+JywLfqzyhdiHzRop/GQDxxNhLGQ6gFUN github.com/go-toolsmith/typep v1.1.0/go.mod h1:fVIw+7zjdsMxDA3ITWnH1yOiw1rnTQKCsF/sk2H/qig= github.com/go-xmlfmt/xmlfmt v1.1.2 h1:Nea7b4icn8s57fTx1M5AI4qQT5HEM3rVUO8MuE6g80U= github.com/go-xmlfmt/xmlfmt v1.1.2/go.mod h1:aUCEOzzezBEjDBbFBoSiya/gduyIiWYRP6CnSFIV8AM= +github.com/gobuffalo/flect v1.0.2 h1:eqjPGSo2WmjgY2XlpGwo2NXgL3RucAKo4k4qQMNA5sA= +github.com/gobuffalo/flect v1.0.2/go.mod h1:A5msMlrHtLqh9umBSnvabjsMrCcCpAyzglnDvkbYKHs= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee h1:s+21KNqlpePfkah2I+gwHF8xmJWRjooY+5248k6m4A0= @@ -706,8 +708,12 @@ github.com/nishanths/predeclared v0.2.2 h1:V2EPdZPliZymNAn79T8RkNApBjMmVKh5XRpLm github.com/nishanths/predeclared v0.2.2/go.mod h1:RROzoN6TnGQupbC+lqggsOlcgysk3LMK/HI84Mp280c= github.com/nunnatsa/ginkgolinter v0.11.2 h1:xzQpAsEyZe5F1RMy2Z5kn8UFCGiWfKqJOUd2ZzBXA4M= github.com/nunnatsa/ginkgolinter v0.11.2/go.mod h1:dJIGXYXbkBswqa/pIzG0QlVTTDSBMxDoCFwhsl4Uras= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= @@ -1417,6 +1423,8 @@ gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI= gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0/go.mod h1:WDnlLJ4WF5VGsH/HVa3CI79GS0ol3YnhVnKP89i0kNg= @@ -1485,6 +1493,8 @@ rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/controller-runtime v0.16.2 h1:mwXAVuEk3EQf478PQwQ48zGOXvW27UJc8NHktQVuIPU= sigs.k8s.io/controller-runtime v0.16.2/go.mod h1:vpMu3LpI5sYWtujJOa2uPK61nB5rbwlN7BAB8aSLvGU= +sigs.k8s.io/controller-tools v0.13.0 h1:NfrvuZ4bxyolhDBt/rCZhDnx3M2hzlhgo5n3Iv2RykI= +sigs.k8s.io/controller-tools v0.13.0/go.mod h1:5vw3En2NazbejQGCeWKRrE7q4P+CW8/klfVqP8QZkgA= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.3.0 h1:UZbZAZfX0wV2zr7YZorDz6GXROfDFj6LvqCRm4VUVKk= diff --git a/header.txt b/header.txt new file mode 100644 index 000000000..8111cb74e --- /dev/null +++ b/header.txt @@ -0,0 +1,2 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause \ No newline at end of file diff --git a/k8s-operator/apis/doc.go b/k8s-operator/apis/doc.go new file mode 100644 index 000000000..0b431cae1 --- /dev/null +++ b/k8s-operator/apis/doc.go @@ -0,0 +1,8 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package apis + +const GroupName = "tailscale.com" diff --git a/k8s-operator/apis/v1alpha1/doc.go b/k8s-operator/apis/v1alpha1/doc.go new file mode 100644 index 000000000..467e73e17 --- /dev/null +++ b/k8s-operator/apis/v1alpha1/doc.go @@ -0,0 +1,8 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +// +kubebuilder:object:generate=true +// +groupName=tailscale.com +package v1alpha1 diff --git a/k8s-operator/apis/v1alpha1/register.go b/k8s-operator/apis/v1alpha1/register.go new file mode 100644 index 000000000..d8929a9f5 --- /dev/null +++ b/k8s-operator/apis/v1alpha1/register.go @@ -0,0 +1,56 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package v1alpha1 + +import ( + "fmt" + + "tailscale.com/k8s-operator/apis" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes/scheme" +) + +// SchemeGroupVersion is group version used to register these objects +var SchemeGroupVersion = schema.GroupVersion{Group: apis.GroupName, Version: "v1alpha1"} + +// Resource takes an unqualified resource and returns a Group qualified GroupResource +func Resource(resource string) schema.GroupResource { + return SchemeGroupVersion.WithResource(resource).GroupResource() +} + +var ( + SchemeBuilder runtime.SchemeBuilder + localSchemeBuilder = &SchemeBuilder + AddToScheme = localSchemeBuilder.AddToScheme + + GlobalScheme *runtime.Scheme +) + +func init() { + // We only register manually written functions here. The registration of the + // generated functions takes place in the generated files. The separation + // makes the code compile even when the generated files are missing. + localSchemeBuilder.Register(addKnownTypes) + + GlobalScheme = runtime.NewScheme() + if err := scheme.AddToScheme(GlobalScheme); err != nil { + panic(fmt.Sprintf("failed to add k8s.io scheme: %s", err)) + } + if err := AddToScheme(GlobalScheme); err != nil { + panic(fmt.Sprintf("failed to add tailscale.com scheme: %s", err)) + } +} + +// Adds the list of known types to api.Scheme. +func addKnownTypes(scheme *runtime.Scheme) error { + scheme.AddKnownTypes(SchemeGroupVersion, &Connector{}, &ConnectorList{}) + + metav1.AddToGroupVersion(scheme, SchemeGroupVersion) + return nil +} diff --git a/k8s-operator/apis/v1alpha1/types_connector.go b/k8s-operator/apis/v1alpha1/types_connector.go new file mode 100644 index 000000000..9151ca8d1 --- /dev/null +++ b/k8s-operator/apis/v1alpha1/types_connector.go @@ -0,0 +1,155 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// Code comments on these types should be treated as user facing documentation- +// they will appear on the Connector CRD i.e if someone runs kubectl explain connector. + +var ConnectorKind = "Connector" + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:scope=Cluster,shortName=cn +// +kubebuilder:printcolumn:name="SubnetRoutes",type="string",JSONPath=`.status.subnetRouter.routes`,description="Cluster CIDR ranges exposed to tailnet via subnet router" +// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=`.status.conditions[?(@.type == "ConnectorReady")].reason`,description="Status of the components deployed by the connector" + +type Connector struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + // Desired state of the Connector resource. + Spec ConnectorSpec `json:"spec"` + + // Status of the Connector. This is set and managed by the Tailscale operator. + // +optional + Status ConnectorStatus `json:"status"` +} + +// +kubebuilder:object:root=true + +type ConnectorList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata"` + + Items []Connector `json:"items"` +} + +// ConnectorSpec defines the desired state of a ConnectorSpec. +type ConnectorSpec struct { + // SubnetRouter configures a Tailscale subnet router to be deployed in + // the cluster. If unset no subnet router will be deployed. + // https://tailscale.com/kb/1019/subnets/ + SubnetRouter *SubnetRouter `json:"subnetRouter"` +} + +// SubnetRouter describes a subnet router. +// +kubebuilder:validation:XValidation:rule="has(self.tags) == has(oldSelf.tags)",message="Subnetrouter tags cannot be changed. Delete and redeploy the Connector if you need to change it." +type SubnetRouter struct { + // Routes refer to in-cluster CIDRs that the subnet router should make + // available. Route values must be strings that represent a valid IPv4 + // or IPv6 CIDR range. Values can be Tailscale 4via6 subnet routes. + // https://tailscale.com/kb/1201/4via6-subnets/ + Routes []Route `json:"routes"` + // Tags that the Tailscale node will be tagged with. If you want the + // subnet router to be autoapproved, you can configure Tailscale ACLs to + // autoapprove the subnetrouter's CIDRs for these tags. + // See https://tailscale.com/kb/1018/acls/#auto-approvers-for-routes-and-exit-nodes + // Defaults to tag:k8s. + // If you specify custom tags here, you must also make tag:k8s-operator owner of the custom tag. + // See https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator. + // Tags cannot be changed once a Connector has been created. + // Tag values must be in form ^tag:[a-zA-Z][a-zA-Z0-9-]*$. + // +optional + Tags []Tag `json:"tags,omitempty"` + // Hostname is the tailnet hostname that should be assigned to the + // subnet router node. If unset hostname is defaulted to -subnetrouter. Hostname can contain lower case letters, numbers + // and dashes, it must not start or end with a dash and must be between + // 2 and 63 characters long. + // +optional + Hostname Hostname `json:"hostname,omitempty"` +} + +// +kubebuilder:validation:Type=string +// +kubebuilder:validation:Format=cidr +type Route string + +// +kubebuilder:validation:Type=string +// +kubebuilder:validation:Pattern=`^tag:[a-zA-Z][a-zA-Z0-9-]*$` +type Tag string + +// +kubebuilder:validation:Type=string +// +kubebuilder:validation:Pattern=`^[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$` +type Hostname string + +// ConnectorStatus defines the observed state of the Connector. +type ConnectorStatus struct { + + // List of status conditions to indicate the status of the Connector. + // Known condition types are `ConnectorReady`. + // +listType=map + // +listMapKey=type + // +optional + Conditions []ConnectorCondition `json:"conditions"` + // SubnetRouter status is the current status of a subnet router + // +optional + SubnetRouter *SubnetRouterStatus `json:"subnetRouter"` +} + +// SubnetRouter status is the current status of a subnet router if deployed +type SubnetRouterStatus struct { + // Routes are the CIDRs currently exposed via subnet router + Routes string `json:"routes"` + // Ready is the ready status of the subnet router + Ready metav1.ConditionStatus `json:"ready"` + // Reason is the reason for the subnet router status + Reason string `json:"reason"` + // Message is a more verbose reason for the current subnet router status + Message string `json:"message"` +} + +// ConnectorCondition contains condition information for a Connector. +type ConnectorCondition struct { + // Type of the condition, known values are (`SubnetRouterReady`). + Type ConnectorConditionType `json:"type"` + + // Status of the condition, one of ('True', 'False', 'Unknown'). + Status metav1.ConditionStatus `json:"status"` + + // LastTransitionTime is the timestamp corresponding to the last status + // change of this condition. + // +optional + LastTransitionTime *metav1.Time `json:"lastTransitionTime,omitempty"` + + // Reason is a brief machine readable explanation for the condition's last + // transition. + // +optional + Reason string `json:"reason,omitempty"` + + // Message is a human readable description of the details of the last + // transition, complementing reason. + // +optional + Message string `json:"message,omitempty"` + + // If set, this represents the .metadata.generation that the condition was + // set based upon. + // For instance, if .metadata.generation is currently 12, but the + // .status.condition[x].observedGeneration is 9, the condition is out of date + // with respect to the current state of the Connector. + // +optional + ObservedGeneration int64 `json:"observedGeneration,omitempty"` +} + +// ConnectorConditionType represents a Connector condition type +type ConnectorConditionType string + +const ( + ConnectorReady ConnectorConditionType = `ConnectorReady` +) diff --git a/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go b/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 000000000..068d68ada --- /dev/null +++ b/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,177 @@ +//go:build !ignore_autogenerated && !plan9 + +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Connector) DeepCopyInto(out *Connector) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Connector. +func (in *Connector) DeepCopy() *Connector { + if in == nil { + return nil + } + out := new(Connector) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Connector) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ConnectorCondition) DeepCopyInto(out *ConnectorCondition) { + *out = *in + if in.LastTransitionTime != nil { + in, out := &in.LastTransitionTime, &out.LastTransitionTime + *out = (*in).DeepCopy() + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConnectorCondition. +func (in *ConnectorCondition) DeepCopy() *ConnectorCondition { + if in == nil { + return nil + } + out := new(ConnectorCondition) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ConnectorList) DeepCopyInto(out *ConnectorList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Connector, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConnectorList. +func (in *ConnectorList) DeepCopy() *ConnectorList { + if in == nil { + return nil + } + out := new(ConnectorList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ConnectorList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ConnectorSpec) DeepCopyInto(out *ConnectorSpec) { + *out = *in + if in.SubnetRouter != nil { + in, out := &in.SubnetRouter, &out.SubnetRouter + *out = new(SubnetRouter) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConnectorSpec. +func (in *ConnectorSpec) DeepCopy() *ConnectorSpec { + if in == nil { + return nil + } + out := new(ConnectorSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ConnectorStatus) DeepCopyInto(out *ConnectorStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]ConnectorCondition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.SubnetRouter != nil { + in, out := &in.SubnetRouter, &out.SubnetRouter + *out = new(SubnetRouterStatus) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConnectorStatus. +func (in *ConnectorStatus) DeepCopy() *ConnectorStatus { + if in == nil { + return nil + } + out := new(ConnectorStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SubnetRouter) DeepCopyInto(out *SubnetRouter) { + *out = *in + if in.Routes != nil { + in, out := &in.Routes, &out.Routes + *out = make([]Route, len(*in)) + copy(*out, *in) + } + if in.Tags != nil { + in, out := &in.Tags, &out.Tags + *out = make([]Tag, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SubnetRouter. +func (in *SubnetRouter) DeepCopy() *SubnetRouter { + if in == nil { + return nil + } + out := new(SubnetRouter) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SubnetRouterStatus) DeepCopyInto(out *SubnetRouterStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SubnetRouterStatus. +func (in *SubnetRouterStatus) DeepCopy() *SubnetRouterStatus { + if in == nil { + return nil + } + out := new(SubnetRouterStatus) + in.DeepCopyInto(out) + return out +} diff --git a/k8s-operator/conditions.go b/k8s-operator/conditions.go new file mode 100644 index 000000000..f1344e34c --- /dev/null +++ b/k8s-operator/conditions.go @@ -0,0 +1,60 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package kube + +import ( + "slices" + + "go.uber.org/zap" + xslices "golang.org/x/exp/slices" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/tstime" +) + +// SetConnectorCondition ensures that Connector status has a condition with the +// given attributes. LastTransitionTime gets set every time condition's status +// changes +func SetConnectorCondition(cn *tsapi.Connector, conditionType tsapi.ConnectorConditionType, status metav1.ConditionStatus, reason, message string, gen int64, clock tstime.Clock, logger *zap.SugaredLogger) { + newCondition := tsapi.ConnectorCondition{ + Type: conditionType, + Status: status, + Reason: reason, + Message: message, + ObservedGeneration: gen, + } + + nowTime := metav1.NewTime(clock.Now()) + newCondition.LastTransitionTime = &nowTime + + idx := xslices.IndexFunc(cn.Status.Conditions, func(cond tsapi.ConnectorCondition) bool { + return cond.Type == conditionType + }) + + if idx == -1 { + cn.Status.Conditions = append(cn.Status.Conditions, newCondition) + return + } + + // Update the existing condition + cond := cn.Status.Conditions[idx] + // If this update doesn't contain a state transition, we don't update + // the conditions LastTransitionTime to Now() + if cond.Status == status { + newCondition.LastTransitionTime = cond.LastTransitionTime + } else { + logger.Info("Status change for Connector condition %s from %s to %s", conditionType, cond.Status, status) + } + + cn.Status.Conditions[idx] = newCondition +} + +// RemoveConnectorCondition will remove condition of the given type +func RemoveConnectorCondition(conn *tsapi.Connector, conditionType tsapi.ConnectorConditionType) { + conn.Status.Conditions = slices.DeleteFunc(conn.Status.Conditions, func(cond tsapi.ConnectorCondition) bool { + return cond.Type == conditionType + }) +} diff --git a/k8s-operator/conditions_test.go b/k8s-operator/conditions_test.go new file mode 100644 index 000000000..d7d8d6cd8 --- /dev/null +++ b/k8s-operator/conditions_test.go @@ -0,0 +1,102 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package kube + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/tstest" +) + +func TestSetConnectorCondition(t *testing.T) { + cn := tsapi.Connector{} + clock := tstest.NewClock(tstest.ClockOpts{}) + fakeNow := metav1.NewTime(clock.Now()) + fakePast := metav1.NewTime(clock.Now().Add(-5 * time.Minute)) + zl, err := zap.NewDevelopment() + assert.Nil(t, err) + + // Set up a new condition + SetConnectorCondition(&cn, tsapi.ConnectorReady, metav1.ConditionTrue, "someReason", "someMsg", 1, clock, zl.Sugar()) + assert.Equal(t, cn, tsapi.Connector{ + Status: tsapi.ConnectorStatus{ + Conditions: []tsapi.ConnectorCondition{ + { + Type: tsapi.ConnectorReady, + Status: metav1.ConditionTrue, + Reason: "someReason", + Message: "someMsg", + ObservedGeneration: 1, + LastTransitionTime: &fakeNow, + }, + }, + }, + }) + + // Modify status of an existing condition + cn.Status = tsapi.ConnectorStatus{ + Conditions: []tsapi.ConnectorCondition{ + { + Type: tsapi.ConnectorReady, + Status: metav1.ConditionFalse, + Reason: "someReason", + Message: "someMsg", + ObservedGeneration: 1, + LastTransitionTime: &fakePast, + }, + }, + } + SetConnectorCondition(&cn, tsapi.ConnectorReady, metav1.ConditionTrue, "anotherReason", "anotherMsg", 2, clock, zl.Sugar()) + assert.Equal(t, cn, tsapi.Connector{ + Status: tsapi.ConnectorStatus{ + Conditions: []tsapi.ConnectorCondition{ + { + Type: tsapi.ConnectorReady, + Status: metav1.ConditionTrue, + Reason: "anotherReason", + Message: "anotherMsg", + ObservedGeneration: 2, + LastTransitionTime: &fakeNow, + }, + }, + }, + }) + + // Don't modify last transition time if status hasn't changed + cn.Status = tsapi.ConnectorStatus{ + Conditions: []tsapi.ConnectorCondition{ + { + Type: tsapi.ConnectorReady, + Status: metav1.ConditionTrue, + Reason: "someReason", + Message: "someMsg", + ObservedGeneration: 1, + LastTransitionTime: &fakePast, + }, + }, + } + SetConnectorCondition(&cn, tsapi.ConnectorReady, metav1.ConditionTrue, "anotherReason", "anotherMsg", 2, clock, zl.Sugar()) + assert.Equal(t, cn, tsapi.Connector{ + Status: tsapi.ConnectorStatus{ + Conditions: []tsapi.ConnectorCondition{ + { + Type: tsapi.ConnectorReady, + Status: metav1.ConditionTrue, + Reason: "anotherReason", + Message: "anotherMsg", + ObservedGeneration: 2, + LastTransitionTime: &fakePast, + }, + }, + }, + }) + +} diff --git a/scripts/check_license_headers.sh b/scripts/check_license_headers.sh index bbb128e17..aedb83fd5 100755 --- a/scripts/check_license_headers.sh +++ b/scripts/check_license_headers.sh @@ -52,15 +52,21 @@ for file in $(find $1 \( -name '*.go' -or -name '*.tsx' -or -name '*.ts' -not -n $1/util/winutil/testdata/testrestartableprocesses/main.go) # Subprocess test harness code ;; + *$1/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go) + # Generated kube deepcopy funcs file starts with a Go build tag + an empty line + header="$(head -5 $file | tail -n+3 )" + ;; *) - header="$(head -2 $file)" + header="$(head -2 $file)" + ;; + esac + if [ ! -z "$header" ]; then if ! check_file "$header"; then fail=1 echo "${file#$1/} doesn't have the right copyright header:" echo "$header" | sed -e 's/^/ /g' fi - ;; - esac + fi done if [ $fail -ne 0 ]; then diff --git a/scripts/kube-deepcopy.sh b/scripts/kube-deepcopy.sh new file mode 100755 index 000000000..6e6330ee3 --- /dev/null +++ b/scripts/kube-deepcopy.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env sh + +set -eu + +./tool/go run sigs.k8s.io/controller-tools/cmd/controller-gen object:headerFile=./header.txt paths=./k8s-operator/apis/... + +# At the moment controller-gen does not support adding custom tags to generated +# files. We want to exclude all kube-related code from plan9 builds because some +# apimachinery libraries refer to syscalls that are not available for plan9 +# https://github.com/kubernetes/apimachinery/blob/v0.28.2/pkg/util/net/util.go#L42-L63 +sed -i "1 s|$| \&\& \!plan9|" k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go diff --git a/tstest/tools/tools.go b/tstest/tools/tools.go index ca92c5de5..3dda6e763 100644 --- a/tstest/tools/tools.go +++ b/tstest/tools/tools.go @@ -11,4 +11,5 @@ package tools import ( _ "github.com/tailscale/mkctr" _ "honnef.co/go/tools/cmd/staticcheck" + _ "sigs.k8s.io/controller-tools/cmd/controller-gen" )