cmd/k8s-operator: add Connector CRD to Helm chart and static manifests (#10775)

cmd/k8s-operator: add CRD to chart and static manifest

Add functionality to insert CRD to chart at package time.
Insert CRD to static manifests as this is where they are currently consumed from.

Signed-off-by: Irbe Krumina <irbe@tailscale.com>

cmd/k8s-operator/deploy/chart/values.yaml

@@ -9,11 +9,15 @@ oauth: {}
   # clientSecret: ""
 
 # enableConnector determines whether the operator should reconcile
-# connector.tailscale.com custom resources. If set to true you have to install
-# connector CRD in a separate step.
-# You can do so by running 'kubectl apply -f ./cmd/k8s-operator/deploy/crds'.
+# connector.tailscale.com custom resources.
 enableConnector: "false"
 
+# installCRDs determines whether tailscale.com CRDs should be installed as part
+# of chart installation. We do not use Helm's CRD installation mechanism as that
+# does not allow for upgrading CRDs.
+# https://helm.sh/docs/chart_best_practices/custom_resource_definitions/
+installCRDs: "true"
+
 operatorConfig:
   image:
     repo: tailscale/k8s-operator

cmd/k8s-operator/deploy/manifests/operator.yaml

@@ -27,6 +27,132 @@ metadata:
     name: proxies
     namespace: tailscale
 ---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+    annotations:
+        controller-gen.kubebuilder.io/version: v0.13.0
+    name: connectors.tailscale.com
+spec:
+    group: tailscale.com
+    names:
+        kind: Connector
+        listKind: ConnectorList
+        plural: connectors
+        shortNames:
+            - cn
+        singular: connector
+    scope: Cluster
+    versions:
+        - additionalPrinterColumns:
+            - description: CIDR ranges exposed to tailnet by a subnet router defined via this Connector instance.
+              jsonPath: .status.subnetRoutes
+              name: SubnetRoutes
+              type: string
+            - description: Whether this Connector instance defines an exit node.
+              jsonPath: .status.isExitNode
+              name: IsExitNode
+              type: string
+            - description: Status of the deployed Connector resources.
+              jsonPath: .status.conditions[?(@.type == "ConnectorReady")].reason
+              name: Status
+              type: string
+          name: v1alpha1
+          schema:
+            openAPIV3Schema:
+                properties:
+                    apiVersion:
+                        description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+                        type: string
+                    kind:
+                        description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+                        type: string
+                    metadata:
+                        type: object
+                    spec:
+                        description: ConnectorSpec describes the desired Tailscale component.
+                        properties:
+                            exitNode:
+                                description: ExitNode defines whether the Connector node should act as a Tailscale exit node. Defaults to false. https://tailscale.com/kb/1103/exit-nodes
+                                type: boolean
+                            hostname:
+                                description: Hostname is the tailnet hostname that should be assigned to the Connector node. If unset, hostname defaults to <connector name>-connector. Hostname can contain lower case letters, numbers and dashes, it must not start or end with a dash and must be between 2 and 63 characters long.
+                                pattern: ^[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$
+                                type: string
+                            subnetRouter:
+                                description: SubnetRouter defines subnet routes that the Connector node should expose to tailnet. If unset, none are exposed. https://tailscale.com/kb/1019/subnets/
+                                properties:
+                                    advertiseRoutes:
+                                        description: AdvertiseRoutes refer to CIDRs that the subnet router should make available. Route values must be strings that represent a valid IPv4 or IPv6 CIDR range. Values can be Tailscale 4via6 subnet routes. https://tailscale.com/kb/1201/4via6-subnets/
+                                        items:
+                                            format: cidr
+                                            type: string
+                                        minItems: 1
+                                        type: array
+                                required:
+                                    - advertiseRoutes
+                                type: object
+                            tags:
+                                description: Tags that the Tailscale node will be tagged with. Defaults to [tag:k8s]. To autoapprove the subnet routes or exit node defined by a Connector, you can configure Tailscale ACLs to give these tags the necessary permissions. See https://tailscale.com/kb/1018/acls/#auto-approvers-for-routes-and-exit-nodes. If you specify custom tags here, you must also make the operator an owner of these tags. See  https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator. Tags cannot be changed once a Connector node has been created. Tag values must be in form ^tag:[a-zA-Z][a-zA-Z0-9-]*$.
+                                items:
+                                    pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$
+                                    type: string
+                                type: array
+                        type: object
+                        x-kubernetes-validations:
+                            - message: A Connector needs to be either an exit node or a subnet router, or both.
+                              rule: has(self.subnetRouter) || self.exitNode == true
+                    status:
+                        description: ConnectorStatus describes the status of the Connector. This is set and managed by the Tailscale operator.
+                        properties:
+                            conditions:
+                                description: List of status conditions to indicate the status of the Connector. Known condition types are `ConnectorReady`.
+                                items:
+                                    description: ConnectorCondition contains condition information for a Connector.
+                                    properties:
+                                        lastTransitionTime:
+                                            description: LastTransitionTime is the timestamp corresponding to the last status change of this condition.
+                                            format: date-time
+                                            type: string
+                                        message:
+                                            description: Message is a human readable description of the details of the last transition, complementing reason.
+                                            type: string
+                                        observedGeneration:
+                                            description: If set, this represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.condition[x].observedGeneration is 9, the condition is out of date with respect to the current state of the Connector.
+                                            format: int64
+                                            type: integer
+                                        reason:
+                                            description: Reason is a brief machine readable explanation for the condition's last transition.
+                                            type: string
+                                        status:
+                                            description: Status of the condition, one of ('True', 'False', 'Unknown').
+                                            type: string
+                                        type:
+                                            description: Type of the condition, known values are (`SubnetRouterReady`).
+                                            type: string
+                                    required:
+                                        - status
+                                        - type
+                                    type: object
+                                type: array
+                                x-kubernetes-list-map-keys:
+                                    - type
+                                x-kubernetes-list-type: map
+                            isExitNode:
+                                description: IsExitNode is set to true if the Connector acts as an exit node.
+                                type: boolean
+                            subnetRoutes:
+                                description: SubnetRoutes are the routes currently exposed to tailnet via this Connector instance.
+                                type: string
+                        type: object
+                required:
+                    - spec
+                type: object
+          served: true
+          storage: true
+          subresources:
+            status: {}
+---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:

cmd/k8s-operator/generate/main.go

@@ -18,15 +18,49 @@ import (
 	"gopkg.in/yaml.v3"
 )
 
+const (
+	operatorDeploymentFilesPath = "cmd/k8s-operator/deploy"
+	crdPath                     = operatorDeploymentFilesPath + "/crds/tailscale.com_connectors.yaml"
+	helmTemplatesPath           = operatorDeploymentFilesPath + "/chart/templates"
+	crdTemplatePath             = helmTemplatesPath + "/connectors.yaml"
+
+	helmConditionalStart = "{{ if .Values.installCRDs -}}\n"
+	helmConditionalEnd   = "{{- end -}}"
+)
+
 func main() {
+	if len(os.Args) < 2 {
+		log.Fatalf("usage ./generate [staticmanifests|helmcrd]")
+	}
 	repoRoot := "../../"
-	cmd := exec.Command("./tool/helm", "template", "operator", "./cmd/k8s-operator/deploy/chart",
+	switch os.Args[1] {
+	case "helmcrd": // insert CRD to Helm templates behind a installCRDs=true conditional check
+		log.Print("Adding Connector CRD to Helm templates")
+		if err := generate("./"); err != nil {
+			log.Fatalf("error adding Connector CRD to Helm templates: %v", err)
+		}
+		return
+	case "staticmanifests": // generate static manifests from Helm templates (including the CRD)
+	default:
+		log.Fatalf("unknown option %s, known options are 'staticmanifests', 'helmcrd'", os.Args[1])
+	}
+	log.Printf("Inserting CRD into the Helm templates")
+	if err := generate(repoRoot); err != nil {
+		log.Fatalf("error adding Connector CRD to Helm templates: %v", err)
+	}
+	defer func() {
+		if err := cleanup(repoRoot); err != nil {
+			log.Fatalf("error cleaning up generated resources")
+		}
+	}()
+	log.Print("Templating Helm chart contents")
+	helmTmplCmd := exec.Command("./tool/helm", "template", "operator", "./cmd/k8s-operator/deploy/chart",
 		"--namespace=tailscale")
-	cmd.Dir = repoRoot
+	helmTmplCmd.Dir = repoRoot
 	var out bytes.Buffer
-	cmd.Stdout = &out
-	cmd.Stderr = os.Stderr
-	if err := cmd.Run(); err != nil {
+	helmTmplCmd.Stdout = &out
+	helmTmplCmd.Stderr = os.Stderr
+	if err := helmTmplCmd.Run(); err != nil {
 		log.Fatalf("error templating helm manifests: %v", err)
 	}
 
@@ -54,7 +88,6 @@ func main() {
 		if err != nil {
 			log.Fatalf("failed read from input data: %v", err)
 		}
-
 		bytes, err := yaml.Marshal(document)
 		if err != nil {
 			log.Fatalf("failed to marshal YAML document: %v", err)
@@ -72,3 +105,35 @@ func main() {
 		log.Fatalf("error writing new file: %v", err)
 	}
 }
+
+func generate(baseDir string) error {
+	log.Print("Placing Connector CRD into Helm templates..")
+	chartBytes, err := os.ReadFile(filepath.Join(baseDir, crdPath))
+	if err != nil {
+		return fmt.Errorf("error reading CRD contents: %w", err)
+	}
+	// Place a new temporary Helm template file with the templated CRD
+	// contents into Helm templates.
+	file, err := os.Create(filepath.Join(baseDir, crdTemplatePath))
+	if err != nil {
+		return fmt.Errorf("error creating CRD template file: %w", err)
+	}
+	if _, err := file.Write([]byte(helmConditionalStart)); err != nil {
+		return fmt.Errorf("error writing helm if statement start: %w", err)
+	}
+	if _, err := file.Write(chartBytes); err != nil {
+		return fmt.Errorf("error writing chart bytes: %w", err)
+	}
+	if _, err := file.Write([]byte(helmConditionalEnd)); err != nil {
+		return fmt.Errorf("error writing helm if-statement end: %w", err)
+	}
+	return nil
+}
+
+func cleanup(baseDir string) error {
+	log.Print("Cleaning up CRD from Helm templates")
+	if err := os.Remove(filepath.Join(baseDir, crdTemplatePath)); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("error cleaning up CRD template: %w", err)
+	}
+	return nil
+}

cmd/k8s-operator/generate/main_test.go

@@ -0,0 +1,68 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9 && !windows
+
+package main
+
+import (
+	"bytes"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func Test_generate(t *testing.T) {
+	base, err := os.Getwd()
+	base = filepath.Join(base, "../../../")
+	if err != nil {
+		t.Fatalf("error getting current working directory: %v", err)
+	}
+	defer cleanup(base)
+	if err := generate(base); err != nil {
+		t.Fatalf("CRD template generation: %v", err)
+	}
+
+	tempDir := t.TempDir()
+	helmCLIPath := filepath.Join(base, "tool/helm")
+	helmChartTemplatesPath := filepath.Join(base, "cmd/k8s-operator/deploy/chart")
+	helmPackageCmd := exec.Command(helmCLIPath, "package", helmChartTemplatesPath, "--destination", tempDir, "--version", "0.0.1")
+	helmPackageCmd.Stderr = os.Stderr
+	helmPackageCmd.Stdout = os.Stdout
+	if err := helmPackageCmd.Run(); err != nil {
+		t.Fatalf("error packaging Helm chart: %v", err)
+	}
+	helmPackagePath := filepath.Join(tempDir, "tailscale-operator-0.0.1.tgz")
+	helmLintCmd := exec.Command(helmCLIPath, "lint", helmPackagePath)
+	helmLintCmd.Stderr = os.Stderr
+	helmLintCmd.Stdout = os.Stdout
+	if err := helmLintCmd.Run(); err != nil {
+		t.Fatalf("Helm chart linter failed: %v", err)
+	}
+
+	// Test that default Helm install contains the CRD
+	installContentsWithCRD := bytes.NewBuffer([]byte{})
+	helmTemplateWithCRDCmd := exec.Command(helmCLIPath, "template", helmPackagePath)
+	helmTemplateWithCRDCmd.Stderr = os.Stderr
+	helmTemplateWithCRDCmd.Stdout = installContentsWithCRD
+	if err := helmTemplateWithCRDCmd.Run(); err != nil {
+		t.Fatalf("templating Helm chart with CRDs failed: %v", err)
+	}
+	if !strings.Contains(installContentsWithCRD.String(), "name: connectors.tailscale.com") {
+		t.Errorf("CRD not found in default chart install")
+	}
+
+	// Test that CRD can be excluded from Helm chart install
+	installContentsWithoutCRD := bytes.NewBuffer([]byte{})
+	helmTemplateWithoutCRDCmd := exec.Command(helmCLIPath, "template", helmPackagePath, "--set", "installCRDs=false")
+	helmTemplateWithoutCRDCmd.Stderr = os.Stderr
+	helmTemplateWithoutCRDCmd.Stdout = installContentsWithoutCRD
+	if err := helmTemplateWithoutCRDCmd.Run(); err != nil {
+		t.Fatalf("templating Helm chart without CRDs failed: %v", err)
+	}
+	if strings.Contains(installContentsWithoutCRD.String(), "name: connectors.tailscale.com") {
+		t.Errorf("CRD found in chart install that should not contain a CRD")
+	}
+}

cmd/k8s-operator/operator.go

@@ -45,7 +45,7 @@ import (
 )
 
 // Generate static manifests for deploying Tailscale operator on Kubernetes from the operator's Helm chart.
-//go:generate go run tailscale.com/cmd/k8s-operator/generate
+//go:generate go run tailscale.com/cmd/k8s-operator/generate staticmanifests
 
 // Generate Connector CustomResourceDefinition yaml from its Go types.
 //go:generate go run sigs.k8s.io/controller-tools/cmd/controller-gen crd schemapatch:manifests=./deploy/crds output:dir=./deploy/crds paths=../../k8s-operator/apis/...