cmd/k8s-operator: generate static kube manifests from the Helm chart. (#10436)

* cmd/k8s-operator: generate static manifests from Helm charts

This is done to ensure that there is a single source of truth
for the operator kube manifests.
Also adds linux node selector to the static manifests as
this was added as a default to the Helm chart.

Static manifests can now be generated by running
`go generate tailscale.com/cmd/k8s-operator`.

Updates tailscale/tailscale#9222

Signed-off-by: Irbe Krumina <irbe@tailscale.com>

.github/workflows/kubemanifests.yaml

@@ -22,3 +22,9 @@ jobs:
         eval `./tool/go run ./cmd/mkversion`
         ./tool/helm package --app-version="${VERSION_SHORT}" --version=${VERSION_SHORT} './cmd/k8s-operator/deploy/chart'
         ./tool/helm lint "tailscale-operator-${VERSION_SHORT}.tgz"
+    - name: Verify that static manifests are up to date
+      run: |
+        ./tool/go generate tailscale.com/cmd/k8s-operator
+        echo
+        echo
+        git diff --name-only --exit-code || (echo "Static manifests for Tailscale Kubernetes operator are out of date. Please run 'go generate tailscale.com/cmd/k8s-operator' and commit the diff."; exit 1)

.github/workflows/test.yml

@@ -429,7 +429,7 @@ jobs:
       uses: actions/checkout@v4
     - name: check that 'go generate' is clean
       run: |
-        pkgs=$(./tool/go list ./... | grep -v dnsfallback)
+        pkgs=$(./tool/go list ./... | grep -Ev 'dnsfallback|k8s-operator')
         ./tool/go generate $pkgs
         echo
         echo

cmd/k8s-operator/deploy/README.md

@@ -0,0 +1,12 @@
+# Tailscale Kubernetes operator deployment manifests
+
+./cmd/k8s-operator/deploy contain various Tailscale Kubernetes operator deployment manifests.
+
+## Helm chart
+
+`./cmd/k8s-operator/deploy/chart` contains Tailscale operator Helm chart templates.
+The chart templates are also used to generate the static manifest, so developers must ensure that any changes applied to the chart have been propagated to the static manifest by running `go generate tailscale.com/cmd/k8s-operator`
+
+## Static manifests
+
+`./cmd/k8s-operator/deploy/manifests/operator.yaml` is a static manifest for the operator generated from the Helm chart templates for the operator.

cmd/k8s-operator/deploy/manifests/operator.yaml

@@ -7,39 +7,24 @@ metadata:
   name: tailscale
 ---
 apiVersion: v1
-kind: ServiceAccount
+kind: Secret
-metadata:
-  name: proxies
-  namespace: tailscale
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
 metadata:
-  name: proxies
+  name: operator-oauth
   namespace: tailscale
-rules:
+stringData:
-- apiGroups: [""]
+  client_id: # SET CLIENT ID HERE
-  resources: ["secrets"]
+  client_secret: # SET CLIENT SECRET HERE
-  verbs: ["*"]
 ---
-apiVersion: rbac.authorization.k8s.io/v1
+apiVersion: v1
-kind: RoleBinding
+kind: ServiceAccount
 metadata:
-  name: proxies
+    name: operator
-  namespace: tailscale
-subjects:
-- kind: ServiceAccount
-  name: proxies
     namespace: tailscale
-roleRef:
-  kind: Role
-  name: proxies
-  apiGroup: rbac.authorization.k8s.io
 ---
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name: operator
+    name: proxies
     namespace: tailscale
 ---
 apiVersion: rbac.authorization.k8s.io/v1
@@ -47,25 +32,34 @@ kind: ClusterRole
 metadata:
     name: tailscale-operator
 rules:
-- apiGroups: [""]
+    - apiGroups:
-  resources: ["events", "services", "services/status"]
+        - ""
-  verbs: ["*"]
+      resources:
-- apiGroups: ["networking.k8s.io"]
+        - events
-  resources: ["ingresses", "ingresses/status"]
+        - services
-  verbs: ["*"]
+        - services/status
+      verbs:
+        - '*'
+    - apiGroups:
+        - networking.k8s.io
+      resources:
+        - ingresses
+        - ingresses/status
+      verbs:
+        - '*'
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
     name: tailscale-operator
-subjects:
-- kind: ServiceAccount
-  name: operator
-  namespace: tailscale
 roleRef:
+    apiGroup: rbac.authorization.k8s.io
     kind: ClusterRole
     name: tailscale-operator
-  apiGroup: rbac.authorization.k8s.io
+subjects:
+    - kind: ServiceAccount
+      name: operator
+      namespace: tailscale
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
@@ -73,35 +67,59 @@ metadata:
     name: operator
     namespace: tailscale
 rules:
-- apiGroups: [""]
+    - apiGroups:
-  resources: ["secrets"]
+        - ""
-  verbs: ["*"]
+      resources:
-- apiGroups: ["apps"]
+        - secrets
-  resources: ["statefulsets"]
+      verbs:
-  verbs: ["*"]
+        - '*'
+    - apiGroups:
+        - apps
+      resources:
+        - statefulsets
+      verbs:
+        - '*'
 ---
 apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
+kind: Role
 metadata:
-  name: operator
+    name: proxies
     namespace: tailscale
-subjects:
+rules:
-- kind: ServiceAccount
+    - apiGroups:
+        - ""
+      resources:
+        - secrets
+      verbs:
+        - '*'
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
     name: operator
     namespace: tailscale
 roleRef:
+    apiGroup: rbac.authorization.k8s.io
     kind: Role
     name: operator
-  apiGroup: rbac.authorization.k8s.io
+subjects:
+    - kind: ServiceAccount
+      name: operator
+      namespace: tailscale
 ---
-apiVersion: v1
+apiVersion: rbac.authorization.k8s.io/v1
-kind: Secret
+kind: RoleBinding
 metadata:
-  name: operator-oauth
+    name: proxies
+    namespace: tailscale
+roleRef:
+    apiGroup: rbac.authorization.k8s.io
+    kind: Role
+    name: proxies
+subjects:
+    - kind: ServiceAccount
+      name: proxies
       namespace: tailscale
-stringData:
-  client_id: # SET CLIENT ID HERE
-  client_secret: # SET CLIENT SECRET HERE
 ---
 apiVersion: apps/v1
 kind: Deployment
@@ -110,29 +128,18 @@ metadata:
     namespace: tailscale
 spec:
     replicas: 1
-  strategy:
-    type: Recreate
     selector:
         matchLabels:
             app: operator
+    strategy:
+        type: Recreate
     template:
         metadata:
             labels:
                 app: operator
         spec:
-      serviceAccountName: operator
-      volumes:
-      - name: oauth
-        secret:
-          secretName: operator-oauth
             containers:
-        - name: operator
+                - env:
-          image: tailscale/k8s-operator:unstable
-          resources:
-            requests:
-              cpu: 500m
-              memory: 100Mi
-          env:
                     - name: OPERATOR_HOSTNAME
                       value: tailscale-operator
                     - name: OPERATOR_SECRET
@@ -155,7 +162,17 @@ spec:
                       value: "false"
                     - name: PROXY_FIREWALL_MODE
                       value: auto
+                  image: tailscale/k8s-operator:unstable
+                  imagePullPolicy: Always
+                  name: operator
                   volumeMounts:
-          - name: oauth
+                    - mountPath: /oauth
-            mountPath: /oauth
+                      name: oauth
                       readOnly: true
+            nodeSelector:
+                kubernetes.io/os: linux
+            serviceAccountName: operator
+            volumes:
+                - name: oauth
+                  secret:
+                    secretName: operator-oauth

cmd/k8s-operator/deploy/manifests/templates/01-header.yaml

@@ -0,0 +1,3 @@
+# Copyright (c) Tailscale Inc & AUTHORS
+# SPDX-License-Identifier: BSD-3-Clause
+

cmd/k8s-operator/deploy/manifests/templates/02-namespace.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: tailscale
+---

cmd/k8s-operator/deploy/manifests/templates/03-secret.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: operator-oauth
+  namespace: tailscale
+stringData:
+  client_id: # SET CLIENT ID HERE
+  client_secret: # SET CLIENT SECRET HERE
+---

cmd/k8s-operator/generate/main.go

@@ -0,0 +1,74 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"gopkg.in/yaml.v3"
+)
+
+func main() {
+	repoRoot := "../../"
+	cmd := exec.Command("./tool/helm", "template", "operator", "./cmd/k8s-operator/deploy/chart",
+		"--namespace=tailscale")
+	cmd.Dir = repoRoot
+	var out bytes.Buffer
+	cmd.Stdout = &out
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		log.Fatalf("error templating helm manifests: %v", err)
+	}
+
+	var final bytes.Buffer
+
+	templatePath := filepath.Join(repoRoot, "cmd/k8s-operator/deploy/manifests/templates")
+	fileInfos, err := os.ReadDir(templatePath)
+	if err != nil {
+		log.Fatalf("error reading templates: %v", err)
+	}
+	for _, fi := range fileInfos {
+		templateBytes, err := os.ReadFile(filepath.Join(templatePath, fi.Name()))
+		if err != nil {
+			log.Fatalf("error reading template: %v", err)
+		}
+		final.Write(templateBytes)
+	}
+	decoder := yaml.NewDecoder(&out)
+	for {
+		var document any
+		err := decoder.Decode(&document)
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			log.Fatalf("failed read from input data: %v", err)
+		}
+
+		bytes, err := yaml.Marshal(document)
+		if err != nil {
+			log.Fatalf("failed to marshal YAML document: %v", err)
+		}
+		if strings.TrimSpace(string(bytes)) == "null" {
+			continue
+		}
+		if _, err = final.Write(bytes); err != nil {
+			log.Fatalf("error marshaling yaml: %v", err)
+		}
+		fmt.Fprint(&final, "---\n")
+	}
+	finalString, _ := strings.CutSuffix(final.String(), "---\n")
+	if err := os.WriteFile(filepath.Join(repoRoot, "cmd/k8s-operator/deploy/manifests/operator.yaml"), []byte(finalString), 0664); err != nil {
+		log.Fatalf("error writing new file: %v", err)
+	}
+}

cmd/k8s-operator/operator.go

@@ -42,6 +42,8 @@ import (
 	"tailscale.com/version"
 )
 
+//go:generate go run tailscale.com/cmd/k8s-operator/generate
+
 func main() {
 	// Required to use our client API. We're fine with the instability since the
 	// client lives in the same repo as this code.

go.mod

@@ -358,7 +358,7 @@ require (
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
+	gopkg.in/yaml.v3 v3.0.1
 	howett.net/plist v1.0.0 // indirect
 	k8s.io/apiextensions-apiserver v0.28.2 // indirect
 	k8s.io/component-base v0.28.2 // indirect