From 49fd0a62c9bd24eaed7e612d1eeb3ca4b89575fe Mon Sep 17 00:00:00 2001 From: Irbe Krumina Date: Mon, 4 Dec 2023 10:18:07 +0000 Subject: [PATCH] cmd/k8s-operator: generate static kube manifests from the Helm chart. (#10436) * cmd/k8s-operator: generate static manifests from Helm charts This is done to ensure that there is a single source of truth for the operator kube manifests. Also adds linux node selector to the static manifests as this was added as a default to the Helm chart. Static manifests can now be generated by running `go generate tailscale.com/cmd/k8s-operator`. Updates tailscale/tailscale#9222 Signed-off-by: Irbe Krumina --- .github/workflows/kubemanifests.yaml | 6 + .github/workflows/test.yml | 2 +- cmd/k8s-operator/deploy/README.md | 12 + .../deploy/manifests/operator.yaml | 251 ++++++++++-------- .../deploy/manifests/templates/01-header.yaml | 3 + .../manifests/templates/02-namespace.yaml | 5 + .../deploy/manifests/templates/03-secret.yaml | 9 + cmd/k8s-operator/generate/main.go | 74 ++++++ cmd/k8s-operator/operator.go | 2 + go.mod | 2 +- 10 files changed, 247 insertions(+), 119 deletions(-) create mode 100644 cmd/k8s-operator/deploy/README.md create mode 100644 cmd/k8s-operator/deploy/manifests/templates/01-header.yaml create mode 100644 cmd/k8s-operator/deploy/manifests/templates/02-namespace.yaml create mode 100644 cmd/k8s-operator/deploy/manifests/templates/03-secret.yaml create mode 100644 cmd/k8s-operator/generate/main.go diff --git a/.github/workflows/kubemanifests.yaml b/.github/workflows/kubemanifests.yaml index 90902ff75..6f51fb98c 100644 --- a/.github/workflows/kubemanifests.yaml +++ b/.github/workflows/kubemanifests.yaml @@ -22,3 +22,9 @@ jobs: eval `./tool/go run ./cmd/mkversion` ./tool/helm package --app-version="${VERSION_SHORT}" --version=${VERSION_SHORT} './cmd/k8s-operator/deploy/chart' ./tool/helm lint "tailscale-operator-${VERSION_SHORT}.tgz" + - name: Verify that static manifests are up to date + run: | + ./tool/go generate tailscale.com/cmd/k8s-operator + echo + echo + git diff --name-only --exit-code || (echo "Static manifests for Tailscale Kubernetes operator are out of date. Please run 'go generate tailscale.com/cmd/k8s-operator' and commit the diff."; exit 1) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2fd188928..d19b7e27f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -429,7 +429,7 @@ jobs: uses: actions/checkout@v4 - name: check that 'go generate' is clean run: | - pkgs=$(./tool/go list ./... | grep -v dnsfallback) + pkgs=$(./tool/go list ./... | grep -Ev 'dnsfallback|k8s-operator') ./tool/go generate $pkgs echo echo diff --git a/cmd/k8s-operator/deploy/README.md b/cmd/k8s-operator/deploy/README.md new file mode 100644 index 000000000..516d6f9cd --- /dev/null +++ b/cmd/k8s-operator/deploy/README.md @@ -0,0 +1,12 @@ +# Tailscale Kubernetes operator deployment manifests + +./cmd/k8s-operator/deploy contain various Tailscale Kubernetes operator deployment manifests. + +## Helm chart + +`./cmd/k8s-operator/deploy/chart` contains Tailscale operator Helm chart templates. +The chart templates are also used to generate the static manifest, so developers must ensure that any changes applied to the chart have been propagated to the static manifest by running `go generate tailscale.com/cmd/k8s-operator` + +## Static manifests + +`./cmd/k8s-operator/deploy/manifests/operator.yaml` is a static manifest for the operator generated from the Helm chart templates for the operator. \ No newline at end of file diff --git a/cmd/k8s-operator/deploy/manifests/operator.yaml b/cmd/k8s-operator/deploy/manifests/operator.yaml index e2f98c146..73a3895f6 100644 --- a/cmd/k8s-operator/deploy/manifests/operator.yaml +++ b/cmd/k8s-operator/deploy/manifests/operator.yaml @@ -7,155 +7,172 @@ metadata: name: tailscale --- apiVersion: v1 -kind: ServiceAccount -metadata: - name: proxies - namespace: tailscale ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role +kind: Secret metadata: - name: proxies + name: operator-oauth namespace: tailscale -rules: -- apiGroups: [""] - resources: ["secrets"] - verbs: ["*"] +stringData: + client_id: # SET CLIENT ID HERE + client_secret: # SET CLIENT SECRET HERE --- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding +apiVersion: v1 +kind: ServiceAccount metadata: - name: proxies - namespace: tailscale -subjects: -- kind: ServiceAccount - name: proxies - namespace: tailscale -roleRef: - kind: Role - name: proxies - apiGroup: rbac.authorization.k8s.io + name: operator + namespace: tailscale --- apiVersion: v1 kind: ServiceAccount metadata: - name: operator - namespace: tailscale + name: proxies + namespace: tailscale --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: tailscale-operator + name: tailscale-operator rules: -- apiGroups: [""] - resources: ["events", "services", "services/status"] - verbs: ["*"] -- apiGroups: ["networking.k8s.io"] - resources: ["ingresses", "ingresses/status"] - verbs: ["*"] + - apiGroups: + - "" + resources: + - events + - services + - services/status + verbs: + - '*' + - apiGroups: + - networking.k8s.io + resources: + - ingresses + - ingresses/status + verbs: + - '*' --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: tailscale-operator -subjects: -- kind: ServiceAccount - name: operator - namespace: tailscale + name: tailscale-operator roleRef: - kind: ClusterRole - name: tailscale-operator - apiGroup: rbac.authorization.k8s.io + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: tailscale-operator +subjects: + - kind: ServiceAccount + name: operator + namespace: tailscale --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - name: operator - namespace: tailscale + name: operator + namespace: tailscale +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - '*' + - apiGroups: + - apps + resources: + - statefulsets + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: proxies + namespace: tailscale rules: -- apiGroups: [""] - resources: ["secrets"] - verbs: ["*"] -- apiGroups: ["apps"] - resources: ["statefulsets"] - verbs: ["*"] + - apiGroups: + - "" + resources: + - secrets + verbs: + - '*' --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: operator - namespace: tailscale -subjects: -- kind: ServiceAccount - name: operator - namespace: tailscale + name: operator + namespace: tailscale roleRef: - kind: Role - name: operator - apiGroup: rbac.authorization.k8s.io + apiGroup: rbac.authorization.k8s.io + kind: Role + name: operator +subjects: + - kind: ServiceAccount + name: operator + namespace: tailscale --- -apiVersion: v1 -kind: Secret +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding metadata: - name: operator-oauth - namespace: tailscale -stringData: - client_id: # SET CLIENT ID HERE - client_secret: # SET CLIENT SECRET HERE + name: proxies + namespace: tailscale +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: proxies +subjects: + - kind: ServiceAccount + name: proxies + namespace: tailscale --- apiVersion: apps/v1 kind: Deployment metadata: - name: operator - namespace: tailscale + name: operator + namespace: tailscale spec: - replicas: 1 - strategy: - type: Recreate - selector: - matchLabels: - app: operator - template: - metadata: - labels: - app: operator - spec: - serviceAccountName: operator - volumes: - - name: oauth - secret: - secretName: operator-oauth - containers: - - name: operator - image: tailscale/k8s-operator:unstable - resources: - requests: - cpu: 500m - memory: 100Mi - env: - - name: OPERATOR_HOSTNAME - value: tailscale-operator - - name: OPERATOR_SECRET - value: operator - - name: OPERATOR_LOGGING - value: info - - name: OPERATOR_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: CLIENT_ID_FILE - value: /oauth/client_id - - name: CLIENT_SECRET_FILE - value: /oauth/client_secret - - name: PROXY_IMAGE - value: tailscale/tailscale:unstable - - name: PROXY_TAGS - value: tag:k8s - - name: APISERVER_PROXY - value: "false" - - name: PROXY_FIREWALL_MODE - value: auto - volumeMounts: - - name: oauth - mountPath: /oauth - readOnly: true + replicas: 1 + selector: + matchLabels: + app: operator + strategy: + type: Recreate + template: + metadata: + labels: + app: operator + spec: + containers: + - env: + - name: OPERATOR_HOSTNAME + value: tailscale-operator + - name: OPERATOR_SECRET + value: operator + - name: OPERATOR_LOGGING + value: info + - name: OPERATOR_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: CLIENT_ID_FILE + value: /oauth/client_id + - name: CLIENT_SECRET_FILE + value: /oauth/client_secret + - name: PROXY_IMAGE + value: tailscale/tailscale:unstable + - name: PROXY_TAGS + value: tag:k8s + - name: APISERVER_PROXY + value: "false" + - name: PROXY_FIREWALL_MODE + value: auto + image: tailscale/k8s-operator:unstable + imagePullPolicy: Always + name: operator + volumeMounts: + - mountPath: /oauth + name: oauth + readOnly: true + nodeSelector: + kubernetes.io/os: linux + serviceAccountName: operator + volumes: + - name: oauth + secret: + secretName: operator-oauth diff --git a/cmd/k8s-operator/deploy/manifests/templates/01-header.yaml b/cmd/k8s-operator/deploy/manifests/templates/01-header.yaml new file mode 100644 index 000000000..a96d4c37e --- /dev/null +++ b/cmd/k8s-operator/deploy/manifests/templates/01-header.yaml @@ -0,0 +1,3 @@ +# Copyright (c) Tailscale Inc & AUTHORS +# SPDX-License-Identifier: BSD-3-Clause + diff --git a/cmd/k8s-operator/deploy/manifests/templates/02-namespace.yaml b/cmd/k8s-operator/deploy/manifests/templates/02-namespace.yaml new file mode 100644 index 000000000..04d4cbcb3 --- /dev/null +++ b/cmd/k8s-operator/deploy/manifests/templates/02-namespace.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: tailscale +--- diff --git a/cmd/k8s-operator/deploy/manifests/templates/03-secret.yaml b/cmd/k8s-operator/deploy/manifests/templates/03-secret.yaml new file mode 100644 index 000000000..0793a2458 --- /dev/null +++ b/cmd/k8s-operator/deploy/manifests/templates/03-secret.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Secret +metadata: + name: operator-oauth + namespace: tailscale +stringData: + client_id: # SET CLIENT ID HERE + client_secret: # SET CLIENT SECRET HERE +--- diff --git a/cmd/k8s-operator/generate/main.go b/cmd/k8s-operator/generate/main.go new file mode 100644 index 000000000..d5ec08ab9 --- /dev/null +++ b/cmd/k8s-operator/generate/main.go @@ -0,0 +1,74 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package main + +import ( + "bytes" + "fmt" + "io" + "log" + "os" + "os/exec" + "path/filepath" + "strings" + + "gopkg.in/yaml.v3" +) + +func main() { + repoRoot := "../../" + cmd := exec.Command("./tool/helm", "template", "operator", "./cmd/k8s-operator/deploy/chart", + "--namespace=tailscale") + cmd.Dir = repoRoot + var out bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + log.Fatalf("error templating helm manifests: %v", err) + } + + var final bytes.Buffer + + templatePath := filepath.Join(repoRoot, "cmd/k8s-operator/deploy/manifests/templates") + fileInfos, err := os.ReadDir(templatePath) + if err != nil { + log.Fatalf("error reading templates: %v", err) + } + for _, fi := range fileInfos { + templateBytes, err := os.ReadFile(filepath.Join(templatePath, fi.Name())) + if err != nil { + log.Fatalf("error reading template: %v", err) + } + final.Write(templateBytes) + } + decoder := yaml.NewDecoder(&out) + for { + var document any + err := decoder.Decode(&document) + if err == io.EOF { + break + } + if err != nil { + log.Fatalf("failed read from input data: %v", err) + } + + bytes, err := yaml.Marshal(document) + if err != nil { + log.Fatalf("failed to marshal YAML document: %v", err) + } + if strings.TrimSpace(string(bytes)) == "null" { + continue + } + if _, err = final.Write(bytes); err != nil { + log.Fatalf("error marshaling yaml: %v", err) + } + fmt.Fprint(&final, "---\n") + } + finalString, _ := strings.CutSuffix(final.String(), "---\n") + if err := os.WriteFile(filepath.Join(repoRoot, "cmd/k8s-operator/deploy/manifests/operator.yaml"), []byte(finalString), 0664); err != nil { + log.Fatalf("error writing new file: %v", err) + } +} diff --git a/cmd/k8s-operator/operator.go b/cmd/k8s-operator/operator.go index 49af08a47..73b039b4d 100644 --- a/cmd/k8s-operator/operator.go +++ b/cmd/k8s-operator/operator.go @@ -42,6 +42,8 @@ import ( "tailscale.com/version" ) +//go:generate go run tailscale.com/cmd/k8s-operator/generate + func main() { // Required to use our client API. We're fine with the instability since the // client lives in the same repo as this code. diff --git a/go.mod b/go.mod index df60c7e1e..964a19b20 100644 --- a/go.mod +++ b/go.mod @@ -358,7 +358,7 @@ require ( gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect + gopkg.in/yaml.v3 v3.0.1 howett.net/plist v1.0.0 // indirect k8s.io/apiextensions-apiserver v0.28.2 // indirect k8s.io/component-base v0.28.2 // indirect