From 5bd19fd3e30c40721680c08ceb066ffdea21bcd2 Mon Sep 17 00:00:00 2001 From: Irbe Krumina Date: Tue, 13 Feb 2024 05:27:54 +0000 Subject: [PATCH] cmd/k8s-operator,k8s-operator: proxy configuration mechanism via a new ProxyClass custom resource (#11074) * cmd/k8s-operator,k8s-operator: introduce proxy configuration mechanism via ProxyClass custom resource. ProxyClass custom resource can be used to specify customizations for the proxy resources created by the operator. Add a reconciler that validates ProxyClass resources and sets a Ready condition to True or False with a corresponding reason and message. This is required because some fields (labels and annotations) require complex validations that cannot be performed at custom resource apply time. Reconcilers that use the ProxyClass to configure proxy resources are expected to verify that the ProxyClass is Ready and not proceed with resource creation if configuration from a ProxyClass that is not yet Ready is required. If a tailscale ingress/egress Service is annotated with a tailscale.com/proxy-class annotation, look up the corresponding ProxyClass and, if it is Ready, apply the configuration from the ProxyClass to the proxy's StatefulSet. If a tailscale Ingress has a tailscale.com/proxy-class annotation and the referenced ProxyClass custom resource is available and Ready, apply configuration from the ProxyClass to the proxy resources that will be created for the Ingress. Add a new .proxyClass field to the Connector spec. If connector.spec.proxyClass is set to a ProxyClass that is available and Ready, apply configuration from the ProxyClass to the proxy resources created for the Connector. Ensure that when Helm chart is packaged, the ProxyClass yaml is added to chart templates. Ensure that static manifest generator adds ProxyClass yaml to operator.yaml. Regenerate operator.yaml Signed-off-by: Irbe Krumina --- cmd/k8s-operator/connector.go | 12 + cmd/k8s-operator/connector_test.go | 120 ++++- .../deploy/chart/templates/operator-rbac.yaml | 2 +- .../deploy/crds/tailscale.com_connectors.yaml | 3 + .../crds/tailscale.com_proxyclasses.yaml | 494 +++++++++++++++++ .../deploy/examples/proxyclass.yaml | 15 + .../deploy/manifests/operator.yaml | 500 ++++++++++++++++++ cmd/k8s-operator/generate/main.go | 66 ++- cmd/k8s-operator/generate/main_test.go | 14 +- cmd/k8s-operator/ingress.go | 12 + cmd/k8s-operator/ingress_test.go | 135 ++++- cmd/k8s-operator/operator.go | 91 +++- cmd/k8s-operator/operator_test.go | 127 ++++- cmd/k8s-operator/proxyclass.go | 108 ++++ cmd/k8s-operator/proxyclass_test.go | 83 +++ cmd/k8s-operator/sts.go | 150 +++++- cmd/k8s-operator/sts_test.go | 252 +++++++++ cmd/k8s-operator/svc.go | 26 + cmd/k8s-operator/testutils_test.go | 33 +- k8s-operator/apis/v1alpha1/register.go | 2 +- k8s-operator/apis/v1alpha1/types_connector.go | 9 +- .../apis/v1alpha1/types_proxyclass.go | 143 +++++ .../apis/v1alpha1/zz_generated.deepcopy.go | 220 ++++++++ k8s-operator/conditions.go | 54 +- k8s-operator/conditions_test.go | 4 +- 25 files changed, 2584 insertions(+), 91 deletions(-) create mode 100644 cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml create mode 100644 cmd/k8s-operator/deploy/examples/proxyclass.yaml create mode 100644 cmd/k8s-operator/proxyclass.go create mode 100644 cmd/k8s-operator/proxyclass_test.go create mode 100644 k8s-operator/apis/v1alpha1/types_proxyclass.go diff --git a/cmd/k8s-operator/connector.go b/cmd/k8s-operator/connector.go index 997d47932..26ee0b7c6 100644 --- a/cmd/k8s-operator/connector.go +++ b/cmd/k8s-operator/connector.go @@ -164,6 +164,17 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge hostname = string(cn.Spec.Hostname) } crl := childResourceLabels(cn.Name, a.tsnamespace, "connector") + + proxyClass := cn.Spec.ProxyClass + if proxyClass != "" { + if ready, err := proxyClassIsReady(ctx, proxyClass, a.Client); err != nil { + return fmt.Errorf("error verifying ProxyClass for Connector: %w", err) + } else if !ready { + logger.Infof("ProxyClass %s specified for the Connector, but is not (yet) Ready, waiting..", proxyClass) + return nil + } + } + sts := &tailscaleSTSConfig{ ParentResourceName: cn.Name, ParentResourceUID: string(cn.UID), @@ -173,6 +184,7 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge Connector: &connector{ isExitNode: cn.Spec.ExitNode, }, + ProxyClass: proxyClass, } if cn.Spec.SubnetRouter != nil && len(cn.Spec.SubnetRouter.AdvertiseRoutes) > 0 { diff --git a/cmd/k8s-operator/connector_test.go b/cmd/k8s-operator/connector_test.go index 82ca71046..291aaec61 100644 --- a/cmd/k8s-operator/connector_test.go +++ b/cmd/k8s-operator/connector_test.go @@ -77,7 +77,7 @@ func TestConnector(t *testing.T) { confFileHash: "9321660203effb80983eaecc7b5ac5a8c53934926f46e895b9fe295dcfc5a904", } expectEqual(t, fc, expectedSecret(t, opts)) - expectEqual(t, fc, expectedSTS(opts)) + expectEqual(t, fc, expectedSTS(t, fc, opts)) // Add another route to be advertised. mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) { @@ -87,7 +87,7 @@ func TestConnector(t *testing.T) { opts.confFileHash = "fb6c4daf67425f983985750cd8d6f2beae77e614fcb34176604571f5623d6862" expectReconciled(t, cr, "", "test") - expectEqual(t, fc, expectedSTS(opts)) + expectEqual(t, fc, expectedSTS(t, fc, opts)) // Remove a route. mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) { @@ -96,7 +96,7 @@ func TestConnector(t *testing.T) { opts.subnetRoutes = "10.44.0.0/20" opts.confFileHash = "bacba177bcfe3849065cf6fee53d658a9bb4144197ac5b861727d69ea99742bb" expectReconciled(t, cr, "", "test") - expectEqual(t, fc, expectedSTS(opts)) + expectEqual(t, fc, expectedSTS(t, fc, opts)) // Remove the subnet router. mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) { @@ -105,7 +105,7 @@ func TestConnector(t *testing.T) { opts.subnetRoutes = "" opts.confFileHash = "7c421a99128eb80e79a285a82702f19f8f720615542a15bd794858a6275d8079" expectReconciled(t, cr, "", "test") - expectEqual(t, fc, expectedSTS(opts)) + expectEqual(t, fc, expectedSTS(t, fc, opts)) // Re-add the subnet router. mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) { @@ -116,7 +116,7 @@ func TestConnector(t *testing.T) { opts.subnetRoutes = "10.44.0.0/20" opts.confFileHash = "bacba177bcfe3849065cf6fee53d658a9bb4144197ac5b861727d69ea99742bb" expectReconciled(t, cr, "", "test") - expectEqual(t, fc, expectedSTS(opts)) + expectEqual(t, fc, expectedSTS(t, fc, opts)) // Delete the Connector. if err = fc.Delete(context.Background(), cn); err != nil { @@ -161,7 +161,7 @@ func TestConnector(t *testing.T) { confFileHash: "57d922331890c9b1c8c6ae664394cb254334c551d9cd9db14537b5d9da9fb17e", } expectEqual(t, fc, expectedSecret(t, opts)) - expectEqual(t, fc, expectedSTS(opts)) + expectEqual(t, fc, expectedSTS(t, fc, opts)) // Add an exit node. mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) { @@ -170,7 +170,7 @@ func TestConnector(t *testing.T) { opts.isExitNode = true opts.confFileHash = "1499b591fd97a50f0330db6ec09979792c49890cf31f5da5bb6a3f50dba1e77a" expectReconciled(t, cr, "", "test") - expectEqual(t, fc, expectedSTS(opts)) + expectEqual(t, fc, expectedSTS(t, fc, opts)) // Delete the Connector. if err = fc.Delete(context.Background(), cn); err != nil { @@ -183,3 +183,109 @@ func TestConnector(t *testing.T) { expectMissing[appsv1.StatefulSet](t, fc, "operator-ns", shortName) expectMissing[corev1.Secret](t, fc, "operator-ns", fullName) } + +func TestConnectorWithProxyClass(t *testing.T) { + // Setup + pc := &tsapi.ProxyClass{ + ObjectMeta: metav1.ObjectMeta{Name: "custom-metadata"}, + Spec: tsapi.ProxyClassSpec{StatefulSet: &tsapi.StatefulSet{ + Labels: map[string]string{"foo": "bar"}, + Annotations: map[string]string{"bar.io/foo": "some-val"}, + Pod: &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}}}}, + } + cn := &tsapi.Connector{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + UID: types.UID("1234-UID"), + }, + TypeMeta: metav1.TypeMeta{ + Kind: tsapi.ConnectorKind, + APIVersion: "tailscale.io/v1alpha1", + }, + Spec: tsapi.ConnectorSpec{ + SubnetRouter: &tsapi.SubnetRouter{ + AdvertiseRoutes: []tsapi.Route{"10.40.0.0/14"}, + }, + ExitNode: true, + }, + } + fc := fake.NewClientBuilder(). + WithScheme(tsapi.GlobalScheme). + WithObjects(pc, cn). + WithStatusSubresource(pc, cn). + Build() + ft := &fakeTSClient{} + zl, err := zap.NewDevelopment() + if err != nil { + t.Fatal(err) + } + cl := tstest.NewClock(tstest.ClockOpts{}) + cr := &ConnectorReconciler{ + Client: fc, + clock: cl, + ssr: &tailscaleSTSReconciler{ + Client: fc, + tsClient: ft, + defaultTags: []string{"tag:k8s"}, + operatorNamespace: "operator-ns", + proxyImage: "tailscale/tailscale", + }, + logger: zl.Sugar(), + } + + // 1. Connector is created with no ProxyClass specified, create + // resources with the default configuration. + expectReconciled(t, cr, "", "test") + fullName, shortName := findGenName(t, fc, "", "test", "connector") + + opts := configOpts{ + stsName: shortName, + secretName: fullName, + parentType: "connector", + hostname: "test-connector", + shouldUseDeclarativeConfig: true, + isExitNode: true, + subnetRoutes: "10.40.0.0/14", + confFileHash: "9321660203effb80983eaecc7b5ac5a8c53934926f46e895b9fe295dcfc5a904", + } + expectEqual(t, fc, expectedSecret(t, opts)) + expectEqual(t, fc, expectedSTS(t, fc, opts)) + + // 2. Update Connector to specify a ProxyClass. ProxyClass is not yet + // ready, so its configuration is NOT applied to the Connector + // resources. + mustUpdate(t, fc, "", "test", func(conn *tsapi.Connector) { + conn.Spec.ProxyClass = "custom-metadata" + }) + expectReconciled(t, cr, "", "test") + expectEqual(t, fc, expectedSTS(t, fc, opts)) + + // 3. ProxyClass is set to Ready by proxy-class reconciler. Connector + // get reconciled and configuration from the ProxyClass is applied to + // its resources. + mustUpdateStatus(t, fc, "", "custom-metadata", func(pc *tsapi.ProxyClass) { + pc.Status = tsapi.ProxyClassStatus{ + Conditions: []tsapi.ConnectorCondition{{ + Status: metav1.ConditionTrue, + Type: tsapi.ProxyClassready, + ObservedGeneration: pc.Generation, + }}} + }) + opts.proxyClass = pc.Name + // We lose the auth key on second reconcile, because in code it's set to + // StringData, but is actually read from Data. This works with a real + // API server, but not with our test setup here. + opts.confFileHash = "1499b591fd97a50f0330db6ec09979792c49890cf31f5da5bb6a3f50dba1e77a" + expectReconciled(t, cr, "", "test") + expectEqual(t, fc, expectedSTS(t, fc, opts)) + + // 4. Connector.spec.proxyClass field is unset, Connector gets + // reconciled and configuration from the ProxyClass is removed from the + // cluster resources for the Connector. + mustUpdate(t, fc, "", "test", func(conn *tsapi.Connector) { + conn.Spec.ProxyClass = "" + }) + opts.proxyClass = "" + expectReconciled(t, cr, "", "test") + expectEqual(t, fc, expectedSTS(t, fc, opts)) +} diff --git a/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml b/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml index 8ea07e808..ff518e40c 100644 --- a/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml +++ b/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml @@ -22,7 +22,7 @@ rules: resources: ["ingressclasses"] verbs: ["get", "list", "watch"] - apiGroups: ["tailscale.com"] - resources: ["connectors", "connectors/status"] + resources: ["connectors", "connectors/status", "proxyclasses", "proxyclasses/status"] verbs: ["get", "list", "watch", "update"] --- apiVersion: rbac.authorization.k8s.io/v1 diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_connectors.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_connectors.yaml index 5291c7c0a..b9a4e6183 100644 --- a/cmd/k8s-operator/deploy/crds/tailscale.com_connectors.yaml +++ b/cmd/k8s-operator/deploy/crds/tailscale.com_connectors.yaml @@ -54,6 +54,9 @@ spec: description: Hostname is the tailnet hostname that should be assigned to the Connector node. If unset, hostname defaults to -connector. Hostname can contain lower case letters, numbers and dashes, it must not start or end with a dash and must be between 2 and 63 characters long. type: string pattern: ^[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$ + proxyClass: + description: ProxyClass is the name of the ProxyClass custom resource that contains configuration options that should be applied to the resources created for this Connector. If unset, the operator will create resources with the default configuration. + type: string subnetRouter: description: SubnetRouter defines subnet routes that the Connector node should expose to tailnet. If unset, none are exposed. https://tailscale.com/kb/1019/subnets/ type: object diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml new file mode 100644 index 000000000..83504f4c0 --- /dev/null +++ b/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml @@ -0,0 +1,494 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.13.0 + name: proxyclasses.tailscale.com +spec: + group: tailscale.com + names: + kind: ProxyClass + listKind: ProxyClassList + plural: proxyclasses + singular: proxyclass + scope: Cluster + versions: + - additionalPrinterColumns: + - description: Status of the ProxyClass. + jsonPath: .status.conditions[?(@.type == "ProxyClassReady")].reason + name: Status + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + type: object + required: + - spec + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + type: object + required: + - statefulSet + properties: + statefulSet: + description: Proxy's StatefulSet spec. + type: object + properties: + annotations: + description: Annotations that will be added to the StatefulSet created for the proxy. Any Annotations specified here will be merged with the default annotations applied to the StatefulSet by the Tailscale Kubernetes operator as well as any other annotations that might have been applied by other actors. Annotations must be valid Kubernetes annotations. https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set + type: object + additionalProperties: + type: string + labels: + description: Labels that will be added to the StatefulSet created for the proxy. Any labels specified here will be merged with the default labels applied to the StatefulSet by the Tailscale Kubernetes operator as well as any other labels that might have been applied by other actors. Label keys and values must be valid Kubernetes label keys and values. https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set + type: object + additionalProperties: + type: string + pod: + description: Configuration for the proxy Pod. + type: object + properties: + annotations: + description: Annotations that will be added to the proxy Pod. Any annotations specified here will be merged with the default annotations applied to the Pod by the Tailscale Kubernetes operator. Annotations must be valid Kubernetes annotations. https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set + type: object + additionalProperties: + type: string + imagePullSecrets: + description: Proxy Pod's image pull Secrets. https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#PodSpec + type: array + items: + description: LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace. + type: object + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + x-kubernetes-map-type: atomic + labels: + description: Labels that will be added to the proxy Pod. Any labels specified here will be merged with the default labels applied to the Pod by the Tailscale Kubernetes operator. Label keys and values must be valid Kubernetes label keys and values. https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set + type: object + additionalProperties: + type: string + nodeName: + description: Proxy Pod's node name. https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#scheduling + type: string + nodeSelector: + description: Proxy Pod's node selector. By default Tailscale Kubernetes operator does not apply any node selector. https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#scheduling + type: object + additionalProperties: + type: string + securityContext: + description: Proxy Pod's security context. By default Tailscale Kubernetes operator does not apply any Pod security context. https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#security-context-2 + type: object + properties: + fsGroup: + description: "A special supplemental group that applies to all containers in a pod. Some volume types allow the Kubelet to change the ownership of that volume to be owned by the pod: \n 1. The owning GID will be the FSGroup 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) 3. The permission bits are OR'd with rw-rw---- \n If unset, the Kubelet will not modify the ownership and permissions of any volume. Note that this field cannot be set when spec.os.name is windows." + type: integer + format: int64 + fsGroupChangePolicy: + description: 'fsGroupChangePolicy defines behavior of changing ownership and permission of the volume before being exposed inside Pod. This field will only apply to volume types which support fsGroup based ownership(and permissions). It will have no effect on ephemeral volume types such as: secret, configmaps and emptydir. Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. Note that this field cannot be set when spec.os.name is windows.' + type: string + runAsGroup: + description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container. Note that this field cannot be set when spec.os.name is windows. + type: integer + format: int64 + runAsNonRoot: + description: Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container. Note that this field cannot be set when spec.os.name is windows. + type: integer + format: int64 + seLinuxOptions: + description: The SELinux context to be applied to all containers. If unspecified, the container runtime will allocate a random SELinux context for each container. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container. Note that this field cannot be set when spec.os.name is windows. + type: object + properties: + level: + description: Level is SELinux level label that applies to the container. + type: string + role: + description: Role is a SELinux role label that applies to the container. + type: string + type: + description: Type is a SELinux type label that applies to the container. + type: string + user: + description: User is a SELinux user label that applies to the container. + type: string + seccompProfile: + description: The seccomp options to use by the containers in this pod. Note that this field cannot be set when spec.os.name is windows. + type: object + required: + - type + properties: + localhostProfile: + description: localhostProfile indicates a profile defined in a file on the node should be used. The profile must be preconfigured on the node to work. Must be a descending path, relative to the kubelet's configured seccomp profile location. Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: "type indicates which kind of seccomp profile will be applied. Valid options are: \n Localhost - a profile defined in a file on the node should be used. RuntimeDefault - the container runtime default profile should be used. Unconfined - no profile should be applied." + type: string + supplementalGroups: + description: A list of groups applied to the first process run in each container, in addition to the container's primary GID, the fsGroup (if specified), and group memberships defined in the container image for the uid of the container process. If unspecified, no additional groups are added to any container. Note that group memberships defined in the container image for the uid of the container process are still effective, even if they are not included in this list. Note that this field cannot be set when spec.os.name is windows. + type: array + items: + type: integer + format: int64 + sysctls: + description: Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported sysctls (by the container runtime) might fail to launch. Note that this field cannot be set when spec.os.name is windows. + type: array + items: + description: Sysctl defines a kernel parameter to be set + type: object + required: + - name + - value + properties: + name: + description: Name of a property to set + type: string + value: + description: Value of a property to set + type: string + windowsOptions: + description: The Windows specific settings applied to all containers. If unspecified, the options within a container's SecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is linux. + type: object + properties: + gmsaCredentialSpec: + description: GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + type: string + hostProcess: + description: HostProcess determines if a container should be run as a 'Host Process' container. All of a Pod's containers must have the same effective HostProcess value (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + tailscaleContainer: + description: Configuration for the proxy container running tailscale. + type: object + properties: + resources: + description: Container resource requirements. By default Tailscale Kubernetes operator does not apply any resource requirements. The amount of resources required wil depend on the amount of resources the operator needs to parse, usage patterns and cluster size. https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#resources + type: object + properties: + claims: + description: "Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. \n This is an alpha field and requires enabling the DynamicResourceAllocation feature gate. \n This field is immutable. It can only be set for containers." + type: array + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + type: object + required: + - name + properties: + name: + description: Name must match the name of one entry in pod.spec.resourceClaims of the Pod where this field is used. It makes that resource available inside a container. + type: string + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + additionalProperties: + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + requests: + description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. Requests cannot exceed Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + additionalProperties: + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + securityContext: + description: 'Container security context. Security context specified here will override the security context by the operator. By default the operator: - sets ''privileged: true'' for the init container - set NET_ADMIN capability for tailscale container for proxies that are created for Services or Connector. https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#security-context' + type: object + properties: + allowPrivilegeEscalation: + description: 'AllowPrivilegeEscalation controls whether a process can gain more privileges than its parent process. This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged 2) has CAP_SYS_ADMIN Note that this field cannot be set when spec.os.name is windows.' + type: boolean + capabilities: + description: The capabilities to add/drop when running containers. Defaults to the default set of capabilities granted by the container runtime. Note that this field cannot be set when spec.os.name is windows. + type: object + properties: + add: + description: Added capabilities + type: array + items: + description: Capability represent POSIX capabilities type + type: string + drop: + description: Removed capabilities + type: array + items: + description: Capability represent POSIX capabilities type + type: string + privileged: + description: Run container in privileged mode. Processes in privileged containers are essentially equivalent to root on the host. Defaults to false. Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: procMount denotes the type of proc mount to use for the containers. The default is DefaultProcMount which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag to be enabled. Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: Whether this container has a read-only root filesystem. Default is false. Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is windows. + type: integer + format: int64 + runAsNonRoot: + description: Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is windows. + type: integer + format: int64 + seLinuxOptions: + description: The SELinux context to be applied to the container. If unspecified, the container runtime will allocate a random SELinux context for each container. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is windows. + type: object + properties: + level: + description: Level is SELinux level label that applies to the container. + type: string + role: + description: Role is a SELinux role label that applies to the container. + type: string + type: + description: Type is a SELinux type label that applies to the container. + type: string + user: + description: User is a SELinux user label that applies to the container. + type: string + seccompProfile: + description: The seccomp options to use by this container. If seccomp options are provided at both the pod & container level, the container options override the pod options. Note that this field cannot be set when spec.os.name is windows. + type: object + required: + - type + properties: + localhostProfile: + description: localhostProfile indicates a profile defined in a file on the node should be used. The profile must be preconfigured on the node to work. Must be a descending path, relative to the kubelet's configured seccomp profile location. Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: "type indicates which kind of seccomp profile will be applied. Valid options are: \n Localhost - a profile defined in a file on the node should be used. RuntimeDefault - the container runtime default profile should be used. Unconfined - no profile should be applied." + type: string + windowsOptions: + description: The Windows specific settings applied to all containers. If unspecified, the options from the PodSecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is linux. + type: object + properties: + gmsaCredentialSpec: + description: GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + type: string + hostProcess: + description: HostProcess determines if a container should be run as a 'Host Process' container. All of a Pod's containers must have the same effective HostProcess value (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + tailscaleInitContainer: + description: Configuration for the proxy init container that enables forwarding. + type: object + properties: + resources: + description: Container resource requirements. By default Tailscale Kubernetes operator does not apply any resource requirements. The amount of resources required wil depend on the amount of resources the operator needs to parse, usage patterns and cluster size. https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#resources + type: object + properties: + claims: + description: "Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. \n This is an alpha field and requires enabling the DynamicResourceAllocation feature gate. \n This field is immutable. It can only be set for containers." + type: array + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + type: object + required: + - name + properties: + name: + description: Name must match the name of one entry in pod.spec.resourceClaims of the Pod where this field is used. It makes that resource available inside a container. + type: string + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + additionalProperties: + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + requests: + description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. Requests cannot exceed Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + additionalProperties: + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + securityContext: + description: 'Container security context. Security context specified here will override the security context by the operator. By default the operator: - sets ''privileged: true'' for the init container - set NET_ADMIN capability for tailscale container for proxies that are created for Services or Connector. https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#security-context' + type: object + properties: + allowPrivilegeEscalation: + description: 'AllowPrivilegeEscalation controls whether a process can gain more privileges than its parent process. This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged 2) has CAP_SYS_ADMIN Note that this field cannot be set when spec.os.name is windows.' + type: boolean + capabilities: + description: The capabilities to add/drop when running containers. Defaults to the default set of capabilities granted by the container runtime. Note that this field cannot be set when spec.os.name is windows. + type: object + properties: + add: + description: Added capabilities + type: array + items: + description: Capability represent POSIX capabilities type + type: string + drop: + description: Removed capabilities + type: array + items: + description: Capability represent POSIX capabilities type + type: string + privileged: + description: Run container in privileged mode. Processes in privileged containers are essentially equivalent to root on the host. Defaults to false. Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: procMount denotes the type of proc mount to use for the containers. The default is DefaultProcMount which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag to be enabled. Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: Whether this container has a read-only root filesystem. Default is false. Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is windows. + type: integer + format: int64 + runAsNonRoot: + description: Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is windows. + type: integer + format: int64 + seLinuxOptions: + description: The SELinux context to be applied to the container. If unspecified, the container runtime will allocate a random SELinux context for each container. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is windows. + type: object + properties: + level: + description: Level is SELinux level label that applies to the container. + type: string + role: + description: Role is a SELinux role label that applies to the container. + type: string + type: + description: Type is a SELinux type label that applies to the container. + type: string + user: + description: User is a SELinux user label that applies to the container. + type: string + seccompProfile: + description: The seccomp options to use by this container. If seccomp options are provided at both the pod & container level, the container options override the pod options. Note that this field cannot be set when spec.os.name is windows. + type: object + required: + - type + properties: + localhostProfile: + description: localhostProfile indicates a profile defined in a file on the node should be used. The profile must be preconfigured on the node to work. Must be a descending path, relative to the kubelet's configured seccomp profile location. Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: "type indicates which kind of seccomp profile will be applied. Valid options are: \n Localhost - a profile defined in a file on the node should be used. RuntimeDefault - the container runtime default profile should be used. Unconfined - no profile should be applied." + type: string + windowsOptions: + description: The Windows specific settings applied to all containers. If unspecified, the options from the PodSecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is linux. + type: object + properties: + gmsaCredentialSpec: + description: GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + type: string + hostProcess: + description: HostProcess determines if a container should be run as a 'Host Process' container. All of a Pod's containers must have the same effective HostProcess value (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + tolerations: + description: Proxy Pod's tolerations. By default Tailscale Kubernetes operator does not apply any tolerations. https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#scheduling + type: array + items: + description: The pod this Toleration is attached to tolerates any taint that matches the triple using the matching operator . + type: object + properties: + effect: + description: Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Key is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: Operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: TolerationSeconds represents the period of time the toleration (which must be of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) by the system. + type: integer + format: int64 + value: + description: Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + status: + type: object + properties: + conditions: + description: List of status conditions to indicate the status of the ProxyClass. Known condition types are `ProxyClassReady`. + type: array + items: + description: ConnectorCondition contains condition information for a Connector. + type: object + required: + - status + - type + properties: + lastTransitionTime: + description: LastTransitionTime is the timestamp corresponding to the last status change of this condition. + type: string + format: date-time + message: + description: Message is a human readable description of the details of the last transition, complementing reason. + type: string + observedGeneration: + description: If set, this represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.condition[x].observedGeneration is 9, the condition is out of date with respect to the current state of the Connector. + type: integer + format: int64 + reason: + description: Reason is a brief machine readable explanation for the condition's last transition. + type: string + status: + description: Status of the condition, one of ('True', 'False', 'Unknown'). + type: string + type: + description: Type of the condition, known values are (`SubnetRouterReady`). + type: string + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + served: true + storage: true + subresources: + status: {} diff --git a/cmd/k8s-operator/deploy/examples/proxyclass.yaml b/cmd/k8s-operator/deploy/examples/proxyclass.yaml new file mode 100644 index 000000000..121465bab --- /dev/null +++ b/cmd/k8s-operator/deploy/examples/proxyclass.yaml @@ -0,0 +1,15 @@ +apiVersion: tailscale.com/v1alpha1 +kind: ProxyClass +metadata: + name: prod +spec: + statefulSet: + annotations: + platform-component: infra + pod: + labels: + team: eng + nodeSelector: + beta.kubernetes.io/os: "linux" + imagePullSecrets: + - name: "foo" diff --git a/cmd/k8s-operator/deploy/manifests/operator.yaml b/cmd/k8s-operator/deploy/manifests/operator.yaml index 057117b0e..62e444f82 100644 --- a/cmd/k8s-operator/deploy/manifests/operator.yaml +++ b/cmd/k8s-operator/deploy/manifests/operator.yaml @@ -79,6 +79,9 @@ spec: description: Hostname is the tailnet hostname that should be assigned to the Connector node. If unset, hostname defaults to -connector. Hostname can contain lower case letters, numbers and dashes, it must not start or end with a dash and must be between 2 and 63 characters long. pattern: ^[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$ type: string + proxyClass: + description: ProxyClass is the name of the ProxyClass custom resource that contains configuration options that should be applied to the resources created for this Connector. If unset, the operator will create resources with the default configuration. + type: string subnetRouter: description: SubnetRouter defines subnet routes that the Connector node should expose to tailnet. If unset, none are exposed. https://tailscale.com/kb/1019/subnets/ properties: @@ -153,6 +156,501 @@ spec: subresources: status: {} --- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.13.0 + name: proxyclasses.tailscale.com +spec: + group: tailscale.com + names: + kind: ProxyClass + listKind: ProxyClassList + plural: proxyclasses + singular: proxyclass + scope: Cluster + versions: + - additionalPrinterColumns: + - description: Status of the ProxyClass. + jsonPath: .status.conditions[?(@.type == "ProxyClassReady")].reason + name: Status + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + statefulSet: + description: Proxy's StatefulSet spec. + properties: + annotations: + additionalProperties: + type: string + description: Annotations that will be added to the StatefulSet created for the proxy. Any Annotations specified here will be merged with the default annotations applied to the StatefulSet by the Tailscale Kubernetes operator as well as any other annotations that might have been applied by other actors. Annotations must be valid Kubernetes annotations. https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set + type: object + labels: + additionalProperties: + type: string + description: Labels that will be added to the StatefulSet created for the proxy. Any labels specified here will be merged with the default labels applied to the StatefulSet by the Tailscale Kubernetes operator as well as any other labels that might have been applied by other actors. Label keys and values must be valid Kubernetes label keys and values. https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set + type: object + pod: + description: Configuration for the proxy Pod. + properties: + annotations: + additionalProperties: + type: string + description: Annotations that will be added to the proxy Pod. Any annotations specified here will be merged with the default annotations applied to the Pod by the Tailscale Kubernetes operator. Annotations must be valid Kubernetes annotations. https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set + type: object + imagePullSecrets: + description: Proxy Pod's image pull Secrets. https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#PodSpec + items: + description: LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + x-kubernetes-map-type: atomic + type: array + labels: + additionalProperties: + type: string + description: Labels that will be added to the proxy Pod. Any labels specified here will be merged with the default labels applied to the Pod by the Tailscale Kubernetes operator. Label keys and values must be valid Kubernetes label keys and values. https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set + type: object + nodeName: + description: Proxy Pod's node name. https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#scheduling + type: string + nodeSelector: + additionalProperties: + type: string + description: Proxy Pod's node selector. By default Tailscale Kubernetes operator does not apply any node selector. https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#scheduling + type: object + securityContext: + description: Proxy Pod's security context. By default Tailscale Kubernetes operator does not apply any Pod security context. https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#security-context-2 + properties: + fsGroup: + description: "A special supplemental group that applies to all containers in a pod. Some volume types allow the Kubelet to change the ownership of that volume to be owned by the pod: \n 1. The owning GID will be the FSGroup 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) 3. The permission bits are OR'd with rw-rw---- \n If unset, the Kubelet will not modify the ownership and permissions of any volume. Note that this field cannot be set when spec.os.name is windows." + format: int64 + type: integer + fsGroupChangePolicy: + description: 'fsGroupChangePolicy defines behavior of changing ownership and permission of the volume before being exposed inside Pod. This field will only apply to volume types which support fsGroup based ownership(and permissions). It will have no effect on ephemeral volume types such as: secret, configmaps and emptydir. Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. Note that this field cannot be set when spec.os.name is windows.' + type: string + runAsGroup: + description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container. Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container. Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: The SELinux context to be applied to all containers. If unspecified, the container runtime will allocate a random SELinux context for each container. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container. Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies to the container. + type: string + role: + description: Role is a SELinux role label that applies to the container. + type: string + type: + description: Type is a SELinux type label that applies to the container. + type: string + user: + description: User is a SELinux user label that applies to the container. + type: string + type: object + seccompProfile: + description: The seccomp options to use by the containers in this pod. Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: localhostProfile indicates a profile defined in a file on the node should be used. The profile must be preconfigured on the node to work. Must be a descending path, relative to the kubelet's configured seccomp profile location. Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: "type indicates which kind of seccomp profile will be applied. Valid options are: \n Localhost - a profile defined in a file on the node should be used. RuntimeDefault - the container runtime default profile should be used. Unconfined - no profile should be applied." + type: string + required: + - type + type: object + supplementalGroups: + description: A list of groups applied to the first process run in each container, in addition to the container's primary GID, the fsGroup (if specified), and group memberships defined in the container image for the uid of the container process. If unspecified, no additional groups are added to any container. Note that group memberships defined in the container image for the uid of the container process are still effective, even if they are not included in this list. Note that this field cannot be set when spec.os.name is windows. + items: + format: int64 + type: integer + type: array + sysctls: + description: Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported sysctls (by the container runtime) might fail to launch. Note that this field cannot be set when spec.os.name is windows. + items: + description: Sysctl defines a kernel parameter to be set + properties: + name: + description: Name of a property to set + type: string + value: + description: Value of a property to set + type: string + required: + - name + - value + type: object + type: array + windowsOptions: + description: The Windows specific settings applied to all containers. If unspecified, the options within a container's SecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + type: string + hostProcess: + description: HostProcess determines if a container should be run as a 'Host Process' container. All of a Pod's containers must have the same effective HostProcess value (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + tailscaleContainer: + description: Configuration for the proxy container running tailscale. + properties: + resources: + description: Container resource requirements. By default Tailscale Kubernetes operator does not apply any resource requirements. The amount of resources required wil depend on the amount of resources the operator needs to parse, usage patterns and cluster size. https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#resources + properties: + claims: + description: "Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. \n This is an alpha field and requires enabling the DynamicResourceAllocation feature gate. \n This field is immutable. It can only be set for containers." + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name of one entry in pod.spec.resourceClaims of the Pod where this field is used. It makes that resource available inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. Requests cannot exceed Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + securityContext: + description: 'Container security context. Security context specified here will override the security context by the operator. By default the operator: - sets ''privileged: true'' for the init container - set NET_ADMIN capability for tailscale container for proxies that are created for Services or Connector. https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#security-context' + properties: + allowPrivilegeEscalation: + description: 'AllowPrivilegeEscalation controls whether a process can gain more privileges than its parent process. This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged 2) has CAP_SYS_ADMIN Note that this field cannot be set when spec.os.name is windows.' + type: boolean + capabilities: + description: The capabilities to add/drop when running containers. Defaults to the default set of capabilities granted by the container runtime. Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities type + type: string + type: array + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities type + type: string + type: array + type: object + privileged: + description: Run container in privileged mode. Processes in privileged containers are essentially equivalent to root on the host. Defaults to false. Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: procMount denotes the type of proc mount to use for the containers. The default is DefaultProcMount which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag to be enabled. Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: Whether this container has a read-only root filesystem. Default is false. Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: The SELinux context to be applied to the container. If unspecified, the container runtime will allocate a random SELinux context for each container. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies to the container. + type: string + role: + description: Role is a SELinux role label that applies to the container. + type: string + type: + description: Type is a SELinux type label that applies to the container. + type: string + user: + description: User is a SELinux user label that applies to the container. + type: string + type: object + seccompProfile: + description: The seccomp options to use by this container. If seccomp options are provided at both the pod & container level, the container options override the pod options. Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: localhostProfile indicates a profile defined in a file on the node should be used. The profile must be preconfigured on the node to work. Must be a descending path, relative to the kubelet's configured seccomp profile location. Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: "type indicates which kind of seccomp profile will be applied. Valid options are: \n Localhost - a profile defined in a file on the node should be used. RuntimeDefault - the container runtime default profile should be used. Unconfined - no profile should be applied." + type: string + required: + - type + type: object + windowsOptions: + description: The Windows specific settings applied to all containers. If unspecified, the options from the PodSecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + type: string + hostProcess: + description: HostProcess determines if a container should be run as a 'Host Process' container. All of a Pod's containers must have the same effective HostProcess value (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + type: object + tailscaleInitContainer: + description: Configuration for the proxy init container that enables forwarding. + properties: + resources: + description: Container resource requirements. By default Tailscale Kubernetes operator does not apply any resource requirements. The amount of resources required wil depend on the amount of resources the operator needs to parse, usage patterns and cluster size. https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#resources + properties: + claims: + description: "Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. \n This is an alpha field and requires enabling the DynamicResourceAllocation feature gate. \n This field is immutable. It can only be set for containers." + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name of one entry in pod.spec.resourceClaims of the Pod where this field is used. It makes that resource available inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. Requests cannot exceed Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + securityContext: + description: 'Container security context. Security context specified here will override the security context by the operator. By default the operator: - sets ''privileged: true'' for the init container - set NET_ADMIN capability for tailscale container for proxies that are created for Services or Connector. https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#security-context' + properties: + allowPrivilegeEscalation: + description: 'AllowPrivilegeEscalation controls whether a process can gain more privileges than its parent process. This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged 2) has CAP_SYS_ADMIN Note that this field cannot be set when spec.os.name is windows.' + type: boolean + capabilities: + description: The capabilities to add/drop when running containers. Defaults to the default set of capabilities granted by the container runtime. Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities type + type: string + type: array + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities type + type: string + type: array + type: object + privileged: + description: Run container in privileged mode. Processes in privileged containers are essentially equivalent to root on the host. Defaults to false. Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: procMount denotes the type of proc mount to use for the containers. The default is DefaultProcMount which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag to be enabled. Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: Whether this container has a read-only root filesystem. Default is false. Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: The SELinux context to be applied to the container. If unspecified, the container runtime will allocate a random SELinux context for each container. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies to the container. + type: string + role: + description: Role is a SELinux role label that applies to the container. + type: string + type: + description: Type is a SELinux type label that applies to the container. + type: string + user: + description: User is a SELinux user label that applies to the container. + type: string + type: object + seccompProfile: + description: The seccomp options to use by this container. If seccomp options are provided at both the pod & container level, the container options override the pod options. Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: localhostProfile indicates a profile defined in a file on the node should be used. The profile must be preconfigured on the node to work. Must be a descending path, relative to the kubelet's configured seccomp profile location. Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: "type indicates which kind of seccomp profile will be applied. Valid options are: \n Localhost - a profile defined in a file on the node should be used. RuntimeDefault - the container runtime default profile should be used. Unconfined - no profile should be applied." + type: string + required: + - type + type: object + windowsOptions: + description: The Windows specific settings applied to all containers. If unspecified, the options from the PodSecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + type: string + hostProcess: + description: HostProcess determines if a container should be run as a 'Host Process' container. All of a Pod's containers must have the same effective HostProcess value (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + type: object + tolerations: + description: Proxy Pod's tolerations. By default Tailscale Kubernetes operator does not apply any tolerations. https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#scheduling + items: + description: The pod this Toleration is attached to tolerates any taint that matches the triple using the matching operator . + properties: + effect: + description: Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Key is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: Operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: TolerationSeconds represents the period of time the toleration (which must be of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + type: object + type: object + required: + - statefulSet + type: object + status: + properties: + conditions: + description: List of status conditions to indicate the status of the ProxyClass. Known condition types are `ProxyClassReady`. + items: + description: ConnectorCondition contains condition information for a Connector. + properties: + lastTransitionTime: + description: LastTransitionTime is the timestamp corresponding to the last status change of this condition. + format: date-time + type: string + message: + description: Message is a human readable description of the details of the last transition, complementing reason. + type: string + observedGeneration: + description: If set, this represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.condition[x].observedGeneration is 9, the condition is out of date with respect to the current state of the Connector. + format: int64 + type: integer + reason: + description: Reason is a brief machine readable explanation for the condition's last transition. + type: string + status: + description: Status of the condition, one of ('True', 'False', 'Unknown'). + type: string + type: + description: Type of the condition, known values are (`SubnetRouterReady`). + type: string + required: + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -186,6 +684,8 @@ rules: resources: - connectors - connectors/status + - proxyclasses + - proxyclasses/status verbs: - get - list diff --git a/cmd/k8s-operator/generate/main.go b/cmd/k8s-operator/generate/main.go index 531dee2dc..7f2b18bf9 100644 --- a/cmd/k8s-operator/generate/main.go +++ b/cmd/k8s-operator/generate/main.go @@ -19,10 +19,12 @@ import ( ) const ( - operatorDeploymentFilesPath = "cmd/k8s-operator/deploy" - crdPath = operatorDeploymentFilesPath + "/crds/tailscale.com_connectors.yaml" - helmTemplatesPath = operatorDeploymentFilesPath + "/chart/templates" - crdTemplatePath = helmTemplatesPath + "/connectors.yaml" + operatorDeploymentFilesPath = "cmd/k8s-operator/deploy" + connectorCRDPath = operatorDeploymentFilesPath + "/crds/tailscale.com_connectors.yaml" + proxyClassCRDPath = operatorDeploymentFilesPath + "/crds/tailscale.com_proxyclasses.yaml" + helmTemplatesPath = operatorDeploymentFilesPath + "/chart/templates" + connectorCRDHelmTemplatePath = helmTemplatesPath + "/connector.yaml" + proxyClassCRDHelmTemplatePath = helmTemplatesPath + "/proxyclass.yaml" helmConditionalStart = "{{ if .Values.installCRDs -}}\n" helmConditionalEnd = "{{- end -}}" @@ -44,9 +46,9 @@ func main() { default: log.Fatalf("unknown option %s, known options are 'staticmanifests', 'helmcrd'", os.Args[1]) } - log.Printf("Inserting CRD into the Helm templates") + log.Printf("Inserting CRDs Helm templates") if err := generate(repoRoot); err != nil { - log.Fatalf("error adding Connector CRD to Helm templates: %v", err) + log.Fatalf("error adding CRDs to Helm templates: %v", err) } defer func() { if err := cleanup(repoRoot); err != nil { @@ -106,34 +108,48 @@ func main() { } } +// generate places tailscale.com CRDs (currently Connector and ProxyClass) into +// the Helm chart templates behind .Values.installCRDs=true condition (true by +// default). func generate(baseDir string) error { - log.Print("Placing Connector CRD into Helm templates..") - chartBytes, err := os.ReadFile(filepath.Join(baseDir, crdPath)) - if err != nil { - return fmt.Errorf("error reading CRD contents: %w", err) - } - // Place a new temporary Helm template file with the templated CRD - // contents into Helm templates. - file, err := os.Create(filepath.Join(baseDir, crdTemplatePath)) - if err != nil { - return fmt.Errorf("error creating CRD template file: %w", err) - } - if _, err := file.Write([]byte(helmConditionalStart)); err != nil { - return fmt.Errorf("error writing helm if statement start: %w", err) + addCRDToHelm := func(crdPath, crdTemplatePath string) error { + chartBytes, err := os.ReadFile(filepath.Join(baseDir, crdPath)) + if err != nil { + return fmt.Errorf("error reading CRD contents: %w", err) + } + // Place a new temporary Helm template file with the templated CRD + // contents into Helm templates. + file, err := os.Create(filepath.Join(baseDir, crdTemplatePath)) + if err != nil { + return fmt.Errorf("error creating CRD template file: %w", err) + } + if _, err := file.Write([]byte(helmConditionalStart)); err != nil { + return fmt.Errorf("error writing helm if statement start: %w", err) + } + if _, err := file.Write(chartBytes); err != nil { + return fmt.Errorf("error writing chart bytes: %w", err) + } + if _, err := file.Write([]byte(helmConditionalEnd)); err != nil { + return fmt.Errorf("error writing helm if-statement end: %w", err) + } + return nil } - if _, err := file.Write(chartBytes); err != nil { - return fmt.Errorf("error writing chart bytes: %w", err) + if err := addCRDToHelm(connectorCRDPath, connectorCRDHelmTemplatePath); err != nil { + return fmt.Errorf("error adding Connector CRD to Helm templates: %w", err) } - if _, err := file.Write([]byte(helmConditionalEnd)); err != nil { - return fmt.Errorf("error writing helm if-statement end: %w", err) + if err := addCRDToHelm(proxyClassCRDPath, proxyClassCRDHelmTemplatePath); err != nil { + return fmt.Errorf("error adding ProxyClass CRD to Helm templates: %w", err) } return nil } func cleanup(baseDir string) error { log.Print("Cleaning up CRD from Helm templates") - if err := os.Remove(filepath.Join(baseDir, crdTemplatePath)); err != nil && !os.IsNotExist(err) { - return fmt.Errorf("error cleaning up CRD template: %w", err) + if err := os.Remove(filepath.Join(baseDir, connectorCRDHelmTemplatePath)); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("error cleaning up Connector CRD template: %w", err) + } + if err := os.Remove(filepath.Join(baseDir, proxyClassCRDHelmTemplatePath)); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("error cleaning up ProxyClass CRD template: %w", err) } return nil } diff --git a/cmd/k8s-operator/generate/main_test.go b/cmd/k8s-operator/generate/main_test.go index 6f5a054d8..7e309c1fd 100644 --- a/cmd/k8s-operator/generate/main_test.go +++ b/cmd/k8s-operator/generate/main_test.go @@ -42,7 +42,7 @@ func Test_generate(t *testing.T) { t.Fatalf("Helm chart linter failed: %v", err) } - // Test that default Helm install contains the CRD + // Test that default Helm install contains the Connector and ProxyClass CRDs. installContentsWithCRD := bytes.NewBuffer([]byte{}) helmTemplateWithCRDCmd := exec.Command(helmCLIPath, "template", helmPackagePath) helmTemplateWithCRDCmd.Stderr = os.Stderr @@ -51,10 +51,13 @@ func Test_generate(t *testing.T) { t.Fatalf("templating Helm chart with CRDs failed: %v", err) } if !strings.Contains(installContentsWithCRD.String(), "name: connectors.tailscale.com") { - t.Errorf("CRD not found in default chart install") + t.Errorf("Connector CRD not found in default chart install") + } + if !strings.Contains(installContentsWithCRD.String(), "name: proxyclasses.tailscale.com") { + t.Errorf("ProxyClass CRD not found in default chart install") } - // Test that CRD can be excluded from Helm chart install + // Test that CRDs can be excluded from Helm chart install installContentsWithoutCRD := bytes.NewBuffer([]byte{}) helmTemplateWithoutCRDCmd := exec.Command(helmCLIPath, "template", helmPackagePath, "--set", "installCRDs=false") helmTemplateWithoutCRDCmd.Stderr = os.Stderr @@ -63,6 +66,9 @@ func Test_generate(t *testing.T) { t.Fatalf("templating Helm chart without CRDs failed: %v", err) } if strings.Contains(installContentsWithoutCRD.String(), "name: connectors.tailscale.com") { - t.Errorf("CRD found in chart install that should not contain a CRD") + t.Errorf("Connector CRD found in chart install that should not contain a CRD") + } + if strings.Contains(installContentsWithoutCRD.String(), "name: connectors.tailscale.com") { + t.Errorf("ProxyClass CRD found in chart install that should not contain a CRD") } } diff --git a/cmd/k8s-operator/ingress.go b/cmd/k8s-operator/ingress.go index f0c176924..8335506bc 100644 --- a/cmd/k8s-operator/ingress.go +++ b/cmd/k8s-operator/ingress.go @@ -132,6 +132,17 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga return fmt.Errorf("failed to add finalizer: %w", err) } } + + proxyClass := proxyClassForObject(ing) + if proxyClass != "" { + if ready, err := proxyClassIsReady(ctx, proxyClass, a.Client); err != nil { + return fmt.Errorf("error verifying ProxyClass for Ingress: %w", err) + } else if !ready { + logger.Infof("ProxyClass %s specified for the Ingress, but is not (yet) Ready, waiting..", proxyClass) + return nil + } + } + a.mu.Lock() a.managedIngresses.Add(ing.UID) gaugeIngressResources.Set(int64(a.managedIngresses.Len())) @@ -253,6 +264,7 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga ServeConfig: sc, Tags: tags, ChildResourceLabels: crl, + ProxyClass: proxyClass, } if val := ing.GetAnnotations()[AnnotationExperimentalForwardClusterTrafficViaL7IngresProxy]; val == "true" { diff --git a/cmd/k8s-operator/ingress_test.go b/cmd/k8s-operator/ingress_test.go index 3a87e5184..bdbe98d81 100644 --- a/cmd/k8s-operator/ingress_test.go +++ b/cmd/k8s-operator/ingress_test.go @@ -16,6 +16,7 @@ import ( "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client/fake" "tailscale.com/ipn" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/types/ptr" "tailscale.com/util/mak" ) @@ -101,7 +102,7 @@ func TestTailscaleIngress(t *testing.T) { expectEqual(t, fc, expectedSecret(t, opts)) expectEqual(t, fc, expectedHeadlessService(shortName, "ingress")) - expectEqual(t, fc, expectedSTSUserspace(opts)) + expectEqual(t, fc, expectedSTSUserspace(t, fc, opts)) // 2. Ingress status gets updated with ingress proxy's MagicDNS name // once that becomes available. @@ -125,7 +126,7 @@ func TestTailscaleIngress(t *testing.T) { }) opts.shouldEnableForwardingClusterTrafficViaIngress = true expectReconciled(t, ingR, "default", "test") - expectEqual(t, fc, expectedSTS(opts)) + expectEqual(t, fc, expectedSTS(t, fc, opts)) // 4. Resources get cleaned up when Ingress class is unset mustUpdate(t, fc, "default", "test", func(ing *networkingv1.Ingress) { @@ -137,3 +138,133 @@ func TestTailscaleIngress(t *testing.T) { expectMissing[corev1.Service](t, fc, "operator-ns", shortName) expectMissing[corev1.Secret](t, fc, "operator-ns", fullName) } + +func TestTailscaleIngressWithProxyClass(t *testing.T) { + // Setup + pc := &tsapi.ProxyClass{ + ObjectMeta: metav1.ObjectMeta{Name: "custom-metadata"}, + Spec: tsapi.ProxyClassSpec{StatefulSet: &tsapi.StatefulSet{ + Labels: map[string]string{"foo": "bar"}, + Annotations: map[string]string{"bar.io/foo": "some-val"}, + Pod: &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}}}}, + } + tsIngressClass := &networkingv1.IngressClass{ObjectMeta: metav1.ObjectMeta{Name: "tailscale"}, Spec: networkingv1.IngressClassSpec{Controller: "tailscale.com/ts-ingress"}} + fc := fake.NewClientBuilder(). + WithScheme(tsapi.GlobalScheme). + WithObjects(pc, tsIngressClass). + WithStatusSubresource(pc). + Build() + ft := &fakeTSClient{} + fakeTsnetServer := &fakeTSNetServer{certDomains: []string{"foo.com"}} + zl, err := zap.NewDevelopment() + if err != nil { + t.Fatal(err) + } + ingR := &IngressReconciler{ + Client: fc, + ssr: &tailscaleSTSReconciler{ + Client: fc, + tsClient: ft, + tsnetServer: fakeTsnetServer, + defaultTags: []string{"tag:k8s"}, + operatorNamespace: "operator-ns", + proxyImage: "tailscale/tailscale", + }, + logger: zl.Sugar(), + } + + // 1. Ingress is created with no ProxyClass specified, default proxy + // resources get configured. + ing := &networkingv1.Ingress{ + TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + // The apiserver is supposed to set the UID, but the fake client + // doesn't. So, set it explicitly because other code later depends + // on it being set. + UID: types.UID("1234-UID"), + }, + Spec: networkingv1.IngressSpec{ + IngressClassName: ptr.To("tailscale"), + DefaultBackend: &networkingv1.IngressBackend{ + Service: &networkingv1.IngressServiceBackend{ + Name: "test", + Port: networkingv1.ServiceBackendPort{ + Number: 8080, + }, + }, + }, + TLS: []networkingv1.IngressTLS{ + {Hosts: []string{"default-test"}}, + }, + }, + } + mustCreate(t, fc, ing) + mustCreate(t, fc, &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + }, + Spec: corev1.ServiceSpec{ + ClusterIP: "1.2.3.4", + Ports: []corev1.ServicePort{{ + Port: 8080, + Name: "http"}, + }, + }, + }) + + expectReconciled(t, ingR, "default", "test") + + fullName, shortName := findGenName(t, fc, "default", "test", "ingress") + opts := configOpts{ + stsName: shortName, + secretName: fullName, + namespace: "default", + parentType: "ingress", + hostname: "default-test", + } + serveConfig := &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}}, + Web: map[ipn.HostPort]*ipn.WebServerConfig{"${TS_CERT_DOMAIN}:443": {Handlers: map[string]*ipn.HTTPHandler{"/": {Proxy: "http://1.2.3.4:8080/"}}}}, + } + opts.serveConfig = serveConfig + + expectEqual(t, fc, expectedSecret(t, opts)) + expectEqual(t, fc, expectedHeadlessService(shortName, "ingress")) + expectEqual(t, fc, expectedSTSUserspace(t, fc, opts)) + + // 2. Ingress is updated to specify a ProxyClass, ProxyClass is not yet + // ready, so proxy resource configuration does not change. + mustUpdate(t, fc, "default", "test", func(ing *networkingv1.Ingress) { + mak.Set(&ing.ObjectMeta.Labels, LabelProxyClass, "custom-metadata") + }) + expectReconciled(t, ingR, "default", "test") + expectEqual(t, fc, expectedSTSUserspace(t, fc, opts)) + + // 3. ProxyClass is set to Ready by proxy-class reconciler. Ingress get + // reconciled and configuration from the ProxyClass is applied to the + // created proxy resources. + mustUpdateStatus(t, fc, "", "custom-metadata", func(pc *tsapi.ProxyClass) { + pc.Status = tsapi.ProxyClassStatus{ + Conditions: []tsapi.ConnectorCondition{{ + Status: metav1.ConditionTrue, + Type: tsapi.ProxyClassready, + ObservedGeneration: pc.Generation, + }}} + }) + expectReconciled(t, ingR, "default", "test") + opts.proxyClass = pc.Name + expectEqual(t, fc, expectedSTSUserspace(t, fc, opts)) + + // 4. tailscale.com/proxy-class label is removed from the Ingress, the + // Ingress gets reconciled and the custom ProxyClass configuration is + // removed from the proxy resources. + mustUpdate(t, fc, "default", "test", func(ing *networkingv1.Ingress) { + delete(ing.ObjectMeta.Labels, LabelProxyClass) + }) + expectReconciled(t, ingR, "default", "test") + opts.proxyClass = "" + expectEqual(t, fc, expectedSTSUserspace(t, fc, opts)) +} diff --git a/cmd/k8s-operator/operator.go b/cmd/k8s-operator/operator.go index 483a88bba..236c83ad8 100644 --- a/cmd/k8s-operator/operator.go +++ b/cmd/k8s-operator/operator.go @@ -233,6 +233,9 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string svcFilter := handler.EnqueueRequestsFromMapFunc(serviceHandler) svcChildFilter := handler.EnqueueRequestsFromMapFunc(managedResourceHandlerForType("svc")) + // If a ProxyClassChanges, enqueue all Services labeled with that + // ProxyClass's name. + proxyClassFilterForSvc := handler.EnqueueRequestsFromMapFunc(proxyClassHandlerForSvc(mgr.GetClient(), startlog)) eventRecorder := mgr.GetEventRecorderFor("tailscale-operator") ssr := &tailscaleSTSReconciler{ @@ -251,6 +254,7 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string Watches(&corev1.Service{}, svcFilter). Watches(&appsv1.StatefulSet{}, svcChildFilter). Watches(&corev1.Secret{}, svcChildFilter). + Watches(&tsapi.ProxyClass{}, proxyClassFilterForSvc). Complete(&ServiceReconciler{ ssr: ssr, Client: mgr.GetClient(), @@ -259,15 +263,19 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string recorder: eventRecorder, }) if err != nil { - startlog.Fatalf("could not create controller: %v", err) + startlog.Fatalf("could not create service reconciler: %v", err) } ingressChildFilter := handler.EnqueueRequestsFromMapFunc(managedResourceHandlerForType("ingress")) + // If a ProxyClassChanges, enqueue all Ingresses labeled with that + // ProxyClass's name. + proxyClassFilterForIngress := handler.EnqueueRequestsFromMapFunc(proxyClassHandlerForIngress(mgr.GetClient(), startlog)) err = builder. ControllerManagedBy(mgr). For(&networkingv1.Ingress{}). Watches(&appsv1.StatefulSet{}, ingressChildFilter). Watches(&corev1.Secret{}, ingressChildFilter). Watches(&corev1.Service{}, ingressChildFilter). + Watches(&tsapi.ProxyClass{}, proxyClassFilterForIngress). Complete(&IngressReconciler{ ssr: ssr, recorder: eventRecorder, @@ -275,14 +283,18 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string logger: zlog.Named("ingress-reconciler"), }) if err != nil { - startlog.Fatalf("could not create controller: %v", err) + startlog.Fatalf("could not create ingress reconciler: %v", err) } connectorFilter := handler.EnqueueRequestsFromMapFunc(managedResourceHandlerForType("connector")) + // If a ProxyClassChanges, enqueue all Connectors that have + // .spec.proxyClass set to the name of this ProxyClass. + proxyClassFilterForConnector := handler.EnqueueRequestsFromMapFunc(proxyClassHandlerForConnector(mgr.GetClient(), startlog)) err = builder.ControllerManagedBy(mgr). For(&tsapi.Connector{}). Watches(&appsv1.StatefulSet{}, connectorFilter). Watches(&corev1.Secret{}, connectorFilter). + Watches(&tsapi.ProxyClass{}, proxyClassFilterForConnector). Complete(&ConnectorReconciler{ ssr: ssr, recorder: eventRecorder, @@ -293,6 +305,17 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string if err != nil { startlog.Fatal("could not create connector reconciler: %v", err) } + err = builder.ControllerManagedBy(mgr). + For(&tsapi.ProxyClass{}). + Complete(&ProxyClassReconciler{ + Client: mgr.GetClient(), + recorder: eventRecorder, + logger: zlog.Named("proxyclass-reconciler"), + clock: tstime.DefaultClock{}, + }) + if err != nil { + startlog.Fatal("could not create proxyclass reconciler: %v", err) + } startlog.Infof("Startup complete, operator running, version: %s", version.Long()) if err := mgr.Start(signals.SetupSignalHandler()); err != nil { startlog.Fatalf("could not start manager: %v", err) @@ -321,6 +344,7 @@ func parentFromObjectLabels(o client.Object) types.NamespacedName { Name: ls[LabelParentName], } } + func managedResourceHandlerForType(typ string) handler.MapFunc { return func(_ context.Context, o client.Object) []reconcile.Request { if !isManagedByType(o, typ) { @@ -330,14 +354,75 @@ func managedResourceHandlerForType(typ string) handler.MapFunc { {NamespacedName: parentFromObjectLabels(o)}, } } +} +// proxyClassHandlerForSvc returns a handler that, for a given ProxyClass, +// returns a list of reconcile requests for all Services labeled with +// tailscale.com/proxy-class: . +func proxyClassHandlerForSvc(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc { + return func(ctx context.Context, o client.Object) []reconcile.Request { + svcList := new(corev1.ServiceList) + labels := map[string]string{ + LabelProxyClass: o.GetName(), + } + if err := cl.List(ctx, svcList, client.MatchingLabels(labels)); err != nil { + logger.Debugf("error listing Services for ProxyClass: %v", err) + return nil + } + reqs := make([]reconcile.Request, 0) + for _, svc := range svcList.Items { + reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&svc)}) + } + return reqs + } +} + +// proxyClassHandlerForIngress returns a handler that, for a given ProxyClass, +// returns a list of reconcile requests for all Ingresses labeled with +// tailscale.com/proxy-class: . +func proxyClassHandlerForIngress(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc { + return func(ctx context.Context, o client.Object) []reconcile.Request { + ingList := new(networkingv1.IngressList) + labels := map[string]string{ + LabelProxyClass: o.GetName(), + } + if err := cl.List(ctx, ingList, client.MatchingLabels(labels)); err != nil { + logger.Debugf("error listing Ingresses for ProxyClass: %v", err) + return nil + } + reqs := make([]reconcile.Request, 0) + for _, ing := range ingList.Items { + reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&ing)}) + } + return reqs + } +} + +// proxyClassHandlerForConnector returns a handler that, for a given ProxyClass, +// returns a list of reconcile requests for all Connectors that have +// .spec.proxyClass set. +func proxyClassHandlerForConnector(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc { + return func(ctx context.Context, o client.Object) []reconcile.Request { + connList := new(tsapi.ConnectorList) + if err := cl.List(ctx, connList); err != nil { + logger.Debugf("error listing Connectors for ProxyClass: %v", err) + return nil + } + reqs := make([]reconcile.Request, 0) + proxyClassName := o.GetName() + for _, conn := range connList.Items { + if conn.Spec.ProxyClass == proxyClassName { + reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&conn)}) + } + } + return reqs + } } func serviceHandler(_ context.Context, o client.Object) []reconcile.Request { if isManagedByType(o, "svc") { // If this is a Service managed by a Service we want to enqueue its parent return []reconcile.Request{{NamespacedName: parentFromObjectLabels(o)}} - } if isManagedResource(o) { // If this is a Servce managed by a resource that is not a Service, we leave it alone diff --git a/cmd/k8s-operator/operator_test.go b/cmd/k8s-operator/operator_test.go index 942147477..90ed243bb 100644 --- a/cmd/k8s-operator/operator_test.go +++ b/cmd/k8s-operator/operator_test.go @@ -15,7 +15,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client/fake" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/types/ptr" + "tailscale.com/util/mak" ) func TestLoadBalancerClass(t *testing.T) { @@ -69,7 +71,7 @@ func TestLoadBalancerClass(t *testing.T) { expectEqual(t, fc, expectedSecret(t, opts)) expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(opts)) + expectEqual(t, fc, expectedSTS(t, fc, opts)) // Normally the Tailscale proxy pod would come up here and write its info // into the secret. Simulate that, then verify reconcile again and verify @@ -210,7 +212,7 @@ func TestTailnetTargetFQDNAnnotation(t *testing.T) { expectEqual(t, fc, expectedSecret(t, o)) expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(o)) + expectEqual(t, fc, expectedSTS(t, fc, o)) want := &corev1.Service{ TypeMeta: metav1.TypeMeta{ Kind: "Service", @@ -234,7 +236,7 @@ func TestTailnetTargetFQDNAnnotation(t *testing.T) { expectEqual(t, fc, want) expectEqual(t, fc, expectedSecret(t, o)) expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(o)) + expectEqual(t, fc, expectedSTS(t, fc, o)) // Change the tailscale-target-fqdn annotation which should update the // StatefulSet @@ -320,7 +322,7 @@ func TestTailnetTargetIPAnnotation(t *testing.T) { expectEqual(t, fc, expectedSecret(t, o)) expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(o)) + expectEqual(t, fc, expectedSTS(t, fc, o)) want := &corev1.Service{ TypeMeta: metav1.TypeMeta{ Kind: "Service", @@ -344,7 +346,7 @@ func TestTailnetTargetIPAnnotation(t *testing.T) { expectEqual(t, fc, want) expectEqual(t, fc, expectedSecret(t, o)) expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(o)) + expectEqual(t, fc, expectedSTS(t, fc, o)) // Change the tailscale-target-ip annotation which should update the // StatefulSet @@ -427,7 +429,7 @@ func TestAnnotations(t *testing.T) { expectEqual(t, fc, expectedSecret(t, o)) expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(o)) + expectEqual(t, fc, expectedSTS(t, fc, o)) want := &corev1.Service{ TypeMeta: metav1.TypeMeta{ Kind: "Service", @@ -535,7 +537,7 @@ func TestAnnotationIntoLB(t *testing.T) { expectEqual(t, fc, expectedSecret(t, o)) expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(o)) + expectEqual(t, fc, expectedSTS(t, fc, o)) // Normally the Tailscale proxy pod would come up here and write its info // into the secret. Simulate that, since it would have normally happened at @@ -580,7 +582,7 @@ func TestAnnotationIntoLB(t *testing.T) { expectReconciled(t, sr, "default", "test") // None of the proxy machinery should have changed... expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(o)) + expectEqual(t, fc, expectedSTS(t, fc, o)) // ... but the service should have a LoadBalancer status. want = &corev1.Service{ @@ -666,7 +668,7 @@ func TestLBIntoAnnotation(t *testing.T) { expectEqual(t, fc, expectedSecret(t, o)) expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(o)) + expectEqual(t, fc, expectedSTS(t, fc, o)) // Normally the Tailscale proxy pod would come up here and write its info // into the secret. Simulate that, then verify reconcile again and verify @@ -729,7 +731,7 @@ func TestLBIntoAnnotation(t *testing.T) { expectReconciled(t, sr, "default", "test") expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(o)) + expectEqual(t, fc, expectedSTS(t, fc, o)) want = &corev1.Service{ TypeMeta: metav1.TypeMeta{ @@ -807,7 +809,7 @@ func TestCustomHostname(t *testing.T) { expectEqual(t, fc, expectedSecret(t, o)) expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(o)) + expectEqual(t, fc, expectedSTS(t, fc, o)) want := &corev1.Service{ TypeMeta: metav1.TypeMeta{ Kind: "Service", @@ -920,7 +922,104 @@ func TestCustomPriorityClassName(t *testing.T) { clusterTargetIP: "10.20.30.40", } - expectEqual(t, fc, expectedSTS(o)) + expectEqual(t, fc, expectedSTS(t, fc, o)) +} + +func TestProxyClassForService(t *testing.T) { + // Setup + pc := &tsapi.ProxyClass{ + ObjectMeta: metav1.ObjectMeta{Name: "custom-metadata"}, + Spec: tsapi.ProxyClassSpec{StatefulSet: &tsapi.StatefulSet{ + Labels: map[string]string{"foo": "bar"}, + Annotations: map[string]string{"bar.io/foo": "some-val"}, + Pod: &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}}}}, + } + fc := fake.NewClientBuilder(). + WithScheme(tsapi.GlobalScheme). + WithObjects(pc). + WithStatusSubresource(pc). + Build() + ft := &fakeTSClient{} + zl, err := zap.NewDevelopment() + if err != nil { + t.Fatal(err) + } + sr := &ServiceReconciler{ + Client: fc, + ssr: &tailscaleSTSReconciler{ + Client: fc, + tsClient: ft, + defaultTags: []string{"tag:k8s"}, + operatorNamespace: "operator-ns", + proxyImage: "tailscale/tailscale", + }, + logger: zl.Sugar(), + } + + // 1. A new tailscale LoadBalancer Service is created without any + // ProxyClass. Resources get created for it as usual. + mustCreate(t, fc, &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + // The apiserver is supposed to set the UID, but the fake client + // doesn't. So, set it explicitly because other code later depends + // on it being set. + UID: types.UID("1234-UID"), + }, + Spec: corev1.ServiceSpec{ + ClusterIP: "10.20.30.40", + Type: corev1.ServiceTypeLoadBalancer, + LoadBalancerClass: ptr.To("tailscale"), + }, + }) + expectReconciled(t, sr, "default", "test") + fullName, shortName := findGenName(t, fc, "default", "test", "svc") + opts := configOpts{ + stsName: shortName, + secretName: fullName, + namespace: "default", + parentType: "svc", + hostname: "default-test", + clusterTargetIP: "10.20.30.40", + } + expectEqual(t, fc, expectedSecret(t, opts)) + expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) + expectEqual(t, fc, expectedSTS(t, fc, opts)) + + // 2. The Service gets updated with tailscale.com/proxy-class label + // pointing at the 'custom-metadata' ProxyClass. The ProxyClass is not + // yet ready, so no changes are actually applied to the proxy resources. + mustUpdate(t, fc, "default", "test", func(svc *corev1.Service) { + mak.Set(&svc.Labels, LabelProxyClass, "custom-metadata") + }) + expectReconciled(t, sr, "default", "test") + expectEqual(t, fc, expectedSTS(t, fc, opts)) + + // 3. ProxyClass is set to Ready, the Service gets reconciled by the + // services-reconciler and the customization from the ProxyClass is + // applied to the proxy resources. + mustUpdateStatus(t, fc, "", "custom-metadata", func(pc *tsapi.ProxyClass) { + pc.Status = tsapi.ProxyClassStatus{ + Conditions: []tsapi.ConnectorCondition{{ + Status: metav1.ConditionTrue, + Type: tsapi.ProxyClassready, + ObservedGeneration: pc.Generation, + }}} + }) + opts.proxyClass = pc.Name + expectReconciled(t, sr, "default", "test") + expectEqual(t, fc, expectedSTS(t, fc, opts)) + + // 4. tailscale.com/proxy-class label is removed from the Service, the + // configuration from the ProxyClass is removed from the cluster + // resources. + mustUpdate(t, fc, "default", "test", func(svc *corev1.Service) { + delete(svc.Labels, LabelProxyClass) + }) + opts.proxyClass = "" + expectReconciled(t, sr, "default", "test") + expectEqual(t, fc, expectedSTS(t, fc, opts)) } func TestDefaultLoadBalancer(t *testing.T) { @@ -973,7 +1072,7 @@ func TestDefaultLoadBalancer(t *testing.T) { hostname: "default-test", clusterTargetIP: "10.20.30.40", } - expectEqual(t, fc, expectedSTS(o)) + expectEqual(t, fc, expectedSTS(t, fc, o)) } func TestProxyFirewallMode(t *testing.T) { @@ -1026,7 +1125,7 @@ func TestProxyFirewallMode(t *testing.T) { firewallMode: "nftables", clusterTargetIP: "10.20.30.40", } - expectEqual(t, fc, expectedSTS(o)) + expectEqual(t, fc, expectedSTS(t, fc, o)) } diff --git a/cmd/k8s-operator/proxyclass.go b/cmd/k8s-operator/proxyclass.go new file mode 100644 index 000000000..74ed71e6b --- /dev/null +++ b/cmd/k8s-operator/proxyclass.go @@ -0,0 +1,108 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +// tailscale-operator provides a way to expose services running in a Kubernetes +// cluster to your Tailnet. +package main + +import ( + "context" + "fmt" + + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apivalidation "k8s.io/apimachinery/pkg/api/validation" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + metavalidation "k8s.io/apimachinery/pkg/apis/meta/v1/validation" + "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + tsoperator "tailscale.com/k8s-operator" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/tstime" +) + +const ( + reasonProxyClassInvalid = "ProxyClassInvalid" + reasonProxyClassValid = "ProxyClassValid" + messageProxyClassInvalid = "ProxyClass is not valid: %v" +) + +type ProxyClassReconciler struct { + client.Client + + recorder record.EventRecorder + logger *zap.SugaredLogger + clock tstime.Clock +} + +func (pcr *ProxyClassReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) { + logger := pcr.logger.With("ProxyClass", req.Name) + logger.Debugf("starting reconcile") + defer logger.Debugf("reconcile finished") + + pc := new(tsapi.ProxyClass) + err = pcr.Get(ctx, req.NamespacedName, pc) + if apierrors.IsNotFound(err) { + logger.Debugf("ProxyClass not found, assuming it was deleted") + return reconcile.Result{}, nil + } else if err != nil { + return reconcile.Result{}, fmt.Errorf("failed to get tailscale.com ProxyClass: %w", err) + } + if !pc.DeletionTimestamp.IsZero() { + logger.Debugf("ProxyClass is being deleted, do nothing") + return reconcile.Result{}, nil + } + oldPCStatus := pc.Status.DeepCopy() + if errs := pcr.validate(pc); errs != nil { + msg := fmt.Sprintf(messageProxyClassInvalid, errs.ToAggregate().Error()) + pcr.recorder.Event(pc, corev1.EventTypeWarning, reasonProxyClassInvalid, msg) + tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassready, metav1.ConditionFalse, reasonProxyClassInvalid, msg, pc.Generation, pcr.clock, logger) + } else { + tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassready, metav1.ConditionTrue, reasonProxyClassValid, reasonProxyClassValid, pc.Generation, pcr.clock, logger) + } + if !apiequality.Semantic.DeepEqual(oldPCStatus, pc.Status) { + if err := pcr.Client.Status().Update(ctx, pc); err != nil { + logger.Errorf("error updating ProxyClass status: %v", err) + return reconcile.Result{}, err + } + } + return reconcile.Result{}, nil +} + +func (a *ProxyClassReconciler) validate(pc *tsapi.ProxyClass) (violations field.ErrorList) { + if sts := pc.Spec.StatefulSet; sts != nil { + if len(sts.Labels) > 0 { + if errs := metavalidation.ValidateLabels(sts.Labels, field.NewPath(".spec.statefulSet.labels")); errs != nil { + violations = append(violations, errs...) + } + } + if len(sts.Annotations) > 0 { + if errs := apivalidation.ValidateAnnotations(sts.Annotations, field.NewPath(".spec.statefulSet.annotations")); errs != nil { + violations = append(violations, errs...) + } + } + if pod := sts.Pod; pod != nil { + if len(pod.Labels) > 0 { + if errs := metavalidation.ValidateLabels(pod.Labels, field.NewPath(".spec.statefulSet.pod.labels")); errs != nil { + violations = append(violations, errs...) + } + } + if len(pod.Annotations) > 0 { + if errs := apivalidation.ValidateAnnotations(pod.Annotations, field.NewPath(".spec.statefulSet.pod.annotations")); errs != nil { + violations = append(violations, errs...) + } + } + } + } + // We do not validate embedded fields (security context, resource + // requirements etc) as we inherit upstream validation for those fields. + // Invalid values would get rejected by upstream validations at apply + // time. + return violations +} diff --git a/cmd/k8s-operator/proxyclass_test.go b/cmd/k8s-operator/proxyclass_test.go new file mode 100644 index 000000000..aada3b2cb --- /dev/null +++ b/cmd/k8s-operator/proxyclass_test.go @@ -0,0 +1,83 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +// tailscale-operator provides a way to expose services running in a Kubernetes +// cluster to your Tailnet. +package main + +import ( + "testing" + "time" + + "go.uber.org/zap" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + tsoperator "tailscale.com/k8s-operator" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/tstest" +) + +func TestProxyClass(t *testing.T) { + pc := &tsapi.ProxyClass{ + TypeMeta: metav1.TypeMeta{Kind: "ProxyClass", APIVersion: "tailscale.com/v1alpha1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + // The apiserver is supposed to set the UID, but the fake client + // doesn't. So, set it explicitly because other code later depends + // on it being set. + UID: types.UID("1234-UID"), + }, + Spec: tsapi.ProxyClassSpec{ + StatefulSet: &tsapi.StatefulSet{ + Labels: map[string]string{"foo": "bar", "xyz1234": "abc567"}, + Annotations: map[string]string{"foo.io/bar": "{'key': 'val1232'}"}, + Pod: &tsapi.Pod{ + Labels: map[string]string{"foo": "bar", "xyz1234": "abc567"}, + Annotations: map[string]string{"foo.io/bar": "{'key': 'val1232'}"}, + }, + }, + }, + } + fc := fake.NewClientBuilder(). + WithScheme(tsapi.GlobalScheme). + WithObjects(pc). + WithStatusSubresource(pc). + Build() + zl, err := zap.NewDevelopment() + if err != nil { + t.Fatal(err) + } + cl := tstest.NewClock(tstest.ClockOpts{}) + pcr := &ProxyClassReconciler{ + Client: fc, + logger: zl.Sugar(), + clock: cl, + recorder: record.NewFakeRecorder(1), + } + expectReconciled(t, pcr, "", "test") + + // 1. A valid ProxyClass resource gets its status updated to Ready. + pc.Status.Conditions = append(pc.Status.Conditions, tsapi.ConnectorCondition{ + Type: tsapi.ProxyClassready, + Status: metav1.ConditionTrue, + Reason: reasonProxyClassValid, + Message: reasonProxyClassValid, + LastTransitionTime: &metav1.Time{Time: cl.Now().Truncate(time.Second)}, + }) + + expectEqual(t, fc, pc) + + // 2. An invalid ProxyClass resource gets its status updated to Invalid. + pc.Spec.StatefulSet.Labels["foo"] = "?!someVal" + mustUpdate(t, fc, "", "test", func(proxyClass *tsapi.ProxyClass) { + proxyClass.Spec.StatefulSet.Labels = pc.Spec.StatefulSet.Labels + }) + expectReconciled(t, pcr, "", "test") + msg := `ProxyClass is not valid: .spec.statefulSet.labels: Invalid value: "?!someVal": a valid label must be an empty string or consist of alphanumeric characters, '-', '_' or '.', and must start and end with an alphanumeric character (e.g. 'MyValue', or 'my_value', or '12345', regex used for validation is '(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?')` + tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassready, metav1.ConditionFalse, reasonProxyClassInvalid, msg, 0, cl, zl.Sugar()) + expectEqual(t, fc, pc) +} diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go index cd1a585ca..87f114b63 100644 --- a/cmd/k8s-operator/sts.go +++ b/cmd/k8s-operator/sts.go @@ -14,6 +14,7 @@ import ( "fmt" "net/http" "os" + "slices" "strings" "go.uber.org/zap" @@ -22,11 +23,14 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" "k8s.io/apiserver/pkg/storage/names" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/yaml" "tailscale.com/client/tailscale" "tailscale.com/ipn" + tsoperator "tailscale.com/k8s-operator" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/net/netutil" "tailscale.com/tailcfg" "tailscale.com/types/opt" @@ -35,11 +39,20 @@ import ( ) const ( + // Labels that the operator sets on StatefulSets and Pods. If you add a + // new label here, do also add it to tailscaleManagedLabels var to + // ensure that it does not get overwritten by ProxyClass configuration. LabelManaged = "tailscale.com/managed" LabelParentType = "tailscale.com/parent-resource-type" LabelParentName = "tailscale.com/parent-resource" LabelParentNamespace = "tailscale.com/parent-resource-ns" + // LabelProxyClass can be set by users on Connectors, tailscale + // Ingresses and Services that define cluster ingress or cluster egress, + // to specify that configuration in this ProxyClass should be applied to + // resources created for the Connector, Ingress or Service. + LabelProxyClass = "tailscale.com/proxy-class" + FinalizerName = "tailscale.com/finalizer" // Annotations settable by users on services. @@ -68,7 +81,10 @@ const ( AnnotationExperimentalForwardClusterTrafficViaL7IngresProxy = "tailscale.com/experimental-forward-cluster-traffic-via-ingress" // Annotations set by the operator on pods to trigger restarts when the - // hostname, IP, FQDN or tailscaled config changes. + // hostname, IP, FQDN or tailscaled config changes. If you add a new + // annotation here, also add it to tailscaleManagedAnnotations var to + // ensure that it does not get removed when a ProxyClass configuration + // is applied. podAnnotationLastSetClusterIP = "tailscale.com/operator-last-set-cluster-ip" podAnnotationLastSetHostname = "tailscale.com/operator-last-set-hostname" podAnnotationLastSetTailnetTargetIP = "tailscale.com/operator-last-set-ts-tailnet-target-ip" @@ -81,6 +97,13 @@ const ( tailscaledConfigKey = "tailscaled" ) +var ( + // tailscaleManagedLabels are label keys that tailscale operator sets on StatefulSets and Pods. + tailscaleManagedLabels = []string{LabelManaged, LabelParentType, LabelParentName, LabelParentNamespace, "app"} + // tailscaleManagedAnnotations are annotation keys that tailscale operator sets on StatefulSets and Pods. + tailscaleManagedAnnotations = []string{podAnnotationLastSetClusterIP, podAnnotationLastSetHostname, podAnnotationLastSetTailnetTargetIP, podAnnotationLastSetTailnetTargetFQDN, podAnnotationLastSetConfigFileHash} +) + type tailscaleSTSConfig struct { ParentResourceName string ParentResourceUID string @@ -102,6 +125,8 @@ type tailscaleSTSConfig struct { // Connector specifies a configuration of a Connector instance if that's // what this StatefulSet should be created for. Connector *connector + + ProxyClass string } type connector struct { @@ -398,10 +423,10 @@ var proxyYaml []byte var userspaceProxyYaml []byte func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.SugaredLogger, sts *tailscaleSTSConfig, headlessSvc *corev1.Service, proxySecret, tsConfigHash string) (*appsv1.StatefulSet, error) { - var ss appsv1.StatefulSet + ss := new(appsv1.StatefulSet) if sts.ServeConfig != nil && sts.ForwardClusterTrafficViaL7IngressProxy != true { // If forwarding cluster traffic via is required we need non-userspace + NET_ADMIN + forwarding if err := yaml.Unmarshal(userspaceProxyYaml, &ss); err != nil { - return nil, fmt.Errorf("failed to unmarshal proxy spec: %w", err) + return nil, fmt.Errorf("failed to unmarshal userspace proxy spec: %v", err) } } else { if err := yaml.Unmarshal(proxyYaml, &ss); err != nil { @@ -415,12 +440,25 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S } } } - container := &ss.Spec.Template.Spec.Containers[0] + pod := &ss.Spec.Template + container := &pod.Spec.Containers[0] + proxyClass := new(tsapi.ProxyClass) + if sts.ProxyClass != "" { + if err := a.Get(ctx, types.NamespacedName{Name: sts.ProxyClass}, proxyClass); err != nil { + return nil, fmt.Errorf("failed to get ProxyClass: %w", err) + } + if !tsoperator.ProxyClassIsReady(proxyClass) { + logger.Infof("ProxyClass %s specified for the proxy, but it is not (yet) in a ready state, waiting..") + return nil, nil + } + } container.Image = a.proxyImage ss.ObjectMeta = metav1.ObjectMeta{ Name: headlessSvc.Name, Namespace: a.operatorNamespace, - Labels: sts.ChildResourceLabels, + } + for key, val := range sts.ChildResourceLabels { + mak.Set(&ss.ObjectMeta.Labels, key, val) } ss.Spec.ServiceName = headlessSvc.Name ss.Spec.Selector = &metav1.LabelSelector{ @@ -428,9 +466,9 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S "app": sts.ParentResourceUID, }, } - mak.Set(&ss.Spec.Template.Labels, "app", sts.ParentResourceUID) + mak.Set(&pod.Labels, "app", sts.ParentResourceUID) for key, val := range sts.ChildResourceLabels { - ss.Spec.Template.Labels[key] = val // sync StatefulSet labels to Pod to make it easier for users to select the Pod + pod.Labels[key] = val // sync StatefulSet labels to Pod to make it easier for users to select the Pod } // Generic containerboot configuration options. @@ -455,12 +493,12 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S // it is passed via an environment variable. So we need to restart the // container when the value changes. We do this by adding an annotation to // the pod template that contains the last value we set. - mak.Set(&ss.Spec.Template.Annotations, podAnnotationLastSetHostname, sts.Hostname) + mak.Set(&pod.Annotations, podAnnotationLastSetHostname, sts.Hostname) } // Configure containeboot to run tailscaled with a configfile read from the state Secret. if shouldDoTailscaledDeclarativeConfig(sts) { mak.Set(&ss.Spec.Template.Annotations, podAnnotationLastSetConfigFileHash, tsConfigHash) - ss.Spec.Template.Spec.Volumes = append(ss.Spec.Template.Spec.Volumes, corev1.Volume{ + pod.Spec.Volumes = append(ss.Spec.Template.Spec.Volumes, corev1.Volume{ Name: "tailscaledconfig", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ @@ -489,7 +527,7 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S Value: a.tsFirewallMode, }) } - ss.Spec.Template.Spec.PriorityClassName = a.proxyPriorityClassName + pod.Spec.PriorityClassName = a.proxyPriorityClassName // Ingress/egress proxy configuration options. if sts.ClusterTargetIP != "" { @@ -520,7 +558,7 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S ReadOnly: true, MountPath: "/etc/tailscaled", }) - ss.Spec.Template.Spec.Volumes = append(ss.Spec.Template.Spec.Volumes, corev1.Volume{ + pod.Spec.Volumes = append(ss.Spec.Template.Spec.Volumes, corev1.Volume{ Name: "serve-config", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ @@ -534,7 +572,95 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S }) } logger.Debugf("reconciling statefulset %s/%s", ss.GetNamespace(), ss.GetName()) - return createOrUpdate(ctx, a.Client, a.operatorNamespace, &ss, func(s *appsv1.StatefulSet) { s.Spec = ss.Spec }) + if sts.ProxyClass != "" { + logger.Debugf("configuring proxy resources with ProxyClass %s", sts.ProxyClass) + ss = applyProxyClassToStatefulSet(proxyClass, ss) + } + updateSS := func(s *appsv1.StatefulSet) { + s.Spec = ss.Spec + s.ObjectMeta.Labels = ss.Labels + s.ObjectMeta.Annotations = ss.Annotations + } + return createOrUpdate(ctx, a.Client, a.operatorNamespace, ss, updateSS) +} + +// mergeStatefulSetLabelsOrAnnots returns a map that contains all keys/values +// present in 'custom' map as well as those keys/values from the current map +// whose keys are present in the 'managed' map. The reason why this merge is +// necessary is to ensure that labels/annotations applied from a ProxyClass get removed +// if they are removed from a ProxyClass or if the ProxyClass no longer applies +// to this StatefulSet whilst any tailscale managed labels/annotations remain present. +func mergeStatefulSetLabelsOrAnnots(current, custom map[string]string, managed []string) map[string]string { + if custom == nil { + custom = make(map[string]string) + } + if current == nil { + return custom + } + for key, val := range current { + if slices.Contains(managed, key) { + custom[key] = val + } + } + return custom +} + +func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet) *appsv1.StatefulSet { + if pc == nil || ss == nil || pc.Spec.StatefulSet == nil { + return ss + } + + // Update StatefulSet metadata. + if wantsSSLabels := pc.Spec.StatefulSet.Labels; len(wantsSSLabels) > 0 { + ss.ObjectMeta.Labels = mergeStatefulSetLabelsOrAnnots(ss.ObjectMeta.Labels, wantsSSLabels, tailscaleManagedLabels) + } + if wantsSSAnnots := pc.Spec.StatefulSet.Annotations; len(wantsSSAnnots) > 0 { + ss.ObjectMeta.Annotations = mergeStatefulSetLabelsOrAnnots(ss.ObjectMeta.Annotations, wantsSSAnnots, tailscaleManagedAnnotations) + } + + // Update Pod fields. + if pc.Spec.StatefulSet.Pod == nil { + return ss + } + wantsPod := pc.Spec.StatefulSet.Pod + if wantsPodLabels := wantsPod.Labels; len(wantsPodLabels) > 0 { + ss.Spec.Template.ObjectMeta.Labels = mergeStatefulSetLabelsOrAnnots(ss.Spec.Template.ObjectMeta.Labels, wantsPodLabels, tailscaleManagedLabels) + } + if wantsPodAnnots := wantsPod.Annotations; len(wantsPodAnnots) > 0 { + ss.Spec.Template.ObjectMeta.Annotations = mergeStatefulSetLabelsOrAnnots(ss.Spec.Template.ObjectMeta.Annotations, wantsPodAnnots, tailscaleManagedAnnotations) + } + ss.Spec.Template.Spec.SecurityContext = wantsPod.SecurityContext + ss.Spec.Template.Spec.ImagePullSecrets = wantsPod.ImagePullSecrets + ss.Spec.Template.Spec.NodeName = wantsPod.NodeName + ss.Spec.Template.Spec.NodeSelector = wantsPod.NodeSelector + ss.Spec.Template.Spec.Tolerations = wantsPod.Tolerations + + // Update containers. + updateContainer := func(overlay *tsapi.Container, base corev1.Container) corev1.Container { + if overlay == nil { + return base + } + if overlay.SecurityContext != nil { + base.SecurityContext = overlay.SecurityContext + } + base.Resources = overlay.Resources + return base + } + for i, c := range ss.Spec.Template.Spec.Containers { + if c.Name == "tailscale" { + ss.Spec.Template.Spec.Containers[i] = updateContainer(wantsPod.TailscaleContainer, ss.Spec.Template.Spec.Containers[i]) + break + } + } + if initContainers := ss.Spec.Template.Spec.InitContainers; len(initContainers) > 0 { + for i, c := range initContainers { + if c.Name == "sysctler" { + ss.Spec.Template.Spec.InitContainers[i] = updateContainer(wantsPod.TailscaleInitContainer, initContainers[i]) + break + } + } + } + return ss } // tailscaledConfig takes a proxy config, a newly generated auth key if diff --git a/cmd/k8s-operator/sts_test.go b/cmd/k8s-operator/sts_test.go index 0d0fe746b..b1acdfd91 100644 --- a/cmd/k8s-operator/sts_test.go +++ b/cmd/k8s-operator/sts_test.go @@ -6,10 +6,20 @@ package main import ( + _ "embed" "fmt" + "reflect" "regexp" "strings" "testing" + + "github.com/google/go-cmp/cmp" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "sigs.k8s.io/yaml" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/types/ptr" ) // Test_statefulSetNameBase tests that parent name portion in a StatefulSet name @@ -39,3 +49,245 @@ func Test_statefulSetNameBase(t *testing.T) { } } } + +func Test_applyProxyClassToStatefulSet(t *testing.T) { + // Setup + proxyClassAllOpts := &tsapi.ProxyClass{ + Spec: tsapi.ProxyClassSpec{ + StatefulSet: &tsapi.StatefulSet{ + Labels: map[string]string{"foo": "bar"}, + Annotations: map[string]string{"foo.io/bar": "foo"}, + Pod: &tsapi.Pod{ + Labels: map[string]string{"bar": "foo"}, + Annotations: map[string]string{"bar.io/foo": "foo"}, + SecurityContext: &corev1.PodSecurityContext{ + RunAsUser: ptr.To(int64(0)), + }, + ImagePullSecrets: []corev1.LocalObjectReference{{Name: "docker-creds"}}, + NodeName: "some-node", + NodeSelector: map[string]string{"beta.kubernetes.io/os": "linux"}, + Tolerations: []corev1.Toleration{{Key: "", Operator: "Exists"}}, + TailscaleContainer: &tsapi.Container{ + SecurityContext: &corev1.SecurityContext{ + Privileged: ptr.To(true), + }, + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1000m"), corev1.ResourceMemory: resource.MustParse("128Mi")}, + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("500m"), corev1.ResourceMemory: resource.MustParse("64Mi")}, + }, + }, + TailscaleInitContainer: &tsapi.Container{ + SecurityContext: &corev1.SecurityContext{ + Privileged: ptr.To(true), + RunAsUser: ptr.To(int64(0)), + }, + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1000m"), corev1.ResourceMemory: resource.MustParse("128Mi")}, + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("500m"), corev1.ResourceMemory: resource.MustParse("64Mi")}, + }, + }, + }, + }, + }, + } + proxyClassJustLabels := &tsapi.ProxyClass{ + Spec: tsapi.ProxyClassSpec{ + StatefulSet: &tsapi.StatefulSet{ + Labels: map[string]string{"foo": "bar"}, + Annotations: map[string]string{"foo.io/bar": "foo"}, + Pod: &tsapi.Pod{ + Labels: map[string]string{"bar": "foo"}, + Annotations: map[string]string{"bar.io/foo": "foo"}, + }, + }, + }, + } + var userspaceProxySS, nonUserspaceProxySS appsv1.StatefulSet + if err := yaml.Unmarshal(userspaceProxyYaml, &userspaceProxySS); err != nil { + t.Fatalf("unmarshaling userspace proxy template: %v", err) + } + if err := yaml.Unmarshal(proxyYaml, &nonUserspaceProxySS); err != nil { + t.Fatalf("unmarshaling non-userspace proxy template: %v", err) + } + // Set a couple additional fields so we can test that we don't + // mistakenly override those. + labels := map[string]string{ + LabelManaged: "true", + LabelParentName: "foo", + } + annots := map[string]string{ + podAnnotationLastSetClusterIP: "1.2.3.4", + } + env := []corev1.EnvVar{{Name: "TS_HOSTNAME", Value: "nginx"}} + userspaceProxySS.Labels = labels + userspaceProxySS.Annotations = annots + userspaceProxySS.Spec.Template.Spec.Containers[0].Env = env + nonUserspaceProxySS.ObjectMeta.Labels = labels + nonUserspaceProxySS.ObjectMeta.Annotations = annots + nonUserspaceProxySS.Spec.Template.Spec.Containers[0].Env = env + + // 1. Test that a ProxyClass with all fields set gets correctly applied + // to a Statefulset built from non-userspace proxy template. + wantSS := nonUserspaceProxySS.DeepCopy() + wantSS.ObjectMeta.Labels = mergeMapKeys(wantSS.ObjectMeta.Labels, proxyClassAllOpts.Spec.StatefulSet.Labels) + wantSS.ObjectMeta.Annotations = mergeMapKeys(wantSS.ObjectMeta.Annotations, proxyClassAllOpts.Spec.StatefulSet.Annotations) + wantSS.Spec.Template.Labels = proxyClassAllOpts.Spec.StatefulSet.Pod.Labels + wantSS.Spec.Template.Annotations = proxyClassAllOpts.Spec.StatefulSet.Pod.Annotations + wantSS.Spec.Template.Spec.SecurityContext = proxyClassAllOpts.Spec.StatefulSet.Pod.SecurityContext + wantSS.Spec.Template.Spec.ImagePullSecrets = proxyClassAllOpts.Spec.StatefulSet.Pod.ImagePullSecrets + wantSS.Spec.Template.Spec.NodeName = proxyClassAllOpts.Spec.StatefulSet.Pod.NodeName + wantSS.Spec.Template.Spec.NodeSelector = proxyClassAllOpts.Spec.StatefulSet.Pod.NodeSelector + wantSS.Spec.Template.Spec.Tolerations = proxyClassAllOpts.Spec.StatefulSet.Pod.Tolerations + wantSS.Spec.Template.Spec.Containers[0].SecurityContext = proxyClassAllOpts.Spec.StatefulSet.Pod.TailscaleContainer.SecurityContext + wantSS.Spec.Template.Spec.InitContainers[0].SecurityContext = proxyClassAllOpts.Spec.StatefulSet.Pod.TailscaleInitContainer.SecurityContext + wantSS.Spec.Template.Spec.Containers[0].Resources = proxyClassAllOpts.Spec.StatefulSet.Pod.TailscaleContainer.Resources + wantSS.Spec.Template.Spec.InitContainers[0].Resources = proxyClassAllOpts.Spec.StatefulSet.Pod.TailscaleInitContainer.Resources + + gotSS := applyProxyClassToStatefulSet(proxyClassAllOpts, nonUserspaceProxySS.DeepCopy()) + if diff := cmp.Diff(gotSS, wantSS); diff != "" { + t.Fatalf("Unexpected result applying ProxyClass with all fields set to a StatefulSet for non-userspace proxy (-got +want):\n%s", diff) + } + + // 2. Test that a ProxyClass with custom labels and annotations for + // StatefulSet and Pod set gets correctly applied to a Statefulset built + // from non-userspace proxy template. + wantSS = nonUserspaceProxySS.DeepCopy() + wantSS.ObjectMeta.Labels = mergeMapKeys(wantSS.ObjectMeta.Labels, proxyClassJustLabels.Spec.StatefulSet.Labels) + wantSS.ObjectMeta.Annotations = mergeMapKeys(wantSS.ObjectMeta.Annotations, proxyClassJustLabels.Spec.StatefulSet.Annotations) + wantSS.Spec.Template.Labels = proxyClassJustLabels.Spec.StatefulSet.Pod.Labels + wantSS.Spec.Template.Annotations = proxyClassJustLabels.Spec.StatefulSet.Pod.Annotations + gotSS = applyProxyClassToStatefulSet(proxyClassJustLabels, nonUserspaceProxySS.DeepCopy()) + if diff := cmp.Diff(gotSS, wantSS); diff != "" { + t.Fatalf("Unexpected result applying ProxyClass with custom labels and annotations to a StatefulSet for non-userspace proxy (-got +want):\n%s", diff) + } + + // 3. Test that a ProxyClass with all fields set gets correctly applied + // to a Statefulset built from a userspace proxy template. + wantSS = userspaceProxySS.DeepCopy() + wantSS.ObjectMeta.Labels = mergeMapKeys(wantSS.ObjectMeta.Labels, proxyClassAllOpts.Spec.StatefulSet.Labels) + wantSS.ObjectMeta.Annotations = mergeMapKeys(wantSS.ObjectMeta.Annotations, proxyClassAllOpts.Spec.StatefulSet.Annotations) + wantSS.Spec.Template.Labels = proxyClassAllOpts.Spec.StatefulSet.Pod.Labels + wantSS.Spec.Template.Annotations = proxyClassAllOpts.Spec.StatefulSet.Pod.Annotations + wantSS.Spec.Template.Spec.SecurityContext = proxyClassAllOpts.Spec.StatefulSet.Pod.SecurityContext + wantSS.Spec.Template.Spec.ImagePullSecrets = proxyClassAllOpts.Spec.StatefulSet.Pod.ImagePullSecrets + wantSS.Spec.Template.Spec.NodeName = proxyClassAllOpts.Spec.StatefulSet.Pod.NodeName + wantSS.Spec.Template.Spec.NodeSelector = proxyClassAllOpts.Spec.StatefulSet.Pod.NodeSelector + wantSS.Spec.Template.Spec.Tolerations = proxyClassAllOpts.Spec.StatefulSet.Pod.Tolerations + wantSS.Spec.Template.Spec.Containers[0].SecurityContext = proxyClassAllOpts.Spec.StatefulSet.Pod.TailscaleContainer.SecurityContext + wantSS.Spec.Template.Spec.Containers[0].Resources = proxyClassAllOpts.Spec.StatefulSet.Pod.TailscaleContainer.Resources + gotSS = applyProxyClassToStatefulSet(proxyClassAllOpts, userspaceProxySS.DeepCopy()) + if diff := cmp.Diff(gotSS, wantSS); diff != "" { + t.Fatalf("Unexpected result applying ProxyClass with custom labels and annotations to a StatefulSet for a userspace proxy (-got +want):\n%s", diff) + } + + // 4. Test that a ProxyClass with custom labels and annotations gets correctly applied + // to a Statefulset built from a userspace proxy template. + wantSS = userspaceProxySS.DeepCopy() + wantSS.ObjectMeta.Labels = mergeMapKeys(wantSS.ObjectMeta.Labels, proxyClassJustLabels.Spec.StatefulSet.Labels) + wantSS.ObjectMeta.Annotations = mergeMapKeys(wantSS.ObjectMeta.Annotations, proxyClassJustLabels.Spec.StatefulSet.Annotations) + wantSS.Spec.Template.Labels = proxyClassJustLabels.Spec.StatefulSet.Pod.Labels + wantSS.Spec.Template.Annotations = proxyClassJustLabels.Spec.StatefulSet.Pod.Annotations + gotSS = applyProxyClassToStatefulSet(proxyClassJustLabels, userspaceProxySS.DeepCopy()) + if diff := cmp.Diff(gotSS, wantSS); diff != "" { + t.Fatalf("Unexpected result applying ProxyClass with custom labels and annotations to a StatefulSet for a userspace proxy (-got +want):\n%s", diff) + } +} + +func mergeMapKeys(a, b map[string]string) map[string]string { + for key, val := range b { + a[key] = val + } + return a +} + +func Test_mergeStatefulSetLabelsOrAnnots(t *testing.T) { + tests := []struct { + name string + current map[string]string + custom map[string]string + managed []string + want map[string]string + }{ + { + name: "no custom labels specified and none present in current labels, return current labels", + current: map[string]string{LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"}, + want: map[string]string{LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"}, + managed: tailscaleManagedLabels, + }, + { + name: "no custom labels specified, but some present in current labels, return tailscale managed labels only from the current labels", + current: map[string]string{"foo": "bar", "something.io/foo": "bar", LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"}, + want: map[string]string{LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"}, + managed: tailscaleManagedLabels, + }, + { + name: "custom labels specified, current labels only contain tailscale managed labels, return a union of both", + current: map[string]string{LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"}, + custom: map[string]string{"foo": "bar", "something.io/foo": "bar"}, + want: map[string]string{"foo": "bar", "something.io/foo": "bar", LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"}, + managed: tailscaleManagedLabels, + }, + { + name: "custom labels specified, current labels contain tailscale managed labels and custom labels, some of which re not present in the new custom labels, return a union of managed labels and the desired custom labels", + current: map[string]string{"foo": "bar", "bar": "baz", "app": "1234", LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"}, + custom: map[string]string{"foo": "bar", "something.io/foo": "bar"}, + want: map[string]string{"foo": "bar", "something.io/foo": "bar", "app": "1234", LabelManaged: "true", LabelParentName: "foo", LabelParentType: "svc", LabelParentNamespace: "foo"}, + managed: tailscaleManagedLabels, + }, + { + name: "no current labels present, return custom labels only", + custom: map[string]string{"foo": "bar", "something.io/foo": "bar"}, + want: map[string]string{"foo": "bar", "something.io/foo": "bar"}, + managed: tailscaleManagedLabels, + }, + { + name: "no current labels present, no custom labels specified, return empty map", + want: map[string]string{}, + managed: tailscaleManagedLabels, + }, + { + name: "no custom annots specified and none present in current annots, return current annots", + current: map[string]string{podAnnotationLastSetClusterIP: "1.2.3.4", podAnnotationLastSetHostname: "foo"}, + want: map[string]string{podAnnotationLastSetClusterIP: "1.2.3.4", podAnnotationLastSetHostname: "foo"}, + managed: tailscaleManagedAnnotations, + }, + { + name: "no custom annots specified, but some present in current annots, return tailscale managed annots only from the current annots", + current: map[string]string{"foo": "bar", "something.io/foo": "bar", podAnnotationLastSetClusterIP: "1.2.3.4", podAnnotationLastSetHostname: "foo"}, + want: map[string]string{podAnnotationLastSetClusterIP: "1.2.3.4", podAnnotationLastSetHostname: "foo"}, + managed: tailscaleManagedAnnotations, + }, + { + name: "custom annots specified, current annots only contain tailscale managed annots, return a union of both", + current: map[string]string{podAnnotationLastSetClusterIP: "1.2.3.4", podAnnotationLastSetHostname: "foo"}, + custom: map[string]string{"foo": "bar", "something.io/foo": "bar"}, + want: map[string]string{"foo": "bar", "something.io/foo": "bar", podAnnotationLastSetClusterIP: "1.2.3.4", podAnnotationLastSetHostname: "foo"}, + managed: tailscaleManagedAnnotations, + }, + { + name: "custom annots specified, current annots contain tailscale managed annots and custom annots, some of which are not present in the new custom annots, return a union of managed annots and the desired custom annots", + current: map[string]string{"foo": "bar", "something.io/foo": "bar", podAnnotationLastSetClusterIP: "1.2.3.4", podAnnotationLastSetHostname: "foo"}, + custom: map[string]string{"something.io/foo": "bar"}, + want: map[string]string{"something.io/foo": "bar", podAnnotationLastSetClusterIP: "1.2.3.4", podAnnotationLastSetHostname: "foo"}, + managed: tailscaleManagedAnnotations, + }, + { + name: "no current annots present, return custom annots only", + custom: map[string]string{"foo": "bar", "something.io/foo": "bar"}, + want: map[string]string{"foo": "bar", "something.io/foo": "bar"}, + managed: tailscaleManagedAnnotations, + }, + { + name: "no current labels present, no custom labels specified, return empty map", + want: map[string]string{}, + managed: tailscaleManagedAnnotations, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := mergeStatefulSetLabelsOrAnnots(tt.current, tt.custom, tt.managed); !reflect.DeepEqual(got, tt.want) { + t.Errorf("mergeStatefulSetLabels() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/cmd/k8s-operator/svc.go b/cmd/k8s-operator/svc.go index d6b810e73..8820a3554 100644 --- a/cmd/k8s-operator/svc.go +++ b/cmd/k8s-operator/svc.go @@ -20,6 +20,8 @@ import ( "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" + tsoperator "tailscale.com/k8s-operator" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/util/clientmetric" "tailscale.com/util/set" ) @@ -155,6 +157,17 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga a.logger.Error(msg) return nil } + + proxyClass := proxyClassForObject(svc) + if proxyClass != "" { + if ready, err := proxyClassIsReady(ctx, proxyClass, a.Client); err != nil { + return fmt.Errorf("error verifying ProxyClass for Service: %w", err) + } else if !ready { + logger.Infof("ProxyClass %s specified for the Service, but is not (yet) Ready, waiting..", proxyClass) + return nil + } + } + hostname, err := nameForService(svc) if err != nil { return err @@ -183,6 +196,7 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga Hostname: hostname, Tags: tags, ChildResourceLabels: crl, + ProxyClass: proxyClass, } a.mu.Lock() @@ -318,3 +332,15 @@ func (a *ServiceReconciler) tailnetTargetAnnotation(svc *corev1.Service) string } return svc.Annotations[annotationTailnetTargetIPOld] } + +func proxyClassForObject(o client.Object) string { + return o.GetLabels()[LabelProxyClass] +} + +func proxyClassIsReady(ctx context.Context, name string, cl client.Client) (bool, error) { + proxyClass := new(tsapi.ProxyClass) + if err := cl.Get(ctx, types.NamespacedName{Name: name}, proxyClass); err != nil { + return false, fmt.Errorf("error getting ProxyClass %s: %w", name, err) + } + return tsoperator.ProxyClassIsReady(proxyClass), nil +} diff --git a/cmd/k8s-operator/testutils_test.go b/cmd/k8s-operator/testutils_test.go index 5d025539e..8e4834873 100644 --- a/cmd/k8s-operator/testutils_test.go +++ b/cmd/k8s-operator/testutils_test.go @@ -24,6 +24,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "tailscale.com/client/tailscale" "tailscale.com/ipn" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/types/ptr" "tailscale.com/util/mak" ) @@ -47,9 +48,11 @@ type configOpts struct { confFileHash string serveConfig *ipn.ServeConfig shouldEnableForwardingClusterTrafficViaIngress bool + proxyClass string // configuration from the named ProxyClass should be applied to proxy resources } -func expectedSTS(opts configOpts) *appsv1.StatefulSet { +func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.StatefulSet { + t.Helper() tsContainer := corev1.Container{ Name: "tailscale", Image: "tailscale/tailscale", @@ -141,7 +144,7 @@ func expectedSTS(opts configOpts) *appsv1.StatefulSet { }) tsContainer.VolumeMounts = append(tsContainer.VolumeMounts, corev1.VolumeMount{Name: "serve-config", ReadOnly: true, MountPath: "/etc/tailscaled"}) } - return &appsv1.StatefulSet{ + ss := &appsv1.StatefulSet{ TypeMeta: metav1.TypeMeta{ Kind: "StatefulSet", APIVersion: "apps/v1", @@ -194,9 +197,20 @@ func expectedSTS(opts configOpts) *appsv1.StatefulSet { }, }, } + // If opts.proxyClass is set, retrieve the ProxyClass and apply + // configuration from that to the StatefulSet. + if opts.proxyClass != "" { + t.Logf("applying configuration from ProxyClass %s", opts.proxyClass) + proxyClass := new(tsapi.ProxyClass) + if err := cl.Get(context.Background(), types.NamespacedName{Name: opts.proxyClass}, proxyClass); err != nil { + t.Fatalf("error getting ProxyClass: %v", err) + } + return applyProxyClassToStatefulSet(proxyClass, ss) + } + return ss } -func expectedSTSUserspace(opts configOpts) *appsv1.StatefulSet { +func expectedSTSUserspace(t *testing.T, cl client.Client, opts configOpts) *appsv1.StatefulSet { tsContainer := corev1.Container{ Name: "tailscale", Image: "tailscale/tailscale", @@ -213,7 +227,7 @@ func expectedSTSUserspace(opts configOpts) *appsv1.StatefulSet { annots := make(map[string]string) volumes := []corev1.Volume{{Name: "serve-config", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: opts.secretName, Items: []corev1.KeyToPath{{Key: "serve-config", Path: "serve-config"}}}}}} annots["tailscale.com/operator-last-set-hostname"] = opts.hostname - return &appsv1.StatefulSet{ + ss := &appsv1.StatefulSet{ TypeMeta: metav1.TypeMeta{ Kind: "StatefulSet", APIVersion: "apps/v1", @@ -255,6 +269,17 @@ func expectedSTSUserspace(opts configOpts) *appsv1.StatefulSet { }, }, } + // If opts.proxyClass is set, retrieve the ProxyClass and apply + // configuration from that to the StatefulSet. + if opts.proxyClass != "" { + t.Logf("applying configuration from ProxyClass %s", opts.proxyClass) + proxyClass := new(tsapi.ProxyClass) + if err := cl.Get(context.Background(), types.NamespacedName{Name: opts.proxyClass}, proxyClass); err != nil { + t.Fatalf("error getting ProxyClass: %v", err) + } + return applyProxyClassToStatefulSet(proxyClass, ss) + } + return ss } func expectedHeadlessService(name string, parentType string) *corev1.Service { diff --git a/k8s-operator/apis/v1alpha1/register.go b/k8s-operator/apis/v1alpha1/register.go index d8929a9f5..4e6bfda64 100644 --- a/k8s-operator/apis/v1alpha1/register.go +++ b/k8s-operator/apis/v1alpha1/register.go @@ -49,7 +49,7 @@ func init() { // Adds the list of known types to api.Scheme. func addKnownTypes(scheme *runtime.Scheme) error { - scheme.AddKnownTypes(SchemeGroupVersion, &Connector{}, &ConnectorList{}) + scheme.AddKnownTypes(SchemeGroupVersion, &Connector{}, &ConnectorList{}, &ProxyClass{}, &ProxyClassList{}) metav1.AddToGroupVersion(scheme, SchemeGroupVersion) return nil diff --git a/k8s-operator/apis/v1alpha1/types_connector.go b/k8s-operator/apis/v1alpha1/types_connector.go index ba6952265..c5333f5a1 100644 --- a/k8s-operator/apis/v1alpha1/types_connector.go +++ b/k8s-operator/apis/v1alpha1/types_connector.go @@ -68,6 +68,12 @@ type ConnectorSpec struct { // and 63 characters long. // +optional Hostname Hostname `json:"hostname,omitempty"` + // ProxyClass is the name of the ProxyClass custom resource that + // contains configuration options that should be applied to the + // resources created for this Connector. If unset, the operator will + // create resources with the default configuration. + // +optional + ProxyClass string `json:"proxyClass,omitempty"` // SubnetRouter defines subnet routes that the Connector node should // expose to tailnet. If unset, none are exposed. // https://tailscale.com/kb/1019/subnets/ @@ -180,5 +186,6 @@ type ConnectorCondition struct { type ConnectorConditionType string const ( - ConnectorReady ConnectorConditionType = `ConnectorReady` + ConnectorReady ConnectorConditionType = `ConnectorReady` + ProxyClassready ConnectorConditionType = `ProxyClassReady` ) diff --git a/k8s-operator/apis/v1alpha1/types_proxyclass.go b/k8s-operator/apis/v1alpha1/types_proxyclass.go new file mode 100644 index 000000000..98d7b5753 --- /dev/null +++ b/k8s-operator/apis/v1alpha1/types_proxyclass.go @@ -0,0 +1,143 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package v1alpha1 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var ProxyClassKind = "ProxyClass" + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:scope=Cluster +// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=`.status.conditions[?(@.type == "ProxyClassReady")].reason`,description="Status of the ProxyClass." + +type ProxyClass struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec ProxyClassSpec `json:"spec"` + + // +optional + Status ProxyClassStatus `json:"status"` +} + +// +kubebuilder:object:root=true +type ProxyClassList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata"` + + Items []ProxyClass `json:"items"` +} + +type ProxyClassSpec struct { + // Proxy's StatefulSet spec. + StatefulSet *StatefulSet `json:"statefulSet"` +} + +type StatefulSet struct { + // Labels that will be added to the StatefulSet created for the proxy. + // Any labels specified here will be merged with the default labels + // applied to the StatefulSet by the Tailscale Kubernetes operator as + // well as any other labels that might have been applied by other + // actors. + // Label keys and values must be valid Kubernetes label keys and values. + // https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set + // +optional + Labels map[string]string `json:"labels,omitempty"` + // Annotations that will be added to the StatefulSet created for the proxy. + // Any Annotations specified here will be merged with the default annotations + // applied to the StatefulSet by the Tailscale Kubernetes operator as + // well as any other annotations that might have been applied by other + // actors. + // Annotations must be valid Kubernetes annotations. + // https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set + // +optional + Annotations map[string]string `json:"annotations,omitempty"` + // Configuration for the proxy Pod. + // +optional + Pod *Pod `json:"pod,omitempty"` +} + +type Pod struct { + // Labels that will be added to the proxy Pod. + // Any labels specified here will be merged with the default labels + // applied to the Pod by the Tailscale Kubernetes operator. + // Label keys and values must be valid Kubernetes label keys and values. + // https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set + // +optional + Labels map[string]string `json:"labels,omitempty"` + // Annotations that will be added to the proxy Pod. + // Any annotations specified here will be merged with the default + // annotations applied to the Pod by the Tailscale Kubernetes operator. + // Annotations must be valid Kubernetes annotations. + // https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set + // +optional + Annotations map[string]string `json:"annotations,omitempty"` + // Configuration for the proxy container running tailscale. + // +optional + TailscaleContainer *Container `json:"tailscaleContainer,omitempty"` + // Configuration for the proxy init container that enables forwarding. + // +optional + TailscaleInitContainer *Container `json:"tailscaleInitContainer,omitempty"` + // Proxy Pod's security context. + // By default Tailscale Kubernetes operator does not apply any Pod + // security context. + // https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#security-context-2 + // +optional + SecurityContext *corev1.PodSecurityContext `json:"securityContext,omitempty"` + // Proxy Pod's image pull Secrets. + // https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#PodSpec + // +optional + ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"` + // Proxy Pod's node name. + // https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#scheduling + // +optional + NodeName string `json:"nodeName,omitempty"` + // Proxy Pod's node selector. + // By default Tailscale Kubernetes operator does not apply any node + // selector. + // https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#scheduling + // +optional + NodeSelector map[string]string `json:"nodeSelector,omitempty"` + // Proxy Pod's tolerations. + // By default Tailscale Kubernetes operator does not apply any + // tolerations. + // https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#scheduling + // +optional + Tolerations []corev1.Toleration `json:"tolerations,omitempty"` +} + +type Container struct { + // Container security context. + // Security context specified here will override the security context by the operator. + // By default the operator: + // - sets 'privileged: true' for the init container + // - set NET_ADMIN capability for tailscale container for proxies that + // are created for Services or Connector. + // https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#security-context + // +optional + SecurityContext *corev1.SecurityContext `json:"securityContext,omitempty"` + // Container resource requirements. + // By default Tailscale Kubernetes operator does not apply any resource + // requirements. The amount of resources required wil depend on the + // amount of resources the operator needs to parse, usage patterns and + // cluster size. + // https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#resources + // +optional + Resources corev1.ResourceRequirements `json:"resources,omitempty"` +} + +type ProxyClassStatus struct { + // List of status conditions to indicate the status of the ProxyClass. + // Known condition types are `ProxyClassReady`. + // +listType=map + // +listMapKey=type + // +optional + Conditions []ConnectorCondition `json:"conditions,omitempty"` +} diff --git a/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go b/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go index f0f54b533..efd202eee 100644 --- a/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go +++ b/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go @@ -8,6 +8,7 @@ package v1alpha1 import ( + "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" ) @@ -136,6 +137,191 @@ func (in *ConnectorStatus) DeepCopy() *ConnectorStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Container) DeepCopyInto(out *Container) { + *out = *in + if in.SecurityContext != nil { + in, out := &in.SecurityContext, &out.SecurityContext + *out = new(v1.SecurityContext) + (*in).DeepCopyInto(*out) + } + in.Resources.DeepCopyInto(&out.Resources) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Container. +func (in *Container) DeepCopy() *Container { + if in == nil { + return nil + } + out := new(Container) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Pod) DeepCopyInto(out *Pod) { + *out = *in + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.TailscaleContainer != nil { + in, out := &in.TailscaleContainer, &out.TailscaleContainer + *out = new(Container) + (*in).DeepCopyInto(*out) + } + if in.TailscaleInitContainer != nil { + in, out := &in.TailscaleInitContainer, &out.TailscaleInitContainer + *out = new(Container) + (*in).DeepCopyInto(*out) + } + if in.SecurityContext != nil { + in, out := &in.SecurityContext, &out.SecurityContext + *out = new(v1.PodSecurityContext) + (*in).DeepCopyInto(*out) + } + if in.ImagePullSecrets != nil { + in, out := &in.ImagePullSecrets, &out.ImagePullSecrets + *out = make([]v1.LocalObjectReference, len(*in)) + copy(*out, *in) + } + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Tolerations != nil { + in, out := &in.Tolerations, &out.Tolerations + *out = make([]v1.Toleration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Pod. +func (in *Pod) DeepCopy() *Pod { + if in == nil { + return nil + } + out := new(Pod) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProxyClass) DeepCopyInto(out *ProxyClass) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyClass. +func (in *ProxyClass) DeepCopy() *ProxyClass { + if in == nil { + return nil + } + out := new(ProxyClass) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ProxyClass) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProxyClassList) DeepCopyInto(out *ProxyClassList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]ProxyClass, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyClassList. +func (in *ProxyClassList) DeepCopy() *ProxyClassList { + if in == nil { + return nil + } + out := new(ProxyClassList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ProxyClassList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProxyClassSpec) DeepCopyInto(out *ProxyClassSpec) { + *out = *in + if in.StatefulSet != nil { + in, out := &in.StatefulSet, &out.StatefulSet + *out = new(StatefulSet) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyClassSpec. +func (in *ProxyClassSpec) DeepCopy() *ProxyClassSpec { + if in == nil { + return nil + } + out := new(ProxyClassSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProxyClassStatus) DeepCopyInto(out *ProxyClassStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]ConnectorCondition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyClassStatus. +func (in *ProxyClassStatus) DeepCopy() *ProxyClassStatus { + if in == nil { + return nil + } + out := new(ProxyClassStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in Routes) DeepCopyInto(out *Routes) { { @@ -155,6 +341,40 @@ func (in Routes) DeepCopy() Routes { return *out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StatefulSet) DeepCopyInto(out *StatefulSet) { + *out = *in + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Pod != nil { + in, out := &in.Pod, &out.Pod + *out = new(Pod) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StatefulSet. +func (in *StatefulSet) DeepCopy() *StatefulSet { + if in == nil { + return nil + } + out := new(StatefulSet) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SubnetRouter) DeepCopyInto(out *SubnetRouter) { *out = *in diff --git a/k8s-operator/conditions.go b/k8s-operator/conditions.go index f1344e34c..146487fb8 100644 --- a/k8s-operator/conditions.go +++ b/k8s-operator/conditions.go @@ -7,6 +7,7 @@ package kube import ( "slices" + "time" "go.uber.org/zap" xslices "golang.org/x/exp/slices" @@ -17,8 +18,28 @@ import ( // SetConnectorCondition ensures that Connector status has a condition with the // given attributes. LastTransitionTime gets set every time condition's status -// changes +// changes. func SetConnectorCondition(cn *tsapi.Connector, conditionType tsapi.ConnectorConditionType, status metav1.ConditionStatus, reason, message string, gen int64, clock tstime.Clock, logger *zap.SugaredLogger) { + conds := updateCondition(cn.Status.Conditions, conditionType, status, reason, message, gen, clock, logger) + cn.Status.Conditions = conds +} + +// RemoveConnectorCondition will remove condition of the given type. +func RemoveConnectorCondition(conn *tsapi.Connector, conditionType tsapi.ConnectorConditionType) { + conn.Status.Conditions = slices.DeleteFunc(conn.Status.Conditions, func(cond tsapi.ConnectorCondition) bool { + return cond.Type == conditionType + }) +} + +// SetProxyClassCondition ensures that ProxyClass status has a condition with the +// given attributes. LastTransitionTime gets set every time condition's status +// changes. +func SetProxyClassCondition(pc *tsapi.ProxyClass, conditionType tsapi.ConnectorConditionType, status metav1.ConditionStatus, reason, message string, gen int64, clock tstime.Clock, logger *zap.SugaredLogger) { + conds := updateCondition(pc.Status.Conditions, conditionType, status, reason, message, gen, clock, logger) + pc.Status.Conditions = conds +} + +func updateCondition(conds []tsapi.ConnectorCondition, conditionType tsapi.ConnectorConditionType, status metav1.ConditionStatus, reason, message string, gen int64, clock tstime.Clock, logger *zap.SugaredLogger) []tsapi.ConnectorCondition { newCondition := tsapi.ConnectorCondition{ Type: conditionType, Status: status, @@ -27,34 +48,37 @@ func SetConnectorCondition(cn *tsapi.Connector, conditionType tsapi.ConnectorCon ObservedGeneration: gen, } - nowTime := metav1.NewTime(clock.Now()) + nowTime := metav1.NewTime(clock.Now().Truncate(time.Second)) newCondition.LastTransitionTime = &nowTime - idx := xslices.IndexFunc(cn.Status.Conditions, func(cond tsapi.ConnectorCondition) bool { + idx := xslices.IndexFunc(conds, func(cond tsapi.ConnectorCondition) bool { return cond.Type == conditionType }) if idx == -1 { - cn.Status.Conditions = append(cn.Status.Conditions, newCondition) - return + conds = append(conds, newCondition) + return conds } - // Update the existing condition - cond := cn.Status.Conditions[idx] + cond := conds[idx] // update the existing condition // If this update doesn't contain a state transition, we don't update - // the conditions LastTransitionTime to Now() + // the conditions LastTransitionTime to Now(). if cond.Status == status { newCondition.LastTransitionTime = cond.LastTransitionTime } else { - logger.Info("Status change for Connector condition %s from %s to %s", conditionType, cond.Status, status) + logger.Infof("Status change for condition %s from %s to %s", conditionType, cond.Status, status) } - - cn.Status.Conditions[idx] = newCondition + conds[idx] = newCondition + return conds } -// RemoveConnectorCondition will remove condition of the given type -func RemoveConnectorCondition(conn *tsapi.Connector, conditionType tsapi.ConnectorConditionType) { - conn.Status.Conditions = slices.DeleteFunc(conn.Status.Conditions, func(cond tsapi.ConnectorCondition) bool { - return cond.Type == conditionType +func ProxyClassIsReady(pc *tsapi.ProxyClass) bool { + idx := xslices.IndexFunc(pc.Status.Conditions, func(cond tsapi.ConnectorCondition) bool { + return cond.Type == tsapi.ProxyClassready }) + if idx == -1 { + return false + } + cond := pc.Status.Conditions[idx] + return cond.Status == metav1.ConditionTrue && cond.ObservedGeneration == pc.Generation } diff --git a/k8s-operator/conditions_test.go b/k8s-operator/conditions_test.go index d7d8d6cd8..624891d75 100644 --- a/k8s-operator/conditions_test.go +++ b/k8s-operator/conditions_test.go @@ -19,8 +19,8 @@ import ( func TestSetConnectorCondition(t *testing.T) { cn := tsapi.Connector{} clock := tstest.NewClock(tstest.ClockOpts{}) - fakeNow := metav1.NewTime(clock.Now()) - fakePast := metav1.NewTime(clock.Now().Add(-5 * time.Minute)) + fakeNow := metav1.NewTime(clock.Now().Truncate(time.Second)) + fakePast := metav1.NewTime(clock.Now().Truncate(time.Second).Add(-5 * time.Minute)) zl, err := zap.NewDevelopment() assert.Nil(t, err)