diff --git a/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml b/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml index 8ea07e808..ff518e40c 100644 --- a/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml +++ b/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml @@ -22,7 +22,7 @@ rules: resources: ["ingressclasses"] verbs: ["get", "list", "watch"] - apiGroups: ["tailscale.com"] - resources: ["connectors", "connectors/status"] + resources: ["connectors", "connectors/status", "proxyclasses", "proxyclasses/status"] verbs: ["get", "list", "watch", "update"] --- apiVersion: rbac.authorization.k8s.io/v1 diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml new file mode 100644 index 000000000..68935c884 --- /dev/null +++ b/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml @@ -0,0 +1,465 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.13.0 + name: proxyclasses.tailscale.com +spec: + group: tailscale.com + names: + kind: ProxyClass + listKind: ProxyClassList + plural: proxyclasses + shortNames: + - pc + singular: proxyclass + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + type: object + required: + - spec + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + type: object + properties: + service: + description: Configuration for the headless Service, not actually used in this prototype, but is here to better illustrate the API structure + type: object + properties: + labels: + type: object + additionalProperties: + type: string + statefulSet: + type: object + properties: + annotations: + type: object + additionalProperties: + type: string + labels: + type: object + additionalProperties: + type: string + pod: + type: object + properties: + annotations: + type: object + additionalProperties: + type: string + imagePullSecrets: + type: array + items: + description: LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace. + type: object + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + x-kubernetes-map-type: atomic + labels: + description: Or should we just sync statefulset.labels, statefulset.annotations? + type: object + additionalProperties: + type: string + nodeName: + type: string + nodeSelector: + type: object + additionalProperties: + type: string + patches: + type: array + items: + description: RFC 6902 JSON patch + type: object + required: + - op + - path + properties: + op: + type: string + path: + type: string + value: + type: string + podSecurityContext: + description: PodSecurityContext holds pod-level security attributes and common container settings. Some fields are also present in container.securityContext. Field values of container.securityContext take precedence over field values of PodSecurityContext. + type: object + properties: + fsGroup: + description: "A special supplemental group that applies to all containers in a pod. Some volume types allow the Kubelet to change the ownership of that volume to be owned by the pod: \n 1. The owning GID will be the FSGroup 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) 3. The permission bits are OR'd with rw-rw---- \n If unset, the Kubelet will not modify the ownership and permissions of any volume. Note that this field cannot be set when spec.os.name is windows." + type: integer + format: int64 + fsGroupChangePolicy: + description: 'fsGroupChangePolicy defines behavior of changing ownership and permission of the volume before being exposed inside Pod. This field will only apply to volume types which support fsGroup based ownership(and permissions). It will have no effect on ephemeral volume types such as: secret, configmaps and emptydir. Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. Note that this field cannot be set when spec.os.name is windows.' + type: string + runAsGroup: + description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container. Note that this field cannot be set when spec.os.name is windows. + type: integer + format: int64 + runAsNonRoot: + description: Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container. Note that this field cannot be set when spec.os.name is windows. + type: integer + format: int64 + seLinuxOptions: + description: The SELinux context to be applied to all containers. If unspecified, the container runtime will allocate a random SELinux context for each container. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container. Note that this field cannot be set when spec.os.name is windows. + type: object + properties: + level: + description: Level is SELinux level label that applies to the container. + type: string + role: + description: Role is a SELinux role label that applies to the container. + type: string + type: + description: Type is a SELinux type label that applies to the container. + type: string + user: + description: User is a SELinux user label that applies to the container. + type: string + seccompProfile: + description: The seccomp options to use by the containers in this pod. Note that this field cannot be set when spec.os.name is windows. + type: object + required: + - type + properties: + localhostProfile: + description: localhostProfile indicates a profile defined in a file on the node should be used. The profile must be preconfigured on the node to work. Must be a descending path, relative to the kubelet's configured seccomp profile location. Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: "type indicates which kind of seccomp profile will be applied. Valid options are: \n Localhost - a profile defined in a file on the node should be used. RuntimeDefault - the container runtime default profile should be used. Unconfined - no profile should be applied." + type: string + supplementalGroups: + description: A list of groups applied to the first process run in each container, in addition to the container's primary GID, the fsGroup (if specified), and group memberships defined in the container image for the uid of the container process. If unspecified, no additional groups are added to any container. Note that group memberships defined in the container image for the uid of the container process are still effective, even if they are not included in this list. Note that this field cannot be set when spec.os.name is windows. + type: array + items: + type: integer + format: int64 + sysctls: + description: Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported sysctls (by the container runtime) might fail to launch. Note that this field cannot be set when spec.os.name is windows. + type: array + items: + description: Sysctl defines a kernel parameter to be set + type: object + required: + - name + - value + properties: + name: + description: Name of a property to set + type: string + value: + description: Value of a property to set + type: string + windowsOptions: + description: The Windows specific settings applied to all containers. If unspecified, the options within a container's SecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is linux. + type: object + properties: + gmsaCredentialSpec: + description: GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + type: string + hostProcess: + description: HostProcess determines if a container should be run as a 'Host Process' container. All of a Pod's containers must have the same effective HostProcess value (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + tailscaleContainer: + type: object + properties: + resources: + description: ResourceRequirements describes the compute resource requirements. + type: object + properties: + claims: + description: "Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. \n This is an alpha field and requires enabling the DynamicResourceAllocation feature gate. \n This field is immutable. It can only be set for containers." + type: array + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + type: object + required: + - name + properties: + name: + description: Name must match the name of one entry in pod.spec.resourceClaims of the Pod where this field is used. It makes that resource available inside a container. + type: string + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + additionalProperties: + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + requests: + description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. Requests cannot exceed Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + additionalProperties: + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + securityContext: + description: SecurityContext holds security configuration that will be applied to a container. Some fields are present in both SecurityContext and PodSecurityContext. When both are set, the values in SecurityContext take precedence. + type: object + properties: + allowPrivilegeEscalation: + description: 'AllowPrivilegeEscalation controls whether a process can gain more privileges than its parent process. This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged 2) has CAP_SYS_ADMIN Note that this field cannot be set when spec.os.name is windows.' + type: boolean + capabilities: + description: The capabilities to add/drop when running containers. Defaults to the default set of capabilities granted by the container runtime. Note that this field cannot be set when spec.os.name is windows. + type: object + properties: + add: + description: Added capabilities + type: array + items: + description: Capability represent POSIX capabilities type + type: string + drop: + description: Removed capabilities + type: array + items: + description: Capability represent POSIX capabilities type + type: string + privileged: + description: Run container in privileged mode. Processes in privileged containers are essentially equivalent to root on the host. Defaults to false. Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: procMount denotes the type of proc mount to use for the containers. The default is DefaultProcMount which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag to be enabled. Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: Whether this container has a read-only root filesystem. Default is false. Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is windows. + type: integer + format: int64 + runAsNonRoot: + description: Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is windows. + type: integer + format: int64 + seLinuxOptions: + description: The SELinux context to be applied to the container. If unspecified, the container runtime will allocate a random SELinux context for each container. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is windows. + type: object + properties: + level: + description: Level is SELinux level label that applies to the container. + type: string + role: + description: Role is a SELinux role label that applies to the container. + type: string + type: + description: Type is a SELinux type label that applies to the container. + type: string + user: + description: User is a SELinux user label that applies to the container. + type: string + seccompProfile: + description: The seccomp options to use by this container. If seccomp options are provided at both the pod & container level, the container options override the pod options. Note that this field cannot be set when spec.os.name is windows. + type: object + required: + - type + properties: + localhostProfile: + description: localhostProfile indicates a profile defined in a file on the node should be used. The profile must be preconfigured on the node to work. Must be a descending path, relative to the kubelet's configured seccomp profile location. Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: "type indicates which kind of seccomp profile will be applied. Valid options are: \n Localhost - a profile defined in a file on the node should be used. RuntimeDefault - the container runtime default profile should be used. Unconfined - no profile should be applied." + type: string + windowsOptions: + description: The Windows specific settings applied to all containers. If unspecified, the options from the PodSecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is linux. + type: object + properties: + gmsaCredentialSpec: + description: GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + type: string + hostProcess: + description: HostProcess determines if a container should be run as a 'Host Process' container. All of a Pod's containers must have the same effective HostProcess value (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + tailscaleInitContainer: + type: object + properties: + resources: + description: ResourceRequirements describes the compute resource requirements. + type: object + properties: + claims: + description: "Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. \n This is an alpha field and requires enabling the DynamicResourceAllocation feature gate. \n This field is immutable. It can only be set for containers." + type: array + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + type: object + required: + - name + properties: + name: + description: Name must match the name of one entry in pod.spec.resourceClaims of the Pod where this field is used. It makes that resource available inside a container. + type: string + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + additionalProperties: + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + requests: + description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. Requests cannot exceed Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + additionalProperties: + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + securityContext: + description: SecurityContext holds security configuration that will be applied to a container. Some fields are present in both SecurityContext and PodSecurityContext. When both are set, the values in SecurityContext take precedence. + type: object + properties: + allowPrivilegeEscalation: + description: 'AllowPrivilegeEscalation controls whether a process can gain more privileges than its parent process. This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged 2) has CAP_SYS_ADMIN Note that this field cannot be set when spec.os.name is windows.' + type: boolean + capabilities: + description: The capabilities to add/drop when running containers. Defaults to the default set of capabilities granted by the container runtime. Note that this field cannot be set when spec.os.name is windows. + type: object + properties: + add: + description: Added capabilities + type: array + items: + description: Capability represent POSIX capabilities type + type: string + drop: + description: Removed capabilities + type: array + items: + description: Capability represent POSIX capabilities type + type: string + privileged: + description: Run container in privileged mode. Processes in privileged containers are essentially equivalent to root on the host. Defaults to false. Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: procMount denotes the type of proc mount to use for the containers. The default is DefaultProcMount which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag to be enabled. Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: Whether this container has a read-only root filesystem. Default is false. Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is windows. + type: integer + format: int64 + runAsNonRoot: + description: Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is windows. + type: integer + format: int64 + seLinuxOptions: + description: The SELinux context to be applied to the container. If unspecified, the container runtime will allocate a random SELinux context for each container. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is windows. + type: object + properties: + level: + description: Level is SELinux level label that applies to the container. + type: string + role: + description: Role is a SELinux role label that applies to the container. + type: string + type: + description: Type is a SELinux type label that applies to the container. + type: string + user: + description: User is a SELinux user label that applies to the container. + type: string + seccompProfile: + description: The seccomp options to use by this container. If seccomp options are provided at both the pod & container level, the container options override the pod options. Note that this field cannot be set when spec.os.name is windows. + type: object + required: + - type + properties: + localhostProfile: + description: localhostProfile indicates a profile defined in a file on the node should be used. The profile must be preconfigured on the node to work. Must be a descending path, relative to the kubelet's configured seccomp profile location. Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: "type indicates which kind of seccomp profile will be applied. Valid options are: \n Localhost - a profile defined in a file on the node should be used. RuntimeDefault - the container runtime default profile should be used. Unconfined - no profile should be applied." + type: string + windowsOptions: + description: The Windows specific settings applied to all containers. If unspecified, the options from the PodSecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is linux. + type: object + properties: + gmsaCredentialSpec: + description: GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + type: string + hostProcess: + description: HostProcess determines if a container should be run as a 'Host Process' container. All of a Pod's containers must have the same effective HostProcess value (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + tolerations: + type: array + items: + description: The pod this Toleration is attached to tolerates any taint that matches the triple using the matching operator . + type: object + properties: + effect: + description: Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Key is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: Operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: TolerationSeconds represents the period of time the toleration (which must be of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) by the system. + type: integer + format: int64 + value: + description: Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + served: true + storage: true + subresources: + status: {} diff --git a/cmd/k8s-operator/deploy/examples/proxclass.yaml b/cmd/k8s-operator/deploy/examples/proxclass.yaml new file mode 100644 index 000000000..5fa661806 --- /dev/null +++ b/cmd/k8s-operator/deploy/examples/proxclass.yaml @@ -0,0 +1,11 @@ +apiVersion: tailscale.com/v1alpha1 +kind: ProxyClass +metadata: + name: prod +spec: + statefulSet: + pod: + nodeSelector: + beta.kubernetes.io/os: "linux" + imagePullSecrets: + - name: "foo" diff --git a/cmd/k8s-operator/deploy/examples/proxyclassremoveinit.yaml b/cmd/k8s-operator/deploy/examples/proxyclassremoveinit.yaml new file mode 100644 index 000000000..a2d278fc8 --- /dev/null +++ b/cmd/k8s-operator/deploy/examples/proxyclassremoveinit.yaml @@ -0,0 +1,10 @@ +apiVersion: tailscale.com/v1alpha1 +kind: ProxyClass +metadata: + name: removeinit +spec: + statefulSet: + pod: + patches: + - op: remove + path: "/spec/initContainers/0" diff --git a/cmd/k8s-operator/deploy/manifests/operator.yaml b/cmd/k8s-operator/deploy/manifests/operator.yaml index 151eec620..873217497 100644 --- a/cmd/k8s-operator/deploy/manifests/operator.yaml +++ b/cmd/k8s-operator/deploy/manifests/operator.yaml @@ -186,6 +186,8 @@ rules: resources: - connectors - connectors/status + - proxyclasses + - proxyclasses/status verbs: - get - list diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go index 1c8063521..b08bc4826 100644 --- a/cmd/k8s-operator/sts.go +++ b/cmd/k8s-operator/sts.go @@ -16,6 +16,7 @@ import ( "os" "strings" + jsonpatch "github.com/evanphx/json-patch" "go.uber.org/zap" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -27,6 +28,7 @@ import ( "sigs.k8s.io/yaml" "tailscale.com/client/tailscale" "tailscale.com/ipn" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/net/netutil" "tailscale.com/tailcfg" "tailscale.com/tsnet" @@ -52,6 +54,9 @@ const ( //MagicDNS name of tailnet node. AnnotationTailnetTargetFQDN = "tailscale.com/tailnet-fqdn" + // Users can set this on a Service or Ingress. This prototype only looks at Services + AnnotationProxyClass = "tailscale.com/proxy-class" + // Annotations settable by users on ingresses. AnnotationFunnel = "tailscale.com/funnel" @@ -87,6 +92,8 @@ type tailscaleSTSConfig struct { // Connector specifies a configuration of a Connector instance if that's // what this StatefulSet should be created for. Connector *connector + + ProxyClass string } type connector struct { @@ -397,7 +404,102 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S } } } - container := &ss.Spec.Template.Spec.Containers[0] + pod := &ss.Spec.Template + container := &pod.Spec.Containers[0] + // if proxyclass is set + // get the proxy class + // get the pod template thing from there + // how to merge? + if sts.ProxyClass != "" { + logger.Infof("looking at proxy class %s", sts.ProxyClass) + proxyClass := &tsapi.ProxyClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: sts.ProxyClass, + }, + } + if err := a.Get(ctx, client.ObjectKeyFromObject(proxyClass), proxyClass); err != nil { + return nil, fmt.Errorf("failed to get ProxyClass: %w", err) + } + logger.Infof("retrieved proxy class %#+v", proxyClass.Spec) + // only look at pod spec for this prototype + if ssOverlay := proxyClass.Spec.StatefulSet; ssOverlay != nil && ssOverlay.Pod != nil { + pod.Labels = ssOverlay.Pod.Labels + pod.Annotations = ssOverlay.Pod.Annotations + pod.Spec.NodeName = ssOverlay.Pod.NodeName + pod.Spec.NodeSelector = ssOverlay.Pod.NodeSelector + logger.Infof("Setting pod node selctor: %+#v", ssOverlay.Pod.NodeSelector) + pod.Spec.ImagePullSecrets = ssOverlay.Pod.ImagePullSecrets + pod.Spec.Tolerations = ssOverlay.Pod.Tolerations + if ssOverlay.Pod.PodSecurityContext != nil { + pod.Spec.SecurityContext = ssOverlay.Pod.PodSecurityContext + } + if contOverlay := proxyClass.Spec.StatefulSet.Pod.TailscaleContainer; contOverlay != nil { + if contOverlay.SecurityContext != nil { + // alternatively we could merge this with the existing security context + container.SecurityContext = contOverlay.SecurityContext + } + container.Resources = contOverlay.Resources + } + if initContOverlay := proxyClass.Spec.StatefulSet.Pod.TailscaleInitContainer; initContOverlay != nil { + if initContOverlay.SecurityContext != nil { + pod.Spec.InitContainers[0].SecurityContext = initContOverlay.SecurityContext + } + } + if len(ssOverlay.Pod.Patches) > 0 { + logger.Info("applying overlay patches") + // logger.Infof("before modifying pod's init containers are %#+v", pod.Spec.InitContainers) + // get all patches together + var patches []byte + for _, patch := range ssOverlay.Pod.Patches { + jsonBytes, err := json.Marshal(patch) + if err != nil { + return nil, fmt.Errorf("error marshaling JSON patch: %w", err) + } + // there is definitely a better way + jsonBytes = []byte("[" + string(jsonBytes) + "]") + // patch, err := jsonpatch.DecodePatch(jsonBytes) + // if err != nil { + // return nil, fmt.Errorf("error decoding JSON patch: %w", err) + // } + if len(patches) == 0 { + patches = jsonBytes + } else { + patches, err = jsonpatch.MergeMergePatches(patches, jsonBytes) + if err != nil { + return nil, fmt.Errorf("error merging patches: %w", err) + } + } + logger.Infof("patch before merging : %+v\n", string(jsonBytes)) + } + + // this can be done better + podBytes, err := json.Marshal(pod) + if err != nil { + return nil, fmt.Errorf("error marshaling Pod spec to JSON: %w", err) + } + logger.Infof("patches before unmarshal: %+#v\n", string(patches)) + mergePatch, err := jsonpatch.DecodePatch(patches) + if err != nil { + return nil, fmt.Errorf("error decoding JSON patches: %w", err) + } + modifiedPodBytes, err := mergePatch.Apply(podBytes) + if err != nil { + return nil, fmt.Errorf("error applying patch: %w", err) + } + // modifiedPodBytes, err := jsonpatch.MergePatch(podBytes, patches) + // if err != nil { + // return nil, fmt.Errorf("error updating Pod spec using merge patch: %w", err) + // } + // if jsonpatch.Equal(podBytes, modifiedPodBytes) { + // logger.Info("no change was applied") + // } + logger.Infof("modified pod's init containers are %#+v", string(modifiedPodBytes)) + if err = json.Unmarshal(modifiedPodBytes, pod); err != nil { + return nil, fmt.Errorf("error umarshaling pod bytes: %w", err) + } + } + } + } container.Image = a.proxyImage ss.ObjectMeta = metav1.ObjectMeta{ Name: headlessSvc.Name, @@ -410,9 +512,9 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S "app": sts.ParentResourceUID, }, } - mak.Set(&ss.Spec.Template.Labels, "app", sts.ParentResourceUID) + mak.Set(&pod.Labels, "app", sts.ParentResourceUID) for key, val := range sts.ChildResourceLabels { - ss.Spec.Template.Labels[key] = val // sync StatefulSet labels to Pod to make it easier for users to select the Pod + pod.Labels[key] = val // sync StatefulSet labels to Pod to make it easier for users to select the Pod } // Generic containerboot configuration options. @@ -431,12 +533,12 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S // it is passed via an environment variable. So we need to restart the // container when the value changes. We do this by adding an annotation to // the pod template that contains the last value we set. - mak.Set(&ss.Spec.Template.Annotations, podAnnotationLastSetHostname, sts.Hostname) + mak.Set(&pod.Annotations, podAnnotationLastSetHostname, sts.Hostname) } // Configure containeboot to run tailscaled with a configfile read from the state Secret. if shouldDoTailscaledDeclarativeConfig(sts) { mak.Set(&ss.Spec.Template.Annotations, podAnnotationLastSetConfigFileHash, tsConfigHash) - ss.Spec.Template.Spec.Volumes = append(ss.Spec.Template.Spec.Volumes, corev1.Volume{ + pod.Spec.Volumes = append(ss.Spec.Template.Spec.Volumes, corev1.Volume{ Name: "tailscaledconfig", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ @@ -465,7 +567,7 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S Value: a.tsFirewallMode, }) } - ss.Spec.Template.Spec.PriorityClassName = a.proxyPriorityClassName + pod.Spec.PriorityClassName = a.proxyPriorityClassName // Ingress/egress proxy configuration options. if sts.ClusterTargetIP != "" { @@ -496,7 +598,7 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S ReadOnly: true, MountPath: "/etc/tailscaled", }) - ss.Spec.Template.Spec.Volumes = append(ss.Spec.Template.Spec.Volumes, corev1.Volume{ + pod.Spec.Volumes = append(ss.Spec.Template.Spec.Volumes, corev1.Volume{ Name: "serve-config", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ diff --git a/cmd/k8s-operator/svc.go b/cmd/k8s-operator/svc.go index d6b810e73..c6207c49e 100644 --- a/cmd/k8s-operator/svc.go +++ b/cmd/k8s-operator/svc.go @@ -183,6 +183,7 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga Hostname: hostname, Tags: tags, ChildResourceLabels: crl, + ProxyClass: svc.GetAnnotations()[AnnotationProxyClass], // nil? } a.mu.Lock() diff --git a/k8s-operator/apis/v1alpha1/register.go b/k8s-operator/apis/v1alpha1/register.go index d8929a9f5..4e6bfda64 100644 --- a/k8s-operator/apis/v1alpha1/register.go +++ b/k8s-operator/apis/v1alpha1/register.go @@ -49,7 +49,7 @@ func init() { // Adds the list of known types to api.Scheme. func addKnownTypes(scheme *runtime.Scheme) error { - scheme.AddKnownTypes(SchemeGroupVersion, &Connector{}, &ConnectorList{}) + scheme.AddKnownTypes(SchemeGroupVersion, &Connector{}, &ConnectorList{}, &ProxyClass{}, &ProxyClassList{}) metav1.AddToGroupVersion(scheme, SchemeGroupVersion) return nil diff --git a/k8s-operator/apis/v1alpha1/types_proxyclass.go b/k8s-operator/apis/v1alpha1/types_proxyclass.go new file mode 100644 index 000000000..b458cd0fb --- /dev/null +++ b/k8s-operator/apis/v1alpha1/types_proxyclass.go @@ -0,0 +1,88 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package v1alpha1 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var ProxyClassKind = "ProxyClass" + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:scope=Cluster,shortName=pc + +type ProxyClass struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec ProxyClassSpec `json:"spec"` + + // This would need status if we do any validation in operator. + // +optional + // Status ProxyClassStatus `json:"status"` +} + +// +kubebuilder:object:root=true + +type ProxyClassList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata"` + + Items []ProxyClass `json:"items"` +} + +type ProxyClassSpec struct { + // +optional + Service `json:"service,omitempty"` + // +optional + StatefulSet *StatefulSet `json:"statefulSet,omitempty"` +} + +// Configuration for the headless Service, not actually used in this prototype, +// but is here to better illustrate the API structure +type Service struct { + Labels map[string]string `json:"labels,omitempty"` +} + +type StatefulSet struct { + // +optional + Labels map[string]string `json:"labels,omitempty"` + // +optional + Annotations map[string]string `json:"annotations,omitempty"` + // +optional + Pod *Pod `json:"pod,omitempty"` +} + +type Pod struct { + // Or should we just sync statefulset.labels, statefulset.annotations? + // +optional + Labels map[string]string `json:"labels,omitempty"` + // +optional + Annotations map[string]string `json:"annotations,omitempty"` + TailscaleContainer *Container `json:"tailscaleContainer,omitempty"` + TailscaleInitContainer *Container `json:"tailscaleInitContainer,omitempty"` + PodSecurityContext *corev1.PodSecurityContext `json:"podSecurityContext,omitempty"` + ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"` + NodeName string `json:"nodeName,omitempty"` + NodeSelector map[string]string `json:"nodeSelector,omitempty"` + Tolerations []corev1.Toleration `json:"tolerations,omitempty"` + Patches []Patch `json:"patches,omitempty"` +} + +type Container struct { + SecurityContext *corev1.SecurityContext `json:"securityContext,omitempty"` + Resources corev1.ResourceRequirements `json:"resources,omitempty"` +} + +// RFC 6902 JSON patch +type Patch struct { + Path string `json:"path"` + // +optional + Value string `json:"value,omitempty"` + Op string `json:"op"` +} diff --git a/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go b/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go index f0f54b533..4123e3f1a 100644 --- a/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go +++ b/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go @@ -8,6 +8,7 @@ package v1alpha1 import ( + "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" ) @@ -136,6 +137,189 @@ func (in *ConnectorStatus) DeepCopy() *ConnectorStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Container) DeepCopyInto(out *Container) { + *out = *in + if in.SecurityContext != nil { + in, out := &in.SecurityContext, &out.SecurityContext + *out = new(v1.SecurityContext) + (*in).DeepCopyInto(*out) + } + in.Resources.DeepCopyInto(&out.Resources) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Container. +func (in *Container) DeepCopy() *Container { + if in == nil { + return nil + } + out := new(Container) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Patch) DeepCopyInto(out *Patch) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Patch. +func (in *Patch) DeepCopy() *Patch { + if in == nil { + return nil + } + out := new(Patch) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Pod) DeepCopyInto(out *Pod) { + *out = *in + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.TailscaleContainer != nil { + in, out := &in.TailscaleContainer, &out.TailscaleContainer + *out = new(Container) + (*in).DeepCopyInto(*out) + } + if in.TailscaleInitContainer != nil { + in, out := &in.TailscaleInitContainer, &out.TailscaleInitContainer + *out = new(Container) + (*in).DeepCopyInto(*out) + } + if in.PodSecurityContext != nil { + in, out := &in.PodSecurityContext, &out.PodSecurityContext + *out = new(v1.PodSecurityContext) + (*in).DeepCopyInto(*out) + } + if in.ImagePullSecrets != nil { + in, out := &in.ImagePullSecrets, &out.ImagePullSecrets + *out = make([]v1.LocalObjectReference, len(*in)) + copy(*out, *in) + } + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Tolerations != nil { + in, out := &in.Tolerations, &out.Tolerations + *out = make([]v1.Toleration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Patches != nil { + in, out := &in.Patches, &out.Patches + *out = make([]Patch, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Pod. +func (in *Pod) DeepCopy() *Pod { + if in == nil { + return nil + } + out := new(Pod) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProxyClass) DeepCopyInto(out *ProxyClass) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyClass. +func (in *ProxyClass) DeepCopy() *ProxyClass { + if in == nil { + return nil + } + out := new(ProxyClass) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ProxyClass) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProxyClassList) DeepCopyInto(out *ProxyClassList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]ProxyClass, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyClassList. +func (in *ProxyClassList) DeepCopy() *ProxyClassList { + if in == nil { + return nil + } + out := new(ProxyClassList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ProxyClassList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProxyClassSpec) DeepCopyInto(out *ProxyClassSpec) { + *out = *in + in.Service.DeepCopyInto(&out.Service) + if in.StatefulSet != nil { + in, out := &in.StatefulSet, &out.StatefulSet + *out = new(StatefulSet) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyClassSpec. +func (in *ProxyClassSpec) DeepCopy() *ProxyClassSpec { + if in == nil { + return nil + } + out := new(ProxyClassSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in Routes) DeepCopyInto(out *Routes) { { @@ -155,6 +339,62 @@ func (in Routes) DeepCopy() Routes { return *out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Service) DeepCopyInto(out *Service) { + *out = *in + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Service. +func (in *Service) DeepCopy() *Service { + if in == nil { + return nil + } + out := new(Service) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StatefulSet) DeepCopyInto(out *StatefulSet) { + *out = *in + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Pod != nil { + in, out := &in.Pod, &out.Pod + *out = new(Pod) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StatefulSet. +func (in *StatefulSet) DeepCopy() *StatefulSet { + if in == nil { + return nil + } + out := new(StatefulSet) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SubnetRouter) DeepCopyInto(out *SubnetRouter) { *out = *in