mirror of https://github.com/tailscale/tailscale/
cmd/k8s-operator/e2e: run e2e tests with fewer setup requirements
Adds orchestration for more of the e2e testing setup requirements to make it easier to run them in CI, but also run them locally in a way that's consistent with CI. Updates tailscale/corp#32085 Change-Id: Ia7bff38af3801fd141ad17452aa5a68b7e724ca6 Signed-off-by: Tom Proctor <tomhjp@users.noreply.github.com>tomhjp/k8s-e2e
Tom Proctor
2 months ago
parent
f4ffc32e42
commit
f120809f80
@ -0,0 +1,19 @@
|
||||
-----BEGIN CERTIFICATE-----
|
||||
MIIDCTCCAfGgAwIBAgIIJOLbes8sTr4wDQYJKoZIhvcNAQELBQAwIDEeMBwGA1UE
|
||||
AxMVbWluaWNhIHJvb3QgY2EgMjRlMmRiMCAXDTE3MTIwNjE5NDIxMFoYDzIxMTcx
|
||||
MjA2MTk0MjEwWjAgMR4wHAYDVQQDExVtaW5pY2Egcm9vdCBjYSAyNGUyZGIwggEi
|
||||
MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC5WgZNoVJandj43kkLyU50vzCZ
|
||||
alozvdRo3OFiKoDtmqKPNWRNO2hC9AUNxTDJco51Yc42u/WV3fPbbhSznTiOOVtn
|
||||
Ajm6iq4I5nZYltGGZetGDOQWr78y2gWY+SG078MuOO2hyDIiKtVc3xiXYA+8Hluu
|
||||
9F8KbqSS1h55yxZ9b87eKR+B0zu2ahzBCIHKmKWgc6N13l7aDxxY3D6uq8gtJRU0
|
||||
toumyLbdzGcupVvjbjDP11nl07RESDWBLG1/g3ktJvqIa4BWgU2HMh4rND6y8OD3
|
||||
Hy3H8MY6CElL+MOCbFJjWqhtOxeFyZZV9q3kYnk9CAuQJKMEGuN4GU6tzhW1AgMB
|
||||
AAGjRTBDMA4GA1UdDwEB/wQEAwIChDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYB
|
||||
BQUHAwIwEgYDVR0TAQH/BAgwBgEB/wIBADANBgkqhkiG9w0BAQsFAAOCAQEAF85v
|
||||
d40HK1ouDAtWeO1PbnWfGEmC5Xa478s9ddOd9Clvp2McYzNlAFfM7kdcj6xeiNhF
|
||||
WPIfaGAi/QdURSL/6C1KsVDqlFBlTs9zYfh2g0UXGvJtj1maeih7zxFLvet+fqll
|
||||
xseM4P9EVJaQxwuK/F78YBt0tCNfivC6JNZMgxKF59h0FBpH70ytUSHXdz7FKwix
|
||||
Mfn3qEb9BXSk0Q3prNV5sOV3vgjEtB4THfDxSz9z3+DepVnW3vbbqwEbkXdk3j82
|
||||
2muVldgOUgTwK8eT+XdofVdntzU/kzygSAtAQwLJfn51fS1GvEcYGBc1bDryIqmF
|
||||
p9BI7gVKtWSZYegicA==
|
||||
-----END CERTIFICATE-----
|
||||
@ -0,0 +1,171 @@
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
appsv1 "k8s.io/api/apps/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/intstr"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"tailscale.com/types/ptr"
|
||||
)
|
||||
|
||||
func applyPebbleResources(ctx context.Context, cl client.Client) error {
|
||||
owner := client.FieldOwner("k8s-test")
|
||||
|
||||
if err := cl.Patch(ctx, pebbleDeployment(pebbleTag), client.Apply, owner); err != nil {
|
||||
return fmt.Errorf("failed to apply pebble Deployment: %w", err)
|
||||
}
|
||||
if err := cl.Patch(ctx, pebbleService(), client.Apply, owner); err != nil {
|
||||
return fmt.Errorf("failed to apply pebble Service: %w", err)
|
||||
}
|
||||
if err := cl.Patch(ctx, tailscaleNamespace(), client.Apply, owner); err != nil {
|
||||
return fmt.Errorf("failed to apply tailscale Namespace: %w", err)
|
||||
}
|
||||
if err := cl.Patch(ctx, pebbleExternalNameService(), client.Apply, owner); err != nil {
|
||||
return fmt.Errorf("failed to apply pebble ExternalName Service: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func pebbleDeployment(tag string) *appsv1.Deployment {
|
||||
return &appsv1.Deployment{
|
||||
TypeMeta: metav1.TypeMeta{
|
||||
Kind: "Deployment",
|
||||
APIVersion: "apps/v1",
|
||||
},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "pebble",
|
||||
Namespace: ns,
|
||||
},
|
||||
Spec: appsv1.DeploymentSpec{
|
||||
Replicas: ptr.To[int32](1),
|
||||
Selector: &metav1.LabelSelector{
|
||||
MatchLabels: map[string]string{
|
||||
"app": "pebble",
|
||||
},
|
||||
},
|
||||
Template: corev1.PodTemplateSpec{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Labels: map[string]string{
|
||||
"app": "pebble",
|
||||
},
|
||||
},
|
||||
Spec: corev1.PodSpec{
|
||||
Containers: []corev1.Container{
|
||||
{
|
||||
Name: "pebble",
|
||||
Image: fmt.Sprintf("ghcr.io/letsencrypt/pebble:%s", tag),
|
||||
ImagePullPolicy: corev1.PullIfNotPresent,
|
||||
Args: []string{
|
||||
"-dnsserver=localhost:8053",
|
||||
"-strict",
|
||||
},
|
||||
Ports: []corev1.ContainerPort{
|
||||
{
|
||||
Name: "acme",
|
||||
ContainerPort: 14000,
|
||||
},
|
||||
{
|
||||
Name: "pebble-api",
|
||||
ContainerPort: 15000,
|
||||
},
|
||||
},
|
||||
Env: []corev1.EnvVar{
|
||||
{
|
||||
Name: "PEBBLE_VA_NOSLEEP",
|
||||
Value: "1",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "challtestsrv",
|
||||
Image: fmt.Sprintf("ghcr.io/letsencrypt/pebble-challtestsrv:%s", tag),
|
||||
ImagePullPolicy: corev1.PullIfNotPresent,
|
||||
Args: []string{"-defaultIPv6="},
|
||||
Ports: []corev1.ContainerPort{
|
||||
{
|
||||
Name: "mgmt-api",
|
||||
ContainerPort: 8055,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func pebbleService() *corev1.Service {
|
||||
return &corev1.Service{
|
||||
TypeMeta: metav1.TypeMeta{
|
||||
Kind: "Service",
|
||||
APIVersion: "v1",
|
||||
},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "pebble",
|
||||
Namespace: ns,
|
||||
},
|
||||
Spec: corev1.ServiceSpec{
|
||||
Type: corev1.ServiceTypeClusterIP,
|
||||
Selector: map[string]string{
|
||||
"app": "pebble",
|
||||
},
|
||||
Ports: []corev1.ServicePort{
|
||||
{
|
||||
Name: "acme",
|
||||
Port: 14000,
|
||||
TargetPort: intstr.FromInt(14000),
|
||||
},
|
||||
{
|
||||
Name: "pebble-api",
|
||||
Port: 15000,
|
||||
TargetPort: intstr.FromInt(15000),
|
||||
},
|
||||
{
|
||||
Name: "mgmt-api",
|
||||
Port: 8055,
|
||||
TargetPort: intstr.FromInt(8055),
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func tailscaleNamespace() *corev1.Namespace {
|
||||
return &corev1.Namespace{
|
||||
TypeMeta: metav1.TypeMeta{
|
||||
Kind: "Namespace",
|
||||
APIVersion: "v1",
|
||||
},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "tailscale",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// pebbleExternalNameService ensures the operator in the tailscale namespace
|
||||
// can reach pebble on a DNS name (pebble) that matches its TLS cert.
|
||||
func pebbleExternalNameService() *corev1.Service {
|
||||
return &corev1.Service{
|
||||
TypeMeta: metav1.TypeMeta{
|
||||
Kind: "Service",
|
||||
APIVersion: "v1",
|
||||
},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "pebble",
|
||||
Namespace: "tailscale",
|
||||
},
|
||||
Spec: corev1.ServiceSpec{
|
||||
Type: corev1.ServiceTypeExternalName,
|
||||
Selector: map[string]string{
|
||||
"app": "pebble",
|
||||
},
|
||||
ExternalName: "pebble.default.svc.cluster.local",
|
||||
},
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,690 @@
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
_ "embed"
|
||||
jsonv1 "encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-logr/zapr"
|
||||
"github.com/google/go-containerregistry/pkg/name"
|
||||
"github.com/google/go-containerregistry/pkg/v1/daemon"
|
||||
"github.com/google/go-containerregistry/pkg/v1/tarball"
|
||||
"go.uber.org/zap"
|
||||
"golang.org/x/oauth2/clientcredentials"
|
||||
"helm.sh/helm/v3/pkg/action"
|
||||
"helm.sh/helm/v3/pkg/chart"
|
||||
"helm.sh/helm/v3/pkg/chart/loader"
|
||||
"helm.sh/helm/v3/pkg/cli"
|
||||
"helm.sh/helm/v3/pkg/release"
|
||||
"helm.sh/helm/v3/pkg/storage/driver"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime/serializer"
|
||||
"k8s.io/apimachinery/pkg/watch"
|
||||
"k8s.io/client-go/rest"
|
||||
"k8s.io/client-go/tools/clientcmd"
|
||||
"k8s.io/client-go/tools/portforward"
|
||||
"k8s.io/client-go/transport/spdy"
|
||||
"k8s.io/client-go/util/apply"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
klog "sigs.k8s.io/controller-runtime/pkg/log"
|
||||
kzap "sigs.k8s.io/controller-runtime/pkg/log/zap"
|
||||
"sigs.k8s.io/kind/pkg/cluster"
|
||||
"sigs.k8s.io/kind/pkg/cluster/nodeutils"
|
||||
"sigs.k8s.io/kind/pkg/cmd"
|
||||
"tailscale.com/client/tailscale/v2"
|
||||
"tailscale.com/ipn"
|
||||
"tailscale.com/ipn/store/mem"
|
||||
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
|
||||
"tailscale.com/tsnet"
|
||||
)
|
||||
|
||||
const (
|
||||
pebbleTag = "2.8.0"
|
||||
ns = "default"
|
||||
tmp = "/tmp/k8s-operator-e2e"
|
||||
kindClusterName = "k8s-operator-e2e"
|
||||
)
|
||||
|
||||
var (
|
||||
tsClient = &tailscale.Client{Tailnet: "-"} // For API calls to control.
|
||||
tnClient *tsnet.Server // For testing real tailnet traffic.
|
||||
kubeClient client.WithWatch // For k8s API calls.
|
||||
|
||||
//go:embed certs/pebble.minica.crt
|
||||
pebbleMiniCACert []byte
|
||||
|
||||
// Either nil (system) or pebble CAs if pebble is deployed for devcontrol.
|
||||
// pebble has a static "mini" CA that its ACME directory URL serves a cert
|
||||
// from, and also dynamically generates a different CA for issuing certs.
|
||||
testCAs *x509.CertPool
|
||||
|
||||
//go:embed acl.hujson
|
||||
requiredACLs []byte
|
||||
|
||||
fDevcontrol = flag.Bool("devcontrol", false, "if true, connect to devcontrol at http://localhost:31544. Run devcontrol with "+`
|
||||
./tool/go run ./cmd/devcontrol \
|
||||
--generate-test-devices=k8s-operator-e2e \
|
||||
--addr=localhost:31544 \
|
||||
--dir=/tmp/devcontrol \
|
||||
--scenario-output-dir=/tmp/k8s-operator-e2e \
|
||||
--test-dns=http://localhost:8055`)
|
||||
fSkipCleanup = flag.Bool("skip-cleanup", false, "if true, do not delete the kind cluster (if created) or tmp dir on exit")
|
||||
fCluster = flag.Bool("cluster", false, "if true, create or use a pre-existing kind cluster named k8s-operator-e2e; otherwise assume a usable cluster already exists in kubeconfig")
|
||||
fBuild = flag.Bool("build", false, "if true, build and deploy the operator and container images from the current checkout; otherwise assume the operator is already set up")
|
||||
)
|
||||
|
||||
func runTests(m *testing.M) (int, error) {
|
||||
logger := kzap.NewRaw().Sugar()
|
||||
klog.SetLogger(zapr.NewLogger(logger.Desugar()))
|
||||
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT)
|
||||
defer cancel()
|
||||
|
||||
ossDir, err := gitRootDir()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := os.MkdirAll(tmp, 0755); err != nil {
|
||||
return 0, fmt.Errorf("failed to create temp dir: %w", err)
|
||||
}
|
||||
if !*fSkipCleanup {
|
||||
defer os.RemoveAll(tmp)
|
||||
}
|
||||
|
||||
logger.Infof("temp dir: %q", tmp)
|
||||
logger.Infof("oss dir: %q", ossDir)
|
||||
|
||||
var (
|
||||
kubeconfig string
|
||||
kindProvider *cluster.Provider
|
||||
)
|
||||
if *fCluster {
|
||||
kubeconfig = filepath.Join(tmp, "kubeconfig")
|
||||
kindProvider = cluster.NewProvider(
|
||||
cluster.ProviderWithLogger(cmd.NewLogger()),
|
||||
)
|
||||
clusters, err := kindProvider.List()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to list kind clusters: %w", err)
|
||||
}
|
||||
if !slices.Contains(clusters, kindClusterName) {
|
||||
if err := kindProvider.Create(kindClusterName,
|
||||
cluster.CreateWithWaitForReady(5*time.Minute),
|
||||
cluster.CreateWithKubeconfigPath(kubeconfig),
|
||||
cluster.CreateWithNodeImage("kindest/node:v1.30.0"),
|
||||
); err != nil {
|
||||
return 0, fmt.Errorf("failed to create kind cluster: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if !*fSkipCleanup {
|
||||
defer kindProvider.Delete(kindClusterName, kubeconfig)
|
||||
}
|
||||
}
|
||||
|
||||
// Cluster client setup.
|
||||
restCfg, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("error loading kubeconfig: %w", err)
|
||||
}
|
||||
kubeClient, err = client.NewWithWatch(restCfg, client.Options{Scheme: tsapi.GlobalScheme})
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("error creating Kubernetes client: %w", err)
|
||||
}
|
||||
|
||||
var (
|
||||
clusterLoginServer string // Login server from cluster Pod point of view.
|
||||
clientID, clientSecret string // OAuth client for the operator to use.
|
||||
caPaths []string // Extra CA cert file paths to add to images.
|
||||
|
||||
certsDir string = filepath.Join(tmp, "certs") // Directory containing extra CA certs to add to images.
|
||||
)
|
||||
if *fDevcontrol {
|
||||
// Deploy pebble and get its certs.
|
||||
if err := applyPebbleResources(ctx, kubeClient); err != nil {
|
||||
return 0, fmt.Errorf("failed to apply pebble resources: %w", err)
|
||||
}
|
||||
pebblePod, err := waitForPodReady(ctx, logger, kubeClient, ns, client.MatchingLabels{"app": "pebble"})
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("pebble pod not ready: %w", err)
|
||||
}
|
||||
if err := forwardLocalPortToPod(ctx, logger, restCfg, ns, pebblePod, 15000); err != nil {
|
||||
return 0, fmt.Errorf("failed to set up port forwarding to pebble: %w", err)
|
||||
}
|
||||
testCAs = x509.NewCertPool()
|
||||
if ok := testCAs.AppendCertsFromPEM(pebbleMiniCACert); !ok {
|
||||
return 0, fmt.Errorf("failed to parse pebble minica cert")
|
||||
}
|
||||
var pebbleCAChain []byte
|
||||
for _, path := range []string{"/intermediates/0", "/roots/0"} {
|
||||
pem, err := pebbleGet(ctx, 15000, path)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
pebbleCAChain = append(pebbleCAChain, pem...)
|
||||
}
|
||||
if ok := testCAs.AppendCertsFromPEM(pebbleCAChain); !ok {
|
||||
return 0, fmt.Errorf("failed to parse pebble ca chain cert")
|
||||
}
|
||||
if err := os.MkdirAll(certsDir, 0755); err != nil {
|
||||
return 0, fmt.Errorf("failed to create certs dir: %w", err)
|
||||
}
|
||||
pebbleCAChainPath := filepath.Join(certsDir, "pebble-ca-chain.crt")
|
||||
if err := os.WriteFile(pebbleCAChainPath, pebbleCAChain, 0644); err != nil {
|
||||
return 0, fmt.Errorf("failed to write pebble CA chain: %w", err)
|
||||
}
|
||||
pebbleMiniCACertPath := filepath.Join(certsDir, "pebble.minica.crt")
|
||||
if err := os.WriteFile(pebbleMiniCACertPath, pebbleMiniCACert, 0644); err != nil {
|
||||
return 0, fmt.Errorf("failed to write pebble minica: %w", err)
|
||||
}
|
||||
caPaths = []string{pebbleCAChainPath, pebbleMiniCACertPath}
|
||||
|
||||
// Set up network connectivity between cluster and devcontrol.
|
||||
//
|
||||
// For devcontrol -> pebble (DNS mgmt for ACME challenges):
|
||||
// * Port forward from localhost port 8055 to in-cluster pebble port 8055.
|
||||
//
|
||||
// For Pods -> devcontrol (tailscale clients joining the tailnet):
|
||||
// * Create ssh-server Deployment in cluster.
|
||||
// * Create reverse ssh tunnel that goes from ssh-server port 31544 to localhost:31544.
|
||||
if err := forwardLocalPortToPod(ctx, logger, restCfg, ns, pebblePod, 8055); err != nil {
|
||||
return 0, fmt.Errorf("failed to set up port forwarding to pebble: %w", err)
|
||||
}
|
||||
privateKey, publicKey, err := readOrGenerateSSHKey(tmp)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to read or generate SSH key: %w", err)
|
||||
}
|
||||
sshServiceIP, err := connectClusterToDevcontrol(ctx, logger, kubeClient, restCfg, privateKey, publicKey)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to set up cluster->devcontrol connection: %w", err)
|
||||
}
|
||||
if !*fSkipCleanup {
|
||||
defer func() {
|
||||
if err := cleanupSSHResources(context.Background(), kubeClient); err != nil {
|
||||
logger.Infof("failed to clean up ssh-server resources: %v", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Address cluster workloads can reach devcontrol at. Must be a private
|
||||
// IP to make sure tailscale client code recognises it shouldn't try an
|
||||
// https fallback. See [controlclient.NewNoiseClient] for details.
|
||||
clusterLoginServer = fmt.Sprintf("http://%s:31544", sshServiceIP)
|
||||
|
||||
b, err := os.ReadFile(filepath.Join(tmp, "api-key.json"))
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to read api-key.json: %w", err)
|
||||
}
|
||||
var apiKeyData struct {
|
||||
APIKey string `json:"apiKey"`
|
||||
}
|
||||
if err := jsonv1.Unmarshal(b, &apiKeyData); err != nil {
|
||||
return 0, fmt.Errorf("failed to parse api-key.json: %w", err)
|
||||
}
|
||||
if apiKeyData.APIKey == "" {
|
||||
return 0, fmt.Errorf("api-key.json did not contain an API key")
|
||||
}
|
||||
logger.Infof("got API key from devcontrol: %s", apiKeyData.APIKey)
|
||||
|
||||
// Finish setting up tsClient.
|
||||
baseURL, err := url.Parse("http://localhost:31544")
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("parse url: %w", err)
|
||||
}
|
||||
tsClient.BaseURL = baseURL
|
||||
tsClient.APIKey = apiKeyData.APIKey
|
||||
tsClient.HTTP = &http.Client{}
|
||||
|
||||
// Set ACLs and create OAuth client.
|
||||
if err := tsClient.PolicyFile().Set(ctx, string(requiredACLs), ""); err != nil {
|
||||
return 0, fmt.Errorf("failed to set ACLs: %w", err)
|
||||
}
|
||||
logger.Infof("ACLs configured")
|
||||
|
||||
key, err := tsClient.Keys().CreateOAuthClient(ctx, tailscale.CreateOAuthClientRequest{
|
||||
Scopes: []string{"auth_keys", "devices:core", "services"},
|
||||
Tags: []string{"tag:k8s-operator"},
|
||||
Description: "k8s-operator client for e2e tests",
|
||||
})
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to create OAuth client: %w", err)
|
||||
}
|
||||
clientID = key.ID
|
||||
clientSecret = key.Key
|
||||
} else {
|
||||
clientSecret = os.Getenv("TS_API_CLIENT_SECRET")
|
||||
if clientSecret == "" {
|
||||
return 0, fmt.Errorf("must use --devcontrol or set TS_API_CLIENT_SECRET to an OAuth client suitable for the operator")
|
||||
}
|
||||
// Format is "tskey-client-<id>-<random>".
|
||||
parts := strings.Split(clientSecret, "-")
|
||||
if len(parts) != 4 {
|
||||
return 0, fmt.Errorf("TS_API_CLIENT_SECRET is not valid")
|
||||
}
|
||||
clientID = parts[2]
|
||||
credentials := clientcredentials.Config{
|
||||
ClientID: clientID,
|
||||
ClientSecret: clientSecret,
|
||||
TokenURL: fmt.Sprintf("%s/api/v2/oauth/token", ipn.DefaultControlURL),
|
||||
Scopes: []string{"auth_keys"},
|
||||
}
|
||||
baseURL, _ := url.Parse(ipn.DefaultControlURL)
|
||||
tsClient = &tailscale.Client{
|
||||
Tailnet: "-",
|
||||
HTTP: credentials.Client(ctx),
|
||||
BaseURL: baseURL,
|
||||
}
|
||||
}
|
||||
|
||||
var ossTag string
|
||||
if *fBuild {
|
||||
// TODO(tomhjp): proper support for --build=false and layering pebble certs on top of existing images.
|
||||
// TODO(tomhjp): support non-local platform.
|
||||
// TODO(tomhjp): build tsrecorder as well.
|
||||
|
||||
// Build tailscale/k8s-operator, tailscale/tailscale, tailscale/k8s-proxy, with pebble CAs added.
|
||||
ossTag, err = tagForRepo(ossDir)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
logger.Infof("using OSS image tag: %q", ossTag)
|
||||
ossImageToTarget := map[string]string{
|
||||
"local/k8s-operator": "publishdevoperator",
|
||||
"local/tailscale": "publishdevimage",
|
||||
"local/k8s-proxy": "publishdevproxy",
|
||||
}
|
||||
for img, target := range ossImageToTarget {
|
||||
if err := buildImage(ctx, ossDir, img, target, ossTag, caPaths); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
nodes, err := kindProvider.ListInternalNodes(kindClusterName)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to list kind nodes: %w", err)
|
||||
}
|
||||
// TODO(tomhjp): can be made more efficient and portable if we
|
||||
// stream built image tarballs straight to the node rather than
|
||||
// going via the daemon.
|
||||
// TODO(tomhjp): support --build with non-kind clusters.
|
||||
imgRef, err := name.ParseReference(fmt.Sprintf("%s:%s", img, ossTag))
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to parse image reference: %w", err)
|
||||
}
|
||||
img, err := daemon.Image(imgRef)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to get image from daemon: %w", err)
|
||||
}
|
||||
pr, pw := io.Pipe()
|
||||
go func() {
|
||||
defer pw.Close()
|
||||
if err := tarball.Write(imgRef, img, pw); err != nil {
|
||||
logger.Infof("failed to write image to pipe: %v", err)
|
||||
}
|
||||
}()
|
||||
for _, n := range nodes {
|
||||
if err := nodeutils.LoadImageArchive(n, pr); err != nil {
|
||||
return 0, fmt.Errorf("failed to load image into node %q: %w", n.String(), err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Generate CRDs for the helm chart.
|
||||
cmd := exec.CommandContext(ctx, "go", "run", "tailscale.com/cmd/k8s-operator/generate", "helmcrd")
|
||||
cmd.Dir = ossDir
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to generate CRD: %v: %s", err, out)
|
||||
}
|
||||
|
||||
// Load and install helm chart.
|
||||
chart, err := loader.Load(filepath.Join(ossDir, "cmd", "k8s-operator", "deploy", "chart"))
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to load helm chart: %w", err)
|
||||
}
|
||||
values := map[string]any{
|
||||
"loginServer": clusterLoginServer,
|
||||
"oauth": map[string]any{
|
||||
"clientId": clientID,
|
||||
"clientSecret": clientSecret,
|
||||
},
|
||||
"apiServerProxyConfig": map[string]any{
|
||||
"mode": "true",
|
||||
},
|
||||
"operatorConfig": map[string]any{
|
||||
"logging": "debug",
|
||||
"extraEnv": []map[string]any{
|
||||
{
|
||||
"name": "K8S_PROXY_IMAGE",
|
||||
"value": "local/k8s-proxy:" + ossTag,
|
||||
},
|
||||
{
|
||||
"name": "TS_DEBUG_ACME_DIRECTORY_URL",
|
||||
"value": "https://pebble:14000/dir",
|
||||
},
|
||||
},
|
||||
"image": map[string]any{
|
||||
"repo": "local/k8s-operator",
|
||||
"tag": ossTag,
|
||||
"pullPolicy": "IfNotPresent",
|
||||
},
|
||||
},
|
||||
"proxyConfig": map[string]any{
|
||||
"defaultProxyClass": "default",
|
||||
"image": map[string]any{
|
||||
"repository": "local/tailscale",
|
||||
"tag": ossTag,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
settings := cli.New()
|
||||
settings.KubeConfig = kubeconfig
|
||||
settings.SetNamespace("tailscale")
|
||||
helmCfg := &action.Configuration{}
|
||||
if err := helmCfg.Init(settings.RESTClientGetter(), "tailscale", "", logger.Infof); err != nil {
|
||||
return 0, fmt.Errorf("failed to initialize helm action configuration: %w", err)
|
||||
}
|
||||
|
||||
const relName = "tailscale-operator" // TODO(tomhjp): maybe configurable if others use a different value.
|
||||
f := upgraderOrInstaller(helmCfg, relName)
|
||||
if _, err := f(ctx, relName, chart, values); err != nil {
|
||||
return 0, fmt.Errorf("failed to install %q via helm: %w", relName, err)
|
||||
}
|
||||
|
||||
if err := applyDefaultProxyClass(ctx, restCfg); err != nil {
|
||||
return 0, fmt.Errorf("failed to apply default ProxyClass: %w", err)
|
||||
}
|
||||
|
||||
caps := tailscale.KeyCapabilities{}
|
||||
caps.Devices.Create.Preauthorized = true
|
||||
caps.Devices.Create.Ephemeral = true
|
||||
caps.Devices.Create.Tags = []string{"tag:k8s"}
|
||||
|
||||
authKey, err := tsClient.Keys().CreateAuthKey(ctx, tailscale.CreateKeyRequest{
|
||||
Capabilities: caps,
|
||||
ExpirySeconds: 600,
|
||||
Description: "e2e test authkey",
|
||||
})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer tsClient.Keys().Delete(context.Background(), authKey.ID)
|
||||
|
||||
tnClient = &tsnet.Server{
|
||||
ControlURL: tsClient.BaseURL.String(),
|
||||
Hostname: "test-proxy",
|
||||
Ephemeral: true,
|
||||
Store: &mem.Store{},
|
||||
AuthKey: authKey.Key,
|
||||
}
|
||||
_, err = tnClient.Up(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer tnClient.Close()
|
||||
|
||||
return m.Run(), nil
|
||||
}
|
||||
|
||||
func upgraderOrInstaller(cfg *action.Configuration, releaseName string) helmInstallerFunc {
|
||||
hist := action.NewHistory(cfg)
|
||||
hist.Max = 1
|
||||
helmVersions, err := hist.Run(releaseName)
|
||||
if err == driver.ErrReleaseNotFound || (len(helmVersions) > 0 && helmVersions[0].Info.Status == release.StatusUninstalled) {
|
||||
return helmInstaller(cfg, releaseName)
|
||||
} else {
|
||||
return helmUpgrader(cfg)
|
||||
}
|
||||
}
|
||||
|
||||
func helmUpgrader(cfg *action.Configuration) helmInstallerFunc {
|
||||
upgrade := action.NewUpgrade(cfg)
|
||||
upgrade.Namespace = "tailscale"
|
||||
upgrade.Install = true
|
||||
upgrade.Wait = true
|
||||
upgrade.Timeout = 5 * time.Minute
|
||||
return upgrade.RunWithContext
|
||||
}
|
||||
|
||||
func helmInstaller(cfg *action.Configuration, releaseName string) helmInstallerFunc {
|
||||
install := action.NewInstall(cfg)
|
||||
install.Namespace = "tailscale"
|
||||
install.CreateNamespace = true
|
||||
install.ReleaseName = releaseName
|
||||
install.Wait = true
|
||||
install.Timeout = 5 * time.Minute
|
||||
install.Replace = true
|
||||
return func(ctx context.Context, _ string, chart *chart.Chart, values map[string]interface{}) (*release.Release, error) {
|
||||
return install.RunWithContext(ctx, chart, values)
|
||||
}
|
||||
}
|
||||
|
||||
type helmInstallerFunc func(context.Context, string, *chart.Chart, map[string]interface{}) (*release.Release, error)
|
||||
|
||||
// gitRootDir returns the top-level directory of the current git repo. Expects
|
||||
// to be run from inside a git repo.
|
||||
func gitRootDir() (string, error) {
|
||||
top, err := exec.Command("git", "rev-parse", "--show-toplevel").Output()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to find git top level (not in corp git?): %w", err)
|
||||
}
|
||||
return strings.TrimSpace(string(top)), nil
|
||||
}
|
||||
|
||||
func tagForRepo(dir string) (string, error) {
|
||||
cmd := exec.Command("git", "rev-parse", "--short", "HEAD")
|
||||
cmd.Dir = dir
|
||||
out, err := cmd.Output()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to get latest git tag for repo %q: %w", dir, err)
|
||||
}
|
||||
tag := strings.TrimSpace(string(out))
|
||||
|
||||
// If dirty, append an extra random tag to ensure unique image tags.
|
||||
cmd = exec.Command("git", "status", "--porcelain")
|
||||
cmd.Dir = dir
|
||||
out, err = cmd.Output()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to check git status for repo %q: %w", dir, err)
|
||||
}
|
||||
if strings.TrimSpace(string(out)) != "" {
|
||||
tag += "-" + strings.ToLower(rand.Text())
|
||||
}
|
||||
|
||||
return tag, nil
|
||||
}
|
||||
|
||||
func applyDefaultProxyClass(ctx context.Context, cfg *rest.Config) error {
|
||||
c := *cfg
|
||||
c.APIPath = "/apis"
|
||||
c.GroupVersion = &tsapi.SchemeGroupVersion
|
||||
c.NegotiatedSerializer = serializer.NewCodecFactory(tsapi.GlobalScheme)
|
||||
cl, err := rest.RESTClientFor(&c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
pc := &tsapi.ProxyClass{
|
||||
TypeMeta: metav1.TypeMeta{
|
||||
APIVersion: tsapi.SchemeGroupVersion.String(),
|
||||
Kind: tsapi.ProxyClassKind,
|
||||
},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "default",
|
||||
},
|
||||
Spec: tsapi.ProxyClassSpec{
|
||||
StatefulSet: &tsapi.StatefulSet{
|
||||
Pod: &tsapi.Pod{
|
||||
TailscaleInitContainer: &tsapi.Container{
|
||||
ImagePullPolicy: "IfNotPresent",
|
||||
},
|
||||
TailscaleContainer: &tsapi.Container{
|
||||
ImagePullPolicy: "IfNotPresent",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
req, err := apply.NewRequest(cl, pc)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return req.
|
||||
Resource("proxyclasses").
|
||||
Name("default").
|
||||
Param("fieldManager", "k8s-test").
|
||||
Do(ctx).
|
||||
Error()
|
||||
}
|
||||
|
||||
// forwardLocalPortToPod sets up port forwarding to the specified Pod and remote port.
|
||||
// It runs until the provided ctx is done.
|
||||
func forwardLocalPortToPod(ctx context.Context, logger *zap.SugaredLogger, cfg *rest.Config, ns, podName string, port int) error {
|
||||
transport, upgrader, err := spdy.RoundTripperFor(cfg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create round tripper: %w", err)
|
||||
}
|
||||
|
||||
u, err := url.Parse(fmt.Sprintf("%s%s/api/v1/namespaces/%s/pods/%s/portforward", cfg.Host, cfg.APIPath, ns, podName))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to parse URL: %w", err)
|
||||
}
|
||||
dialer := spdy.NewDialer(upgrader, &http.Client{Transport: transport}, "POST", u)
|
||||
|
||||
stopChan := make(chan struct{}, 1)
|
||||
readyChan := make(chan struct{}, 1)
|
||||
|
||||
ports := []string{fmt.Sprintf("%d:%d", port, port)}
|
||||
|
||||
// TODO(tomhjp): work out how zap logger can be used instead of stdout/err.
|
||||
pf, err := portforward.New(dialer, ports, stopChan, readyChan, os.Stdout, os.Stderr)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create port forwarder: %w", err)
|
||||
}
|
||||
|
||||
go func() {
|
||||
if err := pf.ForwardPorts(); err != nil {
|
||||
logger.Infof("Port forwarding error: %v\n", err)
|
||||
}
|
||||
}()
|
||||
|
||||
var once sync.Once
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
once.Do(func() { close(stopChan) })
|
||||
}()
|
||||
|
||||
// Wait for port forwarding to be ready
|
||||
select {
|
||||
case <-readyChan:
|
||||
logger.Infof("Port forwarding to Pod %s/%s ready", ns, podName)
|
||||
break
|
||||
case <-time.After(10 * time.Second):
|
||||
once.Do(func() { close(stopChan) })
|
||||
return fmt.Errorf("timeout waiting for port forward to be ready")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// waitForPodReady waits for at least 1 Pod matching the label selector to be
|
||||
// in Ready state. It returns the name of the first ready Pod it finds.
|
||||
func waitForPodReady(ctx context.Context, logger *zap.SugaredLogger, cl client.WithWatch, ns string, labelSelector client.MatchingLabels) (string, error) {
|
||||
pods := &corev1.PodList{}
|
||||
w, err := cl.Watch(ctx, pods, client.InNamespace(ns), client.MatchingLabels(labelSelector))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create pod watcher: %v", err)
|
||||
}
|
||||
defer w.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case event, ok := <-w.ResultChan():
|
||||
if !ok {
|
||||
return "", fmt.Errorf("watcher channel closed")
|
||||
}
|
||||
|
||||
switch event.Type {
|
||||
case watch.Added, watch.Modified:
|
||||
if pod, ok := event.Object.(*corev1.Pod); ok {
|
||||
for _, condition := range pod.Status.Conditions {
|
||||
if condition.Type == corev1.PodReady && condition.Status == corev1.ConditionTrue {
|
||||
logger.Infof("pod %s is ready", pod.Name)
|
||||
return pod.Name, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
case watch.Error:
|
||||
return "", fmt.Errorf("watch error: %v", event.Object)
|
||||
}
|
||||
case <-ctx.Done():
|
||||
return "", fmt.Errorf("timeout waiting for pod to be ready")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func pebbleGet(ctx context.Context, port uint16, path string) ([]byte, error) {
|
||||
pebbleClient := &http.Client{
|
||||
Transport: &http.Transport{
|
||||
TLSClientConfig: &tls.Config{
|
||||
RootCAs: testCAs,
|
||||
},
|
||||
},
|
||||
Timeout: 10 * time.Second,
|
||||
}
|
||||
req, _ := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://localhost:%d%s", port, path), nil)
|
||||
resp, err := pebbleClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to fetch pebble root CA: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("HTTP %d when fetching pebble root CA", resp.StatusCode)
|
||||
}
|
||||
b, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read pebble root CA response: %w", err)
|
||||
}
|
||||
|
||||
return b, nil
|
||||
}
|
||||
|
||||
func buildImage(ctx context.Context, dir, repo, target, tag string, extraCACerts []string) error {
|
||||
var files []string
|
||||
for _, f := range extraCACerts {
|
||||
files = append(files, fmt.Sprintf("%s:/etc/ssl/certs/%s", f, filepath.Base(f)))
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, "make", target,
|
||||
"PLATFORM=local",
|
||||
fmt.Sprintf("TAGS=%s", tag),
|
||||
fmt.Sprintf("REPO=%s", repo),
|
||||
fmt.Sprintf("FILES=%s", strings.Join(files, ",")),
|
||||
)
|
||||
cmd.Dir = dir
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
if err := cmd.Run(); err != nil {
|
||||
return fmt.Errorf("failed to build image %q: %w", target, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@ -0,0 +1,348 @@
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/pem"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
"golang.org/x/crypto/ssh"
|
||||
appsv1 "k8s.io/api/apps/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/intstr"
|
||||
"k8s.io/client-go/rest"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
tailscaleroot "tailscale.com"
|
||||
"tailscale.com/types/ptr"
|
||||
)
|
||||
|
||||
const (
|
||||
keysFilePath = "/root/.ssh/authorized_keys"
|
||||
sshdConfig = `
|
||||
Port 8022
|
||||
|
||||
# Allow reverse tunnels
|
||||
GatewayPorts yes
|
||||
AllowTcpForwarding yes
|
||||
|
||||
# Auth
|
||||
PermitRootLogin yes
|
||||
PasswordAuthentication no
|
||||
PubkeyAuthentication yes
|
||||
AuthorizedKeysFile ` + keysFilePath
|
||||
)
|
||||
|
||||
func connectClusterToDevcontrol(ctx context.Context, logger *zap.SugaredLogger, cl client.WithWatch, restConfig *rest.Config, privKey ed25519.PrivateKey, pubKey []byte) (clusterIP string, _ error) {
|
||||
logger.Info("Setting up SSH reverse tunnel from cluster to devcontrol...")
|
||||
var err error
|
||||
if clusterIP, err = applySSHResources(ctx, cl, tailscaleroot.AlpineDockerTag, pubKey); err != nil {
|
||||
return "", fmt.Errorf("failed to apply ssh-server resources: %w", err)
|
||||
}
|
||||
sshPodName, err := waitForPodReady(ctx, logger, cl, ns, client.MatchingLabels{"app": "ssh-server"})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("ssh-server Pod not ready: %w", err)
|
||||
}
|
||||
if err := forwardLocalPortToPod(ctx, logger, restConfig, ns, sshPodName, 8022); err != nil {
|
||||
return "", fmt.Errorf("failed to set up port forwarding to ssh-server: %w", err)
|
||||
}
|
||||
if err := reverseTunnel(ctx, logger, privKey, fmt.Sprintf("localhost:%d", 8022), 31544, "localhost:31544"); err != nil {
|
||||
return "", fmt.Errorf("failed to set up reverse tunnel: %w", err)
|
||||
}
|
||||
|
||||
return clusterIP, nil
|
||||
}
|
||||
|
||||
func reverseTunnel(ctx context.Context, logger *zap.SugaredLogger, privateKey ed25519.PrivateKey, sshHost string, remotePort uint16, fwdTo string) error {
|
||||
signer, err := ssh.NewSignerFromKey(privateKey)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create signer: %w", err)
|
||||
}
|
||||
config := &ssh.ClientConfig{
|
||||
User: "root",
|
||||
Auth: []ssh.AuthMethod{
|
||||
ssh.PublicKeys(signer),
|
||||
},
|
||||
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
|
||||
Timeout: 30 * time.Second,
|
||||
}
|
||||
|
||||
conn, err := ssh.Dial("tcp", sshHost, config)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to connect to SSH server: %w", err)
|
||||
}
|
||||
logger.Infof("Connected to SSH server at %s\n", sshHost)
|
||||
|
||||
go func() {
|
||||
defer conn.Close()
|
||||
|
||||
// Start listening on remote port.
|
||||
remoteAddr := fmt.Sprintf("localhost:%d", remotePort)
|
||||
remoteLn, err := conn.Listen("tcp", remoteAddr)
|
||||
if err != nil {
|
||||
logger.Infof("Failed to listen on remote port %d: %v", remotePort, err)
|
||||
return
|
||||
}
|
||||
defer remoteLn.Close()
|
||||
logger.Infof("Reverse tunnel ready on remote addr %s -> local addr %s", remoteAddr, fwdTo)
|
||||
|
||||
for {
|
||||
remoteConn, err := remoteLn.Accept()
|
||||
if err != nil {
|
||||
logger.Infof("Failed to accept remote connection: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
go handleConnection(ctx, logger, remoteConn, fwdTo)
|
||||
}
|
||||
}()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func handleConnection(ctx context.Context, logger *zap.SugaredLogger, remoteConn net.Conn, fwdTo string) {
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
remoteConn.Close()
|
||||
}()
|
||||
|
||||
var d net.Dialer
|
||||
localConn, err := d.DialContext(ctx, "tcp", fwdTo)
|
||||
if err != nil {
|
||||
logger.Infof("Failed to connect to local service %s: %v", fwdTo, err)
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
localConn.Close()
|
||||
}()
|
||||
|
||||
go func() {
|
||||
if _, err := io.Copy(localConn, remoteConn); err != nil {
|
||||
logger.Infof("Error copying remote->local: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
go func() {
|
||||
if _, err := io.Copy(remoteConn, localConn); err != nil {
|
||||
logger.Infof("Error copying local->remote: %v", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func readOrGenerateSSHKey(tmp string) (ed25519.PrivateKey, []byte, error) {
|
||||
var privateKey ed25519.PrivateKey
|
||||
privateKeyPath := filepath.Join(tmp, "id_ed25519")
|
||||
b, err := os.ReadFile(privateKeyPath)
|
||||
switch {
|
||||
case os.IsNotExist(err):
|
||||
_, privateKey, err = ed25519.GenerateKey(rand.Reader)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to generate key: %w", err)
|
||||
}
|
||||
privKeyPEM, err := ssh.MarshalPrivateKey(privateKey, "")
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to marshal SSH private key: %w", err)
|
||||
}
|
||||
f, err := os.OpenFile(privateKeyPath, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to open SSH private key file: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
if err := pem.Encode(f, privKeyPEM); err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to write SSH private key: %w", err)
|
||||
}
|
||||
case err != nil:
|
||||
return nil, nil, fmt.Errorf("failed to read SSH private key: %w", err)
|
||||
default:
|
||||
pKey, err := ssh.ParseRawPrivateKey(b)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to parse SSH private key: %w", err)
|
||||
}
|
||||
pKeyPointer, ok := pKey.(*ed25519.PrivateKey)
|
||||
if !ok {
|
||||
return nil, nil, fmt.Errorf("SSH private key is not ed25519: %T", pKey)
|
||||
}
|
||||
privateKey = *pKeyPointer
|
||||
}
|
||||
|
||||
sshPublicKey, err := ssh.NewPublicKey(privateKey.Public())
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to create SSH public key: %w", err)
|
||||
}
|
||||
|
||||
return privateKey, ssh.MarshalAuthorizedKey(sshPublicKey), nil
|
||||
}
|
||||
|
||||
func applySSHResources(ctx context.Context, cl client.Client, alpineTag string, pubKey []byte) (string, error) {
|
||||
owner := client.FieldOwner("k8s-test")
|
||||
|
||||
if err := cl.Patch(ctx, sshDeployment(alpineTag, pubKey), client.Apply, owner); err != nil {
|
||||
return "", fmt.Errorf("failed to apply ssh-server Deployment: %w", err)
|
||||
}
|
||||
if err := cl.Patch(ctx, sshConfigMap(pubKey), client.Apply, owner); err != nil {
|
||||
return "", fmt.Errorf("failed to apply ssh-server ConfigMap: %w", err)
|
||||
}
|
||||
svc := sshService()
|
||||
if err := cl.Patch(ctx, svc, client.Apply, owner); err != nil {
|
||||
return "", fmt.Errorf("failed to apply ssh-server Service: %w", err)
|
||||
}
|
||||
|
||||
return svc.Spec.ClusterIP, nil
|
||||
}
|
||||
|
||||
func cleanupSSHResources(ctx context.Context, cl client.Client) error {
|
||||
noGrace := &client.DeleteOptions{
|
||||
GracePeriodSeconds: ptr.To[int64](0),
|
||||
}
|
||||
if err := cl.Delete(ctx, sshDeployment("", nil), noGrace); err != nil {
|
||||
return fmt.Errorf("failed to delete ssh-server Deployment: %w", err)
|
||||
}
|
||||
if err := cl.Delete(ctx, sshConfigMap(nil), noGrace); err != nil {
|
||||
return fmt.Errorf("failed to delete ssh-server ConfigMap: %w", err)
|
||||
}
|
||||
if err := cl.Delete(ctx, sshService(), noGrace); err != nil {
|
||||
return fmt.Errorf("failed to delete control Service: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func sshDeployment(tag string, pubKey []byte) *appsv1.Deployment {
|
||||
return &appsv1.Deployment{
|
||||
TypeMeta: metav1.TypeMeta{
|
||||
Kind: "Deployment",
|
||||
APIVersion: "apps/v1",
|
||||
},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "ssh-server",
|
||||
Namespace: ns,
|
||||
},
|
||||
Spec: appsv1.DeploymentSpec{
|
||||
Replicas: ptr.To[int32](1),
|
||||
Selector: &metav1.LabelSelector{
|
||||
MatchLabels: map[string]string{
|
||||
"app": "ssh-server",
|
||||
},
|
||||
},
|
||||
Template: corev1.PodTemplateSpec{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Labels: map[string]string{
|
||||
"app": "ssh-server",
|
||||
},
|
||||
Annotations: map[string]string{
|
||||
"pubkey": hex.EncodeToString(pubKey), // Ensure new key triggers rollout.
|
||||
},
|
||||
},
|
||||
Spec: corev1.PodSpec{
|
||||
Containers: []corev1.Container{
|
||||
{
|
||||
Name: "ssh-server",
|
||||
Image: fmt.Sprintf("alpine:%s", tag),
|
||||
Command: []string{
|
||||
"sh", "-c",
|
||||
"apk add openssh-server; ssh-keygen -A; /usr/sbin/sshd -D -e",
|
||||
},
|
||||
Ports: []corev1.ContainerPort{
|
||||
{
|
||||
Name: "ctrl-port-fwd",
|
||||
ContainerPort: 31544,
|
||||
Protocol: corev1.ProtocolTCP,
|
||||
},
|
||||
{
|
||||
Name: "ssh",
|
||||
ContainerPort: 8022,
|
||||
Protocol: corev1.ProtocolTCP,
|
||||
},
|
||||
},
|
||||
ReadinessProbe: &corev1.Probe{
|
||||
ProbeHandler: corev1.ProbeHandler{
|
||||
TCPSocket: &corev1.TCPSocketAction{
|
||||
Port: intstr.FromInt(8022),
|
||||
},
|
||||
},
|
||||
InitialDelaySeconds: 1,
|
||||
PeriodSeconds: 1,
|
||||
},
|
||||
VolumeMounts: []corev1.VolumeMount{
|
||||
{
|
||||
Name: "sshd-config",
|
||||
MountPath: "/etc/ssh/sshd_config.d/reverse-tunnel.conf",
|
||||
SubPath: "reverse-tunnel.conf",
|
||||
},
|
||||
{
|
||||
Name: "sshd-config",
|
||||
MountPath: keysFilePath,
|
||||
SubPath: "authorized_keys",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
Volumes: []corev1.Volume{
|
||||
{
|
||||
Name: "sshd-config",
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
ConfigMap: &corev1.ConfigMapVolumeSource{
|
||||
LocalObjectReference: corev1.LocalObjectReference{
|
||||
Name: "ssh-server-config",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func sshConfigMap(pubKey []byte) *corev1.ConfigMap {
|
||||
return &corev1.ConfigMap{
|
||||
TypeMeta: metav1.TypeMeta{
|
||||
Kind: "ConfigMap",
|
||||
APIVersion: "v1",
|
||||
},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "ssh-server-config",
|
||||
Namespace: ns,
|
||||
},
|
||||
Data: map[string]string{
|
||||
"reverse-tunnel.conf": sshdConfig,
|
||||
"authorized_keys": string(pubKey),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func sshService() *corev1.Service {
|
||||
return &corev1.Service{
|
||||
TypeMeta: metav1.TypeMeta{
|
||||
Kind: "Service",
|
||||
APIVersion: "v1",
|
||||
},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "control",
|
||||
Namespace: ns,
|
||||
},
|
||||
Spec: corev1.ServiceSpec{
|
||||
Type: corev1.ServiceTypeClusterIP,
|
||||
Selector: map[string]string{
|
||||
"app": "ssh-server",
|
||||
},
|
||||
Ports: []corev1.ServicePort{
|
||||
{
|
||||
Name: "tunnel",
|
||||
Port: 31544,
|
||||
Protocol: corev1.ProtocolTCP,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue