Add support for kubelet cert

cmd/main.go

@@ -84,6 +84,7 @@ func main() {
 	var secureMetrics bool
 	var enableHTTP2 bool
 	var kubeletAddr, kubeletClientCA string
+	var kubeletServingCSR bool
 	var tlsOpts []func(*tls.Config)
 	flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+
 		"Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.")
@@ -110,6 +111,13 @@ func main() {
 			"serves TLS without client verification, because which CA signs the API server's kubelet "+
 			"client certificate is not portable across distributions; restrict the port with a "+
 			"NetworkPolicy, or set this to your API server's kubelet client CA.")
+	flag.BoolVar(&kubeletServingCSR, "kubelet-serving-csr", false,
+		"Request the kubelet API's serving certificate from the cluster's kubernetes.io/kubelet-serving "+
+			"signer instead of self-signing it. Set this if `kubectl logs` fails with "+
+			"\"certificate signed by unknown authority\", which means the API server runs with "+
+			"--kubelet-certificate-authority. Needs the RBAC in config/rbac/kubelet_serving_role.yaml "+
+			"(it self-approves its own CSR); without it, and on a control plane whose signer is "+
+			"disabled, it logs and falls back to self-signed.")
 	opts := zap.Options{
 		Development: true,
 	}
@@ -270,7 +278,7 @@ func main() {
 	// The kubelet endpoint for `kubectl logs` — one listener shared by every provider's
 	// node, hence built here rather than in setupVirtualNodes. Nil is supported: the
 	// nodes then advertise no address, and logs report NotFound.
-	kubeletSrv := setupKubeletServer(mgr, kubeletAddr, kubeletClientCA)
+	kubeletSrv := setupKubeletServer(mgr, kubeletAddr, kubeletClientCA, kubeletServingCSR)
 
 	// Controller and webhook registration is deferred until the cert exists, so it
 	// runs in a goroutine: the cert cannot be minted until the manager is STARTED
@@ -417,7 +425,7 @@ func setupControllers(mgr ctrl.Manager, blocklist *failover.Blocklist, kubeletSr
 // what the API server dials and nothing substitutes for it: a Service would balance to
 // a non-leader replica, which holds no tracked Pods. Either way only logs degrade, so
 // it is logged loudly and the manager carries on.
-func setupKubeletServer(mgr ctrl.Manager, addr, clientCA string) *vnode.KubeletServer {
+func setupKubeletServer(mgr ctrl.Manager, addr, clientCA string, servingCSR bool) *vnode.KubeletServer {
 	if addr == "" {
 		setupLog.Info("kubelet API disabled by configuration; `kubectl logs` will not work for Nebula pods")
 		return nil
@@ -435,11 +443,27 @@ func setupKubeletServer(mgr ctrl.Manager, addr, clientCA string) *vnode.KubeletS
 		setupLog.Error(err, "unable to set up the kubelet API; `kubectl logs` will not work for Nebula pods")
 		return nil
 	}
+	if servingCSR {
+		// Only reachable because registerProviders ran first; without a provider there is no
+		// virtual node, and nothing to serve logs for. The name is only the CSR's subject —
+		// one issued cert covers every node here, since it is the IP SAN that matters.
+		names := provider.Names()
+		clientset, err := kubernetes.NewForConfig(mgr.GetConfig())
+		switch {
+		case err != nil:
+			setupLog.Error(err, "unable to build a clientset for the kubelet serving CSR; self-signing instead")
+		case len(names) == 0:
+			setupLog.Info("no provider registered; skipping the kubelet serving CSR")
+		default:
+			srv.EnableServingCSR(clientset, vnode.NodeName(names[0]))
+		}
+	}
 	if err := mgr.Add(srv); err != nil {
 		setupLog.Error(err, "unable to add the kubelet API to the manager")
 		return nil
 	}
-	setupLog.Info("kubelet API enabled", "addr", addr, "advertisedIP", podIP, "clientCertRequired", clientCA != "")
+	setupLog.Info("kubelet API enabled", "addr", addr, "advertisedIP", podIP,
+		"clientCertRequired", clientCA != "", "servingCSR", servingCSR)
 	return srv
 }
 

config/manager/kustomization.yaml

@@ -5,4 +5,4 @@ kind: Kustomization
 images:
 - name: controller
   newName: inftyai/nebula-controller
-  newTag: latest
+  newTag: 0819-02

config/manager/manager.yaml

@@ -128,6 +128,11 @@ spec:
         # portable — requiring it would break logs on managed control planes. So
         # anything able to reach this port can read any Nebula pod's logs: restrict it
         # with a NetworkPolicy, or set --kubelet-client-ca to require mTLS.
+        #
+        # The self-signed cert is only accepted by an API server that does not verify it.
+        # If logs fail with "certificate signed by unknown authority", the control plane
+        # sets --kubelet-certificate-authority: add --kubelet-serving-csr and uncomment
+        # kubelet_serving_role.yaml in config/rbac/kustomization.yaml.
         - name: kubelet-api
           containerPort: 10250
           protocol: TCP

config/rbac/kubelet_serving_role.yaml

@@ -0,0 +1,46 @@
+# Opt-in RBAC for --kubelet-serving-csr. Not generated from kubebuilder markers and not
+# applied by default, on purpose: `approve` on the kubelet-serving signer lets the holder
+# obtain a serving certificate for ANY node's kubelet endpoint, so granting it has to be a
+# deliberate act.
+#
+# Needed only where the API server runs with --kubelet-certificate-authority, which makes
+# it verify the certificate the kubelet API presents. Without these rules the manager logs
+# the failure and serves a self-signed certificate, which such an API server rejects — so
+# `kubectl logs` and `kubectl exec` on Nebula pods keep failing.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    app.kubernetes.io/name: nebula
+    app.kubernetes.io/managed-by: kustomize
+  name: kubelet-serving-role
+rules:
+# No update: the request is immutable once created, and a stale one (previous pod IP) is
+# replaced by delete-then-create.
+- apiGroups:
+  - certificates.k8s.io
+  resources:
+  - certificatesigningrequests
+  verbs:
+  - get
+  - create
+  - delete
+# kube-controller-manager auto-approves node CLIENT certificates only, never serving ones,
+# because an approver cannot verify that a requester owns the SANs it asks for. This is the
+# manager asserting that about itself.
+- apiGroups:
+  - certificates.k8s.io
+  resources:
+  - certificatesigningrequests/approval
+  verbs:
+  - update
+# Approval is scoped to one signer: it authorizes nothing about client certificates, which
+# are identities the API server would authenticate.
+- apiGroups:
+  - certificates.k8s.io
+  resources:
+  - signers
+  resourceNames:
+  - kubernetes.io/kubelet-serving
+  verbs:
+  - approve

config/rbac/kubelet_serving_role_binding.yaml

@@ -0,0 +1,17 @@
+# Binds the opt-in kubelet-serving permissions to the manager. Applied together with
+# kubelet_serving_role.yaml — see the rationale there.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    app.kubernetes.io/name: nebula
+    app.kubernetes.io/managed-by: kustomize
+  name: kubelet-serving-rolebinding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kubelet-serving-role
+subjects:
+- kind: ServiceAccount
+  name: controller-manager
+  namespace: system

config/rbac/kustomization.yaml

@@ -9,6 +9,12 @@ resources:
 - role_binding.yaml
 - leader_election_role.yaml
 - leader_election_role_binding.yaml
+# Uncomment these two together with the manager's --kubelet-serving-csr flag, which is
+# needed when the API server runs with --kubelet-certificate-authority. Left out by
+# default because approving on the kubelet-serving signer is a privileged grant — see
+# kubelet_serving_role.yaml.
+#- kubelet_serving_role.yaml
+#- kubelet_serving_role_binding.yaml
 # The following RBAC configurations are used to protect
 # the metrics endpoint with authn/authz. These configurations
 # ensure that only authorized users and service accounts

config/samples/deployment.yaml

@@ -29,7 +29,7 @@ metadata:
   labels:
     app.kubernetes.io/managed-by: nebula
 spec:
-  replicas: 30
+  replicas: 3
   selector:
     matchLabels:
       app: gpu-workload-sample

pkg/vnode/kubelet.go

@@ -38,6 +38,7 @@ import (
 	"github.com/virtual-kubelet/virtual-kubelet/errdefs"
 	vkapi "github.com/virtual-kubelet/virtual-kubelet/node/api"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/client-go/kubernetes"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 )
@@ -48,7 +49,9 @@ import (
 const DefaultKubeletAddr = ":10250"
 
 // certValidity is the lifetime of the self-signed serving cert. Rotation is a process
-// restart — the key never leaves memory, so there is no Secret to keep in step.
+// restart — the key never leaves memory, so there is no Secret to keep in step. A
+// signer-issued cert is different: the cluster picks its lifetime, so it is renewed in
+// place (see rotateServingCert).
 const certValidity = 365 * 24 * time.Hour
 
 // kubeletShutdownGrace is how long a `kubectl logs -f` stream may finish after shutdown
@@ -81,10 +84,10 @@ const (
 // is resolved by asking each registered Handler whether it tracks that Pod — at most one
 // can. Cheaper than a port per provider, and than reading the Pod to learn its node.
 //
-// TLS uses a self-signed in-memory cert, which is what the API server expects: it does
-// not verify a kubelet's serving cert unless --kubelet-certificate-authority is set. The
-// webhook cert rotator cannot help, since it mints for a Service DNS name and this
-// endpoint is dialed by Pod IP.
+// TLS defaults to a self-signed in-memory cert, which is enough for an API server that
+// does not set --kubelet-certificate-authority. Where it does, EnableServingCSR asks the
+// cluster's kubelet-serving signer instead. The webhook cert rotator cannot help either
+// way: it mints for a Service DNS name and this endpoint is dialed by Pod IP.
 //
 // Client certs are verified only when ClientCAPath is set. Off by default because which CA
 // signs the API server's kubelet client cert is not portable (kubeadm uses the cluster CA,
@@ -105,6 +108,18 @@ type KubeletServer struct {
 	// others are refused at the TLS layer. Empty disables verification — see above.
 	clientCAPath string
 
+	// csrClient, when set by EnableServingCSR, asks the cluster's kubelet-serving signer
+	// for the serving cert instead of self-signing it. nil keeps the self-signed default.
+	csrClient   kubernetes.Interface
+	csrNodeName string
+	// csrTimeout overrides csrIssueTimeout, so tests need not wait out the real one.
+	csrTimeout time.Duration
+
+	// certMu guards the served cert, which rotation swaps under live connections.
+	certMu     sync.RWMutex
+	cert       *tls.Certificate
+	certIssued bool
+
 	mu       sync.RWMutex
 	handlers map[string]*Handler
 }
@@ -170,10 +185,13 @@ func (s *KubeletServer) daemonEndpoints() corev1.NodeDaemonEndpoints {
 func (s *KubeletServer) Start(ctx context.Context) error {
 	log := logf.FromContext(ctx).WithName("kubelet-api")
 
-	tlsCfg, err := s.tlsConfig()
+	tlsCfg, err := s.tlsConfig(ctx)
 	if err != nil {
 		return err
 	}
+	if s.csrClient != nil {
+		go s.rotateServingCert(ctx)
+	}
 
 	mux := http.NewServeMux()
 	// Logs and exec are wired; the nil funcs make VK answer NotImplemented on
@@ -211,8 +229,11 @@ func (s *KubeletServer) Start(ctx context.Context) error {
 		}
 	}()
 
+	// signerIssued is the field to read when logs fail with x509 errors: false means the
+	// fallback cert, which only a non-verifying API server accepts.
 	log.Info("serving kubelet api (container logs, exec)",
-		"addr", s.addr, "advertisedIP", s.nodeIP, "clientCertRequired", s.clientCAPath != "")
+		"addr", s.addr, "advertisedIP", s.nodeIP, "clientCertRequired", s.clientCAPath != "",
+		"signerIssued", s.signerIssued())
 	// The cert and key are already in TLSConfig, hence the empty paths.
 	if err := srv.ServeTLS(ln, "", ""); err != nil && !errors.Is(err, http.ErrServerClosed) {
 		return fmt.Errorf("kubelet api: serve: %w", err)
@@ -260,15 +281,20 @@ func (s *KubeletServer) runInContainer(
 	return h.RunInContainer(ctx, namespace, podName, containerName, cmd, attach)
 }
 
-// tlsConfig: a fresh self-signed keypair, plus client verification if a CA is set.
-func (s *KubeletServer) tlsConfig() (*tls.Config, error) {
-	cert, err := selfSignedCert(s.nodeIP)
+// tlsConfig mints the first serving cert — signer-issued or self-signed, see servingCert —
+// and adds client verification if a CA is set.
+//
+// GetCertificate rather than Certificates, so rotation is a field swap that new handshakes
+// pick up without rebuilding the server.
+func (s *KubeletServer) tlsConfig(ctx context.Context) (*tls.Config, error) {
+	cert, issued, err := s.servingCert(ctx)
 	if err != nil {
 		return nil, err
 	}
+	s.setCert(cert, issued)
 	cfg := &tls.Config{
-		Certificates: []tls.Certificate{cert},
-		MinVersion:   tls.VersionTLS12,
+		GetCertificate: func(*tls.ClientHelloInfo) (*tls.Certificate, error) { return s.currentCert(), nil },
+		MinVersion:     tls.VersionTLS12,
 		// http/1.1 only, like a real kubelet: logs need nothing HTTP/2 offers, and this is
 		// the streaming path every kubelet client already exercises.
 		NextProtos: []string{"http/1.1"},

pkg/vnode/kubelet_test.go

@@ -301,7 +301,7 @@ func TestKubeletServer_ClientCAErrorsAreFatal(t *testing.T) {
 		if err != nil {
 			t.Fatalf("NewKubeletServer: %v", err)
 		}
-		if _, err := s.tlsConfig(); err == nil {
+		if _, err := s.tlsConfig(context.Background()); err == nil {
 			t.Fatal("expected an error for a client CA path that does not exist")
 		}
 	})
@@ -315,7 +315,7 @@ func TestKubeletServer_ClientCAErrorsAreFatal(t *testing.T) {
 		if err != nil {
 			t.Fatalf("NewKubeletServer: %v", err)
 		}
-		if _, err := s.tlsConfig(); err == nil {
+		if _, err := s.tlsConfig(context.Background()); err == nil {
 			t.Fatal("expected an error for a client CA file containing no certificates")
 		}
 	})
@@ -327,7 +327,7 @@ func TestKubeletServer_ClientCAErrorsAreFatal(t *testing.T) {
 		if err != nil {
 			t.Fatalf("NewKubeletServer: %v", err)
 		}
-		cfg, err := s.tlsConfig()
+		cfg, err := s.tlsConfig(context.Background())
 		if err != nil {
 			t.Fatalf("tlsConfig: %v", err)
 		}