diff --git a/internal/deployer/crs.go b/internal/deployer/crs.go index 85e6e2b..36ee747 100644 --- a/internal/deployer/crs.go +++ b/internal/deployer/crs.go @@ -17,6 +17,7 @@ import ( "time" "github.com/stackrox/roxie/internal/k8s" + "github.com/stackrox/roxie/internal/logger" ) const ( @@ -111,6 +112,7 @@ func (d *Deployer) centralHTTPClient() (*http.Client, error) { } pool.AddCert(cert) caCertsAdded++ + d.logger.Dimf("CA cert #%d: Subject.CN=%q, Issuer.CN=%q, SubjectKeyId=%x", caCertsAdded, cert.Subject.CommonName, cert.Issuer.CommonName, cert.SubjectKeyId) } d.logger.Infof("Loaded %d CA certificate(s) from %q", caCertsAdded, d.roxCACertFile) tlsConfig.RootCAs = pool @@ -119,7 +121,7 @@ func (d *Deployer) centralHTTPClient() (*http.Client, error) { if err != nil { return nil, fmt.Errorf("parsing central endpoint %q: %w", d.centralEndpoint, err) } - tlsConfig.VerifyPeerCertificate = centralVerifyFunc(host, tlsConfig) + tlsConfig.VerifyPeerCertificate = centralVerifyFunc(d.logger, host, tlsConfig) } return &http.Client{ @@ -187,15 +189,22 @@ func (d *Deployer) isRetryableError(err error) bool { } // centralVerifyFunc returns a custom TLS peer certificate verifier for Central. -// Central's self-signed serving cert uses IP SANs and the internal DNS name -// "central.stackrox" rather than the external hostname roxie connects to (e.g. -// a port-forwarded localhost or LoadBalancer IP). Go's default TLS verification -// rejects the cert because the hostname doesn't match any SAN. We work around -// this by setting InsecureSkipVerify and performing chain verification ourselves: -// first against the actual hostname (which may work if the user added a matching -// SAN), then falling back to "central.stackrox" for certs issued by Central's -// own service CA. -func centralVerifyFunc(hostname string, conf *tls.Config) func([][]byte, [][]*x509.Certificate) error { +// +// Go's default TLS verifier is disabled (InsecureSkipVerify) because roxie +// often connects via port-forward to 127.0.0.1, which never matches any SAN. +// This function performs certificate verification in its place. +// +// Two modes based on connection type: +// +// - Port-forward (hostname is 127.0.0.1): chain verification only. Hostname +// checking is unnecessary because kubectl port-forward provides transport- +// level assurance that we're reaching the correct pod. +// +// - Direct connection (LoadBalancer, service DNS): chain verification plus +// hostname checking. If the hostname doesn't match, falls back to +// "central.stackrox" for certs issued by the StackRox internal CA, since +// those use internal DNS names rather than the external endpoint. +func centralVerifyFunc(log *logger.Logger, hostname string, conf *tls.Config) func([][]byte, [][]*x509.Certificate) error { return func(rawCerts [][]byte, _ [][]*x509.Certificate) error { if len(rawCerts) == 0 { return errors.New("remote peer presented no certificates") @@ -216,28 +225,44 @@ func centralVerifyFunc(hostname string, conf *tls.Config) func([][]byte, [][]*x5 intermediates.AddCert(cert) } - systemVerifyOpts := x509.VerifyOptions{ + log.Dimf("Leaf cert: Subject.CN=%q, Issuer.CN=%q, AuthorityKeyId=%x", + leaf.Subject.CommonName, leaf.Issuer.CommonName, leaf.AuthorityKeyId) + + // Port-forward: chain verification only. + if isLoopback(hostname) { + log.Dim("Port-forward connection: verifying certificate chain only") + _, err := leaf.Verify(x509.VerifyOptions{ + Intermediates: intermediates, + Roots: conf.RootCAs, + }) + return err + } + + // Direct connection: chain + hostname verification. + log.Dimf("Direct connection: verifying against hostname %s", hostname) + verifyOpts := x509.VerifyOptions{ DNSName: hostname, Intermediates: intermediates, Roots: conf.RootCAs, } - - _, systemVerifyErr := leaf.Verify(systemVerifyOpts) - if systemVerifyErr == nil || !isACentralCert(leaf) { - return systemVerifyErr + _, err := leaf.Verify(verifyOpts) + if err == nil { + return nil + } + var hostErr x509.HostnameError + if !errors.As(err, &hostErr) || !isACentralCert(leaf) { + return err } - serviceVerifyOpts := x509.VerifyOptions{ + // Fallback for StackRox service certs: the internal cert uses + // "central.stackrox" as its SAN, not the external endpoint hostname. + log.Dim("Falling back to central.stackrox for StackRox service cert") + _, err = leaf.Verify(x509.VerifyOptions{ DNSName: "central.stackrox", Intermediates: intermediates, Roots: conf.RootCAs, - } - - _, serviceVerifyErr := leaf.Verify(serviceVerifyOpts) - if serviceVerifyErr == nil { - return nil - } - return errors.Join(systemVerifyErr, serviceVerifyErr) + }) + return err } } @@ -252,6 +277,13 @@ func isACentralCert(cert *x509.Certificate) bool { return false } +func isLoopback(hostname string) bool { + if ip := net.ParseIP(hostname); ip != nil { + return ip.IsLoopback() + } + return hostname == "localhost" +} + // applyCRS applies the CRS content to the sensor namespace func (d *Deployer) applyCRS(ctx context.Context, crsContent string) error { d.logger.Info("Applying CRS to sensor namespace") diff --git a/internal/deployer/deploy_via_operator.go b/internal/deployer/deploy_via_operator.go index 2820d11..8358fb7 100644 --- a/internal/deployer/deploy_via_operator.go +++ b/internal/deployer/deploy_via_operator.go @@ -3,10 +3,12 @@ package deployer import ( "bytes" "context" + "crypto/x509" + "encoding/base64" + "encoding/pem" "errors" "fmt" "os" - "os/exec" "strings" "time" @@ -558,27 +560,33 @@ func (d *Deployer) waitForLoadBalancer(ctx context.Context, namespace, serviceNa return "", errors.New("timeout waiting for LoadBalancer to get external address") } -// fetchCentralCACert fetches the Central CA certificate -func (d *Deployer) fetchCentralCACert(ctx context.Context) error { - d.logger.Info("Fetching Central CA certificate...") +// fetchCentralCACerts fetches CA certificates needed to verify Central's TLS cert. +// It always fetches the internal StackRox CA from the central-tls secret, and +// additionally fetches CA certs from the defaultTLSSecret if one is configured +// in the Central CR spec. +func (d *Deployer) fetchCentralCACerts(ctx context.Context) error { + var caPEMs [][]byte - result, err := d.runKubectl(ctx, k8s.KubectlOptions{ - Args: []string{"get", "secret", "central-tls", "-n", d.config.Central.Namespace, "-o", "jsonpath={.data.ca\\.pem}"}, - }) + // Fetch the internal StackRox CA from the central-tls secret. + d.logger.Info("Fetching internal CA certificate from central-tls secret...") + internalCA, err := d.fetchSecretField(ctx, "central-tls", "ca\\.pem") if err != nil { - return fmt.Errorf("failed to get CA cert from secret: %w", err) - } - - caCertBase64 := strings.TrimSpace(result.Stdout) - if caCertBase64 == "" { - return errors.New("CA certificate is empty") + return fmt.Errorf("failed to get CA cert from central-tls secret: %w", err) } + caPEMs = append(caPEMs, internalCA) - decodeCmd := exec.CommandContext(ctx, "base64", "-d") - decodeCmd.Stdin = strings.NewReader(caCertBase64) - caCert, err := decodeCmd.Output() - if err != nil { - return fmt.Errorf("failed to decode CA cert: %w", err) + // If a custom defaultTLSSecret is configured, add its certs to the + // trust pool. This includes the leaf — Central itself does the same + // when building the trust bundle for Sensor. + if secretName := d.defaultTLSSecretName(); secretName != "" { + d.logger.Infof("Fetching custom TLS certificates from secret %s...", secretName) + customCerts, err := d.fetchCustomTLSCerts(ctx, secretName) + if err != nil { + d.logger.Warningf("Could not fetch custom TLS certs from secret %s: %v", secretName, err) + // Try to continue. + } else { + caPEMs = append(caPEMs, customCerts...) + } } caCertFile, err := os.CreateTemp(d.tempDir, "roxie-central-ca-*.pem") @@ -586,20 +594,77 @@ func (d *Deployer) fetchCentralCACert(ctx context.Context) error { return fmt.Errorf("failed to create temp file for CA cert: %w", err) } - d.roxCACertFile = caCertFile.Name() - if _, err := caCertFile.Write(caCert); err != nil { - _ = caCertFile.Close() - _ = os.Remove(d.roxCACertFile) - return fmt.Errorf("failed to write CA cert: %w", err) + fileName := caCertFile.Name() + for _, pemData := range caPEMs { + pemData = append(bytes.TrimRight(pemData, "\n"), '\n') + if _, err := caCertFile.Write(pemData); err != nil { + _ = caCertFile.Close() + _ = os.Remove(fileName) + return fmt.Errorf("failed to write CA cert: %w", err) + } } if err := caCertFile.Close(); err != nil { + _ = os.Remove(fileName) return fmt.Errorf("failed to close CA cert file: %w", err) } - d.logger.Successf("✓ CA certificate saved to: %s", d.roxCACertFile) + d.roxCACertFile = fileName + d.logger.Successf("✓ CA certificates saved to: %s", d.roxCACertFile) return nil } +// defaultTLSSecretName returns the name of the defaultTLSSecret from the Central +// CR spec, or "" if none is configured. +func (d *Deployer) defaultTLSSecretName() string { + name, _, _ := unstructured.NestedString(d.config.Central.Spec, "central", "defaultTLSSecret", "name") + return name +} + +// fetchSecretField fetches a single base64-encoded field from a Kubernetes secret +// in the Central namespace and returns the decoded PEM bytes. +func (d *Deployer) fetchSecretField(ctx context.Context, secretName, jsonpathField string) ([]byte, error) { + result, err := d.runKubectl(ctx, k8s.KubectlOptions{ + Args: []string{"get", "secret", secretName, "-n", d.config.Central.Namespace, "-o", fmt.Sprintf("jsonpath={.data.%s}", jsonpathField)}, + }) + if err != nil { + return nil, fmt.Errorf("kubectl get secret %s: %w", secretName, err) + } + + b64 := strings.TrimSpace(result.Stdout) + if b64 == "" { + return nil, fmt.Errorf("field %s in secret %s is empty", jsonpathField, secretName) + } + + decoded, err := base64.StdEncoding.DecodeString(b64) + if err != nil { + return nil, fmt.Errorf("base64 decode of %s from %s: %w", jsonpathField, secretName, err) + } + return decoded, nil +} + +// fetchCustomTLSCerts fetches the tls.crt field from a Kubernetes TLS secret +// and returns all certificates (leaf, intermediates, root) as PEM blocks. +// All certs are added to the trust pool, matching how Central itself builds +// the trust bundle for Sensor. +func (d *Deployer) fetchCustomTLSCerts(ctx context.Context, secretName string) ([][]byte, error) { + certBundle, err := d.fetchSecretField(ctx, secretName, "tls\\.crt") + if err != nil { + return nil, fmt.Errorf("fetching certificates from secret %s: %w", secretName, err) + } + + var pems [][]byte + for block, rest := pem.Decode(certBundle); block != nil; block, rest = pem.Decode(rest) { + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return nil, fmt.Errorf("parsing certificate from secret %s: %w", secretName, err) + } + d.logger.Infof("Found certificate in %s: Subject.CN=%q, IsCA=%v", + secretName, cert.Subject.CommonName, cert.IsCA) + pems = append(pems, pem.EncodeToMemory(block)) + } + return pems, nil +} + // configureCentralEndpoint configures the central endpoint in the Deployer based on exposure settings. func (d *Deployer) configureCentralEndpoint(ctx context.Context) error { exposure := d.config.Central.GetExposure() @@ -639,8 +704,8 @@ func (d *Deployer) configureCentralEndpoint(ctx context.Context) error { d.centralEndpoint = "central." + d.config.Central.Namespace + ".svc:443" } - if err := d.fetchCentralCACert(ctx); err != nil { - d.logger.Warningf("Could not fetch CA cert: %v", err) + if err := d.fetchCentralCACerts(ctx); err != nil { + d.logger.Warningf("Could not fetch Central CA certs: %v", err) } if env.RunningInteractively { diff --git a/tests/e2e/custom_tls_test.go b/tests/e2e/custom_tls_test.go new file mode 100644 index 0000000..295e9ef --- /dev/null +++ b/tests/e2e/custom_tls_test.go @@ -0,0 +1,161 @@ +//go:build e2e + +package e2e + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "net" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestDeployWithCustomTLSCert(t *testing.T) { + dumpClusterStateOnFailure(t) + + const ( + sharedNamespace = "stackrox" + secretName = "custom-central-tls" + ) + + caCertPEM, certChainPEM, keyPEM := generateCentralTLSCert(t) + + ensureNamespace(t, sharedNamespace) + createTLSSecret(t, sharedNamespace, secretName, certChainPEM, keyPEM) + + envrcFile, err := os.CreateTemp(t.TempDir(), ".envrc.roxie-test-*") + require.NoError(t, err) + envrcPath := envrcFile.Name() + envrcFile.Close() + + t.Log("=== Deploying both components with custom TLS certificate ===") + args := append([]string{ + roxieBinary, "deploy", "--single-namespace", "--early-readiness", "both", + "--exposure=none", "--port-forwarding", + "--set", "central.spec.central.defaultTLSSecret.name=" + secretName, + "--envrc", envrcPath, + }, commonDeployArgs...) + runCommand(t, deployTimeout*2, nil, args...) + + verifyCentralInstalled(t, sharedNamespace) + verifySecuredClusterInstalled(t, sharedNamespace) + + env, err := loadEnvrcFile(envrcPath) + require.NoError(t, err, "Failed to load envrc file") + require.NotEmpty(t, env["ROX_CA_CERT_FILE"], "ROX_CA_CERT_FILE should be set in envrc") + + verifyCACertFileContainsCustomCA(t, env["ROX_CA_CERT_FILE"], caCertPEM) + + t.Log("=== Cleaning up ===") + teardownArgs := []string{roxieBinary, "teardown", "--single-namespace", "--skip-user-config", "both"} + runCommand(t, teardownTimeout, env, teardownArgs...) + + verifyCentralNotInstalled(t, sharedNamespace) + verifySecuredClusterNotInstalled(t, sharedNamespace) +} + +// generateCentralTLSCert creates a self-signed CA and a leaf certificate for +// Central's HTTPS endpoint. It returns PEM-encoded bytes: the CA certificate, +// the full certificate chain (leaf + CA), and the leaf private key. +func generateCentralTLSCert(t *testing.T) (caCertPEM, certChainPEM, keyPEM []byte) { + t.Helper() + + caKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + + caTemplate := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{ + CommonName: "roxie-test-ca", + }, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign, + BasicConstraintsValid: true, + IsCA: true, + } + + caCertDER, err := x509.CreateCertificate(rand.Reader, caTemplate, caTemplate, &caKey.PublicKey, caKey) + require.NoError(t, err) + + caCert, err := x509.ParseCertificate(caCertDER) + require.NoError(t, err) + + leafKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + + leafTemplate := &x509.Certificate{ + SerialNumber: big.NewInt(2), + Subject: pkix.Name{ + CommonName: "custom-central.arthur.dent", + }, + DNSNames: []string{ + "custom-central.arthur.dent", + }, + IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1)}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(24 * time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + } + + leafCertDER, err := x509.CreateCertificate(rand.Reader, leafTemplate, caCert, &leafKey.PublicKey, caKey) + require.NoError(t, err) + + caCertPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: caCertDER}) + leafCertPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: leafCertDER}) + + leafKeyDER, err := x509.MarshalECPrivateKey(leafKey) + require.NoError(t, err) + keyPEM = pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: leafKeyDER}) + + certChainPEM = append(leafCertPEM, caCertPEM...) + return caCertPEM, certChainPEM, keyPEM +} + +func ensureNamespace(t *testing.T, name string) { + t.Helper() + cmd := exec.Command("kubectl", "create", "namespace", name) + output, err := cmd.CombinedOutput() + if err != nil && !strings.Contains(string(output), "already exists") { + t.Fatalf("Failed to create namespace %s: %s", name, string(output)) + } +} + +func createTLSSecret(t *testing.T, namespace, name string, certPEM, keyPEM []byte) { + t.Helper() + dir := t.TempDir() + certFile := filepath.Join(dir, "tls.crt") + keyFile := filepath.Join(dir, "tls.key") + require.NoError(t, os.WriteFile(certFile, certPEM, 0600)) + require.NoError(t, os.WriteFile(keyFile, keyPEM, 0600)) + + // Delete any leftover secret from a previous run. + del := exec.Command("kubectl", "-n", namespace, "delete", "secret", name, "--ignore-not-found") + del.CombinedOutput() + + cmd := exec.Command("kubectl", "-n", namespace, "create", "secret", "tls", name, + "--cert="+certFile, "--key="+keyFile) + output, err := cmd.CombinedOutput() + require.NoError(t, err, "Failed to create TLS secret: %s", string(output)) +} + +func verifyCACertFileContainsCustomCA(t *testing.T, caCertFile string, expectedCAPEM []byte) { + t.Helper() + contents, err := os.ReadFile(caCertFile) + require.NoError(t, err, "Failed to read CA cert file %s", caCertFile) + require.Contains(t, string(contents), string(expectedCAPEM), + "CA cert file should contain the custom CA certificate") +}