feat: support egress policy

api/v1alpha1/groupversion_info.go

@@ -119,6 +119,15 @@ const (
 	// handler's built-in default.
 	BlocklistTTLAnnotation = "nebula.inftyai.com/blocklist-ttl"
 
+	// EgressAnnotation carries the pool's EgressPolicy.Mode and EgressTargetsAnnotation its
+	// Targets, comma-separated (no CIDR or hostname contains one). Same flow and
+	// reason as BlocklistTTLAnnotation: pool policy the VK handler must honour but never
+	// sees the pool to read.
+	//
+	// Absent means Open. Mode is authoritative — without it the target list is ignored.
+	EgressAnnotation        = "nebula.inftyai.com/egress"
+	EgressTargetsAnnotation = "nebula.inftyai.com/egress-targets"
+
 	// EndpointAnnotation carries the reachable address of the external instance (a DNS
 	// name, an IP, or a URL, in the provider's own form). It is the only way to reach
 	// the workload, and PodIP cannot hold it — the API server validates PodIP as a

api/v1alpha1/nodepool_types.go

@@ -69,6 +69,65 @@ type NodePoolSpec struct {
 	// RunPod reports no capacity) is temporarily excluded and re-tried.
 	// +optional
 	Failover *FailoverPolicy `json:"failover,omitempty"`
+
+	// Egress restricts OUTBOUND connections from this pool's workloads; omitted means
+	// Open. Inbound is never affected — a Blocked sandbox still serves its consumer's
+	// tunnel and connect token, it just cannot call out.
+	//
+	// +optional
+	Egress *EgressPolicy `json:"egress,omitempty"`
+}
+
+// EgressPolicy is a pool's outbound network policy. The rules below keep Blocked and
+// Allowlist disjoint, so "no egress" has one spelling instead of three.
+// +kubebuilder:validation:XValidation:rule="self.mode == 'Allowlist' || !has(self.targets)",message="targets is only valid with mode Allowlist"
+// +kubebuilder:validation:XValidation:rule="self.mode != 'Allowlist' || (has(self.targets) && self.targets.size() > 0)",message="mode Allowlist requires at least one target; use mode Blocked to permit nothing"
+type EgressPolicy struct {
+	// Mode is required once spec.egress is set, so a half-written policy is rejected
+	// rather than defaulted into a weaker one.
+	Mode EgressMode `json:"mode"`
+
+	// Targets is what mode Allowlist permits: CIDRs, bare IPs and domain names with an
+	// optional wildcard, mixed in one list, e.g. ["10.0.0.0/8", "*.huggingface.co"].
+	// +optional
+	// +kubebuilder:validation:MaxItems=64
+	// +kubebuilder:validation:items:MaxLength=253
+	Targets []string `json:"targets,omitempty"`
+}
+
+// EgressMode is how a pool treats outbound traffic. No mode restricts inbound.
+// +kubebuilder:validation:Enum=Open;Blocked;Allowlist
+type EgressMode string
+
+const (
+	// EgressOpen places no restriction, and is what an omitted spec.egress means.
+	EgressOpen EgressMode = "Open"
+	// EgressBlocked permits no outbound connection at all.
+	EgressBlocked EgressMode = "Blocked"
+	// EgressAllowlist permits EgressPolicy.Targets and nothing else.
+	EgressAllowlist EgressMode = "Allowlist"
+)
+
+// ModeOrOpen reads a nil policy as Open, since an omitted spec.egress and an explicit
+// Open are the same thing and no caller should nil-check for it.
+func (p *EgressPolicy) ModeOrOpen() EgressMode {
+	if p == nil || p.Mode == "" {
+		return EgressOpen
+	}
+	return p.Mode
+}
+
+// GetTargets reads Targets off a possibly-nil policy, for the same reason as ModeOrOpen.
+func (p *EgressPolicy) GetTargets() []string {
+	if p == nil {
+		return nil
+	}
+	return p.Targets
+}
+
+// RestrictsEgress reports whether the policy needs a provider to enforce anything.
+func (p *EgressPolicy) RestrictsEgress() bool {
+	return p.ModeOrOpen() != EgressOpen
 }
 
 // ProviderSpec is one provider's entry in a pool: which provider, and the

config/crd/bases/nebula.inftyai.com_nodepools.yaml

@@ -94,6 +94,40 @@ spec:
                   type: string
                 minItems: 1
                 type: array
+              egress:
+                description: |-
+                  Egress restricts OUTBOUND connections from this pool's workloads; omitted means
+                  Open. Inbound is never affected — a Blocked sandbox still serves its consumer's
+                  tunnel and connect token, it just cannot call out.
+                properties:
+                  mode:
+                    description: |-
+                      Mode is required once spec.egress is set, so a half-written policy is rejected
+                      rather than defaulted into a weaker one.
+                    enum:
+                    - Open
+                    - Blocked
+                    - Allowlist
+                    type: string
+                  targets:
+                    description: |-
+                      Targets is what mode Allowlist permits: CIDRs, bare IPs and domain names with an
+                      optional wildcard, mixed in one list, e.g. ["10.0.0.0/8", "*.huggingface.co"].
+                    items:
+                      maxLength: 253
+                      type: string
+                    maxItems: 64
+                    type: array
+                required:
+                - mode
+                type: object
+                x-kubernetes-validations:
+                - message: targets is only valid with mode Allowlist
+                  rule: self.mode == 'Allowlist' || !has(self.targets)
+                - message: mode Allowlist requires at least one target; use mode Blocked
+                    to permit nothing
+                  rule: self.mode != 'Allowlist' || (has(self.targets) && self.targets.size()
+                    > 0)
               failover:
                 description: |-
                   Failover controls how a provider that fails at provision time (e.g.

config/samples/nodepool.yaml

@@ -28,3 +28,19 @@ spec:
   strategy: Ordered
   failover:
     blocklistTTL: 30s
+  # Outbound network policy. Omitted (as here) means Open. Inbound is never restricted, so a
+  # sandbox stays reachable through its connect URL and token under every mode — it just
+  # cannot call out.
+  #
+  # Blocked permits nothing:
+  # egress:
+  #   mode: Blocked
+  #
+  # Allowlist permits only what it lists — CIDRs, bare IPs and domain names (wildcards
+  # allowed) in ONE list; the adapter sorts them by kind. Use this when the workload pulls
+  # model weights at startup, which hangs under Blocked:
+  # egress:
+  #   mode: Allowlist
+  #   targets:
+  #   - "*.huggingface.co"
+  #   - 10.0.0.0/8

internal/controller/nodeclaim_controller_test.go

@@ -48,13 +48,14 @@ type fakeProvider struct {
 	terminateErr error               // if set, Terminate fails
 	gpus         []string            // accelerators MapAccelerator offers; nil = offer any
 	spot         bool                // Capabilities().SupportsSpot (placement skips Spot without it)
+	egress       bool                // Capabilities().SupportsEgressPolicy (placement skips restricted pools without it)
 	// expandRegions overrides ExpandRegions; nil = pass the declaration through.
 	expandRegions func([]string) []string
 }
 
 func (f *fakeProvider) Name() string { return f.name }
 func (f *fakeProvider) Capabilities() provider.Capabilities {
-	return provider.Capabilities{SupportsSpot: f.spot}
+	return provider.Capabilities{SupportsSpot: f.spot, SupportsEgressPolicy: f.egress}
 }
 func (f *fakeProvider) Provision(context.Context, *corev1.Pod, provider.ProvisionRequest) (provider.ProvisionResult, error) {
 	return provider.ProvisionResult{}, nil

internal/controller/pod_placement_controller_test.go

@@ -597,6 +597,83 @@ func TestPlacement_SpotOnlyPoolStaysGatedOnOnDemandOnlyProvider(t *testing.T) {
 	}
 }
 
+func TestPlacement_StampsEgressAnnotationsFromPool(t *testing.T) {
+	// The VK handler never sees the pool, so the policy has to ride the Pod. Both
+	// annotations must land: the mode alone is authoritative, and without the targets
+	// an Allowlist pool would reach the adapter as "permit nothing" — silently Blocked.
+	pod := gatedPod("p1", "default", "uid-1", "pool-a", "H100")
+	pool := poolWith("pool-a", []nebulav1alpha1.CapacityType{nebulav1alpha1.CapacityOnDemand}, provider.ProviderModal)
+	pool.Spec.Egress = &nebulav1alpha1.EgressPolicy{
+		Mode:    nebulav1alpha1.EgressAllowlist,
+		Targets: []string{"10.0.0.0/8", "*.huggingface.co"},
+	}
+	prov := &fakeProvider{name: provider.ProviderModal, gpus: []string{"H100"}, egress: true}
+	r, c := newPlacementReconciler(t, []client.Object{pod, pool}, prov)
+
+	reconcilePod(t, r, "default", "p1")
+
+	got := getPod(t, c, "default", "p1")
+	if hasGateNamed(got) {
+		t.Fatal("expected the Pod placed on a provider that enforces egress")
+	}
+	if v := got.Annotations[nebulav1alpha1.EgressAnnotation]; v != string(nebulav1alpha1.EgressAllowlist) {
+		t.Errorf("egress annotation = %q, want %q", v, nebulav1alpha1.EgressAllowlist)
+	}
+	if v := got.Annotations[nebulav1alpha1.EgressTargetsAnnotation]; v != "10.0.0.0/8,*.huggingface.co" {
+		t.Errorf("egress-targets annotation = %q, want the comma-joined list", v)
+	}
+}
+
+func TestPlacement_OpenPoolStampsNoEgressAnnotation(t *testing.T) {
+	// Absence IS Open (see EgressAnnotation), so an unrestricted pool must leave the Pod
+	// clean rather than stamping "Open" — otherwise every Pod in the cluster grows an
+	// annotation that means nothing.
+	pod := gatedPod("p1", "default", "uid-1", "pool-a", "H100")
+	pool := poolWith("pool-a", []nebulav1alpha1.CapacityType{nebulav1alpha1.CapacityOnDemand}, provider.ProviderModal)
+	// No egress on the pool at all, which is the common case.
+	prov := &fakeProvider{name: provider.ProviderModal, gpus: []string{"H100"}} // egress: false
+	r, c := newPlacementReconciler(t, []client.Object{pod, pool}, prov)
+
+	reconcilePod(t, r, "default", "p1")
+
+	got := getPod(t, c, "default", "p1")
+	if hasGateNamed(got) {
+		t.Fatal("expected an Open pool to place on a provider without egress support")
+	}
+	if _, ok := got.Annotations[nebulav1alpha1.EgressAnnotation]; ok {
+		t.Errorf("expected no egress annotation for an Open pool, got %q",
+			got.Annotations[nebulav1alpha1.EgressAnnotation])
+	}
+}
+
+func TestPlacement_RestrictedPoolStaysGatedWhenNoProviderEnforcesEgress(t *testing.T) {
+	// The provider cannot enforce the policy, so there is no candidate and the Pod stays
+	// visibly unplaceable. This is the whole point of the capability gate: placing it
+	// anyway would provision a workload with open internet access under a pool that says
+	// Blocked, with nothing to reveal the substitution. No requeue hint — only a pool edit
+	// or a provider gaining support fixes it, and both emit their own event.
+	pod := gatedPod("p1", "default", "uid-1", "pool-a", "H100")
+	pool := poolWith("pool-a", []nebulav1alpha1.CapacityType{nebulav1alpha1.CapacityOnDemand}, provider.ProviderModal)
+	pool.Spec.Egress = &nebulav1alpha1.EgressPolicy{Mode: nebulav1alpha1.EgressBlocked}
+	prov := &fakeProvider{name: provider.ProviderModal, gpus: []string{"H100"}} // egress: false
+	r, c := newPlacementReconciler(t, []client.Object{pod, pool}, prov)
+
+	res, err := r.Reconcile(context.Background(), reconcile.Request{
+		NamespacedName: types.NamespacedName{Namespace: "default", Name: "p1"},
+	})
+	if err != nil {
+		t.Fatalf("reconcile: %v", err)
+	}
+	if res.RequeueAfter != 0 {
+		t.Fatalf("expected no requeue hint for a capability gap, got %v", res.RequeueAfter)
+	}
+
+	got := getPod(t, c, "default", "p1")
+	if !hasGateNamed(got) {
+		t.Fatal("expected the Pod to stay gated when no provider can enforce the egress policy")
+	}
+}
+
 func TestPlacement_AllCandidatesBlockedRequeuesForBlockExpiry(t *testing.T) {
 	// Every (tier, provider, region) candidate is blocked (DenyAll on the provider),
 	// but the candidates are servable — the block is a transient failover exclusion.

internal/controller/pod_placement_helpers.go

@@ -19,6 +19,7 @@ package controller
 import (
 	"context"
 	"hash/fnv"
+	"strings"
 	"time"
 
 	corev1 "k8s.io/api/core/v1"
@@ -116,12 +117,18 @@ func (r *PodPlacementReconciler) selectPlacement(ctx context.Context, pod *corev
 					"provider", ref.Name, "capacityType", tier)
 				continue // unregistered; NodePool status surfaces this separately
 			}
-			if !servesCapacity(prov, tier) {
+			if !servesCapacityTier(prov, tier) {
 				metrics.RecordCandidateSkip(ref.Name, tier, "", metrics.SkipCapacityUnsupported)
 				log.V(1).Info("skipping candidate: provider does not offer the capacity tier",
 					"provider", ref.Name, "capacityType", tier)
 				continue
 			}
+			if !servesEgress(prov, pool.Spec.Egress) {
+				metrics.RecordCandidateSkip(ref.Name, tier, "", metrics.SkipEgressUnsupported)
+				log.V(1).Info("skipping candidate: provider cannot enforce the pool's egress policy",
+					"provider", ref.Name, "egressMode", pool.Spec.Egress.ModeOrOpen())
+				continue
+			}
 			// A CPU-only Pod (no accelerator) matches any provider; an accelerator
 			// Pod only matches a provider whose catalog serves that (type, count).
 			// MapAccelerator is consulted only for that servability check — the block
@@ -205,7 +212,7 @@ func capacityTiers(pool *nebulav1alpha1.NodePool) []nebulav1alpha1.CapacityType
 	return pool.Spec.CapacityTypes
 }
 
-// servesCapacity reports whether prov can deliver the candidate's capacity tier. Only Spot
+// servesCapacityTier reports whether prov can deliver the candidate's capacity tier. Only Spot
 // is ever refused: an OnDemand-only provider (Modal) has no interruptible tier, so placing a
 // Spot candidate there would stamp CapacityType=Spot on the Pod, hand it to an adapter that
 // drops the field, and bill OnDemand rates for capacity the user asked to be cheap — with no
@@ -214,13 +221,27 @@ func capacityTiers(pool *nebulav1alpha1.NodePool) []nebulav1alpha1.CapacityType
 // leaves the Pod visibly unplaceable rather than quietly overcharged.
 //
 // The empty tier is "the provider's default", which every provider serves, so it passes.
-func servesCapacity(prov provider.Provider, tier nebulav1alpha1.CapacityType) bool {
+func servesCapacityTier(prov provider.Provider, tier nebulav1alpha1.CapacityType) bool {
 	if tier != nebulav1alpha1.CapacitySpot {
 		return true
 	}
 	return prov.Capabilities().SupportsSpot
 }
 
+// servesEgress reports whether prov can enforce the pool's egress policy. Open needs no
+// enforcement, so every provider serves it; anything else needs SupportsEgressPolicy.
+//
+// Same reasoning as servesCapacity, and load-bearing for a different reason: a provider
+// that drops the field would put the workload on the open internet while the pool claims
+// containment. Skipping makes that visible — an AWS-only pool asking for Blocked leaves the
+// Pod unplaceable instead of silently unprotected.
+func servesEgress(prov provider.Provider, policy *nebulav1alpha1.EgressPolicy) bool {
+	if !policy.RestrictsEgress() {
+		return true
+	}
+	return prov.Capabilities().SupportsEgressPolicy
+}
+
 // regionsFor is the inner axis for one provider ref: the concrete regions to try, in
 // expansion order. The pool's declaration is a CONSTRAINT, not a list of regions —
 // it may be omitted (unconstrained), name a geography group ("us"), or name regions
@@ -358,6 +379,15 @@ func (r *PodPlacementReconciler) place(ctx context.Context, pod *corev1.Pod, poo
 	if pool.Spec.Failover != nil && pool.Spec.Failover.BlocklistTTL.Duration > 0 {
 		setAnnotation(pod, nebulav1alpha1.BlocklistTTLAnnotation, pool.Spec.Failover.BlocklistTTL.Duration.String())
 	}
+	// Same for the egress policy. Stamped only when it restricts something: the absence of
+	// the annotation IS Open, and selectPlacement has already ensured p.provider can enforce
+	// whatever is written here.
+	if pool.Spec.Egress.RestrictsEgress() {
+		setAnnotation(pod, nebulav1alpha1.EgressAnnotation, string(pool.Spec.Egress.ModeOrOpen()))
+		if len(pool.Spec.Egress.Targets) > 0 {
+			setAnnotation(pod, nebulav1alpha1.EgressTargetsAnnotation, strings.Join(pool.Spec.Egress.Targets, ","))
+		}
+	}
 
 	// Remove our gate, releasing the Pod to the scheduler. Preserve any other
 	// gates a different controller may hold.

pkg/metrics/placement.go

@@ -51,6 +51,7 @@ const (
 	SkipProviderUnregistered   = "provider_unregistered"
 	SkipCapacityUnsupported    = "capacity_type_unsupported"
 	SkipAcceleratorUnsupported = "accelerator_unsupported"
+	SkipEgressUnsupported      = "egress_policy_unsupported"
 	SkipBlocked                = "blocked"
 )
 

pkg/provider/aws/aws.go

@@ -390,12 +390,16 @@ func (p *Provider) clientFor(ctx context.Context, region string) (Client, error)
 // trait is set the way it is.
 func (p *Provider) Capabilities() provider.Capabilities {
 	return provider.Capabilities{
-		SupportsStop:     true,             // EC2 instances stop/start
-		SupportsSpot:     true,             // real interruptible tier
-		NativeTags:       true,             // EC2 tags carry identity
-		PreemptionNotice: preemptionNotice, // Spot 2-minute warning
-		PollInterval:     spotPollInterval, // Spot reclaims are abrupt; poll faster than default
-		ProvisionTimeout: provisionTimeout, // caps the per-zone capacity failover loop
+		SupportsStop: true, // EC2 instances stop/start
+		SupportsSpot: true, // real interruptible tier
+		// Instances launch into the default VPC, whose security group allows all egress and
+		// which routes to an internet gateway. Enforcing a pool's policy means managing SG
+		// egress rules (and no NAT for the Blocked case), so it is unsupported until then.
+		SupportsEgressPolicy: false,
+		NativeTags:           true,             // EC2 tags carry identity
+		PreemptionNotice:     preemptionNotice, // Spot 2-minute warning
+		PollInterval:         spotPollInterval, // Spot reclaims are abrupt; poll faster than default
+		ProvisionTimeout:     provisionTimeout, // caps the per-zone capacity failover loop
 	}
 }