diff --git a/rust/operator-binary/src/controller/build/mod.rs b/rust/operator-binary/src/controller/build/mod.rs index 41875e98..1c5dac7a 100644 --- a/rust/operator-binary/src/controller/build/mod.rs +++ b/rust/operator-binary/src/controller/build/mod.rs @@ -3,12 +3,119 @@ use std::str::FromStr; -use stackable_operator::v2::types::operator::RoleGroupName; +use snafu::{ResultExt, Snafu}; +use stackable_operator::{ + utils::cluster_info::KubernetesClusterInfo, v2::types::operator::RoleGroupName, +}; + +use crate::{ + controller::{ + KubernetesResources, ValidatedCluster, + build::resource::{ + config_map::{self, build_rolegroup_config_map}, + discovery::{self, build_discovery_config_map}, + pdb::build_pdb, + service::{build_rolegroup_metrics_service, build_rolegroup_service}, + statefulset::{self, build_rolegroup_statefulset}, + }, + }, + crd::HbaseRole, +}; // Placeholder role-group name used for the recommended labels of the role-level discovery // `ConfigMap` (which is not tied to a single role group). stackable_operator::constant!(pub(crate) PLACEHOLDER_DISCOVERY_ROLE_GROUP: RoleGroupName = "discovery"); +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display("failed to build ConfigMap for role {hbase_role} role group {role_group}"))] + ConfigMap { + source: config_map::Error, + hbase_role: HbaseRole, + role_group: RoleGroupName, + }, + + #[snafu(display("failed to build StatefulSet for role {hbase_role} role group {role_group}"))] + StatefulSet { + source: statefulset::Error, + hbase_role: HbaseRole, + role_group: RoleGroupName, + }, + + #[snafu(display("failed to build discovery ConfigMap"))] + Discovery { source: discovery::Error }, +} + +/// Builds every Kubernetes resource for the given validated cluster. +/// +/// Does not need a Kubernetes client: every reference to another Kubernetes resource is already +/// dereferenced and validated by this point, so the errors returned here are resource-assembly +/// failures only. `cluster_info` is static cluster metadata (not a client call), and +/// `service_account_name` is the name of the RBAC `ServiceAccount` the role-group Pods run under +/// (RBAC resources are built and applied separately, in the reconcile step). +pub fn build( + cluster: &ValidatedCluster, + cluster_info: &KubernetesClusterInfo, + service_account_name: &str, +) -> Result { + let mut stateful_sets = vec![]; + let mut services = vec![]; + let mut config_maps = vec![]; + let mut pod_disruption_budgets = vec![]; + + for (hbase_role, role_group_configs) in &cluster.role_group_configs { + for (role_group_name, rg_config) in role_group_configs { + services.push(build_rolegroup_service( + cluster, + hbase_role, + role_group_name, + )); + services.push(build_rolegroup_metrics_service( + cluster, + hbase_role, + role_group_name, + )); + config_maps.push( + build_rolegroup_config_map(cluster, cluster_info, hbase_role, role_group_name) + .with_context(|_| ConfigMapSnafu { + hbase_role: hbase_role.clone(), + role_group: role_group_name.clone(), + })?, + ); + stateful_sets.push( + build_rolegroup_statefulset( + cluster, + hbase_role, + role_group_name, + rg_config, + service_account_name, + ) + .with_context(|_| StatefulSetSnafu { + hbase_role: hbase_role.clone(), + role_group: role_group_name.clone(), + })?, + ); + } + + if let Some(role_config) = cluster.role_configs.get(hbase_role) + && let Some(pdb) = build_pdb(&role_config.pdb, cluster, hbase_role) + { + pod_disruption_budgets.push(pdb); + } + } + + // The role-level discovery ConfigMap advertises the cluster's connection information; it is + // deterministic (derived only from the validated cluster and static cluster info). + config_maps.push(build_discovery_config_map(cluster, cluster_info).context(DiscoverySnafu)?); + + Ok(KubernetesResources { + stateful_sets, + services, + config_maps, + pod_disruption_budgets, + }) +} + pub mod graceful_shutdown; pub mod jvm; pub mod kerberos; @@ -17,3 +124,66 @@ pub mod properties; pub mod region_mover; pub mod resource; pub mod role; + +#[cfg(test)] +mod tests { + use stackable_operator::kube::Resource; + + use super::build; + use crate::test_utils; + + /// Collects the `.metadata.name`s of the given resources, sorted for stable comparison. + fn sorted_names(resources: &[impl Resource]) -> Vec<&str> { + let mut names: Vec<&str> = resources + .iter() + .filter_map(|resource| resource.meta().name.as_deref()) + .collect(); + names.sort(); + names + } + + #[test] + fn build_produces_expected_resource_names() { + let cluster = test_utils::validated_cluster(); + let cluster_info = test_utils::cluster_info(); + let resources = + build(&cluster, &cluster_info, "hbase-serviceaccount").expect("build succeeds"); + + // One StatefulSet per role group (one `default` group for each of the three roles). + assert_eq!( + sorted_names(&resources.stateful_sets), + [ + "hbase-master-default", + "hbase-regionserver-default", + "hbase-restserver-default", + ] + ); + // One headless and one metrics Service per role group. + assert_eq!( + sorted_names(&resources.services), + [ + "hbase-master-default-headless", + "hbase-master-default-metrics", + "hbase-regionserver-default-headless", + "hbase-regionserver-default-metrics", + "hbase-restserver-default-headless", + "hbase-restserver-default-metrics", + ] + ); + // One ConfigMap per role group plus the cluster-wide discovery ConfigMap (`hbase`). + assert_eq!( + sorted_names(&resources.config_maps), + [ + "hbase", + "hbase-master-default", + "hbase-regionserver-default", + "hbase-restserver-default", + ] + ); + // A default PodDisruptionBudget per role. + assert_eq!( + sorted_names(&resources.pod_disruption_budgets), + ["hbase-master", "hbase-regionserver", "hbase-restserver"] + ); + } +} diff --git a/rust/operator-binary/src/controller/build/resource/statefulset.rs b/rust/operator-binary/src/controller/build/resource/statefulset.rs index 06271f3e..1ae98a9c 100644 --- a/rust/operator-binary/src/controller/build/resource/statefulset.rs +++ b/rust/operator-binary/src/controller/build/resource/statefulset.rs @@ -15,14 +15,10 @@ use stackable_operator::{ DeepMerge, api::{ apps::v1::{StatefulSet, StatefulSetSpec}, - core::v1::{ - ConfigMapVolumeSource, ContainerPort, Probe, ServiceAccount, TCPSocketAction, - Volume, - }, + core::v1::{ConfigMapVolumeSource, ContainerPort, Probe, TCPSocketAction, Volume}, }, apimachinery::pkg::{apis::meta::v1::LabelSelector, util::intstr::IntOrString}, }, - kube::ResourceExt, product_logging, v2::{ builder::pod::container::{EnvVarName, EnvVarSet, new_container_builder}, @@ -110,7 +106,7 @@ pub fn build_rolegroup_statefulset( hbase_role: &HbaseRole, role_group_name: &RoleGroupName, validated_rg_config: &HbaseRoleGroupConfig, - service_account: &ServiceAccount, + service_account_name: &str, ) -> Result { let resolved_product_image = &cluster.image; let merged_config = &validated_rg_config.config.config; @@ -243,7 +239,7 @@ pub fn build_rolegroup_statefulset( )), ) .context(AddVolumeSnafu)? - .service_account_name(service_account.name_any()) + .service_account_name(service_account_name) .security_context(PodSecurityContextBuilder::new().fs_group(1000).build()); // The HBase container's log config ConfigMap: either the operator-generated one (the diff --git a/rust/operator-binary/src/controller/mod.rs b/rust/operator-binary/src/controller/mod.rs index e2ffc079..2cb2c9e6 100644 --- a/rust/operator-binary/src/controller/mod.rs +++ b/rust/operator-binary/src/controller/mod.rs @@ -10,7 +10,14 @@ pub use stackable_operator::v2::types::operator::RoleGroupName; use stackable_operator::{ builder::meta::ObjectMetaBuilder, commons::product_image_selection::ResolvedProductImage, - k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta, + k8s_openapi::{ + api::{ + apps::v1::StatefulSet, + core::v1::{ConfigMap, Service}, + policy::v1::PodDisruptionBudget, + }, + apimachinery::pkg::apis::meta::v1::ObjectMeta, + }, kube::Resource, kvp::Labels, v2::{ @@ -51,6 +58,17 @@ pub(crate) fn controller_name() -> ControllerName { .expect("the controller name is a valid label value") } +/// The complete set of Kubernetes resources built for a [`ValidatedCluster`], ready to be applied. +/// +/// hbase exposes its listeners as volume/PVC sources inside the `StatefulSet` rather than as +/// top-level `Listener` objects, so (unlike some sibling operators) there is no `listeners` field. +pub struct KubernetesResources { + pub stateful_sets: Vec, + pub services: Vec, + pub config_maps: Vec, + pub pod_disruption_budgets: Vec, +} + /// The validated cluster: proves that config merging and validation succeeded for /// every role and role group before any resources are created. #[derive(Clone, Debug)] diff --git a/rust/operator-binary/src/hbase_controller.rs b/rust/operator-binary/src/hbase_controller.rs index 666d58d7..61d6ba46 100644 --- a/rust/operator-binary/src/hbase_controller.rs +++ b/rust/operator-binary/src/hbase_controller.rs @@ -13,6 +13,7 @@ use stackable_operator::{ cluster_resources::ClusterResourceApplyStrategy, commons::rbac::build_rbac_resources, kube::{ + ResourceExt, core::{DeserializeGuard, error_boundary}, runtime::controller::Action, }, @@ -28,17 +29,7 @@ use stackable_operator::{ use strum::{EnumDiscriminants, IntoStaticStr}; use crate::{ - controller::{ - RoleGroupName, - build::resource::{ - config_map::build_rolegroup_config_map, - discovery::build_discovery_config_map, - pdb::build_pdb, - service::{build_rolegroup_metrics_service, build_rolegroup_service}, - statefulset::build_rolegroup_statefulset, - }, - controller_name, operator_name, product_name, - }, + controller::{build, controller_name, operator_name, product_name}, crd::{APP_NAME, HbaseClusterStatus, OPERATOR_NAME, v1alpha1}, }; @@ -73,47 +64,11 @@ pub enum Error { source: stackable_operator::cluster_resources::Error, }, - #[snafu(display("failed to apply Service for role group {role_group}"))] - ApplyRoleGroupService { - source: stackable_operator::cluster_resources::Error, - role_group: RoleGroupName, - }, - - #[snafu(display("failed to build rolegroup ConfigMap"))] - BuildRolegroupConfigMap { - source: crate::controller::build::resource::config_map::Error, - }, - - #[snafu(display("failed to apply ConfigMap for role group {role_group}"))] - ApplyRoleGroupConfig { - source: stackable_operator::cluster_resources::Error, - role_group: RoleGroupName, - }, - - #[snafu(display("failed to build StatefulSet for role group {role_group}"))] - BuildRoleGroupStatefulSet { - source: crate::controller::build::resource::statefulset::Error, - role_group: RoleGroupName, - }, - - #[snafu(display("failed to apply StatefulSet for role group {role_group}"))] - ApplyRoleGroupStatefulSet { - source: stackable_operator::cluster_resources::Error, - role_group: RoleGroupName, - }, - - #[snafu(display("failed to apply PodDisruptionBudget"))] - ApplyPdb { - source: stackable_operator::cluster_resources::Error, - }, - - #[snafu(display("failed to build discovery configmap"))] - BuildDiscoveryConfigMap { - source: crate::controller::build::resource::discovery::Error, - }, + #[snafu(display("failed to build cluster resources"))] + BuildResources { source: build::Error }, - #[snafu(display("failed to apply discovery configmap"))] - ApplyDiscoveryConfigMap { + #[snafu(display("failed to apply cluster resource"))] + ApplyResource { source: stackable_operator::cluster_resources::Error, }, @@ -199,85 +154,49 @@ pub async fn reconcile_hbase( .await .context(ApplyRoleBindingSnafu)?; + // The ServiceAccount name is deterministic on the built object, so the build step does not + // depend on the applied ServiceAccount. + let service_account_name = rbac_sa.name_any(); + + let resources = build::build( + &validated_cluster, + &client.kubernetes_cluster_info, + &service_account_name, + ) + .context(BuildResourcesSnafu)?; + let mut ss_cond_builder = StatefulSetConditionBuilder::default(); - for (hbase_role, role_group_configs) in &validated_cluster.role_group_configs { - for (role_group_name, validated_rg_config) in role_group_configs { - let rg_service = - build_rolegroup_service(&validated_cluster, hbase_role, role_group_name); - - let rg_metrics_service = - build_rolegroup_metrics_service(&validated_cluster, hbase_role, role_group_name); - - let rg_configmap = build_rolegroup_config_map( - &validated_cluster, - &client.kubernetes_cluster_info, - hbase_role, - role_group_name, - ) - .context(BuildRolegroupConfigMapSnafu)?; - let rg_statefulset = build_rolegroup_statefulset( - &validated_cluster, - hbase_role, - role_group_name, - validated_rg_config, - &rbac_sa, - ) - .with_context(|_| BuildRoleGroupStatefulSetSnafu { - role_group: role_group_name.clone(), - })?; - cluster_resources - .add(client, rg_service) - .await - .with_context(|_| ApplyRoleGroupServiceSnafu { - role_group: role_group_name.clone(), - })?; - cluster_resources - .add(client, rg_metrics_service) - .await - .with_context(|_| ApplyRoleGroupServiceSnafu { - role_group: role_group_name.clone(), - })?; - cluster_resources - .add(client, rg_configmap) - .await - .with_context(|_| ApplyRoleGroupConfigSnafu { - role_group: role_group_name.clone(), - })?; - - // Note: The StatefulSet needs to be applied after all ConfigMaps and Secrets it mounts - // to prevent unnecessary Pod restarts. - // See https://github.com/stackabletech/commons-operator/issues/111 for details. - ss_cond_builder.add( - cluster_resources - .add(client, rg_statefulset) - .await - .with_context(|_| ApplyRoleGroupStatefulSetSnafu { - role_group: role_group_name.clone(), - })?, - ); - } - - if let Some(role_config) = validated_cluster.role_configs.get(hbase_role) - && let Some(pdb) = build_pdb(&role_config.pdb, &validated_cluster, hbase_role) - { + // Apply order: everything before the StatefulSets, StatefulSets last. A changed ConfigMap or + // Secret a Pod mounts must exist before the Pod restarts, otherwise the Pod restarts again + // unnecessarily. See https://github.com/stackabletech/commons-operator/issues/111 for details. + for service in resources.services { + cluster_resources + .add(client, service) + .await + .context(ApplyResourceSnafu)?; + } + for config_map in resources.config_maps { + cluster_resources + .add(client, config_map) + .await + .context(ApplyResourceSnafu)?; + } + for pdb in resources.pod_disruption_budgets { + cluster_resources + .add(client, pdb) + .await + .context(ApplyResourceSnafu)?; + } + for statefulset in resources.stateful_sets { + ss_cond_builder.add( cluster_resources - .add(client, pdb) + .add(client, statefulset) .await - .context(ApplyPdbSnafu)?; - } + .context(ApplyResourceSnafu)?, + ); } - // Discovery CM will fail to build until the rest of the cluster has been deployed, so do it last - // so that failure won't inhibit the rest of the cluster from booting up. - let discovery_cm = - build_discovery_config_map(&validated_cluster, &client.kubernetes_cluster_info) - .context(BuildDiscoveryConfigMapSnafu)?; - cluster_resources - .add(client, discovery_cm) - .await - .context(ApplyDiscoveryConfigMapSnafu)?; - let cluster_operation_cond_builder = ClusterOperationsConditionBuilder::new(&hbase.spec.cluster_operation); diff --git a/rust/operator-binary/src/test_utils.rs b/rust/operator-binary/src/test_utils.rs index ad888f71..998e4eb3 100644 --- a/rust/operator-binary/src/test_utils.rs +++ b/rust/operator-binary/src/test_utils.rs @@ -7,7 +7,10 @@ use std::str::FromStr; -use stackable_operator::v2::types::operator::RoleGroupName; +use stackable_operator::{ + commons::networking::DomainName, utils::cluster_info::KubernetesClusterInfo, + v2::types::operator::RoleGroupName, +}; use crate::{ controller::{ @@ -80,6 +83,14 @@ pub fn role_group_name(name: &str) -> RoleGroupName { RoleGroupName::from_str(name).expect("valid role group name") } +/// A fixed [`KubernetesClusterInfo`] (`cluster.local` domain) for builders that need cluster +/// metadata such as the discovery `ConfigMap` and Kerberos principals. +pub fn cluster_info() -> KubernetesClusterInfo { + KubernetesClusterInfo { + cluster_domain: DomainName::from_str("cluster.local").expect("valid cluster domain"), + } +} + /// The merged [`AnyServiceConfig`] for the given `role` and `role_group`. pub fn merged_config_for<'a>( validated_cluster: &'a ValidatedCluster,