diff --git a/crates/temps-entities/src/status_monitors.rs b/crates/temps-entities/src/status_monitors.rs index d2c1f674a..02f01a11a 100644 --- a/crates/temps-entities/src/status_monitors.rs +++ b/crates/temps-entities/src/status_monitors.rs @@ -17,6 +17,7 @@ pub struct Model { pub name: String, pub monitor_type: String, // web, api, desktop pub check_path: Option, + pub check_path_revision: i64, pub check_interval_seconds: i32, pub is_active: bool, pub is_managed: bool, diff --git a/crates/temps-migrations/src/migration/m20260908_000001_reconcile_legacy_status_monitors.rs b/crates/temps-migrations/src/migration/m20260908_000001_reconcile_legacy_status_monitors.rs new file mode 100644 index 000000000..2203bf5cb --- /dev/null +++ b/crates/temps-migrations/src/migration/m20260908_000001_reconcile_legacy_status_monitors.rs @@ -0,0 +1,221 @@ +// SPDX-FileCopyrightText: 2024-2026 Temps Contributors +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + // Before `is_managed` existed, Temps' automatic environment monitor + // used the deterministic "{environment} Monitor" name. A previous + // migration left those rows unmanaged and startup reconciliation then + // created a replacement, splitting uptime history across duplicate + // rows. Keep the oldest legacy row as the canonical managed monitor, + // move references from every duplicate automatic row to it, and remove + // only duplicate rows with the reserved environment-monitor name or + // explicit managed provenance. Custom-named monitors remain untouched. + manager + .get_connection() + .execute_unprepared( + "ALTER TABLE status_monitors \ + ADD COLUMN check_path_revision BIGINT NOT NULL DEFAULT 0; \ + LOCK TABLE status_monitors, status_checks, status_incidents \ + IN SHARE ROW EXCLUSIVE MODE; \ + CREATE TEMP TABLE _temps_monitor_canonical ON COMMIT DROP AS \ + SELECT DISTINCT ON (monitor.environment_id) \ + monitor.environment_id, monitor.id AS canonical_id \ + FROM status_monitors AS monitor \ + JOIN environments AS environment \ + ON environment.id = monitor.environment_id \ + AND environment.project_id = monitor.project_id \ + WHERE monitor.environment_id IS NOT NULL \ + AND monitor.name = environment.name || ' Monitor' \ + ORDER BY monitor.environment_id, monitor.id; \ + CREATE TEMP TABLE _temps_monitor_duplicates ON COMMIT DROP AS \ + SELECT canonical.environment_id, canonical.canonical_id, \ + monitor.id AS duplicate_id, monitor.is_managed AS was_managed \ + FROM _temps_monitor_canonical AS canonical \ + JOIN status_monitors AS monitor \ + ON monitor.environment_id = canonical.environment_id \ + JOIN environments AS environment \ + ON environment.id = canonical.environment_id \ + AND environment.project_id = monitor.project_id \ + WHERE monitor.id <> canonical.canonical_id \ + AND (monitor.is_managed = TRUE \ + OR monitor.name = environment.name || ' Monitor'); \ + CREATE TABLE _temps_m20260908_monitor_canonical_backup AS \ + SELECT monitor.id, monitor.is_managed, monitor.check_path, \ + monitor.check_path_revision, \ + monitor.check_path AS reconciled_check_path, \ + monitor.check_path_revision AS reconciled_check_path_revision \ + FROM status_monitors AS monitor \ + JOIN _temps_monitor_canonical AS canonical \ + ON canonical.canonical_id = monitor.id; \ + ALTER TABLE _temps_m20260908_monitor_canonical_backup \ + ADD PRIMARY KEY (id); \ + CREATE TABLE _temps_m20260908_monitor_duplicate_backup AS \ + SELECT monitor.*, mapping.canonical_id \ + FROM status_monitors AS monitor \ + JOIN _temps_monitor_duplicates AS mapping \ + ON mapping.duplicate_id = monitor.id; \ + ALTER TABLE _temps_m20260908_monitor_duplicate_backup \ + ADD PRIMARY KEY (id); \ + CREATE TABLE _temps_m20260908_status_check_backup AS \ + SELECT status_check.id, status_check.checked_at, \ + status_check.monitor_id \ + FROM status_checks AS status_check \ + JOIN _temps_monitor_duplicates AS mapping \ + ON mapping.duplicate_id = status_check.monitor_id; \ + ALTER TABLE _temps_m20260908_status_check_backup \ + ADD PRIMARY KEY (id, checked_at); \ + CREATE TABLE _temps_m20260908_status_incident_backup AS \ + SELECT incident.id, incident.monitor_id \ + FROM status_incidents AS incident \ + JOIN _temps_monitor_duplicates AS mapping \ + ON mapping.duplicate_id = incident.monitor_id; \ + ALTER TABLE _temps_m20260908_status_incident_backup \ + ADD PRIMARY KEY (id); \ + UPDATE status_monitors AS duplicate \ + SET is_managed = FALSE \ + FROM _temps_monitor_duplicates AS mapping \ + WHERE duplicate.id = mapping.duplicate_id; \ + UPDATE status_monitors AS canonical \ + SET is_managed = TRUE, \ + check_path = CASE \ + WHEN EXISTS ( \ + SELECT 1 \ + FROM _temps_monitor_duplicates AS mapping \ + WHERE mapping.canonical_id = canonical.id \ + ) THEN ( \ + SELECT duplicate.check_path \ + FROM _temps_monitor_duplicates AS mapping \ + JOIN status_monitors AS duplicate \ + ON duplicate.id = mapping.duplicate_id \ + WHERE mapping.canonical_id = canonical.id \ + ORDER BY mapping.was_managed DESC, \ + duplicate.updated_at DESC, \ + duplicate.id DESC \ + LIMIT 1 \ + ) \ + ELSE canonical.check_path \ + END, \ + check_path_revision = GREATEST( \ + canonical.check_path_revision, \ + COALESCE(( \ + SELECT MAX(duplicate.check_path_revision) \ + FROM _temps_monitor_duplicates AS mapping \ + JOIN status_monitors AS duplicate \ + ON duplicate.id = mapping.duplicate_id \ + WHERE mapping.canonical_id = canonical.id \ + ), canonical.check_path_revision) \ + ) + 1 \ + FROM _temps_monitor_canonical AS selected \ + WHERE canonical.id = selected.canonical_id; \ + UPDATE _temps_m20260908_monitor_canonical_backup AS backup \ + SET reconciled_check_path = canonical.check_path, \ + reconciled_check_path_revision = canonical.check_path_revision \ + FROM status_monitors AS canonical \ + WHERE canonical.id = backup.id; \ + UPDATE status_checks AS status_check \ + SET monitor_id = mapping.canonical_id \ + FROM _temps_monitor_duplicates AS mapping \ + WHERE status_check.monitor_id = mapping.duplicate_id; \ + UPDATE status_incidents AS incident \ + SET monitor_id = mapping.canonical_id \ + FROM _temps_monitor_duplicates AS mapping \ + WHERE incident.monitor_id = mapping.duplicate_id; \ + DELETE FROM status_monitors AS duplicate \ + USING _temps_monitor_duplicates AS mapping \ + WHERE duplicate.id = mapping.duplicate_id; \ + DROP TABLE IF EXISTS \ + _temps_m20260904_managed_monitor_ownership_backup", + ) + .await?; + + Ok(()) + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + // Restore only the rows and associations captured by up(). Checks or + // incidents created on the canonical monitor after the migration stay + // there; they did not belong to a duplicate in the pre-migration state. + manager + .get_connection() + .execute_unprepared( + "LOCK TABLE status_monitors, status_checks, status_incidents \ + IN SHARE ROW EXCLUSIVE MODE; \ + UPDATE status_monitors AS canonical \ + SET is_managed = FALSE \ + FROM _temps_m20260908_monitor_canonical_backup AS backup \ + WHERE canonical.id = backup.id; \ + INSERT INTO status_monitors \ + (id, project_id, environment_id, name, monitor_type, check_path, \ + check_path_revision, check_interval_seconds, is_active, is_managed, \ + created_at, updated_at) \ + SELECT id, project_id, environment_id, name, monitor_type, check_path, \ + check_path_revision, check_interval_seconds, is_active, is_managed, \ + created_at, updated_at \ + FROM _temps_m20260908_monitor_duplicate_backup; \ + WITH changed_canonical AS ( \ + SELECT canonical.id, canonical.check_path, \ + canonical.check_path_revision, canonical.updated_at \ + FROM status_monitors AS canonical \ + JOIN _temps_m20260908_monitor_canonical_backup AS backup \ + ON backup.id = canonical.id \ + WHERE canonical.check_path_revision <> \ + backup.reconciled_check_path_revision \ + ), handoff AS ( \ + SELECT DISTINCT ON (duplicate.canonical_id) \ + duplicate.id, changed.check_path, \ + changed.check_path_revision, changed.updated_at \ + FROM _temps_m20260908_monitor_duplicate_backup AS duplicate \ + JOIN changed_canonical AS changed \ + ON changed.id = duplicate.canonical_id \ + ORDER BY duplicate.canonical_id, duplicate.is_managed DESC, \ + duplicate.updated_at DESC, duplicate.id DESC \ + ) \ + UPDATE status_monitors AS restored \ + SET check_path = handoff.check_path, \ + check_path_revision = handoff.check_path_revision, \ + updated_at = handoff.updated_at \ + FROM handoff \ + WHERE restored.id = handoff.id; \ + UPDATE status_checks AS status_check \ + SET monitor_id = backup.monitor_id \ + FROM _temps_m20260908_status_check_backup AS backup \ + WHERE status_check.id = backup.id \ + AND status_check.checked_at = backup.checked_at; \ + UPDATE status_incidents AS incident \ + SET monitor_id = backup.monitor_id \ + FROM _temps_m20260908_status_incident_backup AS backup \ + WHERE incident.id = backup.id; \ + UPDATE status_monitors AS canonical \ + SET is_managed = backup.is_managed, \ + check_path = CASE \ + WHEN canonical.check_path_revision = \ + backup.reconciled_check_path_revision \ + THEN backup.check_path \ + ELSE canonical.check_path \ + END, \ + check_path_revision = CASE \ + WHEN canonical.check_path_revision = \ + backup.reconciled_check_path_revision \ + THEN backup.check_path_revision \ + ELSE canonical.check_path_revision \ + END \ + FROM _temps_m20260908_monitor_canonical_backup AS backup \ + WHERE canonical.id = backup.id; \ + DROP TABLE _temps_m20260908_status_incident_backup; \ + DROP TABLE _temps_m20260908_status_check_backup; \ + DROP TABLE _temps_m20260908_monitor_duplicate_backup; \ + DROP TABLE _temps_m20260908_monitor_canonical_backup; \ + ALTER TABLE status_monitors DROP COLUMN check_path_revision", + ) + .await?; + + Ok(()) + } +} diff --git a/crates/temps-migrations/src/migration/mod.rs b/crates/temps-migrations/src/migration/mod.rs index 78043a0dc..5ebb3c50d 100644 --- a/crates/temps-migrations/src/migration/mod.rs +++ b/crates/temps-migrations/src/migration/mod.rs @@ -254,6 +254,7 @@ mod m20260904_000001_reset_ambiguous_managed_status_monitors; mod m20260904_000002_add_lifecycle_reconcile_generation_to_s3_sources; mod m20260904_000003_add_continuous_archive_source_to_external_services; mod m20260907_000001_add_mfa_pending_origin_to_sessions; +mod m20260908_000001_reconcile_legacy_status_monitors; pub struct Migrator; @@ -564,6 +565,7 @@ impl MigratorTrait for Migrator { Box::new(m20260903_000002_harden_application_workspaces::Migration), Box::new(m20260903_000003_application_workspace_quarantine::Migration), Box::new(m20260903_000004_repair_application_primary_projects::Migration), + Box::new(m20260908_000001_reconcile_legacy_status_monitors::Migration), ] } } @@ -615,6 +617,10 @@ mod registry_tests { "m20260904_000001_reset_ambiguous_managed_status_monitors", "m20260831_000001_ai_first_applications", ), + ( + "m20260903_000004_repair_application_primary_projects", + "m20260908_000001_reconcile_legacy_status_monitors", + ), ] { let shipped_position = names .iter() diff --git a/crates/temps-migrations/tests/migration_tests.rs b/crates/temps-migrations/tests/migration_tests.rs index 1e2cfde39..2f6b64e9c 100644 --- a/crates/temps-migrations/tests/migration_tests.rs +++ b/crates/temps-migrations/tests/migration_tests.rs @@ -622,6 +622,413 @@ async fn test_managed_monitor_migration_down_restores_state_from_previous_up_imp Ok(()) } +#[tokio::test] +async fn test_legacy_monitor_reconciliation_merges_duplicates_and_preserves_history( +) -> anyhow::Result<()> { + if external_db_configured() { + println!("Skipping legacy-monitor reconciliation test: external database configured"); + return Ok(()); + } + let container = match GenericImage::new("timescale/timescaledb-ha", "pg18") + .with_wait_for(postgres_ready_wait_for()) + .with_exposed_port(ContainerPort::Tcp(5432)) + .with_env_var("POSTGRES_DB", "postgres") + .with_env_var("POSTGRES_USER", "postgres") + .with_env_var("POSTGRES_PASSWORD", "postgres") + .with_env_var("POSTGRES_HOST_AUTH_METHOD", "trust") + .with_cmd(vec![ + "postgres", + "-c", + "timescaledb.max_background_workers=0", + ]) + .with_startup_timeout(CONTAINER_STARTUP_TIMEOUT) + .start() + .await + { + Ok(container) => container, + Err(error) => { + eprintln!("Skipping legacy-monitor reconciliation test: Docker unavailable: {error}"); + return Ok(()); + } + }; + let port = container.get_host_port_ipv4(5432).await?; + let db = connect_with_retries(&format!( + "postgresql://postgres:postgres@localhost:{port}/postgres" + )) + .await?; + + let target = "m20260908_000001_reconcile_legacy_status_monitors"; + let target_position = Migrator::migrations() + .iter() + .position(|migration| migration.name() == target) + .unwrap_or_else(|| panic!("migration {target} not found in Migrator")); + let legacy_ownership_target = "m20260904_000001_reset_ambiguous_managed_status_monitors"; + let legacy_ownership_position = Migrator::migrations() + .iter() + .position(|migration| migration.name() == legacy_ownership_target) + .unwrap_or_else(|| panic!("migration {legacy_ownership_target} not found in Migrator")); + assert!(legacy_ownership_position < target_position); + let rollback_through_legacy = (target_position - legacy_ownership_position) as u32; + let reapply_through_target = rollback_through_legacy + 1; + Migrator::up(&db, Some(target_position as u32)).await?; + + db.execute_unprepared( + "INSERT INTO projects (name, repo_name, repo_owner, directory, main_branch, preset, \ + created_at, updated_at, slug) \ + VALUES ('monitor-reconcile-test', 'repo', 'owner', '.', 'main', 'nodejs', \ + now(), now(), 'monitor-reconcile-test'); \ + INSERT INTO environments \ + (name, slug, subdomain, host, upstreams, created_at, updated_at, project_id) \ + SELECT 'production', 'production', 'monitor-reconcile-production', \ + 'monitor-reconcile.test', '[]', now(), now(), id \ + FROM projects WHERE slug = 'monitor-reconcile-test'; \ + INSERT INTO status_monitors \ + (project_id, environment_id, name, monitor_type, check_path, \ + check_interval_seconds, is_active, is_managed, created_at, updated_at) \ + SELECT project_id, id, 'production Monitor', 'web', '/legacy-health', \ + 60, true, false, now() - interval '30 days', now() - interval '30 days' \ + FROM environments WHERE subdomain = 'monitor-reconcile-production'; \ + INSERT INTO status_monitors \ + (project_id, environment_id, name, monitor_type, check_path, \ + check_interval_seconds, is_active, is_managed, created_at, updated_at) \ + SELECT project_id, id, 'Custom API check', 'web', '/custom', \ + 120, true, false, now() - interval '20 days', now() - interval '20 days' \ + FROM environments WHERE subdomain = 'monitor-reconcile-production'; \ + INSERT INTO status_monitors \ + (project_id, environment_id, name, monitor_type, check_path, \ + check_interval_seconds, is_active, is_managed, created_at, updated_at) \ + SELECT project_id, id, 'production Monitor', 'web', NULL, \ + 60, true, false, now() - interval '2 days', now() - interval '2 days' \ + FROM environments WHERE subdomain = 'monitor-reconcile-production'; \ + INSERT INTO status_monitors \ + (project_id, environment_id, name, monitor_type, check_path, \ + check_interval_seconds, is_active, is_managed, created_at, updated_at) \ + SELECT project_id, id, 'production Monitor', 'web', '/from-temps-yaml', \ + 60, true, true, now() - interval '1 day', now() - interval '1 day' \ + FROM environments WHERE subdomain = 'monitor-reconcile-production'; \ + INSERT INTO status_monitors \ + (project_id, environment_id, name, monitor_type, check_path, \ + check_interval_seconds, is_active, is_managed, created_at, updated_at) \ + SELECT project_id, id, 'production Monitor', 'web', '/user-health', \ + 90, true, false, now() - interval '10 days', now() - interval '10 days' \ + FROM environments WHERE subdomain = 'monitor-reconcile-production'", + ) + .await?; + + let monitor_rows = db + .query_all(sea_orm::Statement::from_string( + sea_orm::DatabaseBackend::Postgres, + "SELECT id, name FROM status_monitors ORDER BY id".to_string(), + )) + .await?; + let canonical_id = monitor_rows[0].try_get::("", "id")?; + let repeated_duplicate_id = monitor_rows[2].try_get::("", "id")?; + let managed_duplicate_id = monitor_rows[3].try_get::("", "id")?; + let explicit_user_monitor_id = monitor_rows[4].try_get::("", "id")?; + + db.execute_unprepared(&format!( + "INSERT INTO status_checks (monitor_id, status, checked_at, created_at) VALUES \ + ({canonical_id}, 'operational', now() - interval '3 days', now() - interval '3 days'), \ + ({repeated_duplicate_id}, 'operational', now() - interval '2 days', now() - interval '2 days'), \ + ({managed_duplicate_id}, 'degraded', now() - interval '1 day', now() - interval '1 day'), \ + ({explicit_user_monitor_id}, 'unknown', now() - interval '12 hours', now() - interval '12 hours'); \ + UPDATE status_checks \ + SET error_message = 'Monitor created - awaiting first health check' \ + WHERE monitor_id = {explicit_user_monitor_id}; \ + INSERT INTO status_incidents \ + (project_id, environment_id, monitor_id, title, severity, status, \ + started_at, created_at, updated_at) \ + SELECT project_id, id, {managed_duplicate_id}, 'Deployment health failed', \ + 'minor', 'resolved', now() - interval '1 day', \ + now() - interval '1 day', now() - interval '1 day' \ + FROM environments WHERE subdomain = 'monitor-reconcile-production'; \ + CREATE TABLE _temps_m20260904_managed_monitor_ownership_backup ( \ + monitor_id INTEGER PRIMARY KEY REFERENCES status_monitors(id) ON DELETE CASCADE \ + ); \ + INSERT INTO _temps_m20260904_managed_monitor_ownership_backup (monitor_id) \ + VALUES ({canonical_id})" + )) + .await?; + + Migrator::up(&db, Some(1)).await?; + + let reconciled = db + .query_all(sea_orm::Statement::from_string( + sea_orm::DatabaseBackend::Postgres, + "SELECT id, name, check_path, is_managed \ + FROM status_monitors ORDER BY id" + .to_string(), + )) + .await?; + assert_eq!( + reconciled.len(), + 2, + "the reserved environment-monitor rows are integrated while the custom monitor remains" + ); + assert_eq!(reconciled[0].try_get::("", "id")?, canonical_id); + assert_eq!( + reconciled[0].try_get::("", "name")?, + "production Monitor" + ); + assert_eq!( + reconciled[0].try_get::("", "check_path")?, + "/from-temps-yaml" + ); + assert!(reconciled[0].try_get::("", "is_managed")?); + assert_eq!( + reconciled[1].try_get::("", "name")?, + "Custom API check" + ); + assert!(!reconciled[1].try_get::("", "is_managed")?); + + let check_count = db + .query_one(sea_orm::Statement::from_string( + sea_orm::DatabaseBackend::Postgres, + format!( + "SELECT COUNT(*) AS count FROM status_checks WHERE monitor_id = {canonical_id}" + ), + )) + .await? + .expect("canonical status-check count"); + assert_eq!(check_count.try_get::("", "count")?, 4); + + let incident_monitor_id = db + .query_one(sea_orm::Statement::from_string( + sea_orm::DatabaseBackend::Postgres, + "SELECT monitor_id FROM status_incidents WHERE title = 'Deployment health failed'" + .to_string(), + )) + .await? + .expect("migrated incident"); + assert_eq!( + incident_monitor_id.try_get::("", "monitor_id")?, + canonical_id + ); + + let reconciliation_backup_exists = db + .query_one(sea_orm::Statement::from_string( + sea_orm::DatabaseBackend::Postgres, + "SELECT to_regclass('_temps_m20260908_monitor_duplicate_backup') IS NOT NULL AS present" + .to_string(), + )) + .await? + .expect("reconciliation backup-table lookup"); + assert!(reconciliation_backup_exists.try_get::("", "present")?); + let stale_ownership_backup_retired = db + .query_one(sea_orm::Statement::from_string( + sea_orm::DatabaseBackend::Postgres, + "SELECT to_regclass('_temps_m20260904_managed_monitor_ownership_backup') IS NULL AS gone" + .to_string(), + )) + .await? + .expect("legacy ownership backup lookup"); + assert!(stale_ownership_backup_retired.try_get::("", "gone")?); + + db.execute_unprepared(&format!( + "UPDATE status_monitors \ + SET is_active = FALSE, updated_at = now() \ + WHERE id = {canonical_id}" + )) + .await?; + Migrator::down(&db, Some(1)).await?; + let after_unrelated_update = db + .query_one(sea_orm::Statement::from_string( + sea_orm::DatabaseBackend::Postgres, + format!("SELECT check_path, is_active FROM status_monitors WHERE id = {canonical_id}"), + )) + .await? + .expect("canonical monitor after unrelated-update rollback"); + assert_eq!( + after_unrelated_update.try_get::("", "check_path")?, + "/legacy-health" + ); + assert!(!after_unrelated_update.try_get::("", "is_active")?); + + Migrator::up(&db, Some(1)).await?; + let after_unrelated_update_reapply = db + .query_one(sea_orm::Statement::from_string( + sea_orm::DatabaseBackend::Postgres, + format!("SELECT check_path FROM status_monitors WHERE id = {canonical_id}"), + )) + .await? + .expect("canonical monitor after unrelated-update reapply"); + assert_eq!( + after_unrelated_update_reapply.try_get::("", "check_path")?, + "/from-temps-yaml" + ); + + db.execute_unprepared(&format!( + "UPDATE status_monitors \ + SET check_path = '/from-temps-yaml', \ + check_path_revision = check_path_revision + 1, \ + updated_at = now() \ + WHERE id = {canonical_id}" + )) + .await?; + Migrator::down(&db, Some(1)).await?; + let equal_path_write_after_rollback = db + .query_one(sea_orm::Statement::from_string( + sea_orm::DatabaseBackend::Postgres, + format!("SELECT check_path FROM status_monitors WHERE id = {canonical_id}"), + )) + .await? + .expect("canonical monitor after equal-valued path write and rollback"); + assert_eq!( + equal_path_write_after_rollback.try_get::("", "check_path")?, + "/from-temps-yaml" + ); + Migrator::up(&db, Some(1)).await?; + + db.execute_unprepared(&format!( + "UPDATE status_monitors \ + SET check_path = '/post-migration-deploy', \ + check_path_revision = check_path_revision + 1, \ + updated_at = now() \ + WHERE id = {canonical_id}" + )) + .await?; + + Migrator::down(&db, Some(1)).await?; + let restored = db + .query_all(sea_orm::Statement::from_string( + sea_orm::DatabaseBackend::Postgres, + "SELECT id, name, check_path, is_managed FROM status_monitors ORDER BY id".to_string(), + )) + .await?; + assert_eq!(restored.len(), 5); + assert_eq!(restored[0].try_get::("", "id")?, canonical_id); + assert_eq!( + restored[0].try_get::("", "check_path")?, + "/post-migration-deploy" + ); + assert!(!restored[0].try_get::("", "is_managed")?); + assert_eq!(restored[2].try_get::("", "id")?, repeated_duplicate_id); + assert_eq!(restored[3].try_get::("", "id")?, managed_duplicate_id); + assert!(restored[3].try_get::("", "is_managed")?); + + let restored_check_owners = db + .query_all(sea_orm::Statement::from_string( + sea_orm::DatabaseBackend::Postgres, + "SELECT monitor_id FROM status_checks ORDER BY checked_at".to_string(), + )) + .await? + .into_iter() + .map(|row| row.try_get::("", "monitor_id")) + .collect::, _>>()?; + assert_eq!( + restored_check_owners, + vec![ + canonical_id, + repeated_duplicate_id, + managed_duplicate_id, + explicit_user_monitor_id, + ] + ); + let restored_incident_owner = db + .query_one(sea_orm::Statement::from_string( + sea_orm::DatabaseBackend::Postgres, + "SELECT monitor_id FROM status_incidents WHERE title = 'Deployment health failed'" + .to_string(), + )) + .await? + .expect("restored incident") + .try_get::("", "monitor_id")?; + assert_eq!(restored_incident_owner, managed_duplicate_id); + + let reconciliation_backup_gone = db + .query_one(sea_orm::Statement::from_string( + sea_orm::DatabaseBackend::Postgres, + "SELECT to_regclass('_temps_m20260908_monitor_duplicate_backup') IS NULL AS gone" + .to_string(), + )) + .await? + .expect("reconciliation backup-table lookup after down"); + assert!(reconciliation_backup_gone.try_get::("", "gone")?); + + Migrator::down(&db, Some(rollback_through_legacy)).await?; + let after_legacy_rollback = db + .query_all(sea_orm::Statement::from_string( + sea_orm::DatabaseBackend::Postgres, + "SELECT id, check_path, is_managed FROM status_monitors ORDER BY id".to_string(), + )) + .await?; + assert_eq!(after_legacy_rollback.len(), 5); + assert!(!after_legacy_rollback[0].try_get::("", "is_managed")?); + assert_eq!( + after_legacy_rollback[0].try_get::("", "check_path")?, + "/post-migration-deploy" + ); + assert!(after_legacy_rollback[3].try_get::("", "is_managed")?); + + Migrator::up(&db, Some(reapply_through_target)).await?; + let reconciled_after_reapply = db + .query_all(sea_orm::Statement::from_string( + sea_orm::DatabaseBackend::Postgres, + "SELECT id, name, check_path, is_managed FROM status_monitors ORDER BY id".to_string(), + )) + .await?; + assert_eq!(reconciled_after_reapply.len(), 2); + assert_eq!( + reconciled_after_reapply[0].try_get::("", "id")?, + canonical_id + ); + assert_eq!( + reconciled_after_reapply[0].try_get::("", "check_path")?, + "/post-migration-deploy" + ); + assert!(reconciled_after_reapply[0].try_get::("", "is_managed")?); + + db.execute_unprepared(&format!( + "UPDATE status_monitors \ + SET check_path = NULL, \ + check_path_revision = check_path_revision + 1, \ + updated_at = now() + interval '1 second' \ + WHERE id = {canonical_id}" + )) + .await?; + Migrator::down(&db, Some(1)).await?; + let cleared_after_target_rollback = db + .query_one(sea_orm::Statement::from_string( + sea_orm::DatabaseBackend::Postgres, + format!("SELECT check_path FROM status_monitors WHERE id = {canonical_id}"), + )) + .await? + .expect("canonical monitor after rolling back reconciliation"); + assert_eq!( + cleared_after_target_rollback.try_get::>("", "check_path")?, + None + ); + + Migrator::down(&db, Some(rollback_through_legacy)).await?; + let cleared_after_legacy_rollback = db + .query_one(sea_orm::Statement::from_string( + sea_orm::DatabaseBackend::Postgres, + format!("SELECT check_path FROM status_monitors WHERE id = {canonical_id}"), + )) + .await? + .expect("canonical monitor after rolling back legacy ownership migration"); + assert_eq!( + cleared_after_legacy_rollback.try_get::>("", "check_path")?, + None + ); + + Migrator::up(&db, Some(reapply_through_target)).await?; + let cleared_after_reapply = db + .query_one(sea_orm::Statement::from_string( + sea_orm::DatabaseBackend::Postgres, + format!("SELECT check_path, is_managed FROM status_monitors WHERE id = {canonical_id}"), + )) + .await? + .expect("canonical monitor after reapplying reconciliation"); + assert_eq!( + cleared_after_reapply.try_get::>("", "check_path")?, + None + ); + assert!(cleared_after_reapply.try_get::("", "is_managed")?); + + Ok(()) +} + #[tokio::test] async fn test_preview_inclusion_default_migration_up_and_down() -> anyhow::Result<()> { if external_db_configured() { diff --git a/crates/temps-monitoring/src/outage.rs b/crates/temps-monitoring/src/outage.rs index 1a10a8802..5167bd708 100644 --- a/crates/temps-monitoring/src/outage.rs +++ b/crates/temps-monitoring/src/outage.rs @@ -1350,6 +1350,7 @@ mod tests { name: "API Health".to_string(), monitor_type: "web".to_string(), check_path: None, + check_path_revision: 0, check_interval_seconds: 60, is_active: true, is_managed: false, diff --git a/crates/temps-status-page/src/routes/status_page.rs b/crates/temps-status-page/src/routes/status_page.rs index 0fae1a4e0..1fa6965db 100644 --- a/crates/temps-status-page/src/routes/status_page.rs +++ b/crates/temps-status-page/src/routes/status_page.rs @@ -1174,6 +1174,7 @@ fn map_error(error: StatusPageError) -> Problem { )) .build(), error @ (StatusPageError::EnvironmentOwnershipLookup { .. } + | StatusPageError::ManagedMonitorReconciliation { .. } | StatusPageError::MonitorOwnershipLookup { .. }) => { tracing::error!(error = %error, "status-page association ownership lookup failed"); internal_server_error() diff --git a/crates/temps-status-page/src/services/health_check_service.rs b/crates/temps-status-page/src/services/health_check_service.rs index d165241fa..6ecd7c827 100644 --- a/crates/temps-status-page/src/services/health_check_service.rs +++ b/crates/temps-status-page/src/services/health_check_service.rs @@ -896,6 +896,7 @@ mod tests { name: format!("monitor-{}", id), monitor_type: "web".to_string(), check_path: None, + check_path_revision: 0, check_interval_seconds: 60, is_active: true, is_managed: false, diff --git a/crates/temps-status-page/src/services/monitor_service.rs b/crates/temps-status-page/src/services/monitor_service.rs index 9b4ce220c..8f4fe641b 100644 --- a/crates/temps-status-page/src/services/monitor_service.rs +++ b/crates/temps-status-page/src/services/monitor_service.rs @@ -3,9 +3,10 @@ use chrono::Utc; use futures::future::BoxFuture; +use sea_orm::sea_query::Expr; use sea_orm::{ - ActiveModelTrait, ColumnTrait, DatabaseConnection, EntityTrait, FromQueryResult, QueryFilter, - QueryOrder, QuerySelect, Set, + ActiveModelTrait, ColumnTrait, DatabaseConnection, DbErr, EntityTrait, FromQueryResult, + QueryFilter, QueryOrder, QuerySelect, Set, }; use std::sync::Arc; use std::time::Duration; @@ -20,6 +21,15 @@ use super::types::{ StatusPageError, UptimeDataPoint, UptimeHistoryResponse, }; +const USER_CREATED_MONITOR_BOOTSTRAP_MESSAGE: &str = + "Monitor created - awaiting first health check"; + +fn is_managed_monitor_unique_violation(error: &DbErr) -> bool { + let rendered = error.to_string(); + rendered.contains("idx_status_monitors_managed_environment") + && (rendered.contains("23505") || rendered.contains("duplicate key")) +} + /// Service for managing status monitors and their health checks pub struct MonitorService { db: Arc, @@ -201,24 +211,122 @@ impl MonitorService { environment_id: i32, environment_name: &str, ) -> Result { - // Check if a monitor already exists for this environment + let environment_project_id = environments::Entity::find_by_id(environment_id) + .select_only() + .column(environments::Column::ProjectId) + .into_tuple::() + .one(self.db.as_ref()) + .await + .map_err(|source| StatusPageError::EnvironmentOwnershipLookup { + environment_id, + project_id, + source, + })?; + if environment_project_id != Some(project_id) { + return Err(StatusPageError::EnvironmentNotInProject { + environment_id, + project_id, + }); + } + + // Prefer the canonical managed monitor when one already exists. let existing = status_monitors::Entity::find() .filter(status_monitors::Column::ProjectId.eq(project_id)) .filter(status_monitors::Column::EnvironmentId.eq(Some(environment_id))) .filter(status_monitors::Column::IsManaged.eq(true)) .one(self.db.as_ref()) - .await?; + .await + .map_err(|source| StatusPageError::ManagedMonitorReconciliation { + operation: "find", + environment_id, + project_id, + source, + })?; if let Some(monitor) = existing { let response: MonitorResponse = monitor.into(); return Ok(self.populate_monitor_url(response).await); } + // This deterministic name is the reserved environment-monitor slot. + // Adopt an existing row instead of creating a second monitor: its ID + // owns the environment's existing uptime history, and successful + // deployments (including `.temps.yaml` health paths) must continue + // updating that same monitor. + let legacy_name = format!("{} Monitor", environment_name); + if let Some(legacy) = status_monitors::Entity::find() + .filter(status_monitors::Column::ProjectId.eq(project_id)) + .filter(status_monitors::Column::EnvironmentId.eq(Some(environment_id))) + .filter(status_monitors::Column::Name.eq(&legacy_name)) + .order_by_asc(status_monitors::Column::Id) + .one(self.db.as_ref()) + .await + .map_err(|source| StatusPageError::ManagedMonitorReconciliation { + operation: "find the existing environment monitor for", + environment_id, + project_id, + source, + })? + { + let mut adopted: status_monitors::ActiveModel = legacy.into(); + adopted.is_managed = Set(true); + + let adopted = match adopted.update(self.db.as_ref()).await { + Ok(monitor) => monitor, + Err(update_error) if is_managed_monitor_unique_violation(&update_error) => { + // Environment-created jobs are at-least-once and may race + // startup reconciliation. The partial unique index decides + // the winner; if another worker established ownership, + // return that canonical monitor. + if let Some(existing) = status_monitors::Entity::find() + .filter(status_monitors::Column::ProjectId.eq(project_id)) + .filter(status_monitors::Column::EnvironmentId.eq(Some(environment_id))) + .filter(status_monitors::Column::IsManaged.eq(true)) + .one(self.db.as_ref()) + .await + .map_err(|source| StatusPageError::ManagedMonitorReconciliation { + operation: "resolve a concurrent adoption of", + environment_id, + project_id, + source, + })? + { + existing + } else { + return Err(StatusPageError::ManagedMonitorReconciliation { + operation: "adopt", + environment_id, + project_id, + source: update_error, + }); + } + } + Err(source) => { + return Err(StatusPageError::ManagedMonitorReconciliation { + operation: "adopt", + environment_id, + project_id, + source, + }); + } + }; + + tracing::info!( + monitor_id = adopted.id, + environment_id, + project_id, + "Adopted existing environment monitor" + ); + + let response: MonitorResponse = adopted.into(); + return Ok(self.populate_monitor_url(response).await); + } + // Create a new monitor for this environment let monitor = status_monitors::ActiveModel { project_id: Set(project_id), environment_id: Set(Some(environment_id)), - name: Set(format!("{} Monitor", environment_name)), + name: Set(legacy_name), monitor_type: Set("web".to_string()), check_interval_seconds: Set(60), // Check every minute is_active: Set(true), @@ -230,7 +338,7 @@ impl MonitorService { let result = match monitor.insert(self.db.as_ref()).await { Ok(monitor) => monitor, - Err(insert_error) => { + Err(insert_error) if is_managed_monitor_unique_violation(&insert_error) => { // Environment creation events are at-least-once and can race. // The partial unique index is authoritative; if another worker // won, return that managed monitor instead of surfacing a false @@ -240,13 +348,32 @@ impl MonitorService { .filter(status_monitors::Column::EnvironmentId.eq(Some(environment_id))) .filter(status_monitors::Column::IsManaged.eq(true)) .one(self.db.as_ref()) - .await? + .await + .map_err(|source| StatusPageError::ManagedMonitorReconciliation { + operation: "resolve a concurrent creation of", + environment_id, + project_id, + source, + })? { existing } else { - return Err(StatusPageError::Database(insert_error)); + return Err(StatusPageError::ManagedMonitorReconciliation { + operation: "create", + environment_id, + project_id, + source: insert_error, + }); } } + Err(source) => { + return Err(StatusPageError::ManagedMonitorReconciliation { + operation: "create", + environment_id, + project_id, + source, + }); + } }; tracing::info!( @@ -311,7 +438,7 @@ impl MonitorService { result.id, "unknown".to_string(), None, - Some("Monitor created - awaiting first health check".to_string()), + Some(USER_CREATED_MONITOR_BOOTSTRAP_MESSAGE.to_string()), ) .await?; @@ -490,7 +617,7 @@ impl MonitorService { } /// Set or clear the deployment-discovered path on Temps' managed monitor. - /// User-created monitors retain their independently configured endpoints. + /// Custom-named monitors retain their independently configured endpoints. pub async fn update_managed_check_path_for_environment( &self, project_id: i32, @@ -501,19 +628,22 @@ impl MonitorService { validate_check_path(check_path)?; } - let monitors = status_monitors::Entity::find() + status_monitors::Entity::update_many() + .col_expr( + status_monitors::Column::CheckPath, + Expr::value(check_path.map(str::to_string)), + ) + .col_expr( + status_monitors::Column::CheckPathRevision, + Expr::col(status_monitors::Column::CheckPathRevision).add(1_i64), + ) + .col_expr(status_monitors::Column::UpdatedAt, Expr::value(Utc::now())) .filter(status_monitors::Column::ProjectId.eq(project_id)) .filter(status_monitors::Column::EnvironmentId.eq(Some(environment_id))) .filter(status_monitors::Column::IsManaged.eq(true)) - .all(self.db.as_ref()) + .exec(self.db.as_ref()) .await?; - for monitor in monitors { - let mut active: status_monitors::ActiveModel = monitor.into(); - active.check_path = Set(check_path.map(str::to_string)); - active.update(self.db.as_ref()).await?; - } - Ok(()) } @@ -1214,6 +1344,18 @@ mod tests { .update_managed_check_path_for_environment(project.id, environment.id, Some("/docs")) .await .unwrap(); + let changed_managed = status_monitors::Entity::find_by_id(managed.id) + .one(db.as_ref()) + .await + .unwrap() + .unwrap(); + let unchanged_custom = status_monitors::Entity::find_by_id(custom.id) + .one(db.as_ref()) + .await + .unwrap() + .unwrap(); + assert_eq!(changed_managed.check_path_revision, 1); + assert_eq!(unchanged_custom.check_path_revision, 0); assert_eq!( service .get_monitor(managed.id) @@ -1233,10 +1375,35 @@ mod tests { Some("/api/ready") ); + service + .update_managed_check_path_for_environment(project.id, environment.id, Some("/docs")) + .await + .unwrap(); + let equal_value_write = status_monitors::Entity::find_by_id(managed.id) + .one(db.as_ref()) + .await + .unwrap() + .unwrap(); + assert_eq!(equal_value_write.check_path.as_deref(), Some("/docs")); + assert_eq!(equal_value_write.check_path_revision, 2); + service .update_managed_check_path_for_environment(project.id, environment.id, None) .await .unwrap(); + let cleared_managed = status_monitors::Entity::find_by_id(managed.id) + .one(db.as_ref()) + .await + .unwrap() + .unwrap(); + let still_unchanged_custom = status_monitors::Entity::find_by_id(custom.id) + .one(db.as_ref()) + .await + .unwrap() + .unwrap(); + assert_eq!(cleared_managed.check_path, None); + assert_eq!(cleared_managed.check_path_revision, 3); + assert_eq!(still_unchanged_custom.check_path_revision, 0); assert_eq!( service.get_monitor(managed.id).await.unwrap().check_path, None @@ -1253,7 +1420,71 @@ mod tests { } #[tokio::test] - async fn managed_monitor_does_not_claim_a_user_monitor_with_the_default_name() { + async fn managed_monitor_adopts_a_legacy_monitor_with_the_default_name() { + let Ok(test_db) = TestDatabase::with_migrations().await else { + println!("Docker not available, skipping"); + return; + }; + let db = test_db.connection_arc(); + let service = MonitorService::new(db.clone(), create_mock_config_service(&db)); + let project = create_test_project(&db).await; + let environment = create_test_environment(&db, project.id).await; + let legacy_monitor = status_monitors::ActiveModel { + project_id: Set(project.id), + environment_id: Set(Some(environment.id)), + name: Set(format!("{} Monitor", environment.name)), + monitor_type: Set("web".to_string()), + check_path: Set(Some("/legacy-health".to_string())), + check_interval_seconds: Set(60), + is_active: Set(true), + is_managed: Set(false), + ..Default::default() + } + .insert(db.as_ref()) + .await + .unwrap(); + + let managed_monitor = service + .ensure_monitor_for_environment(project.id, environment.id, &environment.name) + .await + .unwrap(); + + assert_eq!(managed_monitor.id, legacy_monitor.id); + let restarted_service = MonitorService::new(db.clone(), create_mock_config_service(&db)); + let after_restart = restarted_service + .ensure_monitor_for_environment(project.id, environment.id, &environment.name) + .await + .unwrap(); + assert_eq!(after_restart.id, legacy_monitor.id); + service + .update_managed_check_path_for_environment( + project.id, + environment.id, + Some("/deployed-health"), + ) + .await + .unwrap(); + assert_eq!( + service + .get_monitor(legacy_monitor.id) + .await + .unwrap() + .check_path + .as_deref(), + Some("/deployed-health") + ); + + let monitors = status_monitors::Entity::find() + .filter(status_monitors::Column::EnvironmentId.eq(Some(environment.id))) + .all(db.as_ref()) + .await + .unwrap(); + assert_eq!(monitors.len(), 1); + assert!(monitors[0].is_managed); + } + + #[tokio::test] + async fn ensure_monitor_for_environment_adopts_an_existing_default_name() { let Ok(test_db) = TestDatabase::with_migrations().await else { println!("Docker not available, skipping"); return; @@ -1269,7 +1500,7 @@ mod tests { name: format!("{} Monitor", environment.name), monitor_type: "web".to_string(), environment_id: environment.id, - check_interval_seconds: Some(60), + check_interval_seconds: Some(90), check_path: Some("/user-health".to_string()), }, ) @@ -1281,33 +1512,82 @@ mod tests { .await .unwrap(); - assert_ne!(managed_monitor.id, user_monitor.id); + assert_eq!(managed_monitor.id, user_monitor.id); + assert_eq!(managed_monitor.check_path.as_deref(), Some("/user-health")); + assert_eq!(managed_monitor.check_interval_seconds, 90); + let managed_row = status_monitors::Entity::find_by_id(managed_monitor.id) + .one(db.as_ref()) + .await + .unwrap() + .unwrap(); + assert!(managed_row.is_managed); service .update_managed_check_path_for_environment( project.id, environment.id, - Some("/deployed-health"), + Some("/from-temps-yaml"), ) .await .unwrap(); - assert_eq!( - service - .get_monitor(user_monitor.id) - .await - .unwrap() - .check_path - .as_deref(), - Some("/user-health") + let updated = service.get_monitor(user_monitor.id).await.unwrap(); + assert_eq!(updated.check_path.as_deref(), Some("/from-temps-yaml")); + + let monitor_count = status_monitors::Entity::find() + .filter(status_monitors::Column::EnvironmentId.eq(Some(environment.id))) + .count(db.as_ref()) + .await + .unwrap(); + assert_eq!(monitor_count, 1); + } + + #[tokio::test] + async fn ensure_monitor_for_environment_rejects_an_environment_from_another_project() { + let Ok(test_db) = TestDatabase::with_migrations().await else { + println!("Docker not available, skipping"); + return; + }; + let db = test_db.connection_arc(); + let service = MonitorService::new(db.clone(), create_mock_config_service(&db)); + let owning_project = create_test_project(&db).await; + let other_project = create_test_project(&db).await; + let environment = create_test_environment(&db, owning_project.id).await; + + let result = service + .ensure_monitor_for_environment(other_project.id, environment.id, &environment.name) + .await; + + assert!(matches!( + result, + Err(StatusPageError::EnvironmentNotInProject { + environment_id, + project_id, + }) if environment_id == environment.id && project_id == other_project.id + )); + let monitor_count = status_monitors::Entity::find() + .filter(status_monitors::Column::EnvironmentId.eq(Some(environment.id))) + .count(db.as_ref()) + .await + .unwrap(); + assert_eq!(monitor_count, 0); + } + + #[test] + fn managed_monitor_unique_violation_requires_the_expected_constraint() { + let expected = DbErr::Custom( + "database error 23505: duplicate key violates constraint \ + idx_status_monitors_managed_environment" + .to_string(), ); - assert_eq!( - service - .get_monitor(managed_monitor.id) - .await - .unwrap() - .check_path - .as_deref(), - Some("/deployed-health") + let unrelated = DbErr::Custom( + "database error 23505: duplicate key violates constraint other_index".to_string(), ); + let connection = DbErr::Custom( + "connection failed while reading idx_status_monitors_managed_environment".to_string(), + ); + + assert!(is_managed_monitor_unique_violation(&expected)); + assert!(!is_managed_monitor_unique_violation(&unrelated)); + assert!(!is_managed_monitor_unique_violation(&connection)); } #[tokio::test] diff --git a/crates/temps-status-page/src/services/types.rs b/crates/temps-status-page/src/services/types.rs index 7d65d164b..2c07789ad 100644 --- a/crates/temps-status-page/src/services/types.rs +++ b/crates/temps-status-page/src/services/types.rs @@ -40,6 +40,16 @@ pub enum StatusPageError { #[source] source: sea_orm::DbErr, }, + #[error( + "failed to {operation} the managed monitor for environment {environment_id} in project {project_id}: {source}" + )] + ManagedMonitorReconciliation { + operation: &'static str, + environment_id: i32, + project_id: i32, + #[source] + source: sea_orm::DbErr, + }, #[error( "failed to validate monitor {monitor_id} ownership for project {project_id}: {source}" )]