From 3279b7ada24423e50da69788643b715d456c5459 Mon Sep 17 00:00:00 2001 From: BernardLee Date: Sat, 24 Jan 2026 14:58:10 +0800 Subject: [PATCH] feat: implement hot_standby and OpLog interfaces --- dependencies.sh | 1 + mooncake-store/include/etcd_oplog_store.h | 202 ++++++ mooncake-store/include/ha_metric_manager.h | 192 ++++++ mooncake-store/include/hot_standby_service.h | 244 +++++++ mooncake-store/include/metadata_store.h | 113 +++ mooncake-store/include/oplog_applier.h | 170 +++++ mooncake-store/include/oplog_manager.h | 139 ++++ mooncake-store/include/oplog_watcher.h | 158 +++++ mooncake-store/include/snapshot_provider.h | 55 ++ .../include/standby_state_machine.h | 334 +++++++++ mooncake-store/include/types.h | 61 ++ mooncake-store/src/CMakeLists.txt | 23 +- mooncake-store/src/etcd_oplog_store.cpp | 651 ++++++++++++++++++ mooncake-store/src/ha_metric_manager.cpp | 281 ++++++++ mooncake-store/src/hot_standby_service.cpp | 605 ++++++++++++++++ mooncake-store/src/oplog_applier.cpp | 584 ++++++++++++++++ mooncake-store/src/oplog_manager.cpp | 171 +++++ mooncake-store/src/oplog_watcher.cpp | 610 ++++++++++++++++ mooncake-store/src/standby_state_machine.cpp | 283 ++++++++ 19 files changed, 4875 insertions(+), 2 deletions(-) create mode 100644 mooncake-store/include/etcd_oplog_store.h create mode 100644 mooncake-store/include/ha_metric_manager.h create mode 100644 mooncake-store/include/hot_standby_service.h create mode 100644 mooncake-store/include/metadata_store.h create mode 100644 mooncake-store/include/oplog_applier.h create mode 100644 mooncake-store/include/oplog_manager.h create mode 100644 mooncake-store/include/oplog_watcher.h create mode 100644 mooncake-store/include/snapshot_provider.h create mode 100644 mooncake-store/include/standby_state_machine.h create mode 100644 mooncake-store/src/etcd_oplog_store.cpp create mode 100644 mooncake-store/src/ha_metric_manager.cpp create mode 100644 mooncake-store/src/hot_standby_service.cpp create mode 100644 mooncake-store/src/oplog_applier.cpp create mode 100644 mooncake-store/src/oplog_manager.cpp create mode 100644 mooncake-store/src/oplog_watcher.cpp create mode 100644 mooncake-store/src/standby_state_machine.cpp diff --git a/dependencies.sh b/dependencies.sh index e4f3a8589e..f241ba2be7 100755 --- a/dependencies.sh +++ b/dependencies.sh @@ -124,6 +124,7 @@ SYSTEM_PACKAGES="build-essential \ libhiredis-dev \ liburing-dev \ libjemalloc-dev \ + libxxhash-dev \ pkg-config \ patchelf \ libc6-dev \ diff --git a/mooncake-store/include/etcd_oplog_store.h b/mooncake-store/include/etcd_oplog_store.h new file mode 100644 index 0000000000..013c38eea3 --- /dev/null +++ b/mooncake-store/include/etcd_oplog_store.h @@ -0,0 +1,202 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "oplog_manager.h" +#include "types.h" + +namespace mooncake { + +/** + * @brief Store for OpLog entries in etcd. + * + * This class is responsible for writing OpLog entries to etcd and reading them back. + * OpLog entries are stored with keys in the format: + * /oplog/{cluster_id}/{sequence_id} + * + * The latest sequence_id is also stored at: + * /oplog/{cluster_id}/latest + */ +class EtcdOpLogStore { + public: + /** + * @brief Constructor. + * @param cluster_id: The cluster ID for this OpLog store. + * @param enable_latest_seq_batch_update: Whether to start background thread + * to batch-update `/latest`. Readers (Standby) should set this to false + * to avoid unnecessary thread creation. + */ + explicit EtcdOpLogStore(const std::string& cluster_id, + bool enable_latest_seq_batch_update = false); + + /** + * @brief Write an OpLog entry to etcd. + * @param entry: The OpLog entry to write. + * @return: Error code. + */ + ErrorCode WriteOpLog(const OpLogEntry& entry); + + /** + * @brief Read an OpLog entry from etcd by sequence_id. + * @param sequence_id: The sequence ID of the entry to read. + * @param entry: Output param, the OpLog entry. + * @return: Error code. + */ + ErrorCode ReadOpLog(uint64_t sequence_id, OpLogEntry& entry); + + /** + * @brief Read OpLog entries starting from a given sequence_id. + * @param start_sequence_id: The starting sequence ID (exclusive). + * @param limit: Maximum number of entries to read (default: 1000). + * @param entries: Output param, vector of OpLog entries. + * @return: Error code. + */ + ErrorCode ReadOpLogSince(uint64_t start_sequence_id, size_t limit, + std::vector& entries); + + // Like ReadOpLogSince, but also returns the etcd revision for consistent + // "read then watch(from revision+1)" startup. + ErrorCode ReadOpLogSinceWithRevision(uint64_t start_sequence_id, size_t limit, + std::vector& entries, + EtcdRevisionId& revision_id); + + /** + * @brief Get the latest sequence_id from etcd. + * @param sequence_id: Output param, the latest sequence_id. + * @return: Error code. ETCD_KEY_NOT_EXIST if no OpLog exists yet. + */ + ErrorCode GetLatestSequenceId(uint64_t& sequence_id); + + // Stronger (than `/latest`) best-effort query: return the maximum existing + // sequence_id by scanning etcd keys under /oplog/{cluster_id}/ with + // descending key order. + // Return ETCD_KEY_NOT_EXIST if no OpLog exists yet. + ErrorCode GetMaxSequenceId(uint64_t& sequence_id); + + /** + * @brief Update the latest sequence_id in etcd. + * @param sequence_id: The latest sequence_id to update. + * @return: Error code. + */ + ErrorCode UpdateLatestSequenceId(uint64_t sequence_id); + + /** + * @brief Record the sequence_id corresponding to a snapshot. + * @param snapshot_id: The snapshot ID. + * @param sequence_id: The sequence_id at which the snapshot was taken. + * @return: Error code. + */ + ErrorCode RecordSnapshotSequenceId(const std::string& snapshot_id, + uint64_t sequence_id); + + /** + * @brief Get the sequence_id for a given snapshot. + * @param snapshot_id: The snapshot ID. + * @param sequence_id: Output param, the sequence_id. + * @return: Error code. ETCD_KEY_NOT_EXIST if snapshot not found. + */ + ErrorCode GetSnapshotSequenceId(const std::string& snapshot_id, + uint64_t& sequence_id); + + /** + * @brief Clean up OpLog entries before a given sequence_id. + * @param before_sequence_id: All entries with sequence_id < before_sequence_id + * will be deleted. + * @return: Error code. + */ + ErrorCode CleanupOpLogBefore(uint64_t before_sequence_id); + + /** + * @brief Destructor - stops batch update thread. + */ + ~EtcdOpLogStore(); + + private: + /** + * @brief Build the etcd key for an OpLog entry. + * @param sequence_id: The sequence ID. + * @return: The etcd key. + */ + std::string BuildOpLogKey(uint64_t sequence_id) const; + + /** + * @brief Build the etcd key for the latest sequence_id. + * @return: The etcd key. + */ + std::string BuildLatestKey() const; + + /** + * @brief Build the etcd key for a snapshot sequence_id. + * @param snapshot_id: The snapshot ID. + * @return: The etcd key. + */ + std::string BuildSnapshotKey(const std::string& snapshot_id) const; + + // Best-effort: find the minimum existing OpLog sequence_id in etcd. + // Used for robust cleanup (Scheme 3) so we don't rely on a persisted + // "cleaned_upto" marker. + std::optional GetMinSequenceId() const; + + // Best-effort: find the maximum existing OpLog sequence_id in etcd. + std::optional GetMaxSequenceIdInternal() const; + + /** + * @brief Serialize an OpLogEntry to JSON string. + * @param entry: The OpLog entry to serialize. + * @return: The JSON string. + */ + std::string SerializeOpLogEntry(const OpLogEntry& entry) const; + + /** + * @brief Deserialize a JSON string to OpLogEntry. + * @param json_str: The JSON string. + * @param entry: Output param, the OpLog entry. + * @return: true if successful, false otherwise. + */ + bool DeserializeOpLogEntry(const std::string& json_str, + OpLogEntry& entry) const; + + /** + * @brief Batch update thread function. + * Periodically updates latest_sequence_id in etcd. + */ + void BatchUpdateThread(); + + /** + * @brief Trigger immediate batch update if threshold is reached. + */ + void TriggerBatchUpdateIfNeeded(); + + /** + * @brief Perform the actual batch update to etcd. + */ + void DoBatchUpdate(); + + std::string cluster_id_; + static constexpr const char* kOpLogPrefix = "/oplog/"; + static constexpr const char* kLatestSuffix = "/latest"; + static constexpr const char* kSnapshotPrefix = "/oplog/"; + static constexpr const char* kSnapshotSuffix = "/snapshot/"; + + // Batch update mechanism for latest_sequence_id + const bool enable_latest_seq_batch_update_{false}; + std::atomic pending_latest_seq_id_{0}; + std::atomic pending_count_{0}; + std::atomic batch_update_running_{false}; + std::mutex batch_update_mutex_; + std::thread batch_update_thread_; + std::chrono::steady_clock::time_point last_update_time_; + + // Batch update configuration + static constexpr size_t kBatchSize = 100; // Update every 100 entries + static constexpr int kBatchIntervalMs = 1000; // Or every 1 second +}; + +} // namespace mooncake diff --git a/mooncake-store/include/ha_metric_manager.h b/mooncake-store/include/ha_metric_manager.h new file mode 100644 index 0000000000..feebcc59ed --- /dev/null +++ b/mooncake-store/include/ha_metric_manager.h @@ -0,0 +1,192 @@ +#pragma once + +#include +#include +#include +#include + +#include "ylt/metric/counter.hpp" +#include "ylt/metric/gauge.hpp" +#include "ylt/metric/histogram.hpp" + +namespace mooncake { + +/** + * @brief Singleton manager for High Availability (HA) related metrics. + * + * This class provides metrics for monitoring the health and performance + * of the OpLog replication system, including: + * - OpLog sequence tracking + * - Standby replication lag + * - Error counters (checksum failures, skipped entries) + * - Performance histograms (etcd write latency) + * - Queue sizes (pending mutations) + */ +class HAMetricManager { + public: + // --- Singleton Access --- + static HAMetricManager& instance(); + + HAMetricManager(const HAMetricManager&) = delete; + HAMetricManager& operator=(const HAMetricManager&) = delete; + HAMetricManager(HAMetricManager&&) = delete; + HAMetricManager& operator=(HAMetricManager&&) = delete; + + // ========== OpLog Sequence Metrics (Gauge) ========== + + /** + * @brief Set the latest OpLog sequence ID on Primary + */ + void set_oplog_last_sequence_id(int64_t seq_id); + int64_t get_oplog_last_sequence_id(); + + /** + * @brief Set the Standby's applied sequence ID + */ + void set_oplog_applied_sequence_id(int64_t seq_id); + int64_t get_oplog_applied_sequence_id(); + + /** + * @brief Set the replication lag (entries behind Primary) + */ + void set_oplog_standby_lag(int64_t lag); + int64_t get_oplog_standby_lag(); + + /** + * @brief Set the number of pending (out-of-order) entries in OpLogApplier + */ + void set_oplog_pending_entries(int64_t count); + int64_t get_oplog_pending_entries(); + + /** + * @brief Set the pending mutation queue size (retry queue) + */ + void set_pending_mutation_queue_size(int64_t size); + int64_t get_pending_mutation_queue_size(); + + // ========== Error Counters ========== + + /** + * @brief Increment counter for skipped OpLog entries + */ + void inc_oplog_skipped_entries(int64_t val = 1); + int64_t get_oplog_skipped_entries_total(); + + /** + * @brief Increment counter for checksum verification failures + */ + void inc_oplog_checksum_failures(int64_t val = 1); + int64_t get_oplog_checksum_failures_total(); + + /** + * @brief Increment counter for gap resolve attempts + */ + void inc_oplog_gap_resolve_attempts(int64_t val = 1); + int64_t get_oplog_gap_resolve_attempts_total(); + + /** + * @brief Increment counter for successful gap resolves + */ + void inc_oplog_gap_resolve_success(int64_t val = 1); + int64_t get_oplog_gap_resolve_success_total(); + + /** + * @brief Increment counter for etcd write failures + */ + void inc_oplog_etcd_write_failures(int64_t val = 1); + int64_t get_oplog_etcd_write_failures_total(); + + /** + * @brief Increment counter for etcd write retries + */ + void inc_oplog_etcd_write_retries(int64_t val = 1); + int64_t get_oplog_etcd_write_retries_total(); + + /** + * @brief Increment counter for watch disconnections + */ + void inc_oplog_watch_disconnections(int64_t val = 1); + int64_t get_oplog_watch_disconnections_total(); + + /** + * @brief Increment counter for successfully applied OpLog entries + */ + void inc_oplog_applied_entries(int64_t val = 1); + int64_t get_oplog_applied_entries_total(); + + // ========== Latency Histograms ========== + + /** + * @brief Record etcd write latency in microseconds + */ + void observe_oplog_etcd_write_latency_us(int64_t latency_us); + + /** + * @brief Record OpLog apply latency in microseconds + */ + void observe_oplog_apply_latency_us(int64_t latency_us); + + // ========== State Machine Metrics ========== + + /** + * @brief Set the current Standby state (as integer for Prometheus) + * @param state_value Integer representation of StandbyState + */ + void set_standby_state(int64_t state_value); + int64_t get_standby_state(); + + /** + * @brief Increment state transition counter + */ + void inc_state_transitions(int64_t val = 1); + int64_t get_state_transitions_total(); + + // ========== Serialization ========== + + /** + * @brief Serializes all HA metrics into Prometheus text format. + * @return A string containing the metrics in Prometheus format. + */ + std::string serialize_metrics(); + + /** + * @brief Generates a concise, human-readable summary of HA metrics. + * @return A string containing the formatted summary. + */ + std::string get_summary_string(); + + private: + // --- Private Constructor & Destructor --- + HAMetricManager(); + ~HAMetricManager() = default; + + // --- Metric Members --- + + // OpLog Sequence Gauges + ylt::metric::gauge_t oplog_last_sequence_id_; + ylt::metric::gauge_t oplog_applied_sequence_id_; + ylt::metric::gauge_t oplog_standby_lag_; + ylt::metric::gauge_t oplog_pending_entries_; + ylt::metric::gauge_t pending_mutation_queue_size_; + + // Error Counters + ylt::metric::counter_t oplog_skipped_entries_total_; + ylt::metric::counter_t oplog_checksum_failures_total_; + ylt::metric::counter_t oplog_gap_resolve_attempts_total_; + ylt::metric::counter_t oplog_gap_resolve_success_total_; + ylt::metric::counter_t oplog_etcd_write_failures_total_; + ylt::metric::counter_t oplog_etcd_write_retries_total_; + ylt::metric::counter_t oplog_watch_disconnections_total_; + ylt::metric::counter_t oplog_applied_entries_total_; + + // Latency Histograms (buckets in microseconds: 100us, 500us, 1ms, 5ms, 10ms, 50ms, 100ms, 500ms, 1s) + ylt::metric::histogram_t oplog_etcd_write_latency_us_; + ylt::metric::histogram_t oplog_apply_latency_us_; + + // State Machine + ylt::metric::gauge_t standby_state_; + ylt::metric::counter_t state_transitions_total_; +}; + +} // namespace mooncake + diff --git a/mooncake-store/include/hot_standby_service.h b/mooncake-store/include/hot_standby_service.h new file mode 100644 index 0000000000..29544775ed --- /dev/null +++ b/mooncake-store/include/hot_standby_service.h @@ -0,0 +1,244 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "metadata_store.h" +#include "oplog_applier.h" +#include "oplog_manager.h" +#include "oplog_watcher.h" +#include "snapshot_provider.h" +#include "standby_state_machine.h" +#include "types.h" + +namespace mooncake { + +// Forward declarations +class MasterService; +class ReplicationStream; + +/** + * @brief Configuration for HotStandbyService + */ +struct HotStandbyConfig { + std::string standby_id; + std::string primary_address; + uint32_t replication_port{0}; + uint32_t verification_interval_sec{30}; + uint32_t max_replication_lag_entries{1000}; + bool enable_verification{true}; + + // Snapshot bootstrap (optional): + // If provided, Standby will try to load a snapshot first, then replay OpLog + // from snapshot_sequence_id. + bool enable_snapshot_bootstrap{false}; +}; + +/** + * @brief Sync status information for HotStandbyService + */ +struct StandbySyncStatus { + uint64_t applied_seq_id{0}; + uint64_t primary_seq_id{0}; + uint64_t lag_entries{0}; + std::chrono::milliseconds lag_time{0}; + bool is_syncing{false}; + bool is_connected{false}; + StandbyState state{StandbyState::STOPPED}; + std::chrono::milliseconds time_in_state{0}; +}; + +/** + * @brief HotStandbyService manages standby replication and promotion + * + * This service runs on Standby Master nodes and is responsible for: + * - Connecting to Primary and receiving OpLog entries + * - Applying OpLog entries to local metadata store + * - Periodically verifying data consistency with Primary + * - Promoting to Primary when elected as new Leader + * + * For now, this is a skeleton implementation without actual network + * communication. The gRPC integration will be added later. + */ +class HotStandbyService { + public: + explicit HotStandbyService(const HotStandbyConfig& config); + ~HotStandbyService(); + + /** + * @brief Start connecting to Primary and begin replication + * @param primary_address Address of the Primary Master (not used with etcd-based sync) + * @param etcd_endpoints Comma-separated etcd endpoints + * @param cluster_id Cluster identifier for OpLog path + * @return ErrorCode::OK on success + */ + ErrorCode Start(const std::string& primary_address, + const std::string& etcd_endpoints, + const std::string& cluster_id); + + /** + * @brief Stop replication and disconnect from Primary + */ + void Stop(); + + /** + * @brief Get current synchronization status + * @return StandbySyncStatus with current sync state + */ + StandbySyncStatus GetSyncStatus() const; + + /** + * @brief Check if standby is ready for promotion + * @return true if replication lag is within threshold + */ + bool IsReadyForPromotion() const; + + /** + * @brief Promote this standby to Primary + * + * This method should be called after successful leader election. + * It returns a MasterService instance initialized with the replicated + * metadata, ready to serve as the new Primary. + * + * @return Unique pointer to MasterService, or nullptr on failure + */ + std::unique_ptr Promote(); + + /** + * @brief Get the number of metadata entries in the local store + */ + size_t GetMetadataCount() const; + + /** + * @brief Get the latest applied sequence ID after promotion + * + * This should be called after Promote() to get the sequence_id + * that the new Primary's OpLogManager should start from. + * + * @return Latest applied sequence ID, or 0 if not available + */ + uint64_t GetLatestAppliedSequenceId() const; + + // Export a point-in-time snapshot of all replicated metadata. + // This is used by MasterServiceSupervisor to initialize the new Primary + // after leader election (fast recovery). + bool ExportMetadataSnapshot( + std::vector>& out) const; + + // Inject a snapshot provider (from external snapshot implementation). + void SetSnapshotProvider(std::unique_ptr provider); + + /** + * @brief Get current state from state machine + */ + StandbyState GetState() const { return state_machine_.GetState(); } + + /** + * @brief Get state machine for monitoring/debugging + */ + const StandbyStateMachine& GetStateMachine() const { return state_machine_; } + + /** + * @brief Callback for OpLogWatcher state changes + * @param event The event to process + */ + void OnWatcherEvent(StandbyEvent event); + + private: + /** + * @brief Main replication loop (runs in background thread) + */ + void ReplicationLoop(); + + /** + * @brief Verification loop (runs in background thread) + */ + void VerificationLoop(); + + /** + * @brief Apply a single OpLog entry to local metadata store + * @param entry The OpLog entry to apply + * @deprecated Use OpLogApplier instead + */ + void ApplyOpLogEntry(const OpLogEntry& entry); + + /** + * @brief Connect to Primary and establish replication stream + * @return true on success, false on failure + */ + bool ConnectToPrimary(); + + /** + * @brief Disconnect from Primary + */ + void DisconnectFromPrimary(); + + /** + * @brief Process a batch of OpLog entries received from Primary + * @param entries Batch of OpLog entries + */ + void ProcessOpLogBatch(const std::vector& entries); + + HotStandbyConfig config_; + + // Simple in-memory metadata store implementation + class StandbyMetadataStore : public MetadataStore { + public: + bool PutMetadata(const std::string& key, + const StandbyObjectMetadata& metadata) override; + bool Put(const std::string& key, + const std::string& payload = std::string()) override; + const StandbyObjectMetadata* GetMetadata(const std::string& key) const override; + bool Remove(const std::string& key) override; + bool Exists(const std::string& key) const override; + size_t GetKeyCount() const override; + + // Snapshot for promotion/restore. + void Snapshot( + std::vector>& out) const; + + private: + mutable std::mutex mutex_; + std::unordered_map store_; + }; + std::unique_ptr metadata_store_; + std::unique_ptr snapshot_provider_{std::make_unique()}; + + // OpLog replication components + std::unique_ptr oplog_applier_; + std::unique_ptr oplog_watcher_; + + // Configuration for etcd-based OpLog sync + std::string etcd_endpoints_; + std::string cluster_id_; + + // Replication state + std::shared_ptr replication_stream_; + std::atomic applied_seq_id_{0}; + std::atomic primary_seq_id_{0}; + + // State machine for managing service lifecycle + StandbyStateMachine state_machine_; + + // Helper methods for state machine + bool IsRunning() const { return state_machine_.IsRunning(); } + bool IsConnected() const { return state_machine_.IsConnected(); } + + // Background threads + std::thread replication_thread_; + std::thread verification_thread_; + + // Synchronization + mutable std::mutex mutex_; +}; + +} // namespace mooncake + diff --git a/mooncake-store/include/metadata_store.h b/mooncake-store/include/metadata_store.h new file mode 100644 index 0000000000..1e9066c685 --- /dev/null +++ b/mooncake-store/include/metadata_store.h @@ -0,0 +1,113 @@ +#pragma once + +#include +#include +#include +#include + +#include "replica.h" +#include "types.h" + +namespace mooncake { + +/** + * @brief Metadata structure for Standby to store and restore object information + * + * This structure contains all essential metadata information needed by Standby + * to immediately serve as Primary when promoted. + */ +struct StandbyObjectMetadata { + UUID client_id{0, 0}; + uint64_t size{0}; + std::vector replicas; + // NOTE: Lease information is NOT stored because: + // 1. Standby does not perform eviction, so lease info is not used + // 2. After promotion, new Primary should grant fresh leases, not restore old ones + uint64_t last_sequence_id{0}; // Last OpLog sequence ID that modified this key + + StandbyObjectMetadata() = default; + + // Check if this metadata has valid replicas + bool HasReplicas() const { return !replicas.empty(); } +}; + +/** + * @brief Payload structure for struct_pack serialization (msgpack binary format) + * + * Now uses UUID directly since struct_pack natively supports std::pair. + */ +struct MetadataPayload { + UUID client_id{0, 0}; + uint64_t size{0}; + std::vector replicas; + // NOTE: Lease information removed - not needed by Standby + + YLT_REFL(MetadataPayload, client_id, size, replicas); + + // Convert to StandbyObjectMetadata + StandbyObjectMetadata ToStandbyMetadata(uint64_t sequence_id) const { + StandbyObjectMetadata meta; + meta.client_id = client_id; + meta.size = size; + meta.replicas = replicas; + meta.last_sequence_id = sequence_id; + return meta; + } +}; + +/** + * @brief Abstract interface for metadata storage on Standby + * + * This interface provides basic operations for storing and managing object metadata. + * In a full implementation, this would mirror MasterService's metadata_shards_ structure. + */ +class MetadataStore { + public: + virtual ~MetadataStore() = default; + + /** + * @brief Put or update metadata for a key with structured metadata + * @param key Object key + * @param metadata Structured metadata object + * @return true on success, false on failure + */ + virtual bool PutMetadata(const std::string& key, const StandbyObjectMetadata& metadata) = 0; + + /** + * @brief Put or update metadata for a key (legacy interface for backward compatibility) + * @param key Object key + * @param payload Optional payload data (JSON serialized metadata) + * @return true on success, false on failure + */ + virtual bool Put(const std::string& key, const std::string& payload = std::string()) = 0; + + /** + * @brief Get metadata for a key + * @param key Object key + * @return Pointer to metadata if found, nullptr otherwise + */ + virtual const StandbyObjectMetadata* GetMetadata(const std::string& key) const = 0; + + /** + * @brief Remove metadata for a key + * @param key Object key + * @return true if key was found and removed, false otherwise + */ + virtual bool Remove(const std::string& key) = 0; + + /** + * @brief Check if a key exists + * @param key Object key + * @return true if key exists, false otherwise + */ + virtual bool Exists(const std::string& key) const = 0; + + /** + * @brief Get the count of keys in the store + * @return Number of keys + */ + virtual size_t GetKeyCount() const = 0; +}; + +} // namespace mooncake + diff --git a/mooncake-store/include/oplog_applier.h b/mooncake-store/include/oplog_applier.h new file mode 100644 index 0000000000..993ef81f5c --- /dev/null +++ b/mooncake-store/include/oplog_applier.h @@ -0,0 +1,170 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "oplog_manager.h" +#include "metadata_store.h" + +namespace mooncake { + +// Forward declaration +class EtcdOpLogStore; + +/** + * @brief Apply OpLog entries to Standby metadata store with ordering guarantee + * + * This class applies OpLog entries to the Standby metadata store, + * ensuring both global and per-key ordering. + */ +class OpLogApplier { + public: + /** + * @brief Constructor + * @param metadata_store Metadata store to apply changes to + * @param cluster_id Cluster ID for accessing etcd OpLog (optional, for requesting missing OpLog) + */ + explicit OpLogApplier(MetadataStore* metadata_store, + const std::string& cluster_id = std::string()); + + /** + * @brief Apply a single OpLog entry (with ordering checks) + * @param entry OpLog entry to apply + * @return true on success, false on failure or ordering violation + */ + bool ApplyOpLogEntry(const OpLogEntry& entry); + + /** + * @brief Apply multiple OpLog entries + * @param entries OpLog entries to apply + * @return Number of successfully applied entries + */ + size_t ApplyOpLogEntries(const std::vector& entries); + + /** + * @brief Get the current sequence ID for a key (DEPRECATED) + * @param key Object key + * @return Always returns 0 - global sequence_id is used for ordering + * @deprecated Use global sequence_id for ordering + */ + uint64_t GetKeySequenceId(const std::string& key) const; + + /** + * @brief Get the expected global sequence ID + * @return Expected global sequence ID + */ + uint64_t GetExpectedSequenceId() const; + + /** + * @brief Recover from a given sequence ID + * @param last_applied_sequence_id Last applied sequence ID + */ + void Recover(uint64_t last_applied_sequence_id); + + /** + * @brief Process pending entries (entries with non-continuous sequence IDs) + * @return Number of entries processed + */ + size_t ProcessPendingEntries(); + + // Promotion helper: + // Try to resolve current gaps ONCE (no waiting) by fetching missing/skipped + // sequence_ids from etcd. If an entry arrives late: + // - REMOVE / PUT_REVOKE: delete the key + // - PUT_END: discard + // + // This is used during Standby promotion so we don't block promotion on gaps, + // but still best-effort clean up potentially stale metadata. + struct GapResolveResult { + size_t attempted{0}; + size_t fetched{0}; + size_t applied_deletes{0}; + }; + GapResolveResult TryResolveGapsOnceForPromotion(size_t max_ids = 1024); + + private: + /** + * @brief Check if the entry's sequence order is valid + * @param entry OpLog entry + * @return true if order is valid, false otherwise + */ + bool CheckSequenceOrder(const OpLogEntry& entry); + + /** + * @brief Apply PUT_END operation + * @param entry OpLog entry + */ + void ApplyPutEnd(const OpLogEntry& entry); + + /** + * @brief Apply PUT_REVOKE operation + * @param entry OpLog entry + */ + void ApplyPutRevoke(const OpLogEntry& entry); + + /** + * @brief Apply REMOVE operation + * @param entry OpLog entry + */ + void ApplyRemove(const OpLogEntry& entry); + + /** + * @brief Request missing OpLog entry from etcd + * @param missing_seq_id Missing sequence ID + * @return true if entry was found and applied, false otherwise + */ + bool RequestMissingOpLog(uint64_t missing_seq_id); + + /** + * @brief Schedule wait for missing entries + * @param missing_seq_id Missing sequence ID + */ + void ScheduleWaitForMissingEntries(uint64_t missing_seq_id); + + MetadataStore* metadata_store_; + + // EtcdOpLogStore for requesting missing OpLog entries (optional) + std::string cluster_id_; + mutable std::mutex etcd_oplog_store_mutex_; + mutable std::unique_ptr etcd_oplog_store_; + + /** + * @brief Get or create EtcdOpLogStore instance (lazy initialization) + * @return Pointer to EtcdOpLogStore, or nullptr if cluster_id is not set + */ + EtcdOpLogStore* GetEtcdOpLogStore() const; + + // Note: key_sequence_map_ has been removed. + // Global sequence_id is sufficient for ordering guarantee. + + // Track pending entries (entries with non-continuous sequence IDs) + mutable std::mutex pending_mutex_; + std::map pending_entries_; + + // Track missing sequence IDs that we're waiting for + std::map missing_sequence_ids_; + + // Sequence IDs we chose to skip (gap-timeout). If the late entry arrives: + // - REMOVE / PUT_REVOKE: delete the key (safe) + // - PUT_END: discard (do not resurrect potentially stale metadata) + std::map skipped_sequence_ids_; + + // Next expected global sequence_id. Read frequently from monitoring thread, + // updated by watch/apply thread. Use atomic to avoid data races. + std::atomic expected_sequence_id_{1}; + + // Constants for missing entry handling + // IMPORTANT: request must happen BEFORE skip, otherwise we will never request. + static constexpr int kMissingEntryRequestSeconds = 1; // request from etcd after 1s + static constexpr int kMissingEntrySkipSeconds = 3; // skip after 3s (avoid global stall) + static constexpr int kMaxPendingEntries = 1000; // Max pending entries before giving up +}; + +} // namespace mooncake + diff --git a/mooncake-store/include/oplog_manager.h b/mooncake-store/include/oplog_manager.h new file mode 100644 index 0000000000..df76ce6eeb --- /dev/null +++ b/mooncake-store/include/oplog_manager.h @@ -0,0 +1,139 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "types.h" + +namespace mooncake { + +// Forward declaration +class EtcdOpLogStore; + +// Operation types for hot-standby replication. +// This is a minimal subset that can be extended later. +enum class OpType : uint8_t { + PUT_END = 1, + PUT_REVOKE = 2, + REMOVE = 3, + // Deprecated: LEASE_RENEW is intentionally not recorded in OpLog in the + // current etcd-based hot-standby design (Standby relies on Primary DELETE operations). + LEASE_RENEW = 4, +}; + +// A single operation log entry. +// Note: Payload contains JSON serialized MetadataPayload (defined in metadata_store.h) +// for PUT_END operations, allowing Standby to restore complete metadata. +struct OpLogEntry { + uint64_t sequence_id{0}; // Monotonically increasing global sequence + uint64_t timestamp_ms{0}; // Logical timestamp in milliseconds + OpType op_type{OpType::PUT_END}; + std::string object_key; // Target object key + std::string payload; // Serialized extra data (optional) + uint32_t checksum{0}; // Checksum of payload (implementation-defined) + uint32_t prefix_hash{0}; // Hash of the entire key (for verification and optimization) +}; + +/** + * @brief In-memory operation log manager. + * + * This class is intentionally simple: it keeps a bounded deque of OpLogEntry + * and provides append / get-since primitives. It can later be extended to + * or to spill to disk if needed. In the new etcd-based design, OpLog will be written to etcd. + */ +class OpLogManager { + public: + OpLogManager(); + + // Set the EtcdOpLogStore for writing OpLog to etcd (optional). + // If not set, OpLog will only be stored in memory buffer. + void SetEtcdOpLogStore(std::shared_ptr etcd_oplog_store); + + // Append a new entry and return the assigned sequence_id. + uint64_t Append(OpType type, const std::string& key, + const std::string& payload = std::string()); + + // Allocate a new OpLogEntry with a reserved sequence_id, append it to the + // in-memory buffer, and return the full entry. + // + // IMPORTANT: This will advance last_seq_id_ even if the caller later fails + // to persist it to etcd. This supports "seq pre-allocation" semantics where + // retries use the same (smaller) sequence_id. + OpLogEntry AllocateEntry(OpType type, const std::string& key, + const std::string& payload = std::string()); + + // Persist an already-allocated entry to etcd using its sequence_id. + // Does NOT modify sequence counters. + ErrorCode PersistEntryToEtcd(const OpLogEntry& entry) const; + + // Append a new entry and durably persist it to etcd (if EtcdOpLogStore is set). + // + // This is intended for operations that may free/reuse memory (e.g. REMOVE), + // where best-effort replication is unsafe: Standby must observe the DELETE + // before promotion, otherwise it may return stale descriptors that point to + // reused memory and cause silent data corruption. + // + // Design (updated for seq pre-allocation): + // - sequence_id is allocated first and never reused. + // - If etcd write fails, caller may retry PersistEntryToEtcd with the same + // entry (sequence_id fixed and "smaller" than later entries). + tl::expected AppendAndPersist( + OpType type, const std::string& key, + const std::string& payload = std::string()); + + // Get the latest assigned sequence id. Returns 0 if no entry exists. + uint64_t GetLastSequenceId() const; + + // Set the initial sequence ID (used when promoting Standby to Primary). + // This ensures the new Primary's OpLogManager continues from the correct sequence_id. + void SetInitialSequenceId(uint64_t sequence_id); + + // Current number of entries in the buffer. + size_t GetEntryCount() const; + + // Verify checksum of an OpLogEntry payload. + // Returns true if checksum matches, false otherwise. + // This is public so OpLogWatcher and OpLogApplier can validate entries. + static bool VerifyChecksum(const OpLogEntry& entry); + + // Basic DoS protection for externally sourced OpLog entries (etcd watch / reads). + // Enforce conservative bounds on key/payload sizes before parsing/applying. + static constexpr size_t kMaxObjectKeySize = 4096; // 4 KiB + static constexpr size_t kMaxPayloadSize = 10 * 1024 * 1024; // 10 MiB + + // Validate OpLogEntry key/payload sizes. If invalid, returns false and + // optionally sets a human-readable reason. + static bool ValidateEntrySize(const OpLogEntry& entry, + std::string* reason = nullptr); + + private: + static uint64_t NowMs(); + static uint32_t ComputeChecksum(const std::string& data); + static uint32_t ComputePrefixHash(const std::string& key); + + mutable std::shared_mutex mutex_; + std::deque buffer_; + uint64_t first_seq_id_{1}; // sequence_id of buffer_.front() + uint64_t last_seq_id_{0}; // last assigned sequence_id + + // Note: We removed key_sequence_map_ and key_remove_time_map_. + // Global sequence_id is sufficient for ordering guarantee. + // All operations are applied in sequence_id order, which ensures consistency. + + // Optional etcd OpLog store for persistent storage + std::shared_ptr etcd_oplog_store_; + + // Simple bounds to avoid unbounded memory growth. + static constexpr size_t kMaxBufferEntries_ = 100000; +}; + +} // namespace mooncake + + diff --git a/mooncake-store/include/oplog_watcher.h b/mooncake-store/include/oplog_watcher.h new file mode 100644 index 0000000000..9c430e87be --- /dev/null +++ b/mooncake-store/include/oplog_watcher.h @@ -0,0 +1,158 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "oplog_manager.h" +#include "standby_state_machine.h" +#include "types.h" + +namespace mooncake { + +// Forward declaration +class OpLogApplier; + +// Callback type for state events +using WatcherStateCallback = std::function; + +/** + * @brief Watch etcd for OpLog changes and apply them to Standby + * + * This class watches etcd for new OpLog entries and forwards them + * to OpLogApplier for processing. + */ +class OpLogWatcher { + public: + /** + * @brief Constructor + * @param etcd_endpoints Comma-separated etcd endpoints + * @param cluster_id Cluster identifier + * @param applier OpLog applier to process entries + */ + OpLogWatcher(const std::string& etcd_endpoints, + const std::string& cluster_id, OpLogApplier* applier); + + ~OpLogWatcher(); + + /** + * @brief Start watching etcd for OpLog changes + */ + void Start(); + + /** + * @brief Start from a known last-applied sequence_id. + * + * It will read historical OpLogs at a consistent etcd revision, then start + * watch from revision+1 to close the gap between "read" and "watch". + */ + bool StartFromSequenceId(uint64_t start_seq_id); + + /** + * @brief Stop watching + */ + void Stop(); + + /** + * @brief Get the last processed sequence ID + * @return Last processed sequence ID + */ + uint64_t GetLastProcessedSequenceId() const; + + /** + * @brief Set callback for state events + * @param callback Callback function to invoke on state events + */ + void SetStateCallback(WatcherStateCallback callback) { + state_callback_ = std::move(callback); + } + + /** + * @brief Check if watch is healthy + */ + bool IsWatchHealthy() const { return watch_healthy_.load(); } + + private: + /** + * @brief Notify state callback + */ + void NotifyStateEvent(StandbyEvent event) { + if (state_callback_) { + state_callback_(event); + } + } + + bool ReadOpLogSince(uint64_t start_seq_id, + std::vector& entries, + EtcdRevisionId& revision_id); + // Callback includes etcd KV mod_revision for precise resume. + static void WatchCallback(void* context, const char* key, size_t key_size, + const char* value, size_t value_size, int event_type, + int64_t mod_revision); + + /** + * @brief Watch etcd OpLog changes (runs in background thread) + */ + void WatchOpLog(); + + /** + * @brief Process a Watch event + * @param key etcd key + * @param value etcd value (JSON string for PUT events, empty for DELETE events) + * @param event_type Event type (0 = PUT, 1 = DELETE) + */ + void HandleWatchEvent(const std::string& key, const std::string& value, + int event_type); + void HandleWatchEvent(const std::string& key, const std::string& value, + int event_type, int64_t mod_revision); + + /** + * @brief Deserialize OpLogEntry from JSON string + * @param json_str JSON string + * @param entry Output OpLog entry + * @return true on success, false on failure + */ + bool DeserializeOpLogEntry(const std::string& json_str, OpLogEntry& entry); + + /** + * @brief Attempt to reconnect after watch failure + */ + void TryReconnect(); + + /** + * @brief Sync missed OpLog entries after reconnection + * @return true if sync was successful + */ + bool SyncMissedEntries(); + + // Next watch revision (0 means from now). Updated by consistent reads. + std::atomic next_watch_revision_{0}; + + std::string etcd_endpoints_; + std::string cluster_id_; + OpLogApplier* applier_; + std::atomic running_{false}; + std::thread watch_thread_; + std::atomic last_processed_sequence_id_{0}; + + // Error handling and recovery + std::atomic consecutive_errors_{0}; + std::atomic reconnect_count_{0}; + std::atomic watch_healthy_{false}; + + // State callback for notifying HotStandbyService + WatcherStateCallback state_callback_; + + // Constants for error handling + static constexpr int kMaxConsecutiveErrors = 10; + static constexpr int kReconnectDelayMs = 1000; + static constexpr int kMaxReconnectDelayMs = 30000; + static constexpr int kSyncBatchSize = 1000; +}; + +} // namespace mooncake + diff --git a/mooncake-store/include/snapshot_provider.h b/mooncake-store/include/snapshot_provider.h new file mode 100644 index 0000000000..ae8041511c --- /dev/null +++ b/mooncake-store/include/snapshot_provider.h @@ -0,0 +1,55 @@ +#pragma once + +#include +#include +#include +#include + +#include "metadata_store.h" + +namespace mooncake { + +/** + * @brief SnapshotProvider is an abstraction for loading metadata snapshots. + * + * Assumption: snapshot functionality exists (implemented by another team), but + * may not be synced into this repo yet. We keep Mooncake-store code progressing + * by depending on this narrow interface. + * + * Snapshot semantics for hot-standby: + * - A snapshot represents a consistent metadata baseline at `snapshot_sequence_id`. + * - Standby should: load snapshot -> recover applier to snapshot_sequence_id -> + * replay OpLog entries with sequence_id > snapshot_sequence_id. + */ +class SnapshotProvider { + public: + virtual ~SnapshotProvider() = default; + + // Load the latest available snapshot for `cluster_id`. + // Returns true on success and fills: + // - snapshot_id: opaque identifier (e.g. timestamp/version) + // - snapshot_sequence_id: global OpLog sequence_id at snapshot boundary + // - snapshot: full metadata baseline as key -> StandbyObjectMetadata + virtual bool LoadLatestSnapshot( + const std::string& cluster_id, std::string& snapshot_id, + uint64_t& snapshot_sequence_id, + std::vector>& snapshot) = 0; +}; + +// Default no-op provider: behaves as if "no snapshot available". +class NoopSnapshotProvider final : public SnapshotProvider { + public: + bool LoadLatestSnapshot( + const std::string& /*cluster_id*/, std::string& snapshot_id, + uint64_t& snapshot_sequence_id, + std::vector>& snapshot) override { + snapshot_id.clear(); + snapshot_sequence_id = 0; + snapshot.clear(); + return false; + } +}; + +} // namespace mooncake + + diff --git a/mooncake-store/include/standby_state_machine.h b/mooncake-store/include/standby_state_machine.h new file mode 100644 index 0000000000..649b462566 --- /dev/null +++ b/mooncake-store/include/standby_state_machine.h @@ -0,0 +1,334 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace mooncake { + +/** + * @brief Standby service states + * + * State transition diagram: + * + * ┌─────────┐ + * │ STOPPED │◄──────────────────────────────────────┐ + * └────┬────┘ │ + * │ Start() │ Stop()/Error + * ▼ │ + * ┌─────────────┐ │ + * │ CONNECTING │◄──────────────────────┐ │ + * └──────┬──────┘ │ │ + * │ Connected │ Reconnect │ + * ▼ │ │ + * ┌─────────────┐ Error/Gap ┌────┴─────┐ │ + * │ SYNCING │─────────────────►│RECOVERING│─────┤ + * └──────┬──────┘ └──────────┘ │ + * │ Sync complete │ + * ▼ │ + * ┌─────────────┐ Watch broken ┌────────────┐ │ + * │ WATCHING │─────────────────►│RECONNECTING│───┤ + * └──────┬──────┘ └────────────┘ │ + * │ Promote() │ + * ▼ │ + * ┌─────────────┐ │ + * │ PROMOTING │───────────────────────────────────┤ + * └──────┬──────┘ │ + * │ Success │ + * ▼ │ + * ┌─────────────┐ │ + * │ PROMOTED │───────────────────────────────────┘ + * └─────────────┘ + */ +enum class StandbyState : uint8_t { + // Initial state, service not started + STOPPED = 0, + + // Connecting to etcd cluster + CONNECTING = 1, + + // Initial sync: reading historical OpLog entries + SYNCING = 2, + + // Normal operation: watching for new OpLog entries + WATCHING = 3, + + // Recovering from error: re-syncing missed entries + RECOVERING = 4, + + // Reconnecting after watch failure + RECONNECTING = 5, + + // Promotion in progress: final catch-up before becoming Primary + PROMOTING = 6, + + // Successfully promoted to Primary + PROMOTED = 7, + + // Fatal error, cannot recover + FAILED = 8, +}; + +/** + * @brief Get human-readable state name + */ +inline const char* StandbyStateToString(StandbyState state) { + switch (state) { + case StandbyState::STOPPED: + return "STOPPED"; + case StandbyState::CONNECTING: + return "CONNECTING"; + case StandbyState::SYNCING: + return "SYNCING"; + case StandbyState::WATCHING: + return "WATCHING"; + case StandbyState::RECOVERING: + return "RECOVERING"; + case StandbyState::RECONNECTING: + return "RECONNECTING"; + case StandbyState::PROMOTING: + return "PROMOTING"; + case StandbyState::PROMOTED: + return "PROMOTED"; + case StandbyState::FAILED: + return "FAILED"; + default: + return "UNKNOWN"; + } +} + +/** + * @brief Events that trigger state transitions + */ +enum class StandbyEvent : uint8_t { + // User/system actions + START, // Start() called + STOP, // Stop() called + PROMOTE, // Promote() called + + // Connection events + CONNECTED, // Successfully connected to etcd + CONNECTION_FAILED, // Failed to connect to etcd + DISCONNECTED, // Connection lost + + // Sync events + SYNC_COMPLETE, // Initial sync completed + SYNC_FAILED, // Sync failed + + // Watch events + WATCH_HEALTHY, // Watch is healthy and receiving events + WATCH_BROKEN, // Watch connection broken + + // Recovery events + RECOVERY_SUCCESS, // Successfully recovered from error + RECOVERY_FAILED, // Recovery failed + + // Promotion events + PROMOTION_SUCCESS, // Successfully promoted + PROMOTION_FAILED, // Promotion failed + + // Error events + MAX_ERRORS_REACHED, // Too many consecutive errors + FATAL_ERROR, // Unrecoverable error +}; + +inline const char* StandbyEventToString(StandbyEvent event) { + switch (event) { + case StandbyEvent::START: + return "START"; + case StandbyEvent::STOP: + return "STOP"; + case StandbyEvent::PROMOTE: + return "PROMOTE"; + case StandbyEvent::CONNECTED: + return "CONNECTED"; + case StandbyEvent::CONNECTION_FAILED: + return "CONNECTION_FAILED"; + case StandbyEvent::DISCONNECTED: + return "DISCONNECTED"; + case StandbyEvent::SYNC_COMPLETE: + return "SYNC_COMPLETE"; + case StandbyEvent::SYNC_FAILED: + return "SYNC_FAILED"; + case StandbyEvent::WATCH_HEALTHY: + return "WATCH_HEALTHY"; + case StandbyEvent::WATCH_BROKEN: + return "WATCH_BROKEN"; + case StandbyEvent::RECOVERY_SUCCESS: + return "RECOVERY_SUCCESS"; + case StandbyEvent::RECOVERY_FAILED: + return "RECOVERY_FAILED"; + case StandbyEvent::PROMOTION_SUCCESS: + return "PROMOTION_SUCCESS"; + case StandbyEvent::PROMOTION_FAILED: + return "PROMOTION_FAILED"; + case StandbyEvent::MAX_ERRORS_REACHED: + return "MAX_ERRORS_REACHED"; + case StandbyEvent::FATAL_ERROR: + return "FATAL_ERROR"; + default: + return "UNKNOWN"; + } +} + +/** + * @brief State transition result + */ +struct StateTransitionResult { + bool allowed{false}; + StandbyState old_state{StandbyState::STOPPED}; + StandbyState new_state{StandbyState::STOPPED}; + std::string reason; +}; + +/** + * @brief Callback for state transition notifications + */ +using StateChangeCallback = + std::function; + +/** + * @brief Standby State Machine + * + * Thread-safe state machine for managing Standby service lifecycle. + * All state transitions are explicit and logged. + */ +class StandbyStateMachine { + public: + StandbyStateMachine(); + + /** + * @brief Get current state (thread-safe) + */ + StandbyState GetState() const { return current_state_.load(std::memory_order_acquire); } + + /** + * @brief Check if in a specific state + */ + bool IsInState(StandbyState state) const { return GetState() == state; } + + /** + * @brief Check if service is running (SYNCING, WATCHING, RECOVERING, RECONNECTING, + * PROMOTING) + */ + bool IsRunning() const { + StandbyState s = GetState(); + return s == StandbyState::SYNCING || s == StandbyState::WATCHING || + s == StandbyState::RECOVERING || s == StandbyState::RECONNECTING || + s == StandbyState::PROMOTING; + } + + /** + * @brief Check if connected to etcd + */ + bool IsConnected() const { + StandbyState s = GetState(); + return s == StandbyState::SYNCING || s == StandbyState::WATCHING || + s == StandbyState::RECOVERING || s == StandbyState::PROMOTING; + } + + /** + * @brief Check if watch is healthy + */ + bool IsWatchHealthy() const { return GetState() == StandbyState::WATCHING; } + + /** + * @brief Check if ready for promotion + */ + bool IsReadyForPromotion() const { return GetState() == StandbyState::WATCHING; } + + /** + * @brief Process an event and perform state transition + * @param event The event to process + * @return Result indicating if transition was allowed and new state + */ + StateTransitionResult ProcessEvent(StandbyEvent event); + + /** + * @brief Register a callback for state change notifications + */ + void RegisterCallback(StateChangeCallback callback); + + /** + * @brief State transition record for debugging + */ + struct TransitionRecord { + std::chrono::steady_clock::time_point timestamp; + StandbyState from_state; + StandbyState to_state; + StandbyEvent event; + }; + + /** + * @brief Get state transition history (for debugging) + */ + std::vector GetTransitionHistory(size_t max_records = 100) const; + + /** + * @brief Get time spent in current state + */ + std::chrono::milliseconds GetTimeInCurrentState() const; + + /** + * @brief Get consecutive error count + */ + int GetConsecutiveErrors() const { return consecutive_errors_.load(); } + + /** + * @brief Increment consecutive error count + * @return New error count + */ + int IncrementErrors(); + + /** + * @brief Reset consecutive error count + */ + void ResetErrors() { consecutive_errors_.store(0); } + + /** + * @brief Get reconnect attempt count + */ + int GetReconnectCount() const { return reconnect_count_.load(); } + + /** + * @brief Increment reconnect count + */ + void IncrementReconnectCount() { reconnect_count_.fetch_add(1); } + + /** + * @brief Reset reconnect count + */ + void ResetReconnectCount() { reconnect_count_.store(0); } + + // Constants + static constexpr int kMaxConsecutiveErrors = 10; + static constexpr int kMaxReconnectAttempts = 100; + + private: + /** + * @brief Check if a transition is valid and get new state + */ + StateTransitionResult ValidateTransition(StandbyState from, StandbyEvent event) const; + + /** + * @brief Notify all registered callbacks + */ + void NotifyCallbacks(StandbyState old_state, StandbyState new_state, StandbyEvent event); + + std::atomic current_state_{StandbyState::STOPPED}; + std::atomic consecutive_errors_{0}; + std::atomic reconnect_count_{0}; + std::chrono::steady_clock::time_point state_enter_time_; + + mutable std::mutex mutex_; + std::vector callbacks_; + std::vector transition_history_; + + static constexpr size_t kMaxHistorySize = 1000; +}; + +} // namespace mooncake + diff --git a/mooncake-store/include/types.h b/mooncake-store/include/types.h index 21ed99852a..f86b2190c5 100644 --- a/mooncake-store/include/types.h +++ b/mooncake-store/include/types.h @@ -7,11 +7,13 @@ #include #include #include +#include #include #include "Slab.h" #include "ylt/struct_json/json_reader.h" #include "ylt/struct_json/json_writer.h" +#include "ylt/struct_pack.hpp" #ifdef STORE_USE_ETCD #include "libetcd_wrapper.h" @@ -23,6 +25,65 @@ namespace mooncake { static constexpr uint64_t WRONG_VERSION = 0; static constexpr uint64_t DEFAULT_VALUE = UINT64_MAX; static constexpr uint64_t ERRNO_BASE = DEFAULT_VALUE - 1000; + +// Sequence ID comparison utilities for wrap-around safety. +// These functions use signed difference to correctly handle uint64_t overflow +// (from UINT64_MAX wrapping to 0). Assumes sequence IDs won't differ by more +// than 2^63, which is reasonable for practical systems. +// +// Example: If sequence_id wraps from UINT64_MAX to 0, then: +// IsSequenceNewer(0, UINT64_MAX) = true (0 is newer after wrap) +// IsSequenceNewer(UINT64_MAX, 0) = false (UINT64_MAX is older before wrap) +// +// Note: We use 'static inline' here to give these small helpers internal +// linkage and avoid any potential ODR/linkage issues in large codebases. +static inline bool IsSequenceNewer(uint64_t a, uint64_t b) { + // Cast to int64_t to get signed difference, then check if positive. + // This correctly handles wrap-around: if a wrapped from UINT64_MAX to 0, + // then (int64_t)(a - b) will be positive (assuming gap < 2^63). + return static_cast(a - b) > 0; +} + +static inline bool IsSequenceOlder(uint64_t a, uint64_t b) { + return static_cast(a - b) < 0; +} + +static inline bool IsSequenceEqual(uint64_t a, uint64_t b) { + return a == b; +} + +static inline bool IsSequenceNewerOrEqual(uint64_t a, uint64_t b) { + return a == b || static_cast(a - b) > 0; +} + +static inline bool IsSequenceOlderOrEqual(uint64_t a, uint64_t b) { + return a == b || static_cast(a - b) < 0; +} + +// Cluster ID validation utilities. +// +// cluster_id is used to construct etcd key prefixes (e.g. "/oplog//..."). +// To avoid key-prefix injection / accidental cross-cluster overlap, we restrict the +// allowed characters to a conservative safe set. We validate the "component" form +// (without trailing slash). Trailing slashes should be normalized away before +// validation. +static inline bool IsValidClusterIdComponent(const std::string& cluster_id) { + if (cluster_id.empty()) { + return false; + } + if (cluster_id.size() > 128) { + return false; + } + for (unsigned char c : cluster_id) { + const bool ok = + (c >= '0' && c <= '9') || (c >= 'A' && c <= 'Z') || + (c >= 'a' && c <= 'z') || c == '_' || c == '-' || c == '.'; + if (!ok) { + return false; + } + } + return true; +} static constexpr uint64_t DEFAULT_DEFAULT_KV_LEASE_TTL = 5000; // in milliseconds static constexpr uint64_t DEFAULT_KV_SOFT_PIN_TTL_MS = diff --git a/mooncake-store/src/CMakeLists.txt b/mooncake-store/src/CMakeLists.txt index 09a560d0bd..db9d344c4e 100644 --- a/mooncake-store/src/CMakeLists.txt +++ b/mooncake-store/src/CMakeLists.txt @@ -16,8 +16,6 @@ set(MOONCAKE_STORE_SOURCES ha_helper.cpp segment.cpp transfer_task.cpp - etcd_helper.cpp - ha_helper.cpp rpc_service.cpp offset_allocator.cpp posix_file.cpp @@ -27,10 +25,28 @@ set(MOONCAKE_STORE_SOURCES http_metadata_server.cpp file_storage.cpp task_manager.cpp + oplog_manager.cpp + etcd_oplog_store.cpp + oplog_watcher.cpp + oplog_applier.cpp + hot_standby_service.cpp + standby_state_machine.cpp + ha_metric_manager.cpp + # replication_service.cpp removed - using etcd-based OpLog sync instead ) set(EXTRA_LIBS "") +# Find xxHash (required for ComputeChecksum) +find_path(XXHASH_INCLUDE_DIR NAMES xxhash.h PATHS /usr/include /usr/local/include) +find_library(XXHASH_LIBRARY NAMES xxhash libxxhash PATHS /usr/lib /usr/local/lib /usr/lib64) +if (XXHASH_INCLUDE_DIR AND XXHASH_LIBRARY) + message(STATUS "Found xxHash: include=${XXHASH_INCLUDE_DIR} lib=${XXHASH_LIBRARY}") + list(APPEND MASTER_EXTRA_INCS ${XXHASH_INCLUDE_DIR}) +else() + message(FATAL_ERROR "xxHash library/header not found. Please install xxhash (development headers) and try again.") +endif() + if(USE_3FS) add_subdirectory(hf3fs) list(APPEND MOONCAKE_STORE_SOURCES ${HF3FS_SOURCES}) @@ -44,6 +60,8 @@ endif() # The cache_allocator library include_directories(${Python3_INCLUDE_DIRS}) add_library(mooncake_store ${MOONCAKE_STORE_SOURCES}) +target_include_directories(mooncake_store PUBLIC ${XXHASH_INCLUDE_DIR}) +target_link_libraries(mooncake_store PUBLIC ${XXHASH_LIBRARY}) # Note: transfer_engine is PRIVATE to avoid propagating its dependencies (e.g., vendor-specific hardware) # to targets that don't need it (e.g., mooncake_master). # Targets that need transfer_engine should link it explicitly. @@ -57,6 +75,7 @@ target_link_libraries(mooncake_store PRIVATE transfer_engine ) + if (STORE_USE_ETCD) add_dependencies(mooncake_store build_etcd_wrapper) endif() diff --git a/mooncake-store/src/etcd_oplog_store.cpp b/mooncake-store/src/etcd_oplog_store.cpp new file mode 100644 index 0000000000..ec5fb4f961 --- /dev/null +++ b/mooncake-store/src/etcd_oplog_store.cpp @@ -0,0 +1,651 @@ +#include "etcd_oplog_store.h" + +#include +#include +#include + +#if __has_include() +#include // Ubuntu +#else +#include // CentOS +#endif + +#include "etcd_helper.h" + +namespace mooncake { + +EtcdOpLogStore::EtcdOpLogStore(const std::string& cluster_id, + bool enable_latest_seq_batch_update) + : cluster_id_(cluster_id), + enable_latest_seq_batch_update_(enable_latest_seq_batch_update), + last_update_time_(std::chrono::steady_clock::now()) { + // Normalize cluster_id to avoid accidental double slashes in etcd keys when + // caller passes a trailing '/' (master_view_key uses trailing '/', OpLog keys don't). + while (!cluster_id_.empty() && cluster_id_.back() == '/') { + cluster_id_.pop_back(); + } + + if (!cluster_id_.empty() && !IsValidClusterIdComponent(cluster_id_)) { + LOG(FATAL) << "Invalid cluster_id for EtcdOpLogStore: '" << cluster_id_ + << "'. Allowed chars: [A-Za-z0-9_.-], max_len=128, no slashes."; + } + + // Initialize /latest key to 0 if it doesn't exist (first startup). + // This avoids "key not found" errors when querying the latest sequence ID. + // Important: Only initialize if the key doesn't exist to avoid overwriting existing data. + if (!cluster_id_.empty()) { + std::string latest_key = BuildLatestKey(); + std::string existing_value; + EtcdRevisionId revision_id; + ErrorCode get_err = EtcdHelper::Get(latest_key.c_str(), latest_key.size(), + existing_value, revision_id); + if (get_err == ErrorCode::ETCD_KEY_NOT_EXIST) { + // Key doesn't exist, safe to initialize to 0 + std::string initial_value = "0"; + ErrorCode create_err = EtcdHelper::Create(latest_key.c_str(), latest_key.size(), + initial_value.c_str(), initial_value.size()); + if (create_err == ErrorCode::OK) { + LOG(INFO) << "Initialized /latest key to 0 for cluster_id=" << cluster_id_; + } else if (create_err == ErrorCode::ETCD_TRANSACTION_FAIL) { + // Race condition: another instance created it between Get and Create + LOG(INFO) << "/latest key was created by another instance for cluster_id=" + << cluster_id_; + } else { + // Other errors (e.g., etcd not connected) are logged but don't fail construction + // The key will be created when the first OpLog entry is written + LOG(WARNING) << "Failed to initialize /latest key (error=" << create_err + << "), will be created on first OpLog write"; + } + } else if (get_err == ErrorCode::OK) { + // Key already exists, do nothing - preserve existing value + LOG(INFO) << "/latest key already exists (value=" << existing_value + << ") for cluster_id=" << cluster_id_; + } else { + // Other errors (e.g., etcd not connected) are logged but don't fail construction + LOG(WARNING) << "Failed to check /latest key existence (error=" << get_err + << "), will be created on first OpLog write"; + } + } + + // Start batch update thread only for writers. + if (enable_latest_seq_batch_update_) { + batch_update_running_.store(true); + batch_update_thread_ = + std::thread(&EtcdOpLogStore::BatchUpdateThread, this); + } +} + +EtcdOpLogStore::~EtcdOpLogStore() { + if (!enable_latest_seq_batch_update_) { + return; + } + + // Stop batch update thread + batch_update_running_.store(false); + if (batch_update_thread_.joinable()) { + batch_update_thread_.join(); + } + + // Perform final update if there are pending updates + if (pending_count_.load() > 0) { + DoBatchUpdate(); + } +} + +ErrorCode EtcdOpLogStore::WriteOpLog(const OpLogEntry& entry) { + std::string key = BuildOpLogKey(entry.sequence_id); + std::string value = SerializeOpLogEntry(entry); + + // Fence: never overwrite an existing OpLog key. + // - If this is a retry of the same entry: key exists with same value => OK. + // - If key exists with different value: conflict => error (signals seq regression / bug). + ErrorCode err = EtcdHelper::Create(key.c_str(), key.size(), value.c_str(), value.size()); + if (err == ErrorCode::ETCD_TRANSACTION_FAIL) { + std::string existing; + EtcdRevisionId rev = 0; + ErrorCode get_err = EtcdHelper::Get(key.c_str(), key.size(), existing, rev); + if (get_err == ErrorCode::OK && existing == value) { + // Idempotent retry. + err = ErrorCode::OK; + } else { + LOG(ERROR) << "OpLog key conflict: seq=" << entry.sequence_id + << ", get_err=" << get_err; + return ErrorCode::ETCD_OPERATION_ERROR; + } + } + if (err != ErrorCode::OK) { + LOG(ERROR) << "Failed to write OpLog entry, sequence_id=" << entry.sequence_id; + return err; + } + + // Update `/latest`. + // - Writers: batch update to reduce etcd write pressure. + // - Readers / tests: update immediately for simplicity. + if (!enable_latest_seq_batch_update_) { + return UpdateLatestSequenceId(entry.sequence_id); + } + + pending_latest_seq_id_.store(entry.sequence_id); + size_t count = pending_count_.fetch_add(1) + 1; + if (count >= kBatchSize) { + DoBatchUpdate(); + } + + return ErrorCode::OK; +} + +ErrorCode EtcdOpLogStore::ReadOpLog(uint64_t sequence_id, + OpLogEntry& entry) { + std::string key = BuildOpLogKey(sequence_id); + std::string value; + EtcdRevisionId revision_id; + ErrorCode err = EtcdHelper::Get(key.c_str(), key.size(), value, revision_id); + if (err != ErrorCode::OK) { + return err; + } + + if (!DeserializeOpLogEntry(value, entry)) { + LOG(ERROR) << "Failed to deserialize OpLog entry, sequence_id=" + << sequence_id; + return ErrorCode::INTERNAL_ERROR; + } + + return ErrorCode::OK; +} + +ErrorCode EtcdOpLogStore::ReadOpLogSince(uint64_t start_sequence_id, + size_t limit, + std::vector& entries) { + EtcdRevisionId rev = 0; + return ReadOpLogSinceWithRevision(start_sequence_id, limit, entries, rev); +} + +ErrorCode EtcdOpLogStore::ReadOpLogSinceWithRevision(uint64_t start_sequence_id, + size_t limit, + std::vector& entries, + EtcdRevisionId& revision_id) { + entries.clear(); + entries.reserve(limit); + + // Range is limited to OpLog entry keys only. + const std::string prefix = std::string(kOpLogPrefix) + cluster_id_ + "/"; + std::string current_start_key = BuildOpLogKey(start_sequence_id + 1); + + // Compute prefix range end (etcd prefix end). + auto prefix_end = [](std::string p) -> std::string { + for (int i = static_cast(p.size()) - 1; i >= 0; --i) { + unsigned char c = static_cast(p[i]); + if (c < 0xFF) { + p[i] = static_cast(c + 1); + p.resize(i + 1); + return p; + } + } + return std::string(1, '\0'); + }; + const std::string end_key = prefix_end(prefix); + + // Pagination: + // - Use range-get with limit + // - Start next page from lastKey + '\0' (lexicographically just after lastKey) + // This avoids repeating the last key without adding new Go/C++ APIs. + revision_id = 0; + while (entries.size() < limit) { + const size_t page_limit = limit - entries.size(); + std::string json; + EtcdRevisionId page_rev = 0; + ErrorCode err = + EtcdHelper::GetRangeAsJson(current_start_key.c_str(), + current_start_key.size(), end_key.c_str(), + end_key.size(), page_limit, json, page_rev); + if (err != ErrorCode::OK) { + return err; + } + if (page_rev > revision_id) { + revision_id = page_rev; + } + + // Parse kv list: [{"key":"...","value":"..."}] + Json::Value root; + Json::CharReaderBuilder reader; + std::string errs; + std::istringstream s(json); + if (!Json::parseFromStream(reader, s, &root, &errs)) { + LOG(ERROR) << "Failed to parse range JSON: " << errs; + return ErrorCode::INTERNAL_ERROR; + } + if (!root.isArray()) { + return ErrorCode::INTERNAL_ERROR; + } + if (root.empty()) { + break; // no more data + } + + std::string last_key_in_page; + for (const auto& kv : root) { + const std::string key = kv.get("key", "").asString(); + last_key_in_page = key; + if (key.empty() || key.find("/latest") != std::string::npos || + key.find("/snapshot/") != std::string::npos) { + continue; + } + + // Parse seq from key suffix and filter (handles legacy keys too). + size_t pos = key.rfind('/'); + if (pos == std::string::npos || pos + 1 >= key.size()) { + continue; + } + uint64_t seq = 0; + try { + seq = static_cast(std::stoull(key.substr(pos + 1))); + } catch (...) { + continue; + } + if (IsSequenceOlderOrEqual(seq, start_sequence_id)) { + continue; + } + + OpLogEntry entry; + const std::string value = kv.get("value", "").asString(); + if (!DeserializeOpLogEntry(value, entry)) { + LOG(ERROR) << "Failed to deserialize OpLog entry from key=" << key; + return ErrorCode::INTERNAL_ERROR; + } + entries.push_back(std::move(entry)); + if (entries.size() >= limit) { + break; + } + } + + // Advance start key for next page. + if (last_key_in_page.empty()) { + break; + } + current_start_key = last_key_in_page; + current_start_key.push_back('\0'); + } + + return ErrorCode::OK; +} + +ErrorCode EtcdOpLogStore::GetLatestSequenceId(uint64_t& sequence_id) { + std::string key = BuildLatestKey(); + std::string value; + EtcdRevisionId revision_id; + ErrorCode err = EtcdHelper::Get(key.c_str(), key.size(), value, revision_id); + if (err != ErrorCode::OK) { + return err; + } + + try { + sequence_id = std::stoull(value); + } catch (const std::exception& e) { + LOG(ERROR) << "Failed to parse latest sequence_id: " << e.what(); + return ErrorCode::INTERNAL_ERROR; + } + + return ErrorCode::OK; +} + +ErrorCode EtcdOpLogStore::GetMaxSequenceId(uint64_t& sequence_id) { + auto max_seq_opt = GetMaxSequenceIdInternal(); + if (!max_seq_opt.has_value()) { + return ErrorCode::ETCD_KEY_NOT_EXIST; + } + sequence_id = max_seq_opt.value(); + return ErrorCode::OK; +} + +ErrorCode EtcdOpLogStore::UpdateLatestSequenceId(uint64_t sequence_id) { + std::string key = BuildLatestKey(); + std::string value = std::to_string(sequence_id); + return EtcdHelper::Put(key.c_str(), key.size(), value.c_str(), value.size()); +} + +ErrorCode EtcdOpLogStore::RecordSnapshotSequenceId( + const std::string& snapshot_id, uint64_t sequence_id) { + std::string key = BuildSnapshotKey(snapshot_id); + std::string value = std::to_string(sequence_id); + return EtcdHelper::Put(key.c_str(), key.size(), value.c_str(), value.size()); +} + +ErrorCode EtcdOpLogStore::GetSnapshotSequenceId( + const std::string& snapshot_id, uint64_t& sequence_id) { + std::string key = BuildSnapshotKey(snapshot_id); + std::string value; + EtcdRevisionId revision_id; + ErrorCode err = EtcdHelper::Get(key.c_str(), key.size(), value, revision_id); + if (err != ErrorCode::OK) { + return err; + } + + try { + sequence_id = std::stoull(value); + } catch (const std::exception& e) { + LOG(ERROR) << "Failed to parse snapshot sequence_id: " << e.what(); + return ErrorCode::INTERNAL_ERROR; + } + + return ErrorCode::OK; +} + +ErrorCode EtcdOpLogStore::CleanupOpLogBefore(uint64_t before_sequence_id) { + // Robust cleanup (Scheme 3): + // - Determine current minimum sequence_id in etcd + // - DeleteRange [min_key, before_key) + // + // IMPORTANT: This relies on lexicographical ordering of keys, so the + // sequence_id portion MUST be fixed-width (zero-padded). + auto min_seq_opt = GetMinSequenceId(); + if (!min_seq_opt.has_value()) { + return ErrorCode::OK; // nothing to cleanup + } + + uint64_t min_seq = min_seq_opt.value(); + if (before_sequence_id <= min_seq) { + return ErrorCode::OK; + } + + std::string start_key = BuildOpLogKey(min_seq); + std::string end_key = BuildOpLogKey(before_sequence_id); // delete < before_sequence_id + + return EtcdHelper::DeleteRange(start_key.c_str(), start_key.size(), + end_key.c_str(), end_key.size()); +} + +std::string EtcdOpLogStore::BuildOpLogKey(uint64_t sequence_id) const { + std::ostringstream oss; + // Fixed-width encoding for correct etcd lexicographical range operations. + // 20 digits is enough for uint64_t max (18446744073709551615). + oss << kOpLogPrefix << cluster_id_ << "/" + << std::setw(20) << std::setfill('0') << sequence_id; + return oss.str(); +} + +std::optional EtcdOpLogStore::GetMinSequenceId() const { + std::string prefix = std::string(kOpLogPrefix) + cluster_id_ + "/"; + std::string first_key; + ErrorCode err = + EtcdHelper::GetFirstKeyWithPrefix(prefix.c_str(), prefix.size(), first_key); + if (err != ErrorCode::OK) { + return std::nullopt; + } + + // Skip non-entry keys if any (e.g. "/latest" or "/snapshot/..."). + // Entries are expected to be ".../<20-digit-seq>". + // If the first key isn't an entry key, fall back to nullopt (safe no-op). + if (first_key.find("/latest") != std::string::npos || + first_key.find("/snapshot/") != std::string::npos) { + return std::nullopt; + } + + size_t pos = first_key.rfind('/'); + if (pos == std::string::npos || pos + 1 >= first_key.size()) { + return std::nullopt; + } + std::string seq_str = first_key.substr(pos + 1); + try { + return static_cast(std::stoull(seq_str)); + } catch (...) { + return std::nullopt; + } +} + +std::optional EtcdOpLogStore::GetMaxSequenceIdInternal() const { + // Entry keys are fixed-width 20-digit numbers, which (in practice) start with '0'. + // Use "/0" to avoid picking up "/latest" which is lexicographically after digits. + std::string prefix = std::string(kOpLogPrefix) + cluster_id_ + "/0"; + std::string last_key; + ErrorCode err = + EtcdHelper::GetLastKeyWithPrefix(prefix.c_str(), prefix.size(), last_key); + if (err != ErrorCode::OK) { + return std::nullopt; + } + + size_t pos = last_key.rfind('/'); + if (pos == std::string::npos || pos + 1 >= last_key.size()) { + return std::nullopt; + } + std::string seq_str = last_key.substr(pos + 1); + try { + return static_cast(std::stoull(seq_str)); + } catch (...) { + return std::nullopt; + } +} + +std::string EtcdOpLogStore::BuildLatestKey() const { + std::ostringstream oss; + oss << kOpLogPrefix << cluster_id_ << kLatestSuffix; + return oss.str(); +} + +std::string EtcdOpLogStore::BuildSnapshotKey( + const std::string& snapshot_id) const { + std::ostringstream oss; + oss << kOpLogPrefix << cluster_id_ << kSnapshotSuffix << snapshot_id + << "/sequence_id"; + return oss.str(); +} + +namespace { + +// Base64 encoding for binary payload +// JsonCpp treats strings as UTF-8, so we must encode binary data +std::string Base64Encode(const std::string& data) { + static const char base64_chars[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + + std::string result; + result.reserve(((data.size() + 2) / 3) * 4); + + size_t i = 0; + size_t data_len = data.size(); + + // Process 3 bytes at a time + while (i + 2 < data_len) { + uint32_t octet_a = static_cast(data[i++]); + uint32_t octet_b = static_cast(data[i++]); + uint32_t octet_c = static_cast(data[i++]); + + uint32_t triple = (octet_a << 16) | (octet_b << 8) | octet_c; + + result.push_back(base64_chars[(triple >> 18) & 0x3F]); + result.push_back(base64_chars[(triple >> 12) & 0x3F]); + result.push_back(base64_chars[(triple >> 6) & 0x3F]); + result.push_back(base64_chars[triple & 0x3F]); + } + + // Handle remaining bytes + size_t remaining = data_len - i; + if (remaining > 0) { + uint32_t octet_a = static_cast(data[i++]); + uint32_t octet_b = (remaining > 1) ? static_cast(data[i++]) : 0; + uint32_t octet_c = 0; + + uint32_t triple = (octet_a << 16) | (octet_b << 8) | octet_c; + + result.push_back(base64_chars[(triple >> 18) & 0x3F]); + result.push_back(base64_chars[(triple >> 12) & 0x3F]); + result.push_back((remaining > 1) ? base64_chars[(triple >> 6) & 0x3F] : '='); + result.push_back('='); + } + + return result; +} + +std::string Base64Decode(const std::string& encoded) { + static const unsigned char decode_table[256] = { + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 62, 64, 64, 64, 63, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 64, 64, 64, 64, 64, 64, + 64, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 64, 64, 64, 64, 64, + 64, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 + }; + + std::string result; + result.reserve((encoded.size() * 3) / 4); + + size_t i = 0; + while (i < encoded.size()) { + // Skip whitespace and invalid chars + while (i < encoded.size() && (encoded[i] == ' ' || encoded[i] == '\n' || encoded[i] == '\r' || encoded[i] == '\t')) { + i++; + } + if (i >= encoded.size()) break; + + uint32_t sextet_a = decode_table[static_cast(encoded[i++])]; + if (i >= encoded.size() || sextet_a == 64) break; + + uint32_t sextet_b = decode_table[static_cast(encoded[i++])]; + if (sextet_b == 64) break; + + uint32_t sextet_c = (i < encoded.size()) ? decode_table[static_cast(encoded[i++])] : 64; + uint32_t sextet_d = (i < encoded.size()) ? decode_table[static_cast(encoded[i++])] : 64; + + uint32_t triple = (sextet_a << 18) | (sextet_b << 12) | + ((sextet_c != 64) ? (sextet_c << 6) : 0) | + ((sextet_d != 64) ? sextet_d : 0); + + result.push_back(static_cast((triple >> 16) & 0xFF)); + if (sextet_c != 64) { + result.push_back(static_cast((triple >> 8) & 0xFF)); + } + if (sextet_d != 64) { + result.push_back(static_cast(triple & 0xFF)); + } + } + + return result; +} + +} // namespace + +std::string EtcdOpLogStore::SerializeOpLogEntry( + const OpLogEntry& entry) const { + Json::Value root; + root["sequence_id"] = static_cast(entry.sequence_id); + root["timestamp_ms"] = static_cast(entry.timestamp_ms); + root["op_type"] = static_cast(entry.op_type); + root["object_key"] = entry.object_key; + // CRITICAL: Base64 encode binary payload to prevent UTF-8 corruption in JSON + root["payload"] = Base64Encode(entry.payload); + root["checksum"] = static_cast(entry.checksum); + root["prefix_hash"] = static_cast(entry.prefix_hash); + + Json::StreamWriterBuilder builder; + builder["indentation"] = ""; // Compact format + std::unique_ptr writer(builder.newStreamWriter()); + std::ostringstream oss; + writer->write(root, &oss); + return oss.str(); +} + +bool EtcdOpLogStore::DeserializeOpLogEntry(const std::string& json_str, + OpLogEntry& entry) const { + Json::Value root; + Json::CharReaderBuilder builder; + std::unique_ptr reader(builder.newCharReader()); + std::string errors; + + if (!reader->parse(json_str.data(), json_str.data() + json_str.size(), + &root, &errors)) { + LOG(ERROR) << "Failed to parse JSON: " << errors; + return false; + } + + try { + entry.sequence_id = root["sequence_id"].asUInt64(); + entry.timestamp_ms = root["timestamp_ms"].asUInt64(); + entry.op_type = static_cast(root["op_type"].asInt()); + entry.object_key = root["object_key"].asString(); + // CRITICAL: Base64 decode payload to restore binary data + entry.payload = Base64Decode(root["payload"].asString()); + entry.checksum = root["checksum"].asUInt(); + entry.prefix_hash = root["prefix_hash"].asUInt(); + } catch (const std::exception& e) { + LOG(ERROR) << "Failed to deserialize OpLogEntry: " << e.what(); + return false; + } + + std::string size_reason; + if (!OpLogManager::ValidateEntrySize(entry, &size_reason)) { + LOG(ERROR) << "EtcdOpLogStore: entry size rejected, sequence_id=" + << entry.sequence_id << ", key=" << entry.object_key + << ", reason=" << size_reason; + return false; + } + + return true; +} + +void EtcdOpLogStore::BatchUpdateThread() { + if (!enable_latest_seq_batch_update_) { + return; + } + while (batch_update_running_.load()) { + std::this_thread::sleep_for( + std::chrono::milliseconds(kBatchIntervalMs)); + + // Check if we need to update based on time interval + auto now = std::chrono::steady_clock::now(); + auto elapsed = std::chrono::duration_cast( + now - last_update_time_).count(); + + if (pending_count_.load() > 0 && elapsed >= kBatchIntervalMs) { + DoBatchUpdate(); + } + } +} + +void EtcdOpLogStore::TriggerBatchUpdateIfNeeded() { + // This method is kept for potential future use (e.g., manual trigger) + // Currently, DoBatchUpdate() is called directly from WriteOpLog + // when batch size threshold is reached + if (pending_count_.load() >= kBatchSize) { + DoBatchUpdate(); + } +} + +void EtcdOpLogStore::DoBatchUpdate() { + if (!enable_latest_seq_batch_update_) { + return; + } + std::lock_guard lock(batch_update_mutex_); + + // Get the pending sequence_id and reset counters + uint64_t seq_id_to_update = pending_latest_seq_id_.load(); + size_t count = pending_count_.exchange(0); + + if (count == 0) { + return; // Nothing to update + } + + // Update latest_sequence_id in etcd + ErrorCode err = UpdateLatestSequenceId(seq_id_to_update); + if (err != ErrorCode::OK) { + LOG(WARNING) << "Failed to batch update latest_sequence_id=" + << seq_id_to_update << ", error=" << err + << ". Will retry in next batch."; + // Restore the count so it will be retried + pending_count_.fetch_add(count); + } else { + last_update_time_ = std::chrono::steady_clock::now(); + VLOG(2) << "Batch updated latest_sequence_id=" << seq_id_to_update + << " (count=" << count << " entries)"; + } +} + +} // namespace mooncake + diff --git a/mooncake-store/src/ha_metric_manager.cpp b/mooncake-store/src/ha_metric_manager.cpp new file mode 100644 index 0000000000..b16c007aca --- /dev/null +++ b/mooncake-store/src/ha_metric_manager.cpp @@ -0,0 +1,281 @@ +#include "ha_metric_manager.h" + +#include + +#include +#include + +namespace mooncake { + +// --- Singleton Instance --- +HAMetricManager& HAMetricManager::instance() { + static HAMetricManager static_instance; + return static_instance; +} + +// --- Constructor --- +HAMetricManager::HAMetricManager() + // OpLog Sequence Gauges + : oplog_last_sequence_id_( + "ha_oplog_last_sequence_id", + "Latest OpLog sequence ID written by Primary"), + oplog_applied_sequence_id_( + "ha_oplog_applied_sequence_id", + "Latest OpLog sequence ID applied by Standby"), + oplog_standby_lag_( + "ha_oplog_standby_lag", + "Number of OpLog entries Standby is behind Primary"), + oplog_pending_entries_( + "ha_oplog_pending_entries", + "Number of out-of-order entries waiting in OpLogApplier"), + pending_mutation_queue_size_( + "ha_pending_mutation_queue_size", + "Number of mutations pending etcd write retry"), + + // Error Counters + oplog_skipped_entries_total_( + "ha_oplog_skipped_entries_total", + "Total number of OpLog entries skipped due to timeout"), + oplog_checksum_failures_total_( + "ha_oplog_checksum_failures_total", + "Total number of OpLog entries with checksum verification failures"), + oplog_gap_resolve_attempts_total_( + "ha_oplog_gap_resolve_attempts_total", + "Total number of attempts to resolve missing OpLog entries"), + oplog_gap_resolve_success_total_( + "ha_oplog_gap_resolve_success_total", + "Total number of successfully resolved missing OpLog entries"), + oplog_etcd_write_failures_total_( + "ha_oplog_etcd_write_failures_total", + "Total number of failed etcd write operations"), + oplog_etcd_write_retries_total_( + "ha_oplog_etcd_write_retries_total", + "Total number of etcd write retry attempts"), + oplog_watch_disconnections_total_( + "ha_oplog_watch_disconnections_total", + "Total number of OpLog watch disconnections"), + oplog_applied_entries_total_( + "ha_oplog_applied_entries_total", + "Total number of OpLog entries successfully applied"), + + // Latency Histograms (buckets in microseconds) + // 100us, 500us, 1ms, 5ms, 10ms, 50ms, 100ms, 500ms, 1s, 5s + oplog_etcd_write_latency_us_( + "ha_oplog_etcd_write_latency_us", + "Latency of etcd write operations in microseconds", + {100, 500, 1000, 5000, 10000, 50000, 100000, 500000, 1000000, 5000000}), + oplog_apply_latency_us_( + "ha_oplog_apply_latency_us", + "Latency of OpLog entry application in microseconds", + {10, 50, 100, 500, 1000, 5000, 10000, 50000, 100000}), + + // State Machine + standby_state_( + "ha_standby_state", + "Current state of the Standby service (0=STOPPED, 1=CONNECTING, " + "2=SYNCING, 3=WATCHING, 4=RECOVERING, 5=RECONNECTING, " + "6=PROMOTING, 7=PROMOTED, 8=FAILED)"), + state_transitions_total_( + "ha_state_transitions_total", + "Total number of Standby state machine transitions") { + // Initialize gauges to 0 for proper Prometheus output + oplog_last_sequence_id_.update(0); + oplog_applied_sequence_id_.update(0); + oplog_standby_lag_.update(0); + oplog_pending_entries_.update(0); + pending_mutation_queue_size_.update(0); + standby_state_.update(0); +} + +// ========== OpLog Sequence Metrics (Gauge) ========== + +void HAMetricManager::set_oplog_last_sequence_id(int64_t seq_id) { + oplog_last_sequence_id_.update(seq_id); +} + +int64_t HAMetricManager::get_oplog_last_sequence_id() { + return static_cast(oplog_last_sequence_id_.value()); +} + +void HAMetricManager::set_oplog_applied_sequence_id(int64_t seq_id) { + oplog_applied_sequence_id_.update(seq_id); +} + +int64_t HAMetricManager::get_oplog_applied_sequence_id() { + return static_cast(oplog_applied_sequence_id_.value()); +} + +void HAMetricManager::set_oplog_standby_lag(int64_t lag) { + oplog_standby_lag_.update(lag); +} + +int64_t HAMetricManager::get_oplog_standby_lag() { + return static_cast(oplog_standby_lag_.value()); +} + +void HAMetricManager::set_oplog_pending_entries(int64_t count) { + oplog_pending_entries_.update(count); +} + +int64_t HAMetricManager::get_oplog_pending_entries() { + return static_cast(oplog_pending_entries_.value()); +} + +void HAMetricManager::set_pending_mutation_queue_size(int64_t size) { + pending_mutation_queue_size_.update(size); +} + +int64_t HAMetricManager::get_pending_mutation_queue_size() { + return static_cast(pending_mutation_queue_size_.value()); +} + +// ========== Error Counters ========== + +void HAMetricManager::inc_oplog_skipped_entries(int64_t val) { + oplog_skipped_entries_total_.inc(val); +} + +int64_t HAMetricManager::get_oplog_skipped_entries_total() { + return static_cast(oplog_skipped_entries_total_.value()); +} + +void HAMetricManager::inc_oplog_checksum_failures(int64_t val) { + oplog_checksum_failures_total_.inc(val); +} + +int64_t HAMetricManager::get_oplog_checksum_failures_total() { + return static_cast(oplog_checksum_failures_total_.value()); +} + +void HAMetricManager::inc_oplog_gap_resolve_attempts(int64_t val) { + oplog_gap_resolve_attempts_total_.inc(val); +} + +int64_t HAMetricManager::get_oplog_gap_resolve_attempts_total() { + return static_cast(oplog_gap_resolve_attempts_total_.value()); +} + +void HAMetricManager::inc_oplog_gap_resolve_success(int64_t val) { + oplog_gap_resolve_success_total_.inc(val); +} + +int64_t HAMetricManager::get_oplog_gap_resolve_success_total() { + return static_cast(oplog_gap_resolve_success_total_.value()); +} + +void HAMetricManager::inc_oplog_etcd_write_failures(int64_t val) { + oplog_etcd_write_failures_total_.inc(val); +} + +int64_t HAMetricManager::get_oplog_etcd_write_failures_total() { + return static_cast(oplog_etcd_write_failures_total_.value()); +} + +void HAMetricManager::inc_oplog_etcd_write_retries(int64_t val) { + oplog_etcd_write_retries_total_.inc(val); +} + +int64_t HAMetricManager::get_oplog_etcd_write_retries_total() { + return static_cast(oplog_etcd_write_retries_total_.value()); +} + +void HAMetricManager::inc_oplog_watch_disconnections(int64_t val) { + oplog_watch_disconnections_total_.inc(val); +} + +int64_t HAMetricManager::get_oplog_watch_disconnections_total() { + return static_cast(oplog_watch_disconnections_total_.value()); +} + +void HAMetricManager::inc_oplog_applied_entries(int64_t val) { + oplog_applied_entries_total_.inc(val); +} + +int64_t HAMetricManager::get_oplog_applied_entries_total() { + return static_cast(oplog_applied_entries_total_.value()); +} + +// ========== Latency Histograms ========== + +void HAMetricManager::observe_oplog_etcd_write_latency_us(int64_t latency_us) { + oplog_etcd_write_latency_us_.observe(latency_us); +} + +void HAMetricManager::observe_oplog_apply_latency_us(int64_t latency_us) { + oplog_apply_latency_us_.observe(latency_us); +} + +// ========== State Machine Metrics ========== + +void HAMetricManager::set_standby_state(int64_t state_value) { + standby_state_.update(state_value); +} + +int64_t HAMetricManager::get_standby_state() { + return static_cast(standby_state_.value()); +} + +void HAMetricManager::inc_state_transitions(int64_t val) { + state_transitions_total_.inc(val); +} + +int64_t HAMetricManager::get_state_transitions_total() { + return static_cast(state_transitions_total_.value()); +} + +// ========== Serialization ========== + +std::string HAMetricManager::serialize_metrics() { + std::stringstream ss; + + // Helper lambda to serialize a metric + auto serialize_metric = [&ss](auto& metric) { + std::string metric_str; + metric.serialize(metric_str); + ss << metric_str; + }; + + // Gauges + serialize_metric(oplog_last_sequence_id_); + serialize_metric(oplog_applied_sequence_id_); + serialize_metric(oplog_standby_lag_); + serialize_metric(oplog_pending_entries_); + serialize_metric(pending_mutation_queue_size_); + serialize_metric(standby_state_); + + // Counters + serialize_metric(oplog_skipped_entries_total_); + serialize_metric(oplog_checksum_failures_total_); + serialize_metric(oplog_gap_resolve_attempts_total_); + serialize_metric(oplog_gap_resolve_success_total_); + serialize_metric(oplog_etcd_write_failures_total_); + serialize_metric(oplog_etcd_write_retries_total_); + serialize_metric(oplog_watch_disconnections_total_); + serialize_metric(oplog_applied_entries_total_); + serialize_metric(state_transitions_total_); + + // Histograms + serialize_metric(oplog_etcd_write_latency_us_); + serialize_metric(oplog_apply_latency_us_); + + return ss.str(); +} + +std::string HAMetricManager::get_summary_string() { + std::stringstream ss; + ss << "HA Metrics Summary: "; + ss << "last_seq=" << get_oplog_last_sequence_id(); + ss << ", applied_seq=" << get_oplog_applied_sequence_id(); + ss << ", lag=" << get_oplog_standby_lag(); + ss << ", pending=" << get_oplog_pending_entries(); + ss << ", mutation_queue=" << get_pending_mutation_queue_size(); + ss << ", skipped=" << get_oplog_skipped_entries_total(); + ss << ", checksum_fail=" << get_oplog_checksum_failures_total(); + ss << ", etcd_fail=" << get_oplog_etcd_write_failures_total(); + ss << ", watch_disconn=" << get_oplog_watch_disconnections_total(); + ss << ", state=" << get_standby_state(); + return ss.str(); +} + +} // namespace mooncake + diff --git a/mooncake-store/src/hot_standby_service.cpp b/mooncake-store/src/hot_standby_service.cpp new file mode 100644 index 0000000000..3ff3940bab --- /dev/null +++ b/mooncake-store/src/hot_standby_service.cpp @@ -0,0 +1,605 @@ +#include "hot_standby_service.h" + +#include + +#include +#include + +#include "etcd_helper.h" +#include "etcd_oplog_store.h" +#include "ha_metric_manager.h" +#include "master_service.h" +#include "oplog_applier.h" +#include "oplog_manager.h" +#include "oplog_watcher.h" + +namespace mooncake { + +HotStandbyService::HotStandbyService(const HotStandbyConfig& config) + : config_(config) { + metadata_store_ = std::make_unique(); + // OpLogApplier will be re-created in Start() with the resolved cluster_id + // to enable etcd-based operations (e.g. requesting missing OpLog entries). + // Here we construct a minimal instance so that local metadata operations + // are available before etcd wiring is completed. + oplog_applier_ = std::make_unique(metadata_store_.get()); + + // Register callback for state change logging and metrics. + state_machine_.RegisterCallback([](StandbyState old_state, StandbyState new_state, StandbyEvent event) { + LOG(INFO) << "HotStandbyService state changed: " + << StandbyStateToString(old_state) << " -> " + << StandbyStateToString(new_state) + << " (event: " << StandbyEventToString(event) << ")"; + + // Update HA metrics + HAMetricManager::instance().set_standby_state(static_cast(new_state)); + HAMetricManager::instance().inc_state_transitions(); + + // Track watch disconnections + if (event == StandbyEvent::WATCH_BROKEN || event == StandbyEvent::DISCONNECTED) { + HAMetricManager::instance().inc_oplog_watch_disconnections(); + } + }); +} + +// StandbyMetadataStore implementation +bool HotStandbyService::StandbyMetadataStore::PutMetadata( + const std::string& key, const StandbyObjectMetadata& metadata) { + std::lock_guard lock(mutex_); + store_[key] = metadata; + VLOG(2) << "StandbyMetadataStore: stored metadata for key=" << key + << ", replicas=" << metadata.replicas.size() + << ", size=" << metadata.size; + return true; +} + +bool HotStandbyService::StandbyMetadataStore::Put(const std::string& key, + const std::string& payload) { + // Legacy interface - create empty metadata + StandbyObjectMetadata metadata; + std::lock_guard lock(mutex_); + store_[key] = metadata; + return true; +} + +const StandbyObjectMetadata* HotStandbyService::StandbyMetadataStore::GetMetadata( + const std::string& key) const { + std::lock_guard lock(mutex_); + auto it = store_.find(key); + if (it != store_.end()) { + return &it->second; + } + return nullptr; +} + +bool HotStandbyService::StandbyMetadataStore::Remove(const std::string& key) { + std::lock_guard lock(mutex_); + auto it = store_.find(key); + if (it != store_.end()) { + store_.erase(it); + return true; + } + return false; +} + +bool HotStandbyService::StandbyMetadataStore::Exists( + const std::string& key) const { + std::lock_guard lock(mutex_); + return store_.find(key) != store_.end(); +} + +size_t HotStandbyService::StandbyMetadataStore::GetKeyCount() const { + std::lock_guard lock(mutex_); + return store_.size(); +} + +void HotStandbyService::StandbyMetadataStore::Snapshot( + std::vector>& out) const { + std::lock_guard lock(mutex_); + out.clear(); + out.reserve(store_.size()); + for (const auto& kv : store_) { + out.emplace_back(kv.first, kv.second); + } +} + +HotStandbyService::~HotStandbyService() { + // Always ensure threads are joined, regardless of state + // This prevents std::terminate() if threads are still joinable + Stop(); + + // Double-check: ensure all threads are joined even if Stop() had early return + if (replication_thread_.joinable()) { + replication_thread_.join(); + } + if (verification_thread_.joinable()) { + verification_thread_.join(); + } +} + +ErrorCode HotStandbyService::Start(const std::string& primary_address, + const std::string& etcd_endpoints, + const std::string& cluster_id) { + std::lock_guard lock(mutex_); + + // Use state machine to check if already running + if (IsRunning()) { + LOG(WARNING) << "HotStandbyService is already running"; + return ErrorCode::OK; + } + + // Trigger START event + auto result = state_machine_.ProcessEvent(StandbyEvent::START); + if (!result.allowed) { + LOG(ERROR) << "Cannot start HotStandbyService: " << result.reason; + return ErrorCode::INTERNAL_ERROR; // State machine rejected START + } + + config_.primary_address = primary_address; + etcd_endpoints_ = etcd_endpoints; + cluster_id_ = cluster_id; + +#ifdef STORE_USE_ETCD + // Connect to etcd + ErrorCode err = EtcdHelper::ConnectToEtcdStoreClient(etcd_endpoints.c_str()); + if (err != ErrorCode::OK) { + LOG(ERROR) << "Failed to connect to etcd: " << etcd_endpoints; + state_machine_.ProcessEvent(StandbyEvent::CONNECTION_FAILED); + return err; + } + + // Transition to SYNCING state + state_machine_.ProcessEvent(StandbyEvent::CONNECTED); + + // Preserve existing local state if HotStandbyService is restarted in-process: + // - metadata_store_ may already contain real-time metadata + // - oplog_applier_ may already have expected_sequence_id_ + uint64_t local_last_seq_id = 0; + if (oplog_applier_) { + uint64_t expected = oplog_applier_->GetExpectedSequenceId(); + local_last_seq_id = expected > 0 ? expected - 1 : 0; + } + const bool has_local_metadata = + metadata_store_ && metadata_store_->GetKeyCount() > 0; + const bool has_local_state = has_local_metadata && local_last_seq_id > 0; + + // Recreate OpLogApplier with cluster_id (for requesting missing OpLog). + // If we had local state, recover to keep sequence continuity. + oplog_applier_ = std::make_unique(metadata_store_.get(), cluster_id); + if (has_local_state) { + LOG(INFO) << "Standby warm start: reuse local metadata (keys=" + << metadata_store_->GetKeyCount() + << "), recover last_seq_id=" << local_last_seq_id; + oplog_applier_->Recover(local_last_seq_id); + } + + // Create OpLogWatcher with state machine callback + oplog_watcher_ = std::make_unique( + etcd_endpoints, cluster_id, oplog_applier_.get()); + + // Register callback for watcher events + oplog_watcher_->SetStateCallback([this](StandbyEvent event) { + OnWatcherEvent(event); + }); + + // Bootstrap: + // - If we already have local state (warm start), do NOT reload snapshot. + // - Otherwise (cold start/new standby), try snapshot (if enabled) then replay OpLog. + uint64_t baseline_seq_id = has_local_state ? local_last_seq_id : 0; + if (!has_local_state && config_.enable_snapshot_bootstrap && snapshot_provider_) { + std::string snapshot_id; + uint64_t snapshot_seq_id = 0; + std::vector> snapshot; + if (snapshot_provider_->LoadLatestSnapshot(cluster_id_, snapshot_id, snapshot_seq_id, + snapshot)) { + LOG(INFO) << "Loaded snapshot: snapshot_id=" << snapshot_id + << ", snapshot_seq_id=" << snapshot_seq_id + << ", keys=" << snapshot.size(); + // Apply snapshot into local standby store. + for (const auto& kv : snapshot) { + metadata_store_->PutMetadata(kv.first, kv.second); + } + // Align applier to snapshot boundary. + oplog_applier_->Recover(snapshot_seq_id); + baseline_seq_id = snapshot_seq_id; + } else { + LOG(INFO) << "No snapshot available (or provider not ready), falling back to OpLog-only bootstrap"; + } + } + + // Read historical OpLog entries since baseline_seq_id. + uint64_t last_applied_seq_id = baseline_seq_id; + + // Start OpLogWatcher with a consistent "read then watch(from revision+1)" sequence. + if (!oplog_watcher_->StartFromSequenceId(last_applied_seq_id)) { + LOG(WARNING) << "Failed to start OpLogWatcher from sequence_id=" + << last_applied_seq_id << ", continuing anyway"; + state_machine_.ProcessEvent(StandbyEvent::SYNC_FAILED); + } else { + // Transition to WATCHING state after successful sync + state_machine_.ProcessEvent(StandbyEvent::SYNC_COMPLETE); + } + + // Start background threads + replication_thread_ = std::thread(&HotStandbyService::ReplicationLoop, this); + if (config_.enable_verification) { + verification_thread_ = + std::thread(&HotStandbyService::VerificationLoop, this); + } + + LOG(INFO) << "HotStandbyService started, watching etcd OpLog for cluster: " + << cluster_id << ", state=" << StandbyStateToString(GetState()); + return ErrorCode::OK; +#else + state_machine_.ProcessEvent(StandbyEvent::FATAL_ERROR); + LOG(ERROR) << "STORE_USE_ETCD is not enabled, cannot start HotStandbyService"; + return ErrorCode::INTERNAL_ERROR; +#endif +} + +void HotStandbyService::OnWatcherEvent(StandbyEvent event) { + state_machine_.ProcessEvent(event); +} + +void HotStandbyService::Stop() { + // Check if already stopped (to avoid duplicate processing) + bool was_running = IsRunning(); + StandbyState current_state = GetState(); + + if (!was_running && current_state != StandbyState::PROMOTING && + !replication_thread_.joinable() && !verification_thread_.joinable()) { + // Already fully stopped and threads are joined + return; + } + + // Trigger STOP event + state_machine_.ProcessEvent(StandbyEvent::STOP); + + // Stop OpLogWatcher + if (oplog_watcher_) { + oplog_watcher_->Stop(); + oplog_watcher_.reset(); + } + + // Wait for threads to finish + if (replication_thread_.joinable()) { + replication_thread_.join(); + } + if (verification_thread_.joinable()) { + verification_thread_.join(); + } + + LOG(INFO) << "HotStandbyService stopped, final_state=" << StandbyStateToString(GetState()); +} + +StandbySyncStatus HotStandbyService::GetSyncStatus() const { + StandbySyncStatus status; + + // Get applied sequence ID from OpLogApplier + if (oplog_applier_) { + status.applied_seq_id = oplog_applier_->GetExpectedSequenceId() - 1; + if (status.applied_seq_id == 0) { + status.applied_seq_id = applied_seq_id_.load(); // Fallback + } + } else { + status.applied_seq_id = applied_seq_id_.load(); + } + + // Primary sequence ID (best-effort): updated by ReplicationLoop via etcd `/latest`. + status.primary_seq_id = primary_seq_id_.load(); + + // Use state machine for connection status + status.is_connected = IsConnected(); + status.state = GetState(); + status.time_in_state = state_machine_.GetTimeInCurrentState(); + + if (status.primary_seq_id > status.applied_seq_id) { + status.lag_entries = status.primary_seq_id - status.applied_seq_id; + } else { + status.lag_entries = 0; + } + + // Lag time is currently reported as 0; if needed we can extend the + // protocol to propagate primary timestamps and compute a real value. + status.lag_time = std::chrono::milliseconds(0); + status.is_syncing = IsRunning() && IsConnected(); + + return status; +} + +bool HotStandbyService::IsReadyForPromotion() const { + // Use state machine to check if ready for promotion + if (!state_machine_.IsReadyForPromotion()) { + return false; + } + + StandbySyncStatus status = GetSyncStatus(); + + // Allow promotion even with large lag - the new Primary can continue + // syncing remaining OpLog entries from etcd after promotion. + // Log a warning if lag is large, but don't block promotion. + if (status.lag_entries > config_.max_replication_lag_entries) { + LOG(WARNING) << "Standby has large replication lag: " << status.lag_entries + << " entries (threshold: " << config_.max_replication_lag_entries + << "). Promotion will proceed, but remaining OpLog entries " + << "will be synced after promotion."; + } + + return true; +} + +std::unique_ptr HotStandbyService::Promote() { + std::lock_guard lock(mutex_); + + if (!IsReadyForPromotion()) { + LOG(ERROR) << "Standby is not ready for promotion, state=" + << StandbyStateToString(GetState()); + return nullptr; + } + + // Trigger PROMOTE event + auto result = state_machine_.ProcessEvent(StandbyEvent::PROMOTE); + if (!result.allowed) { + LOG(ERROR) << "Cannot promote: " << result.reason; + return nullptr; + } + + StandbySyncStatus status = GetSyncStatus(); + uint64_t current_applied_seq_id = status.applied_seq_id; + + LOG(INFO) << "Promoting Standby to Primary. Applied seq_id: " + << current_applied_seq_id + << ", lag: " << status.lag_entries << " entries" + << ", state: " << StandbyStateToString(GetState()); + + // Final catch-up sync before promotion. + // IMPORTANT: + // - Do NOT rely on `lag_entries` here because primary_seq_id_ is best-effort. + // - Stop OpLogWatcher first to avoid concurrent Apply from watch callbacks. + if (oplog_watcher_) { + oplog_watcher_->Stop(); + } + + // Best-effort: resolve any outstanding gaps with retry before promotion. + // Do NOT block promotion if gaps cannot be fetched after max retries. + static constexpr int kMaxGapResolveRetries = 3; + if (oplog_applier_) { + for (int retry = 0; retry < kMaxGapResolveRetries; ++retry) { + auto res = oplog_applier_->TryResolveGapsOnceForPromotion(/*max_ids=*/1024); + if (res.attempted == 0) { + // No gaps to resolve + break; + } + LOG(INFO) << "Promotion gap resolve (attempt " << (retry + 1) << "/" + << kMaxGapResolveRetries << "): attempted=" << res.attempted + << ", fetched=" << res.fetched + << ", applied_deletes=" << res.applied_deletes; + if (res.fetched == res.attempted) { + // All gaps resolved successfully + break; + } + // Some gaps failed, retry after short delay + if (retry + 1 < kMaxGapResolveRetries) { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + } + } + + LOG(INFO) << "Final catch-up sync from etcd before promotion..."; + EtcdOpLogStore oplog_store(cluster_id_, /*enable_latest_seq_batch_update=*/false); + const size_t batch_size = 1000; + + // P0 fix: Prevent underflow when current_applied_seq_id is 0 + // ReadOpLogSince reads entries with seq > given_seq, so we pass current_applied_seq_id directly + uint64_t read_from_seq = current_applied_seq_id; // Will read entries with seq > read_from_seq + + // P1 fix: Add timeout control to prevent infinite blocking + static constexpr size_t kMaxCatchUpBatches = 100; // Max 100 batches * 1000 = 100k entries + static constexpr auto kMaxCatchUpDuration = std::chrono::seconds(30); + auto catch_up_start = std::chrono::steady_clock::now(); + + size_t total_applied = 0; + size_t batch_count = 0; + + for (;;) { + // Check timeout + auto elapsed = std::chrono::steady_clock::now() - catch_up_start; + if (elapsed > kMaxCatchUpDuration) { + LOG(WARNING) << "Final catch-up: timeout after " + << std::chrono::duration_cast(elapsed).count() + << "s. Proceeding with promotion. total_applied=" << total_applied; + break; + } + + // Check batch limit + if (batch_count >= kMaxCatchUpBatches) { + LOG(WARNING) << "Final catch-up: reached max batch limit (" << kMaxCatchUpBatches + << "). Proceeding with promotion. total_applied=" << total_applied; + break; + } + + std::vector batch; + ErrorCode read_err = oplog_store.ReadOpLogSince(read_from_seq, batch_size, batch); + if (read_err != ErrorCode::OK) { + LOG(WARNING) << "Final catch-up: failed to read OpLog since seq=" + << read_from_seq << ", err=" << static_cast(read_err) + << ". Proceeding with promotion."; + break; + } + if (batch.empty()) { + break; + } + size_t applied = oplog_applier_->ApplyOpLogEntries(batch); + total_applied += applied; + read_from_seq = batch.back().sequence_id; // Next read will get entries > this seq + ++batch_count; + } + LOG(INFO) << "Final catch-up sync done. total_applied=" << total_applied + << ", batches=" << batch_count; + + // Transition to PROMOTED state + state_machine_.ProcessEvent(StandbyEvent::PROMOTION_SUCCESS); + + // Stop replication (OpLogWatcher will stop watching). + // Note: This will trigger STOP event, transitioning to STOPPED. + Stop(); + + // Design note: MasterService creation and initialization are handled by + // MasterServiceSupervisor::Start() after leader election. The + // responsibility of HotStandbyService::Promote() is limited to ensuring + // that all remaining OpLog entries are applied before the new Primary + // starts serving requests. + + LOG(INFO) << "Standby promoted to Primary successfully. " + << "All remaining OpLog entries have been synced."; + + // Return nullptr - actual MasterService creation happens externally + // The caller (MasterServiceSupervisor) will create the MasterService + // with the appropriate configuration. + return nullptr; +} + +size_t HotStandbyService::GetMetadataCount() const { + std::lock_guard lock(mutex_); + return metadata_store_ ? metadata_store_->GetKeyCount() : 0; +} + +uint64_t HotStandbyService::GetLatestAppliedSequenceId() const { + std::lock_guard lock(mutex_); + if (oplog_applier_) { + uint64_t expected_seq = oplog_applier_->GetExpectedSequenceId(); + // GetExpectedSequenceId returns the next expected sequence_id, + // so the latest applied is expected_seq - 1 + return expected_seq > 0 ? expected_seq - 1 : 0; + } + return applied_seq_id_.load(); +} + +bool HotStandbyService::ExportMetadataSnapshot( + std::vector>& out) const { + std::lock_guard lock(mutex_); + if (!metadata_store_) { + out.clear(); + return false; + } + metadata_store_->Snapshot(out); + return true; +} + +void HotStandbyService::SetSnapshotProvider(std::unique_ptr provider) { + std::lock_guard lock(mutex_); + if (provider) { + snapshot_provider_ = std::move(provider); + } else { + snapshot_provider_ = std::make_unique(); + } +} + +void HotStandbyService::ReplicationLoop() { + LOG(INFO) << "Replication loop started (etcd-based OpLog sync)"; + + // With etcd-based OpLog sync, OpLogWatcher handles the actual watching + // in its own thread. This loop now just monitors the status and updates + // metrics. + + while (IsRunning()) { + if (!IsConnected()) { + // Not connected - wait a bit before checking again + std::this_thread::sleep_for(std::chrono::seconds(1)); + continue; + } + + // Update applied_seq_id from OpLogApplier + if (oplog_applier_) { + uint64_t current_applied = oplog_applier_->GetExpectedSequenceId() - 1; + if (current_applied > 0) { + applied_seq_id_.store(current_applied); + } + } + + // Update primary_seq_id by querying etcd `/latest` (best-effort). + // Note: `/latest` is batch-updated on Primary, so this is for monitoring only. +#ifdef STORE_USE_ETCD + if (!cluster_id_.empty()) { + EtcdOpLogStore oplog_store(cluster_id_, /*enable_latest_seq_batch_update=*/false); + uint64_t latest_seq = 0; + ErrorCode err = oplog_store.GetLatestSequenceId(latest_seq); + if (err == ErrorCode::OK) { + primary_seq_id_.store(latest_seq); + } + } +#endif + + // Sleep and check again + std::this_thread::sleep_for(std::chrono::milliseconds(1000)); + } + + LOG(INFO) << "Replication loop stopped"; +} + +void HotStandbyService::VerificationLoop() { + LOG(INFO) << "Verification loop started"; + + while (IsRunning()) { + std::this_thread::sleep_for( + std::chrono::seconds(config_.verification_interval_sec)); + + if (!IsConnected()) { + continue; + } + + // Verification is not yet implemented. When enabled, this loop is + // expected to: + // 1) sample keys from the local metadata store, + // 2) calculate checksums, + // 3) send a verification request to the Primary, and + // 4) handle any mismatches that are detected. + VLOG(1) << "Verification check skipped (feature not implemented), state=" + << StandbyStateToString(GetState()); + } + + LOG(INFO) << "Verification loop stopped"; +} + +void HotStandbyService::ApplyOpLogEntry(const OpLogEntry& entry) { + // NOTE: This method is deprecated. OpLog entries are now applied via + // OpLogApplier, which is called by OpLogWatcher. This method is kept + // for backward compatibility but should not be used in the new etcd-based + // implementation. + + // Update applied_seq_id for status tracking + applied_seq_id_.store(entry.sequence_id); + + // The actual application is handled by OpLogApplier via OpLogWatcher + VLOG(2) << "ApplyOpLogEntry called (deprecated), sequence_id=" + << entry.sequence_id << ", op_type=" << static_cast(entry.op_type) + << ", key=" << entry.object_key; +} + +void HotStandbyService::ProcessOpLogBatch( + const std::vector& entries) { + for (const auto& entry : entries) { + ApplyOpLogEntry(entry); + } +} + +bool HotStandbyService::ConnectToPrimary() { + // With etcd-based OpLog sync, connection is handled by OpLogWatcher + // This method is kept for compatibility but is no longer used + LOG(INFO) << "ConnectToPrimary called (no-op with etcd-based sync)"; + return true; +} + +void HotStandbyService::DisconnectFromPrimary() { + // With etcd-based OpLog sync, disconnection is handled by OpLogWatcher + // This method is kept for compatibility + if (IsConnected()) { + state_machine_.ProcessEvent(StandbyEvent::DISCONNECTED); + replication_stream_.reset(); + LOG(INFO) << "Disconnected from Primary (etcd-based sync), state=" + << StandbyStateToString(GetState()); + } +} + +} // namespace mooncake + diff --git a/mooncake-store/src/oplog_applier.cpp b/mooncake-store/src/oplog_applier.cpp new file mode 100644 index 0000000000..3f30307f45 --- /dev/null +++ b/mooncake-store/src/oplog_applier.cpp @@ -0,0 +1,584 @@ +#include "oplog_applier.h" + +#include + + +#include +#include + +#include "etcd_oplog_store.h" +#include "ha_metric_manager.h" +#include "metadata_store.h" +#include "oplog_manager.h" + +namespace mooncake { + +OpLogApplier::OpLogApplier(MetadataStore* metadata_store, + const std::string& cluster_id) + : metadata_store_(metadata_store), + cluster_id_(cluster_id), + expected_sequence_id_(1) { + if (metadata_store_ == nullptr) { + LOG(FATAL) << "OpLogApplier: metadata_store cannot be null"; + } + + // Validate cluster_id if provided (required for etcd operations). + // Normalize by stripping trailing slashes for validation. + std::string normalized = cluster_id_; + while (!normalized.empty() && normalized.back() == '/') { + normalized.pop_back(); + } + if (!normalized.empty() && !IsValidClusterIdComponent(normalized)) { + LOG(FATAL) << "Invalid cluster_id for OpLogApplier: '" << cluster_id_ + << "' (normalized: '" << normalized + << "'). Allowed chars: [A-Za-z0-9_.-], max_len=128, no slashes."; + } +} + +EtcdOpLogStore* OpLogApplier::GetEtcdOpLogStore() const { +#ifdef STORE_USE_ETCD + if (cluster_id_.empty()) { + return nullptr; + } + + std::lock_guard lock(etcd_oplog_store_mutex_); + if (!etcd_oplog_store_) { + // Reader: do not start `/latest` batch update thread. + etcd_oplog_store_ = + std::make_unique(cluster_id_, /*enable_latest_seq_batch_update=*/false); + } + return etcd_oplog_store_.get(); +#else + return nullptr; +#endif +} + +bool OpLogApplier::ApplyOpLogEntry(const OpLogEntry& entry) { + // Basic DoS protection: validate key/payload sizes before parsing/applying. + std::string size_reason; + if (!OpLogManager::ValidateEntrySize(entry, &size_reason)) { + LOG(ERROR) << "OpLogApplier: entry size rejected, sequence_id=" << entry.sequence_id + << ", key=" << entry.object_key << ", reason=" << size_reason; + return false; + } + + // Verify checksum to detect data corruption or tampering. + if (!OpLogManager::VerifyChecksum(entry)) { + LOG(ERROR) << "OpLogApplier: checksum mismatch, sequence_id=" << entry.sequence_id + << ", key=" << entry.object_key + << ". Possible data corruption or tampering. Discarding entry."; + HAMetricManager::instance().inc_oplog_checksum_failures(); + return false; + } + + // Global ordering only. + // + // IMPORTANT: + // - Watch callbacks / retries may deliver duplicate or already-applied entries. + // - Those must be treated as no-op, not as "out-of-order pending", otherwise + // pending_entries_ can grow and the applier may appear stuck. + const uint64_t expected = expected_sequence_id_.load(); + if (IsSequenceOlder(entry.sequence_id, expected)) { + // Late arrival of a previously-skipped gap entry: apply only if it's a delete/revoke. + bool was_skipped = false; + { + std::lock_guard lock(pending_mutex_); + auto it = skipped_sequence_ids_.find(entry.sequence_id); + if (it != skipped_sequence_ids_.end()) { + was_skipped = true; + skipped_sequence_ids_.erase(it); + } + } + if (was_skipped) { + if (entry.op_type == OpType::REMOVE || entry.op_type == OpType::PUT_REVOKE) { + // Safe: ensure we don't keep stale metadata. + if (entry.op_type == OpType::REMOVE) { + ApplyRemove(entry); + } else { + ApplyPutRevoke(entry); + } + return true; + } + // PUT_END (or others): discard to avoid resurrecting stale state. + VLOG(1) << "OpLogApplier: discard late skipped entry, op_type=" + << static_cast(entry.op_type) + << ", sequence_id=" << entry.sequence_id + << ", key=" << entry.object_key; + return true; + } + + VLOG(2) << "OpLogApplier: skip already-applied entry, sequence_id=" + << entry.sequence_id << ", expected=" << expected + << ", key=" << entry.object_key; + return true; // consumed (no-op) + } + if (IsSequenceNewer(entry.sequence_id, expected)) { + // Future entry - store into pending, wait for the gap to be filled. + std::lock_guard lock(pending_mutex_); + + if (pending_entries_.size() >= static_cast(kMaxPendingEntries)) { + LOG(ERROR) << "OpLogApplier: too many pending entries (" + << pending_entries_.size() << "), discarding entry sequence_id=" + << entry.sequence_id << ", key=" << entry.object_key; + return false; + } + + pending_entries_[entry.sequence_id] = entry; + VLOG(1) << "OpLogApplier: future entry buffered, sequence_id=" + << entry.sequence_id << ", expected=" << expected + << ", key=" << entry.object_key + << ", pending_entries=" << pending_entries_.size(); + return false; + } + + // Apply the operation based on type + switch (entry.op_type) { + case OpType::PUT_END: + ApplyPutEnd(entry); + break; + case OpType::PUT_REVOKE: + ApplyPutRevoke(entry); + break; + case OpType::REMOVE: + ApplyRemove(entry); + break; + default: + LOG(ERROR) << "OpLogApplier: unsupported op_type=" + << static_cast(entry.op_type) + << ", sequence_id=" << entry.sequence_id + << ", key=" << entry.object_key; + return false; + } + + // Update expected sequence ID + expected_sequence_id_.store(entry.sequence_id + 1); + + // Update metrics + HAMetricManager::instance().inc_oplog_applied_entries(); + HAMetricManager::instance().set_oplog_applied_sequence_id( + static_cast(entry.sequence_id)); + + // Try to process pending entries + ProcessPendingEntries(); + + return true; +} + +size_t OpLogApplier::ApplyOpLogEntries(const std::vector& entries) { + size_t applied_count = 0; + for (const auto& entry : entries) { + if (ApplyOpLogEntry(entry)) { + applied_count++; + } + } + return applied_count; +} + +uint64_t OpLogApplier::GetKeySequenceId(const std::string& key) const { + // Global sequence_id is used for ordering. + (void)key; // Suppress unused parameter warning + return 0; +} + +uint64_t OpLogApplier::GetExpectedSequenceId() const { + return expected_sequence_id_.load(); +} + +void OpLogApplier::Recover(uint64_t last_applied_sequence_id) { + expected_sequence_id_.store(last_applied_sequence_id + 1); + LOG(INFO) << "OpLogApplier: recovered from sequence_id=" + << last_applied_sequence_id + << ", expected_sequence_id set to=" << expected_sequence_id_.load(); +} + +size_t OpLogApplier::ProcessPendingEntries() { + // Check for missing sequence IDs, possibly skip after timeout, and/or request them. + uint64_t missing_seq_to_request = 0; + uint64_t skipped_count = 0; + { + std::lock_guard lock(pending_mutex_); + auto now = std::chrono::steady_clock::now(); + for (;;) { + if (pending_entries_.empty()) { + break; + } + const uint64_t first_pending_seq = pending_entries_.begin()->first; + const uint64_t expected = expected_sequence_id_.load(); + if (IsSequenceOlderOrEqual(first_pending_seq, expected)) { + break; + } + + // There's a gap: expected is missing. + const uint64_t missing_seq = expected; + auto it = missing_sequence_ids_.find(missing_seq); + if (it == missing_sequence_ids_.end()) { + missing_sequence_ids_[missing_seq] = now; + ScheduleWaitForMissingEntries(missing_seq); + break; + } + + const auto waited = std::chrono::duration_cast(now - it->second); + + // Skip after timeout to avoid global stall (user requested behavior). + if (waited.count() >= kMissingEntrySkipSeconds) { + skipped_sequence_ids_[missing_seq] = now; + missing_sequence_ids_.erase(missing_seq); + expected_sequence_id_.store(missing_seq + 1); + skipped_count++; + HAMetricManager::instance().inc_oplog_skipped_entries(); + LOG(WARNING) << "OpLogApplier: skipped missing entry seq=" << missing_seq + << " after " << waited.count() << "s timeout"; + continue; // may skip multiple consecutive gaps + } + + // Best-effort request from etcd (before skip triggers). + if (waited.count() >= kMissingEntryRequestSeconds) { + missing_seq_to_request = missing_seq; + break; + } + break; + } + } + + // Request missing OpLog if needed (outside the lock to avoid deadlock) + bool retrieved_missing = false; + if (missing_seq_to_request > 0) { + retrieved_missing = RequestMissingOpLog(missing_seq_to_request); + if (retrieved_missing) { + std::lock_guard lock(pending_mutex_); + missing_sequence_ids_.erase(missing_seq_to_request); + } + } + + size_t processed_count = 0; + for (;;) { + OpLogEntry entry_copy; + bool has_entry = false; + + { + std::lock_guard lock(pending_mutex_); + if (pending_entries_.empty()) { + break; + } + + auto it = pending_entries_.begin(); + const uint64_t expected = expected_sequence_id_.load(); + if (!IsSequenceEqual(it->first, expected)) { + break; // still waiting for earlier sequence_id + } + + entry_copy = it->second; + pending_entries_.erase(it); + has_entry = true; + } + + if (!has_entry) { + break; + } + + // Apply outside lock. + switch (entry_copy.op_type) { + case OpType::PUT_END: + ApplyPutEnd(entry_copy); + break; + case OpType::PUT_REVOKE: + ApplyPutRevoke(entry_copy); + break; + case OpType::REMOVE: + ApplyRemove(entry_copy); + break; + default: + LOG(ERROR) << "OpLogApplier: unsupported op_type in pending entry"; + break; + } + + expected_sequence_id_.store(entry_copy.sequence_id + 1); + + { + std::lock_guard lock(pending_mutex_); + missing_sequence_ids_.erase(entry_copy.sequence_id); + } + + processed_count++; + } + + // Clean up old missing sequence IDs (older than 1 minute) + { + std::lock_guard lock(pending_mutex_); + auto now = std::chrono::steady_clock::now(); + for (auto it = missing_sequence_ids_.begin(); it != missing_sequence_ids_.end();) { + auto age = std::chrono::duration_cast(now - it->second); + if (age.count() > 60) { + LOG(WARNING) << "OpLogApplier: giving up on missing sequence_id=" + << it->first << " after " << age.count() << " seconds"; + it = missing_sequence_ids_.erase(it); + } else { + ++it; + } + } + + // Clean up old skipped sequence IDs too (avoid unbounded growth). + for (auto it = skipped_sequence_ids_.begin(); it != skipped_sequence_ids_.end();) { + auto age = std::chrono::duration_cast(now - it->second); + if (age.count() > 60) { + it = skipped_sequence_ids_.erase(it); + } else { + ++it; + } + } + } + + if (skipped_count > 0) { + LOG(WARNING) << "OpLogApplier: skipped " << skipped_count + << " missing sequence_id(s) after timeout, expected_sequence_id now=" + << expected_sequence_id_.load(); + } + + if (processed_count > 0) { + LOG(INFO) << "OpLogApplier: processed " << processed_count + << " pending entries, expected_sequence_id now=" + << expected_sequence_id_.load(); + } + + // Update pending entries metric + { + std::lock_guard lock(pending_mutex_); + HAMetricManager::instance().set_oplog_pending_entries( + static_cast(pending_entries_.size())); + } + + return processed_count; +} + +OpLogApplier::GapResolveResult OpLogApplier::TryResolveGapsOnceForPromotion( + size_t max_ids) { + GapResolveResult r; +#ifdef STORE_USE_ETCD + EtcdOpLogStore* store = GetEtcdOpLogStore(); + if (store == nullptr) { + return r; + } + + std::vector gap_ids; + gap_ids.reserve(max_ids); + { + std::lock_guard lock(pending_mutex_); + for (const auto& kv : missing_sequence_ids_) { + if (gap_ids.size() >= max_ids) break; + gap_ids.push_back(kv.first); + } + for (const auto& kv : skipped_sequence_ids_) { + if (gap_ids.size() >= max_ids) break; + gap_ids.push_back(kv.first); + } + } + + if (gap_ids.empty()) { + return r; + } + + std::sort(gap_ids.begin(), gap_ids.end()); + gap_ids.erase(std::unique(gap_ids.begin(), gap_ids.end()), gap_ids.end()); + + r.attempted = gap_ids.size(); + std::vector successfully_processed; + for (uint64_t seq : gap_ids) { + OpLogEntry e; + ErrorCode err = store->ReadOpLog(seq, e); + if (err != ErrorCode::OK) { + // Log failed gap for monitoring, but don't clear it so it can be retried later. + LOG(WARNING) << "Promotion gap resolve: failed to fetch seq=" << seq + << ", err=" << static_cast(err); + continue; + } + r.fetched++; + + // Apply policy: only delete/revoke; drop PUT_END. + if (e.op_type == OpType::REMOVE) { + ApplyRemove(e); + r.applied_deletes++; + successfully_processed.push_back(seq); + } else if (e.op_type == OpType::PUT_REVOKE) { + ApplyPutRevoke(e); + r.applied_deletes++; + successfully_processed.push_back(seq); + } else { + // PUT_END or others: mark as processed (dropped) so we don't retry. + successfully_processed.push_back(seq); + } + } + + // Only clear gaps we successfully fetched and processed. + // Failed gaps remain in missing_sequence_ids_/skipped_sequence_ids_ for potential + // retry or monitoring. + if (!successfully_processed.empty()) { + std::lock_guard lock(pending_mutex_); + for (uint64_t seq : successfully_processed) { + missing_sequence_ids_.erase(seq); + skipped_sequence_ids_.erase(seq); + } + } + return r; +#else + (void)max_ids; + return r; +#endif +} + +bool OpLogApplier::CheckSequenceOrder(const OpLogEntry& entry) { + // Only check global sequence order. + // Use IsSequenceEqual for wrap-around safety (though equality check doesn't + // need special handling, we use it for consistency). + return IsSequenceEqual(entry.sequence_id, expected_sequence_id_.load()); +} + +void OpLogApplier::ApplyPutEnd(const OpLogEntry& entry) { + // Payload contains serialized metadata (replicas, size, etc.) in JSON format. + // Deserialize the payload immediately and store structured metadata. + // This allows Standby to serve requests immediately after promotion. + + if (entry.payload.empty()) { + // No payload - create empty metadata (legacy compatibility) + LOG(WARNING) << "OpLogApplier: PUT_END without payload, key=" << entry.object_key + << ", sequence_id=" << entry.sequence_id; + StandbyObjectMetadata empty_metadata; + empty_metadata.last_sequence_id = entry.sequence_id; + if (!metadata_store_->PutMetadata(entry.object_key, empty_metadata)) { + LOG(ERROR) << "OpLogApplier: failed to PutMetadata key=" << entry.object_key + << ", sequence_id=" << entry.sequence_id; + } + return; + } + + // Deserialize payload using struct_pack (msgpack binary format) + MetadataPayload payload; + bool parse_success = false; + auto result = struct_pack::deserialize_to(payload, entry.payload); + if (result == struct_pack::errc::ok) { + parse_success = true; + } else { + LOG(ERROR) << "OpLogApplier: failed to deserialize payload for key=" << entry.object_key + << ", sequence_id=" << entry.sequence_id + << ", payload_size=" << entry.payload.size() + << ", error_code=" << static_cast(result); + } + + if (!parse_success) { + // Fallback to empty metadata if parsing fails + StandbyObjectMetadata empty_metadata; + empty_metadata.last_sequence_id = entry.sequence_id; + metadata_store_->PutMetadata(entry.object_key, empty_metadata); + return; + } + + // Convert to StandbyObjectMetadata and store + StandbyObjectMetadata metadata = payload.ToStandbyMetadata(entry.sequence_id); + + if (!metadata_store_->PutMetadata(entry.object_key, metadata)) { + LOG(ERROR) << "OpLogApplier: failed to PutMetadata key=" << entry.object_key + << ", sequence_id=" << entry.sequence_id; + } else { + VLOG(1) << "OpLogApplier: applied PUT_END, key=" << entry.object_key + << ", sequence_id=" << entry.sequence_id + << ", replicas=" << metadata.replicas.size() + << ", size=" << metadata.size; + } +} + +void OpLogApplier::ApplyPutRevoke(const OpLogEntry& entry) { + // PUT_REVOKE means the object should be removed from metadata store + // (but the key itself may still exist if there are other replicas). + // Current implementation removes the entire key; if we later support + // partial replica revocation this logic will need to be refined. + if (!metadata_store_->Remove(entry.object_key)) { + LOG(WARNING) << "OpLogApplier: failed to Remove key=" << entry.object_key + << " in PUT_REVOKE, sequence_id=" << entry.sequence_id + << " (key may not exist)"; + } else { + VLOG(1) << "OpLogApplier: applied PUT_REVOKE, key=" << entry.object_key + << ", sequence_id=" << entry.sequence_id; + } +} + +void OpLogApplier::ApplyRemove(const OpLogEntry& entry) { + if (!metadata_store_->Remove(entry.object_key)) { + LOG(WARNING) << "OpLogApplier: failed to Remove key=" << entry.object_key + << ", sequence_id=" << entry.sequence_id + << " (key may not exist)"; + } else { + VLOG(1) << "OpLogApplier: applied REMOVE, key=" << entry.object_key + << ", sequence_id=" << entry.sequence_id; + } +} + +bool OpLogApplier::RequestMissingOpLog(uint64_t missing_seq_id) { +#ifdef STORE_USE_ETCD + HAMetricManager::instance().inc_oplog_gap_resolve_attempts(); + + EtcdOpLogStore* oplog_store = GetEtcdOpLogStore(); + if (oplog_store == nullptr) { + LOG(WARNING) << "OpLogApplier: cannot request missing OpLog, cluster_id not set"; + return false; + } + + OpLogEntry entry; + ErrorCode err = oplog_store->ReadOpLog(missing_seq_id, entry); + if (err == ErrorCode::ETCD_KEY_NOT_EXIST) { + LOG(INFO) << "OpLogApplier: missing OpLog entry not found in etcd, sequence_id=" + << missing_seq_id; + return false; + } + if (err != ErrorCode::OK) { + LOG(ERROR) << "OpLogApplier: failed to read missing OpLog from etcd, sequence_id=" + << missing_seq_id << ", error=" << static_cast(err); + return false; + } + + std::string size_reason; + if (!OpLogManager::ValidateEntrySize(entry, &size_reason)) { + LOG(ERROR) << "OpLogApplier: missing entry size rejected, sequence_id=" + << missing_seq_id << ", key=" << entry.object_key + << ", reason=" << size_reason; + return false; + } + + // Verify checksum before adding to pending entries. + if (!OpLogManager::VerifyChecksum(entry)) { + LOG(ERROR) << "OpLogApplier: checksum mismatch for retrieved missing entry, sequence_id=" + << missing_seq_id << ", key=" << entry.object_key + << ". Possible data corruption. Discarding entry."; + HAMetricManager::instance().inc_oplog_checksum_failures(); + return false; + } + + // Successfully retrieved the missing OpLog entry + LOG(INFO) << "OpLogApplier: retrieved missing OpLog entry, sequence_id=" + << missing_seq_id << ", op_type=" << static_cast(entry.op_type) + << ", key=" << entry.object_key; + HAMetricManager::instance().inc_oplog_gap_resolve_success(); + + // Add to pending entries + // Note: We don't call ProcessPendingEntries() here to avoid potential recursion. + // The caller (ProcessPendingEntries itself) will process the entry in the next loop. + { + std::lock_guard lock(pending_mutex_); + pending_entries_[entry.sequence_id] = entry; + } + + return true; +#else + LOG(WARNING) << "OpLogApplier: STORE_USE_ETCD not enabled, cannot request missing OpLog"; + return false; +#endif +} + +void OpLogApplier::ScheduleWaitForMissingEntries(uint64_t missing_seq_id) { + // This method is called when we first detect a missing sequence_id. + // The actual waiting and requesting is handled in ProcessPendingEntries(). + // We just log it here for tracking. + VLOG(1) << "OpLogApplier: scheduling wait for missing sequence_id=" << missing_seq_id + << ", will request after " << kMissingEntryRequestSeconds << " seconds"; +} + +} // namespace mooncake + diff --git a/mooncake-store/src/oplog_manager.cpp b/mooncake-store/src/oplog_manager.cpp new file mode 100644 index 0000000000..17d4da84ba --- /dev/null +++ b/mooncake-store/src/oplog_manager.cpp @@ -0,0 +1,171 @@ +#include "oplog_manager.h" + +#include +#include +#include +#include + +#include "etcd_oplog_store.h" + +namespace mooncake { + +OpLogManager::OpLogManager() = default; + +void OpLogManager::SetEtcdOpLogStore( + std::shared_ptr etcd_oplog_store) { + std::unique_lock lock(mutex_); + etcd_oplog_store_ = etcd_oplog_store; +} + +uint64_t OpLogManager::Append(OpType type, const std::string& key, + const std::string& payload) { + OpLogEntry entry; + entry.op_type = type; + entry.object_key = key; + entry.payload = payload; + entry.timestamp_ms = NowMs(); + entry.checksum = ComputeChecksum(entry.payload); + entry.prefix_hash = ComputePrefixHash(entry.object_key); + + std::unique_lock lock(mutex_); + entry.sequence_id = ++last_seq_id_; + + if (buffer_.size() >= kMaxBufferEntries_) { + buffer_.pop_front(); + ++first_seq_id_; + } + + buffer_.emplace_back(entry); // Copy entry to buffer + + // Write to etcd if EtcdOpLogStore is set + if (etcd_oplog_store_) { + // Release lock before writing to etcd to avoid blocking + // We use the original entry (before it was copied to buffer) + lock.unlock(); + ErrorCode err = etcd_oplog_store_->WriteOpLog(entry); + if (err != ErrorCode::OK) { + // Log error but don't fail the operation + // The entry is already in the memory buffer + LOG(WARNING) << "Failed to write OpLog to etcd, sequence_id=" + << entry.sequence_id + << ", but entry is in memory buffer"; + } + } + + return last_seq_id_; +} + +OpLogEntry OpLogManager::AllocateEntry(OpType type, const std::string& key, + const std::string& payload) { + OpLogEntry entry; + entry.op_type = type; + entry.object_key = key; + entry.payload = payload; + entry.timestamp_ms = NowMs(); + entry.checksum = ComputeChecksum(entry.payload); + entry.prefix_hash = ComputePrefixHash(entry.object_key); + + std::unique_lock lock(mutex_); + entry.sequence_id = ++last_seq_id_; + + if (buffer_.size() >= kMaxBufferEntries_) { + buffer_.pop_front(); + ++first_seq_id_; + } + buffer_.emplace_back(entry); + return entry; +} + +ErrorCode OpLogManager::PersistEntryToEtcd(const OpLogEntry& entry) const { + std::shared_lock lock(mutex_); + auto store = etcd_oplog_store_; + lock.unlock(); + if (!store) { + return ErrorCode::ETCD_OPERATION_ERROR; + } + return store->WriteOpLog(entry); +} + +tl::expected OpLogManager::AppendAndPersist( + OpType type, const std::string& key, const std::string& payload) { + // Seq pre-allocation semantics: allocate first, then persist. + OpLogEntry entry = AllocateEntry(type, key, payload); + ErrorCode err = PersistEntryToEtcd(entry); + if (err != ErrorCode::OK) { + return tl::make_unexpected(err); + } + return entry.sequence_id; +} + +uint64_t OpLogManager::GetLastSequenceId() const { + std::shared_lock lock(mutex_); + return last_seq_id_; +} + +void OpLogManager::SetInitialSequenceId(uint64_t sequence_id) { + std::unique_lock lock(mutex_); + if (last_seq_id_ == 0 && buffer_.empty()) { + // Only allow setting initial sequence_id if OpLogManager is empty + last_seq_id_ = sequence_id; + first_seq_id_ = sequence_id + 1; // first_seq_id_ should be > last_seq_id_ when empty + LOG(INFO) << "OpLogManager initial sequence_id set to " << sequence_id; + } else { + LOG(WARNING) << "Cannot set initial sequence_id: OpLogManager is not empty " + << "(last_seq_id_=" << last_seq_id_ << ", buffer_size=" << buffer_.size() << ")"; + } +} + +size_t OpLogManager::GetEntryCount() const { + std::shared_lock lock(mutex_); + return buffer_.size(); +} + +uint64_t OpLogManager::NowMs() { + using namespace std::chrono; + return duration_cast(steady_clock::now().time_since_epoch()) + .count(); +} + +uint32_t OpLogManager::ComputeChecksum(const std::string& data) { + // Use xxHash XXH32 for a fast, deterministic 32-bit checksum. + // Requires linking against xxHash (e.g., libxxhash) and including . + return static_cast(XXH32(data.data(), data.size(), 0)); +} + +uint32_t OpLogManager::ComputePrefixHash(const std::string& key) { + if (key.empty()) { + return 0; + } + // Use XXH32 for consistency with ComputeChecksum and better performance. + // XXH32 provides faster hashing and lower collision rate than std::hash. + // Computing hash for the entire key ensures better distribution and fewer collisions. + return static_cast(XXH32(key.data(), key.size(), 0)); +} + +bool OpLogManager::VerifyChecksum(const OpLogEntry& entry) { + uint32_t computed = ComputeChecksum(entry.payload); + return computed == entry.checksum; +} + +bool OpLogManager::ValidateEntrySize(const OpLogEntry& entry, + std::string* reason) { + if (entry.object_key.size() > kMaxObjectKeySize) { + if (reason) { + *reason = "object_key too large: size=" + + std::to_string(entry.object_key.size()); + } + return false; + } + if (entry.payload.size() > kMaxPayloadSize) { + if (reason) { + *reason = + "payload too large: size=" + std::to_string(entry.payload.size()); + } + return false; + } + return true; +} + +} // namespace mooncake + + diff --git a/mooncake-store/src/oplog_watcher.cpp b/mooncake-store/src/oplog_watcher.cpp new file mode 100644 index 0000000000..3ce96ed3bc --- /dev/null +++ b/mooncake-store/src/oplog_watcher.cpp @@ -0,0 +1,610 @@ +#include "oplog_watcher.h" + +#include +#include +#include +#include +#include + +#ifdef STORE_USE_ETCD +#include "etcd_helper.h" +#include "etcd_oplog_store.h" +#include "ha_metric_manager.h" +#include "oplog_applier.h" +#include "oplog_manager.h" + +#if __has_include() +#include // Ubuntu +#else +#include // CentOS +#endif + +namespace mooncake { + +namespace { + +// Base64 decoding for binary payload (must match encoding in etcd_oplog_store.cpp) +std::string Base64Decode(const std::string& encoded) { + static const unsigned char decode_table[256] = { + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 62, 64, 64, 64, 63, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 64, 64, 64, 64, 64, 64, + 64, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 64, 64, 64, 64, 64, + 64, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 + }; + + std::string result; + result.reserve((encoded.size() * 3) / 4); + + size_t i = 0; + while (i < encoded.size()) { + // Skip whitespace and invalid chars + while (i < encoded.size() && (encoded[i] == ' ' || encoded[i] == '\n' || encoded[i] == '\r' || encoded[i] == '\t')) { + i++; + } + if (i >= encoded.size()) break; + + uint32_t sextet_a = decode_table[static_cast(encoded[i++])]; + if (i >= encoded.size() || sextet_a == 64) break; + + uint32_t sextet_b = decode_table[static_cast(encoded[i++])]; + if (sextet_b == 64) break; + + uint32_t sextet_c = (i < encoded.size()) ? decode_table[static_cast(encoded[i++])] : 64; + uint32_t sextet_d = (i < encoded.size()) ? decode_table[static_cast(encoded[i++])] : 64; + + uint32_t triple = (sextet_a << 18) | (sextet_b << 12) | + ((sextet_c != 64) ? (sextet_c << 6) : 0) | + ((sextet_d != 64) ? sextet_d : 0); + + result.push_back(static_cast((triple >> 16) & 0xFF)); + if (sextet_c != 64) { + result.push_back(static_cast((triple >> 8) & 0xFF)); + } + if (sextet_d != 64) { + result.push_back(static_cast(triple & 0xFF)); + } + } + + return result; +} + +} // namespace + +OpLogWatcher::OpLogWatcher(const std::string& etcd_endpoints, + const std::string& cluster_id, OpLogApplier* applier) + : etcd_endpoints_(etcd_endpoints), cluster_id_(cluster_id), applier_(applier) { + if (applier_ == nullptr) { + LOG(FATAL) << "OpLogApplier cannot be null"; + } + // Normalize cluster_id to avoid double slashes in watch prefix. + while (!cluster_id_.empty() && cluster_id_.back() == '/') { + cluster_id_.pop_back(); + } + if (!cluster_id_.empty() && !IsValidClusterIdComponent(cluster_id_)) { + LOG(FATAL) << "Invalid cluster_id for OpLogWatcher: '" << cluster_id_ + << "'. Allowed chars: [A-Za-z0-9_.-], max_len=128, no slashes."; + } +} + +OpLogWatcher::~OpLogWatcher() { + Stop(); +} + +void OpLogWatcher::Start() { + // Backward-compatible: start from the last processed sequence id. + (void)StartFromSequenceId(last_processed_sequence_id_.load()); +} + +bool OpLogWatcher::StartFromSequenceId(uint64_t start_seq_id) { + if (running_.load()) { + LOG(WARNING) << "OpLogWatcher is already running"; + return true; + } + +#ifdef STORE_USE_ETCD + uint64_t read_seq_id = start_seq_id; + EtcdRevisionId last_read_rev = 0; + size_t total_applied = 0; + + for (;;) { + std::vector batch; + EtcdRevisionId rev = 0; + if (!ReadOpLogSince(read_seq_id, batch, rev)) { + last_read_rev = 0; + break; + } + last_read_rev = rev; + if (!batch.empty()) { + for (const auto& e : batch) { + if (applier_->ApplyOpLogEntry(e)) { + last_processed_sequence_id_.store(e.sequence_id); + read_seq_id = e.sequence_id; + total_applied++; + } + } + } + if (batch.size() < kSyncBatchSize) { + break; + } + } + + if (last_read_rev > 0) { + next_watch_revision_.store(static_cast(last_read_rev + 1)); + } else { + next_watch_revision_.store(0); + } + + LOG(INFO) << "OpLogWatcher initial sync done: applied=" << total_applied + << ", last_seq=" << last_processed_sequence_id_.load() + << ", next_watch_revision=" << next_watch_revision_.load(); +#endif + + running_.store(true); + watch_thread_ = std::thread(&OpLogWatcher::WatchOpLog, this); + LOG(INFO) << "OpLogWatcher started for cluster_id=" << cluster_id_; + return true; +} + +void OpLogWatcher::Stop() { + if (!running_.load()) { + return; + } + + running_.store(false); + +#ifdef STORE_USE_ETCD + // Wait for watch thread to finish first. This ensures that the watch thread + // has exited before we cancel the watch, reducing the chance of race conditions. + if (watch_thread_.joinable()) { + watch_thread_.join(); + } + + // Now cancel the watch. This will trigger the Go goroutine to exit. + // The watch thread has already stopped, so we won't have race conditions + // with it trying to access the watcher object. + std::string watch_prefix = "/oplog/" + cluster_id_ + "/"; + ErrorCode err = EtcdHelper::CancelWatchWithPrefix(watch_prefix.c_str(), watch_prefix.size()); + if (err != ErrorCode::OK) { + LOG(WARNING) << "Failed to cancel watch for prefix " << watch_prefix + << ", error=" << static_cast(err); + } + + // Wait for Go watch goroutine to fully exit (no more callbacks). + // This avoids callback-after-free without relying on sleeps. + (void)EtcdHelper::WaitWatchWithPrefixStopped(watch_prefix.c_str(), + watch_prefix.size(), + /*timeout_ms=*/5000); +#endif + + LOG(INFO) << "OpLogWatcher stopped"; +} + +bool OpLogWatcher::ReadOpLogSince(uint64_t start_seq_id, + std::vector& entries, + EtcdRevisionId& revision_id) { +#ifdef STORE_USE_ETCD + EtcdOpLogStore oplog_store(cluster_id_, /*enable_latest_seq_batch_update=*/false); + ErrorCode err = oplog_store.ReadOpLogSinceWithRevision( + start_seq_id, kSyncBatchSize, entries, revision_id); + if (err != ErrorCode::OK) { + LOG(ERROR) << "Failed to read OpLog since sequence_id=" << start_seq_id + << ", error=" << static_cast(err); + return false; + } + return true; +#else + (void)start_seq_id; + (void)entries; + (void)revision_id; + return false; +#endif +} + +uint64_t OpLogWatcher::GetLastProcessedSequenceId() const { + return last_processed_sequence_id_.load(); +} + +void OpLogWatcher::WatchCallback(void* context, const char* key, size_t key_size, + const char* value, size_t value_size, + int event_type, int64_t mod_revision) { + // Use try-catch to prevent crashes if object is destroyed + try { + OpLogWatcher* watcher = static_cast(context); + if (watcher == nullptr) { + // Context is null, ignore callback + return; + } + + // Early exit check: First, try to read running_ flag with minimal object access. + // If object is destroyed, this access might cause SIGSEGV, which will be caught + // by signal handler or cause immediate crash (better than accessing more members). + // We use memory_order_acquire for consistency, but if object is destroyed, + // even this access can fail. + // + // Note: There's no perfect way to check if a C++ object is still valid without + // potentially accessing invalid memory. The best we can do is: + // 1. Check quickly and exit early if stopped + // 2. Use try-catch for C++ exceptions (won't catch SIGSEGV) + // 3. Ensure Stop() waits long enough for all callbacks to complete + bool is_running = false; + try { + is_running = watcher->running_.load(std::memory_order_acquire); + } catch (...) { + // Object may be destroyed, ignore callback + return; + } + + if (!is_running) { + // Watcher is being stopped, ignore callback + return; + } + + std::string key_str; + if (key != nullptr && key_size > 0) { + key_str.assign(key, key_size); + } + std::string value_str; + if (value != nullptr && value_size > 0) { + value_str = std::string(value, value_size); + } + watcher->HandleWatchEvent(key_str, value_str, event_type, mod_revision); + } catch (const std::exception& e) { + // C++ object may have been destroyed, ignore the exception + LOG(WARNING) << "Exception in WatchCallback (likely object destroyed): " << e.what(); + } catch (...) { + // Catch all other exceptions (including access violations) + LOG(WARNING) << "Unknown exception in WatchCallback (likely object destroyed)"; + } +} + +void OpLogWatcher::WatchOpLog() { +#ifdef STORE_USE_ETCD + LOG(INFO) << "OpLog watch thread started for cluster_id=" << cluster_id_; + + std::string watch_prefix = "/oplog/" + cluster_id_ + "/"; + + while (running_.load()) { + // Cancel any existing watch before starting a new one + // This prevents "prefix already being watched" errors + (void)EtcdHelper::CancelWatchWithPrefix(watch_prefix.c_str(), watch_prefix.size()); + (void)EtcdHelper::WaitWatchWithPrefixStopped(watch_prefix.c_str(), + watch_prefix.size(), + /*timeout_ms=*/5000); + + // Start watching - pass static callback function and this pointer as context + EtcdRevisionId start_rev = + static_cast(next_watch_revision_.load()); + // Use watcher with mod_revision so we can update next_watch_revision_ precisely. + ErrorCode err = EtcdHelper::WatchWithPrefixFromRevision( + watch_prefix.c_str(), watch_prefix.size(), start_rev, this, WatchCallback); + + if (err != ErrorCode::OK) { + LOG(ERROR) << "Failed to start watch for prefix " << watch_prefix + << ", error=" << static_cast(err); + watch_healthy_.store(false); + NotifyStateEvent(StandbyEvent::WATCH_BROKEN); + + // Wait a bit longer before retrying, to ensure old goroutines have time to exit + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + + // Try to reconnect + TryReconnect(); + continue; + } + + LOG(INFO) << "Watch started for prefix " << watch_prefix; + watch_healthy_.store(true); + consecutive_errors_.store(0); + NotifyStateEvent(StandbyEvent::WATCH_HEALTHY); + + // The watch is now running in the background (via Go goroutine) + // We just need to keep the thread alive until Stop() is called or watch fails + while (running_.load() && watch_healthy_.load()) { + // Drive pending/missing handling even when no new watch events arrive. + // Without this, a single out-of-order arrival could park entries in + // pending_entries_ forever if the missing entry isn't delivered via watch + // (but exists in etcd and could be fetched). + (void)applier_->ProcessPendingEntries(); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + // Periodically check watch health + if (consecutive_errors_.load() >= kMaxConsecutiveErrors) { + LOG(WARNING) << "Too many consecutive errors (" << consecutive_errors_.load() + << "), reconnecting watch..."; + watch_healthy_.store(false); + NotifyStateEvent(StandbyEvent::MAX_ERRORS_REACHED); + break; + } + } + + if (running_.load() && !watch_healthy_.load()) { + // Cancel current watch before reconnecting + (void)EtcdHelper::CancelWatchWithPrefix(watch_prefix.c_str(), watch_prefix.size()); + (void)EtcdHelper::WaitWatchWithPrefixStopped(watch_prefix.c_str(), + watch_prefix.size(), + /*timeout_ms=*/5000); + NotifyStateEvent(StandbyEvent::WATCH_BROKEN); + TryReconnect(); + } + } + + LOG(INFO) << "OpLog watch thread stopped"; +#else + LOG(ERROR) << "STORE_USE_ETCD is not enabled, cannot watch OpLog from etcd"; + running_.store(false); +#endif +} + +void OpLogWatcher::TryReconnect() { + if (!running_.load()) { + return; + } + + int reconnect_attempt = reconnect_count_.fetch_add(1) + 1; + + // Calculate delay with exponential backoff + int delay_ms = std::min(kReconnectDelayMs * reconnect_attempt, kMaxReconnectDelayMs); + + LOG(INFO) << "Attempting to reconnect watch (attempt #" << reconnect_attempt + << "), waiting " << delay_ms << "ms..."; + + std::this_thread::sleep_for(std::chrono::milliseconds(delay_ms)); + + // Sync any missed entries before resuming watch + if (SyncMissedEntries()) { + LOG(INFO) << "Successfully synced missed OpLog entries"; + NotifyStateEvent(StandbyEvent::RECOVERY_SUCCESS); + } else { + LOG(WARNING) << "Failed to sync missed OpLog entries, continuing anyway"; + NotifyStateEvent(StandbyEvent::RECOVERY_FAILED); + } +} + +bool OpLogWatcher::SyncMissedEntries() { +#ifdef STORE_USE_ETCD + uint64_t last_seq = last_processed_sequence_id_.load(); + if (last_seq == 0) { + // No entries processed yet, nothing to sync + return true; + } + + LOG(INFO) << "Syncing missed OpLog entries since sequence_id=" << last_seq; + + std::vector entries; + EtcdRevisionId rev = 0; + if (!ReadOpLogSince(last_seq, entries, rev)) { + LOG(ERROR) << "Failed to read missed OpLog entries"; + return false; + } + if (rev > 0) { + next_watch_revision_.store(static_cast(rev + 1)); + } + + if (entries.empty()) { + LOG(INFO) << "No missed OpLog entries to sync"; + return true; + } + + LOG(INFO) << "Syncing " << entries.size() << " missed OpLog entries"; + + for (const auto& entry : entries) { + if (applier_->ApplyOpLogEntry(entry)) { + last_processed_sequence_id_.store(entry.sequence_id); + } else { + LOG(WARNING) << "Failed to apply missed OpLog entry, sequence_id=" + << entry.sequence_id; + } + } + + return true; +#else + return false; +#endif +} + +void OpLogWatcher::HandleWatchEvent(const std::string& key, const std::string& value, + int event_type) { + HandleWatchEvent(key, value, event_type, /*mod_revision=*/0); +} + +void OpLogWatcher::HandleWatchEvent(const std::string& key, const std::string& value, + int event_type, int64_t mod_revision) { + // event_type: + // 0 = PUT, 1 = DELETE, 2 = WATCH_BROKEN (Go watcher terminated; should reconnect) + if (event_type == 2) { + LOG(WARNING) << "OpLog watch broken, will reconnect. cluster_id=" << cluster_id_ + << ", next_watch_revision=" << next_watch_revision_.load() + << ", last_seq=" << last_processed_sequence_id_.load(); + watch_healthy_.store(false); + consecutive_errors_.fetch_add(1); + return; + } + + if (mod_revision > 0) { + // Keep next_watch_revision_ monotonic: next = max(next, modRev+1) + int64_t candidate = mod_revision + 1; + int64_t cur = next_watch_revision_.load(); + while (candidate > cur && + !next_watch_revision_.compare_exchange_weak(cur, candidate)) { + // retry + } + } + // event_type: 0 = PUT, 1 = DELETE + if (event_type == 1) { + // DELETE event - OpLog entry was cleaned up + VLOG(1) << "OpLog entry deleted: " << key; + consecutive_errors_.store(0); // Watch is working + return; + } + + if (event_type != 0) { + LOG(WARNING) << "Unknown event type: " << event_type << " for key: " << key; + consecutive_errors_.fetch_add(1); + return; + } + + // Skip the "latest" key and snapshot keys + if (key.find("/latest") != std::string::npos || + key.find("/snapshot/") != std::string::npos) { + return; + } + + // Parse the OpLog entry from JSON + OpLogEntry entry; + if (!DeserializeOpLogEntry(value, entry)) { + LOG(ERROR) << "Failed to deserialize OpLog entry from key: " << key; + consecutive_errors_.fetch_add(1); + return; + } + + // Basic DoS protection: validate key/payload sizes before further processing. + std::string size_reason; + if (!OpLogManager::ValidateEntrySize(entry, &size_reason)) { + LOG(ERROR) << "OpLog entry size rejected: sequence_id=" << entry.sequence_id + << ", key=" << entry.object_key << ", reason=" << size_reason; + consecutive_errors_.fetch_add(1); + return; + } + + // Verify checksum to detect data corruption or tampering. + if (!OpLogManager::VerifyChecksum(entry)) { + LOG(ERROR) << "OpLog entry checksum mismatch: sequence_id=" << entry.sequence_id + << ", key=" << entry.object_key + << ". Possible data corruption or tampering. Discarding entry."; + consecutive_errors_.fetch_add(1); + HAMetricManager::instance().inc_oplog_checksum_failures(); + return; + } + + // Apply the OpLog entry + if (applier_->ApplyOpLogEntry(entry)) { + // last_processed_sequence_id_ must be monotonic. We may "consume" duplicate + // / already-applied entries (entry.sequence_id < expected) as no-ops, so + // never regress this counter. + uint64_t cur = last_processed_sequence_id_.load(); + while (IsSequenceNewer(entry.sequence_id, cur) && + !last_processed_sequence_id_.compare_exchange_weak(cur, entry.sequence_id)) { + // retry + } + consecutive_errors_.store(0); // Reset error counter on success + reconnect_count_.store(0); // Reset reconnect counter on success + VLOG(2) << "Applied OpLog entry: sequence_id=" << entry.sequence_id + << ", op_type=" << static_cast(entry.op_type) + << ", key=" << entry.object_key; + } else { + // ApplyOpLogEntry returns false for out-of-order entries, + // which is expected behavior, not an error + VLOG(1) << "OpLog entry not applied (may be out of order): sequence_id=" + << entry.sequence_id; + } +} + +bool OpLogWatcher::DeserializeOpLogEntry(const std::string& json_str, + OpLogEntry& entry) { + Json::Value root; + Json::CharReaderBuilder reader; + std::string errs; + std::istringstream s(json_str); + + if (!Json::parseFromStream(reader, s, &root, &errs)) { + LOG(ERROR) << "Failed to parse OpLogEntry JSON: " << errs; + return false; + } + + entry.sequence_id = root.get("sequence_id", 0).asUInt64(); + entry.timestamp_ms = root.get("timestamp_ms", 0).asUInt64(); + entry.op_type = static_cast(root.get("op_type", 0).asInt()); + entry.object_key = root.get("object_key", "").asString(); + + // CRITICAL: Base64 decode payload to restore binary data + std::string encoded_payload = root.get("payload", "").asString(); + entry.payload = Base64Decode(encoded_payload); + + entry.checksum = root.get("checksum", 0).asUInt(); + entry.prefix_hash = root.get("prefix_hash", 0).asUInt(); + return true; +} + +} // namespace mooncake + +#else // STORE_USE_ETCD not defined + +namespace mooncake { + +OpLogWatcher::OpLogWatcher(const std::string& etcd_endpoints, + const std::string& cluster_id, OpLogApplier* applier) + : etcd_endpoints_(etcd_endpoints), cluster_id_(cluster_id), applier_(applier) { + LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; +} + +OpLogWatcher::~OpLogWatcher() { + Stop(); +} + +void OpLogWatcher::Start() { + LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; +} + +bool OpLogWatcher::StartFromSequenceId(uint64_t /*start_seq_id*/) { + LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; + return false; +} + +void OpLogWatcher::Stop() { + // No-op when STORE_USE_ETCD is not enabled +} + +bool OpLogWatcher::ReadOpLogSince(uint64_t /*start_seq_id*/, + std::vector& /*entries*/, + EtcdRevisionId& /*revision_id*/) { + LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; + return false; +} + +uint64_t OpLogWatcher::GetLastProcessedSequenceId() const { + return last_processed_sequence_id_.load(); +} + +void OpLogWatcher::WatchOpLog() { + LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; +} + +void OpLogWatcher::HandleWatchEvent(const std::string& key, const std::string& value, + int event_type) { + LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; +} + +void OpLogWatcher::HandleWatchEvent(const std::string& key, const std::string& value, + int event_type, int64_t mod_revision) { + (void)key; + (void)value; + (void)event_type; + (void)mod_revision; + LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; +} + +void OpLogWatcher::TryReconnect() { + LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; +} + +bool OpLogWatcher::SyncMissedEntries() { + LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; + return false; +} + +} // namespace mooncake + +#endif // STORE_USE_ETCD + diff --git a/mooncake-store/src/standby_state_machine.cpp b/mooncake-store/src/standby_state_machine.cpp new file mode 100644 index 0000000000..616649a454 --- /dev/null +++ b/mooncake-store/src/standby_state_machine.cpp @@ -0,0 +1,283 @@ +#include "standby_state_machine.h" + +#include + +namespace mooncake { + +StandbyStateMachine::StandbyStateMachine() + : state_enter_time_(std::chrono::steady_clock::now()) {} + +StateTransitionResult StandbyStateMachine::ValidateTransition(StandbyState from, + StandbyEvent event) const { + StateTransitionResult result; + result.allowed = false; + result.old_state = from; + result.new_state = from; + + // State transition table + switch (from) { + case StandbyState::STOPPED: + if (event == StandbyEvent::START) { + result.allowed = true; + result.new_state = StandbyState::CONNECTING; + } + break; + + case StandbyState::CONNECTING: + switch (event) { + case StandbyEvent::CONNECTED: + result.allowed = true; + result.new_state = StandbyState::SYNCING; + break; + case StandbyEvent::CONNECTION_FAILED: + case StandbyEvent::FATAL_ERROR: + result.allowed = true; + result.new_state = StandbyState::FAILED; + break; + case StandbyEvent::STOP: + result.allowed = true; + result.new_state = StandbyState::STOPPED; + break; + default: + break; + } + break; + + case StandbyState::SYNCING: + switch (event) { + case StandbyEvent::SYNC_COMPLETE: + result.allowed = true; + result.new_state = StandbyState::WATCHING; + break; + case StandbyEvent::SYNC_FAILED: + case StandbyEvent::DISCONNECTED: + result.allowed = true; + result.new_state = StandbyState::RECONNECTING; + break; + case StandbyEvent::STOP: + result.allowed = true; + result.new_state = StandbyState::STOPPED; + break; + case StandbyEvent::FATAL_ERROR: + result.allowed = true; + result.new_state = StandbyState::FAILED; + break; + default: + break; + } + break; + + case StandbyState::WATCHING: + switch (event) { + case StandbyEvent::WATCH_BROKEN: + case StandbyEvent::DISCONNECTED: + result.allowed = true; + result.new_state = StandbyState::RECONNECTING; + break; + case StandbyEvent::MAX_ERRORS_REACHED: + result.allowed = true; + result.new_state = StandbyState::RECOVERING; + break; + case StandbyEvent::PROMOTE: + result.allowed = true; + result.new_state = StandbyState::PROMOTING; + break; + case StandbyEvent::STOP: + result.allowed = true; + result.new_state = StandbyState::STOPPED; + break; + case StandbyEvent::FATAL_ERROR: + result.allowed = true; + result.new_state = StandbyState::FAILED; + break; + // WATCH_HEALTHY in WATCHING state is a no-op (stay in WATCHING) + case StandbyEvent::WATCH_HEALTHY: + result.allowed = true; + result.new_state = StandbyState::WATCHING; + break; + default: + break; + } + break; + + case StandbyState::RECOVERING: + switch (event) { + case StandbyEvent::RECOVERY_SUCCESS: + result.allowed = true; + result.new_state = StandbyState::WATCHING; + break; + case StandbyEvent::RECOVERY_FAILED: + case StandbyEvent::DISCONNECTED: + result.allowed = true; + result.new_state = StandbyState::RECONNECTING; + break; + case StandbyEvent::STOP: + result.allowed = true; + result.new_state = StandbyState::STOPPED; + break; + case StandbyEvent::FATAL_ERROR: + result.allowed = true; + result.new_state = StandbyState::FAILED; + break; + default: + break; + } + break; + + case StandbyState::RECONNECTING: + switch (event) { + case StandbyEvent::CONNECTED: + result.allowed = true; + result.new_state = StandbyState::SYNCING; + break; + case StandbyEvent::MAX_ERRORS_REACHED: + case StandbyEvent::FATAL_ERROR: + result.allowed = true; + result.new_state = StandbyState::FAILED; + break; + case StandbyEvent::STOP: + result.allowed = true; + result.new_state = StandbyState::STOPPED; + break; + default: + break; + } + break; + + case StandbyState::PROMOTING: + switch (event) { + case StandbyEvent::PROMOTION_SUCCESS: + result.allowed = true; + result.new_state = StandbyState::PROMOTED; + break; + case StandbyEvent::PROMOTION_FAILED: + result.allowed = true; + result.new_state = StandbyState::FAILED; + break; + case StandbyEvent::STOP: + result.allowed = true; + result.new_state = StandbyState::STOPPED; + break; + default: + break; + } + break; + + case StandbyState::PROMOTED: + if (event == StandbyEvent::STOP) { + result.allowed = true; + result.new_state = StandbyState::STOPPED; + } + break; + + case StandbyState::FAILED: + if (event == StandbyEvent::STOP) { + result.allowed = true; + result.new_state = StandbyState::STOPPED; + } else if (event == StandbyEvent::START) { + // Allow restart from FAILED state + result.allowed = true; + result.new_state = StandbyState::CONNECTING; + } + break; + } + + if (!result.allowed) { + result.reason = std::string("Invalid transition from ") + StandbyStateToString(from) + + " on event " + StandbyEventToString(event); + } + + return result; +} + +StateTransitionResult StandbyStateMachine::ProcessEvent(StandbyEvent event) { + StandbyState old_state = current_state_.load(std::memory_order_acquire); + StateTransitionResult result = ValidateTransition(old_state, event); + + if (result.allowed && result.new_state != old_state) { + std::lock_guard lock(mutex_); + + // Double-check state hasn't changed (compare-and-swap pattern) + StandbyState current = current_state_.load(std::memory_order_acquire); + if (current != old_state) { + // State changed by another thread, re-validate + result = ValidateTransition(current, event); + old_state = current; + result.old_state = current; + if (!result.allowed || result.new_state == old_state) { + return result; + } + } + + // Record transition + TransitionRecord record; + record.timestamp = std::chrono::steady_clock::now(); + record.from_state = old_state; + record.to_state = result.new_state; + record.event = event; + + transition_history_.push_back(record); + if (transition_history_.size() > kMaxHistorySize) { + transition_history_.erase(transition_history_.begin()); + } + + // Update state + current_state_.store(result.new_state, std::memory_order_release); + state_enter_time_ = record.timestamp; + + LOG(INFO) << "Standby state transition: " << StandbyStateToString(old_state) << " -> " + << StandbyStateToString(result.new_state) + << " (event: " << StandbyEventToString(event) << ")"; + + // Make a copy of callbacks to release the lock before calling them + std::vector callbacks_copy = callbacks_; + + // Release lock by ending scope, then notify callbacks + // Note: We need to unlock before calling callbacks to avoid deadlock + // So we copy callbacks and call after the lock_guard scope ends + for (const auto& callback : callbacks_copy) { + if (callback) { + callback(old_state, result.new_state, event); + } + } + } else if (!result.allowed) { + VLOG(1) << "Standby state transition rejected: " << result.reason; + } + + return result; +} + +void StandbyStateMachine::RegisterCallback(StateChangeCallback callback) { + std::lock_guard lock(mutex_); + callbacks_.push_back(std::move(callback)); +} + +std::vector StandbyStateMachine::GetTransitionHistory( + size_t max_records) const { + std::lock_guard lock(mutex_); + + if (transition_history_.size() <= max_records) { + return transition_history_; + } + + return std::vector(transition_history_.end() - max_records, + transition_history_.end()); +} + +std::chrono::milliseconds StandbyStateMachine::GetTimeInCurrentState() const { + auto now = std::chrono::steady_clock::now(); + std::lock_guard lock(mutex_); + return std::chrono::duration_cast(now - state_enter_time_); +} + +int StandbyStateMachine::IncrementErrors() { + int new_count = consecutive_errors_.fetch_add(1) + 1; + if (new_count >= kMaxConsecutiveErrors) { + // Trigger MAX_ERRORS_REACHED event + ProcessEvent(StandbyEvent::MAX_ERRORS_REACHED); + } + return new_count; +} + +} // namespace mooncake +